/*
- * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
*
- * @APPLE_LICENSE_OSREFERENCE_HEADER_START@
- *
- * This file contains Original Code and/or Modifications of Original Code
- * as defined in and that are subject to the Apple Public Source License
- * Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. The rights granted to you under the
- * License may not be used to create, or enable the creation or
- * redistribution of, unlawful or unlicensed copies of an Apple operating
- * system, or to circumvent, violate, or enable the circumvention or
- * violation of, any terms of an Apple operating system software license
- * agreement.
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
- * Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this
- * file.
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
*
- * The Original Code and all software distributed under the License are
- * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
- * Please see the License for the specific language governing rights and
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ *
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and
* limitations under the License.
*
- * @APPLE_LICENSE_OSREFERENCE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
/*
* @OSF_COPYRIGHT@
*/
-/*
+/*
* Mach Operating System
* Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
* All Rights Reserved.
- *
+ *
* Permission to use, copy, modify and distribute this software and its
* documentation is hereby granted, provided that both the copyright
* notice and this permission notice appear in all copies of the
* software, derivative works or modified versions, and any portions
* thereof, and that both notices appear in supporting documentation.
- *
+ *
* CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
* CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
* ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
- *
+ *
* Carnegie Mellon requests users of this software to return to
- *
+ *
* Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
* School of Computer Science
* Carnegie Mellon University
* Pittsburgh PA 15213-3890
- *
+ *
* any improvements or extensions that they make and grant Carnegie Mellon
* the rights to redistribute these changes.
*/
*/
#include <stdint.h>
+#include <ptrauth.h>
#include <debug.h>
#include <mach_pagemap.h>
#include <mach_cluster_stats.h>
-#include <mach_kdb.h>
-#include <advisory_pageout.h>
#include <mach/mach_types.h>
#include <mach/memory_object.h>
#include <mach/vm_map.h>
#include <mach/vm_param.h>
#include <mach/vm_statistics.h>
+#include <mach/sdt.h>
#include <kern/kern_types.h>
#include <kern/counters.h>
#include <kern/host_statistics.h>
#include <kern/machine.h>
#include <kern/misc_protos.h>
+#include <kern/sched.h>
#include <kern/thread.h>
-#include <kern/xpr.h>
#include <kern/kalloc.h>
+#include <kern/zalloc_internal.h>
+#include <kern/policy_internal.h>
+#include <kern/thread_group.h>
#include <machine/vm_tuning.h>
+#include <machine/commpage.h>
#include <vm/pmap.h>
+#include <vm/vm_compressor_pager.h>
#include <vm/vm_fault.h>
#include <vm/vm_map.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>
#include <vm/vm_pageout.h>
#include <vm/vm_protos.h> /* must be last */
+#include <vm/memory_object.h>
+#include <vm/vm_purgeable_internal.h>
+#include <vm/vm_shared_region.h>
+#include <vm/vm_compressor.h>
-/*
- * ENCRYPTED SWAP:
- */
-#ifdef __ppc__
-#include <ppc/mappings.h>
-#endif /* __ppc__ */
-#include <../bsd/crypto/aes/aes.h>
-
-extern ipc_port_t memory_manager_default;
+#include <san/kasan.h>
+#if CONFIG_PHANTOM_CACHE
+#include <vm/vm_phantom_cache.h>
+#endif
-#ifndef VM_PAGEOUT_BURST_ACTIVE_THROTTLE
-#define VM_PAGEOUT_BURST_ACTIVE_THROTTLE 10000 /* maximum iterations of the active queue to move pages to inactive */
+#if UPL_DEBUG
+#include <libkern/OSDebug.h>
#endif
-#ifndef VM_PAGEOUT_BURST_INACTIVE_THROTTLE
-#define VM_PAGEOUT_BURST_INACTIVE_THROTTLE 4096 /* maximum iterations of the inactive queue w/o stealing/cleaning a page */
+extern int cs_debug;
+
+extern void mbuf_drain(boolean_t);
+
+#if VM_PRESSURE_EVENTS
+#if CONFIG_JETSAM
+extern unsigned int memorystatus_available_pages;
+extern unsigned int memorystatus_available_pages_pressure;
+extern unsigned int memorystatus_available_pages_critical;
+#else /* CONFIG_JETSAM */
+extern uint64_t memorystatus_available_pages;
+extern uint64_t memorystatus_available_pages_pressure;
+extern uint64_t memorystatus_available_pages_critical;
+#endif /* CONFIG_JETSAM */
+
+extern unsigned int memorystatus_frozen_count;
+extern unsigned int memorystatus_suspended_count;
+extern vm_pressure_level_t memorystatus_vm_pressure_level;
+
+extern lck_mtx_t memorystatus_jetsam_fg_band_lock;
+extern uint32_t memorystatus_jetsam_fg_band_waiters;
+
+void vm_pressure_response(void);
+extern void consider_vm_pressure_events(void);
+
+#define MEMORYSTATUS_SUSPENDED_THRESHOLD 4
+#endif /* VM_PRESSURE_EVENTS */
+
+thread_t vm_pageout_scan_thread = THREAD_NULL;
+boolean_t vps_dynamic_priority_enabled = FALSE;
+
+#ifndef VM_PAGEOUT_BURST_INACTIVE_THROTTLE /* maximum iterations of the inactive queue w/o stealing/cleaning a page */
+#ifdef CONFIG_EMBEDDED
+#define VM_PAGEOUT_BURST_INACTIVE_THROTTLE 1024
+#else
+#define VM_PAGEOUT_BURST_INACTIVE_THROTTLE 4096
+#endif
#endif
#ifndef VM_PAGEOUT_DEADLOCK_RELIEF
-#define VM_PAGEOUT_DEADLOCK_RELIEF 100 /* number of pages to move to break deadlock */
+#define VM_PAGEOUT_DEADLOCK_RELIEF 100 /* number of pages to move to break deadlock */
#endif
-#ifndef VM_PAGEOUT_INACTIVE_RELIEF
-#define VM_PAGEOUT_INACTIVE_RELIEF 50 /* minimum number of pages to move to the inactive q */
-#endif
+#ifndef VM_PAGE_LAUNDRY_MAX
+#define VM_PAGE_LAUNDRY_MAX 128UL /* maximum pageouts on a given pageout queue */
+#endif /* VM_PAGEOUT_LAUNDRY_MAX */
+
+#ifndef VM_PAGEOUT_BURST_WAIT
+#define VM_PAGEOUT_BURST_WAIT 1 /* milliseconds */
+#endif /* VM_PAGEOUT_BURST_WAIT */
+
+#ifndef VM_PAGEOUT_EMPTY_WAIT
+#define VM_PAGEOUT_EMPTY_WAIT 50 /* milliseconds */
+#endif /* VM_PAGEOUT_EMPTY_WAIT */
-#ifndef VM_PAGE_LAUNDRY_MAX
-#define VM_PAGE_LAUNDRY_MAX 16UL /* maximum pageouts on a given pageout queue */
-#endif /* VM_PAGEOUT_LAUNDRY_MAX */
+#ifndef VM_PAGEOUT_DEADLOCK_WAIT
+#define VM_PAGEOUT_DEADLOCK_WAIT 100 /* milliseconds */
+#endif /* VM_PAGEOUT_DEADLOCK_WAIT */
-#ifndef VM_PAGEOUT_BURST_WAIT
-#define VM_PAGEOUT_BURST_WAIT 30 /* milliseconds per page */
-#endif /* VM_PAGEOUT_BURST_WAIT */
+#ifndef VM_PAGEOUT_IDLE_WAIT
+#define VM_PAGEOUT_IDLE_WAIT 10 /* milliseconds */
+#endif /* VM_PAGEOUT_IDLE_WAIT */
-#ifndef VM_PAGEOUT_EMPTY_WAIT
-#define VM_PAGEOUT_EMPTY_WAIT 200 /* milliseconds */
-#endif /* VM_PAGEOUT_EMPTY_WAIT */
+#ifndef VM_PAGEOUT_SWAP_WAIT
+#define VM_PAGEOUT_SWAP_WAIT 10 /* milliseconds */
+#endif /* VM_PAGEOUT_SWAP_WAIT */
-#ifndef VM_PAGEOUT_DEADLOCK_WAIT
-#define VM_PAGEOUT_DEADLOCK_WAIT 300 /* milliseconds */
-#endif /* VM_PAGEOUT_DEADLOCK_WAIT */
-#ifndef VM_PAGEOUT_IDLE_WAIT
-#define VM_PAGEOUT_IDLE_WAIT 10 /* milliseconds */
-#endif /* VM_PAGEOUT_IDLE_WAIT */
+#ifndef VM_PAGE_SPECULATIVE_TARGET
+#define VM_PAGE_SPECULATIVE_TARGET(total) ((total) * 1 / (100 / vm_pageout_state.vm_page_speculative_percentage))
+#endif /* VM_PAGE_SPECULATIVE_TARGET */
/*
* then the pageout daemon starts running.
*/
-#ifndef VM_PAGE_INACTIVE_TARGET
-#define VM_PAGE_INACTIVE_TARGET(avail) ((avail) * 1 / 3)
-#endif /* VM_PAGE_INACTIVE_TARGET */
+#ifndef VM_PAGE_INACTIVE_TARGET
+#define VM_PAGE_INACTIVE_TARGET(avail) ((avail) * 1 / 2)
+#endif /* VM_PAGE_INACTIVE_TARGET */
/*
* Once the pageout daemon starts running, it keeps going
* until vm_page_free_count meets or exceeds vm_page_free_target.
*/
-#ifndef VM_PAGE_FREE_TARGET
-#define VM_PAGE_FREE_TARGET(free) (15 + (free) / 80)
-#endif /* VM_PAGE_FREE_TARGET */
+#ifndef VM_PAGE_FREE_TARGET
+#ifdef CONFIG_EMBEDDED
+#define VM_PAGE_FREE_TARGET(free) (15 + (free) / 100)
+#else
+#define VM_PAGE_FREE_TARGET(free) (15 + (free) / 80)
+#endif
+#endif /* VM_PAGE_FREE_TARGET */
+
/*
* The pageout daemon always starts running once vm_page_free_count
* falls below vm_page_free_min.
*/
-#ifndef VM_PAGE_FREE_MIN
-#define VM_PAGE_FREE_MIN(free) (10 + (free) / 100)
-#endif /* VM_PAGE_FREE_MIN */
+#ifndef VM_PAGE_FREE_MIN
+#ifdef CONFIG_EMBEDDED
+#define VM_PAGE_FREE_MIN(free) (10 + (free) / 200)
+#else
+#define VM_PAGE_FREE_MIN(free) (10 + (free) / 100)
+#endif
+#endif /* VM_PAGE_FREE_MIN */
+
+#ifdef CONFIG_EMBEDDED
+#define VM_PAGE_FREE_RESERVED_LIMIT 100
+#define VM_PAGE_FREE_MIN_LIMIT 1500
+#define VM_PAGE_FREE_TARGET_LIMIT 2000
+#else
+#define VM_PAGE_FREE_RESERVED_LIMIT 1700
+#define VM_PAGE_FREE_MIN_LIMIT 3500
+#define VM_PAGE_FREE_TARGET_LIMIT 4000
+#endif
/*
* When vm_page_free_count falls below vm_page_free_reserved,
* operation by dipping into the reserved pool of pages.
*/
-#ifndef VM_PAGE_FREE_RESERVED
-#define VM_PAGE_FREE_RESERVED(n) \
- ((6 * VM_PAGE_LAUNDRY_MAX) + (n))
-#endif /* VM_PAGE_FREE_RESERVED */
-
+#ifndef VM_PAGE_FREE_RESERVED
+#define VM_PAGE_FREE_RESERVED(n) \
+ ((unsigned) (6 * VM_PAGE_LAUNDRY_MAX) + (n))
+#endif /* VM_PAGE_FREE_RESERVED */
/*
- * must hold the page queues lock to
- * manipulate this structure
+ * When we dequeue pages from the inactive list, they are
+ * reactivated (ie, put back on the active queue) if referenced.
+ * However, it is possible to starve the free list if other
+ * processors are referencing pages faster than we can turn off
+ * the referenced bit. So we limit the number of reactivations
+ * we will make per call of vm_pageout_scan().
*/
-struct vm_pageout_queue {
- queue_head_t pgo_pending; /* laundry pages to be processed by pager's iothread */
- unsigned int pgo_laundry; /* current count of laundry pages on queue or in flight */
- unsigned int pgo_maxlaundry;
-
- unsigned int pgo_idle:1, /* iothread is blocked waiting for work to do */
- pgo_busy:1, /* iothread is currently processing request from pgo_pending */
- pgo_throttled:1,/* vm_pageout_scan thread needs a wakeup when pgo_laundry drops */
- :0;
-};
-
-#define VM_PAGE_Q_THROTTLED(q) \
- ((q)->pgo_laundry >= (q)->pgo_maxlaundry)
-
+#define VM_PAGE_REACTIVATE_LIMIT_MAX 20000
-/*
- * Exported variable used to broadcast the activation of the pageout scan
- * Working Set uses this to throttle its use of pmap removes. In this
- * way, code which runs within memory in an uncontested context does
- * not keep encountering soft faults.
- */
+#ifndef VM_PAGE_REACTIVATE_LIMIT
+#ifdef CONFIG_EMBEDDED
+#define VM_PAGE_REACTIVATE_LIMIT(avail) (VM_PAGE_INACTIVE_TARGET(avail) / 2)
+#else
+#define VM_PAGE_REACTIVATE_LIMIT(avail) (MAX((avail) * 1 / 20,VM_PAGE_REACTIVATE_LIMIT_MAX))
+#endif
+#endif /* VM_PAGE_REACTIVATE_LIMIT */
+#define VM_PAGEOUT_INACTIVE_FORCE_RECLAIM 1000
-unsigned int vm_pageout_scan_event_counter = 0;
+extern boolean_t hibernate_cleaning_in_progress;
/*
* Forward declarations for internal routines.
*/
+struct cq {
+ struct vm_pageout_queue *q;
+ void *current_chead;
+ char *scratch_buf;
+ int id;
+};
-static void vm_pageout_garbage_collect(int);
-static void vm_pageout_iothread_continue(struct vm_pageout_queue *);
-static void vm_pageout_iothread_external(void);
-static void vm_pageout_iothread_internal(void);
-static void vm_pageout_queue_steal(vm_page_t);
+struct cq ciq[MAX_COMPRESSOR_THREAD_COUNT];
-extern void vm_pageout_continue(void);
-extern void vm_pageout_scan(void);
-unsigned int vm_pageout_reserved_internal = 0;
-unsigned int vm_pageout_reserved_really = 0;
+#if VM_PRESSURE_EVENTS
+void vm_pressure_thread(void);
-unsigned int vm_pageout_idle_wait = 0; /* milliseconds */
-unsigned int vm_pageout_empty_wait = 0; /* milliseconds */
-unsigned int vm_pageout_burst_wait = 0; /* milliseconds */
-unsigned int vm_pageout_deadlock_wait = 0; /* milliseconds */
-unsigned int vm_pageout_deadlock_relief = 0;
-unsigned int vm_pageout_inactive_relief = 0;
-unsigned int vm_pageout_burst_active_throttle = 0;
-unsigned int vm_pageout_burst_inactive_throttle = 0;
+boolean_t VM_PRESSURE_NORMAL_TO_WARNING(void);
+boolean_t VM_PRESSURE_WARNING_TO_CRITICAL(void);
-/*
- * Protection against zero fill flushing live working sets derived
- * from existing backing store and files
- */
-unsigned int vm_accellerate_zf_pageout_trigger = 400;
-unsigned int vm_zf_iterator;
-unsigned int vm_zf_iterator_count = 40;
-unsigned int last_page_zf;
-unsigned int vm_zf_count = 0;
+boolean_t VM_PRESSURE_WARNING_TO_NORMAL(void);
+boolean_t VM_PRESSURE_CRITICAL_TO_WARNING(void);
+#endif
-/*
- * These variables record the pageout daemon's actions:
- * how many pages it looks at and what happens to those pages.
- * No locking needed because only one thread modifies the variables.
- */
+void vm_pageout_garbage_collect(int);
+static void vm_pageout_iothread_external(void);
+static void vm_pageout_iothread_internal(struct cq *cq);
+static void vm_pageout_adjust_eq_iothrottle(struct vm_pageout_queue *, boolean_t);
-unsigned int vm_pageout_active = 0; /* debugging */
-unsigned int vm_pageout_inactive = 0; /* debugging */
-unsigned int vm_pageout_inactive_throttled = 0; /* debugging */
-unsigned int vm_pageout_inactive_forced = 0; /* debugging */
-unsigned int vm_pageout_inactive_nolock = 0; /* debugging */
-unsigned int vm_pageout_inactive_avoid = 0; /* debugging */
-unsigned int vm_pageout_inactive_busy = 0; /* debugging */
-unsigned int vm_pageout_inactive_absent = 0; /* debugging */
-unsigned int vm_pageout_inactive_used = 0; /* debugging */
-unsigned int vm_pageout_inactive_clean = 0; /* debugging */
-unsigned int vm_pageout_inactive_dirty = 0; /* debugging */
-unsigned int vm_pageout_dirty_no_pager = 0; /* debugging */
-unsigned int vm_pageout_purged_objects = 0; /* debugging */
-unsigned int vm_stat_discard = 0; /* debugging */
-unsigned int vm_stat_discard_sent = 0; /* debugging */
-unsigned int vm_stat_discard_failure = 0; /* debugging */
-unsigned int vm_stat_discard_throttle = 0; /* debugging */
-
-unsigned int vm_pageout_scan_active_throttled = 0;
-unsigned int vm_pageout_scan_inactive_throttled = 0;
-unsigned int vm_pageout_scan_throttle = 0; /* debugging */
-unsigned int vm_pageout_scan_burst_throttle = 0; /* debugging */
-unsigned int vm_pageout_scan_empty_throttle = 0; /* debugging */
-unsigned int vm_pageout_scan_deadlock_detected = 0; /* debugging */
-unsigned int vm_pageout_scan_active_throttle_success = 0; /* debugging */
-unsigned int vm_pageout_scan_inactive_throttle_success = 0; /* debugging */
-/*
- * Backing store throttle when BS is exhausted
- */
-unsigned int vm_backing_store_low = 0;
+extern void vm_pageout_continue(void);
+extern void vm_pageout_scan(void);
-unsigned int vm_pageout_out_of_line = 0;
-unsigned int vm_pageout_in_place = 0;
+boolean_t vm_pageout_running = FALSE;
-/*
- * ENCRYPTED SWAP:
- * counters and statistics...
- */
-unsigned long vm_page_decrypt_counter = 0;
-unsigned long vm_page_decrypt_for_upl_counter = 0;
-unsigned long vm_page_encrypt_counter = 0;
-unsigned long vm_page_encrypt_abort_counter = 0;
-unsigned long vm_page_encrypt_already_encrypted_counter = 0;
-boolean_t vm_pages_encrypted = FALSE; /* are there encrypted pages ? */
+uint32_t vm_page_upl_tainted = 0;
+uint32_t vm_page_iopl_tainted = 0;
+#if !CONFIG_EMBEDDED
+static boolean_t vm_pageout_waiter = FALSE;
+#endif /* !CONFIG_EMBEDDED */
-struct vm_pageout_queue vm_pageout_queue_internal;
-struct vm_pageout_queue vm_pageout_queue_external;
+#if DEVELOPMENT || DEBUG
+struct vm_pageout_debug vm_pageout_debug;
+#endif
+struct vm_pageout_vminfo vm_pageout_vminfo;
+struct vm_pageout_state vm_pageout_state;
+struct vm_config vm_config;
-/*
- * Routine: vm_backing_store_disable
- * Purpose:
- * Suspend non-privileged threads wishing to extend
- * backing store when we are low on backing store
- * (Synchronized by caller)
- */
-void
-vm_backing_store_disable(
- boolean_t disable)
-{
- if(disable) {
- vm_backing_store_low = 1;
- } else {
- if(vm_backing_store_low) {
- vm_backing_store_low = 0;
- thread_wakeup((event_t) &vm_backing_store_low);
- }
- }
-}
+struct vm_pageout_queue vm_pageout_queue_internal VM_PAGE_PACKED_ALIGNED;
+struct vm_pageout_queue vm_pageout_queue_external VM_PAGE_PACKED_ALIGNED;
+int vm_upl_wait_for_pages = 0;
+vm_object_t vm_pageout_scan_wants_object = VM_OBJECT_NULL;
-/*
- * Routine: vm_pageout_object_allocate
- * Purpose:
- * Allocate an object for use as out-of-line memory in a
- * data_return/data_initialize message.
- * The page must be in an unlocked object.
- *
- * If the page belongs to a trusted pager, cleaning in place
- * will be used, which utilizes a special "pageout object"
- * containing private alias pages for the real page frames.
- * Untrusted pagers use normal out-of-line memory.
- */
-vm_object_t
-vm_pageout_object_allocate(
- vm_page_t m,
- vm_size_t size,
- vm_object_offset_t offset)
-{
- vm_object_t object = m->object;
- vm_object_t new_object;
+boolean_t(*volatile consider_buffer_cache_collect)(int) = NULL;
- assert(object->pager_ready);
+int vm_debug_events = 0;
- new_object = vm_object_allocate(size);
+LCK_GRP_DECLARE(vm_pageout_lck_grp, "vm_pageout");
- if (object->pager_trusted) {
- assert (offset < object->size);
+#if CONFIG_MEMORYSTATUS
+extern boolean_t memorystatus_kill_on_VM_page_shortage(boolean_t async);
- vm_object_lock(new_object);
- new_object->pageout = TRUE;
- new_object->shadow = object;
- new_object->can_persist = FALSE;
- new_object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
- new_object->shadow_offset = offset;
- vm_object_unlock(new_object);
+uint32_t vm_pageout_memorystatus_fb_factor_nr = 5;
+uint32_t vm_pageout_memorystatus_fb_factor_dr = 2;
- /*
- * Take a paging reference on the object. This will be dropped
- * in vm_pageout_object_terminate()
- */
- vm_object_lock(object);
- vm_object_paging_begin(object);
- vm_page_lock_queues();
- vm_page_unlock_queues();
- vm_object_unlock(object);
+#endif
- vm_pageout_in_place++;
- } else
- vm_pageout_out_of_line++;
- return(new_object);
-}
+#if __AMP__
+int vm_compressor_ebound = 1;
+int vm_pgo_pbound = 0;
+extern void thread_bind_cluster_type(thread_t, char, bool);
+#endif /* __AMP__ */
-#if MACH_CLUSTER_STATS
-unsigned long vm_pageout_cluster_dirtied = 0;
-unsigned long vm_pageout_cluster_cleaned = 0;
-unsigned long vm_pageout_cluster_collisions = 0;
-unsigned long vm_pageout_cluster_clusters = 0;
-unsigned long vm_pageout_cluster_conversions = 0;
-unsigned long vm_pageout_target_collisions = 0;
-unsigned long vm_pageout_target_page_dirtied = 0;
-unsigned long vm_pageout_target_page_freed = 0;
-#define CLUSTER_STAT(clause) clause
-#else /* MACH_CLUSTER_STATS */
-#define CLUSTER_STAT(clause)
-#endif /* MACH_CLUSTER_STATS */
-
-/*
+
+/*
* Routine: vm_pageout_object_terminate
* Purpose:
- * Destroy the pageout_object allocated by
- * vm_pageout_object_allocate(), and perform all of the
+ * Destroy the pageout_object, and perform all of the
* required cleanup actions.
- *
+ *
* In/Out conditions:
* The object must be locked, and will be returned locked.
*/
void
vm_pageout_object_terminate(
- vm_object_t object)
+ vm_object_t object)
{
- vm_object_t shadow_object;
- boolean_t shadow_internal;
+ vm_object_t shadow_object;
/*
* Deal with the deallocation (last reference) of a pageout object
assert(object->pageout);
shadow_object = object->shadow;
vm_object_lock(shadow_object);
- shadow_internal = shadow_object->internal;
- while (!queue_empty(&object->memq)) {
- vm_page_t p, m;
- vm_object_offset_t offset;
+ while (!vm_page_queue_empty(&object->memq)) {
+ vm_page_t p, m;
+ vm_object_offset_t offset;
- p = (vm_page_t) queue_first(&object->memq);
+ p = (vm_page_t) vm_page_queue_first(&object->memq);
- assert(p->private);
- assert(p->pageout);
- p->pageout = FALSE;
- assert(!p->cleaning);
+ assert(p->vmp_private);
+ assert(p->vmp_free_when_done);
+ p->vmp_free_when_done = FALSE;
+ assert(!p->vmp_cleaning);
+ assert(!p->vmp_laundry);
- offset = p->offset;
+ offset = p->vmp_offset;
VM_PAGE_FREE(p);
p = VM_PAGE_NULL;
m = vm_page_lookup(shadow_object,
- offset + object->shadow_offset);
+ offset + object->vo_shadow_offset);
- if(m == VM_PAGE_NULL)
+ if (m == VM_PAGE_NULL) {
continue;
- assert(m->cleaning);
- /* used as a trigger on upl_commit etc to recognize the */
- /* pageout daemon's subseqent desire to pageout a cleaning */
- /* page. When the bit is on the upl commit code will */
- /* respect the pageout bit in the target page over the */
- /* caller's page list indication */
- m->dump_cleaning = FALSE;
+ }
- /*
- * Account for the paging reference taken when
- * m->cleaning was set on this page.
- */
- vm_object_paging_end(shadow_object);
- assert((m->dirty) || (m->precious) ||
- (m->busy && m->cleaning));
+ assert((m->vmp_dirty) || (m->vmp_precious) ||
+ (m->vmp_busy && m->vmp_cleaning));
/*
* Handle the trusted pager throttle.
* Also decrement the burst throttle (if external).
*/
vm_page_lock_queues();
- if (m->laundry) {
+ if (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q) {
vm_pageout_throttle_up(m);
}
* pages may have been modified between the selection as an
* adjacent page and conversion to a target.
*/
- if (m->pageout) {
- assert(m->busy);
- assert(m->wire_count == 1);
- m->cleaning = FALSE;
- m->pageout = FALSE;
-#if MACH_CLUSTER_STATS
- if (m->wanted) vm_pageout_target_collisions++;
-#endif
+ if (m->vmp_free_when_done) {
+ assert(m->vmp_busy);
+ assert(m->vmp_q_state == VM_PAGE_IS_WIRED);
+ assert(m->vmp_wire_count == 1);
+ m->vmp_cleaning = FALSE;
+ m->vmp_free_when_done = FALSE;
/*
* Revoke all access to the page. Since the object is
* locked, and the page is busy, this prevents the page
* can detect whether the page was redirtied during
* pageout by checking the modify state.
*/
- if (pmap_disconnect(m->phys_page) & VM_MEM_MODIFIED)
- m->dirty = TRUE;
- else
- m->dirty = FALSE;
-
- if (m->dirty) {
- CLUSTER_STAT(vm_pageout_target_page_dirtied++;)
- vm_page_unwire(m);/* reactivates */
- VM_STAT(reactivations++);
+ if (pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)) & VM_MEM_MODIFIED) {
+ SET_PAGE_DIRTY(m, FALSE);
+ } else {
+ m->vmp_dirty = FALSE;
+ }
+
+ if (m->vmp_dirty) {
+ vm_page_unwire(m, TRUE); /* reactivates */
+ VM_STAT_INCR(reactivations);
PAGE_WAKEUP_DONE(m);
} else {
- CLUSTER_STAT(vm_pageout_target_page_freed++;)
- vm_page_free(m);/* clears busy, etc. */
+ vm_page_free(m); /* clears busy, etc. */
}
vm_page_unlock_queues();
continue;
* If prep_pin_count is nonzero, then someone is using the
* page, so make it active.
*/
- if (!m->active && !m->inactive && !m->private) {
- if (m->reference)
+ if ((m->vmp_q_state == VM_PAGE_NOT_ON_Q) && !m->vmp_private) {
+ if (m->vmp_reference) {
vm_page_activate(m);
- else
+ } else {
vm_page_deactivate(m);
+ }
}
- if((m->busy) && (m->cleaning)) {
-
- /* the request_page_list case, (COPY_OUT_FROM FALSE) */
- m->busy = FALSE;
-
- /* We do not re-set m->dirty ! */
- /* The page was busy so no extraneous activity */
- /* could have occurred. COPY_INTO is a read into the */
- /* new pages. CLEAN_IN_PLACE does actually write */
- /* out the pages but handling outside of this code */
- /* will take care of resetting dirty. We clear the */
- /* modify however for the Programmed I/O case. */
- pmap_clear_modify(m->phys_page);
- if(m->absent) {
- m->absent = FALSE;
- if(shadow_object->absent_count == 1)
- vm_object_absent_release(shadow_object);
- else
- shadow_object->absent_count--;
- }
- m->overwriting = FALSE;
- } else if (m->overwriting) {
- /* alternate request page list, write to page_list */
- /* case. Occurs when the original page was wired */
- /* at the time of the list request */
- assert(m->wire_count != 0);
- vm_page_unwire(m);/* reactivates */
- m->overwriting = FALSE;
- } else {
- /*
- * Set the dirty state according to whether or not the page was
- * modified during the pageout. Note that we purposefully do
- * NOT call pmap_clear_modify since the page is still mapped.
- * If the page were to be dirtied between the 2 calls, this
- * this fact would be lost. This code is only necessary to
- * maintain statistics, since the pmap module is always
- * consulted if m->dirty is false.
- */
-#if MACH_CLUSTER_STATS
- m->dirty = pmap_is_modified(m->phys_page);
+ if (m->vmp_overwriting) {
+ /*
+ * the (COPY_OUT_FROM == FALSE) request_page_list case
+ */
+ if (m->vmp_busy) {
+ /*
+ * We do not re-set m->vmp_dirty !
+ * The page was busy so no extraneous activity
+ * could have occurred. COPY_INTO is a read into the
+ * new pages. CLEAN_IN_PLACE does actually write
+ * out the pages but handling outside of this code
+ * will take care of resetting dirty. We clear the
+ * modify however for the Programmed I/O case.
+ */
+ pmap_clear_modify(VM_PAGE_GET_PHYS_PAGE(m));
- if (m->dirty) vm_pageout_cluster_dirtied++;
- else vm_pageout_cluster_cleaned++;
- if (m->wanted) vm_pageout_cluster_collisions++;
-#else
- m->dirty = 0;
-#endif
+ m->vmp_busy = FALSE;
+ m->vmp_absent = FALSE;
+ } else {
+ /*
+ * alternate (COPY_OUT_FROM == FALSE) request_page_list case
+ * Occurs when the original page was wired
+ * at the time of the list request
+ */
+ assert(VM_PAGE_WIRED(m));
+ vm_page_unwire(m, TRUE); /* reactivates */
+ }
+ m->vmp_overwriting = FALSE;
+ } else {
+ m->vmp_dirty = FALSE;
}
- m->cleaning = FALSE;
+ m->vmp_cleaning = FALSE;
/*
* Wakeup any thread waiting for the page to be un-cleaning.
/*
* Account for the paging reference taken in vm_paging_object_allocate.
*/
- vm_object_paging_end(shadow_object);
+ vm_object_activity_end(shadow_object);
vm_object_unlock(shadow_object);
assert(object->ref_count == 0);
assert(object->paging_in_progress == 0);
+ assert(object->activity_in_progress == 0);
assert(object->resident_page_count == 0);
return;
}
-/*
- * Routine: vm_pageout_setup
- * Purpose:
- * Set up a page for pageout (clean & flush).
- *
- * Move the page to a new object, as part of which it will be
- * sent to its memory manager in a memory_object_data_write or
- * memory_object_initialize message.
- *
- * The "new_object" and "new_offset" arguments
- * indicate where the page should be moved.
- *
- * In/Out conditions:
- * The page in question must not be on any pageout queues,
- * and must be busy. The object to which it belongs
- * must be unlocked, and the caller must hold a paging
- * reference to it. The new_object must not be locked.
- *
- * This routine returns a pointer to a place-holder page,
- * inserted at the same offset, to block out-of-order
- * requests for the page. The place-holder page must
- * be freed after the data_write or initialize message
- * has been sent.
- *
- * The original page is put on a paging queue and marked
- * not busy on exit.
- */
-vm_page_t
-vm_pageout_setup(
- register vm_page_t m,
- register vm_object_t new_object,
- vm_object_offset_t new_offset)
-{
- register vm_object_t old_object = m->object;
- vm_object_offset_t paging_offset;
- vm_object_offset_t offset;
- register vm_page_t holding_page;
- register vm_page_t new_m;
- boolean_t need_to_wire = FALSE;
-
-
- XPR(XPR_VM_PAGEOUT,
- "vm_pageout_setup, obj 0x%X off 0x%X page 0x%X new obj 0x%X offset 0x%X\n",
- (integer_t)m->object, (integer_t)m->offset,
- (integer_t)m, (integer_t)new_object,
- (integer_t)new_offset);
- assert(m && m->busy && !m->absent && !m->fictitious && !m->error &&
- !m->restart);
-
- assert(m->dirty || m->precious);
-
- /*
- * Create a place-holder page where the old one was, to prevent
- * attempted pageins of this page while we're unlocked.
- */
- VM_PAGE_GRAB_FICTITIOUS(holding_page);
-
- vm_object_lock(old_object);
-
- offset = m->offset;
- paging_offset = offset + old_object->paging_offset;
-
- if (old_object->pager_trusted) {
- /*
- * This pager is trusted, so we can clean this page
- * in place. Leave it in the old object, and mark it
- * cleaning & pageout.
- */
- new_m = holding_page;
- holding_page = VM_PAGE_NULL;
-
- /*
- * Set up new page to be private shadow of real page.
- */
- new_m->phys_page = m->phys_page;
- new_m->fictitious = FALSE;
- new_m->pageout = TRUE;
-
- /*
- * Mark real page as cleaning (indicating that we hold a
- * paging reference to be released via m_o_d_r_c) and
- * pageout (indicating that the page should be freed
- * when the pageout completes).
- */
- pmap_clear_modify(m->phys_page);
- vm_page_lock_queues();
- new_m->private = TRUE;
- vm_page_wire(new_m);
- m->cleaning = TRUE;
- m->pageout = TRUE;
-
- vm_page_wire(m);
- assert(m->wire_count == 1);
- vm_page_unlock_queues();
-
- m->dirty = TRUE;
- m->precious = FALSE;
- m->page_lock = VM_PROT_NONE;
- m->unusual = FALSE;
- m->unlock_request = VM_PROT_NONE;
- } else {
- /*
- * Cannot clean in place, so rip the old page out of the
- * object, and stick the holding page in. Set new_m to the
- * page in the new object.
- */
- vm_page_lock_queues();
- VM_PAGE_QUEUES_REMOVE(m);
- vm_page_remove(m);
-
- vm_page_insert(holding_page, old_object, offset);
- vm_page_unlock_queues();
-
- m->dirty = TRUE;
- m->precious = FALSE;
- new_m = m;
- new_m->page_lock = VM_PROT_NONE;
- new_m->unlock_request = VM_PROT_NONE;
-
- if (old_object->internal)
- need_to_wire = TRUE;
- }
- /*
- * Record that this page has been written out
- */
-#if MACH_PAGEMAP
- vm_external_state_set(old_object->existence_map, offset);
-#endif /* MACH_PAGEMAP */
-
- vm_object_unlock(old_object);
-
- vm_object_lock(new_object);
-
- /*
- * Put the page into the new object. If it is a not wired
- * (if it's the real page) it will be activated.
- */
-
- vm_page_lock_queues();
- vm_page_insert(new_m, new_object, new_offset);
- if (need_to_wire)
- vm_page_wire(new_m);
- else
- vm_page_activate(new_m);
- PAGE_WAKEUP_DONE(new_m);
- vm_page_unlock_queues();
-
- vm_object_unlock(new_object);
-
- /*
- * Return the placeholder page to simplify cleanup.
- */
- return (holding_page);
-}
-
/*
* Routine: vm_pageclean_setup
*
* necessarily flushed from the VM page cache.
* This is accomplished by cleaning in place.
*
- * The page must not be busy, and the object and page
- * queues must be locked.
- *
+ * The page must not be busy, and new_object
+ * must be locked.
+ *
*/
-void
+static void
vm_pageclean_setup(
- vm_page_t m,
- vm_page_t new_m,
- vm_object_t new_object,
- vm_object_offset_t new_offset)
+ vm_page_t m,
+ vm_page_t new_m,
+ vm_object_t new_object,
+ vm_object_offset_t new_offset)
{
- vm_object_t old_object = m->object;
- assert(!m->busy);
- assert(!m->cleaning);
-
- XPR(XPR_VM_PAGEOUT,
- "vm_pageclean_setup, obj 0x%X off 0x%X page 0x%X new 0x%X new_off 0x%X\n",
- (integer_t)old_object, m->offset, (integer_t)m,
- (integer_t)new_m, new_offset);
-
- pmap_clear_modify(m->phys_page);
- vm_object_paging_begin(old_object);
+ assert(!m->vmp_busy);
+#if 0
+ assert(!m->vmp_cleaning);
+#endif
- /*
- * Record that this page has been written out
- */
-#if MACH_PAGEMAP
- vm_external_state_set(old_object->existence_map, m->offset);
-#endif /*MACH_PAGEMAP*/
+ pmap_clear_modify(VM_PAGE_GET_PHYS_PAGE(m));
/*
* Mark original page as cleaning in place.
*/
- m->cleaning = TRUE;
- m->dirty = TRUE;
- m->precious = FALSE;
+ m->vmp_cleaning = TRUE;
+ SET_PAGE_DIRTY(m, FALSE);
+ m->vmp_precious = FALSE;
/*
* Convert the fictitious page to a private shadow of
* the real page.
*/
- assert(new_m->fictitious);
- new_m->fictitious = FALSE;
- new_m->private = TRUE;
- new_m->pageout = TRUE;
- new_m->phys_page = m->phys_page;
- vm_page_wire(new_m);
-
- vm_page_insert(new_m, new_object, new_offset);
- assert(!new_m->wanted);
- new_m->busy = FALSE;
-}
-
-void
-vm_pageclean_copy(
- vm_page_t m,
- vm_page_t new_m,
- vm_object_t new_object,
- vm_object_offset_t new_offset)
-{
- XPR(XPR_VM_PAGEOUT,
- "vm_pageclean_copy, page 0x%X new_m 0x%X new_obj 0x%X offset 0x%X\n",
- m, new_m, new_object, new_offset, 0);
-
- assert((!m->busy) && (!m->cleaning));
-
- assert(!new_m->private && !new_m->fictitious);
-
- pmap_clear_modify(m->phys_page);
-
- m->busy = TRUE;
- vm_object_paging_begin(m->object);
+ assert(new_m->vmp_fictitious);
+ assert(VM_PAGE_GET_PHYS_PAGE(new_m) == vm_page_fictitious_addr);
+ new_m->vmp_fictitious = FALSE;
+ new_m->vmp_private = TRUE;
+ new_m->vmp_free_when_done = TRUE;
+ VM_PAGE_SET_PHYS_PAGE(new_m, VM_PAGE_GET_PHYS_PAGE(m));
+
+ vm_page_lockspin_queues();
+ vm_page_wire(new_m, VM_KERN_MEMORY_NONE, TRUE);
vm_page_unlock_queues();
- vm_object_unlock(m->object);
-
- /*
- * Copy the original page to the new page.
- */
- vm_page_copy(m, new_m);
-
- /*
- * Mark the old page as clean. A request to pmap_is_modified
- * will get the right answer.
- */
- vm_object_lock(m->object);
- m->dirty = FALSE;
-
- vm_object_paging_end(m->object);
-
- vm_page_lock_queues();
- if (!m->active && !m->inactive)
- vm_page_activate(m);
- PAGE_WAKEUP_DONE(m);
- vm_page_insert(new_m, new_object, new_offset);
- vm_page_activate(new_m);
- new_m->busy = FALSE; /* No other thread can be waiting */
+ vm_page_insert_wired(new_m, new_object, new_offset, VM_KERN_MEMORY_NONE);
+ assert(!new_m->vmp_wanted);
+ new_m->vmp_busy = FALSE;
}
-
/*
* Routine: vm_pageout_initialize_page
* Purpose:
* Implementation:
* Move this page to a completely new object.
*/
-void
+void
vm_pageout_initialize_page(
- vm_page_t m)
+ vm_page_t m)
{
- vm_object_t object;
- vm_object_offset_t paging_offset;
- vm_page_t holding_page;
+ vm_object_t object;
+ vm_object_offset_t paging_offset;
+ memory_object_t pager;
+
+ assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
+ object = VM_PAGE_OBJECT(m);
- XPR(XPR_VM_PAGEOUT,
- "vm_pageout_initialize_page, page 0x%X\n",
- (integer_t)m, 0, 0, 0, 0);
- assert(m->busy);
+ assert(m->vmp_busy);
+ assert(object->internal);
/*
* Verify that we really want to clean this page
*/
- assert(!m->absent);
- assert(!m->error);
- assert(m->dirty);
+ assert(!m->vmp_absent);
+ assert(!m->vmp_error);
+ assert(m->vmp_dirty);
/*
* Create a paging reference to let us play with the object.
*/
- object = m->object;
- paging_offset = m->offset + object->paging_offset;
- vm_object_paging_begin(object);
- if (m->absent || m->error || m->restart ||
- (!m->dirty && !m->precious)) {
- VM_PAGE_FREE(m);
+ paging_offset = m->vmp_offset + object->paging_offset;
+
+ if (m->vmp_absent || m->vmp_error || m->vmp_restart || (!m->vmp_dirty && !m->vmp_precious)) {
panic("reservation without pageout?"); /* alan */
- vm_object_unlock(object);
+
+ VM_PAGE_FREE(m);
+ vm_object_unlock(object);
+
return;
}
- /* set the page for future call to vm_fault_list_request */
- holding_page = NULL;
- vm_page_lock_queues();
- pmap_clear_modify(m->phys_page);
- m->dirty = TRUE;
- m->busy = TRUE;
- m->list_req_pending = TRUE;
- m->cleaning = TRUE;
- m->pageout = TRUE;
- vm_page_wire(m);
- vm_page_unlock_queues();
+ /*
+ * If there's no pager, then we can't clean the page. This should
+ * never happen since this should be a copy object and therefore not
+ * an external object, so the pager should always be there.
+ */
+
+ pager = object->pager;
+
+ if (pager == MEMORY_OBJECT_NULL) {
+ panic("missing pager for copy object");
+
+ VM_PAGE_FREE(m);
+ return;
+ }
+
+ /*
+ * set the page for future call to vm_fault_list_request
+ */
+ pmap_clear_modify(VM_PAGE_GET_PHYS_PAGE(m));
+ SET_PAGE_DIRTY(m, FALSE);
+
+ /*
+ * keep the object from collapsing or terminating
+ */
+ vm_object_paging_begin(object);
vm_object_unlock(object);
/*
* [The object reference from its allocation is donated
* to the eventual recipient.]
*/
- memory_object_data_initialize(object->pager,
- paging_offset,
- PAGE_SIZE);
+ memory_object_data_initialize(pager, paging_offset, PAGE_SIZE);
vm_object_lock(object);
+ vm_object_paging_end(object);
}
-#if MACH_CLUSTER_STATS
-#define MAXCLUSTERPAGES 16
-struct {
- unsigned long pages_in_cluster;
- unsigned long pages_at_higher_offsets;
- unsigned long pages_at_lower_offsets;
-} cluster_stats[MAXCLUSTERPAGES];
-#endif /* MACH_CLUSTER_STATS */
-
-boolean_t allow_clustered_pageouts = FALSE;
/*
* vm_pageout_cluster:
* which will page it out and attempt to clean adjacent pages
* in the same operation.
*
- * The page must be busy, and the object and queues locked. We will take a
+ * The object and queues must be locked. We will take a
* paging reference to prevent deallocation or collapse when we
* release the object lock back at the call site. The I/O thread
* is responsible for consuming this reference
*
* The page must not be on any pageout queue.
*/
+#if DEVELOPMENT || DEBUG
+vmct_stats_t vmct_stats;
+
+int32_t vmct_active = 0;
+uint64_t vm_compressor_epoch_start = 0;
+uint64_t vm_compressor_epoch_stop = 0;
+
+typedef enum vmct_state_t {
+ VMCT_IDLE,
+ VMCT_AWAKENED,
+ VMCT_ACTIVE,
+} vmct_state_t;
+vmct_state_t vmct_state[MAX_COMPRESSOR_THREAD_COUNT];
+#endif
+
void
vm_pageout_cluster(vm_page_t m)
{
- vm_object_t object = m->object;
- struct vm_pageout_queue *q;
+ vm_object_t object = VM_PAGE_OBJECT(m);
+ struct vm_pageout_queue *q;
-
- XPR(XPR_VM_PAGEOUT,
- "vm_pageout_cluster, object 0x%X offset 0x%X page 0x%X\n",
- (integer_t)object, m->offset, (integer_t)m, 0, 0);
+ VM_PAGE_CHECK(m);
+ LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
+ vm_object_lock_assert_exclusive(object);
/*
* Only a certain kind of page is appreciated here.
*/
- assert(m->busy && (m->dirty || m->precious) && (m->wire_count == 0));
- assert(!m->cleaning && !m->pageout && !m->inactive && !m->active);
+ assert((m->vmp_dirty || m->vmp_precious) && (!VM_PAGE_WIRED(m)));
+ assert(!m->vmp_cleaning && !m->vmp_laundry);
+ assert(m->vmp_q_state == VM_PAGE_NOT_ON_Q);
/*
- * protect the object from collapse -
- * locking in the object's paging_offset.
+ * protect the object from collapse or termination
*/
- vm_object_paging_begin(object);
+ vm_object_activity_begin(object);
+
+ if (object->internal == TRUE) {
+ assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
+
+ m->vmp_busy = TRUE;
+
+ q = &vm_pageout_queue_internal;
+ } else {
+ q = &vm_pageout_queue_external;
+ }
/*
- * set the page for future call to vm_fault_list_request
- * page should already be marked busy
+ * pgo_laundry count is tied to the laundry bit
*/
- vm_page_wire(m);
- m->list_req_pending = TRUE;
- m->cleaning = TRUE;
- m->pageout = TRUE;
- m->laundry = TRUE;
-
- if (object->internal == TRUE)
- q = &vm_pageout_queue_internal;
- else
- q = &vm_pageout_queue_external;
+ m->vmp_laundry = TRUE;
q->pgo_laundry++;
- m->pageout_queue = TRUE;
- queue_enter(&q->pgo_pending, m, vm_page_t, pageq);
-
+ m->vmp_q_state = VM_PAGE_ON_PAGEOUT_Q;
+ vm_page_queue_enter(&q->pgo_pending, m, vmp_pageq);
+
if (q->pgo_idle == TRUE) {
- q->pgo_idle = FALSE;
- thread_wakeup((event_t) &q->pgo_pending);
+ q->pgo_idle = FALSE;
+ thread_wakeup((event_t) &q->pgo_pending);
}
+ VM_PAGE_CHECK(m);
}
-unsigned long vm_pageout_throttle_up_count = 0;
-
/*
- * A page is back from laundry. See if there are some pages waiting to
+ * A page is back from laundry or we are stealing it back from
+ * the laundering state. See if there are some pages waiting to
* go to laundry and if we can let some of them go now.
*
* Object and page queues must be locked.
*/
void
vm_pageout_throttle_up(
- vm_page_t m)
+ vm_page_t m)
{
- struct vm_pageout_queue *q;
+ struct vm_pageout_queue *q;
+ vm_object_t m_object;
- vm_pageout_throttle_up_count++;
+ m_object = VM_PAGE_OBJECT(m);
- assert(m->laundry);
- assert(m->object != VM_OBJECT_NULL);
- assert(m->object != kernel_object);
+ assert(m_object != VM_OBJECT_NULL);
+ assert(m_object != kernel_object);
- if (m->object->internal == TRUE)
- q = &vm_pageout_queue_internal;
- else
- q = &vm_pageout_queue_external;
+ LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
+ vm_object_lock_assert_exclusive(m_object);
- m->laundry = FALSE;
- q->pgo_laundry--;
-
- if (q->pgo_throttled == TRUE) {
- q->pgo_throttled = FALSE;
- thread_wakeup((event_t) &q->pgo_laundry);
+ if (m_object->internal == TRUE) {
+ q = &vm_pageout_queue_internal;
+ } else {
+ q = &vm_pageout_queue_external;
}
-}
+ if (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q) {
+ vm_page_queue_remove(&q->pgo_pending, m, vmp_pageq);
+ m->vmp_q_state = VM_PAGE_NOT_ON_Q;
-/*
- * vm_pageout_scan does the dirty work for the pageout daemon.
- * It returns with vm_page_queue_free_lock held and
- * vm_page_free_wanted == 0.
- */
-
-#define DELAYED_UNLOCK_LIMIT (3 * MAX_UPL_TRANSFER)
+ VM_PAGE_ZERO_PAGEQ_ENTRY(m);
-#define FCS_IDLE 0
-#define FCS_DELAYED 1
-#define FCS_DEADLOCK_DETECTED 2
+ vm_object_activity_end(m_object);
-struct flow_control {
- int state;
- mach_timespec_t ts;
-};
+ VM_PAGEOUT_DEBUG(vm_page_steal_pageout_page, 1);
+ }
+ if (m->vmp_laundry == TRUE) {
+ m->vmp_laundry = FALSE;
+ q->pgo_laundry--;
-extern kern_return_t sysclk_gettime(mach_timespec_t *);
+ if (q->pgo_throttled == TRUE) {
+ q->pgo_throttled = FALSE;
+ thread_wakeup((event_t) &q->pgo_laundry);
+ }
+ if (q->pgo_draining == TRUE && q->pgo_laundry == 0) {
+ q->pgo_draining = FALSE;
+ thread_wakeup((event_t) (&q->pgo_laundry + 1));
+ }
+ VM_PAGEOUT_DEBUG(vm_pageout_throttle_up_count, 1);
+ }
+}
-void
-vm_pageout_scan(void)
+static void
+vm_pageout_throttle_up_batch(
+ struct vm_pageout_queue *q,
+ int batch_cnt)
{
- unsigned int loop_count = 0;
- unsigned int inactive_burst_count = 0;
- unsigned int active_burst_count = 0;
- vm_page_t local_freeq = 0;
- int local_freed = 0;
- int delayed_unlock = 0;
- int need_internal_inactive = 0;
- int refmod_state = 0;
- int vm_pageout_deadlock_target = 0;
- struct vm_pageout_queue *iq;
- struct vm_pageout_queue *eq;
- struct flow_control flow_control;
- boolean_t active_throttled = FALSE;
- boolean_t inactive_throttled = FALSE;
- mach_timespec_t ts;
- unsigned int msecs = 0;
- vm_object_t object;
-
+ LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
- flow_control.state = FCS_IDLE;
- iq = &vm_pageout_queue_internal;
- eq = &vm_pageout_queue_external;
+ VM_PAGEOUT_DEBUG(vm_pageout_throttle_up_count, batch_cnt);
- XPR(XPR_VM_PAGEOUT, "vm_pageout_scan\n", 0, 0, 0, 0, 0);
+ q->pgo_laundry -= batch_cnt;
-/*???*/ /*
- * We want to gradually dribble pages from the active queue
- * to the inactive queue. If we let the inactive queue get
- * very small, and then suddenly dump many pages into it,
- * those pages won't get a sufficient chance to be referenced
- * before we start taking them from the inactive queue.
- *
- * We must limit the rate at which we send pages to the pagers.
- * data_write messages consume memory, for message buffers and
- * for map-copy objects. If we get too far ahead of the pagers,
- * we can potentially run out of memory.
- *
- * We can use the laundry count to limit directly the number
- * of pages outstanding to the default pager. A similar
- * strategy for external pagers doesn't work, because
- * external pagers don't have to deallocate the pages sent them,
- * and because we might have to send pages to external pagers
- * even if they aren't processing writes. So we also
- * use a burst count to limit writes to external pagers.
- *
- * When memory is very tight, we can't rely on external pagers to
- * clean pages. They probably aren't running, because they
- * aren't vm-privileged. If we kept sending dirty pages to them,
- * we could exhaust the free list.
- */
- vm_page_lock_queues();
- delayed_unlock = 1;
+ if (q->pgo_throttled == TRUE) {
+ q->pgo_throttled = FALSE;
+ thread_wakeup((event_t) &q->pgo_laundry);
+ }
+ if (q->pgo_draining == TRUE && q->pgo_laundry == 0) {
+ q->pgo_draining = FALSE;
+ thread_wakeup((event_t) (&q->pgo_laundry + 1));
+ }
+}
-Restart:
- /*
- * Recalculate vm_page_inactivate_target.
- */
- vm_page_inactive_target = VM_PAGE_INACTIVE_TARGET(vm_page_active_count +
- vm_page_inactive_count);
- object = NULL;
- for (;;) {
- vm_page_t m;
+/*
+ * VM memory pressure monitoring.
+ *
+ * vm_pageout_scan() keeps track of the number of pages it considers and
+ * reclaims, in the currently active vm_pageout_stat[vm_pageout_stat_now].
+ *
+ * compute_memory_pressure() is called every second from compute_averages()
+ * and moves "vm_pageout_stat_now" forward, to start accumulating the number
+ * of recalimed pages in a new vm_pageout_stat[] bucket.
+ *
+ * mach_vm_pressure_monitor() collects past statistics about memory pressure.
+ * The caller provides the number of seconds ("nsecs") worth of statistics
+ * it wants, up to 30 seconds.
+ * It computes the number of pages reclaimed in the past "nsecs" seconds and
+ * also returns the number of pages the system still needs to reclaim at this
+ * moment in time.
+ */
+#if DEVELOPMENT || DEBUG
+#define VM_PAGEOUT_STAT_SIZE (30 * 8) + 1
+#else
+#define VM_PAGEOUT_STAT_SIZE (1 * 8) + 1
+#endif
+struct vm_pageout_stat {
+ unsigned long vm_page_active_count;
+ unsigned long vm_page_speculative_count;
+ unsigned long vm_page_inactive_count;
+ unsigned long vm_page_anonymous_count;
- if (delayed_unlock == 0)
- vm_page_lock_queues();
+ unsigned long vm_page_free_count;
+ unsigned long vm_page_wire_count;
+ unsigned long vm_page_compressor_count;
- active_burst_count = vm_page_active_count;
+ unsigned long vm_page_pages_compressed;
+ unsigned long vm_page_pageable_internal_count;
+ unsigned long vm_page_pageable_external_count;
+ unsigned long vm_page_xpmapped_external_count;
- if (active_burst_count > vm_pageout_burst_active_throttle)
- active_burst_count = vm_pageout_burst_active_throttle;
+ unsigned int pages_grabbed;
+ unsigned int pages_freed;
- /*
- * Move pages from active to inactive.
- */
- while ((need_internal_inactive ||
- vm_page_inactive_count < vm_page_inactive_target) &&
- !queue_empty(&vm_page_queue_active) &&
- ((active_burst_count--) > 0)) {
+ unsigned int pages_compressed;
+ unsigned int pages_grabbed_by_compressor;
+ unsigned int failed_compressions;
- vm_pageout_active++;
+ unsigned int pages_evicted;
+ unsigned int pages_purged;
- m = (vm_page_t) queue_first(&vm_page_queue_active);
+ unsigned int considered;
+ unsigned int considered_bq_internal;
+ unsigned int considered_bq_external;
- assert(m->active && !m->inactive);
- assert(!m->laundry);
- assert(m->object != kernel_object);
+ unsigned int skipped_external;
+ unsigned int filecache_min_reactivations;
- /*
- * Try to lock object; since we've already got the
- * page queues lock, we can only 'try' for this one.
- * if the 'try' fails, we need to do a mutex_pause
- * to allow the owner of the object lock a chance to
- * run... otherwise, we're likely to trip over this
- * object in the same state as we work our way through
- * the queue... clumps of pages associated with the same
- * object are fairly typical on the inactive and active queues
- */
- if (m->object != object) {
- if (object != NULL) {
- vm_object_unlock(object);
- object = NULL;
- }
- if (!vm_object_lock_try(m->object)) {
- /*
- * move page to end of active queue and continue
- */
- queue_remove(&vm_page_queue_active, m,
- vm_page_t, pageq);
- queue_enter(&vm_page_queue_active, m,
- vm_page_t, pageq);
-
- goto done_with_activepage;
- }
- object = m->object;
- }
- /*
- * if the page is BUSY, then we pull it
- * off the active queue and leave it alone.
- * when BUSY is cleared, it will get stuck
- * back on the appropriate queue
- */
- if (m->busy) {
- queue_remove(&vm_page_queue_active, m,
- vm_page_t, pageq);
- m->pageq.next = NULL;
- m->pageq.prev = NULL;
-
- if (!m->fictitious)
- vm_page_active_count--;
- m->active = FALSE;
-
- goto done_with_activepage;
- }
- if (need_internal_inactive) {
- /*
- * If we're unable to make forward progress
- * with the current set of pages on the
- * inactive queue due to busy objects or
- * throttled pageout queues, then
- * move a page that is already clean
- * or belongs to a pageout queue that
- * isn't currently throttled
- */
- active_throttled = FALSE;
+ unsigned int freed_speculative;
+ unsigned int freed_cleaned;
+ unsigned int freed_internal;
+ unsigned int freed_external;
- if (object->internal) {
- if ((VM_PAGE_Q_THROTTLED(iq) || !IP_VALID(memory_manager_default)))
- active_throttled = TRUE;
- } else if (VM_PAGE_Q_THROTTLED(eq)) {
- active_throttled = TRUE;
- }
- if (active_throttled == TRUE) {
- if (!m->dirty) {
- refmod_state = pmap_get_refmod(m->phys_page);
-
- if (refmod_state & VM_MEM_REFERENCED)
- m->reference = TRUE;
- if (refmod_state & VM_MEM_MODIFIED)
- m->dirty = TRUE;
- }
- if (m->dirty || m->precious) {
- /*
- * page is dirty and targets a THROTTLED queue
- * so all we can do is move it back to the
- * end of the active queue to get it out
- * of the way
- */
- queue_remove(&vm_page_queue_active, m,
- vm_page_t, pageq);
- queue_enter(&vm_page_queue_active, m,
- vm_page_t, pageq);
+ unsigned int cleaned_dirty_external;
+ unsigned int cleaned_dirty_internal;
- vm_pageout_scan_active_throttled++;
+ unsigned int inactive_referenced;
+ unsigned int inactive_nolock;
+ unsigned int reactivation_limit_exceeded;
+ unsigned int forced_inactive_reclaim;
- goto done_with_activepage;
- }
- }
- vm_pageout_scan_active_throttle_success++;
- need_internal_inactive--;
- }
- /*
- * Deactivate the page while holding the object
- * locked, so we know the page is still not busy.
- * This should prevent races between pmap_enter
- * and pmap_clear_reference. The page might be
- * absent or fictitious, but vm_page_deactivate
- * can handle that.
- */
- vm_page_deactivate(m);
-done_with_activepage:
- if (delayed_unlock++ > DELAYED_UNLOCK_LIMIT) {
+ unsigned int throttled_internal_q;
+ unsigned int throttled_external_q;
- if (object != NULL) {
- vm_object_unlock(object);
- object = NULL;
- }
- if (local_freeq) {
- vm_page_free_list(local_freeq);
-
- local_freeq = 0;
- local_freed = 0;
- }
- delayed_unlock = 0;
- vm_page_unlock_queues();
+ unsigned int phantom_ghosts_found;
+ unsigned int phantom_ghosts_added;
+} vm_pageout_stats[VM_PAGEOUT_STAT_SIZE] = {{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, };
- mutex_pause();
- vm_page_lock_queues();
- /*
- * continue the while loop processing
- * the active queue... need to hold
- * the page queues lock
- */
- continue;
- }
- }
+unsigned int vm_pageout_stat_now = 0;
+#define VM_PAGEOUT_STAT_BEFORE(i) \
+ (((i) == 0) ? VM_PAGEOUT_STAT_SIZE - 1 : (i) - 1)
+#define VM_PAGEOUT_STAT_AFTER(i) \
+ (((i) == VM_PAGEOUT_STAT_SIZE - 1) ? 0 : (i) + 1)
+#if VM_PAGE_BUCKETS_CHECK
+int vm_page_buckets_check_interval = 80; /* in eighths of a second */
+#endif /* VM_PAGE_BUCKETS_CHECK */
- /**********************************************************************
- * above this point we're playing with the active queue
- * below this point we're playing with the throttling mechanisms
- * and the inactive queue
- **********************************************************************/
+void
+record_memory_pressure(void);
+void
+record_memory_pressure(void)
+{
+ unsigned int vm_pageout_next;
+#if VM_PAGE_BUCKETS_CHECK
+ /* check the consistency of VM page buckets at regular interval */
+ static int counter = 0;
+ if ((++counter % vm_page_buckets_check_interval) == 0) {
+ vm_page_buckets_check();
+ }
+#endif /* VM_PAGE_BUCKETS_CHECK */
- /*
- * We are done if we have met our target *and*
- * nobody is still waiting for a page.
- */
- if (vm_page_free_count + local_freed >= vm_page_free_target) {
- if (object != NULL) {
- vm_object_unlock(object);
- object = NULL;
- }
- if (local_freeq) {
- vm_page_free_list(local_freeq);
-
- local_freeq = 0;
- local_freed = 0;
- }
- mutex_lock(&vm_page_queue_free_lock);
+ vm_pageout_state.vm_memory_pressure =
+ vm_pageout_stats[VM_PAGEOUT_STAT_BEFORE(vm_pageout_stat_now)].freed_speculative +
+ vm_pageout_stats[VM_PAGEOUT_STAT_BEFORE(vm_pageout_stat_now)].freed_cleaned +
+ vm_pageout_stats[VM_PAGEOUT_STAT_BEFORE(vm_pageout_stat_now)].freed_internal +
+ vm_pageout_stats[VM_PAGEOUT_STAT_BEFORE(vm_pageout_stat_now)].freed_external;
- if ((vm_page_free_count >= vm_page_free_target) &&
- (vm_page_free_wanted == 0)) {
+ commpage_set_memory_pressure((unsigned int)vm_pageout_state.vm_memory_pressure );
- vm_page_unlock_queues();
+ /* move "now" forward */
+ vm_pageout_next = VM_PAGEOUT_STAT_AFTER(vm_pageout_stat_now);
- thread_wakeup((event_t) &vm_pageout_garbage_collect);
- return;
- }
- mutex_unlock(&vm_page_queue_free_lock);
- }
+ bzero(&vm_pageout_stats[vm_pageout_next], sizeof(struct vm_pageout_stat));
+ vm_pageout_stat_now = vm_pageout_next;
+}
- /*
- * Sometimes we have to pause:
- * 1) No inactive pages - nothing to do.
- * 2) Flow control - default pageout queue is full
- * 3) Loop control - no acceptable pages found on the inactive queue
- * within the last vm_pageout_burst_inactive_throttle iterations
- */
- if ((queue_empty(&vm_page_queue_inactive) && queue_empty(&vm_page_queue_zf))) {
- vm_pageout_scan_empty_throttle++;
- msecs = vm_pageout_empty_wait;
- goto vm_pageout_scan_delay;
-
- } else if (inactive_burst_count >= vm_pageout_burst_inactive_throttle) {
- vm_pageout_scan_burst_throttle++;
- msecs = vm_pageout_burst_wait;
- goto vm_pageout_scan_delay;
-
- } else if (VM_PAGE_Q_THROTTLED(iq)) {
-
- switch (flow_control.state) {
-
- case FCS_IDLE:
-reset_deadlock_timer:
- ts.tv_sec = vm_pageout_deadlock_wait / 1000;
- ts.tv_nsec = (vm_pageout_deadlock_wait % 1000) * 1000 * NSEC_PER_USEC;
- sysclk_gettime(&flow_control.ts);
- ADD_MACH_TIMESPEC(&flow_control.ts, &ts);
-
- flow_control.state = FCS_DELAYED;
- msecs = vm_pageout_deadlock_wait;
- break;
-
- case FCS_DELAYED:
- sysclk_gettime(&ts);
-
- if (CMP_MACH_TIMESPEC(&ts, &flow_control.ts) >= 0) {
- /*
- * the pageout thread for the default pager is potentially
- * deadlocked since the
- * default pager queue has been throttled for more than the
- * allowable time... we need to move some clean pages or dirty
- * pages belonging to the external pagers if they aren't throttled
- * vm_page_free_wanted represents the number of threads currently
- * blocked waiting for pages... we'll move one page for each of
- * these plus a fixed amount to break the logjam... once we're done
- * moving this number of pages, we'll re-enter the FSC_DELAYED state
- * with a new timeout target since we have no way of knowing
- * whether we've broken the deadlock except through observation
- * of the queue associated with the default pager... we need to
- * stop moving pagings and allow the system to run to see what
- * state it settles into.
- */
- vm_pageout_deadlock_target = vm_pageout_deadlock_relief + vm_page_free_wanted;
- vm_pageout_scan_deadlock_detected++;
- flow_control.state = FCS_DEADLOCK_DETECTED;
+/*
+ * IMPORTANT
+ * mach_vm_ctl_page_free_wanted() is called indirectly, via
+ * mach_vm_pressure_monitor(), when taking a stackshot. Therefore,
+ * it must be safe in the restricted stackshot context. Locks and/or
+ * blocking are not allowable.
+ */
+unsigned int
+mach_vm_ctl_page_free_wanted(void)
+{
+ unsigned int page_free_target, page_free_count, page_free_wanted;
- thread_wakeup((event_t) &vm_pageout_garbage_collect);
- goto consider_inactive;
- }
- /*
- * just resniff instead of trying
- * to compute a new delay time... we're going to be
- * awakened immediately upon a laundry completion,
- * so we won't wait any longer than necessary
- */
- msecs = vm_pageout_idle_wait;
- break;
+ page_free_target = vm_page_free_target;
+ page_free_count = vm_page_free_count;
+ if (page_free_target > page_free_count) {
+ page_free_wanted = page_free_target - page_free_count;
+ } else {
+ page_free_wanted = 0;
+ }
+
+ return page_free_wanted;
+}
+
+
+/*
+ * IMPORTANT:
+ * mach_vm_pressure_monitor() is called when taking a stackshot, with
+ * wait_for_pressure FALSE, so that code path must remain safe in the
+ * restricted stackshot context. No blocking or locks are allowable.
+ * on that code path.
+ */
- case FCS_DEADLOCK_DETECTED:
- if (vm_pageout_deadlock_target)
- goto consider_inactive;
- goto reset_deadlock_timer;
+kern_return_t
+mach_vm_pressure_monitor(
+ boolean_t wait_for_pressure,
+ unsigned int nsecs_monitored,
+ unsigned int *pages_reclaimed_p,
+ unsigned int *pages_wanted_p)
+{
+ wait_result_t wr;
+ unsigned int vm_pageout_then, vm_pageout_now;
+ unsigned int pages_reclaimed;
+ unsigned int units_of_monitor;
+
+ units_of_monitor = 8 * nsecs_monitored;
+ /*
+ * We don't take the vm_page_queue_lock here because we don't want
+ * vm_pressure_monitor() to get in the way of the vm_pageout_scan()
+ * thread when it's trying to reclaim memory. We don't need fully
+ * accurate monitoring anyway...
+ */
+ if (wait_for_pressure) {
+ /* wait until there's memory pressure */
+ while (vm_page_free_count >= vm_page_free_target) {
+ wr = assert_wait((event_t) &vm_page_free_wanted,
+ THREAD_INTERRUPTIBLE);
+ if (wr == THREAD_WAITING) {
+ wr = thread_block(THREAD_CONTINUE_NULL);
}
- vm_pageout_scan_throttle++;
- iq->pgo_throttled = TRUE;
-vm_pageout_scan_delay:
- if (object != NULL) {
- vm_object_unlock(object);
- object = NULL;
+ if (wr == THREAD_INTERRUPTED) {
+ return KERN_ABORTED;
}
- if (local_freeq) {
- vm_page_free_list(local_freeq);
-
- local_freeq = 0;
- local_freed = 0;
+ if (wr == THREAD_AWAKENED) {
+ /*
+ * The memory pressure might have already
+ * been relieved but let's not block again
+ * and let's report that there was memory
+ * pressure at some point.
+ */
+ break;
}
- assert_wait_timeout((event_t) &iq->pgo_laundry, THREAD_INTERRUPTIBLE, msecs, 1000*NSEC_PER_USEC);
+ }
+ }
- counter(c_vm_pageout_scan_block++);
+ /* provide the number of pages the system wants to reclaim */
+ if (pages_wanted_p != NULL) {
+ *pages_wanted_p = mach_vm_ctl_page_free_wanted();
+ }
- vm_page_unlock_queues();
-
- thread_block(THREAD_CONTINUE_NULL);
+ if (pages_reclaimed_p == NULL) {
+ return KERN_SUCCESS;
+ }
- vm_page_lock_queues();
- delayed_unlock = 1;
+ /* provide number of pages reclaimed in the last "nsecs_monitored" */
+ vm_pageout_now = vm_pageout_stat_now;
+ pages_reclaimed = 0;
+ for (vm_pageout_then =
+ VM_PAGEOUT_STAT_BEFORE(vm_pageout_now);
+ vm_pageout_then != vm_pageout_now &&
+ units_of_monitor-- != 0;
+ vm_pageout_then =
+ VM_PAGEOUT_STAT_BEFORE(vm_pageout_then)) {
+ pages_reclaimed += vm_pageout_stats[vm_pageout_then].freed_speculative;
+ pages_reclaimed += vm_pageout_stats[vm_pageout_then].freed_cleaned;
+ pages_reclaimed += vm_pageout_stats[vm_pageout_then].freed_internal;
+ pages_reclaimed += vm_pageout_stats[vm_pageout_then].freed_external;
+ }
+ *pages_reclaimed_p = pages_reclaimed;
- iq->pgo_throttled = FALSE;
+ return KERN_SUCCESS;
+}
- if (loop_count >= vm_page_inactive_count) {
- if (VM_PAGE_Q_THROTTLED(eq) || VM_PAGE_Q_THROTTLED(iq)) {
- /*
- * Make sure we move enough "appropriate"
- * pages to the inactive queue before trying
- * again.
- */
- need_internal_inactive = vm_pageout_inactive_relief;
- }
- loop_count = 0;
- }
- inactive_burst_count = 0;
- goto Restart;
- /*NOTREACHED*/
- }
+#if DEVELOPMENT || DEBUG
- flow_control.state = FCS_IDLE;
-consider_inactive:
- loop_count++;
- inactive_burst_count++;
- vm_pageout_inactive++;
+static void
+vm_pageout_disconnect_all_pages_in_queue(vm_page_queue_head_t *, int);
- if (!queue_empty(&vm_page_queue_inactive)) {
- m = (vm_page_t) queue_first(&vm_page_queue_inactive);
-
- if (m->clustered && (m->no_isync == TRUE)) {
- goto use_this_page;
- }
- }
- if (vm_zf_count < vm_accellerate_zf_pageout_trigger) {
- vm_zf_iterator = 0;
- } else {
- last_page_zf = 0;
- if((vm_zf_iterator+=1) >= vm_zf_iterator_count) {
- vm_zf_iterator = 0;
- }
- }
- if (queue_empty(&vm_page_queue_zf) ||
- (((last_page_zf) || (vm_zf_iterator == 0)) &&
- !queue_empty(&vm_page_queue_inactive))) {
- m = (vm_page_t) queue_first(&vm_page_queue_inactive);
- last_page_zf = 0;
- } else {
- m = (vm_page_t) queue_first(&vm_page_queue_zf);
- last_page_zf = 1;
- }
-use_this_page:
- assert(!m->active && m->inactive);
- assert(!m->laundry);
- assert(m->object != kernel_object);
+/*
+ * condition variable used to make sure there is
+ * only a single sweep going on at a time
+ */
+boolean_t vm_pageout_disconnect_all_pages_active = FALSE;
- /*
- * Try to lock object; since we've alread got the
- * page queues lock, we can only 'try' for this one.
- * if the 'try' fails, we need to do a mutex_pause
- * to allow the owner of the object lock a chance to
- * run... otherwise, we're likely to trip over this
- * object in the same state as we work our way through
- * the queue... clumps of pages associated with the same
- * object are fairly typical on the inactive and active queues
- */
- if (m->object != object) {
- if (object != NULL) {
- vm_object_unlock(object);
- object = NULL;
- }
- if (!vm_object_lock_try(m->object)) {
- /*
- * Move page to end and continue.
- * Don't re-issue ticket
- */
- if (m->zero_fill) {
- queue_remove(&vm_page_queue_zf, m,
- vm_page_t, pageq);
- queue_enter(&vm_page_queue_zf, m,
- vm_page_t, pageq);
- } else {
- queue_remove(&vm_page_queue_inactive, m,
- vm_page_t, pageq);
- queue_enter(&vm_page_queue_inactive, m,
- vm_page_t, pageq);
- }
- vm_pageout_inactive_nolock++;
- /*
- * force us to dump any collected free pages
- * and to pause before moving on
- */
- delayed_unlock = DELAYED_UNLOCK_LIMIT + 1;
+void
+vm_pageout_disconnect_all_pages()
+{
+ vm_page_lock_queues();
- goto done_with_inactivepage;
- }
- object = m->object;
- }
- /*
- * If the page belongs to a purgable object with no pending copies
- * against it, then we reap all of the pages in the object
- * and note that the object has been "emptied". It'll be up to the
- * application the discover this and recreate its contents if desired.
- */
- if ((object->purgable == VM_OBJECT_PURGABLE_VOLATILE ||
- object->purgable == VM_OBJECT_PURGABLE_EMPTY) &&
- object->copy == VM_OBJECT_NULL) {
+ if (vm_pageout_disconnect_all_pages_active == TRUE) {
+ vm_page_unlock_queues();
+ return;
+ }
+ vm_pageout_disconnect_all_pages_active = TRUE;
+ vm_page_unlock_queues();
- (void) vm_object_purge(object);
- vm_pageout_purged_objects++;
- /*
- * we've just taken all of the pages from this object,
- * so drop the lock now since we're not going to find
- * any more pages belonging to it anytime soon
- */
- vm_object_unlock(object);
- object = NULL;
+ vm_pageout_disconnect_all_pages_in_queue(&vm_page_queue_throttled, vm_page_throttled_count);
+ vm_pageout_disconnect_all_pages_in_queue(&vm_page_queue_anonymous, vm_page_anonymous_count);
+ vm_pageout_disconnect_all_pages_in_queue(&vm_page_queue_active, vm_page_active_count);
- inactive_burst_count = 0;
+ vm_pageout_disconnect_all_pages_active = FALSE;
+}
- goto done_with_inactivepage;
- }
+
+void
+vm_pageout_disconnect_all_pages_in_queue(vm_page_queue_head_t *q, int qcount)
+{
+ vm_page_t m;
+ vm_object_t t_object = NULL;
+ vm_object_t l_object = NULL;
+ vm_object_t m_object = NULL;
+ int delayed_unlock = 0;
+ int try_failed_count = 0;
+ int disconnected_count = 0;
+ int paused_count = 0;
+ int object_locked_count = 0;
+
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_DISCONNECT_ALL_PAGE_MAPPINGS)) | DBG_FUNC_START,
+ q, qcount, 0, 0, 0);
+
+ vm_page_lock_queues();
+
+ while (qcount && !vm_page_queue_empty(q)) {
+ LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
+
+ m = (vm_page_t) vm_page_queue_first(q);
+ m_object = VM_PAGE_OBJECT(m);
/*
- * Paging out pages of external objects which
- * are currently being created must be avoided.
- * The pager may claim for memory, thus leading to a
- * possible dead lock between it and the pageout thread,
- * if such pages are finally chosen. The remaining assumption
- * is that there will finally be enough available pages in the
- * inactive pool to page out in order to satisfy all memory
- * claimed by the thread which concurrently creates the pager.
+ * check to see if we currently are working
+ * with the same object... if so, we've
+ * already got the lock
*/
- if (!object->pager_initialized && object->pager_created) {
+ if (m_object != l_object) {
/*
- * Move page to end and continue, hoping that
- * there will be enough other inactive pages to
- * page out so that the thread which currently
- * initializes the pager will succeed.
- * Don't re-grant the ticket, the page should
- * pulled from the queue and paged out whenever
- * one of its logically adjacent fellows is
- * targeted.
+ * the object associated with candidate page is
+ * different from the one we were just working
+ * with... dump the lock if we still own it
*/
- if (m->zero_fill) {
- queue_remove(&vm_page_queue_zf, m,
- vm_page_t, pageq);
- queue_enter(&vm_page_queue_zf, m,
- vm_page_t, pageq);
- last_page_zf = 1;
- vm_zf_iterator = vm_zf_iterator_count - 1;
- } else {
- queue_remove(&vm_page_queue_inactive, m,
- vm_page_t, pageq);
- queue_enter(&vm_page_queue_inactive, m,
- vm_page_t, pageq);
- last_page_zf = 0;
- vm_zf_iterator = 1;
+ if (l_object != NULL) {
+ vm_object_unlock(l_object);
+ l_object = NULL;
+ }
+ if (m_object != t_object) {
+ try_failed_count = 0;
}
- vm_pageout_inactive_avoid++;
- goto done_with_inactivepage;
- }
- /*
- * Remove the page from the inactive list.
- */
- if (m->zero_fill) {
- queue_remove(&vm_page_queue_zf, m, vm_page_t, pageq);
- } else {
- queue_remove(&vm_page_queue_inactive, m, vm_page_t, pageq);
- }
- m->pageq.next = NULL;
- m->pageq.prev = NULL;
- m->inactive = FALSE;
- if (!m->fictitious)
- vm_page_inactive_count--;
+ /*
+ * Try to lock object; since we've alread got the
+ * page queues lock, we can only 'try' for this one.
+ * if the 'try' fails, we need to do a mutex_pause
+ * to allow the owner of the object lock a chance to
+ * run...
+ */
+ if (!vm_object_lock_try_scan(m_object)) {
+ if (try_failed_count > 20) {
+ goto reenter_pg_on_q;
+ }
+ vm_page_unlock_queues();
+ mutex_pause(try_failed_count++);
+ vm_page_lock_queues();
+ delayed_unlock = 0;
+
+ paused_count++;
+
+ t_object = m_object;
+ continue;
+ }
+ object_locked_count++;
- if (m->busy || !object->alive) {
+ l_object = m_object;
+ }
+ if (!m_object->alive || m->vmp_cleaning || m->vmp_laundry || m->vmp_busy || m->vmp_absent || m->vmp_error || m->vmp_free_when_done) {
/*
- * Somebody is already playing with this page.
- * Leave it off the pageout queues.
+ * put it back on the head of its queue
*/
- vm_pageout_inactive_busy++;
+ goto reenter_pg_on_q;
+ }
+ if (m->vmp_pmapped == TRUE) {
+ pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
- goto done_with_inactivepage;
+ disconnected_count++;
}
+reenter_pg_on_q:
+ vm_page_queue_remove(q, m, vmp_pageq);
+ vm_page_queue_enter(q, m, vmp_pageq);
- /*
- * If it's absent or in error, we can reclaim the page.
- */
+ qcount--;
+ try_failed_count = 0;
- if (m->absent || m->error) {
- vm_pageout_inactive_absent++;
-reclaim_page:
- if (vm_pageout_deadlock_target) {
- vm_pageout_scan_inactive_throttle_success++;
- vm_pageout_deadlock_target--;
+ if (delayed_unlock++ > 128) {
+ if (l_object != NULL) {
+ vm_object_unlock(l_object);
+ l_object = NULL;
}
- if (m->tabled)
- vm_page_remove(m); /* clears tabled, object, offset */
- if (m->absent)
- vm_object_absent_release(object);
+ lck_mtx_yield(&vm_page_queue_lock);
+ delayed_unlock = 0;
+ }
+ }
+ if (l_object != NULL) {
+ vm_object_unlock(l_object);
+ l_object = NULL;
+ }
+ vm_page_unlock_queues();
- assert(m->pageq.next == NULL &&
- m->pageq.prev == NULL);
- m->pageq.next = (queue_entry_t)local_freeq;
- local_freeq = m;
- local_freed++;
+ KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_DISCONNECT_ALL_PAGE_MAPPINGS)) | DBG_FUNC_END,
+ q, disconnected_count, object_locked_count, paused_count, 0);
+}
- inactive_burst_count = 0;
+#endif
- goto done_with_inactivepage;
- }
- assert(!m->private);
- assert(!m->fictitious);
+static void
+vm_pageout_page_queue(vm_page_queue_head_t *, int);
- /*
- * If already cleaning this page in place, convert from
- * "adjacent" to "target". We can leave the page mapped,
- * and vm_pageout_object_terminate will determine whether
- * to free or reactivate.
- */
+/*
+ * condition variable used to make sure there is
+ * only a single sweep going on at a time
+ */
+boolean_t vm_pageout_anonymous_pages_active = FALSE;
- if (m->cleaning) {
- m->busy = TRUE;
- m->pageout = TRUE;
- m->dump_cleaning = TRUE;
- vm_page_wire(m);
- CLUSTER_STAT(vm_pageout_cluster_conversions++);
+void
+vm_pageout_anonymous_pages()
+{
+ if (VM_CONFIG_COMPRESSOR_IS_PRESENT) {
+ vm_page_lock_queues();
- inactive_burst_count = 0;
+ if (vm_pageout_anonymous_pages_active == TRUE) {
+ vm_page_unlock_queues();
+ return;
+ }
+ vm_pageout_anonymous_pages_active = TRUE;
+ vm_page_unlock_queues();
- goto done_with_inactivepage;
+ vm_pageout_page_queue(&vm_page_queue_throttled, vm_page_throttled_count);
+ vm_pageout_page_queue(&vm_page_queue_anonymous, vm_page_anonymous_count);
+ vm_pageout_page_queue(&vm_page_queue_active, vm_page_active_count);
+
+ if (VM_CONFIG_SWAP_IS_PRESENT) {
+ vm_consider_swapping();
}
- /*
- * If it's being used, reactivate.
- * (Fictitious pages are either busy or absent.)
- */
- if ( (!m->reference) ) {
- refmod_state = pmap_get_refmod(m->phys_page);
-
- if (refmod_state & VM_MEM_REFERENCED)
- m->reference = TRUE;
- if (refmod_state & VM_MEM_MODIFIED)
- m->dirty = TRUE;
- }
- if (m->reference) {
-was_referenced:
- vm_page_activate(m);
- VM_STAT(reactivations++);
+ vm_page_lock_queues();
+ vm_pageout_anonymous_pages_active = FALSE;
+ vm_page_unlock_queues();
+ }
+}
- vm_pageout_inactive_used++;
- last_page_zf = 0;
- inactive_burst_count = 0;
- goto done_with_inactivepage;
- }
+void
+vm_pageout_page_queue(vm_page_queue_head_t *q, int qcount)
+{
+ vm_page_t m;
+ vm_object_t t_object = NULL;
+ vm_object_t l_object = NULL;
+ vm_object_t m_object = NULL;
+ int delayed_unlock = 0;
+ int try_failed_count = 0;
+ int refmod_state;
+ int pmap_options;
+ struct vm_pageout_queue *iq;
+ ppnum_t phys_page;
- XPR(XPR_VM_PAGEOUT,
- "vm_pageout_scan, replace object 0x%X offset 0x%X page 0x%X\n",
- (integer_t)object, (integer_t)m->offset, (integer_t)m, 0,0);
- /*
- * we've got a candidate page to steal...
- *
- * m->dirty is up to date courtesy of the
- * preceding check for m->reference... if
- * we get here, then m->reference had to be
- * FALSE which means we did a pmap_get_refmod
- * and updated both m->reference and m->dirty
- *
- * if it's dirty or precious we need to
- * see if the target queue is throtttled
- * it if is, we need to skip over it by moving it back
- * to the end of the inactive queue
- */
- inactive_throttled = FALSE;
+ iq = &vm_pageout_queue_internal;
- if (m->dirty || m->precious) {
- if (object->internal) {
- if ((VM_PAGE_Q_THROTTLED(iq) || !IP_VALID(memory_manager_default)))
- inactive_throttled = TRUE;
- } else if (VM_PAGE_Q_THROTTLED(eq)) {
- inactive_throttled = TRUE;
- }
- }
- if (inactive_throttled == TRUE) {
- if (m->zero_fill) {
- queue_enter(&vm_page_queue_zf, m,
- vm_page_t, pageq);
- } else {
- queue_enter(&vm_page_queue_inactive, m,
- vm_page_t, pageq);
+ vm_page_lock_queues();
+
+ while (qcount && !vm_page_queue_empty(q)) {
+ LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
+
+ if (VM_PAGE_Q_THROTTLED(iq)) {
+ if (l_object != NULL) {
+ vm_object_unlock(l_object);
+ l_object = NULL;
}
- if (!m->fictitious)
- vm_page_inactive_count++;
- m->inactive = TRUE;
+ iq->pgo_draining = TRUE;
+
+ assert_wait((event_t) (&iq->pgo_laundry + 1), THREAD_INTERRUPTIBLE);
+ vm_page_unlock_queues();
- vm_pageout_scan_inactive_throttled++;
+ thread_block(THREAD_CONTINUE_NULL);
- goto done_with_inactivepage;
+ vm_page_lock_queues();
+ delayed_unlock = 0;
+ continue;
}
+ m = (vm_page_t) vm_page_queue_first(q);
+ m_object = VM_PAGE_OBJECT(m);
+
/*
- * we've got a page that we can steal...
- * eliminate all mappings and make sure
- * we have the up-to-date modified state
- * first take the page BUSY, so that no new
- * mappings can be made
+ * check to see if we currently are working
+ * with the same object... if so, we've
+ * already got the lock
*/
- m->busy = TRUE;
-
- /*
- * if we need to do a pmap_disconnect then we
- * need to re-evaluate m->dirty since the pmap_disconnect
- * provides the true state atomically... the
- * page was still mapped up to the pmap_disconnect
- * and may have been dirtied at the last microsecond
- *
- * we also check for the page being referenced 'late'
- * if it was, we first need to do a WAKEUP_DONE on it
- * since we already set m->busy = TRUE, before
- * going off to reactivate it
- *
- * if we don't need the pmap_disconnect, then
- * m->dirty is up to date courtesy of the
- * earlier check for m->reference... if
- * we get here, then m->reference had to be
- * FALSE which means we did a pmap_get_refmod
- * and updated both m->reference and m->dirty...
- */
- if (m->no_isync == FALSE) {
- refmod_state = pmap_disconnect(m->phys_page);
+ if (m_object != l_object) {
+ if (!m_object->internal) {
+ goto reenter_pg_on_q;
+ }
- if (refmod_state & VM_MEM_MODIFIED)
- m->dirty = TRUE;
- if (refmod_state & VM_MEM_REFERENCED) {
- m->reference = TRUE;
+ /*
+ * the object associated with candidate page is
+ * different from the one we were just working
+ * with... dump the lock if we still own it
+ */
+ if (l_object != NULL) {
+ vm_object_unlock(l_object);
+ l_object = NULL;
+ }
+ if (m_object != t_object) {
+ try_failed_count = 0;
+ }
- PAGE_WAKEUP_DONE(m);
- goto was_referenced;
+ /*
+ * Try to lock object; since we've alread got the
+ * page queues lock, we can only 'try' for this one.
+ * if the 'try' fails, we need to do a mutex_pause
+ * to allow the owner of the object lock a chance to
+ * run...
+ */
+ if (!vm_object_lock_try_scan(m_object)) {
+ if (try_failed_count > 20) {
+ goto reenter_pg_on_q;
+ }
+ vm_page_unlock_queues();
+ mutex_pause(try_failed_count++);
+ vm_page_lock_queues();
+ delayed_unlock = 0;
+
+ t_object = m_object;
+ continue;
}
+ l_object = m_object;
}
- /*
- * If it's clean and not precious, we can free the page.
- */
- if (!m->dirty && !m->precious) {
- vm_pageout_inactive_clean++;
- goto reclaim_page;
+ if (!m_object->alive || m->vmp_cleaning || m->vmp_laundry || m->vmp_busy || m->vmp_absent || m->vmp_error || m->vmp_free_when_done) {
+ /*
+ * page is not to be cleaned
+ * put it back on the head of its queue
+ */
+ goto reenter_pg_on_q;
}
- vm_pageout_cluster(m);
+ phys_page = VM_PAGE_GET_PHYS_PAGE(m);
- vm_pageout_inactive_dirty++;
+ if (m->vmp_reference == FALSE && m->vmp_pmapped == TRUE) {
+ refmod_state = pmap_get_refmod(phys_page);
- inactive_burst_count = 0;
+ if (refmod_state & VM_MEM_REFERENCED) {
+ m->vmp_reference = TRUE;
+ }
+ if (refmod_state & VM_MEM_MODIFIED) {
+ SET_PAGE_DIRTY(m, FALSE);
+ }
+ }
+ if (m->vmp_reference == TRUE) {
+ m->vmp_reference = FALSE;
+ pmap_clear_refmod_options(phys_page, VM_MEM_REFERENCED, PMAP_OPTIONS_NOFLUSH, (void *)NULL);
+ goto reenter_pg_on_q;
+ }
+ if (m->vmp_pmapped == TRUE) {
+ if (m->vmp_dirty || m->vmp_precious) {
+ pmap_options = PMAP_OPTIONS_COMPRESSOR;
+ } else {
+ pmap_options = PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED;
+ }
+ refmod_state = pmap_disconnect_options(phys_page, pmap_options, NULL);
+ if (refmod_state & VM_MEM_MODIFIED) {
+ SET_PAGE_DIRTY(m, FALSE);
+ }
+ }
-done_with_inactivepage:
- if (delayed_unlock++ > DELAYED_UNLOCK_LIMIT) {
+ if (!m->vmp_dirty && !m->vmp_precious) {
+ vm_page_unlock_queues();
+ VM_PAGE_FREE(m);
+ vm_page_lock_queues();
+ delayed_unlock = 0;
- if (object != NULL) {
- vm_object_unlock(object);
- object = NULL;
+ goto next_pg;
+ }
+ if (!m_object->pager_initialized || m_object->pager == MEMORY_OBJECT_NULL) {
+ if (!m_object->pager_initialized) {
+ vm_page_unlock_queues();
+
+ vm_object_collapse(m_object, (vm_object_offset_t) 0, TRUE);
+
+ if (!m_object->pager_initialized) {
+ vm_object_compressor_pager_create(m_object);
+ }
+
+ vm_page_lock_queues();
+ delayed_unlock = 0;
}
- if (local_freeq) {
- vm_page_free_list(local_freeq);
-
- local_freeq = 0;
- local_freed = 0;
+ if (!m_object->pager_initialized || m_object->pager == MEMORY_OBJECT_NULL) {
+ goto reenter_pg_on_q;
}
- delayed_unlock = 0;
- vm_page_unlock_queues();
- mutex_pause();
+ /*
+ * vm_object_compressor_pager_create will drop the object lock
+ * which means 'm' may no longer be valid to use
+ */
+ continue;
}
/*
- * back to top of pageout scan loop
+ * we've already factored out pages in the laundry which
+ * means this page can't be on the pageout queue so it's
+ * safe to do the vm_page_queues_remove
*/
- }
-}
+ vm_page_queues_remove(m, TRUE);
+ LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
-int vm_page_free_count_init;
-
-void
-vm_page_free_reserve(
- int pages)
-{
- int free_after_reserve;
+ vm_pageout_cluster(m);
- vm_page_free_reserved += pages;
+ goto next_pg;
- free_after_reserve = vm_page_free_count_init - vm_page_free_reserved;
+reenter_pg_on_q:
+ vm_page_queue_remove(q, m, vmp_pageq);
+ vm_page_queue_enter(q, m, vmp_pageq);
+next_pg:
+ qcount--;
+ try_failed_count = 0;
- vm_page_free_min = vm_page_free_reserved +
- VM_PAGE_FREE_MIN(free_after_reserve);
+ if (delayed_unlock++ > 128) {
+ if (l_object != NULL) {
+ vm_object_unlock(l_object);
+ l_object = NULL;
+ }
+ lck_mtx_yield(&vm_page_queue_lock);
+ delayed_unlock = 0;
+ }
+ }
+ if (l_object != NULL) {
+ vm_object_unlock(l_object);
+ l_object = NULL;
+ }
+ vm_page_unlock_queues();
+}
- vm_page_free_target = vm_page_free_reserved +
- VM_PAGE_FREE_TARGET(free_after_reserve);
- if (vm_page_free_target < vm_page_free_min + 5)
- vm_page_free_target = vm_page_free_min + 5;
-}
/*
- * vm_pageout is the high level pageout daemon.
+ * function in BSD to apply I/O throttle to the pageout thread
*/
+extern void vm_pageout_io_throttle(void);
+
+#define VM_PAGEOUT_SCAN_HANDLE_REUSABLE_PAGE(m, obj) \
+ MACRO_BEGIN \
+ /* \
+ * If a "reusable" page somehow made it back into \
+ * the active queue, it's been re-used and is not \
+ * quite re-usable. \
+ * If the VM object was "all_reusable", consider it \
+ * as "all re-used" instead of converting it to \
+ * "partially re-used", which could be expensive. \
+ */ \
+ assert(VM_PAGE_OBJECT((m)) == (obj)); \
+ if ((m)->vmp_reusable || \
+ (obj)->all_reusable) { \
+ vm_object_reuse_pages((obj), \
+ (m)->vmp_offset, \
+ (m)->vmp_offset + PAGE_SIZE_64, \
+ FALSE); \
+ } \
+ MACRO_END
+
+
+#define VM_PAGEOUT_DELAYED_UNLOCK_LIMIT 64
+#define VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX 1024
+
+#define FCS_IDLE 0
+#define FCS_DELAYED 1
+#define FCS_DEADLOCK_DETECTED 2
-void
-vm_pageout_continue(void)
-{
- vm_pageout_scan_event_counter++;
- vm_pageout_scan();
- /* we hold vm_page_queue_free_lock now */
- assert(vm_page_free_wanted == 0);
- assert_wait((event_t) &vm_page_free_wanted, THREAD_UNINT);
- mutex_unlock(&vm_page_queue_free_lock);
+struct flow_control {
+ int state;
+ mach_timespec_t ts;
+};
- counter(c_vm_pageout_block++);
- thread_block((thread_continue_t)vm_pageout_continue);
- /*NOTREACHED*/
-}
+#if CONFIG_BACKGROUND_QUEUE
+uint64_t vm_pageout_rejected_bq_internal = 0;
+uint64_t vm_pageout_rejected_bq_external = 0;
+uint64_t vm_pageout_skipped_bq_internal = 0;
+#endif
-/*
- * must be called with the
- * queues and object locks held
- */
-static void
-vm_pageout_queue_steal(vm_page_t m)
-{
- struct vm_pageout_queue *q;
+#define ANONS_GRABBED_LIMIT 2
- if (m->object->internal == TRUE)
- q = &vm_pageout_queue_internal;
- else
- q = &vm_pageout_queue_external;
- m->laundry = FALSE;
- m->pageout_queue = FALSE;
- queue_remove(&q->pgo_pending, m, vm_page_t, pageq);
+#if 0
+static void vm_pageout_delayed_unlock(int *, int *, vm_page_t *);
+#endif
+static void vm_pageout_prepare_to_block(vm_object_t *, int *, vm_page_t *, int *, int);
- m->pageq.next = NULL;
- m->pageq.prev = NULL;
+#define VM_PAGEOUT_PB_NO_ACTION 0
+#define VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER 1
+#define VM_PAGEOUT_PB_THREAD_YIELD 2
- vm_object_paging_end(m->object);
- q->pgo_laundry--;
-}
+#if 0
+static void
+vm_pageout_delayed_unlock(int *delayed_unlock, int *local_freed, vm_page_t *local_freeq)
+{
+ if (*local_freeq) {
+ vm_page_unlock_queues();
+ VM_DEBUG_CONSTANT_EVENT(
+ vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_START,
+ vm_page_free_count, 0, 0, 1);
-#ifdef FAKE_DEADLOCK
+ vm_page_free_list(*local_freeq, TRUE);
-#define FAKE_COUNT 5000
+ VM_DEBUG_CONSTANT_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_END,
+ vm_page_free_count, *local_freed, 0, 1);
-int internal_count = 0;
-int fake_deadlock = 0;
+ *local_freeq = NULL;
+ *local_freed = 0;
+ vm_page_lock_queues();
+ } else {
+ lck_mtx_yield(&vm_page_queue_lock);
+ }
+ *delayed_unlock = 1;
+}
#endif
+
static void
-vm_pageout_iothread_continue(struct vm_pageout_queue *q)
+vm_pageout_prepare_to_block(vm_object_t *object, int *delayed_unlock,
+ vm_page_t *local_freeq, int *local_freed, int action)
{
- vm_page_t m = NULL;
- vm_object_t object;
- boolean_t need_wakeup;
+ vm_page_unlock_queues();
+
+ if (*object != NULL) {
+ vm_object_unlock(*object);
+ *object = NULL;
+ }
+ if (*local_freeq) {
+ vm_page_free_list(*local_freeq, TRUE);
+ *local_freeq = NULL;
+ *local_freed = 0;
+ }
+ *delayed_unlock = 1;
+
+ switch (action) {
+ case VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER:
+ vm_consider_waking_compactor_swapper();
+ break;
+ case VM_PAGEOUT_PB_THREAD_YIELD:
+ thread_yield_internal(1);
+ break;
+ case VM_PAGEOUT_PB_NO_ACTION:
+ default:
+ break;
+ }
vm_page_lock_queues();
+}
- while ( !queue_empty(&q->pgo_pending) ) {
- q->pgo_busy = TRUE;
- queue_remove_first(&q->pgo_pending, m, vm_page_t, pageq);
- m->pageout_queue = FALSE;
- vm_page_unlock_queues();
+static struct vm_pageout_vminfo last;
- m->pageq.next = NULL;
- m->pageq.prev = NULL;
-#ifdef FAKE_DEADLOCK
- if (q == &vm_pageout_queue_internal) {
- vm_offset_t addr;
- int pg_count;
+uint64_t last_vm_page_pages_grabbed = 0;
- internal_count++;
+extern uint32_t c_segment_pages_compressed;
- if ((internal_count == FAKE_COUNT)) {
+extern uint64_t shared_region_pager_reclaimed;
+extern struct memory_object_pager_ops shared_region_pager_ops;
- pg_count = vm_page_free_count + vm_page_free_reserved;
+void
+update_vm_info(void)
+{
+ unsigned long tmp;
+ uint64_t tmp64;
- if (kmem_alloc(kernel_map, &addr, PAGE_SIZE * pg_count) == KERN_SUCCESS) {
- kmem_free(kernel_map, addr, PAGE_SIZE * pg_count);
- }
- internal_count = 0;
- fake_deadlock++;
- }
- }
-#endif
- object = m->object;
-
- if (!object->pager_initialized) {
- vm_object_lock(object);
-
- /*
- * If there is no memory object for the page, create
- * one and hand it to the default pager.
- */
-
- if (!object->pager_initialized)
- vm_object_collapse(object, (vm_object_offset_t)0);
- if (!object->pager_initialized)
- vm_object_pager_create(object);
- if (!object->pager_initialized) {
- /*
- * Still no pager for the object.
- * Reactivate the page.
- *
- * Should only happen if there is no
- * default pager.
- */
- m->list_req_pending = FALSE;
- m->cleaning = FALSE;
- m->pageout = FALSE;
- vm_page_unwire(m);
-
- vm_pageout_throttle_up(m);
-
- vm_page_lock_queues();
- vm_pageout_dirty_no_pager++;
- vm_page_activate(m);
- vm_page_unlock_queues();
-
- /*
- * And we are done with it.
- */
- PAGE_WAKEUP_DONE(m);
-
- vm_object_paging_end(object);
- vm_object_unlock(object);
-
- vm_page_lock_queues();
- continue;
- } else if (object->pager == MEMORY_OBJECT_NULL) {
- /*
- * This pager has been destroyed by either
- * memory_object_destroy or vm_object_destroy, and
- * so there is nowhere for the page to go.
- * Just free the page... VM_PAGE_FREE takes
- * care of cleaning up all the state...
- * including doing the vm_pageout_throttle_up
- */
- VM_PAGE_FREE(m);
-
- vm_object_paging_end(object);
- vm_object_unlock(object);
-
- vm_page_lock_queues();
- continue;
- }
- vm_object_unlock(object);
- }
- /*
- * we expect the paging_in_progress reference to have
- * already been taken on the object before it was added
- * to the appropriate pageout I/O queue... this will
- * keep the object from being terminated and/or the
- * paging_offset from changing until the I/O has
- * completed... therefore no need to lock the object to
- * pull the paging_offset from it.
- *
- * Send the data to the pager.
- * any pageout clustering happens there
- */
- memory_object_data_return(object->pager,
- m->offset + object->paging_offset,
- PAGE_SIZE,
- NULL,
- NULL,
- FALSE,
- FALSE,
- 0);
-
- vm_object_lock(object);
- vm_object_paging_end(object);
- vm_object_unlock(object);
-
- vm_page_lock_queues();
- }
- assert_wait((event_t) q, THREAD_UNINT);
-
-
- if (q->pgo_throttled == TRUE && !VM_PAGE_Q_THROTTLED(q)) {
- q->pgo_throttled = FALSE;
- need_wakeup = TRUE;
- } else
- need_wakeup = FALSE;
+ vm_pageout_stats[vm_pageout_stat_now].vm_page_active_count = vm_page_active_count;
+ vm_pageout_stats[vm_pageout_stat_now].vm_page_speculative_count = vm_page_speculative_count;
+ vm_pageout_stats[vm_pageout_stat_now].vm_page_inactive_count = vm_page_inactive_count;
+ vm_pageout_stats[vm_pageout_stat_now].vm_page_anonymous_count = vm_page_anonymous_count;
- q->pgo_busy = FALSE;
- q->pgo_idle = TRUE;
- vm_page_unlock_queues();
+ vm_pageout_stats[vm_pageout_stat_now].vm_page_free_count = vm_page_free_count;
+ vm_pageout_stats[vm_pageout_stat_now].vm_page_wire_count = vm_page_wire_count;
+ vm_pageout_stats[vm_pageout_stat_now].vm_page_compressor_count = VM_PAGE_COMPRESSOR_COUNT;
- if (need_wakeup == TRUE)
- thread_wakeup((event_t) &q->pgo_laundry);
+ vm_pageout_stats[vm_pageout_stat_now].vm_page_pages_compressed = c_segment_pages_compressed;
+ vm_pageout_stats[vm_pageout_stat_now].vm_page_pageable_internal_count = vm_page_pageable_internal_count;
+ vm_pageout_stats[vm_pageout_stat_now].vm_page_pageable_external_count = vm_page_pageable_external_count;
+ vm_pageout_stats[vm_pageout_stat_now].vm_page_xpmapped_external_count = vm_page_xpmapped_external_count;
- thread_block_parameter((thread_continue_t)vm_pageout_iothread_continue, (void *) &q->pgo_pending);
- /*NOTREACHED*/
-}
+ tmp = vm_pageout_vminfo.vm_pageout_considered_page;
+ vm_pageout_stats[vm_pageout_stat_now].considered = (unsigned int)(tmp - last.vm_pageout_considered_page);
+ last.vm_pageout_considered_page = tmp;
-static void
-vm_pageout_iothread_external(void)
-{
+ tmp64 = vm_pageout_vminfo.vm_pageout_compressions;
+ vm_pageout_stats[vm_pageout_stat_now].pages_compressed = (unsigned int)(tmp64 - last.vm_pageout_compressions);
+ last.vm_pageout_compressions = tmp64;
- vm_pageout_iothread_continue(&vm_pageout_queue_external);
- /*NOTREACHED*/
-}
+ tmp = vm_pageout_vminfo.vm_compressor_failed;
+ vm_pageout_stats[vm_pageout_stat_now].failed_compressions = (unsigned int)(tmp - last.vm_compressor_failed);
+ last.vm_compressor_failed = tmp;
+ tmp64 = vm_pageout_vminfo.vm_compressor_pages_grabbed;
+ vm_pageout_stats[vm_pageout_stat_now].pages_grabbed_by_compressor = (unsigned int)(tmp64 - last.vm_compressor_pages_grabbed);
+ last.vm_compressor_pages_grabbed = tmp64;
-static void
-vm_pageout_iothread_internal(void)
-{
- thread_t self = current_thread();
+ tmp = vm_pageout_vminfo.vm_phantom_cache_found_ghost;
+ vm_pageout_stats[vm_pageout_stat_now].phantom_ghosts_found = (unsigned int)(tmp - last.vm_phantom_cache_found_ghost);
+ last.vm_phantom_cache_found_ghost = tmp;
- self->options |= TH_OPT_VMPRIV;
+ tmp = vm_pageout_vminfo.vm_phantom_cache_added_ghost;
+ vm_pageout_stats[vm_pageout_stat_now].phantom_ghosts_added = (unsigned int)(tmp - last.vm_phantom_cache_added_ghost);
+ last.vm_phantom_cache_added_ghost = tmp;
- vm_pageout_iothread_continue(&vm_pageout_queue_internal);
- /*NOTREACHED*/
-}
+ tmp64 = get_pages_grabbed_count();
+ vm_pageout_stats[vm_pageout_stat_now].pages_grabbed = (unsigned int)(tmp64 - last_vm_page_pages_grabbed);
+ last_vm_page_pages_grabbed = tmp64;
-static void
-vm_pageout_garbage_collect(int collect)
-{
- if (collect) {
- stack_collect();
+ tmp = vm_pageout_vminfo.vm_page_pages_freed;
+ vm_pageout_stats[vm_pageout_stat_now].pages_freed = (unsigned int)(tmp - last.vm_page_pages_freed);
+ last.vm_page_pages_freed = tmp;
- /*
- * consider_zone_gc should be last, because the other operations
- * might return memory to zones.
- */
- consider_machine_collect();
- consider_zone_gc();
- consider_machine_adjust();
- }
+ if (vm_pageout_stats[vm_pageout_stat_now].considered) {
+ tmp = vm_pageout_vminfo.vm_pageout_pages_evicted;
+ vm_pageout_stats[vm_pageout_stat_now].pages_evicted = (unsigned int)(tmp - last.vm_pageout_pages_evicted);
+ last.vm_pageout_pages_evicted = tmp;
- assert_wait((event_t) &vm_pageout_garbage_collect, THREAD_UNINT);
+ tmp = vm_pageout_vminfo.vm_pageout_pages_purged;
+ vm_pageout_stats[vm_pageout_stat_now].pages_purged = (unsigned int)(tmp - last.vm_pageout_pages_purged);
+ last.vm_pageout_pages_purged = tmp;
- thread_block_parameter((thread_continue_t) vm_pageout_garbage_collect, (void *)1);
- /*NOTREACHED*/
-}
+ tmp = vm_pageout_vminfo.vm_pageout_freed_speculative;
+ vm_pageout_stats[vm_pageout_stat_now].freed_speculative = (unsigned int)(tmp - last.vm_pageout_freed_speculative);
+ last.vm_pageout_freed_speculative = tmp;
+ tmp = vm_pageout_vminfo.vm_pageout_freed_external;
+ vm_pageout_stats[vm_pageout_stat_now].freed_external = (unsigned int)(tmp - last.vm_pageout_freed_external);
+ last.vm_pageout_freed_external = tmp;
+ tmp = vm_pageout_vminfo.vm_pageout_inactive_referenced;
+ vm_pageout_stats[vm_pageout_stat_now].inactive_referenced = (unsigned int)(tmp - last.vm_pageout_inactive_referenced);
+ last.vm_pageout_inactive_referenced = tmp;
-void
-vm_pageout(void)
-{
- thread_t self = current_thread();
- thread_t thread;
- kern_return_t result;
- spl_t s;
+ tmp = vm_pageout_vminfo.vm_pageout_scan_inactive_throttled_external;
+ vm_pageout_stats[vm_pageout_stat_now].throttled_external_q = (unsigned int)(tmp - last.vm_pageout_scan_inactive_throttled_external);
+ last.vm_pageout_scan_inactive_throttled_external = tmp;
- /*
- * Set thread privileges.
- */
- s = splsched();
- thread_lock(self);
- self->priority = BASEPRI_PREEMPT - 1;
- set_sched_pri(self, self->priority);
- thread_unlock(self);
- splx(s);
+ tmp = vm_pageout_vminfo.vm_pageout_inactive_dirty_external;
+ vm_pageout_stats[vm_pageout_stat_now].cleaned_dirty_external = (unsigned int)(tmp - last.vm_pageout_inactive_dirty_external);
+ last.vm_pageout_inactive_dirty_external = tmp;
- /*
- * Initialize some paging parameters.
- */
+ tmp = vm_pageout_vminfo.vm_pageout_freed_cleaned;
+ vm_pageout_stats[vm_pageout_stat_now].freed_cleaned = (unsigned int)(tmp - last.vm_pageout_freed_cleaned);
+ last.vm_pageout_freed_cleaned = tmp;
+
+ tmp = vm_pageout_vminfo.vm_pageout_inactive_nolock;
+ vm_pageout_stats[vm_pageout_stat_now].inactive_nolock = (unsigned int)(tmp - last.vm_pageout_inactive_nolock);
+ last.vm_pageout_inactive_nolock = tmp;
+
+ tmp = vm_pageout_vminfo.vm_pageout_scan_inactive_throttled_internal;
+ vm_pageout_stats[vm_pageout_stat_now].throttled_internal_q = (unsigned int)(tmp - last.vm_pageout_scan_inactive_throttled_internal);
+ last.vm_pageout_scan_inactive_throttled_internal = tmp;
+
+ tmp = vm_pageout_vminfo.vm_pageout_skipped_external;
+ vm_pageout_stats[vm_pageout_stat_now].skipped_external = (unsigned int)(tmp - last.vm_pageout_skipped_external);
+ last.vm_pageout_skipped_external = tmp;
+
+ tmp = vm_pageout_vminfo.vm_pageout_reactivation_limit_exceeded;
+ vm_pageout_stats[vm_pageout_stat_now].reactivation_limit_exceeded = (unsigned int)(tmp - last.vm_pageout_reactivation_limit_exceeded);
+ last.vm_pageout_reactivation_limit_exceeded = tmp;
+
+ tmp = vm_pageout_vminfo.vm_pageout_inactive_force_reclaim;
+ vm_pageout_stats[vm_pageout_stat_now].forced_inactive_reclaim = (unsigned int)(tmp - last.vm_pageout_inactive_force_reclaim);
+ last.vm_pageout_inactive_force_reclaim = tmp;
+
+ tmp = vm_pageout_vminfo.vm_pageout_freed_internal;
+ vm_pageout_stats[vm_pageout_stat_now].freed_internal = (unsigned int)(tmp - last.vm_pageout_freed_internal);
+ last.vm_pageout_freed_internal = tmp;
- if (vm_pageout_idle_wait == 0)
- vm_pageout_idle_wait = VM_PAGEOUT_IDLE_WAIT;
+ tmp = vm_pageout_vminfo.vm_pageout_considered_bq_internal;
+ vm_pageout_stats[vm_pageout_stat_now].considered_bq_internal = (unsigned int)(tmp - last.vm_pageout_considered_bq_internal);
+ last.vm_pageout_considered_bq_internal = tmp;
- if (vm_pageout_burst_wait == 0)
- vm_pageout_burst_wait = VM_PAGEOUT_BURST_WAIT;
+ tmp = vm_pageout_vminfo.vm_pageout_considered_bq_external;
+ vm_pageout_stats[vm_pageout_stat_now].considered_bq_external = (unsigned int)(tmp - last.vm_pageout_considered_bq_external);
+ last.vm_pageout_considered_bq_external = tmp;
- if (vm_pageout_empty_wait == 0)
- vm_pageout_empty_wait = VM_PAGEOUT_EMPTY_WAIT;
+ tmp = vm_pageout_vminfo.vm_pageout_filecache_min_reactivated;
+ vm_pageout_stats[vm_pageout_stat_now].filecache_min_reactivations = (unsigned int)(tmp - last.vm_pageout_filecache_min_reactivated);
+ last.vm_pageout_filecache_min_reactivated = tmp;
- if (vm_pageout_deadlock_wait == 0)
- vm_pageout_deadlock_wait = VM_PAGEOUT_DEADLOCK_WAIT;
+ tmp = vm_pageout_vminfo.vm_pageout_inactive_dirty_internal;
+ vm_pageout_stats[vm_pageout_stat_now].cleaned_dirty_internal = (unsigned int)(tmp - last.vm_pageout_inactive_dirty_internal);
+ last.vm_pageout_inactive_dirty_internal = tmp;
+ }
- if (vm_pageout_deadlock_relief == 0)
- vm_pageout_deadlock_relief = VM_PAGEOUT_DEADLOCK_RELIEF;
+ KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO1)) | DBG_FUNC_NONE,
+ vm_pageout_stats[vm_pageout_stat_now].vm_page_active_count,
+ vm_pageout_stats[vm_pageout_stat_now].vm_page_speculative_count,
+ vm_pageout_stats[vm_pageout_stat_now].vm_page_inactive_count,
+ vm_pageout_stats[vm_pageout_stat_now].vm_page_anonymous_count,
+ 0);
+
+ KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO2)) | DBG_FUNC_NONE,
+ vm_pageout_stats[vm_pageout_stat_now].vm_page_free_count,
+ vm_pageout_stats[vm_pageout_stat_now].vm_page_wire_count,
+ vm_pageout_stats[vm_pageout_stat_now].vm_page_compressor_count,
+ 0,
+ 0);
+
+ KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO3)) | DBG_FUNC_NONE,
+ vm_pageout_stats[vm_pageout_stat_now].vm_page_pages_compressed,
+ vm_pageout_stats[vm_pageout_stat_now].vm_page_pageable_internal_count,
+ vm_pageout_stats[vm_pageout_stat_now].vm_page_pageable_external_count,
+ vm_pageout_stats[vm_pageout_stat_now].vm_page_xpmapped_external_count,
+ 0);
+
+ if (vm_pageout_stats[vm_pageout_stat_now].considered ||
+ vm_pageout_stats[vm_pageout_stat_now].pages_compressed ||
+ vm_pageout_stats[vm_pageout_stat_now].failed_compressions) {
+ KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO4)) | DBG_FUNC_NONE,
+ vm_pageout_stats[vm_pageout_stat_now].considered,
+ vm_pageout_stats[vm_pageout_stat_now].freed_speculative,
+ vm_pageout_stats[vm_pageout_stat_now].freed_external,
+ vm_pageout_stats[vm_pageout_stat_now].inactive_referenced,
+ 0);
+
+ KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO5)) | DBG_FUNC_NONE,
+ vm_pageout_stats[vm_pageout_stat_now].throttled_external_q,
+ vm_pageout_stats[vm_pageout_stat_now].cleaned_dirty_external,
+ vm_pageout_stats[vm_pageout_stat_now].freed_cleaned,
+ vm_pageout_stats[vm_pageout_stat_now].inactive_nolock,
+ 0);
+
+ KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO6)) | DBG_FUNC_NONE,
+ vm_pageout_stats[vm_pageout_stat_now].throttled_internal_q,
+ vm_pageout_stats[vm_pageout_stat_now].pages_compressed,
+ vm_pageout_stats[vm_pageout_stat_now].pages_grabbed_by_compressor,
+ vm_pageout_stats[vm_pageout_stat_now].skipped_external,
+ 0);
+
+ KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO7)) | DBG_FUNC_NONE,
+ vm_pageout_stats[vm_pageout_stat_now].reactivation_limit_exceeded,
+ vm_pageout_stats[vm_pageout_stat_now].forced_inactive_reclaim,
+ vm_pageout_stats[vm_pageout_stat_now].failed_compressions,
+ vm_pageout_stats[vm_pageout_stat_now].freed_internal,
+ 0);
+
+ KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO8)) | DBG_FUNC_NONE,
+ vm_pageout_stats[vm_pageout_stat_now].considered_bq_internal,
+ vm_pageout_stats[vm_pageout_stat_now].considered_bq_external,
+ vm_pageout_stats[vm_pageout_stat_now].filecache_min_reactivations,
+ vm_pageout_stats[vm_pageout_stat_now].cleaned_dirty_internal,
+ 0);
+ }
+ KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO9)) | DBG_FUNC_NONE,
+ vm_pageout_stats[vm_pageout_stat_now].pages_grabbed,
+ vm_pageout_stats[vm_pageout_stat_now].pages_freed,
+ vm_pageout_stats[vm_pageout_stat_now].phantom_ghosts_found,
+ vm_pageout_stats[vm_pageout_stat_now].phantom_ghosts_added,
+ 0);
+
+ record_memory_pressure();
+}
- if (vm_pageout_inactive_relief == 0)
- vm_pageout_inactive_relief = VM_PAGEOUT_INACTIVE_RELIEF;
+extern boolean_t hibernation_vmqueues_inspection;
- if (vm_pageout_burst_active_throttle == 0)
- vm_pageout_burst_active_throttle = VM_PAGEOUT_BURST_ACTIVE_THROTTLE;
+/*
+ * Return values for functions called by vm_pageout_scan
+ * that control its flow.
+ *
+ * PROCEED -- vm_pageout_scan will keep making forward progress.
+ * DONE_RETURN -- page demand satisfied, work is done -> vm_pageout_scan returns.
+ * NEXT_ITERATION -- restart the 'for' loop in vm_pageout_scan aka continue.
+ */
- if (vm_pageout_burst_inactive_throttle == 0)
- vm_pageout_burst_inactive_throttle = VM_PAGEOUT_BURST_INACTIVE_THROTTLE;
+#define VM_PAGEOUT_SCAN_PROCEED (0)
+#define VM_PAGEOUT_SCAN_DONE_RETURN (1)
+#define VM_PAGEOUT_SCAN_NEXT_ITERATION (2)
+/*
+ * This function is called only from vm_pageout_scan and
+ * it moves overflow secluded pages (one-at-a-time) to the
+ * batched 'local' free Q or active Q.
+ */
+static void
+vps_deal_with_secluded_page_overflow(vm_page_t *local_freeq, int *local_freed)
+{
+#if CONFIG_SECLUDED_MEMORY
/*
- * Set kernel task to low backing store privileged
- * status
+ * Deal with secluded_q overflow.
*/
- task_lock(kernel_task);
- kernel_task->priv_flags |= VM_BACKING_STORE_PRIV;
- task_unlock(kernel_task);
+ if (vm_page_secluded_count > vm_page_secluded_target) {
+ vm_page_t secluded_page;
+
+ /*
+ * SECLUDED_AGING_BEFORE_ACTIVE:
+ * Excess secluded pages go to the active queue and
+ * will later go to the inactive queue.
+ */
+ assert((vm_page_secluded_count_free +
+ vm_page_secluded_count_inuse) ==
+ vm_page_secluded_count);
+ secluded_page = (vm_page_t)vm_page_queue_first(&vm_page_queue_secluded);
+ assert(secluded_page->vmp_q_state == VM_PAGE_ON_SECLUDED_Q);
+
+ vm_page_queues_remove(secluded_page, FALSE);
+ assert(!secluded_page->vmp_fictitious);
+ assert(!VM_PAGE_WIRED(secluded_page));
+
+ if (secluded_page->vmp_object == 0) {
+ /* transfer to free queue */
+ assert(secluded_page->vmp_busy);
+ secluded_page->vmp_snext = *local_freeq;
+ *local_freeq = secluded_page;
+ *local_freed += 1;
+ } else {
+ /* transfer to head of active queue */
+ vm_page_enqueue_active(secluded_page, FALSE);
+ secluded_page = VM_PAGE_NULL;
+ }
+ }
+#else /* CONFIG_SECLUDED_MEMORY */
- vm_page_free_count_init = vm_page_free_count;
- vm_zf_iterator = 0;
+#pragma unused(local_freeq)
+#pragma unused(local_freed)
+
+ return;
+
+#endif /* CONFIG_SECLUDED_MEMORY */
+}
+
+/*
+ * This function is called only from vm_pageout_scan and
+ * it initializes the loop targets for vm_pageout_scan().
+ */
+static void
+vps_init_page_targets(void)
+{
/*
- * even if we've already called vm_page_free_reserve
- * call it again here to insure that the targets are
- * accurately calculated (it uses vm_page_free_count_init)
- * calling it with an arg of 0 will not change the reserve
- * but will re-calculate free_min and free_target
+ * LD TODO: Other page targets should be calculated here too.
*/
- if (vm_page_free_reserved < VM_PAGE_FREE_RESERVED(processor_count)) {
- vm_page_free_reserve((VM_PAGE_FREE_RESERVED(processor_count)) - vm_page_free_reserved);
- } else
- vm_page_free_reserve(0);
+ vm_page_anonymous_min = vm_page_inactive_target / 20;
+ if (vm_pageout_state.vm_page_speculative_percentage > 50) {
+ vm_pageout_state.vm_page_speculative_percentage = 50;
+ } else if (vm_pageout_state.vm_page_speculative_percentage <= 0) {
+ vm_pageout_state.vm_page_speculative_percentage = 1;
+ }
- queue_init(&vm_pageout_queue_external.pgo_pending);
- vm_pageout_queue_external.pgo_maxlaundry = VM_PAGE_LAUNDRY_MAX;
- vm_pageout_queue_external.pgo_laundry = 0;
- vm_pageout_queue_external.pgo_idle = FALSE;
- vm_pageout_queue_external.pgo_busy = FALSE;
- vm_pageout_queue_external.pgo_throttled = FALSE;
+ vm_pageout_state.vm_page_speculative_target = VM_PAGE_SPECULATIVE_TARGET(vm_page_active_count +
+ vm_page_inactive_count);
+}
- queue_init(&vm_pageout_queue_internal.pgo_pending);
- vm_pageout_queue_internal.pgo_maxlaundry = VM_PAGE_LAUNDRY_MAX;
- vm_pageout_queue_internal.pgo_laundry = 0;
- vm_pageout_queue_internal.pgo_idle = FALSE;
- vm_pageout_queue_internal.pgo_busy = FALSE;
- vm_pageout_queue_internal.pgo_throttled = FALSE;
+/*
+ * This function is called only from vm_pageout_scan and
+ * it purges a single VM object at-a-time and will either
+ * make vm_pageout_scan() restart the loop or keeping moving forward.
+ */
+static int
+vps_purge_object()
+{
+ int force_purge;
+ assert(available_for_purge >= 0);
+ force_purge = 0; /* no force-purging */
- result = kernel_thread_start_priority((thread_continue_t)vm_pageout_iothread_internal, NULL, BASEPRI_PREEMPT - 1, &thread);
- if (result != KERN_SUCCESS)
- panic("vm_pageout_iothread_internal: create failed");
+#if VM_PRESSURE_EVENTS
+ vm_pressure_level_t pressure_level;
- thread_deallocate(thread);
+ pressure_level = memorystatus_vm_pressure_level;
+ if (pressure_level > kVMPressureNormal) {
+ if (pressure_level >= kVMPressureCritical) {
+ force_purge = vm_pageout_state.memorystatus_purge_on_critical;
+ } else if (pressure_level >= kVMPressureUrgent) {
+ force_purge = vm_pageout_state.memorystatus_purge_on_urgent;
+ } else if (pressure_level >= kVMPressureWarning) {
+ force_purge = vm_pageout_state.memorystatus_purge_on_warning;
+ }
+ }
+#endif /* VM_PRESSURE_EVENTS */
- result = kernel_thread_start_priority((thread_continue_t)vm_pageout_iothread_external, NULL, BASEPRI_PREEMPT - 1, &thread);
- if (result != KERN_SUCCESS)
- panic("vm_pageout_iothread_external: create failed");
+ if (available_for_purge || force_purge) {
+ memoryshot(VM_PAGEOUT_PURGEONE, DBG_FUNC_START);
- thread_deallocate(thread);
+ VM_DEBUG_EVENT(vm_pageout_purgeone, VM_PAGEOUT_PURGEONE, DBG_FUNC_START, vm_page_free_count, 0, 0, 0);
+ if (vm_purgeable_object_purge_one(force_purge, C_DONT_BLOCK)) {
+ VM_PAGEOUT_DEBUG(vm_pageout_purged_objects, 1);
+ VM_DEBUG_EVENT(vm_pageout_purgeone, VM_PAGEOUT_PURGEONE, DBG_FUNC_END, vm_page_free_count, 0, 0, 0);
+ memoryshot(VM_PAGEOUT_PURGEONE, DBG_FUNC_END);
+ return VM_PAGEOUT_SCAN_NEXT_ITERATION;
+ }
+ VM_DEBUG_EVENT(vm_pageout_purgeone, VM_PAGEOUT_PURGEONE, DBG_FUNC_END, 0, 0, 0, -1);
+ memoryshot(VM_PAGEOUT_PURGEONE, DBG_FUNC_END);
+ }
- result = kernel_thread_start_priority((thread_continue_t)vm_pageout_garbage_collect, NULL, BASEPRI_PREEMPT - 2, &thread);
- if (result != KERN_SUCCESS)
- panic("vm_pageout_garbage_collect: create failed");
+ return VM_PAGEOUT_SCAN_PROCEED;
+}
- thread_deallocate(thread);
+/*
+ * This function is called only from vm_pageout_scan and
+ * it will try to age the next speculative Q if the oldest
+ * one is empty.
+ */
+static int
+vps_age_speculative_queue(boolean_t force_speculative_aging)
+{
+#define DELAY_SPECULATIVE_AGE 1000
+ /*
+ * try to pull pages from the aging bins...
+ * see vm_page.h for an explanation of how
+ * this mechanism works
+ */
+ boolean_t can_steal = FALSE;
+ int num_scanned_queues;
+ static int delay_speculative_age = 0; /* depends the # of times we go through the main pageout_scan loop.*/
+ mach_timespec_t ts;
+ struct vm_speculative_age_q *aq;
+ struct vm_speculative_age_q *sq;
- vm_pageout_continue();
- /*NOTREACHED*/
-}
+ sq = &vm_page_queue_speculative[VM_PAGE_SPECULATIVE_AGED_Q];
+ aq = &vm_page_queue_speculative[speculative_steal_index];
-static upl_t
-upl_create(
- int flags,
- upl_size_t size)
-{
- upl_t upl;
- int page_field_size; /* bit field in word size buf */
+ num_scanned_queues = 0;
+ while (vm_page_queue_empty(&aq->age_q) &&
+ num_scanned_queues++ != VM_PAGE_MAX_SPECULATIVE_AGE_Q) {
+ speculative_steal_index++;
- page_field_size = 0;
- if (flags & UPL_CREATE_LITE) {
- page_field_size = ((size/PAGE_SIZE) + 7) >> 3;
- page_field_size = (page_field_size + 3) & 0xFFFFFFFC;
+ if (speculative_steal_index > VM_PAGE_MAX_SPECULATIVE_AGE_Q) {
+ speculative_steal_index = VM_PAGE_MIN_SPECULATIVE_AGE_Q;
+ }
+
+ aq = &vm_page_queue_speculative[speculative_steal_index];
}
- if(flags & UPL_CREATE_INTERNAL) {
- upl = (upl_t)kalloc(sizeof(struct upl)
- + (sizeof(struct upl_page_info)*(size/PAGE_SIZE))
- + page_field_size);
- } else {
- upl = (upl_t)kalloc(sizeof(struct upl) + page_field_size);
+
+ if (num_scanned_queues == VM_PAGE_MAX_SPECULATIVE_AGE_Q + 1) {
+ /*
+ * XXX We've scanned all the speculative
+ * queues but still haven't found one
+ * that is not empty, even though
+ * vm_page_speculative_count is not 0.
+ */
+ if (!vm_page_queue_empty(&sq->age_q)) {
+ return VM_PAGEOUT_SCAN_NEXT_ITERATION;
+ }
+#if DEVELOPMENT || DEBUG
+ panic("vm_pageout_scan: vm_page_speculative_count=%d but queues are empty", vm_page_speculative_count);
+#endif
+ /* readjust... */
+ vm_page_speculative_count = 0;
+ /* ... and continue */
+ return VM_PAGEOUT_SCAN_NEXT_ITERATION;
}
- upl->flags = 0;
- upl->src_object = NULL;
- upl->kaddr = (vm_offset_t)0;
- upl->size = 0;
- upl->map_object = NULL;
- upl->ref_count = 1;
- upl_lock_init(upl);
-#ifdef UPL_DEBUG
- upl->ubc_alias1 = 0;
- upl->ubc_alias2 = 0;
-#endif /* UPL_DEBUG */
- return(upl);
-}
-static void
-upl_destroy(
- upl_t upl)
-{
- int page_field_size; /* bit field in word size buf */
+ if (vm_page_speculative_count > vm_pageout_state.vm_page_speculative_target || force_speculative_aging == TRUE) {
+ can_steal = TRUE;
+ } else {
+ if (!delay_speculative_age) {
+ mach_timespec_t ts_fully_aged;
-#ifdef UPL_DEBUG
- {
- upl_t upl_ele;
- vm_object_t object;
- if (upl->map_object->pageout) {
- object = upl->map_object->shadow;
+ ts_fully_aged.tv_sec = (VM_PAGE_MAX_SPECULATIVE_AGE_Q * vm_pageout_state.vm_page_speculative_q_age_ms) / 1000;
+ ts_fully_aged.tv_nsec = ((VM_PAGE_MAX_SPECULATIVE_AGE_Q * vm_pageout_state.vm_page_speculative_q_age_ms) % 1000)
+ * 1000 * NSEC_PER_USEC;
+
+ ADD_MACH_TIMESPEC(&ts_fully_aged, &aq->age_ts);
+
+ clock_sec_t sec;
+ clock_nsec_t nsec;
+ clock_get_system_nanotime(&sec, &nsec);
+ ts.tv_sec = (unsigned int) sec;
+ ts.tv_nsec = nsec;
+
+ if (CMP_MACH_TIMESPEC(&ts, &ts_fully_aged) >= 0) {
+ can_steal = TRUE;
+ } else {
+ delay_speculative_age++;
+ }
} else {
- object = upl->map_object;
- }
- vm_object_lock(object);
- queue_iterate(&object->uplq, upl_ele, upl_t, uplq) {
- if(upl_ele == upl) {
- queue_remove(&object->uplq,
- upl_ele, upl_t, uplq);
- break;
+ delay_speculative_age++;
+ if (delay_speculative_age == DELAY_SPECULATIVE_AGE) {
+ delay_speculative_age = 0;
}
}
- vm_object_unlock(object);
}
-#endif /* UPL_DEBUG */
- /* drop a reference on the map_object whether or */
- /* not a pageout object is inserted */
- if(upl->map_object->pageout)
- vm_object_deallocate(upl->map_object);
+ if (can_steal == TRUE) {
+ vm_page_speculate_ageit(aq);
+ }
+
+ return VM_PAGEOUT_SCAN_PROCEED;
+}
+
+/*
+ * This function is called only from vm_pageout_scan and
+ * it evicts a single VM object from the cache.
+ */
+static int inline
+vps_object_cache_evict(vm_object_t *object_to_unlock)
+{
+ static int cache_evict_throttle = 0;
+ struct vm_speculative_age_q *sq;
+
+ sq = &vm_page_queue_speculative[VM_PAGE_SPECULATIVE_AGED_Q];
+
+ if (vm_page_queue_empty(&sq->age_q) && cache_evict_throttle == 0) {
+ int pages_evicted;
+
+ if (*object_to_unlock != NULL) {
+ vm_object_unlock(*object_to_unlock);
+ *object_to_unlock = NULL;
+ }
+ KERNEL_DEBUG_CONSTANT(0x13001ec | DBG_FUNC_START, 0, 0, 0, 0, 0);
+
+ pages_evicted = vm_object_cache_evict(100, 10);
+
+ KERNEL_DEBUG_CONSTANT(0x13001ec | DBG_FUNC_END, pages_evicted, 0, 0, 0, 0);
+
+ if (pages_evicted) {
+ vm_pageout_vminfo.vm_pageout_pages_evicted += pages_evicted;
+
+ VM_DEBUG_EVENT(vm_pageout_cache_evict, VM_PAGEOUT_CACHE_EVICT, DBG_FUNC_NONE,
+ vm_page_free_count, pages_evicted, vm_pageout_vminfo.vm_pageout_pages_evicted, 0);
+ memoryshot(VM_PAGEOUT_CACHE_EVICT, DBG_FUNC_NONE);
+
+ /*
+ * we just freed up to 100 pages,
+ * so go back to the top of the main loop
+ * and re-evaulate the memory situation
+ */
+ return VM_PAGEOUT_SCAN_NEXT_ITERATION;
+ } else {
+ cache_evict_throttle = 1000;
+ }
+ }
+ if (cache_evict_throttle) {
+ cache_evict_throttle--;
+ }
+
+ return VM_PAGEOUT_SCAN_PROCEED;
+}
+
+
+/*
+ * This function is called only from vm_pageout_scan and
+ * it calculates the filecache min. that needs to be maintained
+ * as we start to steal pages.
+ */
+static void
+vps_calculate_filecache_min(void)
+{
+ int divisor = vm_pageout_state.vm_page_filecache_min_divisor;
+
+#if CONFIG_JETSAM
+ /*
+ * don't let the filecache_min fall below 15% of available memory
+ * on systems with an active compressor that isn't nearing its
+ * limits w/r to accepting new data
+ *
+ * on systems w/o the compressor/swapper, the filecache is always
+ * a very large percentage of the AVAILABLE_NON_COMPRESSED_MEMORY
+ * since most (if not all) of the anonymous pages are in the
+ * throttled queue (which isn't counted as available) which
+ * effectively disables this filter
+ */
+ if (vm_compressor_low_on_space() || divisor == 0) {
+ vm_pageout_state.vm_page_filecache_min = 0;
+ } else {
+ vm_pageout_state.vm_page_filecache_min =
+ ((AVAILABLE_NON_COMPRESSED_MEMORY) * 10) / divisor;
+ }
+#else
+ if (vm_compressor_out_of_space() || divisor == 0) {
+ vm_pageout_state.vm_page_filecache_min = 0;
+ } else {
+ /*
+ * don't let the filecache_min fall below the specified critical level
+ */
+ vm_pageout_state.vm_page_filecache_min =
+ ((AVAILABLE_NON_COMPRESSED_MEMORY) * 10) / divisor;
+ }
+#endif
+ if (vm_page_free_count < (vm_page_free_reserved / 4)) {
+ vm_pageout_state.vm_page_filecache_min = 0;
+ }
+}
+
+/*
+ * This function is called only from vm_pageout_scan and
+ * it updates the flow control time to detect if VM pageoutscan
+ * isn't making progress.
+ */
+static void
+vps_flow_control_reset_deadlock_timer(struct flow_control *flow_control)
+{
+ mach_timespec_t ts;
+ clock_sec_t sec;
+ clock_nsec_t nsec;
+
+ ts.tv_sec = vm_pageout_state.vm_pageout_deadlock_wait / 1000;
+ ts.tv_nsec = (vm_pageout_state.vm_pageout_deadlock_wait % 1000) * 1000 * NSEC_PER_USEC;
+ clock_get_system_nanotime(&sec, &nsec);
+ flow_control->ts.tv_sec = (unsigned int) sec;
+ flow_control->ts.tv_nsec = nsec;
+ ADD_MACH_TIMESPEC(&flow_control->ts, &ts);
+
+ flow_control->state = FCS_DELAYED;
+
+ vm_pageout_vminfo.vm_pageout_scan_inactive_throttled_internal++;
+}
+
+/*
+ * This function is called only from vm_pageout_scan and
+ * it is the flow control logic of VM pageout scan which
+ * controls if it should block and for how long.
+ * Any blocking of vm_pageout_scan happens ONLY in this function.
+ */
+static int
+vps_flow_control(struct flow_control *flow_control, int *anons_grabbed, vm_object_t *object, int *delayed_unlock,
+ vm_page_t *local_freeq, int *local_freed, int *vm_pageout_deadlock_target, unsigned int inactive_burst_count)
+{
+ boolean_t exceeded_burst_throttle = FALSE;
+ unsigned int msecs = 0;
+ uint32_t inactive_external_count;
+ mach_timespec_t ts;
+ struct vm_pageout_queue *iq;
+ struct vm_pageout_queue *eq;
+ struct vm_speculative_age_q *sq;
+
+ iq = &vm_pageout_queue_internal;
+ eq = &vm_pageout_queue_external;
+ sq = &vm_page_queue_speculative[VM_PAGE_SPECULATIVE_AGED_Q];
+
+ /*
+ * Sometimes we have to pause:
+ * 1) No inactive pages - nothing to do.
+ * 2) Loop control - no acceptable pages found on the inactive queue
+ * within the last vm_pageout_burst_inactive_throttle iterations
+ * 3) Flow control - default pageout queue is full
+ */
+ if (vm_page_queue_empty(&vm_page_queue_inactive) &&
+ vm_page_queue_empty(&vm_page_queue_anonymous) &&
+ vm_page_queue_empty(&vm_page_queue_cleaned) &&
+ vm_page_queue_empty(&sq->age_q)) {
+ VM_PAGEOUT_DEBUG(vm_pageout_scan_empty_throttle, 1);
+ msecs = vm_pageout_state.vm_pageout_empty_wait;
+ } else if (inactive_burst_count >=
+ MIN(vm_pageout_state.vm_pageout_burst_inactive_throttle,
+ (vm_page_inactive_count +
+ vm_page_speculative_count))) {
+ VM_PAGEOUT_DEBUG(vm_pageout_scan_burst_throttle, 1);
+ msecs = vm_pageout_state.vm_pageout_burst_wait;
+
+ exceeded_burst_throttle = TRUE;
+ } else if (VM_PAGE_Q_THROTTLED(iq) &&
+ VM_DYNAMIC_PAGING_ENABLED()) {
+ clock_sec_t sec;
+ clock_nsec_t nsec;
+
+ switch (flow_control->state) {
+ case FCS_IDLE:
+ if ((vm_page_free_count + *local_freed) < vm_page_free_target &&
+ vm_pageout_state.vm_restricted_to_single_processor == FALSE) {
+ /*
+ * since the compressor is running independently of vm_pageout_scan
+ * let's not wait for it just yet... as long as we have a healthy supply
+ * of filecache pages to work with, let's keep stealing those.
+ */
+ inactive_external_count = vm_page_inactive_count - vm_page_anonymous_count;
+
+ if (vm_page_pageable_external_count > vm_pageout_state.vm_page_filecache_min &&
+ (inactive_external_count >= VM_PAGE_INACTIVE_TARGET(vm_page_pageable_external_count))) {
+ *anons_grabbed = ANONS_GRABBED_LIMIT;
+ VM_PAGEOUT_DEBUG(vm_pageout_scan_throttle_deferred, 1);
+ return VM_PAGEOUT_SCAN_PROCEED;
+ }
+ }
+
+ vps_flow_control_reset_deadlock_timer(flow_control);
+ msecs = vm_pageout_state.vm_pageout_deadlock_wait;
+
+ break;
+
+ case FCS_DELAYED:
+ clock_get_system_nanotime(&sec, &nsec);
+ ts.tv_sec = (unsigned int) sec;
+ ts.tv_nsec = nsec;
+
+ if (CMP_MACH_TIMESPEC(&ts, &flow_control->ts) >= 0) {
+ /*
+ * the pageout thread for the default pager is potentially
+ * deadlocked since the
+ * default pager queue has been throttled for more than the
+ * allowable time... we need to move some clean pages or dirty
+ * pages belonging to the external pagers if they aren't throttled
+ * vm_page_free_wanted represents the number of threads currently
+ * blocked waiting for pages... we'll move one page for each of
+ * these plus a fixed amount to break the logjam... once we're done
+ * moving this number of pages, we'll re-enter the FSC_DELAYED state
+ * with a new timeout target since we have no way of knowing
+ * whether we've broken the deadlock except through observation
+ * of the queue associated with the default pager... we need to
+ * stop moving pages and allow the system to run to see what
+ * state it settles into.
+ */
+
+ *vm_pageout_deadlock_target = vm_pageout_state.vm_pageout_deadlock_relief +
+ vm_page_free_wanted + vm_page_free_wanted_privileged;
+ VM_PAGEOUT_DEBUG(vm_pageout_scan_deadlock_detected, 1);
+ flow_control->state = FCS_DEADLOCK_DETECTED;
+ thread_wakeup((event_t) &vm_pageout_garbage_collect);
+ return VM_PAGEOUT_SCAN_PROCEED;
+ }
+ /*
+ * just resniff instead of trying
+ * to compute a new delay time... we're going to be
+ * awakened immediately upon a laundry completion,
+ * so we won't wait any longer than necessary
+ */
+ msecs = vm_pageout_state.vm_pageout_idle_wait;
+ break;
+
+ case FCS_DEADLOCK_DETECTED:
+ if (*vm_pageout_deadlock_target) {
+ return VM_PAGEOUT_SCAN_PROCEED;
+ }
+
+ vps_flow_control_reset_deadlock_timer(flow_control);
+ msecs = vm_pageout_state.vm_pageout_deadlock_wait;
+
+ break;
+ }
+ } else {
+ /*
+ * No need to pause...
+ */
+ return VM_PAGEOUT_SCAN_PROCEED;
+ }
+
+ vm_pageout_scan_wants_object = VM_OBJECT_NULL;
+
+ vm_pageout_prepare_to_block(object, delayed_unlock, local_freeq, local_freed,
+ VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER);
+
+ if (vm_page_free_count >= vm_page_free_target) {
+ /*
+ * we're here because
+ * 1) someone else freed up some pages while we had
+ * the queues unlocked above
+ * and we've hit one of the 3 conditions that
+ * cause us to pause the pageout scan thread
+ *
+ * since we already have enough free pages,
+ * let's avoid stalling and return normally
+ *
+ * before we return, make sure the pageout I/O threads
+ * are running throttled in case there are still requests
+ * in the laundry... since we have enough free pages
+ * we don't need the laundry to be cleaned in a timely
+ * fashion... so let's avoid interfering with foreground
+ * activity
+ *
+ * we don't want to hold vm_page_queue_free_lock when
+ * calling vm_pageout_adjust_eq_iothrottle (since it
+ * may cause other locks to be taken), we do the intitial
+ * check outside of the lock. Once we take the lock,
+ * we recheck the condition since it may have changed.
+ * if it has, no problem, we will make the threads
+ * non-throttled before actually blocking
+ */
+ vm_pageout_adjust_eq_iothrottle(eq, TRUE);
+ }
+ lck_mtx_lock(&vm_page_queue_free_lock);
+
+ if (vm_page_free_count >= vm_page_free_target &&
+ (vm_page_free_wanted == 0) && (vm_page_free_wanted_privileged == 0)) {
+ return VM_PAGEOUT_SCAN_DONE_RETURN;
+ }
+ lck_mtx_unlock(&vm_page_queue_free_lock);
+
+ if ((vm_page_free_count + vm_page_cleaned_count) < vm_page_free_target) {
+ /*
+ * we're most likely about to block due to one of
+ * the 3 conditions that cause vm_pageout_scan to
+ * not be able to make forward progress w/r
+ * to providing new pages to the free queue,
+ * so unthrottle the I/O threads in case we
+ * have laundry to be cleaned... it needs
+ * to be completed ASAP.
+ *
+ * even if we don't block, we want the io threads
+ * running unthrottled since the sum of free +
+ * clean pages is still under our free target
+ */
+ vm_pageout_adjust_eq_iothrottle(eq, FALSE);
+ }
+ if (vm_page_cleaned_count > 0 && exceeded_burst_throttle == FALSE) {
+ /*
+ * if we get here we're below our free target and
+ * we're stalling due to a full laundry queue or
+ * we don't have any inactive pages other then
+ * those in the clean queue...
+ * however, we have pages on the clean queue that
+ * can be moved to the free queue, so let's not
+ * stall the pageout scan
+ */
+ flow_control->state = FCS_IDLE;
+ return VM_PAGEOUT_SCAN_PROCEED;
+ }
+ if (flow_control->state == FCS_DELAYED && !VM_PAGE_Q_THROTTLED(iq)) {
+ flow_control->state = FCS_IDLE;
+ return VM_PAGEOUT_SCAN_PROCEED;
+ }
+
+ VM_CHECK_MEMORYSTATUS;
+
+ if (flow_control->state != FCS_IDLE) {
+ VM_PAGEOUT_DEBUG(vm_pageout_scan_throttle, 1);
+ }
+
+ iq->pgo_throttled = TRUE;
+ assert_wait_timeout((event_t) &iq->pgo_laundry, THREAD_INTERRUPTIBLE, msecs, 1000 * NSEC_PER_USEC);
+
+ counter(c_vm_pageout_scan_block++);
+
+ vm_page_unlock_queues();
+
+ assert(vm_pageout_scan_wants_object == VM_OBJECT_NULL);
+
+ VM_DEBUG_EVENT(vm_pageout_thread_block, VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_START,
+ iq->pgo_laundry, iq->pgo_maxlaundry, msecs, 0);
+ memoryshot(VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_START);
+
+ thread_block(THREAD_CONTINUE_NULL);
+
+ VM_DEBUG_EVENT(vm_pageout_thread_block, VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_END,
+ iq->pgo_laundry, iq->pgo_maxlaundry, msecs, 0);
+ memoryshot(VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_END);
+
+ vm_page_lock_queues();
+
+ iq->pgo_throttled = FALSE;
+
+ vps_init_page_targets();
+
+ return VM_PAGEOUT_SCAN_NEXT_ITERATION;
+}
+
+/*
+ * This function is called only from vm_pageout_scan and
+ * it will find and return the most appropriate page to be
+ * reclaimed.
+ */
+static int
+vps_choose_victim_page(vm_page_t *victim_page, int *anons_grabbed, boolean_t *grab_anonymous, boolean_t force_anonymous,
+ boolean_t *is_page_from_bg_q, unsigned int *reactivated_this_call)
+{
+ vm_page_t m = NULL;
+ vm_object_t m_object = VM_OBJECT_NULL;
+ uint32_t inactive_external_count;
+ struct vm_speculative_age_q *sq;
+ struct vm_pageout_queue *iq;
+ int retval = VM_PAGEOUT_SCAN_PROCEED;
+
+ sq = &vm_page_queue_speculative[VM_PAGE_SPECULATIVE_AGED_Q];
+ iq = &vm_pageout_queue_internal;
+
+ *is_page_from_bg_q = FALSE;
+
+ m = NULL;
+ m_object = VM_OBJECT_NULL;
+
+ if (VM_DYNAMIC_PAGING_ENABLED()) {
+ assert(vm_page_throttled_count == 0);
+ assert(vm_page_queue_empty(&vm_page_queue_throttled));
+ }
+
+ /*
+ * Try for a clean-queue inactive page.
+ * These are pages that vm_pageout_scan tried to steal earlier, but
+ * were dirty and had to be cleaned. Pick them up now that they are clean.
+ */
+ if (!vm_page_queue_empty(&vm_page_queue_cleaned)) {
+ m = (vm_page_t) vm_page_queue_first(&vm_page_queue_cleaned);
+
+ assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q);
+
+ goto found_page;
+ }
+
+ /*
+ * The next most eligible pages are ones we paged in speculatively,
+ * but which have not yet been touched and have been aged out.
+ */
+ if (!vm_page_queue_empty(&sq->age_q)) {
+ m = (vm_page_t) vm_page_queue_first(&sq->age_q);
+
+ assert(m->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q);
+
+ if (!m->vmp_dirty || force_anonymous == FALSE) {
+ goto found_page;
+ } else {
+ m = NULL;
+ }
+ }
+
+#if CONFIG_BACKGROUND_QUEUE
+ if (vm_page_background_mode != VM_PAGE_BG_DISABLED && (vm_page_background_count > vm_page_background_target)) {
+ vm_object_t bg_m_object = NULL;
+
+ m = (vm_page_t) vm_page_queue_first(&vm_page_queue_background);
+
+ bg_m_object = VM_PAGE_OBJECT(m);
+
+ if (!VM_PAGE_PAGEABLE(m)) {
+ /*
+ * This page is on the background queue
+ * but not on a pageable queue. This is
+ * likely a transient state and whoever
+ * took it out of its pageable queue
+ * will likely put it back on a pageable
+ * queue soon but we can't deal with it
+ * at this point, so let's ignore this
+ * page.
+ */
+ } else if (force_anonymous == FALSE || bg_m_object->internal) {
+ if (bg_m_object->internal &&
+ (VM_PAGE_Q_THROTTLED(iq) ||
+ vm_compressor_out_of_space() == TRUE ||
+ vm_page_free_count < (vm_page_free_reserved / 4))) {
+ vm_pageout_skipped_bq_internal++;
+ } else {
+ *is_page_from_bg_q = TRUE;
+
+ if (bg_m_object->internal) {
+ vm_pageout_vminfo.vm_pageout_considered_bq_internal++;
+ } else {
+ vm_pageout_vminfo.vm_pageout_considered_bq_external++;
+ }
+ goto found_page;
+ }
+ }
+ }
+#endif /* CONFIG_BACKGROUND_QUEUE */
+
+ inactive_external_count = vm_page_inactive_count - vm_page_anonymous_count;
+
+ if ((vm_page_pageable_external_count < vm_pageout_state.vm_page_filecache_min || force_anonymous == TRUE) ||
+ (inactive_external_count < VM_PAGE_INACTIVE_TARGET(vm_page_pageable_external_count))) {
+ *grab_anonymous = TRUE;
+ *anons_grabbed = 0;
+
+ vm_pageout_vminfo.vm_pageout_skipped_external++;
+ goto want_anonymous;
+ }
+ *grab_anonymous = (vm_page_anonymous_count > vm_page_anonymous_min);
+
+#if CONFIG_JETSAM
+ /* If the file-backed pool has accumulated
+ * significantly more pages than the jetsam
+ * threshold, prefer to reclaim those
+ * inline to minimise compute overhead of reclaiming
+ * anonymous pages.
+ * This calculation does not account for the CPU local
+ * external page queues, as those are expected to be
+ * much smaller relative to the global pools.
+ */
+
+ struct vm_pageout_queue *eq = &vm_pageout_queue_external;
+
+ if (*grab_anonymous == TRUE && !VM_PAGE_Q_THROTTLED(eq)) {
+ if (vm_page_pageable_external_count >
+ vm_pageout_state.vm_page_filecache_min) {
+ if ((vm_page_pageable_external_count *
+ vm_pageout_memorystatus_fb_factor_dr) >
+ (memorystatus_available_pages_critical *
+ vm_pageout_memorystatus_fb_factor_nr)) {
+ *grab_anonymous = FALSE;
+
+ VM_PAGEOUT_DEBUG(vm_grab_anon_overrides, 1);
+ }
+ }
+ if (*grab_anonymous) {
+ VM_PAGEOUT_DEBUG(vm_grab_anon_nops, 1);
+ }
+ }
+#endif /* CONFIG_JETSAM */
+
+want_anonymous:
+ if (*grab_anonymous == FALSE || *anons_grabbed >= ANONS_GRABBED_LIMIT || vm_page_queue_empty(&vm_page_queue_anonymous)) {
+ if (!vm_page_queue_empty(&vm_page_queue_inactive)) {
+ m = (vm_page_t) vm_page_queue_first(&vm_page_queue_inactive);
+
+ assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_EXTERNAL_Q);
+ *anons_grabbed = 0;
+
+ if (vm_page_pageable_external_count < vm_pageout_state.vm_page_filecache_min) {
+ if (!vm_page_queue_empty(&vm_page_queue_anonymous)) {
+ if ((++(*reactivated_this_call) % 100)) {
+ vm_pageout_vminfo.vm_pageout_filecache_min_reactivated++;
+
+ vm_page_activate(m);
+ VM_STAT_INCR(reactivations);
+#if CONFIG_BACKGROUND_QUEUE
+#if DEVELOPMENT || DEBUG
+ if (*is_page_from_bg_q == TRUE) {
+ if (m_object->internal) {
+ vm_pageout_rejected_bq_internal++;
+ } else {
+ vm_pageout_rejected_bq_external++;
+ }
+ }
+#endif /* DEVELOPMENT || DEBUG */
+#endif /* CONFIG_BACKGROUND_QUEUE */
+ vm_pageout_state.vm_pageout_inactive_used++;
+
+ m = NULL;
+ retval = VM_PAGEOUT_SCAN_NEXT_ITERATION;
+
+ goto found_page;
+ }
+
+ /*
+ * steal 1 of the file backed pages even if
+ * we are under the limit that has been set
+ * for a healthy filecache
+ */
+ }
+ }
+ goto found_page;
+ }
+ }
+ if (!vm_page_queue_empty(&vm_page_queue_anonymous)) {
+ m = (vm_page_t) vm_page_queue_first(&vm_page_queue_anonymous);
+
+ assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q);
+ *anons_grabbed += 1;
+
+ goto found_page;
+ }
+
+ m = NULL;
+
+found_page:
+ *victim_page = m;
+
+ return retval;
+}
+
+/*
+ * This function is called only from vm_pageout_scan and
+ * it will put a page back on the active/inactive queue
+ * if we can't reclaim it for some reason.
+ */
+static void
+vps_requeue_page(vm_page_t m, int page_prev_q_state, __unused boolean_t page_from_bg_q)
+{
+ if (page_prev_q_state == VM_PAGE_ON_SPECULATIVE_Q) {
+ vm_page_enqueue_inactive(m, FALSE);
+ } else {
+ vm_page_activate(m);
+ }
+
+#if CONFIG_BACKGROUND_QUEUE
+#if DEVELOPMENT || DEBUG
+ vm_object_t m_object = VM_PAGE_OBJECT(m);
+
+ if (page_from_bg_q == TRUE) {
+ if (m_object->internal) {
+ vm_pageout_rejected_bq_internal++;
+ } else {
+ vm_pageout_rejected_bq_external++;
+ }
+ }
+#endif /* DEVELOPMENT || DEBUG */
+#endif /* CONFIG_BACKGROUND_QUEUE */
+}
+
+/*
+ * This function is called only from vm_pageout_scan and
+ * it will try to grab the victim page's VM object (m_object)
+ * which differs from the previous victim page's object (object).
+ */
+static int
+vps_switch_object(vm_page_t m, vm_object_t m_object, vm_object_t *object, int page_prev_q_state, boolean_t avoid_anon_pages, boolean_t page_from_bg_q)
+{
+ struct vm_speculative_age_q *sq;
+
+ sq = &vm_page_queue_speculative[VM_PAGE_SPECULATIVE_AGED_Q];
+
+ /*
+ * the object associated with candidate page is
+ * different from the one we were just working
+ * with... dump the lock if we still own it
+ */
+ if (*object != NULL) {
+ vm_object_unlock(*object);
+ *object = NULL;
+ }
+ /*
+ * Try to lock object; since we've alread got the
+ * page queues lock, we can only 'try' for this one.
+ * if the 'try' fails, we need to do a mutex_pause
+ * to allow the owner of the object lock a chance to
+ * run... otherwise, we're likely to trip over this
+ * object in the same state as we work our way through
+ * the queue... clumps of pages associated with the same
+ * object are fairly typical on the inactive and active queues
+ */
+ if (!vm_object_lock_try_scan(m_object)) {
+ vm_page_t m_want = NULL;
+
+ vm_pageout_vminfo.vm_pageout_inactive_nolock++;
+
+ if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) {
+ VM_PAGEOUT_DEBUG(vm_pageout_cleaned_nolock, 1);
+ }
+
+ pmap_clear_reference(VM_PAGE_GET_PHYS_PAGE(m));
+
+ m->vmp_reference = FALSE;
+
+ if (!m_object->object_is_shared_cache) {
+ /*
+ * don't apply this optimization if this is the shared cache
+ * object, it's too easy to get rid of very hot and important
+ * pages...
+ * m->vmp_object must be stable since we hold the page queues lock...
+ * we can update the scan_collisions field sans the object lock
+ * since it is a separate field and this is the only spot that does
+ * a read-modify-write operation and it is never executed concurrently...
+ * we can asynchronously set this field to 0 when creating a UPL, so it
+ * is possible for the value to be a bit non-determistic, but that's ok
+ * since it's only used as a hint
+ */
+ m_object->scan_collisions = 1;
+ }
+ if (!vm_page_queue_empty(&vm_page_queue_cleaned)) {
+ m_want = (vm_page_t) vm_page_queue_first(&vm_page_queue_cleaned);
+ } else if (!vm_page_queue_empty(&sq->age_q)) {
+ m_want = (vm_page_t) vm_page_queue_first(&sq->age_q);
+ } else if ((avoid_anon_pages || vm_page_queue_empty(&vm_page_queue_anonymous)) &&
+ !vm_page_queue_empty(&vm_page_queue_inactive)) {
+ m_want = (vm_page_t) vm_page_queue_first(&vm_page_queue_inactive);
+ } else if (!vm_page_queue_empty(&vm_page_queue_anonymous)) {
+ m_want = (vm_page_t) vm_page_queue_first(&vm_page_queue_anonymous);
+ }
+
+ /*
+ * this is the next object we're going to be interested in
+ * try to make sure its available after the mutex_pause
+ * returns control
+ */
+ if (m_want) {
+ vm_pageout_scan_wants_object = VM_PAGE_OBJECT(m_want);
+ }
+
+ vps_requeue_page(m, page_prev_q_state, page_from_bg_q);
+
+ return VM_PAGEOUT_SCAN_NEXT_ITERATION;
+ } else {
+ *object = m_object;
+ vm_pageout_scan_wants_object = VM_OBJECT_NULL;
+ }
+
+ return VM_PAGEOUT_SCAN_PROCEED;
+}
+
+/*
+ * This function is called only from vm_pageout_scan and
+ * it notices that pageout scan may be rendered ineffective
+ * due to a FS deadlock and will jetsam a process if possible.
+ * If jetsam isn't supported, it'll move the page to the active
+ * queue to try and get some different pages pushed onwards so
+ * we can try to get out of this scenario.
+ */
+static void
+vps_deal_with_throttled_queues(vm_page_t m, vm_object_t *object, uint32_t *vm_pageout_inactive_external_forced_reactivate_limit,
+ int *delayed_unlock, boolean_t *force_anonymous, __unused boolean_t is_page_from_bg_q)
+{
+ struct vm_pageout_queue *eq;
+ vm_object_t cur_object = VM_OBJECT_NULL;
+
+ cur_object = *object;
+
+ eq = &vm_pageout_queue_external;
+
+ if (cur_object->internal == FALSE) {
+ /*
+ * we need to break up the following potential deadlock case...
+ * a) The external pageout thread is stuck on the truncate lock for a file that is being extended i.e. written.
+ * b) The thread doing the writing is waiting for pages while holding the truncate lock
+ * c) Most of the pages in the inactive queue belong to this file.
+ *
+ * we are potentially in this deadlock because...
+ * a) the external pageout queue is throttled
+ * b) we're done with the active queue and moved on to the inactive queue
+ * c) we've got a dirty external page
+ *
+ * since we don't know the reason for the external pageout queue being throttled we
+ * must suspect that we are deadlocked, so move the current page onto the active queue
+ * in an effort to cause a page from the active queue to 'age' to the inactive queue
+ *
+ * if we don't have jetsam configured (i.e. we have a dynamic pager), set
+ * 'force_anonymous' to TRUE to cause us to grab a page from the cleaned/anonymous
+ * pool the next time we select a victim page... if we can make enough new free pages,
+ * the deadlock will break, the external pageout queue will empty and it will no longer
+ * be throttled
+ *
+ * if we have jetsam configured, keep a count of the pages reactivated this way so
+ * that we can try to find clean pages in the active/inactive queues before
+ * deciding to jetsam a process
+ */
+ vm_pageout_vminfo.vm_pageout_scan_inactive_throttled_external++;
+
+ vm_page_check_pageable_safe(m);
+ assert(m->vmp_q_state == VM_PAGE_NOT_ON_Q);
+ vm_page_queue_enter(&vm_page_queue_active, m, vmp_pageq);
+ m->vmp_q_state = VM_PAGE_ON_ACTIVE_Q;
+ vm_page_active_count++;
+ vm_page_pageable_external_count++;
+
+ vm_pageout_adjust_eq_iothrottle(eq, FALSE);
+
+#if CONFIG_MEMORYSTATUS && CONFIG_JETSAM
+
+#pragma unused(force_anonymous)
+
+ *vm_pageout_inactive_external_forced_reactivate_limit -= 1;
+
+ if (*vm_pageout_inactive_external_forced_reactivate_limit <= 0) {
+ *vm_pageout_inactive_external_forced_reactivate_limit = vm_page_active_count + vm_page_inactive_count;
+ /*
+ * Possible deadlock scenario so request jetsam action
+ */
+
+ assert(cur_object);
+ vm_object_unlock(cur_object);
+
+ cur_object = VM_OBJECT_NULL;
+
+ /*
+ * VM pageout scan needs to know we have dropped this lock and so set the
+ * object variable we got passed in to NULL.
+ */
+ *object = VM_OBJECT_NULL;
+
+ vm_page_unlock_queues();
+
+ VM_DEBUG_CONSTANT_EVENT(vm_pageout_jetsam, VM_PAGEOUT_JETSAM, DBG_FUNC_START,
+ vm_page_active_count, vm_page_inactive_count, vm_page_free_count, vm_page_free_count);
+
+ /* Kill first suitable process. If this call returned FALSE, we might have simply purged a process instead. */
+ if (memorystatus_kill_on_VM_page_shortage(FALSE) == TRUE) {
+ VM_PAGEOUT_DEBUG(vm_pageout_inactive_external_forced_jetsam_count, 1);
+ }
+
+ VM_DEBUG_CONSTANT_EVENT(vm_pageout_jetsam, VM_PAGEOUT_JETSAM, DBG_FUNC_END,
+ vm_page_active_count, vm_page_inactive_count, vm_page_free_count, vm_page_free_count);
+
+ vm_page_lock_queues();
+ *delayed_unlock = 1;
+ }
+#else /* CONFIG_MEMORYSTATUS && CONFIG_JETSAM */
+
+#pragma unused(vm_pageout_inactive_external_forced_reactivate_limit)
+#pragma unused(delayed_unlock)
+
+ *force_anonymous = TRUE;
+#endif /* CONFIG_MEMORYSTATUS && CONFIG_JETSAM */
+ } else {
+ vm_page_activate(m);
+ VM_STAT_INCR(reactivations);
+
+#if CONFIG_BACKGROUND_QUEUE
+#if DEVELOPMENT || DEBUG
+ if (is_page_from_bg_q == TRUE) {
+ if (cur_object->internal) {
+ vm_pageout_rejected_bq_internal++;
+ } else {
+ vm_pageout_rejected_bq_external++;
+ }
+ }
+#endif /* DEVELOPMENT || DEBUG */
+#endif /* CONFIG_BACKGROUND_QUEUE */
+
+ vm_pageout_state.vm_pageout_inactive_used++;
+ }
+}
+
+
+void
+vm_page_balance_inactive(int max_to_move)
+{
+ vm_page_t m;
+
+ LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
+
+ if (hibernation_vmqueues_inspection || hibernate_cleaning_in_progress) {
+ /*
+ * It is likely that the hibernation code path is
+ * dealing with these very queues as we are about
+ * to move pages around in/from them and completely
+ * change the linkage of the pages.
+ *
+ * And so we skip the rebalancing of these queues.
+ */
+ return;
+ }
+ vm_page_inactive_target = VM_PAGE_INACTIVE_TARGET(vm_page_active_count +
+ vm_page_inactive_count +
+ vm_page_speculative_count);
+
+ while (max_to_move-- && (vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_target) {
+ VM_PAGEOUT_DEBUG(vm_pageout_balanced, 1);
+
+ m = (vm_page_t) vm_page_queue_first(&vm_page_queue_active);
+
+ assert(m->vmp_q_state == VM_PAGE_ON_ACTIVE_Q);
+ assert(!m->vmp_laundry);
+ assert(VM_PAGE_OBJECT(m) != kernel_object);
+ assert(VM_PAGE_GET_PHYS_PAGE(m) != vm_page_guard_addr);
+
+ DTRACE_VM2(scan, int, 1, (uint64_t *), NULL);
+
+ /*
+ * by not passing in a pmap_flush_context we will forgo any TLB flushing, local or otherwise...
+ *
+ * a TLB flush isn't really needed here since at worst we'll miss the reference bit being
+ * updated in the PTE if a remote processor still has this mapping cached in its TLB when the
+ * new reference happens. If no futher references happen on the page after that remote TLB flushes
+ * we'll see a clean, non-referenced page when it eventually gets pulled out of the inactive queue
+ * by pageout_scan, which is just fine since the last reference would have happened quite far
+ * in the past (TLB caches don't hang around for very long), and of course could just as easily
+ * have happened before we moved the page
+ */
+ if (m->vmp_pmapped == TRUE) {
+ pmap_clear_refmod_options(VM_PAGE_GET_PHYS_PAGE(m), VM_MEM_REFERENCED, PMAP_OPTIONS_NOFLUSH, (void *)NULL);
+ }
+
+ /*
+ * The page might be absent or busy,
+ * but vm_page_deactivate can handle that.
+ * FALSE indicates that we don't want a H/W clear reference
+ */
+ vm_page_deactivate_internal(m, FALSE);
+ }
+}
+
+
+/*
+ * vm_pageout_scan does the dirty work for the pageout daemon.
+ * It returns with both vm_page_queue_free_lock and vm_page_queue_lock
+ * held and vm_page_free_wanted == 0.
+ */
+void
+vm_pageout_scan(void)
+{
+ unsigned int loop_count = 0;
+ unsigned int inactive_burst_count = 0;
+ unsigned int reactivated_this_call;
+ unsigned int reactivate_limit;
+ vm_page_t local_freeq = NULL;
+ int local_freed = 0;
+ int delayed_unlock;
+ int delayed_unlock_limit = 0;
+ int refmod_state = 0;
+ int vm_pageout_deadlock_target = 0;
+ struct vm_pageout_queue *iq;
+ struct vm_pageout_queue *eq;
+ struct vm_speculative_age_q *sq;
+ struct flow_control flow_control = { .state = 0, .ts = { .tv_sec = 0, .tv_nsec = 0 } };
+ boolean_t inactive_throttled = FALSE;
+ vm_object_t object = NULL;
+ uint32_t inactive_reclaim_run;
+ boolean_t grab_anonymous = FALSE;
+ boolean_t force_anonymous = FALSE;
+ boolean_t force_speculative_aging = FALSE;
+ int anons_grabbed = 0;
+ int page_prev_q_state = 0;
+ boolean_t page_from_bg_q = FALSE;
+ uint32_t vm_pageout_inactive_external_forced_reactivate_limit = 0;
+ vm_object_t m_object = VM_OBJECT_NULL;
+ int retval = 0;
+ boolean_t lock_yield_check = FALSE;
+
+
+ VM_DEBUG_CONSTANT_EVENT(vm_pageout_scan, VM_PAGEOUT_SCAN, DBG_FUNC_START,
+ vm_pageout_vminfo.vm_pageout_freed_speculative,
+ vm_pageout_state.vm_pageout_inactive_clean,
+ vm_pageout_vminfo.vm_pageout_inactive_dirty_internal,
+ vm_pageout_vminfo.vm_pageout_inactive_dirty_external);
+
+ flow_control.state = FCS_IDLE;
+ iq = &vm_pageout_queue_internal;
+ eq = &vm_pageout_queue_external;
+ sq = &vm_page_queue_speculative[VM_PAGE_SPECULATIVE_AGED_Q];
+
+ /* Ask the pmap layer to return any pages it no longer needs. */
+ uint64_t pmap_wired_pages_freed = pmap_release_pages_fast();
+
+ vm_page_lock_queues();
+
+ vm_page_wire_count -= pmap_wired_pages_freed;
+
+ delayed_unlock = 1;
+
+ /*
+ * Calculate the max number of referenced pages on the inactive
+ * queue that we will reactivate.
+ */
+ reactivated_this_call = 0;
+ reactivate_limit = VM_PAGE_REACTIVATE_LIMIT(vm_page_active_count +
+ vm_page_inactive_count);
+ inactive_reclaim_run = 0;
+
+ vm_pageout_inactive_external_forced_reactivate_limit = vm_page_active_count + vm_page_inactive_count;
+
+ /*
+ * We must limit the rate at which we send pages to the pagers
+ * so that we don't tie up too many pages in the I/O queues.
+ * We implement a throttling mechanism using the laundry count
+ * to limit the number of pages outstanding to the default
+ * and external pagers. We can bypass the throttles and look
+ * for clean pages if the pageout queues don't drain in a timely
+ * fashion since this may indicate that the pageout paths are
+ * stalled waiting for memory, which only we can provide.
+ */
+
+ vps_init_page_targets();
+ assert(object == NULL);
+ assert(delayed_unlock != 0);
+
+ for (;;) {
+ vm_page_t m;
+
+ DTRACE_VM2(rev, int, 1, (uint64_t *), NULL);
+
+ if (lock_yield_check) {
+ lock_yield_check = FALSE;
+
+ if (delayed_unlock++ > delayed_unlock_limit) {
+ int freed = local_freed;
+
+ vm_pageout_prepare_to_block(&object, &delayed_unlock, &local_freeq, &local_freed,
+ VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER);
+ if (freed == 0) {
+ lck_mtx_yield(&vm_page_queue_lock);
+ }
+ } else if (vm_pageout_scan_wants_object) {
+ vm_page_unlock_queues();
+ mutex_pause(0);
+ vm_page_lock_queues();
+ }
+ }
+
+ if (vm_upl_wait_for_pages < 0) {
+ vm_upl_wait_for_pages = 0;
+ }
+
+ delayed_unlock_limit = VM_PAGEOUT_DELAYED_UNLOCK_LIMIT + vm_upl_wait_for_pages;
+
+ if (delayed_unlock_limit > VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX) {
+ delayed_unlock_limit = VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX;
+ }
+
+ vps_deal_with_secluded_page_overflow(&local_freeq, &local_freed);
+
+ assert(delayed_unlock);
+
+ /*
+ * maintain our balance
+ */
+ vm_page_balance_inactive(1);
+
+
+ /**********************************************************************
+ * above this point we're playing with the active and secluded queues
+ * below this point we're playing with the throttling mechanisms
+ * and the inactive queue
+ **********************************************************************/
+
+ if (vm_page_free_count + local_freed >= vm_page_free_target) {
+ vm_pageout_scan_wants_object = VM_OBJECT_NULL;
+
+ vm_pageout_prepare_to_block(&object, &delayed_unlock, &local_freeq, &local_freed,
+ VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER);
+ /*
+ * make sure the pageout I/O threads are running
+ * throttled in case there are still requests
+ * in the laundry... since we have met our targets
+ * we don't need the laundry to be cleaned in a timely
+ * fashion... so let's avoid interfering with foreground
+ * activity
+ */
+ vm_pageout_adjust_eq_iothrottle(eq, TRUE);
+
+ lck_mtx_lock(&vm_page_queue_free_lock);
+
+ if ((vm_page_free_count >= vm_page_free_target) &&
+ (vm_page_free_wanted == 0) && (vm_page_free_wanted_privileged == 0)) {
+ /*
+ * done - we have met our target *and*
+ * there is no one waiting for a page.
+ */
+return_from_scan:
+ assert(vm_pageout_scan_wants_object == VM_OBJECT_NULL);
+
+ VM_DEBUG_CONSTANT_EVENT(vm_pageout_scan, VM_PAGEOUT_SCAN, DBG_FUNC_NONE,
+ vm_pageout_state.vm_pageout_inactive,
+ vm_pageout_state.vm_pageout_inactive_used, 0, 0);
+ VM_DEBUG_CONSTANT_EVENT(vm_pageout_scan, VM_PAGEOUT_SCAN, DBG_FUNC_END,
+ vm_pageout_vminfo.vm_pageout_freed_speculative,
+ vm_pageout_state.vm_pageout_inactive_clean,
+ vm_pageout_vminfo.vm_pageout_inactive_dirty_internal,
+ vm_pageout_vminfo.vm_pageout_inactive_dirty_external);
+
+ return;
+ }
+ lck_mtx_unlock(&vm_page_queue_free_lock);
+ }
+
+ /*
+ * Before anything, we check if we have any ripe volatile
+ * objects around. If so, try to purge the first object.
+ * If the purge fails, fall through to reclaim a page instead.
+ * If the purge succeeds, go back to the top and reevalute
+ * the new memory situation.
+ */
+ retval = vps_purge_object();
+
+ if (retval == VM_PAGEOUT_SCAN_NEXT_ITERATION) {
+ /*
+ * Success
+ */
+ if (object != NULL) {
+ vm_object_unlock(object);
+ object = NULL;
+ }
+
+ lock_yield_check = FALSE;
+ continue;
+ }
+
+ /*
+ * If our 'aged' queue is empty and we have some speculative pages
+ * in the other queues, let's go through and see if we need to age
+ * them.
+ *
+ * If we succeeded in aging a speculative Q or just that everything
+ * looks normal w.r.t queue age and queue counts, we keep going onward.
+ *
+ * If, for some reason, we seem to have a mismatch between the spec.
+ * page count and the page queues, we reset those variables and
+ * restart the loop (LD TODO: Track this better?).
+ */
+ if (vm_page_queue_empty(&sq->age_q) && vm_page_speculative_count) {
+ retval = vps_age_speculative_queue(force_speculative_aging);
+
+ if (retval == VM_PAGEOUT_SCAN_NEXT_ITERATION) {
+ lock_yield_check = FALSE;
+ continue;
+ }
+ }
+ force_speculative_aging = FALSE;
+
+ /*
+ * Check to see if we need to evict objects from the cache.
+ *
+ * Note: 'object' here doesn't have anything to do with
+ * the eviction part. We just need to make sure we have dropped
+ * any object lock we might be holding if we need to go down
+ * into the eviction logic.
+ */
+ retval = vps_object_cache_evict(&object);
+
+ if (retval == VM_PAGEOUT_SCAN_NEXT_ITERATION) {
+ lock_yield_check = FALSE;
+ continue;
+ }
+
+
+ /*
+ * Calculate our filecache_min that will affect the loop
+ * going forward.
+ */
+ vps_calculate_filecache_min();
+
+ /*
+ * LD TODO: Use a structure to hold all state variables for a single
+ * vm_pageout_scan iteration and pass that structure to this function instead.
+ */
+ retval = vps_flow_control(&flow_control, &anons_grabbed, &object,
+ &delayed_unlock, &local_freeq, &local_freed,
+ &vm_pageout_deadlock_target, inactive_burst_count);
+
+ if (retval == VM_PAGEOUT_SCAN_NEXT_ITERATION) {
+ if (loop_count >= vm_page_inactive_count) {
+ loop_count = 0;
+ }
+
+ inactive_burst_count = 0;
+
+ assert(object == NULL);
+ assert(delayed_unlock != 0);
+
+ lock_yield_check = FALSE;
+ continue;
+ } else if (retval == VM_PAGEOUT_SCAN_DONE_RETURN) {
+ goto return_from_scan;
+ }
+
+ flow_control.state = FCS_IDLE;
+
+ vm_pageout_inactive_external_forced_reactivate_limit = MIN((vm_page_active_count + vm_page_inactive_count),
+ vm_pageout_inactive_external_forced_reactivate_limit);
+ loop_count++;
+ inactive_burst_count++;
+ vm_pageout_state.vm_pageout_inactive++;
+
+ /*
+ * Choose a victim.
+ */
+
+ m = NULL;
+ retval = vps_choose_victim_page(&m, &anons_grabbed, &grab_anonymous, force_anonymous, &page_from_bg_q, &reactivated_this_call);
+
+ if (m == NULL) {
+ if (retval == VM_PAGEOUT_SCAN_NEXT_ITERATION) {
+ inactive_burst_count = 0;
+
+ if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) {
+ VM_PAGEOUT_DEBUG(vm_pageout_cleaned_reactivated, 1);
+ }
+
+ lock_yield_check = TRUE;
+ continue;
+ }
+
+ /*
+ * if we've gotten here, we have no victim page.
+ * check to see if we've not finished balancing the queues
+ * or we have a page on the aged speculative queue that we
+ * skipped due to force_anonymous == TRUE.. or we have
+ * speculative pages that we can prematurely age... if
+ * one of these cases we'll keep going, else panic
+ */
+ force_anonymous = FALSE;
+ VM_PAGEOUT_DEBUG(vm_pageout_no_victim, 1);
+
+ if (!vm_page_queue_empty(&sq->age_q)) {
+ lock_yield_check = TRUE;
+ continue;
+ }
+
+ if (vm_page_speculative_count) {
+ force_speculative_aging = TRUE;
+ lock_yield_check = TRUE;
+ continue;
+ }
+ panic("vm_pageout: no victim");
+
+ /* NOTREACHED */
+ }
+
+ assert(VM_PAGE_PAGEABLE(m));
+ m_object = VM_PAGE_OBJECT(m);
+ force_anonymous = FALSE;
+
+ page_prev_q_state = m->vmp_q_state;
+ /*
+ * we just found this page on one of our queues...
+ * it can't also be on the pageout queue, so safe
+ * to call vm_page_queues_remove
+ */
+ vm_page_queues_remove(m, TRUE);
+
+ assert(!m->vmp_laundry);
+ assert(!m->vmp_private);
+ assert(!m->vmp_fictitious);
+ assert(m_object != kernel_object);
+ assert(VM_PAGE_GET_PHYS_PAGE(m) != vm_page_guard_addr);
+
+ vm_pageout_vminfo.vm_pageout_considered_page++;
+
+ DTRACE_VM2(scan, int, 1, (uint64_t *), NULL);
+
+ /*
+ * check to see if we currently are working
+ * with the same object... if so, we've
+ * already got the lock
+ */
+ if (m_object != object) {
+ boolean_t avoid_anon_pages = (grab_anonymous == FALSE || anons_grabbed >= ANONS_GRABBED_LIMIT);
+
+ /*
+ * vps_switch_object() will always drop the 'object' lock first
+ * and then try to acquire the 'm_object' lock. So 'object' has to point to
+ * either 'm_object' or NULL.
+ */
+ retval = vps_switch_object(m, m_object, &object, page_prev_q_state, avoid_anon_pages, page_from_bg_q);
+
+ if (retval == VM_PAGEOUT_SCAN_NEXT_ITERATION) {
+ lock_yield_check = TRUE;
+ continue;
+ }
+ }
+ assert(m_object == object);
+ assert(VM_PAGE_OBJECT(m) == m_object);
+
+ if (m->vmp_busy) {
+ /*
+ * Somebody is already playing with this page.
+ * Put it back on the appropriate queue
+ *
+ */
+ VM_PAGEOUT_DEBUG(vm_pageout_inactive_busy, 1);
+
+ if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) {
+ VM_PAGEOUT_DEBUG(vm_pageout_cleaned_busy, 1);
+ }
+
+ vps_requeue_page(m, page_prev_q_state, page_from_bg_q);
+
+ lock_yield_check = TRUE;
+ continue;
+ }
+
+ /*
+ * if (m->vmp_cleaning && !m->vmp_free_when_done)
+ * If already cleaning this page in place
+ * just leave if off the paging queues.
+ * We can leave the page mapped, and upl_commit_range
+ * will put it on the clean queue.
+ *
+ * if (m->vmp_free_when_done && !m->vmp_cleaning)
+ * an msync INVALIDATE is in progress...
+ * this page has been marked for destruction
+ * after it has been cleaned,
+ * but not yet gathered into a UPL
+ * where 'cleaning' will be set...
+ * just leave it off the paging queues
+ *
+ * if (m->vmp_free_when_done && m->vmp_clenaing)
+ * an msync INVALIDATE is in progress
+ * and the UPL has already gathered this page...
+ * just leave it off the paging queues
+ */
+ if (m->vmp_free_when_done || m->vmp_cleaning) {
+ lock_yield_check = TRUE;
+ continue;
+ }
+
+
+ /*
+ * If it's absent, in error or the object is no longer alive,
+ * we can reclaim the page... in the no longer alive case,
+ * there are 2 states the page can be in that preclude us
+ * from reclaiming it - busy or cleaning - that we've already
+ * dealt with
+ */
+ if (m->vmp_absent || m->vmp_error || !object->alive) {
+ if (m->vmp_absent) {
+ VM_PAGEOUT_DEBUG(vm_pageout_inactive_absent, 1);
+ } else if (!object->alive) {
+ VM_PAGEOUT_DEBUG(vm_pageout_inactive_notalive, 1);
+ } else {
+ VM_PAGEOUT_DEBUG(vm_pageout_inactive_error, 1);
+ }
+reclaim_page:
+ if (vm_pageout_deadlock_target) {
+ VM_PAGEOUT_DEBUG(vm_pageout_scan_inactive_throttle_success, 1);
+ vm_pageout_deadlock_target--;
+ }
+
+ DTRACE_VM2(dfree, int, 1, (uint64_t *), NULL);
+
+ if (object->internal) {
+ DTRACE_VM2(anonfree, int, 1, (uint64_t *), NULL);
+ } else {
+ DTRACE_VM2(fsfree, int, 1, (uint64_t *), NULL);
+ }
+ assert(!m->vmp_cleaning);
+ assert(!m->vmp_laundry);
+
+ if (!object->internal &&
+ object->pager != NULL &&
+ object->pager->mo_pager_ops == &shared_region_pager_ops) {
+ shared_region_pager_reclaimed++;
+ }
+
+ m->vmp_busy = TRUE;
+
+ /*
+ * remove page from object here since we're already
+ * behind the object lock... defer the rest of the work
+ * we'd normally do in vm_page_free_prepare_object
+ * until 'vm_page_free_list' is called
+ */
+ if (m->vmp_tabled) {
+ vm_page_remove(m, TRUE);
+ }
+
+ assert(m->vmp_pageq.next == 0 && m->vmp_pageq.prev == 0);
+ m->vmp_snext = local_freeq;
+ local_freeq = m;
+ local_freed++;
+
+ if (page_prev_q_state == VM_PAGE_ON_SPECULATIVE_Q) {
+ vm_pageout_vminfo.vm_pageout_freed_speculative++;
+ } else if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) {
+ vm_pageout_vminfo.vm_pageout_freed_cleaned++;
+ } else if (page_prev_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q) {
+ vm_pageout_vminfo.vm_pageout_freed_internal++;
+ } else {
+ vm_pageout_vminfo.vm_pageout_freed_external++;
+ }
+
+ inactive_burst_count = 0;
+
+ lock_yield_check = TRUE;
+ continue;
+ }
+ if (object->copy == VM_OBJECT_NULL) {
+ /*
+ * No one else can have any interest in this page.
+ * If this is an empty purgable object, the page can be
+ * reclaimed even if dirty.
+ * If the page belongs to a volatile purgable object, we
+ * reactivate it if the compressor isn't active.
+ */
+ if (object->purgable == VM_PURGABLE_EMPTY) {
+ if (m->vmp_pmapped == TRUE) {
+ /* unmap the page */
+ refmod_state = pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
+ if (refmod_state & VM_MEM_MODIFIED) {
+ SET_PAGE_DIRTY(m, FALSE);
+ }
+ }
+ if (m->vmp_dirty || m->vmp_precious) {
+ /* we saved the cost of cleaning this page ! */
+ vm_page_purged_count++;
+ }
+ goto reclaim_page;
+ }
+
+ if (VM_CONFIG_COMPRESSOR_IS_ACTIVE) {
+ /*
+ * With the VM compressor, the cost of
+ * reclaiming a page is much lower (no I/O),
+ * so if we find a "volatile" page, it's better
+ * to let it get compressed rather than letting
+ * it occupy a full page until it gets purged.
+ * So no need to check for "volatile" here.
+ */
+ } else if (object->purgable == VM_PURGABLE_VOLATILE) {
+ /*
+ * Avoid cleaning a "volatile" page which might
+ * be purged soon.
+ */
+
+ /* if it's wired, we can't put it on our queue */
+ assert(!VM_PAGE_WIRED(m));
+
+ /* just stick it back on! */
+ reactivated_this_call++;
+
+ if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) {
+ VM_PAGEOUT_DEBUG(vm_pageout_cleaned_volatile_reactivated, 1);
+ }
+
+ goto reactivate_page;
+ }
+ }
+ /*
+ * If it's being used, reactivate.
+ * (Fictitious pages are either busy or absent.)
+ * First, update the reference and dirty bits
+ * to make sure the page is unreferenced.
+ */
+ refmod_state = -1;
+
+ if (m->vmp_reference == FALSE && m->vmp_pmapped == TRUE) {
+ refmod_state = pmap_get_refmod(VM_PAGE_GET_PHYS_PAGE(m));
+
+ if (refmod_state & VM_MEM_REFERENCED) {
+ m->vmp_reference = TRUE;
+ }
+ if (refmod_state & VM_MEM_MODIFIED) {
+ SET_PAGE_DIRTY(m, FALSE);
+ }
+ }
+
+ if (m->vmp_reference || m->vmp_dirty) {
+ /* deal with a rogue "reusable" page */
+ VM_PAGEOUT_SCAN_HANDLE_REUSABLE_PAGE(m, m_object);
+ }
+
+ if (vm_pageout_state.vm_page_xpmapped_min_divisor == 0) {
+ vm_pageout_state.vm_page_xpmapped_min = 0;
+ } else {
+ vm_pageout_state.vm_page_xpmapped_min = (vm_page_external_count * 10) / vm_pageout_state.vm_page_xpmapped_min_divisor;
+ }
+
+ if (!m->vmp_no_cache &&
+ page_from_bg_q == FALSE &&
+ (m->vmp_reference || (m->vmp_xpmapped && !object->internal &&
+ (vm_page_xpmapped_external_count < vm_pageout_state.vm_page_xpmapped_min)))) {
+ /*
+ * The page we pulled off the inactive list has
+ * been referenced. It is possible for other
+ * processors to be touching pages faster than we
+ * can clear the referenced bit and traverse the
+ * inactive queue, so we limit the number of
+ * reactivations.
+ */
+ if (++reactivated_this_call >= reactivate_limit) {
+ vm_pageout_vminfo.vm_pageout_reactivation_limit_exceeded++;
+ } else if (++inactive_reclaim_run >= VM_PAGEOUT_INACTIVE_FORCE_RECLAIM) {
+ vm_pageout_vminfo.vm_pageout_inactive_force_reclaim++;
+ } else {
+ uint32_t isinuse;
+
+ if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) {
+ VM_PAGEOUT_DEBUG(vm_pageout_cleaned_reference_reactivated, 1);
+ }
+
+ vm_pageout_vminfo.vm_pageout_inactive_referenced++;
+reactivate_page:
+ if (!object->internal && object->pager != MEMORY_OBJECT_NULL &&
+ vnode_pager_get_isinuse(object->pager, &isinuse) == KERN_SUCCESS && !isinuse) {
+ /*
+ * no explict mappings of this object exist
+ * and it's not open via the filesystem
+ */
+ vm_page_deactivate(m);
+ VM_PAGEOUT_DEBUG(vm_pageout_inactive_deactivated, 1);
+ } else {
+ /*
+ * The page was/is being used, so put back on active list.
+ */
+ vm_page_activate(m);
+ VM_STAT_INCR(reactivations);
+ inactive_burst_count = 0;
+ }
+#if CONFIG_BACKGROUND_QUEUE
+#if DEVELOPMENT || DEBUG
+ if (page_from_bg_q == TRUE) {
+ if (m_object->internal) {
+ vm_pageout_rejected_bq_internal++;
+ } else {
+ vm_pageout_rejected_bq_external++;
+ }
+ }
+#endif /* DEVELOPMENT || DEBUG */
+#endif /* CONFIG_BACKGROUND_QUEUE */
+
+ if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) {
+ VM_PAGEOUT_DEBUG(vm_pageout_cleaned_reactivated, 1);
+ }
+ vm_pageout_state.vm_pageout_inactive_used++;
+
+ lock_yield_check = TRUE;
+ continue;
+ }
+ /*
+ * Make sure we call pmap_get_refmod() if it
+ * wasn't already called just above, to update
+ * the dirty bit.
+ */
+ if ((refmod_state == -1) && !m->vmp_dirty && m->vmp_pmapped) {
+ refmod_state = pmap_get_refmod(VM_PAGE_GET_PHYS_PAGE(m));
+ if (refmod_state & VM_MEM_MODIFIED) {
+ SET_PAGE_DIRTY(m, FALSE);
+ }
+ }
+ }
+
+ /*
+ * we've got a candidate page to steal...
+ *
+ * m->vmp_dirty is up to date courtesy of the
+ * preceding check for m->vmp_reference... if
+ * we get here, then m->vmp_reference had to be
+ * FALSE (or possibly "reactivate_limit" was
+ * exceeded), but in either case we called
+ * pmap_get_refmod() and updated both
+ * m->vmp_reference and m->vmp_dirty
+ *
+ * if it's dirty or precious we need to
+ * see if the target queue is throtttled
+ * it if is, we need to skip over it by moving it back
+ * to the end of the inactive queue
+ */
+
+ inactive_throttled = FALSE;
+
+ if (m->vmp_dirty || m->vmp_precious) {
+ if (object->internal) {
+ if (VM_PAGE_Q_THROTTLED(iq)) {
+ inactive_throttled = TRUE;
+ }
+ } else if (VM_PAGE_Q_THROTTLED(eq)) {
+ inactive_throttled = TRUE;
+ }
+ }
+throttle_inactive:
+ if (!VM_DYNAMIC_PAGING_ENABLED() &&
+ object->internal && m->vmp_dirty &&
+ (object->purgable == VM_PURGABLE_DENY ||
+ object->purgable == VM_PURGABLE_NONVOLATILE ||
+ object->purgable == VM_PURGABLE_VOLATILE)) {
+ vm_page_check_pageable_safe(m);
+ assert(m->vmp_q_state == VM_PAGE_NOT_ON_Q);
+ vm_page_queue_enter(&vm_page_queue_throttled, m, vmp_pageq);
+ m->vmp_q_state = VM_PAGE_ON_THROTTLED_Q;
+ vm_page_throttled_count++;
+
+ VM_PAGEOUT_DEBUG(vm_pageout_scan_reclaimed_throttled, 1);
+
+ inactive_burst_count = 0;
+
+ lock_yield_check = TRUE;
+ continue;
+ }
+ if (inactive_throttled == TRUE) {
+ vps_deal_with_throttled_queues(m, &object, &vm_pageout_inactive_external_forced_reactivate_limit,
+ &delayed_unlock, &force_anonymous, page_from_bg_q);
+
+ inactive_burst_count = 0;
+
+ if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) {
+ VM_PAGEOUT_DEBUG(vm_pageout_cleaned_reactivated, 1);
+ }
+
+ lock_yield_check = TRUE;
+ continue;
+ }
+
+ /*
+ * we've got a page that we can steal...
+ * eliminate all mappings and make sure
+ * we have the up-to-date modified state
+ *
+ * if we need to do a pmap_disconnect then we
+ * need to re-evaluate m->vmp_dirty since the pmap_disconnect
+ * provides the true state atomically... the
+ * page was still mapped up to the pmap_disconnect
+ * and may have been dirtied at the last microsecond
+ *
+ * Note that if 'pmapped' is FALSE then the page is not
+ * and has not been in any map, so there is no point calling
+ * pmap_disconnect(). m->vmp_dirty could have been set in anticipation
+ * of likely usage of the page.
+ */
+ if (m->vmp_pmapped == TRUE) {
+ int pmap_options;
+
+ /*
+ * Don't count this page as going into the compressor
+ * if any of these are true:
+ * 1) compressed pager isn't enabled
+ * 2) Freezer enabled device with compressed pager
+ * backend (exclusive use) i.e. most of the VM system
+ * (including vm_pageout_scan) has no knowledge of
+ * the compressor
+ * 3) This page belongs to a file and hence will not be
+ * sent into the compressor
+ */
+ if (!VM_CONFIG_COMPRESSOR_IS_ACTIVE ||
+ object->internal == FALSE) {
+ pmap_options = 0;
+ } else if (m->vmp_dirty || m->vmp_precious) {
+ /*
+ * VM knows that this page is dirty (or
+ * precious) and needs to be compressed
+ * rather than freed.
+ * Tell the pmap layer to count this page
+ * as "compressed".
+ */
+ pmap_options = PMAP_OPTIONS_COMPRESSOR;
+ } else {
+ /*
+ * VM does not know if the page needs to
+ * be preserved but the pmap layer might tell
+ * us if any mapping has "modified" it.
+ * Let's the pmap layer to count this page
+ * as compressed if and only if it has been
+ * modified.
+ */
+ pmap_options =
+ PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED;
+ }
+ refmod_state = pmap_disconnect_options(VM_PAGE_GET_PHYS_PAGE(m),
+ pmap_options,
+ NULL);
+ if (refmod_state & VM_MEM_MODIFIED) {
+ SET_PAGE_DIRTY(m, FALSE);
+ }
+ }
+
+ /*
+ * reset our count of pages that have been reclaimed
+ * since the last page was 'stolen'
+ */
+ inactive_reclaim_run = 0;
+
+ /*
+ * If it's clean and not precious, we can free the page.
+ */
+ if (!m->vmp_dirty && !m->vmp_precious) {
+ vm_pageout_state.vm_pageout_inactive_clean++;
+
+ /*
+ * OK, at this point we have found a page we are going to free.
+ */
+#if CONFIG_PHANTOM_CACHE
+ if (!object->internal) {
+ vm_phantom_cache_add_ghost(m);
+ }
+#endif
+ goto reclaim_page;
+ }
+
+ /*
+ * The page may have been dirtied since the last check
+ * for a throttled target queue (which may have been skipped
+ * if the page was clean then). With the dirty page
+ * disconnected here, we can make one final check.
+ */
+ if (object->internal) {
+ if (VM_PAGE_Q_THROTTLED(iq)) {
+ inactive_throttled = TRUE;
+ }
+ } else if (VM_PAGE_Q_THROTTLED(eq)) {
+ inactive_throttled = TRUE;
+ }
+
+ if (inactive_throttled == TRUE) {
+ goto throttle_inactive;
+ }
+
+#if VM_PRESSURE_EVENTS
+#if CONFIG_JETSAM
+
+ /*
+ * If Jetsam is enabled, then the sending
+ * of memory pressure notifications is handled
+ * from the same thread that takes care of high-water
+ * and other jetsams i.e. the memorystatus_thread.
+ */
+
+#else /* CONFIG_JETSAM */
+
+ vm_pressure_response();
+
+#endif /* CONFIG_JETSAM */
+#endif /* VM_PRESSURE_EVENTS */
+
+ if (page_prev_q_state == VM_PAGE_ON_SPECULATIVE_Q) {
+ VM_PAGEOUT_DEBUG(vm_pageout_speculative_dirty, 1);
+ }
+
+ if (object->internal) {
+ vm_pageout_vminfo.vm_pageout_inactive_dirty_internal++;
+ } else {
+ vm_pageout_vminfo.vm_pageout_inactive_dirty_external++;
+ }
+
+ /*
+ * internal pages will go to the compressor...
+ * external pages will go to the appropriate pager to be cleaned
+ * and upon completion will end up on 'vm_page_queue_cleaned' which
+ * is a preferred queue to steal from
+ */
+ vm_pageout_cluster(m);
+ inactive_burst_count = 0;
+
+ /*
+ * back to top of pageout scan loop
+ */
+ }
+}
+
+
+void
+vm_page_free_reserve(
+ int pages)
+{
+ int free_after_reserve;
+
+ if (VM_CONFIG_COMPRESSOR_IS_PRESENT) {
+ if ((vm_page_free_reserved + pages + COMPRESSOR_FREE_RESERVED_LIMIT) >= (VM_PAGE_FREE_RESERVED_LIMIT + COMPRESSOR_FREE_RESERVED_LIMIT)) {
+ vm_page_free_reserved = VM_PAGE_FREE_RESERVED_LIMIT + COMPRESSOR_FREE_RESERVED_LIMIT;
+ } else {
+ vm_page_free_reserved += (pages + COMPRESSOR_FREE_RESERVED_LIMIT);
+ }
+ } else {
+ if ((vm_page_free_reserved + pages) >= VM_PAGE_FREE_RESERVED_LIMIT) {
+ vm_page_free_reserved = VM_PAGE_FREE_RESERVED_LIMIT;
+ } else {
+ vm_page_free_reserved += pages;
+ }
+ }
+ free_after_reserve = vm_pageout_state.vm_page_free_count_init - vm_page_free_reserved;
+
+ vm_page_free_min = vm_page_free_reserved +
+ VM_PAGE_FREE_MIN(free_after_reserve);
+
+ if (vm_page_free_min > VM_PAGE_FREE_MIN_LIMIT) {
+ vm_page_free_min = VM_PAGE_FREE_MIN_LIMIT;
+ }
+
+ vm_page_free_target = vm_page_free_reserved +
+ VM_PAGE_FREE_TARGET(free_after_reserve);
+
+ if (vm_page_free_target > VM_PAGE_FREE_TARGET_LIMIT) {
+ vm_page_free_target = VM_PAGE_FREE_TARGET_LIMIT;
+ }
+
+ if (vm_page_free_target < vm_page_free_min + 5) {
+ vm_page_free_target = vm_page_free_min + 5;
+ }
+
+ vm_page_throttle_limit = vm_page_free_target - (vm_page_free_target / 2);
+}
+
+/*
+ * vm_pageout is the high level pageout daemon.
+ */
+
+void
+vm_pageout_continue(void)
+{
+ DTRACE_VM2(pgrrun, int, 1, (uint64_t *), NULL);
+ VM_PAGEOUT_DEBUG(vm_pageout_scan_event_counter, 1);
+
+ lck_mtx_lock(&vm_page_queue_free_lock);
+ vm_pageout_running = TRUE;
+ lck_mtx_unlock(&vm_page_queue_free_lock);
+
+ vm_pageout_scan();
+ /*
+ * we hold both the vm_page_queue_free_lock
+ * and the vm_page_queues_lock at this point
+ */
+ assert(vm_page_free_wanted == 0);
+ assert(vm_page_free_wanted_privileged == 0);
+ assert_wait((event_t) &vm_page_free_wanted, THREAD_UNINT);
+
+ vm_pageout_running = FALSE;
+#if !CONFIG_EMBEDDED
+ if (vm_pageout_waiter) {
+ vm_pageout_waiter = FALSE;
+ thread_wakeup((event_t)&vm_pageout_waiter);
+ }
+#endif /* !CONFIG_EMBEDDED */
+
+ lck_mtx_unlock(&vm_page_queue_free_lock);
+ vm_page_unlock_queues();
+
+ counter(c_vm_pageout_block++);
+ thread_block((thread_continue_t)vm_pageout_continue);
+ /*NOTREACHED*/
+}
+
+#if !CONFIG_EMBEDDED
+kern_return_t
+vm_pageout_wait(uint64_t deadline)
+{
+ kern_return_t kr;
+
+ lck_mtx_lock(&vm_page_queue_free_lock);
+ for (kr = KERN_SUCCESS; vm_pageout_running && (KERN_SUCCESS == kr);) {
+ vm_pageout_waiter = TRUE;
+ if (THREAD_AWAKENED != lck_mtx_sleep_deadline(
+ &vm_page_queue_free_lock, LCK_SLEEP_DEFAULT,
+ (event_t) &vm_pageout_waiter, THREAD_UNINT, deadline)) {
+ kr = KERN_OPERATION_TIMED_OUT;
+ }
+ }
+ lck_mtx_unlock(&vm_page_queue_free_lock);
+
+ return kr;
+}
+#endif /* !CONFIG_EMBEDDED */
+
+
+static void
+vm_pageout_iothread_external_continue(struct vm_pageout_queue *q)
+{
+ vm_page_t m = NULL;
+ vm_object_t object;
+ vm_object_offset_t offset;
+ memory_object_t pager;
+
+ /* On systems with a compressor, the external IO thread clears its
+ * VM privileged bit to accommodate large allocations (e.g. bulk UPL
+ * creation)
+ */
+ if (vm_pageout_state.vm_pageout_internal_iothread != THREAD_NULL) {
+ current_thread()->options &= ~TH_OPT_VMPRIV;
+ }
+
+ vm_page_lockspin_queues();
+
+ while (!vm_page_queue_empty(&q->pgo_pending)) {
+ q->pgo_busy = TRUE;
+ vm_page_queue_remove_first(&q->pgo_pending, m, vmp_pageq);
+
+ assert(m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q);
+ VM_PAGE_CHECK(m);
+ /*
+ * grab a snapshot of the object and offset this
+ * page is tabled in so that we can relookup this
+ * page after we've taken the object lock - these
+ * fields are stable while we hold the page queues lock
+ * but as soon as we drop it, there is nothing to keep
+ * this page in this object... we hold an activity_in_progress
+ * on this object which will keep it from terminating
+ */
+ object = VM_PAGE_OBJECT(m);
+ offset = m->vmp_offset;
+
+ m->vmp_q_state = VM_PAGE_NOT_ON_Q;
+ VM_PAGE_ZERO_PAGEQ_ENTRY(m);
+
+ vm_page_unlock_queues();
+
+ vm_object_lock(object);
+
+ m = vm_page_lookup(object, offset);
+
+ if (m == NULL || m->vmp_busy || m->vmp_cleaning ||
+ !m->vmp_laundry || (m->vmp_q_state != VM_PAGE_NOT_ON_Q)) {
+ /*
+ * it's either the same page that someone else has
+ * started cleaning (or it's finished cleaning or
+ * been put back on the pageout queue), or
+ * the page has been freed or we have found a
+ * new page at this offset... in all of these cases
+ * we merely need to release the activity_in_progress
+ * we took when we put the page on the pageout queue
+ */
+ vm_object_activity_end(object);
+ vm_object_unlock(object);
+
+ vm_page_lockspin_queues();
+ continue;
+ }
+ pager = object->pager;
+
+ if (pager == MEMORY_OBJECT_NULL) {
+ /*
+ * This pager has been destroyed by either
+ * memory_object_destroy or vm_object_destroy, and
+ * so there is nowhere for the page to go.
+ */
+ if (m->vmp_free_when_done) {
+ /*
+ * Just free the page... VM_PAGE_FREE takes
+ * care of cleaning up all the state...
+ * including doing the vm_pageout_throttle_up
+ */
+ VM_PAGE_FREE(m);
+ } else {
+ vm_page_lockspin_queues();
+
+ vm_pageout_throttle_up(m);
+ vm_page_activate(m);
+
+ vm_page_unlock_queues();
+
+ /*
+ * And we are done with it.
+ */
+ }
+ vm_object_activity_end(object);
+ vm_object_unlock(object);
+
+ vm_page_lockspin_queues();
+ continue;
+ }
+#if 0
+ /*
+ * we don't hold the page queue lock
+ * so this check isn't safe to make
+ */
+ VM_PAGE_CHECK(m);
+#endif
+ /*
+ * give back the activity_in_progress reference we
+ * took when we queued up this page and replace it
+ * it with a paging_in_progress reference that will
+ * also hold the paging offset from changing and
+ * prevent the object from terminating
+ */
+ vm_object_activity_end(object);
+ vm_object_paging_begin(object);
+ vm_object_unlock(object);
+
+ /*
+ * Send the data to the pager.
+ * any pageout clustering happens there
+ */
+ memory_object_data_return(pager,
+ m->vmp_offset + object->paging_offset,
+ PAGE_SIZE,
+ NULL,
+ NULL,
+ FALSE,
+ FALSE,
+ 0);
+
+ vm_object_lock(object);
+ vm_object_paging_end(object);
+ vm_object_unlock(object);
+
+ vm_pageout_io_throttle();
+
+ vm_page_lockspin_queues();
+ }
+ q->pgo_busy = FALSE;
+ q->pgo_idle = TRUE;
+
+ assert_wait((event_t) &q->pgo_pending, THREAD_UNINT);
+ vm_page_unlock_queues();
+
+ thread_block_parameter((thread_continue_t)vm_pageout_iothread_external_continue, (void *) q);
+ /*NOTREACHED*/
+}
+
+
+#define MAX_FREE_BATCH 32
+uint32_t vm_compressor_time_thread; /* Set via sysctl to record time accrued by
+ * this thread.
+ */
+
+
+void
+vm_pageout_iothread_internal_continue(struct cq *);
+void
+vm_pageout_iothread_internal_continue(struct cq *cq)
+{
+ struct vm_pageout_queue *q;
+ vm_page_t m = NULL;
+ boolean_t pgo_draining;
+ vm_page_t local_q;
+ int local_cnt;
+ vm_page_t local_freeq = NULL;
+ int local_freed = 0;
+ int local_batch_size;
+#if DEVELOPMENT || DEBUG
+ int ncomps = 0;
+ boolean_t marked_active = FALSE;
+#endif
+ KERNEL_DEBUG(0xe040000c | DBG_FUNC_END, 0, 0, 0, 0, 0);
+
+ q = cq->q;
+#if __AMP__
+ if (vm_compressor_ebound && (vm_pageout_state.vm_compressor_thread_count > 1)) {
+ local_batch_size = (q->pgo_maxlaundry >> 3);
+ local_batch_size = MAX(local_batch_size, 16);
+ } else {
+ local_batch_size = q->pgo_maxlaundry / (vm_pageout_state.vm_compressor_thread_count * 2);
+ }
+#else
+ local_batch_size = q->pgo_maxlaundry / (vm_pageout_state.vm_compressor_thread_count * 2);
+#endif
+
+#if RECORD_THE_COMPRESSED_DATA
+ if (q->pgo_laundry) {
+ c_compressed_record_init();
+ }
+#endif
+ while (TRUE) {
+ int pages_left_on_q = 0;
+
+ local_cnt = 0;
+ local_q = NULL;
+
+ KERNEL_DEBUG(0xe0400014 | DBG_FUNC_START, 0, 0, 0, 0, 0);
+
+ vm_page_lock_queues();
+#if DEVELOPMENT || DEBUG
+ if (marked_active == FALSE) {
+ vmct_active++;
+ vmct_state[cq->id] = VMCT_ACTIVE;
+ marked_active = TRUE;
+ if (vmct_active == 1) {
+ vm_compressor_epoch_start = mach_absolute_time();
+ }
+ }
+#endif
+ KERNEL_DEBUG(0xe0400014 | DBG_FUNC_END, 0, 0, 0, 0, 0);
+
+ KERNEL_DEBUG(0xe0400018 | DBG_FUNC_START, q->pgo_laundry, 0, 0, 0, 0);
+
+ while (!vm_page_queue_empty(&q->pgo_pending) && local_cnt < local_batch_size) {
+ vm_page_queue_remove_first(&q->pgo_pending, m, vmp_pageq);
+ assert(m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q);
+ VM_PAGE_CHECK(m);
+
+ m->vmp_q_state = VM_PAGE_NOT_ON_Q;
+ VM_PAGE_ZERO_PAGEQ_ENTRY(m);
+ m->vmp_laundry = FALSE;
+
+ m->vmp_snext = local_q;
+ local_q = m;
+ local_cnt++;
+ }
+ if (local_q == NULL) {
+ break;
+ }
+
+ q->pgo_busy = TRUE;
+
+ if ((pgo_draining = q->pgo_draining) == FALSE) {
+ vm_pageout_throttle_up_batch(q, local_cnt);
+ pages_left_on_q = q->pgo_laundry;
+ } else {
+ pages_left_on_q = q->pgo_laundry - local_cnt;
+ }
+
+ vm_page_unlock_queues();
+
+#if !RECORD_THE_COMPRESSED_DATA
+ if (pages_left_on_q >= local_batch_size && cq->id < (vm_pageout_state.vm_compressor_thread_count - 1)) {
+ thread_wakeup((event_t) ((uintptr_t)&q->pgo_pending + cq->id + 1));
+ }
+#endif
+ KERNEL_DEBUG(0xe0400018 | DBG_FUNC_END, q->pgo_laundry, 0, 0, 0, 0);
+
+ while (local_q) {
+ KERNEL_DEBUG(0xe0400024 | DBG_FUNC_START, local_cnt, 0, 0, 0, 0);
+
+ m = local_q;
+ local_q = m->vmp_snext;
+ m->vmp_snext = NULL;
+
+ if (vm_pageout_compress_page(&cq->current_chead, cq->scratch_buf, m) == KERN_SUCCESS) {
+#if DEVELOPMENT || DEBUG
+ ncomps++;
+#endif
+ KERNEL_DEBUG(0xe0400024 | DBG_FUNC_END, local_cnt, 0, 0, 0, 0);
+
+ m->vmp_snext = local_freeq;
+ local_freeq = m;
+ local_freed++;
+
+ if (local_freed >= MAX_FREE_BATCH) {
+ OSAddAtomic64(local_freed, &vm_pageout_vminfo.vm_pageout_compressions);
+
+ vm_page_free_list(local_freeq, TRUE);
+
+ local_freeq = NULL;
+ local_freed = 0;
+ }
+ }
+#if !CONFIG_JETSAM
+ while (vm_page_free_count < COMPRESSOR_FREE_RESERVED_LIMIT) {
+ kern_return_t wait_result;
+ int need_wakeup = 0;
+
+ if (local_freeq) {
+ OSAddAtomic64(local_freed, &vm_pageout_vminfo.vm_pageout_compressions);
+
+ vm_page_free_list(local_freeq, TRUE);
+ local_freeq = NULL;
+ local_freed = 0;
+
+ continue;
+ }
+ lck_mtx_lock_spin(&vm_page_queue_free_lock);
+
+ if (vm_page_free_count < COMPRESSOR_FREE_RESERVED_LIMIT) {
+ if (vm_page_free_wanted_privileged++ == 0) {
+ need_wakeup = 1;
+ }
+ wait_result = assert_wait((event_t)&vm_page_free_wanted_privileged, THREAD_UNINT);
+
+ lck_mtx_unlock(&vm_page_queue_free_lock);
+
+ if (need_wakeup) {
+ thread_wakeup((event_t)&vm_page_free_wanted);
+ }
+
+ if (wait_result == THREAD_WAITING) {
+ thread_block(THREAD_CONTINUE_NULL);
+ }
+ } else {
+ lck_mtx_unlock(&vm_page_queue_free_lock);
+ }
+ }
+#endif
+ }
+ if (local_freeq) {
+ OSAddAtomic64(local_freed, &vm_pageout_vminfo.vm_pageout_compressions);
+
+ vm_page_free_list(local_freeq, TRUE);
+ local_freeq = NULL;
+ local_freed = 0;
+ }
+ if (pgo_draining == TRUE) {
+ vm_page_lockspin_queues();
+ vm_pageout_throttle_up_batch(q, local_cnt);
+ vm_page_unlock_queues();
+ }
+ }
+ KERNEL_DEBUG(0xe040000c | DBG_FUNC_START, 0, 0, 0, 0, 0);
+
+ /*
+ * queue lock is held and our q is empty
+ */
+ q->pgo_busy = FALSE;
+ q->pgo_idle = TRUE;
+
+ assert_wait((event_t) ((uintptr_t)&q->pgo_pending + cq->id), THREAD_UNINT);
+#if DEVELOPMENT || DEBUG
+ if (marked_active == TRUE) {
+ vmct_active--;
+ vmct_state[cq->id] = VMCT_IDLE;
+
+ if (vmct_active == 0) {
+ vm_compressor_epoch_stop = mach_absolute_time();
+ assertf(vm_compressor_epoch_stop >= vm_compressor_epoch_start,
+ "Compressor epoch non-monotonic: 0x%llx -> 0x%llx",
+ vm_compressor_epoch_start, vm_compressor_epoch_stop);
+ /* This interval includes intervals where one or more
+ * compressor threads were pre-empted
+ */
+ vmct_stats.vmct_cthreads_total += vm_compressor_epoch_stop - vm_compressor_epoch_start;
+ }
+ }
+#endif
+ vm_page_unlock_queues();
+#if DEVELOPMENT || DEBUG
+ if (__improbable(vm_compressor_time_thread)) {
+ vmct_stats.vmct_runtimes[cq->id] = thread_get_runtime_self();
+ vmct_stats.vmct_pages[cq->id] += ncomps;
+ vmct_stats.vmct_iterations[cq->id]++;
+ if (ncomps > vmct_stats.vmct_maxpages[cq->id]) {
+ vmct_stats.vmct_maxpages[cq->id] = ncomps;
+ }
+ if (ncomps < vmct_stats.vmct_minpages[cq->id]) {
+ vmct_stats.vmct_minpages[cq->id] = ncomps;
+ }
+ }
+#endif
+
+ KERNEL_DEBUG(0xe0400018 | DBG_FUNC_END, 0, 0, 0, 0, 0);
+
+ thread_block_parameter((thread_continue_t)vm_pageout_iothread_internal_continue, (void *) cq);
+ /*NOTREACHED*/
+}
+
+
+kern_return_t
+vm_pageout_compress_page(void **current_chead, char *scratch_buf, vm_page_t m)
+{
+ vm_object_t object;
+ memory_object_t pager;
+ int compressed_count_delta;
+ kern_return_t retval;
+
+ object = VM_PAGE_OBJECT(m);
+
+ assert(!m->vmp_free_when_done);
+ assert(!m->vmp_laundry);
+
+ pager = object->pager;
+
+ if (!object->pager_initialized || pager == MEMORY_OBJECT_NULL) {
+ KERNEL_DEBUG(0xe0400010 | DBG_FUNC_START, object, pager, 0, 0, 0);
+
+ vm_object_lock(object);
+
+ /*
+ * If there is no memory object for the page, create
+ * one and hand it to the compression pager.
+ */
+
+ if (!object->pager_initialized) {
+ vm_object_collapse(object, (vm_object_offset_t) 0, TRUE);
+ }
+ if (!object->pager_initialized) {
+ vm_object_compressor_pager_create(object);
+ }
+
+ pager = object->pager;
+
+ if (!object->pager_initialized || pager == MEMORY_OBJECT_NULL) {
+ /*
+ * Still no pager for the object,
+ * or the pager has been destroyed.
+ * Reactivate the page.
+ *
+ * Should only happen if there is no
+ * compression pager
+ */
+ PAGE_WAKEUP_DONE(m);
+
+ vm_page_lockspin_queues();
+ vm_page_activate(m);
+ VM_PAGEOUT_DEBUG(vm_pageout_dirty_no_pager, 1);
+ vm_page_unlock_queues();
+
+ /*
+ * And we are done with it.
+ */
+ vm_object_activity_end(object);
+ vm_object_unlock(object);
+
+ return KERN_FAILURE;
+ }
+ vm_object_unlock(object);
+
+ KERNEL_DEBUG(0xe0400010 | DBG_FUNC_END, object, pager, 0, 0, 0);
+ }
+ assert(object->pager_initialized && pager != MEMORY_OBJECT_NULL);
+ assert(object->activity_in_progress > 0);
+
+ retval = vm_compressor_pager_put(
+ pager,
+ m->vmp_offset + object->paging_offset,
+ VM_PAGE_GET_PHYS_PAGE(m),
+ current_chead,
+ scratch_buf,
+ &compressed_count_delta);
+
+ vm_object_lock(object);
+
+ assert(object->activity_in_progress > 0);
+ assert(VM_PAGE_OBJECT(m) == object);
+ assert( !VM_PAGE_WIRED(m));
+
+ vm_compressor_pager_count(pager,
+ compressed_count_delta,
+ FALSE, /* shared_lock */
+ object);
+
+ if (retval == KERN_SUCCESS) {
+ /*
+ * If the object is purgeable, its owner's
+ * purgeable ledgers will be updated in
+ * vm_page_remove() but the page still
+ * contributes to the owner's memory footprint,
+ * so account for it as such.
+ */
+ if ((object->purgable != VM_PURGABLE_DENY ||
+ object->vo_ledger_tag) &&
+ object->vo_owner != NULL) {
+ /* one more compressed purgeable/tagged page */
+ vm_object_owner_compressed_update(object,
+ +1);
+ }
+ VM_STAT_INCR(compressions);
+
+ if (m->vmp_tabled) {
+ vm_page_remove(m, TRUE);
+ }
+ } else {
+ PAGE_WAKEUP_DONE(m);
+
+ vm_page_lockspin_queues();
+
+ vm_page_activate(m);
+ vm_pageout_vminfo.vm_compressor_failed++;
+
+ vm_page_unlock_queues();
+ }
+ vm_object_activity_end(object);
+ vm_object_unlock(object);
+
+ return retval;
+}
+
+
+static void
+vm_pageout_adjust_eq_iothrottle(struct vm_pageout_queue *eq, boolean_t req_lowpriority)
+{
+ uint32_t policy;
+
+ if (hibernate_cleaning_in_progress == TRUE) {
+ req_lowpriority = FALSE;
+ }
+
+ if (eq->pgo_inited == TRUE && eq->pgo_lowpriority != req_lowpriority) {
+ vm_page_unlock_queues();
+
+ if (req_lowpriority == TRUE) {
+ policy = THROTTLE_LEVEL_PAGEOUT_THROTTLED;
+ DTRACE_VM(laundrythrottle);
+ } else {
+ policy = THROTTLE_LEVEL_PAGEOUT_UNTHROTTLED;
+ DTRACE_VM(laundryunthrottle);
+ }
+ proc_set_thread_policy_with_tid(kernel_task, eq->pgo_tid,
+ TASK_POLICY_EXTERNAL, TASK_POLICY_IO, policy);
+
+ vm_page_lock_queues();
+ eq->pgo_lowpriority = req_lowpriority;
+ }
+}
+
+
+static void
+vm_pageout_iothread_external(void)
+{
+ thread_t self = current_thread();
+
+ self->options |= TH_OPT_VMPRIV;
+
+ DTRACE_VM2(laundrythrottle, int, 1, (uint64_t *), NULL);
+
+ proc_set_thread_policy(self, TASK_POLICY_EXTERNAL,
+ TASK_POLICY_IO, THROTTLE_LEVEL_PAGEOUT_THROTTLED);
+
+ vm_page_lock_queues();
+
+ vm_pageout_queue_external.pgo_tid = self->thread_id;
+ vm_pageout_queue_external.pgo_lowpriority = TRUE;
+ vm_pageout_queue_external.pgo_inited = TRUE;
+
+ vm_page_unlock_queues();
+
+ vm_pageout_iothread_external_continue(&vm_pageout_queue_external);
+
+ /*NOTREACHED*/
+}
+
+
+static void
+vm_pageout_iothread_internal(struct cq *cq)
+{
+ thread_t self = current_thread();
+
+ self->options |= TH_OPT_VMPRIV;
+
+ vm_page_lock_queues();
+
+ vm_pageout_queue_internal.pgo_tid = self->thread_id;
+ vm_pageout_queue_internal.pgo_lowpriority = TRUE;
+ vm_pageout_queue_internal.pgo_inited = TRUE;
+
+ vm_page_unlock_queues();
+
+ if (vm_pageout_state.vm_restricted_to_single_processor == TRUE) {
+ thread_vm_bind_group_add();
+ }
+
+#if CONFIG_THREAD_GROUPS
+ thread_group_vm_add();
+#endif /* CONFIG_THREAD_GROUPS */
+
+#if __AMP__
+ if (vm_compressor_ebound) {
+ /*
+ * Use the soft bound option for vm_compressor to allow it to run on
+ * P-cores if E-cluster is unavailable.
+ */
+ thread_bind_cluster_type(self, 'E', true);
+ }
+#endif /* __AMP__ */
+
+ thread_set_thread_name(current_thread(), "VM_compressor");
+#if DEVELOPMENT || DEBUG
+ vmct_stats.vmct_minpages[cq->id] = INT32_MAX;
+#endif
+ vm_pageout_iothread_internal_continue(cq);
+
+ /*NOTREACHED*/
+}
+
+kern_return_t
+vm_set_buffer_cleanup_callout(boolean_t (*func)(int))
+{
+ if (OSCompareAndSwapPtr(NULL, ptrauth_nop_cast(void *, func), (void * volatile *) &consider_buffer_cache_collect)) {
+ return KERN_SUCCESS;
+ } else {
+ return KERN_FAILURE; /* Already set */
+ }
+}
+
+extern boolean_t memorystatus_manual_testing_on;
+extern unsigned int memorystatus_level;
+
+
+#if VM_PRESSURE_EVENTS
+
+boolean_t vm_pressure_events_enabled = FALSE;
+
+void
+vm_pressure_response(void)
+{
+ vm_pressure_level_t old_level = kVMPressureNormal;
+ int new_level = -1;
+ unsigned int total_pages;
+ uint64_t available_memory = 0;
+
+ if (vm_pressure_events_enabled == FALSE) {
+ return;
+ }
+
+#if CONFIG_EMBEDDED
+
+ available_memory = (uint64_t) memorystatus_available_pages;
+
+#else /* CONFIG_EMBEDDED */
+
+ available_memory = (uint64_t) AVAILABLE_NON_COMPRESSED_MEMORY;
+ memorystatus_available_pages = (uint64_t) AVAILABLE_NON_COMPRESSED_MEMORY;
+
+#endif /* CONFIG_EMBEDDED */
+
+ total_pages = (unsigned int) atop_64(max_mem);
+#if CONFIG_SECLUDED_MEMORY
+ total_pages -= vm_page_secluded_count;
+#endif /* CONFIG_SECLUDED_MEMORY */
+ memorystatus_level = (unsigned int) ((available_memory * 100) / total_pages);
+
+ if (memorystatus_manual_testing_on) {
+ return;
+ }
+
+ old_level = memorystatus_vm_pressure_level;
+
+ switch (memorystatus_vm_pressure_level) {
+ case kVMPressureNormal:
+ {
+ if (VM_PRESSURE_WARNING_TO_CRITICAL()) {
+ new_level = kVMPressureCritical;
+ } else if (VM_PRESSURE_NORMAL_TO_WARNING()) {
+ new_level = kVMPressureWarning;
+ }
+ break;
+ }
+
+ case kVMPressureWarning:
+ case kVMPressureUrgent:
+ {
+ if (VM_PRESSURE_WARNING_TO_NORMAL()) {
+ new_level = kVMPressureNormal;
+ } else if (VM_PRESSURE_WARNING_TO_CRITICAL()) {
+ new_level = kVMPressureCritical;
+ }
+ break;
+ }
+
+ case kVMPressureCritical:
+ {
+ if (VM_PRESSURE_WARNING_TO_NORMAL()) {
+ new_level = kVMPressureNormal;
+ } else if (VM_PRESSURE_CRITICAL_TO_WARNING()) {
+ new_level = kVMPressureWarning;
+ }
+ break;
+ }
+
+ default:
+ return;
+ }
+
+ if (new_level != -1) {
+ memorystatus_vm_pressure_level = (vm_pressure_level_t) new_level;
+
+ if (new_level != (int) old_level) {
+ VM_DEBUG_CONSTANT_EVENT(vm_pressure_level_change, VM_PRESSURE_LEVEL_CHANGE, DBG_FUNC_NONE,
+ new_level, old_level, 0, 0);
+ }
+
+ if ((memorystatus_vm_pressure_level != kVMPressureNormal) || (old_level != memorystatus_vm_pressure_level)) {
+ if (vm_pageout_state.vm_pressure_thread_running == FALSE) {
+ thread_wakeup(&vm_pressure_thread);
+ }
+
+ if (old_level != memorystatus_vm_pressure_level) {
+ thread_wakeup(&vm_pageout_state.vm_pressure_changed);
+ }
+ }
+ }
+}
+#endif /* VM_PRESSURE_EVENTS */
+
+/*
+ * Function called by a kernel thread to either get the current pressure level or
+ * wait until memory pressure changes from a given level.
+ */
+kern_return_t
+mach_vm_pressure_level_monitor(__unused boolean_t wait_for_pressure, __unused unsigned int *pressure_level)
+{
+#if !VM_PRESSURE_EVENTS
+
+ return KERN_FAILURE;
+
+#else /* VM_PRESSURE_EVENTS */
+
+ wait_result_t wr = 0;
+ vm_pressure_level_t old_level = memorystatus_vm_pressure_level;
+
+ if (pressure_level == NULL) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ if (*pressure_level == kVMPressureJetsam) {
+ if (!wait_for_pressure) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ lck_mtx_lock(&memorystatus_jetsam_fg_band_lock);
+ wr = assert_wait((event_t)&memorystatus_jetsam_fg_band_waiters,
+ THREAD_INTERRUPTIBLE);
+ if (wr == THREAD_WAITING) {
+ ++memorystatus_jetsam_fg_band_waiters;
+ lck_mtx_unlock(&memorystatus_jetsam_fg_band_lock);
+ wr = thread_block(THREAD_CONTINUE_NULL);
+ } else {
+ lck_mtx_unlock(&memorystatus_jetsam_fg_band_lock);
+ }
+ if (wr != THREAD_AWAKENED) {
+ return KERN_ABORTED;
+ }
+ *pressure_level = kVMPressureJetsam;
+ return KERN_SUCCESS;
+ }
+
+ if (wait_for_pressure == TRUE) {
+ while (old_level == *pressure_level) {
+ wr = assert_wait((event_t) &vm_pageout_state.vm_pressure_changed,
+ THREAD_INTERRUPTIBLE);
+ if (wr == THREAD_WAITING) {
+ wr = thread_block(THREAD_CONTINUE_NULL);
+ }
+ if (wr == THREAD_INTERRUPTED) {
+ return KERN_ABORTED;
+ }
+
+ if (wr == THREAD_AWAKENED) {
+ old_level = memorystatus_vm_pressure_level;
+ }
+ }
+ }
+
+ *pressure_level = old_level;
+ return KERN_SUCCESS;
+#endif /* VM_PRESSURE_EVENTS */
+}
+
+#if VM_PRESSURE_EVENTS
+void
+vm_pressure_thread(void)
+{
+ static boolean_t thread_initialized = FALSE;
+
+ if (thread_initialized == TRUE) {
+ vm_pageout_state.vm_pressure_thread_running = TRUE;
+ consider_vm_pressure_events();
+ vm_pageout_state.vm_pressure_thread_running = FALSE;
+ }
+
+ thread_set_thread_name(current_thread(), "VM_pressure");
+ thread_initialized = TRUE;
+ assert_wait((event_t) &vm_pressure_thread, THREAD_UNINT);
+ thread_block((thread_continue_t)vm_pressure_thread);
+}
+#endif /* VM_PRESSURE_EVENTS */
+
+
+/*
+ * called once per-second via "compute_averages"
+ */
+void
+compute_pageout_gc_throttle(__unused void *arg)
+{
+ if (vm_pageout_vminfo.vm_pageout_considered_page != vm_pageout_state.vm_pageout_considered_page_last) {
+ vm_pageout_state.vm_pageout_considered_page_last = vm_pageout_vminfo.vm_pageout_considered_page;
+
+ thread_wakeup((event_t) &vm_pageout_garbage_collect);
+ }
+}
+
+/*
+ * vm_pageout_garbage_collect can also be called when the zone allocator needs
+ * to call zone_gc on a different thread in order to trigger zone-map-exhaustion
+ * jetsams. We need to check if the zone map size is above its jetsam limit to
+ * decide if this was indeed the case.
+ *
+ * We need to do this on a different thread because of the following reasons:
+ *
+ * 1. In the case of synchronous jetsams, the leaking process can try to jetsam
+ * itself causing the system to hang. We perform synchronous jetsams if we're
+ * leaking in the VM map entries zone, so the leaking process could be doing a
+ * zalloc for a VM map entry while holding its vm_map lock, when it decides to
+ * jetsam itself. We also need the vm_map lock on the process termination path,
+ * which would now lead the dying process to deadlock against itself.
+ *
+ * 2. The jetsam path might need to allocate zone memory itself. We could try
+ * using the non-blocking variant of zalloc for this path, but we can still
+ * end up trying to do a kernel_memory_allocate when the zone maps are almost
+ * full.
+ */
+
+void
+vm_pageout_garbage_collect(int collect)
+{
+ if (collect) {
+ if (is_zone_map_nearing_exhaustion()) {
+ /*
+ * Woken up by the zone allocator for zone-map-exhaustion jetsams.
+ *
+ * Bail out after calling zone_gc (which triggers the
+ * zone-map-exhaustion jetsams). If we fall through, the subsequent
+ * operations that clear out a bunch of caches might allocate zone
+ * memory themselves (for eg. vm_map operations would need VM map
+ * entries). Since the zone map is almost full at this point, we
+ * could end up with a panic. We just need to quickly jetsam a
+ * process and exit here.
+ *
+ * It could so happen that we were woken up to relieve memory
+ * pressure and the zone map also happened to be near its limit at
+ * the time, in which case we'll skip out early. But that should be
+ * ok; if memory pressure persists, the thread will simply be woken
+ * up again.
+ */
+ consider_zone_gc(TRUE);
+ } else {
+ /* Woken up by vm_pageout_scan or compute_pageout_gc_throttle. */
+ boolean_t buf_large_zfree = FALSE;
+ boolean_t first_try = TRUE;
+
+ stack_collect();
+
+ consider_machine_collect();
+ mbuf_drain(FALSE);
+
+ do {
+ if (consider_buffer_cache_collect != NULL) {
+ buf_large_zfree = (*consider_buffer_cache_collect)(0);
+ }
+ if (first_try == TRUE || buf_large_zfree == TRUE) {
+ /*
+ * consider_zone_gc should be last, because the other operations
+ * might return memory to zones.
+ */
+ consider_zone_gc(FALSE);
+ }
+ first_try = FALSE;
+ } while (buf_large_zfree == TRUE && vm_page_free_count < vm_page_free_target);
+
+ consider_machine_adjust();
+ }
+ }
+
+ assert_wait((event_t) &vm_pageout_garbage_collect, THREAD_UNINT);
+
+ thread_block_parameter((thread_continue_t) vm_pageout_garbage_collect, (void *)1);
+ /*NOTREACHED*/
+}
+
+
+#if VM_PAGE_BUCKETS_CHECK
+#if VM_PAGE_FAKE_BUCKETS
+extern vm_map_offset_t vm_page_fake_buckets_start, vm_page_fake_buckets_end;
+#endif /* VM_PAGE_FAKE_BUCKETS */
+#endif /* VM_PAGE_BUCKETS_CHECK */
+
+
+
+void
+vm_set_restrictions(unsigned int num_cpus)
+{
+ int vm_restricted_to_single_processor = 0;
+
+ if (PE_parse_boot_argn("vm_restricted_to_single_processor", &vm_restricted_to_single_processor, sizeof(vm_restricted_to_single_processor))) {
+ kprintf("Overriding vm_restricted_to_single_processor to %d\n", vm_restricted_to_single_processor);
+ vm_pageout_state.vm_restricted_to_single_processor = (vm_restricted_to_single_processor ? TRUE : FALSE);
+ } else {
+ assert(num_cpus > 0);
+
+ if (num_cpus <= 3) {
+ /*
+ * on systems with a limited number of CPUS, bind the
+ * 4 major threads that can free memory and that tend to use
+ * a fair bit of CPU under pressured conditions to a single processor.
+ * This insures that these threads don't hog all of the available CPUs
+ * (important for camera launch), while allowing them to run independently
+ * w/r to locks... the 4 threads are
+ * vm_pageout_scan, vm_pageout_iothread_internal (compressor),
+ * vm_compressor_swap_trigger_thread (minor and major compactions),
+ * memorystatus_thread (jetsams).
+ *
+ * the first time the thread is run, it is responsible for checking the
+ * state of vm_restricted_to_single_processor, and if TRUE it calls
+ * thread_bind_master... someday this should be replaced with a group
+ * scheduling mechanism and KPI.
+ */
+ vm_pageout_state.vm_restricted_to_single_processor = TRUE;
+ } else {
+ vm_pageout_state.vm_restricted_to_single_processor = FALSE;
+ }
+ }
+}
+
+void
+vm_pageout(void)
+{
+ thread_t self = current_thread();
+ thread_t thread;
+ kern_return_t result;
+ spl_t s;
+
+ /*
+ * Set thread privileges.
+ */
+ s = splsched();
+
+ vm_pageout_scan_thread = self;
+
+#if CONFIG_VPS_DYNAMIC_PRIO
+
+ int vps_dynprio_bootarg = 0;
+
+ if (PE_parse_boot_argn("vps_dynamic_priority_enabled", &vps_dynprio_bootarg, sizeof(vps_dynprio_bootarg))) {
+ vps_dynamic_priority_enabled = (vps_dynprio_bootarg ? TRUE : FALSE);
+ kprintf("Overriding vps_dynamic_priority_enabled to %d\n", vps_dynamic_priority_enabled);
+ } else {
+ if (vm_pageout_state.vm_restricted_to_single_processor == TRUE) {
+ vps_dynamic_priority_enabled = TRUE;
+ } else {
+ vps_dynamic_priority_enabled = FALSE;
+ }
+ }
+
+ if (vps_dynamic_priority_enabled) {
+ sched_set_kernel_thread_priority(self, MAXPRI_THROTTLE);
+ thread_set_eager_preempt(self);
+ } else {
+ sched_set_kernel_thread_priority(self, BASEPRI_VM);
+ }
+
+#else /* CONFIG_VPS_DYNAMIC_PRIO */
+
+ vps_dynamic_priority_enabled = FALSE;
+ sched_set_kernel_thread_priority(self, BASEPRI_VM);
+
+#endif /* CONFIG_VPS_DYNAMIC_PRIO */
+
+ thread_lock(self);
+ self->options |= TH_OPT_VMPRIV;
+ thread_unlock(self);
+
+ if (!self->reserved_stack) {
+ self->reserved_stack = self->kernel_stack;
+ }
+
+ if (vm_pageout_state.vm_restricted_to_single_processor == TRUE &&
+ vps_dynamic_priority_enabled == FALSE) {
+ thread_vm_bind_group_add();
+ }
+
+
+#if CONFIG_THREAD_GROUPS
+ thread_group_vm_add();
+#endif /* CONFIG_THREAD_GROUPS */
+
+#if __AMP__
+ PE_parse_boot_argn("vmpgo_pcluster", &vm_pgo_pbound, sizeof(vm_pgo_pbound));
+ if (vm_pgo_pbound) {
+ /*
+ * Use the soft bound option for vm pageout to allow it to run on
+ * E-cores if P-cluster is unavailable.
+ */
+ thread_bind_cluster_type(self, 'P', true);
+ }
+#endif /* __AMP__ */
+
+ splx(s);
+
+ thread_set_thread_name(current_thread(), "VM_pageout_scan");
+
+ /*
+ * Initialize some paging parameters.
+ */
+
+ vm_pageout_state.vm_pressure_thread_running = FALSE;
+ vm_pageout_state.vm_pressure_changed = FALSE;
+ vm_pageout_state.memorystatus_purge_on_warning = 2;
+ vm_pageout_state.memorystatus_purge_on_urgent = 5;
+ vm_pageout_state.memorystatus_purge_on_critical = 8;
+ vm_pageout_state.vm_page_speculative_q_age_ms = VM_PAGE_SPECULATIVE_Q_AGE_MS;
+ vm_pageout_state.vm_page_speculative_percentage = 5;
+ vm_pageout_state.vm_page_speculative_target = 0;
+
+ vm_pageout_state.vm_pageout_external_iothread = THREAD_NULL;
+ vm_pageout_state.vm_pageout_internal_iothread = THREAD_NULL;
+
+ vm_pageout_state.vm_pageout_swap_wait = 0;
+ vm_pageout_state.vm_pageout_idle_wait = 0;
+ vm_pageout_state.vm_pageout_empty_wait = 0;
+ vm_pageout_state.vm_pageout_burst_wait = 0;
+ vm_pageout_state.vm_pageout_deadlock_wait = 0;
+ vm_pageout_state.vm_pageout_deadlock_relief = 0;
+ vm_pageout_state.vm_pageout_burst_inactive_throttle = 0;
+
+ vm_pageout_state.vm_pageout_inactive = 0;
+ vm_pageout_state.vm_pageout_inactive_used = 0;
+ vm_pageout_state.vm_pageout_inactive_clean = 0;
+
+ vm_pageout_state.vm_memory_pressure = 0;
+ vm_pageout_state.vm_page_filecache_min = 0;
+#if CONFIG_JETSAM
+ vm_pageout_state.vm_page_filecache_min_divisor = 70;
+ vm_pageout_state.vm_page_xpmapped_min_divisor = 40;
+#else
+ vm_pageout_state.vm_page_filecache_min_divisor = 27;
+ vm_pageout_state.vm_page_xpmapped_min_divisor = 36;
+#endif
+ vm_pageout_state.vm_page_free_count_init = vm_page_free_count;
+
+ vm_pageout_state.vm_pageout_considered_page_last = 0;
+
+ if (vm_pageout_state.vm_pageout_swap_wait == 0) {
+ vm_pageout_state.vm_pageout_swap_wait = VM_PAGEOUT_SWAP_WAIT;
+ }
+
+ if (vm_pageout_state.vm_pageout_idle_wait == 0) {
+ vm_pageout_state.vm_pageout_idle_wait = VM_PAGEOUT_IDLE_WAIT;
+ }
+
+ if (vm_pageout_state.vm_pageout_burst_wait == 0) {
+ vm_pageout_state.vm_pageout_burst_wait = VM_PAGEOUT_BURST_WAIT;
+ }
+
+ if (vm_pageout_state.vm_pageout_empty_wait == 0) {
+ vm_pageout_state.vm_pageout_empty_wait = VM_PAGEOUT_EMPTY_WAIT;
+ }
+
+ if (vm_pageout_state.vm_pageout_deadlock_wait == 0) {
+ vm_pageout_state.vm_pageout_deadlock_wait = VM_PAGEOUT_DEADLOCK_WAIT;
+ }
+
+ if (vm_pageout_state.vm_pageout_deadlock_relief == 0) {
+ vm_pageout_state.vm_pageout_deadlock_relief = VM_PAGEOUT_DEADLOCK_RELIEF;
+ }
+
+ if (vm_pageout_state.vm_pageout_burst_inactive_throttle == 0) {
+ vm_pageout_state.vm_pageout_burst_inactive_throttle = VM_PAGEOUT_BURST_INACTIVE_THROTTLE;
+ }
+ /*
+ * even if we've already called vm_page_free_reserve
+ * call it again here to insure that the targets are
+ * accurately calculated (it uses vm_page_free_count_init)
+ * calling it with an arg of 0 will not change the reserve
+ * but will re-calculate free_min and free_target
+ */
+ if (vm_page_free_reserved < VM_PAGE_FREE_RESERVED(processor_count)) {
+ vm_page_free_reserve((VM_PAGE_FREE_RESERVED(processor_count)) - vm_page_free_reserved);
+ } else {
+ vm_page_free_reserve(0);
+ }
+
+
+ vm_page_queue_init(&vm_pageout_queue_external.pgo_pending);
+ vm_pageout_queue_external.pgo_maxlaundry = VM_PAGE_LAUNDRY_MAX;
+ vm_pageout_queue_external.pgo_laundry = 0;
+ vm_pageout_queue_external.pgo_idle = FALSE;
+ vm_pageout_queue_external.pgo_busy = FALSE;
+ vm_pageout_queue_external.pgo_throttled = FALSE;
+ vm_pageout_queue_external.pgo_draining = FALSE;
+ vm_pageout_queue_external.pgo_lowpriority = FALSE;
+ vm_pageout_queue_external.pgo_tid = -1;
+ vm_pageout_queue_external.pgo_inited = FALSE;
+
+ vm_page_queue_init(&vm_pageout_queue_internal.pgo_pending);
+ vm_pageout_queue_internal.pgo_maxlaundry = 0;
+ vm_pageout_queue_internal.pgo_laundry = 0;
+ vm_pageout_queue_internal.pgo_idle = FALSE;
+ vm_pageout_queue_internal.pgo_busy = FALSE;
+ vm_pageout_queue_internal.pgo_throttled = FALSE;
+ vm_pageout_queue_internal.pgo_draining = FALSE;
+ vm_pageout_queue_internal.pgo_lowpriority = FALSE;
+ vm_pageout_queue_internal.pgo_tid = -1;
+ vm_pageout_queue_internal.pgo_inited = FALSE;
+
+ /* internal pageout thread started when default pager registered first time */
+ /* external pageout and garbage collection threads started here */
+
+ result = kernel_thread_start_priority((thread_continue_t)vm_pageout_iothread_external, NULL,
+ BASEPRI_VM,
+ &vm_pageout_state.vm_pageout_external_iothread);
+ if (result != KERN_SUCCESS) {
+ panic("vm_pageout_iothread_external: create failed");
+ }
+ thread_set_thread_name(vm_pageout_state.vm_pageout_external_iothread, "VM_pageout_external_iothread");
+ thread_deallocate(vm_pageout_state.vm_pageout_external_iothread);
+
+ result = kernel_thread_start_priority((thread_continue_t)vm_pageout_garbage_collect, NULL,
+ BASEPRI_DEFAULT,
+ &thread);
+ if (result != KERN_SUCCESS) {
+ panic("vm_pageout_garbage_collect: create failed");
+ }
+ thread_set_thread_name(thread, "VM_pageout_garbage_collect");
+ thread_deallocate(thread);
+
+#if VM_PRESSURE_EVENTS
+ result = kernel_thread_start_priority((thread_continue_t)vm_pressure_thread, NULL,
+ BASEPRI_DEFAULT,
+ &thread);
+
+ if (result != KERN_SUCCESS) {
+ panic("vm_pressure_thread: create failed");
+ }
+
+ thread_deallocate(thread);
+#endif
+
+ vm_object_reaper_init();
+
+
+ bzero(&vm_config, sizeof(vm_config));
+
+ switch (vm_compressor_mode) {
+ case VM_PAGER_DEFAULT:
+ printf("mapping deprecated VM_PAGER_DEFAULT to VM_PAGER_COMPRESSOR_WITH_SWAP\n");
+ OS_FALLTHROUGH;
+
+ case VM_PAGER_COMPRESSOR_WITH_SWAP:
+ vm_config.compressor_is_present = TRUE;
+ vm_config.swap_is_present = TRUE;
+ vm_config.compressor_is_active = TRUE;
+ vm_config.swap_is_active = TRUE;
+ break;
+
+ case VM_PAGER_COMPRESSOR_NO_SWAP:
+ vm_config.compressor_is_present = TRUE;
+ vm_config.swap_is_present = TRUE;
+ vm_config.compressor_is_active = TRUE;
+ break;
+
+ case VM_PAGER_FREEZER_DEFAULT:
+ printf("mapping deprecated VM_PAGER_FREEZER_DEFAULT to VM_PAGER_FREEZER_COMPRESSOR_NO_SWAP\n");
+ OS_FALLTHROUGH;
+
+ case VM_PAGER_FREEZER_COMPRESSOR_NO_SWAP:
+ vm_config.compressor_is_present = TRUE;
+ vm_config.swap_is_present = TRUE;
+ break;
+
+ case VM_PAGER_COMPRESSOR_NO_SWAP_PLUS_FREEZER_COMPRESSOR_WITH_SWAP:
+ vm_config.compressor_is_present = TRUE;
+ vm_config.swap_is_present = TRUE;
+ vm_config.compressor_is_active = TRUE;
+ vm_config.freezer_swap_is_active = TRUE;
+ break;
+
+ case VM_PAGER_NOT_CONFIGURED:
+ break;
+
+ default:
+ printf("unknown compressor mode - %x\n", vm_compressor_mode);
+ break;
+ }
+ if (VM_CONFIG_COMPRESSOR_IS_PRESENT) {
+ vm_compressor_pager_init();
+ }
+
+#if VM_PRESSURE_EVENTS
+ vm_pressure_events_enabled = TRUE;
+#endif /* VM_PRESSURE_EVENTS */
+
+#if CONFIG_PHANTOM_CACHE
+ vm_phantom_cache_init();
+#endif
+#if VM_PAGE_BUCKETS_CHECK
+#if VM_PAGE_FAKE_BUCKETS
+ printf("**** DEBUG: protecting fake buckets [0x%llx:0x%llx]\n",
+ (uint64_t) vm_page_fake_buckets_start,
+ (uint64_t) vm_page_fake_buckets_end);
+ pmap_protect(kernel_pmap,
+ vm_page_fake_buckets_start,
+ vm_page_fake_buckets_end,
+ VM_PROT_READ);
+// *(char *) vm_page_fake_buckets_start = 'x'; /* panic! */
+#endif /* VM_PAGE_FAKE_BUCKETS */
+#endif /* VM_PAGE_BUCKETS_CHECK */
+
+#if VM_OBJECT_TRACKING
+ vm_object_tracking_init();
+#endif /* VM_OBJECT_TRACKING */
+
+ vm_pageout_continue();
+
+ /*
+ * Unreached code!
+ *
+ * The vm_pageout_continue() call above never returns, so the code below is never
+ * executed. We take advantage of this to declare several DTrace VM related probe
+ * points that our kernel doesn't have an analog for. These are probe points that
+ * exist in Solaris and are in the DTrace documentation, so people may have written
+ * scripts that use them. Declaring the probe points here means their scripts will
+ * compile and execute which we want for portability of the scripts, but since this
+ * section of code is never reached, the probe points will simply never fire. Yes,
+ * this is basically a hack. The problem is the DTrace probe points were chosen with
+ * Solaris specific VM events in mind, not portability to different VM implementations.
+ */
+
+ DTRACE_VM2(execfree, int, 1, (uint64_t *), NULL);
+ DTRACE_VM2(execpgin, int, 1, (uint64_t *), NULL);
+ DTRACE_VM2(execpgout, int, 1, (uint64_t *), NULL);
+ DTRACE_VM2(pgswapin, int, 1, (uint64_t *), NULL);
+ DTRACE_VM2(pgswapout, int, 1, (uint64_t *), NULL);
+ DTRACE_VM2(swapin, int, 1, (uint64_t *), NULL);
+ DTRACE_VM2(swapout, int, 1, (uint64_t *), NULL);
+ /*NOTREACHED*/
+}
+
+
+
+kern_return_t
+vm_pageout_internal_start(void)
+{
+ kern_return_t result;
+ host_basic_info_data_t hinfo;
+ vm_offset_t buf, bufsize;
+
+ assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
+
+ mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
+#define BSD_HOST 1
+ host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count);
+
+ assert(hinfo.max_cpus > 0);
+
+#if CONFIG_EMBEDDED
+ vm_pageout_state.vm_compressor_thread_count = 1;
+#else
+ if (hinfo.max_cpus > 4) {
+ vm_pageout_state.vm_compressor_thread_count = 2;
+ } else {
+ vm_pageout_state.vm_compressor_thread_count = 1;
+ }
+#endif
+ PE_parse_boot_argn("vmcomp_threads", &vm_pageout_state.vm_compressor_thread_count,
+ sizeof(vm_pageout_state.vm_compressor_thread_count));
+
+#if __AMP__
+ PE_parse_boot_argn("vmcomp_ecluster", &vm_compressor_ebound, sizeof(vm_compressor_ebound));
+ if (vm_compressor_ebound) {
+ vm_pageout_state.vm_compressor_thread_count = 2;
+ }
+#endif
+ if (vm_pageout_state.vm_compressor_thread_count >= hinfo.max_cpus) {
+ vm_pageout_state.vm_compressor_thread_count = hinfo.max_cpus - 1;
+ }
+ if (vm_pageout_state.vm_compressor_thread_count <= 0) {
+ vm_pageout_state.vm_compressor_thread_count = 1;
+ } else if (vm_pageout_state.vm_compressor_thread_count > MAX_COMPRESSOR_THREAD_COUNT) {
+ vm_pageout_state.vm_compressor_thread_count = MAX_COMPRESSOR_THREAD_COUNT;
+ }
+
+ vm_pageout_queue_internal.pgo_maxlaundry =
+ (vm_pageout_state.vm_compressor_thread_count * 4) * VM_PAGE_LAUNDRY_MAX;
+
+ PE_parse_boot_argn("vmpgoi_maxlaundry",
+ &vm_pageout_queue_internal.pgo_maxlaundry,
+ sizeof(vm_pageout_queue_internal.pgo_maxlaundry));
+
+ bufsize = COMPRESSOR_SCRATCH_BUF_SIZE;
+ if (kernel_memory_allocate(kernel_map, &buf,
+ bufsize * vm_pageout_state.vm_compressor_thread_count,
+ 0, KMA_KOBJECT | KMA_PERMANENT, VM_KERN_MEMORY_COMPRESSOR)) {
+ panic("vm_pageout_internal_start: Unable to allocate %zd bytes",
+ (size_t)(bufsize * vm_pageout_state.vm_compressor_thread_count));
+ }
+
+ for (int i = 0; i < vm_pageout_state.vm_compressor_thread_count; i++) {
+ ciq[i].id = i;
+ ciq[i].q = &vm_pageout_queue_internal;
+ ciq[i].current_chead = NULL;
+ ciq[i].scratch_buf = (char *)(buf + i * bufsize);
+
+ result = kernel_thread_start_priority((thread_continue_t)vm_pageout_iothread_internal,
+ (void *)&ciq[i], BASEPRI_VM,
+ &vm_pageout_state.vm_pageout_internal_iothread);
+
+ if (result == KERN_SUCCESS) {
+ thread_deallocate(vm_pageout_state.vm_pageout_internal_iothread);
+ } else {
+ break;
+ }
+ }
+ return result;
+}
+
+#if CONFIG_IOSCHED
+/*
+ * To support I/O Expedite for compressed files we mark the upls with special flags.
+ * The way decmpfs works is that we create a big upl which marks all the pages needed to
+ * represent the compressed file as busy. We tag this upl with the flag UPL_DECMP_REQ. Decmpfs
+ * then issues smaller I/Os for compressed I/Os, deflates them and puts the data into the pages
+ * being held in the big original UPL. We mark each of these smaller UPLs with the flag
+ * UPL_DECMP_REAL_IO. Any outstanding real I/O UPL is tracked by the big req upl using the
+ * decmp_io_upl field (in the upl structure). This link is protected in the forward direction
+ * by the req upl lock (the reverse link doesnt need synch. since we never inspect this link
+ * unless the real I/O upl is being destroyed).
+ */
+
+
+static void
+upl_set_decmp_info(upl_t upl, upl_t src_upl)
+{
+ assert((src_upl->flags & UPL_DECMP_REQ) != 0);
+
+ upl_lock(src_upl);
+ if (src_upl->decmp_io_upl) {
+ /*
+ * If there is already an alive real I/O UPL, ignore this new UPL.
+ * This case should rarely happen and even if it does, it just means
+ * that we might issue a spurious expedite which the driver is expected
+ * to handle.
+ */
+ upl_unlock(src_upl);
+ return;
+ }
+ src_upl->decmp_io_upl = (void *)upl;
+ src_upl->ref_count++;
+
+ upl->flags |= UPL_DECMP_REAL_IO;
+ upl->decmp_io_upl = (void *)src_upl;
+ upl_unlock(src_upl);
+}
+#endif /* CONFIG_IOSCHED */
+
+#if UPL_DEBUG
+int upl_debug_enabled = 1;
+#else
+int upl_debug_enabled = 0;
+#endif
+
+static upl_t
+upl_create(int type, int flags, upl_size_t size)
+{
+ upl_t upl;
+ vm_size_t page_field_size = 0;
+ int upl_flags = 0;
+ vm_size_t upl_size = sizeof(struct upl);
+
+ assert(page_aligned(size));
+
+ size = round_page_32(size);
+
+ if (type & UPL_CREATE_LITE) {
+ page_field_size = (atop(size) + 7) >> 3;
+ page_field_size = (page_field_size + 3) & 0xFFFFFFFC;
+
+ upl_flags |= UPL_LITE;
+ }
+ if (type & UPL_CREATE_INTERNAL) {
+ upl_size += sizeof(struct upl_page_info) * atop(size);
+
+ upl_flags |= UPL_INTERNAL;
+ }
+ upl = (upl_t)kalloc(upl_size + page_field_size);
+
+ if (page_field_size) {
+ bzero((char *)upl + upl_size, page_field_size);
+ }
+
+ upl->flags = upl_flags | flags;
+ upl->kaddr = (vm_offset_t)0;
+ upl->u_offset = 0;
+ upl->u_size = 0;
+ upl->map_object = NULL;
+ upl->ref_count = 1;
+ upl->ext_ref_count = 0;
+ upl->highest_page = 0;
+ upl_lock_init(upl);
+ upl->vector_upl = NULL;
+ upl->associated_upl = NULL;
+ upl->upl_iodone = NULL;
+#if CONFIG_IOSCHED
+ if (type & UPL_CREATE_IO_TRACKING) {
+ upl->upl_priority = proc_get_effective_thread_policy(current_thread(), TASK_POLICY_IO);
+ }
+
+ upl->upl_reprio_info = 0;
+ upl->decmp_io_upl = 0;
+ if ((type & UPL_CREATE_INTERNAL) && (type & UPL_CREATE_EXPEDITE_SUP)) {
+ /* Only support expedite on internal UPLs */
+ thread_t curthread = current_thread();
+ upl->upl_reprio_info = (uint64_t *)kalloc(sizeof(uint64_t) * atop(size));
+ bzero(upl->upl_reprio_info, (sizeof(uint64_t) * atop(size)));
+ upl->flags |= UPL_EXPEDITE_SUPPORTED;
+ if (curthread->decmp_upl != NULL) {
+ upl_set_decmp_info(upl, curthread->decmp_upl);
+ }
+ }
+#endif
+#if CONFIG_IOSCHED || UPL_DEBUG
+ if ((type & UPL_CREATE_IO_TRACKING) || upl_debug_enabled) {
+ upl->upl_creator = current_thread();
+ upl->uplq.next = 0;
+ upl->uplq.prev = 0;
+ upl->flags |= UPL_TRACKED_BY_OBJECT;
+ }
+#endif
+
+#if UPL_DEBUG
+ upl->ubc_alias1 = 0;
+ upl->ubc_alias2 = 0;
+
+ upl->upl_state = 0;
+ upl->upl_commit_index = 0;
+ bzero(&upl->upl_commit_records[0], sizeof(upl->upl_commit_records));
+
+ (void) OSBacktrace(&upl->upl_create_retaddr[0], UPL_DEBUG_STACK_FRAMES);
+#endif /* UPL_DEBUG */
+
+ return upl;
+}
+
+static void
+upl_destroy(upl_t upl)
+{
+ int page_field_size; /* bit field in word size buf */
+ int size;
+
+// DEBUG4K_UPL("upl %p (u_offset 0x%llx u_size 0x%llx) object %p\n", upl, (uint64_t)upl->u_offset, (uint64_t)upl->u_size, upl->map_object);
+
+ if (upl->ext_ref_count) {
+ panic("upl(%p) ext_ref_count", upl);
+ }
+
+#if CONFIG_IOSCHED
+ if ((upl->flags & UPL_DECMP_REAL_IO) && upl->decmp_io_upl) {
+ upl_t src_upl;
+ src_upl = upl->decmp_io_upl;
+ assert((src_upl->flags & UPL_DECMP_REQ) != 0);
+ upl_lock(src_upl);
+ src_upl->decmp_io_upl = NULL;
+ upl_unlock(src_upl);
+ upl_deallocate(src_upl);
+ }
+#endif /* CONFIG_IOSCHED */
+
+#if CONFIG_IOSCHED || UPL_DEBUG
+ if (((upl->flags & UPL_TRACKED_BY_OBJECT) || upl_debug_enabled) &&
+ !(upl->flags & UPL_VECTOR)) {
+ vm_object_t object;
+
+ if (upl->flags & UPL_SHADOWED) {
+ object = upl->map_object->shadow;
+ } else {
+ object = upl->map_object;
+ }
+
+ vm_object_lock(object);
+ queue_remove(&object->uplq, upl, upl_t, uplq);
+ vm_object_activity_end(object);
+ vm_object_collapse(object, 0, TRUE);
+ vm_object_unlock(object);
+ }
+#endif
+ /*
+ * drop a reference on the map_object whether or
+ * not a pageout object is inserted
+ */
+ if (upl->flags & UPL_SHADOWED) {
+ vm_object_deallocate(upl->map_object);
+ }
+
+ if (upl->flags & UPL_DEVICE_MEMORY) {
+ size = PAGE_SIZE;
+ } else {
+ size = upl_adjusted_size(upl, PAGE_MASK);
+ }
+ page_field_size = 0;
- page_field_size = 0;
if (upl->flags & UPL_LITE) {
- page_field_size = ((upl->size/PAGE_SIZE) + 7) >> 3;
+ page_field_size = ((size / PAGE_SIZE) + 7) >> 3;
page_field_size = (page_field_size + 3) & 0xFFFFFFFC;
}
- if(upl->flags & UPL_INTERNAL) {
+ upl_lock_destroy(upl);
+ upl->vector_upl = (vector_upl_t) 0xfeedbeef;
+
+#if CONFIG_IOSCHED
+ if (upl->flags & UPL_EXPEDITE_SUPPORTED) {
+ kfree(upl->upl_reprio_info, sizeof(uint64_t) * (size / PAGE_SIZE));
+ }
+#endif
+
+ if (upl->flags & UPL_INTERNAL) {
kfree(upl,
- sizeof(struct upl) +
- (sizeof(struct upl_page_info) * (upl->size/PAGE_SIZE))
- + page_field_size);
+ sizeof(struct upl) +
+ (sizeof(struct upl_page_info) * (size / PAGE_SIZE))
+ + page_field_size);
} else {
kfree(upl, sizeof(struct upl) + page_field_size);
}
}
-void uc_upl_dealloc(upl_t upl);
-__private_extern__ void
-uc_upl_dealloc(
- upl_t upl)
+void
+upl_deallocate(upl_t upl)
{
- upl->ref_count -= 1;
- if(upl->ref_count == 0) {
+ upl_lock(upl);
+
+ if (--upl->ref_count == 0) {
+ if (vector_upl_is_valid(upl)) {
+ vector_upl_deallocate(upl);
+ }
+ upl_unlock(upl);
+
+ if (upl->upl_iodone) {
+ upl_callout_iodone(upl);
+ }
+
upl_destroy(upl);
+ } else {
+ upl_unlock(upl);
}
}
+#if CONFIG_IOSCHED
void
-upl_deallocate(
- upl_t upl)
+upl_mark_decmp(upl_t upl)
{
-
- upl->ref_count -= 1;
- if(upl->ref_count == 0) {
- upl_destroy(upl);
+ if (upl->flags & UPL_TRACKED_BY_OBJECT) {
+ upl->flags |= UPL_DECMP_REQ;
+ upl->upl_creator->decmp_upl = (void *)upl;
+ }
+}
+
+void
+upl_unmark_decmp(upl_t upl)
+{
+ if (upl && (upl->flags & UPL_DECMP_REQ)) {
+ upl->upl_creator->decmp_upl = NULL;
+ }
+}
+
+#endif /* CONFIG_IOSCHED */
+
+#define VM_PAGE_Q_BACKING_UP(q) \
+ ((q)->pgo_laundry >= (((q)->pgo_maxlaundry * 8) / 10))
+
+boolean_t must_throttle_writes(void);
+
+boolean_t
+must_throttle_writes()
+{
+ if (VM_PAGE_Q_BACKING_UP(&vm_pageout_queue_external) &&
+ vm_page_pageable_external_count > (AVAILABLE_NON_COMPRESSED_MEMORY * 6) / 10) {
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+#define MIN_DELAYED_WORK_CTX_ALLOCATED (16)
+#define MAX_DELAYED_WORK_CTX_ALLOCATED (512)
+
+int vm_page_delayed_work_ctx_needed = 0;
+zone_t dw_ctx_zone = ZONE_NULL;
+
+void
+vm_page_delayed_work_init_ctx(void)
+{
+ int nelems = 0, elem_size = 0;
+
+ elem_size = sizeof(struct vm_page_delayed_work_ctx);
+
+ dw_ctx_zone = zone_create_ext("delayed-work-ctx", elem_size,
+ ZC_NOGC, ZONE_ID_ANY, ^(zone_t z) {
+ zone_set_exhaustible(z, MAX_DELAYED_WORK_CTX_ALLOCATED * elem_size);
+ });
+
+ nelems = zfill(dw_ctx_zone, MIN_DELAYED_WORK_CTX_ALLOCATED);
+ if (nelems < MIN_DELAYED_WORK_CTX_ALLOCATED) {
+ printf("vm_page_delayed_work_init_ctx: Failed to preallocate minimum delayed work contexts (%d vs %d).\n", nelems, MIN_DELAYED_WORK_CTX_ALLOCATED);
+#if DEVELOPMENT || DEBUG
+ panic("Failed to preallocate minimum delayed work contexts (%d vs %d).\n", nelems, MIN_DELAYED_WORK_CTX_ALLOCATED);
+#endif /* DEVELOPMENT || DEBUG */
+ }
+}
+
+struct vm_page_delayed_work*
+vm_page_delayed_work_get_ctx(void)
+{
+ struct vm_page_delayed_work_ctx * dw_ctx = NULL;
+
+ dw_ctx = (struct vm_page_delayed_work_ctx*) zalloc_noblock(dw_ctx_zone);
+
+ if (dw_ctx) {
+ dw_ctx->delayed_owner = current_thread();
+ } else {
+ vm_page_delayed_work_ctx_needed++;
}
+ return dw_ctx ? dw_ctx->dwp : NULL;
+}
+
+void
+vm_page_delayed_work_finish_ctx(struct vm_page_delayed_work* dwp)
+{
+ struct vm_page_delayed_work_ctx *ldw_ctx;
+
+ ldw_ctx = (struct vm_page_delayed_work_ctx *)dwp;
+ ldw_ctx->delayed_owner = NULL;
+
+ zfree(dw_ctx_zone, ldw_ctx);
}
/*
- * Statistics about UPL enforcement of copy-on-write obligations.
- */
-unsigned long upl_cow = 0;
-unsigned long upl_cow_again = 0;
-unsigned long upl_cow_contiguous = 0;
-unsigned long upl_cow_pages = 0;
-unsigned long upl_cow_again_pages = 0;
-unsigned long upl_cow_contiguous_pages = 0;
-
-/*
- * Routine: vm_object_upl_request
- * Purpose:
+ * Routine: vm_object_upl_request
+ * Purpose:
* Cause the population of a portion of a vm_object.
* Depending on the nature of the request, the pages
* returned may be contain valid data or be uninitialized.
* IMPORTANT NOTE: The caller must still respect the relationship
* between the vm_object and its backing memory object. The
* caller MUST NOT substitute changes in the backing file
- * without first doing a memory_object_lock_request on the
+ * without first doing a memory_object_lock_request on the
* target range unless it is know that the pages are not
* shared with another entity at the pager level.
* Copy_in_to:
* all mapped pages. Where a page does not exist
* map a zero filled one. Leave pages busy in
* the original object. If a page list structure
- * is not specified, this call is a no-op.
+ * is not specified, this call is a no-op.
*
* Note: access of default pager objects has a rather interesting
* twist. The caller of this routine, presumably the file system
* against a default pager backed object. Only the default
* pager will make requests on backing store related vm_objects
* In this way the default pager can maintain the relationship
- * between backing store files (abstract memory objects) and
+ * between backing store files (abstract memory objects) and
* the vm_objects (cache objects), they support.
*
*/
__private_extern__ kern_return_t
vm_object_upl_request(
- vm_object_t object,
- vm_object_offset_t offset,
- upl_size_t size,
- upl_t *upl_ptr,
- upl_page_info_array_t user_page_list,
- unsigned int *page_list_count,
- int cntrl_flags)
+ vm_object_t object,
+ vm_object_offset_t offset,
+ upl_size_t size,
+ upl_t *upl_ptr,
+ upl_page_info_array_t user_page_list,
+ unsigned int *page_list_count,
+ upl_control_flags_t cntrl_flags,
+ vm_tag_t tag)
{
- vm_page_t dst_page = VM_PAGE_NULL;
- vm_object_offset_t dst_offset = offset;
- upl_size_t xfer_size = size;
- boolean_t do_m_lock = FALSE;
- boolean_t dirty;
- boolean_t hw_dirty;
- upl_t upl = NULL;
- unsigned int entry;
-#if MACH_CLUSTER_STATS
- boolean_t encountered_lrp = FALSE;
-#endif
- vm_page_t alias_page = NULL;
- int page_ticket;
- int refmod_state;
- wpl_array_t lite_list = NULL;
- vm_object_t last_copy_object;
-
+ vm_page_t dst_page = VM_PAGE_NULL;
+ vm_object_offset_t dst_offset;
+ upl_size_t xfer_size;
+ unsigned int size_in_pages;
+ boolean_t dirty;
+ boolean_t hw_dirty;
+ upl_t upl = NULL;
+ unsigned int entry;
+ vm_page_t alias_page = NULL;
+ int refmod_state = 0;
+ wpl_array_t lite_list = NULL;
+ vm_object_t last_copy_object;
+ struct vm_page_delayed_work dw_array;
+ struct vm_page_delayed_work *dwp, *dwp_start;
+ bool dwp_finish_ctx = TRUE;
+ int dw_count;
+ int dw_limit;
+ int io_tracking_flag = 0;
+ int grab_options;
+ int page_grab_count = 0;
+ ppnum_t phys_page;
+ pmap_flush_context pmap_flush_context_storage;
+ boolean_t pmap_flushes_delayed = FALSE;
+#if DEVELOPMENT || DEBUG
+ task_t task = current_task();
+#endif /* DEVELOPMENT || DEBUG */
+
+ dwp_start = dwp = NULL;
if (cntrl_flags & ~UPL_VALID_FLAGS) {
/*
*/
return KERN_INVALID_VALUE;
}
+ if ((!object->internal) && (object->paging_offset != 0)) {
+ panic("vm_object_upl_request: external object with non-zero paging offset\n");
+ }
+ if (object->phys_contiguous) {
+ panic("vm_object_upl_request: contiguous object specified\n");
+ }
- page_ticket = (cntrl_flags & UPL_PAGE_TICKET_MASK)
- >> UPL_PAGE_TICKET_SHIFT;
+ assertf(page_aligned(offset) && page_aligned(size),
+ "offset 0x%llx size 0x%x",
+ offset, size);
- if(((size/PAGE_SIZE) > MAX_UPL_TRANSFER) && !object->phys_contiguous) {
- size = MAX_UPL_TRANSFER * PAGE_SIZE;
+ VM_DEBUG_CONSTANT_EVENT(vm_object_upl_request, VM_UPL_REQUEST, DBG_FUNC_START, size, cntrl_flags, 0, 0);
+
+ dw_count = 0;
+ dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT);
+ dwp_start = vm_page_delayed_work_get_ctx();
+ if (dwp_start == NULL) {
+ dwp_start = &dw_array;
+ dw_limit = 1;
+ dwp_finish_ctx = FALSE;
}
- if(cntrl_flags & UPL_SET_INTERNAL)
- if(page_list_count != NULL)
- *page_list_count = MAX_UPL_TRANSFER;
+ dwp = dwp_start;
- if((!object->internal) && (object->paging_offset != 0))
- panic("vm_object_upl_request: vnode object with non-zero paging offset\n");
+ if (size > MAX_UPL_SIZE_BYTES) {
+ size = MAX_UPL_SIZE_BYTES;
+ }
- if((cntrl_flags & UPL_COPYOUT_FROM) && (upl_ptr == NULL)) {
- return KERN_SUCCESS;
+ if ((cntrl_flags & UPL_SET_INTERNAL) && page_list_count != NULL) {
+ *page_list_count = MAX_UPL_SIZE_BYTES >> PAGE_SHIFT;
}
- vm_object_lock(object);
- vm_object_paging_begin(object);
- vm_object_unlock(object);
+#if CONFIG_IOSCHED || UPL_DEBUG
+ if (object->io_tracking || upl_debug_enabled) {
+ io_tracking_flag |= UPL_CREATE_IO_TRACKING;
+ }
+#endif
+#if CONFIG_IOSCHED
+ if (object->io_tracking) {
+ io_tracking_flag |= UPL_CREATE_EXPEDITE_SUP;
+ }
+#endif
- if(upl_ptr) {
- if(cntrl_flags & UPL_SET_INTERNAL) {
- if(cntrl_flags & UPL_SET_LITE) {
- uintptr_t page_field_size;
- upl = upl_create(
- UPL_CREATE_INTERNAL | UPL_CREATE_LITE,
- size);
- user_page_list = (upl_page_info_t *)
- (((uintptr_t)upl) + sizeof(struct upl));
- lite_list = (wpl_array_t)
- (((uintptr_t)user_page_list) +
- ((size/PAGE_SIZE) *
- sizeof(upl_page_info_t)));
- page_field_size = ((size/PAGE_SIZE) + 7) >> 3;
- page_field_size =
- (page_field_size + 3) & 0xFFFFFFFC;
- bzero((char *)lite_list, page_field_size);
- upl->flags =
- UPL_LITE | UPL_INTERNAL;
- } else {
- upl = upl_create(UPL_CREATE_INTERNAL, size);
- user_page_list = (upl_page_info_t *)
- (((uintptr_t)upl) + sizeof(struct upl));
- upl->flags = UPL_INTERNAL;
+ if (cntrl_flags & UPL_SET_INTERNAL) {
+ if (cntrl_flags & UPL_SET_LITE) {
+ upl = upl_create(UPL_CREATE_INTERNAL | UPL_CREATE_LITE | io_tracking_flag, 0, size);
+
+ user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl));
+ lite_list = (wpl_array_t)
+ (((uintptr_t)user_page_list) +
+ ((size / PAGE_SIZE) * sizeof(upl_page_info_t)));
+ if (size == 0) {
+ user_page_list = NULL;
+ lite_list = NULL;
}
} else {
- if(cntrl_flags & UPL_SET_LITE) {
- uintptr_t page_field_size;
- upl = upl_create(UPL_CREATE_LITE, size);
- lite_list = (wpl_array_t)
- (((uintptr_t)upl) + sizeof(struct upl));
- page_field_size = ((size/PAGE_SIZE) + 7) >> 3;
- page_field_size =
- (page_field_size + 3) & 0xFFFFFFFC;
- bzero((char *)lite_list, page_field_size);
- upl->flags = UPL_LITE;
- } else {
- upl = upl_create(UPL_CREATE_EXTERNAL, size);
- upl->flags = 0;
- }
- }
+ upl = upl_create(UPL_CREATE_INTERNAL | io_tracking_flag, 0, size);
- if (object->phys_contiguous) {
- if ((cntrl_flags & UPL_WILL_MODIFY) &&
- object->copy != VM_OBJECT_NULL) {
- /* Honor copy-on-write obligations */
-
- /*
- * XXX FBDP
- * We could still have a race...
- * A is here building the UPL for a write().
- * A pushes the pages to the current copy
- * object.
- * A returns the UPL to the caller.
- * B comes along and establishes another
- * private mapping on this object, inserting
- * a new copy object between the original
- * object and the old copy object.
- * B reads a page and gets the original contents
- * from the original object.
- * A modifies the page in the original object.
- * B reads the page again and sees A's changes,
- * which is wrong...
- *
- * The problem is that the pages are not
- * marked "busy" in the original object, so
- * nothing prevents B from reading it before
- * before A's changes are completed.
- *
- * The "paging_in_progress" might protect us
- * from the insertion of a new copy object
- * though... To be verified.
- */
- vm_object_lock_request(object,
- offset,
- size,
- FALSE,
- MEMORY_OBJECT_COPY_SYNC,
- VM_PROT_NO_CHANGE);
- upl_cow_contiguous++;
- upl_cow_contiguous_pages += size >> PAGE_SHIFT;
- }
-
- upl->map_object = object;
- /* don't need any shadow mappings for this one */
- /* since it is already I/O memory */
- upl->flags |= UPL_DEVICE_MEMORY;
-
-
- /* paging_in_progress protects paging_offset */
- upl->offset = offset + object->paging_offset;
- upl->size = size;
- *upl_ptr = upl;
- if(user_page_list) {
- user_page_list[0].phys_addr =
- (offset + object->shadow_offset)>>PAGE_SHIFT;
- user_page_list[0].device = TRUE;
- }
-
- if(page_list_count != NULL) {
- if (upl->flags & UPL_INTERNAL) {
- *page_list_count = 0;
- } else {
- *page_list_count = 1;
- }
+ user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl));
+ if (size == 0) {
+ user_page_list = NULL;
}
-
- return KERN_SUCCESS;
}
+ } else {
+ if (cntrl_flags & UPL_SET_LITE) {
+ upl = upl_create(UPL_CREATE_EXTERNAL | UPL_CREATE_LITE | io_tracking_flag, 0, size);
- if(user_page_list)
- user_page_list[0].device = FALSE;
-
- if(cntrl_flags & UPL_SET_LITE) {
- upl->map_object = object;
+ lite_list = (wpl_array_t) (((uintptr_t)upl) + sizeof(struct upl));
+ if (size == 0) {
+ lite_list = NULL;
+ }
} else {
- upl->map_object = vm_object_allocate(size);
- /*
- * No neeed to lock the new object: nobody else knows
- * about it yet, so it's all ours so far.
- */
- upl->map_object->shadow = object;
- upl->map_object->pageout = TRUE;
- upl->map_object->can_persist = FALSE;
- upl->map_object->copy_strategy =
- MEMORY_OBJECT_COPY_NONE;
- upl->map_object->shadow_offset = offset;
- upl->map_object->wimg_bits = object->wimg_bits;
+ upl = upl_create(UPL_CREATE_EXTERNAL | io_tracking_flag, 0, size);
}
+ }
+ *upl_ptr = upl;
+ if (user_page_list) {
+ user_page_list[0].device = FALSE;
}
- if (!(cntrl_flags & UPL_SET_LITE)) {
+
+ if (cntrl_flags & UPL_SET_LITE) {
+ upl->map_object = object;
+ } else {
+ upl->map_object = vm_object_allocate(size);
+ /*
+ * No neeed to lock the new object: nobody else knows
+ * about it yet, so it's all ours so far.
+ */
+ upl->map_object->shadow = object;
+ upl->map_object->pageout = TRUE;
+ upl->map_object->can_persist = FALSE;
+ upl->map_object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
+ upl->map_object->vo_shadow_offset = offset;
+ upl->map_object->wimg_bits = object->wimg_bits;
+ assertf(page_aligned(upl->map_object->vo_shadow_offset),
+ "object %p shadow_offset 0x%llx",
+ upl->map_object, upl->map_object->vo_shadow_offset);
+
VM_PAGE_GRAB_FICTITIOUS(alias_page);
- }
- /*
- * ENCRYPTED SWAP:
- * Just mark the UPL as "encrypted" here.
- * We'll actually encrypt the pages later,
- * in upl_encrypt(), when the caller has
- * selected which pages need to go to swap.
- */
- if (cntrl_flags & UPL_ENCRYPT) {
- upl->flags |= UPL_ENCRYPTED;
+ upl->flags |= UPL_SHADOWED;
}
if (cntrl_flags & UPL_FOR_PAGEOUT) {
upl->flags |= UPL_PAGEOUT;
}
+
vm_object_lock(object);
+ vm_object_activity_begin(object);
+
+ grab_options = 0;
+#if CONFIG_SECLUDED_MEMORY
+ if (object->can_grab_secluded) {
+ grab_options |= VM_PAGE_GRAB_SECLUDED;
+ }
+#endif /* CONFIG_SECLUDED_MEMORY */
+
+ /*
+ * we can lock in the paging_offset once paging_in_progress is set
+ */
+ upl->u_size = size;
+ upl->u_offset = offset + object->paging_offset;
- /* we can lock in the paging_offset once paging_in_progress is set */
- if(upl_ptr) {
- upl->size = size;
- upl->offset = offset + object->paging_offset;
- *upl_ptr = upl;
-#ifdef UPL_DEBUG
+#if CONFIG_IOSCHED || UPL_DEBUG
+ if (object->io_tracking || upl_debug_enabled) {
+ vm_object_activity_begin(object);
queue_enter(&object->uplq, upl, upl_t, uplq);
-#endif /* UPL_DEBUG */
+ }
+#endif
+ if ((cntrl_flags & UPL_WILL_MODIFY) && object->copy != VM_OBJECT_NULL) {
+ /*
+ * Honor copy-on-write obligations
+ *
+ * The caller is gathering these pages and
+ * might modify their contents. We need to
+ * make sure that the copy object has its own
+ * private copies of these pages before we let
+ * the caller modify them.
+ */
+ vm_object_update(object,
+ offset,
+ size,
+ NULL,
+ NULL,
+ FALSE, /* should_return */
+ MEMORY_OBJECT_COPY_SYNC,
+ VM_PROT_NO_CHANGE);
+
+ VM_PAGEOUT_DEBUG(upl_cow, 1);
+ VM_PAGEOUT_DEBUG(upl_cow_pages, (size >> PAGE_SHIFT));
+ }
+ /*
+ * remember which copy object we synchronized with
+ */
+ last_copy_object = object->copy;
+ entry = 0;
+
+ xfer_size = size;
+ dst_offset = offset;
+ size_in_pages = size / PAGE_SIZE;
+
+ if (vm_page_free_count > (vm_page_free_target + size_in_pages) ||
+ object->resident_page_count < ((MAX_UPL_SIZE_BYTES * 2) >> PAGE_SHIFT)) {
+ object->scan_collisions = 0;
}
- if ((cntrl_flags & UPL_WILL_MODIFY) &&
- object->copy != VM_OBJECT_NULL) {
- /* Honor copy-on-write obligations */
+ if ((cntrl_flags & UPL_WILL_MODIFY) && must_throttle_writes() == TRUE) {
+ boolean_t isSSD = FALSE;
+
+#if CONFIG_EMBEDDED
+ isSSD = TRUE;
+#else
+ vnode_pager_get_isSSD(object->pager, &isSSD);
+#endif
+ vm_object_unlock(object);
+
+ OSAddAtomic(size_in_pages, &vm_upl_wait_for_pages);
+
+ if (isSSD == TRUE) {
+ delay(1000 * size_in_pages);
+ } else {
+ delay(5000 * size_in_pages);
+ }
+ OSAddAtomic(-size_in_pages, &vm_upl_wait_for_pages);
+
+ vm_object_lock(object);
+ }
- /*
- * The caller is gathering these pages and
- * might modify their contents. We need to
- * make sure that the copy object has its own
- * private copies of these pages before we let
- * the caller modify them.
- */
- vm_object_update(object,
- offset,
- size,
- NULL,
- NULL,
- FALSE, /* should_return */
- MEMORY_OBJECT_COPY_SYNC,
- VM_PROT_NO_CHANGE);
- upl_cow++;
- upl_cow_pages += size >> PAGE_SHIFT;
-
- }
- /* remember which copy object we synchronized with */
- last_copy_object = object->copy;
+ while (xfer_size) {
+ dwp->dw_mask = 0;
- entry = 0;
- if(cntrl_flags & UPL_COPYOUT_FROM) {
- upl->flags |= UPL_PAGE_SYNC_DONE;
+ if ((alias_page == NULL) && !(cntrl_flags & UPL_SET_LITE)) {
+ vm_object_unlock(object);
+ VM_PAGE_GRAB_FICTITIOUS(alias_page);
+ vm_object_lock(object);
+ }
+ if (cntrl_flags & UPL_COPYOUT_FROM) {
+ upl->flags |= UPL_PAGE_SYNC_DONE;
+
+ if (((dst_page = vm_page_lookup(object, dst_offset)) == VM_PAGE_NULL) ||
+ dst_page->vmp_fictitious ||
+ dst_page->vmp_absent ||
+ dst_page->vmp_error ||
+ dst_page->vmp_cleaning ||
+ (VM_PAGE_WIRED(dst_page))) {
+ if (user_page_list) {
+ user_page_list[entry].phys_addr = 0;
+ }
- while (xfer_size) {
- if((alias_page == NULL) &&
- !(cntrl_flags & UPL_SET_LITE)) {
- vm_object_unlock(object);
- VM_PAGE_GRAB_FICTITIOUS(alias_page);
- vm_object_lock(object);
+ goto try_next_page;
}
- if ( ((dst_page = vm_page_lookup(object, dst_offset)) == VM_PAGE_NULL) ||
- dst_page->fictitious ||
- dst_page->absent ||
- dst_page->error ||
- (dst_page->wire_count && !dst_page->pageout) ||
+ phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page);
- ((!dst_page->inactive) && (cntrl_flags & UPL_FOR_PAGEOUT) &&
- (dst_page->page_ticket != page_ticket) &&
- ((dst_page->page_ticket+1) != page_ticket)) ) {
+ /*
+ * grab this up front...
+ * a high percentange of the time we're going to
+ * need the hardware modification state a bit later
+ * anyway... so we can eliminate an extra call into
+ * the pmap layer by grabbing it here and recording it
+ */
+ if (dst_page->vmp_pmapped) {
+ refmod_state = pmap_get_refmod(phys_page);
+ } else {
+ refmod_state = 0;
+ }
- if (user_page_list)
- user_page_list[entry].phys_addr = 0;
- } else {
- /*
- * grab this up front...
- * a high percentange of the time we're going to
- * need the hardware modification state a bit later
- * anyway... so we can eliminate an extra call into
- * the pmap layer by grabbing it here and recording it
+ if ((refmod_state & VM_MEM_REFERENCED) && VM_PAGE_INACTIVE(dst_page)) {
+ /*
+ * page is on inactive list and referenced...
+ * reactivate it now... this gets it out of the
+ * way of vm_pageout_scan which would have to
+ * reactivate it upon tripping over it
*/
- refmod_state = pmap_get_refmod(dst_page->phys_page);
-
- if (cntrl_flags & UPL_RET_ONLY_DIRTY) {
- /*
- * we're only asking for DIRTY pages to be returned
- */
-
- if (dst_page->list_req_pending || !(cntrl_flags & UPL_FOR_PAGEOUT)) {
- /*
- * if we were the page stolen by vm_pageout_scan to be
- * cleaned (as opposed to a buddy being clustered in
- * or this request is not being driven by a PAGEOUT cluster
- * then we only need to check for the page being diry or
- * precious to decide whether to return it
- */
- if (dst_page->dirty || dst_page->precious ||
- (refmod_state & VM_MEM_MODIFIED)) {
- goto check_busy;
- }
- }
+ dwp->dw_mask |= DW_vm_page_activate;
+ }
+ if (cntrl_flags & UPL_RET_ONLY_DIRTY) {
+ /*
+ * we're only asking for DIRTY pages to be returned
+ */
+ if (dst_page->vmp_laundry || !(cntrl_flags & UPL_FOR_PAGEOUT)) {
/*
- * this is a request for a PAGEOUT cluster and this page
- * is merely along for the ride as a 'buddy'... not only
- * does it have to be dirty to be returned, but it also
- * can't have been referenced recently... note that we've
- * already filtered above based on whether this page is
- * currently on the inactive queue or it meets the page
- * ticket (generation count) check
+ * if we were the page stolen by vm_pageout_scan to be
+ * cleaned (as opposed to a buddy being clustered in
+ * or this request is not being driven by a PAGEOUT cluster
+ * then we only need to check for the page being dirty or
+ * precious to decide whether to return it
*/
- if ( !(refmod_state & VM_MEM_REFERENCED) &&
- ((refmod_state & VM_MEM_MODIFIED) ||
- dst_page->dirty || dst_page->precious) ) {
- goto check_busy;
+ if (dst_page->vmp_dirty || dst_page->vmp_precious || (refmod_state & VM_MEM_MODIFIED)) {
+ goto check_busy;
}
- /*
- * if we reach here, we're not to return
- * the page... go on to the next one
- */
- if (user_page_list)
- user_page_list[entry].phys_addr = 0;
- entry++;
- dst_offset += PAGE_SIZE_64;
- xfer_size -= PAGE_SIZE;
- continue;
+ goto dont_return;
}
-check_busy:
- if(dst_page->busy &&
- (!(dst_page->list_req_pending &&
- dst_page->pageout))) {
- if(cntrl_flags & UPL_NOBLOCK) {
- if(user_page_list) {
- user_page_list[entry].phys_addr = 0;
- }
- entry++;
- dst_offset += PAGE_SIZE_64;
- xfer_size -= PAGE_SIZE;
- continue;
- }
+ /*
+ * this is a request for a PAGEOUT cluster and this page
+ * is merely along for the ride as a 'buddy'... not only
+ * does it have to be dirty to be returned, but it also
+ * can't have been referenced recently...
+ */
+ if ((hibernate_cleaning_in_progress == TRUE ||
+ (!((refmod_state & VM_MEM_REFERENCED) || dst_page->vmp_reference) ||
+ (dst_page->vmp_q_state == VM_PAGE_ON_THROTTLED_Q))) &&
+ ((refmod_state & VM_MEM_MODIFIED) || dst_page->vmp_dirty || dst_page->vmp_precious)) {
+ goto check_busy;
+ }
+dont_return:
+ /*
+ * if we reach here, we're not to return
+ * the page... go on to the next one
+ */
+ if (dst_page->vmp_laundry == TRUE) {
/*
- * someone else is playing with the
- * page. We will have to wait.
+ * if we get here, the page is not 'cleaning' (filtered out above).
+ * since it has been referenced, remove it from the laundry
+ * so we don't pay the cost of an I/O to clean a page
+ * we're just going to take back
*/
- PAGE_SLEEP(object, dst_page, THREAD_UNINT);
- continue;
+ vm_page_lockspin_queues();
+
+ vm_pageout_steal_laundry(dst_page, TRUE);
+ vm_page_activate(dst_page);
+
+ vm_page_unlock_queues();
}
- /* Someone else already cleaning the page? */
- if((dst_page->cleaning || dst_page->absent ||
- dst_page->wire_count != 0) &&
- !dst_page->list_req_pending) {
- if(user_page_list) {
- user_page_list[entry].phys_addr = 0;
- }
- entry++;
- dst_offset += PAGE_SIZE_64;
- xfer_size -= PAGE_SIZE;
- continue;
+ if (user_page_list) {
+ user_page_list[entry].phys_addr = 0;
}
- /* eliminate all mappings from the */
- /* original object and its prodigy */
-
- vm_page_lock_queues();
- if (dst_page->pageout_queue == TRUE)
- /*
+ goto try_next_page;
+ }
+check_busy:
+ if (dst_page->vmp_busy) {
+ if (cntrl_flags & UPL_NOBLOCK) {
+ if (user_page_list) {
+ user_page_list[entry].phys_addr = 0;
+ }
+ dwp->dw_mask = 0;
+
+ goto try_next_page;
+ }
+ /*
+ * someone else is playing with the
+ * page. We will have to wait.
+ */
+ PAGE_SLEEP(object, dst_page, THREAD_UNINT);
+
+ continue;
+ }
+ if (dst_page->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q) {
+ vm_page_lockspin_queues();
+
+ if (dst_page->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q) {
+ /*
* we've buddied up a page for a clustered pageout
* that has already been moved to the pageout
* queue by pageout_scan... we need to remove
* it from the queue and drop the laundry count
* on that queue
*/
- vm_pageout_queue_steal(dst_page);
-#if MACH_CLUSTER_STATS
- /* pageout statistics gathering. count */
- /* all the pages we will page out that */
- /* were not counted in the initial */
- /* vm_pageout_scan work */
- if(dst_page->list_req_pending)
- encountered_lrp = TRUE;
- if((dst_page->dirty ||
- (dst_page->object->internal &&
- dst_page->precious)) &&
- (dst_page->list_req_pending
- == FALSE)) {
- if(encountered_lrp) {
- CLUSTER_STAT
- (pages_at_higher_offsets++;)
- } else {
- CLUSTER_STAT
- (pages_at_lower_offsets++;)
- }
+ vm_pageout_throttle_up(dst_page);
}
-#endif
- /* Turn off busy indication on pending */
- /* pageout. Note: we can only get here */
- /* in the request pending case. */
- dst_page->list_req_pending = FALSE;
- dst_page->busy = FALSE;
- dst_page->cleaning = FALSE;
-
- hw_dirty = refmod_state & VM_MEM_MODIFIED;
- dirty = hw_dirty ? TRUE : dst_page->dirty;
-
- if(cntrl_flags & UPL_SET_LITE) {
- int pg_num;
- pg_num = (dst_offset-offset)/PAGE_SIZE;
- lite_list[pg_num>>5] |=
- 1 << (pg_num & 31);
- if (hw_dirty)
- pmap_clear_modify(dst_page->phys_page);
- /*
- * Record that this page has been
- * written out
- */
-#if MACH_PAGEMAP
- vm_external_state_set(
- object->existence_map,
- dst_page->offset);
-#endif /*MACH_PAGEMAP*/
+ vm_page_unlock_queues();
+ }
+ hw_dirty = refmod_state & VM_MEM_MODIFIED;
+ dirty = hw_dirty ? TRUE : dst_page->vmp_dirty;
- /*
- * Mark original page as cleaning
- * in place.
- */
- dst_page->cleaning = TRUE;
- dst_page->dirty = TRUE;
- dst_page->precious = FALSE;
- } else {
- /* use pageclean setup, it is more */
- /* convenient even for the pageout */
- /* cases here */
-
- vm_object_lock(upl->map_object);
- vm_pageclean_setup(dst_page,
- alias_page, upl->map_object,
- size - xfer_size);
- vm_object_unlock(upl->map_object);
-
- alias_page->absent = FALSE;
- alias_page = NULL;
- }
-
- if(!dirty) {
- dst_page->dirty = FALSE;
- dst_page->precious = TRUE;
- }
+ if (phys_page > upl->highest_page) {
+ upl->highest_page = phys_page;
+ }
- if(dst_page->pageout)
- dst_page->busy = TRUE;
-
- if ( (cntrl_flags & UPL_ENCRYPT) ) {
- /*
- * ENCRYPTED SWAP:
- * We want to deny access to the target page
- * because its contents are about to be
- * encrypted and the user would be very
- * confused to see encrypted data instead
- * of their data.
- */
- dst_page->busy = TRUE;
- }
- if ( !(cntrl_flags & UPL_CLEAN_IN_PLACE) ) {
- /*
- * deny access to the target page
- * while it is being worked on
- */
- if ((!dst_page->pageout) &&
- (dst_page->wire_count == 0)) {
- dst_page->busy = TRUE;
- dst_page->pageout = TRUE;
- vm_page_wire(dst_page);
- }
- }
+ assert(!pmap_is_noencrypt(phys_page));
+
+ if (cntrl_flags & UPL_SET_LITE) {
+ unsigned int pg_num;
+
+ pg_num = (unsigned int) ((dst_offset - offset) / PAGE_SIZE);
+ assert(pg_num == (dst_offset - offset) / PAGE_SIZE);
+ lite_list[pg_num >> 5] |= 1U << (pg_num & 31);
- if(user_page_list) {
- user_page_list[entry].phys_addr
- = dst_page->phys_page;
- user_page_list[entry].dirty =
- dst_page->dirty;
- user_page_list[entry].pageout =
- dst_page->pageout;
- user_page_list[entry].absent =
- dst_page->absent;
- user_page_list[entry].precious =
- dst_page->precious;
+ if (hw_dirty) {
+ if (pmap_flushes_delayed == FALSE) {
+ pmap_flush_context_init(&pmap_flush_context_storage);
+ pmap_flushes_delayed = TRUE;
+ }
+ pmap_clear_refmod_options(phys_page,
+ VM_MEM_MODIFIED,
+ PMAP_OPTIONS_NOFLUSH | PMAP_OPTIONS_CLEAR_WRITE,
+ &pmap_flush_context_storage);
}
- vm_page_unlock_queues();
/*
- * ENCRYPTED SWAP:
- * The caller is gathering this page and might
- * access its contents later on. Decrypt the
- * page before adding it to the UPL, so that
- * the caller never sees encrypted data.
+ * Mark original page as cleaning
+ * in place.
*/
- if (! (cntrl_flags & UPL_ENCRYPT) &&
- dst_page->encrypted) {
- assert(dst_page->busy);
-
- vm_page_decrypt(dst_page, 0);
- vm_page_decrypt_for_upl_counter++;
+ dst_page->vmp_cleaning = TRUE;
+ dst_page->vmp_precious = FALSE;
+ } else {
+ /*
+ * use pageclean setup, it is more
+ * convenient even for the pageout
+ * cases here
+ */
+ vm_object_lock(upl->map_object);
+ vm_pageclean_setup(dst_page, alias_page, upl->map_object, size - xfer_size);
+ vm_object_unlock(upl->map_object);
- /*
- * Retry this page, since anything
- * could have changed while we were
- * decrypting.
- */
- continue;
- }
+ alias_page->vmp_absent = FALSE;
+ alias_page = NULL;
}
- entry++;
- dst_offset += PAGE_SIZE_64;
- xfer_size -= PAGE_SIZE;
- }
- } else {
- while (xfer_size) {
- if((alias_page == NULL) &&
- !(cntrl_flags & UPL_SET_LITE)) {
- vm_object_unlock(object);
- VM_PAGE_GRAB_FICTITIOUS(alias_page);
- vm_object_lock(object);
+ if (dirty) {
+ SET_PAGE_DIRTY(dst_page, FALSE);
+ } else {
+ dst_page->vmp_dirty = FALSE;
}
- if ((cntrl_flags & UPL_WILL_MODIFY) &&
- object->copy != last_copy_object) {
- /* Honor copy-on-write obligations */
+ if (!dirty) {
+ dst_page->vmp_precious = TRUE;
+ }
+ if (!(cntrl_flags & UPL_CLEAN_IN_PLACE)) {
+ if (!VM_PAGE_WIRED(dst_page)) {
+ dst_page->vmp_free_when_done = TRUE;
+ }
+ }
+ } else {
+ if ((cntrl_flags & UPL_WILL_MODIFY) && object->copy != last_copy_object) {
/*
+ * Honor copy-on-write obligations
+ *
* The copy object has changed since we
* last synchronized for copy-on-write.
* Another copy object might have been
xfer_size, /* remaining size */
NULL,
NULL,
- FALSE, /* should_return */
+ FALSE, /* should_return */
MEMORY_OBJECT_COPY_SYNC,
VM_PROT_NO_CHANGE);
- upl_cow_again++;
- upl_cow_again_pages +=
- xfer_size >> PAGE_SHIFT;
+
+ VM_PAGEOUT_DEBUG(upl_cow_again, 1);
+ VM_PAGEOUT_DEBUG(upl_cow_again_pages, (xfer_size >> PAGE_SHIFT));
}
- /* remember the copy object we synced with */
+ /*
+ * remember the copy object we synced with
+ */
last_copy_object = object->copy;
}
-
dst_page = vm_page_lookup(object, dst_offset);
-
- if(dst_page != VM_PAGE_NULL) {
- if((cntrl_flags & UPL_RET_ONLY_ABSENT) &&
- !((dst_page->list_req_pending)
- && (dst_page->absent))) {
- /* we are doing extended range */
- /* requests. we want to grab */
- /* pages around some which are */
- /* already present. */
- if(user_page_list) {
- user_page_list[entry].phys_addr = 0;
+
+ if (dst_page != VM_PAGE_NULL) {
+ if ((cntrl_flags & UPL_RET_ONLY_ABSENT)) {
+ /*
+ * skip over pages already present in the cache
+ */
+ if (user_page_list) {
+ user_page_list[entry].phys_addr = 0;
}
- entry++;
- dst_offset += PAGE_SIZE_64;
- xfer_size -= PAGE_SIZE;
- continue;
+
+ goto try_next_page;
}
- if((dst_page->cleaning) &&
- !(dst_page->list_req_pending)) {
- /*someone else is writing to the */
- /* page. We will have to wait. */
- PAGE_SLEEP(object,dst_page,THREAD_UNINT);
- continue;
+ if (dst_page->vmp_fictitious) {
+ panic("need corner case for fictitious page");
}
- if ((dst_page->fictitious &&
- dst_page->list_req_pending)) {
- /* dump the fictitious page */
- dst_page->list_req_pending = FALSE;
- dst_page->clustered = FALSE;
-
- vm_page_lock_queues();
- vm_page_free(dst_page);
- vm_page_unlock_queues();
- dst_page = NULL;
- } else if ((dst_page->absent &&
- dst_page->list_req_pending)) {
- /* the default_pager case */
- dst_page->list_req_pending = FALSE;
- dst_page->busy = FALSE;
+ if (dst_page->vmp_busy || dst_page->vmp_cleaning) {
+ /*
+ * someone else is playing with the
+ * page. We will have to wait.
+ */
+ PAGE_SLEEP(object, dst_page, THREAD_UNINT);
+
+ continue;
}
- }
- if(dst_page == VM_PAGE_NULL) {
- if(object->private) {
- /*
- * This is a nasty wrinkle for users
- * of upl who encounter device or
- * private memory however, it is
+ if (dst_page->vmp_laundry) {
+ vm_pageout_steal_laundry(dst_page, FALSE);
+ }
+ } else {
+ if (object->private) {
+ /*
+ * This is a nasty wrinkle for users
+ * of upl who encounter device or
+ * private memory however, it is
* unavoidable, only a fault can
- * reslove the actual backing
+ * resolve the actual backing
* physical page by asking the
* backing device.
*/
- if(user_page_list) {
+ if (user_page_list) {
user_page_list[entry].phys_addr = 0;
}
- entry++;
- dst_offset += PAGE_SIZE_64;
- xfer_size -= PAGE_SIZE;
- continue;
+
+ goto try_next_page;
+ }
+ if (object->scan_collisions) {
+ /*
+ * the pageout_scan thread is trying to steal
+ * pages from this object, but has run into our
+ * lock... grab 2 pages from the head of the object...
+ * the first is freed on behalf of pageout_scan, the
+ * 2nd is for our own use... we use vm_object_page_grab
+ * in both cases to avoid taking pages from the free
+ * list since we are under memory pressure and our
+ * lock on this object is getting in the way of
+ * relieving it
+ */
+ dst_page = vm_object_page_grab(object);
+
+ if (dst_page != VM_PAGE_NULL) {
+ vm_page_release(dst_page,
+ FALSE);
+ }
+
+ dst_page = vm_object_page_grab(object);
+ }
+ if (dst_page == VM_PAGE_NULL) {
+ /*
+ * need to allocate a page
+ */
+ dst_page = vm_page_grab_options(grab_options);
+ if (dst_page != VM_PAGE_NULL) {
+ page_grab_count++;
+ }
}
- /* need to allocate a page */
- dst_page = vm_page_alloc(object, dst_offset);
if (dst_page == VM_PAGE_NULL) {
+ if ((cntrl_flags & (UPL_RET_ONLY_ABSENT | UPL_NOBLOCK)) == (UPL_RET_ONLY_ABSENT | UPL_NOBLOCK)) {
+ /*
+ * we don't want to stall waiting for pages to come onto the free list
+ * while we're already holding absent pages in this UPL
+ * the caller will deal with the empty slots
+ */
+ if (user_page_list) {
+ user_page_list[entry].phys_addr = 0;
+ }
+
+ goto try_next_page;
+ }
+ /*
+ * no pages available... wait
+ * then try again for the same
+ * offset...
+ */
vm_object_unlock(object);
+
+ OSAddAtomic(size_in_pages, &vm_upl_wait_for_pages);
+
+ VM_DEBUG_EVENT(vm_upl_page_wait, VM_UPL_PAGE_WAIT, DBG_FUNC_START, vm_upl_wait_for_pages, 0, 0, 0);
+
VM_PAGE_WAIT();
+ OSAddAtomic(-size_in_pages, &vm_upl_wait_for_pages);
+
+ VM_DEBUG_EVENT(vm_upl_page_wait, VM_UPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, 0);
+
vm_object_lock(object);
+
continue;
}
- dst_page->busy = FALSE;
-#if 0
- if(cntrl_flags & UPL_NO_SYNC) {
- dst_page->page_lock = 0;
- dst_page->unlock_request = 0;
- }
-#endif
- if(cntrl_flags & UPL_RET_ONLY_ABSENT) {
- /*
+ vm_page_insert(dst_page, object, dst_offset);
+
+ dst_page->vmp_absent = TRUE;
+ dst_page->vmp_busy = FALSE;
+
+ if (cntrl_flags & UPL_RET_ONLY_ABSENT) {
+ /*
* if UPL_RET_ONLY_ABSENT was specified,
* than we're definitely setting up a
- * upl for a clustered read/pagein
+ * upl for a clustered read/pagein
* operation... mark the pages as clustered
- * so vm_fault can correctly attribute them
- * to the 'pagein' bucket the first time
- * a fault happens on them
+ * so upl_commit_range can put them on the
+ * speculative list
*/
- dst_page->clustered = TRUE;
+ dst_page->vmp_clustered = TRUE;
+
+ if (!(cntrl_flags & UPL_FILE_IO)) {
+ VM_STAT_INCR(pageins);
+ }
}
- dst_page->absent = TRUE;
- object->absent_count++;
}
-#if 1
- if(cntrl_flags & UPL_NO_SYNC) {
- dst_page->page_lock = 0;
- dst_page->unlock_request = 0;
+ phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page);
+
+ dst_page->vmp_overwriting = TRUE;
+
+ if (dst_page->vmp_pmapped) {
+ if (!(cntrl_flags & UPL_FILE_IO)) {
+ /*
+ * eliminate all mappings from the
+ * original object and its prodigy
+ */
+ refmod_state = pmap_disconnect(phys_page);
+ } else {
+ refmod_state = pmap_get_refmod(phys_page);
+ }
+ } else {
+ refmod_state = 0;
}
-#endif /* 1 */
- /*
- * ENCRYPTED SWAP:
- */
- if (cntrl_flags & UPL_ENCRYPT) {
+ hw_dirty = refmod_state & VM_MEM_MODIFIED;
+ dirty = hw_dirty ? TRUE : dst_page->vmp_dirty;
+
+ if (cntrl_flags & UPL_SET_LITE) {
+ unsigned int pg_num;
+
+ pg_num = (unsigned int) ((dst_offset - offset) / PAGE_SIZE);
+ assert(pg_num == (dst_offset - offset) / PAGE_SIZE);
+ lite_list[pg_num >> 5] |= 1U << (pg_num & 31);
+
+ if (hw_dirty) {
+ pmap_clear_modify(phys_page);
+ }
+
/*
- * The page is going to be encrypted when we
- * get it from the pager, so mark it so.
+ * Mark original page as cleaning
+ * in place.
*/
- dst_page->encrypted = TRUE;
+ dst_page->vmp_cleaning = TRUE;
+ dst_page->vmp_precious = FALSE;
} else {
/*
- * Otherwise, the page will not contain
- * encrypted data.
+ * use pageclean setup, it is more
+ * convenient even for the pageout
+ * cases here
*/
- dst_page->encrypted = FALSE;
- }
+ vm_object_lock(upl->map_object);
+ vm_pageclean_setup(dst_page, alias_page, upl->map_object, size - xfer_size);
+ vm_object_unlock(upl->map_object);
- dst_page->overwriting = TRUE;
- if(dst_page->fictitious) {
- panic("need corner case for fictitious page");
- }
- if(dst_page->page_lock) {
- do_m_lock = TRUE;
+ alias_page->vmp_absent = FALSE;
+ alias_page = NULL;
}
- if(upl_ptr) {
- /* eliminate all mappings from the */
- /* original object and its prodigy */
-
- if(dst_page->busy) {
- /*someone else is playing with the */
- /* page. We will have to wait. */
- PAGE_SLEEP(object, dst_page, THREAD_UNINT);
- continue;
+ if (cntrl_flags & UPL_REQUEST_SET_DIRTY) {
+ upl->flags &= ~UPL_CLEAR_DIRTY;
+ upl->flags |= UPL_SET_DIRTY;
+ dirty = TRUE;
+ /*
+ * Page belonging to a code-signed object is about to
+ * be written. Mark it tainted and disconnect it from
+ * all pmaps so processes have to fault it back in and
+ * deal with the tainted bit.
+ */
+ if (object->code_signed && dst_page->vmp_cs_tainted != VMP_CS_ALL_TRUE) {
+ dst_page->vmp_cs_tainted = VMP_CS_ALL_TRUE;
+ vm_page_upl_tainted++;
+ if (dst_page->vmp_pmapped) {
+ refmod_state = pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(dst_page));
+ if (refmod_state & VM_MEM_REFERENCED) {
+ dst_page->vmp_reference = TRUE;
+ }
+ }
}
- vm_page_lock_queues();
-
- if( !(cntrl_flags & UPL_FILE_IO))
- hw_dirty = pmap_disconnect(dst_page->phys_page) & VM_MEM_MODIFIED;
- else
- hw_dirty = pmap_get_refmod(dst_page->phys_page) & VM_MEM_MODIFIED;
- dirty = hw_dirty ? TRUE : dst_page->dirty;
-
- if(cntrl_flags & UPL_SET_LITE) {
- int pg_num;
- pg_num = (dst_offset-offset)/PAGE_SIZE;
- lite_list[pg_num>>5] |=
- 1 << (pg_num & 31);
- if (hw_dirty)
- pmap_clear_modify(dst_page->phys_page);
- /*
- * Record that this page has been
- * written out
- */
-#if MACH_PAGEMAP
- vm_external_state_set(
- object->existence_map,
- dst_page->offset);
-#endif /*MACH_PAGEMAP*/
+ } else if (cntrl_flags & UPL_CLEAN_IN_PLACE) {
+ /*
+ * clean in place for read implies
+ * that a write will be done on all
+ * the pages that are dirty before
+ * a upl commit is done. The caller
+ * is obligated to preserve the
+ * contents of all pages marked dirty
+ */
+ upl->flags |= UPL_CLEAR_DIRTY;
+ }
+ dst_page->vmp_dirty = dirty;
- /*
- * Mark original page as cleaning
- * in place.
- */
- dst_page->cleaning = TRUE;
- dst_page->dirty = TRUE;
- dst_page->precious = FALSE;
- } else {
- /* use pageclean setup, it is more */
- /* convenient even for the pageout */
- /* cases here */
- vm_object_lock(upl->map_object);
- vm_pageclean_setup(dst_page,
- alias_page, upl->map_object,
- size - xfer_size);
- vm_object_unlock(upl->map_object);
-
- alias_page->absent = FALSE;
- alias_page = NULL;
- }
+ if (!dirty) {
+ dst_page->vmp_precious = TRUE;
+ }
- if(cntrl_flags & UPL_CLEAN_IN_PLACE) {
- /* clean in place for read implies */
- /* that a write will be done on all */
- /* the pages that are dirty before */
- /* a upl commit is done. The caller */
- /* is obligated to preserve the */
- /* contents of all pages marked */
- /* dirty. */
- upl->flags |= UPL_CLEAR_DIRTY;
- }
+ if (!VM_PAGE_WIRED(dst_page)) {
+ /*
+ * deny access to the target page while
+ * it is being worked on
+ */
+ dst_page->vmp_busy = TRUE;
+ } else {
+ dwp->dw_mask |= DW_vm_page_wire;
+ }
- if(!dirty) {
- dst_page->dirty = FALSE;
- dst_page->precious = TRUE;
- }
-
- if (dst_page->wire_count == 0) {
- /* deny access to the target page while */
- /* it is being worked on */
- dst_page->busy = TRUE;
- } else {
- vm_page_wire(dst_page);
- }
- if(cntrl_flags & UPL_RET_ONLY_ABSENT) {
- /*
- * expect the page not to be used
- * since it's coming in as part
- * of a cluster and could be
- * speculative... pages that
- * are 'consumed' will get a
- * hardware reference
- */
- dst_page->reference = FALSE;
+ /*
+ * We might be about to satisfy a fault which has been
+ * requested. So no need for the "restart" bit.
+ */
+ dst_page->vmp_restart = FALSE;
+ if (!dst_page->vmp_absent && !(cntrl_flags & UPL_WILL_MODIFY)) {
+ /*
+ * expect the page to be used
+ */
+ dwp->dw_mask |= DW_set_reference;
+ }
+ if (cntrl_flags & UPL_PRECIOUS) {
+ if (object->internal) {
+ SET_PAGE_DIRTY(dst_page, FALSE);
+ dst_page->vmp_precious = FALSE;
} else {
- /*
- * expect the page to be used
- */
- dst_page->reference = TRUE;
- }
- dst_page->precious =
- (cntrl_flags & UPL_PRECIOUS)
- ? TRUE : FALSE;
- if(user_page_list) {
- user_page_list[entry].phys_addr
- = dst_page->phys_page;
- user_page_list[entry].dirty =
- dst_page->dirty;
- user_page_list[entry].pageout =
- dst_page->pageout;
- user_page_list[entry].absent =
- dst_page->absent;
- user_page_list[entry].precious =
- dst_page->precious;
+ dst_page->vmp_precious = TRUE;
}
- vm_page_unlock_queues();
+ } else {
+ dst_page->vmp_precious = FALSE;
}
- entry++;
- dst_offset += PAGE_SIZE_64;
- xfer_size -= PAGE_SIZE;
}
- }
-
- if (upl->flags & UPL_INTERNAL) {
- if(page_list_count != NULL)
- *page_list_count = 0;
- } else if (*page_list_count > entry) {
- if(page_list_count != NULL)
- *page_list_count = entry;
- }
+ if (dst_page->vmp_busy) {
+ upl->flags |= UPL_HAS_BUSY;
+ }
- if(alias_page != NULL) {
- vm_page_lock_queues();
- vm_page_free(alias_page);
- vm_page_unlock_queues();
- }
+ if (phys_page > upl->highest_page) {
+ upl->highest_page = phys_page;
+ }
+ assert(!pmap_is_noencrypt(phys_page));
+ if (user_page_list) {
+ user_page_list[entry].phys_addr = phys_page;
+ user_page_list[entry].free_when_done = dst_page->vmp_free_when_done;
+ user_page_list[entry].absent = dst_page->vmp_absent;
+ user_page_list[entry].dirty = dst_page->vmp_dirty;
+ user_page_list[entry].precious = dst_page->vmp_precious;
+ user_page_list[entry].device = FALSE;
+ user_page_list[entry].needed = FALSE;
+ if (dst_page->vmp_clustered == TRUE) {
+ user_page_list[entry].speculative = (dst_page->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q) ? TRUE : FALSE;
+ } else {
+ user_page_list[entry].speculative = FALSE;
+ }
+ user_page_list[entry].cs_validated = dst_page->vmp_cs_validated;
+ user_page_list[entry].cs_tainted = dst_page->vmp_cs_tainted;
+ user_page_list[entry].cs_nx = dst_page->vmp_cs_nx;
+ user_page_list[entry].mark = FALSE;
+ }
+ /*
+ * if UPL_RET_ONLY_ABSENT is set, then
+ * we are working with a fresh page and we've
+ * just set the clustered flag on it to
+ * indicate that it was drug in as part of a
+ * speculative cluster... so leave it alone
+ */
+ if (!(cntrl_flags & UPL_RET_ONLY_ABSENT)) {
+ /*
+ * someone is explicitly grabbing this page...
+ * update clustered and speculative state
+ *
+ */
+ if (dst_page->vmp_clustered) {
+ VM_PAGE_CONSUME_CLUSTERED(dst_page);
+ }
+ }
+try_next_page:
+ if (dwp->dw_mask) {
+ if (dwp->dw_mask & DW_vm_page_activate) {
+ VM_STAT_INCR(reactivations);
+ }
- if(do_m_lock) {
- vm_prot_t access_required;
- /* call back all associated pages from other users of the pager */
- /* all future updates will be on data which is based on the */
- /* changes we are going to make here. Note: it is assumed that */
- /* we already hold copies of the data so we will not be seeing */
- /* an avalanche of incoming data from the pager */
- access_required = (cntrl_flags & UPL_COPYOUT_FROM)
- ? VM_PROT_READ : VM_PROT_WRITE;
- while (TRUE) {
- kern_return_t rc;
+ VM_PAGE_ADD_DELAYED_WORK(dwp, dst_page, dw_count);
- if(!object->pager_ready) {
- wait_result_t wait_result;
+ if (dw_count >= dw_limit) {
+ vm_page_do_delayed_work(object, tag, dwp_start, dw_count);
- wait_result = vm_object_sleep(object,
- VM_OBJECT_EVENT_PAGER_READY,
- THREAD_UNINT);
- if (wait_result != THREAD_AWAKENED) {
- vm_object_unlock(object);
- return KERN_FAILURE;
- }
- continue;
+ dwp = dwp_start;
+ dw_count = 0;
+ }
}
+ entry++;
+ dst_offset += PAGE_SIZE_64;
+ xfer_size -= PAGE_SIZE;
+ }
+ if (dw_count) {
+ vm_page_do_delayed_work(object, tag, dwp_start, dw_count);
+ dwp = dwp_start;
+ dw_count = 0;
+ }
- vm_object_unlock(object);
- rc = memory_object_data_unlock(
- object->pager,
- dst_offset + object->paging_offset,
- size,
- access_required);
- if (rc != KERN_SUCCESS && rc != MACH_SEND_INTERRUPTED)
- return KERN_FAILURE;
- vm_object_lock(object);
+ if (alias_page != NULL) {
+ VM_PAGE_FREE(alias_page);
+ }
+ if (pmap_flushes_delayed == TRUE) {
+ pmap_flush(&pmap_flush_context_storage);
+ }
- if (rc == KERN_SUCCESS)
- break;
- }
-
- /* lets wait on the last page requested */
- /* NOTE: we will have to update lock completed routine to signal */
- if(dst_page != VM_PAGE_NULL &&
- (access_required & dst_page->page_lock) != access_required) {
- PAGE_ASSERT_WAIT(dst_page, THREAD_UNINT);
- vm_object_unlock(object);
- thread_block(THREAD_CONTINUE_NULL);
- return KERN_SUCCESS;
- }
+ if (page_list_count != NULL) {
+ if (upl->flags & UPL_INTERNAL) {
+ *page_list_count = 0;
+ } else if (*page_list_count > entry) {
+ *page_list_count = entry;
+ }
}
-
+#if UPL_DEBUG
+ upl->upl_state = 1;
+#endif
vm_object_unlock(object);
- return KERN_SUCCESS;
-}
-
-/* JMM - Backward compatability for now */
-kern_return_t
-vm_fault_list_request( /* forward */
- memory_object_control_t control,
- vm_object_offset_t offset,
- upl_size_t size,
- upl_t *upl_ptr,
- upl_page_info_t **user_page_list_ptr,
- int page_list_count,
- int cntrl_flags);
-kern_return_t
-vm_fault_list_request(
- memory_object_control_t control,
- vm_object_offset_t offset,
- upl_size_t size,
- upl_t *upl_ptr,
- upl_page_info_t **user_page_list_ptr,
- int page_list_count,
- int cntrl_flags)
-{
- int local_list_count;
- upl_page_info_t *user_page_list;
- kern_return_t kr;
- if (user_page_list_ptr != NULL) {
- local_list_count = page_list_count;
- user_page_list = *user_page_list_ptr;
- } else {
- local_list_count = 0;
- user_page_list = NULL;
- }
- kr = memory_object_upl_request(control,
- offset,
- size,
- upl_ptr,
- user_page_list,
- &local_list_count,
- cntrl_flags);
-
- if(kr != KERN_SUCCESS)
- return kr;
+ VM_DEBUG_CONSTANT_EVENT(vm_object_upl_request, VM_UPL_REQUEST, DBG_FUNC_END, page_grab_count, 0, 0, 0);
+#if DEVELOPMENT || DEBUG
+ if (task != NULL) {
+ ledger_credit(task->ledger, task_ledgers.pages_grabbed_upl, page_grab_count);
+ }
+#endif /* DEVELOPMENT || DEBUG */
- if ((user_page_list_ptr != NULL) && (cntrl_flags & UPL_INTERNAL)) {
- *user_page_list_ptr = UPL_GET_INTERNAL_PAGE_LIST(*upl_ptr);
+ if (dwp_start && dwp_finish_ctx) {
+ vm_page_delayed_work_finish_ctx(dwp_start);
+ dwp_start = dwp = NULL;
}
return KERN_SUCCESS;
}
-
-
-/*
+/*
* Routine: vm_object_super_upl_request
- * Purpose:
+ * Purpose:
* Cause the population of a portion of a vm_object
* in much the same way as memory_object_upl_request.
* Depending on the nature of the request, the pages
__private_extern__ kern_return_t
vm_object_super_upl_request(
vm_object_t object,
- vm_object_offset_t offset,
- upl_size_t size,
- upl_size_t super_cluster,
- upl_t *upl,
- upl_page_info_t *user_page_list,
- unsigned int *page_list_count,
- int cntrl_flags)
+ vm_object_offset_t offset,
+ upl_size_t size,
+ upl_size_t super_cluster,
+ upl_t *upl,
+ upl_page_info_t *user_page_list,
+ unsigned int *page_list_count,
+ upl_control_flags_t cntrl_flags,
+ vm_tag_t tag)
{
- vm_page_t target_page;
- int ticket;
-
-
- if(object->paging_offset > offset)
+ if (object->paging_offset > offset || ((cntrl_flags & UPL_VECTOR) == UPL_VECTOR)) {
return KERN_FAILURE;
+ }
assert(object->paging_in_progress);
offset = offset - object->paging_offset;
- if(cntrl_flags & UPL_FOR_PAGEOUT) {
-
- vm_object_lock(object);
-
- if((target_page = vm_page_lookup(object, offset))
- != VM_PAGE_NULL) {
- ticket = target_page->page_ticket;
- cntrl_flags = cntrl_flags & ~(int)UPL_PAGE_TICKET_MASK;
- cntrl_flags = cntrl_flags |
- ((ticket << UPL_PAGE_TICKET_SHIFT)
- & UPL_PAGE_TICKET_MASK);
- }
- vm_object_unlock(object);
- }
-
if (super_cluster > size) {
-
- vm_object_offset_t base_offset;
- upl_size_t super_size;
-
- base_offset = (offset &
- ~((vm_object_offset_t) super_cluster - 1));
- super_size = (offset+size) > (base_offset + super_cluster) ?
- super_cluster<<1 : super_cluster;
- super_size = ((base_offset + super_size) > object->size) ?
- (object->size - base_offset) : super_size;
- if(offset > (base_offset + super_size))
- panic("vm_object_super_upl_request: Missed target pageout"
- " %#llx,%#llx, %#x, %#x, %#x, %#llx\n",
- offset, base_offset, super_size, super_cluster,
- size, object->paging_offset);
+ vm_object_offset_t base_offset;
+ upl_size_t super_size;
+ vm_object_size_t super_size_64;
+
+ base_offset = (offset & ~((vm_object_offset_t) super_cluster - 1));
+ super_size = (offset + size) > (base_offset + super_cluster) ? super_cluster << 1 : super_cluster;
+ super_size_64 = ((base_offset + super_size) > object->vo_size) ? (object->vo_size - base_offset) : super_size;
+ super_size = (upl_size_t) super_size_64;
+ assert(super_size == super_size_64);
+
+ if (offset > (base_offset + super_size)) {
+ panic("vm_object_super_upl_request: Missed target pageout"
+ " %#llx,%#llx, %#x, %#x, %#x, %#llx\n",
+ offset, base_offset, super_size, super_cluster,
+ size, object->paging_offset);
+ }
/*
* apparently there is a case where the vm requests a
* page to be written out who's offset is beyond the
* object size
*/
- if((offset + size) > (base_offset + super_size))
- super_size = (offset + size) - base_offset;
+ if ((offset + size) > (base_offset + super_size)) {
+ super_size_64 = (offset + size) - base_offset;
+ super_size = (upl_size_t) super_size_64;
+ assert(super_size == super_size_64);
+ }
offset = base_offset;
size = super_size;
}
- return vm_object_upl_request(object, offset, size,
- upl, user_page_list, page_list_count,
- cntrl_flags);
+ return vm_object_upl_request(object, offset, size, upl, user_page_list, page_list_count, cntrl_flags, tag);
}
-
+int cs_executable_create_upl = 0;
+extern int proc_selfpid(void);
+extern char *proc_name_address(void *p);
+
kern_return_t
vm_map_create_upl(
- vm_map_t map,
- vm_map_address_t offset,
- upl_size_t *upl_size,
- upl_t *upl,
- upl_page_info_array_t page_list,
- unsigned int *count,
- int *flags)
+ vm_map_t map,
+ vm_map_address_t offset,
+ upl_size_t *upl_size,
+ upl_t *upl,
+ upl_page_info_array_t page_list,
+ unsigned int *count,
+ upl_control_flags_t *flags,
+ vm_tag_t tag)
{
- vm_map_entry_t entry;
- int caller_flags;
- int force_data_sync;
- int sync_cow_data;
- vm_object_t local_object;
- vm_map_offset_t local_offset;
- vm_map_offset_t local_start;
- kern_return_t ret;
+ vm_map_entry_t entry;
+ upl_control_flags_t caller_flags;
+ int force_data_sync;
+ int sync_cow_data;
+ vm_object_t local_object;
+ vm_map_offset_t local_offset;
+ vm_map_offset_t local_start;
+ kern_return_t ret;
+ vm_map_address_t original_offset;
+ vm_map_size_t original_size, adjusted_size;
+ vm_map_offset_t local_entry_start;
+ vm_object_offset_t local_entry_offset;
+ vm_object_offset_t offset_in_mapped_page;
+ boolean_t release_map = FALSE;
+
+start_with_map:
+
+ original_offset = offset;
+ original_size = *upl_size;
+ adjusted_size = original_size;
caller_flags = *flags;
* For forward compatibility's sake,
* reject any unknown flag.
*/
- return KERN_INVALID_VALUE;
+ ret = KERN_INVALID_VALUE;
+ goto done;
}
-
force_data_sync = (caller_flags & UPL_FORCE_DATA_SYNC);
sync_cow_data = !(caller_flags & UPL_COPYOUT_FROM);
- if(upl == NULL)
- return KERN_INVALID_ARGUMENT;
-
+ if (upl == NULL) {
+ ret = KERN_INVALID_ARGUMENT;
+ goto done;
+ }
REDISCOVER_ENTRY:
- vm_map_lock(map);
- if (vm_map_lookup_entry(map, offset, &entry)) {
- if (entry->object.vm_object == VM_OBJECT_NULL ||
- !entry->object.vm_object->phys_contiguous) {
- if((*upl_size/page_size) > MAX_UPL_TRANSFER) {
- *upl_size = MAX_UPL_TRANSFER * page_size;
+ vm_map_lock_read(map);
+
+ if (!vm_map_lookup_entry(map, offset, &entry)) {
+ vm_map_unlock_read(map);
+ ret = KERN_FAILURE;
+ goto done;
+ }
+
+ local_entry_start = entry->vme_start;
+ local_entry_offset = VME_OFFSET(entry);
+
+ if (VM_MAP_PAGE_SHIFT(map) < PAGE_SHIFT) {
+ DEBUG4K_UPL("map %p (%d) offset 0x%llx size 0x%x flags 0x%llx\n", map, VM_MAP_PAGE_SHIFT(map), (uint64_t)offset, *upl_size, *flags);
+ }
+
+ if (entry->vme_end - original_offset < adjusted_size) {
+ adjusted_size = entry->vme_end - original_offset;
+ assert(adjusted_size > 0);
+ *upl_size = (upl_size_t) adjusted_size;
+ assert(*upl_size == adjusted_size);
+ }
+
+ if (caller_flags & UPL_QUERY_OBJECT_TYPE) {
+ *flags = 0;
+
+ if (!entry->is_sub_map &&
+ VME_OBJECT(entry) != VM_OBJECT_NULL) {
+ if (VME_OBJECT(entry)->private) {
+ *flags = UPL_DEV_MEMORY;
+ }
+
+ if (VME_OBJECT(entry)->phys_contiguous) {
+ *flags |= UPL_PHYS_CONTIG;
}
}
- if((entry->vme_end - offset) < *upl_size) {
- *upl_size = entry->vme_end - offset;
+ vm_map_unlock_read(map);
+ ret = KERN_SUCCESS;
+ goto done;
+ }
+
+ offset_in_mapped_page = 0;
+ if (VM_MAP_PAGE_SIZE(map) < PAGE_SIZE) {
+ offset = vm_map_trunc_page(original_offset, VM_MAP_PAGE_MASK(map));
+ *upl_size = (upl_size_t)
+ (vm_map_round_page(original_offset + adjusted_size,
+ VM_MAP_PAGE_MASK(map))
+ - offset);
+
+ offset_in_mapped_page = original_offset - offset;
+ assert(offset_in_mapped_page < VM_MAP_PAGE_SIZE(map));
+
+ DEBUG4K_UPL("map %p (%d) offset 0x%llx size 0x%llx flags 0x%llx -> offset 0x%llx adjusted_size 0x%llx *upl_size 0x%x offset_in_mapped_page 0x%llx\n", map, VM_MAP_PAGE_SHIFT(map), (uint64_t)original_offset, (uint64_t)original_size, *flags, (uint64_t)offset, (uint64_t)adjusted_size, *upl_size, offset_in_mapped_page);
+ }
+
+ if (VME_OBJECT(entry) == VM_OBJECT_NULL ||
+ !VME_OBJECT(entry)->phys_contiguous) {
+ if (*upl_size > MAX_UPL_SIZE_BYTES) {
+ *upl_size = MAX_UPL_SIZE_BYTES;
}
- if (caller_flags & UPL_QUERY_OBJECT_TYPE) {
- if (entry->object.vm_object == VM_OBJECT_NULL) {
- *flags = 0;
- } else if (entry->object.vm_object->private) {
- *flags = UPL_DEV_MEMORY;
- if (entry->object.vm_object->phys_contiguous) {
- *flags |= UPL_PHYS_CONTIG;
- }
- } else {
- *flags = 0;
- }
- vm_map_unlock(map);
- return KERN_SUCCESS;
+ }
+
+ /*
+ * Create an object if necessary.
+ */
+ if (VME_OBJECT(entry) == VM_OBJECT_NULL) {
+ if (vm_map_lock_read_to_write(map)) {
+ goto REDISCOVER_ENTRY;
}
+
+ VME_OBJECT_SET(entry,
+ vm_object_allocate((vm_size_t)
+ vm_object_round_page((entry->vme_end - entry->vme_start))));
+ VME_OFFSET_SET(entry, 0);
+ assert(entry->use_pmap);
+
+ vm_map_lock_write_to_read(map);
+ }
+
+ if (!(caller_flags & UPL_COPYOUT_FROM) &&
+ !entry->is_sub_map &&
+ !(entry->protection & VM_PROT_WRITE)) {
+ vm_map_unlock_read(map);
+ ret = KERN_PROTECTION_FAILURE;
+ goto done;
+ }
+
+#if CONFIG_EMBEDDED
+ if (map->pmap != kernel_pmap &&
+ (caller_flags & UPL_COPYOUT_FROM) &&
+ (entry->protection & VM_PROT_EXECUTE) &&
+ !(entry->protection & VM_PROT_WRITE)) {
+ vm_offset_t kaddr;
+ vm_size_t ksize;
+
/*
- * Create an object if necessary.
+ * We're about to create a read-only UPL backed by
+ * memory from an executable mapping.
+ * Wiring the pages would result in the pages being copied
+ * (due to the "MAP_PRIVATE" mapping) and no longer
+ * code-signed, so no longer eligible for execution.
+ * Instead, let's copy the data into a kernel buffer and
+ * create the UPL from this kernel buffer.
+ * The kernel buffer is then freed, leaving the UPL holding
+ * the last reference on the VM object, so the memory will
+ * be released when the UPL is committed.
*/
- if (entry->object.vm_object == VM_OBJECT_NULL) {
- entry->object.vm_object = vm_object_allocate(
- (vm_size_t)(entry->vme_end - entry->vme_start));
- entry->offset = 0;
- }
- if (!(caller_flags & UPL_COPYOUT_FROM)) {
- if (!(entry->protection & VM_PROT_WRITE)) {
- vm_map_unlock(map);
- return KERN_PROTECTION_FAILURE;
- }
- if (entry->needs_copy) {
- vm_map_t local_map;
- vm_object_t object;
- vm_map_offset_t offset_hi;
- vm_map_offset_t offset_lo;
- vm_object_offset_t new_offset;
- vm_prot_t prot;
- boolean_t wired;
- vm_behavior_t behavior;
- vm_map_version_t version;
- vm_map_t real_map;
-
- local_map = map;
- vm_map_lock_write_to_read(map);
- if(vm_map_lookup_locked(&local_map,
- offset, VM_PROT_WRITE,
- &version, &object,
- &new_offset, &prot, &wired,
- &behavior, &offset_lo,
- &offset_hi, &real_map)) {
- vm_map_unlock(local_map);
- return KERN_FAILURE;
- }
- if (real_map != map) {
- vm_map_unlock(real_map);
- }
- vm_object_unlock(object);
- vm_map_unlock(local_map);
- goto REDISCOVER_ENTRY;
- }
+ vm_map_unlock_read(map);
+ entry = VM_MAP_ENTRY_NULL;
+ /* allocate kernel buffer */
+ ksize = round_page(*upl_size);
+ kaddr = 0;
+ ret = kmem_alloc_pageable(kernel_map,
+ &kaddr,
+ ksize,
+ tag);
+ if (ret == KERN_SUCCESS) {
+ /* copyin the user data */
+ ret = copyinmap(map, offset, (void *)kaddr, *upl_size);
}
- if (entry->is_sub_map) {
- vm_map_t submap;
-
- submap = entry->object.sub_map;
- local_start = entry->vme_start;
- local_offset = entry->offset;
- vm_map_reference(submap);
- vm_map_unlock(map);
-
- ret = (vm_map_create_upl(submap,
- local_offset + (offset - local_start),
- upl_size, upl, page_list, count,
- flags));
-
- vm_map_deallocate(submap);
- return ret;
- }
-
- if (sync_cow_data) {
- if (entry->object.vm_object->shadow
- || entry->object.vm_object->copy) {
-
- local_object = entry->object.vm_object;
- local_start = entry->vme_start;
- local_offset = entry->offset;
- vm_object_reference(local_object);
- vm_map_unlock(map);
-
- if (entry->object.vm_object->shadow &&
- entry->object.vm_object->copy) {
- vm_object_lock_request(
- local_object->shadow,
- (vm_object_offset_t)
- ((offset - local_start) +
- local_offset) +
- local_object->shadow_offset,
- *upl_size, FALSE,
- MEMORY_OBJECT_DATA_SYNC,
- VM_PROT_NO_CHANGE);
- }
- sync_cow_data = FALSE;
- vm_object_deallocate(local_object);
- goto REDISCOVER_ENTRY;
+ if (ret == KERN_SUCCESS) {
+ if (ksize > *upl_size) {
+ /* zero out the extra space in kernel buffer */
+ memset((void *)(kaddr + *upl_size),
+ 0,
+ ksize - *upl_size);
}
+ /* create the UPL from the kernel buffer */
+ vm_object_offset_t offset_in_object;
+ vm_object_offset_t offset_in_object_page;
+
+ offset_in_object = offset - local_entry_start + local_entry_offset;
+ offset_in_object_page = offset_in_object - vm_object_trunc_page(offset_in_object);
+ assert(offset_in_object_page < PAGE_SIZE);
+ assert(offset_in_object_page + offset_in_mapped_page < PAGE_SIZE);
+ *upl_size -= offset_in_object_page + offset_in_mapped_page;
+ ret = vm_map_create_upl(kernel_map,
+ (vm_map_address_t)(kaddr + offset_in_object_page + offset_in_mapped_page),
+ upl_size, upl, page_list, count, flags, tag);
}
+ if (kaddr != 0) {
+ /* free the kernel buffer */
+ kmem_free(kernel_map, kaddr, ksize);
+ kaddr = 0;
+ ksize = 0;
+ }
+#if DEVELOPMENT || DEBUG
+ DTRACE_VM4(create_upl_from_executable,
+ vm_map_t, map,
+ vm_map_address_t, offset,
+ upl_size_t, *upl_size,
+ kern_return_t, ret);
+#endif /* DEVELOPMENT || DEBUG */
+ goto done;
+ }
+#endif /* CONFIG_EMBEDDED */
+
+ local_object = VME_OBJECT(entry);
+ assert(local_object != VM_OBJECT_NULL);
+
+ if (!entry->is_sub_map &&
+ !entry->needs_copy &&
+ *upl_size != 0 &&
+ local_object->vo_size > *upl_size && /* partial UPL */
+ entry->wired_count == 0 && /* No COW for entries that are wired */
+ (map->pmap != kernel_pmap) && /* alias checks */
+ (vm_map_entry_should_cow_for_true_share(entry) /* case 1 */
+ ||
+ ( /* case 2 */
+ local_object->internal &&
+ (local_object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) &&
+ local_object->ref_count > 1))) {
+ vm_prot_t prot;
- if (force_data_sync) {
-
- local_object = entry->object.vm_object;
- local_start = entry->vme_start;
- local_offset = entry->offset;
- vm_object_reference(local_object);
- vm_map_unlock(map);
+ /*
+ * Case 1:
+ * Set up the targeted range for copy-on-write to avoid
+ * applying true_share/copy_delay to the entire object.
+ *
+ * Case 2:
+ * This map entry covers only part of an internal
+ * object. There could be other map entries covering
+ * other areas of this object and some of these map
+ * entries could be marked as "needs_copy", which
+ * assumes that the object is COPY_SYMMETRIC.
+ * To avoid marking this object as COPY_DELAY and
+ * "true_share", let's shadow it and mark the new
+ * (smaller) object as "true_share" and COPY_DELAY.
+ */
- vm_object_lock_request(
- local_object,
- (vm_object_offset_t)
- ((offset - local_start) + local_offset),
- (vm_object_size_t)*upl_size, FALSE,
- MEMORY_OBJECT_DATA_SYNC,
- VM_PROT_NO_CHANGE);
- force_data_sync = FALSE;
- vm_object_deallocate(local_object);
+ if (vm_map_lock_read_to_write(map)) {
goto REDISCOVER_ENTRY;
}
+ vm_map_lock_assert_exclusive(map);
+ assert(VME_OBJECT(entry) == local_object);
+
+ vm_map_clip_start(map,
+ entry,
+ vm_map_trunc_page(offset,
+ VM_MAP_PAGE_MASK(map)));
+ vm_map_clip_end(map,
+ entry,
+ vm_map_round_page(offset + *upl_size,
+ VM_MAP_PAGE_MASK(map)));
+ if ((entry->vme_end - offset) < *upl_size) {
+ *upl_size = (upl_size_t) (entry->vme_end - offset);
+ assert(*upl_size == entry->vme_end - offset);
+ }
- if(!(entry->object.vm_object->private)) {
- if(*upl_size > (MAX_UPL_TRANSFER*PAGE_SIZE))
- *upl_size = (MAX_UPL_TRANSFER*PAGE_SIZE);
- if(entry->object.vm_object->phys_contiguous) {
- *flags = UPL_PHYS_CONTIG;
- } else {
- *flags = 0;
+ prot = entry->protection & ~VM_PROT_WRITE;
+ if (override_nx(map, VME_ALIAS(entry)) && prot) {
+ prot |= VM_PROT_EXECUTE;
+ }
+ vm_object_pmap_protect(local_object,
+ VME_OFFSET(entry),
+ entry->vme_end - entry->vme_start,
+ ((entry->is_shared ||
+ map->mapped_in_other_pmaps)
+ ? PMAP_NULL
+ : map->pmap),
+ VM_MAP_PAGE_SIZE(map),
+ entry->vme_start,
+ prot);
+
+ assert(entry->wired_count == 0);
+
+ /*
+ * Lock the VM object and re-check its status: if it's mapped
+ * in another address space, we could still be racing with
+ * another thread holding that other VM map exclusively.
+ */
+ vm_object_lock(local_object);
+ if (local_object->true_share) {
+ /* object is already in proper state: no COW needed */
+ assert(local_object->copy_strategy !=
+ MEMORY_OBJECT_COPY_SYMMETRIC);
+ } else {
+ /* not true_share: ask for copy-on-write below */
+ assert(local_object->copy_strategy ==
+ MEMORY_OBJECT_COPY_SYMMETRIC);
+ entry->needs_copy = TRUE;
+ }
+ vm_object_unlock(local_object);
+
+ vm_map_lock_write_to_read(map);
+ }
+
+ if (entry->needs_copy) {
+ /*
+ * Honor copy-on-write for COPY_SYMMETRIC
+ * strategy.
+ */
+ vm_map_t local_map;
+ vm_object_t object;
+ vm_object_offset_t new_offset;
+ vm_prot_t prot;
+ boolean_t wired;
+ vm_map_version_t version;
+ vm_map_t real_map;
+ vm_prot_t fault_type;
+
+ if (entry->vme_start < VM_MAP_TRUNC_PAGE(offset, VM_MAP_PAGE_MASK(map)) ||
+ entry->vme_end > VM_MAP_ROUND_PAGE(offset + *upl_size, VM_MAP_PAGE_MASK(map))) {
+ /*
+ * Clip the requested range first to minimize the
+ * amount of potential copying...
+ */
+ if (vm_map_lock_read_to_write(map)) {
+ goto REDISCOVER_ENTRY;
}
+ vm_map_lock_assert_exclusive(map);
+ assert(VME_OBJECT(entry) == local_object);
+ vm_map_clip_start(map, entry,
+ VM_MAP_TRUNC_PAGE(offset, VM_MAP_PAGE_MASK(map)));
+ vm_map_clip_end(map, entry,
+ VM_MAP_ROUND_PAGE(offset + *upl_size, VM_MAP_PAGE_MASK(map)));
+ vm_map_lock_write_to_read(map);
+ }
+
+ local_map = map;
+
+ if (caller_flags & UPL_COPYOUT_FROM) {
+ fault_type = VM_PROT_READ | VM_PROT_COPY;
+ vm_counters.create_upl_extra_cow++;
+ vm_counters.create_upl_extra_cow_pages +=
+ (entry->vme_end - entry->vme_start) / PAGE_SIZE;
} else {
- *flags = UPL_DEV_MEMORY | UPL_PHYS_CONTIG;
+ fault_type = VM_PROT_WRITE;
+ }
+ if (vm_map_lookup_locked(&local_map,
+ offset, fault_type,
+ OBJECT_LOCK_EXCLUSIVE,
+ &version, &object,
+ &new_offset, &prot, &wired,
+ NULL,
+ &real_map, NULL) != KERN_SUCCESS) {
+ if (fault_type == VM_PROT_WRITE) {
+ vm_counters.create_upl_lookup_failure_write++;
+ } else {
+ vm_counters.create_upl_lookup_failure_copy++;
+ }
+ vm_map_unlock_read(local_map);
+ ret = KERN_FAILURE;
+ goto done;
+ }
+ if (real_map != local_map) {
+ vm_map_unlock(real_map);
+ }
+ vm_map_unlock_read(local_map);
+
+ vm_object_unlock(object);
+
+ goto REDISCOVER_ENTRY;
+ }
+
+ if (entry->is_sub_map) {
+ vm_map_t submap;
+
+ submap = VME_SUBMAP(entry);
+ local_start = entry->vme_start;
+ local_offset = (vm_map_offset_t)VME_OFFSET(entry);
+
+ vm_map_reference(submap);
+ vm_map_unlock_read(map);
+
+ DEBUG4K_UPL("map %p offset 0x%llx (0x%llx) size 0x%x (adjusted 0x%llx original 0x%llx) offset_in_mapped_page 0x%llx submap %p\n", map, (uint64_t)offset, (uint64_t)original_offset, *upl_size, (uint64_t)adjusted_size, (uint64_t)original_size, offset_in_mapped_page, submap);
+ offset += offset_in_mapped_page;
+ *upl_size -= offset_in_mapped_page;
+
+ if (release_map) {
+ vm_map_deallocate(map);
}
- local_object = entry->object.vm_object;
- local_offset = entry->offset;
+ map = submap;
+ release_map = TRUE;
+ offset = local_offset + (offset - local_start);
+ goto start_with_map;
+ }
+
+ if (sync_cow_data &&
+ (VME_OBJECT(entry)->shadow ||
+ VME_OBJECT(entry)->copy)) {
+ local_object = VME_OBJECT(entry);
local_start = entry->vme_start;
+ local_offset = (vm_map_offset_t)VME_OFFSET(entry);
+
vm_object_reference(local_object);
- vm_map_unlock(map);
- if(caller_flags & UPL_SET_IO_WIRE) {
- ret = (vm_object_iopl_request(local_object,
- (vm_object_offset_t)
- ((offset - local_start)
- + local_offset),
- *upl_size,
- upl,
- page_list,
- count,
- caller_flags));
- } else {
- ret = (vm_object_upl_request(local_object,
- (vm_object_offset_t)
- ((offset - local_start)
- + local_offset),
- *upl_size,
- upl,
- page_list,
- count,
- caller_flags));
+ vm_map_unlock_read(map);
+
+ if (local_object->shadow && local_object->copy) {
+ vm_object_lock_request(local_object->shadow,
+ ((vm_object_offset_t)
+ ((offset - local_start) +
+ local_offset) +
+ local_object->vo_shadow_offset),
+ *upl_size, FALSE,
+ MEMORY_OBJECT_DATA_SYNC,
+ VM_PROT_NO_CHANGE);
}
+ sync_cow_data = FALSE;
+ vm_object_deallocate(local_object);
+
+ goto REDISCOVER_ENTRY;
+ }
+ if (force_data_sync) {
+ local_object = VME_OBJECT(entry);
+ local_start = entry->vme_start;
+ local_offset = (vm_map_offset_t)VME_OFFSET(entry);
+
+ vm_object_reference(local_object);
+ vm_map_unlock_read(map);
+
+ vm_object_lock_request(local_object,
+ ((vm_object_offset_t)
+ ((offset - local_start) +
+ local_offset)),
+ (vm_object_size_t)*upl_size,
+ FALSE,
+ MEMORY_OBJECT_DATA_SYNC,
+ VM_PROT_NO_CHANGE);
+
+ force_data_sync = FALSE;
vm_object_deallocate(local_object);
- return(ret);
- }
- vm_map_unlock(map);
- return(KERN_FAILURE);
+ goto REDISCOVER_ENTRY;
+ }
+ if (VME_OBJECT(entry)->private) {
+ *flags = UPL_DEV_MEMORY;
+ } else {
+ *flags = 0;
+ }
+
+ if (VME_OBJECT(entry)->phys_contiguous) {
+ *flags |= UPL_PHYS_CONTIG;
+ }
+
+ local_object = VME_OBJECT(entry);
+ local_offset = (vm_map_offset_t)VME_OFFSET(entry);
+ local_start = entry->vme_start;
+
+ /*
+ * Wiring will copy the pages to the shadow object.
+ * The shadow object will not be code-signed so
+ * attempting to execute code from these copied pages
+ * would trigger a code-signing violation.
+ */
+ if (entry->protection & VM_PROT_EXECUTE) {
+#if MACH_ASSERT
+ printf("pid %d[%s] create_upl out of executable range from "
+ "0x%llx to 0x%llx: side effects may include "
+ "code-signing violations later on\n",
+ proc_selfpid(),
+ (current_task()->bsd_info
+ ? proc_name_address(current_task()->bsd_info)
+ : "?"),
+ (uint64_t) entry->vme_start,
+ (uint64_t) entry->vme_end);
+#endif /* MACH_ASSERT */
+ DTRACE_VM2(cs_executable_create_upl,
+ uint64_t, (uint64_t)entry->vme_start,
+ uint64_t, (uint64_t)entry->vme_end);
+ cs_executable_create_upl++;
+ }
+
+ vm_object_lock(local_object);
+
+ /*
+ * Ensure that this object is "true_share" and "copy_delay" now,
+ * while we're still holding the VM map lock. After we unlock the map,
+ * anything could happen to that mapping, including some copy-on-write
+ * activity. We need to make sure that the IOPL will point at the
+ * same memory as the mapping.
+ */
+ if (local_object->true_share) {
+ assert(local_object->copy_strategy !=
+ MEMORY_OBJECT_COPY_SYMMETRIC);
+ } else if (local_object != kernel_object &&
+ local_object != compressor_object &&
+ !local_object->phys_contiguous) {
+#if VM_OBJECT_TRACKING_OP_TRUESHARE
+ if (!local_object->true_share &&
+ vm_object_tracking_inited) {
+ void *bt[VM_OBJECT_TRACKING_BTDEPTH];
+ int num = 0;
+ num = OSBacktrace(bt,
+ VM_OBJECT_TRACKING_BTDEPTH);
+ btlog_add_entry(vm_object_tracking_btlog,
+ local_object,
+ VM_OBJECT_TRACKING_OP_TRUESHARE,
+ bt,
+ num);
+ }
+#endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */
+ local_object->true_share = TRUE;
+ if (local_object->copy_strategy ==
+ MEMORY_OBJECT_COPY_SYMMETRIC) {
+ local_object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
+ }
+ }
+
+ vm_object_reference_locked(local_object);
+ vm_object_unlock(local_object);
+
+ vm_map_unlock_read(map);
+
+ offset += offset_in_mapped_page;
+ assert(*upl_size > offset_in_mapped_page);
+ *upl_size -= offset_in_mapped_page;
+
+ ret = vm_object_iopl_request(local_object,
+ ((vm_object_offset_t)
+ ((offset - local_start) + local_offset)),
+ *upl_size,
+ upl,
+ page_list,
+ count,
+ caller_flags,
+ tag);
+ vm_object_deallocate(local_object);
+
+done:
+ if (release_map) {
+ vm_map_deallocate(map);
+ }
+ return ret;
}
/*
* Internal routine to enter a UPL into a VM map.
- *
+ *
* JMM - This should just be doable through the standard
* vm_map_enter() API.
*/
kern_return_t
vm_map_enter_upl(
- vm_map_t map,
- upl_t upl,
- vm_map_offset_t *dst_addr)
+ vm_map_t map,
+ upl_t upl,
+ vm_map_offset_t *dst_addr)
{
- vm_map_size_t size;
- vm_object_offset_t offset;
- vm_map_offset_t addr;
- vm_page_t m;
- kern_return_t kr;
-
- if (upl == UPL_NULL)
+ vm_map_size_t size;
+ vm_object_offset_t offset;
+ vm_map_offset_t addr;
+ vm_page_t m;
+ kern_return_t kr;
+ int isVectorUPL = 0, curr_upl = 0;
+ upl_t vector_upl = NULL;
+ vm_offset_t vector_upl_dst_addr = 0;
+ vm_map_t vector_upl_submap = NULL;
+ upl_offset_t subupl_offset = 0;
+ upl_size_t subupl_size = 0;
+
+ if (upl == UPL_NULL) {
return KERN_INVALID_ARGUMENT;
+ }
- upl_lock(upl);
+ DEBUG4K_UPL("map %p upl %p flags 0x%x object %p offset 0x%llx size 0x%x \n", map, upl, upl->flags, upl->map_object, upl->u_offset, upl->u_size);
+ assert(map == kernel_map);
- /* check to see if already mapped */
- if(UPL_PAGE_LIST_MAPPED & upl->flags) {
- upl_unlock(upl);
- return KERN_FAILURE;
+ if ((isVectorUPL = vector_upl_is_valid(upl))) {
+ int mapped = 0, valid_upls = 0;
+ vector_upl = upl;
+
+ upl_lock(vector_upl);
+ for (curr_upl = 0; curr_upl < MAX_VECTOR_UPL_ELEMENTS; curr_upl++) {
+ upl = vector_upl_subupl_byindex(vector_upl, curr_upl );
+ if (upl == NULL) {
+ continue;
+ }
+ valid_upls++;
+ if (UPL_PAGE_LIST_MAPPED & upl->flags) {
+ mapped++;
+ }
+ }
+
+ if (mapped) {
+ if (mapped != valid_upls) {
+ panic("Only %d of the %d sub-upls within the Vector UPL are alread mapped\n", mapped, valid_upls);
+ } else {
+ upl_unlock(vector_upl);
+ return KERN_FAILURE;
+ }
+ }
+
+ if (VM_MAP_PAGE_MASK(map) < PAGE_MASK) {
+ panic("TODO4K: vector UPL not implemented");
+ }
+
+ kr = kmem_suballoc(map, &vector_upl_dst_addr,
+ vector_upl->u_size,
+ FALSE,
+ VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_NONE,
+ &vector_upl_submap);
+ if (kr != KERN_SUCCESS) {
+ panic("Vector UPL submap allocation failed\n");
+ }
+ map = vector_upl_submap;
+ vector_upl_set_submap(vector_upl, vector_upl_submap, vector_upl_dst_addr);
+ curr_upl = 0;
+ } else {
+ upl_lock(upl);
}
- if((!(upl->map_object->pageout)) &&
- !((upl->flags & (UPL_DEVICE_MEMORY | UPL_IO_WIRE)) ||
- (upl->map_object->phys_contiguous))) {
- vm_object_t object;
- vm_page_t alias_page;
- vm_object_offset_t new_offset;
- int pg_num;
- wpl_array_t lite_list;
-
- if(upl->flags & UPL_INTERNAL) {
- lite_list = (wpl_array_t)
- ((((uintptr_t)upl) + sizeof(struct upl))
- + ((upl->size/PAGE_SIZE)
- * sizeof(upl_page_info_t)));
- } else {
+process_upl_to_enter:
+ if (isVectorUPL) {
+ if (curr_upl == MAX_VECTOR_UPL_ELEMENTS) {
+ *dst_addr = vector_upl_dst_addr;
+ upl_unlock(vector_upl);
+ return KERN_SUCCESS;
+ }
+ upl = vector_upl_subupl_byindex(vector_upl, curr_upl++ );
+ if (upl == NULL) {
+ goto process_upl_to_enter;
+ }
+
+ vector_upl_get_iostate(vector_upl, upl, &subupl_offset, &subupl_size);
+ *dst_addr = (vm_map_offset_t)(vector_upl_dst_addr + (vm_map_offset_t)subupl_offset);
+ } else {
+ /*
+ * check to see if already mapped
+ */
+ if (UPL_PAGE_LIST_MAPPED & upl->flags) {
+ upl_unlock(upl);
+ return KERN_FAILURE;
+ }
+ }
+
+ size = upl_adjusted_size(upl, VM_MAP_PAGE_MASK(map));
+
+ if ((!(upl->flags & UPL_SHADOWED)) &&
+ ((upl->flags & UPL_HAS_BUSY) ||
+ !((upl->flags & (UPL_DEVICE_MEMORY | UPL_IO_WIRE)) || (upl->map_object->phys_contiguous)))) {
+ vm_object_t object;
+ vm_page_t alias_page;
+ vm_object_offset_t new_offset;
+ unsigned int pg_num;
+ wpl_array_t lite_list;
+
+ if (upl->flags & UPL_INTERNAL) {
lite_list = (wpl_array_t)
- (((uintptr_t)upl) + sizeof(struct upl));
+ ((((uintptr_t)upl) + sizeof(struct upl))
+ + ((size / PAGE_SIZE) * sizeof(upl_page_info_t)));
+ } else {
+ lite_list = (wpl_array_t)(((uintptr_t)upl) + sizeof(struct upl));
}
object = upl->map_object;
- upl->map_object = vm_object_allocate(upl->size);
+ upl->map_object = vm_object_allocate(vm_object_round_page(size));
+
vm_object_lock(upl->map_object);
+
upl->map_object->shadow = object;
upl->map_object->pageout = TRUE;
upl->map_object->can_persist = FALSE;
- upl->map_object->copy_strategy =
- MEMORY_OBJECT_COPY_NONE;
- upl->map_object->shadow_offset =
- upl->offset - object->paging_offset;
+ upl->map_object->copy_strategy = MEMORY_OBJECT_COPY_NONE;
+ upl->map_object->vo_shadow_offset = upl_adjusted_offset(upl, PAGE_MASK) - object->paging_offset;
+ assertf(page_aligned(upl->map_object->vo_shadow_offset),
+ "object %p shadow_offset 0x%llx",
+ upl->map_object,
+ (uint64_t)upl->map_object->vo_shadow_offset);
upl->map_object->wimg_bits = object->wimg_bits;
- offset = upl->map_object->shadow_offset;
+ assertf(page_aligned(upl->map_object->vo_shadow_offset),
+ "object %p shadow_offset 0x%llx",
+ upl->map_object, upl->map_object->vo_shadow_offset);
+ offset = upl->map_object->vo_shadow_offset;
new_offset = 0;
- size = upl->size;
+ size = upl_adjusted_size(upl, VM_MAP_PAGE_MASK(map));
- vm_object_lock(object);
+ upl->flags |= UPL_SHADOWED;
- while(size) {
- pg_num = (new_offset)/PAGE_SIZE;
- if(lite_list[pg_num>>5] & (1 << (pg_num & 31))) {
- vm_object_unlock(object);
- VM_PAGE_GRAB_FICTITIOUS(alias_page);
- vm_object_lock(object);
- m = vm_page_lookup(object, offset);
- if (m == VM_PAGE_NULL) {
- panic("vm_upl_map: page missing\n");
- }
+ while (size) {
+ pg_num = (unsigned int) (new_offset / PAGE_SIZE);
+ assert(pg_num == new_offset / PAGE_SIZE);
- vm_object_paging_begin(object);
+ if (lite_list[pg_num >> 5] & (1U << (pg_num & 31))) {
+ VM_PAGE_GRAB_FICTITIOUS(alias_page);
- /*
- * Convert the fictitious page to a private
- * shadow of the real page.
- */
- assert(alias_page->fictitious);
- alias_page->fictitious = FALSE;
- alias_page->private = TRUE;
- alias_page->pageout = TRUE;
- alias_page->phys_page = m->phys_page;
+ vm_object_lock(object);
- vm_page_lock_queues();
- vm_page_wire(alias_page);
- vm_page_unlock_queues();
+ m = vm_page_lookup(object, offset);
+ if (m == VM_PAGE_NULL) {
+ panic("vm_upl_map: page missing\n");
+ }
- /*
- * ENCRYPTED SWAP:
- * The virtual page ("m") has to be wired in some way
- * here or its physical page ("m->phys_page") could
- * be recycled at any time.
- * Assuming this is enforced by the caller, we can't
- * get an encrypted page here. Since the encryption
- * key depends on the VM page's "pager" object and
- * the "paging_offset", we couldn't handle 2 pageable
- * VM pages (with different pagers and paging_offsets)
- * sharing the same physical page: we could end up
- * encrypting with one key (via one VM page) and
- * decrypting with another key (via the alias VM page).
- */
- ASSERT_PAGE_DECRYPTED(m);
+ /*
+ * Convert the fictitious page to a private
+ * shadow of the real page.
+ */
+ assert(alias_page->vmp_fictitious);
+ alias_page->vmp_fictitious = FALSE;
+ alias_page->vmp_private = TRUE;
+ alias_page->vmp_free_when_done = TRUE;
+ /*
+ * since m is a page in the upl it must
+ * already be wired or BUSY, so it's
+ * safe to assign the underlying physical
+ * page to the alias
+ */
+ VM_PAGE_SET_PHYS_PAGE(alias_page, VM_PAGE_GET_PHYS_PAGE(m));
- vm_page_insert(alias_page,
- upl->map_object, new_offset);
- assert(!alias_page->wanted);
- alias_page->busy = FALSE;
- alias_page->absent = FALSE;
- }
+ vm_object_unlock(object);
+
+ vm_page_lockspin_queues();
+ vm_page_wire(alias_page, VM_KERN_MEMORY_NONE, TRUE);
+ vm_page_unlock_queues();
+
+ vm_page_insert_wired(alias_page, upl->map_object, new_offset, VM_KERN_MEMORY_NONE);
- size -= PAGE_SIZE;
- offset += PAGE_SIZE_64;
- new_offset += PAGE_SIZE_64;
+ assert(!alias_page->vmp_wanted);
+ alias_page->vmp_busy = FALSE;
+ alias_page->vmp_absent = FALSE;
+ }
+ size -= PAGE_SIZE;
+ offset += PAGE_SIZE_64;
+ new_offset += PAGE_SIZE_64;
}
- vm_object_unlock(object);
vm_object_unlock(upl->map_object);
}
- if ((upl->flags & (UPL_DEVICE_MEMORY | UPL_IO_WIRE)) || upl->map_object->phys_contiguous)
- offset = upl->offset - upl->map_object->paging_offset;
- else
- offset = 0;
-
- size = upl->size;
-
- vm_object_lock(upl->map_object);
- upl->map_object->ref_count++;
- vm_object_res_reference(upl->map_object);
- vm_object_unlock(upl->map_object);
+ if (upl->flags & UPL_SHADOWED) {
+ offset = 0;
+ } else {
+ offset = upl_adjusted_offset(upl, VM_MAP_PAGE_MASK(map)) - upl->map_object->paging_offset;
+ }
- *dst_addr = 0;
+ size = upl_adjusted_size(upl, VM_MAP_PAGE_MASK(map));
+ vm_object_reference(upl->map_object);
- /* NEED A UPL_MAP ALIAS */
- kr = vm_map_enter(map, dst_addr, (vm_map_size_t)size, (vm_map_offset_t) 0,
- VM_FLAGS_ANYWHERE, upl->map_object, offset, FALSE,
- VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
+ if (!isVectorUPL) {
+ *dst_addr = 0;
+ /*
+ * NEED A UPL_MAP ALIAS
+ */
+ kr = vm_map_enter(map, dst_addr, (vm_map_size_t)size, (vm_map_offset_t) 0,
+ VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_OSFMK,
+ upl->map_object, offset, FALSE,
+ VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
- if (kr != KERN_SUCCESS) {
- upl_unlock(upl);
- return(kr);
+ if (kr != KERN_SUCCESS) {
+ vm_object_deallocate(upl->map_object);
+ upl_unlock(upl);
+ return kr;
+ }
+ } else {
+ kr = vm_map_enter(map, dst_addr, (vm_map_size_t)size, (vm_map_offset_t) 0,
+ VM_FLAGS_FIXED, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_OSFMK,
+ upl->map_object, offset, FALSE,
+ VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
+ if (kr) {
+ panic("vm_map_enter failed for a Vector UPL\n");
+ }
}
-
vm_object_lock(upl->map_object);
- for(addr=*dst_addr; size > 0; size-=PAGE_SIZE,addr+=PAGE_SIZE) {
+ for (addr = *dst_addr; size > 0; size -= PAGE_SIZE, addr += PAGE_SIZE) {
m = vm_page_lookup(upl->map_object, offset);
- if(m) {
- unsigned int cache_attr;
- cache_attr = ((unsigned int)m->object->wimg_bits) & VM_WIMG_MASK;
-
- PMAP_ENTER(map->pmap, addr,
- m, VM_PROT_ALL,
- cache_attr, TRUE);
+
+ if (m) {
+ m->vmp_pmapped = TRUE;
+
+ /* CODE SIGNING ENFORCEMENT: page has been wpmapped,
+ * but only in kernel space. If this was on a user map,
+ * we'd have to set the wpmapped bit. */
+ /* m->vmp_wpmapped = TRUE; */
+ assert(map->pmap == kernel_pmap);
+
+ PMAP_ENTER(map->pmap, addr, m, VM_PROT_DEFAULT, VM_PROT_NONE, 0, TRUE, kr);
+
+ assert(kr == KERN_SUCCESS);
+#if KASAN
+ kasan_notify_address(addr, PAGE_SIZE_64);
+#endif
}
- offset+=PAGE_SIZE_64;
+ offset += PAGE_SIZE_64;
}
vm_object_unlock(upl->map_object);
- upl->ref_count++; /* hold a reference for the mapping */
+ /*
+ * hold a reference for the mapping
+ */
+ upl->ref_count++;
upl->flags |= UPL_PAGE_LIST_MAPPED;
- upl->kaddr = *dst_addr;
+ upl->kaddr = (vm_offset_t) *dst_addr;
+ assert(upl->kaddr == *dst_addr);
+
+ if (isVectorUPL) {
+ goto process_upl_to_enter;
+ }
+
+ if (!isVectorUPL) {
+ vm_map_offset_t addr_adjustment;
+
+ addr_adjustment = (vm_map_offset_t)(upl->u_offset - upl_adjusted_offset(upl, VM_MAP_PAGE_MASK(map)));
+ if (addr_adjustment) {
+ assert(VM_MAP_PAGE_MASK(map) != PAGE_MASK);
+ DEBUG4K_UPL("dst_addr 0x%llx (+ 0x%llx) -> 0x%llx\n", (uint64_t)*dst_addr, (uint64_t)addr_adjustment, (uint64_t)(*dst_addr + addr_adjustment));
+ *dst_addr += addr_adjustment;
+ }
+ }
+
upl_unlock(upl);
+
return KERN_SUCCESS;
}
-
+
/*
* Internal routine to remove a UPL mapping from a VM map.
*
*/
kern_return_t
vm_map_remove_upl(
- vm_map_t map,
- upl_t upl)
+ vm_map_t map,
+ upl_t upl)
{
- vm_address_t addr;
- upl_size_t size;
+ vm_address_t addr;
+ upl_size_t size;
+ int isVectorUPL = 0, curr_upl = 0;
+ upl_t vector_upl = NULL;
+
+ if (upl == UPL_NULL) {
+ return KERN_INVALID_ARGUMENT;
+ }
+
+ if ((isVectorUPL = vector_upl_is_valid(upl))) {
+ int unmapped = 0, valid_upls = 0;
+ vector_upl = upl;
+ upl_lock(vector_upl);
+ for (curr_upl = 0; curr_upl < MAX_VECTOR_UPL_ELEMENTS; curr_upl++) {
+ upl = vector_upl_subupl_byindex(vector_upl, curr_upl );
+ if (upl == NULL) {
+ continue;
+ }
+ valid_upls++;
+ if (!(UPL_PAGE_LIST_MAPPED & upl->flags)) {
+ unmapped++;
+ }
+ }
+
+ if (unmapped) {
+ if (unmapped != valid_upls) {
+ panic("%d of the %d sub-upls within the Vector UPL is/are not mapped\n", unmapped, valid_upls);
+ } else {
+ upl_unlock(vector_upl);
+ return KERN_FAILURE;
+ }
+ }
+ curr_upl = 0;
+ } else {
+ upl_lock(upl);
+ }
- if (upl == UPL_NULL)
- return KERN_INVALID_ARGUMENT;
+process_upl_to_remove:
+ if (isVectorUPL) {
+ if (curr_upl == MAX_VECTOR_UPL_ELEMENTS) {
+ vm_map_t v_upl_submap;
+ vm_offset_t v_upl_submap_dst_addr;
+ vector_upl_get_submap(vector_upl, &v_upl_submap, &v_upl_submap_dst_addr);
+
+ vm_map_remove(map, v_upl_submap_dst_addr,
+ v_upl_submap_dst_addr + vector_upl->u_size,
+ VM_MAP_REMOVE_NO_FLAGS);
+ vm_map_deallocate(v_upl_submap);
+ upl_unlock(vector_upl);
+ return KERN_SUCCESS;
+ }
- upl_lock(upl);
- if(upl->flags & UPL_PAGE_LIST_MAPPED) {
+ upl = vector_upl_subupl_byindex(vector_upl, curr_upl++ );
+ if (upl == NULL) {
+ goto process_upl_to_remove;
+ }
+ }
+
+ if (upl->flags & UPL_PAGE_LIST_MAPPED) {
addr = upl->kaddr;
- size = upl->size;
+ size = upl_adjusted_size(upl, VM_MAP_PAGE_MASK(map));
+
assert(upl->ref_count > 1);
- upl->ref_count--; /* removing mapping ref */
+ upl->ref_count--; /* removing mapping ref */
+
upl->flags &= ~UPL_PAGE_LIST_MAPPED;
upl->kaddr = (vm_offset_t) 0;
- upl_unlock(upl);
- vm_map_remove( map,
- vm_map_trunc_page(addr),
- vm_map_round_page(addr + size),
- VM_MAP_NO_FLAGS);
- return KERN_SUCCESS;
+ if (!isVectorUPL) {
+ upl_unlock(upl);
+
+ vm_map_remove(
+ map,
+ vm_map_trunc_page(addr,
+ VM_MAP_PAGE_MASK(map)),
+ vm_map_round_page(addr + size,
+ VM_MAP_PAGE_MASK(map)),
+ VM_MAP_REMOVE_NO_FLAGS);
+ return KERN_SUCCESS;
+ } else {
+ /*
+ * If it's a Vectored UPL, we'll be removing the entire
+ * submap anyways, so no need to remove individual UPL
+ * element mappings from within the submap
+ */
+ goto process_upl_to_remove;
+ }
}
upl_unlock(upl);
+
return KERN_FAILURE;
}
+
kern_return_t
upl_commit_range(
- upl_t upl,
- upl_offset_t offset,
- upl_size_t size,
- int flags,
- upl_page_info_t *page_list,
- mach_msg_type_number_t count,
- boolean_t *empty)
+ upl_t upl,
+ upl_offset_t offset,
+ upl_size_t size,
+ int flags,
+ upl_page_info_t *page_list,
+ mach_msg_type_number_t count,
+ boolean_t *empty)
{
- upl_size_t xfer_size = size;
- vm_object_t shadow_object;
- vm_object_t object = upl->map_object;
- vm_object_offset_t target_offset;
- int entry;
- wpl_array_t lite_list;
- int occupied;
- int delayed_unlock = 0;
- int clear_refmod = 0;
- boolean_t shadow_internal;
-
+ upl_size_t xfer_size, subupl_size;
+ vm_object_t shadow_object;
+ vm_object_t object;
+ vm_object_t m_object;
+ vm_object_offset_t target_offset;
+ upl_offset_t subupl_offset = offset;
+ int entry;
+ wpl_array_t lite_list;
+ int occupied;
+ int clear_refmod = 0;
+ int pgpgout_count = 0;
+ struct vm_page_delayed_work dw_array;
+ struct vm_page_delayed_work *dwp, *dwp_start;
+ bool dwp_finish_ctx = TRUE;
+ int dw_count;
+ int dw_limit;
+ int isVectorUPL = 0;
+ upl_t vector_upl = NULL;
+ boolean_t should_be_throttled = FALSE;
+
+ vm_page_t nxt_page = VM_PAGE_NULL;
+ int fast_path_possible = 0;
+ int fast_path_full_commit = 0;
+ int throttle_page = 0;
+ int unwired_count = 0;
+ int local_queue_count = 0;
+ vm_page_t first_local, last_local;
+ vm_object_offset_t obj_start, obj_end, obj_offset;
+ kern_return_t kr = KERN_SUCCESS;
+
+// DEBUG4K_UPL("upl %p (u_offset 0x%llx u_size 0x%llx) object %p offset 0x%llx size 0x%llx flags 0x%x\n", upl, (uint64_t)upl->u_offset, (uint64_t)upl->u_size, upl->map_object, (uint64_t)offset, (uint64_t)size, flags);
+
+ dwp_start = dwp = NULL;
+
+ subupl_size = size;
*empty = FALSE;
- if (upl == UPL_NULL)
+ if (upl == UPL_NULL) {
return KERN_INVALID_ARGUMENT;
+ }
+
+ dw_count = 0;
+ dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT);
+ dwp_start = vm_page_delayed_work_get_ctx();
+ if (dwp_start == NULL) {
+ dwp_start = &dw_array;
+ dw_limit = 1;
+ dwp_finish_ctx = FALSE;
+ }
+ dwp = dwp_start;
- if (count == 0)
+ if (count == 0) {
page_list = NULL;
+ }
- if (object->pageout) {
- shadow_object = object->shadow;
+ if ((isVectorUPL = vector_upl_is_valid(upl))) {
+ vector_upl = upl;
+ upl_lock(vector_upl);
} else {
- shadow_object = object;
+ upl_lock(upl);
}
- upl_lock(upl);
+process_upl_to_commit:
- if (upl->flags & UPL_ACCESS_BLOCKED) {
- /*
- * We used this UPL to block access to the pages by marking
- * them "busy". Now we need to clear the "busy" bit to allow
- * access to these pages again.
- */
- flags |= UPL_COMMIT_ALLOW_ACCESS;
+ if (isVectorUPL) {
+ size = subupl_size;
+ offset = subupl_offset;
+ if (size == 0) {
+ upl_unlock(vector_upl);
+ kr = KERN_SUCCESS;
+ goto done;
+ }
+ upl = vector_upl_subupl_byoffset(vector_upl, &offset, &size);
+ if (upl == NULL) {
+ upl_unlock(vector_upl);
+ kr = KERN_FAILURE;
+ goto done;
+ }
+ page_list = UPL_GET_INTERNAL_PAGE_LIST_SIMPLE(upl);
+ subupl_size -= size;
+ subupl_offset += size;
}
- if (upl->flags & UPL_CLEAR_DIRTY)
- flags |= UPL_COMMIT_CLEAR_DIRTY;
+#if UPL_DEBUG
+ if (upl->upl_commit_index < UPL_DEBUG_COMMIT_RECORDS) {
+ (void) OSBacktrace(&upl->upl_commit_records[upl->upl_commit_index].c_retaddr[0], UPL_DEBUG_STACK_FRAMES);
+ upl->upl_commit_records[upl->upl_commit_index].c_beg = offset;
+ upl->upl_commit_records[upl->upl_commit_index].c_end = (offset + size);
+
+ upl->upl_commit_index++;
+ }
+#endif
if (upl->flags & UPL_DEVICE_MEMORY) {
xfer_size = 0;
- } else if ((offset + size) > upl->size) {
- upl_unlock(upl);
- return KERN_FAILURE;
+ } else if ((offset + size) <= upl_adjusted_size(upl, PAGE_MASK)) {
+ xfer_size = size;
+ } else {
+ if (!isVectorUPL) {
+ upl_unlock(upl);
+ } else {
+ upl_unlock(vector_upl);
+ }
+ DEBUG4K_ERROR("upl %p (u_offset 0x%llx u_size 0x%x) offset 0x%x size 0x%x\n", upl, upl->u_offset, upl->u_size, offset, size);
+ kr = KERN_FAILURE;
+ goto done;
+ }
+ if (upl->flags & UPL_SET_DIRTY) {
+ flags |= UPL_COMMIT_SET_DIRTY;
+ }
+ if (upl->flags & UPL_CLEAR_DIRTY) {
+ flags |= UPL_COMMIT_CLEAR_DIRTY;
}
if (upl->flags & UPL_INTERNAL) {
- lite_list = (wpl_array_t)
- ((((uintptr_t)upl) + sizeof(struct upl))
- + ((upl->size/PAGE_SIZE) * sizeof(upl_page_info_t)));
+ lite_list = (wpl_array_t) ((((uintptr_t)upl) + sizeof(struct upl))
+ + ((upl_adjusted_size(upl, PAGE_MASK) / PAGE_SIZE) * sizeof(upl_page_info_t)));
} else {
- lite_list = (wpl_array_t)
- (((uintptr_t)upl) + sizeof(struct upl));
+ lite_list = (wpl_array_t) (((uintptr_t)upl) + sizeof(struct upl));
}
- if (object != shadow_object)
- vm_object_lock(object);
- vm_object_lock(shadow_object);
- shadow_internal = shadow_object->internal;
+ object = upl->map_object;
- entry = offset/PAGE_SIZE;
+ if (upl->flags & UPL_SHADOWED) {
+ vm_object_lock(object);
+ shadow_object = object->shadow;
+ } else {
+ shadow_object = object;
+ }
+ entry = offset / PAGE_SIZE;
target_offset = (vm_object_offset_t)offset;
- while (xfer_size) {
- vm_page_t t,m;
- upl_page_info_t *p;
+ if (upl->flags & UPL_KERNEL_OBJECT) {
+ vm_object_lock_shared(shadow_object);
+ } else {
+ vm_object_lock(shadow_object);
+ }
+
+ VM_OBJECT_WIRED_PAGE_UPDATE_START(shadow_object);
+
+ if (upl->flags & UPL_ACCESS_BLOCKED) {
+ assert(shadow_object->blocked_access);
+ shadow_object->blocked_access = FALSE;
+ vm_object_wakeup(object, VM_OBJECT_EVENT_UNBLOCKED);
+ }
+
+ if (shadow_object->code_signed) {
+ /*
+ * CODE SIGNING:
+ * If the object is code-signed, do not let this UPL tell
+ * us if the pages are valid or not. Let the pages be
+ * validated by VM the normal way (when they get mapped or
+ * copied).
+ */
+ flags &= ~UPL_COMMIT_CS_VALIDATED;
+ }
+ if (!page_list) {
+ /*
+ * No page list to get the code-signing info from !?
+ */
+ flags &= ~UPL_COMMIT_CS_VALIDATED;
+ }
+ if (!VM_DYNAMIC_PAGING_ENABLED() && shadow_object->internal) {
+ should_be_throttled = TRUE;
+ }
+
+ if ((upl->flags & UPL_IO_WIRE) &&
+ !(flags & UPL_COMMIT_FREE_ABSENT) &&
+ !isVectorUPL &&
+ shadow_object->purgable != VM_PURGABLE_VOLATILE &&
+ shadow_object->purgable != VM_PURGABLE_EMPTY) {
+ if (!vm_page_queue_empty(&shadow_object->memq)) {
+ if (size == shadow_object->vo_size) {
+ nxt_page = (vm_page_t)vm_page_queue_first(&shadow_object->memq);
+ fast_path_full_commit = 1;
+ }
+ fast_path_possible = 1;
+
+ if (!VM_DYNAMIC_PAGING_ENABLED() && shadow_object->internal &&
+ (shadow_object->purgable == VM_PURGABLE_DENY ||
+ shadow_object->purgable == VM_PURGABLE_NONVOLATILE ||
+ shadow_object->purgable == VM_PURGABLE_VOLATILE)) {
+ throttle_page = 1;
+ }
+ }
+ }
+ first_local = VM_PAGE_NULL;
+ last_local = VM_PAGE_NULL;
+
+ obj_start = target_offset + upl->u_offset - shadow_object->paging_offset;
+ obj_end = obj_start + xfer_size;
+ obj_start = vm_object_trunc_page(obj_start);
+ obj_end = vm_object_round_page(obj_end);
+ for (obj_offset = obj_start;
+ obj_offset < obj_end;
+ obj_offset += PAGE_SIZE) {
+ vm_page_t t, m;
+
+ dwp->dw_mask = 0;
+ clear_refmod = 0;
m = VM_PAGE_NULL;
if (upl->flags & UPL_LITE) {
- int pg_num;
+ unsigned int pg_num;
+
+ if (nxt_page != VM_PAGE_NULL) {
+ m = nxt_page;
+ nxt_page = (vm_page_t)vm_page_queue_next(&nxt_page->vmp_listq);
+ target_offset = m->vmp_offset;
+ }
+ pg_num = (unsigned int) (target_offset / PAGE_SIZE);
+ assert(pg_num == target_offset / PAGE_SIZE);
- pg_num = target_offset/PAGE_SIZE;
+ if (lite_list[pg_num >> 5] & (1U << (pg_num & 31))) {
+ lite_list[pg_num >> 5] &= ~(1U << (pg_num & 31));
- if (lite_list[pg_num>>5] & (1 << (pg_num & 31))) {
- lite_list[pg_num>>5] &= ~(1 << (pg_num & 31));
- m = vm_page_lookup(shadow_object,
- target_offset + (upl->offset -
- shadow_object->paging_offset));
+ if (!(upl->flags & UPL_KERNEL_OBJECT) && m == VM_PAGE_NULL) {
+ m = vm_page_lookup(shadow_object, obj_offset);
+ }
+ } else {
+ m = NULL;
}
}
- if (object->pageout) {
- if ((t = vm_page_lookup(object, target_offset)) != NULL) {
- t->pageout = FALSE;
+ if (upl->flags & UPL_SHADOWED) {
+ if ((t = vm_page_lookup(object, target_offset)) != VM_PAGE_NULL) {
+ t->vmp_free_when_done = FALSE;
- if (delayed_unlock) {
- delayed_unlock = 0;
- vm_page_unlock_queues();
- }
VM_PAGE_FREE(t);
- if (m == NULL) {
- m = vm_page_lookup(
- shadow_object,
- target_offset +
- object->shadow_offset);
+ if (!(upl->flags & UPL_KERNEL_OBJECT) && m == VM_PAGE_NULL) {
+ m = vm_page_lookup(shadow_object, target_offset + object->vo_shadow_offset);
}
- if (m != VM_PAGE_NULL)
- vm_object_paging_end(m->object);
}
}
- if (m != VM_PAGE_NULL) {
+ if (m == VM_PAGE_NULL) {
+ goto commit_next_page;
+ }
- clear_refmod = 0;
+ m_object = VM_PAGE_OBJECT(m);
- if (upl->flags & UPL_IO_WIRE) {
+ if (m->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) {
+ assert(m->vmp_busy);
- if (delayed_unlock == 0)
- vm_page_lock_queues();
+ dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
+ goto commit_next_page;
+ }
- vm_page_unwire(m);
+ if (flags & UPL_COMMIT_CS_VALIDATED) {
+ /*
+ * CODE SIGNING:
+ * Set the code signing bits according to
+ * what the UPL says they should be.
+ */
+ m->vmp_cs_validated |= page_list[entry].cs_validated;
+ m->vmp_cs_tainted |= page_list[entry].cs_tainted;
+ m->vmp_cs_nx |= page_list[entry].cs_nx;
+ }
+ if (flags & UPL_COMMIT_WRITTEN_BY_KERNEL) {
+ m->vmp_written_by_kernel = TRUE;
+ }
- if (delayed_unlock++ > DELAYED_UNLOCK_LIMIT) {
- delayed_unlock = 0;
- vm_page_unlock_queues();
- }
- if (page_list) {
+ if (upl->flags & UPL_IO_WIRE) {
+ if (page_list) {
page_list[entry].phys_addr = 0;
}
- if (flags & UPL_COMMIT_SET_DIRTY) {
- m->dirty = TRUE;
- } else if (flags & UPL_COMMIT_CLEAR_DIRTY) {
- m->dirty = FALSE;
+
+ if (flags & UPL_COMMIT_SET_DIRTY) {
+ SET_PAGE_DIRTY(m, FALSE);
+ } else if (flags & UPL_COMMIT_CLEAR_DIRTY) {
+ m->vmp_dirty = FALSE;
+
+ if (!(flags & UPL_COMMIT_CS_VALIDATED) &&
+ m->vmp_cs_validated &&
+ m->vmp_cs_tainted != VMP_CS_ALL_TRUE) {
+ /*
+ * CODE SIGNING:
+ * This page is no longer dirty
+ * but could have been modified,
+ * so it will need to be
+ * re-validated.
+ */
+ m->vmp_cs_validated = VMP_CS_ALL_FALSE;
+
+ VM_PAGEOUT_DEBUG(vm_cs_validated_resets, 1);
+
+ pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
+ }
clear_refmod |= VM_MEM_MODIFIED;
- }
- if (flags & UPL_COMMIT_INACTIVATE) {
- m->reference = FALSE;
- clear_refmod |= VM_MEM_REFERENCED;
- vm_page_deactivate(m);
}
- if (clear_refmod)
- pmap_clear_refmod(m->phys_page, clear_refmod);
-
- if (flags & UPL_COMMIT_ALLOW_ACCESS) {
+ if (upl->flags & UPL_ACCESS_BLOCKED) {
/*
* We blocked access to the pages in this UPL.
* Clear the "busy" bit and wake up any waiter
* for this page.
*/
- PAGE_WAKEUP_DONE(m);
+ dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
}
+ if (fast_path_possible) {
+ assert(m_object->purgable != VM_PURGABLE_EMPTY);
+ assert(m_object->purgable != VM_PURGABLE_VOLATILE);
+ if (m->vmp_absent) {
+ assert(m->vmp_q_state == VM_PAGE_NOT_ON_Q);
+ assert(m->vmp_wire_count == 0);
+ assert(m->vmp_busy);
+
+ m->vmp_absent = FALSE;
+ dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
+ } else {
+ if (m->vmp_wire_count == 0) {
+ panic("wire_count == 0, m = %p, obj = %p\n", m, shadow_object);
+ }
+ assert(m->vmp_q_state == VM_PAGE_IS_WIRED);
- target_offset += PAGE_SIZE_64;
- xfer_size -= PAGE_SIZE;
- entry++;
- continue;
- }
- if (delayed_unlock == 0)
- vm_page_lock_queues();
- /*
- * make sure to clear the hardware
- * modify or reference bits before
- * releasing the BUSY bit on this page
- * otherwise we risk losing a legitimate
- * change of state
- */
- if (flags & UPL_COMMIT_CLEAR_DIRTY) {
- m->dirty = FALSE;
- clear_refmod |= VM_MEM_MODIFIED;
- }
- if (flags & UPL_COMMIT_INACTIVATE)
- clear_refmod |= VM_MEM_REFERENCED;
-
- if (clear_refmod)
- pmap_clear_refmod(m->phys_page, clear_refmod);
-
- if (page_list) {
- p = &(page_list[entry]);
- if(p->phys_addr && p->pageout && !m->pageout) {
- m->busy = TRUE;
- m->pageout = TRUE;
- vm_page_wire(m);
- } else if (page_list[entry].phys_addr &&
- !p->pageout && m->pageout &&
- !m->dump_cleaning) {
- m->pageout = FALSE;
- m->absent = FALSE;
- m->overwriting = FALSE;
- vm_page_unwire(m);
- PAGE_WAKEUP_DONE(m);
+ /*
+ * XXX FBDP need to update some other
+ * counters here (purgeable_wired_count)
+ * (ledgers), ...
+ */
+ assert(m->vmp_wire_count > 0);
+ m->vmp_wire_count--;
+
+ if (m->vmp_wire_count == 0) {
+ m->vmp_q_state = VM_PAGE_NOT_ON_Q;
+ unwired_count++;
+ }
+ }
+ if (m->vmp_wire_count == 0) {
+ assert(m->vmp_pageq.next == 0 && m->vmp_pageq.prev == 0);
+
+ if (last_local == VM_PAGE_NULL) {
+ assert(first_local == VM_PAGE_NULL);
+
+ last_local = m;
+ first_local = m;
+ } else {
+ assert(first_local != VM_PAGE_NULL);
+
+ m->vmp_pageq.next = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(first_local);
+ first_local->vmp_pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(m);
+ first_local = m;
+ }
+ local_queue_count++;
+
+ if (throttle_page) {
+ m->vmp_q_state = VM_PAGE_ON_THROTTLED_Q;
+ } else {
+ if (flags & UPL_COMMIT_INACTIVATE) {
+ if (shadow_object->internal) {
+ m->vmp_q_state = VM_PAGE_ON_INACTIVE_INTERNAL_Q;
+ } else {
+ m->vmp_q_state = VM_PAGE_ON_INACTIVE_EXTERNAL_Q;
+ }
+ } else {
+ m->vmp_q_state = VM_PAGE_ON_ACTIVE_Q;
+ }
+ }
+ }
+ } else {
+ if (flags & UPL_COMMIT_INACTIVATE) {
+ dwp->dw_mask |= DW_vm_page_deactivate_internal;
+ clear_refmod |= VM_MEM_REFERENCED;
+ }
+ if (m->vmp_absent) {
+ if (flags & UPL_COMMIT_FREE_ABSENT) {
+ dwp->dw_mask |= DW_vm_page_free;
+ } else {
+ m->vmp_absent = FALSE;
+ dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
+
+ if (!(dwp->dw_mask & DW_vm_page_deactivate_internal)) {
+ dwp->dw_mask |= DW_vm_page_activate;
+ }
+ }
+ } else {
+ dwp->dw_mask |= DW_vm_page_unwire;
+ }
}
+ goto commit_next_page;
+ }
+ assert(m->vmp_q_state != VM_PAGE_USED_BY_COMPRESSOR);
+
+ if (page_list) {
page_list[entry].phys_addr = 0;
- }
- m->dump_cleaning = FALSE;
- if(m->laundry) {
- vm_pageout_throttle_up(m);
- }
- if(m->pageout) {
- m->cleaning = FALSE;
- m->pageout = FALSE;
-#if MACH_CLUSTER_STATS
- if (m->wanted) vm_pageout_target_collisions++;
-#endif
- if (pmap_disconnect(m->phys_page) & VM_MEM_MODIFIED)
- m->dirty = TRUE;
- else
- m->dirty = FALSE;
-
- if(m->dirty) {
- vm_page_unwire(m);/* reactivates */
-
- if (upl->flags & UPL_PAGEOUT) {
- CLUSTER_STAT(vm_pageout_target_page_dirtied++;)
- VM_STAT(reactivations++);
- }
- PAGE_WAKEUP_DONE(m);
- } else {
- vm_page_free(m);/* clears busy, etc. */
-
- if (upl->flags & UPL_PAGEOUT) {
- CLUSTER_STAT(vm_pageout_target_page_freed++;)
-
- if (page_list[entry].dirty)
- VM_STAT(pageouts++);
- }
- }
- if (delayed_unlock++ > DELAYED_UNLOCK_LIMIT) {
- delayed_unlock = 0;
- vm_page_unlock_queues();
- }
- target_offset += PAGE_SIZE_64;
- xfer_size -= PAGE_SIZE;
- entry++;
- continue;
- }
-#if MACH_CLUSTER_STATS
- m->dirty = pmap_is_modified(m->phys_page);
-
- if (m->dirty) vm_pageout_cluster_dirtied++;
- else vm_pageout_cluster_cleaned++;
- if (m->wanted) vm_pageout_cluster_collisions++;
-#else
- m->dirty = 0;
+ }
+
+ /*
+ * make sure to clear the hardware
+ * modify or reference bits before
+ * releasing the BUSY bit on this page
+ * otherwise we risk losing a legitimate
+ * change of state
+ */
+ if (flags & UPL_COMMIT_CLEAR_DIRTY) {
+ m->vmp_dirty = FALSE;
+
+ clear_refmod |= VM_MEM_MODIFIED;
+ }
+ if (m->vmp_laundry) {
+ dwp->dw_mask |= DW_vm_pageout_throttle_up;
+ }
+
+ if (VM_PAGE_WIRED(m)) {
+ m->vmp_free_when_done = FALSE;
+ }
+
+ if (!(flags & UPL_COMMIT_CS_VALIDATED) &&
+ m->vmp_cs_validated &&
+ m->vmp_cs_tainted != VMP_CS_ALL_TRUE) {
+ /*
+ * CODE SIGNING:
+ * This page is no longer dirty
+ * but could have been modified,
+ * so it will need to be
+ * re-validated.
+ */
+ m->vmp_cs_validated = VMP_CS_ALL_FALSE;
+
+ VM_PAGEOUT_DEBUG(vm_cs_validated_resets, 1);
+
+ pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
+ }
+ if (m->vmp_overwriting) {
+ /*
+ * the (COPY_OUT_FROM == FALSE) request_page_list case
+ */
+ if (m->vmp_busy) {
+#if CONFIG_PHANTOM_CACHE
+ if (m->vmp_absent && !m_object->internal) {
+ dwp->dw_mask |= DW_vm_phantom_cache_update;
+ }
#endif
+ m->vmp_absent = FALSE;
- if((m->busy) && (m->cleaning)) {
- /* the request_page_list case */
- if(m->absent) {
- m->absent = FALSE;
- if(shadow_object->absent_count == 1)
- vm_object_absent_release(shadow_object);
- else
- shadow_object->absent_count--;
- }
- m->overwriting = FALSE;
- m->busy = FALSE;
- m->dirty = FALSE;
- } else if (m->overwriting) {
- /* alternate request page list, write to
- * page_list case. Occurs when the original
- * page was wired at the time of the list
- * request */
- assert(m->wire_count != 0);
- vm_page_unwire(m);/* reactivates */
- m->overwriting = FALSE;
- }
- m->cleaning = FALSE;
-
- /* It is a part of the semantic of COPYOUT_FROM */
- /* UPLs that a commit implies cache sync */
- /* between the vm page and the backing store */
- /* this can be used to strip the precious bit */
- /* as well as clean */
- if (upl->flags & UPL_PAGE_SYNC_DONE)
- m->precious = FALSE;
-
- if (flags & UPL_COMMIT_SET_DIRTY)
- m->dirty = TRUE;
-
- if (flags & UPL_COMMIT_INACTIVATE) {
- m->reference = FALSE;
- vm_page_deactivate(m);
- } else if (!m->active && !m->inactive) {
- if (m->reference)
- vm_page_activate(m);
- else
- vm_page_deactivate(m);
- }
-
- if (flags & UPL_COMMIT_ALLOW_ACCESS) {
- /*
- * We blocked access to the pages in this URL.
- * Clear the "busy" bit on this page before we
- * wake up any waiter.
- */
- m->busy = FALSE;
- }
-
- /*
- * Wakeup any thread waiting for the page to be un-cleaning.
- */
- PAGE_WAKEUP(m);
-
- if (delayed_unlock++ > DELAYED_UNLOCK_LIMIT) {
- delayed_unlock = 0;
- vm_page_unlock_queues();
- }
+ dwp->dw_mask |= DW_clear_busy;
+ } else {
+ /*
+ * alternate (COPY_OUT_FROM == FALSE) page_list case
+ * Occurs when the original page was wired
+ * at the time of the list request
+ */
+ assert(VM_PAGE_WIRED(m));
+
+ dwp->dw_mask |= DW_vm_page_unwire; /* reactivates */
+ }
+ m->vmp_overwriting = FALSE;
+ }
+ m->vmp_cleaning = FALSE;
+
+ if (m->vmp_free_when_done) {
+ /*
+ * With the clean queue enabled, UPL_PAGEOUT should
+ * no longer set the pageout bit. Its pages now go
+ * to the clean queue.
+ *
+ * We don't use the cleaned Q anymore and so this
+ * assert isn't correct. The code for the clean Q
+ * still exists and might be used in the future. If we
+ * go back to the cleaned Q, we will re-enable this
+ * assert.
+ *
+ * assert(!(upl->flags & UPL_PAGEOUT));
+ */
+ assert(!m_object->internal);
+
+ m->vmp_free_when_done = FALSE;
+
+ if ((flags & UPL_COMMIT_SET_DIRTY) ||
+ (m->vmp_pmapped && (pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)) & VM_MEM_MODIFIED))) {
+ /*
+ * page was re-dirtied after we started
+ * the pageout... reactivate it since
+ * we don't know whether the on-disk
+ * copy matches what is now in memory
+ */
+ SET_PAGE_DIRTY(m, FALSE);
+
+ dwp->dw_mask |= DW_vm_page_activate | DW_PAGE_WAKEUP;
+
+ if (upl->flags & UPL_PAGEOUT) {
+ VM_STAT_INCR(reactivations);
+ DTRACE_VM2(pgrec, int, 1, (uint64_t *), NULL);
+ }
+ } else {
+ /*
+ * page has been successfully cleaned
+ * go ahead and free it for other use
+ */
+ if (m_object->internal) {
+ DTRACE_VM2(anonpgout, int, 1, (uint64_t *), NULL);
+ } else {
+ DTRACE_VM2(fspgout, int, 1, (uint64_t *), NULL);
+ }
+ m->vmp_dirty = FALSE;
+ m->vmp_busy = TRUE;
+
+ dwp->dw_mask |= DW_vm_page_free;
+ }
+ goto commit_next_page;
+ }
+ /*
+ * It is a part of the semantic of COPYOUT_FROM
+ * UPLs that a commit implies cache sync
+ * between the vm page and the backing store
+ * this can be used to strip the precious bit
+ * as well as clean
+ */
+ if ((upl->flags & UPL_PAGE_SYNC_DONE) || (flags & UPL_COMMIT_CLEAR_PRECIOUS)) {
+ m->vmp_precious = FALSE;
+ }
+
+ if (flags & UPL_COMMIT_SET_DIRTY) {
+ SET_PAGE_DIRTY(m, FALSE);
+ } else {
+ m->vmp_dirty = FALSE;
+ }
+
+ /* with the clean queue on, move *all* cleaned pages to the clean queue */
+ if (hibernate_cleaning_in_progress == FALSE && !m->vmp_dirty && (upl->flags & UPL_PAGEOUT)) {
+ pgpgout_count++;
+
+ VM_STAT_INCR(pageouts);
+ DTRACE_VM2(pgout, int, 1, (uint64_t *), NULL);
+
+ dwp->dw_mask |= DW_enqueue_cleaned;
+ } else if (should_be_throttled == TRUE && (m->vmp_q_state == VM_PAGE_NOT_ON_Q)) {
+ /*
+ * page coming back in from being 'frozen'...
+ * it was dirty before it was frozen, so keep it so
+ * the vm_page_activate will notice that it really belongs
+ * on the throttle queue and put it there
+ */
+ SET_PAGE_DIRTY(m, FALSE);
+ dwp->dw_mask |= DW_vm_page_activate;
+ } else {
+ if ((flags & UPL_COMMIT_INACTIVATE) && !m->vmp_clustered && (m->vmp_q_state != VM_PAGE_ON_SPECULATIVE_Q)) {
+ dwp->dw_mask |= DW_vm_page_deactivate_internal;
+ clear_refmod |= VM_MEM_REFERENCED;
+ } else if (!VM_PAGE_PAGEABLE(m)) {
+ if (m->vmp_clustered || (flags & UPL_COMMIT_SPECULATE)) {
+ dwp->dw_mask |= DW_vm_page_speculate;
+ } else if (m->vmp_reference) {
+ dwp->dw_mask |= DW_vm_page_activate;
+ } else {
+ dwp->dw_mask |= DW_vm_page_deactivate_internal;
+ clear_refmod |= VM_MEM_REFERENCED;
+ }
+ }
+ }
+ if (upl->flags & UPL_ACCESS_BLOCKED) {
+ /*
+ * We blocked access to the pages in this URL.
+ * Clear the "busy" bit on this page before we
+ * wake up any waiter.
+ */
+ dwp->dw_mask |= DW_clear_busy;
+ }
+ /*
+ * Wakeup any thread waiting for the page to be un-cleaning.
+ */
+ dwp->dw_mask |= DW_PAGE_WAKEUP;
+
+commit_next_page:
+ if (clear_refmod) {
+ pmap_clear_refmod(VM_PAGE_GET_PHYS_PAGE(m), clear_refmod);
}
+
target_offset += PAGE_SIZE_64;
xfer_size -= PAGE_SIZE;
entry++;
+
+ if (dwp->dw_mask) {
+ if (dwp->dw_mask & ~(DW_clear_busy | DW_PAGE_WAKEUP)) {
+ VM_PAGE_ADD_DELAYED_WORK(dwp, m, dw_count);
+
+ if (dw_count >= dw_limit) {
+ vm_page_do_delayed_work(shadow_object, VM_KERN_MEMORY_NONE, dwp_start, dw_count);
+
+ dwp = dwp_start;
+ dw_count = 0;
+ }
+ } else {
+ if (dwp->dw_mask & DW_clear_busy) {
+ m->vmp_busy = FALSE;
+ }
+
+ if (dwp->dw_mask & DW_PAGE_WAKEUP) {
+ PAGE_WAKEUP(m);
+ }
+ }
+ }
+ }
+ if (dw_count) {
+ vm_page_do_delayed_work(shadow_object, VM_KERN_MEMORY_NONE, dwp_start, dw_count);
+ dwp = dwp_start;
+ dw_count = 0;
}
- if (delayed_unlock)
- vm_page_unlock_queues();
+ if (fast_path_possible) {
+ assert(shadow_object->purgable != VM_PURGABLE_VOLATILE);
+ assert(shadow_object->purgable != VM_PURGABLE_EMPTY);
+
+ if (local_queue_count || unwired_count) {
+ if (local_queue_count) {
+ vm_page_t first_target;
+ vm_page_queue_head_t *target_queue;
+
+ if (throttle_page) {
+ target_queue = &vm_page_queue_throttled;
+ } else {
+ if (flags & UPL_COMMIT_INACTIVATE) {
+ if (shadow_object->internal) {
+ target_queue = &vm_page_queue_anonymous;
+ } else {
+ target_queue = &vm_page_queue_inactive;
+ }
+ } else {
+ target_queue = &vm_page_queue_active;
+ }
+ }
+ /*
+ * Transfer the entire local queue to a regular LRU page queues.
+ */
+ vm_page_lockspin_queues();
+
+ first_target = (vm_page_t) vm_page_queue_first(target_queue);
+
+ if (vm_page_queue_empty(target_queue)) {
+ target_queue->prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(last_local);
+ } else {
+ first_target->vmp_pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(last_local);
+ }
+
+ target_queue->next = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(first_local);
+ first_local->vmp_pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(target_queue);
+ last_local->vmp_pageq.next = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(first_target);
+
+ /*
+ * Adjust the global page counts.
+ */
+ if (throttle_page) {
+ vm_page_throttled_count += local_queue_count;
+ } else {
+ if (flags & UPL_COMMIT_INACTIVATE) {
+ if (shadow_object->internal) {
+ vm_page_anonymous_count += local_queue_count;
+ }
+ vm_page_inactive_count += local_queue_count;
+
+ token_new_pagecount += local_queue_count;
+ } else {
+ vm_page_active_count += local_queue_count;
+ }
+
+ if (shadow_object->internal) {
+ vm_page_pageable_internal_count += local_queue_count;
+ } else {
+ vm_page_pageable_external_count += local_queue_count;
+ }
+ }
+ } else {
+ vm_page_lockspin_queues();
+ }
+ if (unwired_count) {
+ vm_page_wire_count -= unwired_count;
+ VM_CHECK_MEMORYSTATUS;
+ }
+ vm_page_unlock_queues();
+
+ VM_OBJECT_WIRED_PAGE_COUNT(shadow_object, -unwired_count);
+ }
+ }
occupied = 1;
- if (upl->flags & UPL_DEVICE_MEMORY) {
+ if (upl->flags & UPL_DEVICE_MEMORY) {
occupied = 0;
} else if (upl->flags & UPL_LITE) {
- int pg_num;
- int i;
- pg_num = upl->size/PAGE_SIZE;
- pg_num = (pg_num + 31) >> 5;
+ int pg_num;
+ int i;
+
occupied = 0;
- for(i= 0; i<pg_num; i++) {
- if(lite_list[i] != 0) {
- occupied = 1;
- break;
+
+ if (!fast_path_full_commit) {
+ pg_num = upl_adjusted_size(upl, PAGE_MASK) / PAGE_SIZE;
+ pg_num = (pg_num + 31) >> 5;
+
+ for (i = 0; i < pg_num; i++) {
+ if (lite_list[i] != 0) {
+ occupied = 1;
+ break;
+ }
}
}
} else {
- if(queue_empty(&upl->map_object->memq)) {
+ if (vm_page_queue_empty(&upl->map_object->memq)) {
occupied = 0;
}
}
-
- if(occupied == 0) {
- if(upl->flags & UPL_COMMIT_NOTIFY_EMPTY) {
+ if (occupied == 0) {
+ /*
+ * If this UPL element belongs to a Vector UPL and is
+ * empty, then this is the right function to deallocate
+ * it. So go ahead set the *empty variable. The flag
+ * UPL_COMMIT_NOTIFY_EMPTY, from the caller's point of view
+ * should be considered relevant for the Vector UPL and not
+ * the internal UPLs.
+ */
+ if ((upl->flags & UPL_COMMIT_NOTIFY_EMPTY) || isVectorUPL) {
*empty = TRUE;
}
- if(object == shadow_object)
- vm_object_paging_end(shadow_object);
+
+ if (object == shadow_object && !(upl->flags & UPL_KERNEL_OBJECT)) {
+ /*
+ * this is not a paging object
+ * so we need to drop the paging reference
+ * that was taken when we created the UPL
+ * against this object
+ */
+ vm_object_activity_end(shadow_object);
+ vm_object_collapse(shadow_object, 0, TRUE);
+ } else {
+ /*
+ * we dontated the paging reference to
+ * the map object... vm_pageout_object_terminate
+ * will drop this reference
+ */
+ }
}
+ VM_OBJECT_WIRED_PAGE_UPDATE_END(shadow_object, shadow_object->wire_tag);
vm_object_unlock(shadow_object);
- if (object != shadow_object)
- vm_object_unlock(object);
- upl_unlock(upl);
+ if (object != shadow_object) {
+ vm_object_unlock(object);
+ }
- return KERN_SUCCESS;
+ if (!isVectorUPL) {
+ upl_unlock(upl);
+ } else {
+ /*
+ * If we completed our operations on an UPL that is
+ * part of a Vectored UPL and if empty is TRUE, then
+ * we should go ahead and deallocate this UPL element.
+ * Then we check if this was the last of the UPL elements
+ * within that Vectored UPL. If so, set empty to TRUE
+ * so that in ubc_upl_commit_range or ubc_upl_commit, we
+ * can go ahead and deallocate the Vector UPL too.
+ */
+ if (*empty == TRUE) {
+ *empty = vector_upl_set_subupl(vector_upl, upl, 0);
+ upl_deallocate(upl);
+ }
+ goto process_upl_to_commit;
+ }
+ if (pgpgout_count) {
+ DTRACE_VM2(pgpgout, int, pgpgout_count, (uint64_t *), NULL);
+ }
+
+ kr = KERN_SUCCESS;
+done:
+ if (dwp_start && dwp_finish_ctx) {
+ vm_page_delayed_work_finish_ctx(dwp_start);
+ dwp_start = dwp = NULL;
+ }
+
+ return kr;
}
kern_return_t
upl_abort_range(
- upl_t upl,
- upl_offset_t offset,
- upl_size_t size,
- int error,
- boolean_t *empty)
+ upl_t upl,
+ upl_offset_t offset,
+ upl_size_t size,
+ int error,
+ boolean_t *empty)
{
- upl_size_t xfer_size = size;
- vm_object_t shadow_object;
- vm_object_t object = upl->map_object;
- vm_object_offset_t target_offset;
- int entry;
- wpl_array_t lite_list;
- int occupied;
- boolean_t shadow_internal;
-
+ upl_page_info_t *user_page_list = NULL;
+ upl_size_t xfer_size, subupl_size;
+ vm_object_t shadow_object;
+ vm_object_t object;
+ vm_object_offset_t target_offset;
+ upl_offset_t subupl_offset = offset;
+ int entry;
+ wpl_array_t lite_list;
+ int occupied;
+ struct vm_page_delayed_work dw_array;
+ struct vm_page_delayed_work *dwp, *dwp_start;
+ bool dwp_finish_ctx = TRUE;
+ int dw_count;
+ int dw_limit;
+ int isVectorUPL = 0;
+ upl_t vector_upl = NULL;
+ vm_object_offset_t obj_start, obj_end, obj_offset;
+ kern_return_t kr = KERN_SUCCESS;
+
+// DEBUG4K_UPL("upl %p (u_offset 0x%llx u_size 0x%llx) object %p offset 0x%llx size 0x%llx error 0x%x\n", upl, (uint64_t)upl->u_offset, (uint64_t)upl->u_size, upl->map_object, (uint64_t)offset, (uint64_t)size, error);
+
+ dwp_start = dwp = NULL;
+
+ subupl_size = size;
*empty = FALSE;
- if (upl == UPL_NULL)
+ if (upl == UPL_NULL) {
return KERN_INVALID_ARGUMENT;
+ }
- if (upl->flags & UPL_IO_WIRE) {
- return upl_commit_range(upl,
- offset, size, 0,
- NULL, 0, empty);
+ if ((upl->flags & UPL_IO_WIRE) && !(error & UPL_ABORT_DUMP_PAGES)) {
+ return upl_commit_range(upl, offset, size, UPL_COMMIT_FREE_ABSENT, NULL, 0, empty);
}
- if(object->pageout) {
- shadow_object = object->shadow;
+ dw_count = 0;
+ dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT);
+ dwp_start = vm_page_delayed_work_get_ctx();
+ if (dwp_start == NULL) {
+ dwp_start = &dw_array;
+ dw_limit = 1;
+ dwp_finish_ctx = FALSE;
+ }
+
+ dwp = dwp_start;
+
+ if ((isVectorUPL = vector_upl_is_valid(upl))) {
+ vector_upl = upl;
+ upl_lock(vector_upl);
} else {
- shadow_object = object;
+ upl_lock(upl);
}
- upl_lock(upl);
- if(upl->flags & UPL_DEVICE_MEMORY) {
+process_upl_to_abort:
+ if (isVectorUPL) {
+ size = subupl_size;
+ offset = subupl_offset;
+ if (size == 0) {
+ upl_unlock(vector_upl);
+ kr = KERN_SUCCESS;
+ goto done;
+ }
+ upl = vector_upl_subupl_byoffset(vector_upl, &offset, &size);
+ if (upl == NULL) {
+ upl_unlock(vector_upl);
+ kr = KERN_FAILURE;
+ goto done;
+ }
+ subupl_size -= size;
+ subupl_offset += size;
+ }
+
+ *empty = FALSE;
+
+#if UPL_DEBUG
+ if (upl->upl_commit_index < UPL_DEBUG_COMMIT_RECORDS) {
+ (void) OSBacktrace(&upl->upl_commit_records[upl->upl_commit_index].c_retaddr[0], UPL_DEBUG_STACK_FRAMES);
+
+ upl->upl_commit_records[upl->upl_commit_index].c_beg = offset;
+ upl->upl_commit_records[upl->upl_commit_index].c_end = (offset + size);
+ upl->upl_commit_records[upl->upl_commit_index].c_aborted = 1;
+
+ upl->upl_commit_index++;
+ }
+#endif
+ if (upl->flags & UPL_DEVICE_MEMORY) {
xfer_size = 0;
- } else if ((offset + size) > upl->size) {
- upl_unlock(upl);
- return KERN_FAILURE;
+ } else if ((offset + size) <= upl_adjusted_size(upl, PAGE_MASK)) {
+ xfer_size = size;
+ } else {
+ if (!isVectorUPL) {
+ upl_unlock(upl);
+ } else {
+ upl_unlock(vector_upl);
+ }
+ DEBUG4K_ERROR("upl %p (u_offset 0x%llx u_size 0x%x) offset 0x%x size 0x%x\n", upl, upl->u_offset, upl->u_size, offset, size);
+ kr = KERN_FAILURE;
+ goto done;
}
- if (object != shadow_object)
- vm_object_lock(object);
- vm_object_lock(shadow_object);
+ if (upl->flags & UPL_INTERNAL) {
+ lite_list = (wpl_array_t)
+ ((((uintptr_t)upl) + sizeof(struct upl))
+ + ((upl_adjusted_size(upl, PAGE_MASK) / PAGE_SIZE) * sizeof(upl_page_info_t)));
- shadow_internal = shadow_object->internal;
+ user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl));
+ } else {
+ lite_list = (wpl_array_t)
+ (((uintptr_t)upl) + sizeof(struct upl));
+ }
+ object = upl->map_object;
- if(upl->flags & UPL_INTERNAL) {
- lite_list = (wpl_array_t)
- ((((uintptr_t)upl) + sizeof(struct upl))
- + ((upl->size/PAGE_SIZE) * sizeof(upl_page_info_t)));
+ if (upl->flags & UPL_SHADOWED) {
+ vm_object_lock(object);
+ shadow_object = object->shadow;
} else {
- lite_list = (wpl_array_t)
- (((uintptr_t)upl) + sizeof(struct upl));
+ shadow_object = object;
}
- entry = offset/PAGE_SIZE;
+ entry = offset / PAGE_SIZE;
target_offset = (vm_object_offset_t)offset;
- while(xfer_size) {
- vm_page_t t,m;
+ if (upl->flags & UPL_KERNEL_OBJECT) {
+ vm_object_lock_shared(shadow_object);
+ } else {
+ vm_object_lock(shadow_object);
+ }
+
+ if (upl->flags & UPL_ACCESS_BLOCKED) {
+ assert(shadow_object->blocked_access);
+ shadow_object->blocked_access = FALSE;
+ vm_object_wakeup(object, VM_OBJECT_EVENT_UNBLOCKED);
+ }
+
+ if ((error & UPL_ABORT_DUMP_PAGES) && (upl->flags & UPL_KERNEL_OBJECT)) {
+ panic("upl_abort_range: kernel_object being DUMPED");
+ }
+
+ obj_start = target_offset + upl->u_offset - shadow_object->paging_offset;
+ obj_end = obj_start + xfer_size;
+ obj_start = vm_object_trunc_page(obj_start);
+ obj_end = vm_object_round_page(obj_end);
+ for (obj_offset = obj_start;
+ obj_offset < obj_end;
+ obj_offset += PAGE_SIZE) {
+ vm_page_t t, m;
+ unsigned int pg_num;
+ boolean_t needed;
+
+ pg_num = (unsigned int) (target_offset / PAGE_SIZE);
+ assert(pg_num == target_offset / PAGE_SIZE);
+
+ needed = FALSE;
+
+ if (user_page_list) {
+ needed = user_page_list[pg_num].needed;
+ }
+
+ dwp->dw_mask = 0;
m = VM_PAGE_NULL;
- if(upl->flags & UPL_LITE) {
- int pg_num;
- pg_num = target_offset/PAGE_SIZE;
- if(lite_list[pg_num>>5] & (1 << (pg_num & 31))) {
- lite_list[pg_num>>5] &= ~(1 << (pg_num & 31));
- m = vm_page_lookup(shadow_object,
- target_offset + (upl->offset -
- shadow_object->paging_offset));
- }
- }
- if(object->pageout) {
- if ((t = vm_page_lookup(object, target_offset))
- != NULL) {
- t->pageout = FALSE;
+
+ if (upl->flags & UPL_LITE) {
+ if (lite_list[pg_num >> 5] & (1U << (pg_num & 31))) {
+ lite_list[pg_num >> 5] &= ~(1U << (pg_num & 31));
+
+ if (!(upl->flags & UPL_KERNEL_OBJECT)) {
+ m = vm_page_lookup(shadow_object, obj_offset);
+ }
+ }
+ }
+ if (upl->flags & UPL_SHADOWED) {
+ if ((t = vm_page_lookup(object, target_offset)) != VM_PAGE_NULL) {
+ t->vmp_free_when_done = FALSE;
+
VM_PAGE_FREE(t);
- if(m == NULL) {
- m = vm_page_lookup(
- shadow_object,
- target_offset +
- object->shadow_offset);
+
+ if (m == VM_PAGE_NULL) {
+ m = vm_page_lookup(shadow_object, target_offset + object->vo_shadow_offset);
}
- if(m != VM_PAGE_NULL)
- vm_object_paging_end(m->object);
}
}
- if(m != VM_PAGE_NULL) {
- vm_page_lock_queues();
- if(m->absent) {
- boolean_t must_free = TRUE;
-
- /* COPYOUT = FALSE case */
- /* check for error conditions which must */
- /* be passed back to the pages customer */
- if(error & UPL_ABORT_RESTART) {
- m->restart = TRUE;
- m->absent = FALSE;
- vm_object_absent_release(m->object);
- m->page_error = KERN_MEMORY_ERROR;
- m->error = TRUE;
+ if ((upl->flags & UPL_KERNEL_OBJECT)) {
+ goto abort_next_page;
+ }
+
+ if (m != VM_PAGE_NULL) {
+ assert(m->vmp_q_state != VM_PAGE_USED_BY_COMPRESSOR);
+
+ if (m->vmp_absent) {
+ boolean_t must_free = TRUE;
+
+ /*
+ * COPYOUT = FALSE case
+ * check for error conditions which must
+ * be passed back to the pages customer
+ */
+ if (error & UPL_ABORT_RESTART) {
+ m->vmp_restart = TRUE;
+ m->vmp_absent = FALSE;
+ m->vmp_unusual = TRUE;
must_free = FALSE;
- } else if(error & UPL_ABORT_UNAVAILABLE) {
- m->restart = FALSE;
- m->unusual = TRUE;
+ } else if (error & UPL_ABORT_UNAVAILABLE) {
+ m->vmp_restart = FALSE;
+ m->vmp_unusual = TRUE;
must_free = FALSE;
- } else if(error & UPL_ABORT_ERROR) {
- m->restart = FALSE;
- m->absent = FALSE;
- vm_object_absent_release(m->object);
- m->page_error = KERN_MEMORY_ERROR;
- m->error = TRUE;
+ } else if (error & UPL_ABORT_ERROR) {
+ m->vmp_restart = FALSE;
+ m->vmp_absent = FALSE;
+ m->vmp_error = TRUE;
+ m->vmp_unusual = TRUE;
must_free = FALSE;
}
+ if (m->vmp_clustered && needed == FALSE) {
+ /*
+ * This page was a part of a speculative
+ * read-ahead initiated by the kernel
+ * itself. No one is expecting this
+ * page and no one will clean up its
+ * error state if it ever becomes valid
+ * in the future.
+ * We have to free it here.
+ */
+ must_free = TRUE;
+ }
+ m->vmp_cleaning = FALSE;
- /*
- * ENCRYPTED SWAP:
- * If the page was already encrypted,
- * we don't really need to decrypt it
- * now. It will get decrypted later,
- * on demand, as soon as someone needs
- * to access its contents.
- */
+ if (m->vmp_overwriting && !m->vmp_busy) {
+ /*
+ * this shouldn't happen since
+ * this is an 'absent' page, but
+ * it doesn't hurt to check for
+ * the 'alternate' method of
+ * stabilizing the page...
+ * we will mark 'busy' to be cleared
+ * in the following code which will
+ * take care of the primary stabilzation
+ * method (i.e. setting 'busy' to TRUE)
+ */
+ dwp->dw_mask |= DW_vm_page_unwire;
+ }
+ m->vmp_overwriting = FALSE;
- m->cleaning = FALSE;
- m->overwriting = FALSE;
- PAGE_WAKEUP_DONE(m);
+ dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
if (must_free == TRUE) {
- vm_page_free(m);
+ dwp->dw_mask |= DW_vm_page_free;
} else {
- vm_page_activate(m);
+ dwp->dw_mask |= DW_vm_page_activate;
}
- vm_page_unlock_queues();
-
- target_offset += PAGE_SIZE_64;
- xfer_size -= PAGE_SIZE;
- entry++;
- continue;
- }
- /*
- * Handle the trusted pager throttle.
- */
- if (m->laundry) {
- vm_pageout_throttle_up(m);
- }
- if(m->pageout) {
- assert(m->busy);
- assert(m->wire_count == 1);
- m->pageout = FALSE;
- vm_page_unwire(m);
- }
- m->dump_cleaning = FALSE;
- m->cleaning = FALSE;
- m->overwriting = FALSE;
-#if MACH_PAGEMAP
- vm_external_state_clr(
- m->object->existence_map, m->offset);
-#endif /* MACH_PAGEMAP */
- if(error & UPL_ABORT_DUMP_PAGES) {
- vm_page_free(m);
- pmap_disconnect(m->phys_page);
} else {
- PAGE_WAKEUP_DONE(m);
+ /*
+ * Handle the trusted pager throttle.
+ */
+ if (m->vmp_laundry) {
+ dwp->dw_mask |= DW_vm_pageout_throttle_up;
+ }
+
+ if (upl->flags & UPL_ACCESS_BLOCKED) {
+ /*
+ * We blocked access to the pages in this UPL.
+ * Clear the "busy" bit and wake up any waiter
+ * for this page.
+ */
+ dwp->dw_mask |= DW_clear_busy;
+ }
+ if (m->vmp_overwriting) {
+ if (m->vmp_busy) {
+ dwp->dw_mask |= DW_clear_busy;
+ } else {
+ /*
+ * deal with the 'alternate' method
+ * of stabilizing the page...
+ * we will either free the page
+ * or mark 'busy' to be cleared
+ * in the following code which will
+ * take care of the primary stabilzation
+ * method (i.e. setting 'busy' to TRUE)
+ */
+ dwp->dw_mask |= DW_vm_page_unwire;
+ }
+ m->vmp_overwriting = FALSE;
+ }
+ m->vmp_free_when_done = FALSE;
+ m->vmp_cleaning = FALSE;
+
+ if (error & UPL_ABORT_DUMP_PAGES) {
+ pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
+
+ dwp->dw_mask |= DW_vm_page_free;
+ } else {
+ if (!(dwp->dw_mask & DW_vm_page_unwire)) {
+ if (error & UPL_ABORT_REFERENCE) {
+ /*
+ * we've been told to explictly
+ * reference this page... for
+ * file I/O, this is done by
+ * implementing an LRU on the inactive q
+ */
+ dwp->dw_mask |= DW_vm_page_lru;
+ } else if (!VM_PAGE_PAGEABLE(m)) {
+ dwp->dw_mask |= DW_vm_page_deactivate_internal;
+ }
+ }
+ dwp->dw_mask |= DW_PAGE_WAKEUP;
+ }
}
- vm_page_unlock_queues();
}
+abort_next_page:
target_offset += PAGE_SIZE_64;
xfer_size -= PAGE_SIZE;
entry++;
+
+ if (dwp->dw_mask) {
+ if (dwp->dw_mask & ~(DW_clear_busy | DW_PAGE_WAKEUP)) {
+ VM_PAGE_ADD_DELAYED_WORK(dwp, m, dw_count);
+
+ if (dw_count >= dw_limit) {
+ vm_page_do_delayed_work(shadow_object, VM_KERN_MEMORY_NONE, dwp_start, dw_count);
+
+ dwp = dwp_start;
+ dw_count = 0;
+ }
+ } else {
+ if (dwp->dw_mask & DW_clear_busy) {
+ m->vmp_busy = FALSE;
+ }
+
+ if (dwp->dw_mask & DW_PAGE_WAKEUP) {
+ PAGE_WAKEUP(m);
+ }
+ }
+ }
}
+ if (dw_count) {
+ vm_page_do_delayed_work(shadow_object, VM_KERN_MEMORY_NONE, dwp_start, dw_count);
+ dwp = dwp_start;
+ dw_count = 0;
+ }
+
occupied = 1;
- if (upl->flags & UPL_DEVICE_MEMORY) {
+
+ if (upl->flags & UPL_DEVICE_MEMORY) {
occupied = 0;
} else if (upl->flags & UPL_LITE) {
- int pg_num;
- int i;
- pg_num = upl->size/PAGE_SIZE;
+ int pg_num;
+ int i;
+
+ pg_num = upl_adjusted_size(upl, PAGE_MASK) / PAGE_SIZE;
pg_num = (pg_num + 31) >> 5;
occupied = 0;
- for(i= 0; i<pg_num; i++) {
- if(lite_list[i] != 0) {
+
+ for (i = 0; i < pg_num; i++) {
+ if (lite_list[i] != 0) {
occupied = 1;
break;
}
}
} else {
- if(queue_empty(&upl->map_object->memq)) {
+ if (vm_page_queue_empty(&upl->map_object->memq)) {
occupied = 0;
}
}
-
- if(occupied == 0) {
- if(upl->flags & UPL_COMMIT_NOTIFY_EMPTY) {
+ if (occupied == 0) {
+ /*
+ * If this UPL element belongs to a Vector UPL and is
+ * empty, then this is the right function to deallocate
+ * it. So go ahead set the *empty variable. The flag
+ * UPL_COMMIT_NOTIFY_EMPTY, from the caller's point of view
+ * should be considered relevant for the Vector UPL and
+ * not the internal UPLs.
+ */
+ if ((upl->flags & UPL_COMMIT_NOTIFY_EMPTY) || isVectorUPL) {
*empty = TRUE;
}
- if(object == shadow_object)
- vm_object_paging_end(shadow_object);
+
+ if (object == shadow_object && !(upl->flags & UPL_KERNEL_OBJECT)) {
+ /*
+ * this is not a paging object
+ * so we need to drop the paging reference
+ * that was taken when we created the UPL
+ * against this object
+ */
+ vm_object_activity_end(shadow_object);
+ vm_object_collapse(shadow_object, 0, TRUE);
+ } else {
+ /*
+ * we dontated the paging reference to
+ * the map object... vm_pageout_object_terminate
+ * will drop this reference
+ */
+ }
}
vm_object_unlock(shadow_object);
- if (object != shadow_object)
- vm_object_unlock(object);
+ if (object != shadow_object) {
+ vm_object_unlock(object);
+ }
- upl_unlock(upl);
+ if (!isVectorUPL) {
+ upl_unlock(upl);
+ } else {
+ /*
+ * If we completed our operations on an UPL that is
+ * part of a Vectored UPL and if empty is TRUE, then
+ * we should go ahead and deallocate this UPL element.
+ * Then we check if this was the last of the UPL elements
+ * within that Vectored UPL. If so, set empty to TRUE
+ * so that in ubc_upl_abort_range or ubc_upl_abort, we
+ * can go ahead and deallocate the Vector UPL too.
+ */
+ if (*empty == TRUE) {
+ *empty = vector_upl_set_subupl(vector_upl, upl, 0);
+ upl_deallocate(upl);
+ }
+ goto process_upl_to_abort;
+ }
- return KERN_SUCCESS;
+ kr = KERN_SUCCESS;
+
+done:
+ if (dwp_start && dwp_finish_ctx) {
+ vm_page_delayed_work_finish_ctx(dwp_start);
+ dwp_start = dwp = NULL;
+ }
+
+ return kr;
}
+
kern_return_t
upl_abort(
- upl_t upl,
- int error)
+ upl_t upl,
+ int error)
{
- vm_object_t object = NULL;
- vm_object_t shadow_object = NULL;
- vm_object_offset_t offset;
- vm_object_offset_t shadow_offset;
- vm_object_offset_t target_offset;
- upl_size_t i;
- wpl_array_t lite_list;
- vm_page_t t,m;
- int occupied;
- boolean_t shadow_internal;
-
- if (upl == UPL_NULL)
+ boolean_t empty;
+
+ if (upl == UPL_NULL) {
return KERN_INVALID_ARGUMENT;
+ }
+
+ return upl_abort_range(upl, 0, upl->u_size, error, &empty);
+}
+
+
+/* an option on commit should be wire */
+kern_return_t
+upl_commit(
+ upl_t upl,
+ upl_page_info_t *page_list,
+ mach_msg_type_number_t count)
+{
+ boolean_t empty;
- if (upl->flags & UPL_IO_WIRE) {
- boolean_t empty;
- return upl_commit_range(upl,
- 0, upl->size, 0,
- NULL, 0, &empty);
+ if (upl == UPL_NULL) {
+ return KERN_INVALID_ARGUMENT;
}
- upl_lock(upl);
- if(upl->flags & UPL_DEVICE_MEMORY) {
- upl_unlock(upl);
- return KERN_SUCCESS;
+ return upl_commit_range(upl, 0, upl->u_size, 0,
+ page_list, count, &empty);
+}
+
+
+void
+iopl_valid_data(
+ upl_t upl,
+ vm_tag_t tag)
+{
+ vm_object_t object;
+ vm_offset_t offset;
+ vm_page_t m, nxt_page = VM_PAGE_NULL;
+ upl_size_t size;
+ int wired_count = 0;
+
+ if (upl == NULL) {
+ panic("iopl_valid_data: NULL upl");
+ }
+ if (vector_upl_is_valid(upl)) {
+ panic("iopl_valid_data: vector upl");
+ }
+ if ((upl->flags & (UPL_DEVICE_MEMORY | UPL_SHADOWED | UPL_ACCESS_BLOCKED | UPL_IO_WIRE | UPL_INTERNAL)) != UPL_IO_WIRE) {
+ panic("iopl_valid_data: unsupported upl, flags = %x", upl->flags);
}
object = upl->map_object;
- if (object == NULL) {
- panic("upl_abort: upl object is not backed by an object");
- upl_unlock(upl);
- return KERN_INVALID_ARGUMENT;
+ if (object == kernel_object || object == compressor_object) {
+ panic("iopl_valid_data: object == kernel or compressor");
}
- if(object->pageout) {
- shadow_object = object->shadow;
- shadow_offset = object->shadow_offset;
- } else {
- shadow_object = object;
- shadow_offset = upl->offset - object->paging_offset;
+ if (object->purgable == VM_PURGABLE_VOLATILE ||
+ object->purgable == VM_PURGABLE_EMPTY) {
+ panic("iopl_valid_data: object %p purgable %d",
+ object, object->purgable);
}
- if(upl->flags & UPL_INTERNAL) {
- lite_list = (wpl_array_t)
- ((((uintptr_t)upl) + sizeof(struct upl))
- + ((upl->size/PAGE_SIZE) * sizeof(upl_page_info_t)));
+ size = upl_adjusted_size(upl, PAGE_MASK);
+
+ vm_object_lock(object);
+ VM_OBJECT_WIRED_PAGE_UPDATE_START(object);
+
+ if (object->vo_size == size && object->resident_page_count == (size / PAGE_SIZE)) {
+ nxt_page = (vm_page_t)vm_page_queue_first(&object->memq);
} else {
- lite_list = (wpl_array_t)
- (((uintptr_t)upl) + sizeof(struct upl));
+ offset = (vm_offset_t)(upl_adjusted_offset(upl, PAGE_MASK) - object->paging_offset);
}
- offset = 0;
-
- if (object != shadow_object)
- vm_object_lock(object);
- vm_object_lock(shadow_object);
- shadow_internal = shadow_object->internal;
+ while (size) {
+ if (nxt_page != VM_PAGE_NULL) {
+ m = nxt_page;
+ nxt_page = (vm_page_t)vm_page_queue_next(&nxt_page->vmp_listq);
+ } else {
+ m = vm_page_lookup(object, offset);
+ offset += PAGE_SIZE;
- for(i = 0; i<(upl->size); i+=PAGE_SIZE, offset += PAGE_SIZE_64) {
- m = VM_PAGE_NULL;
- target_offset = offset + shadow_offset;
- if(upl->flags & UPL_LITE) {
- int pg_num;
- pg_num = offset/PAGE_SIZE;
- if(lite_list[pg_num>>5] & (1 << (pg_num & 31))) {
- lite_list[pg_num>>5] &= ~(1 << (pg_num & 31));
- m = vm_page_lookup(
- shadow_object, target_offset);
- }
- }
- if(object->pageout) {
- if ((t = vm_page_lookup(object, offset)) != NULL) {
- t->pageout = FALSE;
- VM_PAGE_FREE(t);
- if(m == NULL) {
- m = vm_page_lookup(
- shadow_object, target_offset);
- }
- if(m != VM_PAGE_NULL)
- vm_object_paging_end(m->object);
+ if (m == VM_PAGE_NULL) {
+ panic("iopl_valid_data: missing expected page at offset %lx", (long)offset);
}
}
- if(m != VM_PAGE_NULL) {
- vm_page_lock_queues();
- if(m->absent) {
- boolean_t must_free = TRUE;
-
- /* COPYOUT = FALSE case */
- /* check for error conditions which must */
- /* be passed back to the pages customer */
- if(error & UPL_ABORT_RESTART) {
- m->restart = TRUE;
- m->absent = FALSE;
- vm_object_absent_release(m->object);
- m->page_error = KERN_MEMORY_ERROR;
- m->error = TRUE;
- must_free = FALSE;
- } else if(error & UPL_ABORT_UNAVAILABLE) {
- m->restart = FALSE;
- m->unusual = TRUE;
- must_free = FALSE;
- } else if(error & UPL_ABORT_ERROR) {
- m->restart = FALSE;
- m->absent = FALSE;
- vm_object_absent_release(m->object);
- m->page_error = KERN_MEMORY_ERROR;
- m->error = TRUE;
- must_free = FALSE;
- }
-
- /*
- * ENCRYPTED SWAP:
- * If the page was already encrypted,
- * we don't really need to decrypt it
- * now. It will get decrypted later,
- * on demand, as soon as someone needs
- * to access its contents.
- */
-
- m->cleaning = FALSE;
- m->overwriting = FALSE;
- PAGE_WAKEUP_DONE(m);
+ if (m->vmp_busy) {
+ if (!m->vmp_absent) {
+ panic("iopl_valid_data: busy page w/o absent");
+ }
- if (must_free == TRUE) {
- vm_page_free(m);
- } else {
- vm_page_activate(m);
- }
- vm_page_unlock_queues();
- continue;
+ if (m->vmp_pageq.next || m->vmp_pageq.prev) {
+ panic("iopl_valid_data: busy+absent page on page queue");
}
- /*
- * Handle the trusted pager throttle.
- */
- if (m->laundry) {
- vm_pageout_throttle_up(m);
- }
- if(m->pageout) {
- assert(m->busy);
- assert(m->wire_count == 1);
- m->pageout = FALSE;
- vm_page_unwire(m);
- }
- m->dump_cleaning = FALSE;
- m->cleaning = FALSE;
- m->overwriting = FALSE;
-#if MACH_PAGEMAP
- vm_external_state_clr(
- m->object->existence_map, m->offset);
-#endif /* MACH_PAGEMAP */
- if(error & UPL_ABORT_DUMP_PAGES) {
- vm_page_free(m);
- pmap_disconnect(m->phys_page);
+ if (m->vmp_reusable) {
+ panic("iopl_valid_data: %p is reusable", m);
+ }
+
+ m->vmp_absent = FALSE;
+ m->vmp_dirty = TRUE;
+ assert(m->vmp_q_state == VM_PAGE_NOT_ON_Q);
+ assert(m->vmp_wire_count == 0);
+ m->vmp_wire_count++;
+ assert(m->vmp_wire_count);
+ if (m->vmp_wire_count == 1) {
+ m->vmp_q_state = VM_PAGE_IS_WIRED;
+ wired_count++;
} else {
- PAGE_WAKEUP_DONE(m);
+ panic("iopl_valid_data: %p already wired\n", m);
}
- vm_page_unlock_queues();
+
+ PAGE_WAKEUP_DONE(m);
+ }
+ size -= PAGE_SIZE;
+ }
+ if (wired_count) {
+ VM_OBJECT_WIRED_PAGE_COUNT(object, wired_count);
+ assert(object->resident_page_count >= object->wired_page_count);
+
+ /* no need to adjust purgeable accounting for this object: */
+ assert(object->purgable != VM_PURGABLE_VOLATILE);
+ assert(object->purgable != VM_PURGABLE_EMPTY);
+
+ vm_page_lockspin_queues();
+ vm_page_wire_count += wired_count;
+ vm_page_unlock_queues();
+ }
+ VM_OBJECT_WIRED_PAGE_UPDATE_END(object, tag);
+ vm_object_unlock(object);
+}
+
+
+void
+vm_object_set_pmap_cache_attr(
+ vm_object_t object,
+ upl_page_info_array_t user_page_list,
+ unsigned int num_pages,
+ boolean_t batch_pmap_op)
+{
+ unsigned int cache_attr = 0;
+
+ cache_attr = object->wimg_bits & VM_WIMG_MASK;
+ assert(user_page_list);
+ if (cache_attr != VM_WIMG_USE_DEFAULT) {
+ PMAP_BATCH_SET_CACHE_ATTR(object, user_page_list, cache_attr, num_pages, batch_pmap_op);
+ }
+}
+
+
+boolean_t vm_object_iopl_wire_full(vm_object_t, upl_t, upl_page_info_array_t, wpl_array_t, upl_control_flags_t, vm_tag_t);
+kern_return_t vm_object_iopl_wire_empty(vm_object_t, upl_t, upl_page_info_array_t, wpl_array_t, upl_control_flags_t, vm_tag_t, vm_object_offset_t *, int, int*);
+
+
+
+boolean_t
+vm_object_iopl_wire_full(vm_object_t object, upl_t upl, upl_page_info_array_t user_page_list,
+ wpl_array_t lite_list, upl_control_flags_t cntrl_flags, vm_tag_t tag)
+{
+ vm_page_t dst_page;
+ unsigned int entry;
+ int page_count;
+ int delayed_unlock = 0;
+ boolean_t retval = TRUE;
+ ppnum_t phys_page;
+
+ vm_object_lock_assert_exclusive(object);
+ assert(object->purgable != VM_PURGABLE_VOLATILE);
+ assert(object->purgable != VM_PURGABLE_EMPTY);
+ assert(object->pager == NULL);
+ assert(object->copy == NULL);
+ assert(object->shadow == NULL);
+
+ page_count = object->resident_page_count;
+ dst_page = (vm_page_t)vm_page_queue_first(&object->memq);
+
+ vm_page_lock_queues();
+
+ while (page_count--) {
+ if (dst_page->vmp_busy ||
+ dst_page->vmp_fictitious ||
+ dst_page->vmp_absent ||
+ dst_page->vmp_error ||
+ dst_page->vmp_cleaning ||
+ dst_page->vmp_restart ||
+ dst_page->vmp_laundry) {
+ retval = FALSE;
+ goto done;
+ }
+ if ((cntrl_flags & UPL_REQUEST_FORCE_COHERENCY) && dst_page->vmp_written_by_kernel == TRUE) {
+ retval = FALSE;
+ goto done;
+ }
+ dst_page->vmp_reference = TRUE;
+
+ vm_page_wire(dst_page, tag, FALSE);
+
+ if (!(cntrl_flags & UPL_COPYOUT_FROM)) {
+ SET_PAGE_DIRTY(dst_page, FALSE);
}
- }
- occupied = 1;
- if (upl->flags & UPL_DEVICE_MEMORY) {
- occupied = 0;
- } else if (upl->flags & UPL_LITE) {
- int pg_num;
- int j;
- pg_num = upl->size/PAGE_SIZE;
- pg_num = (pg_num + 31) >> 5;
- occupied = 0;
- for(j= 0; j<pg_num; j++) {
- if(lite_list[j] != 0) {
- occupied = 1;
- break;
- }
+ entry = (unsigned int)(dst_page->vmp_offset / PAGE_SIZE);
+ assert(entry >= 0 && entry < object->resident_page_count);
+ lite_list[entry >> 5] |= 1U << (entry & 31);
+
+ phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page);
+
+ if (phys_page > upl->highest_page) {
+ upl->highest_page = phys_page;
}
- } else {
- if(queue_empty(&upl->map_object->memq)) {
- occupied = 0;
+
+ if (user_page_list) {
+ user_page_list[entry].phys_addr = phys_page;
+ user_page_list[entry].absent = dst_page->vmp_absent;
+ user_page_list[entry].dirty = dst_page->vmp_dirty;
+ user_page_list[entry].free_when_done = dst_page->vmp_free_when_done;
+ user_page_list[entry].precious = dst_page->vmp_precious;
+ user_page_list[entry].device = FALSE;
+ user_page_list[entry].speculative = FALSE;
+ user_page_list[entry].cs_validated = FALSE;
+ user_page_list[entry].cs_tainted = FALSE;
+ user_page_list[entry].cs_nx = FALSE;
+ user_page_list[entry].needed = FALSE;
+ user_page_list[entry].mark = FALSE;
}
- }
+ if (delayed_unlock++ > 256) {
+ delayed_unlock = 0;
+ lck_mtx_yield(&vm_page_queue_lock);
- if(occupied == 0) {
- if(object == shadow_object)
- vm_object_paging_end(shadow_object);
+ VM_CHECK_MEMORYSTATUS;
+ }
+ dst_page = (vm_page_t)vm_page_queue_next(&dst_page->vmp_listq);
}
- vm_object_unlock(shadow_object);
- if (object != shadow_object)
- vm_object_unlock(object);
+done:
+ vm_page_unlock_queues();
- upl_unlock(upl);
- return KERN_SUCCESS;
+ VM_CHECK_MEMORYSTATUS;
+
+ return retval;
}
-/* an option on commit should be wire */
+
kern_return_t
-upl_commit(
- upl_t upl,
- upl_page_info_t *page_list,
- mach_msg_type_number_t count)
+vm_object_iopl_wire_empty(vm_object_t object, upl_t upl, upl_page_info_array_t user_page_list,
+ wpl_array_t lite_list, upl_control_flags_t cntrl_flags, vm_tag_t tag, vm_object_offset_t *dst_offset,
+ int page_count, int* page_grab_count)
{
- if (upl == UPL_NULL)
- return KERN_INVALID_ARGUMENT;
+ vm_page_t dst_page;
+ boolean_t no_zero_fill = FALSE;
+ int interruptible;
+ int pages_wired = 0;
+ int pages_inserted = 0;
+ int entry = 0;
+ uint64_t delayed_ledger_update = 0;
+ kern_return_t ret = KERN_SUCCESS;
+ int grab_options;
+ ppnum_t phys_page;
+
+ vm_object_lock_assert_exclusive(object);
+ assert(object->purgable != VM_PURGABLE_VOLATILE);
+ assert(object->purgable != VM_PURGABLE_EMPTY);
+ assert(object->pager == NULL);
+ assert(object->copy == NULL);
+ assert(object->shadow == NULL);
+
+ if (cntrl_flags & UPL_SET_INTERRUPTIBLE) {
+ interruptible = THREAD_ABORTSAFE;
+ } else {
+ interruptible = THREAD_UNINT;
+ }
- if(upl->flags & (UPL_LITE | UPL_IO_WIRE)) {
- boolean_t empty;
- return upl_commit_range(upl, 0, upl->size, 0,
- page_list, count, &empty);
+ if (cntrl_flags & (UPL_NOZEROFILL | UPL_NOZEROFILLIO)) {
+ no_zero_fill = TRUE;
}
- if (count == 0)
- page_list = NULL;
+ grab_options = 0;
+#if CONFIG_SECLUDED_MEMORY
+ if (object->can_grab_secluded) {
+ grab_options |= VM_PAGE_GRAB_SECLUDED;
+ }
+#endif /* CONFIG_SECLUDED_MEMORY */
- upl_lock(upl);
- if (upl->flags & UPL_DEVICE_MEMORY)
- page_list = NULL;
+ while (page_count--) {
+ while ((dst_page = vm_page_grab_options(grab_options))
+ == VM_PAGE_NULL) {
+ OSAddAtomic(page_count, &vm_upl_wait_for_pages);
- if (upl->flags & UPL_ENCRYPTED) {
- /*
- * ENCRYPTED SWAP:
- * This UPL was encrypted, but we don't need
- * to decrypt here. We'll decrypt each page
- * later, on demand, as soon as someone needs
- * to access the page's contents.
- */
- }
+ VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_START, vm_upl_wait_for_pages, 0, 0, 0);
- if ((upl->flags & UPL_CLEAR_DIRTY) ||
- (upl->flags & UPL_PAGE_SYNC_DONE) || page_list) {
- vm_object_t shadow_object = upl->map_object->shadow;
- vm_object_t object = upl->map_object;
- vm_object_offset_t target_offset;
- upl_size_t xfer_end;
- int entry;
+ if (vm_page_wait(interruptible) == FALSE) {
+ /*
+ * interrupted case
+ */
+ OSAddAtomic(-page_count, &vm_upl_wait_for_pages);
- vm_page_t t, m;
- upl_page_info_t *p;
+ VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, -1);
- if (object != shadow_object)
- vm_object_lock(object);
- vm_object_lock(shadow_object);
+ ret = MACH_SEND_INTERRUPTED;
+ goto done;
+ }
+ OSAddAtomic(-page_count, &vm_upl_wait_for_pages);
- entry = 0;
- target_offset = object->shadow_offset;
- xfer_end = upl->size + object->shadow_offset;
+ VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, 0);
+ }
+ if (no_zero_fill == FALSE) {
+ vm_page_zero_fill(dst_page);
+ } else {
+ dst_page->vmp_absent = TRUE;
+ }
- while(target_offset < xfer_end) {
+ dst_page->vmp_reference = TRUE;
- if ((t = vm_page_lookup(object,
- target_offset - object->shadow_offset))
- == NULL) {
- target_offset += PAGE_SIZE_64;
- entry++;
- continue;
- }
+ if (!(cntrl_flags & UPL_COPYOUT_FROM)) {
+ SET_PAGE_DIRTY(dst_page, FALSE);
+ }
+ if (dst_page->vmp_absent == FALSE) {
+ assert(dst_page->vmp_q_state == VM_PAGE_NOT_ON_Q);
+ assert(dst_page->vmp_wire_count == 0);
+ dst_page->vmp_wire_count++;
+ dst_page->vmp_q_state = VM_PAGE_IS_WIRED;
+ assert(dst_page->vmp_wire_count);
+ pages_wired++;
+ PAGE_WAKEUP_DONE(dst_page);
+ }
+ pages_inserted++;
- m = vm_page_lookup(shadow_object, target_offset);
- if(m != VM_PAGE_NULL) {
- /*
- * ENCRYPTED SWAP:
- * If this page was encrypted, we
- * don't need to decrypt it here.
- * We'll decrypt it later, on demand,
- * as soon as someone needs to access
- * its contents.
- */
-
- if (upl->flags & UPL_CLEAR_DIRTY) {
- pmap_clear_modify(m->phys_page);
- m->dirty = FALSE;
- }
- /* It is a part of the semantic of */
- /* COPYOUT_FROM UPLs that a commit */
- /* implies cache sync between the */
- /* vm page and the backing store */
- /* this can be used to strip the */
- /* precious bit as well as clean */
- if (upl->flags & UPL_PAGE_SYNC_DONE)
- m->precious = FALSE;
-
- if(page_list) {
- p = &(page_list[entry]);
- if(page_list[entry].phys_addr &&
- p->pageout && !m->pageout) {
- vm_page_lock_queues();
- m->busy = TRUE;
- m->pageout = TRUE;
- vm_page_wire(m);
- vm_page_unlock_queues();
- } else if (page_list[entry].phys_addr &&
- !p->pageout && m->pageout &&
- !m->dump_cleaning) {
- vm_page_lock_queues();
- m->pageout = FALSE;
- m->absent = FALSE;
- m->overwriting = FALSE;
- vm_page_unwire(m);
- PAGE_WAKEUP_DONE(m);
- vm_page_unlock_queues();
- }
- page_list[entry].phys_addr = 0;
- }
- }
- target_offset += PAGE_SIZE_64;
- entry++;
+ vm_page_insert_internal(dst_page, object, *dst_offset, tag, FALSE, TRUE, TRUE, TRUE, &delayed_ledger_update);
+
+ lite_list[entry >> 5] |= 1U << (entry & 31);
+
+ phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page);
+
+ if (phys_page > upl->highest_page) {
+ upl->highest_page = phys_page;
}
- vm_object_unlock(shadow_object);
- if (object != shadow_object)
- vm_object_unlock(object);
+ if (user_page_list) {
+ user_page_list[entry].phys_addr = phys_page;
+ user_page_list[entry].absent = dst_page->vmp_absent;
+ user_page_list[entry].dirty = dst_page->vmp_dirty;
+ user_page_list[entry].free_when_done = FALSE;
+ user_page_list[entry].precious = FALSE;
+ user_page_list[entry].device = FALSE;
+ user_page_list[entry].speculative = FALSE;
+ user_page_list[entry].cs_validated = FALSE;
+ user_page_list[entry].cs_tainted = FALSE;
+ user_page_list[entry].cs_nx = FALSE;
+ user_page_list[entry].needed = FALSE;
+ user_page_list[entry].mark = FALSE;
+ }
+ entry++;
+ *dst_offset += PAGE_SIZE_64;
}
- if (upl->flags & UPL_DEVICE_MEMORY) {
- vm_object_lock(upl->map_object->shadow);
- if(upl->map_object == upl->map_object->shadow)
- vm_object_paging_end(upl->map_object->shadow);
- vm_object_unlock(upl->map_object->shadow);
+done:
+ if (pages_wired) {
+ vm_page_lockspin_queues();
+ vm_page_wire_count += pages_wired;
+ vm_page_unlock_queues();
}
- upl_unlock(upl);
- return KERN_SUCCESS;
+ if (pages_inserted) {
+ if (object->internal) {
+ OSAddAtomic(pages_inserted, &vm_page_internal_count);
+ } else {
+ OSAddAtomic(pages_inserted, &vm_page_external_count);
+ }
+ }
+ if (delayed_ledger_update) {
+ task_t owner;
+ int ledger_idx_volatile;
+ int ledger_idx_nonvolatile;
+ int ledger_idx_volatile_compressed;
+ int ledger_idx_nonvolatile_compressed;
+ boolean_t do_footprint;
+
+ owner = VM_OBJECT_OWNER(object);
+ assert(owner);
+
+ vm_object_ledger_tag_ledgers(object,
+ &ledger_idx_volatile,
+ &ledger_idx_nonvolatile,
+ &ledger_idx_volatile_compressed,
+ &ledger_idx_nonvolatile_compressed,
+ &do_footprint);
+
+ /* more non-volatile bytes */
+ ledger_credit(owner->ledger,
+ ledger_idx_nonvolatile,
+ delayed_ledger_update);
+ if (do_footprint) {
+ /* more footprint */
+ ledger_credit(owner->ledger,
+ task_ledgers.phys_footprint,
+ delayed_ledger_update);
+ }
+ }
+
+ assert(page_grab_count);
+ *page_grab_count = pages_inserted;
+
+ return ret;
}
kern_return_t
vm_object_iopl_request(
- vm_object_t object,
- vm_object_offset_t offset,
- upl_size_t size,
- upl_t *upl_ptr,
- upl_page_info_array_t user_page_list,
- unsigned int *page_list_count,
- int cntrl_flags)
+ vm_object_t object,
+ vm_object_offset_t offset,
+ upl_size_t size,
+ upl_t *upl_ptr,
+ upl_page_info_array_t user_page_list,
+ unsigned int *page_list_count,
+ upl_control_flags_t cntrl_flags,
+ vm_tag_t tag)
{
- vm_page_t dst_page;
- vm_object_offset_t dst_offset = offset;
- upl_size_t xfer_size = size;
- upl_t upl = NULL;
- unsigned int entry;
- wpl_array_t lite_list = NULL;
- int page_field_size;
- int delayed_unlock = 0;
- int no_zero_fill = FALSE;
- vm_page_t alias_page = NULL;
- kern_return_t ret;
- vm_prot_t prot;
-
+ vm_page_t dst_page;
+ vm_object_offset_t dst_offset;
+ upl_size_t xfer_size;
+ upl_t upl = NULL;
+ unsigned int entry;
+ wpl_array_t lite_list = NULL;
+ int no_zero_fill = FALSE;
+ unsigned int size_in_pages;
+ int page_grab_count = 0;
+ u_int32_t psize;
+ kern_return_t ret;
+ vm_prot_t prot;
+ struct vm_object_fault_info fault_info = {};
+ struct vm_page_delayed_work dw_array;
+ struct vm_page_delayed_work *dwp, *dwp_start;
+ bool dwp_finish_ctx = TRUE;
+ int dw_count;
+ int dw_limit;
+ int dw_index;
+ boolean_t caller_lookup;
+ int io_tracking_flag = 0;
+ int interruptible;
+ ppnum_t phys_page;
+
+ boolean_t set_cache_attr_needed = FALSE;
+ boolean_t free_wired_pages = FALSE;
+ boolean_t fast_path_empty_req = FALSE;
+ boolean_t fast_path_full_req = FALSE;
+
+#if DEVELOPMENT || DEBUG
+ task_t task = current_task();
+#endif /* DEVELOPMENT || DEBUG */
+
+ dwp_start = dwp = NULL;
+
+ vm_object_offset_t original_offset = offset;
+ upl_size_t original_size = size;
+
+// DEBUG4K_UPL("object %p offset 0x%llx size 0x%llx cntrl_flags 0x%llx\n", object, (uint64_t)offset, (uint64_t)size, cntrl_flags);
+
+ size = (upl_size_t)(vm_object_round_page(offset + size) - vm_object_trunc_page(offset));
+ offset = vm_object_trunc_page(offset);
+ if (size != original_size || offset != original_offset) {
+ DEBUG4K_IOKIT("flags 0x%llx object %p offset 0x%llx size 0x%x -> offset 0x%llx size 0x%x\n", cntrl_flags, object, original_offset, original_size, offset, size);
+ }
if (cntrl_flags & ~UPL_VALID_FLAGS) {
/*
*/
return KERN_INVALID_VALUE;
}
-
- if (cntrl_flags & UPL_ENCRYPT) {
- /*
- * ENCRYPTED SWAP:
- * The paging path doesn't use this interface,
- * so we don't support the UPL_ENCRYPT flag
- * here. We won't encrypt the pages.
- */
- assert(! (cntrl_flags & UPL_ENCRYPT));
+ if (vm_lopage_needed == FALSE) {
+ cntrl_flags &= ~UPL_NEED_32BIT_ADDR;
}
- if (cntrl_flags & UPL_NOZEROFILL)
- no_zero_fill = TRUE;
+ if (cntrl_flags & UPL_NEED_32BIT_ADDR) {
+ if ((cntrl_flags & (UPL_SET_IO_WIRE | UPL_SET_LITE)) != (UPL_SET_IO_WIRE | UPL_SET_LITE)) {
+ return KERN_INVALID_VALUE;
+ }
+
+ if (object->phys_contiguous) {
+ if ((offset + object->vo_shadow_offset) >= (vm_object_offset_t)max_valid_dma_address) {
+ return KERN_INVALID_ADDRESS;
+ }
+
+ if (((offset + object->vo_shadow_offset) + size) >= (vm_object_offset_t)max_valid_dma_address) {
+ return KERN_INVALID_ADDRESS;
+ }
+ }
+ }
+ if (cntrl_flags & (UPL_NOZEROFILL | UPL_NOZEROFILLIO)) {
+ no_zero_fill = TRUE;
+ }
- if (cntrl_flags & UPL_COPYOUT_FROM)
+ if (cntrl_flags & UPL_COPYOUT_FROM) {
prot = VM_PROT_READ;
- else
+ } else {
prot = VM_PROT_READ | VM_PROT_WRITE;
+ }
- if(((size/page_size) > MAX_UPL_TRANSFER) && !object->phys_contiguous) {
- size = MAX_UPL_TRANSFER * page_size;
+ if ((!object->internal) && (object->paging_offset != 0)) {
+ panic("vm_object_iopl_request: external object with non-zero paging offset\n");
}
- if(cntrl_flags & UPL_SET_INTERNAL)
- if(page_list_count != NULL)
- *page_list_count = MAX_UPL_TRANSFER;
- if(((cntrl_flags & UPL_SET_INTERNAL) && !(object->phys_contiguous)) &&
- ((page_list_count != NULL) && (*page_list_count != 0)
- && *page_list_count < (size/page_size)))
- return KERN_INVALID_ARGUMENT;
- if((!object->internal) && (object->paging_offset != 0))
- panic("vm_object_upl_request: vnode object with non-zero paging offset\n");
-
- if(object->phys_contiguous) {
- /* No paging operations are possible against this memory */
- /* and so no need for map object, ever */
- cntrl_flags |= UPL_SET_LITE;
- }
-
- if(upl_ptr) {
- if(cntrl_flags & UPL_SET_INTERNAL) {
- if(cntrl_flags & UPL_SET_LITE) {
- upl = upl_create(
- UPL_CREATE_INTERNAL | UPL_CREATE_LITE,
- size);
- user_page_list = (upl_page_info_t *)
- (((uintptr_t)upl) + sizeof(struct upl));
- lite_list = (wpl_array_t)
- (((uintptr_t)user_page_list) +
- ((size/PAGE_SIZE) *
- sizeof(upl_page_info_t)));
- page_field_size = ((size/PAGE_SIZE) + 7) >> 3;
- page_field_size =
- (page_field_size + 3) & 0xFFFFFFFC;
- bzero((char *)lite_list, page_field_size);
- upl->flags =
- UPL_LITE | UPL_INTERNAL | UPL_IO_WIRE;
- } else {
- upl = upl_create(UPL_CREATE_INTERNAL, size);
- user_page_list = (upl_page_info_t *)
- (((uintptr_t)upl)
- + sizeof(struct upl));
- upl->flags = UPL_INTERNAL | UPL_IO_WIRE;
- }
- } else {
- if(cntrl_flags & UPL_SET_LITE) {
- upl = upl_create(UPL_CREATE_LITE, size);
- lite_list = (wpl_array_t)
- (((uintptr_t)upl) + sizeof(struct upl));
- page_field_size = ((size/PAGE_SIZE) + 7) >> 3;
- page_field_size =
- (page_field_size + 3) & 0xFFFFFFFC;
- bzero((char *)lite_list, page_field_size);
- upl->flags = UPL_LITE | UPL_IO_WIRE;
- } else {
- upl = upl_create(UPL_CREATE_EXTERNAL, size);
- upl->flags = UPL_IO_WIRE;
- }
+ VM_DEBUG_CONSTANT_EVENT(vm_object_iopl_request, VM_IOPL_REQUEST, DBG_FUNC_START, size, cntrl_flags, prot, 0);
+
+#if CONFIG_IOSCHED || UPL_DEBUG
+ if ((object->io_tracking && object != kernel_object) || upl_debug_enabled) {
+ io_tracking_flag |= UPL_CREATE_IO_TRACKING;
+ }
+#endif
+
+#if CONFIG_IOSCHED
+ if (object->io_tracking) {
+ /* Check if we're dealing with the kernel object. We do not support expedite on kernel object UPLs */
+ if (object != kernel_object) {
+ io_tracking_flag |= UPL_CREATE_EXPEDITE_SUP;
}
+ }
+#endif
- if(object->phys_contiguous) {
- upl->map_object = object;
- /* don't need any shadow mappings for this one */
- /* since it is already I/O memory */
- upl->flags |= UPL_DEVICE_MEMORY;
+ if (object->phys_contiguous) {
+ psize = PAGE_SIZE;
+ } else {
+ psize = size;
+
+ dw_count = 0;
+ dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT);
+ dwp_start = vm_page_delayed_work_get_ctx();
+ if (dwp_start == NULL) {
+ dwp_start = &dw_array;
+ dw_limit = 1;
+ dwp_finish_ctx = FALSE;
+ }
- vm_object_lock(object);
- vm_object_paging_begin(object);
- vm_object_unlock(object);
+ dwp = dwp_start;
+ }
- /* paging in progress also protects the paging_offset */
- upl->offset = offset + object->paging_offset;
- upl->size = size;
- *upl_ptr = upl;
- if(user_page_list) {
- user_page_list[0].phys_addr =
- (offset + object->shadow_offset)>>PAGE_SHIFT;
- user_page_list[0].device = TRUE;
- }
+ if (cntrl_flags & UPL_SET_INTERNAL) {
+ upl = upl_create(UPL_CREATE_INTERNAL | UPL_CREATE_LITE | io_tracking_flag, UPL_IO_WIRE, psize);
- if(page_list_count != NULL) {
- if (upl->flags & UPL_INTERNAL) {
- *page_list_count = 0;
- } else {
- *page_list_count = 1;
- }
- }
- return KERN_SUCCESS;
+ user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl));
+ lite_list = (wpl_array_t) (((uintptr_t)user_page_list) +
+ ((psize / PAGE_SIZE) * sizeof(upl_page_info_t)));
+ if (size == 0) {
+ user_page_list = NULL;
+ lite_list = NULL;
}
- if(user_page_list)
- user_page_list[0].device = FALSE;
-
- if(cntrl_flags & UPL_SET_LITE) {
- upl->map_object = object;
- } else {
- upl->map_object = vm_object_allocate(size);
- vm_object_lock(upl->map_object);
- upl->map_object->shadow = object;
- upl->map_object->pageout = TRUE;
- upl->map_object->can_persist = FALSE;
- upl->map_object->copy_strategy =
- MEMORY_OBJECT_COPY_NONE;
- upl->map_object->shadow_offset = offset;
- upl->map_object->wimg_bits = object->wimg_bits;
- vm_object_unlock(upl->map_object);
+ } else {
+ upl = upl_create(UPL_CREATE_LITE | io_tracking_flag, UPL_IO_WIRE, psize);
+
+ lite_list = (wpl_array_t) (((uintptr_t)upl) + sizeof(struct upl));
+ if (size == 0) {
+ lite_list = NULL;
}
}
- vm_object_lock(object);
- vm_object_paging_begin(object);
-
- if (!object->phys_contiguous) {
- /* Protect user space from future COW operations */
- object->true_share = TRUE;
- if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
- object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
+ if (user_page_list) {
+ user_page_list[0].device = FALSE;
+ }
+ *upl_ptr = upl;
+
+ if (cntrl_flags & UPL_NOZEROFILLIO) {
+ DTRACE_VM4(upl_nozerofillio,
+ vm_object_t, object,
+ vm_object_offset_t, offset,
+ upl_size_t, size,
+ upl_t, upl);
}
- /* we can lock the upl offset now that paging_in_progress is set */
- if(upl_ptr) {
- upl->size = size;
- upl->offset = offset + object->paging_offset;
- *upl_ptr = upl;
-#ifdef UPL_DEBUG
- queue_enter(&object->uplq, upl, upl_t, uplq);
-#endif /* UPL_DEBUG */
+ upl->map_object = object;
+ upl->u_offset = original_offset;
+ upl->u_size = original_size;
+
+ size_in_pages = size / PAGE_SIZE;
+
+ if (object == kernel_object &&
+ !(cntrl_flags & (UPL_NEED_32BIT_ADDR | UPL_BLOCK_ACCESS))) {
+ upl->flags |= UPL_KERNEL_OBJECT;
+#if UPL_DEBUG
+ vm_object_lock(object);
+#else
+ vm_object_lock_shared(object);
+#endif
+ } else {
+ vm_object_lock(object);
+ vm_object_activity_begin(object);
}
+ /*
+ * paging in progress also protects the paging_offset
+ */
+ upl->u_offset = original_offset + object->paging_offset;
if (cntrl_flags & UPL_BLOCK_ACCESS) {
/*
- * The user requested that access to the pages in this URL
+ * The user requested that access to the pages in this UPL
* be blocked until the UPL is commited or aborted.
*/
upl->flags |= UPL_ACCESS_BLOCKED;
}
+#if CONFIG_IOSCHED || UPL_DEBUG
+ if ((upl->flags & UPL_TRACKED_BY_OBJECT) || upl_debug_enabled) {
+ vm_object_activity_begin(object);
+ queue_enter(&object->uplq, upl, upl_t, uplq);
+ }
+#endif
+
+ if (object->phys_contiguous) {
+ if (upl->flags & UPL_ACCESS_BLOCKED) {
+ assert(!object->blocked_access);
+ object->blocked_access = TRUE;
+ }
+
+ vm_object_unlock(object);
+
+ /*
+ * don't need any shadow mappings for this one
+ * since it is already I/O memory
+ */
+ upl->flags |= UPL_DEVICE_MEMORY;
+
+ upl->highest_page = (ppnum_t) ((offset + object->vo_shadow_offset + size - 1) >> PAGE_SHIFT);
+
+ if (user_page_list) {
+ user_page_list[0].phys_addr = (ppnum_t) ((offset + object->vo_shadow_offset) >> PAGE_SHIFT);
+ user_page_list[0].device = TRUE;
+ }
+ if (page_list_count != NULL) {
+ if (upl->flags & UPL_INTERNAL) {
+ *page_list_count = 0;
+ } else {
+ *page_list_count = 1;
+ }
+ }
+
+ VM_DEBUG_CONSTANT_EVENT(vm_object_iopl_request, VM_IOPL_REQUEST, DBG_FUNC_END, page_grab_count, KERN_SUCCESS, 0, 0);
+#if DEVELOPMENT || DEBUG
+ if (task != NULL) {
+ ledger_credit(task->ledger, task_ledgers.pages_grabbed_iopl, page_grab_count);
+ }
+#endif /* DEVELOPMENT || DEBUG */
+ return KERN_SUCCESS;
+ }
+ if (object != kernel_object && object != compressor_object) {
+ /*
+ * Protect user space from future COW operations
+ */
+#if VM_OBJECT_TRACKING_OP_TRUESHARE
+ if (!object->true_share &&
+ vm_object_tracking_inited) {
+ void *bt[VM_OBJECT_TRACKING_BTDEPTH];
+ int num = 0;
+
+ num = OSBacktrace(bt,
+ VM_OBJECT_TRACKING_BTDEPTH);
+ btlog_add_entry(vm_object_tracking_btlog,
+ object,
+ VM_OBJECT_TRACKING_OP_TRUESHARE,
+ bt,
+ num);
+ }
+#endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */
+
+ vm_object_lock_assert_exclusive(object);
+ object->true_share = TRUE;
+
+ if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) {
+ object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
+ }
+ }
+
+ if (!(cntrl_flags & UPL_COPYOUT_FROM) &&
+ object->copy != VM_OBJECT_NULL) {
+ /*
+ * Honor copy-on-write obligations
+ *
+ * The caller is gathering these pages and
+ * might modify their contents. We need to
+ * make sure that the copy object has its own
+ * private copies of these pages before we let
+ * the caller modify them.
+ *
+ * NOTE: someone else could map the original object
+ * after we've done this copy-on-write here, and they
+ * could then see an inconsistent picture of the memory
+ * while it's being modified via the UPL. To prevent this,
+ * we would have to block access to these pages until the
+ * UPL is released. We could use the UPL_BLOCK_ACCESS
+ * code path for that...
+ */
+ vm_object_update(object,
+ offset,
+ size,
+ NULL,
+ NULL,
+ FALSE, /* should_return */
+ MEMORY_OBJECT_COPY_SYNC,
+ VM_PROT_NO_CHANGE);
+ VM_PAGEOUT_DEBUG(iopl_cow, 1);
+ VM_PAGEOUT_DEBUG(iopl_cow_pages, (size >> PAGE_SHIFT));
+ }
+ if (!(cntrl_flags & (UPL_NEED_32BIT_ADDR | UPL_BLOCK_ACCESS)) &&
+ object->purgable != VM_PURGABLE_VOLATILE &&
+ object->purgable != VM_PURGABLE_EMPTY &&
+ object->copy == NULL &&
+ size == object->vo_size &&
+ offset == 0 &&
+ object->shadow == NULL &&
+ object->pager == NULL) {
+ if (object->resident_page_count == size_in_pages) {
+ assert(object != compressor_object);
+ assert(object != kernel_object);
+ fast_path_full_req = TRUE;
+ } else if (object->resident_page_count == 0) {
+ assert(object != compressor_object);
+ assert(object != kernel_object);
+ fast_path_empty_req = TRUE;
+ set_cache_attr_needed = TRUE;
+ }
+ }
+
+ if (cntrl_flags & UPL_SET_INTERRUPTIBLE) {
+ interruptible = THREAD_ABORTSAFE;
+ } else {
+ interruptible = THREAD_UNINT;
+ }
+
entry = 0;
+
+ xfer_size = size;
+ dst_offset = offset;
+
+ if (fast_path_full_req) {
+ if (vm_object_iopl_wire_full(object, upl, user_page_list, lite_list, cntrl_flags, tag) == TRUE) {
+ goto finish;
+ }
+ /*
+ * we couldn't complete the processing of this request on the fast path
+ * so fall through to the slow path and finish up
+ */
+ } else if (fast_path_empty_req) {
+ if (cntrl_flags & UPL_REQUEST_NO_FAULT) {
+ ret = KERN_MEMORY_ERROR;
+ goto return_err;
+ }
+ ret = vm_object_iopl_wire_empty(object, upl, user_page_list, lite_list, cntrl_flags, tag, &dst_offset, size_in_pages, &page_grab_count);
+
+ if (ret) {
+ free_wired_pages = TRUE;
+ goto return_err;
+ }
+ goto finish;
+ }
+
+ fault_info.behavior = VM_BEHAVIOR_SEQUENTIAL;
+ fault_info.lo_offset = offset;
+ fault_info.hi_offset = offset + xfer_size;
+ fault_info.mark_zf_absent = TRUE;
+ fault_info.interruptible = interruptible;
+ fault_info.batch_pmap_op = TRUE;
+
while (xfer_size) {
- if((alias_page == NULL) && !(cntrl_flags & UPL_SET_LITE)) {
- if (delayed_unlock) {
- delayed_unlock = 0;
- vm_page_unlock_queues();
+ vm_fault_return_t result;
+
+ dwp->dw_mask = 0;
+
+ if (fast_path_full_req) {
+ /*
+ * if we get here, it means that we ran into a page
+ * state we couldn't handle in the fast path and
+ * bailed out to the slow path... since the order
+ * we look at pages is different between the 2 paths,
+ * the following check is needed to determine whether
+ * this page was already processed in the fast path
+ */
+ if (lite_list[entry >> 5] & (1 << (entry & 31))) {
+ goto skip_page;
}
- vm_object_unlock(object);
- VM_PAGE_GRAB_FICTITIOUS(alias_page);
- vm_object_lock(object);
}
dst_page = vm_page_lookup(object, dst_offset);
- /*
- * ENCRYPTED SWAP:
- * If the page is encrypted, we need to decrypt it,
- * so force a soft page fault.
- */
- if ((dst_page == VM_PAGE_NULL) || (dst_page->busy) ||
- (dst_page->encrypted) ||
- (dst_page->unusual && (dst_page->error ||
- dst_page->restart ||
- dst_page->absent ||
- dst_page->fictitious ||
- (prot & dst_page->page_lock)))) {
- vm_fault_return_t result;
- do {
- vm_page_t top_page;
- kern_return_t error_code;
- int interruptible;
-
- vm_object_offset_t lo_offset = offset;
- vm_object_offset_t hi_offset = offset + size;
-
-
- if (delayed_unlock) {
- delayed_unlock = 0;
- vm_page_unlock_queues();
- }
-
- if(cntrl_flags & UPL_SET_INTERRUPTIBLE) {
- interruptible = THREAD_ABORTSAFE;
- } else {
- interruptible = THREAD_UNINT;
+ if (dst_page == VM_PAGE_NULL ||
+ dst_page->vmp_busy ||
+ dst_page->vmp_error ||
+ dst_page->vmp_restart ||
+ dst_page->vmp_absent ||
+ dst_page->vmp_fictitious) {
+ if (object == kernel_object) {
+ panic("vm_object_iopl_request: missing/bad page in kernel object\n");
+ }
+ if (object == compressor_object) {
+ panic("vm_object_iopl_request: missing/bad page in compressor object\n");
}
- result = vm_fault_page(object, dst_offset,
- prot | VM_PROT_WRITE, FALSE,
- interruptible,
- lo_offset, hi_offset,
- VM_BEHAVIOR_SEQUENTIAL,
- &prot, &dst_page, &top_page,
- (int *)0,
- &error_code, no_zero_fill, FALSE, NULL, 0);
+ if (cntrl_flags & UPL_REQUEST_NO_FAULT) {
+ ret = KERN_MEMORY_ERROR;
+ goto return_err;
+ }
+ set_cache_attr_needed = TRUE;
- switch(result) {
- case VM_FAULT_SUCCESS:
+ /*
+ * We just looked up the page and the result remains valid
+ * until the object lock is release, so send it to
+ * vm_fault_page() (as "dst_page"), to avoid having to
+ * look it up again there.
+ */
+ caller_lookup = TRUE;
- PAGE_WAKEUP_DONE(dst_page);
+ do {
+ vm_page_t top_page;
+ kern_return_t error_code;
- /*
- * Release paging references and
- * top-level placeholder page, if any.
- */
+ fault_info.cluster_size = xfer_size;
+
+ vm_object_paging_begin(object);
+
+ result = vm_fault_page(object, dst_offset,
+ prot | VM_PROT_WRITE, FALSE,
+ caller_lookup,
+ &prot, &dst_page, &top_page,
+ (int *)0,
+ &error_code, no_zero_fill,
+ FALSE, &fault_info);
+
+ /* our lookup is no longer valid at this point */
+ caller_lookup = FALSE;
+
+ switch (result) {
+ case VM_FAULT_SUCCESS:
+ page_grab_count++;
+
+ if (!dst_page->vmp_absent) {
+ PAGE_WAKEUP_DONE(dst_page);
+ } else {
+ /*
+ * we only get back an absent page if we
+ * requested that it not be zero-filled
+ * because we are about to fill it via I/O
+ *
+ * absent pages should be left BUSY
+ * to prevent them from being faulted
+ * into an address space before we've
+ * had a chance to complete the I/O on
+ * them since they may contain info that
+ * shouldn't be seen by the faulting task
+ */
+ }
+ /*
+ * Release paging references and
+ * top-level placeholder page, if any.
+ */
+ if (top_page != VM_PAGE_NULL) {
+ vm_object_t local_object;
+
+ local_object = VM_PAGE_OBJECT(top_page);
+
+ /*
+ * comparing 2 packed pointers
+ */
+ if (top_page->vmp_object != dst_page->vmp_object) {
+ vm_object_lock(local_object);
+ VM_PAGE_FREE(top_page);
+ vm_object_paging_end(local_object);
+ vm_object_unlock(local_object);
+ } else {
+ VM_PAGE_FREE(top_page);
+ vm_object_paging_end(local_object);
+ }
+ }
+ vm_object_paging_end(object);
+ break;
+
+ case VM_FAULT_RETRY:
+ vm_object_lock(object);
+ break;
+
+ case VM_FAULT_MEMORY_SHORTAGE:
+ OSAddAtomic((size_in_pages - entry), &vm_upl_wait_for_pages);
+
+ VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_START, vm_upl_wait_for_pages, 0, 0, 0);
+
+ if (vm_page_wait(interruptible)) {
+ OSAddAtomic(-(size_in_pages - entry), &vm_upl_wait_for_pages);
+
+ VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, 0);
+ vm_object_lock(object);
+
+ break;
+ }
+ OSAddAtomic(-(size_in_pages - entry), &vm_upl_wait_for_pages);
+
+ VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, -1);
+
+ OS_FALLTHROUGH;
+
+ case VM_FAULT_INTERRUPTED:
+ error_code = MACH_SEND_INTERRUPTED;
+ OS_FALLTHROUGH;
+ case VM_FAULT_MEMORY_ERROR:
+memory_error:
+ ret = (error_code ? error_code: KERN_MEMORY_ERROR);
+
+ vm_object_lock(object);
+ goto return_err;
+
+ case VM_FAULT_SUCCESS_NO_VM_PAGE:
+ /* success but no page: fail */
+ vm_object_paging_end(object);
+ vm_object_unlock(object);
+ goto memory_error;
+
+ default:
+ panic("vm_object_iopl_request: unexpected error"
+ " 0x%x from vm_fault_page()\n", result);
+ }
+ } while (result != VM_FAULT_SUCCESS);
+ }
+ phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page);
+
+ if (upl->flags & UPL_KERNEL_OBJECT) {
+ goto record_phys_addr;
+ }
+
+ if (dst_page->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) {
+ dst_page->vmp_busy = TRUE;
+ goto record_phys_addr;
+ }
+
+ if (dst_page->vmp_cleaning) {
+ /*
+ * Someone else is cleaning this page in place.
+ * In theory, we should be able to proceed and use this
+ * page but they'll probably end up clearing the "busy"
+ * bit on it in upl_commit_range() but they didn't set
+ * it, so they would clear our "busy" bit and open
+ * us to race conditions.
+ * We'd better wait for the cleaning to complete and
+ * then try again.
+ */
+ VM_PAGEOUT_DEBUG(vm_object_iopl_request_sleep_for_cleaning, 1);
+ PAGE_SLEEP(object, dst_page, THREAD_UNINT);
+ continue;
+ }
+ if (dst_page->vmp_laundry) {
+ vm_pageout_steal_laundry(dst_page, FALSE);
+ }
+
+ if ((cntrl_flags & UPL_NEED_32BIT_ADDR) &&
+ phys_page >= (max_valid_dma_address >> PAGE_SHIFT)) {
+ vm_page_t low_page;
+ int refmod;
+
+ /*
+ * support devices that can't DMA above 32 bits
+ * by substituting pages from a pool of low address
+ * memory for any pages we find above the 4G mark
+ * can't substitute if the page is already wired because
+ * we don't know whether that physical address has been
+ * handed out to some other 64 bit capable DMA device to use
+ */
+ if (VM_PAGE_WIRED(dst_page)) {
+ ret = KERN_PROTECTION_FAILURE;
+ goto return_err;
+ }
+ low_page = vm_page_grablo();
+
+ if (low_page == VM_PAGE_NULL) {
+ ret = KERN_RESOURCE_SHORTAGE;
+ goto return_err;
+ }
+ /*
+ * from here until the vm_page_replace completes
+ * we musn't drop the object lock... we don't
+ * want anyone refaulting this page in and using
+ * it after we disconnect it... we want the fault
+ * to find the new page being substituted.
+ */
+ if (dst_page->vmp_pmapped) {
+ refmod = pmap_disconnect(phys_page);
+ } else {
+ refmod = 0;
+ }
- if(top_page != VM_PAGE_NULL) {
- vm_object_t local_object;
- local_object =
- top_page->object;
- if(top_page->object
- != dst_page->object) {
- vm_object_lock(
- local_object);
- VM_PAGE_FREE(top_page);
- vm_object_paging_end(
- local_object);
- vm_object_unlock(
- local_object);
- } else {
- VM_PAGE_FREE(top_page);
- vm_object_paging_end(
- local_object);
- }
- }
+ if (!dst_page->vmp_absent) {
+ vm_page_copy(dst_page, low_page);
+ }
- break;
-
-
- case VM_FAULT_RETRY:
- vm_object_lock(object);
- vm_object_paging_begin(object);
- break;
+ low_page->vmp_reference = dst_page->vmp_reference;
+ low_page->vmp_dirty = dst_page->vmp_dirty;
+ low_page->vmp_absent = dst_page->vmp_absent;
- case VM_FAULT_FICTITIOUS_SHORTAGE:
- vm_page_more_fictitious();
- vm_object_lock(object);
- vm_object_paging_begin(object);
- break;
+ if (refmod & VM_MEM_REFERENCED) {
+ low_page->vmp_reference = TRUE;
+ }
+ if (refmod & VM_MEM_MODIFIED) {
+ SET_PAGE_DIRTY(low_page, FALSE);
+ }
- case VM_FAULT_MEMORY_SHORTAGE:
- if (vm_page_wait(interruptible)) {
- vm_object_lock(object);
- vm_object_paging_begin(object);
- break;
- }
- /* fall thru */
+ vm_page_replace(low_page, object, dst_offset);
- case VM_FAULT_INTERRUPTED:
- error_code = MACH_SEND_INTERRUPTED;
- case VM_FAULT_MEMORY_ERROR:
- ret = (error_code ? error_code:
- KERN_MEMORY_ERROR);
- vm_object_lock(object);
- for(; offset < dst_offset;
- offset += PAGE_SIZE) {
- dst_page = vm_page_lookup(
- object, offset);
- if(dst_page == VM_PAGE_NULL)
- panic("vm_object_iopl_request: Wired pages missing. \n");
- vm_page_lock_queues();
- vm_page_unwire(dst_page);
- vm_page_unlock_queues();
- VM_STAT(reactivations++);
- }
- vm_object_unlock(object);
- upl_destroy(upl);
- return ret;
+ dst_page = low_page;
+ /*
+ * vm_page_grablo returned the page marked
+ * BUSY... we don't need a PAGE_WAKEUP_DONE
+ * here, because we've never dropped the object lock
+ */
+ if (!dst_page->vmp_absent) {
+ dst_page->vmp_busy = FALSE;
}
- } while ((result != VM_FAULT_SUCCESS)
- || (result == VM_FAULT_INTERRUPTED));
+
+ phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page);
+ }
+ if (!dst_page->vmp_busy) {
+ dwp->dw_mask |= DW_vm_page_wire;
}
- if (delayed_unlock == 0)
- vm_page_lock_queues();
- vm_page_wire(dst_page);
if (cntrl_flags & UPL_BLOCK_ACCESS) {
/*
* Mark the page "busy" to block any future page fault
- * on this page. We'll also remove the mapping
+ * on this page in addition to wiring it.
+ * We'll also remove the mapping
* of all these pages before leaving this routine.
*/
- assert(!dst_page->fictitious);
- dst_page->busy = TRUE;
+ assert(!dst_page->vmp_fictitious);
+ dst_page->vmp_busy = TRUE;
}
+ /*
+ * expect the page to be used
+ * page queues lock must be held to set 'reference'
+ */
+ dwp->dw_mask |= DW_set_reference;
- if (upl_ptr) {
- if (cntrl_flags & UPL_SET_LITE) {
- int pg_num;
- pg_num = (dst_offset-offset)/PAGE_SIZE;
- lite_list[pg_num>>5] |= 1 << (pg_num & 31);
+ if (!(cntrl_flags & UPL_COPYOUT_FROM)) {
+ SET_PAGE_DIRTY(dst_page, TRUE);
+ /*
+ * Page belonging to a code-signed object is about to
+ * be written. Mark it tainted and disconnect it from
+ * all pmaps so processes have to fault it back in and
+ * deal with the tainted bit.
+ */
+ if (object->code_signed && dst_page->vmp_cs_tainted != VMP_CS_ALL_TRUE) {
+ dst_page->vmp_cs_tainted = VMP_CS_ALL_TRUE;
+ vm_page_iopl_tainted++;
+ if (dst_page->vmp_pmapped) {
+ int refmod = pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(dst_page));
+ if (refmod & VM_MEM_REFERENCED) {
+ dst_page->vmp_reference = TRUE;
+ }
+ }
+ }
+ }
+ if ((cntrl_flags & UPL_REQUEST_FORCE_COHERENCY) && dst_page->vmp_written_by_kernel == TRUE) {
+ pmap_sync_page_attributes_phys(phys_page);
+ dst_page->vmp_written_by_kernel = FALSE;
+ }
+
+record_phys_addr:
+ if (dst_page->vmp_busy) {
+ upl->flags |= UPL_HAS_BUSY;
+ }
+
+ lite_list[entry >> 5] |= 1U << (entry & 31);
+
+ if (phys_page > upl->highest_page) {
+ upl->highest_page = phys_page;
+ }
+
+ if (user_page_list) {
+ user_page_list[entry].phys_addr = phys_page;
+ user_page_list[entry].free_when_done = dst_page->vmp_free_when_done;
+ user_page_list[entry].absent = dst_page->vmp_absent;
+ user_page_list[entry].dirty = dst_page->vmp_dirty;
+ user_page_list[entry].precious = dst_page->vmp_precious;
+ user_page_list[entry].device = FALSE;
+ user_page_list[entry].needed = FALSE;
+ if (dst_page->vmp_clustered == TRUE) {
+ user_page_list[entry].speculative = (dst_page->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q) ? TRUE : FALSE;
} else {
- /*
- * Convert the fictitious page to a
- * private shadow of the real page.
- */
- assert(alias_page->fictitious);
- alias_page->fictitious = FALSE;
- alias_page->private = TRUE;
- alias_page->pageout = TRUE;
- alias_page->phys_page = dst_page->phys_page;
- vm_page_wire(alias_page);
-
- vm_page_insert(alias_page,
- upl->map_object, size - xfer_size);
- assert(!alias_page->wanted);
- alias_page->busy = FALSE;
- alias_page->absent = FALSE;
- }
-
- /* expect the page to be used */
- dst_page->reference = TRUE;
-
- if (!(cntrl_flags & UPL_COPYOUT_FROM))
- dst_page->dirty = TRUE;
- alias_page = NULL;
-
- if (user_page_list) {
- user_page_list[entry].phys_addr
- = dst_page->phys_page;
- user_page_list[entry].dirty =
- dst_page->dirty;
- user_page_list[entry].pageout =
- dst_page->pageout;
- user_page_list[entry].absent =
- dst_page->absent;
- user_page_list[entry].precious =
- dst_page->precious;
- }
- }
- if (delayed_unlock++ > DELAYED_UNLOCK_LIMIT) {
- delayed_unlock = 0;
- vm_page_unlock_queues();
+ user_page_list[entry].speculative = FALSE;
+ }
+ user_page_list[entry].cs_validated = dst_page->vmp_cs_validated;
+ user_page_list[entry].cs_tainted = dst_page->vmp_cs_tainted;
+ user_page_list[entry].cs_nx = dst_page->vmp_cs_nx;
+ user_page_list[entry].mark = FALSE;
+ }
+ if (object != kernel_object && object != compressor_object) {
+ /*
+ * someone is explicitly grabbing this page...
+ * update clustered and speculative state
+ *
+ */
+ if (dst_page->vmp_clustered) {
+ VM_PAGE_CONSUME_CLUSTERED(dst_page);
+ }
}
+skip_page:
entry++;
dst_offset += PAGE_SIZE_64;
xfer_size -= PAGE_SIZE;
- }
- if (delayed_unlock)
- vm_page_unlock_queues();
- if (upl->flags & UPL_INTERNAL) {
- if(page_list_count != NULL)
- *page_list_count = 0;
- } else if (*page_list_count > entry) {
- if(page_list_count != NULL)
- *page_list_count = entry;
+ if (dwp->dw_mask) {
+ VM_PAGE_ADD_DELAYED_WORK(dwp, dst_page, dw_count);
+
+ if (dw_count >= dw_limit) {
+ vm_page_do_delayed_work(object, tag, dwp_start, dw_count);
+
+ dwp = dwp_start;
+ dw_count = 0;
+ }
+ }
}
+ assert(entry == size_in_pages);
- if (alias_page != NULL) {
- vm_page_lock_queues();
- vm_page_free(alias_page);
- vm_page_unlock_queues();
+ if (dw_count) {
+ vm_page_do_delayed_work(object, tag, dwp_start, dw_count);
+ dwp = dwp_start;
+ dw_count = 0;
+ }
+finish:
+ if (user_page_list && set_cache_attr_needed == TRUE) {
+ vm_object_set_pmap_cache_attr(object, user_page_list, size_in_pages, TRUE);
}
+ if (page_list_count != NULL) {
+ if (upl->flags & UPL_INTERNAL) {
+ *page_list_count = 0;
+ } else if (*page_list_count > size_in_pages) {
+ *page_list_count = size_in_pages;
+ }
+ }
vm_object_unlock(object);
if (cntrl_flags & UPL_BLOCK_ACCESS) {
* can't be accessed without causing a page fault.
*/
vm_object_pmap_protect(object, offset, (vm_object_size_t)size,
- PMAP_NULL, 0, VM_PROT_NONE);
+ PMAP_NULL,
+ PAGE_SIZE,
+ 0, VM_PROT_NONE);
+ assert(!object->blocked_access);
+ object->blocked_access = TRUE;
+ }
+
+ VM_DEBUG_CONSTANT_EVENT(vm_object_iopl_request, VM_IOPL_REQUEST, DBG_FUNC_END, page_grab_count, KERN_SUCCESS, 0, 0);
+#if DEVELOPMENT || DEBUG
+ if (task != NULL) {
+ ledger_credit(task->ledger, task_ledgers.pages_grabbed_iopl, page_grab_count);
+ }
+#endif /* DEVELOPMENT || DEBUG */
+
+ if (dwp_start && dwp_finish_ctx) {
+ vm_page_delayed_work_finish_ctx(dwp_start);
+ dwp_start = dwp = NULL;
}
return KERN_SUCCESS;
+
+return_err:
+ dw_index = 0;
+
+ for (; offset < dst_offset; offset += PAGE_SIZE) {
+ boolean_t need_unwire;
+
+ dst_page = vm_page_lookup(object, offset);
+
+ if (dst_page == VM_PAGE_NULL) {
+ panic("vm_object_iopl_request: Wired page missing. \n");
+ }
+
+ /*
+ * if we've already processed this page in an earlier
+ * dw_do_work, we need to undo the wiring... we will
+ * leave the dirty and reference bits on if they
+ * were set, since we don't have a good way of knowing
+ * what the previous state was and we won't get here
+ * under any normal circumstances... we will always
+ * clear BUSY and wakeup any waiters via vm_page_free
+ * or PAGE_WAKEUP_DONE
+ */
+ need_unwire = TRUE;
+
+ if (dw_count) {
+ if ((dwp_start)[dw_index].dw_m == dst_page) {
+ /*
+ * still in the deferred work list
+ * which means we haven't yet called
+ * vm_page_wire on this page
+ */
+ need_unwire = FALSE;
+
+ dw_index++;
+ dw_count--;
+ }
+ }
+ vm_page_lock_queues();
+
+ if (dst_page->vmp_absent || free_wired_pages == TRUE) {
+ vm_page_free(dst_page);
+
+ need_unwire = FALSE;
+ } else {
+ if (need_unwire == TRUE) {
+ vm_page_unwire(dst_page, TRUE);
+ }
+
+ PAGE_WAKEUP_DONE(dst_page);
+ }
+ vm_page_unlock_queues();
+
+ if (need_unwire == TRUE) {
+ VM_STAT_INCR(reactivations);
+ }
+ }
+#if UPL_DEBUG
+ upl->upl_state = 2;
+#endif
+ if (!(upl->flags & UPL_KERNEL_OBJECT)) {
+ vm_object_activity_end(object);
+ vm_object_collapse(object, 0, TRUE);
+ }
+ vm_object_unlock(object);
+ upl_destroy(upl);
+
+ VM_DEBUG_CONSTANT_EVENT(vm_object_iopl_request, VM_IOPL_REQUEST, DBG_FUNC_END, page_grab_count, ret, 0, 0);
+#if DEVELOPMENT || DEBUG
+ if (task != NULL) {
+ ledger_credit(task->ledger, task_ledgers.pages_grabbed_iopl, page_grab_count);
+ }
+#endif /* DEVELOPMENT || DEBUG */
+
+ if (dwp_start && dwp_finish_ctx) {
+ vm_page_delayed_work_finish_ctx(dwp_start);
+ dwp_start = dwp = NULL;
+ }
+ return ret;
}
kern_return_t
upl_transpose(
- upl_t upl1,
- upl_t upl2)
+ upl_t upl1,
+ upl_t upl2)
{
- kern_return_t retval;
- boolean_t upls_locked;
- vm_object_t object1, object2;
+ kern_return_t retval;
+ boolean_t upls_locked;
+ vm_object_t object1, object2;
- if (upl1 == UPL_NULL || upl2 == UPL_NULL || upl1 == upl2) {
+ if (upl1 == UPL_NULL || upl2 == UPL_NULL || upl1 == upl2 || ((upl1->flags & UPL_VECTOR) == UPL_VECTOR) || ((upl2->flags & UPL_VECTOR) == UPL_VECTOR)) {
return KERN_INVALID_ARGUMENT;
}
-
+
upls_locked = FALSE;
/*
upl_lock(upl2);
upl_lock(upl1);
}
- upls_locked = TRUE; /* the UPLs will need to be unlocked */
+ upls_locked = TRUE; /* the UPLs will need to be unlocked */
object1 = upl1->map_object;
object2 = upl2->map_object;
- if (upl1->offset != 0 || upl2->offset != 0 ||
- upl1->size != upl2->size) {
+ if (upl1->u_offset != 0 || upl2->u_offset != 0 ||
+ upl1->u_size != upl2->u_size) {
/*
* We deal only with full objects, not subsets.
* That's because we exchange the entire backing store info
* Tranpose the VM objects' backing store.
*/
retval = vm_object_transpose(object1, object2,
- (vm_object_size_t) upl1->size);
+ upl_adjusted_size(upl1, PAGE_MASK));
if (retval == KERN_SUCCESS) {
/*
* Make each UPL point to the correct VM object, i.e. the
* object holding the pages that the UPL refers to...
*/
+#if CONFIG_IOSCHED || UPL_DEBUG
+ if ((upl1->flags & UPL_TRACKED_BY_OBJECT) || (upl2->flags & UPL_TRACKED_BY_OBJECT)) {
+ vm_object_lock(object1);
+ vm_object_lock(object2);
+ }
+ if ((upl1->flags & UPL_TRACKED_BY_OBJECT) || upl_debug_enabled) {
+ queue_remove(&object1->uplq, upl1, upl_t, uplq);
+ }
+ if ((upl2->flags & UPL_TRACKED_BY_OBJECT) || upl_debug_enabled) {
+ queue_remove(&object2->uplq, upl2, upl_t, uplq);
+ }
+#endif
upl1->map_object = object2;
upl2->map_object = object1;
+
+#if CONFIG_IOSCHED || UPL_DEBUG
+ if ((upl1->flags & UPL_TRACKED_BY_OBJECT) || upl_debug_enabled) {
+ queue_enter(&object2->uplq, upl1, upl_t, uplq);
+ }
+ if ((upl2->flags & UPL_TRACKED_BY_OBJECT) || upl_debug_enabled) {
+ queue_enter(&object1->uplq, upl2, upl_t, uplq);
+ }
+ if ((upl1->flags & UPL_TRACKED_BY_OBJECT) || (upl2->flags & UPL_TRACKED_BY_OBJECT)) {
+ vm_object_unlock(object2);
+ vm_object_unlock(object1);
+ }
+#endif
}
done:
return retval;
}
+void
+upl_range_needed(
+ upl_t upl,
+ int index,
+ int count)
+{
+ upl_page_info_t *user_page_list;
+ int size_in_pages;
+
+ if (!(upl->flags & UPL_INTERNAL) || count <= 0) {
+ return;
+ }
+
+ size_in_pages = upl_adjusted_size(upl, PAGE_MASK) / PAGE_SIZE;
+
+ user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl));
+
+ while (count-- && index < size_in_pages) {
+ user_page_list[index++].needed = TRUE;
+ }
+}
+
+
/*
- * ENCRYPTED SWAP:
- *
- * Rationale: the user might have some encrypted data on disk (via
- * FileVault or any other mechanism). That data is then decrypted in
- * memory, which is safe as long as the machine is secure. But that
- * decrypted data in memory could be paged out to disk by the default
- * pager. The data would then be stored on disk in clear (not encrypted)
- * and it could be accessed by anyone who gets physical access to the
- * disk (if the laptop or the disk gets stolen for example). This weakens
- * the security offered by FileVault.
- *
- * Solution: the default pager will optionally request that all the
- * pages it gathers for pageout be encrypted, via the UPL interfaces,
- * before it sends this UPL to disk via the vnode_pageout() path.
- *
- * Notes:
- *
- * To avoid disrupting the VM LRU algorithms, we want to keep the
- * clean-in-place mechanisms, which allow us to send some extra pages to
- * swap (clustering) without actually removing them from the user's
- * address space. We don't want the user to unknowingly access encrypted
- * data, so we have to actually remove the encrypted pages from the page
- * table. When the user accesses the data, the hardware will fail to
- * locate the virtual page in its page table and will trigger a page
- * fault. We can then decrypt the page and enter it in the page table
- * again. Whenever we allow the user to access the contents of a page,
- * we have to make sure it's not encrypted.
- *
- *
- */
-/*
- * ENCRYPTED SWAP:
* Reserve of virtual addresses in the kernel address space.
* We need to map the physical pages in the kernel, so that we
- * can call the encryption/decryption routines with a kernel
+ * can call the code-signing or slide routines with a kernel
* virtual address. We keep this pool of pre-allocated kernel
* virtual addresses so that we don't have to scan the kernel's
- * virtaul address space each time we need to encrypt or decrypt
+ * virtaul address space each time we need to work with
* a physical page.
- * It would be nice to be able to encrypt and decrypt in physical
- * mode but that might not always be more efficient...
*/
-decl_simple_lock_data(,vm_paging_lock)
-#define VM_PAGING_NUM_PAGES 64
+SIMPLE_LOCK_DECLARE(vm_paging_lock, 0);
+#define VM_PAGING_NUM_PAGES 64
vm_map_offset_t vm_paging_base_address = 0;
-boolean_t vm_paging_page_inuse[VM_PAGING_NUM_PAGES] = { FALSE, };
-int vm_paging_max_index = 0;
-unsigned long vm_paging_no_kernel_page = 0;
-unsigned long vm_paging_objects_mapped = 0;
-unsigned long vm_paging_pages_mapped = 0;
-unsigned long vm_paging_objects_mapped_slow = 0;
-unsigned long vm_paging_pages_mapped_slow = 0;
+boolean_t vm_paging_page_inuse[VM_PAGING_NUM_PAGES] = { FALSE, };
+int vm_paging_max_index = 0;
+int vm_paging_page_waiter = 0;
+int vm_paging_page_waiter_total = 0;
+
+unsigned long vm_paging_no_kernel_page = 0;
+unsigned long vm_paging_objects_mapped = 0;
+unsigned long vm_paging_pages_mapped = 0;
+unsigned long vm_paging_objects_mapped_slow = 0;
+unsigned long vm_paging_pages_mapped_slow = 0;
+
+__startup_func
+void
+vm_paging_map_init(void)
+{
+ kern_return_t kr;
+ vm_map_offset_t page_map_offset;
+ vm_map_entry_t map_entry;
+
+ assert(vm_paging_base_address == 0);
+
+ /*
+ * Initialize our pool of pre-allocated kernel
+ * virtual addresses.
+ */
+ page_map_offset = 0;
+ kr = vm_map_find_space(kernel_map,
+ &page_map_offset,
+ VM_PAGING_NUM_PAGES * PAGE_SIZE,
+ 0,
+ 0,
+ VM_MAP_KERNEL_FLAGS_NONE,
+ VM_KERN_MEMORY_NONE,
+ &map_entry);
+ if (kr != KERN_SUCCESS) {
+ panic("vm_paging_map_init: kernel_map full\n");
+ }
+ VME_OBJECT_SET(map_entry, kernel_object);
+ VME_OFFSET_SET(map_entry, page_map_offset);
+ map_entry->protection = VM_PROT_NONE;
+ map_entry->max_protection = VM_PROT_NONE;
+ map_entry->permanent = TRUE;
+ vm_object_reference(kernel_object);
+ vm_map_unlock(kernel_map);
+
+ assert(vm_paging_base_address == 0);
+ vm_paging_base_address = page_map_offset;
+}
/*
- * ENCRYPTED SWAP:
* vm_paging_map_object:
* Maps part of a VM object's pages in the kernel
- * virtual address space, using the pre-allocated
+ * virtual address space, using the pre-allocated
* kernel virtual addresses, if possible.
* Context:
- * The VM object is locked. This lock will get
- * dropped and re-acquired though.
+ * The VM object is locked. This lock will get
+ * dropped and re-acquired though, so the caller
+ * must make sure the VM object is kept alive
+ * (by holding a VM map that has a reference
+ * on it, for example, or taking an extra reference).
+ * The page should also be kept busy to prevent
+ * it from being reclaimed.
*/
kern_return_t
vm_paging_map_object(
- vm_map_offset_t *address,
- vm_page_t page,
- vm_object_t object,
- vm_object_offset_t offset,
- vm_map_size_t *size)
+ vm_page_t page,
+ vm_object_t object,
+ vm_object_offset_t offset,
+ vm_prot_t protection,
+ boolean_t can_unlock_object,
+ vm_map_size_t *size, /* IN/OUT */
+ vm_map_offset_t *address, /* OUT */
+ boolean_t *need_unmap) /* OUT */
{
- kern_return_t kr;
- vm_map_offset_t page_map_offset;
- vm_map_size_t map_size;
- vm_object_offset_t object_offset;
-#ifdef __ppc__
- int i;
- vm_map_entry_t map_entry;
-#endif /* __ppc__ */
-
+ kern_return_t kr;
+ vm_map_offset_t page_map_offset;
+ vm_map_size_t map_size;
+ vm_object_offset_t object_offset;
+ int i;
-#ifdef __ppc__
if (page != VM_PAGE_NULL && *size == PAGE_SIZE) {
+ /* use permanent 1-to-1 kernel mapping of physical memory ? */
+ *address = (vm_map_offset_t)
+ phystokv((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(page) << PAGE_SHIFT);
+ *need_unmap = FALSE;
+ return KERN_SUCCESS;
+
+ assert(page->vmp_busy);
/*
- * Optimization for the PowerPC.
* Use one of the pre-allocated kernel virtual addresses
* and just enter the VM page in the kernel address space
* at that virtual address.
*/
- vm_object_unlock(object);
- simple_lock(&vm_paging_lock);
-
- if (vm_paging_base_address == 0) {
- /*
- * Initialize our pool of pre-allocated kernel
- * virtual addresses.
- */
- simple_unlock(&vm_paging_lock);
- page_map_offset = 0;
- kr = vm_map_find_space(kernel_map,
- &page_map_offset,
- VM_PAGING_NUM_PAGES * PAGE_SIZE,
- 0,
- &map_entry);
- if (kr != KERN_SUCCESS) {
- panic("vm_paging_map_object: "
- "kernel_map full\n");
- }
- map_entry->object.vm_object = kernel_object;
- map_entry->offset =
- page_map_offset - VM_MIN_KERNEL_ADDRESS;
- vm_object_reference(kernel_object);
- vm_map_unlock(kernel_map);
-
- simple_lock(&vm_paging_lock);
- if (vm_paging_base_address != 0) {
- /* someone raced us and won: undo */
- simple_unlock(&vm_paging_lock);
- kr = vm_map_remove(kernel_map,
- page_map_offset,
- page_map_offset +
- (VM_PAGING_NUM_PAGES
- * PAGE_SIZE),
- VM_MAP_NO_FLAGS);
- assert(kr == KERN_SUCCESS);
- simple_lock(&vm_paging_lock);
- } else {
- vm_paging_base_address = page_map_offset;
- }
- }
+ simple_lock(&vm_paging_lock, &vm_pageout_lck_grp);
/*
* Try and find an available kernel virtual address
* from our pre-allocated pool.
*/
page_map_offset = 0;
- for (i = 0; i < VM_PAGING_NUM_PAGES; i++) {
- if (vm_paging_page_inuse[i] == FALSE) {
- page_map_offset = vm_paging_base_address +
- (i * PAGE_SIZE);
+ for (;;) {
+ for (i = 0; i < VM_PAGING_NUM_PAGES; i++) {
+ if (vm_paging_page_inuse[i] == FALSE) {
+ page_map_offset =
+ vm_paging_base_address +
+ (i * PAGE_SIZE);
+ break;
+ }
+ }
+ if (page_map_offset != 0) {
+ /* found a space to map our page ! */
+ break;
+ }
+
+ if (can_unlock_object) {
+ /*
+ * If we can afford to unlock the VM object,
+ * let's take the slow path now...
+ */
break;
}
+ /*
+ * We can't afford to unlock the VM object, so
+ * let's wait for a space to become available...
+ */
+ vm_paging_page_waiter_total++;
+ vm_paging_page_waiter++;
+ kr = assert_wait((event_t)&vm_paging_page_waiter, THREAD_UNINT);
+ if (kr == THREAD_WAITING) {
+ simple_unlock(&vm_paging_lock);
+ kr = thread_block(THREAD_CONTINUE_NULL);
+ simple_lock(&vm_paging_lock, &vm_pageout_lck_grp);
+ }
+ vm_paging_page_waiter--;
+ /* ... and try again */
}
if (page_map_offset != 0) {
}
vm_paging_page_inuse[i] = TRUE;
simple_unlock(&vm_paging_lock);
- pmap_map_block(kernel_pmap,
- page_map_offset,
- page->phys_page,
- 1, /* Size is number of 4k pages */
- VM_PROT_DEFAULT,
- ((int) page->object->wimg_bits &
- VM_WIMG_MASK),
- 0);
+
+ page->vmp_pmapped = TRUE;
+
+ /*
+ * Keep the VM object locked over the PMAP_ENTER
+ * and the actual use of the page by the kernel,
+ * or this pmap mapping might get undone by a
+ * vm_object_pmap_protect() call...
+ */
+ PMAP_ENTER(kernel_pmap,
+ page_map_offset,
+ page,
+ protection,
+ VM_PROT_NONE,
+ 0,
+ TRUE,
+ kr);
+ assert(kr == KERN_SUCCESS);
vm_paging_objects_mapped++;
- vm_paging_pages_mapped++;
+ vm_paging_pages_mapped++;
*address = page_map_offset;
- vm_object_lock(object);
+ *need_unmap = TRUE;
+
+#if KASAN
+ kasan_notify_address(page_map_offset, PAGE_SIZE);
+#endif
/* all done and mapped, ready to use ! */
return KERN_SUCCESS;
*/
vm_paging_no_kernel_page++;
simple_unlock(&vm_paging_lock);
- vm_object_lock(object);
}
-#endif /* __ppc__ */
+
+ if (!can_unlock_object) {
+ *address = 0;
+ *size = 0;
+ *need_unmap = FALSE;
+ return KERN_NOT_SUPPORTED;
+ }
object_offset = vm_object_trunc_page(offset);
- map_size = vm_map_round_page(*size);
+ map_size = vm_map_round_page(*size,
+ VM_MAP_PAGE_MASK(kernel_map));
/*
* Try and map the required range of the object
* in the kernel_map
*/
- /* don't go beyond the object's end... */
- if (object_offset >= object->size) {
- map_size = 0;
- } else if (map_size > object->size - offset) {
- map_size = object->size - offset;
- }
-
- vm_object_reference_locked(object); /* for the map entry */
+ vm_object_reference_locked(object); /* for the map entry */
vm_object_unlock(object);
kr = vm_map_enter(kernel_map,
- address,
- map_size,
- 0,
- VM_FLAGS_ANYWHERE,
- object,
- object_offset,
- FALSE,
- VM_PROT_DEFAULT,
- VM_PROT_ALL,
- VM_INHERIT_NONE);
+ address,
+ map_size,
+ 0,
+ VM_FLAGS_ANYWHERE,
+ VM_MAP_KERNEL_FLAGS_NONE,
+ VM_KERN_MEMORY_NONE,
+ object,
+ object_offset,
+ FALSE,
+ protection,
+ VM_PROT_ALL,
+ VM_INHERIT_NONE);
if (kr != KERN_SUCCESS) {
*address = 0;
*size = 0;
- vm_object_deallocate(object); /* for the map entry */
+ *need_unmap = FALSE;
+ vm_object_deallocate(object); /* for the map entry */
+ vm_object_lock(object);
return kr;
}
* Enter the mapped pages in the page table now.
*/
vm_object_lock(object);
- for (page_map_offset = 0;
- map_size != 0;
- map_size -= PAGE_SIZE_64, page_map_offset += PAGE_SIZE_64) {
- unsigned int cache_attr;
+ /*
+ * VM object must be kept locked from before PMAP_ENTER()
+ * until after the kernel is done accessing the page(s).
+ * Otherwise, the pmap mappings in the kernel could be
+ * undone by a call to vm_object_pmap_protect().
+ */
+ for (page_map_offset = 0;
+ map_size != 0;
+ map_size -= PAGE_SIZE_64, page_map_offset += PAGE_SIZE_64) {
page = vm_page_lookup(object, offset + page_map_offset);
if (page == VM_PAGE_NULL) {
- panic("vm_paging_map_object: no page !?");
- }
- if (page->no_isync == TRUE) {
- pmap_sync_page_data_phys(page->phys_page);
+ printf("vm_paging_map_object: no page !?");
+ vm_object_unlock(object);
+ kr = vm_map_remove(kernel_map, *address, *size,
+ VM_MAP_REMOVE_NO_FLAGS);
+ assert(kr == KERN_SUCCESS);
+ *address = 0;
+ *size = 0;
+ *need_unmap = FALSE;
+ vm_object_lock(object);
+ return KERN_MEMORY_ERROR;
}
- cache_attr = ((unsigned int) object->wimg_bits) & VM_WIMG_MASK;
+ page->vmp_pmapped = TRUE;
+ //assert(pmap_verify_free(VM_PAGE_GET_PHYS_PAGE(page)));
PMAP_ENTER(kernel_pmap,
- *address + page_map_offset,
- page,
- VM_PROT_DEFAULT,
- cache_attr,
- FALSE);
+ *address + page_map_offset,
+ page,
+ protection,
+ VM_PROT_NONE,
+ 0,
+ TRUE,
+ kr);
+ assert(kr == KERN_SUCCESS);
+#if KASAN
+ kasan_notify_address(*address + page_map_offset, PAGE_SIZE);
+#endif
}
-
+
vm_paging_objects_mapped_slow++;
- vm_paging_pages_mapped_slow += map_size / PAGE_SIZE_64;
+ vm_paging_pages_mapped_slow += (unsigned long) (map_size / PAGE_SIZE_64);
+
+ *need_unmap = TRUE;
return KERN_SUCCESS;
}
/*
- * ENCRYPTED SWAP:
* vm_paging_unmap_object:
* Unmaps part of a VM object's pages from the kernel
- * virtual address space.
+ * virtual address space.
* Context:
- * The VM object is locked. This lock will get
- * dropped and re-acquired though.
+ * The VM object is locked. This lock will get
+ * dropped and re-acquired though.
*/
void
vm_paging_unmap_object(
- vm_object_t object,
- vm_map_offset_t start,
- vm_map_offset_t end)
+ vm_object_t object,
+ vm_map_offset_t start,
+ vm_map_offset_t end)
{
- kern_return_t kr;
-#ifdef __ppc__
- int i;
-#endif /* __ppc__ */
-
- if ((vm_paging_base_address != 0) &&
- ((start < vm_paging_base_address) ||
- (end > (vm_paging_base_address
- + (VM_PAGING_NUM_PAGES * PAGE_SIZE))))) {
+ kern_return_t kr;
+ int i;
+
+ if ((vm_paging_base_address == 0) ||
+ (start < vm_paging_base_address) ||
+ (end > (vm_paging_base_address
+ + (VM_PAGING_NUM_PAGES * PAGE_SIZE)))) {
/*
* We didn't use our pre-allocated pool of
* kernel virtual address. Deallocate the
if (object != VM_OBJECT_NULL) {
vm_object_unlock(object);
}
- kr = vm_map_remove(kernel_map, start, end, VM_MAP_NO_FLAGS);
+ kr = vm_map_remove(kernel_map, start, end,
+ VM_MAP_REMOVE_NO_FLAGS);
if (object != VM_OBJECT_NULL) {
vm_object_lock(object);
}
* pre-allocated pool. Put it back in the pool
* for next time.
*/
-#ifdef __ppc__
assert(end - start == PAGE_SIZE);
- i = (start - vm_paging_base_address) >> PAGE_SHIFT;
+ i = (int) ((start - vm_paging_base_address) >> PAGE_SHIFT);
+ assert(i >= 0 && i < VM_PAGING_NUM_PAGES);
/* undo the pmap mapping */
- mapping_remove(kernel_pmap, start);
+ pmap_remove(kernel_pmap, start, end);
- simple_lock(&vm_paging_lock);
+ simple_lock(&vm_paging_lock, &vm_pageout_lck_grp);
vm_paging_page_inuse[i] = FALSE;
+ if (vm_paging_page_waiter) {
+ thread_wakeup(&vm_paging_page_waiter);
+ }
simple_unlock(&vm_paging_lock);
-#endif /* __ppc__ */
}
}
-/*
- * Encryption data.
- * "iv" is the "initial vector". Ideally, we want to
- * have a different one for each page we encrypt, so that
- * crackers can't find encryption patterns too easily.
- */
-#define SWAP_CRYPT_AES_KEY_SIZE 128 /* XXX 192 and 256 don't work ! */
-boolean_t swap_crypt_ctx_initialized = FALSE;
-aes_32t swap_crypt_key[8]; /* big enough for a 256 key */
-aes_ctx swap_crypt_ctx;
-const unsigned char swap_crypt_null_iv[AES_BLOCK_SIZE] = {0xa, };
-
-#if DEBUG
-boolean_t swap_crypt_ctx_tested = FALSE;
-unsigned char swap_crypt_test_page_ref[4096] __attribute__((aligned(4096)));
-unsigned char swap_crypt_test_page_encrypt[4096] __attribute__((aligned(4096)));
-unsigned char swap_crypt_test_page_decrypt[4096] __attribute__((aligned(4096)));
-#endif /* DEBUG */
-
-extern u_long random(void);
/*
- * Initialize the encryption context: key and key size.
+ * page->vmp_object must be locked
*/
-void swap_crypt_ctx_initialize(void); /* forward */
void
-swap_crypt_ctx_initialize(void)
+vm_pageout_steal_laundry(vm_page_t page, boolean_t queues_locked)
{
- unsigned int i;
+ if (!queues_locked) {
+ vm_page_lockspin_queues();
+ }
+ page->vmp_free_when_done = FALSE;
/*
- * No need for locking to protect swap_crypt_ctx_initialized
- * because the first use of encryption will come from the
- * pageout thread (we won't pagein before there's been a pageout)
- * and there's only one pageout thread.
- */
- if (swap_crypt_ctx_initialized == FALSE) {
- for (i = 0;
- i < (sizeof (swap_crypt_key) /
- sizeof (swap_crypt_key[0]));
- i++) {
- swap_crypt_key[i] = random();
- }
- aes_encrypt_key((const unsigned char *) swap_crypt_key,
- SWAP_CRYPT_AES_KEY_SIZE,
- &swap_crypt_ctx.encrypt);
- aes_decrypt_key((const unsigned char *) swap_crypt_key,
- SWAP_CRYPT_AES_KEY_SIZE,
- &swap_crypt_ctx.decrypt);
- swap_crypt_ctx_initialized = TRUE;
- }
-
-#if DEBUG
- /*
- * Validate the encryption algorithms.
+ * need to drop the laundry count...
+ * we may also need to remove it
+ * from the I/O paging queue...
+ * vm_pageout_throttle_up handles both cases
+ *
+ * the laundry and pageout_queue flags are cleared...
*/
- if (swap_crypt_ctx_tested == FALSE) {
- /* initialize */
- for (i = 0; i < 4096; i++) {
- swap_crypt_test_page_ref[i] = (char) i;
- }
- /* encrypt */
- aes_encrypt_cbc(swap_crypt_test_page_ref,
- swap_crypt_null_iv,
- PAGE_SIZE / AES_BLOCK_SIZE,
- swap_crypt_test_page_encrypt,
- &swap_crypt_ctx.encrypt);
- /* decrypt */
- aes_decrypt_cbc(swap_crypt_test_page_encrypt,
- swap_crypt_null_iv,
- PAGE_SIZE / AES_BLOCK_SIZE,
- swap_crypt_test_page_decrypt,
- &swap_crypt_ctx.decrypt);
- /* compare result with original */
- for (i = 0; i < 4096; i ++) {
- if (swap_crypt_test_page_decrypt[i] !=
- swap_crypt_test_page_ref[i]) {
- panic("encryption test failed");
- }
- }
-
- /* encrypt again */
- aes_encrypt_cbc(swap_crypt_test_page_decrypt,
- swap_crypt_null_iv,
- PAGE_SIZE / AES_BLOCK_SIZE,
- swap_crypt_test_page_decrypt,
- &swap_crypt_ctx.encrypt);
- /* decrypt in place */
- aes_decrypt_cbc(swap_crypt_test_page_decrypt,
- swap_crypt_null_iv,
- PAGE_SIZE / AES_BLOCK_SIZE,
- swap_crypt_test_page_decrypt,
- &swap_crypt_ctx.decrypt);
- for (i = 0; i < 4096; i ++) {
- if (swap_crypt_test_page_decrypt[i] !=
- swap_crypt_test_page_ref[i]) {
- panic("in place encryption test failed");
- }
- }
-
- swap_crypt_ctx_tested = TRUE;
- }
-#endif /* DEBUG */
+ vm_pageout_throttle_up(page);
+
+ if (!queues_locked) {
+ vm_page_unlock_queues();
+ }
}
-/*
- * ENCRYPTED SWAP:
- * vm_page_encrypt:
- * Encrypt the given page, for secure paging.
- * The page might already be mapped at kernel virtual
- * address "kernel_mapping_offset". Otherwise, we need
- * to map it.
- *
- * Context:
- * The page's object is locked, but this lock will be released
- * and re-acquired.
- * The page is busy and not accessible by users (not entered in any pmap).
- */
-void
-vm_page_encrypt(
- vm_page_t page,
- vm_map_offset_t kernel_mapping_offset)
+upl_t
+vector_upl_create(vm_offset_t upl_offset)
{
- int clear_refmod = 0;
- kern_return_t kr;
- boolean_t page_was_referenced;
- boolean_t page_was_modified;
- vm_map_size_t kernel_mapping_size;
- vm_offset_t kernel_vaddr;
- union {
- unsigned char aes_iv[AES_BLOCK_SIZE];
- struct {
- memory_object_t pager_object;
- vm_object_offset_t paging_offset;
- } vm;
- } encrypt_iv;
-
- if (! vm_pages_encrypted) {
- vm_pages_encrypted = TRUE;
- }
-
- assert(page->busy);
- assert(page->dirty || page->precious);
-
- if (page->encrypted) {
- /*
- * Already encrypted: no need to do it again.
- */
- vm_page_encrypt_already_encrypted_counter++;
- return;
+ int vector_upl_size = sizeof(struct _vector_upl);
+ int i = 0;
+ upl_t upl;
+ vector_upl_t vector_upl = (vector_upl_t)kalloc(vector_upl_size);
+
+ upl = upl_create(0, UPL_VECTOR, 0);
+ upl->vector_upl = vector_upl;
+ upl->u_offset = upl_offset;
+ vector_upl->size = 0;
+ vector_upl->offset = upl_offset;
+ vector_upl->invalid_upls = 0;
+ vector_upl->num_upls = 0;
+ vector_upl->pagelist = NULL;
+
+ for (i = 0; i < MAX_VECTOR_UPL_ELEMENTS; i++) {
+ vector_upl->upl_iostates[i].size = 0;
+ vector_upl->upl_iostates[i].offset = 0;
}
- ASSERT_PAGE_DECRYPTED(page);
-
- /*
- * Gather the "reference" and "modified" status of the page.
- * We'll restore these values after the encryption, so that
- * the encryption is transparent to the rest of the system
- * and doesn't impact the VM's LRU logic.
- */
- page_was_referenced =
- (page->reference || pmap_is_referenced(page->phys_page));
- page_was_modified =
- (page->dirty || pmap_is_modified(page->phys_page));
+ return upl;
+}
- if (kernel_mapping_offset == 0) {
- /*
- * The page hasn't already been mapped in kernel space
- * by the caller. Map it now, so that we can access
- * its contents and encrypt them.
- */
- kernel_mapping_size = PAGE_SIZE;
- kr = vm_paging_map_object(&kernel_mapping_offset,
- page,
- page->object,
- page->offset,
- &kernel_mapping_size);
- if (kr != KERN_SUCCESS) {
- panic("vm_page_encrypt: "
- "could not map page in kernel: 0x%x\n",
- kr);
+void
+vector_upl_deallocate(upl_t upl)
+{
+ if (upl) {
+ vector_upl_t vector_upl = upl->vector_upl;
+ if (vector_upl) {
+ if (vector_upl->invalid_upls != vector_upl->num_upls) {
+ panic("Deallocating non-empty Vectored UPL\n");
+ }
+ kfree(vector_upl->pagelist, (sizeof(struct upl_page_info) * (vector_upl->size / PAGE_SIZE)));
+ vector_upl->invalid_upls = 0;
+ vector_upl->num_upls = 0;
+ vector_upl->pagelist = NULL;
+ vector_upl->size = 0;
+ vector_upl->offset = 0;
+ kfree(vector_upl, sizeof(struct _vector_upl));
+ vector_upl = (vector_upl_t)0xfeedfeed;
+ } else {
+ panic("vector_upl_deallocate was passed a non-vectored upl\n");
}
} else {
- kernel_mapping_size = 0;
- }
- kernel_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping_offset);
-
- if (swap_crypt_ctx_initialized == FALSE) {
- swap_crypt_ctx_initialize();
- }
- assert(swap_crypt_ctx_initialized);
-
- /*
- * Prepare an "initial vector" for the encryption.
- * We use the "pager" and the "paging_offset" for that
- * page to obfuscate the encrypted data a bit more and
- * prevent crackers from finding patterns that they could
- * use to break the key.
- */
- bzero(&encrypt_iv.aes_iv[0], sizeof (encrypt_iv.aes_iv));
- encrypt_iv.vm.pager_object = page->object->pager;
- encrypt_iv.vm.paging_offset =
- page->object->paging_offset + page->offset;
-
- vm_object_unlock(page->object);
-
- /* encrypt the "initial vector" */
- aes_encrypt_cbc((const unsigned char *) &encrypt_iv.aes_iv[0],
- swap_crypt_null_iv,
- 1,
- &encrypt_iv.aes_iv[0],
- &swap_crypt_ctx.encrypt);
-
- /*
- * Encrypt the page.
- */
- aes_encrypt_cbc((const unsigned char *) kernel_vaddr,
- &encrypt_iv.aes_iv[0],
- PAGE_SIZE / AES_BLOCK_SIZE,
- (unsigned char *) kernel_vaddr,
- &swap_crypt_ctx.encrypt);
-
- vm_page_encrypt_counter++;
-
- vm_object_lock(page->object);
-
- /*
- * Unmap the page from the kernel's address space,
- * if we had to map it ourselves. Otherwise, let
- * the caller undo the mapping if needed.
- */
- if (kernel_mapping_size != 0) {
- vm_paging_unmap_object(page->object,
- kernel_mapping_offset,
- kernel_mapping_offset + kernel_mapping_size);
+ panic("vector_upl_deallocate was passed a NULL upl\n");
}
+}
- /*
- * Restore the "reference" and "modified" bits.
- * This should clean up any impact the encryption had
- * on them.
- */
- if (! page_was_referenced) {
- clear_refmod |= VM_MEM_REFERENCED;
- page->reference = FALSE;
- }
- if (! page_was_modified) {
- clear_refmod |= VM_MEM_MODIFIED;
- page->dirty = FALSE;
+boolean_t
+vector_upl_is_valid(upl_t upl)
+{
+ if (upl && ((upl->flags & UPL_VECTOR) == UPL_VECTOR)) {
+ vector_upl_t vector_upl = upl->vector_upl;
+ if (vector_upl == NULL || vector_upl == (vector_upl_t)0xfeedfeed || vector_upl == (vector_upl_t)0xfeedbeef) {
+ return FALSE;
+ } else {
+ return TRUE;
+ }
}
- if (clear_refmod)
- pmap_clear_refmod(page->phys_page, clear_refmod);
-
- page->encrypted = TRUE;
+ return FALSE;
}
-/*
- * ENCRYPTED SWAP:
- * vm_page_decrypt:
- * Decrypt the given page.
- * The page might already be mapped at kernel virtual
- * address "kernel_mapping_offset". Otherwise, we need
- * to map it.
- *
- * Context:
- * The page's VM object is locked but will be unlocked and relocked.
- * The page is busy and not accessible by users (not entered in any pmap).
- */
-void
-vm_page_decrypt(
- vm_page_t page,
- vm_map_offset_t kernel_mapping_offset)
+boolean_t
+vector_upl_set_subupl(upl_t upl, upl_t subupl, uint32_t io_size)
{
- int clear_refmod = 0;
- kern_return_t kr;
- vm_map_size_t kernel_mapping_size;
- vm_offset_t kernel_vaddr;
- boolean_t page_was_referenced;
- union {
- unsigned char aes_iv[AES_BLOCK_SIZE];
- struct {
- memory_object_t pager_object;
- vm_object_offset_t paging_offset;
- } vm;
- } decrypt_iv;
-
- assert(page->busy);
- assert(page->encrypted);
-
- /*
- * Gather the "reference" status of the page.
- * We'll restore its value after the decryption, so that
- * the decryption is transparent to the rest of the system
- * and doesn't impact the VM's LRU logic.
- */
- page_was_referenced =
- (page->reference || pmap_is_referenced(page->phys_page));
+ if (vector_upl_is_valid(upl)) {
+ vector_upl_t vector_upl = upl->vector_upl;
+
+ if (vector_upl) {
+ if (subupl) {
+ if (io_size) {
+ if (io_size < PAGE_SIZE) {
+ io_size = PAGE_SIZE;
+ }
+ subupl->vector_upl = (void*)vector_upl;
+ vector_upl->upl_elems[vector_upl->num_upls++] = subupl;
+ vector_upl->size += io_size;
+ upl->u_size += io_size;
+ } else {
+ uint32_t i = 0, invalid_upls = 0;
+ for (i = 0; i < vector_upl->num_upls; i++) {
+ if (vector_upl->upl_elems[i] == subupl) {
+ break;
+ }
+ }
+ if (i == vector_upl->num_upls) {
+ panic("Trying to remove sub-upl when none exists");
+ }
- if (kernel_mapping_offset == 0) {
- /*
- * The page hasn't already been mapped in kernel space
- * by the caller. Map it now, so that we can access
- * its contents and decrypt them.
- */
- kernel_mapping_size = PAGE_SIZE;
- kr = vm_paging_map_object(&kernel_mapping_offset,
- page,
- page->object,
- page->offset,
- &kernel_mapping_size);
- if (kr != KERN_SUCCESS) {
- panic("vm_page_decrypt: "
- "could not map page in kernel: 0x%x\n");
+ vector_upl->upl_elems[i] = NULL;
+ invalid_upls = os_atomic_inc(&(vector_upl)->invalid_upls,
+ relaxed);
+ if (invalid_upls == vector_upl->num_upls) {
+ return TRUE;
+ } else {
+ return FALSE;
+ }
+ }
+ } else {
+ panic("vector_upl_set_subupl was passed a NULL upl element\n");
+ }
+ } else {
+ panic("vector_upl_set_subupl was passed a non-vectored upl\n");
}
} else {
- kernel_mapping_size = 0;
+ panic("vector_upl_set_subupl was passed a NULL upl\n");
}
- kernel_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping_offset);
-
- assert(swap_crypt_ctx_initialized);
- /*
- * Prepare an "initial vector" for the decryption.
- * It has to be the same as the "initial vector" we
- * used to encrypt that page.
- */
- bzero(&decrypt_iv.aes_iv[0], sizeof (decrypt_iv.aes_iv));
- decrypt_iv.vm.pager_object = page->object->pager;
- decrypt_iv.vm.paging_offset =
- page->object->paging_offset + page->offset;
-
- vm_object_unlock(page->object);
+ return FALSE;
+}
- /* encrypt the "initial vector" */
- aes_encrypt_cbc((const unsigned char *) &decrypt_iv.aes_iv[0],
- swap_crypt_null_iv,
- 1,
- &decrypt_iv.aes_iv[0],
- &swap_crypt_ctx.encrypt);
+void
+vector_upl_set_pagelist(upl_t upl)
+{
+ if (vector_upl_is_valid(upl)) {
+ uint32_t i = 0;
+ vector_upl_t vector_upl = upl->vector_upl;
- /*
- * Decrypt the page.
- */
- aes_decrypt_cbc((const unsigned char *) kernel_vaddr,
- &decrypt_iv.aes_iv[0],
- PAGE_SIZE / AES_BLOCK_SIZE,
- (unsigned char *) kernel_vaddr,
- &swap_crypt_ctx.decrypt);
- vm_page_decrypt_counter++;
+ if (vector_upl) {
+ vm_offset_t pagelist_size = 0, cur_upl_pagelist_size = 0;
- vm_object_lock(page->object);
+ vector_upl->pagelist = (upl_page_info_array_t)kalloc(sizeof(struct upl_page_info) * (vector_upl->size / PAGE_SIZE));
- /*
- * Unmap the page from the kernel's address space,
- * if we had to map it ourselves. Otherwise, let
- * the caller undo the mapping if needed.
- */
- if (kernel_mapping_size != 0) {
- vm_paging_unmap_object(page->object,
- kernel_vaddr,
- kernel_vaddr + PAGE_SIZE);
+ for (i = 0; i < vector_upl->num_upls; i++) {
+ cur_upl_pagelist_size = sizeof(struct upl_page_info) * upl_adjusted_size(vector_upl->upl_elems[i], PAGE_MASK) / PAGE_SIZE;
+ bcopy(UPL_GET_INTERNAL_PAGE_LIST_SIMPLE(vector_upl->upl_elems[i]), (char*)vector_upl->pagelist + pagelist_size, cur_upl_pagelist_size);
+ pagelist_size += cur_upl_pagelist_size;
+ if (vector_upl->upl_elems[i]->highest_page > upl->highest_page) {
+ upl->highest_page = vector_upl->upl_elems[i]->highest_page;
+ }
+ }
+ assert( pagelist_size == (sizeof(struct upl_page_info) * (vector_upl->size / PAGE_SIZE)));
+ } else {
+ panic("vector_upl_set_pagelist was passed a non-vectored upl\n");
+ }
+ } else {
+ panic("vector_upl_set_pagelist was passed a NULL upl\n");
}
+}
- /*
- * After decryption, the page is actually clean.
- * It was encrypted as part of paging, which "cleans"
- * the "dirty" pages.
- * Noone could access it after it was encrypted
- * and the decryption doesn't count.
- */
- page->dirty = FALSE;
- clear_refmod = VM_MEM_MODIFIED;
+upl_t
+vector_upl_subupl_byindex(upl_t upl, uint32_t index)
+{
+ if (vector_upl_is_valid(upl)) {
+ vector_upl_t vector_upl = upl->vector_upl;
+ if (vector_upl) {
+ if (index < vector_upl->num_upls) {
+ return vector_upl->upl_elems[index];
+ }
+ } else {
+ panic("vector_upl_subupl_byindex was passed a non-vectored upl\n");
+ }
+ }
+ return NULL;
+}
- /* restore the "reference" bit */
- if (! page_was_referenced) {
- page->reference = FALSE;
- clear_refmod |= VM_MEM_REFERENCED;
+upl_t
+vector_upl_subupl_byoffset(upl_t upl, upl_offset_t *upl_offset, upl_size_t *upl_size)
+{
+ if (vector_upl_is_valid(upl)) {
+ uint32_t i = 0;
+ vector_upl_t vector_upl = upl->vector_upl;
+
+ if (vector_upl) {
+ upl_t subupl = NULL;
+ vector_upl_iostates_t subupl_state;
+
+ for (i = 0; i < vector_upl->num_upls; i++) {
+ subupl = vector_upl->upl_elems[i];
+ subupl_state = vector_upl->upl_iostates[i];
+ if (*upl_offset <= (subupl_state.offset + subupl_state.size - 1)) {
+ /* We could have been passed an offset/size pair that belongs
+ * to an UPL element that has already been committed/aborted.
+ * If so, return NULL.
+ */
+ if (subupl == NULL) {
+ return NULL;
+ }
+ if ((subupl_state.offset + subupl_state.size) < (*upl_offset + *upl_size)) {
+ *upl_size = (subupl_state.offset + subupl_state.size) - *upl_offset;
+ if (*upl_size > subupl_state.size) {
+ *upl_size = subupl_state.size;
+ }
+ }
+ if (*upl_offset >= subupl_state.offset) {
+ *upl_offset -= subupl_state.offset;
+ } else if (i) {
+ panic("Vector UPL offset miscalculation\n");
+ }
+ return subupl;
+ }
+ }
+ } else {
+ panic("vector_upl_subupl_byoffset was passed a non-vectored UPL\n");
+ }
}
- pmap_clear_refmod(page->phys_page, clear_refmod);
+ return NULL;
+}
- page->encrypted = FALSE;
+void
+vector_upl_get_submap(upl_t upl, vm_map_t *v_upl_submap, vm_offset_t *submap_dst_addr)
+{
+ *v_upl_submap = NULL;
- /*
- * We've just modified the page's contents via the data cache and part
- * of the new contents might still be in the cache and not yet in RAM.
- * Since the page is now available and might get gathered in a UPL to
- * be part of a DMA transfer from a driver that expects the memory to
- * be coherent at this point, we have to flush the data cache.
- */
- pmap_sync_page_data_phys(page->phys_page);
- /*
- * Since the page is not mapped yet, some code might assume that it
- * doesn't need to invalidate the instruction cache when writing to
- * that page. That code relies on "no_isync" being set, so that the
- * caches get syncrhonized when the page is first mapped. So we need
- * to set "no_isync" here too, despite the fact that we just
- * synchronized the caches above...
- */
- page->no_isync = TRUE;
+ if (vector_upl_is_valid(upl)) {
+ vector_upl_t vector_upl = upl->vector_upl;
+ if (vector_upl) {
+ *v_upl_submap = vector_upl->submap;
+ *submap_dst_addr = vector_upl->submap_dst_addr;
+ } else {
+ panic("vector_upl_get_submap was passed a non-vectored UPL\n");
+ }
+ } else {
+ panic("vector_upl_get_submap was passed a null UPL\n");
+ }
}
-unsigned long upl_encrypt_upls = 0;
-unsigned long upl_encrypt_pages = 0;
-
-/*
- * ENCRYPTED SWAP:
- *
- * upl_encrypt:
- * Encrypts all the pages in the UPL, within the specified range.
- *
- */
void
-upl_encrypt(
- upl_t upl,
- upl_offset_t crypt_offset,
- upl_size_t crypt_size)
+vector_upl_set_submap(upl_t upl, vm_map_t submap, vm_offset_t submap_dst_addr)
{
- upl_size_t upl_size;
- upl_offset_t upl_offset;
- vm_object_t upl_object;
- vm_page_t page;
- vm_object_t shadow_object;
- vm_object_offset_t shadow_offset;
- vm_object_offset_t paging_offset;
- vm_object_offset_t base_offset;
-
- upl_encrypt_upls++;
- upl_encrypt_pages += crypt_size / PAGE_SIZE;
-
- upl_lock(upl);
+ if (vector_upl_is_valid(upl)) {
+ vector_upl_t vector_upl = upl->vector_upl;
+ if (vector_upl) {
+ vector_upl->submap = submap;
+ vector_upl->submap_dst_addr = submap_dst_addr;
+ } else {
+ panic("vector_upl_get_submap was passed a non-vectored UPL\n");
+ }
+ } else {
+ panic("vector_upl_get_submap was passed a NULL UPL\n");
+ }
+}
- upl_object = upl->map_object;
- upl_offset = upl->offset;
- upl_size = upl->size;
+void
+vector_upl_set_iostate(upl_t upl, upl_t subupl, upl_offset_t offset, upl_size_t size)
+{
+ if (vector_upl_is_valid(upl)) {
+ uint32_t i = 0;
+ vector_upl_t vector_upl = upl->vector_upl;
- upl_unlock(upl);
+ if (vector_upl) {
+ for (i = 0; i < vector_upl->num_upls; i++) {
+ if (vector_upl->upl_elems[i] == subupl) {
+ break;
+ }
+ }
- vm_object_lock(upl_object);
+ if (i == vector_upl->num_upls) {
+ panic("setting sub-upl iostate when none exists");
+ }
- /*
- * Find the VM object that contains the actual pages.
- */
- if (upl_object->pageout) {
- shadow_object = upl_object->shadow;
- /*
- * The offset in the shadow object is actually also
- * accounted for in upl->offset. It possibly shouldn't be
- * this way, but for now don't account for it twice.
- */
- shadow_offset = 0;
- assert(upl_object->paging_offset == 0); /* XXX ? */
- vm_object_lock(shadow_object);
+ vector_upl->upl_iostates[i].offset = offset;
+ if (size < PAGE_SIZE) {
+ size = PAGE_SIZE;
+ }
+ vector_upl->upl_iostates[i].size = size;
+ } else {
+ panic("vector_upl_set_iostate was passed a non-vectored UPL\n");
+ }
} else {
- shadow_object = upl_object;
- shadow_offset = 0;
+ panic("vector_upl_set_iostate was passed a NULL UPL\n");
}
+}
- paging_offset = shadow_object->paging_offset;
- vm_object_paging_begin(shadow_object);
+void
+vector_upl_get_iostate(upl_t upl, upl_t subupl, upl_offset_t *offset, upl_size_t *size)
+{
+ if (vector_upl_is_valid(upl)) {
+ uint32_t i = 0;
+ vector_upl_t vector_upl = upl->vector_upl;
- if (shadow_object != upl_object) {
- vm_object_unlock(shadow_object);
- }
- vm_object_unlock(upl_object);
+ if (vector_upl) {
+ for (i = 0; i < vector_upl->num_upls; i++) {
+ if (vector_upl->upl_elems[i] == subupl) {
+ break;
+ }
+ }
- base_offset = shadow_offset;
- base_offset += upl_offset;
- base_offset += crypt_offset;
- base_offset -= paging_offset;
- /*
- * Unmap the pages, so that nobody can continue accessing them while
- * they're encrypted. After that point, all accesses to these pages
- * will cause a page fault and block while the page is being encrypted
- * (busy). After the encryption completes, any access will cause a
- * page fault and the page gets decrypted at that time.
- */
- assert(crypt_offset + crypt_size <= upl_size);
- vm_object_pmap_protect(shadow_object,
- base_offset,
- (vm_object_size_t)crypt_size,
- PMAP_NULL,
- 0,
- VM_PROT_NONE);
-
- /* XXX FBDP could the object have changed significantly here ? */
- vm_object_lock(shadow_object);
+ if (i == vector_upl->num_upls) {
+ panic("getting sub-upl iostate when none exists");
+ }
- for (upl_offset = 0;
- upl_offset < crypt_size;
- upl_offset += PAGE_SIZE) {
- page = vm_page_lookup(shadow_object,
- base_offset + upl_offset);
- if (page == VM_PAGE_NULL) {
- panic("upl_encrypt: "
- "no page for (obj=%p,off=%lld+%d)!\n",
- shadow_object,
- base_offset,
- upl_offset);
+ *offset = vector_upl->upl_iostates[i].offset;
+ *size = vector_upl->upl_iostates[i].size;
+ } else {
+ panic("vector_upl_get_iostate was passed a non-vectored UPL\n");
}
- vm_page_encrypt(page, 0);
+ } else {
+ panic("vector_upl_get_iostate was passed a NULL UPL\n");
}
+}
- vm_object_paging_end(shadow_object);
- vm_object_unlock(shadow_object);
+void
+vector_upl_get_iostate_byindex(upl_t upl, uint32_t index, upl_offset_t *offset, upl_size_t *size)
+{
+ if (vector_upl_is_valid(upl)) {
+ vector_upl_t vector_upl = upl->vector_upl;
+ if (vector_upl) {
+ if (index < vector_upl->num_upls) {
+ *offset = vector_upl->upl_iostates[index].offset;
+ *size = vector_upl->upl_iostates[index].size;
+ } else {
+ *offset = *size = 0;
+ }
+ } else {
+ panic("vector_upl_get_iostate_byindex was passed a non-vectored UPL\n");
+ }
+ } else {
+ panic("vector_upl_get_iostate_byindex was passed a NULL UPL\n");
+ }
+}
+
+upl_page_info_t *
+upl_get_internal_vectorupl_pagelist(upl_t upl)
+{
+ return ((vector_upl_t)(upl->vector_upl))->pagelist;
+}
+
+void *
+upl_get_internal_vectorupl(upl_t upl)
+{
+ return upl->vector_upl;
}
vm_size_t
}
void
-upl_set_dirty(
- upl_t upl)
+upl_clear_dirty(
+ upl_t upl,
+ boolean_t value)
{
- upl->flags |= UPL_CLEAR_DIRTY;
+ if (value) {
+ upl->flags |= UPL_CLEAR_DIRTY;
+ } else {
+ upl->flags &= ~UPL_CLEAR_DIRTY;
+ }
}
void
-upl_clear_dirty(
- upl_t upl)
+upl_set_referenced(
+ upl_t upl,
+ boolean_t value)
+{
+ upl_lock(upl);
+ if (value) {
+ upl->ext_ref_count++;
+ } else {
+ if (!upl->ext_ref_count) {
+ panic("upl_set_referenced not %p\n", upl);
+ }
+ upl->ext_ref_count--;
+ }
+ upl_unlock(upl);
+}
+
+#if CONFIG_IOSCHED
+void
+upl_set_blkno(
+ upl_t upl,
+ vm_offset_t upl_offset,
+ int io_size,
+ int64_t blkno)
{
- upl->flags &= ~UPL_CLEAR_DIRTY;
+ int i, j;
+ if ((upl->flags & UPL_EXPEDITE_SUPPORTED) == 0) {
+ return;
+ }
+
+ assert(upl->upl_reprio_info != 0);
+ for (i = (int)(upl_offset / PAGE_SIZE), j = 0; j < io_size; i++, j += PAGE_SIZE) {
+ UPL_SET_REPRIO_INFO(upl, i, blkno, io_size);
+ }
}
+#endif
+void inline
+memoryshot(unsigned int event, unsigned int control)
+{
+ if (vm_debug_events) {
+ KERNEL_DEBUG_CONSTANT1((MACHDBG_CODE(DBG_MACH_VM_PRESSURE, event)) | control,
+ vm_page_active_count, vm_page_inactive_count,
+ vm_page_free_count, vm_page_speculative_count,
+ vm_page_throttled_count);
+ } else {
+ (void) event;
+ (void) control;
+ }
+}
#ifdef MACH_BSD
-boolean_t upl_page_present(upl_page_info_t *upl, int index)
+boolean_t
+upl_device_page(upl_page_info_t *upl)
{
- return(UPL_PAGE_PRESENT(upl, index));
+ return UPL_DEVICE_PAGE(upl);
}
-boolean_t upl_dirty_page(upl_page_info_t *upl, int index)
+boolean_t
+upl_page_present(upl_page_info_t *upl, int index)
{
- return(UPL_DIRTY_PAGE(upl, index));
+ return UPL_PAGE_PRESENT(upl, index);
}
-boolean_t upl_valid_page(upl_page_info_t *upl, int index)
+boolean_t
+upl_speculative_page(upl_page_info_t *upl, int index)
{
- return(UPL_VALID_PAGE(upl, index));
+ return UPL_SPECULATIVE_PAGE(upl, index);
}
-ppnum_t upl_phys_page(upl_page_info_t *upl, int index)
+boolean_t
+upl_dirty_page(upl_page_info_t *upl, int index)
{
- return(UPL_PHYS_PAGE(upl, index));
+ return UPL_DIRTY_PAGE(upl, index);
+}
+boolean_t
+upl_valid_page(upl_page_info_t *upl, int index)
+{
+ return UPL_VALID_PAGE(upl, index);
+}
+ppnum_t
+upl_phys_page(upl_page_info_t *upl, int index)
+{
+ return UPL_PHYS_PAGE(upl, index);
+}
+
+void
+upl_page_set_mark(upl_page_info_t *upl, int index, boolean_t v)
+{
+ upl[index].mark = v;
+}
+
+boolean_t
+upl_page_get_mark(upl_page_info_t *upl, int index)
+{
+ return upl[index].mark;
}
void
int precpages;
- dpages=0;
- pgopages=0;
- precpages=0;
+ dpages = 0;
+ pgopages = 0;
+ precpages = 0;
vm_page_lock_queues();
- m = (vm_page_t) queue_first(&vm_page_queue_inactive);
+ m = (vm_page_t) vm_page_queue_first(&vm_page_queue_inactive);
do {
- if (m ==(vm_page_t )0) break;
-
- if(m->dirty) dpages++;
- if(m->pageout) pgopages++;
- if(m->precious) precpages++;
+ if (m == (vm_page_t)0) {
+ break;
+ }
- assert(m->object != kernel_object);
- m = (vm_page_t) queue_next(&m->pageq);
- if (m ==(vm_page_t )0) break;
+ if (m->vmp_dirty) {
+ dpages++;
+ }
+ if (m->vmp_free_when_done) {
+ pgopages++;
+ }
+ if (m->vmp_precious) {
+ precpages++;
+ }
- } while (!queue_end(&vm_page_queue_inactive,(queue_entry_t) m));
+ assert(VM_PAGE_OBJECT(m) != kernel_object);
+ m = (vm_page_t) vm_page_queue_next(&m->vmp_pageq);
+ if (m == (vm_page_t)0) {
+ break;
+ }
+ } while (!vm_page_queue_end(&vm_page_queue_inactive, (vm_page_queue_entry_t) m));
vm_page_unlock_queues();
vm_page_lock_queues();
- m = (vm_page_t) queue_first(&vm_page_queue_zf);
+ m = (vm_page_t) vm_page_queue_first(&vm_page_queue_throttled);
do {
- if (m ==(vm_page_t )0) break;
+ if (m == (vm_page_t)0) {
+ break;
+ }
+
+ dpages++;
+ assert(m->vmp_dirty);
+ assert(!m->vmp_free_when_done);
+ assert(VM_PAGE_OBJECT(m) != kernel_object);
+ m = (vm_page_t) vm_page_queue_next(&m->vmp_pageq);
+ if (m == (vm_page_t)0) {
+ break;
+ }
+ } while (!vm_page_queue_end(&vm_page_queue_throttled, (vm_page_queue_entry_t) m));
+ vm_page_unlock_queues();
- if(m->dirty) dpages++;
- if(m->pageout) pgopages++;
- if(m->precious) precpages++;
+ vm_page_lock_queues();
+ m = (vm_page_t) vm_page_queue_first(&vm_page_queue_anonymous);
+ do {
+ if (m == (vm_page_t)0) {
+ break;
+ }
- assert(m->object != kernel_object);
- m = (vm_page_t) queue_next(&m->pageq);
- if (m ==(vm_page_t )0) break;
+ if (m->vmp_dirty) {
+ dpages++;
+ }
+ if (m->vmp_free_when_done) {
+ pgopages++;
+ }
+ if (m->vmp_precious) {
+ precpages++;
+ }
- } while (!queue_end(&vm_page_queue_zf,(queue_entry_t) m));
+ assert(VM_PAGE_OBJECT(m) != kernel_object);
+ m = (vm_page_t) vm_page_queue_next(&m->vmp_pageq);
+ if (m == (vm_page_t)0) {
+ break;
+ }
+ } while (!vm_page_queue_end(&vm_page_queue_anonymous, (vm_page_queue_entry_t) m));
vm_page_unlock_queues();
printf("IN Q: %d : %d : %d\n", dpages, pgopages, precpages);
- dpages=0;
- pgopages=0;
- precpages=0;
+ dpages = 0;
+ pgopages = 0;
+ precpages = 0;
vm_page_lock_queues();
- m = (vm_page_t) queue_first(&vm_page_queue_active);
+ m = (vm_page_t) vm_page_queue_first(&vm_page_queue_active);
do {
- if(m == (vm_page_t )0) break;
- if(m->dirty) dpages++;
- if(m->pageout) pgopages++;
- if(m->precious) precpages++;
-
- assert(m->object != kernel_object);
- m = (vm_page_t) queue_next(&m->pageq);
- if(m == (vm_page_t )0) break;
+ if (m == (vm_page_t)0) {
+ break;
+ }
+ if (m->vmp_dirty) {
+ dpages++;
+ }
+ if (m->vmp_free_when_done) {
+ pgopages++;
+ }
+ if (m->vmp_precious) {
+ precpages++;
+ }
- } while (!queue_end(&vm_page_queue_active,(queue_entry_t) m));
+ assert(VM_PAGE_OBJECT(m) != kernel_object);
+ m = (vm_page_t) vm_page_queue_next(&m->vmp_pageq);
+ if (m == (vm_page_t)0) {
+ break;
+ }
+ } while (!vm_page_queue_end(&vm_page_queue_active, (vm_page_queue_entry_t) m));
vm_page_unlock_queues();
printf("AC Q: %d : %d : %d\n", dpages, pgopages, precpages);
-
}
#endif /* MACH_BSD */
-#ifdef UPL_DEBUG
-kern_return_t upl_ubc_alias_set(upl_t upl, unsigned int alias1, unsigned int alias2)
+
+#if CONFIG_IOSCHED
+int
+upl_get_cached_tier(upl_t upl)
+{
+ assert(upl);
+ if (upl->flags & UPL_TRACKED_BY_OBJECT) {
+ return upl->upl_priority;
+ }
+ return -1;
+}
+#endif /* CONFIG_IOSCHED */
+
+
+void
+upl_callout_iodone(upl_t upl)
+{
+ struct upl_io_completion *upl_ctx = upl->upl_iodone;
+
+ if (upl_ctx) {
+ void (*iodone_func)(void *, int) = upl_ctx->io_done;
+
+ assert(upl_ctx->io_done);
+
+ (*iodone_func)(upl_ctx->io_context, upl_ctx->io_error);
+ }
+}
+
+void
+upl_set_iodone(upl_t upl, void *upl_iodone)
+{
+ upl->upl_iodone = (struct upl_io_completion *)upl_iodone;
+}
+
+void
+upl_set_iodone_error(upl_t upl, int error)
+{
+ struct upl_io_completion *upl_ctx = upl->upl_iodone;
+
+ if (upl_ctx) {
+ upl_ctx->io_error = error;
+ }
+}
+
+
+ppnum_t
+upl_get_highest_page(
+ upl_t upl)
+{
+ return upl->highest_page;
+}
+
+upl_size_t
+upl_get_size(
+ upl_t upl)
+{
+ return upl_adjusted_size(upl, PAGE_MASK);
+}
+
+upl_size_t
+upl_adjusted_size(
+ upl_t upl,
+ vm_map_offset_t pgmask)
+{
+ vm_object_offset_t start_offset, end_offset;
+
+ start_offset = trunc_page_mask_64(upl->u_offset, pgmask);
+ end_offset = round_page_mask_64(upl->u_offset + upl->u_size, pgmask);
+
+ return (upl_size_t)(end_offset - start_offset);
+}
+
+vm_object_offset_t
+upl_adjusted_offset(
+ upl_t upl,
+ vm_map_offset_t pgmask)
+{
+ return trunc_page_mask_64(upl->u_offset, pgmask);
+}
+
+vm_object_offset_t
+upl_get_data_offset(
+ upl_t upl)
+{
+ return upl->u_offset - upl_adjusted_offset(upl, PAGE_MASK);
+}
+
+upl_t
+upl_associated_upl(upl_t upl)
+{
+ return upl->associated_upl;
+}
+
+void
+upl_set_associated_upl(upl_t upl, upl_t associated_upl)
+{
+ upl->associated_upl = associated_upl;
+}
+
+struct vnode *
+upl_lookup_vnode(upl_t upl)
+{
+ if (!upl->map_object->internal) {
+ return vnode_pager_lookup_vnode(upl->map_object->pager);
+ } else {
+ return NULL;
+ }
+}
+
+#if UPL_DEBUG
+kern_return_t
+upl_ubc_alias_set(upl_t upl, uintptr_t alias1, uintptr_t alias2)
{
upl->ubc_alias1 = alias1;
upl->ubc_alias2 = alias2;
return KERN_SUCCESS;
}
-int upl_ubc_alias_get(upl_t upl, unsigned int * al, unsigned int * al2)
+int
+upl_ubc_alias_get(upl_t upl, uintptr_t * al, uintptr_t * al2)
{
- if(al)
+ if (al) {
*al = upl->ubc_alias1;
- if(al2)
+ }
+ if (al2) {
*al2 = upl->ubc_alias2;
+ }
return KERN_SUCCESS;
}
#endif /* UPL_DEBUG */
+#if VM_PRESSURE_EVENTS
+/*
+ * Upward trajectory.
+ */
+extern boolean_t vm_compressor_low_on_space(void);
-
-#if MACH_KDB
-#include <ddb/db_output.h>
-#include <ddb/db_print.h>
-#include <vm/vm_print.h>
-
-#define printf kdbprintf
-void db_pageout(void);
-
-void
-db_vm(void)
+boolean_t
+VM_PRESSURE_NORMAL_TO_WARNING(void)
{
-
- iprintf("VM Statistics:\n");
- db_indent += 2;
- iprintf("pages:\n");
- db_indent += 2;
- iprintf("activ %5d inact %5d free %5d",
- vm_page_active_count, vm_page_inactive_count,
- vm_page_free_count);
- printf(" wire %5d gobbl %5d\n",
- vm_page_wire_count, vm_page_gobble_count);
- db_indent -= 2;
- iprintf("target:\n");
- db_indent += 2;
- iprintf("min %5d inact %5d free %5d",
- vm_page_free_min, vm_page_inactive_target,
- vm_page_free_target);
- printf(" resrv %5d\n", vm_page_free_reserved);
- db_indent -= 2;
- iprintf("pause:\n");
- db_pageout();
- db_indent -= 2;
+ if (!VM_CONFIG_COMPRESSOR_IS_ACTIVE) {
+ /* Available pages below our threshold */
+ if (memorystatus_available_pages < memorystatus_available_pages_pressure) {
+ /* No frozen processes to kill */
+ if (memorystatus_frozen_count == 0) {
+ /* Not enough suspended processes available. */
+ if (memorystatus_suspended_count < MEMORYSTATUS_SUSPENDED_THRESHOLD) {
+ return TRUE;
+ }
+ }
+ }
+ return FALSE;
+ } else {
+ return (AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD) ? 1 : 0;
+ }
}
-#if MACH_COUNTERS
-extern int c_laundry_pages_freed;
-#endif /* MACH_COUNTERS */
+boolean_t
+VM_PRESSURE_WARNING_TO_CRITICAL(void)
+{
+ if (!VM_CONFIG_COMPRESSOR_IS_ACTIVE) {
+ /* Available pages below our threshold */
+ if (memorystatus_available_pages < memorystatus_available_pages_critical) {
+ return TRUE;
+ }
+ return FALSE;
+ } else {
+ return vm_compressor_low_on_space() || (AVAILABLE_NON_COMPRESSED_MEMORY < ((12 * VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) / 10)) ? 1 : 0;
+ }
+}
-void
-db_pageout(void)
+/*
+ * Downward trajectory.
+ */
+boolean_t
+VM_PRESSURE_WARNING_TO_NORMAL(void)
{
- iprintf("Pageout Statistics:\n");
- db_indent += 2;
- iprintf("active %5d inactv %5d\n",
- vm_pageout_active, vm_pageout_inactive);
- iprintf("nolock %5d avoid %5d busy %5d absent %5d\n",
- vm_pageout_inactive_nolock, vm_pageout_inactive_avoid,
- vm_pageout_inactive_busy, vm_pageout_inactive_absent);
- iprintf("used %5d clean %5d dirty %5d\n",
- vm_pageout_inactive_used, vm_pageout_inactive_clean,
- vm_pageout_inactive_dirty);
-#if MACH_COUNTERS
- iprintf("laundry_pages_freed %d\n", c_laundry_pages_freed);
-#endif /* MACH_COUNTERS */
-#if MACH_CLUSTER_STATS
- iprintf("Cluster Statistics:\n");
- db_indent += 2;
- iprintf("dirtied %5d cleaned %5d collisions %5d\n",
- vm_pageout_cluster_dirtied, vm_pageout_cluster_cleaned,
- vm_pageout_cluster_collisions);
- iprintf("clusters %5d conversions %5d\n",
- vm_pageout_cluster_clusters, vm_pageout_cluster_conversions);
- db_indent -= 2;
- iprintf("Target Statistics:\n");
- db_indent += 2;
- iprintf("collisions %5d page_dirtied %5d page_freed %5d\n",
- vm_pageout_target_collisions, vm_pageout_target_page_dirtied,
- vm_pageout_target_page_freed);
- db_indent -= 2;
-#endif /* MACH_CLUSTER_STATS */
- db_indent -= 2;
+ if (!VM_CONFIG_COMPRESSOR_IS_ACTIVE) {
+ /* Available pages above our threshold */
+ unsigned int target_threshold = (unsigned int) (memorystatus_available_pages_pressure + ((15 * memorystatus_available_pages_pressure) / 100));
+ if (memorystatus_available_pages > target_threshold) {
+ return TRUE;
+ }
+ return FALSE;
+ } else {
+ return (AVAILABLE_NON_COMPRESSED_MEMORY > ((12 * VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD) / 10)) ? 1 : 0;
+ }
}
-#endif /* MACH_KDB */
+boolean_t
+VM_PRESSURE_CRITICAL_TO_WARNING(void)
+{
+ if (!VM_CONFIG_COMPRESSOR_IS_ACTIVE) {
+ /* Available pages above our threshold */
+ unsigned int target_threshold = (unsigned int)(memorystatus_available_pages_critical + ((15 * memorystatus_available_pages_critical) / 100));
+ if (memorystatus_available_pages > target_threshold) {
+ return TRUE;
+ }
+ return FALSE;
+ } else {
+ return (AVAILABLE_NON_COMPRESSED_MEMORY > ((14 * VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) / 10)) ? 1 : 0;
+ }
+}
+#endif /* VM_PRESSURE_EVENTS */
projects / apple / xnu.git / blobdiff