xnu-4903.270.47.tar.gz

[apple/xnu.git] / osfmk / vm / vm_pageout.c

diff --git a/osfmk/vm/vm_pageout.c b/osfmk/vm/vm_pageout.c
index ada634f90184ce8c39a7c073eb578ffcfb778bb6..6b6e3d04d6796dff5f35142bcf8b61a0012aacc8 100644 (file)
--- a/osfmk/vm/vm_pageout.c
+++ b/osfmk/vm/vm_pageout.c
@@ -2,7 +2,7 @@
  * Copyright (c) 2000-2014 Apple Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  * Copyright (c) 2000-2014 Apple Inc. All rights reserved.
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

- * 
+ *

  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in


@@ -11,10 +11,10 @@
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.

- * 
+ *

  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.

- * 
+ *

  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,


@@ -22,34 +22,34 @@
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.

- * 
+ *

  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
  */
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
  */

-/* 
+/*

  * Mach Operating System
  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
  * All Rights Reserved.
  * Mach Operating System
  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
  * All Rights Reserved.

- * 
+ *

  * Permission to use, copy, modify and distribute this software and its
  * documentation is hereby granted, provided that both the copyright
  * notice and this permission notice appear in all copies of the
  * software, derivative works or modified versions, and any portions
  * thereof, and that both notices appear in supporting documentation.
  * Permission to use, copy, modify and distribute this software and its
  * documentation is hereby granted, provided that both the copyright
  * notice and this permission notice appear in all copies of the
  * software, derivative works or modified versions, and any portions
  * thereof, and that both notices appear in supporting documentation.

- * 
+ *

  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.

- * 
+ *

  * Carnegie Mellon requests users of this software to return to
  * Carnegie Mellon requests users of this software to return to

- * 
+ *

  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  *  School of Computer Science
  *  Carnegie Mellon University
  *  Pittsburgh PA 15213-3890
  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  *  School of Computer Science
  *  Carnegie Mellon University
  *  Pittsburgh PA 15213-3890

- * 
+ *

  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */


@@ -114,13 +114,13 @@
 #include <vm/vm_phantom_cache.h>
 #endif
 
 #include <vm/vm_phantom_cache.h>
 #endif
 

-extern int cs_debug;
-

 #if UPL_DEBUG
 #include <libkern/OSDebug.h>
 #endif
 
 #if UPL_DEBUG
 #include <libkern/OSDebug.h>
 #endif
 

-extern void m_drain(void);
+extern int cs_debug;
+
+extern void mbuf_drain(boolean_t);

 
 #if VM_PRESSURE_EVENTS
 #if CONFIG_JETSAM
 
 #if VM_PRESSURE_EVENTS
 #if CONFIG_JETSAM


@@ -135,27 +135,17 @@ extern uint64_t memorystatus_available_pages_critical;
 
 extern unsigned int memorystatus_frozen_count;
 extern unsigned int memorystatus_suspended_count;
 
 extern unsigned int memorystatus_frozen_count;
 extern unsigned int memorystatus_suspended_count;

-

 extern vm_pressure_level_t memorystatus_vm_pressure_level;
 extern vm_pressure_level_t memorystatus_vm_pressure_level;

-int memorystatus_purge_on_warning = 2;
-int memorystatus_purge_on_urgent = 5;
-int memorystatus_purge_on_critical = 8;

 
 void vm_pressure_response(void);
 
 void vm_pressure_response(void);

-boolean_t vm_pressure_thread_running = FALSE;

 extern void consider_vm_pressure_events(void);
 
 #define MEMORYSTATUS_SUSPENDED_THRESHOLD  4
 #endif /* VM_PRESSURE_EVENTS */
 
 extern void consider_vm_pressure_events(void);
 
 #define MEMORYSTATUS_SUSPENDED_THRESHOLD  4
 #endif /* VM_PRESSURE_EVENTS */
 

-boolean_t      vm_pressure_changed = FALSE;
-
-#ifndef VM_PAGEOUT_BURST_ACTIVE_THROTTLE   /* maximum iterations of the active queue to move pages to inactive */
-#define VM_PAGEOUT_BURST_ACTIVE_THROTTLE  100
-#endif

 
 #ifndef VM_PAGEOUT_BURST_INACTIVE_THROTTLE  /* maximum iterations of the inactive queue w/o stealing/cleaning a page */
 
 #ifndef VM_PAGEOUT_BURST_INACTIVE_THROTTLE  /* maximum iterations of the inactive queue w/o stealing/cleaning a page */

-#ifdef CONFIG_EMBEDDED
+#ifdef  CONFIG_EMBEDDED

 #define VM_PAGEOUT_BURST_INACTIVE_THROTTLE 1024
 #else
 #define VM_PAGEOUT_BURST_INACTIVE_THROTTLE 4096
 #define VM_PAGEOUT_BURST_INACTIVE_THROTTLE 1024
 #else
 #define VM_PAGEOUT_BURST_INACTIVE_THROTTLE 4096


@@ -163,50 +153,36 @@ boolean_t vm_pressure_changed = FALSE;
 #endif
 
 #ifndef VM_PAGEOUT_DEADLOCK_RELIEF
 #endif
 
 #ifndef VM_PAGEOUT_DEADLOCK_RELIEF

-#define VM_PAGEOUT_DEADLOCK_RELIEF 100 /* number of pages to move to break deadlock */
-#endif
-
-#ifndef VM_PAGEOUT_INACTIVE_RELIEF
-#define VM_PAGEOUT_INACTIVE_RELIEF 50  /* minimum number of pages to move to the inactive q */
+#define VM_PAGEOUT_DEADLOCK_RELIEF 100  /* number of pages to move to break deadlock */

 #endif
 
 #endif
 

-#ifndef        VM_PAGE_LAUNDRY_MAX
-#define        VM_PAGE_LAUNDRY_MAX     128UL   /* maximum pageouts on a given pageout queue */
-#endif /* VM_PAGEOUT_LAUNDRY_MAX */
-
-#ifndef        VM_PAGEOUT_BURST_WAIT
-#define        VM_PAGEOUT_BURST_WAIT   10      /* milliseconds */
-#endif /* VM_PAGEOUT_BURST_WAIT */
+#ifndef VM_PAGE_LAUNDRY_MAX
+#define VM_PAGE_LAUNDRY_MAX     128UL   /* maximum pageouts on a given pageout queue */
+#endif  /* VM_PAGEOUT_LAUNDRY_MAX */

 
 

-#ifndef        VM_PAGEOUT_EMPTY_WAIT
-#define VM_PAGEOUT_EMPTY_WAIT  200     /* milliseconds */
-#endif /* VM_PAGEOUT_EMPTY_WAIT */
+#ifndef VM_PAGEOUT_BURST_WAIT
+#define VM_PAGEOUT_BURST_WAIT   1       /* milliseconds */
+#endif  /* VM_PAGEOUT_BURST_WAIT */

 
 

-#ifndef        VM_PAGEOUT_DEADLOCK_WAIT
-#define VM_PAGEOUT_DEADLOCK_WAIT       300     /* milliseconds */
-#endif /* VM_PAGEOUT_DEADLOCK_WAIT */
+#ifndef VM_PAGEOUT_EMPTY_WAIT
+#define VM_PAGEOUT_EMPTY_WAIT   50      /* milliseconds */
+#endif  /* VM_PAGEOUT_EMPTY_WAIT */

 
 

-#ifndef        VM_PAGEOUT_IDLE_WAIT
-#define VM_PAGEOUT_IDLE_WAIT   10      /* milliseconds */
-#endif /* VM_PAGEOUT_IDLE_WAIT */
+#ifndef VM_PAGEOUT_DEADLOCK_WAIT
+#define VM_PAGEOUT_DEADLOCK_WAIT 100    /* milliseconds */
+#endif  /* VM_PAGEOUT_DEADLOCK_WAIT */

 
 

-#ifndef        VM_PAGEOUT_SWAP_WAIT
-#define VM_PAGEOUT_SWAP_WAIT   50      /* milliseconds */
-#endif /* VM_PAGEOUT_SWAP_WAIT */
+#ifndef VM_PAGEOUT_IDLE_WAIT
+#define VM_PAGEOUT_IDLE_WAIT    10      /* milliseconds */
+#endif  /* VM_PAGEOUT_IDLE_WAIT */

 
 

-#ifndef VM_PAGEOUT_PRESSURE_PAGES_CONSIDERED
-#define VM_PAGEOUT_PRESSURE_PAGES_CONSIDERED           1000    /* maximum pages considered before we issue a pressure event */
-#endif /* VM_PAGEOUT_PRESSURE_PAGES_CONSIDERED */
+#ifndef VM_PAGEOUT_SWAP_WAIT
+#define VM_PAGEOUT_SWAP_WAIT    10      /* milliseconds */
+#endif  /* VM_PAGEOUT_SWAP_WAIT */

 
 

-#ifndef VM_PAGEOUT_PRESSURE_EVENT_MONITOR_SECS
-#define VM_PAGEOUT_PRESSURE_EVENT_MONITOR_SECS         5       /* seconds */
-#endif /* VM_PAGEOUT_PRESSURE_EVENT_MONITOR_SECS */
-
-unsigned int   vm_page_speculative_q_age_ms = VM_PAGE_SPECULATIVE_Q_AGE_MS;
-unsigned int   vm_page_speculative_percentage = 5;

 
 #ifndef VM_PAGE_SPECULATIVE_TARGET
 
 #ifndef VM_PAGE_SPECULATIVE_TARGET

-#define VM_PAGE_SPECULATIVE_TARGET(total) ((total) * 1 / (100 / vm_page_speculative_percentage))
+#define VM_PAGE_SPECULATIVE_TARGET(total) ((total) * 1 / (100 / vm_pageout_state.vm_page_speculative_percentage))

 #endif /* VM_PAGE_SPECULATIVE_TARGET */
 
 
 #endif /* VM_PAGE_SPECULATIVE_TARGET */
 
 


@@ -222,26 +198,22 @@ unsigned int      vm_page_speculative_percentage = 5;
  *     then the pageout daemon starts running.
  */
 
  *     then the pageout daemon starts running.
  */
 

-#ifndef        VM_PAGE_INACTIVE_TARGET
-#ifdef CONFIG_EMBEDDED
-#define        VM_PAGE_INACTIVE_TARGET(avail)  ((avail) * 1 / 3)
-#else
-#define        VM_PAGE_INACTIVE_TARGET(avail)  ((avail) * 1 / 2)
-#endif
-#endif /* VM_PAGE_INACTIVE_TARGET */
+#ifndef VM_PAGE_INACTIVE_TARGET
+#define VM_PAGE_INACTIVE_TARGET(avail)  ((avail) * 1 / 2)
+#endif  /* VM_PAGE_INACTIVE_TARGET */

 
 /*
  *     Once the pageout daemon starts running, it keeps going
  *     until vm_page_free_count meets or exceeds vm_page_free_target.
  */
 
 
 /*
  *     Once the pageout daemon starts running, it keeps going
  *     until vm_page_free_count meets or exceeds vm_page_free_target.
  */
 

-#ifndef        VM_PAGE_FREE_TARGET
-#ifdef CONFIG_EMBEDDED
-#define        VM_PAGE_FREE_TARGET(free)       (15 + (free) / 100)
+#ifndef VM_PAGE_FREE_TARGET
+#ifdef  CONFIG_EMBEDDED
+#define VM_PAGE_FREE_TARGET(free)       (15 + (free) / 100)

 #else
 #else

-#define        VM_PAGE_FREE_TARGET(free)       (15 + (free) / 80)
+#define VM_PAGE_FREE_TARGET(free)       (15 + (free) / 80)

 #endif
 #endif

-#endif /* VM_PAGE_FREE_TARGET */
+#endif  /* VM_PAGE_FREE_TARGET */

 
 
 /*
 
 
 /*


@@ -249,22 +221,22 @@ unsigned int      vm_page_speculative_percentage = 5;
  *     falls below vm_page_free_min.
  */
 
  *     falls below vm_page_free_min.
  */
 

-#ifndef        VM_PAGE_FREE_MIN
-#ifdef CONFIG_EMBEDDED
-#define        VM_PAGE_FREE_MIN(free)          (10 + (free) / 200)
+#ifndef VM_PAGE_FREE_MIN
+#ifdef  CONFIG_EMBEDDED
+#define VM_PAGE_FREE_MIN(free)          (10 + (free) / 200)

 #else
 #else

-#define        VM_PAGE_FREE_MIN(free)          (10 + (free) / 100)
+#define VM_PAGE_FREE_MIN(free)          (10 + (free) / 100)

 #endif
 #endif

-#endif /* VM_PAGE_FREE_MIN */
+#endif  /* VM_PAGE_FREE_MIN */

 
 

-#ifdef CONFIG_EMBEDDED
-#define VM_PAGE_FREE_RESERVED_LIMIT    100
-#define VM_PAGE_FREE_MIN_LIMIT         1500
-#define VM_PAGE_FREE_TARGET_LIMIT      2000
+#ifdef  CONFIG_EMBEDDED
+#define VM_PAGE_FREE_RESERVED_LIMIT     100
+#define VM_PAGE_FREE_MIN_LIMIT          1500
+#define VM_PAGE_FREE_TARGET_LIMIT       2000

 #else
 #else

-#define VM_PAGE_FREE_RESERVED_LIMIT    1700
-#define VM_PAGE_FREE_MIN_LIMIT         3500
-#define VM_PAGE_FREE_TARGET_LIMIT      4000
+#define VM_PAGE_FREE_RESERVED_LIMIT     1700
+#define VM_PAGE_FREE_MIN_LIMIT          3500
+#define VM_PAGE_FREE_TARGET_LIMIT       4000

 #endif
 
 /*
 #endif
 
 /*


@@ -275,10 +247,10 @@ unsigned int      vm_page_speculative_percentage = 5;
  *     operation by dipping into the reserved pool of pages.
  */
 
  *     operation by dipping into the reserved pool of pages.
  */
 

-#ifndef        VM_PAGE_FREE_RESERVED
-#define        VM_PAGE_FREE_RESERVED(n)        \
+#ifndef VM_PAGE_FREE_RESERVED
+#define VM_PAGE_FREE_RESERVED(n)        \

        ((unsigned) (6 * VM_PAGE_LAUNDRY_MAX) + (n))
        ((unsigned) (6 * VM_PAGE_LAUNDRY_MAX) + (n))

-#endif /* VM_PAGE_FREE_RESERVED */
+#endif  /* VM_PAGE_FREE_RESERVED */

 
 /*
  *     When we dequeue pages from the inactive list, they are
 
 /*
  *     When we dequeue pages from the inactive list, they are


@@ -289,35 +261,26 @@ unsigned int      vm_page_speculative_percentage = 5;
  *     we will make per call of vm_pageout_scan().
  */
 #define VM_PAGE_REACTIVATE_LIMIT_MAX 20000
  *     we will make per call of vm_pageout_scan().
  */
 #define VM_PAGE_REACTIVATE_LIMIT_MAX 20000

-#ifndef        VM_PAGE_REACTIVATE_LIMIT
-#ifdef CONFIG_EMBEDDED
-#define        VM_PAGE_REACTIVATE_LIMIT(avail) (VM_PAGE_INACTIVE_TARGET(avail) / 2)
+
+#ifndef VM_PAGE_REACTIVATE_LIMIT
+#ifdef  CONFIG_EMBEDDED
+#define VM_PAGE_REACTIVATE_LIMIT(avail) (VM_PAGE_INACTIVE_TARGET(avail) / 2)

 #else
 #else

-#define        VM_PAGE_REACTIVATE_LIMIT(avail) (MAX((avail) * 1 / 20,VM_PAGE_REACTIVATE_LIMIT_MAX))
+#define VM_PAGE_REACTIVATE_LIMIT(avail) (MAX((avail) * 1 / 20,VM_PAGE_REACTIVATE_LIMIT_MAX))

 #endif
 #endif

-#endif /* VM_PAGE_REACTIVATE_LIMIT */
-#define VM_PAGEOUT_INACTIVE_FORCE_RECLAIM      1000
-
+#endif  /* VM_PAGE_REACTIVATE_LIMIT */
+#define VM_PAGEOUT_INACTIVE_FORCE_RECLAIM       1000

 
 extern boolean_t hibernate_cleaning_in_progress;
 
 
 extern boolean_t hibernate_cleaning_in_progress;
 

-/*
- * Exported variable used to broadcast the activation of the pageout scan
- * Working Set uses this to throttle its use of pmap removes.  In this
- * way, code which runs within memory in an uncontested context does
- * not keep encountering soft faults.
- */
-
-unsigned int   vm_pageout_scan_event_counter = 0;
-

 /*
  * Forward declarations for internal routines.
  */
 struct cq {
        struct vm_pageout_queue *q;
 /*
  * Forward declarations for internal routines.
  */
 struct cq {
        struct vm_pageout_queue *q;

-       void                    *current_chead;
-       char                    *scratch_buf;
-       int                     id;
+       void                    *current_chead;
+       char                    *scratch_buf;
+       int                     id;

 };
 
 struct cq ciq[MAX_COMPRESSOR_THREAD_COUNT];
 };
 
 struct cq ciq[MAX_COMPRESSOR_THREAD_COUNT];


@@ -332,6 +295,7 @@ boolean_t VM_PRESSURE_WARNING_TO_CRITICAL(void);
 boolean_t VM_PRESSURE_WARNING_TO_NORMAL(void);
 boolean_t VM_PRESSURE_CRITICAL_TO_WARNING(void);
 #endif
 boolean_t VM_PRESSURE_WARNING_TO_NORMAL(void);
 boolean_t VM_PRESSURE_CRITICAL_TO_WARNING(void);
 #endif

+

 void vm_pageout_garbage_collect(int);
 static void vm_pageout_iothread_external(void);
 static void vm_pageout_iothread_internal(struct cq *cq);
 void vm_pageout_garbage_collect(int);
 static void vm_pageout_iothread_external(void);
 static void vm_pageout_iothread_internal(struct cq *cq);


@@ -339,184 +303,58 @@ static void vm_pageout_adjust_eq_iothrottle(struct vm_pageout_queue *, boolean_t
 
 extern void vm_pageout_continue(void);
 extern void vm_pageout_scan(void);
 
 extern void vm_pageout_continue(void);
 extern void vm_pageout_scan(void);

+

 void vm_tests(void); /* forward */
 
 void vm_tests(void); /* forward */
 

-boolean_t      vm_restricted_to_single_processor = FALSE;

 #if !CONFIG_EMBEDDED
 static boolean_t vm_pageout_waiter  = FALSE;
 static boolean_t vm_pageout_running = FALSE;
 #endif /* !CONFIG_EMBEDDED */
 
 
 #if !CONFIG_EMBEDDED
 static boolean_t vm_pageout_waiter  = FALSE;
 static boolean_t vm_pageout_running = FALSE;
 #endif /* !CONFIG_EMBEDDED */
 
 

-static thread_t        vm_pageout_external_iothread = THREAD_NULL;
-static thread_t        vm_pageout_internal_iothread = THREAD_NULL;
-
-unsigned int vm_pageout_reserved_internal = 0;
-unsigned int vm_pageout_reserved_really = 0;
-
-unsigned int vm_pageout_swap_wait = 0;
-unsigned int vm_pageout_idle_wait = 0;         /* milliseconds */
-unsigned int vm_pageout_empty_wait = 0;                /* milliseconds */
-unsigned int vm_pageout_burst_wait = 0;                /* milliseconds */
-unsigned int vm_pageout_deadlock_wait = 0;     /* milliseconds */
-unsigned int vm_pageout_deadlock_relief = 0;
-unsigned int vm_pageout_inactive_relief = 0;
-unsigned int vm_pageout_burst_active_throttle = 0;
-unsigned int vm_pageout_burst_inactive_throttle = 0;
-
-int    vm_upl_wait_for_pages = 0;
-
-
-/*
- *     These variables record the pageout daemon's actions:
- *     how many pages it looks at and what happens to those pages.
- *     No locking needed because only one thread modifies the variables.
- */
-
-unsigned int vm_pageout_active = 0;            /* debugging */
-unsigned int vm_pageout_inactive = 0;          /* debugging */
-unsigned int vm_pageout_inactive_throttled = 0;        /* debugging */
-unsigned int vm_pageout_inactive_forced = 0;   /* debugging */
-unsigned int vm_pageout_inactive_nolock = 0;   /* debugging */
-unsigned int vm_pageout_inactive_avoid = 0;    /* debugging */
-unsigned int vm_pageout_inactive_busy = 0;     /* debugging */
-unsigned int vm_pageout_inactive_error = 0;    /* debugging */
-unsigned int vm_pageout_inactive_absent = 0;   /* debugging */
-unsigned int vm_pageout_inactive_notalive = 0; /* debugging */
-unsigned int vm_pageout_inactive_used = 0;     /* debugging */
-unsigned int vm_pageout_cache_evicted = 0;     /* debugging */
-unsigned int vm_pageout_inactive_clean = 0;    /* debugging */
-unsigned int vm_pageout_speculative_clean = 0; /* debugging */
-unsigned int vm_pageout_speculative_dirty = 0; /* debugging */
-
-unsigned int vm_pageout_freed_from_cleaned = 0;
-unsigned int vm_pageout_freed_from_speculative = 0;
-unsigned int vm_pageout_freed_from_inactive_clean = 0;
-unsigned int vm_pageout_freed_after_compression = 0;
-
-extern uint32_t vm_compressor_pages_grabbed;
-extern  uint32_t c_segment_pages_compressed;
-
-unsigned int vm_pageout_enqueued_cleaned_from_inactive_dirty = 0;
-
-unsigned int vm_pageout_cleaned_reclaimed = 0;         /* debugging; how many cleaned pages are reclaimed by the pageout scan */
-unsigned int vm_pageout_cleaned_reactivated = 0;       /* debugging; how many cleaned pages are found to be referenced on pageout (and are therefore reactivated) */
-unsigned int vm_pageout_cleaned_reference_reactivated = 0;
-unsigned int vm_pageout_cleaned_volatile_reactivated = 0;
-unsigned int vm_pageout_cleaned_fault_reactivated = 0;
-unsigned int vm_pageout_cleaned_commit_reactivated = 0;        /* debugging; how many cleaned pages are found to be referenced on commit (and are therefore reactivated) */
-unsigned int vm_pageout_cleaned_busy = 0;
-unsigned int vm_pageout_cleaned_nolock = 0;
-
-unsigned int vm_pageout_inactive_dirty_internal = 0;   /* debugging */
-unsigned int vm_pageout_inactive_dirty_external = 0;   /* debugging */
-unsigned int vm_pageout_inactive_deactivated = 0;      /* debugging */
-unsigned int vm_pageout_inactive_anonymous = 0;        /* debugging */
-unsigned int vm_pageout_dirty_no_pager = 0;    /* debugging */
-unsigned int vm_pageout_purged_objects = 0;    /* used for sysctl vm stats */
-unsigned int vm_stat_discard = 0;              /* debugging */
-unsigned int vm_stat_discard_sent = 0;         /* debugging */
-unsigned int vm_stat_discard_failure = 0;      /* debugging */
-unsigned int vm_stat_discard_throttle = 0;     /* debugging */
-unsigned int vm_pageout_reactivation_limit_exceeded = 0;       /* debugging */
-unsigned int vm_pageout_inactive_force_reclaim = 0;    /* debugging */
-unsigned int vm_pageout_skipped_external = 0;   /* debugging */
-
-unsigned int vm_pageout_scan_reclaimed_throttled = 0;
-unsigned int vm_pageout_scan_active_throttled = 0;
-unsigned int vm_pageout_scan_inactive_throttled_internal = 0;
-unsigned int vm_pageout_scan_inactive_throttled_external = 0;
-unsigned int vm_pageout_scan_throttle = 0;                     /* debugging */
-unsigned int vm_pageout_scan_burst_throttle = 0;               /* debugging */
-unsigned int vm_pageout_scan_empty_throttle = 0;               /* debugging */
-unsigned int vm_pageout_scan_swap_throttle = 0;                /* debugging */
-unsigned int vm_pageout_scan_deadlock_detected = 0;            /* debugging */
-unsigned int vm_pageout_scan_active_throttle_success = 0;      /* debugging */
-unsigned int vm_pageout_scan_inactive_throttle_success = 0;    /* debugging */
-unsigned int vm_pageout_inactive_external_forced_jetsam_count = 0;     /* debugging */
-unsigned int vm_pageout_scan_throttle_deferred = 0;            /* debugging */
-unsigned int vm_pageout_scan_yield_unthrottled = 0;            /* debugging */
-unsigned int vm_page_speculative_count_drifts = 0;
-unsigned int vm_page_speculative_count_drift_max = 0;
-
-uint32_t vm_compressor_failed;
-
-/*
- * Backing store throttle when BS is exhausted
- */
-unsigned int   vm_backing_store_low = 0;
-
-unsigned int vm_pageout_out_of_line  = 0;
-unsigned int vm_pageout_in_place  = 0;
-
-unsigned int vm_page_steal_pageout_page = 0;
-
-struct vm_config       vm_config;
-
-struct vm_pageout_queue vm_pageout_queue_internal __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT)));
-struct vm_pageout_queue vm_pageout_queue_external __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT)));
+#if DEVELOPMENT || DEBUG
+struct vm_pageout_debug vm_pageout_debug;
+#endif
+struct vm_pageout_vminfo vm_pageout_vminfo;
+struct vm_pageout_state  vm_pageout_state;
+struct vm_config         vm_config;

 
 

-unsigned int vm_page_speculative_target = 0;
+struct  vm_pageout_queue vm_pageout_queue_internal __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT)));
+struct  vm_pageout_queue vm_pageout_queue_external __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT)));

 
 

-vm_object_t    vm_pageout_scan_wants_object = VM_OBJECT_NULL;
+int         vm_upl_wait_for_pages = 0;
+vm_object_t vm_pageout_scan_wants_object = VM_OBJECT_NULL;

 
 

-boolean_t (* volatile consider_buffer_cache_collect)(int) = NULL;
+boolean_t(*volatile consider_buffer_cache_collect)(int) = NULL;

 
 

-#if DEVELOPMENT || DEBUG
-unsigned long vm_cs_validated_resets = 0;
-#endif
+int     vm_debug_events = 0;

 
 

-int    vm_debug_events = 0;
+lck_grp_t vm_pageout_lck_grp;

 
 #if CONFIG_MEMORYSTATUS
 
 #if CONFIG_MEMORYSTATUS

-#if !CONFIG_JETSAM
-extern boolean_t memorystatus_idle_exit_from_VM(void);
-#endif

 extern boolean_t memorystatus_kill_on_VM_page_shortage(boolean_t async);
 extern boolean_t memorystatus_kill_on_VM_page_shortage(boolean_t async);

-extern void memorystatus_on_pageout_scan_end(void);

 
 uint32_t vm_pageout_memorystatus_fb_factor_nr = 5;
 uint32_t vm_pageout_memorystatus_fb_factor_dr = 2;
 
 uint32_t vm_pageout_memorystatus_fb_factor_nr = 5;
 uint32_t vm_pageout_memorystatus_fb_factor_dr = 2;

-#if DEVELOPMENT || DEBUG
-uint32_t vm_grab_anon_overrides = 0;
-uint32_t vm_grab_anon_nops = 0;
-#endif

 
 #endif
 
 
 #endif
 

-#if MACH_CLUSTER_STATS
-unsigned long vm_pageout_cluster_dirtied = 0;
-unsigned long vm_pageout_cluster_cleaned = 0;
-unsigned long vm_pageout_cluster_collisions = 0;
-unsigned long vm_pageout_cluster_clusters = 0;
-unsigned long vm_pageout_cluster_conversions = 0;
-unsigned long vm_pageout_target_collisions = 0;
-unsigned long vm_pageout_target_page_dirtied = 0;
-unsigned long vm_pageout_target_page_freed = 0;
-#define CLUSTER_STAT(clause)   clause
-#else  /* MACH_CLUSTER_STATS */
-#define CLUSTER_STAT(clause)
-#endif /* MACH_CLUSTER_STATS */

 
 
 
 

-#if DEVELOPMENT || DEBUG
-vmct_stats_t vmct_stats;
-#endif
-
-/* 
+/*

  *     Routine:        vm_pageout_object_terminate
  *     Purpose:
  *             Destroy the pageout_object, and perform all of the
  *             required cleanup actions.
  *     Routine:        vm_pageout_object_terminate
  *     Purpose:
  *             Destroy the pageout_object, and perform all of the
  *             required cleanup actions.

- * 
+ *

  *     In/Out conditions:
  *             The object must be locked, and will be returned locked.
  */
 void
 vm_pageout_object_terminate(
  *     In/Out conditions:
  *             The object must be locked, and will be returned locked.
  */
 void
 vm_pageout_object_terminate(

-       vm_object_t     object)
+       vm_object_t     object)

 {
 {

-       vm_object_t     shadow_object;
+       vm_object_t     shadow_object;

 
        /*
         * Deal with the deallocation (last reference) of a pageout object
 
        /*
         * Deal with the deallocation (last reference) of a pageout object


@@ -529,37 +367,39 @@ vm_pageout_object_terminate(
        vm_object_lock(shadow_object);
 
        while (!vm_page_queue_empty(&object->memq)) {
        vm_object_lock(shadow_object);
 
        while (!vm_page_queue_empty(&object->memq)) {

-               vm_page_t               p, m;
-               vm_object_offset_t      offset;
+               vm_page_t               p, m;
+               vm_object_offset_t      offset;

 
                p = (vm_page_t) vm_page_queue_first(&object->memq);
 
 
                p = (vm_page_t) vm_page_queue_first(&object->memq);
 

-               assert(p->private);
-               assert(p->free_when_done);
-               p->free_when_done = FALSE;
-               assert(!p->cleaning);
-               assert(!p->laundry);
+               assert(p->vmp_private);
+               assert(p->vmp_free_when_done);
+               p->vmp_free_when_done = FALSE;
+               assert(!p->vmp_cleaning);
+               assert(!p->vmp_laundry);

 
 

-               offset = p->offset;
+               offset = p->vmp_offset;

                VM_PAGE_FREE(p);
                p = VM_PAGE_NULL;
 
                m = vm_page_lookup(shadow_object,
                VM_PAGE_FREE(p);
                p = VM_PAGE_NULL;
 
                m = vm_page_lookup(shadow_object,

-                       offset + object->vo_shadow_offset);
+                   offset + object->vo_shadow_offset);

 
 

-               if(m == VM_PAGE_NULL)
+               if (m == VM_PAGE_NULL) {

                        continue;
                        continue;

+               }

 
 

-               assert((m->dirty) || (m->precious) ||
-                               (m->busy && m->cleaning));
+               assert((m->vmp_dirty) || (m->vmp_precious) ||
+                   (m->vmp_busy && m->vmp_cleaning));

 
                /*
                 * Handle the trusted pager throttle.
                 * Also decrement the burst throttle (if external).
                 */
                vm_page_lock_queues();
 
                /*
                 * Handle the trusted pager throttle.
                 * Also decrement the burst throttle (if external).
                 */
                vm_page_lock_queues();

-               if (m->vm_page_q_state == VM_PAGE_ON_PAGEOUT_Q)
+               if (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q) {

                        vm_pageout_throttle_up(m);
                        vm_pageout_throttle_up(m);

+               }

 
                /*
                 * Handle the "target" page(s). These pages are to be freed if
 
                /*
                 * Handle the "target" page(s). These pages are to be freed if


@@ -569,15 +409,12 @@ vm_pageout_object_terminate(
                 * pages may have been modified between the selection as an
                 * adjacent page and conversion to a target.
                 */
                 * pages may have been modified between the selection as an
                 * adjacent page and conversion to a target.
                 */

-               if (m->free_when_done) {
-                       assert(m->busy);
-                       assert(m->vm_page_q_state == VM_PAGE_IS_WIRED);
-                       assert(m->wire_count == 1);
-                       m->cleaning = FALSE;
-                       m->free_when_done = FALSE;
-#if MACH_CLUSTER_STATS
-                       if (m->wanted) vm_pageout_target_collisions++;
-#endif
+               if (m->vmp_free_when_done) {
+                       assert(m->vmp_busy);
+                       assert(m->vmp_q_state == VM_PAGE_IS_WIRED);
+                       assert(m->vmp_wire_count == 1);
+                       m->vmp_cleaning = FALSE;
+                       m->vmp_free_when_done = FALSE;

                        /*
                         * Revoke all access to the page. Since the object is
                         * locked, and the page is busy, this prevents the page
                        /*
                         * Revoke all access to the page. Since the object is
                         * locked, and the page is busy, this prevents the page


@@ -591,17 +428,15 @@ vm_pageout_object_terminate(
                        if (pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)) & VM_MEM_MODIFIED) {
                                SET_PAGE_DIRTY(m, FALSE);
                        } else {
                        if (pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)) & VM_MEM_MODIFIED) {
                                SET_PAGE_DIRTY(m, FALSE);
                        } else {

-                               m->dirty = FALSE;
+                               m->vmp_dirty = FALSE;

                        }
 
                        }
 

-                       if (m->dirty) {
-                               CLUSTER_STAT(vm_pageout_target_page_dirtied++;)
-                               vm_page_unwire(m, TRUE);        /* reactivates */
+                       if (m->vmp_dirty) {
+                               vm_page_unwire(m, TRUE);        /* reactivates */

                                VM_STAT_INCR(reactivations);
                                PAGE_WAKEUP_DONE(m);
                        } else {
                                VM_STAT_INCR(reactivations);
                                PAGE_WAKEUP_DONE(m);
                        } else {

-                               CLUSTER_STAT(vm_pageout_target_page_freed++;)
-                               vm_page_free(m);/* clears busy, etc. */
+                               vm_page_free(m);  /* clears busy, etc. */

                        }
                        vm_page_unlock_queues();
                        continue;
                        }
                        vm_page_unlock_queues();
                        continue;


@@ -612,19 +447,20 @@ vm_pageout_object_terminate(
                 * If prep_pin_count is nonzero, then someone is using the
                 * page, so make it active.
                 */
                 * If prep_pin_count is nonzero, then someone is using the
                 * page, so make it active.
                 */

-               if ((m->vm_page_q_state == VM_PAGE_NOT_ON_Q) && !m->private) {
-                       if (m->reference)
+               if ((m->vmp_q_state == VM_PAGE_NOT_ON_Q) && !m->vmp_private) {
+                       if (m->vmp_reference) {

                                vm_page_activate(m);
                                vm_page_activate(m);

-                       else
+                       } else {

                                vm_page_deactivate(m);
                                vm_page_deactivate(m);

+                       }

                }
                }

-               if (m->overwriting) {
+               if (m->vmp_overwriting) {

                        /*
                         * the (COPY_OUT_FROM == FALSE) request_page_list case
                         */
                        /*
                         * the (COPY_OUT_FROM == FALSE) request_page_list case
                         */

-                       if (m->busy) {
+                       if (m->vmp_busy) {

                                /*
                                /*

-                                * We do not re-set m->dirty !
+                                * We do not re-set m->vmp_dirty !

                                 * The page was busy so no extraneous activity
                                 * could have occurred. COPY_INTO is a read into the
                                 * new pages. CLEAN_IN_PLACE does actually write
                                 * The page was busy so no extraneous activity
                                 * could have occurred. COPY_INTO is a read into the
                                 * new pages. CLEAN_IN_PLACE does actually write


@@ -634,39 +470,22 @@ vm_pageout_object_terminate(
                                 */
                                pmap_clear_modify(VM_PAGE_GET_PHYS_PAGE(m));
 
                                 */
                                pmap_clear_modify(VM_PAGE_GET_PHYS_PAGE(m));
 

-                               m->busy = FALSE;
-                               m->absent = FALSE;
+                               m->vmp_busy = FALSE;
+                               m->vmp_absent = FALSE;

                        } else {
                                /*
                                 * alternate (COPY_OUT_FROM == FALSE) request_page_list case
                                 * Occurs when the original page was wired
                                 * at the time of the list request
                                 */
                        } else {
                                /*
                                 * alternate (COPY_OUT_FROM == FALSE) request_page_list case
                                 * Occurs when the original page was wired
                                 * at the time of the list request
                                 */

-                                assert(VM_PAGE_WIRED(m));
-                                vm_page_unwire(m, TRUE);       /* reactivates */
+                               assert(VM_PAGE_WIRED(m));
+                               vm_page_unwire(m, TRUE);        /* reactivates */

                        }
                        }

-                       m->overwriting = FALSE;
+                       m->vmp_overwriting = FALSE;

                } else {
                } else {

-                       /*
-                        * Set the dirty state according to whether or not the page was
-                        * modified during the pageout. Note that we purposefully do
-                        * NOT call pmap_clear_modify since the page is still mapped.
-                        * If the page were to be dirtied between the 2 calls, this
-                        * this fact would be lost. This code is only necessary to
-                        * maintain statistics, since the pmap module is always
-                        * consulted if m->dirty is false.
-                        */
-#if MACH_CLUSTER_STATS
-                       m->dirty = pmap_is_modified(VM_PAGE_GET_PHYS_PAGE(m));
-
-                       if (m->dirty)   vm_pageout_cluster_dirtied++;
-                       else            vm_pageout_cluster_cleaned++;
-                       if (m->wanted)  vm_pageout_cluster_collisions++;
-#else
-                       m->dirty = FALSE;
-#endif
+                       m->vmp_dirty = FALSE;

                }
                }

-               m->cleaning = FALSE;
+               m->vmp_cleaning = FALSE;

 
                /*
                 * Wakeup any thread waiting for the page to be un-cleaning.
 
                /*
                 * Wakeup any thread waiting for the page to be un-cleaning.


@@ -700,39 +519,39 @@ vm_pageout_object_terminate(
  */
 static void
 vm_pageclean_setup(
  */
 static void
 vm_pageclean_setup(

-       vm_page_t               m,
-       vm_page_t               new_m,
-       vm_object_t             new_object,
-       vm_object_offset_t      new_offset)
+       vm_page_t               m,
+       vm_page_t               new_m,
+       vm_object_t             new_object,
+       vm_object_offset_t      new_offset)

 {
 {

-       assert(!m->busy);
+       assert(!m->vmp_busy);

 #if 0
 #if 0

-       assert(!m->cleaning);
+       assert(!m->vmp_cleaning);

 #endif
 
        XPR(XPR_VM_PAGEOUT,
            "vm_pageclean_setup, obj 0x%X off 0x%X page 0x%X new 0x%X new_off 0x%X\n",
 #endif
 
        XPR(XPR_VM_PAGEOUT,
            "vm_pageclean_setup, obj 0x%X off 0x%X page 0x%X new 0x%X new_off 0x%X\n",

-               VM_PAGE_OBJECT(m), m->offset, m, 
-               new_m, new_offset);
+           VM_PAGE_OBJECT(m), m->vmp_offset, m,
+           new_m, new_offset);

 
        pmap_clear_modify(VM_PAGE_GET_PHYS_PAGE(m));
 
        /*
         * Mark original page as cleaning in place.
         */
 
        pmap_clear_modify(VM_PAGE_GET_PHYS_PAGE(m));
 
        /*
         * Mark original page as cleaning in place.
         */

-       m->cleaning = TRUE;
+       m->vmp_cleaning = TRUE;

        SET_PAGE_DIRTY(m, FALSE);
        SET_PAGE_DIRTY(m, FALSE);

-       m->precious = FALSE;
+       m->vmp_precious = FALSE;

 
        /*
         * Convert the fictitious page to a private shadow of
         * the real page.
         */
 
        /*
         * Convert the fictitious page to a private shadow of
         * the real page.
         */

-       assert(new_m->fictitious);
+       assert(new_m->vmp_fictitious);

        assert(VM_PAGE_GET_PHYS_PAGE(new_m) == vm_page_fictitious_addr);
        assert(VM_PAGE_GET_PHYS_PAGE(new_m) == vm_page_fictitious_addr);

-       new_m->fictitious = FALSE;
-       new_m->private = TRUE;
-       new_m->free_when_done = TRUE;
+       new_m->vmp_fictitious = FALSE;
+       new_m->vmp_private = TRUE;
+       new_m->vmp_free_when_done = TRUE;

        VM_PAGE_SET_PHYS_PAGE(new_m, VM_PAGE_GET_PHYS_PAGE(m));
 
        vm_page_lockspin_queues();
        VM_PAGE_SET_PHYS_PAGE(new_m, VM_PAGE_GET_PHYS_PAGE(m));
 
        vm_page_lockspin_queues();


@@ -740,8 +559,8 @@ vm_pageclean_setup(
        vm_page_unlock_queues();
 
        vm_page_insert_wired(new_m, new_object, new_offset, VM_KERN_MEMORY_NONE);
        vm_page_unlock_queues();
 
        vm_page_insert_wired(new_m, new_object, new_offset, VM_KERN_MEMORY_NONE);

-       assert(!new_m->wanted);
-       new_m->busy = FALSE;
+       assert(!new_m->vmp_wanted);
+       new_m->vmp_busy = FALSE;

 }
 
 /*
 }
 
 /*


@@ -762,38 +581,38 @@ vm_pageclean_setup(
  *     Implementation:
  *             Move this page to a completely new object.
  */
  *     Implementation:
  *             Move this page to a completely new object.
  */

-void   
+void

 vm_pageout_initialize_page(
 vm_pageout_initialize_page(

-       vm_page_t       m)
+       vm_page_t       m)

 {
 {

-       vm_object_t             object;
-       vm_object_offset_t      paging_offset;
-       memory_object_t         pager;
+       vm_object_t             object;
+       vm_object_offset_t      paging_offset;
+       memory_object_t         pager;

 
        XPR(XPR_VM_PAGEOUT,
 
        XPR(XPR_VM_PAGEOUT,

-               "vm_pageout_initialize_page, page 0x%X\n",
-               m, 0, 0, 0, 0);
+           "vm_pageout_initialize_page, page 0x%X\n",
+           m, 0, 0, 0, 0);

 
        assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
 
        object = VM_PAGE_OBJECT(m);
 
 
        assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
 
        object = VM_PAGE_OBJECT(m);
 

-       assert(m->busy);
+       assert(m->vmp_busy);

        assert(object->internal);
 
        /*
         *      Verify that we really want to clean this page
         */
        assert(object->internal);
 
        /*
         *      Verify that we really want to clean this page
         */

-       assert(!m->absent);
-       assert(!m->error);
-       assert(m->dirty);
+       assert(!m->vmp_absent);
+       assert(!m->vmp_error);
+       assert(m->vmp_dirty);

 
        /*
         *      Create a paging reference to let us play with the object.
         */
 
        /*
         *      Create a paging reference to let us play with the object.
         */

-       paging_offset = m->offset + object->paging_offset;
+       paging_offset = m->vmp_offset + object->paging_offset;

 
 

-       if (m->absent || m->error || m->restart || (!m->dirty && !m->precious)) {
+       if (m->vmp_absent || m->vmp_error || m->vmp_restart || (!m->vmp_dirty && !m->vmp_precious)) {

                panic("reservation without pageout?"); /* alan */
 
                VM_PAGE_FREE(m);
                panic("reservation without pageout?"); /* alan */
 
                VM_PAGE_FREE(m);


@@ -803,7 +622,7 @@ vm_pageout_initialize_page(
        }
 
        /*
        }
 
        /*

-        * If there's no pager, then we can't clean the page.  This should 
+        * If there's no pager, then we can't clean the page.  This should

         * never happen since this should be a copy object and therefore not
         * an external object, so the pager should always be there.
         */
         * never happen since this should be a copy object and therefore not
         * an external object, so the pager should always be there.
         */


@@ -843,15 +662,6 @@ vm_pageout_initialize_page(
        vm_object_paging_end(object);
 }
 
        vm_object_paging_end(object);
 }
 

-#if    MACH_CLUSTER_STATS
-#define MAXCLUSTERPAGES        16
-struct {
-       unsigned long pages_in_cluster;
-       unsigned long pages_at_higher_offsets;
-       unsigned long pages_at_lower_offsets;
-} cluster_stats[MAXCLUSTERPAGES];
-#endif /* MACH_CLUSTER_STATS */
-

 
 /*
  * vm_pageout_cluster:
 
 /*
  * vm_pageout_cluster:


@@ -867,24 +677,32 @@ struct {
  *
  * The page must not be on any pageout queue.
  */
  *
  * The page must not be on any pageout queue.
  */

+#if DEVELOPMENT || DEBUG
+vmct_stats_t vmct_stats;
+

 int32_t vmct_active = 0;
 int32_t vmct_active = 0;

+uint64_t vm_compressor_epoch_start = 0;
+uint64_t vm_compressor_epoch_stop = 0;
+

 typedef enum vmct_state_t {
        VMCT_IDLE,
        VMCT_AWAKENED,
        VMCT_ACTIVE,
 } vmct_state_t;
 vmct_state_t vmct_state[MAX_COMPRESSOR_THREAD_COUNT];
 typedef enum vmct_state_t {
        VMCT_IDLE,
        VMCT_AWAKENED,
        VMCT_ACTIVE,
 } vmct_state_t;
 vmct_state_t vmct_state[MAX_COMPRESSOR_THREAD_COUNT];

+#endif
+

 
 void
 vm_pageout_cluster(vm_page_t m)
 {
 
 void
 vm_pageout_cluster(vm_page_t m)
 {

-       vm_object_t     object = VM_PAGE_OBJECT(m);
-        struct         vm_pageout_queue *q;
+       vm_object_t     object = VM_PAGE_OBJECT(m);
+       struct          vm_pageout_queue *q;

 
 
        XPR(XPR_VM_PAGEOUT,
 
 
        XPR(XPR_VM_PAGEOUT,

-               "vm_pageout_cluster, object 0x%X offset 0x%X page 0x%X\n",
-               object, m->offset, m, 0, 0);
+           "vm_pageout_cluster, object 0x%X offset 0x%X page 0x%X\n",
+           object, m->vmp_offset, m, 0, 0);

 
        VM_PAGE_CHECK(m);
        LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
 
        VM_PAGE_CHECK(m);
        LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);


@@ -893,9 +711,9 @@ vm_pageout_cluster(vm_page_t m)
        /*
         * Only a certain kind of page is appreciated here.
         */
        /*
         * Only a certain kind of page is appreciated here.
         */

-       assert((m->dirty || m->precious) && (!VM_PAGE_WIRED(m)));
-       assert(!m->cleaning && !m->laundry);
-       assert(m->vm_page_q_state == VM_PAGE_NOT_ON_Q);
+       assert((m->vmp_dirty || m->vmp_precious) && (!VM_PAGE_WIRED(m)));
+       assert(!m->vmp_cleaning && !m->vmp_laundry);
+       assert(m->vmp_q_state == VM_PAGE_NOT_ON_Q);

 
        /*
         * protect the object from collapse or termination
 
        /*
         * protect the object from collapse or termination


@@ -905,20 +723,21 @@ vm_pageout_cluster(vm_page_t m)
        if (object->internal == TRUE) {
                assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
 
        if (object->internal == TRUE) {
                assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
 

-               m->busy = TRUE;
+               m->vmp_busy = TRUE;

 
 

-               q = &vm_pageout_queue_internal;
-       } else
-               q = &vm_pageout_queue_external;
+               q = &vm_pageout_queue_internal;
+       } else {
+               q = &vm_pageout_queue_external;
+       }

 
 

-       /* 
+       /*

         * pgo_laundry count is tied to the laundry bit
         */
         * pgo_laundry count is tied to the laundry bit
         */

-       m->laundry = TRUE;
+       m->vmp_laundry = TRUE;

        q->pgo_laundry++;
 
        q->pgo_laundry++;
 

-       m->vm_page_q_state = VM_PAGE_ON_PAGEOUT_Q;
-       vm_page_queue_enter(&q->pgo_pending, m, vm_page_t, pageq);
+       m->vmp_q_state = VM_PAGE_ON_PAGEOUT_Q;
+       vm_page_queue_enter(&q->pgo_pending, m, vmp_pageq);

 
        if (q->pgo_idle == TRUE) {
                q->pgo_idle = FALSE;
 
        if (q->pgo_idle == TRUE) {
                q->pgo_idle = FALSE;


@@ -928,10 +747,8 @@ vm_pageout_cluster(vm_page_t m)
 }
 
 
 }
 
 

-unsigned long vm_pageout_throttle_up_count = 0;
-

 /*
 /*

- * A page is back from laundry or we are stealing it back from 
+ * A page is back from laundry or we are stealing it back from

  * the laundering state.  See if there are some pages waiting to
  * go to laundry and if we can let some of them go now.
  *
  * the laundering state.  See if there are some pages waiting to
  * go to laundry and if we can let some of them go now.
  *


@@ -939,48 +756,48 @@ unsigned long vm_pageout_throttle_up_count = 0;
  */
 void
 vm_pageout_throttle_up(
  */
 void
 vm_pageout_throttle_up(

-       vm_page_t       m)
+       vm_page_t       m)

 {
 {

-       struct vm_pageout_queue *q;
-       vm_object_t     m_object;
-
-       m_object = VM_PAGE_OBJECT(m);
-
-       assert(m_object != VM_OBJECT_NULL);
-       assert(m_object != kernel_object);
+       struct vm_pageout_queue *q;
+       vm_object_t      m_object;

 
 

-       LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
-       vm_object_lock_assert_exclusive(m_object);
+       m_object = VM_PAGE_OBJECT(m);

 
 

-       vm_pageout_throttle_up_count++;
+       assert(m_object != VM_OBJECT_NULL);
+       assert(m_object != kernel_object);

 
 

-       if (m_object->internal == TRUE)
-               q = &vm_pageout_queue_internal;
-       else
-               q = &vm_pageout_queue_external;
+       LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
+       vm_object_lock_assert_exclusive(m_object);

 
 

-       if (m->vm_page_q_state == VM_PAGE_ON_PAGEOUT_Q) {
+       if (m_object->internal == TRUE) {
+               q = &vm_pageout_queue_internal;
+       } else {
+               q = &vm_pageout_queue_external;
+       }

 
 

-              vm_page_queue_remove(&q->pgo_pending, m, vm_page_t, pageq);
-              m->vm_page_q_state = VM_PAGE_NOT_ON_Q;
+       if (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q) {
+               vm_page_queue_remove(&q->pgo_pending, m, vmp_pageq);
+               m->vmp_q_state = VM_PAGE_NOT_ON_Q;

 
 

-              VM_PAGE_ZERO_PAGEQ_ENTRY(m);
+               VM_PAGE_ZERO_PAGEQ_ENTRY(m);

 
 

-              vm_object_activity_end(m_object);
-       }
-       if (m->laundry == TRUE) {
+               vm_object_activity_end(m_object);

 
 

-              m->laundry = FALSE;
-              q->pgo_laundry--;
+               VM_PAGEOUT_DEBUG(vm_page_steal_pageout_page, 1);
+       }
+       if (m->vmp_laundry == TRUE) {
+               m->vmp_laundry = FALSE;
+               q->pgo_laundry--;

 
 

-              if (q->pgo_throttled == TRUE) {
-                      q->pgo_throttled = FALSE;
-                       thread_wakeup((event_t) &q->pgo_laundry);
-               }
-              if (q->pgo_draining == TRUE && q->pgo_laundry == 0) {
-                      q->pgo_draining = FALSE;
-                      thread_wakeup((event_t) (&q->pgo_laundry+1));
-              }
+               if (q->pgo_throttled == TRUE) {
+                       q->pgo_throttled = FALSE;
+                       thread_wakeup((event_t) &q->pgo_laundry);
+               }
+               if (q->pgo_draining == TRUE && q->pgo_laundry == 0) {
+                       q->pgo_draining = FALSE;
+                       thread_wakeup((event_t) (&q->pgo_laundry + 1));
+               }
+               VM_PAGEOUT_DEBUG(vm_pageout_throttle_up_count, 1);

        }
 }
 
        }
 }
 


@@ -988,22 +805,22 @@ vm_pageout_throttle_up(
 static void
 vm_pageout_throttle_up_batch(
        struct vm_pageout_queue *q,
 static void
 vm_pageout_throttle_up_batch(
        struct vm_pageout_queue *q,

-       int             batch_cnt)
+       int             batch_cnt)

 {
 {

-       LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
+       LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);

 
 

-       vm_pageout_throttle_up_count += batch_cnt;
+       VM_PAGEOUT_DEBUG(vm_pageout_throttle_up_count, batch_cnt);

 
 

-       q->pgo_laundry -= batch_cnt;
+       q->pgo_laundry -= batch_cnt;

 
 

-       if (q->pgo_throttled == TRUE) {
-              q->pgo_throttled = FALSE;
-              thread_wakeup((event_t) &q->pgo_laundry);
-       }
-       if (q->pgo_draining == TRUE && q->pgo_laundry == 0) {
-              q->pgo_draining = FALSE;
-              thread_wakeup((event_t) (&q->pgo_laundry+1));
-       }
+       if (q->pgo_throttled == TRUE) {
+               q->pgo_throttled = FALSE;
+               thread_wakeup((event_t) &q->pgo_laundry);
+       }
+       if (q->pgo_draining == TRUE && q->pgo_laundry == 0) {
+               q->pgo_draining = FALSE;
+               thread_wakeup((event_t) (&q->pgo_laundry + 1));
+       }

 }
 
 
 }
 
 


@@ -1025,20 +842,64 @@ vm_pageout_throttle_up_batch(
  * also returns the number of pages the system still needs to reclaim at this
  * moment in time.
  */
  * also returns the number of pages the system still needs to reclaim at this
  * moment in time.
  */

-#define VM_PAGEOUT_STAT_SIZE   31
+#if DEVELOPMENT || DEBUG
+#define VM_PAGEOUT_STAT_SIZE    (30 * 8) + 1
+#else
+#define VM_PAGEOUT_STAT_SIZE    (1 * 8) + 1
+#endif

 struct vm_pageout_stat {
 struct vm_pageout_stat {

-       unsigned int considered;
-       unsigned int reclaimed_clean;
+       unsigned long vm_page_active_count;
+       unsigned long vm_page_speculative_count;
+       unsigned long vm_page_inactive_count;
+       unsigned long vm_page_anonymous_count;
+
+       unsigned long vm_page_free_count;
+       unsigned long vm_page_wire_count;
+       unsigned long vm_page_compressor_count;
+
+       unsigned long vm_page_pages_compressed;
+       unsigned long vm_page_pageable_internal_count;
+       unsigned long vm_page_pageable_external_count;
+       unsigned long vm_page_xpmapped_external_count;
+
+       unsigned int pages_grabbed;
+       unsigned int pages_freed;
+

        unsigned int pages_compressed;
        unsigned int pages_grabbed_by_compressor;
        unsigned int pages_compressed;
        unsigned int pages_grabbed_by_compressor;

+       unsigned int failed_compressions;
+
+       unsigned int pages_evicted;
+       unsigned int pages_purged;
+
+       unsigned int considered;
+       unsigned int considered_bq_internal;
+       unsigned int considered_bq_external;
+
+       unsigned int skipped_external;
+       unsigned int filecache_min_reactivations;
+
+       unsigned int freed_speculative;
+       unsigned int freed_cleaned;
+       unsigned int freed_internal;
+       unsigned int freed_external;
+

        unsigned int cleaned_dirty_external;
        unsigned int cleaned_dirty_external;

+       unsigned int cleaned_dirty_internal;
+
+       unsigned int inactive_referenced;
+       unsigned int inactive_nolock;
+       unsigned int reactivation_limit_exceeded;
+       unsigned int forced_inactive_reclaim;
+

        unsigned int throttled_internal_q;
        unsigned int throttled_external_q;
        unsigned int throttled_internal_q;
        unsigned int throttled_external_q;

-       unsigned int failed_compressions;
-} vm_pageout_stats[VM_PAGEOUT_STAT_SIZE] = {{0,0,0,0,0,0,0,0}, };
+
+       unsigned int phantom_ghosts_found;
+       unsigned int phantom_ghosts_added;
+} vm_pageout_stats[VM_PAGEOUT_STAT_SIZE] = {{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, };

 
 unsigned int vm_pageout_stat_now = 0;
 
 unsigned int vm_pageout_stat_now = 0;

-unsigned int vm_memory_pressure = 0;

 
 #define VM_PAGEOUT_STAT_BEFORE(i) \
        (((i) == 0) ? VM_PAGEOUT_STAT_SIZE - 1 : (i) - 1)
 
 #define VM_PAGEOUT_STAT_BEFORE(i) \
        (((i) == 0) ? VM_PAGEOUT_STAT_SIZE - 1 : (i) - 1)


@@ -1046,15 +907,14 @@ unsigned int vm_memory_pressure = 0;
        (((i) == VM_PAGEOUT_STAT_SIZE - 1) ? 0 : (i) + 1)
 
 #if VM_PAGE_BUCKETS_CHECK
        (((i) == VM_PAGEOUT_STAT_SIZE - 1) ? 0 : (i) + 1)
 
 #if VM_PAGE_BUCKETS_CHECK

-int vm_page_buckets_check_interval = 10; /* in seconds */
+int vm_page_buckets_check_interval = 80; /* in eighths of a second */

 #endif /* VM_PAGE_BUCKETS_CHECK */
 
 #endif /* VM_PAGE_BUCKETS_CHECK */
 

-/*
- * Called from compute_averages().
- */
+

 void
 void

-compute_memory_pressure(
-       __unused void *arg)
+record_memory_pressure(void);
+void
+record_memory_pressure(void)

 {
        unsigned int vm_pageout_next;
 
 {
        unsigned int vm_pageout_next;
 


@@ -1066,21 +926,18 @@ compute_memory_pressure(
        }
 #endif /* VM_PAGE_BUCKETS_CHECK */
 
        }
 #endif /* VM_PAGE_BUCKETS_CHECK */
 

-       vm_memory_pressure =
-               vm_pageout_stats[VM_PAGEOUT_STAT_BEFORE(vm_pageout_stat_now)].reclaimed_clean;
+       vm_pageout_state.vm_memory_pressure =
+           vm_pageout_stats[VM_PAGEOUT_STAT_BEFORE(vm_pageout_stat_now)].freed_speculative +
+           vm_pageout_stats[VM_PAGEOUT_STAT_BEFORE(vm_pageout_stat_now)].freed_cleaned +
+           vm_pageout_stats[VM_PAGEOUT_STAT_BEFORE(vm_pageout_stat_now)].freed_internal +
+           vm_pageout_stats[VM_PAGEOUT_STAT_BEFORE(vm_pageout_stat_now)].freed_external;

 
 

-       commpage_set_memory_pressure( vm_memory_pressure );
+       commpage_set_memory_pressure((unsigned int)vm_pageout_state.vm_memory_pressure );

 
        /* move "now" forward */
        vm_pageout_next = VM_PAGEOUT_STAT_AFTER(vm_pageout_stat_now);
 
        /* move "now" forward */
        vm_pageout_next = VM_PAGEOUT_STAT_AFTER(vm_pageout_stat_now);

-       vm_pageout_stats[vm_pageout_next].considered = 0;
-       vm_pageout_stats[vm_pageout_next].reclaimed_clean = 0;
-       vm_pageout_stats[vm_pageout_next].throttled_internal_q = 0;
-       vm_pageout_stats[vm_pageout_next].throttled_external_q = 0;
-       vm_pageout_stats[vm_pageout_next].cleaned_dirty_external = 0;
-       vm_pageout_stats[vm_pageout_next].pages_compressed = 0;
-       vm_pageout_stats[vm_pageout_next].pages_grabbed_by_compressor = 0;
-       vm_pageout_stats[vm_pageout_next].failed_compressions = 0;
+
+       bzero(&vm_pageout_stats[vm_pageout_next], sizeof(struct vm_pageout_stat));

 
        vm_pageout_stat_now = vm_pageout_next;
 }
 
        vm_pageout_stat_now = vm_pageout_next;
 }


@@ -1089,8 +946,8 @@ compute_memory_pressure(
 /*
  * IMPORTANT
  * mach_vm_ctl_page_free_wanted() is called indirectly, via
 /*
  * IMPORTANT
  * mach_vm_ctl_page_free_wanted() is called indirectly, via

- * mach_vm_pressure_monitor(), when taking a stackshot. Therefore, 
- * it must be safe in the restricted stackshot context. Locks and/or 
+ * mach_vm_pressure_monitor(), when taking a stackshot. Therefore,
+ * it must be safe in the restricted stackshot context. Locks and/or

  * blocking are not allowable.
  */
 unsigned int
  * blocking are not allowable.
  */
 unsigned int


@@ -1112,7 +969,7 @@ mach_vm_ctl_page_free_wanted(void)
 
 /*
  * IMPORTANT:
 
 /*
  * IMPORTANT:

- * mach_vm_pressure_monitor() is called when taking a stackshot, with 
+ * mach_vm_pressure_monitor() is called when taking a stackshot, with

  * wait_for_pressure FALSE, so that code path must remain safe in the
  * restricted stackshot context. No blocking or locks are allowable.
  * on that code path.
  * wait_for_pressure FALSE, so that code path must remain safe in the
  * restricted stackshot context. No blocking or locks are allowable.
  * on that code path.


@@ -1120,15 +977,17 @@ mach_vm_ctl_page_free_wanted(void)
 
 kern_return_t
 mach_vm_pressure_monitor(
 
 kern_return_t
 mach_vm_pressure_monitor(

-       boolean_t       wait_for_pressure,
-       unsigned int    nsecs_monitored,
-       unsigned int    *pages_reclaimed_p,
-       unsigned int    *pages_wanted_p)
+       boolean_t       wait_for_pressure,
+       unsigned int    nsecs_monitored,
+       unsigned int    *pages_reclaimed_p,
+       unsigned int    *pages_wanted_p)

 {
 {

-       wait_result_t   wr;
-       unsigned int    vm_pageout_then, vm_pageout_now;
-       unsigned int    pages_reclaimed;
+       wait_result_t   wr;
+       unsigned int    vm_pageout_then, vm_pageout_now;
+       unsigned int    pages_reclaimed;
+       unsigned int    units_of_monitor;

 
 

+       units_of_monitor = 8 * nsecs_monitored;

        /*
         * We don't take the vm_page_queue_lock here because we don't want
         * vm_pressure_monitor() to get in the way of the vm_pageout_scan()
        /*
         * We don't take the vm_page_queue_lock here because we don't want
         * vm_pressure_monitor() to get in the way of the vm_pageout_scan()


@@ -1140,7 +999,7 @@ mach_vm_pressure_monitor(
                /* wait until there's memory pressure */
                while (vm_page_free_count >= vm_page_free_target) {
                        wr = assert_wait((event_t) &vm_page_free_wanted,
                /* wait until there's memory pressure */
                while (vm_page_free_count >= vm_page_free_target) {
                        wr = assert_wait((event_t) &vm_page_free_wanted,

-                                        THREAD_INTERRUPTIBLE);
+                           THREAD_INTERRUPTIBLE);

                        if (wr == THREAD_WAITING) {
                                wr = thread_block(THREAD_CONTINUE_NULL);
                        }
                        if (wr == THREAD_WAITING) {
                                wr = thread_block(THREAD_CONTINUE_NULL);
                        }


@@ -1172,12 +1031,15 @@ mach_vm_pressure_monitor(
        vm_pageout_now = vm_pageout_stat_now;
        pages_reclaimed = 0;
        for (vm_pageout_then =
        vm_pageout_now = vm_pageout_stat_now;
        pages_reclaimed = 0;
        for (vm_pageout_then =

-                    VM_PAGEOUT_STAT_BEFORE(vm_pageout_now);
-            vm_pageout_then != vm_pageout_now &&
-                    nsecs_monitored-- != 0;
-            vm_pageout_then =
-                    VM_PAGEOUT_STAT_BEFORE(vm_pageout_then)) {
-               pages_reclaimed += vm_pageout_stats[vm_pageout_then].reclaimed_clean;
+           VM_PAGEOUT_STAT_BEFORE(vm_pageout_now);
+           vm_pageout_then != vm_pageout_now &&
+           units_of_monitor-- != 0;
+           vm_pageout_then =
+           VM_PAGEOUT_STAT_BEFORE(vm_pageout_then)) {
+               pages_reclaimed += vm_pageout_stats[vm_pageout_then].freed_speculative;
+               pages_reclaimed += vm_pageout_stats[vm_pageout_then].freed_cleaned;
+               pages_reclaimed += vm_pageout_stats[vm_pageout_then].freed_internal;
+               pages_reclaimed += vm_pageout_stats[vm_pageout_then].freed_external;

        }
        *pages_reclaimed_p = pages_reclaimed;
 
        }
        *pages_reclaimed_p = pages_reclaimed;
 


@@ -1195,7 +1057,7 @@ vm_pageout_disconnect_all_pages_in_queue(vm_page_queue_head_t *, int);
  * condition variable used to make sure there is
  * only a single sweep going on at a time
  */
  * condition variable used to make sure there is
  * only a single sweep going on at a time
  */

-boolean_t      vm_pageout_disconnect_all_pages_active = FALSE;
+boolean_t       vm_pageout_disconnect_all_pages_active = FALSE;

 
 
 void
 
 
 void


@@ -1221,23 +1083,22 @@ vm_pageout_disconnect_all_pages()
 void
 vm_pageout_disconnect_all_pages_in_queue(vm_page_queue_head_t *q, int qcount)
 {
 void
 vm_pageout_disconnect_all_pages_in_queue(vm_page_queue_head_t *q, int qcount)
 {

-       vm_page_t       m;
-       vm_object_t     t_object = NULL;
-       vm_object_t     l_object = NULL;
-       vm_object_t     m_object = NULL;
-       int             delayed_unlock = 0;
-       int             try_failed_count = 0;
-       int             disconnected_count = 0;
-       int             paused_count = 0;
-       int             object_locked_count = 0;
+       vm_page_t       m;
+       vm_object_t     t_object = NULL;
+       vm_object_t     l_object = NULL;
+       vm_object_t     m_object = NULL;
+       int             delayed_unlock = 0;
+       int             try_failed_count = 0;
+       int             disconnected_count = 0;
+       int             paused_count = 0;
+       int             object_locked_count = 0;

 
        KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_DISCONNECT_ALL_PAGE_MAPPINGS)) | DBG_FUNC_START,
 
        KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_DISCONNECT_ALL_PAGE_MAPPINGS)) | DBG_FUNC_START,

-                                 q, qcount, 0, 0, 0);
+           q, qcount, 0, 0, 0);

 
        vm_page_lock_queues();
 
        while (qcount && !vm_page_queue_empty(q)) {
 
        vm_page_lock_queues();
 
        while (qcount && !vm_page_queue_empty(q)) {

-

                LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
 
                m = (vm_page_t) vm_page_queue_first(q);
                LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
 
                m = (vm_page_t) vm_page_queue_first(q);


@@ -1249,27 +1110,27 @@ vm_pageout_disconnect_all_pages_in_queue(vm_page_queue_head_t *q, int qcount)
                 * already got the lock
                 */
                if (m_object != l_object) {
                 * already got the lock
                 */
                if (m_object != l_object) {

-                       /*
-                        * the object associated with candidate page is 
+                       /*
+                        * the object associated with candidate page is

                         * different from the one we were just working
                         * with... dump the lock if we still own it
                         */
                         * different from the one we were just working
                         * with... dump the lock if we still own it
                         */

-                       if (l_object != NULL) {
-                               vm_object_unlock(l_object);
+                       if (l_object != NULL) {
+                               vm_object_unlock(l_object);

                                l_object = NULL;
                        }
                                l_object = NULL;
                        }

-                       if (m_object != t_object)
+                       if (m_object != t_object) {

                                try_failed_count = 0;
                                try_failed_count = 0;

+                       }

 
                        /*
                         * Try to lock object; since we've alread got the
                         * page queues lock, we can only 'try' for this one.
                         * if the 'try' fails, we need to do a mutex_pause
                         * to allow the owner of the object lock a chance to
 
                        /*
                         * Try to lock object; since we've alread got the
                         * page queues lock, we can only 'try' for this one.
                         * if the 'try' fails, we need to do a mutex_pause
                         * to allow the owner of the object lock a chance to

-                        * run... 
+                        * run...

                         */
                         */

-                       if ( !vm_object_lock_try_scan(m_object)) {
-
+                       if (!vm_object_lock_try_scan(m_object)) {

                                if (try_failed_count > 20) {
                                        goto reenter_pg_on_q;
                                }
                                if (try_failed_count > 20) {
                                        goto reenter_pg_on_q;
                                }


@@ -1287,27 +1148,25 @@ vm_pageout_disconnect_all_pages_in_queue(vm_page_queue_head_t *q, int qcount)
 
                        l_object = m_object;
                }
 
                        l_object = m_object;
                }

-               if ( !m_object->alive || m->cleaning || m->laundry || m->busy || m->absent || m->error || m->free_when_done) {
+               if (!m_object->alive || m->vmp_cleaning || m->vmp_laundry || m->vmp_busy || m->vmp_absent || m->vmp_error || m->vmp_free_when_done) {

                        /*
                         * put it back on the head of its queue
                         */
                        goto reenter_pg_on_q;
                }
                        /*
                         * put it back on the head of its queue
                         */
                        goto reenter_pg_on_q;
                }

-               if (m->pmapped == TRUE) {
-
+               if (m->vmp_pmapped == TRUE) {

                        pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
 
                        disconnected_count++;
                }
 reenter_pg_on_q:
                        pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
 
                        disconnected_count++;
                }
 reenter_pg_on_q:

-               vm_page_queue_remove(q, m, vm_page_t, pageq);
-               vm_page_queue_enter(q, m, vm_page_t, pageq);
+               vm_page_queue_remove(q, m, vmp_pageq);
+               vm_page_queue_enter(q, m, vmp_pageq);

 
                qcount--;
                try_failed_count = 0;
 
                if (delayed_unlock++ > 128) {
 
                qcount--;
                try_failed_count = 0;
 
                if (delayed_unlock++ > 128) {

-

                        if (l_object != NULL) {
                                vm_object_unlock(l_object);
                                l_object = NULL;
                        if (l_object != NULL) {
                                vm_object_unlock(l_object);
                                l_object = NULL;


@@ -1323,7 +1182,7 @@ reenter_pg_on_q:
        vm_page_unlock_queues();
 
        KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_DISCONNECT_ALL_PAGE_MAPPINGS)) | DBG_FUNC_END,
        vm_page_unlock_queues();
 
        KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, (MACHDBG_CODE(DBG_MACH_WORKINGSET, VM_DISCONNECT_ALL_PAGE_MAPPINGS)) | DBG_FUNC_END,

-                                 q, disconnected_count, object_locked_count, paused_count, 0);
+           q, disconnected_count, object_locked_count, paused_count, 0);

 }
 
 #endif
 }
 
 #endif


@@ -1336,14 +1195,13 @@ vm_pageout_page_queue(vm_page_queue_head_t *, int);
  * condition variable used to make sure there is
  * only a single sweep going on at a time
  */
  * condition variable used to make sure there is
  * only a single sweep going on at a time
  */

-boolean_t      vm_pageout_anonymous_pages_active = FALSE;
+boolean_t       vm_pageout_anonymous_pages_active = FALSE;

 
 
 void
 vm_pageout_anonymous_pages()
 {
        if (VM_CONFIG_COMPRESSOR_IS_PRESENT) {
 
 
 void
 vm_pageout_anonymous_pages()
 {
        if (VM_CONFIG_COMPRESSOR_IS_PRESENT) {

-

                vm_page_lock_queues();
 
                if (vm_pageout_anonymous_pages_active == TRUE) {
                vm_page_lock_queues();
 
                if (vm_pageout_anonymous_pages_active == TRUE) {


@@ -1357,8 +1215,9 @@ vm_pageout_anonymous_pages()
                vm_pageout_page_queue(&vm_page_queue_anonymous, vm_page_anonymous_count);
                vm_pageout_page_queue(&vm_page_queue_active, vm_page_active_count);
 
                vm_pageout_page_queue(&vm_page_queue_anonymous, vm_page_anonymous_count);
                vm_pageout_page_queue(&vm_page_queue_active, vm_page_active_count);
 

-               if (VM_CONFIG_SWAP_IS_PRESENT)
+               if (VM_CONFIG_SWAP_IS_PRESENT) {

                        vm_consider_swapping();
                        vm_consider_swapping();

+               }

 
                vm_page_lock_queues();
                vm_pageout_anonymous_pages_active = FALSE;
 
                vm_page_lock_queues();
                vm_pageout_anonymous_pages_active = FALSE;


@@ -1370,39 +1229,37 @@ vm_pageout_anonymous_pages()
 void
 vm_pageout_page_queue(vm_page_queue_head_t *q, int qcount)
 {
 void
 vm_pageout_page_queue(vm_page_queue_head_t *q, int qcount)
 {

-       vm_page_t       m;
-       vm_object_t     t_object = NULL;
-       vm_object_t     l_object = NULL;
-       vm_object_t     m_object = NULL;
-       int             delayed_unlock = 0;
-       int             try_failed_count = 0;
-       int             refmod_state;
-       int             pmap_options;
-       struct          vm_pageout_queue *iq;
-       ppnum_t         phys_page;
+       vm_page_t       m;
+       vm_object_t     t_object = NULL;
+       vm_object_t     l_object = NULL;
+       vm_object_t     m_object = NULL;
+       int             delayed_unlock = 0;
+       int             try_failed_count = 0;
+       int             refmod_state;
+       int             pmap_options;
+       struct          vm_pageout_queue *iq;
+       ppnum_t         phys_page;

 
 
        iq = &vm_pageout_queue_internal;
 
 
        iq = &vm_pageout_queue_internal;

-       
+

        vm_page_lock_queues();
 
        while (qcount && !vm_page_queue_empty(q)) {
        vm_page_lock_queues();
 
        while (qcount && !vm_page_queue_empty(q)) {

-

                LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
 
                if (VM_PAGE_Q_THROTTLED(iq)) {
                LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
 
                if (VM_PAGE_Q_THROTTLED(iq)) {

-
-                       if (l_object != NULL) {
-                               vm_object_unlock(l_object);
+                       if (l_object != NULL) {
+                               vm_object_unlock(l_object);

                                l_object = NULL;
                        }
                        iq->pgo_draining = TRUE;
                                l_object = NULL;
                        }
                        iq->pgo_draining = TRUE;

-                                       
+

                        assert_wait((event_t) (&iq->pgo_laundry + 1), THREAD_INTERRUPTIBLE);
                        vm_page_unlock_queues();
                        assert_wait((event_t) (&iq->pgo_laundry + 1), THREAD_INTERRUPTIBLE);
                        vm_page_unlock_queues();

-                                       
+

                        thread_block(THREAD_CONTINUE_NULL);
                        thread_block(THREAD_CONTINUE_NULL);

-                       
+

                        vm_page_lock_queues();
                        delayed_unlock = 0;
                        continue;
                        vm_page_lock_queues();
                        delayed_unlock = 0;
                        continue;


@@ -1416,30 +1273,31 @@ vm_pageout_page_queue(vm_page_queue_head_t *q, int qcount)
                 * already got the lock
                 */
                if (m_object != l_object) {
                 * already got the lock
                 */
                if (m_object != l_object) {

-                       if ( !m_object->internal) 
+                       if (!m_object->internal) {

                                goto reenter_pg_on_q;
                                goto reenter_pg_on_q;

+                       }

 
 

-                       /*
-                        * the object associated with candidate page is 
+                       /*
+                        * the object associated with candidate page is

                         * different from the one we were just working
                         * with... dump the lock if we still own it
                         */
                         * different from the one we were just working
                         * with... dump the lock if we still own it
                         */

-                       if (l_object != NULL) {
-                               vm_object_unlock(l_object);
+                       if (l_object != NULL) {
+                               vm_object_unlock(l_object);

                                l_object = NULL;
                        }
                                l_object = NULL;
                        }

-                       if (m_object != t_object)
+                       if (m_object != t_object) {

                                try_failed_count = 0;
                                try_failed_count = 0;

+                       }

 
                        /*
                         * Try to lock object; since we've alread got the
                         * page queues lock, we can only 'try' for this one.
                         * if the 'try' fails, we need to do a mutex_pause
                         * to allow the owner of the object lock a chance to
 
                        /*
                         * Try to lock object; since we've alread got the
                         * page queues lock, we can only 'try' for this one.
                         * if the 'try' fails, we need to do a mutex_pause
                         * to allow the owner of the object lock a chance to

-                        * run... 
+                        * run...

                         */
                         */

-                       if ( !vm_object_lock_try_scan(m_object)) {
-
+                       if (!vm_object_lock_try_scan(m_object)) {

                                if (try_failed_count > 20) {
                                        goto reenter_pg_on_q;
                                }
                                if (try_failed_count > 20) {
                                        goto reenter_pg_on_q;
                                }


@@ -1453,7 +1311,7 @@ vm_pageout_page_queue(vm_page_queue_head_t *q, int qcount)
                        }
                        l_object = m_object;
                }
                        }
                        l_object = m_object;
                }

-               if ( !m_object->alive || m->cleaning || m->laundry || m->busy || m->absent || m->error || m->free_when_done) {
+               if (!m_object->alive || m->vmp_cleaning || m->vmp_laundry || m->vmp_busy || m->vmp_absent || m->vmp_error || m->vmp_free_when_done) {

                        /*
                         * page is not to be cleaned
                         * put it back on the head of its queue
                        /*
                         * page is not to be cleaned
                         * put it back on the head of its queue


@@ -1462,22 +1320,23 @@ vm_pageout_page_queue(vm_page_queue_head_t *q, int qcount)
                }
                phys_page = VM_PAGE_GET_PHYS_PAGE(m);
 
                }
                phys_page = VM_PAGE_GET_PHYS_PAGE(m);
 

-               if (m->reference == FALSE && m->pmapped == TRUE) {
+               if (m->vmp_reference == FALSE && m->vmp_pmapped == TRUE) {

                        refmod_state = pmap_get_refmod(phys_page);
                        refmod_state = pmap_get_refmod(phys_page);

-                 
-                       if (refmod_state & VM_MEM_REFERENCED)
-                               m->reference = TRUE;
+
+                       if (refmod_state & VM_MEM_REFERENCED) {
+                               m->vmp_reference = TRUE;
+                       }

                        if (refmod_state & VM_MEM_MODIFIED) {
                        if (refmod_state & VM_MEM_MODIFIED) {

-                               SET_PAGE_DIRTY(m, FALSE);
+                               SET_PAGE_DIRTY(m, FALSE);

                        }
                }
                        }
                }

-               if (m->reference == TRUE) {
-                       m->reference = FALSE;
+               if (m->vmp_reference == TRUE) {
+                       m->vmp_reference = FALSE;

                        pmap_clear_refmod_options(phys_page, VM_MEM_REFERENCED, PMAP_OPTIONS_NOFLUSH, (void *)NULL);
                        goto reenter_pg_on_q;
                }
                        pmap_clear_refmod_options(phys_page, VM_MEM_REFERENCED, PMAP_OPTIONS_NOFLUSH, (void *)NULL);
                        goto reenter_pg_on_q;
                }

-               if (m->pmapped == TRUE) {
-                       if (m->dirty || m->precious) {
+               if (m->vmp_pmapped == TRUE) {
+                       if (m->vmp_dirty || m->vmp_precious) {

                                pmap_options = PMAP_OPTIONS_COMPRESSOR;
                        } else {
                                pmap_options = PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED;
                                pmap_options = PMAP_OPTIONS_COMPRESSOR;
                        } else {
                                pmap_options = PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED;


@@ -1487,7 +1346,8 @@ vm_pageout_page_queue(vm_page_queue_head_t *q, int qcount)
                                SET_PAGE_DIRTY(m, FALSE);
                        }
                }
                                SET_PAGE_DIRTY(m, FALSE);
                        }
                }

-               if ( !m->dirty && !m->precious) {
+
+               if (!m->vmp_dirty && !m->vmp_precious) {

                        vm_page_unlock_queues();
                        VM_PAGE_FREE(m);
                        vm_page_lock_queues();
                        vm_page_unlock_queues();
                        VM_PAGE_FREE(m);
                        vm_page_lock_queues();


@@ -1495,22 +1355,22 @@ vm_pageout_page_queue(vm_page_queue_head_t *q, int qcount)
 
                        goto next_pg;
                }
 
                        goto next_pg;
                }

-               if (!m_object->pager_initialized || m_object->pager == MEMORY_OBJECT_NULL)  {
-                       
+               if (!m_object->pager_initialized || m_object->pager == MEMORY_OBJECT_NULL) {

                        if (!m_object->pager_initialized) {
                        if (!m_object->pager_initialized) {

-

                                vm_page_unlock_queues();
 
                                vm_object_collapse(m_object, (vm_object_offset_t) 0, TRUE);
 
                                vm_page_unlock_queues();
 
                                vm_object_collapse(m_object, (vm_object_offset_t) 0, TRUE);
 

-                               if (!m_object->pager_initialized)
+                               if (!m_object->pager_initialized) {

                                        vm_object_compressor_pager_create(m_object);
                                        vm_object_compressor_pager_create(m_object);

+                               }

 
                                vm_page_lock_queues();
                                delayed_unlock = 0;
                        }
 
                                vm_page_lock_queues();
                                delayed_unlock = 0;
                        }

-                       if (!m_object->pager_initialized || m_object->pager == MEMORY_OBJECT_NULL)
+                       if (!m_object->pager_initialized || m_object->pager == MEMORY_OBJECT_NULL) {

                                goto reenter_pg_on_q;
                                goto reenter_pg_on_q;

+                       }

                        /*
                         * vm_object_compressor_pager_create will drop the object lock
                         * which means 'm' may no longer be valid to use
                        /*
                         * vm_object_compressor_pager_create will drop the object lock
                         * which means 'm' may no longer be valid to use


@@ -1531,14 +1391,13 @@ vm_pageout_page_queue(vm_page_queue_head_t *q, int qcount)
                goto next_pg;
 
 reenter_pg_on_q:
                goto next_pg;
 
 reenter_pg_on_q:

-               vm_page_queue_remove(q, m, vm_page_t, pageq);
-               vm_page_queue_enter(q, m, vm_page_t, pageq);
+               vm_page_queue_remove(q, m, vmp_pageq);
+               vm_page_queue_enter(q, m, vmp_pageq);

 next_pg:
                qcount--;
                try_failed_count = 0;
 
                if (delayed_unlock++ > 128) {
 next_pg:
                qcount--;
                try_failed_count = 0;
 
                if (delayed_unlock++ > 128) {

-

                        if (l_object != NULL) {
                                vm_object_unlock(l_object);
                                l_object = NULL;
                        if (l_object != NULL) {
                                vm_object_unlock(l_object);
                                l_object = NULL;


@@ -1561,81 +1420,74 @@ next_pg:
  */
 extern void vm_pageout_io_throttle(void);
 
  */
 extern void vm_pageout_io_throttle(void);
 

-#define VM_PAGEOUT_SCAN_HANDLE_REUSABLE_PAGE(m, obj)                   \
-        MACRO_BEGIN                                                     \
-        /*                                                              \
-         * If a "reusable" page somehow made it back into               \
-         * the active queue, it's been re-used and is not               \
-         * quite re-usable.                                             \
-         * If the VM object was "all_reusable", consider it             \
-         * as "all re-used" instead of converting it to                 \
-         * "partially re-used", which could be expensive.               \
-         */                                                             \
-       assert(VM_PAGE_OBJECT((m)) == (obj));                           \
-        if ((m)->reusable ||                                            \
-            (obj)->all_reusable) {                                     \
-               vm_object_reuse_pages((obj),                            \
-                                      (m)->offset,                      \
-                                      (m)->offset + PAGE_SIZE_64,       \
-                                      FALSE);                           \
-        }                                                               \
-        MACRO_END
-
-
-#define VM_PAGEOUT_DELAYED_UNLOCK_LIMIT        64
-#define VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX    1024
-
-#define        FCS_IDLE                0
-#define FCS_DELAYED            1
-#define FCS_DEADLOCK_DETECTED  2
+#define VM_PAGEOUT_SCAN_HANDLE_REUSABLE_PAGE(m, obj)                    \
+       MACRO_BEGIN                                                     \
+       /* \
+        * If a "reusable" page somehow made it back into \
+        * the active queue, it's been re-used and is not \
+        * quite re-usable. \
+        * If the VM object was "all_reusable", consider it \
+        * as "all re-used" instead of converting it to \
+        * "partially re-used", which could be expensive. \
+        */                                                             \
+       assert(VM_PAGE_OBJECT((m)) == (obj));                           \
+       if ((m)->vmp_reusable ||                                        \
+           (obj)->all_reusable) {                                      \
+               vm_object_reuse_pages((obj),                            \
+                                     (m)->vmp_offset,                  \
+                                     (m)->vmp_offset + PAGE_SIZE_64,   \
+                                     FALSE);                           \
+       }                                                               \
+       MACRO_END
+
+
+#define VM_PAGEOUT_DELAYED_UNLOCK_LIMIT         64
+#define VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX     1024
+
+#define FCS_IDLE                0
+#define FCS_DELAYED             1
+#define FCS_DEADLOCK_DETECTED   2

 
 struct flow_control {
 
 struct flow_control {

-        int            state;
-        mach_timespec_t        ts;
+       int             state;
+       mach_timespec_t ts;

 };
 
 };
 

+

 #if CONFIG_BACKGROUND_QUEUE
 #if CONFIG_BACKGROUND_QUEUE

-uint64_t vm_pageout_skipped_bq_internal = 0;
-uint64_t vm_pageout_considered_bq_internal = 0;
-uint64_t vm_pageout_considered_bq_external = 0;

 uint64_t vm_pageout_rejected_bq_internal = 0;
 uint64_t vm_pageout_rejected_bq_external = 0;
 uint64_t vm_pageout_rejected_bq_internal = 0;
 uint64_t vm_pageout_rejected_bq_external = 0;

+uint64_t vm_pageout_skipped_bq_internal = 0;

 #endif
 
 #endif
 

-uint32_t vm_pageout_no_victim = 0;
-uint32_t vm_pageout_considered_page = 0;
-uint32_t vm_page_filecache_min = 0;
-
-#define ANONS_GRABBED_LIMIT    2
-
-#if CONFIG_SECLUDED_MEMORY
-extern vm_page_t vm_page_grab_secluded(void);
-uint64_t vm_pageout_secluded_burst_count = 0;
-#endif /* CONFIG_SECLUDED_MEMORY */
+#define ANONS_GRABBED_LIMIT     2

 
 
 
 

+#if 0

 static void vm_pageout_delayed_unlock(int *, int *, vm_page_t *);
 static void vm_pageout_delayed_unlock(int *, int *, vm_page_t *);

+#endif

 static void vm_pageout_prepare_to_block(vm_object_t *, int *, vm_page_t *, int *, int);
 
 static void vm_pageout_prepare_to_block(vm_object_t *, int *, vm_page_t *, int *, int);
 

-#define        VM_PAGEOUT_PB_NO_ACTION                         0
-#define        VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER 1
-#define        VM_PAGEOUT_PB_THREAD_YIELD                      2
+#define VM_PAGEOUT_PB_NO_ACTION                         0
+#define VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER 1
+#define VM_PAGEOUT_PB_THREAD_YIELD                      2

 
 
 
 

+#if 0

 static void
 vm_pageout_delayed_unlock(int *delayed_unlock, int *local_freed, vm_page_t *local_freeq)
 {
        if (*local_freeq) {
                vm_page_unlock_queues();
 
 static void
 vm_pageout_delayed_unlock(int *delayed_unlock, int *local_freed, vm_page_t *local_freeq)
 {
        if (*local_freeq) {
                vm_page_unlock_queues();
 

-               VM_DEBUG_EVENT(
+               VM_DEBUG_CONSTANT_EVENT(

                        vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_START,
                        vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_START,

-                       vm_page_free_count, *local_freed, 0, 1);
+                       vm_page_free_count, 0, 0, 1);

 
                vm_page_free_list(*local_freeq, TRUE);
 
 
                vm_page_free_list(*local_freeq, TRUE);
 

-               VM_DEBUG_EVENT(vm_pageout_freelist,VM_PAGEOUT_FREELIST, DBG_FUNC_END,
-                              vm_page_free_count, 0, 0, 1);
+               VM_DEBUG_CONSTANT_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_END,
+                   vm_page_free_count, *local_freed, 0, 1);

 
                *local_freeq = NULL;
                *local_freed = 0;
 
                *local_freeq = NULL;
                *local_freed = 0;


@@ -1646,11 +1498,12 @@ vm_pageout_delayed_unlock(int *delayed_unlock, int *local_freed, vm_page_t *loca
        }
        *delayed_unlock = 1;
 }
        }
        *delayed_unlock = 1;
 }

+#endif

 
 
 static void
 vm_pageout_prepare_to_block(vm_object_t *object, int *delayed_unlock,
 
 
 static void
 vm_pageout_prepare_to_block(vm_object_t *object, int *delayed_unlock,

-                           vm_page_t *local_freeq, int *local_freed, int action)
+    vm_page_t *local_freeq, int *local_freed, int action)

 {
        vm_page_unlock_queues();
 
 {
        vm_page_unlock_queues();
 


@@ -1658,17 +1511,8 @@ vm_pageout_prepare_to_block(vm_object_t *object, int *delayed_unlock,
                vm_object_unlock(*object);
                *object = NULL;
        }
                vm_object_unlock(*object);
                *object = NULL;
        }

-       vm_pageout_scan_wants_object = VM_OBJECT_NULL;
-

        if (*local_freeq) {
        if (*local_freeq) {

-
-               VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_START,
-                              vm_page_free_count, *local_freed, 0, 2);
-

                vm_page_free_list(*local_freeq, TRUE);
                vm_page_free_list(*local_freeq, TRUE);

-                                       
-               VM_DEBUG_EVENT(vm_pageout_freelist, VM_PAGEOUT_FREELIST, DBG_FUNC_END,
-                              vm_page_free_count, 0, 0, 2);

 
                *local_freeq = NULL;
                *local_freed = 0;
 
                *local_freeq = NULL;
                *local_freed = 0;


@@ -1676,7 +1520,6 @@ vm_pageout_prepare_to_block(vm_object_t *object, int *delayed_unlock,
        *delayed_unlock = 1;
 
        switch (action) {
        *delayed_unlock = 1;
 
        switch (action) {

-

        case VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER:
                vm_consider_waking_compactor_swapper();
                break;
        case VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER:
                vm_consider_waking_compactor_swapper();
                break;


@@ -1691,130 +1534,269 @@ vm_pageout_prepare_to_block(vm_object_t *object, int *delayed_unlock,
 }
 
 
 }
 
 

-int    last_vm_pageout_freed_from_inactive_clean = 0;
-int    last_vm_pageout_freed_from_cleaned = 0;
-int    last_vm_pageout_freed_from_speculative = 0;
-int    last_vm_pageout_freed_after_compression = 0;
-int    last_vm_pageout_enqueued_cleaned_from_inactive_dirty = 0;
-int    last_vm_pageout_inactive_force_reclaim = 0;
-int    last_vm_pageout_scan_inactive_throttled_external = 0;
-int    last_vm_pageout_scan_inactive_throttled_internal = 0;
-int    last_vm_pageout_reactivation_limit_exceeded = 0;
-int    last_vm_pageout_considered_page = 0;
-int    last_vm_compressor_pages_grabbed = 0;
-int    last_vm_compressor_failed = 0;
-int     last_vm_pageout_skipped_external = 0;
+static struct vm_pageout_vminfo last;
+
+uint64_t last_vm_page_pages_grabbed = 0;

 
 

+extern  uint32_t c_segment_pages_compressed;
+
+extern uint64_t shared_region_pager_reclaimed;
+extern struct memory_object_pager_ops shared_region_pager_ops;

 
 

-void update_vm_info(void)
+void
+update_vm_info(void)

 {
 {

-        int    tmp1, tmp2, tmp3, tmp4;
+       uint64_t tmp;
+
+       vm_pageout_stats[vm_pageout_stat_now].vm_page_active_count = vm_page_active_count;
+       vm_pageout_stats[vm_pageout_stat_now].vm_page_speculative_count = vm_page_speculative_count;
+       vm_pageout_stats[vm_pageout_stat_now].vm_page_inactive_count = vm_page_inactive_count;
+       vm_pageout_stats[vm_pageout_stat_now].vm_page_anonymous_count = vm_page_anonymous_count;
+
+       vm_pageout_stats[vm_pageout_stat_now].vm_page_free_count = vm_page_free_count;
+       vm_pageout_stats[vm_pageout_stat_now].vm_page_wire_count = vm_page_wire_count;
+       vm_pageout_stats[vm_pageout_stat_now].vm_page_compressor_count = VM_PAGE_COMPRESSOR_COUNT;
+
+       vm_pageout_stats[vm_pageout_stat_now].vm_page_pages_compressed = c_segment_pages_compressed;
+       vm_pageout_stats[vm_pageout_stat_now].vm_page_pageable_internal_count = vm_page_pageable_internal_count;
+       vm_pageout_stats[vm_pageout_stat_now].vm_page_pageable_external_count = vm_page_pageable_external_count;
+       vm_pageout_stats[vm_pageout_stat_now].vm_page_xpmapped_external_count = vm_page_xpmapped_external_count;
+
+
+       tmp = vm_pageout_vminfo.vm_pageout_considered_page;
+       vm_pageout_stats[vm_pageout_stat_now].considered = (unsigned int)(tmp - last.vm_pageout_considered_page);
+       last.vm_pageout_considered_page = tmp;
+
+       tmp = vm_pageout_vminfo.vm_pageout_compressions;
+       vm_pageout_stats[vm_pageout_stat_now].pages_compressed = (unsigned int)(tmp - last.vm_pageout_compressions);
+       last.vm_pageout_compressions = tmp;
+
+       tmp = vm_pageout_vminfo.vm_compressor_failed;
+       vm_pageout_stats[vm_pageout_stat_now].failed_compressions = (unsigned int)(tmp - last.vm_compressor_failed);
+       last.vm_compressor_failed = tmp;
+
+       tmp = vm_pageout_vminfo.vm_compressor_pages_grabbed;
+       vm_pageout_stats[vm_pageout_stat_now].pages_grabbed_by_compressor = (unsigned int)(tmp - last.vm_compressor_pages_grabbed);
+       last.vm_compressor_pages_grabbed = tmp;
+
+       tmp = vm_pageout_vminfo.vm_phantom_cache_found_ghost;
+       vm_pageout_stats[vm_pageout_stat_now].phantom_ghosts_found = (unsigned int)(tmp - last.vm_phantom_cache_found_ghost);
+       last.vm_phantom_cache_found_ghost = tmp;
+
+       tmp = vm_pageout_vminfo.vm_phantom_cache_added_ghost;
+       vm_pageout_stats[vm_pageout_stat_now].phantom_ghosts_added = (unsigned int)(tmp - last.vm_phantom_cache_added_ghost);
+       last.vm_phantom_cache_added_ghost = tmp;
+
+       tmp = get_pages_grabbed_count();
+       vm_pageout_stats[vm_pageout_stat_now].pages_grabbed = (unsigned int)(tmp - last_vm_page_pages_grabbed);
+       last_vm_page_pages_grabbed = tmp;
+
+       tmp = vm_pageout_vminfo.vm_page_pages_freed;
+       vm_pageout_stats[vm_pageout_stat_now].pages_freed = (unsigned int)(tmp - last.vm_page_pages_freed);
+       last.vm_page_pages_freed = tmp;
+
+
+       if (vm_pageout_stats[vm_pageout_stat_now].considered) {
+               tmp = vm_pageout_vminfo.vm_pageout_pages_evicted;
+               vm_pageout_stats[vm_pageout_stat_now].pages_evicted = (unsigned int)(tmp - last.vm_pageout_pages_evicted);
+               last.vm_pageout_pages_evicted = tmp;
+
+               tmp = vm_pageout_vminfo.vm_pageout_pages_purged;
+               vm_pageout_stats[vm_pageout_stat_now].pages_purged = (unsigned int)(tmp - last.vm_pageout_pages_purged);
+               last.vm_pageout_pages_purged = tmp;
+
+               tmp = vm_pageout_vminfo.vm_pageout_freed_speculative;
+               vm_pageout_stats[vm_pageout_stat_now].freed_speculative = (unsigned int)(tmp - last.vm_pageout_freed_speculative);
+               last.vm_pageout_freed_speculative = tmp;
+
+               tmp = vm_pageout_vminfo.vm_pageout_freed_external;
+               vm_pageout_stats[vm_pageout_stat_now].freed_external = (unsigned int)(tmp - last.vm_pageout_freed_external);
+               last.vm_pageout_freed_external = tmp;
+
+               tmp = vm_pageout_vminfo.vm_pageout_inactive_referenced;
+               vm_pageout_stats[vm_pageout_stat_now].inactive_referenced = (unsigned int)(tmp - last.vm_pageout_inactive_referenced);
+               last.vm_pageout_inactive_referenced = tmp;
+
+               tmp = vm_pageout_vminfo.vm_pageout_scan_inactive_throttled_external;
+               vm_pageout_stats[vm_pageout_stat_now].throttled_external_q = (unsigned int)(tmp - last.vm_pageout_scan_inactive_throttled_external);
+               last.vm_pageout_scan_inactive_throttled_external = tmp;
+
+               tmp = vm_pageout_vminfo.vm_pageout_inactive_dirty_external;
+               vm_pageout_stats[vm_pageout_stat_now].cleaned_dirty_external = (unsigned int)(tmp - last.vm_pageout_inactive_dirty_external);
+               last.vm_pageout_inactive_dirty_external = tmp;
+
+               tmp = vm_pageout_vminfo.vm_pageout_freed_cleaned;
+               vm_pageout_stats[vm_pageout_stat_now].freed_cleaned = (unsigned int)(tmp - last.vm_pageout_freed_cleaned);
+               last.vm_pageout_freed_cleaned = tmp;
+
+               tmp = vm_pageout_vminfo.vm_pageout_inactive_nolock;
+               vm_pageout_stats[vm_pageout_stat_now].inactive_nolock = (unsigned int)(tmp - last.vm_pageout_inactive_nolock);
+               last.vm_pageout_inactive_nolock = tmp;
+
+               tmp = vm_pageout_vminfo.vm_pageout_scan_inactive_throttled_internal;
+               vm_pageout_stats[vm_pageout_stat_now].throttled_internal_q = (unsigned int)(tmp - last.vm_pageout_scan_inactive_throttled_internal);
+               last.vm_pageout_scan_inactive_throttled_internal = tmp;
+
+               tmp = vm_pageout_vminfo.vm_pageout_skipped_external;
+               vm_pageout_stats[vm_pageout_stat_now].skipped_external = (unsigned int)(tmp - last.vm_pageout_skipped_external);
+               last.vm_pageout_skipped_external = tmp;
+
+               tmp = vm_pageout_vminfo.vm_pageout_reactivation_limit_exceeded;
+               vm_pageout_stats[vm_pageout_stat_now].reactivation_limit_exceeded = (unsigned int)(tmp - last.vm_pageout_reactivation_limit_exceeded);
+               last.vm_pageout_reactivation_limit_exceeded = tmp;
+
+               tmp = vm_pageout_vminfo.vm_pageout_inactive_force_reclaim;
+               vm_pageout_stats[vm_pageout_stat_now].forced_inactive_reclaim = (unsigned int)(tmp - last.vm_pageout_inactive_force_reclaim);
+               last.vm_pageout_inactive_force_reclaim = tmp;
+
+               tmp = vm_pageout_vminfo.vm_pageout_freed_internal;
+               vm_pageout_stats[vm_pageout_stat_now].freed_internal = (unsigned int)(tmp - last.vm_pageout_freed_internal);
+               last.vm_pageout_freed_internal = tmp;
+
+               tmp = vm_pageout_vminfo.vm_pageout_considered_bq_internal;
+               vm_pageout_stats[vm_pageout_stat_now].considered_bq_internal = (unsigned int)(tmp - last.vm_pageout_considered_bq_internal);
+               last.vm_pageout_considered_bq_internal = tmp;
+
+               tmp = vm_pageout_vminfo.vm_pageout_considered_bq_external;
+               vm_pageout_stats[vm_pageout_stat_now].considered_bq_external = (unsigned int)(tmp - last.vm_pageout_considered_bq_external);
+               last.vm_pageout_considered_bq_external = tmp;
+
+               tmp = vm_pageout_vminfo.vm_pageout_filecache_min_reactivated;
+               vm_pageout_stats[vm_pageout_stat_now].filecache_min_reactivations = (unsigned int)(tmp - last.vm_pageout_filecache_min_reactivated);
+               last.vm_pageout_filecache_min_reactivated = tmp;
+
+               tmp = vm_pageout_vminfo.vm_pageout_inactive_dirty_internal;
+               vm_pageout_stats[vm_pageout_stat_now].cleaned_dirty_internal = (unsigned int)(tmp - last.vm_pageout_inactive_dirty_internal);
+               last.vm_pageout_inactive_dirty_internal = tmp;
+       }

 
 

-       if (!kdebug_enable)
-               return;
-       

        KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO1)) | DBG_FUNC_NONE,
        KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO1)) | DBG_FUNC_NONE,

-                             vm_page_active_count,
-                             vm_page_speculative_count,
-                             vm_page_inactive_count,
-                             vm_page_anonymous_count,
-                             0);
+           vm_pageout_stats[vm_pageout_stat_now].vm_page_active_count,
+           vm_pageout_stats[vm_pageout_stat_now].vm_page_speculative_count,
+           vm_pageout_stats[vm_pageout_stat_now].vm_page_inactive_count,
+           vm_pageout_stats[vm_pageout_stat_now].vm_page_anonymous_count,
+           0);

 
        KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO2)) | DBG_FUNC_NONE,
 
        KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO2)) | DBG_FUNC_NONE,

-                             vm_page_free_count,
-                             vm_page_wire_count,
-                             VM_PAGE_COMPRESSOR_COUNT,
-                             0, 0);
+           vm_pageout_stats[vm_pageout_stat_now].vm_page_free_count,
+           vm_pageout_stats[vm_pageout_stat_now].vm_page_wire_count,
+           vm_pageout_stats[vm_pageout_stat_now].vm_page_compressor_count,
+           0,
+           0);

 
        KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO3)) | DBG_FUNC_NONE,
 
        KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO3)) | DBG_FUNC_NONE,

-                             c_segment_pages_compressed, 
-                             vm_page_internal_count,
-                             vm_page_external_count,
-                             vm_page_xpmapped_external_count,
-                             0);
-
-
-       if ((vm_pageout_considered_page - last_vm_pageout_considered_page) == 0 &&
-           (vm_pageout_enqueued_cleaned_from_inactive_dirty - last_vm_pageout_enqueued_cleaned_from_inactive_dirty == 0) &&
-           (vm_pageout_freed_after_compression - last_vm_pageout_freed_after_compression == 0))
-               return;
-
-
-       tmp1 = vm_pageout_considered_page;
-       tmp2 = vm_pageout_freed_from_speculative;
-       tmp3 = vm_pageout_freed_from_inactive_clean;
-
-       KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO4)) | DBG_FUNC_NONE,
-                             tmp1 - last_vm_pageout_considered_page,
-                             tmp2 - last_vm_pageout_freed_from_speculative,
-                             tmp3 - last_vm_pageout_freed_from_inactive_clean,
-                             0, 0);
-       
-       last_vm_pageout_considered_page = tmp1;
-       last_vm_pageout_freed_from_speculative = tmp2;
-       last_vm_pageout_freed_from_inactive_clean = tmp3;
-
-
-       tmp1 = vm_pageout_scan_inactive_throttled_external;
-       tmp2 = vm_pageout_enqueued_cleaned_from_inactive_dirty;
-       tmp3 = vm_pageout_freed_from_cleaned;
-
-       KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO5)) | DBG_FUNC_NONE,
-                             tmp1 - last_vm_pageout_scan_inactive_throttled_external,
-                             tmp2 - last_vm_pageout_enqueued_cleaned_from_inactive_dirty,
-                             tmp3 - last_vm_pageout_freed_from_cleaned,
-                             0, 0);
-
-       vm_pageout_stats[vm_pageout_stat_now].throttled_external_q += (tmp1 - last_vm_pageout_scan_inactive_throttled_external);
-       vm_pageout_stats[vm_pageout_stat_now].cleaned_dirty_external += (tmp2 - last_vm_pageout_enqueued_cleaned_from_inactive_dirty);
-
-       last_vm_pageout_scan_inactive_throttled_external = tmp1;
-       last_vm_pageout_enqueued_cleaned_from_inactive_dirty = tmp2;
-       last_vm_pageout_freed_from_cleaned = tmp3;
-
-
-       tmp1 = vm_pageout_scan_inactive_throttled_internal;
-       tmp2 = vm_pageout_freed_after_compression;
-       tmp3 = vm_compressor_pages_grabbed;
-       tmp4 = vm_pageout_skipped_external;
-
-       KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO6)) | DBG_FUNC_NONE,
-                             tmp1 - last_vm_pageout_scan_inactive_throttled_internal,
-                             tmp2 - last_vm_pageout_freed_after_compression,
-                             tmp3 - last_vm_compressor_pages_grabbed,
-                             tmp4 - last_vm_pageout_skipped_external,
-                             0);
-                             
-       vm_pageout_stats[vm_pageout_stat_now].throttled_internal_q += (tmp1 - last_vm_pageout_scan_inactive_throttled_internal);
-       vm_pageout_stats[vm_pageout_stat_now].pages_compressed += (tmp2 - last_vm_pageout_freed_after_compression);
-       vm_pageout_stats[vm_pageout_stat_now].pages_grabbed_by_compressor += (tmp3 - last_vm_compressor_pages_grabbed);
-
-       last_vm_pageout_scan_inactive_throttled_internal = tmp1;
-       last_vm_pageout_freed_after_compression = tmp2;
-       last_vm_compressor_pages_grabbed = tmp3;
-       last_vm_pageout_skipped_external = tmp4;
-
-
-       if ((vm_pageout_reactivation_limit_exceeded - last_vm_pageout_reactivation_limit_exceeded) == 0 &&
-           (vm_pageout_inactive_force_reclaim - last_vm_pageout_inactive_force_reclaim) == 0 &&
-           (vm_compressor_failed - last_vm_compressor_failed) == 0)
-               return;
-
-       tmp1 = vm_pageout_reactivation_limit_exceeded;
-       tmp2 = vm_pageout_inactive_force_reclaim;
-       tmp3 = vm_compressor_failed;
-
-       KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO7)) | DBG_FUNC_NONE,
-                             tmp1 - last_vm_pageout_reactivation_limit_exceeded,
-                             tmp2 - last_vm_pageout_inactive_force_reclaim,
-                             tmp3 - last_vm_compressor_failed,
-                             0, 0);
-
-       vm_pageout_stats[vm_pageout_stat_now].failed_compressions += (tmp3 - last_vm_compressor_failed);
-
-       last_vm_pageout_reactivation_limit_exceeded = tmp1;
-       last_vm_pageout_inactive_force_reclaim = tmp2;
-       last_vm_compressor_failed = tmp3;
+           vm_pageout_stats[vm_pageout_stat_now].vm_page_pages_compressed,
+           vm_pageout_stats[vm_pageout_stat_now].vm_page_pageable_internal_count,
+           vm_pageout_stats[vm_pageout_stat_now].vm_page_pageable_external_count,
+           vm_pageout_stats[vm_pageout_stat_now].vm_page_xpmapped_external_count,
+           0);
+
+       if (vm_pageout_stats[vm_pageout_stat_now].considered ||
+           vm_pageout_stats[vm_pageout_stat_now].pages_compressed ||
+           vm_pageout_stats[vm_pageout_stat_now].failed_compressions) {
+               KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO4)) | DBG_FUNC_NONE,
+                   vm_pageout_stats[vm_pageout_stat_now].considered,
+                   vm_pageout_stats[vm_pageout_stat_now].freed_speculative,
+                   vm_pageout_stats[vm_pageout_stat_now].freed_external,
+                   vm_pageout_stats[vm_pageout_stat_now].inactive_referenced,
+                   0);
+
+               KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO5)) | DBG_FUNC_NONE,
+                   vm_pageout_stats[vm_pageout_stat_now].throttled_external_q,
+                   vm_pageout_stats[vm_pageout_stat_now].cleaned_dirty_external,
+                   vm_pageout_stats[vm_pageout_stat_now].freed_cleaned,
+                   vm_pageout_stats[vm_pageout_stat_now].inactive_nolock,
+                   0);
+
+               KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO6)) | DBG_FUNC_NONE,
+                   vm_pageout_stats[vm_pageout_stat_now].throttled_internal_q,
+                   vm_pageout_stats[vm_pageout_stat_now].pages_compressed,
+                   vm_pageout_stats[vm_pageout_stat_now].pages_grabbed_by_compressor,
+                   vm_pageout_stats[vm_pageout_stat_now].skipped_external,
+                   0);
+
+               KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO7)) | DBG_FUNC_NONE,
+                   vm_pageout_stats[vm_pageout_stat_now].reactivation_limit_exceeded,
+                   vm_pageout_stats[vm_pageout_stat_now].forced_inactive_reclaim,
+                   vm_pageout_stats[vm_pageout_stat_now].failed_compressions,
+                   vm_pageout_stats[vm_pageout_stat_now].freed_internal,
+                   0);
+
+               KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO8)) | DBG_FUNC_NONE,
+                   vm_pageout_stats[vm_pageout_stat_now].considered_bq_internal,
+                   vm_pageout_stats[vm_pageout_stat_now].considered_bq_external,
+                   vm_pageout_stats[vm_pageout_stat_now].filecache_min_reactivations,
+                   vm_pageout_stats[vm_pageout_stat_now].cleaned_dirty_internal,
+                   0);
+       }
+       KERNEL_DEBUG_CONSTANT((MACHDBG_CODE(DBG_MACH_VM, VM_INFO9)) | DBG_FUNC_NONE,
+           vm_pageout_stats[vm_pageout_stat_now].pages_grabbed,
+           vm_pageout_stats[vm_pageout_stat_now].pages_freed,
+           vm_pageout_stats[vm_pageout_stat_now].phantom_ghosts_found,
+           vm_pageout_stats[vm_pageout_stat_now].phantom_ghosts_added,
+           0);
+
+       record_memory_pressure();
+}
+
+extern boolean_t hibernation_vmqueues_inspection;
+
+void
+vm_page_balance_inactive(int max_to_move)
+{
+       vm_page_t m;
+
+       LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
+
+       if (hibernation_vmqueues_inspection == TRUE) {
+               /*
+                * It is likely that the hibernation code path is
+                * dealing with these very queues as we are about
+                * to move pages around in/from them and completely
+                * change the linkage of the pages.
+                *
+                * And so we skip the rebalancing of these queues.
+                */
+               return;
+       }
+       vm_page_inactive_target = VM_PAGE_INACTIVE_TARGET(vm_page_active_count +
+           vm_page_inactive_count +
+           vm_page_speculative_count);
+
+       while (max_to_move-- && (vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_target) {
+               VM_PAGEOUT_DEBUG(vm_pageout_balanced, 1);
+
+               m = (vm_page_t) vm_page_queue_first(&vm_page_queue_active);
+
+               assert(m->vmp_q_state == VM_PAGE_ON_ACTIVE_Q);
+               assert(!m->vmp_laundry);
+               assert(VM_PAGE_OBJECT(m) != kernel_object);
+               assert(VM_PAGE_GET_PHYS_PAGE(m) != vm_page_guard_addr);
+
+               DTRACE_VM2(scan, int, 1, (uint64_t *), NULL);
+
+               /*
+                * by not passing in a pmap_flush_context we will forgo any TLB flushing, local or otherwise...
+                *
+                * a TLB flush isn't really needed here since at worst we'll miss the reference bit being
+                * updated in the PTE if a remote processor still has this mapping cached in its TLB when the
+                * new reference happens. If no futher references happen on the page after that remote TLB flushes
+                * we'll see a clean, non-referenced page when it eventually gets pulled out of the inactive queue
+                * by pageout_scan, which is just fine since the last reference would have happened quite far
+                * in the past (TLB caches don't hang around for very long), and of course could just as easily
+                * have happened before we moved the page
+                */
+               if (m->vmp_pmapped == TRUE) {
+                       pmap_clear_refmod_options(VM_PAGE_GET_PHYS_PAGE(m), VM_MEM_REFERENCED, PMAP_OPTIONS_NOFLUSH, (void *)NULL);
+               }
+
+               /*
+                * The page might be absent or busy,
+                * but vm_page_deactivate can handle that.
+                * FALSE indicates that we don't want a H/W clear reference
+                */
+               vm_page_deactivate_internal(m, FALSE);
+       }

 }
 
 
 }
 
 


@@ -1828,48 +1810,50 @@ vm_pageout_scan(void)
 {
        unsigned int loop_count = 0;
        unsigned int inactive_burst_count = 0;
 {
        unsigned int loop_count = 0;
        unsigned int inactive_burst_count = 0;

-       unsigned int active_burst_count = 0;

        unsigned int reactivated_this_call;
        unsigned int reactivate_limit;
        vm_page_t   local_freeq = NULL;
        int         local_freed = 0;
        int         delayed_unlock;
        unsigned int reactivated_this_call;
        unsigned int reactivate_limit;
        vm_page_t   local_freeq = NULL;
        int         local_freed = 0;
        int         delayed_unlock;

-       int         delayed_unlock_limit = 0;
-       int         refmod_state = 0;
-        int    vm_pageout_deadlock_target = 0;
-       struct  vm_pageout_queue *iq;
-       struct  vm_pageout_queue *eq;
-        struct vm_speculative_age_q *sq;
-       struct  flow_control    flow_control = { 0, { 0, 0 } };
-        boolean_t inactive_throttled = FALSE;
-       boolean_t try_failed;
-       mach_timespec_t ts;
-       unsigned        int msecs = 0;
-       vm_object_t     object = NULL;
-       uint32_t        inactive_reclaim_run;
-       boolean_t       exceeded_burst_throttle;
-       boolean_t       grab_anonymous = FALSE;
-       boolean_t       force_anonymous = FALSE;
-       boolean_t       force_speculative_aging = FALSE;
-       int             anons_grabbed = 0;
-       int             page_prev_q_state = 0;
+       int         delayed_unlock_limit = 0;
+       int         refmod_state = 0;
+       int     vm_pageout_deadlock_target = 0;
+       struct  vm_pageout_queue *iq;
+       struct  vm_pageout_queue *eq;
+       struct  vm_speculative_age_q *sq;
+       struct  flow_control    flow_control = { 0, { 0, 0 } };
+       boolean_t inactive_throttled = FALSE;
+       mach_timespec_t ts;
+       unsigned        int msecs = 0;
+       vm_object_t     object = NULL;
+       uint32_t        inactive_reclaim_run;
+       boolean_t       exceeded_burst_throttle;
+       boolean_t       grab_anonymous = FALSE;
+       boolean_t       force_anonymous = FALSE;
+       boolean_t       force_speculative_aging = FALSE;
+       int             anons_grabbed = 0;
+       int             page_prev_q_state = 0;

 #if CONFIG_BACKGROUND_QUEUE
 #if CONFIG_BACKGROUND_QUEUE

-       boolean_t       page_from_bg_q = FALSE;
+       boolean_t       page_from_bg_q = FALSE;

 #endif
 #endif

-       int             cache_evict_throttle = 0;
-       uint32_t        vm_pageout_inactive_external_forced_reactivate_limit = 0;
-       int             force_purge = 0;
-#define        DELAY_SPECULATIVE_AGE   1000
-       int             delay_speculative_age = 0;
-       vm_object_t     m_object = VM_OBJECT_NULL;
+       int             cache_evict_throttle = 0;
+       uint32_t        vm_pageout_inactive_external_forced_reactivate_limit = 0;
+       uint32_t        inactive_external_count;
+       int             force_purge = 0;
+       int             divisor;
+#define DELAY_SPECULATIVE_AGE   1000
+       int             delay_speculative_age = 0;
+       vm_object_t     m_object = VM_OBJECT_NULL;

 
 #if VM_PRESSURE_EVENTS
        vm_pressure_level_t pressure_level;
 #endif /* VM_PRESSURE_EVENTS */
 
        VM_DEBUG_CONSTANT_EVENT(vm_pageout_scan, VM_PAGEOUT_SCAN, DBG_FUNC_START,
 
 #if VM_PRESSURE_EVENTS
        vm_pressure_level_t pressure_level;
 #endif /* VM_PRESSURE_EVENTS */
 
        VM_DEBUG_CONSTANT_EVENT(vm_pageout_scan, VM_PAGEOUT_SCAN, DBG_FUNC_START,

-                      vm_pageout_speculative_clean, vm_pageout_inactive_clean,
-                      vm_pageout_inactive_dirty_internal, vm_pageout_inactive_dirty_external);
+           vm_pageout_vminfo.vm_pageout_freed_speculative,
+           vm_pageout_state.vm_pageout_inactive_clean,
+           vm_pageout_vminfo.vm_pageout_inactive_dirty_internal,
+           vm_pageout_vminfo.vm_pageout_inactive_dirty_external);

 
        flow_control.state = FCS_IDLE;
        iq = &vm_pageout_queue_internal;
 
        flow_control.state = FCS_IDLE;
        iq = &vm_pageout_queue_internal;


@@ -1877,12 +1861,15 @@ vm_pageout_scan(void)
        sq = &vm_page_queue_speculative[VM_PAGE_SPECULATIVE_AGED_Q];
 
 
        sq = &vm_page_queue_speculative[VM_PAGE_SPECULATIVE_AGED_Q];
 
 

-        XPR(XPR_VM_PAGEOUT, "vm_pageout_scan\n", 0, 0, 0, 0, 0);
+       XPR(XPR_VM_PAGEOUT, "vm_pageout_scan\n", 0, 0, 0, 0, 0);

 
        /* Ask the pmap layer to return any pages it no longer needs. */
 
        /* Ask the pmap layer to return any pages it no longer needs. */

-       pmap_release_pages_fast();
+       uint64_t pmap_wired_pages_freed = pmap_release_pages_fast();

 
        vm_page_lock_queues();
 
        vm_page_lock_queues();

+
+       vm_page_wire_count -= pmap_wired_pages_freed;
+

        delayed_unlock = 1;
 
        /*
        delayed_unlock = 1;
 
        /*


@@ -1891,125 +1878,85 @@ vm_pageout_scan(void)
         */
        reactivated_this_call = 0;
        reactivate_limit = VM_PAGE_REACTIVATE_LIMIT(vm_page_active_count +
         */
        reactivated_this_call = 0;
        reactivate_limit = VM_PAGE_REACTIVATE_LIMIT(vm_page_active_count +

-                                                   vm_page_inactive_count);
+           vm_page_inactive_count);

        inactive_reclaim_run = 0;
 
        vm_pageout_inactive_external_forced_reactivate_limit = vm_page_active_count + vm_page_inactive_count;
 
        /*
        inactive_reclaim_run = 0;
 
        vm_pageout_inactive_external_forced_reactivate_limit = vm_page_active_count + vm_page_inactive_count;
 
        /*

-        *      We want to gradually dribble pages from the active queue
-        *      to the inactive queue.  If we let the inactive queue get
-        *      very small, and then suddenly dump many pages into it,
-        *      those pages won't get a sufficient chance to be referenced
-        *      before we start taking them from the inactive queue.
-        *

         *      We must limit the rate at which we send pages to the pagers
         *      so that we don't tie up too many pages in the I/O queues.
         *      We implement a throttling mechanism using the laundry count
         *      We must limit the rate at which we send pages to the pagers
         *      so that we don't tie up too many pages in the I/O queues.
         *      We implement a throttling mechanism using the laundry count

-        *      to limit the number of pages outstanding to the default
+        *      to limit the number of pages outstanding to the default

         *      and external pagers.  We can bypass the throttles and look
         *      for clean pages if the pageout queues don't drain in a timely
         *      fashion since this may indicate that the pageout paths are
         *      stalled waiting for memory, which only we can provide.
         */
 
         *      and external pagers.  We can bypass the throttles and look
         *      for clean pages if the pageout queues don't drain in a timely
         *      fashion since this may indicate that the pageout paths are
         *      stalled waiting for memory, which only we can provide.
         */
 

-

 Restart:
 
        assert(object == NULL);
        assert(delayed_unlock != 0);
 Restart:
 
        assert(object == NULL);
        assert(delayed_unlock != 0);

-       
-       /*
-        *      Recalculate vm_page_inactivate_target.
-        */
-       vm_page_inactive_target = VM_PAGE_INACTIVE_TARGET(vm_page_active_count +
-                                                         vm_page_inactive_count +
-                                                         vm_page_speculative_count);

 
        vm_page_anonymous_min = vm_page_inactive_target / 20;
 
 
        vm_page_anonymous_min = vm_page_inactive_target / 20;
 

+       if (vm_pageout_state.vm_page_speculative_percentage > 50) {
+               vm_pageout_state.vm_page_speculative_percentage = 50;
+       } else if (vm_pageout_state.vm_page_speculative_percentage <= 0) {
+               vm_pageout_state.vm_page_speculative_percentage = 1;
+       }

 
 

-       /*
-        * don't want to wake the pageout_scan thread up everytime we fall below
-        * the targets... set a low water mark at 0.25% below the target
-        */
-       vm_page_inactive_min = vm_page_inactive_target - (vm_page_inactive_target / 400);
-
-       if (vm_page_speculative_percentage > 50)
-               vm_page_speculative_percentage = 50;
-       else if (vm_page_speculative_percentage <= 0)
-               vm_page_speculative_percentage = 1;
-
-       vm_page_speculative_target = VM_PAGE_SPECULATIVE_TARGET(vm_page_active_count +
-                                                               vm_page_inactive_count);
-
-       try_failed = FALSE;
+       vm_pageout_state.vm_page_speculative_target = VM_PAGE_SPECULATIVE_TARGET(vm_page_active_count +
+           vm_page_inactive_count);

 
        for (;;) {
                vm_page_t m;
 
                DTRACE_VM2(rev, int, 1, (uint64_t *), NULL);
 
 
        for (;;) {
                vm_page_t m;
 
                DTRACE_VM2(rev, int, 1, (uint64_t *), NULL);
 

-               if (vm_upl_wait_for_pages < 0)
+               if (vm_upl_wait_for_pages < 0) {

                        vm_upl_wait_for_pages = 0;
                        vm_upl_wait_for_pages = 0;

+               }

 
                delayed_unlock_limit = VM_PAGEOUT_DELAYED_UNLOCK_LIMIT + vm_upl_wait_for_pages;
 
 
                delayed_unlock_limit = VM_PAGEOUT_DELAYED_UNLOCK_LIMIT + vm_upl_wait_for_pages;
 

-               if (delayed_unlock_limit > VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX)
+               if (delayed_unlock_limit > VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX) {

                        delayed_unlock_limit = VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX;
                        delayed_unlock_limit = VM_PAGEOUT_DELAYED_UNLOCK_LIMIT_MAX;

+               }

 
 #if CONFIG_SECLUDED_MEMORY
                /*
                 * Deal with secluded_q overflow.
                 */
                if (vm_page_secluded_count > vm_page_secluded_target) {
 
 #if CONFIG_SECLUDED_MEMORY
                /*
                 * Deal with secluded_q overflow.
                 */
                if (vm_page_secluded_count > vm_page_secluded_target) {

-                       unsigned int secluded_overflow;

                        vm_page_t secluded_page;
 
                        vm_page_t secluded_page;
 

-                       if (object != NULL) {
-                               vm_object_unlock(object);
-                               object = NULL;
-                               vm_pageout_scan_wants_object = VM_OBJECT_NULL;
-                       }

                        /*
                         * SECLUDED_AGING_BEFORE_ACTIVE:
                         * Excess secluded pages go to the active queue and
                         * will later go to the inactive queue.
                         */
                        /*
                         * SECLUDED_AGING_BEFORE_ACTIVE:
                         * Excess secluded pages go to the active queue and
                         * will later go to the inactive queue.
                         */

-                       active_burst_count = MIN(vm_pageout_burst_active_throttle,
-                                                vm_page_secluded_count_inuse);
-                       secluded_overflow = (vm_page_secluded_count -
-                                            vm_page_secluded_target);
-                       while (secluded_overflow-- > 0 &&
-                              vm_page_secluded_count > vm_page_secluded_target) {
-                               assert((vm_page_secluded_count_free +
-                                       vm_page_secluded_count_inuse) ==
-                                      vm_page_secluded_count);
-                               secluded_page = (vm_page_t)vm_page_queue_first(&vm_page_queue_secluded);
-                               assert(secluded_page->vm_page_q_state ==
-                                      VM_PAGE_ON_SECLUDED_Q);
-                               vm_page_queues_remove(secluded_page, FALSE);
-                               assert(!secluded_page->fictitious);
-                               assert(!VM_PAGE_WIRED(secluded_page));
-                               if (secluded_page->vm_page_object == 0) {
-                                       /* transfer to free queue */
-                                       assert(secluded_page->busy);
-                                       secluded_page->snext = local_freeq;
-                                       local_freeq = secluded_page;
-                                       local_freed++;
-                               } else {
-                                       /* transfer to head of active queue */
-                                       vm_page_enqueue_active(secluded_page, FALSE);
-                                       if (active_burst_count-- == 0) {
-                                               vm_pageout_secluded_burst_count++;
-                                               break;
-                                       }
-                               }
+                       assert((vm_page_secluded_count_free +
+                           vm_page_secluded_count_inuse) ==
+                           vm_page_secluded_count);
+                       secluded_page = (vm_page_t)vm_page_queue_first(&vm_page_queue_secluded);
+                       assert(secluded_page->vmp_q_state == VM_PAGE_ON_SECLUDED_Q);
+
+                       vm_page_queues_remove(secluded_page, FALSE);
+                       assert(!secluded_page->vmp_fictitious);
+                       assert(!VM_PAGE_WIRED(secluded_page));
+
+                       if (secluded_page->vmp_object == 0) {
+                               /* transfer to free queue */
+                               assert(secluded_page->vmp_busy);
+                               secluded_page->vmp_snext = local_freeq;
+                               local_freeq = secluded_page;
+                               local_freed++;
+                       } else {
+                               /* transfer to head of active queue */
+                               vm_page_enqueue_active(secluded_page, FALSE);

                                secluded_page = VM_PAGE_NULL;
                                secluded_page = VM_PAGE_NULL;

-
-                               if (delayed_unlock++ > delayed_unlock_limit) {
-                                       vm_pageout_delayed_unlock(&delayed_unlock, &local_freed, &local_freeq);
-                               }

                        }
                }
 #endif /* CONFIG_SECLUDED_MEMORY */
                        }
                }
 #endif /* CONFIG_SECLUDED_MEMORY */


@@ -2017,88 +1964,25 @@ Restart:
                assert(delayed_unlock);
 
                /*
                assert(delayed_unlock);
 
                /*

-                * Move pages from active to inactive if we're below the target
+                * maintain our balance

                 */
                 */

-               if ((vm_page_inactive_count + vm_page_speculative_count) >= vm_page_inactive_target)
-                       goto done_moving_active_pages;
+               vm_page_balance_inactive(1);

 
 

-               if (object != NULL) {
-                       vm_object_unlock(object);
-                       object = NULL;
-                       vm_pageout_scan_wants_object = VM_OBJECT_NULL;
-               }
-               /*
-                * Don't sweep through active queue more than the throttle
-                * which should be kept relatively low
-                */
-               active_burst_count = MIN(vm_pageout_burst_active_throttle, vm_page_active_count);
-
-               VM_DEBUG_EVENT(vm_pageout_balance, VM_PAGEOUT_BALANCE, DBG_FUNC_START,
-                              vm_pageout_inactive, vm_pageout_inactive_used, vm_page_free_count, local_freed);
-
-               VM_DEBUG_EVENT(vm_pageout_balance, VM_PAGEOUT_BALANCE, DBG_FUNC_NONE,
-                              vm_pageout_speculative_clean, vm_pageout_inactive_clean,
-                              vm_pageout_inactive_dirty_internal, vm_pageout_inactive_dirty_external);
-               memoryshot(VM_PAGEOUT_BALANCE, DBG_FUNC_START);
-
-
-               while (!vm_page_queue_empty(&vm_page_queue_active) && active_burst_count--) {
-
-                       vm_pageout_active++;
-
-                       m = (vm_page_t) vm_page_queue_first(&vm_page_queue_active);
-
-                       assert(m->vm_page_q_state == VM_PAGE_ON_ACTIVE_Q);
-                       assert(!m->laundry);
-                       assert(VM_PAGE_OBJECT(m) != kernel_object);
-                       assert(VM_PAGE_GET_PHYS_PAGE(m) != vm_page_guard_addr);
-
-                       DTRACE_VM2(scan, int, 1, (uint64_t *), NULL);
-
-                       /*
-                        * by not passing in a pmap_flush_context we will forgo any TLB flushing, local or otherwise...
-                        *
-                        * a TLB flush isn't really needed here since at worst we'll miss the reference bit being
-                        * updated in the PTE if a remote processor still has this mapping cached in its TLB when the
-                        * new reference happens. If no futher references happen on the page after that remote TLB flushes
-                        * we'll see a clean, non-referenced page when it eventually gets pulled out of the inactive queue
-                        * by pageout_scan, which is just fine since the last reference would have happened quite far
-                        * in the past (TLB caches don't hang around for very long), and of course could just as easily
-                        * have happened before we moved the page
-                        */
-                       pmap_clear_refmod_options(VM_PAGE_GET_PHYS_PAGE(m), VM_MEM_REFERENCED, PMAP_OPTIONS_NOFLUSH, (void *)NULL);
-
-                       /*
-                        * The page might be absent or busy,
-                        * but vm_page_deactivate can handle that.
-                        * FALSE indicates that we don't want a H/W clear reference
-                        */
-                       vm_page_deactivate_internal(m, FALSE);
-
-                       if (delayed_unlock++ > delayed_unlock_limit) {
-                               vm_pageout_delayed_unlock(&delayed_unlock, &local_freed, &local_freeq);
-                       }
-               }
-
-               VM_DEBUG_EVENT(vm_pageout_balance, VM_PAGEOUT_BALANCE, DBG_FUNC_END,
-                              vm_page_active_count, vm_page_inactive_count, vm_page_speculative_count, vm_page_inactive_target);
-               memoryshot(VM_PAGEOUT_BALANCE, DBG_FUNC_END);

 
                /**********************************************************************
 
                /**********************************************************************

-                * above this point we're playing with the active and secluded queues
-                * below this point we're playing with the throttling mechanisms
-                * and the inactive queue
-                **********************************************************************/
+               * above this point we're playing with the active and secluded queues
+               * below this point we're playing with the throttling mechanisms
+               * and the inactive queue
+               **********************************************************************/

 
 

-done_moving_active_pages:
+               if (vm_page_free_count + local_freed >= vm_page_free_target) {
+                       vm_pageout_scan_wants_object = VM_OBJECT_NULL;

 
 

-               if (vm_page_free_count + local_freed >= vm_page_free_target)
-               {

                        vm_pageout_prepare_to_block(&object, &delayed_unlock, &local_freeq, &local_freed,
                        vm_pageout_prepare_to_block(&object, &delayed_unlock, &local_freeq, &local_freed,

-                                                   VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER);
+                           VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER);

                        /*
                         * make sure the pageout I/O threads are running
                        /*
                         * make sure the pageout I/O threads are running

-                        * throttled in case there are still requests 
+                        * throttled in case there are still requests

                         * in the laundry... since we have met our targets
                         * we don't need the laundry to be cleaned in a timely
                         * fashion... so let's avoid interfering with foreground
                         * in the laundry... since we have met our targets
                         * we don't need the laundry to be cleaned in a timely
                         * fashion... so let's avoid interfering with foreground


@@ -2106,23 +1990,7 @@ done_moving_active_pages:
                         */
                        vm_pageout_adjust_eq_iothrottle(eq, TRUE);
 
                         */
                        vm_pageout_adjust_eq_iothrottle(eq, TRUE);
 

-                       /*
-                        * recalculate vm_page_inactivate_target
-                        */
-                       vm_page_inactive_target = VM_PAGE_INACTIVE_TARGET(vm_page_active_count +
-                                                                         vm_page_inactive_count +
-                                                                         vm_page_speculative_count);
-#ifndef        CONFIG_EMBEDDED
-                       if (((vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_target) &&
-                           !vm_page_queue_empty(&vm_page_queue_active)) {
-                               /*
-                                * inactive target still not met... keep going
-                                * until we get the queues balanced...
-                                */
-                               continue;
-                       }
-#endif
-                       lck_mtx_lock(&vm_page_queue_free_lock);
+                       lck_mtx_lock(&vm_page_queue_free_lock);

 
                        if ((vm_page_free_count >= vm_page_free_target) &&
                            (vm_page_free_wanted == 0) && (vm_page_free_wanted_privileged == 0)) {
 
                        if ((vm_page_free_count >= vm_page_free_target) &&
                            (vm_page_free_wanted == 0) && (vm_page_free_wanted_privileged == 0)) {


@@ -2134,46 +2002,47 @@ return_from_scan:
                                assert(vm_pageout_scan_wants_object == VM_OBJECT_NULL);
 
                                VM_DEBUG_CONSTANT_EVENT(vm_pageout_scan, VM_PAGEOUT_SCAN, DBG_FUNC_NONE,
                                assert(vm_pageout_scan_wants_object == VM_OBJECT_NULL);
 
                                VM_DEBUG_CONSTANT_EVENT(vm_pageout_scan, VM_PAGEOUT_SCAN, DBG_FUNC_NONE,

-                                              vm_pageout_inactive, vm_pageout_inactive_used, 0, 0);
+                                   vm_pageout_state.vm_pageout_inactive,
+                                   vm_pageout_state.vm_pageout_inactive_used, 0, 0);

                                VM_DEBUG_CONSTANT_EVENT(vm_pageout_scan, VM_PAGEOUT_SCAN, DBG_FUNC_END,
                                VM_DEBUG_CONSTANT_EVENT(vm_pageout_scan, VM_PAGEOUT_SCAN, DBG_FUNC_END,

-                                              vm_pageout_speculative_clean, vm_pageout_inactive_clean,
-                                              vm_pageout_inactive_dirty_internal, vm_pageout_inactive_dirty_external);
+                                   vm_pageout_vminfo.vm_pageout_freed_speculative,
+                                   vm_pageout_state.vm_pageout_inactive_clean,
+                                   vm_pageout_vminfo.vm_pageout_inactive_dirty_internal,
+                                   vm_pageout_vminfo.vm_pageout_inactive_dirty_external);

 
                                return;
                        }
                        lck_mtx_unlock(&vm_page_queue_free_lock);
                }
 
                                return;
                        }
                        lck_mtx_unlock(&vm_page_queue_free_lock);
                }

-               
+

                /*
                /*

-                * Before anything, we check if we have any ripe volatile 
+                * Before anything, we check if we have any ripe volatile

                 * objects around. If so, try to purge the first object.
                 * If the purge fails, fall through to reclaim a page instead.
                 * If the purge succeeds, go back to the top and reevalute
                 * the new memory situation.
                 */
                 * objects around. If so, try to purge the first object.
                 * If the purge fails, fall through to reclaim a page instead.
                 * If the purge succeeds, go back to the top and reevalute
                 * the new memory situation.
                 */

-               
-               assert (available_for_purge>=0);
+
+               assert(available_for_purge >= 0);

                force_purge = 0; /* no force-purging */
 
 #if VM_PRESSURE_EVENTS
                pressure_level = memorystatus_vm_pressure_level;
 
                if (pressure_level > kVMPressureNormal) {
                force_purge = 0; /* no force-purging */
 
 #if VM_PRESSURE_EVENTS
                pressure_level = memorystatus_vm_pressure_level;
 
                if (pressure_level > kVMPressureNormal) {

-

                        if (pressure_level >= kVMPressureCritical) {
                        if (pressure_level >= kVMPressureCritical) {

-                               force_purge = memorystatus_purge_on_critical;
+                               force_purge = vm_pageout_state.memorystatus_purge_on_critical;

                        } else if (pressure_level >= kVMPressureUrgent) {
                        } else if (pressure_level >= kVMPressureUrgent) {

-                               force_purge = memorystatus_purge_on_urgent;
+                               force_purge = vm_pageout_state.memorystatus_purge_on_urgent;

                        } else if (pressure_level >= kVMPressureWarning) {
                        } else if (pressure_level >= kVMPressureWarning) {

-                               force_purge = memorystatus_purge_on_warning;
+                               force_purge = vm_pageout_state.memorystatus_purge_on_warning;

                        }
                }
 #endif /* VM_PRESSURE_EVENTS */
 
                if (available_for_purge || force_purge) {
                        }
                }
 #endif /* VM_PRESSURE_EVENTS */
 
                if (available_for_purge || force_purge) {

-
-                       if (object != NULL) {
-                               vm_object_unlock(object);
+                       if (object != NULL) {
+                               vm_object_unlock(object);

                                object = NULL;
                        }
 
                                object = NULL;
                        }
 


@@ -2181,7 +2050,7 @@ return_from_scan:
 
                        VM_DEBUG_EVENT(vm_pageout_purgeone, VM_PAGEOUT_PURGEONE, DBG_FUNC_START, vm_page_free_count, 0, 0, 0);
                        if (vm_purgeable_object_purge_one(force_purge, C_DONT_BLOCK)) {
 
                        VM_DEBUG_EVENT(vm_pageout_purgeone, VM_PAGEOUT_PURGEONE, DBG_FUNC_START, vm_page_free_count, 0, 0, 0);
                        if (vm_purgeable_object_purge_one(force_purge, C_DONT_BLOCK)) {

-                               vm_pageout_purged_objects++;
+                               VM_PAGEOUT_DEBUG(vm_pageout_purged_objects, 1);

                                VM_DEBUG_EVENT(vm_pageout_purgeone, VM_PAGEOUT_PURGEONE, DBG_FUNC_END, vm_page_free_count, 0, 0, 0);
                                memoryshot(VM_PAGEOUT_PURGEONE, DBG_FUNC_END);
                                continue;
                                VM_DEBUG_EVENT(vm_pageout_purgeone, VM_PAGEOUT_PURGEONE, DBG_FUNC_END, vm_page_free_count, 0, 0, 0);
                                memoryshot(VM_PAGEOUT_PURGEONE, DBG_FUNC_END);
                                continue;


@@ -2191,26 +2060,26 @@ return_from_scan:
                }
 
                if (vm_page_queue_empty(&sq->age_q) && vm_page_speculative_count) {
                }
 
                if (vm_page_queue_empty(&sq->age_q) && vm_page_speculative_count) {

-                       /*
+                       /*

                         * try to pull pages from the aging bins...
                         * see vm_page.h for an explanation of how
                         * this mechanism works
                         */
                         * try to pull pages from the aging bins...
                         * see vm_page.h for an explanation of how
                         * this mechanism works
                         */

-                       struct vm_speculative_age_q     *aq;
-                       boolean_t       can_steal = FALSE;
+                       struct vm_speculative_age_q     *aq;
+                       boolean_t       can_steal = FALSE;

                        int num_scanned_queues;
                        int num_scanned_queues;

-                      
+

                        aq = &vm_page_queue_speculative[speculative_steal_index];
 
                        num_scanned_queues = 0;
                        while (vm_page_queue_empty(&aq->age_q) &&
                        aq = &vm_page_queue_speculative[speculative_steal_index];
 
                        num_scanned_queues = 0;
                        while (vm_page_queue_empty(&aq->age_q) &&

-                              num_scanned_queues++ != VM_PAGE_MAX_SPECULATIVE_AGE_Q) {
+                           num_scanned_queues++ != VM_PAGE_MAX_SPECULATIVE_AGE_Q) {
+                               speculative_steal_index++;

 
 

-                               speculative_steal_index++;
+                               if (speculative_steal_index > VM_PAGE_MAX_SPECULATIVE_AGE_Q) {
+                                       speculative_steal_index = VM_PAGE_MIN_SPECULATIVE_AGE_Q;
+                               }

 
 

-                               if (speculative_steal_index > VM_PAGE_MAX_SPECULATIVE_AGE_Q)
-                                       speculative_steal_index = VM_PAGE_MIN_SPECULATIVE_AGE_Q;
-                               

                                aq = &vm_page_queue_speculative[speculative_steal_index];
                        }
 
                                aq = &vm_page_queue_speculative[speculative_steal_index];
                        }
 


@@ -2220,34 +2089,28 @@ return_from_scan:
                                 * queues but still haven't found one
                                 * that is not empty, even though
                                 * vm_page_speculative_count is not 0.
                                 * queues but still haven't found one
                                 * that is not empty, even though
                                 * vm_page_speculative_count is not 0.

-                                *
-                                * report the anomaly...

                                 */
                                 */

-                               printf("vm_pageout_scan: "
-                                      "all speculative queues empty "
-                                      "but count=%d.  Re-adjusting.\n",
-                                      vm_page_speculative_count);
-                               if (vm_page_speculative_count > vm_page_speculative_count_drift_max)
-                                       vm_page_speculative_count_drift_max = vm_page_speculative_count;
-                               vm_page_speculative_count_drifts++;
+                               if (!vm_page_queue_empty(&sq->age_q)) {
+                                       continue;
+                               }

 #if DEVELOPMENT || DEBUG
                                panic("vm_pageout_scan: vm_page_speculative_count=%d but queues are empty", vm_page_speculative_count);
 #if DEVELOPMENT || DEBUG
                                panic("vm_pageout_scan: vm_page_speculative_count=%d but queues are empty", vm_page_speculative_count);

-#endif /* DEVELOPMENT || DEBUG */
+#endif

                                /* readjust... */
                                vm_page_speculative_count = 0;
                                /* ... and continue */
                                continue;
                        }
 
                                /* readjust... */
                                vm_page_speculative_count = 0;
                                /* ... and continue */
                                continue;
                        }
 

-                       if (vm_page_speculative_count > vm_page_speculative_target || force_speculative_aging == TRUE)
-                               can_steal = TRUE;
-                       else {
+                       if (vm_page_speculative_count > vm_pageout_state.vm_page_speculative_target || force_speculative_aging == TRUE) {
+                               can_steal = TRUE;
+                       } else {

                                if (!delay_speculative_age) {
                                if (!delay_speculative_age) {

-                                       mach_timespec_t ts_fully_aged;
+                                       mach_timespec_t ts_fully_aged;

 
 

-                                       ts_fully_aged.tv_sec = (VM_PAGE_MAX_SPECULATIVE_AGE_Q * vm_page_speculative_q_age_ms) / 1000;
-                                       ts_fully_aged.tv_nsec = ((VM_PAGE_MAX_SPECULATIVE_AGE_Q * vm_page_speculative_q_age_ms) % 1000)
-                                               * 1000 * NSEC_PER_USEC;
+                                       ts_fully_aged.tv_sec = (VM_PAGE_MAX_SPECULATIVE_AGE_Q * vm_pageout_state.vm_page_speculative_q_age_ms) / 1000;
+                                       ts_fully_aged.tv_nsec = ((VM_PAGE_MAX_SPECULATIVE_AGE_Q * vm_pageout_state.vm_page_speculative_q_age_ms) % 1000)
+                                           * 1000 * NSEC_PER_USEC;

 
                                        ADD_MACH_TIMESPEC(&ts_fully_aged, &aq->age_ts);
 
 
                                        ADD_MACH_TIMESPEC(&ts_fully_aged, &aq->age_ts);
 


@@ -2257,42 +2120,42 @@ return_from_scan:
                                        ts.tv_sec = (unsigned int) sec;
                                        ts.tv_nsec = nsec;
 
                                        ts.tv_sec = (unsigned int) sec;
                                        ts.tv_nsec = nsec;
 

-                                       if (CMP_MACH_TIMESPEC(&ts, &ts_fully_aged) >= 0)
+                                       if (CMP_MACH_TIMESPEC(&ts, &ts_fully_aged) >= 0) {

                                                can_steal = TRUE;
                                                can_steal = TRUE;

-                                       else
+                                       } else {

                                                delay_speculative_age++;
                                                delay_speculative_age++;

+                                       }

                                } else {
                                        delay_speculative_age++;
                                } else {
                                        delay_speculative_age++;

-                                       if (delay_speculative_age == DELAY_SPECULATIVE_AGE)
+                                       if (delay_speculative_age == DELAY_SPECULATIVE_AGE) {

                                                delay_speculative_age = 0;
                                                delay_speculative_age = 0;

+                                       }

                                }
                        }
                                }
                        }

-                       if (can_steal == TRUE)
+                       if (can_steal == TRUE) {

                                vm_page_speculate_ageit(aq);
                                vm_page_speculate_ageit(aq);

+                       }

                }
                force_speculative_aging = FALSE;
 
                }
                force_speculative_aging = FALSE;
 

-#if CONFIG_BACKGROUND_QUEUE
-               if (vm_page_queue_empty(&sq->age_q) && cache_evict_throttle == 0 &&
-                   ((vm_page_background_mode == VM_PAGE_BG_DISABLED) || (vm_page_background_count <= vm_page_background_target)))
-#else
-               if (vm_page_queue_empty(&sq->age_q) && cache_evict_throttle == 0)
-#endif
-               {
-                       int     pages_evicted;
+               if (vm_page_queue_empty(&sq->age_q) && cache_evict_throttle == 0) {
+                       int     pages_evicted;

 
 

-                       if (object != NULL) {
-                               vm_object_unlock(object);
+                       if (object != NULL) {
+                               vm_object_unlock(object);

                                object = NULL;
                        }
                                object = NULL;
                        }

+                       KERNEL_DEBUG_CONSTANT(0x13001ec | DBG_FUNC_START, 0, 0, 0, 0, 0);
+

                        pages_evicted = vm_object_cache_evict(100, 10);
 
                        pages_evicted = vm_object_cache_evict(100, 10);
 

-                       if (pages_evicted) {
+                       KERNEL_DEBUG_CONSTANT(0x13001ec | DBG_FUNC_END, pages_evicted, 0, 0, 0, 0);

 
 

-                               vm_pageout_cache_evicted += pages_evicted;
+                       if (pages_evicted) {
+                               vm_pageout_vminfo.vm_pageout_pages_evicted += pages_evicted;

 
                                VM_DEBUG_EVENT(vm_pageout_cache_evict, VM_PAGEOUT_CACHE_EVICT, DBG_FUNC_NONE,
 
                                VM_DEBUG_EVENT(vm_pageout_cache_evict, VM_PAGEOUT_CACHE_EVICT, DBG_FUNC_NONE,

-                                              vm_page_free_count, pages_evicted, vm_pageout_cache_evicted, 0);
+                                   vm_page_free_count, pages_evicted, vm_pageout_vminfo.vm_pageout_pages_evicted, 0);

                                memoryshot(VM_PAGEOUT_CACHE_EVICT, DBG_FUNC_NONE);
 
                                /*
                                memoryshot(VM_PAGEOUT_CACHE_EVICT, DBG_FUNC_NONE);
 
                                /*


@@ -2301,11 +2164,15 @@ return_from_scan:
                                 * and re-evaulate the memory situation
                                 */
                                continue;
                                 * and re-evaulate the memory situation
                                 */
                                continue;

-                       } else
+                       } else {

                                cache_evict_throttle = 1000;
                                cache_evict_throttle = 1000;

+                       }

                }
                }

-               if  (cache_evict_throttle)
+               if (cache_evict_throttle) {

                        cache_evict_throttle--;
                        cache_evict_throttle--;

+               }
+
+               divisor = vm_pageout_state.vm_page_filecache_min_divisor;

 
 #if CONFIG_JETSAM
                /*
 
 #if CONFIG_JETSAM
                /*


@@ -2319,22 +2186,26 @@ return_from_scan:
                 * throttled queue (which isn't counted as available) which
                 * effectively disables this filter
                 */
                 * throttled queue (which isn't counted as available) which
                 * effectively disables this filter
                 */

-               if (vm_compressor_low_on_space())
-                       vm_page_filecache_min = 0;
-               else
-                       vm_page_filecache_min = (AVAILABLE_NON_COMPRESSED_MEMORY / 7);
+               if (vm_compressor_low_on_space() || divisor == 0) {
+                       vm_pageout_state.vm_page_filecache_min = 0;
+               } else {
+                       vm_pageout_state.vm_page_filecache_min =
+                           ((AVAILABLE_NON_COMPRESSED_MEMORY) * 10) / divisor;
+               }

 #else
 #else

-               if (vm_compressor_out_of_space())
-                       vm_page_filecache_min = 0;
-               else {
+               if (vm_compressor_out_of_space() || divisor == 0) {
+                       vm_pageout_state.vm_page_filecache_min = 0;
+               } else {

                        /*
                        /*

-                        * don't let the filecache_min fall below 33% of available memory...
+                        * don't let the filecache_min fall below the specified critical level

                         */
                         */

-                       vm_page_filecache_min = (AVAILABLE_NON_COMPRESSED_MEMORY / 3);
+                       vm_pageout_state.vm_page_filecache_min =
+                           ((AVAILABLE_NON_COMPRESSED_MEMORY) * 10) / divisor;

                }
 #endif
                }
 #endif

-               if (vm_page_free_count < (vm_page_free_reserved / 4))
-                       vm_page_filecache_min = 0;
+               if (vm_page_free_count < (vm_page_free_reserved / 4)) {
+                       vm_pageout_state.vm_page_filecache_min = 0;
+               }

 
                exceeded_burst_throttle = FALSE;
                /*
 
                exceeded_burst_throttle = FALSE;
                /*


@@ -2346,74 +2217,66 @@ return_from_scan:
                 */
                if (vm_page_queue_empty(&vm_page_queue_inactive) &&
                    vm_page_queue_empty(&vm_page_queue_anonymous) &&
                 */
                if (vm_page_queue_empty(&vm_page_queue_inactive) &&
                    vm_page_queue_empty(&vm_page_queue_anonymous) &&

+                   vm_page_queue_empty(&vm_page_queue_cleaned) &&

                    vm_page_queue_empty(&sq->age_q)) {
                    vm_page_queue_empty(&sq->age_q)) {

-                       vm_pageout_scan_empty_throttle++;
-                       msecs = vm_pageout_empty_wait;
+                       VM_PAGEOUT_DEBUG(vm_pageout_scan_empty_throttle, 1);
+                       msecs = vm_pageout_state.vm_pageout_empty_wait;

                        goto vm_pageout_scan_delay;
                        goto vm_pageout_scan_delay;

-
-               } else if (inactive_burst_count >= 
-                          MIN(vm_pageout_burst_inactive_throttle,
-                              (vm_page_inactive_count +
-                               vm_page_speculative_count))) {
-                       vm_pageout_scan_burst_throttle++;
-                       msecs = vm_pageout_burst_wait;
+               } else if (inactive_burst_count >=
+                   MIN(vm_pageout_state.vm_pageout_burst_inactive_throttle,
+                   (vm_page_inactive_count +
+                   vm_page_speculative_count))) {
+                       VM_PAGEOUT_DEBUG(vm_pageout_scan_burst_throttle, 1);
+                       msecs = vm_pageout_state.vm_pageout_burst_wait;

 
                        exceeded_burst_throttle = TRUE;
                        goto vm_pageout_scan_delay;
 
                        exceeded_burst_throttle = TRUE;
                        goto vm_pageout_scan_delay;

-
-               } else if (vm_page_free_count > (vm_page_free_reserved / 4) &&
-                          VM_PAGEOUT_SCAN_NEEDS_TO_THROTTLE()) {
-                       vm_pageout_scan_swap_throttle++;
-                       msecs = vm_pageout_swap_wait;
-                       goto vm_pageout_scan_delay;
-
-               } else if (VM_PAGE_Q_THROTTLED(iq) && 
-                                 VM_DYNAMIC_PAGING_ENABLED()) {
+               } else if (VM_PAGE_Q_THROTTLED(iq) &&
+                   VM_DYNAMIC_PAGING_ENABLED()) {

                        clock_sec_t sec;
                        clock_nsec_t nsec;
 
                        clock_sec_t sec;
                        clock_nsec_t nsec;
 

-                       switch (flow_control.state) {
-
+                       switch (flow_control.state) {

                        case FCS_IDLE:
                        case FCS_IDLE:

-                               if ((vm_page_free_count + local_freed) < vm_page_free_target) {
+                               if ((vm_page_free_count + local_freed) < vm_page_free_target &&
+                                   vm_pageout_state.vm_restricted_to_single_processor == FALSE) {
+                                       /*
+                                        * since the compressor is running independently of vm_pageout_scan
+                                        * let's not wait for it just yet... as long as we have a healthy supply
+                                        * of filecache pages to work with, let's keep stealing those.
+                                        */
+                                       inactive_external_count = vm_page_inactive_count - vm_page_anonymous_count;

 
 

-                                       vm_pageout_prepare_to_block(&object, &delayed_unlock, &local_freeq, &local_freed,
-                                                                   VM_PAGEOUT_PB_THREAD_YIELD);
-                                       if (!VM_PAGE_Q_THROTTLED(iq)) {
-                                               vm_pageout_scan_yield_unthrottled++;
-                                               continue;
-                                       }
-                                       if (vm_page_pageable_external_count > vm_page_filecache_min &&
-                                           !vm_page_queue_empty(&vm_page_queue_inactive)) {
+                                       if (vm_page_pageable_external_count > vm_pageout_state.vm_page_filecache_min &&
+                                           (inactive_external_count >= VM_PAGE_INACTIVE_TARGET(vm_page_pageable_external_count))) {

                                                anons_grabbed = ANONS_GRABBED_LIMIT;
                                                anons_grabbed = ANONS_GRABBED_LIMIT;

-                                               vm_pageout_scan_throttle_deferred++;
+                                               VM_PAGEOUT_DEBUG(vm_pageout_scan_throttle_deferred, 1);

                                                goto consider_inactive;
                                        }
                                                goto consider_inactive;
                                        }

-                                       if (((vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_target) && vm_page_active_count)
-                                               continue;

                                }
 reset_deadlock_timer:
                                }
 reset_deadlock_timer:

-                               ts.tv_sec = vm_pageout_deadlock_wait / 1000;
-                               ts.tv_nsec = (vm_pageout_deadlock_wait % 1000) * 1000 * NSEC_PER_USEC;
-                               clock_get_system_nanotime(&sec, &nsec);
+                               ts.tv_sec = vm_pageout_state.vm_pageout_deadlock_wait / 1000;
+                               ts.tv_nsec = (vm_pageout_state.vm_pageout_deadlock_wait % 1000) * 1000 * NSEC_PER_USEC;
+                               clock_get_system_nanotime(&sec, &nsec);

                                flow_control.ts.tv_sec = (unsigned int) sec;
                                flow_control.ts.tv_nsec = nsec;
                                ADD_MACH_TIMESPEC(&flow_control.ts, &ts);
                                flow_control.ts.tv_sec = (unsigned int) sec;
                                flow_control.ts.tv_nsec = nsec;
                                ADD_MACH_TIMESPEC(&flow_control.ts, &ts);

-                               
+

                                flow_control.state = FCS_DELAYED;
                                flow_control.state = FCS_DELAYED;

-                               msecs = vm_pageout_deadlock_wait;
+                               msecs = vm_pageout_state.vm_pageout_deadlock_wait;

 
 

+                               vm_pageout_vminfo.vm_pageout_scan_inactive_throttled_internal++;

                                break;
                                break;

-                                       
+

                        case FCS_DELAYED:
                        case FCS_DELAYED:

-                               clock_get_system_nanotime(&sec, &nsec);
+                               clock_get_system_nanotime(&sec, &nsec);

                                ts.tv_sec = (unsigned int) sec;
                                ts.tv_nsec = nsec;
 
                                if (CMP_MACH_TIMESPEC(&ts, &flow_control.ts) >= 0) {
                                ts.tv_sec = (unsigned int) sec;
                                ts.tv_nsec = nsec;
 
                                if (CMP_MACH_TIMESPEC(&ts, &flow_control.ts) >= 0) {

-                                       /*
+                                       /*

                                         * the pageout thread for the default pager is potentially
                                         * the pageout thread for the default pager is potentially

-                                        * deadlocked since the 
+                                        * deadlocked since the

                                         * default pager queue has been throttled for more than the
                                         * allowable time... we need to move some clean pages or dirty
                                         * pages belonging to the external pagers if they aren't throttled
                                         * default pager queue has been throttled for more than the
                                         * allowable time... we need to move some clean pages or dirty
                                         * pages belonging to the external pagers if they aren't throttled


@@ -2421,14 +2284,15 @@ reset_deadlock_timer:
                                         * blocked waiting for pages... we'll move one page for each of
                                         * these plus a fixed amount to break the logjam... once we're done
                                         * moving this number of pages, we'll re-enter the FSC_DELAYED state
                                         * blocked waiting for pages... we'll move one page for each of
                                         * these plus a fixed amount to break the logjam... once we're done
                                         * moving this number of pages, we'll re-enter the FSC_DELAYED state

-                                        * with a new timeout target since we have no way of knowing 
+                                        * with a new timeout target since we have no way of knowing

                                         * whether we've broken the deadlock except through observation
                                         * of the queue associated with the default pager... we need to
                                         * stop moving pages and allow the system to run to see what
                                         * state it settles into.
                                         */
                                         * whether we've broken the deadlock except through observation
                                         * of the queue associated with the default pager... we need to
                                         * stop moving pages and allow the system to run to see what
                                         * state it settles into.
                                         */

-                                       vm_pageout_deadlock_target = vm_pageout_deadlock_relief + vm_page_free_wanted + vm_page_free_wanted_privileged;
-                                       vm_pageout_scan_deadlock_detected++;
+                                       vm_pageout_deadlock_target = vm_pageout_state.vm_pageout_deadlock_relief +
+                                           vm_page_free_wanted + vm_page_free_wanted_privileged;
+                                       VM_PAGEOUT_DEBUG(vm_pageout_scan_deadlock_detected, 1);

                                        flow_control.state = FCS_DEADLOCK_DETECTED;
                                        thread_wakeup((event_t) &vm_pageout_garbage_collect);
                                        goto consider_inactive;
                                        flow_control.state = FCS_DEADLOCK_DETECTED;
                                        thread_wakeup((event_t) &vm_pageout_garbage_collect);
                                        goto consider_inactive;


@@ -2439,38 +2303,34 @@ reset_deadlock_timer:
                                 * awakened immediately upon a laundry completion,
                                 * so we won't wait any longer than necessary
                                 */
                                 * awakened immediately upon a laundry completion,
                                 * so we won't wait any longer than necessary
                                 */

-                               msecs = vm_pageout_idle_wait;
+                               msecs = vm_pageout_state.vm_pageout_idle_wait;

                                break;
 
                        case FCS_DEADLOCK_DETECTED:
                                break;
 
                        case FCS_DEADLOCK_DETECTED:

-                               if (vm_pageout_deadlock_target)
-                                       goto consider_inactive;
+                               if (vm_pageout_deadlock_target) {
+                                       goto consider_inactive;
+                               }

                                goto reset_deadlock_timer;
                                goto reset_deadlock_timer;

-

                        }
 vm_pageout_scan_delay:
                        }
 vm_pageout_scan_delay:

-                       vm_pageout_prepare_to_block(&object, &delayed_unlock, &local_freeq, &local_freed, 
-                                                   VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER);
+                       vm_pageout_scan_wants_object = VM_OBJECT_NULL;
+
+                       vm_pageout_prepare_to_block(&object, &delayed_unlock, &local_freeq, &local_freed,
+                           VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER);

 
 

-                       if (flow_control.state == FCS_DELAYED &&
-                           !VM_PAGE_Q_THROTTLED(iq)) {
-                               flow_control.state = FCS_IDLE;
-                               goto consider_inactive;
-                       }
-                       

                        if (vm_page_free_count >= vm_page_free_target) {
                                /*
                                 * we're here because
                                 *  1) someone else freed up some pages while we had
                                 *     the queues unlocked above
                        if (vm_page_free_count >= vm_page_free_target) {
                                /*
                                 * we're here because
                                 *  1) someone else freed up some pages while we had
                                 *     the queues unlocked above

-                                * and we've hit one of the 3 conditions that 
+                                * and we've hit one of the 3 conditions that

                                 * cause us to pause the pageout scan thread
                                 *
                                 * since we already have enough free pages,
                                 * let's avoid stalling and return normally
                                 *
                                 * before we return, make sure the pageout I/O threads
                                 * cause us to pause the pageout scan thread
                                 *
                                 * since we already have enough free pages,
                                 * let's avoid stalling and return normally
                                 *
                                 * before we return, make sure the pageout I/O threads

-                                * are running throttled in case there are still requests 
+                                * are running throttled in case there are still requests

                                 * in the laundry... since we have enough free pages
                                 * we don't need the laundry to be cleaned in a timely
                                 * fashion... so let's avoid interfering with foreground
                                 * in the laundry... since we have enough free pages
                                 * we don't need the laundry to be cleaned in a timely
                                 * fashion... so let's avoid interfering with foreground


@@ -2493,7 +2353,7 @@ vm_pageout_scan_delay:
                                goto return_from_scan;
                        }
                        lck_mtx_unlock(&vm_page_queue_free_lock);
                                goto return_from_scan;
                        }
                        lck_mtx_unlock(&vm_page_queue_free_lock);

-                       
+

                        if ((vm_page_free_count + vm_page_cleaned_count) < vm_page_free_target) {
                                /*
                                 * we're most likely about to block due to one of
                        if ((vm_page_free_count + vm_page_cleaned_count) < vm_page_free_target) {
                                /*
                                 * we're most likely about to block due to one of


@@ -2523,35 +2383,43 @@ vm_pageout_scan_delay:
                                flow_control.state = FCS_IDLE;
                                goto consider_inactive;
                        }
                                flow_control.state = FCS_IDLE;
                                goto consider_inactive;
                        }

+                       if (flow_control.state == FCS_DELAYED && !VM_PAGE_Q_THROTTLED(iq)) {
+                               flow_control.state = FCS_IDLE;
+                               goto consider_inactive;
+                       }
+

                        VM_CHECK_MEMORYSTATUS;
 
                        VM_CHECK_MEMORYSTATUS;
 

-                       if (flow_control.state != FCS_IDLE)
-                               vm_pageout_scan_throttle++;
+                       if (flow_control.state != FCS_IDLE) {
+                               VM_PAGEOUT_DEBUG(vm_pageout_scan_throttle, 1);
+                       }
+

                        iq->pgo_throttled = TRUE;
                        iq->pgo_throttled = TRUE;

+                       assert_wait_timeout((event_t) &iq->pgo_laundry, THREAD_INTERRUPTIBLE, msecs, 1000 * NSEC_PER_USEC);

 
 

-                       assert_wait_timeout((event_t) &iq->pgo_laundry, THREAD_INTERRUPTIBLE, msecs, 1000*NSEC_PER_USEC);

                        counter(c_vm_pageout_scan_block++);
 
                        vm_page_unlock_queues();
 
                        assert(vm_pageout_scan_wants_object == VM_OBJECT_NULL);
 
                        counter(c_vm_pageout_scan_block++);
 
                        vm_page_unlock_queues();
 
                        assert(vm_pageout_scan_wants_object == VM_OBJECT_NULL);
 

-                       VM_DEBUG_EVENT(vm_pageout_thread_block, VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_START, 
-                                      iq->pgo_laundry, iq->pgo_maxlaundry, msecs, 0);
+                       VM_DEBUG_EVENT(vm_pageout_thread_block, VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_START,
+                           iq->pgo_laundry, iq->pgo_maxlaundry, msecs, 0);

                        memoryshot(VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_START);
 
                        thread_block(THREAD_CONTINUE_NULL);
 
                        VM_DEBUG_EVENT(vm_pageout_thread_block, VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_END,
                        memoryshot(VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_START);
 
                        thread_block(THREAD_CONTINUE_NULL);
 
                        VM_DEBUG_EVENT(vm_pageout_thread_block, VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_END,

-                                      iq->pgo_laundry, iq->pgo_maxlaundry, msecs, 0);
+                           iq->pgo_laundry, iq->pgo_maxlaundry, msecs, 0);

                        memoryshot(VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_END);
 
                        vm_page_lock_queues();
 
                        iq->pgo_throttled = FALSE;
 
                        memoryshot(VM_PAGEOUT_THREAD_BLOCK, DBG_FUNC_END);
 
                        vm_page_lock_queues();
 
                        iq->pgo_throttled = FALSE;
 

-                       if (loop_count >= vm_page_inactive_count)
+                       if (loop_count >= vm_page_inactive_count) {

                                loop_count = 0;
                                loop_count = 0;

+                       }

                        inactive_burst_count = 0;
 
                        goto Restart;
                        inactive_burst_count = 0;
 
                        goto Restart;


@@ -2561,26 +2429,23 @@ vm_pageout_scan_delay:
 
                flow_control.state = FCS_IDLE;
 consider_inactive:
 
                flow_control.state = FCS_IDLE;
 consider_inactive:

-               vm_pageout_inactive_external_forced_reactivate_limit = MIN((vm_page_active_count + vm_page_inactive_count), 
-                                                                           vm_pageout_inactive_external_forced_reactivate_limit);
+               vm_pageout_inactive_external_forced_reactivate_limit = MIN((vm_page_active_count + vm_page_inactive_count),
+                   vm_pageout_inactive_external_forced_reactivate_limit);

                loop_count++;
                inactive_burst_count++;
                loop_count++;
                inactive_burst_count++;

-               vm_pageout_inactive++;
-
+               vm_pageout_state.vm_pageout_inactive++;

 
                /*
                 * Choose a victim.
                 */
                while (1) {
 
                /*
                 * Choose a victim.
                 */
                while (1) {

-                       uint32_t        inactive_external_count;
-

 #if CONFIG_BACKGROUND_QUEUE
                        page_from_bg_q = FALSE;
 #endif /* CONFIG_BACKGROUND_QUEUE */
 
                        m = NULL;
                        m_object = VM_OBJECT_NULL;
 #if CONFIG_BACKGROUND_QUEUE
                        page_from_bg_q = FALSE;
 #endif /* CONFIG_BACKGROUND_QUEUE */
 
                        m = NULL;
                        m_object = VM_OBJECT_NULL;

-                       
+

                        if (VM_DYNAMIC_PAGING_ENABLED()) {
                                assert(vm_page_throttled_count == 0);
                                assert(vm_page_queue_empty(&vm_page_queue_throttled));
                        if (VM_DYNAMIC_PAGING_ENABLED()) {
                                assert(vm_page_throttled_count == 0);
                                assert(vm_page_queue_empty(&vm_page_queue_throttled));


@@ -2593,9 +2458,9 @@ consider_inactive:
                         */
                        if (!vm_page_queue_empty(&vm_page_queue_cleaned)) {
                                m = (vm_page_t) vm_page_queue_first(&vm_page_queue_cleaned);
                         */
                        if (!vm_page_queue_empty(&vm_page_queue_cleaned)) {
                                m = (vm_page_t) vm_page_queue_first(&vm_page_queue_cleaned);

-                    
-                               assert(m->vm_page_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q);
-                    
+
+                               assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q);
+

                                break;
                        }
 
                                break;
                        }
 


@@ -2606,17 +2471,18 @@ consider_inactive:
                        if (!vm_page_queue_empty(&sq->age_q)) {
                                m = (vm_page_t) vm_page_queue_first(&sq->age_q);
 
                        if (!vm_page_queue_empty(&sq->age_q)) {
                                m = (vm_page_t) vm_page_queue_first(&sq->age_q);
 

-                               assert(m->vm_page_q_state == VM_PAGE_ON_SPECULATIVE_Q);
+                               assert(m->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q);

 
 

-                               if (!m->dirty || force_anonymous == FALSE)
-                                       break;
-                               else
-                                       m = NULL;
+                               if (!m->vmp_dirty || force_anonymous == FALSE) {
+                                       break;
+                               } else {
+                                       m = NULL;
+                               }

                        }
 
 #if CONFIG_BACKGROUND_QUEUE
                        if (vm_page_background_mode != VM_PAGE_BG_DISABLED && (vm_page_background_count > vm_page_background_target)) {
                        }
 
 #if CONFIG_BACKGROUND_QUEUE
                        if (vm_page_background_mode != VM_PAGE_BG_DISABLED && (vm_page_background_count > vm_page_background_target)) {

-                               vm_object_t     bg_m_object = NULL;
+                               vm_object_t     bg_m_object = NULL;

 
                                m = (vm_page_t) vm_page_queue_first(&vm_page_queue_background);
 
 
                                m = (vm_page_t) vm_page_queue_first(&vm_page_queue_background);
 


@@ -2634,37 +2500,36 @@ consider_inactive:
                                         * page.
                                         */
                                } else if (force_anonymous == FALSE || bg_m_object->internal) {
                                         * page.
                                         */
                                } else if (force_anonymous == FALSE || bg_m_object->internal) {

-

                                        if (bg_m_object->internal &&
                                        if (bg_m_object->internal &&

-                                           ((vm_compressor_out_of_space() == TRUE) ||
-                                            (vm_page_free_count < (vm_page_free_reserved / 4)))) {
-                                               
+                                           (VM_PAGE_Q_THROTTLED(iq) ||
+                                           vm_compressor_out_of_space() == TRUE ||
+                                           vm_page_free_count < (vm_page_free_reserved / 4))) {

                                                vm_pageout_skipped_bq_internal++;
                                        } else {
                                                page_from_bg_q = TRUE;
                                                vm_pageout_skipped_bq_internal++;
                                        } else {
                                                page_from_bg_q = TRUE;

-                               
-                                               if (bg_m_object->internal)
-                                                       vm_pageout_considered_bq_internal++;
-                                               else
-                                                       vm_pageout_considered_bq_external++;
-                                       
+
+                                               if (bg_m_object->internal) {
+                                                       vm_pageout_vminfo.vm_pageout_considered_bq_internal++;
+                                               } else {
+                                                       vm_pageout_vminfo.vm_pageout_considered_bq_external++;
+                                               }

                                                break;
                                        }
                                }
                        }
 #endif
                                                break;
                                        }
                                }
                        }
 #endif

-
-                       grab_anonymous = (vm_page_anonymous_count > vm_page_anonymous_min);

                        inactive_external_count = vm_page_inactive_count - vm_page_anonymous_count;
 
                        inactive_external_count = vm_page_inactive_count - vm_page_anonymous_count;
 

-                       if ((vm_page_pageable_external_count < vm_page_filecache_min || force_anonymous == TRUE) ||
-                           ((inactive_external_count < vm_page_anonymous_count) && (inactive_external_count < (vm_page_pageable_external_count / 3)))) {
+                       if ((vm_page_pageable_external_count < vm_pageout_state.vm_page_filecache_min || force_anonymous == TRUE) ||
+                           (inactive_external_count < VM_PAGE_INACTIVE_TARGET(vm_page_pageable_external_count))) {

                                grab_anonymous = TRUE;
                                anons_grabbed = 0;
 
                                grab_anonymous = TRUE;
                                anons_grabbed = 0;
 

-                               vm_pageout_skipped_external++;
+                               vm_pageout_vminfo.vm_pageout_skipped_external++;

                                goto want_anonymous;
                        }
                                goto want_anonymous;
                        }

+                       grab_anonymous = (vm_page_anonymous_count > vm_page_anonymous_min);
+

 #if CONFIG_JETSAM
                        /* If the file-backed pool has accumulated
                         * significantly more pages than the jetsam
 #if CONFIG_JETSAM
                        /* If the file-backed pool has accumulated
                         * significantly more pages than the jetsam


@@ -2677,50 +2542,50 @@ consider_inactive:
                         */
                        if (grab_anonymous == TRUE && !VM_PAGE_Q_THROTTLED(eq)) {
                                if (vm_page_pageable_external_count >
                         */
                        if (grab_anonymous == TRUE && !VM_PAGE_Q_THROTTLED(eq)) {
                                if (vm_page_pageable_external_count >

-                                   vm_page_filecache_min) {
+                                   vm_pageout_state.vm_page_filecache_min) {

                                        if ((vm_page_pageable_external_count *
                                        if ((vm_page_pageable_external_count *

-                                               vm_pageout_memorystatus_fb_factor_dr) >
+                                           vm_pageout_memorystatus_fb_factor_dr) >

                                            (memorystatus_available_pages_critical *
                                            vm_pageout_memorystatus_fb_factor_nr)) {
                                                grab_anonymous = FALSE;
                                            (memorystatus_available_pages_critical *
                                            vm_pageout_memorystatus_fb_factor_nr)) {
                                                grab_anonymous = FALSE;

-#if DEVELOPMENT || DEBUG
-                                               vm_grab_anon_overrides++;
-#endif
+
+                                               VM_PAGEOUT_DEBUG(vm_grab_anon_overrides, 1);

                                        }
                                }
                                        }
                                }

-#if DEVELOPMENT || DEBUG

                                if (grab_anonymous) {
                                if (grab_anonymous) {

-                                       vm_grab_anon_nops++;
+                                       VM_PAGEOUT_DEBUG(vm_grab_anon_nops, 1);

                                }
                                }

-#endif

                        }
 #endif /* CONFIG_JETSAM */
 
 want_anonymous:
                        if (grab_anonymous == FALSE || anons_grabbed >= ANONS_GRABBED_LIMIT || vm_page_queue_empty(&vm_page_queue_anonymous)) {
                        }
 #endif /* CONFIG_JETSAM */
 
 want_anonymous:
                        if (grab_anonymous == FALSE || anons_grabbed >= ANONS_GRABBED_LIMIT || vm_page_queue_empty(&vm_page_queue_anonymous)) {

-
-                               if ( !vm_page_queue_empty(&vm_page_queue_inactive) ) {
+                               if (!vm_page_queue_empty(&vm_page_queue_inactive)) {

                                        m = (vm_page_t) vm_page_queue_first(&vm_page_queue_inactive);
                                        m = (vm_page_t) vm_page_queue_first(&vm_page_queue_inactive);

-                               
-                                       assert(m->vm_page_q_state == VM_PAGE_ON_INACTIVE_EXTERNAL_Q);
+
+                                       assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_EXTERNAL_Q);

                                        anons_grabbed = 0;
 
                                        anons_grabbed = 0;
 

-                                       if (vm_page_pageable_external_count < vm_page_filecache_min) {
-                                               if ((++reactivated_this_call % 100))
-                                                       goto must_activate_page;
-                                               /*
-                                                * steal 1% of the file backed pages even if
-                                                * we are under the limit that has been set
-                                                * for a healthy filecache
-                                                */
+                                       if (vm_page_pageable_external_count < vm_pageout_state.vm_page_filecache_min) {
+                                               if (!vm_page_queue_empty(&vm_page_queue_anonymous)) {
+                                                       if ((++reactivated_this_call % 100)) {
+                                                               vm_pageout_vminfo.vm_pageout_filecache_min_reactivated++;
+                                                               goto must_activate_page;
+                                                       }
+                                                       /*
+                                                        * steal 1% of the file backed pages even if
+                                                        * we are under the limit that has been set
+                                                        * for a healthy filecache
+                                                        */
+                                               }

                                        }
                                        break;
                                }
                        }
                                        }
                                        break;
                                }
                        }

-                       if ( !vm_page_queue_empty(&vm_page_queue_anonymous) ) {
+                       if (!vm_page_queue_empty(&vm_page_queue_anonymous)) {

                                m = (vm_page_t) vm_page_queue_first(&vm_page_queue_anonymous);
 
                                m = (vm_page_t) vm_page_queue_first(&vm_page_queue_anonymous);
 

-                               assert(m->vm_page_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q);
+                               assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q);

                                anons_grabbed++;
 
                                break;
                                anons_grabbed++;
 
                                break;


@@ -2729,33 +2594,31 @@ want_anonymous:
                        /*
                         * if we've gotten here, we have no victim page.
                         * check to see if we've not finished balancing the queues
                        /*
                         * if we've gotten here, we have no victim page.
                         * check to see if we've not finished balancing the queues

-                        * or we have a page on the aged speculative queue that we 
+                        * or we have a page on the aged speculative queue that we

                         * skipped due to force_anonymous == TRUE.. or we have
                         * speculative  pages that we can prematurely age... if
                         * one of these cases we'll keep going, else panic
                         */
                        force_anonymous = FALSE;
                         * skipped due to force_anonymous == TRUE.. or we have
                         * speculative  pages that we can prematurely age... if
                         * one of these cases we'll keep going, else panic
                         */
                        force_anonymous = FALSE;

-                       vm_pageout_no_victim++;
+                       VM_PAGEOUT_DEBUG(vm_pageout_no_victim, 1);

 
 

-                       if ((vm_page_inactive_count + vm_page_speculative_count) < vm_page_inactive_target)
-                               goto done_with_inactivepage;
-
-                       if (!vm_page_queue_empty(&sq->age_q))
+                       if (!vm_page_queue_empty(&sq->age_q)) {

                                goto done_with_inactivepage;
                                goto done_with_inactivepage;

+                       }

 
                        if (vm_page_speculative_count) {
                                force_speculative_aging = TRUE;
                                goto done_with_inactivepage;
                        }
                        panic("vm_pageout: no victim");
 
                        if (vm_page_speculative_count) {
                                force_speculative_aging = TRUE;
                                goto done_with_inactivepage;
                        }
                        panic("vm_pageout: no victim");

-                       
+

                        /* NOTREACHED */
                }
                assert(VM_PAGE_PAGEABLE(m));
                m_object = VM_PAGE_OBJECT(m);
                force_anonymous = FALSE;
                        /* NOTREACHED */
                }
                assert(VM_PAGE_PAGEABLE(m));
                m_object = VM_PAGE_OBJECT(m);
                force_anonymous = FALSE;

-               
-               page_prev_q_state = m->vm_page_q_state;
+
+               page_prev_q_state = m->vmp_q_state;

                /*
                 * we just found this page on one of our queues...
                 * it can't also be on the pageout queue, so safe
                /*
                 * we just found this page on one of our queues...
                 * it can't also be on the pageout queue, so safe


@@ -2763,14 +2626,13 @@ want_anonymous:
                 */
                vm_page_queues_remove(m, TRUE);
 
                 */
                vm_page_queues_remove(m, TRUE);
 

-               assert(!m->laundry);
-               assert(!m->private);
-               assert(!m->fictitious);
+               assert(!m->vmp_laundry);
+               assert(!m->vmp_private);
+               assert(!m->vmp_fictitious);

                assert(m_object != kernel_object);
                assert(VM_PAGE_GET_PHYS_PAGE(m) != vm_page_guard_addr);
 
                assert(m_object != kernel_object);
                assert(VM_PAGE_GET_PHYS_PAGE(m) != vm_page_guard_addr);
 

-               vm_pageout_stats[vm_pageout_stat_now].considered++;
-               vm_pageout_considered_page++;
+               vm_pageout_vminfo.vm_pageout_considered_page++;

 
                DTRACE_VM2(scan, int, 1, (uint64_t *), NULL);
 
 
                DTRACE_VM2(scan, int, 1, (uint64_t *), NULL);
 


@@ -2780,15 +2642,14 @@ want_anonymous:
                 * already got the lock
                 */
                if (m_object != object) {
                 * already got the lock
                 */
                if (m_object != object) {

-                       /*
-                        * the object associated with candidate page is 
+                       /*
+                        * the object associated with candidate page is

                         * different from the one we were just working
                         * with... dump the lock if we still own it
                         */
                         * different from the one we were just working
                         * with... dump the lock if we still own it
                         */

-                       if (object != NULL) {
-                               vm_object_unlock(object);
+                       if (object != NULL) {
+                               vm_object_unlock(object);

                                object = NULL;
                                object = NULL;

-                               vm_pageout_scan_wants_object = VM_OBJECT_NULL;

                        }
                        /*
                         * Try to lock object; since we've alread got the
                        }
                        /*
                         * Try to lock object; since we've alread got the


@@ -2803,93 +2664,112 @@ want_anonymous:
                        if (!vm_object_lock_try_scan(m_object)) {
                                vm_page_t m_want = NULL;
 
                        if (!vm_object_lock_try_scan(m_object)) {
                                vm_page_t m_want = NULL;
 

-                               vm_pageout_inactive_nolock++;
+                               vm_pageout_vminfo.vm_pageout_inactive_nolock++;

 
 

-                               if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q)
-                                       vm_pageout_cleaned_nolock++;
+                               if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) {
+                                       VM_PAGEOUT_DEBUG(vm_pageout_cleaned_nolock, 1);
+                               }

 
                                pmap_clear_reference(VM_PAGE_GET_PHYS_PAGE(m));
 
                                pmap_clear_reference(VM_PAGE_GET_PHYS_PAGE(m));

-                               m->reference = FALSE;
-
-#if !CONFIG_EMBEDDED
-                               /*
-                                * m->object must be stable since we hold the page queues lock...
-                                * we can update the scan_collisions field sans the object lock
-                                * since it is a separate field and this is the only spot that does
-                                * a read-modify-write operation and it is never executed concurrently...
-                                * we can asynchronously set this field to 0 when creating a UPL, so it
-                                * is possible for the value to be a bit non-determistic, but that's ok
-                                * since it's only used as a hint
-                                */

 
 

-                               /*
-                                * This is not used on EMBEDDED because having this variable set *could* lead
-                                * us to self-cannibalize pages from m_object to fill a UPL for a pagein.
-                                * And, there's a high probability that the object that vm_pageout_scan
-                                * wants and collides on is a very popular object e.g. the shared cache on EMBEDDED.
-                                * The older pages that we cannibalize from the shared cache could be really
-                                * important text pages e.g. the system call stubs.
-                                */
-                               m_object->scan_collisions = 1;
-#endif /* !CONFIG_EMBEDDED */
+                               m->vmp_reference = FALSE;

 
 

-                               if ( !vm_page_queue_empty(&sq->age_q) )
-                                       m_want = (vm_page_t) vm_page_queue_first(&sq->age_q);
-                               else if ( !vm_page_queue_empty(&vm_page_queue_cleaned))
+                               if (!m_object->object_is_shared_cache) {
+                                       /*
+                                        * don't apply this optimization if this is the shared cache
+                                        * object, it's too easy to get rid of very hot and important
+                                        * pages...
+                                        * m->vmp_object must be stable since we hold the page queues lock...
+                                        * we can update the scan_collisions field sans the object lock
+                                        * since it is a separate field and this is the only spot that does
+                                        * a read-modify-write operation and it is never executed concurrently...
+                                        * we can asynchronously set this field to 0 when creating a UPL, so it
+                                        * is possible for the value to be a bit non-determistic, but that's ok
+                                        * since it's only used as a hint
+                                        */
+                                       m_object->scan_collisions = 1;
+                               }
+                               if (!vm_page_queue_empty(&vm_page_queue_cleaned)) {

                                        m_want = (vm_page_t) vm_page_queue_first(&vm_page_queue_cleaned);
                                        m_want = (vm_page_t) vm_page_queue_first(&vm_page_queue_cleaned);

-                               else if ( !vm_page_queue_empty(&vm_page_queue_inactive) &&
-                                         (anons_grabbed >= ANONS_GRABBED_LIMIT || vm_page_queue_empty(&vm_page_queue_anonymous)))
+                               } else if (!vm_page_queue_empty(&sq->age_q)) {
+                                       m_want = (vm_page_t) vm_page_queue_first(&sq->age_q);
+                               } else if ((grab_anonymous == FALSE || anons_grabbed >= ANONS_GRABBED_LIMIT ||
+                                   vm_page_queue_empty(&vm_page_queue_anonymous)) &&
+                                   !vm_page_queue_empty(&vm_page_queue_inactive)) {

                                        m_want = (vm_page_t) vm_page_queue_first(&vm_page_queue_inactive);
                                        m_want = (vm_page_t) vm_page_queue_first(&vm_page_queue_inactive);

-                               else if ( !vm_page_queue_empty(&vm_page_queue_anonymous))
+                               } else if (!vm_page_queue_empty(&vm_page_queue_anonymous)) {

                                        m_want = (vm_page_t) vm_page_queue_first(&vm_page_queue_anonymous);
                                        m_want = (vm_page_t) vm_page_queue_first(&vm_page_queue_anonymous);

+                               }

 
                                /*
                                 * this is the next object we're going to be interested in
 
                                /*
                                 * this is the next object we're going to be interested in

-                                * try to make sure its available after the mutex_yield
+                                * try to make sure its available after the mutex_pause

                                 * returns control
                                 */
                                 * returns control
                                 */

-                               if (m_want)
+                               if (m_want) {

                                        vm_pageout_scan_wants_object = VM_PAGE_OBJECT(m_want);
                                        vm_pageout_scan_wants_object = VM_PAGE_OBJECT(m_want);

-
-                               /*
-                                * force us to dump any collected free pages
-                                * and to pause before moving on
-                                */
-                               try_failed = TRUE;
+                               }

 
                                goto requeue_page;
                        }
                        object = m_object;
                        vm_pageout_scan_wants_object = VM_OBJECT_NULL;
 
                                goto requeue_page;
                        }
                        object = m_object;
                        vm_pageout_scan_wants_object = VM_OBJECT_NULL;

-
-                       try_failed = FALSE;

                }
                assert(m_object == object);
                assert(VM_PAGE_OBJECT(m) == m_object);
 
                }
                assert(m_object == object);
                assert(VM_PAGE_OBJECT(m) == m_object);
 

-               if (m->busy) {
+               if (m->vmp_busy) {

                        /*
                         *      Somebody is already playing with this page.
                         *      Put it back on the appropriate queue
                         *
                         */
                        /*
                         *      Somebody is already playing with this page.
                         *      Put it back on the appropriate queue
                         *
                         */

-                       vm_pageout_inactive_busy++;
+                       VM_PAGEOUT_DEBUG(vm_pageout_inactive_busy, 1);

 
 

-                       if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q)
-                               vm_pageout_cleaned_busy++;
+                       if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) {
+                               VM_PAGEOUT_DEBUG(vm_pageout_cleaned_busy, 1);
+                       }

 requeue_page:
 requeue_page:

-                       if (page_prev_q_state == VM_PAGE_ON_SPECULATIVE_Q)
+                       if (page_prev_q_state == VM_PAGE_ON_SPECULATIVE_Q) {

                                vm_page_enqueue_inactive(m, FALSE);
                                vm_page_enqueue_inactive(m, FALSE);

-                       else
+                       } else {

                                vm_page_activate(m);
                                vm_page_activate(m);

+                       }

 #if CONFIG_BACKGROUND_QUEUE
 #if CONFIG_BACKGROUND_QUEUE

+#if DEVELOPMENT || DEBUG

                        if (page_from_bg_q == TRUE) {
                        if (page_from_bg_q == TRUE) {

-                               if (m_object->internal)
+                               if (m_object->internal) {

                                        vm_pageout_rejected_bq_internal++;
                                        vm_pageout_rejected_bq_internal++;

-                               else
+                               } else {

                                        vm_pageout_rejected_bq_external++;
                                        vm_pageout_rejected_bq_external++;

+                               }

                        }
 #endif
                        }
 #endif

+#endif
+                       goto done_with_inactivepage;
+               }
+
+               /*
+                *   if (m->vmp_cleaning && !m->vmp_free_when_done)
+                *      If already cleaning this page in place
+                *      just leave if off the paging queues.
+                *      We can leave the page mapped, and upl_commit_range
+                *      will put it on the clean queue.
+                *
+                *   if (m->vmp_free_when_done && !m->vmp_cleaning)
+                *      an msync INVALIDATE is in progress...
+                *      this page has been marked for destruction
+                *      after it has been cleaned,
+                *      but not yet gathered into a UPL
+                *      where 'cleaning' will be set...
+                *      just leave it off the paging queues
+                *
+                *   if (m->vmp_free_when_done && m->vmp_clenaing)
+                *      an msync INVALIDATE is in progress
+                *      and the UPL has already gathered this page...
+                *      just leave it off the paging queues
+                */
+               if (m->vmp_free_when_done || m->vmp_cleaning) {

                        goto done_with_inactivepage;
                }
 
                        goto done_with_inactivepage;
                }
 


@@ -2901,18 +2781,18 @@ requeue_page:
                 *      from reclaiming it - busy or cleaning - that we've already
                 *      dealt with
                 */
                 *      from reclaiming it - busy or cleaning - that we've already
                 *      dealt with
                 */

-               if (m->absent || m->error || !object->alive) {
-
-                       if (m->absent)
-                               vm_pageout_inactive_absent++;
-                       else if (!object->alive)
-                               vm_pageout_inactive_notalive++;
-                       else
-                               vm_pageout_inactive_error++;
-reclaim_page:                  
+               if (m->vmp_absent || m->vmp_error || !object->alive) {
+                       if (m->vmp_absent) {
+                               VM_PAGEOUT_DEBUG(vm_pageout_inactive_absent, 1);
+                       } else if (!object->alive) {
+                               VM_PAGEOUT_DEBUG(vm_pageout_inactive_notalive, 1);
+                       } else {
+                               VM_PAGEOUT_DEBUG(vm_pageout_inactive_error, 1);
+                       }
+reclaim_page:

                        if (vm_pageout_deadlock_target) {
                        if (vm_pageout_deadlock_target) {

-                               vm_pageout_scan_inactive_throttle_success++;
-                               vm_pageout_deadlock_target--;
+                               VM_PAGEOUT_DEBUG(vm_pageout_scan_inactive_throttle_success, 1);
+                               vm_pageout_deadlock_target--;

                        }
 
                        DTRACE_VM2(dfree, int, 1, (uint64_t *), NULL);
                        }
 
                        DTRACE_VM2(dfree, int, 1, (uint64_t *), NULL);


@@ -2922,10 +2802,16 @@ reclaim_page:
                        } else {
                                DTRACE_VM2(fsfree, int, 1, (uint64_t *), NULL);
                        }
                        } else {
                                DTRACE_VM2(fsfree, int, 1, (uint64_t *), NULL);
                        }

-                       assert(!m->cleaning);
-                       assert(!m->laundry);
+                       assert(!m->vmp_cleaning);
+                       assert(!m->vmp_laundry);
+
+                       if (!object->internal &&
+                           object->pager != NULL &&
+                           object->pager->mo_pager_ops == &shared_region_pager_ops) {
+                               shared_region_pager_reclaimed++;
+                       }

 
 

-                       m->busy = TRUE;
+                       m->vmp_busy = TRUE;

 
                        /*
                         * remove page from object here since we're already
 
                        /*
                         * remove page from object here since we're already


@@ -2933,42 +2819,45 @@ reclaim_page:
                         * we'd normally do in vm_page_free_prepare_object
                         * until 'vm_page_free_list' is called
                         */
                         * we'd normally do in vm_page_free_prepare_object
                         * until 'vm_page_free_list' is called
                         */

-                       if (m->tabled)
+                       if (m->vmp_tabled) {

                                vm_page_remove(m, TRUE);
                                vm_page_remove(m, TRUE);

+                       }

 
 

-                       assert(m->pageq.next == 0 && m->pageq.prev == 0);
-                       m->snext = local_freeq;
+                       assert(m->vmp_pageq.next == 0 && m->vmp_pageq.prev == 0);
+                       m->vmp_snext = local_freeq;

                        local_freeq = m;
                        local_freed++;
                        local_freeq = m;
                        local_freed++;

-                       
-                       if (page_prev_q_state == VM_PAGE_ON_SPECULATIVE_Q)
-                               vm_pageout_freed_from_speculative++;
-                       else if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q)
-                               vm_pageout_freed_from_cleaned++;
-                       else
-                               vm_pageout_freed_from_inactive_clean++;

 
 

-                       vm_pageout_stats[vm_pageout_stat_now].reclaimed_clean++;
+                       if (page_prev_q_state == VM_PAGE_ON_SPECULATIVE_Q) {
+                               vm_pageout_vminfo.vm_pageout_freed_speculative++;
+                       } else if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) {
+                               vm_pageout_vminfo.vm_pageout_freed_cleaned++;
+                       } else if (page_prev_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q) {
+                               vm_pageout_vminfo.vm_pageout_freed_internal++;
+                       } else {
+                               vm_pageout_vminfo.vm_pageout_freed_external++;
+                       }

 
                        inactive_burst_count = 0;
                        goto done_with_inactivepage;
                }
 
                        inactive_burst_count = 0;
                        goto done_with_inactivepage;
                }

-               /*
-                * If the object is empty, the page must be reclaimed even
-                * if dirty or used.
-                * If the page belongs to a volatile object, we stick it back
-                * on.
-                */

                if (object->copy == VM_OBJECT_NULL) {
                if (object->copy == VM_OBJECT_NULL) {

+                       /*
+                        * No one else can have any interest in this page.
+                        * If this is an empty purgable object, the page can be
+                        * reclaimed even if dirty.
+                        * If the page belongs to a volatile purgable object, we
+                        * reactivate it if the compressor isn't active.
+                        */

                        if (object->purgable == VM_PURGABLE_EMPTY) {
                        if (object->purgable == VM_PURGABLE_EMPTY) {

-                               if (m->pmapped == TRUE) {
+                               if (m->vmp_pmapped == TRUE) {

                                        /* unmap the page */
                                        refmod_state = pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
                                        if (refmod_state & VM_MEM_MODIFIED) {
                                                SET_PAGE_DIRTY(m, FALSE);
                                        }
                                }
                                        /* unmap the page */
                                        refmod_state = pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
                                        if (refmod_state & VM_MEM_MODIFIED) {
                                                SET_PAGE_DIRTY(m, FALSE);
                                        }
                                }

-                               if (m->dirty || m->precious) {
+                               if (m->vmp_dirty || m->vmp_precious) {

                                        /* we saved the cost of cleaning this page ! */
                                        vm_page_purged_count++;
                                }
                                        /* we saved the cost of cleaning this page ! */
                                        vm_page_purged_count++;
                                }


@@ -2996,8 +2885,9 @@ reclaim_page:
                                /* just stick it back on! */
                                reactivated_this_call++;
 
                                /* just stick it back on! */
                                reactivated_this_call++;
 

-                               if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q)
-                                       vm_pageout_cleaned_volatile_reactivated++;
+                               if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) {
+                                       VM_PAGEOUT_DEBUG(vm_pageout_cleaned_volatile_reactivated, 1);
+                               }

 
                                goto reactivate_page;
                        }
 
                                goto reactivate_page;
                        }


@@ -3010,66 +2900,35 @@ reclaim_page:
                 */
                refmod_state = -1;
 
                 */
                refmod_state = -1;
 

-               if (m->reference == FALSE && m->pmapped == TRUE) {
-                       refmod_state = pmap_get_refmod(VM_PAGE_GET_PHYS_PAGE(m));
-                 
-                       if (refmod_state & VM_MEM_REFERENCED)
-                               m->reference = TRUE;
-                       if (refmod_state & VM_MEM_MODIFIED) {
+               if (m->vmp_reference == FALSE && m->vmp_pmapped == TRUE) {
+                       refmod_state = pmap_get_refmod(VM_PAGE_GET_PHYS_PAGE(m));
+
+                       if (refmod_state & VM_MEM_REFERENCED) {
+                               m->vmp_reference = TRUE;
+                       }
+                       if (refmod_state & VM_MEM_MODIFIED) {

                                SET_PAGE_DIRTY(m, FALSE);
                        }
                }
                                SET_PAGE_DIRTY(m, FALSE);
                        }
                }

-               
-               /*
-                *   if (m->cleaning && !m->free_when_done)
-                *      If already cleaning this page in place and it hasn't
-                *      been recently referenced, just pull off the queue.
-                *      We can leave the page mapped, and upl_commit_range
-                *      will put it on the clean queue.
-                *
-                *   if (m->free_when_done && !m->cleaning)
-                *      an msync INVALIDATE is in progress...
-                *      this page has been marked for destruction
-                *      after it has been cleaned,
-                *      but not yet gathered into a UPL
-                *      where 'cleaning' will be set...
-                *      just leave it off the paging queues
-                *
-                *   if (m->free_when_done && m->clenaing)
-                *      an msync INVALIDATE is in progress
-                *      and the UPL has already gathered this page...
-                *      just leave it off the paging queues
-                */
-               
-               /*
-                * page with m->free_when_done and still on the queues means that an
-                * MS_INVALIDATE is in progress on this page... leave it alone
-                */
-               if (m->free_when_done) {
-                       goto done_with_inactivepage;
-               }
-               
-               /* if cleaning, reactivate if referenced.  otherwise, just pull off queue */
-               if (m->cleaning) {
-                       if (m->reference == TRUE) {
-                               reactivated_this_call++;
-                               goto reactivate_page;
-                       } else {
-                               goto done_with_inactivepage;
-                       }
+
+               if (m->vmp_reference || m->vmp_dirty) {
+                       /* deal with a rogue "reusable" page */
+                       VM_PAGEOUT_SCAN_HANDLE_REUSABLE_PAGE(m, m_object);

                }
                }

+               divisor = vm_pageout_state.vm_page_xpmapped_min_divisor;

 
 

-                if (m->reference || m->dirty) {
-                        /* deal with a rogue "reusable" page */
-                        VM_PAGEOUT_SCAN_HANDLE_REUSABLE_PAGE(m, m_object);
-                }
+               if (divisor == 0) {
+                       vm_pageout_state.vm_page_xpmapped_min = 0;
+               } else {
+                       vm_pageout_state.vm_page_xpmapped_min = (vm_page_external_count * 10) / divisor;
+               }

 
 

-               if (!m->no_cache &&
+               if (!m->vmp_no_cache &&

 #if CONFIG_BACKGROUND_QUEUE
                    page_from_bg_q == FALSE &&
 #endif
 #if CONFIG_BACKGROUND_QUEUE
                    page_from_bg_q == FALSE &&
 #endif

-                   (m->reference ||
-                    (m->xpmapped && !object->internal && (vm_page_xpmapped_external_count < (vm_page_external_count / 4))))) {
+                   (m->vmp_reference || (m->vmp_xpmapped && !object->internal &&
+                   (vm_page_xpmapped_external_count < vm_pageout_state.vm_page_xpmapped_min)))) {

                        /*
                         * The page we pulled off the inactive list has
                         * been referenced.  It is possible for other
                        /*
                         * The page we pulled off the inactive list has
                         * been referenced.  It is possible for other


@@ -3079,23 +2938,26 @@ reclaim_page:
                         * reactivations.
                         */
                        if (++reactivated_this_call >= reactivate_limit) {
                         * reactivations.
                         */
                        if (++reactivated_this_call >= reactivate_limit) {

-                               vm_pageout_reactivation_limit_exceeded++;
+                               vm_pageout_vminfo.vm_pageout_reactivation_limit_exceeded++;

                        } else if (++inactive_reclaim_run >= VM_PAGEOUT_INACTIVE_FORCE_RECLAIM) {
                        } else if (++inactive_reclaim_run >= VM_PAGEOUT_INACTIVE_FORCE_RECLAIM) {

-                               vm_pageout_inactive_force_reclaim++;
+                               vm_pageout_vminfo.vm_pageout_inactive_force_reclaim++;

                        } else {
                                uint32_t isinuse;
 
                        } else {
                                uint32_t isinuse;
 

-                               if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q)
-                                       vm_pageout_cleaned_reference_reactivated++;
+                               if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) {
+                                       VM_PAGEOUT_DEBUG(vm_pageout_cleaned_reference_reactivated, 1);
+                               }
+
+                               vm_pageout_vminfo.vm_pageout_inactive_referenced++;

 reactivate_page:
 reactivate_page:

-                               if ( !object->internal && object->pager != MEMORY_OBJECT_NULL &&
-                                    vnode_pager_get_isinuse(object->pager, &isinuse) == KERN_SUCCESS && !isinuse) {
+                               if (!object->internal && object->pager != MEMORY_OBJECT_NULL &&
+                                   vnode_pager_get_isinuse(object->pager, &isinuse) == KERN_SUCCESS && !isinuse) {

                                        /*
                                         * no explict mappings of this object exist
                                         * and it's not open via the filesystem
                                         */
                                        vm_page_deactivate(m);
                                        /*
                                         * no explict mappings of this object exist
                                         * and it's not open via the filesystem
                                         */
                                        vm_page_deactivate(m);

-                                       vm_pageout_inactive_deactivated++;
+                                       VM_PAGEOUT_DEBUG(vm_pageout_inactive_deactivated, 1);

                                } else {
 must_activate_page:
                                        /*
                                } else {
 must_activate_page:
                                        /*


@@ -3106,25 +2968,29 @@ must_activate_page:
                                        inactive_burst_count = 0;
                                }
 #if CONFIG_BACKGROUND_QUEUE
                                        inactive_burst_count = 0;
                                }
 #if CONFIG_BACKGROUND_QUEUE

+#if DEVELOPMENT || DEBUG

                                if (page_from_bg_q == TRUE) {
                                if (page_from_bg_q == TRUE) {

-                                       if (m_object->internal)
+                                       if (m_object->internal) {

                                                vm_pageout_rejected_bq_internal++;
                                                vm_pageout_rejected_bq_internal++;

-                                       else
+                                       } else {

                                                vm_pageout_rejected_bq_external++;
                                                vm_pageout_rejected_bq_external++;

+                                       }

                                }
 #endif
                                }
 #endif

-                               if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q)
-                                       vm_pageout_cleaned_reactivated++;
-                               vm_pageout_inactive_used++;
+#endif
+                               if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) {
+                                       VM_PAGEOUT_DEBUG(vm_pageout_cleaned_reactivated, 1);
+                               }
+                               vm_pageout_state.vm_pageout_inactive_used++;

 
 

-                                goto done_with_inactivepage;
+                               goto done_with_inactivepage;

                        }
                        }

-                       /* 
+                       /*

                         * Make sure we call pmap_get_refmod() if it
                         * wasn't already called just above, to update
                         * the dirty bit.
                         */
                         * Make sure we call pmap_get_refmod() if it
                         * wasn't already called just above, to update
                         * the dirty bit.
                         */

-                       if ((refmod_state == -1) && !m->dirty && m->pmapped) {
+                       if ((refmod_state == -1) && !m->vmp_dirty && m->vmp_pmapped) {

                                refmod_state = pmap_get_refmod(VM_PAGE_GET_PHYS_PAGE(m));
                                if (refmod_state & VM_MEM_MODIFIED) {
                                        SET_PAGE_DIRTY(m, FALSE);
                                refmod_state = pmap_get_refmod(VM_PAGE_GET_PHYS_PAGE(m));
                                if (refmod_state & VM_MEM_MODIFIED) {
                                        SET_PAGE_DIRTY(m, FALSE);


@@ -3132,20 +2998,20 @@ must_activate_page:
                        }
                }
 
                        }
                }
 

-                XPR(XPR_VM_PAGEOUT,
-                "vm_pageout_scan, replace object 0x%X offset 0x%X page 0x%X\n",
-                object, m->offset, m, 0,0);
+               XPR(XPR_VM_PAGEOUT,
+                   "vm_pageout_scan, replace object 0x%X offset 0x%X page 0x%X\n",
+                   object, m->vmp_offset, m, 0, 0);

 
                /*
                 * we've got a candidate page to steal...
                 *
 
                /*
                 * we've got a candidate page to steal...
                 *

-                * m->dirty is up to date courtesy of the
-                * preceding check for m->reference... if 
-                * we get here, then m->reference had to be
+                * m->vmp_dirty is up to date courtesy of the
+                * preceding check for m->vmp_reference... if
+                * we get here, then m->vmp_reference had to be

                 * FALSE (or possibly "reactivate_limit" was
                 * FALSE (or possibly "reactivate_limit" was

-                 * exceeded), but in either case we called
-                 * pmap_get_refmod() and updated both
-                 * m->reference and m->dirty
+                * exceeded), but in either case we called
+                * pmap_get_refmod() and updated both
+                * m->vmp_reference and m->vmp_dirty

                 *
                 * if it's dirty or precious we need to
                 * see if the target queue is throtttled
                 *
                 * if it's dirty or precious we need to
                 * see if the target queue is throtttled


@@ -3155,36 +3021,35 @@ must_activate_page:
 
                inactive_throttled = FALSE;
 
 
                inactive_throttled = FALSE;
 

-               if (m->dirty || m->precious) {
-                       if (object->internal) {
-                               if (VM_PAGE_Q_THROTTLED(iq))
-                                       inactive_throttled = TRUE;
+               if (m->vmp_dirty || m->vmp_precious) {
+                       if (object->internal) {
+                               if (VM_PAGE_Q_THROTTLED(iq)) {
+                                       inactive_throttled = TRUE;
+                               }

                        } else if (VM_PAGE_Q_THROTTLED(eq)) {
                                inactive_throttled = TRUE;
                        }
                }
 throttle_inactive:
                if (!VM_DYNAMIC_PAGING_ENABLED() &&
                        } else if (VM_PAGE_Q_THROTTLED(eq)) {
                                inactive_throttled = TRUE;
                        }
                }
 throttle_inactive:
                if (!VM_DYNAMIC_PAGING_ENABLED() &&

-                   object->internal && m->dirty &&
+                   object->internal && m->vmp_dirty &&

                    (object->purgable == VM_PURGABLE_DENY ||
                    (object->purgable == VM_PURGABLE_DENY ||

-                    object->purgable == VM_PURGABLE_NONVOLATILE ||
-                    object->purgable == VM_PURGABLE_VOLATILE)) {
+                   object->purgable == VM_PURGABLE_NONVOLATILE ||
+                   object->purgable == VM_PURGABLE_VOLATILE)) {

                        vm_page_check_pageable_safe(m);
                        vm_page_check_pageable_safe(m);

-                       assert(m->vm_page_q_state == VM_PAGE_NOT_ON_Q);
-                       vm_page_queue_enter(&vm_page_queue_throttled, m,
-                                           vm_page_t, pageq);
-                       m->vm_page_q_state = VM_PAGE_ON_THROTTLED_Q;
+                       assert(m->vmp_q_state == VM_PAGE_NOT_ON_Q);
+                       vm_page_queue_enter(&vm_page_queue_throttled, m, vmp_pageq);
+                       m->vmp_q_state = VM_PAGE_ON_THROTTLED_Q;

                        vm_page_throttled_count++;
 
                        vm_page_throttled_count++;
 

-                       vm_pageout_scan_reclaimed_throttled++;
+                       VM_PAGEOUT_DEBUG(vm_pageout_scan_reclaimed_throttled, 1);

 
                        inactive_burst_count = 0;
                        goto done_with_inactivepage;
                }
                if (inactive_throttled == TRUE) {
 
                        inactive_burst_count = 0;
                        goto done_with_inactivepage;
                }
                if (inactive_throttled == TRUE) {

-

                        if (object->internal == FALSE) {
                        if (object->internal == FALSE) {

-                                /*
+                               /*

                                 * we need to break up the following potential deadlock case...
                                 *  a) The external pageout thread is stuck on the truncate lock for a file that is being extended i.e. written.
                                 *  b) The thread doing the writing is waiting for pages while holding the truncate lock
                                 * we need to break up the following potential deadlock case...
                                 *  a) The external pageout thread is stuck on the truncate lock for a file that is being extended i.e. written.
                                 *  b) The thread doing the writing is waiting for pages while holding the truncate lock


@@ -3209,12 +3074,12 @@ throttle_inactive:
                                 * that we can try to find clean pages in the active/inactive queues before
                                 * deciding to jetsam a process
                                 */
                                 * that we can try to find clean pages in the active/inactive queues before
                                 * deciding to jetsam a process
                                 */

-                               vm_pageout_scan_inactive_throttled_external++;
+                               vm_pageout_vminfo.vm_pageout_scan_inactive_throttled_external++;

 
                                vm_page_check_pageable_safe(m);
 
                                vm_page_check_pageable_safe(m);

-                               assert(m->vm_page_q_state == VM_PAGE_NOT_ON_Q);
-                               vm_page_queue_enter(&vm_page_queue_active, m, vm_page_t, pageq);
-                               m->vm_page_q_state = VM_PAGE_ON_ACTIVE_Q;
+                               assert(m->vmp_q_state == VM_PAGE_NOT_ON_Q);
+                               vm_page_queue_enter(&vm_page_queue_active, m, vmp_pageq);
+                               m->vmp_q_state = VM_PAGE_ON_ACTIVE_Q;

                                vm_page_active_count++;
                                vm_page_pageable_external_count++;
 
                                vm_page_active_count++;
                                vm_page_pageable_external_count++;
 


@@ -3234,17 +3099,17 @@ throttle_inactive:
                                        vm_page_unlock_queues();
 
                                        VM_DEBUG_CONSTANT_EVENT(vm_pageout_jetsam, VM_PAGEOUT_JETSAM, DBG_FUNC_START,
                                        vm_page_unlock_queues();
 
                                        VM_DEBUG_CONSTANT_EVENT(vm_pageout_jetsam, VM_PAGEOUT_JETSAM, DBG_FUNC_START,

-                                              vm_page_active_count, vm_page_inactive_count, vm_page_free_count, vm_page_free_count);
+                                           vm_page_active_count, vm_page_inactive_count, vm_page_free_count, vm_page_free_count);

 
 

-                                        /* Kill first suitable process */
-                                       if (memorystatus_kill_on_VM_page_shortage(FALSE) == FALSE) {
-                                               panic("vm_pageout_scan: Jetsam request failed\n");      
+                                       /* Kill first suitable process. If this call returned FALSE, we might have simply purged a process instead. */
+                                       if (memorystatus_kill_on_VM_page_shortage(FALSE) == TRUE) {
+                                               VM_PAGEOUT_DEBUG(vm_pageout_inactive_external_forced_jetsam_count, 1);

                                        }
                                        }

-                                       
-                                       VM_DEBUG_CONSTANT_EVENT(vm_pageout_jetsam, VM_PAGEOUT_JETSAM, DBG_FUNC_END, 0, 0, 0, 0);

 
 

-                                       vm_pageout_inactive_external_forced_jetsam_count++;
-                                       vm_page_lock_queues();  
+                                       VM_DEBUG_CONSTANT_EVENT(vm_pageout_jetsam, VM_PAGEOUT_JETSAM, DBG_FUNC_END,
+                                           vm_page_active_count, vm_page_inactive_count, vm_page_free_count, vm_page_free_count);
+
+                                       vm_page_lock_queues();

                                        delayed_unlock = 1;
                                }
 #else /* CONFIG_MEMORYSTATUS && CONFIG_JETSAM */
                                        delayed_unlock = 1;
                                }
 #else /* CONFIG_MEMORYSTATUS && CONFIG_JETSAM */


@@ -3253,7 +3118,6 @@ throttle_inactive:
                                inactive_burst_count = 0;
                                goto done_with_inactivepage;
                        } else {
                                inactive_burst_count = 0;
                                goto done_with_inactivepage;
                        } else {

-                               vm_pageout_scan_inactive_throttled_internal++;

                                goto must_activate_page;
                        }
                }
                                goto must_activate_page;
                        }
                }


@@ -3264,17 +3128,17 @@ throttle_inactive:
                 * we have the up-to-date modified state
                 *
                 * if we need to do a pmap_disconnect then we
                 * we have the up-to-date modified state
                 *
                 * if we need to do a pmap_disconnect then we

-                * need to re-evaluate m->dirty since the pmap_disconnect
-                * provides the true state atomically... the 
+                * need to re-evaluate m->vmp_dirty since the pmap_disconnect
+                * provides the true state atomically... the

                 * page was still mapped up to the pmap_disconnect
                 * and may have been dirtied at the last microsecond
                 *
                 * Note that if 'pmapped' is FALSE then the page is not
                 * and has not been in any map, so there is no point calling
                 * page was still mapped up to the pmap_disconnect
                 * and may have been dirtied at the last microsecond
                 *
                 * Note that if 'pmapped' is FALSE then the page is not
                 * and has not been in any map, so there is no point calling

-                * pmap_disconnect().  m->dirty could have been set in anticipation
+                * pmap_disconnect().  m->vmp_dirty could have been set in anticipation

                 * of likely usage of the page.
                 */
                 * of likely usage of the page.
                 */

-               if (m->pmapped == TRUE) {
+               if (m->vmp_pmapped == TRUE) {

                        int pmap_options;
 
                        /*
                        int pmap_options;
 
                        /*


@@ -3288,10 +3152,10 @@ throttle_inactive:
                         * 3) This page belongs to a file and hence will not be
                         *    sent into the compressor
                         */
                         * 3) This page belongs to a file and hence will not be
                         *    sent into the compressor
                         */

-                       if ( !VM_CONFIG_COMPRESSOR_IS_ACTIVE ||
+                       if (!VM_CONFIG_COMPRESSOR_IS_ACTIVE ||

                            object->internal == FALSE) {
                                pmap_options = 0;
                            object->internal == FALSE) {
                                pmap_options = 0;

-                       } else if (m->dirty || m->precious) {
+                       } else if (m->vmp_dirty || m->vmp_precious) {

                                /*
                                 * VM knows that this page is dirty (or
                                 * precious) and needs to be compressed
                                /*
                                 * VM knows that this page is dirty (or
                                 * precious) and needs to be compressed


@@ -3310,17 +3174,18 @@ throttle_inactive:
                                 * modified.
                                 */
                                pmap_options =
                                 * modified.
                                 */
                                pmap_options =

-                                       PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED;
+                                   PMAP_OPTIONS_COMPRESSOR_IFF_MODIFIED;

                        }
                        refmod_state = pmap_disconnect_options(VM_PAGE_GET_PHYS_PAGE(m),
                        }
                        refmod_state = pmap_disconnect_options(VM_PAGE_GET_PHYS_PAGE(m),

-                                                              pmap_options,
-                                                              NULL);
+                           pmap_options,
+                           NULL);

                        if (refmod_state & VM_MEM_MODIFIED) {
                                SET_PAGE_DIRTY(m, FALSE);
                        }
                }
                        if (refmod_state & VM_MEM_MODIFIED) {
                                SET_PAGE_DIRTY(m, FALSE);
                        }
                }

+

                /*
                /*

-                * reset our count of pages that have been reclaimed 
+                * reset our count of pages that have been reclaimed

                 * since the last page was 'stolen'
                 */
                inactive_reclaim_run = 0;
                 * since the last page was 'stolen'
                 */
                inactive_reclaim_run = 0;


@@ -3328,24 +3193,16 @@ throttle_inactive:
                /*
                 *      If it's clean and not precious, we can free the page.
                 */
                /*
                 *      If it's clean and not precious, we can free the page.
                 */

-               if (!m->dirty && !m->precious) {
-
-                       if (page_prev_q_state == VM_PAGE_ON_SPECULATIVE_Q)
-                               vm_pageout_speculative_clean++;
-                       else {
-                               if (page_prev_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q)
-                                       vm_pageout_inactive_anonymous++;
-                               else if (page_prev_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q)
-                                       vm_pageout_cleaned_reclaimed++;
+               if (!m->vmp_dirty && !m->vmp_precious) {
+                       vm_pageout_state.vm_pageout_inactive_clean++;

 
 

-                               vm_pageout_inactive_clean++;
-                       }

                        /*
                         * OK, at this point we have found a page we are going to free.
                         */
 #if CONFIG_PHANTOM_CACHE
                        /*
                         * OK, at this point we have found a page we are going to free.
                         */
 #if CONFIG_PHANTOM_CACHE

-                       if (!object->internal)
+                       if (!object->internal) {

                                vm_phantom_cache_add_ghost(m);
                                vm_phantom_cache_add_ghost(m);

+                       }

 #endif
                        goto reclaim_page;
                }
 #endif
                        goto reclaim_page;
                }


@@ -3357,15 +3214,17 @@ throttle_inactive:
                 * disconnected here, we can make one final check.
                 */
                if (object->internal) {
                 * disconnected here, we can make one final check.
                 */
                if (object->internal) {

-                       if (VM_PAGE_Q_THROTTLED(iq))
+                       if (VM_PAGE_Q_THROTTLED(iq)) {

                                inactive_throttled = TRUE;
                                inactive_throttled = TRUE;

+                       }

                } else if (VM_PAGE_Q_THROTTLED(eq)) {
                        inactive_throttled = TRUE;
                }
 
                } else if (VM_PAGE_Q_THROTTLED(eq)) {
                        inactive_throttled = TRUE;
                }
 

-               if (inactive_throttled == TRUE)
+               if (inactive_throttled == TRUE) {

                        goto throttle_inactive;
                        goto throttle_inactive;

-       
+               }
+

 #if VM_PRESSURE_EVENTS
 #if CONFIG_JETSAM
 
 #if VM_PRESSURE_EVENTS
 #if CONFIG_JETSAM
 


@@ -3377,40 +3236,46 @@ throttle_inactive:
                 */
 
 #else /* CONFIG_JETSAM */
                 */
 
 #else /* CONFIG_JETSAM */

-               
+

                vm_pressure_response();
 
 #endif /* CONFIG_JETSAM */
 #endif /* VM_PRESSURE_EVENTS */
                vm_pressure_response();
 
 #endif /* CONFIG_JETSAM */
 #endif /* VM_PRESSURE_EVENTS */

-               
-               if (page_prev_q_state == VM_PAGE_ON_SPECULATIVE_Q)
-                       vm_pageout_speculative_dirty++;
-               else if (page_prev_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q)
-                       vm_pageout_inactive_anonymous++;

 
 

-               if (object->internal)
-                       vm_pageout_inactive_dirty_internal++;
-               else
-                       vm_pageout_inactive_dirty_external++;
+               if (page_prev_q_state == VM_PAGE_ON_SPECULATIVE_Q) {
+                       VM_PAGEOUT_DEBUG(vm_pageout_speculative_dirty, 1);
+               }
+
+               if (object->internal) {
+                       vm_pageout_vminfo.vm_pageout_inactive_dirty_internal++;
+               } else {
+                       vm_pageout_vminfo.vm_pageout_inactive_dirty_external++;
+               }

 
                /*
 
                /*

-                * do NOT set the pageout bit!
-                * sure, we might need free pages, but this page is going to take time to become free 
-                * anyway, so we may as well put it on the clean queue first and take it from there later
-                * if necessary.  that way, we'll ensure we don't free up too much. -mj
+                * internal pages will go to the compressor...
+                * external pages will go to the appropriate pager to be cleaned
+                * and upon completion will end up on 'vm_page_queue_cleaned' which
+                * is a preferred queue to steal from

                 */
                vm_pageout_cluster(m);
                 */
                vm_pageout_cluster(m);

+               inactive_burst_count = 0;

 
 done_with_inactivepage:
 
 
 done_with_inactivepage:
 

-               if (delayed_unlock++ > delayed_unlock_limit || try_failed == TRUE) {
+               if (delayed_unlock++ > delayed_unlock_limit) {
+                       int freed = local_freed;

 
 

-                       vm_pageout_prepare_to_block(&object, &delayed_unlock, &local_freeq, &local_freed,
-                                                   VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER);
-                       if (try_failed == TRUE)
+                       vm_pageout_prepare_to_block(&object, &delayed_unlock, &local_freeq, &local_freed,
+                           VM_PAGEOUT_PB_CONSIDER_WAKING_COMPACTOR_SWAPPER);
+                       if (freed == 0) {

                                lck_mtx_yield(&vm_page_queue_lock);
                                lck_mtx_yield(&vm_page_queue_lock);

+                       }
+               } else if (vm_pageout_scan_wants_object) {
+                       vm_page_unlock_queues();
+                       mutex_pause(0);
+                       vm_page_lock_queues();

                }
                }

-

                /*
                 * back to top of pageout scan loop
                 */
                /*
                 * back to top of pageout scan loop
                 */


@@ -3418,43 +3283,44 @@ done_with_inactivepage:
 }
 
 
 }
 
 

-int vm_page_free_count_init;
-

 void
 vm_page_free_reserve(
        int pages)
 {
 void
 vm_page_free_reserve(
        int pages)
 {

-       int             free_after_reserve;
+       int             free_after_reserve;

 
        if (VM_CONFIG_COMPRESSOR_IS_PRESENT) {
 
        if (VM_CONFIG_COMPRESSOR_IS_PRESENT) {

-
-               if ((vm_page_free_reserved + pages + COMPRESSOR_FREE_RESERVED_LIMIT) >= (VM_PAGE_FREE_RESERVED_LIMIT + COMPRESSOR_FREE_RESERVED_LIMIT))
+               if ((vm_page_free_reserved + pages + COMPRESSOR_FREE_RESERVED_LIMIT) >= (VM_PAGE_FREE_RESERVED_LIMIT + COMPRESSOR_FREE_RESERVED_LIMIT)) {

                        vm_page_free_reserved = VM_PAGE_FREE_RESERVED_LIMIT + COMPRESSOR_FREE_RESERVED_LIMIT;
                        vm_page_free_reserved = VM_PAGE_FREE_RESERVED_LIMIT + COMPRESSOR_FREE_RESERVED_LIMIT;

-               else
+               } else {

                        vm_page_free_reserved += (pages + COMPRESSOR_FREE_RESERVED_LIMIT);
                        vm_page_free_reserved += (pages + COMPRESSOR_FREE_RESERVED_LIMIT);

-
+               }

        } else {
        } else {

-               if ((vm_page_free_reserved + pages) >= VM_PAGE_FREE_RESERVED_LIMIT)
+               if ((vm_page_free_reserved + pages) >= VM_PAGE_FREE_RESERVED_LIMIT) {

                        vm_page_free_reserved = VM_PAGE_FREE_RESERVED_LIMIT;
                        vm_page_free_reserved = VM_PAGE_FREE_RESERVED_LIMIT;

-               else
+               } else {

                        vm_page_free_reserved += pages;
                        vm_page_free_reserved += pages;

+               }

        }
        }

-       free_after_reserve = vm_page_free_count_init - vm_page_free_reserved;
+       free_after_reserve = vm_pageout_state.vm_page_free_count_init - vm_page_free_reserved;

 
        vm_page_free_min = vm_page_free_reserved +
 
        vm_page_free_min = vm_page_free_reserved +

-               VM_PAGE_FREE_MIN(free_after_reserve);
+           VM_PAGE_FREE_MIN(free_after_reserve);

 
 

-       if (vm_page_free_min > VM_PAGE_FREE_MIN_LIMIT)
-               vm_page_free_min = VM_PAGE_FREE_MIN_LIMIT;
+       if (vm_page_free_min > VM_PAGE_FREE_MIN_LIMIT) {
+               vm_page_free_min = VM_PAGE_FREE_MIN_LIMIT;
+       }

 
        vm_page_free_target = vm_page_free_reserved +
 
        vm_page_free_target = vm_page_free_reserved +

-               VM_PAGE_FREE_TARGET(free_after_reserve);
+           VM_PAGE_FREE_TARGET(free_after_reserve);

 
 

-       if (vm_page_free_target > VM_PAGE_FREE_TARGET_LIMIT)
-               vm_page_free_target = VM_PAGE_FREE_TARGET_LIMIT;
+       if (vm_page_free_target > VM_PAGE_FREE_TARGET_LIMIT) {
+               vm_page_free_target = VM_PAGE_FREE_TARGET_LIMIT;
+       }

 
 

-       if (vm_page_free_target < vm_page_free_min + 5)
+       if (vm_page_free_target < vm_page_free_min + 5) {

                vm_page_free_target = vm_page_free_min + 5;
                vm_page_free_target = vm_page_free_min + 5;

+       }

 
        vm_page_throttle_limit = vm_page_free_target - (vm_page_free_target / 2);
 }
 
        vm_page_throttle_limit = vm_page_free_target - (vm_page_free_target / 2);
 }


@@ -3467,7 +3333,7 @@ void
 vm_pageout_continue(void)
 {
        DTRACE_VM2(pgrrun, int, 1, (uint64_t *), NULL);
 vm_pageout_continue(void)
 {
        DTRACE_VM2(pgrrun, int, 1, (uint64_t *), NULL);

-       vm_pageout_scan_event_counter++;
+       VM_PAGEOUT_DEBUG(vm_pageout_scan_event_counter, 1);

 
 #if !CONFIG_EMBEDDED
        lck_mtx_lock(&vm_page_queue_free_lock);
 
 #if !CONFIG_EMBEDDED
        lck_mtx_lock(&vm_page_queue_free_lock);


@@ -3507,17 +3373,17 @@ vm_pageout_wait(uint64_t deadline)
        kern_return_t kr;
 
        lck_mtx_lock(&vm_page_queue_free_lock);
        kern_return_t kr;
 
        lck_mtx_lock(&vm_page_queue_free_lock);

-       for (kr = KERN_SUCCESS; vm_pageout_running && (KERN_SUCCESS == kr); ) {
+       for (kr = KERN_SUCCESS; vm_pageout_running && (KERN_SUCCESS == kr);) {

                vm_pageout_waiter = TRUE;
                if (THREAD_AWAKENED != lck_mtx_sleep_deadline(
                vm_pageout_waiter = TRUE;
                if (THREAD_AWAKENED != lck_mtx_sleep_deadline(

-                               &vm_page_queue_free_lock, LCK_SLEEP_DEFAULT,
-                               (event_t) &vm_pageout_waiter, THREAD_UNINT, deadline)) {
+                           &vm_page_queue_free_lock, LCK_SLEEP_DEFAULT,
+                           (event_t) &vm_pageout_waiter, THREAD_UNINT, deadline)) {

                        kr = KERN_OPERATION_TIMED_OUT;
                }
        }
        lck_mtx_unlock(&vm_page_queue_free_lock);
 
                        kr = KERN_OPERATION_TIMED_OUT;
                }
        }
        lck_mtx_unlock(&vm_page_queue_free_lock);
 

-       return (kr);
+       return kr;

 }
 #endif /* !CONFIG_EMBEDDED */
 
 }
 #endif /* !CONFIG_EMBEDDED */
 


@@ -3525,139 +3391,136 @@ vm_pageout_wait(uint64_t deadline)
 static void
 vm_pageout_iothread_external_continue(struct vm_pageout_queue *q)
 {
 static void
 vm_pageout_iothread_external_continue(struct vm_pageout_queue *q)
 {

-       vm_page_t       m = NULL;
-       vm_object_t     object;
+       vm_page_t       m = NULL;
+       vm_object_t     object;

        vm_object_offset_t offset;
        vm_object_offset_t offset;

-       memory_object_t pager;
+       memory_object_t pager;

 
 

-       /* On systems without a compressor, the external IO thread clears its
+       /* On systems with a compressor, the external IO thread clears its

         * VM privileged bit to accommodate large allocations (e.g. bulk UPL
         * creation)
         */
         * VM privileged bit to accommodate large allocations (e.g. bulk UPL
         * creation)
         */

-       if (vm_pageout_internal_iothread != THREAD_NULL)
+       if (vm_pageout_state.vm_pageout_internal_iothread != THREAD_NULL) {

                current_thread()->options &= ~TH_OPT_VMPRIV;
                current_thread()->options &= ~TH_OPT_VMPRIV;

+       }

 
        vm_page_lockspin_queues();
 
 
        vm_page_lockspin_queues();
 

-        while ( !vm_page_queue_empty(&q->pgo_pending) ) {
-
-                  q->pgo_busy = TRUE;
-                  vm_page_queue_remove_first(&q->pgo_pending, m, vm_page_t, pageq);
-
-                  assert(m->vm_page_q_state == VM_PAGE_ON_PAGEOUT_Q);
-                  VM_PAGE_CHECK(m);
-                  /*
-                   * grab a snapshot of the object and offset this
-                   * page is tabled in so that we can relookup this
-                   * page after we've taken the object lock - these
-                   * fields are stable while we hold the page queues lock
-                   * but as soon as we drop it, there is nothing to keep
-                   * this page in this object... we hold an activity_in_progress
-                   * on this object which will keep it from terminating
-                   */
-                  object = VM_PAGE_OBJECT(m);
-                  offset = m->offset;
-
-                  if (object->object_slid) {
-                          panic("slid page %p not allowed on this path\n", m);
-                  }
-                  m->vm_page_q_state = VM_PAGE_NOT_ON_Q;
-                  VM_PAGE_ZERO_PAGEQ_ENTRY(m);
-
-                  vm_page_unlock_queues();
-
-                  vm_object_lock(object);
-
-                  m = vm_page_lookup(object, offset);
-
-                  if (m == NULL ||
-                      m->busy || m->cleaning || !m->laundry || (m->vm_page_q_state == VM_PAGE_ON_PAGEOUT_Q)) {
-                          /*
-                           * it's either the same page that someone else has
-                           * started cleaning (or it's finished cleaning or
-                           * been put back on the pageout queue), or
-                           * the page has been freed or we have found a
-                           * new page at this offset... in all of these cases
-                           * we merely need to release the activity_in_progress
-                           * we took when we put the page on the pageout queue
-                           */
-                          vm_object_activity_end(object);
-                          vm_object_unlock(object);
-
-                          vm_page_lockspin_queues();
-                          continue;
-                  }
-                  pager = object->pager;
-
-                  if (pager == MEMORY_OBJECT_NULL) {
-                          /*
-                           * This pager has been destroyed by either
-                           * memory_object_destroy or vm_object_destroy, and
-                           * so there is nowhere for the page to go.
-                           */
-                          if (m->free_when_done) {
-                                  /*
-                                   * Just free the page... VM_PAGE_FREE takes
-                                   * care of cleaning up all the state...
-                                   * including doing the vm_pageout_throttle_up
-                                   */
-                                  VM_PAGE_FREE(m);
-                          } else {
-                                  vm_page_lockspin_queues();
-
-                                  vm_pageout_throttle_up(m);
-                                  vm_page_activate(m);
-                                  
-                                  vm_page_unlock_queues();
-
-                                  /*
-                                   *   And we are done with it.
-                                   */
-                          }
-                          vm_object_activity_end(object);
-                          vm_object_unlock(object);
-
-                          vm_page_lockspin_queues();
-                          continue;
-                  }
+       while (!vm_page_queue_empty(&q->pgo_pending)) {
+               q->pgo_busy = TRUE;
+               vm_page_queue_remove_first(&q->pgo_pending, m, vmp_pageq);
+
+               assert(m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q);
+               VM_PAGE_CHECK(m);
+               /*
+                * grab a snapshot of the object and offset this
+                * page is tabled in so that we can relookup this
+                * page after we've taken the object lock - these
+                * fields are stable while we hold the page queues lock
+                * but as soon as we drop it, there is nothing to keep
+                * this page in this object... we hold an activity_in_progress
+                * on this object which will keep it from terminating
+                */
+               object = VM_PAGE_OBJECT(m);
+               offset = m->vmp_offset;
+
+               m->vmp_q_state = VM_PAGE_NOT_ON_Q;
+               VM_PAGE_ZERO_PAGEQ_ENTRY(m);
+
+               vm_page_unlock_queues();
+
+               vm_object_lock(object);
+
+               m = vm_page_lookup(object, offset);
+
+               if (m == NULL || m->vmp_busy || m->vmp_cleaning ||
+                   !m->vmp_laundry || (m->vmp_q_state != VM_PAGE_NOT_ON_Q)) {
+                       /*
+                        * it's either the same page that someone else has
+                        * started cleaning (or it's finished cleaning or
+                        * been put back on the pageout queue), or
+                        * the page has been freed or we have found a
+                        * new page at this offset... in all of these cases
+                        * we merely need to release the activity_in_progress
+                        * we took when we put the page on the pageout queue
+                        */
+                       vm_object_activity_end(object);
+                       vm_object_unlock(object);
+
+                       vm_page_lockspin_queues();
+                       continue;
+               }
+               pager = object->pager;
+
+               if (pager == MEMORY_OBJECT_NULL) {
+                       /*
+                        * This pager has been destroyed by either
+                        * memory_object_destroy or vm_object_destroy, and
+                        * so there is nowhere for the page to go.
+                        */
+                       if (m->vmp_free_when_done) {
+                               /*
+                                * Just free the page... VM_PAGE_FREE takes
+                                * care of cleaning up all the state...
+                                * including doing the vm_pageout_throttle_up
+                                */
+                               VM_PAGE_FREE(m);
+                       } else {
+                               vm_page_lockspin_queues();
+
+                               vm_pageout_throttle_up(m);
+                               vm_page_activate(m);
+
+                               vm_page_unlock_queues();
+
+                               /*
+                                *      And we are done with it.
+                                */
+                       }
+                       vm_object_activity_end(object);
+                       vm_object_unlock(object);
+
+                       vm_page_lockspin_queues();
+                       continue;
+               }

 #if 0
 #if 0

-                  /*
-                   * we don't hold the page queue lock
-                   * so this check isn't safe to make
-                   */
-                  VM_PAGE_CHECK(m);
+               /*
+                * we don't hold the page queue lock
+                * so this check isn't safe to make
+                */
+               VM_PAGE_CHECK(m);

 #endif
 #endif

-                  /*
-                   * give back the activity_in_progress reference we
-                   * took when we queued up this page and replace it
-                   * it with a paging_in_progress reference that will
-                    * also hold the paging offset from changing and
-                    * prevent the object from terminating
-                   */
-                  vm_object_activity_end(object);
-                  vm_object_paging_begin(object);
-                  vm_object_unlock(object);
-
-                   /*
-                   * Send the data to the pager.
-                   * any pageout clustering happens there
-                   */
-                  memory_object_data_return(pager,
-                                            m->offset + object->paging_offset,
-                                            PAGE_SIZE,
-                                            NULL,
-                                            NULL,
-                                            FALSE,
-                                            FALSE,
-                                            0);
-
-                  vm_object_lock(object);
-                  vm_object_paging_end(object);
-                  vm_object_unlock(object);
-
-                  vm_pageout_io_throttle();
-
-                  vm_page_lockspin_queues();
+               /*
+                * give back the activity_in_progress reference we
+                * took when we queued up this page and replace it
+                * it with a paging_in_progress reference that will
+                * also hold the paging offset from changing and
+                * prevent the object from terminating
+                */
+               vm_object_activity_end(object);
+               vm_object_paging_begin(object);
+               vm_object_unlock(object);
+
+               /*
+                * Send the data to the pager.
+                * any pageout clustering happens there
+                */
+               memory_object_data_return(pager,
+                   m->vmp_offset + object->paging_offset,
+                   PAGE_SIZE,
+                   NULL,
+                   NULL,
+                   FALSE,
+                   FALSE,
+                   0);
+
+               vm_object_lock(object);
+               vm_object_paging_end(object);
+               vm_object_unlock(object);
+
+               vm_pageout_io_throttle();
+
+               vm_page_lockspin_queues();

        }
        q->pgo_busy = FALSE;
        q->pgo_idle = TRUE;
        }
        q->pgo_busy = FALSE;
        q->pgo_idle = TRUE;


@@ -3670,15 +3533,11 @@ vm_pageout_iothread_external_continue(struct vm_pageout_queue *q)
 }
 
 
 }
 
 

-#define                MAX_FREE_BATCH          32
+#define         MAX_FREE_BATCH          32

 uint32_t vm_compressor_time_thread; /* Set via sysctl to record time accrued by
 uint32_t vm_compressor_time_thread; /* Set via sysctl to record time accrued by

-                                    * this thread.
-                                    */
-
+                                     * this thread.
+                                     */

 
 

-#if DEVELOPMENT || DEBUG
-uint64_t compressor_epoch_start, compressor_epoch_stop, compressor_threads_runtime;
-#endif

 
 void
 vm_pageout_iothread_internal_continue(struct cq *);
 
 void
 vm_pageout_iothread_internal_continue(struct cq *);


@@ -3686,28 +3545,29 @@ void
 vm_pageout_iothread_internal_continue(struct cq *cq)
 {
        struct vm_pageout_queue *q;
 vm_pageout_iothread_internal_continue(struct cq *cq)
 {
        struct vm_pageout_queue *q;

-       vm_page_t       m = NULL;
-       boolean_t       pgo_draining;
+       vm_page_t       m = NULL;
+       boolean_t       pgo_draining;

        vm_page_t   local_q;
        vm_page_t   local_q;

-       int         local_cnt;
+       int         local_cnt;

        vm_page_t   local_freeq = NULL;
        int         local_freed = 0;
        vm_page_t   local_freeq = NULL;
        int         local_freed = 0;

-       int         local_batch_size;
-       int     ncomps = 0;
+       int         local_batch_size;

 #if DEVELOPMENT || DEBUG
 #if DEVELOPMENT || DEBUG

+       int       ncomps = 0;

        boolean_t marked_active = FALSE;
 #endif
        KERNEL_DEBUG(0xe040000c | DBG_FUNC_END, 0, 0, 0, 0, 0);
 
        q = cq->q;
        boolean_t marked_active = FALSE;
 #endif
        KERNEL_DEBUG(0xe040000c | DBG_FUNC_END, 0, 0, 0, 0, 0);
 
        q = cq->q;

-       local_batch_size = q->pgo_maxlaundry / (vm_compressor_thread_count * 2);
+       local_batch_size = q->pgo_maxlaundry / (vm_pageout_state.vm_compressor_thread_count * 2);

 
 #if RECORD_THE_COMPRESSED_DATA
 
 #if RECORD_THE_COMPRESSED_DATA

-       if (q->pgo_laundry)
+       if (q->pgo_laundry) {

                c_compressed_record_init();
                c_compressed_record_init();

+       }

 #endif
        while (TRUE) {
 #endif
        while (TRUE) {

-               int     pages_left_on_q = 0;
+               int     pages_left_on_q = 0;

 
                local_cnt = 0;
                local_q = NULL;
 
                local_cnt = 0;
                local_q = NULL;


@@ -3721,7 +3581,7 @@ vm_pageout_iothread_internal_continue(struct cq *cq)
                        vmct_state[cq->id] = VMCT_ACTIVE;
                        marked_active = TRUE;
                        if (vmct_active == 1) {
                        vmct_state[cq->id] = VMCT_ACTIVE;
                        marked_active = TRUE;
                        if (vmct_active == 1) {

-                               compressor_epoch_start = mach_absolute_time();
+                               vm_compressor_epoch_start = mach_absolute_time();

                        }
                }
 #endif
                        }
                }
 #endif


@@ -3729,69 +3589,74 @@ vm_pageout_iothread_internal_continue(struct cq *cq)
 
                KERNEL_DEBUG(0xe0400018 | DBG_FUNC_START, q->pgo_laundry, 0, 0, 0, 0);
 
 
                KERNEL_DEBUG(0xe0400018 | DBG_FUNC_START, q->pgo_laundry, 0, 0, 0, 0);
 

-               while ( !vm_page_queue_empty(&q->pgo_pending) && local_cnt <  local_batch_size) {
-
-                       vm_page_queue_remove_first(&q->pgo_pending, m, vm_page_t, pageq);
-                       assert(m->vm_page_q_state == VM_PAGE_ON_PAGEOUT_Q);
+               while (!vm_page_queue_empty(&q->pgo_pending) && local_cnt < local_batch_size) {
+                       vm_page_queue_remove_first(&q->pgo_pending, m, vmp_pageq);
+                       assert(m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q);

                        VM_PAGE_CHECK(m);
                        VM_PAGE_CHECK(m);

-                       
-                       m->vm_page_q_state = VM_PAGE_NOT_ON_Q;
+
+                       m->vmp_q_state = VM_PAGE_NOT_ON_Q;

                        VM_PAGE_ZERO_PAGEQ_ENTRY(m);
                        VM_PAGE_ZERO_PAGEQ_ENTRY(m);

-                       m->laundry = FALSE;
+                       m->vmp_laundry = FALSE;

 
 

-                       m->snext = local_q;
+                       m->vmp_snext = local_q;

                        local_q = m;
                        local_cnt++;
                }
                        local_q = m;
                        local_cnt++;
                }

-               if (local_q == NULL)
+               if (local_q == NULL) {

                        break;
                        break;

+               }

 
                q->pgo_busy = TRUE;
 
                if ((pgo_draining = q->pgo_draining) == FALSE) {
                        vm_pageout_throttle_up_batch(q, local_cnt);
                        pages_left_on_q = q->pgo_laundry;
 
                q->pgo_busy = TRUE;
 
                if ((pgo_draining = q->pgo_draining) == FALSE) {
                        vm_pageout_throttle_up_batch(q, local_cnt);
                        pages_left_on_q = q->pgo_laundry;

-               } else
+               } else {

                        pages_left_on_q = q->pgo_laundry - local_cnt;
                        pages_left_on_q = q->pgo_laundry - local_cnt;

+               }

 
                vm_page_unlock_queues();
 
 #if !RECORD_THE_COMPRESSED_DATA
 
                vm_page_unlock_queues();
 
 #if !RECORD_THE_COMPRESSED_DATA

-               if (pages_left_on_q >= local_batch_size && cq->id < (vm_compressor_thread_count - 1)) {
+               if (pages_left_on_q >= local_batch_size && cq->id < (vm_pageout_state.vm_compressor_thread_count - 1)) {

                        thread_wakeup((event_t) ((uintptr_t)&q->pgo_pending + cq->id + 1));
                }
 #endif
                KERNEL_DEBUG(0xe0400018 | DBG_FUNC_END, q->pgo_laundry, 0, 0, 0, 0);
 
                while (local_q) {
                        thread_wakeup((event_t) ((uintptr_t)&q->pgo_pending + cq->id + 1));
                }
 #endif
                KERNEL_DEBUG(0xe0400018 | DBG_FUNC_END, q->pgo_laundry, 0, 0, 0, 0);
 
                while (local_q) {

-

                        KERNEL_DEBUG(0xe0400024 | DBG_FUNC_START, local_cnt, 0, 0, 0, 0);
 
                        m = local_q;
                        KERNEL_DEBUG(0xe0400024 | DBG_FUNC_START, local_cnt, 0, 0, 0, 0);
 
                        m = local_q;

-                       local_q = m->snext;
-                       m->snext = NULL;
+                       local_q = m->vmp_snext;
+                       m->vmp_snext = NULL;

 
 

-                       if (vm_pageout_compress_page(&cq->current_chead, cq->scratch_buf, m, FALSE) == KERN_SUCCESS) {
+                       if (vm_pageout_compress_page(&cq->current_chead, cq->scratch_buf, m) == KERN_SUCCESS) {
+#if DEVELOPMENT || DEBUG

                                ncomps++;
                                ncomps++;

-                               m->snext = local_freeq;
+#endif
+                               KERNEL_DEBUG(0xe0400024 | DBG_FUNC_END, local_cnt, 0, 0, 0, 0);
+
+                               m->vmp_snext = local_freeq;

                                local_freeq = m;
                                local_freed++;
 
                                if (local_freed >= MAX_FREE_BATCH) {
                                local_freeq = m;
                                local_freed++;
 
                                if (local_freed >= MAX_FREE_BATCH) {

-                                       vm_pageout_freed_after_compression += local_freed;
+                                       OSAddAtomic64(local_freed, &vm_pageout_vminfo.vm_pageout_compressions);

 
                                        vm_page_free_list(local_freeq, TRUE);
 
                                        vm_page_free_list(local_freeq, TRUE);

+

                                        local_freeq = NULL;
                                        local_freed = 0;
                                }
                        }
 #if !CONFIG_JETSAM
                        while (vm_page_free_count < COMPRESSOR_FREE_RESERVED_LIMIT) {
                                        local_freeq = NULL;
                                        local_freed = 0;
                                }
                        }
 #if !CONFIG_JETSAM
                        while (vm_page_free_count < COMPRESSOR_FREE_RESERVED_LIMIT) {

-                               kern_return_t   wait_result;
-                               int             need_wakeup = 0;
+                               kern_return_t   wait_result;
+                               int             need_wakeup = 0;

 
                                if (local_freeq) {
 
                                if (local_freeq) {

-                                       vm_pageout_freed_after_compression += local_freed;
+                                       OSAddAtomic64(local_freed, &vm_pageout_vminfo.vm_pageout_compressions);

 
                                        vm_page_free_list(local_freeq, TRUE);
                                        local_freeq = NULL;
 
                                        vm_page_free_list(local_freeq, TRUE);
                                        local_freeq = NULL;


@@ -3802,26 +3667,28 @@ vm_pageout_iothread_internal_continue(struct cq *cq)
                                lck_mtx_lock_spin(&vm_page_queue_free_lock);
 
                                if (vm_page_free_count < COMPRESSOR_FREE_RESERVED_LIMIT) {
                                lck_mtx_lock_spin(&vm_page_queue_free_lock);
 
                                if (vm_page_free_count < COMPRESSOR_FREE_RESERVED_LIMIT) {

-
-                                       if (vm_page_free_wanted_privileged++ == 0)
+                                       if (vm_page_free_wanted_privileged++ == 0) {

                                                need_wakeup = 1;
                                                need_wakeup = 1;

+                                       }

                                        wait_result = assert_wait((event_t)&vm_page_free_wanted_privileged, THREAD_UNINT);
 
                                        lck_mtx_unlock(&vm_page_queue_free_lock);
 
                                        wait_result = assert_wait((event_t)&vm_page_free_wanted_privileged, THREAD_UNINT);
 
                                        lck_mtx_unlock(&vm_page_queue_free_lock);
 

-                                       if (need_wakeup)
+                                       if (need_wakeup) {

                                                thread_wakeup((event_t)&vm_page_free_wanted);
                                                thread_wakeup((event_t)&vm_page_free_wanted);

+                                       }

 
 

-                                       if (wait_result == THREAD_WAITING)
-
+                                       if (wait_result == THREAD_WAITING) {

                                                thread_block(THREAD_CONTINUE_NULL);
                                                thread_block(THREAD_CONTINUE_NULL);

-                               } else
+                                       }
+                               } else {

                                        lck_mtx_unlock(&vm_page_queue_free_lock);
                                        lck_mtx_unlock(&vm_page_queue_free_lock);

+                               }

                        }
 #endif
                }
                if (local_freeq) {
                        }
 #endif
                }
                if (local_freeq) {

-                       vm_pageout_freed_after_compression += local_freed;
+                       OSAddAtomic64(local_freed, &vm_pageout_vminfo.vm_pageout_compressions);

 
                        vm_page_free_list(local_freeq, TRUE);
                        local_freeq = NULL;
 
                        vm_page_free_list(local_freeq, TRUE);
                        local_freeq = NULL;


@@ -3848,14 +3715,15 @@ vm_pageout_iothread_internal_continue(struct cq *cq)
                vmct_state[cq->id] = VMCT_IDLE;
 
                if (vmct_active == 0) {
                vmct_state[cq->id] = VMCT_IDLE;
 
                if (vmct_active == 0) {

-                       compressor_epoch_stop = mach_absolute_time();
-                       assert(compressor_epoch_stop > compressor_epoch_start);
+                       vm_compressor_epoch_stop = mach_absolute_time();
+                       assertf(vm_compressor_epoch_stop >= vm_compressor_epoch_start,
+                           "Compressor epoch non-monotonic: 0x%llx -> 0x%llx",
+                           vm_compressor_epoch_start, vm_compressor_epoch_stop);

                        /* This interval includes intervals where one or more
                         * compressor threads were pre-empted
                         */
                        /* This interval includes intervals where one or more
                         * compressor threads were pre-empted
                         */

-                       vmct_stats.vmct_cthreads_total += compressor_epoch_stop - compressor_epoch_start;
+                       vmct_stats.vmct_cthreads_total += vm_compressor_epoch_stop - vm_compressor_epoch_start;

                }
                }

-

        }
 #endif
        vm_page_unlock_queues();
        }
 #endif
        vm_page_unlock_queues();


@@ -3881,25 +3749,21 @@ vm_pageout_iothread_internal_continue(struct cq *cq)
 
 
 kern_return_t
 
 
 kern_return_t

-vm_pageout_compress_page(void **current_chead, char *scratch_buf, vm_page_t m, boolean_t object_locked_by_caller) 
+vm_pageout_compress_page(void **current_chead, char *scratch_buf, vm_page_t m)

 {
 {

-       vm_object_t     object;
-       memory_object_t pager;
-       int             compressed_count_delta;
-       kern_return_t   retval;
+       vm_object_t     object;
+       memory_object_t pager;
+       int             compressed_count_delta;
+       kern_return_t   retval;

 
        object = VM_PAGE_OBJECT(m);
 
 
        object = VM_PAGE_OBJECT(m);
 

-       if (object->object_slid) {
-               panic("slid page %p not allowed on this path\n", m);
-       }
-       assert(!m->free_when_done);
-       assert(!m->laundry);
+       assert(!m->vmp_free_when_done);
+       assert(!m->vmp_laundry);

 
        pager = object->pager;
 
 
        pager = object->pager;
 

-       if (object_locked_by_caller == FALSE && (!object->pager_initialized || pager == MEMORY_OBJECT_NULL))  {
-                               
+       if (!object->pager_initialized || pager == MEMORY_OBJECT_NULL) {

                KERNEL_DEBUG(0xe0400010 | DBG_FUNC_START, object, pager, 0, 0, 0);
 
                vm_object_lock(object);
                KERNEL_DEBUG(0xe0400010 | DBG_FUNC_START, object, pager, 0, 0, 0);
 
                vm_object_lock(object);


@@ -3909,10 +3773,12 @@ vm_pageout_compress_page(void **current_chead, char *scratch_buf, vm_page_t m, b
                 * one and hand it to the compression pager.
                 */
 
                 * one and hand it to the compression pager.
                 */
 

-               if (!object->pager_initialized)
+               if (!object->pager_initialized) {

                        vm_object_collapse(object, (vm_object_offset_t) 0, TRUE);
                        vm_object_collapse(object, (vm_object_offset_t) 0, TRUE);

-               if (!object->pager_initialized)
+               }
+               if (!object->pager_initialized) {

                        vm_object_compressor_pager_create(object);
                        vm_object_compressor_pager_create(object);

+               }

 
                pager = object->pager;
 
 
                pager = object->pager;
 


@@ -3929,9 +3795,9 @@ vm_pageout_compress_page(void **current_chead, char *scratch_buf, vm_page_t m, b
 
                        vm_page_lockspin_queues();
                        vm_page_activate(m);
 
                        vm_page_lockspin_queues();
                        vm_page_activate(m);

-                       vm_pageout_dirty_no_pager++;
+                       VM_PAGEOUT_DEBUG(vm_pageout_dirty_no_pager, 1);

                        vm_page_unlock_queues();
                        vm_page_unlock_queues();

-                                       
+

                        /*
                         *      And we are done with it.
                         */
                        /*
                         *      And we are done with it.
                         */


@@ -3941,35 +3807,30 @@ vm_pageout_compress_page(void **current_chead, char *scratch_buf, vm_page_t m, b
                        return KERN_FAILURE;
                }
                vm_object_unlock(object);
                        return KERN_FAILURE;
                }
                vm_object_unlock(object);

-                               
+

                KERNEL_DEBUG(0xe0400010 | DBG_FUNC_END, object, pager, 0, 0, 0);
        }
        assert(object->pager_initialized && pager != MEMORY_OBJECT_NULL);
                KERNEL_DEBUG(0xe0400010 | DBG_FUNC_END, object, pager, 0, 0, 0);
        }
        assert(object->pager_initialized && pager != MEMORY_OBJECT_NULL);

-
-       if (object_locked_by_caller == FALSE)
-               assert(object->activity_in_progress > 0);
+       assert(object->activity_in_progress > 0);

 
        retval = vm_compressor_pager_put(
                pager,
 
        retval = vm_compressor_pager_put(
                pager,

-               m->offset + object->paging_offset,
+               m->vmp_offset + object->paging_offset,

                VM_PAGE_GET_PHYS_PAGE(m),
                current_chead,
                scratch_buf,
                &compressed_count_delta);
 
                VM_PAGE_GET_PHYS_PAGE(m),
                current_chead,
                scratch_buf,
                &compressed_count_delta);
 

-       if (object_locked_by_caller == FALSE) {
-               vm_object_lock(object);
+       vm_object_lock(object);

 
 

-               assert(object->activity_in_progress > 0);
-               assert(VM_PAGE_OBJECT(m) == object);
-       }
+       assert(object->activity_in_progress > 0);
+       assert(VM_PAGE_OBJECT(m) == object);
+       assert( !VM_PAGE_WIRED(m));

 
        vm_compressor_pager_count(pager,
 
        vm_compressor_pager_count(pager,

-                                 compressed_count_delta,
-                                 FALSE, /* shared_lock */
-                                 object);
-
-       assert( !VM_PAGE_WIRED(m));
+           compressed_count_delta,
+           FALSE,                       /* shared_lock */
+           object);

 
        if (retval == KERN_SUCCESS) {
                /*
 
        if (retval == KERN_SUCCESS) {
                /*


@@ -3979,31 +3840,31 @@ vm_pageout_compress_page(void **current_chead, char *scratch_buf, vm_page_t m, b
                 * contributes to the owner's memory footprint,
                 * so account for it as such.
                 */
                 * contributes to the owner's memory footprint,
                 * so account for it as such.
                 */

-               if (object->purgable != VM_PURGABLE_DENY &&
-                   object->vo_purgeable_owner != NULL) {
-                       /* one more compressed purgeable page */
-                       vm_purgeable_compressed_update(object,
-                                                      +1);
+               if ((object->purgable != VM_PURGABLE_DENY ||
+                   object->vo_ledger_tag) &&
+                   object->vo_owner != NULL) {
+                       /* one more compressed purgeable/tagged page */
+                       vm_object_owner_compressed_update(object,
+                           +1);

                }
                VM_STAT_INCR(compressions);
                }
                VM_STAT_INCR(compressions);

-                       
-               if (m->tabled)
-                       vm_page_remove(m, TRUE);

 
 

+               if (m->vmp_tabled) {
+                       vm_page_remove(m, TRUE);
+               }

        } else {
                PAGE_WAKEUP_DONE(m);
 
                vm_page_lockspin_queues();
 
                vm_page_activate(m);
        } else {
                PAGE_WAKEUP_DONE(m);
 
                vm_page_lockspin_queues();
 
                vm_page_activate(m);

-               vm_compressor_failed++;
+               vm_pageout_vminfo.vm_compressor_failed++;

 
                vm_page_unlock_queues();
        }
 
                vm_page_unlock_queues();
        }

-       if (object_locked_by_caller == FALSE) {
-               vm_object_activity_end(object);
-               vm_object_unlock(object);
-       }
+       vm_object_activity_end(object);
+       vm_object_unlock(object);
+

        return retval;
 }
 
        return retval;
 }
 


@@ -4011,13 +3872,13 @@ vm_pageout_compress_page(void **current_chead, char *scratch_buf, vm_page_t m, b
 static void
 vm_pageout_adjust_eq_iothrottle(struct vm_pageout_queue *eq, boolean_t req_lowpriority)
 {
 static void
 vm_pageout_adjust_eq_iothrottle(struct vm_pageout_queue *eq, boolean_t req_lowpriority)
 {

-       uint32_t        policy;
-       
-       if (hibernate_cleaning_in_progress == TRUE)
+       uint32_t        policy;
+
+       if (hibernate_cleaning_in_progress == TRUE) {

                req_lowpriority = FALSE;
                req_lowpriority = FALSE;

+       }

 
        if (eq->pgo_inited == TRUE && eq->pgo_lowpriority != req_lowpriority) {
 
        if (eq->pgo_inited == TRUE && eq->pgo_lowpriority != req_lowpriority) {

-               

                vm_page_unlock_queues();
 
                if (req_lowpriority == TRUE) {
                vm_page_unlock_queues();
 
                if (req_lowpriority == TRUE) {


@@ -4028,7 +3889,7 @@ vm_pageout_adjust_eq_iothrottle(struct vm_pageout_queue *eq, boolean_t req_lowpr
                        DTRACE_VM(laundryunthrottle);
                }
                proc_set_thread_policy_with_tid(kernel_task, eq->pgo_tid,
                        DTRACE_VM(laundryunthrottle);
                }
                proc_set_thread_policy_with_tid(kernel_task, eq->pgo_tid,

-                                               TASK_POLICY_EXTERNAL, TASK_POLICY_IO, policy);
+                   TASK_POLICY_EXTERNAL, TASK_POLICY_IO, policy);

 
                eq->pgo_lowpriority = req_lowpriority;
 
 
                eq->pgo_lowpriority = req_lowpriority;
 


@@ -4040,14 +3901,14 @@ vm_pageout_adjust_eq_iothrottle(struct vm_pageout_queue *eq, boolean_t req_lowpr
 static void
 vm_pageout_iothread_external(void)
 {
 static void
 vm_pageout_iothread_external(void)
 {

-       thread_t        self = current_thread();
+       thread_t        self = current_thread();

 
        self->options |= TH_OPT_VMPRIV;
 
        DTRACE_VM2(laundrythrottle, int, 1, (uint64_t *), NULL);
 
        proc_set_thread_policy(self, TASK_POLICY_EXTERNAL,
 
        self->options |= TH_OPT_VMPRIV;
 
        DTRACE_VM2(laundrythrottle, int, 1, (uint64_t *), NULL);
 
        proc_set_thread_policy(self, TASK_POLICY_EXTERNAL,

-                              TASK_POLICY_IO, THROTTLE_LEVEL_PAGEOUT_THROTTLED);
+           TASK_POLICY_IO, THROTTLE_LEVEL_PAGEOUT_THROTTLED);

 
        vm_page_lock_queues();
 
 
        vm_page_lock_queues();
 


@@ -4066,7 +3927,7 @@ vm_pageout_iothread_external(void)
 static void
 vm_pageout_iothread_internal(struct cq *cq)
 {
 static void
 vm_pageout_iothread_internal(struct cq *cq)
 {

-       thread_t        self = current_thread();
+       thread_t        self = current_thread();

 
        self->options |= TH_OPT_VMPRIV;
 
 
        self->options |= TH_OPT_VMPRIV;
 


@@ -4078,8 +3939,9 @@ vm_pageout_iothread_internal(struct cq *cq)
 
        vm_page_unlock_queues();
 
 
        vm_page_unlock_queues();
 

-       if (vm_restricted_to_single_processor == TRUE)
+       if (vm_pageout_state.vm_restricted_to_single_processor == TRUE) {

                thread_vm_bind_group_add();
                thread_vm_bind_group_add();

+       }

 
 
        thread_set_thread_name(current_thread(), "VM_compressor");
 
 
        thread_set_thread_name(current_thread(), "VM_compressor");


@@ -4092,7 +3954,7 @@ vm_pageout_iothread_internal(struct cq *cq)
 }
 
 kern_return_t
 }
 
 kern_return_t

-vm_set_buffer_cleanup_callout(boolean_t (*func)(int)) 
+vm_set_buffer_cleanup_callout(boolean_t (*func)(int))

 {
        if (OSCompareAndSwapPtr(NULL, func, (void * volatile *) &consider_buffer_cache_collect)) {
                return KERN_SUCCESS;
 {
        if (OSCompareAndSwapPtr(NULL, func, (void * volatile *) &consider_buffer_cache_collect)) {
                return KERN_SUCCESS;


@@ -4101,8 +3963,8 @@ vm_set_buffer_cleanup_callout(boolean_t (*func)(int))
        }
 }
 
        }
 }
 

-extern boolean_t       memorystatus_manual_testing_on;
-extern unsigned int    memorystatus_level;
+extern boolean_t        memorystatus_manual_testing_on;
+extern unsigned int     memorystatus_level;

 
 
 #if VM_PRESSURE_EVENTS
 
 
 #if VM_PRESSURE_EVENTS


@@ -4112,14 +3974,14 @@ boolean_t vm_pressure_events_enabled = FALSE;
 void
 vm_pressure_response(void)
 {
 void
 vm_pressure_response(void)
 {

+       vm_pressure_level_t     old_level = kVMPressureNormal;
+       int                     new_level = -1;
+       unsigned int            total_pages;
+       uint64_t                available_memory = 0;

 
 

-       vm_pressure_level_t     old_level = kVMPressureNormal;
-       int                     new_level = -1;
-       unsigned int            total_pages;
-       uint64_t                available_memory = 0;
-
-       if (vm_pressure_events_enabled == FALSE)
+       if (vm_pressure_events_enabled == FALSE) {

                return;
                return;

+       }

 
 #if CONFIG_EMBEDDED
 
 
 #if CONFIG_EMBEDDED
 


@@ -4141,88 +4003,90 @@ vm_pressure_response(void)
        if (memorystatus_manual_testing_on) {
                return;
        }
        if (memorystatus_manual_testing_on) {
                return;
        }

-       
+

        old_level = memorystatus_vm_pressure_level;
 
        switch (memorystatus_vm_pressure_level) {
        old_level = memorystatus_vm_pressure_level;
 
        switch (memorystatus_vm_pressure_level) {

-
-               case kVMPressureNormal:
-               {
-                       if (VM_PRESSURE_WARNING_TO_CRITICAL()) {
-                               new_level = kVMPressureCritical;
-                       }  else if (VM_PRESSURE_NORMAL_TO_WARNING()) {
-                               new_level = kVMPressureWarning;
-                       }
-                       break;
+       case kVMPressureNormal:
+       {
+               if (VM_PRESSURE_WARNING_TO_CRITICAL()) {
+                       new_level = kVMPressureCritical;
+               } else if (VM_PRESSURE_NORMAL_TO_WARNING()) {
+                       new_level = kVMPressureWarning;

                }
                }

+               break;
+       }

 
 

-               case kVMPressureWarning:
-               case kVMPressureUrgent:
-               {
-                       if (VM_PRESSURE_WARNING_TO_NORMAL()) {
-                               new_level = kVMPressureNormal;
-                       }  else if (VM_PRESSURE_WARNING_TO_CRITICAL()) {
-                               new_level = kVMPressureCritical;
-                       }
-                       break;
+       case kVMPressureWarning:
+       case kVMPressureUrgent:
+       {
+               if (VM_PRESSURE_WARNING_TO_NORMAL()) {
+                       new_level = kVMPressureNormal;
+               } else if (VM_PRESSURE_WARNING_TO_CRITICAL()) {
+                       new_level = kVMPressureCritical;

                }
                }

+               break;
+       }

 
 

-               case kVMPressureCritical:
-               {
-                       if (VM_PRESSURE_WARNING_TO_NORMAL()) {
-                               new_level = kVMPressureNormal;
-                       }  else if (VM_PRESSURE_CRITICAL_TO_WARNING()) {
-                               new_level = kVMPressureWarning;
-                       }
-                       break;
+       case kVMPressureCritical:
+       {
+               if (VM_PRESSURE_WARNING_TO_NORMAL()) {
+                       new_level = kVMPressureNormal;
+               } else if (VM_PRESSURE_CRITICAL_TO_WARNING()) {
+                       new_level = kVMPressureWarning;

                }
                }

+               break;
+       }

 
 

-               default:
-                       return;
+       default:
+               return;

        }
        }

-               
+

        if (new_level != -1) {
                memorystatus_vm_pressure_level = (vm_pressure_level_t) new_level;
 
        if (new_level != -1) {
                memorystatus_vm_pressure_level = (vm_pressure_level_t) new_level;
 

-               if ((memorystatus_vm_pressure_level != kVMPressureNormal) || (old_level != new_level)) {
-                       if (vm_pressure_thread_running == FALSE) {
+               if (new_level != (int) old_level) {
+                       VM_DEBUG_CONSTANT_EVENT(vm_pressure_level_change, VM_PRESSURE_LEVEL_CHANGE, DBG_FUNC_NONE,
+                           new_level, old_level, 0, 0);
+               }
+
+               if ((memorystatus_vm_pressure_level != kVMPressureNormal) || (old_level != memorystatus_vm_pressure_level)) {
+                       if (vm_pageout_state.vm_pressure_thread_running == FALSE) {

                                thread_wakeup(&vm_pressure_thread);
                        }
 
                                thread_wakeup(&vm_pressure_thread);
                        }
 

-                       if (old_level != new_level) {
-                               thread_wakeup(&vm_pressure_changed);
+                       if (old_level != memorystatus_vm_pressure_level) {
+                               thread_wakeup(&vm_pageout_state.vm_pressure_changed);

                        }
                }
        }
                        }
                }
        }

-

 }
 #endif /* VM_PRESSURE_EVENTS */
 
 kern_return_t
 }
 #endif /* VM_PRESSURE_EVENTS */
 
 kern_return_t

-mach_vm_pressure_level_monitor(__unused boolean_t wait_for_pressure, __unused unsigned int *pressure_level) {
-
+mach_vm_pressure_level_monitor(__unused boolean_t wait_for_pressure, __unused unsigned int *pressure_level)
+{

 #if CONFIG_EMBEDDED
 #if CONFIG_EMBEDDED

-       
+

        return KERN_FAILURE;
 
 #elif !VM_PRESSURE_EVENTS
        return KERN_FAILURE;
 
 #elif !VM_PRESSURE_EVENTS

-       
+

        return KERN_FAILURE;
 
 #else /* VM_PRESSURE_EVENTS */
 
        return KERN_FAILURE;
 
 #else /* VM_PRESSURE_EVENTS */
 

-       kern_return_t   kr = KERN_SUCCESS;
+       kern_return_t   kr = KERN_SUCCESS;

 
        if (pressure_level != NULL) {
 
        if (pressure_level != NULL) {

-
-               vm_pressure_level_t     old_level = memorystatus_vm_pressure_level;
+               vm_pressure_level_t     old_level = memorystatus_vm_pressure_level;

 
                if (wait_for_pressure == TRUE) {
 
                if (wait_for_pressure == TRUE) {

-                       wait_result_t           wr = 0;
+                       wait_result_t           wr = 0;

 
                        while (old_level == *pressure_level) {
 
                        while (old_level == *pressure_level) {

-                               wr = assert_wait((event_t) &vm_pressure_changed,
-                                                THREAD_INTERRUPTIBLE);
+                               wr = assert_wait((event_t) &vm_pageout_state.vm_pressure_changed,
+                                   THREAD_INTERRUPTIBLE);

                                if (wr == THREAD_WAITING) {
                                        wr = thread_block(THREAD_CONTINUE_NULL);
                                }
                                if (wr == THREAD_WAITING) {
                                        wr = thread_block(THREAD_CONTINUE_NULL);
                                }


@@ -4230,7 +4094,6 @@ mach_vm_pressure_level_monitor(__unused boolean_t wait_for_pressure, __unused un
                                        return KERN_ABORTED;
                                }
                                if (wr == THREAD_AWAKENED) {
                                        return KERN_ABORTED;
                                }
                                if (wr == THREAD_AWAKENED) {

-                                       

                                        old_level = memorystatus_vm_pressure_level;
 
                                        if (old_level != *pressure_level) {
                                        old_level = memorystatus_vm_pressure_level;
 
                                        if (old_level != *pressure_level) {


@@ -4252,15 +4115,17 @@ mach_vm_pressure_level_monitor(__unused boolean_t wait_for_pressure, __unused un
 
 #if VM_PRESSURE_EVENTS
 void
 
 #if VM_PRESSURE_EVENTS
 void

-vm_pressure_thread(void) {
+vm_pressure_thread(void)
+{

        static boolean_t thread_initialized = FALSE;
 
        if (thread_initialized == TRUE) {
        static boolean_t thread_initialized = FALSE;
 
        if (thread_initialized == TRUE) {

-               vm_pressure_thread_running = TRUE;
+               vm_pageout_state.vm_pressure_thread_running = TRUE;

                consider_vm_pressure_events();
                consider_vm_pressure_events();

-               vm_pressure_thread_running = FALSE;
+               vm_pageout_state.vm_pressure_thread_running = FALSE;

        }
 
        }
 

+       thread_set_thread_name(current_thread(), "VM_pressure");

        thread_initialized = TRUE;
        assert_wait((event_t) &vm_pressure_thread, THREAD_UNINT);
        thread_block((thread_continue_t)vm_pressure_thread);
        thread_initialized = TRUE;
        assert_wait((event_t) &vm_pressure_thread, THREAD_UNINT);
        thread_block((thread_continue_t)vm_pressure_thread);


@@ -4268,17 +4133,14 @@ vm_pressure_thread(void) {
 #endif /* VM_PRESSURE_EVENTS */
 
 
 #endif /* VM_PRESSURE_EVENTS */
 
 

-uint32_t vm_pageout_considered_page_last = 0;
-

 /*
  * called once per-second via "compute_averages"
  */
 void
 compute_pageout_gc_throttle(__unused void *arg)
 {
 /*
  * called once per-second via "compute_averages"
  */
 void
 compute_pageout_gc_throttle(__unused void *arg)
 {

-       if (vm_pageout_considered_page != vm_pageout_considered_page_last) {
-
-               vm_pageout_considered_page_last = vm_pageout_considered_page;
+       if (vm_pageout_vminfo.vm_pageout_considered_page != vm_pageout_state.vm_pageout_considered_page_last) {
+               vm_pageout_state.vm_pageout_considered_page_last = vm_pageout_vminfo.vm_pageout_considered_page;

 
                thread_wakeup((event_t) &vm_pageout_garbage_collect);
        }
 
                thread_wakeup((event_t) &vm_pageout_garbage_collect);
        }


@@ -4330,7 +4192,6 @@ vm_pageout_garbage_collect(int collect)
                         * up again.
                         */
                        consider_zone_gc(TRUE);
                         * up again.
                         */
                        consider_zone_gc(TRUE);

-

                } else {
                        /* Woken up by vm_pageout_scan or compute_pageout_gc_throttle. */
                        boolean_t buf_large_zfree = FALSE;
                } else {
                        /* Woken up by vm_pageout_scan or compute_pageout_gc_throttle. */
                        boolean_t buf_large_zfree = FALSE;


@@ -4339,7 +4200,7 @@ vm_pageout_garbage_collect(int collect)
                        stack_collect();
 
                        consider_machine_collect();
                        stack_collect();
 
                        consider_machine_collect();

-                       m_drain();
+                       mbuf_drain(FALSE);

 
                        do {
                                if (consider_buffer_cache_collect != NULL) {
 
                        do {
                                if (consider_buffer_cache_collect != NULL) {


@@ -4353,7 +4214,6 @@ vm_pageout_garbage_collect(int collect)
                                        consider_zone_gc(FALSE);
                                }
                                first_try = FALSE;
                                        consider_zone_gc(FALSE);
                                }
                                first_try = FALSE;

-

                        } while (buf_large_zfree == TRUE && vm_page_free_count < vm_page_free_target);
 
                        consider_machine_adjust();
                        } while (buf_large_zfree == TRUE && vm_page_free_count < vm_page_free_target);
 
                        consider_machine_adjust();


@@ -4388,13 +4248,13 @@ vm_set_restrictions()
 
        if (hinfo.max_cpus <= 3) {
                /*
 
        if (hinfo.max_cpus <= 3) {
                /*

-                * on systems with a limited number of CPUS, bind the 
+                * on systems with a limited number of CPUS, bind the

                 * 4 major threads that can free memory and that tend to use
                 * a fair bit of CPU under pressured conditions to a single processor.
                 * This insures that these threads don't hog all of the available CPUs
                 * (important for camera launch), while allowing them to run independently
                 * w/r to locks... the 4 threads are
                 * 4 major threads that can free memory and that tend to use
                 * a fair bit of CPU under pressured conditions to a single processor.
                 * This insures that these threads don't hog all of the available CPUs
                 * (important for camera launch), while allowing them to run independently
                 * w/r to locks... the 4 threads are

-                * vm_pageout_scan,  vm_pageout_iothread_internal (compressor), 
+                * vm_pageout_scan,  vm_pageout_iothread_internal (compressor),

                 * vm_compressor_swap_trigger_thread (minor and major compactions),
                 * memorystatus_thread (jetsams).
                 *
                 * vm_compressor_swap_trigger_thread (minor and major compactions),
                 * memorystatus_thread (jetsams).
                 *


@@ -4403,17 +4263,19 @@ vm_set_restrictions()
                 * thread_bind_master...  someday this should be replaced with a group
                 * scheduling mechanism and KPI.
                 */
                 * thread_bind_master...  someday this should be replaced with a group
                 * scheduling mechanism and KPI.
                 */

-               vm_restricted_to_single_processor = TRUE;
+               vm_pageout_state.vm_restricted_to_single_processor = TRUE;
+       } else {
+               vm_pageout_state.vm_restricted_to_single_processor = FALSE;

        }
 }
 
 void
 vm_pageout(void)
 {
        }
 }
 
 void
 vm_pageout(void)
 {

-       thread_t        self = current_thread();
-       thread_t        thread;
-       kern_return_t   result;
-       spl_t           s;
+       thread_t        self = current_thread();
+       thread_t        thread;
+       kern_return_t   result;
+       spl_t           s;

 
        /*
         * Set thread privileges.
 
        /*
         * Set thread privileges.


@@ -4425,11 +4287,13 @@ vm_pageout(void)
        sched_set_thread_base_priority(self, BASEPRI_VM);
        thread_unlock(self);
 
        sched_set_thread_base_priority(self, BASEPRI_VM);
        thread_unlock(self);
 

-       if (!self->reserved_stack)
+       if (!self->reserved_stack) {

                self->reserved_stack = self->kernel_stack;
                self->reserved_stack = self->kernel_stack;

+       }

 
 

-       if (vm_restricted_to_single_processor == TRUE)
+       if (vm_pageout_state.vm_restricted_to_single_processor == TRUE) {

                thread_vm_bind_group_add();
                thread_vm_bind_group_add();

+       }

 
        splx(s);
 
 
        splx(s);
 


@@ -4439,43 +4303,70 @@ vm_pageout(void)
         *      Initialize some paging parameters.
         */
 
         *      Initialize some paging parameters.
         */
 

-       if (vm_pageout_swap_wait == 0)
-               vm_pageout_swap_wait = VM_PAGEOUT_SWAP_WAIT;
-
-       if (vm_pageout_idle_wait == 0)
-               vm_pageout_idle_wait = VM_PAGEOUT_IDLE_WAIT;
-
-       if (vm_pageout_burst_wait == 0)
-               vm_pageout_burst_wait = VM_PAGEOUT_BURST_WAIT;
-
-       if (vm_pageout_empty_wait == 0)
-               vm_pageout_empty_wait = VM_PAGEOUT_EMPTY_WAIT;
+       vm_pageout_state.vm_pressure_thread_running = FALSE;
+       vm_pageout_state.vm_pressure_changed = FALSE;
+       vm_pageout_state.memorystatus_purge_on_warning = 2;
+       vm_pageout_state.memorystatus_purge_on_urgent = 5;
+       vm_pageout_state.memorystatus_purge_on_critical = 8;
+       vm_pageout_state.vm_page_speculative_q_age_ms = VM_PAGE_SPECULATIVE_Q_AGE_MS;
+       vm_pageout_state.vm_page_speculative_percentage = 5;
+       vm_pageout_state.vm_page_speculative_target = 0;
+
+       vm_pageout_state.vm_pageout_external_iothread = THREAD_NULL;
+       vm_pageout_state.vm_pageout_internal_iothread = THREAD_NULL;
+
+       vm_pageout_state.vm_pageout_swap_wait = 0;
+       vm_pageout_state.vm_pageout_idle_wait = 0;
+       vm_pageout_state.vm_pageout_empty_wait = 0;
+       vm_pageout_state.vm_pageout_burst_wait = 0;
+       vm_pageout_state.vm_pageout_deadlock_wait = 0;
+       vm_pageout_state.vm_pageout_deadlock_relief = 0;
+       vm_pageout_state.vm_pageout_burst_inactive_throttle = 0;
+
+       vm_pageout_state.vm_pageout_inactive = 0;
+       vm_pageout_state.vm_pageout_inactive_used = 0;
+       vm_pageout_state.vm_pageout_inactive_clean = 0;
+
+       vm_pageout_state.vm_memory_pressure = 0;
+       vm_pageout_state.vm_page_filecache_min = 0;
+#if CONFIG_JETSAM
+       vm_pageout_state.vm_page_filecache_min_divisor = 70;
+       vm_pageout_state.vm_page_xpmapped_min_divisor = 40;
+#else
+       vm_pageout_state.vm_page_filecache_min_divisor = 27;
+       vm_pageout_state.vm_page_xpmapped_min_divisor = 36;
+#endif
+       vm_pageout_state.vm_page_free_count_init = vm_page_free_count;

 
 

-       if (vm_pageout_deadlock_wait == 0)
-               vm_pageout_deadlock_wait = VM_PAGEOUT_DEADLOCK_WAIT;
+       vm_pageout_state.vm_pageout_considered_page_last = 0;

 
 

-       if (vm_pageout_deadlock_relief == 0)
-               vm_pageout_deadlock_relief = VM_PAGEOUT_DEADLOCK_RELIEF;
+       if (vm_pageout_state.vm_pageout_swap_wait == 0) {
+               vm_pageout_state.vm_pageout_swap_wait = VM_PAGEOUT_SWAP_WAIT;
+       }

 
 

-       if (vm_pageout_inactive_relief == 0)
-               vm_pageout_inactive_relief = VM_PAGEOUT_INACTIVE_RELIEF;
+       if (vm_pageout_state.vm_pageout_idle_wait == 0) {
+               vm_pageout_state.vm_pageout_idle_wait = VM_PAGEOUT_IDLE_WAIT;
+       }

 
 

-       if (vm_pageout_burst_active_throttle == 0)
-               vm_pageout_burst_active_throttle = VM_PAGEOUT_BURST_ACTIVE_THROTTLE;
+       if (vm_pageout_state.vm_pageout_burst_wait == 0) {
+               vm_pageout_state.vm_pageout_burst_wait = VM_PAGEOUT_BURST_WAIT;
+       }

 
 

-       if (vm_pageout_burst_inactive_throttle == 0)
-               vm_pageout_burst_inactive_throttle = VM_PAGEOUT_BURST_INACTIVE_THROTTLE;
+       if (vm_pageout_state.vm_pageout_empty_wait == 0) {
+               vm_pageout_state.vm_pageout_empty_wait = VM_PAGEOUT_EMPTY_WAIT;
+       }

 
 

-       /*
-        * Set kernel task to low backing store privileged 
-        * status
-        */
-       task_lock(kernel_task);
-       kernel_task->priv_flags |= VM_BACKING_STORE_PRIV;
-       task_unlock(kernel_task);
+       if (vm_pageout_state.vm_pageout_deadlock_wait == 0) {
+               vm_pageout_state.vm_pageout_deadlock_wait = VM_PAGEOUT_DEADLOCK_WAIT;
+       }

 
 

-       vm_page_free_count_init = vm_page_free_count;
+       if (vm_pageout_state.vm_pageout_deadlock_relief == 0) {
+               vm_pageout_state.vm_pageout_deadlock_relief = VM_PAGEOUT_DEADLOCK_RELIEF;
+       }

 
 

+       if (vm_pageout_state.vm_pageout_burst_inactive_throttle == 0) {
+               vm_pageout_state.vm_pageout_burst_inactive_throttle = VM_PAGEOUT_BURST_INACTIVE_THROTTLE;
+       }

        /*
         * even if we've already called vm_page_free_reserve
         * call it again here to insure that the targets are
        /*
         * even if we've already called vm_page_free_reserve
         * call it again here to insure that the targets are


@@ -4485,8 +4376,9 @@ vm_pageout(void)
         */
        if (vm_page_free_reserved < VM_PAGE_FREE_RESERVED(processor_count)) {
                vm_page_free_reserve((VM_PAGE_FREE_RESERVED(processor_count)) - vm_page_free_reserved);
         */
        if (vm_page_free_reserved < VM_PAGE_FREE_RESERVED(processor_count)) {
                vm_page_free_reserve((VM_PAGE_FREE_RESERVED(processor_count)) - vm_page_free_reserved);

-       } else
+       } else {

                vm_page_free_reserve(0);
                vm_page_free_reserve(0);

+       }

 
 
        vm_page_queue_init(&vm_pageout_queue_external.pgo_pending);
 
 
        vm_page_queue_init(&vm_pageout_queue_external.pgo_pending);


@@ -4514,29 +4406,32 @@ vm_pageout(void)
        /* internal pageout thread started when default pager registered first time */
        /* external pageout and garbage collection threads started here */
 
        /* internal pageout thread started when default pager registered first time */
        /* external pageout and garbage collection threads started here */
 

-       result = kernel_thread_start_priority((thread_continue_t)vm_pageout_iothread_external, NULL, 
-                                             BASEPRI_VM,
-                                             &vm_pageout_external_iothread);
-       if (result != KERN_SUCCESS)
+       result = kernel_thread_start_priority((thread_continue_t)vm_pageout_iothread_external, NULL,
+           BASEPRI_VM,
+           &vm_pageout_state.vm_pageout_external_iothread);
+       if (result != KERN_SUCCESS) {

                panic("vm_pageout_iothread_external: create failed");
                panic("vm_pageout_iothread_external: create failed");

+       }

 
 

-       thread_deallocate(vm_pageout_external_iothread);
+       thread_deallocate(vm_pageout_state.vm_pageout_external_iothread);

 
        result = kernel_thread_start_priority((thread_continue_t)vm_pageout_garbage_collect, NULL,
 
        result = kernel_thread_start_priority((thread_continue_t)vm_pageout_garbage_collect, NULL,

-                                             BASEPRI_DEFAULT, 
-                                             &thread);
-       if (result != KERN_SUCCESS)
+           BASEPRI_DEFAULT,
+           &thread);
+       if (result != KERN_SUCCESS) {

                panic("vm_pageout_garbage_collect: create failed");
                panic("vm_pageout_garbage_collect: create failed");

+       }

 
        thread_deallocate(thread);
 
 #if VM_PRESSURE_EVENTS
        result = kernel_thread_start_priority((thread_continue_t)vm_pressure_thread, NULL,
 
        thread_deallocate(thread);
 
 #if VM_PRESSURE_EVENTS
        result = kernel_thread_start_priority((thread_continue_t)vm_pressure_thread, NULL,

-                                               BASEPRI_DEFAULT,
-                                               &thread);
+           BASEPRI_DEFAULT,
+           &thread);

 
 

-       if (result != KERN_SUCCESS)
+       if (result != KERN_SUCCESS) {

                panic("vm_pressure_thread: create failed");
                panic("vm_pressure_thread: create failed");

+       }

 
        thread_deallocate(thread);
 #endif
 
        thread_deallocate(thread);
 #endif


@@ -4546,8 +4441,7 @@ vm_pageout(void)
 
        bzero(&vm_config, sizeof(vm_config));
 
 
        bzero(&vm_config, sizeof(vm_config));
 

-       switch(vm_compressor_mode) {
-               
+       switch (vm_compressor_mode) {

        case VM_PAGER_DEFAULT:
                printf("mapping deprecated VM_PAGER_DEFAULT to VM_PAGER_COMPRESSOR_WITH_SWAP\n");
 
        case VM_PAGER_DEFAULT:
                printf("mapping deprecated VM_PAGER_DEFAULT to VM_PAGER_COMPRESSOR_WITH_SWAP\n");
 


@@ -4586,8 +4480,9 @@ vm_pageout(void)
                printf("unknown compressor mode - %x\n", vm_compressor_mode);
                break;
        }
                printf("unknown compressor mode - %x\n", vm_compressor_mode);
                break;
        }

-       if (VM_CONFIG_COMPRESSOR_IS_PRESENT)
+       if (VM_CONFIG_COMPRESSOR_IS_PRESENT) {

                vm_compressor_pager_init();
                vm_compressor_pager_init();

+       }

 
 #if VM_PRESSURE_EVENTS
        vm_pressure_events_enabled = TRUE;
 
 #if VM_PRESSURE_EVENTS
        vm_pressure_events_enabled = TRUE;


@@ -4599,12 +4494,12 @@ vm_pageout(void)
 #if VM_PAGE_BUCKETS_CHECK
 #if VM_PAGE_FAKE_BUCKETS
        printf("**** DEBUG: protecting fake buckets [0x%llx:0x%llx]\n",
 #if VM_PAGE_BUCKETS_CHECK
 #if VM_PAGE_FAKE_BUCKETS
        printf("**** DEBUG: protecting fake buckets [0x%llx:0x%llx]\n",

-              (uint64_t) vm_page_fake_buckets_start,
-              (uint64_t) vm_page_fake_buckets_end);
+           (uint64_t) vm_page_fake_buckets_start,
+           (uint64_t) vm_page_fake_buckets_end);

        pmap_protect(kernel_pmap,
        pmap_protect(kernel_pmap,

-                    vm_page_fake_buckets_start,
-                    vm_page_fake_buckets_end,
-                    VM_PROT_READ);
+           vm_page_fake_buckets_start,
+           vm_page_fake_buckets_end,
+           VM_PROT_READ);

 //     *(char *) vm_page_fake_buckets_start = 'x';     /* panic! */
 #endif /* VM_PAGE_FAKE_BUCKETS */
 #endif /* VM_PAGE_BUCKETS_CHECK */
 //     *(char *) vm_page_fake_buckets_start = 'x';     /* panic! */
 #endif /* VM_PAGE_FAKE_BUCKETS */
 #endif /* VM_PAGE_BUCKETS_CHECK */


@@ -4643,20 +4538,14 @@ vm_pageout(void)
 
 
 
 
 
 

-#if CONFIG_EMBEDDED
-int vm_compressor_thread_count = 1;
-#else
-int vm_compressor_thread_count = 2;
-#endif
-

 kern_return_t
 vm_pageout_internal_start(void)
 {
 kern_return_t
 vm_pageout_internal_start(void)
 {

-       kern_return_t   result;
-       int             i;
+       kern_return_t   result;
+       int             i;

        host_basic_info_data_t hinfo;
 
        host_basic_info_data_t hinfo;
 

-       assert (VM_CONFIG_COMPRESSOR_IS_PRESENT);
+       assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);

 
        mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
 #define BSD_HOST 1
 
        mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
 #define BSD_HOST 1


@@ -4664,30 +4553,47 @@ vm_pageout_internal_start(void)
 
        assert(hinfo.max_cpus > 0);
 
 
        assert(hinfo.max_cpus > 0);
 

-       PE_parse_boot_argn("vmcomp_threads", &vm_compressor_thread_count, sizeof(vm_compressor_thread_count));
-       if (vm_compressor_thread_count >= hinfo.max_cpus)
-               vm_compressor_thread_count = hinfo.max_cpus - 1;
-       if (vm_compressor_thread_count <= 0)
-               vm_compressor_thread_count = 1;
-       else if (vm_compressor_thread_count > MAX_COMPRESSOR_THREAD_COUNT)
-               vm_compressor_thread_count = MAX_COMPRESSOR_THREAD_COUNT;
+       lck_grp_init(&vm_pageout_lck_grp, "vm_pageout", LCK_GRP_ATTR_NULL);
+
+#if CONFIG_EMBEDDED
+       vm_pageout_state.vm_compressor_thread_count = 1;
+#else
+       if (hinfo.max_cpus > 4) {
+               vm_pageout_state.vm_compressor_thread_count = 2;
+       } else {
+               vm_pageout_state.vm_compressor_thread_count = 1;
+       }
+#endif
+       PE_parse_boot_argn("vmcomp_threads", &vm_pageout_state.vm_compressor_thread_count,
+           sizeof(vm_pageout_state.vm_compressor_thread_count));
+
+       if (vm_pageout_state.vm_compressor_thread_count >= hinfo.max_cpus) {
+               vm_pageout_state.vm_compressor_thread_count = hinfo.max_cpus - 1;
+       }
+       if (vm_pageout_state.vm_compressor_thread_count <= 0) {
+               vm_pageout_state.vm_compressor_thread_count = 1;
+       } else if (vm_pageout_state.vm_compressor_thread_count > MAX_COMPRESSOR_THREAD_COUNT) {
+               vm_pageout_state.vm_compressor_thread_count = MAX_COMPRESSOR_THREAD_COUNT;
+       }

 
 

-       vm_pageout_queue_internal.pgo_maxlaundry = (vm_compressor_thread_count * 4) * VM_PAGE_LAUNDRY_MAX;
+       vm_pageout_queue_internal.pgo_maxlaundry = (vm_pageout_state.vm_compressor_thread_count * 4) * VM_PAGE_LAUNDRY_MAX;

 
        PE_parse_boot_argn("vmpgoi_maxlaundry", &vm_pageout_queue_internal.pgo_maxlaundry, sizeof(vm_pageout_queue_internal.pgo_maxlaundry));
 
 
        PE_parse_boot_argn("vmpgoi_maxlaundry", &vm_pageout_queue_internal.pgo_maxlaundry, sizeof(vm_pageout_queue_internal.pgo_maxlaundry));
 

-       for (i = 0; i < vm_compressor_thread_count; i++) {
+       for (i = 0; i < vm_pageout_state.vm_compressor_thread_count; i++) {

                ciq[i].id = i;
                ciq[i].q = &vm_pageout_queue_internal;
                ciq[i].current_chead = NULL;
                ciq[i].scratch_buf = kalloc(COMPRESSOR_SCRATCH_BUF_SIZE);
 
                ciq[i].id = i;
                ciq[i].q = &vm_pageout_queue_internal;
                ciq[i].current_chead = NULL;
                ciq[i].scratch_buf = kalloc(COMPRESSOR_SCRATCH_BUF_SIZE);
 

-               result = kernel_thread_start_priority((thread_continue_t)vm_pageout_iothread_internal, (void *)&ciq[i], BASEPRI_VM, &vm_pageout_internal_iothread);
+               result = kernel_thread_start_priority((thread_continue_t)vm_pageout_iothread_internal, (void *)&ciq[i],
+                   BASEPRI_VM, &vm_pageout_state.vm_pageout_internal_iothread);

 
 

-               if (result == KERN_SUCCESS)
-                       thread_deallocate(vm_pageout_internal_iothread);
-               else
+               if (result == KERN_SUCCESS) {
+                       thread_deallocate(vm_pageout_state.vm_pageout_internal_iothread);
+               } else {

                        break;
                        break;

+               }

        }
        return result;
 }
        }
        return result;
 }


@@ -4709,41 +4615,41 @@ vm_pageout_internal_start(void)
 static void
 upl_set_decmp_info(upl_t upl, upl_t src_upl)
 {
 static void
 upl_set_decmp_info(upl_t upl, upl_t src_upl)
 {

-        assert((src_upl->flags & UPL_DECMP_REQ) != 0);
-
-        upl_lock(src_upl);
-        if (src_upl->decmp_io_upl) {
-                /*
-                 * If there is already an alive real I/O UPL, ignore this new UPL.
-                 * This case should rarely happen and even if it does, it just means
-                 * that we might issue a spurious expedite which the driver is expected
-                 * to handle.
-                 */ 
-                upl_unlock(src_upl);
-                return;
-        }
-        src_upl->decmp_io_upl = (void *)upl;
-        src_upl->ref_count++;
-
-        upl->flags |= UPL_DECMP_REAL_IO;
-        upl->decmp_io_upl = (void *)src_upl;
+       assert((src_upl->flags & UPL_DECMP_REQ) != 0);
+
+       upl_lock(src_upl);
+       if (src_upl->decmp_io_upl) {
+               /*
+                * If there is already an alive real I/O UPL, ignore this new UPL.
+                * This case should rarely happen and even if it does, it just means
+                * that we might issue a spurious expedite which the driver is expected
+                * to handle.
+                */
+               upl_unlock(src_upl);
+               return;
+       }
+       src_upl->decmp_io_upl = (void *)upl;
+       src_upl->ref_count++;
+
+       upl->flags |= UPL_DECMP_REAL_IO;
+       upl->decmp_io_upl = (void *)src_upl;

        upl_unlock(src_upl);
 }
        upl_unlock(src_upl);
 }

-#endif /* CONFIG_IOSCHED */  
+#endif /* CONFIG_IOSCHED */

 
 #if UPL_DEBUG
 
 #if UPL_DEBUG

-int    upl_debug_enabled = 1;
+int     upl_debug_enabled = 1;

 #else
 #else

-int    upl_debug_enabled = 0;
+int     upl_debug_enabled = 0;

 #endif
 
 static upl_t
 upl_create(int type, int flags, upl_size_t size)
 {
 #endif
 
 static upl_t
 upl_create(int type, int flags, upl_size_t size)
 {

-       upl_t   upl;
-       vm_size_t       page_field_size = 0;
-       int     upl_flags = 0;
-       vm_size_t       upl_size  = sizeof(struct upl);
+       upl_t   upl;
+       vm_size_t       page_field_size = 0;
+       int     upl_flags = 0;
+       vm_size_t       upl_size  = sizeof(struct upl);

 
        size = round_page_32(size);
 
 
        size = round_page_32(size);
 


@@ -4760,8 +4666,9 @@ upl_create(int type, int flags, upl_size_t size)
        }
        upl = (upl_t)kalloc(upl_size + page_field_size);
 
        }
        upl = (upl_t)kalloc(upl_size + page_field_size);
 

-       if (page_field_size)
-               bzero((char *)upl + upl_size, page_field_size);
+       if (page_field_size) {
+               bzero((char *)upl + upl_size, page_field_size);
+       }

 
        upl->flags = upl_flags | flags;
        upl->kaddr = (vm_offset_t)0;
 
        upl->flags = upl_flags | flags;
        upl->kaddr = (vm_offset_t)0;


@@ -4773,11 +4680,12 @@ upl_create(int type, int flags, upl_size_t size)
        upl_lock_init(upl);
        upl->vector_upl = NULL;
        upl->associated_upl = NULL;
        upl_lock_init(upl);
        upl->vector_upl = NULL;
        upl->associated_upl = NULL;

+       upl->upl_iodone = NULL;

 #if CONFIG_IOSCHED
        if (type & UPL_CREATE_IO_TRACKING) {
                upl->upl_priority = proc_get_effective_thread_policy(current_thread(), TASK_POLICY_IO);
        }
 #if CONFIG_IOSCHED
        if (type & UPL_CREATE_IO_TRACKING) {
                upl->upl_priority = proc_get_effective_thread_policy(current_thread(), TASK_POLICY_IO);
        }

-       
+

        upl->upl_reprio_info = 0;
        upl->decmp_io_upl = 0;
        if ((type & UPL_CREATE_INTERNAL) && (type & UPL_CREATE_EXPEDITE_SUP)) {
        upl->upl_reprio_info = 0;
        upl->decmp_io_upl = 0;
        if ((type & UPL_CREATE_INTERNAL) && (type & UPL_CREATE_EXPEDITE_SUP)) {


@@ -4786,8 +4694,9 @@ upl_create(int type, int flags, upl_size_t size)
                upl->upl_reprio_info = (uint64_t *)kalloc(sizeof(uint64_t) * atop(size));
                bzero(upl->upl_reprio_info, (sizeof(uint64_t) * atop(size)));
                upl->flags |= UPL_EXPEDITE_SUPPORTED;
                upl->upl_reprio_info = (uint64_t *)kalloc(sizeof(uint64_t) * atop(size));
                bzero(upl->upl_reprio_info, (sizeof(uint64_t) * atop(size)));
                upl->flags |= UPL_EXPEDITE_SUPPORTED;

-               if (curthread->decmp_upl != NULL) 
+               if (curthread->decmp_upl != NULL) {

                        upl_set_decmp_info(upl, curthread->decmp_upl);
                        upl_set_decmp_info(upl, curthread->decmp_upl);

+               }

        }
 #endif
 #if CONFIG_IOSCHED || UPL_DEBUG
        }
 #endif
 #if CONFIG_IOSCHED || UPL_DEBUG


@@ -4810,34 +4719,34 @@ upl_create(int type, int flags, upl_size_t size)
        (void) OSBacktrace(&upl->upl_create_retaddr[0], UPL_DEBUG_STACK_FRAMES);
 #endif /* UPL_DEBUG */
 
        (void) OSBacktrace(&upl->upl_create_retaddr[0], UPL_DEBUG_STACK_FRAMES);
 #endif /* UPL_DEBUG */
 

-       return(upl);
+       return upl;

 }
 
 static void
 upl_destroy(upl_t upl)
 {
 }
 
 static void
 upl_destroy(upl_t upl)
 {

-       int     page_field_size;  /* bit field in word size buf */
-        int    size;
+       int     page_field_size;  /* bit field in word size buf */
+       int     size;

 
        if (upl->ext_ref_count) {
                panic("upl(%p) ext_ref_count", upl);
        }
 
 #if CONFIG_IOSCHED
 
        if (upl->ext_ref_count) {
                panic("upl(%p) ext_ref_count", upl);
        }
 
 #if CONFIG_IOSCHED

-        if ((upl->flags & UPL_DECMP_REAL_IO) && upl->decmp_io_upl) {
-                upl_t src_upl;
-                src_upl = upl->decmp_io_upl;
-                assert((src_upl->flags & UPL_DECMP_REQ) != 0);
-                upl_lock(src_upl);
-                src_upl->decmp_io_upl = NULL;
-                upl_unlock(src_upl);
-                upl_deallocate(src_upl);
-        }
+       if ((upl->flags & UPL_DECMP_REAL_IO) && upl->decmp_io_upl) {
+               upl_t src_upl;
+               src_upl = upl->decmp_io_upl;
+               assert((src_upl->flags & UPL_DECMP_REQ) != 0);
+               upl_lock(src_upl);
+               src_upl->decmp_io_upl = NULL;
+               upl_unlock(src_upl);
+               upl_deallocate(src_upl);
+       }

 #endif /* CONFIG_IOSCHED */
 
 #if CONFIG_IOSCHED || UPL_DEBUG
        if ((upl->flags & UPL_TRACKED_BY_OBJECT) && !(upl->flags & UPL_VECTOR)) {
 #endif /* CONFIG_IOSCHED */
 
 #if CONFIG_IOSCHED || UPL_DEBUG
        if ((upl->flags & UPL_TRACKED_BY_OBJECT) && !(upl->flags & UPL_VECTOR)) {

-               vm_object_t     object;
+               vm_object_t     object;

 
                if (upl->flags & UPL_SHADOWED) {
                        object = upl->map_object->shadow;
 
                if (upl->flags & UPL_SHADOWED) {
                        object = upl->map_object->shadow;


@@ -4856,32 +4765,35 @@ upl_destroy(upl_t upl)
         * drop a reference on the map_object whether or
         * not a pageout object is inserted
         */
         * drop a reference on the map_object whether or
         * not a pageout object is inserted
         */

-       if (upl->flags & UPL_SHADOWED)
+       if (upl->flags & UPL_SHADOWED) {

                vm_object_deallocate(upl->map_object);
                vm_object_deallocate(upl->map_object);

+       }

 
 

-        if (upl->flags & UPL_DEVICE_MEMORY)
-               size = PAGE_SIZE;
-       else
-               size = upl->size;
+       if (upl->flags & UPL_DEVICE_MEMORY) {
+               size = PAGE_SIZE;
+       } else {
+               size = upl->size;
+       }

        page_field_size = 0;
 
        if (upl->flags & UPL_LITE) {
        page_field_size = 0;
 
        if (upl->flags & UPL_LITE) {

-               page_field_size = ((size/PAGE_SIZE) + 7) >> 3;
+               page_field_size = ((size / PAGE_SIZE) + 7) >> 3;

                page_field_size = (page_field_size + 3) & 0xFFFFFFFC;
        }
        upl_lock_destroy(upl);
        upl->vector_upl = (vector_upl_t) 0xfeedbeef;
 
 #if CONFIG_IOSCHED
                page_field_size = (page_field_size + 3) & 0xFFFFFFFC;
        }
        upl_lock_destroy(upl);
        upl->vector_upl = (vector_upl_t) 0xfeedbeef;
 
 #if CONFIG_IOSCHED

-       if (upl->flags & UPL_EXPEDITE_SUPPORTED)
-               kfree(upl->upl_reprio_info, sizeof(uint64_t) * (size/PAGE_SIZE));
+       if (upl->flags & UPL_EXPEDITE_SUPPORTED) {
+               kfree(upl->upl_reprio_info, sizeof(uint64_t) * (size / PAGE_SIZE));
+       }

 #endif
 
        if (upl->flags & UPL_INTERNAL) {
                kfree(upl,
 #endif
 
        if (upl->flags & UPL_INTERNAL) {
                kfree(upl,

-                     sizeof(struct upl) + 
-                     (sizeof(struct upl_page_info) * (size/PAGE_SIZE))
-                     + page_field_size);
+                   sizeof(struct upl) +
+                   (sizeof(struct upl_page_info) * (size / PAGE_SIZE))
+                   + page_field_size);

        } else {
                kfree(upl, sizeof(struct upl) + page_field_size);
        }
        } else {
                kfree(upl, sizeof(struct upl) + page_field_size);
        }


@@ -4891,14 +4803,21 @@ void
 upl_deallocate(upl_t upl)
 {
        upl_lock(upl);
 upl_deallocate(upl_t upl)
 {
        upl_lock(upl);

+

        if (--upl->ref_count == 0) {
        if (--upl->ref_count == 0) {

-               if(vector_upl_is_valid(upl))
+               if (vector_upl_is_valid(upl)) {

                        vector_upl_deallocate(upl);
                        vector_upl_deallocate(upl);

-               upl_unlock(upl);        
+               }
+               upl_unlock(upl);
+
+               if (upl->upl_iodone) {
+                       upl_callout_iodone(upl);
+               }
+

                upl_destroy(upl);
                upl_destroy(upl);

-       }
-       else
+       } else {

                upl_unlock(upl);
                upl_unlock(upl);

+       }

 }
 
 #if CONFIG_IOSCHED
 }
 
 #if CONFIG_IOSCHED


@@ -4908,21 +4827,21 @@ upl_mark_decmp(upl_t upl)
        if (upl->flags & UPL_TRACKED_BY_OBJECT) {
                upl->flags |= UPL_DECMP_REQ;
                upl->upl_creator->decmp_upl = (void *)upl;
        if (upl->flags & UPL_TRACKED_BY_OBJECT) {
                upl->flags |= UPL_DECMP_REQ;
                upl->upl_creator->decmp_upl = (void *)upl;

-       }       
+       }

 }
 
 void
 upl_unmark_decmp(upl_t upl)
 {
 }
 
 void
 upl_unmark_decmp(upl_t upl)
 {

-       if(upl && (upl->flags & UPL_DECMP_REQ)) {
+       if (upl && (upl->flags & UPL_DECMP_REQ)) {

                upl->upl_creator->decmp_upl = NULL;
        }
                upl->upl_creator->decmp_upl = NULL;
        }

-} 
+}

 
 #endif /* CONFIG_IOSCHED */
 
 
 #endif /* CONFIG_IOSCHED */
 

-#define VM_PAGE_Q_BACKING_UP(q)                \
-        ((q)->pgo_laundry >= (((q)->pgo_maxlaundry * 8) / 10))
+#define VM_PAGE_Q_BACKING_UP(q)         \
+       ((q)->pgo_laundry >= (((q)->pgo_maxlaundry * 8) / 10))

 
 boolean_t must_throttle_writes(void);
 
 
 boolean_t must_throttle_writes(void);
 


@@ -4930,29 +4849,17 @@ boolean_t
 must_throttle_writes()
 {
        if (VM_PAGE_Q_BACKING_UP(&vm_pageout_queue_external) &&
 must_throttle_writes()
 {
        if (VM_PAGE_Q_BACKING_UP(&vm_pageout_queue_external) &&

-           vm_page_pageable_external_count > (AVAILABLE_NON_COMPRESSED_MEMORY * 6) / 10)
-               return (TRUE);
+           vm_page_pageable_external_count > (AVAILABLE_NON_COMPRESSED_MEMORY * 6) / 10) {
+               return TRUE;
+       }

 
 

-       return (FALSE);
+       return FALSE;

 }
 
 
 }
 
 

-#if DEVELOPMENT || DEBUG
-/*/*
- * Statistics about UPL enforcement of copy-on-write obligations.
- */
-unsigned long upl_cow = 0;
-unsigned long upl_cow_again = 0;
-unsigned long upl_cow_pages = 0;
-unsigned long upl_cow_again_pages = 0;
-
-unsigned long iopl_cow = 0;
-unsigned long iopl_cow_pages = 0;
-#endif
-
-/*  
- *     Routine:        vm_object_upl_request 
- *     Purpose:        
+/*
+ *     Routine:        vm_object_upl_request
+ *     Purpose:

  *             Cause the population of a portion of a vm_object.
  *             Depending on the nature of the request, the pages
  *             returned may be contain valid data or be uninitialized.
  *             Cause the population of a portion of a vm_object.
  *             Depending on the nature of the request, the pages
  *             returned may be contain valid data or be uninitialized.


@@ -4963,7 +4870,7 @@ unsigned long iopl_cow_pages = 0;
  *             IMPORTANT NOTE: The caller must still respect the relationship
  *             between the vm_object and its backing memory object.  The
  *             caller MUST NOT substitute changes in the backing file
  *             IMPORTANT NOTE: The caller must still respect the relationship
  *             between the vm_object and its backing memory object.  The
  *             caller MUST NOT substitute changes in the backing file

- *             without first doing a memory_object_lock_request on the 
+ *             without first doing a memory_object_lock_request on the

  *             target range unless it is know that the pages are not
  *             shared with another entity at the pager level.
  *             Copy_in_to:
  *             target range unless it is know that the pages are not
  *             shared with another entity at the pager level.
  *             Copy_in_to:


@@ -4981,7 +4888,7 @@ unsigned long iopl_cow_pages = 0;
  *                     all mapped pages.  Where a page does not exist
  *                     map a zero filled one. Leave pages busy in
  *                     the original object.  If a page list structure
  *                     all mapped pages.  Where a page does not exist
  *                     map a zero filled one. Leave pages busy in
  *                     the original object.  If a page list structure

- *                     is not specified, this call is a no-op. 
+ *                     is not specified, this call is a no-op.

  *
  *             Note:  access of default pager objects has a rather interesting
  *             twist.  The caller of this routine, presumably the file system
  *
  *             Note:  access of default pager objects has a rather interesting
  *             twist.  The caller of this routine, presumably the file system


@@ -4989,44 +4896,47 @@ unsigned long iopl_cow_pages = 0;
  *             against a default pager backed object.  Only the default
  *             pager will make requests on backing store related vm_objects
  *             In this way the default pager can maintain the relationship
  *             against a default pager backed object.  Only the default
  *             pager will make requests on backing store related vm_objects
  *             In this way the default pager can maintain the relationship

- *             between backing store files (abstract memory objects) and 
+ *             between backing store files (abstract memory objects) and

  *             the vm_objects (cache objects), they support.
  *
  */
 
 __private_extern__ kern_return_t
 vm_object_upl_request(
  *             the vm_objects (cache objects), they support.
  *
  */
 
 __private_extern__ kern_return_t
 vm_object_upl_request(

-       vm_object_t             object,
-       vm_object_offset_t      offset,
-       upl_size_t              size,
-       upl_t                   *upl_ptr,
-       upl_page_info_array_t   user_page_list,
-       unsigned int            *page_list_count,
-       upl_control_flags_t     cntrl_flags,
-       vm_tag_t                tag)
+       vm_object_t             object,
+       vm_object_offset_t      offset,
+       upl_size_t              size,
+       upl_t                   *upl_ptr,
+       upl_page_info_array_t   user_page_list,
+       unsigned int            *page_list_count,
+       upl_control_flags_t     cntrl_flags,
+       vm_tag_t                tag)

 {
 {

-       vm_page_t               dst_page = VM_PAGE_NULL;
-       vm_object_offset_t      dst_offset;
-       upl_size_t              xfer_size;
-       unsigned int            size_in_pages;
-       boolean_t               dirty;
-       boolean_t               hw_dirty;
-       upl_t                   upl = NULL;
-       unsigned int            entry;
-#if MACH_CLUSTER_STATS
-       boolean_t               encountered_lrp = FALSE;
-#endif
-       vm_page_t               alias_page = NULL;
-        int                    refmod_state = 0;
-       wpl_array_t             lite_list = NULL;
-       vm_object_t             last_copy_object;
-       struct  vm_page_delayed_work    dw_array[DEFAULT_DELAYED_WORK_LIMIT];
-       struct  vm_page_delayed_work    *dwp;
-       int                     dw_count;
-       int                     dw_limit;
-       int                     io_tracking_flag = 0;
-       int                     grab_options;
-       ppnum_t                 phys_page;
+       vm_page_t               dst_page = VM_PAGE_NULL;
+       vm_object_offset_t      dst_offset;
+       upl_size_t              xfer_size;
+       unsigned int            size_in_pages;
+       boolean_t               dirty;
+       boolean_t               hw_dirty;
+       upl_t                   upl = NULL;
+       unsigned int            entry;
+       vm_page_t               alias_page = NULL;
+       int                     refmod_state = 0;
+       wpl_array_t             lite_list = NULL;
+       vm_object_t             last_copy_object;
+       struct  vm_page_delayed_work    dw_array[DEFAULT_DELAYED_WORK_LIMIT];
+       struct  vm_page_delayed_work    *dwp;
+       int                     dw_count;
+       int                     dw_limit;
+       int                     io_tracking_flag = 0;
+       int                     grab_options;
+       int                     page_grab_count = 0;
+       ppnum_t                 phys_page;
+       pmap_flush_context      pmap_flush_context_storage;
+       boolean_t               pmap_flushes_delayed = FALSE;
+#if DEVELOPMENT || DEBUG
+       task_t                  task = current_task();
+#endif /* DEVELOPMENT || DEBUG */

 
        if (cntrl_flags & ~UPL_VALID_FLAGS) {
                /*
 
        if (cntrl_flags & ~UPL_VALID_FLAGS) {
                /*


@@ -5035,42 +4945,48 @@ vm_object_upl_request(
                 */
                return KERN_INVALID_VALUE;
        }
                 */
                return KERN_INVALID_VALUE;
        }

-       if ( (!object->internal) && (object->paging_offset != 0) )
+       if ((!object->internal) && (object->paging_offset != 0)) {

                panic("vm_object_upl_request: external object with non-zero paging offset\n");
                panic("vm_object_upl_request: external object with non-zero paging offset\n");

-       if (object->phys_contiguous)
-               panic("vm_object_upl_request: contiguous object specified\n");
+       }
+       if (object->phys_contiguous) {
+               panic("vm_object_upl_request: contiguous object specified\n");
+       }

 
 

+       VM_DEBUG_CONSTANT_EVENT(vm_object_upl_request, VM_UPL_REQUEST, DBG_FUNC_START, size, cntrl_flags, 0, 0);

 
 

-       if (size > MAX_UPL_SIZE_BYTES)
+       if (size > MAX_UPL_SIZE_BYTES) {

                size = MAX_UPL_SIZE_BYTES;
                size = MAX_UPL_SIZE_BYTES;

+       }

 
 

-       if ( (cntrl_flags & UPL_SET_INTERNAL) && page_list_count != NULL)
-               *page_list_count = MAX_UPL_SIZE_BYTES >> PAGE_SHIFT;
+       if ((cntrl_flags & UPL_SET_INTERNAL) && page_list_count != NULL) {
+               *page_list_count = MAX_UPL_SIZE_BYTES >> PAGE_SHIFT;
+       }

 
 #if CONFIG_IOSCHED || UPL_DEBUG
 
 #if CONFIG_IOSCHED || UPL_DEBUG

-       if (object->io_tracking || upl_debug_enabled)
+       if (object->io_tracking || upl_debug_enabled) {

                io_tracking_flag |= UPL_CREATE_IO_TRACKING;
                io_tracking_flag |= UPL_CREATE_IO_TRACKING;

+       }

 #endif
 #if CONFIG_IOSCHED
 #endif
 #if CONFIG_IOSCHED

-       if (object->io_tracking)
+       if (object->io_tracking) {

                io_tracking_flag |= UPL_CREATE_EXPEDITE_SUP;
                io_tracking_flag |= UPL_CREATE_EXPEDITE_SUP;

+       }

 #endif
 
        if (cntrl_flags & UPL_SET_INTERNAL) {
 #endif
 
        if (cntrl_flags & UPL_SET_INTERNAL) {

-               if (cntrl_flags & UPL_SET_LITE) {
-
+               if (cntrl_flags & UPL_SET_LITE) {

                        upl = upl_create(UPL_CREATE_INTERNAL | UPL_CREATE_LITE | io_tracking_flag, 0, size);
 
                        user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl));
                        lite_list = (wpl_array_t)
                        upl = upl_create(UPL_CREATE_INTERNAL | UPL_CREATE_LITE | io_tracking_flag, 0, size);
 
                        user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl));
                        lite_list = (wpl_array_t)

-                                       (((uintptr_t)user_page_list) + 
-                                       ((size/PAGE_SIZE) * sizeof(upl_page_info_t)));
+                           (((uintptr_t)user_page_list) +
+                           ((size / PAGE_SIZE) * sizeof(upl_page_info_t)));

                        if (size == 0) {
                                user_page_list = NULL;
                                lite_list = NULL;
                        }
                } else {
                        if (size == 0) {
                                user_page_list = NULL;
                                lite_list = NULL;
                        }
                } else {

-                       upl = upl_create(UPL_CREATE_INTERNAL | io_tracking_flag, 0, size);
+                       upl = upl_create(UPL_CREATE_INTERNAL | io_tracking_flag, 0, size);

 
                        user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl));
                        if (size == 0) {
 
                        user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl));
                        if (size == 0) {


@@ -5078,8 +4994,7 @@ vm_object_upl_request(
                        }
                }
        } else {
                        }
                }
        } else {

-               if (cntrl_flags & UPL_SET_LITE) {
-
+               if (cntrl_flags & UPL_SET_LITE) {

                        upl = upl_create(UPL_CREATE_EXTERNAL | UPL_CREATE_LITE | io_tracking_flag, 0, size);
 
                        lite_list = (wpl_array_t) (((uintptr_t)upl) + sizeof(struct upl));
                        upl = upl_create(UPL_CREATE_EXTERNAL | UPL_CREATE_LITE | io_tracking_flag, 0, size);
 
                        lite_list = (wpl_array_t) (((uintptr_t)upl) + sizeof(struct upl));


@@ -5087,18 +5002,19 @@ vm_object_upl_request(
                                lite_list = NULL;
                        }
                } else {
                                lite_list = NULL;
                        }
                } else {

-                       upl = upl_create(UPL_CREATE_EXTERNAL | io_tracking_flag, 0, size);
+                       upl = upl_create(UPL_CREATE_EXTERNAL | io_tracking_flag, 0, size);

                }
        }
        *upl_ptr = upl;
                }
        }
        *upl_ptr = upl;

-       
-       if (user_page_list)
-               user_page_list[0].device = FALSE;
+
+       if (user_page_list) {
+               user_page_list[0].device = FALSE;
+       }

 
        if (cntrl_flags & UPL_SET_LITE) {
 
        if (cntrl_flags & UPL_SET_LITE) {

-               upl->map_object = object;
+               upl->map_object = object;

        } else {
        } else {

-               upl->map_object = vm_object_allocate(size);
+               upl->map_object = vm_object_allocate(size);

                /*
                 * No neeed to lock the new object: nobody else knows
                 * about it yet, so it's all ours so far.
                /*
                 * No neeed to lock the new object: nobody else knows
                 * about it yet, so it's all ours so far.


@@ -5114,8 +5030,9 @@ vm_object_upl_request(
 
                upl->flags |= UPL_SHADOWED;
        }
 
                upl->flags |= UPL_SHADOWED;
        }

-       if (cntrl_flags & UPL_FOR_PAGEOUT)
+       if (cntrl_flags & UPL_FOR_PAGEOUT) {

                upl->flags |= UPL_PAGEOUT;
                upl->flags |= UPL_PAGEOUT;

+       }

 
        vm_object_lock(object);
        vm_object_activity_begin(object);
 
        vm_object_lock(object);
        vm_object_activity_begin(object);


@@ -5150,17 +5067,16 @@ vm_object_upl_request(
                 * the caller modify them.
                 */
                vm_object_update(object,
                 * the caller modify them.
                 */
                vm_object_update(object,

-                                offset,
-                                size,
-                                NULL,
-                                NULL,
-                                FALSE, /* should_return */
-                                MEMORY_OBJECT_COPY_SYNC,
-                                VM_PROT_NO_CHANGE);
-#if DEVELOPMENT || DEBUG
-               upl_cow++;
-               upl_cow_pages += size >> PAGE_SHIFT;
-#endif
+                   offset,
+                   size,
+                   NULL,
+                   NULL,
+                   FALSE,              /* should_return */
+                   MEMORY_OBJECT_COPY_SYNC,
+                   VM_PROT_NO_CHANGE);
+
+               VM_PAGEOUT_DEBUG(upl_cow, 1);
+               VM_PAGEOUT_DEBUG(upl_cow_pages, (size >> PAGE_SHIFT));

        }
        /*
         * remember which copy object we synchronized with
        }
        /*
         * remember which copy object we synchronized with


@@ -5177,11 +5093,12 @@ vm_object_upl_request(
        dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT);
 
        if (vm_page_free_count > (vm_page_free_target + size_in_pages) ||
        dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT);
 
        if (vm_page_free_count > (vm_page_free_target + size_in_pages) ||

-           object->resident_page_count < ((MAX_UPL_SIZE_BYTES * 2) >> PAGE_SHIFT))
+           object->resident_page_count < ((MAX_UPL_SIZE_BYTES * 2) >> PAGE_SHIFT)) {

                object->scan_collisions = 0;
                object->scan_collisions = 0;

+       }

 
        if ((cntrl_flags & UPL_WILL_MODIFY) && must_throttle_writes() == TRUE) {
 
        if ((cntrl_flags & UPL_WILL_MODIFY) && must_throttle_writes() == TRUE) {

-               boolean_t       isSSD = FALSE;
+               boolean_t       isSSD = FALSE;

 
 #if CONFIG_EMBEDDED
                isSSD = TRUE;
 
 #if CONFIG_EMBEDDED
                isSSD = TRUE;


@@ -5189,20 +5106,20 @@ vm_object_upl_request(
                vnode_pager_get_isSSD(object->pager, &isSSD);
 #endif
                vm_object_unlock(object);
                vnode_pager_get_isSSD(object->pager, &isSSD);
 #endif
                vm_object_unlock(object);

-               
+

                OSAddAtomic(size_in_pages, &vm_upl_wait_for_pages);
 
                OSAddAtomic(size_in_pages, &vm_upl_wait_for_pages);
 

-               if (isSSD == TRUE)
+               if (isSSD == TRUE) {

                        delay(1000 * size_in_pages);
                        delay(1000 * size_in_pages);

-               else
+               } else {

                        delay(5000 * size_in_pages);
                        delay(5000 * size_in_pages);

+               }

                OSAddAtomic(-size_in_pages, &vm_upl_wait_for_pages);
 
                vm_object_lock(object);
        }
 
        while (xfer_size) {
                OSAddAtomic(-size_in_pages, &vm_upl_wait_for_pages);
 
                vm_object_lock(object);
        }
 
        while (xfer_size) {

-

                dwp->dw_mask = 0;
 
                if ((alias_page == NULL) && !(cntrl_flags & UPL_SET_LITE)) {
                dwp->dw_mask = 0;
 
                if ((alias_page == NULL) && !(cntrl_flags & UPL_SET_LITE)) {


@@ -5211,17 +5128,17 @@ vm_object_upl_request(
                        vm_object_lock(object);
                }
                if (cntrl_flags & UPL_COPYOUT_FROM) {
                        vm_object_lock(object);
                }
                if (cntrl_flags & UPL_COPYOUT_FROM) {

-                       upl->flags |= UPL_PAGE_SYNC_DONE;
-
-                       if ( ((dst_page = vm_page_lookup(object, dst_offset)) == VM_PAGE_NULL) ||
-                               dst_page->fictitious ||
-                               dst_page->absent ||
-                               dst_page->error ||
-                               dst_page->cleaning ||
-                               (VM_PAGE_WIRED(dst_page))) {
-                               
-                               if (user_page_list)
+                       upl->flags |= UPL_PAGE_SYNC_DONE;
+
+                       if (((dst_page = vm_page_lookup(object, dst_offset)) == VM_PAGE_NULL) ||
+                           dst_page->vmp_fictitious ||
+                           dst_page->vmp_absent ||
+                           dst_page->vmp_error ||
+                           dst_page->vmp_cleaning ||
+                           (VM_PAGE_WIRED(dst_page))) {
+                               if (user_page_list) {

                                        user_page_list[entry].phys_addr = 0;
                                        user_page_list[entry].phys_addr = 0;

+                               }

 
                                goto try_next_page;
                        }
 
                                goto try_next_page;
                        }


@@ -5234,13 +5151,14 @@ vm_object_upl_request(
                         * anyway... so we can eliminate an extra call into
                         * the pmap layer by grabbing it here and recording it
                         */
                         * anyway... so we can eliminate an extra call into
                         * the pmap layer by grabbing it here and recording it
                         */

-                       if (dst_page->pmapped)
-                               refmod_state = pmap_get_refmod(phys_page);
-                       else
-                               refmod_state = 0;
+                       if (dst_page->vmp_pmapped) {
+                               refmod_state = pmap_get_refmod(phys_page);
+                       } else {
+                               refmod_state = 0;
+                       }

 
 

-                       if ( (refmod_state & VM_MEM_REFERENCED) && VM_PAGE_INACTIVE(dst_page)) {
-                               /*
+                       if ((refmod_state & VM_MEM_REFERENCED) && VM_PAGE_INACTIVE(dst_page)) {
+                               /*

                                 * page is on inactive list and referenced...
                                 * reactivate it now... this gets it out of the
                                 * way of vm_pageout_scan which would have to
                                 * page is on inactive list and referenced...
                                 * reactivate it now... this gets it out of the
                                 * way of vm_pageout_scan which would have to


@@ -5249,19 +5167,20 @@ vm_object_upl_request(
                                dwp->dw_mask |= DW_vm_page_activate;
                        }
                        if (cntrl_flags & UPL_RET_ONLY_DIRTY) {
                                dwp->dw_mask |= DW_vm_page_activate;
                        }
                        if (cntrl_flags & UPL_RET_ONLY_DIRTY) {

-                               /*
+                               /*

                                 * we're only asking for DIRTY pages to be returned
                                 */
                                 * we're only asking for DIRTY pages to be returned
                                 */

-                               if (dst_page->laundry || !(cntrl_flags & UPL_FOR_PAGEOUT)) {
-                                       /*
+                               if (dst_page->vmp_laundry || !(cntrl_flags & UPL_FOR_PAGEOUT)) {
+                                       /*

                                         * if we were the page stolen by vm_pageout_scan to be
                                         * if we were the page stolen by vm_pageout_scan to be

-                                        * cleaned (as opposed to a buddy being clustered in 
+                                        * cleaned (as opposed to a buddy being clustered in

                                         * or this request is not being driven by a PAGEOUT cluster
                                         * then we only need to check for the page being dirty or
                                         * precious to decide whether to return it
                                         */
                                         * or this request is not being driven by a PAGEOUT cluster
                                         * then we only need to check for the page being dirty or
                                         * precious to decide whether to return it
                                         */

-                                       if (dst_page->dirty || dst_page->precious || (refmod_state & VM_MEM_MODIFIED))
-                                               goto check_busy;
+                                       if (dst_page->vmp_dirty || dst_page->vmp_precious || (refmod_state & VM_MEM_MODIFIED)) {
+                                               goto check_busy;
+                                       }

                                        goto dont_return;
                                }
                                /*
                                        goto dont_return;
                                }
                                /*


@@ -5270,18 +5189,18 @@ vm_object_upl_request(
                                 * does it have to be dirty to be returned, but it also
                                 * can't have been referenced recently...
                                 */
                                 * does it have to be dirty to be returned, but it also
                                 * can't have been referenced recently...
                                 */

-                               if ( (hibernate_cleaning_in_progress == TRUE ||
-                                     (!((refmod_state & VM_MEM_REFERENCED) || dst_page->reference) ||
-                                      (dst_page->vm_page_q_state == VM_PAGE_ON_THROTTLED_Q))) && 
-                                    ((refmod_state & VM_MEM_MODIFIED) || dst_page->dirty || dst_page->precious) ) {
-                                       goto check_busy;
+                               if ((hibernate_cleaning_in_progress == TRUE ||
+                                   (!((refmod_state & VM_MEM_REFERENCED) || dst_page->vmp_reference) ||
+                                   (dst_page->vmp_q_state == VM_PAGE_ON_THROTTLED_Q))) &&
+                                   ((refmod_state & VM_MEM_MODIFIED) || dst_page->vmp_dirty || dst_page->vmp_precious)) {
+                                       goto check_busy;

                                }
 dont_return:
                                /*
                                 * if we reach here, we're not to return
                                 * the page... go on to the next one
                                 */
                                }
 dont_return:
                                /*
                                 * if we reach here, we're not to return
                                 * the page... go on to the next one
                                 */

-                               if (dst_page->laundry == TRUE) {
+                               if (dst_page->vmp_laundry == TRUE) {

                                        /*
                                         * if we get here, the page is not 'cleaning' (filtered out above).
                                         * since it has been referenced, remove it from the laundry
                                        /*
                                         * if we get here, the page is not 'cleaning' (filtered out above).
                                         * since it has been referenced, remove it from the laundry


@@ -5292,19 +5211,21 @@ dont_return:
 
                                        vm_pageout_steal_laundry(dst_page, TRUE);
                                        vm_page_activate(dst_page);
 
                                        vm_pageout_steal_laundry(dst_page, TRUE);
                                        vm_page_activate(dst_page);

-                                       
+

                                        vm_page_unlock_queues();
                                }
                                        vm_page_unlock_queues();
                                }

-                               if (user_page_list)
-                                       user_page_list[entry].phys_addr = 0;
+                               if (user_page_list) {
+                                       user_page_list[entry].phys_addr = 0;
+                               }

 
                                goto try_next_page;
                        }
 
                                goto try_next_page;
                        }

-check_busy:                    
-                       if (dst_page->busy) {
-                               if (cntrl_flags & UPL_NOBLOCK) {        
-                                       if (user_page_list)
-                                               user_page_list[entry].phys_addr = 0;
+check_busy:
+                       if (dst_page->vmp_busy) {
+                               if (cntrl_flags & UPL_NOBLOCK) {
+                                       if (user_page_list) {
+                                               user_page_list[entry].phys_addr = 0;
+                                       }

                                        dwp->dw_mask = 0;
 
                                        goto try_next_page;
                                        dwp->dw_mask = 0;
 
                                        goto try_next_page;


@@ -5317,11 +5238,10 @@ check_busy:
 
                                continue;
                        }
 
                                continue;
                        }

-                       if (dst_page->vm_page_q_state == VM_PAGE_ON_PAGEOUT_Q) {
-
+                       if (dst_page->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q) {

                                vm_page_lockspin_queues();
 
                                vm_page_lockspin_queues();
 

-                               if (dst_page->vm_page_q_state == VM_PAGE_ON_PAGEOUT_Q) {
+                               if (dst_page->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q) {

                                        /*
                                         * we've buddied up a page for a clustered pageout
                                         * that has already been moved to the pageout
                                        /*
                                         * we've buddied up a page for a clustered pageout
                                         * that has already been moved to the pageout


@@ -5333,71 +5253,66 @@ check_busy:
                                }
                                vm_page_unlock_queues();
                        }
                                }
                                vm_page_unlock_queues();
                        }

-#if MACH_CLUSTER_STATS
-                       /*
-                        * pageout statistics gathering.  count
-                        * all the pages we will page out that
-                        * were not counted in the initial
-                        * vm_pageout_scan work
-                        */
-                       if (dst_page->pageout)
-                               encountered_lrp = TRUE;
-                       if ((dst_page->dirty || (object->internal && dst_page->precious))) {
-                               if (encountered_lrp)
-                                       CLUSTER_STAT(pages_at_higher_offsets++;)
-                               else
-                                       CLUSTER_STAT(pages_at_lower_offsets++;)
-                       }
-#endif

                        hw_dirty = refmod_state & VM_MEM_MODIFIED;
                        hw_dirty = refmod_state & VM_MEM_MODIFIED;

-                       dirty = hw_dirty ? TRUE : dst_page->dirty;
+                       dirty = hw_dirty ? TRUE : dst_page->vmp_dirty;

 
 

-                       if (phys_page > upl->highest_page)
-                               upl->highest_page = phys_page;
+                       if (phys_page > upl->highest_page) {
+                               upl->highest_page = phys_page;
+                       }

 
 

-                       assert (!pmap_is_noencrypt(phys_page));
+                       assert(!pmap_is_noencrypt(phys_page));

 
                        if (cntrl_flags & UPL_SET_LITE) {
 
                        if (cntrl_flags & UPL_SET_LITE) {

-                               unsigned int    pg_num;
+                               unsigned int    pg_num;

 
 

-                               pg_num = (unsigned int) ((dst_offset-offset)/PAGE_SIZE);
-                               assert(pg_num == (dst_offset-offset)/PAGE_SIZE);
-                               lite_list[pg_num>>5] |= 1 << (pg_num & 31);
+                               pg_num = (unsigned int) ((dst_offset - offset) / PAGE_SIZE);
+                               assert(pg_num == (dst_offset - offset) / PAGE_SIZE);
+                               lite_list[pg_num >> 5] |= 1 << (pg_num & 31);

 
 

-                               if (hw_dirty)
-                                       pmap_clear_modify(phys_page);
+                               if (hw_dirty) {
+                                       if (pmap_flushes_delayed == FALSE) {
+                                               pmap_flush_context_init(&pmap_flush_context_storage);
+                                               pmap_flushes_delayed = TRUE;
+                                       }
+                                       pmap_clear_refmod_options(phys_page,
+                                           VM_MEM_MODIFIED,
+                                           PMAP_OPTIONS_NOFLUSH | PMAP_OPTIONS_CLEAR_WRITE,
+                                           &pmap_flush_context_storage);
+                               }

 
                                /*
 
                                /*

-                                * Mark original page as cleaning 
+                                * Mark original page as cleaning

                                 * in place.
                                 */
                                 * in place.
                                 */

-                               dst_page->cleaning = TRUE;
-                               dst_page->precious = FALSE;
+                               dst_page->vmp_cleaning = TRUE;
+                               dst_page->vmp_precious = FALSE;

                        } else {
                        } else {

-                               /*
+                               /*

                                 * use pageclean setup, it is more
                                 * convenient even for the pageout
                                 * cases here
                                 */
                                 * use pageclean setup, it is more
                                 * convenient even for the pageout
                                 * cases here
                                 */

-                               vm_object_lock(upl->map_object);
+                               vm_object_lock(upl->map_object);

                                vm_pageclean_setup(dst_page, alias_page, upl->map_object, size - xfer_size);
                                vm_object_unlock(upl->map_object);
 
                                vm_pageclean_setup(dst_page, alias_page, upl->map_object, size - xfer_size);
                                vm_object_unlock(upl->map_object);
 

-                               alias_page->absent = FALSE;
+                               alias_page->vmp_absent = FALSE;

                                alias_page = NULL;
                        }
                        if (dirty) {
                                SET_PAGE_DIRTY(dst_page, FALSE);
                        } else {
                                alias_page = NULL;
                        }
                        if (dirty) {
                                SET_PAGE_DIRTY(dst_page, FALSE);
                        } else {

-                               dst_page->dirty = FALSE;
+                               dst_page->vmp_dirty = FALSE;

                        }
 
                        }
 

-                       if (!dirty)
-                               dst_page->precious = TRUE;
+                       if (!dirty) {
+                               dst_page->vmp_precious = TRUE;
+                       }

 
 

-                       if ( !(cntrl_flags & UPL_CLEAN_IN_PLACE) ) {
-                               if ( !VM_PAGE_WIRED(dst_page))
-                                       dst_page->free_when_done = TRUE;
+                       if (!(cntrl_flags & UPL_CLEAN_IN_PLACE)) {
+                               if (!VM_PAGE_WIRED(dst_page)) {
+                                       dst_page->vmp_free_when_done = TRUE;
+                               }

                        }
                } else {
                        if ((cntrl_flags & UPL_WILL_MODIFY) && object->copy != last_copy_object) {
                        }
                } else {
                        if ((cntrl_flags & UPL_WILL_MODIFY) && object->copy != last_copy_object) {


@@ -5429,14 +5344,12 @@ check_busy:
                                                xfer_size, /* remaining size */
                                                NULL,
                                                NULL,
                                                xfer_size, /* remaining size */
                                                NULL,
                                                NULL,

-                                               FALSE,     /* should_return */
+                                               FALSE,     /* should_return */

                                                MEMORY_OBJECT_COPY_SYNC,
                                                VM_PROT_NO_CHANGE);
 
                                                MEMORY_OBJECT_COPY_SYNC,
                                                VM_PROT_NO_CHANGE);
 

-#if DEVELOPMENT || DEBUG
-                                       upl_cow_again++;
-                                       upl_cow_again_pages += xfer_size >> PAGE_SHIFT;
-#endif
+                                       VM_PAGEOUT_DEBUG(upl_cow_again, 1);
+                                       VM_PAGEOUT_DEBUG(upl_cow_again_pages, (xfer_size >> PAGE_SHIFT));

                                }
                                /*
                                 * remember the copy object we synced with
                                }
                                /*
                                 * remember the copy object we synced with


@@ -5444,23 +5357,23 @@ check_busy:
                                last_copy_object = object->copy;
                        }
                        dst_page = vm_page_lookup(object, dst_offset);
                                last_copy_object = object->copy;
                        }
                        dst_page = vm_page_lookup(object, dst_offset);

-                       
-                       if (dst_page != VM_PAGE_NULL) {

 
 

+                       if (dst_page != VM_PAGE_NULL) {

                                if ((cntrl_flags & UPL_RET_ONLY_ABSENT)) {
                                        /*
                                         * skip over pages already present in the cache
                                         */
                                if ((cntrl_flags & UPL_RET_ONLY_ABSENT)) {
                                        /*
                                         * skip over pages already present in the cache
                                         */

-                                       if (user_page_list)
+                                       if (user_page_list) {

                                                user_page_list[entry].phys_addr = 0;
                                                user_page_list[entry].phys_addr = 0;

+                                       }

 
                                        goto try_next_page;
                                }
 
                                        goto try_next_page;
                                }

-                               if (dst_page->fictitious) {
+                               if (dst_page->vmp_fictitious) {

                                        panic("need corner case for fictitious page");
                                }
 
                                        panic("need corner case for fictitious page");
                                }
 

-                               if (dst_page->busy || dst_page->cleaning) {
+                               if (dst_page->vmp_busy || dst_page->vmp_cleaning) {

                                        /*
                                         * someone else is playing with the
                                         * page.  We will have to wait.
                                        /*
                                         * someone else is playing with the
                                         * page.  We will have to wait.


@@ -5469,21 +5382,23 @@ check_busy:
 
                                        continue;
                                }
 
                                        continue;
                                }

-                               if (dst_page->laundry)
+                               if (dst_page->vmp_laundry) {

                                        vm_pageout_steal_laundry(dst_page, FALSE);
                                        vm_pageout_steal_laundry(dst_page, FALSE);

+                               }

                        } else {
                                if (object->private) {
                        } else {
                                if (object->private) {

-                                       /* 
-                                        * This is a nasty wrinkle for users 
-                                        * of upl who encounter device or 
-                                        * private memory however, it is 
+                                       /*
+                                        * This is a nasty wrinkle for users
+                                        * of upl who encounter device or
+                                        * private memory however, it is

                                         * unavoidable, only a fault can
                                         * resolve the actual backing
                                         * physical page by asking the
                                         * backing device.
                                         */
                                         * unavoidable, only a fault can
                                         * resolve the actual backing
                                         * physical page by asking the
                                         * backing device.
                                         */

-                                       if (user_page_list)
+                                       if (user_page_list) {

                                                user_page_list[entry].phys_addr = 0;
                                                user_page_list[entry].phys_addr = 0;

+                                       }

 
                                        goto try_next_page;
                                }
 
                                        goto try_next_page;
                                }


@@ -5501,9 +5416,10 @@ check_busy:
                                         */
                                        dst_page = vm_object_page_grab(object);
 
                                         */
                                        dst_page = vm_object_page_grab(object);
 

-                                       if (dst_page != VM_PAGE_NULL)
+                                       if (dst_page != VM_PAGE_NULL) {

                                                vm_page_release(dst_page,
                                                vm_page_release(dst_page,

-                                                               FALSE);
+                                                   FALSE);
+                                       }

 
                                        dst_page = vm_object_page_grab(object);
                                }
 
                                        dst_page = vm_object_page_grab(object);
                                }


@@ -5512,26 +5428,30 @@ check_busy:
                                         * need to allocate a page
                                         */
                                        dst_page = vm_page_grab_options(grab_options);
                                         * need to allocate a page
                                         */
                                        dst_page = vm_page_grab_options(grab_options);

+                                       if (dst_page != VM_PAGE_NULL) {
+                                               page_grab_count++;
+                                       }

                                }
                                if (dst_page == VM_PAGE_NULL) {
                                }
                                if (dst_page == VM_PAGE_NULL) {

-                                       if ( (cntrl_flags & (UPL_RET_ONLY_ABSENT | UPL_NOBLOCK)) == (UPL_RET_ONLY_ABSENT | UPL_NOBLOCK)) {
-                                              /*
-                                               * we don't want to stall waiting for pages to come onto the free list
-                                               * while we're already holding absent pages in this UPL
-                                               * the caller will deal with the empty slots
-                                               */
-                                               if (user_page_list)
-                                                       user_page_list[entry].phys_addr = 0;
+                                       if ((cntrl_flags & (UPL_RET_ONLY_ABSENT | UPL_NOBLOCK)) == (UPL_RET_ONLY_ABSENT | UPL_NOBLOCK)) {
+                                               /*
+                                                * we don't want to stall waiting for pages to come onto the free list
+                                                * while we're already holding absent pages in this UPL
+                                                * the caller will deal with the empty slots
+                                                */
+                                               if (user_page_list) {
+                                                       user_page_list[entry].phys_addr = 0;
+                                               }

 
                                                goto try_next_page;
                                        }
 
                                                goto try_next_page;
                                        }

-                                       /*
+                                       /*

                                         * no pages available... wait
                                         * then try again for the same
                                         * offset...
                                         */
                                        vm_object_unlock(object);
                                         * no pages available... wait
                                         * then try again for the same
                                         * offset...
                                         */
                                        vm_object_unlock(object);

-                                       
+

                                        OSAddAtomic(size_in_pages, &vm_upl_wait_for_pages);
 
                                        VM_DEBUG_EVENT(vm_upl_page_wait, VM_UPL_PAGE_WAIT, DBG_FUNC_START, vm_upl_wait_for_pages, 0, 0, 0);
                                        OSAddAtomic(size_in_pages, &vm_upl_wait_for_pages);
 
                                        VM_DEBUG_EVENT(vm_upl_page_wait, VM_UPL_PAGE_WAIT, DBG_FUNC_START, vm_upl_wait_for_pages, 0, 0, 0);


@@ -5547,70 +5467,74 @@ check_busy:
                                }
                                vm_page_insert(dst_page, object, dst_offset);
 
                                }
                                vm_page_insert(dst_page, object, dst_offset);
 

-                               dst_page->absent = TRUE;
-                               dst_page->busy = FALSE;
+                               dst_page->vmp_absent = TRUE;
+                               dst_page->vmp_busy = FALSE;

 
                                if (cntrl_flags & UPL_RET_ONLY_ABSENT) {
 
                                if (cntrl_flags & UPL_RET_ONLY_ABSENT) {

-                                       /*
+                                       /*

                                         * if UPL_RET_ONLY_ABSENT was specified,
                                         * than we're definitely setting up a
                                         * if UPL_RET_ONLY_ABSENT was specified,
                                         * than we're definitely setting up a

-                                        * upl for a clustered read/pagein 
+                                        * upl for a clustered read/pagein

                                         * operation... mark the pages as clustered
                                         * so upl_commit_range can put them on the
                                         * speculative list
                                         */
                                         * operation... mark the pages as clustered
                                         * so upl_commit_range can put them on the
                                         * speculative list
                                         */

-                                       dst_page->clustered = TRUE;
+                                       dst_page->vmp_clustered = TRUE;

 
 

-                                       if ( !(cntrl_flags & UPL_FILE_IO))
+                                       if (!(cntrl_flags & UPL_FILE_IO)) {

                                                VM_STAT_INCR(pageins);
                                                VM_STAT_INCR(pageins);

+                                       }

                                }
                        }
                        phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page);
 
                                }
                        }
                        phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page);
 

-                       dst_page->overwriting = TRUE;
+                       dst_page->vmp_overwriting = TRUE;

 
 

-                       if (dst_page->pmapped) {
-                               if ( !(cntrl_flags & UPL_FILE_IO))
-                                       /*
+                       if (dst_page->vmp_pmapped) {
+                               if (!(cntrl_flags & UPL_FILE_IO)) {
+                                       /*

                                         * eliminate all mappings from the
                                         * original object and its prodigy
                                         */
                                         * eliminate all mappings from the
                                         * original object and its prodigy
                                         */

-                                       refmod_state = pmap_disconnect(phys_page);
-                               else
-                                       refmod_state = pmap_get_refmod(phys_page);
-                       } else
-                               refmod_state = 0;
+                                       refmod_state = pmap_disconnect(phys_page);
+                               } else {
+                                       refmod_state = pmap_get_refmod(phys_page);
+                               }
+                       } else {
+                               refmod_state = 0;
+                       }

 
                        hw_dirty = refmod_state & VM_MEM_MODIFIED;
 
                        hw_dirty = refmod_state & VM_MEM_MODIFIED;

-                       dirty = hw_dirty ? TRUE : dst_page->dirty;
+                       dirty = hw_dirty ? TRUE : dst_page->vmp_dirty;

 
                        if (cntrl_flags & UPL_SET_LITE) {
 
                        if (cntrl_flags & UPL_SET_LITE) {

-                               unsigned int    pg_num;
+                               unsigned int    pg_num;

 
 

-                               pg_num = (unsigned int) ((dst_offset-offset)/PAGE_SIZE);
-                               assert(pg_num == (dst_offset-offset)/PAGE_SIZE);
-                               lite_list[pg_num>>5] |= 1 << (pg_num & 31);
+                               pg_num = (unsigned int) ((dst_offset - offset) / PAGE_SIZE);
+                               assert(pg_num == (dst_offset - offset) / PAGE_SIZE);
+                               lite_list[pg_num >> 5] |= 1 << (pg_num & 31);

 
 

-                               if (hw_dirty)
-                                       pmap_clear_modify(phys_page);
+                               if (hw_dirty) {
+                                       pmap_clear_modify(phys_page);
+                               }

 
                                /*
 
                                /*

-                                * Mark original page as cleaning 
+                                * Mark original page as cleaning

                                 * in place.
                                 */
                                 * in place.
                                 */

-                               dst_page->cleaning = TRUE;
-                               dst_page->precious = FALSE;
+                               dst_page->vmp_cleaning = TRUE;
+                               dst_page->vmp_precious = FALSE;

                        } else {
                                /*
                                 * use pageclean setup, it is more
                                 * convenient even for the pageout
                                 * cases here
                                 */
                        } else {
                                /*
                                 * use pageclean setup, it is more
                                 * convenient even for the pageout
                                 * cases here
                                 */

-                               vm_object_lock(upl->map_object);
+                               vm_object_lock(upl->map_object);

                                vm_pageclean_setup(dst_page, alias_page, upl->map_object, size - xfer_size);
                                vm_pageclean_setup(dst_page, alias_page, upl->map_object, size - xfer_size);

-                               vm_object_unlock(upl->map_object);
+                               vm_object_unlock(upl->map_object);

 
 

-                               alias_page->absent = FALSE;
+                               alias_page->vmp_absent = FALSE;

                                alias_page = NULL;
                        }
 
                                alias_page = NULL;
                        }
 


@@ -5630,27 +5554,29 @@ check_busy:
                                 */
                                upl->flags |= UPL_CLEAR_DIRTY;
                        }
                                 */
                                upl->flags |= UPL_CLEAR_DIRTY;
                        }

-                       dst_page->dirty = dirty;
+                       dst_page->vmp_dirty = dirty;

 
 

-                       if (!dirty)
-                               dst_page->precious = TRUE;
+                       if (!dirty) {
+                               dst_page->vmp_precious = TRUE;
+                       }

 
 

-                       if ( !VM_PAGE_WIRED(dst_page)) {
-                               /*
+                       if (!VM_PAGE_WIRED(dst_page)) {
+                               /*

                                 * deny access to the target page while
                                 * it is being worked on
                                 */
                                 * deny access to the target page while
                                 * it is being worked on
                                 */

-                               dst_page->busy = TRUE;
-                       } else
+                               dst_page->vmp_busy = TRUE;
+                       } else {

                                dwp->dw_mask |= DW_vm_page_wire;
                                dwp->dw_mask |= DW_vm_page_wire;

+                       }

 
                        /*
                         * We might be about to satisfy a fault which has been
                         * requested. So no need for the "restart" bit.
                         */
 
                        /*
                         * We might be about to satisfy a fault which has been
                         * requested. So no need for the "restart" bit.
                         */

-                       dst_page->restart = FALSE;
-                       if (!dst_page->absent && !(cntrl_flags & UPL_WILL_MODIFY)) {
-                               /*
+                       dst_page->vmp_restart = FALSE;
+                       if (!dst_page->vmp_absent && !(cntrl_flags & UPL_WILL_MODIFY)) {
+                               /*

                                 * expect the page to be used
                                 */
                                dwp->dw_mask |= DW_set_reference;
                                 * expect the page to be used
                                 */
                                dwp->dw_mask |= DW_set_reference;


@@ -5658,57 +5584,62 @@ check_busy:
                        if (cntrl_flags & UPL_PRECIOUS) {
                                if (object->internal) {
                                        SET_PAGE_DIRTY(dst_page, FALSE);
                        if (cntrl_flags & UPL_PRECIOUS) {
                                if (object->internal) {
                                        SET_PAGE_DIRTY(dst_page, FALSE);

-                                       dst_page->precious = FALSE;
+                                       dst_page->vmp_precious = FALSE;

                                } else {
                                } else {

-                                       dst_page->precious = TRUE;
+                                       dst_page->vmp_precious = TRUE;

                                }
                        } else {
                                }
                        } else {

-                               dst_page->precious = FALSE;
+                               dst_page->vmp_precious = FALSE;

                        }
                }
                        }
                }

-               if (dst_page->busy)
+               if (dst_page->vmp_busy) {

                        upl->flags |= UPL_HAS_BUSY;
                        upl->flags |= UPL_HAS_BUSY;

+               }

 
 

-               if (phys_page > upl->highest_page)
-                       upl->highest_page = phys_page;
-               assert (!pmap_is_noencrypt(phys_page));
+               if (phys_page > upl->highest_page) {
+                       upl->highest_page = phys_page;
+               }
+               assert(!pmap_is_noencrypt(phys_page));

                if (user_page_list) {
                        user_page_list[entry].phys_addr = phys_page;
                if (user_page_list) {
                        user_page_list[entry].phys_addr = phys_page;

-                       user_page_list[entry].free_when_done    = dst_page->free_when_done;
-                       user_page_list[entry].absent    = dst_page->absent;
-                       user_page_list[entry].dirty     = dst_page->dirty;
-                       user_page_list[entry].precious  = dst_page->precious;
-                       user_page_list[entry].device    = FALSE;
+                       user_page_list[entry].free_when_done    = dst_page->vmp_free_when_done;
+                       user_page_list[entry].absent    = dst_page->vmp_absent;
+                       user_page_list[entry].dirty     = dst_page->vmp_dirty;
+                       user_page_list[entry].precious  = dst_page->vmp_precious;
+                       user_page_list[entry].device    = FALSE;

                        user_page_list[entry].needed    = FALSE;
                        user_page_list[entry].needed    = FALSE;

-                       if (dst_page->clustered == TRUE)
-                               user_page_list[entry].speculative = (dst_page->vm_page_q_state == VM_PAGE_ON_SPECULATIVE_Q) ? TRUE : FALSE;
-                       else
-                               user_page_list[entry].speculative = FALSE;
-                       user_page_list[entry].cs_validated = dst_page->cs_validated;
-                       user_page_list[entry].cs_tainted = dst_page->cs_tainted;
-                       user_page_list[entry].cs_nx = dst_page->cs_nx;
+                       if (dst_page->vmp_clustered == TRUE) {
+                               user_page_list[entry].speculative = (dst_page->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q) ? TRUE : FALSE;
+                       } else {
+                               user_page_list[entry].speculative = FALSE;
+                       }
+                       user_page_list[entry].cs_validated = dst_page->vmp_cs_validated;
+                       user_page_list[entry].cs_tainted = dst_page->vmp_cs_tainted;
+                       user_page_list[entry].cs_nx = dst_page->vmp_cs_nx;

                        user_page_list[entry].mark      = FALSE;
                }
                        user_page_list[entry].mark      = FALSE;
                }

-               /*
+               /*

                 * if UPL_RET_ONLY_ABSENT is set, then
                 * we are working with a fresh page and we've
                 * just set the clustered flag on it to
                 * indicate that it was drug in as part of a
                 * speculative cluster... so leave it alone
                 */
                 * if UPL_RET_ONLY_ABSENT is set, then
                 * we are working with a fresh page and we've
                 * just set the clustered flag on it to
                 * indicate that it was drug in as part of a
                 * speculative cluster... so leave it alone
                 */

-               if ( !(cntrl_flags & UPL_RET_ONLY_ABSENT)) {
-                       /*
+               if (!(cntrl_flags & UPL_RET_ONLY_ABSENT)) {
+                       /*

                         * someone is explicitly grabbing this page...
                         * update clustered and speculative state
                         * someone is explicitly grabbing this page...
                         * update clustered and speculative state

-                        * 
+                        *

                         */
                         */

-                       if (dst_page->clustered)
+                       if (dst_page->vmp_clustered) {

                                VM_PAGE_CONSUME_CLUSTERED(dst_page);
                                VM_PAGE_CONSUME_CLUSTERED(dst_page);

+                       }

                }
 try_next_page:
                if (dwp->dw_mask) {
                }
 try_next_page:
                if (dwp->dw_mask) {

-                       if (dwp->dw_mask & DW_vm_page_activate)
+                       if (dwp->dw_mask & DW_vm_page_activate) {

                                VM_STAT_INCR(reactivations);
                                VM_STAT_INCR(reactivations);

+                       }

 
                        VM_PAGE_ADD_DELAYED_WORK(dwp, dst_page, dw_count);
 
 
                        VM_PAGE_ADD_DELAYED_WORK(dwp, dst_page, dw_count);
 


@@ -5723,30 +5654,42 @@ try_next_page:
                dst_offset += PAGE_SIZE_64;
                xfer_size -= PAGE_SIZE;
        }
                dst_offset += PAGE_SIZE_64;
                xfer_size -= PAGE_SIZE;
        }

-       if (dw_count)
+       if (dw_count) {

                vm_page_do_delayed_work(object, tag, &dw_array[0], dw_count);
                vm_page_do_delayed_work(object, tag, &dw_array[0], dw_count);

+       }

 
        if (alias_page != NULL) {
                VM_PAGE_FREE(alias_page);
        }
 
        if (alias_page != NULL) {
                VM_PAGE_FREE(alias_page);
        }

+       if (pmap_flushes_delayed == TRUE) {
+               pmap_flush(&pmap_flush_context_storage);
+       }

 
        if (page_list_count != NULL) {
 
        if (page_list_count != NULL) {

-               if (upl->flags & UPL_INTERNAL)
+               if (upl->flags & UPL_INTERNAL) {

                        *page_list_count = 0;
                        *page_list_count = 0;

-               else if (*page_list_count > entry)
+               } else if (*page_list_count > entry) {

                        *page_list_count = entry;
                        *page_list_count = entry;

+               }

        }
 #if UPL_DEBUG
        upl->upl_state = 1;
 #endif
        vm_object_unlock(object);
 
        }
 #if UPL_DEBUG
        upl->upl_state = 1;
 #endif
        vm_object_unlock(object);
 

+       VM_DEBUG_CONSTANT_EVENT(vm_object_upl_request, VM_UPL_REQUEST, DBG_FUNC_END, page_grab_count, 0, 0, 0);
+#if DEVELOPMENT || DEBUG
+       if (task != NULL) {
+               ledger_credit(task->ledger, task_ledgers.pages_grabbed_upl, page_grab_count);
+       }
+#endif /* DEVELOPMENT || DEBUG */
+

        return KERN_SUCCESS;
 }
 
        return KERN_SUCCESS;
 }
 

-/*  
+/*

  *     Routine:        vm_object_super_upl_request
  *     Routine:        vm_object_super_upl_request

- *     Purpose:        
+ *     Purpose:

  *             Cause the population of a portion of a vm_object
  *             in much the same way as memory_object_upl_request.
  *             Depending on the nature of the request, the pages
  *             Cause the population of a portion of a vm_object
  *             in much the same way as memory_object_upl_request.
  *             Depending on the nature of the request, the pages


@@ -5758,38 +5701,38 @@ try_next_page:
 __private_extern__ kern_return_t
 vm_object_super_upl_request(
        vm_object_t object,
 __private_extern__ kern_return_t
 vm_object_super_upl_request(
        vm_object_t object,

-       vm_object_offset_t      offset,
-       upl_size_t              size,
-       upl_size_t              super_cluster,
-       upl_t                   *upl,
-       upl_page_info_t         *user_page_list,
-       unsigned int            *page_list_count,
-       upl_control_flags_t     cntrl_flags,
-       vm_tag_t                tag)
+       vm_object_offset_t      offset,
+       upl_size_t              size,
+       upl_size_t              super_cluster,
+       upl_t                   *upl,
+       upl_page_info_t         *user_page_list,
+       unsigned int            *page_list_count,
+       upl_control_flags_t     cntrl_flags,
+       vm_tag_t                tag)

 {
 {

-       if (object->paging_offset > offset  || ((cntrl_flags & UPL_VECTOR)==UPL_VECTOR))
+       if (object->paging_offset > offset || ((cntrl_flags & UPL_VECTOR) == UPL_VECTOR)) {

                return KERN_FAILURE;
                return KERN_FAILURE;

+       }

 
        assert(object->paging_in_progress);
        offset = offset - object->paging_offset;
 
        if (super_cluster > size) {
 
        assert(object->paging_in_progress);
        offset = offset - object->paging_offset;
 
        if (super_cluster > size) {

-
-               vm_object_offset_t      base_offset;
-               upl_size_t              super_size;
-               vm_object_size_t        super_size_64;
+               vm_object_offset_t      base_offset;
+               upl_size_t              super_size;
+               vm_object_size_t        super_size_64;

 
                base_offset = (offset & ~((vm_object_offset_t) super_cluster - 1));
 
                base_offset = (offset & ~((vm_object_offset_t) super_cluster - 1));

-               super_size = (offset + size) > (base_offset + super_cluster) ? super_cluster<<1 : super_cluster;
+               super_size = (offset + size) > (base_offset + super_cluster) ? super_cluster << 1 : super_cluster;

                super_size_64 = ((base_offset + super_size) > object->vo_size) ? (object->vo_size - base_offset) : super_size;
                super_size = (upl_size_t) super_size_64;
                assert(super_size == super_size_64);
 
                if (offset > (base_offset + super_size)) {
                super_size_64 = ((base_offset + super_size) > object->vo_size) ? (object->vo_size - base_offset) : super_size;
                super_size = (upl_size_t) super_size_64;
                assert(super_size == super_size_64);
 
                if (offset > (base_offset + super_size)) {

-                       panic("vm_object_super_upl_request: Missed target pageout"
-                             " %#llx,%#llx, %#x, %#x, %#x, %#llx\n",
-                             offset, base_offset, super_size, super_cluster,
-                             size, object->paging_offset);
+                       panic("vm_object_super_upl_request: Missed target pageout"
+                           " %#llx,%#llx, %#x, %#x, %#x, %#llx\n",
+                           offset, base_offset, super_size, super_cluster,
+                           size, object->paging_offset);

                }
                /*
                 * apparently there is a case where the vm requests a
                }
                /*
                 * apparently there is a case where the vm requests a


@@ -5797,7 +5740,7 @@ vm_object_super_upl_request(
                 * object size
                 */
                if ((offset + size) > (base_offset + super_size)) {
                 * object size
                 */
                if ((offset + size) > (base_offset + super_size)) {

-                       super_size_64 = (offset + size) - base_offset;
+                       super_size_64 = (offset + size) - base_offset;

                        super_size = (upl_size_t) super_size_64;
                        assert(super_size == super_size_64);
                }
                        super_size = (upl_size_t) super_size_64;
                        assert(super_size == super_size_64);
                }


@@ -5816,23 +5759,23 @@ extern char *proc_name_address(void *p);
 
 kern_return_t
 vm_map_create_upl(
 
 kern_return_t
 vm_map_create_upl(

-       vm_map_t                map,
-       vm_map_address_t        offset,
-       upl_size_t              *upl_size,
-       upl_t                   *upl,
-       upl_page_info_array_t   page_list,
-       unsigned int            *count,
-       upl_control_flags_t     *flags,
-       vm_tag_t                tag)
+       vm_map_t                map,
+       vm_map_address_t        offset,
+       upl_size_t              *upl_size,
+       upl_t                   *upl,
+       upl_page_info_array_t   page_list,
+       unsigned int            *count,
+       upl_control_flags_t     *flags,
+       vm_tag_t                tag)

 {
 {

-       vm_map_entry_t          entry;
-       upl_control_flags_t     caller_flags;
-       int                     force_data_sync;
-       int                     sync_cow_data;
-       vm_object_t             local_object;
-       vm_map_offset_t         local_offset;
-       vm_map_offset_t         local_start;
-       kern_return_t           ret;
+       vm_map_entry_t          entry;
+       upl_control_flags_t     caller_flags;
+       int                     force_data_sync;
+       int                     sync_cow_data;
+       vm_object_t             local_object;
+       vm_map_offset_t         local_offset;
+       vm_map_offset_t         local_start;
+       kern_return_t           ret;

 
        assert(page_aligned(offset));
 
 
        assert(page_aligned(offset));
 


@@ -5848,8 +5791,9 @@ vm_map_create_upl(
        force_data_sync = (caller_flags & UPL_FORCE_DATA_SYNC);
        sync_cow_data = !(caller_flags & UPL_COPYOUT_FROM);
 
        force_data_sync = (caller_flags & UPL_FORCE_DATA_SYNC);
        sync_cow_data = !(caller_flags & UPL_COPYOUT_FROM);
 

-       if (upl == NULL)
+       if (upl == NULL) {

                return KERN_INVALID_ARGUMENT;
                return KERN_INVALID_ARGUMENT;

+       }

 
 REDISCOVER_ENTRY:
        vm_map_lock_read(map);
 
 REDISCOVER_ENTRY:
        vm_map_lock_read(map);


@@ -5869,11 +5813,13 @@ REDISCOVER_ENTRY:
 
                if (!entry->is_sub_map &&
                    VME_OBJECT(entry) != VM_OBJECT_NULL) {
 
                if (!entry->is_sub_map &&
                    VME_OBJECT(entry) != VM_OBJECT_NULL) {

-                       if (VME_OBJECT(entry)->private)
+                       if (VME_OBJECT(entry)->private) {

                                *flags = UPL_DEV_MEMORY;
                                *flags = UPL_DEV_MEMORY;

+                       }

 
 

-                       if (VME_OBJECT(entry)->phys_contiguous)
+                       if (VME_OBJECT(entry)->phys_contiguous) {

                                *flags |= UPL_PHYS_CONTIG;
                                *flags |= UPL_PHYS_CONTIG;

+                       }

                }
                vm_map_unlock_read(map);
                return KERN_SUCCESS;
                }
                vm_map_unlock_read(map);
                return KERN_SUCCESS;


@@ -5881,28 +5827,31 @@ REDISCOVER_ENTRY:
 
        if (VME_OBJECT(entry) == VM_OBJECT_NULL ||
            !VME_OBJECT(entry)->phys_contiguous) {
 
        if (VME_OBJECT(entry) == VM_OBJECT_NULL ||
            !VME_OBJECT(entry)->phys_contiguous) {

-               if (*upl_size > MAX_UPL_SIZE_BYTES)
+               if (*upl_size > MAX_UPL_SIZE_BYTES) {

                        *upl_size = MAX_UPL_SIZE_BYTES;
                        *upl_size = MAX_UPL_SIZE_BYTES;

+               }

        }
 
        /*
         *      Create an object if necessary.
         */
        if (VME_OBJECT(entry) == VM_OBJECT_NULL) {
        }
 
        /*
         *      Create an object if necessary.
         */
        if (VME_OBJECT(entry) == VM_OBJECT_NULL) {

-
-               if (vm_map_lock_read_to_write(map))
+               if (vm_map_lock_read_to_write(map)) {

                        goto REDISCOVER_ENTRY;
                        goto REDISCOVER_ENTRY;

+               }

 
                VME_OBJECT_SET(entry,
 
                VME_OBJECT_SET(entry,

-                              vm_object_allocate((vm_size_t)
-                                                 (entry->vme_end -
-                                                  entry->vme_start)));
+                   vm_object_allocate((vm_size_t)
+                   (entry->vme_end -
+                   entry->vme_start)));

                VME_OFFSET_SET(entry, 0);
                VME_OFFSET_SET(entry, 0);

+               assert(entry->use_pmap);

 
                vm_map_lock_write_to_read(map);
        }
 
        if (!(caller_flags & UPL_COPYOUT_FROM) &&
 
                vm_map_lock_write_to_read(map);
        }
 
        if (!(caller_flags & UPL_COPYOUT_FROM) &&

+           !entry->is_sub_map &&

            !(entry->protection & VM_PROT_WRITE)) {
                vm_map_unlock_read(map);
                return KERN_PROTECTION_FAILURE;
            !(entry->protection & VM_PROT_WRITE)) {
                vm_map_unlock_read(map);
                return KERN_PROTECTION_FAILURE;


@@ -5913,8 +5862,8 @@ REDISCOVER_ENTRY:
            (caller_flags & UPL_COPYOUT_FROM) &&
            (entry->protection & VM_PROT_EXECUTE) &&
            !(entry->protection & VM_PROT_WRITE)) {
            (caller_flags & UPL_COPYOUT_FROM) &&
            (entry->protection & VM_PROT_EXECUTE) &&
            !(entry->protection & VM_PROT_WRITE)) {

-               vm_offset_t     kaddr;
-               vm_size_t       ksize;
+               vm_offset_t     kaddr;
+               vm_size_t       ksize;

 
                /*
                 * We're about to create a read-only UPL backed by
 
                /*
                 * We're about to create a read-only UPL backed by


@@ -5934,9 +5883,9 @@ REDISCOVER_ENTRY:
                ksize = round_page(*upl_size);
                kaddr = 0;
                ret = kmem_alloc_pageable(kernel_map,
                ksize = round_page(*upl_size);
                kaddr = 0;
                ret = kmem_alloc_pageable(kernel_map,

-                                         &kaddr,
-                                         ksize,
-                                         tag);
+                   &kaddr,
+                   ksize,
+                   tag);

                if (ret == KERN_SUCCESS) {
                        /* copyin the user data */
                        assert(page_aligned(offset));
                if (ret == KERN_SUCCESS) {
                        /* copyin the user data */
                        assert(page_aligned(offset));


@@ -5946,12 +5895,12 @@ REDISCOVER_ENTRY:
                        if (ksize > *upl_size) {
                                /* zero out the extra space in kernel buffer */
                                memset((void *)(kaddr + *upl_size),
                        if (ksize > *upl_size) {
                                /* zero out the extra space in kernel buffer */
                                memset((void *)(kaddr + *upl_size),

-                                      0,
-                                      ksize - *upl_size);
+                                   0,
+                                   ksize - *upl_size);

                        }
                        /* create the UPL from the kernel buffer */
                        ret = vm_map_create_upl(kernel_map, kaddr, upl_size,
                        }
                        /* create the UPL from the kernel buffer */
                        ret = vm_map_create_upl(kernel_map, kaddr, upl_size,

-                                               upl, page_list, count, flags, tag);
+                           upl, page_list, count, flags, tag);

                }
                if (kaddr != 0) {
                        /* free the kernel buffer */
                }
                if (kaddr != 0) {
                        /* free the kernel buffer */


@@ -5961,10 +5910,10 @@ REDISCOVER_ENTRY:
                }
 #if DEVELOPMENT || DEBUG
                DTRACE_VM4(create_upl_from_executable,
                }
 #if DEVELOPMENT || DEBUG
                DTRACE_VM4(create_upl_from_executable,

-                          vm_map_t, map,
-                          vm_map_address_t, offset,
-                          upl_size_t, *upl_size,
-                          kern_return_t, ret);
+                   vm_map_t, map,
+                   vm_map_address_t, offset,
+                   upl_size_t, *upl_size,
+                   kern_return_t, ret);

 #endif /* DEVELOPMENT || DEBUG */
                return ret;
        }
 #endif /* DEVELOPMENT || DEBUG */
                return ret;
        }


@@ -5980,12 +5929,12 @@ REDISCOVER_ENTRY:
            entry->wired_count == 0 && /* No COW for entries that are wired */
            (map->pmap != kernel_pmap) && /* alias checks */
            (vm_map_entry_should_cow_for_true_share(entry) /* case 1 */
            entry->wired_count == 0 && /* No COW for entries that are wired */
            (map->pmap != kernel_pmap) && /* alias checks */
            (vm_map_entry_should_cow_for_true_share(entry) /* case 1 */

-            ||
-            (/* case 2 */
-             local_object->internal &&
-             (local_object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) &&
-             local_object->ref_count > 1))) {
-               vm_prot_t       prot;
+           ||
+           ( /* case 2 */
+                   local_object->internal &&
+                   (local_object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) &&
+                   local_object->ref_count > 1))) {
+               vm_prot_t       prot;

 
                /*
                 * Case 1:
 
                /*
                 * Case 1:


@@ -6010,30 +5959,31 @@ REDISCOVER_ENTRY:
                assert(VME_OBJECT(entry) == local_object);
 
                vm_map_clip_start(map,
                assert(VME_OBJECT(entry) == local_object);
 
                vm_map_clip_start(map,

-                                 entry,
-                                 vm_map_trunc_page(offset,
-                                                   VM_MAP_PAGE_MASK(map)));
+                   entry,
+                   vm_map_trunc_page(offset,
+                   VM_MAP_PAGE_MASK(map)));

                vm_map_clip_end(map,
                vm_map_clip_end(map,

-                               entry,
-                               vm_map_round_page(offset + *upl_size,
-                                                 VM_MAP_PAGE_MASK(map)));
+                   entry,
+                   vm_map_round_page(offset + *upl_size,
+                   VM_MAP_PAGE_MASK(map)));

                if ((entry->vme_end - offset) < *upl_size) {
                        *upl_size = (upl_size_t) (entry->vme_end - offset);
                        assert(*upl_size == entry->vme_end - offset);
                }
 
                prot = entry->protection & ~VM_PROT_WRITE;
                if ((entry->vme_end - offset) < *upl_size) {
                        *upl_size = (upl_size_t) (entry->vme_end - offset);
                        assert(*upl_size == entry->vme_end - offset);
                }
 
                prot = entry->protection & ~VM_PROT_WRITE;

-               if (override_nx(map, VME_ALIAS(entry)) && prot)
+               if (override_nx(map, VME_ALIAS(entry)) && prot) {

                        prot |= VM_PROT_EXECUTE;
                        prot |= VM_PROT_EXECUTE;

+               }

                vm_object_pmap_protect(local_object,
                vm_object_pmap_protect(local_object,

-                                      VME_OFFSET(entry),
-                                      entry->vme_end - entry->vme_start,
-                                      ((entry->is_shared ||
-                                        map->mapped_in_other_pmaps)
-                                       ? PMAP_NULL
-                                       : map->pmap),
-                                      entry->vme_start,
-                                      prot);
+                   VME_OFFSET(entry),
+                   entry->vme_end - entry->vme_start,
+                   ((entry->is_shared ||
+                   map->mapped_in_other_pmaps)
+                   ? PMAP_NULL
+                   : map->pmap),
+                   entry->vme_start,
+                   prot);

 
                assert(entry->wired_count == 0);
 
 
                assert(entry->wired_count == 0);
 


@@ -6046,11 +5996,11 @@ REDISCOVER_ENTRY:
                if (local_object->true_share) {
                        /* object is already in proper state: no COW needed */
                        assert(local_object->copy_strategy !=
                if (local_object->true_share) {
                        /* object is already in proper state: no COW needed */
                        assert(local_object->copy_strategy !=

-                              MEMORY_OBJECT_COPY_SYMMETRIC);
+                           MEMORY_OBJECT_COPY_SYMMETRIC);

                } else {
                        /* not true_share: ask for copy-on-write below */
                        assert(local_object->copy_strategy ==
                } else {
                        /* not true_share: ask for copy-on-write below */
                        assert(local_object->copy_strategy ==

-                              MEMORY_OBJECT_COPY_SYMMETRIC);
+                           MEMORY_OBJECT_COPY_SYMMETRIC);

                        entry->needs_copy = TRUE;
                }
                vm_object_unlock(local_object);
                        entry->needs_copy = TRUE;
                }
                vm_object_unlock(local_object);


@@ -6058,19 +6008,19 @@ REDISCOVER_ENTRY:
                vm_map_lock_write_to_read(map);
        }
 
                vm_map_lock_write_to_read(map);
        }
 

-       if (entry->needs_copy)  {
+       if (entry->needs_copy) {

                /*
                 * Honor copy-on-write for COPY_SYMMETRIC
                 * strategy.
                 */
                /*
                 * Honor copy-on-write for COPY_SYMMETRIC
                 * strategy.
                 */

-               vm_map_t                local_map;
-               vm_object_t             object;
-               vm_object_offset_t      new_offset;
-               vm_prot_t               prot;
-               boolean_t               wired;
-               vm_map_version_t        version;
-               vm_map_t                real_map;
-               vm_prot_t               fault_type;
+               vm_map_t                local_map;
+               vm_object_t             object;
+               vm_object_offset_t      new_offset;
+               vm_prot_t               prot;
+               boolean_t               wired;
+               vm_map_version_t        version;
+               vm_map_t                real_map;
+               vm_prot_t               fault_type;

 
                local_map = map;
 
 
                local_map = map;
 


@@ -6078,17 +6028,17 @@ REDISCOVER_ENTRY:
                        fault_type = VM_PROT_READ | VM_PROT_COPY;
                        vm_counters.create_upl_extra_cow++;
                        vm_counters.create_upl_extra_cow_pages +=
                        fault_type = VM_PROT_READ | VM_PROT_COPY;
                        vm_counters.create_upl_extra_cow++;
                        vm_counters.create_upl_extra_cow_pages +=

-                               (entry->vme_end - entry->vme_start) / PAGE_SIZE;
+                           (entry->vme_end - entry->vme_start) / PAGE_SIZE;

                } else {
                        fault_type = VM_PROT_WRITE;
                }
                if (vm_map_lookup_locked(&local_map,
                } else {
                        fault_type = VM_PROT_WRITE;
                }
                if (vm_map_lookup_locked(&local_map,

-                                        offset, fault_type,
-                                        OBJECT_LOCK_EXCLUSIVE,
-                                        &version, &object,
-                                        &new_offset, &prot, &wired,
-                                        NULL,
-                                        &real_map) != KERN_SUCCESS) {
+                   offset, fault_type,
+                   OBJECT_LOCK_EXCLUSIVE,
+                   &version, &object,
+                   &new_offset, &prot, &wired,
+                   NULL,
+                   &real_map) != KERN_SUCCESS) {

                        if (fault_type == VM_PROT_WRITE) {
                                vm_counters.create_upl_lookup_failure_write++;
                        } else {
                        if (fault_type == VM_PROT_WRITE) {
                                vm_counters.create_upl_lookup_failure_write++;
                        } else {


@@ -6097,8 +6047,9 @@ REDISCOVER_ENTRY:
                        vm_map_unlock_read(local_map);
                        return KERN_FAILURE;
                }
                        vm_map_unlock_read(local_map);
                        return KERN_FAILURE;
                }

-               if (real_map != map)
+               if (real_map != map) {

                        vm_map_unlock(real_map);
                        vm_map_unlock(real_map);

+               }

                vm_map_unlock_read(local_map);
 
                vm_object_unlock(object);
                vm_map_unlock_read(local_map);
 
                vm_object_unlock(object);


@@ -6107,7 +6058,7 @@ REDISCOVER_ENTRY:
        }
 
        if (entry->is_sub_map) {
        }
 
        if (entry->is_sub_map) {

-               vm_map_t        submap;
+               vm_map_t        submap;

 
                submap = VME_SUBMAP(entry);
                local_start = entry->vme_start;
 
                submap = VME_SUBMAP(entry);
                local_start = entry->vme_start;


@@ -6116,9 +6067,9 @@ REDISCOVER_ENTRY:
                vm_map_reference(submap);
                vm_map_unlock_read(map);
 
                vm_map_reference(submap);
                vm_map_unlock_read(map);
 

-               ret = vm_map_create_upl(submap, 
-                                       local_offset + (offset - local_start), 
-                                       upl_size, upl, page_list, count, flags, tag);
+               ret = vm_map_create_upl(submap,
+                   local_offset + (offset - local_start),
+                   upl_size, upl, page_list, count, flags, tag);

                vm_map_deallocate(submap);
 
                return ret;
                vm_map_deallocate(submap);
 
                return ret;


@@ -6126,7 +6077,7 @@ REDISCOVER_ENTRY:
 
        if (sync_cow_data &&
            (VME_OBJECT(entry)->shadow ||
 
        if (sync_cow_data &&
            (VME_OBJECT(entry)->shadow ||

-            VME_OBJECT(entry)->copy)) {
+           VME_OBJECT(entry)->copy)) {

                local_object = VME_OBJECT(entry);
                local_start = entry->vme_start;
                local_offset = VME_OFFSET(entry);
                local_object = VME_OBJECT(entry);
                local_start = entry->vme_start;
                local_offset = VME_OFFSET(entry);


@@ -6136,13 +6087,13 @@ REDISCOVER_ENTRY:
 
                if (local_object->shadow && local_object->copy) {
                        vm_object_lock_request(local_object->shadow,
 
                if (local_object->shadow && local_object->copy) {
                        vm_object_lock_request(local_object->shadow,

-                                              ((vm_object_offset_t)
-                                               ((offset - local_start) +
-                                                local_offset) +
-                                               local_object->vo_shadow_offset),
-                                              *upl_size, FALSE, 
-                                              MEMORY_OBJECT_DATA_SYNC,
-                                              VM_PROT_NO_CHANGE);
+                           ((vm_object_offset_t)
+                           ((offset - local_start) +
+                           local_offset) +
+                           local_object->vo_shadow_offset),
+                           *upl_size, FALSE,
+                           MEMORY_OBJECT_DATA_SYNC,
+                           VM_PROT_NO_CHANGE);

                }
                sync_cow_data = FALSE;
                vm_object_deallocate(local_object);
                }
                sync_cow_data = FALSE;
                vm_object_deallocate(local_object);


@@ -6158,26 +6109,28 @@ REDISCOVER_ENTRY:
                vm_map_unlock_read(map);
 
                vm_object_lock_request(local_object,
                vm_map_unlock_read(map);
 
                vm_object_lock_request(local_object,

-                                      ((vm_object_offset_t)
-                                       ((offset - local_start) +
-                                        local_offset)),
-                                      (vm_object_size_t)*upl_size,
-                                      FALSE, 
-                                      MEMORY_OBJECT_DATA_SYNC,
-                                      VM_PROT_NO_CHANGE);
+                   ((vm_object_offset_t)
+                   ((offset - local_start) +
+                   local_offset)),
+                   (vm_object_size_t)*upl_size,
+                   FALSE,
+                   MEMORY_OBJECT_DATA_SYNC,
+                   VM_PROT_NO_CHANGE);

 
                force_data_sync = FALSE;
                vm_object_deallocate(local_object);
 
                goto REDISCOVER_ENTRY;
        }
 
                force_data_sync = FALSE;
                vm_object_deallocate(local_object);
 
                goto REDISCOVER_ENTRY;
        }

-       if (VME_OBJECT(entry)->private)
+       if (VME_OBJECT(entry)->private) {

                *flags = UPL_DEV_MEMORY;
                *flags = UPL_DEV_MEMORY;

-       else
+       } else {

                *flags = 0;
                *flags = 0;

+       }

 
 

-       if (VME_OBJECT(entry)->phys_contiguous)
+       if (VME_OBJECT(entry)->phys_contiguous) {

                *flags |= UPL_PHYS_CONTIG;
                *flags |= UPL_PHYS_CONTIG;

+       }

 
        local_object = VME_OBJECT(entry);
        local_offset = VME_OFFSET(entry);
 
        local_object = VME_OBJECT(entry);
        local_offset = VME_OFFSET(entry);


@@ -6193,18 +6146,18 @@ REDISCOVER_ENTRY:
        if (entry->protection & VM_PROT_EXECUTE) {
 #if MACH_ASSERT
                printf("pid %d[%s] create_upl out of executable range from "
        if (entry->protection & VM_PROT_EXECUTE) {
 #if MACH_ASSERT
                printf("pid %d[%s] create_upl out of executable range from "

-                      "0x%llx to 0x%llx: side effects may include "
-                      "code-signing violations later on\n",
-                      proc_selfpid(),
-                      (current_task()->bsd_info
-                       ? proc_name_address(current_task()->bsd_info)
-                       : "?"),
-                      (uint64_t) entry->vme_start,
-                      (uint64_t) entry->vme_end);
+                   "0x%llx to 0x%llx: side effects may include "
+                   "code-signing violations later on\n",
+                   proc_selfpid(),
+                   (current_task()->bsd_info
+                   ? proc_name_address(current_task()->bsd_info)
+                   : "?"),
+                   (uint64_t) entry->vme_start,
+                   (uint64_t) entry->vme_end);

 #endif /* MACH_ASSERT */
                DTRACE_VM2(cs_executable_create_upl,
 #endif /* MACH_ASSERT */
                DTRACE_VM2(cs_executable_create_upl,

-                          uint64_t, (uint64_t)entry->vme_start,
-                          uint64_t, (uint64_t)entry->vme_end);
+                   uint64_t, (uint64_t)entry->vme_start,
+                   uint64_t, (uint64_t)entry->vme_end);

                cs_executable_create_upl++;
        }
 #endif /* CONFIG_EMBEDDED */
                cs_executable_create_upl++;
        }
 #endif /* CONFIG_EMBEDDED */


@@ -6220,22 +6173,22 @@ REDISCOVER_ENTRY:
         */
        if (local_object->true_share) {
                assert(local_object->copy_strategy !=
         */
        if (local_object->true_share) {
                assert(local_object->copy_strategy !=

-                      MEMORY_OBJECT_COPY_SYMMETRIC);
+                   MEMORY_OBJECT_COPY_SYMMETRIC);

        } else if (local_object != kernel_object &&
        } else if (local_object != kernel_object &&

-                  local_object != compressor_object &&
-                  !local_object->phys_contiguous) {
+           local_object != compressor_object &&
+           !local_object->phys_contiguous) {

 #if VM_OBJECT_TRACKING_OP_TRUESHARE
                if (!local_object->true_share &&
                    vm_object_tracking_inited) {
                        void *bt[VM_OBJECT_TRACKING_BTDEPTH];
                        int num = 0;
                        num = OSBacktrace(bt,
 #if VM_OBJECT_TRACKING_OP_TRUESHARE
                if (!local_object->true_share &&
                    vm_object_tracking_inited) {
                        void *bt[VM_OBJECT_TRACKING_BTDEPTH];
                        int num = 0;
                        num = OSBacktrace(bt,

-                                         VM_OBJECT_TRACKING_BTDEPTH);
+                           VM_OBJECT_TRACKING_BTDEPTH);

                        btlog_add_entry(vm_object_tracking_btlog,
                        btlog_add_entry(vm_object_tracking_btlog,

-                                       local_object,
-                                       VM_OBJECT_TRACKING_OP_TRUESHARE,
-                                       bt,
-                                       num);
+                           local_object,
+                           VM_OBJECT_TRACKING_OP_TRUESHARE,
+                           bt,
+                           num);

                }
 #endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */
                local_object->true_share = TRUE;
                }
 #endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */
                local_object->true_share = TRUE;


@@ -6250,15 +6203,15 @@ REDISCOVER_ENTRY:
 
        vm_map_unlock_read(map);
 
 
        vm_map_unlock_read(map);
 

-       ret = vm_object_iopl_request(local_object, 
-                                    ((vm_object_offset_t)
-                                     ((offset - local_start) + local_offset)),
-                                    *upl_size,
-                                    upl,
-                                    page_list,
-                                    count,
-                                    caller_flags,
-                                    tag);
+       ret = vm_object_iopl_request(local_object,
+           ((vm_object_offset_t)
+           ((offset - local_start) + local_offset)),
+           *upl_size,
+           upl,
+           page_list,
+           count,
+           caller_flags,
+           tag);

        vm_object_deallocate(local_object);
 
        return ret;
        vm_object_deallocate(local_object);
 
        return ret;


@@ -6266,76 +6219,81 @@ REDISCOVER_ENTRY:
 
 /*
  * Internal routine to enter a UPL into a VM map.
 
 /*
  * Internal routine to enter a UPL into a VM map.

- * 
+ *

  * JMM - This should just be doable through the standard
  * vm_map_enter() API.
  */
 kern_return_t
 vm_map_enter_upl(
  * JMM - This should just be doable through the standard
  * vm_map_enter() API.
  */
 kern_return_t
 vm_map_enter_upl(

-       vm_map_t                map, 
-       upl_t                   upl, 
-       vm_map_offset_t         *dst_addr)
+       vm_map_t                map,
+       upl_t                   upl,
+       vm_map_offset_t         *dst_addr)

 {
 {

-       vm_map_size_t           size;
-       vm_object_offset_t      offset;
-       vm_map_offset_t         addr;
-       vm_page_t               m;
-       kern_return_t           kr;
-       int                     isVectorUPL = 0, curr_upl=0;
-       upl_t                   vector_upl = NULL;
-       vm_offset_t             vector_upl_dst_addr = 0;
-       vm_map_t                vector_upl_submap = NULL;
-       upl_offset_t            subupl_offset = 0;
-       upl_size_t              subupl_size = 0;
-
-       if (upl == UPL_NULL)
+       vm_map_size_t           size;
+       vm_object_offset_t      offset;
+       vm_map_offset_t         addr;
+       vm_page_t               m;
+       kern_return_t           kr;
+       int                     isVectorUPL = 0, curr_upl = 0;
+       upl_t                   vector_upl = NULL;
+       vm_offset_t             vector_upl_dst_addr = 0;
+       vm_map_t                vector_upl_submap = NULL;
+       upl_offset_t            subupl_offset = 0;
+       upl_size_t              subupl_size = 0;
+
+       if (upl == UPL_NULL) {

                return KERN_INVALID_ARGUMENT;
                return KERN_INVALID_ARGUMENT;

+       }

 
 

-       if((isVectorUPL = vector_upl_is_valid(upl))) {
-               int mapped=0,valid_upls=0;
+       if ((isVectorUPL = vector_upl_is_valid(upl))) {
+               int mapped = 0, valid_upls = 0;

                vector_upl = upl;
 
                upl_lock(vector_upl);
                vector_upl = upl;
 
                upl_lock(vector_upl);

-               for(curr_upl=0; curr_upl < MAX_VECTOR_UPL_ELEMENTS; curr_upl++) {
+               for (curr_upl = 0; curr_upl < MAX_VECTOR_UPL_ELEMENTS; curr_upl++) {

                        upl =  vector_upl_subupl_byindex(vector_upl, curr_upl );
                        upl =  vector_upl_subupl_byindex(vector_upl, curr_upl );

-                       if(upl == NULL)
+                       if (upl == NULL) {

                                continue;
                                continue;

+                       }

                        valid_upls++;
                        valid_upls++;

-                       if (UPL_PAGE_LIST_MAPPED & upl->flags)
+                       if (UPL_PAGE_LIST_MAPPED & upl->flags) {

                                mapped++;
                                mapped++;

+                       }

                }
 
                }
 

-               if(mapped) { 
-                       if(mapped != valid_upls)
+               if (mapped) {
+                       if (mapped != valid_upls) {

                                panic("Only %d of the %d sub-upls within the Vector UPL are alread mapped\n", mapped, valid_upls);
                                panic("Only %d of the %d sub-upls within the Vector UPL are alread mapped\n", mapped, valid_upls);

-                       else {
+                       } else {

                                upl_unlock(vector_upl);
                                return KERN_FAILURE;
                        }
                }
 
                kr = kmem_suballoc(map, &vector_upl_dst_addr, vector_upl->size, FALSE,
                                upl_unlock(vector_upl);
                                return KERN_FAILURE;
                        }
                }
 
                kr = kmem_suballoc(map, &vector_upl_dst_addr, vector_upl->size, FALSE,

-                                   VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_NONE,
-                                   &vector_upl_submap);
-               if( kr != KERN_SUCCESS )
+                   VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_NONE,
+                   &vector_upl_submap);
+               if (kr != KERN_SUCCESS) {

                        panic("Vector UPL submap allocation failed\n");
                        panic("Vector UPL submap allocation failed\n");

+               }

                map = vector_upl_submap;
                vector_upl_set_submap(vector_upl, vector_upl_submap, vector_upl_dst_addr);
                map = vector_upl_submap;
                vector_upl_set_submap(vector_upl, vector_upl_submap, vector_upl_dst_addr);

-               curr_upl=0;
-       }
-       else
+               curr_upl = 0;
+       } else {

                upl_lock(upl);
                upl_lock(upl);

+       }

 
 process_upl_to_enter:
 
 process_upl_to_enter:

-       if(isVectorUPL){
-               if(curr_upl == MAX_VECTOR_UPL_ELEMENTS) {
+       if (isVectorUPL) {
+               if (curr_upl == MAX_VECTOR_UPL_ELEMENTS) {

                        *dst_addr = vector_upl_dst_addr;
                        upl_unlock(vector_upl);
                        return KERN_SUCCESS;
                }
                upl =  vector_upl_subupl_byindex(vector_upl, curr_upl++ );
                        *dst_addr = vector_upl_dst_addr;
                        upl_unlock(vector_upl);
                        return KERN_SUCCESS;
                }
                upl =  vector_upl_subupl_byindex(vector_upl, curr_upl++ );

-               if(upl == NULL)
+               if (upl == NULL) {

                        goto process_upl_to_enter;
                        goto process_upl_to_enter;

+               }

 
                vector_upl_get_iostate(vector_upl, upl, &subupl_offset, &subupl_size);
                *dst_addr = (vm_map_offset_t)(vector_upl_dst_addr + (vm_map_offset_t)subupl_offset);
 
                vector_upl_get_iostate(vector_upl, upl, &subupl_offset, &subupl_size);
                *dst_addr = (vm_map_offset_t)(vector_upl_dst_addr + (vm_map_offset_t)subupl_offset);


@@ -6350,20 +6308,19 @@ process_upl_to_enter:
        }
        if ((!(upl->flags & UPL_SHADOWED)) &&
            ((upl->flags & UPL_HAS_BUSY) ||
        }
        if ((!(upl->flags & UPL_SHADOWED)) &&
            ((upl->flags & UPL_HAS_BUSY) ||

-            !((upl->flags & (UPL_DEVICE_MEMORY | UPL_IO_WIRE)) || (upl->map_object->phys_contiguous)))) {
-
-               vm_object_t             object;
-               vm_page_t               alias_page;
-               vm_object_offset_t      new_offset;
-               unsigned int            pg_num;
-               wpl_array_t             lite_list;
+           !((upl->flags & (UPL_DEVICE_MEMORY | UPL_IO_WIRE)) || (upl->map_object->phys_contiguous)))) {
+               vm_object_t             object;
+               vm_page_t               alias_page;
+               vm_object_offset_t      new_offset;
+               unsigned int            pg_num;
+               wpl_array_t             lite_list;

 
                if (upl->flags & UPL_INTERNAL) {
 
                if (upl->flags & UPL_INTERNAL) {

-                       lite_list = (wpl_array_t) 
-                               ((((uintptr_t)upl) + sizeof(struct upl))
-                                + ((upl->size/PAGE_SIZE) * sizeof(upl_page_info_t)));
+                       lite_list = (wpl_array_t)
+                           ((((uintptr_t)upl) + sizeof(struct upl))
+                           + ((upl->size / PAGE_SIZE) * sizeof(upl_page_info_t)));

                } else {
                } else {

-                       lite_list = (wpl_array_t)(((uintptr_t)upl) + sizeof(struct upl));
+                       lite_list = (wpl_array_t)(((uintptr_t)upl) + sizeof(struct upl));

                }
                object = upl->map_object;
                upl->map_object = vm_object_allocate(upl->size);
                }
                object = upl->map_object;
                upl->map_object = vm_object_allocate(upl->size);


@@ -6386,25 +6343,24 @@ process_upl_to_enter:
                        pg_num = (unsigned int) (new_offset / PAGE_SIZE);
                        assert(pg_num == new_offset / PAGE_SIZE);
 
                        pg_num = (unsigned int) (new_offset / PAGE_SIZE);
                        assert(pg_num == new_offset / PAGE_SIZE);
 

-                       if (lite_list[pg_num>>5] & (1 << (pg_num & 31))) {
-
+                       if (lite_list[pg_num >> 5] & (1 << (pg_num & 31))) {

                                VM_PAGE_GRAB_FICTITIOUS(alias_page);
 
                                vm_object_lock(object);
 
                                m = vm_page_lookup(object, offset);
                                if (m == VM_PAGE_NULL) {
                                VM_PAGE_GRAB_FICTITIOUS(alias_page);
 
                                vm_object_lock(object);
 
                                m = vm_page_lookup(object, offset);
                                if (m == VM_PAGE_NULL) {

-                                       panic("vm_upl_map: page missing\n");
+                                       panic("vm_upl_map: page missing\n");

                                }
 
                                /*
                                }
 
                                /*

-                                * Convert the fictitious page to a private 
+                                * Convert the fictitious page to a private

                                 * shadow of the real page.
                                 */
                                 * shadow of the real page.
                                 */

-                               assert(alias_page->fictitious);
-                               alias_page->fictitious = FALSE;
-                               alias_page->private = TRUE;
-                               alias_page->free_when_done = TRUE;
+                               assert(alias_page->vmp_fictitious);
+                               alias_page->vmp_fictitious = FALSE;
+                               alias_page->vmp_private = TRUE;
+                               alias_page->vmp_free_when_done = TRUE;

                                /*
                                 * since m is a page in the upl it must
                                 * already be wired or BUSY, so it's
                                /*
                                 * since m is a page in the upl it must
                                 * already be wired or BUSY, so it's


@@ -6413,17 +6369,17 @@ process_upl_to_enter:
                                 */
                                VM_PAGE_SET_PHYS_PAGE(alias_page, VM_PAGE_GET_PHYS_PAGE(m));
 
                                 */
                                VM_PAGE_SET_PHYS_PAGE(alias_page, VM_PAGE_GET_PHYS_PAGE(m));
 

-                               vm_object_unlock(object);
+                               vm_object_unlock(object);

 
                                vm_page_lockspin_queues();
                                vm_page_wire(alias_page, VM_KERN_MEMORY_NONE, TRUE);
                                vm_page_unlock_queues();
 
                                vm_page_lockspin_queues();
                                vm_page_wire(alias_page, VM_KERN_MEMORY_NONE, TRUE);
                                vm_page_unlock_queues();

-                               
+

                                vm_page_insert_wired(alias_page, upl->map_object, new_offset, VM_KERN_MEMORY_NONE);
 
                                vm_page_insert_wired(alias_page, upl->map_object, new_offset, VM_KERN_MEMORY_NONE);
 

-                               assert(!alias_page->wanted);
-                               alias_page->busy = FALSE;
-                               alias_page->absent = FALSE;
+                               assert(!alias_page->vmp_wanted);
+                               alias_page->vmp_busy = FALSE;
+                               alias_page->vmp_absent = FALSE;

                        }
                        size -= PAGE_SIZE;
                        offset += PAGE_SIZE_64;
                        }
                        size -= PAGE_SIZE;
                        offset += PAGE_SIZE_64;


@@ -6431,38 +6387,39 @@ process_upl_to_enter:
                }
                vm_object_unlock(upl->map_object);
        }
                }
                vm_object_unlock(upl->map_object);
        }

-       if (upl->flags & UPL_SHADOWED)
-               offset = 0;
-       else
-               offset = upl->offset - upl->map_object->paging_offset;
+       if (upl->flags & UPL_SHADOWED) {
+               offset = 0;
+       } else {
+               offset = upl->offset - upl->map_object->paging_offset;
+       }

 
        size = upl->size;
 
        size = upl->size;

-       
+

        vm_object_reference(upl->map_object);
 
        vm_object_reference(upl->map_object);
 

-       if(!isVectorUPL) {
+       if (!isVectorUPL) {

                *dst_addr = 0;
                /*
                *dst_addr = 0;
                /*

-               * NEED A UPL_MAP ALIAS
-               */
+                * NEED A UPL_MAP ALIAS
+                */

                kr = vm_map_enter(map, dst_addr, (vm_map_size_t)size, (vm_map_offset_t) 0,
                kr = vm_map_enter(map, dst_addr, (vm_map_size_t)size, (vm_map_offset_t) 0,

-                                 VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_OSFMK,
-                                 upl->map_object, offset, FALSE,
-                                 VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
+                   VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_OSFMK,
+                   upl->map_object, offset, FALSE,
+                   VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);

 
                if (kr != KERN_SUCCESS) {
                        vm_object_deallocate(upl->map_object);
                        upl_unlock(upl);
 
                if (kr != KERN_SUCCESS) {
                        vm_object_deallocate(upl->map_object);
                        upl_unlock(upl);

-                       return(kr);
+                       return kr;

                }
                }

-       }
-       else {
+       } else {

                kr = vm_map_enter(map, dst_addr, (vm_map_size_t)size, (vm_map_offset_t) 0,
                kr = vm_map_enter(map, dst_addr, (vm_map_size_t)size, (vm_map_offset_t) 0,

-                                 VM_FLAGS_FIXED, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_OSFMK,
-                                 upl->map_object, offset, FALSE,
-                                 VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
-               if(kr)
+                   VM_FLAGS_FIXED, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_OSFMK,
+                   upl->map_object, offset, FALSE,
+                   VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
+               if (kr) {

                        panic("vm_map_enter failed for a Vector UPL\n");
                        panic("vm_map_enter failed for a Vector UPL\n");

+               }

        }
        vm_object_lock(upl->map_object);
 
        }
        vm_object_lock(upl->map_object);
 


@@ -6470,14 +6427,14 @@ process_upl_to_enter:
                m = vm_page_lookup(upl->map_object, offset);
 
                if (m) {
                m = vm_page_lookup(upl->map_object, offset);
 
                if (m) {

-                       m->pmapped = TRUE;
+                       m->vmp_pmapped = TRUE;

 
 

-                       /* CODE SIGNING ENFORCEMENT: page has been wpmapped, 
+                       /* CODE SIGNING ENFORCEMENT: page has been wpmapped,

                         * but only in kernel space. If this was on a user map,
                         * we'd have to set the wpmapped bit. */
                         * but only in kernel space. If this was on a user map,
                         * we'd have to set the wpmapped bit. */

-                       /* m->wpmapped = TRUE; */
+                       /* m->vmp_wpmapped = TRUE; */

                        assert(map->pmap == kernel_pmap);
                        assert(map->pmap == kernel_pmap);

-       
+

                        PMAP_ENTER(map->pmap, addr, m, VM_PROT_DEFAULT, VM_PROT_NONE, 0, TRUE, kr);
 
                        assert(kr == KERN_SUCCESS);
                        PMAP_ENTER(map->pmap, addr, m, VM_PROT_DEFAULT, VM_PROT_NONE, 0, TRUE, kr);
 
                        assert(kr == KERN_SUCCESS);


@@ -6496,15 +6453,16 @@ process_upl_to_enter:
        upl->flags |= UPL_PAGE_LIST_MAPPED;
        upl->kaddr = (vm_offset_t) *dst_addr;
        assert(upl->kaddr == *dst_addr);
        upl->flags |= UPL_PAGE_LIST_MAPPED;
        upl->kaddr = (vm_offset_t) *dst_addr;
        assert(upl->kaddr == *dst_addr);

-       
-       if(isVectorUPL)
+
+       if (isVectorUPL) {

                goto process_upl_to_enter;
                goto process_upl_to_enter;

+       }

 
        upl_unlock(upl);
 
        return KERN_SUCCESS;
 }
 
        upl_unlock(upl);
 
        return KERN_SUCCESS;
 }

-       
+

 /*
  * Internal routine to remove a UPL mapping from a VM map.
  *
 /*
  * Internal routine to remove a UPL mapping from a VM map.
  *


@@ -6517,59 +6475,63 @@ process_upl_to_enter:
  */
 kern_return_t
 vm_map_remove_upl(
  */
 kern_return_t
 vm_map_remove_upl(

-       vm_map_t        map, 
-       upl_t           upl)
+       vm_map_t        map,
+       upl_t           upl)

 {
 {

-       vm_address_t    addr;
-       upl_size_t      size;
-       int             isVectorUPL = 0, curr_upl = 0;
-       upl_t           vector_upl = NULL;
+       vm_address_t    addr;
+       upl_size_t      size;
+       int             isVectorUPL = 0, curr_upl = 0;
+       upl_t           vector_upl = NULL;

 
 

-       if (upl == UPL_NULL)
+       if (upl == UPL_NULL) {

                return KERN_INVALID_ARGUMENT;
                return KERN_INVALID_ARGUMENT;

+       }

 
 

-       if((isVectorUPL = vector_upl_is_valid(upl))) {
-               int     unmapped=0, valid_upls=0;
+       if ((isVectorUPL = vector_upl_is_valid(upl))) {
+               int     unmapped = 0, valid_upls = 0;

                vector_upl = upl;
                upl_lock(vector_upl);
                vector_upl = upl;
                upl_lock(vector_upl);

-               for(curr_upl=0; curr_upl < MAX_VECTOR_UPL_ELEMENTS; curr_upl++) {
+               for (curr_upl = 0; curr_upl < MAX_VECTOR_UPL_ELEMENTS; curr_upl++) {

                        upl =  vector_upl_subupl_byindex(vector_upl, curr_upl );
                        upl =  vector_upl_subupl_byindex(vector_upl, curr_upl );

-                       if(upl == NULL)
+                       if (upl == NULL) {

                                continue;
                                continue;

+                       }

                        valid_upls++;
                        valid_upls++;

-                       if (!(UPL_PAGE_LIST_MAPPED & upl->flags))
+                       if (!(UPL_PAGE_LIST_MAPPED & upl->flags)) {

                                unmapped++;
                                unmapped++;

+                       }

                }
 
                }
 

-               if(unmapped) {
-                       if(unmapped != valid_upls)
+               if (unmapped) {
+                       if (unmapped != valid_upls) {

                                panic("%d of the %d sub-upls within the Vector UPL is/are not mapped\n", unmapped, valid_upls);
                                panic("%d of the %d sub-upls within the Vector UPL is/are not mapped\n", unmapped, valid_upls);

-                       else {
+                       } else {

                                upl_unlock(vector_upl);
                                return KERN_FAILURE;
                        }
                }
                                upl_unlock(vector_upl);
                                return KERN_FAILURE;
                        }
                }

-               curr_upl=0;
-       }
-       else
+               curr_upl = 0;
+       } else {

                upl_lock(upl);
                upl_lock(upl);

+       }

 
 process_upl_to_remove:
 
 process_upl_to_remove:

-       if(isVectorUPL) {
-               if(curr_upl == MAX_VECTOR_UPL_ELEMENTS) {
+       if (isVectorUPL) {
+               if (curr_upl == MAX_VECTOR_UPL_ELEMENTS) {

                        vm_map_t v_upl_submap;
                        vm_offset_t v_upl_submap_dst_addr;
                        vector_upl_get_submap(vector_upl, &v_upl_submap, &v_upl_submap_dst_addr);
 
                        vm_map_t v_upl_submap;
                        vm_offset_t v_upl_submap_dst_addr;
                        vector_upl_get_submap(vector_upl, &v_upl_submap, &v_upl_submap_dst_addr);
 

-                       vm_map_remove(map, v_upl_submap_dst_addr, v_upl_submap_dst_addr + vector_upl->size, VM_MAP_NO_FLAGS);
+                       vm_map_remove(map, v_upl_submap_dst_addr, v_upl_submap_dst_addr + vector_upl->size, VM_MAP_REMOVE_NO_FLAGS);

                        vm_map_deallocate(v_upl_submap);
                        upl_unlock(vector_upl);
                        return KERN_SUCCESS;
                }
 
                upl =  vector_upl_subupl_byindex(vector_upl, curr_upl++ );
                        vm_map_deallocate(v_upl_submap);
                        upl_unlock(vector_upl);
                        return KERN_SUCCESS;
                }
 
                upl =  vector_upl_subupl_byindex(vector_upl, curr_upl++ );

-               if(upl == NULL)
-                       goto process_upl_to_remove;     
+               if (upl == NULL) {
+                       goto process_upl_to_remove;
+               }

        }
 
        if (upl->flags & UPL_PAGE_LIST_MAPPED) {
        }
 
        if (upl->flags & UPL_PAGE_LIST_MAPPED) {


@@ -6577,30 +6539,28 @@ process_upl_to_remove:
                size = upl->size;
 
                assert(upl->ref_count > 1);
                size = upl->size;
 
                assert(upl->ref_count > 1);

-               upl->ref_count--;               /* removing mapping ref */
+               upl->ref_count--;               /* removing mapping ref */

 
                upl->flags &= ~UPL_PAGE_LIST_MAPPED;
                upl->kaddr = (vm_offset_t) 0;
 
                upl->flags &= ~UPL_PAGE_LIST_MAPPED;
                upl->kaddr = (vm_offset_t) 0;

-               
-               if(!isVectorUPL) {
+
+               if (!isVectorUPL) {

                        upl_unlock(upl);
                        upl_unlock(upl);

-               
+

                        vm_map_remove(
                                map,
                                vm_map_trunc_page(addr,
                        vm_map_remove(
                                map,
                                vm_map_trunc_page(addr,

-                                                 VM_MAP_PAGE_MASK(map)),
+                               VM_MAP_PAGE_MASK(map)),

                                vm_map_round_page(addr + size,
                                vm_map_round_page(addr + size,

-                                                 VM_MAP_PAGE_MASK(map)),
-                               VM_MAP_NO_FLAGS);
-               
+                               VM_MAP_PAGE_MASK(map)),
+                               VM_MAP_REMOVE_NO_FLAGS);

                        return KERN_SUCCESS;
                        return KERN_SUCCESS;

-               }
-               else {
+               } else {

                        /*
                        /*

-                       * If it's a Vectored UPL, we'll be removing the entire
-                       * submap anyways, so no need to remove individual UPL
-                       * element mappings from within the submap
-                       */      
+                        * If it's a Vectored UPL, we'll be removing the entire
+                        * submap anyways, so no need to remove individual UPL
+                        * element mappings from within the submap
+                        */

                        goto process_upl_to_remove;
                }
        }
                        goto process_upl_to_remove;
                }
        }


@@ -6612,67 +6572,69 @@ process_upl_to_remove:
 
 kern_return_t
 upl_commit_range(
 
 kern_return_t
 upl_commit_range(

-       upl_t                   upl, 
-       upl_offset_t            offset, 
-       upl_size_t              size,
-       int                     flags,
-       upl_page_info_t         *page_list,
-       mach_msg_type_number_t  count,
-       boolean_t               *empty) 
+       upl_t                   upl,
+       upl_offset_t            offset,
+       upl_size_t              size,
+       int                     flags,
+       upl_page_info_t         *page_list,
+       mach_msg_type_number_t  count,
+       boolean_t               *empty)

 {
 {

-       upl_size_t              xfer_size, subupl_size = size;
-       vm_object_t             shadow_object;
-       vm_object_t             object;
-       vm_object_t             m_object;
-       vm_object_offset_t      target_offset;
-       upl_offset_t            subupl_offset = offset;
-       int                     entry;
-       wpl_array_t             lite_list;
-       int                     occupied;
-       int                     clear_refmod = 0;
-       int                     pgpgout_count = 0;
-       struct  vm_page_delayed_work    dw_array[DEFAULT_DELAYED_WORK_LIMIT];
-       struct  vm_page_delayed_work    *dwp;
-       int                     dw_count;
-       int                     dw_limit;
-       int                     isVectorUPL = 0;
-       upl_t                   vector_upl = NULL;
-       boolean_t               should_be_throttled = FALSE;
-
-       vm_page_t               nxt_page = VM_PAGE_NULL;
-       int                     fast_path_possible = 0;
-       int                     fast_path_full_commit = 0;
-       int                     throttle_page = 0;
-       int                     unwired_count = 0;
-       int                     local_queue_count = 0;
-       vm_page_t               first_local, last_local;
+       upl_size_t              xfer_size, subupl_size = size;
+       vm_object_t             shadow_object;
+       vm_object_t             object;
+       vm_object_t             m_object;
+       vm_object_offset_t      target_offset;
+       upl_offset_t            subupl_offset = offset;
+       int                     entry;
+       wpl_array_t             lite_list;
+       int                     occupied;
+       int                     clear_refmod = 0;
+       int                     pgpgout_count = 0;
+       struct  vm_page_delayed_work    dw_array[DEFAULT_DELAYED_WORK_LIMIT];
+       struct  vm_page_delayed_work    *dwp;
+       int                     dw_count;
+       int                     dw_limit;
+       int                     isVectorUPL = 0;
+       upl_t                   vector_upl = NULL;
+       boolean_t               should_be_throttled = FALSE;
+
+       vm_page_t               nxt_page = VM_PAGE_NULL;
+       int                     fast_path_possible = 0;
+       int                     fast_path_full_commit = 0;
+       int                     throttle_page = 0;
+       int                     unwired_count = 0;
+       int                     local_queue_count = 0;
+       vm_page_t               first_local, last_local;

 
        *empty = FALSE;
 
 
        *empty = FALSE;
 

-       if (upl == UPL_NULL)
+       if (upl == UPL_NULL) {

                return KERN_INVALID_ARGUMENT;
                return KERN_INVALID_ARGUMENT;

+       }

 
 

-       if (count == 0)
+       if (count == 0) {

                page_list = NULL;
                page_list = NULL;

+       }

 
 

-       if((isVectorUPL = vector_upl_is_valid(upl))) {
+       if ((isVectorUPL = vector_upl_is_valid(upl))) {

                vector_upl = upl;
                upl_lock(vector_upl);
                vector_upl = upl;
                upl_lock(vector_upl);

-       }
-       else
+       } else {

                upl_lock(upl);
                upl_lock(upl);

+       }

 
 process_upl_to_commit:
 
 
 process_upl_to_commit:
 

-       if(isVectorUPL) {
+       if (isVectorUPL) {

                size = subupl_size;
                offset = subupl_offset;
                size = subupl_size;
                offset = subupl_offset;

-               if(size == 0) {
+               if (size == 0) {

                        upl_unlock(vector_upl);
                        return KERN_SUCCESS;
                }
                upl =  vector_upl_subupl_byoffset(vector_upl, &offset, &size);
                        upl_unlock(vector_upl);
                        return KERN_SUCCESS;
                }
                upl =  vector_upl_subupl_byoffset(vector_upl, &offset, &size);

-               if(upl == NULL) {
+               if (upl == NULL) {

                        upl_unlock(vector_upl);
                        return KERN_FAILURE;
                }
                        upl_unlock(vector_upl);
                        return KERN_FAILURE;
                }


@@ -6684,54 +6646,58 @@ process_upl_to_commit:
 #if UPL_DEBUG
        if (upl->upl_commit_index < UPL_DEBUG_COMMIT_RECORDS) {
                (void) OSBacktrace(&upl->upl_commit_records[upl->upl_commit_index].c_retaddr[0], UPL_DEBUG_STACK_FRAMES);
 #if UPL_DEBUG
        if (upl->upl_commit_index < UPL_DEBUG_COMMIT_RECORDS) {
                (void) OSBacktrace(&upl->upl_commit_records[upl->upl_commit_index].c_retaddr[0], UPL_DEBUG_STACK_FRAMES);

-               
+

                upl->upl_commit_records[upl->upl_commit_index].c_beg = offset;
                upl->upl_commit_records[upl->upl_commit_index].c_end = (offset + size);
 
                upl->upl_commit_index++;
        }
 #endif
                upl->upl_commit_records[upl->upl_commit_index].c_beg = offset;
                upl->upl_commit_records[upl->upl_commit_index].c_end = (offset + size);
 
                upl->upl_commit_index++;
        }
 #endif

-       if (upl->flags & UPL_DEVICE_MEMORY)
+       if (upl->flags & UPL_DEVICE_MEMORY) {

                xfer_size = 0;
                xfer_size = 0;

-       else if ((offset + size) <= upl->size)
-               xfer_size = size;
-       else {
-               if(!isVectorUPL)
+       } else if ((offset + size) <= upl->size) {
+               xfer_size = size;
+       } else {
+               if (!isVectorUPL) {

                        upl_unlock(upl);
                        upl_unlock(upl);

-               else {
+               } else {

                        upl_unlock(vector_upl);
                }
                return KERN_FAILURE;
        }
                        upl_unlock(vector_upl);
                }
                return KERN_FAILURE;
        }

-       if (upl->flags & UPL_SET_DIRTY)
+       if (upl->flags & UPL_SET_DIRTY) {

                flags |= UPL_COMMIT_SET_DIRTY;
                flags |= UPL_COMMIT_SET_DIRTY;

-       if (upl->flags & UPL_CLEAR_DIRTY)
-               flags |= UPL_COMMIT_CLEAR_DIRTY;
+       }
+       if (upl->flags & UPL_CLEAR_DIRTY) {
+               flags |= UPL_COMMIT_CLEAR_DIRTY;
+       }

 
 

-       if (upl->flags & UPL_INTERNAL)
+       if (upl->flags & UPL_INTERNAL) {

                lite_list = (wpl_array_t) ((((uintptr_t)upl) + sizeof(struct upl))
                lite_list = (wpl_array_t) ((((uintptr_t)upl) + sizeof(struct upl))

-                                          + ((upl->size/PAGE_SIZE) * sizeof(upl_page_info_t)));
-       else
+                   + ((upl->size / PAGE_SIZE) * sizeof(upl_page_info_t)));
+       } else {

                lite_list = (wpl_array_t) (((uintptr_t)upl) + sizeof(struct upl));
                lite_list = (wpl_array_t) (((uintptr_t)upl) + sizeof(struct upl));

+       }

 
        object = upl->map_object;
 
        if (upl->flags & UPL_SHADOWED) {
 
        object = upl->map_object;
 
        if (upl->flags & UPL_SHADOWED) {

-               vm_object_lock(object);
+               vm_object_lock(object);

                shadow_object = object->shadow;
        } else {
                shadow_object = object;
        }
                shadow_object = object->shadow;
        } else {
                shadow_object = object;
        }

-       entry = offset/PAGE_SIZE;
+       entry = offset / PAGE_SIZE;

        target_offset = (vm_object_offset_t)offset;
 
        assert(!(target_offset & PAGE_MASK));
        assert(!(xfer_size & PAGE_MASK));
 
        target_offset = (vm_object_offset_t)offset;
 
        assert(!(target_offset & PAGE_MASK));
        assert(!(xfer_size & PAGE_MASK));
 

-       if (upl->flags & UPL_KERNEL_OBJECT)
+       if (upl->flags & UPL_KERNEL_OBJECT) {

                vm_object_lock_shared(shadow_object);
                vm_object_lock_shared(shadow_object);

-       else
+       } else {

                vm_object_lock(shadow_object);
                vm_object_lock(shadow_object);

+       }

 
        VM_OBJECT_WIRED_PAGE_UPDATE_START(shadow_object);
 
 
        VM_OBJECT_WIRED_PAGE_UPDATE_START(shadow_object);
 


@@ -6751,14 +6717,15 @@ process_upl_to_commit:
                 */
                flags &= ~UPL_COMMIT_CS_VALIDATED;
        }
                 */
                flags &= ~UPL_COMMIT_CS_VALIDATED;
        }

-       if (! page_list) {
+       if (!page_list) {

                /*
                 * No page list to get the code-signing info from !?
                 */
                flags &= ~UPL_COMMIT_CS_VALIDATED;
        }
                /*
                 * No page list to get the code-signing info from !?
                 */
                flags &= ~UPL_COMMIT_CS_VALIDATED;
        }

-       if (!VM_DYNAMIC_PAGING_ENABLED() && shadow_object->internal)
+       if (!VM_DYNAMIC_PAGING_ENABLED() && shadow_object->internal) {

                should_be_throttled = TRUE;
                should_be_throttled = TRUE;

+       }

 
        dwp = &dw_array[0];
        dw_count = 0;
 
        dwp = &dw_array[0];
        dw_count = 0;


@@ -6769,9 +6736,7 @@ process_upl_to_commit:
            !isVectorUPL &&
            shadow_object->purgable != VM_PURGABLE_VOLATILE &&
            shadow_object->purgable != VM_PURGABLE_EMPTY) {
            !isVectorUPL &&
            shadow_object->purgable != VM_PURGABLE_VOLATILE &&
            shadow_object->purgable != VM_PURGABLE_EMPTY) {

-

                if (!vm_page_queue_empty(&shadow_object->memq)) {
                if (!vm_page_queue_empty(&shadow_object->memq)) {

-

                        if (size == shadow_object->vo_size) {
                                nxt_page = (vm_page_t)vm_page_queue_first(&shadow_object->memq);
                                fast_path_full_commit = 1;
                        if (size == shadow_object->vo_size) {
                                nxt_page = (vm_page_t)vm_page_queue_first(&shadow_object->memq);
                                fast_path_full_commit = 1;


@@ -6780,8 +6745,8 @@ process_upl_to_commit:
 
                        if (!VM_DYNAMIC_PAGING_ENABLED() && shadow_object->internal &&
                            (shadow_object->purgable == VM_PURGABLE_DENY ||
 
                        if (!VM_DYNAMIC_PAGING_ENABLED() && shadow_object->internal &&
                            (shadow_object->purgable == VM_PURGABLE_DENY ||

-                            shadow_object->purgable == VM_PURGABLE_NONVOLATILE ||
-                            shadow_object->purgable == VM_PURGABLE_VOLATILE)) {
+                           shadow_object->purgable == VM_PURGABLE_NONVOLATILE ||
+                           shadow_object->purgable == VM_PURGABLE_VOLATILE)) {

                                throttle_page = 1;
                        }
                }
                                throttle_page = 1;
                        }
                }


@@ -6790,7 +6755,7 @@ process_upl_to_commit:
        last_local = VM_PAGE_NULL;
 
        while (xfer_size) {
        last_local = VM_PAGE_NULL;
 
        while (xfer_size) {

-               vm_page_t       t, m;
+               vm_page_t       t, m;

 
                dwp->dw_mask = 0;
                clear_refmod = 0;
 
                dwp->dw_mask = 0;
                clear_refmod = 0;


@@ -6798,42 +6763,45 @@ process_upl_to_commit:
                m = VM_PAGE_NULL;
 
                if (upl->flags & UPL_LITE) {
                m = VM_PAGE_NULL;
 
                if (upl->flags & UPL_LITE) {

-                       unsigned int    pg_num;
+                       unsigned int    pg_num;

 
                        if (nxt_page != VM_PAGE_NULL) {
                                m = nxt_page;
 
                        if (nxt_page != VM_PAGE_NULL) {
                                m = nxt_page;

-                               nxt_page = (vm_page_t)vm_page_queue_next(&nxt_page->listq);
-                               target_offset = m->offset;
+                               nxt_page = (vm_page_t)vm_page_queue_next(&nxt_page->vmp_listq);
+                               target_offset = m->vmp_offset;

                        }
                        }

-                       pg_num = (unsigned int) (target_offset/PAGE_SIZE);
-                       assert(pg_num == target_offset/PAGE_SIZE);
+                       pg_num = (unsigned int) (target_offset / PAGE_SIZE);
+                       assert(pg_num == target_offset / PAGE_SIZE);

 
 

-                       if (lite_list[pg_num>>5] & (1 << (pg_num & 31))) {
-                               lite_list[pg_num>>5] &= ~(1 << (pg_num & 31));
+                       if (lite_list[pg_num >> 5] & (1 << (pg_num & 31))) {
+                               lite_list[pg_num >> 5] &= ~(1 << (pg_num & 31));

 
 

-                               if (!(upl->flags & UPL_KERNEL_OBJECT) && m == VM_PAGE_NULL)
+                               if (!(upl->flags & UPL_KERNEL_OBJECT) && m == VM_PAGE_NULL) {

                                        m = vm_page_lookup(shadow_object, target_offset + (upl->offset - shadow_object->paging_offset));
                                        m = vm_page_lookup(shadow_object, target_offset + (upl->offset - shadow_object->paging_offset));

-                       } else
+                               }
+                       } else {

                                m = NULL;
                                m = NULL;

+                       }

                }
                if (upl->flags & UPL_SHADOWED) {
                }
                if (upl->flags & UPL_SHADOWED) {

-                       if ((t = vm_page_lookup(object, target_offset)) != VM_PAGE_NULL) {
-
-                               t->free_when_done = FALSE;
+                       if ((t = vm_page_lookup(object, target_offset)) != VM_PAGE_NULL) {
+                               t->vmp_free_when_done = FALSE;

 
                                VM_PAGE_FREE(t);
 
 
                                VM_PAGE_FREE(t);
 

-                               if (!(upl->flags & UPL_KERNEL_OBJECT) && m == VM_PAGE_NULL)
+                               if (!(upl->flags & UPL_KERNEL_OBJECT) && m == VM_PAGE_NULL) {

                                        m = vm_page_lookup(shadow_object, target_offset + object->vo_shadow_offset);
                                        m = vm_page_lookup(shadow_object, target_offset + object->vo_shadow_offset);

+                               }

                        }
                }
                        }
                }

-               if (m == VM_PAGE_NULL)
+               if (m == VM_PAGE_NULL) {

                        goto commit_next_page;
                        goto commit_next_page;

+               }

 
                m_object = VM_PAGE_OBJECT(m);
 
 
                m_object = VM_PAGE_OBJECT(m);
 

-               if (m->vm_page_q_state == VM_PAGE_USED_BY_COMPRESSOR) {
-                       assert(m->busy);
+               if (m->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) {
+                       assert(m->vmp_busy);

 
                        dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
                        goto commit_next_page;
 
                        dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
                        goto commit_next_page;


@@ -6845,25 +6813,26 @@ process_upl_to_commit:
                         * Set the code signing bits according to
                         * what the UPL says they should be.
                         */
                         * Set the code signing bits according to
                         * what the UPL says they should be.
                         */

-                       m->cs_validated = page_list[entry].cs_validated;
-                       m->cs_tainted = page_list[entry].cs_tainted;
-                       m->cs_nx = page_list[entry].cs_nx;
+                       m->vmp_cs_validated = page_list[entry].cs_validated;
+                       m->vmp_cs_tainted = page_list[entry].cs_tainted;
+                       m->vmp_cs_nx = page_list[entry].cs_nx;
+               }
+               if (flags & UPL_COMMIT_WRITTEN_BY_KERNEL) {
+                       m->vmp_written_by_kernel = TRUE;

                }
                }

-               if (flags & UPL_COMMIT_WRITTEN_BY_KERNEL)
-                       m->written_by_kernel = TRUE;

 
                if (upl->flags & UPL_IO_WIRE) {
 
                if (upl->flags & UPL_IO_WIRE) {

-
-                       if (page_list)
+                       if (page_list) {

                                page_list[entry].phys_addr = 0;
                                page_list[entry].phys_addr = 0;

+                       }

 
                        if (flags & UPL_COMMIT_SET_DIRTY) {
                                SET_PAGE_DIRTY(m, FALSE);
                        } else if (flags & UPL_COMMIT_CLEAR_DIRTY) {
 
                        if (flags & UPL_COMMIT_SET_DIRTY) {
                                SET_PAGE_DIRTY(m, FALSE);
                        } else if (flags & UPL_COMMIT_CLEAR_DIRTY) {

-                               m->dirty = FALSE;
+                               m->vmp_dirty = FALSE;

 
 

-                               if (! (flags & UPL_COMMIT_CS_VALIDATED) &&
-                                   m->cs_validated && !m->cs_tainted) {
+                               if (!(flags & UPL_COMMIT_CS_VALIDATED) &&
+                                   m->vmp_cs_validated && !m->vmp_cs_tainted) {

                                        /*
                                         * CODE SIGNING:
                                         * This page is no longer dirty
                                        /*
                                         * CODE SIGNING:
                                         * This page is no longer dirty


@@ -6871,15 +6840,10 @@ process_upl_to_commit:
                                         * so it will need to be
                                         * re-validated.
                                         */
                                         * so it will need to be
                                         * re-validated.
                                         */

-                                       if (m->slid) {
-                                               panic("upl_commit_range(%p): page %p was slid\n",
-                                                     upl, m);
-                                       }
-                                       assert(!m->slid);
-                                       m->cs_validated = FALSE;
-#if DEVELOPMENT || DEBUG
-                                       vm_cs_validated_resets++;
-#endif
+                                       m->vmp_cs_validated = FALSE;
+
+                                       VM_PAGEOUT_DEBUG(vm_cs_validated_resets, 1);
+

                                        pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
                                }
                                clear_refmod |= VM_MEM_MODIFIED;
                                        pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
                                }
                                clear_refmod |= VM_MEM_MODIFIED;


@@ -6895,33 +6859,34 @@ process_upl_to_commit:
                        if (fast_path_possible) {
                                assert(m_object->purgable != VM_PURGABLE_EMPTY);
                                assert(m_object->purgable != VM_PURGABLE_VOLATILE);
                        if (fast_path_possible) {
                                assert(m_object->purgable != VM_PURGABLE_EMPTY);
                                assert(m_object->purgable != VM_PURGABLE_VOLATILE);

-                               if (m->absent) {
-                                       assert(m->vm_page_q_state == VM_PAGE_NOT_ON_Q);
-                                       assert(m->wire_count == 0);
-                                       assert(m->busy);
+                               if (m->vmp_absent) {
+                                       assert(m->vmp_q_state == VM_PAGE_NOT_ON_Q);
+                                       assert(m->vmp_wire_count == 0);
+                                       assert(m->vmp_busy);

 
 

-                                       m->absent = FALSE;
+                                       m->vmp_absent = FALSE;

                                        dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
                                } else {
                                        dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
                                } else {

-                                       if (m->wire_count == 0)
+                                       if (m->vmp_wire_count == 0) {

                                                panic("wire_count == 0, m = %p, obj = %p\n", m, shadow_object);
                                                panic("wire_count == 0, m = %p, obj = %p\n", m, shadow_object);

-                                       assert(m->vm_page_q_state == VM_PAGE_IS_WIRED);
+                                       }
+                                       assert(m->vmp_q_state == VM_PAGE_IS_WIRED);

 
                                        /*
                                         * XXX FBDP need to update some other
                                         * counters here (purgeable_wired_count)
                                         * (ledgers), ...
                                         */
 
                                        /*
                                         * XXX FBDP need to update some other
                                         * counters here (purgeable_wired_count)
                                         * (ledgers), ...
                                         */

-                                       assert(m->wire_count > 0);
-                                       m->wire_count--;
+                                       assert(m->vmp_wire_count > 0);
+                                       m->vmp_wire_count--;

 
 

-                                       if (m->wire_count == 0) {
-                                               m->vm_page_q_state = VM_PAGE_NOT_ON_Q;
+                                       if (m->vmp_wire_count == 0) {
+                                               m->vmp_q_state = VM_PAGE_NOT_ON_Q;

                                                unwired_count++;
                                        }
                                }
                                                unwired_count++;
                                        }
                                }

-                               if (m->wire_count == 0) {
-                                       assert(m->pageq.next == 0 && m->pageq.prev == 0);
+                               if (m->vmp_wire_count == 0) {
+                                       assert(m->vmp_pageq.next == 0 && m->vmp_pageq.prev == 0);

 
                                        if (last_local == VM_PAGE_NULL) {
                                                assert(first_local == VM_PAGE_NULL);
 
                                        if (last_local == VM_PAGE_NULL) {
                                                assert(first_local == VM_PAGE_NULL);


@@ -6931,22 +6896,24 @@ process_upl_to_commit:
                                        } else {
                                                assert(first_local != VM_PAGE_NULL);
 
                                        } else {
                                                assert(first_local != VM_PAGE_NULL);
 

-                                               m->pageq.next = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(first_local);
-                                               first_local->pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(m);
+                                               m->vmp_pageq.next = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(first_local);
+                                               first_local->vmp_pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(m);

                                                first_local = m;
                                        }
                                        local_queue_count++;
 
                                        if (throttle_page) {
                                                first_local = m;
                                        }
                                        local_queue_count++;
 
                                        if (throttle_page) {

-                                               m->vm_page_q_state = VM_PAGE_ON_THROTTLED_Q;
+                                               m->vmp_q_state = VM_PAGE_ON_THROTTLED_Q;

                                        } else {
                                                if (flags & UPL_COMMIT_INACTIVATE) {
                                        } else {
                                                if (flags & UPL_COMMIT_INACTIVATE) {

-                                                       if (shadow_object->internal)
-                                                               m->vm_page_q_state = VM_PAGE_ON_INACTIVE_INTERNAL_Q;
-                                                       else
-                                                               m->vm_page_q_state = VM_PAGE_ON_INACTIVE_EXTERNAL_Q;
-                                               } else
-                                                       m->vm_page_q_state = VM_PAGE_ON_ACTIVE_Q;
+                                                       if (shadow_object->internal) {
+                                                               m->vmp_q_state = VM_PAGE_ON_INACTIVE_INTERNAL_Q;
+                                                       } else {
+                                                               m->vmp_q_state = VM_PAGE_ON_INACTIVE_EXTERNAL_Q;
+                                                       }
+                                               } else {
+                                                       m->vmp_q_state = VM_PAGE_ON_ACTIVE_Q;
+                                               }

                                        }
                                }
                        } else {
                                        }
                                }
                        } else {


@@ -6954,25 +6921,28 @@ process_upl_to_commit:
                                        dwp->dw_mask |= DW_vm_page_deactivate_internal;
                                        clear_refmod |= VM_MEM_REFERENCED;
                                }
                                        dwp->dw_mask |= DW_vm_page_deactivate_internal;
                                        clear_refmod |= VM_MEM_REFERENCED;
                                }

-                               if (m->absent) {
-                                       if (flags & UPL_COMMIT_FREE_ABSENT)
+                               if (m->vmp_absent) {
+                                       if (flags & UPL_COMMIT_FREE_ABSENT) {

                                                dwp->dw_mask |= DW_vm_page_free;
                                                dwp->dw_mask |= DW_vm_page_free;

-                                       else {
-                                               m->absent = FALSE;
+                                       } else {
+                                               m->vmp_absent = FALSE;

                                                dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
 
                                                dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
 

-                                               if ( !(dwp->dw_mask & DW_vm_page_deactivate_internal))
+                                               if (!(dwp->dw_mask & DW_vm_page_deactivate_internal)) {

                                                        dwp->dw_mask |= DW_vm_page_activate;
                                                        dwp->dw_mask |= DW_vm_page_activate;

+                                               }

                                        }
                                        }

-                               } else
+                               } else {

                                        dwp->dw_mask |= DW_vm_page_unwire;
                                        dwp->dw_mask |= DW_vm_page_unwire;

+                               }

                        }
                        goto commit_next_page;
                }
                        }
                        goto commit_next_page;
                }

-               assert(m->vm_page_q_state != VM_PAGE_USED_BY_COMPRESSOR);
+               assert(m->vmp_q_state != VM_PAGE_USED_BY_COMPRESSOR);

 
 

-               if (page_list)
+               if (page_list) {

                        page_list[entry].phys_addr = 0;
                        page_list[entry].phys_addr = 0;

+               }

 
                /*
                 * make sure to clear the hardware
 
                /*
                 * make sure to clear the hardware


@@ -6982,18 +6952,20 @@ process_upl_to_commit:
                 * change of state
                 */
                if (flags & UPL_COMMIT_CLEAR_DIRTY) {
                 * change of state
                 */
                if (flags & UPL_COMMIT_CLEAR_DIRTY) {

-                       m->dirty = FALSE;
+                       m->vmp_dirty = FALSE;

 
                        clear_refmod |= VM_MEM_MODIFIED;
                }
 
                        clear_refmod |= VM_MEM_MODIFIED;
                }

-               if (m->laundry)
+               if (m->vmp_laundry) {

                        dwp->dw_mask |= DW_vm_pageout_throttle_up;
                        dwp->dw_mask |= DW_vm_pageout_throttle_up;

+               }
+
+               if (VM_PAGE_WIRED(m)) {
+                       m->vmp_free_when_done = FALSE;
+               }

 
 

-               if (VM_PAGE_WIRED(m))
-                       m->free_when_done = FALSE;
-               
-               if (! (flags & UPL_COMMIT_CS_VALIDATED) &&
-                   m->cs_validated && !m->cs_tainted) {
+               if (!(flags & UPL_COMMIT_CS_VALIDATED) &&
+                   m->vmp_cs_validated && !m->vmp_cs_tainted) {

                        /*
                         * CODE SIGNING:
                         * This page is no longer dirty
                        /*
                         * CODE SIGNING:
                         * This page is no longer dirty


@@ -7001,27 +6973,23 @@ process_upl_to_commit:
                         * so it will need to be
                         * re-validated.
                         */
                         * so it will need to be
                         * re-validated.
                         */

-                       if (m->slid) {
-                               panic("upl_commit_range(%p): page %p was slid\n",
-                                     upl, m);
-                       }
-                       assert(!m->slid);
-                       m->cs_validated = FALSE;
-#if DEVELOPMENT || DEBUG
-                       vm_cs_validated_resets++;
-#endif
+                       m->vmp_cs_validated = FALSE;
+
+                       VM_PAGEOUT_DEBUG(vm_cs_validated_resets, 1);
+

                        pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
                }
                        pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
                }

-               if (m->overwriting) {
+               if (m->vmp_overwriting) {

                        /*
                         * the (COPY_OUT_FROM == FALSE) request_page_list case
                         */
                        /*
                         * the (COPY_OUT_FROM == FALSE) request_page_list case
                         */

-                       if (m->busy) {
+                       if (m->vmp_busy) {

 #if CONFIG_PHANTOM_CACHE
 #if CONFIG_PHANTOM_CACHE

-                               if (m->absent && !m_object->internal)
+                               if (m->vmp_absent && !m_object->internal) {

                                        dwp->dw_mask |= DW_vm_phantom_cache_update;
                                        dwp->dw_mask |= DW_vm_phantom_cache_update;

+                               }

 #endif
 #endif

-                               m->absent = FALSE;
+                               m->vmp_absent = FALSE;

 
                                dwp->dw_mask |= DW_clear_busy;
                        } else {
 
                                dwp->dw_mask |= DW_clear_busy;
                        } else {


@@ -7034,37 +7002,34 @@ process_upl_to_commit:
 
                                dwp->dw_mask |= DW_vm_page_unwire; /* reactivates */
                        }
 
                                dwp->dw_mask |= DW_vm_page_unwire; /* reactivates */
                        }

-                       m->overwriting = FALSE;
+                       m->vmp_overwriting = FALSE;

                }
                }

-               m->cleaning = FALSE;
+               m->vmp_cleaning = FALSE;

 
 

-               if (m->free_when_done) {
-                       /* 
+               if (m->vmp_free_when_done) {
+                       /*

                         * With the clean queue enabled, UPL_PAGEOUT should
                         * With the clean queue enabled, UPL_PAGEOUT should

-                        * no longer set the pageout bit. It's pages now go 
+                        * no longer set the pageout bit. It's pages now go

                         * to the clean queue.
                         */
                        assert(!(flags & UPL_PAGEOUT));
                        assert(!m_object->internal);
 
                         * to the clean queue.
                         */
                        assert(!(flags & UPL_PAGEOUT));
                        assert(!m_object->internal);
 

-                       m->free_when_done = FALSE;
-#if MACH_CLUSTER_STATS
-                       if (m->wanted) vm_pageout_target_collisions++;
-#endif
+                       m->vmp_free_when_done = FALSE;
+

                        if ((flags & UPL_COMMIT_SET_DIRTY) ||
                        if ((flags & UPL_COMMIT_SET_DIRTY) ||

-                           (m->pmapped && (pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)) & VM_MEM_MODIFIED))) {
+                           (m->vmp_pmapped && (pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)) & VM_MEM_MODIFIED))) {

                                /*
                                 * page was re-dirtied after we started
                                /*
                                 * page was re-dirtied after we started

-                                * the pageout... reactivate it since 
+                                * the pageout... reactivate it since

                                 * we don't know whether the on-disk
                                 * copy matches what is now in memory
                                 */
                                SET_PAGE_DIRTY(m, FALSE);
                                 * we don't know whether the on-disk
                                 * copy matches what is now in memory
                                 */
                                SET_PAGE_DIRTY(m, FALSE);

-                               
+

                                dwp->dw_mask |= DW_vm_page_activate | DW_PAGE_WAKEUP;
 
                                if (upl->flags & UPL_PAGEOUT) {
                                dwp->dw_mask |= DW_vm_page_activate | DW_PAGE_WAKEUP;
 
                                if (upl->flags & UPL_PAGEOUT) {

-                                       CLUSTER_STAT(vm_pageout_target_page_dirtied++;)

                                        VM_STAT_INCR(reactivations);
                                        DTRACE_VM2(pgrec, int, 1, (uint64_t *), NULL);
                                }
                                        VM_STAT_INCR(reactivations);
                                        DTRACE_VM2(pgrec, int, 1, (uint64_t *), NULL);
                                }


@@ -7078,21 +7043,13 @@ process_upl_to_commit:
                                } else {
                                        DTRACE_VM2(fspgout, int, 1, (uint64_t *), NULL);
                                }
                                } else {
                                        DTRACE_VM2(fspgout, int, 1, (uint64_t *), NULL);
                                }

-                               m->dirty = FALSE;
-                               m->busy = TRUE;
+                               m->vmp_dirty = FALSE;
+                               m->vmp_busy = TRUE;

 
                                dwp->dw_mask |= DW_vm_page_free;
                        }
                        goto commit_next_page;
                }
 
                                dwp->dw_mask |= DW_vm_page_free;
                        }
                        goto commit_next_page;
                }

-#if MACH_CLUSTER_STATS
-               if (m->wpmapped)
-                       m->dirty = pmap_is_modified(VM_PAGE_GET_PHYS_PAGE(m));
-
-               if (m->dirty)   vm_pageout_cluster_dirtied++;
-               else            vm_pageout_cluster_cleaned++;
-               if (m->wanted)  vm_pageout_cluster_collisions++;
-#endif

                /*
                 * It is a part of the semantic of COPYOUT_FROM
                 * UPLs that a commit implies cache sync
                /*
                 * It is a part of the semantic of COPYOUT_FROM
                 * UPLs that a commit implies cache sync


@@ -7100,25 +7057,25 @@ process_upl_to_commit:
                 * this can be used to strip the precious bit
                 * as well as clean
                 */
                 * this can be used to strip the precious bit
                 * as well as clean
                 */

-               if ((upl->flags & UPL_PAGE_SYNC_DONE) || (flags & UPL_COMMIT_CLEAR_PRECIOUS))
-                       m->precious = FALSE;
+               if ((upl->flags & UPL_PAGE_SYNC_DONE) || (flags & UPL_COMMIT_CLEAR_PRECIOUS)) {
+                       m->vmp_precious = FALSE;
+               }

 
                if (flags & UPL_COMMIT_SET_DIRTY) {
                        SET_PAGE_DIRTY(m, FALSE);
                } else {
 
                if (flags & UPL_COMMIT_SET_DIRTY) {
                        SET_PAGE_DIRTY(m, FALSE);
                } else {

-                       m->dirty = FALSE;
+                       m->vmp_dirty = FALSE;

                }
 
                /* with the clean queue on, move *all* cleaned pages to the clean queue */
                }
 
                /* with the clean queue on, move *all* cleaned pages to the clean queue */

-               if (hibernate_cleaning_in_progress == FALSE && !m->dirty && (upl->flags & UPL_PAGEOUT)) {
+               if (hibernate_cleaning_in_progress == FALSE && !m->vmp_dirty && (upl->flags & UPL_PAGEOUT)) {

                        pgpgout_count++;
 
                        VM_STAT_INCR(pageouts);
                        DTRACE_VM2(pgout, int, 1, (uint64_t *), NULL);
 
                        dwp->dw_mask |= DW_enqueue_cleaned;
                        pgpgout_count++;
 
                        VM_STAT_INCR(pageouts);
                        DTRACE_VM2(pgout, int, 1, (uint64_t *), NULL);
 
                        dwp->dw_mask |= DW_enqueue_cleaned;

-                       vm_pageout_enqueued_cleaned_from_inactive_dirty++;
-               } else if (should_be_throttled == TRUE && (m->vm_page_q_state == VM_PAGE_NOT_ON_Q)) {
+               } else if (should_be_throttled == TRUE && (m->vmp_q_state == VM_PAGE_NOT_ON_Q)) {

                        /*
                         * page coming back in from being 'frozen'...
                         * it was dirty before it was frozen, so keep it so
                        /*
                         * page coming back in from being 'frozen'...
                         * it was dirty before it was frozen, so keep it so


@@ -7127,18 +7084,16 @@ process_upl_to_commit:
                         */
                        SET_PAGE_DIRTY(m, FALSE);
                        dwp->dw_mask |= DW_vm_page_activate;
                         */
                        SET_PAGE_DIRTY(m, FALSE);
                        dwp->dw_mask |= DW_vm_page_activate;

-

                } else {
                } else {

-                       if ((flags & UPL_COMMIT_INACTIVATE) && !m->clustered && (m->vm_page_q_state != VM_PAGE_ON_SPECULATIVE_Q)) {
+                       if ((flags & UPL_COMMIT_INACTIVATE) && !m->vmp_clustered && (m->vmp_q_state != VM_PAGE_ON_SPECULATIVE_Q)) {

                                dwp->dw_mask |= DW_vm_page_deactivate_internal;
                                clear_refmod |= VM_MEM_REFERENCED;
                                dwp->dw_mask |= DW_vm_page_deactivate_internal;
                                clear_refmod |= VM_MEM_REFERENCED;

-                       } else if ( !VM_PAGE_PAGEABLE(m)) {
-
-                               if (m->clustered || (flags & UPL_COMMIT_SPECULATE))
+                       } else if (!VM_PAGE_PAGEABLE(m)) {
+                               if (m->vmp_clustered || (flags & UPL_COMMIT_SPECULATE)) {

                                        dwp->dw_mask |= DW_vm_page_speculate;
                                        dwp->dw_mask |= DW_vm_page_speculate;

-                               else if (m->reference)
+                               } else if (m->vmp_reference) {

                                        dwp->dw_mask |= DW_vm_page_activate;
                                        dwp->dw_mask |= DW_vm_page_activate;

-                               else {
+                               } else {

                                        dwp->dw_mask |= DW_vm_page_deactivate_internal;
                                        clear_refmod |= VM_MEM_REFERENCED;
                                }
                                        dwp->dw_mask |= DW_vm_page_deactivate_internal;
                                        clear_refmod |= VM_MEM_REFERENCED;
                                }


@@ -7158,8 +7113,9 @@ process_upl_to_commit:
                dwp->dw_mask |= DW_PAGE_WAKEUP;
 
 commit_next_page:
                dwp->dw_mask |= DW_PAGE_WAKEUP;
 
 commit_next_page:

-               if (clear_refmod)
+               if (clear_refmod) {

                        pmap_clear_refmod(VM_PAGE_GET_PHYS_PAGE(m), clear_refmod);
                        pmap_clear_refmod(VM_PAGE_GET_PHYS_PAGE(m), clear_refmod);

+               }

 
                target_offset += PAGE_SIZE_64;
                xfer_size -= PAGE_SIZE;
 
                target_offset += PAGE_SIZE_64;
                xfer_size -= PAGE_SIZE;


@@ -7171,43 +7127,46 @@ commit_next_page:
 
                                if (dw_count >= dw_limit) {
                                        vm_page_do_delayed_work(shadow_object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count);
 
                                if (dw_count >= dw_limit) {
                                        vm_page_do_delayed_work(shadow_object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count);

-                       
+

                                        dwp = &dw_array[0];
                                        dw_count = 0;
                                }
                        } else {
                                        dwp = &dw_array[0];
                                        dw_count = 0;
                                }
                        } else {

-                               if (dwp->dw_mask & DW_clear_busy)
-                                       m->busy = FALSE;
+                               if (dwp->dw_mask & DW_clear_busy) {
+                                       m->vmp_busy = FALSE;
+                               }

 
 

-                               if (dwp->dw_mask & DW_PAGE_WAKEUP)
+                               if (dwp->dw_mask & DW_PAGE_WAKEUP) {

                                        PAGE_WAKEUP(m);
                                        PAGE_WAKEUP(m);

+                               }

                        }
                }
        }
                        }
                }
        }

-       if (dw_count)
+       if (dw_count) {

                vm_page_do_delayed_work(shadow_object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count);
                vm_page_do_delayed_work(shadow_object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count);

+       }

 
        if (fast_path_possible) {
 
        if (fast_path_possible) {

-

                assert(shadow_object->purgable != VM_PURGABLE_VOLATILE);
                assert(shadow_object->purgable != VM_PURGABLE_EMPTY);
 
                if (local_queue_count || unwired_count) {
                assert(shadow_object->purgable != VM_PURGABLE_VOLATILE);
                assert(shadow_object->purgable != VM_PURGABLE_EMPTY);
 
                if (local_queue_count || unwired_count) {

-

                        if (local_queue_count) {
                        if (local_queue_count) {

-                               vm_page_t       first_target;
-                               vm_page_queue_head_t    *target_queue;
+                               vm_page_t       first_target;
+                               vm_page_queue_head_t    *target_queue;

 
 

-                               if (throttle_page)
+                               if (throttle_page) {

                                        target_queue = &vm_page_queue_throttled;
                                        target_queue = &vm_page_queue_throttled;

-                               else {
+                               } else {

                                        if (flags & UPL_COMMIT_INACTIVATE) {
                                        if (flags & UPL_COMMIT_INACTIVATE) {

-                                               if (shadow_object->internal)
+                                               if (shadow_object->internal) {

                                                        target_queue = &vm_page_queue_anonymous;
                                                        target_queue = &vm_page_queue_anonymous;

-                                               else
+                                               } else {

                                                        target_queue = &vm_page_queue_inactive;
                                                        target_queue = &vm_page_queue_inactive;

-                                       } else
+                                               }
+                                       } else {

                                                target_queue = &vm_page_queue_active;
                                                target_queue = &vm_page_queue_active;

+                                       }

                                }
                                /*
                                 * Transfer the entire local queue to a regular LRU page queues.
                                }
                                /*
                                 * Transfer the entire local queue to a regular LRU page queues.


@@ -7216,14 +7175,15 @@ commit_next_page:
 
                                first_target = (vm_page_t) vm_page_queue_first(target_queue);
 
 
                                first_target = (vm_page_t) vm_page_queue_first(target_queue);
 

-                               if (vm_page_queue_empty(target_queue))
+                               if (vm_page_queue_empty(target_queue)) {

                                        target_queue->prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(last_local);
                                        target_queue->prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(last_local);

-                               else
-                                       first_target->pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(last_local);
+                               } else {
+                                       first_target->vmp_pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(last_local);
+                               }

 
                                target_queue->next = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(first_local);
 
                                target_queue->next = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(first_local);

-                               first_local->pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(target_queue);
-                               last_local->pageq.next = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(first_target);
+                               first_local->vmp_pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(target_queue);
+                               last_local->vmp_pageq.next = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(first_target);

 
                                /*
                                 * Adjust the global page counts.
 
                                /*
                                 * Adjust the global page counts.


@@ -7232,23 +7192,26 @@ commit_next_page:
                                        vm_page_throttled_count += local_queue_count;
                                } else {
                                        if (flags & UPL_COMMIT_INACTIVATE) {
                                        vm_page_throttled_count += local_queue_count;
                                } else {
                                        if (flags & UPL_COMMIT_INACTIVATE) {

-                                               if (shadow_object->internal)
+                                               if (shadow_object->internal) {

                                                        vm_page_anonymous_count += local_queue_count;
                                                        vm_page_anonymous_count += local_queue_count;

+                                               }

                                                vm_page_inactive_count += local_queue_count;
 
                                                token_new_pagecount += local_queue_count;
                                                vm_page_inactive_count += local_queue_count;
 
                                                token_new_pagecount += local_queue_count;

-                                       } else
+                                       } else {

                                                vm_page_active_count += local_queue_count;
                                                vm_page_active_count += local_queue_count;

+                                       }

 
 

-                                       if (shadow_object->internal)
+                                       if (shadow_object->internal) {

                                                vm_page_pageable_internal_count += local_queue_count;
                                                vm_page_pageable_internal_count += local_queue_count;

-                                       else
+                                       } else {

                                                vm_page_pageable_external_count += local_queue_count;
                                                vm_page_pageable_external_count += local_queue_count;

+                                       }

                                }
                        } else {
                                vm_page_lockspin_queues();
                        }
                                }
                        } else {
                                vm_page_lockspin_queues();
                        }

-                       if (unwired_count) {    
+                       if (unwired_count) {

                                vm_page_wire_count -= unwired_count;
                                VM_CHECK_MEMORYSTATUS;
                        }
                                vm_page_wire_count -= unwired_count;
                                VM_CHECK_MEMORYSTATUS;
                        }


@@ -7259,16 +7222,16 @@ commit_next_page:
        }
        occupied = 1;
 
        }
        occupied = 1;
 

-       if (upl->flags & UPL_DEVICE_MEMORY)  {
+       if (upl->flags & UPL_DEVICE_MEMORY) {

                occupied = 0;
        } else if (upl->flags & UPL_LITE) {
                occupied = 0;
        } else if (upl->flags & UPL_LITE) {

-               int     pg_num;
-               int     i;
+               int     pg_num;
+               int     i;

 
                occupied = 0;
 
                if (!fast_path_full_commit) {
 
                occupied = 0;
 
                if (!fast_path_full_commit) {

-                       pg_num = upl->size/PAGE_SIZE;
+                       pg_num = upl->size / PAGE_SIZE;

                        pg_num = (pg_num + 31) >> 5;
 
                        for (i = 0; i < pg_num; i++) {
                        pg_num = (pg_num + 31) >> 5;
 
                        for (i = 0; i < pg_num; i++) {


@@ -7279,8 +7242,9 @@ commit_next_page:
                        }
                }
        } else {
                        }
                }
        } else {

-               if (vm_page_queue_empty(&upl->map_object->memq))
+               if (vm_page_queue_empty(&upl->map_object->memq)) {

                        occupied = 0;
                        occupied = 0;

+               }

        }
        if (occupied == 0) {
                /*
        }
        if (occupied == 0) {
                /*


@@ -7291,11 +7255,12 @@ commit_next_page:
                 * should be considered relevant for the Vector UPL and not
                 * the internal UPLs.
                 */
                 * should be considered relevant for the Vector UPL and not
                 * the internal UPLs.
                 */

-               if ((upl->flags & UPL_COMMIT_NOTIFY_EMPTY) || isVectorUPL)
+               if ((upl->flags & UPL_COMMIT_NOTIFY_EMPTY) || isVectorUPL) {

                        *empty = TRUE;
                        *empty = TRUE;

+               }

 
                if (object == shadow_object && !(upl->flags & UPL_KERNEL_OBJECT)) {
 
                if (object == shadow_object && !(upl->flags & UPL_KERNEL_OBJECT)) {

-                       /*
+                       /*

                         * this is not a paging object
                         * so we need to drop the paging reference
                         * that was taken when we created the UPL
                         * this is not a paging object
                         * so we need to drop the paging reference
                         * that was taken when we created the UPL


@@ -7304,31 +7269,32 @@ commit_next_page:
                        vm_object_activity_end(shadow_object);
                        vm_object_collapse(shadow_object, 0, TRUE);
                } else {
                        vm_object_activity_end(shadow_object);
                        vm_object_collapse(shadow_object, 0, TRUE);
                } else {

-                        /*
-                         * we dontated the paging reference to
-                         * the map object... vm_pageout_object_terminate
-                         * will drop this reference
-                         */
+                       /*
+                        * we dontated the paging reference to
+                        * the map object... vm_pageout_object_terminate
+                        * will drop this reference
+                        */

                }
        }
        VM_OBJECT_WIRED_PAGE_UPDATE_END(shadow_object, shadow_object->wire_tag);
        vm_object_unlock(shadow_object);
                }
        }
        VM_OBJECT_WIRED_PAGE_UPDATE_END(shadow_object, shadow_object->wire_tag);
        vm_object_unlock(shadow_object);

-       if (object != shadow_object)
-               vm_object_unlock(object);
-       
-       if(!isVectorUPL)
+       if (object != shadow_object) {
+               vm_object_unlock(object);
+       }
+
+       if (!isVectorUPL) {

                upl_unlock(upl);
                upl_unlock(upl);

-       else {
-               /* 
+       } else {
+               /*

                 * If we completed our operations on an UPL that is
                 * part of a Vectored UPL and if empty is TRUE, then
                 * If we completed our operations on an UPL that is
                 * part of a Vectored UPL and if empty is TRUE, then

-                * we should go ahead and deallocate this UPL element. 
+                * we should go ahead and deallocate this UPL element.

                 * Then we check if this was the last of the UPL elements
                 * within that Vectored UPL. If so, set empty to TRUE
                 * so that in ubc_upl_commit_range or ubc_upl_commit, we
                 * can go ahead and deallocate the Vector UPL too.
                 */
                 * Then we check if this was the last of the UPL elements
                 * within that Vectored UPL. If so, set empty to TRUE
                 * so that in ubc_upl_commit_range or ubc_upl_commit, we
                 * can go ahead and deallocate the Vector UPL too.
                 */

-               if(*empty==TRUE) {
+               if (*empty == TRUE) {

                        *empty = vector_upl_set_subupl(vector_upl, upl, 0);
                        upl_deallocate(upl);
                }
                        *empty = vector_upl_set_subupl(vector_upl, upl, 0);
                        upl_deallocate(upl);
                }


@@ -7343,53 +7309,55 @@ commit_next_page:
 
 kern_return_t
 upl_abort_range(
 
 kern_return_t
 upl_abort_range(

-       upl_t                   upl, 
-       upl_offset_t            offset, 
-       upl_size_t              size,
-       int                     error,
-       boolean_t               *empty) 
+       upl_t                   upl,
+       upl_offset_t            offset,
+       upl_size_t              size,
+       int                     error,
+       boolean_t               *empty)

 {
 {

-       upl_page_info_t         *user_page_list = NULL;
-       upl_size_t              xfer_size, subupl_size = size;
-       vm_object_t             shadow_object;
-       vm_object_t             object;
-       vm_object_offset_t      target_offset;
-       upl_offset_t            subupl_offset = offset;
-       int                     entry;
-       wpl_array_t             lite_list;
-       int                     occupied;
-       struct  vm_page_delayed_work    dw_array[DEFAULT_DELAYED_WORK_LIMIT];
-       struct  vm_page_delayed_work    *dwp;
-       int                     dw_count;
-       int                     dw_limit;
-       int                     isVectorUPL = 0;
-       upl_t                   vector_upl = NULL;
+       upl_page_info_t         *user_page_list = NULL;
+       upl_size_t              xfer_size, subupl_size = size;
+       vm_object_t             shadow_object;
+       vm_object_t             object;
+       vm_object_offset_t      target_offset;
+       upl_offset_t            subupl_offset = offset;
+       int                     entry;
+       wpl_array_t             lite_list;
+       int                     occupied;
+       struct  vm_page_delayed_work    dw_array[DEFAULT_DELAYED_WORK_LIMIT];
+       struct  vm_page_delayed_work    *dwp;
+       int                     dw_count;
+       int                     dw_limit;
+       int                     isVectorUPL = 0;
+       upl_t                   vector_upl = NULL;

 
        *empty = FALSE;
 
 
        *empty = FALSE;
 

-       if (upl == UPL_NULL)
+       if (upl == UPL_NULL) {

                return KERN_INVALID_ARGUMENT;
                return KERN_INVALID_ARGUMENT;

+       }

 
 

-       if ( (upl->flags & UPL_IO_WIRE) && !(error & UPL_ABORT_DUMP_PAGES) )
+       if ((upl->flags & UPL_IO_WIRE) && !(error & UPL_ABORT_DUMP_PAGES)) {

                return upl_commit_range(upl, offset, size, UPL_COMMIT_FREE_ABSENT, NULL, 0, empty);
                return upl_commit_range(upl, offset, size, UPL_COMMIT_FREE_ABSENT, NULL, 0, empty);

+       }

 
 

-       if((isVectorUPL = vector_upl_is_valid(upl))) {
+       if ((isVectorUPL = vector_upl_is_valid(upl))) {

                vector_upl = upl;
                upl_lock(vector_upl);
                vector_upl = upl;
                upl_lock(vector_upl);

-       }
-       else
+       } else {

                upl_lock(upl);
                upl_lock(upl);

+       }

 
 process_upl_to_abort:
 
 process_upl_to_abort:

-       if(isVectorUPL) {
+       if (isVectorUPL) {

                size = subupl_size;
                offset = subupl_offset;
                size = subupl_size;
                offset = subupl_offset;

-               if(size == 0) {
+               if (size == 0) {

                        upl_unlock(vector_upl);
                        return KERN_SUCCESS;
                }
                upl =  vector_upl_subupl_byoffset(vector_upl, &offset, &size);
                        upl_unlock(vector_upl);
                        return KERN_SUCCESS;
                }
                upl =  vector_upl_subupl_byoffset(vector_upl, &offset, &size);

-               if(upl == NULL) {
+               if (upl == NULL) {

                        upl_unlock(vector_upl);
                        return KERN_FAILURE;
                }
                        upl_unlock(vector_upl);
                        return KERN_FAILURE;
                }


@@ -7402,7 +7370,7 @@ process_upl_to_abort:
 #if UPL_DEBUG
        if (upl->upl_commit_index < UPL_DEBUG_COMMIT_RECORDS) {
                (void) OSBacktrace(&upl->upl_commit_records[upl->upl_commit_index].c_retaddr[0], UPL_DEBUG_STACK_FRAMES);
 #if UPL_DEBUG
        if (upl->upl_commit_index < UPL_DEBUG_COMMIT_RECORDS) {
                (void) OSBacktrace(&upl->upl_commit_records[upl->upl_commit_index].c_retaddr[0], UPL_DEBUG_STACK_FRAMES);

-               
+

                upl->upl_commit_records[upl->upl_commit_index].c_beg = offset;
                upl->upl_commit_records[upl->upl_commit_index].c_end = (offset + size);
                upl->upl_commit_records[upl->upl_commit_index].c_aborted = 1;
                upl->upl_commit_records[upl->upl_commit_index].c_beg = offset;
                upl->upl_commit_records[upl->upl_commit_index].c_end = (offset + size);
                upl->upl_commit_records[upl->upl_commit_index].c_aborted = 1;


@@ -7410,47 +7378,49 @@ process_upl_to_abort:
                upl->upl_commit_index++;
        }
 #endif
                upl->upl_commit_index++;
        }
 #endif

-       if (upl->flags & UPL_DEVICE_MEMORY)
+       if (upl->flags & UPL_DEVICE_MEMORY) {

                xfer_size = 0;
                xfer_size = 0;

-       else if ((offset + size) <= upl->size)
-               xfer_size = size;
-       else {
-               if(!isVectorUPL)
+       } else if ((offset + size) <= upl->size) {
+               xfer_size = size;
+       } else {
+               if (!isVectorUPL) {

                        upl_unlock(upl);
                        upl_unlock(upl);

-               else {
+               } else {

                        upl_unlock(vector_upl);
                }
 
                return KERN_FAILURE;
        }
        if (upl->flags & UPL_INTERNAL) {
                        upl_unlock(vector_upl);
                }
 
                return KERN_FAILURE;
        }
        if (upl->flags & UPL_INTERNAL) {

-               lite_list = (wpl_array_t) 
-                       ((((uintptr_t)upl) + sizeof(struct upl))
-                       + ((upl->size/PAGE_SIZE) * sizeof(upl_page_info_t)));
+               lite_list = (wpl_array_t)
+                   ((((uintptr_t)upl) + sizeof(struct upl))
+                   + ((upl->size / PAGE_SIZE) * sizeof(upl_page_info_t)));

 
                user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl));
        } else {
 
                user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl));
        } else {

-               lite_list = (wpl_array_t) 
-                       (((uintptr_t)upl) + sizeof(struct upl));
+               lite_list = (wpl_array_t)
+                   (((uintptr_t)upl) + sizeof(struct upl));

        }
        object = upl->map_object;
 
        if (upl->flags & UPL_SHADOWED) {
        }
        object = upl->map_object;
 
        if (upl->flags & UPL_SHADOWED) {

-               vm_object_lock(object);
+               vm_object_lock(object);

                shadow_object = object->shadow;
                shadow_object = object->shadow;

-       } else
+       } else {

                shadow_object = object;
                shadow_object = object;

+       }

 
 

-       entry = offset/PAGE_SIZE;
+       entry = offset / PAGE_SIZE;

        target_offset = (vm_object_offset_t)offset;
 
        assert(!(target_offset & PAGE_MASK));
        assert(!(xfer_size & PAGE_MASK));
 
        target_offset = (vm_object_offset_t)offset;
 
        assert(!(target_offset & PAGE_MASK));
        assert(!(xfer_size & PAGE_MASK));
 

-       if (upl->flags & UPL_KERNEL_OBJECT)
+       if (upl->flags & UPL_KERNEL_OBJECT) {

                vm_object_lock_shared(shadow_object);
                vm_object_lock_shared(shadow_object);

-       else
+       } else {

                vm_object_lock(shadow_object);
                vm_object_lock(shadow_object);

+       }

 
        if (upl->flags & UPL_ACCESS_BLOCKED) {
                assert(shadow_object->blocked_access);
 
        if (upl->flags & UPL_ACCESS_BLOCKED) {
                assert(shadow_object->blocked_access);


@@ -7462,54 +7432,57 @@ process_upl_to_abort:
        dw_count = 0;
        dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT);
 
        dw_count = 0;
        dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT);
 

-       if ((error & UPL_ABORT_DUMP_PAGES) && (upl->flags & UPL_KERNEL_OBJECT))
+       if ((error & UPL_ABORT_DUMP_PAGES) && (upl->flags & UPL_KERNEL_OBJECT)) {

                panic("upl_abort_range: kernel_object being DUMPED");
                panic("upl_abort_range: kernel_object being DUMPED");

+       }

 
        while (xfer_size) {
 
        while (xfer_size) {

-               vm_page_t       t, m;
-               unsigned int    pg_num;
-               boolean_t       needed;
+               vm_page_t       t, m;
+               unsigned int    pg_num;
+               boolean_t       needed;

 
 

-               pg_num = (unsigned int) (target_offset/PAGE_SIZE);
-               assert(pg_num == target_offset/PAGE_SIZE);
+               pg_num = (unsigned int) (target_offset / PAGE_SIZE);
+               assert(pg_num == target_offset / PAGE_SIZE);

 
                needed = FALSE;
 
 
                needed = FALSE;
 

-               if (user_page_list)
+               if (user_page_list) {

                        needed = user_page_list[pg_num].needed;
                        needed = user_page_list[pg_num].needed;

+               }

 
                dwp->dw_mask = 0;
                m = VM_PAGE_NULL;
 
                if (upl->flags & UPL_LITE) {
 
                dwp->dw_mask = 0;
                m = VM_PAGE_NULL;
 
                if (upl->flags & UPL_LITE) {

+                       if (lite_list[pg_num >> 5] & (1 << (pg_num & 31))) {
+                               lite_list[pg_num >> 5] &= ~(1 << (pg_num & 31));

 
 

-                       if (lite_list[pg_num>>5] & (1 << (pg_num & 31))) {
-                               lite_list[pg_num>>5] &= ~(1 << (pg_num & 31));
-
-                               if ( !(upl->flags & UPL_KERNEL_OBJECT))
+                               if (!(upl->flags & UPL_KERNEL_OBJECT)) {

                                        m = vm_page_lookup(shadow_object, target_offset +
                                        m = vm_page_lookup(shadow_object, target_offset +

-                                                          (upl->offset - shadow_object->paging_offset));
+                                           (upl->offset - shadow_object->paging_offset));
+                               }

                        }
                }
                if (upl->flags & UPL_SHADOWED) {
                        }
                }
                if (upl->flags & UPL_SHADOWED) {

-                       if ((t = vm_page_lookup(object, target_offset)) != VM_PAGE_NULL) {
-                               t->free_when_done = FALSE;
+                       if ((t = vm_page_lookup(object, target_offset)) != VM_PAGE_NULL) {
+                               t->vmp_free_when_done = FALSE;

 
                                VM_PAGE_FREE(t);
 
 
                                VM_PAGE_FREE(t);
 

-                               if (m == VM_PAGE_NULL)
+                               if (m == VM_PAGE_NULL) {

                                        m = vm_page_lookup(shadow_object, target_offset + object->vo_shadow_offset);
                                        m = vm_page_lookup(shadow_object, target_offset + object->vo_shadow_offset);

+                               }

                        }
                }
                        }
                }

-               if ((upl->flags & UPL_KERNEL_OBJECT))
+               if ((upl->flags & UPL_KERNEL_OBJECT)) {

                        goto abort_next_page;
                        goto abort_next_page;

+               }

 
                if (m != VM_PAGE_NULL) {
 
                if (m != VM_PAGE_NULL) {

+                       assert(m->vmp_q_state != VM_PAGE_USED_BY_COMPRESSOR);

 
 

-                       assert(m->vm_page_q_state != VM_PAGE_USED_BY_COMPRESSOR);
-
-                       if (m->absent) {
-                               boolean_t must_free = TRUE;
+                       if (m->vmp_absent) {
+                               boolean_t must_free = TRUE;

 
                                /*
                                 * COPYOUT = FALSE case
 
                                /*
                                 * COPYOUT = FALSE case


@@ -7517,22 +7490,22 @@ process_upl_to_abort:
                                 * be passed back to the pages customer
                                 */
                                if (error & UPL_ABORT_RESTART) {
                                 * be passed back to the pages customer
                                 */
                                if (error & UPL_ABORT_RESTART) {

-                                       m->restart = TRUE;
-                                       m->absent = FALSE;
-                                       m->unusual = TRUE;
+                                       m->vmp_restart = TRUE;
+                                       m->vmp_absent = FALSE;
+                                       m->vmp_unusual = TRUE;

                                        must_free = FALSE;
                                } else if (error & UPL_ABORT_UNAVAILABLE) {
                                        must_free = FALSE;
                                } else if (error & UPL_ABORT_UNAVAILABLE) {

-                                       m->restart = FALSE;
-                                       m->unusual = TRUE;
+                                       m->vmp_restart = FALSE;
+                                       m->vmp_unusual = TRUE;

                                        must_free = FALSE;
                                } else if (error & UPL_ABORT_ERROR) {
                                        must_free = FALSE;
                                } else if (error & UPL_ABORT_ERROR) {

-                                       m->restart = FALSE;
-                                       m->absent = FALSE;
-                                       m->error = TRUE;
-                                       m->unusual = TRUE;
+                                       m->vmp_restart = FALSE;
+                                       m->vmp_absent = FALSE;
+                                       m->vmp_error = TRUE;
+                                       m->vmp_unusual = TRUE;

                                        must_free = FALSE;
                                }
                                        must_free = FALSE;
                                }

-                               if (m->clustered && needed == FALSE) {
+                               if (m->vmp_clustered && needed == FALSE) {

                                        /*
                                         * This page was a part of a speculative
                                         * read-ahead initiated by the kernel
                                        /*
                                         * This page was a part of a speculative
                                         * read-ahead initiated by the kernel


@@ -7544,14 +7517,14 @@ process_upl_to_abort:
                                         */
                                        must_free = TRUE;
                                }
                                         */
                                        must_free = TRUE;
                                }

-                               m->cleaning = FALSE;
+                               m->vmp_cleaning = FALSE;

 
 

-                               if (m->overwriting && !m->busy) {
+                               if (m->vmp_overwriting && !m->vmp_busy) {

                                        /*
                                         * this shouldn't happen since
                                         * this is an 'absent' page, but
                                         * it doesn't hurt to check for
                                        /*
                                         * this shouldn't happen since
                                         * this is an 'absent' page, but
                                         * it doesn't hurt to check for

-                                        * the 'alternate' method of 
+                                        * the 'alternate' method of

                                         * stabilizing the page...
                                         * we will mark 'busy' to be cleared
                                         * in the following code which will
                                         * stabilizing the page...
                                         * we will mark 'busy' to be cleared
                                         * in the following code which will


@@ -7560,20 +7533,22 @@ process_upl_to_abort:
                                         */
                                        dwp->dw_mask |= DW_vm_page_unwire;
                                }
                                         */
                                        dwp->dw_mask |= DW_vm_page_unwire;
                                }

-                               m->overwriting = FALSE;
+                               m->vmp_overwriting = FALSE;

 
                                dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
 
 
                                dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
 

-                               if (must_free == TRUE)
+                               if (must_free == TRUE) {

                                        dwp->dw_mask |= DW_vm_page_free;
                                        dwp->dw_mask |= DW_vm_page_free;

-                               else
+                               } else {

                                        dwp->dw_mask |= DW_vm_page_activate;
                                        dwp->dw_mask |= DW_vm_page_activate;

+                               }

                        } else {
                        } else {

-                               /*                          
+                               /*

                                 * Handle the trusted pager throttle.
                                 * Handle the trusted pager throttle.

-                                */                     
-                               if (m->laundry)
+                                */
+                               if (m->vmp_laundry) {

                                        dwp->dw_mask |= DW_vm_pageout_throttle_up;
                                        dwp->dw_mask |= DW_vm_pageout_throttle_up;

+                               }

 
                                if (upl->flags & UPL_ACCESS_BLOCKED) {
                                        /*
 
                                if (upl->flags & UPL_ACCESS_BLOCKED) {
                                        /*


@@ -7583,10 +7558,10 @@ process_upl_to_abort:
                                         */
                                        dwp->dw_mask |= DW_clear_busy;
                                }
                                         */
                                        dwp->dw_mask |= DW_clear_busy;
                                }

-                               if (m->overwriting) {
-                                       if (m->busy)
+                               if (m->vmp_overwriting) {
+                                       if (m->vmp_busy) {

                                                dwp->dw_mask |= DW_clear_busy;
                                                dwp->dw_mask |= DW_clear_busy;

-                                       else {
+                                       } else {

                                                /*
                                                 * deal with the 'alternate' method
                                                 * of stabilizing the page...
                                                /*
                                                 * deal with the 'alternate' method
                                                 * of stabilizing the page...


@@ -7598,10 +7573,10 @@ process_upl_to_abort:
                                                 */
                                                dwp->dw_mask |= DW_vm_page_unwire;
                                        }
                                                 */
                                                dwp->dw_mask |= DW_vm_page_unwire;
                                        }

-                                       m->overwriting = FALSE;
+                                       m->vmp_overwriting = FALSE;

                                }
                                }

-                               m->free_when_done = FALSE;
-                               m->cleaning = FALSE;
+                               m->vmp_free_when_done = FALSE;
+                               m->vmp_cleaning = FALSE;

 
                                if (error & UPL_ABORT_DUMP_PAGES) {
                                        pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
 
                                if (error & UPL_ABORT_DUMP_PAGES) {
                                        pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));


@@ -7612,14 +7587,14 @@ process_upl_to_abort:
                                                if (error & UPL_ABORT_REFERENCE) {
                                                        /*
                                                         * we've been told to explictly
                                                if (error & UPL_ABORT_REFERENCE) {
                                                        /*
                                                         * we've been told to explictly

-                                                        * reference this page... for 
+                                                        * reference this page... for

                                                         * file I/O, this is done by
                                                         * implementing an LRU on the inactive q
                                                         */
                                                        dwp->dw_mask |= DW_vm_page_lru;
                                                         * file I/O, this is done by
                                                         * implementing an LRU on the inactive q
                                                         */
                                                        dwp->dw_mask |= DW_vm_page_lru;

-
-                                               } else if ( !VM_PAGE_PAGEABLE(m))
+                                               } else if (!VM_PAGE_PAGEABLE(m)) {

                                                        dwp->dw_mask |= DW_vm_page_deactivate_internal;
                                                        dwp->dw_mask |= DW_vm_page_deactivate_internal;

+                                               }

                                        }
                                        dwp->dw_mask |= DW_PAGE_WAKEUP;
                                }
                                        }
                                        dwp->dw_mask |= DW_PAGE_WAKEUP;
                                }


@@ -7636,31 +7611,34 @@ abort_next_page:
 
                                if (dw_count >= dw_limit) {
                                        vm_page_do_delayed_work(shadow_object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count);
 
                                if (dw_count >= dw_limit) {
                                        vm_page_do_delayed_work(shadow_object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count);

-                               
+

                                        dwp = &dw_array[0];
                                        dw_count = 0;
                                }
                        } else {
                                        dwp = &dw_array[0];
                                        dw_count = 0;
                                }
                        } else {

-                               if (dwp->dw_mask & DW_clear_busy)
-                                       m->busy = FALSE;
+                               if (dwp->dw_mask & DW_clear_busy) {
+                                       m->vmp_busy = FALSE;
+                               }

 
 

-                               if (dwp->dw_mask & DW_PAGE_WAKEUP)
+                               if (dwp->dw_mask & DW_PAGE_WAKEUP) {

                                        PAGE_WAKEUP(m);
                                        PAGE_WAKEUP(m);

+                               }

                        }
                }
        }
                        }
                }
        }

-       if (dw_count)
+       if (dw_count) {

                vm_page_do_delayed_work(shadow_object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count);
                vm_page_do_delayed_work(shadow_object, VM_KERN_MEMORY_NONE, &dw_array[0], dw_count);

+       }

 
        occupied = 1;
 
 
        occupied = 1;
 

-       if (upl->flags & UPL_DEVICE_MEMORY)  {
+       if (upl->flags & UPL_DEVICE_MEMORY) {

                occupied = 0;
        } else if (upl->flags & UPL_LITE) {
                occupied = 0;
        } else if (upl->flags & UPL_LITE) {

-               int     pg_num;
-               int     i;
+               int     pg_num;
+               int     i;

 
 

-               pg_num = upl->size/PAGE_SIZE;
+               pg_num = upl->size / PAGE_SIZE;

                pg_num = (pg_num + 31) >> 5;
                occupied = 0;
 
                pg_num = (pg_num + 31) >> 5;
                occupied = 0;
 


@@ -7671,8 +7649,9 @@ abort_next_page:
                        }
                }
        } else {
                        }
                }
        } else {

-               if (vm_page_queue_empty(&upl->map_object->memq))
+               if (vm_page_queue_empty(&upl->map_object->memq)) {

                        occupied = 0;
                        occupied = 0;

+               }

        }
        if (occupied == 0) {
                /*
        }
        if (occupied == 0) {
                /*


@@ -7683,11 +7662,12 @@ abort_next_page:
                 * should be considered relevant for the Vector UPL and
                 * not the internal UPLs.
                 */
                 * should be considered relevant for the Vector UPL and
                 * not the internal UPLs.
                 */

-               if ((upl->flags & UPL_COMMIT_NOTIFY_EMPTY) || isVectorUPL)
+               if ((upl->flags & UPL_COMMIT_NOTIFY_EMPTY) || isVectorUPL) {

                        *empty = TRUE;
                        *empty = TRUE;

+               }

 
                if (object == shadow_object && !(upl->flags & UPL_KERNEL_OBJECT)) {
 
                if (object == shadow_object && !(upl->flags & UPL_KERNEL_OBJECT)) {

-                       /*
+                       /*

                         * this is not a paging object
                         * so we need to drop the paging reference
                         * that was taken when we created the UPL
                         * this is not a paging object
                         * so we need to drop the paging reference
                         * that was taken when we created the UPL


@@ -7696,31 +7676,32 @@ abort_next_page:
                        vm_object_activity_end(shadow_object);
                        vm_object_collapse(shadow_object, 0, TRUE);
                } else {
                        vm_object_activity_end(shadow_object);
                        vm_object_collapse(shadow_object, 0, TRUE);
                } else {

-                        /*
-                         * we dontated the paging reference to
-                         * the map object... vm_pageout_object_terminate
-                         * will drop this reference
-                         */
+                       /*
+                        * we dontated the paging reference to
+                        * the map object... vm_pageout_object_terminate
+                        * will drop this reference
+                        */

                }
        }
        vm_object_unlock(shadow_object);
                }
        }
        vm_object_unlock(shadow_object);

-       if (object != shadow_object)
-               vm_object_unlock(object);
-       
-       if(!isVectorUPL)
+       if (object != shadow_object) {
+               vm_object_unlock(object);
+       }
+
+       if (!isVectorUPL) {

                upl_unlock(upl);
                upl_unlock(upl);

-       else {
-               /* 
-               * If we completed our operations on an UPL that is
-               * part of a Vectored UPL and if empty is TRUE, then
-               * we should go ahead and deallocate this UPL element. 
-               * Then we check if this was the last of the UPL elements
-               * within that Vectored UPL. If so, set empty to TRUE
-               * so that in ubc_upl_abort_range or ubc_upl_abort, we
-               * can go ahead and deallocate the Vector UPL too.
-               */
-               if(*empty == TRUE) {
-                       *empty = vector_upl_set_subupl(vector_upl, upl,0);
+       } else {
+               /*
+                * If we completed our operations on an UPL that is
+                * part of a Vectored UPL and if empty is TRUE, then
+                * we should go ahead and deallocate this UPL element.
+                * Then we check if this was the last of the UPL elements
+                * within that Vectored UPL. If so, set empty to TRUE
+                * so that in ubc_upl_abort_range or ubc_upl_abort, we
+                * can go ahead and deallocate the Vector UPL too.
+                */
+               if (*empty == TRUE) {
+                       *empty = vector_upl_set_subupl(vector_upl, upl, 0);

                        upl_deallocate(upl);
                }
                goto process_upl_to_abort;
                        upl_deallocate(upl);
                }
                goto process_upl_to_abort;


@@ -7732,13 +7713,14 @@ abort_next_page:
 
 kern_return_t
 upl_abort(
 
 kern_return_t
 upl_abort(

-       upl_t   upl,
-       int     error)
+       upl_t   upl,
+       int     error)

 {
 {

-       boolean_t       empty;
+       boolean_t       empty;

 
 

-       if (upl == UPL_NULL)
+       if (upl == UPL_NULL) {

                return KERN_INVALID_ARGUMENT;
                return KERN_INVALID_ARGUMENT;

+       }

 
        return upl_abort_range(upl, 0, upl->size, error, &empty);
 }
 
        return upl_abort_range(upl, 0, upl->size, error, &empty);
 }


@@ -7747,14 +7729,15 @@ upl_abort(
 /* an option on commit should be wire */
 kern_return_t
 upl_commit(
 /* an option on commit should be wire */
 kern_return_t
 upl_commit(

-       upl_t                   upl,
-       upl_page_info_t         *page_list,
-       mach_msg_type_number_t  count)
+       upl_t                   upl,
+       upl_page_info_t         *page_list,
+       mach_msg_type_number_t  count)

 {
 {

-       boolean_t       empty;
+       boolean_t       empty;

 
 

-       if (upl == UPL_NULL)
+       if (upl == UPL_NULL) {

                return KERN_INVALID_ARGUMENT;
                return KERN_INVALID_ARGUMENT;

+       }

 
        return upl_commit_range(upl, 0, upl->size, 0, page_list, count, &empty);
 }
 
        return upl_commit_range(upl, 0, upl->size, 0, page_list, count, &empty);
 }


@@ -7762,83 +7745,90 @@ upl_commit(
 
 void
 iopl_valid_data(
 
 void
 iopl_valid_data(

-       upl_t    upl,
+       upl_t    upl,

        vm_tag_t tag)
 {
        vm_tag_t tag)
 {

-       vm_object_t     object;
-       vm_offset_t     offset;
-       vm_page_t       m, nxt_page = VM_PAGE_NULL;
-       upl_size_t      size;
-       int             wired_count = 0;
+       vm_object_t     object;
+       vm_offset_t     offset;
+       vm_page_t       m, nxt_page = VM_PAGE_NULL;
+       upl_size_t      size;
+       int             wired_count = 0;

 
 

-       if (upl == NULL)
+       if (upl == NULL) {

                panic("iopl_valid_data: NULL upl");
                panic("iopl_valid_data: NULL upl");

-       if (vector_upl_is_valid(upl))
+       }
+       if (vector_upl_is_valid(upl)) {

                panic("iopl_valid_data: vector upl");
                panic("iopl_valid_data: vector upl");

-       if ((upl->flags & (UPL_DEVICE_MEMORY|UPL_SHADOWED|UPL_ACCESS_BLOCKED|UPL_IO_WIRE|UPL_INTERNAL)) != UPL_IO_WIRE)
+       }
+       if ((upl->flags & (UPL_DEVICE_MEMORY | UPL_SHADOWED | UPL_ACCESS_BLOCKED | UPL_IO_WIRE | UPL_INTERNAL)) != UPL_IO_WIRE) {

                panic("iopl_valid_data: unsupported upl, flags = %x", upl->flags);
                panic("iopl_valid_data: unsupported upl, flags = %x", upl->flags);

+       }

 
        object = upl->map_object;
 
 
        object = upl->map_object;
 

-       if (object == kernel_object || object == compressor_object)
+       if (object == kernel_object || object == compressor_object) {

                panic("iopl_valid_data: object == kernel or compressor");
                panic("iopl_valid_data: object == kernel or compressor");

+       }

 
        if (object->purgable == VM_PURGABLE_VOLATILE ||
 
        if (object->purgable == VM_PURGABLE_VOLATILE ||

-           object->purgable == VM_PURGABLE_EMPTY) 
+           object->purgable == VM_PURGABLE_EMPTY) {

                panic("iopl_valid_data: object %p purgable %d",
                panic("iopl_valid_data: object %p purgable %d",

-                     object, object->purgable);
+                   object, object->purgable);
+       }

 
        size = upl->size;
 
        vm_object_lock(object);
        VM_OBJECT_WIRED_PAGE_UPDATE_START(object);
 
 
        size = upl->size;
 
        vm_object_lock(object);
        VM_OBJECT_WIRED_PAGE_UPDATE_START(object);
 

-       if (object->vo_size == size && object->resident_page_count == (size / PAGE_SIZE))
+       if (object->vo_size == size && object->resident_page_count == (size / PAGE_SIZE)) {

                nxt_page = (vm_page_t)vm_page_queue_first(&object->memq);
                nxt_page = (vm_page_t)vm_page_queue_first(&object->memq);

-       else
+       } else {

                offset = 0 + upl->offset - object->paging_offset;
                offset = 0 + upl->offset - object->paging_offset;

+       }

 
        while (size) {
 
        while (size) {

-

                if (nxt_page != VM_PAGE_NULL) {
                        m = nxt_page;
                if (nxt_page != VM_PAGE_NULL) {
                        m = nxt_page;

-                       nxt_page = (vm_page_t)vm_page_queue_next(&nxt_page->listq);
+                       nxt_page = (vm_page_t)vm_page_queue_next(&nxt_page->vmp_listq);

                } else {
                        m = vm_page_lookup(object, offset);
                        offset += PAGE_SIZE;
 
                } else {
                        m = vm_page_lookup(object, offset);
                        offset += PAGE_SIZE;
 

-                       if (m == VM_PAGE_NULL)
+                       if (m == VM_PAGE_NULL) {

                                panic("iopl_valid_data: missing expected page at offset %lx", (long)offset);
                                panic("iopl_valid_data: missing expected page at offset %lx", (long)offset);

+                       }

                }
                }

-               if (m->busy) {
-                       if (!m->absent)
+               if (m->vmp_busy) {
+                       if (!m->vmp_absent) {

                                panic("iopl_valid_data: busy page w/o absent");
                                panic("iopl_valid_data: busy page w/o absent");

+                       }

 
 

-                       if (m->pageq.next || m->pageq.prev)
+                       if (m->vmp_pageq.next || m->vmp_pageq.prev) {

                                panic("iopl_valid_data: busy+absent page on page queue");
                                panic("iopl_valid_data: busy+absent page on page queue");

-                       if (m->reusable) {
+                       }
+                       if (m->vmp_reusable) {

                                panic("iopl_valid_data: %p is reusable", m);
                        }
 
                                panic("iopl_valid_data: %p is reusable", m);
                        }
 

-                       m->absent = FALSE;
-                       m->dirty = TRUE;
-                       assert(m->vm_page_q_state == VM_PAGE_NOT_ON_Q);
-                       assert(m->wire_count == 0);
-                       m->wire_count++;
-                       assert(m->wire_count);
-                       if (m->wire_count == 1) {
-                               m->vm_page_q_state = VM_PAGE_IS_WIRED;
+                       m->vmp_absent = FALSE;
+                       m->vmp_dirty = TRUE;
+                       assert(m->vmp_q_state == VM_PAGE_NOT_ON_Q);
+                       assert(m->vmp_wire_count == 0);
+                       m->vmp_wire_count++;
+                       assert(m->vmp_wire_count);
+                       if (m->vmp_wire_count == 1) {
+                               m->vmp_q_state = VM_PAGE_IS_WIRED;

                                wired_count++;
                        } else {
                                panic("iopl_valid_data: %p already wired\n", m);
                        }
                                wired_count++;
                        } else {
                                panic("iopl_valid_data: %p already wired\n", m);
                        }

-                       
+

                        PAGE_WAKEUP_DONE(m);
                }
                size -= PAGE_SIZE;
        }
        if (wired_count) {
                        PAGE_WAKEUP_DONE(m);
                }
                size -= PAGE_SIZE;
        }
        if (wired_count) {

-

                VM_OBJECT_WIRED_PAGE_COUNT(object, wired_count);
                assert(object->resident_page_count >= object->wired_page_count);
 
                VM_OBJECT_WIRED_PAGE_COUNT(object, wired_count);
                assert(object->resident_page_count >= object->wired_page_count);
 


@@ -7857,10 +7847,10 @@ iopl_valid_data(
 
 void
 vm_object_set_pmap_cache_attr(
 
 void
 vm_object_set_pmap_cache_attr(

-               vm_object_t             object,
-               upl_page_info_array_t   user_page_list,
-               unsigned int            num_pages,
-               boolean_t               batch_pmap_op)
+       vm_object_t             object,
+       upl_page_info_array_t   user_page_list,
+       unsigned int            num_pages,
+       boolean_t               batch_pmap_op)

 {
        unsigned int    cache_attr = 0;
 
 {
        unsigned int    cache_attr = 0;
 


@@ -7872,21 +7862,21 @@ vm_object_set_pmap_cache_attr(
 }
 
 
 }
 
 

-boolean_t      vm_object_iopl_wire_full(vm_object_t, upl_t, upl_page_info_array_t, wpl_array_t, upl_control_flags_t, vm_tag_t);
-kern_return_t  vm_object_iopl_wire_empty(vm_object_t, upl_t, upl_page_info_array_t, wpl_array_t, upl_control_flags_t, vm_tag_t, vm_object_offset_t *, int);
+boolean_t       vm_object_iopl_wire_full(vm_object_t, upl_t, upl_page_info_array_t, wpl_array_t, upl_control_flags_t, vm_tag_t);
+kern_return_t   vm_object_iopl_wire_empty(vm_object_t, upl_t, upl_page_info_array_t, wpl_array_t, upl_control_flags_t, vm_tag_t, vm_object_offset_t *, int, int*);

 
 
 
 boolean_t
 vm_object_iopl_wire_full(vm_object_t object, upl_t upl, upl_page_info_array_t user_page_list,
 
 
 
 boolean_t
 vm_object_iopl_wire_full(vm_object_t object, upl_t upl, upl_page_info_array_t user_page_list,

-                           wpl_array_t lite_list, upl_control_flags_t cntrl_flags, vm_tag_t tag)
+    wpl_array_t lite_list, upl_control_flags_t cntrl_flags, vm_tag_t tag)

 {
 {

-       vm_page_t       dst_page;
-       unsigned int    entry;
-       int             page_count;
-       int             delayed_unlock = 0;
-       boolean_t       retval = TRUE;
-       ppnum_t         phys_page;
+       vm_page_t       dst_page;
+       unsigned int    entry;
+       int             page_count;
+       int             delayed_unlock = 0;
+       boolean_t       retval = TRUE;
+       ppnum_t         phys_page;

 
        vm_object_lock_assert_exclusive(object);
        assert(object->purgable != VM_PURGABLE_VOLATILE);
 
        vm_object_lock_assert_exclusive(object);
        assert(object->purgable != VM_PURGABLE_VOLATILE);


@@ -7901,48 +7891,48 @@ vm_object_iopl_wire_full(vm_object_t object, upl_t upl, upl_page_info_array_t us
        vm_page_lock_queues();
 
        while (page_count--) {
        vm_page_lock_queues();
 
        while (page_count--) {

-
-               if (dst_page->busy ||
-                   dst_page->fictitious ||
-                   dst_page->absent ||
-                   dst_page->error ||
-                   dst_page->cleaning ||
-                   dst_page->restart ||
-                   dst_page->laundry) {
+               if (dst_page->vmp_busy ||
+                   dst_page->vmp_fictitious ||
+                   dst_page->vmp_absent ||
+                   dst_page->vmp_error ||
+                   dst_page->vmp_cleaning ||
+                   dst_page->vmp_restart ||
+                   dst_page->vmp_laundry) {

                        retval = FALSE;
                        goto done;
                }
                        retval = FALSE;
                        goto done;
                }

-               if ((cntrl_flags & UPL_REQUEST_FORCE_COHERENCY) && dst_page->written_by_kernel == TRUE) {
+               if ((cntrl_flags & UPL_REQUEST_FORCE_COHERENCY) && dst_page->vmp_written_by_kernel == TRUE) {

                        retval = FALSE;
                        goto done;
                }
                        retval = FALSE;
                        goto done;
                }

-               dst_page->reference = TRUE;
+               dst_page->vmp_reference = TRUE;

 
                vm_page_wire(dst_page, tag, FALSE);
 
                if (!(cntrl_flags & UPL_COPYOUT_FROM)) {
                        SET_PAGE_DIRTY(dst_page, FALSE);
                }
 
                vm_page_wire(dst_page, tag, FALSE);
 
                if (!(cntrl_flags & UPL_COPYOUT_FROM)) {
                        SET_PAGE_DIRTY(dst_page, FALSE);
                }

-               entry = (unsigned int)(dst_page->offset / PAGE_SIZE);
+               entry = (unsigned int)(dst_page->vmp_offset / PAGE_SIZE);

                assert(entry >= 0 && entry < object->resident_page_count);
                assert(entry >= 0 && entry < object->resident_page_count);

-               lite_list[entry>>5] |= 1 << (entry & 31);
-               
+               lite_list[entry >> 5] |= 1 << (entry & 31);
+

                phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page);
 
                phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page);
 

-               if (phys_page > upl->highest_page)
+               if (phys_page > upl->highest_page) {

                        upl->highest_page = phys_page;
                        upl->highest_page = phys_page;

+               }

 
                if (user_page_list) {
                        user_page_list[entry].phys_addr = phys_page;
 
                if (user_page_list) {
                        user_page_list[entry].phys_addr = phys_page;

-                       user_page_list[entry].absent    = dst_page->absent;
-                       user_page_list[entry].dirty     = dst_page->dirty;
-                       user_page_list[entry].free_when_done   = dst_page->free_when_done;
-                       user_page_list[entry].precious  = dst_page->precious;
+                       user_page_list[entry].absent    = dst_page->vmp_absent;
+                       user_page_list[entry].dirty     = dst_page->vmp_dirty;
+                       user_page_list[entry].free_when_done   = dst_page->vmp_free_when_done;
+                       user_page_list[entry].precious  = dst_page->vmp_precious;

                        user_page_list[entry].device    = FALSE;
                        user_page_list[entry].speculative = FALSE;
                        user_page_list[entry].cs_validated = FALSE;
                        user_page_list[entry].cs_tainted = FALSE;
                        user_page_list[entry].device    = FALSE;
                        user_page_list[entry].speculative = FALSE;
                        user_page_list[entry].cs_validated = FALSE;
                        user_page_list[entry].cs_tainted = FALSE;

-                       user_page_list[entry].cs_nx     = FALSE;
+                       user_page_list[entry].cs_nx     = FALSE;

                        user_page_list[entry].needed    = FALSE;
                        user_page_list[entry].mark      = FALSE;
                }
                        user_page_list[entry].needed    = FALSE;
                        user_page_list[entry].mark      = FALSE;
                }


@@ -7952,31 +7942,32 @@ vm_object_iopl_wire_full(vm_object_t object, upl_t upl, upl_page_info_array_t us
 
                        VM_CHECK_MEMORYSTATUS;
                }
 
                        VM_CHECK_MEMORYSTATUS;
                }

-               dst_page = (vm_page_t)vm_page_queue_next(&dst_page->listq);
+               dst_page = (vm_page_t)vm_page_queue_next(&dst_page->vmp_listq);

        }
 done:
        vm_page_unlock_queues();
 
        VM_CHECK_MEMORYSTATUS;
 
        }
 done:
        vm_page_unlock_queues();
 
        VM_CHECK_MEMORYSTATUS;
 

-       return (retval);
+       return retval;

 }
 
 
 kern_return_t
 vm_object_iopl_wire_empty(vm_object_t object, upl_t upl, upl_page_info_array_t user_page_list,
 }
 
 
 kern_return_t
 vm_object_iopl_wire_empty(vm_object_t object, upl_t upl, upl_page_info_array_t user_page_list,

-                            wpl_array_t lite_list, upl_control_flags_t cntrl_flags, vm_tag_t tag, vm_object_offset_t *dst_offset, int page_count)
+    wpl_array_t lite_list, upl_control_flags_t cntrl_flags, vm_tag_t tag, vm_object_offset_t *dst_offset,
+    int page_count, int* page_grab_count)

 {
 {

-       vm_page_t       dst_page;
-       boolean_t       no_zero_fill = FALSE;
-       int             interruptible;
-       int             pages_wired = 0;
-       int             pages_inserted = 0;
-       int             entry = 0;
-       uint64_t        delayed_ledger_update = 0;
-       kern_return_t   ret = KERN_SUCCESS;
-       int             grab_options;
-       ppnum_t         phys_page;
+       vm_page_t       dst_page;
+       boolean_t       no_zero_fill = FALSE;
+       int             interruptible;
+       int             pages_wired = 0;
+       int             pages_inserted = 0;
+       int             entry = 0;
+       uint64_t        delayed_ledger_update = 0;
+       kern_return_t   ret = KERN_SUCCESS;
+       int             grab_options;
+       ppnum_t         phys_page;

 
        vm_object_lock_assert_exclusive(object);
        assert(object->purgable != VM_PURGABLE_VOLATILE);
 
        vm_object_lock_assert_exclusive(object);
        assert(object->purgable != VM_PURGABLE_VOLATILE);


@@ -7985,13 +7976,15 @@ vm_object_iopl_wire_empty(vm_object_t object, upl_t upl, upl_page_info_array_t u
        assert(object->copy == NULL);
        assert(object->shadow == NULL);
 
        assert(object->copy == NULL);
        assert(object->shadow == NULL);
 

-       if (cntrl_flags & UPL_SET_INTERRUPTIBLE)
+       if (cntrl_flags & UPL_SET_INTERRUPTIBLE) {

                interruptible = THREAD_ABORTSAFE;
                interruptible = THREAD_ABORTSAFE;

-       else
+       } else {

                interruptible = THREAD_UNINT;
                interruptible = THREAD_UNINT;

+       }

 
 

-       if (cntrl_flags & (UPL_NOZEROFILL | UPL_NOZEROFILLIO))
-               no_zero_fill = TRUE;
+       if (cntrl_flags & (UPL_NOZEROFILL | UPL_NOZEROFILLIO)) {
+               no_zero_fill = TRUE;
+       }

 
        grab_options = 0;
 #if CONFIG_SECLUDED_MEMORY
 
        grab_options = 0;
 #if CONFIG_SECLUDED_MEMORY


@@ -8001,10 +7994,8 @@ vm_object_iopl_wire_empty(vm_object_t object, upl_t upl, upl_page_info_array_t u
 #endif /* CONFIG_SECLUDED_MEMORY */
 
        while (page_count--) {
 #endif /* CONFIG_SECLUDED_MEMORY */
 
        while (page_count--) {

-                       

                while ((dst_page = vm_page_grab_options(grab_options))
                while ((dst_page = vm_page_grab_options(grab_options))

-                      == VM_PAGE_NULL) {
-
+                   == VM_PAGE_NULL) {

                        OSAddAtomic(page_count, &vm_upl_wait_for_pages);
 
                        VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_START, vm_upl_wait_for_pages, 0, 0, 0);
                        OSAddAtomic(page_count, &vm_upl_wait_for_pages);
 
                        VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_START, vm_upl_wait_for_pages, 0, 0, 0);


@@ -8016,7 +8007,7 @@ vm_object_iopl_wire_empty(vm_object_t object, upl_t upl, upl_page_info_array_t u
                                OSAddAtomic(-page_count, &vm_upl_wait_for_pages);
 
                                VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, -1);
                                OSAddAtomic(-page_count, &vm_upl_wait_for_pages);
 
                                VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, -1);

-                               
+

                                ret = MACH_SEND_INTERRUPTED;
                                goto done;
                        }
                                ret = MACH_SEND_INTERRUPTED;
                                goto done;
                        }


@@ -8024,22 +8015,23 @@ vm_object_iopl_wire_empty(vm_object_t object, upl_t upl, upl_page_info_array_t u
 
                        VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, 0);
                }
 
                        VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, 0);
                }

-               if (no_zero_fill == FALSE)
+               if (no_zero_fill == FALSE) {

                        vm_page_zero_fill(dst_page);
                        vm_page_zero_fill(dst_page);

-               else
-                       dst_page->absent = TRUE;
+               } else {
+                       dst_page->vmp_absent = TRUE;
+               }

 
 

-               dst_page->reference = TRUE;
+               dst_page->vmp_reference = TRUE;

 
                if (!(cntrl_flags & UPL_COPYOUT_FROM)) {
 
                if (!(cntrl_flags & UPL_COPYOUT_FROM)) {

-                       SET_PAGE_DIRTY(dst_page, FALSE);        
-               }
-               if (dst_page->absent == FALSE) {
-                       assert(dst_page->vm_page_q_state == VM_PAGE_NOT_ON_Q);
-                       assert(dst_page->wire_count == 0);
-                       dst_page->wire_count++;
-                       dst_page->vm_page_q_state = VM_PAGE_IS_WIRED;
-                       assert(dst_page->wire_count);
+                       SET_PAGE_DIRTY(dst_page, FALSE);
+               }
+               if (dst_page->vmp_absent == FALSE) {
+                       assert(dst_page->vmp_q_state == VM_PAGE_NOT_ON_Q);
+                       assert(dst_page->vmp_wire_count == 0);
+                       dst_page->vmp_wire_count++;
+                       dst_page->vmp_q_state = VM_PAGE_IS_WIRED;
+                       assert(dst_page->vmp_wire_count);

                        pages_wired++;
                        PAGE_WAKEUP_DONE(dst_page);
                }
                        pages_wired++;
                        PAGE_WAKEUP_DONE(dst_page);
                }


@@ -8047,20 +8039,21 @@ vm_object_iopl_wire_empty(vm_object_t object, upl_t upl, upl_page_info_array_t u
 
                vm_page_insert_internal(dst_page, object, *dst_offset, tag, FALSE, TRUE, TRUE, TRUE, &delayed_ledger_update);
 
 
                vm_page_insert_internal(dst_page, object, *dst_offset, tag, FALSE, TRUE, TRUE, TRUE, &delayed_ledger_update);
 

-               lite_list[entry>>5] |= 1 << (entry & 31);
-               
+               lite_list[entry >> 5] |= 1 << (entry & 31);
+

                phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page);
 
                phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page);
 

-               if (phys_page > upl->highest_page)
+               if (phys_page > upl->highest_page) {

                        upl->highest_page = phys_page;
                        upl->highest_page = phys_page;

+               }

 
                if (user_page_list) {
 
                if (user_page_list) {

-                       user_page_list[entry].phys_addr = phys_page;
-                       user_page_list[entry].absent    = dst_page->absent;
-                       user_page_list[entry].dirty     = dst_page->dirty;
-                       user_page_list[entry].free_when_done    = FALSE;
-                       user_page_list[entry].precious  = FALSE;
-                       user_page_list[entry].device    = FALSE;
+                       user_page_list[entry].phys_addr = phys_page;
+                       user_page_list[entry].absent    = dst_page->vmp_absent;
+                       user_page_list[entry].dirty     = dst_page->vmp_dirty;
+                       user_page_list[entry].free_when_done    = FALSE;
+                       user_page_list[entry].precious  = FALSE;
+                       user_page_list[entry].device    = FALSE;

                        user_page_list[entry].speculative = FALSE;
                        user_page_list[entry].cs_validated = FALSE;
                        user_page_list[entry].cs_tainted = FALSE;
                        user_page_list[entry].speculative = FALSE;
                        user_page_list[entry].cs_validated = FALSE;
                        user_page_list[entry].cs_tainted = FALSE;


@@ -8085,64 +8078,85 @@ done:
                }
        }
        if (delayed_ledger_update) {
                }
        }
        if (delayed_ledger_update) {

-               task_t          owner;
-
-               owner = object->vo_purgeable_owner;
+               task_t          owner;
+               int             ledger_idx_volatile;
+               int             ledger_idx_nonvolatile;
+               int             ledger_idx_volatile_compressed;
+               int             ledger_idx_nonvolatile_compressed;
+               boolean_t       do_footprint;
+
+               owner = VM_OBJECT_OWNER(object);

                assert(owner);
 
                assert(owner);
 

+               vm_object_ledger_tag_ledgers(object,
+                   &ledger_idx_volatile,
+                   &ledger_idx_nonvolatile,
+                   &ledger_idx_volatile_compressed,
+                   &ledger_idx_nonvolatile_compressed,
+                   &do_footprint);
+

                /* more non-volatile bytes */
                ledger_credit(owner->ledger,
                /* more non-volatile bytes */
                ledger_credit(owner->ledger,

-                             task_ledgers.purgeable_nonvolatile,
-                             delayed_ledger_update);
-               /* more footprint */
-               ledger_credit(owner->ledger,
-                             task_ledgers.phys_footprint,
-                             delayed_ledger_update);
+                   ledger_idx_nonvolatile,
+                   delayed_ledger_update);
+               if (do_footprint) {
+                       /* more footprint */
+                       ledger_credit(owner->ledger,
+                           task_ledgers.phys_footprint,
+                           delayed_ledger_update);
+               }

        }
        }

-       return (ret);
-}

 
 

+       assert(page_grab_count);
+       *page_grab_count = pages_inserted;
+
+       return ret;
+}

 
 

-unsigned int vm_object_iopl_request_sleep_for_cleaning = 0;

 
 
 kern_return_t
 vm_object_iopl_request(
 
 
 kern_return_t
 vm_object_iopl_request(

-       vm_object_t             object,
-       vm_object_offset_t      offset,
-       upl_size_t              size,
-       upl_t                   *upl_ptr,
-       upl_page_info_array_t   user_page_list,
-       unsigned int            *page_list_count,
-       upl_control_flags_t     cntrl_flags,
-       vm_tag_t                tag)
+       vm_object_t             object,
+       vm_object_offset_t      offset,
+       upl_size_t              size,
+       upl_t                   *upl_ptr,
+       upl_page_info_array_t   user_page_list,
+       unsigned int            *page_list_count,
+       upl_control_flags_t     cntrl_flags,
+       vm_tag_t                tag)

 {
 {

-       vm_page_t               dst_page;
-       vm_object_offset_t      dst_offset;
-       upl_size_t              xfer_size;
-       upl_t                   upl = NULL;
-       unsigned int            entry;
-       wpl_array_t             lite_list = NULL;
-       int                     no_zero_fill = FALSE;
-       unsigned int            size_in_pages;
-       u_int32_t               psize;
-       kern_return_t           ret;
-       vm_prot_t               prot;
-       struct vm_object_fault_info fault_info;
-       struct  vm_page_delayed_work    dw_array[DEFAULT_DELAYED_WORK_LIMIT];
-       struct  vm_page_delayed_work    *dwp;
-       int                     dw_count;
-       int                     dw_limit;
-       int                     dw_index;
-       boolean_t               caller_lookup;
-       int                     io_tracking_flag = 0;
-       int                     interruptible;
-       ppnum_t                 phys_page;
-
-       boolean_t               set_cache_attr_needed = FALSE;
-       boolean_t               free_wired_pages = FALSE;
-       boolean_t               fast_path_empty_req = FALSE;
-       boolean_t               fast_path_full_req = FALSE;
+       vm_page_t               dst_page;
+       vm_object_offset_t      dst_offset;
+       upl_size_t              xfer_size;
+       upl_t                   upl = NULL;
+       unsigned int            entry;
+       wpl_array_t             lite_list = NULL;
+       int                     no_zero_fill = FALSE;
+       unsigned int            size_in_pages;
+       int                     page_grab_count = 0;
+       u_int32_t               psize;
+       kern_return_t           ret;
+       vm_prot_t               prot;
+       struct vm_object_fault_info fault_info = {};
+       struct  vm_page_delayed_work    dw_array[DEFAULT_DELAYED_WORK_LIMIT];
+       struct  vm_page_delayed_work    *dwp;
+       int                     dw_count;
+       int                     dw_limit;
+       int                     dw_index;
+       boolean_t               caller_lookup;
+       int                     io_tracking_flag = 0;
+       int                     interruptible;
+       ppnum_t                 phys_page;
+
+       boolean_t               set_cache_attr_needed = FALSE;
+       boolean_t               free_wired_pages = FALSE;
+       boolean_t               fast_path_empty_req = FALSE;
+       boolean_t               fast_path_full_req = FALSE;
+
+#if DEVELOPMENT || DEBUG
+       task_t                  task = current_task();
+#endif /* DEVELOPMENT || DEBUG */

 
        if (cntrl_flags & ~UPL_VALID_FLAGS) {
                /*
 
        if (cntrl_flags & ~UPL_VALID_FLAGS) {
                /*


@@ -8151,72 +8165,93 @@ vm_object_iopl_request(
                 */
                return KERN_INVALID_VALUE;
        }
                 */
                return KERN_INVALID_VALUE;
        }

-       if (vm_lopage_needed == FALSE)
-               cntrl_flags &= ~UPL_NEED_32BIT_ADDR;
+       if (vm_lopage_needed == FALSE) {
+               cntrl_flags &= ~UPL_NEED_32BIT_ADDR;
+       }

 
        if (cntrl_flags & UPL_NEED_32BIT_ADDR) {
 
        if (cntrl_flags & UPL_NEED_32BIT_ADDR) {

-               if ( (cntrl_flags & (UPL_SET_IO_WIRE | UPL_SET_LITE)) != (UPL_SET_IO_WIRE | UPL_SET_LITE))
-                       return KERN_INVALID_VALUE;
+               if ((cntrl_flags & (UPL_SET_IO_WIRE | UPL_SET_LITE)) != (UPL_SET_IO_WIRE | UPL_SET_LITE)) {
+                       return KERN_INVALID_VALUE;
+               }

 
                if (object->phys_contiguous) {
 
                if (object->phys_contiguous) {

-                       if ((offset + object->vo_shadow_offset) >= (vm_object_offset_t)max_valid_dma_address)
-                               return KERN_INVALID_ADDRESS;
-             
-                       if (((offset + object->vo_shadow_offset) + size) >= (vm_object_offset_t)max_valid_dma_address)
-                               return KERN_INVALID_ADDRESS;
+                       if ((offset + object->vo_shadow_offset) >= (vm_object_offset_t)max_valid_dma_address) {
+                               return KERN_INVALID_ADDRESS;
+                       }
+
+                       if (((offset + object->vo_shadow_offset) + size) >= (vm_object_offset_t)max_valid_dma_address) {
+                               return KERN_INVALID_ADDRESS;
+                       }

                }
        }
                }
        }

-       if (cntrl_flags & (UPL_NOZEROFILL | UPL_NOZEROFILLIO))
-               no_zero_fill = TRUE;
+       if (cntrl_flags & (UPL_NOZEROFILL | UPL_NOZEROFILLIO)) {
+               no_zero_fill = TRUE;
+       }

 
 

-       if (cntrl_flags & UPL_COPYOUT_FROM)
+       if (cntrl_flags & UPL_COPYOUT_FROM) {

                prot = VM_PROT_READ;
                prot = VM_PROT_READ;

-       else
+       } else {

                prot = VM_PROT_READ | VM_PROT_WRITE;
                prot = VM_PROT_READ | VM_PROT_WRITE;

+       }

 
 

-       if ((!object->internal) && (object->paging_offset != 0))
+       if ((!object->internal) && (object->paging_offset != 0)) {

                panic("vm_object_iopl_request: external object with non-zero paging offset\n");
                panic("vm_object_iopl_request: external object with non-zero paging offset\n");

+       }
+
+       VM_DEBUG_CONSTANT_EVENT(vm_object_iopl_request, VM_IOPL_REQUEST, DBG_FUNC_START, size, cntrl_flags, prot, 0);

 
 #if CONFIG_IOSCHED || UPL_DEBUG
 
 #if CONFIG_IOSCHED || UPL_DEBUG

-       if ((object->io_tracking && object != kernel_object) || upl_debug_enabled)
+       if ((object->io_tracking && object != kernel_object) || upl_debug_enabled) {

                io_tracking_flag |= UPL_CREATE_IO_TRACKING;
                io_tracking_flag |= UPL_CREATE_IO_TRACKING;

+       }

 #endif
 
 #if CONFIG_IOSCHED
        if (object->io_tracking) {
                /* Check if we're dealing with the kernel object. We do not support expedite on kernel object UPLs */
 #endif
 
 #if CONFIG_IOSCHED
        if (object->io_tracking) {
                /* Check if we're dealing with the kernel object. We do not support expedite on kernel object UPLs */

-               if (object != kernel_object)
+               if (object != kernel_object) {

                        io_tracking_flag |= UPL_CREATE_EXPEDITE_SUP;
                        io_tracking_flag |= UPL_CREATE_EXPEDITE_SUP;

+               }

        }
 #endif
 
        }
 #endif
 

-       if (object->phys_contiguous)
-               psize = PAGE_SIZE;
-       else
-               psize = size;
+       if (object->phys_contiguous) {
+               psize = PAGE_SIZE;
+       } else {
+               psize = size;
+       }

 
        if (cntrl_flags & UPL_SET_INTERNAL) {
 
        if (cntrl_flags & UPL_SET_INTERNAL) {

-               upl = upl_create(UPL_CREATE_INTERNAL | UPL_CREATE_LITE | io_tracking_flag, UPL_IO_WIRE, psize);
+               upl = upl_create(UPL_CREATE_INTERNAL | UPL_CREATE_LITE | io_tracking_flag, UPL_IO_WIRE, psize);

 
                user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl));
                lite_list = (wpl_array_t) (((uintptr_t)user_page_list) +
 
                user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl));
                lite_list = (wpl_array_t) (((uintptr_t)user_page_list) +

-                                          ((psize / PAGE_SIZE) * sizeof(upl_page_info_t)));
+                   ((psize / PAGE_SIZE) * sizeof(upl_page_info_t)));

                if (size == 0) {
                        user_page_list = NULL;
                        lite_list = NULL;
                }
        } else {
                if (size == 0) {
                        user_page_list = NULL;
                        lite_list = NULL;
                }
        } else {

-               upl = upl_create(UPL_CREATE_LITE | io_tracking_flag, UPL_IO_WIRE, psize);
+               upl = upl_create(UPL_CREATE_LITE | io_tracking_flag, UPL_IO_WIRE, psize);

 
                lite_list = (wpl_array_t) (((uintptr_t)upl) + sizeof(struct upl));
                if (size == 0) {
                        lite_list = NULL;
                }
        }
 
                lite_list = (wpl_array_t) (((uintptr_t)upl) + sizeof(struct upl));
                if (size == 0) {
                        lite_list = NULL;
                }
        }

-       if (user_page_list)
-               user_page_list[0].device = FALSE;
+       if (user_page_list) {
+               user_page_list[0].device = FALSE;
+       }

        *upl_ptr = upl;
 
        *upl_ptr = upl;
 

+       if (cntrl_flags & UPL_NOZEROFILLIO) {
+               DTRACE_VM4(upl_nozerofillio,
+                   vm_object_t, object,
+                   vm_object_offset_t, offset,
+                   upl_size_t, size,
+                   upl_t, upl);
+       }
+

        upl->map_object = object;
        upl->size = size;
 
        upl->map_object = object;
        upl->size = size;
 


@@ -8255,7 +8290,6 @@ vm_object_iopl_request(
 #endif
 
        if (object->phys_contiguous) {
 #endif
 
        if (object->phys_contiguous) {

-

                if (upl->flags & UPL_ACCESS_BLOCKED) {
                        assert(!object->blocked_access);
                        object->blocked_access = TRUE;
                if (upl->flags & UPL_ACCESS_BLOCKED) {
                        assert(!object->blocked_access);
                        object->blocked_access = TRUE;


@@ -8269,18 +8303,26 @@ vm_object_iopl_request(
                 */
                upl->flags |= UPL_DEVICE_MEMORY;
 
                 */
                upl->flags |= UPL_DEVICE_MEMORY;
 

-               upl->highest_page = (ppnum_t) ((offset + object->vo_shadow_offset + size - 1)>>PAGE_SHIFT);
+               upl->highest_page = (ppnum_t) ((offset + object->vo_shadow_offset + size - 1) >> PAGE_SHIFT);

 
                if (user_page_list) {
 
                if (user_page_list) {

-                       user_page_list[0].phys_addr = (ppnum_t) ((offset + object->vo_shadow_offset)>>PAGE_SHIFT);
+                       user_page_list[0].phys_addr = (ppnum_t) ((offset + object->vo_shadow_offset) >> PAGE_SHIFT);

                        user_page_list[0].device = TRUE;
                }
                if (page_list_count != NULL) {
                        user_page_list[0].device = TRUE;
                }
                if (page_list_count != NULL) {

-                       if (upl->flags & UPL_INTERNAL)
-                               *page_list_count = 0;
-                       else
-                               *page_list_count = 1;
+                       if (upl->flags & UPL_INTERNAL) {
+                               *page_list_count = 0;
+                       } else {
+                               *page_list_count = 1;
+                       }
+               }
+
+               VM_DEBUG_CONSTANT_EVENT(vm_object_iopl_request, VM_IOPL_REQUEST, DBG_FUNC_END, page_grab_count, KERN_SUCCESS, 0, 0);
+#if DEVELOPMENT || DEBUG
+               if (task != NULL) {
+                       ledger_credit(task->ledger, task_ledgers.pages_grabbed_iopl, page_grab_count);

                }
                }

+#endif /* DEVELOPMENT || DEBUG */

                return KERN_SUCCESS;
        }
        if (object != kernel_object && object != compressor_object) {
                return KERN_SUCCESS;
        }
        if (object != kernel_object && object != compressor_object) {


@@ -8294,20 +8336,21 @@ vm_object_iopl_request(
                        int num = 0;
 
                        num = OSBacktrace(bt,
                        int num = 0;
 
                        num = OSBacktrace(bt,

-                                         VM_OBJECT_TRACKING_BTDEPTH);
+                           VM_OBJECT_TRACKING_BTDEPTH);

                        btlog_add_entry(vm_object_tracking_btlog,
                        btlog_add_entry(vm_object_tracking_btlog,

-                                       object,
-                                       VM_OBJECT_TRACKING_OP_TRUESHARE,
-                                       bt,
-                                       num);
+                           object,
+                           VM_OBJECT_TRACKING_OP_TRUESHARE,
+                           bt,
+                           num);

                }
 #endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */
 
                vm_object_lock_assert_exclusive(object);
                object->true_share = TRUE;
 
                }
 #endif /* VM_OBJECT_TRACKING_OP_TRUESHARE */
 
                vm_object_lock_assert_exclusive(object);
                object->true_share = TRUE;
 

-               if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC)
+               if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) {

                        object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
                        object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;

+               }

        }
 
        if (!(cntrl_flags & UPL_COPYOUT_FROM) &&
        }
 
        if (!(cntrl_flags & UPL_COPYOUT_FROM) &&


@@ -8330,17 +8373,15 @@ vm_object_iopl_request(
                 * code path for that...
                 */
                vm_object_update(object,
                 * code path for that...
                 */
                vm_object_update(object,

-                                offset,
-                                size,
-                                NULL,
-                                NULL,
-                                FALSE, /* should_return */
-                                MEMORY_OBJECT_COPY_SYNC,
-                                VM_PROT_NO_CHANGE);
-#if DEVELOPMENT || DEBUG
-               iopl_cow++;
-               iopl_cow_pages += size >> PAGE_SHIFT;
-#endif
+                   offset,
+                   size,
+                   NULL,
+                   NULL,
+                   FALSE,              /* should_return */
+                   MEMORY_OBJECT_COPY_SYNC,
+                   VM_PROT_NO_CHANGE);
+               VM_PAGEOUT_DEBUG(iopl_cow, 1);
+               VM_PAGEOUT_DEBUG(iopl_cow_pages, (size >> PAGE_SHIFT));

        }
        if (!(cntrl_flags & (UPL_NEED_32BIT_ADDR | UPL_BLOCK_ACCESS)) &&
            object->purgable != VM_PURGABLE_VOLATILE &&
        }
        if (!(cntrl_flags & (UPL_NEED_32BIT_ADDR | UPL_BLOCK_ACCESS)) &&
            object->purgable != VM_PURGABLE_VOLATILE &&


@@ -8349,16 +8390,12 @@ vm_object_iopl_request(
            size == object->vo_size &&
            offset == 0 &&
            object->shadow == NULL &&
            size == object->vo_size &&
            offset == 0 &&
            object->shadow == NULL &&

-           object->pager == NULL)
-       {
-               if (object->resident_page_count == size_in_pages)
-               {
+           object->pager == NULL) {
+               if (object->resident_page_count == size_in_pages) {

                        assert(object != compressor_object);
                        assert(object != kernel_object);
                        fast_path_full_req = TRUE;
                        assert(object != compressor_object);
                        assert(object != kernel_object);
                        fast_path_full_req = TRUE;

-               }
-               else if (object->resident_page_count == 0)
-               {
+               } else if (object->resident_page_count == 0) {

                        assert(object != compressor_object);
                        assert(object != kernel_object);
                        fast_path_empty_req = TRUE;
                        assert(object != compressor_object);
                        assert(object != kernel_object);
                        fast_path_empty_req = TRUE;


@@ -8366,10 +8403,11 @@ vm_object_iopl_request(
                }
        }
 
                }
        }
 

-       if (cntrl_flags & UPL_SET_INTERRUPTIBLE)
+       if (cntrl_flags & UPL_SET_INTERRUPTIBLE) {

                interruptible = THREAD_ABORTSAFE;
                interruptible = THREAD_ABORTSAFE;

-       else
+       } else {

                interruptible = THREAD_UNINT;
                interruptible = THREAD_UNINT;

+       }

 
        entry = 0;
 
 
        entry = 0;
 


@@ -8378,22 +8416,20 @@ vm_object_iopl_request(
        dw_count = 0;
 
        if (fast_path_full_req) {
        dw_count = 0;
 
        if (fast_path_full_req) {

-
-               if (vm_object_iopl_wire_full(object, upl, user_page_list, lite_list, cntrl_flags, tag) == TRUE)
+               if (vm_object_iopl_wire_full(object, upl, user_page_list, lite_list, cntrl_flags, tag) == TRUE) {

                        goto finish;
                        goto finish;

+               }

                /*
                 * we couldn't complete the processing of this request on the fast path
                 * so fall through to the slow path and finish up
                 */
                /*
                 * we couldn't complete the processing of this request on the fast path
                 * so fall through to the slow path and finish up
                 */

-

        } else if (fast_path_empty_req) {
        } else if (fast_path_empty_req) {

-

                if (cntrl_flags & UPL_REQUEST_NO_FAULT) {
                        ret = KERN_MEMORY_ERROR;
                        goto return_err;
                }
                if (cntrl_flags & UPL_REQUEST_NO_FAULT) {
                        ret = KERN_MEMORY_ERROR;
                        goto return_err;
                }

-               ret = vm_object_iopl_wire_empty(object, upl, user_page_list, lite_list, cntrl_flags, tag, &dst_offset, size_in_pages);
-               
+               ret = vm_object_iopl_wire_empty(object, upl, user_page_list, lite_list, cntrl_flags, tag, &dst_offset, size_in_pages, &page_grab_count);
+

                if (ret) {
                        free_wired_pages = TRUE;
                        goto return_err;
                if (ret) {
                        free_wired_pages = TRUE;
                        goto return_err;


@@ -8402,13 +8438,8 @@ vm_object_iopl_request(
        }
 
        fault_info.behavior = VM_BEHAVIOR_SEQUENTIAL;
        }
 
        fault_info.behavior = VM_BEHAVIOR_SEQUENTIAL;

-       fault_info.user_tag  = 0;

        fault_info.lo_offset = offset;
        fault_info.hi_offset = offset + xfer_size;
        fault_info.lo_offset = offset;
        fault_info.hi_offset = offset + xfer_size;

-       fault_info.no_cache  = FALSE;
-       fault_info.stealth = FALSE;
-       fault_info.io_sync = FALSE;
-       fault_info.cs_bypass = FALSE;

        fault_info.mark_zf_absent = TRUE;
        fault_info.interruptible = interruptible;
        fault_info.batch_pmap_op = TRUE;
        fault_info.mark_zf_absent = TRUE;
        fault_info.interruptible = interruptible;
        fault_info.batch_pmap_op = TRUE;


@@ -8417,7 +8448,7 @@ vm_object_iopl_request(
        dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT);
 
        while (xfer_size) {
        dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT);
 
        while (xfer_size) {

-               vm_fault_return_t       result;
+               vm_fault_return_t       result;

 
                dwp->dw_mask = 0;
 
 
                dwp->dw_mask = 0;
 


@@ -8430,157 +8461,159 @@ vm_object_iopl_request(
                         * the following check is needed to determine whether
                         * this page was already processed in the fast path
                         */
                         * the following check is needed to determine whether
                         * this page was already processed in the fast path
                         */

-                       if (lite_list[entry>>5] & (1 << (entry & 31)))
+                       if (lite_list[entry >> 5] & (1 << (entry & 31))) {

                                goto skip_page;
                                goto skip_page;

+                       }

                }
                dst_page = vm_page_lookup(object, dst_offset);
 
                if (dst_page == VM_PAGE_NULL ||
                }
                dst_page = vm_page_lookup(object, dst_offset);
 
                if (dst_page == VM_PAGE_NULL ||

-                   dst_page->busy ||
-                   dst_page->error || 
-                   dst_page->restart ||
-                   dst_page->absent ||
-                   dst_page->fictitious) {
-
-                  if (object == kernel_object)
-                          panic("vm_object_iopl_request: missing/bad page in kernel object\n");
-                  if (object == compressor_object)
-                          panic("vm_object_iopl_request: missing/bad page in compressor object\n");
-
-                  if (cntrl_flags & UPL_REQUEST_NO_FAULT) {
-                          ret = KERN_MEMORY_ERROR;
-                          goto return_err;
-                  }
-                  set_cache_attr_needed = TRUE;
-
-                   /*
-                   * We just looked up the page and the result remains valid
-                   * until the object lock is release, so send it to
-                   * vm_fault_page() (as "dst_page"), to avoid having to
-                   * look it up again there.
-                   */
-                  caller_lookup = TRUE;
-
-                  do {
-                       vm_page_t       top_page;
-                       kern_return_t   error_code;
-
-                       fault_info.cluster_size = xfer_size;
-
-                       vm_object_paging_begin(object);
-
-                       result = vm_fault_page(object, dst_offset,
-                                              prot | VM_PROT_WRITE, FALSE,
-                                              caller_lookup,
-                                              &prot, &dst_page, &top_page,
-                                              (int *)0,
-                                              &error_code, no_zero_fill,
-                                              FALSE, &fault_info);
-
-                        /* our lookup is no longer valid at this point */
-                       caller_lookup = FALSE;
-
-                       switch (result) {
-
-                       case VM_FAULT_SUCCESS:
-
-                               if ( !dst_page->absent) {
-                                       PAGE_WAKEUP_DONE(dst_page);
-                               } else {
-                                       /*
-                                        * we only get back an absent page if we
-                                        * requested that it not be zero-filled
-                                        * because we are about to fill it via I/O
-                                        * 
-                                        * absent pages should be left BUSY
-                                        * to prevent them from being faulted
-                                        * into an address space before we've
-                                        * had a chance to complete the I/O on
-                                        * them since they may contain info that
-                                        * shouldn't be seen by the faulting task
-                                        */
-                               }
-                               /*
-                                *      Release paging references and
-                                *      top-level placeholder page, if any.
-                                */
-                               if (top_page != VM_PAGE_NULL) {
-                                       vm_object_t local_object;
+                   dst_page->vmp_busy ||
+                   dst_page->vmp_error ||
+                   dst_page->vmp_restart ||
+                   dst_page->vmp_absent ||
+                   dst_page->vmp_fictitious) {
+                       if (object == kernel_object) {
+                               panic("vm_object_iopl_request: missing/bad page in kernel object\n");
+                       }
+                       if (object == compressor_object) {
+                               panic("vm_object_iopl_request: missing/bad page in compressor object\n");
+                       }
+
+                       if (cntrl_flags & UPL_REQUEST_NO_FAULT) {
+                               ret = KERN_MEMORY_ERROR;
+                               goto return_err;
+                       }
+                       set_cache_attr_needed = TRUE;
+
+                       /*
+                        * We just looked up the page and the result remains valid
+                        * until the object lock is release, so send it to
+                        * vm_fault_page() (as "dst_page"), to avoid having to
+                        * look it up again there.
+                        */
+                       caller_lookup = TRUE;
+
+                       do {
+                               vm_page_t       top_page;
+                               kern_return_t   error_code;
+
+                               fault_info.cluster_size = xfer_size;
+
+                               vm_object_paging_begin(object);
+
+                               result = vm_fault_page(object, dst_offset,
+                                   prot | VM_PROT_WRITE, FALSE,
+                                   caller_lookup,
+                                   &prot, &dst_page, &top_page,
+                                   (int *)0,
+                                   &error_code, no_zero_fill,
+                                   FALSE, &fault_info);
+
+                               /* our lookup is no longer valid at this point */
+                               caller_lookup = FALSE;

 
 

-                                       local_object = VM_PAGE_OBJECT(top_page);
-                                       
+                               switch (result) {
+                               case VM_FAULT_SUCCESS:
+                                       page_grab_count++;
+
+                                       if (!dst_page->vmp_absent) {
+                                               PAGE_WAKEUP_DONE(dst_page);
+                                       } else {
+                                               /*
+                                                * we only get back an absent page if we
+                                                * requested that it not be zero-filled
+                                                * because we are about to fill it via I/O
+                                                *
+                                                * absent pages should be left BUSY
+                                                * to prevent them from being faulted
+                                                * into an address space before we've
+                                                * had a chance to complete the I/O on
+                                                * them since they may contain info that
+                                                * shouldn't be seen by the faulting task
+                                                */
+                                       }

                                        /*
                                        /*

-                                        * comparing 2 packed pointers
+                                        *      Release paging references and
+                                        *      top-level placeholder page, if any.

                                         */
                                         */

-                                       if (top_page->vm_page_object != dst_page->vm_page_object) {
-                                               vm_object_lock(local_object);
-                                               VM_PAGE_FREE(top_page);
-                                               vm_object_paging_end(local_object);
-                                               vm_object_unlock(local_object);
-                                       } else {
-                                               VM_PAGE_FREE(top_page);
-                                               vm_object_paging_end(local_object);
+                                       if (top_page != VM_PAGE_NULL) {
+                                               vm_object_t local_object;
+
+                                               local_object = VM_PAGE_OBJECT(top_page);
+
+                                               /*
+                                                * comparing 2 packed pointers
+                                                */
+                                               if (top_page->vmp_object != dst_page->vmp_object) {
+                                                       vm_object_lock(local_object);
+                                                       VM_PAGE_FREE(top_page);
+                                                       vm_object_paging_end(local_object);
+                                                       vm_object_unlock(local_object);
+                                               } else {
+                                                       VM_PAGE_FREE(top_page);
+                                                       vm_object_paging_end(local_object);
+                                               }

                                        }
                                        }

-                               }
-                               vm_object_paging_end(object);
-                               break;
-                       
-                       case VM_FAULT_RETRY:
-                               vm_object_lock(object);
-                               break;
+                                       vm_object_paging_end(object);
+                                       break;

 
 

-                       case VM_FAULT_MEMORY_SHORTAGE:
-                               OSAddAtomic((size_in_pages - entry), &vm_upl_wait_for_pages);
+                               case VM_FAULT_RETRY:
+                                       vm_object_lock(object);
+                                       break;

 
 

-                               VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_START, vm_upl_wait_for_pages, 0, 0, 0);
+                               case VM_FAULT_MEMORY_SHORTAGE:
+                                       OSAddAtomic((size_in_pages - entry), &vm_upl_wait_for_pages);

 
 

-                               if (vm_page_wait(interruptible)) {
-                                       OSAddAtomic(-(size_in_pages - entry), &vm_upl_wait_for_pages);
+                                       VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_START, vm_upl_wait_for_pages, 0, 0, 0);

 
 

-                                       VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, 0);
-                                       vm_object_lock(object);
+                                       if (vm_page_wait(interruptible)) {
+                                               OSAddAtomic(-(size_in_pages - entry), &vm_upl_wait_for_pages);

 
 

-                                       break;
-                               }
-                               OSAddAtomic(-(size_in_pages - entry), &vm_upl_wait_for_pages);
+                                               VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, 0);
+                                               vm_object_lock(object);

 
 

-                               VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, -1);
+                                               break;
+                                       }
+                                       OSAddAtomic(-(size_in_pages - entry), &vm_upl_wait_for_pages);

 
 

-                               /* fall thru */
+                                       VM_DEBUG_EVENT(vm_iopl_page_wait, VM_IOPL_PAGE_WAIT, DBG_FUNC_END, vm_upl_wait_for_pages, 0, 0, -1);

 
 

-                       case VM_FAULT_INTERRUPTED:
-                               error_code = MACH_SEND_INTERRUPTED;
-                       case VM_FAULT_MEMORY_ERROR:
-                       memory_error:
-                               ret = (error_code ? error_code: KERN_MEMORY_ERROR);
+                               /* fall thru */

 
 

-                               vm_object_lock(object);
-                               goto return_err;
+                               case VM_FAULT_INTERRUPTED:
+                                       error_code = MACH_SEND_INTERRUPTED;
+                               case VM_FAULT_MEMORY_ERROR:
+memory_error:
+                                       ret = (error_code ? error_code: KERN_MEMORY_ERROR);

 
 

-                       case VM_FAULT_SUCCESS_NO_VM_PAGE:
-                               /* success but no page: fail */
-                               vm_object_paging_end(object);
-                               vm_object_unlock(object);
-                               goto memory_error;
+                                       vm_object_lock(object);
+                                       goto return_err;

 
 

-                       default:
-                               panic("vm_object_iopl_request: unexpected error"
-                                     " 0x%x from vm_fault_page()\n", result);
-                       }
-                  } while (result != VM_FAULT_SUCCESS);
+                               case VM_FAULT_SUCCESS_NO_VM_PAGE:
+                                       /* success but no page: fail */
+                                       vm_object_paging_end(object);
+                                       vm_object_unlock(object);
+                                       goto memory_error;

 
 

+                               default:
+                                       panic("vm_object_iopl_request: unexpected error"
+                                           " 0x%x from vm_fault_page()\n", result);
+                               }
+                       } while (result != VM_FAULT_SUCCESS);

                }
                phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page);
 
                }
                phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page);
 

-               if (upl->flags & UPL_KERNEL_OBJECT)
+               if (upl->flags & UPL_KERNEL_OBJECT) {

                        goto record_phys_addr;
                        goto record_phys_addr;

+               }

 
 

-               if (dst_page->vm_page_q_state == VM_PAGE_USED_BY_COMPRESSOR) {
-                       dst_page->busy = TRUE;
+               if (dst_page->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) {
+                       dst_page->vmp_busy = TRUE;

                        goto record_phys_addr;
                }
 
                        goto record_phys_addr;
                }
 

-               if (dst_page->cleaning) {
+               if (dst_page->vmp_cleaning) {

                        /*
                         * Someone else is cleaning this page in place.
                         * In theory, we should be able to  proceed and use this
                        /*
                         * Someone else is cleaning this page in place.
                         * In theory, we should be able to  proceed and use this


@@ -8591,17 +8624,18 @@ vm_object_iopl_request(
                         * We'd better wait for the cleaning to complete and
                         * then try again.
                         */
                         * We'd better wait for the cleaning to complete and
                         * then try again.
                         */

-                       vm_object_iopl_request_sleep_for_cleaning++;
+                       VM_PAGEOUT_DEBUG(vm_object_iopl_request_sleep_for_cleaning, 1);

                        PAGE_SLEEP(object, dst_page, THREAD_UNINT);
                        continue;
                }
                        PAGE_SLEEP(object, dst_page, THREAD_UNINT);
                        continue;
                }

-               if (dst_page->laundry)
+               if (dst_page->vmp_laundry) {

                        vm_pageout_steal_laundry(dst_page, FALSE);
                        vm_pageout_steal_laundry(dst_page, FALSE);

+               }

 
 

-               if ( (cntrl_flags & UPL_NEED_32BIT_ADDR) &&
-                    phys_page >= (max_valid_dma_address >> PAGE_SHIFT) ) {
-                       vm_page_t       low_page;
-                       int             refmod;
+               if ((cntrl_flags & UPL_NEED_32BIT_ADDR) &&
+                   phys_page >= (max_valid_dma_address >> PAGE_SHIFT)) {
+                       vm_page_t       low_page;
+                       int             refmod;

 
                        /*
                         * support devices that can't DMA above 32 bits
 
                        /*
                         * support devices that can't DMA above 32 bits


@@ -8612,13 +8646,13 @@ vm_object_iopl_request(
                         * handed out to some other 64 bit capable DMA device to use
                         */
                        if (VM_PAGE_WIRED(dst_page)) {
                         * handed out to some other 64 bit capable DMA device to use
                         */
                        if (VM_PAGE_WIRED(dst_page)) {

-                               ret = KERN_PROTECTION_FAILURE;
+                               ret = KERN_PROTECTION_FAILURE;

                                goto return_err;
                        }
                        low_page = vm_page_grablo();
 
                        if (low_page == VM_PAGE_NULL) {
                                goto return_err;
                        }
                        low_page = vm_page_grablo();
 
                        if (low_page == VM_PAGE_NULL) {

-                               ret = KERN_RESOURCE_SHORTAGE;
+                               ret = KERN_RESOURCE_SHORTAGE;

                                goto return_err;
                        }
                        /*
                                goto return_err;
                        }
                        /*


@@ -8628,22 +8662,25 @@ vm_object_iopl_request(
                         * it after we disconnect it... we want the fault
                         * to find the new page being substituted.
                         */
                         * it after we disconnect it... we want the fault
                         * to find the new page being substituted.
                         */

-                       if (dst_page->pmapped)
-                               refmod = pmap_disconnect(phys_page);
-                       else
-                               refmod = 0;
+                       if (dst_page->vmp_pmapped) {
+                               refmod = pmap_disconnect(phys_page);
+                       } else {
+                               refmod = 0;
+                       }

 
 

-                       if (!dst_page->absent)
+                       if (!dst_page->vmp_absent) {

                                vm_page_copy(dst_page, low_page);
                                vm_page_copy(dst_page, low_page);

-                 
-                       low_page->reference = dst_page->reference;
-                       low_page->dirty     = dst_page->dirty;
-                       low_page->absent    = dst_page->absent;
+                       }
+
+                       low_page->vmp_reference = dst_page->vmp_reference;
+                       low_page->vmp_dirty     = dst_page->vmp_dirty;
+                       low_page->vmp_absent    = dst_page->vmp_absent;

 
 

-                       if (refmod & VM_MEM_REFERENCED)
-                               low_page->reference = TRUE;
+                       if (refmod & VM_MEM_REFERENCED) {
+                               low_page->vmp_reference = TRUE;
+                       }

                        if (refmod & VM_MEM_MODIFIED) {
                        if (refmod & VM_MEM_MODIFIED) {

-                               SET_PAGE_DIRTY(low_page, FALSE);
+                               SET_PAGE_DIRTY(low_page, FALSE);

                        }
 
                        vm_page_replace(low_page, object, dst_offset);
                        }
 
                        vm_page_replace(low_page, object, dst_offset);


@@ -8654,13 +8691,15 @@ vm_object_iopl_request(
                         * BUSY... we don't need a PAGE_WAKEUP_DONE
                         * here, because we've never dropped the object lock
                         */
                         * BUSY... we don't need a PAGE_WAKEUP_DONE
                         * here, because we've never dropped the object lock
                         */

-                       if ( !dst_page->absent)
-                               dst_page->busy = FALSE;
+                       if (!dst_page->vmp_absent) {
+                               dst_page->vmp_busy = FALSE;
+                       }

 
                        phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page);
                }
 
                        phys_page = VM_PAGE_GET_PHYS_PAGE(dst_page);
                }

-               if ( !dst_page->busy)
+               if (!dst_page->vmp_busy) {

                        dwp->dw_mask |= DW_vm_page_wire;
                        dwp->dw_mask |= DW_vm_page_wire;

+               }

 
                if (cntrl_flags & UPL_BLOCK_ACCESS) {
                        /*
 
                if (cntrl_flags & UPL_BLOCK_ACCESS) {
                        /*


@@ -8669,8 +8708,8 @@ vm_object_iopl_request(
                         * We'll also remove the mapping
                         * of all these pages before leaving this routine.
                         */
                         * We'll also remove the mapping
                         * of all these pages before leaving this routine.
                         */

-                       assert(!dst_page->fictitious);
-                       dst_page->busy = TRUE;
+                       assert(!dst_page->vmp_fictitious);
+                       dst_page->vmp_busy = TRUE;

                }
                /*
                 * expect the page to be used
                }
                /*
                 * expect the page to be used


@@ -8678,48 +8717,52 @@ vm_object_iopl_request(
                 */
                dwp->dw_mask |= DW_set_reference;
 
                 */
                dwp->dw_mask |= DW_set_reference;
 

-               if (!(cntrl_flags & UPL_COPYOUT_FROM)) {
-                       SET_PAGE_DIRTY(dst_page, TRUE); 
+               if (!(cntrl_flags & UPL_COPYOUT_FROM)) {
+                       SET_PAGE_DIRTY(dst_page, TRUE);

                }
                }

-               if ((cntrl_flags & UPL_REQUEST_FORCE_COHERENCY) && dst_page->written_by_kernel == TRUE) {
+               if ((cntrl_flags & UPL_REQUEST_FORCE_COHERENCY) && dst_page->vmp_written_by_kernel == TRUE) {

                        pmap_sync_page_attributes_phys(phys_page);
                        pmap_sync_page_attributes_phys(phys_page);

-                       dst_page->written_by_kernel = FALSE;
+                       dst_page->vmp_written_by_kernel = FALSE;

                }
 
 record_phys_addr:
                }
 
 record_phys_addr:

-               if (dst_page->busy)
+               if (dst_page->vmp_busy) {

                        upl->flags |= UPL_HAS_BUSY;
                        upl->flags |= UPL_HAS_BUSY;

+               }

 
 

-               lite_list[entry>>5] |= 1 << (entry & 31);
+               lite_list[entry >> 5] |= 1 << (entry & 31);

 
 

-               if (phys_page > upl->highest_page)
-                       upl->highest_page = phys_page;
+               if (phys_page > upl->highest_page) {
+                       upl->highest_page = phys_page;
+               }

 
                if (user_page_list) {
 
                if (user_page_list) {

-                       user_page_list[entry].phys_addr = phys_page;
-                       user_page_list[entry].free_when_done    = dst_page->free_when_done;
-                       user_page_list[entry].absent    = dst_page->absent;
-                       user_page_list[entry].dirty     = dst_page->dirty;
-                       user_page_list[entry].precious  = dst_page->precious;
-                       user_page_list[entry].device    = FALSE;
+                       user_page_list[entry].phys_addr = phys_page;
+                       user_page_list[entry].free_when_done    = dst_page->vmp_free_when_done;
+                       user_page_list[entry].absent    = dst_page->vmp_absent;
+                       user_page_list[entry].dirty     = dst_page->vmp_dirty;
+                       user_page_list[entry].precious  = dst_page->vmp_precious;
+                       user_page_list[entry].device    = FALSE;

                        user_page_list[entry].needed    = FALSE;
                        user_page_list[entry].needed    = FALSE;

-                       if (dst_page->clustered == TRUE)
-                               user_page_list[entry].speculative = (dst_page->vm_page_q_state == VM_PAGE_ON_SPECULATIVE_Q) ? TRUE : FALSE;
-                       else
-                               user_page_list[entry].speculative = FALSE;
-                       user_page_list[entry].cs_validated = dst_page->cs_validated;
-                       user_page_list[entry].cs_tainted = dst_page->cs_tainted;
-                       user_page_list[entry].cs_nx = dst_page->cs_nx;
+                       if (dst_page->vmp_clustered == TRUE) {
+                               user_page_list[entry].speculative = (dst_page->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q) ? TRUE : FALSE;
+                       } else {
+                               user_page_list[entry].speculative = FALSE;
+                       }
+                       user_page_list[entry].cs_validated = dst_page->vmp_cs_validated;
+                       user_page_list[entry].cs_tainted = dst_page->vmp_cs_tainted;
+                       user_page_list[entry].cs_nx = dst_page->vmp_cs_nx;

                        user_page_list[entry].mark      = FALSE;
                }
                if (object != kernel_object && object != compressor_object) {
                        /*
                         * someone is explicitly grabbing this page...
                         * update clustered and speculative state
                        user_page_list[entry].mark      = FALSE;
                }
                if (object != kernel_object && object != compressor_object) {
                        /*
                         * someone is explicitly grabbing this page...
                         * update clustered and speculative state

-                        * 
+                        *

                         */
                         */

-                       if (dst_page->clustered)
+                       if (dst_page->vmp_clustered) {

                                VM_PAGE_CONSUME_CLUSTERED(dst_page);
                                VM_PAGE_CONSUME_CLUSTERED(dst_page);

+                       }

                }
 skip_page:
                entry++;
                }
 skip_page:
                entry++;


@@ -8731,7 +8774,7 @@ skip_page:
 
                        if (dw_count >= dw_limit) {
                                vm_page_do_delayed_work(object, tag, &dw_array[0], dw_count);
 
                        if (dw_count >= dw_limit) {
                                vm_page_do_delayed_work(object, tag, &dw_array[0], dw_count);

-                               
+

                                dwp = &dw_array[0];
                                dw_count = 0;
                        }
                                dwp = &dw_array[0];
                                dw_count = 0;
                        }


@@ -8739,17 +8782,20 @@ skip_page:
        }
        assert(entry == size_in_pages);
 
        }
        assert(entry == size_in_pages);
 

-       if (dw_count)
+       if (dw_count) {

                vm_page_do_delayed_work(object, tag, &dw_array[0], dw_count);
                vm_page_do_delayed_work(object, tag, &dw_array[0], dw_count);

+       }

 finish:
 finish:

-       if (user_page_list && set_cache_attr_needed == TRUE)
+       if (user_page_list && set_cache_attr_needed == TRUE) {

                vm_object_set_pmap_cache_attr(object, user_page_list, size_in_pages, TRUE);
                vm_object_set_pmap_cache_attr(object, user_page_list, size_in_pages, TRUE);

+       }

 
        if (page_list_count != NULL) {
 
        if (page_list_count != NULL) {

-               if (upl->flags & UPL_INTERNAL)
+               if (upl->flags & UPL_INTERNAL) {

                        *page_list_count = 0;
                        *page_list_count = 0;

-               else if (*page_list_count > size_in_pages)
+               } else if (*page_list_count > size_in_pages) {

                        *page_list_count = size_in_pages;
                        *page_list_count = size_in_pages;

+               }

        }
        vm_object_unlock(object);
 
        }
        vm_object_unlock(object);
 


@@ -8761,11 +8807,17 @@ finish:
                 * can't be accessed without causing a page fault.
                 */
                vm_object_pmap_protect(object, offset, (vm_object_size_t)size,
                 * can't be accessed without causing a page fault.
                 */
                vm_object_pmap_protect(object, offset, (vm_object_size_t)size,

-                                      PMAP_NULL, 0, VM_PROT_NONE);
+                   PMAP_NULL, 0, VM_PROT_NONE);

                assert(!object->blocked_access);
                object->blocked_access = TRUE;
        }
 
                assert(!object->blocked_access);
                object->blocked_access = TRUE;
        }
 

+       VM_DEBUG_CONSTANT_EVENT(vm_object_iopl_request, VM_IOPL_REQUEST, DBG_FUNC_END, page_grab_count, KERN_SUCCESS, 0, 0);
+#if DEVELOPMENT || DEBUG
+       if (task != NULL) {
+               ledger_credit(task->ledger, task_ledgers.pages_grabbed_iopl, page_grab_count);
+       }
+#endif /* DEVELOPMENT || DEBUG */

        return KERN_SUCCESS;
 
 return_err:
        return KERN_SUCCESS;
 
 return_err:


@@ -8774,13 +8826,14 @@ return_err:
        for (; offset < dst_offset; offset += PAGE_SIZE) {
                boolean_t need_unwire;
 
        for (; offset < dst_offset; offset += PAGE_SIZE) {
                boolean_t need_unwire;
 

-               dst_page = vm_page_lookup(object, offset);
+               dst_page = vm_page_lookup(object, offset);

 
 

-               if (dst_page == VM_PAGE_NULL)
-                       panic("vm_object_iopl_request: Wired page missing. \n");
+               if (dst_page == VM_PAGE_NULL) {
+                       panic("vm_object_iopl_request: Wired page missing. \n");
+               }

 
                /*
 
                /*

-                * if we've already processed this page in an earlier 
+                * if we've already processed this page in an earlier

                 * dw_do_work, we need to undo the wiring... we will
                 * leave the dirty and reference bits on if they
                 * were set, since we don't have a good way of knowing
                 * dw_do_work, we need to undo the wiring... we will
                 * leave the dirty and reference bits on if they
                 * were set, since we don't have a good way of knowing


@@ -8806,47 +8859,55 @@ return_err:
                }
                vm_page_lock_queues();
 
                }
                vm_page_lock_queues();
 

-               if (dst_page->absent || free_wired_pages == TRUE) {
+               if (dst_page->vmp_absent || free_wired_pages == TRUE) {

                        vm_page_free(dst_page);
 
                        need_unwire = FALSE;
                } else {
                        vm_page_free(dst_page);
 
                        need_unwire = FALSE;
                } else {

-                       if (need_unwire == TRUE)
+                       if (need_unwire == TRUE) {

                                vm_page_unwire(dst_page, TRUE);
                                vm_page_unwire(dst_page, TRUE);

+                       }

 
                        PAGE_WAKEUP_DONE(dst_page);
                }
                vm_page_unlock_queues();
 
 
                        PAGE_WAKEUP_DONE(dst_page);
                }
                vm_page_unlock_queues();
 

-               if (need_unwire == TRUE)
+               if (need_unwire == TRUE) {

                        VM_STAT_INCR(reactivations);
                        VM_STAT_INCR(reactivations);

+               }

        }
 #if UPL_DEBUG
        upl->upl_state = 2;
 #endif
        }
 #if UPL_DEBUG
        upl->upl_state = 2;
 #endif

-       if (! (upl->flags & UPL_KERNEL_OBJECT)) {
+       if (!(upl->flags & UPL_KERNEL_OBJECT)) {

                vm_object_activity_end(object);
                vm_object_collapse(object, 0, TRUE);
        }
        vm_object_unlock(object);
        upl_destroy(upl);
 
                vm_object_activity_end(object);
                vm_object_collapse(object, 0, TRUE);
        }
        vm_object_unlock(object);
        upl_destroy(upl);
 

+       VM_DEBUG_CONSTANT_EVENT(vm_object_iopl_request, VM_IOPL_REQUEST, DBG_FUNC_END, page_grab_count, ret, 0, 0);
+#if DEVELOPMENT || DEBUG
+       if (task != NULL) {
+               ledger_credit(task->ledger, task_ledgers.pages_grabbed_iopl, page_grab_count);
+       }
+#endif /* DEVELOPMENT || DEBUG */

        return ret;
 }
 
 kern_return_t
 upl_transpose(
        return ret;
 }
 
 kern_return_t
 upl_transpose(

-       upl_t           upl1,
-       upl_t           upl2)
+       upl_t           upl1,
+       upl_t           upl2)

 {
 {

-       kern_return_t           retval;
-       boolean_t               upls_locked;
-       vm_object_t             object1, object2;
+       kern_return_t           retval;
+       boolean_t               upls_locked;
+       vm_object_t             object1, object2;

 
 

-       if (upl1 == UPL_NULL || upl2 == UPL_NULL || upl1 == upl2  || ((upl1->flags & UPL_VECTOR)==UPL_VECTOR)  || ((upl2->flags & UPL_VECTOR)==UPL_VECTOR)) {
+       if (upl1 == UPL_NULL || upl2 == UPL_NULL || upl1 == upl2 || ((upl1->flags & UPL_VECTOR) == UPL_VECTOR) || ((upl2->flags & UPL_VECTOR) == UPL_VECTOR)) {

                return KERN_INVALID_ARGUMENT;
        }
                return KERN_INVALID_ARGUMENT;
        }

-       
+

        upls_locked = FALSE;
 
        /*
        upls_locked = FALSE;
 
        /*


@@ -8860,7 +8921,7 @@ upl_transpose(
                upl_lock(upl2);
                upl_lock(upl1);
        }
                upl_lock(upl2);
                upl_lock(upl1);
        }

-       upls_locked = TRUE;     /* the UPLs will need to be unlocked */
+       upls_locked = TRUE;     /* the UPLs will need to be unlocked */

 
        object1 = upl1->map_object;
        object2 = upl2->map_object;
 
        object1 = upl1->map_object;
        object2 = upl2->map_object;


@@ -8881,7 +8942,7 @@ upl_transpose(
         * Tranpose the VM objects' backing store.
         */
        retval = vm_object_transpose(object1, object2,
         * Tranpose the VM objects' backing store.
         */
        retval = vm_object_transpose(object1, object2,

-                                    (vm_object_size_t) upl1->size);
+           (vm_object_size_t) upl1->size);

 
        if (retval == KERN_SUCCESS) {
                /*
 
        if (retval == KERN_SUCCESS) {
                /*


@@ -8893,19 +8954,23 @@ upl_transpose(
                        vm_object_lock(object1);
                        vm_object_lock(object2);
                }
                        vm_object_lock(object1);
                        vm_object_lock(object2);
                }

-               if (upl1->flags & UPL_TRACKED_BY_OBJECT)
+               if (upl1->flags & UPL_TRACKED_BY_OBJECT) {

                        queue_remove(&object1->uplq, upl1, upl_t, uplq);
                        queue_remove(&object1->uplq, upl1, upl_t, uplq);

-               if (upl2->flags & UPL_TRACKED_BY_OBJECT)
+               }
+               if (upl2->flags & UPL_TRACKED_BY_OBJECT) {

                        queue_remove(&object2->uplq, upl2, upl_t, uplq);
                        queue_remove(&object2->uplq, upl2, upl_t, uplq);

+               }

 #endif
                upl1->map_object = object2;
                upl2->map_object = object1;
 
 #if CONFIG_IOSCHED || UPL_DEBUG
 #endif
                upl1->map_object = object2;
                upl2->map_object = object1;
 
 #if CONFIG_IOSCHED || UPL_DEBUG

-               if (upl1->flags & UPL_TRACKED_BY_OBJECT)
+               if (upl1->flags & UPL_TRACKED_BY_OBJECT) {

                        queue_enter(&object2->uplq, upl1, upl_t, uplq);
                        queue_enter(&object2->uplq, upl1, upl_t, uplq);

-               if (upl2->flags & UPL_TRACKED_BY_OBJECT)
+               }
+               if (upl2->flags & UPL_TRACKED_BY_OBJECT) {

                        queue_enter(&object1->uplq, upl2, upl_t, uplq);
                        queue_enter(&object1->uplq, upl2, upl_t, uplq);

+               }

                if ((upl1->flags & UPL_TRACKED_BY_OBJECT) || (upl2->flags & UPL_TRACKED_BY_OBJECT)) {
                        vm_object_unlock(object2);
                        vm_object_unlock(object1);
                if ((upl1->flags & UPL_TRACKED_BY_OBJECT) || (upl2->flags & UPL_TRACKED_BY_OBJECT)) {
                        vm_object_unlock(object2);
                        vm_object_unlock(object1);


@@ -8928,22 +8993,24 @@ done:
 
 void
 upl_range_needed(
 
 void
 upl_range_needed(

-       upl_t           upl,
-       int             index,
-       int             count)
+       upl_t           upl,
+       int             index,
+       int             count)

 {
 {

-       upl_page_info_t *user_page_list;
-       int             size_in_pages;
+       upl_page_info_t *user_page_list;
+       int             size_in_pages;

 
 

-       if ( !(upl->flags & UPL_INTERNAL) || count <= 0)
+       if (!(upl->flags & UPL_INTERNAL) || count <= 0) {

                return;
                return;

+       }

 
        size_in_pages = upl->size / PAGE_SIZE;
 
        user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl));
 
 
        size_in_pages = upl->size / PAGE_SIZE;
 
        user_page_list = (upl_page_info_t *) (((uintptr_t)upl) + sizeof(struct upl));
 

-       while (count-- && index < size_in_pages)
+       while (count-- && index < size_in_pages) {

                user_page_list[index++].needed = TRUE;
                user_page_list[index++].needed = TRUE;

+       }

 }
 
 
 }
 
 


@@ -8956,25 +9023,26 @@ upl_range_needed(
  * virtaul address space each time we need to work with
  * a physical page.
  */
  * virtaul address space each time we need to work with
  * a physical page.
  */

-decl_simple_lock_data(,vm_paging_lock)
-#define VM_PAGING_NUM_PAGES    64
+decl_simple_lock_data(, vm_paging_lock)
+#define VM_PAGING_NUM_PAGES     64

 vm_map_offset_t vm_paging_base_address = 0;
 vm_map_offset_t vm_paging_base_address = 0;

-boolean_t      vm_paging_page_inuse[VM_PAGING_NUM_PAGES] = { FALSE, };
-int            vm_paging_max_index = 0;
-int            vm_paging_page_waiter = 0;
-int            vm_paging_page_waiter_total = 0;
-unsigned long  vm_paging_no_kernel_page = 0;
-unsigned long  vm_paging_objects_mapped = 0;
-unsigned long  vm_paging_pages_mapped = 0;
-unsigned long  vm_paging_objects_mapped_slow = 0;
-unsigned long  vm_paging_pages_mapped_slow = 0;
+boolean_t       vm_paging_page_inuse[VM_PAGING_NUM_PAGES] = { FALSE, };
+int             vm_paging_max_index = 0;
+int             vm_paging_page_waiter = 0;
+int             vm_paging_page_waiter_total = 0;
+
+unsigned long   vm_paging_no_kernel_page = 0;
+unsigned long   vm_paging_objects_mapped = 0;
+unsigned long   vm_paging_pages_mapped = 0;
+unsigned long   vm_paging_objects_mapped_slow = 0;
+unsigned long   vm_paging_pages_mapped_slow = 0;

 
 void
 vm_paging_map_init(void)
 {
 
 void
 vm_paging_map_init(void)
 {

-       kern_return_t   kr;
-       vm_map_offset_t page_map_offset;
-       vm_map_entry_t  map_entry;
+       kern_return_t   kr;
+       vm_map_offset_t page_map_offset;
+       vm_map_entry_t  map_entry;

 
        assert(vm_paging_base_address == 0);
 
 
        assert(vm_paging_base_address == 0);
 


@@ -8984,13 +9052,13 @@ vm_paging_map_init(void)
         */
        page_map_offset = 0;
        kr = vm_map_find_space(kernel_map,
         */
        page_map_offset = 0;
        kr = vm_map_find_space(kernel_map,

-                              &page_map_offset,
-                              VM_PAGING_NUM_PAGES * PAGE_SIZE,
-                              0,
-                              0,
-                              VM_MAP_KERNEL_FLAGS_NONE,
-                              VM_KERN_MEMORY_NONE,
-                              &map_entry);
+           &page_map_offset,
+           VM_PAGING_NUM_PAGES * PAGE_SIZE,
+           0,
+           0,
+           VM_MAP_KERNEL_FLAGS_NONE,
+           VM_KERN_MEMORY_NONE,
+           &map_entry);

        if (kr != KERN_SUCCESS) {
                panic("vm_paging_map_init: kernel_map full\n");
        }
        if (kr != KERN_SUCCESS) {
                panic("vm_paging_map_init: kernel_map full\n");
        }


@@ -9009,58 +9077,58 @@ vm_paging_map_init(void)
 /*
  * vm_paging_map_object:
  *     Maps part of a VM object's pages in the kernel
 /*
  * vm_paging_map_object:
  *     Maps part of a VM object's pages in the kernel

- *     virtual address space, using the pre-allocated
+ *      virtual address space, using the pre-allocated

  *     kernel virtual addresses, if possible.
  * Context:
  *     kernel virtual addresses, if possible.
  * Context:

- *     The VM object is locked.  This lock will get
- *     dropped and re-acquired though, so the caller
- *     must make sure the VM object is kept alive
+ *      The VM object is locked.  This lock will get
+ *      dropped and re-acquired though, so the caller
+ *      must make sure the VM object is kept alive

  *     (by holding a VM map that has a reference
  *     (by holding a VM map that has a reference

- *     on it, for example, or taking an extra reference).
- *     The page should also be kept busy to prevent
+ *      on it, for example, or taking an extra reference).
+ *      The page should also be kept busy to prevent

  *     it from being reclaimed.
  */
 kern_return_t
 vm_paging_map_object(
  *     it from being reclaimed.
  */
 kern_return_t
 vm_paging_map_object(

-       vm_page_t               page,
-       vm_object_t             object,
-       vm_object_offset_t      offset,
-       vm_prot_t               protection,
-       boolean_t               can_unlock_object,
-       vm_map_size_t           *size,          /* IN/OUT */
-       vm_map_offset_t         *address,       /* OUT */
-       boolean_t               *need_unmap)    /* OUT */
+       vm_page_t               page,
+       vm_object_t             object,
+       vm_object_offset_t      offset,
+       vm_prot_t               protection,
+       boolean_t               can_unlock_object,
+       vm_map_size_t           *size,          /* IN/OUT */
+       vm_map_offset_t         *address,       /* OUT */
+       boolean_t               *need_unmap)    /* OUT */

 {
 {

-       kern_return_t           kr;
-       vm_map_offset_t         page_map_offset;
-       vm_map_size_t           map_size;
-       vm_object_offset_t      object_offset;
-       int                     i;
+       kern_return_t           kr;
+       vm_map_offset_t         page_map_offset;
+       vm_map_size_t           map_size;
+       vm_object_offset_t      object_offset;
+       int                     i;

 
        if (page != VM_PAGE_NULL && *size == PAGE_SIZE) {
                /* use permanent 1-to-1 kernel mapping of physical memory ? */
 #if __x86_64__
                *address = (vm_map_offset_t)
 
        if (page != VM_PAGE_NULL && *size == PAGE_SIZE) {
                /* use permanent 1-to-1 kernel mapping of physical memory ? */
 #if __x86_64__
                *address = (vm_map_offset_t)

-                       PHYSMAP_PTOV((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(page) <<
-                                    PAGE_SHIFT);
+                   PHYSMAP_PTOV((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(page) <<
+                       PAGE_SHIFT);

                *need_unmap = FALSE;
                return KERN_SUCCESS;
 #elif __arm__ || __arm64__
                *address = (vm_map_offset_t)
                *need_unmap = FALSE;
                return KERN_SUCCESS;
 #elif __arm__ || __arm64__
                *address = (vm_map_offset_t)

-                       phystokv((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(page) << PAGE_SHIFT);
+                   phystokv((pmap_paddr_t)VM_PAGE_GET_PHYS_PAGE(page) << PAGE_SHIFT);

                *need_unmap = FALSE;
                return KERN_SUCCESS;
 #else
 #warn "vm_paging_map_object: no 1-to-1 kernel mapping of physical memory..."
 #endif
 
                *need_unmap = FALSE;
                return KERN_SUCCESS;
 #else
 #warn "vm_paging_map_object: no 1-to-1 kernel mapping of physical memory..."
 #endif
 

-               assert(page->busy);
+               assert(page->vmp_busy);

                /*
                 * Use one of the pre-allocated kernel virtual addresses
                 * and just enter the VM page in the kernel address space
                 * at that virtual address.
                 */
                /*
                 * Use one of the pre-allocated kernel virtual addresses
                 * and just enter the VM page in the kernel address space
                 * at that virtual address.
                 */

-               simple_lock(&vm_paging_lock);
+               simple_lock(&vm_paging_lock, &vm_pageout_lck_grp);

 
                /*
                 * Try and find an available kernel virtual address
 
                /*
                 * Try and find an available kernel virtual address


@@ -9071,8 +9139,8 @@ vm_paging_map_object(
                        for (i = 0; i < VM_PAGING_NUM_PAGES; i++) {
                                if (vm_paging_page_inuse[i] == FALSE) {
                                        page_map_offset =
                        for (i = 0; i < VM_PAGING_NUM_PAGES; i++) {
                                if (vm_paging_page_inuse[i] == FALSE) {
                                        page_map_offset =

-                                               vm_paging_base_address +
-                                               (i * PAGE_SIZE);
+                                           vm_paging_base_address +
+                                           (i * PAGE_SIZE);

                                        break;
                                }
                        }
                                        break;
                                }
                        }


@@ -9098,7 +9166,7 @@ vm_paging_map_object(
                        if (kr == THREAD_WAITING) {
                                simple_unlock(&vm_paging_lock);
                                kr = thread_block(THREAD_CONTINUE_NULL);
                        if (kr == THREAD_WAITING) {
                                simple_unlock(&vm_paging_lock);
                                kr = thread_block(THREAD_CONTINUE_NULL);

-                               simple_lock(&vm_paging_lock);
+                               simple_lock(&vm_paging_lock, &vm_pageout_lck_grp);

                        }
                        vm_paging_page_waiter--;
                        /* ... and try again */
                        }
                        vm_paging_page_waiter--;
                        /* ... and try again */


@@ -9115,25 +9183,25 @@ vm_paging_map_object(
                        vm_paging_page_inuse[i] = TRUE;
                        simple_unlock(&vm_paging_lock);
 
                        vm_paging_page_inuse[i] = TRUE;
                        simple_unlock(&vm_paging_lock);
 

-                       page->pmapped = TRUE;
+                       page->vmp_pmapped = TRUE;

 
                        /*
                         * Keep the VM object locked over the PMAP_ENTER
                         * and the actual use of the page by the kernel,
 
                        /*
                         * Keep the VM object locked over the PMAP_ENTER
                         * and the actual use of the page by the kernel,

-                        * or this pmap mapping might get undone by a 
+                        * or this pmap mapping might get undone by a

                         * vm_object_pmap_protect() call...
                         */
                        PMAP_ENTER(kernel_pmap,
                         * vm_object_pmap_protect() call...
                         */
                        PMAP_ENTER(kernel_pmap,

-                                  page_map_offset,
-                                  page,
-                                  protection,
-                                  VM_PROT_NONE,
-                                  0,
-                                  TRUE,
-                                  kr);
+                           page_map_offset,
+                           page,
+                           protection,
+                           VM_PROT_NONE,
+                           0,
+                           TRUE,
+                           kr);

                        assert(kr == KERN_SUCCESS);
                        vm_paging_objects_mapped++;
                        assert(kr == KERN_SUCCESS);
                        vm_paging_objects_mapped++;

-                       vm_paging_pages_mapped++; 
+                       vm_paging_pages_mapped++;

                        *address = page_map_offset;
                        *need_unmap = TRUE;
 
                        *address = page_map_offset;
                        *need_unmap = TRUE;
 


@@ -9154,7 +9222,7 @@ vm_paging_map_object(
                simple_unlock(&vm_paging_lock);
        }
 
                simple_unlock(&vm_paging_lock);
        }
 

-       if (! can_unlock_object) {
+       if (!can_unlock_object) {

                *address = 0;
                *size = 0;
                *need_unmap = FALSE;
                *address = 0;
                *size = 0;
                *need_unmap = FALSE;


@@ -9163,34 +9231,34 @@ vm_paging_map_object(
 
        object_offset = vm_object_trunc_page(offset);
        map_size = vm_map_round_page(*size,
 
        object_offset = vm_object_trunc_page(offset);
        map_size = vm_map_round_page(*size,

-                                    VM_MAP_PAGE_MASK(kernel_map));
+           VM_MAP_PAGE_MASK(kernel_map));

 
        /*
         * Try and map the required range of the object
         * in the kernel_map
         */
 
 
        /*
         * Try and map the required range of the object
         * in the kernel_map
         */
 

-       vm_object_reference_locked(object);     /* for the map entry */
+       vm_object_reference_locked(object);     /* for the map entry */

        vm_object_unlock(object);
 
        kr = vm_map_enter(kernel_map,
        vm_object_unlock(object);
 
        kr = vm_map_enter(kernel_map,

-                         address,
-                         map_size,
-                         0,
-                         VM_FLAGS_ANYWHERE,
-                         VM_MAP_KERNEL_FLAGS_NONE,
-                         VM_KERN_MEMORY_NONE,
-                         object,
-                         object_offset,
-                         FALSE,
-                         protection,
-                         VM_PROT_ALL,
-                         VM_INHERIT_NONE);
+           address,
+           map_size,
+           0,
+           VM_FLAGS_ANYWHERE,
+           VM_MAP_KERNEL_FLAGS_NONE,
+           VM_KERN_MEMORY_NONE,
+           object,
+           object_offset,
+           FALSE,
+           protection,
+           VM_PROT_ALL,
+           VM_INHERIT_NONE);

        if (kr != KERN_SUCCESS) {
                *address = 0;
                *size = 0;
                *need_unmap = FALSE;
        if (kr != KERN_SUCCESS) {
                *address = 0;
                *size = 0;
                *need_unmap = FALSE;

-               vm_object_deallocate(object);   /* for the map entry */
+               vm_object_deallocate(object);   /* for the map entry */

                vm_object_lock(object);
                return kr;
        }
                vm_object_lock(object);
                return kr;
        }


@@ -9209,15 +9277,14 @@ vm_paging_map_object(
         */
 
        for (page_map_offset = 0;
         */
 
        for (page_map_offset = 0;

-            map_size != 0;
-            map_size -= PAGE_SIZE_64, page_map_offset += PAGE_SIZE_64) {
-
+           map_size != 0;
+           map_size -= PAGE_SIZE_64, page_map_offset += PAGE_SIZE_64) {

                page = vm_page_lookup(object, offset + page_map_offset);
                if (page == VM_PAGE_NULL) {
                        printf("vm_paging_map_object: no page !?");
                        vm_object_unlock(object);
                        kr = vm_map_remove(kernel_map, *address, *size,
                page = vm_page_lookup(object, offset + page_map_offset);
                if (page == VM_PAGE_NULL) {
                        printf("vm_paging_map_object: no page !?");
                        vm_object_unlock(object);
                        kr = vm_map_remove(kernel_map, *address, *size,

-                                          VM_MAP_NO_FLAGS);
+                           VM_MAP_REMOVE_NO_FLAGS);

                        assert(kr == KERN_SUCCESS);
                        *address = 0;
                        *size = 0;
                        assert(kr == KERN_SUCCESS);
                        *address = 0;
                        *size = 0;


@@ -9225,23 +9292,23 @@ vm_paging_map_object(
                        vm_object_lock(object);
                        return KERN_MEMORY_ERROR;
                }
                        vm_object_lock(object);
                        return KERN_MEMORY_ERROR;
                }

-               page->pmapped = TRUE;
+               page->vmp_pmapped = TRUE;

 
                //assert(pmap_verify_free(VM_PAGE_GET_PHYS_PAGE(page)));
                PMAP_ENTER(kernel_pmap,
 
                //assert(pmap_verify_free(VM_PAGE_GET_PHYS_PAGE(page)));
                PMAP_ENTER(kernel_pmap,

-                          *address + page_map_offset,
-                          page,
-                          protection,
-                          VM_PROT_NONE,
-                          0,
-                          TRUE,
-                          kr);
+                   *address + page_map_offset,
+                   page,
+                   protection,
+                   VM_PROT_NONE,
+                   0,
+                   TRUE,
+                   kr);

                assert(kr == KERN_SUCCESS);
 #if KASAN
                kasan_notify_address(*address + page_map_offset, PAGE_SIZE);
 #endif
        }
                assert(kr == KERN_SUCCESS);
 #if KASAN
                kasan_notify_address(*address + page_map_offset, PAGE_SIZE);
 #endif
        }

-                          
+

        vm_paging_objects_mapped_slow++;
        vm_paging_pages_mapped_slow += (unsigned long) (map_size / PAGE_SIZE_64);
 
        vm_paging_objects_mapped_slow++;
        vm_paging_pages_mapped_slow += (unsigned long) (map_size / PAGE_SIZE_64);
 


@@ -9253,24 +9320,24 @@ vm_paging_map_object(
 /*
  * vm_paging_unmap_object:
  *     Unmaps part of a VM object's pages from the kernel
 /*
  * vm_paging_unmap_object:
  *     Unmaps part of a VM object's pages from the kernel

- *     virtual address space.
+ *      virtual address space.

  * Context:
  * Context:

- *     The VM object is locked.  This lock will get
- *     dropped and re-acquired though.
+ *      The VM object is locked.  This lock will get
+ *      dropped and re-acquired though.

  */
 void
 vm_paging_unmap_object(
  */
 void
 vm_paging_unmap_object(

-       vm_object_t     object,
-       vm_map_offset_t start,
-       vm_map_offset_t end)
+       vm_object_t     object,
+       vm_map_offset_t start,
+       vm_map_offset_t end)

 {
 {

-       kern_return_t   kr;
-       int             i;
+       kern_return_t   kr;
+       int             i;

 
        if ((vm_paging_base_address == 0) ||
            (start < vm_paging_base_address) ||
            (end > (vm_paging_base_address
 
        if ((vm_paging_base_address == 0) ||
            (start < vm_paging_base_address) ||
            (end > (vm_paging_base_address

-                    + (VM_PAGING_NUM_PAGES * PAGE_SIZE)))) {
+           + (VM_PAGING_NUM_PAGES * PAGE_SIZE)))) {

                /*
                 * We didn't use our pre-allocated pool of
                 * kernel virtual address.  Deallocate the
                /*
                 * We didn't use our pre-allocated pool of
                 * kernel virtual address.  Deallocate the


@@ -9279,7 +9346,8 @@ vm_paging_unmap_object(
                if (object != VM_OBJECT_NULL) {
                        vm_object_unlock(object);
                }
                if (object != VM_OBJECT_NULL) {
                        vm_object_unlock(object);
                }

-               kr = vm_map_remove(kernel_map, start, end, VM_MAP_NO_FLAGS);
+               kr = vm_map_remove(kernel_map, start, end,
+                   VM_MAP_REMOVE_NO_FLAGS);

                if (object != VM_OBJECT_NULL) {
                        vm_object_lock(object);
                }
                if (object != VM_OBJECT_NULL) {
                        vm_object_lock(object);
                }


@@ -9297,7 +9365,7 @@ vm_paging_unmap_object(
                /* undo the pmap mapping */
                pmap_remove(kernel_pmap, start, end);
 
                /* undo the pmap mapping */
                pmap_remove(kernel_pmap, start, end);
 

-               simple_lock(&vm_paging_lock);
+               simple_lock(&vm_paging_lock, &vm_pageout_lck_grp);

                vm_paging_page_inuse[i] = FALSE;
                if (vm_paging_page_waiter) {
                        thread_wakeup(&vm_paging_page_waiter);
                vm_paging_page_inuse[i] = FALSE;
                if (vm_paging_page_waiter) {
                        thread_wakeup(&vm_paging_page_waiter);


@@ -9308,7 +9376,7 @@ vm_paging_unmap_object(
 
 
 /*
 
 
 /*

- * page->object must be locked
+ * page->vmp_object must be locked

  */
 void
 vm_pageout_steal_laundry(vm_page_t page, boolean_t queues_locked)
  */
 void
 vm_pageout_steal_laundry(vm_page_t page, boolean_t queues_locked)


@@ -9317,7 +9385,7 @@ vm_pageout_steal_laundry(vm_page_t page, boolean_t queues_locked)
                vm_page_lockspin_queues();
        }
 
                vm_page_lockspin_queues();
        }
 

-       page->free_when_done = FALSE;
+       page->vmp_free_when_done = FALSE;

        /*
         * need to drop the laundry count...
         * we may also need to remove it
        /*
         * need to drop the laundry count...
         * we may also need to remove it


@@ -9328,8 +9396,6 @@ vm_pageout_steal_laundry(vm_page_t page, boolean_t queues_locked)
         */
        vm_pageout_throttle_up(page);
 
         */
        vm_pageout_throttle_up(page);
 

-       vm_page_steal_pageout_page++;
-

        if (!queues_locked) {
                vm_page_unlock_queues();
        }
        if (!queues_locked) {
                vm_page_unlock_queues();
        }


@@ -9338,24 +9404,23 @@ vm_pageout_steal_laundry(vm_page_t page, boolean_t queues_locked)
 upl_t
 vector_upl_create(vm_offset_t upl_offset)
 {
 upl_t
 vector_upl_create(vm_offset_t upl_offset)
 {

-       int     vector_upl_size  = sizeof(struct _vector_upl);
-       int i=0;
-       upl_t   upl;
+       int     vector_upl_size  = sizeof(struct _vector_upl);
+       int i = 0;
+       upl_t   upl;

        vector_upl_t vector_upl = (vector_upl_t)kalloc(vector_upl_size);
 
        vector_upl_t vector_upl = (vector_upl_t)kalloc(vector_upl_size);
 

-       upl = upl_create(0,UPL_VECTOR,0);
+       upl = upl_create(0, UPL_VECTOR, 0);

        upl->vector_upl = vector_upl;
        upl->offset = upl_offset;
        vector_upl->size = 0;
        vector_upl->offset = upl_offset;
        upl->vector_upl = vector_upl;
        upl->offset = upl_offset;
        vector_upl->size = 0;
        vector_upl->offset = upl_offset;

-       vector_upl->invalid_upls=0;
-       vector_upl->num_upls=0;
+       vector_upl->invalid_upls = 0;
+       vector_upl->num_upls = 0;

        vector_upl->pagelist = NULL;
        vector_upl->pagelist = NULL;

-       
-       for(i=0; i < MAX_VECTOR_UPL_ELEMENTS ; i++) {
+
+       for (i = 0; i < MAX_VECTOR_UPL_ELEMENTS; i++) {

                vector_upl->upl_iostates[i].size = 0;
                vector_upl->upl_iostates[i].offset = 0;
                vector_upl->upl_iostates[i].size = 0;
                vector_upl->upl_iostates[i].offset = 0;

-               

        }
        return upl;
 }
        }
        return upl;
 }


@@ -9363,125 +9428,131 @@ vector_upl_create(vm_offset_t upl_offset)
 void
 vector_upl_deallocate(upl_t upl)
 {
 void
 vector_upl_deallocate(upl_t upl)
 {

-       if(upl) {
+       if (upl) {

                vector_upl_t vector_upl = upl->vector_upl;
                vector_upl_t vector_upl = upl->vector_upl;

-               if(vector_upl) {
-                       if(vector_upl->invalid_upls != vector_upl->num_upls)
+               if (vector_upl) {
+                       if (vector_upl->invalid_upls != vector_upl->num_upls) {

                                panic("Deallocating non-empty Vectored UPL\n");
                                panic("Deallocating non-empty Vectored UPL\n");

-                       kfree(vector_upl->pagelist,(sizeof(struct upl_page_info)*(vector_upl->size/PAGE_SIZE)));
-                       vector_upl->invalid_upls=0;
+                       }
+                       kfree(vector_upl->pagelist, (sizeof(struct upl_page_info) * (vector_upl->size / PAGE_SIZE)));
+                       vector_upl->invalid_upls = 0;

                        vector_upl->num_upls = 0;
                        vector_upl->pagelist = NULL;
                        vector_upl->size = 0;
                        vector_upl->offset = 0;
                        kfree(vector_upl, sizeof(struct _vector_upl));
                        vector_upl = (vector_upl_t)0xfeedfeed;
                        vector_upl->num_upls = 0;
                        vector_upl->pagelist = NULL;
                        vector_upl->size = 0;
                        vector_upl->offset = 0;
                        kfree(vector_upl, sizeof(struct _vector_upl));
                        vector_upl = (vector_upl_t)0xfeedfeed;

-               }
-               else
+               } else {

                        panic("vector_upl_deallocate was passed a non-vectored upl\n");
                        panic("vector_upl_deallocate was passed a non-vectored upl\n");

-       }
-       else
+               }
+       } else {

                panic("vector_upl_deallocate was passed a NULL upl\n");
                panic("vector_upl_deallocate was passed a NULL upl\n");

+       }

 }
 
 boolean_t
 vector_upl_is_valid(upl_t upl)
 {
 }
 
 boolean_t
 vector_upl_is_valid(upl_t upl)
 {

-       if(upl &&  ((upl->flags & UPL_VECTOR)==UPL_VECTOR)) {
+       if (upl && ((upl->flags & UPL_VECTOR) == UPL_VECTOR)) {

                vector_upl_t vector_upl = upl->vector_upl;
                vector_upl_t vector_upl = upl->vector_upl;

-               if(vector_upl == NULL || vector_upl == (vector_upl_t)0xfeedfeed || vector_upl == (vector_upl_t)0xfeedbeef)
+               if (vector_upl == NULL || vector_upl == (vector_upl_t)0xfeedfeed || vector_upl == (vector_upl_t)0xfeedbeef) {

                        return FALSE;
                        return FALSE;

-               else
+               } else {

                        return TRUE;
                        return TRUE;

+               }

        }
        return FALSE;
 }
 
 boolean_t
        }
        return FALSE;
 }
 
 boolean_t

-vector_upl_set_subupl(upl_t upl,upl_t subupl, uint32_t io_size)
+vector_upl_set_subupl(upl_t upl, upl_t subupl, uint32_t io_size)

 {
 {

-       if(vector_upl_is_valid(upl)) {          
+       if (vector_upl_is_valid(upl)) {

                vector_upl_t vector_upl = upl->vector_upl;
                vector_upl_t vector_upl = upl->vector_upl;

-               
-               if(vector_upl) {
-                       if(subupl) {
-                               if(io_size) {
-                                       if(io_size < PAGE_SIZE)
+
+               if (vector_upl) {
+                       if (subupl) {
+                               if (io_size) {
+                                       if (io_size < PAGE_SIZE) {

                                                io_size = PAGE_SIZE;
                                                io_size = PAGE_SIZE;

+                                       }

                                        subupl->vector_upl = (void*)vector_upl;
                                        vector_upl->upl_elems[vector_upl->num_upls++] = subupl;
                                        vector_upl->size += io_size;
                                        upl->size += io_size;
                                        subupl->vector_upl = (void*)vector_upl;
                                        vector_upl->upl_elems[vector_upl->num_upls++] = subupl;
                                        vector_upl->size += io_size;
                                        upl->size += io_size;

-                               }
-                               else {
-                                       uint32_t i=0,invalid_upls=0;
-                                       for(i = 0; i < vector_upl->num_upls; i++) {
-                                               if(vector_upl->upl_elems[i] == subupl)
+                               } else {
+                                       uint32_t i = 0, invalid_upls = 0;
+                                       for (i = 0; i < vector_upl->num_upls; i++) {
+                                               if (vector_upl->upl_elems[i] == subupl) {

                                                        break;
                                                        break;

+                                               }

                                        }
                                        }

-                                       if(i == vector_upl->num_upls)
+                                       if (i == vector_upl->num_upls) {

                                                panic("Trying to remove sub-upl when none exists");
                                                panic("Trying to remove sub-upl when none exists");

-                                       
+                                       }
+

                                        vector_upl->upl_elems[i] = NULL;
                                        vector_upl->upl_elems[i] = NULL;

-                                       invalid_upls = hw_atomic_add(&(vector_upl)->invalid_upls, 1); 
-                                       if(invalid_upls == vector_upl->num_upls)
+                                       invalid_upls = hw_atomic_add(&(vector_upl)->invalid_upls, 1);
+                                       if (invalid_upls == vector_upl->num_upls) {

                                                return TRUE;
                                                return TRUE;

-                                       else 
+                                       } else {

                                                return FALSE;
                                                return FALSE;

+                                       }

                                }
                                }

-                       }
-                       else
+                       } else {

                                panic("vector_upl_set_subupl was passed a NULL upl element\n");
                                panic("vector_upl_set_subupl was passed a NULL upl element\n");

-               }
-               else
+                       }
+               } else {

                        panic("vector_upl_set_subupl was passed a non-vectored upl\n");
                        panic("vector_upl_set_subupl was passed a non-vectored upl\n");

-       }
-       else
+               }
+       } else {

                panic("vector_upl_set_subupl was passed a NULL upl\n");
                panic("vector_upl_set_subupl was passed a NULL upl\n");

+       }

 
        return FALSE;
 
        return FALSE;

-}      
+}

 
 void
 vector_upl_set_pagelist(upl_t upl)
 {
 
 void
 vector_upl_set_pagelist(upl_t upl)
 {

-       if(vector_upl_is_valid(upl)) {          
-               uint32_t i=0;
+       if (vector_upl_is_valid(upl)) {
+               uint32_t i = 0;

                vector_upl_t vector_upl = upl->vector_upl;
 
                vector_upl_t vector_upl = upl->vector_upl;
 

-               if(vector_upl) {
-                       vm_offset_t pagelist_size=0, cur_upl_pagelist_size=0;
+               if (vector_upl) {
+                       vm_offset_t pagelist_size = 0, cur_upl_pagelist_size = 0;
+
+                       vector_upl->pagelist = (upl_page_info_array_t)kalloc(sizeof(struct upl_page_info) * (vector_upl->size / PAGE_SIZE));

 
 

-                       vector_upl->pagelist = (upl_page_info_array_t)kalloc(sizeof(struct upl_page_info)*(vector_upl->size/PAGE_SIZE));
-                       
-                       for(i=0; i < vector_upl->num_upls; i++) {
-                               cur_upl_pagelist_size = sizeof(struct upl_page_info) * vector_upl->upl_elems[i]->size/PAGE_SIZE;
+                       for (i = 0; i < vector_upl->num_upls; i++) {
+                               cur_upl_pagelist_size = sizeof(struct upl_page_info) * vector_upl->upl_elems[i]->size / PAGE_SIZE;

                                bcopy(UPL_GET_INTERNAL_PAGE_LIST_SIMPLE(vector_upl->upl_elems[i]), (char*)vector_upl->pagelist + pagelist_size, cur_upl_pagelist_size);
                                pagelist_size += cur_upl_pagelist_size;
                                bcopy(UPL_GET_INTERNAL_PAGE_LIST_SIMPLE(vector_upl->upl_elems[i]), (char*)vector_upl->pagelist + pagelist_size, cur_upl_pagelist_size);
                                pagelist_size += cur_upl_pagelist_size;

-                               if(vector_upl->upl_elems[i]->highest_page > upl->highest_page)
+                               if (vector_upl->upl_elems[i]->highest_page > upl->highest_page) {

                                        upl->highest_page = vector_upl->upl_elems[i]->highest_page;
                                        upl->highest_page = vector_upl->upl_elems[i]->highest_page;

+                               }

                        }
                        }

-                       assert( pagelist_size == (sizeof(struct upl_page_info)*(vector_upl->size/PAGE_SIZE)) );
-               }
-               else
+                       assert( pagelist_size == (sizeof(struct upl_page_info) * (vector_upl->size / PAGE_SIZE)));
+               } else {

                        panic("vector_upl_set_pagelist was passed a non-vectored upl\n");
                        panic("vector_upl_set_pagelist was passed a non-vectored upl\n");

-       }
-       else
+               }
+       } else {

                panic("vector_upl_set_pagelist was passed a NULL upl\n");
                panic("vector_upl_set_pagelist was passed a NULL upl\n");

-
+       }

 }
 
 upl_t
 vector_upl_subupl_byindex(upl_t upl, uint32_t index)
 {
 }
 
 upl_t
 vector_upl_subupl_byindex(upl_t upl, uint32_t index)
 {

-       if(vector_upl_is_valid(upl)) {          
+       if (vector_upl_is_valid(upl)) {

                vector_upl_t vector_upl = upl->vector_upl;
                vector_upl_t vector_upl = upl->vector_upl;

-               if(vector_upl) {
-                       if(index < vector_upl->num_upls)
+               if (vector_upl) {
+                       if (index < vector_upl->num_upls) {

                                return vector_upl->upl_elems[index];
                                return vector_upl->upl_elems[index];

-               }
-               else
+                       }
+               } else {

                        panic("vector_upl_subupl_byindex was passed a non-vectored upl\n");
                        panic("vector_upl_subupl_byindex was passed a non-vectored upl\n");

+               }

        }
        return NULL;
 }
        }
        return NULL;
 }


@@ -9489,39 +9560,42 @@ vector_upl_subupl_byindex(upl_t upl, uint32_t index)
 upl_t
 vector_upl_subupl_byoffset(upl_t upl, upl_offset_t *upl_offset, upl_size_t *upl_size)
 {
 upl_t
 vector_upl_subupl_byoffset(upl_t upl, upl_offset_t *upl_offset, upl_size_t *upl_size)
 {

-       if(vector_upl_is_valid(upl)) {          
-               uint32_t i=0;
+       if (vector_upl_is_valid(upl)) {
+               uint32_t i = 0;

                vector_upl_t vector_upl = upl->vector_upl;
 
                vector_upl_t vector_upl = upl->vector_upl;
 

-               if(vector_upl) {
+               if (vector_upl) {

                        upl_t subupl = NULL;
                        vector_upl_iostates_t subupl_state;
 
                        upl_t subupl = NULL;
                        vector_upl_iostates_t subupl_state;
 

-                       for(i=0; i < vector_upl->num_upls; i++) {
+                       for (i = 0; i < vector_upl->num_upls; i++) {

                                subupl = vector_upl->upl_elems[i];
                                subupl_state = vector_upl->upl_iostates[i];
                                subupl = vector_upl->upl_elems[i];
                                subupl_state = vector_upl->upl_iostates[i];

-                               if( *upl_offset <= (subupl_state.offset + subupl_state.size - 1)) {
+                               if (*upl_offset <= (subupl_state.offset + subupl_state.size - 1)) {

                                        /* We could have been passed an offset/size pair that belongs
                                         * to an UPL element that has already been committed/aborted.
                                         * If so, return NULL.
                                         */
                                        /* We could have been passed an offset/size pair that belongs
                                         * to an UPL element that has already been committed/aborted.
                                         * If so, return NULL.
                                         */

-                                       if(subupl == NULL)
+                                       if (subupl == NULL) {

                                                return NULL;
                                                return NULL;

-                                       if((subupl_state.offset + subupl_state.size) < (*upl_offset + *upl_size)) {
+                                       }
+                                       if ((subupl_state.offset + subupl_state.size) < (*upl_offset + *upl_size)) {

                                                *upl_size = (subupl_state.offset + subupl_state.size) - *upl_offset;
                                                *upl_size = (subupl_state.offset + subupl_state.size) - *upl_offset;

-                                               if(*upl_size > subupl_state.size)
+                                               if (*upl_size > subupl_state.size) {

                                                        *upl_size = subupl_state.size;
                                                        *upl_size = subupl_state.size;

+                                               }

                                        }
                                        }

-                                       if(*upl_offset >= subupl_state.offset)
+                                       if (*upl_offset >= subupl_state.offset) {

                                                *upl_offset -= subupl_state.offset;
                                                *upl_offset -= subupl_state.offset;

-                                       else if(i)
+                                       } else if (i) {

                                                panic("Vector UPL offset miscalculation\n");
                                                panic("Vector UPL offset miscalculation\n");

+                                       }

                                        return subupl;
                                        return subupl;

-                               }       
+                               }

                        }
                        }

-               }
-               else
+               } else {

                        panic("vector_upl_subupl_byoffset was passed a non-vectored UPL\n");
                        panic("vector_upl_subupl_byoffset was passed a non-vectored UPL\n");

+               }

        }
        return NULL;
 }
        }
        return NULL;
 }


@@ -9531,107 +9605,112 @@ vector_upl_get_submap(upl_t upl, vm_map_t *v_upl_submap, vm_offset_t *submap_dst
 {
        *v_upl_submap = NULL;
 
 {
        *v_upl_submap = NULL;
 

-       if(vector_upl_is_valid(upl)) {          
+       if (vector_upl_is_valid(upl)) {

                vector_upl_t vector_upl = upl->vector_upl;
                vector_upl_t vector_upl = upl->vector_upl;

-               if(vector_upl) {
+               if (vector_upl) {

                        *v_upl_submap = vector_upl->submap;
                        *submap_dst_addr = vector_upl->submap_dst_addr;
                        *v_upl_submap = vector_upl->submap;
                        *submap_dst_addr = vector_upl->submap_dst_addr;

-               }
-               else
+               } else {

                        panic("vector_upl_get_submap was passed a non-vectored UPL\n");
                        panic("vector_upl_get_submap was passed a non-vectored UPL\n");

-       }
-       else
+               }
+       } else {

                panic("vector_upl_get_submap was passed a null UPL\n");
                panic("vector_upl_get_submap was passed a null UPL\n");

+       }

 }
 
 void
 vector_upl_set_submap(upl_t upl, vm_map_t submap, vm_offset_t submap_dst_addr)
 {
 }
 
 void
 vector_upl_set_submap(upl_t upl, vm_map_t submap, vm_offset_t submap_dst_addr)
 {

-       if(vector_upl_is_valid(upl)) {          
+       if (vector_upl_is_valid(upl)) {

                vector_upl_t vector_upl = upl->vector_upl;
                vector_upl_t vector_upl = upl->vector_upl;

-               if(vector_upl) {
+               if (vector_upl) {

                        vector_upl->submap = submap;
                        vector_upl->submap_dst_addr = submap_dst_addr;
                        vector_upl->submap = submap;
                        vector_upl->submap_dst_addr = submap_dst_addr;

-               }
-               else
+               } else {

                        panic("vector_upl_get_submap was passed a non-vectored UPL\n");
                        panic("vector_upl_get_submap was passed a non-vectored UPL\n");

-       }
-       else
+               }
+       } else {

                panic("vector_upl_get_submap was passed a NULL UPL\n");
                panic("vector_upl_get_submap was passed a NULL UPL\n");

+       }

 }
 
 void
 vector_upl_set_iostate(upl_t upl, upl_t subupl, upl_offset_t offset, upl_size_t size)
 {
 }
 
 void
 vector_upl_set_iostate(upl_t upl, upl_t subupl, upl_offset_t offset, upl_size_t size)
 {

-       if(vector_upl_is_valid(upl)) {          
+       if (vector_upl_is_valid(upl)) {

                uint32_t i = 0;
                vector_upl_t vector_upl = upl->vector_upl;
 
                uint32_t i = 0;
                vector_upl_t vector_upl = upl->vector_upl;
 

-               if(vector_upl) {
-                       for(i = 0; i < vector_upl->num_upls; i++) {
-                               if(vector_upl->upl_elems[i] == subupl)
+               if (vector_upl) {
+                       for (i = 0; i < vector_upl->num_upls; i++) {
+                               if (vector_upl->upl_elems[i] == subupl) {

                                        break;
                                        break;

+                               }

                        }
                        }

-                       
-                       if(i == vector_upl->num_upls)
+
+                       if (i == vector_upl->num_upls) {

                                panic("setting sub-upl iostate when none exists");
                                panic("setting sub-upl iostate when none exists");

+                       }

 
                        vector_upl->upl_iostates[i].offset = offset;
 
                        vector_upl->upl_iostates[i].offset = offset;

-                       if(size < PAGE_SIZE)
+                       if (size < PAGE_SIZE) {

                                size = PAGE_SIZE;
                                size = PAGE_SIZE;

+                       }

                        vector_upl->upl_iostates[i].size = size;
                        vector_upl->upl_iostates[i].size = size;

-               }
-               else
+               } else {

                        panic("vector_upl_set_iostate was passed a non-vectored UPL\n");
                        panic("vector_upl_set_iostate was passed a non-vectored UPL\n");

-       }
-       else
+               }
+       } else {

                panic("vector_upl_set_iostate was passed a NULL UPL\n");
                panic("vector_upl_set_iostate was passed a NULL UPL\n");

+       }

 }
 
 void
 vector_upl_get_iostate(upl_t upl, upl_t subupl, upl_offset_t *offset, upl_size_t *size)
 {
 }
 
 void
 vector_upl_get_iostate(upl_t upl, upl_t subupl, upl_offset_t *offset, upl_size_t *size)
 {

-       if(vector_upl_is_valid(upl)) {          
+       if (vector_upl_is_valid(upl)) {

                uint32_t i = 0;
                vector_upl_t vector_upl = upl->vector_upl;
 
                uint32_t i = 0;
                vector_upl_t vector_upl = upl->vector_upl;
 

-               if(vector_upl) {
-                       for(i = 0; i < vector_upl->num_upls; i++) {
-                               if(vector_upl->upl_elems[i] == subupl)
+               if (vector_upl) {
+                       for (i = 0; i < vector_upl->num_upls; i++) {
+                               if (vector_upl->upl_elems[i] == subupl) {

                                        break;
                                        break;

+                               }

                        }
                        }

-                       
-                       if(i == vector_upl->num_upls)
+
+                       if (i == vector_upl->num_upls) {

                                panic("getting sub-upl iostate when none exists");
                                panic("getting sub-upl iostate when none exists");

+                       }

 
                        *offset = vector_upl->upl_iostates[i].offset;
                        *size = vector_upl->upl_iostates[i].size;
 
                        *offset = vector_upl->upl_iostates[i].offset;
                        *size = vector_upl->upl_iostates[i].size;

-               }
-               else
+               } else {

                        panic("vector_upl_get_iostate was passed a non-vectored UPL\n");
                        panic("vector_upl_get_iostate was passed a non-vectored UPL\n");

-       }
-       else
+               }
+       } else {

                panic("vector_upl_get_iostate was passed a NULL UPL\n");
                panic("vector_upl_get_iostate was passed a NULL UPL\n");

+       }

 }
 
 void
 vector_upl_get_iostate_byindex(upl_t upl, uint32_t index, upl_offset_t *offset, upl_size_t *size)
 {
 }
 
 void
 vector_upl_get_iostate_byindex(upl_t upl, uint32_t index, upl_offset_t *offset, upl_size_t *size)
 {

-       if(vector_upl_is_valid(upl)) {          
+       if (vector_upl_is_valid(upl)) {

                vector_upl_t vector_upl = upl->vector_upl;
                vector_upl_t vector_upl = upl->vector_upl;

-               if(vector_upl) {
-                       if(index < vector_upl->num_upls) {
+               if (vector_upl) {
+                       if (index < vector_upl->num_upls) {

                                *offset = vector_upl->upl_iostates[index].offset;
                                *size = vector_upl->upl_iostates[index].size;
                                *offset = vector_upl->upl_iostates[index].offset;
                                *size = vector_upl->upl_iostates[index].size;

-                       }
-                       else
+                       } else {

                                *offset = *size = 0;
                                *offset = *size = 0;

-               }
-               else
+                       }
+               } else {

                        panic("vector_upl_get_iostate_byindex was passed a non-vectored UPL\n");
                        panic("vector_upl_get_iostate_byindex was passed a non-vectored UPL\n");

-       }
-       else
+               }
+       } else {

                panic("vector_upl_get_iostate_byindex was passed a NULL UPL\n");
                panic("vector_upl_get_iostate_byindex was passed a NULL UPL\n");

+       }

 }
 
 upl_page_info_t *
 }
 
 upl_page_info_t *


@@ -9654,8 +9733,8 @@ upl_get_internal_pagelist_offset(void)
 
 void
 upl_clear_dirty(
 
 void
 upl_clear_dirty(

-       upl_t           upl,
-       boolean_t       value)
+       upl_t           upl,
+       boolean_t       value)

 {
        if (value) {
                upl->flags |= UPL_CLEAR_DIRTY;
 {
        if (value) {
                upl->flags |= UPL_CLEAR_DIRTY;


@@ -9666,8 +9745,8 @@ upl_clear_dirty(
 
 void
 upl_set_referenced(
 
 void
 upl_set_referenced(

-       upl_t           upl,
-       boolean_t       value)
+       upl_t           upl,
+       boolean_t       value)

 {
        upl_lock(upl);
        if (value) {
 {
        upl_lock(upl);
        if (value) {


@@ -9684,216 +9763,78 @@ upl_set_referenced(
 #if CONFIG_IOSCHED
 void
 upl_set_blkno(
 #if CONFIG_IOSCHED
 void
 upl_set_blkno(

-       upl_t           upl,
-       vm_offset_t     upl_offset,
-       int             io_size,
-       int64_t         blkno)
-{
-               int i,j;
-               if ((upl->flags & UPL_EXPEDITE_SUPPORTED) == 0)
-                       return;
-                       
-               assert(upl->upl_reprio_info != 0);      
-               for(i = (int)(upl_offset / PAGE_SIZE), j = 0; j < io_size; i++, j += PAGE_SIZE) {
-                       UPL_SET_REPRIO_INFO(upl, i, blkno, io_size);
-               }
-}
-#endif
-
-boolean_t
-vm_page_is_slideable(vm_page_t m)
-{
-       boolean_t result = FALSE;
-       vm_shared_region_slide_info_t si;
-       vm_object_t     m_object;
-
-       m_object = VM_PAGE_OBJECT(m);
-
-       vm_object_lock_assert_held(m_object);
-
-       /* make sure our page belongs to the one object allowed to do this */
-       if (!m_object->object_slid) {
-               goto done;
-       }
-
-       si = m_object->vo_slide_info;
-       if (si == NULL) {
-               goto done;
-       }
-
-       if(!m->slid && (si->start <= m->offset && si->end > m->offset)) {
-               result = TRUE;
-       }
-
-done:
-       return result;
-}
-
-int vm_page_slide_counter = 0;
-int vm_page_slide_errors = 0;
-kern_return_t
-vm_page_slide(
-       vm_page_t       page,
-       vm_map_offset_t kernel_mapping_offset)
+       upl_t           upl,
+       vm_offset_t     upl_offset,
+       int             io_size,
+       int64_t         blkno)

 {
 {

-       kern_return_t           kr;
-       vm_map_size_t           kernel_mapping_size;
-       boolean_t               kernel_mapping_needs_unmap;
-       vm_offset_t             kernel_vaddr;
-       uint32_t                pageIndex;
-       uint32_t                slide_chunk;
-       vm_object_t             page_object;
-
-       page_object = VM_PAGE_OBJECT(page);
-
-       assert(!page->slid);
-       assert(page_object->object_slid);
-       vm_object_lock_assert_exclusive(page_object);
-
-       if (page->error)
-               return KERN_FAILURE;
-       
-       /*
-        * Take a paging-in-progress reference to keep the object
-        * alive even if we have to unlock it (in vm_paging_map_object()
-        * for example)...
-        */
-       vm_object_paging_begin(page_object);
-
-       if (kernel_mapping_offset == 0) {
-               /*
-                * The page hasn't already been mapped in kernel space
-                * by the caller.  Map it now, so that we can access
-                * its contents and decrypt them.
-                */
-               kernel_mapping_size = PAGE_SIZE;
-               kernel_mapping_needs_unmap = FALSE;
-               kr = vm_paging_map_object(page,
-                                         page_object,
-                                         page->offset,
-                                         VM_PROT_READ | VM_PROT_WRITE,
-                                         FALSE,
-                                         &kernel_mapping_size,
-                                         &kernel_mapping_offset,
-                                         &kernel_mapping_needs_unmap);
-               if (kr != KERN_SUCCESS) {
-                       panic("vm_page_slide: "
-                             "could not map page in kernel: 0x%x\n",
-                             kr);
-               }
-       } else {
-               kernel_mapping_size = 0;
-               kernel_mapping_needs_unmap = FALSE;
-       }
-       kernel_vaddr = CAST_DOWN(vm_offset_t, kernel_mapping_offset);
-
-       /*
-        * Slide the pointers on the page.
-        */
-
-       /*assert that slide_file_info.start/end are page-aligned?*/
-
-       assert(!page->slid);
-       assert(page_object->object_slid);
-
-       pageIndex = (uint32_t)((page->offset -
-                               page_object->vo_slide_info->start) /
-                              PAGE_SIZE_FOR_SR_SLIDE);
-       for (slide_chunk = 0;
-            slide_chunk < PAGE_SIZE / PAGE_SIZE_FOR_SR_SLIDE;
-            slide_chunk++) {
-               kr = vm_shared_region_slide_page(page_object->vo_slide_info,
-                                                (kernel_vaddr +
-                                                 (slide_chunk *
-                                                  PAGE_SIZE_FOR_SR_SLIDE)),
-                                                (pageIndex + slide_chunk));
-               if (kr != KERN_SUCCESS) {
-                       break;
-               }
+       int i, j;
+       if ((upl->flags & UPL_EXPEDITE_SUPPORTED) == 0) {
+               return;

        }
 
        }
 

-       vm_page_slide_counter++;
-
-       /*
-        * Unmap the page from the kernel's address space,
-        */
-       if (kernel_mapping_needs_unmap) {
-               vm_paging_unmap_object(page_object,
-                                      kernel_vaddr,
-                                      kernel_vaddr + PAGE_SIZE);
-       }
-       
-       page->dirty = FALSE;
-       pmap_clear_refmod(VM_PAGE_GET_PHYS_PAGE(page), VM_MEM_MODIFIED | VM_MEM_REFERENCED);
-       
-       if (kr != KERN_SUCCESS || cs_debug > 1) {
-               printf("vm_page_slide(%p): "
-                      "obj %p off 0x%llx mobj %p moff 0x%llx\n",
-                      page,
-                      page_object, page->offset,
-                      page_object->pager,
-                      page->offset + page_object->paging_offset);
-       }
-
-       if (kr == KERN_SUCCESS) {
-               page->slid = TRUE;
-       } else {
-               page->error = TRUE;
-               vm_page_slide_errors++;
+       assert(upl->upl_reprio_info != 0);
+       for (i = (int)(upl_offset / PAGE_SIZE), j = 0; j < io_size; i++, j += PAGE_SIZE) {
+               UPL_SET_REPRIO_INFO(upl, i, blkno, io_size);

        }
        }

-
-       vm_object_paging_end(page_object);
-
-       return kr;

 }
 }

+#endif

 
 

-void inline memoryshot(unsigned int event, unsigned int control)
+void inline
+memoryshot(unsigned int event, unsigned int control)

 {
        if (vm_debug_events) {
                KERNEL_DEBUG_CONSTANT1((MACHDBG_CODE(DBG_MACH_VM_PRESSURE, event)) | control,
 {
        if (vm_debug_events) {
                KERNEL_DEBUG_CONSTANT1((MACHDBG_CODE(DBG_MACH_VM_PRESSURE, event)) | control,

-                                       vm_page_active_count, vm_page_inactive_count,
-                                       vm_page_free_count, vm_page_speculative_count,
-                                       vm_page_throttled_count);
+                   vm_page_active_count, vm_page_inactive_count,
+                   vm_page_free_count, vm_page_speculative_count,
+                   vm_page_throttled_count);

        } else {
                (void) event;
                (void) control;
        }
        } else {
                (void) event;
                (void) control;
        }

-

 }
 
 #ifdef MACH_BSD
 
 }
 
 #ifdef MACH_BSD
 

-boolean_t  upl_device_page(upl_page_info_t *upl)
+boolean_t
+upl_device_page(upl_page_info_t *upl)

 {
 {

-       return(UPL_DEVICE_PAGE(upl));
+       return UPL_DEVICE_PAGE(upl);

 }
 }

-boolean_t  upl_page_present(upl_page_info_t *upl, int index)
+boolean_t
+upl_page_present(upl_page_info_t *upl, int index)

 {
 {

-       return(UPL_PAGE_PRESENT(upl, index));
+       return UPL_PAGE_PRESENT(upl, index);

 }
 }

-boolean_t  upl_speculative_page(upl_page_info_t *upl, int index)
+boolean_t
+upl_speculative_page(upl_page_info_t *upl, int index)

 {
 {

-       return(UPL_SPECULATIVE_PAGE(upl, index));
+       return UPL_SPECULATIVE_PAGE(upl, index);

 }
 }

-boolean_t  upl_dirty_page(upl_page_info_t *upl, int index)
+boolean_t
+upl_dirty_page(upl_page_info_t *upl, int index)

 {
 {

-       return(UPL_DIRTY_PAGE(upl, index));
+       return UPL_DIRTY_PAGE(upl, index);

 }
 }

-boolean_t  upl_valid_page(upl_page_info_t *upl, int index)
+boolean_t
+upl_valid_page(upl_page_info_t *upl, int index)

 {
 {

-       return(UPL_VALID_PAGE(upl, index));
+       return UPL_VALID_PAGE(upl, index);

 }
 }

-ppnum_t  upl_phys_page(upl_page_info_t *upl, int index)
+ppnum_t
+upl_phys_page(upl_page_info_t *upl, int index)

 {
 {

-       return(UPL_PHYS_PAGE(upl, index));
+       return UPL_PHYS_PAGE(upl, index);

 }
 
 }
 

-void upl_page_set_mark(upl_page_info_t *upl, int index, boolean_t v)
+void
+upl_page_set_mark(upl_page_info_t *upl, int index, boolean_t v)

 {
        upl[index].mark = v;
 }
 
 {
        upl[index].mark = v;
 }
 

-boolean_t upl_page_get_mark(upl_page_info_t *upl, int index)
+boolean_t
+upl_page_get_mark(upl_page_info_t *upl, int index)

 {
        return upl[index].mark;
 }
 {
        return upl[index].mark;
 }


@@ -9907,138 +9848,211 @@ vm_countdirtypages(void)
        int precpages;
 
 
        int precpages;
 
 

-       dpages=0;
-       pgopages=0;
-       precpages=0;
+       dpages = 0;
+       pgopages = 0;
+       precpages = 0;

 
        vm_page_lock_queues();
        m = (vm_page_t) vm_page_queue_first(&vm_page_queue_inactive);
        do {
 
        vm_page_lock_queues();
        m = (vm_page_t) vm_page_queue_first(&vm_page_queue_inactive);
        do {

-               if (m ==(vm_page_t )0) break;
+               if (m == (vm_page_t)0) {
+                       break;
+               }

 
 

-               if(m->dirty) dpages++;
-               if(m->free_when_done) pgopages++;
-               if(m->precious) precpages++;
+               if (m->vmp_dirty) {
+                       dpages++;
+               }
+               if (m->vmp_free_when_done) {
+                       pgopages++;
+               }
+               if (m->vmp_precious) {
+                       precpages++;
+               }

 
                assert(VM_PAGE_OBJECT(m) != kernel_object);
 
                assert(VM_PAGE_OBJECT(m) != kernel_object);

-               m = (vm_page_t) vm_page_queue_next(&m->pageq);
-               if (m ==(vm_page_t )0) break;
-
+               m = (vm_page_t) vm_page_queue_next(&m->vmp_pageq);
+               if (m == (vm_page_t)0) {
+                       break;
+               }

        } while (!vm_page_queue_end(&vm_page_queue_inactive, (vm_page_queue_entry_t) m));
        vm_page_unlock_queues();
 
        vm_page_lock_queues();
        m = (vm_page_t) vm_page_queue_first(&vm_page_queue_throttled);
        do {
        } while (!vm_page_queue_end(&vm_page_queue_inactive, (vm_page_queue_entry_t) m));
        vm_page_unlock_queues();
 
        vm_page_lock_queues();
        m = (vm_page_t) vm_page_queue_first(&vm_page_queue_throttled);
        do {

-               if (m ==(vm_page_t )0) break;
+               if (m == (vm_page_t)0) {
+                       break;
+               }

 
                dpages++;
 
                dpages++;

-               assert(m->dirty);
-               assert(!m->free_when_done);
+               assert(m->vmp_dirty);
+               assert(!m->vmp_free_when_done);

                assert(VM_PAGE_OBJECT(m) != kernel_object);
                assert(VM_PAGE_OBJECT(m) != kernel_object);

-               m = (vm_page_t) vm_page_queue_next(&m->pageq);
-               if (m ==(vm_page_t )0) break;
-
+               m = (vm_page_t) vm_page_queue_next(&m->vmp_pageq);
+               if (m == (vm_page_t)0) {
+                       break;
+               }

        } while (!vm_page_queue_end(&vm_page_queue_throttled, (vm_page_queue_entry_t) m));
        vm_page_unlock_queues();
 
        vm_page_lock_queues();
        m = (vm_page_t) vm_page_queue_first(&vm_page_queue_anonymous);
        do {
        } while (!vm_page_queue_end(&vm_page_queue_throttled, (vm_page_queue_entry_t) m));
        vm_page_unlock_queues();
 
        vm_page_lock_queues();
        m = (vm_page_t) vm_page_queue_first(&vm_page_queue_anonymous);
        do {

-               if (m ==(vm_page_t )0) break;
+               if (m == (vm_page_t)0) {
+                       break;
+               }

 
 

-               if(m->dirty) dpages++;
-               if(m->free_when_done) pgopages++;
-               if(m->precious) precpages++;
+               if (m->vmp_dirty) {
+                       dpages++;
+               }
+               if (m->vmp_free_when_done) {
+                       pgopages++;
+               }
+               if (m->vmp_precious) {
+                       precpages++;
+               }

 
                assert(VM_PAGE_OBJECT(m) != kernel_object);
 
                assert(VM_PAGE_OBJECT(m) != kernel_object);

-               m = (vm_page_t) vm_page_queue_next(&m->pageq);
-               if (m ==(vm_page_t )0) break;
-
+               m = (vm_page_t) vm_page_queue_next(&m->vmp_pageq);
+               if (m == (vm_page_t)0) {
+                       break;
+               }

        } while (!vm_page_queue_end(&vm_page_queue_anonymous, (vm_page_queue_entry_t) m));
        vm_page_unlock_queues();
 
        printf("IN Q: %d : %d : %d\n", dpages, pgopages, precpages);
 
        } while (!vm_page_queue_end(&vm_page_queue_anonymous, (vm_page_queue_entry_t) m));
        vm_page_unlock_queues();
 
        printf("IN Q: %d : %d : %d\n", dpages, pgopages, precpages);
 

-       dpages=0;
-       pgopages=0;
-       precpages=0;
+       dpages = 0;
+       pgopages = 0;
+       precpages = 0;

 
        vm_page_lock_queues();
        m = (vm_page_t) vm_page_queue_first(&vm_page_queue_active);
 
        do {
 
        vm_page_lock_queues();
        m = (vm_page_t) vm_page_queue_first(&vm_page_queue_active);
 
        do {

-               if(m == (vm_page_t )0) break;
-               if(m->dirty) dpages++;
-               if(m->free_when_done) pgopages++;
-               if(m->precious) precpages++;
+               if (m == (vm_page_t)0) {
+                       break;
+               }
+               if (m->vmp_dirty) {
+                       dpages++;
+               }
+               if (m->vmp_free_when_done) {
+                       pgopages++;
+               }
+               if (m->vmp_precious) {
+                       precpages++;
+               }

 
                assert(VM_PAGE_OBJECT(m) != kernel_object);
 
                assert(VM_PAGE_OBJECT(m) != kernel_object);

-               m = (vm_page_t) vm_page_queue_next(&m->pageq);
-               if(m == (vm_page_t )0) break;
-
+               m = (vm_page_t) vm_page_queue_next(&m->vmp_pageq);
+               if (m == (vm_page_t)0) {
+                       break;
+               }

        } while (!vm_page_queue_end(&vm_page_queue_active, (vm_page_queue_entry_t) m));
        vm_page_unlock_queues();
 
        printf("AC Q: %d : %d : %d\n", dpages, pgopages, precpages);
        } while (!vm_page_queue_end(&vm_page_queue_active, (vm_page_queue_entry_t) m));
        vm_page_unlock_queues();
 
        printf("AC Q: %d : %d : %d\n", dpages, pgopages, precpages);

-

 }
 #endif /* MACH_BSD */
 
 
 #if CONFIG_IOSCHED
 }
 #endif /* MACH_BSD */
 
 
 #if CONFIG_IOSCHED

-int upl_get_cached_tier(upl_t  upl)
+int
+upl_get_cached_tier(upl_t  upl)
+{
+       assert(upl);
+       if (upl->flags & UPL_TRACKED_BY_OBJECT) {
+               return upl->upl_priority;
+       }
+       return -1;
+}
+#endif /* CONFIG_IOSCHED */
+
+
+void
+upl_callout_iodone(upl_t upl)
+{
+       struct upl_io_completion *upl_ctx = upl->upl_iodone;
+
+       if (upl_ctx) {
+               void    (*iodone_func)(void *, int) = upl_ctx->io_done;
+
+               assert(upl_ctx->io_done);
+
+               (*iodone_func)(upl_ctx->io_context, upl_ctx->io_error);
+       }
+}
+
+void
+upl_set_iodone(upl_t upl, void *upl_iodone)
+{
+       upl->upl_iodone = (struct upl_io_completion *)upl_iodone;
+}
+
+void
+upl_set_iodone_error(upl_t upl, int error)

 {
 {

-       assert(upl);
-       if (upl->flags & UPL_TRACKED_BY_OBJECT)
-               return (upl->upl_priority);
-       return (-1);
+       struct upl_io_completion *upl_ctx = upl->upl_iodone;
+
+       if (upl_ctx) {
+               upl_ctx->io_error = error;
+       }

 }
 }

-#endif /* CONFIG_IOSCHED */      

 
 

-ppnum_t upl_get_highest_page(
-                            upl_t                      upl)
+
+ppnum_t
+upl_get_highest_page(
+       upl_t                      upl)

 {
 {

-        return upl->highest_page;
+       return upl->highest_page;

 }
 
 }
 

-upl_size_t upl_get_size(
-                            upl_t                      upl)
+upl_size_t
+upl_get_size(
+       upl_t                      upl)

 {
 {

-        return upl->size;
+       return upl->size;

 }
 
 }
 

-upl_t upl_associated_upl(upl_t upl)
+upl_t
+upl_associated_upl(upl_t upl)

 {
        return upl->associated_upl;
 }
 
 {
        return upl->associated_upl;
 }
 

-void upl_set_associated_upl(upl_t upl, upl_t associated_upl)
+void
+upl_set_associated_upl(upl_t upl, upl_t associated_upl)

 {
        upl->associated_upl = associated_upl;
 }
 
 {
        upl->associated_upl = associated_upl;
 }
 

-struct vnode * upl_lookup_vnode(upl_t upl)
+struct vnode *
+upl_lookup_vnode(upl_t upl)

 {
 {

-       if (!upl->map_object->internal)
+       if (!upl->map_object->internal) {

                return vnode_pager_lookup_vnode(upl->map_object->pager);
                return vnode_pager_lookup_vnode(upl->map_object->pager);

-       else
+       } else {

                return NULL;
                return NULL;

-}      
+       }
+}

 
 #if UPL_DEBUG
 
 #if UPL_DEBUG

-kern_return_t  upl_ubc_alias_set(upl_t upl, uintptr_t alias1, uintptr_t alias2)
+kern_return_t
+upl_ubc_alias_set(upl_t upl, uintptr_t alias1, uintptr_t alias2)

 {
        upl->ubc_alias1 = alias1;
        upl->ubc_alias2 = alias2;
        return KERN_SUCCESS;
 }
 {
        upl->ubc_alias1 = alias1;
        upl->ubc_alias2 = alias2;
        return KERN_SUCCESS;
 }

-int  upl_ubc_alias_get(upl_t upl, uintptr_t * al, uintptr_t * al2)
+int
+upl_ubc_alias_get(upl_t upl, uintptr_t * al, uintptr_t * al2)

 {
 {

-       if(al)
+       if (al) {

                *al = upl->ubc_alias1;
                *al = upl->ubc_alias1;

-       if(al2)
+       }
+       if (al2) {

                *al2 = upl->ubc_alias2;
                *al2 = upl->ubc_alias2;

+       }

        return KERN_SUCCESS;
 }
 #endif /* UPL_DEBUG */
        return KERN_SUCCESS;
 }
 #endif /* UPL_DEBUG */


@@ -10050,10 +10064,9 @@ int  upl_ubc_alias_get(upl_t upl, uintptr_t * al, uintptr_t * al2)
 extern boolean_t vm_compressor_low_on_space(void);
 
 boolean_t
 extern boolean_t vm_compressor_low_on_space(void);
 
 boolean_t

-VM_PRESSURE_NORMAL_TO_WARNING(void)    {
-
-       if ( !VM_CONFIG_COMPRESSOR_IS_ACTIVE) {
-
+VM_PRESSURE_NORMAL_TO_WARNING(void)
+{
+       if (!VM_CONFIG_COMPRESSOR_IS_ACTIVE) {

                /* Available pages below our threshold */
                if (memorystatus_available_pages < memorystatus_available_pages_pressure) {
                        /* No frozen processes to kill */
                /* Available pages below our threshold */
                if (memorystatus_available_pages < memorystatus_available_pages_pressure) {
                        /* No frozen processes to kill */


@@ -10065,24 +10078,22 @@ VM_PRESSURE_NORMAL_TO_WARNING(void)   {
                        }
                }
                return FALSE;
                        }
                }
                return FALSE;

-

        } else {
        } else {

-               return ((AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD) ? 1 : 0);
+               return (AVAILABLE_NON_COMPRESSED_MEMORY < VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD) ? 1 : 0;

        }
 }
 
 boolean_t
        }
 }
 
 boolean_t

-VM_PRESSURE_WARNING_TO_CRITICAL(void) {
-
-       if ( !VM_CONFIG_COMPRESSOR_IS_ACTIVE) {
-
+VM_PRESSURE_WARNING_TO_CRITICAL(void)
+{
+       if (!VM_CONFIG_COMPRESSOR_IS_ACTIVE) {

                /* Available pages below our threshold */
                if (memorystatus_available_pages < memorystatus_available_pages_critical) {
                        return TRUE;
                }
                return FALSE;
        } else {
                /* Available pages below our threshold */
                if (memorystatus_available_pages < memorystatus_available_pages_critical) {
                        return TRUE;
                }
                return FALSE;
        } else {

-               return (vm_compressor_low_on_space() || (AVAILABLE_NON_COMPRESSED_MEMORY < ((12 * VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) / 10)) ? 1 : 0);
+               return vm_compressor_low_on_space() || (AVAILABLE_NON_COMPRESSED_MEMORY < ((12 * VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) / 10)) ? 1 : 0;

        }
 }
 
        }
 }
 


@@ -10090,10 +10101,9 @@ VM_PRESSURE_WARNING_TO_CRITICAL(void) {
  * Downward trajectory.
  */
 boolean_t
  * Downward trajectory.
  */
 boolean_t

-VM_PRESSURE_WARNING_TO_NORMAL(void) {
-
-       if ( !VM_CONFIG_COMPRESSOR_IS_ACTIVE) {
-
+VM_PRESSURE_WARNING_TO_NORMAL(void)
+{
+       if (!VM_CONFIG_COMPRESSOR_IS_ACTIVE) {

                /* Available pages above our threshold */
                unsigned int target_threshold = (unsigned int) (memorystatus_available_pages_pressure + ((15 * memorystatus_available_pages_pressure) / 100));
                if (memorystatus_available_pages > target_threshold) {
                /* Available pages above our threshold */
                unsigned int target_threshold = (unsigned int) (memorystatus_available_pages_pressure + ((15 * memorystatus_available_pages_pressure) / 100));
                if (memorystatus_available_pages > target_threshold) {


@@ -10101,15 +10111,14 @@ VM_PRESSURE_WARNING_TO_NORMAL(void) {
                }
                return FALSE;
        } else {
                }
                return FALSE;
        } else {

-               return ((AVAILABLE_NON_COMPRESSED_MEMORY > ((12 * VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD) / 10)) ? 1 : 0);
+               return (AVAILABLE_NON_COMPRESSED_MEMORY > ((12 * VM_PAGE_COMPRESSOR_COMPACT_THRESHOLD) / 10)) ? 1 : 0;

        }
 }
 
 boolean_t
        }
 }
 
 boolean_t

-VM_PRESSURE_CRITICAL_TO_WARNING(void) {
-
-       if ( !VM_CONFIG_COMPRESSOR_IS_ACTIVE) {
-
+VM_PRESSURE_CRITICAL_TO_WARNING(void)
+{
+       if (!VM_CONFIG_COMPRESSOR_IS_ACTIVE) {

                /* Available pages above our threshold */
                unsigned int target_threshold = (unsigned int)(memorystatus_available_pages_critical + ((15 * memorystatus_available_pages_critical) / 100));
                if (memorystatus_available_pages > target_threshold) {
                /* Available pages above our threshold */
                unsigned int target_threshold = (unsigned int)(memorystatus_available_pages_critical + ((15 * memorystatus_available_pages_critical) / 100));
                if (memorystatus_available_pages > target_threshold) {


@@ -10117,20 +10126,20 @@ VM_PRESSURE_CRITICAL_TO_WARNING(void) {
                }
                return FALSE;
        } else {
                }
                return FALSE;
        } else {

-               return ((AVAILABLE_NON_COMPRESSED_MEMORY > ((14 * VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) / 10)) ? 1 : 0);
+               return (AVAILABLE_NON_COMPRESSED_MEMORY > ((14 * VM_PAGE_COMPRESSOR_SWAP_UNTHROTTLE_THRESHOLD) / 10)) ? 1 : 0;

        }
 }
 #endif /* VM_PRESSURE_EVENTS */
 
 
 
        }
 }
 #endif /* VM_PRESSURE_EVENTS */
 
 
 

-#define VM_TEST_COLLAPSE_COMPRESSOR            0
-#define VM_TEST_WIRE_AND_EXTRACT               0
-#define VM_TEST_PAGE_WIRE_OVERFLOW_PANIC       0
+#define VM_TEST_COLLAPSE_COMPRESSOR             0
+#define VM_TEST_WIRE_AND_EXTRACT                0
+#define VM_TEST_PAGE_WIRE_OVERFLOW_PANIC        0

 #if __arm64__
 #if __arm64__

-#define VM_TEST_KERNEL_OBJECT_FAULT            0
+#define VM_TEST_KERNEL_OBJECT_FAULT             0

 #endif /* __arm64__ */
 #endif /* __arm64__ */

-#define VM_TEST_DEVICE_PAGER_TRANSPOSE         (DEVELOPMENT || DEBUG)
+#define VM_TEST_DEVICE_PAGER_TRANSPOSE          (DEVELOPMENT || DEBUG)

 
 #if VM_TEST_COLLAPSE_COMPRESSOR
 extern boolean_t vm_object_collapse_compressor_allowed;
 
 #if VM_TEST_COLLAPSE_COMPRESSOR
 extern boolean_t vm_object_collapse_compressor_allowed;


@@ -10138,11 +10147,11 @@ extern boolean_t vm_object_collapse_compressor_allowed;
 static void
 vm_test_collapse_compressor(void)
 {
 static void
 vm_test_collapse_compressor(void)
 {

-       vm_object_size_t        backing_size, top_size;
-       vm_object_t             backing_object, top_object;
-       vm_map_offset_t         backing_offset, top_offset;
-       unsigned char           *backing_address, *top_address;
-       kern_return_t           kr;
+       vm_object_size_t        backing_size, top_size;
+       vm_object_t             backing_object, top_object;
+       vm_map_offset_t         backing_offset, top_offset;
+       unsigned char           *backing_address, *top_address;
+       kern_return_t           kr;

 
        printf("VM_TEST_COLLAPSE_COMPRESSOR:\n");
 
 
        printf("VM_TEST_COLLAPSE_COMPRESSOR:\n");
 


@@ -10151,45 +10160,46 @@ vm_test_collapse_compressor(void)
        backing_object = vm_object_allocate(backing_size);
        assert(backing_object != VM_OBJECT_NULL);
        printf("VM_TEST_COLLAPSE_COMPRESSOR: created backing object %p\n",
        backing_object = vm_object_allocate(backing_size);
        assert(backing_object != VM_OBJECT_NULL);
        printf("VM_TEST_COLLAPSE_COMPRESSOR: created backing object %p\n",

-               backing_object);
+           backing_object);

        /* map backing object */
        backing_offset = 0;
        kr = vm_map_enter(kernel_map, &backing_offset, backing_size, 0,
        /* map backing object */
        backing_offset = 0;
        kr = vm_map_enter(kernel_map, &backing_offset, backing_size, 0,

-                         VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE,
-                         backing_object, 0, FALSE,
-                         VM_PROT_DEFAULT, VM_PROT_DEFAULT, VM_INHERIT_DEFAULT);
+           VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE,
+           backing_object, 0, FALSE,
+           VM_PROT_DEFAULT, VM_PROT_DEFAULT, VM_INHERIT_DEFAULT);

        assert(kr == KERN_SUCCESS);
        backing_address = (unsigned char *) backing_offset;
        printf("VM_TEST_COLLAPSE_COMPRESSOR: "
        assert(kr == KERN_SUCCESS);
        backing_address = (unsigned char *) backing_offset;
        printf("VM_TEST_COLLAPSE_COMPRESSOR: "

-              "mapped backing object %p at 0x%llx\n",
-              backing_object, (uint64_t) backing_offset);
+           "mapped backing object %p at 0x%llx\n",
+           backing_object, (uint64_t) backing_offset);

        /* populate with pages to be compressed in backing object */
        /* populate with pages to be compressed in backing object */

-       backing_address[0x1*PAGE_SIZE] = 0xB1;
-       backing_address[0x4*PAGE_SIZE] = 0xB4;
-       backing_address[0x7*PAGE_SIZE] = 0xB7;
-       backing_address[0xa*PAGE_SIZE] = 0xBA;
-       backing_address[0xd*PAGE_SIZE] = 0xBD;
+       backing_address[0x1 * PAGE_SIZE] = 0xB1;
+       backing_address[0x4 * PAGE_SIZE] = 0xB4;
+       backing_address[0x7 * PAGE_SIZE] = 0xB7;
+       backing_address[0xa * PAGE_SIZE] = 0xBA;
+       backing_address[0xd * PAGE_SIZE] = 0xBD;

        printf("VM_TEST_COLLAPSE_COMPRESSOR: "
        printf("VM_TEST_COLLAPSE_COMPRESSOR: "

-              "populated pages to be compressed in "
-              "backing_object %p\n", backing_object);
+           "populated pages to be compressed in "
+           "backing_object %p\n", backing_object);

        /* compress backing object */
        vm_object_pageout(backing_object);
        printf("VM_TEST_COLLAPSE_COMPRESSOR: compressing backing_object %p\n",
        /* compress backing object */
        vm_object_pageout(backing_object);
        printf("VM_TEST_COLLAPSE_COMPRESSOR: compressing backing_object %p\n",

-              backing_object);
+           backing_object);

        /* wait for all the pages to be gone */
        /* wait for all the pages to be gone */

-       while (*(volatile int *)&backing_object->resident_page_count != 0)
+       while (*(volatile int *)&backing_object->resident_page_count != 0) {

                IODelay(10);
                IODelay(10);

+       }

        printf("VM_TEST_COLLAPSE_COMPRESSOR: backing_object %p compressed\n",
        printf("VM_TEST_COLLAPSE_COMPRESSOR: backing_object %p compressed\n",

-              backing_object);
+           backing_object);

        /* populate with pages to be resident in backing object */
        /* populate with pages to be resident in backing object */

-       backing_address[0x0*PAGE_SIZE] = 0xB0;
-       backing_address[0x3*PAGE_SIZE] = 0xB3;
-       backing_address[0x6*PAGE_SIZE] = 0xB6;
-       backing_address[0x9*PAGE_SIZE] = 0xB9;
-       backing_address[0xc*PAGE_SIZE] = 0xBC;
+       backing_address[0x0 * PAGE_SIZE] = 0xB0;
+       backing_address[0x3 * PAGE_SIZE] = 0xB3;
+       backing_address[0x6 * PAGE_SIZE] = 0xB6;
+       backing_address[0x9 * PAGE_SIZE] = 0xB9;
+       backing_address[0xc * PAGE_SIZE] = 0xBC;

        printf("VM_TEST_COLLAPSE_COMPRESSOR: "
        printf("VM_TEST_COLLAPSE_COMPRESSOR: "

-              "populated pages to be resident in "
-              "backing_object %p\n", backing_object);
+           "populated pages to be resident in "
+           "backing_object %p\n", backing_object);

        /* leave the other pages absent */
        /* mess with the paging_offset of the backing_object */
        assert(backing_object->paging_offset == 0);
        /* leave the other pages absent */
        /* mess with the paging_offset of the backing_object */
        assert(backing_object->paging_offset == 0);


@@ -10200,41 +10210,42 @@ vm_test_collapse_compressor(void)
        top_object = vm_object_allocate(top_size);
        assert(top_object != VM_OBJECT_NULL);
        printf("VM_TEST_COLLAPSE_COMPRESSOR: created top object %p\n",
        top_object = vm_object_allocate(top_size);
        assert(top_object != VM_OBJECT_NULL);
        printf("VM_TEST_COLLAPSE_COMPRESSOR: created top object %p\n",

-               top_object);
+           top_object);

        /* map top object */
        top_offset = 0;
        kr = vm_map_enter(kernel_map, &top_offset, top_size, 0,
        /* map top object */
        top_offset = 0;
        kr = vm_map_enter(kernel_map, &top_offset, top_size, 0,

-                         VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE,
-                         top_object, 0, FALSE,
-                         VM_PROT_DEFAULT, VM_PROT_DEFAULT, VM_INHERIT_DEFAULT);
+           VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE,
+           top_object, 0, FALSE,
+           VM_PROT_DEFAULT, VM_PROT_DEFAULT, VM_INHERIT_DEFAULT);

        assert(kr == KERN_SUCCESS);
        top_address = (unsigned char *) top_offset;
        printf("VM_TEST_COLLAPSE_COMPRESSOR: "
        assert(kr == KERN_SUCCESS);
        top_address = (unsigned char *) top_offset;
        printf("VM_TEST_COLLAPSE_COMPRESSOR: "

-              "mapped top object %p at 0x%llx\n",
-              top_object, (uint64_t) top_offset);
+           "mapped top object %p at 0x%llx\n",
+           top_object, (uint64_t) top_offset);

        /* populate with pages to be compressed in top object */
        /* populate with pages to be compressed in top object */

-       top_address[0x3*PAGE_SIZE] = 0xA3;
-       top_address[0x4*PAGE_SIZE] = 0xA4;
-       top_address[0x5*PAGE_SIZE] = 0xA5;
+       top_address[0x3 * PAGE_SIZE] = 0xA3;
+       top_address[0x4 * PAGE_SIZE] = 0xA4;
+       top_address[0x5 * PAGE_SIZE] = 0xA5;

        printf("VM_TEST_COLLAPSE_COMPRESSOR: "
        printf("VM_TEST_COLLAPSE_COMPRESSOR: "

-              "populated pages to be compressed in "
-              "top_object %p\n", top_object);
+           "populated pages to be compressed in "
+           "top_object %p\n", top_object);

        /* compress top object */
        vm_object_pageout(top_object);
        printf("VM_TEST_COLLAPSE_COMPRESSOR: compressing top_object %p\n",
        /* compress top object */
        vm_object_pageout(top_object);
        printf("VM_TEST_COLLAPSE_COMPRESSOR: compressing top_object %p\n",

-              top_object);
+           top_object);

        /* wait for all the pages to be gone */
        /* wait for all the pages to be gone */

-       while (top_object->resident_page_count != 0)
+       while (top_object->resident_page_count != 0) {

                IODelay(10);
                IODelay(10);

+       }

        printf("VM_TEST_COLLAPSE_COMPRESSOR: top_object %p compressed\n",
        printf("VM_TEST_COLLAPSE_COMPRESSOR: top_object %p compressed\n",

-              top_object);
+           top_object);

        /* populate with pages to be resident in top object */
        /* populate with pages to be resident in top object */

-       top_address[0x0*PAGE_SIZE] = 0xA0;
-       top_address[0x1*PAGE_SIZE] = 0xA1;
-       top_address[0x2*PAGE_SIZE] = 0xA2;
+       top_address[0x0 * PAGE_SIZE] = 0xA0;
+       top_address[0x1 * PAGE_SIZE] = 0xA1;
+       top_address[0x2 * PAGE_SIZE] = 0xA2;

        printf("VM_TEST_COLLAPSE_COMPRESSOR: "
        printf("VM_TEST_COLLAPSE_COMPRESSOR: "

-              "populated pages to be resident in "
-              "top_object %p\n", top_object);
+           "populated pages to be resident in "
+           "top_object %p\n", top_object);

        /* leave the other pages absent */
 
        /* link the 2 objects */
        /* leave the other pages absent */
 
        /* link the 2 objects */


@@ -10242,18 +10253,18 @@ vm_test_collapse_compressor(void)
        top_object->shadow = backing_object;
        top_object->vo_shadow_offset = 0x3000;
        printf("VM_TEST_COLLAPSE_COMPRESSOR: linked %p and %p\n",
        top_object->shadow = backing_object;
        top_object->vo_shadow_offset = 0x3000;
        printf("VM_TEST_COLLAPSE_COMPRESSOR: linked %p and %p\n",

-              top_object, backing_object);
+           top_object, backing_object);

 
        /* unmap backing object */
        vm_map_remove(kernel_map,
 
        /* unmap backing object */
        vm_map_remove(kernel_map,

-                     backing_offset,
-                     backing_offset + backing_size,
-                     0);
+           backing_offset,
+           backing_offset + backing_size,
+           VM_MAP_REMOVE_NO_FLAGS);

        printf("VM_TEST_COLLAPSE_COMPRESSOR: "
        printf("VM_TEST_COLLAPSE_COMPRESSOR: "

-              "unmapped backing_object %p [0x%llx:0x%llx]\n",
-              backing_object,
-              (uint64_t) backing_offset,
-              (uint64_t) (backing_offset + backing_size));
+           "unmapped backing_object %p [0x%llx:0x%llx]\n",
+           backing_object,
+           (uint64_t) backing_offset,
+           (uint64_t) (backing_offset + backing_size));

 
        /* collapse */
        printf("VM_TEST_COLLAPSE_COMPRESSOR: collapsing %p\n", top_object);
 
        /* collapse */
        printf("VM_TEST_COLLAPSE_COMPRESSOR: collapsing %p\n", top_object);


@@ -10272,32 +10283,32 @@ vm_test_collapse_compressor(void)
        } else {
                /* check the contents of the mapping */
                unsigned char expect[9] =
        } else {
                /* check the contents of the mapping */
                unsigned char expect[9] =

-                       { 0xA0, 0xA1, 0xA2,     /* resident in top */
-                         0xA3, 0xA4, 0xA5,     /* compressed in top */
-                         0xB9, /* resident in backing + shadow_offset */
-                         0xBD, /* compressed in backing + shadow_offset + paging_offset */
-                         0x00 };               /* absent in both */
+               { 0xA0, 0xA1, 0xA2,             /* resident in top */
+                 0xA3, 0xA4, 0xA5,             /* compressed in top */
+                 0xB9,         /* resident in backing + shadow_offset */
+                 0xBD,         /* compressed in backing + shadow_offset + paging_offset */
+                 0x00 };                       /* absent in both */

                unsigned char actual[9];
                unsigned int i, errors;
 
                errors = 0;
                unsigned char actual[9];
                unsigned int i, errors;
 
                errors = 0;

-               for (i = 0; i < sizeof (actual); i++) {
-                       actual[i] = (unsigned char) top_address[i*PAGE_SIZE];
+               for (i = 0; i < sizeof(actual); i++) {
+                       actual[i] = (unsigned char) top_address[i * PAGE_SIZE];

                        if (actual[i] != expect[i]) {
                                errors++;
                        }
                }
                printf("VM_TEST_COLLAPSE_COMPRESSOR: "
                        if (actual[i] != expect[i]) {
                                errors++;
                        }
                }
                printf("VM_TEST_COLLAPSE_COMPRESSOR: "

-                      "actual [%x %x %x %x %x %x %x %x %x] "
-                      "expect [%x %x %x %x %x %x %x %x %x] "
-                      "%d errors\n",
-                      actual[0], actual[1], actual[2], actual[3],
-                      actual[4], actual[5], actual[6], actual[7],
-                      actual[8],
-                      expect[0], expect[1], expect[2], expect[3],
-                      expect[4], expect[5], expect[6], expect[7],
-                      expect[8],
-                      errors);
+                   "actual [%x %x %x %x %x %x %x %x %x] "
+                   "expect [%x %x %x %x %x %x %x %x %x] "
+                   "%d errors\n",
+                   actual[0], actual[1], actual[2], actual[3],
+                   actual[4], actual[5], actual[6], actual[7],
+                   actual[8],
+                   expect[0], expect[1], expect[2], expect[3],
+                   expect[4], expect[5], expect[6], expect[7],
+                   expect[8],
+                   errors);

                if (errors) {
                        panic("VM_TEST_COLLAPSE_COMPRESSOR: FAIL\n");
                } else {
                if (errors) {
                        panic("VM_TEST_COLLAPSE_COMPRESSOR: FAIL\n");
                } else {


@@ -10310,69 +10321,69 @@ vm_test_collapse_compressor(void)
 #endif /* VM_TEST_COLLAPSE_COMPRESSOR */
 
 #if VM_TEST_WIRE_AND_EXTRACT
 #endif /* VM_TEST_COLLAPSE_COMPRESSOR */
 
 #if VM_TEST_WIRE_AND_EXTRACT

-extern ledger_template_t       task_ledger_template;
+extern ledger_template_t        task_ledger_template;

 #include <mach/mach_vm.h>
 extern ppnum_t vm_map_get_phys_page(vm_map_t map,
 #include <mach/mach_vm.h>
 extern ppnum_t vm_map_get_phys_page(vm_map_t map,

-                                   vm_offset_t offset);
+    vm_offset_t offset);

 static void
 vm_test_wire_and_extract(void)
 {
 static void
 vm_test_wire_and_extract(void)
 {

-       ledger_t                ledger;
-       vm_map_t                user_map, wire_map;
-       mach_vm_address_t       user_addr, wire_addr;
-       mach_vm_size_t          user_size, wire_size;
-       mach_vm_offset_t        cur_offset;
-       vm_prot_t               cur_prot, max_prot;
-       ppnum_t                 user_ppnum, wire_ppnum;
-       kern_return_t           kr;
+       ledger_t                ledger;
+       vm_map_t                user_map, wire_map;
+       mach_vm_address_t       user_addr, wire_addr;
+       mach_vm_size_t          user_size, wire_size;
+       mach_vm_offset_t        cur_offset;
+       vm_prot_t               cur_prot, max_prot;
+       ppnum_t                 user_ppnum, wire_ppnum;
+       kern_return_t           kr;

 
        ledger = ledger_instantiate(task_ledger_template,
 
        ledger = ledger_instantiate(task_ledger_template,

-                                   LEDGER_CREATE_ACTIVE_ENTRIES);
+           LEDGER_CREATE_ACTIVE_ENTRIES);

        user_map = vm_map_create(pmap_create(ledger, 0, PMAP_CREATE_64BIT),
        user_map = vm_map_create(pmap_create(ledger, 0, PMAP_CREATE_64BIT),

-                                0x100000000ULL,
-                                0x200000000ULL,
-                                TRUE);
+           0x100000000ULL,
+           0x200000000ULL,
+           TRUE);

        wire_map = vm_map_create(NULL,
        wire_map = vm_map_create(NULL,

-                                0x100000000ULL,
-                                0x200000000ULL,
-                                TRUE);
+           0x100000000ULL,
+           0x200000000ULL,
+           TRUE);

        user_addr = 0;
        user_size = 0x10000;
        kr = mach_vm_allocate(user_map,
        user_addr = 0;
        user_size = 0x10000;
        kr = mach_vm_allocate(user_map,

-                             &user_addr,
-                             user_size,
-                             VM_FLAGS_ANYWHERE);
+           &user_addr,
+           user_size,
+           VM_FLAGS_ANYWHERE);

        assert(kr == KERN_SUCCESS);
        wire_addr = 0;
        wire_size = user_size;
        kr = mach_vm_remap(wire_map,
        assert(kr == KERN_SUCCESS);
        wire_addr = 0;
        wire_size = user_size;
        kr = mach_vm_remap(wire_map,

-                          &wire_addr,
-                          wire_size,
-                          0,
-                          VM_FLAGS_ANYWHERE,
-                          user_map,
-                          user_addr,
-                          FALSE,
-                          &cur_prot,
-                          &max_prot,
-                          VM_INHERIT_NONE);
+           &wire_addr,
+           wire_size,
+           0,
+           VM_FLAGS_ANYWHERE,
+           user_map,
+           user_addr,
+           FALSE,
+           &cur_prot,
+           &max_prot,
+           VM_INHERIT_NONE);

        assert(kr == KERN_SUCCESS);
        for (cur_offset = 0;
        assert(kr == KERN_SUCCESS);
        for (cur_offset = 0;

-            cur_offset < wire_size;
-            cur_offset += PAGE_SIZE) {
+           cur_offset < wire_size;
+           cur_offset += PAGE_SIZE) {

                kr = vm_map_wire_and_extract(wire_map,
                kr = vm_map_wire_and_extract(wire_map,

-                                            wire_addr + cur_offset,
-                                            VM_PROT_DEFAULT | VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_OSFMK),
-                                            TRUE,
-                                            &wire_ppnum);
+                   wire_addr + cur_offset,
+                   VM_PROT_DEFAULT | VM_PROT_MEMORY_TAG_MAKE(VM_KERN_MEMORY_OSFMK),
+                   TRUE,
+                   &wire_ppnum);

                assert(kr == KERN_SUCCESS);
                user_ppnum = vm_map_get_phys_page(user_map,
                assert(kr == KERN_SUCCESS);
                user_ppnum = vm_map_get_phys_page(user_map,

-                                                 user_addr + cur_offset);
+                   user_addr + cur_offset);

                printf("VM_TEST_WIRE_AND_EXTRACT: kr=0x%x "
                printf("VM_TEST_WIRE_AND_EXTRACT: kr=0x%x "

-                      "user[%p:0x%llx:0x%x] wire[%p:0x%llx:0x%x]\n",
-                      kr,
-                      user_map, user_addr + cur_offset, user_ppnum,
-                      wire_map, wire_addr + cur_offset, wire_ppnum);
+                   "user[%p:0x%llx:0x%x] wire[%p:0x%llx:0x%x]\n",
+                   kr,
+                   user_map, user_addr + cur_offset, user_ppnum,
+                   wire_map, wire_addr + cur_offset, wire_ppnum);

                if (kr != KERN_SUCCESS ||
                    wire_ppnum == 0 ||
                    wire_ppnum != user_ppnum) {
                if (kr != KERN_SUCCESS ||
                    wire_ppnum == 0 ||
                    wire_ppnum != user_ppnum) {


@@ -10381,16 +10392,16 @@ vm_test_wire_and_extract(void)
        }
        cur_offset -= PAGE_SIZE;
        kr = vm_map_wire_and_extract(wire_map,
        }
        cur_offset -= PAGE_SIZE;
        kr = vm_map_wire_and_extract(wire_map,

-                                    wire_addr + cur_offset,
-                                    VM_PROT_DEFAULT,
-                                    TRUE,
-                                    &wire_ppnum);
+           wire_addr + cur_offset,
+           VM_PROT_DEFAULT,
+           TRUE,
+           &wire_ppnum);

        assert(kr == KERN_SUCCESS);
        printf("VM_TEST_WIRE_AND_EXTRACT: re-wire kr=0x%x "
        assert(kr == KERN_SUCCESS);
        printf("VM_TEST_WIRE_AND_EXTRACT: re-wire kr=0x%x "

-              "user[%p:0x%llx:0x%x] wire[%p:0x%llx:0x%x]\n",
-              kr,
-              user_map, user_addr + cur_offset, user_ppnum,
-              wire_map, wire_addr + cur_offset, wire_ppnum);
+           "user[%p:0x%llx:0x%x] wire[%p:0x%llx:0x%x]\n",
+           kr,
+           user_map, user_addr + cur_offset, user_ppnum,
+           wire_map, wire_addr + cur_offset, wire_ppnum);

        if (kr != KERN_SUCCESS ||
            wire_ppnum == 0 ||
            wire_ppnum != user_ppnum) {
        if (kr != KERN_SUCCESS ||
            wire_ppnum == 0 ||
            wire_ppnum != user_ppnum) {


@@ -10422,7 +10433,7 @@ vm_test_page_wire_overflow_panic(void)
        vm_page_unlock_queues();
        vm_object_unlock(object);
        panic("FBDP(%p,%p): wire_count overflow not detected\n",
        vm_page_unlock_queues();
        vm_object_unlock(object);
        panic("FBDP(%p,%p): wire_count overflow not detected\n",

-             object, page);
+           object, page);

 }
 #else /* VM_TEST_PAGE_WIRE_OVERFLOW_PANIC */
 #define vm_test_page_wire_overflow_panic()
 }
 #else /* VM_TEST_PAGE_WIRE_OVERFLOW_PANIC */
 #define vm_test_page_wire_overflow_panic()


@@ -10439,11 +10450,11 @@ vm_test_kernel_object_fault(void)
        int ret;
 
        kr = kernel_memory_allocate(kernel_map, &stack,
        int ret;
 
        kr = kernel_memory_allocate(kernel_map, &stack,

-                                   kernel_stack_size + (2*PAGE_SIZE),
-                                   0,
-                                   (KMA_KSTACK | KMA_KOBJECT |
-                                    KMA_GUARD_FIRST | KMA_GUARD_LAST),
-                                   VM_KERN_MEMORY_STACK);
+           kernel_stack_size + (2 * PAGE_SIZE),
+           0,
+           (KMA_KSTACK | KMA_KOBJECT |
+           KMA_GUARD_FIRST | KMA_GUARD_LAST),
+           VM_KERN_MEMORY_STACK);

        if (kr != KERN_SUCCESS) {
                panic("VM_TEST_KERNEL_OBJECT_FAULT: kernel_memory_allocate kr 0x%x\n", kr);
        }
        if (kr != KERN_SUCCESS) {
                panic("VM_TEST_KERNEL_OBJECT_FAULT: kernel_memory_allocate kr 0x%x\n", kr);
        }


@@ -10454,9 +10465,9 @@ vm_test_kernel_object_fault(void)
                printf("VM_TEST_KERNEL_OBJECT_FAULT: FAIL\n");
        }
        vm_map_remove(kernel_map,
                printf("VM_TEST_KERNEL_OBJECT_FAULT: FAIL\n");
        }
        vm_map_remove(kernel_map,

-                     stack,
-                     stack + kernel_stack_size + (2*PAGE_SIZE),
-                     VM_MAP_REMOVE_KUNWIRE);
+           stack,
+           stack + kernel_stack_size + (2 * PAGE_SIZE),
+           VM_MAP_REMOVE_KUNWIRE);

        stack = 0;
 }
 #else /* __arm64__ && VM_TEST_KERNEL_OBJECT_FAULT */
        stack = 0;
 }
 #else /* __arm64__ && VM_TEST_KERNEL_OBJECT_FAULT */


@@ -10467,11 +10478,11 @@ vm_test_kernel_object_fault(void)
 static void
 vm_test_device_pager_transpose(void)
 {
 static void
 vm_test_device_pager_transpose(void)
 {

-       memory_object_t device_pager;
-       vm_object_t     anon_object, device_object;
-       vm_size_t       size;
-       vm_map_offset_t anon_mapping, device_mapping;
-       kern_return_t   kr;
+       memory_object_t device_pager;
+       vm_object_t     anon_object, device_object;
+       vm_size_t       size;
+       vm_map_offset_t device_mapping;
+       kern_return_t   kr;

 
        size = 3 * PAGE_SIZE;
        anon_object = vm_object_allocate(size);
 
        size = 3 * PAGE_SIZE;
        anon_object = vm_object_allocate(size);


@@ -10480,20 +10491,27 @@ vm_test_device_pager_transpose(void)
        assert(device_pager != NULL);
        device_object = memory_object_to_vm_object(device_pager);
        assert(device_object != VM_OBJECT_NULL);
        assert(device_pager != NULL);
        device_object = memory_object_to_vm_object(device_pager);
        assert(device_object != VM_OBJECT_NULL);

-       anon_mapping = 0;
+#if 0
+       /*
+        * Can't actually map this, since another thread might do a
+        * vm_map_enter() that gets coalesced into this object, which
+        * would cause the test to fail.
+        */
+       vm_map_offset_t anon_mapping = 0;

        kr = vm_map_enter(kernel_map, &anon_mapping, size, 0,
        kr = vm_map_enter(kernel_map, &anon_mapping, size, 0,

-                         VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_NONE,
-                         anon_object, 0, FALSE, VM_PROT_DEFAULT, VM_PROT_ALL,
-                         VM_INHERIT_DEFAULT);
+           VM_FLAGS_ANYWHERE, VM_MAP_KERNEL_FLAGS_NONE, VM_KERN_MEMORY_NONE,
+           anon_object, 0, FALSE, VM_PROT_DEFAULT, VM_PROT_ALL,
+           VM_INHERIT_DEFAULT);

        assert(kr == KERN_SUCCESS);
        assert(kr == KERN_SUCCESS);

+#endif

        device_mapping = 0;
        kr = vm_map_enter_mem_object(kernel_map, &device_mapping, size, 0,
        device_mapping = 0;
        kr = vm_map_enter_mem_object(kernel_map, &device_mapping, size, 0,

-                                    VM_FLAGS_ANYWHERE,
-                                    VM_MAP_KERNEL_FLAGS_NONE,
-                                    VM_KERN_MEMORY_NONE,
-                                    (void *)device_pager, 0, FALSE,
-                                    VM_PROT_DEFAULT, VM_PROT_ALL,
-                                    VM_INHERIT_DEFAULT);
+           VM_FLAGS_ANYWHERE,
+           VM_MAP_KERNEL_FLAGS_NONE,
+           VM_KERN_MEMORY_NONE,
+           (void *)device_pager, 0, FALSE,
+           VM_PROT_DEFAULT, VM_PROT_ALL,
+           VM_INHERIT_DEFAULT);

        assert(kr == KERN_SUCCESS);
        memory_object_deallocate(device_pager);
 
        assert(kr == KERN_SUCCESS);
        memory_object_deallocate(device_pager);
 


@@ -10525,8 +10543,10 @@ vm_test_device_pager_transpose(void)
 
        assert(anon_object->ref_count == 2);
        assert(anon_object->named);
 
        assert(anon_object->ref_count == 2);
        assert(anon_object->named);

+#if 0

        kr = vm_deallocate(kernel_map, anon_mapping, size);
        assert(kr == KERN_SUCCESS);
        kr = vm_deallocate(kernel_map, anon_mapping, size);
        assert(kr == KERN_SUCCESS);

+#endif

        assert(device_object->ref_count == 1);
        assert(!device_object->named);
        kr = vm_deallocate(kernel_map, device_mapping, size);
        assert(device_object->ref_count == 1);
        assert(!device_object->named);
        kr = vm_deallocate(kernel_map, device_mapping, size);