xnu-7195.50.7.100.1.tar.gz

[apple/xnu.git] / osfmk / vm / vm_resident.c

diff --git a/osfmk/vm/vm_resident.c b/osfmk/vm/vm_resident.c
index 860bde4d5314d1739ea8d9288f426ea122be1ab2..45430fce92cd1ac84e884eebc4969412d1b640c2 100644 (file)
--- a/osfmk/vm/vm_resident.c
+++ b/osfmk/vm/vm_resident.c
@@ -1,8 +1,8 @@
 /*
 /*

- * Copyright (c) 2000-2009 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2020 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

- * 
+ *

  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in


@@ -11,10 +11,10 @@
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.

- * 
+ *

  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.

- * 
+ *

  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,


@@ -22,34 +22,34 @@
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.

- * 
+ *

  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
  */
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @OSF_COPYRIGHT@
  */

-/* 
+/*

  * Mach Operating System
  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
  * All Rights Reserved.
  * Mach Operating System
  * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
  * All Rights Reserved.

- * 
+ *

  * Permission to use, copy, modify and distribute this software and its
  * documentation is hereby granted, provided that both the copyright
  * notice and this permission notice appear in all copies of the
  * software, derivative works or modified versions, and any portions
  * thereof, and that both notices appear in supporting documentation.
  * Permission to use, copy, modify and distribute this software and its
  * documentation is hereby granted, provided that both the copyright
  * notice and this permission notice appear in all copies of the
  * software, derivative works or modified versions, and any portions
  * thereof, and that both notices appear in supporting documentation.

- * 
+ *

  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
  * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.

- * 
+ *

  * Carnegie Mellon requests users of this software to return to
  * Carnegie Mellon requests users of this software to return to

- * 
+ *

  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  *  School of Computer Science
  *  Carnegie Mellon University
  *  Pittsburgh PA 15213-3890
  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
  *  School of Computer Science
  *  Carnegie Mellon University
  *  Pittsburgh PA 15213-3890

- * 
+ *

  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */
  * any improvements or extensions that they make and grant Carnegie Mellon
  * the rights to redistribute these changes.
  */


@@ -71,22 +71,21 @@
 #include <mach/vm_statistics.h>
 #include <mach/sdt.h>
 #include <kern/counters.h>
 #include <mach/vm_statistics.h>
 #include <mach/sdt.h>
 #include <kern/counters.h>

+#include <kern/host_statistics.h>

 #include <kern/sched_prim.h>
 #include <kern/policy_internal.h>
 #include <kern/task.h>
 #include <kern/thread.h>
 #include <kern/kalloc.h>
 #include <kern/sched_prim.h>
 #include <kern/policy_internal.h>
 #include <kern/task.h>
 #include <kern/thread.h>
 #include <kern/kalloc.h>

-#include <kern/zalloc.h>
-#include <kern/xpr.h>
+#include <kern/zalloc_internal.h>

 #include <kern/ledger.h>
 #include <vm/pmap.h>
 #include <vm/vm_init.h>
 #include <vm/vm_map.h>
 #include <vm/vm_page.h>
 #include <vm/vm_pageout.h>
 #include <kern/ledger.h>
 #include <vm/pmap.h>
 #include <vm/vm_init.h>
 #include <vm/vm_map.h>
 #include <vm/vm_page.h>
 #include <vm/vm_pageout.h>

-#include <vm/vm_kern.h>                        /* kernel_memory_allocate() */
+#include <vm/vm_kern.h>                 /* kernel_memory_allocate() */

 #include <kern/misc_protos.h>
 #include <kern/misc_protos.h>

-#include <zone_debug.h>

 #include <mach_debug/zone_info.h>
 #include <vm/cpm.h>
 #include <pexpert/pexpert.h>
 #include <mach_debug/zone_info.h>
 #include <vm/cpm.h>
 #include <pexpert/pexpert.h>


@@ -96,57 +95,87 @@
 #include <vm/memory_object.h>
 #include <vm/vm_purgeable_internal.h>
 #include <vm/vm_compressor.h>
 #include <vm/memory_object.h>
 #include <vm/vm_purgeable_internal.h>
 #include <vm/vm_compressor.h>

+#if defined (__x86_64__)
+#include <i386/misc_protos.h>
+#endif

 
 #if CONFIG_PHANTOM_CACHE
 #include <vm/vm_phantom_cache.h>
 #endif
 
 
 #if CONFIG_PHANTOM_CACHE
 #include <vm/vm_phantom_cache.h>
 #endif
 

+#if HIBERNATION

 #include <IOKit/IOHibernatePrivate.h>
 #include <IOKit/IOHibernatePrivate.h>

+#include <machine/pal_hibernate.h>
+#endif /* HIBERNATION */

 
 #include <sys/kdebug.h>
 
 
 #include <sys/kdebug.h>
 

+#if defined(HAS_APPLE_PAC)
+#include <ptrauth.h>
+#endif
+#if defined(__arm64__)
+#include <arm/cpu_internal.h>
+#endif /* defined(__arm64__) */
+
+#if MACH_ASSERT
+
+#define ASSERT_PMAP_FREE(mem) pmap_assert_free(VM_PAGE_GET_PHYS_PAGE(mem))
+
+#else /* MACH_ASSERT */
+
+#define ASSERT_PMAP_FREE(mem) /* nothing */

 
 

+#endif /* MACH_ASSERT */

 
 

-char   vm_page_inactive_states[VM_PAGE_Q_STATE_ARRAY_SIZE];
-char   vm_page_pageable_states[VM_PAGE_Q_STATE_ARRAY_SIZE];
-char   vm_page_non_speculative_pageable_states[VM_PAGE_Q_STATE_ARRAY_SIZE];
-char   vm_page_active_or_inactive_states[VM_PAGE_Q_STATE_ARRAY_SIZE];
+extern boolean_t vm_pageout_running;
+extern thread_t  vm_pageout_scan_thread;
+extern boolean_t vps_dynamic_priority_enabled;
+
+char    vm_page_inactive_states[VM_PAGE_Q_STATE_ARRAY_SIZE];
+char    vm_page_pageable_states[VM_PAGE_Q_STATE_ARRAY_SIZE];
+char    vm_page_non_speculative_pageable_states[VM_PAGE_Q_STATE_ARRAY_SIZE];
+char    vm_page_active_or_inactive_states[VM_PAGE_Q_STATE_ARRAY_SIZE];

 
 #if CONFIG_SECLUDED_MEMORY
 struct vm_page_secluded_data vm_page_secluded;
 
 #if CONFIG_SECLUDED_MEMORY
 struct vm_page_secluded_data vm_page_secluded;

-void secluded_suppression_init(void);

 #endif /* CONFIG_SECLUDED_MEMORY */
 
 #endif /* CONFIG_SECLUDED_MEMORY */
 

-boolean_t      hibernate_cleaning_in_progress = FALSE;
-boolean_t      vm_page_free_verify = TRUE;
+#if DEVELOPMENT || DEBUG
+extern struct memory_object_pager_ops shared_region_pager_ops;
+unsigned int shared_region_pagers_resident_count = 0;
+unsigned int shared_region_pagers_resident_peak = 0;
+#endif /* DEVELOPMENT || DEBUG */

 
 

-uint32_t       vm_lopage_free_count = 0;
-uint32_t       vm_lopage_free_limit = 0;
-uint32_t       vm_lopage_lowater    = 0;
-boolean_t      vm_lopage_refill = FALSE;
-boolean_t      vm_lopage_needed = FALSE;
+int             PERCPU_DATA(start_color);
+vm_page_t       PERCPU_DATA(free_pages);
+boolean_t       hibernate_cleaning_in_progress = FALSE;
+boolean_t       vm_page_free_verify = TRUE;

 
 

-lck_mtx_ext_t  vm_page_queue_lock_ext;
-lck_mtx_ext_t  vm_page_queue_free_lock_ext;
-lck_mtx_ext_t  vm_purgeable_queue_lock_ext;
+uint32_t        vm_lopage_free_count = 0;
+uint32_t        vm_lopage_free_limit = 0;
+uint32_t        vm_lopage_lowater    = 0;
+boolean_t       vm_lopage_refill = FALSE;
+boolean_t       vm_lopage_needed = FALSE;

 
 

-int            speculative_age_index = 0;
-int            speculative_steal_index = 0;
-struct vm_speculative_age_q vm_page_queue_speculative[VM_PAGE_MAX_SPECULATIVE_AGE_Q + 1];
+lck_mtx_ext_t   vm_page_queue_lock_ext;
+lck_mtx_ext_t   vm_page_queue_free_lock_ext;
+lck_mtx_ext_t   vm_purgeable_queue_lock_ext;

 
 

-boolean_t      hibernation_vmqueues_inspection = FALSE; /* Tracks if the hibernation code is looking at the VM queues.
-                                                         * Updated and checked behind the vm_page_queues_lock. */
+int             speculative_age_index = 0;
+int             speculative_steal_index = 0;
+struct vm_speculative_age_q vm_page_queue_speculative[VM_PAGE_MAX_SPECULATIVE_AGE_Q + 1];

 
 

-__private_extern__ void                vm_page_init_lck_grp(void);
+boolean_t       hibernation_vmqueues_inspection = FALSE; /* Tracks if the hibernation code is looking at the VM queues.
+                                                          * Updated and checked behind the vm_page_queues_lock. */

 
 

-static void            vm_page_free_prepare(vm_page_t  page);
-static vm_page_t       vm_page_grab_fictitious_common(ppnum_t phys_addr);
+static void             vm_page_free_prepare(vm_page_t  page);
+static vm_page_t        vm_page_grab_fictitious_common(ppnum_t phys_addr);

 
 static void vm_tag_init(void);
 
 
 static void vm_tag_init(void);
 

-uint64_t       vm_min_kernel_and_kext_address = VM_MIN_KERNEL_AND_KEXT_ADDRESS;
-uint32_t       vm_packed_from_vm_pages_array_mask = VM_PACKED_FROM_VM_PAGES_ARRAY;
-uint32_t       vm_packed_pointer_shift = VM_PACKED_POINTER_SHIFT;
+/* for debugging purposes */
+SECURITY_READ_ONLY_EARLY(vm_packing_params_t) vm_page_packing_params =
+    VM_PACKING_PARAMS(VM_PAGE_PACKED_PTR);

 
 /*
  *     Associated with page of user-allocatable memory is a
 
 /*
  *     Associated with page of user-allocatable memory is a


@@ -161,7 +190,7 @@ uint32_t    vm_packed_pointer_shift = VM_PACKED_POINTER_SHIFT;
 
 vm_offset_t virtual_space_start;
 vm_offset_t virtual_space_end;
 
 vm_offset_t virtual_space_start;
 vm_offset_t virtual_space_end;

-uint32_t       vm_page_pages;
+uint32_t        vm_page_pages;

 
 /*
  *     The vm_page_lookup() routine, which provides for fast
 
 /*
  *     The vm_page_lookup() routine, which provides for fast


@@ -173,21 +202,21 @@ uint32_t  vm_page_pages;
  */
 typedef struct {
        vm_page_packed_t page_list;
  */
 typedef struct {
        vm_page_packed_t page_list;

-#if    MACH_PAGE_HASH_STATS
-       int             cur_count;              /* current count */
-       int             hi_count;               /* high water mark */
+#if     MACH_PAGE_HASH_STATS
+       int             cur_count;              /* current count */
+       int             hi_count;               /* high water mark */

 #endif /* MACH_PAGE_HASH_STATS */
 } vm_page_bucket_t;
 
 
 #endif /* MACH_PAGE_HASH_STATS */
 } vm_page_bucket_t;
 
 

-#define BUCKETS_PER_LOCK       16
+#define BUCKETS_PER_LOCK        16

 
 

-vm_page_bucket_t *vm_page_buckets;             /* Array of buckets */
-unsigned int   vm_page_bucket_count = 0;       /* How big is array? */
-unsigned int   vm_page_hash_mask;              /* Mask for hash function */
-unsigned int   vm_page_hash_shift;             /* Shift for hash function */
-uint32_t       vm_page_bucket_hash;            /* Basic bucket hash */
-unsigned int   vm_page_bucket_lock_count = 0;          /* How big is array of locks? */
+vm_page_bucket_t *vm_page_buckets;              /* Array of buckets */
+unsigned int    vm_page_bucket_count = 0;       /* How big is array? */
+unsigned int    vm_page_hash_mask;              /* Mask for hash function */
+unsigned int    vm_page_hash_shift;             /* Shift for hash function */
+uint32_t        vm_page_bucket_hash;            /* Basic bucket hash */
+unsigned int    vm_page_bucket_lock_count = 0;          /* How big is array of locks? */

 
 #ifndef VM_TAG_ACTIVE_UPDATE
 #error VM_TAG_ACTIVE_UPDATE
 
 #ifndef VM_TAG_ACTIVE_UPDATE
 #error VM_TAG_ACTIVE_UPDATE


@@ -197,9 +226,7 @@ unsigned int        vm_page_bucket_lock_count = 0;          /* How big is array of locks? */
 #endif
 
 boolean_t   vm_tag_active_update = VM_TAG_ACTIVE_UPDATE;
 #endif
 
 boolean_t   vm_tag_active_update = VM_TAG_ACTIVE_UPDATE;

-lck_spin_t     *vm_page_bucket_locks;
-lck_spin_t     vm_objects_wired_lock;
-lck_spin_t     vm_allocation_sites_lock;
+lck_spin_t      *vm_page_bucket_locks;

 
 vm_allocation_site_t            vm_allocation_sites_static[VM_KERN_MEMORY_FIRST_DYNAMIC + 1];
 vm_allocation_site_t *          vm_allocation_sites[VM_MAX_TAG_VALUE];
 
 vm_allocation_site_t            vm_allocation_sites_static[VM_KERN_MEMORY_FIRST_DYNAMIC + 1];
 vm_allocation_site_t *          vm_allocation_sites[VM_MAX_TAG_VALUE];


@@ -212,14 +239,14 @@ vm_tag_t vm_allocation_tag_highest;
 #if VM_PAGE_BUCKETS_CHECK
 boolean_t vm_page_buckets_check_ready = FALSE;
 #if VM_PAGE_FAKE_BUCKETS
 #if VM_PAGE_BUCKETS_CHECK
 boolean_t vm_page_buckets_check_ready = FALSE;
 #if VM_PAGE_FAKE_BUCKETS

-vm_page_bucket_t *vm_page_fake_buckets;        /* decoy buckets */
+vm_page_bucket_t *vm_page_fake_buckets; /* decoy buckets */

 vm_map_offset_t vm_page_fake_buckets_start, vm_page_fake_buckets_end;
 #endif /* VM_PAGE_FAKE_BUCKETS */
 #endif /* VM_PAGE_BUCKETS_CHECK */
 
 
 
 vm_map_offset_t vm_page_fake_buckets_start, vm_page_fake_buckets_end;
 #endif /* VM_PAGE_FAKE_BUCKETS */
 #endif /* VM_PAGE_BUCKETS_CHECK */
 
 
 

-#if    MACH_PAGE_HASH_STATS
+#if     MACH_PAGE_HASH_STATS

 /* This routine is only for debug.  It is intended to be called by
  * hand by a developer using a kernel debugger.  This routine prints
  * out vm_page_hash table statistics to the kernel debug console.
 /* This routine is only for debug.  It is intended to be called by
  * hand by a developer using a kernel debugger.  This routine prints
  * out vm_page_hash table statistics to the kernel debug console.


@@ -227,29 +254,30 @@ vm_map_offset_t vm_page_fake_buckets_start, vm_page_fake_buckets_end;
 void
 hash_debug(void)
 {
 void
 hash_debug(void)
 {

-       int     i;
-       int     numbuckets = 0;
-       int     highsum = 0;
-       int     maxdepth = 0;
+       int     i;
+       int     numbuckets = 0;
+       int     highsum = 0;
+       int     maxdepth = 0;

 
        for (i = 0; i < vm_page_bucket_count; i++) {
                if (vm_page_buckets[i].hi_count) {
                        numbuckets++;
                        highsum += vm_page_buckets[i].hi_count;
 
        for (i = 0; i < vm_page_bucket_count; i++) {
                if (vm_page_buckets[i].hi_count) {
                        numbuckets++;
                        highsum += vm_page_buckets[i].hi_count;

-                       if (vm_page_buckets[i].hi_count > maxdepth)
+                       if (vm_page_buckets[i].hi_count > maxdepth) {

                                maxdepth = vm_page_buckets[i].hi_count;
                                maxdepth = vm_page_buckets[i].hi_count;

+                       }

                }
        }
        printf("Total number of buckets: %d\n", vm_page_bucket_count);
        printf("Number used buckets:     %d = %d%%\n",
                }
        }
        printf("Total number of buckets: %d\n", vm_page_bucket_count);
        printf("Number used buckets:     %d = %d%%\n",

-               numbuckets, 100*numbuckets/vm_page_bucket_count);
+           numbuckets, 100 * numbuckets / vm_page_bucket_count);

        printf("Number unused buckets:   %d = %d%%\n",
        printf("Number unused buckets:   %d = %d%%\n",

-               vm_page_bucket_count - numbuckets,
-               100*(vm_page_bucket_count-numbuckets)/vm_page_bucket_count);
+           vm_page_bucket_count - numbuckets,
+           100 * (vm_page_bucket_count - numbuckets) / vm_page_bucket_count);

        printf("Sum of bucket max depth: %d\n", highsum);
        printf("Average bucket depth:    %d.%2d\n",
        printf("Sum of bucket max depth: %d\n", highsum);
        printf("Average bucket depth:    %d.%2d\n",

-               highsum/vm_page_bucket_count,
-               highsum%vm_page_bucket_count);
+           highsum / vm_page_bucket_count,
+           highsum % vm_page_bucket_count);

        printf("Maximum bucket depth:    %d\n", maxdepth);
 }
 #endif /* MACH_PAGE_HASH_STATS */
        printf("Maximum bucket depth:    %d\n", maxdepth);
 }
 #endif /* MACH_PAGE_HASH_STATS */


@@ -266,57 +294,45 @@ hash_debug(void)
  *     constants.
  */
 #if defined(__arm__) || defined(__arm64__)
  *     constants.
  */
 #if defined(__arm__) || defined(__arm64__)

-vm_size_t      page_size;
-vm_size_t      page_mask;
-int            page_shift;
+vm_size_t       page_size;
+vm_size_t       page_mask;
+int             page_shift;

 #else
 #else

-vm_size_t      page_size  = PAGE_SIZE;
-vm_size_t      page_mask  = PAGE_MASK;
-int            page_shift = PAGE_SHIFT;
+vm_size_t       page_size  = PAGE_SIZE;
+vm_size_t       page_mask  = PAGE_MASK;
+int             page_shift = PAGE_SHIFT;

 #endif
 
 #endif
 

-/*
- *     Resident page structures are initialized from
- *     a template (see vm_page_alloc).
- *
- *     When adding a new field to the virtual memory
- *     object structure, be sure to add initialization
- *     (see vm_page_bootstrap).
- */
-struct vm_page vm_page_template;
-
-vm_page_t      vm_pages = VM_PAGE_NULL;
-vm_page_t      vm_page_array_beginning_addr;
-vm_page_t      vm_page_array_ending_addr;
-vm_page_t      vm_page_array_boundary;
+SECURITY_READ_ONLY_LATE(vm_page_t) vm_pages = VM_PAGE_NULL;
+SECURITY_READ_ONLY_LATE(vm_page_t) vm_page_array_beginning_addr;
+vm_page_t                          vm_page_array_ending_addr;

 
 

-unsigned int   vm_pages_count = 0;
-ppnum_t                vm_page_lowest = 0;
+unsigned int    vm_pages_count = 0;

 
 /*
  *     Resident pages that represent real memory
  *     are allocated from a set of free lists,
  *     one per color.
  */
 
 /*
  *     Resident pages that represent real memory
  *     are allocated from a set of free lists,
  *     one per color.
  */

-unsigned int   vm_colors;
-unsigned int    vm_color_mask;                 /* mask is == (vm_colors-1) */
-unsigned int   vm_cache_geometry_colors = 0;   /* set by hw dependent code during startup */
-unsigned int   vm_free_magazine_refill_limit = 0;
+unsigned int    vm_colors;
+unsigned int    vm_color_mask;                  /* mask is == (vm_colors-1) */
+unsigned int    vm_cache_geometry_colors = 0;   /* set by hw dependent code during startup */
+unsigned int    vm_free_magazine_refill_limit = 0;

 
 
 struct vm_page_queue_free_head {
 
 
 struct vm_page_queue_free_head {

-       vm_page_queue_head_t    qhead;
-} __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT)));
+       vm_page_queue_head_t    qhead;
+} VM_PAGE_PACKED_ALIGNED;

 
 

-struct vm_page_queue_free_head vm_page_queue_free[MAX_COLORS];
+struct vm_page_queue_free_head  vm_page_queue_free[MAX_COLORS];

 
 
 
 

-unsigned int   vm_page_free_wanted;
-unsigned int   vm_page_free_wanted_privileged;
+unsigned int    vm_page_free_wanted;
+unsigned int    vm_page_free_wanted_privileged;

 #if CONFIG_SECLUDED_MEMORY
 #if CONFIG_SECLUDED_MEMORY

-unsigned int   vm_page_free_wanted_secluded;
+unsigned int    vm_page_free_wanted_secluded;

 #endif /* CONFIG_SECLUDED_MEMORY */
 #endif /* CONFIG_SECLUDED_MEMORY */

-unsigned int   vm_page_free_count;
+unsigned int    vm_page_free_count;

 
 /*
  *     Occasionally, the virtual memory system uses
 
 /*
  *     Occasionally, the virtual memory system uses


@@ -327,16 +343,23 @@ unsigned int      vm_page_free_count;
  *     These page structures are allocated the way
  *     most other kernel structures are.
  */
  *     These page structures are allocated the way
  *     most other kernel structures are.
  */

-zone_t vm_page_array_zone;
-zone_t vm_page_zone;
+SECURITY_READ_ONLY_LATE(zone_t) vm_page_zone;

 vm_locks_array_t vm_page_locks;
 vm_locks_array_t vm_page_locks;

-decl_lck_mtx_data(,vm_page_alloc_lock)
-lck_mtx_ext_t vm_page_alloc_lock_ext;

 
 

-unsigned int   vm_page_local_q_count = 0;
-unsigned int   vm_page_local_q_soft_limit = 250;
-unsigned int   vm_page_local_q_hard_limit = 500;
-struct vplq     *vm_page_local_q = NULL;
+LCK_ATTR_DECLARE(vm_page_lck_attr, 0, 0);
+LCK_GRP_DECLARE(vm_page_lck_grp_free, "vm_page_free");
+LCK_GRP_DECLARE(vm_page_lck_grp_queue, "vm_page_queue");
+LCK_GRP_DECLARE(vm_page_lck_grp_local, "vm_page_queue_local");
+LCK_GRP_DECLARE(vm_page_lck_grp_purge, "vm_page_purge");
+LCK_GRP_DECLARE(vm_page_lck_grp_alloc, "vm_page_alloc");
+LCK_GRP_DECLARE(vm_page_lck_grp_bucket, "vm_page_bucket");
+LCK_MTX_EARLY_DECLARE_ATTR(vm_page_alloc_lock, &vm_page_lck_grp_alloc, &vm_page_lck_attr);
+LCK_SPIN_DECLARE_ATTR(vm_objects_wired_lock, &vm_page_lck_grp_bucket, &vm_page_lck_attr);
+LCK_SPIN_DECLARE_ATTR(vm_allocation_sites_lock, &vm_page_lck_grp_bucket, &vm_page_lck_attr);
+
+unsigned int    vm_page_local_q_soft_limit = 250;
+unsigned int    vm_page_local_q_hard_limit = 500;
+struct vpl     *__zpercpu vm_page_local_q;

 
 /* N.B. Guard and fictitious pages must not
  * be assigned a zero phys_page value.
 
 /* N.B. Guard and fictitious pages must not
  * be assigned a zero phys_page value.


@@ -351,7 +374,7 @@ const ppnum_t vm_page_fictitious_addr = (ppnum_t) -1;
 
 /*
  *     Guard pages are not accessible so they don't
 
 /*
  *     Guard pages are not accessible so they don't

- *     need a physical address, but we need to enter
+ *      need a physical address, but we need to enter

  *     one in the pmap.
  *     Let's make it recognizable and make sure that
  *     we don't use a real physical page with that
  *     one in the pmap.
  *     Let's make it recognizable and make sure that
  *     we don't use a real physical page with that


@@ -364,80 +387,84 @@ const ppnum_t vm_page_guard_addr = (ppnum_t) -2;
  *     queues that are used by the page replacement
  *     system (pageout daemon).  These queues are
  *     defined here, but are shared by the pageout
  *     queues that are used by the page replacement
  *     system (pageout daemon).  These queues are
  *     defined here, but are shared by the pageout

- *     module.  The inactive queue is broken into 
- *     file backed and anonymous for convenience as the 
- *     pageout daemon often assignes a higher 
+ *     module.  The inactive queue is broken into
+ *     file backed and anonymous for convenience as the
+ *     pageout daemon often assignes a higher

  *     importance to anonymous pages (less likely to pick)
  */
  *     importance to anonymous pages (less likely to pick)
  */

-vm_page_queue_head_t   vm_page_queue_active __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT)));
-vm_page_queue_head_t   vm_page_queue_inactive __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT)));
+vm_page_queue_head_t    vm_page_queue_active VM_PAGE_PACKED_ALIGNED;
+vm_page_queue_head_t    vm_page_queue_inactive VM_PAGE_PACKED_ALIGNED;

 #if CONFIG_SECLUDED_MEMORY
 #if CONFIG_SECLUDED_MEMORY

-vm_page_queue_head_t   vm_page_queue_secluded __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT)));
+vm_page_queue_head_t    vm_page_queue_secluded VM_PAGE_PACKED_ALIGNED;

 #endif /* CONFIG_SECLUDED_MEMORY */
 #endif /* CONFIG_SECLUDED_MEMORY */

-vm_page_queue_head_t   vm_page_queue_anonymous __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT)));  /* inactive memory queue for anonymous pages */
-vm_page_queue_head_t   vm_page_queue_throttled __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT)));
+vm_page_queue_head_t    vm_page_queue_anonymous VM_PAGE_PACKED_ALIGNED;  /* inactive memory queue for anonymous pages */
+vm_page_queue_head_t    vm_page_queue_throttled VM_PAGE_PACKED_ALIGNED;

 
 

-queue_head_t   vm_objects_wired;
+queue_head_t    vm_objects_wired;

 
 void vm_update_darkwake_mode(boolean_t);
 
 #if CONFIG_BACKGROUND_QUEUE
 
 void vm_update_darkwake_mode(boolean_t);
 
 #if CONFIG_BACKGROUND_QUEUE

-vm_page_queue_head_t   vm_page_queue_background __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT)));
-uint32_t       vm_page_background_target;
-uint32_t       vm_page_background_target_snapshot;
-uint32_t       vm_page_background_count;
-uint64_t       vm_page_background_promoted_count;
+vm_page_queue_head_t    vm_page_queue_background VM_PAGE_PACKED_ALIGNED;
+uint32_t        vm_page_background_target;
+uint32_t        vm_page_background_target_snapshot;
+uint32_t        vm_page_background_count;
+uint64_t        vm_page_background_promoted_count;

 
 

-uint32_t       vm_page_background_internal_count;
-uint32_t       vm_page_background_external_count;
+uint32_t        vm_page_background_internal_count;
+uint32_t        vm_page_background_external_count;

 
 

-uint32_t       vm_page_background_mode;
-uint32_t       vm_page_background_exclude_external;
+uint32_t        vm_page_background_mode;
+uint32_t        vm_page_background_exclude_external;

 #endif
 
 #endif
 

-unsigned int   vm_page_active_count;
-unsigned int   vm_page_inactive_count;
+unsigned int    vm_page_active_count;
+unsigned int    vm_page_inactive_count;
+unsigned int    vm_page_kernelcache_count;

 #if CONFIG_SECLUDED_MEMORY
 #if CONFIG_SECLUDED_MEMORY

-unsigned int   vm_page_secluded_count;
-unsigned int   vm_page_secluded_count_free;
-unsigned int   vm_page_secluded_count_inuse;
+unsigned int    vm_page_secluded_count;
+unsigned int    vm_page_secluded_count_free;
+unsigned int    vm_page_secluded_count_inuse;
+unsigned int    vm_page_secluded_count_over_target;

 #endif /* CONFIG_SECLUDED_MEMORY */
 #endif /* CONFIG_SECLUDED_MEMORY */

-unsigned int   vm_page_anonymous_count;
-unsigned int   vm_page_throttled_count;
-unsigned int   vm_page_speculative_count;
-
-unsigned int   vm_page_wire_count;
-unsigned int   vm_page_wire_count_on_boot = 0;
-unsigned int   vm_page_stolen_count;
-unsigned int   vm_page_wire_count_initial;
-unsigned int   vm_page_pages_initial;
-unsigned int   vm_page_gobble_count = 0;
-
-#define        VM_PAGE_WIRE_COUNT_WARNING      0
-#define VM_PAGE_GOBBLE_COUNT_WARNING   0
-
-unsigned int   vm_page_purgeable_count = 0; /* # of pages purgeable now */
-unsigned int   vm_page_purgeable_wired_count = 0; /* # of purgeable pages that are wired now */
-uint64_t       vm_page_purged_count = 0;    /* total count of purged pages */
-
-unsigned int   vm_page_xpmapped_external_count = 0;
-unsigned int   vm_page_external_count = 0;
-unsigned int   vm_page_internal_count = 0;
-unsigned int   vm_page_pageable_external_count = 0;
-unsigned int   vm_page_pageable_internal_count = 0;
+unsigned int    vm_page_anonymous_count;
+unsigned int    vm_page_throttled_count;
+unsigned int    vm_page_speculative_count;
+
+unsigned int    vm_page_wire_count;
+unsigned int    vm_page_wire_count_on_boot = 0;
+unsigned int    vm_page_stolen_count = 0;
+unsigned int    vm_page_wire_count_initial;
+unsigned int    vm_page_gobble_count = 0;
+unsigned int    vm_page_kern_lpage_count = 0;
+
+uint64_t        booter_size;  /* external so it can be found in core dumps */
+
+#define VM_PAGE_WIRE_COUNT_WARNING      0
+#define VM_PAGE_GOBBLE_COUNT_WARNING    0
+
+unsigned int    vm_page_purgeable_count = 0; /* # of pages purgeable now */
+unsigned int    vm_page_purgeable_wired_count = 0; /* # of purgeable pages that are wired now */
+uint64_t        vm_page_purged_count = 0;    /* total count of purged pages */
+
+unsigned int    vm_page_xpmapped_external_count = 0;
+unsigned int    vm_page_external_count = 0;
+unsigned int    vm_page_internal_count = 0;
+unsigned int    vm_page_pageable_external_count = 0;
+unsigned int    vm_page_pageable_internal_count = 0;

 
 #if DEVELOPMENT || DEBUG
 
 #if DEVELOPMENT || DEBUG

-unsigned int   vm_page_speculative_recreated = 0;
-unsigned int   vm_page_speculative_created = 0;
-unsigned int   vm_page_speculative_used = 0;
+unsigned int    vm_page_speculative_recreated = 0;
+unsigned int    vm_page_speculative_created = 0;
+unsigned int    vm_page_speculative_used = 0;

 #endif
 
 #endif
 

-vm_page_queue_head_t    vm_page_queue_cleaned __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT)));
+vm_page_queue_head_t    vm_page_queue_cleaned VM_PAGE_PACKED_ALIGNED;

 
 

-unsigned int   vm_page_cleaned_count = 0;
+unsigned int    vm_page_cleaned_count = 0;

 
 

-uint64_t       max_valid_dma_address = 0xffffffffffffffffULL;
-ppnum_t                max_valid_low_ppnum = 0xffffffff;
+uint64_t        max_valid_dma_address = 0xffffffffffffffffULL;
+ppnum_t         max_valid_low_ppnum = PPNUM_MAX;

 
 
 /*
 
 
 /*


@@ -446,15 +473,15 @@ ppnum_t           max_valid_low_ppnum = 0xffffffff;
  *     (done here in vm_page_alloc) can trigger the
  *     pageout daemon.
  */
  *     (done here in vm_page_alloc) can trigger the
  *     pageout daemon.
  */

-unsigned int   vm_page_free_target = 0;
-unsigned int   vm_page_free_min = 0;
-unsigned int   vm_page_throttle_limit = 0;
-unsigned int   vm_page_inactive_target = 0;
+unsigned int    vm_page_free_target = 0;
+unsigned int    vm_page_free_min = 0;
+unsigned int    vm_page_throttle_limit = 0;
+unsigned int    vm_page_inactive_target = 0;

 #if CONFIG_SECLUDED_MEMORY
 #if CONFIG_SECLUDED_MEMORY

-unsigned int   vm_page_secluded_target = 0;
+unsigned int    vm_page_secluded_target = 0;

 #endif /* CONFIG_SECLUDED_MEMORY */
 #endif /* CONFIG_SECLUDED_MEMORY */

-unsigned int   vm_page_anonymous_min = 0;
-unsigned int   vm_page_free_reserved = 0;
+unsigned int    vm_page_anonymous_min = 0;
+unsigned int    vm_page_free_reserved = 0;

 
 
 /*
 
 
 /*


@@ -468,7 +495,7 @@ unsigned int        vm_page_free_reserved = 0;
 boolean_t vm_page_deactivate_hint = TRUE;
 
 struct vm_page_stats_reusable vm_page_stats_reusable;
 boolean_t vm_page_deactivate_hint = TRUE;
 
 struct vm_page_stats_reusable vm_page_stats_reusable;

-       
+

 /*
  *     vm_set_page_size:
  *
 /*
  *     vm_set_page_size:
  *


@@ -485,12 +512,15 @@ vm_set_page_size(void)
        page_mask  = PAGE_MASK;
        page_shift = PAGE_SHIFT;
 
        page_mask  = PAGE_MASK;
        page_shift = PAGE_SHIFT;
 

-       if ((page_mask & page_size) != 0)
+       if ((page_mask & page_size) != 0) {

                panic("vm_set_page_size: page size not a power of two");
                panic("vm_set_page_size: page size not a power of two");

+       }

 
 

-       for (page_shift = 0; ; page_shift++)
-               if ((1U << page_shift) == page_size)
+       for (page_shift = 0;; page_shift++) {
+               if ((1U << page_shift) == page_size) {

                        break;
                        break;

+               }
+       }

 }
 
 #if defined (__x86_64__)
 }
 
 #if defined (__x86_64__)


@@ -501,13 +531,17 @@ vm_set_page_size(void)
 unsigned int vm_clump_size, vm_clump_mask, vm_clump_shift, vm_clump_promote_threshold;
 
 #if DEVELOPMENT || DEBUG
 unsigned int vm_clump_size, vm_clump_mask, vm_clump_shift, vm_clump_promote_threshold;
 
 #if DEVELOPMENT || DEBUG

-unsigned long vm_clump_stats[MAX_CLUMP_SIZE+1];
+unsigned long vm_clump_stats[MAX_CLUMP_SIZE + 1];

 unsigned long vm_clump_allocs, vm_clump_inserts, vm_clump_inrange, vm_clump_promotes;
 
 unsigned long vm_clump_allocs, vm_clump_inserts, vm_clump_inrange, vm_clump_promotes;
 

-static inline void vm_clump_update_stats(unsigned int c) {
-    assert(c<=vm_clump_size);
-    if(c>0 && c<=vm_clump_size) vm_clump_stats[c]+=c;
-    vm_clump_allocs+=c;
+static inline void
+vm_clump_update_stats(unsigned int c)
+{
+       assert(c <= vm_clump_size);
+       if (c > 0 && c <= vm_clump_size) {
+               vm_clump_stats[c] += c;
+       }
+       vm_clump_allocs += c;

 }
 #endif  /*  if DEVELOPMENT || DEBUG */
 
 }
 #endif  /*  if DEVELOPMENT || DEBUG */
 


@@ -515,135 +549,337 @@ static inline void vm_clump_update_stats(unsigned int c) {
 static void
 vm_page_setup_clump( void )
 {
 static void
 vm_page_setup_clump( void )
 {

-    unsigned int override, n;
+       unsigned int override, n;

 
 

-    vm_clump_size = DEFAULT_CLUMP_SIZE;
-    if ( PE_parse_boot_argn("clump_size", &override, sizeof (override)) ) vm_clump_size = override;
+       vm_clump_size = DEFAULT_CLUMP_SIZE;
+       if (PE_parse_boot_argn("clump_size", &override, sizeof(override))) {
+               vm_clump_size = override;
+       }

 
 

-    if(vm_clump_size > MAX_CLUMP_SIZE) panic("vm_page_setup_clump:: clump_size is too large!");
-    if(vm_clump_size < 1) panic("vm_page_setup_clump:: clump_size must be >= 1");
-    if((vm_clump_size & (vm_clump_size-1)) != 0)  panic("vm_page_setup_clump:: clump_size must be a power of 2");
+       if (vm_clump_size > MAX_CLUMP_SIZE) {
+               panic("vm_page_setup_clump:: clump_size is too large!");
+       }
+       if (vm_clump_size < 1) {
+               panic("vm_page_setup_clump:: clump_size must be >= 1");
+       }
+       if ((vm_clump_size & (vm_clump_size - 1)) != 0) {
+               panic("vm_page_setup_clump:: clump_size must be a power of 2");
+       }

 
 

-    vm_clump_promote_threshold = vm_clump_size;
-    vm_clump_mask = vm_clump_size - 1;
-    for(vm_clump_shift=0, n=vm_clump_size; n>1; n>>=1, vm_clump_shift++);
+       vm_clump_promote_threshold = vm_clump_size;
+       vm_clump_mask = vm_clump_size - 1;
+       for (vm_clump_shift = 0, n = vm_clump_size; n > 1; n >>= 1, vm_clump_shift++) {
+               ;
+       }

 
 #if DEVELOPMENT || DEBUG
 
 #if DEVELOPMENT || DEBUG

-    bzero(vm_clump_stats, sizeof(vm_clump_stats));
-    vm_clump_allocs = vm_clump_inserts = vm_clump_inrange = vm_clump_promotes = 0;
+       bzero(vm_clump_stats, sizeof(vm_clump_stats));
+       vm_clump_allocs = vm_clump_inserts = vm_clump_inrange = vm_clump_promotes = 0;

 #endif  /*  if DEVELOPMENT || DEBUG */
 }
 
 #endif  /*  if DEVELOPMENT || DEBUG */
 }
 

-#endif /* #if defined (__x86_64__) */
+#endif  /* #if defined (__x86_64__) */

 
 

-#define COLOR_GROUPS_TO_STEAL  4
+#define COLOR_GROUPS_TO_STEAL   4

 
 /* Called once during statup, once the cache geometry is known.
  */
 static void
 vm_page_set_colors( void )
 {
 
 /* Called once during statup, once the cache geometry is known.
  */
 static void
 vm_page_set_colors( void )
 {

-       unsigned int    n, override;
+       unsigned int    n, override;

 
 

-#if defined (__x86_64__)       
+#if defined (__x86_64__)

        /* adjust #colors because we need to color outside the clump boundary */
        vm_cache_geometry_colors >>= vm_clump_shift;
 #endif
        /* adjust #colors because we need to color outside the clump boundary */
        vm_cache_geometry_colors >>= vm_clump_shift;
 #endif

-       if ( PE_parse_boot_argn("colors", &override, sizeof (override)) )               /* colors specified as a boot-arg? */
-               n = override;   
-       else if ( vm_cache_geometry_colors )                    /* do we know what the cache geometry is? */
+       if (PE_parse_boot_argn("colors", &override, sizeof(override))) {                /* colors specified as a boot-arg? */
+               n = override;
+       } else if (vm_cache_geometry_colors) {                  /* do we know what the cache geometry is? */

                n = vm_cache_geometry_colors;
                n = vm_cache_geometry_colors;

-       else    n = DEFAULT_COLORS;                             /* use default if all else fails */
-
-       if ( n == 0 )
+       } else {
+               n = DEFAULT_COLORS;                             /* use default if all else fails */
+       }
+       if (n == 0) {

                n = 1;
                n = 1;

-       if ( n > MAX_COLORS )
+       }
+       if (n > MAX_COLORS) {

                n = MAX_COLORS;
                n = MAX_COLORS;

-               
+       }
+

        /* the count must be a power of 2  */
        /* the count must be a power of 2  */

-       if ( ( n & (n - 1)) != 0  )
-               n = DEFAULT_COLORS;                             /* use default if all else fails */
-       
+       if ((n & (n - 1)) != 0) {
+               n = DEFAULT_COLORS;                             /* use default if all else fails */
+       }

        vm_colors = n;
        vm_color_mask = n - 1;
 
        vm_free_magazine_refill_limit = vm_colors * COLOR_GROUPS_TO_STEAL;
 
 #if defined (__x86_64__)
        vm_colors = n;
        vm_color_mask = n - 1;
 
        vm_free_magazine_refill_limit = vm_colors * COLOR_GROUPS_TO_STEAL;
 
 #if defined (__x86_64__)

-        /* adjust for reduction in colors due to clumping and multiple cores */
-       if (real_ncpus)
+       /* adjust for reduction in colors due to clumping and multiple cores */
+       if (real_ncpus) {

                vm_free_magazine_refill_limit *= (vm_clump_size * real_ncpus);
                vm_free_magazine_refill_limit *= (vm_clump_size * real_ncpus);

+       }

 #endif
 }
 
 #endif
 }
 

+/*
+ * During single threaded early boot we don't initialize all pages.
+ * This avoids some delay during boot. They'll be initialized and
+ * added to the free list as needed or after we are multithreaded by
+ * what becomes the pageout thread.
+ */
+static boolean_t fill = FALSE;
+static unsigned int fillval;
+uint_t vm_delayed_count = 0;    /* when non-zero, indicates we may have more pages to init */
+ppnum_t delay_above_pnum = PPNUM_MAX;

 
 

-lck_grp_t              vm_page_lck_grp_free;
-lck_grp_t              vm_page_lck_grp_queue;
-lck_grp_t              vm_page_lck_grp_local;
-lck_grp_t              vm_page_lck_grp_purge;
-lck_grp_t              vm_page_lck_grp_alloc;
-lck_grp_t              vm_page_lck_grp_bucket;
-lck_grp_attr_t         vm_page_lck_grp_attr;
-lck_attr_t             vm_page_lck_attr;
+/*
+ * For x86 first 8 Gig initializes quickly and gives us lots of lowmem + mem above to start off with.
+ * If ARM ever uses delayed page initialization, this value may need to be quite different.
+ */
+#define DEFAULT_DELAY_ABOVE_PHYS_GB (8)

 
 

+/*
+ * When we have to dip into more delayed pages due to low memory, free up
+ * a large chunk to get things back to normal. This avoids contention on the
+ * delayed code allocating page by page.
+ */
+#define VM_DELAY_PAGE_CHUNK ((1024 * 1024 * 1024) / PAGE_SIZE)

 
 

-__private_extern__ void
-vm_page_init_lck_grp(void)
+/*
+ * Get and initialize the next delayed page.
+ */
+static vm_page_t
+vm_get_delayed_page(int grab_options)

 {
 {

+       vm_page_t p;
+       ppnum_t   pnum;
+
+       /*
+        * Get a new page if we have one.
+        */
+       lck_mtx_lock(&vm_page_queue_free_lock);
+       if (vm_delayed_count == 0) {
+               lck_mtx_unlock(&vm_page_queue_free_lock);
+               return NULL;
+       }
+       if (!pmap_next_page(&pnum)) {
+               vm_delayed_count = 0;
+               lck_mtx_unlock(&vm_page_queue_free_lock);
+               return NULL;
+       }
+
+       assert(vm_delayed_count > 0);
+       --vm_delayed_count;
+
+#if defined(__x86_64__)
+       /* x86 cluster code requires increasing phys_page in vm_pages[] */
+       if (vm_pages_count > 0) {
+               assert(pnum > vm_pages[vm_pages_count - 1].vmp_phys_page);
+       }
+#endif
+       p = &vm_pages[vm_pages_count];
+       assert(p < vm_page_array_ending_addr);
+       vm_page_init(p, pnum, FALSE);
+       ++vm_pages_count;
+       ++vm_page_pages;
+       lck_mtx_unlock(&vm_page_queue_free_lock);
+

        /*
        /*

-        * initialze the vm_page lock world
+        * These pages were initially counted as wired, undo that now.

         */
         */

-       lck_grp_attr_setdefault(&vm_page_lck_grp_attr);
-       lck_grp_init(&vm_page_lck_grp_free, "vm_page_free", &vm_page_lck_grp_attr);
-       lck_grp_init(&vm_page_lck_grp_queue, "vm_page_queue", &vm_page_lck_grp_attr);
-       lck_grp_init(&vm_page_lck_grp_local, "vm_page_queue_local", &vm_page_lck_grp_attr);
-       lck_grp_init(&vm_page_lck_grp_purge, "vm_page_purge", &vm_page_lck_grp_attr);
-       lck_grp_init(&vm_page_lck_grp_alloc, "vm_page_alloc", &vm_page_lck_grp_attr);
-       lck_grp_init(&vm_page_lck_grp_bucket, "vm_page_bucket", &vm_page_lck_grp_attr);
-       lck_attr_setdefault(&vm_page_lck_attr);
-       lck_mtx_init_ext(&vm_page_alloc_lock, &vm_page_alloc_lock_ext, &vm_page_lck_grp_alloc, &vm_page_lck_attr);
+       if (grab_options & VM_PAGE_GRAB_Q_LOCK_HELD) {
+               LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
+       } else {
+               LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_NOTOWNED);
+               vm_page_lockspin_queues();
+       }
+       --vm_page_wire_count;
+       --vm_page_wire_count_initial;
+       if (vm_page_wire_count_on_boot != 0) {
+               --vm_page_wire_count_on_boot;
+       }
+       if (!(grab_options & VM_PAGE_GRAB_Q_LOCK_HELD)) {
+               vm_page_unlock_queues();
+       }

 
 

-       vm_compressor_init_locks();
+
+       if (fill) {
+               fillPage(pnum, fillval);
+       }
+       return p;

 }
 
 }
 

-#define ROUNDUP_NEXTP2(X) (1U << (32 - __builtin_clz((X) - 1)))
+static void vm_page_module_init_delayed(void);

 
 

+/*
+ * Free all remaining delayed pages to the free lists.
+ */

 void
 void

-vm_page_init_local_q()
+vm_free_delayed_pages(void)
+{
+       vm_page_t   p;
+       vm_page_t   list = NULL;
+       uint_t      cnt = 0;
+       vm_offset_t start_free_va;
+       int64_t     free_size;
+
+       while ((p = vm_get_delayed_page(VM_PAGE_GRAB_OPTIONS_NONE)) != NULL) {
+               if (vm_himemory_mode) {
+                       vm_page_release(p, FALSE);
+               } else {
+                       p->vmp_snext = list;
+                       list = p;
+               }
+               ++cnt;
+       }
+
+       /*
+        * Free the pages in reverse order if not himemory mode.
+        * Hence the low memory pages will be first on free lists. (LIFO)
+        */
+       while (list != NULL) {
+               p = list;
+               list = p->vmp_snext;
+               p->vmp_snext = NULL;
+               vm_page_release(p, FALSE);
+       }
+#if DEVELOPMENT || DEBUG
+       kprintf("vm_free_delayed_pages: initialized %d free pages\n", cnt);
+#endif
+
+       /*
+        * Free up any unused full pages at the end of the vm_pages[] array
+        */
+       start_free_va = round_page((vm_offset_t)&vm_pages[vm_pages_count]);
+
+#if defined(__x86_64__)
+       /*
+        * Since x86 might have used large pages for vm_pages[], we can't
+        * free starting in the middle of a partially used large page.
+        */
+       if (pmap_query_pagesize(kernel_pmap, start_free_va) == I386_LPGBYTES) {
+               start_free_va = ((start_free_va + I386_LPGMASK) & ~I386_LPGMASK);
+       }
+#endif
+       if (start_free_va < (vm_offset_t)vm_page_array_ending_addr) {
+               free_size = trunc_page((vm_offset_t)vm_page_array_ending_addr - start_free_va);
+               if (free_size > 0) {
+                       ml_static_mfree(start_free_va, (vm_offset_t)free_size);
+                       vm_page_array_ending_addr = (void *)start_free_va;
+
+                       /*
+                        * Note there's no locking here, as only this thread will ever change this value.
+                        * The reader, vm_page_diagnose, doesn't grab any locks for the counts it looks at.
+                        */
+                       vm_page_stolen_count -= (free_size >> PAGE_SHIFT);
+
+#if DEVELOPMENT || DEBUG
+                       kprintf("Freeing final unused %ld bytes from vm_pages[] at 0x%lx\n",
+                           (long)free_size, (long)start_free_va);
+#endif
+               }
+       }
+
+
+       /*
+        * now we can create the VM page array zone
+        */
+       vm_page_module_init_delayed();
+}
+
+/*
+ * Try and free up enough delayed pages to match a contig memory allocation.
+ */
+static void
+vm_free_delayed_pages_contig(
+       uint_t    npages,
+       ppnum_t   max_pnum,
+       ppnum_t   pnum_mask)

 {
 {

-       unsigned int            num_cpus;
-       unsigned int            i;
-       struct vplq             *t_local_q;
+       vm_page_t p;
+       ppnum_t   pnum;
+       uint_t    cnt = 0;
+
+       /*
+        * Treat 0 as the absolute max page number.
+        */
+       if (max_pnum == 0) {
+               max_pnum = PPNUM_MAX;
+       }
+
+       /*
+        * Free till we get a properly aligned start page
+        */
+       for (;;) {
+               p = vm_get_delayed_page(VM_PAGE_GRAB_OPTIONS_NONE);
+               if (p == NULL) {
+                       return;
+               }
+               pnum = VM_PAGE_GET_PHYS_PAGE(p);
+               vm_page_release(p, FALSE);
+               if (pnum >= max_pnum) {
+                       return;
+               }
+               if ((pnum & pnum_mask) == 0) {
+                       break;
+               }
+       }
+
+       /*
+        * Having a healthy pool of free pages will help performance. We don't
+        * want to fall back to the delayed code for every page allocation.
+        */
+       if (vm_page_free_count < VM_DELAY_PAGE_CHUNK) {
+               npages += VM_DELAY_PAGE_CHUNK;
+       }
+
+       /*
+        * Now free up the pages
+        */
+       for (cnt = 1; cnt < npages; ++cnt) {
+               p = vm_get_delayed_page(VM_PAGE_GRAB_OPTIONS_NONE);
+               if (p == NULL) {
+                       return;
+               }
+               vm_page_release(p, FALSE);
+       }
+}
+
+#define ROUNDUP_NEXTP2(X) (1U << (32 - __builtin_clz((X) - 1)))

 
 

-       num_cpus = ml_get_max_cpus();
+void
+vm_page_init_local_q(unsigned int num_cpus)
+{
+       struct vpl *t_local_q;

 
        /*
         * no point in this for a uni-processor system
         */
        if (num_cpus >= 2) {
 
        /*
         * no point in this for a uni-processor system
         */
        if (num_cpus >= 2) {

-#if KASAN
-               /* KASAN breaks the expectation of a size-aligned object by adding a
-                * redzone, so explicitly align. */
-               t_local_q = (struct vplq *)kalloc(num_cpus * sizeof(struct vplq) + VM_PACKED_POINTER_ALIGNMENT);
-               t_local_q = (void *)(((uintptr_t)t_local_q + (VM_PACKED_POINTER_ALIGNMENT-1)) & ~(VM_PACKED_POINTER_ALIGNMENT-1));
-#else
-               /* round the size up to the nearest power of two */
-               t_local_q = (struct vplq *)kalloc(ROUNDUP_NEXTP2(num_cpus * sizeof(struct vplq)));
-#endif
+               ml_cpu_info_t cpu_info;
+
+               /*
+                * Force the allocation alignment to a cacheline,
+                * because the `vpl` struct has a lock and will be taken
+                * cross CPU so we want to isolate the rest of the per-CPU
+                * data to avoid false sharing due to this lock being taken.
+                */
+
+               ml_cpu_get_info(&cpu_info);

 
 

-               for (i = 0; i < num_cpus; i++) {
-                       struct vpl      *lq;
+               t_local_q = zalloc_percpu_permanent(sizeof(struct vpl),
+                   cpu_info.cache_line_size - 1);

 
 

-                       lq = &t_local_q[i].vpl_un.vpl;
+               zpercpu_foreach(lq, t_local_q) {

                        VPL_LOCK_INIT(lq, &vm_page_lck_grp_local, &vm_page_lck_attr);
                        vm_page_queue_init(&lq->vpl_queue);
                        VPL_LOCK_INIT(lq, &vm_page_lck_grp_local, &vm_page_lck_attr);
                        vm_page_queue_init(&lq->vpl_queue);

-                       lq->vpl_count = 0;
-                       lq->vpl_internal_count = 0;
-                       lq->vpl_external_count = 0;

                }
                }

-               vm_page_local_q_count = num_cpus;

 
 

-               vm_page_local_q = (struct vplq *)t_local_q;
+               /* make the initialization visible to all cores */
+               os_atomic_store(&vm_page_local_q, t_local_q, release);

        }
 }
 
        }
 }
 


@@ -655,7 +891,9 @@ vm_page_init_local_q()
 void
 vm_init_before_launchd()
 {
 void
 vm_init_before_launchd()
 {

+       vm_page_lockspin_queues();

        vm_page_wire_count_on_boot = vm_page_wire_count;
        vm_page_wire_count_on_boot = vm_page_wire_count;

+       vm_page_unlock_queues();

 }
 
 
 }
 
 


@@ -669,93 +907,33 @@ vm_init_before_launchd()
  *     Each page cell is initialized and placed on the free list.
  *     Returns the range of available kernel virtual memory.
  */
  *     Each page cell is initialized and placed on the free list.
  *     Returns the range of available kernel virtual memory.
  */

-
+__startup_func

 void
 vm_page_bootstrap(
 void
 vm_page_bootstrap(

-       vm_offset_t             *startp,
-       vm_offset_t             *endp)
+       vm_offset_t             *startp,
+       vm_offset_t             *endp)

 {
 {

-       vm_page_t               m;
-       unsigned int            i;
-       unsigned int            log1;
-       unsigned int            log2;
-       unsigned int            size;
-
-       /*
-        *      Initialize the vm_page template.
-        */
-
-       m = &vm_page_template;
-       bzero(m, sizeof (*m));
-
-#if CONFIG_BACKGROUND_QUEUE
-       m->vmp_backgroundq.next = 0;
-       m->vmp_backgroundq.prev = 0;
-       m->vmp_in_background = FALSE;
-       m->vmp_on_backgroundq = FALSE;
-#endif
-
-       VM_PAGE_ZERO_PAGEQ_ENTRY(m);
-       m->vmp_listq.next = 0;
-       m->vmp_listq.prev = 0;
-       m->vmp_next_m = 0;
-
-       m->vmp_object = 0;                      /* reset later */
-       m->vmp_offset = (vm_object_offset_t) -1;        /* reset later */
-
-       m->vmp_wire_count = 0;
-       m->vmp_q_state = VM_PAGE_NOT_ON_Q;
-       m->vmp_laundry = FALSE;
-       m->vmp_reference = FALSE;
-       m->vmp_gobbled = FALSE;
-       m->vmp_private = FALSE;
-       m->vmp_unused_page_bits = 0;
-
-#if    !defined(__arm__) && !defined(__arm64__)
-       VM_PAGE_SET_PHYS_PAGE(m, 0);            /* reset later */
-#endif
-       m->vmp_busy = TRUE;
-       m->vmp_wanted = FALSE;
-       m->vmp_tabled = FALSE;
-       m->vmp_hashed = FALSE;
-       m->vmp_fictitious = FALSE;
-       m->vmp_pmapped = FALSE;
-       m->vmp_wpmapped = FALSE;
-       m->vmp_free_when_done = FALSE;
-       m->vmp_absent = FALSE;
-       m->vmp_error = FALSE;
-       m->vmp_dirty = FALSE;
-       m->vmp_cleaning = FALSE;
-       m->vmp_precious = FALSE;
-       m->vmp_clustered = FALSE;
-       m->vmp_overwriting = FALSE;
-       m->vmp_restart = FALSE;
-       m->vmp_unusual = FALSE;
-       m->vmp_cs_validated = FALSE;
-       m->vmp_cs_tainted = FALSE;
-       m->vmp_cs_nx = FALSE;
-       m->vmp_no_cache = FALSE;
-       m->vmp_reusable = FALSE;
-       m->vmp_xpmapped = FALSE;
-       m->vmp_written_by_kernel = FALSE;
-       m->vmp_unused_object_bits = 0;
+       unsigned int            i;
+       unsigned int            log1;
+       unsigned int            log2;
+       unsigned int            size;

 
        /*
         *      Initialize the page queues.
         */
 
        /*
         *      Initialize the page queues.
         */

-       vm_page_init_lck_grp();
-       
+

        lck_mtx_init_ext(&vm_page_queue_free_lock, &vm_page_queue_free_lock_ext, &vm_page_lck_grp_free, &vm_page_lck_attr);
        lck_mtx_init_ext(&vm_page_queue_lock, &vm_page_queue_lock_ext, &vm_page_lck_grp_queue, &vm_page_lck_attr);
        lck_mtx_init_ext(&vm_purgeable_queue_lock, &vm_purgeable_queue_lock_ext, &vm_page_lck_grp_purge, &vm_page_lck_attr);
        lck_mtx_init_ext(&vm_page_queue_free_lock, &vm_page_queue_free_lock_ext, &vm_page_lck_grp_free, &vm_page_lck_attr);
        lck_mtx_init_ext(&vm_page_queue_lock, &vm_page_queue_lock_ext, &vm_page_lck_grp_queue, &vm_page_lck_attr);
        lck_mtx_init_ext(&vm_purgeable_queue_lock, &vm_purgeable_queue_lock_ext, &vm_page_lck_grp_purge, &vm_page_lck_attr);

-    
+

        for (i = 0; i < PURGEABLE_Q_TYPE_MAX; i++) {
                int group;
 
                purgeable_queues[i].token_q_head = 0;
                purgeable_queues[i].token_q_tail = 0;
        for (i = 0; i < PURGEABLE_Q_TYPE_MAX; i++) {
                int group;
 
                purgeable_queues[i].token_q_head = 0;
                purgeable_queues[i].token_q_tail = 0;

-               for (group = 0; group < NUM_VOLATILE_GROUPS; group++)
-                       queue_init(&purgeable_queues[i].objq[group]);
+               for (group = 0; group < NUM_VOLATILE_GROUPS; group++) {
+                       queue_init(&purgeable_queues[i].objq[group]);
+               }

 
                purgeable_queues[i].type = i;
                purgeable_queues[i].new_pages = 0;
 
                purgeable_queues[i].type = i;
                purgeable_queues[i].new_pages = 0;


@@ -763,12 +941,14 @@ vm_page_bootstrap(
                purgeable_queues[i].debug_count_tokens = 0;
                purgeable_queues[i].debug_count_objects = 0;
 #endif
                purgeable_queues[i].debug_count_tokens = 0;
                purgeable_queues[i].debug_count_objects = 0;
 #endif

-       };
+       }
+       ;

        purgeable_nonvolatile_count = 0;
        queue_init(&purgeable_nonvolatile_queue);
        purgeable_nonvolatile_count = 0;
        queue_init(&purgeable_nonvolatile_queue);

-    
-       for (i = 0; i < MAX_COLORS; i++ )
+
+       for (i = 0; i < MAX_COLORS; i++) {

                vm_page_queue_init(&vm_page_queue_free[i].qhead);
                vm_page_queue_init(&vm_page_queue_free[i].qhead);

+       }

 
        vm_page_queue_init(&vm_lopage_queue_free);
        vm_page_queue_init(&vm_page_queue_active);
 
        vm_page_queue_init(&vm_lopage_queue_free);
        vm_page_queue_init(&vm_page_queue_active);


@@ -781,7 +961,7 @@ vm_page_bootstrap(
        vm_page_queue_init(&vm_page_queue_anonymous);
        queue_init(&vm_objects_wired);
 
        vm_page_queue_init(&vm_page_queue_anonymous);
        queue_init(&vm_objects_wired);
 

-       for ( i = 0; i <= VM_PAGE_MAX_SPECULATIVE_AGE_Q; i++ ) {
+       for (i = 0; i <= VM_PAGE_MAX_SPECULATIVE_AGE_Q; i++) {

                vm_page_queue_init(&vm_page_queue_speculative[i].age_q);
 
                vm_page_queue_speculative[i].age_ts.tv_sec = 0;
                vm_page_queue_init(&vm_page_queue_speculative[i].age_q);
 
                vm_page_queue_speculative[i].age_ts.tv_sec = 0;


@@ -797,8 +977,9 @@ vm_page_bootstrap(
 
        vm_page_background_target = (unsigned int)(atop_64(max_mem) / 25);
 
 
        vm_page_background_target = (unsigned int)(atop_64(max_mem) / 25);
 

-       if (vm_page_background_target > VM_PAGE_BACKGROUND_TARGET_MAX)
+       if (vm_page_background_target > VM_PAGE_BACKGROUND_TARGET_MAX) {

                vm_page_background_target = VM_PAGE_BACKGROUND_TARGET_MAX;
                vm_page_background_target = VM_PAGE_BACKGROUND_TARGET_MAX;

+       }

 
        vm_page_background_mode = VM_PAGE_BG_LEVEL_1;
        vm_page_background_exclude_external = 0;
 
        vm_page_background_mode = VM_PAGE_BG_LEVEL_1;
        vm_page_background_exclude_external = 0;


@@ -807,15 +988,16 @@ vm_page_bootstrap(
        PE_parse_boot_argn("vm_page_bg_exclude_external", &vm_page_background_exclude_external, sizeof(vm_page_background_exclude_external));
        PE_parse_boot_argn("vm_page_bg_target", &vm_page_background_target, sizeof(vm_page_background_target));
 
        PE_parse_boot_argn("vm_page_bg_exclude_external", &vm_page_background_exclude_external, sizeof(vm_page_background_exclude_external));
        PE_parse_boot_argn("vm_page_bg_target", &vm_page_background_target, sizeof(vm_page_background_target));
 

-       if (vm_page_background_mode > VM_PAGE_BG_LEVEL_1)
+       if (vm_page_background_mode > VM_PAGE_BG_LEVEL_1) {

                vm_page_background_mode = VM_PAGE_BG_LEVEL_1;
                vm_page_background_mode = VM_PAGE_BG_LEVEL_1;

+       }

 #endif
        vm_page_free_wanted = 0;
        vm_page_free_wanted_privileged = 0;
 #if CONFIG_SECLUDED_MEMORY
        vm_page_free_wanted_secluded = 0;
 #endif /* CONFIG_SECLUDED_MEMORY */
 #endif
        vm_page_free_wanted = 0;
        vm_page_free_wanted_privileged = 0;
 #if CONFIG_SECLUDED_MEMORY
        vm_page_free_wanted_secluded = 0;
 #endif /* CONFIG_SECLUDED_MEMORY */

-       
+

 #if defined (__x86_64__)
        /* this must be called before vm_page_set_colors() */
        vm_page_setup_clump();
 #if defined (__x86_64__)
        /* this must be called before vm_page_set_colors() */
        vm_page_setup_clump();


@@ -848,7 +1030,7 @@ vm_page_bootstrap(
 #if CONFIG_SECLUDED_MEMORY
        vm_page_non_speculative_pageable_states[VM_PAGE_ON_SECLUDED_Q] = 1;
 #endif /* CONFIG_SECLUDED_MEMORY */
 #if CONFIG_SECLUDED_MEMORY
        vm_page_non_speculative_pageable_states[VM_PAGE_ON_SECLUDED_Q] = 1;
 #endif /* CONFIG_SECLUDED_MEMORY */

-       
+

        bzero(vm_page_active_or_inactive_states, sizeof(vm_page_active_or_inactive_states));
        vm_page_active_or_inactive_states[VM_PAGE_ON_INACTIVE_INTERNAL_Q] = 1;
        vm_page_active_or_inactive_states[VM_PAGE_ON_INACTIVE_EXTERNAL_Q] = 1;
        bzero(vm_page_active_or_inactive_states, sizeof(vm_page_active_or_inactive_states));
        vm_page_active_or_inactive_states[VM_PAGE_ON_INACTIVE_INTERNAL_Q] = 1;
        vm_page_active_or_inactive_states[VM_PAGE_ON_INACTIVE_EXTERNAL_Q] = 1;


@@ -858,11 +1040,10 @@ vm_page_bootstrap(
        vm_page_active_or_inactive_states[VM_PAGE_ON_SECLUDED_Q] = 1;
 #endif /* CONFIG_SECLUDED_MEMORY */
 
        vm_page_active_or_inactive_states[VM_PAGE_ON_SECLUDED_Q] = 1;
 #endif /* CONFIG_SECLUDED_MEMORY */
 

-       for (i = 0; i < VM_KERN_MEMORY_FIRST_DYNAMIC; i++)
-       {
-               vm_allocation_sites_static[i].refcount = 2;
-               vm_allocation_sites_static[i].tag = i;
-               vm_allocation_sites[i] = &vm_allocation_sites_static[i];
+       for (vm_tag_t t = 0; t < VM_KERN_MEMORY_FIRST_DYNAMIC; t++) {
+               vm_allocation_sites_static[t].refcount = 2;
+               vm_allocation_sites_static[t].tag = t;
+               vm_allocation_sites[t] = &vm_allocation_sites_static[t];

        }
        vm_allocation_sites_static[VM_KERN_MEMORY_FIRST_DYNAMIC].refcount = 2;
        vm_allocation_sites_static[VM_KERN_MEMORY_FIRST_DYNAMIC].tag = VM_KERN_MEMORY_ANY;
        }
        vm_allocation_sites_static[VM_KERN_MEMORY_FIRST_DYNAMIC].refcount = 2;
        vm_allocation_sites_static[VM_KERN_MEMORY_FIRST_DYNAMIC].tag = VM_KERN_MEMORY_ANY;


@@ -871,11 +1052,7 @@ vm_page_bootstrap(
        /*
         *      Steal memory for the map and zone subsystems.
         */
        /*
         *      Steal memory for the map and zone subsystems.
         */

-#if CONFIG_GZALLOC
-       gzalloc_configure();
-#endif
-       kernel_debug_string_early("vm_map_steal_memory");
-       vm_map_steal_memory();
+       kernel_startup_initialize_upto(STARTUP_SUB_PMAP_STEAL);

 
        /*
         *      Allocate (and initialize) the virtual-to-physical
 
        /*
         *      Allocate (and initialize) the virtual-to-physical


@@ -891,8 +1068,9 @@ vm_page_bootstrap(
                unsigned int npages = pmap_free_pages();
 
                vm_page_bucket_count = 1;
                unsigned int npages = pmap_free_pages();
 
                vm_page_bucket_count = 1;

-               while (vm_page_bucket_count < npages)
+               while (vm_page_bucket_count < npages) {

                        vm_page_bucket_count <<= 1;
                        vm_page_bucket_count <<= 1;

+               }

        }
        vm_page_bucket_lock_count = (vm_page_bucket_count + BUCKETS_PER_LOCK - 1) / BUCKETS_PER_LOCK;
 
        }
        vm_page_bucket_lock_count = (vm_page_bucket_count + BUCKETS_PER_LOCK - 1) / BUCKETS_PER_LOCK;
 


@@ -906,19 +1084,22 @@ vm_page_bootstrap(
         *              B/2 - O
         */
        size = vm_page_bucket_count;
         *              B/2 - O
         */
        size = vm_page_bucket_count;

-       for (log1 = 0; size > 1; log1++) 
+       for (log1 = 0; size > 1; log1++) {

                size /= 2;
                size /= 2;

+       }

        size = sizeof(struct vm_object);
        size = sizeof(struct vm_object);

-       for (log2 = 0; size > 1; log2++) 
+       for (log2 = 0; size > 1; log2++) {

                size /= 2;
                size /= 2;

-       vm_page_hash_shift = log1/2 - log2 + 1;
-       
-       vm_page_bucket_hash = 1 << ((log1 + 1) >> 1);           /* Get (ceiling of sqrt of table size) */
-       vm_page_bucket_hash |= 1 << ((log1 + 1) >> 2);          /* Get (ceiling of quadroot of table size) */
-       vm_page_bucket_hash |= 1;                                                       /* Set bit and add 1 - always must be 1 to insure unique series */
+       }
+       vm_page_hash_shift = log1 / 2 - log2 + 1;
+
+       vm_page_bucket_hash = 1 << ((log1 + 1) >> 1);           /* Get (ceiling of sqrt of table size) */
+       vm_page_bucket_hash |= 1 << ((log1 + 1) >> 2);          /* Get (ceiling of quadroot of table size) */
+       vm_page_bucket_hash |= 1;                                                       /* Set bit and add 1 - always must be 1 to insure unique series */

 
 

-       if (vm_page_hash_mask & vm_page_bucket_count)
+       if (vm_page_hash_mask & vm_page_bucket_count) {

                printf("vm_page_bootstrap: WARNING -- strange page hash\n");
                printf("vm_page_bootstrap: WARNING -- strange page hash\n");

+       }

 
 #if VM_PAGE_BUCKETS_CHECK
 #if VM_PAGE_FAKE_BUCKETS
 
 #if VM_PAGE_BUCKETS_CHECK
 #if VM_PAGE_FAKE_BUCKETS


@@ -927,18 +1108,18 @@ vm_page_bootstrap(
         * any stomping there.
         */
        vm_page_fake_buckets = (vm_page_bucket_t *)
         * any stomping there.
         */
        vm_page_fake_buckets = (vm_page_bucket_t *)

-               pmap_steal_memory(vm_page_bucket_count *
-                                 sizeof(vm_page_bucket_t));
+           pmap_steal_memory(vm_page_bucket_count *
+           sizeof(vm_page_bucket_t));

        vm_page_fake_buckets_start = (vm_map_offset_t) vm_page_fake_buckets;
        vm_page_fake_buckets_end =
        vm_page_fake_buckets_start = (vm_map_offset_t) vm_page_fake_buckets;
        vm_page_fake_buckets_end =

-               vm_map_round_page((vm_page_fake_buckets_start +
-                                  (vm_page_bucket_count *
-                                   sizeof (vm_page_bucket_t))),
-                                 PAGE_MASK);
+           vm_map_round_page((vm_page_fake_buckets_start +
+           (vm_page_bucket_count *
+           sizeof(vm_page_bucket_t))),
+           PAGE_MASK);

        char *cp;
        for (cp = (char *)vm_page_fake_buckets_start;
        char *cp;
        for (cp = (char *)vm_page_fake_buckets_start;

-            cp < (char *)vm_page_fake_buckets_end;
-            cp++) {
+           cp < (char *)vm_page_fake_buckets_end;
+           cp++) {

                *cp = 0x5a;
        }
 #endif /* VM_PAGE_FAKE_BUCKETS */
                *cp = 0x5a;
        }
 #endif /* VM_PAGE_FAKE_BUCKETS */


@@ -946,13 +1127,13 @@ vm_page_bootstrap(
 
        kernel_debug_string_early("vm_page_buckets");
        vm_page_buckets = (vm_page_bucket_t *)
 
        kernel_debug_string_early("vm_page_buckets");
        vm_page_buckets = (vm_page_bucket_t *)

-               pmap_steal_memory(vm_page_bucket_count *
-                                 sizeof(vm_page_bucket_t));
+           pmap_steal_memory(vm_page_bucket_count *
+           sizeof(vm_page_bucket_t));

 
        kernel_debug_string_early("vm_page_bucket_locks");
        vm_page_bucket_locks = (lck_spin_t *)
 
        kernel_debug_string_early("vm_page_bucket_locks");
        vm_page_bucket_locks = (lck_spin_t *)

-               pmap_steal_memory(vm_page_bucket_lock_count *
-                                 sizeof(lck_spin_t));
+           pmap_steal_memory(vm_page_bucket_lock_count *
+           sizeof(lck_spin_t));

 
        for (i = 0; i < vm_page_bucket_count; i++) {
                vm_page_bucket_t *bucket = &vm_page_buckets[i];
 
        for (i = 0; i < vm_page_bucket_count; i++) {
                vm_page_bucket_t *bucket = &vm_page_buckets[i];


@@ -964,11 +1145,10 @@ vm_page_bootstrap(
 #endif /* MACH_PAGE_HASH_STATS */
        }
 
 #endif /* MACH_PAGE_HASH_STATS */
        }
 

-       for (i = 0; i < vm_page_bucket_lock_count; i++)
-               lck_spin_init(&vm_page_bucket_locks[i], &vm_page_lck_grp_bucket, &vm_page_lck_attr);
+       for (i = 0; i < vm_page_bucket_lock_count; i++) {
+               lck_spin_init(&vm_page_bucket_locks[i], &vm_page_lck_grp_bucket, &vm_page_lck_attr);
+       }

 
 

-       lck_spin_init(&vm_objects_wired_lock, &vm_page_lck_grp_bucket, &vm_page_lck_attr);
-       lck_spin_init(&vm_allocation_sites_lock, &vm_page_lck_grp_bucket, &vm_page_lck_attr);

        vm_tag_init();
 
 #if VM_PAGE_BUCKETS_CHECK
        vm_tag_init();
 
 #if VM_PAGE_BUCKETS_CHECK


@@ -993,103 +1173,127 @@ vm_page_bootstrap(
 
        /*
         *      Compute the initial "wire" count.
 
        /*
         *      Compute the initial "wire" count.

-        *      Up until now, the pages which have been set aside are not under 
+        *      Up until now, the pages which have been set aside are not under

         *      the VM system's control, so although they aren't explicitly
         *      wired, they nonetheless can't be moved. At this moment,
         *      all VM managed pages are "free", courtesy of pmap_startup.
         */
        assert((unsigned int) atop_64(max_mem) == atop_64(max_mem));
         *      the VM system's control, so although they aren't explicitly
         *      wired, they nonetheless can't be moved. At this moment,
         *      all VM managed pages are "free", courtesy of pmap_startup.
         */
        assert((unsigned int) atop_64(max_mem) == atop_64(max_mem));

-       vm_page_wire_count = ((unsigned int) atop_64(max_mem)) - vm_page_free_count - vm_lopage_free_count;     /* initial value */
+       vm_page_wire_count = ((unsigned int) atop_64(max_mem)) -
+           vm_page_free_count - vm_lopage_free_count;

 #if CONFIG_SECLUDED_MEMORY
        vm_page_wire_count -= vm_page_secluded_count;
 #endif
        vm_page_wire_count_initial = vm_page_wire_count;
 #if CONFIG_SECLUDED_MEMORY
        vm_page_wire_count -= vm_page_secluded_count;
 #endif
        vm_page_wire_count_initial = vm_page_wire_count;

-       vm_page_pages_initial = vm_page_pages;

 
 

-       printf("vm_page_bootstrap: %d free pages and %d wired pages\n",
-              vm_page_free_count, vm_page_wire_count);
+       /* capture this for later use */
+       booter_size = ml_get_booter_memory_size();
+
+       printf("vm_page_bootstrap: %d free pages, %d wired pages, (up to %d of which are delayed free)\n",
+           vm_page_free_count, vm_page_wire_count, vm_delayed_count);

 
        kernel_debug_string_early("vm_page_bootstrap complete");
 
        kernel_debug_string_early("vm_page_bootstrap complete");

-       simple_lock_init(&vm_paging_lock, 0);

 }
 
 }
 

-#ifndef        MACHINE_PAGES
+#ifndef MACHINE_PAGES

 /*
 /*

- *     We implement pmap_steal_memory and pmap_startup with the help
- *     of two simpler functions, pmap_virtual_space and pmap_next_page.
+ * This is the early boot time allocator for data structures needed to bootstrap the VM system.
+ * On x86 it will allocate large pages if size is sufficiently large. We don't need to do this
+ * on ARM yet, due to the combination of a large base page size and smaller RAM devices.

  */
  */

-
-void *
-pmap_steal_memory(
-       vm_size_t size)
+static void *
+pmap_steal_memory_internal(
+       vm_size_t size,
+       boolean_t might_free)

 {
        kern_return_t kr;
 {
        kern_return_t kr;

-       vm_offset_t addr, vaddr;
+       vm_offset_t addr;
+       vm_offset_t map_addr;

        ppnum_t phys_page;
 
        /*
        ppnum_t phys_page;
 
        /*

-        *      We round the size to a round multiple.
+        * Size needs to be aligned to word size.

         */
         */

-
-       size = (size + sizeof (void *) - 1) &~ (sizeof (void *) - 1);
+       size = (size + sizeof(void *) - 1) & ~(sizeof(void *) - 1);

 
        /*
 
        /*

-        *      If this is the first call to pmap_steal_memory,
-        *      we have to initialize ourself.
+        * On the first call, get the initial values for virtual address space
+        * and page align them.

         */
         */

-

        if (virtual_space_start == virtual_space_end) {
                pmap_virtual_space(&virtual_space_start, &virtual_space_end);
        if (virtual_space_start == virtual_space_end) {
                pmap_virtual_space(&virtual_space_start, &virtual_space_end);

+               virtual_space_start = round_page(virtual_space_start);
+               virtual_space_end = trunc_page(virtual_space_end);

 
 

+#if defined(__x86_64__)

                /*
                /*

-                *      The initial values must be aligned properly, and
-                *      we don't trust the pmap module to do it right.
+                * Release remaining unused section of preallocated KVA and the 4K page tables
+                * that map it. This makes the VA available for large page mappings.

                 */
                 */

-
-               virtual_space_start = round_page(virtual_space_start);
-               virtual_space_end = trunc_page(virtual_space_end);
+               Idle_PTs_release(virtual_space_start, virtual_space_end);
+#endif

        }
 
        /*
        }
 
        /*

-        *      Allocate virtual memory for this request.
+        * Allocate the virtual space for this request. On x86, we'll align to a large page
+        * address if the size is big enough to back with at least 1 large page.

         */
         */

-
+#if defined(__x86_64__)
+       if (size >= I386_LPGBYTES) {
+               virtual_space_start = ((virtual_space_start + I386_LPGMASK) & ~I386_LPGMASK);
+       }
+#endif

        addr = virtual_space_start;
        virtual_space_start += size;
 
        //kprintf("pmap_steal_memory: %08lX - %08lX; size=%08lX\n", (long)addr, (long)virtual_space_start, (long)size); /* (TEST/DEBUG) */
 
        /*
        addr = virtual_space_start;
        virtual_space_start += size;
 
        //kprintf("pmap_steal_memory: %08lX - %08lX; size=%08lX\n", (long)addr, (long)virtual_space_start, (long)size); /* (TEST/DEBUG) */
 
        /*

-        *      Allocate and map physical pages to back new virtual pages.
+        * Allocate and map physical pages to back the new virtual space.

         */
         */

-
-       for (vaddr = round_page(addr);
-            vaddr < addr + size;
-            vaddr += PAGE_SIZE) {
-
-               if (!pmap_next_page_hi(&phys_page))
-                       panic("pmap_steal_memory() size: 0x%llx\n", (uint64_t)size);
-
+       map_addr = round_page(addr);
+       while (map_addr < addr + size) {
+#if defined(__x86_64__)

                /*
                /*

-                *      XXX Logically, these mappings should be wired,
-                *      but some pmap modules barf if they are.
+                * Back with a large page if properly aligned on x86

                 */
                 */

-#if defined(__LP64__)
-#ifdef __arm64__
-               /* ARM64_TODO: verify that we really don't need this */
-#else
-               pmap_pre_expand(kernel_pmap, vaddr);
+               if ((map_addr & I386_LPGMASK) == 0 &&
+                   map_addr + I386_LPGBYTES <= addr + size &&
+                   pmap_pre_expand_large(kernel_pmap, map_addr) == KERN_SUCCESS &&
+                   pmap_next_page_large(&phys_page) == KERN_SUCCESS) {
+                       kr = pmap_enter(kernel_pmap, map_addr, phys_page,
+                           VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE,
+                           VM_WIMG_USE_DEFAULT | VM_MEM_SUPERPAGE, FALSE);
+
+                       if (kr != KERN_SUCCESS) {
+                               panic("pmap_steal_memory: pmap_enter() large failed, new_addr=%#lx, phys_page=%u",
+                                   (unsigned long)map_addr, phys_page);
+                       }
+                       map_addr += I386_LPGBYTES;
+                       vm_page_wire_count += I386_LPGBYTES >> PAGE_SHIFT;
+                       vm_page_stolen_count += I386_LPGBYTES >> PAGE_SHIFT;
+                       vm_page_kern_lpage_count++;
+                       continue;
+               }

 #endif
 #endif

+
+               if (!pmap_next_page_hi(&phys_page, might_free)) {
+                       panic("pmap_steal_memory() size: 0x%llx\n", (uint64_t)size);
+               }
+
+#if defined(__x86_64__)
+               pmap_pre_expand(kernel_pmap, map_addr);

 #endif
 
 #endif
 

-               kr = pmap_enter(kernel_pmap, vaddr, phys_page,
-                               VM_PROT_READ|VM_PROT_WRITE, VM_PROT_NONE,
-                               VM_WIMG_USE_DEFAULT, FALSE);
+               kr = pmap_enter(kernel_pmap, map_addr, phys_page,
+                   VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE,
+                   VM_WIMG_USE_DEFAULT, FALSE);

 
                if (kr != KERN_SUCCESS) {
 
                if (kr != KERN_SUCCESS) {

-                       panic("pmap_steal_memory() pmap_enter failed, vaddr=%#lx, phys_page=%u",
-                             (unsigned long)vaddr, phys_page);
+                       panic("pmap_steal_memory() pmap_enter failed, map_addr=%#lx, phys_page=%u",
+                           (unsigned long)map_addr, phys_page);

                }
                }

+               map_addr += PAGE_SIZE;

 
                /*
                 * Account for newly stolen memory
 
                /*
                 * Account for newly stolen memory


@@ -1098,22 +1302,46 @@ pmap_steal_memory(
                vm_page_stolen_count++;
        }
 
                vm_page_stolen_count++;
        }
 

+#if defined(__x86_64__)
+       /*
+        * The call with might_free is currently the last use of pmap_steal_memory*().
+        * Notify the pmap layer to record which high pages were allocated so far.
+        */
+       if (might_free) {
+               pmap_hi_pages_done();
+       }
+#endif

 #if KASAN
        kasan_notify_address(round_page(addr), size);
 #endif
        return (void *) addr;
 }
 
 #if KASAN
        kasan_notify_address(round_page(addr), size);
 #endif
        return (void *) addr;
 }
 

+void *
+pmap_steal_memory(
+       vm_size_t size)
+{
+       return pmap_steal_memory_internal(size, FALSE);
+}
+
+void *
+pmap_steal_freeable_memory(
+       vm_size_t size)
+{
+       return pmap_steal_memory_internal(size, TRUE);
+}
+

 #if CONFIG_SECLUDED_MEMORY
 /* boot-args to control secluded memory */
 #if CONFIG_SECLUDED_MEMORY
 /* boot-args to control secluded memory */

-unsigned int secluded_mem_mb = 0;      /* # of MBs of RAM to seclude */
-int secluded_for_iokit = 1;            /* IOKit can use secluded memory */
-int secluded_for_apps = 1;             /* apps can use secluded memory */
-int secluded_for_filecache = 2;                /* filecache can use seclude memory */
+unsigned int secluded_mem_mb = 0;       /* # of MBs of RAM to seclude */
+int secluded_for_iokit = 1;             /* IOKit can use secluded memory */
+int secluded_for_apps = 1;              /* apps can use secluded memory */
+int secluded_for_filecache = 2;         /* filecache can use seclude memory */

 #if 11
 int secluded_for_fbdp = 0;
 #endif
 uint64_t secluded_shutoff_trigger = 0;
 #if 11
 int secluded_for_fbdp = 0;
 #endif
 uint64_t secluded_shutoff_trigger = 0;

+uint64_t secluded_shutoff_headroom = 150 * 1024 * 1024; /* original value from N56 */

 #endif /* CONFIG_SECLUDED_MEMORY */
 
 
 #endif /* CONFIG_SECLUDED_MEMORY */
 
 


@@ -1122,16 +1350,18 @@ extern void patch_low_glo_vm_page_info(void *, void *, uint32_t);
 unsigned int vm_first_phys_ppnum = 0;
 #endif
 
 unsigned int vm_first_phys_ppnum = 0;
 #endif
 

-

 void vm_page_release_startup(vm_page_t mem);
 void
 pmap_startup(
 void vm_page_release_startup(vm_page_t mem);
 void
 pmap_startup(

-       vm_offset_t *startp,
-       vm_offset_t *endp)
+       vm_offset_t     *startp,
+       vm_offset_t     *endp)

 {
 {

-       unsigned int i, npages, pages_initialized, fill, fillval;
-       ppnum_t         phys_page;
-       addr64_t        tmpaddr;
+       unsigned int    i, npages;
+       ppnum_t         phys_page;
+       uint64_t        mem_sz;
+       uint64_t        start_ns;
+       uint64_t        now_ns;
+       uint_t          low_page_count = 0;

 
 #if    defined(__LP64__)
        /*
 
 #if    defined(__LP64__)
        /*


@@ -1139,256 +1369,282 @@ pmap_startup(
         * for VM_PAGE_PACK_PTR (it clips off the low-order
         * 6 bits of the pointer)
         */
         * for VM_PAGE_PACK_PTR (it clips off the low-order
         * 6 bits of the pointer)
         */

-       if (virtual_space_start != virtual_space_end)
+       if (virtual_space_start != virtual_space_end) {

                virtual_space_start = round_page(virtual_space_start);
                virtual_space_start = round_page(virtual_space_start);

+       }

 #endif
 
        /*
 #endif
 
        /*

-        *      We calculate how many page frames we will have
-        *      and then allocate the page structures in one chunk.
+        * We calculate how many page frames we will have
+        * and then allocate the page structures in one chunk.
+        *
+        * Note that the calculation here doesn't take into account
+        * the memory needed to map what's being allocated, i.e. the page
+        * table entries. So the actual number of pages we get will be
+        * less than this. To do someday: include that in the computation.

         */
         */

+       mem_sz = pmap_free_pages() * (uint64_t)PAGE_SIZE;
+       mem_sz += round_page(virtual_space_start) - virtual_space_start;        /* Account for any slop */
+       npages = (uint_t)(mem_sz / (PAGE_SIZE + sizeof(*vm_pages)));    /* scaled to include the vm_page_ts */

 
 

-       tmpaddr = (addr64_t)pmap_free_pages() * (addr64_t)PAGE_SIZE;    /* Get the amount of memory left */
-       tmpaddr = tmpaddr + (addr64_t)(round_page(virtual_space_start) - virtual_space_start);  /* Account for any slop */
-       npages = (unsigned int)(tmpaddr / (addr64_t)(PAGE_SIZE + sizeof(*vm_pages)));   /* Figure size of all vm_page_ts, including enough to hold the vm_page_ts */
+       vm_pages = (vm_page_t) pmap_steal_freeable_memory(npages * sizeof *vm_pages);

 
 

-       vm_pages = (vm_page_t) pmap_steal_memory(npages * sizeof *vm_pages);
-
-       /*
-        *      Initialize the page frames.
-        */
-       kernel_debug_string_early("Initialize the page frames");
-
-       vm_page_array_beginning_addr = &vm_pages[0];
-       vm_page_array_ending_addr = &vm_pages[npages];
-
-       for (i = 0, pages_initialized = 0; i < npages; i++) {
-               if (!pmap_next_page(&phys_page))
-                       break;
-#if defined(__arm__) || defined(__arm64__)
-               if (pages_initialized == 0) {
-                       vm_first_phys_ppnum = phys_page;
-                       patch_low_glo_vm_page_info((void *)vm_page_array_beginning_addr, (void *)vm_page_array_ending_addr, vm_first_phys_ppnum);
-               }
-               assert((i + vm_first_phys_ppnum) == phys_page);
-#endif
-               if (pages_initialized == 0 || phys_page < vm_page_lowest)
-                       vm_page_lowest = phys_page;
-
-               vm_page_init(&vm_pages[i], phys_page, FALSE);
-               vm_page_pages++;
-               pages_initialized++;
-       }
-       vm_pages_count = pages_initialized;
-       vm_page_array_boundary = &vm_pages[pages_initialized];
-
-#if    defined(__LP64__)
-
-       if ((vm_page_t)(VM_PAGE_UNPACK_PTR(VM_PAGE_PACK_PTR(&vm_pages[0]))) != &vm_pages[0])
-               panic("VM_PAGE_PACK_PTR failed on &vm_pages[0] - %p", (void *)&vm_pages[0]);
-
-       if ((vm_page_t)(VM_PAGE_UNPACK_PTR(VM_PAGE_PACK_PTR(&vm_pages[vm_pages_count-1]))) != &vm_pages[vm_pages_count-1])
-               panic("VM_PAGE_PACK_PTR failed on &vm_pages[vm_pages_count-1] - %p", (void *)&vm_pages[vm_pages_count-1]);
-#endif
-       kernel_debug_string_early("page fill/release");

        /*
         * Check if we want to initialize pages to a known value
         */
        /*
         * Check if we want to initialize pages to a known value
         */

-       fill = 0;                                                               /* Assume no fill */
-       if (PE_parse_boot_argn("fill", &fillval, sizeof (fillval))) fill = 1;                   /* Set fill */
-#if    DEBUG
+       if (PE_parse_boot_argn("fill", &fillval, sizeof(fillval))) {
+               fill = TRUE;
+       }
+#if     DEBUG

        /* This slows down booting the DEBUG kernel, particularly on
         * large memory systems, but is worthwhile in deterministically
         * trapping uninitialized memory usage.
         */
        /* This slows down booting the DEBUG kernel, particularly on
         * large memory systems, but is worthwhile in deterministically
         * trapping uninitialized memory usage.
         */

-       if (fill == 0) {
-               fill = 1;
+       if (!fill) {
+               fill = TRUE;

                fillval = 0xDEB8F177;
        }
 #endif
                fillval = 0xDEB8F177;
        }
 #endif

-       if (fill)
+       if (fill) {

                kprintf("Filling vm_pages with pattern: 0x%x\n", fillval);
                kprintf("Filling vm_pages with pattern: 0x%x\n", fillval);

+       }

 
 #if CONFIG_SECLUDED_MEMORY
 
 #if CONFIG_SECLUDED_MEMORY

-       /* default: no secluded mem */
+       /*
+        * Figure out how much secluded memory to have before we start
+        * release pages to free lists.
+        * The default, if specified nowhere else, is no secluded mem.
+        */

        secluded_mem_mb = 0;
        secluded_mem_mb = 0;

-       if (max_mem > 1*1024*1024*1024) {
+       if (max_mem > 1 * 1024 * 1024 * 1024) {

                /* default to 90MB for devices with > 1GB of RAM */
                secluded_mem_mb = 90;
        }
        /* override with value from device tree, if provided */
        PE_get_default("kern.secluded_mem_mb",
                /* default to 90MB for devices with > 1GB of RAM */
                secluded_mem_mb = 90;
        }
        /* override with value from device tree, if provided */
        PE_get_default("kern.secluded_mem_mb",

-                      &secluded_mem_mb, sizeof(secluded_mem_mb));
+           &secluded_mem_mb, sizeof(secluded_mem_mb));

        /* override with value from boot-args, if provided */
        PE_parse_boot_argn("secluded_mem_mb",
        /* override with value from boot-args, if provided */
        PE_parse_boot_argn("secluded_mem_mb",

-                          &secluded_mem_mb,
-                          sizeof (secluded_mem_mb));
+           &secluded_mem_mb,
+           sizeof(secluded_mem_mb));

 
        vm_page_secluded_target = (unsigned int)
 
        vm_page_secluded_target = (unsigned int)

-               ((secluded_mem_mb * 1024ULL * 1024ULL) / PAGE_SIZE);
+           ((secluded_mem_mb * 1024ULL * 1024ULL) / PAGE_SIZE);

        PE_parse_boot_argn("secluded_for_iokit",
        PE_parse_boot_argn("secluded_for_iokit",

-                          &secluded_for_iokit,
-                          sizeof (secluded_for_iokit));
+           &secluded_for_iokit,
+           sizeof(secluded_for_iokit));

        PE_parse_boot_argn("secluded_for_apps",
        PE_parse_boot_argn("secluded_for_apps",

-                          &secluded_for_apps,
-                          sizeof (secluded_for_apps));
+           &secluded_for_apps,
+           sizeof(secluded_for_apps));

        PE_parse_boot_argn("secluded_for_filecache",
        PE_parse_boot_argn("secluded_for_filecache",

-                          &secluded_for_filecache,
-                          sizeof (secluded_for_filecache));
+           &secluded_for_filecache,
+           sizeof(secluded_for_filecache));

 #if 11
        PE_parse_boot_argn("secluded_for_fbdp",
 #if 11
        PE_parse_boot_argn("secluded_for_fbdp",

-                          &secluded_for_fbdp,
-                          sizeof (secluded_for_fbdp));
+           &secluded_for_fbdp,
+           sizeof(secluded_for_fbdp));

 #endif
 
        /*
 #endif
 
        /*

-        * On small devices, allow a large app to effectively suppress
-        * secluded memory until it exits.
+        * Allow a really large app to effectively use secluded memory until it exits.

         */
         */

-       if (max_mem <= 1 * 1024 * 1024 * 1024 && vm_page_secluded_target != 0) {
-
+       if (vm_page_secluded_target != 0) {

                /*
                /*

-                * Get an amount from boot-args, else use 500MB.
-                * 500MB was chosen from a Peace daemon tentpole test which used munch
-                * to induce jetsam thrashing of false idle daemons.
+                * Get an amount from boot-args, else use 1/2 of max_mem.
+                * 1/2 max_mem was chosen from a Peace daemon tentpole test which
+                * used munch to induce jetsam thrashing of false idle daemons on N56.

                 */
                int secluded_shutoff_mb;
                if (PE_parse_boot_argn("secluded_shutoff_mb", &secluded_shutoff_mb,
                 */
                int secluded_shutoff_mb;
                if (PE_parse_boot_argn("secluded_shutoff_mb", &secluded_shutoff_mb,

-                   sizeof (secluded_shutoff_mb)))
+                   sizeof(secluded_shutoff_mb))) {

                        secluded_shutoff_trigger = (uint64_t)secluded_shutoff_mb * 1024 * 1024;
                        secluded_shutoff_trigger = (uint64_t)secluded_shutoff_mb * 1024 * 1024;

-               else
-                       secluded_shutoff_trigger = 500 * 1024 * 1024;
+               } else {
+                       secluded_shutoff_trigger = max_mem / 2;
+               }

 
 

-               if (secluded_shutoff_trigger != 0)
-                       secluded_suppression_init();
+               /* ensure the headroom value is sensible and avoid underflows */
+               assert(secluded_shutoff_trigger == 0 || secluded_shutoff_trigger > secluded_shutoff_headroom);

        }
 
 #endif /* CONFIG_SECLUDED_MEMORY */
 
        }
 
 #endif /* CONFIG_SECLUDED_MEMORY */
 

+#if defined(__x86_64__)
+

        /*
        /*

-        * By default release pages in reverse order so that physical pages
-        * initially get allocated in ascending addresses. This keeps
-        * the devices (which must address physical memory) happy if
-        * they require several consecutive pages.
-        *
-        * For debugging, you can reverse this ordering and/or fill
-        * all pages with a known value.
+        * Decide how much memory we delay freeing at boot time.

         */
         */

-       if (vm_himemory_mode == 2) {
-               for (i = 0; i < pages_initialized; i++) {
-                       if (fill)
-                               fillPage(VM_PAGE_GET_PHYS_PAGE(&vm_pages[i]), fillval);
-                       vm_page_release_startup(&vm_pages[i]);
-               }
+       uint32_t delay_above_gb;
+       if (!PE_parse_boot_argn("delay_above_gb", &delay_above_gb, sizeof(delay_above_gb))) {
+               delay_above_gb = DEFAULT_DELAY_ABOVE_PHYS_GB;
+       }
+
+       if (delay_above_gb == 0) {
+               delay_above_pnum = PPNUM_MAX;

        } else {
        } else {

-               for (i = pages_initialized; i-- > 0; ) {
-                       if (fill)
-                               fillPage(VM_PAGE_GET_PHYS_PAGE(&vm_pages[i]), fillval);
-                       vm_page_release_startup(&vm_pages[i]);
-               }
+               delay_above_pnum = delay_above_gb * (1024 * 1024 * 1024 / PAGE_SIZE);

        }
 
        }
 

-       VM_CHECK_MEMORYSTATUS;
-       
-#if 0
-       {
-               vm_page_t xx, xxo, xxl;
-               int i, j, k, l;
-       
-               j = 0;                                                                                                  /* (BRINGUP) */
-               xxl = 0;
-               
-               for( i = 0; i < vm_colors; i++ ) {
-                       queue_iterate(&vm_page_queue_free[i].qhead,
-                                     xx,
-                                     vm_page_t,
-                                     vmp_pageq) {      /* BRINGUP */
-                               j++;                                                                                            /* (BRINGUP) */
-                               if(j > vm_page_free_count) {                                            /* (BRINGUP) */
-                                       panic("pmap_startup: too many pages, xx = %08X, xxl = %08X\n", xx, xxl);
-                               }
-                               
-                               l = vm_page_free_count - j;                                                     /* (BRINGUP) */
-                               k = 0;                                                                                          /* (BRINGUP) */
-                               
-                               if(((j - 1) & 0xFFFF) == 0) kprintf("checking number %d of %d\n", j, vm_page_free_count);
-
-                               for(xxo = xx->pageq.next; xxo != &vm_page_queue_free[i].qhead; xxo = xxo->pageq.next) { /* (BRINGUP) */
-                                       k++;
-                                       if(k > l) panic("pmap_startup: too many in secondary check %d %d\n", k, l);
-                                       if((xx->phys_page & 0xFFFFFFFF) == (xxo->phys_page & 0xFFFFFFFF)) {     /* (BRINGUP) */
-                                               panic("pmap_startup: duplicate physaddr, xx = %08X, xxo = %08X\n", xx, xxo);
-                                       }
-                               }
+       /* make sure we have sane breathing room: 1G above low memory */
+       if (delay_above_pnum <= max_valid_low_ppnum) {
+               delay_above_pnum = max_valid_low_ppnum + ((1024 * 1024 * 1024) >> PAGE_SHIFT);
+       }

 
 

-                               xxl = xx;
-                       }
+       if (delay_above_pnum < PPNUM_MAX) {
+               printf("pmap_startup() delaying init/free of page nums > 0x%x\n", delay_above_pnum);
+       }
+
+#endif /* defined(__x86_64__) */
+
+       /*
+        * Initialize and release the page frames.
+        */
+       kernel_debug_string_early("page_frame_init");
+
+       vm_page_array_beginning_addr = &vm_pages[0];
+       vm_page_array_ending_addr = &vm_pages[npages];  /* used by ptr packing/unpacking code */
+#if VM_PAGE_PACKED_FROM_ARRAY
+       if (npages >= VM_PAGE_PACKED_FROM_ARRAY) {
+               panic("pmap_startup(): too many pages to support vm_page packing");
+       }
+#endif
+
+       vm_delayed_count = 0;
+
+       absolutetime_to_nanoseconds(mach_absolute_time(), &start_ns);
+       vm_pages_count = 0;
+       for (i = 0; i < npages; i++) {
+               /* Did we run out of pages? */
+               if (!pmap_next_page(&phys_page)) {
+                       break;
+               }
+
+               if (phys_page < max_valid_low_ppnum) {
+                       ++low_page_count;
+               }
+
+               /* Are we at high enough pages to delay the rest? */
+               if (low_page_count > vm_lopage_free_limit && phys_page > delay_above_pnum) {
+                       vm_delayed_count = pmap_free_pages();
+                       break;
+               }
+
+#if defined(__arm__) || defined(__arm64__)
+               if (i == 0) {
+                       vm_first_phys_ppnum = phys_page;
+                       patch_low_glo_vm_page_info((void *)vm_page_array_beginning_addr,
+                           (void *)vm_page_array_ending_addr, vm_first_phys_ppnum);
+               }
+               assert((i + vm_first_phys_ppnum) == phys_page);
+#endif
+
+#if defined(__x86_64__)
+               /* The x86 clump freeing code requires increasing ppn's to work correctly */
+               if (i > 0) {
+                       assert(phys_page > vm_pages[i - 1].vmp_phys_page);

                }
                }

-               
-               if(j != vm_page_free_count) {                                           /* (BRINGUP) */
-                       panic("pmap_startup: vm_page_free_count does not match, calc =  %d, vm_page_free_count = %08X\n", j, vm_page_free_count);
+#endif
+               ++vm_pages_count;
+               vm_page_init(&vm_pages[i], phys_page, FALSE);
+               if (fill) {
+                       fillPage(phys_page, fillval);
+               }
+               if (vm_himemory_mode) {
+                       vm_page_release_startup(&vm_pages[i]);

                }
        }
                }
        }

+       vm_page_pages = vm_pages_count; /* used to report to user space */
+
+       if (!vm_himemory_mode) {
+               do {
+                       vm_page_release_startup(&vm_pages[--i]);
+               } while (i != 0);
+       }
+
+       absolutetime_to_nanoseconds(mach_absolute_time(), &now_ns);
+       printf("pmap_startup() init/release time: %lld microsec\n", (now_ns - start_ns) / NSEC_PER_USEC);
+       printf("pmap_startup() delayed init/release of %d pages\n", vm_delayed_count);
+
+#if defined(__LP64__)
+       if ((vm_page_t)(VM_PAGE_UNPACK_PTR(VM_PAGE_PACK_PTR(&vm_pages[0]))) != &vm_pages[0]) {
+               panic("VM_PAGE_PACK_PTR failed on &vm_pages[0] - %p", (void *)&vm_pages[0]);
+       }
+
+       if ((vm_page_t)(VM_PAGE_UNPACK_PTR(VM_PAGE_PACK_PTR(&vm_pages[vm_pages_count - 1]))) != &vm_pages[vm_pages_count - 1]) {
+               panic("VM_PAGE_PACK_PTR failed on &vm_pages[vm_pages_count-1] - %p", (void *)&vm_pages[vm_pages_count - 1]);
+       }

 #endif
 
 #endif
 

+       VM_CHECK_MEMORYSTATUS;

 
        /*
 
        /*

-        *      We have to re-align virtual_space_start,
-        *      because pmap_steal_memory has been using it.
+        * We have to re-align virtual_space_start,
+        * because pmap_steal_memory has been using it.

         */
         */

-

        virtual_space_start = round_page(virtual_space_start);
        virtual_space_start = round_page(virtual_space_start);

-

        *startp = virtual_space_start;
        *endp = virtual_space_end;
 }
        *startp = virtual_space_start;
        *endp = virtual_space_end;
 }

-#endif /* MACHINE_PAGES */
+#endif  /* MACHINE_PAGES */

 
 /*
 
 /*

- *     Routine:        vm_page_module_init
- *     Purpose:
- *             Second initialization pass, to be done after
- *             the basic VM system is ready.
+ * Create the zone that represents the vm_pages[] array. Nothing ever allocates
+ * or frees to this zone. It's just here for reporting purposes via zprint command.
+ * This needs to be done after all initially delayed pages are put on the free lists.

  */
  */

-void
-vm_page_module_init(void)
+static void
+vm_page_module_init_delayed(void)

 {
 {

-       uint64_t vm_page_zone_pages, vm_page_array_zone_data_size;
-       vm_size_t       vm_page_with_ppnum_size;
+       (void)zone_create_ext("vm pages array", sizeof(struct vm_page),
+           ZC_NOGZALLOC, ZONE_ID_ANY, ^(zone_t z) {
+               uint64_t vm_page_zone_pages, vm_page_array_zone_data_size;

 
 

-       vm_page_array_zone = zinit((vm_size_t) sizeof(struct vm_page),
-                            0, PAGE_SIZE, "vm pages array");
+               zone_set_exhaustible(z, 0);
+               /*
+                * Reflect size and usage information for vm_pages[].
+                */

 
 

-       zone_change(vm_page_array_zone, Z_CALLERACCT, FALSE);
-       zone_change(vm_page_array_zone, Z_EXPAND, FALSE);
-       zone_change(vm_page_array_zone, Z_EXHAUST, TRUE);
-       zone_change(vm_page_array_zone, Z_FOREIGN, TRUE);
-       zone_change(vm_page_array_zone, Z_GZALLOC_EXEMPT, TRUE);
-       /*
-        * Adjust zone statistics to account for the real pages allocated
-        * in vm_page_create(). [Q: is this really what we want?]
-        */
-       vm_page_array_zone->count += vm_page_pages;
-       vm_page_array_zone->sum_count += vm_page_pages;
-       vm_page_array_zone_data_size = vm_page_pages * vm_page_array_zone->elem_size;
-       vm_page_array_zone->cur_size += vm_page_array_zone_data_size;
-       vm_page_zone_pages = ((round_page(vm_page_array_zone_data_size)) / PAGE_SIZE);
-       OSAddAtomic64(vm_page_zone_pages, &(vm_page_array_zone->page_count));
-       /* since zone accounts for these, take them out of stolen */
-       VM_PAGE_MOVE_STOLEN(vm_page_zone_pages);
+               z->countavail = (uint32_t)(vm_page_array_ending_addr - vm_pages);
+               z->countfree = z->countavail - vm_pages_count;
+               zpercpu_get_cpu(z->z_stats, 0)->zs_mem_allocated =
+               vm_pages_count * sizeof(struct vm_page);
+               vm_page_array_zone_data_size = (uintptr_t)((void *)vm_page_array_ending_addr - (void *)vm_pages);
+               vm_page_zone_pages = atop(round_page((vm_offset_t)vm_page_array_zone_data_size));
+               z->page_count += vm_page_zone_pages;
+               /* since zone accounts for these, take them out of stolen */
+               VM_PAGE_MOVE_STOLEN(vm_page_zone_pages);
+       });
+}
+
+/*
+ * Create the vm_pages zone. This is used for the vm_page structures for the pages
+ * that are scavanged from other boot time usages by ml_static_mfree(). As such,
+ * this needs to happen in early VM bootstrap.
+ */

 
 

-       vm_page_with_ppnum_size = (sizeof(struct vm_page_with_ppnum) + (VM_PACKED_POINTER_ALIGNMENT-1)) & ~(VM_PACKED_POINTER_ALIGNMENT - 1);
+__startup_func
+static void
+vm_page_module_init(void)
+{
+       vm_size_t vm_page_with_ppnum_size;

 
 

-       vm_page_zone = zinit(vm_page_with_ppnum_size,
-                            0, PAGE_SIZE, "vm pages");
+       /*
+        * Since the pointers to elements in this zone will be packed, they
+        * must have appropriate size. Not strictly what sizeof() reports.
+        */
+       vm_page_with_ppnum_size =
+           (sizeof(struct vm_page_with_ppnum) + (VM_PAGE_PACKED_PTR_ALIGNMENT - 1)) &
+           ~(VM_PAGE_PACKED_PTR_ALIGNMENT - 1);

 
 

-       zone_change(vm_page_zone, Z_CALLERACCT, FALSE);
-       zone_change(vm_page_zone, Z_EXPAND, FALSE);
-       zone_change(vm_page_zone, Z_EXHAUST, TRUE);
-       zone_change(vm_page_zone, Z_FOREIGN, TRUE);
-       zone_change(vm_page_zone, Z_GZALLOC_EXEMPT, TRUE);
-       zone_change(vm_page_zone, Z_ALIGNMENT_REQUIRED, TRUE);
+       vm_page_zone = zone_create_ext("vm pages", vm_page_with_ppnum_size,
+           ZC_ALLOW_FOREIGN | ZC_NOGZALLOC | ZC_ALIGNMENT_REQUIRED |
+           ZC_NOCALLOUT, ZONE_ID_ANY, ^(zone_t z) {
+#if defined(__LP64__)
+               zone_set_submap_idx(z, Z_SUBMAP_IDX_VA_RESTRICTED_MAP);
+#endif
+               zone_set_exhaustible(z, 0);
+       });

 }
 }

+STARTUP(ZALLOC, STARTUP_RANK_SECOND, vm_page_module_init);

 
 /*
  *     Routine:        vm_page_create
 
 /*
  *     Routine:        vm_page_create


@@ -1404,20 +1660,23 @@ vm_page_create(
        ppnum_t start,
        ppnum_t end)
 {
        ppnum_t start,
        ppnum_t end)
 {

-       ppnum_t         phys_page;
-       vm_page_t       m;
+       ppnum_t         phys_page;
+       vm_page_t       m;

 
        for (phys_page = start;
 
        for (phys_page = start;

-            phys_page < end;
-            phys_page++) {
+           phys_page < end;
+           phys_page++) {

                while ((m = (vm_page_t) vm_page_grab_fictitious_common(phys_page))
                while ((m = (vm_page_t) vm_page_grab_fictitious_common(phys_page))

-                       == VM_PAGE_NULL)
+                   == VM_PAGE_NULL) {

                        vm_page_more_fictitious();
                        vm_page_more_fictitious();

+               }

 
                m->vmp_fictitious = FALSE;
                pmap_clear_noencrypt(phys_page);
 
 
                m->vmp_fictitious = FALSE;
                pmap_clear_noencrypt(phys_page);
 

+               lck_mtx_lock(&vm_page_queue_free_lock);

                vm_page_pages++;
                vm_page_pages++;

+               lck_mtx_unlock(&vm_page_queue_free_lock);

                vm_page_release(m, FALSE);
        }
 }
                vm_page_release(m, FALSE);
        }
 }


@@ -1444,18 +1703,18 @@ vm_page_create(
  */
 void
 vm_page_insert(
  */
 void
 vm_page_insert(

-       vm_page_t               mem,
-       vm_object_t             object,
-       vm_object_offset_t      offset)
+       vm_page_t               mem,
+       vm_object_t             object,
+       vm_object_offset_t      offset)

 {
        vm_page_insert_internal(mem, object, offset, VM_KERN_MEMORY_NONE, FALSE, TRUE, FALSE, FALSE, NULL);
 }
 
 void
 vm_page_insert_wired(
 {
        vm_page_insert_internal(mem, object, offset, VM_KERN_MEMORY_NONE, FALSE, TRUE, FALSE, FALSE, NULL);
 }
 
 void
 vm_page_insert_wired(

-       vm_page_t               mem,
-       vm_object_t             object,
-       vm_object_offset_t      offset,
+       vm_page_t               mem,
+       vm_object_t             object,
+       vm_object_offset_t      offset,

        vm_tag_t                tag)
 {
        vm_page_insert_internal(mem, object, offset, tag, FALSE, TRUE, FALSE, FALSE, NULL);
        vm_tag_t                tag)
 {
        vm_page_insert_internal(mem, object, offset, tag, FALSE, TRUE, FALSE, FALSE, NULL);


@@ -1463,29 +1722,26 @@ vm_page_insert_wired(
 
 void
 vm_page_insert_internal(
 
 void
 vm_page_insert_internal(

-       vm_page_t               mem,
-       vm_object_t             object,
-       vm_object_offset_t      offset,
+       vm_page_t               mem,
+       vm_object_t             object,
+       vm_object_offset_t      offset,

        vm_tag_t                tag,
        vm_tag_t                tag,

-       boolean_t               queues_lock_held,
-       boolean_t               insert_in_hash,
-       boolean_t               batch_pmap_op,
-        boolean_t               batch_accounting,
-       uint64_t                *delayed_ledger_update)
-{
-       vm_page_bucket_t        *bucket;
-       lck_spin_t              *bucket_lock;
-       int                     hash_id;
-       task_t                  owner;
-       int                     ledger_idx_volatile;
-       int                     ledger_idx_nonvolatile;
-       int                     ledger_idx_volatile_compressed;
-       int                     ledger_idx_nonvolatile_compressed;
-       boolean_t               do_footprint;
-
-        XPR(XPR_VM_PAGE,
-                "vm_page_insert, object 0x%X offset 0x%X page 0x%X\n",
-                object, offset, mem, 0,0);
+       boolean_t               queues_lock_held,
+       boolean_t               insert_in_hash,
+       boolean_t               batch_pmap_op,
+       boolean_t               batch_accounting,
+       uint64_t                *delayed_ledger_update)
+{
+       vm_page_bucket_t        *bucket;
+       lck_spin_t              *bucket_lock;
+       int                     hash_id;
+       task_t                  owner;
+       int                     ledger_idx_volatile;
+       int                     ledger_idx_nonvolatile;
+       int                     ledger_idx_volatile_compressed;
+       int                     ledger_idx_nonvolatile_compressed;
+       boolean_t               do_footprint;
+

 #if 0
        /*
         * we may not hold the page queue lock
 #if 0
        /*
         * we may not hold the page queue lock


@@ -1494,7 +1750,7 @@ vm_page_insert_internal(
        VM_PAGE_CHECK(mem);
 #endif
 
        VM_PAGE_CHECK(mem);
 #endif
 

-       assert(page_aligned(offset));
+       assertf(page_aligned(offset), "0x%llx\n", offset);

 
        assert(!VM_PAGE_WIRED(mem) || mem->vmp_private || mem->vmp_fictitious || (tag != VM_KERN_MEMORY_NONE));
 
 
        assert(!VM_PAGE_WIRED(mem) || mem->vmp_private || mem->vmp_fictitious || (tag != VM_KERN_MEMORY_NONE));
 


@@ -1503,26 +1759,28 @@ vm_page_insert_internal(
 
        vm_object_lock_assert_exclusive(object);
        LCK_MTX_ASSERT(&vm_page_queue_lock,
 
        vm_object_lock_assert_exclusive(object);
        LCK_MTX_ASSERT(&vm_page_queue_lock,

-                      queues_lock_held ? LCK_MTX_ASSERT_OWNED
-                                       : LCK_MTX_ASSERT_NOTOWNED);
+           queues_lock_held ? LCK_MTX_ASSERT_OWNED
+           : LCK_MTX_ASSERT_NOTOWNED);

 
 

-       if (queues_lock_held == FALSE)
+       if (queues_lock_held == FALSE) {

                assert(!VM_PAGE_PAGEABLE(mem));
                assert(!VM_PAGE_PAGEABLE(mem));

+       }

 
        if (insert_in_hash == TRUE) {
 
        if (insert_in_hash == TRUE) {

-#if DEBUG || VM_PAGE_CHECK_BUCKETS
-               if (mem->vmp_tabled || mem->vmp_object)
+#if DEBUG || VM_PAGE_BUCKETS_CHECK
+               if (mem->vmp_tabled || mem->vmp_object) {

                        panic("vm_page_insert: page %p for (obj=%p,off=0x%llx) "
                        panic("vm_page_insert: page %p for (obj=%p,off=0x%llx) "

-                             "already in (obj=%p,off=0x%llx)",
-                             mem, object, offset, VM_PAGE_OBJECT(mem), mem->vmp_offset);
+                           "already in (obj=%p,off=0x%llx)",
+                           mem, object, offset, VM_PAGE_OBJECT(mem), mem->vmp_offset);
+               }

 #endif
                if (object->internal && (offset >= object->vo_size)) {
                        panic("vm_page_insert_internal: (page=%p,obj=%p,off=0x%llx,size=0x%llx) inserted at offset past object bounds",
 #endif
                if (object->internal && (offset >= object->vo_size)) {
                        panic("vm_page_insert_internal: (page=%p,obj=%p,off=0x%llx,size=0x%llx) inserted at offset past object bounds",

-                             mem, object, offset, object->vo_size);
+                           mem, object, offset, object->vo_size);

                }
 
                assert(vm_page_lookup(object, offset) == VM_PAGE_NULL);
                }
 
                assert(vm_page_lookup(object, offset) == VM_PAGE_NULL);

-               
+

                /*
                 *      Record the object/offset pair in this page
                 */
                /*
                 *      Record the object/offset pair in this page
                 */


@@ -1542,22 +1800,23 @@ vm_page_insert_internal(
                hash_id = vm_page_hash(object, offset);
                bucket = &vm_page_buckets[hash_id];
                bucket_lock = &vm_page_bucket_locks[hash_id / BUCKETS_PER_LOCK];
                hash_id = vm_page_hash(object, offset);
                bucket = &vm_page_buckets[hash_id];
                bucket_lock = &vm_page_bucket_locks[hash_id / BUCKETS_PER_LOCK];

-       
-               lck_spin_lock(bucket_lock);
+
+               lck_spin_lock_grp(bucket_lock, &vm_page_lck_grp_bucket);

 
                mem->vmp_next_m = bucket->page_list;
                bucket->page_list = VM_PAGE_PACK_PTR(mem);
                assert(mem == (vm_page_t)(VM_PAGE_UNPACK_PTR(bucket->page_list)));
 
 #if     MACH_PAGE_HASH_STATS
 
                mem->vmp_next_m = bucket->page_list;
                bucket->page_list = VM_PAGE_PACK_PTR(mem);
                assert(mem == (vm_page_t)(VM_PAGE_UNPACK_PTR(bucket->page_list)));
 
 #if     MACH_PAGE_HASH_STATS

-               if (++bucket->cur_count > bucket->hi_count)
+               if (++bucket->cur_count > bucket->hi_count) {

                        bucket->hi_count = bucket->cur_count;
                        bucket->hi_count = bucket->cur_count;

+               }

 #endif /* MACH_PAGE_HASH_STATS */
                mem->vmp_hashed = TRUE;
                lck_spin_unlock(bucket_lock);
        }
 
 #endif /* MACH_PAGE_HASH_STATS */
                mem->vmp_hashed = TRUE;
                lck_spin_unlock(bucket_lock);
        }
 

-       {       
+       {

                unsigned int    cache_attr;
 
                cache_attr = object->wimg_bits & VM_WIMG_MASK;
                unsigned int    cache_attr;
 
                cache_attr = object->wimg_bits & VM_WIMG_MASK;


@@ -1569,7 +1828,7 @@ vm_page_insert_internal(
        /*
         *      Now link into the object's list of backed pages.
         */
        /*
         *      Now link into the object's list of backed pages.
         */

-       vm_page_queue_enter(&object->memq, mem, vm_page_t, vmp_listq);
+       vm_page_queue_enter(&object->memq, mem, vmp_listq);

        object->memq_hint = mem;
        mem->vmp_tabled = TRUE;
 
        object->memq_hint = mem;
        mem->vmp_tabled = TRUE;
 


@@ -1579,14 +1838,28 @@ vm_page_insert_internal(
 
        object->resident_page_count++;
        if (VM_PAGE_WIRED(mem)) {
 
        object->resident_page_count++;
        if (VM_PAGE_WIRED(mem)) {

-           assert(mem->vmp_wire_count > 0);
-           VM_OBJECT_WIRED_PAGE_UPDATE_START(object);
-           VM_OBJECT_WIRED_PAGE_ADD(object, mem);
-           VM_OBJECT_WIRED_PAGE_UPDATE_END(object, tag);
+               assert(mem->vmp_wire_count > 0);
+               VM_OBJECT_WIRED_PAGE_UPDATE_START(object);
+               VM_OBJECT_WIRED_PAGE_ADD(object, mem);
+               VM_OBJECT_WIRED_PAGE_UPDATE_END(object, tag);

        }
        assert(object->resident_page_count >= object->wired_page_count);
 
        }
        assert(object->resident_page_count >= object->wired_page_count);
 

-        if (batch_accounting == FALSE) {
+#if DEVELOPMENT || DEBUG
+       if (object->object_is_shared_cache &&
+           object->pager != NULL &&
+           object->pager->mo_pager_ops == &shared_region_pager_ops) {
+               int new, old;
+               assert(!object->internal);
+               new = OSAddAtomic(+1, &shared_region_pagers_resident_count);
+               do {
+                       old = shared_region_pagers_resident_peak;
+               } while (old < new &&
+                   !OSCompareAndSwap(old, new, &shared_region_pagers_resident_peak));
+       }
+#endif /* DEVELOPMENT || DEBUG */
+
+       if (batch_accounting == FALSE) {

                if (object->internal) {
                        OSAddAtomic(1, &vm_page_internal_count);
                } else {
                if (object->internal) {
                        OSAddAtomic(1, &vm_page_internal_count);
                } else {


@@ -1612,45 +1885,43 @@ vm_page_insert_internal(
        }
 
        if (object->purgable == VM_PURGABLE_DENY &&
        }
 
        if (object->purgable == VM_PURGABLE_DENY &&

-           ! object->vo_ledger_tag) {
+           !object->vo_ledger_tag) {

                owner = TASK_NULL;
        } else {
                owner = VM_OBJECT_OWNER(object);
                vm_object_ledger_tag_ledgers(object,
                owner = TASK_NULL;
        } else {
                owner = VM_OBJECT_OWNER(object);
                vm_object_ledger_tag_ledgers(object,

-                                            &ledger_idx_volatile,
-                                            &ledger_idx_nonvolatile,
-                                            &ledger_idx_volatile_compressed,
-                                            &ledger_idx_nonvolatile_compressed,
-                                            &do_footprint);
+                   &ledger_idx_volatile,
+                   &ledger_idx_nonvolatile,
+                   &ledger_idx_volatile_compressed,
+                   &ledger_idx_nonvolatile_compressed,
+                   &do_footprint);

        }
        if (owner &&
            (object->purgable == VM_PURGABLE_NONVOLATILE ||
        }
        if (owner &&
            (object->purgable == VM_PURGABLE_NONVOLATILE ||

-            object->purgable == VM_PURGABLE_DENY ||
-            VM_PAGE_WIRED(mem))) {
-
-               if (delayed_ledger_update)
+           object->purgable == VM_PURGABLE_DENY ||
+           VM_PAGE_WIRED(mem))) {
+               if (delayed_ledger_update) {

                        *delayed_ledger_update += PAGE_SIZE;
                        *delayed_ledger_update += PAGE_SIZE;

-               else {
+               } else {

                        /* more non-volatile bytes */
                        ledger_credit(owner->ledger,
                        /* more non-volatile bytes */
                        ledger_credit(owner->ledger,

-                                     ledger_idx_nonvolatile,
-                                     PAGE_SIZE);
+                           ledger_idx_nonvolatile,
+                           PAGE_SIZE);

                        if (do_footprint) {
                                /* more footprint */
                                ledger_credit(owner->ledger,
                        if (do_footprint) {
                                /* more footprint */
                                ledger_credit(owner->ledger,

-                                             task_ledgers.phys_footprint,
-                                             PAGE_SIZE);
+                                   task_ledgers.phys_footprint,
+                                   PAGE_SIZE);

                        }
                }
                        }
                }

-

        } else if (owner &&
        } else if (owner &&

-                  (object->purgable == VM_PURGABLE_VOLATILE ||
-                   object->purgable == VM_PURGABLE_EMPTY)) {
-               assert(! VM_PAGE_WIRED(mem));
+           (object->purgable == VM_PURGABLE_VOLATILE ||
+           object->purgable == VM_PURGABLE_EMPTY)) {
+               assert(!VM_PAGE_WIRED(mem));

                /* more volatile bytes */
                ledger_credit(owner->ledger,
                /* more volatile bytes */
                ledger_credit(owner->ledger,

-                             ledger_idx_volatile,
-                             PAGE_SIZE);
+                   ledger_idx_volatile,
+                   PAGE_SIZE);

        }
 
        if (object->purgable == VM_PURGABLE_VOLATILE) {
        }
 
        if (object->purgable == VM_PURGABLE_VOLATILE) {


@@ -1660,7 +1931,7 @@ vm_page_insert_internal(
                        OSAddAtomic(+1, &vm_page_purgeable_count);
                }
        } else if (object->purgable == VM_PURGABLE_EMPTY &&
                        OSAddAtomic(+1, &vm_page_purgeable_count);
                }
        } else if (object->purgable == VM_PURGABLE_EMPTY &&

-                  mem->vmp_q_state == VM_PAGE_ON_THROTTLED_Q) {
+           mem->vmp_q_state == VM_PAGE_ON_THROTTLED_Q) {

                /*
                 * This page belongs to a purged VM object but hasn't
                 * been purged (because it was "busy").
                /*
                 * This page belongs to a purged VM object but hasn't
                 * been purged (because it was "busy").


@@ -1669,11 +1940,13 @@ vm_page_insert_internal(
                 * queue, so that it can eventually be reclaimed, instead
                 * of lingering in the "empty" object.
                 */
                 * queue, so that it can eventually be reclaimed, instead
                 * of lingering in the "empty" object.
                 */

-               if (queues_lock_held == FALSE)
+               if (queues_lock_held == FALSE) {

                        vm_page_lockspin_queues();
                        vm_page_lockspin_queues();

+               }

                vm_page_deactivate(mem);
                vm_page_deactivate(mem);

-               if (queues_lock_held == FALSE)
+               if (queues_lock_held == FALSE) {

                        vm_page_unlock_queues();
                        vm_page_unlock_queues();

+               }

        }
 
 #if VM_OBJECT_TRACKING_OP_MODIFIED
        }
 
 #if VM_OBJECT_TRACKING_OP_MODIFIED


@@ -1686,12 +1959,12 @@ vm_page_insert_internal(
                void *bt[VM_OBJECT_TRACKING_BTDEPTH];
                int numsaved = 0;
 
                void *bt[VM_OBJECT_TRACKING_BTDEPTH];
                int numsaved = 0;
 

-               numsaved =OSBacktrace(bt, VM_OBJECT_TRACKING_BTDEPTH);
+               numsaved = OSBacktrace(bt, VM_OBJECT_TRACKING_BTDEPTH);

                btlog_add_entry(vm_object_tracking_btlog,
                btlog_add_entry(vm_object_tracking_btlog,

-                               object,
-                               VM_OBJECT_TRACKING_OP_MODIFIED,
-                               bt,
-                               numsaved);
+                   object,
+                   VM_OBJECT_TRACKING_OP_MODIFIED,
+                   bt,
+                   numsaved);

        }
 #endif /* VM_OBJECT_TRACKING_OP_MODIFIED */
 }
        }
 #endif /* VM_OBJECT_TRACKING_OP_MODIFIED */
 }


@@ -1706,14 +1979,14 @@ vm_page_insert_internal(
  */
 void
 vm_page_replace(
  */
 void
 vm_page_replace(

-       vm_page_t               mem,
-       vm_object_t             object,
-       vm_object_offset_t      offset)
+       vm_page_t               mem,
+       vm_object_t             object,
+       vm_object_offset_t      offset)

 {
        vm_page_bucket_t *bucket;
 {
        vm_page_bucket_t *bucket;

-       vm_page_t        found_m = VM_PAGE_NULL;
-       lck_spin_t      *bucket_lock;
-       int             hash_id;
+       vm_page_t        found_m = VM_PAGE_NULL;
+       lck_spin_t      *bucket_lock;
+       int             hash_id;

 
 #if 0
        /*
 
 #if 0
        /*


@@ -1723,11 +1996,12 @@ vm_page_replace(
        VM_PAGE_CHECK(mem);
 #endif
        vm_object_lock_assert_exclusive(object);
        VM_PAGE_CHECK(mem);
 #endif
        vm_object_lock_assert_exclusive(object);

-#if DEBUG || VM_PAGE_CHECK_BUCKETS
-       if (mem->vmp_tabled || mem->vmp_object)
+#if DEBUG || VM_PAGE_BUCKETS_CHECK
+       if (mem->vmp_tabled || mem->vmp_object) {

                panic("vm_page_replace: page %p for (obj=%p,off=0x%llx) "
                panic("vm_page_replace: page %p for (obj=%p,off=0x%llx) "

-                     "already in (obj=%p,off=0x%llx)",
-                     mem, object, offset, VM_PAGE_OBJECT(mem), mem->vmp_offset);
+                   "already in (obj=%p,off=0x%llx)",
+                   mem, object, offset, VM_PAGE_OBJECT(mem), mem->vmp_offset);
+       }

 #endif
        LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_NOTOWNED);
 
 #endif
        LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_NOTOWNED);
 


@@ -1748,7 +2022,7 @@ vm_page_replace(
        bucket = &vm_page_buckets[hash_id];
        bucket_lock = &vm_page_bucket_locks[hash_id / BUCKETS_PER_LOCK];
 
        bucket = &vm_page_buckets[hash_id];
        bucket_lock = &vm_page_bucket_locks[hash_id / BUCKETS_PER_LOCK];
 

-       lck_spin_lock(bucket_lock);
+       lck_spin_lock_grp(bucket_lock, &vm_page_lck_grp_bucket);

 
        if (bucket->page_list) {
                vm_page_packed_t *mp = &bucket->page_list;
 
        if (bucket->page_list) {
                vm_page_packed_t *mp = &bucket->page_list;


@@ -1785,7 +2059,7 @@ vm_page_replace(
        lck_spin_unlock(bucket_lock);
 
        if (found_m) {
        lck_spin_unlock(bucket_lock);
 
        if (found_m) {

-               /*
+               /*

                 * there was already a page at the specified
                 * offset for this object... remove it from
                 * the object and free it back to the free list
                 * there was already a page at the specified
                 * offset for this object... remove it from
                 * the object and free it back to the free list


@@ -1806,28 +2080,23 @@ vm_page_replace(
 
 void
 vm_page_remove(
 
 void
 vm_page_remove(

-       vm_page_t       mem,
-       boolean_t       remove_from_hash)
+       vm_page_t       mem,
+       boolean_t       remove_from_hash)

 {
        vm_page_bucket_t *bucket;
 {
        vm_page_bucket_t *bucket;

-       vm_page_t       this;
-       lck_spin_t      *bucket_lock;
-       int             hash_id;
-       task_t          owner;
-       vm_object_t     m_object;
-       int             ledger_idx_volatile;
-       int             ledger_idx_nonvolatile;
-       int             ledger_idx_volatile_compressed;
-       int             ledger_idx_nonvolatile_compressed;
-       int             do_footprint;
+       vm_page_t       this;
+       lck_spin_t      *bucket_lock;
+       int             hash_id;
+       task_t          owner;
+       vm_object_t     m_object;
+       int             ledger_idx_volatile;
+       int             ledger_idx_nonvolatile;
+       int             ledger_idx_volatile_compressed;
+       int             ledger_idx_nonvolatile_compressed;
+       int             do_footprint;

 
        m_object = VM_PAGE_OBJECT(mem);
 
 
        m_object = VM_PAGE_OBJECT(mem);
 

-        XPR(XPR_VM_PAGE,
-                "vm_page_remove, object 0x%X offset 0x%X page 0x%X\n",
-                m_object, mem->vmp_offset, 
-               mem, 0,0);
-

        vm_object_lock_assert_exclusive(m_object);
        assert(mem->vmp_tabled);
        assert(!mem->vmp_cleaning);
        vm_object_lock_assert_exclusive(m_object);
        assert(mem->vmp_tabled);
        assert(!mem->vmp_cleaning);


@@ -1851,19 +2120,20 @@ vm_page_remove(
                bucket = &vm_page_buckets[hash_id];
                bucket_lock = &vm_page_bucket_locks[hash_id / BUCKETS_PER_LOCK];
 
                bucket = &vm_page_buckets[hash_id];
                bucket_lock = &vm_page_bucket_locks[hash_id / BUCKETS_PER_LOCK];
 

-               lck_spin_lock(bucket_lock);
+               lck_spin_lock_grp(bucket_lock, &vm_page_lck_grp_bucket);

 
                if ((this = (vm_page_t)(VM_PAGE_UNPACK_PTR(bucket->page_list))) == mem) {
                        /* optimize for common case */
 
                        bucket->page_list = mem->vmp_next_m;
                } else {
 
                if ((this = (vm_page_t)(VM_PAGE_UNPACK_PTR(bucket->page_list))) == mem) {
                        /* optimize for common case */
 
                        bucket->page_list = mem->vmp_next_m;
                } else {

-                       vm_page_packed_t        *prev;
+                       vm_page_packed_t        *prev;

 
                        for (prev = &this->vmp_next_m;
 
                        for (prev = &this->vmp_next_m;

-                            (this = (vm_page_t)(VM_PAGE_UNPACK_PTR(*prev))) != mem;
-                            prev = &this->vmp_next_m)
+                           (this = (vm_page_t)(VM_PAGE_UNPACK_PTR(*prev))) != mem;
+                           prev = &this->vmp_next_m) {

                                continue;
                                continue;

+                       }

                        *prev = this->vmp_next_m;
                }
 #if     MACH_PAGE_HASH_STATS
                        *prev = this->vmp_next_m;
                }
 #if     MACH_PAGE_HASH_STATS


@@ -1887,6 +2157,15 @@ vm_page_remove(
        assert(m_object->resident_page_count > 0);
        m_object->resident_page_count--;
 
        assert(m_object->resident_page_count > 0);
        m_object->resident_page_count--;
 

+#if DEVELOPMENT || DEBUG
+       if (m_object->object_is_shared_cache &&
+           m_object->pager != NULL &&
+           m_object->pager->mo_pager_ops == &shared_region_pager_ops) {
+               assert(!m_object->internal);
+               OSAddAtomic(-1, &shared_region_pagers_resident_count);
+       }
+#endif /* DEVELOPMENT || DEBUG */
+

        if (m_object->internal) {
 #if DEBUG
                assert(vm_page_internal_count);
        if (m_object->internal) {
 #if DEBUG
                assert(vm_page_internal_count);


@@ -1905,8 +2184,9 @@ vm_page_remove(
        if (!m_object->internal &&
            m_object->cached_list.next &&
            m_object->cached_list.prev) {
        if (!m_object->internal &&
            m_object->cached_list.next &&
            m_object->cached_list.prev) {

-               if (m_object->resident_page_count == 0)
+               if (m_object->resident_page_count == 0) {

                        vm_object_cache_remove(m_object);
                        vm_object_cache_remove(m_object);

+               }

        }
 
        if (VM_PAGE_WIRED(mem)) {
        }
 
        if (VM_PAGE_WIRED(mem)) {


@@ -1916,12 +2196,12 @@ vm_page_remove(
                VM_OBJECT_WIRED_PAGE_UPDATE_END(m_object, m_object->wire_tag);
        }
        assert(m_object->resident_page_count >=
                VM_OBJECT_WIRED_PAGE_UPDATE_END(m_object, m_object->wire_tag);
        }
        assert(m_object->resident_page_count >=

-              m_object->wired_page_count);
+           m_object->wired_page_count);

        if (mem->vmp_reusable) {
                assert(m_object->reusable_page_count > 0);
                m_object->reusable_page_count--;
                assert(m_object->reusable_page_count <=
        if (mem->vmp_reusable) {
                assert(m_object->reusable_page_count > 0);
                m_object->reusable_page_count--;
                assert(m_object->reusable_page_count <=

-                      m_object->resident_page_count);
+                   m_object->resident_page_count);

                mem->vmp_reusable = FALSE;
                OSAddAtomic(-1, &vm_page_stats_reusable.reusable_count);
                vm_page_stats_reusable.reused_remove++;
                mem->vmp_reusable = FALSE;
                OSAddAtomic(-1, &vm_page_stats_reusable.reusable_count);
                vm_page_stats_reusable.reused_remove++;


@@ -1931,39 +2211,39 @@ vm_page_remove(
        }
 
        if (m_object->purgable == VM_PURGABLE_DENY &&
        }
 
        if (m_object->purgable == VM_PURGABLE_DENY &&

-           ! m_object->vo_ledger_tag) {
+           !m_object->vo_ledger_tag) {

                owner = TASK_NULL;
        } else {
                owner = VM_OBJECT_OWNER(m_object);
                vm_object_ledger_tag_ledgers(m_object,
                owner = TASK_NULL;
        } else {
                owner = VM_OBJECT_OWNER(m_object);
                vm_object_ledger_tag_ledgers(m_object,

-                                            &ledger_idx_volatile,
-                                            &ledger_idx_nonvolatile,
-                                            &ledger_idx_volatile_compressed,
-                                            &ledger_idx_nonvolatile_compressed,
-                                            &do_footprint);
+                   &ledger_idx_volatile,
+                   &ledger_idx_nonvolatile,
+                   &ledger_idx_volatile_compressed,
+                   &ledger_idx_nonvolatile_compressed,
+                   &do_footprint);

        }
        if (owner &&
            (m_object->purgable == VM_PURGABLE_NONVOLATILE ||
        }
        if (owner &&
            (m_object->purgable == VM_PURGABLE_NONVOLATILE ||

-            m_object->purgable == VM_PURGABLE_DENY ||
-            VM_PAGE_WIRED(mem))) {
+           m_object->purgable == VM_PURGABLE_DENY ||
+           VM_PAGE_WIRED(mem))) {

                /* less non-volatile bytes */
                ledger_debit(owner->ledger,
                /* less non-volatile bytes */
                ledger_debit(owner->ledger,

-                            ledger_idx_nonvolatile,
-                            PAGE_SIZE);
+                   ledger_idx_nonvolatile,
+                   PAGE_SIZE);

                if (do_footprint) {
                        /* less footprint */
                        ledger_debit(owner->ledger,
                if (do_footprint) {
                        /* less footprint */
                        ledger_debit(owner->ledger,

-                                    task_ledgers.phys_footprint,
-                                    PAGE_SIZE);
+                           task_ledgers.phys_footprint,
+                           PAGE_SIZE);

                }
        } else if (owner &&
                }
        } else if (owner &&

-                  (m_object->purgable == VM_PURGABLE_VOLATILE ||
-                   m_object->purgable == VM_PURGABLE_EMPTY)) {
-               assert(! VM_PAGE_WIRED(mem));
+           (m_object->purgable == VM_PURGABLE_VOLATILE ||
+           m_object->purgable == VM_PURGABLE_EMPTY)) {
+               assert(!VM_PAGE_WIRED(mem));

                /* less volatile bytes */
                ledger_debit(owner->ledger,
                /* less volatile bytes */
                ledger_debit(owner->ledger,

-                            ledger_idx_volatile,
-                            PAGE_SIZE);
+                   ledger_idx_volatile,
+                   PAGE_SIZE);

        }
        if (m_object->purgable == VM_PURGABLE_VOLATILE) {
                if (VM_PAGE_WIRED(mem)) {
        }
        if (m_object->purgable == VM_PURGABLE_VOLATILE) {
                if (VM_PAGE_WIRED(mem)) {


@@ -1975,8 +2255,9 @@ vm_page_remove(
                }
        }
 
                }
        }
 

-       if (m_object->set_cache_attr == TRUE)
+       if (m_object->set_cache_attr == TRUE) {

                pmap_set_cache_attributes(VM_PAGE_GET_PHYS_PAGE(mem), 0);
                pmap_set_cache_attributes(VM_PAGE_GET_PHYS_PAGE(mem), 0);

+       }

 
        mem->vmp_tabled = FALSE;
        mem->vmp_object = 0;
 
        mem->vmp_tabled = FALSE;
        mem->vmp_object = 0;


@@ -1993,34 +2274,34 @@ vm_page_remove(
  *     The object must be locked.  No side effects.
  */
 
  *     The object must be locked.  No side effects.
  */
 

-#define        VM_PAGE_HASH_LOOKUP_THRESHOLD   10
+#define VM_PAGE_HASH_LOOKUP_THRESHOLD   10

 
 #if DEBUG_VM_PAGE_LOOKUP
 
 struct {
 
 #if DEBUG_VM_PAGE_LOOKUP
 
 struct {

-       uint64_t        vpl_total;
-       uint64_t        vpl_empty_obj;
-       uint64_t        vpl_bucket_NULL;
-       uint64_t        vpl_hit_hint;
-       uint64_t        vpl_hit_hint_next;
-       uint64_t        vpl_hit_hint_prev;
-       uint64_t        vpl_fast;
-       uint64_t        vpl_slow;
-       uint64_t        vpl_hit;
-       uint64_t        vpl_miss;
-
-       uint64_t        vpl_fast_elapsed;
-       uint64_t        vpl_slow_elapsed;
+       uint64_t        vpl_total;
+       uint64_t        vpl_empty_obj;
+       uint64_t        vpl_bucket_NULL;
+       uint64_t        vpl_hit_hint;
+       uint64_t        vpl_hit_hint_next;
+       uint64_t        vpl_hit_hint_prev;
+       uint64_t        vpl_fast;
+       uint64_t        vpl_slow;
+       uint64_t        vpl_hit;
+       uint64_t        vpl_miss;
+
+       uint64_t        vpl_fast_elapsed;
+       uint64_t        vpl_slow_elapsed;

 } vm_page_lookup_stats __attribute__((aligned(8)));
 
 #endif
 
 } vm_page_lookup_stats __attribute__((aligned(8)));
 
 #endif
 

-#define        KDP_VM_PAGE_WALK_MAX    1000
+#define KDP_VM_PAGE_WALK_MAX    1000

 
 vm_page_t
 kdp_vm_page_lookup(
 
 vm_page_t
 kdp_vm_page_lookup(

-       vm_object_t             object,
-       vm_object_offset_t      offset)
+       vm_object_t             object,
+       vm_object_offset_t      offset)

 {
        vm_page_t cur_page;
        int num_traversed = 0;
 {
        vm_page_t cur_page;
        int num_traversed = 0;


@@ -2029,7 +2310,7 @@ kdp_vm_page_lookup(
                panic("panic: kdp_vm_page_lookup done outside of kernel debugger");
        }
 
                panic("panic: kdp_vm_page_lookup done outside of kernel debugger");
        }
 

-       vm_page_queue_iterate(&object->memq, cur_page, vm_page_t, vmp_listq) {
+       vm_page_queue_iterate(&object->memq, cur_page, vmp_listq) {

                if (cur_page->vmp_offset == offset) {
                        return cur_page;
                }
                if (cur_page->vmp_offset == offset) {
                        return cur_page;
                }


@@ -2045,26 +2326,27 @@ kdp_vm_page_lookup(
 
 vm_page_t
 vm_page_lookup(
 
 vm_page_t
 vm_page_lookup(

-       vm_object_t             object,
-       vm_object_offset_t      offset)
+       vm_object_t             object,
+       vm_object_offset_t      offset)

 {
 {

-       vm_page_t       mem;
+       vm_page_t       mem;

        vm_page_bucket_t *bucket;
        vm_page_bucket_t *bucket;

-       vm_page_queue_entry_t   qe;
-       lck_spin_t      *bucket_lock = NULL;
-       int             hash_id;
+       vm_page_queue_entry_t   qe;
+       lck_spin_t      *bucket_lock = NULL;
+       int             hash_id;

 #if DEBUG_VM_PAGE_LOOKUP
 #if DEBUG_VM_PAGE_LOOKUP

-       uint64_t        start, elapsed;
+       uint64_t        start, elapsed;

 
        OSAddAtomic64(1, &vm_page_lookup_stats.vpl_total);
 #endif
        vm_object_lock_assert_held(object);
 
        OSAddAtomic64(1, &vm_page_lookup_stats.vpl_total);
 #endif
        vm_object_lock_assert_held(object);

+       assertf(page_aligned(offset), "offset 0x%llx\n", offset);

 
        if (object->resident_page_count == 0) {
 #if DEBUG_VM_PAGE_LOOKUP
                OSAddAtomic64(1, &vm_page_lookup_stats.vpl_empty_obj);
 #endif
 
        if (object->resident_page_count == 0) {
 #if DEBUG_VM_PAGE_LOOKUP
                OSAddAtomic64(1, &vm_page_lookup_stats.vpl_empty_obj);
 #endif

-               return (VM_PAGE_NULL);
+               return VM_PAGE_NULL;

        }
 
        mem = object->memq_hint;
        }
 
        mem = object->memq_hint;


@@ -2076,12 +2358,12 @@ vm_page_lookup(
 #if DEBUG_VM_PAGE_LOOKUP
                        OSAddAtomic64(1, &vm_page_lookup_stats.vpl_hit_hint);
 #endif
 #if DEBUG_VM_PAGE_LOOKUP
                        OSAddAtomic64(1, &vm_page_lookup_stats.vpl_hit_hint);
 #endif

-                       return (mem);
+                       return mem;

                }
                qe = (vm_page_queue_entry_t)vm_page_queue_next(&mem->vmp_listq);
 
                }
                qe = (vm_page_queue_entry_t)vm_page_queue_next(&mem->vmp_listq);
 

-               if (! vm_page_queue_end(&object->memq, qe)) {
-                       vm_page_t       next_page;
+               if (!vm_page_queue_end(&object->memq, qe)) {
+                       vm_page_t       next_page;

 
                        next_page = (vm_page_t)((uintptr_t)qe);
                        assert(VM_PAGE_OBJECT(next_page) == object);
 
                        next_page = (vm_page_t)((uintptr_t)qe);
                        assert(VM_PAGE_OBJECT(next_page) == object);


@@ -2091,12 +2373,12 @@ vm_page_lookup(
 #if DEBUG_VM_PAGE_LOOKUP
                                OSAddAtomic64(1, &vm_page_lookup_stats.vpl_hit_hint_next);
 #endif
 #if DEBUG_VM_PAGE_LOOKUP
                                OSAddAtomic64(1, &vm_page_lookup_stats.vpl_hit_hint_next);
 #endif

-                               return (next_page);
+                               return next_page;

                        }
                }
                qe = (vm_page_queue_entry_t)vm_page_queue_prev(&mem->vmp_listq);
 
                        }
                }
                qe = (vm_page_queue_entry_t)vm_page_queue_prev(&mem->vmp_listq);
 

-               if (! vm_page_queue_end(&object->memq, qe)) {
+               if (!vm_page_queue_end(&object->memq, qe)) {

                        vm_page_t prev_page;
 
                        prev_page = (vm_page_t)((uintptr_t)qe);
                        vm_page_t prev_page;
 
                        prev_page = (vm_page_t)((uintptr_t)qe);


@@ -2107,7 +2389,7 @@ vm_page_lookup(
 #if DEBUG_VM_PAGE_LOOKUP
                                OSAddAtomic64(1, &vm_page_lookup_stats.vpl_hit_hint_prev);
 #endif
 #if DEBUG_VM_PAGE_LOOKUP
                                OSAddAtomic64(1, &vm_page_lookup_stats.vpl_hit_hint_prev);
 #endif

-                               return (prev_page);
+                               return prev_page;

                        }
                }
        }
                        }
                }
        }


@@ -2122,14 +2404,14 @@ vm_page_lookup(
         * new pages can be inserted into this object... this in turn
         * guarantess that the page we're looking for can't exist
         * if the bucket it hashes to is currently NULL even when looked
         * new pages can be inserted into this object... this in turn
         * guarantess that the page we're looking for can't exist
         * if the bucket it hashes to is currently NULL even when looked

-        * at outside the scope of the hash bucket lock... this is a 
+        * at outside the scope of the hash bucket lock... this is a

         * really cheap optimiztion to avoid taking the lock
         */
        if (!bucket->page_list) {
 #if DEBUG_VM_PAGE_LOOKUP
                OSAddAtomic64(1, &vm_page_lookup_stats.vpl_bucket_NULL);
 #endif
         * really cheap optimiztion to avoid taking the lock
         */
        if (!bucket->page_list) {
 #if DEBUG_VM_PAGE_LOOKUP
                OSAddAtomic64(1, &vm_page_lookup_stats.vpl_bucket_NULL);
 #endif

-               return (VM_PAGE_NULL);
+               return VM_PAGE_NULL;

        }
 
 #if DEBUG_VM_PAGE_LOOKUP
        }
 
 #if DEBUG_VM_PAGE_LOOKUP


@@ -2143,26 +2425,27 @@ vm_page_lookup(
                mem = (vm_page_t)vm_page_queue_first(&object->memq);
 
                while (!vm_page_queue_end(&object->memq, (vm_page_queue_entry_t)mem)) {
                mem = (vm_page_t)vm_page_queue_first(&object->memq);
 
                while (!vm_page_queue_end(&object->memq, (vm_page_queue_entry_t)mem)) {

-
-                       if (mem->vmp_offset == offset)
+                       if (mem->vmp_offset == offset) {

                                break;
                                break;

+                       }

 
                        mem = (vm_page_t)vm_page_queue_next(&mem->vmp_listq);
                }
 
                        mem = (vm_page_t)vm_page_queue_next(&mem->vmp_listq);
                }

-               if (vm_page_queue_end(&object->memq, (vm_page_queue_entry_t)mem))
+               if (vm_page_queue_end(&object->memq, (vm_page_queue_entry_t)mem)) {

                        mem = NULL;
                        mem = NULL;

+               }

        } else {
        } else {

-               vm_page_object_t        packed_object;
+               vm_page_object_t        packed_object;

 
                packed_object = VM_PAGE_PACK_OBJECT(object);
 
                bucket_lock = &vm_page_bucket_locks[hash_id / BUCKETS_PER_LOCK];
 
 
                packed_object = VM_PAGE_PACK_OBJECT(object);
 
                bucket_lock = &vm_page_bucket_locks[hash_id / BUCKETS_PER_LOCK];
 

-               lck_spin_lock(bucket_lock);
+               lck_spin_lock_grp(bucket_lock, &vm_page_lck_grp_bucket);

 
                for (mem = (vm_page_t)(VM_PAGE_UNPACK_PTR(bucket->page_list));
 
                for (mem = (vm_page_t)(VM_PAGE_UNPACK_PTR(bucket->page_list));

-                    mem != VM_PAGE_NULL;
-                    mem = (vm_page_t)(VM_PAGE_UNPACK_PTR(mem->vmp_next_m))) {
+                   mem != VM_PAGE_NULL;
+                   mem = (vm_page_t)(VM_PAGE_UNPACK_PTR(mem->vmp_next_m))) {

 #if 0
                        /*
                         * we don't hold the page queue lock
 #if 0
                        /*
                         * we don't hold the page queue lock


@@ -2170,8 +2453,9 @@ vm_page_lookup(
                         */
                        VM_PAGE_CHECK(mem);
 #endif
                         */
                        VM_PAGE_CHECK(mem);
 #endif

-                       if ((mem->vmp_object == packed_object) && (mem->vmp_offset == offset))
+                       if ((mem->vmp_object == packed_object) && (mem->vmp_offset == offset)) {

                                break;
                                break;

+                       }

                }
                lck_spin_unlock(bucket_lock);
        }
                }
                lck_spin_unlock(bucket_lock);
        }


@@ -2186,17 +2470,18 @@ vm_page_lookup(
                OSAddAtomic64(1, &vm_page_lookup_stats.vpl_fast);
                OSAddAtomic64(elapsed, &vm_page_lookup_stats.vpl_fast_elapsed);
        }
                OSAddAtomic64(1, &vm_page_lookup_stats.vpl_fast);
                OSAddAtomic64(elapsed, &vm_page_lookup_stats.vpl_fast_elapsed);
        }

-       if (mem != VM_PAGE_NULL)
+       if (mem != VM_PAGE_NULL) {

                OSAddAtomic64(1, &vm_page_lookup_stats.vpl_hit);
                OSAddAtomic64(1, &vm_page_lookup_stats.vpl_hit);

-       else
-               OSAddAtomic64(1, &vm_page_lookup_stats.vpl_miss);
+       } else {
+               OSAddAtomic64(1, &vm_page_lookup_stats.vpl_miss);
+       }

 #endif
        if (mem != VM_PAGE_NULL) {
                assert(VM_PAGE_OBJECT(mem) == object);
 
                object->memq_hint = mem;
        }
 #endif
        if (mem != VM_PAGE_NULL) {
                assert(VM_PAGE_OBJECT(mem) == object);
 
                object->memq_hint = mem;
        }

-       return (mem);
+       return mem;

 }
 
 
 }
 
 


@@ -2210,23 +2495,18 @@ vm_page_lookup(
  */
 void
 vm_page_rename(
  */
 void
 vm_page_rename(

-       vm_page_t               mem,
-       vm_object_t             new_object,
-       vm_object_offset_t      new_offset)
+       vm_page_t               mem,
+       vm_object_t             new_object,
+       vm_object_offset_t      new_offset)

 {
 {

-       boolean_t       internal_to_external, external_to_internal;
-       vm_tag_t        tag;
-       vm_object_t     m_object;
+       boolean_t       internal_to_external, external_to_internal;
+       vm_tag_t        tag;
+       vm_object_t     m_object;

 
        m_object = VM_PAGE_OBJECT(mem);
 
        assert(m_object != new_object);
 
        m_object = VM_PAGE_OBJECT(mem);
 
        assert(m_object != new_object);

-        assert(m_object);
-
-        XPR(XPR_VM_PAGE,
-                "vm_page_rename, new object 0x%X, offset 0x%X page 0x%X\n",
-                new_object, new_offset, 
-               mem, 0,0);
+       assert(m_object);

 
        /*
         *      Changes to mem->vmp_object require the page lock because
 
        /*
         *      Changes to mem->vmp_object require the page lock because


@@ -2257,7 +2537,7 @@ vm_page_rename(
        }
 
        tag = m_object->wire_tag;
        }
 
        tag = m_object->wire_tag;

-       vm_page_remove(mem, TRUE);
+       vm_page_remove(mem, TRUE);

        vm_page_insert_internal(mem, new_object, new_offset, tag, TRUE, TRUE, FALSE, FALSE, NULL);
 
        if (internal_to_external) {
        vm_page_insert_internal(mem, new_object, new_offset, tag, TRUE, TRUE, FALSE, FALSE, NULL);
 
        if (internal_to_external) {


@@ -2280,20 +2560,38 @@ vm_page_rename(
  */
 void
 vm_page_init(
  */
 void
 vm_page_init(

-       vm_page_t       mem,
-       ppnum_t         phys_page,
-       boolean_t       lopage)
+       vm_page_t mem,
+       ppnum_t   phys_page,
+       boolean_t lopage)

 {
 {

+       uint_t    i;
+       uintptr_t *p;
+

        assert(phys_page);
 
        assert(phys_page);
 

-#if    DEBUG
+#if DEBUG

        if ((phys_page != vm_page_fictitious_addr) && (phys_page != vm_page_guard_addr)) {
                if (!(pmap_valid_page(phys_page))) {
                        panic("vm_page_init: non-DRAM phys_page 0x%x\n", phys_page);
                }
        }
        if ((phys_page != vm_page_fictitious_addr) && (phys_page != vm_page_guard_addr)) {
                if (!(pmap_valid_page(phys_page))) {
                        panic("vm_page_init: non-DRAM phys_page 0x%x\n", phys_page);
                }
        }

-#endif
-       *mem = vm_page_template;
+#endif /* DEBUG */
+
+       /*
+        * Initialize the fields of the vm_page. If adding any new fields to vm_page,
+        * try to use initial values which match 0. This minimizes the number of writes
+        * needed for boot-time initialization.
+        *
+        * Kernel bzero() isn't an inline yet, so do it by hand for performance.
+        */
+       assert(VM_PAGE_NOT_ON_Q == 0);
+       assert(sizeof(*mem) % sizeof(uintptr_t) == 0);
+       for (p = (uintptr_t *)(void *)mem, i = sizeof(*mem) / sizeof(uintptr_t); i != 0; --i) {
+               *p++ = 0;
+       }
+       mem->vmp_offset = (vm_object_offset_t)-1;
+       mem->vmp_busy = TRUE;
+       mem->vmp_lopage = lopage;

 
        VM_PAGE_SET_PHYS_PAGE(mem, phys_page);
 #if 0
 
        VM_PAGE_SET_PHYS_PAGE(mem, phys_page);
 #if 0


@@ -2309,13 +2607,12 @@ vm_page_init(
 
        /*
         * make sure both the h/w referenced and modified bits are
 
        /*
         * make sure both the h/w referenced and modified bits are

-        * clear at this point... we are especially dependent on 
+        * clear at this point... we are especially dependent on

         * not finding a 'stale' h/w modified in a number of spots
         * once this page goes back into use
         */
        pmap_clear_refmod(phys_page, VM_MEM_MODIFIED | VM_MEM_REFERENCED);
 #endif
         * not finding a 'stale' h/w modified in a number of spots
         * once this page goes back into use
         */
        pmap_clear_refmod(phys_page, VM_MEM_MODIFIED | VM_MEM_REFERENCED);
 #endif

-       mem->vmp_lopage = lopage;

 }
 
 /*
 }
 
 /*


@@ -2324,25 +2621,25 @@ vm_page_init(
  *     Remove a fictitious page from the free list.
  *     Returns VM_PAGE_NULL if there are no free pages.
  */
  *     Remove a fictitious page from the free list.
  *     Returns VM_PAGE_NULL if there are no free pages.
  */

-int    c_vm_page_grab_fictitious = 0;
-int    c_vm_page_grab_fictitious_failed = 0;
-int    c_vm_page_release_fictitious = 0;
-int    c_vm_page_more_fictitious = 0;
+int     c_vm_page_grab_fictitious = 0;
+int     c_vm_page_grab_fictitious_failed = 0;
+int     c_vm_page_release_fictitious = 0;
+int     c_vm_page_more_fictitious = 0;

 
 vm_page_t
 vm_page_grab_fictitious_common(
        ppnum_t phys_addr)
 {
 
 vm_page_t
 vm_page_grab_fictitious_common(
        ppnum_t phys_addr)
 {

-       vm_page_t       m;
-
-       if ((m = (vm_page_t)zget(vm_page_zone))) {
+       vm_page_t       m;

 
 

+       if ((m = (vm_page_t)zalloc_noblock(vm_page_zone))) {

                vm_page_init(m, phys_addr, FALSE);
                m->vmp_fictitious = TRUE;
 
                c_vm_page_grab_fictitious++;
                vm_page_init(m, phys_addr, FALSE);
                m->vmp_fictitious = TRUE;
 
                c_vm_page_grab_fictitious++;

-       } else
+       } else {

                c_vm_page_grab_fictitious_failed++;
                c_vm_page_grab_fictitious_failed++;

+       }

 
        return m;
 }
 
        return m;
 }


@@ -2361,7 +2658,9 @@ vm_page_grab_guard(void)
 {
        vm_page_t page;
        page = vm_page_grab_fictitious_common(vm_page_guard_addr);
 {
        vm_page_t page;
        page = vm_page_grab_fictitious_common(vm_page_guard_addr);

-    if (page) OSAddAtomic(1, &vm_guard_count);
+       if (page) {
+               OSAddAtomic(1, &vm_guard_count);
+       }

        return page;
 }
 
        return page;
 }
 


@@ -2378,10 +2677,12 @@ vm_page_release_fictitious(
        assert((m->vmp_q_state == VM_PAGE_NOT_ON_Q) || (m->vmp_q_state == VM_PAGE_IS_WIRED));
        assert(m->vmp_fictitious);
        assert(VM_PAGE_GET_PHYS_PAGE(m) == vm_page_fictitious_addr ||
        assert((m->vmp_q_state == VM_PAGE_NOT_ON_Q) || (m->vmp_q_state == VM_PAGE_IS_WIRED));
        assert(m->vmp_fictitious);
        assert(VM_PAGE_GET_PHYS_PAGE(m) == vm_page_fictitious_addr ||

-              VM_PAGE_GET_PHYS_PAGE(m) == vm_page_guard_addr);
+           VM_PAGE_GET_PHYS_PAGE(m) == vm_page_guard_addr);

 
 
 
 

-if (VM_PAGE_GET_PHYS_PAGE(m) == vm_page_guard_addr) OSAddAtomic(-1, &vm_guard_count);
+       if (VM_PAGE_GET_PHYS_PAGE(m) == vm_page_guard_addr) {
+               OSAddAtomic(-1, &vm_guard_count);
+       }

 
        c_vm_page_release_fictitious++;
 
 
        c_vm_page_release_fictitious++;
 


@@ -2396,7 +2697,7 @@ if (VM_PAGE_GET_PHYS_PAGE(m) == vm_page_guard_addr) OSAddAtomic(-1, &vm_guard_co
  *     with the zones code, for several reasons:
  *     1. we need to carve some page structures out of physical
  *        memory before zones work, so they _cannot_ come from
  *     with the zones code, for several reasons:
  *     1. we need to carve some page structures out of physical
  *        memory before zones work, so they _cannot_ come from

- *        the zone_map.
+ *        the zone restricted submap.

  *     2. the zone needs to be collectable in order to prevent
  *        growth without bound. These structures are used by
  *        the device pager (by the hundreds and thousands), as
  *     2. the zone needs to be collectable in order to prevent
  *        growth without bound. These structures are used by
  *        the device pager (by the hundreds and thousands), as


@@ -2408,27 +2709,29 @@ if (VM_PAGE_GET_PHYS_PAGE(m) == vm_page_guard_addr) OSAddAtomic(-1, &vm_guard_co
  *        zone.
  */
 
  *        zone.
  */
 

-void vm_page_more_fictitious(void)
+void
+vm_page_more_fictitious(void)

 {
 {

-       vm_offset_t     addr;
-       kern_return_t   retval;
+       vm_offset_t     addr;
+       kern_return_t   retval;

 
        c_vm_page_more_fictitious++;
 
        /*
 
        c_vm_page_more_fictitious++;
 
        /*

-        * Allocate a single page from the zone_map. Do not wait if no physical
-        * pages are immediately available, and do not zero the space. We need
-        * our own blocking lock here to prevent having multiple,
-        * simultaneous requests from piling up on the zone_map lock. Exactly
-        * one (of our) threads should be potentially waiting on the map lock.
-        * If winner is not vm-privileged, then the page allocation will fail,
-        * and it will temporarily block here in the vm_page_wait().
+        * Allocate a single page from the zone restricted submap. Do not wait
+        * if no physical pages are immediately available, and do not zero the
+        * space. We need our own blocking lock here to prevent having multiple,
+        * simultaneous requests from piling up on the zone restricted submap
+        * lock.
+        * Exactly one (of our) threads should be potentially waiting on the map
+        * lock.  If winner is not vm-privileged, then the page allocation will
+        * fail, and it will temporarily block here in the vm_page_wait().

         */
        lck_mtx_lock(&vm_page_alloc_lock);
        /*
         * If another thread allocated space, just bail out now.
         */
         */
        lck_mtx_lock(&vm_page_alloc_lock);
        /*
         * If another thread allocated space, just bail out now.
         */

-       if (zone_free_count(vm_page_zone) > 5) {
+       if (os_atomic_load(&vm_page_zone->countfree, relaxed) > 5) {

                /*
                 * The number "5" is a small number that is larger than the
                 * number of fictitious pages that any single caller will
                /*
                 * The number "5" is a small number that is larger than the
                 * number of fictitious pages that any single caller will


@@ -2445,10 +2748,11 @@ void vm_page_more_fictitious(void)
                return;
        }
 
                return;
        }
 

-       retval = kernel_memory_allocate(zone_map,
-                                       &addr, PAGE_SIZE, 0,
-                                       KMA_KOBJECT|KMA_NOPAGEWAIT, VM_KERN_MEMORY_ZONE);
-       if (retval != KERN_SUCCESS) { 
+       retval = kernel_memory_allocate(zone_submap(vm_page_zone),
+           &addr, PAGE_SIZE, 0, KMA_ZERO | KMA_KOBJECT | KMA_NOPAGEWAIT,
+           VM_KERN_MEMORY_ZONE);
+
+       if (retval != KERN_SUCCESS) {

                /*
                 * No page was available. Drop the
                 * lock to give another thread a chance at it, and
                /*
                 * No page was available. Drop the
                 * lock to give another thread a chance at it, and


@@ -2476,7 +2780,7 @@ int
 vm_pool_low(void)
 {
        /* No locking, at worst we will fib. */
 vm_pool_low(void)
 {
        /* No locking, at worst we will fib. */

-       return( vm_page_free_count <= vm_page_free_reserved );
+       return vm_page_free_count <= vm_page_free_reserved;

 }
 
 boolean_t vm_darkwake_mode = FALSE;
 }
 
 boolean_t vm_darkwake_mode = FALSE;


@@ -2520,7 +2824,6 @@ vm_update_darkwake_mode(boolean_t darkwake_mode)
                vm_page_background_target = 0;
 
 #endif /* CONFIG_BACKGROUND_QUEUE */
                vm_page_background_target = 0;
 
 #endif /* CONFIG_BACKGROUND_QUEUE */

-

        } else if (vm_darkwake_mode == FALSE) {
 #if CONFIG_BACKGROUND_QUEUE
 
        } else if (vm_darkwake_mode == FALSE) {
 #if CONFIG_BACKGROUND_QUEUE
 


@@ -2537,13 +2840,15 @@ vm_update_darkwake_mode(boolean_t darkwake_mode)
 void
 vm_page_update_background_state(vm_page_t mem)
 {
 void
 vm_page_update_background_state(vm_page_t mem)
 {

-       if (vm_page_background_mode == VM_PAGE_BG_DISABLED)
+       if (vm_page_background_mode == VM_PAGE_BG_DISABLED) {

                return;
                return;

+       }

 
 

-       if (mem->vmp_in_background == FALSE)
+       if (mem->vmp_in_background == FALSE) {

                return;
                return;

+       }

 
 

-       task_t  my_task = current_task();
+       task_t  my_task = current_task();

 
        if (my_task) {
                if (task_get_darkwake_mode(my_task)) {
 
        if (my_task) {
                if (task_get_darkwake_mode(my_task)) {


@@ -2552,12 +2857,14 @@ vm_page_update_background_state(vm_page_t mem)
        }
 
 #if BACKGROUNDQ_BASED_ON_QOS
        }
 
 #if BACKGROUNDQ_BASED_ON_QOS

-        if (proc_get_effective_thread_policy(current_thread(), TASK_POLICY_QOS) <= THREAD_QOS_LEGACY)
+       if (proc_get_effective_thread_policy(current_thread(), TASK_POLICY_QOS) <= THREAD_QOS_LEGACY) {

                return;
                return;

+       }

 #else
        if (my_task) {
 #else
        if (my_task) {

-               if (proc_get_effective_task_policy(my_task, TASK_POLICY_DARWIN_BG))
+               if (proc_get_effective_task_policy(my_task, TASK_POLICY_DARWIN_BG)) {

                        return;
                        return;

+               }

        }
 #endif
        vm_page_lockspin_queues();
        }
 #endif
        vm_page_lockspin_queues();


@@ -2574,10 +2881,11 @@ vm_page_update_background_state(vm_page_t mem)
 void
 vm_page_assign_background_state(vm_page_t mem)
 {
 void
 vm_page_assign_background_state(vm_page_t mem)
 {

-       if (vm_page_background_mode == VM_PAGE_BG_DISABLED)
+       if (vm_page_background_mode == VM_PAGE_BG_DISABLED) {

                return;
                return;

+       }

 
 

-       task_t  my_task = current_task();
+       task_t  my_task = current_task();

 
        if (my_task) {
                if (task_get_darkwake_mode(my_task)) {
 
        if (my_task) {
                if (task_get_darkwake_mode(my_task)) {


@@ -2587,124 +2895,128 @@ vm_page_assign_background_state(vm_page_t mem)
        }
 
 #if BACKGROUNDQ_BASED_ON_QOS
        }
 
 #if BACKGROUNDQ_BASED_ON_QOS

-        if (proc_get_effective_thread_policy(current_thread(), TASK_POLICY_QOS) <= THREAD_QOS_LEGACY)
-                mem->vmp_in_background = TRUE;
-       else
-                mem->vmp_in_background = FALSE;
+       if (proc_get_effective_thread_policy(current_thread(), TASK_POLICY_QOS) <= THREAD_QOS_LEGACY) {
+               mem->vmp_in_background = TRUE;
+       } else {
+               mem->vmp_in_background = FALSE;
+       }

 #else
 #else

-       if (my_task)
+       if (my_task) {

                mem->vmp_in_background = proc_get_effective_task_policy(my_task, TASK_POLICY_DARWIN_BG);
                mem->vmp_in_background = proc_get_effective_task_policy(my_task, TASK_POLICY_DARWIN_BG);

+       }

 #endif
 }
 
 
 void
 vm_page_remove_from_backgroundq(
 #endif
 }
 
 
 void
 vm_page_remove_from_backgroundq(

-       vm_page_t       mem)
+       vm_page_t       mem)

 {
 {

-       vm_object_t     m_object;
+       vm_object_t     m_object;

 
        LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
 
        if (mem->vmp_on_backgroundq) {
 
        LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
 
        if (mem->vmp_on_backgroundq) {

-               vm_page_queue_remove(&vm_page_queue_background, mem, vm_page_t, vmp_backgroundq);
+               vm_page_queue_remove(&vm_page_queue_background, mem, vmp_backgroundq);

 
                mem->vmp_backgroundq.next = 0;
                mem->vmp_backgroundq.prev = 0;
                mem->vmp_on_backgroundq = FALSE;
 
                mem->vmp_backgroundq.next = 0;
                mem->vmp_backgroundq.prev = 0;
                mem->vmp_on_backgroundq = FALSE;

-               
+

                vm_page_background_count--;
 
                m_object = VM_PAGE_OBJECT(mem);
 
                vm_page_background_count--;
 
                m_object = VM_PAGE_OBJECT(mem);
 

-               if (m_object->internal)
+               if (m_object->internal) {

                        vm_page_background_internal_count--;
                        vm_page_background_internal_count--;

-               else
+               } else {

                        vm_page_background_external_count--;
                        vm_page_background_external_count--;

+               }

        } else {
                assert(VM_PAGE_UNPACK_PTR(mem->vmp_backgroundq.next) == (uintptr_t)NULL &&
        } else {
                assert(VM_PAGE_UNPACK_PTR(mem->vmp_backgroundq.next) == (uintptr_t)NULL &&

-                      VM_PAGE_UNPACK_PTR(mem->vmp_backgroundq.prev) == (uintptr_t)NULL);
+                   VM_PAGE_UNPACK_PTR(mem->vmp_backgroundq.prev) == (uintptr_t)NULL);

        }
 }
 
 
 void
 vm_page_add_to_backgroundq(
        }
 }
 
 
 void
 vm_page_add_to_backgroundq(

-       vm_page_t       mem,
-       boolean_t       first)
-{                              
-       vm_object_t     m_object;
+       vm_page_t       mem,
+       boolean_t       first)
+{
+       vm_object_t     m_object;

 
        LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
 
 
        LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
 

-       if (vm_page_background_mode == VM_PAGE_BG_DISABLED)
+       if (vm_page_background_mode == VM_PAGE_BG_DISABLED) {

                return;
                return;

+       }

 
        if (mem->vmp_on_backgroundq == FALSE) {
 
        if (mem->vmp_on_backgroundq == FALSE) {

-

                m_object = VM_PAGE_OBJECT(mem);
 
                m_object = VM_PAGE_OBJECT(mem);
 

-               if (vm_page_background_exclude_external && !m_object->internal)
+               if (vm_page_background_exclude_external && !m_object->internal) {

                        return;
                        return;

+               }

 
 

-               if (first == TRUE)
-                       vm_page_queue_enter_first(&vm_page_queue_background, mem, vm_page_t, vmp_backgroundq);
-               else
-                       vm_page_queue_enter(&vm_page_queue_background, mem, vm_page_t, vmp_backgroundq);
+               if (first == TRUE) {
+                       vm_page_queue_enter_first(&vm_page_queue_background, mem, vmp_backgroundq);
+               } else {
+                       vm_page_queue_enter(&vm_page_queue_background, mem, vmp_backgroundq);
+               }

                mem->vmp_on_backgroundq = TRUE;
                mem->vmp_on_backgroundq = TRUE;

-               
+

                vm_page_background_count++;
 
                vm_page_background_count++;
 

-               if (m_object->internal)
+               if (m_object->internal) {

                        vm_page_background_internal_count++;
                        vm_page_background_internal_count++;

-               else
+               } else {

                        vm_page_background_external_count++;
                        vm_page_background_external_count++;

+               }

        }
 }
 
 #endif /* CONFIG_BACKGROUND_QUEUE */
 
 /*
        }
 }
 
 #endif /* CONFIG_BACKGROUND_QUEUE */
 
 /*

- * this is an interface to support bring-up of drivers
- * on platforms with physical memory > 4G...
+ * This can be switched to FALSE to help debug drivers
+ * that are having problems with memory > 4G.

  */
  */

-int            vm_himemory_mode = 2;
-
+boolean_t       vm_himemory_mode = TRUE;

 
 /*
  * this interface exists to support hardware controllers
  * incapable of generating DMAs with more than 32 bits
  * of address on platforms with physical memory > 4G...
  */
 
 /*
  * this interface exists to support hardware controllers
  * incapable of generating DMAs with more than 32 bits
  * of address on platforms with physical memory > 4G...
  */

-unsigned int   vm_lopages_allocated_q = 0;
-unsigned int   vm_lopages_allocated_cpm_success = 0;
-unsigned int   vm_lopages_allocated_cpm_failed = 0;
-vm_page_queue_head_t   vm_lopage_queue_free __attribute__((aligned(VM_PACKED_POINTER_ALIGNMENT)));
+unsigned int    vm_lopages_allocated_q = 0;
+unsigned int    vm_lopages_allocated_cpm_success = 0;
+unsigned int    vm_lopages_allocated_cpm_failed = 0;
+vm_page_queue_head_t    vm_lopage_queue_free VM_PAGE_PACKED_ALIGNED;

 
 vm_page_t
 vm_page_grablo(void)
 {
 
 vm_page_t
 vm_page_grablo(void)
 {

-       vm_page_t       mem;
+       vm_page_t       mem;

 
 

-       if (vm_lopage_needed == FALSE)
-               return (vm_page_grab());
+       if (vm_lopage_needed == FALSE) {
+               return vm_page_grab();
+       }

 
        lck_mtx_lock_spin(&vm_page_queue_free_lock);
 
 
        lck_mtx_lock_spin(&vm_page_queue_free_lock);
 

-        if ( !vm_page_queue_empty(&vm_lopage_queue_free)) {
-                vm_page_queue_remove_first(&vm_lopage_queue_free,
-                                   mem,
-                                   vm_page_t,
-                                   vmp_pageq);
+       if (!vm_page_queue_empty(&vm_lopage_queue_free)) {
+               vm_page_queue_remove_first(&vm_lopage_queue_free, mem, vmp_pageq);

                assert(vm_lopage_free_count);
                assert(mem->vmp_q_state == VM_PAGE_ON_FREE_LOPAGE_Q);
                mem->vmp_q_state = VM_PAGE_NOT_ON_Q;
 
                assert(vm_lopage_free_count);
                assert(mem->vmp_q_state == VM_PAGE_ON_FREE_LOPAGE_Q);
                mem->vmp_q_state = VM_PAGE_NOT_ON_Q;
 

-                vm_lopage_free_count--;
+               vm_lopage_free_count--;

                vm_lopages_allocated_q++;
 
                vm_lopages_allocated_q++;
 

-               if (vm_lopage_free_count < vm_lopage_lowater)
+               if (vm_lopage_free_count < vm_lopage_lowater) {

                        vm_lopage_refill = TRUE;
                        vm_lopage_refill = TRUE;

+               }

 
                lck_mtx_unlock(&vm_page_queue_free_lock);
 
 
                lck_mtx_unlock(&vm_page_queue_free_lock);
 


@@ -2714,20 +3026,19 @@ vm_page_grablo(void)
        } else {
                lck_mtx_unlock(&vm_page_queue_free_lock);
 
        } else {
                lck_mtx_unlock(&vm_page_queue_free_lock);
 

-               if (cpm_allocate(PAGE_SIZE, &mem, atop(0xffffffff), 0, FALSE, KMA_LOMEM) != KERN_SUCCESS) {
-
+               if (cpm_allocate(PAGE_SIZE, &mem, atop(PPNUM_MAX), 0, FALSE, KMA_LOMEM) != KERN_SUCCESS) {

                        lck_mtx_lock_spin(&vm_page_queue_free_lock);
                        vm_lopages_allocated_cpm_failed++;
                        lck_mtx_unlock(&vm_page_queue_free_lock);
 
                        lck_mtx_lock_spin(&vm_page_queue_free_lock);
                        vm_lopages_allocated_cpm_failed++;
                        lck_mtx_unlock(&vm_page_queue_free_lock);
 

-                       return (VM_PAGE_NULL);
+                       return VM_PAGE_NULL;

                }
                assert(mem->vmp_q_state == VM_PAGE_NOT_ON_Q);
 
                mem->vmp_busy = TRUE;
 
                vm_page_lockspin_queues();
                }
                assert(mem->vmp_q_state == VM_PAGE_NOT_ON_Q);
 
                mem->vmp_busy = TRUE;
 
                vm_page_lockspin_queues();

-               
+

                mem->vmp_gobbled = FALSE;
                vm_page_gobble_count--;
                vm_page_wire_count--;
                mem->vmp_gobbled = FALSE;
                vm_page_gobble_count--;
                vm_page_wire_count--;


@@ -2743,11 +3054,11 @@ vm_page_grablo(void)
        VM_PAGE_ZERO_PAGEQ_ENTRY(mem);
 
        disable_preemption();
        VM_PAGE_ZERO_PAGEQ_ENTRY(mem);
 
        disable_preemption();

-       PROCESSOR_DATA(current_processor(), page_grab_count) += 1;
+       *PERCPU_GET(vm_page_grab_count) += 1;

        VM_DEBUG_EVENT(vm_page_grab, VM_PAGE_GRAB, DBG_FUNC_NONE, 0, 1, 0, 0);
        enable_preemption();
 
        VM_DEBUG_EVENT(vm_page_grab, VM_PAGE_GRAB, DBG_FUNC_NONE, 0, 1, 0, 0);
        enable_preemption();
 

-       return (mem);
+       return mem;

 }
 
 
 }
 
 


@@ -2756,18 +3067,18 @@ vm_page_grablo(void)
  *
  *     first try to grab a page from the per-cpu free list...
  *     this must be done while pre-emption is disabled... if
  *
  *     first try to grab a page from the per-cpu free list...
  *     this must be done while pre-emption is disabled... if

- *     a page is available, we're done... 
+ *      a page is available, we're done...

  *     if no page is available, grab the vm_page_queue_free_lock
  *     and see if current number of free pages would allow us
  *     if no page is available, grab the vm_page_queue_free_lock
  *     and see if current number of free pages would allow us

- *     to grab at least 1... if not, return VM_PAGE_NULL as before... 
+ *      to grab at least 1... if not, return VM_PAGE_NULL as before...

  *     if there are pages available, disable preemption and
  *     if there are pages available, disable preemption and

- *     recheck the state of the per-cpu free list... we could
+ *      recheck the state of the per-cpu free list... we could

  *     have been preempted and moved to a different cpu, or
  *     have been preempted and moved to a different cpu, or

- *     some other thread could have re-filled it... if still
+ *      some other thread could have re-filled it... if still

  *     empty, figure out how many pages we can steal from the
  *     global free queue and move to the per-cpu queue...
  *     return 1 of these pages when done... only wakeup the
  *     empty, figure out how many pages we can steal from the
  *     global free queue and move to the per-cpu queue...
  *     return 1 of these pages when done... only wakeup the

- *     pageout_scan thread if we moved pages from the global
+ *      pageout_scan thread if we moved pages from the global

  *     list... no need for the wakeup if we've satisfied the
  *     request from the per-cpu queue.
  */
  *     list... no need for the wakeup if we've satisfied the
  *     request from the per-cpu queue.
  */


@@ -2776,10 +3087,13 @@ vm_page_grablo(void)
 vm_page_t vm_page_grab_secluded(void);
 #endif /* CONFIG_SECLUDED_MEMORY */
 
 vm_page_t vm_page_grab_secluded(void);
 #endif /* CONFIG_SECLUDED_MEMORY */
 

+static inline void
+vm_page_grab_diags(void);
+

 vm_page_t
 vm_page_grab(void)
 {
 vm_page_t
 vm_page_grab(void)
 {

-       return vm_page_grab_options(0);
+       return vm_page_grab_options(VM_PAGE_GRAB_OPTIONS_NONE);

 }
 
 #if HIBERNATION
 }
 
 #if HIBERNATION


@@ -2790,11 +3104,11 @@ vm_page_t
 vm_page_grab_options(
        int grab_options)
 {
 vm_page_grab_options(
        int grab_options)
 {

-       vm_page_t       mem;
+       vm_page_t       mem;

 
        disable_preemption();
 
 
        disable_preemption();
 

-       if ((mem = PROCESSOR_DATA(current_processor(), free_pages))) {
+       if ((mem = *PERCPU_GET(free_pages))) {

 return_page_from_cpu_list:
                assert(mem->vmp_q_state == VM_PAGE_ON_FREE_LOCAL_Q);
 
 return_page_from_cpu_list:
                assert(mem->vmp_q_state == VM_PAGE_ON_FREE_LOCAL_Q);
 


@@ -2803,11 +3117,15 @@ return_page_from_cpu_list:
                        panic("%s:%d should not modify cpu->free_pages while hibernating", __FUNCTION__, __LINE__);
                }
 #endif /* HIBERNATION */
                        panic("%s:%d should not modify cpu->free_pages while hibernating", __FUNCTION__, __LINE__);
                }
 #endif /* HIBERNATION */

-               PROCESSOR_DATA(current_processor(), page_grab_count) += 1;
-               PROCESSOR_DATA(current_processor(), free_pages) = mem->vmp_snext;
+
+               vm_page_grab_diags();
+
+               vm_offset_t pcpu_base = current_percpu_base();
+               *PERCPU_GET_WITH_BASE(pcpu_base, vm_page_grab_count) += 1;
+               *PERCPU_GET_WITH_BASE(pcpu_base, free_pages) = mem->vmp_snext;

                VM_DEBUG_EVENT(vm_page_grab, VM_PAGE_GRAB, DBG_FUNC_NONE, grab_options, 0, 0, 0);
 
                VM_DEBUG_EVENT(vm_page_grab, VM_PAGE_GRAB, DBG_FUNC_NONE, grab_options, 0, 0, 0);
 

-               enable_preemption();
+               enable_preemption();

                VM_PAGE_ZERO_PAGEQ_ENTRY(mem);
                mem->vmp_q_state = VM_PAGE_NOT_ON_Q;
 
                VM_PAGE_ZERO_PAGEQ_ENTRY(mem);
                mem->vmp_q_state = VM_PAGE_NOT_ON_Q;
 


@@ -2815,7 +3133,7 @@ return_page_from_cpu_list:
                assert(mem->vmp_tabled == FALSE);
                assert(mem->vmp_object == 0);
                assert(!mem->vmp_laundry);
                assert(mem->vmp_tabled == FALSE);
                assert(mem->vmp_object == 0);
                assert(!mem->vmp_laundry);

-               assertf(pmap_verify_free(VM_PAGE_GET_PHYS_PAGE(mem)), "page = 0x%llx", (uint64_t)VM_PAGE_GET_PHYS_PAGE(mem));
+               ASSERT_PMAP_FREE(mem);

                assert(mem->vmp_busy);
                assert(!mem->vmp_pmapped);
                assert(!mem->vmp_wpmapped);
                assert(mem->vmp_busy);
                assert(!mem->vmp_pmapped);
                assert(!mem->vmp_wpmapped);


@@ -2836,16 +3154,26 @@ return_page_from_cpu_list:
 #if VM_PAGE_WIRE_COUNT_WARNING
        if (vm_page_wire_count >= VM_PAGE_WIRE_COUNT_WARNING) {
                printf("mk: vm_page_grab(): high wired page count of %d\n",
 #if VM_PAGE_WIRE_COUNT_WARNING
        if (vm_page_wire_count >= VM_PAGE_WIRE_COUNT_WARNING) {
                printf("mk: vm_page_grab(): high wired page count of %d\n",

-                       vm_page_wire_count);
+                   vm_page_wire_count);

        }
 #endif
 #if VM_PAGE_GOBBLE_COUNT_WARNING
        if (vm_page_gobble_count >= VM_PAGE_GOBBLE_COUNT_WARNING) {
                printf("mk: vm_page_grab(): high gobbled page count of %d\n",
        }
 #endif
 #if VM_PAGE_GOBBLE_COUNT_WARNING
        if (vm_page_gobble_count >= VM_PAGE_GOBBLE_COUNT_WARNING) {
                printf("mk: vm_page_grab(): high gobbled page count of %d\n",

-                       vm_page_gobble_count);
+                   vm_page_gobble_count);

        }
 #endif
 
        }
 #endif
 

+       /*
+        * If free count is low and we have delayed pages from early boot,
+        * get one of those instead.
+        */
+       if (__improbable(vm_delayed_count > 0 &&
+           vm_page_free_count <= vm_page_free_target &&
+           (mem = vm_get_delayed_page(grab_options)) != NULL)) {
+               return mem;
+       }
+

        lck_mtx_lock_spin(&vm_page_queue_free_lock);
 
        /*
        lck_mtx_lock_spin(&vm_page_queue_free_lock);
 
        /*


@@ -2862,7 +3190,7 @@ return_page_from_cpu_list:
                /* ... but can we try and grab from the secluded queue? */
                if (vm_page_secluded_count > 0 &&
                    ((grab_options & VM_PAGE_GRAB_SECLUDED) ||
                /* ... but can we try and grab from the secluded queue? */
                if (vm_page_secluded_count > 0 &&
                    ((grab_options & VM_PAGE_GRAB_SECLUDED) ||

-                    task_can_use_secluded_mem(current_task(), TRUE))) {
+                   task_can_use_secluded_mem(current_task(), TRUE))) {

                        mem = vm_page_grab_secluded();
                        if (grab_options & VM_PAGE_GRAB_SECLUDED) {
                                vm_page_secluded.grab_for_iokit++;
                        mem = vm_page_grab_secluded();
                        if (grab_options & VM_PAGE_GRAB_SECLUDED) {
                                vm_page_secluded.grab_for_iokit++;


@@ -2874,7 +3202,8 @@ return_page_from_cpu_list:
                                VM_CHECK_MEMORYSTATUS;
 
                                disable_preemption();
                                VM_CHECK_MEMORYSTATUS;
 
                                disable_preemption();

-                               PROCESSOR_DATA(current_processor(), page_grab_count) += 1;
+                               vm_page_grab_diags();
+                               *PERCPU_GET(vm_page_grab_count) += 1;

                                VM_DEBUG_EVENT(vm_page_grab, VM_PAGE_GRAB, DBG_FUNC_NONE, grab_options, 0, 0, 0);
                                enable_preemption();
 
                                VM_DEBUG_EVENT(vm_page_grab, VM_PAGE_GRAB, DBG_FUNC_NONE, grab_options, 0, 0, 0);
                                enable_preemption();
 


@@ -2884,20 +3213,18 @@ return_page_from_cpu_list:
 #else /* CONFIG_SECLUDED_MEMORY */
                (void) grab_options;
 #endif /* CONFIG_SECLUDED_MEMORY */
 #else /* CONFIG_SECLUDED_MEMORY */
                (void) grab_options;
 #endif /* CONFIG_SECLUDED_MEMORY */

-       }
-       else {
-              vm_page_t        head;
-              vm_page_t        tail;
-              unsigned int     pages_to_steal;
-              unsigned int     color;
-              unsigned int clump_end, sub_count;
-
-              while ( vm_page_free_count == 0 ) {
+       } else {
+               vm_page_t        head;
+               vm_page_t        tail;
+               unsigned int     pages_to_steal;
+               unsigned int     color;
+               unsigned int clump_end, sub_count;

 
 

+               while (vm_page_free_count == 0) {

                        lck_mtx_unlock(&vm_page_queue_free_lock);
                        /*
                         * must be a privileged thread to be
                        lck_mtx_unlock(&vm_page_queue_free_lock);
                        /*
                         * must be a privileged thread to be

-                        * in this state since a non-privileged 
+                        * in this state since a non-privileged

                         * thread would have bailed if we were
                         * under the vm_page_free_reserved mark
                         */
                         * thread would have bailed if we were
                         * under the vm_page_free_reserved mark
                         */


@@ -2907,44 +3234,40 @@ return_page_from_cpu_list:
 
                disable_preemption();
 
 
                disable_preemption();
 

-               if ((mem = PROCESSOR_DATA(current_processor(), free_pages))) {
+               if ((mem = *PERCPU_GET(free_pages))) {

                        lck_mtx_unlock(&vm_page_queue_free_lock);
 
                        lck_mtx_unlock(&vm_page_queue_free_lock);
 

-                       /*
+                       /*

                         * we got preempted and moved to another processor
                         * or we got preempted and someone else ran and filled the cache
                         */
                        goto return_page_from_cpu_list;
                }
                         * we got preempted and moved to another processor
                         * or we got preempted and someone else ran and filled the cache
                         */
                        goto return_page_from_cpu_list;
                }

-               if (vm_page_free_count <= vm_page_free_reserved)
-                       pages_to_steal = 1;
-               else {
-                       if (vm_free_magazine_refill_limit <= (vm_page_free_count - vm_page_free_reserved))
+               if (vm_page_free_count <= vm_page_free_reserved) {
+                       pages_to_steal = 1;
+               } else {
+                       if (vm_free_magazine_refill_limit <= (vm_page_free_count - vm_page_free_reserved)) {

                                pages_to_steal = vm_free_magazine_refill_limit;
                                pages_to_steal = vm_free_magazine_refill_limit;

-                       else
-                               pages_to_steal = (vm_page_free_count - vm_page_free_reserved);
+                       } else {
+                               pages_to_steal = (vm_page_free_count - vm_page_free_reserved);
+                       }

                }
                }

-               color = PROCESSOR_DATA(current_processor(), start_color);
+               color = *PERCPU_GET(start_color);

                head = tail = NULL;
 
                vm_page_free_count -= pages_to_steal;
                clump_end = sub_count = 0;
 
                while (pages_to_steal--) {
                head = tail = NULL;
 
                vm_page_free_count -= pages_to_steal;
                clump_end = sub_count = 0;
 
                while (pages_to_steal--) {

-
-                       while (vm_page_queue_empty(&vm_page_queue_free[color].qhead))
-                               color = (color + 1) & vm_color_mask;
+                       while (vm_page_queue_empty(&vm_page_queue_free[color].qhead)) {
+                               color = (color + 1) & vm_color_mask;
+                       }

 #if defined(__x86_64__)
                        vm_page_queue_remove_first_with_clump(&vm_page_queue_free[color].qhead,
 #if defined(__x86_64__)
                        vm_page_queue_remove_first_with_clump(&vm_page_queue_free[color].qhead,

-                                                             mem,
-                                                             vm_page_t,
-                                                             vmp_pageq,
-                                                             clump_end);
+                           mem, clump_end);

 #else
                        vm_page_queue_remove_first(&vm_page_queue_free[color].qhead,
 #else
                        vm_page_queue_remove_first(&vm_page_queue_free[color].qhead,

-                                                             mem,
-                                                             vm_page_t,
-                                                             vmp_pageq);
+                           mem, vmp_pageq);

 #endif
 
                        assert(mem->vmp_q_state == VM_PAGE_ON_FREE_Q);
 #endif
 
                        assert(mem->vmp_q_state == VM_PAGE_ON_FREE_Q);


@@ -2964,17 +3287,20 @@ return_page_from_cpu_list:
                                color = (color + 1) & vm_color_mask;
                        }
 #else
                                color = (color + 1) & vm_color_mask;
                        }
 #else

-                       if (clump_end) color = (color + 1) & vm_color_mask;
+                       if (clump_end) {
+                               color = (color + 1) & vm_color_mask;
+                       }

 
 #endif /* if DEVELOPMENT || DEBUG */
 
 #endif  /* if defined(__arm__) || defined(__arm64__) */
 
 
 #endif /* if DEVELOPMENT || DEBUG */
 
 #endif  /* if defined(__arm__) || defined(__arm64__) */
 

-                       if (head == NULL)
+                       if (head == NULL) {

                                head = mem;
                                head = mem;

-                       else
-                               tail->vmp_snext = mem;
-                       tail = mem;
+                       } else {
+                               tail->vmp_snext = mem;
+                       }
+                       tail = mem;

 
                        assert(mem->vmp_listq.next == 0 && mem->vmp_listq.prev == 0);
                        assert(mem->vmp_tabled == FALSE);
 
                        assert(mem->vmp_listq.next == 0 && mem->vmp_listq.prev == 0);
                        assert(mem->vmp_tabled == FALSE);


@@ -2983,7 +3309,7 @@ return_page_from_cpu_list:
 
                        mem->vmp_q_state = VM_PAGE_ON_FREE_LOCAL_Q;
 
 
                        mem->vmp_q_state = VM_PAGE_ON_FREE_LOCAL_Q;
 

-                       assertf(pmap_verify_free(VM_PAGE_GET_PHYS_PAGE(mem)), "page = 0x%llx", (uint64_t)VM_PAGE_GET_PHYS_PAGE(mem));
+                       ASSERT_PMAP_FREE(mem);

                        assert(mem->vmp_busy);
                        assert(!mem->vmp_pmapped);
                        assert(!mem->vmp_wpmapped);
                        assert(mem->vmp_busy);
                        assert(!mem->vmp_pmapped);
                        assert(!mem->vmp_wpmapped);


@@ -2999,13 +3325,15 @@ return_page_from_cpu_list:
                        panic("%s:%d should not modify cpu->free_pages while hibernating", __FUNCTION__, __LINE__);
                }
 #endif /* HIBERNATION */
                        panic("%s:%d should not modify cpu->free_pages while hibernating", __FUNCTION__, __LINE__);
                }
 #endif /* HIBERNATION */

-               PROCESSOR_DATA(current_processor(), free_pages) = head->vmp_snext;
-               PROCESSOR_DATA(current_processor(), start_color) = color;
+               vm_offset_t pcpu_base = current_percpu_base();
+               *PERCPU_GET_WITH_BASE(pcpu_base, free_pages) = head->vmp_snext;
+               *PERCPU_GET_WITH_BASE(pcpu_base, start_color) = color;

 
                /*
                 * satisfy this request
                 */
 
                /*
                 * satisfy this request
                 */

-               PROCESSOR_DATA(current_processor(), page_grab_count) += 1;
+               vm_page_grab_diags();
+               *PERCPU_GET_WITH_BASE(pcpu_base, vm_page_grab_count) += 1;

                VM_DEBUG_EVENT(vm_page_grab, VM_PAGE_GRAB, DBG_FUNC_NONE, grab_options, 0, 0, 0);
                mem = head;
                assert(mem->vmp_q_state == VM_PAGE_ON_FREE_LOCAL_Q);
                VM_DEBUG_EVENT(vm_page_grab, VM_PAGE_GRAB, DBG_FUNC_NONE, grab_options, 0, 0, 0);
                mem = head;
                assert(mem->vmp_q_state == VM_PAGE_ON_FREE_LOCAL_Q);


@@ -3018,15 +3346,23 @@ return_page_from_cpu_list:
        /*
         *      Decide if we should poke the pageout daemon.
         *      We do this if the free count is less than the low
        /*
         *      Decide if we should poke the pageout daemon.
         *      We do this if the free count is less than the low

-        *      water mark, or if the free count is less than the high
-        *      water mark (but above the low water mark) and the inactive
-        *      count is less than its target.
+        *      water mark. VM Pageout Scan will keep running till
+        *      the free_count > free_target (& hence above free_min).
+        *      This wakeup is to catch the possibility of the counts
+        *      dropping between VM Pageout Scan parking and this check.

         *
         *      We don't have the counts locked ... if they change a little,
         *      it doesn't really matter.
         */
         *
         *      We don't have the counts locked ... if they change a little,
         *      it doesn't really matter.
         */

-       if (vm_page_free_count < vm_page_free_min)
-                thread_wakeup((event_t) &vm_page_free_wanted);
+       if (vm_page_free_count < vm_page_free_min) {
+               lck_mtx_lock(&vm_page_queue_free_lock);
+               if (vm_pageout_running == FALSE) {
+                       lck_mtx_unlock(&vm_page_queue_free_lock);
+                       thread_wakeup((event_t) &vm_page_free_wanted);
+               } else {
+                       lck_mtx_unlock(&vm_page_queue_free_lock);
+               }
+       }

 
        VM_CHECK_MEMORYSTATUS;
 
 
        VM_CHECK_MEMORYSTATUS;
 


@@ -3044,9 +3380,9 @@ return_page_from_cpu_list:
 vm_page_t
 vm_page_grab_secluded(void)
 {
 vm_page_t
 vm_page_grab_secluded(void)
 {

-       vm_page_t       mem;
-       vm_object_t     object;
-       int             refmod_state;
+       vm_page_t       mem;
+       vm_object_t     object;
+       int             refmod_state;

 
        if (vm_page_secluded_count == 0) {
                /* no secluded pages to grab... */
 
        if (vm_page_secluded_count == 0) {
                /* no secluded pages to grab... */


@@ -3066,7 +3402,7 @@ vm_page_grab_secluded(void)
        /* can we grab from the secluded queue? */
        if (vm_page_secluded_count > vm_page_secluded_target ||
            (vm_page_secluded_count > 0 &&
        /* can we grab from the secluded queue? */
        if (vm_page_secluded_count > vm_page_secluded_target ||
            (vm_page_secluded_count > 0 &&

-            task_can_use_secluded_mem(current_task(), TRUE))) {
+           task_can_use_secluded_mem(current_task(), TRUE))) {

                /* OK */
        } else {
                /* can't grab from secluded queue... */
                /* OK */
        } else {
                /* can't grab from secluded queue... */


@@ -3077,8 +3413,8 @@ vm_page_grab_secluded(void)
 
        /* we can grab a page from secluded queue! */
        assert((vm_page_secluded_count_free +
 
        /* we can grab a page from secluded queue! */
        assert((vm_page_secluded_count_free +

-               vm_page_secluded_count_inuse) ==
-              vm_page_secluded_count);
+           vm_page_secluded_count_inuse) ==
+           vm_page_secluded_count);

        if (current_task()->task_can_use_secluded_mem) {
                assert(num_tasks_can_use_secluded_mem > 0);
        }
        if (current_task()->task_can_use_secluded_mem) {
                assert(num_tasks_can_use_secluded_mem > 0);
        }


@@ -3118,7 +3454,7 @@ vm_page_grab_secluded(void)
        if (!vm_object_lock_try(object)) {
 //             printf("SECLUDED: page %p: object %p locked\n", mem, object);
                vm_page_secluded.grab_failure_locked++;
        if (!vm_object_lock_try(object)) {
 //             printf("SECLUDED: page %p: object %p locked\n", mem, object);
                vm_page_secluded.grab_failure_locked++;

-       reactivate_secluded_page:
+reactivate_secluded_page:

                vm_page_activate(mem);
                vm_page_unlock_queues();
                return VM_PAGE_NULL;
                vm_page_activate(mem);
                vm_page_unlock_queues();
                return VM_PAGE_NULL;


@@ -3159,7 +3495,7 @@ vm_page_grab_secluded(void)
        vm_object_unlock(object);
        object = VM_OBJECT_NULL;
        if (vm_page_free_verify) {
        vm_object_unlock(object);
        object = VM_OBJECT_NULL;
        if (vm_page_free_verify) {

-               assertf(pmap_verify_free(VM_PAGE_GET_PHYS_PAGE(mem)), "page = 0x%llx", (uint64_t)VM_PAGE_GET_PHYS_PAGE(mem));
+               ASSERT_PMAP_FREE(mem);

        }
        pmap_clear_noencrypt(VM_PAGE_GET_PHYS_PAGE(mem));
        vm_page_secluded.grab_success_other++;
        }
        pmap_clear_noencrypt(VM_PAGE_GET_PHYS_PAGE(mem));
        vm_page_secluded.grab_success_other++;


@@ -3179,8 +3515,82 @@ vm_page_grab_secluded(void)
 
        return mem;
 }
 
        return mem;
 }

+
+uint64_t
+vm_page_secluded_drain(void)
+{
+       vm_page_t local_freeq;
+       int local_freed;
+       uint64_t num_reclaimed;
+       unsigned int saved_secluded_count, saved_secluded_target;
+
+       num_reclaimed = 0;
+       local_freeq = NULL;
+       local_freed = 0;
+
+       vm_page_lock_queues();
+
+       saved_secluded_count = vm_page_secluded_count;
+       saved_secluded_target = vm_page_secluded_target;
+       vm_page_secluded_target = 0;
+       VM_PAGE_SECLUDED_COUNT_OVER_TARGET_UPDATE();
+       while (vm_page_secluded_count) {
+               vm_page_t secluded_page;
+
+               assert((vm_page_secluded_count_free +
+                   vm_page_secluded_count_inuse) ==
+                   vm_page_secluded_count);
+               secluded_page = (vm_page_t)vm_page_queue_first(&vm_page_queue_secluded);
+               assert(secluded_page->vmp_q_state == VM_PAGE_ON_SECLUDED_Q);
+
+               vm_page_queues_remove(secluded_page, FALSE);
+               assert(!secluded_page->vmp_fictitious);
+               assert(!VM_PAGE_WIRED(secluded_page));
+
+               if (secluded_page->vmp_object == 0) {
+                       /* transfer to free queue */
+                       assert(secluded_page->vmp_busy);
+                       secluded_page->vmp_snext = local_freeq;
+                       local_freeq = secluded_page;
+                       local_freed += 1;
+               } else {
+                       /* transfer to head of active queue */
+                       vm_page_enqueue_active(secluded_page, FALSE);
+                       secluded_page = VM_PAGE_NULL;
+               }
+               num_reclaimed++;
+       }
+       vm_page_secluded_target = saved_secluded_target;
+       VM_PAGE_SECLUDED_COUNT_OVER_TARGET_UPDATE();
+
+//     printf("FBDP %s:%d secluded_count %d->%d, target %d, reclaimed %lld\n", __FUNCTION__, __LINE__, saved_secluded_count, vm_page_secluded_count, vm_page_secluded_target, num_reclaimed);
+
+       vm_page_unlock_queues();
+
+       if (local_freed) {
+               vm_page_free_list(local_freeq, TRUE);
+               local_freeq = NULL;
+               local_freed = 0;
+       }
+
+       return num_reclaimed;
+}

 #endif /* CONFIG_SECLUDED_MEMORY */
 
 #endif /* CONFIG_SECLUDED_MEMORY */
 

+
+static inline void
+vm_page_grab_diags()
+{
+#if DEVELOPMENT || DEBUG
+       task_t task = current_task();
+       if (task == NULL) {
+               return;
+       }
+
+       ledger_credit(task->ledger, task_ledgers.pages_grabbed, 1);
+#endif /* DEVELOPMENT || DEBUG */
+}
+

 /*
  *     vm_page_release:
  *
 /*
  *     vm_page_release:
  *


@@ -3189,15 +3599,16 @@ vm_page_grab_secluded(void)
 
 void
 vm_page_release(
 
 void
 vm_page_release(

-       vm_page_t       mem,
-       boolean_t       page_queues_locked)
+       vm_page_t       mem,
+       boolean_t       page_queues_locked)

 {
 {

-       unsigned int    color;
-       int     need_wakeup = 0;
-       int     need_priv_wakeup = 0;
+       unsigned int    color;
+       int     need_wakeup = 0;
+       int     need_priv_wakeup = 0;

 #if CONFIG_SECLUDED_MEMORY
 #if CONFIG_SECLUDED_MEMORY

-       int     need_secluded_wakeup = 0;
+       int     need_secluded_wakeup = 0;

 #endif /* CONFIG_SECLUDED_MEMORY */
 #endif /* CONFIG_SECLUDED_MEMORY */

+       event_t wakeup_event = NULL;

 
        if (page_queues_locked) {
                LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
 
        if (page_queues_locked) {
                LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);


@@ -3207,7 +3618,7 @@ vm_page_release(
 
        assert(!mem->vmp_private && !mem->vmp_fictitious);
        if (vm_page_free_verify) {
 
        assert(!mem->vmp_private && !mem->vmp_fictitious);
        if (vm_page_free_verify) {

-               assertf(pmap_verify_free(VM_PAGE_GET_PHYS_PAGE(mem)), "page = 0x%llx", (uint64_t)VM_PAGE_GET_PHYS_PAGE(mem));
+               ASSERT_PMAP_FREE(mem);

        }
 //     dbgLog(VM_PAGE_GET_PHYS_PAGE(mem), vm_page_free_count, vm_page_wire_count, 5);  /* (TEST/DEBUG) */
 
        }
 //     dbgLog(VM_PAGE_GET_PHYS_PAGE(mem), vm_page_free_count, vm_page_wire_count, 5);  /* (TEST/DEBUG) */
 


@@ -3223,32 +3634,30 @@ vm_page_release(
        assert(mem->vmp_listq.next == 0 && mem->vmp_listq.prev == 0);
 #if CONFIG_BACKGROUND_QUEUE
        assert(mem->vmp_backgroundq.next == 0 &&
        assert(mem->vmp_listq.next == 0 && mem->vmp_listq.prev == 0);
 #if CONFIG_BACKGROUND_QUEUE
        assert(mem->vmp_backgroundq.next == 0 &&

-              mem->vmp_backgroundq.prev == 0 &&
-              mem->vmp_on_backgroundq == FALSE);
-#endif 
+           mem->vmp_backgroundq.prev == 0 &&
+           mem->vmp_on_backgroundq == FALSE);
+#endif

        if ((mem->vmp_lopage == TRUE || vm_lopage_refill == TRUE) &&
            vm_lopage_free_count < vm_lopage_free_limit &&
            VM_PAGE_GET_PHYS_PAGE(mem) < max_valid_low_ppnum) {
        if ((mem->vmp_lopage == TRUE || vm_lopage_refill == TRUE) &&
            vm_lopage_free_count < vm_lopage_free_limit &&
            VM_PAGE_GET_PHYS_PAGE(mem) < max_valid_low_ppnum) {

-               /*
+               /*

                 * this exists to support hardware controllers
                 * incapable of generating DMAs with more than 32 bits
                 * of address on platforms with physical memory > 4G...
                 */
                 * this exists to support hardware controllers
                 * incapable of generating DMAs with more than 32 bits
                 * of address on platforms with physical memory > 4G...
                 */

-               vm_page_queue_enter_first(&vm_lopage_queue_free,
-                                         mem,
-                                         vm_page_t,
-                                         vmp_pageq);
+               vm_page_queue_enter_first(&vm_lopage_queue_free, mem, vmp_pageq);

                vm_lopage_free_count++;
 
                vm_lopage_free_count++;
 

-               if (vm_lopage_free_count >= vm_lopage_free_limit)
+               if (vm_lopage_free_count >= vm_lopage_free_limit) {

                        vm_lopage_refill = FALSE;
                        vm_lopage_refill = FALSE;

+               }

 
                mem->vmp_q_state = VM_PAGE_ON_FREE_LOPAGE_Q;
                mem->vmp_lopage = TRUE;
 #if CONFIG_SECLUDED_MEMORY
        } else if (vm_page_free_count > vm_page_free_reserved &&
 
                mem->vmp_q_state = VM_PAGE_ON_FREE_LOPAGE_Q;
                mem->vmp_lopage = TRUE;
 #if CONFIG_SECLUDED_MEMORY
        } else if (vm_page_free_count > vm_page_free_reserved &&

-                  vm_page_secluded_count < vm_page_secluded_target &&
-                  num_tasks_can_use_secluded_mem == 0) {
+           vm_page_secluded_count < vm_page_secluded_target &&
+           num_tasks_can_use_secluded_mem == 0) {

                /*
                 * XXX FBDP TODO: also avoid refilling secluded queue
                 * when some IOKit objects are already grabbing from it...
                /*
                 * XXX FBDP TODO: also avoid refilling secluded queue
                 * when some IOKit objects are already grabbing from it...


@@ -3263,19 +3672,17 @@ vm_page_release(
                }
                mem->vmp_lopage = FALSE;
                LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
                }
                mem->vmp_lopage = FALSE;
                LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);

-               vm_page_queue_enter_first(&vm_page_queue_secluded,
-                                         mem,
-                                         vm_page_t,
-                                         vmp_pageq);
+               vm_page_queue_enter_first(&vm_page_queue_secluded, mem, vmp_pageq);

                mem->vmp_q_state = VM_PAGE_ON_SECLUDED_Q;
                vm_page_secluded_count++;
                mem->vmp_q_state = VM_PAGE_ON_SECLUDED_Q;
                vm_page_secluded_count++;

+               VM_PAGE_SECLUDED_COUNT_OVER_TARGET_UPDATE();

                vm_page_secluded_count_free++;
                if (!page_queues_locked) {
                        vm_page_unlock_queues();
                }
                LCK_MTX_ASSERT(&vm_page_queue_free_lock, LCK_MTX_ASSERT_OWNED);
                if (vm_page_free_wanted_secluded > 0) {
                vm_page_secluded_count_free++;
                if (!page_queues_locked) {
                        vm_page_unlock_queues();
                }
                LCK_MTX_ASSERT(&vm_page_queue_free_lock, LCK_MTX_ASSERT_OWNED);
                if (vm_page_free_wanted_secluded > 0) {

-                       vm_page_free_wanted_secluded--;
+                       vm_page_free_wanted_secluded--;

                        need_secluded_wakeup = 1;
                }
 #endif /* CONFIG_SECLUDED_MEMORY */
                        need_secluded_wakeup = 1;
                }
 #endif /* CONFIG_SECLUDED_MEMORY */


@@ -3285,15 +3692,9 @@ vm_page_release(
 
                color = VM_PAGE_GET_COLOR(mem);
 #if defined(__x86_64__)
 
                color = VM_PAGE_GET_COLOR(mem);
 #if defined(__x86_64__)

-               vm_page_queue_enter_clump(&vm_page_queue_free[color].qhead,
-                                         mem,
-                                         vm_page_t,
-                                         vmp_pageq);
+               vm_page_queue_enter_clump(&vm_page_queue_free[color].qhead, mem);

 #else
 #else

-               vm_page_queue_enter(&vm_page_queue_free[color].qhead,
-                                         mem,
-                                         vm_page_t,
-                                         vmp_pageq);
+               vm_page_queue_enter(&vm_page_queue_free[color].qhead, mem, vmp_pageq);

 #endif
                vm_page_free_count++;
                /*
 #endif
                vm_page_free_count++;
                /*


@@ -3318,17 +3719,17 @@ vm_page_release(
 
                assert(vm_page_free_count > 0);
                if (vm_page_free_wanted_privileged > 0) {
 
                assert(vm_page_free_count > 0);
                if (vm_page_free_wanted_privileged > 0) {

-                       vm_page_free_wanted_privileged--;
+                       vm_page_free_wanted_privileged--;

                        need_priv_wakeup = 1;
 #if CONFIG_SECLUDED_MEMORY
                } else if (vm_page_free_wanted_secluded > 0 &&
                        need_priv_wakeup = 1;
 #if CONFIG_SECLUDED_MEMORY
                } else if (vm_page_free_wanted_secluded > 0 &&

-                          vm_page_free_count > vm_page_free_reserved) {
+                   vm_page_free_count > vm_page_free_reserved) {

                        vm_page_free_wanted_secluded--;
                        need_secluded_wakeup = 1;
 #endif /* CONFIG_SECLUDED_MEMORY */
                } else if (vm_page_free_wanted > 0 &&
                        vm_page_free_wanted_secluded--;
                        need_secluded_wakeup = 1;
 #endif /* CONFIG_SECLUDED_MEMORY */
                } else if (vm_page_free_wanted > 0 &&

-                          vm_page_free_count > vm_page_free_reserved) {
-                       vm_page_free_wanted--;
+                   vm_page_free_count > vm_page_free_reserved) {
+                       vm_page_free_wanted--;

                        need_wakeup = 1;
                }
        }
                        need_wakeup = 1;
                }
        }


@@ -3338,29 +3739,42 @@ vm_page_release(
 
        lck_mtx_unlock(&vm_page_queue_free_lock);
 
 
        lck_mtx_unlock(&vm_page_queue_free_lock);
 

-       if (need_priv_wakeup)
-               thread_wakeup_one((event_t) &vm_page_free_wanted_privileged);
+       if (need_priv_wakeup) {
+               wakeup_event = &vm_page_free_wanted_privileged;
+       }

 #if CONFIG_SECLUDED_MEMORY
 #if CONFIG_SECLUDED_MEMORY

-       else if (need_secluded_wakeup)
-               thread_wakeup_one((event_t) &vm_page_free_wanted_secluded);
+       else if (need_secluded_wakeup) {
+               wakeup_event = &vm_page_free_wanted_secluded;
+       }

 #endif /* CONFIG_SECLUDED_MEMORY */
 #endif /* CONFIG_SECLUDED_MEMORY */

-       else if (need_wakeup)
-               thread_wakeup_one((event_t) &vm_page_free_count);
+       else if (need_wakeup) {
+               wakeup_event = &vm_page_free_count;
+       }
+
+       if (wakeup_event) {
+               if (vps_dynamic_priority_enabled == TRUE) {
+                       thread_t thread_woken = NULL;
+                       wakeup_one_with_inheritor((event_t) wakeup_event, THREAD_AWAKENED, LCK_WAKE_DO_NOT_TRANSFER_PUSH, &thread_woken);
+                       thread_deallocate(thread_woken);
+               } else {
+                       thread_wakeup_one((event_t) wakeup_event);
+               }
+       }

 
        VM_CHECK_MEMORYSTATUS;
 }
 
 /*
  * This version of vm_page_release() is used only at startup
 
        VM_CHECK_MEMORYSTATUS;
 }
 
 /*
  * This version of vm_page_release() is used only at startup

- * when we are single-threaded and pages are being released 
+ * when we are single-threaded and pages are being released

  * for the first time. Hence, no locking or unnecessary checks are made.
  * Note: VM_CHECK_MEMORYSTATUS invoked by the caller.
  */
 void
 vm_page_release_startup(
  * for the first time. Hence, no locking or unnecessary checks are made.
  * Note: VM_CHECK_MEMORYSTATUS invoked by the caller.
  */
 void
 vm_page_release_startup(

-       vm_page_t       mem)
+       vm_page_t       mem)

 {
 {

-       vm_page_queue_t queue_free;
+       vm_page_queue_t queue_free;

 
        if (vm_lopage_free_count < vm_lopage_free_limit &&
            VM_PAGE_GET_PHYS_PAGE(mem) < max_valid_low_ppnum) {
 
        if (vm_lopage_free_count < vm_lopage_free_limit &&
            VM_PAGE_GET_PHYS_PAGE(mem) < max_valid_low_ppnum) {


@@ -3373,6 +3787,7 @@ vm_page_release_startup(
                mem->vmp_lopage = FALSE;
                mem->vmp_q_state = VM_PAGE_ON_SECLUDED_Q;
                vm_page_secluded_count++;
                mem->vmp_lopage = FALSE;
                mem->vmp_q_state = VM_PAGE_ON_SECLUDED_Q;
                vm_page_secluded_count++;

+               VM_PAGE_SECLUDED_COUNT_OVER_TARGET_UPDATE();

                vm_page_secluded_count_free++;
                queue_free = &vm_page_queue_secluded;
 #endif /* CONFIG_SECLUDED_MEMORY */
                vm_page_secluded_count_free++;
                queue_free = &vm_page_queue_secluded;
 #endif /* CONFIG_SECLUDED_MEMORY */


@@ -3384,12 +3799,13 @@ vm_page_release_startup(
        }
        if (mem->vmp_q_state == VM_PAGE_ON_FREE_Q) {
 #if defined(__x86_64__)
        }
        if (mem->vmp_q_state == VM_PAGE_ON_FREE_Q) {
 #if defined(__x86_64__)

-               vm_page_queue_enter_clump(queue_free, mem, vm_page_t, vmp_pageq);
+               vm_page_queue_enter_clump(queue_free, mem);

 #else
 #else

-               vm_page_queue_enter(queue_free, mem, vm_page_t, vmp_pageq);
+               vm_page_queue_enter(queue_free, mem, vmp_pageq);

 #endif
 #endif

-       } else
-               vm_page_queue_enter_first(queue_free, mem, vm_page_t, vmp_pageq);
+       } else {
+               vm_page_queue_enter_first(queue_free, mem, vmp_pageq);
+       }

 }
 
 /*
 }
 
 /*


@@ -3405,7 +3821,7 @@ vm_page_release_startup(
 
 boolean_t
 vm_page_wait(
 
 boolean_t
 vm_page_wait(

-       int     interruptible )
+       int     interruptible )

 {
        /*
         *      We can't use vm_page_free_reserved to make this
 {
        /*
         *      We can't use vm_page_free_reserved to make this


@@ -3414,9 +3830,10 @@ vm_page_wait(
         *      succeeds, the second fails.  After the first page is freed,
         *      a call to vm_page_wait must really block.
         */
         *      succeeds, the second fails.  After the first page is freed,
         *      a call to vm_page_wait must really block.
         */

-       kern_return_t   wait_result;
-       int             need_wakeup = 0;
-       int             is_privileged = current_thread()->options & TH_OPT_VMPRIV;
+       kern_return_t   wait_result;
+       int             need_wakeup = 0;
+       int             is_privileged = current_thread()->options & TH_OPT_VMPRIV;
+       event_t         wait_event = NULL;

 
        lck_mtx_lock_spin(&vm_page_queue_free_lock);
 
 
        lck_mtx_lock_spin(&vm_page_queue_free_lock);
 


@@ -3431,12 +3848,13 @@ vm_page_wait(
        }
 
        if (is_privileged) {
        }
 
        if (is_privileged) {

-               if (vm_page_free_wanted_privileged++ == 0)
+               if (vm_page_free_wanted_privileged++ == 0) {

                        need_wakeup = 1;
                        need_wakeup = 1;

-               wait_result = assert_wait((event_t)&vm_page_free_wanted_privileged, interruptible);
+               }
+               wait_event = (event_t)&vm_page_free_wanted_privileged;

 #if CONFIG_SECLUDED_MEMORY
        } else if (secluded_for_apps &&
 #if CONFIG_SECLUDED_MEMORY
        } else if (secluded_for_apps &&

-                  task_can_use_secluded_mem(current_task(), FALSE)) {
+           task_can_use_secluded_mem(current_task(), FALSE)) {

 #if 00
                /* XXX FBDP: need pageq lock for this... */
                /* XXX FBDP: might wait even if pages available, */
 #if 00
                /* XXX FBDP: need pageq lock for this... */
                /* XXX FBDP: might wait even if pages available, */


@@ -3449,38 +3867,83 @@ vm_page_wait(
                if (vm_page_free_wanted_secluded++ == 0) {
                        need_wakeup = 1;
                }
                if (vm_page_free_wanted_secluded++ == 0) {
                        need_wakeup = 1;
                }

-               wait_result = assert_wait(
-                       (event_t)&vm_page_free_wanted_secluded,
-                       interruptible);
+               wait_event = (event_t)&vm_page_free_wanted_secluded;

 #endif /* CONFIG_SECLUDED_MEMORY */
        } else {
 #endif /* CONFIG_SECLUDED_MEMORY */
        } else {

-               if (vm_page_free_wanted++ == 0)
+               if (vm_page_free_wanted++ == 0) {

                        need_wakeup = 1;
                        need_wakeup = 1;

-               wait_result = assert_wait((event_t)&vm_page_free_count,
-                                         interruptible);
+               }
+               wait_event = (event_t)&vm_page_free_count;

        }
        }

-       lck_mtx_unlock(&vm_page_queue_free_lock);
-       counter(c_vm_page_wait_block++);

 
 

-       if (need_wakeup)
-               thread_wakeup((event_t)&vm_page_free_wanted);
+       /*
+        * We don't do a vm_pageout_scan wakeup if we already have
+        * some waiters because vm_pageout_scan checks for waiters
+        * before it returns and does so behind the vm_page_queue_free_lock,
+        * which we own when we bump the waiter counts.
+        */
+
+       if (vps_dynamic_priority_enabled == TRUE) {
+               /*
+                * We are waking up vm_pageout_scan here. If it needs
+                * the vm_page_queue_free_lock before we unlock it
+                * we'll end up just blocking and incur an extra
+                * context switch. Could be a perf. issue.
+                */
+
+               counter(c_vm_page_wait_block++);
+
+               if (need_wakeup) {
+                       thread_wakeup((event_t)&vm_page_free_wanted);
+               }

 
 

-       if (wait_result == THREAD_WAITING) {
+               /*
+                * LD: This event is going to get recorded every time because
+                * we don't get back THREAD_WAITING from lck_mtx_sleep_with_inheritor.
+                * We just block in that routine.
+                */

                VM_DEBUG_CONSTANT_EVENT(vm_page_wait_block, VM_PAGE_WAIT_BLOCK, DBG_FUNC_START,
                VM_DEBUG_CONSTANT_EVENT(vm_page_wait_block, VM_PAGE_WAIT_BLOCK, DBG_FUNC_START,

-                              vm_page_free_wanted_privileged,
-                              vm_page_free_wanted,
+                   vm_page_free_wanted_privileged,
+                   vm_page_free_wanted,

 #if CONFIG_SECLUDED_MEMORY
 #if CONFIG_SECLUDED_MEMORY

-                              vm_page_free_wanted_secluded,
+                   vm_page_free_wanted_secluded,

 #else /* CONFIG_SECLUDED_MEMORY */
 #else /* CONFIG_SECLUDED_MEMORY */

-                              0,
+                   0,

 #endif /* CONFIG_SECLUDED_MEMORY */
 #endif /* CONFIG_SECLUDED_MEMORY */

-                              0);
-               wait_result = thread_block(THREAD_CONTINUE_NULL);
-               VM_DEBUG_CONSTANT_EVENT(vm_page_wait_block,
-                                  VM_PAGE_WAIT_BLOCK, DBG_FUNC_END, 0, 0, 0, 0);
+                   0);
+               wait_result =  lck_mtx_sleep_with_inheritor(&vm_page_queue_free_lock,
+                   LCK_SLEEP_UNLOCK,
+                   wait_event,
+                   vm_pageout_scan_thread,
+                   interruptible,
+                   0);
+       } else {
+               wait_result = assert_wait(wait_event, interruptible);
+
+               lck_mtx_unlock(&vm_page_queue_free_lock);
+               counter(c_vm_page_wait_block++);
+
+               if (need_wakeup) {
+                       thread_wakeup((event_t)&vm_page_free_wanted);
+               }
+
+               if (wait_result == THREAD_WAITING) {
+                       VM_DEBUG_CONSTANT_EVENT(vm_page_wait_block, VM_PAGE_WAIT_BLOCK, DBG_FUNC_START,
+                           vm_page_free_wanted_privileged,
+                           vm_page_free_wanted,
+#if CONFIG_SECLUDED_MEMORY
+                           vm_page_free_wanted_secluded,
+#else /* CONFIG_SECLUDED_MEMORY */
+                           0,
+#endif /* CONFIG_SECLUDED_MEMORY */
+                           0);
+                       wait_result = thread_block(THREAD_CONTINUE_NULL);
+                       VM_DEBUG_CONSTANT_EVENT(vm_page_wait_block,
+                           VM_PAGE_WAIT_BLOCK, DBG_FUNC_END, 0, 0, 0, 0);
+               }

        }
 
        }
 

-       return (wait_result == THREAD_AWAKENED);
+       return (wait_result == THREAD_AWAKENED) || (wait_result == THREAD_NOT_WAITING);

 }
 
 /*
 }
 
 /*


@@ -3494,11 +3957,11 @@ vm_page_wait(
 
 vm_page_t
 vm_page_alloc(
 
 vm_page_t
 vm_page_alloc(

-       vm_object_t             object,
-       vm_object_offset_t      offset)
+       vm_object_t             object,
+       vm_object_offset_t      offset)

 {
 {

-       vm_page_t       mem;
-       int             grab_options;
+       vm_page_t       mem;
+       int             grab_options;

 
        vm_object_lock_assert_exclusive(object);
        grab_options = 0;
 
        vm_object_lock_assert_exclusive(object);
        grab_options = 0;


@@ -3508,41 +3971,43 @@ vm_page_alloc(
        }
 #endif /* CONFIG_SECLUDED_MEMORY */
        mem = vm_page_grab_options(grab_options);
        }
 #endif /* CONFIG_SECLUDED_MEMORY */
        mem = vm_page_grab_options(grab_options);

-       if (mem == VM_PAGE_NULL)
+       if (mem == VM_PAGE_NULL) {

                return VM_PAGE_NULL;
                return VM_PAGE_NULL;

+       }

 
        vm_page_insert(mem, object, offset);
 
 
        vm_page_insert(mem, object, offset);
 

-       return(mem);
+       return mem;

 }
 
 /*
  *     vm_page_alloc_guard:
 }
 
 /*
  *     vm_page_alloc_guard:

- *     
- *     Allocate a fictitious page which will be used
+ *
+ *      Allocate a fictitious page which will be used

  *     as a guard page.  The page will be inserted into
  *     the object and returned to the caller.
  */
 
 vm_page_t
 vm_page_alloc_guard(
  *     as a guard page.  The page will be inserted into
  *     the object and returned to the caller.
  */
 
 vm_page_t
 vm_page_alloc_guard(

-       vm_object_t             object,
-       vm_object_offset_t      offset)
+       vm_object_t             object,
+       vm_object_offset_t      offset)

 {
 {

-       vm_page_t       mem;
+       vm_page_t       mem;

 
        vm_object_lock_assert_exclusive(object);
        mem = vm_page_grab_guard();
 
        vm_object_lock_assert_exclusive(object);
        mem = vm_page_grab_guard();

-       if (mem == VM_PAGE_NULL)
+       if (mem == VM_PAGE_NULL) {

                return VM_PAGE_NULL;
                return VM_PAGE_NULL;

+       }

 
        vm_page_insert(mem, object, offset);
 
 
        vm_page_insert(mem, object, offset);
 

-       return(mem);
+       return mem;

 }
 
 
 }
 
 

-counter(unsigned int c_laundry_pages_freed = 0;)
+counter(unsigned int c_laundry_pages_freed = 0; )

 
 /*
  *     vm_page_free_prepare:
 
 /*
  *     vm_page_free_prepare:


@@ -3554,7 +4019,7 @@ counter(unsigned int c_laundry_pages_freed = 0;)
  */
 static void
 vm_page_free_prepare(
  */
 static void
 vm_page_free_prepare(

-       vm_page_t       mem)
+       vm_page_t       mem)

 {
        vm_page_free_prepare_queues(mem);
        vm_page_free_prepare_object(mem, TRUE);
 {
        vm_page_free_prepare_queues(mem);
        vm_page_free_prepare_object(mem, TRUE);


@@ -3563,9 +4028,9 @@ vm_page_free_prepare(
 
 void
 vm_page_free_prepare_queues(
 
 void
 vm_page_free_prepare_queues(

-       vm_page_t       mem)
+       vm_page_t       mem)

 {
 {

-       vm_object_t     m_object;
+       vm_object_t     m_object;

 
        VM_PAGE_CHECK(mem);
 
 
        VM_PAGE_CHECK(mem);
 


@@ -3588,20 +4053,19 @@ vm_page_free_prepare_queues(
                vm_pageout_steal_laundry(mem, TRUE);
                counter(++c_laundry_pages_freed);
        }
                vm_pageout_steal_laundry(mem, TRUE);
                counter(++c_laundry_pages_freed);
        }

-       
+

        vm_page_queues_remove(mem, TRUE);
 
        if (VM_PAGE_WIRED(mem)) {
                assert(mem->vmp_wire_count > 0);
 
                if (m_object) {
        vm_page_queues_remove(mem, TRUE);
 
        if (VM_PAGE_WIRED(mem)) {
                assert(mem->vmp_wire_count > 0);
 
                if (m_object) {

-

                        VM_OBJECT_WIRED_PAGE_UPDATE_START(m_object);
                        VM_OBJECT_WIRED_PAGE_REMOVE(m_object, mem);
                        VM_OBJECT_WIRED_PAGE_UPDATE_END(m_object, m_object->wire_tag);
 
                        assert(m_object->resident_page_count >=
                        VM_OBJECT_WIRED_PAGE_UPDATE_START(m_object);
                        VM_OBJECT_WIRED_PAGE_REMOVE(m_object, mem);
                        VM_OBJECT_WIRED_PAGE_UPDATE_END(m_object, m_object->wire_tag);
 
                        assert(m_object->resident_page_count >=

-                              m_object->wired_page_count);
+                           m_object->wired_page_count);

 
                        if (m_object->purgable == VM_PURGABLE_VOLATILE) {
                                OSAddAtomic(+1, &vm_page_purgeable_count);
 
                        if (m_object->purgable == VM_PURGABLE_VOLATILE) {
                                OSAddAtomic(+1, &vm_page_purgeable_count);


@@ -3609,14 +4073,14 @@ vm_page_free_prepare_queues(
                                OSAddAtomic(-1, &vm_page_purgeable_wired_count);
                        }
                        if ((m_object->purgable == VM_PURGABLE_VOLATILE ||
                                OSAddAtomic(-1, &vm_page_purgeable_wired_count);
                        }
                        if ((m_object->purgable == VM_PURGABLE_VOLATILE ||

-                            m_object->purgable == VM_PURGABLE_EMPTY) &&
+                           m_object->purgable == VM_PURGABLE_EMPTY) &&

                            m_object->vo_owner != TASK_NULL) {
                            m_object->vo_owner != TASK_NULL) {

-                               task_t          owner;
-                               int             ledger_idx_volatile;
-                               int             ledger_idx_nonvolatile;
-                               int             ledger_idx_volatile_compressed;
-                               int             ledger_idx_nonvolatile_compressed;
-                               boolean_t       do_footprint;
+                               task_t          owner;
+                               int             ledger_idx_volatile;
+                               int             ledger_idx_nonvolatile;
+                               int             ledger_idx_volatile_compressed;
+                               int             ledger_idx_nonvolatile_compressed;
+                               boolean_t       do_footprint;

 
                                owner = VM_OBJECT_OWNER(m_object);
                                vm_object_ledger_tag_ledgers(
 
                                owner = VM_OBJECT_OWNER(m_object);
                                vm_object_ledger_tag_ledgers(


@@ -3633,29 +4097,31 @@ vm_page_free_prepare_queues(
                                 */
                                /* one less "non-volatile"... */
                                ledger_debit(owner->ledger,
                                 */
                                /* one less "non-volatile"... */
                                ledger_debit(owner->ledger,

-                                            ledger_idx_nonvolatile,
-                                            PAGE_SIZE);
+                                   ledger_idx_nonvolatile,
+                                   PAGE_SIZE);

                                if (do_footprint) {
                                        /* ... and "phys_footprint" */
                                        ledger_debit(owner->ledger,
                                if (do_footprint) {
                                        /* ... and "phys_footprint" */
                                        ledger_debit(owner->ledger,

-                                                    task_ledgers.phys_footprint,
-                                                    PAGE_SIZE);
+                                           task_ledgers.phys_footprint,
+                                           PAGE_SIZE);

                                }
                                /* one more "volatile" */
                                ledger_credit(owner->ledger,
                                }
                                /* one more "volatile" */
                                ledger_credit(owner->ledger,

-                                             ledger_idx_volatile,
-                                             PAGE_SIZE);
+                                   ledger_idx_volatile,
+                                   PAGE_SIZE);

                        }
                }
                        }
                }

-               if (!mem->vmp_private && !mem->vmp_fictitious)
+               if (!mem->vmp_private && !mem->vmp_fictitious) {

                        vm_page_wire_count--;
                        vm_page_wire_count--;

+               }

 
                mem->vmp_q_state = VM_PAGE_NOT_ON_Q;
                mem->vmp_wire_count = 0;
                assert(!mem->vmp_gobbled);
        } else if (mem->vmp_gobbled) {
 
                mem->vmp_q_state = VM_PAGE_NOT_ON_Q;
                mem->vmp_wire_count = 0;
                assert(!mem->vmp_gobbled);
        } else if (mem->vmp_gobbled) {

-               if (!mem->vmp_private && !mem->vmp_fictitious)
+               if (!mem->vmp_private && !mem->vmp_fictitious) {

                        vm_page_wire_count--;
                        vm_page_wire_count--;

+               }

                vm_page_gobble_count--;
        }
 }
                vm_page_gobble_count--;
        }
 }


@@ -3663,20 +4129,20 @@ vm_page_free_prepare_queues(
 
 void
 vm_page_free_prepare_object(
 
 void
 vm_page_free_prepare_object(

-       vm_page_t       mem,
-       boolean_t       remove_from_hash)
+       vm_page_t       mem,
+       boolean_t       remove_from_hash)

 {
 {

-       if (mem->vmp_tabled)
-               vm_page_remove(mem, remove_from_hash);  /* clears tabled, object, offset */
-
-       PAGE_WAKEUP(mem);               /* clears wanted */
+       if (mem->vmp_tabled) {
+               vm_page_remove(mem, remove_from_hash);  /* clears tabled, object, offset */
+       }
+       PAGE_WAKEUP(mem);               /* clears wanted */

 
        if (mem->vmp_private) {
                mem->vmp_private = FALSE;
                mem->vmp_fictitious = TRUE;
                VM_PAGE_SET_PHYS_PAGE(mem, vm_page_fictitious_addr);
        }
 
        if (mem->vmp_private) {
                mem->vmp_private = FALSE;
                mem->vmp_fictitious = TRUE;
                VM_PAGE_SET_PHYS_PAGE(mem, vm_page_fictitious_addr);
        }

-       if ( !mem->vmp_fictitious) {
+       if (!mem->vmp_fictitious) {

                assert(mem->vmp_pageq.next == 0);
                assert(mem->vmp_pageq.prev == 0);
                assert(mem->vmp_listq.next == 0);
                assert(mem->vmp_pageq.next == 0);
                assert(mem->vmp_pageq.prev == 0);
                assert(mem->vmp_listq.next == 0);


@@ -3686,6 +4152,7 @@ vm_page_free_prepare_object(
                assert(mem->vmp_backgroundq.prev == 0);
 #endif /* CONFIG_BACKGROUND_QUEUE */
                assert(mem->vmp_next_m == 0);
                assert(mem->vmp_backgroundq.prev == 0);
 #endif /* CONFIG_BACKGROUND_QUEUE */
                assert(mem->vmp_next_m == 0);

+               ASSERT_PMAP_FREE(mem);

                vm_page_init(mem, VM_PAGE_GET_PHYS_PAGE(mem), mem->vmp_lopage);
        }
 }
                vm_page_init(mem, VM_PAGE_GET_PHYS_PAGE(mem), mem->vmp_lopage);
        }
 }


@@ -3701,7 +4168,7 @@ vm_page_free_prepare_object(
  */
 void
 vm_page_free(
  */
 void
 vm_page_free(

-       vm_page_t       mem)
+       vm_page_t       mem)

 {
        vm_page_free_prepare(mem);
 
 {
        vm_page_free_prepare(mem);
 


@@ -3709,15 +4176,15 @@ vm_page_free(
                vm_page_release_fictitious(mem);
        } else {
                vm_page_release(mem,
                vm_page_release_fictitious(mem);
        } else {
                vm_page_release(mem,

-                               TRUE); /* page queues are locked */
+                   TRUE);             /* page queues are locked */

        }
 }
 
 
 void
 vm_page_free_unlocked(
        }
 }
 
 
 void
 vm_page_free_unlocked(

-       vm_page_t       mem,
-       boolean_t       remove_from_hash)
+       vm_page_t       mem,
+       boolean_t       remove_from_hash)

 {
        vm_page_lockspin_queues();
        vm_page_free_prepare_queues(mem);
 {
        vm_page_lockspin_queues();
        vm_page_free_prepare_queues(mem);


@@ -3744,19 +4211,18 @@ vm_page_free_unlocked(
  */
 void
 vm_page_free_list(
  */
 void
 vm_page_free_list(

-       vm_page_t       freeq,
-       boolean_t       prepare_object)
+       vm_page_t       freeq,
+       boolean_t       prepare_object)

 {
 {

-        vm_page_t      mem;
-        vm_page_t      nxt;
-       vm_page_t       local_freeq;
-       int             pg_count;
+       vm_page_t       mem;
+       vm_page_t       nxt;
+       vm_page_t       local_freeq;
+       int             pg_count;

 
        LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_NOTOWNED);
        LCK_MTX_ASSERT(&vm_page_queue_free_lock, LCK_MTX_ASSERT_NOTOWNED);
 
        while (freeq) {
 
        LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_NOTOWNED);
        LCK_MTX_ASSERT(&vm_page_queue_free_lock, LCK_MTX_ASSERT_NOTOWNED);
 
        while (freeq) {

-

                pg_count = 0;
                local_freeq = VM_PAGE_NULL;
                mem = freeq;
                pg_count = 0;
                local_freeq = VM_PAGE_NULL;
                mem = freeq;


@@ -3768,23 +4234,23 @@ vm_page_free_list(
                 * contention on the global free queue lock
                 */
                while (mem && pg_count < 64) {
                 * contention on the global free queue lock
                 */
                while (mem && pg_count < 64) {

-

                        assert((mem->vmp_q_state == VM_PAGE_NOT_ON_Q) ||
                        assert((mem->vmp_q_state == VM_PAGE_NOT_ON_Q) ||

-                              (mem->vmp_q_state == VM_PAGE_IS_WIRED));
+                           (mem->vmp_q_state == VM_PAGE_IS_WIRED));

 #if CONFIG_BACKGROUND_QUEUE
                        assert(mem->vmp_backgroundq.next == 0 &&
 #if CONFIG_BACKGROUND_QUEUE
                        assert(mem->vmp_backgroundq.next == 0 &&

-                              mem->vmp_backgroundq.prev == 0 &&
-                              mem->vmp_on_backgroundq == FALSE);
+                           mem->vmp_backgroundq.prev == 0 &&
+                           mem->vmp_on_backgroundq == FALSE);

 #endif
                        nxt = mem->vmp_snext;
                        mem->vmp_snext = NULL;
                        assert(mem->vmp_pageq.prev == 0);
 
                        if (vm_page_free_verify && !mem->vmp_fictitious && !mem->vmp_private) {
 #endif
                        nxt = mem->vmp_snext;
                        mem->vmp_snext = NULL;
                        assert(mem->vmp_pageq.prev == 0);
 
                        if (vm_page_free_verify && !mem->vmp_fictitious && !mem->vmp_private) {

-                               assertf(pmap_verify_free(VM_PAGE_GET_PHYS_PAGE(mem)), "page = 0x%llx", (uint64_t)VM_PAGE_GET_PHYS_PAGE(mem));
+                               ASSERT_PMAP_FREE(mem);

                        }
                        }

-                       if (prepare_object == TRUE)
+                       if (prepare_object == TRUE) {

                                vm_page_free_prepare_object(mem, TRUE);
                                vm_page_free_prepare_object(mem, TRUE);

+                       }

 
                        if (!mem->vmp_fictitious) {
                                assert(mem->vmp_busy);
 
                        if (!mem->vmp_fictitious) {
                                assert(mem->vmp_busy);


@@ -3795,9 +4261,9 @@ vm_page_free_list(
                                        vm_page_release(mem, FALSE); /* page queues are not locked */
 #if CONFIG_SECLUDED_MEMORY
                                } else if (vm_page_secluded_count < vm_page_secluded_target &&
                                        vm_page_release(mem, FALSE); /* page queues are not locked */
 #if CONFIG_SECLUDED_MEMORY
                                } else if (vm_page_secluded_count < vm_page_secluded_target &&

-                                          num_tasks_can_use_secluded_mem == 0) {
+                                   num_tasks_can_use_secluded_mem == 0) {

                                        vm_page_release(mem,
                                        vm_page_release(mem,

-                                                       FALSE); /* page queues are not locked */
+                                           FALSE);             /* page queues are not locked */

 #endif /* CONFIG_SECLUDED_MEMORY */
                                } else {
                                        /*
 #endif /* CONFIG_SECLUDED_MEMORY */
                                } else {
                                        /*


@@ -3817,25 +4283,27 @@ vm_page_free_list(
                                }
                        } else {
                                assert(VM_PAGE_GET_PHYS_PAGE(mem) == vm_page_fictitious_addr ||
                                }
                        } else {
                                assert(VM_PAGE_GET_PHYS_PAGE(mem) == vm_page_fictitious_addr ||

-                                      VM_PAGE_GET_PHYS_PAGE(mem) == vm_page_guard_addr);
+                                   VM_PAGE_GET_PHYS_PAGE(mem) == vm_page_guard_addr);

                                vm_page_release_fictitious(mem);
                        }
                        mem = nxt;
                }
                freeq = mem;
 
                                vm_page_release_fictitious(mem);
                        }
                        mem = nxt;
                }
                freeq = mem;
 

-               if ( (mem = local_freeq) ) {
-                       unsigned int    avail_free_count;
-                       unsigned int    need_wakeup = 0;
-                       unsigned int    need_priv_wakeup = 0;
+               if ((mem = local_freeq)) {
+                       unsigned int    avail_free_count;
+                       unsigned int    need_wakeup = 0;
+                       unsigned int    need_priv_wakeup = 0;

 #if CONFIG_SECLUDED_MEMORY
 #if CONFIG_SECLUDED_MEMORY

-                       unsigned int    need_wakeup_secluded = 0;
+                       unsigned int    need_wakeup_secluded = 0;

 #endif /* CONFIG_SECLUDED_MEMORY */
 #endif /* CONFIG_SECLUDED_MEMORY */

-         
+                       event_t         priv_wakeup_event, secluded_wakeup_event, normal_wakeup_event;
+                       boolean_t       priv_wakeup_all, secluded_wakeup_all, normal_wakeup_all;
+

                        lck_mtx_lock_spin(&vm_page_queue_free_lock);
 
                        while (mem) {
                        lck_mtx_lock_spin(&vm_page_queue_free_lock);
 
                        while (mem) {

-                               int     color;
+                               int     color;

 
                                nxt = mem->vmp_snext;
 
 
                                nxt = mem->vmp_snext;
 


@@ -3846,15 +4314,10 @@ vm_page_free_list(
 
                                color = VM_PAGE_GET_COLOR(mem);
 #if defined(__x86_64__)
 
                                color = VM_PAGE_GET_COLOR(mem);
 #if defined(__x86_64__)

-                               vm_page_queue_enter_clump(&vm_page_queue_free[color].qhead,
-                                                         mem,
-                                                         vm_page_t,
-                                                         vmp_pageq);
+                               vm_page_queue_enter_clump(&vm_page_queue_free[color].qhead, mem);

 #else
                                vm_page_queue_enter(&vm_page_queue_free[color].qhead,
 #else
                                vm_page_queue_enter(&vm_page_queue_free[color].qhead,

-                                                         mem,
-                                                         vm_page_t,
-                                                         vmp_pageq);
+                                   mem, vmp_pageq);

 #endif
                                mem = nxt;
                        }
 #endif
                                mem = nxt;
                        }


@@ -3865,7 +4328,6 @@ vm_page_free_list(
                        VM_DEBUG_CONSTANT_EVENT(vm_page_release, VM_PAGE_RELEASE, DBG_FUNC_NONE, pg_count, 0, 0, 0);
 
                        if (vm_page_free_wanted_privileged > 0 && avail_free_count > 0) {
                        VM_DEBUG_CONSTANT_EVENT(vm_page_release, VM_PAGE_RELEASE, DBG_FUNC_NONE, pg_count, 0, 0, 0);
 
                        if (vm_page_free_wanted_privileged > 0 && avail_free_count > 0) {

-

                                if (avail_free_count < vm_page_free_wanted_privileged) {
                                        need_priv_wakeup = avail_free_count;
                                        vm_page_free_wanted_privileged -= avail_free_count;
                                if (avail_free_count < vm_page_free_wanted_privileged) {
                                        need_priv_wakeup = avail_free_count;
                                        vm_page_free_wanted_privileged -= avail_free_count;


@@ -3881,18 +4343,18 @@ vm_page_free_list(
                            avail_free_count > vm_page_free_reserved) {
                                unsigned int available_pages;
                                available_pages = (avail_free_count -
                            avail_free_count > vm_page_free_reserved) {
                                unsigned int available_pages;
                                available_pages = (avail_free_count -

-                                                  vm_page_free_reserved);
+                                   vm_page_free_reserved);

                                if (available_pages <
                                    vm_page_free_wanted_secluded) {
                                        need_wakeup_secluded = available_pages;
                                        vm_page_free_wanted_secluded -=
                                if (available_pages <
                                    vm_page_free_wanted_secluded) {
                                        need_wakeup_secluded = available_pages;
                                        vm_page_free_wanted_secluded -=

-                                               available_pages;
+                                           available_pages;

                                        avail_free_count -= available_pages;
                                } else {
                                        need_wakeup_secluded =
                                        avail_free_count -= available_pages;
                                } else {
                                        need_wakeup_secluded =

-                                               vm_page_free_wanted_secluded;
+                                           vm_page_free_wanted_secluded;

                                        avail_free_count -=
                                        avail_free_count -=

-                                               vm_page_free_wanted_secluded;
+                                           vm_page_free_wanted_secluded;

                                        vm_page_free_wanted_secluded = 0;
                                }
                        }
                                        vm_page_free_wanted_secluded = 0;
                                }
                        }


@@ -3912,27 +4374,32 @@ vm_page_free_list(
                        }
                        lck_mtx_unlock(&vm_page_queue_free_lock);
 
                        }
                        lck_mtx_unlock(&vm_page_queue_free_lock);
 

+                       priv_wakeup_event = NULL;
+                       secluded_wakeup_event = NULL;
+                       normal_wakeup_event = NULL;
+
+                       priv_wakeup_all = FALSE;
+                       secluded_wakeup_all = FALSE;
+                       normal_wakeup_all = FALSE;
+
+

                        if (need_priv_wakeup != 0) {
                                /*
                                 * There shouldn't be that many VM-privileged threads,
                                 * so let's wake them all up, even if we don't quite
                                 * have enough pages to satisfy them all.
                                 */
                        if (need_priv_wakeup != 0) {
                                /*
                                 * There shouldn't be that many VM-privileged threads,
                                 * so let's wake them all up, even if we don't quite
                                 * have enough pages to satisfy them all.
                                 */

-                               thread_wakeup((event_t)&vm_page_free_wanted_privileged);
+                               priv_wakeup_event = (event_t)&vm_page_free_wanted_privileged;
+                               priv_wakeup_all = TRUE;

                        }
 #if CONFIG_SECLUDED_MEMORY
                        if (need_wakeup_secluded != 0 &&
                            vm_page_free_wanted_secluded == 0) {
                        }
 #if CONFIG_SECLUDED_MEMORY
                        if (need_wakeup_secluded != 0 &&
                            vm_page_free_wanted_secluded == 0) {

-                               thread_wakeup((event_t)
-                                             &vm_page_free_wanted_secluded);
+                               secluded_wakeup_event = (event_t)&vm_page_free_wanted_secluded;
+                               secluded_wakeup_all = TRUE;
+                               need_wakeup_secluded = 0;

                        } else {
                        } else {

-                               for (;
-                                    need_wakeup_secluded != 0;
-                                    need_wakeup_secluded--) {
-                                       thread_wakeup_one(
-                                               (event_t)
-                                               &vm_page_free_wanted_secluded);
-                               }
+                               secluded_wakeup_event = (event_t)&vm_page_free_wanted_secluded;

                        }
 #endif /* CONFIG_SECLUDED_MEMORY */
                        if (need_wakeup != 0 && vm_page_free_wanted == 0) {
                        }
 #endif /* CONFIG_SECLUDED_MEMORY */
                        if (need_wakeup != 0 && vm_page_free_wanted == 0) {


@@ -3941,12 +4408,83 @@ vm_page_free_list(
                                 * after this, so let's wake them all up at
                                 * once.
                                 */
                                 * after this, so let's wake them all up at
                                 * once.
                                 */

-                               thread_wakeup((event_t) &vm_page_free_count);
-                       } else for (; need_wakeup != 0; need_wakeup--) {
-                               /*
-                                * Wake up one waiter per page we just released.
-                                */
-                               thread_wakeup_one((event_t) &vm_page_free_count);
+                               normal_wakeup_event = (event_t) &vm_page_free_count;
+                               normal_wakeup_all = TRUE;
+                               need_wakeup = 0;
+                       } else {
+                               normal_wakeup_event = (event_t) &vm_page_free_count;
+                       }
+
+                       if (priv_wakeup_event ||
+#if CONFIG_SECLUDED_MEMORY
+                           secluded_wakeup_event ||
+#endif /* CONFIG_SECLUDED_MEMORY */
+                           normal_wakeup_event) {
+                               if (vps_dynamic_priority_enabled == TRUE) {
+                                       thread_t thread_woken = NULL;
+
+                                       if (priv_wakeup_all == TRUE) {
+                                               wakeup_all_with_inheritor(priv_wakeup_event, THREAD_AWAKENED);
+                                       }
+
+#if CONFIG_SECLUDED_MEMORY
+                                       if (secluded_wakeup_all == TRUE) {
+                                               wakeup_all_with_inheritor(secluded_wakeup_event, THREAD_AWAKENED);
+                                       }
+
+                                       while (need_wakeup_secluded-- != 0) {
+                                               /*
+                                                * Wake up one waiter per page we just released.
+                                                */
+                                               wakeup_one_with_inheritor(secluded_wakeup_event, THREAD_AWAKENED, LCK_WAKE_DO_NOT_TRANSFER_PUSH, &thread_woken);
+                                               thread_deallocate(thread_woken);
+                                       }
+#endif /* CONFIG_SECLUDED_MEMORY */
+
+                                       if (normal_wakeup_all == TRUE) {
+                                               wakeup_all_with_inheritor(normal_wakeup_event, THREAD_AWAKENED);
+                                       }
+
+                                       while (need_wakeup-- != 0) {
+                                               /*
+                                                * Wake up one waiter per page we just released.
+                                                */
+                                               wakeup_one_with_inheritor(normal_wakeup_event, THREAD_AWAKENED, LCK_WAKE_DO_NOT_TRANSFER_PUSH, &thread_woken);
+                                               thread_deallocate(thread_woken);
+                                       }
+                               } else {
+                                       /*
+                                        * Non-priority-aware wakeups.
+                                        */
+
+                                       if (priv_wakeup_all == TRUE) {
+                                               thread_wakeup(priv_wakeup_event);
+                                       }
+
+#if CONFIG_SECLUDED_MEMORY
+                                       if (secluded_wakeup_all == TRUE) {
+                                               thread_wakeup(secluded_wakeup_event);
+                                       }
+
+                                       while (need_wakeup_secluded-- != 0) {
+                                               /*
+                                                * Wake up one waiter per page we just released.
+                                                */
+                                               thread_wakeup_one(secluded_wakeup_event);
+                                       }
+
+#endif /* CONFIG_SECLUDED_MEMORY */
+                                       if (normal_wakeup_all == TRUE) {
+                                               thread_wakeup(normal_wakeup_event);
+                                       }
+
+                                       while (need_wakeup-- != 0) {
+                                               /*
+                                                * Wake up one waiter per page we just released.
+                                                */
+                                               thread_wakeup_one(normal_wakeup_event);
+                                       }
+                               }

                        }
 
                        VM_CHECK_MEMORYSTATUS;
                        }
 
                        VM_CHECK_MEMORYSTATUS;


@@ -3970,9 +4508,9 @@ void
 vm_page_wire(
        vm_page_t mem,
        vm_tag_t           tag,
 vm_page_wire(
        vm_page_t mem,
        vm_tag_t           tag,

-       boolean_t          check_memorystatus)
+       boolean_t          check_memorystatus)

 {
 {

-       vm_object_t     m_object;
+       vm_object_t     m_object;

 
        m_object = VM_PAGE_OBJECT(mem);
 
 
        m_object = VM_PAGE_OBJECT(mem);
 


@@ -3993,10 +4531,10 @@ vm_page_wire(
                 */
        }
        LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
                 */
        }
        LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);

-       if ( !VM_PAGE_WIRED(mem)) {
-
-               if (mem->vmp_laundry)
+       if (!VM_PAGE_WIRED(mem)) {
+               if (mem->vmp_laundry) {

                        vm_pageout_steal_laundry(mem, TRUE);
                        vm_pageout_steal_laundry(mem, TRUE);

+               }

 
                vm_page_queues_remove(mem, TRUE);
 
 
                vm_page_queues_remove(mem, TRUE);
 


@@ -4004,27 +4542,26 @@ vm_page_wire(
                mem->vmp_q_state = VM_PAGE_IS_WIRED;
 
                if (m_object) {
                mem->vmp_q_state = VM_PAGE_IS_WIRED;
 
                if (m_object) {

-

                        VM_OBJECT_WIRED_PAGE_UPDATE_START(m_object);
                        VM_OBJECT_WIRED_PAGE_ADD(m_object, mem);
                        VM_OBJECT_WIRED_PAGE_UPDATE_END(m_object, tag);
 
                        assert(m_object->resident_page_count >=
                        VM_OBJECT_WIRED_PAGE_UPDATE_START(m_object);
                        VM_OBJECT_WIRED_PAGE_ADD(m_object, mem);
                        VM_OBJECT_WIRED_PAGE_UPDATE_END(m_object, tag);
 
                        assert(m_object->resident_page_count >=

-                              m_object->wired_page_count);
+                           m_object->wired_page_count);

                        if (m_object->purgable == VM_PURGABLE_VOLATILE) {
                                assert(vm_page_purgeable_count > 0);
                                OSAddAtomic(-1, &vm_page_purgeable_count);
                                OSAddAtomic(1, &vm_page_purgeable_wired_count);
                        }
                        if ((m_object->purgable == VM_PURGABLE_VOLATILE ||
                        if (m_object->purgable == VM_PURGABLE_VOLATILE) {
                                assert(vm_page_purgeable_count > 0);
                                OSAddAtomic(-1, &vm_page_purgeable_count);
                                OSAddAtomic(1, &vm_page_purgeable_wired_count);
                        }
                        if ((m_object->purgable == VM_PURGABLE_VOLATILE ||

-                            m_object->purgable == VM_PURGABLE_EMPTY) &&
+                           m_object->purgable == VM_PURGABLE_EMPTY) &&

                            m_object->vo_owner != TASK_NULL) {
                            m_object->vo_owner != TASK_NULL) {

-                               task_t          owner;
-                               int             ledger_idx_volatile;
-                               int             ledger_idx_nonvolatile;
-                               int             ledger_idx_volatile_compressed;
-                               int             ledger_idx_nonvolatile_compressed;
-                               boolean_t       do_footprint;
+                               task_t          owner;
+                               int             ledger_idx_volatile;
+                               int             ledger_idx_nonvolatile;
+                               int             ledger_idx_volatile_compressed;
+                               int             ledger_idx_nonvolatile_compressed;
+                               boolean_t       do_footprint;

 
                                owner = VM_OBJECT_OWNER(m_object);
                                vm_object_ledger_tag_ledgers(
 
                                owner = VM_OBJECT_OWNER(m_object);
                                vm_object_ledger_tag_ledgers(


@@ -4036,17 +4573,17 @@ vm_page_wire(
                                        &do_footprint);
                                /* less volatile bytes */
                                ledger_debit(owner->ledger,
                                        &do_footprint);
                                /* less volatile bytes */
                                ledger_debit(owner->ledger,

-                                            ledger_idx_volatile,
-                                            PAGE_SIZE);
+                                   ledger_idx_volatile,
+                                   PAGE_SIZE);

                                /* more not-quite-volatile bytes */
                                ledger_credit(owner->ledger,
                                /* more not-quite-volatile bytes */
                                ledger_credit(owner->ledger,

-                                             ledger_idx_nonvolatile,
-                                             PAGE_SIZE);
+                                   ledger_idx_nonvolatile,
+                                   PAGE_SIZE);

                                if (do_footprint) {
                                        /* more footprint */
                                        ledger_credit(owner->ledger,
                                if (do_footprint) {
                                        /* more footprint */
                                        ledger_credit(owner->ledger,

-                                                     task_ledgers.phys_footprint,
-                                                     PAGE_SIZE);
+                                           task_ledgers.phys_footprint,
+                                           PAGE_SIZE);

                                }
                        }
                        if (m_object->all_reusable) {
                                }
                        }
                        if (m_object->all_reusable) {


@@ -4062,17 +4599,19 @@ vm_page_wire(
                                 * accounting.
                                 */
                                vm_object_reuse_pages(m_object,
                                 * accounting.
                                 */
                                vm_object_reuse_pages(m_object,

-                                                     mem->vmp_offset,
-                                                     mem->vmp_offset+PAGE_SIZE_64,
-                                                     FALSE);
+                                   mem->vmp_offset,
+                                   mem->vmp_offset + PAGE_SIZE_64,
+                                   FALSE);

                        }
                }
                assert(!mem->vmp_reusable);
 
                        }
                }
                assert(!mem->vmp_reusable);
 

-               if (!mem->vmp_private && !mem->vmp_fictitious && !mem->vmp_gobbled)
+               if (!mem->vmp_private && !mem->vmp_fictitious && !mem->vmp_gobbled) {

                        vm_page_wire_count++;
                        vm_page_wire_count++;

-               if (mem->vmp_gobbled)
+               }
+               if (mem->vmp_gobbled) {

                        vm_page_gobble_count--;
                        vm_page_gobble_count--;

+               }

                mem->vmp_gobbled = FALSE;
 
                if (check_memorystatus == TRUE) {
                mem->vmp_gobbled = FALSE;
 
                if (check_memorystatus == TRUE) {


@@ -4098,10 +4637,10 @@ vm_page_wire(
  */
 void
 vm_page_unwire(
  */
 void
 vm_page_unwire(

-       vm_page_t       mem,
-       boolean_t       queueit)
+       vm_page_t       mem,
+       boolean_t       queueit)

 {
 {

-       vm_object_t     m_object;
+       vm_object_t     m_object;

 
        m_object = VM_PAGE_OBJECT(mem);
 
 
        m_object = VM_PAGE_OBJECT(mem);
 


@@ -4115,7 +4654,6 @@ vm_page_unwire(
        vm_object_lock_assert_exclusive(m_object);
        LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
        if (--mem->vmp_wire_count == 0) {
        vm_object_lock_assert_exclusive(m_object);
        LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
        if (--mem->vmp_wire_count == 0) {

-

                mem->vmp_q_state = VM_PAGE_NOT_ON_Q;
 
                VM_OBJECT_WIRED_PAGE_UPDATE_START(m_object);
                mem->vmp_q_state = VM_PAGE_NOT_ON_Q;
 
                VM_OBJECT_WIRED_PAGE_UPDATE_START(m_object);


@@ -4126,21 +4664,21 @@ vm_page_unwire(
                }
 
                assert(m_object->resident_page_count >=
                }
 
                assert(m_object->resident_page_count >=

-                      m_object->wired_page_count);
+                   m_object->wired_page_count);

                if (m_object->purgable == VM_PURGABLE_VOLATILE) {
                        OSAddAtomic(+1, &vm_page_purgeable_count);
                        assert(vm_page_purgeable_wired_count > 0);
                        OSAddAtomic(-1, &vm_page_purgeable_wired_count);
                }
                if ((m_object->purgable == VM_PURGABLE_VOLATILE ||
                if (m_object->purgable == VM_PURGABLE_VOLATILE) {
                        OSAddAtomic(+1, &vm_page_purgeable_count);
                        assert(vm_page_purgeable_wired_count > 0);
                        OSAddAtomic(-1, &vm_page_purgeable_wired_count);
                }
                if ((m_object->purgable == VM_PURGABLE_VOLATILE ||

-                    m_object->purgable == VM_PURGABLE_EMPTY) &&
+                   m_object->purgable == VM_PURGABLE_EMPTY) &&

                    m_object->vo_owner != TASK_NULL) {
                    m_object->vo_owner != TASK_NULL) {

-                       task_t          owner;
-                       int             ledger_idx_volatile;
-                       int             ledger_idx_nonvolatile;
-                       int             ledger_idx_volatile_compressed;
-                       int             ledger_idx_nonvolatile_compressed;
-                       boolean_t       do_footprint;
+                       task_t          owner;
+                       int             ledger_idx_volatile;
+                       int             ledger_idx_nonvolatile;
+                       int             ledger_idx_volatile_compressed;
+                       int             ledger_idx_nonvolatile_compressed;
+                       boolean_t       do_footprint;

 
                        owner = VM_OBJECT_OWNER(m_object);
                        vm_object_ledger_tag_ledgers(
 
                        owner = VM_OBJECT_OWNER(m_object);
                        vm_object_ledger_tag_ledgers(


@@ -4152,17 +4690,17 @@ vm_page_unwire(
                                &do_footprint);
                        /* more volatile bytes */
                        ledger_credit(owner->ledger,
                                &do_footprint);
                        /* more volatile bytes */
                        ledger_credit(owner->ledger,

-                                     ledger_idx_volatile,
-                                     PAGE_SIZE);
+                           ledger_idx_volatile,
+                           PAGE_SIZE);

                        /* less not-quite-volatile bytes */
                        ledger_debit(owner->ledger,
                        /* less not-quite-volatile bytes */
                        ledger_debit(owner->ledger,

-                                    ledger_idx_nonvolatile,
-                                    PAGE_SIZE);
+                           ledger_idx_nonvolatile,
+                           PAGE_SIZE);

                        if (do_footprint) {
                                /* less footprint */
                                ledger_debit(owner->ledger,
                        if (do_footprint) {
                                /* less footprint */
                                ledger_debit(owner->ledger,

-                                            task_ledgers.phys_footprint,
-                                            PAGE_SIZE);
+                                   task_ledgers.phys_footprint,
+                                   PAGE_SIZE);

                        }
                }
                assert(m_object != kernel_object);
                        }
                }
                assert(m_object != kernel_object);


@@ -4177,7 +4715,6 @@ vm_page_unwire(
                }
 
                VM_CHECK_MEMORYSTATUS;
                }
 
                VM_CHECK_MEMORYSTATUS;

-               

        }
        VM_PAGE_CHECK(mem);
 }
        }
        VM_PAGE_CHECK(mem);
 }


@@ -4193,7 +4730,7 @@ vm_page_unwire(
  */
 void
 vm_page_deactivate(
  */
 void
 vm_page_deactivate(

-       vm_page_t       m)
+       vm_page_t       m)

 {
        vm_page_deactivate_internal(m, TRUE);
 }
 {
        vm_page_deactivate_internal(m, TRUE);
 }


@@ -4201,10 +4738,10 @@ vm_page_deactivate(
 
 void
 vm_page_deactivate_internal(
 
 void
 vm_page_deactivate_internal(

-       vm_page_t       m,
-       boolean_t       clear_hw_reference)
+       vm_page_t       m,
+       boolean_t       clear_hw_reference)

 {
 {

-       vm_object_t     m_object;
+       vm_object_t     m_object;

 
        m_object = VM_PAGE_OBJECT(m);
 
 
        m_object = VM_PAGE_OBJECT(m);
 


@@ -4221,13 +4758,14 @@ vm_page_deactivate_internal(
         *      inactive queue.  Note wired pages should not have
         *      their reference bit cleared.
         */
         *      inactive queue.  Note wired pages should not have
         *      their reference bit cleared.
         */

-       assert ( !(m->vmp_absent && !m->vmp_unusual));
+       assert( !(m->vmp_absent && !m->vmp_unusual));

 
 

-       if (m->vmp_gobbled) {           /* can this happen? */
+       if (m->vmp_gobbled) {           /* can this happen? */

                assert( !VM_PAGE_WIRED(m));
 
                assert( !VM_PAGE_WIRED(m));
 

-               if (!m->vmp_private && !m->vmp_fictitious)
+               if (!m->vmp_private && !m->vmp_fictitious) {

                        vm_page_wire_count--;
                        vm_page_wire_count--;

+               }

                vm_page_gobble_count--;
                m->vmp_gobbled = FALSE;
        }
                vm_page_gobble_count--;
                m->vmp_gobbled = FALSE;
        }


@@ -4243,29 +4781,30 @@ vm_page_deactivate_internal(
            (m->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) ||
            (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q) ||
            VM_PAGE_WIRED(m)) {
            (m->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) ||
            (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q) ||
            VM_PAGE_WIRED(m)) {

-               return;
+               return;

        }
        }

-       if (!m->vmp_absent && clear_hw_reference == TRUE)
+       if (!m->vmp_absent && clear_hw_reference == TRUE) {

                pmap_clear_reference(VM_PAGE_GET_PHYS_PAGE(m));
                pmap_clear_reference(VM_PAGE_GET_PHYS_PAGE(m));

+       }

 
        m->vmp_reference = FALSE;
        m->vmp_no_cache = FALSE;
 
 
        m->vmp_reference = FALSE;
        m->vmp_no_cache = FALSE;
 

-       if ( !VM_PAGE_INACTIVE(m)) {
+       if (!VM_PAGE_INACTIVE(m)) {

                vm_page_queues_remove(m, FALSE);
 
                if (!VM_DYNAMIC_PAGING_ENABLED() &&
                    m->vmp_dirty && m_object->internal &&
                    (m_object->purgable == VM_PURGABLE_DENY ||
                vm_page_queues_remove(m, FALSE);
 
                if (!VM_DYNAMIC_PAGING_ENABLED() &&
                    m->vmp_dirty && m_object->internal &&
                    (m_object->purgable == VM_PURGABLE_DENY ||

-                    m_object->purgable == VM_PURGABLE_NONVOLATILE ||
-                    m_object->purgable == VM_PURGABLE_VOLATILE)) {
+                   m_object->purgable == VM_PURGABLE_NONVOLATILE ||
+                   m_object->purgable == VM_PURGABLE_VOLATILE)) {

                        vm_page_check_pageable_safe(m);
                        vm_page_check_pageable_safe(m);

-                       vm_page_queue_enter(&vm_page_queue_throttled, m, vm_page_t, vmp_pageq);
+                       vm_page_queue_enter(&vm_page_queue_throttled, m, vmp_pageq);

                        m->vmp_q_state = VM_PAGE_ON_THROTTLED_Q;
                        vm_page_throttled_count++;
                } else {
                        if (m_object->named && m_object->ref_count == 1) {
                        m->vmp_q_state = VM_PAGE_ON_THROTTLED_Q;
                        vm_page_throttled_count++;
                } else {
                        if (m_object->named && m_object->ref_count == 1) {

-                               vm_page_speculate(m, FALSE);
+                               vm_page_speculate(m, FALSE);

 #if DEVELOPMENT || DEBUG
                                vm_page_speculative_recreated++;
 #endif
 #if DEVELOPMENT || DEBUG
                                vm_page_speculative_recreated++;
 #endif


@@ -4286,9 +4825,10 @@ vm_page_deactivate_internal(
  * Call with the queues lock held.
  */
 
  * Call with the queues lock held.
  */
 

-void vm_page_enqueue_cleaned(vm_page_t m)
+void
+vm_page_enqueue_cleaned(vm_page_t m)

 {
 {

-       vm_object_t     m_object;
+       vm_object_t     m_object;

 
        m_object = VM_PAGE_OBJECT(m);
 
 
        m_object = VM_PAGE_OBJECT(m);
 


@@ -4301,8 +4841,9 @@ void vm_page_enqueue_cleaned(vm_page_t m)
        }
 
        if (m->vmp_gobbled) {
        }
 
        if (m->vmp_gobbled) {

-               if (!m->vmp_private && !m->vmp_fictitious)
+               if (!m->vmp_private && !m->vmp_fictitious) {

                        vm_page_wire_count--;
                        vm_page_wire_count--;

+               }

                vm_page_gobble_count--;
                m->vmp_gobbled = FALSE;
        }
                vm_page_gobble_count--;
                m->vmp_gobbled = FALSE;
        }


@@ -4317,12 +4858,12 @@ void vm_page_enqueue_cleaned(vm_page_t m)
        if (m->vmp_laundry || m->vmp_private || m->vmp_fictitious ||
            (m->vmp_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) ||
            (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q)) {
        if (m->vmp_laundry || m->vmp_private || m->vmp_fictitious ||
            (m->vmp_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q) ||
            (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q)) {

-                   return;
+               return;

        }
        vm_page_queues_remove(m, FALSE);
 
        vm_page_check_pageable_safe(m);
        }
        vm_page_queues_remove(m, FALSE);
 
        vm_page_check_pageable_safe(m);

-       vm_page_queue_enter(&vm_page_queue_cleaned, m, vm_page_t, vmp_pageq);
+       vm_page_queue_enter(&vm_page_queue_cleaned, m, vmp_pageq);

        m->vmp_q_state = VM_PAGE_ON_INACTIVE_CLEANED_Q;
        vm_page_cleaned_count++;
 
        m->vmp_q_state = VM_PAGE_ON_INACTIVE_CLEANED_Q;
        vm_page_cleaned_count++;
 


@@ -4333,8 +4874,9 @@ void vm_page_enqueue_cleaned(vm_page_t m)
                vm_page_pageable_external_count++;
        }
 #if CONFIG_BACKGROUND_QUEUE
                vm_page_pageable_external_count++;
        }
 #if CONFIG_BACKGROUND_QUEUE

-       if (m->vmp_in_background)
+       if (m->vmp_in_background) {

                vm_page_add_to_backgroundq(m, TRUE);
                vm_page_add_to_backgroundq(m, TRUE);

+       }

 #endif
        VM_PAGEOUT_DEBUG(vm_pageout_enqueued_cleaned, 1);
 }
 #endif
        VM_PAGEOUT_DEBUG(vm_pageout_enqueued_cleaned, 1);
 }


@@ -4349,14 +4891,14 @@ void vm_page_enqueue_cleaned(vm_page_t m)
 
 void
 vm_page_activate(
 
 void
 vm_page_activate(

-       vm_page_t       m)
+       vm_page_t       m)

 {
 {

-       vm_object_t     m_object;
+       vm_object_t     m_object;

 
        m_object = VM_PAGE_OBJECT(m);
 
        VM_PAGE_CHECK(m);
 
        m_object = VM_PAGE_OBJECT(m);
 
        VM_PAGE_CHECK(m);

-#ifdef FIXME_4778297
+#ifdef  FIXME_4778297

        assert(m_object != kernel_object);
 #endif
        assert(VM_PAGE_GET_PHYS_PAGE(m) != vm_page_guard_addr);
        assert(m_object != kernel_object);
 #endif
        assert(VM_PAGE_GET_PHYS_PAGE(m) != vm_page_guard_addr);


@@ -4365,8 +4907,9 @@ vm_page_activate(
 
        if (m->vmp_gobbled) {
                assert( !VM_PAGE_WIRED(m));
 
        if (m->vmp_gobbled) {
                assert( !VM_PAGE_WIRED(m));

-               if (!m->vmp_private && !m->vmp_fictitious)
+               if (!m->vmp_private && !m->vmp_fictitious) {

                        vm_page_wire_count--;
                        vm_page_wire_count--;

+               }

                vm_page_gobble_count--;
                m->vmp_gobbled = FALSE;
        }
                vm_page_gobble_count--;
                m->vmp_gobbled = FALSE;
        }


@@ -4380,29 +4923,31 @@ vm_page_activate(
         */
        if (m->vmp_laundry || m->vmp_private || m->vmp_fictitious ||
            (m->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) ||
         */
        if (m->vmp_laundry || m->vmp_private || m->vmp_fictitious ||
            (m->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) ||

-           (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q))
+           (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q)) {

                return;
                return;

+       }

 
 #if DEBUG
 
 #if DEBUG

-       if (m->vmp_q_state == VM_PAGE_ON_ACTIVE_Q)
-               panic("vm_page_activate: already active");
+       if (m->vmp_q_state == VM_PAGE_ON_ACTIVE_Q) {
+               panic("vm_page_activate: already active");
+       }

 #endif
 
        if (m->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q) {
                DTRACE_VM2(pgrec, int, 1, (uint64_t *), NULL);
                DTRACE_VM2(pgfrec, int, 1, (uint64_t *), NULL);
        }
 #endif
 
        if (m->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q) {
                DTRACE_VM2(pgrec, int, 1, (uint64_t *), NULL);
                DTRACE_VM2(pgfrec, int, 1, (uint64_t *), NULL);
        }

-       
+

        vm_page_queues_remove(m, FALSE);
 
        vm_page_queues_remove(m, FALSE);
 

-       if ( !VM_PAGE_WIRED(m)) {
+       if (!VM_PAGE_WIRED(m)) {

                vm_page_check_pageable_safe(m);
                vm_page_check_pageable_safe(m);

-               if (!VM_DYNAMIC_PAGING_ENABLED() && 
-                   m->vmp_dirty && m_object->internal && 
+               if (!VM_DYNAMIC_PAGING_ENABLED() &&
+                   m->vmp_dirty && m_object->internal &&

                    (m_object->purgable == VM_PURGABLE_DENY ||
                    (m_object->purgable == VM_PURGABLE_DENY ||

-                    m_object->purgable == VM_PURGABLE_NONVOLATILE ||
-                    m_object->purgable == VM_PURGABLE_VOLATILE)) {
-                       vm_page_queue_enter(&vm_page_queue_throttled, m, vm_page_t, vmp_pageq);
+                   m_object->purgable == VM_PURGABLE_NONVOLATILE ||
+                   m_object->purgable == VM_PURGABLE_VOLATILE)) {
+                       vm_page_queue_enter(&vm_page_queue_throttled, m, vmp_pageq);

                        m->vmp_q_state = VM_PAGE_ON_THROTTLED_Q;
                        vm_page_throttled_count++;
                } else {
                        m->vmp_q_state = VM_PAGE_ON_THROTTLED_Q;
                        vm_page_throttled_count++;
                } else {


@@ -4411,10 +4956,10 @@ vm_page_activate(
                            vm_page_secluded_target != 0 &&
                            num_tasks_can_use_secluded_mem == 0 &&
                            m_object->eligible_for_secluded) {
                            vm_page_secluded_target != 0 &&
                            num_tasks_can_use_secluded_mem == 0 &&
                            m_object->eligible_for_secluded) {

-                               vm_page_queue_enter(&vm_page_queue_secluded, m,
-                                                   vm_page_t, vmp_pageq);
+                               vm_page_queue_enter(&vm_page_queue_secluded, m, vmp_pageq);

                                m->vmp_q_state = VM_PAGE_ON_SECLUDED_Q;
                                vm_page_secluded_count++;
                                m->vmp_q_state = VM_PAGE_ON_SECLUDED_Q;
                                vm_page_secluded_count++;

+                               VM_PAGE_SECLUDED_COUNT_OVER_TARGET_UPDATE();

                                vm_page_secluded_count_inuse++;
                                assert(!m_object->internal);
 //                             vm_page_pageable_external_count++;
                                vm_page_secluded_count_inuse++;
                                assert(!m_object->internal);
 //                             vm_page_pageable_external_count++;


@@ -4438,11 +4983,11 @@ vm_page_activate(
  */
 void
 vm_page_speculate(
  */
 void
 vm_page_speculate(

-       vm_page_t       m,
-       boolean_t       new)
+       vm_page_t       m,
+       boolean_t       new)

 {
 {

-        struct vm_speculative_age_q    *aq;
-       vm_object_t     m_object;
+       struct vm_speculative_age_q     *aq;
+       vm_object_t     m_object;

 
        m_object = VM_PAGE_OBJECT(m);
 
 
        m_object = VM_PAGE_OBJECT(m);
 


@@ -4464,55 +5009,59 @@ vm_page_speculate(
         */
        if (m->vmp_laundry || m->vmp_private || m->vmp_fictitious ||
            (m->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) ||
         */
        if (m->vmp_laundry || m->vmp_private || m->vmp_fictitious ||
            (m->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) ||

-           (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q))
+           (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q)) {

                return;
                return;

+       }

 
        vm_page_queues_remove(m, FALSE);
 
 
        vm_page_queues_remove(m, FALSE);
 

-       if ( !VM_PAGE_WIRED(m)) {
-               mach_timespec_t         ts;
+       if (!VM_PAGE_WIRED(m)) {
+               mach_timespec_t         ts;

                clock_sec_t sec;
                clock_nsec_t nsec;
 
                clock_sec_t sec;
                clock_nsec_t nsec;
 

-               clock_get_system_nanotime(&sec, &nsec);
+               clock_get_system_nanotime(&sec, &nsec);

                ts.tv_sec = (unsigned int) sec;
                ts.tv_nsec = nsec;
 
                if (vm_page_speculative_count == 0) {
                ts.tv_sec = (unsigned int) sec;
                ts.tv_nsec = nsec;
 
                if (vm_page_speculative_count == 0) {

-

                        speculative_age_index = VM_PAGE_MIN_SPECULATIVE_AGE_Q;
                        speculative_steal_index = VM_PAGE_MIN_SPECULATIVE_AGE_Q;
 
                        aq = &vm_page_queue_speculative[speculative_age_index];
 
                        speculative_age_index = VM_PAGE_MIN_SPECULATIVE_AGE_Q;
                        speculative_steal_index = VM_PAGE_MIN_SPECULATIVE_AGE_Q;
 
                        aq = &vm_page_queue_speculative[speculative_age_index];
 

-                       /*
+                       /*

                         * set the timer to begin a new group
                         */
                        aq->age_ts.tv_sec = vm_pageout_state.vm_page_speculative_q_age_ms / 1000;
                        aq->age_ts.tv_nsec = (vm_pageout_state.vm_page_speculative_q_age_ms % 1000) * 1000 * NSEC_PER_USEC;
                         * set the timer to begin a new group
                         */
                        aq->age_ts.tv_sec = vm_pageout_state.vm_page_speculative_q_age_ms / 1000;
                        aq->age_ts.tv_nsec = (vm_pageout_state.vm_page_speculative_q_age_ms % 1000) * 1000 * NSEC_PER_USEC;

+

                        ADD_MACH_TIMESPEC(&aq->age_ts, &ts);
                } else {
                        aq = &vm_page_queue_speculative[speculative_age_index];
 
                        if (CMP_MACH_TIMESPEC(&ts, &aq->age_ts) >= 0) {
                        ADD_MACH_TIMESPEC(&aq->age_ts, &ts);
                } else {
                        aq = &vm_page_queue_speculative[speculative_age_index];
 
                        if (CMP_MACH_TIMESPEC(&ts, &aq->age_ts) >= 0) {

+                               speculative_age_index++;

 
 

-                               speculative_age_index++;
-
-                               if (speculative_age_index > VM_PAGE_MAX_SPECULATIVE_AGE_Q)
-                                       speculative_age_index = VM_PAGE_MIN_SPECULATIVE_AGE_Q;
+                               if (speculative_age_index > VM_PAGE_MAX_SPECULATIVE_AGE_Q) {
+                                       speculative_age_index = VM_PAGE_MIN_SPECULATIVE_AGE_Q;
+                               }

                                if (speculative_age_index == speculative_steal_index) {
                                if (speculative_age_index == speculative_steal_index) {

-                                       speculative_steal_index = speculative_age_index + 1;
+                                       speculative_steal_index = speculative_age_index + 1;

 
 

-                                       if (speculative_steal_index > VM_PAGE_MAX_SPECULATIVE_AGE_Q)
-                                               speculative_steal_index = VM_PAGE_MIN_SPECULATIVE_AGE_Q;
+                                       if (speculative_steal_index > VM_PAGE_MAX_SPECULATIVE_AGE_Q) {
+                                               speculative_steal_index = VM_PAGE_MIN_SPECULATIVE_AGE_Q;
+                                       }

                                }
                                aq = &vm_page_queue_speculative[speculative_age_index];
 
                                }
                                aq = &vm_page_queue_speculative[speculative_age_index];
 

-                               if (!vm_page_queue_empty(&aq->age_q))
-                                       vm_page_speculate_ageit(aq);
+                               if (!vm_page_queue_empty(&aq->age_q)) {
+                                       vm_page_speculate_ageit(aq);
+                               }

 
                                aq->age_ts.tv_sec = vm_pageout_state.vm_page_speculative_q_age_ms / 1000;
                                aq->age_ts.tv_nsec = (vm_pageout_state.vm_page_speculative_q_age_ms % 1000) * 1000 * NSEC_PER_USEC;
 
                                aq->age_ts.tv_sec = vm_pageout_state.vm_page_speculative_q_age_ms / 1000;
                                aq->age_ts.tv_nsec = (vm_pageout_state.vm_page_speculative_q_age_ms % 1000) * 1000 * NSEC_PER_USEC;

+

                                ADD_MACH_TIMESPEC(&aq->age_ts, &ts);
                        }
                }
                                ADD_MACH_TIMESPEC(&aq->age_ts, &ts);
                        }
                }


@@ -4524,7 +5073,7 @@ vm_page_speculate(
                if (new == TRUE) {
                        vm_object_lock_assert_exclusive(m_object);
 
                if (new == TRUE) {
                        vm_object_lock_assert_exclusive(m_object);
 

-                       m_object->pages_created++;
+                       m_object->pages_created++;

 #if DEVELOPMENT || DEBUG
                        vm_page_speculative_created++;
 #endif
 #if DEVELOPMENT || DEBUG
                        vm_page_speculative_created++;
 #endif


@@ -4543,24 +5092,24 @@ vm_page_speculate(
 void
 vm_page_speculate_ageit(struct vm_speculative_age_q *aq)
 {
 void
 vm_page_speculate_ageit(struct vm_speculative_age_q *aq)
 {

-        struct vm_speculative_age_q    *sq;
-       vm_page_t       t;
+       struct vm_speculative_age_q     *sq;
+       vm_page_t       t;

 
        sq = &vm_page_queue_speculative[VM_PAGE_SPECULATIVE_AGED_Q];
 
        if (vm_page_queue_empty(&sq->age_q)) {
 
        sq = &vm_page_queue_speculative[VM_PAGE_SPECULATIVE_AGED_Q];
 
        if (vm_page_queue_empty(&sq->age_q)) {

-               sq->age_q.next = aq->age_q.next;
+               sq->age_q.next = aq->age_q.next;

                sq->age_q.prev = aq->age_q.prev;
                sq->age_q.prev = aq->age_q.prev;

-               
+

                t = (vm_page_t)VM_PAGE_UNPACK_PTR(sq->age_q.next);
                t->vmp_pageq.prev = VM_PAGE_PACK_PTR(&sq->age_q);
 
                t = (vm_page_t)VM_PAGE_UNPACK_PTR(sq->age_q.prev);
                t->vmp_pageq.next = VM_PAGE_PACK_PTR(&sq->age_q);
        } else {
                t = (vm_page_t)VM_PAGE_UNPACK_PTR(sq->age_q.next);
                t->vmp_pageq.prev = VM_PAGE_PACK_PTR(&sq->age_q);
 
                t = (vm_page_t)VM_PAGE_UNPACK_PTR(sq->age_q.prev);
                t->vmp_pageq.next = VM_PAGE_PACK_PTR(&sq->age_q);
        } else {

-               t = (vm_page_t)VM_PAGE_UNPACK_PTR(sq->age_q.prev);
+               t = (vm_page_t)VM_PAGE_UNPACK_PTR(sq->age_q.prev);

                t->vmp_pageq.next = aq->age_q.next;
                t->vmp_pageq.next = aq->age_q.next;

-                                               
+

                t = (vm_page_t)VM_PAGE_UNPACK_PTR(aq->age_q.next);
                t->vmp_pageq.prev = sq->age_q.prev;
 
                t = (vm_page_t)VM_PAGE_UNPACK_PTR(aq->age_q.next);
                t->vmp_pageq.prev = sq->age_q.prev;
 


@@ -4575,7 +5124,7 @@ vm_page_speculate_ageit(struct vm_speculative_age_q *aq)
 
 void
 vm_page_lru(
 
 void
 vm_page_lru(

-       vm_page_t       m)
+       vm_page_t       m)

 {
        VM_PAGE_CHECK(m);
        assert(VM_PAGE_OBJECT(m) != kernel_object);
 {
        VM_PAGE_CHECK(m);
        assert(VM_PAGE_OBJECT(m) != kernel_object);


@@ -4584,18 +5133,18 @@ vm_page_lru(
        LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
 
        if (m->vmp_q_state == VM_PAGE_ON_INACTIVE_EXTERNAL_Q) {
        LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
 
        if (m->vmp_q_state == VM_PAGE_ON_INACTIVE_EXTERNAL_Q) {

-               /*
-                * we don't need to do all the other work that 
+               /*
+                * we don't need to do all the other work that

                 * vm_page_queues_remove and vm_page_enqueue_inactive
                 * bring along for the ride
                 */
                 * vm_page_queues_remove and vm_page_enqueue_inactive
                 * bring along for the ride
                 */

-               assert(!m->vmp_laundry);
+               assert(!m->vmp_laundry);

                assert(!m->vmp_private);
                assert(!m->vmp_private);

-               
+

                m->vmp_no_cache = FALSE;
 
                m->vmp_no_cache = FALSE;
 

-               vm_page_queue_remove(&vm_page_queue_inactive, m, vm_page_t, vmp_pageq);
-               vm_page_queue_enter(&vm_page_queue_inactive, m, vm_page_t, vmp_pageq);
+               vm_page_queue_remove(&vm_page_queue_inactive, m, vmp_pageq);
+               vm_page_queue_enter(&vm_page_queue_inactive, m, vmp_pageq);

 
                return;
        }
 
                return;
        }


@@ -4610,8 +5159,9 @@ vm_page_lru(
        if (m->vmp_laundry || m->vmp_private ||
            (m->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) ||
            (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q) ||
        if (m->vmp_laundry || m->vmp_private ||
            (m->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) ||
            (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q) ||

-           VM_PAGE_WIRED(m))
+           VM_PAGE_WIRED(m)) {

                return;
                return;

+       }

 
        m->vmp_no_cache = FALSE;
 
 
        m->vmp_no_cache = FALSE;
 


@@ -4624,25 +5174,26 @@ vm_page_lru(
 void
 vm_page_reactivate_all_throttled(void)
 {
 void
 vm_page_reactivate_all_throttled(void)
 {

-       vm_page_t       first_throttled, last_throttled;
-       vm_page_t       first_active;
-       vm_page_t       m;
-       int             extra_active_count;
-       int             extra_internal_count, extra_external_count;
-       vm_object_t     m_object;
-
-       if (!VM_DYNAMIC_PAGING_ENABLED())
+       vm_page_t       first_throttled, last_throttled;
+       vm_page_t       first_active;
+       vm_page_t       m;
+       int             extra_active_count;
+       int             extra_internal_count, extra_external_count;
+       vm_object_t     m_object;
+
+       if (!VM_DYNAMIC_PAGING_ENABLED()) {

                return;
                return;

+       }

 
        extra_active_count = 0;
        extra_internal_count = 0;
        extra_external_count = 0;
        vm_page_lock_queues();
 
        extra_active_count = 0;
        extra_internal_count = 0;
        extra_external_count = 0;
        vm_page_lock_queues();

-       if (! vm_page_queue_empty(&vm_page_queue_throttled)) {
+       if (!vm_page_queue_empty(&vm_page_queue_throttled)) {

                /*
                 * Switch "throttled" pages to "active".
                 */
                /*
                 * Switch "throttled" pages to "active".
                 */

-               vm_page_queue_iterate(&vm_page_queue_throttled, m, vm_page_t, vmp_pageq) {
+               vm_page_queue_iterate(&vm_page_queue_throttled, m, vmp_pageq) {

                        VM_PAGE_CHECK(m);
                        assert(m->vmp_q_state == VM_PAGE_ON_THROTTLED_Q);
 
                        VM_PAGE_CHECK(m);
                        assert(m->vmp_q_state == VM_PAGE_ON_THROTTLED_Q);
 


@@ -4658,8 +5209,9 @@ vm_page_reactivate_all_throttled(void)
                        m->vmp_q_state = VM_PAGE_ON_ACTIVE_Q;
                        VM_PAGE_CHECK(m);
 #if CONFIG_BACKGROUND_QUEUE
                        m->vmp_q_state = VM_PAGE_ON_ACTIVE_Q;
                        VM_PAGE_CHECK(m);
 #if CONFIG_BACKGROUND_QUEUE

-                       if (m->vmp_in_background)
+                       if (m->vmp_in_background) {

                                vm_page_add_to_backgroundq(m, FALSE);
                                vm_page_add_to_backgroundq(m, FALSE);

+                       }

 #endif
                }
 
 #endif
                }
 


@@ -4709,23 +5261,26 @@ vm_page_reactivate_all_throttled(void)
 void
 vm_page_reactivate_local(uint32_t lid, boolean_t force, boolean_t nolocks)
 {
 void
 vm_page_reactivate_local(uint32_t lid, boolean_t force, boolean_t nolocks)
 {

-       struct vpl      *lq;
-       vm_page_t       first_local, last_local;
-       vm_page_t       first_active;
-       vm_page_t       m;
-       uint32_t        count = 0;
+       struct vpl      *lq;
+       vm_page_t       first_local, last_local;
+       vm_page_t       first_active;
+       vm_page_t       m;
+       uint32_t        count = 0;

 
 

-       if (vm_page_local_q == NULL)
+       if (vm_page_local_q == NULL) {

                return;
                return;

+       }

 
 

-       lq = &vm_page_local_q[lid].vpl_un.vpl;
+       lq = zpercpu_get_cpu(vm_page_local_q, lid);

 
        if (nolocks == FALSE) {
                if (lq->vpl_count < vm_page_local_q_hard_limit && force == FALSE) {
 
        if (nolocks == FALSE) {
                if (lq->vpl_count < vm_page_local_q_hard_limit && force == FALSE) {

-                       if ( !vm_page_trylockspin_queues())
+                       if (!vm_page_trylockspin_queues()) {

                                return;
                                return;

-               } else
+                       }
+               } else {

                        vm_page_lockspin_queues();
                        vm_page_lockspin_queues();

+               }

 
                VPL_LOCK(&lq->vpl_lock);
        }
 
                VPL_LOCK(&lq->vpl_lock);
        }


@@ -4735,26 +5290,29 @@ vm_page_reactivate_local(uint32_t lid, boolean_t force, boolean_t nolocks)
                 */
                assert(!vm_page_queue_empty(&lq->vpl_queue));
 
                 */
                assert(!vm_page_queue_empty(&lq->vpl_queue));
 

-               vm_page_queue_iterate(&lq->vpl_queue, m, vm_page_t, vmp_pageq) {
+               vm_page_queue_iterate(&lq->vpl_queue, m, vmp_pageq) {

                        VM_PAGE_CHECK(m);
                        vm_page_check_pageable_safe(m);
                        assert(m->vmp_q_state == VM_PAGE_ON_ACTIVE_LOCAL_Q);
                        assert(!m->vmp_fictitious);
 
                        VM_PAGE_CHECK(m);
                        vm_page_check_pageable_safe(m);
                        assert(m->vmp_q_state == VM_PAGE_ON_ACTIVE_LOCAL_Q);
                        assert(!m->vmp_fictitious);
 

-                       if (m->vmp_local_id != lid)
+                       if (m->vmp_local_id != lid) {

                                panic("vm_page_reactivate_local: found vm_page_t(%p) with wrong cpuid", m);
                                panic("vm_page_reactivate_local: found vm_page_t(%p) with wrong cpuid", m);

-                       
+                       }
+

                        m->vmp_local_id = 0;
                        m->vmp_q_state = VM_PAGE_ON_ACTIVE_Q;
                        VM_PAGE_CHECK(m);
 #if CONFIG_BACKGROUND_QUEUE
                        m->vmp_local_id = 0;
                        m->vmp_q_state = VM_PAGE_ON_ACTIVE_Q;
                        VM_PAGE_CHECK(m);
 #if CONFIG_BACKGROUND_QUEUE

-                       if (m->vmp_in_background)
+                       if (m->vmp_in_background) {

                                vm_page_add_to_backgroundq(m, FALSE);
                                vm_page_add_to_backgroundq(m, FALSE);

+                       }

 #endif
                        count++;
                }
 #endif
                        count++;
                }

-               if (count != lq->vpl_count)
+               if (count != lq->vpl_count) {

                        panic("vm_page_reactivate_local: count = %d, vm_page_local_count = %d\n", count, lq->vpl_count);
                        panic("vm_page_reactivate_local: count = %d, vm_page_local_count = %d\n", count, lq->vpl_count);

+               }

 
                /*
                 * Transfer the entire local queue to a regular LRU page queues.
 
                /*
                 * Transfer the entire local queue to a regular LRU page queues.


@@ -4801,11 +5359,10 @@ vm_page_reactivate_local(uint32_t lid, boolean_t force, boolean_t nolocks)
 #define PMAP_ZERO_PART_PAGE_IMPLEMENTED
 void
 vm_page_part_zero_fill(
 #define PMAP_ZERO_PART_PAGE_IMPLEMENTED
 void
 vm_page_part_zero_fill(

-       vm_page_t       m,
-       vm_offset_t     m_pa,
-       vm_size_t       len)
+       vm_page_t       m,
+       vm_offset_t     m_pa,
+       vm_size_t       len)

 {
 {

-

 #if 0
        /*
         * we don't hold the page queue lock
 #if 0
        /*
         * we don't hold the page queue lock


@@ -4817,27 +5374,26 @@ vm_page_part_zero_fill(
 #ifdef PMAP_ZERO_PART_PAGE_IMPLEMENTED
        pmap_zero_part_page(VM_PAGE_GET_PHYS_PAGE(m), m_pa, len);
 #else
 #ifdef PMAP_ZERO_PART_PAGE_IMPLEMENTED
        pmap_zero_part_page(VM_PAGE_GET_PHYS_PAGE(m), m_pa, len);
 #else

-       vm_page_t       tmp;
+       vm_page_t       tmp;

        while (1) {
        while (1) {

-                       tmp = vm_page_grab();
+               tmp = vm_page_grab();

                if (tmp == VM_PAGE_NULL) {
                        vm_page_wait(THREAD_UNINT);
                        continue;
                }
                if (tmp == VM_PAGE_NULL) {
                        vm_page_wait(THREAD_UNINT);
                        continue;
                }

-               break;  
+               break;

        }
        vm_page_zero_fill(tmp);
        }
        vm_page_zero_fill(tmp);

-       if(m_pa != 0) {
+       if (m_pa != 0) {

                vm_page_part_copy(m, 0, tmp, 0, m_pa);
        }
                vm_page_part_copy(m, 0, tmp, 0, m_pa);
        }

-       if((m_pa + len) <  PAGE_SIZE) {
-               vm_page_part_copy(m, m_pa + len, tmp, 
-                               m_pa + len, PAGE_SIZE - (m_pa + len));
+       if ((m_pa + len) < PAGE_SIZE) {
+               vm_page_part_copy(m, m_pa + len, tmp,
+                   m_pa + len, PAGE_SIZE - (m_pa + len));

        }
        }

-       vm_page_copy(tmp,m);
-       VM_PAGE_FREE(tmp); 
+       vm_page_copy(tmp, m);
+       VM_PAGE_FREE(tmp);

 #endif
 #endif

-

 }
 
 /*
 }
 
 /*


@@ -4847,11 +5403,8 @@ vm_page_part_zero_fill(
  */
 void
 vm_page_zero_fill(
  */
 void
 vm_page_zero_fill(

-       vm_page_t       m)
+       vm_page_t       m)

 {
 {

-        XPR(XPR_VM_PAGE,
-           "vm_page_zero_fill, object 0x%X offset 0x%X page 0x%X\n",
-           VM_PAGE_OBJECT(m), m->vmp_offset, m, 0,0);

 #if 0
        /*
         * we don't hold the page queue lock
 #if 0
        /*
         * we don't hold the page queue lock


@@ -4872,11 +5425,11 @@ vm_page_zero_fill(
 
 void
 vm_page_part_copy(
 
 void
 vm_page_part_copy(

-       vm_page_t       src_m,
-       vm_offset_t     src_pa,
-       vm_page_t       dst_m,
-       vm_offset_t     dst_pa,
-       vm_size_t       len)
+       vm_page_t       src_m,
+       vm_offset_t     src_pa,
+       vm_page_t       dst_m,
+       vm_offset_t     dst_pa,
+       vm_size_t       len)

 {
 #if 0
        /*
 {
 #if 0
        /*


@@ -4887,7 +5440,7 @@ vm_page_part_copy(
        VM_PAGE_CHECK(dst_m);
 #endif
        pmap_copy_part_page(VM_PAGE_GET_PHYS_PAGE(src_m), src_pa,
        VM_PAGE_CHECK(dst_m);
 #endif
        pmap_copy_part_page(VM_PAGE_GET_PHYS_PAGE(src_m), src_pa,

-                           VM_PAGE_GET_PHYS_PAGE(dst_m), dst_pa, len);
+           VM_PAGE_GET_PHYS_PAGE(dst_m), dst_pa, len);

 }
 
 /*
 }
 
 /*


@@ -4901,18 +5454,13 @@ int vm_page_copy_cs_tainted = 0;
 
 void
 vm_page_copy(
 
 void
 vm_page_copy(

-       vm_page_t       src_m,
-       vm_page_t       dest_m)
+       vm_page_t       src_m,
+       vm_page_t       dest_m)

 {
 {

-       vm_object_t     src_m_object;
+       vm_object_t     src_m_object;

 
        src_m_object = VM_PAGE_OBJECT(src_m);
 
 
        src_m_object = VM_PAGE_OBJECT(src_m);
 

-        XPR(XPR_VM_PAGE,
-           "vm_page_copy, object 0x%X offset 0x%X to object 0x%X offset 0x%X\n",
-           src_m_object, src_m->vmp_offset, 
-           VM_PAGE_OBJECT(dest_m), dest_m->vmp_offset,
-           0);

 #if 0
        /*
         * we don't hold the page queue lock
 #if 0
        /*
         * we don't hold the page queue lock


@@ -4932,15 +5480,14 @@ vm_page_copy(
                 * source page now.
                 */
                vm_page_copy_cs_validations++;
                 * source page now.
                 */
                vm_page_copy_cs_validations++;

-               vm_page_validate_cs(src_m);
+               vm_page_validate_cs(src_m, PAGE_SIZE, 0);

 #if DEVELOPMENT || DEBUG
                DTRACE_VM4(codesigned_copy,
 #if DEVELOPMENT || DEBUG
                DTRACE_VM4(codesigned_copy,

-                          vm_object_t, src_m_object,
-                          vm_object_offset_t, src_m->vmp_offset,
-                          int, src_m->vmp_cs_validated,
-                          int, src_m->vmp_cs_tainted);
+                   vm_object_t, src_m_object,
+                   vm_object_offset_t, src_m->vmp_offset,
+                   int, src_m->vmp_cs_validated,
+                   int, src_m->vmp_cs_tainted);

 #endif /* DEVELOPMENT || DEBUG */
 #endif /* DEVELOPMENT || DEBUG */

-

        }
 
        /*
        }
 
        /*


@@ -4948,6 +5495,7 @@ vm_page_copy(
         * the cs_validated bit.
         */
        dest_m->vmp_cs_tainted = src_m->vmp_cs_tainted;
         * the cs_validated bit.
         */
        dest_m->vmp_cs_tainted = src_m->vmp_cs_tainted;

+       dest_m->vmp_cs_nx = src_m->vmp_cs_nx;

        if (dest_m->vmp_cs_tainted) {
                vm_page_copy_cs_tainted++;
        }
        if (dest_m->vmp_cs_tainted) {
                vm_page_copy_cs_tainted++;
        }


@@ -4958,49 +5506,49 @@ vm_page_copy(
 #if MACH_ASSERT
 static void
 _vm_page_print(
 #if MACH_ASSERT
 static void
 _vm_page_print(

-       vm_page_t       p)
+       vm_page_t       p)

 {
        printf("vm_page %p: \n", p);
        printf("  pageq: next=%p prev=%p\n",
 {
        printf("vm_page %p: \n", p);
        printf("  pageq: next=%p prev=%p\n",

-              (vm_page_t)VM_PAGE_UNPACK_PTR(p->vmp_pageq.next),
-              (vm_page_t)VM_PAGE_UNPACK_PTR(p->vmp_pageq.prev));
+           (vm_page_t)VM_PAGE_UNPACK_PTR(p->vmp_pageq.next),
+           (vm_page_t)VM_PAGE_UNPACK_PTR(p->vmp_pageq.prev));

        printf("  listq: next=%p prev=%p\n",
        printf("  listq: next=%p prev=%p\n",

-              (vm_page_t)(VM_PAGE_UNPACK_PTR(p->vmp_listq.next)),
-              (vm_page_t)(VM_PAGE_UNPACK_PTR(p->vmp_listq.prev)));
+           (vm_page_t)(VM_PAGE_UNPACK_PTR(p->vmp_listq.next)),
+           (vm_page_t)(VM_PAGE_UNPACK_PTR(p->vmp_listq.prev)));

        printf("  next=%p\n", (vm_page_t)(VM_PAGE_UNPACK_PTR(p->vmp_next_m)));
        printf("  next=%p\n", (vm_page_t)(VM_PAGE_UNPACK_PTR(p->vmp_next_m)));

-       printf("  object=%p offset=0x%llx\n",VM_PAGE_OBJECT(p), p->vmp_offset);
+       printf("  object=%p offset=0x%llx\n", VM_PAGE_OBJECT(p), p->vmp_offset);

        printf("  wire_count=%u\n", p->vmp_wire_count);
        printf("  q_state=%u\n", p->vmp_q_state);
 
        printf("  %slaundry, %sref, %sgobbled, %sprivate\n",
        printf("  wire_count=%u\n", p->vmp_wire_count);
        printf("  q_state=%u\n", p->vmp_q_state);
 
        printf("  %slaundry, %sref, %sgobbled, %sprivate\n",

-              (p->vmp_laundry ? "" : "!"),
-              (p->vmp_reference ? "" : "!"),
-              (p->vmp_gobbled ? "" : "!"),
-              (p->vmp_private ? "" : "!"));
+           (p->vmp_laundry ? "" : "!"),
+           (p->vmp_reference ? "" : "!"),
+           (p->vmp_gobbled ? "" : "!"),
+           (p->vmp_private ? "" : "!"));

        printf("  %sbusy, %swanted, %stabled, %sfictitious, %spmapped, %swpmapped\n",
        printf("  %sbusy, %swanted, %stabled, %sfictitious, %spmapped, %swpmapped\n",

-              (p->vmp_busy ? "" : "!"),
-              (p->vmp_wanted ? "" : "!"),
-              (p->vmp_tabled ? "" : "!"),
-              (p->vmp_fictitious ? "" : "!"),
-              (p->vmp_pmapped ? "" : "!"),
-              (p->vmp_wpmapped ? "" : "!"));
+           (p->vmp_busy ? "" : "!"),
+           (p->vmp_wanted ? "" : "!"),
+           (p->vmp_tabled ? "" : "!"),
+           (p->vmp_fictitious ? "" : "!"),
+           (p->vmp_pmapped ? "" : "!"),
+           (p->vmp_wpmapped ? "" : "!"));

        printf("  %sfree_when_done, %sabsent, %serror, %sdirty, %scleaning, %sprecious, %sclustered\n",
        printf("  %sfree_when_done, %sabsent, %serror, %sdirty, %scleaning, %sprecious, %sclustered\n",

-              (p->vmp_free_when_done ? "" : "!"),
-              (p->vmp_absent ? "" : "!"),
-              (p->vmp_error ? "" : "!"),
-              (p->vmp_dirty ? "" : "!"),
-              (p->vmp_cleaning ? "" : "!"),
-              (p->vmp_precious ? "" : "!"),
-              (p->vmp_clustered ? "" : "!"));
+           (p->vmp_free_when_done ? "" : "!"),
+           (p->vmp_absent ? "" : "!"),
+           (p->vmp_error ? "" : "!"),
+           (p->vmp_dirty ? "" : "!"),
+           (p->vmp_cleaning ? "" : "!"),
+           (p->vmp_precious ? "" : "!"),
+           (p->vmp_clustered ? "" : "!"));

        printf("  %soverwriting, %srestart, %sunusual\n",
        printf("  %soverwriting, %srestart, %sunusual\n",

-              (p->vmp_overwriting ? "" : "!"),
-              (p->vmp_restart ? "" : "!"),
-              (p->vmp_unusual ? "" : "!"));
-       printf("  %scs_validated, %scs_tainted, %scs_nx, %sno_cache\n",
-              (p->vmp_cs_validated ? "" : "!"),
-              (p->vmp_cs_tainted ? "" : "!"),
-              (p->vmp_cs_nx ? "" : "!"),
-              (p->vmp_no_cache ? "" : "!"));
+           (p->vmp_overwriting ? "" : "!"),
+           (p->vmp_restart ? "" : "!"),
+           (p->vmp_unusual ? "" : "!"));
+       printf("  cs_validated=%d, cs_tainted=%d, cs_nx=%d, %sno_cache\n",
+           p->vmp_cs_validated,
+           p->vmp_cs_tainted,
+           p->vmp_cs_nx,
+           (p->vmp_no_cache ? "" : "!"));

 
        printf("phys_page=0x%x\n", VM_PAGE_GET_PHYS_PAGE(p));
 }
 
        printf("phys_page=0x%x\n", VM_PAGE_GET_PHYS_PAGE(p));
 }


@@ -5011,19 +5559,19 @@ _vm_page_print(
  */
 static int
 vm_page_verify_contiguous(
  */
 static int
 vm_page_verify_contiguous(

-       vm_page_t       pages,
-       unsigned int    npages)
+       vm_page_t       pages,
+       unsigned int    npages)

 {
 {

-       vm_page_t               m;
-       unsigned int            page_count;
-       vm_offset_t             prev_addr;
+       vm_page_t               m;
+       unsigned int            page_count;
+       vm_offset_t             prev_addr;

 
        prev_addr = VM_PAGE_GET_PHYS_PAGE(pages);
        page_count = 1;
        for (m = NEXT_PAGE(pages); m != VM_PAGE_NULL; m = NEXT_PAGE(m)) {
                if (VM_PAGE_GET_PHYS_PAGE(m) != prev_addr + 1) {
                        printf("m %p prev_addr 0x%lx, current addr 0x%x\n",
 
        prev_addr = VM_PAGE_GET_PHYS_PAGE(pages);
        page_count = 1;
        for (m = NEXT_PAGE(pages); m != VM_PAGE_NULL; m = NEXT_PAGE(m)) {
                if (VM_PAGE_GET_PHYS_PAGE(m) != prev_addr + 1) {
                        printf("m %p prev_addr 0x%lx, current addr 0x%x\n",

-                              m, (long)prev_addr, VM_PAGE_GET_PHYS_PAGE(m));
+                           m, (long)prev_addr, VM_PAGE_GET_PHYS_PAGE(m));

                        printf("pages %p page_count %d npages %d\n", pages, page_count, npages);
                        panic("vm_page_verify_contiguous:  not contiguous!");
                }
                        printf("pages %p page_count %d npages %d\n", pages, page_count, npages);
                        panic("vm_page_verify_contiguous:  not contiguous!");
                }


@@ -5032,7 +5580,7 @@ vm_page_verify_contiguous(
        }
        if (page_count != npages) {
                printf("pages %p actual count 0x%x but requested 0x%x\n",
        }
        if (page_count != npages) {
                printf("pages %p actual count 0x%x but requested 0x%x\n",

-                      pages, page_count, npages);
+                   pages, page_count, npages);

                panic("vm_page_verify_contiguous:  count error");
        }
        return 1;
                panic("vm_page_verify_contiguous:  count error");
        }
        return 1;


@@ -5045,48 +5593,50 @@ vm_page_verify_contiguous(
 static boolean_t vm_page_verify_this_free_list_enabled = FALSE;
 static unsigned int
 vm_page_verify_free_list(
 static boolean_t vm_page_verify_this_free_list_enabled = FALSE;
 static unsigned int
 vm_page_verify_free_list(

-       vm_page_queue_head_t    *vm_page_queue,
-       unsigned int    color,
-       vm_page_t       look_for_page,
-       boolean_t       expect_page)
+       vm_page_queue_head_t    *vm_page_queue,
+       unsigned int    color,
+       vm_page_t       look_for_page,
+       boolean_t       expect_page)

 {
 {

-       unsigned int    npages;
-       vm_page_t       m;
-       vm_page_t       prev_m;
-       boolean_t       found_page;
+       unsigned int    npages;
+       vm_page_t       m;
+       vm_page_t       prev_m;
+       boolean_t       found_page;

 
 

-       if (! vm_page_verify_this_free_list_enabled)
+       if (!vm_page_verify_this_free_list_enabled) {

                return 0;
                return 0;

+       }

 
        found_page = FALSE;
        npages = 0;
        prev_m = (vm_page_t)((uintptr_t)vm_page_queue);
 
 
        found_page = FALSE;
        npages = 0;
        prev_m = (vm_page_t)((uintptr_t)vm_page_queue);
 

-       vm_page_queue_iterate(vm_page_queue,
-                             m,
-                             vm_page_t,
-                             vmp_pageq) {
-
+       vm_page_queue_iterate(vm_page_queue, m, vmp_pageq) {

                if (m == look_for_page) {
                        found_page = TRUE;
                }
                if (m == look_for_page) {
                        found_page = TRUE;
                }

-               if ((vm_page_t)VM_PAGE_UNPACK_PTR(m->vmp_pageq.prev) != prev_m)
+               if ((vm_page_t)VM_PAGE_UNPACK_PTR(m->vmp_pageq.prev) != prev_m) {

                        panic("vm_page_verify_free_list(color=%u, npages=%u): page %p corrupted prev ptr %p instead of %p\n",
                        panic("vm_page_verify_free_list(color=%u, npages=%u): page %p corrupted prev ptr %p instead of %p\n",

-                             color, npages, m, (vm_page_t)VM_PAGE_UNPACK_PTR(m->vmp_pageq.prev), prev_m);
-               if ( ! m->vmp_busy )
+                           color, npages, m, (vm_page_t)VM_PAGE_UNPACK_PTR(m->vmp_pageq.prev), prev_m);
+               }
+               if (!m->vmp_busy) {

                        panic("vm_page_verify_free_list(color=%u, npages=%u): page %p not busy\n",
                        panic("vm_page_verify_free_list(color=%u, npages=%u): page %p not busy\n",

-                             color, npages, m);
+                           color, npages, m);
+               }

                if (color != (unsigned int) -1) {
                if (color != (unsigned int) -1) {

-                       if (VM_PAGE_GET_COLOR(m) != color)
+                       if (VM_PAGE_GET_COLOR(m) != color) {

                                panic("vm_page_verify_free_list(color=%u, npages=%u): page %p wrong color %u instead of %u\n",
                                panic("vm_page_verify_free_list(color=%u, npages=%u): page %p wrong color %u instead of %u\n",

-                                     color, npages, m, VM_PAGE_GET_COLOR(m), color);
-                       if (m->vmp_q_state != VM_PAGE_ON_FREE_Q)
+                                   color, npages, m, VM_PAGE_GET_COLOR(m), color);
+                       }
+                       if (m->vmp_q_state != VM_PAGE_ON_FREE_Q) {

                                panic("vm_page_verify_free_list(color=%u, npages=%u): page %p - expecting q_state == VM_PAGE_ON_FREE_Q, found %d\n",
                                panic("vm_page_verify_free_list(color=%u, npages=%u): page %p - expecting q_state == VM_PAGE_ON_FREE_Q, found %d\n",

-                                     color, npages, m, m->vmp_q_state);
+                                   color, npages, m, m->vmp_q_state);
+                       }

                } else {
                } else {

-                       if (m->vmp_q_state != VM_PAGE_ON_FREE_LOCAL_Q)
+                       if (m->vmp_q_state != VM_PAGE_ON_FREE_LOCAL_Q) {

                                panic("vm_page_verify_free_list(npages=%u): local page %p - expecting q_state == VM_PAGE_ON_FREE_LOCAL_Q, found %d\n",
                                panic("vm_page_verify_free_list(npages=%u): local page %p - expecting q_state == VM_PAGE_ON_FREE_LOCAL_Q, found %d\n",

-                                     npages, m, m->vmp_q_state);
+                                   npages, m, m->vmp_q_state);
+                       }

                }
                ++npages;
                prev_m = m;
                }
                ++npages;
                prev_m = m;


@@ -5096,25 +5646,26 @@ vm_page_verify_free_list(
 
                if (expect_page && !found_page) {
                        printf("vm_page_verify_free_list(color=%u, npages=%u): page %p not found phys=%u\n",
 
                if (expect_page && !found_page) {
                        printf("vm_page_verify_free_list(color=%u, npages=%u): page %p not found phys=%u\n",

-                              color, npages, look_for_page, VM_PAGE_GET_PHYS_PAGE(look_for_page));
+                           color, npages, look_for_page, VM_PAGE_GET_PHYS_PAGE(look_for_page));

                        _vm_page_print(look_for_page);
                        for (other_color = 0;
                        _vm_page_print(look_for_page);
                        for (other_color = 0;

-                            other_color < vm_colors;
-                            other_color++) {
-                               if (other_color == color)
+                           other_color < vm_colors;
+                           other_color++) {
+                               if (other_color == color) {

                                        continue;
                                        continue;

+                               }

                                vm_page_verify_free_list(&vm_page_queue_free[other_color].qhead,
                                vm_page_verify_free_list(&vm_page_queue_free[other_color].qhead,

-                                                        other_color, look_for_page, FALSE);
+                                   other_color, look_for_page, FALSE);

                        }
                        if (color == (unsigned int) -1) {
                                vm_page_verify_free_list(&vm_lopage_queue_free,
                        }
                        if (color == (unsigned int) -1) {
                                vm_page_verify_free_list(&vm_lopage_queue_free,

-                                                        (unsigned int) -1, look_for_page, FALSE);
+                                   (unsigned int) -1, look_for_page, FALSE);

                        }
                        panic("vm_page_verify_free_list(color=%u)\n", color);
                }
                if (!expect_page && found_page) {
                        printf("vm_page_verify_free_list(color=%u, npages=%u): page %p found phys=%u\n",
                        }
                        panic("vm_page_verify_free_list(color=%u)\n", color);
                }
                if (!expect_page && found_page) {
                        printf("vm_page_verify_free_list(color=%u, npages=%u): page %p found phys=%u\n",

-                              color, npages, look_for_page, VM_PAGE_GET_PHYS_PAGE(look_for_page));
+                           color, npages, look_for_page, VM_PAGE_GET_PHYS_PAGE(look_for_page));

                }
        }
        return npages;
                }
        }
        return npages;


@@ -5124,16 +5675,17 @@ static boolean_t vm_page_verify_all_free_lists_enabled = FALSE;
 static void
 vm_page_verify_free_lists( void )
 {
 static void
 vm_page_verify_free_lists( void )
 {

-       unsigned int    color, npages, nlopages;
-       boolean_t       toggle = TRUE;
+       unsigned int    color, npages, nlopages;
+       boolean_t       toggle = TRUE;

 
 

-       if (! vm_page_verify_all_free_lists_enabled)
+       if (!vm_page_verify_all_free_lists_enabled) {

                return;
                return;

+       }

 
        npages = 0;
 
        lck_mtx_lock(&vm_page_queue_free_lock);
 
        npages = 0;
 
        lck_mtx_lock(&vm_page_queue_free_lock);

-       
+

        if (vm_page_verify_this_free_list_enabled == TRUE) {
                /*
                 * This variable has been set globally for extra checking of
        if (vm_page_verify_this_free_list_enabled == TRUE) {
                /*
                 * This variable has been set globally for extra checking of


@@ -5147,17 +5699,18 @@ vm_page_verify_free_lists( void )
                vm_page_verify_this_free_list_enabled = TRUE;
        }
 
                vm_page_verify_this_free_list_enabled = TRUE;
        }
 

-       for( color = 0; color < vm_colors; color++ ) {
+       for (color = 0; color < vm_colors; color++) {

                npages += vm_page_verify_free_list(&vm_page_queue_free[color].qhead,
                npages += vm_page_verify_free_list(&vm_page_queue_free[color].qhead,

-                                                  color, VM_PAGE_NULL, FALSE);
+                   color, VM_PAGE_NULL, FALSE);

        }
        nlopages = vm_page_verify_free_list(&vm_lopage_queue_free,
        }
        nlopages = vm_page_verify_free_list(&vm_lopage_queue_free,

-                                           (unsigned int) -1,
-                                           VM_PAGE_NULL, FALSE);
-       if (npages != vm_page_free_count || nlopages != vm_lopage_free_count)
+           (unsigned int) -1,
+           VM_PAGE_NULL, FALSE);
+       if (npages != vm_page_free_count || nlopages != vm_lopage_free_count) {

                panic("vm_page_verify_free_lists:  "
                panic("vm_page_verify_free_lists:  "

-                     "npages %u free_count %d nlopages %u lo_free_count %u",
-                     npages, vm_page_free_count, nlopages, vm_lopage_free_count);
+                   "npages %u free_count %d nlopages %u lo_free_count %u",
+                   npages, vm_page_free_count, nlopages, vm_lopage_free_count);
+       }

 
        if (toggle == TRUE) {
                vm_page_verify_this_free_list_enabled = FALSE;
 
        if (toggle == TRUE) {
                vm_page_verify_this_free_list_enabled = FALSE;


@@ -5166,49 +5719,10 @@ vm_page_verify_free_lists( void )
        lck_mtx_unlock(&vm_page_queue_free_lock);
 }
 
        lck_mtx_unlock(&vm_page_queue_free_lock);
 }
 

-#endif /* MACH_ASSERT */
-
-
-
-#if __arm64__
-/*
- *     1 or more clients (currently only SEP) ask for a large contiguous chunk of memory 
- *     after the system has 'aged'. To ensure that other allocation requests don't mess
- *     with the chances of that request being satisfied, we pre-allocate a single contiguous
- *     10MB buffer and hand it out to the first request of >= 4MB.
- */
-
-kern_return_t cpm_preallocate_early(void);
-
-vm_page_t cpm_preallocated_pages_list = NULL;
-boolean_t preallocated_buffer_available = FALSE;
-
-#define        PREALLOCATED_CONTIG_BUFFER_PAGES_COUNT                  ((10 * 1024 * 1024) / PAGE_SIZE_64) /* 10 MB */
-#define MIN_CONTIG_PAGES_REQUEST_FOR_PREALLOCATED_BUFFER       ((4 * 1024 *1024) / PAGE_SIZE_64)   /* 4 MB */
-
-kern_return_t
-cpm_preallocate_early(void)
-{
-
-       kern_return_t   kr = KERN_SUCCESS;
-       vm_map_size_t   prealloc_size = (PREALLOCATED_CONTIG_BUFFER_PAGES_COUNT * PAGE_SIZE_64);
-
-       printf("cpm_preallocate_early called to preallocate contiguous buffer of %llu pages\n", PREALLOCATED_CONTIG_BUFFER_PAGES_COUNT);
-
-       kr = cpm_allocate(CAST_DOWN(vm_size_t, prealloc_size), &cpm_preallocated_pages_list, 0, 0, TRUE, 0);
-
-       if (kr != KERN_SUCCESS) {
-               printf("cpm_allocate for preallocated contig buffer failed with %d.\n", kr);
-       } else {
-               preallocated_buffer_available = TRUE;
-       }
-
-       return kr;
-}
-#endif /* __arm64__ */
+#endif  /* MACH_ASSERT */

 
 
 
 

-extern boolean_t (* volatile consider_buffer_cache_collect)(int);
+extern boolean_t(*volatile consider_buffer_cache_collect)(int);

 
 /*
  *     CONTIGUOUS PAGE ALLOCATION
 
 /*
  *     CONTIGUOUS PAGE ALLOCATION


@@ -5219,27 +5733,27 @@ extern boolean_t (* volatile consider_buffer_cache_collect)(int);
  *     This is done by traversing the vm_page_t array in a linear fashion
  *     we assume that the vm_page_t array has the avaiable physical pages in an
  *     ordered, ascending list... this is currently true of all our implementations
  *     This is done by traversing the vm_page_t array in a linear fashion
  *     we assume that the vm_page_t array has the avaiable physical pages in an
  *     ordered, ascending list... this is currently true of all our implementations

- *     and must remain so... there can be 'holes' in the array...  we also can
+ *      and must remain so... there can be 'holes' in the array...  we also can

  *     no longer tolerate the vm_page_t's in the list being 'freed' and reclaimed
  *     no longer tolerate the vm_page_t's in the list being 'freed' and reclaimed

- *     which use to happen via 'vm_page_convert'... that function was no longer
- *     being called and was removed...
- *     
- *     The basic flow consists of stabilizing some of the interesting state of 
+ *      which use to happen via 'vm_page_convert'... that function was no longer
+ *      being called and was removed...
+ *
+ *     The basic flow consists of stabilizing some of the interesting state of

  *     a vm_page_t behind the vm_page_queue and vm_page_free locks... we start our
  *     sweep at the beginning of the array looking for pages that meet our criterea
  *     for a 'stealable' page... currently we are pretty conservative... if the page
  *     meets this criterea and is physically contiguous to the previous page in the 'run'
  *     a vm_page_t behind the vm_page_queue and vm_page_free locks... we start our
  *     sweep at the beginning of the array looking for pages that meet our criterea
  *     for a 'stealable' page... currently we are pretty conservative... if the page
  *     meets this criterea and is physically contiguous to the previous page in the 'run'

- *     we keep developing it.  If we hit a page that doesn't fit, we reset our state
+ *      we keep developing it.  If we hit a page that doesn't fit, we reset our state

  *     and start to develop a new run... if at this point we've already considered
  *     and start to develop a new run... if at this point we've already considered

- *     at least MAX_CONSIDERED_BEFORE_YIELD pages, we'll drop the 2 locks we hold,
- *     and mutex_pause (which will yield the processor), to keep the latency low w/r 
+ *      at least MAX_CONSIDERED_BEFORE_YIELD pages, we'll drop the 2 locks we hold,
+ *     and mutex_pause (which will yield the processor), to keep the latency low w/r

  *     to other threads trying to acquire free pages (or move pages from q to q),
  *     and then continue from the spot we left off... we only make 1 pass through the
  *     array.  Once we have a 'run' that is long enough, we'll go into the loop which
  *     to other threads trying to acquire free pages (or move pages from q to q),
  *     and then continue from the spot we left off... we only make 1 pass through the
  *     array.  Once we have a 'run' that is long enough, we'll go into the loop which

- *     which steals the pages from the queues they're currently on... pages on the free
+ *      which steals the pages from the queues they're currently on... pages on the free

  *     queue can be stolen directly... pages that are on any of the other queues
  *     must be removed from the object they are tabled on... this requires taking the
  *     queue can be stolen directly... pages that are on any of the other queues
  *     must be removed from the object they are tabled on... this requires taking the

- *     object lock... we do this as a 'try' to prevent deadlocks... if the 'try' fails
+ *      object lock... we do this as a 'try' to prevent deadlocks... if the 'try' fails

  *     or if the state of the page behind the vm_object lock is no longer viable, we'll
  *     dump the pages we've currently stolen back to the free list, and pick up our
  *     scan from the point where we aborted the 'current' run.
  *     or if the state of the page behind the vm_object lock is no longer viable, we'll
  *     dump the pages we've currently stolen back to the free list, and pick up our
  *     scan from the point where we aborted the 'current' run.


@@ -5253,17 +5767,17 @@ extern boolean_t (* volatile consider_buffer_cache_collect)(int);
  * Algorithm:
  */
 
  * Algorithm:
  */
 

-#define        MAX_CONSIDERED_BEFORE_YIELD     1000
+#define MAX_CONSIDERED_BEFORE_YIELD     1000

 
 
 
 

-#define RESET_STATE_OF_RUN()   \
-       MACRO_BEGIN             \
-       prevcontaddr = -2;      \
-       start_pnum = -1;        \
-       free_considered = 0;    \
-       substitute_needed = 0;  \
-       npages = 0;             \
-       MACRO_END                       
+#define RESET_STATE_OF_RUN()    \
+       MACRO_BEGIN             \
+       prevcontaddr = -2;      \
+       start_pnum = -1;        \
+       free_considered = 0;    \
+       substitute_needed = 0;  \
+       npages = 0;             \
+       MACRO_END

 
 /*
  * Can we steal in-use (i.e. not free) pages when searching for
 
 /*
  * Can we steal in-use (i.e. not free) pages when searching for


@@ -5271,42 +5785,43 @@ extern boolean_t (* volatile consider_buffer_cache_collect)(int);
  */
 #define VM_PAGE_FIND_CONTIGUOUS_CAN_STEAL 1
 
  */
 #define VM_PAGE_FIND_CONTIGUOUS_CAN_STEAL 1
 

-static unsigned int vm_page_find_contiguous_last_idx = 0,  vm_page_lomem_find_contiguous_last_idx = 0;
+static unsigned int vm_page_find_contiguous_last_idx = 0, vm_page_lomem_find_contiguous_last_idx = 0;

 #if DEBUG
 int vm_page_find_contig_debug = 0;
 #endif
 
 static vm_page_t
 vm_page_find_contiguous(
 #if DEBUG
 int vm_page_find_contig_debug = 0;
 #endif
 
 static vm_page_t
 vm_page_find_contiguous(

-       unsigned int    contig_pages,
-       ppnum_t         max_pnum,
+       unsigned int    contig_pages,
+       ppnum_t         max_pnum,

        ppnum_t     pnum_mask,
        ppnum_t     pnum_mask,

-       boolean_t       wire,
-       int             flags)
-{
-       vm_page_t       m = NULL;
-       ppnum_t         prevcontaddr = 0;
-       ppnum_t         start_pnum = 0;
-       unsigned int    npages = 0, considered = 0, scanned = 0;
-       unsigned int    page_idx = 0, start_idx = 0, last_idx = 0, orig_last_idx = 0;
-       unsigned int    idx_last_contig_page_found = 0;
-       int             free_considered = 0, free_available = 0;
-       int             substitute_needed = 0;
-       boolean_t       wrapped, zone_gc_called = FALSE;
-       kern_return_t   kr;
+       boolean_t       wire,
+       int             flags)
+{
+       vm_page_t       m = NULL;
+       ppnum_t         prevcontaddr = 0;
+       ppnum_t         start_pnum = 0;
+       unsigned int    npages = 0, considered = 0, scanned = 0;
+       unsigned int    page_idx = 0, start_idx = 0, last_idx = 0, orig_last_idx = 0;
+       unsigned int    idx_last_contig_page_found = 0;
+       int             free_considered = 0, free_available = 0;
+       int             substitute_needed = 0;
+       boolean_t       wrapped, zone_gc_called = FALSE;
+       kern_return_t   kr;

 #if DEBUG
 #if DEBUG

-       clock_sec_t     tv_start_sec = 0, tv_end_sec = 0;
-       clock_usec_t    tv_start_usec = 0, tv_end_usec = 0;
+       clock_sec_t     tv_start_sec = 0, tv_end_sec = 0;
+       clock_usec_t    tv_start_usec = 0, tv_end_usec = 0;

 #endif
 
 #endif
 

-       int             yielded = 0;
-       int             dumped_run = 0;
-       int             stolen_pages = 0;
-       int             compressed_pages = 0;
+       int             yielded = 0;
+       int             dumped_run = 0;
+       int             stolen_pages = 0;
+       int             compressed_pages = 0;

 
 
 
 

-       if (contig_pages == 0)
+       if (contig_pages == 0) {

                return VM_PAGE_NULL;
                return VM_PAGE_NULL;

+       }

 
 full_scan_again:
 
 
 full_scan_again:
 


@@ -5318,60 +5833,14 @@ full_scan_again:
 #endif
        PAGE_REPLACEMENT_ALLOWED(TRUE);
 
 #endif
        PAGE_REPLACEMENT_ALLOWED(TRUE);
 

-       vm_page_lock_queues();
-
-#if __arm64__
-       if (preallocated_buffer_available) {
-
-               if ((contig_pages >= MIN_CONTIG_PAGES_REQUEST_FOR_PREALLOCATED_BUFFER) && (contig_pages <= PREALLOCATED_CONTIG_BUFFER_PAGES_COUNT)) {
-
-                       m = cpm_preallocated_pages_list;
-
-                       start_idx = (unsigned int) (m - &vm_pages[0]);
-
-                       if (wire == FALSE) {
-                       
-                               last_idx = start_idx;
-
-                               for(npages = 0; npages < contig_pages; npages++, last_idx++) {
-
-                                       assert(vm_pages[last_idx].vmp_gobbled == FALSE);
-
-                                       vm_pages[last_idx].vmp_gobbled = TRUE;
-                                       vm_page_gobble_count++;
-
-                                       assert(1 == vm_pages[last_idx].vmp_wire_count);
-                                       /*
-                                        * Gobbled pages are counted as wired pages. So no need to drop
-                                        * the global wired page count. Just the page's wire count is fine.
-                                        */
-                                       vm_pages[last_idx].vmp_wire_count--;
-                                       vm_pages[last_idx].vmp_q_state = VM_PAGE_NOT_ON_Q;
-                               }
-                       
-                       }
-
-                       last_idx = start_idx + contig_pages - 1;
-
-                       vm_pages[last_idx].vmp_snext = NULL;
-
-                       printf("Using preallocated buffer: Requested size (pages):%d... index range: %d-%d...freeing %llu pages\n", contig_pages, start_idx, last_idx, PREALLOCATED_CONTIG_BUFFER_PAGES_COUNT - contig_pages);
-
-                       last_idx += 1;
-                       for(npages = contig_pages; npages < PREALLOCATED_CONTIG_BUFFER_PAGES_COUNT; npages++, last_idx++) {
-
-                               VM_PAGE_ZERO_PAGEQ_ENTRY(&vm_pages[last_idx]);
-                               vm_page_free(&vm_pages[last_idx]);
-                       }
-
-                       cpm_preallocated_pages_list = NULL;
-                       preallocated_buffer_available = FALSE;
-
-                       goto done_scanning;
-               }
+       /*
+        * If there are still delayed pages, try to free up some that match.
+        */
+       if (__improbable(vm_delayed_count != 0 && contig_pages != 0)) {
+               vm_free_delayed_pages_contig(contig_pages, max_pnum, pnum_mask);

        }
        }

-#endif /* __arm64__ */

 
 

+       vm_page_lock_queues();

        lck_mtx_lock(&vm_page_queue_free_lock);
 
        RESET_STATE_OF_RUN();
        lck_mtx_lock(&vm_page_queue_free_lock);
 
        RESET_STATE_OF_RUN();


@@ -5381,18 +5850,19 @@ full_scan_again:
        free_available = vm_page_free_count - vm_page_free_reserved;
 
        wrapped = FALSE;
        free_available = vm_page_free_count - vm_page_free_reserved;
 
        wrapped = FALSE;

-       
-       if(flags & KMA_LOMEM) 
+
+       if (flags & KMA_LOMEM) {

                idx_last_contig_page_found = vm_page_lomem_find_contiguous_last_idx;
                idx_last_contig_page_found = vm_page_lomem_find_contiguous_last_idx;

-       else
+       } else {

                idx_last_contig_page_found =  vm_page_find_contiguous_last_idx;
                idx_last_contig_page_found =  vm_page_find_contiguous_last_idx;

+       }

 
        orig_last_idx = idx_last_contig_page_found;
        last_idx = orig_last_idx;
 
        for (page_idx = last_idx, start_idx = last_idx;
 
        orig_last_idx = idx_last_contig_page_found;
        last_idx = orig_last_idx;
 
        for (page_idx = last_idx, start_idx = last_idx;

-            npages < contig_pages && page_idx < vm_pages_count;
-            page_idx++) {
+           npages < contig_pages && page_idx < vm_pages_count;
+           page_idx++) {

 retry:
                if (wrapped &&
                    npages == 0 &&
 retry:
                if (wrapped &&
                    npages == 0 &&


@@ -5419,10 +5889,9 @@ retry:
                         * not aligned
                         */
                        RESET_STATE_OF_RUN();
                         * not aligned
                         */
                        RESET_STATE_OF_RUN();

-

                } else if (VM_PAGE_WIRED(m) || m->vmp_gobbled ||
                } else if (VM_PAGE_WIRED(m) || m->vmp_gobbled ||

-                          m->vmp_laundry || m->vmp_wanted ||
-                          m->vmp_cleaning || m->vmp_overwriting || m->vmp_free_when_done) {
+                   m->vmp_laundry || m->vmp_wanted ||
+                   m->vmp_cleaning || m->vmp_overwriting || m->vmp_free_when_done) {

                        /*
                         * page is in a transient state
                         * or a state we don't want to deal
                        /*
                         * page is in a transient state
                         * or a state we don't want to deal


@@ -5430,11 +5899,10 @@ retry:
                         * means starting a new run
                         */
                        RESET_STATE_OF_RUN();
                         * means starting a new run
                         */
                        RESET_STATE_OF_RUN();

-

                } else if ((m->vmp_q_state == VM_PAGE_NOT_ON_Q) ||
                } else if ((m->vmp_q_state == VM_PAGE_NOT_ON_Q) ||

-                          (m->vmp_q_state == VM_PAGE_ON_FREE_LOCAL_Q) ||
-                          (m->vmp_q_state == VM_PAGE_ON_FREE_LOPAGE_Q) ||
-                          (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q)) {
+                   (m->vmp_q_state == VM_PAGE_ON_FREE_LOCAL_Q) ||
+                   (m->vmp_q_state == VM_PAGE_ON_FREE_LOPAGE_Q) ||
+                   (m->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q)) {

                        /*
                         * page needs to be on one of our queues (other then the pageout or special free queues)
                         * or it needs to belong to the compressor pool (which is now indicated
                        /*
                         * page needs to be on one of our queues (other then the pageout or special free queues)
                         * or it needs to belong to the compressor pool (which is now indicated


@@ -5446,7 +5914,6 @@ retry:
                         * means starting a new run
                         */
                        RESET_STATE_OF_RUN();
                         * means starting a new run
                         */
                        RESET_STATE_OF_RUN();

-

                } else if ((m->vmp_q_state != VM_PAGE_ON_FREE_Q) && (!m->vmp_tabled || m->vmp_busy)) {
                        /*
                         * pages on the free list are always 'busy'
                } else if ((m->vmp_q_state != VM_PAGE_ON_FREE_Q) && (!m->vmp_tabled || m->vmp_busy)) {
                        /*
                         * pages on the free list are always 'busy'


@@ -5455,12 +5922,11 @@ retry:
                         * 'free' are never 'tabled', so we also couldn't
                         * test for 'tabled'.  So we check here to make
                         * sure that a non-free page is not busy and is
                         * 'free' are never 'tabled', so we also couldn't
                         * test for 'tabled'.  So we check here to make
                         * sure that a non-free page is not busy and is

-                        * tabled on an object... 
+                        * tabled on an object...

                         * if not, don't consider it which
                         * means starting a new run
                         */
                        RESET_STATE_OF_RUN();
                         * if not, don't consider it which
                         * means starting a new run
                         */
                        RESET_STATE_OF_RUN();

-

                } else {
                        if (VM_PAGE_GET_PHYS_PAGE(m) != prevcontaddr + 1) {
                                if ((VM_PAGE_GET_PHYS_PAGE(m) & pnum_mask) != 0) {
                } else {
                        if (VM_PAGE_GET_PHYS_PAGE(m) != prevcontaddr + 1) {
                                if ((VM_PAGE_GET_PHYS_PAGE(m) & pnum_mask) != 0) {


@@ -5475,7 +5941,7 @@ retry:
                                npages++;
                        }
                        prevcontaddr = VM_PAGE_GET_PHYS_PAGE(m);
                                npages++;
                        }
                        prevcontaddr = VM_PAGE_GET_PHYS_PAGE(m);

-                       
+

                        VM_PAGE_CHECK(m);
                        if (m->vmp_q_state == VM_PAGE_ON_FREE_Q) {
                                free_considered++;
                        VM_PAGE_CHECK(m);
                        if (m->vmp_q_state == VM_PAGE_ON_FREE_Q) {
                                free_considered++;


@@ -5498,7 +5964,7 @@ retry:
 #endif
                        }
 
 #endif
                        }
 

-                       if ((free_considered + substitute_needed) > free_available) {   
+                       if ((free_considered + substitute_needed) > free_available) {

                                /*
                                 * if we let this run continue
                                 * we will end up dropping the vm_page_free_count
                                /*
                                 * if we let this run continue
                                 * we will end up dropping the vm_page_free_count


@@ -5525,7 +5991,6 @@ retry:
                }
 did_consider:
                if (considered > MAX_CONSIDERED_BEFORE_YIELD && npages <= 1) {
                }
 did_consider:
                if (considered > MAX_CONSIDERED_BEFORE_YIELD && npages <= 1) {

-

                        PAGE_REPLACEMENT_ALLOWED(FALSE);
 
                        lck_mtx_unlock(&vm_page_queue_free_lock);
                        PAGE_REPLACEMENT_ALLOWED(FALSE);
 
                        lck_mtx_unlock(&vm_page_queue_free_lock);


@@ -5562,10 +6027,11 @@ did_consider:
                         * Start again from the very first page.
                         */
                        RESET_STATE_OF_RUN();
                         * Start again from the very first page.
                         */
                        RESET_STATE_OF_RUN();

-                       if( flags & KMA_LOMEM)
+                       if (flags & KMA_LOMEM) {

                                idx_last_contig_page_found  = vm_page_lomem_find_contiguous_last_idx = 0;
                                idx_last_contig_page_found  = vm_page_lomem_find_contiguous_last_idx = 0;

-                       else
+                       } else {

                                idx_last_contig_page_found = vm_page_find_contiguous_last_idx = 0;
                                idx_last_contig_page_found = vm_page_find_contiguous_last_idx = 0;

+                       }

                        last_idx = 0;
                        page_idx = last_idx;
                        wrapped = TRUE;
                        last_idx = 0;
                        page_idx = last_idx;
                        wrapped = TRUE;


@@ -5573,13 +6039,13 @@ did_consider:
                }
                lck_mtx_unlock(&vm_page_queue_free_lock);
        } else {
                }
                lck_mtx_unlock(&vm_page_queue_free_lock);
        } else {

-               vm_page_t       m1;
-               vm_page_t       m2;
-               unsigned int    cur_idx;
-               unsigned int    tmp_start_idx;
-               vm_object_t     locked_object = VM_OBJECT_NULL;
-               boolean_t       abort_run = FALSE;
-               
+               vm_page_t       m1;
+               vm_page_t       m2;
+               unsigned int    cur_idx;
+               unsigned int    tmp_start_idx;
+               vm_object_t     locked_object = VM_OBJECT_NULL;
+               boolean_t       abort_run = FALSE;
+

                assert(page_idx - start_idx == contig_pages);
 
                tmp_start_idx = start_idx;
                assert(page_idx - start_idx == contig_pages);
 
                tmp_start_idx = start_idx;


@@ -5587,14 +6053,13 @@ did_consider:
                /*
                 * first pass through to pull the free pages
                 * off of the free queue so that in case we
                /*
                 * first pass through to pull the free pages
                 * off of the free queue so that in case we

-                * need substitute pages, we won't grab any 
+                * need substitute pages, we won't grab any

                 * of the free pages in the run... we'll clear
                 * the 'free' bit in the 2nd pass, and even in
                 * an abort_run case, we'll collect all of the
                 * free pages in this run and return them to the free list
                 */
                while (start_idx < page_idx) {
                 * of the free pages in the run... we'll clear
                 * the 'free' bit in the 2nd pass, and even in
                 * an abort_run case, we'll collect all of the
                 * free pages in this run and return them to the free list
                 */
                while (start_idx < page_idx) {

-

                        m1 = &vm_pages[start_idx++];
 
 #if !VM_PAGE_FIND_CONTIGUOUS_CAN_STEAL
                        m1 = &vm_pages[start_idx++];
 
 #if !VM_PAGE_FIND_CONTIGUOUS_CAN_STEAL


@@ -5608,10 +6073,7 @@ did_consider:
 #if MACH_ASSERT
                                vm_page_verify_free_list(&vm_page_queue_free[color].qhead, color, m1, TRUE);
 #endif
 #if MACH_ASSERT
                                vm_page_verify_free_list(&vm_page_queue_free[color].qhead, color, m1, TRUE);
 #endif

-                               vm_page_queue_remove(&vm_page_queue_free[color].qhead,
-                                                    m1,
-                                                    vm_page_t,
-                                                    vmp_pageq);
+                               vm_page_queue_remove(&vm_page_queue_free[color].qhead, m1, vmp_pageq);

 
                                VM_PAGE_ZERO_PAGEQ_ENTRY(m1);
 #if MACH_ASSERT
 
                                VM_PAGE_ZERO_PAGEQ_ENTRY(m1);
 #if MACH_ASSERT


@@ -5628,11 +6090,12 @@ did_consider:
                                vm_page_free_count--;
                        }
                }
                                vm_page_free_count--;
                        }
                }

-               if( flags & KMA_LOMEM)
+               if (flags & KMA_LOMEM) {

                        vm_page_lomem_find_contiguous_last_idx = page_idx;
                        vm_page_lomem_find_contiguous_last_idx = page_idx;

-               else 
+               } else {

                        vm_page_find_contiguous_last_idx = page_idx;
                        vm_page_find_contiguous_last_idx = page_idx;

-               
+               }
+

                /*
                 * we can drop the free queue lock at this point since
                 * we've pulled any 'free' candidates off of the list
                /*
                 * we can drop the free queue lock at this point since
                 * we've pulled any 'free' candidates off of the list


@@ -5667,8 +6130,9 @@ did_consider:
                                int refmod;
                                boolean_t disconnected, reusable;
 
                                int refmod;
                                boolean_t disconnected, reusable;
 

-                               if (abort_run == TRUE)
+                               if (abort_run == TRUE) {

                                        continue;
                                        continue;

+                               }

 
                                assert(m1->vmp_q_state != VM_PAGE_NOT_ON_Q);
 
 
                                assert(m1->vmp_q_state != VM_PAGE_NOT_ON_Q);
 


@@ -5679,15 +6143,15 @@ did_consider:
                                                vm_object_unlock(locked_object);
                                                locked_object = VM_OBJECT_NULL;
                                        }
                                                vm_object_unlock(locked_object);
                                                locked_object = VM_OBJECT_NULL;
                                        }

-                                       if (vm_object_lock_try(object))
+                                       if (vm_object_lock_try(object)) {

                                                locked_object = object;
                                                locked_object = object;

+                                       }

                                }
                                }

-                               if (locked_object == VM_OBJECT_NULL || 
+                               if (locked_object == VM_OBJECT_NULL ||

                                    (VM_PAGE_WIRED(m1) || m1->vmp_gobbled ||
                                    (VM_PAGE_WIRED(m1) || m1->vmp_gobbled ||

-                                    m1->vmp_laundry || m1->vmp_wanted ||
-                                    m1->vmp_cleaning || m1->vmp_overwriting || m1->vmp_free_when_done || m1->vmp_busy) ||
+                                   m1->vmp_laundry || m1->vmp_wanted ||
+                                   m1->vmp_cleaning || m1->vmp_overwriting || m1->vmp_free_when_done || m1->vmp_busy) ||

                                    (m1->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q)) {
                                    (m1->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q)) {

-

                                        if (locked_object) {
                                                vm_object_unlock(locked_object);
                                                locked_object = VM_OBJECT_NULL;
                                        if (locked_object) {
                                                vm_object_unlock(locked_object);
                                                locked_object = VM_OBJECT_NULL;


@@ -5701,7 +6165,7 @@ did_consider:
                                reusable = FALSE;
 
                                if ((m1->vmp_reusable ||
                                reusable = FALSE;
 
                                if ((m1->vmp_reusable ||

-                                    object->all_reusable) &&
+                                   object->all_reusable) &&

                                    (m1->vmp_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q) &&
                                    !m1->vmp_dirty &&
                                    !m1->vmp_reference) {
                                    (m1->vmp_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q) &&
                                    !m1->vmp_dirty &&
                                    !m1->vmp_reference) {


@@ -5718,12 +6182,12 @@ did_consider:
                                }
 
                                if ((m1->vmp_pmapped &&
                                }
 
                                if ((m1->vmp_pmapped &&

-                                    ! reusable) ||
+                                   !reusable) ||

                                    m1->vmp_dirty ||
                                    m1->vmp_precious) {
                                        vm_object_offset_t offset;
 
                                    m1->vmp_dirty ||
                                    m1->vmp_precious) {
                                        vm_object_offset_t offset;
 

-                                       m2 = vm_page_grab();
+                                       m2 = vm_page_grab_options(VM_PAGE_GRAB_Q_LOCK_HELD);

 
                                        if (m2 == VM_PAGE_NULL) {
                                                if (locked_object) {
 
                                        if (m2 == VM_PAGE_NULL) {
                                                if (locked_object) {


@@ -5734,11 +6198,12 @@ did_consider:
                                                abort_run = TRUE;
                                                continue;
                                        }
                                                abort_run = TRUE;
                                                continue;
                                        }

-                                       if (! disconnected) {
-                                               if (m1->vmp_pmapped)
+                                       if (!disconnected) {
+                                               if (m1->vmp_pmapped) {

                                                        refmod = pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m1));
                                                        refmod = pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m1));

-                                               else
+                                               } else {

                                                        refmod = 0;
                                                        refmod = 0;

+                                               }

                                        }
 
                                        /* copy the page's contents */
                                        }
 
                                        /* copy the page's contents */


@@ -5748,29 +6213,29 @@ did_consider:
                                        assert(m1->vmp_q_state != VM_PAGE_ON_FREE_Q);
                                        assert(m1->vmp_q_state != VM_PAGE_ON_PAGEOUT_Q);
                                        assert(!m1->vmp_laundry);
                                        assert(m1->vmp_q_state != VM_PAGE_ON_FREE_Q);
                                        assert(m1->vmp_q_state != VM_PAGE_ON_PAGEOUT_Q);
                                        assert(!m1->vmp_laundry);

-                                       m2->vmp_reference       = m1->vmp_reference;
+                                       m2->vmp_reference       = m1->vmp_reference;

                                        assert(!m1->vmp_gobbled);
                                        assert(!m1->vmp_private);
                                        assert(!m1->vmp_gobbled);
                                        assert(!m1->vmp_private);

-                                       m2->vmp_no_cache        = m1->vmp_no_cache;
-                                       m2->vmp_xpmapped        = 0;
+                                       m2->vmp_no_cache        = m1->vmp_no_cache;
+                                       m2->vmp_xpmapped        = 0;

                                        assert(!m1->vmp_busy);
                                        assert(!m1->vmp_wanted);
                                        assert(!m1->vmp_fictitious);
                                        assert(!m1->vmp_busy);
                                        assert(!m1->vmp_wanted);
                                        assert(!m1->vmp_fictitious);

-                                       m2->vmp_pmapped = m1->vmp_pmapped; /* should flush cache ? */
-                                       m2->vmp_wpmapped        = m1->vmp_wpmapped;
+                                       m2->vmp_pmapped = m1->vmp_pmapped; /* should flush cache ? */
+                                       m2->vmp_wpmapped        = m1->vmp_wpmapped;

                                        assert(!m1->vmp_free_when_done);
                                        assert(!m1->vmp_free_when_done);

-                                       m2->vmp_absent  = m1->vmp_absent;
-                                       m2->vmp_error   = m1->vmp_error;
-                                       m2->vmp_dirty   = m1->vmp_dirty;
+                                       m2->vmp_absent  = m1->vmp_absent;
+                                       m2->vmp_error   = m1->vmp_error;
+                                       m2->vmp_dirty   = m1->vmp_dirty;

                                        assert(!m1->vmp_cleaning);
                                        assert(!m1->vmp_cleaning);

-                                       m2->vmp_precious        = m1->vmp_precious;
-                                       m2->vmp_clustered       = m1->vmp_clustered;
+                                       m2->vmp_precious        = m1->vmp_precious;
+                                       m2->vmp_clustered       = m1->vmp_clustered;

                                        assert(!m1->vmp_overwriting);
                                        assert(!m1->vmp_overwriting);

-                                       m2->vmp_restart = m1->vmp_restart;
-                                       m2->vmp_unusual = m1->vmp_unusual;
+                                       m2->vmp_restart = m1->vmp_restart;
+                                       m2->vmp_unusual = m1->vmp_unusual;

                                        m2->vmp_cs_validated = m1->vmp_cs_validated;
                                        m2->vmp_cs_validated = m1->vmp_cs_validated;

-                                       m2->vmp_cs_tainted      = m1->vmp_cs_tainted;
-                                       m2->vmp_cs_nx   = m1->vmp_cs_nx;
+                                       m2->vmp_cs_tainted      = m1->vmp_cs_tainted;
+                                       m2->vmp_cs_nx   = m1->vmp_cs_nx;

 
                                        /*
                                         * If m1 had really been reusable,
 
                                        /*
                                         * If m1 had really been reusable,


@@ -5784,8 +6249,9 @@ did_consider:
 
                                        // assert(!m1->vmp_lopage);
 
 
                                        // assert(!m1->vmp_lopage);
 

-                                       if (m1->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR)
+                                       if (m1->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) {

                                                m2->vmp_q_state = VM_PAGE_USED_BY_COMPRESSOR;
                                                m2->vmp_q_state = VM_PAGE_USED_BY_COMPRESSOR;

+                                       }

 
                                        /*
                                         * page may need to be flushed if
 
                                        /*
                                         * page may need to be flushed if


@@ -5803,8 +6269,9 @@ did_consider:
                                         */
                                        pmap_clear_refmod(VM_PAGE_GET_PHYS_PAGE(m2), VM_MEM_MODIFIED | VM_MEM_REFERENCED);
 
                                         */
                                        pmap_clear_refmod(VM_PAGE_GET_PHYS_PAGE(m2), VM_MEM_MODIFIED | VM_MEM_REFERENCED);
 

-                                       if (refmod & VM_MEM_REFERENCED)
+                                       if (refmod & VM_MEM_REFERENCED) {

                                                m2->vmp_reference = TRUE;
                                                m2->vmp_reference = TRUE;

+                                       }

                                        if (refmod & VM_MEM_MODIFIED) {
                                                SET_PAGE_DIRTY(m2, TRUE);
                                        }
                                        if (refmod & VM_MEM_MODIFIED) {
                                                SET_PAGE_DIRTY(m2, TRUE);
                                        }


@@ -5829,21 +6296,20 @@ did_consider:
                                                m2->vmp_pmapped = TRUE;
                                                m2->vmp_wpmapped = TRUE;
 
                                                m2->vmp_pmapped = TRUE;
                                                m2->vmp_wpmapped = TRUE;
 

-                                               PMAP_ENTER(kernel_pmap, m2->vmp_offset, m2,
-                                                          VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE, 0, TRUE, kr);
+                                               PMAP_ENTER(kernel_pmap, (vm_map_offset_t)m2->vmp_offset, m2,
+                                                   VM_PROT_READ | VM_PROT_WRITE, VM_PROT_NONE, 0, TRUE, kr);

 
                                                assert(kr == KERN_SUCCESS);
 
                                                compressed_pages++;
 
                                                assert(kr == KERN_SUCCESS);
 
                                                compressed_pages++;

-

                                        } else {
                                        } else {

-                                               if (m2->vmp_reference)
+                                               if (m2->vmp_reference) {

                                                        vm_page_activate(m2);
                                                        vm_page_activate(m2);

-                                               else
+                                               } else {

                                                        vm_page_deactivate(m2);
                                                        vm_page_deactivate(m2);

+                                               }

                                        }
                                        PAGE_WAKEUP_DONE(m2);
                                        }
                                        PAGE_WAKEUP_DONE(m2);

-

                                } else {
                                        assert(m1->vmp_q_state != VM_PAGE_USED_BY_COMPRESSOR);
 
                                } else {
                                        assert(m1->vmp_q_state != VM_PAGE_USED_BY_COMPRESSOR);
 


@@ -5858,7 +6324,6 @@ did_consider:
                                }
 
                                stolen_pages++;
                                }
 
                                stolen_pages++;

-

                        }
 #if CONFIG_BACKGROUND_QUEUE
                        vm_page_assign_background_state(m1);
                        }
 #if CONFIG_BACKGROUND_QUEUE
                        vm_page_assign_background_state(m1);


@@ -5896,21 +6361,22 @@ did_consider:
                                wrapped = TRUE;
                        }
                        abort_run = FALSE;
                                wrapped = TRUE;
                        }
                        abort_run = FALSE;

-               
+

                        /*
                         * We didn't find a contiguous range but we didn't
                         * start from the very first page.
                         * Start again from the very first page.
                         */
                        RESET_STATE_OF_RUN();
                        /*
                         * We didn't find a contiguous range but we didn't
                         * start from the very first page.
                         * Start again from the very first page.
                         */
                        RESET_STATE_OF_RUN();

-                       
-                       if( flags & KMA_LOMEM)
+
+                       if (flags & KMA_LOMEM) {

                                idx_last_contig_page_found  = vm_page_lomem_find_contiguous_last_idx = page_idx;
                                idx_last_contig_page_found  = vm_page_lomem_find_contiguous_last_idx = page_idx;

-                       else
+                       } else {

                                idx_last_contig_page_found = vm_page_find_contiguous_last_idx = page_idx;
                                idx_last_contig_page_found = vm_page_find_contiguous_last_idx = page_idx;

-                       
+                       }
+

                        last_idx = page_idx;
                        last_idx = page_idx;

-                       
+

                        if (m != VM_PAGE_NULL) {
                                vm_page_unlock_queues();
                                vm_page_free_list(m, FALSE);
                        if (m != VM_PAGE_NULL) {
                                vm_page_unlock_queues();
                                vm_page_free_list(m, FALSE);


@@ -5921,33 +6387,34 @@ did_consider:
 
                        lck_mtx_lock(&vm_page_queue_free_lock);
                        /*
 
                        lck_mtx_lock(&vm_page_queue_free_lock);
                        /*

-                       * reset our free page limit since we
-                       * dropped the lock protecting the vm_page_free_queue
-                       */
+                        * reset our free page limit since we
+                        * dropped the lock protecting the vm_page_free_queue
+                        */

                        free_available = vm_page_free_count - vm_page_free_reserved;
                        goto retry;
                }
 
                for (m1 = m; m1 != VM_PAGE_NULL; m1 = NEXT_PAGE(m1)) {
                        free_available = vm_page_free_count - vm_page_free_reserved;
                        goto retry;
                }
 
                for (m1 = m; m1 != VM_PAGE_NULL; m1 = NEXT_PAGE(m1)) {

-

                        assert(m1->vmp_q_state == VM_PAGE_NOT_ON_Q);
                        assert(m1->vmp_wire_count == 0);
 
                        if (wire == TRUE) {
                                m1->vmp_wire_count++;
                                m1->vmp_q_state = VM_PAGE_IS_WIRED;
                        assert(m1->vmp_q_state == VM_PAGE_NOT_ON_Q);
                        assert(m1->vmp_wire_count == 0);
 
                        if (wire == TRUE) {
                                m1->vmp_wire_count++;
                                m1->vmp_q_state = VM_PAGE_IS_WIRED;

-                       } else
+                       } else {

                                m1->vmp_gobbled = TRUE;
                                m1->vmp_gobbled = TRUE;

+                       }

                }
                }

-               if (wire == FALSE)
+               if (wire == FALSE) {

                        vm_page_gobble_count += npages;
                        vm_page_gobble_count += npages;

+               }

 
                /*
                 * gobbled pages are also counted as wired pages
                 */
                vm_page_wire_count += npages;
 
 
                /*
                 * gobbled pages are also counted as wired pages
                 */
                vm_page_wire_count += npages;
 

-               assert(vm_page_verify_contiguous(m, npages));
+               assert(vm_page_verify_contiguous(m, npages));

        }
 done_scanning:
        PAGE_REPLACEMENT_ALLOWED(FALSE);
        }
 done_scanning:
        PAGE_REPLACEMENT_ALLOWED(FALSE);


@@ -5969,9 +6436,9 @@ done_scanning:
        }
        if (vm_page_find_contig_debug) {
                printf("%s(num=%d,low=%d): found %d pages at 0x%llx in %ld.%06ds...  started at %d...  scanned %d pages...  yielded %d times...  dumped run %d times... stole %d pages... stole %d compressed pages\n",
        }
        if (vm_page_find_contig_debug) {
                printf("%s(num=%d,low=%d): found %d pages at 0x%llx in %ld.%06ds...  started at %d...  scanned %d pages...  yielded %d times...  dumped run %d times... stole %d pages... stole %d compressed pages\n",

-                      __func__, contig_pages, max_pnum, npages, (vm_object_offset_t)start_pnum << PAGE_SHIFT,
-                      (long)tv_end_sec, tv_end_usec, orig_last_idx,
-                      scanned, yielded, dumped_run, stolen_pages, compressed_pages);
+                   __func__, contig_pages, max_pnum, npages, (vm_object_offset_t)start_pnum << PAGE_SHIFT,
+                   (long)tv_end_sec, tv_end_usec, orig_last_idx,
+                       scanned, yielded, dumped_run, stolen_pages, compressed_pages);

        }
 
 #endif
        }
 
 #endif


@@ -5980,8 +6447,8 @@ done_scanning:
 #endif
        if (m == NULL && zone_gc_called == FALSE) {
                printf("%s(num=%d,low=%d): found %d pages at 0x%llx...scanned %d pages...  yielded %d times...  dumped run %d times... stole %d pages... stole %d compressed pages... wired count is %d\n",
 #endif
        if (m == NULL && zone_gc_called == FALSE) {
                printf("%s(num=%d,low=%d): found %d pages at 0x%llx...scanned %d pages...  yielded %d times...  dumped run %d times... stole %d pages... stole %d compressed pages... wired count is %d\n",

-                      __func__, contig_pages, max_pnum, npages, (vm_object_offset_t)start_pnum << PAGE_SHIFT,
-                      scanned, yielded, dumped_run, stolen_pages, compressed_pages, vm_page_wire_count);
+                   __func__, contig_pages, max_pnum, npages, (vm_object_offset_t)start_pnum << PAGE_SHIFT,
+                       scanned, yielded, dumped_run, stolen_pages, compressed_pages, vm_page_wire_count);

 
                if (consider_buffer_cache_collect != NULL) {
                        (void)(*consider_buffer_cache_collect)(1);
 
                if (consider_buffer_cache_collect != NULL) {
                        (void)(*consider_buffer_cache_collect)(1);


@@ -6003,18 +6470,19 @@ done_scanning:
  */
 kern_return_t
 cpm_allocate(
  */
 kern_return_t
 cpm_allocate(

-       vm_size_t       size,
-       vm_page_t       *list,
-       ppnum_t         max_pnum,
-       ppnum_t         pnum_mask,
-       boolean_t       wire,
-       int             flags)
+       vm_size_t       size,
+       vm_page_t       *list,
+       ppnum_t         max_pnum,
+       ppnum_t         pnum_mask,
+       boolean_t       wire,
+       int             flags)

 {
 {

-       vm_page_t               pages;
-       unsigned int            npages;
+       vm_page_t               pages;
+       unsigned int            npages;

 
 

-       if (size % PAGE_SIZE != 0)
+       if (size % PAGE_SIZE != 0) {

                return KERN_INVALID_ARGUMENT;
                return KERN_INVALID_ARGUMENT;

+       }

 
        npages = (unsigned int) (size / PAGE_SIZE);
        if (npages != size / PAGE_SIZE) {
 
        npages = (unsigned int) (size / PAGE_SIZE);
        if (npages != size / PAGE_SIZE) {


@@ -6029,16 +6497,24 @@ cpm_allocate(
         */
        pages = vm_page_find_contiguous(npages, max_pnum, pnum_mask, wire, flags);
 
         */
        pages = vm_page_find_contiguous(npages, max_pnum, pnum_mask, wire, flags);
 

-       if (pages == VM_PAGE_NULL)
+       if (pages == VM_PAGE_NULL) {

                return KERN_NO_SPACE;
                return KERN_NO_SPACE;

+       }

        /*
         * determine need for wakeups
         */
        /*
         * determine need for wakeups
         */

-       if (vm_page_free_count < vm_page_free_min)
-               thread_wakeup((event_t) &vm_page_free_wanted);
-               
+       if (vm_page_free_count < vm_page_free_min) {
+               lck_mtx_lock(&vm_page_queue_free_lock);
+               if (vm_pageout_running == FALSE) {
+                       lck_mtx_unlock(&vm_page_queue_free_lock);
+                       thread_wakeup((event_t) &vm_page_free_wanted);
+               } else {
+                       lck_mtx_unlock(&vm_page_queue_free_lock);
+               }
+       }
+

        VM_CHECK_MEMORYSTATUS;
        VM_CHECK_MEMORYSTATUS;

-       
+

        /*
         *      The CPM pages should now be available and
         *      ordered by ascending physical address.
        /*
         *      The CPM pages should now be available and
         *      ordered by ascending physical address.


@@ -6053,7 +6529,7 @@ cpm_allocate(
 unsigned int vm_max_delayed_work_limit = DEFAULT_DELAYED_WORK_LIMIT;
 
 /*
 unsigned int vm_max_delayed_work_limit = DEFAULT_DELAYED_WORK_LIMIT;
 
 /*

- * when working on a 'run' of pages, it is necessary to hold 
+ * when working on a 'run' of pages, it is necessary to hold

  * the vm_page_queue_lock (a hot global lock) for certain operations
  * on the page... however, the majority of the work can be done
  * while merely holding the object lock... in fact there are certain
  * the vm_page_queue_lock (a hot global lock) for certain operations
  * on the page... however, the majority of the work can be done
  * while merely holding the object lock... in fact there are certain


@@ -6073,14 +6549,14 @@ unsigned int vm_max_delayed_work_limit = DEFAULT_DELAYED_WORK_LIMIT;
 
 void
 vm_page_do_delayed_work(
 
 void
 vm_page_do_delayed_work(

-       vm_object_t     object,
+       vm_object_t     object,

        vm_tag_t        tag,
        struct vm_page_delayed_work *dwp,
        vm_tag_t        tag,
        struct vm_page_delayed_work *dwp,

-       int             dw_count)
+       int             dw_count)

 {
 {

-       int             j;
-       vm_page_t       m;
-        vm_page_t       local_free_q = VM_PAGE_NULL;
+       int             j;
+       vm_page_t       m;
+       vm_page_t       local_free_q = VM_PAGE_NULL;

 
        /*
         * pageout_scan takes the vm_page_lock_queues first
 
        /*
         * pageout_scan takes the vm_page_lock_queues first


@@ -6098,31 +6574,45 @@ vm_page_do_delayed_work(
        if (!vm_page_trylockspin_queues()) {
                vm_object_unlock(object);
 
        if (!vm_page_trylockspin_queues()) {
                vm_object_unlock(object);
 

+               /*
+                * "Turnstile enabled vm_pageout_scan" can be runnable
+                * for a very long time without getting on a core.
+                * If this is a higher priority thread it could be
+                * waiting here for a very long time respecting the fact
+                * that pageout_scan would like its object after VPS does
+                * a mutex_pause(0).
+                * So we cap the number of yields in the vm_object_lock_avoid()
+                * case to a single mutex_pause(0) which will give vm_pageout_scan
+                * 10us to run and grab the object if needed.
+                */

                vm_page_lockspin_queues();
 
                vm_page_lockspin_queues();
 

-               for (j = 0; ; j++) {
-                       if (!vm_object_lock_avoid(object) &&
-                           _vm_object_lock_try(object))
+               for (j = 0;; j++) {
+                       if ((!vm_object_lock_avoid(object) ||
+                           (vps_dynamic_priority_enabled && (j > 0))) &&
+                           _vm_object_lock_try(object)) {

                                break;
                                break;

+                       }

                        vm_page_unlock_queues();
                        mutex_pause(j);
                        vm_page_lockspin_queues();
                }
        }
        for (j = 0; j < dw_count; j++, dwp++) {
                        vm_page_unlock_queues();
                        mutex_pause(j);
                        vm_page_lockspin_queues();
                }
        }
        for (j = 0; j < dw_count; j++, dwp++) {

-

                m = dwp->dw_m;
 
                m = dwp->dw_m;
 

-               if (dwp->dw_mask & DW_vm_pageout_throttle_up)
+               if (dwp->dw_mask & DW_vm_pageout_throttle_up) {

                        vm_pageout_throttle_up(m);
                        vm_pageout_throttle_up(m);

+               }

 #if CONFIG_PHANTOM_CACHE
 #if CONFIG_PHANTOM_CACHE

-               if (dwp->dw_mask & DW_vm_phantom_cache_update)
+               if (dwp->dw_mask & DW_vm_phantom_cache_update) {

                        vm_phantom_cache_update(m);
                        vm_phantom_cache_update(m);

+               }

 #endif
 #endif

-               if (dwp->dw_mask & DW_vm_page_wire)
+               if (dwp->dw_mask & DW_vm_page_wire) {

                        vm_page_wire(m, tag, FALSE);
                        vm_page_wire(m, tag, FALSE);

-               else if (dwp->dw_mask & DW_vm_page_unwire) {
-                       boolean_t       queueit;
+               } else if (dwp->dw_mask & DW_vm_page_unwire) {
+                       boolean_t       queueit;

 
                        queueit = (dwp->dw_mask & (DW_vm_page_free | DW_vm_page_deactivate_internal)) ? FALSE : TRUE;
 
 
                        queueit = (dwp->dw_mask & (DW_vm_page_free | DW_vm_page_deactivate_internal)) ? FALSE : TRUE;
 


@@ -6139,16 +6629,15 @@ vm_page_do_delayed_work(
                        m->vmp_snext = local_free_q;
                        local_free_q = m;
                } else {
                        m->vmp_snext = local_free_q;
                        local_free_q = m;
                } else {

-                       if (dwp->dw_mask & DW_vm_page_deactivate_internal)
+                       if (dwp->dw_mask & DW_vm_page_deactivate_internal) {

                                vm_page_deactivate_internal(m, FALSE);
                                vm_page_deactivate_internal(m, FALSE);

-                       else if (dwp->dw_mask & DW_vm_page_activate) {
+                       } else if (dwp->dw_mask & DW_vm_page_activate) {

                                if (m->vmp_q_state != VM_PAGE_ON_ACTIVE_Q) {
                                        vm_page_activate(m);
                                }
                                if (m->vmp_q_state != VM_PAGE_ON_ACTIVE_Q) {
                                        vm_page_activate(m);
                                }

-                       }
-                       else if (dwp->dw_mask & DW_vm_page_speculate)
+                       } else if (dwp->dw_mask & DW_vm_page_speculate) {

                                vm_page_speculate(m, TRUE);
                                vm_page_speculate(m, TRUE);

-                       else if (dwp->dw_mask & DW_enqueue_cleaned) {
+                       } else if (dwp->dw_mask & DW_enqueue_cleaned) {

                                /*
                                 * if we didn't hold the object lock and did this,
                                 * we might disconnect the page, then someone might
                                /*
                                 * if we didn't hold the object lock and did this,
                                 * we might disconnect the page, then someone might


@@ -6163,26 +6652,28 @@ vm_page_do_delayed_work(
                                         * this page has been touched since it got cleaned; let's activate it
                                         * if it hasn't already been
                                         */
                                         * this page has been touched since it got cleaned; let's activate it
                                         * if it hasn't already been
                                         */

-                                       VM_PAGEOUT_DEBUG(vm_pageout_enqueued_cleaned, 1);
+                                       VM_PAGEOUT_DEBUG(vm_pageout_enqueued_cleaned, 1);

                                        VM_PAGEOUT_DEBUG(vm_pageout_cleaned_reactivated, 1);
 
                                        VM_PAGEOUT_DEBUG(vm_pageout_cleaned_reactivated, 1);
 

-                                       if (m->vmp_q_state != VM_PAGE_ON_ACTIVE_Q)
+                                       if (m->vmp_q_state != VM_PAGE_ON_ACTIVE_Q) {

                                                vm_page_activate(m);
                                                vm_page_activate(m);

+                                       }

                                } else {
                                        m->vmp_reference = FALSE;
                                        vm_page_enqueue_cleaned(m);
                                }
                                } else {
                                        m->vmp_reference = FALSE;
                                        vm_page_enqueue_cleaned(m);
                                }

-                       }
-                       else if (dwp->dw_mask & DW_vm_page_lru)
+                       } else if (dwp->dw_mask & DW_vm_page_lru) {

                                vm_page_lru(m);
                                vm_page_lru(m);

-                       else if (dwp->dw_mask & DW_VM_PAGE_QUEUES_REMOVE) {
-                               if (m->vmp_q_state != VM_PAGE_ON_PAGEOUT_Q)
+                       } else if (dwp->dw_mask & DW_VM_PAGE_QUEUES_REMOVE) {
+                               if (m->vmp_q_state != VM_PAGE_ON_PAGEOUT_Q) {

                                        vm_page_queues_remove(m, TRUE);
                                        vm_page_queues_remove(m, TRUE);

+                               }

                        }
                        }

-                       if (dwp->dw_mask & DW_set_reference)
+                       if (dwp->dw_mask & DW_set_reference) {

                                m->vmp_reference = TRUE;
                                m->vmp_reference = TRUE;

-                       else if (dwp->dw_mask & DW_clear_reference)
+                       } else if (dwp->dw_mask & DW_clear_reference) {

                                m->vmp_reference = FALSE;
                                m->vmp_reference = FALSE;

+                       }

 
                        if (dwp->dw_mask & DW_move_page) {
                                if (m->vmp_q_state != VM_PAGE_ON_PAGEOUT_Q) {
 
                        if (dwp->dw_mask & DW_move_page) {
                                if (m->vmp_q_state != VM_PAGE_ON_PAGEOUT_Q) {


@@ -6193,53 +6684,56 @@ vm_page_do_delayed_work(
                                        vm_page_enqueue_inactive(m, FALSE);
                                }
                        }
                                        vm_page_enqueue_inactive(m, FALSE);
                                }
                        }

-                       if (dwp->dw_mask & DW_clear_busy)
+                       if (dwp->dw_mask & DW_clear_busy) {

                                m->vmp_busy = FALSE;
                                m->vmp_busy = FALSE;

+                       }

 
 

-                       if (dwp->dw_mask & DW_PAGE_WAKEUP)
+                       if (dwp->dw_mask & DW_PAGE_WAKEUP) {

                                PAGE_WAKEUP(m);
                                PAGE_WAKEUP(m);

+                       }

                }
        }
        vm_page_unlock_queues();
 
                }
        }
        vm_page_unlock_queues();
 

-       if (local_free_q)
+       if (local_free_q) {

                vm_page_free_list(local_free_q, TRUE);
                vm_page_free_list(local_free_q, TRUE);

-       
-       VM_CHECK_MEMORYSTATUS;
+       }

 
 

+       VM_CHECK_MEMORYSTATUS;

 }
 
 kern_return_t
 vm_page_alloc_list(
 }
 
 kern_return_t
 vm_page_alloc_list(

-       int     page_count,
-       int     flags,
+       int     page_count,
+       int     flags,

        vm_page_t *list)
 {
        vm_page_t *list)
 {

-       vm_page_t       lo_page_list = VM_PAGE_NULL;
-       vm_page_t       mem;
-       int             i;
+       vm_page_t       lo_page_list = VM_PAGE_NULL;
+       vm_page_t       mem;
+       int             i;

 
 

-       if ( !(flags & KMA_LOMEM))
+       if (!(flags & KMA_LOMEM)) {

                panic("vm_page_alloc_list: called w/o KMA_LOMEM");
                panic("vm_page_alloc_list: called w/o KMA_LOMEM");

+       }

 
        for (i = 0; i < page_count; i++) {
 
        for (i = 0; i < page_count; i++) {

-

                mem = vm_page_grablo();
 
                if (mem == VM_PAGE_NULL) {
                mem = vm_page_grablo();
 
                if (mem == VM_PAGE_NULL) {

-                       if (lo_page_list)
+                       if (lo_page_list) {

                                vm_page_free_list(lo_page_list, FALSE);
                                vm_page_free_list(lo_page_list, FALSE);

+                       }

 
                        *list = VM_PAGE_NULL;
 
 
                        *list = VM_PAGE_NULL;
 

-                       return (KERN_RESOURCE_SHORTAGE);
+                       return KERN_RESOURCE_SHORTAGE;

                }
                mem->vmp_snext = lo_page_list;
                lo_page_list = mem;
        }
        *list = lo_page_list;
 
                }
                mem->vmp_snext = lo_page_list;
                lo_page_list = mem;
        }
        *list = lo_page_list;
 

-       return (KERN_SUCCESS);
+       return KERN_SUCCESS;

 }
 
 void
 }
 
 void


@@ -6251,22 +6745,22 @@ vm_page_set_offset(vm_page_t page, vm_object_offset_t offset)
 vm_page_t
 vm_page_get_next(vm_page_t page)
 {
 vm_page_t
 vm_page_get_next(vm_page_t page)
 {

-       return (page->vmp_snext);
+       return page->vmp_snext;

 }
 
 vm_object_offset_t
 vm_page_get_offset(vm_page_t page)
 {
 }
 
 vm_object_offset_t
 vm_page_get_offset(vm_page_t page)
 {

-       return (page->vmp_offset);
+       return page->vmp_offset;

 }
 
 ppnum_t
 vm_page_get_phys_page(vm_page_t page)
 {
 }
 
 ppnum_t
 vm_page_get_phys_page(vm_page_t page)
 {

-       return (VM_PAGE_GET_PHYS_PAGE(page));
+       return VM_PAGE_GET_PHYS_PAGE(page);

 }
 }

-       
-       
+
+

 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 #if HIBERNATION
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 #if HIBERNATION


@@ -6281,12 +6775,11 @@ void hibernate_flush_wait(void);
 void hibernate_mark_in_progress(void);
 void hibernate_clear_in_progress(void);
 
 void hibernate_mark_in_progress(void);
 void hibernate_clear_in_progress(void);
 

-void           hibernate_free_range(int, int);
-void           hibernate_hash_insert_page(vm_page_t);
-uint32_t       hibernate_mark_as_unneeded(addr64_t, addr64_t, hibernate_page_list_t *, hibernate_page_list_t *);
-void           hibernate_rebuild_vm_structs(void);
-uint32_t       hibernate_teardown_vm_structs(hibernate_page_list_t *, hibernate_page_list_t *);
-ppnum_t                hibernate_lookup_paddr(unsigned int);
+void            hibernate_free_range(int, int);
+void            hibernate_hash_insert_page(vm_page_t);
+uint32_t        hibernate_mark_as_unneeded(addr64_t, addr64_t, hibernate_page_list_t *, hibernate_page_list_t *);
+uint32_t        hibernate_teardown_vm_structs(hibernate_page_list_t *, hibernate_page_list_t *);
+ppnum_t         hibernate_lookup_paddr(unsigned int);

 
 struct hibernate_statistics {
        int hibernate_considered;
 
 struct hibernate_statistics {
        int hibernate_considered;


@@ -6327,21 +6820,20 @@ struct hibernate_statistics {
  * so that we don't overrun the estimated image size, which would
  * result in a hibernation failure.
  */
  * so that we don't overrun the estimated image size, which would
  * result in a hibernation failure.
  */

-#define        HIBERNATE_XPMAPPED_LIMIT        40000
+#define HIBERNATE_XPMAPPED_LIMIT        40000

 
 
 static int
 hibernate_drain_pageout_queue(struct vm_pageout_queue *q)
 {
 
 
 static int
 hibernate_drain_pageout_queue(struct vm_pageout_queue *q)
 {

-       wait_result_t   wait_result;
+       wait_result_t   wait_result;

 
        vm_page_lock_queues();
 
 
        vm_page_lock_queues();
 

-       while ( !vm_page_queue_empty(&q->pgo_pending) ) {
-
+       while (!vm_page_queue_empty(&q->pgo_pending)) {

                q->pgo_draining = TRUE;
 
                q->pgo_draining = TRUE;
 

-               assert_wait_timeout((event_t) (&q->pgo_laundry+1), THREAD_INTERRUPTIBLE, 5000, 1000*NSEC_PER_USEC);
+               assert_wait_timeout((event_t) (&q->pgo_laundry + 1), THREAD_INTERRUPTIBLE, 5000, 1000 * NSEC_PER_USEC);

 
                vm_page_unlock_queues();
 
 
                vm_page_unlock_queues();
 


@@ -6349,11 +6841,12 @@ hibernate_drain_pageout_queue(struct vm_pageout_queue *q)
 
                if (wait_result == THREAD_TIMED_OUT && !vm_page_queue_empty(&q->pgo_pending)) {
                        hibernate_stats.hibernate_drain_timeout++;
 
                if (wait_result == THREAD_TIMED_OUT && !vm_page_queue_empty(&q->pgo_pending)) {
                        hibernate_stats.hibernate_drain_timeout++;

-                       
-                       if (q == &vm_pageout_queue_external)
-                               return (0);
-                       
-                       return (1);
+
+                       if (q == &vm_pageout_queue_external) {
+                               return 0;
+                       }
+
+                       return 1;

                }
                vm_page_lock_queues();
 
                }
                vm_page_lock_queues();
 


@@ -6361,7 +6854,7 @@ hibernate_drain_pageout_queue(struct vm_pageout_queue *q)
        }
        vm_page_unlock_queues();
 
        }
        vm_page_unlock_queues();
 

-       return (0);
+       return 0;

 }
 
 
 }
 
 


@@ -6370,19 +6863,19 @@ boolean_t hibernate_skip_external = FALSE;
 static int
 hibernate_flush_queue(vm_page_queue_head_t *q, int qcount)
 {
 static int
 hibernate_flush_queue(vm_page_queue_head_t *q, int qcount)
 {

-       vm_page_t       m;
-       vm_object_t     l_object = NULL;
-       vm_object_t     m_object = NULL;
-       int             refmod_state = 0;
-       int             try_failed_count = 0;
-       int             retval = 0;
-       int             current_run = 0;
-       struct  vm_pageout_queue *iq;
-       struct  vm_pageout_queue *eq;
-       struct  vm_pageout_queue *tq;
+       vm_page_t       m;
+       vm_object_t     l_object = NULL;
+       vm_object_t     m_object = NULL;
+       int             refmod_state = 0;
+       int             try_failed_count = 0;
+       int             retval = 0;
+       int             current_run = 0;
+       struct  vm_pageout_queue *iq;
+       struct  vm_pageout_queue *eq;
+       struct  vm_pageout_queue *tq;

 
        KDBG(IOKDBG_CODE(DBG_HIBERNATE, 4) | DBG_FUNC_START,
 
        KDBG(IOKDBG_CODE(DBG_HIBERNATE, 4) | DBG_FUNC_START,

-                       VM_KERNEL_UNSLIDE_OR_PERM(q), qcount);
+           VM_KERNEL_UNSLIDE_OR_PERM(q), qcount);

 
        iq = &vm_pageout_queue_internal;
        eq = &vm_pageout_queue_external;
 
        iq = &vm_pageout_queue_internal;
        eq = &vm_pageout_queue_external;


@@ -6390,7 +6883,6 @@ hibernate_flush_queue(vm_page_queue_head_t *q, int qcount)
        vm_page_lock_queues();
 
        while (qcount && !vm_page_queue_empty(q)) {
        vm_page_lock_queues();
 
        while (qcount && !vm_page_queue_empty(q)) {

-

                if (current_run++ == 1000) {
                        if (hibernate_should_abort()) {
                                retval = 1;
                if (current_run++ == 1000) {
                        if (hibernate_should_abort()) {
                                retval = 1;


@@ -6408,13 +6900,13 @@ hibernate_flush_queue(vm_page_queue_head_t *q, int qcount)
                 * already got the lock
                 */
                if (m_object != l_object) {
                 * already got the lock
                 */
                if (m_object != l_object) {

-                       /*
-                        * the object associated with candidate page is 
+                       /*
+                        * the object associated with candidate page is

                         * different from the one we were just working
                         * with... dump the lock if we still own it
                         */
                         * different from the one we were just working
                         * with... dump the lock if we still own it
                         */

-                       if (l_object != NULL) {
-                               vm_object_unlock(l_object);
+                       if (l_object != NULL) {
+                               vm_object_unlock(l_object);

                                l_object = NULL;
                        }
                        /*
                                l_object = NULL;
                        }
                        /*


@@ -6422,10 +6914,9 @@ hibernate_flush_queue(vm_page_queue_head_t *q, int qcount)
                         * page queues lock, we can only 'try' for this one.
                         * if the 'try' fails, we need to do a mutex_pause
                         * to allow the owner of the object lock a chance to
                         * page queues lock, we can only 'try' for this one.
                         * if the 'try' fails, we need to do a mutex_pause
                         * to allow the owner of the object lock a chance to

-                        * run... 
+                        * run...

                         */
                         */

-                       if ( !vm_object_lock_try_scan(m_object)) {
-
+                       if (!vm_object_lock_try_scan(m_object)) {

                                if (try_failed_count > 20) {
                                        hibernate_stats.hibernate_queue_nolock++;
 
                                if (try_failed_count > 20) {
                                        hibernate_stats.hibernate_queue_nolock++;
 


@@ -6442,15 +6933,16 @@ hibernate_flush_queue(vm_page_queue_head_t *q, int qcount)
                                l_object = m_object;
                        }
                }
                                l_object = m_object;
                        }
                }

-               if ( !m_object->alive || m->vmp_cleaning || m->vmp_laundry || m->vmp_busy || m->vmp_absent || m->vmp_error) {
+               if (!m_object->alive || m->vmp_cleaning || m->vmp_laundry || m->vmp_busy || m->vmp_absent || m->vmp_error) {

                        /*
                         * page is not to be cleaned
                         * put it back on the head of its queue
                         */
                        /*
                         * page is not to be cleaned
                         * put it back on the head of its queue
                         */

-                       if (m->vmp_cleaning)
+                       if (m->vmp_cleaning) {

                                hibernate_stats.hibernate_skipped_cleaning++;
                                hibernate_stats.hibernate_skipped_cleaning++;

-                       else
+                       } else {

                                hibernate_stats.hibernate_skipped_transient++;
                                hibernate_stats.hibernate_skipped_transient++;

+                       }

 
                        goto reenter_pg_on_q;
                }
 
                        goto reenter_pg_on_q;
                }


@@ -6463,54 +6955,56 @@ hibernate_flush_queue(vm_page_queue_head_t *q, int qcount)
                                goto reenter_pg_on_q;
                        }
                }
                                goto reenter_pg_on_q;
                        }
                }

-               if ( !m->vmp_dirty && m->vmp_pmapped) {
-                       refmod_state = pmap_get_refmod(VM_PAGE_GET_PHYS_PAGE(m));
+               if (!m->vmp_dirty && m->vmp_pmapped) {
+                       refmod_state = pmap_get_refmod(VM_PAGE_GET_PHYS_PAGE(m));

 
                        if ((refmod_state & VM_MEM_MODIFIED)) {
                                SET_PAGE_DIRTY(m, FALSE);
                        }
 
                        if ((refmod_state & VM_MEM_MODIFIED)) {
                                SET_PAGE_DIRTY(m, FALSE);
                        }

-               } else
+               } else {

                        refmod_state = 0;
                        refmod_state = 0;

+               }

 
 

-               if ( !m->vmp_dirty) {
+               if (!m->vmp_dirty) {

                        /*
                         * page is not to be cleaned
                         * put it back on the head of its queue
                         */
                        /*
                         * page is not to be cleaned
                         * put it back on the head of its queue
                         */

-                       if (m->vmp_precious)
+                       if (m->vmp_precious) {

                                hibernate_stats.hibernate_skipped_precious++;
                                hibernate_stats.hibernate_skipped_precious++;

+                       }

 
                        goto reenter_pg_on_q;
                }
 
                if (hibernate_skip_external == TRUE && !m_object->internal) {
 
                        goto reenter_pg_on_q;
                }
 
                if (hibernate_skip_external == TRUE && !m_object->internal) {

-

                        hibernate_stats.hibernate_skipped_external++;
                        hibernate_stats.hibernate_skipped_external++;

-                       
+

                        goto reenter_pg_on_q;
                }
                tq = NULL;
 
                if (m_object->internal) {
                        goto reenter_pg_on_q;
                }
                tq = NULL;
 
                if (m_object->internal) {

-                       if (VM_PAGE_Q_THROTTLED(iq))
+                       if (VM_PAGE_Q_THROTTLED(iq)) {

                                tq = iq;
                                tq = iq;

-               } else if (VM_PAGE_Q_THROTTLED(eq))
+                       }
+               } else if (VM_PAGE_Q_THROTTLED(eq)) {

                        tq = eq;
                        tq = eq;

+               }

 
                if (tq != NULL) {
 
                if (tq != NULL) {

-                       wait_result_t   wait_result;
-                       int             wait_count = 5;
+                       wait_result_t   wait_result;
+                       int             wait_count = 5;

 
 

-                       if (l_object != NULL) {
-                               vm_object_unlock(l_object);
+                       if (l_object != NULL) {
+                               vm_object_unlock(l_object);

                                l_object = NULL;
                        }
 
                        while (retval == 0) {
                                l_object = NULL;
                        }
 
                        while (retval == 0) {

-

                                tq->pgo_throttled = TRUE;
 
                                tq->pgo_throttled = TRUE;
 

-                               assert_wait_timeout((event_t) &tq->pgo_laundry, THREAD_INTERRUPTIBLE, 1000, 1000*NSEC_PER_USEC);
+                               assert_wait_timeout((event_t) &tq->pgo_laundry, THREAD_INTERRUPTIBLE, 1000, 1000 * NSEC_PER_USEC);

 
                                vm_page_unlock_queues();
 
 
                                vm_page_unlock_queues();
 


@@ -6518,16 +7012,18 @@ hibernate_flush_queue(vm_page_queue_head_t *q, int qcount)
 
                                vm_page_lock_queues();
 
 
                                vm_page_lock_queues();
 

-                               if (wait_result != THREAD_TIMED_OUT)
+                               if (wait_result != THREAD_TIMED_OUT) {

                                        break;
                                        break;

-                                if (!VM_PAGE_Q_THROTTLED(tq))
-                                        break;
+                               }
+                               if (!VM_PAGE_Q_THROTTLED(tq)) {
+                                       break;
+                               }

 
 

-                               if (hibernate_should_abort())
+                               if (hibernate_should_abort()) {

                                        retval = 1;
                                        retval = 1;

+                               }

 
                                if (--wait_count == 0) {
 
                                if (--wait_count == 0) {

-

                                        hibernate_stats.hibernate_throttle_timeout++;
 
                                        if (tq == eq) {
                                        hibernate_stats.hibernate_throttle_timeout++;
 
                                        if (tq == eq) {


@@ -6537,8 +7033,9 @@ hibernate_flush_queue(vm_page_queue_head_t *q, int qcount)
                                        retval = 1;
                                }
                        }
                                        retval = 1;
                                }
                        }

-                       if (retval)
+                       if (retval) {

                                break;
                                break;

+                       }

 
                        hibernate_stats.hibernate_throttled++;
 
 
                        hibernate_stats.hibernate_throttled++;
 


@@ -6551,8 +7048,9 @@ hibernate_flush_queue(vm_page_queue_head_t *q, int qcount)
                 */
                vm_page_queues_remove(m, TRUE);
 
                 */
                vm_page_queues_remove(m, TRUE);
 

-               if (m_object->internal == TRUE)
+               if (m_object->internal == TRUE) {

                        pmap_disconnect_options(VM_PAGE_GET_PHYS_PAGE(m), PMAP_OPTIONS_COMPRESSOR, NULL);
                        pmap_disconnect_options(VM_PAGE_GET_PHYS_PAGE(m), PMAP_OPTIONS_COMPRESSOR, NULL);

+               }

 
                vm_pageout_cluster(m);
 
 
                vm_pageout_cluster(m);
 


@@ -6561,8 +7059,8 @@ hibernate_flush_queue(vm_page_queue_head_t *q, int qcount)
                goto next_pg;
 
 reenter_pg_on_q:
                goto next_pg;
 
 reenter_pg_on_q:

-               vm_page_queue_remove(q, m, vm_page_t, vmp_pageq);
-               vm_page_queue_enter(q, m, vm_page_t, vmp_pageq);
+               vm_page_queue_remove(q, m, vmp_pageq);
+               vm_page_queue_enter(q, m, vmp_pageq);

 
                hibernate_stats.hibernate_reentered_on_q++;
 next_pg:
 
                hibernate_stats.hibernate_reentered_on_q++;
 next_pg:


@@ -6580,77 +7078,85 @@ next_pg:
 
        KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 4) | DBG_FUNC_END, hibernate_stats.hibernate_found_dirty, retval, 0, 0, 0);
 
 
        KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 4) | DBG_FUNC_END, hibernate_stats.hibernate_found_dirty, retval, 0, 0, 0);
 

-       return (retval);
+       return retval;

 }
 
 
 static int
 hibernate_flush_dirty_pages(int pass)
 {
 }
 
 
 static int
 hibernate_flush_dirty_pages(int pass)
 {

-       struct vm_speculative_age_q     *aq;
-       uint32_t        i;
+       struct vm_speculative_age_q     *aq;
+       uint32_t        i;

 
        if (vm_page_local_q) {
 
        if (vm_page_local_q) {

-               for (i = 0; i < vm_page_local_q_count; i++)
-                       vm_page_reactivate_local(i, TRUE, FALSE);
+               zpercpu_foreach_cpu(lid) {
+                       vm_page_reactivate_local(lid, TRUE, FALSE);
+               }

        }
 
        for (i = 0; i <= VM_PAGE_MAX_SPECULATIVE_AGE_Q; i++) {
        }
 
        for (i = 0; i <= VM_PAGE_MAX_SPECULATIVE_AGE_Q; i++) {

-               int             qcount;
-               vm_page_t       m;
+               int             qcount;
+               vm_page_t       m;

 
                aq = &vm_page_queue_speculative[i];
 
 
                aq = &vm_page_queue_speculative[i];
 

-               if (vm_page_queue_empty(&aq->age_q))
+               if (vm_page_queue_empty(&aq->age_q)) {

                        continue;
                        continue;

+               }

                qcount = 0;
 
                vm_page_lockspin_queues();
 
                qcount = 0;
 
                vm_page_lockspin_queues();
 

-               vm_page_queue_iterate(&aq->age_q,
-                             m,
-                             vm_page_t,
-                             vmp_pageq)
-               {
+               vm_page_queue_iterate(&aq->age_q, m, vmp_pageq) {

                        qcount++;
                }
                vm_page_unlock_queues();
 
                if (qcount) {
                        qcount++;
                }
                vm_page_unlock_queues();
 
                if (qcount) {

-                       if (hibernate_flush_queue(&aq->age_q, qcount))
-                               return (1);
+                       if (hibernate_flush_queue(&aq->age_q, qcount)) {
+                               return 1;
+                       }

                }
        }
                }
        }

-       if (hibernate_flush_queue(&vm_page_queue_inactive, vm_page_inactive_count - vm_page_anonymous_count - vm_page_cleaned_count))
-               return (1);
+       if (hibernate_flush_queue(&vm_page_queue_inactive, vm_page_inactive_count - vm_page_anonymous_count - vm_page_cleaned_count)) {
+               return 1;
+       }

        /* XXX FBDP TODO: flush secluded queue */
        /* XXX FBDP TODO: flush secluded queue */

-       if (hibernate_flush_queue(&vm_page_queue_anonymous, vm_page_anonymous_count))
-               return (1);
-       if (hibernate_flush_queue(&vm_page_queue_cleaned, vm_page_cleaned_count))
-               return (1);
-       if (hibernate_drain_pageout_queue(&vm_pageout_queue_internal))
-               return (1);
-
-       if (pass == 1)
+       if (hibernate_flush_queue(&vm_page_queue_anonymous, vm_page_anonymous_count)) {
+               return 1;
+       }
+       if (hibernate_flush_queue(&vm_page_queue_cleaned, vm_page_cleaned_count)) {
+               return 1;
+       }
+       if (hibernate_drain_pageout_queue(&vm_pageout_queue_internal)) {
+               return 1;
+       }
+
+       if (pass == 1) {

                vm_compressor_record_warmup_start();
                vm_compressor_record_warmup_start();

+       }

 
        if (hibernate_flush_queue(&vm_page_queue_active, vm_page_active_count)) {
 
        if (hibernate_flush_queue(&vm_page_queue_active, vm_page_active_count)) {

-               if (pass == 1)
+               if (pass == 1) {

                        vm_compressor_record_warmup_end();
                        vm_compressor_record_warmup_end();

-               return (1);
+               }
+               return 1;

        }
        if (hibernate_drain_pageout_queue(&vm_pageout_queue_internal)) {
        }
        if (hibernate_drain_pageout_queue(&vm_pageout_queue_internal)) {

-               if (pass == 1)
+               if (pass == 1) {

                        vm_compressor_record_warmup_end();
                        vm_compressor_record_warmup_end();

-               return (1);
+               }
+               return 1;

        }
        }

-       if (pass == 1)
+       if (pass == 1) {

                vm_compressor_record_warmup_end();
                vm_compressor_record_warmup_end();

+       }

 
 

-       if (hibernate_skip_external == FALSE && hibernate_drain_pageout_queue(&vm_pageout_queue_external))
-               return (1);
+       if (hibernate_skip_external == FALSE && hibernate_drain_pageout_queue(&vm_pageout_queue_external)) {
+               return 1;
+       }

 
 

-       return (0);
+       return 0;

 }
 
 
 }
 
 


@@ -6664,7 +7170,7 @@ hibernate_reset_stats()
 int
 hibernate_flush_memory()
 {
 int
 hibernate_flush_memory()
 {

-       int     retval;
+       int     retval;

 
        assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
 
 
        assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
 


@@ -6674,7 +7180,6 @@ hibernate_flush_memory()
        hibernate_skip_external = FALSE;
 
        if ((retval = hibernate_flush_dirty_pages(1)) == 0) {
        hibernate_skip_external = FALSE;
 
        if ((retval = hibernate_flush_dirty_pages(1)) == 0) {

-

                KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 10) | DBG_FUNC_START, VM_PAGE_COMPRESSOR_COUNT, 0, 0, 0, 0);
 
                vm_compressor_flush();
                KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 10) | DBG_FUNC_START, VM_PAGE_COMPRESSOR_COUNT, 0, 0, 0, 0);
 
                vm_compressor_flush();


@@ -6686,7 +7191,7 @@ hibernate_flush_memory()
 
                        KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 7) | DBG_FUNC_START, 0, 0, 0, 0, 0);
                        orig_wire_count = vm_page_wire_count;
 
                        KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 7) | DBG_FUNC_START, 0, 0, 0, 0, 0);
                        orig_wire_count = vm_page_wire_count;

-                       
+

                        (void)(*consider_buffer_cache_collect)(1);
                        consider_zone_gc(FALSE);
 
                        (void)(*consider_buffer_cache_collect)(1);
                        consider_zone_gc(FALSE);
 


@@ -6699,832 +7204,846 @@ hibernate_flush_memory()
 
        KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 3) | DBG_FUNC_END, vm_page_free_count, hibernate_stats.hibernate_found_dirty, retval, 0, 0);
 
 
        KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 3) | DBG_FUNC_END, vm_page_free_count, hibernate_stats.hibernate_found_dirty, retval, 0, 0);
 

-       if (retval)
+       if (retval) {

                HIBLOG("hibernate_flush_memory() failed to finish - vm_page_compressor_count(%d)\n", VM_PAGE_COMPRESSOR_COUNT);
                HIBLOG("hibernate_flush_memory() failed to finish - vm_page_compressor_count(%d)\n", VM_PAGE_COMPRESSOR_COUNT);

+       }

 
 
 
 

-    HIBPRINT("hibernate_flush_memory() considered(%d) reentered_on_q(%d) found_dirty(%d)\n",
-                hibernate_stats.hibernate_considered,
-                hibernate_stats.hibernate_reentered_on_q,
-                hibernate_stats.hibernate_found_dirty);
-    HIBPRINT("   skipped_cleaning(%d) skipped_transient(%d) skipped_precious(%d) skipped_external(%d) queue_nolock(%d)\n",
-                hibernate_stats.hibernate_skipped_cleaning,
-                hibernate_stats.hibernate_skipped_transient,
-                hibernate_stats.hibernate_skipped_precious,
-                hibernate_stats.hibernate_skipped_external,
-                hibernate_stats.hibernate_queue_nolock);
-    HIBPRINT("   queue_paused(%d) throttled(%d) throttle_timeout(%d) drained(%d) drain_timeout(%d)\n",
-                hibernate_stats.hibernate_queue_paused,
-                hibernate_stats.hibernate_throttled,
-                hibernate_stats.hibernate_throttle_timeout,
-                hibernate_stats.hibernate_drained,
-                hibernate_stats.hibernate_drain_timeout);
+       HIBPRINT("hibernate_flush_memory() considered(%d) reentered_on_q(%d) found_dirty(%d)\n",
+           hibernate_stats.hibernate_considered,
+           hibernate_stats.hibernate_reentered_on_q,
+           hibernate_stats.hibernate_found_dirty);
+       HIBPRINT("   skipped_cleaning(%d) skipped_transient(%d) skipped_precious(%d) skipped_external(%d) queue_nolock(%d)\n",
+           hibernate_stats.hibernate_skipped_cleaning,
+           hibernate_stats.hibernate_skipped_transient,
+           hibernate_stats.hibernate_skipped_precious,
+           hibernate_stats.hibernate_skipped_external,
+           hibernate_stats.hibernate_queue_nolock);
+       HIBPRINT("   queue_paused(%d) throttled(%d) throttle_timeout(%d) drained(%d) drain_timeout(%d)\n",
+           hibernate_stats.hibernate_queue_paused,
+           hibernate_stats.hibernate_throttled,
+           hibernate_stats.hibernate_throttle_timeout,
+           hibernate_stats.hibernate_drained,
+           hibernate_stats.hibernate_drain_timeout);

 
 

-       return (retval);
+       return retval;

 }
 
 
 static void
 hibernate_page_list_zero(hibernate_page_list_t *list)
 {
 }
 
 
 static void
 hibernate_page_list_zero(hibernate_page_list_t *list)
 {

-    uint32_t             bank;
-    hibernate_bitmap_t * bitmap;
-
-    bitmap = &list->bank_bitmap[0];
-    for (bank = 0; bank < list->bank_count; bank++)
-    {
-        uint32_t last_bit;
-
-       bzero((void *) &bitmap->bitmap[0], bitmap->bitmapwords << 2); 
-        // set out-of-bound bits at end of bitmap.
-        last_bit = ((bitmap->last_page - bitmap->first_page + 1) & 31);
-       if (last_bit)
-           bitmap->bitmap[bitmap->bitmapwords - 1] = (0xFFFFFFFF >> last_bit);
+       uint32_t             bank;
+       hibernate_bitmap_t * bitmap;
+
+       bitmap = &list->bank_bitmap[0];
+       for (bank = 0; bank < list->bank_count; bank++) {
+               uint32_t last_bit;
+
+               bzero((void *) &bitmap->bitmap[0], bitmap->bitmapwords << 2);
+               // set out-of-bound bits at end of bitmap.
+               last_bit = ((bitmap->last_page - bitmap->first_page + 1) & 31);
+               if (last_bit) {
+                       bitmap->bitmap[bitmap->bitmapwords - 1] = (0xFFFFFFFF >> last_bit);
+               }

 
 

-       bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
-    }
+               bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
+       }

 }
 
 void
 hibernate_free_gobble_pages(void)
 {
 }
 
 void
 hibernate_free_gobble_pages(void)
 {

-    vm_page_t m, next;
-    uint32_t  count = 0;
+       vm_page_t m, next;
+       uint32_t  count = 0;

 
 

-    m = (vm_page_t) hibernate_gobble_queue;
-    while(m)
-    {
-        next = m->vmp_snext;
-        vm_page_free(m);
-        count++;
-        m = next;
-    }
-    hibernate_gobble_queue = VM_PAGE_NULL;
-    
-    if (count)
-        HIBLOG("Freed %d pages\n", count);
+       m = (vm_page_t) hibernate_gobble_queue;
+       while (m) {
+               next = m->vmp_snext;
+               vm_page_free(m);
+               count++;
+               m = next;
+       }
+       hibernate_gobble_queue = VM_PAGE_NULL;
+
+       if (count) {
+               HIBLOG("Freed %d pages\n", count);
+       }

 }
 
 }
 

-static boolean_t 
+static boolean_t

 hibernate_consider_discard(vm_page_t m, boolean_t preflight)
 {
 hibernate_consider_discard(vm_page_t m, boolean_t preflight)
 {

-    vm_object_t object = NULL;
-    int                  refmod_state;
-    boolean_t            discard = FALSE;
+       vm_object_t object = NULL;
+       int                  refmod_state;
+       boolean_t            discard = FALSE;

 
 

-    do
-    {
-        if (m->vmp_private)
-            panic("hibernate_consider_discard: private");
+       do{
+               if (m->vmp_private) {
+                       panic("hibernate_consider_discard: private");
+               }

 
 

-       object = VM_PAGE_OBJECT(m);
+               object = VM_PAGE_OBJECT(m);

 
 

-        if (!vm_object_lock_try(object)) {
-           object = NULL;
-           if (!preflight) hibernate_stats.cd_lock_failed++;
-            break;
-       }
-       if (VM_PAGE_WIRED(m)) {
-           if (!preflight) hibernate_stats.cd_found_wired++;
-            break;
-       }
-        if (m->vmp_precious) {
-           if (!preflight) hibernate_stats.cd_found_precious++;
-            break;
-       }
-        if (m->vmp_busy || !object->alive) {
-           /*
-            *  Somebody is playing with this page.
-            */
-           if (!preflight) hibernate_stats.cd_found_busy++;
-            break;
-       }
-        if (m->vmp_absent || m->vmp_unusual || m->vmp_error) {
-           /*
-            * If it's unusual in anyway, ignore it
-            */
-           if (!preflight) hibernate_stats.cd_found_unusual++;
-            break;
-       }
-        if (m->vmp_cleaning) {
-           if (!preflight) hibernate_stats.cd_found_cleaning++;
-            break;
-       }
-       if (m->vmp_laundry) {
-           if (!preflight) hibernate_stats.cd_found_laundry++;
-            break;
-       }
-        if (!m->vmp_dirty)
-        {
-               refmod_state = pmap_get_refmod(VM_PAGE_GET_PHYS_PAGE(m));
-        
-            if (refmod_state & VM_MEM_REFERENCED)
-                m->vmp_reference = TRUE;
-            if (refmod_state & VM_MEM_MODIFIED) {
-               SET_PAGE_DIRTY(m, FALSE);
-           }
-        }
-   
-        /*
-         * If it's clean or purgeable we can discard the page on wakeup.
-         */
-        discard = (!m->vmp_dirty) 
+               if (!vm_object_lock_try(object)) {
+                       object = NULL;
+                       if (!preflight) {
+                               hibernate_stats.cd_lock_failed++;
+                       }
+                       break;
+               }
+               if (VM_PAGE_WIRED(m)) {
+                       if (!preflight) {
+                               hibernate_stats.cd_found_wired++;
+                       }
+                       break;
+               }
+               if (m->vmp_precious) {
+                       if (!preflight) {
+                               hibernate_stats.cd_found_precious++;
+                       }
+                       break;
+               }
+               if (m->vmp_busy || !object->alive) {
+                       /*
+                        *      Somebody is playing with this page.
+                        */
+                       if (!preflight) {
+                               hibernate_stats.cd_found_busy++;
+                       }
+                       break;
+               }
+               if (m->vmp_absent || m->vmp_unusual || m->vmp_error) {
+                       /*
+                        * If it's unusual in anyway, ignore it
+                        */
+                       if (!preflight) {
+                               hibernate_stats.cd_found_unusual++;
+                       }
+                       break;
+               }
+               if (m->vmp_cleaning) {
+                       if (!preflight) {
+                               hibernate_stats.cd_found_cleaning++;
+                       }
+                       break;
+               }
+               if (m->vmp_laundry) {
+                       if (!preflight) {
+                               hibernate_stats.cd_found_laundry++;
+                       }
+                       break;
+               }
+               if (!m->vmp_dirty) {
+                       refmod_state = pmap_get_refmod(VM_PAGE_GET_PHYS_PAGE(m));
+
+                       if (refmod_state & VM_MEM_REFERENCED) {
+                               m->vmp_reference = TRUE;
+                       }
+                       if (refmod_state & VM_MEM_MODIFIED) {
+                               SET_PAGE_DIRTY(m, FALSE);
+                       }
+               }
+
+               /*
+                * If it's clean or purgeable we can discard the page on wakeup.
+                */
+               discard = (!m->vmp_dirty)

                    || (VM_PURGABLE_VOLATILE == object->purgable)
                    || (VM_PURGABLE_VOLATILE == object->purgable)

-                   || (VM_PURGABLE_EMPTY    == object->purgable);
+                   || (VM_PURGABLE_EMPTY == object->purgable);

 
 
 
 

-        if (discard == FALSE) {
-               if (!preflight)
-                       hibernate_stats.cd_found_dirty++;
-        } else if (m->vmp_xpmapped && m->vmp_reference && !object->internal) {
-               if (hibernate_stats.cd_found_xpmapped < HIBERNATE_XPMAPPED_LIMIT) {
-                       if (!preflight)
-                               hibernate_stats.cd_found_xpmapped++;
-                       discard = FALSE;
-               } else {
-                       if (!preflight)
-                               hibernate_stats.cd_skipped_xpmapped++;
+               if (discard == FALSE) {
+                       if (!preflight) {
+                               hibernate_stats.cd_found_dirty++;
+                       }
+               } else if (m->vmp_xpmapped && m->vmp_reference && !object->internal) {
+                       if (hibernate_stats.cd_found_xpmapped < HIBERNATE_XPMAPPED_LIMIT) {
+                               if (!preflight) {
+                                       hibernate_stats.cd_found_xpmapped++;
+                               }
+                               discard = FALSE;
+                       } else {
+                               if (!preflight) {
+                                       hibernate_stats.cd_skipped_xpmapped++;
+                               }
+                       }

                }
                }

-        }
-    }
-    while (FALSE);
+       }while (FALSE);

 
 

-    if (object)
-        vm_object_unlock(object);
+       if (object) {
+               vm_object_unlock(object);
+       }

 
 

-    return (discard);
+       return discard;

 }
 
 
 static void
 hibernate_discard_page(vm_page_t m)
 {
 }
 
 
 static void
 hibernate_discard_page(vm_page_t m)
 {

-    vm_object_t        m_object;
+       vm_object_t m_object;

 
 

-    if (m->vmp_absent || m->vmp_unusual || m->vmp_error)
-       /*
-        * If it's unusual in anyway, ignore
-        */
-        return;
+       if (m->vmp_absent || m->vmp_unusual || m->vmp_error) {
+               /*
+                * If it's unusual in anyway, ignore
+                */
+               return;
+       }

 
 

-    m_object = VM_PAGE_OBJECT(m);
+       m_object = VM_PAGE_OBJECT(m);

 
 #if MACH_ASSERT || DEBUG
 
 #if MACH_ASSERT || DEBUG

-    if (!vm_object_lock_try(m_object))
-       panic("hibernate_discard_page(%p) !vm_object_lock_try", m);
+       if (!vm_object_lock_try(m_object)) {
+               panic("hibernate_discard_page(%p) !vm_object_lock_try", m);
+       }

 #else
 #else

-    /* No need to lock page queue for token delete, hibernate_vm_unlock() 
-       makes sure these locks are uncontended before sleep */
+       /* No need to lock page queue for token delete, hibernate_vm_unlock()
+        *  makes sure these locks are uncontended before sleep */

 #endif /* MACH_ASSERT || DEBUG */
 
 #endif /* MACH_ASSERT || DEBUG */
 

-    if (m->vmp_pmapped == TRUE) 
-    {
-       __unused int refmod_state = pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
-    }
-
-    if (m->vmp_laundry)
-        panic("hibernate_discard_page(%p) laundry", m);
-    if (m->vmp_private)
-        panic("hibernate_discard_page(%p) private", m);
-    if (m->vmp_fictitious)
-        panic("hibernate_discard_page(%p) fictitious", m);
-
-    if (VM_PURGABLE_VOLATILE == m_object->purgable)
-    {
-       /* object should be on a queue */
-        assert((m_object->objq.next != NULL) && (m_object->objq.prev != NULL));
-        purgeable_q_t old_queue = vm_purgeable_object_remove(m_object);
-        assert(old_queue);
-       if (m_object->purgeable_when_ripe) {
-               vm_purgeable_token_delete_first(old_queue);
+       if (m->vmp_pmapped == TRUE) {
+               __unused int refmod_state = pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));

        }
        }

-       vm_object_lock_assert_exclusive(m_object);
-        m_object->purgable = VM_PURGABLE_EMPTY;

 
 

-       /*
-        * Purgeable ledgers:  pages of VOLATILE and EMPTY objects are
-        * accounted in the "volatile" ledger, so no change here.
-        * We have to update vm_page_purgeable_count, though, since we're
-        * effectively purging this object.
-        */
-       unsigned int delta;
-       assert(m_object->resident_page_count >= m_object->wired_page_count);
-       delta = (m_object->resident_page_count - m_object->wired_page_count);
-       assert(vm_page_purgeable_count >= delta);
-       assert(delta > 0);
-       OSAddAtomic(-delta, (SInt32 *)&vm_page_purgeable_count);
-    }
-       
-    vm_page_free(m);
+       if (m->vmp_laundry) {
+               panic("hibernate_discard_page(%p) laundry", m);
+       }
+       if (m->vmp_private) {
+               panic("hibernate_discard_page(%p) private", m);
+       }
+       if (m->vmp_fictitious) {
+               panic("hibernate_discard_page(%p) fictitious", m);
+       }
+
+       if (VM_PURGABLE_VOLATILE == m_object->purgable) {
+               /* object should be on a queue */
+               assert((m_object->objq.next != NULL) && (m_object->objq.prev != NULL));
+               purgeable_q_t old_queue = vm_purgeable_object_remove(m_object);
+               assert(old_queue);
+               if (m_object->purgeable_when_ripe) {
+                       vm_purgeable_token_delete_first(old_queue);
+               }
+               vm_object_lock_assert_exclusive(m_object);
+               m_object->purgable = VM_PURGABLE_EMPTY;
+
+               /*
+                * Purgeable ledgers:  pages of VOLATILE and EMPTY objects are
+                * accounted in the "volatile" ledger, so no change here.
+                * We have to update vm_page_purgeable_count, though, since we're
+                * effectively purging this object.
+                */
+               unsigned int delta;
+               assert(m_object->resident_page_count >= m_object->wired_page_count);
+               delta = (m_object->resident_page_count - m_object->wired_page_count);
+               assert(vm_page_purgeable_count >= delta);
+               assert(delta > 0);
+               OSAddAtomic(-delta, (SInt32 *)&vm_page_purgeable_count);
+       }
+
+       vm_page_free(m);

 
 #if MACH_ASSERT || DEBUG
 
 #if MACH_ASSERT || DEBUG

-    vm_object_unlock(m_object);
-#endif /* MACH_ASSERT || DEBUG */
+       vm_object_unlock(m_object);
+#endif  /* MACH_ASSERT || DEBUG */

 }
 
 /*
 }
 
 /*

- Grab locks for hibernate_page_list_setall()
-*/
+ *  Grab locks for hibernate_page_list_setall()
+ */

 void
 hibernate_vm_lock_queues(void)
 {
 void
 hibernate_vm_lock_queues(void)
 {

-    vm_object_lock(compressor_object);
-    vm_page_lock_queues();
-    lck_mtx_lock(&vm_page_queue_free_lock);
-    lck_mtx_lock(&vm_purgeable_queue_lock);
+       vm_object_lock(compressor_object);
+       vm_page_lock_queues();
+       lck_mtx_lock(&vm_page_queue_free_lock);
+       lck_mtx_lock(&vm_purgeable_queue_lock);

 
 

-    if (vm_page_local_q) {
-       uint32_t  i;
-       for (i = 0; i < vm_page_local_q_count; i++) {
-           struct vpl  *lq;
-           lq = &vm_page_local_q[i].vpl_un.vpl;
-           VPL_LOCK(&lq->vpl_lock);
+       if (vm_page_local_q) {
+               zpercpu_foreach(lq, vm_page_local_q) {
+                       VPL_LOCK(&lq->vpl_lock);
+               }

        }
        }

-    }

 }
 
 void
 hibernate_vm_unlock_queues(void)
 {
 }
 
 void
 hibernate_vm_unlock_queues(void)
 {

-    if (vm_page_local_q) {
-       uint32_t  i;
-       for (i = 0; i < vm_page_local_q_count; i++) {
-           struct vpl  *lq;
-           lq = &vm_page_local_q[i].vpl_un.vpl;
-           VPL_UNLOCK(&lq->vpl_lock);
+       if (vm_page_local_q) {
+               zpercpu_foreach(lq, vm_page_local_q) {
+                       VPL_UNLOCK(&lq->vpl_lock);
+               }

        }
        }

-    }
-    lck_mtx_unlock(&vm_purgeable_queue_lock);
-    lck_mtx_unlock(&vm_page_queue_free_lock);
-    vm_page_unlock_queues();
-    vm_object_unlock(compressor_object);
+       lck_mtx_unlock(&vm_purgeable_queue_lock);
+       lck_mtx_unlock(&vm_page_queue_free_lock);
+       vm_page_unlock_queues();
+       vm_object_unlock(compressor_object);

 }
 
 /*
 }
 
 /*

- Bits zero in the bitmaps => page needs to be saved. All pages default to be saved,
- pages known to VM to not need saving are subtracted.
- Wired pages to be saved are present in page_list_wired, pageable in page_list.
-*/
+ *  Bits zero in the bitmaps => page needs to be saved. All pages default to be saved,
+ *  pages known to VM to not need saving are subtracted.
+ *  Wired pages to be saved are present in page_list_wired, pageable in page_list.
+ */

 
 void
 hibernate_page_list_setall(hibernate_page_list_t * page_list,
 
 void
 hibernate_page_list_setall(hibernate_page_list_t * page_list,

-                          hibernate_page_list_t * page_list_wired,
-                          hibernate_page_list_t * page_list_pal,
-                          boolean_t preflight, 
-                          boolean_t will_discard,
-                          uint32_t * pagesOut)
-{
-    uint64_t start, end, nsec;
-    vm_page_t m;
-    vm_page_t next;
-    uint32_t pages = page_list->page_count;
-    uint32_t count_anonymous = 0, count_throttled = 0, count_compressor = 0;
-    uint32_t count_inactive = 0, count_active = 0, count_speculative = 0, count_cleaned = 0;
-    uint32_t count_wire = pages;
-    uint32_t count_discard_active    = 0;
-    uint32_t count_discard_inactive  = 0;
-    uint32_t count_discard_cleaned   = 0;
-    uint32_t count_discard_purgeable = 0;
-    uint32_t count_discard_speculative = 0;
-    uint32_t count_discard_vm_struct_pages = 0;
-    uint32_t i;
-    uint32_t             bank;
-    hibernate_bitmap_t * bitmap;
-    hibernate_bitmap_t * bitmap_wired;
-    boolean_t                   discard_all;
-    boolean_t            discard;
-
-    HIBLOG("hibernate_page_list_setall(preflight %d) start\n", preflight);
-
-    if (preflight) {
-        page_list       = NULL;
-        page_list_wired = NULL;
-        page_list_pal   = NULL;
+    hibernate_page_list_t * page_list_wired,
+    hibernate_page_list_t * page_list_pal,
+    boolean_t preflight,
+    boolean_t will_discard,
+    uint32_t * pagesOut)
+{
+       uint64_t start, end, nsec;
+       vm_page_t m;
+       vm_page_t next;
+       uint32_t pages = page_list->page_count;
+       uint32_t count_anonymous = 0, count_throttled = 0, count_compressor = 0;
+       uint32_t count_inactive = 0, count_active = 0, count_speculative = 0, count_cleaned = 0;
+       uint32_t count_wire = pages;
+       uint32_t count_discard_active    = 0;
+       uint32_t count_discard_inactive  = 0;
+       uint32_t count_discard_cleaned   = 0;
+       uint32_t count_discard_purgeable = 0;
+       uint32_t count_discard_speculative = 0;
+       uint32_t count_discard_vm_struct_pages = 0;
+       uint32_t i;
+       uint32_t             bank;
+       hibernate_bitmap_t * bitmap;
+       hibernate_bitmap_t * bitmap_wired;
+       boolean_t                    discard_all;
+       boolean_t            discard;
+
+       HIBLOG("hibernate_page_list_setall(preflight %d) start\n", preflight);
+
+       if (preflight) {
+               page_list       = NULL;
+               page_list_wired = NULL;
+               page_list_pal   = NULL;

                discard_all     = FALSE;
                discard_all     = FALSE;

-    } else {
+       } else {

                discard_all     = will_discard;
                discard_all     = will_discard;

-    }
+       }

 
 #if MACH_ASSERT || DEBUG
 
 #if MACH_ASSERT || DEBUG

-    if (!preflight)
-    {
-        assert(hibernate_vm_locks_are_safe());
-        vm_page_lock_queues();
-       if (vm_page_local_q) {
-           for (i = 0; i < vm_page_local_q_count; i++) {
-               struct vpl      *lq;
-               lq = &vm_page_local_q[i].vpl_un.vpl;
-               VPL_LOCK(&lq->vpl_lock);
-           }
+       if (!preflight) {
+               assert(hibernate_vm_locks_are_safe());
+               vm_page_lock_queues();
+               if (vm_page_local_q) {
+                       zpercpu_foreach(lq, vm_page_local_q) {
+                               VPL_LOCK(&lq->vpl_lock);
+                       }
+               }

        }
        }

-    }

 #endif  /* MACH_ASSERT || DEBUG */
 
 
 #endif  /* MACH_ASSERT || DEBUG */
 
 

-    KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 8) | DBG_FUNC_START, count_wire, 0, 0, 0, 0);
+       KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 8) | DBG_FUNC_START, count_wire, 0, 0, 0, 0);

 
 

-    clock_get_uptime(&start);
+       clock_get_uptime(&start);

 
 

-    if (!preflight) {
-       hibernate_page_list_zero(page_list);
-       hibernate_page_list_zero(page_list_wired);
-       hibernate_page_list_zero(page_list_pal);
-    
-       hibernate_stats.cd_vm_page_wire_count = vm_page_wire_count;
-       hibernate_stats.cd_pages = pages;
-    }
+       if (!preflight) {
+               hibernate_page_list_zero(page_list);
+               hibernate_page_list_zero(page_list_wired);
+               hibernate_page_list_zero(page_list_pal);

 
 

-    if (vm_page_local_q) {
-           for (i = 0; i < vm_page_local_q_count; i++)
-                   vm_page_reactivate_local(i, TRUE, !preflight);
-    }
+               hibernate_stats.cd_vm_page_wire_count = vm_page_wire_count;
+               hibernate_stats.cd_pages = pages;
+       }

 
 

-    if (preflight) {
-       vm_object_lock(compressor_object);
-       vm_page_lock_queues();
-       lck_mtx_lock(&vm_page_queue_free_lock);
-    }
+       if (vm_page_local_q) {
+               zpercpu_foreach_cpu(lid) {
+                       vm_page_reactivate_local(lid, TRUE, !preflight);
+               }
+       }
+
+       if (preflight) {
+               vm_object_lock(compressor_object);
+               vm_page_lock_queues();
+               lck_mtx_lock(&vm_page_queue_free_lock);
+       }

 
 

-    LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
+       LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
+
+       hibernation_vmqueues_inspection = TRUE;

 
 

-    hibernation_vmqueues_inspection = TRUE;
+       m = (vm_page_t) hibernate_gobble_queue;
+       while (m) {
+               pages--;
+               count_wire--;
+               if (!preflight) {
+                       hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
+                       hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
+               }
+               m = m->vmp_snext;
+       }

 
 

-    m = (vm_page_t) hibernate_gobble_queue;
-    while (m)
-    {
-       pages--;
-       count_wire--;

        if (!preflight) {
        if (!preflight) {

-           hibernate_page_bitset(page_list,       TRUE, VM_PAGE_GET_PHYS_PAGE(m));
-           hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
+               percpu_foreach(free_pages_head, free_pages) {
+                       for (m = *free_pages_head; m; m = m->vmp_snext) {
+                               assert(m->vmp_q_state == VM_PAGE_ON_FREE_LOCAL_Q);
+
+                               pages--;
+                               count_wire--;
+                               hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
+                               hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
+
+                               hibernate_stats.cd_local_free++;
+                               hibernate_stats.cd_total_free++;
+                       }
+               }

        }
        }

-       m = m->vmp_snext;
-    }

 
 

-    if (!preflight) for( i = 0; i < real_ncpus; i++ )
-    {
-       if (cpu_data_ptr[i] && cpu_data_ptr[i]->cpu_processor)
-       {
-       for (m = PROCESSOR_DATA(cpu_data_ptr[i]->cpu_processor, free_pages); m; m = m->vmp_snext)
-           {
-               assert(m->vmp_q_state == VM_PAGE_ON_FREE_LOCAL_Q);
+       for (i = 0; i < vm_colors; i++) {
+               vm_page_queue_iterate(&vm_page_queue_free[i].qhead, m, vmp_pageq) {
+                       assert(m->vmp_q_state == VM_PAGE_ON_FREE_Q);
+
+                       pages--;
+                       count_wire--;
+                       if (!preflight) {
+                               hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
+                               hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
+
+                               hibernate_stats.cd_total_free++;
+                       }
+               }
+       }
+
+       vm_page_queue_iterate(&vm_lopage_queue_free, m, vmp_pageq) {
+               assert(m->vmp_q_state == VM_PAGE_ON_FREE_LOPAGE_Q);

 
                pages--;
                count_wire--;
 
                pages--;
                count_wire--;

-               hibernate_page_bitset(page_list,       TRUE, VM_PAGE_GET_PHYS_PAGE(m));
-               hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
+               if (!preflight) {
+                       hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
+                       hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));

 
 

-               hibernate_stats.cd_local_free++;
-               hibernate_stats.cd_total_free++;
-           }
+                       hibernate_stats.cd_total_free++;
+               }
+       }
+
+       m = (vm_page_t) vm_page_queue_first(&vm_page_queue_throttled);
+       while (m && !vm_page_queue_end(&vm_page_queue_throttled, (vm_page_queue_entry_t)m)) {
+               assert(m->vmp_q_state == VM_PAGE_ON_THROTTLED_Q);
+
+               next = (vm_page_t)VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
+               discard = FALSE;
+               if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode)
+                   && hibernate_consider_discard(m, preflight)) {
+                       if (!preflight) {
+                               hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
+                       }
+                       count_discard_inactive++;
+                       discard = discard_all;
+               } else {
+                       count_throttled++;
+               }
+               count_wire--;
+               if (!preflight) {
+                       hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
+               }
+
+               if (discard) {
+                       hibernate_discard_page(m);
+               }
+               m = next;
+       }
+
+       m = (vm_page_t)vm_page_queue_first(&vm_page_queue_anonymous);
+       while (m && !vm_page_queue_end(&vm_page_queue_anonymous, (vm_page_queue_entry_t)m)) {
+               assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q);
+
+               next = (vm_page_t)VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
+               discard = FALSE;
+               if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode) &&
+                   hibernate_consider_discard(m, preflight)) {
+                       if (!preflight) {
+                               hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
+                       }
+                       if (m->vmp_dirty) {
+                               count_discard_purgeable++;
+                       } else {
+                               count_discard_inactive++;
+                       }
+                       discard = discard_all;
+               } else {
+                       count_anonymous++;
+               }
+               count_wire--;
+               if (!preflight) {
+                       hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
+               }
+               if (discard) {
+                       hibernate_discard_page(m);
+               }
+               m = next;
+       }
+
+       m = (vm_page_t) vm_page_queue_first(&vm_page_queue_cleaned);
+       while (m && !vm_page_queue_end(&vm_page_queue_cleaned, (vm_page_queue_entry_t)m)) {
+               assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q);
+
+               next = (vm_page_t)VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
+               discard = FALSE;
+               if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode) &&
+                   hibernate_consider_discard(m, preflight)) {
+                       if (!preflight) {
+                               hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
+                       }
+                       if (m->vmp_dirty) {
+                               count_discard_purgeable++;
+                       } else {
+                               count_discard_cleaned++;
+                       }
+                       discard = discard_all;
+               } else {
+                       count_cleaned++;
+               }
+               count_wire--;
+               if (!preflight) {
+                       hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
+               }
+               if (discard) {
+                       hibernate_discard_page(m);
+               }
+               m = next;
+       }
+
+       m = (vm_page_t) vm_page_queue_first(&vm_page_queue_active);
+       while (m && !vm_page_queue_end(&vm_page_queue_active, (vm_page_queue_entry_t)m)) {
+               assert(m->vmp_q_state == VM_PAGE_ON_ACTIVE_Q);
+
+               next = (vm_page_t)VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
+               discard = FALSE;
+               if ((kIOHibernateModeDiscardCleanActive & gIOHibernateMode) &&
+                   hibernate_consider_discard(m, preflight)) {
+                       if (!preflight) {
+                               hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
+                       }
+                       if (m->vmp_dirty) {
+                               count_discard_purgeable++;
+                       } else {
+                               count_discard_active++;
+                       }
+                       discard = discard_all;
+               } else {
+                       count_active++;
+               }
+               count_wire--;
+               if (!preflight) {
+                       hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
+               }
+               if (discard) {
+                       hibernate_discard_page(m);
+               }
+               m = next;
+       }
+
+       m = (vm_page_t) vm_page_queue_first(&vm_page_queue_inactive);
+       while (m && !vm_page_queue_end(&vm_page_queue_inactive, (vm_page_queue_entry_t)m)) {
+               assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_EXTERNAL_Q);
+
+               next = (vm_page_t)VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
+               discard = FALSE;
+               if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode) &&
+                   hibernate_consider_discard(m, preflight)) {
+                       if (!preflight) {
+                               hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
+                       }
+                       if (m->vmp_dirty) {
+                               count_discard_purgeable++;
+                       } else {
+                               count_discard_inactive++;
+                       }
+                       discard = discard_all;
+               } else {
+                       count_inactive++;
+               }
+               count_wire--;
+               if (!preflight) {
+                       hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
+               }
+               if (discard) {
+                       hibernate_discard_page(m);
+               }
+               m = next;
+       }
+       /* XXX FBDP TODO: secluded queue */
+
+       for (i = 0; i <= VM_PAGE_MAX_SPECULATIVE_AGE_Q; i++) {
+               m = (vm_page_t) vm_page_queue_first(&vm_page_queue_speculative[i].age_q);
+               while (m && !vm_page_queue_end(&vm_page_queue_speculative[i].age_q, (vm_page_queue_entry_t)m)) {
+                       assertf(m->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q,
+                           "Bad page: %p (0x%x:0x%x) on queue %d has state: %d (Discard: %d, Preflight: %d)",
+                           m, m->vmp_pageq.next, m->vmp_pageq.prev, i, m->vmp_q_state, discard, preflight);
+
+                       next = (vm_page_t)VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
+                       discard = FALSE;
+                       if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode) &&
+                           hibernate_consider_discard(m, preflight)) {
+                               if (!preflight) {
+                                       hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
+                               }
+                               count_discard_speculative++;
+                               discard = discard_all;
+                       } else {
+                               count_speculative++;
+                       }
+                       count_wire--;
+                       if (!preflight) {
+                               hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
+                       }
+                       if (discard) {
+                               hibernate_discard_page(m);
+                       }
+                       m = next;
+               }
+       }
+
+       vm_page_queue_iterate(&compressor_object->memq, m, vmp_listq) {
+               assert(m->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR);
+
+               count_compressor++;
+               count_wire--;
+               if (!preflight) {
+                       hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
+               }
+       }
+
+       if (preflight == FALSE && discard_all == TRUE) {
+               KDBG(IOKDBG_CODE(DBG_HIBERNATE, 12) | DBG_FUNC_START);
+
+               HIBLOG("hibernate_teardown started\n");
+               count_discard_vm_struct_pages = hibernate_teardown_vm_structs(page_list, page_list_wired);
+               HIBLOG("hibernate_teardown completed - discarded %d\n", count_discard_vm_struct_pages);
+
+               pages -= count_discard_vm_struct_pages;
+               count_wire -= count_discard_vm_struct_pages;
+
+               hibernate_stats.cd_vm_struct_pages_unneeded = count_discard_vm_struct_pages;
+
+               KDBG(IOKDBG_CODE(DBG_HIBERNATE, 12) | DBG_FUNC_END);

        }
        }

-    }

 
 

-    for( i = 0; i < vm_colors; i++ )
-    {
-       vm_page_queue_iterate(&vm_page_queue_free[i].qhead,
-                             m,
-                             vm_page_t,
-                             vmp_pageq)
-       {
-           assert(m->vmp_q_state == VM_PAGE_ON_FREE_Q);
-
-           pages--;
-           count_wire--;
-           if (!preflight) {
-               hibernate_page_bitset(page_list,       TRUE, VM_PAGE_GET_PHYS_PAGE(m));
-               hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
-    
-               hibernate_stats.cd_total_free++;
-           }
-       }
-    }
-
-    vm_page_queue_iterate(&vm_lopage_queue_free,
-                         m,
-                         vm_page_t,
-                         vmp_pageq)
-    {
-       assert(m->vmp_q_state == VM_PAGE_ON_FREE_LOPAGE_Q);
-
-       pages--;
-       count_wire--;

        if (!preflight) {
        if (!preflight) {

-           hibernate_page_bitset(page_list,       TRUE, VM_PAGE_GET_PHYS_PAGE(m));
-           hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
-    
-           hibernate_stats.cd_total_free++;
-       }
-    }
-
-    m = (vm_page_t) vm_page_queue_first(&vm_page_queue_throttled);
-    while (m && !vm_page_queue_end(&vm_page_queue_throttled, (vm_page_queue_entry_t)m))
-    {
-       assert(m->vmp_q_state == VM_PAGE_ON_THROTTLED_Q);
-
-        next = (vm_page_t)VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
-       discard = FALSE;
-        if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode) 
-         && hibernate_consider_discard(m, preflight))
-        {
-           if (!preflight) hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
-            count_discard_inactive++;
-            discard = discard_all;
-        }
-        else
-            count_throttled++;
-       count_wire--;
-       if (!preflight) hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
-
-        if (discard) hibernate_discard_page(m);
-       m = next;
-    }
-
-    m = (vm_page_t) vm_page_queue_first(&vm_page_queue_anonymous);
-    while (m && !vm_page_queue_end(&vm_page_queue_anonymous, (vm_page_queue_entry_t)m))
-    {
-       assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q);
-
-        next = (vm_page_t)VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
-       discard = FALSE;
-        if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode) 
-         && hibernate_consider_discard(m, preflight))
-        {
-           if (!preflight) hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
-           if (m->vmp_dirty)
-               count_discard_purgeable++;
-           else
-               count_discard_inactive++;
-            discard = discard_all;
-        }
-        else
-            count_anonymous++;
-       count_wire--;
-       if (!preflight) hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
-        if (discard)    hibernate_discard_page(m);
-       m = next;
-    }
-
-    m = (vm_page_t) vm_page_queue_first(&vm_page_queue_cleaned);
-    while (m && !vm_page_queue_end(&vm_page_queue_cleaned, (vm_page_queue_entry_t)m))
-    {
-       assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q);
-
-        next = (vm_page_t)VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
-       discard = FALSE;
-        if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode) 
-         && hibernate_consider_discard(m, preflight))
-        {
-           if (!preflight) hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
-           if (m->vmp_dirty)
-               count_discard_purgeable++;
-           else
-               count_discard_cleaned++;
-            discard = discard_all;
-        }
-        else
-            count_cleaned++;
-       count_wire--;
-       if (!preflight) hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
-        if (discard)    hibernate_discard_page(m);
-       m = next;
-    }
-
-    m = (vm_page_t) vm_page_queue_first(&vm_page_queue_active);
-    while (m && !vm_page_queue_end(&vm_page_queue_active, (vm_page_queue_entry_t)m))
-    {
-       assert(m->vmp_q_state == VM_PAGE_ON_ACTIVE_Q);
-
-        next = (vm_page_t)VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
-       discard = FALSE;
-        if ((kIOHibernateModeDiscardCleanActive & gIOHibernateMode) 
-         && hibernate_consider_discard(m, preflight))
-        {
-           if (!preflight) hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
-           if (m->vmp_dirty)
-               count_discard_purgeable++;
-           else
-               count_discard_active++;
-            discard = discard_all;
-        }
-        else
-            count_active++;
-       count_wire--;
-       if (!preflight) hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
-        if (discard)    hibernate_discard_page(m);
-       m = next;
-    }
-
-    m = (vm_page_t) vm_page_queue_first(&vm_page_queue_inactive);
-    while (m && !vm_page_queue_end(&vm_page_queue_inactive, (vm_page_queue_entry_t)m))
-    {
-       assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_EXTERNAL_Q);
-
-        next = (vm_page_t)VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
-       discard = FALSE;
-        if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode) 
-         && hibernate_consider_discard(m, preflight))
-        {
-           if (!preflight) hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
-           if (m->vmp_dirty)
-               count_discard_purgeable++;
-           else
-               count_discard_inactive++;
-            discard = discard_all;
-        }
-        else
-            count_inactive++;
-       count_wire--;
-       if (!preflight) hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
-        if (discard)    hibernate_discard_page(m);
-       m = next;
-    }
-    /* XXX FBDP TODO: secluded queue */
-
-    for( i = 0; i <= VM_PAGE_MAX_SPECULATIVE_AGE_Q; i++ )
-    {
-       m = (vm_page_t) vm_page_queue_first(&vm_page_queue_speculative[i].age_q);
-       while (m && !vm_page_queue_end(&vm_page_queue_speculative[i].age_q, (vm_page_queue_entry_t)m))
-       {
-           assert(m->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q);
-           assertf(m->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q,
-                   "Bad page: %p (0x%x:0x%x) on queue %d has state: %d (Discard: %d, Preflight: %d)",
-                    m, m->vmp_pageq.next, m->vmp_pageq.prev, i, m->vmp_q_state, discard, preflight);
-
-           next = (vm_page_t)VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
-           discard = FALSE;
-           if ((kIOHibernateModeDiscardCleanInactive & gIOHibernateMode) 
-            && hibernate_consider_discard(m, preflight))
-           {
-               if (!preflight) hibernate_page_bitset(page_list, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
-               count_discard_speculative++;
-               discard = discard_all;
-           }
-           else
-               count_speculative++;
-           count_wire--;
-           if (!preflight) hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
-           if (discard)    hibernate_discard_page(m);
-           m = next;
-       }
-    }
-
-    vm_page_queue_iterate(&compressor_object->memq, m, vm_page_t, vmp_listq)
-    {
-       assert(m->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR);
-
-        count_compressor++;
-       count_wire--;
-       if (!preflight) hibernate_page_bitset(page_list_wired, TRUE, VM_PAGE_GET_PHYS_PAGE(m));
-    }
-
-    if (preflight == FALSE && discard_all == TRUE) {
-           KDBG(IOKDBG_CODE(DBG_HIBERNATE, 12) | DBG_FUNC_START);
-
-           HIBLOG("hibernate_teardown started\n");
-           count_discard_vm_struct_pages = hibernate_teardown_vm_structs(page_list, page_list_wired);
-           HIBLOG("hibernate_teardown completed - discarded %d\n", count_discard_vm_struct_pages);
-
-           pages -= count_discard_vm_struct_pages;
-           count_wire -= count_discard_vm_struct_pages;
-
-           hibernate_stats.cd_vm_struct_pages_unneeded = count_discard_vm_struct_pages;
-
-           KDBG(IOKDBG_CODE(DBG_HIBERNATE, 12) | DBG_FUNC_END);
-    }
-
-    if (!preflight) {
-       // pull wired from hibernate_bitmap
-       bitmap = &page_list->bank_bitmap[0];
-       bitmap_wired = &page_list_wired->bank_bitmap[0];
-       for (bank = 0; bank < page_list->bank_count; bank++)
-       {
-           for (i = 0; i < bitmap->bitmapwords; i++)
-               bitmap->bitmap[i] = bitmap->bitmap[i] | ~bitmap_wired->bitmap[i];
-           bitmap       = (hibernate_bitmap_t *) &bitmap->bitmap      [bitmap->bitmapwords];
-           bitmap_wired = (hibernate_bitmap_t *) &bitmap_wired->bitmap[bitmap_wired->bitmapwords];
+               // pull wired from hibernate_bitmap
+               bitmap = &page_list->bank_bitmap[0];
+               bitmap_wired = &page_list_wired->bank_bitmap[0];
+               for (bank = 0; bank < page_list->bank_count; bank++) {
+                       for (i = 0; i < bitmap->bitmapwords; i++) {
+                               bitmap->bitmap[i] = bitmap->bitmap[i] | ~bitmap_wired->bitmap[i];
+                       }
+                       bitmap = (hibernate_bitmap_t *)&bitmap->bitmap[bitmap->bitmapwords];
+                       bitmap_wired = (hibernate_bitmap_t *) &bitmap_wired->bitmap[bitmap_wired->bitmapwords];
+               }

        }
        }

-    }

 
 

-    // machine dependent adjustments
-    hibernate_page_list_setall_machine(page_list, page_list_wired, preflight, &pages);
+       // machine dependent adjustments
+       hibernate_page_list_setall_machine(page_list, page_list_wired, preflight, &pages);

 
 

-    if (!preflight) {
-       hibernate_stats.cd_count_wire = count_wire;
-       hibernate_stats.cd_discarded = count_discard_active + count_discard_inactive + count_discard_purgeable +
-               count_discard_speculative + count_discard_cleaned + count_discard_vm_struct_pages;
-    }
+       if (!preflight) {
+               hibernate_stats.cd_count_wire = count_wire;
+               hibernate_stats.cd_discarded = count_discard_active + count_discard_inactive + count_discard_purgeable +
+                   count_discard_speculative + count_discard_cleaned + count_discard_vm_struct_pages;
+       }

 
 

-    clock_get_uptime(&end);
-    absolutetime_to_nanoseconds(end - start, &nsec);
-    HIBLOG("hibernate_page_list_setall time: %qd ms\n", nsec / 1000000ULL);
+       clock_get_uptime(&end);
+       absolutetime_to_nanoseconds(end - start, &nsec);
+       HIBLOG("hibernate_page_list_setall time: %qd ms\n", nsec / 1000000ULL);

 
 

-    HIBLOG("pages %d, wire %d, act %d, inact %d, cleaned %d spec %d, zf %d, throt %d, compr %d, xpmapped %d\n  %s discard act %d inact %d purgeable %d spec %d cleaned %d\n", 
-          pages, count_wire, count_active, count_inactive, count_cleaned, count_speculative, count_anonymous, count_throttled, count_compressor, hibernate_stats.cd_found_xpmapped,
-               discard_all ? "did" : "could",
-               count_discard_active, count_discard_inactive, count_discard_purgeable, count_discard_speculative, count_discard_cleaned);
+       HIBLOG("pages %d, wire %d, act %d, inact %d, cleaned %d spec %d, zf %d, throt %d, compr %d, xpmapped %d\n  %s discard act %d inact %d purgeable %d spec %d cleaned %d\n",
+           pages, count_wire, count_active, count_inactive, count_cleaned, count_speculative, count_anonymous, count_throttled, count_compressor, hibernate_stats.cd_found_xpmapped,
+           discard_all ? "did" : "could",
+           count_discard_active, count_discard_inactive, count_discard_purgeable, count_discard_speculative, count_discard_cleaned);

 
 

-    if (hibernate_stats.cd_skipped_xpmapped)
-           HIBLOG("WARNING: hibernate_page_list_setall skipped %d xpmapped pages\n", hibernate_stats.cd_skipped_xpmapped);
+       if (hibernate_stats.cd_skipped_xpmapped) {
+               HIBLOG("WARNING: hibernate_page_list_setall skipped %d xpmapped pages\n", hibernate_stats.cd_skipped_xpmapped);
+       }

 
 

-    *pagesOut = pages - count_discard_active - count_discard_inactive - count_discard_purgeable - count_discard_speculative - count_discard_cleaned;
+       *pagesOut = pages - count_discard_active - count_discard_inactive - count_discard_purgeable - count_discard_speculative - count_discard_cleaned;

 
 

-    if (preflight && will_discard) *pagesOut -= count_compressor + count_throttled + count_anonymous + count_inactive + count_cleaned + count_speculative + count_active;
+       if (preflight && will_discard) {
+               *pagesOut -= count_compressor + count_throttled + count_anonymous + count_inactive + count_cleaned + count_speculative + count_active;
+       }

 
 

-    hibernation_vmqueues_inspection = FALSE;
+       hibernation_vmqueues_inspection = FALSE;

 
 #if MACH_ASSERT || DEBUG
 
 #if MACH_ASSERT || DEBUG

-    if (!preflight)
-    {
+       if (!preflight) {
+               if (vm_page_local_q) {
+                       zpercpu_foreach(lq, vm_page_local_q) {
+                               VPL_UNLOCK(&lq->vpl_lock);
+                       }
+               }
+               vm_page_unlock_queues();
+       }
+#endif  /* MACH_ASSERT || DEBUG */
+
+       if (preflight) {
+               lck_mtx_unlock(&vm_page_queue_free_lock);
+               vm_page_unlock_queues();
+               vm_object_unlock(compressor_object);
+       }
+
+       KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 8) | DBG_FUNC_END, count_wire, *pagesOut, 0, 0, 0);
+}
+
+void
+hibernate_page_list_discard(hibernate_page_list_t * page_list)
+{
+       uint64_t  start, end, nsec;
+       vm_page_t m;
+       vm_page_t next;
+       uint32_t  i;
+       uint32_t  count_discard_active    = 0;
+       uint32_t  count_discard_inactive  = 0;
+       uint32_t  count_discard_purgeable = 0;
+       uint32_t  count_discard_cleaned   = 0;
+       uint32_t  count_discard_speculative = 0;
+
+
+#if MACH_ASSERT || DEBUG
+       vm_page_lock_queues();

        if (vm_page_local_q) {
        if (vm_page_local_q) {

-           for (i = 0; i < vm_page_local_q_count; i++) {
-               struct vpl      *lq;
-               lq = &vm_page_local_q[i].vpl_un.vpl;
-               VPL_UNLOCK(&lq->vpl_lock);
-           }
+               zpercpu_foreach(lq, vm_page_local_q) {
+                       VPL_LOCK(&lq->vpl_lock);
+               }
+       }
+#endif  /* MACH_ASSERT || DEBUG */
+
+       clock_get_uptime(&start);
+
+       m = (vm_page_t) vm_page_queue_first(&vm_page_queue_anonymous);
+       while (m && !vm_page_queue_end(&vm_page_queue_anonymous, (vm_page_queue_entry_t)m)) {
+               assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q);
+
+               next = (vm_page_t) VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
+               if (hibernate_page_bittst(page_list, VM_PAGE_GET_PHYS_PAGE(m))) {
+                       if (m->vmp_dirty) {
+                               count_discard_purgeable++;
+                       } else {
+                               count_discard_inactive++;
+                       }
+                       hibernate_discard_page(m);
+               }
+               m = next;

        }
        }

-        vm_page_unlock_queues();
-    }
-#endif  /* MACH_ASSERT || DEBUG */

 
 

-    if (preflight) {
-       lck_mtx_unlock(&vm_page_queue_free_lock);
-       vm_page_unlock_queues();
-       vm_object_unlock(compressor_object);
-    }
+       for (i = 0; i <= VM_PAGE_MAX_SPECULATIVE_AGE_Q; i++) {
+               m = (vm_page_t) vm_page_queue_first(&vm_page_queue_speculative[i].age_q);
+               while (m && !vm_page_queue_end(&vm_page_queue_speculative[i].age_q, (vm_page_queue_entry_t)m)) {
+                       assert(m->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q);
+
+                       next = (vm_page_t) VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
+                       if (hibernate_page_bittst(page_list, VM_PAGE_GET_PHYS_PAGE(m))) {
+                               count_discard_speculative++;
+                               hibernate_discard_page(m);
+                       }
+                       m = next;
+               }
+       }

 
 

-    KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 8) | DBG_FUNC_END, count_wire, *pagesOut, 0, 0, 0);
-}
+       m = (vm_page_t) vm_page_queue_first(&vm_page_queue_inactive);
+       while (m && !vm_page_queue_end(&vm_page_queue_inactive, (vm_page_queue_entry_t)m)) {
+               assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_EXTERNAL_Q);

 
 

-void
-hibernate_page_list_discard(hibernate_page_list_t * page_list)
-{
-    uint64_t  start, end, nsec;
-    vm_page_t m;
-    vm_page_t next;
-    uint32_t  i;
-    uint32_t  count_discard_active    = 0;
-    uint32_t  count_discard_inactive  = 0;
-    uint32_t  count_discard_purgeable = 0;
-    uint32_t  count_discard_cleaned   = 0;
-    uint32_t  count_discard_speculative = 0;
+               next = (vm_page_t) VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
+               if (hibernate_page_bittst(page_list, VM_PAGE_GET_PHYS_PAGE(m))) {
+                       if (m->vmp_dirty) {
+                               count_discard_purgeable++;
+                       } else {
+                               count_discard_inactive++;
+                       }
+                       hibernate_discard_page(m);
+               }
+               m = next;
+       }
+       /* XXX FBDP TODO: secluded queue */

 
 

+       m = (vm_page_t) vm_page_queue_first(&vm_page_queue_active);
+       while (m && !vm_page_queue_end(&vm_page_queue_active, (vm_page_queue_entry_t)m)) {
+               assert(m->vmp_q_state == VM_PAGE_ON_ACTIVE_Q);

 
 

-#if MACH_ASSERT || DEBUG
-        vm_page_lock_queues();
-       if (vm_page_local_q) {
-           for (i = 0; i < vm_page_local_q_count; i++) {
-               struct vpl      *lq;
-               lq = &vm_page_local_q[i].vpl_un.vpl;
-               VPL_LOCK(&lq->vpl_lock);
-           }
+               next = (vm_page_t) VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
+               if (hibernate_page_bittst(page_list, VM_PAGE_GET_PHYS_PAGE(m))) {
+                       if (m->vmp_dirty) {
+                               count_discard_purgeable++;
+                       } else {
+                               count_discard_active++;
+                       }
+                       hibernate_discard_page(m);
+               }
+               m = next;

        }
        }

-#endif  /* MACH_ASSERT || DEBUG */

 
 

-    clock_get_uptime(&start);
-
-    m = (vm_page_t) vm_page_queue_first(&vm_page_queue_anonymous);
-    while (m && !vm_page_queue_end(&vm_page_queue_anonymous, (vm_page_queue_entry_t)m))
-    {
-       assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_INTERNAL_Q);
-
-        next = (vm_page_t) VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
-        if (hibernate_page_bittst(page_list, VM_PAGE_GET_PHYS_PAGE(m)))
-        {
-           if (m->vmp_dirty)
-               count_discard_purgeable++;
-           else
-               count_discard_inactive++;
-            hibernate_discard_page(m);
-        }
-        m = next;
-    }
-
-    for( i = 0; i <= VM_PAGE_MAX_SPECULATIVE_AGE_Q; i++ )
-    {
-       m = (vm_page_t) vm_page_queue_first(&vm_page_queue_speculative[i].age_q);
-       while (m && !vm_page_queue_end(&vm_page_queue_speculative[i].age_q, (vm_page_queue_entry_t)m))
-       {
-          assert(m->vmp_q_state == VM_PAGE_ON_SPECULATIVE_Q);
-
-           next = (vm_page_t) VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
-           if (hibernate_page_bittst(page_list, VM_PAGE_GET_PHYS_PAGE(m)))
-           {
-               count_discard_speculative++;
-               hibernate_discard_page(m);
-           }
-           m = next;
-       }
-    }
-
-    m = (vm_page_t) vm_page_queue_first(&vm_page_queue_inactive);
-    while (m && !vm_page_queue_end(&vm_page_queue_inactive, (vm_page_queue_entry_t)m))
-    {
-       assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_EXTERNAL_Q);
-
-        next = (vm_page_t) VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
-        if (hibernate_page_bittst(page_list, VM_PAGE_GET_PHYS_PAGE(m)))
-        {
-           if (m->vmp_dirty)
-               count_discard_purgeable++;
-           else
-               count_discard_inactive++;
-            hibernate_discard_page(m);
-        }
-        m = next;
-    }
-    /* XXX FBDP TODO: secluded queue */
-
-    m = (vm_page_t) vm_page_queue_first(&vm_page_queue_active);
-    while (m && !vm_page_queue_end(&vm_page_queue_active, (vm_page_queue_entry_t)m))
-    {
-       assert(m->vmp_q_state == VM_PAGE_ON_ACTIVE_Q);
-
-        next = (vm_page_t) VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
-        if (hibernate_page_bittst(page_list, VM_PAGE_GET_PHYS_PAGE(m)))
-        {
-           if (m->vmp_dirty)
-               count_discard_purgeable++;
-           else
-               count_discard_active++;
-            hibernate_discard_page(m);
-        }
-        m = next;
-    }
-
-    m = (vm_page_t) vm_page_queue_first(&vm_page_queue_cleaned);
-    while (m && !vm_page_queue_end(&vm_page_queue_cleaned, (vm_page_queue_entry_t)m))
-    {
-       assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q);
-
-        next = (vm_page_t) VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
-        if (hibernate_page_bittst(page_list, VM_PAGE_GET_PHYS_PAGE(m)))
-        {
-           if (m->vmp_dirty)
-               count_discard_purgeable++;
-           else
-               count_discard_cleaned++;
-            hibernate_discard_page(m);
-        }
-        m = next;
-    }
+       m = (vm_page_t) vm_page_queue_first(&vm_page_queue_cleaned);
+       while (m && !vm_page_queue_end(&vm_page_queue_cleaned, (vm_page_queue_entry_t)m)) {
+               assert(m->vmp_q_state == VM_PAGE_ON_INACTIVE_CLEANED_Q);
+
+               next = (vm_page_t) VM_PAGE_UNPACK_PTR(m->vmp_pageq.next);
+               if (hibernate_page_bittst(page_list, VM_PAGE_GET_PHYS_PAGE(m))) {
+                       if (m->vmp_dirty) {
+                               count_discard_purgeable++;
+                       } else {
+                               count_discard_cleaned++;
+                       }
+                       hibernate_discard_page(m);
+               }
+               m = next;
+       }

 
 #if MACH_ASSERT || DEBUG
        if (vm_page_local_q) {
 
 #if MACH_ASSERT || DEBUG
        if (vm_page_local_q) {

-           for (i = 0; i < vm_page_local_q_count; i++) {
-               struct vpl      *lq;
-               lq = &vm_page_local_q[i].vpl_un.vpl;
-               VPL_UNLOCK(&lq->vpl_lock);
-           }
+               zpercpu_foreach(lq, vm_page_local_q) {
+                       VPL_UNLOCK(&lq->vpl_lock);
+               }

        }
        }

-        vm_page_unlock_queues();
+       vm_page_unlock_queues();

 #endif  /* MACH_ASSERT || DEBUG */
 
 #endif  /* MACH_ASSERT || DEBUG */
 

-    clock_get_uptime(&end);
-    absolutetime_to_nanoseconds(end - start, &nsec);
-    HIBLOG("hibernate_page_list_discard time: %qd ms, discarded act %d inact %d purgeable %d spec %d cleaned %d\n",
-                nsec / 1000000ULL,
-               count_discard_active, count_discard_inactive, count_discard_purgeable, count_discard_speculative, count_discard_cleaned);
+       clock_get_uptime(&end);
+       absolutetime_to_nanoseconds(end - start, &nsec);
+       HIBLOG("hibernate_page_list_discard time: %qd ms, discarded act %d inact %d purgeable %d spec %d cleaned %d\n",
+           nsec / 1000000ULL,
+           count_discard_active, count_discard_inactive, count_discard_purgeable, count_discard_speculative, count_discard_cleaned);

 }
 
 boolean_t       hibernate_paddr_map_inited = FALSE;
 }
 
 boolean_t       hibernate_paddr_map_inited = FALSE;

-unsigned int   hibernate_teardown_last_valid_compact_indx = -1;
-vm_page_t      hibernate_rebuild_hash_list = NULL;
+unsigned int    hibernate_teardown_last_valid_compact_indx = -1;
+vm_page_t       hibernate_rebuild_hash_list = NULL;

 
 

-unsigned int   hibernate_teardown_found_tabled_pages = 0;
-unsigned int   hibernate_teardown_found_created_pages = 0;
-unsigned int   hibernate_teardown_found_free_pages = 0;
-unsigned int   hibernate_teardown_vm_page_free_count;
+unsigned int    hibernate_teardown_found_tabled_pages = 0;
+unsigned int    hibernate_teardown_found_created_pages = 0;
+unsigned int    hibernate_teardown_found_free_pages = 0;
+unsigned int    hibernate_teardown_vm_page_free_count;

 
 
 struct ppnum_mapping {
 
 
 struct ppnum_mapping {

-       struct ppnum_mapping    *ppnm_next;
-       ppnum_t                 ppnm_base_paddr;
-       unsigned int            ppnm_sindx;
-       unsigned int            ppnm_eindx;
+       struct ppnum_mapping    *ppnm_next;
+       ppnum_t                 ppnm_base_paddr;
+       unsigned int            ppnm_sindx;
+       unsigned int            ppnm_eindx;

 };
 
 };
 

-struct ppnum_mapping   *ppnm_head;
-struct ppnum_mapping   *ppnm_last_found = NULL;
+struct ppnum_mapping    *ppnm_head;
+struct ppnum_mapping    *ppnm_last_found = NULL;

 
 
 void
 
 
 void

-hibernate_create_paddr_map() 
+hibernate_create_paddr_map(void)

 {
 {

-       unsigned int    i;
-       ppnum_t         next_ppnum_in_run = 0;
+       unsigned int    i;
+       ppnum_t         next_ppnum_in_run = 0;

        struct ppnum_mapping *ppnm = NULL;
 
        if (hibernate_paddr_map_inited == FALSE) {
        struct ppnum_mapping *ppnm = NULL;
 
        if (hibernate_paddr_map_inited == FALSE) {

-

                for (i = 0; i < vm_pages_count; i++) {
                for (i = 0; i < vm_pages_count; i++) {

-
-                       if (ppnm)
+                       if (ppnm) {

                                ppnm->ppnm_eindx = i;
                                ppnm->ppnm_eindx = i;

+                       }

 
                        if (ppnm == NULL || VM_PAGE_GET_PHYS_PAGE(&vm_pages[i]) != next_ppnum_in_run) {
 
                        if (ppnm == NULL || VM_PAGE_GET_PHYS_PAGE(&vm_pages[i]) != next_ppnum_in_run) {

-
-                               ppnm = kalloc(sizeof(struct ppnum_mapping));
+                               ppnm = zalloc_permanent_type(struct ppnum_mapping);

 
                                ppnm->ppnm_next = ppnm_head;
                                ppnm_head = ppnm;
 
                                ppnm->ppnm_next = ppnm_head;
                                ppnm_head = ppnm;


@@ -7544,51 +8063,51 @@ ppnum_t
 hibernate_lookup_paddr(unsigned int indx)
 {
        struct ppnum_mapping *ppnm = NULL;
 hibernate_lookup_paddr(unsigned int indx)
 {
        struct ppnum_mapping *ppnm = NULL;

-       
+

        ppnm = ppnm_last_found;
 
        if (ppnm) {
        ppnm = ppnm_last_found;
 
        if (ppnm) {

-               if (indx >= ppnm->ppnm_sindx && indx < ppnm->ppnm_eindx)
+               if (indx >= ppnm->ppnm_sindx && indx < ppnm->ppnm_eindx) {

                        goto done;
                        goto done;

+               }

        }
        for (ppnm = ppnm_head; ppnm; ppnm = ppnm->ppnm_next) {
        }
        for (ppnm = ppnm_head; ppnm; ppnm = ppnm->ppnm_next) {

-

                if (indx >= ppnm->ppnm_sindx && indx < ppnm->ppnm_eindx) {
                        ppnm_last_found = ppnm;
                        break;
                }
        }
                if (indx >= ppnm->ppnm_sindx && indx < ppnm->ppnm_eindx) {
                        ppnm_last_found = ppnm;
                        break;
                }
        }

-       if (ppnm == NULL)
+       if (ppnm == NULL) {

                panic("hibernate_lookup_paddr of %d failed\n", indx);
                panic("hibernate_lookup_paddr of %d failed\n", indx);

+       }

 done:
 done:

-       return (ppnm->ppnm_base_paddr + (indx - ppnm->ppnm_sindx));
+       return ppnm->ppnm_base_paddr + (indx - ppnm->ppnm_sindx);

 }
 
 
 uint32_t
 hibernate_mark_as_unneeded(addr64_t saddr, addr64_t eaddr, hibernate_page_list_t *page_list, hibernate_page_list_t *page_list_wired)
 {
 }
 
 
 uint32_t
 hibernate_mark_as_unneeded(addr64_t saddr, addr64_t eaddr, hibernate_page_list_t *page_list, hibernate_page_list_t *page_list_wired)
 {

-       addr64_t        saddr_aligned;
-       addr64_t        eaddr_aligned;
-       addr64_t        addr;
-       ppnum_t         paddr;
-       unsigned int    mark_as_unneeded_pages = 0;
+       addr64_t        saddr_aligned;
+       addr64_t        eaddr_aligned;
+       addr64_t        addr;
+       ppnum_t         paddr;
+       unsigned int    mark_as_unneeded_pages = 0;

 
        saddr_aligned = (saddr + PAGE_MASK_64) & ~PAGE_MASK_64;
        eaddr_aligned = eaddr & ~PAGE_MASK_64;
 
        for (addr = saddr_aligned; addr < eaddr_aligned; addr += PAGE_SIZE_64) {
 
        saddr_aligned = (saddr + PAGE_MASK_64) & ~PAGE_MASK_64;
        eaddr_aligned = eaddr & ~PAGE_MASK_64;
 
        for (addr = saddr_aligned; addr < eaddr_aligned; addr += PAGE_SIZE_64) {

-

                paddr = pmap_find_phys(kernel_pmap, addr);
 
                assert(paddr);
 
                paddr = pmap_find_phys(kernel_pmap, addr);
 
                assert(paddr);
 

-               hibernate_page_bitset(page_list,       TRUE, paddr);
+               hibernate_page_bitset(page_list, TRUE, paddr);

                hibernate_page_bitset(page_list_wired, TRUE, paddr);
 
                mark_as_unneeded_pages++;
        }
                hibernate_page_bitset(page_list_wired, TRUE, paddr);
 
                mark_as_unneeded_pages++;
        }

-       return (mark_as_unneeded_pages);
+       return mark_as_unneeded_pages;

 }
 
 
 }
 
 


@@ -7596,8 +8115,8 @@ void
 hibernate_hash_insert_page(vm_page_t mem)
 {
        vm_page_bucket_t *bucket;
 hibernate_hash_insert_page(vm_page_t mem)
 {
        vm_page_bucket_t *bucket;

-       int             hash_id;
-       vm_object_t     m_object;
+       int             hash_id;
+       vm_object_t     m_object;

 
        m_object = VM_PAGE_OBJECT(mem);
 
 
        m_object = VM_PAGE_OBJECT(mem);
 


@@ -7619,8 +8138,8 @@ hibernate_hash_insert_page(vm_page_t mem)
 void
 hibernate_free_range(int sindx, int eindx)
 {
 void
 hibernate_free_range(int sindx, int eindx)
 {

-       vm_page_t       mem;
-       unsigned int    color;
+       vm_page_t       mem;
+       unsigned int    color;

 
        while (sindx < eindx) {
                mem = &vm_pages[sindx];
 
        while (sindx < eindx) {
                mem = &vm_pages[sindx];


@@ -7632,15 +8151,9 @@ hibernate_free_range(int sindx, int eindx)
 
                color = VM_PAGE_GET_COLOR(mem);
 #if defined(__x86_64__)
 
                color = VM_PAGE_GET_COLOR(mem);
 #if defined(__x86_64__)

-               vm_page_queue_enter_clump(&vm_page_queue_free[color].qhead,
-                                         mem,
-                                         vm_page_t,
-                                         vmp_pageq);
+               vm_page_queue_enter_clump(&vm_page_queue_free[color].qhead, mem);

 #else
 #else

-               vm_page_queue_enter(&vm_page_queue_free[color].qhead,
-                                         mem,
-                                         vm_page_t,
-                                         vmp_pageq);
+               vm_page_queue_enter(&vm_page_queue_free[color].qhead, mem, vmp_pageq);

 #endif
                vm_page_free_count++;
 
 #endif
                vm_page_free_count++;
 


@@ -7648,40 +8161,38 @@ hibernate_free_range(int sindx, int eindx)
        }
 }
 
        }
 }
 

-
-extern void hibernate_rebuild_pmap_structs(void);
-

 void
 hibernate_rebuild_vm_structs(void)
 {
 void
 hibernate_rebuild_vm_structs(void)
 {

-       int             i, cindx, sindx, eindx;
-       vm_page_t       mem, tmem, mem_next;
-       AbsoluteTime    startTime, endTime;
-       uint64_t        nsec;
+       int             i, cindx, sindx, eindx;
+       vm_page_t       mem, tmem, mem_next;
+       AbsoluteTime    startTime, endTime;
+       uint64_t        nsec;

 
 

-       if (hibernate_rebuild_needed == FALSE)
+       if (hibernate_rebuild_needed == FALSE) {

                return;
                return;

+       }

 
        KDBG(IOKDBG_CODE(DBG_HIBERNATE, 13) | DBG_FUNC_START);
        HIBLOG("hibernate_rebuild started\n");
 
        clock_get_uptime(&startTime);
 
 
        KDBG(IOKDBG_CODE(DBG_HIBERNATE, 13) | DBG_FUNC_START);
        HIBLOG("hibernate_rebuild started\n");
 
        clock_get_uptime(&startTime);
 

-       hibernate_rebuild_pmap_structs();
+       pal_hib_rebuild_pmap_structs();

 
        bzero(&vm_page_buckets[0], vm_page_bucket_count * sizeof(vm_page_bucket_t));
        eindx = vm_pages_count;
 
        /*
 
        bzero(&vm_page_buckets[0], vm_page_bucket_count * sizeof(vm_page_bucket_t));
        eindx = vm_pages_count;
 
        /*

-        * Mark all the vm_pages[] that have not been initialized yet as being 
+        * Mark all the vm_pages[] that have not been initialized yet as being

         * transient. This is needed to ensure that buddy page search is corrrect.
         * transient. This is needed to ensure that buddy page search is corrrect.

-        * Without this random data in these vm_pages[] can trip the buddy search 
+        * Without this random data in these vm_pages[] can trip the buddy search

         */
         */

-       for (i = hibernate_teardown_last_valid_compact_indx+1; i < eindx; ++i) 
+       for (i = hibernate_teardown_last_valid_compact_indx + 1; i < eindx; ++i) {

                vm_pages[i].vmp_q_state = VM_PAGE_NOT_ON_Q;
                vm_pages[i].vmp_q_state = VM_PAGE_NOT_ON_Q;

+       }

 
        for (cindx = hibernate_teardown_last_valid_compact_indx; cindx >= 0; cindx--) {
 
        for (cindx = hibernate_teardown_last_valid_compact_indx; cindx >= 0; cindx--) {

-               

                mem = &vm_pages[cindx];
                assert(mem->vmp_q_state != VM_PAGE_ON_FREE_Q);
                /*
                mem = &vm_pages[cindx];
                assert(mem->vmp_q_state != VM_PAGE_ON_FREE_Q);
                /*


@@ -7701,25 +8212,27 @@ hibernate_rebuild_vm_structs(void)
                        *tmem = *mem;
                        mem = tmem;
                }
                        *tmem = *mem;
                        mem = tmem;
                }

-               if (mem->vmp_hashed)
+               if (mem->vmp_hashed) {

                        hibernate_hash_insert_page(mem);
                        hibernate_hash_insert_page(mem);

+               }

                /*
                 * the 'hole' between this vm_page_t and the previous
                /*
                 * the 'hole' between this vm_page_t and the previous

-                * vm_page_t we moved needs to be initialized as 
+                * vm_page_t we moved needs to be initialized as

                 * a range of free vm_page_t's
                 */
                hibernate_free_range(sindx + 1, eindx);
 
                eindx = sindx;
        }
                 * a range of free vm_page_t's
                 */
                hibernate_free_range(sindx + 1, eindx);
 
                eindx = sindx;
        }

-       if (sindx)
+       if (sindx) {

                hibernate_free_range(0, sindx);
                hibernate_free_range(0, sindx);

+       }

 
        assert(vm_page_free_count == hibernate_teardown_vm_page_free_count);
 
        /*
         * process the list of vm_page_t's that were entered in the hash,
 
        assert(vm_page_free_count == hibernate_teardown_vm_page_free_count);
 
        /*
         * process the list of vm_page_t's that were entered in the hash,

-        * but were not located in the vm_pages arrary... these are 
+        * but were not located in the vm_pages arrary... these are

         * vm_page_t's that were created on the fly (i.e. fictitious)
         */
        for (mem = hibernate_rebuild_hash_list; mem; mem = mem_next) {
         * vm_page_t's that were created on the fly (i.e. fictitious)
         */
        for (mem = hibernate_rebuild_hash_list; mem; mem = mem_next) {


@@ -7730,9 +8243,9 @@ hibernate_rebuild_vm_structs(void)
        }
        hibernate_rebuild_hash_list = NULL;
 
        }
        hibernate_rebuild_hash_list = NULL;
 

-        clock_get_uptime(&endTime);
-        SUB_ABSOLUTETIME(&endTime, &startTime);
-        absolutetime_to_nanoseconds(endTime, &nsec);
+       clock_get_uptime(&endTime);
+       SUB_ABSOLUTETIME(&endTime, &startTime);
+       absolutetime_to_nanoseconds(endTime, &nsec);

 
        HIBLOG("hibernate_rebuild completed - took %qd msecs\n", nsec / 1000000ULL);
 
 
        HIBLOG("hibernate_rebuild completed - took %qd msecs\n", nsec / 1000000ULL);
 


@@ -7741,35 +8254,32 @@ hibernate_rebuild_vm_structs(void)
        KDBG(IOKDBG_CODE(DBG_HIBERNATE, 13) | DBG_FUNC_END);
 }
 
        KDBG(IOKDBG_CODE(DBG_HIBERNATE, 13) | DBG_FUNC_END);
 }
 

-
-extern void hibernate_teardown_pmap_structs(addr64_t *, addr64_t *);
-

 uint32_t
 hibernate_teardown_vm_structs(hibernate_page_list_t *page_list, hibernate_page_list_t *page_list_wired)
 {
 uint32_t
 hibernate_teardown_vm_structs(hibernate_page_list_t *page_list, hibernate_page_list_t *page_list_wired)
 {

-       unsigned int    i;
-       unsigned int    compact_target_indx;
-       vm_page_t       mem, mem_next;
+       unsigned int    i;
+       unsigned int    compact_target_indx;
+       vm_page_t       mem, mem_next;

        vm_page_bucket_t *bucket;
        vm_page_bucket_t *bucket;

-       unsigned int    mark_as_unneeded_pages = 0;
-       unsigned int    unneeded_vm_page_bucket_pages = 0;
-       unsigned int    unneeded_vm_pages_pages = 0;
-       unsigned int    unneeded_pmap_pages = 0;
-       addr64_t        start_of_unneeded = 0;
-       addr64_t        end_of_unneeded = 0;
+       unsigned int    mark_as_unneeded_pages = 0;
+       unsigned int    unneeded_vm_page_bucket_pages = 0;
+       unsigned int    unneeded_vm_pages_pages = 0;
+       unsigned int    unneeded_pmap_pages = 0;
+       addr64_t        start_of_unneeded = 0;
+       addr64_t        end_of_unneeded = 0;
+

 
 

-       
-       if (hibernate_should_abort())
-               return (0);
+       if (hibernate_should_abort()) {
+               return 0;
+       }

 
        hibernate_rebuild_needed = TRUE;
 
        HIBLOG("hibernate_teardown: wired_pages %d, free_pages %d, active_pages %d, inactive_pages %d, speculative_pages %d, cleaned_pages %d, compressor_pages %d\n",
 
        hibernate_rebuild_needed = TRUE;
 
        HIBLOG("hibernate_teardown: wired_pages %d, free_pages %d, active_pages %d, inactive_pages %d, speculative_pages %d, cleaned_pages %d, compressor_pages %d\n",

-              vm_page_wire_count, vm_page_free_count, vm_page_active_count, vm_page_inactive_count, vm_page_speculative_count,
-              vm_page_cleaned_count, compressor_object->resident_page_count);
+           vm_page_wire_count, vm_page_free_count, vm_page_active_count, vm_page_inactive_count, vm_page_speculative_count,
+           vm_page_cleaned_count, compressor_object->resident_page_count);

 
        for (i = 0; i < vm_page_bucket_count; i++) {
 
        for (i = 0; i < vm_page_bucket_count; i++) {

-

                bucket = &vm_page_buckets[i];
 
                for (mem = (vm_page_t)(VM_PAGE_UNPACK_PTR(bucket->page_list)); mem != VM_PAGE_NULL; mem = mem_next) {
                bucket = &vm_page_buckets[i];
 
                for (mem = (vm_page_t)(VM_PAGE_UNPACK_PTR(bucket->page_list)); mem != VM_PAGE_NULL; mem = mem_next) {


@@ -7791,7 +8301,6 @@ hibernate_teardown_vm_structs(hibernate_page_list_t *page_list, hibernate_page_l
        compact_target_indx = 0;
 
        for (i = 0; i < vm_pages_count; i++) {
        compact_target_indx = 0;
 
        for (i = 0; i < vm_pages_count; i++) {

-

                mem = &vm_pages[i];
 
                if (mem->vmp_q_state == VM_PAGE_ON_FREE_Q) {
                mem = &vm_pages[i];
 
                if (mem->vmp_q_state == VM_PAGE_ON_FREE_Q) {


@@ -7802,10 +8311,7 @@ hibernate_teardown_vm_structs(hibernate_page_list_t *page_list, hibernate_page_l
 
                        color = VM_PAGE_GET_COLOR(mem);
 
 
                        color = VM_PAGE_GET_COLOR(mem);
 

-                       vm_page_queue_remove(&vm_page_queue_free[color].qhead,
-                                            mem,
-                                            vm_page_t,
-                                            vmp_pageq);
+                       vm_page_queue_remove(&vm_page_queue_free[color].qhead, mem, vmp_pageq);

 
                        VM_PAGE_ZERO_PAGEQ_ENTRY(mem);
 
 
                        VM_PAGE_ZERO_PAGEQ_ENTRY(mem);
 


@@ -7813,8 +8319,9 @@ hibernate_teardown_vm_structs(hibernate_page_list_t *page_list, hibernate_page_l
 
                        hibernate_teardown_found_free_pages++;
 
 
                        hibernate_teardown_found_free_pages++;
 

-                       if (vm_pages[compact_target_indx].vmp_q_state != VM_PAGE_ON_FREE_Q)
+                       if (vm_pages[compact_target_indx].vmp_q_state != VM_PAGE_ON_FREE_Q) {

                                compact_target_indx = i;
                                compact_target_indx = i;

+                       }

                } else {
                        /*
                         * record this vm_page_t's original location
                } else {
                        /*
                         * record this vm_page_t's original location


@@ -7834,15 +8341,16 @@ hibernate_teardown_vm_structs(hibernate_page_list_t *page_list, hibernate_page_l
 
                                hibernate_teardown_last_valid_compact_indx = compact_target_indx;
                                compact_target_indx++;
 
                                hibernate_teardown_last_valid_compact_indx = compact_target_indx;
                                compact_target_indx++;

-                       } else
+                       } else {

                                hibernate_teardown_last_valid_compact_indx = i;
                                hibernate_teardown_last_valid_compact_indx = i;

+                       }

                }
        }
                }
        }

-       unneeded_vm_pages_pages = hibernate_mark_as_unneeded((addr64_t)&vm_pages[hibernate_teardown_last_valid_compact_indx+1],
-                                                            (addr64_t)&vm_pages[vm_pages_count-1], page_list, page_list_wired);
+       unneeded_vm_pages_pages = hibernate_mark_as_unneeded((addr64_t)&vm_pages[hibernate_teardown_last_valid_compact_indx + 1],
+           (addr64_t)&vm_pages[vm_pages_count - 1], page_list, page_list_wired);

        mark_as_unneeded_pages += unneeded_vm_pages_pages;
 
        mark_as_unneeded_pages += unneeded_vm_pages_pages;
 

-       hibernate_teardown_pmap_structs(&start_of_unneeded, &end_of_unneeded);
+       pal_hib_teardown_pmap_structs(&start_of_unneeded, &end_of_unneeded);

 
        if (start_of_unneeded) {
                unneeded_pmap_pages = hibernate_mark_as_unneeded(start_of_unneeded, end_of_unneeded, page_list, page_list_wired);
 
        if (start_of_unneeded) {
                unneeded_pmap_pages = hibernate_mark_as_unneeded(start_of_unneeded, end_of_unneeded, page_list, page_list_wired);


@@ -7850,7 +8358,7 @@ hibernate_teardown_vm_structs(hibernate_page_list_t *page_list, hibernate_page_l
        }
        HIBLOG("hibernate_teardown: mark_as_unneeded_pages %d, %d, %d\n", unneeded_vm_page_bucket_pages, unneeded_vm_pages_pages, unneeded_pmap_pages);
 
        }
        HIBLOG("hibernate_teardown: mark_as_unneeded_pages %d, %d, %d\n", unneeded_vm_page_bucket_pages, unneeded_vm_pages_pages, unneeded_pmap_pages);
 

-       return (mark_as_unneeded_pages);
+       return mark_as_unneeded_pages;

 }
 
 
 }
 
 


@@ -7859,7 +8367,7 @@ hibernate_teardown_vm_structs(hibernate_page_list_t *page_list, hibernate_page_l
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 #include <mach_vm_debug.h>
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 #include <mach_vm_debug.h>

-#if    MACH_VM_DEBUG
+#if     MACH_VM_DEBUG

 
 #include <mach_debug/hash_info.h>
 #include <vm/vm_debug.h>
 
 #include <mach_debug/hash_info.h>
 #include <vm/vm_debug.h>


@@ -7881,10 +8389,11 @@ vm_page_info(
        unsigned int count)
 {
        unsigned int i;
        unsigned int count)
 {
        unsigned int i;

-       lck_spin_t      *bucket_lock;
+       lck_spin_t      *bucket_lock;

 
 

-       if (vm_page_bucket_count < count)
+       if (vm_page_bucket_count < count) {

                count = vm_page_bucket_count;
                count = vm_page_bucket_count;

+       }

 
        for (i = 0; i < count; i++) {
                vm_page_bucket_t *bucket = &vm_page_buckets[i];
 
        for (i = 0; i < count; i++) {
                vm_page_bucket_t *bucket = &vm_page_buckets[i];


@@ -7892,12 +8401,13 @@ vm_page_info(
                vm_page_t m;
 
                bucket_lock = &vm_page_bucket_locks[i / BUCKETS_PER_LOCK];
                vm_page_t m;
 
                bucket_lock = &vm_page_bucket_locks[i / BUCKETS_PER_LOCK];

-               lck_spin_lock(bucket_lock);
+               lck_spin_lock_grp(bucket_lock, &vm_page_lck_grp_bucket);

 
                for (m = (vm_page_t)(VM_PAGE_UNPACK_PTR(bucket->page_list));
 
                for (m = (vm_page_t)(VM_PAGE_UNPACK_PTR(bucket->page_list));

-                    m != VM_PAGE_NULL;
-                    m = (vm_page_t)(VM_PAGE_UNPACK_PTR(m->vmp_next_m)))
+                   m != VM_PAGE_NULL;
+                   m = (vm_page_t)(VM_PAGE_UNPACK_PTR(m->vmp_next_m))) {

                        bucket_count++;
                        bucket_count++;

+               }

 
                lck_spin_unlock(bucket_lock);
 
 
                lck_spin_unlock(bucket_lock);
 


@@ -7907,7 +8417,7 @@ vm_page_info(
 
        return vm_page_bucket_count;
 }
 
        return vm_page_bucket_count;
 }

-#endif /* MACH_VM_DEBUG */
+#endif  /* MACH_VM_DEBUG */

 
 #if VM_PAGE_BUCKETS_CHECK
 void
 
 #if VM_PAGE_BUCKETS_CHECK
 void


@@ -7917,7 +8427,7 @@ vm_page_buckets_check(void)
        vm_page_t p;
        unsigned int p_hash;
        vm_page_bucket_t *bucket;
        vm_page_t p;
        unsigned int p_hash;
        vm_page_bucket_t *bucket;

-       lck_spin_t      *bucket_lock;
+       lck_spin_t      *bucket_lock;

 
        if (!vm_page_buckets_check_ready) {
                return;
 
        if (!vm_page_buckets_check_ready) {
                return;


@@ -7927,29 +8437,29 @@ vm_page_buckets_check(void)
        if (hibernate_rebuild_needed ||
            hibernate_rebuild_hash_list) {
                panic("BUCKET_CHECK: hibernation in progress: "
        if (hibernate_rebuild_needed ||
            hibernate_rebuild_hash_list) {
                panic("BUCKET_CHECK: hibernation in progress: "

-                     "rebuild_needed=%d rebuild_hash_list=%p\n",
-                     hibernate_rebuild_needed,
-                     hibernate_rebuild_hash_list);
+                   "rebuild_needed=%d rebuild_hash_list=%p\n",
+                   hibernate_rebuild_needed,
+                   hibernate_rebuild_hash_list);

        }
 #endif /* HIBERNATION */
 
 #if VM_PAGE_FAKE_BUCKETS
        char *cp;
        for (cp = (char *) vm_page_fake_buckets_start;
        }
 #endif /* HIBERNATION */
 
 #if VM_PAGE_FAKE_BUCKETS
        char *cp;
        for (cp = (char *) vm_page_fake_buckets_start;

-            cp < (char *) vm_page_fake_buckets_end;
-            cp++) {
+           cp < (char *) vm_page_fake_buckets_end;
+           cp++) {

                if (*cp != 0x5a) {
                        panic("BUCKET_CHECK: corruption at %p in fake buckets "
                if (*cp != 0x5a) {
                        panic("BUCKET_CHECK: corruption at %p in fake buckets "

-                             "[0x%llx:0x%llx]\n",
-                             cp,
-                             (uint64_t) vm_page_fake_buckets_start,
-                             (uint64_t) vm_page_fake_buckets_end);
+                           "[0x%llx:0x%llx]\n",
+                           cp,
+                           (uint64_t) vm_page_fake_buckets_start,
+                           (uint64_t) vm_page_fake_buckets_end);

                }
        }
 #endif /* VM_PAGE_FAKE_BUCKETS */
 
        for (i = 0; i < vm_page_bucket_count; i++) {
                }
        }
 #endif /* VM_PAGE_FAKE_BUCKETS */
 
        for (i = 0; i < vm_page_bucket_count; i++) {

-               vm_object_t     p_object;
+               vm_object_t     p_object;

 
                bucket = &vm_page_buckets[i];
                if (!bucket->page_list) {
 
                bucket = &vm_page_buckets[i];
                if (!bucket->page_list) {


@@ -7957,7 +8467,7 @@ vm_page_buckets_check(void)
                }
 
                bucket_lock = &vm_page_bucket_locks[i / BUCKETS_PER_LOCK];
                }
 
                bucket_lock = &vm_page_bucket_locks[i / BUCKETS_PER_LOCK];

-               lck_spin_lock(bucket_lock);
+               lck_spin_lock_grp(bucket_lock, &vm_page_lck_grp_bucket);

                p = (vm_page_t)(VM_PAGE_UNPACK_PTR(bucket->page_list));
 
                while (p != VM_PAGE_NULL) {
                p = (vm_page_t)(VM_PAGE_UNPACK_PTR(bucket->page_list));
 
                while (p != VM_PAGE_NULL) {


@@ -7965,18 +8475,18 @@ vm_page_buckets_check(void)
 
                        if (!p->vmp_hashed) {
                                panic("BUCKET_CHECK: page %p (%p,0x%llx) "
 
                        if (!p->vmp_hashed) {
                                panic("BUCKET_CHECK: page %p (%p,0x%llx) "

-                                     "hash %d in bucket %d at %p "
-                                     "is not hashed\n",
-                                     p, p_object, p->vmp_offset,
-                                     p_hash, i, bucket);
+                                   "hash %d in bucket %d at %p "
+                                   "is not hashed\n",
+                                   p, p_object, p->vmp_offset,
+                                   p_hash, i, bucket);

                        }
                        p_hash = vm_page_hash(p_object, p->vmp_offset);
                        if (p_hash != i) {
                                panic("BUCKET_CHECK: corruption in bucket %d "
                        }
                        p_hash = vm_page_hash(p_object, p->vmp_offset);
                        if (p_hash != i) {
                                panic("BUCKET_CHECK: corruption in bucket %d "

-                                     "at %p: page %p object %p offset 0x%llx "
-                                     "hash %d\n",
-                                     i, bucket, p, p_object, p->vmp_offset,
-                                     p_hash);
+                                   "at %p: page %p object %p offset 0x%llx "
+                                   "hash %d\n",
+                                   i, bucket, p, p_object, p->vmp_offset,
+                                   p_hash);

                        }
                        p = (vm_page_t)(VM_PAGE_UNPACK_PTR(p->vmp_next_m));
                }
                        }
                        p = (vm_page_t)(VM_PAGE_UNPACK_PTR(p->vmp_next_m));
                }


@@ -8011,15 +8521,14 @@ void
 vm_page_queues_remove(vm_page_t mem, boolean_t __unused remove_from_backgroundq)
 #endif
 {
 vm_page_queues_remove(vm_page_t mem, boolean_t __unused remove_from_backgroundq)
 #endif
 {

-       boolean_t       was_pageable = TRUE;
-       vm_object_t     m_object;
+       boolean_t       was_pageable = TRUE;
+       vm_object_t     m_object;

 
        m_object = VM_PAGE_OBJECT(mem);
 
        LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
 
 
        m_object = VM_PAGE_OBJECT(mem);
 
        LCK_MTX_ASSERT(&vm_page_queue_lock, LCK_MTX_ASSERT_OWNED);
 

-       if (mem->vmp_q_state == VM_PAGE_NOT_ON_Q)
-       {
+       if (mem->vmp_q_state == VM_PAGE_NOT_ON_Q) {

                assert(mem->vmp_pageq.next == 0 && mem->vmp_pageq.prev == 0);
 #if CONFIG_BACKGROUND_QUEUE
                if (remove_from_backgroundq == TRUE) {
                assert(mem->vmp_pageq.next == 0 && mem->vmp_pageq.prev == 0);
 #if CONFIG_BACKGROUND_QUEUE
                if (remove_from_backgroundq == TRUE) {


@@ -8036,13 +8545,12 @@ vm_page_queues_remove(vm_page_t mem, boolean_t __unused remove_from_backgroundq)
                return;
        }
 
                return;
        }
 

-       if (mem->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR)
-       {
+       if (mem->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) {

                assert(mem->vmp_pageq.next == 0 && mem->vmp_pageq.prev == 0);
 #if CONFIG_BACKGROUND_QUEUE
                assert(mem->vmp_backgroundq.next == 0 &&
                assert(mem->vmp_pageq.next == 0 && mem->vmp_pageq.prev == 0);
 #if CONFIG_BACKGROUND_QUEUE
                assert(mem->vmp_backgroundq.next == 0 &&

-                      mem->vmp_backgroundq.prev == 0 &&
-                      mem->vmp_on_backgroundq == FALSE);
+                   mem->vmp_backgroundq.prev == 0 &&
+                   mem->vmp_on_backgroundq == FALSE);

 #endif
                return;
        }
 #endif
                return;
        }


@@ -8054,8 +8562,8 @@ vm_page_queues_remove(vm_page_t mem, boolean_t __unused remove_from_backgroundq)
                assert(mem->vmp_pageq.next == 0 && mem->vmp_pageq.prev == 0);
 #if CONFIG_BACKGROUND_QUEUE
                assert(mem->vmp_backgroundq.next == 0 &&
                assert(mem->vmp_pageq.next == 0 && mem->vmp_pageq.prev == 0);
 #if CONFIG_BACKGROUND_QUEUE
                assert(mem->vmp_backgroundq.next == 0 &&

-                      mem->vmp_backgroundq.prev == 0 &&
-                      mem->vmp_on_backgroundq == FALSE);
+                   mem->vmp_backgroundq.prev == 0 &&
+                   mem->vmp_on_backgroundq == FALSE);

 #endif
                return;
        }
 #endif
                return;
        }


@@ -8065,16 +8573,14 @@ vm_page_queues_remove(vm_page_t mem, boolean_t __unused remove_from_backgroundq)
        assert(m_object != vm_submap_object);
        assert(!mem->vmp_fictitious);
 
        assert(m_object != vm_submap_object);
        assert(!mem->vmp_fictitious);
 

-       switch(mem->vmp_q_state) {
-               
+       switch (mem->vmp_q_state) {

        case VM_PAGE_ON_ACTIVE_LOCAL_Q:
        {
        case VM_PAGE_ON_ACTIVE_LOCAL_Q:
        {

-               struct vpl      *lq;
+               struct vpl      *lq;

 
 

-               lq = &vm_page_local_q[mem->vmp_local_id].vpl_un.vpl;
+               lq = zpercpu_get_cpu(vm_page_local_q, mem->vmp_local_id);

                VPL_LOCK(&lq->vpl_lock);
                VPL_LOCK(&lq->vpl_lock);

-               vm_page_queue_remove(&lq->vpl_queue,
-                                    mem, vm_page_t, vmp_pageq);
+               vm_page_queue_remove(&lq->vpl_queue, mem, vmp_pageq);

                mem->vmp_local_id = 0;
                lq->vpl_count--;
                if (m_object->internal) {
                mem->vmp_local_id = 0;
                lq->vpl_count--;
                if (m_object->internal) {


@@ -8088,8 +8594,7 @@ vm_page_queues_remove(vm_page_t mem, boolean_t __unused remove_from_backgroundq)
        }
        case VM_PAGE_ON_ACTIVE_Q:
        {
        }
        case VM_PAGE_ON_ACTIVE_Q:
        {

-               vm_page_queue_remove(&vm_page_queue_active,
-                                    mem, vm_page_t, vmp_pageq);
+               vm_page_queue_remove(&vm_page_queue_active, mem, vmp_pageq);

                vm_page_active_count--;
                break;
        }
                vm_page_active_count--;
                break;
        }


@@ -8099,8 +8604,7 @@ vm_page_queues_remove(vm_page_t mem, boolean_t __unused remove_from_backgroundq)
                assert(m_object->internal == TRUE);
 
                vm_page_inactive_count--;
                assert(m_object->internal == TRUE);
 
                vm_page_inactive_count--;

-               vm_page_queue_remove(&vm_page_queue_anonymous,
-                                    mem, vm_page_t, vmp_pageq);
+               vm_page_queue_remove(&vm_page_queue_anonymous, mem, vmp_pageq);

                vm_page_anonymous_count--;
 
                vm_purgeable_q_advance_all();
                vm_page_anonymous_count--;
 
                vm_purgeable_q_advance_all();


@@ -8113,8 +8617,7 @@ vm_page_queues_remove(vm_page_t mem, boolean_t __unused remove_from_backgroundq)
                assert(m_object->internal == FALSE);
 
                vm_page_inactive_count--;
                assert(m_object->internal == FALSE);
 
                vm_page_inactive_count--;

-               vm_page_queue_remove(&vm_page_queue_inactive,
-                                    mem, vm_page_t, vmp_pageq);
+               vm_page_queue_remove(&vm_page_queue_inactive, mem, vmp_pageq);

                vm_purgeable_q_advance_all();
                vm_page_balance_inactive(3);
                break;
                vm_purgeable_q_advance_all();
                vm_page_balance_inactive(3);
                break;


@@ -8125,8 +8628,7 @@ vm_page_queues_remove(vm_page_t mem, boolean_t __unused remove_from_backgroundq)
                assert(m_object->internal == FALSE);
 
                vm_page_inactive_count--;
                assert(m_object->internal == FALSE);
 
                vm_page_inactive_count--;

-               vm_page_queue_remove(&vm_page_queue_cleaned,
-                                    mem, vm_page_t, vmp_pageq);
+               vm_page_queue_remove(&vm_page_queue_cleaned, mem, vmp_pageq);

                vm_page_cleaned_count--;
                vm_page_balance_inactive(3);
                break;
                vm_page_cleaned_count--;
                vm_page_balance_inactive(3);
                break;


@@ -8136,8 +8638,7 @@ vm_page_queues_remove(vm_page_t mem, boolean_t __unused remove_from_backgroundq)
        {
                assert(m_object->internal == TRUE);
 
        {
                assert(m_object->internal == TRUE);
 

-               vm_page_queue_remove(&vm_page_queue_throttled,
-                                    mem, vm_page_t, vmp_pageq);
+               vm_page_queue_remove(&vm_page_queue_throttled, mem, vmp_pageq);

                vm_page_throttled_count--;
                was_pageable = FALSE;
                break;
                vm_page_throttled_count--;
                was_pageable = FALSE;
                break;


@@ -8147,7 +8648,7 @@ vm_page_queues_remove(vm_page_t mem, boolean_t __unused remove_from_backgroundq)
        {
                assert(m_object->internal == FALSE);
 
        {
                assert(m_object->internal == FALSE);
 

-                vm_page_remque(&mem->vmp_pageq);
+               vm_page_remque(&mem->vmp_pageq);

                vm_page_speculative_count--;
                vm_page_balance_inactive(3);
                break;
                vm_page_speculative_count--;
                vm_page_balance_inactive(3);
                break;


@@ -8156,9 +8657,9 @@ vm_page_queues_remove(vm_page_t mem, boolean_t __unused remove_from_backgroundq)
 #if CONFIG_SECLUDED_MEMORY
        case VM_PAGE_ON_SECLUDED_Q:
        {
 #if CONFIG_SECLUDED_MEMORY
        case VM_PAGE_ON_SECLUDED_Q:
        {

-               vm_page_queue_remove(&vm_page_queue_secluded,
-                                    mem, vm_page_t, vmp_pageq);
+               vm_page_queue_remove(&vm_page_queue_secluded, mem, vmp_pageq);

                vm_page_secluded_count--;
                vm_page_secluded_count--;

+               VM_PAGE_SECLUDED_COUNT_OVER_TARGET_UPDATE();

                if (m_object == VM_OBJECT_NULL) {
                        vm_page_secluded_count_free--;
                        was_pageable = FALSE;
                if (m_object == VM_OBJECT_NULL) {
                        vm_page_secluded_count_free--;
                        was_pageable = FALSE;


@@ -8176,10 +8677,10 @@ vm_page_queues_remove(vm_page_t mem, boolean_t __unused remove_from_backgroundq)
        {
                /*
                 *      if (mem->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q)
        {
                /*
                 *      if (mem->vmp_q_state == VM_PAGE_ON_PAGEOUT_Q)

-                *              NOTE: vm_page_queues_remove does not deal with removing pages from the pageout queue...
-                *              the caller is responsible for determing if the page is on that queue, and if so, must
-                *              either first remove it (it needs both the page queues lock and the object lock to do
-                *              this via vm_pageout_steal_laundry), or avoid the call to vm_page_queues_remove
+                *              NOTE: vm_page_queues_remove does not deal with removing pages from the pageout queue...
+                *              the caller is responsible for determing if the page is on that queue, and if so, must
+                *              either first remove it (it needs both the page queues lock and the object lock to do
+                *              this via vm_pageout_steal_laundry), or avoid the call to vm_page_queues_remove

                 *
                 *      we also don't expect to encounter VM_PAGE_ON_FREE_Q, VM_PAGE_ON_FREE_LOCAL_Q, VM_PAGE_ON_FREE_LOPAGE_Q
                 *      or any of the undefined states
                 *
                 *      we also don't expect to encounter VM_PAGE_ON_FREE_Q, VM_PAGE_ON_FREE_LOCAL_Q, VM_PAGE_ON_FREE_LOPAGE_Q
                 *      or any of the undefined states


@@ -8187,14 +8688,14 @@ vm_page_queues_remove(vm_page_t mem, boolean_t __unused remove_from_backgroundq)
                panic("vm_page_queues_remove - bad page q_state (%p, %d)\n", mem, mem->vmp_q_state);
                break;
        }
                panic("vm_page_queues_remove - bad page q_state (%p, %d)\n", mem, mem->vmp_q_state);
                break;
        }

-

        }
        VM_PAGE_ZERO_PAGEQ_ENTRY(mem);
        mem->vmp_q_state = VM_PAGE_NOT_ON_Q;
 
 #if CONFIG_BACKGROUND_QUEUE
        }
        VM_PAGE_ZERO_PAGEQ_ENTRY(mem);
        mem->vmp_q_state = VM_PAGE_NOT_ON_Q;
 
 #if CONFIG_BACKGROUND_QUEUE

-       if (remove_from_backgroundq == TRUE)
+       if (remove_from_backgroundq == TRUE) {

                vm_page_remove_from_backgroundq(mem);
                vm_page_remove_from_backgroundq(mem);

+       }

 #endif
        if (was_pageable) {
                if (m_object->internal) {
 #endif
        if (was_pageable) {
                if (m_object->internal) {


@@ -8210,8 +8711,8 @@ vm_page_remove_internal(vm_page_t page)
 {
        vm_object_t __object = VM_PAGE_OBJECT(page);
        if (page == __object->memq_hint) {
 {
        vm_object_t __object = VM_PAGE_OBJECT(page);
        if (page == __object->memq_hint) {

-               vm_page_t       __new_hint;
-               vm_page_queue_entry_t   __qe;
+               vm_page_t       __new_hint;
+               vm_page_queue_entry_t   __qe;

                __qe = (vm_page_queue_entry_t)vm_page_queue_next(&page->vmp_listq);
                if (vm_page_queue_end(&__object->memq, __qe)) {
                        __qe = (vm_page_queue_entry_t)vm_page_queue_prev(&page->vmp_listq);
                __qe = (vm_page_queue_entry_t)vm_page_queue_next(&page->vmp_listq);
                if (vm_page_queue_end(&__object->memq, __qe)) {
                        __qe = (vm_page_queue_entry_t)vm_page_queue_prev(&page->vmp_listq);


@@ -8222,7 +8723,7 @@ vm_page_remove_internal(vm_page_t page)
                __new_hint = (vm_page_t)((uintptr_t) __qe);
                __object->memq_hint = __new_hint;
        }
                __new_hint = (vm_page_t)((uintptr_t) __qe);
                __object->memq_hint = __new_hint;
        }

-       vm_page_queue_remove(&__object->memq, page, vm_page_t, vmp_listq);
+       vm_page_queue_remove(&__object->memq, page, vmp_listq);

 #if CONFIG_SECLUDED_MEMORY
        if (__object->eligible_for_secluded) {
                vm_page_secluded.eligible_for_secluded--;
 #if CONFIG_SECLUDED_MEMORY
        if (__object->eligible_for_secluded) {
                vm_page_secluded.eligible_for_secluded--;


@@ -8233,7 +8734,7 @@ vm_page_remove_internal(vm_page_t page)
 void
 vm_page_enqueue_inactive(vm_page_t mem, boolean_t first)
 {
 void
 vm_page_enqueue_inactive(vm_page_t mem, boolean_t first)
 {

-       vm_object_t     m_object;
+       vm_object_t     m_object;

 
        m_object = VM_PAGE_OBJECT(mem);
 
 
        m_object = VM_PAGE_OBJECT(mem);
 


@@ -8246,20 +8747,22 @@ vm_page_enqueue_inactive(vm_page_t mem, boolean_t first)
        if (m_object->internal) {
                mem->vmp_q_state = VM_PAGE_ON_INACTIVE_INTERNAL_Q;
 
        if (m_object->internal) {
                mem->vmp_q_state = VM_PAGE_ON_INACTIVE_INTERNAL_Q;
 

-               if (first == TRUE)
-                       vm_page_queue_enter_first(&vm_page_queue_anonymous, mem, vm_page_t, vmp_pageq);
-               else
-                       vm_page_queue_enter(&vm_page_queue_anonymous, mem, vm_page_t, vmp_pageq);
+               if (first == TRUE) {
+                       vm_page_queue_enter_first(&vm_page_queue_anonymous, mem, vmp_pageq);
+               } else {
+                       vm_page_queue_enter(&vm_page_queue_anonymous, mem, vmp_pageq);
+               }

 
                vm_page_anonymous_count++;
                vm_page_pageable_internal_count++;
        } else {
                mem->vmp_q_state = VM_PAGE_ON_INACTIVE_EXTERNAL_Q;
 
 
                vm_page_anonymous_count++;
                vm_page_pageable_internal_count++;
        } else {
                mem->vmp_q_state = VM_PAGE_ON_INACTIVE_EXTERNAL_Q;
 

-               if (first == TRUE)
-                       vm_page_queue_enter_first(&vm_page_queue_inactive, mem, vm_page_t, vmp_pageq);
-               else
-                       vm_page_queue_enter(&vm_page_queue_inactive, mem, vm_page_t, vmp_pageq);
+               if (first == TRUE) {
+                       vm_page_queue_enter_first(&vm_page_queue_inactive, mem, vmp_pageq);
+               } else {
+                       vm_page_queue_enter(&vm_page_queue_inactive, mem, vmp_pageq);
+               }

 
                vm_page_pageable_external_count++;
        }
 
                vm_page_pageable_external_count++;
        }


@@ -8267,15 +8770,16 @@ vm_page_enqueue_inactive(vm_page_t mem, boolean_t first)
        token_new_pagecount++;
 
 #if CONFIG_BACKGROUND_QUEUE
        token_new_pagecount++;
 
 #if CONFIG_BACKGROUND_QUEUE

-       if (mem->vmp_in_background)
+       if (mem->vmp_in_background) {

                vm_page_add_to_backgroundq(mem, FALSE);
                vm_page_add_to_backgroundq(mem, FALSE);

+       }

 #endif
 }
 
 void
 vm_page_enqueue_active(vm_page_t mem, boolean_t first)
 {
 #endif
 }
 
 void
 vm_page_enqueue_active(vm_page_t mem, boolean_t first)
 {

-       vm_object_t     m_object;
+       vm_object_t     m_object;

 
        m_object = VM_PAGE_OBJECT(mem);
 
 
        m_object = VM_PAGE_OBJECT(mem);
 


@@ -8286,10 +8790,11 @@ vm_page_enqueue_active(vm_page_t mem, boolean_t first)
        vm_page_check_pageable_safe(mem);
 
        mem->vmp_q_state = VM_PAGE_ON_ACTIVE_Q;
        vm_page_check_pageable_safe(mem);
 
        mem->vmp_q_state = VM_PAGE_ON_ACTIVE_Q;

-       if (first == TRUE)
-               vm_page_queue_enter_first(&vm_page_queue_active, mem, vm_page_t, vmp_pageq);
-       else
-               vm_page_queue_enter(&vm_page_queue_active, mem, vm_page_t, vmp_pageq);
+       if (first == TRUE) {
+               vm_page_queue_enter_first(&vm_page_queue_active, mem, vmp_pageq);
+       } else {
+               vm_page_queue_enter(&vm_page_queue_active, mem, vmp_pageq);
+       }

        vm_page_active_count++;
 
        if (m_object->internal) {
        vm_page_active_count++;
 
        if (m_object->internal) {


@@ -8299,8 +8804,9 @@ vm_page_enqueue_active(vm_page_t mem, boolean_t first)
        }
 
 #if CONFIG_BACKGROUND_QUEUE
        }
 
 #if CONFIG_BACKGROUND_QUEUE

-       if (mem->vmp_in_background)
+       if (mem->vmp_in_background) {

                vm_page_add_to_backgroundq(mem, FALSE);
                vm_page_add_to_backgroundq(mem, FALSE);

+       }

 #endif
        vm_page_balance_inactive(3);
 }
 #endif
        vm_page_balance_inactive(3);
 }


@@ -8312,264 +8818,289 @@ vm_page_enqueue_active(vm_page_t mem, boolean_t first)
 void
 vm_page_check_pageable_safe(vm_page_t page)
 {
 void
 vm_page_check_pageable_safe(vm_page_t page)
 {

-       vm_object_t     page_object;
+       vm_object_t     page_object;

 
        page_object = VM_PAGE_OBJECT(page);
 
        if (page_object == kernel_object) {
                panic("vm_page_check_pageable_safe: trying to add page" \
 
        page_object = VM_PAGE_OBJECT(page);
 
        if (page_object == kernel_object) {
                panic("vm_page_check_pageable_safe: trying to add page" \

-                        "from kernel object (%p) to pageable queue", kernel_object);
+                   "from kernel object (%p) to pageable queue", kernel_object);

        }
 
        if (page_object == compressor_object) {
                panic("vm_page_check_pageable_safe: trying to add page" \
        }
 
        if (page_object == compressor_object) {
                panic("vm_page_check_pageable_safe: trying to add page" \

-                        "from compressor object (%p) to pageable queue", compressor_object);
+                   "from compressor object (%p) to pageable queue", compressor_object);

        }
 
        if (page_object == vm_submap_object) {
                panic("vm_page_check_pageable_safe: trying to add page" \
        }
 
        if (page_object == vm_submap_object) {
                panic("vm_page_check_pageable_safe: trying to add page" \

-                       "from submap object (%p) to pageable queue", vm_submap_object);
+                   "from submap object (%p) to pageable queue", vm_submap_object);

        }
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
        }
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *

- * wired page diagnose
- * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+* wired page diagnose
+* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

 
 #include <libkern/OSKextLibPrivate.h>
 
 
 #include <libkern/OSKextLibPrivate.h>
 

-#define KA_SIZE(namelen, subtotalscount)       \
+#define KA_SIZE(namelen, subtotalscount)        \

        (sizeof(struct vm_allocation_site) + (namelen) + 1 + ((subtotalscount) * sizeof(struct vm_allocation_total)))
 
        (sizeof(struct vm_allocation_site) + (namelen) + 1 + ((subtotalscount) * sizeof(struct vm_allocation_total)))
 

-#define KA_NAME(alloc) \
+#define KA_NAME(alloc)  \

        ((char *)(&(alloc)->subtotals[(alloc->subtotalscount)]))
 
        ((char *)(&(alloc)->subtotals[(alloc->subtotalscount)]))
 

-#define KA_NAME_LEN(alloc)     \
+#define KA_NAME_LEN(alloc)      \

     (VM_TAG_NAME_LEN_MAX & (alloc->flags >> VM_TAG_NAME_LEN_SHIFT))
 
     (VM_TAG_NAME_LEN_MAX & (alloc->flags >> VM_TAG_NAME_LEN_SHIFT))
 

-vm_tag_t 
+vm_tag_t

 vm_tag_bt(void)
 {
 vm_tag_bt(void)
 {

-    uintptr_t* frameptr;
-    uintptr_t* frameptr_next;
-    uintptr_t retaddr;
-    uintptr_t kstackb, kstackt;
-    const vm_allocation_site_t * site;
-    thread_t cthread;
-    kern_allocation_name_t name;
-    
-    cthread = current_thread();
-    if (__improbable(cthread == NULL)) return VM_KERN_MEMORY_OSFMK;
-
-    if ((name = thread_get_kernel_state(cthread)->allocation_name))
-    {
-       if (!name->tag) vm_tag_alloc(name);
-       return name->tag;
-    }
-
-    kstackb = cthread->kernel_stack;
-    kstackt = kstackb + kernel_stack_size;
-
-    /* Load stack frame pointer (EBP on x86) into frameptr */
-    frameptr = __builtin_frame_address(0);
-    site = NULL;
-    while (frameptr != NULL) 
-    {
-       /* Verify thread stack bounds */
-       if (((uintptr_t)(frameptr + 2) > kstackt) || ((uintptr_t)frameptr < kstackb)) break;
-
-       /* Next frame pointer is pointed to by the previous one */
-       frameptr_next = (uintptr_t*) *frameptr;
-
-       /* Pull return address from one spot above the frame pointer */
-       retaddr = *(frameptr + 1);
-
-
-       if (((retaddr < vm_kernel_builtinkmod_text_end) && (retaddr >= vm_kernel_builtinkmod_text))
-               || (retaddr < vm_kernel_stext) || (retaddr > vm_kernel_top))
-       {
-               site = OSKextGetAllocationSiteForCaller(retaddr);
-               break;
+       uintptr_t* frameptr;
+       uintptr_t* frameptr_next;
+       uintptr_t retaddr;
+       uintptr_t kstackb, kstackt;
+       const vm_allocation_site_t * site;
+       thread_t cthread;
+       kern_allocation_name_t name;
+
+       cthread = current_thread();
+       if (__improbable(cthread == NULL)) {
+               return VM_KERN_MEMORY_OSFMK;
+       }
+
+       if ((name = thread_get_kernel_state(cthread)->allocation_name)) {
+               if (!name->tag) {
+                       vm_tag_alloc(name);
+               }
+               return name->tag;
+       }
+
+       kstackb = cthread->kernel_stack;
+       kstackt = kstackb + kernel_stack_size;
+
+       /* Load stack frame pointer (EBP on x86) into frameptr */
+       frameptr = __builtin_frame_address(0);
+       site = NULL;
+       while (frameptr != NULL) {
+               /* Verify thread stack bounds */
+               if (((uintptr_t)(frameptr + 2) > kstackt) || ((uintptr_t)frameptr < kstackb)) {
+                       break;
+               }
+
+               /* Next frame pointer is pointed to by the previous one */
+               frameptr_next = (uintptr_t*) *frameptr;
+
+               /* Pull return address from one spot above the frame pointer */
+               retaddr = *(frameptr + 1);
+
+#if defined(HAS_APPLE_PAC)
+               retaddr = (uintptr_t) ptrauth_strip((void *)retaddr, ptrauth_key_return_address);
+#endif
+
+               if (((retaddr < vm_kernel_builtinkmod_text_end) && (retaddr >= vm_kernel_builtinkmod_text))
+                   || (retaddr < vm_kernel_stext) || (retaddr > vm_kernel_top)) {
+                       site = OSKextGetAllocationSiteForCaller(retaddr);
+                       break;
+               }
+               frameptr = frameptr_next;

        }
        }

-       frameptr = frameptr_next;
-    }

 
 

-    return (site ? site->tag : VM_KERN_MEMORY_NONE);
+       return site ? site->tag : VM_KERN_MEMORY_NONE;

 }
 
 }
 

-static uint64_t free_tag_bits[VM_MAX_TAG_VALUE/64];
+static uint64_t free_tag_bits[VM_MAX_TAG_VALUE / 64];

 
 void
 vm_tag_alloc_locked(vm_allocation_site_t * site, vm_allocation_site_t ** releasesiteP)
 {
 
 void
 vm_tag_alloc_locked(vm_allocation_site_t * site, vm_allocation_site_t ** releasesiteP)
 {

-    vm_tag_t tag;
-    uint64_t avail;
-    uint32_t idx;
-    vm_allocation_site_t * prev;
+       vm_tag_t tag;
+       uint64_t avail;
+       uint32_t idx;
+       vm_allocation_site_t * prev;

 
 

-    if (site->tag) return;
+       if (site->tag) {
+               return;
+       }

 
 

-    idx = 0;
-    while (TRUE)
-    {
+       idx = 0;
+       while (TRUE) {

                avail = free_tag_bits[idx];
                avail = free_tag_bits[idx];

-               if (avail)
-               {
-                   tag = __builtin_clzll(avail);
-                   avail &= ~(1ULL << (63 - tag));
-                   free_tag_bits[idx] = avail;
-                   tag += (idx << 6);
-                   break;
+               if (avail) {
+                       tag = (vm_tag_t)__builtin_clzll(avail);
+                       avail &= ~(1ULL << (63 - tag));
+                       free_tag_bits[idx] = avail;
+                       tag += (idx << 6);
+                       break;

                }
                idx++;
                }
                idx++;

-               if (idx >= ARRAY_COUNT(free_tag_bits))
-               {
-                   for (idx = 0; idx < ARRAY_COUNT(vm_allocation_sites); idx++)
-                   {
+               if (idx >= ARRAY_COUNT(free_tag_bits)) {
+                       for (idx = 0; idx < ARRAY_COUNT(vm_allocation_sites); idx++) {

                                prev = vm_allocation_sites[idx];
                                prev = vm_allocation_sites[idx];

-                               if (!prev)               continue;
-                               if (!KA_NAME_LEN(prev))  continue;
-                               if (!prev->tag)          continue;
-                               if (prev->total)         continue;
-                               if (1 != prev->refcount) continue;
+                               if (!prev) {
+                                       continue;
+                               }
+                               if (!KA_NAME_LEN(prev)) {
+                                       continue;
+                               }
+                               if (!prev->tag) {
+                                       continue;
+                               }
+                               if (prev->total) {
+                                       continue;
+                               }
+                               if (1 != prev->refcount) {
+                                       continue;
+                               }

 
                                assert(idx == prev->tag);
 
                                assert(idx == prev->tag);

-                               tag = idx;
+                               tag = (vm_tag_t)idx;

                                prev->tag = VM_KERN_MEMORY_NONE;
                                *releasesiteP = prev;
                                break;
                                prev->tag = VM_KERN_MEMORY_NONE;
                                *releasesiteP = prev;
                                break;

-                   }
-                   if (idx >= ARRAY_COUNT(vm_allocation_sites))
-                       {
+                       }
+                       if (idx >= ARRAY_COUNT(vm_allocation_sites)) {

                                tag = VM_KERN_MEMORY_ANY;
                        }
                                tag = VM_KERN_MEMORY_ANY;
                        }

-                   break;
+                       break;

                }
                }

-    }
-    site->tag = tag;
+       }
+       site->tag = tag;

 
 

-    OSAddAtomic16(1, &site->refcount);
+       OSAddAtomic16(1, &site->refcount);

 
 

-    if (VM_KERN_MEMORY_ANY != tag) vm_allocation_sites[tag] = site;
+       if (VM_KERN_MEMORY_ANY != tag) {
+               vm_allocation_sites[tag] = site;
+       }

 
 

-    if (tag > vm_allocation_tag_highest) vm_allocation_tag_highest = tag;
+       if (tag > vm_allocation_tag_highest) {
+               vm_allocation_tag_highest = tag;
+       }

 }
 
 static void
 vm_tag_free_locked(vm_tag_t tag)
 {
 }
 
 static void
 vm_tag_free_locked(vm_tag_t tag)
 {

-    uint64_t avail;
-    uint32_t idx;
-    uint64_t bit;
+       uint64_t avail;
+       uint32_t idx;
+       uint64_t bit;

 
 

-    if (VM_KERN_MEMORY_ANY == tag) return;
+       if (VM_KERN_MEMORY_ANY == tag) {
+               return;
+       }

 
 

-    idx = (tag >> 6);
-    avail = free_tag_bits[idx];
-    tag &= 63;
-    bit = (1ULL << (63 - tag));
-    assert(!(avail & bit));
-    free_tag_bits[idx] = (avail | bit);
+       idx = (tag >> 6);
+       avail = free_tag_bits[idx];
+       tag &= 63;
+       bit = (1ULL << (63 - tag));
+       assert(!(avail & bit));
+       free_tag_bits[idx] = (avail | bit);

 }
 
 static void
 vm_tag_init(void)
 {
 }
 
 static void
 vm_tag_init(void)
 {

-    vm_tag_t tag;
-    for (tag = VM_KERN_MEMORY_FIRST_DYNAMIC; tag < VM_KERN_MEMORY_ANY; tag++)
-    {
-        vm_tag_free_locked(tag);
-    }
+       vm_tag_t tag;
+       for (tag = VM_KERN_MEMORY_FIRST_DYNAMIC; tag < VM_KERN_MEMORY_ANY; tag++) {
+               vm_tag_free_locked(tag);
+       }

 
 

-    for (tag = VM_KERN_MEMORY_ANY + 1; tag < VM_MAX_TAG_VALUE; tag++)
-    {
-        vm_tag_free_locked(tag);
-    }
+       for (tag = VM_KERN_MEMORY_ANY + 1; tag < VM_MAX_TAG_VALUE; tag++) {
+               vm_tag_free_locked(tag);
+       }

 }
 
 vm_tag_t
 vm_tag_alloc(vm_allocation_site_t * site)
 {
 }
 
 vm_tag_t
 vm_tag_alloc(vm_allocation_site_t * site)
 {

-    vm_tag_t tag;
-    vm_allocation_site_t * releasesite;
+       vm_tag_t tag;
+       vm_allocation_site_t * releasesite;

 
 

-    if (VM_TAG_BT & site->flags)
-    {
+       if (VM_TAG_BT & site->flags) {

                tag = vm_tag_bt();
                tag = vm_tag_bt();

-               if (VM_KERN_MEMORY_NONE != tag) return (tag);
-    }
+               if (VM_KERN_MEMORY_NONE != tag) {
+                       return tag;
+               }
+       }

 
 

-    if (!site->tag) 
-    {
+       if (!site->tag) {

                releasesite = NULL;
                lck_spin_lock(&vm_allocation_sites_lock);
                vm_tag_alloc_locked(site, &releasesite);
                lck_spin_unlock(&vm_allocation_sites_lock);
                releasesite = NULL;
                lck_spin_lock(&vm_allocation_sites_lock);
                vm_tag_alloc_locked(site, &releasesite);
                lck_spin_unlock(&vm_allocation_sites_lock);

-        if (releasesite) kern_allocation_name_release(releasesite);
-    }
+               if (releasesite) {
+                       kern_allocation_name_release(releasesite);
+               }
+       }

 
 

-    return (site->tag);
+       return site->tag;

 }
 
 void
 vm_tag_update_size(vm_tag_t tag, int64_t delta)
 {
 }
 
 void
 vm_tag_update_size(vm_tag_t tag, int64_t delta)
 {

-    vm_allocation_site_t * allocation;
-    uint64_t prior;
+       vm_allocation_site_t * allocation;
+       uint64_t prior;

 
 

-    assert(VM_KERN_MEMORY_NONE != tag);
-    assert(tag < VM_MAX_TAG_VALUE);
+       assert(VM_KERN_MEMORY_NONE != tag);
+       assert(tag < VM_MAX_TAG_VALUE);

 
 

-    allocation = vm_allocation_sites[tag];
-    assert(allocation);
+       allocation = vm_allocation_sites[tag];
+       assert(allocation);

 
 

-    if (delta < 0) {
+       if (delta < 0) {

                assertf(allocation->total >= ((uint64_t)-delta), "tag %d, site %p", tag, allocation);
                assertf(allocation->total >= ((uint64_t)-delta), "tag %d, site %p", tag, allocation);

-    }
-    prior = OSAddAtomic64(delta, &allocation->total);
+       }
+       prior = OSAddAtomic64(delta, &allocation->total);

 
 #if DEBUG || DEVELOPMENT
 
 
 #if DEBUG || DEVELOPMENT
 

-    uint64_t new, peak;
+       uint64_t new, peak;

        new = prior + delta;
        new = prior + delta;

-    do
-    {
-        peak = allocation->peak;
-        if (new <= peak) break;
-    }
-    while (!OSCompareAndSwap64(peak, new, &allocation->peak));
+       do{
+               peak = allocation->peak;
+               if (new <= peak) {
+                       break;
+               }
+       }while (!OSCompareAndSwap64(peak, new, &allocation->peak));

 
 #endif /* DEBUG || DEVELOPMENT */
 
 
 #endif /* DEBUG || DEVELOPMENT */
 

-    if (tag < VM_KERN_MEMORY_FIRST_DYNAMIC) return;
+       if (tag < VM_KERN_MEMORY_FIRST_DYNAMIC) {
+               return;
+       }

 
 

-    if (!prior && !allocation->tag) vm_tag_alloc(allocation);
+       if (!prior && !allocation->tag) {
+               vm_tag_alloc(allocation);
+       }

 }
 
 void
 kern_allocation_update_size(kern_allocation_name_t allocation, int64_t delta)
 {
 }
 
 void
 kern_allocation_update_size(kern_allocation_name_t allocation, int64_t delta)
 {

-    uint64_t prior;
+       uint64_t prior;

 
 

-    if (delta < 0) {
+       if (delta < 0) {

                assertf(allocation->total >= ((uint64_t)-delta), "name %p", allocation);
                assertf(allocation->total >= ((uint64_t)-delta), "name %p", allocation);

-    }
-    prior = OSAddAtomic64(delta, &allocation->total);
+       }
+       prior = OSAddAtomic64(delta, &allocation->total);

 
 #if DEBUG || DEVELOPMENT
 
 
 #if DEBUG || DEVELOPMENT
 

-    uint64_t new, peak;
+       uint64_t new, peak;

        new = prior + delta;
        new = prior + delta;

-    do
-    {
-        peak = allocation->peak;
-        if (new <= peak) break;
-    }
-    while (!OSCompareAndSwap64(peak, new, &allocation->peak));
+       do{
+               peak = allocation->peak;
+               if (new <= peak) {
+                       break;
+               }
+       }while (!OSCompareAndSwap64(peak, new, &allocation->peak));

 
 #endif /* DEBUG || DEVELOPMENT */
 
 
 #endif /* DEBUG || DEVELOPMENT */
 

-    if (!prior && !allocation->tag) vm_tag_alloc(allocation);
+       if (!prior && !allocation->tag) {
+               vm_tag_alloc(allocation);
+       }

 }
 
 #if VM_MAX_TAG_ZONES
 }
 
 #if VM_MAX_TAG_ZONES


@@ -8577,86 +9108,91 @@ kern_allocation_update_size(kern_allocation_name_t allocation, int64_t delta)
 void
 vm_allocation_zones_init(void)
 {
 void
 vm_allocation_zones_init(void)
 {

-    kern_return_t ret;
-       vm_offset_t       addr;
+       kern_return_t ret;
+       vm_offset_t       addr;

        vm_size_t     size;
 
        vm_size_t     size;
 

-    size = VM_MAX_TAG_VALUE * sizeof(vm_allocation_zone_total_t **)
-         + 2 * VM_MAX_TAG_ZONES * sizeof(vm_allocation_zone_total_t);
+       size = VM_MAX_TAG_VALUE * sizeof(vm_allocation_zone_total_t * *)
+           + 2 * VM_MAX_TAG_ZONES * sizeof(vm_allocation_zone_total_t);

 
        ret = kernel_memory_allocate(kernel_map,
 
        ret = kernel_memory_allocate(kernel_map,

-                                       &addr, round_page(size), 0,
-                                       KMA_ZERO, VM_KERN_MEMORY_DIAG);
-    assert(KERN_SUCCESS == ret);
+           &addr, round_page(size), 0,
+           KMA_ZERO, VM_KERN_MEMORY_DIAG);
+       assert(KERN_SUCCESS == ret);

 
 

-    vm_allocation_zone_totals = (vm_allocation_zone_total_t **) addr;
-    addr += VM_MAX_TAG_VALUE * sizeof(vm_allocation_zone_total_t **);
+       vm_allocation_zone_totals = (vm_allocation_zone_total_t **) addr;
+       addr += VM_MAX_TAG_VALUE * sizeof(vm_allocation_zone_total_t * *);

 
 

-    // prepopulate VM_KERN_MEMORY_DIAG & VM_KERN_MEMORY_KALLOC so allocations
-    // in vm_tag_update_zone_size() won't recurse
-    vm_allocation_zone_totals[VM_KERN_MEMORY_DIAG]   = (vm_allocation_zone_total_t *) addr;
-    addr += VM_MAX_TAG_ZONES * sizeof(vm_allocation_zone_total_t);
-    vm_allocation_zone_totals[VM_KERN_MEMORY_KALLOC] = (vm_allocation_zone_total_t *) addr;
+       // prepopulate VM_KERN_MEMORY_DIAG & VM_KERN_MEMORY_KALLOC so allocations
+       // in vm_tag_update_zone_size() won't recurse
+       vm_allocation_zone_totals[VM_KERN_MEMORY_DIAG]   = (vm_allocation_zone_total_t *) addr;
+       addr += VM_MAX_TAG_ZONES * sizeof(vm_allocation_zone_total_t);
+       vm_allocation_zone_totals[VM_KERN_MEMORY_KALLOC] = (vm_allocation_zone_total_t *) addr;

 }
 
 void
 vm_tag_will_update_zone(vm_tag_t tag, uint32_t zidx)
 {
 }
 
 void
 vm_tag_will_update_zone(vm_tag_t tag, uint32_t zidx)
 {

-    vm_allocation_zone_total_t * zone;
+       vm_allocation_zone_total_t * zone;

 
 

-    assert(VM_KERN_MEMORY_NONE != tag);
-    assert(tag < VM_MAX_TAG_VALUE);
+       assert(VM_KERN_MEMORY_NONE != tag);
+       assert(tag < VM_MAX_TAG_VALUE);

 
 

-    if (zidx >= VM_MAX_TAG_ZONES) return;
+       if (zidx >= VM_MAX_TAG_ZONES) {
+               return;
+       }

 
        zone = vm_allocation_zone_totals[tag];
 
        zone = vm_allocation_zone_totals[tag];

-    if (!zone)
-    {
-        zone = kalloc_tag(VM_MAX_TAG_ZONES * sizeof(*zone), VM_KERN_MEMORY_DIAG);
-        if (!zone) return;
-        bzero(zone, VM_MAX_TAG_ZONES * sizeof(*zone));
-        if (!OSCompareAndSwapPtr(NULL, zone, &vm_allocation_zone_totals[tag]))
-               {
+       if (!zone) {
+               zone = kalloc_tag(VM_MAX_TAG_ZONES * sizeof(*zone), VM_KERN_MEMORY_DIAG);
+               if (!zone) {
+                       return;
+               }
+               bzero(zone, VM_MAX_TAG_ZONES * sizeof(*zone));
+               if (!OSCompareAndSwapPtr(NULL, zone, &vm_allocation_zone_totals[tag])) {

                        kfree(zone, VM_MAX_TAG_ZONES * sizeof(*zone));
                }
                        kfree(zone, VM_MAX_TAG_ZONES * sizeof(*zone));
                }

-    }
+       }

 }
 
 void
 vm_tag_update_zone_size(vm_tag_t tag, uint32_t zidx, int64_t delta, int64_t dwaste)
 {
 }
 
 void
 vm_tag_update_zone_size(vm_tag_t tag, uint32_t zidx, int64_t delta, int64_t dwaste)
 {

-    vm_allocation_zone_total_t * zone;
-    uint32_t new;
+       vm_allocation_zone_total_t * zone;
+       uint32_t new;

 
 

-    assert(VM_KERN_MEMORY_NONE != tag);
-    assert(tag < VM_MAX_TAG_VALUE);
+       assert(VM_KERN_MEMORY_NONE != tag);
+       assert(tag < VM_MAX_TAG_VALUE);

 
 

-    if (zidx >= VM_MAX_TAG_ZONES) return;
+       if (zidx >= VM_MAX_TAG_ZONES) {
+               return;
+       }

 
        zone = vm_allocation_zone_totals[tag];
 
        zone = vm_allocation_zone_totals[tag];

-    assert(zone);
-    zone += zidx;
+       assert(zone);
+       zone += zidx;

 
 

-    /* the zone is locked */
-    if (delta < 0)
-    {
+       /* the zone is locked */
+       if (delta < 0) {

                assertf(zone->total >= ((uint64_t)-delta), "zidx %d, tag %d, %p", zidx, tag, zone);
                assertf(zone->total >= ((uint64_t)-delta), "zidx %d, tag %d, %p", zidx, tag, zone);

-           zone->total += delta;
-    }
-    else
-    {

                zone->total += delta;
                zone->total += delta;

-               if (zone->total > zone->peak) zone->peak = zone->total;
-        if (dwaste)
-        {
+       } else {
+               zone->total += delta;
+               if (zone->total > zone->peak) {
+                       zone->peak = zone->total;
+               }
+               if (dwaste) {

                        new = zone->waste;
                        new = zone->waste;

-                       if (zone->wastediv < 65536) zone->wastediv++;
-                       else                        new -= (new >> 16);
+                       if (zone->wastediv < 65536) {
+                               zone->wastediv++;
+                       } else {
+                               new -= (new >> 16);
+                       }

                        __assert_only bool ov = os_add_overflow(new, dwaste, &new);
                        assert(!ov);
                        zone->waste = new;
                        __assert_only bool ov = os_add_overflow(new, dwaste, &new);
                        assert(!ov);
                        zone->waste = new;

-        }
-    }
+               }
+       }

 }
 
 #endif /* VM_MAX_TAG_ZONES */
 }
 
 #endif /* VM_MAX_TAG_ZONES */


@@ -8664,380 +9200,458 @@ vm_tag_update_zone_size(vm_tag_t tag, uint32_t zidx, int64_t delta, int64_t dwas
 void
 kern_allocation_update_subtotal(kern_allocation_name_t allocation, uint32_t subtag, int64_t delta)
 {
 void
 kern_allocation_update_subtotal(kern_allocation_name_t allocation, uint32_t subtag, int64_t delta)
 {

-    kern_allocation_name_t other;
+       kern_allocation_name_t other;

        struct vm_allocation_total * total;
        struct vm_allocation_total * total;

-    uint32_t subidx;
+       uint32_t subidx;

 
 

-    subidx = 0;
-    assert(VM_KERN_MEMORY_NONE != subtag);
-       for (; subidx < allocation->subtotalscount; subidx++)
-       {
-               if (VM_KERN_MEMORY_NONE == allocation->subtotals[subidx].tag)
-               {
-                       allocation->subtotals[subidx].tag = subtag;
+       subidx = 0;
+       assert(VM_KERN_MEMORY_NONE != subtag);
+       lck_spin_lock(&vm_allocation_sites_lock);
+       for (; subidx < allocation->subtotalscount; subidx++) {
+               if (VM_KERN_MEMORY_NONE == allocation->subtotals[subidx].tag) {
+                       allocation->subtotals[subidx].tag = (vm_tag_t)subtag;
+                       break;
+               }
+               if (subtag == allocation->subtotals[subidx].tag) {

                        break;
                }
                        break;
                }

-               if (subtag == allocation->subtotals[subidx].tag) break;

        }
        }

-    assert(subidx < allocation->subtotalscount);
-    if (subidx >= allocation->subtotalscount) return;
+       lck_spin_unlock(&vm_allocation_sites_lock);
+       assert(subidx < allocation->subtotalscount);
+       if (subidx >= allocation->subtotalscount) {
+               return;
+       }

 
 

-    total = &allocation->subtotals[subidx];
-    other = vm_allocation_sites[subtag];
-    assert(other);
+       total = &allocation->subtotals[subidx];
+       other = vm_allocation_sites[subtag];
+       assert(other);

 
 

-    if (delta < 0)
-    {
+       if (delta < 0) {

                assertf(total->total >= ((uint64_t)-delta), "name %p", allocation);
                assertf(total->total >= ((uint64_t)-delta), "name %p", allocation);

-           OSAddAtomic64(delta, &total->total);

                assertf(other->mapped >= ((uint64_t)-delta), "other %p", other);
                assertf(other->mapped >= ((uint64_t)-delta), "other %p", other);

-        OSAddAtomic64(delta, &other->mapped);
-    }
-    else
-    {
-        OSAddAtomic64(delta, &other->mapped);
-           OSAddAtomic64(delta, &total->total);
-    }
+       }
+       OSAddAtomic64(delta, &other->mapped);
+       OSAddAtomic64(delta, &total->total);

 }
 
 const char *
 kern_allocation_get_name(kern_allocation_name_t allocation)
 {
 }
 
 const char *
 kern_allocation_get_name(kern_allocation_name_t allocation)
 {

-    return (KA_NAME(allocation));
+       return KA_NAME(allocation);

 }
 
 kern_allocation_name_t
 }
 
 kern_allocation_name_t

-kern_allocation_name_allocate(const char * name, uint32_t subtotalscount)
+kern_allocation_name_allocate(const char * name, uint16_t subtotalscount)

 {
 {

-    uint32_t namelen;
+       uint16_t namelen;

 
 

-    namelen = (uint32_t) strnlen(name, MACH_MEMORY_INFO_NAME_MAX_LEN - 1);
+       namelen = (uint16_t)strnlen(name, MACH_MEMORY_INFO_NAME_MAX_LEN - 1);

 
 

-    kern_allocation_name_t allocation;
-    allocation = kalloc(KA_SIZE(namelen, subtotalscount));
-    bzero(allocation, KA_SIZE(namelen, subtotalscount));
+       kern_allocation_name_t allocation;
+       allocation = kheap_alloc(KHEAP_DATA_BUFFERS,
+           KA_SIZE(namelen, subtotalscount), Z_WAITOK);
+       bzero(allocation, KA_SIZE(namelen, subtotalscount));

 
 

-    allocation->refcount       = 1;
-    allocation->subtotalscount = subtotalscount;
-    allocation->flags          = (namelen << VM_TAG_NAME_LEN_SHIFT);
-    strlcpy(KA_NAME(allocation), name, namelen + 1);
+       allocation->refcount       = 1;
+       allocation->subtotalscount = subtotalscount;
+       allocation->flags          = (uint16_t)(namelen << VM_TAG_NAME_LEN_SHIFT);
+       strlcpy(KA_NAME(allocation), name, namelen + 1);

 
 

-    return (allocation);
+       return allocation;

 }
 
 void
 kern_allocation_name_release(kern_allocation_name_t allocation)
 {
 }
 
 void
 kern_allocation_name_release(kern_allocation_name_t allocation)
 {

-    assert(allocation->refcount > 0);
-    if (1 == OSAddAtomic16(-1, &allocation->refcount))
-    {
-        kfree(allocation, KA_SIZE(KA_NAME_LEN(allocation), allocation->subtotalscount));
-    }
+       assert(allocation->refcount > 0);
+       if (1 == OSAddAtomic16(-1, &allocation->refcount)) {
+               kheap_free(KHEAP_DATA_BUFFERS, allocation,
+                   KA_SIZE(KA_NAME_LEN(allocation), allocation->subtotalscount));
+       }

 }
 
 vm_tag_t
 kern_allocation_name_get_vm_tag(kern_allocation_name_t allocation)
 {
 }
 
 vm_tag_t
 kern_allocation_name_get_vm_tag(kern_allocation_name_t allocation)
 {

-    return (vm_tag_alloc(allocation));
+       return vm_tag_alloc(allocation);

 }
 
 }
 

-#if ! VM_TAG_ACTIVE_UPDATE
-static void 
+#if !VM_TAG_ACTIVE_UPDATE
+static void

 vm_page_count_object(mach_memory_info_t * info, unsigned int __unused num_info, vm_object_t object)
 {
 vm_page_count_object(mach_memory_info_t * info, unsigned int __unused num_info, vm_object_t object)
 {

-    if (!object->wired_page_count) return;
-    if (object != kernel_object)
-    {
+       if (!object->wired_page_count) {
+               return;
+       }
+       if (object != kernel_object) {

                assert(object->wire_tag < num_info);
                info[object->wire_tag].size += ptoa_64(object->wired_page_count);
                assert(object->wire_tag < num_info);
                info[object->wire_tag].size += ptoa_64(object->wired_page_count);

-    }
+       }

 }
 
 typedef void (*vm_page_iterate_proc)(mach_memory_info_t * info,
 }
 
 typedef void (*vm_page_iterate_proc)(mach_memory_info_t * info,

-                                    unsigned int num_info, vm_object_t object);
+    unsigned int num_info, vm_object_t object);

 
 

-static void 
+static void

 vm_page_iterate_purgeable_objects(mach_memory_info_t * info, unsigned int num_info,
 vm_page_iterate_purgeable_objects(mach_memory_info_t * info, unsigned int num_info,

-                                 vm_page_iterate_proc proc, purgeable_q_t queue, 
-                                 int group)
+    vm_page_iterate_proc proc, purgeable_q_t queue,
+    int group)

 {
 {

-    vm_object_t object;
+       vm_object_t object;

 
 

-    for (object = (vm_object_t) queue_first(&queue->objq[group]);
-               !queue_end(&queue->objq[group], (queue_entry_t) object);
-               object = (vm_object_t) queue_next(&object->objq))
-    {
+       for (object = (vm_object_t) queue_first(&queue->objq[group]);
+           !queue_end(&queue->objq[group], (queue_entry_t) object);
+           object = (vm_object_t) queue_next(&object->objq)) {

                proc(info, num_info, object);
                proc(info, num_info, object);

-    }
+       }

 }
 
 }
 

-static void 
+static void

 vm_page_iterate_objects(mach_memory_info_t * info, unsigned int num_info,
 vm_page_iterate_objects(mach_memory_info_t * info, unsigned int num_info,

-                       vm_page_iterate_proc proc)
+    vm_page_iterate_proc proc)

 {
 {

-    vm_object_t     object;
+       vm_object_t     object;

 
 

-    lck_spin_lock(&vm_objects_wired_lock);
-    queue_iterate(&vm_objects_wired,
-                 object,
-                 vm_object_t,
-                 wired_objq)
-    {
+       lck_spin_lock_grp(&vm_objects_wired_lock, &vm_page_lck_grp_bucket);
+       queue_iterate(&vm_objects_wired,
+           object,
+           vm_object_t,
+           wired_objq)
+       {

                proc(info, num_info, object);
                proc(info, num_info, object);

-    }
-    lck_spin_unlock(&vm_objects_wired_lock);
+       }
+       lck_spin_unlock(&vm_objects_wired_lock);

 }
 #endif /* ! VM_TAG_ACTIVE_UPDATE */
 
 static uint64_t
 }
 #endif /* ! VM_TAG_ACTIVE_UPDATE */
 
 static uint64_t

-process_account(mach_memory_info_t * info, unsigned int num_info, uint64_t zones_collectable_bytes, boolean_t iterated)
+process_account(mach_memory_info_t * info, unsigned int num_info,
+    uint64_t zones_collectable_bytes, boolean_t iterated)

 {
 {

-    size_t                 namelen;
-    unsigned int           idx, count, nextinfo;
-    vm_allocation_site_t * site;
+       size_t                 namelen;
+       unsigned int           idx, count, nextinfo;
+       vm_allocation_site_t * site;

        lck_spin_lock(&vm_allocation_sites_lock);
 
        lck_spin_lock(&vm_allocation_sites_lock);
 

-       for (idx = 0; idx <= vm_allocation_tag_highest; idx++)
-    {
+       for (idx = 0; idx <= vm_allocation_tag_highest; idx++) {

                site = vm_allocation_sites[idx];
                site = vm_allocation_sites[idx];

-               if (!site) continue;
+               if (!site) {
+                       continue;
+               }

                info[idx].mapped = site->mapped;
                info[idx].tag    = site->tag;
                info[idx].mapped = site->mapped;
                info[idx].tag    = site->tag;

-        if (!iterated)
-        {
+               if (!iterated) {

                        info[idx].size = site->total;
 #if DEBUG || DEVELOPMENT
                        info[idx].peak = site->peak;
 #endif /* DEBUG || DEVELOPMENT */
                        info[idx].size = site->total;
 #if DEBUG || DEVELOPMENT
                        info[idx].peak = site->peak;
 #endif /* DEBUG || DEVELOPMENT */

-        }
-        else
-        {
-                       if (!site->subtotalscount && (site->total != info[idx].size))
-                       {
-                           printf("tag mismatch[%d] 0x%qx, iter 0x%qx\n", idx, site->total, info[idx].size);
-                           info[idx].size = site->total;
-                       }
-        }
-    }
-
-    nextinfo = (vm_allocation_tag_highest + 1);
-    count    = nextinfo;
-    if (count >= num_info) count = num_info;
-
-    for (idx = 0; idx < count; idx++)
-    {
-               site = vm_allocation_sites[idx];
-               if (!site) continue;
+               } else {
+                       if (!site->subtotalscount && (site->total != info[idx].size)) {
+                               printf("tag mismatch[%d] 0x%qx, iter 0x%qx\n", idx, site->total, info[idx].size);
+                               info[idx].size = site->total;
+                       }
+               }

                info[idx].flags |= VM_KERN_SITE_WIRED;
                info[idx].flags |= VM_KERN_SITE_WIRED;

-               if (idx < VM_KERN_MEMORY_FIRST_DYNAMIC)
-               {
-                   info[idx].site   = idx;
-                   info[idx].flags |= VM_KERN_SITE_TAG;
-                   if (VM_KERN_MEMORY_ZONE == idx)
-                   {
+               if (idx < VM_KERN_MEMORY_FIRST_DYNAMIC) {
+                       info[idx].site   = idx;
+                       info[idx].flags |= VM_KERN_SITE_TAG;
+                       if (VM_KERN_MEMORY_ZONE == idx) {

                                info[idx].flags |= VM_KERN_SITE_HIDE;
                                info[idx].flags &= ~VM_KERN_SITE_WIRED;
                                info[idx].collectable_bytes = zones_collectable_bytes;
                        }
                                info[idx].flags |= VM_KERN_SITE_HIDE;
                                info[idx].flags &= ~VM_KERN_SITE_WIRED;
                                info[idx].collectable_bytes = zones_collectable_bytes;
                        }

+               } else if ((namelen = (VM_TAG_NAME_LEN_MAX & (site->flags >> VM_TAG_NAME_LEN_SHIFT)))) {
+                       info[idx].site   = 0;
+                       info[idx].flags |= VM_KERN_SITE_NAMED;
+                       if (namelen > sizeof(info[idx].name)) {
+                               namelen = sizeof(info[idx].name);
+                       }
+                       strncpy(&info[idx].name[0], KA_NAME(site), namelen);
+               } else if (VM_TAG_KMOD & site->flags) {
+                       info[idx].site   = OSKextGetKmodIDForSite(site, NULL, 0);
+                       info[idx].flags |= VM_KERN_SITE_KMOD;
+               } else {
+                       info[idx].site   = VM_KERNEL_UNSLIDE(site);
+                       info[idx].flags |= VM_KERN_SITE_KERNEL;

                }
                }

-               else if ((namelen = (VM_TAG_NAME_LEN_MAX & (site->flags >> VM_TAG_NAME_LEN_SHIFT))))
-               {
-                   info[idx].site   = 0;
-                   info[idx].flags |= VM_KERN_SITE_NAMED;
-                   if (namelen > sizeof(info[idx].name)) namelen = sizeof(info[idx].name);
-                   strncpy(&info[idx].name[0], KA_NAME(site), namelen);
-               }
-               else if (VM_TAG_KMOD & site->flags)
-               {
-                   info[idx].site   = OSKextGetKmodIDForSite(site, NULL, 0);
-                   info[idx].flags |= VM_KERN_SITE_KMOD;
-               }
-               else
-               {
-                   info[idx].site   = VM_KERNEL_UNSLIDE(site);
-                   info[idx].flags |= VM_KERN_SITE_KERNEL;
+       }
+
+       nextinfo = (vm_allocation_tag_highest + 1);
+       count    = nextinfo;
+       if (count >= num_info) {
+               count = num_info;
+       }
+
+       for (idx = 0; idx < count; idx++) {
+               site = vm_allocation_sites[idx];
+               if (!site) {
+                       continue;

                }
 #if VM_MAX_TAG_ZONES
                vm_allocation_zone_total_t * zone;
                unsigned int                 zidx;
                vm_size_t                    elem_size;
 
                }
 #if VM_MAX_TAG_ZONES
                vm_allocation_zone_total_t * zone;
                unsigned int                 zidx;
                vm_size_t                    elem_size;
 

-        if (vm_allocation_zone_totals
-                       && (zone = vm_allocation_zone_totals[idx])
-                       && (nextinfo < num_info))
-        {
-            for (zidx = 0; zidx < VM_MAX_TAG_ZONES; zidx++)
-            {
-                if (!zone[zidx].peak)              continue;
-                               info[nextinfo]                   = info[idx];
-                               info[nextinfo].zone              = zone_index_from_tag_index(zidx, &elem_size);
-                               info[nextinfo].flags            &= ~VM_KERN_SITE_WIRED;
-                               info[nextinfo].flags            |= VM_KERN_SITE_ZONE;
-                               info[nextinfo].size              = zone[zidx].total;
-                               info[nextinfo].peak              = zone[zidx].peak;
-                               info[nextinfo].mapped            = 0;
-                if (zone[zidx].wastediv)
-                               {
+               if (vm_allocation_zone_totals
+                   && (zone = vm_allocation_zone_totals[idx])
+                   && (nextinfo < num_info)) {
+                       for (zidx = 0; zidx < VM_MAX_TAG_ZONES; zidx++) {
+                               if (!zone[zidx].peak) {
+                                       continue;
+                               }
+                               info[nextinfo]        = info[idx];
+                               info[nextinfo].zone   = (uint16_t)zone_index_from_tag_index(zidx, &elem_size);
+                               info[nextinfo].flags  &= ~VM_KERN_SITE_WIRED;
+                               info[nextinfo].flags  |= VM_KERN_SITE_ZONE;
+                               info[nextinfo].size   = zone[zidx].total;
+                               info[nextinfo].peak   = zone[zidx].peak;
+                               info[nextinfo].mapped = 0;
+                               if (zone[zidx].wastediv) {

                                        info[nextinfo].collectable_bytes = ((zone[zidx].waste * zone[zidx].total / elem_size) / zone[zidx].wastediv);
                                }
                                        info[nextinfo].collectable_bytes = ((zone[zidx].waste * zone[zidx].total / elem_size) / zone[zidx].wastediv);
                                }

-                nextinfo++;
-             }
-        }
+                               nextinfo++;
+                       }
+               }

 #endif /* VM_MAX_TAG_ZONES */
 #endif /* VM_MAX_TAG_ZONES */

-        if (site->subtotalscount)
-        {
+               if (site->subtotalscount) {

                        uint64_t mapped, mapcost, take;
                        uint32_t sub;
                        vm_tag_t alloctag;
 
                        uint64_t mapped, mapcost, take;
                        uint32_t sub;
                        vm_tag_t alloctag;
 

-            info[idx].size = site->total;
-            mapped = info[idx].size;
-            info[idx].mapped = mapped;
-            mapcost = 0;
-            for (sub = 0; sub < site->subtotalscount; sub++)
-            {
+                       info[idx].size = site->total;
+                       mapped = info[idx].size;
+                       info[idx].mapped = mapped;
+                       mapcost = 0;
+                       for (sub = 0; sub < site->subtotalscount; sub++) {

                                alloctag = site->subtotals[sub].tag;
                                assert(alloctag < num_info);
                                alloctag = site->subtotals[sub].tag;
                                assert(alloctag < num_info);

-                               if (info[alloctag].name[0]) continue;
-                               take = info[alloctag].mapped;
-                               if (take > info[alloctag].size) take = info[alloctag].size;
-                               if (take > mapped) take = mapped;
+                               if (info[alloctag].name[0]) {
+                                       continue;
+                               }
+                               take = site->subtotals[sub].total;
+                               if (take > info[alloctag].size) {
+                                       take = info[alloctag].size;
+                               }
+                               if (take > mapped) {
+                                       take = mapped;
+                               }

                                info[alloctag].mapped  -= take;
                                info[alloctag].size    -= take;
                                mapped                 -= take;
                                mapcost                += take;
                                info[alloctag].mapped  -= take;
                                info[alloctag].size    -= take;
                                mapped                 -= take;
                                mapcost                += take;

-            }
-            info[idx].size = mapcost;
-        }
+                       }
+                       info[idx].size = mapcost;
+               }

        }
        lck_spin_unlock(&vm_allocation_sites_lock);
 
        }
        lck_spin_unlock(&vm_allocation_sites_lock);
 

-    return (0);
+       return 0;

 }
 
 uint32_t
 vm_page_diagnose_estimate(void)
 {
 }
 
 uint32_t
 vm_page_diagnose_estimate(void)
 {

-    vm_allocation_site_t * site;
-    uint32_t               count;
-    uint32_t               idx;
+       vm_allocation_site_t * site;
+       uint32_t               count = zone_view_count;
+       uint32_t               idx;

 
        lck_spin_lock(&vm_allocation_sites_lock);
 
        lck_spin_lock(&vm_allocation_sites_lock);

-       for (count = idx = 0; idx < VM_MAX_TAG_VALUE; idx++)
-    {
+       for (idx = 0; idx < VM_MAX_TAG_VALUE; idx++) {

                site = vm_allocation_sites[idx];
                site = vm_allocation_sites[idx];

-               if (!site) continue;
+               if (!site) {
+                       continue;
+               }

                count++;
 #if VM_MAX_TAG_ZONES
                count++;
 #if VM_MAX_TAG_ZONES

-               if (vm_allocation_zone_totals)
-               {
+               if (vm_allocation_zone_totals) {

                        vm_allocation_zone_total_t * zone;
                        zone = vm_allocation_zone_totals[idx];
                        vm_allocation_zone_total_t * zone;
                        zone = vm_allocation_zone_totals[idx];

-                       if (!zone) continue;
-                       for (uint32_t zidx = 0; zidx < VM_MAX_TAG_ZONES; zidx++) if (zone[zidx].peak) count++;
+                       if (!zone) {
+                               continue;
+                       }
+                       for (uint32_t zidx = 0; zidx < VM_MAX_TAG_ZONES; zidx++) {
+                               if (zone[zidx].peak) {
+                                       count++;
+                               }
+                       }

                }
 #endif
                }
 #endif

-    }
+       }

        lck_spin_unlock(&vm_allocation_sites_lock);
 
        lck_spin_unlock(&vm_allocation_sites_lock);
 

-    /* some slop for new tags created */
-    count += 8;
-    count += VM_KERN_COUNTER_COUNT;
+       /* some slop for new tags created */
+       count += 8;
+       count += VM_KERN_COUNTER_COUNT;

 
 

-    return (count);
+       return count;

 }
 
 }
 

+static void
+vm_page_diagnose_zone_stats(mach_memory_info_t *info, zone_stats_t zstats,
+    bool percpu)
+{
+       zpercpu_foreach(zs, zstats) {
+               info->size += zs->zs_mem_allocated - zs->zs_mem_freed;
+       }
+       if (percpu) {
+               info->size *= zpercpu_count();
+       }
+       info->flags |= VM_KERN_SITE_NAMED | VM_KERN_SITE_ZONE_VIEW;
+}
+
+static void
+vm_page_diagnose_zone(mach_memory_info_t *info, zone_t z)
+{
+       vm_page_diagnose_zone_stats(info, z->z_stats, z->percpu);
+       snprintf(info->name, sizeof(info->name),
+           "%s%s[raw]", zone_heap_name(z), z->z_name);
+}
+
+static int
+vm_page_diagnose_heap(mach_memory_info_t *info, kalloc_heap_t kheap)
+{
+       struct kheap_zones *zones = kheap->kh_zones;
+       int i = 0;
+
+       for (; i < zones->max_k_zone; i++) {
+               vm_page_diagnose_zone(info + i, zones->k_zone[i]);
+       }
+
+       for (kalloc_heap_t kh = zones->views; kh; kh = kh->kh_next, i++) {
+               vm_page_diagnose_zone_stats(info + i, kh->kh_stats, false);
+               snprintf(info[i].name, sizeof(info[i].name),
+                   "%skalloc[%s]", kheap->kh_name, kh->kh_name);
+       }

 
 

-kern_return_t 
+       return i;
+}
+
+kern_return_t

 vm_page_diagnose(mach_memory_info_t * info, unsigned int num_info, uint64_t zones_collectable_bytes)
 {
 vm_page_diagnose(mach_memory_info_t * info, unsigned int num_info, uint64_t zones_collectable_bytes)
 {

-    uint64_t                wired_size;
-    uint64_t                wired_managed_size;
-    uint64_t                wired_reserved_size;
-    uint64_t                booter_size;
-    boolean_t                iterate;
-    mach_memory_info_t     * counts;
+       uint64_t                 wired_size;
+       uint64_t                 wired_managed_size;
+       uint64_t                 wired_reserved_size;
+       boolean_t                iterate;
+       mach_memory_info_t     * counts;
+       uint32_t                 i;

 
 

-    bzero(info, num_info * sizeof(mach_memory_info_t));
+       bzero(info, num_info * sizeof(mach_memory_info_t));

 
 

-    if (!vm_page_wire_count_initial) return (KERN_ABORTED);
+       if (!vm_page_wire_count_initial) {
+               return KERN_ABORTED;
+       }

 
 #if CONFIG_EMBEDDED
 
 #if CONFIG_EMBEDDED

-    wired_size         = ptoa_64(vm_page_wire_count);
-    wired_reserved_size = ptoa_64(vm_page_wire_count_initial - vm_page_stolen_count);
+       wired_size          = ptoa_64(vm_page_wire_count);
+       wired_reserved_size = ptoa_64(vm_page_wire_count_initial - vm_page_stolen_count);

 #else
 #else

-    wired_size          = ptoa_64(vm_page_wire_count + vm_lopage_free_count + vm_page_throttled_count);
-    wired_reserved_size = ptoa_64(vm_page_wire_count_initial - vm_page_stolen_count + vm_page_throttled_count);
+       wired_size          = ptoa_64(vm_page_wire_count + vm_lopage_free_count + vm_page_throttled_count);
+       wired_reserved_size = ptoa_64(vm_page_wire_count_initial - vm_page_stolen_count + vm_page_throttled_count);

 #endif
 #endif

-    wired_managed_size  = ptoa_64(vm_page_wire_count - vm_page_wire_count_initial);
+       wired_managed_size  = ptoa_64(vm_page_wire_count - vm_page_wire_count_initial);

 
 

-    booter_size = ml_get_booter_memory_size();
-    wired_size += booter_size;
+       wired_size += booter_size;

 
 

-    assert(num_info >= VM_KERN_COUNTER_COUNT);
-    num_info -= VM_KERN_COUNTER_COUNT;
-    counts = &info[num_info];
+       assert(num_info >= VM_KERN_COUNTER_COUNT);
+       num_info -= VM_KERN_COUNTER_COUNT;
+       counts = &info[num_info];

 
 

-#define SET_COUNT(xcount, xsize, xflags)                       \
+#define SET_COUNT(xcount, xsize, xflags)                        \

     counts[xcount].tag   = VM_MAX_TAG_VALUE + xcount;   \
     counts[xcount].tag   = VM_MAX_TAG_VALUE + xcount;   \

-    counts[xcount].site  = (xcount);                           \
-    counts[xcount].size  = (xsize);                                \
-    counts[xcount].mapped  = (xsize);                          \
+    counts[xcount].site  = (xcount);                            \
+    counts[xcount].size  = (xsize);                                 \
+    counts[xcount].mapped  = (xsize);                           \

     counts[xcount].flags = VM_KERN_SITE_COUNTER | xflags;
 
     counts[xcount].flags = VM_KERN_SITE_COUNTER | xflags;
 

-    SET_COUNT(VM_KERN_COUNT_MANAGED,             ptoa_64(vm_page_pages),        0);
-    SET_COUNT(VM_KERN_COUNT_WIRED,               wired_size,                    0);
-    SET_COUNT(VM_KERN_COUNT_WIRED_MANAGED,       wired_managed_size,            0);
-    SET_COUNT(VM_KERN_COUNT_RESERVED,            wired_reserved_size,           VM_KERN_SITE_WIRED);
-    SET_COUNT(VM_KERN_COUNT_STOLEN,              ptoa_64(vm_page_stolen_count), VM_KERN_SITE_WIRED);
-    SET_COUNT(VM_KERN_COUNT_LOPAGE,              ptoa_64(vm_lopage_free_count), VM_KERN_SITE_WIRED);
-    SET_COUNT(VM_KERN_COUNT_WIRED_BOOT,                  ptoa_64(vm_page_wire_count_on_boot), 0);
-    SET_COUNT(VM_KERN_COUNT_BOOT_STOLEN,         booter_size,                   VM_KERN_SITE_WIRED);
-
-#define SET_MAP(xcount, xsize, xfree, xlargest)        \
-    counts[xcount].site    = (xcount);                 \
-    counts[xcount].size    = (xsize);                  \
-    counts[xcount].mapped  = (xsize);                  \
-    counts[xcount].free    = (xfree);                  \
-    counts[xcount].largest = (xlargest);               \
+       SET_COUNT(VM_KERN_COUNT_MANAGED, ptoa_64(vm_page_pages), 0);
+       SET_COUNT(VM_KERN_COUNT_WIRED, wired_size, 0);
+       SET_COUNT(VM_KERN_COUNT_WIRED_MANAGED, wired_managed_size, 0);
+       SET_COUNT(VM_KERN_COUNT_RESERVED, wired_reserved_size, VM_KERN_SITE_WIRED);
+       SET_COUNT(VM_KERN_COUNT_STOLEN, ptoa_64(vm_page_stolen_count), VM_KERN_SITE_WIRED);
+       SET_COUNT(VM_KERN_COUNT_LOPAGE, ptoa_64(vm_lopage_free_count), VM_KERN_SITE_WIRED);
+       SET_COUNT(VM_KERN_COUNT_WIRED_BOOT, ptoa_64(vm_page_wire_count_on_boot), 0);
+       SET_COUNT(VM_KERN_COUNT_BOOT_STOLEN, booter_size, VM_KERN_SITE_WIRED);
+       SET_COUNT(VM_KERN_COUNT_WIRED_STATIC_KERNELCACHE, ptoa_64(vm_page_kernelcache_count), 0);
+
+#define SET_MAP(xcount, xsize, xfree, xlargest) \
+    counts[xcount].site    = (xcount);                  \
+    counts[xcount].size    = (xsize);                   \
+    counts[xcount].mapped  = (xsize);                   \
+    counts[xcount].free    = (xfree);                   \
+    counts[xcount].largest = (xlargest);                \

     counts[xcount].flags   = VM_KERN_SITE_COUNTER;
 
     counts[xcount].flags   = VM_KERN_SITE_COUNTER;
 

-    vm_map_size_t map_size, map_free, map_largest;
+       vm_map_size_t map_size, map_free, map_largest;

 
 

-    vm_map_sizes(kernel_map, &map_size, &map_free, &map_largest);
-    SET_MAP(VM_KERN_COUNT_MAP_KERNEL, map_size, map_free, map_largest);
+       vm_map_sizes(kernel_map, &map_size, &map_free, &map_largest);
+       SET_MAP(VM_KERN_COUNT_MAP_KERNEL, map_size, map_free, map_largest);

 
 

-    vm_map_sizes(zone_map, &map_size, &map_free, &map_largest);
-    SET_MAP(VM_KERN_COUNT_MAP_ZONE, map_size, map_free, map_largest);
+       zone_map_sizes(&map_size, &map_free, &map_largest);
+       SET_MAP(VM_KERN_COUNT_MAP_ZONE, map_size, map_free, map_largest);

 
 

-    vm_map_sizes(kalloc_map, &map_size, &map_free, &map_largest);
-    SET_MAP(VM_KERN_COUNT_MAP_KALLOC, map_size, map_free, map_largest);
+       vm_map_sizes(kalloc_map, &map_size, &map_free, &map_largest);
+       SET_MAP(VM_KERN_COUNT_MAP_KALLOC, map_size, map_free, map_largest);

 
 

-    iterate = !VM_TAG_ACTIVE_UPDATE;
-    if (iterate)
-       {
-               enum                       { kMaxKernelDepth = 1 };
-               vm_map_t                     maps   [kMaxKernelDepth];
-               vm_map_entry_t               entries[kMaxKernelDepth];
-               vm_map_t                     map;
-               vm_map_entry_t               entry;
-               vm_object_offset_t           offset;
-               vm_page_t                    page;
-               int                          stackIdx, count;
-
-#if ! VM_TAG_ACTIVE_UPDATE
-           vm_page_iterate_objects(info, num_info, &vm_page_count_object);
+       assert(num_info >= zone_view_count);
+       num_info -= zone_view_count;
+       counts = &info[num_info];
+       i = 0;
+
+       i += vm_page_diagnose_heap(counts + i, KHEAP_DEFAULT);
+       if (KHEAP_DATA_BUFFERS->kh_heap_id == KHEAP_ID_DATA_BUFFERS) {
+               i += vm_page_diagnose_heap(counts + i, KHEAP_DATA_BUFFERS);
+       }
+       if (KHEAP_KEXT->kh_heap_id == KHEAP_ID_KEXT) {
+               i += vm_page_diagnose_heap(counts + i, KHEAP_KEXT);
+       }
+       assert(i <= zone_view_count);
+
+       zone_index_foreach(zidx) {
+               zone_t z = &zone_array[zidx];
+               zone_view_t zv = z->z_views;
+
+               if (zv == NULL) {
+                       continue;
+               }
+
+               if (z->kalloc_heap == KHEAP_ID_NONE) {
+                       vm_page_diagnose_zone(counts + i, z);
+                       i++;
+                       assert(i <= zone_view_count);
+               }
+
+               for (; zv; zv = zv->zv_next) {
+                       vm_page_diagnose_zone_stats(counts + i, zv->zv_stats,
+                           z->percpu);
+                       snprintf(counts[i].name, sizeof(counts[i].name), "%s%s[%s]",
+                           zone_heap_name(z), z->z_name, zv->zv_name);
+                       i++;
+                       assert(i <= zone_view_count);
+               }
+       }
+
+       iterate = !VM_TAG_ACTIVE_UPDATE;
+       if (iterate) {
+               enum                       { kMaxKernelDepth = 1 };
+               vm_map_t                     maps[kMaxKernelDepth];
+               vm_map_entry_t               entries[kMaxKernelDepth];
+               vm_map_t                     map;
+               vm_map_entry_t               entry;
+               vm_object_offset_t           offset;
+               vm_page_t                    page;
+               int                          stackIdx, count;
+
+#if !VM_TAG_ACTIVE_UPDATE
+               vm_page_iterate_objects(info, num_info, &vm_page_count_object);

 #endif /* ! VM_TAG_ACTIVE_UPDATE */
 
 #endif /* ! VM_TAG_ACTIVE_UPDATE */
 

-           map = kernel_map;
-           stackIdx = 0;
-           while (map)
-           {
+               map = kernel_map;
+               stackIdx = 0;
+               while (map) {

                        vm_map_lock(map);
                        vm_map_lock(map);

-                       for (entry = map->hdr.links.next; map; entry = entry->links.next)
-                       {
-                           if (entry->is_sub_map)
-                           {
+                       for (entry = map->hdr.links.next; map; entry = entry->links.next) {
+                               if (entry->is_sub_map) {

                                        assert(stackIdx < kMaxKernelDepth);
                                        maps[stackIdx] = map;
                                        entries[stackIdx] = entry;
                                        assert(stackIdx < kMaxKernelDepth);
                                        maps[stackIdx] = map;
                                        entries[stackIdx] = entry;


@@ -9045,43 +9659,41 @@ vm_page_diagnose(mach_memory_info_t * info, unsigned int num_info, uint64_t zone
                                        map = VME_SUBMAP(entry);
                                        entry = NULL;
                                        break;
                                        map = VME_SUBMAP(entry);
                                        entry = NULL;
                                        break;

-                           }
-                           if (VME_OBJECT(entry) == kernel_object)
-                           {
+                               }
+                               if (VME_OBJECT(entry) == kernel_object) {

                                        count = 0;
                                        vm_object_lock(VME_OBJECT(entry));
                                        count = 0;
                                        vm_object_lock(VME_OBJECT(entry));

-                                       for (offset = entry->links.start; offset < entry->links.end; offset += page_size)
-                                       {
+                                       for (offset = entry->links.start; offset < entry->links.end; offset += page_size) {

                                                page = vm_page_lookup(VME_OBJECT(entry), offset);
                                                page = vm_page_lookup(VME_OBJECT(entry), offset);

-                                               if (page && VM_PAGE_WIRED(page)) count++;
+                                               if (page && VM_PAGE_WIRED(page)) {
+                                                       count++;
+                                               }

                                        }
                                        vm_object_unlock(VME_OBJECT(entry));
 
                                        }
                                        vm_object_unlock(VME_OBJECT(entry));
 

-                                       if (count)
-                                       {
-                                           assert(VME_ALIAS(entry) != VM_KERN_MEMORY_NONE);
-                                           assert(VME_ALIAS(entry) < num_info);
-                                           info[VME_ALIAS(entry)].size += ptoa_64(count);
+                                       if (count) {
+                                               assert(VME_ALIAS(entry) != VM_KERN_MEMORY_NONE);
+                                               assert(VME_ALIAS(entry) < num_info);
+                                               info[VME_ALIAS(entry)].size += ptoa_64(count);

                                        }
                                        }

-                           }
-                           while (map && (entry == vm_map_last_entry(map)))
-                           {
+                               }
+                               while (map && (entry == vm_map_last_entry(map))) {

                                        vm_map_unlock(map);
                                        vm_map_unlock(map);

-                                       if (!stackIdx) map = NULL;
-                                       else
-                                       {
-                                           --stackIdx;
-                                           map = maps[stackIdx];
-                                           entry = entries[stackIdx];
+                                       if (!stackIdx) {
+                                               map = NULL;
+                                       } else {
+                                               --stackIdx;
+                                               map = maps[stackIdx];
+                                               entry = entries[stackIdx];

                                        }
                                        }

-                           }
+                               }

                        }
                        }

-           }
-    }
+               }
+       }

 
 

-    process_account(info, num_info, zones_collectable_bytes, iterate);
-    
-    return (KERN_SUCCESS);
+       process_account(info, num_info, zones_collectable_bytes, iterate);
+
+       return KERN_SUCCESS;

 }
 
 #if DEBUG || DEVELOPMENT
 }
 
 #if DEBUG || DEVELOPMENT


@@ -9089,40 +9701,45 @@ vm_page_diagnose(mach_memory_info_t * info, unsigned int num_info, uint64_t zone
 kern_return_t
 vm_kern_allocation_info(uintptr_t addr, vm_size_t * size, vm_tag_t * tag, vm_size_t * zone_size)
 {
 kern_return_t
 vm_kern_allocation_info(uintptr_t addr, vm_size_t * size, vm_tag_t * tag, vm_size_t * zone_size)
 {

-    kern_return_t  ret;
-    vm_size_t      zsize;
-    vm_map_t       map;
-    vm_map_entry_t entry;
+       kern_return_t  ret;
+       vm_size_t      zsize;
+       vm_map_t       map;
+       vm_map_entry_t entry;

 
 

-    zsize = zone_element_info((void *) addr, tag);
-    if (zsize)
-    {
+       zsize = zone_element_info((void *) addr, tag);
+       if (zsize) {

                *zone_size = *size = zsize;
                *zone_size = *size = zsize;

-               return (KERN_SUCCESS);
-    }
+               return KERN_SUCCESS;
+       }

 
        *zone_size = 0;
 
        *zone_size = 0;

-    ret = KERN_INVALID_ADDRESS;
-    for (map = kernel_map; map; )
-       {
+       ret = KERN_INVALID_ADDRESS;
+       for (map = kernel_map; map;) {

                vm_map_lock(map);
                vm_map_lock(map);

-               if (!vm_map_lookup_entry(map, addr, &entry)) break;
-           if (entry->is_sub_map)
-           {
-            if (map != kernel_map)                   break;
+               if (!vm_map_lookup_entry(map, addr, &entry)) {
+                       break;
+               }
+               if (entry->is_sub_map) {
+                       if (map != kernel_map) {
+                               break;
+                       }

                        map = VME_SUBMAP(entry);
                        continue;
                        map = VME_SUBMAP(entry);
                        continue;

-           }
-               if (entry->vme_start != addr)                break;
-               *tag = VME_ALIAS(entry);
+               }
+               if (entry->vme_start != addr) {
+                       break;
+               }
+               *tag = (vm_tag_t)VME_ALIAS(entry);

                *size = (entry->vme_end - addr);
                ret = KERN_SUCCESS;
                break;
        }
                *size = (entry->vme_end - addr);
                ret = KERN_SUCCESS;
                break;
        }

-       if (map != kernel_map) vm_map_unlock(map);
+       if (map != kernel_map) {
+               vm_map_unlock(map);
+       }

        vm_map_unlock(kernel_map);
 
        vm_map_unlock(kernel_map);
 

-       return (ret);
+       return ret;

 }
 
 #endif /* DEBUG || DEVELOPMENT */
 }
 
 #endif /* DEBUG || DEVELOPMENT */


@@ -9130,21 +9747,19 @@ vm_kern_allocation_info(uintptr_t addr, vm_size_t * size, vm_tag_t * tag, vm_siz
 uint32_t
 vm_tag_get_kext(vm_tag_t tag, char * name, vm_size_t namelen)
 {
 uint32_t
 vm_tag_get_kext(vm_tag_t tag, char * name, vm_size_t namelen)
 {

-    vm_allocation_site_t * site;
-    uint32_t               kmodId;
+       vm_allocation_site_t * site;
+       uint32_t               kmodId;

 
 

-    kmodId = 0;
-    lck_spin_lock(&vm_allocation_sites_lock);
-    if ((site = vm_allocation_sites[tag]))
-    {
-        if (VM_TAG_KMOD & site->flags)
-        {
-            kmodId = OSKextGetKmodIDForSite(site, name, namelen);
-        }
-    }
-    lck_spin_unlock(&vm_allocation_sites_lock);
+       kmodId = 0;
+       lck_spin_lock(&vm_allocation_sites_lock);
+       if ((site = vm_allocation_sites[tag])) {
+               if (VM_TAG_KMOD & site->flags) {
+                       kmodId = OSKextGetKmodIDForSite(site, name, namelen);
+               }
+       }
+       lck_spin_unlock(&vm_allocation_sites_lock);

 
 

-    return (kmodId);
+       return kmodId;

 }
 
 
 }
 
 


@@ -9160,33 +9775,21 @@ vm_tag_get_kext(vm_tag_t tag, char * name, vm_size_t namelen)
 unsigned int vm_page_secluded_suppress_cnt = 0;
 unsigned int vm_page_secluded_save_target;
 
 unsigned int vm_page_secluded_suppress_cnt = 0;
 unsigned int vm_page_secluded_save_target;
 

-
-lck_grp_attr_t secluded_suppress_slock_grp_attr;
-lck_grp_t      secluded_suppress_slock_grp;
-lck_attr_t     secluded_suppress_slock_attr;
-lck_spin_t     secluded_suppress_slock;
-
-void
-secluded_suppression_init(void)
-{
-       lck_grp_attr_setdefault(&secluded_suppress_slock_grp_attr);
-       lck_grp_init(&secluded_suppress_slock_grp,
-           "secluded_suppress_slock",  &secluded_suppress_slock_grp_attr);
-       lck_attr_setdefault(&secluded_suppress_slock_attr);
-       lck_spin_init(&secluded_suppress_slock,
-           &secluded_suppress_slock_grp, &secluded_suppress_slock_attr);
-}
+LCK_GRP_DECLARE(secluded_suppress_slock_grp, "secluded_suppress_slock");
+LCK_SPIN_DECLARE(secluded_suppress_slock, &secluded_suppress_slock_grp);

 
 void
 start_secluded_suppression(task_t task)
 {
 
 void
 start_secluded_suppression(task_t task)
 {

-       if (task->task_suppressed_secluded)
+       if (task->task_suppressed_secluded) {

                return;
                return;

+       }

        lck_spin_lock(&secluded_suppress_slock);
        if (!task->task_suppressed_secluded && vm_page_secluded_suppress_cnt++ == 0) {
                task->task_suppressed_secluded = TRUE;
                vm_page_secluded_save_target = vm_page_secluded_target;
                vm_page_secluded_target = 0;
        lck_spin_lock(&secluded_suppress_slock);
        if (!task->task_suppressed_secluded && vm_page_secluded_suppress_cnt++ == 0) {
                task->task_suppressed_secluded = TRUE;
                vm_page_secluded_save_target = vm_page_secluded_target;
                vm_page_secluded_target = 0;

+               VM_PAGE_SECLUDED_COUNT_OVER_TARGET_UPDATE();

        }
        lck_spin_unlock(&secluded_suppress_slock);
 }
        }
        lck_spin_unlock(&secluded_suppress_slock);
 }


@@ -9198,6 +9801,7 @@ stop_secluded_suppression(task_t task)
        if (task->task_suppressed_secluded && --vm_page_secluded_suppress_cnt == 0) {
                task->task_suppressed_secluded = FALSE;
                vm_page_secluded_target = vm_page_secluded_save_target;
        if (task->task_suppressed_secluded && --vm_page_secluded_suppress_cnt == 0) {
                task->task_suppressed_secluded = FALSE;
                vm_page_secluded_target = vm_page_secluded_save_target;

+               VM_PAGE_SECLUDED_COUNT_OVER_TARGET_UPDATE();

        }
        lck_spin_unlock(&secluded_suppress_slock);
 }
        }
        lck_spin_unlock(&secluded_suppress_slock);
 }