xnu-7195.101.1.tar.gz

[apple/xnu.git] / osfmk / kern / gzalloc.c

diff --git a/osfmk/kern/gzalloc.c b/osfmk/kern/gzalloc.c
index 915b2278276d9ae546d3c51ad7949185f8c10101..54ec12be25c702b806c247de3ff9dc7fa2f79cf5 100644 (file)
--- a/osfmk/kern/gzalloc.c
+++ b/osfmk/kern/gzalloc.c
@@ -1,8 +1,8 @@
 /*
 /*

- * Copyright (c) 2000-2012 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2020 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

- * 
+ *

  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in


@@ -11,10 +11,10 @@
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.
  * unlawful or unlicensed copies of an Apple operating system, or to
  * circumvent, violate, or enable the circumvention or violation of, any
  * terms of an Apple operating system software license agreement.

- * 
+ *

  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  * Please obtain a copy of the License at
  * http://www.opensource.apple.com/apsl/ and read it before using this file.

- * 
+ *

  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,


@@ -22,7 +22,7 @@
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.
  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  * Please see the License for the specific language governing rights and
  * limitations under the License.

- * 
+ *

  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 /*


@@ -37,7 +37,7 @@
  *     gzalloc_size=<size>: target all zones with elements of <size> bytes
  *     gzalloc_min=<size>: target zones with elements >= size
  *     gzalloc_max=<size>: target zones with elements <= size
  *     gzalloc_size=<size>: target all zones with elements of <size> bytes
  *     gzalloc_min=<size>: target zones with elements >= size
  *     gzalloc_max=<size>: target zones with elements <= size

- *     gzalloc_min/max can be specified in conjunction to target a range of
+ *      gzalloc_min/max can be specified in conjunction to target a range of

  *     sizes
  *     gzalloc_fc_size=<size>: number of zone elements (effectively page
  *     multiple sized) to retain in the free VA cache. This cache is evicted
  *     sizes
  *     gzalloc_fc_size=<size>: number of zone elements (effectively page
  *     multiple sized) to retain in the free VA cache. This cache is evicted


@@ -57,7 +57,7 @@
  *     -gzalloc_noconsistency: disable consistency checks that flag mismatched
  *     frees, corruptions of the header/trailer signatures etc.
  *     -nogzalloc_mode: Disables the guard mode allocator. The DEBUG kernel
  *     -gzalloc_noconsistency: disable consistency checks that flag mismatched
  *     frees, corruptions of the header/trailer signatures etc.
  *     -nogzalloc_mode: Disables the guard mode allocator. The DEBUG kernel

- *     enables the guard allocator for zones sized 8K-16K (if present) by
+ *     enables the guard allocator for zones sized 1K (if present) by

  *     default, this option can disable that behaviour.
  *     gzname=<name> target a zone by name. Can be coupled with size-based
  *     targeting. Naming conventions match those of the zlog boot-arg, i.e.
  *     default, this option can disable that behaviour.
  *     gzname=<name> target a zone by name. Can be coupled with size-based
  *     targeting. Naming conventions match those of the zlog boot-arg, i.e.


@@ -66,8 +66,6 @@
  *     gzalloc_zscale=<value> specify size multiplier for the dedicated gzalloc submap
  */
 
  *     gzalloc_zscale=<value> specify size multiplier for the dedicated gzalloc submap
  */
 

-#include <zone_debug.h>
-

 #include <mach/mach_types.h>
 #include <mach/vm_param.h>
 #include <mach/kern_return.h>
 #include <mach/mach_types.h>
 #include <mach/vm_param.h>
 #include <mach/kern_return.h>


@@ -80,8 +78,7 @@
 #include <kern/sched.h>
 #include <kern/locks.h>
 #include <kern/misc_protos.h>
 #include <kern/sched.h>
 #include <kern/locks.h>
 #include <kern/misc_protos.h>

-#include <kern/zalloc.h>
-#include <kern/kalloc.h>
+#include <kern/zalloc_internal.h>

 
 #include <vm/pmap.h>
 #include <vm/vm_map.h>
 
 #include <vm/pmap.h>
 #include <vm/vm_map.h>


@@ -96,11 +93,10 @@
 #include <libkern/OSAtomic.h>
 #include <sys/kdebug.h>
 
 #include <libkern/OSAtomic.h>
 #include <sys/kdebug.h>
 

-extern boolean_t vm_kernel_ready, kmem_ready;

 boolean_t gzalloc_mode = FALSE;
 uint32_t pdzalloc_count, pdzfree_count;
 
 boolean_t gzalloc_mode = FALSE;
 uint32_t pdzalloc_count, pdzfree_count;
 

-#define        GZALLOC_MIN_DEFAULT (1024)
+#define GZALLOC_MIN_DEFAULT (1024)

 #define GZDEADZONE ((zone_t) 0xDEAD201E)
 #define GZALLOC_SIGNATURE (0xABADCAFE)
 #define GZALLOC_RESERVE_SIZE_DEFAULT (2 * 1024 * 1024)
 #define GZDEADZONE ((zone_t) 0xDEAD201E)
 #define GZALLOC_SIGNATURE (0xABADCAFE)
 #define GZALLOC_RESERVE_SIZE_DEFAULT (2 * 1024 * 1024)


@@ -136,115 +132,123 @@ extern zone_t vm_page_zone;
 static zone_t gztrackzone = NULL;
 static char gznamedzone[MAX_ZONE_NAME] = "";
 
 static zone_t gztrackzone = NULL;
 static char gznamedzone[MAX_ZONE_NAME] = "";
 

-void gzalloc_reconfigure(__unused zone_t z) {
-       /* Nothing for now */
-}
-
-boolean_t gzalloc_enabled(void) {
+boolean_t
+gzalloc_enabled(void)
+{

        return gzalloc_mode;
 }
 
        return gzalloc_mode;
 }
 

-static inline boolean_t gzalloc_tracked(zone_t z) {
-       return (gzalloc_mode &&
-           (((z->elem_size >= gzalloc_min) && (z->elem_size <= gzalloc_max)) || (z == gztrackzone)) &&
-           (z->gzalloc_exempt == 0));
-}
+void
+gzalloc_zone_init(zone_t z)
+{
+       if (gzalloc_mode == 0) {
+               return;
+       }

 
 

-void gzalloc_zone_init(zone_t z) {
-       if (gzalloc_mode) {
-               bzero(&z->gz, sizeof(z->gz));
+       bzero(&z->gz, sizeof(z->gz));

 
 

-               if (track_this_zone(z->zone_name, gznamedzone)) {
-                       gztrackzone = z;
-               }
+       if (track_this_zone(z->z_name, gznamedzone)) {
+               gztrackzone = z;
+       }

 
 

-               if (gzfc_size &&
-                   gzalloc_tracked(z)) {
-                       vm_size_t gzfcsz = round_page(sizeof(*z->gz.gzfc) * gzfc_size);
-
-                       /* If the VM/kmem system aren't yet configured, carve
-                        * out the free element cache structure directly from the
-                        * gzalloc_reserve supplied by the pmap layer.
-                       */
-                       if (!kmem_ready) {
-                               if (gzalloc_reserve_size < gzfcsz)
-                                       panic("gzalloc reserve exhausted");
-
-                               z->gz.gzfc = (vm_offset_t *)gzalloc_reserve;
-                               gzalloc_reserve += gzfcsz;
-                               gzalloc_reserve_size -= gzfcsz;
-                       } else {
-                               kern_return_t kr;
-
-                               if ((kr = kernel_memory_allocate(kernel_map, (vm_offset_t *)&z->gz.gzfc, gzfcsz, 0, KMA_KOBJECT, VM_KERN_MEMORY_OSFMK)) != KERN_SUCCESS) {
-                                       panic("zinit/gzalloc: kernel_memory_allocate failed (%d) for 0x%lx bytes", kr, (unsigned long) gzfcsz);
-                               }
+       if (!z->gzalloc_exempt) {
+               z->gzalloc_tracked = (z == gztrackzone) ||
+                   ((zone_elem_size(z) >= gzalloc_min) && (zone_elem_size(z) <= gzalloc_max));
+       }
+
+       if (gzfc_size && z->gzalloc_tracked) {
+               vm_size_t gzfcsz = round_page(sizeof(*z->gz.gzfc) * gzfc_size);
+               kern_return_t kr;
+
+               /* If the VM/kmem system aren't yet configured, carve
+                * out the free element cache structure directly from the
+                * gzalloc_reserve supplied by the pmap layer.
+                */
+               if (__improbable(startup_phase < STARTUP_SUB_KMEM)) {
+                       if (gzalloc_reserve_size < gzfcsz) {
+                               panic("gzalloc reserve exhausted");
+                       }
+
+                       z->gz.gzfc = (vm_offset_t *)gzalloc_reserve;
+                       gzalloc_reserve += gzfcsz;
+                       gzalloc_reserve_size -= gzfcsz;
+                       bzero(z->gz.gzfc, gzfcsz);
+               } else {
+                       kr = kernel_memory_allocate(kernel_map,
+                           (vm_offset_t *)&z->gz.gzfc, gzfcsz, 0,
+                           KMA_KOBJECT | KMA_ZERO, VM_KERN_MEMORY_OSFMK);
+                       if (kr != KERN_SUCCESS) {
+                               panic("%s: kernel_memory_allocate failed (%d) for 0x%lx bytes",
+                                   __func__, kr, (unsigned long)gzfcsz);

                        }
                        }

-                       bzero((void *)z->gz.gzfc, gzfcsz);

                }
        }
 }
 
 /* Called by zdestroy() to dump the free cache elements so the zone count can drop to zero. */
                }
        }
 }
 
 /* Called by zdestroy() to dump the free cache elements so the zone count can drop to zero. */

-void gzalloc_empty_free_cache(zone_t zone) {
-       if (__improbable(gzalloc_tracked(zone))) {
-               kern_return_t kr;
-               int freed_elements = 0;
-               vm_offset_t free_addr = 0;
-               vm_offset_t rounded_size = round_page(zone->elem_size + GZHEADER_SIZE);
-               vm_offset_t gzfcsz = round_page(sizeof(*zone->gz.gzfc) * gzfc_size);
-               vm_offset_t gzfc_copy;
+void
+gzalloc_empty_free_cache(zone_t zone)
+{
+       kern_return_t kr;
+       int freed_elements = 0;
+       vm_offset_t free_addr = 0;
+       vm_offset_t rounded_size = round_page(zone_elem_size(zone) + GZHEADER_SIZE);
+       vm_offset_t gzfcsz = round_page(sizeof(*zone->gz.gzfc) * gzfc_size);
+       vm_offset_t gzfc_copy;

 
 

-               kr = kmem_alloc(kernel_map, &gzfc_copy, gzfcsz, VM_KERN_MEMORY_OSFMK);
-               if (kr != KERN_SUCCESS) {
-                       panic("gzalloc_empty_free_cache: kmem_alloc: 0x%x", kr);
-               }
+       assert(zone->gzalloc_tracked); // the caller is responsible for checking

 
 

-               /* Reset gzalloc_data. */
-               lock_zone(zone);
-               memcpy((void *)gzfc_copy, (void *)zone->gz.gzfc, gzfcsz);
-               bzero((void *)zone->gz.gzfc, gzfcsz);
-               zone->gz.gzfc_index = 0;
-               unlock_zone(zone);
-
-               /* Free up all the cached elements. */
-               for (uint32_t index = 0; index < gzfc_size; index++) {
-                       free_addr = ((vm_offset_t *)gzfc_copy)[index];
-                       if (free_addr && free_addr >= gzalloc_map_min && free_addr < gzalloc_map_max) {
-                               kr = vm_map_remove(
-                                               gzalloc_map,
-                                               free_addr,
-                                               free_addr + rounded_size + (1 * PAGE_SIZE),
-                                               VM_MAP_REMOVE_KUNWIRE);
-                               if (kr != KERN_SUCCESS) {
-                                       panic("gzalloc_empty_free_cache: vm_map_remove: %p, 0x%x", (void *)free_addr, kr);
-                               }
-                               OSAddAtomic64((SInt32)rounded_size, &gzalloc_freed);
-                               OSAddAtomic64(-((SInt32) (rounded_size - zone->elem_size)), &gzalloc_wasted);
+       kr = kmem_alloc(kernel_map, &gzfc_copy, gzfcsz, VM_KERN_MEMORY_OSFMK);
+       if (kr != KERN_SUCCESS) {
+               panic("gzalloc_empty_free_cache: kmem_alloc: 0x%x", kr);
+       }

 
 

-                               freed_elements++;
+       /* Reset gzalloc_data. */
+       zone_lock(zone);
+       memcpy((void *)gzfc_copy, (void *)zone->gz.gzfc, gzfcsz);
+       bzero((void *)zone->gz.gzfc, gzfcsz);
+       zone->gz.gzfc_index = 0;
+       zone_unlock(zone);
+
+       /* Free up all the cached elements. */
+       for (uint32_t index = 0; index < gzfc_size; index++) {
+               free_addr = ((vm_offset_t *)gzfc_copy)[index];
+               if (free_addr && free_addr >= gzalloc_map_min && free_addr < gzalloc_map_max) {
+                       kr = vm_map_remove(gzalloc_map, free_addr,
+                           free_addr + rounded_size + (1 * PAGE_SIZE),
+                           VM_MAP_REMOVE_KUNWIRE);
+                       if (kr != KERN_SUCCESS) {
+                               panic("gzalloc_empty_free_cache: vm_map_remove: %p, 0x%x", (void *)free_addr, kr);

                        }
                        }

-               }
-               /*
-                * TODO: Consider freeing up zone->gz.gzfc as well if it didn't come from the gzalloc_reserve pool.
-                * For now we're reusing this buffer across zdestroy's. We would have to allocate it again on a
-                * subsequent zinit() as well.
-                */
-
-               /* Decrement zone counters. */
-               lock_zone(zone);
-               zone->count -= freed_elements;
-               zone->cur_size -= (freed_elements * rounded_size);
-               unlock_zone(zone);
+                       OSAddAtomic64((SInt32)rounded_size, &gzalloc_freed);
+                       OSAddAtomic64(-((SInt32) (rounded_size - zone_elem_size(zone))), &gzalloc_wasted);

 
 

-               kmem_free(kernel_map, gzfc_copy, gzfcsz);
+                       freed_elements++;
+               }

        }
        }

+       /*
+        * TODO: Consider freeing up zone->gz.gzfc as well if it didn't come from the gzalloc_reserve pool.
+        * For now we're reusing this buffer across zdestroy's. We would have to allocate it again on a
+        * subsequent zinit() as well.
+        */
+
+       /* Decrement zone counters. */
+       zone_lock(zone);
+       zone->z_elems_free += freed_elements;
+       zone->z_wired_cur -= freed_elements;
+       zone_unlock(zone);
+
+       kmem_free(kernel_map, gzfc_copy, gzfcsz);

 }
 
 }
 

-void gzalloc_configure(void) {
+__startup_func
+static void
+gzalloc_configure(void)
+{
+#if !KASAN_ZALLOC

        char temp_buf[16];
 
        char temp_buf[16];
 

-       if (PE_parse_boot_argn("-gzalloc_mode", temp_buf, sizeof (temp_buf))) {
+       if (PE_parse_boot_argn("-gzalloc_mode", temp_buf, sizeof(temp_buf))) {

                gzalloc_mode = TRUE;
                gzalloc_min = GZALLOC_MIN_DEFAULT;
                gzalloc_max = ~0U;
                gzalloc_mode = TRUE;
                gzalloc_min = GZALLOC_MIN_DEFAULT;
                gzalloc_max = ~0U;


@@ -257,8 +261,9 @@ void gzalloc_configure(void) {
 
        if (PE_parse_boot_argn("gzalloc_max", &gzalloc_max, sizeof(gzalloc_max))) {
                gzalloc_mode = TRUE;
 
        if (PE_parse_boot_argn("gzalloc_max", &gzalloc_max, sizeof(gzalloc_max))) {
                gzalloc_mode = TRUE;

-               if (gzalloc_min == ~0U)
+               if (gzalloc_min == ~0U) {

                        gzalloc_min = 0;
                        gzalloc_min = 0;

+               }

        }
 
        if (PE_parse_boot_argn("gzalloc_size", &gzalloc_size, sizeof(gzalloc_size))) {
        }
 
        if (PE_parse_boot_argn("gzalloc_size", &gzalloc_size, sizeof(gzalloc_size))) {


@@ -268,11 +273,11 @@ void gzalloc_configure(void) {
 
        (void)PE_parse_boot_argn("gzalloc_fc_size", &gzfc_size, sizeof(gzfc_size));
 
 
        (void)PE_parse_boot_argn("gzalloc_fc_size", &gzfc_size, sizeof(gzfc_size));
 

-       if (PE_parse_boot_argn("-gzalloc_wp", temp_buf, sizeof (temp_buf))) {
+       if (PE_parse_boot_argn("-gzalloc_wp", temp_buf, sizeof(temp_buf))) {

                gzalloc_prot = VM_PROT_READ;
        }
 
                gzalloc_prot = VM_PROT_READ;
        }
 

-       if (PE_parse_boot_argn("-gzalloc_uf_mode", temp_buf, sizeof (temp_buf))) {
+       if (PE_parse_boot_argn("-gzalloc_uf_mode", temp_buf, sizeof(temp_buf))) {

                gzalloc_uf_mode = TRUE;
                gzalloc_guard = KMA_GUARD_FIRST;
        }
                gzalloc_uf_mode = TRUE;
                gzalloc_guard = KMA_GUARD_FIRST;
        }


@@ -283,7 +288,7 @@ void gzalloc_configure(void) {
 
        (void) PE_parse_boot_argn("gzalloc_zscale", &gzalloc_zonemap_scale, sizeof(gzalloc_zonemap_scale));
 
 
        (void) PE_parse_boot_argn("gzalloc_zscale", &gzalloc_zonemap_scale, sizeof(gzalloc_zonemap_scale));
 

-       if (PE_parse_boot_argn("-gzalloc_noconsistency", temp_buf, sizeof (temp_buf))) {
+       if (PE_parse_boot_argn("-gzalloc_noconsistency", temp_buf, sizeof(temp_buf))) {

                gzalloc_consistency_checks = FALSE;
        }
 
                gzalloc_consistency_checks = FALSE;
        }
 


@@ -299,16 +304,21 @@ void gzalloc_configure(void) {
                gzalloc_mode = TRUE;
        }
 #endif
                gzalloc_mode = TRUE;
        }
 #endif

-       if (PE_parse_boot_argn("-nogzalloc_mode", temp_buf, sizeof (temp_buf)))
+       if (PE_parse_boot_argn("-nogzalloc_mode", temp_buf, sizeof(temp_buf))) {

                gzalloc_mode = FALSE;
                gzalloc_mode = FALSE;

+       }

 
        if (gzalloc_mode) {
                gzalloc_reserve_size = GZALLOC_RESERVE_SIZE_DEFAULT;
                gzalloc_reserve = (vm_offset_t) pmap_steal_memory(gzalloc_reserve_size);
        }
 
        if (gzalloc_mode) {
                gzalloc_reserve_size = GZALLOC_RESERVE_SIZE_DEFAULT;
                gzalloc_reserve = (vm_offset_t) pmap_steal_memory(gzalloc_reserve_size);
        }

+#endif

 }
 }

+STARTUP(PMAP_STEAL, STARTUP_RANK_FIRST, gzalloc_configure);

 
 

-void gzalloc_init(vm_size_t max_zonemap_size) {
+void
+gzalloc_init(vm_size_t max_zonemap_size)
+{

        kern_return_t retval;
 
        if (gzalloc_mode) {
        kern_return_t retval;
 
        if (gzalloc_mode) {


@@ -317,72 +327,80 @@ void gzalloc_init(vm_size_t max_zonemap_size) {
                vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
                vmk_flags.vmkf_permanent = TRUE;
                retval = kmem_suballoc(kernel_map, &gzalloc_map_min, (max_zonemap_size * gzalloc_zonemap_scale),
                vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
                vmk_flags.vmkf_permanent = TRUE;
                retval = kmem_suballoc(kernel_map, &gzalloc_map_min, (max_zonemap_size * gzalloc_zonemap_scale),

-                                      FALSE, VM_FLAGS_ANYWHERE, vmk_flags, VM_KERN_MEMORY_ZONE,
-                                      &gzalloc_map);
-       
+                   FALSE, VM_FLAGS_ANYWHERE, vmk_flags, VM_KERN_MEMORY_ZONE,
+                   &gzalloc_map);
+

                if (retval != KERN_SUCCESS) {
                if (retval != KERN_SUCCESS) {

-                       panic("zone_init: kmem_suballoc(gzalloc_map, 0x%lx, %u) failed", max_zonemap_size, gzalloc_zonemap_scale);
+                       panic("zone_init: kmem_suballoc(gzalloc_map, 0x%lx, %u) failed",
+                           max_zonemap_size, gzalloc_zonemap_scale);

                }
                gzalloc_map_max = gzalloc_map_min + (max_zonemap_size * gzalloc_zonemap_scale);
        }
 }
 
 vm_offset_t
                }
                gzalloc_map_max = gzalloc_map_min + (max_zonemap_size * gzalloc_zonemap_scale);
        }
 }
 
 vm_offset_t

-gzalloc_alloc(zone_t zone, boolean_t canblock) {
+gzalloc_alloc(zone_t zone, zone_stats_t zstats, zalloc_flags_t flags)
+{

        vm_offset_t addr = 0;
 
        vm_offset_t addr = 0;
 

-       if (__improbable(gzalloc_tracked(zone))) {
+       assert(zone->gzalloc_tracked); // the caller is responsible for checking

 
 

-               if (get_preemption_level() != 0) {
-                       if (canblock == TRUE) {
-                               pdzalloc_count++;
-                       }
-                       else
-                               return 0;
+       if (get_preemption_level() != 0) {
+               if (flags & Z_NOWAIT) {
+                       return 0;

                }
                }

+               pdzalloc_count++;
+       }

 
 

-               vm_offset_t rounded_size = round_page(zone->elem_size + GZHEADER_SIZE);
-               vm_offset_t residue = rounded_size - zone->elem_size;
-               vm_offset_t gzaddr = 0;
-               gzhdr_t *gzh, *gzhcopy = NULL;
+       bool kmem_ready = (startup_phase >= STARTUP_SUB_KMEM);
+       vm_offset_t rounded_size = round_page(zone_elem_size(zone) + GZHEADER_SIZE);
+       vm_offset_t residue = rounded_size - zone_elem_size(zone);
+       vm_offset_t gzaddr = 0;
+       gzhdr_t *gzh, *gzhcopy = NULL;
+       bool new_va = false;

 
 

-               if (!kmem_ready || (vm_page_zone == ZONE_NULL)) {
-                       /* Early allocations are supplied directly from the
-                        * reserve.
-                        */
-                       if (gzalloc_reserve_size < (rounded_size + PAGE_SIZE))
-                               panic("gzalloc reserve exhausted");
-                       gzaddr = gzalloc_reserve;
-                       /* No guard page for these early allocations, just
-                        * waste an additional page.
-                        */
-                       gzalloc_reserve += rounded_size + PAGE_SIZE;
-                       gzalloc_reserve_size -= rounded_size + PAGE_SIZE;
-                       OSAddAtomic64((SInt32) (rounded_size), &gzalloc_early_alloc);
+       if (!kmem_ready || (vm_page_zone == ZONE_NULL)) {
+               /* Early allocations are supplied directly from the
+                * reserve.
+                */
+               if (gzalloc_reserve_size < (rounded_size + PAGE_SIZE)) {
+                       panic("gzalloc reserve exhausted");

                }
                }

-               else {
-                       kern_return_t kr = kernel_memory_allocate(gzalloc_map,
-                           &gzaddr, rounded_size + (1*PAGE_SIZE),
-                           0, KMA_KOBJECT | KMA_ATOMIC | gzalloc_guard,
-                           VM_KERN_MEMORY_OSFMK);
-                       if (kr != KERN_SUCCESS)
-                               panic("gzalloc: kernel_memory_allocate for size 0x%llx failed with %d", (uint64_t)rounded_size, kr);
-
+               gzaddr = gzalloc_reserve;
+               /* No guard page for these early allocations, just
+                * waste an additional page.
+                */
+               gzalloc_reserve += rounded_size + PAGE_SIZE;
+               gzalloc_reserve_size -= rounded_size + PAGE_SIZE;
+               OSAddAtomic64((SInt32) (rounded_size), &gzalloc_early_alloc);
+       } else {
+               kern_return_t kr = kernel_memory_allocate(gzalloc_map,
+                   &gzaddr, rounded_size + (1 * PAGE_SIZE),
+                   0, KMA_KOBJECT | KMA_ATOMIC | gzalloc_guard,
+                   VM_KERN_MEMORY_OSFMK);
+               if (kr != KERN_SUCCESS) {
+                       panic("gzalloc: kernel_memory_allocate for size 0x%llx failed with %d",
+                           (uint64_t)rounded_size, kr);

                }
                }

+               new_va = true;
+       }

 
 

-               if (gzalloc_uf_mode) {
-                       gzaddr += PAGE_SIZE;
-                       /* The "header" becomes a "footer" in underflow
-                        * mode.
-                        */
-                       gzh = (gzhdr_t *) (gzaddr + zone->elem_size);
-                       addr = gzaddr;
-                       gzhcopy = (gzhdr_t *) (gzaddr + rounded_size - sizeof(gzhdr_t));
-               } else {
-                       gzh = (gzhdr_t *) (gzaddr + residue - GZHEADER_SIZE);
-                       addr = (gzaddr + residue);
-               }
+       if (gzalloc_uf_mode) {
+               gzaddr += PAGE_SIZE;
+               /* The "header" becomes a "footer" in underflow
+                * mode.
+                */
+               gzh = (gzhdr_t *) (gzaddr + zone_elem_size(zone));
+               addr = gzaddr;
+               gzhcopy = (gzhdr_t *) (gzaddr + rounded_size - sizeof(gzhdr_t));
+       } else {
+               gzh = (gzhdr_t *) (gzaddr + residue - GZHEADER_SIZE);
+               addr = (gzaddr + residue);
+       }

 
 

+       if (zone->z_free_zeroes) {
+               bzero((void *)gzaddr, rounded_size);
+       } else {

                /* Fill with a pattern on allocation to trap uninitialized
                 * data use. Since the element size may be "rounded up"
                 * by higher layers such as the kalloc layer, this may
                /* Fill with a pattern on allocation to trap uninitialized
                 * data use. Since the element size may be "rounded up"
                 * by higher layers such as the kalloc layer, this may


@@ -394,202 +412,236 @@ gzalloc_alloc(zone_t zone, boolean_t canblock) {
                 * prefixed to the allocation.
                 */
                memset((void *)gzaddr, gzalloc_fill_pattern, rounded_size);
                 * prefixed to the allocation.
                 */
                memset((void *)gzaddr, gzalloc_fill_pattern, rounded_size);

+       }

 
 

-               gzh->gzone = (kmem_ready && vm_page_zone) ? zone : GZDEADZONE;
-               gzh->gzsize = (uint32_t) zone->elem_size;
-               gzh->gzsig = GZALLOC_SIGNATURE;
-
-               /* In underflow detection mode, stash away a copy of the
-                * metadata at the edge of the allocated range, for
-                * retrieval by gzalloc_element_size()
-                */
-               if (gzhcopy) {
-                       *gzhcopy = *gzh;
-               }
+       gzh->gzone = (kmem_ready && vm_page_zone) ? zone : GZDEADZONE;
+       gzh->gzsize = (uint32_t)zone_elem_size(zone);
+       gzh->gzsig = GZALLOC_SIGNATURE;

 
 

-               lock_zone(zone);
-               assert(zone->zone_valid);
-               zone->count++;
-               zone->sum_count++;
-               zone->cur_size += rounded_size;
-               unlock_zone(zone);
+       /* In underflow detection mode, stash away a copy of the
+        * metadata at the edge of the allocated range, for
+        * retrieval by gzalloc_element_size()
+        */
+       if (gzhcopy) {
+               *gzhcopy = *gzh;
+       }

 
 

-               OSAddAtomic64((SInt32) rounded_size, &gzalloc_allocated);
-               OSAddAtomic64((SInt32) (rounded_size - zone->elem_size), &gzalloc_wasted);
+       zone_lock(zone);
+       assert(zone->z_self == zone);
+       zone->z_elems_free--;
+       if (new_va) {
+               zone->z_va_cur += 1;

        }
        }

+       zone->z_wired_cur += 1;
+       zpercpu_get(zstats)->zs_mem_allocated += rounded_size;
+       zone_unlock(zone);
+
+       OSAddAtomic64((SInt32) rounded_size, &gzalloc_allocated);
+       OSAddAtomic64((SInt32) (rounded_size - zone_elem_size(zone)), &gzalloc_wasted);
+

        return addr;
 }
 
        return addr;
 }
 

-boolean_t gzalloc_free(zone_t zone, void *addr) {
-       boolean_t gzfreed = FALSE;
+void
+gzalloc_free(zone_t zone, zone_stats_t zstats, void *addr)
+{

        kern_return_t kr;
 
        kern_return_t kr;
 

-       if (__improbable(gzalloc_tracked(zone))) {
-               gzhdr_t *gzh;
-               vm_offset_t rounded_size = round_page(zone->elem_size + GZHEADER_SIZE);
-               vm_offset_t residue = rounded_size - zone->elem_size;
-               vm_offset_t saddr;
-               vm_offset_t free_addr = 0;
+       assert(zone->gzalloc_tracked); // the caller is responsible for checking

 
 

-               if (gzalloc_uf_mode) {
-                       gzh = (gzhdr_t *)((vm_offset_t)addr + zone->elem_size);
-                       saddr = (vm_offset_t) addr - PAGE_SIZE;
-               } else {
-                       gzh = (gzhdr_t *)((vm_offset_t)addr - GZHEADER_SIZE);
-                       saddr = ((vm_offset_t)addr) - residue;
-               }
+       gzhdr_t *gzh;
+       vm_offset_t rounded_size = round_page(zone_elem_size(zone) + GZHEADER_SIZE);
+       vm_offset_t residue = rounded_size - zone_elem_size(zone);
+       vm_offset_t saddr;
+       vm_offset_t free_addr = 0;

 
 

-               if ((saddr & PAGE_MASK) != 0) {
-                       panic("gzalloc_free: invalid address supplied: %p (adjusted: 0x%lx) for zone with element sized 0x%lx\n", addr, saddr, zone->elem_size);
-               }
+       if (gzalloc_uf_mode) {
+               gzh = (gzhdr_t *)((vm_offset_t)addr + zone_elem_size(zone));
+               saddr = (vm_offset_t) addr - PAGE_SIZE;
+       } else {
+               gzh = (gzhdr_t *)((vm_offset_t)addr - GZHEADER_SIZE);
+               saddr = ((vm_offset_t)addr) - residue;
+       }

 
 

-               if (gzfc_size) {
-                       if (gzalloc_dfree_check) {
-                               uint32_t gd;
+       if ((saddr & PAGE_MASK) != 0) {
+               panic("%s: invalid address supplied: "
+                   "%p (adjusted: 0x%lx) for zone with element sized 0x%lx\n",
+                   __func__, addr, saddr, zone_elem_size(zone));
+       }

 
 

-                               lock_zone(zone);
-                               assert(zone->zone_valid);
-                               for (gd = 0; gd < gzfc_size; gd++) {
-                                       if (zone->gz.gzfc[gd] == saddr) {
-                                               panic("gzalloc: double free detected, freed address: 0x%lx, current free cache index: %d, freed index: %d", saddr, zone->gz.gzfc_index, gd);
-                                       }
-                               }
-                               unlock_zone(zone);
+       if (gzfc_size && gzalloc_dfree_check) {
+               zone_lock(zone);
+               assert(zone->z_self == zone);
+               for (uint32_t gd = 0; gd < gzfc_size; gd++) {
+                       if (zone->gz.gzfc[gd] != saddr) {
+                               continue;

                        }
                        }

+                       panic("%s: double free detected, freed address: 0x%lx, "
+                           "current free cache index: %d, freed index: %d",
+                           __func__, saddr, zone->gz.gzfc_index, gd);

                }
                }

+               zone_unlock(zone);
+       }

 
 

-               if (gzalloc_consistency_checks) {
-                       if (gzh->gzsig != GZALLOC_SIGNATURE) {
-                               panic("GZALLOC signature mismatch for element %p, expected 0x%x, found 0x%x", addr, GZALLOC_SIGNATURE, gzh->gzsig);
-                       }
-
-                       if (gzh->gzone != zone && (gzh->gzone != GZDEADZONE))
-                               panic("%s: Mismatched zone or under/overflow, current zone: %p, recorded zone: %p, address: %p", __FUNCTION__, zone, gzh->gzone, (void *)addr);
-                       /* Partially redundant given the zone check, but may flag header corruption */
-                       if (gzh->gzsize != zone->elem_size) {
-                               panic("Mismatched zfree or under/overflow for zone %p, recorded size: 0x%x, element size: 0x%x, address: %p\n", zone, gzh->gzsize, (uint32_t) zone->elem_size, (void *)addr);
-                       }
-
-                       char *gzc, *checkstart, *checkend;
-                       if (gzalloc_uf_mode) {
-                               checkstart = (char *) ((uintptr_t) gzh + sizeof(gzh));
-                               checkend = (char *) ((((vm_offset_t)addr) & ~PAGE_MASK) + PAGE_SIZE);
-                       } else {
-                               checkstart = (char *) trunc_page_64(addr);
-                               checkend = (char *)gzh;
-                       }
-
-                       for (gzc = checkstart; gzc < checkend; gzc++) {
-                               if (*gzc != gzalloc_fill_pattern) {
-                                       panic("GZALLOC: detected over/underflow, byte at %p, element %p, contents 0x%x from 0x%lx byte sized zone (%s) doesn't match fill pattern (%c)", gzc, addr, *gzc, zone->elem_size, zone->zone_name, gzalloc_fill_pattern);
-                               }
-                       }
+       if (gzalloc_consistency_checks) {
+               if (gzh->gzsig != GZALLOC_SIGNATURE) {
+                       panic("GZALLOC signature mismatch for element %p, "
+                           "expected 0x%x, found 0x%x",
+                           addr, GZALLOC_SIGNATURE, gzh->gzsig);

                }
 
                }
 

-               if (!kmem_ready || gzh->gzone == GZDEADZONE) {
-                       /* For now, just leak frees of early allocations
-                        * performed before kmem is fully configured.
-                        * They don't seem to get freed currently;
-                        * consider ml_static_mfree in the future.
-                        */
-                       OSAddAtomic64((SInt32) (rounded_size), &gzalloc_early_free);
-                       return TRUE;
+               if (gzh->gzone != zone && (gzh->gzone != GZDEADZONE)) {
+                       panic("%s: Mismatched zone or under/overflow, "
+                           "current zone: %p, recorded zone: %p, address: %p",
+                           __func__, zone, gzh->gzone, (void *)addr);

                }
                }

-
-               if (get_preemption_level() != 0) {
-                               pdzfree_count++;
+               /* Partially redundant given the zone check, but may flag header corruption */
+               if (gzh->gzsize != zone_elem_size(zone)) {
+                       panic("Mismatched zfree or under/overflow for zone %p, "
+                           "recorded size: 0x%x, element size: 0x%x, address: %p",
+                           zone, gzh->gzsize, (uint32_t)zone_elem_size(zone), (void *)addr);

                }
 
                }
 

-               if (gzfc_size) {
-                       /* Either write protect or unmap the newly freed
-                        * allocation
-                        */
-                       kr = vm_map_protect(
-                               gzalloc_map,
-                               saddr,
-                               saddr + rounded_size + (1 * PAGE_SIZE),
-                               gzalloc_prot,
-                               FALSE);
-                       if (kr != KERN_SUCCESS)
-                               panic("%s: vm_map_protect: %p, 0x%x", __FUNCTION__, (void *)saddr, kr);
+               char *gzc, *checkstart, *checkend;
+               if (gzalloc_uf_mode) {
+                       checkstart = (char *) ((uintptr_t) gzh + sizeof(gzh));
+                       checkend = (char *) ((((vm_offset_t)addr) & ~PAGE_MASK) + PAGE_SIZE);

                } else {
                } else {

-                       free_addr = saddr;
+                       checkstart = (char *) trunc_page_64(addr);
+                       checkend = (char *)gzh;

                }
 
                }
 

-               lock_zone(zone);
-               assert(zone->zone_valid);
-
-               /* Insert newly freed element into the protected free element
-                * cache, and rotate out the LRU element.
-                */
-               if (gzfc_size) {
-                       if (zone->gz.gzfc_index >= gzfc_size) {
-                               zone->gz.gzfc_index = 0;
+               for (gzc = checkstart; gzc < checkend; gzc++) {
+                       if (*gzc == gzalloc_fill_pattern) {
+                               continue;

                        }
                        }

-                       free_addr = zone->gz.gzfc[zone->gz.gzfc_index];
-                       zone->gz.gzfc[zone->gz.gzfc_index++] = saddr;
+                       panic("%s: detected over/underflow, byte at %p, element %p, "
+                           "contents 0x%x from 0x%lx byte sized zone (%s%s) "
+                           "doesn't match fill pattern (%c)",
+                           __func__, gzc, addr, *gzc, zone_elem_size(zone),
+                           zone_heap_name(zone), zone->z_name, gzalloc_fill_pattern);

                }
                }

+       }
+
+       if ((startup_phase < STARTUP_SUB_KMEM) || gzh->gzone == GZDEADZONE) {
+               /* For now, just leak frees of early allocations
+                * performed before kmem is fully configured.
+                * They don't seem to get freed currently;
+                * consider ml_static_mfree in the future.
+                */
+               OSAddAtomic64((SInt32) (rounded_size), &gzalloc_early_free);
+               return;
+       }

 
 

-               if (free_addr) {
-                       zone->count--;
-                       zone->cur_size -= rounded_size;
+       if (get_preemption_level() != 0) {
+               pdzfree_count++;
+       }
+
+       if (gzfc_size) {
+               /* Either write protect or unmap the newly freed
+                * allocation
+                */
+               kr = vm_map_protect(gzalloc_map, saddr,
+                   saddr + rounded_size + (1 * PAGE_SIZE),
+                   gzalloc_prot, FALSE);
+               if (kr != KERN_SUCCESS) {
+                       panic("%s: vm_map_protect: %p, 0x%x", __func__, (void *)saddr, kr);

                }
                }

+       } else {
+               free_addr = saddr;
+       }

 
 

-               unlock_zone(zone);
-
-               if (free_addr) {
-                       // TODO: consider using physical reads to check for
-                       // corruption while on the protected freelist
-                       // (i.e. physical corruption)
-                       kr = vm_map_remove(
-                               gzalloc_map,
-                               free_addr,
-                               free_addr + rounded_size + (1 * PAGE_SIZE),
-                               VM_MAP_REMOVE_KUNWIRE);
-                       if (kr != KERN_SUCCESS)
-                               panic("gzfree: vm_map_remove: %p, 0x%x", (void *)free_addr, kr);
-                       // TODO: sysctl-ize for quick reference
-                       OSAddAtomic64((SInt32)rounded_size, &gzalloc_freed);
-                       OSAddAtomic64(-((SInt32) (rounded_size - zone->elem_size)), &gzalloc_wasted);
+       zone_lock(zone);
+       assert(zone->z_self == zone);
+
+       /* Insert newly freed element into the protected free element
+        * cache, and rotate out the LRU element.
+        */
+       if (gzfc_size) {
+               if (zone->gz.gzfc_index >= gzfc_size) {
+                       zone->gz.gzfc_index = 0;

                }
                }

+               free_addr = zone->gz.gzfc[zone->gz.gzfc_index];
+               zone->gz.gzfc[zone->gz.gzfc_index++] = saddr;
+       }
+
+       if (free_addr) {
+               zone->z_elems_free++;
+               zone->z_wired_cur -= 1;
+       }

 
 

-               gzfreed = TRUE;
+       zpercpu_get(zstats)->zs_mem_freed += rounded_size;
+       zone_unlock(zone);
+
+       if (free_addr) {
+               // TODO: consider using physical reads to check for
+               // corruption while on the protected freelist
+               // (i.e. physical corruption)
+               kr = vm_map_remove(gzalloc_map, free_addr,
+                   free_addr + rounded_size + (1 * PAGE_SIZE),
+                   VM_MAP_REMOVE_KUNWIRE);
+               if (kr != KERN_SUCCESS) {
+                       panic("gzfree: vm_map_remove: %p, 0x%x", (void *)free_addr, kr);
+               }
+               // TODO: sysctl-ize for quick reference
+               OSAddAtomic64((SInt32)rounded_size, &gzalloc_freed);
+               OSAddAtomic64(-((SInt32) (rounded_size - zone_elem_size(zone))),
+                   &gzalloc_wasted);

        }
        }

-       return gzfreed;

 }
 
 }
 

-boolean_t gzalloc_element_size(void *gzaddr, zone_t *z, vm_size_t *gzsz) {
+boolean_t
+gzalloc_element_size(void *gzaddr, zone_t *z, vm_size_t *gzsz)
+{

        uintptr_t a = (uintptr_t)gzaddr;
        if (__improbable(gzalloc_mode && (a >= gzalloc_map_min) && (a < gzalloc_map_max))) {
                gzhdr_t *gzh;
        uintptr_t a = (uintptr_t)gzaddr;
        if (__improbable(gzalloc_mode && (a >= gzalloc_map_min) && (a < gzalloc_map_max))) {
                gzhdr_t *gzh;

+               boolean_t       vmef;
+               vm_map_entry_t  gzvme = NULL;
+               vm_map_lock_read(gzalloc_map);
+               vmef = vm_map_lookup_entry(gzalloc_map, (vm_map_offset_t)a, &gzvme);
+               vm_map_unlock(gzalloc_map);
+               if (vmef == FALSE) {
+                       panic("GZALLOC: unable to locate map entry for %p\n", (void *)a);
+               }
+               assertf(gzvme->vme_atomic != 0, "GZALLOC: VM map entry inconsistency, "
+                   "vme: %p, start: %llu end: %llu", gzvme, gzvme->vme_start, gzvme->vme_end);

 
                /* Locate the gzalloc metadata adjoining the element */
                if (gzalloc_uf_mode == TRUE) {
 
                /* Locate the gzalloc metadata adjoining the element */
                if (gzalloc_uf_mode == TRUE) {

-                       boolean_t       vmef;
-                       vm_map_entry_t  gzvme = NULL;
-

                        /* In underflow detection mode, locate the map entry describing
                         * the element, and then locate the copy of the gzalloc
                         * header at the trailing edge of the range.
                         */
                        /* In underflow detection mode, locate the map entry describing
                         * the element, and then locate the copy of the gzalloc
                         * header at the trailing edge of the range.
                         */

-                       vm_map_lock_read(gzalloc_map);
-                       vmef = vm_map_lookup_entry(gzalloc_map, (vm_map_offset_t)a, &gzvme);
-                       vm_map_unlock(gzalloc_map);
-                       if (vmef == FALSE) {
-                               panic("GZALLOC: unable to locate map entry for %p\n", (void *)a);
-                       }
-                       assertf(gzvme->vme_atomic != 0, "GZALLOC: VM map entry inconsistency, vme: %p, start: %llu end: %llu", gzvme, gzvme->vme_start, gzvme->vme_end);

                        gzh = (gzhdr_t *)(gzvme->vme_end - GZHEADER_SIZE);
                } else {
                        gzh = (gzhdr_t *)(gzvme->vme_end - GZHEADER_SIZE);
                } else {

-                       gzh = (gzhdr_t *)(a - GZHEADER_SIZE);
+                       /* In overflow detection mode, scan forward from
+                        * the base of the map entry to locate the
+                        * gzalloc header.
+                        */
+                       uint32_t *p = (uint32_t*) gzvme->vme_start;
+                       while (p < (uint32_t *) gzvme->vme_end) {
+                               if (*p == GZALLOC_SIGNATURE) {
+                                       break;
+                               } else {
+                                       p++;
+                               }
+                       }
+                       if (p >= (uint32_t *) gzvme->vme_end) {
+                               panic("GZALLOC signature missing addr %p, zone %p", gzaddr, z);
+                       }
+                       p++;
+                       uintptr_t q = (uintptr_t) p;
+                       gzh = (gzhdr_t *) (q - sizeof(gzhdr_t));

                }
 
                if (gzh->gzsig != GZALLOC_SIGNATURE) {
                }
 
                if (gzh->gzsig != GZALLOC_SIGNATURE) {

-                       panic("GZALLOC signature mismatch for element %p, expected 0x%x, found 0x%x", (void *)a, GZALLOC_SIGNATURE, gzh->gzsig);
+                       panic("GZALLOC signature mismatch for element %p, expected 0x%x, found 0x%x",
+                           (void *)a, GZALLOC_SIGNATURE, gzh->gzsig);

                }
 
                }
 

-               *gzsz = gzh->gzone->elem_size;
-               if (__improbable((gzalloc_tracked(gzh->gzone)) == FALSE)) {
+               *gzsz = zone_elem_size(gzh->gzone);
+               if (__improbable(!gzh->gzone->gzalloc_tracked)) {

                        panic("GZALLOC: zone mismatch (%p)\n", gzh->gzone);
                }
 
                        panic("GZALLOC: zone mismatch (%p)\n", gzh->gzone);
                }