* -gzalloc_noconsistency: disable consistency checks that flag mismatched
* frees, corruptions of the header/trailer signatures etc.
* -nogzalloc_mode: Disables the guard mode allocator. The DEBUG kernel
- * enables the guard allocator for zones sized 8K-16K (if present) by
+ * enables the guard allocator for zones sized 1K (if present) by
* default, this option can disable that behaviour.
+ * gzname=<name> target a zone by name. Can be coupled with size-based
+ * targeting. Naming conventions match those of the zlog boot-arg, i.e.
+ * "a period in the logname will match a space in the zone name"
+ * -gzalloc_no_dfree_check Eliminate double free checks
+ * gzalloc_zscale=<value> specify size multiplier for the dedicated gzalloc submap
*/
#include <zone_debug.h>
#define GZDEADZONE ((zone_t) 0xDEAD201E)
#define GZALLOC_SIGNATURE (0xABADCAFE)
#define GZALLOC_RESERVE_SIZE_DEFAULT (2 * 1024 * 1024)
-#define GZFC_DEFAULT_SIZE (1024)
+#define GZFC_DEFAULT_SIZE (1536)
char gzalloc_fill_pattern = 0x67; /* 'g' */
uint32_t gzalloc_max = 0;
uint32_t gzalloc_size = 0;
uint64_t gzalloc_allocated, gzalloc_freed, gzalloc_early_alloc, gzalloc_early_free, gzalloc_wasted;
-boolean_t gzalloc_uf_mode = FALSE, gzalloc_consistency_checks = TRUE;
+boolean_t gzalloc_uf_mode = FALSE, gzalloc_consistency_checks = TRUE, gzalloc_dfree_check = TRUE;
vm_prot_t gzalloc_prot = VM_PROT_NONE;
uint32_t gzalloc_guard = KMA_GUARD_LAST;
uint32_t gzfc_size = GZFC_DEFAULT_SIZE;
+uint32_t gzalloc_zonemap_scale = 6;
vm_map_t gzalloc_map;
vm_offset_t gzalloc_map_min, gzalloc_map_max;
extern zone_t vm_page_zone;
+static zone_t gztrackzone = NULL;
+static char gznamedzone[MAX_ZONE_NAME] = "";
+
void gzalloc_reconfigure(__unused zone_t z) {
/* Nothing for now */
}
return gzalloc_mode;
}
+static inline boolean_t gzalloc_tracked(zone_t z) {
+ return (gzalloc_mode &&
+ (((z->elem_size >= gzalloc_min) && (z->elem_size <= gzalloc_max)) || (z == gztrackzone)) &&
+ (z->gzalloc_exempt == 0));
+}
+
void gzalloc_zone_init(zone_t z) {
if (gzalloc_mode) {
bzero(&z->gz, sizeof(z->gz));
- if (gzfc_size && (z->elem_size >= gzalloc_min) && (z->elem_size <= gzalloc_max) && (z->gzalloc_exempt == FALSE)) {
+ if (track_this_zone(z->zone_name, gznamedzone)) {
+ gztrackzone = z;
+ }
+
+ if (gzfc_size &&
+ gzalloc_tracked(z)) {
vm_size_t gzfcsz = round_page(sizeof(*z->gz.gzfc) * gzfc_size);
/* If the VM/kmem system aren't yet configured, carve
}
}
+/* Called by zdestroy() to dump the free cache elements so the zone count can drop to zero. */
+void gzalloc_empty_free_cache(zone_t zone) {
+ if (__improbable(gzalloc_tracked(zone))) {
+ kern_return_t kr;
+ int freed_elements = 0;
+ vm_offset_t free_addr = 0;
+ vm_offset_t rounded_size = round_page(zone->elem_size + GZHEADER_SIZE);
+ vm_offset_t gzfcsz = round_page(sizeof(*zone->gz.gzfc) * gzfc_size);
+ vm_offset_t gzfc_copy;
+
+ kr = kmem_alloc(kernel_map, &gzfc_copy, gzfcsz, VM_KERN_MEMORY_OSFMK);
+ if (kr != KERN_SUCCESS) {
+ panic("gzalloc_empty_free_cache: kmem_alloc: 0x%x", kr);
+ }
+
+ /* Reset gzalloc_data. */
+ lock_zone(zone);
+ memcpy((void *)gzfc_copy, (void *)zone->gz.gzfc, gzfcsz);
+ bzero((void *)zone->gz.gzfc, gzfcsz);
+ zone->gz.gzfc_index = 0;
+ unlock_zone(zone);
+
+ /* Free up all the cached elements. */
+ for (uint32_t index = 0; index < gzfc_size; index++) {
+ free_addr = ((vm_offset_t *)gzfc_copy)[index];
+ if (free_addr && free_addr >= gzalloc_map_min && free_addr < gzalloc_map_max) {
+ kr = vm_map_remove(
+ gzalloc_map,
+ free_addr,
+ free_addr + rounded_size + (1 * PAGE_SIZE),
+ VM_MAP_REMOVE_KUNWIRE);
+ if (kr != KERN_SUCCESS) {
+ panic("gzalloc_empty_free_cache: vm_map_remove: %p, 0x%x", (void *)free_addr, kr);
+ }
+ OSAddAtomic64((SInt32)rounded_size, &gzalloc_freed);
+ OSAddAtomic64(-((SInt32) (rounded_size - zone->elem_size)), &gzalloc_wasted);
+
+ freed_elements++;
+ }
+ }
+ /*
+ * TODO: Consider freeing up zone->gz.gzfc as well if it didn't come from the gzalloc_reserve pool.
+ * For now we're reusing this buffer across zdestroy's. We would have to allocate it again on a
+ * subsequent zinit() as well.
+ */
+
+ /* Decrement zone counters. */
+ lock_zone(zone);
+ zone->count -= freed_elements;
+ zone->cur_size -= (freed_elements * rounded_size);
+ unlock_zone(zone);
+
+ kmem_free(kernel_map, gzfc_copy, gzfcsz);
+ }
+}
+
void gzalloc_configure(void) {
char temp_buf[16];
gzalloc_guard = KMA_GUARD_FIRST;
}
+ if (PE_parse_boot_argn("-gzalloc_no_dfree_check", temp_buf, sizeof(temp_buf))) {
+ gzalloc_dfree_check = FALSE;
+ }
+
+ (void) PE_parse_boot_argn("gzalloc_zscale", &gzalloc_zonemap_scale, sizeof(gzalloc_zonemap_scale));
+
if (PE_parse_boot_argn("-gzalloc_noconsistency", temp_buf, sizeof (temp_buf))) {
gzalloc_consistency_checks = FALSE;
}
-#if DEBUG
+
+ if (PE_parse_boot_argn("gzname", gznamedzone, sizeof(gznamedzone))) {
+ gzalloc_mode = TRUE;
+ }
+#if DEBUG
if (gzalloc_mode == FALSE) {
- gzalloc_min = 8192;
- gzalloc_max = 16384;
+ gzalloc_min = 1024;
+ gzalloc_max = 1024;
+ strlcpy(gznamedzone, "pmap", sizeof(gznamedzone));
gzalloc_prot = VM_PROT_READ;
gzalloc_mode = TRUE;
}
kern_return_t retval;
if (gzalloc_mode) {
- retval = kmem_suballoc(kernel_map, &gzalloc_map_min, (max_zonemap_size << 2),
- FALSE, VM_FLAGS_ANYWHERE | VM_FLAGS_PERMANENT | VM_MAKE_TAG(VM_KERN_MEMORY_ZONE),
- &gzalloc_map);
+ vm_map_kernel_flags_t vmk_flags;
+
+ vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
+ vmk_flags.vmkf_permanent = TRUE;
+ retval = kmem_suballoc(kernel_map, &gzalloc_map_min, (max_zonemap_size * gzalloc_zonemap_scale),
+ FALSE, VM_FLAGS_ANYWHERE, vmk_flags, VM_KERN_MEMORY_ZONE,
+ &gzalloc_map);
- if (retval != KERN_SUCCESS)
- panic("zone_init: kmem_suballoc(gzalloc) failed");
- gzalloc_map_max = gzalloc_map_min + (max_zonemap_size << 2);
+ if (retval != KERN_SUCCESS) {
+ panic("zone_init: kmem_suballoc(gzalloc_map, 0x%lx, %u) failed", max_zonemap_size, gzalloc_zonemap_scale);
+ }
+ gzalloc_map_max = gzalloc_map_min + (max_zonemap_size * gzalloc_zonemap_scale);
}
}
gzalloc_alloc(zone_t zone, boolean_t canblock) {
vm_offset_t addr = 0;
- if (__improbable(gzalloc_mode &&
- (((zone->elem_size >= gzalloc_min) &&
- (zone->elem_size <= gzalloc_max))) &&
- (zone->gzalloc_exempt == 0))) {
+ if (__improbable(gzalloc_tracked(zone))) {
if (get_preemption_level() != 0) {
if (canblock == TRUE) {
/* Early allocations are supplied directly from the
* reserve.
*/
- if (gzalloc_reserve_size < rounded_size)
+ if (gzalloc_reserve_size < (rounded_size + PAGE_SIZE))
panic("gzalloc reserve exhausted");
gzaddr = gzalloc_reserve;
/* No guard page for these early allocations, just
}
lock_zone(zone);
+ assert(zone->zone_valid);
zone->count++;
zone->sum_count++;
zone->cur_size += rounded_size;
boolean_t gzfreed = FALSE;
kern_return_t kr;
- if (__improbable(gzalloc_mode &&
- (((zone->elem_size >= gzalloc_min) &&
- (zone->elem_size <= gzalloc_max))) &&
- (zone->gzalloc_exempt == 0))) {
+ if (__improbable(gzalloc_tracked(zone))) {
gzhdr_t *gzh;
vm_offset_t rounded_size = round_page(zone->elem_size + GZHEADER_SIZE);
vm_offset_t residue = rounded_size - zone->elem_size;
saddr = ((vm_offset_t)addr) - residue;
}
- assert((saddr & PAGE_MASK) == 0);
+ if ((saddr & PAGE_MASK) != 0) {
+ panic("gzalloc_free: invalid address supplied: %p (adjusted: 0x%lx) for zone with element sized 0x%lx\n", addr, saddr, zone->elem_size);
+ }
+
+ if (gzfc_size) {
+ if (gzalloc_dfree_check) {
+ uint32_t gd;
+
+ lock_zone(zone);
+ assert(zone->zone_valid);
+ for (gd = 0; gd < gzfc_size; gd++) {
+ if (zone->gz.gzfc[gd] == saddr) {
+ panic("gzalloc: double free detected, freed address: 0x%lx, current free cache index: %d, freed index: %d", saddr, zone->gz.gzfc_index, gd);
+ }
+ }
+ unlock_zone(zone);
+ }
+ }
if (gzalloc_consistency_checks) {
if (gzh->gzsig != GZALLOC_SIGNATURE) {
if (gzh->gzsize != zone->elem_size) {
panic("Mismatched zfree or under/overflow for zone %p, recorded size: 0x%x, element size: 0x%x, address: %p\n", zone, gzh->gzsize, (uint32_t) zone->elem_size, (void *)addr);
}
+
+ char *gzc, *checkstart, *checkend;
+ if (gzalloc_uf_mode) {
+ checkstart = (char *) ((uintptr_t) gzh + sizeof(gzh));
+ checkend = (char *) ((((vm_offset_t)addr) & ~PAGE_MASK) + PAGE_SIZE);
+ } else {
+ checkstart = (char *) trunc_page_64(addr);
+ checkend = (char *)gzh;
+ }
+
+ for (gzc = checkstart; gzc < checkend; gzc++) {
+ if (*gzc != gzalloc_fill_pattern) {
+ panic("GZALLOC: detected over/underflow, byte at %p, element %p, contents 0x%x from 0x%lx byte sized zone (%s) doesn't match fill pattern (%c)", gzc, addr, *gzc, zone->elem_size, zone->zone_name, gzalloc_fill_pattern);
+ }
+ }
}
if (!kmem_ready || gzh->gzone == GZDEADZONE) {
}
lock_zone(zone);
+ assert(zone->zone_valid);
/* Insert newly freed element into the protected free element
* cache, and rotate out the LRU element.
unlock_zone(zone);
if (free_addr) {
+ // TODO: consider using physical reads to check for
+ // corruption while on the protected freelist
+ // (i.e. physical corruption)
kr = vm_map_remove(
gzalloc_map,
free_addr,
VM_MAP_REMOVE_KUNWIRE);
if (kr != KERN_SUCCESS)
panic("gzfree: vm_map_remove: %p, 0x%x", (void *)free_addr, kr);
-
+ // TODO: sysctl-ize for quick reference
OSAddAtomic64((SInt32)rounded_size, &gzalloc_freed);
OSAddAtomic64(-((SInt32) (rounded_size - zone->elem_size)), &gzalloc_wasted);
}
boolean_t gzalloc_element_size(void *gzaddr, zone_t *z, vm_size_t *gzsz) {
uintptr_t a = (uintptr_t)gzaddr;
- if (__improbable(gzalloc_mode && (a >= gzalloc_map_min) && (a <= gzalloc_map_max))) {
+ if (__improbable(gzalloc_mode && (a >= gzalloc_map_min) && (a < gzalloc_map_max))) {
gzhdr_t *gzh;
/* Locate the gzalloc metadata adjoining the element */
}
*gzsz = gzh->gzone->elem_size;
- if ((*gzsz < gzalloc_min) || (*gzsz > gzalloc_max)) {
- panic("GZALLOC: invalid element size %lu\n", *gzsz);
+ if (__improbable((gzalloc_tracked(gzh->gzone)) == FALSE)) {
+ panic("GZALLOC: zone mismatch (%p)\n", gzh->gzone);
}
if (z) {