[apple/xnu.git] / osfmk / kern / gzalloc.c

/*
 * Copyright (c) 2000-2012 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
/*
 *	File:	kern/gzalloc.c
 *	Author:	Derek Kumar
 *
 *	"Guard mode" zone allocator, used to trap use-after-free errors,
 *	overruns, underruns, mismatched allocations/frees, uninitialized
 *	zone element use, timing dependent races etc.
 *
 *	The allocator is configured by these boot-args:
 *	gzalloc_size=<size>: target all zones with elements of <size> bytes
 *	gzalloc_min=<size>: target zones with elements >= size
 *	gzalloc_max=<size>: target zones with elements <= size
 * 	gzalloc_min/max can be specified in conjunction to target a range of
 *	sizes
 *	gzalloc_fc_size=<size>: number of zone elements (effectively page
 *	multiple sized) to retain in the free VA cache. This cache is evicted
 *	(backing pages and VA released) in a least-recently-freed fashion.
 *	Larger free VA caches allow for a longer window of opportunity to trap
 *	delayed use-after-free operations, but use more memory.
 *	-gzalloc_wp: Write protect, rather than unmap, freed allocations
 *	lingering in the free VA cache. Useful to disambiguate between
 *	read-after-frees/read overruns and writes. Also permits direct inspection
 *	of the freed element in the cache via the kernel debugger. As each
 *	element has a "header" (trailer in underflow detection mode), the zone
 *	of origin of the element can be easily determined in this mode.
 *	-gzalloc_uf_mode: Underflow detection mode, where the guard page
 *	adjoining each element is placed *before* the element page rather than
 *	after. The element is also located at the top of the page, rather than
 *	abutting the bottom as with the standard overflow detection mode.
 *	-gzalloc_noconsistency: disable consistency checks that flag mismatched
 *	frees, corruptions of the header/trailer signatures etc.
 *	-nogzalloc_mode: Disables the guard mode allocator. The DEBUG kernel
 *	enables the guard allocator for zones sized 8K-16K (if present) by
 *	default, this option can disable that behaviour.
 */

#include <zone_debug.h>
#include <zone_alias_addr.h>

#include <mach/mach_types.h>
#include <mach/vm_param.h>
#include <mach/kern_return.h>
#include <mach/machine/vm_types.h>
#include <mach_debug/zone_info.h>
#include <mach/vm_map.h>

#include <kern/kern_types.h>
#include <kern/assert.h>
#include <kern/sched.h>
#include <kern/locks.h>
#include <kern/misc_protos.h>
#include <kern/zalloc.h>
#include <kern/kalloc.h>

#include <vm/pmap.h>
#include <vm/vm_map.h>
#include <vm/vm_kern.h>
#include <vm/vm_page.h>

#include <pexpert/pexpert.h>

#include <machine/machparam.h>

#include <libkern/OSDebug.h>
#include <libkern/OSAtomic.h>
#include <sys/kdebug.h>

extern boolean_t vm_kernel_ready, kmem_ready;
boolean_t gzalloc_mode = FALSE;
uint32_t pdzalloc_count, pdzfree_count;

#define	GZALLOC_MIN_DEFAULT (1024)
#define GZDEADZONE ((zone_t) 0xDEAD201E)
#define GZALLOC_SIGNATURE (0xABADCAFE)
#define GZALLOC_RESERVE_SIZE_DEFAULT (2 * 1024 * 1024)
#define GZFC_DEFAULT_SIZE (1024)

char gzalloc_fill_pattern = 0x67; /* 'g' */

uint32_t gzalloc_min = ~0U;
uint32_t gzalloc_max = 0;
uint32_t gzalloc_size = 0;
uint64_t gzalloc_allocated, gzalloc_freed, gzalloc_early_alloc, gzalloc_early_free, gzalloc_wasted;
boolean_t gzalloc_uf_mode = FALSE, gzalloc_consistency_checks = TRUE;
vm_prot_t gzalloc_prot = VM_PROT_NONE;
uint32_t gzalloc_guard = KMA_GUARD_LAST;
uint32_t gzfc_size = GZFC_DEFAULT_SIZE;

vm_map_t gzalloc_map;
vm_offset_t gzalloc_map_min, gzalloc_map_max;
vm_offset_t gzalloc_reserve;
vm_size_t gzalloc_reserve_size;

typedef struct gzalloc_header {
	zone_t gzone;
	uint32_t  gzsize;
	uint32_t  gzsig;
} gzhdr_t;

#define GZHEADER_SIZE (sizeof(gzhdr_t))

extern zone_t vm_page_zone;

void gzalloc_reconfigure(__unused zone_t z) {
	/* Nothing for now */
}

boolean_t gzalloc_enabled(void) {
	return gzalloc_mode;
}

void gzalloc_zone_init(zone_t z) {
	if (gzalloc_mode) {
		bzero(&z->gz, sizeof(z->gz));

		if (gzfc_size && (z->elem_size >= gzalloc_min) && (z->elem_size <= gzalloc_max) && (z->gzalloc_exempt == FALSE)) {
			vm_size_t gzfcsz = round_page(sizeof(*z->gz.gzfc) * gzfc_size);

			/* If the VM/kmem system aren't yet configured, carve
			 * out the free element cache structure directly from the
			 * gzalloc_reserve supplied by the pmap layer.
			*/
			if (!kmem_ready) {
				if (gzalloc_reserve_size < gzfcsz)
					panic("gzalloc reserve exhausted");

				z->gz.gzfc = (vm_offset_t *)gzalloc_reserve;
				gzalloc_reserve += gzfcsz;
				gzalloc_reserve_size -= gzfcsz;
			} else {
				kern_return_t kr;

				if ((kr = kernel_memory_allocate(kernel_map, (vm_offset_t *)&z->gz.gzfc, gzfcsz, 0, KMA_KOBJECT, VM_KERN_MEMORY_OSFMK)) != KERN_SUCCESS) {
					panic("zinit/gzalloc: kernel_memory_allocate failed (%d) for 0x%lx bytes", kr, (unsigned long) gzfcsz);
				}
			}
			bzero((void *)z->gz.gzfc, gzfcsz);
		}
	}
}

void gzalloc_configure(void) {
	char temp_buf[16];

	if (PE_parse_boot_argn("-gzalloc_mode", temp_buf, sizeof (temp_buf))) {
		gzalloc_mode = TRUE;
		gzalloc_min = GZALLOC_MIN_DEFAULT;
#if	ZONE_DEBUG		
		gzalloc_min += (typeof(gzalloc_min))ZONE_DEBUG_OFFSET;
#endif
		gzalloc_max = ~0U;
	}

	if (PE_parse_boot_argn("gzalloc_min", &gzalloc_min, sizeof(gzalloc_min))) {
#if	ZONE_DEBUG		
		gzalloc_min += (typeof(gzalloc_min))ZONE_DEBUG_OFFSET;
#endif
		gzalloc_mode = TRUE;
		gzalloc_max = ~0U;
	}

	if (PE_parse_boot_argn("gzalloc_max", &gzalloc_max, sizeof(gzalloc_max))) {
#if	ZONE_DEBUG		
		gzalloc_max += (typeof(gzalloc_min))ZONE_DEBUG_OFFSET;
#endif
		gzalloc_mode = TRUE;
		if (gzalloc_min == ~0U)
			gzalloc_min = 0;
	}

	if (PE_parse_boot_argn("gzalloc_size", &gzalloc_size, sizeof(gzalloc_size))) {
#if	ZONE_DEBUG		
		gzalloc_size += (typeof(gzalloc_min))ZONE_DEBUG_OFFSET;
#endif
		gzalloc_min = gzalloc_max = gzalloc_size;
		gzalloc_mode = TRUE;
	}

	(void)PE_parse_boot_argn("gzalloc_fc_size", &gzfc_size, sizeof(gzfc_size));

	if (PE_parse_boot_argn("-gzalloc_wp", temp_buf, sizeof (temp_buf))) {
		gzalloc_prot = VM_PROT_READ;
	}

	if (PE_parse_boot_argn("-gzalloc_uf_mode", temp_buf, sizeof (temp_buf))) {
		gzalloc_uf_mode = TRUE;
		gzalloc_guard = KMA_GUARD_FIRST;
	}

	if (PE_parse_boot_argn("-gzalloc_noconsistency", temp_buf, sizeof (temp_buf))) {
		gzalloc_consistency_checks = FALSE;
	}
#if	DEBUG
	if (gzalloc_mode == FALSE) {
		gzalloc_min = 8192;
		gzalloc_max = 16384;
		gzalloc_prot = VM_PROT_READ;
		gzalloc_mode = TRUE;
	}
#endif
	if (PE_parse_boot_argn("-nogzalloc_mode", temp_buf, sizeof (temp_buf)))
		gzalloc_mode = FALSE;

	if (gzalloc_mode) {
		gzalloc_reserve_size = GZALLOC_RESERVE_SIZE_DEFAULT;
		gzalloc_reserve = (vm_offset_t) pmap_steal_memory(gzalloc_reserve_size);
	}
}

void gzalloc_init(vm_size_t max_zonemap_size) {
	kern_return_t retval;

	if (gzalloc_mode) {
		retval = kmem_suballoc(kernel_map, &gzalloc_map_min, (max_zonemap_size << 2),
		    FALSE, VM_FLAGS_ANYWHERE | VM_FLAGS_PERMANENT | VM_MAKE_TAG(VM_KERN_MEMORY_ZONE),
		    &gzalloc_map);
	
		if (retval != KERN_SUCCESS)
			panic("zone_init: kmem_suballoc(gzalloc) failed");
		gzalloc_map_max = gzalloc_map_min + (max_zonemap_size << 2);
	}
}

vm_offset_t
gzalloc_alloc(zone_t zone, boolean_t canblock) {
	vm_offset_t addr = 0;

	if (__improbable(gzalloc_mode &&
		(((zone->elem_size >= gzalloc_min) &&
		    (zone->elem_size <= gzalloc_max))) &&
		(zone->gzalloc_exempt == 0))) {

		if (get_preemption_level() != 0) {
			if (canblock == TRUE) {
				pdzalloc_count++;
			}
			else
				return 0;
		}

		vm_offset_t rounded_size = round_page(zone->elem_size + GZHEADER_SIZE);
		vm_offset_t residue = rounded_size - zone->elem_size;
		vm_offset_t gzaddr = 0;
		gzhdr_t *gzh;

		if (!kmem_ready || (vm_page_zone == ZONE_NULL)) {
			/* Early allocations are supplied directly from the
			 * reserve.
			 */
			if (gzalloc_reserve_size < rounded_size)
				panic("gzalloc reserve exhausted");
			gzaddr = gzalloc_reserve;
			/* No guard page for these early allocations, just
			 * waste an additional page.
			 */
			gzalloc_reserve += rounded_size + PAGE_SIZE;
			gzalloc_reserve_size -= rounded_size + PAGE_SIZE;
			OSAddAtomic64((SInt32) (rounded_size), &gzalloc_early_alloc);
		}
		else {
			kern_return_t kr = kernel_memory_allocate(gzalloc_map,
			    &gzaddr, rounded_size + (1*PAGE_SIZE),
			    0, KMA_KOBJECT | gzalloc_guard,
			    VM_KERN_MEMORY_OSFMK);
			if (kr != KERN_SUCCESS)
				panic("gzalloc: kernel_memory_allocate for size 0x%llx failed with %d", (uint64_t)rounded_size, kr);

		}

		if (gzalloc_uf_mode) {
			gzaddr += PAGE_SIZE;
			/* The "header" becomes a "footer" in underflow
			 * mode.
			 */
			gzh = (gzhdr_t *) (gzaddr + zone->elem_size);
			addr = gzaddr;
		} else {
			gzh = (gzhdr_t *) (gzaddr + residue - GZHEADER_SIZE);
			addr = (gzaddr + residue);
		}

		/* Fill with a pattern on allocation to trap uninitialized
		 * data use. Since the element size may be "rounded up"
		 * by higher layers such as the kalloc layer, this may
		 * also identify overruns between the originally requested
		 * size and the rounded size via visual inspection.
		 * TBD: plumb through the originally requested size,
		 * prior to rounding by kalloc/IOMalloc etc.
		 * We also add a signature and the zone of origin in a header
		 * prefixed to the allocation.
		 */
		memset((void *)gzaddr, gzalloc_fill_pattern, rounded_size);

		gzh->gzone = (kmem_ready && vm_page_zone) ? zone : GZDEADZONE;
		gzh->gzsize = (uint32_t) zone->elem_size;
		gzh->gzsig = GZALLOC_SIGNATURE;

		lock_zone(zone);
		zone->count++;
		zone->sum_count++;
		zone->cur_size += rounded_size;
		unlock_zone(zone);

		OSAddAtomic64((SInt32) rounded_size, &gzalloc_allocated);
		OSAddAtomic64((SInt32) (rounded_size - zone->elem_size), &gzalloc_wasted);
	}
	return addr;
}

boolean_t gzalloc_free(zone_t zone, void *addr) {
	boolean_t gzfreed = FALSE;
	kern_return_t kr;

	if (__improbable(gzalloc_mode &&
		(((zone->elem_size >= gzalloc_min) &&
		    (zone->elem_size <= gzalloc_max))) &&
		(zone->gzalloc_exempt == 0))) {
		gzhdr_t *gzh;
		vm_offset_t rounded_size = round_page(zone->elem_size + GZHEADER_SIZE);
		vm_offset_t residue = rounded_size - zone->elem_size;
		vm_offset_t saddr;
		vm_offset_t free_addr = 0;

		if (gzalloc_uf_mode) {
			gzh = (gzhdr_t *)((vm_offset_t)addr + zone->elem_size);
			saddr = (vm_offset_t) addr - PAGE_SIZE;
		} else {
			gzh = (gzhdr_t *)((vm_offset_t)addr - GZHEADER_SIZE);
			saddr = ((vm_offset_t)addr) - residue;
		}

		assert((saddr & PAGE_MASK) == 0);

		if (gzalloc_consistency_checks) {
			if (gzh->gzsig != GZALLOC_SIGNATURE) {
				panic("GZALLOC signature mismatch for element %p, expected 0x%x, found 0x%x", addr, GZALLOC_SIGNATURE, gzh->gzsig);
			}

			if (gzh->gzone != zone && (gzh->gzone != GZDEADZONE))
				panic("%s: Mismatched zone or under/overflow, current zone: %p, recorded zone: %p, address: %p", __FUNCTION__, zone, gzh->gzone, (void *)addr);
			/* Partially redundant given the zone check, but may flag header corruption */
			if (gzh->gzsize != zone->elem_size) {
				panic("Mismatched zfree or under/overflow for zone %p, recorded size: 0x%x, element size: 0x%x, address: %p\n", zone, gzh->gzsize, (uint32_t) zone->elem_size, (void *)addr);
			}
		}

		if (!kmem_ready || gzh->gzone == GZDEADZONE) {
			/* For now, just leak frees of early allocations
			 * performed before kmem is fully configured.
			 * They don't seem to get freed currently;
			 * consider ml_static_mfree in the future.
			 */
			OSAddAtomic64((SInt32) (rounded_size), &gzalloc_early_free);
			return TRUE;
		}

		if (get_preemption_level() != 0) {
				pdzfree_count++;
		}

		if (gzfc_size) {
			/* Either write protect or unmap the newly freed
			 * allocation
			 */
			kr = vm_map_protect(
				gzalloc_map,
				saddr,
				saddr + rounded_size + (1 * PAGE_SIZE),
				gzalloc_prot,
				FALSE);
			if (kr != KERN_SUCCESS)
				panic("%s: vm_map_protect: %p, 0x%x", __FUNCTION__, (void *)saddr, kr);
		} else {
			free_addr = saddr;
		}

		lock_zone(zone);

		/* Insert newly freed element into the protected free element
		 * cache, and rotate out the LRU element.
		 */
		if (gzfc_size) {
			if (zone->gz.gzfc_index >= gzfc_size) {
				zone->gz.gzfc_index = 0;
			}
			free_addr = zone->gz.gzfc[zone->gz.gzfc_index];
			zone->gz.gzfc[zone->gz.gzfc_index++] = saddr;
		}

		if (free_addr) {
			zone->count--;
			zone->cur_size -= rounded_size;
		}

		unlock_zone(zone);

		if (free_addr) {
			kr = vm_map_remove(
				gzalloc_map,
				free_addr,
				free_addr + rounded_size + (1 * PAGE_SIZE),
				VM_MAP_REMOVE_KUNWIRE);
			if (kr != KERN_SUCCESS)
				panic("gzfree: vm_map_remove: %p, 0x%x", (void *)free_addr, kr);

			OSAddAtomic64((SInt32)rounded_size, &gzalloc_freed);
			OSAddAtomic64(-((SInt32) (rounded_size - zone->elem_size)), &gzalloc_wasted);
		}

		gzfreed = TRUE;
	}
	return gzfreed;
}
Commit	Line	Data
316670eb A	1	/*
	2	* Copyright (c) 2000-2012 Apple Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28	/*
	29	* File: kern/gzalloc.c
	30	* Author: Derek Kumar
	31	*
	32	* "Guard mode" zone allocator, used to trap use-after-free errors,
	33	* overruns, underruns, mismatched allocations/frees, uninitialized
	34	* zone element use, timing dependent races etc.
	35	*
	36	* The allocator is configured by these boot-args:
	37	* gzalloc_size=<size>: target all zones with elements of <size> bytes
	38	* gzalloc_min=<size>: target zones with elements >= size
	39	* gzalloc_max=<size>: target zones with elements <= size
	40	* gzalloc_min/max can be specified in conjunction to target a range of
	41	* sizes
	42	* gzalloc_fc_size=<size>: number of zone elements (effectively page
	43	* multiple sized) to retain in the free VA cache. This cache is evicted
	44	* (backing pages and VA released) in a least-recently-freed fashion.
	45	* Larger free VA caches allow for a longer window of opportunity to trap
	46	* delayed use-after-free operations, but use more memory.
	47	* -gzalloc_wp: Write protect, rather than unmap, freed allocations
	48	* lingering in the free VA cache. Useful to disambiguate between
	49	* read-after-frees/read overruns and writes. Also permits direct inspection
	50	* of the freed element in the cache via the kernel debugger. As each
	51	* element has a "header" (trailer in underflow detection mode), the zone
	52	* of origin of the element can be easily determined in this mode.
	53	* -gzalloc_uf_mode: Underflow detection mode, where the guard page
	54	* adjoining each element is placed before the element page rather than
	55	* after. The element is also located at the top of the page, rather than
	56	* abutting the bottom as with the standard overflow detection mode.
	57	* -gzalloc_noconsistency: disable consistency checks that flag mismatched
	58	* frees, corruptions of the header/trailer signatures etc.
	59	* -nogzalloc_mode: Disables the guard mode allocator. The DEBUG kernel
	60	* enables the guard allocator for zones sized 8K-16K (if present) by
	61	* default, this option can disable that behaviour.
	62	*/
	63
	64	#include <zone_debug.h>
65	#include <zone_alias_addr.h>
66
67	#include <mach/mach_types.h>
68	#include <mach/vm_param.h>
69	#include <mach/kern_return.h>
70	#include <mach/machine/vm_types.h>
71	#include <mach_debug/zone_info.h>
72	#include <mach/vm_map.h>
73
74	#include <kern/kern_types.h>
75	#include <kern/assert.h>
76	#include <kern/sched.h>
77	#include <kern/locks.h>
78	#include <kern/misc_protos.h>
79	#include <kern/zalloc.h>
80	#include <kern/kalloc.h>
81
82	#include <vm/pmap.h>
83	#include <vm/vm_map.h>
84	#include <vm/vm_kern.h>
85	#include <vm/vm_page.h>
86
87	#include <pexpert/pexpert.h>
88
89	#include <machine/machparam.h>
90
91	#include <libkern/OSDebug.h>
92	#include <libkern/OSAtomic.h>
93	#include <sys/kdebug.h>
94
95	extern boolean_t vm_kernel_ready, kmem_ready;
96	boolean_t gzalloc_mode = FALSE;
97	uint32_t pdzalloc_count, pdzfree_count;
98
99	#define GZALLOC_MIN_DEFAULT (1024)
100	#define GZDEADZONE ((zone_t) 0xDEAD201E)
101	#define GZALLOC_SIGNATURE (0xABADCAFE)
102	#define GZALLOC_RESERVE_SIZE_DEFAULT (2 * 1024 * 1024)
103	#define GZFC_DEFAULT_SIZE (1024)
104
105	char gzalloc_fill_pattern = 0x67; /* 'g' */
106
107	uint32_t gzalloc_min = ~0U;
108	uint32_t gzalloc_max = 0;
109	uint32_t gzalloc_size = 0;
110	uint64_t gzalloc_allocated, gzalloc_freed, gzalloc_early_alloc, gzalloc_early_free, gzalloc_wasted;
111	boolean_t gzalloc_uf_mode = FALSE, gzalloc_consistency_checks = TRUE;
112	vm_prot_t gzalloc_prot = VM_PROT_NONE;
113	uint32_t gzalloc_guard = KMA_GUARD_LAST;
114	uint32_t gzfc_size = GZFC_DEFAULT_SIZE;
115
116	vm_map_t gzalloc_map;
117	vm_offset_t gzalloc_map_min, gzalloc_map_max;
118	vm_offset_t gzalloc_reserve;
119	vm_size_t gzalloc_reserve_size;
120
121	typedef struct gzalloc_header {
122	zone_t gzone;
123	uint32_t gzsize;
124	uint32_t gzsig;
125	} gzhdr_t;
126
127	#define GZHEADER_SIZE (sizeof(gzhdr_t))
128
129	extern zone_t vm_page_zone;
130
131	void gzalloc_reconfigure(__unused zone_t z) {
132	/* Nothing for now */
133	}
134
135	boolean_t gzalloc_enabled(void) {
136	return gzalloc_mode;
137	}
138
139	void gzalloc_zone_init(zone_t z) {
140	if (gzalloc_mode) {
141	bzero(&z->gz, sizeof(z->gz));
142
143	if (gzfc_size && (z->elem_size >= gzalloc_min) && (z->elem_size <= gzalloc_max) && (z->gzalloc_exempt == FALSE)) {
144	vm_size_t gzfcsz = round_page(sizeof(z->gz.gzfc) gzfc_size);
145
146	/* If the VM/kmem system aren't yet configured, carve
147	* out the free element cache structure directly from the
148	* gzalloc_reserve supplied by the pmap layer.
149	*/
150	if (!kmem_ready) {
151	if (gzalloc_reserve_size < gzfcsz)
152	panic("gzalloc reserve exhausted");
153
154	z->gz.gzfc = (vm_offset_t *)gzalloc_reserve;
155	gzalloc_reserve += gzfcsz;
156	gzalloc_reserve_size -= gzfcsz;
157	} else {
158	kern_return_t kr;
159
3e170ce0	160	if ((kr = kernel_memory_allocate(kernel_map, (vm_offset_t *)&z->gz.gzfc, gzfcsz, 0, KMA_KOBJECT, VM_KERN_MEMORY_OSFMK)) != KERN_SUCCESS) {
316670eb A	161	panic("zinit/gzalloc: kernel_memory_allocate failed (%d) for 0x%lx bytes", kr, (unsigned long) gzfcsz);
	162	}
	163	}
	164	bzero((void *)z->gz.gzfc, gzfcsz);
	165	}
	166	}
	167	}
	168
	169	void gzalloc_configure(void) {
	170	char temp_buf[16];
	171
	172	if (PE_parse_boot_argn("-gzalloc_mode", temp_buf, sizeof (temp_buf))) {
	173	gzalloc_mode = TRUE;
	174	gzalloc_min = GZALLOC_MIN_DEFAULT;
	175	#if ZONE_DEBUG
	176	gzalloc_min += (typeof(gzalloc_min))ZONE_DEBUG_OFFSET;
	177	#endif
	178	gzalloc_max = ~0U;
	179	}
	180
	181	if (PE_parse_boot_argn("gzalloc_min", &gzalloc_min, sizeof(gzalloc_min))) {
	182	#if ZONE_DEBUG
	183	gzalloc_min += (typeof(gzalloc_min))ZONE_DEBUG_OFFSET;
	184	#endif
	185	gzalloc_mode = TRUE;
	186	gzalloc_max = ~0U;
	187	}
	188
	189	if (PE_parse_boot_argn("gzalloc_max", &gzalloc_max, sizeof(gzalloc_max))) {
	190	#if ZONE_DEBUG
	191	gzalloc_max += (typeof(gzalloc_min))ZONE_DEBUG_OFFSET;
	192	#endif
	193	gzalloc_mode = TRUE;
	194	if (gzalloc_min == ~0U)
	195	gzalloc_min = 0;
	196	}
	197
	198	if (PE_parse_boot_argn("gzalloc_size", &gzalloc_size, sizeof(gzalloc_size))) {
	199	#if ZONE_DEBUG
	200	gzalloc_size += (typeof(gzalloc_min))ZONE_DEBUG_OFFSET;
	201	#endif
	202	gzalloc_min = gzalloc_max = gzalloc_size;
	203	gzalloc_mode = TRUE;
	204	}
	205
	206	(void)PE_parse_boot_argn("gzalloc_fc_size", &gzfc_size, sizeof(gzfc_size));
	207
	208	if (PE_parse_boot_argn("-gzalloc_wp", temp_buf, sizeof (temp_buf))) {
	209	gzalloc_prot = VM_PROT_READ;
	210	}
	211
	212	if (PE_parse_boot_argn("-gzalloc_uf_mode", temp_buf, sizeof (temp_buf))) {
	213	gzalloc_uf_mode = TRUE;
	214	gzalloc_guard = KMA_GUARD_FIRST;
	215	}
	216
	217	if (PE_parse_boot_argn("-gzalloc_noconsistency", temp_buf, sizeof (temp_buf))) {
	218	gzalloc_consistency_checks = FALSE;
	219	}
	220	#if DEBUG
	221	if (gzalloc_mode == FALSE) {
	222	gzalloc_min = 8192;
	223	gzalloc_max = 16384;
	224	gzalloc_prot = VM_PROT_READ;
225	gzalloc_mode = TRUE;
226	}
227	#endif
228	if (PE_parse_boot_argn("-nogzalloc_mode", temp_buf, sizeof (temp_buf)))
229	gzalloc_mode = FALSE;
230
231	if (gzalloc_mode) {
232	gzalloc_reserve_size = GZALLOC_RESERVE_SIZE_DEFAULT;
233	gzalloc_reserve = (vm_offset_t) pmap_steal_memory(gzalloc_reserve_size);
234	}
235	}
236
237	void gzalloc_init(vm_size_t max_zonemap_size) {
238	kern_return_t retval;
239
240	if (gzalloc_mode) {
241	retval = kmem_suballoc(kernel_map, &gzalloc_map_min, (max_zonemap_size << 2),
3e170ce0	242	FALSE, VM_FLAGS_ANYWHERE \| VM_FLAGS_PERMANENT \| VM_MAKE_TAG(VM_KERN_MEMORY_ZONE),
316670eb A	243	&gzalloc_map);
	244
	245	if (retval != KERN_SUCCESS)
	246	panic("zone_init: kmem_suballoc(gzalloc) failed");
	247	gzalloc_map_max = gzalloc_map_min + (max_zonemap_size << 2);
	248	}
	249	}
	250
	251	vm_offset_t
	252	gzalloc_alloc(zone_t zone, boolean_t canblock) {
	253	vm_offset_t addr = 0;
	254
	255	if (__improbable(gzalloc_mode &&
	256	(((zone->elem_size >= gzalloc_min) &&
	257	(zone->elem_size <= gzalloc_max))) &&
	258	(zone->gzalloc_exempt == 0))) {
	259
	260	if (get_preemption_level() != 0) {
	261	if (canblock == TRUE) {
	262	pdzalloc_count++;
	263	}
	264	else
	265	return 0;
	266	}
	267
	268	vm_offset_t rounded_size = round_page(zone->elem_size + GZHEADER_SIZE);
	269	vm_offset_t residue = rounded_size - zone->elem_size;
	270	vm_offset_t gzaddr = 0;
	271	gzhdr_t *gzh;
	272
	273	if (!kmem_ready \|\| (vm_page_zone == ZONE_NULL)) {
	274	/* Early allocations are supplied directly from the
	275	* reserve.
	276	*/
	277	if (gzalloc_reserve_size < rounded_size)
	278	panic("gzalloc reserve exhausted");
	279	gzaddr = gzalloc_reserve;
	280	/* No guard page for these early allocations, just
	281	* waste an additional page.
	282	*/
	283	gzalloc_reserve += rounded_size + PAGE_SIZE;
	284	gzalloc_reserve_size -= rounded_size + PAGE_SIZE;
	285	OSAddAtomic64((SInt32) (rounded_size), &gzalloc_early_alloc);
	286	}
	287	else {
	288	kern_return_t kr = kernel_memory_allocate(gzalloc_map,
	289	&gzaddr, rounded_size + (1*PAGE_SIZE),
3e170ce0 A	290	0, KMA_KOBJECT \| gzalloc_guard,
3e170ce0 A	291	VM_KERN_MEMORY_OSFMK);
316670eb A	292	if (kr != KERN_SUCCESS)
	293	panic("gzalloc: kernel_memory_allocate for size 0x%llx failed with %d", (uint64_t)rounded_size, kr);
	294
	295	}
	296
	297	if (gzalloc_uf_mode) {
	298	gzaddr += PAGE_SIZE;
	299	/* The "header" becomes a "footer" in underflow
	300	* mode.
	301	*/
	302	gzh = (gzhdr_t *) (gzaddr + zone->elem_size);
	303	addr = gzaddr;
	304	} else {
	305	gzh = (gzhdr_t *) (gzaddr + residue - GZHEADER_SIZE);
	306	addr = (gzaddr + residue);
	307	}
	308
	309	/* Fill with a pattern on allocation to trap uninitialized
	310	* data use. Since the element size may be "rounded up"
	311	* by higher layers such as the kalloc layer, this may
	312	* also identify overruns between the originally requested
	313	* size and the rounded size via visual inspection.
	314	* TBD: plumb through the originally requested size,
	315	* prior to rounding by kalloc/IOMalloc etc.
	316	* We also add a signature and the zone of origin in a header
	317	* prefixed to the allocation.
	318	*/
	319	memset((void *)gzaddr, gzalloc_fill_pattern, rounded_size);
	320
	321	gzh->gzone = (kmem_ready && vm_page_zone) ? zone : GZDEADZONE;
	322	gzh->gzsize = (uint32_t) zone->elem_size;
	323	gzh->gzsig = GZALLOC_SIGNATURE;
	324
	325	lock_zone(zone);
	326	zone->count++;
	327	zone->sum_count++;
	328	zone->cur_size += rounded_size;
	329	unlock_zone(zone);
	330
	331	OSAddAtomic64((SInt32) rounded_size, &gzalloc_allocated);
	332	OSAddAtomic64((SInt32) (rounded_size - zone->elem_size), &gzalloc_wasted);
	333	}
	334	return addr;
	335	}
	336
	337	boolean_t gzalloc_free(zone_t zone, void *addr) {
	338	boolean_t gzfreed = FALSE;
	339	kern_return_t kr;
	340
	341	if (__improbable(gzalloc_mode &&
	342	(((zone->elem_size >= gzalloc_min) &&
	343	(zone->elem_size <= gzalloc_max))) &&
	344	(zone->gzalloc_exempt == 0))) {
	345	gzhdr_t *gzh;
	346	vm_offset_t rounded_size = round_page(zone->elem_size + GZHEADER_SIZE);
	347	vm_offset_t residue = rounded_size - zone->elem_size;
	348	vm_offset_t saddr;
	349	vm_offset_t free_addr = 0;
	350
	351	if (gzalloc_uf_mode) {
	352	gzh = (gzhdr_t *)((vm_offset_t)addr + zone->elem_size);
	353	saddr = (vm_offset_t) addr - PAGE_SIZE;
	354	} else {
	355	gzh = (gzhdr_t *)((vm_offset_t)addr - GZHEADER_SIZE);
356	saddr = ((vm_offset_t)addr) - residue;
357	}
358
359	assert((saddr & PAGE_MASK) == 0);
360
361	if (gzalloc_consistency_checks) {
362	if (gzh->gzsig != GZALLOC_SIGNATURE) {
363	panic("GZALLOC signature mismatch for element %p, expected 0x%x, found 0x%x", addr, GZALLOC_SIGNATURE, gzh->gzsig);
364	}
365
366	if (gzh->gzone != zone && (gzh->gzone != GZDEADZONE))
367	panic("%s: Mismatched zone or under/overflow, current zone: %p, recorded zone: %p, address: %p", __FUNCTION__, zone, gzh->gzone, (void *)addr);
368	/* Partially redundant given the zone check, but may flag header corruption */
369	if (gzh->gzsize != zone->elem_size) {
370	panic("Mismatched zfree or under/overflow for zone %p, recorded size: 0x%x, element size: 0x%x, address: %p\n", zone, gzh->gzsize, (uint32_t) zone->elem_size, (void *)addr);
371	}
372	}
373
374	if (!kmem_ready \|\| gzh->gzone == GZDEADZONE) {
375	/* For now, just leak frees of early allocations
376	* performed before kmem is fully configured.
377	* They don't seem to get freed currently;
378	* consider ml_static_mfree in the future.
379	*/
380	OSAddAtomic64((SInt32) (rounded_size), &gzalloc_early_free);
381	return TRUE;
382	}
383
384	if (get_preemption_level() != 0) {
385	pdzfree_count++;
386	}
387
388	if (gzfc_size) {
389	/* Either write protect or unmap the newly freed
390	* allocation
391	*/
392	kr = vm_map_protect(
393	gzalloc_map,
394	saddr,
395	saddr + rounded_size + (1 * PAGE_SIZE),
396	gzalloc_prot,
397	FALSE);
398	if (kr != KERN_SUCCESS)
399	panic("%s: vm_map_protect: %p, 0x%x", __FUNCTION__, (void *)saddr, kr);
400	} else {
401	free_addr = saddr;
402	}
403
404	lock_zone(zone);
405
406	/* Insert newly freed element into the protected free element
407	* cache, and rotate out the LRU element.
408	*/
409	if (gzfc_size) {
410	if (zone->gz.gzfc_index >= gzfc_size) {
411	zone->gz.gzfc_index = 0;
412	}
413	free_addr = zone->gz.gzfc[zone->gz.gzfc_index];
414	zone->gz.gzfc[zone->gz.gzfc_index++] = saddr;
415	}
416
417	if (free_addr) {
418	zone->count--;
419	zone->cur_size -= rounded_size;
420	}
421
422	unlock_zone(zone);
423
424	if (free_addr) {
425	kr = vm_map_remove(
426	gzalloc_map,
427	free_addr,
428	free_addr + rounded_size + (1 * PAGE_SIZE),
429	VM_MAP_REMOVE_KUNWIRE);
430	if (kr != KERN_SUCCESS)
431	panic("gzfree: vm_map_remove: %p, 0x%x", (void *)free_addr, kr);
432
433	OSAddAtomic64((SInt32)rounded_size, &gzalloc_freed);
434	OSAddAtomic64(-((SInt32) (rounded_size - zone->elem_size)), &gzalloc_wasted);
435	}
436
437	gzfreed = TRUE;
438	}
439	return gzfreed;
440	}