#include <zone_debug.h>
#include <mach/boolean.h>
+#include <mach/sdt.h>
#include <mach/machine/vm_types.h>
#include <mach/vm_param.h>
#include <kern/misc_protos.h>
#include <libkern/OSMalloc.h>
#include <sys/kdebug.h>
+#include <san/kasan.h>
+
#ifdef MACH_BSD
zone_t kalloc_zone(vm_size_t);
#endif
}
/*
- * All allocations of size less than kalloc_max are rounded to the
- * next nearest sized zone. This allocator is built on top of
- * the zone allocator. A zone is created for each potential size
- * that we are willing to get in small blocks.
+ * All allocations of size less than kalloc_max are rounded to the next nearest
+ * sized zone. This allocator is built on top of the zone allocator. A zone
+ * is created for each potential size that we are willing to get in small
+ * blocks.
*
- * We assume that kalloc_max is not greater than 64K;
+ * We assume that kalloc_max is not greater than 64K;
*
- * Note that kalloc_max is somewhat confusingly named.
- * It represents the first power of two for which no zone exists.
- * kalloc_max_prerounded is the smallest allocation size, before
- * rounding, for which no zone exists.
+ * Note that kalloc_max is somewhat confusingly named. It represents the first
+ * power of two for which no zone exists. kalloc_max_prerounded is the
+ * smallest allocation size, before rounding, for which no zone exists.
*
- * Also if the allocation size is more than kalloc_kernmap_size
- * then allocate from kernel map rather than kalloc_map.
+ * Also if the allocation size is more than kalloc_kernmap_size then allocate
+ * from kernel map rather than kalloc_map.
*/
-#if KALLOC_MINSIZE == 16 && KALLOC_LOG2_MINALIGN == 4
+#define KALLOC_MINALIGN (1 << KALLOC_LOG2_MINALIGN)
+#define KiB(x) (1024 * (x))
-#define K_ZONE_SIZES \
- 16, \
- 32, \
- 48, \
-/* 3 */ 64, \
- 80, \
- 96, \
-/* 6 */ 128, \
- 160, 192, \
- 256, \
-/* 9 */ 288, \
- 512, 576, \
- 1024, 1152, \
-/* C */ 1280, \
- 2048, \
- 4096
-
-#define K_ZONE_NAMES \
- "kalloc.16", \
- "kalloc.32", \
- "kalloc.48", \
-/* 3 */ "kalloc.64", \
- "kalloc.80", \
- "kalloc.96", \
-/* 6 */ "kalloc.128", \
- "kalloc.160", \
- "kalloc.192", \
- "kalloc.256", \
-/* 9 */ "kalloc.288", \
- "kalloc.512", \
- "kalloc.576", \
- "kalloc.1024", \
- "kalloc.1152", \
-/* C */ "kalloc.1280", \
- "kalloc.2048", \
- "kalloc.4096"
+static const struct kalloc_zone_config {
+ int kzc_size;
+ const char *kzc_name;
+} k_zone_config[] = {
+#define KZC_ENTRY(SIZE) { .kzc_size = (SIZE), .kzc_name = "kalloc." #SIZE }
+#if KALLOC_MINSIZE == 16 && KALLOC_LOG2_MINALIGN == 4
+ /* 64-bit targets, generally */
+ KZC_ENTRY(16),
+ KZC_ENTRY(32),
+ KZC_ENTRY(48),
+ KZC_ENTRY(64),
+ KZC_ENTRY(80),
+ KZC_ENTRY(96),
+ KZC_ENTRY(128),
+ KZC_ENTRY(160),
+ KZC_ENTRY(192),
+ KZC_ENTRY(224),
+ KZC_ENTRY(256),
+ KZC_ENTRY(288),
+ KZC_ENTRY(368),
+ KZC_ENTRY(400),
+ KZC_ENTRY(512),
+ KZC_ENTRY(576),
+ KZC_ENTRY(768),
+ KZC_ENTRY(1024),
+ KZC_ENTRY(1152),
+ KZC_ENTRY(1280),
+ KZC_ENTRY(1664),
+ KZC_ENTRY(2048),
#elif KALLOC_MINSIZE == 8 && KALLOC_LOG2_MINALIGN == 3
-
-/*
- * Tweaked for ARM (and x64) in 04/2011
- */
-
-#define K_ZONE_SIZES \
-/* 3 */ 8, \
- 16, 24, \
- 32, 40, 48, \
-/* 6 */ 64, 72, 88, 112, \
- 128, 192, \
- 256, 288, 384, 440, \
-/* 9 */ 512, 576, 768, \
- 1024, 1152, 1536, \
- 2048, 2128, 3072, \
- 4096, 6144
-
-#define K_ZONE_NAMES \
-/* 3 */ "kalloc.8", \
- "kalloc.16", "kalloc.24", \
- "kalloc.32", "kalloc.40", "kalloc.48", \
-/* 6 */ "kalloc.64", "kalloc.72", "kalloc.88", "kalloc.112", \
- "kalloc.128", "kalloc.192", \
- "kalloc.256", "kalloc.288", "kalloc.384", "kalloc.440", \
-/* 9 */ "kalloc.512", "kalloc.576", "kalloc.768", \
- "kalloc.1024", "kalloc.1152", "kalloc.1536", \
- "kalloc.2048", "kalloc.2128", "kalloc.3072", \
- "kalloc.4096", "kalloc.6144"
-
+ /* 32-bit targets, generally */
+ KZC_ENTRY(8),
+ KZC_ENTRY(16),
+ KZC_ENTRY(24),
+ KZC_ENTRY(32),
+ KZC_ENTRY(40),
+ KZC_ENTRY(48),
+ KZC_ENTRY(64),
+ KZC_ENTRY(72),
+ KZC_ENTRY(88),
+ KZC_ENTRY(112),
+ KZC_ENTRY(128),
+ KZC_ENTRY(192),
+ KZC_ENTRY(256),
+ KZC_ENTRY(288),
+ KZC_ENTRY(384),
+ KZC_ENTRY(440),
+ KZC_ENTRY(512),
+ KZC_ENTRY(576),
+ KZC_ENTRY(768),
+ KZC_ENTRY(1024),
+ KZC_ENTRY(1152),
+ KZC_ENTRY(1536),
+ KZC_ENTRY(2048),
+ KZC_ENTRY(2128),
+ KZC_ENTRY(3072),
#else
-#error missing zone size parameters for kalloc
+#error missing or invalid zone size parameters for kalloc
#endif
-#define KALLOC_MINALIGN (1 << KALLOC_LOG2_MINALIGN)
-#define KiB(x) (1024 * (x))
-
-static const int k_zone_size[] = {
- K_ZONE_SIZES,
- KiB(8),
- KiB(16),
- KiB(32)
-};
-
-#define MAX_K_ZONE (sizeof (k_zone_size) / sizeof (k_zone_size[0]))
-
-static const char *k_zone_name[MAX_K_ZONE] = {
- K_ZONE_NAMES,
- "kalloc.8192",
- "kalloc.16384",
- "kalloc.32768"
+ /* all configurations get these zones */
+ KZC_ENTRY(4096),
+ KZC_ENTRY(6144),
+ KZC_ENTRY(8192),
+ KZC_ENTRY(16384),
+ KZC_ENTRY(32768),
+#undef KZC_ENTRY
};
+#define MAX_K_ZONE (int)(sizeof(k_zone_config) / sizeof(k_zone_config[0]))
/*
* Many kalloc() allocations are for small structures containing a few
kern_return_t retval;
vm_offset_t min;
vm_size_t size, kalloc_map_size;
- int i;
+ vm_map_kernel_flags_t vmk_flags;
/*
* Scale the kalloc_map_size to physical memory size: stay below
if (kalloc_map_size < KALLOC_MAP_SIZE_MIN)
kalloc_map_size = KALLOC_MAP_SIZE_MIN;
+ vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
+ vmk_flags.vmkf_permanent = TRUE;
+
retval = kmem_suballoc(kernel_map, &min, kalloc_map_size,
- FALSE, VM_FLAGS_ANYWHERE | VM_FLAGS_PERMANENT | VM_MAKE_TAG(0),
+ FALSE,
+ (VM_FLAGS_ANYWHERE),
+ vmk_flags,
+ VM_KERN_MEMORY_KALLOC,
&kalloc_map);
if (retval != KERN_SUCCESS)
kalloc_map_max = min + kalloc_map_size - 1;
/*
- * Create zones up to a least 2 pages because small page-multiples are common
- * allocations. Also ensure that zones up to size 8192 bytes exist. This is
- * desirable because messages are allocated with kalloc(), and messages up
- * through size 8192 are common.
+ * Create zones up to a least 4 pages because small page-multiples are
+ * common allocations. Also ensure that zones up to size 16KB bytes exist.
+ * This is desirable because messages are allocated with kalloc(), and
+ * messages up through size 8192 are common.
*/
kalloc_max = PAGE_SIZE << 2;
if (kalloc_max < KiB(16)) {
kalloc_largest_allocated = kalloc_kernmap_size;
/*
- * Allocate a zone for each size we are going to handle. Don't charge the
- * caller for the allocation, as we aren't sure how the memory will be
- * handled.
+ * Allocate a zone for each size we are going to handle.
*/
- for (i = 0; i < (int)MAX_K_ZONE && (size = k_zone_size[i]) < kalloc_max; i++) {
- k_zone[i] = zinit(size, size, size, k_zone_name[i]);
+ for (int i = 0; i < MAX_K_ZONE && (size = k_zone_config[i].kzc_size) < kalloc_max; i++) {
+ k_zone[i] = zinit(size, size, size, k_zone_config[i].kzc_name);
+
+ /*
+ * Don't charge the caller for the allocation, as we aren't sure how
+ * the memory will be handled.
+ */
zone_change(k_zone[i], Z_CALLERACCT, FALSE);
+#if VM_MAX_TAG_ZONES
+ if (zone_tagging_on) zone_change(k_zone[i], Z_TAGS_ENABLED, TRUE);
+#endif
+ zone_change(k_zone[i], Z_KASAN_QUARANTINE, FALSE);
}
/*
* Build the Direct LookUp Table for small allocations
*/
- for (i = 0, size = 0; i <= N_K_ZDLUT; i++, size += KALLOC_MINALIGN) {
+ size = 0;
+ for (int i = 0; i <= N_K_ZDLUT; i++, size += KALLOC_MINALIGN) {
int zindex = 0;
- while ((vm_size_t)k_zone_size[zindex] < size)
+ while ((vm_size_t)k_zone_config[zindex].kzc_size < size)
zindex++;
if (i == N_K_ZDLUT) {
* Useful when debugging/tweaking the array of zone sizes.
* Cache misses probably more critical than compare-branches!
*/
- for (i = 0; i < (int)MAX_K_ZONE; i++) {
- vm_size_t testsize = (vm_size_t)k_zone_size[i] - 1;
+ for (int i = 0; i < MAX_K_ZONE; i++) {
+ vm_size_t testsize = (vm_size_t)k_zone_config[i].kzc_size - 1;
int compare = 0;
int zindex;
compare += 2; /* 'if' (F), 'if' (T) */
zindex = k_zindex_start;
- while ((vm_size_t)k_zone_size[zindex] < testsize) {
+ while ((vm_size_t)k_zone_config[zindex].kzc_size < testsize) {
zindex++;
compare++; /* 'while' (T) */
}
return (k_zone[zindex]);
}
-/* As above, but linear search k_zone_size[] for the next zone that fits. */
+/* As above, but linear search k_zone_config[] for the next zone that fits. */
static __inline zone_t
get_zone_search(vm_size_t size, int zindex)
{
assert(size < kalloc_max_prerounded);
- while ((vm_size_t)k_zone_size[zindex] < size)
+ while ((vm_size_t)k_zone_config[zindex].kzc_size < size)
zindex++;
- assert((unsigned)zindex < MAX_K_ZONE &&
- (vm_size_t)k_zone_size[zindex] < kalloc_max);
+ assert(zindex < MAX_K_ZONE &&
+ (vm_size_t)k_zone_config[zindex].kzc_size < kalloc_max);
return (k_zone[zindex]);
}
return (vm_entry->vme_end - vm_entry->vme_start);
}
+#if KASAN_KALLOC
+/*
+ * KASAN kalloc stashes the original user-requested size away in the poisoned
+ * area. Return that directly.
+ */
+vm_size_t
+kalloc_size(void *addr)
+{
+ (void)vm_map_lookup_kalloc_entry_locked; /* silence warning */
+ return kasan_user_size((vm_offset_t)addr);
+}
+#else
vm_size_t
kalloc_size(
void *addr)
vm_map_unlock_read(map);
return size;
}
+#endif
vm_size_t
kalloc_bucket_size(
return vm_map_round_page(size, VM_MAP_PAGE_MASK(map));
}
+#if KASAN_KALLOC
+vm_size_t
+kfree_addr(void *addr)
+{
+ vm_size_t origsz = kalloc_size(addr);
+ kfree(addr, origsz);
+ return origsz;
+}
+#else
vm_size_t
kfree_addr(
void *addr)
size = zone_element_size(addr, &z);
if (size) {
+ DTRACE_VM3(kfree, vm_size_t, -1, vm_size_t, z->elem_size, void*, addr);
zfree(z, addr);
return size;
}
addr, map, ret);
}
vm_map_unlock(map);
+ DTRACE_VM3(kfree, vm_size_t, -1, vm_size_t, size, void*, addr);
kalloc_spin_lock();
kalloc_large_total -= size;
KALLOC_ZINFO_SFREE(size);
return size;
}
-
+#endif
+
void *
kalloc_canblock(
vm_size_t * psize,
{
zone_t z;
vm_size_t size;
+ void *addr;
+ vm_tag_t tag;
+ tag = VM_KERN_MEMORY_KALLOC;
size = *psize;
+#if KASAN_KALLOC
+ /* expand the allocation to accomodate redzones */
+ vm_size_t req_size = size;
+ size = kasan_alloc_resize(req_size);
+#endif
+
if (size < MAX_SIZE_ZDLUT)
z = get_zone_dlut(size);
else if (size < kalloc_max_prerounded)
* krealloc can use kmem_realloc.)
*/
vm_map_t alloc_map;
- void *addr;
/* kmem_alloc could block so we return if noblock */
if (!canblock) {
return(NULL);
}
+#if KASAN_KALLOC
+ /* large allocation - use guard pages instead of small redzones */
+ size = round_page(req_size + 2 * PAGE_SIZE);
+ assert(size >= MAX_SIZE_ZDLUT && size >= kalloc_max_prerounded);
+#endif
+
if (size >= kalloc_kernmap_size)
alloc_map = kernel_map;
else
alloc_map = kalloc_map;
- vm_tag_t tag;
- tag = (site ? tag = vm_tag_alloc(site) : VM_KERN_MEMORY_KALLOC);
+ if (site) tag = vm_tag_alloc(site);
if (kmem_alloc_flags(alloc_map, (vm_offset_t *)&addr, size, tag, KMA_ATOMIC) != KERN_SUCCESS) {
if (alloc_map != kernel_map) {
KALLOC_ZINFO_SALLOC(size);
}
+#if KASAN_KALLOC
+ /* fixup the return address to skip the redzone */
+ addr = (void *)kasan_alloc((vm_offset_t)addr, size, req_size, PAGE_SIZE);
+#else
*psize = round_page(size);
+#endif
+ DTRACE_VM3(kalloc, vm_size_t, size, vm_size_t, *psize, void*, addr);
return(addr);
}
#ifdef KALLOC_DEBUG
panic("%s: z %p (%s) but requested size %lu", __func__,
z, z->zone_name, (unsigned long)size);
#endif
+
assert(size <= z->elem_size);
+
+#if VM_MAX_TAG_ZONES
+ if (z->tags && site)
+ {
+ tag = vm_tag_alloc(site);
+ if (!canblock && !vm_allocation_zone_totals[tag]) tag = VM_KERN_MEMORY_KALLOC;
+ }
+#endif
+
+ addr = zalloc_canblock_tag(z, canblock, size, tag);
+
+#if KASAN_KALLOC
+ /* fixup the return address to skip the redzone */
+ addr = (void *)kasan_alloc((vm_offset_t)addr, z->elem_size, req_size, KASAN_GUARD_SIZE);
+
+ /* For KASan, the redzone lives in any additional space, so don't
+ * expand the allocation. */
+#else
*psize = z->elem_size;
- void *addr = zalloc_canblock(z, canblock);
+#endif
+
+ DTRACE_VM3(kalloc, vm_size_t, size, vm_size_t, *psize, void*, addr);
return addr;
}
{
zone_t z;
+#if KASAN_KALLOC
+ /*
+ * Resize back to the real allocation size and hand off to the KASan
+ * quarantine. `data` may then point to a different allocation.
+ */
+ vm_size_t user_size = size;
+ kasan_check_free((vm_address_t)data, size, KASAN_HEAP_KALLOC);
+ data = (void *)kasan_dealloc((vm_address_t)data, &size);
+ kasan_free(&data, &size, KASAN_HEAP_KALLOC, NULL, user_size, true);
+ if (!data) {
+ return;
+ }
+#endif
+
if (size < MAX_SIZE_ZDLUT)
z = get_zone_dlut(size);
else if (size < kalloc_max_prerounded)
kalloc_unlock();
+#if !KASAN_KALLOC
+ DTRACE_VM3(kfree, vm_size_t, size, vm_size_t, size, void*, data);
+#endif
+
KALLOC_ZINFO_SFREE(size);
return;
}
z, z->zone_name, (unsigned long)size);
#endif
assert(size <= z->elem_size);
+ DTRACE_VM3(kfree, vm_size_t, size, vm_size_t, z->elem_size, void*, data);
zfree(z, data);
}
}
#endif
-void
-kalloc_fake_zone_init(int zone_index)
-{
- kalloc_fake_zone_index = zone_index;
-}
-
-void
-kalloc_fake_zone_info(int *count,
- vm_size_t *cur_size, vm_size_t *max_size, vm_size_t *elem_size, vm_size_t *alloc_size,
- uint64_t *sum_size, int *collectable, int *exhaustable, int *caller_acct)
-{
- *count = kalloc_large_inuse;
- *cur_size = kalloc_large_total;
- *max_size = kalloc_large_max;
-
- if (kalloc_large_inuse) {
- *elem_size = kalloc_large_total / kalloc_large_inuse;
- *alloc_size = kalloc_large_total / kalloc_large_inuse;
- } else {
- *elem_size = 0;
- *alloc_size = 0;
- }
- *sum_size = kalloc_large_sum;
- *collectable = 0;
- *exhaustable = 0;
- *caller_acct = 0;
-}
-
-
void
OSMalloc_init(
void)
projects / apple / xnu.git / blobdiff