/*
- * Copyright (c) 2000-2010 Apple Inc. All rights reserved.
+ * Copyright (c) 2000-2014 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
#ifdef MACH_KERNEL_PRIVATE
#include <zone_debug.h>
-#include <kern/lock.h>
#include <kern/locks.h>
#include <kern/queue.h>
#include <kern/thread_call.h>
+#include <kern/btlog.h>
+
+#if KASAN
+#include <sys/queue.h>
+#include <san/kasan.h>
+#endif
#if CONFIG_GZALLOC
typedef struct gzalloc_data {
*
*/
+struct zone_free_element;
+struct zone_page_metadata;
+
struct zone {
+ struct zone_free_element *free_elements; /* free elements directly linked */
+ struct {
+ queue_head_t any_free_foreign; /* foreign pages crammed into zone */
+ queue_head_t all_free;
+ queue_head_t intermediate;
+ queue_head_t all_used;
+ } pages; /* list of zone_page_metadata structs, which maintain per-page free element lists */
int count; /* Number of elements used now */
- vm_offset_t free_elements;
+ int countfree; /* Number of free elements */
+ int count_all_free_pages; /* Number of pages collectable by GC */
+ lck_attr_t lock_attr; /* zone lock attribute */
decl_lck_mtx_data(,lock) /* zone lock */
lck_mtx_ext_t lock_ext; /* placeholder for indirect mutex */
- lck_attr_t lock_attr; /* zone lock attribute */
- lck_grp_t lock_grp; /* zone lock group */
- lck_grp_attr_t lock_grp_attr; /* zone lock group attribute */
vm_size_t cur_size; /* current memory utilization */
vm_size_t max_size; /* how large can this zone grow */
vm_size_t elem_size; /* size of an element */
vm_size_t alloc_size; /* size used for more memory */
+ uint64_t page_count __attribute__((aligned(8))); /* number of pages used by this zone */
uint64_t sum_count; /* count of allocs (life of zone) */
- unsigned int
- /* boolean_t */ exhaustible :1, /* (F) merely return if empty? */
- /* boolean_t */ collectable :1, /* (F) garbage collect empty pages */
- /* boolean_t */ expandable :1, /* (T) expand zone (with message)? */
- /* boolean_t */ allows_foreign :1,/* (F) allow non-zalloc space */
- /* boolean_t */ doing_alloc :1, /* is zone expanding now? */
- /* boolean_t */ waiting :1, /* is thread waiting for expansion? */
- /* boolean_t */ async_pending :1, /* asynchronous allocation pending? */
-#if CONFIG_ZLEAKS
- /* boolean_t */ zleak_on :1, /* Are we collecting allocation information? */
-#endif /* CONFIG_ZLEAKS */
- /* boolean_t */ caller_acct: 1, /* do we account allocation/free to the caller? */
- /* boolean_t */ doing_gc :1, /* garbage collect in progress? */
- /* boolean_t */ noencrypt :1,
- /* boolean_t */ no_callout:1,
- /* boolean_t */ async_prio_refill:1,
- /* boolean_t */ gzalloc_exempt:1,
- /* boolean_t */ alignment_required:1;
+ uint32_t
+ /* boolean_t */ exhaustible :1, /* (F) merely return if empty? */
+ /* boolean_t */ collectable :1, /* (F) garbage collect empty pages */
+ /* boolean_t */ expandable :1, /* (T) expand zone (with message)? */
+ /* boolean_t */ allows_foreign :1, /* (F) allow non-zalloc space */
+ /* boolean_t */ doing_alloc_without_vm_priv:1, /* is zone expanding now via a non-vm_privileged thread? */
+ /* boolean_t */ doing_alloc_with_vm_priv:1, /* is zone expanding now via a vm_privileged thread? */
+ /* boolean_t */ waiting :1, /* is thread waiting for expansion? */
+ /* boolean_t */ async_pending :1, /* asynchronous allocation pending? */
+ /* boolean_t */ zleak_on :1, /* Are we collecting allocation information? */
+ /* boolean_t */ caller_acct :1, /* do we account allocation/free to the caller? */
+ /* boolean_t */ noencrypt :1,
+ /* boolean_t */ no_callout :1,
+ /* boolean_t */ async_prio_refill :1,
+ /* boolean_t */ gzalloc_exempt :1,
+ /* boolean_t */ alignment_required :1,
+ /* boolean_t */ zone_logging :1, /* Enable zone logging for this zone. */
+ /* boolean_t */ zone_replenishing :1,
+ /* boolean_t */ kasan_quarantine :1,
+ /* boolean_t */ tags :1,
+ /* boolean_t */ tags_inline :1,
+ /* future */ tag_zone_index :6,
+ /* boolean_t */ zone_valid :1,
+ /* future */ _reserved :5;
+
int index; /* index into zone_info arrays for this zone */
- struct zone * next_zone; /* Link for all-zones list */
- thread_call_data_t call_async_alloc; /* callout for asynchronous alloc */
const char *zone_name; /* a name for the zone */
-#if ZONE_DEBUG
- queue_head_t active_zones; /* active elements */
-#endif /* ZONE_DEBUG */
#if CONFIG_ZLEAKS
- uint32_t num_allocs; /* alloc stats for zleak benchmarks */
- uint32_t num_frees; /* free stats for zleak benchmarks */
uint32_t zleak_capture; /* per-zone counter for capturing every N allocations */
#endif /* CONFIG_ZLEAKS */
- uint32_t free_check_count; /* counter for poisoning/checking every N frees */
+ uint32_t zp_count; /* counter for poisoning every N frees */
vm_size_t prio_refill_watermark;
thread_t zone_replenish_thread;
#if CONFIG_GZALLOC
gzalloc_data_t gz;
#endif /* CONFIG_GZALLOC */
+
+#if KASAN_ZALLOC
+ vm_size_t kasan_redzone;
+#endif
+
+ btlog_t *zlog_btlog; /* zone logging structure to hold stacks and element references to those stacks. */
};
/*
uint64_t alloc __attribute__((aligned(8))); /* allocation counter */
uint64_t free __attribute__((aligned(8))); /* free counter */
} zinfo_usage_store_t;
-typedef zinfo_usage_store_t *zinfo_usage_t;
-extern void zone_gc(boolean_t);
-extern void consider_zone_gc(boolean_t);
+/*
+ * For sysctl kern.zones_collectable_bytes used by memory_maintenance to check if a
+ * userspace reboot is needed. The only other way to query for this information
+ * is via mach_memory_info() which is unavailable on release kernels.
+ */
+extern uint64_t get_zones_collectable_bytes(void);
-/* Steal memory for zone module */
-extern void zone_steal_memory(void);
+/*
+ * zone_gc also checks if the zone_map is getting close to full and triggers jetsams if needed, provided
+ * consider_jetsams is set to TRUE. To avoid deadlocks, we only pass a value of TRUE from within the
+ * vm_pageout_garbage_collect thread.
+ */
+extern void zone_gc(boolean_t consider_jetsams);
+extern void consider_zone_gc(boolean_t consider_jetsams);
+extern void drop_free_elements(zone_t z);
+
+/* Debug logging for zone-map-exhaustion jetsams. */
+extern void get_zone_map_size(uint64_t *current_size, uint64_t *capacity);
+extern void get_largest_zone_info(char *zone_name, size_t zone_name_len, uint64_t *zone_size);
/* Bootstrap zone module (create zone zone) */
-extern void zone_bootstrap(void) __attribute__((section("__TEXT, initcode")));
+extern void zone_bootstrap(void);
/* Init zone module */
extern void zone_init(
- vm_size_t map_size) __attribute__((section("__TEXT, initcode")));
-
-/* Handle per-task zone info */
-extern void zinfo_task_init(task_t task);
-extern void zinfo_task_free(task_t task);
-
+ vm_size_t map_size);
/* Stack use statistics */
extern void stack_fake_zone_init(int zone_index);
#define ZONE_DEBUG_OFFSET ROUNDUP(sizeof(queue_chain_t),16)
#endif /* ZONE_DEBUG */
+extern unsigned int num_zones;
+extern struct zone zone_array[];
+
+/* zindex and page_count must pack into 16 bits
+ * update tools/lldbmacros/memory.py:GetRealMetadata
+ * when these values change */
+
+#define ZINDEX_BITS (10U)
+#define PAGECOUNT_BITS (16U - ZINDEX_BITS)
+#define MULTIPAGE_METADATA_MAGIC ((1UL << ZINDEX_BITS) - 1)
+#define ZONE_CHUNK_MAXPAGES ((1UL << PAGECOUNT_BITS) - 1)
+
+/*
+ * The max # of elements in a chunk should fit into zone_page_metadata.free_count (uint16_t).
+ * Update this if the type of free_count changes.
+ */
+#define ZONE_CHUNK_MAXELEMENTS (UINT16_MAX)
+
#endif /* MACH_KERNEL_PRIVATE */
__BEGIN_DECLS
-#ifdef XNU_KERNEL_PRIVATE
-/* Allocate from zone */
-extern void * zalloc(
- zone_t zone);
+/* Item definitions for zalloc/zinit/zone_change */
+#define Z_EXHAUST 1 /* Make zone exhaustible */
+#define Z_COLLECT 2 /* Make zone collectable */
+#define Z_EXPAND 3 /* Make zone expandable */
+#define Z_FOREIGN 4 /* Allow collectable zone to contain foreign elements */
+#define Z_CALLERACCT 5 /* Account alloc/free against the caller */
+#define Z_NOENCRYPT 6 /* Don't encrypt zone during hibernation */
+#define Z_NOCALLOUT 7 /* Don't asynchronously replenish the zone via callouts */
+#define Z_ALIGNMENT_REQUIRED 8
+#define Z_GZALLOC_EXEMPT 9 /* Not tracked in guard allocation mode */
+#define Z_KASAN_QUARANTINE 10 /* Allow zone elements to be quarantined on free */
+#ifdef XNU_KERNEL_PRIVATE
+#define Z_TAGS_ENABLED 11 /* Store tags */
+#endif /* XNU_KERNEL_PRIVATE */
-/* Free zone element */
-extern void zfree(
- zone_t zone,
- void *elem);
+#ifdef XNU_KERNEL_PRIVATE
-/* Create zone */
-extern zone_t zinit(
- vm_size_t size, /* the size of an element */
- vm_size_t maxmem, /* maximum memory to use */
- vm_size_t alloc, /* allocation size */
- const char *name); /* a name for the zone */
+extern vm_offset_t zone_map_min_address;
+extern vm_offset_t zone_map_max_address;
-/* Non-blocking version of zalloc */
-extern void * zalloc_noblock(
+/* Non-waiting for memory version of zalloc */
+extern void * zalloc_nopagewait(
zone_t zone);
-/* direct (non-wrappered) interface */
+/* selective version of zalloc */
extern void * zalloc_canblock(
zone_t zone,
boolean_t canblock);
+/* selective version of zalloc */
+extern void * zalloc_canblock_tag(
+ zone_t zone,
+ boolean_t canblock,
+ vm_size_t reqsize,
+ vm_tag_t tag);
+
/* Get from zone free list */
extern void * zget(
zone_t zone);
zone_t zone,
int nelem);
-/* Change zone parameters */
-extern void zone_change(
- zone_t zone,
- unsigned int item,
- boolean_t value);
extern void zone_prio_refill_configure(zone_t, vm_size_t);
-/* Item definitions */
-#define Z_EXHAUST 1 /* Make zone exhaustible */
-#define Z_COLLECT 2 /* Make zone collectable */
-#define Z_EXPAND 3 /* Make zone expandable */
-#define Z_FOREIGN 4 /* Allow collectable zone to contain foreign elements */
-#define Z_CALLERACCT 5 /* Account alloc/free against the caller */
-#define Z_NOENCRYPT 6 /* Don't encrypt zone during hibernation */
-#define Z_NOCALLOUT 7 /* Don't asynchronously replenish the zone via
- * callouts
- */
-#define Z_ALIGNMENT_REQUIRED 8
-#define Z_GZALLOC_EXEMPT 9 /* Not tracked in guard allocation mode */
+
+/* See above/top of file. Z_* definitions moved so they would be usable by kexts */
+
/* Preallocate space for zone from zone map */
extern void zprealloc(
zone_t zone,
extern integer_t zone_free_count(
zone_t zone);
+extern vm_size_t zone_element_size(
+ void *addr,
+ zone_t *z);
+
/*
* MAX_ZTRACE_DEPTH configures how deep of a stack trace is taken on each zalloc in the zone of interest. 15
* levels is usually enough to get past all the layers of code in kalloc and IOKit and see who the actual
#endif /* CONFIG_ZLEAKS */
+#ifndef VM_MAX_TAG_ZONES
+#error MAX_TAG_ZONES
+#endif
+
+#if VM_MAX_TAG_ZONES
+
+extern boolean_t zone_tagging_on;
+extern uint32_t zone_index_from_tag_index(uint32_t tag_zone_index, vm_size_t * elem_size);
+
+#endif /* VM_MAX_TAG_ZONES */
+
/* These functions used for leak detection both in zalloc.c and mbuf.c */
-extern uint32_t fastbacktrace(uintptr_t* bt, uint32_t max_frames) __attribute__((noinline));
extern uintptr_t hash_mix(uintptr_t);
extern uint32_t hashbacktrace(uintptr_t *, uint32_t, uint32_t);
extern uint32_t hashaddr(uintptr_t, uint32_t);
#define lock_zone(zone) \
MACRO_BEGIN \
- lck_mtx_lock_spin(&(zone)->lock); \
+ lck_mtx_lock_spin_always(&(zone)->lock); \
MACRO_END
#define unlock_zone(zone) \
void gzalloc_zone_init(zone_t);
void gzalloc_configure(void);
void gzalloc_reconfigure(zone_t);
+void gzalloc_empty_free_cache(zone_t);
boolean_t gzalloc_enabled(void);
vm_offset_t gzalloc_alloc(zone_t, boolean_t);
boolean_t gzalloc_free(zone_t, void *);
+boolean_t gzalloc_element_size(void *, zone_t *, vm_size_t *);
#endif /* CONFIG_GZALLOC */
+/* Callbacks for btlog lock/unlock */
+void zlog_btlog_lock(__unused void *);
+void zlog_btlog_unlock(__unused void *);
+
+#ifdef MACH_KERNEL_PRIVATE
+#define MAX_ZONE_NAME 32 /* max length of a zone name we can take from the boot-args */
+int track_this_zone(const char *zonename, const char *logname);
+#endif
+
+#if DEBUG || DEVELOPMENT
+extern boolean_t run_zone_test(void);
+extern vm_size_t zone_element_info(void *addr, vm_tag_t * ptag);
+#endif /* DEBUG || DEVELOPMENT */
+
#endif /* XNU_KERNEL_PRIVATE */
+/* Allocate from zone */
+extern void * zalloc(
+ zone_t zone);
+
+/* Non-blocking version of zalloc */
+extern void * zalloc_noblock(
+ zone_t zone);
+
+/* Free zone element */
+extern void zfree(
+ zone_t zone,
+ void *elem);
+
+/* Create zone */
+extern zone_t zinit(
+ vm_size_t size, /* the size of an element */
+ vm_size_t maxmem, /* maximum memory to use */
+ vm_size_t alloc, /* allocation size */
+ const char *name); /* a name for the zone */
+
+/* Change zone parameters */
+extern void zone_change(
+ zone_t zone,
+ unsigned int item,
+ boolean_t value);
+
+/* Destroy the zone */
+extern void zdestroy(
+ zone_t zone);
+
__END_DECLS
#endif /* _KERN_ZALLOC_H_ */
projects / apple / xnu.git / blobdiff