[apple/xnu.git] / osfmk / kern / zalloc.h

/*
 * Copyright (c) 2000-2014 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
/*
 * @OSF_COPYRIGHT@
 */
/* 
 * Mach Operating System
 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
 * All Rights Reserved.
 * 
 * Permission to use, copy, modify and distribute this software and its
 * documentation is hereby granted, provided that both the copyright
 * notice and this permission notice appear in all copies of the
 * software, derivative works or modified versions, and any portions
 * thereof, and that both notices appear in supporting documentation.
 * 
 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
 * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
 * 
 * Carnegie Mellon requests users of this software to return to
 * 
 *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
 *  School of Computer Science
 *  Carnegie Mellon University
 *  Pittsburgh PA 15213-3890
 * 
 * any improvements or extensions that they make and grant Carnegie Mellon
 * the rights to redistribute these changes.
 */
/*
 */
/*
 *	File:	zalloc.h
 *	Author:	Avadis Tevanian, Jr.
 *	Date:	 1985
 *
 */

#ifdef	KERNEL_PRIVATE

#ifndef	_KERN_ZALLOC_H_
#define _KERN_ZALLOC_H_

#include <mach/machine/vm_types.h>
#include <mach_debug/zone_info.h>
#include <kern/kern_types.h>
#include <sys/cdefs.h>

#ifdef	MACH_KERNEL_PRIVATE

#include <zone_debug.h>
#include <kern/locks.h>
#include <kern/queue.h>
#include <kern/thread_call.h>
#include <kern/btlog.h>

#if KASAN
#include <sys/queue.h>
#include <san/kasan.h>
#endif

#ifdef CONFIG_ZCACHE
#include <kern/zcache.h>
#endif

#if	CONFIG_GZALLOC
typedef struct gzalloc_data {
	uint32_t	gzfc_index;
	vm_offset_t 	*gzfc;
} gzalloc_data_t;
#endif

/*
 *	A zone is a collection of fixed size blocks for which there
 *	is fast allocation/deallocation access.  Kernel routines can
 *	use zones to manage data structures dynamically, creating a zone
 *	for each type of data structure to be managed.
 *
 */

struct zone_free_element;
struct zone_page_metadata;

struct zone {
#ifdef	 CONFIG_ZCACHE
	struct zone_cache *zcache;
#endif	/* CONFIG_ZCACHE */
	struct zone_free_element *free_elements;	/* free elements directly linked */
	struct {
		queue_head_t			any_free_foreign;	/* foreign pages crammed into zone */
		queue_head_t			all_free;
		queue_head_t			intermediate;
		queue_head_t			all_used;
	} pages;		/* list of zone_page_metadata structs, which maintain per-page free element lists */
	int		count;		/* Number of elements used now */
	int		countfree;	/* Number of free elements */
	int 	count_all_free_pages;  /* Number of pages collectable by GC */
	lck_attr_t      lock_attr;	/* zone lock attribute */
	decl_lck_mtx_data(,lock)	/* zone lock */
	lck_mtx_ext_t   lock_ext;	/* placeholder for indirect mutex */
	vm_size_t	cur_size;	/* current memory utilization */
	vm_size_t	max_size;	/* how large can this zone grow */
	vm_size_t	elem_size;	/* size of an element */
	vm_size_t	alloc_size;	/* size used for more memory */
	uint64_t	page_count __attribute__((aligned(8)));   /* number of pages used by this zone */
	uint64_t	sum_count;	/* count of allocs (life of zone) */
	uint32_t
	/* boolean_t */ exhaustible        :1,	/* (F) merely return if empty? */
	/* boolean_t */	collectable        :1,	/* (F) garbage collect empty pages */
	/* boolean_t */	expandable         :1,	/* (T) expand zone (with message)? */
	/* boolean_t */ allows_foreign     :1,  /* (F) allow non-zalloc space */
	/* boolean_t */	doing_alloc_without_vm_priv:1,	/* is zone expanding now via a non-vm_privileged thread? */
	/* boolean_t */ doing_alloc_with_vm_priv:1, /* is zone expanding now via a vm_privileged thread? */
	/* boolean_t */	waiting            :1,	/* is thread waiting for expansion? */
	/* boolean_t */	async_pending      :1,	/* asynchronous allocation pending? */
	/* boolean_t */ zleak_on           :1,	/* Are we collecting allocation information? */
	/* boolean_t */	caller_acct        :1,  /* do we account allocation/free to the caller? */  
	/* boolean_t */ noencrypt          :1,
	/* boolean_t */	no_callout         :1,
	/* boolean_t */	async_prio_refill  :1,
	/* boolean_t */	gzalloc_exempt     :1,
	/* boolean_t */	alignment_required :1,
	/* boolean_t */ zone_logging	   :1,	/* Enable zone logging for this zone. */
	/* boolean_t */ zone_replenishing  :1,
	/* boolean_t */ kasan_quarantine   :1,
	/* boolean_t */ tags               :1,
	/* boolean_t */ tags_inline        :1,
	/* future    */ tag_zone_index     :6,
	/* boolean_t */ zone_valid         :1,
	/* boolean_t */ cpu_cache_enable_when_ready  :1,
	/* boolean_t */ cpu_cache_enabled  :1,
	/* future    */ _reserved          :3;

	int		index;		/* index into zone_info arrays for this zone */
	const char	*zone_name;	/* a name for the zone */

#if CONFIG_ZLEAKS
	uint32_t zleak_capture;		/* per-zone counter for capturing every N allocations */
#endif /* CONFIG_ZLEAKS */
	uint32_t zp_count;              /* counter for poisoning every N frees */
	vm_size_t	prio_refill_watermark;
	thread_t	zone_replenish_thread;
#if	CONFIG_GZALLOC
	gzalloc_data_t	gz;
#endif /* CONFIG_GZALLOC */

#if KASAN_ZALLOC
	vm_size_t kasan_redzone;
#endif

	btlog_t		*zlog_btlog;		/* zone logging structure to hold stacks and element references to those stacks. */
};

/*
 *	structure for tracking zone usage
 *	Used either one per task/thread for all zones or <per-task,per-zone>.
 */
typedef struct zinfo_usage_store_t {
	/* These fields may be updated atomically, and so must be 8 byte aligned */
	uint64_t	alloc __attribute__((aligned(8)));		/* allocation counter */
	uint64_t	free __attribute__((aligned(8)));		/* free counter */
} zinfo_usage_store_t;

/*
 * For sysctl kern.zones_collectable_bytes used by memory_maintenance to check if a
 * userspace reboot is needed. The only other way to query for this information
 * is via mach_memory_info() which is unavailable on release kernels.
 */
extern uint64_t get_zones_collectable_bytes(void);

/*
 * zone_gc also checks if the zone_map is getting close to full and triggers jetsams if needed, provided
 * consider_jetsams is set to TRUE. To avoid deadlocks, we only pass a value of TRUE from within the
 * vm_pageout_garbage_collect thread.
 */
extern void		zone_gc(boolean_t consider_jetsams);
extern void		consider_zone_gc(boolean_t consider_jetsams);
extern void		drop_free_elements(zone_t z);

/* Debug logging for zone-map-exhaustion jetsams. */
extern void		get_zone_map_size(uint64_t *current_size, uint64_t *capacity);
extern void		get_largest_zone_info(char *zone_name, size_t zone_name_len, uint64_t *zone_size);

/* Bootstrap zone module (create zone zone) */
extern void		zone_bootstrap(void);

/* Init zone module */
extern void		zone_init(
					vm_size_t	map_size);

/* Stack use statistics */
extern void		stack_fake_zone_init(int zone_index);
extern void		stack_fake_zone_info(
					int			*count, 
					vm_size_t	*cur_size, 
					vm_size_t	*max_size,
					vm_size_t	*elem_size,
					vm_size_t	*alloc_size, 
					uint64_t	*sum_size,
					int			*collectable, 
					int			*exhaustable,
					int		*caller_acct);

#if		ZONE_DEBUG

extern void		zone_debug_enable(
				zone_t		z);

extern void		zone_debug_disable(
				zone_t		z);

#define zone_debug_enabled(z) z->active_zones.next
#define	ROUNDUP(x,y)		((((x)+(y)-1)/(y))*(y))
#define ZONE_DEBUG_OFFSET	ROUNDUP(sizeof(queue_chain_t),16)
#endif	/* ZONE_DEBUG */

extern unsigned int            num_zones;
extern struct zone             zone_array[];

/* zindex and page_count must pack into 16 bits
 * update tools/lldbmacros/memory.py:GetRealMetadata
 * when these values change */

#define ZINDEX_BITS              (10U)
#define PAGECOUNT_BITS           (16U - ZINDEX_BITS)
#define MULTIPAGE_METADATA_MAGIC ((1UL << ZINDEX_BITS) - 1)
#define ZONE_CHUNK_MAXPAGES      ((1UL << PAGECOUNT_BITS) - 1)

/*
 * The max # of elements in a chunk should fit into zone_page_metadata.free_count (uint16_t).
 * Update this if the type of free_count changes.
 */
#define ZONE_CHUNK_MAXELEMENTS   (UINT16_MAX)

#endif	/* MACH_KERNEL_PRIVATE */

__BEGIN_DECLS


/* Item definitions for zalloc/zinit/zone_change */
#define Z_EXHAUST	1	/* Make zone exhaustible	*/
#define Z_COLLECT	2	/* Make zone collectable	*/
#define Z_EXPAND	3	/* Make zone expandable		*/
#define	Z_FOREIGN	4	/* Allow collectable zone to contain foreign elements */
#define Z_CALLERACCT	5	/* Account alloc/free against the caller */
#define Z_NOENCRYPT	6	/* Don't encrypt zone during hibernation */
#define Z_NOCALLOUT 	7	/* Don't asynchronously replenish the zone via callouts */
#define Z_ALIGNMENT_REQUIRED 8
#define Z_GZALLOC_EXEMPT 9	/* Not tracked in guard allocation mode */
#define Z_KASAN_QUARANTINE	10 /* Allow zone elements to be quarantined on free */
#ifdef	XNU_KERNEL_PRIVATE
#define Z_TAGS_ENABLED	11      /* Store tags */
#endif  /* XNU_KERNEL_PRIVATE */
#define Z_CACHING_ENABLED 12	/*enable and initialize per-cpu caches for the zone*/

#ifdef	XNU_KERNEL_PRIVATE

extern vm_offset_t     zone_map_min_address;
extern vm_offset_t     zone_map_max_address;

/* free an element with no regard for gzalloc, zleaks, or kasan*/
extern void 	zfree_direct(		zone_t 		zone,
					vm_offset_t 	elem);

/* attempts to allocate an element with no regard for gzalloc, zleaks, or kasan*/
extern void *	zalloc_attempt(		zone_t		zone);

/* Non-waiting for memory version of zalloc */
extern void *	zalloc_nopagewait(
					zone_t		zone);

/* selective version of zalloc */
extern void *	zalloc_canblock(
					zone_t		zone,
					boolean_t	canblock);

/* selective version of zalloc */
extern void *	zalloc_canblock_tag(
					zone_t		zone,
					boolean_t	canblock,
					vm_size_t	reqsize,
					vm_tag_t    tag);

/* Get from zone free list */
extern void *	zget(
					zone_t		zone);

/* Fill zone with memory */
extern void		zcram(
					zone_t		zone,
					vm_offset_t	newmem,
					vm_size_t	size);

/* Initially fill zone with specified number of elements */
extern int		zfill(
					zone_t		zone,
					int			nelem);

extern void		zone_prio_refill_configure(zone_t, vm_size_t);

/* See above/top of file. Z_* definitions moved so they would be usable by kexts */

/* Preallocate space for zone from zone map */
extern void		zprealloc(
					zone_t		zone,
					vm_size_t	size);

extern integer_t	zone_free_count(
						zone_t		zone);

extern vm_size_t 	zone_element_size(
						void 		*addr,
						zone_t 		*z);

/* 
 *  Structure for keeping track of a backtrace, used for leak detection.
 *  This is in the .h file because it is used during panic, see kern/debug.c
 *  A non-zero size indicates that the trace is in use.
 */
struct ztrace {
	vm_size_t		zt_size;			/* How much memory are all the allocations referring to this trace taking up? */
	uint32_t		zt_depth;			/* depth of stack (0 to MAX_ZTRACE_DEPTH) */
	void*			zt_stack[MAX_ZTRACE_DEPTH];	/* series of return addresses from OSBacktrace */
	uint32_t		zt_collisions;			/* How many times did a different stack land here while it was occupied? */
	uint32_t		zt_hit_count;			/* for determining effectiveness of hash function */
};

#if CONFIG_ZLEAKS

/* support for the kern.zleak.* sysctls */

extern kern_return_t zleak_activate(void);
extern vm_size_t zleak_max_zonemap_size;
extern vm_size_t zleak_global_tracking_threshold;
extern vm_size_t zleak_per_zone_tracking_threshold;

extern int get_zleak_state(void);

#endif	/* CONFIG_ZLEAKS */

#ifndef VM_MAX_TAG_ZONES
#error MAX_TAG_ZONES
#endif

#if VM_MAX_TAG_ZONES

extern boolean_t zone_tagging_on;
extern uint32_t  zone_index_from_tag_index(uint32_t tag_zone_index, vm_size_t * elem_size);

#endif /* VM_MAX_TAG_ZONES */

/* These functions used for leak detection both in zalloc.c and mbuf.c */
extern uintptr_t hash_mix(uintptr_t);
extern uint32_t hashbacktrace(uintptr_t *, uint32_t, uint32_t);
extern uint32_t hashaddr(uintptr_t, uint32_t);

#define lock_zone(zone)					\
MACRO_BEGIN						\
	lck_mtx_lock_spin_always(&(zone)->lock);	\
MACRO_END

#define unlock_zone(zone)				\
MACRO_BEGIN						\
	lck_mtx_unlock(&(zone)->lock);			\
MACRO_END

#if	CONFIG_GZALLOC
void gzalloc_init(vm_size_t);
void gzalloc_zone_init(zone_t);
void gzalloc_configure(void);
void gzalloc_reconfigure(zone_t);
void gzalloc_empty_free_cache(zone_t);
boolean_t gzalloc_enabled(void);

vm_offset_t gzalloc_alloc(zone_t, boolean_t);
boolean_t gzalloc_free(zone_t, void *);
boolean_t gzalloc_element_size(void *, zone_t *, vm_size_t *);
#endif /* CONFIG_GZALLOC */

/* Callbacks for btlog lock/unlock */
void zlog_btlog_lock(__unused void *);
void zlog_btlog_unlock(__unused void *);

#ifdef MACH_KERNEL_PRIVATE
#define MAX_ZONE_NAME	32	/* max length of a zone name we can take from the boot-args */
int track_this_zone(const char *zonename, const char *logname);
#endif

#if DEBUG || DEVELOPMENT
extern boolean_t run_zone_test(void);
extern vm_size_t zone_element_info(void *addr, vm_tag_t * ptag);
#endif /* DEBUG || DEVELOPMENT */

#endif	/* XNU_KERNEL_PRIVATE */

/* Allocate from zone */
extern void *	zalloc(
					zone_t		zone);

/* Non-blocking version of zalloc */
extern void *	zalloc_noblock(
					zone_t		zone);

/* Free zone element */
extern void		zfree(
					zone_t		zone,
					void 		*elem);

/* Create zone */
extern zone_t	zinit(
					vm_size_t	size,		/* the size of an element */
					vm_size_t	maxmem,		/* maximum memory to use */
					vm_size_t	alloc,		/* allocation size */
					const char	*name);		/* a name for the zone */

/* Change zone parameters */
extern void		zone_change(
					zone_t			zone,
					unsigned int	item,
					boolean_t		value);

/* Destroy the zone */
extern void		zdestroy(
					zone_t		zone);

__END_DECLS

#endif	/* _KERN_ZALLOC_H_ */

#endif	/* KERNEL_PRIVATE */
Commit	Line	Data
1c79356b	1	/*
fe8ab488	2	* Copyright (c) 2000-2014 Apple Inc. All rights reserved.
1c79356b	3	*
2d21ac55	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
1c79356b	5	*
2d21ac55 A	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
8f6c56a5	14	*
2d21ac55 A	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5 A	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
8f6c56a5 A	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55 A	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
8f6c56a5	25	*
2d21ac55	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
1c79356b A	27	*/
	28	/*
	29	* @OSF_COPYRIGHT@
	30	*/
	31	/*
	32	* Mach Operating System
	33	* Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
	34	* All Rights Reserved.
	35	*
	36	* Permission to use, copy, modify and distribute this software and its
	37	* documentation is hereby granted, provided that both the copyright
	38	* notice and this permission notice appear in all copies of the
	39	* software, derivative works or modified versions, and any portions
	40	* thereof, and that both notices appear in supporting documentation.
	41	*
	42	* CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
	43	* CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
	44	* ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
	45	*
	46	* Carnegie Mellon requests users of this software to return to
	47	*
	48	* Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
	49	* School of Computer Science
	50	* Carnegie Mellon University
	51	* Pittsburgh PA 15213-3890
	52	*
	53	* any improvements or extensions that they make and grant Carnegie Mellon
	54	* the rights to redistribute these changes.
	55	*/
	56	/*
	57	*/
	58	/*
	59	* File: zalloc.h
	60	* Author: Avadis Tevanian, Jr.
	61	* Date: 1985
	62	*
	63	*/
	64
91447636 A	65	#ifdef KERNEL_PRIVATE
91447636 A	66
1c79356b A	67	#ifndef _KERN_ZALLOC_H_
	68	#define _KERN_ZALLOC_H_
	69
	70	#include <mach/machine/vm_types.h>
d9a64523	71	#include <mach_debug/zone_info.h>
1c79356b	72	#include <kern/kern_types.h>
91447636	73	#include <sys/cdefs.h>
9bccf70c A	74
	75	#ifdef MACH_KERNEL_PRIVATE
	76
1c79356b	77	#include <zone_debug.h>
2d21ac55	78	#include <kern/locks.h>
1c79356b	79	#include <kern/queue.h>
316670eb	80	#include <kern/thread_call.h>
39037602	81	#include <kern/btlog.h>
316670eb	82
5ba3f43e A	83	#if KASAN
	84	#include <sys/queue.h>
	85	#include <san/kasan.h>
	86	#endif
	87
d9a64523 A	88	#ifdef CONFIG_ZCACHE
	89	#include <kern/zcache.h>
	90	#endif
	91
316670eb A	92	#if CONFIG_GZALLOC
	93	typedef struct gzalloc_data {
	94	uint32_t gzfc_index;
	95	vm_offset_t *gzfc;
	96	} gzalloc_data_t;
	97	#endif
1c79356b A	98
	99	/*
	100	* A zone is a collection of fixed size blocks for which there
	101	* is fast allocation/deallocation access. Kernel routines can
	102	* use zones to manage data structures dynamically, creating a zone
	103	* for each type of data structure to be managed.
	104	*
	105	*/
	106
39236c6e A	107	struct zone_free_element;
	108	struct zone_page_metadata;
	109
1c79356b	110	struct zone {
d9a64523 A	111	#ifdef CONFIG_ZCACHE
	112	struct zone_cache *zcache;
	113	#endif /* CONFIG_ZCACHE */
39236c6e A	114	struct zone_free_element free_elements; / free elements directly linked */
	115	struct {
	116	queue_head_t any_free_foreign; /* foreign pages crammed into zone */
	117	queue_head_t all_free;
	118	queue_head_t intermediate;
	119	queue_head_t all_used;
	120	} pages; /* list of zone_page_metadata structs, which maintain per-page free element lists */
1c79356b	121	int count; /* Number of elements used now */
39236c6e	122	int countfree; /* Number of free elements */
39037602	123	int count_all_free_pages; /* Number of pages collectable by GC */
39236c6e	124	lck_attr_t lock_attr; /* zone lock attribute */
2d21ac55 A	125	decl_lck_mtx_data(,lock) /* zone lock */
2d21ac55 A	126	lck_mtx_ext_t lock_ext; /* placeholder for indirect mutex */
1c79356b A	127	vm_size_t cur_size; /* current memory utilization */
	128	vm_size_t max_size; /* how large can this zone grow */
	129	vm_size_t elem_size; /* size of an element */
	130	vm_size_t alloc_size; /* size used for more memory */
39236c6e	131	uint64_t page_count __attribute__((aligned(8))); /* number of pages used by this zone */
6d2010ae	132	uint64_t sum_count; /* count of allocs (life of zone) */
39236c6e A	133	uint32_t
	134	/* boolean_t / exhaustible :1, / (F) merely return if empty? */
	135	/* boolean_t / collectable :1, / (F) garbage collect empty pages */
	136	/* boolean_t / expandable :1, / (T) expand zone (with message)? */
	137	/* boolean_t / allows_foreign :1, / (F) allow non-zalloc space */
3e170ce0 A	138	/* boolean_t / doing_alloc_without_vm_priv:1, / is zone expanding now via a non-vm_privileged thread? */
3e170ce0 A	139	/* boolean_t / doing_alloc_with_vm_priv:1, / is zone expanding now via a vm_privileged thread? */
39236c6e A	140	/* boolean_t / waiting :1, / is thread waiting for expansion? */
	141	/* boolean_t / async_pending :1, / asynchronous allocation pending? */
	142	/* boolean_t / zleak_on :1, / Are we collecting allocation information? */
	143	/* boolean_t / caller_acct :1, / do we account allocation/free to the caller? */
39236c6e A	144	/* boolean_t */ noencrypt :1,
	145	/* boolean_t */ no_callout :1,
	146	/* boolean_t */ async_prio_refill :1,
	147	/* boolean_t */ gzalloc_exempt :1,
	148	/* boolean_t */ alignment_required :1,
39037602 A	149	/* boolean_t / zone_logging :1, / Enable zone logging for this zone. */
39037602 A	150	/* boolean_t */ zone_replenishing :1,
5ba3f43e A	151	/* boolean_t */ kasan_quarantine :1,
	152	/* boolean_t */ tags :1,
	153	/* boolean_t */ tags_inline :1,
	154	/* future */ tag_zone_index :6,
	155	/* boolean_t */ zone_valid :1,
d9a64523 A	156	/* boolean_t */ cpu_cache_enable_when_ready :1,
	157	/* boolean_t */ cpu_cache_enabled :1,
	158	/* future */ _reserved :3;
39236c6e	159
6d2010ae	160	int index; /* index into zone_info arrays for this zone */
91447636	161	const char zone_name; / a name for the zone */
6d2010ae A	162
6d2010ae A	163	#if CONFIG_ZLEAKS
316670eb	164	uint32_t zleak_capture; /* per-zone counter for capturing every N allocations */
6d2010ae	165	#endif /* CONFIG_ZLEAKS */
39236c6e	166	uint32_t zp_count; /* counter for poisoning every N frees */
7ddcb079 A	167	vm_size_t prio_refill_watermark;
7ddcb079 A	168	thread_t zone_replenish_thread;
316670eb A	169	#if CONFIG_GZALLOC
	170	gzalloc_data_t gz;
	171	#endif /* CONFIG_GZALLOC */
39037602	172
5ba3f43e A	173	#if KASAN_ZALLOC
	174	vm_size_t kasan_redzone;
	175	#endif
	176
39037602	177	btlog_t zlog_btlog; / zone logging structure to hold stacks and element references to those stacks. */
1c79356b A	178	};
1c79356b A	179
6d2010ae A	180	/*
	181	* structure for tracking zone usage
	182	* Used either one per task/thread for all zones or <per-task,per-zone>.
	183	*/
	184	typedef struct zinfo_usage_store_t {
	185	/* These fields may be updated atomically, and so must be 8 byte aligned */
	186	uint64_t alloc __attribute__((aligned(8))); /* allocation counter */
	187	uint64_t free __attribute__((aligned(8))); /* free counter */
	188	} zinfo_usage_store_t;
6d2010ae	189
5ba3f43e A	190	/*
	191	* For sysctl kern.zones_collectable_bytes used by memory_maintenance to check if a
	192	* userspace reboot is needed. The only other way to query for this information
	193	* is via mach_memory_info() which is unavailable on release kernels.
	194	*/
	195	extern uint64_t get_zones_collectable_bytes(void);
	196
	197	/*
	198	* zone_gc also checks if the zone_map is getting close to full and triggers jetsams if needed, provided
	199	* consider_jetsams is set to TRUE. To avoid deadlocks, we only pass a value of TRUE from within the
	200	* vm_pageout_garbage_collect thread.
	201	*/
	202	extern void zone_gc(boolean_t consider_jetsams);
	203	extern void consider_zone_gc(boolean_t consider_jetsams);
	204	extern void drop_free_elements(zone_t z);
	205
	206	/* Debug logging for zone-map-exhaustion jetsams. */
	207	extern void get_zone_map_size(uint64_t current_size, uint64_t capacity);
	208	extern void get_largest_zone_info(char zone_name, size_t zone_name_len, uint64_t zone_size);
1c79356b A	209
1c79356b A	210	/* Bootstrap zone module (create zone zone) */
39236c6e	211	extern void zone_bootstrap(void);
1c79356b A	212
1c79356b A	213	/* Init zone module */
91447636	214	extern void zone_init(
39236c6e	215	vm_size_t map_size);
1c79356b	216
91447636	217	/* Stack use statistics */
6d2010ae	218	extern void stack_fake_zone_init(int zone_index);
91447636 A	219	extern void stack_fake_zone_info(
	220	int *count,
	221	vm_size_t *cur_size,
	222	vm_size_t *max_size,
	223	vm_size_t *elem_size,
	224	vm_size_t *alloc_size,
6d2010ae	225	uint64_t *sum_size,
91447636	226	int *collectable,
6d2010ae A	227	int *exhaustable,
6d2010ae A	228	int *caller_acct);
1c79356b	229
91447636	230	#if ZONE_DEBUG
1c79356b	231
91447636 A	232	extern void zone_debug_enable(
	233	zone_t z);
	234
	235	extern void zone_debug_disable(
	236	zone_t z);
	237
316670eb A	238	#define zone_debug_enabled(z) z->active_zones.next
	239	#define ROUNDUP(x,y) ((((x)+(y)-1)/(y))*(y))
	240	#define ZONE_DEBUG_OFFSET ROUNDUP(sizeof(queue_chain_t),16)
91447636 A	241	#endif /* ZONE_DEBUG */
91447636 A	242
39037602 A	243	extern unsigned int num_zones;
	244	extern struct zone zone_array[];
	245
5ba3f43e A	246	/* zindex and page_count must pack into 16 bits
	247	* update tools/lldbmacros/memory.py:GetRealMetadata
	248	* when these values change */
	249
	250	#define ZINDEX_BITS (10U)
	251	#define PAGECOUNT_BITS (16U - ZINDEX_BITS)
	252	#define MULTIPAGE_METADATA_MAGIC ((1UL << ZINDEX_BITS) - 1)
	253	#define ZONE_CHUNK_MAXPAGES ((1UL << PAGECOUNT_BITS) - 1)
	254
	255	/*
	256	* The max # of elements in a chunk should fit into zone_page_metadata.free_count (uint16_t).
	257	* Update this if the type of free_count changes.
	258	*/
	259	#define ZONE_CHUNK_MAXELEMENTS (UINT16_MAX)
	260
91447636 A	261	#endif /* MACH_KERNEL_PRIVATE */
	262
	263	__BEGIN_DECLS
	264
91447636	265
813fb2f6 A	266	/* Item definitions for zalloc/zinit/zone_change */
	267	#define Z_EXHAUST 1 /* Make zone exhaustible */
	268	#define Z_COLLECT 2 /* Make zone collectable */
	269	#define Z_EXPAND 3 /* Make zone expandable */
	270	#define Z_FOREIGN 4 /* Allow collectable zone to contain foreign elements */
	271	#define Z_CALLERACCT 5 /* Account alloc/free against the caller */
	272	#define Z_NOENCRYPT 6 /* Don't encrypt zone during hibernation */
	273	#define Z_NOCALLOUT 7 /* Don't asynchronously replenish the zone via callouts */
	274	#define Z_ALIGNMENT_REQUIRED 8
	275	#define Z_GZALLOC_EXEMPT 9 /* Not tracked in guard allocation mode */
5ba3f43e A	276	#define Z_KASAN_QUARANTINE 10 /* Allow zone elements to be quarantined on free */
	277	#ifdef XNU_KERNEL_PRIVATE
	278	#define Z_TAGS_ENABLED 11 /* Store tags */
	279	#endif /* XNU_KERNEL_PRIVATE */
d9a64523	280	#define Z_CACHING_ENABLED 12 /enable and initialize per-cpu caches for the zone/
1c79356b	281
813fb2f6	282	#ifdef XNU_KERNEL_PRIVATE
91447636	283
813fb2f6 A	284	extern vm_offset_t zone_map_min_address;
813fb2f6 A	285	extern vm_offset_t zone_map_max_address;
91447636	286
d9a64523 A	287	/* free an element with no regard for gzalloc, zleaks, or kasan*/
	288	extern void zfree_direct( zone_t zone,
	289	vm_offset_t elem);
	290
	291	/* attempts to allocate an element with no regard for gzalloc, zleaks, or kasan*/
	292	extern void * zalloc_attempt( zone_t zone);
91447636	293
fe8ab488 A	294	/* Non-waiting for memory version of zalloc */
	295	extern void * zalloc_nopagewait(
	296	zone_t zone);
	297
fe8ab488	298	/* selective version of zalloc */
91447636 A	299	extern void * zalloc_canblock(
	300	zone_t zone,
	301	boolean_t canblock);
	302
5ba3f43e A	303	/* selective version of zalloc */
	304	extern void * zalloc_canblock_tag(
	305	zone_t zone,
	306	boolean_t canblock,
	307	vm_size_t reqsize,
	308	vm_tag_t tag);
	309
91447636 A	310	/* Get from zone free list */
	311	extern void * zget(
	312	zone_t zone);
1c79356b A	313
	314	/* Fill zone with memory */
	315	extern void zcram(
91447636	316	zone_t zone,
7ddcb079	317	vm_offset_t newmem,
91447636	318	vm_size_t size);
1c79356b A	319
	320	/* Initially fill zone with specified number of elements */
	321	extern int zfill(
91447636 A	322	zone_t zone,
	323	int nelem);
	324
7ddcb079	325	extern void zone_prio_refill_configure(zone_t, vm_size_t);
813fb2f6 A	326
813fb2f6 A	327	/* See above/top of file. Z_* definitions moved so they would be usable by kexts */
39236c6e	328
91447636 A	329	/* Preallocate space for zone from zone map */
	330	extern void zprealloc(
	331	zone_t zone,
	332	vm_size_t size);
9bccf70c	333
91447636 A	334	extern integer_t zone_free_count(
91447636 A	335	zone_t zone);
9bccf70c	336
39037602 A	337	extern vm_size_t zone_element_size(
	338	void *addr,
	339	zone_t *z);
	340
6d2010ae A	341	/*
	342	* Structure for keeping track of a backtrace, used for leak detection.
	343	* This is in the .h file because it is used during panic, see kern/debug.c
	344	* A non-zero size indicates that the trace is in use.
	345	*/
	346	struct ztrace {
	347	vm_size_t zt_size; /* How much memory are all the allocations referring to this trace taking up? */
	348	uint32_t zt_depth; /* depth of stack (0 to MAX_ZTRACE_DEPTH) */
	349	void* zt_stack[MAX_ZTRACE_DEPTH]; /* series of return addresses from OSBacktrace */
	350	uint32_t zt_collisions; /* How many times did a different stack land here while it was occupied? */
	351	uint32_t zt_hit_count; /* for determining effectiveness of hash function */
	352	};
	353
	354	#if CONFIG_ZLEAKS
	355
	356	/* support for the kern.zleak.* sysctls */
	357
	358	extern kern_return_t zleak_activate(void);
	359	extern vm_size_t zleak_max_zonemap_size;
	360	extern vm_size_t zleak_global_tracking_threshold;
	361	extern vm_size_t zleak_per_zone_tracking_threshold;
	362
	363	extern int get_zleak_state(void);
	364
	365	#endif /* CONFIG_ZLEAKS */
	366
5ba3f43e A	367	#ifndef VM_MAX_TAG_ZONES
	368	#error MAX_TAG_ZONES
	369	#endif
	370
	371	#if VM_MAX_TAG_ZONES
	372
	373	extern boolean_t zone_tagging_on;
	374	extern uint32_t zone_index_from_tag_index(uint32_t tag_zone_index, vm_size_t * elem_size);
	375
	376	#endif /* VM_MAX_TAG_ZONES */
	377
6d2010ae	378	/* These functions used for leak detection both in zalloc.c and mbuf.c */
316670eb A	379	extern uintptr_t hash_mix(uintptr_t);
	380	extern uint32_t hashbacktrace(uintptr_t *, uint32_t, uint32_t);
	381	extern uint32_t hashaddr(uintptr_t, uint32_t);
	382
	383	#define lock_zone(zone) \
	384	MACRO_BEGIN \
5ba3f43e	385	lck_mtx_lock_spin_always(&(zone)->lock); \
316670eb A	386	MACRO_END
	387
	388	#define unlock_zone(zone) \
	389	MACRO_BEGIN \
	390	lck_mtx_unlock(&(zone)->lock); \
	391	MACRO_END
	392
	393	#if CONFIG_GZALLOC
	394	void gzalloc_init(vm_size_t);
	395	void gzalloc_zone_init(zone_t);
	396	void gzalloc_configure(void);
	397	void gzalloc_reconfigure(zone_t);
5ba3f43e	398	void gzalloc_empty_free_cache(zone_t);
316670eb A	399	boolean_t gzalloc_enabled(void);
	400
	401	vm_offset_t gzalloc_alloc(zone_t, boolean_t);
	402	boolean_t gzalloc_free(zone_t, void *);
39037602	403	boolean_t gzalloc_element_size(void , zone_t , vm_size_t *);
316670eb	404	#endif /* CONFIG_GZALLOC */
6d2010ae	405
39037602 A	406	/* Callbacks for btlog lock/unlock */
	407	void zlog_btlog_lock(__unused void *);
	408	void zlog_btlog_unlock(__unused void *);
	409
5ba3f43e A	410	#ifdef MACH_KERNEL_PRIVATE
	411	#define MAX_ZONE_NAME 32 /* max length of a zone name we can take from the boot-args */
	412	int track_this_zone(const char zonename, const char logname);
	413	#endif
	414
	415	#if DEBUG \|\| DEVELOPMENT
	416	extern boolean_t run_zone_test(void);
	417	extern vm_size_t zone_element_info(void addr, vm_tag_t ptag);
	418	#endif /* DEBUG \|\| DEVELOPMENT */
	419
91447636	420	#endif /* XNU_KERNEL_PRIVATE */
9bccf70c	421
813fb2f6 A	422	/* Allocate from zone */
	423	extern void * zalloc(
	424	zone_t zone);
	425
5ba3f43e A	426	/* Non-blocking version of zalloc */
	427	extern void * zalloc_noblock(
	428	zone_t zone);
	429
813fb2f6 A	430	/* Free zone element */
	431	extern void zfree(
	432	zone_t zone,
	433	void *elem);
	434
	435	/* Create zone */
	436	extern zone_t zinit(
	437	vm_size_t size, /* the size of an element */
	438	vm_size_t maxmem, /* maximum memory to use */
	439	vm_size_t alloc, /* allocation size */
	440	const char name); / a name for the zone */
	441
	442	/* Change zone parameters */
	443	extern void zone_change(
	444	zone_t zone,
	445	unsigned int item,
	446	boolean_t value);
	447
5ba3f43e A	448	/* Destroy the zone */
	449	extern void zdestroy(
	450	zone_t zone);
813fb2f6	451
91447636	452	__END_DECLS
1c79356b A	453
1c79356b A	454	#endif /* _KERN_ZALLOC_H_ */
91447636 A	455
91447636 A	456	#endif /* KERNEL_PRIVATE */