[apple/xnu.git] / osfmk / kern / zalloc.h

/*
 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
/*
 * @OSF_COPYRIGHT@
 */
/*
 * Mach Operating System
 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
 * All Rights Reserved.
 *
 * Permission to use, copy, modify and distribute this software and its
 * documentation is hereby granted, provided that both the copyright
 * notice and this permission notice appear in all copies of the
 * software, derivative works or modified versions, and any portions
 * thereof, and that both notices appear in supporting documentation.
 *
 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
 * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
 *
 * Carnegie Mellon requests users of this software to return to
 *
 *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
 *  School of Computer Science
 *  Carnegie Mellon University
 *  Pittsburgh PA 15213-3890
 *
 * any improvements or extensions that they make and grant Carnegie Mellon
 * the rights to redistribute these changes.
 */
/*
 */
/*
 *      File:   zalloc.h
 *      Author: Avadis Tevanian, Jr.
 *      Date:    1985
 *
 */

#ifdef  KERNEL_PRIVATE

#ifndef _KERN_ZALLOC_H_
#define _KERN_ZALLOC_H_

#include <mach/machine/vm_types.h>
#include <mach_debug/zone_info.h>
#include <kern/kern_types.h>
#include <sys/cdefs.h>

#if XNU_KERNEL_PRIVATE && !defined(ZALLOC_ALLOW_DEPRECATED)
#define __zalloc_deprecated(msg)       __deprecated_msg(msg)
#else
#define __zalloc_deprecated(msg)
#endif

__BEGIN_DECLS

/*!
 * @typedef zone_id_t
 *
 * @abstract
 * The type for a zone ID.
 */
typedef uint16_t zone_id_t;

/**
 * @enum zone_create_flags_t
 *
 * @abstract
 * Set of flags to pass to zone_create().
 *
 * @discussion
 * Some kernel-wide policies affect all possible created zones.
 * Explicit @c ZC_* win over such policies.
 */
__options_decl(zone_create_flags_t, uint64_t, {
        /** The default value to pass to zone_create() */
        ZC_NONE                 = 0x00000000,

        /** Force the created zone to use VA sequestering */
        ZC_SEQUESTER            = 0x00000001,
        /** Force the created zone @b NOT to use VA sequestering */
        ZC_NOSEQUESTER          = 0x00000002,

        /** Enable per-CPU zone caching for this zone */
        ZC_CACHING              = 0x00000010,
        /** Disable per-CPU zone caching for this zone */
        ZC_NOCACHING            = 0x00000020,


        /** Mark zone as a per-cpu zone */
        ZC_PERCPU               = 0x01000000,

        /** Force the created zone to clear every allocation on free */
        ZC_ZFREE_CLEARMEM       = 0x02000000,

        /** Mark zone as non collectable by zone_gc() */
        ZC_NOGC                 = 0x04000000,

        /** Do not encrypt this zone during hibernation */
        ZC_NOENCRYPT            = 0x08000000,

        /** Type requires alignment to be preserved */
        ZC_ALIGNMENT_REQUIRED   = 0x10000000,

        /** Do not track this zone when gzalloc is engaged */
        ZC_NOGZALLOC            = 0x20000000,

        /** Don't asynchronously replenish the zone via callouts */
        ZC_NOCALLOUT            = 0x40000000,

        /** Can be zdestroy()ed, not default unlike zinit() */
        ZC_DESTRUCTIBLE         = 0x80000000,

#ifdef XNU_KERNEL_PRIVATE

        /** This zone will back a kalloc heap */
        ZC_KALLOC_HEAP          = 0x0800000000000000,

        /** This zone can be crammed with foreign pages */
        ZC_ALLOW_FOREIGN        = 0x1000000000000000,

        /** This zone contains bytes / data buffers only */
        ZC_DATA_BUFFERS         = 0x2000000000000000,

        /** Disable kasan quarantine for this zone */
        ZC_KASAN_NOQUARANTINE   = 0x4000000000000000,

        /** Disable kasan redzones for this zone */
        ZC_KASAN_NOREDZONE      = 0x8000000000000000,
#endif
});

/*!
 * @union zone_or_view
 *
 * @abstract
 * A type used for calls that admit both a zone or a zone view.
 *
 * @discussion
 * @c zalloc() and @c zfree() and their variants can act on both
 * zones and zone views.
 */
union zone_or_view {
        struct zone_view *zov_view;
        struct zone      *zov_zone;
#ifdef __cplusplus
        inline zone_or_view(struct zone_view *zv) : zov_view(zv) {
        }
        inline zone_or_view(struct zone *z) : zov_zone(z) {
        }
#endif
};
#ifdef __cplusplus
typedef union zone_or_view zone_or_view_t;
#else
typedef union zone_or_view zone_or_view_t __attribute__((transparent_union));
#endif

/*!
 * @function zone_create
 *
 * @abstract
 * Creates a zone with the specified parameters.
 *
 * @discussion
 * A Zone is a slab allocator that returns objects of a given size very quickly.
 *
 * @param name          the name for the new zone.
 * @param size          the size of the elements returned by this zone.
 * @param flags         a set of @c zone_create_flags_t flags.
 *
 * @returns             the created zone, this call never fails.
 */
extern zone_t   zone_create(
        const char             *name,
        vm_size_t               size,
        zone_create_flags_t     flags);

/*!
 * @function zdestroy
 *
 * @abstract
 * Destroys a zone previously made with zone_create.
 *
 * @discussion
 * Zones must have been made destructible for @c zdestroy() to be allowed,
 * passing @c ZC_DESTRUCTIBLE at @c zone_create() time.
 *
 * @param zone          the zone to destroy.
 */
extern void     zdestroy(
        zone_t          zone);

/*!
 * @function zone_require
 *
 * @abstract
 * Requires for a given pointer to belong to the specified zone.
 *
 * @discussion
 * The function panics if the check fails as it indicates that the kernel
 * internals have been compromised.
 *
 * Note that zone_require() can only work with:
 * - zones not allowing foreign memory
 * - zones in the general submap.
 *
 * @param zone          the zone the address needs to belong to.
 * @param addr          the element address to check.
 */
extern void     zone_require(
        zone_t          zone,
        void           *addr);

/*!
 * @enum zalloc_flags_t
 *
 * @brief
 * Flags that can be passed to @c zalloc_internal or @c zalloc_flags.
 *
 * @discussion
 * It is encouraged that any callsite passing flags uses exactly one of:
 * @c Z_WAITOK, @c Z_NOWAIT or @c Z_NOPAGEWAIT, the default being @c Z_WAITOK
 * if nothing else was specified.
 *
 * If any @c Z_NO*WAIT flag is passed alongside @c Z_WAITOK,
 * then @c Z_WAITOK is ignored.
 *
 * @const Z_WAITOK
 * Means that it's OK for zalloc() to block to wait for memory,
 * when Z_WAITOK is passed, zalloc will never return NULL.
 *
 * @const Z_NOWAIT
 * Passing this flag means that zalloc is not allowed to ever block.
 *
 * @const Z_NOPAGEWAIT
 * Passing this flag means that zalloc is allowed to wait due to lock
 * contention, but will not wait for the VM to wait for pages when
 * under memory pressure.
 *
 * @const Z_ZERO
 * Passing this flags means that the returned memory has been zeroed out.
 *
 * @const Z_NOFAIL
 * Passing this flag means that the caller expects the allocation to always
 * succeed. This will result in a panic if this assumption isn't correct.
 *
 * This flag is incompatible with @c Z_NOWAIT or @c Z_NOPAGEWAIT. It also can't
 * be used on exhaustible zones.
 *
 #if XNU_KERNEL_PRIVATE
 *
 * @const Z_VM_TAG_MASK
 * Represents bits in which a vm_tag_t for the allocation can be passed.
 * (used by kalloc for the zone tagging debugging feature).
 #endif
 */
__options_decl(zalloc_flags_t, uint32_t, {
        // values smaller than 0xff are shared with the M_* flags from BSD MALLOC
        Z_WAITOK        = 0x0000,
        Z_NOWAIT        = 0x0001,
        Z_NOPAGEWAIT    = 0x0002,
        Z_ZERO          = 0x0004,

        Z_NOFAIL        = 0x8000,
#if XNU_KERNEL_PRIVATE
        /** used by kalloc to propagate vm tags for -zt */
        Z_VM_TAG_MASK   = 0xffff0000,

#define Z_VM_TAG_SHIFT        16
#define Z_VM_TAG(tag)         ((zalloc_flags_t)(tag) << Z_VM_TAG_SHIFT)
#endif
});

/*!
 * @function zalloc
 *
 * @abstract
 * Allocates an element from a specified zone.
 *
 * @discussion
 * If the zone isn't exhaustible and is expandable, this call never fails.
 *
 * @param zone_or_view  the zone or zone view to allocate from
 *
 * @returns             NULL or the allocated element
 */
extern void    *zalloc(
        zone_or_view_t  zone_or_view);

/*!
 * @function zalloc_noblock
 *
 * @abstract
 * Allocates an element from a specified zone, but never blocks.
 *
 * @discussion
 * This call is suitable for preemptible code, however allocation
 * isn't allowed from interrupt context.
 *
 * @param zone_or_view  the zone or zone view to allocate from
 *
 * @returns             NULL or the allocated element
 */
extern void    *zalloc_noblock(
        zone_or_view_t  zone_or_view);

/*!
 * @function zalloc_flags()
 *
 * @abstract
 * Allocates an element from a specified zone, with flags.
 *
 * @param zone_or_view  the zone or zone view to allocate from
 * @param flags         a collection of @c zalloc_flags_t.
 *
 * @returns             NULL or the allocated element
 */
extern void    *zalloc_flags(
        zone_or_view_t  zone_or_view,
        zalloc_flags_t  flags);

/*!
 * @function zfree
 *
 * @abstract
 * Frees an element allocated with @c zalloc*.
 *
 * @discussion
 * If the element being freed doesn't belong to the specified zone,
 * then this call will panic.
 *
 * @param zone_or_view  the zone or zone view to free the element to.
 * @param elem          the element to free
 */
extern void     zfree(
        zone_or_view_t  zone_or_view,
        void            *elem);

/* deprecated KPIS */

__zalloc_deprecated("use zone_create()")
extern zone_t   zinit(
        vm_size_t       size,           /* the size of an element */
        vm_size_t       maxmem,         /* maximum memory to use */
        vm_size_t       alloc,          /* allocation size */
        const char      *name);         /* a name for the zone */

#ifdef XNU_KERNEL_PRIVATE
#pragma mark - XNU only interfaces
#include <kern/startup.h>
#include <kern/cpu_number.h>

#pragma GCC visibility push(hidden)

#pragma mark XNU only: zalloc (extended)

#define ZALIGN_NONE             (sizeof(uint8_t)  - 1)
#define ZALIGN_16               (sizeof(uint16_t) - 1)
#define ZALIGN_32               (sizeof(uint32_t) - 1)
#define ZALIGN_PTR              (sizeof(void *)   - 1)
#define ZALIGN_64               (sizeof(uint64_t) - 1)
#define ZALIGN(t)               (_Alignof(t)      - 1)


/*!
 * @function zalloc_permanent()
 *
 * @abstract
 * Allocates a permanent element from the permanent zone
 *
 * @discussion
 * Memory returned by this function is always 0-initialized.
 * Note that the size of this allocation can not be determined
 * by zone_element_size so it should not be used for copyio.
 *
 * @param size          the element size (must be smaller than PAGE_SIZE)
 * @param align_mask    the required alignment for this allocation
 *
 * @returns             the allocated element
 */
extern void    *zalloc_permanent(
        vm_size_t       size,
        vm_offset_t     align_mask);

/*!
 * @function zalloc_permanent_type()
 *
 * @abstract
 * Allocates a permanent element of a given type with its natural alignment.
 *
 * @discussion
 * Memory returned by this function is always 0-initialized.
 *
 * @param type_t        the element type
 *
 * @returns             the allocated element
 */
#define zalloc_permanent_type(type_t) \
        ((type_t *)zalloc_permanent(sizeof(type_t), ZALIGN(type_t)))

#pragma mark XNU only: per-cpu allocations

/*!
 * @macro __zpercpu
 *
 * @abstract
 * Annotation that helps denoting a per-cpu pointer that requires usage of
 * @c zpercpu_*() for access.
 */
#define __zpercpu

/*!
 * @macro zpercpu_get_cpu()
 *
 * @abstract
 * Get a pointer to a specific CPU slot of a given per-cpu variable.
 *
 * @param ptr           the per-cpu pointer (returned by @c zalloc_percpu*()).
 * @param cpu           the specified CPU number as returned by @c cpu_number()
 *
 * @returns             the per-CPU slot for @c ptr for the specified CPU.
 */
#define zpercpu_get_cpu(ptr, cpu) \
        __zpcpu_cast(ptr, __zpcpu_demangle(ptr) + ptoa((unsigned)cpu))

/*!
 * @macro zpercpu_get()
 *
 * @abstract
 * Get a pointer to the current CPU slot of a given per-cpu variable.
 *
 * @param ptr           the per-cpu pointer (returned by @c zalloc_percpu*()).
 *
 * @returns             the per-CPU slot for @c ptr for the current CPU.
 */
#define zpercpu_get(ptr) \
        zpercpu_get_cpu(ptr, cpu_number())

/*!
 * @macro zpercpu_foreach()
 *
 * @abstract
 * Enumerate all per-CPU slots by address.
 *
 * @param it            the name for the iterator
 * @param ptr           the per-cpu pointer (returned by @c zalloc_percpu*()).
 */
#define zpercpu_foreach(it, ptr) \
        for (typeof(ptr) it = zpercpu_get_cpu(ptr, 0), \
            __end_##it = zpercpu_get_cpu(ptr, zpercpu_count()); \
            it < __end_##it; it = __zpcpu_next(it))

/*!
 * @macro zpercpu_foreach_cpu()
 *
 * @abstract
 * Enumerate all per-CPU slots by CPU slot number.
 *
 * @param cpu           the name for cpu number iterator.
 */
#define zpercpu_foreach_cpu(cpu) \
        for (unsigned cpu = 0; cpu < zpercpu_count(); cpu++)

/*!
 * @function zalloc_percpu()
 *
 * @abstract
 * Allocates an element from a per-cpu zone.
 *
 * @discussion
 * The returned pointer cannot be used directly and must be manipulated
 * through the @c zpercpu_get*() interfaces.
 *
 * @param zone_or_view  the zone or zone view to allocate from
 * @param flags         a collection of @c zalloc_flags_t.
 *
 * @returns             NULL or the allocated element
 */
extern void    *zalloc_percpu(
        zone_or_view_t  zone_or_view,
        zalloc_flags_t  flags);

/*!
 * @function zfree_percpu()
 *
 * @abstract
 * Frees an element previously allocated with @c zalloc_percpu().
 *
 * @param zone_or_view  the zone or zone view to free the element to.
 * @param addr          the address to free
 */
extern void     zfree_percpu(
        zone_or_view_t  zone_or_view,
        void           *addr);

/*!
 * @function zalloc_percpu_permanent()
 *
 * @abstract
 * Allocates a permanent percpu-element from the permanent percpu zone.
 *
 * @discussion
 * Memory returned by this function is always 0-initialized.
 *
 * @param size          the element size (must be smaller than PAGE_SIZE)
 * @param align_mask    the required alignment for this allocation
 *
 * @returns             the allocated element
 */
extern void    *zalloc_percpu_permanent(
        vm_size_t       size,
        vm_offset_t     align_mask);

/*!
 * @function zalloc_percpu_permanent_type()
 *
 * @abstract
 * Allocates a permanent percpu-element from the permanent percpu zone of a given
 * type with its natural alignment.
 *
 * @discussion
 * Memory returned by this function is always 0-initialized.
 *
 * @param type_t        the element type
 *
 * @returns             the allocated element
 */
#define zalloc_percpu_permanent_type(type_t) \
        ((type_t *)zalloc_percpu_permanent(sizeof(type_t), ZALIGN(type_t)))

#pragma mark XNU only: zone views

/*!
 * @enum zone_kheap_id_t
 *
 * @brief
 * Enumerate a particular kalloc heap.
 *
 * @discussion
 * More documentation about heaps is available in @c <kern/kalloc.h>.
 *
 * @const KHEAP_ID_NONE
 * This value denotes regular zones, not used by kalloc.
 *
 * @const KHEAP_ID_DEFAULT
 * Indicates zones part of the KHEAP_DEFAULT heap.
 *
 * @const KHEAP_ID_DATA_BUFFERS
 * Indicates zones part of the KHEAP_DATA_BUFFERS heap.
 *
 * @const KHEAP_ID_KEXT
 * Indicates zones part of the KHEAP_KEXT heap.
 */
__enum_decl(zone_kheap_id_t, uint32_t, {
        KHEAP_ID_NONE,
        KHEAP_ID_DEFAULT,
        KHEAP_ID_DATA_BUFFERS,
        KHEAP_ID_KEXT,

#define KHEAP_ID_COUNT (KHEAP_ID_KEXT + 1)
});

/*!
 * @typedef zone_stats_t
 *
 * @abstract
 * The opaque type for per-cpu zone stats that are accumulated per zone
 * or per zone-view.
 */
typedef struct zone_stats *__zpercpu zone_stats_t;

/*!
 * @typedef zone_view_t
 *
 * @abstract
 * A view on a zone for accounting purposes.
 *
 * @discussion
 * A zone view uses the zone it references for the allocations backing store,
 * but does the allocation accounting at the view level.
 *
 * These accounting are surfaced by @b zprint(1) and similar tools,
 * which allow for cheap but finer grained understanding of allocations
 * without any fragmentation cost.
 *
 * Zone views are protected by the kernel lockdown and can't be initialized
 * dynamically. They must be created using @c ZONE_VIEW_DEFINE().
 */
typedef struct zone_view *zone_view_t;
struct zone_view {
        zone_t          zv_zone;
        zone_stats_t    zv_stats;
        const char     *zv_name;
        zone_view_t     zv_next;
};

/*!
 * @macro ZONE_VIEW_DECLARE
 *
 * @abstract
 * (optionally) declares a zone view (in a header).
 *
 * @param var           the name for the zone view.
 */
#define ZONE_VIEW_DECLARE(var) \
        extern struct zone_view var[1]

/*!
 * @macro ZONE_VIEW_DEFINE
 *
 * @abstract
 * Defines a given zone view and what it points to.
 *
 * @discussion
 * Zone views can either share a pre-existing zone,
 * or perform a lookup into a kalloc heap for the zone
 * backing the bucket of the proper size.
 *
 * Zone views are initialized during the @c STARTUP_SUB_ZALLOC phase,
 * as the last rank. If views on zones are created, these must have been
 * created before this stage.
 *
 * @param var           the name for the zone view.
 * @param name          a string describing the zone view.
 * @param heap_or_zone  a @c KHEAP_ID_* constant or a pointer to a zone.
 * @param size          the element size to be allocated from this view.
 */
#define ZONE_VIEW_DEFINE(var, name, heap_or_zone, size) \
        SECURITY_READ_ONLY_LATE(struct zone_view) var[1] = { { \
            .zv_name = name, \
        } }; \
        static __startup_data struct zone_view_startup_spec \
        __startup_zone_view_spec_ ## var = { var, { heap_or_zone }, size }; \
        STARTUP_ARG(ZALLOC, STARTUP_RANK_LAST, zone_view_startup_init, \
            &__startup_zone_view_spec_ ## var)


#pragma mark XNU only: zone creation (extended)

/*!
 * @enum zone_reserved_id_t
 *
 * @abstract
 * Well known pre-registered zones, allowing use of zone_id_require()
 *
 * @discussion
 * @c ZONE_ID__* aren't real zone IDs.
 *
 * @c ZONE_ID__ZERO reserves zone index 0 so that it can't be used, as 0 is too
 * easy a value to produce (by malice or accident).
 *
 * @c ZONE_ID__FIRST_DYNAMIC is the first dynamic zone ID that can be used by
 * @c zone_create().
 */
__enum_decl(zone_reserved_id_t, zone_id_t, {
        ZONE_ID__ZERO,

        ZONE_ID_PERMANENT,
        ZONE_ID_PERCPU_PERMANENT,

        ZONE_ID_IPC_PORT,
        ZONE_ID_IPC_PORT_SET,
        ZONE_ID_IPC_VOUCHERS,
        ZONE_ID_TASK,
        ZONE_ID_PROC,
        ZONE_ID_VM_MAP_COPY,
        ZONE_ID_PMAP,

        ZONE_ID__FIRST_DYNAMIC,
});

/*!
 * @const ZONE_ID_ANY
 * The value to pass to @c zone_create_ext() to allocate a non pre-registered
 * Zone ID.
 */
#define ZONE_ID_ANY ((zone_id_t)-1)

/**!
 * @function zone_name
 *
 * @param zone          the specified zone
 * @returns             the name of the specified zone.
 */
const char     *zone_name(
        zone_t                  zone);

/**!
 * @function zone_heap_name
 *
 * @param zone          the specified zone
 * @returns             the name of the heap this zone is part of, or "".
 */
const char     *zone_heap_name(
        zone_t                  zone);

/**!
 * @function zone_submap
 *
 * @param zone          the specified zone
 * @returns             the zone (sub)map this zone allocates from.
 */
extern vm_map_t zone_submap(
        zone_t                  zone);

/*!
 * @function zone_create_ext
 *
 * @abstract
 * Creates a zone with the specified parameters.
 *
 * @discussion
 * This is an extended version of @c zone_create().
 *
 * @param name          the name for the new zone.
 * @param size          the size of the elements returned by this zone.
 * @param flags         a set of @c zone_create_flags_t flags.
 * @param desired_zid   a @c zone_reserved_id_t value or @c ZONE_ID_ANY.
 *
 * @param extra_setup   a block that can perform non trivial initialization
 *                      on the zone before it is marked valid.
 *                      This block can call advanced setups like:
 *                      - zone_set_submap_idx()
 *                      - zone_set_exhaustible()
 *                      - zone_set_noexpand()
 *
 * @returns             the created zone, this call never fails.
 */
extern zone_t   zone_create_ext(
        const char             *name,
        vm_size_t               size,
        zone_create_flags_t     flags,
        zone_id_t               desired_zid,
        void                  (^extra_setup)(zone_t));

/*!
 * @macro ZONE_DECLARE
 *
 * @abstract
 * Declares a zone variable to automatically initialize with the specified
 * parameters.
 *
 * @param var           the name of the variable to declare.
 * @param name          the name for the zone
 * @param size          the size of the elements returned by this zone.
 * @param flags         a set of @c zone_create_flags_t flags.
 */
#define ZONE_DECLARE(var, name, size, flags) \
        SECURITY_READ_ONLY_LATE(zone_t) var; \
        static_assert(((flags) & ZC_DESTRUCTIBLE) == 0); \
        static __startup_data struct zone_create_startup_spec \
        __startup_zone_spec_ ## var = { &var, name, size, flags, \
            ZONE_ID_ANY, NULL }; \
        STARTUP_ARG(ZALLOC, STARTUP_RANK_MIDDLE, zone_create_startup, \
            &__startup_zone_spec_ ## var)

/*!
 * @macro ZONE_INIT
 *
 * @abstract
 * Initializes a given zone automatically during startup with the specified
 * parameters.
 *
 * @param var           the name of the variable to initialize.
 * @param name          the name for the zone
 * @param size          the size of the elements returned by this zone.
 * @param flags         a set of @c zone_create_flags_t flags.
 * @param desired_zid   a @c zone_reserved_id_t value or @c ZONE_ID_ANY.
 * @param extra_setup   a block that can perform non trivial initialization
 *                      (@see @c zone_create_ext()).
 */
#define ZONE_INIT(var, name, size, flags, desired_zid, extra_setup) \
        __ZONE_INIT(__LINE__, var, name, size, flags, desired_zid, extra_setup)

/*!
 * @function zone_id_require
 *
 * @abstract
 * Requires for a given pointer to belong to the specified zone, by ID and size.
 *
 * @discussion
 * The function panics if the check fails as it indicates that the kernel
 * internals have been compromised.
 *
 * This is a variant of @c zone_require() which:
 * - isn't sensitive to @c zone_t::elem_size being compromised,
 * - is slightly faster as it saves one load and a multiplication.
 *
 * @param zone_id       the zone ID the address needs to belong to.
 * @param elem_size     the size of elements for this zone.
 * @param addr          the element address to check.
 */
extern void     zone_id_require(
        zone_id_t               zone_id,
        vm_size_t               elem_size,
        void                   *addr);

/*
 * Zone submap indices
 *
 * Z_SUBMAP_IDX_VA_RESTRICTED_MAP (LP64)
 * used to restrict VM allocations lower in the kernel VA space,
 * for pointer packing
 *
 * Z_SUBMAP_IDX_GENERAL_MAP
 * used for unrestricted allocations
 *
 * Z_SUBMAP_IDX_BAG_OF_BYTES_MAP
 * used to sequester bags of bytes from all other allocations and allow VA reuse
 * within the map
 */
#if !defined(__LP64__)
#define Z_SUBMAP_IDX_GENERAL_MAP        0
#define Z_SUBMAP_IDX_BAG_OF_BYTES_MAP   1
#define Z_SUBMAP_IDX_COUNT              2
#else
#define Z_SUBMAP_IDX_VA_RESTRICTED_MAP  0
#define Z_SUBMAP_IDX_GENERAL_MAP        1
#define Z_SUBMAP_IDX_BAG_OF_BYTES_MAP   2
#define Z_SUBMAP_IDX_COUNT              3
#endif

/* Change zone sub-map, to be called from the zone_create_ext() setup hook */
extern void     zone_set_submap_idx(
        zone_t          zone,
        unsigned int    submap_idx);

/* Make zone as non expandable, to be called from the zone_create_ext() setup hook */
extern void     zone_set_noexpand(
        zone_t          zone,
        vm_size_t       maxsize);

/* Make zone exhaustible, to be called from the zone_create_ext() setup hook */
extern void     zone_set_exhaustible(
        zone_t          zone,
        vm_size_t       maxsize);

/* Initially fill zone with specified number of elements */
extern int      zfill(
        zone_t          zone,
        int             nelem);

/* Fill zone with memory */
extern void     zcram(
        zone_t          zone,
        vm_offset_t     newmem,
        vm_size_t       size);

#pragma mark XNU only: misc & implementation details

/*
 * This macro sets "elem" to NULL on free.
 *
 * Note: all values passed to zfree() might be in the element to be freed,
 *       temporaries must be taken, and the resetting to be done prior to free.
 */
#define zfree(zone, elem) ({ \
        _Static_assert(sizeof(elem) == sizeof(void *), "elem isn't pointer sized"); \
        __auto_type __zfree_zone = (zone); \
        __auto_type __zfree_eptr = &(elem); \
        __auto_type __zfree_elem = *__zfree_eptr; \
        *__zfree_eptr = (__typeof__(__zfree_elem))NULL; \
        (zfree)(__zfree_zone, (void *)__zfree_elem); \
})

struct zone_create_startup_spec {
        zone_t                 *z_var;
        const char             *z_name;
        vm_size_t               z_size;
        zone_create_flags_t     z_flags;
        zone_id_t               z_zid;
        void                  (^z_setup)(zone_t);
};

extern void     zone_create_startup(
        struct zone_create_startup_spec *spec);

#define __ZONE_INIT1(ns, var, name, size, flags, zid, setup) \
        static __startup_data struct zone_create_startup_spec \
        __startup_zone_spec_ ## ns = { var, name, size, flags, zid, setup }; \
        STARTUP_ARG(ZALLOC, STARTUP_RANK_MIDDLE, zone_create_startup, \
            &__startup_zone_spec_ ## ns)

#define __ZONE_INIT(ns, var, name, size, flags, zid, setup) \
        __ZONE_INIT1(ns, var, name, size, flags, zid, setup) \

struct zone_view_startup_spec {
        zone_view_t         zv_view;
        union {
                zone_kheap_id_t zv_heapid;
                zone_t          zv_zone;
        };
        vm_size_t           zv_size;
};

extern void zone_view_startup_init(
        struct zone_view_startup_spec *spec);


#if DEBUG || DEVELOPMENT
#  if __LP64__
#    define ZPCPU_MANGLE_BIT    (1ul << 63)
#  else /* !__LP64__ */
#    define ZPCPU_MANGLE_BIT    (1ul << 31)
#  endif /* !__LP64__ */
#else /* !(DEBUG || DEVELOPMENT) */
#  define ZPCPU_MANGLE_BIT      0ul
#endif /* !(DEBUG || DEVELOPMENT) */

#define __zpcpu_mangle(ptr)     (__zpcpu_addr(ptr) & ~ZPCPU_MANGLE_BIT)
#define __zpcpu_demangle(ptr)   (__zpcpu_addr(ptr) | ZPCPU_MANGLE_BIT)
#define __zpcpu_addr(e)         ((vm_address_t)(e))
#define __zpcpu_cast(ptr, e)    ((typeof(ptr))(e))
#define __zpcpu_next(ptr)       __zpcpu_cast(ptr, __zpcpu_addr(ptr) + PAGE_SIZE)

extern unsigned zpercpu_count(void) __pure2;


/* These functions used for leak detection both in zalloc.c and mbuf.c */
extern uintptr_t hash_mix(uintptr_t);
extern uint32_t hashbacktrace(uintptr_t *, uint32_t, uint32_t);
extern uint32_t hashaddr(uintptr_t, uint32_t);

#if CONFIG_ZLEAKS
/* support for the kern.zleak.* sysctls */

extern kern_return_t zleak_activate(void);
extern vm_size_t zleak_max_zonemap_size;
extern vm_size_t zleak_global_tracking_threshold;
extern vm_size_t zleak_per_zone_tracking_threshold;

extern int get_zleak_state(void);

#endif /* CONFIG_ZLEAKS */
#if DEBUG || DEVELOPMENT

extern boolean_t run_zone_test(void);
extern void zone_gc_replenish_test(void);
extern void zone_alloc_replenish_test(void);

#endif /* DEBUG || DEVELOPMENT */

#pragma GCC visibility pop
#endif /* XNU_KERNEL_PRIVATE */

__END_DECLS

#endif  /* _KERN_ZALLOC_H_ */

#endif  /* KERNEL_PRIVATE */