xnu-7195.60.75.tar.gz

[apple/xnu.git] / osfmk / kern / percpu.h

/*
 * Copyright (c) 2020 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#ifndef _KERN_PERCPU_H_
#define _KERN_PERCPU_H_

#include <mach/vm_types.h>

__BEGIN_DECLS

#if XNU_KERNEL_PRIVATE
#include <libkern/section_keywords.h>
#include <os/atomic_private.h>

#pragma GCC visibility push(hidden)

/*!
 * @macro PERCPU_DECL
 *
 * @abstract
 * Declares a per-CPU variable in a header.
 *
 * @param type_t        the per-CPU variable type
 * @param name          the per-CPU variable name
 */
#define PERCPU_DECL(type_t, name) \
	extern type_t __PERCPU_NAME(name)

/*!
 * @macro PERCPU_DATA
 *
 * @abstract
 * Defines a per-CPU variable in a translation unit.
 *
 * @discussion
 * @c PERCPU_DECL can be used in headers to export the variable to clients.
 *
 * By default, per-cpu data is 0-initialized. Per-CPU data is allocated during
 * the STARTUP_SUB_KMEM_ALLOC phase and can be initialized with a STARTUP
 * callback in any later phase.
 *
 * Usage is:
 * <code>
 *   [ static ] type PERCPU_DATA(name);
 * </code>
 *
 * @param name          the per-CPU variable name
 */
#define PERCPU_DATA(name) \
	__percpu __PERCPU_NAME(name) = {0}

/*!
 * @macro PERCPU_GET
 *
 * @abstract
 * Gets a pointer to the per-CPU instance of the variable for the processor the
 * code is currently running on.
 *
 * @discussion
 * It is expected that preemption or interrupts are disabled when this is used,
 * as a context-switch might move the current thread to another CPU.
 *
 * It is also valid in code that wasn't already disabling preemption and cares
 * about code-gen size a lot to use this outside of a preemption-disabled
 * section provided that the data is modified using atomics.
 *
 * Note that if several per-CPU pointers are acquired in short succession,
 * @c PERCPU_GET_WITH_BASE can be used to avoid the repeated calls to
 * @c current_percpu_base() which the compiler wont't elide.
 *
 * @param name          the per-CPU variable name
 */
#define PERCPU_GET(name) \
	__PERCPU_CAST(name, current_percpu_base() + __PERCPU_ADDR(name))

/*!
 * @function current_percpu_base()
 *
 * @abstract
 * Returns an offset that can be passed to @c PERCPU_GET_WITH_BASE().
 *
 * @see PERCPU_GET() for conditions of use.
 */
extern vm_offset_t current_percpu_base(void);

/*!
 * @macro PERCPU_GET_MASTER
 *
 * @abstract
 * Gets a pointer to the master per-CPU instance of the variable.
 *
 * @param base          the per-CPU base to use
 * @param name          the per-CPU variable name
 */
#define PERCPU_GET_MASTER(name) \
	(&__PERCPU_NAME(name))

/*!
 * @macro PERCPU_GET_WITH_BASE
 *
 * @abstract
 * Gets a pointer to the per-CPU instance of the variable for the specified
 * base.
 *
 * @param base          the per-CPU base to use
 * @param name          the per-CPU variable name
 */
#define PERCPU_GET_WITH_BASE(base, name) \
	__PERCPU_CAST(name, base + __PERCPU_ADDR(name))

/*!
 * @macro PERCPU_GET_RELATIVE
 *
 * @abstract
 * Gets a pointer to the per-CPU instance of a variable relative to another
 * known one.
 *
 * @description
 * When a per-CPU slot address is known, but the caller doesn't know the base
 * from which it was derived, then this allows to compute another per-CPU slot
 * address for a different variable but for the same CPU, without any loads.
 *
 * @param name          the per-CPU variable name
 * @param other         the other per-CPU variable name
 * @param ptr           a pointer to the other variable slot
 */
#define PERCPU_GET_RELATIVE(name, other, ptr) ({ \
	__PERCPU_TYPE(other) __other_ptr = (ptr); /* type check */ \
	vm_offset_t __offs = __PERCPU_ADDR(name) - __PERCPU_ADDR(other); \
	__PERCPU_CAST(name, (vm_address_t)__other_ptr + __offs); \
})

/*!
 * @macro percpu_foreach_base()
 *
 * @abstract
 * Enumerates all Per-CPU variable bases.
 *
 * @param it            the name of the iterator
 */
#define percpu_foreach_base(it) \
	for (vm_offset_t it = 0, \
	    __next_ ## it = percpu_base.start, \
	    __end_ ## it = percpu_base.end; \
        \
	    it <= __end_ ## it; \
        \
	    it = __next_ ## it, \
	    __next_ ## it += percpu_section_size())

/*!
 * @macro percpu_foreach()
 *
 * @abstract
 * Enumerates all Per-CPU variable instances.
 *
 * @param it            the name of the iterator
 * @param name          the per-CPU variable name
 */
#define percpu_foreach(it, name) \
	for (__PERCPU_TYPE(name) it, \
	    __base_ ## it = NULL, \
	    __next_ ## it = (typeof(it))percpu_base.start, \
	    __end_ ## it = (typeof(it))percpu_base.end; \
        \
	    (it = (typeof(it))(__PERCPU_ADDR(name) + (vm_address_t)__base_ ## it), \
	    __base_ ## it <= __end_ ## it); \
        \
	    __base_ ## it = __next_ ## it, \
	    __next_ ## it = (typeof(it))((vm_address_t)__base_ ## it + percpu_section_size()))

/*!
 * @macro percpu_foreach_secondary_base()
 *
 * @abstract
 * Enumerates all Per-CPU variable bases, skipping the master slot.
 *
 * @param it            the name of the iterator
 */
#define percpu_foreach_secondary_base(it) \
	for (vm_offset_t it = percpu_base.start, __end_ ## it = percpu_base.end; \
	    it <= __end_ ## it; it += percpu_section_size())

/*!
 * @macro percpu_foreach_secondary()
 *
 * @abstract
 * Enumerates all Per-CPU variable instances, skipping the master slot.
 *
 * @param it            the name of the iterator
 * @param name          the per-CPU variable name
 */
#define percpu_foreach_secondary(it, name) \
	for (__PERCPU_TYPE(name) it, \
	    __base_ ## it = (typeof(it))percpu_base.start, \
	    __end_ ## it = (typeof(it))percpu_base.end; \
        \
	    (it = (typeof(it))(__PERCPU_ADDR(name) + (vm_address_t)__base_ ## it), \
	    __base_ ## it <= __end_ ## it); \
        \
	    __base_ ## it = (typeof(it))((vm_address_t)__base_ ## it + percpu_section_size()))

#pragma mark - implementation details

/*
 * Below this point are implementation details that should not be used directly,
 * except by the macros above, or architecture specific code.
 */

#define __percpu                        __attribute__((section("__DATA, __percpu")))
#define __PERCPU_NAME(name)             percpu_slot_ ## name
#define __PERCPU_ADDR(name)             ((vm_offset_t)&__PERCPU_NAME(name))
#define __PERCPU_TYPE(name)             typeof(&__PERCPU_NAME(name))
#define __PERCPU_CAST(name, expr)       ((__PERCPU_TYPE(name))(expr))

/*
 * Note for implementors:
 *
 * A `base` represents a pointer in the percpu allocation offset by
 * `percpu_section_start()` so that PERCPU_GET() is a single addition.
 *
 * percpu_base.end is inclusive, so that percpu_foreach() and
 * percpu_foreach_base() can do a `<=` comparison.
 *
 * Because the first base is `0` (because the master CPU is using the static
 * percpu section), it allows for the compiler to know that for the first
 * iteration the comparison is always true.
 */
extern struct percpu_base {
	vm_address_t start;
	vm_address_t end;
	vm_offset_t  size;
} percpu_base;

static __pure2 inline vm_offset_t
percpu_section_start(void)
{
	extern char __percpu_section_start[] __SECTION_START_SYM("__DATA", "__percpu");
	return (vm_offset_t)__percpu_section_start;
}

static __pure2 inline vm_offset_t
percpu_section_end(void)
{
	extern char __percpu_section_end[] __SECTION_END_SYM("__DATA", "__percpu");
	return (vm_offset_t)__percpu_section_end;
}

static __pure2 inline vm_size_t
percpu_section_size(void)
{
	return percpu_section_end() - percpu_section_start();
}

#pragma GCC visibility pop
#endif /* XNU_KERNEL_PRIVATE */

__END_DECLS

#endif /* _KERN_PERCPU_H_ */