osfmk/kern/percpu.h

   1 /*
   2  * Copyright (c) 2020 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #ifndef _KERN_PERCPU_H_
  30 #define _KERN_PERCPU_H_
  31
  32 #include <mach/vm_types.h>
  33
  34 __BEGIN_DECLS
  35
  36 #if XNU_KERNEL_PRIVATE
  37 #include <libkern/section_keywords.h>
  38 #include <os/atomic_private.h>
  39
  40 #pragma GCC visibility push(hidden)
  41
  42 /*!
  43  * @macro PERCPU_DECL
  44  *
  45  * @abstract
  46  * Declares a per-CPU variable in a header.
  47  *
  48  * @param type_t        the per-CPU variable type
  49  * @param name          the per-CPU variable name
  50  */
  51 #define PERCPU_DECL(type_t, name) \
  52         extern type_t __PERCPU_NAME(name)
  53
  54 /*!
  55  * @macro PERCPU_DATA
  56  *
  57  * @abstract
  58  * Defines a per-CPU variable in a translation unit.
  59  *
  60  * @discussion
  61  * @c PERCPU_DECL can be used in headers to export the variable to clients.
  62  *
  63  * By default, per-cpu data is 0-initialized. Per-CPU data is allocated during
  64  * the STARTUP_SUB_KMEM_ALLOC phase and can be initialized with a STARTUP
  65  * callback in any later phase.
  66  *
  67  * Usage is:
  68  * <code>
  69  *   [ static ] type PERCPU_DATA(name);
  70  * </code>
  71  *
  72  * @param name          the per-CPU variable name
  73  */
  74 #define PERCPU_DATA(name) \
  75         __percpu __PERCPU_NAME(name) = {0}
  76
  77 /*!
  78  * @macro PERCPU_GET
  79  *
  80  * @abstract
  81  * Gets a pointer to the per-CPU instance of the variable for the processor the
  82  * code is currently running on.
  83  *
  84  * @discussion
  85  * It is expected that preemption or interrupts are disabled when this is used,
  86  * as a context-switch might move the current thread to another CPU.
  87  *
  88  * It is also valid in code that wasn't already disabling preemption and cares
  89  * about code-gen size a lot to use this outside of a preemption-disabled
  90  * section provided that the data is modified using atomics.
  91  *
  92  * Note that if several per-CPU pointers are acquired in short succession,
  93  * @c PERCPU_GET_WITH_BASE can be used to avoid the repeated calls to
  94  * @c current_percpu_base() which the compiler wont't elide.
  95  *
  96  * @param name          the per-CPU variable name
  97  */
  98 #define PERCPU_GET(name) \
  99         __PERCPU_CAST(name, current_percpu_base() + __PERCPU_ADDR(name))
 100
 101 /*!
 102  * @function current_percpu_base()
 103  *
 104  * @abstract
 105  * Returns an offset that can be passed to @c PERCPU_GET_WITH_BASE().
 106  *
 107  * @see PERCPU_GET() for conditions of use.
 108  */
 109 extern vm_offset_t current_percpu_base(void);
 110
 111 /*!
 112  * @macro PERCPU_GET_MASTER
 113  *
 114  * @abstract
 115  * Gets a pointer to the master per-CPU instance of the variable.
 116  *
 117  * @param base          the per-CPU base to use
 118  * @param name          the per-CPU variable name
 119  */
 120 #define PERCPU_GET_MASTER(name) \
 121         (&__PERCPU_NAME(name))
 122
 123 /*!
 124  * @macro PERCPU_GET_WITH_BASE
 125  *
 126  * @abstract
 127  * Gets a pointer to the per-CPU instance of the variable for the specified
 128  * base.
 129  *
 130  * @param base          the per-CPU base to use
 131  * @param name          the per-CPU variable name
 132  */
 133 #define PERCPU_GET_WITH_BASE(base, name) \
 134         __PERCPU_CAST(name, base + __PERCPU_ADDR(name))
 135
 136 /*!
 137  * @macro PERCPU_GET_RELATIVE
 138  *
 139  * @abstract
 140  * Gets a pointer to the per-CPU instance of a variable relative to another
 141  * known one.
 142  *
 143  * @description
 144  * When a per-CPU slot address is known, but the caller doesn't know the base
 145  * from which it was derived, then this allows to compute another per-CPU slot
 146  * address for a different variable but for the same CPU, without any loads.
 147  *
 148  * @param name          the per-CPU variable name
 149  * @param other         the other per-CPU variable name
 150  * @param ptr           a pointer to the other variable slot
 151  */
 152 #define PERCPU_GET_RELATIVE(name, other, ptr) ({ \
 153         __PERCPU_TYPE(other) __other_ptr = (ptr); /* type check */ \
 154         vm_offset_t __offs = __PERCPU_ADDR(name) - __PERCPU_ADDR(other); \
 155         __PERCPU_CAST(name, (vm_address_t)__other_ptr + __offs); \
 156 })
 157
 158 /*!
 159  * @macro percpu_foreach_base()
 160  *
 161  * @abstract
 162  * Enumerates all Per-CPU variable bases.
 163  *
 164  * @param it            the name of the iterator
 165  */
 166 #define percpu_foreach_base(it) \
 167         for (vm_offset_t it = 0, \
 168             __next_ ## it = percpu_base.start, \
 169             __end_ ## it = percpu_base.end; \
 170         \
 171             it <= __end_ ## it; \
 172         \
 173             it = __next_ ## it, \
 174             __next_ ## it += percpu_section_size())
 175
 176 /*!
 177  * @macro percpu_foreach()
 178  *
 179  * @abstract
 180  * Enumerates all Per-CPU variable instances.
 181  *
 182  * @param it            the name of the iterator
 183  * @param name          the per-CPU variable name
 184  */
 185 #define percpu_foreach(it, name) \
 186         for (__PERCPU_TYPE(name) it, \
 187             __base_ ## it = NULL, \
 188             __next_ ## it = (typeof(it))percpu_base.start, \
 189             __end_ ## it = (typeof(it))percpu_base.end; \
 190         \
 191             (it = (typeof(it))(__PERCPU_ADDR(name) + (vm_address_t)__base_ ## it), \
 192             __base_ ## it <= __end_ ## it); \
 193         \
 194             __base_ ## it = __next_ ## it, \
 195             __next_ ## it = (typeof(it))((vm_address_t)__base_ ## it + percpu_section_size()))
 196
 197 /*!
 198  * @macro percpu_foreach_secondary_base()
 199  *
 200  * @abstract
 201  * Enumerates all Per-CPU variable bases, skipping the master slot.
 202  *
 203  * @param it            the name of the iterator
 204  */
 205 #define percpu_foreach_secondary_base(it) \
 206         for (vm_offset_t it = percpu_base.start, __end_ ## it = percpu_base.end; \
 207             it <= __end_ ## it; it += percpu_section_size())
 208
 209 /*!
 210  * @macro percpu_foreach_secondary()
 211  *
 212  * @abstract
 213  * Enumerates all Per-CPU variable instances, skipping the master slot.
 214  *
 215  * @param it            the name of the iterator
 216  * @param name          the per-CPU variable name
 217  */
 218 #define percpu_foreach_secondary(it, name) \
 219         for (__PERCPU_TYPE(name) it, \
 220             __base_ ## it = (typeof(it))percpu_base.start, \
 221             __end_ ## it = (typeof(it))percpu_base.end; \
 222         \
 223             (it = (typeof(it))(__PERCPU_ADDR(name) + (vm_address_t)__base_ ## it), \
 224             __base_ ## it <= __end_ ## it); \
 225         \
 226             __base_ ## it = (typeof(it))((vm_address_t)__base_ ## it + percpu_section_size()))
 227
 228 #pragma mark - implementation details
 229
 230 /*
 231  * Below this point are implementation details that should not be used directly,
 232  * except by the macros above, or architecture specific code.
 233  */
 234
 235 #define __percpu                        __attribute__((section("__DATA, __percpu")))
 236 #define __PERCPU_NAME(name)             percpu_slot_ ## name
 237 #define __PERCPU_ADDR(name)             ((vm_offset_t)&__PERCPU_NAME(name))
 238 #define __PERCPU_TYPE(name)             typeof(&__PERCPU_NAME(name))
 239 #define __PERCPU_CAST(name, expr)       ((__PERCPU_TYPE(name))(expr))
 240
 241 /*
 242  * Note for implementors:
 243  *
 244  * A `base` represents a pointer in the percpu allocation offset by
 245  * `percpu_section_start()` so that PERCPU_GET() is a single addition.
 246  *
 247  * percpu_base.end is inclusive, so that percpu_foreach() and
 248  * percpu_foreach_base() can do a `<=` comparison.
 249  *
 250  * Because the first base is `0` (because the master CPU is using the static
 251  * percpu section), it allows for the compiler to know that for the first
 252  * iteration the comparison is always true.
 253  */
 254 extern struct percpu_base {
 255         vm_address_t start;
 256         vm_address_t end;
 257         vm_offset_t  size;
 258 } percpu_base;
 259
 260 static __pure2 inline vm_offset_t
 261 percpu_section_start(void)
 262 {
 263         extern char __percpu_section_start[] __SECTION_START_SYM("__DATA", "__percpu");
 264         return (vm_offset_t)__percpu_section_start;
 265 }
 266
 267 static __pure2 inline vm_offset_t
 268 percpu_section_end(void)
 269 {
 270         extern char __percpu_section_end[] __SECTION_END_SYM("__DATA", "__percpu");
 271         return (vm_offset_t)__percpu_section_end;
 272 }
 273
 274 static __pure2 inline vm_size_t
 275 percpu_section_size(void)
 276 {
 277         return percpu_section_end() - percpu_section_start();
 278 }
 279
 280 #pragma GCC visibility pop
 281 #endif /* XNU_KERNEL_PRIVATE */
 282
 283 __END_DECLS
 284
 285 #endif /* _KERN_PERCPU_H_ */