[apple/xnu.git] / osfmk / kern / counter.h

/*
 * Copyright (c) 2020 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
#ifdef XNU_KERNEL_PRIVATE

#ifndef _KERN_COUNTER_H
#define _KERN_COUNTER_H

/*!
 * @file <kern/counter.h>
 *
 * @brief
 * Module for working with 64bit relaxed atomic counters.
 *
 * @discussion
 * Different counter types have different speed-memory tradeoffs, but
 * they all share a common interface.
 *
 * Counters can be statically allocated or dynamically allocated.
 *
 * Statically allocated counters are always backed by per-cpu storage which means
 * writes take place on the current CPUs value and reads sum all of the per-cpu values.
 *
 * Dynamically allocated counters can be either per-cpu or use a single 64bit value.
 * To create a per-cpu counter, use the scalable_counter_t type. Note that this
 * trades of additional memory for better scalability.
 * To create a single 64bit counter, use the atomic_counter_t type.
 *
 * For most counters you can just use the counter_t type and the choice of
 * scalable or atomic will be made at compile time based on the target.
 *
 * The counter types are opaque handles. They ARE NOT COPYABLE. If you need
 * to make a copy of a counter, you should do so like this:
 * <code>
 * counter_t original;
 * ...
 * counter_t copy;
 * counter_alloc(&copy);
 * counter_add(&copy, counter_load(&original));
 * ...
 * // Make sure to free them at some point.
 * counter_free(&original);
 * counter_free(&copy);
 * </code>
 *
 * Static counter example:
 * <code>
 * SCALABLE_COUNTER_DEFINE(my_counter);
 * ...
 * counter_inc(&my_counter);
 * assert(counter_load(&my_counter) == 1);
 * </code>
 *
 * Dynamic Counter Example:
 * <code>
 * scalable_counter_t my_percpu_counter;
 * atomic_counter_t my_atomic_counter;
 * counter_t my_counter;
 *
 * // All three counters share the same interface. So to change the speed-memory
 * // tradeoff just change the type.
 * counter_init(&my_scalable_counter);
 * counter_init(&my_atomic_counter);
 * counter_init(&my_counter);
 *
 * counter_inc(&my_scalable_counter);
 * counter_inc(&my_atomic_counter);
 * counter_inc(&my_counter);
 *
 * assert(counter_load(&my_scalable_counter) == 1);
 * assert(counter_load(&my_atomic_counter) == 1);
 * assert(counter_load(&my_counter) == 1);
 * </code>
 */

#include <mach/mach_types.h>
#include <kern/macro_help.h>
#include <kern/startup.h>
#include <kern/zalloc.h>

typedef __zpercpu uint64_t *scalable_counter_t;
typedef uint64_t atomic_counter_t;
/* Generic counter base type. Does not have an implementation. */
struct generic_counter_t;

/*!
 * @macro SCALABLE_COUNTER_DECLARE
 *
 * @abstract
 * (optionally) declares a static per-cpu counter (in a header).
 *
 * @param var           the name of the counter.
 */
#define SCALABLE_COUNTER_DECLARE(name) \
	extern scalable_counter_t name;

/*!
 * @macro SCALABLE_COUNTER_DEFINE
 *
 * @abstract
 * Defines a static per-cpu counter.
 * Counter can only be accessed after the TUNABLES phase of startup.
 *
 * @param var           the name of the counter.
 */
#define SCALABLE_COUNTER_DEFINE(name) \
	__startup_data uint64_t __ ##name##_early_storage = 0;                                   \
	scalable_counter_t name = {&__##name##_early_storage};                                   \
	STARTUP_ARG(TUNABLES, STARTUP_RANK_MIDDLE, scalable_counter_static_boot_mangle, &name);  \
	STARTUP_ARG(PERCPU, STARTUP_RANK_SECOND, scalable_counter_static_init, &name);

/*
 * Initialize a per-cpu counter.
 * May block and will never fail.
 * This counter must be freed with counter_free.
 */
OS_OVERLOADABLE
extern void counter_alloc(struct generic_counter_t *);

OS_OVERLOADABLE
extern void counter_free(struct generic_counter_t *);
/*
 * Add amount to counter.
 * @param amount: The amount to add.
 */
OS_OVERLOADABLE
extern void counter_add(struct generic_counter_t *, uint64_t amount);

/*
 * Add 1 to this counter.
 */
OS_OVERLOADABLE
extern void counter_inc(struct generic_counter_t *);

/*
 * Subtract 1 from this counter.
 */
OS_OVERLOADABLE
extern void counter_dec(struct generic_counter_t *);

/* Variants of the above operations where the caller takes responsibility for disabling preemption. */
OS_OVERLOADABLE
extern void counter_add_preemption_disabled(struct generic_counter_t *, uint64_t amount);
OS_OVERLOADABLE
extern void counter_inc_preemption_disabled(struct generic_counter_t *);
OS_OVERLOADABLE
extern void counter_dec_preemption_disabled(struct generic_counter_t *);

/*
 * Read the value of the percpu counter.
 * Note that this will cause synchronization of all the sharded values.
 */
OS_OVERLOADABLE
extern uint64_t counter_load(struct generic_counter_t *);

#pragma mark implementation details
/* NB: Nothing below here should be used directly. */

__startup_func void scalable_counter_static_boot_mangle(scalable_counter_t *counter);
__startup_func void scalable_counter_static_init(scalable_counter_t *counter);

#if XNU_TARGET_OS_WATCH || XNU_TARGET_OS_TV
#define ATOMIC_COUNTER_USE_PERCPU 0
#else
#define ATOMIC_COUNTER_USE_PERCPU 1
#endif /* XNU_TARGET_OS_OSX */

#if ATOMIC_COUNTER_USE_PERCPU
typedef scalable_counter_t counter_t;
#else
typedef atomic_counter_t counter_t;
#endif /* ATOMIC_COUNTER_USE_PERCPU */

#define COUNTER_MAKE_PROTOTYPES(counter_t)                                 \
OS_OVERLOADABLE                                                            \
extern void counter_alloc(counter_t *);                                    \
                                                                           \
OS_OVERLOADABLE                                                            \
extern void counter_free(counter_t *);                                     \
                                                                           \
OS_OVERLOADABLE                                                            \
extern void counter_add(counter_t *, uint64_t amount);                     \
                                                                           \
OS_OVERLOADABLE                                                            \
extern void counter_inc(counter_t *);                                      \
                                                                           \
OS_OVERLOADABLE                                                            \
extern void counter_dec(counter_t *);                                      \
                                                                           \
OS_OVERLOADABLE                                                            \
extern void counter_add_preemption_disabled(counter_t *, uint64_t amount); \
                                                                           \
OS_OVERLOADABLE                                                            \
extern void counter_inc_preemption_disabled(counter_t *);                  \
                                                                           \
OS_OVERLOADABLE                                                            \
extern void counter_dec_preemption_disabled(counter_t *);                  \
                                                                           \
OS_OVERLOADABLE                                                            \
extern uint64_t counter_load(counter_t *);

COUNTER_MAKE_PROTOTYPES(scalable_counter_t);
COUNTER_MAKE_PROTOTYPES(atomic_counter_t);

#endif /* _KERN_COUNTER_H */

#endif /* XNU_KERNEL_PRIVATE */
Commit	Line	Data
c3c9b80d A	1	/*
	2	* Copyright (c) 2020 Apple Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28	#ifdef XNU_KERNEL_PRIVATE
	29
	30	#ifndef _KERN_COUNTER_H
	31	#define _KERN_COUNTER_H
	32
	33	/*!
	34	* @file <kern/counter.h>
	35	*
	36	* @brief
	37	* Module for working with 64bit relaxed atomic counters.
	38	*
	39	* @discussion
	40	* Different counter types have different speed-memory tradeoffs, but
	41	* they all share a common interface.
	42	*
	43	* Counters can be statically allocated or dynamically allocated.
	44	*
	45	* Statically allocated counters are always backed by per-cpu storage which means
	46	* writes take place on the current CPUs value and reads sum all of the per-cpu values.
	47	*
	48	* Dynamically allocated counters can be either per-cpu or use a single 64bit value.
	49	* To create a per-cpu counter, use the scalable_counter_t type. Note that this
	50	* trades of additional memory for better scalability.
	51	* To create a single 64bit counter, use the atomic_counter_t type.
	52	*
	53	* For most counters you can just use the counter_t type and the choice of
	54	* scalable or atomic will be made at compile time based on the target.
	55	*
	56	* The counter types are opaque handles. They ARE NOT COPYABLE. If you need
	57	* to make a copy of a counter, you should do so like this:
	58	* <code>
	59	* counter_t original;
	60	* ...
	61	* counter_t copy;
	62	* counter_alloc(&copy);
	63	* counter_add(&copy, counter_load(&original));
	64	* ...
65	* // Make sure to free them at some point.
66	* counter_free(&original);
67	* counter_free(&copy);
68	* </code>
69	*
70	* Static counter example:
71	* <code>
72	* SCALABLE_COUNTER_DEFINE(my_counter);
73	* ...
74	* counter_inc(&my_counter);
75	* assert(counter_load(&my_counter) == 1);
76	* </code>
77	*
78	* Dynamic Counter Example:
79	* <code>
80	* scalable_counter_t my_percpu_counter;
81	* atomic_counter_t my_atomic_counter;
82	* counter_t my_counter;
83	*
84	* // All three counters share the same interface. So to change the speed-memory
85	* // tradeoff just change the type.
86	* counter_init(&my_scalable_counter);
87	* counter_init(&my_atomic_counter);
88	* counter_init(&my_counter);
89	*
90	* counter_inc(&my_scalable_counter);
91	* counter_inc(&my_atomic_counter);
92	* counter_inc(&my_counter);
93	*
94	* assert(counter_load(&my_scalable_counter) == 1);
95	* assert(counter_load(&my_atomic_counter) == 1);
96	* assert(counter_load(&my_counter) == 1);
97	* </code>
98	*/
99
100	#include <mach/mach_types.h>
101	#include <kern/macro_help.h>
102	#include <kern/startup.h>
103	#include <kern/zalloc.h>
104
105	typedef __zpercpu uint64_t *scalable_counter_t;
106	typedef uint64_t atomic_counter_t;
107	/* Generic counter base type. Does not have an implementation. */
108	struct generic_counter_t;
109
110	/*!
111	* @macro SCALABLE_COUNTER_DECLARE
112	*
113	* @abstract
114	* (optionally) declares a static per-cpu counter (in a header).
115	*
116	* @param var the name of the counter.
117	*/
118	#define SCALABLE_COUNTER_DECLARE(name) \
119	extern scalable_counter_t name;
120
121	/*!
122	* @macro SCALABLE_COUNTER_DEFINE
123	*
124	* @abstract
125	* Defines a static per-cpu counter.
126	* Counter can only be accessed after the TUNABLES phase of startup.
127	*
128	* @param var the name of the counter.
129	*/
130	#define SCALABLE_COUNTER_DEFINE(name) \
131	__startup_data uint64_t __ ##name##_early_storage = 0; \
132	scalable_counter_t name = {&__##name##_early_storage}; \
133	STARTUP_ARG(TUNABLES, STARTUP_RANK_MIDDLE, scalable_counter_static_boot_mangle, &name); \
134	STARTUP_ARG(PERCPU, STARTUP_RANK_SECOND, scalable_counter_static_init, &name);
135
136	/*
137	* Initialize a per-cpu counter.
138	* May block and will never fail.
139	* This counter must be freed with counter_free.
140	*/
141	OS_OVERLOADABLE
142	extern void counter_alloc(struct generic_counter_t *);
143
144	OS_OVERLOADABLE
145	extern void counter_free(struct generic_counter_t *);
146	/*
147	* Add amount to counter.
148	* @param amount: The amount to add.
149	*/
150	OS_OVERLOADABLE
151	extern void counter_add(struct generic_counter_t *, uint64_t amount);
152
153	/*
154	* Add 1 to this counter.
155	*/
156	OS_OVERLOADABLE
157	extern void counter_inc(struct generic_counter_t *);
158
159	/*
160	* Subtract 1 from this counter.
161	*/
162	OS_OVERLOADABLE
163	extern void counter_dec(struct generic_counter_t *);
164
165	/* Variants of the above operations where the caller takes responsibility for disabling preemption. */
166	OS_OVERLOADABLE
167	extern void counter_add_preemption_disabled(struct generic_counter_t *, uint64_t amount);
168	OS_OVERLOADABLE
169	extern void counter_inc_preemption_disabled(struct generic_counter_t *);
170	OS_OVERLOADABLE
171	extern void counter_dec_preemption_disabled(struct generic_counter_t *);
172
173	/*
174	* Read the value of the percpu counter.
175	* Note that this will cause synchronization of all the sharded values.
176	*/
177	OS_OVERLOADABLE
178	extern uint64_t counter_load(struct generic_counter_t *);
179
180	#pragma mark implementation details
181	/* NB: Nothing below here should be used directly. */
182
183	__startup_func void scalable_counter_static_boot_mangle(scalable_counter_t *counter);
184	__startup_func void scalable_counter_static_init(scalable_counter_t *counter);
185
186	#if XNU_TARGET_OS_WATCH \|\| XNU_TARGET_OS_TV
187	#define ATOMIC_COUNTER_USE_PERCPU 0
188	#else
189	#define ATOMIC_COUNTER_USE_PERCPU 1
190	#endif /* XNU_TARGET_OS_OSX */
191
192	#if ATOMIC_COUNTER_USE_PERCPU
193	typedef scalable_counter_t counter_t;
194	#else
195	typedef atomic_counter_t counter_t;
196	#endif /* ATOMIC_COUNTER_USE_PERCPU */
197
198	#define COUNTER_MAKE_PROTOTYPES(counter_t) \
199	OS_OVERLOADABLE \
200	extern void counter_alloc(counter_t *); \
201	\
202	OS_OVERLOADABLE \
203	extern void counter_free(counter_t *); \
204	\
205	OS_OVERLOADABLE \
206	extern void counter_add(counter_t *, uint64_t amount); \
207	\
208	OS_OVERLOADABLE \
209	extern void counter_inc(counter_t *); \
210	\
211	OS_OVERLOADABLE \
212	extern void counter_dec(counter_t *); \
213	\
214	OS_OVERLOADABLE \
215	extern void counter_add_preemption_disabled(counter_t *, uint64_t amount); \
216	\
217	OS_OVERLOADABLE \
218	extern void counter_inc_preemption_disabled(counter_t *); \
219	\
220	OS_OVERLOADABLE \
221	extern void counter_dec_preemption_disabled(counter_t *); \
222	\
223	OS_OVERLOADABLE \
224	extern uint64_t counter_load(counter_t *);
225
226	COUNTER_MAKE_PROTOTYPES(scalable_counter_t);
227	COUNTER_MAKE_PROTOTYPES(atomic_counter_t);
228
229	#endif /* _KERN_COUNTER_H */
230
231	#endif /* XNU_KERNEL_PRIVATE */