osfmk/machine/atomic.h

   1 /*
   2  * Copyright (c) 2015-2018 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #ifndef _MACHINE_ATOMIC_H
  30 #define _MACHINE_ATOMIC_H
  31
  32 /*
  33  * Internal implementation details are in a separate header
  34  */
  35 #include <machine/atomic_impl.h>
  36
  37 /*!
  38  * @file <machine/atomic.h>
  39  *
  40  * @brief
  41  * This file defines nicer (terser and safer) wrappers for C11's <stdatomic.h>.
  42  *
  43  * @discussion
  44  * @see xnu.git::doc/atomics.md which provides more extensive documentation
  45  * about this header.
  46  *
  47  * Note that some of the macros defined in this file may be overridden by
  48  * architecture specific headers.
  49  *
  50  * All the os_atomic* functions take an operation ordering argument that can be:
  51  * - C11 memory orders: relaxed, acquire, release, acq_rel or seq_cst which
  52  *   imply a memory fence on SMP machines, and always carry the matching
  53  *   compiler barrier semantics.
  54  *
  55  * - the os_atomic-specific `dependency` memory ordering that is used to
  56  *   document intent to a carry a data or address dependency.
  57  *   See doc/atomics.md for more information.
  58  *
  59  * - a compiler barrier: compiler_acquire, compiler_release, compiler_acq_rel
  60  *   without a corresponding memory fence.
  61  */
  62
  63 /*!
  64  * @function os_compiler_barrier
  65  *
  66  * @brief
  67  * Provide a compiler barrier according to the specified ordering.
  68  *
  69  * @param m
  70  * An optional ordering among `acquire`, `release` or `acq_rel` which defaults
  71  * to `acq_rel` when not specified.
  72  * These are equivalent to the `compiler_acquire`, `compiler_release` and
  73  * `compiler_acq_rel` orderings taken by the os_atomic* functions
  74  */
  75 #define os_compiler_barrier(b...) \
  76                 atomic_signal_fence(_os_compiler_barrier_##b)
  77
  78 /*!
  79  * @function os_atomic_thread_fence
  80  *
  81  * @brief
  82  * Memory fence which is elided in non-SMP mode, but always carries the
  83  * corresponding compiler barrier.
  84  *
  85  * @param m
  86  * The ordering for this fence.
  87  */
  88 #define os_atomic_thread_fence(m)  ({ \
  89                 atomic_thread_fence(memory_order_##m##_smp); \
  90                 atomic_signal_fence(memory_order_##m); \
  91 })
  92
  93 /*!
  94  * @function os_atomic_init
  95  *
  96  * @brief
  97  * Wrapper for C11 atomic_init()
  98  *
  99  * @discussion
 100  * This initialization is not performed atomically, and so must only be used as
 101  * part of object initialization before the object is made visible to other
 102  * threads/cores.
 103  *
 104  * @param p
 105  * A pointer to an atomic variable.
 106  *
 107  * @param v
 108  * The value to initialize the variable with.
 109  *
 110  * @returns
 111  * The value loaded from @a p.
 112  */
 113 #define os_atomic_init(p, v) \
 114                 atomic_init(_os_atomic_c11_atomic(p), v)
 115
 116 /*!
 117  * @function os_atomic_load_is_plain, os_atomic_store_is_plain
 118  *
 119  * @brief
 120  * Return whether a relaxed atomic load (resp. store) to an atomic variable
 121  * is implemented as a single plain load (resp. store) instruction.
 122  *
 123  * @discussion
 124  * Non-relaxed loads/stores may involve additional memory fence instructions
 125  * or more complex atomic instructions.
 126  *
 127  * This is a construct that can safely be used in static asserts.
 128  *
 129  * @param p
 130  * A pointer to an atomic variable.
 131  *
 132  * @returns
 133  * True when relaxed atomic loads (resp. stores) compile to a plain load
 134  * (resp. store) instruction, false otherwise.
 135  */
 136 #define os_atomic_load_is_plain(p)  (sizeof(*(p)) <= sizeof(void *))
 137 #define os_atomic_store_is_plain(p) os_atomic_load_is_plain(p)
 138
 139 /*!
 140  * @function os_atomic_load
 141  *
 142  * @brief
 143  * Wrapper for C11 atomic_load_explicit(), guaranteed to compile to a single
 144  * plain load instruction (when @a m is `relaxed`).
 145  *
 146  * @param p
 147  * A pointer to an atomic variable.
 148  *
 149  * @param m
 150  * The ordering to use.
 151  *
 152  * @returns
 153  * The value loaded from @a p.
 154  */
 155 #define os_atomic_load(p, m)  ({ \
 156                 _Static_assert(os_atomic_load_is_plain(p), "Load is wide"); \
 157                 _os_atomic_basetypeof(p) _r; \
 158                 _os_compiler_barrier_before_atomic(m); \
 159                 _r = atomic_load_explicit(_os_atomic_c11_atomic(p), \
 160                                 memory_order_##m##_smp); \
 161                 _os_compiler_barrier_after_atomic(m); \
 162                 _r; \
 163 })
 164
 165 /*!
 166  * @function os_atomic_load_wide
 167  *
 168  * @brief
 169  * Wrapper for C11 atomic_load_explicit(), which may be implemented by a
 170  * compare-exchange loop for double-wide variables.
 171  *
 172  * @param p
 173  * A pointer to an atomic variable.
 174  *
 175  * @param m
 176  * The ordering to use.
 177  *
 178  * @returns
 179  * The value loaded from @a p.
 180  */
 181 #define os_atomic_load_wide(p, m)  ({ \
 182                 _os_atomic_basetypeof(p) _r; \
 183                 _os_compiler_barrier_before_atomic(m); \
 184                 _r = atomic_load_explicit(_os_atomic_c11_atomic(p), \
 185                                 memory_order_##m##_smp); \
 186                 _os_compiler_barrier_after_atomic(m); \
 187                 _r; \
 188 })
 189
 190 /*!
 191  * @function os_atomic_store
 192  *
 193  * @brief
 194  * Wrapper for C11 atomic_store_explicit(), guaranteed to compile to a single
 195  * plain store instruction (when @a m is `relaxed`).
 196  *
 197  * @param p
 198  * A pointer to an atomic variable.
 199  *
 200  * @param v
 201  * The value to store.
 202  *
 203  * @param m
 204  * The ordering to use.
 205  *
 206  * @returns
 207  * The value stored at @a p.
 208  */
 209 #define os_atomic_store(p, v, m)  ({ \
 210                 _Static_assert(os_atomic_store_is_plain(p), "Store is wide"); \
 211                 _os_atomic_basetypeof(p) _v = (v); \
 212                 _os_compiler_barrier_before_atomic(m); \
 213                 atomic_store_explicit(_os_atomic_c11_atomic(p), _v, \
 214                                 memory_order_##m##_smp); \
 215                 _os_compiler_barrier_after_atomic(m); \
 216                 _v; \
 217 })
 218
 219 /*!
 220  * @function os_atomic_store_wide
 221  *
 222  * @brief
 223  * Wrapper for C11 atomic_store_explicit(), which may be implemented by a
 224  * compare-exchange loop for double-wide variables.
 225  *
 226  * @param p
 227  * A pointer to an atomic variable.
 228  *
 229  * @param v
 230  * The value to store.
 231  *
 232  * @param m
 233  * The ordering to use.
 234  *
 235  * @returns
 236  * The value stored at @a p.
 237  */
 238 #define os_atomic_store_wide(p, v, m)  ({ \
 239                 _os_atomic_basetypeof(p) _v = (v); \
 240                 _os_compiler_barrier_before_atomic(m); \
 241                 atomic_store_explicit(_os_atomic_c11_atomic(p), _v, \
 242                                 memory_order_##m##_smp); \
 243                 _os_compiler_barrier_after_atomic(m); \
 244                 _v; \
 245 })
 246
 247 /*!
 248  * @function os_atomic_add, os_atomic_add_orig
 249  *
 250  * @brief
 251  * Wrappers for C11 atomic_fetch_add_explicit().
 252  *
 253  * @param p
 254  * A pointer to an atomic variable.
 255  *
 256  * @param v
 257  * The value to add.
 258  *
 259  * @param m
 260  * The ordering to use.
 261  *
 262  * @returns
 263  * os_atomic_add_orig returns the value of the variable before the atomic add,
 264  * os_atomic_add returns the value of the variable after the atomic add.
 265  */
 266 #define os_atomic_add_orig(p, v, m) _os_atomic_c11_op_orig(p, v, m, fetch_add)
 267 #define os_atomic_add(p, v, m)      _os_atomic_c11_op(p, v, m, fetch_add, +)
 268
 269 /*!
 270  * @function os_atomic_inc, os_atomic_inc_orig
 271  *
 272  * @brief
 273  * Perform an atomic increment.
 274  *
 275  * @param p
 276  * A pointer to an atomic variable.
 277  *
 278  * @param m
 279  * The ordering to use.
 280  *
 281  * @returns
 282  * os_atomic_inc_orig returns the value of the variable before the atomic increment,
 283  * os_atomic_inc returns the value of the variable after the atomic increment.
 284  */
 285 #define os_atomic_inc_orig(p, m)    _os_atomic_c11_op_orig(p, 1, m, fetch_add)
 286 #define os_atomic_inc(p, m)         _os_atomic_c11_op(p, 1, m, fetch_add, +)
 287
 288 /*!
 289  * @function os_atomic_sub, os_atomic_sub_orig
 290  *
 291  * @brief
 292  * Wrappers for C11 atomic_fetch_sub_explicit().
 293  *
 294  * @param p
 295  * A pointer to an atomic variable.
 296  *
 297  * @param v
 298  * The value to subtract.
 299  *
 300  * @param m
 301  * The ordering to use.
 302  *
 303  * @returns
 304  * os_atomic_sub_orig returns the value of the variable before the atomic subtract,
 305  * os_atomic_sub returns the value of the variable after the atomic subtract.
 306  */
 307 #define os_atomic_sub_orig(p, v, m) _os_atomic_c11_op_orig(p, v, m, fetch_sub)
 308 #define os_atomic_sub(p, v, m)      _os_atomic_c11_op(p, v, m, fetch_sub, -)
 309
 310 /*!
 311  * @function os_atomic_dec, os_atomic_dec_orig
 312  *
 313  * @brief
 314  * Perform an atomic decrement.
 315  *
 316  * @param p
 317  * A pointer to an atomic variable.
 318  *
 319  * @param m
 320  * The ordering to use.
 321  *
 322  * @returns
 323  * os_atomic_dec_orig returns the value of the variable before the atomic decrement,
 324  * os_atomic_dec returns the value of the variable after the atomic decrement.
 325  */
 326 #define os_atomic_dec_orig(p, m)    _os_atomic_c11_op_orig(p, 1, m, fetch_sub)
 327 #define os_atomic_dec(p, m)         _os_atomic_c11_op(p, 1, m, fetch_sub, -)
 328
 329 /*!
 330  * @function os_atomic_and, os_atomic_and_orig
 331  *
 332  * @brief
 333  * Wrappers for C11 atomic_fetch_and_explicit().
 334  *
 335  * @param p
 336  * A pointer to an atomic variable.
 337  *
 338  * @param v
 339  * The value to and.
 340  *
 341  * @param m
 342  * The ordering to use.
 343  *
 344  * @returns
 345  * os_atomic_and_orig returns the value of the variable before the atomic and,
 346  * os_atomic_and returns the value of the variable after the atomic and.
 347  */
 348 #define os_atomic_and_orig(p, v, m) _os_atomic_c11_op_orig(p, v, m, fetch_and)
 349 #define os_atomic_and(p, v, m)      _os_atomic_c11_op(p, v, m, fetch_and, &)
 350
 351 /*!
 352  * @function os_atomic_andnot, os_atomic_andnot_orig
 353  *
 354  * @brief
 355  * Wrappers for C11 atomic_fetch_and_explicit(p, ~value).
 356  *
 357  * @param p
 358  * A pointer to an atomic variable.
 359  *
 360  * @param v
 361  * The value whose complement to and.
 362  *
 363  * @param m
 364  * The ordering to use.
 365  *
 366  * @returns
 367  * os_atomic_andnot_orig returns the value of the variable before the atomic andnot,
 368  * os_atomic_andnot returns the value of the variable after the atomic andnot.
 369  */
 370 #define os_atomic_andnot_orig(p, v, m) _os_atomic_c11_op_orig(p, ~(v), m, fetch_and)
 371 #define os_atomic_andnot(p, v, m)      _os_atomic_c11_op(p, ~(v), m, fetch_and, &)
 372
 373 /*!
 374  * @function os_atomic_or, os_atomic_or_orig
 375  *
 376  * @brief
 377  * Wrappers for C11 atomic_fetch_or_explicit().
 378  *
 379  * @param p
 380  * A pointer to an atomic variable.
 381  *
 382  * @param v
 383  * The value to or.
 384  *
 385  * @param m
 386  * The ordering to use.
 387  *
 388  * @returns
 389  * os_atomic_or_orig returns the value of the variable before the atomic or,
 390  * os_atomic_or returns the value of the variable after the atomic or.
 391  */
 392 #define os_atomic_or_orig(p, v, m)  _os_atomic_c11_op_orig(p, v, m, fetch_or)
 393 #define os_atomic_or(p, v, m)       _os_atomic_c11_op(p, v, m, fetch_or, |)
 394
 395 /*!
 396  * @function os_atomic_xor, os_atomic_xor_orig
 397  *
 398  * @brief
 399  * Wrappers for C11 atomic_fetch_xor_explicit().
 400  *
 401  * @param p
 402  * A pointer to an atomic variable.
 403  *
 404  * @param v
 405  * The value to xor.
 406  *
 407  * @param m
 408  * The ordering to use.
 409  *
 410  * @returns
 411  * os_atomic_xor_orig returns the value of the variable before the atomic xor,
 412  * os_atomic_xor returns the value of the variable after the atomic xor.
 413  */
 414 #define os_atomic_xor_orig(p, v, m) _os_atomic_c11_op_orig(p, v, m, fetch_xor)
 415 #define os_atomic_xor(p, v, m)      _os_atomic_c11_op(p, v, m, fetch_xor, ^)
 416
 417 /*!
 418  * @function os_atomic_min, os_atomic_min_orig
 419  *
 420  * @brief
 421  * Wrappers for Clang's __atomic_fetch_min()
 422  *
 423  * @param p
 424  * A pointer to an atomic variable.
 425  *
 426  * @param v
 427  * The value to minimize.
 428  *
 429  * @param m
 430  * The ordering to use.
 431  *
 432  * @returns
 433  * os_atomic_min_orig returns the value of the variable before the atomic min,
 434  * os_atomic_min returns the value of the variable after the atomic min.
 435  */
 436 #define os_atomic_min_orig(p, v, m) _os_atomic_clang_op_orig(p, v, m, fetch_min)
 437 #define os_atomic_min(p, v, m)      _os_atomic_clang_op(p, v, m, fetch_min, MIN)
 438
 439 /*!
 440  * @function os_atomic_max, os_atomic_max_orig
 441  *
 442  * @brief
 443  * Wrappers for Clang's __atomic_fetch_max()
 444  *
 445  * @param p
 446  * A pointer to an atomic variable.
 447  *
 448  * @param v
 449  * The value to maximize.
 450  *
 451  * @param m
 452  * The ordering to use.
 453  *
 454  * @returns
 455  * os_atomic_max_orig returns the value of the variable before the atomic max,
 456  * os_atomic_max returns the value of the variable after the atomic max.
 457  */
 458 #define os_atomic_max_orig(p, v, m) _os_atomic_clang_op_orig(p, v, m, fetch_max)
 459 #define os_atomic_max(p, v, m)      _os_atomic_clang_op(p, v, m, fetch_max, MAX)
 460
 461 /*!
 462  * @function os_atomic_xchg
 463  *
 464  * @brief
 465  * Wrapper for C11 atomic_exchange_explicit().
 466  *
 467  * @param p
 468  * A pointer to an atomic variable.
 469  *
 470  * @param v
 471  * The value to exchange with.
 472  *
 473  * @param m
 474  * The ordering to use.
 475  *
 476  * @returns
 477  * The value of the variable before the exchange.
 478  */
 479 #define os_atomic_xchg(p, v, m)     _os_atomic_c11_op_orig(p, v, m, exchange)
 480
 481 /*!
 482  * @function os_atomic_cmpxchg
 483  *
 484  * @brief
 485  * Wrapper for C11 atomic_compare_exchange_strong_explicit().
 486  *
 487  * @discussion
 488  * Loops around os_atomic_cmpxchg() may want to consider using the
 489  * os_atomic_rmw_loop() construct instead to take advantage of the C11 weak
 490  * compare-exchange operation.
 491  *
 492  * @param p
 493  * A pointer to an atomic variable.
 494  *
 495  * @param e
 496  * The value expected in the atomic variable.
 497  *
 498  * @param v
 499  * The value to store if the atomic variable has the expected value @a e.
 500  *
 501  * @param m
 502  * The ordering to use in case of success.
 503  * The ordering in case of failure is always `relaxed`.
 504  *
 505  * @returns
 506  * 0 if the compare-exchange failed.
 507  * 1 if the compare-exchange succeeded.
 508  */
 509 #define os_atomic_cmpxchg(p, e, v, m)  ({ \
 510                 _os_atomic_basetypeof(p) _r = (e); int _b; \
 511                 _os_compiler_barrier_before_atomic(m); \
 512                 _b = atomic_compare_exchange_strong_explicit(_os_atomic_c11_atomic(p), \
 513                                 &_r, v, memory_order_##m##_smp, memory_order_relaxed); \
 514                 _os_compiler_barrier_after_atomic(m); \
 515                 _b; \
 516 })
 517
 518 /*!
 519  * @function os_atomic_cmpxchgv
 520  *
 521  * @brief
 522  * Wrapper for C11 atomic_compare_exchange_strong_explicit().
 523  *
 524  * @discussion
 525  * Loops around os_atomic_cmpxchgv() may want to consider using the
 526  * os_atomic_rmw_loop() construct instead to take advantage of the C11 weak
 527  * compare-exchange operation.
 528  *
 529  * @param p
 530  * A pointer to an atomic variable.
 531  *
 532  * @param e
 533  * The value expected in the atomic variable.
 534  *
 535  * @param v
 536  * The value to store if the atomic variable has the expected value @a e.
 537  *
 538  * @param g
 539  * A pointer to a location that is filled with the value that was present in
 540  * the atomic variable before the compare-exchange (whether successful or not).
 541  * This can be used to redrive compare-exchange loops.
 542  *
 543  * @param m
 544  * The ordering to use in case of success.
 545  * The ordering in case of failure is always `relaxed`.
 546  *
 547  * @returns
 548  * 0 if the compare-exchange failed.
 549  * 1 if the compare-exchange succeeded.
 550  */
 551 #define os_atomic_cmpxchgv(p, e, v, g, m)  ({ \
 552                 _os_atomic_basetypeof(p) _r = (e); int _b; \
 553                 _os_compiler_barrier_before_atomic(m); \
 554                 _b = atomic_compare_exchange_strong_explicit(_os_atomic_c11_atomic(p), \
 555                                 &_r, v, memory_order_##m##_smp, memory_order_relaxed); \
 556                 _os_compiler_barrier_after_atomic(m); \
 557                 *(g) = _r; _b; \
 558 })
 559
 560 /*!
 561  * @function os_atomic_rmw_loop
 562  *
 563  * @brief
 564  * Advanced read-modify-write construct to wrap compare-exchange loops.
 565  *
 566  * @param p
 567  * A pointer to an atomic variable to be modified.
 568  *
 569  * @param ov
 570  * The name of the variable that will contain the original value of the atomic
 571  * variable (reloaded every iteration of the loop).
 572  *
 573  * @param nv
 574  * The name of the variable that will contain the new value to compare-exchange
 575  * the atomic variable to (typically computed from @a ov every iteration of the
 576  * loop).
 577  *
 578  * @param m
 579  * The ordering to use in case of success.
 580  * The ordering in case of failure is always `relaxed`.
 581  *
 582  * @param ...
 583  * Code block that validates the value of @p ov and computes the new value of
 584  * @p nv that the atomic variable will be compare-exchanged to in an iteration
 585  * of the loop.
 586  *
 587  * The loop can be aborted using os_atomic_rmw_loop_give_up(), e.g. when the
 588  * value of @p ov is found to be "invalid" for the ovarall operation.
 589  * `continue` cannot be used in this context.
 590  *
 591  * No stores to memory should be performed within the code block as it may cause
 592  * LL/SC transactions used to implement compare-exchange to fail persistently.
 593  *
 594  * @returns
 595  * 0 if the loop was aborted with os_atomic_rmw_loop_give_up().
 596  * 1 if the loop completed.
 597  */
 598 #define os_atomic_rmw_loop(p, ov, nv, m, ...)  ({ \
 599                 int _result = 0; \
 600                 typeof(p) _p = (p); \
 601                 _os_compiler_barrier_before_atomic(m); \
 602                 ov = atomic_load_explicit(_os_atomic_c11_atomic(_p), \
 603                                 memory_order_relaxed); \
 604                 do { \
 605                         __VA_ARGS__; \
 606                         _result = atomic_compare_exchange_weak_explicit( \
 607                                         _os_atomic_c11_atomic(_p), &ov, nv, \
 608                                         memory_order_##m##_smp, memory_order_relaxed); \
 609                 } while (__builtin_expect(!_result, 0)); \
 610                 _os_compiler_barrier_after_atomic(m); \
 611                 _result; \
 612         })
 613
 614 /*!
 615  * @function os_atomic_rmw_loop_give_up
 616  *
 617  * @brief
 618  * Abort an os_atomic_rmw_loop() loop.
 619  *
 620  * @param ...
 621  * Optional code block to execute before the `break` out of the loop. May
 622  * further alter the control flow (e.g. using `return`, `goto`, ...).
 623  */
 624 #define os_atomic_rmw_loop_give_up(...) ({ __VA_ARGS__; break; })
 625
 626 /*!
 627  * @typedef os_atomic_dependency_t
 628  *
 629  * @brief
 630  * Type for dependency tokens that can be derived from loads with dependency
 631  * and injected into various expressions.
 632  *
 633  * @warning
 634  * The implementation of atomic dependencies makes painstakingly sure that the
 635  * compiler doesn't know that os_atomic_dependency_t::__opaque_zero is always 0.
 636  *
 637  * Users of os_atomic_dependency_t MUST NOT test its value (even with an
 638  * assert), as doing so would allow the compiler to reason about the value and
 639  * elide its use to inject hardware dependencies (thwarting the entire purpose
 640  * of the construct).
 641  */
 642 typedef struct { unsigned long __opaque_zero; } os_atomic_dependency_t;
 643
 644 /*!
 645  * @const OS_ATOMIC_DEPENDENCY_NONE
 646  *
 647  * @brief
 648  * A value to pass to functions that can carry dependencies, to indicate that
 649  * no dependency should be carried.
 650  */
 651 #define OS_ATOMIC_DEPENDENCY_NONE \
 652                 ((os_atomic_dependency_t){ 0UL })
 653
 654 /*!
 655  * @function os_atomic_make_dependency
 656  *
 657  * @brief
 658  * Create a dependency token that can be injected into expressions to force a
 659  * hardware dependency.
 660  *
 661  * @discussion
 662  * This function is only useful for cases where the dependency needs to be used
 663  * several times.
 664  *
 665  * os_atomic_load_with_dependency_on() and os_atomic_inject_dependency() are
 666  * otherwise capable of automatically creating dependency tokens.
 667  *
 668  * @param v
 669  * The result of:
 670  * - an os_atomic_load(..., dependency),
 671  * - an os_atomic_inject_dependency(),
 672  * - an os_atomic_load_with_dependency_on().
 673  *
 674  * Note that due to implementation limitations, the type of @p v must be
 675  * register-sized, if necessary an explicit cast is required.
 676  *
 677  * @returns
 678  * An os_atomic_dependency_t token that can be used to prolongate dependency
 679  * chains.
 680  *
 681  * The token value is always 0, but the compiler must never be able to reason
 682  * about that fact (c.f. os_atomic_dependency_t)
 683  */
 684 #define os_atomic_make_dependency(v) \
 685                 ((void)(v), OS_ATOMIC_DEPENDENCY_NONE)
 686
 687 /*!
 688  * @function os_atomic_inject_dependency
 689  *
 690  * @brief
 691  * Inject a hardware dependency resulting from a `dependency` load into a
 692  * specified pointer.
 693  *
 694  * @param p
 695  * A pointer to inject the dependency into.
 696  *
 697  * @param e
 698  * - a dependency token returned from os_atomic_make_dependency(),
 699  *
 700  * - OS_ATOMIC_DEPENDENCY_NONE, which turns this operation into a no-op,
 701  *
 702  * - any value accepted by os_atomic_make_dependency().
 703  *
 704  * @returns
 705  * A value equal to @a p but that prolongates the dependency chain rooted at
 706  * @a e.
 707  */
 708 #define os_atomic_inject_dependency(p, e) \
 709                 ((typeof(*(p)) *)((p) + _os_atomic_auto_dependency(e).__opaque_zero))
 710
 711 /*!
 712  * @function os_atomic_load_with_dependency_on
 713  *
 714  * @brief
 715  * Load that prolongates the dependency chain rooted at `v`.
 716  *
 717  * @discussion
 718  * This is shorthand for:
 719  *
 720  * <code>
 721  *   os_atomic_load(os_atomic_inject_dependency(p, e), dependency)
 722  * </code>
 723  *
 724  * @param p
 725  * A pointer to an atomic variable.
 726  *
 727  * @param e
 728  * - a dependency token returned from os_atomic_make_dependency(),
 729  *
 730  * - OS_ATOMIC_DEPENDENCY_NONE, which turns this operation into a no-op,
 731  *
 732  * - any value accepted by os_atomic_make_dependency().
 733  *
 734  * @returns
 735  * The value loaded from @a p.
 736  */
 737 #define os_atomic_load_with_dependency_on(p, e) \
 738                 os_atomic_load(os_atomic_inject_dependency(p, e), dependency)
 739
 740 /*!
 741  * @const OS_ATOMIC_HAS_LLSC
 742  *
 743  * @brief
 744  * Whether the platform has LL/SC features.
 745  *
 746  * @discussion
 747  * When set, the os_atomic_*_exclusive() macros are defined.
 748  */
 749 #define OS_ATOMIC_HAS_LLSC  0
 750
 751 /*!
 752  * @const OS_ATOMIC_USE_LLSC
 753  *
 754  * @brief
 755  * Whether os_atomic* use LL/SC internally.
 756  *
 757  * @discussion
 758  * OS_ATOMIC_USE_LLSC implies OS_ATOMIC_HAS_LLSC.
 759  */
 760 #define OS_ATOMIC_USE_LLSC  0
 761
 762 #if defined (__x86_64__)
 763 #include "i386/atomic.h"
 764 #elif defined (__arm__) || defined (__arm64__)
 765 #include "arm/atomic.h"
 766 #else
 767 #error architecture not supported
 768 #endif
 769
 770 #endif /* _MACHINE_ATOMIC_H */