X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/0a7de7458d150b5d4dffc935ba399be265ef0a1a..HEAD:/libkern/firehose/chunk_private.h diff --git a/libkern/firehose/chunk_private.h b/libkern/firehose/chunk_private.h index 80d5969fa..513fd2082 100644 --- a/libkern/firehose/chunk_private.h +++ b/libkern/firehose/chunk_private.h @@ -21,12 +21,7 @@ #ifndef __FIREHOSE_CHUNK_PRIVATE__ #define __FIREHOSE_CHUNK_PRIVATE__ -#if KERNEL -#include <machine/atomic.h> -#endif -#include <stdatomic.h> #include <sys/param.h> -#include <os/base.h> #include "firehose_types_private.h" #include "tracepoint_private.h" @@ -42,7 +37,7 @@ __BEGIN_DECLS ((((pos).fcp_pos >> 48) & 0x1ff) == (uint16_t)stream) typedef union { - _Atomic(uint64_t) fcp_atomic_pos; + os_atomic(uint64_t) fcp_atomic_pos; uint64_t fcp_pos; struct { uint16_t fcp_next_entry_offs; @@ -69,18 +64,20 @@ typedef struct firehose_chunk_range_s { uint16_t fcr_length; } *firehose_chunk_range_t; +#if __has_include(<os/atomic_private.h>) #if defined(KERNEL) || defined(OS_FIREHOSE_SPI) OS_ALWAYS_INLINE static inline bool -firehose_chunk_pos_fits(firehose_chunk_pos_u pos, uint16_t size) +firehose_chunk_pos_fits(firehose_chunk_pos_u *pos, uint16_t size) { - return pos.fcp_next_entry_offs + size <= pos.fcp_private_offs; + return pos->fcp_next_entry_offs + size <= pos->fcp_private_offs; } #define FIREHOSE_CHUNK_TRY_RESERVE_FAIL_ENQUEUE (-1) #define FIREHOSE_CHUNK_TRY_RESERVE_FAIL ( 0) +#if OS_ATOMIC_HAS_STARVATION_FREE_RMW || !OS_ATOMIC_CONFIG_STARVATION_FREE_ONLY OS_ALWAYS_INLINE static inline long firehose_chunk_tracepoint_try_reserve(firehose_chunk_t fc, uint64_t stamp, @@ -109,8 +106,8 @@ firehose_chunk_tracepoint_try_reserve(firehose_chunk_t fc, uint64_t stamp, // - read the chunk to find a very old thing os_atomic_rmw_loop_give_up(return FIREHOSE_CHUNK_TRY_RESERVE_FAIL); } - pos = orig; - if (!firehose_chunk_pos_fits(orig, + pos.fcp_pos = orig.fcp_pos; + if (!firehose_chunk_pos_fits(&orig, ft_size + pubsize + privsize) || !stamp_delta_fits) { pos.fcp_flag_full = true; reservation_failed = true; @@ -126,7 +123,7 @@ firehose_chunk_tracepoint_try_reserve(firehose_chunk_t fc, uint64_t stamp, pos.fcp_pos -= privsize * FIREHOSE_CHUNK_POS_PRIVATE_OFFS_INC; pos.fcp_pos += FIREHOSE_CHUNK_POS_REFCNT_INC; const uint16_t minimum_payload_size = 16; - if (!firehose_chunk_pos_fits(pos, + if (!firehose_chunk_pos_fits(&pos, roundup(ft_size + minimum_payload_size, 8))) { // if we can't even have minimum_payload_size bytes of payload // for the next tracepoint, just flush right away @@ -146,7 +143,7 @@ firehose_chunk_tracepoint_try_reserve(firehose_chunk_t fc, uint64_t stamp, return FIREHOSE_CHUNK_TRY_RESERVE_FAIL_ENQUEUE; } if (privptr) { - *privptr = fc->fc_start + pos.fcp_private_offs; + *privptr = (uint8_t *)((uintptr_t)fc->fc_start + pos.fcp_private_offs); } return orig.fcp_next_entry_offs; } @@ -157,13 +154,13 @@ firehose_chunk_tracepoint_begin(firehose_chunk_t fc, uint64_t stamp, uint16_t pubsize, uint64_t thread_id, long offset) { firehose_tracepoint_t ft = (firehose_tracepoint_t) - __builtin_assume_aligned(fc->fc_start + offset, 8); + __builtin_assume_aligned((void *)((uintptr_t)fc->fc_start + (uintptr_t)offset), 8); stamp -= fc->fc_timestamp; stamp |= (uint64_t)pubsize << 48; // The compiler barrier is needed for userland process death handling, see // (tracepoint-begin) in libdispatch's firehose_buffer_stream_chunk_install. - atomic_store_explicit(&ft->ft_atomic_stamp_and_length, stamp, - memory_order_relaxed); + os_atomic_std(atomic_store_explicit)(&ft->ft_atomic_stamp_and_length, stamp, + os_atomic_std(memory_order_relaxed)); __asm__ __volatile__ ("" ::: "memory"); ft->ft_thread = thread_id; return ft; @@ -176,14 +173,16 @@ firehose_chunk_tracepoint_end(firehose_chunk_t fc, { firehose_chunk_pos_u pos; - atomic_store_explicit(&ft->ft_id.ftid_atomic_value, - ftid.ftid_value, memory_order_release); - pos.fcp_pos = atomic_fetch_sub_explicit(&fc->fc_pos.fcp_atomic_pos, - FIREHOSE_CHUNK_POS_REFCNT_INC, memory_order_relaxed); + os_atomic_std(atomic_store_explicit)(&ft->ft_id.ftid_atomic_value, + ftid.ftid_value, os_atomic_std(memory_order_release)); + pos.fcp_pos = os_atomic_std(atomic_fetch_sub_explicit)(&fc->fc_pos.fcp_atomic_pos, + FIREHOSE_CHUNK_POS_REFCNT_INC, os_atomic_std(memory_order_relaxed)); return pos.fcp_refcnt == 1 && pos.fcp_flag_full; } +#endif // OS_ATOMIC_HAS_STARVATION_FREE_RMW || !OS_ATOMIC_CONFIG_STARVATION_FREE_ONLY #endif // defined(KERNEL) || defined(OS_FIREHOSE_SPI) +#endif // __has_include(<os/atomic_private.h>) __END_DECLS