libkern/firehose/chunk_private.h

   1 /*
   2  * Copyright (c) 2016 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_APACHE_LICENSE_HEADER_START@
   5  *
   6  * Licensed under the Apache License, Version 2.0 (the "License");
   7  * you may not use this file except in compliance with the License.
   8  * You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  *
  18  * @APPLE_APACHE_LICENSE_HEADER_END@
  19  */
  20
  21 #ifndef __FIREHOSE_CHUNK_PRIVATE__
  22 #define __FIREHOSE_CHUNK_PRIVATE__
  23
  24 #include <sys/param.h>
  25 #include "firehose_types_private.h"
  26 #include "tracepoint_private.h"
  27
  28 __BEGIN_DECLS
  29
  30 #define FIREHOSE_CHUNK_SIZE                     4096ul
  31
  32 #define FIREHOSE_CHUNK_POS_ENTRY_OFFS_INC       (1ULL <<  0)
  33 #define FIREHOSE_CHUNK_POS_PRIVATE_OFFS_INC     (1ULL << 16)
  34 #define FIREHOSE_CHUNK_POS_REFCNT_INC           (1ULL << 32)
  35 #define FIREHOSE_CHUNK_POS_FULL_BIT             (1ULL << 56)
  36 #define FIREHOSE_CHUNK_POS_USABLE_FOR_STREAM(pos, stream) \
  37                 ((((pos).fcp_pos >> 48) & 0x1ff) == (uint16_t)stream)
  38
  39 typedef union {
  40         os_atomic(uint64_t) fcp_atomic_pos;
  41         uint64_t fcp_pos;
  42         struct {
  43                 uint16_t fcp_next_entry_offs;
  44                 uint16_t fcp_private_offs;
  45                 uint8_t  fcp_refcnt;
  46                 uint8_t  fcp_qos;
  47                 uint8_t  fcp_stream;
  48                 uint8_t  fcp_flag_full : 1;
  49                 uint8_t  fcp_flag_io : 1;
  50                 uint8_t  fcp_quarantined : 1;
  51                 uint8_t  _fcp_flag_unused : 5;
  52         };
  53 } firehose_chunk_pos_u;
  54
  55 typedef struct firehose_chunk_s {
  56         uint8_t  fc_start[0];
  57         firehose_chunk_pos_u fc_pos;
  58         uint64_t fc_timestamp;
  59         uint8_t  fc_data[FIREHOSE_CHUNK_SIZE - 8 - 8];
  60 } *firehose_chunk_t;
  61
  62 typedef struct firehose_chunk_range_s {
  63         uint16_t fcr_offset; // offset from the start of the chunk
  64         uint16_t fcr_length;
  65 } *firehose_chunk_range_t;
  66
  67 #if defined(KERNEL) || defined(OS_FIREHOSE_SPI)
  68
  69 OS_ALWAYS_INLINE
  70 static inline bool
  71 firehose_chunk_pos_fits(firehose_chunk_pos_u *pos, uint16_t size)
  72 {
  73         return pos->fcp_next_entry_offs + size <= pos->fcp_private_offs;
  74 }
  75
  76 #define FIREHOSE_CHUNK_TRY_RESERVE_FAIL_ENQUEUE  (-1)
  77 #define FIREHOSE_CHUNK_TRY_RESERVE_FAIL          ( 0)
  78
  79 #if OS_ATOMIC_HAS_STARVATION_FREE_RMW || !OS_ATOMIC_CONFIG_STARVATION_FREE_ONLY
  80 OS_ALWAYS_INLINE
  81 static inline long
  82 firehose_chunk_tracepoint_try_reserve(firehose_chunk_t fc, uint64_t stamp,
  83     firehose_stream_t stream, uint8_t qos, uint16_t pubsize,
  84     uint16_t privsize, uint8_t **privptr)
  85 {
  86         const uint16_t ft_size = offsetof(struct firehose_tracepoint_s, ft_data);
  87         firehose_chunk_pos_u orig, pos;
  88         bool reservation_failed, stamp_delta_fits;
  89
  90         stamp_delta_fits = ((stamp - fc->fc_timestamp) >> 48) == 0;
  91
  92         // no acquire barrier because the returned space is written to only
  93         os_atomic_rmw_loop(&fc->fc_pos.fcp_atomic_pos,
  94             orig.fcp_pos, pos.fcp_pos, relaxed, {
  95                 if (orig.fcp_pos == 0) {
  96                         // we acquired a really really old reference, and we probably
  97                         // just faulted in a new page
  98                         os_atomic_rmw_loop_give_up(return FIREHOSE_CHUNK_TRY_RESERVE_FAIL);
  99                 }
 100                 if (!FIREHOSE_CHUNK_POS_USABLE_FOR_STREAM(orig, stream)) {
 101                         // nothing to do if the chunk is full, or the stream doesn't match,
 102                         // in which case the thread probably:
 103                         // - loaded the chunk ref
 104                         // - been suspended a long while
 105                         // - read the chunk to find a very old thing
 106                         os_atomic_rmw_loop_give_up(return FIREHOSE_CHUNK_TRY_RESERVE_FAIL);
 107                 }
 108                 pos.fcp_pos = orig.fcp_pos;
 109                 if (!firehose_chunk_pos_fits(&orig,
 110                 ft_size + pubsize + privsize) || !stamp_delta_fits) {
 111                         pos.fcp_flag_full = true;
 112                         reservation_failed = true;
 113                 } else {
 114                         if (qos > pos.fcp_qos) {
 115                                 pos.fcp_qos = qos;
 116                         }
 117                         // using these *_INC macros is so that the compiler generates better
 118                         // assembly: using the struct individual fields forces the compiler
 119                         // to handle carry propagations, and we know it won't happen
 120                         pos.fcp_pos += roundup(ft_size + pubsize, 8) *
 121                         FIREHOSE_CHUNK_POS_ENTRY_OFFS_INC;
 122                         pos.fcp_pos -= privsize * FIREHOSE_CHUNK_POS_PRIVATE_OFFS_INC;
 123                         pos.fcp_pos += FIREHOSE_CHUNK_POS_REFCNT_INC;
 124                         const uint16_t minimum_payload_size = 16;
 125                         if (!firehose_chunk_pos_fits(&pos,
 126                         roundup(ft_size + minimum_payload_size, 8))) {
 127                                 // if we can't even have minimum_payload_size bytes of payload
 128                                 // for the next tracepoint, just flush right away
 129                                 pos.fcp_flag_full = true;
 130                         }
 131                         reservation_failed = false;
 132                 }
 133         });
 134
 135         if (reservation_failed) {
 136                 if (pos.fcp_refcnt) {
 137                         // nothing to do, there is a thread writing that will pick up
 138                         // the "FULL" flag on flush and push as a consequence
 139                         return FIREHOSE_CHUNK_TRY_RESERVE_FAIL;
 140                 }
 141                 // caller must enqueue chunk
 142                 return FIREHOSE_CHUNK_TRY_RESERVE_FAIL_ENQUEUE;
 143         }
 144         if (privptr) {
 145                 *privptr = (uint8_t *)((uintptr_t)fc->fc_start + pos.fcp_private_offs);
 146         }
 147         return orig.fcp_next_entry_offs;
 148 }
 149
 150 OS_ALWAYS_INLINE
 151 static inline firehose_tracepoint_t
 152 firehose_chunk_tracepoint_begin(firehose_chunk_t fc, uint64_t stamp,
 153     uint16_t pubsize, uint64_t thread_id, long offset)
 154 {
 155         firehose_tracepoint_t ft = (firehose_tracepoint_t)
 156             __builtin_assume_aligned((void *)((uintptr_t)fc->fc_start + (uintptr_t)offset), 8);
 157         stamp -= fc->fc_timestamp;
 158         stamp |= (uint64_t)pubsize << 48;
 159         // The compiler barrier is needed for userland process death handling, see
 160         // (tracepoint-begin) in libdispatch's firehose_buffer_stream_chunk_install.
 161         os_atomic_std(atomic_store_explicit)(&ft->ft_atomic_stamp_and_length, stamp,
 162             os_atomic_std(memory_order_relaxed));
 163         __asm__ __volatile__ ("" ::: "memory");
 164         ft->ft_thread = thread_id;
 165         return ft;
 166 }
 167
 168 OS_ALWAYS_INLINE
 169 static inline bool
 170 firehose_chunk_tracepoint_end(firehose_chunk_t fc,
 171     firehose_tracepoint_t ft, firehose_tracepoint_id_u ftid)
 172 {
 173         firehose_chunk_pos_u pos;
 174
 175         os_atomic_std(atomic_store_explicit)(&ft->ft_id.ftid_atomic_value,
 176             ftid.ftid_value, os_atomic_std(memory_order_release));
 177         pos.fcp_pos = os_atomic_std(atomic_fetch_sub_explicit)(&fc->fc_pos.fcp_atomic_pos,
 178             FIREHOSE_CHUNK_POS_REFCNT_INC, os_atomic_std(memory_order_relaxed));
 179         return pos.fcp_refcnt == 1 && pos.fcp_flag_full;
 180 }
 181 #endif // OS_ATOMIC_HAS_STARVATION_FREE_RMW || !OS_ATOMIC_CONFIG_STARVATION_FREE_ONLY
 182
 183 #endif // defined(KERNEL) || defined(OS_FIREHOSE_SPI)
 184
 185 __END_DECLS
 186
 187 #endif // __FIREHOSE_CHUNK_PRIVATE__