src/firehose/firehose_buffer_internal.h

   1 /*
   2  * Copyright (c) 2015 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_APACHE_LICENSE_HEADER_START@
   5  *
   6  * Licensed under the Apache License, Version 2.0 (the "License");
   7  * you may not use this file except in compliance with the License.
   8  * You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  *
  18  * @APPLE_APACHE_LICENSE_HEADER_END@
  19  */
  20
  21 #ifndef __FIREHOSE_BUFFER_INTERNAL__
  22 #define __FIREHOSE_BUFFER_INTERNAL__
  23
  24 #if BYTE_ORDER != LITTLE_ENDIAN
  25 #error unsupported byte order
  26 #endif
  27
  28 #ifndef KERNEL
  29 #include <os/lock_private.h>
  30 #endif
  31
  32 // firehose buffer is CHUNK_COUNT * CHUNK_SIZE big == 256k
  33 #define FIREHOSE_BUFFER_CHUNK_COUNT                                     64ul
  34 #ifdef KERNEL
  35 #define FIREHOSE_BUFFER_CHUNK_PREALLOCATED_COUNT        15
  36 #else
  37 #define FIREHOSE_BUFFER_CHUNK_PREALLOCATED_COUNT        4
  38 #define FIREHOSE_BUFFER_MADVISE_CHUNK_COUNT                     4
  39 #endif
  40
  41 static const unsigned long firehose_stream_uses_io_bank =
  42         (1UL << firehose_stream_persist) |
  43         (1UL << firehose_stream_special);
  44
  45 typedef union {
  46 #define FIREHOSE_BANK_SHIFT(bank)                       (16 * (bank))
  47 #define FIREHOSE_BANK_INC(bank)                         (1ULL << FIREHOSE_BANK_SHIFT(bank))
  48 #define FIREHOSE_BANK_UNAVAIL_BIT                       ((uint16_t)0x8000)
  49 #define FIREHOSE_BANK_UNAVAIL_MASK(bank)        (FIREHOSE_BANK_INC(bank) << 15)
  50         uint64_t fbs_atomic_state;
  51         struct {
  52                 uint16_t fbs_mem_bank;
  53                 uint16_t fbs_io_bank;
  54                 uint16_t fbs_max_ref;
  55                 uint16_t fbs_unused;
  56         };
  57 } firehose_bank_state_u;
  58
  59 #if __has_feature(c_static_assert)
  60 _Static_assert(8 * offsetof(firehose_bank_state_u, fbs_mem_bank)
  61                 == FIREHOSE_BANK_SHIFT(0), "mem bank shift");
  62 _Static_assert(8 * offsetof(firehose_bank_state_u, fbs_io_bank)
  63                 == FIREHOSE_BANK_SHIFT(1), "mem bank shift");
  64 #endif
  65
  66 typedef struct firehose_buffer_bank_s {
  67         firehose_bank_state_u volatile fbb_state;
  68         uint64_t volatile fbb_metadata_bitmap;
  69         uint64_t volatile fbb_mem_flushed;
  70         uint64_t volatile fbb_mem_notifs;
  71         uint64_t volatile fbb_mem_sync_pushes;
  72         uint64_t volatile fbb_io_flushed;
  73         uint64_t volatile fbb_io_notifs;
  74         uint64_t volatile fbb_io_sync_pushes;
  75 #define FIREHOSE_BUFFER_BANK_FLAG_LOW_MEMORY    (1UL << 0)
  76 #define FIREHOSE_BUFFER_BANK_FLAG_HIGH_RATE             (1UL << 1)
  77         unsigned long volatile fbb_flags;
  78
  79         uint64_t fbb_bitmap; // protected by fbb_lock
  80         firehose_bank_state_u fbb_limits; // protected by fbb_lock
  81 #ifdef KERNEL
  82         uint32_t _fbb_unused;
  83 #else
  84         dispatch_unfair_lock_s fbb_lock;
  85 #endif
  86 } OS_ALIGNED(64) *firehose_buffer_bank_t;
  87
  88 typedef union {
  89         uint64_t fss_atomic_state;
  90         dispatch_gate_s fss_gate;
  91         struct {
  92                 uint32_t fss_allocator;
  93 #define FIREHOSE_STREAM_STATE_PRISTINE          0xffff
  94                 uint16_t fss_current;
  95                 uint16_t fss_generation;
  96         };
  97 } firehose_stream_state_u;
  98
  99 typedef struct firehose_buffer_stream_s {
 100         firehose_stream_state_u fbs_state;
 101 } OS_ALIGNED(128) *firehose_buffer_stream_t;
 102
 103 typedef union {
 104         uint64_t frp_atomic_tail;
 105         struct {
 106                 uint16_t frp_mem_tail;
 107                 uint16_t frp_mem_flushed;
 108                 uint16_t frp_io_tail;
 109                 uint16_t frp_io_flushed;
 110         };
 111 } firehose_ring_tail_u;
 112
 113 #define FIREHOSE_RING_POS_GEN_INC               ((uint16_t)(FIREHOSE_BUFFER_CHUNK_COUNT))
 114 #define FIREHOSE_RING_POS_IDX_MASK              ((uint16_t)(FIREHOSE_RING_POS_GEN_INC - 1))
 115 #define FIREHOSE_RING_POS_GEN_MASK              ((uint16_t)~FIREHOSE_RING_POS_IDX_MASK)
 116
 117 /*
 118  * Rings are circular buffers with CHUNK_COUNT entries, with 3 important markers
 119  *
 120  * +--------+-------------------------+------------+---------------------------+
 121  * |xxxxxxxx|                         |............|xxxxxxxxxxxxxxxxxxxxxxxxxxx|
 122  * +--------+-------------------------+------------+---------------------------+
 123  *          ^                         ^            ^
 124  *        head                       tail       flushed
 125  *
 126  * A ring position is a uint16_t made of a generation (see GEN_MASK) and an
 127  * index (see IDX_MASK). Slots of that ring hold tagged page references. These
 128  * are made from a generation (see GEN_MASK) and a page reference.
 129  *
 130  * A generation is how many times the head wrapped around.
 131  *
 132  * These conditions hold:
 133  *   (uint16_t)(flushed - tail) < FIREHOSE_BUFFER_CHUNK_COUNT
 134  *   (uint16_t)(head - flushed) < FIREHOSE_BUFFER_CHUNK_COUNT
 135  * which really means, on the circular buffer, tail <= flushed <= head.
 136  *
 137  * Page references span from 1 to (CHUNK_COUNT - 1). 0 is an invalid page
 138  * (corresponds to the buffer header) and means "unused".
 139  *
 140  *
 141  * - Entries situated between tail and flushed hold references to pages that
 142  *   the firehose consumer (logd) has flushed, and can be reused.
 143  *
 144  * - Entries situated between flushed and head are references to pages waiting
 145  *   to be flushed.
 146  *
 147  * - Entries not situated between tail and head are either slots being modified
 148  *   or that should be set to Empty. Empty is the 0 page reference associated
 149  *   with the generation count the head will have the next time it will go over
 150  *   that slot.
 151  */
 152 typedef struct firehose_buffer_header_s {
 153         uint16_t volatile                               fbh_mem_ring[FIREHOSE_BUFFER_CHUNK_COUNT];
 154         uint16_t volatile                               fbh_io_ring[FIREHOSE_BUFFER_CHUNK_COUNT];
 155
 156         firehose_ring_tail_u volatile   fbh_ring_tail OS_ALIGNED(64);
 157         uint32_t                                                fbh_spi_version;
 158         uint16_t volatile                               fbh_ring_mem_head OS_ALIGNED(64);
 159         uint16_t volatile                               fbh_ring_io_head OS_ALIGNED(64);
 160         struct firehose_buffer_bank_s   fbh_bank;
 161         struct firehose_buffer_stream_s fbh_stream[_firehose_stream_max];
 162
 163         uint64_t                                                fbh_uniquepid;
 164         pid_t                                                   fbh_pid;
 165         mach_port_t                                             fbh_logd_port;
 166         mach_port_t volatile                    fbh_sendp;
 167         mach_port_t                                             fbh_recvp;
 168
 169         // past that point fields may be aligned differently between 32 and 64bits
 170 #ifndef KERNEL
 171         dispatch_once_t                                 fbh_notifs_pred OS_ALIGNED(64);
 172         dispatch_source_t                               fbh_notifs_source;
 173         dispatch_unfair_lock_s                  fbh_logd_lock;
 174 #endif
 175         uint64_t                                                fbh_unused[0];
 176 } OS_ALIGNED(FIREHOSE_BUFFER_CHUNK_SIZE) *firehose_buffer_header_t;
 177
 178 union firehose_buffer_u {
 179         struct firehose_buffer_header_s fb_header;
 180         struct firehose_buffer_chunk_s fb_chunks[FIREHOSE_BUFFER_CHUNK_COUNT];
 181 };
 182
 183 // used to let the compiler pack these values in 1 or 2 registers
 184 typedef struct firehose_tracepoint_query_s {
 185         uint16_t pubsize;
 186         uint16_t privsize;
 187         firehose_stream_t stream;
 188         bool     is_bank_ok;
 189         bool     for_io;
 190         uint64_t stamp;
 191 } *firehose_tracepoint_query_t;
 192
 193 #ifndef FIREHOSE_SERVER
 194
 195 firehose_buffer_t
 196 firehose_buffer_create(mach_port_t logd_port, uint64_t unique_pid,
 197                 unsigned long bank_flags);
 198
 199 firehose_tracepoint_t
 200 firehose_buffer_tracepoint_reserve_slow(firehose_buffer_t fb,
 201                 firehose_tracepoint_query_t ask, uint8_t **privptr);
 202
 203 void
 204 firehose_buffer_update_limits(firehose_buffer_t fb);
 205
 206 void
 207 firehose_buffer_ring_enqueue(firehose_buffer_t fb, uint16_t ref);
 208
 209 #endif
 210
 211 #endif // __FIREHOSE_BUFFER_INTERNAL__