2 * Copyright (c) 2015 Apple Inc. All rights reserved.
4 * @APPLE_APACHE_LICENSE_HEADER_START@
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
10 * http://www.apache.org/licenses/LICENSE-2.0
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
18 * @APPLE_APACHE_LICENSE_HEADER_END@
21 #ifndef __FIREHOSE_BUFFER_INTERNAL__
22 #define __FIREHOSE_BUFFER_INTERNAL__
24 #if BYTE_ORDER != LITTLE_ENDIAN
25 #error unsupported byte order
29 #include <os/lock_private.h>
32 // firehose buffer is CHUNK_COUNT * CHUNK_SIZE big == 256k
33 #define FIREHOSE_BUFFER_CHUNK_COUNT 64ul
35 #define FIREHOSE_BUFFER_CHUNK_PREALLOCATED_COUNT 15
37 #define FIREHOSE_BUFFER_CHUNK_PREALLOCATED_COUNT 4
38 #define FIREHOSE_BUFFER_MADVISE_CHUNK_COUNT 4
41 static const unsigned long firehose_stream_uses_io_bank
=
42 (1UL << firehose_stream_persist
) |
43 (1UL << firehose_stream_special
);
46 #define FIREHOSE_BANK_SHIFT(bank) (16 * (bank))
47 #define FIREHOSE_BANK_INC(bank) (1ULL << FIREHOSE_BANK_SHIFT(bank))
48 #define FIREHOSE_BANK_UNAVAIL_BIT ((uint16_t)0x8000)
49 #define FIREHOSE_BANK_UNAVAIL_MASK(bank) (FIREHOSE_BANK_INC(bank) << 15)
50 uint64_t fbs_atomic_state
;
52 uint16_t fbs_mem_bank
;
57 } firehose_bank_state_u
;
59 #if __has_feature(c_static_assert)
60 _Static_assert(8 * offsetof(firehose_bank_state_u
, fbs_mem_bank
)
61 == FIREHOSE_BANK_SHIFT(0), "mem bank shift");
62 _Static_assert(8 * offsetof(firehose_bank_state_u
, fbs_io_bank
)
63 == FIREHOSE_BANK_SHIFT(1), "mem bank shift");
66 typedef struct firehose_buffer_bank_s
{
67 firehose_bank_state_u
volatile fbb_state
;
68 uint64_t volatile fbb_metadata_bitmap
;
69 uint64_t volatile fbb_mem_flushed
;
70 uint64_t volatile fbb_mem_notifs
;
71 uint64_t volatile fbb_mem_sync_pushes
;
72 uint64_t volatile fbb_io_flushed
;
73 uint64_t volatile fbb_io_notifs
;
74 uint64_t volatile fbb_io_sync_pushes
;
75 #define FIREHOSE_BUFFER_BANK_FLAG_LOW_MEMORY (1UL << 0)
76 #define FIREHOSE_BUFFER_BANK_FLAG_HIGH_RATE (1UL << 1)
77 unsigned long volatile fbb_flags
;
79 uint64_t fbb_bitmap
; // protected by fbb_lock
80 firehose_bank_state_u fbb_limits
; // protected by fbb_lock
84 dispatch_unfair_lock_s fbb_lock
;
86 } OS_ALIGNED(64) *firehose_buffer_bank_t
;
89 uint64_t fss_atomic_state
;
90 dispatch_gate_s fss_gate
;
92 uint32_t fss_allocator
;
93 #define FIREHOSE_STREAM_STATE_PRISTINE 0xffff
95 uint16_t fss_generation
;
97 } firehose_stream_state_u
;
99 typedef struct firehose_buffer_stream_s
{
100 firehose_stream_state_u fbs_state
;
101 } OS_ALIGNED(128) *firehose_buffer_stream_t
;
104 uint64_t frp_atomic_tail
;
106 uint16_t frp_mem_tail
;
107 uint16_t frp_mem_flushed
;
108 uint16_t frp_io_tail
;
109 uint16_t frp_io_flushed
;
111 } firehose_ring_tail_u
;
113 #define FIREHOSE_RING_POS_GEN_INC ((uint16_t)(FIREHOSE_BUFFER_CHUNK_COUNT))
114 #define FIREHOSE_RING_POS_IDX_MASK ((uint16_t)(FIREHOSE_RING_POS_GEN_INC - 1))
115 #define FIREHOSE_RING_POS_GEN_MASK ((uint16_t)~FIREHOSE_RING_POS_IDX_MASK)
118 * Rings are circular buffers with CHUNK_COUNT entries, with 3 important markers
120 * +--------+-------------------------+------------+---------------------------+
121 * |xxxxxxxx| |............|xxxxxxxxxxxxxxxxxxxxxxxxxxx|
122 * +--------+-------------------------+------------+---------------------------+
126 * A ring position is a uint16_t made of a generation (see GEN_MASK) and an
127 * index (see IDX_MASK). Slots of that ring hold tagged page references. These
128 * are made from a generation (see GEN_MASK) and a page reference.
130 * A generation is how many times the head wrapped around.
132 * These conditions hold:
133 * (uint16_t)(flushed - tail) < FIREHOSE_BUFFER_CHUNK_COUNT
134 * (uint16_t)(head - flushed) < FIREHOSE_BUFFER_CHUNK_COUNT
135 * which really means, on the circular buffer, tail <= flushed <= head.
137 * Page references span from 1 to (CHUNK_COUNT - 1). 0 is an invalid page
138 * (corresponds to the buffer header) and means "unused".
141 * - Entries situated between tail and flushed hold references to pages that
142 * the firehose consumer (logd) has flushed, and can be reused.
144 * - Entries situated between flushed and head are references to pages waiting
147 * - Entries not situated between tail and head are either slots being modified
148 * or that should be set to Empty. Empty is the 0 page reference associated
149 * with the generation count the head will have the next time it will go over
152 typedef struct firehose_buffer_header_s
{
153 uint16_t volatile fbh_mem_ring
[FIREHOSE_BUFFER_CHUNK_COUNT
];
154 uint16_t volatile fbh_io_ring
[FIREHOSE_BUFFER_CHUNK_COUNT
];
156 firehose_ring_tail_u
volatile fbh_ring_tail
OS_ALIGNED(64);
157 uint32_t fbh_spi_version
;
158 uint16_t volatile fbh_ring_mem_head
OS_ALIGNED(64);
159 uint16_t volatile fbh_ring_io_head
OS_ALIGNED(64);
160 struct firehose_buffer_bank_s fbh_bank
;
161 struct firehose_buffer_stream_s fbh_stream
[_firehose_stream_max
];
163 uint64_t fbh_uniquepid
;
165 mach_port_t fbh_logd_port
;
166 mach_port_t
volatile fbh_sendp
;
167 mach_port_t fbh_recvp
;
169 // past that point fields may be aligned differently between 32 and 64bits
171 dispatch_once_t fbh_notifs_pred
OS_ALIGNED(64);
172 dispatch_source_t fbh_notifs_source
;
173 dispatch_unfair_lock_s fbh_logd_lock
;
174 #define FBH_QUARANTINE_NONE 0
175 #define FBH_QUARANTINE_PENDING 1
176 #define FBH_QUARANTINE_STARTED 2
177 uint8_t volatile fbh_quarantined_state
;
178 bool fbh_quarantined
;
180 uint64_t fbh_unused
[0];
181 } OS_ALIGNED(FIREHOSE_CHUNK_SIZE
) *firehose_buffer_header_t
;
183 union firehose_buffer_u
{
184 struct firehose_buffer_header_s fb_header
;
185 struct firehose_chunk_s fb_chunks
[FIREHOSE_BUFFER_CHUNK_COUNT
];
188 // used to let the compiler pack these values in 1 or 2 registers
189 typedef struct firehose_tracepoint_query_s
{
192 firehose_stream_t stream
;
197 } *firehose_tracepoint_query_t
;
199 #ifndef FIREHOSE_SERVER
202 firehose_buffer_create(mach_port_t logd_port
, uint64_t unique_pid
,
203 unsigned long bank_flags
);
205 firehose_tracepoint_t
206 firehose_buffer_tracepoint_reserve_slow(firehose_buffer_t fb
,
207 firehose_tracepoint_query_t ask
, uint8_t **privptr
);
210 firehose_buffer_update_limits(firehose_buffer_t fb
);
213 firehose_buffer_ring_enqueue(firehose_buffer_t fb
, uint16_t ref
);
216 firehose_buffer_force_connect(firehose_buffer_t fb
);
220 #endif // __FIREHOSE_BUFFER_INTERNAL__