[apple/xnu.git] / libkern / firehose / chunk_private.h

/*
 * Copyright (c) 2016 Apple Inc. All rights reserved.
 *
 * @APPLE_APACHE_LICENSE_HEADER_START@
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * @APPLE_APACHE_LICENSE_HEADER_END@
 */

#ifndef __FIREHOSE_CHUNK_PRIVATE__
#define __FIREHOSE_CHUNK_PRIVATE__

#include <sys/param.h>
#include "firehose_types_private.h"
#include "tracepoint_private.h"

__BEGIN_DECLS

#define FIREHOSE_CHUNK_SIZE                     4096ul

#define FIREHOSE_CHUNK_POS_ENTRY_OFFS_INC       (1ULL <<  0)
#define FIREHOSE_CHUNK_POS_PRIVATE_OFFS_INC     (1ULL << 16)
#define FIREHOSE_CHUNK_POS_REFCNT_INC           (1ULL << 32)
#define FIREHOSE_CHUNK_POS_FULL_BIT             (1ULL << 56)
#define FIREHOSE_CHUNK_POS_USABLE_FOR_STREAM(pos, stream) \
	        ((((pos).fcp_pos >> 48) & 0x1ff) == (uint16_t)stream)

typedef union {
	os_atomic(uint64_t) fcp_atomic_pos;
	uint64_t fcp_pos;
	struct {
		uint16_t fcp_next_entry_offs;
		uint16_t fcp_private_offs;
		uint8_t  fcp_refcnt;
		uint8_t  fcp_qos;
		uint8_t  fcp_stream;
		uint8_t  fcp_flag_full : 1;
		uint8_t  fcp_flag_io : 1;
		uint8_t  fcp_quarantined : 1;
		uint8_t  _fcp_flag_unused : 5;
	};
} firehose_chunk_pos_u;

typedef struct firehose_chunk_s {
	uint8_t  fc_start[0];
	firehose_chunk_pos_u fc_pos;
	uint64_t fc_timestamp;
	uint8_t  fc_data[FIREHOSE_CHUNK_SIZE - 8 - 8];
} *firehose_chunk_t;

typedef struct firehose_chunk_range_s {
	uint16_t fcr_offset; // offset from the start of the chunk
	uint16_t fcr_length;
} *firehose_chunk_range_t;

#if defined(KERNEL) || defined(OS_FIREHOSE_SPI)

OS_ALWAYS_INLINE
static inline bool
firehose_chunk_pos_fits(firehose_chunk_pos_u *pos, uint16_t size)
{
	return pos->fcp_next_entry_offs + size <= pos->fcp_private_offs;
}

#define FIREHOSE_CHUNK_TRY_RESERVE_FAIL_ENQUEUE  (-1)
#define FIREHOSE_CHUNK_TRY_RESERVE_FAIL          ( 0)

#if OS_ATOMIC_HAS_STARVATION_FREE_RMW || !OS_ATOMIC_CONFIG_STARVATION_FREE_ONLY
OS_ALWAYS_INLINE
static inline long
firehose_chunk_tracepoint_try_reserve(firehose_chunk_t fc, uint64_t stamp,
    firehose_stream_t stream, uint8_t qos, uint16_t pubsize,
    uint16_t privsize, uint8_t **privptr)
{
	const uint16_t ft_size = offsetof(struct firehose_tracepoint_s, ft_data);
	firehose_chunk_pos_u orig, pos;
	bool reservation_failed, stamp_delta_fits;

	stamp_delta_fits = ((stamp - fc->fc_timestamp) >> 48) == 0;

	// no acquire barrier because the returned space is written to only
	os_atomic_rmw_loop(&fc->fc_pos.fcp_atomic_pos,
	    orig.fcp_pos, pos.fcp_pos, relaxed, {
		if (orig.fcp_pos == 0) {
		        // we acquired a really really old reference, and we probably
		        // just faulted in a new page
		        os_atomic_rmw_loop_give_up(return FIREHOSE_CHUNK_TRY_RESERVE_FAIL);
		}
		if (!FIREHOSE_CHUNK_POS_USABLE_FOR_STREAM(orig, stream)) {
		        // nothing to do if the chunk is full, or the stream doesn't match,
		        // in which case the thread probably:
		        // - loaded the chunk ref
		        // - been suspended a long while
		        // - read the chunk to find a very old thing
		        os_atomic_rmw_loop_give_up(return FIREHOSE_CHUNK_TRY_RESERVE_FAIL);
		}
		pos.fcp_pos = orig.fcp_pos;
		if (!firehose_chunk_pos_fits(&orig,
		ft_size + pubsize + privsize) || !stamp_delta_fits) {
		        pos.fcp_flag_full = true;
		        reservation_failed = true;
		} else {
		        if (qos > pos.fcp_qos) {
		                pos.fcp_qos = qos;
			}
		        // using these *_INC macros is so that the compiler generates better
		        // assembly: using the struct individual fields forces the compiler
		        // to handle carry propagations, and we know it won't happen
		        pos.fcp_pos += roundup(ft_size + pubsize, 8) *
		        FIREHOSE_CHUNK_POS_ENTRY_OFFS_INC;
		        pos.fcp_pos -= privsize * FIREHOSE_CHUNK_POS_PRIVATE_OFFS_INC;
		        pos.fcp_pos += FIREHOSE_CHUNK_POS_REFCNT_INC;
		        const uint16_t minimum_payload_size = 16;
		        if (!firehose_chunk_pos_fits(&pos,
		        roundup(ft_size + minimum_payload_size, 8))) {
		                // if we can't even have minimum_payload_size bytes of payload
		                // for the next tracepoint, just flush right away
		                pos.fcp_flag_full = true;
			}
		        reservation_failed = false;
		}
	});

	if (reservation_failed) {
		if (pos.fcp_refcnt) {
			// nothing to do, there is a thread writing that will pick up
			// the "FULL" flag on flush and push as a consequence
			return FIREHOSE_CHUNK_TRY_RESERVE_FAIL;
		}
		// caller must enqueue chunk
		return FIREHOSE_CHUNK_TRY_RESERVE_FAIL_ENQUEUE;
	}
	if (privptr) {
		*privptr = (uint8_t *)((uintptr_t)fc->fc_start + pos.fcp_private_offs);
	}
	return orig.fcp_next_entry_offs;
}

OS_ALWAYS_INLINE
static inline firehose_tracepoint_t
firehose_chunk_tracepoint_begin(firehose_chunk_t fc, uint64_t stamp,
    uint16_t pubsize, uint64_t thread_id, long offset)
{
	firehose_tracepoint_t ft = (firehose_tracepoint_t)
	    __builtin_assume_aligned((void *)((uintptr_t)fc->fc_start + (uintptr_t)offset), 8);
	stamp -= fc->fc_timestamp;
	stamp |= (uint64_t)pubsize << 48;
	// The compiler barrier is needed for userland process death handling, see
	// (tracepoint-begin) in libdispatch's firehose_buffer_stream_chunk_install.
	os_atomic_std(atomic_store_explicit)(&ft->ft_atomic_stamp_and_length, stamp,
	    os_atomic_std(memory_order_relaxed));
	__asm__ __volatile__ ("" ::: "memory");
	ft->ft_thread = thread_id;
	return ft;
}

OS_ALWAYS_INLINE
static inline bool
firehose_chunk_tracepoint_end(firehose_chunk_t fc,
    firehose_tracepoint_t ft, firehose_tracepoint_id_u ftid)
{
	firehose_chunk_pos_u pos;

	os_atomic_std(atomic_store_explicit)(&ft->ft_id.ftid_atomic_value,
	    ftid.ftid_value, os_atomic_std(memory_order_release));
	pos.fcp_pos = os_atomic_std(atomic_fetch_sub_explicit)(&fc->fc_pos.fcp_atomic_pos,
	    FIREHOSE_CHUNK_POS_REFCNT_INC, os_atomic_std(memory_order_relaxed));
	return pos.fcp_refcnt == 1 && pos.fcp_flag_full;
}
#endif // OS_ATOMIC_HAS_STARVATION_FREE_RMW || !OS_ATOMIC_CONFIG_STARVATION_FREE_ONLY

#endif // defined(KERNEL) || defined(OS_FIREHOSE_SPI)

__END_DECLS

#endif // __FIREHOSE_CHUNK_PRIVATE__
Commit	Line	Data
813fb2f6 A	1	/*
	2	* Copyright (c) 2016 Apple Inc. All rights reserved.
	3	*
	4	* @APPLE_APACHE_LICENSE_HEADER_START@
	5	*
	6	* Licensed under the Apache License, Version 2.0 (the "License");
	7	* you may not use this file except in compliance with the License.
	8	* You may obtain a copy of the License at
	9	*
	10	* http://www.apache.org/licenses/LICENSE-2.0
	11	*
	12	* Unless required by applicable law or agreed to in writing, software
	13	* distributed under the License is distributed on an "AS IS" BASIS,
	14	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	15	* See the License for the specific language governing permissions and
	16	* limitations under the License.
	17	*
	18	* @APPLE_APACHE_LICENSE_HEADER_END@
	19	*/
	20
	21	#ifndef __FIREHOSE_CHUNK_PRIVATE__
	22	#define __FIREHOSE_CHUNK_PRIVATE__
	23
813fb2f6	24	#include <sys/param.h>
813fb2f6 A	25	#include "firehose_types_private.h"
	26	#include "tracepoint_private.h"
	27
	28	__BEGIN_DECLS
	29
	30	#define FIREHOSE_CHUNK_SIZE 4096ul
	31
	32	#define FIREHOSE_CHUNK_POS_ENTRY_OFFS_INC (1ULL << 0)
	33	#define FIREHOSE_CHUNK_POS_PRIVATE_OFFS_INC (1ULL << 16)
	34	#define FIREHOSE_CHUNK_POS_REFCNT_INC (1ULL << 32)
	35	#define FIREHOSE_CHUNK_POS_FULL_BIT (1ULL << 56)
	36	#define FIREHOSE_CHUNK_POS_USABLE_FOR_STREAM(pos, stream) \
0a7de745	37	((((pos).fcp_pos >> 48) & 0x1ff) == (uint16_t)stream)
813fb2f6 A	38
813fb2f6 A	39	typedef union {
f427ee49	40	os_atomic(uint64_t) fcp_atomic_pos;
813fb2f6 A	41	uint64_t fcp_pos;
	42	struct {
	43	uint16_t fcp_next_entry_offs;
	44	uint16_t fcp_private_offs;
	45	uint8_t fcp_refcnt;
	46	uint8_t fcp_qos;
	47	uint8_t fcp_stream;
	48	uint8_t fcp_flag_full : 1;
	49	uint8_t fcp_flag_io : 1;
5ba3f43e A	50	uint8_t fcp_quarantined : 1;
5ba3f43e A	51	uint8_t _fcp_flag_unused : 5;
813fb2f6 A	52	};
	53	} firehose_chunk_pos_u;
	54
	55	typedef struct firehose_chunk_s {
	56	uint8_t fc_start[0];
	57	firehose_chunk_pos_u fc_pos;
	58	uint64_t fc_timestamp;
	59	uint8_t fc_data[FIREHOSE_CHUNK_SIZE - 8 - 8];
	60	} *firehose_chunk_t;
	61
	62	typedef struct firehose_chunk_range_s {
	63	uint16_t fcr_offset; // offset from the start of the chunk
	64	uint16_t fcr_length;
	65	} *firehose_chunk_range_t;
	66
	67	#if defined(KERNEL) \|\| defined(OS_FIREHOSE_SPI)
	68
	69	OS_ALWAYS_INLINE
	70	static inline bool
f427ee49	71	firehose_chunk_pos_fits(firehose_chunk_pos_u *pos, uint16_t size)
813fb2f6	72	{
f427ee49	73	return pos->fcp_next_entry_offs + size <= pos->fcp_private_offs;
813fb2f6 A	74	}
	75
	76	#define FIREHOSE_CHUNK_TRY_RESERVE_FAIL_ENQUEUE (-1)
	77	#define FIREHOSE_CHUNK_TRY_RESERVE_FAIL ( 0)
	78
f427ee49	79	#if OS_ATOMIC_HAS_STARVATION_FREE_RMW \|\| !OS_ATOMIC_CONFIG_STARVATION_FREE_ONLY
813fb2f6 A	80	OS_ALWAYS_INLINE
	81	static inline long
	82	firehose_chunk_tracepoint_try_reserve(firehose_chunk_t fc, uint64_t stamp,
0a7de745 A	83	firehose_stream_t stream, uint8_t qos, uint16_t pubsize,
0a7de745 A	84	uint16_t privsize, uint8_t **privptr)
813fb2f6 A	85	{
	86	const uint16_t ft_size = offsetof(struct firehose_tracepoint_s, ft_data);
	87	firehose_chunk_pos_u orig, pos;
	88	bool reservation_failed, stamp_delta_fits;
	89
	90	stamp_delta_fits = ((stamp - fc->fc_timestamp) >> 48) == 0;
	91
	92	// no acquire barrier because the returned space is written to only
	93	os_atomic_rmw_loop(&fc->fc_pos.fcp_atomic_pos,
0a7de745	94	orig.fcp_pos, pos.fcp_pos, relaxed, {
813fb2f6	95	if (orig.fcp_pos == 0) {
0a7de745 A	96	// we acquired a really really old reference, and we probably
	97	// just faulted in a new page
	98	os_atomic_rmw_loop_give_up(return FIREHOSE_CHUNK_TRY_RESERVE_FAIL);
813fb2f6 A	99	}
813fb2f6 A	100	if (!FIREHOSE_CHUNK_POS_USABLE_FOR_STREAM(orig, stream)) {
0a7de745 A	101	// nothing to do if the chunk is full, or the stream doesn't match,
	102	// in which case the thread probably:
	103	// - loaded the chunk ref
	104	// - been suspended a long while
	105	// - read the chunk to find a very old thing
	106	os_atomic_rmw_loop_give_up(return FIREHOSE_CHUNK_TRY_RESERVE_FAIL);
813fb2f6	107	}
f427ee49 A	108	pos.fcp_pos = orig.fcp_pos;
f427ee49 A	109	if (!firehose_chunk_pos_fits(&orig,
0a7de745 A	110	ft_size + pubsize + privsize) \|\| !stamp_delta_fits) {
	111	pos.fcp_flag_full = true;
	112	reservation_failed = true;
813fb2f6	113	} else {
0a7de745 A	114	if (qos > pos.fcp_qos) {
0a7de745 A	115	pos.fcp_qos = qos;
813fb2f6	116	}
0a7de745 A	117	// using these *_INC macros is so that the compiler generates better
	118	// assembly: using the struct individual fields forces the compiler
	119	// to handle carry propagations, and we know it won't happen
	120	pos.fcp_pos += roundup(ft_size + pubsize, 8) *
	121	FIREHOSE_CHUNK_POS_ENTRY_OFFS_INC;
	122	pos.fcp_pos -= privsize * FIREHOSE_CHUNK_POS_PRIVATE_OFFS_INC;
	123	pos.fcp_pos += FIREHOSE_CHUNK_POS_REFCNT_INC;
	124	const uint16_t minimum_payload_size = 16;
f427ee49	125	if (!firehose_chunk_pos_fits(&pos,
0a7de745 A	126	roundup(ft_size + minimum_payload_size, 8))) {
	127	// if we can't even have minimum_payload_size bytes of payload
	128	// for the next tracepoint, just flush right away
	129	pos.fcp_flag_full = true;
	130	}
	131	reservation_failed = false;
813fb2f6 A	132	}
	133	});
	134
	135	if (reservation_failed) {
	136	if (pos.fcp_refcnt) {
	137	// nothing to do, there is a thread writing that will pick up
	138	// the "FULL" flag on flush and push as a consequence
	139	return FIREHOSE_CHUNK_TRY_RESERVE_FAIL;
	140	}
	141	// caller must enqueue chunk
	142	return FIREHOSE_CHUNK_TRY_RESERVE_FAIL_ENQUEUE;
	143	}
	144	if (privptr) {
cb323159	145	privptr = (uint8_t )((uintptr_t)fc->fc_start + pos.fcp_private_offs);
813fb2f6 A	146	}
	147	return orig.fcp_next_entry_offs;
	148	}
	149
	150	OS_ALWAYS_INLINE
	151	static inline firehose_tracepoint_t
	152	firehose_chunk_tracepoint_begin(firehose_chunk_t fc, uint64_t stamp,
0a7de745	153	uint16_t pubsize, uint64_t thread_id, long offset)
813fb2f6 A	154	{
813fb2f6 A	155	firehose_tracepoint_t ft = (firehose_tracepoint_t)
cb323159	156	__builtin_assume_aligned((void *)((uintptr_t)fc->fc_start + (uintptr_t)offset), 8);
813fb2f6 A	157	stamp -= fc->fc_timestamp;
	158	stamp \|= (uint64_t)pubsize << 48;
	159	// The compiler barrier is needed for userland process death handling, see
	160	// (tracepoint-begin) in libdispatch's firehose_buffer_stream_chunk_install.
f427ee49 A	161	os_atomic_std(atomic_store_explicit)(&ft->ft_atomic_stamp_and_length, stamp,
f427ee49 A	162	os_atomic_std(memory_order_relaxed));
0a7de745	163	__asm__ __volatile__ ("" ::: "memory");
813fb2f6 A	164	ft->ft_thread = thread_id;
	165	return ft;
	166	}
	167
	168	OS_ALWAYS_INLINE
	169	static inline bool
	170	firehose_chunk_tracepoint_end(firehose_chunk_t fc,
0a7de745	171	firehose_tracepoint_t ft, firehose_tracepoint_id_u ftid)
813fb2f6 A	172	{
	173	firehose_chunk_pos_u pos;
	174
f427ee49 A	175	os_atomic_std(atomic_store_explicit)(&ft->ft_id.ftid_atomic_value,
	176	ftid.ftid_value, os_atomic_std(memory_order_release));
	177	pos.fcp_pos = os_atomic_std(atomic_fetch_sub_explicit)(&fc->fc_pos.fcp_atomic_pos,
	178	FIREHOSE_CHUNK_POS_REFCNT_INC, os_atomic_std(memory_order_relaxed));
813fb2f6 A	179	return pos.fcp_refcnt == 1 && pos.fcp_flag_full;
813fb2f6 A	180	}
f427ee49	181	#endif // OS_ATOMIC_HAS_STARVATION_FREE_RMW \|\| !OS_ATOMIC_CONFIG_STARVATION_FREE_ONLY
813fb2f6 A	182
	183	#endif // defined(KERNEL) \|\| defined(OS_FIREHOSE_SPI)
	184
	185	__END_DECLS
	186
	187	#endif // __FIREHOSE_CHUNK_PRIVATE__