+DISPATCH_ALWAYS_INLINE
+static inline bool
+_dispatch_data_leaf(struct dispatch_data_s *dd)
+{
+ return dd->num_records == 0;
+}
+
+/*
+ * This is about the number of records required to hold that dispatch data
+ * if it's not a leaf. Callers either want that value, or have to special
+ * case the case when the dispatch data *is* a leaf before (and that the actual
+ * embedded record count of that dispatch data is 0)
+ */
+DISPATCH_ALWAYS_INLINE
+static inline size_t
+_dispatch_data_num_records(struct dispatch_data_s *dd)
+{
+ return dd->num_records ?: 1;
+}
+
+typedef dispatch_data_t (*dispatch_transform_t)(dispatch_data_t data);
+
+struct dispatch_data_format_type_s {
+ uint64_t type;
+ uint64_t input_mask;
+ uint64_t output_mask;
+ dispatch_transform_t decode;
+ dispatch_transform_t encode;
+};
+
+void _dispatch_data_init_with_bytes(dispatch_data_t data, const void *buffer,
+ size_t size, dispatch_block_t destructor);
+void _dispatch_data_dispose(dispatch_data_t data, bool *allow_free);
+void _dispatch_data_set_target_queue(struct dispatch_data_s *dd,
+ dispatch_queue_t tq);
+size_t _dispatch_data_debug(dispatch_data_t data, char* buf, size_t bufsiz);
+const void* _dispatch_data_get_flattened_bytes(struct dispatch_data_s *dd);
+
+#if !defined(__cplusplus)
+extern const dispatch_block_t _dispatch_data_destructor_inline;
+#define DISPATCH_DATA_DESTRUCTOR_INLINE (_dispatch_data_destructor_inline)
+
+/*
+ * the out parameters are about seeing "through" trivial subranges
+ * so for something like this: dd = { subrange [ dd1, offset1 ] },
+ * this will return { dd1, offset + offset1 }
+ *
+ * If the dispatch object isn't a trivial subrange, it returns { dd, offset }
+ */
+DISPATCH_ALWAYS_INLINE
+static inline const void*
+_dispatch_data_map_direct(struct dispatch_data_s *dd, size_t offset,
+ struct dispatch_data_s **dd_out, size_t *from_out)
+{
+ const void *buffer = NULL;
+
+ dispatch_assert(dd->size);
+ if (slowpath(!_dispatch_data_leaf(dd)) &&
+ _dispatch_data_num_records(dd) == 1) {
+ offset += dd->records[0].from;
+ dd = (struct dispatch_data_s *)dd->records[0].data_object;
+ }
+
+ if (fastpath(_dispatch_data_leaf(dd))) {
+ buffer = dd->buf + offset;
+ } else {
+ buffer = os_atomic_load((void **)&dd->buf, relaxed);
+ if (buffer) {
+ buffer += offset;
+ }
+ }
+ if (dd_out) *dd_out = dd;
+ if (from_out) *from_out = offset;
+ return buffer;
+}
+
+#endif // !defined(__cplusplus)
+