src/firehose/firehose_inline_internal.h

   1 /*
   2  * Copyright (c) 2015 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_APACHE_LICENSE_HEADER_START@
   5  *
   6  * Licensed under the Apache License, Version 2.0 (the "License");
   7  * you may not use this file except in compliance with the License.
   8  * You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  *
  18  * @APPLE_APACHE_LICENSE_HEADER_END@
  19  */
  20
  21 #ifndef __FIREHOSE_INLINE_INTERNAL__
  22 #define __FIREHOSE_INLINE_INTERNAL__
  23
  24 #define firehose_atomic_maxv2o(p, f, v, o, m) \
  25                 os_atomic_rmw_loop2o(p, f, *(o), (v), m, { \
  26                         if (*(o) >= (v)) os_atomic_rmw_loop_give_up(break); \
  27                 })
  28
  29 #define firehose_atomic_max2o(p, f, v, m)   ({ \
  30                 typeof((p)->f) _old; \
  31                 firehose_atomic_maxv2o(p, f, v, &_old, m); \
  32         })
  33
  34 #ifndef KERNEL
  35 // caller must test for non zero first
  36 OS_ALWAYS_INLINE
  37 static inline uint16_t
  38 firehose_bitmap_first_set(uint64_t bitmap)
  39 {
  40         dispatch_assert(bitmap != 0);
  41         // this builtin returns 0 if bitmap is 0, or (first bit set + 1)
  42         return (uint16_t)__builtin_ffsll((long long)bitmap) - 1;
  43 }
  44 #endif
  45
  46 #pragma mark -
  47 #pragma mark Mach Misc.
  48 #ifndef KERNEL
  49
  50 OS_ALWAYS_INLINE
  51 static inline mach_port_t
  52 firehose_mach_port_allocate(uint32_t flags, void *ctx)
  53 {
  54         mach_port_t port = MACH_PORT_NULL;
  55         mach_port_options_t opts = {
  56                 .flags = flags,
  57         };
  58         kern_return_t kr;
  59
  60         for (;;) {
  61                 kr = mach_port_construct(mach_task_self(), &opts,
  62                                 (mach_port_context_t)ctx, &port);
  63                 if (fastpath(kr == KERN_SUCCESS)) {
  64                         break;
  65                 }
  66                 DISPATCH_VERIFY_MIG(kr);
  67                 dispatch_assume_zero(kr);
  68                 _dispatch_temporary_resource_shortage();
  69         }
  70         return port;
  71 }
  72
  73 OS_ALWAYS_INLINE
  74 static inline kern_return_t
  75 firehose_mach_port_recv_dispose(mach_port_t port, void *ctx)
  76 {
  77         kern_return_t kr;
  78         kr = mach_port_destruct(mach_task_self(), port, 0,
  79                         (mach_port_context_t)ctx);
  80         DISPATCH_VERIFY_MIG(kr);
  81         return kr;
  82 }
  83
  84 OS_ALWAYS_INLINE
  85 static inline void
  86 firehose_mach_port_send_release(mach_port_t port)
  87 {
  88         kern_return_t kr = mach_port_deallocate(mach_task_self(), port);
  89         DISPATCH_VERIFY_MIG(kr);
  90         dispatch_assume_zero(kr);
  91 }
  92
  93 OS_ALWAYS_INLINE
  94 static inline void
  95 firehose_mach_port_guard(mach_port_t port, bool strict, void *ctx)
  96 {
  97         kern_return_t kr = mach_port_guard(mach_task_self(), port,
  98                         (mach_port_context_t)ctx, strict);
  99         DISPATCH_VERIFY_MIG(kr);
 100         dispatch_assume_zero(kr);
 101 }
 102
 103 OS_ALWAYS_INLINE
 104 static inline void
 105 firehose_mig_server(dispatch_mig_callback_t demux, size_t maxmsgsz,
 106                 mach_msg_header_t *hdr)
 107 {
 108         mig_reply_error_t *msg_reply = (mig_reply_error_t *)alloca(maxmsgsz);
 109         kern_return_t rc = KERN_SUCCESS;
 110         bool expects_reply = false;
 111
 112         if (MACH_MSGH_BITS_REMOTE(hdr->msgh_bits) == MACH_MSG_TYPE_MOVE_SEND_ONCE) {
 113                 expects_reply = true;
 114         }
 115
 116         if (!fastpath(demux(hdr, &msg_reply->Head))) {
 117                 rc = MIG_BAD_ID;
 118         } else if (msg_reply->Head.msgh_bits & MACH_MSGH_BITS_COMPLEX) {
 119                 rc = KERN_SUCCESS;
 120         } else {
 121                 // if MACH_MSGH_BITS_COMPLEX is _not_ set, then msg_reply->RetCode
 122                 // is present
 123                 rc = msg_reply->RetCode;
 124         }
 125
 126         if (slowpath(rc == KERN_SUCCESS && expects_reply)) {
 127                 // if crashing here, some handler returned KERN_SUCCESS
 128                 // hoping for firehose_mig_server to perform the mach_msg()
 129                 // call to reply, and it doesn't know how to do that
 130                 DISPATCH_INTERNAL_CRASH(msg_reply->Head.msgh_id,
 131                                 "firehose_mig_server doesn't handle replies");
 132         }
 133         if (slowpath(rc != KERN_SUCCESS && rc != MIG_NO_REPLY)) {
 134                 // destroy the request - but not the reply port
 135                 hdr->msgh_remote_port = 0;
 136                 mach_msg_destroy(hdr);
 137         }
 138 }
 139
 140 #endif // !KERNEL
 141 #pragma mark -
 142 #pragma mark firehose buffer
 143
 144 OS_ALWAYS_INLINE
 145 static inline firehose_chunk_t
 146 firehose_buffer_chunk_for_address(void *addr)
 147 {
 148         uintptr_t chunk_addr = (uintptr_t)addr & ~(FIREHOSE_CHUNK_SIZE - 1);
 149         return (firehose_chunk_t)chunk_addr;
 150 }
 151
 152 OS_ALWAYS_INLINE
 153 static inline uint16_t
 154 firehose_buffer_chunk_to_ref(firehose_buffer_t fb, firehose_chunk_t fbc)
 155 {
 156         return (uint16_t)(fbc - fb->fb_chunks);
 157 }
 158
 159 OS_ALWAYS_INLINE
 160 static inline firehose_chunk_t
 161 firehose_buffer_ref_to_chunk(firehose_buffer_t fb, uint16_t ref)
 162 {
 163         return fb->fb_chunks + ref;
 164 }
 165
 166 #ifndef FIREHOSE_SERVER
 167 #if DISPATCH_PURE_C
 168
 169 OS_ALWAYS_INLINE
 170 static inline uint8_t
 171 firehose_buffer_qos_bits_propagate(void)
 172 {
 173 #ifndef KERNEL
 174         pthread_priority_t pp = _dispatch_priority_propagate();
 175
 176         pp &= _PTHREAD_PRIORITY_QOS_CLASS_MASK;
 177         return (uint8_t)(pp >> _PTHREAD_PRIORITY_QOS_CLASS_SHIFT);
 178 #else
 179         return 0;
 180 #endif
 181 }
 182
 183 OS_ALWAYS_INLINE
 184 static inline void
 185 firehose_buffer_stream_flush(firehose_buffer_t fb, firehose_stream_t stream)
 186 {
 187         firehose_buffer_stream_t fbs = &fb->fb_header.fbh_stream[stream];
 188         firehose_stream_state_u old_state, new_state;
 189         firehose_chunk_t fc;
 190         uint64_t stamp = UINT64_MAX; // will cause the reservation to fail
 191         uint16_t ref;
 192         long result;
 193
 194         old_state.fss_atomic_state =
 195                         os_atomic_load2o(fbs, fbs_state.fss_atomic_state, relaxed);
 196         ref = old_state.fss_current;
 197         if (!ref || ref == FIREHOSE_STREAM_STATE_PRISTINE) {
 198                 // there is no installed page, nothing to flush, go away
 199 #ifndef KERNEL
 200                 firehose_buffer_force_connect(fb);
 201 #endif
 202                 return;
 203         }
 204
 205         fc = firehose_buffer_ref_to_chunk(fb, old_state.fss_current);
 206         result = firehose_chunk_tracepoint_try_reserve(fc, stamp, stream,
 207                         firehose_buffer_qos_bits_propagate(), 1, 0, NULL);
 208         if (likely(result < 0)) {
 209                 firehose_buffer_ring_enqueue(fb, old_state.fss_current);
 210         }
 211         if (unlikely(result > 0)) {
 212                 // because we pass a silly stamp that requires a flush
 213                 DISPATCH_INTERNAL_CRASH(result, "Allocation should always fail");
 214         }
 215
 216         // as a best effort try to uninstall the page we just flushed
 217         // but failing is okay, let's not contend stupidly for something
 218         // allocators know how to handle in the first place
 219         new_state = old_state;
 220         new_state.fss_current = 0;
 221         (void)os_atomic_cmpxchg2o(fbs, fbs_state.fss_atomic_state,
 222                         old_state.fss_atomic_state, new_state.fss_atomic_state, relaxed);
 223 }
 224
 225 /**
 226  * @function firehose_buffer_tracepoint_reserve
 227  *
 228  * @abstract
 229  * Reserves space in the firehose buffer for the tracepoint with specified
 230  * characteristics.
 231  *
 232  * @discussion
 233  * This returns a slot, with the length of the tracepoint already set, so
 234  * that in case of a crash, we maximize our chance to be able to skip the
 235  * tracepoint in case of a partial write.
 236  *
 237  * Once the tracepoint has been written, firehose_buffer_tracepoint_flush()
 238  * must be called.
 239  *
 240  * @param fb
 241  * The buffer to allocate from.
 242  *
 243  * @param stream
 244  * The buffer stream to use.
 245  *
 246  * @param pubsize
 247  * The size of the public data for this tracepoint, cannot be 0, doesn't
 248  * take the size of the tracepoint header into account.
 249  *
 250  * @param privsize
 251  * The size of the private data for this tracepoint, can be 0.
 252  *
 253  * @param privptr
 254  * The pointer to the private buffer, can be NULL
 255  *
 256  * @result
 257  * The pointer to the tracepoint.
 258  */
 259 OS_ALWAYS_INLINE
 260 static inline firehose_tracepoint_t
 261 firehose_buffer_tracepoint_reserve(firehose_buffer_t fb, uint64_t stamp,
 262                 firehose_stream_t stream, uint16_t pubsize,
 263                 uint16_t privsize, uint8_t **privptr)
 264 {
 265         firehose_buffer_stream_t fbs = &fb->fb_header.fbh_stream[stream];
 266         firehose_stream_state_u old_state, new_state;
 267         firehose_chunk_t fc;
 268 #if KERNEL
 269         bool failable = false;
 270 #endif
 271         bool success;
 272         long result;
 273         uint16_t ref;
 274
 275         // cannot use os_atomic_rmw_loop2o, _page_try_reserve does a store
 276         old_state.fss_atomic_state =
 277                         os_atomic_load2o(fbs, fbs_state.fss_atomic_state, relaxed);
 278         for (;;) {
 279                 new_state = old_state;
 280
 281                 ref = old_state.fss_current;
 282                 if (likely(ref && ref != FIREHOSE_STREAM_STATE_PRISTINE)) {
 283                         fc = firehose_buffer_ref_to_chunk(fb, ref);
 284                         result = firehose_chunk_tracepoint_try_reserve(fc, stamp, stream,
 285                                         firehose_buffer_qos_bits_propagate(),
 286                                         pubsize, privsize, privptr);
 287                         if (likely(result > 0)) {
 288                                 uint64_t thread;
 289 #ifdef KERNEL
 290                                 thread = thread_tid(current_thread());
 291 #else
 292                                 thread = _pthread_threadid_self_np_direct();
 293 #endif
 294                                 return firehose_chunk_tracepoint_begin(fc,
 295                                                 stamp, pubsize, thread, result);
 296                         }
 297                         if (likely(result < 0)) {
 298                                 firehose_buffer_ring_enqueue(fb, old_state.fss_current);
 299                         }
 300                         new_state.fss_current = 0;
 301                 }
 302 #if KERNEL
 303                 if (failable) {
 304                         return NULL;
 305                 }
 306 #endif
 307
 308                 if (unlikely(old_state.fss_allocator)) {
 309                         _dispatch_gate_wait(&fbs->fbs_state.fss_gate,
 310                                         DLOCK_LOCK_DATA_CONTENTION);
 311                         old_state.fss_atomic_state =
 312                                         os_atomic_load2o(fbs, fbs_state.fss_atomic_state, relaxed);
 313 #if KERNEL
 314                         failable = true;
 315 #endif
 316                         continue;
 317                 }
 318
 319                 // if the thread doing the allocation is a low priority one
 320                 // we may starve high priority ones.
 321                 // so disable preemption before we become an allocator
 322                 // the reenabling of the preemption is in
 323                 // firehose_buffer_stream_chunk_install
 324                 __firehose_critical_region_enter();
 325 #if KERNEL
 326                 new_state.fss_allocator = (uint32_t)cpu_number();
 327 #else
 328                 new_state.fss_allocator = _dispatch_tid_self();
 329 #endif
 330                 success = os_atomic_cmpxchgvw2o(fbs, fbs_state.fss_atomic_state,
 331                                 old_state.fss_atomic_state, new_state.fss_atomic_state,
 332                                 &old_state.fss_atomic_state, relaxed);
 333                 if (likely(success)) {
 334                         break;
 335                 }
 336                 __firehose_critical_region_leave();
 337         }
 338
 339         struct firehose_tracepoint_query_s ask = {
 340                 .pubsize = pubsize,
 341                 .privsize = privsize,
 342                 .stream = stream,
 343                 .for_io = (firehose_stream_uses_io_bank & (1UL << stream)) != 0,
 344                 .stamp = stamp,
 345         };
 346         return firehose_buffer_tracepoint_reserve_slow(fb, &ask, privptr);
 347 }
 348
 349 /**
 350  * @function firehose_buffer_tracepoint_flush
 351  *
 352  * @abstract
 353  * Flushes a firehose tracepoint, and sends the chunk to the daemon when full
 354  * and this was the last tracepoint writer for this chunk.
 355  *
 356  * @param fb
 357  * The buffer the tracepoint belongs to.
 358  *
 359  * @param ft
 360  * The tracepoint to flush.
 361  *
 362  * @param ftid
 363  * The firehose tracepoint ID for that tracepoint.
 364  * It is written last, preventing compiler reordering, so that its absence
 365  * on crash recovery means the tracepoint is partial.
 366  */
 367 OS_ALWAYS_INLINE
 368 static inline void
 369 firehose_buffer_tracepoint_flush(firehose_buffer_t fb,
 370                 firehose_tracepoint_t ft, firehose_tracepoint_id_u ftid)
 371 {
 372         firehose_chunk_t fc = firehose_buffer_chunk_for_address(ft);
 373
 374         // Needed for process death handling (tracepoint-flush):
 375         // We want to make sure the observers
 376         // will see memory effects in program (asm) order.
 377         // 1. write all the data to the tracepoint
 378         // 2. write the tracepoint ID, so that seeing it means the tracepoint
 379         //    is valid
 380         if (firehose_chunk_tracepoint_end(fc, ft, ftid)) {
 381                 firehose_buffer_ring_enqueue(fb, firehose_buffer_chunk_to_ref(fb, fc));
 382         }
 383 }
 384
 385 #ifndef KERNEL
 386 OS_ALWAYS_INLINE
 387 static inline void
 388 firehose_buffer_clear_bank_flags(firehose_buffer_t fb, unsigned long bits)
 389 {
 390         firehose_buffer_bank_t fbb = &fb->fb_header.fbh_bank;
 391         unsigned long orig_flags;
 392
 393         orig_flags = os_atomic_and_orig2o(fbb, fbb_flags, ~bits, relaxed);
 394         if (orig_flags != (orig_flags & ~bits)) {
 395                 firehose_buffer_update_limits(fb);
 396         }
 397 }
 398
 399 OS_ALWAYS_INLINE
 400 static inline void
 401 firehose_buffer_set_bank_flags(firehose_buffer_t fb, unsigned long bits)
 402 {
 403         firehose_buffer_bank_t fbb = &fb->fb_header.fbh_bank;
 404         unsigned long orig_flags;
 405
 406         orig_flags = os_atomic_or_orig2o(fbb, fbb_flags, bits, relaxed);
 407         if (orig_flags != (orig_flags | bits)) {
 408                 firehose_buffer_update_limits(fb);
 409         }
 410 }
 411 #endif // !KERNEL
 412
 413 #endif // !defined(FIREHOSE_SERVER)
 414
 415 #endif // DISPATCH_PURE_C
 416
 417 #endif // __FIREHOSE_INLINE_INTERNAL__