src/event/event_internal.h

   1 /*
   2  * Copyright (c) 2008-2016 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_APACHE_LICENSE_HEADER_START@
   5  *
   6  * Licensed under the Apache License, Version 2.0 (the "License");
   7  * you may not use this file except in compliance with the License.
   8  * You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  *
  18  * @APPLE_APACHE_LICENSE_HEADER_END@
  19  */
  20
  21 /*
  22  * IMPORTANT: This header file describes INTERNAL interfaces to libdispatch
  23  * which are subject to change in future releases of Mac OS X. Any applications
  24  * relying on these interfaces WILL break.
  25  */
  26
  27 #ifndef __DISPATCH_EVENT_EVENT_INTERNAL__
  28 #define __DISPATCH_EVENT_EVENT_INTERNAL__
  29
  30 #include "event_config.h"
  31
  32 struct dispatch_sync_context_s;
  33 typedef struct dispatch_wlh_s *dispatch_wlh_t; // opaque handle
  34 #define DISPATCH_WLH_ANON       ((dispatch_wlh_t)(void*)(~0ul))
  35 #define DISPATCH_WLH_MANAGER    ((dispatch_wlh_t)(void*)(~2ul))
  36
  37 #define DISPATCH_UNOTE_DATA_ACTION_SIZE 2
  38
  39 #define DISPATCH_UNOTE_CLASS_HEADER() \
  40         dispatch_source_type_t du_type; \
  41         uintptr_t du_owner_wref; /* "weak" back reference to the owner object */ \
  42         dispatch_wlh_t du_wlh; \
  43         uint32_t  du_ident; \
  44         int8_t    du_filter; \
  45         os_atomic(bool) dmsr_notification_armed; \
  46         uint16_t  du_data_action : DISPATCH_UNOTE_DATA_ACTION_SIZE; \
  47         uint16_t  du_is_direct : 1; \
  48         uint16_t  du_is_timer : 1; \
  49         uint16_t  du_memorypressure_override : 1; \
  50         uint16_t  du_vmpressure_override : 1; \
  51         uint16_t  du_can_be_wlh : 1; \
  52         uint16_t  dmr_async_reply : 1; \
  53         uint16_t  dmrr_handler_is_block : 1; \
  54         uint16_t  du_unused : 7; \
  55         uint32_t  du_fflags; \
  56         dispatch_priority_t du_priority
  57
  58 #define _dispatch_ptr2wref(ptr) (~(uintptr_t)(ptr))
  59 #define _dispatch_wref2ptr(ref) ((void*)~(ref))
  60 #define _dispatch_source_from_refs(dr) \
  61                 ((dispatch_source_t)_dispatch_wref2ptr((dr)->du_owner_wref))
  62
  63 DISPATCH_ENUM(dispatch_unote_action, uint8_t,
  64     DISPATCH_UNOTE_ACTION_DATA_OR = 0,
  65     DISPATCH_UNOTE_ACTION_DATA_OR_STATUS_SET,
  66     DISPATCH_UNOTE_ACTION_DATA_SET,
  67     DISPATCH_UNOTE_ACTION_DATA_ADD,
  68         DISPATCH_UNOTE_ACTION_LAST = DISPATCH_UNOTE_ACTION_DATA_ADD
  69 );
  70 _Static_assert(DISPATCH_UNOTE_ACTION_LAST <
  71                 (1 << DISPATCH_UNOTE_DATA_ACTION_SIZE),
  72                 "DISPATCH_UNOTE_ACTION_LAST too large for du_data_action field");
  73
  74 typedef struct dispatch_unote_class_s {
  75         DISPATCH_UNOTE_CLASS_HEADER();
  76 } *dispatch_unote_class_t;
  77
  78
  79 enum {
  80         DS_EVENT_HANDLER = 0,
  81         DS_CANCEL_HANDLER,
  82         DS_REGISTN_HANDLER,
  83 };
  84
  85 #define DISPATCH_SOURCE_REFS_HEADER() \
  86         DISPATCH_UNOTE_CLASS_HEADER(); \
  87         struct dispatch_continuation_s *volatile ds_handler[3]
  88
  89 // Source state which may contain references to the source object
  90 // Separately allocated so that 'leaks' can see sources <rdar://problem/9050566>
  91 typedef struct dispatch_source_refs_s {
  92         DISPATCH_SOURCE_REFS_HEADER();
  93 } *dispatch_source_refs_t;
  94
  95 typedef struct dispatch_timer_delay_s {
  96         uint64_t delay, leeway;
  97 } dispatch_timer_delay_s;
  98
  99 #define DTH_INVALID_ID  (~0u)
 100 #define DTH_TARGET_ID   0u
 101 #define DTH_DEADLINE_ID 1u
 102 #define DTH_ID_COUNT    2u
 103
 104 typedef struct dispatch_timer_source_s {
 105         union {
 106                 struct {
 107                         uint64_t target;
 108                         uint64_t deadline;
 109                 };
 110                 uint64_t heap_key[DTH_ID_COUNT];
 111         };
 112         uint64_t interval;
 113 } *dispatch_timer_source_t;
 114
 115 typedef struct dispatch_timer_config_s {
 116         struct dispatch_timer_source_s dtc_timer;
 117         dispatch_clock_t dtc_clock;
 118 } *dispatch_timer_config_t;
 119
 120 typedef struct dispatch_timer_source_refs_s {
 121         DISPATCH_SOURCE_REFS_HEADER();
 122         struct dispatch_timer_source_s dt_timer;
 123         struct dispatch_timer_config_s *dt_pending_config;
 124         uint32_t dt_heap_entry[DTH_ID_COUNT];
 125 } *dispatch_timer_source_refs_t;
 126
 127 typedef struct dispatch_timer_heap_s {
 128         uint64_t dth_target, dth_deadline;
 129         uint32_t dth_count;
 130         uint16_t dth_segments;
 131 #define DTH_ARMED  1u
 132         uint16_t dth_flags;
 133         dispatch_timer_source_refs_t dth_min[DTH_ID_COUNT];
 134         void **dth_heap;
 135 } *dispatch_timer_heap_t;
 136
 137 #if HAVE_MACH
 138 #if DISPATCH_MACHPORT_DEBUG
 139 void dispatch_debug_machport(mach_port_t name, const char *str);
 140 #define _dispatch_debug_machport(name) \
 141                 dispatch_debug_machport((name), __func__)
 142 #else
 143 #define _dispatch_debug_machport(name) ((void)(name))
 144 #endif // DISPATCH_MACHPORT_DEBUG
 145
 146 // Mach channel state which may contain references to the channel object
 147 // layout must match dispatch_source_refs_s
 148 struct dispatch_mach_recv_refs_s {
 149         DISPATCH_UNOTE_CLASS_HEADER();
 150         dispatch_mach_handler_function_t dmrr_handler_func;
 151         void *dmrr_handler_ctxt;
 152 };
 153 typedef struct dispatch_mach_recv_refs_s *dispatch_mach_recv_refs_t;
 154
 155 struct dispatch_mach_reply_refs_s {
 156         DISPATCH_UNOTE_CLASS_HEADER();
 157         dispatch_priority_t dmr_priority;
 158         void *dmr_ctxt;
 159         voucher_t dmr_voucher;
 160         TAILQ_ENTRY(dispatch_mach_reply_refs_s) dmr_list;
 161         mach_port_t dmr_waiter_tid;
 162 };
 163 typedef struct dispatch_mach_reply_refs_s *dispatch_mach_reply_refs_t;
 164
 165 #define _DISPATCH_MACH_STATE_UNUSED_MASK        0xffffffa000000000ull
 166 #define DISPATCH_MACH_STATE_DIRTY               0x0000002000000000ull
 167 #define DISPATCH_MACH_STATE_PENDING_BARRIER     0x0000001000000000ull
 168 #define DISPATCH_MACH_STATE_RECEIVED_OVERRIDE   0x0000000800000000ull
 169 #define DISPATCH_MACH_STATE_MAX_QOS_MASK        0x0000000700000000ull
 170 #define DISPATCH_MACH_STATE_MAX_QOS_SHIFT       32
 171 #define DISPATCH_MACH_STATE_UNLOCK_MASK         0x00000000ffffffffull
 172
 173 struct dispatch_mach_send_refs_s {
 174         DISPATCH_UNOTE_CLASS_HEADER();
 175         dispatch_mach_msg_t dmsr_checkin;
 176         TAILQ_HEAD(, dispatch_mach_reply_refs_s) dmsr_replies;
 177         dispatch_unfair_lock_s dmsr_replies_lock;
 178 #define DISPATCH_MACH_DISCONNECT_MAGIC_BASE (0x80000000)
 179 #define DISPATCH_MACH_NEVER_INSTALLED (DISPATCH_MACH_DISCONNECT_MAGIC_BASE + 0)
 180 #define DISPATCH_MACH_NEVER_CONNECTED (DISPATCH_MACH_DISCONNECT_MAGIC_BASE + 1)
 181         uint32_t volatile dmsr_disconnect_cnt;
 182         DISPATCH_UNION_LE(uint64_t volatile dmsr_state,
 183                 dispatch_unfair_lock_s dmsr_state_lock,
 184                 uint32_t dmsr_state_bits
 185         ) DISPATCH_ATOMIC64_ALIGN;
 186         struct dispatch_object_s *volatile dmsr_tail;
 187         struct dispatch_object_s *volatile dmsr_head;
 188         mach_port_t dmsr_send, dmsr_checkin_port;
 189 };
 190 typedef struct dispatch_mach_send_refs_s *dispatch_mach_send_refs_t;
 191
 192 void _dispatch_mach_notification_set_armed(dispatch_mach_send_refs_t dmsr);
 193
 194 struct dispatch_xpc_term_refs_s {
 195         DISPATCH_UNOTE_CLASS_HEADER();
 196 };
 197 typedef struct dispatch_xpc_term_refs_s *dispatch_xpc_term_refs_t;
 198 #endif // HAVE_MACH
 199
 200 typedef union dispatch_unote_u {
 201         dispatch_unote_class_t _du;
 202         dispatch_source_refs_t _dr;
 203         dispatch_timer_source_refs_t _dt;
 204 #if HAVE_MACH
 205         dispatch_mach_recv_refs_t _dmrr;
 206         dispatch_mach_send_refs_t _dmsr;
 207         dispatch_mach_reply_refs_t _dmr;
 208         dispatch_xpc_term_refs_t _dxtr;
 209 #endif
 210 } dispatch_unote_t DISPATCH_TRANSPARENT_UNION;
 211
 212 #define DISPATCH_UNOTE_NULL ((dispatch_unote_t){ ._du = NULL })
 213
 214 #if TARGET_OS_EMBEDDED
 215 #define DSL_HASH_SIZE  64u // must be a power of two
 216 #else
 217 #define DSL_HASH_SIZE 256u // must be a power of two
 218 #endif
 219 #define DSL_HASH(x) ((x) & (DSL_HASH_SIZE - 1))
 220
 221 typedef struct dispatch_unote_linkage_s {
 222         TAILQ_ENTRY(dispatch_unote_linkage_s) du_link;
 223         struct dispatch_muxnote_s *du_muxnote;
 224 } DISPATCH_ATOMIC64_ALIGN *dispatch_unote_linkage_t;
 225
 226 #define DU_UNREGISTER_IMMEDIATE_DELETE 0x01
 227 #define DU_UNREGISTER_ALREADY_DELETED  0x02
 228 #define DU_UNREGISTER_DISCONNECTED     0x04
 229 #define DU_UNREGISTER_REPLY_REMOVE     0x08
 230
 231 typedef struct dispatch_source_type_s {
 232         const char *dst_kind;
 233         int8_t     dst_filter;
 234         uint8_t    dst_per_trigger_qos : 1;
 235         uint16_t   dst_flags;
 236         uint32_t   dst_fflags;
 237         uint32_t   dst_mask;
 238         uint32_t   dst_size;
 239 #if DISPATCH_EVENT_BACKEND_KEVENT
 240         uint32_t   dst_data;
 241 #endif
 242
 243         dispatch_unote_t (*dst_create)(dispatch_source_type_t dst,
 244                         uintptr_t handle, unsigned long mask);
 245 #if DISPATCH_EVENT_BACKEND_KEVENT
 246         bool (*dst_update_mux)(struct dispatch_muxnote_s *dmn);
 247 #endif
 248         void (*dst_merge_evt)(dispatch_unote_t du, uint32_t flags, uintptr_t data,
 249                         uintptr_t status, pthread_priority_t pp);
 250 #if HAVE_MACH
 251         void (*dst_merge_msg)(dispatch_unote_t du, uint32_t flags,
 252                         mach_msg_header_t *msg, mach_msg_size_t sz);
 253 #endif
 254 } dispatch_source_type_s;
 255
 256 #define dux_create(dst, handle, mask)   (dst)->dst_create(dst, handle, mask)
 257 #define dux_merge_evt(du, ...)  (du)->du_type->dst_merge_evt(du, __VA_ARGS__)
 258 #define dux_merge_msg(du, ...)  (du)->du_type->dst_merge_msg(du, __VA_ARGS__)
 259
 260 extern const dispatch_source_type_s _dispatch_source_type_after;
 261
 262 #if HAVE_MACH
 263 extern const dispatch_source_type_s _dispatch_source_type_mach_recv_direct;
 264 extern const dispatch_source_type_s _dispatch_mach_type_send;
 265 extern const dispatch_source_type_s _dispatch_mach_type_recv;
 266 extern const dispatch_source_type_s _dispatch_mach_type_reply;
 267 extern const dispatch_source_type_s _dispatch_xpc_type_sigterm;
 268 #endif
 269
 270 #pragma mark -
 271 #pragma mark deferred items
 272
 273 #if DISPATCH_EVENT_BACKEND_KEVENT
 274 #if DISPATCH_USE_KEVENT_QOS
 275 typedef struct kevent_qos_s dispatch_kevent_s;
 276 #else
 277 typedef struct kevent dispatch_kevent_s;
 278 #endif
 279 typedef dispatch_kevent_s *dispatch_kevent_t;
 280 #endif // DISPATCH_EVENT_BACKEND_KEVENT
 281
 282 #define DISPATCH_DEFERRED_ITEMS_EVENT_COUNT 16
 283
 284 typedef struct dispatch_deferred_items_s {
 285         dispatch_queue_t ddi_stashed_rq;
 286         dispatch_object_t ddi_stashed_dou;
 287         dispatch_qos_t ddi_stashed_qos;
 288 #if DISPATCH_EVENT_BACKEND_KEVENT
 289         dispatch_kevent_t ddi_eventlist;
 290         uint16_t ddi_nevents;
 291         uint16_t ddi_maxevents;
 292         bool     ddi_can_stash;
 293         uint16_t ddi_wlh_needs_delete : 1;
 294         uint16_t ddi_wlh_needs_update : 1;
 295         uint16_t ddi_wlh_servicing : 1;
 296 #endif
 297 } dispatch_deferred_items_s, *dispatch_deferred_items_t;
 298
 299 #pragma mark -
 300 #pragma mark inlines
 301
 302 #if DISPATCH_PURE_C
 303
 304 DISPATCH_ALWAYS_INLINE
 305 static inline void
 306 _dispatch_deferred_items_set(dispatch_deferred_items_t ddi)
 307 {
 308         _dispatch_thread_setspecific(dispatch_deferred_items_key, (void *)ddi);
 309 }
 310
 311 DISPATCH_ALWAYS_INLINE
 312 static inline dispatch_deferred_items_t
 313 _dispatch_deferred_items_get(void)
 314 {
 315         return (dispatch_deferred_items_t)
 316                         _dispatch_thread_getspecific(dispatch_deferred_items_key);
 317 }
 318
 319 DISPATCH_ALWAYS_INLINE
 320 static inline bool
 321 _dispatch_needs_to_return_to_kernel(void)
 322 {
 323         return (uintptr_t)_dispatch_thread_getspecific(dispatch_r2k_key) != 0;
 324 }
 325
 326 DISPATCH_ALWAYS_INLINE
 327 static inline void
 328 _dispatch_set_return_to_kernel(void)
 329 {
 330         _dispatch_thread_setspecific(dispatch_r2k_key, (void *)1);
 331 }
 332
 333 DISPATCH_ALWAYS_INLINE
 334 static inline void
 335 _dispatch_clear_return_to_kernel(void)
 336 {
 337         _dispatch_thread_setspecific(dispatch_r2k_key, (void *)0);
 338 }
 339
 340 DISPATCH_ALWAYS_INLINE
 341 static inline bool
 342 _dispatch_unote_registered(dispatch_unote_t du)
 343 {
 344         return du._du->du_wlh != NULL;
 345 }
 346
 347 DISPATCH_ALWAYS_INLINE
 348 static inline bool
 349 _dispatch_unote_wlh_changed(dispatch_unote_t du, dispatch_wlh_t expected_wlh)
 350 {
 351         dispatch_wlh_t wlh = du._du->du_wlh;
 352         return wlh && wlh != DISPATCH_WLH_ANON && wlh != expected_wlh;
 353 }
 354
 355 DISPATCH_ALWAYS_INLINE
 356 static inline dispatch_unote_linkage_t
 357 _dispatch_unote_get_linkage(dispatch_unote_t du)
 358 {
 359         dispatch_assert(!du._du->du_is_direct);
 360         return (dispatch_unote_linkage_t)((char *)du._du
 361                         - sizeof(struct dispatch_unote_linkage_s));
 362 }
 363
 364 DISPATCH_ALWAYS_INLINE
 365 static inline bool
 366 _dispatch_unote_needs_rearm(dispatch_unote_t du)
 367 {
 368         return du._du->du_type->dst_flags & (EV_ONESHOT | EV_DISPATCH);
 369 }
 370
 371 DISPATCH_ALWAYS_INLINE
 372 static inline dispatch_unote_t
 373 _dispatch_unote_linkage_get_unote(dispatch_unote_linkage_t dul)
 374 {
 375         return (dispatch_unote_t){ ._du = (dispatch_unote_class_t)(dul + 1) };
 376 }
 377
 378 #endif // DISPATCH_PURE_C
 379
 380 #pragma mark -
 381 #pragma mark prototypes
 382
 383 #if DISPATCH_HAVE_TIMER_QOS
 384 #define DISPATCH_TIMER_QOS_NORMAL       0u
 385 #define DISPATCH_TIMER_QOS_CRITICAL     1u
 386 #define DISPATCH_TIMER_QOS_BACKGROUND   2u
 387 #define DISPATCH_TIMER_QOS_COUNT        3u
 388 #else
 389 #define DISPATCH_TIMER_QOS_NORMAL       0u
 390 #define DISPATCH_TIMER_QOS_COUNT        1u
 391 #endif
 392
 393 #define DISPATCH_TIMER_QOS(tidx)   (((uintptr_t)(tidx) >> 1) & 3u)
 394 #define DISPATCH_TIMER_CLOCK(tidx) (dispatch_clock_t)((tidx) & 1u)
 395
 396 #define DISPATCH_TIMER_INDEX(clock, qos) ((qos) << 1 | (clock))
 397 #define DISPATCH_TIMER_COUNT \
 398                 DISPATCH_TIMER_INDEX(0, DISPATCH_TIMER_QOS_COUNT)
 399 #define DISPATCH_TIMER_IDENT_CANCELED    (~0u)
 400
 401 extern struct dispatch_timer_heap_s _dispatch_timers_heap[DISPATCH_TIMER_COUNT];
 402 extern bool _dispatch_timers_reconfigure, _dispatch_timers_expired;
 403 extern uint32_t _dispatch_timers_processing_mask;
 404 #if DISPATCH_USE_DTRACE
 405 extern uint32_t _dispatch_timers_will_wake;
 406 #endif
 407
 408 dispatch_unote_t _dispatch_unote_create_with_handle(dispatch_source_type_t dst,
 409                 uintptr_t handle, unsigned long mask);
 410 dispatch_unote_t _dispatch_unote_create_with_fd(dispatch_source_type_t dst,
 411                 uintptr_t handle, unsigned long mask);
 412 dispatch_unote_t _dispatch_unote_create_without_handle(
 413                 dispatch_source_type_t dst, uintptr_t handle, unsigned long mask);
 414
 415 bool _dispatch_unote_register(dispatch_unote_t du, dispatch_wlh_t wlh,
 416                 dispatch_priority_t pri);
 417 void _dispatch_unote_resume(dispatch_unote_t du);
 418 bool _dispatch_unote_unregister(dispatch_unote_t du, uint32_t flags);
 419 void _dispatch_unote_dispose(dispatch_unote_t du);
 420
 421 void _dispatch_event_loop_atfork_child(void);
 422 #define DISPATCH_EVENT_LOOP_CONSUME_2 DISPATCH_WAKEUP_CONSUME_2
 423 #define DISPATCH_EVENT_LOOP_OVERRIDE  0x80000000
 424 void _dispatch_event_loop_poke(dispatch_wlh_t wlh, uint64_t dq_state,
 425                 uint32_t flags);
 426 void _dispatch_event_loop_wake_owner(struct dispatch_sync_context_s *dsc,
 427                 dispatch_wlh_t wlh, uint64_t old_state, uint64_t new_state);
 428 void _dispatch_event_loop_wait_for_ownership(
 429                 struct dispatch_sync_context_s *dsc);
 430 void _dispatch_event_loop_end_ownership(dispatch_wlh_t wlh,
 431                 uint64_t old_state, uint64_t new_state, uint32_t flags);
 432 #if DISPATCH_WLH_DEBUG
 433 void _dispatch_event_loop_assert_not_owned(dispatch_wlh_t wlh);
 434 #else
 435 #undef _dispatch_event_loop_assert_not_owned
 436 #define _dispatch_event_loop_assert_not_owned(wlh) ((void)wlh)
 437 #endif
 438 void _dispatch_event_loop_leave_immediate(dispatch_wlh_t wlh, uint64_t dq_state);
 439 #if DISPATCH_EVENT_BACKEND_KEVENT
 440 void _dispatch_event_loop_leave_deferred(dispatch_wlh_t wlh,
 441                 uint64_t dq_state);
 442 void _dispatch_event_loop_merge(dispatch_kevent_t events, int nevents);
 443 #endif
 444 void _dispatch_event_loop_drain(uint32_t flags);
 445 void _dispatch_event_loop_timer_arm(unsigned int tidx,
 446                 dispatch_timer_delay_s range, dispatch_clock_now_cache_t nows);
 447 void _dispatch_event_loop_timer_delete(unsigned int tidx);
 448
 449 #endif /* __DISPATCH_EVENT_EVENT_INTERNAL__ */