src/source_internal.h

   1 /*
   2  * Copyright (c) 2008-2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_APACHE_LICENSE_HEADER_START@
   5  *
   6  * Licensed under the Apache License, Version 2.0 (the "License");
   7  * you may not use this file except in compliance with the License.
   8  * You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  *
  18  * @APPLE_APACHE_LICENSE_HEADER_END@
  19  */
  20
  21 /*
  22  * IMPORTANT: This header file describes INTERNAL interfaces to libdispatch
  23  * which are subject to change in future releases of Mac OS X. Any applications
  24  * relying on these interfaces WILL break.
  25  */
  26
  27 #ifndef __DISPATCH_SOURCE_INTERNAL__
  28 #define __DISPATCH_SOURCE_INTERNAL__
  29
  30 #ifndef __DISPATCH_INDIRECT__
  31 #error "Please #include <dispatch/dispatch.h> instead of this file directly."
  32 #include <dispatch/base.h> // for HeaderDoc
  33 #endif
  34
  35 #define DISPATCH_EVFILT_TIMER           (-EVFILT_SYSCOUNT - 1)
  36 #define DISPATCH_EVFILT_CUSTOM_ADD      (-EVFILT_SYSCOUNT - 2)
  37 #define DISPATCH_EVFILT_CUSTOM_OR       (-EVFILT_SYSCOUNT - 3)
  38 #define DISPATCH_EVFILT_MACH_NOTIFICATION       (-EVFILT_SYSCOUNT - 4)
  39 #define DISPATCH_EVFILT_SYSCOUNT        ( EVFILT_SYSCOUNT + 4)
  40
  41 #if HAVE_MACH
  42 // NOTE: dispatch_source_mach_send_flags_t and dispatch_source_mach_recv_flags_t
  43 //       bit values must not overlap as they share the same kevent fflags !
  44
  45 /*!
  46  * @enum dispatch_source_mach_send_flags_t
  47  *
  48  * @constant DISPATCH_MACH_SEND_DELETED
  49  * Port-deleted notification. Disabled for source registration.
  50  */
  51 enum {
  52         DISPATCH_MACH_SEND_DELETED = 0x4,
  53 };
  54 /*!
  55  * @enum dispatch_source_mach_recv_flags_t
  56  *
  57  * @constant DISPATCH_MACH_RECV_MESSAGE
  58  * Receive right has pending messages
  59  *
  60  * @constant DISPATCH_MACH_RECV_MESSAGE_DIRECT
  61  * Receive messages from receive right directly via kevent64()
  62  *
  63  * @constant DISPATCH_MACH_RECV_NO_SENDERS
  64  * Receive right has no more senders. TODO <rdar://problem/8132399>
  65  */
  66 enum {
  67         DISPATCH_MACH_RECV_MESSAGE = 0x2,
  68         DISPATCH_MACH_RECV_MESSAGE_DIRECT = 0x10,
  69         DISPATCH_MACH_RECV_MESSAGE_DIRECT_ONCE = 0x20,
  70         DISPATCH_MACH_RECV_NO_SENDERS = 0x40,
  71 };
  72 #endif // HAVE_MACH
  73
  74 enum {
  75         /* DISPATCH_TIMER_STRICT 0x1 */
  76         /* DISPATCH_TIMER_BACKGROUND = 0x2, */
  77         DISPATCH_TIMER_WALL_CLOCK = 0x4,
  78         DISPATCH_TIMER_INTERVAL = 0x8,
  79         DISPATCH_TIMER_WITH_AGGREGATE = 0x10,
  80         /* DISPATCH_INTERVAL_UI_ANIMATION = 0x20 */
  81         DISPATCH_TIMER_AFTER = 0x40,
  82 };
  83
  84 #define DISPATCH_TIMER_QOS_NORMAL 0u
  85 #define DISPATCH_TIMER_QOS_CRITICAL 1u
  86 #define DISPATCH_TIMER_QOS_BACKGROUND 2u
  87 #define DISPATCH_TIMER_QOS_COUNT (DISPATCH_TIMER_QOS_BACKGROUND + 1)
  88 #define DISPATCH_TIMER_QOS(tidx) (((uintptr_t)(tidx) >> 1) & 0x3ul)
  89
  90 #define DISPATCH_TIMER_KIND_WALL 0u
  91 #define DISPATCH_TIMER_KIND_MACH 1u
  92 #define DISPATCH_TIMER_KIND_COUNT (DISPATCH_TIMER_KIND_MACH + 1)
  93 #define DISPATCH_TIMER_KIND(tidx) ((uintptr_t)(tidx) & 0x1ul)
  94
  95 #define DISPATCH_TIMER_INDEX(kind, qos) ((qos) << 1 | (kind))
  96 #define DISPATCH_TIMER_INDEX_DISARM \
  97                 DISPATCH_TIMER_INDEX(0, DISPATCH_TIMER_QOS_COUNT)
  98 #define DISPATCH_TIMER_INDEX_COUNT (DISPATCH_TIMER_INDEX_DISARM + 1)
  99 #define DISPATCH_TIMER_IDENT(flags) ({ unsigned long f = (flags); \
 100                 DISPATCH_TIMER_INDEX(f & DISPATCH_TIMER_WALL_CLOCK ? \
 101                 DISPATCH_TIMER_KIND_WALL : DISPATCH_TIMER_KIND_MACH, \
 102                 f & DISPATCH_TIMER_STRICT ? DISPATCH_TIMER_QOS_CRITICAL : \
 103                 f & DISPATCH_TIMER_BACKGROUND ? DISPATCH_TIMER_QOS_BACKGROUND : \
 104                 DISPATCH_TIMER_QOS_NORMAL); })
 105
 106 struct dispatch_kevent_s {
 107         TAILQ_ENTRY(dispatch_kevent_s) dk_list;
 108         TAILQ_HEAD(, dispatch_source_refs_s) dk_sources;
 109         _dispatch_kevent_qos_s dk_kevent;
 110 };
 111
 112 typedef struct dispatch_kevent_s *dispatch_kevent_t;
 113
 114 typedef typeof(((dispatch_kevent_t)NULL)->dk_kevent.udata) _dispatch_kevent_qos_udata_t;
 115
 116 #define DISPATCH_KEV_CUSTOM_ADD ((dispatch_kevent_t)DISPATCH_EVFILT_CUSTOM_ADD)
 117 #define DISPATCH_KEV_CUSTOM_OR  ((dispatch_kevent_t)DISPATCH_EVFILT_CUSTOM_OR)
 118
 119 struct dispatch_source_type_s {
 120         _dispatch_kevent_qos_s ke;
 121         uint64_t mask;
 122         void (*init)(dispatch_source_t ds, dispatch_source_type_t type,
 123                         uintptr_t handle, unsigned long mask, dispatch_queue_t q);
 124 };
 125
 126 struct dispatch_timer_source_s {
 127         uint64_t target;
 128         uint64_t deadline;
 129         uint64_t last_fire;
 130         uint64_t interval;
 131         uint64_t leeway;
 132         unsigned long flags; // dispatch_timer_flags_t
 133         unsigned long missed;
 134 };
 135
 136 enum {
 137         DS_EVENT_HANDLER = 0,
 138         DS_CANCEL_HANDLER,
 139         DS_REGISTN_HANDLER,
 140 };
 141
 142 // Source state which may contain references to the source object
 143 // Separately allocated so that 'leaks' can see sources <rdar://problem/9050566>
 144 typedef struct dispatch_source_refs_s {
 145         TAILQ_ENTRY(dispatch_source_refs_s) dr_list;
 146         uintptr_t dr_source_wref; // "weak" backref to dispatch_source_t
 147         dispatch_continuation_t volatile ds_handler[3];
 148 } *dispatch_source_refs_t;
 149
 150 typedef struct dispatch_timer_source_refs_s {
 151         struct dispatch_source_refs_s _ds_refs;
 152         struct dispatch_timer_source_s _ds_timer;
 153         TAILQ_ENTRY(dispatch_timer_source_refs_s) dt_list;
 154 } *dispatch_timer_source_refs_t;
 155
 156 typedef struct dispatch_timer_source_aggregate_refs_s {
 157         struct dispatch_timer_source_refs_s _dsa_refs;
 158         TAILQ_ENTRY(dispatch_timer_source_aggregate_refs_s) dra_list;
 159         TAILQ_ENTRY(dispatch_timer_source_aggregate_refs_s) dta_list;
 160 } *dispatch_timer_source_aggregate_refs_t;
 161
 162 #define _dispatch_ptr2wref(ptr) (~(uintptr_t)(ptr))
 163 #define _dispatch_wref2ptr(ref) ((void*)~(ref))
 164 #define _dispatch_source_from_refs(dr) \
 165                 ((dispatch_source_t)_dispatch_wref2ptr((dr)->dr_source_wref))
 166 #define ds_timer(dr) \
 167                 (((dispatch_timer_source_refs_t)(dr))->_ds_timer)
 168 #define ds_timer_aggregate(ds) \
 169                 ((dispatch_timer_aggregate_t)((ds)->dq_specific_q))
 170
 171 DISPATCH_ALWAYS_INLINE
 172 static inline unsigned int
 173 _dispatch_source_timer_idx(dispatch_source_refs_t dr)
 174 {
 175         return DISPATCH_TIMER_IDENT(ds_timer(dr).flags);
 176 }
 177
 178 #define _DISPATCH_SOURCE_HEADER(refs) \
 179         DISPATCH_QUEUE_HEADER(refs); \
 180         /* LP64: fills 32bit hole in QUEUE_HEADER */ \
 181         unsigned int \
 182                 ds_is_level:1, \
 183                 ds_is_adder:1, \
 184                 ds_is_installed:1, \
 185                 ds_is_direct_kevent:1, \
 186                 ds_is_custom_source:1, \
 187                 ds_needs_rearm:1, \
 188                 ds_is_timer:1, \
 189                 ds_vmpressure_override:1, \
 190                 ds_memorypressure_override:1, \
 191                 dm_handler_is_block:1, \
 192                 dm_connect_handler_called:1, \
 193                 dm_cancel_handler_called:1; \
 194         dispatch_kevent_t ds_dkev; \
 195         dispatch_##refs##_refs_t ds_refs; \
 196         unsigned long ds_pending_data_mask;
 197
 198 #define DISPATCH_SOURCE_HEADER(refs) \
 199         struct dispatch_source_s _as_ds[0]; \
 200         _DISPATCH_SOURCE_HEADER(refs)
 201
 202 DISPATCH_CLASS_DECL_BARE(source);
 203 _OS_OBJECT_CLASS_IMPLEMENTS_PROTOCOL(dispatch_source, dispatch_object);
 204
 205 #if DISPATCH_PURE_C
 206 struct dispatch_source_s {
 207         _DISPATCH_SOURCE_HEADER(source);
 208         unsigned long ds_ident_hack;
 209         unsigned long ds_data;
 210         unsigned long ds_pending_data;
 211 } DISPATCH_QUEUE_ALIGN;
 212 #endif
 213
 214 #if HAVE_MACH
 215 // Mach channel state which may contain references to the channel object
 216 // layout must match dispatch_source_refs_s
 217 struct dispatch_mach_refs_s {
 218         TAILQ_ENTRY(dispatch_mach_refs_s) dr_list;
 219         uintptr_t dr_source_wref; // "weak" backref to dispatch_mach_t
 220         dispatch_mach_handler_function_t dm_handler_func;
 221         void *dm_handler_ctxt;
 222 };
 223 typedef struct dispatch_mach_refs_s *dispatch_mach_refs_t;
 224
 225 struct dispatch_mach_reply_refs_s {
 226         TAILQ_ENTRY(dispatch_mach_reply_refs_s) dr_list;
 227         uintptr_t dr_source_wref; // "weak" backref to dispatch_mach_t
 228         dispatch_kevent_t dmr_dkev;
 229         void *dmr_ctxt;
 230         mach_port_t dmr_reply;
 231         dispatch_priority_t dmr_priority;
 232         voucher_t dmr_voucher;
 233         TAILQ_ENTRY(dispatch_mach_reply_refs_s) dmr_list;
 234 };
 235 typedef struct dispatch_mach_reply_refs_s *dispatch_mach_reply_refs_t;
 236
 237 #define _DISPATCH_MACH_STATE_UNUSED_MASK_2       0xff00000000000000ull
 238 #define DISPATCH_MACH_STATE_OVERRIDE_MASK        0x00ffff0000000000ull
 239 #define _DISPATCH_MACH_STATE_UNUSED_MASK_1       0x000000f000000000ull
 240 #define DISPATCH_MACH_STATE_DIRTY                0x0000000800000000ull
 241 #define DISPATCH_MACH_STATE_RECEIVED_OVERRIDE    0x0000000400000000ull
 242 #define _DISPATCH_MACH_STATE_UNUSED_MASK_0       0x0000000200000000ull
 243 #define DISPATCH_MACH_STATE_PENDING_BARRIER      0x0000000100000000ull
 244 #define DISPATCH_MACH_STATE_UNLOCK_MASK          0x00000000ffffffffull
 245
 246 struct dispatch_mach_send_refs_s {
 247         TAILQ_ENTRY(dispatch_mach_send_refs_s) dr_list;
 248         uintptr_t dr_source_wref; // "weak" backref to dispatch_mach_t
 249         dispatch_mach_msg_t dm_checkin;
 250         TAILQ_HEAD(, dispatch_mach_reply_refs_s) dm_replies;
 251         dispatch_unfair_lock_s dm_replies_lock;
 252 #define DISPATCH_MACH_DISCONNECT_MAGIC_BASE (0x80000000)
 253 #define DISPATCH_MACH_NEVER_INSTALLED (DISPATCH_MACH_DISCONNECT_MAGIC_BASE + 0)
 254 #define DISPATCH_MACH_NEVER_CONNECTED (DISPATCH_MACH_DISCONNECT_MAGIC_BASE + 1)
 255         uint32_t volatile dm_disconnect_cnt;
 256         union {
 257                 uint64_t volatile dm_state;
 258                 DISPATCH_STRUCT_LITTLE_ENDIAN_2(
 259                         dispatch_unfair_lock_s dm_state_lock,
 260                         uint32_t dm_state_bits
 261                 );
 262         };
 263         unsigned int dm_needs_mgr:1;
 264         struct dispatch_object_s *volatile dm_tail;
 265         struct dispatch_object_s *volatile dm_head;
 266         mach_port_t dm_send, dm_checkin_port;
 267 };
 268 typedef struct dispatch_mach_send_refs_s *dispatch_mach_send_refs_t;
 269
 270 DISPATCH_CLASS_DECL(mach);
 271 #if DISPATCH_PURE_C
 272 struct dispatch_mach_s {
 273         DISPATCH_SOURCE_HEADER(mach);
 274         dispatch_kevent_t dm_dkev;
 275         dispatch_mach_send_refs_t dm_refs;
 276 } DISPATCH_QUEUE_ALIGN;
 277 #endif
 278
 279 DISPATCH_CLASS_DECL(mach_msg);
 280 struct dispatch_mach_msg_s {
 281         DISPATCH_OBJECT_HEADER(mach_msg);
 282         union {
 283                 mach_msg_option_t dmsg_options;
 284                 mach_error_t dmsg_error;
 285         };
 286         mach_port_t dmsg_reply;
 287         pthread_priority_t dmsg_priority;
 288         voucher_t dmsg_voucher;
 289         dispatch_mach_msg_destructor_t dmsg_destructor;
 290         size_t dmsg_size;
 291         union {
 292                 mach_msg_header_t *dmsg_msg;
 293                 char dmsg_buf[0];
 294         };
 295 };
 296 #endif // HAVE_MACH
 297
 298 extern const struct dispatch_source_type_s _dispatch_source_type_after;
 299
 300 #if TARGET_OS_EMBEDDED
 301 #define DSL_HASH_SIZE  64u // must be a power of two
 302 #else
 303 #define DSL_HASH_SIZE 256u // must be a power of two
 304 #endif
 305
 306 dispatch_source_t
 307 _dispatch_source_create_mach_msg_direct_recv(mach_port_t recvp,
 308                 const struct dispatch_continuation_s *dc);
 309 void _dispatch_source_xref_dispose(dispatch_source_t ds);
 310 void _dispatch_source_dispose(dispatch_source_t ds);
 311 void _dispatch_source_finalize_activation(dispatch_source_t ds);
 312 void _dispatch_source_invoke(dispatch_source_t ds, dispatch_invoke_flags_t flags);
 313 void _dispatch_source_wakeup(dispatch_source_t ds, pthread_priority_t pp,
 314                 dispatch_wakeup_flags_t flags);
 315 size_t _dispatch_source_debug(dispatch_source_t ds, char* buf, size_t bufsiz);
 316 void _dispatch_source_set_interval(dispatch_source_t ds, uint64_t interval);
 317 void _dispatch_source_set_event_handler_continuation(dispatch_source_t ds,
 318                 dispatch_continuation_t dc);
 319 DISPATCH_EXPORT // for firehose server
 320 void _dispatch_source_merge_data(dispatch_source_t ds, pthread_priority_t pp,
 321                 unsigned long val);
 322
 323 #if HAVE_MACH
 324 void _dispatch_mach_dispose(dispatch_mach_t dm);
 325 void _dispatch_mach_finalize_activation(dispatch_mach_t dm);
 326 void _dispatch_mach_invoke(dispatch_mach_t dm, dispatch_invoke_flags_t flags);
 327 void _dispatch_mach_wakeup(dispatch_mach_t dm, pthread_priority_t pp,
 328                 dispatch_wakeup_flags_t flags);
 329 size_t _dispatch_mach_debug(dispatch_mach_t dm, char* buf, size_t bufsiz);
 330
 331 void _dispatch_mach_msg_dispose(dispatch_mach_msg_t dmsg);
 332 void _dispatch_mach_msg_invoke(dispatch_mach_msg_t dmsg,
 333                 dispatch_invoke_flags_t flags);
 334 size_t _dispatch_mach_msg_debug(dispatch_mach_msg_t dmsg, char* buf,
 335                 size_t bufsiz);
 336
 337 void _dispatch_mach_send_barrier_drain_invoke(dispatch_continuation_t dc,
 338                 dispatch_invoke_flags_t flags);
 339 void _dispatch_mach_barrier_invoke(dispatch_continuation_t dc,
 340                 dispatch_invoke_flags_t flags);
 341 #endif // HAVE_MACH
 342
 343 void _dispatch_mgr_queue_wakeup(dispatch_queue_t dq, pthread_priority_t pp,
 344                 dispatch_wakeup_flags_t flags);
 345 void _dispatch_mgr_thread(dispatch_queue_t dq, dispatch_invoke_flags_t flags);
 346 #if DISPATCH_USE_KEVENT_WORKQUEUE
 347 void _dispatch_kevent_worker_thread(_dispatch_kevent_qos_s **events,
 348                 int *nevents);
 349 #endif
 350
 351 #endif /* __DISPATCH_SOURCE_INTERNAL__ */