src/object_internal.h

   1 /*
   2  * Copyright (c) 2008-2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_APACHE_LICENSE_HEADER_START@
   5  *
   6  * Licensed under the Apache License, Version 2.0 (the "License");
   7  * you may not use this file except in compliance with the License.
   8  * You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  *
  18  * @APPLE_APACHE_LICENSE_HEADER_END@
  19  */
  20
  21 /*
  22  * IMPORTANT: This header file describes INTERNAL interfaces to libdispatch
  23  * which are subject to change in future releases of Mac OS X. Any applications
  24  * relying on these interfaces WILL break.
  25  */
  26
  27 #ifndef __DISPATCH_OBJECT_INTERNAL__
  28 #define __DISPATCH_OBJECT_INTERNAL__
  29
  30 #if !OS_OBJECT_USE_OBJC
  31 #define OS_OBJECT_DECL(name)  DISPATCH_DECL(name)
  32 #define OS_OBJECT_DECL_SUBCLASS(name, super)  DISPATCH_DECL(name)
  33 #endif
  34
  35 #if USE_OBJC
  36 #define OS_OBJECT_EXTRA_VTABLE_SYMBOL(name) _OS_##name##_vtable
  37 #define DISPATCH_CLASS_SYMBOL(name) OS_dispatch_##name##_class
  38 #define DISPATCH_CLASS_RAW_SYMBOL_NAME(name) \
  39                 OS_OBJC_CLASS_RAW_SYMBOL_NAME(DISPATCH_CLASS(name))
  40 #else
  41 #define OS_OBJECT_CLASS_SYMBOL(name) _##name##_vtable
  42 #define OS_OBJC_CLASS_RAW_SYMBOL_NAME(name) \
  43                 "__" OS_STRINGIFY(name) "_vtable"
  44 #define DISPATCH_CLASS_SYMBOL(name) _dispatch_##name##_vtable
  45 #define DISPATCH_CLASS_RAW_SYMBOL_NAME(name) \
  46                 "__dispatch_" OS_STRINGIFY(name) "_vtable"
  47 #endif
  48
  49 #define DISPATCH_CLASS(name) OS_dispatch_##name
  50 #if USE_OBJC
  51 #define DISPATCH_OBJC_CLASS_DECL(name) \
  52                 extern void *DISPATCH_CLASS_SYMBOL(name) \
  53                                 asm(DISPATCH_CLASS_RAW_SYMBOL_NAME(name))
  54 #endif
  55
  56 // define a new proper class
  57 #define OS_OBJECT_CLASS_DECL(name, super, ...) \
  58                 struct name##_s; \
  59                 struct name##_extra_vtable_s { \
  60                         __VA_ARGS__; \
  61                 }; \
  62                 struct name##_vtable_s { \
  63                         _OS_OBJECT_CLASS_HEADER(); \
  64                         struct name##_extra_vtable_s _os_obj_vtable; \
  65                 }; \
  66                 OS_OBJECT_EXTRA_VTABLE_DECL(name, name) \
  67                 extern const struct name##_vtable_s OS_OBJECT_CLASS_SYMBOL(name) \
  68                                 asm(OS_OBJC_CLASS_RAW_SYMBOL_NAME(OS_OBJECT_CLASS(name)))
  69
  70 #if OS_OBJECT_SWIFT3
  71 #define OS_OBJECT_INTERNAL_CLASS_DECL(name, super, ...) \
  72                 OS_OBJECT_OBJC_RUNTIME_VISIBLE \
  73                 OS_OBJECT_DECL_IMPL_CLASS(name, OS_OBJECT_CLASS(super)); \
  74                 OS_OBJECT_CLASS_DECL(name, super, ## __VA_ARGS__)
  75 #elif OS_OBJECT_USE_OBJC
  76 #define OS_OBJECT_INTERNAL_CLASS_DECL(name, super, ...) \
  77                 OS_OBJECT_DECL(name); \
  78                 OS_OBJECT_CLASS_DECL(name, super, ## __VA_ARGS__)
  79 #else
  80 #define OS_OBJECT_INTERNAL_CLASS_DECL(name, super, ...) \
  81                 typedef struct name##_s *name##_t; \
  82                 OS_OBJECT_CLASS_DECL(name, super, ## __VA_ARGS__)
  83 #endif
  84
  85 #define DISPATCH_CLASS_DECL_BARE(name) \
  86                 OS_OBJECT_CLASS_DECL(dispatch_##name, dispatch_object, \
  87                                 DISPATCH_OBJECT_VTABLE_HEADER(dispatch_##name))
  88
  89 #define DISPATCH_CLASS_DECL(name) \
  90                 _OS_OBJECT_DECL_PROTOCOL(dispatch_##name, dispatch_object) \
  91                 _OS_OBJECT_CLASS_IMPLEMENTS_PROTOCOL(dispatch_##name, dispatch_##name) \
  92                 DISPATCH_CLASS_DECL_BARE(name)
  93
  94 #define DISPATCH_INTERNAL_CLASS_DECL(name) \
  95                 DISPATCH_DECL(dispatch_##name); \
  96                 DISPATCH_CLASS_DECL(name)
  97
  98 // define a new subclass used in a cluster
  99 #define OS_OBJECT_SUBCLASS_DECL(name, super) \
 100                 _OS_OBJECT_DECL_SUBCLASS_INTERFACE(name, super) \
 101                 struct name##_s; \
 102                 OS_OBJECT_EXTRA_VTABLE_DECL(name, super) \
 103                 extern const struct super##_vtable_s OS_OBJECT_CLASS_SYMBOL(name) \
 104                                 asm(OS_OBJC_CLASS_RAW_SYMBOL_NAME(OS_OBJECT_CLASS(name)))
 105
 106 #define DISPATCH_SUBCLASS_DECL(name, super) \
 107                 OS_OBJECT_SUBCLASS_DECL(dispatch_##name, super)
 108
 109 #if OS_OBJECT_SWIFT3
 110 // define a new internal subclass used in a class cluster
 111 #define OS_OBJECT_INTERNAL_SUBCLASS_DECL(name, super) \
 112                 _OS_OBJECT_DECL_PROTOCOL(name, super); \
 113                 OS_OBJECT_SUBCLASS_DECL(name, super)
 114
 115 #define DISPATCH_INTERNAL_SUBCLASS_DECL(name, super) \
 116                 _OS_OBJECT_DECL_PROTOCOL(dispatch_##name, dispatch_##super) \
 117                 DISPATCH_SUBCLASS_DECL(name, dispatch_##super)
 118 #else
 119 // define a new internal subclass used in a class cluster
 120 #define OS_OBJECT_INTERNAL_SUBCLASS_DECL(name, super) \
 121                 OS_OBJECT_DECL_SUBCLASS(name, super); \
 122                 OS_OBJECT_SUBCLASS_DECL(name, super)
 123
 124 #define DISPATCH_INTERNAL_SUBCLASS_DECL(name, super) \
 125                 OS_OBJECT_DECL_SUBCLASS(dispatch_##name, dispatch_##super); \
 126                 DISPATCH_SUBCLASS_DECL(name, dispatch_##super)
 127 #endif
 128
 129 // vtable symbols
 130 #define OS_OBJECT_VTABLE(name)          (&OS_OBJECT_CLASS_SYMBOL(name))
 131 #define DISPATCH_OBJC_CLASS(name)       (&DISPATCH_CLASS_SYMBOL(name))
 132
 133 // vtables for subclasses used in a class cluster
 134 #if USE_OBJC
 135 // ObjC classes and dispatch vtables are co-located via linker order and alias
 136 // files rdar://10640168
 137 #if OS_OBJECT_HAVE_OBJC2
 138 #define OS_OBJECT_VTABLE_SUBCLASS_INSTANCE(name, super, xdispose, dispose, ...) \
 139                 __attribute__((section("__DATA,__objc_data"), used)) \
 140                 const struct super##_extra_vtable_s \
 141                 OS_OBJECT_EXTRA_VTABLE_SYMBOL(name) = { __VA_ARGS__ }
 142 #define OS_OBJECT_EXTRA_VTABLE_DECL(name, super)
 143 #define DISPATCH_VTABLE(name) DISPATCH_OBJC_CLASS(name)
 144 #else
 145 #define OS_OBJECT_VTABLE_SUBCLASS_INSTANCE(name, super, xdispose, dispose, ...) \
 146                 const struct super##_vtable_s \
 147                 OS_OBJECT_EXTRA_VTABLE_SYMBOL(name) = { \
 148                         ._os_obj_objc_isa = &OS_OBJECT_CLASS_SYMBOL(name), \
 149                         ._os_obj_vtable = { __VA_ARGS__ }, \
 150                 }
 151 #define OS_OBJECT_EXTRA_VTABLE_DECL(name, super) \
 152                 extern const struct super##_vtable_s \
 153                                 OS_OBJECT_EXTRA_VTABLE_SYMBOL(name);
 154 #define DISPATCH_VTABLE(name) &OS_OBJECT_EXTRA_VTABLE_SYMBOL(dispatch_##name)
 155 #endif
 156 #else
 157 #define OS_OBJECT_VTABLE_SUBCLASS_INSTANCE(name, super, xdispose, dispose, ...) \
 158                 const struct super##_vtable_s OS_OBJECT_CLASS_SYMBOL(name) = { \
 159                         ._os_obj_xref_dispose = xdispose, \
 160                         ._os_obj_dispose = dispose, \
 161                         ._os_obj_vtable = { __VA_ARGS__ }, \
 162                 }
 163 #define OS_OBJECT_EXTRA_VTABLE_DECL(name, super)
 164 #define DISPATCH_VTABLE(name) DISPATCH_OBJC_CLASS(name)
 165 #endif // USE_OBJC
 166
 167 #define DISPATCH_VTABLE_SUBCLASS_INSTANCE(name, super, ...) \
 168                 OS_OBJECT_VTABLE_SUBCLASS_INSTANCE(dispatch_##name, dispatch_##super, \
 169                                 _dispatch_xref_dispose, _dispatch_dispose, __VA_ARGS__)
 170
 171 // vtables for proper classes
 172 #define OS_OBJECT_VTABLE_INSTANCE(name, xdispose, dispose, ...) \
 173                 OS_OBJECT_VTABLE_SUBCLASS_INSTANCE(name, name, \
 174                                 xdispose, dispose, __VA_ARGS__)
 175
 176 #define DISPATCH_VTABLE_INSTANCE(name, ...) \
 177                 DISPATCH_VTABLE_SUBCLASS_INSTANCE(name, name, __VA_ARGS__)
 178
 179 #define DISPATCH_INVOKABLE_VTABLE_HEADER(x) \
 180         unsigned long const do_type; \
 181         const char *const do_kind; \
 182         void (*const do_invoke)(struct x##_s *, dispatch_invoke_context_t, \
 183                         dispatch_invoke_flags_t); \
 184         void (*const do_push)(struct x##_s *, dispatch_object_t, \
 185                         dispatch_qos_t)
 186
 187 #define DISPATCH_QUEUEABLE_VTABLE_HEADER(x) \
 188         DISPATCH_INVOKABLE_VTABLE_HEADER(x); \
 189         void (*const do_wakeup)(struct x##_s *, \
 190                         dispatch_qos_t, dispatch_wakeup_flags_t); \
 191         void (*const do_dispose)(struct x##_s *, bool *allow_free)
 192
 193 #define DISPATCH_OBJECT_VTABLE_HEADER(x) \
 194         DISPATCH_QUEUEABLE_VTABLE_HEADER(x); \
 195         void (*const do_set_targetq)(struct x##_s *, dispatch_queue_t); \
 196         void (*const do_suspend)(struct x##_s *); \
 197         void (*const do_resume)(struct x##_s *, bool activate); \
 198         void (*const do_finalize_activation)(struct x##_s *, bool *allow_resume); \
 199         size_t (*const do_debug)(struct x##_s *, char *, size_t)
 200
 201 #define dx_vtable(x) (&(x)->do_vtable->_os_obj_vtable)
 202 #define dx_type(x) dx_vtable(x)->do_type
 203 #define dx_subtype(x) (dx_vtable(x)->do_type & _DISPATCH_SUB_TYPE_MASK)
 204 #define dx_metatype(x) (dx_vtable(x)->do_type & _DISPATCH_META_TYPE_MASK)
 205 #define dx_hastypeflag(x, f) (dx_vtable(x)->do_type & _DISPATCH_##f##_TYPEFLAG)
 206 #define dx_kind(x) dx_vtable(x)->do_kind
 207 #define dx_debug(x, y, z) dx_vtable(x)->do_debug((x), (y), (z))
 208 #define dx_dispose(x, y) dx_vtable(x)->do_dispose(x, y)
 209 #define dx_invoke(x, y, z) dx_vtable(x)->do_invoke(x, y, z)
 210 #define dx_push(x, y, z) dx_vtable(x)->do_push(x, y, z)
 211 #define dx_wakeup(x, y, z) dx_vtable(x)->do_wakeup(x, y, z)
 212
 213 #define DISPATCH_OBJECT_GLOBAL_REFCNT           _OS_OBJECT_GLOBAL_REFCNT
 214
 215 #if OS_OBJECT_HAVE_OBJC1
 216 #define DISPATCH_GLOBAL_OBJECT_HEADER(name) \
 217         .do_vtable = DISPATCH_VTABLE(name), \
 218         ._objc_isa = DISPATCH_OBJC_CLASS(name), \
 219         .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, \
 220         .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT
 221 #else
 222 #define DISPATCH_GLOBAL_OBJECT_HEADER(name) \
 223         .do_vtable = DISPATCH_VTABLE(name), \
 224         .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, \
 225         .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT
 226 #endif
 227
 228 #if DISPATCH_SIZEOF_PTR == 8
 229 // the bottom nibble must not be zero, the rest of the bits should be random
 230 // we sign extend the 64-bit version so that a better instruction encoding is
 231 // generated on Intel
 232 #define DISPATCH_OBJECT_LISTLESS ((void *)0xffffffff89abcdef)
 233 #else
 234 #define DISPATCH_OBJECT_LISTLESS ((void *)0x89abcdef)
 235 #endif
 236
 237 DISPATCH_ENUM(dispatch_wakeup_flags, uint32_t,
 238         // The caller of dx_wakeup owns two internal refcounts on the object being
 239         // woken up. Two are needed for WLH wakeups where two threads need
 240         // the object to remain valid in a non-coordinated way
 241         // - the thread doing the poke for the duration of the poke
 242         // - drainers for the duration of their drain
 243         DISPATCH_WAKEUP_CONSUME_2               = 0x00000001,
 244
 245         // Some change to the object needs to be published to drainers.
 246         // If the drainer isn't the same thread, some scheme such as the dispatch
 247         // queue DIRTY bit must be used and a release barrier likely has to be
 248         // involved before dx_wakeup returns
 249         DISPATCH_WAKEUP_MAKE_DIRTY              = 0x00000002,
 250
 251         // This wakeup is made by a sync owner that still holds the drain lock
 252         DISPATCH_WAKEUP_BARRIER_COMPLETE        = 0x00000004,
 253
 254         // This wakeup is caused by a dispatch_block_wait()
 255         DISPATCH_WAKEUP_BLOCK_WAIT              = 0x00000008,
 256 );
 257
 258 typedef struct dispatch_invoke_context_s {
 259         struct dispatch_object_s *dic_deferred;
 260 #if HAVE_PTHREAD_WORKQUEUE_NARROWING
 261         uint64_t dic_next_narrow_check;
 262 #endif
 263 #if DISPATCH_COCOA_COMPAT
 264         void *dic_autorelease_pool;
 265 #endif
 266 } dispatch_invoke_context_s, *dispatch_invoke_context_t;
 267
 268 #if HAVE_PTHREAD_WORKQUEUE_NARROWING
 269 #define DISPATCH_THREAD_IS_NARROWING 1
 270
 271 #define dispatch_with_disabled_narrowing(dic, ...) ({ \
 272                 uint64_t suspend_narrow_check = dic->dic_next_narrow_check; \
 273                 dic->dic_next_narrow_check = 0; \
 274                 __VA_ARGS__; \
 275                 dic->dic_next_narrow_check = suspend_narrow_check; \
 276         })
 277 #else
 278 #define dispatch_with_disabled_narrowing(dic, ...) __VA_ARGS__
 279 #endif
 280
 281 DISPATCH_ENUM(dispatch_invoke_flags, uint32_t,
 282         DISPATCH_INVOKE_NONE                                    = 0x00000000,
 283
 284         // Invoke modes
 285         //
 286         // @const DISPATCH_INVOKE_STEALING
 287         // This invoke is a stealer, meaning that it doesn't own the
 288         // enqueue lock at drain lock time.
 289         //
 290         // @const DISPATCH_INVOKE_WLH
 291         // This invoke is for a bottom WLH
 292         //
 293         DISPATCH_INVOKE_STEALING                                = 0x00000001,
 294         DISPATCH_INVOKE_WLH                                             = 0x00000002,
 295
 296         // Misc flags
 297         //
 298         // @const DISPATCH_INVOKE_ASYNC_REPLY
 299         // An asynchronous reply to a message is being handled.
 300         //
 301         // @const DISPATCH_INVOKE_DISALLOW_SYNC_WAITERS
 302         // The next serial drain should not allow sync waiters.
 303         //
 304         DISPATCH_INVOKE_ASYNC_REPLY                             = 0x00000004,
 305         DISPATCH_INVOKE_DISALLOW_SYNC_WAITERS   = 0x00000008,
 306
 307         // Below this point flags are propagated to recursive calls to drain(),
 308         // continuation pop() or dx_invoke().
 309 #define _DISPATCH_INVOKE_PROPAGATE_MASK           0xffff0000u
 310
 311         // Drain modes
 312         //
 313         // @const DISPATCH_INVOKE_WORKER_DRAIN
 314         // Invoke has been issued by a worker thread (work queue thread, or
 315         // pthread root queue) drain. This flag is NOT set when the main queue,
 316         // manager queue or runloop queues are drained
 317         //
 318         // @const DISPATCH_INVOKE_REDIRECTING_DRAIN
 319         // Has only been draining concurrent queues so far
 320         // Implies DISPATCH_INVOKE_WORKER_DRAIN
 321         //
 322         // @const DISPATCH_INVOKE_MANAGER_DRAIN
 323         // We're draining from a manager context
 324         //
 325         DISPATCH_INVOKE_WORKER_DRAIN                    = 0x00010000,
 326         DISPATCH_INVOKE_REDIRECTING_DRAIN               = 0x00020000,
 327         DISPATCH_INVOKE_MANAGER_DRAIN                   = 0x00040000,
 328 #define _DISPATCH_INVOKE_DRAIN_MODE_MASK          0x000f0000u
 329
 330         // Autoreleasing modes
 331         //
 332         // @const DISPATCH_INVOKE_AUTORELEASE_ALWAYS
 333         // Always use autoreleasepools around callouts
 334         //
 335         // @const DISPATCH_INVOKE_AUTORELEASE_NEVER
 336         // Never use autoreleasepools around callouts
 337         //
 338         DISPATCH_INVOKE_AUTORELEASE_ALWAYS              = 0x00100000,
 339         DISPATCH_INVOKE_AUTORELEASE_NEVER               = 0x00200000,
 340 #define _DISPATCH_INVOKE_AUTORELEASE_MASK         0x00300000u
 341 );
 342
 343 enum {
 344         _DISPATCH_META_TYPE_MASK                = 0xffff0000, // mask for object meta-types
 345         _DISPATCH_TYPEFLAGS_MASK                = 0x0000ff00, // mask for object typeflags
 346         _DISPATCH_SUB_TYPE_MASK                 = 0x000000ff, // mask for object sub-types
 347
 348         _DISPATCH_CONTINUATION_TYPE             =    0x00000, // meta-type for continuations
 349         _DISPATCH_QUEUE_TYPE                    =    0x10000, // meta-type for queues
 350         _DISPATCH_SOURCE_TYPE                   =    0x20000, // meta-type for sources
 351         _DISPATCH_SEMAPHORE_TYPE                =    0x30000, // meta-type for semaphores
 352         _DISPATCH_NODE_TYPE                             =    0x40000, // meta-type for data node
 353         _DISPATCH_IO_TYPE                               =    0x50000, // meta-type for io channels
 354         _DISPATCH_OPERATION_TYPE                =    0x60000, // meta-type for io operations
 355         _DISPATCH_DISK_TYPE                             =    0x70000, // meta-type for io disks
 356
 357         _DISPATCH_QUEUE_ROOT_TYPEFLAG   =     0x0100, // bit set for any root queues
 358         _DISPATCH_QUEUE_BASE_TYPEFLAG   =     0x0200, // base of a hierarchy
 359                                                       // targets a root queue
 360
 361 #define DISPATCH_CONTINUATION_TYPE(name)  \
 362                 (_DISPATCH_CONTINUATION_TYPE | DC_##name##_TYPE)
 363         DISPATCH_DATA_TYPE                                      = 1 | _DISPATCH_NODE_TYPE,
 364         DISPATCH_MACH_MSG_TYPE                          = 2 | _DISPATCH_NODE_TYPE,
 365         DISPATCH_QUEUE_ATTR_TYPE                        = 3 | _DISPATCH_NODE_TYPE,
 366
 367         DISPATCH_IO_TYPE                                        = 0 | _DISPATCH_IO_TYPE,
 368         DISPATCH_OPERATION_TYPE                         = 0 | _DISPATCH_OPERATION_TYPE,
 369         DISPATCH_DISK_TYPE                                      = 0 | _DISPATCH_DISK_TYPE,
 370
 371         DISPATCH_QUEUE_LEGACY_TYPE                      = 1 | _DISPATCH_QUEUE_TYPE,
 372         DISPATCH_QUEUE_SERIAL_TYPE                      = 2 | _DISPATCH_QUEUE_TYPE,
 373         DISPATCH_QUEUE_CONCURRENT_TYPE          = 3 | _DISPATCH_QUEUE_TYPE,
 374         DISPATCH_QUEUE_GLOBAL_ROOT_TYPE         = 4 | _DISPATCH_QUEUE_TYPE |
 375                         _DISPATCH_QUEUE_ROOT_TYPEFLAG,
 376         DISPATCH_QUEUE_NETWORK_EVENT_TYPE       = 5 | _DISPATCH_QUEUE_TYPE |
 377                         _DISPATCH_QUEUE_BASE_TYPEFLAG,
 378         DISPATCH_QUEUE_RUNLOOP_TYPE                     = 6 | _DISPATCH_QUEUE_TYPE |
 379                         _DISPATCH_QUEUE_BASE_TYPEFLAG,
 380         DISPATCH_QUEUE_MGR_TYPE                         = 7 | _DISPATCH_QUEUE_TYPE |
 381                         _DISPATCH_QUEUE_BASE_TYPEFLAG,
 382         DISPATCH_QUEUE_SPECIFIC_TYPE            = 8 | _DISPATCH_QUEUE_TYPE,
 383
 384         DISPATCH_SEMAPHORE_TYPE                         = 1 | _DISPATCH_SEMAPHORE_TYPE,
 385         DISPATCH_GROUP_TYPE                                     = 2 | _DISPATCH_SEMAPHORE_TYPE,
 386
 387         DISPATCH_SOURCE_KEVENT_TYPE                     = 1 | _DISPATCH_SOURCE_TYPE,
 388         DISPATCH_MACH_CHANNEL_TYPE                      = 2 | _DISPATCH_SOURCE_TYPE,
 389
 390 };
 391
 392 typedef struct _os_object_vtable_s {
 393         _OS_OBJECT_CLASS_HEADER();
 394 } _os_object_vtable_s;
 395
 396 typedef struct _os_object_s {
 397         _OS_OBJECT_HEADER(
 398         const _os_object_vtable_s *os_obj_isa,
 399         os_obj_ref_cnt,
 400         os_obj_xref_cnt);
 401 } _os_object_s;
 402
 403 #if OS_OBJECT_HAVE_OBJC1
 404 #define OS_OBJECT_STRUCT_HEADER(x) \
 405         _OS_OBJECT_HEADER(\
 406         const void *_objc_isa, \
 407         do_ref_cnt, \
 408         do_xref_cnt); \
 409         const struct x##_vtable_s *do_vtable
 410 #else
 411 #define OS_OBJECT_STRUCT_HEADER(x) \
 412         _OS_OBJECT_HEADER(\
 413         const struct x##_vtable_s *do_vtable, \
 414         do_ref_cnt, \
 415         do_xref_cnt)
 416 #endif
 417
 418 #define _DISPATCH_OBJECT_HEADER(x) \
 419         struct _os_object_s _as_os_obj[0]; \
 420         OS_OBJECT_STRUCT_HEADER(dispatch_##x); \
 421         struct dispatch_##x##_s *volatile do_next; \
 422         struct dispatch_queue_s *do_targetq; \
 423         void *do_ctxt; \
 424         void *do_finalizer
 425
 426 #define DISPATCH_OBJECT_HEADER(x) \
 427         struct dispatch_object_s _as_do[0]; \
 428         _DISPATCH_OBJECT_HEADER(x)
 429
 430 // Swift-unavailable -init requires method in each class.
 431 #define DISPATCH_UNAVAILABLE_INIT() \
 432         - (instancetype)init { \
 433                 DISPATCH_CLIENT_CRASH(0, "-init called directly"); \
 434                 return [super init]; \
 435         }
 436
 437 _OS_OBJECT_DECL_PROTOCOL(dispatch_object, object);
 438
 439 OS_OBJECT_CLASS_DECL(dispatch_object, object,
 440                 DISPATCH_OBJECT_VTABLE_HEADER(dispatch_object));
 441
 442 struct dispatch_object_s {
 443         _DISPATCH_OBJECT_HEADER(object);
 444 };
 445
 446 #if OS_OBJECT_HAVE_OBJC1
 447 #define _OS_MPSC_QUEUE_FIELDS(ns, __state_field__) \
 448         DISPATCH_UNION_LE(uint64_t volatile __state_field__, \
 449                         dispatch_lock __state_field__##_lock, \
 450                         uint32_t __state_field__##_bits \
 451         ) DISPATCH_ATOMIC64_ALIGN; \
 452         struct dispatch_object_s *volatile ns##_items_head; \
 453         unsigned long ns##_serialnum; \
 454         const char *ns##_label; \
 455         struct dispatch_object_s *volatile ns##_items_tail; \
 456         dispatch_priority_t ns##_priority; \
 457         int volatile ns##_sref_cnt
 458 #else
 459 #define _OS_MPSC_QUEUE_FIELDS(ns, __state_field__) \
 460         struct dispatch_object_s *volatile ns##_items_head; \
 461         DISPATCH_UNION_LE(uint64_t volatile __state_field__, \
 462                         dispatch_lock __state_field__##_lock, \
 463                         uint32_t __state_field__##_bits \
 464         ) DISPATCH_ATOMIC64_ALIGN; \
 465         /* LP64 global queue cacheline boundary */ \
 466         unsigned long ns##_serialnum; \
 467         const char *ns##_label; \
 468         struct dispatch_object_s *volatile ns##_items_tail; \
 469         dispatch_priority_t ns##_priority; \
 470         int volatile ns##_sref_cnt
 471 #endif
 472
 473 OS_OBJECT_INTERNAL_CLASS_DECL(os_mpsc_queue, object,
 474                 DISPATCH_QUEUEABLE_VTABLE_HEADER(os_mpsc_queue));
 475
 476 struct os_mpsc_queue_s {
 477         struct _os_object_s _as_os_obj[0];
 478         OS_OBJECT_STRUCT_HEADER(os_mpsc_queue);
 479         struct dispatch_object_s *volatile oq_next;
 480         void *oq_opaque1; // do_targetq
 481         void *oq_opaque2; // do_ctxt
 482         void *oq_opaque3; // do_finalizer
 483         _OS_MPSC_QUEUE_FIELDS(oq, __oq_state_do_not_use);
 484 };
 485
 486 size_t _dispatch_object_debug_attr(dispatch_object_t dou, char* buf,
 487                 size_t bufsiz);
 488 void *_dispatch_object_alloc(const void *vtable, size_t size);
 489 void _dispatch_object_finalize(dispatch_object_t dou);
 490 void _dispatch_object_dealloc(dispatch_object_t dou);
 491 #if !USE_OBJC
 492 void _dispatch_xref_dispose(dispatch_object_t dou);
 493 #endif
 494 void _dispatch_dispose(dispatch_object_t dou);
 495 #if DISPATCH_COCOA_COMPAT
 496 #if USE_OBJC
 497 #include <objc/runtime.h>
 498 #if __has_include(<objc/objc-internal.h>)
 499 #include <objc/objc-internal.h>
 500 #else
 501 extern void *objc_autoreleasePoolPush(void);
 502 extern void objc_autoreleasePoolPop(void *context);
 503 #endif // __has_include(<objc/objc-internal.h>)
 504 #define _dispatch_autorelease_pool_push() \
 505                 objc_autoreleasePoolPush()
 506 #define _dispatch_autorelease_pool_pop(context) \
 507                 objc_autoreleasePoolPop(context)
 508 #else
 509 void *_dispatch_autorelease_pool_push(void);
 510 void _dispatch_autorelease_pool_pop(void *context);
 511 #endif
 512 void _dispatch_last_resort_autorelease_pool_push(dispatch_invoke_context_t dic);
 513 void _dispatch_last_resort_autorelease_pool_pop(dispatch_invoke_context_t dic);
 514
 515 #define dispatch_invoke_with_autoreleasepool(flags, ...)  ({ \
 516                 void *pool = NULL; \
 517                 if ((flags) & DISPATCH_INVOKE_AUTORELEASE_ALWAYS) { \
 518                         pool = _dispatch_autorelease_pool_push(); \
 519                         DISPATCH_COMPILER_CAN_ASSUME(pool); \
 520                 }; \
 521                 __VA_ARGS__; \
 522                 if (pool) _dispatch_autorelease_pool_pop(pool); \
 523         })
 524 #else
 525 #define dispatch_invoke_with_autoreleasepool(flags, ...) \
 526         do { (void)flags; __VA_ARGS__; } while (0)
 527 #endif
 528
 529 #if USE_OBJC
 530 OS_OBJECT_OBJC_CLASS_DECL(object);
 531 #endif
 532
 533 #if OS_OBJECT_HAVE_OBJC2
 534 // ObjC toll-free bridging, keep in sync with libdispatch.order file
 535 //
 536 // This is required by the dispatch_data_t/NSData bridging, which is not
 537 // supported on the old runtime.
 538 #define DISPATCH_OBJECT_TFB(f, o, ...) \
 539         if (slowpath((uintptr_t)((o)._os_obj->os_obj_isa) & 1) || \
 540                         slowpath((Class)((o)._os_obj->os_obj_isa) < \
 541                                         (Class)OS_OBJECT_VTABLE(dispatch_object)) || \
 542                         slowpath((Class)((o)._os_obj->os_obj_isa) >= \
 543                                         (Class)OS_OBJECT_VTABLE(object))) { \
 544                 return f((o), ##__VA_ARGS__); \
 545         }
 546
 547 id _dispatch_objc_alloc(Class cls, size_t size);
 548 void _dispatch_objc_retain(dispatch_object_t dou);
 549 void _dispatch_objc_release(dispatch_object_t dou);
 550 void _dispatch_objc_set_context(dispatch_object_t dou, void *context);
 551 void *_dispatch_objc_get_context(dispatch_object_t dou);
 552 void _dispatch_objc_set_finalizer_f(dispatch_object_t dou,
 553                 dispatch_function_t finalizer);
 554 void _dispatch_objc_set_target_queue(dispatch_object_t dou,
 555                 dispatch_queue_t queue);
 556 void _dispatch_objc_suspend(dispatch_object_t dou);
 557 void _dispatch_objc_resume(dispatch_object_t dou);
 558 void _dispatch_objc_activate(dispatch_object_t dou);
 559 size_t _dispatch_objc_debug(dispatch_object_t dou, char* buf, size_t bufsiz);
 560
 561 #if __OBJC2__
 562 @interface NSObject (DISPATCH_CONCAT(_,DISPATCH_CLASS(object)))
 563 - (void)_setContext:(void*)context;
 564 - (void*)_getContext;
 565 - (void)_setFinalizer:(dispatch_function_t)finalizer;
 566 - (void)_setTargetQueue:(dispatch_queue_t)queue;
 567 - (void)_suspend;
 568 - (void)_resume;
 569 - (void)_activate;
 570 @end
 571 #endif // __OBJC2__
 572 #else
 573 #define DISPATCH_OBJECT_TFB(f, o, ...)
 574 #endif // OS_OBJECT_HAVE_OBJC2
 575
 576 #pragma mark -
 577 #pragma mark _os_object_s
 578
 579 /*
 580  * Low level _os_atomic_refcnt_* actions
 581  *
 582  * _os_atomic_refcnt_inc2o(o, f):
 583  *   performs a refcount increment and returns the new refcount value
 584  *
 585  * _os_atomic_refcnt_dec2o(o, f):
 586  *   performs a refcount decrement and returns the new refcount value
 587  *
 588  * _os_atomic_refcnt_dispose_barrier2o(o, f):
 589  *   a barrier to perform prior to tearing down an object when the refcount
 590  *   reached -1.
 591  */
 592 #define _os_atomic_refcnt_perform2o(o, f, op, n, m)   ({ \
 593                 typeof(o) _o = (o); \
 594                 int _ref_cnt = _o->f; \
 595                 if (fastpath(_ref_cnt != _OS_OBJECT_GLOBAL_REFCNT)) { \
 596                         _ref_cnt = os_atomic_##op##2o(_o, f, n, m); \
 597                 } \
 598                 _ref_cnt; \
 599         })
 600
 601 #define _os_atomic_refcnt_add2o(o, m, n) \
 602                 _os_atomic_refcnt_perform2o(o, m, add, n, relaxed)
 603
 604 #define _os_atomic_refcnt_sub2o(o, m, n) \
 605                 _os_atomic_refcnt_perform2o(o, m, sub, n, release)
 606
 607 #define _os_atomic_refcnt_dispose_barrier2o(o, m) \
 608                 (void)os_atomic_load2o(o, m, acquire)
 609
 610
 611 /*
 612  * Higher level _os_object_{x,}refcnt_* actions
 613  *
 614  * _os_atomic_{x,}refcnt_inc(o):
 615  *   increment the external (resp. internal) refcount and
 616  *   returns the new refcount value
 617  *
 618  * _os_atomic_{x,}refcnt_dec(o):
 619  *   decrement the external (resp. internal) refcount and
 620  *   returns the new refcount value
 621  *
 622  * _os_atomic_{x,}refcnt_dispose_barrier(o):
 623  *   performs the pre-teardown barrier for the external
 624  *   (resp. internal) refcount
 625  *
 626  */
 627 #define _os_object_xrefcnt_inc(o) \
 628                 _os_atomic_refcnt_add2o(o, os_obj_xref_cnt, 1)
 629
 630 #define _os_object_xrefcnt_dec(o) \
 631                 _os_atomic_refcnt_sub2o(o, os_obj_xref_cnt, 1)
 632
 633 #define _os_object_xrefcnt_dispose_barrier(o) \
 634                 _os_atomic_refcnt_dispose_barrier2o(o, os_obj_xref_cnt)
 635
 636 #define _os_object_refcnt_add(o, n) \
 637                 _os_atomic_refcnt_add2o(o, os_obj_ref_cnt, n)
 638
 639 #define _os_object_refcnt_sub(o, n) \
 640                 _os_atomic_refcnt_sub2o(o, os_obj_ref_cnt, n)
 641
 642 #define _os_object_refcnt_dispose_barrier(o) \
 643                 _os_atomic_refcnt_dispose_barrier2o(o, os_obj_ref_cnt)
 644
 645 void _os_object_atfork_child(void);
 646 void _os_object_atfork_parent(void);
 647 void _os_object_atfork_prepare(void);
 648 void _os_object_init(void);
 649 unsigned long _os_object_retain_count(_os_object_t obj);
 650 bool _os_object_retain_weak(_os_object_t obj);
 651 bool _os_object_allows_weak_reference(_os_object_t obj);
 652 void _os_object_dispose(_os_object_t obj);
 653 void _os_object_xref_dispose(_os_object_t obj);
 654
 655 #endif // __DISPATCH_OBJECT_INTERNAL__