src/object_internal.h

   1 /*
   2  * Copyright (c) 2008-2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_APACHE_LICENSE_HEADER_START@
   5  *
   6  * Licensed under the Apache License, Version 2.0 (the "License");
   7  * you may not use this file except in compliance with the License.
   8  * You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  *
  18  * @APPLE_APACHE_LICENSE_HEADER_END@
  19  */
  20
  21 /*
  22  * IMPORTANT: This header file describes INTERNAL interfaces to libdispatch
  23  * which are subject to change in future releases of Mac OS X. Any applications
  24  * relying on these interfaces WILL break.
  25  */
  26
  27 #ifndef __DISPATCH_OBJECT_INTERNAL__
  28 #define __DISPATCH_OBJECT_INTERNAL__
  29
  30 #if !OS_OBJECT_USE_OBJC
  31 #define OS_OBJECT_DECL(name)  DISPATCH_DECL(name)
  32 #define OS_OBJECT_DECL_SUBCLASS(name, super)  DISPATCH_DECL(name)
  33 #endif
  34
  35 #if USE_OBJC
  36 #define OS_OBJECT_EXTRA_VTABLE_SYMBOL(name) _OS_##name##_vtable
  37 #define DISPATCH_CLASS_SYMBOL(name) OS_dispatch_##name##_class
  38 #define DISPATCH_CLASS_RAW_SYMBOL_NAME(name) \
  39                 OS_OBJC_CLASS_RAW_SYMBOL_NAME(DISPATCH_CLASS(name))
  40 #else
  41 #define OS_OBJECT_CLASS_SYMBOL(name) _##name##_vtable
  42 #define OS_OBJC_CLASS_RAW_SYMBOL_NAME(name) \
  43                 "__" OS_STRINGIFY(name) "_vtable"
  44 #define DISPATCH_CLASS_SYMBOL(name) _dispatch_##name##_vtable
  45 #define DISPATCH_CLASS_RAW_SYMBOL_NAME(name) \
  46                 "__dispatch_" OS_STRINGIFY(name) "_vtable"
  47 #endif
  48
  49 #define DISPATCH_CLASS(name) OS_dispatch_##name
  50 #if USE_OBJC
  51 #define DISPATCH_OBJC_CLASS_DECL(name) \
  52                 extern void *DISPATCH_CLASS_SYMBOL(name) \
  53                                 asm(DISPATCH_CLASS_RAW_SYMBOL_NAME(name))
  54 #endif
  55
  56 // define a new proper class
  57 #define OS_OBJECT_CLASS_DECL(name, super, ...) \
  58                 struct name##_s; \
  59                 struct name##_extra_vtable_s { \
  60                         __VA_ARGS__; \
  61                 }; \
  62                 struct name##_vtable_s { \
  63                         _OS_OBJECT_CLASS_HEADER(); \
  64                         struct name##_extra_vtable_s _os_obj_vtable; \
  65                 }; \
  66                 OS_OBJECT_EXTRA_VTABLE_DECL(name, name) \
  67                 extern const struct name##_vtable_s OS_OBJECT_CLASS_SYMBOL(name) \
  68                                 asm(OS_OBJC_CLASS_RAW_SYMBOL_NAME(OS_OBJECT_CLASS(name)))
  69
  70 #if OS_OBJECT_SWIFT3
  71 #define OS_OBJECT_INTERNAL_CLASS_DECL(name, super, ...) \
  72                 OS_OBJECT_OBJC_RUNTIME_VISIBLE \
  73                 OS_OBJECT_DECL_IMPL_CLASS(name, OS_OBJECT_CLASS(super)); \
  74                 OS_OBJECT_CLASS_DECL(name, super, ## __VA_ARGS__)
  75 #elif OS_OBJECT_USE_OBJC
  76 #define OS_OBJECT_INTERNAL_CLASS_DECL(name, super, ...) \
  77                 OS_OBJECT_DECL(name); \
  78                 OS_OBJECT_CLASS_DECL(name, super, ## __VA_ARGS__)
  79 #else
  80 #define OS_OBJECT_INTERNAL_CLASS_DECL(name, super, ...) \
  81                 typedef struct name##_s *name##_t; \
  82                 OS_OBJECT_CLASS_DECL(name, super, ## __VA_ARGS__)
  83 #endif
  84
  85 #define DISPATCH_CLASS_DECL_BARE(name) \
  86                 OS_OBJECT_CLASS_DECL(dispatch_##name, dispatch_object, \
  87                                 DISPATCH_OBJECT_VTABLE_HEADER(dispatch_##name))
  88
  89 #define DISPATCH_CLASS_DECL(name) \
  90                 _OS_OBJECT_DECL_PROTOCOL(dispatch_##name, dispatch_object) \
  91                 _OS_OBJECT_CLASS_IMPLEMENTS_PROTOCOL(dispatch_##name, dispatch_##name) \
  92                 DISPATCH_CLASS_DECL_BARE(name)
  93
  94 #define DISPATCH_INTERNAL_CLASS_DECL(name) \
  95                 DISPATCH_DECL(dispatch_##name); \
  96                 DISPATCH_CLASS_DECL(name)
  97
  98 // define a new subclass used in a cluster
  99 #define OS_OBJECT_SUBCLASS_DECL(name, super) \
 100                 _OS_OBJECT_DECL_SUBCLASS_INTERFACE(name, super) \
 101                 struct name##_s; \
 102                 OS_OBJECT_EXTRA_VTABLE_DECL(name, super) \
 103                 extern const struct super##_vtable_s OS_OBJECT_CLASS_SYMBOL(name) \
 104                                 asm(OS_OBJC_CLASS_RAW_SYMBOL_NAME(OS_OBJECT_CLASS(name)))
 105
 106 #define DISPATCH_SUBCLASS_DECL(name, super) \
 107                 OS_OBJECT_SUBCLASS_DECL(dispatch_##name, super)
 108
 109 #if OS_OBJECT_SWIFT3
 110 // define a new internal subclass used in a class cluster
 111 #define OS_OBJECT_INTERNAL_SUBCLASS_DECL(name, super) \
 112                 _OS_OBJECT_DECL_PROTOCOL(name, super); \
 113                 OS_OBJECT_SUBCLASS_DECL(name, super)
 114
 115 #define DISPATCH_INTERNAL_SUBCLASS_DECL(name, super) \
 116                 _OS_OBJECT_DECL_PROTOCOL(dispatch_##name, dispatch_##super) \
 117                 DISPATCH_SUBCLASS_DECL(name, dispatch_##super)
 118 #else
 119 // define a new internal subclass used in a class cluster
 120 #define OS_OBJECT_INTERNAL_SUBCLASS_DECL(name, super) \
 121                 OS_OBJECT_DECL_SUBCLASS(name, super); \
 122                 OS_OBJECT_SUBCLASS_DECL(name, super)
 123
 124 #define DISPATCH_INTERNAL_SUBCLASS_DECL(name, super) \
 125                 OS_OBJECT_DECL_SUBCLASS(dispatch_##name, dispatch_##super); \
 126                 DISPATCH_SUBCLASS_DECL(name, dispatch_##super)
 127 #endif
 128
 129 // vtable symbols
 130 #define OS_OBJECT_VTABLE(name)          (&OS_OBJECT_CLASS_SYMBOL(name))
 131 #define DISPATCH_OBJC_CLASS(name)       (&DISPATCH_CLASS_SYMBOL(name))
 132
 133 // vtables for subclasses used in a class cluster
 134 #if USE_OBJC
 135 // ObjC classes and dispatch vtables are co-located via linker order and alias
 136 // files rdar://10640168
 137 #if OS_OBJECT_HAVE_OBJC2
 138 #define OS_OBJECT_VTABLE_SUBCLASS_INSTANCE(name, super, xdispose, dispose, ...) \
 139                 __attribute__((section("__DATA,__objc_data"), used)) \
 140                 const struct super##_extra_vtable_s \
 141                 OS_OBJECT_EXTRA_VTABLE_SYMBOL(name) = { __VA_ARGS__ }
 142 #define OS_OBJECT_EXTRA_VTABLE_DECL(name, super)
 143 #define DISPATCH_VTABLE(name) DISPATCH_OBJC_CLASS(name)
 144 #else
 145 #define OS_OBJECT_VTABLE_SUBCLASS_INSTANCE(name, super, xdispose, dispose, ...) \
 146                 const struct super##_vtable_s \
 147                 OS_OBJECT_EXTRA_VTABLE_SYMBOL(name) = { \
 148                         ._os_obj_objc_isa = &OS_OBJECT_CLASS_SYMBOL(name), \
 149                         ._os_obj_vtable = { __VA_ARGS__ }, \
 150                 }
 151 #define OS_OBJECT_EXTRA_VTABLE_DECL(name, super) \
 152                 extern const struct super##_vtable_s \
 153                                 OS_OBJECT_EXTRA_VTABLE_SYMBOL(name);
 154 #define DISPATCH_VTABLE(name) &OS_OBJECT_EXTRA_VTABLE_SYMBOL(dispatch_##name)
 155 #endif
 156 #else
 157 #define OS_OBJECT_VTABLE_SUBCLASS_INSTANCE(name, super, xdispose, dispose, ...) \
 158                 const struct super##_vtable_s OS_OBJECT_CLASS_SYMBOL(name) = { \
 159                         ._os_obj_xref_dispose = xdispose, \
 160                         ._os_obj_dispose = dispose, \
 161                         ._os_obj_vtable = { __VA_ARGS__ }, \
 162                 }
 163 #define OS_OBJECT_EXTRA_VTABLE_DECL(name, super)
 164 #define DISPATCH_VTABLE(name) DISPATCH_OBJC_CLASS(name)
 165 #endif // USE_OBJC
 166
 167 #define DISPATCH_VTABLE_SUBCLASS_INSTANCE(name, super, ...) \
 168                 OS_OBJECT_VTABLE_SUBCLASS_INSTANCE(dispatch_##name, dispatch_##super, \
 169                                 _dispatch_xref_dispose, _dispatch_dispose, __VA_ARGS__)
 170
 171 // vtables for proper classes
 172 #define OS_OBJECT_VTABLE_INSTANCE(name, xdispose, dispose, ...) \
 173                 OS_OBJECT_VTABLE_SUBCLASS_INSTANCE(name, name, \
 174                                 xdispose, dispose, __VA_ARGS__)
 175
 176 #define DISPATCH_VTABLE_INSTANCE(name, ...) \
 177                 DISPATCH_VTABLE_SUBCLASS_INSTANCE(name, name, __VA_ARGS__)
 178
 179 #define DISPATCH_INVOKABLE_VTABLE_HEADER(x) \
 180         unsigned long const do_type; \
 181         const char *const do_kind; \
 182         void (*const do_invoke)(struct x##_s *, dispatch_invoke_flags_t)
 183
 184 #define DISPATCH_QUEUEABLE_VTABLE_HEADER(x) \
 185         DISPATCH_INVOKABLE_VTABLE_HEADER(x); \
 186         void (*const do_wakeup)(struct x##_s *, \
 187                         pthread_priority_t, dispatch_wakeup_flags_t); \
 188         void (*const do_dispose)(struct x##_s *)
 189
 190 #define DISPATCH_OBJECT_VTABLE_HEADER(x) \
 191         DISPATCH_QUEUEABLE_VTABLE_HEADER(x); \
 192         void (*const do_set_targetq)(struct x##_s *, dispatch_queue_t); \
 193         void (*const do_suspend)(struct x##_s *); \
 194         void (*const do_resume)(struct x##_s *, bool activate); \
 195         void (*const do_finalize_activation)(struct x##_s *); \
 196         size_t (*const do_debug)(struct x##_s *, char *, size_t)
 197
 198 #define dx_vtable(x) (&(x)->do_vtable->_os_obj_vtable)
 199 #define dx_type(x) dx_vtable(x)->do_type
 200 #define dx_subtype(x) (dx_vtable(x)->do_type & _DISPATCH_SUB_TYPE_MASK)
 201 #define dx_metatype(x) (dx_vtable(x)->do_type & _DISPATCH_META_TYPE_MASK)
 202 #define dx_hastypeflag(x, f) (dx_vtable(x)->do_type & _DISPATCH_##f##_TYPEFLAG)
 203 #define dx_kind(x) dx_vtable(x)->do_kind
 204 #define dx_debug(x, y, z) dx_vtable(x)->do_debug((x), (y), (z))
 205 #define dx_dispose(x) dx_vtable(x)->do_dispose(x)
 206 #define dx_invoke(x, z) dx_vtable(x)->do_invoke(x, z)
 207 #define dx_wakeup(x, y, z) dx_vtable(x)->do_wakeup(x, y, z)
 208
 209 #define DISPATCH_OBJECT_GLOBAL_REFCNT           _OS_OBJECT_GLOBAL_REFCNT
 210
 211 #if OS_OBJECT_HAVE_OBJC1
 212 #define DISPATCH_GLOBAL_OBJECT_HEADER(name) \
 213         .do_vtable = DISPATCH_VTABLE(name), \
 214         ._objc_isa = DISPATCH_OBJC_CLASS(name), \
 215         .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, \
 216         .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT
 217 #else
 218 #define DISPATCH_GLOBAL_OBJECT_HEADER(name) \
 219         .do_vtable = DISPATCH_VTABLE(name), \
 220         .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, \
 221         .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT
 222 #endif
 223
 224 #ifdef __LP64__
 225 // the bottom nibble must not be zero, the rest of the bits should be random
 226 // we sign extend the 64-bit version so that a better instruction encoding is
 227 // generated on Intel
 228 #define DISPATCH_OBJECT_LISTLESS ((void *)0xffffffff89abcdef)
 229 #else
 230 #define DISPATCH_OBJECT_LISTLESS ((void *)0x89abcdef)
 231 #endif
 232
 233 DISPATCH_ENUM(dispatch_wakeup_flags, uint32_t,
 234         // The caller of dx_wakeup owns an internal refcount on the object being
 235         // woken up
 236         DISPATCH_WAKEUP_CONSUME                 = 0x00000001,
 237
 238         // Some change to the object needs to be published to drainers.
 239         // If the drainer isn't the same thread, some scheme such as the dispatch
 240         // queue DIRTY bit must be used and a release barrier likely has to be
 241         // involved before dx_wakeup returns
 242         DISPATCH_WAKEUP_FLUSH                                   = 0x00000002,
 243
 244         // A slow waiter was just enqueued
 245         DISPATCH_WAKEUP_SLOW_WAITER                             = 0x00000004,
 246
 247         // The caller desires to apply an override on the object being woken up
 248         // and has already adjusted the `oq_override` field. When this flag is
 249         // passed, the priority passed to dx_wakeup() should not be 0
 250         DISPATCH_WAKEUP_OVERRIDING              = 0x00000008,
 251
 252         // At the time this queue was woken up it had an override that must be
 253         // preserved (used to solve a race with _dispatch_queue_drain_try_unlock())
 254         DISPATCH_WAKEUP_WAS_OVERRIDDEN          = 0x00000010,
 255
 256 #define _DISPATCH_WAKEUP_OVERRIDE_BITS \
 257                 ((dispatch_wakeup_flags_t)(DISPATCH_WAKEUP_OVERRIDING | \
 258                 DISPATCH_WAKEUP_WAS_OVERRIDDEN))
 259 );
 260
 261 DISPATCH_ENUM(dispatch_invoke_flags, uint32_t,
 262         DISPATCH_INVOKE_NONE                                    = 0x00000000,
 263
 264         // Invoke modes
 265         //
 266         // @const DISPATCH_INVOKE_STEALING
 267         // This invoke is a stealer, meaning that it doesn't own the
 268         // enqueue lock at drain lock time.
 269         //
 270         // @const DISPATCH_INVOKE_OVERRIDING
 271         // This invoke is draining the hierarchy on another root queue and needs
 272         // to fake the identity of the original one.
 273         //
 274         DISPATCH_INVOKE_STEALING                                = 0x00000001,
 275         DISPATCH_INVOKE_OVERRIDING                              = 0x00000002,
 276
 277         // Below this point flags are propagated to recursive calls to drain(),
 278         // continuation pop() or dx_invoke().
 279 #define _DISPATCH_INVOKE_PROPAGATE_MASK           0xffff0000u
 280
 281         // Drain modes
 282         //
 283         // @const DISPATCH_INVOKE_WORKER_DRAIN
 284         // Invoke has been issued by a worker thread (work queue thread, or
 285         // pthread root queue) drain. This flag is NOT set when the main queue,
 286         // manager queue or runloop queues are drained
 287         //
 288         // @const DISPATCH_INVOKE_REDIRECTING_DRAIN
 289         // Has only been draining concurrent queues so far
 290         // Implies DISPATCH_INVOKE_WORKER_DRAIN
 291         //
 292         // @const DISPATCH_INVOKE_MANAGER_DRAIN
 293         // We're draining from a manager context
 294         //
 295         DISPATCH_INVOKE_WORKER_DRAIN                    = 0x00010000,
 296         DISPATCH_INVOKE_REDIRECTING_DRAIN               = 0x00020000,
 297         DISPATCH_INVOKE_MANAGER_DRAIN                   = 0x00040000,
 298 #define _DISPATCH_INVOKE_DRAIN_MODE_MASK          0x000f0000u
 299
 300         // Autoreleasing modes
 301         //
 302         // @const DISPATCH_INVOKE_AUTORELEASE_ALWAYS
 303         // Always use autoreleasepools around callouts
 304         //
 305         // @const DISPATCH_INVOKE_AUTORELEASE_NEVER
 306         // Never use autoreleasepools around callouts
 307         //
 308         DISPATCH_INVOKE_AUTORELEASE_ALWAYS              = 0x00100000,
 309         DISPATCH_INVOKE_AUTORELEASE_NEVER               = 0x00200000,
 310 #define _DISPATCH_INVOKE_AUTORELEASE_MASK         0x00300000u
 311 );
 312
 313 enum {
 314         _DISPATCH_META_TYPE_MASK                = 0xffff0000, // mask for object meta-types
 315         _DISPATCH_TYPEFLAGS_MASK                = 0x0000ff00, // mask for object typeflags
 316         _DISPATCH_SUB_TYPE_MASK                 = 0x000000ff, // mask for object sub-types
 317
 318         _DISPATCH_CONTINUATION_TYPE             =    0x00000, // meta-type for continuations
 319         _DISPATCH_QUEUE_TYPE                    =    0x10000, // meta-type for queues
 320         _DISPATCH_SOURCE_TYPE                   =    0x20000, // meta-type for sources
 321         _DISPATCH_SEMAPHORE_TYPE                =    0x30000, // meta-type for semaphores
 322         _DISPATCH_NODE_TYPE                             =    0x40000, // meta-type for data node
 323         _DISPATCH_IO_TYPE                               =    0x50000, // meta-type for io channels
 324         _DISPATCH_OPERATION_TYPE                =    0x60000, // meta-type for io operations
 325         _DISPATCH_DISK_TYPE                             =    0x70000, // meta-type for io disks
 326
 327         _DISPATCH_QUEUE_ROOT_TYPEFLAG   =     0x0100, // bit set for any root queues
 328
 329 #define DISPATCH_CONTINUATION_TYPE(name)  \
 330                 (_DISPATCH_CONTINUATION_TYPE | DC_##name##_TYPE)
 331         DISPATCH_DATA_TYPE                              = 1 | _DISPATCH_NODE_TYPE,
 332         DISPATCH_MACH_MSG_TYPE                  = 2 | _DISPATCH_NODE_TYPE,
 333         DISPATCH_QUEUE_ATTR_TYPE                = 3 | _DISPATCH_NODE_TYPE,
 334
 335         DISPATCH_IO_TYPE                                = 0 | _DISPATCH_IO_TYPE,
 336         DISPATCH_OPERATION_TYPE                 = 0 | _DISPATCH_OPERATION_TYPE,
 337         DISPATCH_DISK_TYPE                              = 0 | _DISPATCH_DISK_TYPE,
 338
 339         DISPATCH_QUEUE_LEGACY_TYPE              = 1 | _DISPATCH_QUEUE_TYPE,
 340         DISPATCH_QUEUE_SERIAL_TYPE              = 2 | _DISPATCH_QUEUE_TYPE,
 341         DISPATCH_QUEUE_CONCURRENT_TYPE  = 3 | _DISPATCH_QUEUE_TYPE,
 342         DISPATCH_QUEUE_GLOBAL_ROOT_TYPE = 4 | _DISPATCH_QUEUE_TYPE |
 343                         _DISPATCH_QUEUE_ROOT_TYPEFLAG,
 344         DISPATCH_QUEUE_RUNLOOP_TYPE             = 5 | _DISPATCH_QUEUE_TYPE |
 345                         _DISPATCH_QUEUE_ROOT_TYPEFLAG,
 346         DISPATCH_QUEUE_MGR_TYPE                 = 6 | _DISPATCH_QUEUE_TYPE,
 347         DISPATCH_QUEUE_SPECIFIC_TYPE    = 7 | _DISPATCH_QUEUE_TYPE,
 348
 349         DISPATCH_SEMAPHORE_TYPE                 = 1 | _DISPATCH_SEMAPHORE_TYPE,
 350         DISPATCH_GROUP_TYPE                             = 2 | _DISPATCH_SEMAPHORE_TYPE,
 351
 352         DISPATCH_SOURCE_KEVENT_TYPE             = 1 | _DISPATCH_SOURCE_TYPE,
 353         DISPATCH_MACH_CHANNEL_TYPE              = 2 | _DISPATCH_SOURCE_TYPE,
 354
 355 };
 356
 357 typedef struct _os_object_vtable_s {
 358         _OS_OBJECT_CLASS_HEADER();
 359 } _os_object_vtable_s;
 360
 361 typedef struct _os_object_s {
 362         _OS_OBJECT_HEADER(
 363         const _os_object_vtable_s *os_obj_isa,
 364         os_obj_ref_cnt,
 365         os_obj_xref_cnt);
 366 } _os_object_s;
 367
 368 #if OS_OBJECT_HAVE_OBJC1
 369 #define OS_OBJECT_STRUCT_HEADER(x) \
 370         _OS_OBJECT_HEADER(\
 371         const void *_objc_isa, \
 372         do_ref_cnt, \
 373         do_xref_cnt); \
 374         const struct x##_vtable_s *do_vtable
 375 #else
 376 #define OS_OBJECT_STRUCT_HEADER(x) \
 377         _OS_OBJECT_HEADER(\
 378         const struct x##_vtable_s *do_vtable, \
 379         do_ref_cnt, \
 380         do_xref_cnt)
 381 #endif
 382
 383 #define _DISPATCH_OBJECT_HEADER(x) \
 384         struct _os_object_s _as_os_obj[0]; \
 385         OS_OBJECT_STRUCT_HEADER(dispatch_##x); \
 386         struct dispatch_##x##_s *volatile do_next; \
 387         struct dispatch_queue_s *do_targetq; \
 388         void *do_ctxt; \
 389         void *do_finalizer
 390
 391 #define DISPATCH_OBJECT_HEADER(x) \
 392         struct dispatch_object_s _as_do[0]; \
 393         _DISPATCH_OBJECT_HEADER(x)
 394
 395 // Swift-unavailable -init requires method in each class.
 396 #define DISPATCH_UNAVAILABLE_INIT() \
 397         - (instancetype)init { \
 398                 DISPATCH_CLIENT_CRASH(0, "-init called directly"); \
 399                 return [super init]; \
 400         }
 401
 402 _OS_OBJECT_DECL_PROTOCOL(dispatch_object, object);
 403
 404 OS_OBJECT_CLASS_DECL(dispatch_object, object,
 405                 DISPATCH_OBJECT_VTABLE_HEADER(dispatch_object));
 406
 407 struct dispatch_object_s {
 408         _DISPATCH_OBJECT_HEADER(object);
 409 };
 410
 411 #if OS_OBJECT_HAVE_OBJC1
 412 #define _OS_MPSC_QUEUE_FIELDS(ns, __state_field__) \
 413         struct dispatch_object_s *volatile ns##_items_head; \
 414         unsigned long ns##_serialnum; \
 415         union { \
 416                 uint64_t volatile __state_field__; \
 417                 DISPATCH_STRUCT_LITTLE_ENDIAN_2( \
 418                         dispatch_lock __state_field__##_lock, \
 419                         uint32_t __state_field__##_bits \
 420                 ); \
 421         }; /* needs to be 64-bit aligned */ \
 422         /* LP64 global queue cacheline boundary */ \
 423         const char *ns##_label; \
 424         voucher_t ns##_override_voucher; \
 425         dispatch_priority_t ns##_priority; \
 426         dispatch_priority_t volatile ns##_override; \
 427         struct dispatch_object_s *volatile ns##_items_tail
 428 #else
 429 #define _OS_MPSC_QUEUE_FIELDS(ns, __state_field__) \
 430         struct dispatch_object_s *volatile ns##_items_head; \
 431         union { \
 432                 uint64_t volatile __state_field__; \
 433                 DISPATCH_STRUCT_LITTLE_ENDIAN_2( \
 434                         dispatch_lock __state_field__##_lock, \
 435                         uint32_t __state_field__##_bits \
 436                 ); \
 437         }; /* needs to be 64-bit aligned */ \
 438         /* LP64 global queue cacheline boundary */ \
 439         unsigned long ns##_serialnum; \
 440         const char *ns##_label; \
 441         voucher_t ns##_override_voucher; \
 442         dispatch_priority_t ns##_priority; \
 443         dispatch_priority_t volatile ns##_override; \
 444         struct dispatch_object_s *volatile ns##_items_tail
 445 #endif
 446
 447 OS_OBJECT_INTERNAL_CLASS_DECL(os_mpsc_queue, object,
 448                 DISPATCH_QUEUEABLE_VTABLE_HEADER(os_mpsc_queue));
 449
 450 struct os_mpsc_queue_s {
 451         struct _os_object_s _as_os_obj[0];
 452         OS_OBJECT_STRUCT_HEADER(os_mpsc_queue);
 453         struct dispatch_object_s *volatile oq_next;
 454         void *oq_opaque1; // do_targetq
 455         void *oq_opaque2; // do_ctxt
 456         void *oq_opaque3; // do_finalizer
 457         _OS_MPSC_QUEUE_FIELDS(oq, __oq_state_do_not_use);
 458 };
 459
 460 size_t _dispatch_object_debug_attr(dispatch_object_t dou, char* buf,
 461                 size_t bufsiz);
 462 void *_dispatch_alloc(const void *vtable, size_t size);
 463 #if !USE_OBJC
 464 void _dispatch_xref_dispose(dispatch_object_t dou);
 465 #endif
 466 void _dispatch_dispose(dispatch_object_t dou);
 467 #if DISPATCH_COCOA_COMPAT
 468 #if USE_OBJC
 469 #include <objc/runtime.h>
 470 #include <objc/objc-internal.h>
 471 #define _dispatch_autorelease_pool_push() \
 472         objc_autoreleasePoolPush()
 473 #define _dispatch_autorelease_pool_pop(context) \
 474         objc_autoreleasePoolPop(context)
 475 #else
 476 void *_dispatch_autorelease_pool_push(void);
 477 void _dispatch_autorelease_pool_pop(void *context);
 478 #endif
 479 void *_dispatch_last_resort_autorelease_pool_push(void);
 480 void _dispatch_last_resort_autorelease_pool_pop(void *context);
 481
 482 #define dispatch_invoke_with_autoreleasepool(flags, ...)  ({ \
 483                 void *pool = NULL; \
 484                 if ((flags) & DISPATCH_INVOKE_AUTORELEASE_ALWAYS) { \
 485                         pool = _dispatch_autorelease_pool_push(); \
 486                         DISPATCH_COMPILER_CAN_ASSUME(pool); \
 487                 }; \
 488                 __VA_ARGS__; \
 489                 if (pool) _dispatch_autorelease_pool_pop(pool); \
 490         })
 491 #else
 492 #define dispatch_invoke_with_autoreleasepool(flags, ...) \
 493         do { (void)flags; __VA_ARGS__; } while (0)
 494 #endif
 495
 496
 497 #if USE_OBJC
 498 OS_OBJECT_OBJC_CLASS_DECL(object);
 499 #endif
 500
 501 #if OS_OBJECT_HAVE_OBJC2
 502 // ObjC toll-free bridging, keep in sync with libdispatch.order file
 503 //
 504 // This is required by the dispatch_data_t/NSData bridging, which is not
 505 // supported on the old runtime.
 506 #define DISPATCH_OBJECT_TFB(f, o, ...) \
 507         if (slowpath((uintptr_t)((o)._os_obj->os_obj_isa) & 1) || \
 508                         slowpath((Class)((o)._os_obj->os_obj_isa) < \
 509                                         (Class)OS_OBJECT_VTABLE(dispatch_object)) || \
 510                         slowpath((Class)((o)._os_obj->os_obj_isa) >= \
 511                                         (Class)OS_OBJECT_VTABLE(object))) { \
 512                 return f((o), ##__VA_ARGS__); \
 513         }
 514
 515 id _dispatch_objc_alloc(Class cls, size_t size);
 516 void _dispatch_objc_retain(dispatch_object_t dou);
 517 void _dispatch_objc_release(dispatch_object_t dou);
 518 void _dispatch_objc_set_context(dispatch_object_t dou, void *context);
 519 void *_dispatch_objc_get_context(dispatch_object_t dou);
 520 void _dispatch_objc_set_finalizer_f(dispatch_object_t dou,
 521                 dispatch_function_t finalizer);
 522 void _dispatch_objc_set_target_queue(dispatch_object_t dou,
 523                 dispatch_queue_t queue);
 524 void _dispatch_objc_suspend(dispatch_object_t dou);
 525 void _dispatch_objc_resume(dispatch_object_t dou);
 526 void _dispatch_objc_activate(dispatch_object_t dou);
 527 size_t _dispatch_objc_debug(dispatch_object_t dou, char* buf, size_t bufsiz);
 528
 529 #if __OBJC2__
 530 @interface NSObject (DISPATCH_CONCAT(_,DISPATCH_CLASS(object)))
 531 - (void)_setContext:(void*)context;
 532 - (void*)_getContext;
 533 - (void)_setFinalizer:(dispatch_function_t)finalizer;
 534 - (void)_setTargetQueue:(dispatch_queue_t)queue;
 535 - (void)_suspend;
 536 - (void)_resume;
 537 - (void)_activate;
 538 @end
 539 #endif // __OBJC2__
 540 #else
 541 #define DISPATCH_OBJECT_TFB(f, o, ...)
 542 #endif // OS_OBJECT_HAVE_OBJC2
 543
 544 #pragma mark -
 545 #pragma mark _os_object_s
 546
 547 /*
 548  * Low level _os_atomic_refcnt_* actions
 549  *
 550  * _os_atomic_refcnt_inc2o(o, f):
 551  *   performs a refcount increment and returns the new refcount value
 552  *
 553  * _os_atomic_refcnt_dec2o(o, f):
 554  *   performs a refcount decrement and returns the new refcount value
 555  *
 556  * _os_atomic_refcnt_dispose_barrier2o(o, f):
 557  *   a barrier to perform prior to tearing down an object when the refcount
 558  *   reached -1.
 559  */
 560 #define _os_atomic_refcnt_perform2o(o, f, op, m)   ({ \
 561                 typeof(o) _o = (o); \
 562                 int _ref_cnt = _o->f; \
 563                 if (fastpath(_ref_cnt != _OS_OBJECT_GLOBAL_REFCNT)) { \
 564                         _ref_cnt = os_atomic_##op##2o(_o, f, m); \
 565                 } \
 566                 _ref_cnt; \
 567         })
 568
 569 #define _os_atomic_refcnt_inc2o(o, m) \
 570                 _os_atomic_refcnt_perform2o(o, m, inc, relaxed)
 571
 572 #define _os_atomic_refcnt_dec2o(o, m) \
 573                 _os_atomic_refcnt_perform2o(o, m, dec, release)
 574
 575 #define _os_atomic_refcnt_dispose_barrier2o(o, m) \
 576                 (void)os_atomic_load2o(o, m, acquire)
 577
 578
 579 /*
 580  * Higher level _os_object_{x,}refcnt_* actions
 581  *
 582  * _os_atomic_{x,}refcnt_inc(o):
 583  *   increment the external (resp. internal) refcount and
 584  *   returns the new refcount value
 585  *
 586  * _os_atomic_{x,}refcnt_dec(o):
 587  *   decrement the external (resp. internal) refcount and
 588  *   returns the new refcount value
 589  *
 590  * _os_atomic_{x,}refcnt_dispose_barrier(o):
 591  *   performs the pre-teardown barrier for the external
 592  *   (resp. internal) refcount
 593  *
 594  */
 595 #define _os_object_xrefcnt_inc(o) \
 596                 _os_atomic_refcnt_inc2o(o, os_obj_xref_cnt)
 597
 598 #define _os_object_xrefcnt_dec(o) \
 599                 _os_atomic_refcnt_dec2o(o, os_obj_xref_cnt)
 600
 601 #define _os_object_xrefcnt_dispose_barrier(o) \
 602                 _os_atomic_refcnt_dispose_barrier2o(o, os_obj_xref_cnt)
 603
 604 #define _os_object_refcnt_inc(o) \
 605                 _os_atomic_refcnt_inc2o(o, os_obj_ref_cnt)
 606
 607 #define _os_object_refcnt_dec(o) \
 608                 _os_atomic_refcnt_dec2o(o, os_obj_ref_cnt)
 609
 610 #define _os_object_refcnt_dispose_barrier(o) \
 611                 _os_atomic_refcnt_dispose_barrier2o(o, os_obj_ref_cnt)
 612
 613 void _os_object_init(void);
 614 unsigned long _os_object_retain_count(_os_object_t obj);
 615 bool _os_object_retain_weak(_os_object_t obj);
 616 bool _os_object_allows_weak_reference(_os_object_t obj);
 617 void _os_object_dispose(_os_object_t obj);
 618 void _os_object_xref_dispose(_os_object_t obj);
 619
 620 #endif // __DISPATCH_OBJECT_INTERNAL__