src/object_internal.h

   1 /*
   2  * Copyright (c) 2008-2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_APACHE_LICENSE_HEADER_START@
   5  *
   6  * Licensed under the Apache License, Version 2.0 (the "License");
   7  * you may not use this file except in compliance with the License.
   8  * You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  *
  18  * @APPLE_APACHE_LICENSE_HEADER_END@
  19  */
  20
  21 /*
  22  * IMPORTANT: This header file describes INTERNAL interfaces to libdispatch
  23  * which are subject to change in future releases of Mac OS X. Any applications
  24  * relying on these interfaces WILL break.
  25  */
  26
  27 #ifndef __DISPATCH_OBJECT_INTERNAL__
  28 #define __DISPATCH_OBJECT_INTERNAL__
  29
  30 #if !OS_OBJECT_USE_OBJC
  31 #define OS_OBJECT_DECL(name)  DISPATCH_DECL(name)
  32 #define OS_OBJECT_DECL_SUBCLASS(name, super)  DISPATCH_DECL(name)
  33 #endif
  34
  35 #if USE_OBJC
  36 #define OS_OBJECT_EXTRA_VTABLE_SYMBOL(name) _OS_##name##_vtable
  37 #define DISPATCH_CLASS_SYMBOL(name) OS_dispatch_##name##_class
  38 #define DISPATCH_CLASS_RAW_SYMBOL_NAME(name) \
  39                 OS_OBJC_CLASS_RAW_SYMBOL_NAME(DISPATCH_CLASS(name))
  40 #else
  41 #define OS_OBJECT_CLASS_SYMBOL(name) _##name##_vtable
  42 #define OS_OBJC_CLASS_RAW_SYMBOL_NAME(name) \
  43                 "__" OS_STRINGIFY(name) "_vtable"
  44 #define DISPATCH_CLASS_SYMBOL(name) _dispatch_##name##_vtable
  45 #define DISPATCH_CLASS_RAW_SYMBOL_NAME(name) \
  46                 "__dispatch_" OS_STRINGIFY(name) "_vtable"
  47 #endif
  48
  49 #define DISPATCH_CLASS(name) OS_dispatch_##name
  50 #if USE_OBJC
  51 #define DISPATCH_OBJC_CLASS_DECL(name) \
  52                 extern void *DISPATCH_CLASS_SYMBOL(name) \
  53                                 asm(DISPATCH_CLASS_RAW_SYMBOL_NAME(name))
  54 #endif
  55
  56 // define a new proper class
  57 #define OS_OBJECT_CLASS_DECL(name, super, ...) \
  58                 struct name##_s; \
  59                 struct name##_extra_vtable_s { \
  60                         __VA_ARGS__; \
  61                 }; \
  62                 struct name##_vtable_s { \
  63                         _OS_OBJECT_CLASS_HEADER(); \
  64                         struct name##_extra_vtable_s _os_obj_vtable; \
  65                 }; \
  66                 OS_OBJECT_EXTRA_VTABLE_DECL(name, name) \
  67                 extern const struct name##_vtable_s OS_OBJECT_CLASS_SYMBOL(name) \
  68                                 asm(OS_OBJC_CLASS_RAW_SYMBOL_NAME(OS_OBJECT_CLASS(name)))
  69
  70 #if OS_OBJECT_SWIFT3
  71 #define OS_OBJECT_INTERNAL_CLASS_DECL(name, super, ...) \
  72                 OS_OBJECT_OBJC_RUNTIME_VISIBLE \
  73                 OS_OBJECT_DECL_IMPL_CLASS(name, OS_OBJECT_CLASS(super)); \
  74                 OS_OBJECT_CLASS_DECL(name, super, ## __VA_ARGS__)
  75 #elif OS_OBJECT_USE_OBJC
  76 #define OS_OBJECT_INTERNAL_CLASS_DECL(name, super, ...) \
  77                 OS_OBJECT_DECL(name); \
  78                 OS_OBJECT_CLASS_DECL(name, super, ## __VA_ARGS__)
  79 #else
  80 #define OS_OBJECT_INTERNAL_CLASS_DECL(name, super, ...) \
  81                 typedef struct name##_s *name##_t; \
  82                 OS_OBJECT_CLASS_DECL(name, super, ## __VA_ARGS__)
  83 #endif
  84
  85 #define DISPATCH_CLASS_DECL_BARE(name) \
  86                 OS_OBJECT_CLASS_DECL(dispatch_##name, dispatch_object, \
  87                                 DISPATCH_OBJECT_VTABLE_HEADER(dispatch_##name))
  88
  89 #define DISPATCH_CLASS_DECL(name) \
  90                 _OS_OBJECT_DECL_PROTOCOL(dispatch_##name, dispatch_object) \
  91                 _OS_OBJECT_CLASS_IMPLEMENTS_PROTOCOL(dispatch_##name, dispatch_##name) \
  92                 DISPATCH_CLASS_DECL_BARE(name)
  93
  94 #define DISPATCH_INTERNAL_CLASS_DECL(name) \
  95                 DISPATCH_DECL(dispatch_##name); \
  96                 DISPATCH_CLASS_DECL(name)
  97
  98 // define a new subclass used in a cluster
  99 #define OS_OBJECT_SUBCLASS_DECL(name, super) \
 100                 _OS_OBJECT_DECL_SUBCLASS_INTERFACE(name, super) \
 101                 struct name##_s; \
 102                 OS_OBJECT_EXTRA_VTABLE_DECL(name, super) \
 103                 extern const struct super##_vtable_s OS_OBJECT_CLASS_SYMBOL(name) \
 104                                 asm(OS_OBJC_CLASS_RAW_SYMBOL_NAME(OS_OBJECT_CLASS(name)))
 105
 106 #define DISPATCH_SUBCLASS_DECL(name, super) \
 107                 OS_OBJECT_SUBCLASS_DECL(dispatch_##name, super)
 108
 109 #if OS_OBJECT_SWIFT3
 110 // define a new internal subclass used in a class cluster
 111 #define OS_OBJECT_INTERNAL_SUBCLASS_DECL(name, super) \
 112                 _OS_OBJECT_DECL_PROTOCOL(name, super); \
 113                 OS_OBJECT_SUBCLASS_DECL(name, super)
 114
 115 #define DISPATCH_INTERNAL_SUBCLASS_DECL(name, super) \
 116                 _OS_OBJECT_DECL_PROTOCOL(dispatch_##name, dispatch_##super) \
 117                 DISPATCH_SUBCLASS_DECL(name, dispatch_##super)
 118 #else
 119 // define a new internal subclass used in a class cluster
 120 #define OS_OBJECT_INTERNAL_SUBCLASS_DECL(name, super) \
 121                 OS_OBJECT_DECL_SUBCLASS(name, super); \
 122                 OS_OBJECT_SUBCLASS_DECL(name, super)
 123
 124 #define DISPATCH_INTERNAL_SUBCLASS_DECL(name, super) \
 125                 OS_OBJECT_DECL_SUBCLASS(dispatch_##name, dispatch_##super); \
 126                 DISPATCH_SUBCLASS_DECL(name, dispatch_##super)
 127 #endif
 128
 129 // vtable symbols
 130 #define OS_OBJECT_VTABLE(name)          (&OS_OBJECT_CLASS_SYMBOL(name))
 131 #define DISPATCH_OBJC_CLASS(name)       (&DISPATCH_CLASS_SYMBOL(name))
 132
 133 // vtables for subclasses used in a class cluster
 134 #if USE_OBJC
 135 // ObjC classes and dispatch vtables are co-located via linker order and alias
 136 // files rdar://10640168
 137 #if OS_OBJECT_HAVE_OBJC2
 138 #define OS_OBJECT_VTABLE_SUBCLASS_INSTANCE(name, super, xdispose, dispose, ...) \
 139                 __attribute__((section("__DATA,__objc_data"), used)) \
 140                 const struct super##_extra_vtable_s \
 141                 OS_OBJECT_EXTRA_VTABLE_SYMBOL(name) = { __VA_ARGS__ }
 142 #define OS_OBJECT_EXTRA_VTABLE_DECL(name, super)
 143 #define DISPATCH_VTABLE(name) DISPATCH_OBJC_CLASS(name)
 144 #else
 145 #define OS_OBJECT_VTABLE_SUBCLASS_INSTANCE(name, super, xdispose, dispose, ...) \
 146                 const struct super##_vtable_s \
 147                 OS_OBJECT_EXTRA_VTABLE_SYMBOL(name) = { \
 148                         ._os_obj_objc_isa = &OS_OBJECT_CLASS_SYMBOL(name), \
 149                         ._os_obj_vtable = { __VA_ARGS__ }, \
 150                 }
 151 #define OS_OBJECT_EXTRA_VTABLE_DECL(name, super) \
 152                 extern const struct super##_vtable_s \
 153                                 OS_OBJECT_EXTRA_VTABLE_SYMBOL(name);
 154 #define DISPATCH_VTABLE(name) &OS_OBJECT_EXTRA_VTABLE_SYMBOL(dispatch_##name)
 155 #endif
 156 #else
 157 #define OS_OBJECT_VTABLE_SUBCLASS_INSTANCE(name, super, xdispose, dispose, ...) \
 158                 const struct super##_vtable_s OS_OBJECT_CLASS_SYMBOL(name) = { \
 159                         ._os_obj_xref_dispose = xdispose, \
 160                         ._os_obj_dispose = dispose, \
 161                         ._os_obj_vtable = { __VA_ARGS__ }, \
 162                 }
 163 #define OS_OBJECT_EXTRA_VTABLE_DECL(name, super)
 164 #define DISPATCH_VTABLE(name) DISPATCH_OBJC_CLASS(name)
 165 #endif // USE_OBJC
 166
 167 #define DISPATCH_VTABLE_SUBCLASS_INSTANCE(name, super, ...) \
 168                 OS_OBJECT_VTABLE_SUBCLASS_INSTANCE(dispatch_##name, dispatch_##super, \
 169                                 _dispatch_xref_dispose, _dispatch_dispose, __VA_ARGS__)
 170
 171 // vtables for proper classes
 172 #define OS_OBJECT_VTABLE_INSTANCE(name, xdispose, dispose, ...) \
 173                 OS_OBJECT_VTABLE_SUBCLASS_INSTANCE(name, name, \
 174                                 xdispose, dispose, __VA_ARGS__)
 175
 176 #define DISPATCH_VTABLE_INSTANCE(name, ...) \
 177                 DISPATCH_VTABLE_SUBCLASS_INSTANCE(name, name, __VA_ARGS__)
 178
 179 #define DISPATCH_INVOKABLE_VTABLE_HEADER(x) \
 180         unsigned long const do_type; \
 181         const char *const do_kind; \
 182         void (*const do_invoke)(struct x##_s *, dispatch_invoke_flags_t)
 183
 184 #define DISPATCH_QUEUEABLE_VTABLE_HEADER(x) \
 185         DISPATCH_INVOKABLE_VTABLE_HEADER(x); \
 186         void (*const do_wakeup)(struct x##_s *, \
 187                         pthread_priority_t, dispatch_wakeup_flags_t); \
 188         void (*const do_dispose)(struct x##_s *)
 189
 190 #define DISPATCH_OBJECT_VTABLE_HEADER(x) \
 191         DISPATCH_QUEUEABLE_VTABLE_HEADER(x); \
 192         void (*const do_set_targetq)(struct x##_s *, dispatch_queue_t); \
 193         void (*const do_suspend)(struct x##_s *); \
 194         void (*const do_resume)(struct x##_s *, bool activate); \
 195         void (*const do_finalize_activation)(struct x##_s *); \
 196         size_t (*const do_debug)(struct x##_s *, char *, size_t)
 197
 198 #define dx_vtable(x) (&(x)->do_vtable->_os_obj_vtable)
 199 #define dx_type(x) dx_vtable(x)->do_type
 200 #define dx_subtype(x) (dx_vtable(x)->do_type & _DISPATCH_SUB_TYPE_MASK)
 201 #define dx_metatype(x) (dx_vtable(x)->do_type & _DISPATCH_META_TYPE_MASK)
 202 #define dx_hastypeflag(x, f) (dx_vtable(x)->do_type & _DISPATCH_##f##_TYPEFLAG)
 203 #define dx_kind(x) dx_vtable(x)->do_kind
 204 #define dx_debug(x, y, z) dx_vtable(x)->do_debug((x), (y), (z))
 205 #define dx_dispose(x) dx_vtable(x)->do_dispose(x)
 206 #define dx_invoke(x, z) dx_vtable(x)->do_invoke(x, z)
 207 #define dx_wakeup(x, y, z) dx_vtable(x)->do_wakeup(x, y, z)
 208
 209 #define DISPATCH_OBJECT_GLOBAL_REFCNT           _OS_OBJECT_GLOBAL_REFCNT
 210
 211 #if OS_OBJECT_HAVE_OBJC1
 212 #define DISPATCH_GLOBAL_OBJECT_HEADER(name) \
 213         .do_vtable = DISPATCH_VTABLE(name), \
 214         ._objc_isa = DISPATCH_OBJC_CLASS(name), \
 215         .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, \
 216         .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT
 217 #else
 218 #define DISPATCH_GLOBAL_OBJECT_HEADER(name) \
 219         .do_vtable = DISPATCH_VTABLE(name), \
 220         .do_ref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT, \
 221         .do_xref_cnt = DISPATCH_OBJECT_GLOBAL_REFCNT
 222 #endif
 223
 224 #ifdef __LP64__
 225 // the bottom nibble must not be zero, the rest of the bits should be random
 226 // we sign extend the 64-bit version so that a better instruction encoding is
 227 // generated on Intel
 228 #define DISPATCH_OBJECT_LISTLESS ((void *)0xffffffff89abcdef)
 229 #else
 230 #define DISPATCH_OBJECT_LISTLESS ((void *)0x89abcdef)
 231 #endif
 232
 233 DISPATCH_ENUM(dispatch_wakeup_flags, uint32_t,
 234         // The caller of dx_wakeup owns an internal refcount on the object being
 235         // woken up
 236         DISPATCH_WAKEUP_CONSUME                 = 0x00000001,
 237
 238         // Some change to the object needs to be published to drainers.
 239         // If the drainer isn't the same thread, some scheme such as the dispatch
 240         // queue DIRTY bit must be used and a release barrier likely has to be
 241         // involved before dx_wakeup returns
 242         DISPATCH_WAKEUP_FLUSH                                   = 0x00000002,
 243
 244         // A slow waiter was just enqueued
 245         DISPATCH_WAKEUP_SLOW_WAITER                             = 0x00000004,
 246
 247         // The caller desires to apply an override on the object being woken up
 248         // and has already adjusted the `oq_override` field. When this flag is
 249         // passed, the priority passed to dx_wakeup() should not be 0
 250         DISPATCH_WAKEUP_OVERRIDING              = 0x00000008,
 251
 252         // At the time this queue was woken up it had an override that must be
 253         // preserved (used to solve a race with _dispatch_queue_drain_try_unlock())
 254         DISPATCH_WAKEUP_WAS_OVERRIDDEN          = 0x00000010,
 255
 256         // This wakeup is caused by a handoff from a slow waiter.
 257         DISPATCH_WAKEUP_WAITER_HANDOFF          = 0x00000020,
 258
 259 #define _DISPATCH_WAKEUP_OVERRIDE_BITS \
 260                 ((dispatch_wakeup_flags_t)(DISPATCH_WAKEUP_OVERRIDING | \
 261                 DISPATCH_WAKEUP_WAS_OVERRIDDEN))
 262 );
 263
 264 DISPATCH_ENUM(dispatch_invoke_flags, uint32_t,
 265         DISPATCH_INVOKE_NONE                                    = 0x00000000,
 266
 267         // Invoke modes
 268         //
 269         // @const DISPATCH_INVOKE_STEALING
 270         // This invoke is a stealer, meaning that it doesn't own the
 271         // enqueue lock at drain lock time.
 272         //
 273         // @const DISPATCH_INVOKE_OVERRIDING
 274         // This invoke is draining the hierarchy on another root queue and needs
 275         // to fake the identity of the original one.
 276         //
 277         DISPATCH_INVOKE_STEALING                                = 0x00000001,
 278         DISPATCH_INVOKE_OVERRIDING                              = 0x00000002,
 279
 280         // Below this point flags are propagated to recursive calls to drain(),
 281         // continuation pop() or dx_invoke().
 282 #define _DISPATCH_INVOKE_PROPAGATE_MASK           0xffff0000u
 283
 284         // Drain modes
 285         //
 286         // @const DISPATCH_INVOKE_WORKER_DRAIN
 287         // Invoke has been issued by a worker thread (work queue thread, or
 288         // pthread root queue) drain. This flag is NOT set when the main queue,
 289         // manager queue or runloop queues are drained
 290         //
 291         // @const DISPATCH_INVOKE_REDIRECTING_DRAIN
 292         // Has only been draining concurrent queues so far
 293         // Implies DISPATCH_INVOKE_WORKER_DRAIN
 294         //
 295         // @const DISPATCH_INVOKE_MANAGER_DRAIN
 296         // We're draining from a manager context
 297         //
 298         DISPATCH_INVOKE_WORKER_DRAIN                    = 0x00010000,
 299         DISPATCH_INVOKE_REDIRECTING_DRAIN               = 0x00020000,
 300         DISPATCH_INVOKE_MANAGER_DRAIN                   = 0x00040000,
 301 #define _DISPATCH_INVOKE_DRAIN_MODE_MASK          0x000f0000u
 302
 303         // Autoreleasing modes
 304         //
 305         // @const DISPATCH_INVOKE_AUTORELEASE_ALWAYS
 306         // Always use autoreleasepools around callouts
 307         //
 308         // @const DISPATCH_INVOKE_AUTORELEASE_NEVER
 309         // Never use autoreleasepools around callouts
 310         //
 311         DISPATCH_INVOKE_AUTORELEASE_ALWAYS              = 0x00100000,
 312         DISPATCH_INVOKE_AUTORELEASE_NEVER               = 0x00200000,
 313 #define _DISPATCH_INVOKE_AUTORELEASE_MASK         0x00300000u
 314 );
 315
 316 enum {
 317         _DISPATCH_META_TYPE_MASK                = 0xffff0000, // mask for object meta-types
 318         _DISPATCH_TYPEFLAGS_MASK                = 0x0000ff00, // mask for object typeflags
 319         _DISPATCH_SUB_TYPE_MASK                 = 0x000000ff, // mask for object sub-types
 320
 321         _DISPATCH_CONTINUATION_TYPE             =    0x00000, // meta-type for continuations
 322         _DISPATCH_QUEUE_TYPE                    =    0x10000, // meta-type for queues
 323         _DISPATCH_SOURCE_TYPE                   =    0x20000, // meta-type for sources
 324         _DISPATCH_SEMAPHORE_TYPE                =    0x30000, // meta-type for semaphores
 325         _DISPATCH_NODE_TYPE                             =    0x40000, // meta-type for data node
 326         _DISPATCH_IO_TYPE                               =    0x50000, // meta-type for io channels
 327         _DISPATCH_OPERATION_TYPE                =    0x60000, // meta-type for io operations
 328         _DISPATCH_DISK_TYPE                             =    0x70000, // meta-type for io disks
 329
 330         _DISPATCH_QUEUE_ROOT_TYPEFLAG   =     0x0100, // bit set for any root queues
 331
 332 #define DISPATCH_CONTINUATION_TYPE(name)  \
 333                 (_DISPATCH_CONTINUATION_TYPE | DC_##name##_TYPE)
 334         DISPATCH_DATA_TYPE                              = 1 | _DISPATCH_NODE_TYPE,
 335         DISPATCH_MACH_MSG_TYPE                  = 2 | _DISPATCH_NODE_TYPE,
 336         DISPATCH_QUEUE_ATTR_TYPE                = 3 | _DISPATCH_NODE_TYPE,
 337
 338         DISPATCH_IO_TYPE                                = 0 | _DISPATCH_IO_TYPE,
 339         DISPATCH_OPERATION_TYPE                 = 0 | _DISPATCH_OPERATION_TYPE,
 340         DISPATCH_DISK_TYPE                              = 0 | _DISPATCH_DISK_TYPE,
 341
 342         DISPATCH_QUEUE_LEGACY_TYPE              = 1 | _DISPATCH_QUEUE_TYPE,
 343         DISPATCH_QUEUE_SERIAL_TYPE              = 2 | _DISPATCH_QUEUE_TYPE,
 344         DISPATCH_QUEUE_CONCURRENT_TYPE  = 3 | _DISPATCH_QUEUE_TYPE,
 345         DISPATCH_QUEUE_GLOBAL_ROOT_TYPE = 4 | _DISPATCH_QUEUE_TYPE |
 346                         _DISPATCH_QUEUE_ROOT_TYPEFLAG,
 347         DISPATCH_QUEUE_RUNLOOP_TYPE             = 5 | _DISPATCH_QUEUE_TYPE |
 348                         _DISPATCH_QUEUE_ROOT_TYPEFLAG,
 349         DISPATCH_QUEUE_MGR_TYPE                 = 6 | _DISPATCH_QUEUE_TYPE,
 350         DISPATCH_QUEUE_SPECIFIC_TYPE    = 7 | _DISPATCH_QUEUE_TYPE,
 351
 352         DISPATCH_SEMAPHORE_TYPE                 = 1 | _DISPATCH_SEMAPHORE_TYPE,
 353         DISPATCH_GROUP_TYPE                             = 2 | _DISPATCH_SEMAPHORE_TYPE,
 354
 355         DISPATCH_SOURCE_KEVENT_TYPE             = 1 | _DISPATCH_SOURCE_TYPE,
 356         DISPATCH_MACH_CHANNEL_TYPE              = 2 | _DISPATCH_SOURCE_TYPE,
 357
 358 };
 359
 360 typedef struct _os_object_vtable_s {
 361         _OS_OBJECT_CLASS_HEADER();
 362 } _os_object_vtable_s;
 363
 364 typedef struct _os_object_s {
 365         _OS_OBJECT_HEADER(
 366         const _os_object_vtable_s *os_obj_isa,
 367         os_obj_ref_cnt,
 368         os_obj_xref_cnt);
 369 } _os_object_s;
 370
 371 #if OS_OBJECT_HAVE_OBJC1
 372 #define OS_OBJECT_STRUCT_HEADER(x) \
 373         _OS_OBJECT_HEADER(\
 374         const void *_objc_isa, \
 375         do_ref_cnt, \
 376         do_xref_cnt); \
 377         const struct x##_vtable_s *do_vtable
 378 #else
 379 #define OS_OBJECT_STRUCT_HEADER(x) \
 380         _OS_OBJECT_HEADER(\
 381         const struct x##_vtable_s *do_vtable, \
 382         do_ref_cnt, \
 383         do_xref_cnt)
 384 #endif
 385
 386 #define _DISPATCH_OBJECT_HEADER(x) \
 387         struct _os_object_s _as_os_obj[0]; \
 388         OS_OBJECT_STRUCT_HEADER(dispatch_##x); \
 389         struct dispatch_##x##_s *volatile do_next; \
 390         struct dispatch_queue_s *do_targetq; \
 391         void *do_ctxt; \
 392         void *do_finalizer
 393
 394 #define DISPATCH_OBJECT_HEADER(x) \
 395         struct dispatch_object_s _as_do[0]; \
 396         _DISPATCH_OBJECT_HEADER(x)
 397
 398 // Swift-unavailable -init requires method in each class.
 399 #define DISPATCH_UNAVAILABLE_INIT() \
 400         - (instancetype)init { \
 401                 DISPATCH_CLIENT_CRASH(0, "-init called directly"); \
 402                 return [super init]; \
 403         }
 404
 405 _OS_OBJECT_DECL_PROTOCOL(dispatch_object, object);
 406
 407 OS_OBJECT_CLASS_DECL(dispatch_object, object,
 408                 DISPATCH_OBJECT_VTABLE_HEADER(dispatch_object));
 409
 410 struct dispatch_object_s {
 411         _DISPATCH_OBJECT_HEADER(object);
 412 };
 413
 414 #if OS_OBJECT_HAVE_OBJC1
 415 #define _OS_MPSC_QUEUE_FIELDS(ns, __state_field__) \
 416         struct dispatch_object_s *volatile ns##_items_head; \
 417         unsigned long ns##_serialnum; \
 418         union { \
 419                 uint64_t volatile __state_field__; \
 420                 DISPATCH_STRUCT_LITTLE_ENDIAN_2( \
 421                         dispatch_lock __state_field__##_lock, \
 422                         uint32_t __state_field__##_bits \
 423                 ); \
 424         }; /* needs to be 64-bit aligned */ \
 425         /* LP64 global queue cacheline boundary */ \
 426         const char *ns##_label; \
 427         voucher_t ns##_override_voucher; \
 428         dispatch_priority_t ns##_priority; \
 429         dispatch_priority_t volatile ns##_override; \
 430         struct dispatch_object_s *volatile ns##_items_tail
 431 #else
 432 #define _OS_MPSC_QUEUE_FIELDS(ns, __state_field__) \
 433         struct dispatch_object_s *volatile ns##_items_head; \
 434         union { \
 435                 uint64_t volatile __state_field__; \
 436                 DISPATCH_STRUCT_LITTLE_ENDIAN_2( \
 437                         dispatch_lock __state_field__##_lock, \
 438                         uint32_t __state_field__##_bits \
 439                 ); \
 440         }; /* needs to be 64-bit aligned */ \
 441         /* LP64 global queue cacheline boundary */ \
 442         unsigned long ns##_serialnum; \
 443         const char *ns##_label; \
 444         voucher_t ns##_override_voucher; \
 445         dispatch_priority_t ns##_priority; \
 446         dispatch_priority_t volatile ns##_override; \
 447         struct dispatch_object_s *volatile ns##_items_tail
 448 #endif
 449
 450 OS_OBJECT_INTERNAL_CLASS_DECL(os_mpsc_queue, object,
 451                 DISPATCH_QUEUEABLE_VTABLE_HEADER(os_mpsc_queue));
 452
 453 struct os_mpsc_queue_s {
 454         struct _os_object_s _as_os_obj[0];
 455         OS_OBJECT_STRUCT_HEADER(os_mpsc_queue);
 456         struct dispatch_object_s *volatile oq_next;
 457         void *oq_opaque1; // do_targetq
 458         void *oq_opaque2; // do_ctxt
 459         void *oq_opaque3; // do_finalizer
 460         _OS_MPSC_QUEUE_FIELDS(oq, __oq_state_do_not_use);
 461 };
 462
 463 size_t _dispatch_object_debug_attr(dispatch_object_t dou, char* buf,
 464                 size_t bufsiz);
 465 void *_dispatch_alloc(const void *vtable, size_t size);
 466 #if !USE_OBJC
 467 void _dispatch_xref_dispose(dispatch_object_t dou);
 468 #endif
 469 void _dispatch_dispose(dispatch_object_t dou);
 470 #if DISPATCH_COCOA_COMPAT
 471 #if USE_OBJC
 472 #include <objc/runtime.h>
 473 #include <objc/objc-internal.h>
 474 #define _dispatch_autorelease_pool_push() \
 475         objc_autoreleasePoolPush()
 476 #define _dispatch_autorelease_pool_pop(context) \
 477         objc_autoreleasePoolPop(context)
 478 #else
 479 void *_dispatch_autorelease_pool_push(void);
 480 void _dispatch_autorelease_pool_pop(void *context);
 481 #endif
 482 void *_dispatch_last_resort_autorelease_pool_push(void);
 483 void _dispatch_last_resort_autorelease_pool_pop(void *context);
 484
 485 #define dispatch_invoke_with_autoreleasepool(flags, ...)  ({ \
 486                 void *pool = NULL; \
 487                 if ((flags) & DISPATCH_INVOKE_AUTORELEASE_ALWAYS) { \
 488                         pool = _dispatch_autorelease_pool_push(); \
 489                         DISPATCH_COMPILER_CAN_ASSUME(pool); \
 490                 }; \
 491                 __VA_ARGS__; \
 492                 if (pool) _dispatch_autorelease_pool_pop(pool); \
 493         })
 494 #else
 495 #define dispatch_invoke_with_autoreleasepool(flags, ...) \
 496         do { (void)flags; __VA_ARGS__; } while (0)
 497 #endif
 498
 499
 500 #if USE_OBJC
 501 OS_OBJECT_OBJC_CLASS_DECL(object);
 502 #endif
 503
 504 #if OS_OBJECT_HAVE_OBJC2
 505 // ObjC toll-free bridging, keep in sync with libdispatch.order file
 506 //
 507 // This is required by the dispatch_data_t/NSData bridging, which is not
 508 // supported on the old runtime.
 509 #define DISPATCH_OBJECT_TFB(f, o, ...) \
 510         if (slowpath((uintptr_t)((o)._os_obj->os_obj_isa) & 1) || \
 511                         slowpath((Class)((o)._os_obj->os_obj_isa) < \
 512                                         (Class)OS_OBJECT_VTABLE(dispatch_object)) || \
 513                         slowpath((Class)((o)._os_obj->os_obj_isa) >= \
 514                                         (Class)OS_OBJECT_VTABLE(object))) { \
 515                 return f((o), ##__VA_ARGS__); \
 516         }
 517
 518 id _dispatch_objc_alloc(Class cls, size_t size);
 519 void _dispatch_objc_retain(dispatch_object_t dou);
 520 void _dispatch_objc_release(dispatch_object_t dou);
 521 void _dispatch_objc_set_context(dispatch_object_t dou, void *context);
 522 void *_dispatch_objc_get_context(dispatch_object_t dou);
 523 void _dispatch_objc_set_finalizer_f(dispatch_object_t dou,
 524                 dispatch_function_t finalizer);
 525 void _dispatch_objc_set_target_queue(dispatch_object_t dou,
 526                 dispatch_queue_t queue);
 527 void _dispatch_objc_suspend(dispatch_object_t dou);
 528 void _dispatch_objc_resume(dispatch_object_t dou);
 529 void _dispatch_objc_activate(dispatch_object_t dou);
 530 size_t _dispatch_objc_debug(dispatch_object_t dou, char* buf, size_t bufsiz);
 531
 532 #if __OBJC2__
 533 @interface NSObject (DISPATCH_CONCAT(_,DISPATCH_CLASS(object)))
 534 - (void)_setContext:(void*)context;
 535 - (void*)_getContext;
 536 - (void)_setFinalizer:(dispatch_function_t)finalizer;
 537 - (void)_setTargetQueue:(dispatch_queue_t)queue;
 538 - (void)_suspend;
 539 - (void)_resume;
 540 - (void)_activate;
 541 @end
 542 #endif // __OBJC2__
 543 #else
 544 #define DISPATCH_OBJECT_TFB(f, o, ...)
 545 #endif // OS_OBJECT_HAVE_OBJC2
 546
 547 #pragma mark -
 548 #pragma mark _os_object_s
 549
 550 /*
 551  * Low level _os_atomic_refcnt_* actions
 552  *
 553  * _os_atomic_refcnt_inc2o(o, f):
 554  *   performs a refcount increment and returns the new refcount value
 555  *
 556  * _os_atomic_refcnt_dec2o(o, f):
 557  *   performs a refcount decrement and returns the new refcount value
 558  *
 559  * _os_atomic_refcnt_dispose_barrier2o(o, f):
 560  *   a barrier to perform prior to tearing down an object when the refcount
 561  *   reached -1.
 562  */
 563 #define _os_atomic_refcnt_perform2o(o, f, op, m)   ({ \
 564                 typeof(o) _o = (o); \
 565                 int _ref_cnt = _o->f; \
 566                 if (fastpath(_ref_cnt != _OS_OBJECT_GLOBAL_REFCNT)) { \
 567                         _ref_cnt = os_atomic_##op##2o(_o, f, m); \
 568                 } \
 569                 _ref_cnt; \
 570         })
 571
 572 #define _os_atomic_refcnt_inc2o(o, m) \
 573                 _os_atomic_refcnt_perform2o(o, m, inc, relaxed)
 574
 575 #define _os_atomic_refcnt_dec2o(o, m) \
 576                 _os_atomic_refcnt_perform2o(o, m, dec, release)
 577
 578 #define _os_atomic_refcnt_dispose_barrier2o(o, m) \
 579                 (void)os_atomic_load2o(o, m, acquire)
 580
 581
 582 /*
 583  * Higher level _os_object_{x,}refcnt_* actions
 584  *
 585  * _os_atomic_{x,}refcnt_inc(o):
 586  *   increment the external (resp. internal) refcount and
 587  *   returns the new refcount value
 588  *
 589  * _os_atomic_{x,}refcnt_dec(o):
 590  *   decrement the external (resp. internal) refcount and
 591  *   returns the new refcount value
 592  *
 593  * _os_atomic_{x,}refcnt_dispose_barrier(o):
 594  *   performs the pre-teardown barrier for the external
 595  *   (resp. internal) refcount
 596  *
 597  */
 598 #define _os_object_xrefcnt_inc(o) \
 599                 _os_atomic_refcnt_inc2o(o, os_obj_xref_cnt)
 600
 601 #define _os_object_xrefcnt_dec(o) \
 602                 _os_atomic_refcnt_dec2o(o, os_obj_xref_cnt)
 603
 604 #define _os_object_xrefcnt_dispose_barrier(o) \
 605                 _os_atomic_refcnt_dispose_barrier2o(o, os_obj_xref_cnt)
 606
 607 #define _os_object_refcnt_inc(o) \
 608                 _os_atomic_refcnt_inc2o(o, os_obj_ref_cnt)
 609
 610 #define _os_object_refcnt_dec(o) \
 611                 _os_atomic_refcnt_dec2o(o, os_obj_ref_cnt)
 612
 613 #define _os_object_refcnt_dispose_barrier(o) \
 614                 _os_atomic_refcnt_dispose_barrier2o(o, os_obj_ref_cnt)
 615
 616 void _os_object_atfork_child(void);
 617 void _os_object_atfork_parent(void);
 618 void _os_object_atfork_prepare(void);
 619 void _os_object_init(void);
 620 unsigned long _os_object_retain_count(_os_object_t obj);
 621 bool _os_object_retain_weak(_os_object_t obj);
 622 bool _os_object_allows_weak_reference(_os_object_t obj);
 623 void _os_object_dispose(_os_object_t obj);
 624 void _os_object_xref_dispose(_os_object_t obj);
 625
 626 #endif // __DISPATCH_OBJECT_INTERNAL__