src/queue_internal.h

   1 /*
   2  * Copyright (c) 2008-2011 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_APACHE_LICENSE_HEADER_START@
   5  *
   6  * Licensed under the Apache License, Version 2.0 (the "License");
   7  * you may not use this file except in compliance with the License.
   8  * You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  *
  18  * @APPLE_APACHE_LICENSE_HEADER_END@
  19  */
  20
  21 /*
  22  * IMPORTANT: This header file describes INTERNAL interfaces to libdispatch
  23  * which are subject to change in future releases of Mac OS X. Any applications
  24  * relying on these interfaces WILL break.
  25  */
  26
  27 #ifndef __DISPATCH_QUEUE_INTERNAL__
  28 #define __DISPATCH_QUEUE_INTERNAL__
  29
  30 #ifndef __DISPATCH_INDIRECT__
  31 #error "Please #include <dispatch/dispatch.h> instead of this file directly."
  32 #include <dispatch/base.h> // for HeaderDoc
  33 #endif
  34
  35 // If dc_vtable is less than 127, then the object is a continuation.
  36 // Otherwise, the object has a private layout and memory management rules. The
  37 // first two words must align with normal objects.
  38 #define DISPATCH_CONTINUATION_HEADER(x) \
  39         const void *do_vtable; \
  40         struct x *volatile do_next; \
  41         dispatch_function_t dc_func; \
  42         void *dc_ctxt
  43
  44 #define DISPATCH_OBJ_ASYNC_BIT          0x1
  45 #define DISPATCH_OBJ_BARRIER_BIT        0x2
  46 #define DISPATCH_OBJ_GROUP_BIT          0x4
  47 #define DISPATCH_OBJ_SYNC_SLOW_BIT      0x8
  48 // vtables are pointers far away from the low page in memory
  49 #define DISPATCH_OBJ_IS_VTABLE(x) ((unsigned long)(x)->do_vtable > 127ul)
  50
  51 struct dispatch_continuation_s {
  52         DISPATCH_CONTINUATION_HEADER(dispatch_continuation_s);
  53         dispatch_group_t dc_group;
  54         void *dc_data[3];
  55 };
  56
  57 typedef struct dispatch_continuation_s *dispatch_continuation_t;
  58
  59 struct dispatch_queue_attr_vtable_s {
  60         DISPATCH_VTABLE_HEADER(dispatch_queue_attr_s);
  61 };
  62
  63 struct dispatch_queue_attr_s {
  64         DISPATCH_STRUCT_HEADER(dispatch_queue_attr_s, dispatch_queue_attr_vtable_s);
  65 };
  66
  67 struct dispatch_queue_vtable_s {
  68         DISPATCH_VTABLE_HEADER(dispatch_queue_s);
  69 };
  70
  71 #define DISPATCH_QUEUE_MIN_LABEL_SIZE 64
  72
  73 #ifdef __LP64__
  74 #define DISPATCH_QUEUE_CACHELINE_PAD 32
  75 #else
  76 #define DISPATCH_QUEUE_CACHELINE_PAD 8
  77 #endif
  78
  79 #define DISPATCH_QUEUE_HEADER \
  80         uint32_t volatile dq_running; \
  81         uint32_t dq_width; \
  82         struct dispatch_object_s *volatile dq_items_tail; \
  83         struct dispatch_object_s *volatile dq_items_head; \
  84         unsigned long dq_serialnum; \
  85         dispatch_queue_t dq_specific_q;
  86
  87 struct dispatch_queue_s {
  88         DISPATCH_STRUCT_HEADER(dispatch_queue_s, dispatch_queue_vtable_s);
  89         DISPATCH_QUEUE_HEADER;
  90         char dq_label[DISPATCH_QUEUE_MIN_LABEL_SIZE]; // must be last
  91         char _dq_pad[DISPATCH_QUEUE_CACHELINE_PAD]; // for static queues only
  92 };
  93
  94 extern struct dispatch_queue_s _dispatch_mgr_q;
  95
  96 void _dispatch_queue_dispose(dispatch_queue_t dq);
  97 void _dispatch_queue_invoke(dispatch_queue_t dq);
  98 void _dispatch_queue_push_list_slow(dispatch_queue_t dq,
  99                 struct dispatch_object_s *obj);
 100
 101 DISPATCH_ALWAYS_INLINE
 102 static inline void
 103 _dispatch_queue_push_list(dispatch_queue_t dq, dispatch_object_t _head,
 104                 dispatch_object_t _tail)
 105 {
 106         struct dispatch_object_s *prev, *head = _head._do, *tail = _tail._do;
 107
 108         tail->do_next = NULL;
 109         dispatch_atomic_store_barrier();
 110         prev = fastpath(dispatch_atomic_xchg2o(dq, dq_items_tail, tail));
 111         if (prev) {
 112                 // if we crash here with a value less than 0x1000, then we are at a
 113                 // known bug in client code for example, see _dispatch_queue_dispose
 114                 // or _dispatch_atfork_child
 115                 prev->do_next = head;
 116         } else {
 117                 _dispatch_queue_push_list_slow(dq, head);
 118         }
 119 }
 120
 121 #define _dispatch_queue_push(x, y) _dispatch_queue_push_list((x), (y), (y))
 122
 123 #if DISPATCH_DEBUG
 124 void dispatch_debug_queue(dispatch_queue_t dq, const char* str);
 125 #else
 126 static inline void dispatch_debug_queue(dispatch_queue_t dq DISPATCH_UNUSED,
 127                 const char* str DISPATCH_UNUSED) {}
 128 #endif
 129
 130 size_t dispatch_queue_debug(dispatch_queue_t dq, char* buf, size_t bufsiz);
 131 size_t _dispatch_queue_debug_attr(dispatch_queue_t dq, char* buf,
 132                 size_t bufsiz);
 133
 134 DISPATCH_ALWAYS_INLINE
 135 static inline dispatch_queue_t
 136 _dispatch_queue_get_current(void)
 137 {
 138         return _dispatch_thread_getspecific(dispatch_queue_key);
 139 }
 140
 141 #define DISPATCH_QUEUE_PRIORITY_COUNT 4
 142 #define DISPATCH_ROOT_QUEUE_COUNT (DISPATCH_QUEUE_PRIORITY_COUNT * 2)
 143
 144 // overcommit priority index values need bit 1 set
 145 enum {
 146         DISPATCH_ROOT_QUEUE_IDX_LOW_PRIORITY = 0,
 147         DISPATCH_ROOT_QUEUE_IDX_LOW_OVERCOMMIT_PRIORITY,
 148         DISPATCH_ROOT_QUEUE_IDX_DEFAULT_PRIORITY,
 149         DISPATCH_ROOT_QUEUE_IDX_DEFAULT_OVERCOMMIT_PRIORITY,
 150         DISPATCH_ROOT_QUEUE_IDX_HIGH_PRIORITY,
 151         DISPATCH_ROOT_QUEUE_IDX_HIGH_OVERCOMMIT_PRIORITY,
 152         DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_PRIORITY,
 153         DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_OVERCOMMIT_PRIORITY,
 154 };
 155
 156 extern const struct dispatch_queue_attr_vtable_s dispatch_queue_attr_vtable;
 157 extern const struct dispatch_queue_vtable_s _dispatch_queue_vtable;
 158 extern unsigned long _dispatch_queue_serial_numbers;
 159 extern struct dispatch_queue_s _dispatch_root_queues[];
 160
 161 DISPATCH_ALWAYS_INLINE DISPATCH_CONST
 162 static inline dispatch_queue_t
 163 _dispatch_get_root_queue(long priority, bool overcommit)
 164 {
 165         if (overcommit) switch (priority) {
 166         case DISPATCH_QUEUE_PRIORITY_LOW:
 167                 return &_dispatch_root_queues[
 168                                 DISPATCH_ROOT_QUEUE_IDX_LOW_OVERCOMMIT_PRIORITY];
 169         case DISPATCH_QUEUE_PRIORITY_DEFAULT:
 170                 return &_dispatch_root_queues[
 171                                 DISPATCH_ROOT_QUEUE_IDX_DEFAULT_OVERCOMMIT_PRIORITY];
 172         case DISPATCH_QUEUE_PRIORITY_HIGH:
 173                 return &_dispatch_root_queues[
 174                                 DISPATCH_ROOT_QUEUE_IDX_HIGH_OVERCOMMIT_PRIORITY];
 175         case DISPATCH_QUEUE_PRIORITY_BACKGROUND:
 176                 return &_dispatch_root_queues[
 177                                 DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_OVERCOMMIT_PRIORITY];
 178         }
 179         switch (priority) {
 180         case DISPATCH_QUEUE_PRIORITY_LOW:
 181                 return &_dispatch_root_queues[DISPATCH_ROOT_QUEUE_IDX_LOW_PRIORITY];
 182         case DISPATCH_QUEUE_PRIORITY_DEFAULT:
 183                 return &_dispatch_root_queues[DISPATCH_ROOT_QUEUE_IDX_DEFAULT_PRIORITY];
 184         case DISPATCH_QUEUE_PRIORITY_HIGH:
 185                 return &_dispatch_root_queues[DISPATCH_ROOT_QUEUE_IDX_HIGH_PRIORITY];
 186         case DISPATCH_QUEUE_PRIORITY_BACKGROUND:
 187                 return &_dispatch_root_queues[
 188                                 DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_PRIORITY];
 189         default:
 190                 return NULL;
 191         }
 192 }
 193
 194 // Note to later developers: ensure that any initialization changes are
 195 // made for statically allocated queues (i.e. _dispatch_main_q).
 196 static inline void
 197 _dispatch_queue_init(dispatch_queue_t dq)
 198 {
 199         dq->do_vtable = &_dispatch_queue_vtable;
 200         dq->do_next = DISPATCH_OBJECT_LISTLESS;
 201         dq->do_ref_cnt = 1;
 202         dq->do_xref_cnt = 1;
 203         // Default target queue is overcommit!
 204         dq->do_targetq = _dispatch_get_root_queue(0, true);
 205         dq->dq_running = 0;
 206         dq->dq_width = 1;
 207         dq->dq_serialnum = dispatch_atomic_inc(&_dispatch_queue_serial_numbers) - 1;
 208 }
 209
 210 #endif