kern/workqueue_internal.h

   1 /*
   2  * Copyright (c) 2014 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #ifndef _WORKQUEUE_INTERNAL_H_
  30 #define _WORKQUEUE_INTERNAL_H_
  31
  32 /* These definitions are shared between the kext and userspace inside the pthread project. Consolidating
  33  * duplicate definitions that used to exist in both projects, when separate.
  34  */
  35
  36 /* workq_kernreturn commands */
  37 #define WQOPS_THREAD_RETURN        0x04 /* parks the thread back into the kernel */
  38 #define WQOPS_QUEUE_NEWSPISUPP     0x10 /* this is to check for newer SPI support */
  39 #define WQOPS_QUEUE_REQTHREADS     0x20 /* request number of threads of a prio */
  40 #define WQOPS_QUEUE_REQTHREADS2    0x30 /* request a number of threads in a given priority bucket */
  41 #define WQOPS_THREAD_KEVENT_RETURN 0x40 /* parks the thread after delivering the passed kevent array */
  42 #define WQOPS_SET_EVENT_MANAGER_PRIORITY 0x80   /* max() in the provided priority in the the priority of the event manager */
  43
  44 /* flag values for reuse field in the libc side _pthread_wqthread */
  45 #define WQ_FLAG_THREAD_PRIOMASK         0x0000ffff
  46 #define WQ_FLAG_THREAD_PRIOSHIFT        (8ull)
  47 #define WQ_FLAG_THREAD_OVERCOMMIT       0x00010000      /* thread is with overcommit prio */
  48 #define WQ_FLAG_THREAD_REUSE            0x00020000      /* thread is being reused */
  49 #define WQ_FLAG_THREAD_NEWSPI           0x00040000      /* the call is with new SPIs */
  50 #define WQ_FLAG_THREAD_KEVENT       0x00080000  /* thread is response to kevent req */
  51 #define WQ_FLAG_THREAD_EVENT_MANAGER    0x00100000  /* event manager thread */
  52
  53 /* These definitions are only available to the kext, to avoid bleeding constants and types across the boundary to
  54  * the userspace library.
  55  */
  56 #ifdef KERNEL
  57
  58 /* These defines come from kern/thread.h but are XNU_KERNEL_PRIVATE so do not get
  59  * exported to kernel extensions.
  60  */
  61 #define SCHED_CALL_BLOCK 0x1
  62 #define SCHED_CALL_UNBLOCK 0x2
  63
  64 // kwe_state
  65 enum {
  66         KWE_THREAD_INWAIT = 1,
  67         KWE_THREAD_PREPOST,
  68         KWE_THREAD_BROADCAST,
  69 };
  70
  71 /* old workq priority scheme */
  72
  73 #define WORKQUEUE_HIGH_PRIOQUEUE    0       /* high priority queue */
  74 #define WORKQUEUE_DEFAULT_PRIOQUEUE 1       /* default priority queue */
  75 #define WORKQUEUE_LOW_PRIOQUEUE     2       /* low priority queue */
  76 #define WORKQUEUE_BG_PRIOQUEUE      3       /* background priority queue */
  77
  78 #define WORKQUEUE_NUM_BUCKETS 7
  79
  80 // Sometimes something gets passed a bucket number and we need a way to express
  81 // that it's actually the event manager.  Use the (n+1)th bucket for that.
  82 #define WORKQUEUE_EVENT_MANAGER_BUCKET (WORKQUEUE_NUM_BUCKETS-1)
  83
  84 /* wq_max_constrained_threads = max(64, N_CPU * WORKQUEUE_CONSTRAINED_FACTOR)
  85  * This used to be WORKQUEUE_NUM_BUCKETS + 1 when NUM_BUCKETS was 4, yielding
  86  * N_CPU * 5. When NUM_BUCKETS changed, we decided that the limit should
  87  * not change. So the factor is now always 5.
  88  */
  89 #define WORKQUEUE_CONSTRAINED_FACTOR 5
  90
  91 #define WORKQUEUE_OVERCOMMIT    0x10000
  92
  93 struct threadlist {
  94         TAILQ_ENTRY(threadlist) th_entry;
  95         thread_t th_thread;
  96         int      th_flags;
  97         uint8_t  th_priority;
  98         uint8_t  th_policy;
  99         struct workqueue *th_workq;
 100         mach_vm_size_t th_stacksize;
 101         mach_vm_size_t th_allocsize;
 102         mach_vm_offset_t th_stackaddr;
 103         mach_port_name_t th_thport;
 104 };
 105 #define TH_LIST_INITED          0x01
 106 #define TH_LIST_RUNNING         0x02
 107 #define TH_LIST_BLOCKED         0x04
 108 #define TH_LIST_SUSPENDED       0x08
 109 #define TH_LIST_BUSY            0x10
 110 #define TH_LIST_NEED_WAKEUP     0x20
 111 #define TH_LIST_CONSTRAINED     0x40
 112 #define TH_LIST_EVENT_MGR_SCHED_PRI     0x80
 113
 114
 115 struct workqueue {
 116         proc_t          wq_proc;
 117         vm_map_t        wq_map;
 118         task_t          wq_task;
 119         thread_call_t   wq_atimer_call;
 120         int             wq_flags;  // updated atomically
 121         int                     wq_lflags; // protected by wqueue lock
 122         uint64_t        wq_thread_yielded_timestamp;
 123         uint32_t        wq_thread_yielded_count;
 124         uint32_t        wq_timer_interval;
 125         uint32_t        wq_max_concurrency;
 126         uint32_t        wq_threads_scheduled;
 127         uint32_t        wq_constrained_threads_scheduled;
 128         uint32_t        wq_nthreads;
 129         uint32_t        wq_thidlecount;
 130         TAILQ_HEAD(, threadlist) wq_thrunlist;
 131         TAILQ_HEAD(, threadlist) wq_thidlelist;
 132
 133         /* Counters for how many requests we have outstanding.  The invariants here:
 134          *   - reqcount == SUM(requests) + (event manager ? 1 : 0)
 135          *   - SUM(ocrequests) + SUM(kevent_requests) + SUM(kevent_ocrequests) <= SUM(requests)
 136          *   - # of constrained requests is difference between quantities above
 137          * i.e. a kevent+overcommit request will incrument reqcount, requests and
 138          * kevent_ocrequests only.
 139          */
 140         uint32_t        wq_reqcount;
 141         uint16_t        wq_requests[WORKQUEUE_NUM_BUCKETS];
 142         uint16_t        wq_ocrequests[WORKQUEUE_NUM_BUCKETS];
 143         uint16_t        wq_kevent_requests[WORKQUEUE_NUM_BUCKETS];
 144         uint16_t        wq_kevent_ocrequests[WORKQUEUE_NUM_BUCKETS];
 145
 146         uint16_t        wq_reqconc[WORKQUEUE_NUM_BUCKETS];                      /* requested concurrency for each priority level */
 147         uint16_t        wq_thscheduled_count[WORKQUEUE_NUM_BUCKETS];
 148         uint32_t        wq_thactive_count[WORKQUEUE_NUM_BUCKETS] __attribute__((aligned(4))); /* must be uint32_t since we OSAddAtomic on these */
 149         uint64_t        wq_lastblocked_ts[WORKQUEUE_NUM_BUCKETS] __attribute__((aligned(8))); /* XXX: why per bucket? */
 150
 151         uint32_t wq_event_manager_priority;
 152 };
 153 #define WQ_LIST_INITED          0x01
 154 #define WQ_ATIMER_RUNNING       0x02
 155 #define WQ_EXITING              0x04
 156
 157 #define WQL_ATIMER_BUSY         0x01
 158 #define WQL_ATIMER_WAITING      0x02
 159 #define WQL_EXCEEDED_CONSTRAINED_THREAD_LIMIT    0x04
 160 #define WQL_EXCEEDED_TOTAL_THREAD_LIMIT          0x08
 161
 162 #define WQ_VECT_SET_BIT(vector, bit)    \
 163         vector[(bit) / 32] |= (1 << ((bit) % 32))
 164
 165 #define WQ_VECT_CLEAR_BIT(vector, bit)  \
 166         vector[(bit) / 32] &= ~(1 << ((bit) % 32))
 167
 168 #define WQ_VECT_TEST_BIT(vector, bit)   \
 169         vector[(bit) / 32] & (1 << ((bit) % 32))
 170
 171 #define WORKQUEUE_MAXTHREADS            512
 172 #define WQ_YIELDED_THRESHOLD            2000
 173 #define WQ_YIELDED_WINDOW_USECS         30000
 174 #define WQ_STALLED_WINDOW_USECS         200
 175 #define WQ_REDUCE_POOL_WINDOW_USECS     5000000
 176 #define WQ_MAX_TIMER_INTERVAL_USECS     50000
 177
 178 #endif // KERNEL
 179
 180 #endif // _WORKQUEUE_INTERNAL_H_