bsd/pthread/workqueue_internal.h

   1 /*
   2  * Copyright (c) 2014 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #ifndef _WORKQUEUE_INTERNAL_H_
  30 #define _WORKQUEUE_INTERNAL_H_
  31
  32 // Sometimes something gets passed a bucket number and we need a way to express
  33 // that it's actually the event manager.  Use the (0)th bucket for that.
  34 #define WORKQ_THREAD_QOS_MIN        (THREAD_QOS_MAINTENANCE)
  35 #define WORKQ_THREAD_QOS_MAX        (THREAD_QOS_LAST)
  36 #define WORKQ_THREAD_QOS_CLEANUP    (THREAD_QOS_LEGACY)
  37 #define WORKQ_THREAD_QOS_ABOVEUI    (THREAD_QOS_LAST)
  38 #define WORKQ_THREAD_QOS_MANAGER    (THREAD_QOS_LAST + 1) // outside of MIN/MAX
  39
  40 #define WORKQ_NUM_QOS_BUCKETS       (WORKQ_THREAD_QOS_MAX - 1)  // MT/BG shared
  41 #define WORKQ_NUM_BUCKETS           (WORKQ_NUM_QOS_BUCKETS + 1) // + mgr
  42
  43 /* These definitions are only available to the kext, to avoid bleeding
  44  * constants and types across the boundary to the userspace library.
  45  */
  46 #ifdef KERNEL
  47 #pragma mark wq structs
  48
  49 /* These defines come from kern/thread.h but are XNU_KERNEL_PRIVATE so do not get
  50  * exported to kernel extensions.
  51  */
  52 #define SCHED_CALL_BLOCK 0x1
  53 #define SCHED_CALL_UNBLOCK 0x2
  54
  55 /* old workq priority scheme */
  56
  57 #define WORKQUEUE_HIGH_PRIOQUEUE    0       /* high priority queue */
  58 #define WORKQUEUE_DEFAULT_PRIOQUEUE 1       /* default priority queue */
  59 #define WORKQUEUE_LOW_PRIOQUEUE     2       /* low priority queue */
  60 #define WORKQUEUE_BG_PRIOQUEUE      3       /* background priority queue */
  61
  62 /* wq_max_constrained_threads = max(64, N_CPU * WORKQUEUE_CONSTRAINED_FACTOR)
  63  * This used to be WORKQ_NUM_BUCKETS + 1 when NUM_BUCKETS was 4, yielding
  64  * N_CPU * 5. When NUM_BUCKETS changed, we decided that the limit should
  65  * not change. So the factor is now always 5.
  66  */
  67 #define WORKQUEUE_CONSTRAINED_FACTOR 5
  68
  69 #if BSD_KERNEL_PRIVATE
  70 #include <kern/priority_queue.h>
  71 #include <kern/thread_call.h>
  72 #include <kern/turnstile.h>
  73 #include <mach/kern_return.h>
  74 #include <sys/queue.h>
  75 #include <sys/kernel_types.h>
  76
  77 /* struct uthread::uu_workq_flags */
  78 #define UT_WORKQ_NEW                   0x01 /* First return to userspace */
  79 #define UT_WORKQ_RUNNING               0x02 /* On thrunlist, not parked. */
  80 #define UT_WORKQ_DYING                 0x04 /* Thread is being killed */
  81 #define UT_WORKQ_OVERCOMMIT            0x08 /* Overcommit thread. */
  82 #define UT_WORKQ_OUTSIDE_QOS           0x10 /* Thread should avoid send QoS changes to kernel */
  83 #define UT_WORKQ_IDLE_CLEANUP          0x20 /* Thread is removing its voucher or stack */
  84 #define UT_WORKQ_EARLY_BOUND           0x40 /* Thread has been bound early */
  85 #define UT_WORKQ_CPUPERCENT            0x80 /* Thread has CPU percent policy active */
  86
  87 typedef union workq_threadreq_param_s {
  88         struct {
  89                 uint16_t trp_flags;
  90                 uint8_t trp_pri;
  91                 uint8_t trp_pol;
  92                 uint32_t trp_cpupercent: 8,
  93                                 trp_refillms: 24;
  94         };
  95         uint64_t trp_value;
  96 } workq_threadreq_param_t;
  97
  98 #define TRP_PRIORITY            0x1
  99 #define TRP_POLICY                      0x2
 100 #define TRP_CPUPERCENT          0x4
 101 #define TRP_RELEASED            0x8000
 102
 103 typedef struct workq_threadreq_s {
 104         union {
 105                 struct priority_queue_entry tr_entry;
 106                 thread_t tr_binding_thread;
 107         };
 108         uint32_t     tr_flags;
 109         uint8_t      tr_state;
 110         thread_qos_t tr_qos;
 111         uint16_t     tr_count;
 112 } *workq_threadreq_t;
 113
 114 TAILQ_HEAD(threadreq_head, workq_threadreq_s);
 115
 116 #define TR_STATE_IDLE           0  /* request isn't in flight       */
 117 #define TR_STATE_NEW            1  /* request is being initiated    */
 118 #define TR_STATE_QUEUED         2  /* request is being queued       */
 119 #define TR_STATE_BINDING        4  /* request is preposted for bind */
 120
 121 #define TR_FLAG_KEVENT                  0x01
 122 #define TR_FLAG_WORKLOOP                0x02
 123 #define TR_FLAG_OVERCOMMIT              0x04
 124 #define TR_FLAG_WL_PARAMS               0x08
 125 #define TR_FLAG_WL_OUTSIDE_QOS  0x10
 126
 127 #if defined(__LP64__)
 128 typedef unsigned __int128 wq_thactive_t;
 129 #else
 130 typedef uint64_t wq_thactive_t;
 131 #endif
 132
 133 typedef enum {
 134         WQ_EXITING                  = 0x0001,
 135         WQ_PROC_SUSPENDED           = 0x0002,
 136         WQ_DEATH_CALL_SCHEDULED     = 0x0004,
 137
 138         WQ_DELAYED_CALL_SCHEDULED   = 0x0010,
 139         WQ_DELAYED_CALL_PENDED      = 0x0020,
 140         WQ_IMMEDIATE_CALL_SCHEDULED = 0x0040,
 141         WQ_IMMEDIATE_CALL_PENDED    = 0x0080,
 142 } workq_state_flags_t;
 143
 144 TAILQ_HEAD(workq_uthread_head, uthread);
 145
 146 struct workqueue {
 147         thread_call_t   wq_delayed_call;
 148         thread_call_t   wq_immediate_call;
 149         thread_call_t   wq_death_call;
 150         struct turnstile *wq_turnstile;
 151
 152         lck_spin_t      wq_lock;
 153
 154         uint64_t        wq_thread_call_last_run;
 155         struct os_refcnt wq_refcnt;
 156         workq_state_flags_t _Atomic wq_flags;
 157         uint32_t        wq_fulfilled;
 158         uint32_t        wq_creations;
 159         uint32_t        wq_timer_interval;
 160         uint32_t        wq_event_manager_priority;
 161         uint32_t        wq_reqcount;  /* number of elements on the wq_*_reqlists */
 162         uint16_t        wq_thdying_count;
 163         uint16_t        wq_threads_scheduled;
 164         uint16_t        wq_constrained_threads_scheduled;
 165         uint16_t        wq_nthreads;
 166         uint16_t        wq_thidlecount;
 167         uint16_t        wq_thscheduled_count[WORKQ_NUM_BUCKETS]; // incl. manager
 168
 169         _Atomic wq_thactive_t wq_thactive;
 170         _Atomic uint64_t wq_lastblocked_ts[WORKQ_NUM_QOS_BUCKETS];
 171
 172         struct proc    *wq_proc;
 173         struct uthread *wq_creator;
 174         thread_t wq_turnstile_updater; // thread doing a turnstile_update_ineritor
 175         struct workq_uthread_head wq_thrunlist;
 176         struct workq_uthread_head wq_thnewlist;
 177         struct workq_uthread_head wq_thidlelist;
 178
 179         struct priority_queue wq_overcommit_queue;
 180         struct priority_queue wq_constrained_queue;
 181         struct priority_queue wq_special_queue;
 182         workq_threadreq_t wq_event_manager_threadreq;
 183 };
 184
 185 static_assert(offsetof(struct workqueue, wq_lock) >= sizeof(struct queue_entry),
 186                 "Make sure workq_deallocate_enqueue can cast the workqueue");
 187
 188 #define WORKQUEUE_MAXTHREADS            512
 189 #define WQ_STALLED_WINDOW_USECS         200
 190 #define WQ_REDUCE_POOL_WINDOW_USECS     5000000
 191 #define WQ_MAX_TIMER_INTERVAL_USECS     50000
 192
 193 #pragma mark definitions
 194
 195 struct kqrequest;
 196 uint32_t _get_pwq_state_kdp(proc_t p);
 197
 198 void workq_exit(struct proc *p);
 199 void workq_mark_exiting(struct proc *p);
 200
 201 bool workq_is_exiting(struct proc *p);
 202
 203 struct turnstile *workq_turnstile(struct proc *p);
 204
 205 void workq_thread_set_max_qos(struct proc *p, struct kqrequest *kqr);
 206
 207 void workq_thread_terminate(struct proc *p, struct uthread *uth);
 208
 209 #define WORKQ_THREADREQ_SET_AST_ON_FAILURE  0x01
 210 #define WORKQ_THREADREQ_ATTEMPT_REBIND      0x02
 211 #define WORKQ_THREADREQ_CAN_CREATE_THREADS  0x04
 212 #define WORKQ_THREADREQ_CREATOR_TRANSFER    0x08
 213 #define WORKQ_THREADREQ_CREATOR_SYNC_UPDATE 0x10
 214
 215 // called with the kq req lock held
 216 bool workq_kern_threadreq_initiate(struct proc *p, struct kqrequest *kqr,
 217                 struct turnstile *ts, thread_qos_t qos, int flags);
 218
 219 // called with the kq req lock held
 220 void workq_kern_threadreq_modify(struct proc *p, struct kqrequest *kqr,
 221                 thread_qos_t qos, int flags);
 222
 223 // called with the kq req lock held
 224 void workq_kern_threadreq_update_inheritor(struct proc *p, struct kqrequest *kqr,
 225                 thread_t owner, struct turnstile *ts, turnstile_update_flags_t flags);
 226
 227 void workq_kern_threadreq_lock(struct proc *p);
 228 void workq_kern_threadreq_unlock(struct proc *p);
 229
 230 void workq_kern_threadreq_redrive(struct proc *p, int flags);
 231
 232 enum workq_set_self_flags {
 233         WORKQ_SET_SELF_QOS_FLAG = 0x1,
 234         WORKQ_SET_SELF_VOUCHER_FLAG = 0x2,
 235         WORKQ_SET_SELF_FIXEDPRIORITY_FLAG = 0x4,
 236         WORKQ_SET_SELF_TIMESHARE_FLAG = 0x8,
 237         WORKQ_SET_SELF_WQ_KEVENT_UNBIND = 0x10,
 238 };
 239
 240 void workq_proc_suspended(struct proc *p);
 241 void workq_proc_resumed(struct proc *p);
 242
 243 #endif // BSD_KERNEL_PRIVATE
 244
 245 void workq_init(void);
 246
 247 #endif // KERNEL
 248
 249 #endif // _WORKQUEUE_INTERNAL_H_