[apple/xnu.git] / bsd / pthread / workqueue_internal.h

/*
 * Copyright (c) 2014 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#ifndef _WORKQUEUE_INTERNAL_H_
#define _WORKQUEUE_INTERNAL_H_

// Sometimes something gets passed a bucket number and we need a way to express
// that it's actually the event manager.  Use the (0)th bucket for that.
#define WORKQ_THREAD_QOS_MIN        (THREAD_QOS_MAINTENANCE)
#define WORKQ_THREAD_QOS_MAX        (THREAD_QOS_LAST)
#define WORKQ_THREAD_QOS_CLEANUP    (THREAD_QOS_LEGACY)
#define WORKQ_THREAD_QOS_ABOVEUI    (THREAD_QOS_LAST)
#define WORKQ_THREAD_QOS_MANAGER    (THREAD_QOS_LAST + 1) // outside of MIN/MAX

#define WORKQ_NUM_QOS_BUCKETS       (WORKQ_THREAD_QOS_MAX - 1)  // MT/BG shared
#define WORKQ_NUM_BUCKETS           (WORKQ_NUM_QOS_BUCKETS + 1) // + mgr

/* These definitions are only available to the kext, to avoid bleeding
 * constants and types across the boundary to the userspace library.
 */
#ifdef KERNEL
#pragma mark wq structs

/* These defines come from kern/thread.h but are XNU_KERNEL_PRIVATE so do not get
 * exported to kernel extensions.
 */
#define SCHED_CALL_BLOCK 0x1
#define SCHED_CALL_UNBLOCK 0x2

/* old workq priority scheme */

#define WORKQUEUE_HIGH_PRIOQUEUE    0       /* high priority queue */
#define WORKQUEUE_DEFAULT_PRIOQUEUE 1       /* default priority queue */
#define WORKQUEUE_LOW_PRIOQUEUE     2       /* low priority queue */
#define WORKQUEUE_BG_PRIOQUEUE      3       /* background priority queue */

/* wq_max_constrained_threads = max(64, N_CPU * WORKQUEUE_CONSTRAINED_FACTOR)
 * This used to be WORKQ_NUM_BUCKETS + 1 when NUM_BUCKETS was 4, yielding
 * N_CPU * 5. When NUM_BUCKETS changed, we decided that the limit should
 * not change. So the factor is now always 5.
 */
#define WORKQUEUE_CONSTRAINED_FACTOR 5

#if BSD_KERNEL_PRIVATE
#include <kern/priority_queue.h>
#include <kern/thread_call.h>
#include <kern/turnstile.h>
#include <mach/kern_return.h>
#include <sys/queue.h>
#include <sys/kernel_types.h>

/* struct uthread::uu_workq_flags */
#define UT_WORKQ_NEW                   0x01 /* First return to userspace */
#define UT_WORKQ_RUNNING               0x02 /* On thrunlist, not parked. */
#define UT_WORKQ_DYING                 0x04 /* Thread is being killed */
#define UT_WORKQ_OVERCOMMIT            0x08 /* Overcommit thread. */
#define UT_WORKQ_OUTSIDE_QOS           0x10 /* Thread should avoid send QoS changes to kernel */
#define UT_WORKQ_IDLE_CLEANUP          0x20 /* Thread is removing its voucher or stack */
#define UT_WORKQ_EARLY_BOUND           0x40 /* Thread has been bound early */
#define UT_WORKQ_CPUPERCENT            0x80 /* Thread has CPU percent policy active */

typedef union workq_threadreq_param_s {
	struct {
		uint16_t trp_flags;
		uint8_t trp_pri;
		uint8_t trp_pol;
		uint32_t trp_cpupercent: 8,
				trp_refillms: 24;
	};
	uint64_t trp_value;
} workq_threadreq_param_t;

#define TRP_PRIORITY		0x1
#define TRP_POLICY			0x2
#define TRP_CPUPERCENT		0x4
#define TRP_RELEASED		0x8000

typedef struct workq_threadreq_s {
	union {
		struct priority_queue_entry tr_entry;
		thread_t tr_binding_thread;
	};
	uint32_t     tr_flags;
	uint8_t      tr_state;
	thread_qos_t tr_qos;
	uint16_t     tr_count;
} *workq_threadreq_t;

TAILQ_HEAD(threadreq_head, workq_threadreq_s);

#define TR_STATE_IDLE		0  /* request isn't in flight       */
#define TR_STATE_NEW		1  /* request is being initiated    */
#define TR_STATE_QUEUED		2  /* request is being queued       */
#define TR_STATE_BINDING	4  /* request is preposted for bind */

#define TR_FLAG_KEVENT			0x01
#define TR_FLAG_WORKLOOP		0x02
#define TR_FLAG_OVERCOMMIT		0x04
#define TR_FLAG_WL_PARAMS		0x08
#define TR_FLAG_WL_OUTSIDE_QOS	0x10

#if defined(__LP64__)
typedef unsigned __int128 wq_thactive_t;
#else
typedef uint64_t wq_thactive_t;
#endif

typedef enum {
	WQ_EXITING                  = 0x0001,
	WQ_PROC_SUSPENDED           = 0x0002,
	WQ_DEATH_CALL_SCHEDULED     = 0x0004,

	WQ_DELAYED_CALL_SCHEDULED   = 0x0010,
	WQ_DELAYED_CALL_PENDED      = 0x0020,
	WQ_IMMEDIATE_CALL_SCHEDULED = 0x0040,
	WQ_IMMEDIATE_CALL_PENDED    = 0x0080,
} workq_state_flags_t;

TAILQ_HEAD(workq_uthread_head, uthread);

struct workqueue {
	thread_call_t	wq_delayed_call;
	thread_call_t	wq_immediate_call;
	thread_call_t   wq_death_call;
	struct turnstile *wq_turnstile;

	lck_spin_t	wq_lock;

	uint64_t	wq_thread_call_last_run;
	struct os_refcnt wq_refcnt;
	workq_state_flags_t _Atomic wq_flags;
	uint32_t	wq_fulfilled;
	uint32_t	wq_creations;
	uint32_t	wq_timer_interval;
	uint32_t	wq_event_manager_priority;
	uint32_t	wq_reqcount;  /* number of elements on the wq_*_reqlists */
	uint16_t	wq_thdying_count;
	uint16_t	wq_threads_scheduled;
	uint16_t	wq_constrained_threads_scheduled;
	uint16_t	wq_nthreads;
	uint16_t	wq_thidlecount;
	uint16_t	wq_thscheduled_count[WORKQ_NUM_BUCKETS]; // incl. manager

	_Atomic wq_thactive_t wq_thactive;
	_Atomic uint64_t wq_lastblocked_ts[WORKQ_NUM_QOS_BUCKETS];

	struct proc    *wq_proc;
	struct uthread *wq_creator;
	thread_t wq_turnstile_updater; // thread doing a turnstile_update_ineritor
	struct workq_uthread_head wq_thrunlist;
	struct workq_uthread_head wq_thnewlist;
	struct workq_uthread_head wq_thidlelist;

	struct priority_queue wq_overcommit_queue;
	struct priority_queue wq_constrained_queue;
	struct priority_queue wq_special_queue;
	workq_threadreq_t wq_event_manager_threadreq;
};

static_assert(offsetof(struct workqueue, wq_lock) >= sizeof(struct queue_entry),
		"Make sure workq_deallocate_enqueue can cast the workqueue");

#define WORKQUEUE_MAXTHREADS		512
#define WQ_STALLED_WINDOW_USECS		200
#define WQ_REDUCE_POOL_WINDOW_USECS	5000000
#define	WQ_MAX_TIMER_INTERVAL_USECS	50000

#pragma mark definitions

struct kqrequest;
uint32_t _get_pwq_state_kdp(proc_t p);

void workq_exit(struct proc *p);
void workq_mark_exiting(struct proc *p);

bool workq_is_exiting(struct proc *p);

struct turnstile *workq_turnstile(struct proc *p);

void workq_thread_set_max_qos(struct proc *p, struct kqrequest *kqr);

void workq_thread_terminate(struct proc *p, struct uthread *uth);

#define WORKQ_THREADREQ_SET_AST_ON_FAILURE  0x01
#define WORKQ_THREADREQ_ATTEMPT_REBIND      0x02
#define WORKQ_THREADREQ_CAN_CREATE_THREADS  0x04
#define WORKQ_THREADREQ_CREATOR_TRANSFER    0x08
#define WORKQ_THREADREQ_CREATOR_SYNC_UPDATE 0x10

// called with the kq req lock held
bool workq_kern_threadreq_initiate(struct proc *p, struct kqrequest *kqr,
		struct turnstile *ts, thread_qos_t qos, int flags);

// called with the kq req lock held
void workq_kern_threadreq_modify(struct proc *p, struct kqrequest *kqr,
		thread_qos_t qos, int flags);

// called with the kq req lock held
void workq_kern_threadreq_update_inheritor(struct proc *p, struct kqrequest *kqr,
		thread_t owner, struct turnstile *ts, turnstile_update_flags_t flags);

void workq_kern_threadreq_lock(struct proc *p);
void workq_kern_threadreq_unlock(struct proc *p);

void workq_kern_threadreq_redrive(struct proc *p, int flags);

enum workq_set_self_flags {
	WORKQ_SET_SELF_QOS_FLAG = 0x1,
	WORKQ_SET_SELF_VOUCHER_FLAG = 0x2,
	WORKQ_SET_SELF_FIXEDPRIORITY_FLAG = 0x4,
	WORKQ_SET_SELF_TIMESHARE_FLAG = 0x8,
	WORKQ_SET_SELF_WQ_KEVENT_UNBIND = 0x10,
};

void workq_proc_suspended(struct proc *p);
void workq_proc_resumed(struct proc *p);

#endif // BSD_KERNEL_PRIVATE

void workq_init(void);

#endif // KERNEL

#endif // _WORKQUEUE_INTERNAL_H_
Commit	Line	Data
d9a64523 A	1	/*
	2	* Copyright (c) 2014 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29	#ifndef _WORKQUEUE_INTERNAL_H_
	30	#define _WORKQUEUE_INTERNAL_H_
	31
	32	// Sometimes something gets passed a bucket number and we need a way to express
	33	// that it's actually the event manager. Use the (0)th bucket for that.
	34	#define WORKQ_THREAD_QOS_MIN (THREAD_QOS_MAINTENANCE)
	35	#define WORKQ_THREAD_QOS_MAX (THREAD_QOS_LAST)
	36	#define WORKQ_THREAD_QOS_CLEANUP (THREAD_QOS_LEGACY)
	37	#define WORKQ_THREAD_QOS_ABOVEUI (THREAD_QOS_LAST)
	38	#define WORKQ_THREAD_QOS_MANAGER (THREAD_QOS_LAST + 1) // outside of MIN/MAX
	39
	40	#define WORKQ_NUM_QOS_BUCKETS (WORKQ_THREAD_QOS_MAX - 1) // MT/BG shared
	41	#define WORKQ_NUM_BUCKETS (WORKQ_NUM_QOS_BUCKETS + 1) // + mgr
	42
	43	/* These definitions are only available to the kext, to avoid bleeding
	44	* constants and types across the boundary to the userspace library.
	45	*/
	46	#ifdef KERNEL
	47	#pragma mark wq structs
	48
	49	/* These defines come from kern/thread.h but are XNU_KERNEL_PRIVATE so do not get
	50	* exported to kernel extensions.
	51	*/
	52	#define SCHED_CALL_BLOCK 0x1
	53	#define SCHED_CALL_UNBLOCK 0x2
	54
	55	/* old workq priority scheme */
	56
	57	#define WORKQUEUE_HIGH_PRIOQUEUE 0 /* high priority queue */
	58	#define WORKQUEUE_DEFAULT_PRIOQUEUE 1 /* default priority queue */
	59	#define WORKQUEUE_LOW_PRIOQUEUE 2 /* low priority queue */
	60	#define WORKQUEUE_BG_PRIOQUEUE 3 /* background priority queue */
	61
	62	/* wq_max_constrained_threads = max(64, N_CPU * WORKQUEUE_CONSTRAINED_FACTOR)
	63	* This used to be WORKQ_NUM_BUCKETS + 1 when NUM_BUCKETS was 4, yielding
	64	* N_CPU * 5. When NUM_BUCKETS changed, we decided that the limit should
65	* not change. So the factor is now always 5.
66	*/
67	#define WORKQUEUE_CONSTRAINED_FACTOR 5
68
69	#if BSD_KERNEL_PRIVATE
70	#include <kern/priority_queue.h>
71	#include <kern/thread_call.h>
72	#include <kern/turnstile.h>
73	#include <mach/kern_return.h>
74	#include <sys/queue.h>
75	#include <sys/kernel_types.h>
76
77	/* struct uthread::uu_workq_flags */
78	#define UT_WORKQ_NEW 0x01 /* First return to userspace */
79	#define UT_WORKQ_RUNNING 0x02 /* On thrunlist, not parked. */
80	#define UT_WORKQ_DYING 0x04 /* Thread is being killed */
81	#define UT_WORKQ_OVERCOMMIT 0x08 /* Overcommit thread. */
82	#define UT_WORKQ_OUTSIDE_QOS 0x10 /* Thread should avoid send QoS changes to kernel */
83	#define UT_WORKQ_IDLE_CLEANUP 0x20 /* Thread is removing its voucher or stack */
84	#define UT_WORKQ_EARLY_BOUND 0x40 /* Thread has been bound early */
85	#define UT_WORKQ_CPUPERCENT 0x80 /* Thread has CPU percent policy active */
86
87	typedef union workq_threadreq_param_s {
88	struct {
89	uint16_t trp_flags;
90	uint8_t trp_pri;
91	uint8_t trp_pol;
92	uint32_t trp_cpupercent: 8,
93	trp_refillms: 24;
94	};
95	uint64_t trp_value;
96	} workq_threadreq_param_t;
97
98	#define TRP_PRIORITY 0x1
99	#define TRP_POLICY 0x2
100	#define TRP_CPUPERCENT 0x4
101	#define TRP_RELEASED 0x8000
102
103	typedef struct workq_threadreq_s {
104	union {
105	struct priority_queue_entry tr_entry;
106	thread_t tr_binding_thread;
107	};
108	uint32_t tr_flags;
109	uint8_t tr_state;
110	thread_qos_t tr_qos;
111	uint16_t tr_count;
112	} *workq_threadreq_t;
113
114	TAILQ_HEAD(threadreq_head, workq_threadreq_s);
115
116	#define TR_STATE_IDLE 0 /* request isn't in flight */
117	#define TR_STATE_NEW 1 /* request is being initiated */
118	#define TR_STATE_QUEUED 2 /* request is being queued */
119	#define TR_STATE_BINDING 4 /* request is preposted for bind */
120
121	#define TR_FLAG_KEVENT 0x01
122	#define TR_FLAG_WORKLOOP 0x02
123	#define TR_FLAG_OVERCOMMIT 0x04
124	#define TR_FLAG_WL_PARAMS 0x08
125	#define TR_FLAG_WL_OUTSIDE_QOS 0x10
126
127	#if defined(__LP64__)
128	typedef unsigned __int128 wq_thactive_t;
129	#else
130	typedef uint64_t wq_thactive_t;
131	#endif
132
133	typedef enum {
134	WQ_EXITING = 0x0001,
135	WQ_PROC_SUSPENDED = 0x0002,
136	WQ_DEATH_CALL_SCHEDULED = 0x0004,
137
138	WQ_DELAYED_CALL_SCHEDULED = 0x0010,
139	WQ_DELAYED_CALL_PENDED = 0x0020,
140	WQ_IMMEDIATE_CALL_SCHEDULED = 0x0040,
141	WQ_IMMEDIATE_CALL_PENDED = 0x0080,
142	} workq_state_flags_t;
143
144	TAILQ_HEAD(workq_uthread_head, uthread);
145
146	struct workqueue {
147	thread_call_t wq_delayed_call;
148	thread_call_t wq_immediate_call;
149	thread_call_t wq_death_call;
150	struct turnstile *wq_turnstile;
151
152	lck_spin_t wq_lock;
153
154	uint64_t wq_thread_call_last_run;
155	struct os_refcnt wq_refcnt;
156	workq_state_flags_t _Atomic wq_flags;
157	uint32_t wq_fulfilled;
158	uint32_t wq_creations;
159	uint32_t wq_timer_interval;
160	uint32_t wq_event_manager_priority;
161	uint32_t wq_reqcount; /* number of elements on the wq__reqlists /
162	uint16_t wq_thdying_count;
163	uint16_t wq_threads_scheduled;
164	uint16_t wq_constrained_threads_scheduled;
165	uint16_t wq_nthreads;
166	uint16_t wq_thidlecount;
167	uint16_t wq_thscheduled_count[WORKQ_NUM_BUCKETS]; // incl. manager
168
169	_Atomic wq_thactive_t wq_thactive;
170	_Atomic uint64_t wq_lastblocked_ts[WORKQ_NUM_QOS_BUCKETS];
171
172	struct proc *wq_proc;
173	struct uthread *wq_creator;
174	thread_t wq_turnstile_updater; // thread doing a turnstile_update_ineritor
175	struct workq_uthread_head wq_thrunlist;
176	struct workq_uthread_head wq_thnewlist;
177	struct workq_uthread_head wq_thidlelist;
178
179	struct priority_queue wq_overcommit_queue;
180	struct priority_queue wq_constrained_queue;
181	struct priority_queue wq_special_queue;
182	workq_threadreq_t wq_event_manager_threadreq;
183	};
184
185	static_assert(offsetof(struct workqueue, wq_lock) >= sizeof(struct queue_entry),
186	"Make sure workq_deallocate_enqueue can cast the workqueue");
187
188	#define WORKQUEUE_MAXTHREADS 512
189	#define WQ_STALLED_WINDOW_USECS 200
190	#define WQ_REDUCE_POOL_WINDOW_USECS 5000000
191	#define WQ_MAX_TIMER_INTERVAL_USECS 50000
192
193	#pragma mark definitions
194
195	struct kqrequest;
196	uint32_t _get_pwq_state_kdp(proc_t p);
197
198	void workq_exit(struct proc *p);
199	void workq_mark_exiting(struct proc *p);
200
201	bool workq_is_exiting(struct proc *p);
202
203	struct turnstile workq_turnstile(struct proc p);
204
205	void workq_thread_set_max_qos(struct proc p, struct kqrequest kqr);
206
207	void workq_thread_terminate(struct proc p, struct uthread uth);
208
209	#define WORKQ_THREADREQ_SET_AST_ON_FAILURE 0x01
210	#define WORKQ_THREADREQ_ATTEMPT_REBIND 0x02
211	#define WORKQ_THREADREQ_CAN_CREATE_THREADS 0x04
212	#define WORKQ_THREADREQ_CREATOR_TRANSFER 0x08
213	#define WORKQ_THREADREQ_CREATOR_SYNC_UPDATE 0x10
214
215	// called with the kq req lock held
216	bool workq_kern_threadreq_initiate(struct proc p, struct kqrequest kqr,
217	struct turnstile *ts, thread_qos_t qos, int flags);
218
219	// called with the kq req lock held
220	void workq_kern_threadreq_modify(struct proc p, struct kqrequest kqr,
221	thread_qos_t qos, int flags);
222
223	// called with the kq req lock held
224	void workq_kern_threadreq_update_inheritor(struct proc p, struct kqrequest kqr,
225	thread_t owner, struct turnstile *ts, turnstile_update_flags_t flags);
226
227	void workq_kern_threadreq_lock(struct proc *p);
228	void workq_kern_threadreq_unlock(struct proc *p);
229
230	void workq_kern_threadreq_redrive(struct proc *p, int flags);
231
232	enum workq_set_self_flags {
233	WORKQ_SET_SELF_QOS_FLAG = 0x1,
234	WORKQ_SET_SELF_VOUCHER_FLAG = 0x2,
235	WORKQ_SET_SELF_FIXEDPRIORITY_FLAG = 0x4,
236	WORKQ_SET_SELF_TIMESHARE_FLAG = 0x8,
237	WORKQ_SET_SELF_WQ_KEVENT_UNBIND = 0x10,
238	};
239
240	void workq_proc_suspended(struct proc *p);
241	void workq_proc_resumed(struct proc *p);
242
243	#endif // BSD_KERNEL_PRIVATE
244
245	void workq_init(void);
246
247	#endif // KERNEL
248
249	#endif // _WORKQUEUE_INTERNAL_H_