X-Git-Url: https://git.saurik.com/apple/libpthread.git/blobdiff_plain/215aeb0310146fee62c0f2602966571180f8eb63..0691f650c3dd130e4d2ecd2130734ac6b243a0f5:/kern/workqueue_internal.h diff --git a/kern/workqueue_internal.h b/kern/workqueue_internal.h index cad0b50..28d870e 100644 --- a/kern/workqueue_internal.h +++ b/kern/workqueue_internal.h @@ -34,17 +34,30 @@ */ /* workq_kernreturn commands */ -#define WQOPS_THREAD_RETURN 4 -#define WQOPS_QUEUE_NEWSPISUPP 0x10 /* this is to check for newer SPI support */ -#define WQOPS_QUEUE_REQTHREADS 0x20 /* request number of threads of a prio */ -#define WQOPS_QUEUE_REQTHREADS2 0x30 /* request a number of threads in a given priority bucket */ - -/* flag values for reuse field in the libc side _pthread_wqthread */ -#define WQ_FLAG_THREAD_PRIOMASK 0x0000ffff -#define WQ_FLAG_THREAD_PRIOSHIFT (8ull) -#define WQ_FLAG_THREAD_OVERCOMMIT 0x00010000 /* thread is with overcommit prio */ -#define WQ_FLAG_THREAD_REUSE 0x00020000 /* thread is being reused */ -#define WQ_FLAG_THREAD_NEWSPI 0x00040000 /* the call is with new SPIs */ +#define WQOPS_THREAD_RETURN 0x04 /* parks the thread back into the kernel */ +#define WQOPS_QUEUE_NEWSPISUPP 0x10 /* this is to check for newer SPI support */ +#define WQOPS_QUEUE_REQTHREADS 0x20 /* request number of threads of a prio */ +#define WQOPS_QUEUE_REQTHREADS2 0x30 /* request a number of threads in a given priority bucket */ +#define WQOPS_THREAD_KEVENT_RETURN 0x40 /* parks the thread after delivering the passed kevent array */ +#define WQOPS_SET_EVENT_MANAGER_PRIORITY 0x80 /* max() in the provided priority in the the priority of the event manager */ +#define WQOPS_THREAD_WORKLOOP_RETURN 0x100 /* parks the thread after delivering the passed kevent array */ +#define WQOPS_SHOULD_NARROW 0x200 /* checks whether we should narrow our concurrency */ + +/* flag values for upcall flags field, only 8 bits per struct threadlist */ +#define WQ_FLAG_THREAD_PRIOMASK 0x0000ffff +#define WQ_FLAG_THREAD_PRIOSHIFT 16 +#define WQ_FLAG_THREAD_OVERCOMMIT 0x00010000 /* thread is with overcommit prio */ +#define WQ_FLAG_THREAD_REUSE 0x00020000 /* thread is being reused */ +#define WQ_FLAG_THREAD_NEWSPI 0x00040000 /* the call is with new SPIs */ +#define WQ_FLAG_THREAD_KEVENT 0x00080000 /* thread is response to kevent req */ +#define WQ_FLAG_THREAD_EVENT_MANAGER 0x00100000 /* event manager thread */ +#define WQ_FLAG_THREAD_TSD_BASE_SET 0x00200000 /* tsd base has already been set */ +#define WQ_FLAG_THREAD_WORKLOOP 0x00400000 /* workloop thread */ + +#define WQ_THREAD_CLEANUP_QOS QOS_CLASS_DEFAULT + +#define WQ_KEVENT_LIST_LEN 16 // WORKQ_KEVENT_EVENT_BUFFER_LEN +#define WQ_KEVENT_DATA_SIZE (32 * 1024) /* These definitions are only available to the kext, to avoid bleeding constants and types across the boundary to * the userspace library. @@ -71,7 +84,11 @@ enum { #define WORKQUEUE_LOW_PRIOQUEUE 2 /* low priority queue */ #define WORKQUEUE_BG_PRIOQUEUE 3 /* background priority queue */ -#define WORKQUEUE_NUM_BUCKETS 6 +#define WORKQUEUE_NUM_BUCKETS 7 + +// Sometimes something gets passed a bucket number and we need a way to express +// that it's actually the event manager. Use the (n+1)th bucket for that. +#define WORKQUEUE_EVENT_MANAGER_BUCKET (WORKQUEUE_NUM_BUCKETS-1) /* wq_max_constrained_threads = max(64, N_CPU * WORKQUEUE_CONSTRAINED_FACTOR) * This used to be WORKQUEUE_NUM_BUCKETS + 1 when NUM_BUCKETS was 4, yielding @@ -82,78 +99,109 @@ enum { #define WORKQUEUE_OVERCOMMIT 0x10000 +/* + * A thread which is scheduled may read its own th_priority field without + * taking the workqueue lock. Other fields should be assumed to require the + * lock. + */ struct threadlist { TAILQ_ENTRY(threadlist) th_entry; thread_t th_thread; - int th_flags; - uint8_t th_priority; - uint8_t th_policy; struct workqueue *th_workq; - mach_vm_size_t th_stacksize; - mach_vm_size_t th_allocsize; mach_vm_offset_t th_stackaddr; mach_port_name_t th_thport; + uint16_t th_flags; + uint8_t th_upcall_flags; + uint8_t th_priority; +}; + +#define TH_LIST_INITED 0x0001 /* Set at thread creation. */ +#define TH_LIST_RUNNING 0x0002 /* On thrunlist, not parked. */ +#define TH_LIST_KEVENT 0x0004 /* Thread requested by kevent */ +#define TH_LIST_NEW 0x0008 /* First return to userspace */ +#define TH_LIST_BUSY 0x0010 /* Removed from idle list but not ready yet. */ +#define TH_LIST_KEVENT_BOUND 0x0020 /* Thread bound to kqueues */ +#define TH_LIST_CONSTRAINED 0x0040 /* Non-overcommit thread. */ +#define TH_LIST_EVENT_MGR_SCHED_PRI 0x0080 /* Non-QoS Event Manager */ +#define TH_LIST_UNBINDING 0x0100 /* Thread is unbinding during park */ +#define TH_LIST_REMOVING_VOUCHER 0x0200 /* Thread is removing its voucher */ +#define TH_LIST_PACING 0x0400 /* Thread is participating in pacing */ + +struct threadreq { + TAILQ_ENTRY(threadreq) tr_entry; + uint16_t tr_flags; + uint8_t tr_state; + uint8_t tr_priority; }; -#define TH_LIST_INITED 0x01 -#define TH_LIST_RUNNING 0x02 -#define TH_LIST_BLOCKED 0x04 -#define TH_LIST_SUSPENDED 0x08 -#define TH_LIST_BUSY 0x10 -#define TH_LIST_NEED_WAKEUP 0x20 -#define TH_LIST_CONSTRAINED 0x40 +TAILQ_HEAD(threadreq_head, threadreq); +#define TR_STATE_NEW 0 /* Not yet enqueued */ +#define TR_STATE_WAITING 1 /* Waiting to be serviced - on reqlist */ +#define TR_STATE_COMPLETE 2 /* Request handled - for caller to free */ +#define TR_STATE_DEAD 3 + +#define TR_FLAG_KEVENT 0x01 +#define TR_FLAG_OVERCOMMIT 0x02 +#define TR_FLAG_ONSTACK 0x04 +#define TR_FLAG_WORKLOOP 0x08 +#define TR_FLAG_NO_PACING 0x10 + +#if defined(__LP64__) +typedef unsigned __int128 wq_thactive_t; +#else +typedef uint64_t wq_thactive_t; +#endif struct workqueue { proc_t wq_proc; vm_map_t wq_map; task_t wq_task; - thread_call_t wq_atimer_call; - int wq_flags; - int wq_lflags; - uint64_t wq_thread_yielded_timestamp; - uint32_t wq_thread_yielded_count; + + lck_spin_t wq_lock; + + thread_call_t wq_atimer_delayed_call; + thread_call_t wq_atimer_immediate_call; + + uint32_t _Atomic wq_flags; uint32_t wq_timer_interval; - uint32_t wq_max_concurrency; uint32_t wq_threads_scheduled; uint32_t wq_constrained_threads_scheduled; uint32_t wq_nthreads; uint32_t wq_thidlecount; - uint32_t wq_reqcount; + uint32_t wq_event_manager_priority; + uint8_t wq_lflags; // protected by wqueue lock + uint8_t wq_paced; // protected by wqueue lock + uint16_t __wq_unused; + TAILQ_HEAD(, threadlist) wq_thrunlist; TAILQ_HEAD(, threadlist) wq_thidlelist; - uint16_t wq_requests[WORKQUEUE_NUM_BUCKETS]; - uint16_t wq_ocrequests[WORKQUEUE_NUM_BUCKETS]; - uint16_t wq_reqconc[WORKQUEUE_NUM_BUCKETS]; /* requested concurrency for each priority level */ + TAILQ_HEAD(, threadlist) wq_thidlemgrlist; + + uint32_t wq_reqcount; /* number of elements on the following lists */ + struct threadreq_head wq_overcommit_reqlist[WORKQUEUE_EVENT_MANAGER_BUCKET]; + struct threadreq_head wq_reqlist[WORKQUEUE_EVENT_MANAGER_BUCKET]; + struct threadreq wq_event_manager_threadreq; + + struct threadreq *wq_cached_threadreq; + uint16_t wq_thscheduled_count[WORKQUEUE_NUM_BUCKETS]; - uint32_t wq_thactive_count[WORKQUEUE_NUM_BUCKETS] __attribute__((aligned(4))); /* must be uint32_t since we OSAddAtomic on these */ - uint64_t wq_lastblocked_ts[WORKQUEUE_NUM_BUCKETS] __attribute__((aligned(8))); + _Atomic wq_thactive_t wq_thactive; + _Atomic uint64_t wq_lastblocked_ts[WORKQUEUE_NUM_BUCKETS]; }; -#define WQ_LIST_INITED 0x01 -#define WQ_ATIMER_RUNNING 0x02 -#define WQ_EXITING 0x04 +#define WQ_EXITING 0x01 +#define WQ_ATIMER_DELAYED_RUNNING 0x02 +#define WQ_ATIMER_IMMEDIATE_RUNNING 0x04 #define WQL_ATIMER_BUSY 0x01 #define WQL_ATIMER_WAITING 0x02 -#define WQL_EXCEEDED_CONSTRAINED_THREAD_LIMIT 0x04 -#define WQL_EXCEEDED_TOTAL_THREAD_LIMIT 0x08 - - -#define WQ_VECT_SET_BIT(vector, bit) \ - vector[(bit) / 32] |= (1 << ((bit) % 32)) - -#define WQ_VECT_CLEAR_BIT(vector, bit) \ - vector[(bit) / 32] &= ~(1 << ((bit) % 32)) - -#define WQ_VECT_TEST_BIT(vector, bit) \ - vector[(bit) / 32] & (1 << ((bit) % 32)) #define WORKQUEUE_MAXTHREADS 512 -#define WQ_YIELDED_THRESHOLD 2000 -#define WQ_YIELDED_WINDOW_USECS 30000 #define WQ_STALLED_WINDOW_USECS 200 #define WQ_REDUCE_POOL_WINDOW_USECS 5000000 #define WQ_MAX_TIMER_INTERVAL_USECS 50000 +#define WQ_THREADLIST_EXITING_POISON (void *)~0ul + #endif // KERNEL #endif // _WORKQUEUE_INTERNAL_H_