X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/d26ffc64f583ab2d29df48f13518685602bc8832..d9a64523371fa019c4575bb400cbbc3a50ac9903:/bsd/pthread/workqueue_internal.h diff --git a/bsd/pthread/workqueue_internal.h b/bsd/pthread/workqueue_internal.h new file mode 100644 index 000000000..a072d35ef --- /dev/null +++ b/bsd/pthread/workqueue_internal.h @@ -0,0 +1,249 @@ +/* + * Copyright (c) 2014 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ + +#ifndef _WORKQUEUE_INTERNAL_H_ +#define _WORKQUEUE_INTERNAL_H_ + +// Sometimes something gets passed a bucket number and we need a way to express +// that it's actually the event manager. Use the (0)th bucket for that. +#define WORKQ_THREAD_QOS_MIN (THREAD_QOS_MAINTENANCE) +#define WORKQ_THREAD_QOS_MAX (THREAD_QOS_LAST) +#define WORKQ_THREAD_QOS_CLEANUP (THREAD_QOS_LEGACY) +#define WORKQ_THREAD_QOS_ABOVEUI (THREAD_QOS_LAST) +#define WORKQ_THREAD_QOS_MANAGER (THREAD_QOS_LAST + 1) // outside of MIN/MAX + +#define WORKQ_NUM_QOS_BUCKETS (WORKQ_THREAD_QOS_MAX - 1) // MT/BG shared +#define WORKQ_NUM_BUCKETS (WORKQ_NUM_QOS_BUCKETS + 1) // + mgr + +/* These definitions are only available to the kext, to avoid bleeding + * constants and types across the boundary to the userspace library. + */ +#ifdef KERNEL +#pragma mark wq structs + +/* These defines come from kern/thread.h but are XNU_KERNEL_PRIVATE so do not get + * exported to kernel extensions. + */ +#define SCHED_CALL_BLOCK 0x1 +#define SCHED_CALL_UNBLOCK 0x2 + +/* old workq priority scheme */ + +#define WORKQUEUE_HIGH_PRIOQUEUE 0 /* high priority queue */ +#define WORKQUEUE_DEFAULT_PRIOQUEUE 1 /* default priority queue */ +#define WORKQUEUE_LOW_PRIOQUEUE 2 /* low priority queue */ +#define WORKQUEUE_BG_PRIOQUEUE 3 /* background priority queue */ + +/* wq_max_constrained_threads = max(64, N_CPU * WORKQUEUE_CONSTRAINED_FACTOR) + * This used to be WORKQ_NUM_BUCKETS + 1 when NUM_BUCKETS was 4, yielding + * N_CPU * 5. When NUM_BUCKETS changed, we decided that the limit should + * not change. So the factor is now always 5. + */ +#define WORKQUEUE_CONSTRAINED_FACTOR 5 + +#if BSD_KERNEL_PRIVATE +#include +#include +#include +#include +#include +#include + +/* struct uthread::uu_workq_flags */ +#define UT_WORKQ_NEW 0x01 /* First return to userspace */ +#define UT_WORKQ_RUNNING 0x02 /* On thrunlist, not parked. */ +#define UT_WORKQ_DYING 0x04 /* Thread is being killed */ +#define UT_WORKQ_OVERCOMMIT 0x08 /* Overcommit thread. */ +#define UT_WORKQ_OUTSIDE_QOS 0x10 /* Thread should avoid send QoS changes to kernel */ +#define UT_WORKQ_IDLE_CLEANUP 0x20 /* Thread is removing its voucher or stack */ +#define UT_WORKQ_EARLY_BOUND 0x40 /* Thread has been bound early */ +#define UT_WORKQ_CPUPERCENT 0x80 /* Thread has CPU percent policy active */ + +typedef union workq_threadreq_param_s { + struct { + uint16_t trp_flags; + uint8_t trp_pri; + uint8_t trp_pol; + uint32_t trp_cpupercent: 8, + trp_refillms: 24; + }; + uint64_t trp_value; +} workq_threadreq_param_t; + +#define TRP_PRIORITY 0x1 +#define TRP_POLICY 0x2 +#define TRP_CPUPERCENT 0x4 +#define TRP_RELEASED 0x8000 + +typedef struct workq_threadreq_s { + union { + struct priority_queue_entry tr_entry; + thread_t tr_binding_thread; + }; + uint32_t tr_flags; + uint8_t tr_state; + thread_qos_t tr_qos; + uint16_t tr_count; +} *workq_threadreq_t; + +TAILQ_HEAD(threadreq_head, workq_threadreq_s); + +#define TR_STATE_IDLE 0 /* request isn't in flight */ +#define TR_STATE_NEW 1 /* request is being initiated */ +#define TR_STATE_QUEUED 2 /* request is being queued */ +#define TR_STATE_BINDING 4 /* request is preposted for bind */ + +#define TR_FLAG_KEVENT 0x01 +#define TR_FLAG_WORKLOOP 0x02 +#define TR_FLAG_OVERCOMMIT 0x04 +#define TR_FLAG_WL_PARAMS 0x08 +#define TR_FLAG_WL_OUTSIDE_QOS 0x10 + +#if defined(__LP64__) +typedef unsigned __int128 wq_thactive_t; +#else +typedef uint64_t wq_thactive_t; +#endif + +typedef enum { + WQ_EXITING = 0x0001, + WQ_PROC_SUSPENDED = 0x0002, + WQ_DEATH_CALL_SCHEDULED = 0x0004, + + WQ_DELAYED_CALL_SCHEDULED = 0x0010, + WQ_DELAYED_CALL_PENDED = 0x0020, + WQ_IMMEDIATE_CALL_SCHEDULED = 0x0040, + WQ_IMMEDIATE_CALL_PENDED = 0x0080, +} workq_state_flags_t; + +TAILQ_HEAD(workq_uthread_head, uthread); + +struct workqueue { + thread_call_t wq_delayed_call; + thread_call_t wq_immediate_call; + thread_call_t wq_death_call; + struct turnstile *wq_turnstile; + + lck_spin_t wq_lock; + + uint64_t wq_thread_call_last_run; + struct os_refcnt wq_refcnt; + workq_state_flags_t _Atomic wq_flags; + uint32_t wq_fulfilled; + uint32_t wq_creations; + uint32_t wq_timer_interval; + uint32_t wq_event_manager_priority; + uint32_t wq_reqcount; /* number of elements on the wq_*_reqlists */ + uint16_t wq_thdying_count; + uint16_t wq_threads_scheduled; + uint16_t wq_constrained_threads_scheduled; + uint16_t wq_nthreads; + uint16_t wq_thidlecount; + uint16_t wq_thscheduled_count[WORKQ_NUM_BUCKETS]; // incl. manager + + _Atomic wq_thactive_t wq_thactive; + _Atomic uint64_t wq_lastblocked_ts[WORKQ_NUM_QOS_BUCKETS]; + + struct proc *wq_proc; + struct uthread *wq_creator; + thread_t wq_turnstile_updater; // thread doing a turnstile_update_ineritor + struct workq_uthread_head wq_thrunlist; + struct workq_uthread_head wq_thnewlist; + struct workq_uthread_head wq_thidlelist; + + struct priority_queue wq_overcommit_queue; + struct priority_queue wq_constrained_queue; + struct priority_queue wq_special_queue; + workq_threadreq_t wq_event_manager_threadreq; +}; + +static_assert(offsetof(struct workqueue, wq_lock) >= sizeof(struct queue_entry), + "Make sure workq_deallocate_enqueue can cast the workqueue"); + +#define WORKQUEUE_MAXTHREADS 512 +#define WQ_STALLED_WINDOW_USECS 200 +#define WQ_REDUCE_POOL_WINDOW_USECS 5000000 +#define WQ_MAX_TIMER_INTERVAL_USECS 50000 + +#pragma mark definitions + +struct kqrequest; +uint32_t _get_pwq_state_kdp(proc_t p); + +void workq_exit(struct proc *p); +void workq_mark_exiting(struct proc *p); + +bool workq_is_exiting(struct proc *p); + +struct turnstile *workq_turnstile(struct proc *p); + +void workq_thread_set_max_qos(struct proc *p, struct kqrequest *kqr); + +void workq_thread_terminate(struct proc *p, struct uthread *uth); + +#define WORKQ_THREADREQ_SET_AST_ON_FAILURE 0x01 +#define WORKQ_THREADREQ_ATTEMPT_REBIND 0x02 +#define WORKQ_THREADREQ_CAN_CREATE_THREADS 0x04 +#define WORKQ_THREADREQ_CREATOR_TRANSFER 0x08 +#define WORKQ_THREADREQ_CREATOR_SYNC_UPDATE 0x10 + +// called with the kq req lock held +bool workq_kern_threadreq_initiate(struct proc *p, struct kqrequest *kqr, + struct turnstile *ts, thread_qos_t qos, int flags); + +// called with the kq req lock held +void workq_kern_threadreq_modify(struct proc *p, struct kqrequest *kqr, + thread_qos_t qos, int flags); + +// called with the kq req lock held +void workq_kern_threadreq_update_inheritor(struct proc *p, struct kqrequest *kqr, + thread_t owner, struct turnstile *ts, turnstile_update_flags_t flags); + +void workq_kern_threadreq_lock(struct proc *p); +void workq_kern_threadreq_unlock(struct proc *p); + +void workq_kern_threadreq_redrive(struct proc *p, int flags); + +enum workq_set_self_flags { + WORKQ_SET_SELF_QOS_FLAG = 0x1, + WORKQ_SET_SELF_VOUCHER_FLAG = 0x2, + WORKQ_SET_SELF_FIXEDPRIORITY_FLAG = 0x4, + WORKQ_SET_SELF_TIMESHARE_FLAG = 0x8, + WORKQ_SET_SELF_WQ_KEVENT_UNBIND = 0x10, +}; + +void workq_proc_suspended(struct proc *p); +void workq_proc_resumed(struct proc *p); + +#endif // BSD_KERNEL_PRIVATE + +void workq_init(void); + +#endif // KERNEL + +#endif // _WORKQUEUE_INTERNAL_H_