#include <kern/clock.h>
#include <kern/kern_types.h>
+#include <kern/kcdata.h>
#include <kern/locks.h>
+#include <sys/user.h>
#include <sys/_types.h>
#include <sys/_types/_sigset_t.h>
#include <sys/kernel_types.h>
-#include <sys/proc_info.h>
#ifndef PTHREAD_INTERNAL
struct uthread;
#define M_PROC 41
#endif
-#ifdef NEEDS_SCHED_CALL_T
+#if !defined(_SCHED_CALL_T_DEFINED)
+#define _SCHED_CALL_T_DEFINED
typedef void (*sched_call_t)(int type, thread_t thread);
#endif
typedef struct workq_reqthreads_req_s {unsigned long priority; int count;} *workq_reqthreads_req_t;
+typedef struct workq_threadreq_s { void *opaqueptr[2]; uint32_t opaqueint[2];} *workq_threadreq_t;
+enum workq_threadreq_type {
+ WORKQ_THREADREQ_KEVENT = 1,
+ WORKQ_THREADREQ_WORKLOOP = 2,
+ WORKQ_THREADREQ_WORKLOOP_NO_THREAD_CALL = 3,
+ WORKQ_THREADREQ_REDRIVE = 4,
+};
+enum workq_threadreq_op {
+ WORKQ_THREADREQ_CHANGE_PRI = 1,
+ WORKQ_THREADREQ_CANCEL = 2,
+ WORKQ_THREADREQ_CHANGE_PRI_NO_THREAD_CALL = 3,
+};
+#define WORKQ_THREADREQ_FLAG_NOEMERGENCY 0x1
+
/*
* Increment each time new reserved slots are used. When the pthread
*/
#define PTHREAD_FUNCTIONS_TABLE_VERSION 1
-typedef struct pthread_functions_s {
+typedef const struct pthread_functions_s {
int version;
/* internal calls, kernel core -> kext */
void (*pthread_init)(void);
- int (*fill_procworkqueue)(proc_t p, struct proc_workqueueinfo * pwqinfo);
- void (*workqueue_init_lock)(proc_t p);
- void (*workqueue_destroy_lock)(proc_t p);
+ int (*fill_procworkqueue)(proc_t p, void* pwqinfo);
+
+ void (*__unused1)(void);
+ void (*__unused2)(void);
+
void (*workqueue_exit)(struct proc *p);
void (*workqueue_mark_exiting)(struct proc *p);
void (*workqueue_thread_yielded)(void);
/* Resolve a pthread_priority_t to a QoS/relative pri */
integer_t (*thread_qos_from_pthread_priority)(unsigned long pthread_priority, unsigned long *flags);
+ /* try to get wq flags in debugger context */
+ uint32_t (*get_pwq_state_kdp)(proc_t p);
+
+ void (*__unused3)(void);
+ unsigned long (*pthread_priority_canonicalize2)(unsigned long pthread_priority, boolean_t propagation);
+
+ /* Returns true on success, false on mismatch */
+ boolean_t (*workq_thread_has_been_unbound)(thread_t th, int qos_class);
+
+ void (*pthread_find_owner)(thread_t thread, struct stackshot_thread_waitinfo *waitinfo);
+ void *(*pthread_get_thread_kwq)(thread_t thread);
+
+ /*
+ * Submits a threadreq to the workq system.
+ *
+ * If type is WORKQ_THREADREQ_KEVENT, the semantics are similar to a call
+ * to workq_reqthreads and the kevent bind function will be called to
+ * indicate the thread fulfilling the request. The req argument is ignored.
+ *
+ * If type is WORKQ_THREADREQ_WORKLOOP, The req argument should point to
+ * allocated memory of at least the sizeof(workq_threadreq_t). That memory
+ * is lent to the workq system until workloop_fulfill_threadreq is called
+ * and passed the pointer, at which point it may be freed.
+ *
+ * The properties of the request are passed in the (pthread) priority and flags arguments.
+ *
+ * Will return zero upon success or an error value on failure. An error of
+ * ENOTSUP means the type argument was not understood.
+ */
+ int (*workq_threadreq)(struct proc *p, workq_threadreq_t req,
+ enum workq_threadreq_type, unsigned long priority, int flags);
+
+ /*
+ * Modifies an already submitted thread request.
+ *
+ * If operation is WORKQ_THREADREQ_CHANGE_PRI, arg1 is the new priority and arg2 is unused.
+ *
+ * If operation is WORKQ_THREADREQ_CANCEL, arg1 and arg2 are unused.
+ *
+ * Will return zero upon success or an error value on failure. An error of
+ * ENOTSUP means the operation argument was not understood.
+ */
+ int (*workq_threadreq_modify)(struct proc *t, workq_threadreq_t req,
+ enum workq_threadreq_op operation,
+ unsigned long arg1, unsigned long arg2);
+
/* padding for future */
- void* _pad[95];
-} *pthread_functions_t;
+ void * _pad[87];
+} * pthread_functions_t;
-typedef struct pthread_callbacks_s {
+typedef const struct pthread_callbacks_s {
int version;
/* config information */
void (*proc_set_wqthread)(struct proc *t, user_addr_t addr);
int (*proc_get_pthsize)(struct proc *t);
void (*proc_set_pthsize)(struct proc *t, int size);
- user_addr_t (*proc_get_targconc)(struct proc *t);
- void (*proc_set_targconc)(struct proc *t, user_addr_t addr);
+#if defined(__arm64__)
+ unsigned __int128 (*atomic_fetch_add_128_relaxed)(_Atomic unsigned __int128 *ptr,
+ unsigned __int128 value);
+ unsigned __int128 (*atomic_load_128_relaxed)(_Atomic unsigned __int128 *ptr);
+#else
+ void *unused_was_proc_get_targconc;
+ void *unused_was_proc_set_targconc;
+#endif
uint64_t (*proc_get_dispatchqueue_offset)(struct proc *t);
void (*proc_set_dispatchqueue_offset)(struct proc *t, uint64_t offset);
- lck_spin_t* (*proc_get_wqlockptr)(struct proc *t);
- boolean_t* (*proc_get_wqinitingptr)(struct proc *t);
+ void *unused_was_proc_get_wqlockptr;
+ void *unused_was_proc_get_wqinitingptr;
void* (*proc_get_wqptr)(struct proc *t);
void (*proc_set_wqptr)(struct proc *t, void* ptr);
- int (*proc_get_wqsize)(struct proc *t);
- void (*proc_set_wqsize)(struct proc *t, int sz);
+ void *unused_was_proc_get_wqsize;
+ void *unused_was_proc_set_wqsize;
void (*proc_lock)(struct proc *t);
void (*proc_unlock)(struct proc *t);
task_t (*proc_get_task)(struct proc *t);
/* wq functions */
kern_return_t (*thread_set_wq_state32)(thread_t thread, thread_state_t state);
+#if !defined(__arm__)
kern_return_t (*thread_set_wq_state64)(thread_t thread, thread_state_t state);
+#endif
/* sched_prim.h */
- void (*thread_exception_return)();
- void (*thread_bootstrap_return)();
+ void (*thread_exception_return)(void);
+ void (*thread_bootstrap_return)(void);
/* kern/clock.h */
void (*absolutetime_to_microtime)(uint64_t abstime, clock_sec_t *secs, clock_usec_t *microsecs);
- /* osfmk/kern/task.h */
- int (*proc_restore_workq_bgthreadpolicy)(thread_t t);
- int (*proc_apply_workq_bgthreadpolicy)(thread_t t);
+ kern_return_t (*thread_set_workq_pri)(thread_t thread, integer_t priority, integer_t policy);
+ kern_return_t (*thread_set_workq_qos)(thread_t thread, int qos_tier, int relprio);
/* osfmk/kern/thread.h */
struct uthread* (*get_bsdthread_info)(thread_t th);
/* osfmk/<arch>/machine_routines.h */
int (*ml_get_max_cpus)(void);
+ #if defined(__arm__)
+ uint32_t (*map_is_1gb)(vm_map_t);
+ #endif
/* <rdar://problem/12809089> xnu: struct proc p_dispatchqueue_serialno_offset additions */
uint64_t (*proc_get_dispatchqueue_serialno_offset)(struct proc *p);
void (*proc_set_dispatchqueue_serialno_offset)(struct proc *p, uint64_t offset);
- user_addr_t (*proc_get_stack_addr_hint)(struct proc *p);
- void (*proc_set_stack_addr_hint)(struct proc *p, user_addr_t stack_addr_hint);
+ int (*proc_usynch_thread_qos_add_override_for_resource_check_owner)(thread_t thread, int override_qos, boolean_t first_override_for_resource,
+ user_addr_t resource, int resource_type, user_addr_t user_lock_addr, mach_port_name_t user_lock_owner);
+ void *unused_was_proc_set_stack_addr_hint;
uint32_t (*proc_get_pthread_tsd_offset)(struct proc *p);
void (*proc_set_pthread_tsd_offset)(struct proc *p, uint32_t pthread_tsd_offset);
kern_return_t (*thread_set_tsd_base)(thread_t thread, mach_vm_offset_t tsd_base);
int (*proc_usynch_get_requested_thread_qos)(struct uthread *);
- boolean_t (*proc_usynch_thread_qos_add_override)(struct uthread *, uint64_t tid, int override_qos, boolean_t first_override_for_resource);
- boolean_t (*proc_usynch_thread_qos_remove_override)(struct uthread *, uint64_t tid);
+ uint64_t (*proc_get_mach_thread_self_tsd_offset)(struct proc *p);
+ void (*proc_set_mach_thread_self_tsd_offset)(struct proc *p, uint64_t mach_thread_self_tsd_offset);
kern_return_t (*thread_policy_get)(thread_t t, thread_policy_flavor_t flavor, thread_policy_t info, mach_msg_type_number_t *count, boolean_t *get_default);
boolean_t (*qos_main_thread_active)(void);
boolean_t (*proc_usynch_thread_qos_remove_override_for_resource)(task_t task, struct uthread *, uint64_t tid, user_addr_t resource, int resource_type);
boolean_t (*proc_usynch_thread_qos_reset_override_for_resource)(task_t task, struct uthread *, uint64_t tid, user_addr_t resource, int resource_type);
+ boolean_t (*proc_init_wqptr_or_wait)(proc_t proc);
+
+ uint16_t (*thread_set_tag)(thread_t thread, uint16_t tag);
+ uint16_t (*thread_get_tag)(thread_t thread);
+
+ int (*proc_usynch_thread_qos_squash_override_for_resource)(thread_t thread, user_addr_t resource, int resource_type);
+ int (*task_get_default_manager_qos)(task_t task);
+
+ int (*thread_create_workq_waiting)(task_t task, thread_continue_t thread_return, event_t event, thread_t *new_thread);
+
+ user_addr_t (*proc_get_stack_addr_hint)(struct proc *p);
+ void (*proc_set_stack_addr_hint)(struct proc *p, user_addr_t stack_addr_hint);
+
+ uint64_t (*proc_get_return_to_kernel_offset)(struct proc *t);
+ void (*proc_set_return_to_kernel_offset)(struct proc *t, uint64_t offset);
+
+ /* indicates call is being made synchronously with workq_threadreq call */
+# define WORKLOOP_FULFILL_THREADREQ_SYNC 0x1
+# define WORKLOOP_FULFILL_THREADREQ_CANCEL 0x2
+ int (*workloop_fulfill_threadreq)(struct proc *p, workq_threadreq_t req, thread_t thread, int flags);
+ void (*thread_will_park_or_terminate)(thread_t thread);
+
+ /* For getting maximum parallelism for a given QoS */
+ uint32_t (*qos_max_parallelism)(int qos, uint64_t options);
+
+ /* proc_internal.h: struct proc user_stack accessor */
+ user_addr_t (*proc_get_user_stack)(struct proc *p);
+ void (*proc_set_user_stack)(struct proc *p, user_addr_t user_stack);
+
/* padding for future */
- void* _pad[84];
+ void* _pad[69];
} *pthread_callbacks_t;
projects / apple / xnu.git / blobdiff