[apple/libpthread.git] / kern / kern_internal.h

/*
 * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#ifndef _SYS_PTHREAD_INTERNAL_H_
#define _SYS_PTHREAD_INTERNAL_H_

#include <pthread/bsdthread_private.h>
#include <pthread/priority_private.h>
#include <pthread/workqueue_syscalls.h>

#ifdef KERNEL
struct ksyn_waitq_element;
#include <stdatomic.h>
#include <kern/thread_call.h>
#include <kern/kcdata.h>
#include <sys/pthread_shims.h>
#include <sys/queue.h>
#include <sys/proc_info.h>
#endif // KERNEL

#include "kern/synch_internal.h"
#include "kern/kern_trace.h"
#include "pthread/qos.h"
#include "pthread/qos_private.h"

/* pthread userspace SPI feature checking, these constants are returned from bsdthread_register,
 * as a bitmask, to inform userspace of the supported feature set. Old releases of OS X return
 * from this call either zero or -1, allowing us to return a positive number for feature bits.
 */
#define PTHREAD_FEATURE_DISPATCHFUNC	0x01		/* same as WQOPS_QUEUE_NEWSPISUPP, checks for dispatch function support */
#define PTHREAD_FEATURE_FINEPRIO		0x02		/* are fine grained prioirities available */
#define PTHREAD_FEATURE_BSDTHREADCTL	0x04		/* is the bsdthread_ctl syscall available */
#define PTHREAD_FEATURE_SETSELF			0x08		/* is the BSDTHREAD_CTL_SET_SELF command of bsdthread_ctl available */
#define PTHREAD_FEATURE_QOS_MAINTENANCE	0x10		/* is QOS_CLASS_MAINTENANCE available */
#define PTHREAD_FEATURE_RESERVED		0x20		/* burnt, shipped in OSX 10.11 & iOS 9 with partial kevent delivery support */
#define PTHREAD_FEATURE_KEVENT          0x40		/* supports direct kevent delivery */
#define PTHREAD_FEATURE_WORKLOOP          0x80		/* supports workloops */
#define PTHREAD_FEATURE_QOS_DEFAULT		0x40000000	/* the kernel supports QOS_CLASS_DEFAULT */

/* userspace <-> kernel registration struct, for passing data to/from the kext during main thread init. */
struct _pthread_registration_data {
	/*
	 * version == sizeof(struct _pthread_registration_data)
	 *
	 * The structure can only grow, so we use its size as the version.
	 * Userspace initializes this to the size of its structure and the kext
	 * will copy out the version that was actually consumed.
	 *
	 * n.b. you must make sure the size of this structure isn't LP64-dependent
	 */
	uint64_t version;

	uint64_t dispatch_queue_offset; /* copy-in */
	uint64_t /* pthread_priority_t */ main_qos; /* copy-out */
	uint32_t tsd_offset; /* copy-in */
	uint32_t return_to_kernel_offset; /* copy-in */
	uint32_t mach_thread_self_offset; /* copy-in */
	mach_vm_address_t stack_addr_hint; /* copy-out */
#define _PTHREAD_REG_DEFAULT_POLICY_MASK 0xff
#define _PTHREAD_REG_DEFAULT_USE_ULOCK 0x100
#define _PTHREAD_REG_DEFAULT_USE_ADAPTIVE_SPIN 0x200
	uint32_t mutex_default_policy; /* copy-out */
	uint32_t joinable_offset_bits; /* copy-in */
} __attribute__ ((packed));

/*
 * "error" flags returned by fail condvar syscalls
 */
#define ECVCLEARED	0x100
#define ECVPREPOST	0x200

#ifdef KERNEL

/* The set of features, from the feature bits above, that we support. */
#define PTHREAD_FEATURE_SUPPORTED	( \
	PTHREAD_FEATURE_DISPATCHFUNC | \
	PTHREAD_FEATURE_FINEPRIO | \
	PTHREAD_FEATURE_BSDTHREADCTL | \
	PTHREAD_FEATURE_SETSELF | \
	PTHREAD_FEATURE_QOS_MAINTENANCE | \
	PTHREAD_FEATURE_QOS_DEFAULT | \
	PTHREAD_FEATURE_KEVENT | \
	PTHREAD_FEATURE_WORKLOOP )

extern pthread_callbacks_t pthread_kern;

struct ksyn_waitq_element {
	TAILQ_ENTRY(ksyn_waitq_element) kwe_list;	/* link to other list members */
	void *          kwe_kwqqueue;            	/* queue blocked on */
	thread_t        kwe_thread;
	uint16_t        kwe_state;			/* state */
	uint16_t        kwe_flags;
	uint32_t        kwe_lockseq;			/* the sequence of the entry */
	uint32_t	kwe_count;			/* upper bound on number of matches still pending */
	uint32_t 	kwe_psynchretval;		/* thread retval */
	void		*kwe_uth;			/* uthread */
};
typedef struct ksyn_waitq_element * ksyn_waitq_element_t;

#define PTH_DEFAULT_STACKSIZE 512*1024
#define MAX_PTHREAD_SIZE 64*1024

/* exported from the kernel but not present in any headers. */
extern thread_t port_name_to_thread(mach_port_name_t port_name);

/* function declarations for pthread_kext.c */
void pthread_init(void);
void psynch_zoneinit(void);
void _pth_proc_hashinit(proc_t p);
void _pth_proc_hashdelete(proc_t p);
void pth_global_hashinit(void);
void psynch_wq_cleanup(void*, void*);

void _pthread_init(void);
int _fill_procworkqueue(proc_t p, struct proc_workqueueinfo * pwqinfo);
uint32_t _get_pwq_state_kdp(proc_t p);
void _workqueue_exit(struct proc *p);
void _workqueue_mark_exiting(struct proc *p);
void _workqueue_thread_yielded(void);
sched_call_t _workqueue_get_sched_callback(void);

int _bsdthread_create(struct proc *p, user_addr_t user_func, user_addr_t user_funcarg, user_addr_t user_stack, user_addr_t user_pthread, uint32_t flags, user_addr_t *retval);
int _bsdthread_register(struct proc *p, user_addr_t threadstart, user_addr_t wqthread, int pthsize, user_addr_t dummy_value, user_addr_t targetconc_ptr, uint64_t dispatchqueue_offset, int32_t *retval);
int _bsdthread_terminate(struct proc *p, user_addr_t stackaddr, size_t size, uint32_t kthport, uint32_t sem, int32_t *retval);
int _bsdthread_ctl_set_qos(struct proc *p, user_addr_t cmd, mach_port_name_t kport, user_addr_t tsd_priority_addr, user_addr_t arg3, int *retval);
int _bsdthread_ctl_set_self(struct proc *p, user_addr_t cmd, pthread_priority_t priority, mach_port_name_t voucher, _pthread_set_flags_t flags, int *retval);
int _bsdthread_ctl_qos_override_start(struct proc *p, user_addr_t cmd, mach_port_name_t kport, pthread_priority_t priority, user_addr_t resource, int *retval);
int _bsdthread_ctl_qos_override_end(struct proc *p, user_addr_t cmd, mach_port_name_t kport, user_addr_t resource, user_addr_t arg3, int *retval);
int _bsdthread_ctl_qos_override_dispatch(struct proc __unused *p, user_addr_t __unused cmd, mach_port_name_t kport, pthread_priority_t priority, user_addr_t arg3, int __unused *retval);
int _bsdthread_ctl_qos_override_reset(struct proc __unused *p, user_addr_t __unused cmd, user_addr_t arg1, user_addr_t arg2, user_addr_t arg3, int __unused *retval);
int _bsdthread_ctl_qos_dispatch_asynchronous_override_add(struct proc __unused *p, user_addr_t __unused cmd, mach_port_name_t kport, pthread_priority_t priority, user_addr_t resource, int __unused *retval);
int _bsdthread_ctl_qos_dispatch_asynchronous_override_reset(struct proc __unused *p, user_addr_t __unused cmd, int reset_all, user_addr_t resource, user_addr_t arg3, int __unused *retval);
int _bsdthread_ctl(struct proc *p, user_addr_t cmd, user_addr_t arg1, user_addr_t arg2, user_addr_t arg3, int *retval);
int _thread_selfid(__unused struct proc *p, uint64_t *retval);
int _workq_kernreturn(struct proc *p, int options, user_addr_t item, int arg2, int arg3, int32_t *retval);
int _workq_open(struct proc *p, int32_t *retval);

int _psynch_mutexwait(proc_t p, user_addr_t mutex,  uint32_t mgen, uint32_t  ugen, uint64_t tid, uint32_t flags, uint32_t * retval);
int _psynch_mutexdrop(proc_t p, user_addr_t mutex,  uint32_t mgen, uint32_t  ugen, uint64_t tid, uint32_t flags, uint32_t * retval);
int _psynch_cvbroad(proc_t p, user_addr_t cv, uint64_t cvlsgen, uint64_t cvudgen, uint32_t flags, user_addr_t mutex,  uint64_t mugen, uint64_t tid, uint32_t *retval);
int _psynch_cvsignal(proc_t p, user_addr_t cv, uint64_t cvlsgen, uint32_t cvugen, int thread_port, user_addr_t mutex,  uint64_t mugen, uint64_t tid, uint32_t flags, uint32_t * retval);
int _psynch_cvwait(proc_t p, user_addr_t cv, uint64_t cvlsgen, uint32_t cvugen, user_addr_t mutex,  uint64_t mugen, uint32_t flags, int64_t sec, uint32_t nsec, uint32_t * retval);
int _psynch_cvclrprepost(proc_t p, user_addr_t cv, uint32_t cvgen, uint32_t cvugen, uint32_t cvsgen, uint32_t prepocnt, uint32_t preposeq, uint32_t flags, int *retval);
int _psynch_rw_longrdlock(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t * retval);
int _psynch_rw_rdlock(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t *retval);
int _psynch_rw_unlock(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t *retval);
int _psynch_rw_wrlock(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t *retval);
int _psynch_rw_yieldwrlock(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t *retval);

void _pthread_find_owner(thread_t thread, struct stackshot_thread_waitinfo *waitinfo);
void * _pthread_get_thread_kwq(thread_t thread);

extern lck_grp_attr_t *pthread_lck_grp_attr;
extern lck_grp_t *pthread_lck_grp;
extern lck_attr_t *pthread_lck_attr;
extern lck_mtx_t *pthread_list_mlock;
extern thread_call_t psynch_thcall;

struct uthread* current_uthread(void);

int
workq_create_threadstack(proc_t p, vm_map_t vmap, mach_vm_offset_t *out_addr);

int
workq_destroy_threadstack(proc_t p, vm_map_t vmap, mach_vm_offset_t stackaddr);

void
workq_setup_thread(proc_t p, thread_t th, vm_map_t map, user_addr_t stackaddr,
		mach_port_name_t kport, int th_qos, int setup_flags, int upcall_flags);

int
workq_handle_stack_events(proc_t p, thread_t th, vm_map_t map,
		user_addr_t stackaddr, mach_port_name_t kport,
		user_addr_t events, int nevents, int upcall_flags);

void
workq_markfree_threadstack(proc_t p, thread_t th, vm_map_t vmap,
		user_addr_t stackaddr);

#endif // KERNEL

// magical `nkevents` values for _pthread_wqthread
#define WORKQ_EXIT_THREAD_NKEVENT   (-1)

#endif /* _SYS_PTHREAD_INTERNAL_H_ */
Commit	Line	Data
f1a1da6c A	1	/*
	2	* Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
a0619f9c	5	*
f1a1da6c A	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
a0619f9c	14	*
f1a1da6c A	15	* Please obtain a copy of the License at
f1a1da6c A	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
a0619f9c	17	*
f1a1da6c A	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
a0619f9c	25	*
f1a1da6c A	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29	#ifndef _SYS_PTHREAD_INTERNAL_H_
	30	#define _SYS_PTHREAD_INTERNAL_H_
	31
214d78a2 A	32	#include <pthread/bsdthread_private.h>
	33	#include <pthread/priority_private.h>
	34	#include <pthread/workqueue_syscalls.h>
	35
f1a1da6c	36	#ifdef KERNEL
214d78a2	37	struct ksyn_waitq_element;
a0619f9c	38	#include <stdatomic.h>
f1a1da6c	39	#include <kern/thread_call.h>
a0619f9c	40	#include <kern/kcdata.h>
f1a1da6c A	41	#include <sys/pthread_shims.h>
f1a1da6c A	42	#include <sys/queue.h>
a0619f9c	43	#include <sys/proc_info.h>
a0619f9c	44	#endif // KERNEL
f1a1da6c A	45
f1a1da6c A	46	#include "kern/synch_internal.h"
f1a1da6c A	47	#include "kern/kern_trace.h"
f1a1da6c A	48	#include "pthread/qos.h"
c1f56ec9	49	#include "pthread/qos_private.h"
f1a1da6c A	50
	51	/* pthread userspace SPI feature checking, these constants are returned from bsdthread_register,
	52	* as a bitmask, to inform userspace of the supported feature set. Old releases of OS X return
	53	* from this call either zero or -1, allowing us to return a positive number for feature bits.
	54	*/
	55	#define PTHREAD_FEATURE_DISPATCHFUNC 0x01 /* same as WQOPS_QUEUE_NEWSPISUPP, checks for dispatch function support */
	56	#define PTHREAD_FEATURE_FINEPRIO 0x02 /* are fine grained prioirities available */
	57	#define PTHREAD_FEATURE_BSDTHREADCTL 0x04 /* is the bsdthread_ctl syscall available */
	58	#define PTHREAD_FEATURE_SETSELF 0x08 /* is the BSDTHREAD_CTL_SET_SELF command of bsdthread_ctl available */
	59	#define PTHREAD_FEATURE_QOS_MAINTENANCE 0x10 /* is QOS_CLASS_MAINTENANCE available */
2546420a A	60	#define PTHREAD_FEATURE_RESERVED 0x20 /* burnt, shipped in OSX 10.11 & iOS 9 with partial kevent delivery support */
2546420a A	61	#define PTHREAD_FEATURE_KEVENT 0x40 /* supports direct kevent delivery */
a0619f9c	62	#define PTHREAD_FEATURE_WORKLOOP 0x80 /* supports workloops */
f1a1da6c A	63	#define PTHREAD_FEATURE_QOS_DEFAULT 0x40000000 /* the kernel supports QOS_CLASS_DEFAULT */
f1a1da6c A	64
f1a1da6c A	65	/* userspace <-> kernel registration struct, for passing data to/from the kext during main thread init. */
f1a1da6c A	66	struct _pthread_registration_data {
2546420a A	67	/*
	68	* version == sizeof(struct _pthread_registration_data)
	69	*
	70	* The structure can only grow, so we use its size as the version.
	71	* Userspace initializes this to the size of its structure and the kext
	72	* will copy out the version that was actually consumed.
	73	*
	74	* n.b. you must make sure the size of this structure isn't LP64-dependent
	75	*/
	76	uint64_t version;
	77
f1a1da6c	78	uint64_t dispatch_queue_offset; /* copy-in */
2546420a A	79	uint64_t /* pthread_priority_t / main_qos; / copy-out */
2546420a A	80	uint32_t tsd_offset; /* copy-in */
a0619f9c A	81	uint32_t return_to_kernel_offset; /* copy-in */
a0619f9c A	82	uint32_t mach_thread_self_offset; /* copy-in */
214d78a2	83	mach_vm_address_t stack_addr_hint; /* copy-out */
c1f56ec9 A	84	#define _PTHREAD_REG_DEFAULT_POLICY_MASK 0xff
	85	#define _PTHREAD_REG_DEFAULT_USE_ULOCK 0x100
	86	#define _PTHREAD_REG_DEFAULT_USE_ADAPTIVE_SPIN 0x200
76b7b9a2	87	uint32_t mutex_default_policy; /* copy-out */
c1f56ec9	88	uint32_t joinable_offset_bits; /* copy-in */
2546420a	89	} __attribute__ ((packed));
f1a1da6c	90
214d78a2 A	91	/*
	92	* "error" flags returned by fail condvar syscalls
	93	*/
	94	#define ECVCLEARED 0x100
	95	#define ECVPREPOST 0x200
	96
f1a1da6c A	97	#ifdef KERNEL
	98
	99	/* The set of features, from the feature bits above, that we support. */
	100	#define PTHREAD_FEATURE_SUPPORTED ( \
	101	PTHREAD_FEATURE_DISPATCHFUNC \| \
	102	PTHREAD_FEATURE_FINEPRIO \| \
	103	PTHREAD_FEATURE_BSDTHREADCTL \| \
	104	PTHREAD_FEATURE_SETSELF \| \
	105	PTHREAD_FEATURE_QOS_MAINTENANCE \| \
964d3577	106	PTHREAD_FEATURE_QOS_DEFAULT \| \
a0619f9c A	107	PTHREAD_FEATURE_KEVENT \| \
a0619f9c A	108	PTHREAD_FEATURE_WORKLOOP )
f1a1da6c A	109
	110	extern pthread_callbacks_t pthread_kern;
	111
	112	struct ksyn_waitq_element {
	113	TAILQ_ENTRY(ksyn_waitq_element) kwe_list; /* link to other list members */
	114	void * kwe_kwqqueue; /* queue blocked on */
214d78a2 A	115	thread_t kwe_thread;
	116	uint16_t kwe_state; /* state */
	117	uint16_t kwe_flags;
f1a1da6c A	118	uint32_t kwe_lockseq; /* the sequence of the entry */
	119	uint32_t kwe_count; /* upper bound on number of matches still pending */
	120	uint32_t kwe_psynchretval; /* thread retval */
	121	void kwe_uth; / uthread */
f1a1da6c A	122	};
	123	typedef struct ksyn_waitq_element * ksyn_waitq_element_t;
	124
f1a1da6c A	125	#define PTH_DEFAULT_STACKSIZE 512*1024
	126	#define MAX_PTHREAD_SIZE 64*1024
	127
	128	/* exported from the kernel but not present in any headers. */
	129	extern thread_t port_name_to_thread(mach_port_name_t port_name);
	130
	131	/* function declarations for pthread_kext.c */
	132	void pthread_init(void);
	133	void psynch_zoneinit(void);
	134	void _pth_proc_hashinit(proc_t p);
	135	void _pth_proc_hashdelete(proc_t p);
	136	void pth_global_hashinit(void);
	137	void psynch_wq_cleanup(void, void);
	138
	139	void _pthread_init(void);
	140	int _fill_procworkqueue(proc_t p, struct proc_workqueueinfo * pwqinfo);
2546420a	141	uint32_t _get_pwq_state_kdp(proc_t p);
f1a1da6c A	142	void _workqueue_exit(struct proc *p);
	143	void _workqueue_mark_exiting(struct proc *p);
	144	void _workqueue_thread_yielded(void);
	145	sched_call_t _workqueue_get_sched_callback(void);
	146
	147	int _bsdthread_create(struct proc p, user_addr_t user_func, user_addr_t user_funcarg, user_addr_t user_stack, user_addr_t user_pthread, uint32_t flags, user_addr_t retval);
	148	int _bsdthread_register(struct proc p, user_addr_t threadstart, user_addr_t wqthread, int pthsize, user_addr_t dummy_value, user_addr_t targetconc_ptr, uint64_t dispatchqueue_offset, int32_t retval);
	149	int _bsdthread_terminate(struct proc p, user_addr_t stackaddr, size_t size, uint32_t kthport, uint32_t sem, int32_t retval);
	150	int _bsdthread_ctl_set_qos(struct proc p, user_addr_t cmd, mach_port_name_t kport, user_addr_t tsd_priority_addr, user_addr_t arg3, int retval);
	151	int _bsdthread_ctl_set_self(struct proc p, user_addr_t cmd, pthread_priority_t priority, mach_port_name_t voucher, _pthread_set_flags_t flags, int retval);
215aeb03 A	152	int _bsdthread_ctl_qos_override_start(struct proc p, user_addr_t cmd, mach_port_name_t kport, pthread_priority_t priority, user_addr_t resource, int retval);
215aeb03 A	153	int _bsdthread_ctl_qos_override_end(struct proc p, user_addr_t cmd, mach_port_name_t kport, user_addr_t resource, user_addr_t arg3, int retval);
f1a1da6c A	154	int _bsdthread_ctl_qos_override_dispatch(struct proc __unused p, user_addr_t __unused cmd, mach_port_name_t kport, pthread_priority_t priority, user_addr_t arg3, int __unused retval);
f1a1da6c A	155	int _bsdthread_ctl_qos_override_reset(struct proc __unused p, user_addr_t __unused cmd, user_addr_t arg1, user_addr_t arg2, user_addr_t arg3, int __unused retval);
215aeb03 A	156	int _bsdthread_ctl_qos_dispatch_asynchronous_override_add(struct proc __unused p, user_addr_t __unused cmd, mach_port_name_t kport, pthread_priority_t priority, user_addr_t resource, int __unused retval);
215aeb03 A	157	int _bsdthread_ctl_qos_dispatch_asynchronous_override_reset(struct proc __unused p, user_addr_t __unused cmd, int reset_all, user_addr_t resource, user_addr_t arg3, int __unused retval);
f1a1da6c A	158	int _bsdthread_ctl(struct proc p, user_addr_t cmd, user_addr_t arg1, user_addr_t arg2, user_addr_t arg3, int retval);
	159	int _thread_selfid(__unused struct proc p, uint64_t retval);
	160	int _workq_kernreturn(struct proc p, int options, user_addr_t item, int arg2, int arg3, int32_t retval);
	161	int _workq_open(struct proc p, int32_t retval);
	162
	163	int _psynch_mutexwait(proc_t p, user_addr_t mutex, uint32_t mgen, uint32_t ugen, uint64_t tid, uint32_t flags, uint32_t * retval);
	164	int _psynch_mutexdrop(proc_t p, user_addr_t mutex, uint32_t mgen, uint32_t ugen, uint64_t tid, uint32_t flags, uint32_t * retval);
	165	int _psynch_cvbroad(proc_t p, user_addr_t cv, uint64_t cvlsgen, uint64_t cvudgen, uint32_t flags, user_addr_t mutex, uint64_t mugen, uint64_t tid, uint32_t *retval);
	166	int _psynch_cvsignal(proc_t p, user_addr_t cv, uint64_t cvlsgen, uint32_t cvugen, int thread_port, user_addr_t mutex, uint64_t mugen, uint64_t tid, uint32_t flags, uint32_t * retval);
	167	int _psynch_cvwait(proc_t p, user_addr_t cv, uint64_t cvlsgen, uint32_t cvugen, user_addr_t mutex, uint64_t mugen, uint32_t flags, int64_t sec, uint32_t nsec, uint32_t * retval);
	168	int _psynch_cvclrprepost(proc_t p, user_addr_t cv, uint32_t cvgen, uint32_t cvugen, uint32_t cvsgen, uint32_t prepocnt, uint32_t preposeq, uint32_t flags, int *retval);
	169	int _psynch_rw_longrdlock(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t * retval);
	170	int _psynch_rw_rdlock(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t *retval);
	171	int _psynch_rw_unlock(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t *retval);
	172	int _psynch_rw_wrlock(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t *retval);
	173	int _psynch_rw_yieldwrlock(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t *retval);
	174
010efe49 A	175	void _pthread_find_owner(thread_t thread, struct stackshot_thread_waitinfo *waitinfo);
	176	void * _pthread_get_thread_kwq(thread_t thread);
	177
f1a1da6c A	178	extern lck_grp_attr_t *pthread_lck_grp_attr;
	179	extern lck_grp_t *pthread_lck_grp;
	180	extern lck_attr_t *pthread_lck_attr;
	181	extern lck_mtx_t *pthread_list_mlock;
	182	extern thread_call_t psynch_thcall;
	183
	184	struct uthread* current_uthread(void);
	185
214d78a2 A	186	int
214d78a2 A	187	workq_create_threadstack(proc_t p, vm_map_t vmap, mach_vm_offset_t *out_addr);
964d3577	188
214d78a2 A	189	int
214d78a2 A	190	workq_destroy_threadstack(proc_t p, vm_map_t vmap, mach_vm_offset_t stackaddr);
964d3577	191
214d78a2 A	192	void
	193	workq_setup_thread(proc_t p, thread_t th, vm_map_t map, user_addr_t stackaddr,
	194	mach_port_name_t kport, int th_qos, int setup_flags, int upcall_flags);
a0619f9c	195
214d78a2 A	196	int
	197	workq_handle_stack_events(proc_t p, thread_t th, vm_map_t map,
	198	user_addr_t stackaddr, mach_port_name_t kport,
	199	user_addr_t events, int nevents, int upcall_flags);
a0619f9c	200
214d78a2 A	201	void
	202	workq_markfree_threadstack(proc_t p, thread_t th, vm_map_t vmap,
	203	user_addr_t stackaddr);
a0619f9c	204
f1a1da6c A	205	#endif // KERNEL
f1a1da6c A	206
c1f56ec9 A	207	// magical `nkevents` values for _pthread_wqthread
	208	#define WORKQ_EXIT_THREAD_NKEVENT (-1)
	209
f1a1da6c A	210	#endif /* _SYS_PTHREAD_INTERNAL_H_ */
f1a1da6c A	211