kern/kern_internal.h

   1 /*
   2  * Copyright (c) 2000-2003 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #ifndef _SYS_PTHREAD_INTERNAL_H_
  30 #define _SYS_PTHREAD_INTERNAL_H_
  31
  32 #include <pthread/bsdthread_private.h>
  33 #include <pthread/priority_private.h>
  34 #include <pthread/workqueue_syscalls.h>
  35
  36 #ifdef KERNEL
  37 struct ksyn_waitq_element;
  38 #include <stdatomic.h>
  39 #include <kern/thread_call.h>
  40 #include <kern/kcdata.h>
  41 #include <sys/pthread_shims.h>
  42 #include <sys/queue.h>
  43 #include <sys/proc_info.h>
  44 #endif // KERNEL
  45
  46 #include "kern/synch_internal.h"
  47 #include "kern/kern_trace.h"
  48 #include "pthread/qos.h"
  49 #include "pthread/qos_private.h"
  50
  51 /* pthread userspace SPI feature checking, these constants are returned from bsdthread_register,
  52  * as a bitmask, to inform userspace of the supported feature set. Old releases of OS X return
  53  * from this call either zero or -1, allowing us to return a positive number for feature bits.
  54  */
  55 #define PTHREAD_FEATURE_DISPATCHFUNC    0x01            /* same as WQOPS_QUEUE_NEWSPISUPP, checks for dispatch function support */
  56 #define PTHREAD_FEATURE_FINEPRIO                0x02            /* are fine grained prioirities available */
  57 #define PTHREAD_FEATURE_BSDTHREADCTL    0x04            /* is the bsdthread_ctl syscall available */
  58 #define PTHREAD_FEATURE_SETSELF                 0x08            /* is the BSDTHREAD_CTL_SET_SELF command of bsdthread_ctl available */
  59 #define PTHREAD_FEATURE_QOS_MAINTENANCE 0x10            /* is QOS_CLASS_MAINTENANCE available */
  60 #define PTHREAD_FEATURE_RESERVED                0x20            /* burnt, shipped in OSX 10.11 & iOS 9 with partial kevent delivery support */
  61 #define PTHREAD_FEATURE_KEVENT          0x40            /* supports direct kevent delivery */
  62 #define PTHREAD_FEATURE_WORKLOOP          0x80          /* supports workloops */
  63 #define PTHREAD_FEATURE_QOS_DEFAULT             0x40000000      /* the kernel supports QOS_CLASS_DEFAULT */
  64
  65 /* userspace <-> kernel registration struct, for passing data to/from the kext during main thread init. */
  66 struct _pthread_registration_data {
  67         /*
  68          * version == sizeof(struct _pthread_registration_data)
  69          *
  70          * The structure can only grow, so we use its size as the version.
  71          * Userspace initializes this to the size of its structure and the kext
  72          * will copy out the version that was actually consumed.
  73          *
  74          * n.b. you must make sure the size of this structure isn't LP64-dependent
  75          */
  76         uint64_t version;
  77
  78         uint64_t dispatch_queue_offset; /* copy-in */
  79         uint64_t /* pthread_priority_t */ main_qos; /* copy-out */
  80         uint32_t tsd_offset; /* copy-in */
  81         uint32_t return_to_kernel_offset; /* copy-in */
  82         uint32_t mach_thread_self_offset; /* copy-in */
  83         mach_vm_address_t stack_addr_hint; /* copy-out */
  84 #define _PTHREAD_REG_DEFAULT_POLICY_MASK 0xff
  85 #define _PTHREAD_REG_DEFAULT_USE_ULOCK 0x100
  86 #define _PTHREAD_REG_DEFAULT_USE_ADAPTIVE_SPIN 0x200
  87         uint32_t mutex_default_policy; /* copy-out */
  88         uint32_t joinable_offset_bits; /* copy-in */
  89 } __attribute__ ((packed));
  90
  91 /*
  92  * "error" flags returned by fail condvar syscalls
  93  */
  94 #define ECVCLEARED      0x100
  95 #define ECVPREPOST      0x200
  96
  97 #ifdef KERNEL
  98
  99 /* The set of features, from the feature bits above, that we support. */
 100 #define PTHREAD_FEATURE_SUPPORTED       ( \
 101         PTHREAD_FEATURE_DISPATCHFUNC | \
 102         PTHREAD_FEATURE_FINEPRIO | \
 103         PTHREAD_FEATURE_BSDTHREADCTL | \
 104         PTHREAD_FEATURE_SETSELF | \
 105         PTHREAD_FEATURE_QOS_MAINTENANCE | \
 106         PTHREAD_FEATURE_QOS_DEFAULT | \
 107         PTHREAD_FEATURE_KEVENT | \
 108         PTHREAD_FEATURE_WORKLOOP )
 109
 110 extern pthread_callbacks_t pthread_kern;
 111
 112 struct ksyn_waitq_element {
 113         TAILQ_ENTRY(ksyn_waitq_element) kwe_list;       /* link to other list members */
 114         void *          kwe_kwqqueue;                   /* queue blocked on */
 115         thread_t        kwe_thread;
 116         uint16_t        kwe_state;                      /* state */
 117         uint16_t        kwe_flags;
 118         uint32_t        kwe_lockseq;                    /* the sequence of the entry */
 119         uint32_t        kwe_count;                      /* upper bound on number of matches still pending */
 120         uint32_t        kwe_psynchretval;               /* thread retval */
 121         void            *kwe_uth;                       /* uthread */
 122 };
 123 typedef struct ksyn_waitq_element * ksyn_waitq_element_t;
 124
 125 #define PTH_DEFAULT_STACKSIZE 512*1024
 126 #define MAX_PTHREAD_SIZE 64*1024
 127
 128 /* exported from the kernel but not present in any headers. */
 129 extern thread_t port_name_to_thread(mach_port_name_t port_name);
 130
 131 /* function declarations for pthread_kext.c */
 132 void pthread_init(void);
 133 void psynch_zoneinit(void);
 134 void _pth_proc_hashinit(proc_t p);
 135 void _pth_proc_hashdelete(proc_t p);
 136 void pth_global_hashinit(void);
 137 void psynch_wq_cleanup(void*, void*);
 138
 139 void _pthread_init(void);
 140 int _fill_procworkqueue(proc_t p, struct proc_workqueueinfo * pwqinfo);
 141 uint32_t _get_pwq_state_kdp(proc_t p);
 142 void _workqueue_exit(struct proc *p);
 143 void _workqueue_mark_exiting(struct proc *p);
 144 void _workqueue_thread_yielded(void);
 145 sched_call_t _workqueue_get_sched_callback(void);
 146
 147 int _bsdthread_create(struct proc *p, user_addr_t user_func, user_addr_t user_funcarg, user_addr_t user_stack, user_addr_t user_pthread, uint32_t flags, user_addr_t *retval);
 148 int _bsdthread_register(struct proc *p, user_addr_t threadstart, user_addr_t wqthread, int pthsize, user_addr_t dummy_value, user_addr_t targetconc_ptr, uint64_t dispatchqueue_offset, int32_t *retval);
 149 int _bsdthread_terminate(struct proc *p, user_addr_t stackaddr, size_t size, uint32_t kthport, uint32_t sem, int32_t *retval);
 150 int _bsdthread_ctl_set_qos(struct proc *p, user_addr_t cmd, mach_port_name_t kport, user_addr_t tsd_priority_addr, user_addr_t arg3, int *retval);
 151 int _bsdthread_ctl_set_self(struct proc *p, user_addr_t cmd, pthread_priority_t priority, mach_port_name_t voucher, _pthread_set_flags_t flags, int *retval);
 152 int _bsdthread_ctl_qos_override_start(struct proc *p, user_addr_t cmd, mach_port_name_t kport, pthread_priority_t priority, user_addr_t resource, int *retval);
 153 int _bsdthread_ctl_qos_override_end(struct proc *p, user_addr_t cmd, mach_port_name_t kport, user_addr_t resource, user_addr_t arg3, int *retval);
 154 int _bsdthread_ctl_qos_override_dispatch(struct proc __unused *p, user_addr_t __unused cmd, mach_port_name_t kport, pthread_priority_t priority, user_addr_t arg3, int __unused *retval);
 155 int _bsdthread_ctl_qos_override_reset(struct proc __unused *p, user_addr_t __unused cmd, user_addr_t arg1, user_addr_t arg2, user_addr_t arg3, int __unused *retval);
 156 int _bsdthread_ctl_qos_dispatch_asynchronous_override_add(struct proc __unused *p, user_addr_t __unused cmd, mach_port_name_t kport, pthread_priority_t priority, user_addr_t resource, int __unused *retval);
 157 int _bsdthread_ctl_qos_dispatch_asynchronous_override_reset(struct proc __unused *p, user_addr_t __unused cmd, int reset_all, user_addr_t resource, user_addr_t arg3, int __unused *retval);
 158 int _bsdthread_ctl(struct proc *p, user_addr_t cmd, user_addr_t arg1, user_addr_t arg2, user_addr_t arg3, int *retval);
 159 int _thread_selfid(__unused struct proc *p, uint64_t *retval);
 160 int _workq_kernreturn(struct proc *p, int options, user_addr_t item, int arg2, int arg3, int32_t *retval);
 161 int _workq_open(struct proc *p, int32_t *retval);
 162
 163 int _psynch_mutexwait(proc_t p, user_addr_t mutex,  uint32_t mgen, uint32_t  ugen, uint64_t tid, uint32_t flags, uint32_t * retval);
 164 int _psynch_mutexdrop(proc_t p, user_addr_t mutex,  uint32_t mgen, uint32_t  ugen, uint64_t tid, uint32_t flags, uint32_t * retval);
 165 int _psynch_cvbroad(proc_t p, user_addr_t cv, uint64_t cvlsgen, uint64_t cvudgen, uint32_t flags, user_addr_t mutex,  uint64_t mugen, uint64_t tid, uint32_t *retval);
 166 int _psynch_cvsignal(proc_t p, user_addr_t cv, uint64_t cvlsgen, uint32_t cvugen, int thread_port, user_addr_t mutex,  uint64_t mugen, uint64_t tid, uint32_t flags, uint32_t * retval);
 167 int _psynch_cvwait(proc_t p, user_addr_t cv, uint64_t cvlsgen, uint32_t cvugen, user_addr_t mutex,  uint64_t mugen, uint32_t flags, int64_t sec, uint32_t nsec, uint32_t * retval);
 168 int _psynch_cvclrprepost(proc_t p, user_addr_t cv, uint32_t cvgen, uint32_t cvugen, uint32_t cvsgen, uint32_t prepocnt, uint32_t preposeq, uint32_t flags, int *retval);
 169 int _psynch_rw_longrdlock(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t * retval);
 170 int _psynch_rw_rdlock(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t *retval);
 171 int _psynch_rw_unlock(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t *retval);
 172 int _psynch_rw_wrlock(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t *retval);
 173 int _psynch_rw_yieldwrlock(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t *retval);
 174
 175 void _pthread_find_owner(thread_t thread, struct stackshot_thread_waitinfo *waitinfo);
 176 void * _pthread_get_thread_kwq(thread_t thread);
 177
 178 extern lck_grp_attr_t *pthread_lck_grp_attr;
 179 extern lck_grp_t *pthread_lck_grp;
 180 extern lck_attr_t *pthread_lck_attr;
 181 extern lck_mtx_t *pthread_list_mlock;
 182 extern thread_call_t psynch_thcall;
 183
 184 struct uthread* current_uthread(void);
 185
 186 int
 187 workq_create_threadstack(proc_t p, vm_map_t vmap, mach_vm_offset_t *out_addr);
 188
 189 int
 190 workq_destroy_threadstack(proc_t p, vm_map_t vmap, mach_vm_offset_t stackaddr);
 191
 192 void
 193 workq_setup_thread(proc_t p, thread_t th, vm_map_t map, user_addr_t stackaddr,
 194                 mach_port_name_t kport, int th_qos, int setup_flags, int upcall_flags);
 195
 196 int
 197 workq_handle_stack_events(proc_t p, thread_t th, vm_map_t map,
 198                 user_addr_t stackaddr, mach_port_name_t kport,
 199                 user_addr_t events, int nevents, int upcall_flags);
 200
 201 void
 202 workq_markfree_threadstack(proc_t p, thread_t th, vm_map_t vmap,
 203                 user_addr_t stackaddr);
 204
 205 #endif // KERNEL
 206
 207 // magical `nkevents` values for _pthread_wqthread
 208 #define WORKQ_EXIT_THREAD_NKEVENT   (-1)
 209
 210 #endif /* _SYS_PTHREAD_INTERNAL_H_ */
 211