bsd/sys/pthread_shims.h

   1 /*
   2  * Copyright (c) 2012 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #ifdef KERNEL_PRIVATE
  30
  31 #ifndef _PTHREAD_SHIMS_H_
  32 #define _PTHREAD_SHIMS_H_
  33
  34 #ifndef ASSEMBLER
  35
  36 #include <kern/clock.h>
  37 #include <kern/kern_types.h>
  38 #include <kern/kcdata.h>
  39 #include <kern/locks.h>
  40 #include <sys/user.h>
  41 #include <sys/_types.h>
  42 #include <sys/_types/_sigset_t.h>
  43 #include <sys/kernel_types.h>
  44
  45 #ifndef PTHREAD_INTERNAL
  46 struct uthread;
  47 #define M_PROC 41
  48 #endif
  49
  50 #if !defined(_SCHED_CALL_T_DEFINED)
  51 #define _SCHED_CALL_T_DEFINED
  52 typedef void (*sched_call_t)(int type, thread_t thread);
  53 #endif
  54
  55 typedef struct workq_reqthreads_req_s {unsigned long priority; int count;} *workq_reqthreads_req_t;
  56 typedef struct workq_threadreq_s { void *opaqueptr[2]; uint32_t opaqueint[2];} *workq_threadreq_t;
  57 enum workq_threadreq_type {
  58         WORKQ_THREADREQ_KEVENT = 1,
  59         WORKQ_THREADREQ_WORKLOOP = 2,
  60         WORKQ_THREADREQ_WORKLOOP_NO_THREAD_CALL = 3,
  61         WORKQ_THREADREQ_REDRIVE = 4,
  62 };
  63 enum workq_threadreq_op {
  64         WORKQ_THREADREQ_CHANGE_PRI = 1,
  65         WORKQ_THREADREQ_CANCEL = 2,
  66         WORKQ_THREADREQ_CHANGE_PRI_NO_THREAD_CALL = 3,
  67 };
  68 #define WORKQ_THREADREQ_FLAG_NOEMERGENCY 0x1
  69
  70
  71 /*
  72  * Increment each time new reserved slots are used. When the pthread
  73  * kext registers this table, it will include the version of the xnu
  74  * headers that it was built against.
  75  */
  76 #define PTHREAD_FUNCTIONS_TABLE_VERSION 1
  77
  78 typedef const struct pthread_functions_s {
  79         int version;
  80
  81         /* internal calls, kernel core -> kext */
  82         void (*pthread_init)(void);
  83         int (*fill_procworkqueue)(proc_t p, void* pwqinfo);
  84
  85         void (*__unused1)(void);
  86         void (*__unused2)(void);
  87
  88         void (*workqueue_exit)(struct proc *p);
  89         void (*workqueue_mark_exiting)(struct proc *p);
  90         void (*workqueue_thread_yielded)(void);
  91         void (*pth_proc_hashinit)(proc_t p);
  92         void (*pth_proc_hashdelete)(proc_t p);
  93
  94         /* syscall stubs */
  95         int (*bsdthread_create)(struct proc *p, user_addr_t user_func, user_addr_t user_funcarg, user_addr_t user_stack, user_addr_t user_pthread, uint32_t flags, user_addr_t *retval);
  96         int (*bsdthread_register)(struct proc *p, user_addr_t threadstart, user_addr_t wqthread, int pthsize, user_addr_t dummy_value, user_addr_t targetconc_ptr, uint64_t dispatchqueue_offset, int32_t *retval);
  97         int (*bsdthread_terminate)(struct proc *p, user_addr_t stackaddr, size_t size, uint32_t kthport, uint32_t sem, int32_t *retval);
  98         int (*thread_selfid)(struct proc *p, uint64_t *retval);
  99         int (*workq_kernreturn)(struct proc *p, int options, user_addr_t item, int affinity, int prio, int32_t *retval);
 100         int (*workq_open)(struct proc *p, int32_t *retval);
 101
 102         /* psynch syscalls */
 103         int (*psynch_mutexwait)(proc_t p, user_addr_t mutex,  uint32_t mgen, uint32_t  ugen, uint64_t tid, uint32_t flags, uint32_t *retval);
 104         int (*psynch_mutexdrop)(proc_t p, user_addr_t mutex,  uint32_t mgen, uint32_t  ugen, uint64_t tid, uint32_t flags, uint32_t *retval);
 105         int (*psynch_cvbroad)(proc_t p, user_addr_t cv, uint64_t cvlsgen, uint64_t cvudgen, uint32_t flags, user_addr_t mutex, uint64_t mugen, uint64_t tid, uint32_t *retval);
 106         int (*psynch_cvsignal)(proc_t p, user_addr_t cv, uint64_t cvlsgen, uint32_t cvugen, int thread_port, user_addr_t mutex, uint64_t mugen, uint64_t tid, uint32_t flags, uint32_t *retval);
 107         int (*psynch_cvwait)(proc_t p, user_addr_t cv, uint64_t cvlsgen, uint32_t cvugen, user_addr_t mutex,  uint64_t mugen, uint32_t flags, int64_t sec, uint32_t nsec, uint32_t * retval);
 108         int (*psynch_cvclrprepost)(proc_t p, user_addr_t cv, uint32_t cvgen, uint32_t cvugen, uint32_t cvsgen, uint32_t prepocnt, uint32_t preposeq, uint32_t flags, int *retval);
 109         int (*psynch_rw_longrdlock)(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t *retval);
 110         int (*psynch_rw_rdlock)(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t *retval);
 111         int (*psynch_rw_unlock)(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t *retval);
 112         int (*psynch_rw_wrlock)(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t *retval);
 113         int (*psynch_rw_yieldwrlock)(proc_t p, user_addr_t rwlock, uint32_t lgenval, uint32_t ugenval, uint32_t rw_wc, int flags, uint32_t *retval);
 114
 115         sched_call_t (*workqueue_get_sched_callback)(void);
 116
 117         /* New register function with TSD offset */
 118         int (*bsdthread_register2)(struct proc *p, user_addr_t threadstart, user_addr_t wqthread, uint32_t flags, user_addr_t stack_addr_hint, user_addr_t targetconc_ptr, uint32_t dispatchqueue_offset, uint32_t tsd_offset, int32_t *retval);
 119
 120         /* New pthreadctl system. */
 121         int (*bsdthread_ctl)(struct proc *p, user_addr_t cmd, user_addr_t arg1, user_addr_t arg2, user_addr_t arg3, int *retval);
 122
 123     /* Request threads to deliver kevents */
 124     thread_t (*workq_reqthreads)(struct proc *p, int requests_count, workq_reqthreads_req_t requests);
 125
 126     /* Resolve a pthread_priority_t to a QoS/relative pri */
 127     integer_t (*thread_qos_from_pthread_priority)(unsigned long pthread_priority, unsigned long *flags);
 128
 129         /* try to get wq flags in debugger context */
 130         uint32_t (*get_pwq_state_kdp)(proc_t p);
 131
 132         void (*__unused3)(void);
 133         unsigned long (*pthread_priority_canonicalize2)(unsigned long pthread_priority, boolean_t propagation);
 134
 135         /* Returns true on success, false on mismatch */
 136         boolean_t (*workq_thread_has_been_unbound)(thread_t th, int qos_class);
 137
 138         void (*pthread_find_owner)(thread_t thread, struct stackshot_thread_waitinfo *waitinfo);
 139         void *(*pthread_get_thread_kwq)(thread_t thread);
 140
 141         /*
 142          * Submits a threadreq to the workq system.
 143          *
 144          * If type is WORKQ_THREADREQ_KEVENT, the semantics are similar to a call
 145          * to workq_reqthreads and the kevent bind function will be called to
 146          * indicate the thread fulfilling the request.  The req argument is ignored.
 147          *
 148          * If type is WORKQ_THREADREQ_WORKLOOP, The req argument should point to
 149          * allocated memory of at least the sizeof(workq_threadreq_t).  That memory
 150          * is lent to the workq system until workloop_fulfill_threadreq is called
 151          * and passed the pointer, at which point it may be freed.
 152          *
 153          * The properties of the request are passed in the (pthread) priority and flags arguments.
 154          *
 155          * Will return zero upon success or an error value on failure.  An error of
 156          * ENOTSUP means the type argument was not understood.
 157          */
 158         int (*workq_threadreq)(struct proc *p, workq_threadreq_t req,
 159                 enum workq_threadreq_type, unsigned long priority, int flags);
 160
 161         /*
 162          * Modifies an already submitted thread request.
 163          *
 164          * If operation is WORKQ_THREADREQ_CHANGE_PRI, arg1 is the new priority and arg2 is unused.
 165          *
 166          * If operation is WORKQ_THREADREQ_CANCEL, arg1 and arg2 are unused.
 167          *
 168          * Will return zero upon success or an error value on failure.  An error of
 169          * ENOTSUP means the operation argument was not understood.
 170          */
 171         int (*workq_threadreq_modify)(struct proc *t, workq_threadreq_t req,
 172                         enum workq_threadreq_op operation,
 173                         unsigned long arg1, unsigned long arg2);
 174
 175         /* padding for future */
 176         void * _pad[87];
 177 } * pthread_functions_t;
 178
 179 typedef const struct pthread_callbacks_s {
 180         int version;
 181
 182         /* config information */
 183         uint32_t config_thread_max;
 184         uint32_t (*get_task_threadmax)(void);
 185
 186         /* proc.h accessors */
 187         uint64_t (*proc_get_register)(struct proc *t);
 188         void (*proc_set_register)(struct proc *t);
 189
 190         user_addr_t (*proc_get_threadstart)(struct proc *t);
 191         void (*proc_set_threadstart)(struct proc *t, user_addr_t addr);
 192         user_addr_t (*proc_get_wqthread)(struct proc *t);
 193         void (*proc_set_wqthread)(struct proc *t, user_addr_t addr);
 194         int (*proc_get_pthsize)(struct proc *t);
 195         void (*proc_set_pthsize)(struct proc *t, int size);
 196 #if defined(__arm64__)
 197         unsigned __int128 (*atomic_fetch_add_128_relaxed)(_Atomic unsigned __int128 *ptr,
 198                         unsigned __int128 value);
 199         unsigned __int128 (*atomic_load_128_relaxed)(_Atomic unsigned __int128 *ptr);
 200 #else
 201         void *unused_was_proc_get_targconc;
 202         void *unused_was_proc_set_targconc;
 203 #endif
 204         uint64_t (*proc_get_dispatchqueue_offset)(struct proc *t);
 205         void (*proc_set_dispatchqueue_offset)(struct proc *t, uint64_t offset);
 206         void *unused_was_proc_get_wqlockptr;
 207         void *unused_was_proc_get_wqinitingptr;
 208         void* (*proc_get_wqptr)(struct proc *t);
 209         void (*proc_set_wqptr)(struct proc *t, void* ptr);
 210         void *unused_was_proc_get_wqsize;
 211         void *unused_was_proc_set_wqsize;
 212         void (*proc_lock)(struct proc *t);
 213         void (*proc_unlock)(struct proc *t);
 214         task_t (*proc_get_task)(struct proc *t);
 215         void* (*proc_get_pthhash)(struct proc *t);
 216         void (*proc_set_pthhash)(struct proc *t, void* ptr);
 217
 218         /* bsd/sys/user.h */
 219         void* (*uthread_get_threadlist)(struct uthread *t);
 220         void (*uthread_set_threadlist)(struct uthread *t, void* threadlist);
 221         sigset_t (*uthread_get_sigmask)(struct uthread *t);
 222         void (*uthread_set_sigmask)(struct uthread *t, sigset_t s);
 223         void* (*uthread_get_uukwe)(struct uthread *t);
 224         int (*uthread_get_returnval)(struct uthread *t);
 225         void (*uthread_set_returnval)(struct uthread *t, int val);
 226         int (*uthread_is_cancelled)(struct uthread *t);
 227
 228         /* vm_protos.h calls */
 229         ipc_space_t (*task_get_ipcspace)(task_t t);
 230         mach_port_name_t (*ipc_port_copyout_send)(ipc_port_t sright, ipc_space_t space);
 231
 232         /* osfmk/vm/vm_map.h */
 233         kern_return_t (*vm_map_page_info)(vm_map_t map, vm_map_offset_t offset, vm_page_info_flavor_t flavor, vm_page_info_t info, mach_msg_type_number_t *count);
 234         vm_map_t (*vm_map_switch)(vm_map_t map);
 235
 236         /* wq functions */
 237         kern_return_t (*thread_set_wq_state32)(thread_t thread, thread_state_t state);
 238 #if !defined(__arm__)
 239         kern_return_t (*thread_set_wq_state64)(thread_t thread, thread_state_t state);
 240 #endif
 241
 242         /* sched_prim.h */
 243         void (*thread_exception_return)(void);
 244         void (*thread_bootstrap_return)(void);
 245
 246         /* kern/clock.h */
 247         void (*absolutetime_to_microtime)(uint64_t abstime, clock_sec_t *secs, clock_usec_t *microsecs);
 248
 249         kern_return_t (*thread_set_workq_pri)(thread_t thread, integer_t priority, integer_t policy);
 250         kern_return_t (*thread_set_workq_qos)(thread_t thread, int qos_tier, int relprio);
 251
 252         /* osfmk/kern/thread.h */
 253         struct uthread* (*get_bsdthread_info)(thread_t th);
 254         void (*thread_sched_call)(thread_t t, sched_call_t call);
 255         void (*thread_static_param)(thread_t t, boolean_t state);
 256         kern_return_t (*thread_create_workq)(task_t t, thread_continue_t c, thread_t *new_t);
 257         kern_return_t (*thread_policy_set_internal)(thread_t t, thread_policy_flavor_t flavour, thread_policy_t info, mach_msg_type_number_t count);
 258
 259         /* osfmk/kern/affinity.h */
 260         kern_return_t (*thread_affinity_set)(thread_t thread, uint32_t tag);
 261
 262         /* bsd/sys/systm.h */
 263         void (*unix_syscall_return)(int error);
 264
 265         /* osfmk/kern/zalloc.h */
 266         void* (*zalloc)(zone_t zone);
 267         void (*zfree)(zone_t zone, void* ptr);
 268         zone_t (*zinit)(vm_size_t, vm_size_t maxmem, vm_size_t alloc, const char *name);
 269
 270         /* bsd/kerb/kern_sig.c */
 271         void (*__pthread_testcancel)(int);
 272
 273         /* calls without portfolio */
 274         kern_return_t (*mach_port_deallocate)(ipc_space_t space, mach_port_name_t name);
 275         kern_return_t (*semaphore_signal_internal_trap)(mach_port_name_t sema_name);
 276         vm_map_t (*current_map)(void);
 277
 278         /* osfmk/kern/thread.h */
 279         ipc_port_t (*convert_thread_to_port)(thread_t th);
 280
 281         /* mach/task.h */
 282         kern_return_t (*thread_create)(task_t parent_task, thread_act_t *child_act);
 283
 284         /* mach/thread_act.h */
 285         kern_return_t (*thread_resume)(thread_act_t target_act);
 286
 287         /* osfmk/<arch>/machine_routines.h */
 288         int (*ml_get_max_cpus)(void);
 289
 290         #if defined(__arm__)
 291         uint32_t (*map_is_1gb)(vm_map_t);
 292         #endif
 293
 294         /* <rdar://problem/12809089> xnu: struct proc p_dispatchqueue_serialno_offset additions */
 295         uint64_t (*proc_get_dispatchqueue_serialno_offset)(struct proc *p);
 296         void (*proc_set_dispatchqueue_serialno_offset)(struct proc *p, uint64_t offset);
 297
 298         int (*proc_usynch_thread_qos_add_override_for_resource_check_owner)(thread_t thread, int override_qos, boolean_t first_override_for_resource,
 299                         user_addr_t resource, int resource_type, user_addr_t user_lock_addr, mach_port_name_t user_lock_owner);
 300         void *unused_was_proc_set_stack_addr_hint;
 301
 302         uint32_t (*proc_get_pthread_tsd_offset)(struct proc *p);
 303         void (*proc_set_pthread_tsd_offset)(struct proc *p, uint32_t pthread_tsd_offset);
 304
 305         kern_return_t (*thread_set_tsd_base)(thread_t thread, mach_vm_offset_t tsd_base);
 306
 307         int     (*proc_usynch_get_requested_thread_qos)(struct uthread *);
 308         uint64_t (*proc_get_mach_thread_self_tsd_offset)(struct proc *p);
 309         void (*proc_set_mach_thread_self_tsd_offset)(struct proc *p, uint64_t mach_thread_self_tsd_offset);
 310
 311         kern_return_t (*thread_policy_get)(thread_t t, thread_policy_flavor_t flavor, thread_policy_t info, mach_msg_type_number_t *count, boolean_t *get_default);
 312         boolean_t (*qos_main_thread_active)(void);
 313
 314         kern_return_t (*thread_set_voucher_name)(mach_port_name_t voucher_name);
 315
 316         boolean_t (*proc_usynch_thread_qos_add_override_for_resource)(task_t task, struct uthread *, uint64_t tid, int override_qos, boolean_t first_override_for_resource, user_addr_t resource, int resource_type);
 317         boolean_t (*proc_usynch_thread_qos_remove_override_for_resource)(task_t task, struct uthread *, uint64_t tid, user_addr_t resource, int resource_type);
 318         boolean_t (*proc_usynch_thread_qos_reset_override_for_resource)(task_t task, struct uthread *, uint64_t tid, user_addr_t resource, int resource_type);
 319
 320         boolean_t (*proc_init_wqptr_or_wait)(proc_t proc);
 321
 322         uint16_t (*thread_set_tag)(thread_t thread, uint16_t tag);
 323         uint16_t (*thread_get_tag)(thread_t thread);
 324
 325         int (*proc_usynch_thread_qos_squash_override_for_resource)(thread_t thread, user_addr_t resource, int resource_type);
 326         int (*task_get_default_manager_qos)(task_t task);
 327
 328         int (*thread_create_workq_waiting)(task_t task, thread_continue_t thread_return, event_t event, thread_t *new_thread);
 329
 330         user_addr_t (*proc_get_stack_addr_hint)(struct proc *p);
 331         void (*proc_set_stack_addr_hint)(struct proc *p, user_addr_t stack_addr_hint);
 332
 333         uint64_t (*proc_get_return_to_kernel_offset)(struct proc *t);
 334         void (*proc_set_return_to_kernel_offset)(struct proc *t, uint64_t offset);
 335
 336         /* indicates call is being made synchronously with workq_threadreq call */
 337 #       define WORKLOOP_FULFILL_THREADREQ_SYNC   0x1
 338 #       define WORKLOOP_FULFILL_THREADREQ_CANCEL 0x2
 339         int (*workloop_fulfill_threadreq)(struct proc *p, workq_threadreq_t req, thread_t thread, int flags);
 340         void (*thread_will_park_or_terminate)(thread_t thread);
 341
 342         /* For getting maximum parallelism for a given QoS */
 343         uint32_t (*qos_max_parallelism)(int qos, uint64_t options);
 344
 345         /* proc_internal.h: struct proc user_stack accessor */
 346         user_addr_t (*proc_get_user_stack)(struct proc *p);
 347         void (*proc_set_user_stack)(struct proc *p, user_addr_t user_stack);
 348
 349         /* padding for future */
 350         void* _pad[69];
 351
 352 } *pthread_callbacks_t;
 353
 354 void
 355 pthread_kext_register(pthread_functions_t fns, pthread_callbacks_t *callbacks);
 356
 357 #ifdef BSD_KERNEL_PRIVATE
 358 void workqueue_mark_exiting(struct proc *);
 359 void workqueue_exit(struct proc *);
 360 void workqueue_thread_yielded(void);
 361 sched_call_t workqueue_get_sched_callback(void);
 362 void pthread_init(void);
 363
 364 extern pthread_callbacks_t pthread_kern;
 365 extern pthread_functions_t pthread_functions;
 366 #endif
 367
 368 #endif /* ASSEMBLER */
 369 #endif /* _PTHREAD_SHIMS_H_ */
 370 #endif /* KERNEL_PRIVATE */