bsd/kern/pthread_shims.c

   1 /*
   2  * Copyright (c) 2012-2016 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #define PTHREAD_INTERNAL 1
  30
  31 #include <kern/debug.h>
  32 #include <kern/mach_param.h>
  33 #include <kern/sched_prim.h>
  34 #include <kern/task.h>
  35 #include <kern/thread.h>
  36 #include <kern/affinity.h>
  37 #include <kern/zalloc.h>
  38 #include <kern/policy_internal.h>
  39
  40 #include <machine/machine_routines.h>
  41 #include <mach/task.h>
  42 #include <mach/thread_act.h>
  43 #include <sys/param.h>
  44 #include <sys/pthread_shims.h>
  45 #include <sys/proc_internal.h>
  46 #include <sys/sysproto.h>
  47 #include <sys/systm.h>
  48 #include <vm/vm_map.h>
  49 #include <vm/vm_protos.h>
  50 #include <kern/kcdata.h>
  51
  52 /* version number of the in-kernel shims given to pthread.kext */
  53 #define PTHREAD_SHIMS_VERSION 1
  54
  55 /* on arm, the callbacks function has two #ifdef arm ponters */
  56 #define PTHREAD_CALLBACK_MEMBER ml_get_max_cpus
  57
  58 /* compile time asserts to check the length of structures in pthread_shims.h */
  59 static_assert((sizeof(struct pthread_functions_s) - offsetof(struct pthread_functions_s, psynch_rw_yieldwrlock) - sizeof(void*)) == (sizeof(void*) * 100));
  60 static_assert((sizeof(struct pthread_callbacks_s) - offsetof(struct pthread_callbacks_s, PTHREAD_CALLBACK_MEMBER) - sizeof(void*)) == (sizeof(void*) * 100));
  61
  62 /* old pthread code had definitions for these as they don't exist in headers */
  63 extern kern_return_t mach_port_deallocate(ipc_space_t, mach_port_name_t);
  64 extern kern_return_t semaphore_signal_internal_trap(mach_port_name_t);
  65
  66 #define PTHREAD_STRUCT_ACCESSOR(get, set, rettype, structtype, member) \
  67         static rettype \
  68         get(structtype x) { \
  69                 return (x)->member; \
  70         } \
  71         static void \
  72         set(structtype x, rettype y) { \
  73                 (x)->member = y; \
  74         }
  75
  76 PTHREAD_STRUCT_ACCESSOR(proc_get_threadstart, proc_set_threadstart, user_addr_t, struct proc*, p_threadstart);
  77 PTHREAD_STRUCT_ACCESSOR(proc_get_pthsize, proc_set_pthsize, int, struct proc*, p_pthsize);
  78 PTHREAD_STRUCT_ACCESSOR(proc_get_wqthread, proc_set_wqthread, user_addr_t, struct proc*, p_wqthread);
  79 PTHREAD_STRUCT_ACCESSOR(proc_get_stack_addr_hint, proc_set_stack_addr_hint, user_addr_t, struct proc *, p_stack_addr_hint);
  80 PTHREAD_STRUCT_ACCESSOR(proc_get_dispatchqueue_offset, proc_set_dispatchqueue_offset, uint64_t, struct proc*, p_dispatchqueue_offset);
  81 PTHREAD_STRUCT_ACCESSOR(proc_get_dispatchqueue_serialno_offset, proc_set_dispatchqueue_serialno_offset, uint64_t, struct proc*, p_dispatchqueue_serialno_offset);
  82 PTHREAD_STRUCT_ACCESSOR(proc_get_pthread_tsd_offset, proc_set_pthread_tsd_offset, uint32_t, struct proc *, p_pth_tsd_offset);
  83 PTHREAD_STRUCT_ACCESSOR(proc_get_pthhash, proc_set_pthhash, void*, struct proc*, p_pthhash);
  84
  85 PTHREAD_STRUCT_ACCESSOR(uthread_get_threadlist, uthread_set_threadlist, void*, struct uthread*, uu_threadlist);
  86 PTHREAD_STRUCT_ACCESSOR(uthread_get_sigmask, uthread_set_sigmask, sigset_t, struct uthread*, uu_sigmask);
  87 PTHREAD_STRUCT_ACCESSOR(uthread_get_returnval, uthread_set_returnval, int, struct uthread*, uu_rval[0]);
  88
  89 #define WQPTR_IS_INITING_VALUE ((void *)~(uintptr_t)0)
  90
  91 static void *
  92 proc_get_wqptr(struct proc *p) {
  93         void *wqptr =  p->p_wqptr;
  94         return (wqptr == WQPTR_IS_INITING_VALUE) ? NULL : wqptr;
  95 }
  96 static void
  97 proc_set_wqptr(struct proc *p, void *y) {
  98         proc_lock(p);
  99
 100         assert(y == NULL || p->p_wqptr == WQPTR_IS_INITING_VALUE);
 101
 102         p->p_wqptr = y;
 103
 104         if (y != NULL){
 105                 wakeup(&p->p_wqptr);
 106         }
 107
 108         proc_unlock(p);
 109 }
 110 static boolean_t
 111 proc_init_wqptr_or_wait(struct proc *p) {
 112         proc_lock(p);
 113
 114         if (p->p_wqptr == NULL){
 115                 p->p_wqptr = WQPTR_IS_INITING_VALUE;
 116                 proc_unlock(p);
 117
 118                 return TRUE;
 119         } else if (p->p_wqptr == WQPTR_IS_INITING_VALUE){
 120                 assert_wait(&p->p_wqptr, THREAD_UNINT);
 121                 proc_unlock(p);
 122                 thread_block(THREAD_CONTINUE_NULL);
 123
 124                 return FALSE;
 125         } else {
 126                 proc_unlock(p);
 127
 128                 return FALSE;
 129         }
 130 }
 131
 132 __attribute__((noreturn))
 133 static void
 134 pthread_returning_to_userspace(void)
 135 {
 136         thread_exception_return();
 137 }
 138
 139 static uint32_t
 140 get_task_threadmax(void) {
 141         return task_threadmax;
 142 }
 143
 144 static task_t
 145 proc_get_task(struct proc *p) {
 146         return p->task;
 147 }
 148
 149 static uint64_t
 150 proc_get_register(struct proc *p) {
 151         return (p->p_lflag & P_LREGISTER);
 152 }
 153
 154 static void
 155 proc_set_register(struct proc *p) {
 156         proc_setregister(p);
 157 }
 158
 159 static void*
 160 uthread_get_uukwe(struct uthread *t)
 161 {
 162         return &t->uu_kevent.uu_kwe;
 163 }
 164
 165 static int
 166 uthread_is_cancelled(struct uthread *t)
 167 {
 168         return (t->uu_flag & (UT_CANCELDISABLE | UT_CANCEL | UT_CANCELED)) == UT_CANCEL;
 169 }
 170
 171 static vm_map_t
 172 _current_map(void)
 173 {
 174         return current_map();
 175 }
 176
 177 static boolean_t
 178 qos_main_thread_active(void)
 179 {
 180         return TRUE;
 181 }
 182
 183
 184 static int proc_usynch_get_requested_thread_qos(struct uthread *uth)
 185 {
 186         thread_t        thread = uth ? uth->uu_thread : current_thread();
 187         int                     requested_qos;
 188
 189         requested_qos = proc_get_thread_policy(thread, TASK_POLICY_ATTRIBUTE, TASK_POLICY_QOS);
 190
 191         /*
 192          * For the purposes of userspace synchronization, it doesn't make sense to
 193          * place an override of UNSPECIFIED on another thread, if the current thread
 194          * doesn't have any QoS set. In these cases, upgrade to
 195          * THREAD_QOS_USER_INTERACTIVE.
 196          */
 197         if (requested_qos == THREAD_QOS_UNSPECIFIED) {
 198                 requested_qos = THREAD_QOS_USER_INTERACTIVE;
 199         }
 200
 201         return requested_qos;
 202 }
 203
 204 static int
 205 proc_usynch_thread_qos_add_override_for_resource_check_owner(thread_t thread,
 206                 int override_qos, boolean_t first_override_for_resource,
 207                 user_addr_t resource, int resource_type,
 208                 user_addr_t user_lock_addr, mach_port_name_t user_lock_owner)
 209 {
 210         return proc_thread_qos_add_override_check_owner(thread, override_qos,
 211                         first_override_for_resource, resource, resource_type,
 212                         user_lock_addr, user_lock_owner);
 213 }
 214
 215 static boolean_t
 216 proc_usynch_thread_qos_add_override_for_resource(task_t task, struct uthread *uth,
 217                 uint64_t tid, int override_qos, boolean_t first_override_for_resource,
 218                 user_addr_t resource, int resource_type)
 219 {
 220         thread_t thread = uth ? uth->uu_thread : THREAD_NULL;
 221
 222         return proc_thread_qos_add_override(task, thread, tid, override_qos,
 223                         first_override_for_resource, resource, resource_type);
 224 }
 225
 226 static boolean_t
 227 proc_usynch_thread_qos_remove_override_for_resource(task_t task,
 228                 struct uthread *uth, uint64_t tid, user_addr_t resource, int resource_type)
 229 {
 230         thread_t thread = uth ? uth->uu_thread : THREAD_NULL;
 231
 232         return proc_thread_qos_remove_override(task, thread, tid, resource, resource_type);
 233 }
 234
 235 static boolean_t
 236 proc_usynch_thread_qos_reset_override_for_resource(task_t task,
 237                 struct uthread *uth, uint64_t tid, user_addr_t resource, int resource_type)
 238 {
 239         thread_t thread = uth ? uth->uu_thread : THREAD_NULL;
 240
 241         return proc_thread_qos_reset_override(task, thread, tid, resource, resource_type);
 242 }
 243
 244 static boolean_t
 245 proc_usynch_thread_qos_squash_override_for_resource(thread_t thread,
 246                 user_addr_t resource, int resource_type)
 247 {
 248         return proc_thread_qos_squash_override(thread, resource, resource_type);
 249 }
 250
 251 /* kernel (core) to kext shims */
 252
 253 void
 254 pthread_init(void)
 255 {
 256         if (!pthread_functions) {
 257                 panic("pthread kernel extension not loaded (function table is NULL).");
 258         }
 259         pthread_functions->pthread_init();
 260 }
 261
 262 int
 263 fill_procworkqueue(proc_t p, struct proc_workqueueinfo * pwqinfo)
 264 {
 265         return pthread_functions->fill_procworkqueue(p, pwqinfo);
 266 }
 267
 268 /*
 269  * Returns true if the workqueue flags are available, and will fill
 270  * in exceeded_total and exceeded_constrained.
 271  */
 272 boolean_t
 273 workqueue_get_pwq_exceeded(void *v, boolean_t *exceeded_total,
 274                            boolean_t *exceeded_constrained)
 275 {
 276         proc_t p = v;
 277         struct proc_workqueueinfo pwqinfo;
 278         int err;
 279
 280         assert(p != NULL);
 281         assert(exceeded_total != NULL);
 282         assert(exceeded_constrained != NULL);
 283
 284         err = fill_procworkqueue(p, &pwqinfo);
 285         if (err) {
 286                 return FALSE;
 287         }
 288         if (!(pwqinfo.pwq_state & WQ_FLAGS_AVAILABLE)) {
 289                 return FALSE;
 290         }
 291
 292         *exceeded_total = (pwqinfo.pwq_state & WQ_EXCEEDED_TOTAL_THREAD_LIMIT);
 293         *exceeded_constrained = (pwqinfo.pwq_state & WQ_EXCEEDED_CONSTRAINED_THREAD_LIMIT);
 294
 295         return TRUE;
 296 }
 297
 298 uint32_t
 299 workqueue_get_pwq_state_kdp(void * v)
 300 {
 301         static_assert((WQ_EXCEEDED_CONSTRAINED_THREAD_LIMIT << 17) == kTaskWqExceededConstrainedThreadLimit);
 302         static_assert((WQ_EXCEEDED_TOTAL_THREAD_LIMIT << 17) == kTaskWqExceededTotalThreadLimit);
 303         static_assert((WQ_FLAGS_AVAILABLE << 17) == kTaskWqFlagsAvailable);
 304         static_assert((WQ_FLAGS_AVAILABLE | WQ_EXCEEDED_TOTAL_THREAD_LIMIT | WQ_EXCEEDED_CONSTRAINED_THREAD_LIMIT) == 0x7);
 305         proc_t p = v;
 306         if (pthread_functions == NULL || pthread_functions->get_pwq_state_kdp == NULL)
 307                 return 0;
 308         else
 309                 return pthread_functions->get_pwq_state_kdp(p);
 310 }
 311
 312 void
 313 workqueue_exit(struct proc *p)
 314 {
 315         pthread_functions->workqueue_exit(p);
 316 }
 317
 318 void
 319 workqueue_mark_exiting(struct proc *p)
 320 {
 321         pthread_functions->workqueue_mark_exiting(p);
 322 }
 323
 324 void
 325 workqueue_thread_yielded(void)
 326 {
 327         pthread_functions->workqueue_thread_yielded();
 328 }
 329
 330 sched_call_t
 331 workqueue_get_sched_callback(void)
 332 {
 333         if (pthread_functions->workqueue_get_sched_callback) {
 334                 return pthread_functions->workqueue_get_sched_callback();
 335         }
 336         return NULL;
 337 }
 338
 339 void
 340 pth_proc_hashinit(proc_t p)
 341 {
 342         pthread_functions->pth_proc_hashinit(p);
 343 }
 344
 345 void
 346 pth_proc_hashdelete(proc_t p)
 347 {
 348         pthread_functions->pth_proc_hashdelete(p);
 349 }
 350
 351 /* syscall shims */
 352 int
 353 bsdthread_create(struct proc *p, struct bsdthread_create_args *uap, user_addr_t *retval)
 354 {
 355         return pthread_functions->bsdthread_create(p, uap->func, uap->func_arg, uap->stack, uap->pthread, uap->flags, retval);
 356 }
 357
 358 int
 359 bsdthread_register(struct proc *p, struct bsdthread_register_args *uap, __unused int32_t *retval)
 360 {
 361         if (pthread_functions->version >= 1) {
 362                 return pthread_functions->bsdthread_register2(p, uap->threadstart, uap->wqthread,
 363                                                                                                           uap->flags, uap->stack_addr_hint,
 364                                                                                                           uap->targetconc_ptr, uap->dispatchqueue_offset,
 365                                                                                                           uap->tsd_offset, retval);
 366         } else {
 367                 return pthread_functions->bsdthread_register(p, uap->threadstart, uap->wqthread,
 368                                                                                                          uap->flags, uap->stack_addr_hint,
 369                                                                                                          uap->targetconc_ptr, uap->dispatchqueue_offset,
 370                                                                                                          retval);
 371         }
 372 }
 373
 374 int
 375 bsdthread_terminate(struct proc *p, struct bsdthread_terminate_args *uap, int32_t *retval)
 376 {
 377         return pthread_functions->bsdthread_terminate(p, uap->stackaddr, uap->freesize, uap->port, uap->sem, retval);
 378 }
 379
 380 int
 381 bsdthread_ctl(struct proc *p, struct bsdthread_ctl_args *uap, int *retval)
 382 {
 383     return pthread_functions->bsdthread_ctl(p, uap->cmd, uap->arg1, uap->arg2, uap->arg3, retval);
 384 }
 385
 386
 387 int
 388 thread_selfid(struct proc *p, __unused struct thread_selfid_args *uap, uint64_t *retval)
 389 {
 390         return pthread_functions->thread_selfid(p, retval);
 391 }
 392
 393 int
 394 workq_kernreturn(struct proc *p, struct workq_kernreturn_args *uap, int32_t *retval)
 395 {
 396         return pthread_functions->workq_kernreturn(p, uap->options, uap->item, uap->affinity, uap->prio, retval);
 397 }
 398
 399 int
 400 workq_open(struct proc *p, __unused struct workq_open_args  *uap, int32_t *retval)
 401 {
 402         return pthread_functions->workq_open(p, retval);
 403 }
 404
 405 /* pthread synchroniser syscalls */
 406
 407 int
 408 psynch_mutexwait(proc_t p, struct psynch_mutexwait_args *uap, uint32_t *retval)
 409 {
 410         return pthread_functions->psynch_mutexwait(p, uap->mutex, uap->mgen, uap->ugen, uap->tid, uap->flags, retval);
 411 }
 412
 413 int
 414 psynch_mutexdrop(proc_t p, struct psynch_mutexdrop_args *uap, uint32_t *retval)
 415 {
 416         return pthread_functions->psynch_mutexdrop(p, uap->mutex, uap->mgen, uap->ugen, uap->tid, uap->flags, retval);
 417 }
 418
 419 int
 420 psynch_cvbroad(proc_t p, struct psynch_cvbroad_args *uap, uint32_t *retval)
 421 {
 422         return pthread_functions->psynch_cvbroad(p, uap->cv, uap->cvlsgen, uap->cvudgen, uap->flags, uap->mutex, uap->mugen, uap->tid, retval);
 423 }
 424
 425 int
 426 psynch_cvsignal(proc_t p, struct psynch_cvsignal_args *uap, uint32_t *retval)
 427 {
 428         return pthread_functions->psynch_cvsignal(p, uap->cv, uap->cvlsgen, uap->cvugen, uap->thread_port, uap->mutex, uap->mugen, uap->tid, uap->flags, retval);
 429 }
 430
 431 int
 432 psynch_cvwait(proc_t p, struct psynch_cvwait_args * uap, uint32_t * retval)
 433 {
 434         return pthread_functions->psynch_cvwait(p, uap->cv, uap->cvlsgen, uap->cvugen, uap->mutex, uap->mugen, uap->flags, uap->sec, uap->nsec, retval);
 435 }
 436
 437 int
 438 psynch_cvclrprepost(proc_t p, struct psynch_cvclrprepost_args * uap, int *retval)
 439 {
 440         return pthread_functions->psynch_cvclrprepost(p, uap->cv, uap->cvgen, uap->cvugen, uap->cvsgen, uap->prepocnt, uap->preposeq, uap->flags, retval);
 441 }
 442
 443 int
 444 psynch_rw_longrdlock(proc_t p, struct psynch_rw_longrdlock_args * uap,  uint32_t *retval)
 445 {
 446         return pthread_functions->psynch_rw_longrdlock(p, uap->rwlock, uap->lgenval, uap->ugenval, uap->rw_wc, uap->flags, retval);
 447 }
 448
 449 int
 450 psynch_rw_rdlock(proc_t p, struct psynch_rw_rdlock_args * uap, uint32_t * retval)
 451 {
 452         return pthread_functions->psynch_rw_rdlock(p, uap->rwlock, uap->lgenval, uap->ugenval, uap->rw_wc, uap->flags, retval);
 453 }
 454
 455 int
 456 psynch_rw_unlock(proc_t p, struct psynch_rw_unlock_args *uap, uint32_t *retval)
 457 {
 458         return pthread_functions->psynch_rw_unlock(p, uap->rwlock, uap->lgenval, uap->ugenval, uap->rw_wc, uap->flags, retval);
 459 }
 460
 461 int
 462 psynch_rw_unlock2(__unused proc_t p, __unused struct psynch_rw_unlock2_args *uap, __unused uint32_t *retval)
 463 {
 464         return ENOTSUP;
 465 }
 466
 467 int
 468 psynch_rw_wrlock(proc_t p, struct psynch_rw_wrlock_args *uap, uint32_t *retval)
 469 {
 470         return pthread_functions->psynch_rw_wrlock(p, uap->rwlock, uap->lgenval, uap->ugenval, uap->rw_wc, uap->flags, retval);
 471 }
 472
 473 int
 474 psynch_rw_yieldwrlock(proc_t p, struct psynch_rw_yieldwrlock_args *uap, uint32_t *retval)
 475 {
 476         return pthread_functions->psynch_rw_yieldwrlock(p, uap->rwlock, uap->lgenval, uap->ugenval, uap->rw_wc, uap->flags, retval);
 477 }
 478
 479 int
 480 psynch_rw_upgrade(__unused proc_t p, __unused struct psynch_rw_upgrade_args * uap, __unused uint32_t *retval)
 481 {
 482         return 0;
 483 }
 484
 485 int
 486 psynch_rw_downgrade(__unused proc_t p, __unused struct psynch_rw_downgrade_args * uap, __unused int *retval)
 487 {
 488         return 0;
 489 }
 490
 491 int
 492 thread_qos_from_pthread_priority(unsigned long priority, unsigned long *flags)
 493 {
 494         return pthread_functions->thread_qos_from_pthread_priority(priority, flags);
 495 }
 496
 497 unsigned long
 498 pthread_priority_canonicalize(unsigned long priority, boolean_t propagation)
 499 {
 500         if (pthread_functions->pthread_priority_canonicalize2) {
 501                 return pthread_functions->pthread_priority_canonicalize2(priority, propagation);
 502         } else {
 503                 return pthread_functions->pthread_priority_canonicalize(priority);
 504         }
 505 }
 506
 507 /*
 508  * The callbacks structure (defined in pthread_shims.h) contains a collection
 509  * of kernel functions that were not deemed sensible to expose as a KPI to all
 510  * kernel extensions. So the kext is given them in the form of a structure of
 511  * function pointers.
 512  */
 513 static const struct pthread_callbacks_s pthread_callbacks = {
 514         .version = PTHREAD_SHIMS_VERSION,
 515         .config_thread_max = CONFIG_THREAD_MAX,
 516         .get_task_threadmax = get_task_threadmax,
 517
 518         .proc_get_threadstart = proc_get_threadstart,
 519         .proc_set_threadstart = proc_set_threadstart,
 520         .proc_get_pthsize = proc_get_pthsize,
 521         .proc_set_pthsize = proc_set_pthsize,
 522         .proc_get_wqthread = proc_get_wqthread,
 523         .proc_set_wqthread = proc_set_wqthread,
 524         .proc_get_dispatchqueue_offset = proc_get_dispatchqueue_offset,
 525         .proc_set_dispatchqueue_offset = proc_set_dispatchqueue_offset,
 526         .proc_get_wqptr = proc_get_wqptr,
 527         .proc_set_wqptr = proc_set_wqptr,
 528         .proc_get_pthhash = proc_get_pthhash,
 529         .proc_set_pthhash = proc_set_pthhash,
 530         .proc_get_task = proc_get_task,
 531         .proc_lock = proc_lock,
 532         .proc_unlock = proc_unlock,
 533         .proc_get_register = proc_get_register,
 534         .proc_set_register = proc_set_register,
 535
 536         /* kernel IPI interfaces */
 537         .ipc_port_copyout_send = ipc_port_copyout_send,
 538         .task_get_ipcspace = get_task_ipcspace,
 539         .vm_map_page_info = vm_map_page_info,
 540         .vm_map_switch = vm_map_switch,
 541         .thread_set_wq_state32 = thread_set_wq_state32,
 542         .thread_set_wq_state64 = thread_set_wq_state64,
 543
 544         .uthread_get_threadlist = uthread_get_threadlist,
 545         .uthread_set_threadlist = uthread_set_threadlist,
 546         .uthread_get_sigmask = uthread_get_sigmask,
 547         .uthread_set_sigmask = uthread_set_sigmask,
 548         .uthread_get_uukwe = uthread_get_uukwe,
 549         .uthread_get_returnval = uthread_get_returnval,
 550         .uthread_set_returnval = uthread_set_returnval,
 551         .uthread_is_cancelled = uthread_is_cancelled,
 552
 553         .thread_exception_return = pthread_returning_to_userspace,
 554         .thread_bootstrap_return = thread_bootstrap_return,
 555         .unix_syscall_return = unix_syscall_return,
 556
 557         .absolutetime_to_microtime = absolutetime_to_microtime,
 558
 559         .thread_set_workq_pri = thread_set_workq_pri,
 560         .thread_set_workq_qos = thread_set_workq_qos,
 561
 562         .get_bsdthread_info = (void*)get_bsdthread_info,
 563         .thread_sched_call = thread_sched_call,
 564         .thread_static_param = thread_static_param,
 565         .thread_create_workq = thread_create_workq,
 566         .thread_policy_set_internal = thread_policy_set_internal,
 567         .thread_policy_get = thread_policy_get,
 568         .thread_set_voucher_name = thread_set_voucher_name,
 569
 570         .thread_affinity_set = thread_affinity_set,
 571
 572         .zalloc = zalloc,
 573         .zfree = zfree,
 574         .zinit = zinit,
 575
 576         .__pthread_testcancel = __pthread_testcancel,
 577
 578         .mach_port_deallocate = mach_port_deallocate,
 579         .semaphore_signal_internal_trap = semaphore_signal_internal_trap,
 580         .current_map = _current_map,
 581         .thread_create = thread_create,
 582         .thread_resume = thread_resume,
 583
 584         .convert_thread_to_port = convert_thread_to_port,
 585         .ml_get_max_cpus = (void*)ml_get_max_cpus,
 586
 587
 588         .proc_get_dispatchqueue_serialno_offset = proc_get_dispatchqueue_serialno_offset,
 589         .proc_set_dispatchqueue_serialno_offset = proc_set_dispatchqueue_serialno_offset,
 590
 591         .proc_get_stack_addr_hint = proc_get_stack_addr_hint,
 592         .proc_set_stack_addr_hint = proc_set_stack_addr_hint,
 593         .proc_get_pthread_tsd_offset = proc_get_pthread_tsd_offset,
 594         .proc_set_pthread_tsd_offset = proc_set_pthread_tsd_offset,
 595
 596         .thread_set_tsd_base = thread_set_tsd_base,
 597
 598         .proc_usynch_get_requested_thread_qos = proc_usynch_get_requested_thread_qos,
 599
 600         .qos_main_thread_active = qos_main_thread_active,
 601
 602         .proc_usynch_thread_qos_add_override_for_resource_check_owner = proc_usynch_thread_qos_add_override_for_resource_check_owner,
 603         .proc_usynch_thread_qos_add_override_for_resource = proc_usynch_thread_qos_add_override_for_resource,
 604         .proc_usynch_thread_qos_remove_override_for_resource = proc_usynch_thread_qos_remove_override_for_resource,
 605         .proc_usynch_thread_qos_reset_override_for_resource = proc_usynch_thread_qos_reset_override_for_resource,
 606
 607         .proc_init_wqptr_or_wait = proc_init_wqptr_or_wait,
 608
 609         .thread_set_tag = thread_set_tag,
 610         .thread_get_tag = thread_get_tag,
 611
 612         .proc_usynch_thread_qos_squash_override_for_resource = proc_usynch_thread_qos_squash_override_for_resource,
 613         .task_get_default_manager_qos = task_get_default_manager_qos,
 614         .thread_create_workq_waiting = thread_create_workq_waiting,
 615 };
 616
 617 pthread_callbacks_t pthread_kern = &pthread_callbacks;
 618 pthread_functions_t pthread_functions = NULL;
 619
 620 /*
 621  * pthread_kext_register is called by pthread.kext upon load, it has to provide
 622  * us with a function pointer table of pthread internal calls. In return, this
 623  * file provides it with a table of function pointers it needs.
 624  */
 625
 626 void
 627 pthread_kext_register(pthread_functions_t fns, pthread_callbacks_t *callbacks)
 628 {
 629         if (pthread_functions != NULL) {
 630                 panic("Re-initialisation of pthread kext callbacks.");
 631         }
 632
 633         if (callbacks != NULL) {
 634                 *callbacks = &pthread_callbacks;
 635         } else {
 636                 panic("pthread_kext_register called without callbacks pointer.");
 637         }
 638
 639         if (fns) {
 640                 pthread_functions = fns;
 641         }
 642 }