bsd/kern/pthread_shims.c

   1 /*
   2  * Copyright (c) 2012 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #define PTHREAD_INTERNAL 1
  30
  31 #include <kern/debug.h>
  32 #include <kern/mach_param.h>
  33 #include <kern/sched_prim.h>
  34 #include <kern/task.h>
  35 #include <kern/thread.h>
  36 #include <kern/affinity.h>
  37 #include <kern/zalloc.h>
  38 #include <machine/machine_routines.h>
  39 #include <mach/task.h>
  40 #include <mach/thread_act.h>
  41 #include <sys/param.h>
  42 #include <sys/pthread_shims.h>
  43 #include <sys/proc_internal.h>
  44 #include <sys/sysproto.h>
  45 #include <sys/systm.h>
  46 #include <vm/vm_map.h>
  47 #include <vm/vm_protos.h>
  48
  49 /* version number of the in-kernel shims given to pthread.kext */
  50 #define PTHREAD_SHIMS_VERSION 1
  51
  52 /* old pthread code had definitions for these as they don't exist in headers */
  53 extern kern_return_t mach_port_deallocate(ipc_space_t, mach_port_name_t);
  54 extern kern_return_t semaphore_signal_internal_trap(mach_port_name_t);
  55
  56 #define PTHREAD_STRUCT_ACCESSOR(get, set, rettype, structtype, member) \
  57         static rettype \
  58         get(structtype x) { \
  59                 return (x)->member; \
  60         } \
  61         static void \
  62         set(structtype x, rettype y) { \
  63                 (x)->member = y; \
  64         }
  65
  66 PTHREAD_STRUCT_ACCESSOR(proc_get_threadstart, proc_set_threadstart, user_addr_t, struct proc*, p_threadstart);
  67 PTHREAD_STRUCT_ACCESSOR(proc_get_pthsize, proc_set_pthsize, int, struct proc*, p_pthsize);
  68 PTHREAD_STRUCT_ACCESSOR(proc_get_wqthread, proc_set_wqthread, user_addr_t, struct proc*, p_wqthread);
  69 PTHREAD_STRUCT_ACCESSOR(proc_get_targconc, proc_set_targconc, user_addr_t, struct proc*, p_targconc);
  70 PTHREAD_STRUCT_ACCESSOR(proc_get_dispatchqueue_offset, proc_set_dispatchqueue_offset, uint64_t, struct proc*, p_dispatchqueue_offset);
  71 PTHREAD_STRUCT_ACCESSOR(proc_get_dispatchqueue_serialno_offset, proc_set_dispatchqueue_serialno_offset, uint64_t, struct proc*, p_dispatchqueue_serialno_offset);
  72 PTHREAD_STRUCT_ACCESSOR(proc_get_wqptr, proc_set_wqptr, void*, struct proc*, p_wqptr);
  73 PTHREAD_STRUCT_ACCESSOR(proc_get_wqsize, proc_set_wqsize, int, struct proc*, p_wqsize);
  74 PTHREAD_STRUCT_ACCESSOR(proc_get_pthhash, proc_set_pthhash, void*, struct proc*, p_pthhash);
  75
  76 PTHREAD_STRUCT_ACCESSOR(uthread_get_threadlist, uthread_set_threadlist, void*, struct uthread*, uu_threadlist);
  77 PTHREAD_STRUCT_ACCESSOR(uthread_get_sigmask, uthread_set_sigmask, sigset_t, struct uthread*, uu_sigmask);
  78 PTHREAD_STRUCT_ACCESSOR(uthread_get_returnval, uthread_set_returnval, int, struct uthread*, uu_rval[0]);
  79
  80 static void
  81 pthread_returning_to_userspace(void)
  82 {
  83         thread_exception_return();
  84 }
  85
  86 static uint32_t
  87 get_task_threadmax(void) {
  88         return task_threadmax;
  89 }
  90
  91 static task_t
  92 proc_get_task(struct proc *p) {
  93         return p->task;
  94 }
  95
  96 static lck_spin_t*
  97 proc_get_wqlockptr(struct proc *p) {
  98         return &(p->p_wqlock);
  99 }
 100
 101 static boolean_t*
 102 proc_get_wqinitingptr(struct proc *p) {
 103         return &(p->p_wqiniting);
 104 }
 105
 106 static uint64_t
 107 proc_get_register(struct proc *p) {
 108         return (p->p_lflag & P_LREGISTER);
 109 }
 110
 111 static void
 112 proc_set_register(struct proc *p) {
 113         proc_setregister(p);
 114 }
 115
 116 static void*
 117 uthread_get_uukwe(struct uthread *t)
 118 {
 119         return &t->uu_kevent.uu_kwe;
 120 }
 121
 122 static int
 123 uthread_is_cancelled(struct uthread *t)
 124 {
 125         return (t->uu_flag & (UT_CANCELDISABLE | UT_CANCEL | UT_CANCELED)) == UT_CANCEL;
 126 }
 127
 128 static vm_map_t
 129 _current_map(void)
 130 {
 131         return current_map();
 132 }
 133
 134
 135 /* kernel (core) to kext shims */
 136
 137 void
 138 pthread_init(void)
 139 {
 140         if (!pthread_functions) {
 141                 panic("pthread kernel extension not loaded (function table is NULL).");
 142         }
 143         pthread_functions->pthread_init();
 144 }
 145
 146 int
 147 fill_procworkqueue(proc_t p, struct proc_workqueueinfo * pwqinfo)
 148 {
 149         return pthread_functions->fill_procworkqueue(p, pwqinfo);
 150 }
 151
 152 void
 153 workqueue_init_lock(proc_t p)
 154 {
 155         pthread_functions->workqueue_init_lock(p);
 156 }
 157
 158 void
 159 workqueue_destroy_lock(proc_t p)
 160 {
 161         pthread_functions->workqueue_destroy_lock(p);
 162 }
 163
 164 void
 165 workqueue_exit(struct proc *p)
 166 {
 167         pthread_functions->workqueue_exit(p);
 168 }
 169
 170 void
 171 workqueue_mark_exiting(struct proc *p)
 172 {
 173         pthread_functions->workqueue_mark_exiting(p);
 174 }
 175
 176 void
 177 workqueue_thread_yielded(void)
 178 {
 179         pthread_functions->workqueue_thread_yielded();
 180 }
 181
 182 sched_call_t
 183 workqueue_get_sched_callback(void)
 184 {
 185         if (pthread_functions->workqueue_get_sched_callback) {
 186                 return pthread_functions->workqueue_get_sched_callback();
 187         }
 188         return NULL;
 189 }
 190
 191 void
 192 pth_proc_hashinit(proc_t p)
 193 {
 194         pthread_functions->pth_proc_hashinit(p);
 195 }
 196
 197 void
 198 pth_proc_hashdelete(proc_t p)
 199 {
 200         pthread_functions->pth_proc_hashdelete(p);
 201 }
 202
 203 /* syscall shims */
 204 int
 205 bsdthread_create(struct proc *p, struct bsdthread_create_args *uap, user_addr_t *retval)
 206 {
 207         return pthread_functions->bsdthread_create(p, uap->func, uap->func_arg, uap->stack, uap->pthread, uap->flags, retval);
 208 }
 209
 210 int
 211 bsdthread_register(struct proc *p, struct bsdthread_register_args *uap, __unused int32_t *retval)
 212 {
 213         return pthread_functions->bsdthread_register(p, uap->threadstart, uap->wqthread, uap->pthsize, uap->dummy_value,
 214                         uap->targetconc_ptr, uap->dispatchqueue_offset, retval);
 215 }
 216
 217 int
 218 bsdthread_terminate(struct proc *p, struct bsdthread_terminate_args *uap, int32_t *retval)
 219 {
 220         return pthread_functions->bsdthread_terminate(p, uap->stackaddr, uap->freesize, uap->port, uap->sem, retval);
 221 }
 222
 223 int
 224 thread_selfid(struct proc *p, __unused struct thread_selfid_args *uap, uint64_t *retval)
 225 {
 226         return pthread_functions->thread_selfid(p, retval);
 227 }
 228
 229 int
 230 workq_kernreturn(struct proc *p, struct workq_kernreturn_args *uap, int32_t *retval)
 231 {
 232         return pthread_functions->workq_kernreturn(p, uap->options, uap->item, uap->affinity, uap->prio, retval);
 233 }
 234
 235 int
 236 workq_open(struct proc *p, __unused struct workq_open_args  *uap, int32_t *retval)
 237 {
 238         return pthread_functions->workq_open(p, retval);
 239 }
 240
 241 /* pthread synchroniser syscalls */
 242
 243 int
 244 psynch_mutexwait(proc_t p, struct psynch_mutexwait_args *uap, uint32_t *retval)
 245 {
 246         return pthread_functions->psynch_mutexwait(p, uap->mutex, uap->mgen, uap->ugen, uap->tid, uap->flags, retval);
 247 }
 248
 249 int
 250 psynch_mutexdrop(proc_t p, struct psynch_mutexdrop_args *uap, uint32_t *retval)
 251 {
 252         return pthread_functions->psynch_mutexdrop(p, uap->mutex, uap->mgen, uap->ugen, uap->tid, uap->flags, retval);
 253 }
 254
 255 int
 256 psynch_cvbroad(proc_t p, struct psynch_cvbroad_args *uap, uint32_t *retval)
 257 {
 258         return pthread_functions->psynch_cvbroad(p, uap->cv, uap->cvlsgen, uap->cvudgen, uap->flags, uap->mutex, uap->mugen, uap->tid, retval);
 259 }
 260
 261 int
 262 psynch_cvsignal(proc_t p, struct psynch_cvsignal_args *uap, uint32_t *retval)
 263 {
 264         return pthread_functions->psynch_cvsignal(p, uap->cv, uap->cvlsgen, uap->cvugen, uap->thread_port, uap->mutex, uap->mugen, uap->tid, uap->flags, retval);
 265 }
 266
 267 int
 268 psynch_cvwait(proc_t p, struct psynch_cvwait_args * uap, uint32_t * retval)
 269 {
 270         return pthread_functions->psynch_cvwait(p, uap->cv, uap->cvlsgen, uap->cvugen, uap->mutex, uap->mugen, uap->flags, uap->sec, uap->nsec, retval);
 271 }
 272
 273 int
 274 psynch_cvclrprepost(proc_t p, struct psynch_cvclrprepost_args * uap, int *retval)
 275 {
 276         return pthread_functions->psynch_cvclrprepost(p, uap->cv, uap->cvgen, uap->cvugen, uap->cvsgen, uap->prepocnt, uap->preposeq, uap->flags, retval);
 277 }
 278
 279 int
 280 psynch_rw_longrdlock(proc_t p, struct psynch_rw_longrdlock_args * uap,  uint32_t *retval)
 281 {
 282         return pthread_functions->psynch_rw_longrdlock(p, uap->rwlock, uap->lgenval, uap->ugenval, uap->rw_wc, uap->flags, retval);
 283 }
 284
 285 int
 286 psynch_rw_rdlock(proc_t p, struct psynch_rw_rdlock_args * uap, uint32_t * retval)
 287 {
 288         return pthread_functions->psynch_rw_rdlock(p, uap->rwlock, uap->lgenval, uap->ugenval, uap->rw_wc, uap->flags, retval);
 289 }
 290
 291 int
 292 psynch_rw_unlock(proc_t p, struct psynch_rw_unlock_args *uap, uint32_t *retval)
 293 {
 294         return pthread_functions->psynch_rw_unlock(p, uap->rwlock, uap->lgenval, uap->ugenval, uap->rw_wc, uap->flags, retval);
 295 }
 296
 297 int
 298 psynch_rw_unlock2(__unused proc_t p, __unused struct psynch_rw_unlock2_args *uap, __unused uint32_t *retval)
 299 {
 300         return ENOTSUP;
 301 }
 302
 303 int
 304 psynch_rw_wrlock(proc_t p, struct psynch_rw_wrlock_args *uap, uint32_t *retval)
 305 {
 306         return pthread_functions->psynch_rw_wrlock(p, uap->rwlock, uap->lgenval, uap->ugenval, uap->rw_wc, uap->flags, retval);
 307 }
 308
 309 int
 310 psynch_rw_yieldwrlock(proc_t p, struct psynch_rw_yieldwrlock_args *uap, uint32_t *retval)
 311 {
 312         return pthread_functions->psynch_rw_yieldwrlock(p, uap->rwlock, uap->lgenval, uap->ugenval, uap->rw_wc, uap->flags, retval);
 313 }
 314
 315 int
 316 psynch_rw_upgrade(__unused proc_t p, __unused struct psynch_rw_upgrade_args * uap, __unused uint32_t *retval)
 317 {
 318         return 0;
 319 }
 320
 321 int
 322 psynch_rw_downgrade(__unused proc_t p, __unused struct psynch_rw_downgrade_args * uap, __unused int *retval)
 323 {
 324         return 0;
 325 }
 326
 327 /* unimplemented guard */
 328
 329 // static void
 330 // unhooked_panic(void)
 331 // {
 332 //      panic("pthread system call not hooked up");
 333 // }
 334
 335 /*
 336  * The callbacks structure (defined in pthread_shims.h) contains a collection
 337  * of kernel functions that were not deemed sensible to expose as a KPI to all
 338  * kernel extensions. So the kext is given them in the form of a structure of
 339  * function pointers.
 340  */
 341 static struct pthread_callbacks_s pthread_callbacks = {
 342         .version = PTHREAD_SHIMS_VERSION,
 343         .config_thread_max = CONFIG_THREAD_MAX,
 344         .get_task_threadmax = get_task_threadmax,
 345
 346         .proc_get_threadstart = proc_get_threadstart,
 347         .proc_set_threadstart = proc_set_threadstart,
 348         .proc_get_pthsize = proc_get_pthsize,
 349         .proc_set_pthsize = proc_set_pthsize,
 350         .proc_get_wqthread = proc_get_wqthread,
 351         .proc_set_wqthread = proc_set_wqthread,
 352         .proc_get_targconc = proc_get_targconc,
 353         .proc_set_targconc = proc_set_targconc,
 354         .proc_get_dispatchqueue_offset = proc_get_dispatchqueue_offset,
 355         .proc_set_dispatchqueue_offset = proc_set_dispatchqueue_offset,
 356         .proc_get_wqptr = proc_get_wqptr,
 357         .proc_set_wqptr = proc_set_wqptr,
 358         .proc_get_wqsize = proc_get_wqsize,
 359         .proc_set_wqsize = proc_set_wqsize,
 360         .proc_get_wqlockptr = proc_get_wqlockptr,
 361         .proc_get_wqinitingptr = proc_get_wqinitingptr,
 362         .proc_get_pthhash = proc_get_pthhash,
 363         .proc_set_pthhash = proc_set_pthhash,
 364         .proc_get_task = proc_get_task,
 365         .proc_lock = proc_lock,
 366         .proc_unlock = proc_unlock,
 367         .proc_get_register = proc_get_register,
 368         .proc_set_register = proc_set_register,
 369
 370         /* kernel IPI interfaces */
 371         .ipc_port_copyout_send = ipc_port_copyout_send,
 372         .task_get_ipcspace = get_task_ipcspace,
 373         .vm_map_page_info = vm_map_page_info,
 374         .vm_map_switch = vm_map_switch,
 375         .thread_set_wq_state32 = thread_set_wq_state32,
 376         .thread_set_wq_state64 = thread_set_wq_state64,
 377
 378         .uthread_get_threadlist = uthread_get_threadlist,
 379         .uthread_set_threadlist = uthread_set_threadlist,
 380         .uthread_get_sigmask = uthread_get_sigmask,
 381         .uthread_set_sigmask = uthread_set_sigmask,
 382         .uthread_get_uukwe = uthread_get_uukwe,
 383         .uthread_get_returnval = uthread_get_returnval,
 384         .uthread_set_returnval = uthread_set_returnval,
 385         .uthread_is_cancelled = uthread_is_cancelled,
 386
 387         .thread_exception_return = pthread_returning_to_userspace,
 388         .thread_bootstrap_return = thread_bootstrap_return,
 389         .unix_syscall_return = unix_syscall_return,
 390
 391         .absolutetime_to_microtime = absolutetime_to_microtime,
 392
 393         .proc_restore_workq_bgthreadpolicy = proc_restore_workq_bgthreadpolicy,
 394         .proc_apply_workq_bgthreadpolicy = proc_apply_workq_bgthreadpolicy,
 395
 396         .get_bsdthread_info = (void*)get_bsdthread_info,
 397         .thread_sched_call = thread_sched_call,
 398         .thread_static_param = thread_static_param,
 399         .thread_create_workq = thread_create_workq,
 400         .thread_policy_set_internal = thread_policy_set_internal,
 401
 402         .thread_affinity_set = thread_affinity_set,
 403
 404         .zalloc = zalloc,
 405         .zfree = zfree,
 406         .zinit = zinit,
 407
 408         .__pthread_testcancel = __pthread_testcancel,
 409
 410         .mach_port_deallocate = mach_port_deallocate,
 411         .semaphore_signal_internal_trap = semaphore_signal_internal_trap,
 412         .current_map = _current_map,
 413         .thread_create = thread_create,
 414         .thread_resume = thread_resume,
 415
 416         .convert_thread_to_port = convert_thread_to_port,
 417         .ml_get_max_cpus = (void*)ml_get_max_cpus,
 418
 419
 420         .proc_get_dispatchqueue_serialno_offset = proc_get_dispatchqueue_serialno_offset,
 421         .proc_set_dispatchqueue_serialno_offset = proc_set_dispatchqueue_serialno_offset,
 422 };
 423
 424 pthread_callbacks_t pthread_kern = &pthread_callbacks;
 425 pthread_functions_t pthread_functions = NULL;
 426
 427 /*
 428  * pthread_kext_register is called by pthread.kext upon load, it has to provide
 429  * us with a function pointer table of pthread internal calls. In return, this
 430  * file provides it with a table of function pointers it needs.
 431  */
 432
 433 void
 434 pthread_kext_register(pthread_functions_t fns, pthread_callbacks_t *callbacks)
 435 {
 436         if (pthread_functions != NULL) {
 437                 panic("Re-initialisation of pthread kext callbacks.");
 438         }
 439
 440         if (callbacks != NULL) {
 441                 *callbacks = &pthread_callbacks;
 442         } else {
 443                 panic("pthread_kext_register called without callbacks pointer.");
 444         }
 445
 446         if (fns) {
 447                 pthread_functions = fns;
 448         }
 449 }