]>
Commit | Line | Data |
---|---|---|
39236c6e | 1 | /* |
39037602 | 2 | * Copyright (c) 2012-2016 Apple Inc. All rights reserved. |
39236c6e A |
3 | * |
4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ | |
d9a64523 | 5 | * |
39236c6e A |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License | |
8 | * Version 2.0 (the 'License'). You may not use this file except in | |
9 | * compliance with the License. The rights granted to you under the License | |
10 | * may not be used to create, or enable the creation or redistribution of, | |
11 | * unlawful or unlicensed copies of an Apple operating system, or to | |
12 | * circumvent, violate, or enable the circumvention or violation of, any | |
13 | * terms of an Apple operating system software license agreement. | |
d9a64523 | 14 | * |
39236c6e A |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. | |
d9a64523 | 17 | * |
39236c6e A |
18 | * The Original Code and all software distributed under the License are |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER | |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, | |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, | |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. | |
23 | * Please see the License for the specific language governing rights and | |
24 | * limitations under the License. | |
d9a64523 | 25 | * |
39236c6e A |
26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
27 | */ | |
d9a64523 | 28 | |
39236c6e A |
29 | #define PTHREAD_INTERNAL 1 |
30 | ||
5ba3f43e | 31 | #include <stdatomic.h> |
39236c6e A |
32 | #include <kern/debug.h> |
33 | #include <kern/mach_param.h> | |
34 | #include <kern/sched_prim.h> | |
35 | #include <kern/task.h> | |
36 | #include <kern/thread.h> | |
37 | #include <kern/affinity.h> | |
38 | #include <kern/zalloc.h> | |
39037602 A |
39 | #include <kern/policy_internal.h> |
40 | ||
39236c6e A |
41 | #include <machine/machine_routines.h> |
42 | #include <mach/task.h> | |
43 | #include <mach/thread_act.h> | |
44 | #include <sys/param.h> | |
5ba3f43e | 45 | #include <sys/eventvar.h> |
39236c6e | 46 | #include <sys/pthread_shims.h> |
d9a64523 A |
47 | #include <pthread/workqueue_internal.h> |
48 | #include <sys/cdefs.h> | |
5ba3f43e | 49 | #include <sys/proc_info.h> |
39236c6e A |
50 | #include <sys/proc_internal.h> |
51 | #include <sys/sysproto.h> | |
52 | #include <sys/systm.h> | |
53 | #include <vm/vm_map.h> | |
54 | #include <vm/vm_protos.h> | |
39037602 | 55 | #include <kern/kcdata.h> |
39236c6e A |
56 | |
57 | /* version number of the in-kernel shims given to pthread.kext */ | |
58 | #define PTHREAD_SHIMS_VERSION 1 | |
59 | ||
fe8ab488 | 60 | /* on arm, the callbacks function has two #ifdef arm ponters */ |
5ba3f43e | 61 | #if defined(__arm__) |
d9a64523 | 62 | #define PTHREAD_CALLBACK_MEMBER __unused_was_map_is_1gb |
5ba3f43e | 63 | #else |
d9a64523 | 64 | #define PTHREAD_CALLBACK_MEMBER __unused_was_ml_get_max_cpus |
5ba3f43e | 65 | #endif |
fe8ab488 A |
66 | |
67 | /* compile time asserts to check the length of structures in pthread_shims.h */ | |
39037602 A |
68 | static_assert((sizeof(struct pthread_functions_s) - offsetof(struct pthread_functions_s, psynch_rw_yieldwrlock) - sizeof(void*)) == (sizeof(void*) * 100)); |
69 | static_assert((sizeof(struct pthread_callbacks_s) - offsetof(struct pthread_callbacks_s, PTHREAD_CALLBACK_MEMBER) - sizeof(void*)) == (sizeof(void*) * 100)); | |
fe8ab488 | 70 | |
39236c6e A |
71 | /* old pthread code had definitions for these as they don't exist in headers */ |
72 | extern kern_return_t mach_port_deallocate(ipc_space_t, mach_port_name_t); | |
73 | extern kern_return_t semaphore_signal_internal_trap(mach_port_name_t); | |
d9a64523 | 74 | extern void thread_deallocate_safe(thread_t thread); |
39236c6e A |
75 | |
76 | #define PTHREAD_STRUCT_ACCESSOR(get, set, rettype, structtype, member) \ | |
77 | static rettype \ | |
78 | get(structtype x) { \ | |
79 | return (x)->member; \ | |
80 | } \ | |
81 | static void \ | |
82 | set(structtype x, rettype y) { \ | |
83 | (x)->member = y; \ | |
84 | } | |
39037602 | 85 | |
39236c6e A |
86 | PTHREAD_STRUCT_ACCESSOR(proc_get_threadstart, proc_set_threadstart, user_addr_t, struct proc*, p_threadstart); |
87 | PTHREAD_STRUCT_ACCESSOR(proc_get_pthsize, proc_set_pthsize, int, struct proc*, p_pthsize); | |
88 | PTHREAD_STRUCT_ACCESSOR(proc_get_wqthread, proc_set_wqthread, user_addr_t, struct proc*, p_wqthread); | |
fe8ab488 | 89 | PTHREAD_STRUCT_ACCESSOR(proc_get_stack_addr_hint, proc_set_stack_addr_hint, user_addr_t, struct proc *, p_stack_addr_hint); |
fe8ab488 | 90 | PTHREAD_STRUCT_ACCESSOR(proc_get_pthread_tsd_offset, proc_set_pthread_tsd_offset, uint32_t, struct proc *, p_pth_tsd_offset); |
5ba3f43e | 91 | PTHREAD_STRUCT_ACCESSOR(proc_get_mach_thread_self_tsd_offset, proc_set_mach_thread_self_tsd_offset, uint64_t, struct proc *, p_mach_thread_self_offset); |
39236c6e | 92 | PTHREAD_STRUCT_ACCESSOR(proc_get_pthhash, proc_set_pthhash, void*, struct proc*, p_pthhash); |
39236c6e | 93 | |
39037602 A |
94 | #define WQPTR_IS_INITING_VALUE ((void *)~(uintptr_t)0) |
95 | ||
39037602 | 96 | static void |
d9a64523 A |
97 | proc_set_dispatchqueue_offset(struct proc *p, uint64_t offset) |
98 | { | |
99 | p->p_dispatchqueue_offset = offset; | |
39037602 | 100 | } |
39037602 | 101 | |
d9a64523 A |
102 | static void |
103 | proc_set_return_to_kernel_offset(struct proc *p, uint64_t offset) | |
104 | { | |
105 | p->p_return_to_kernel_offset = offset; | |
106 | } | |
39037602 | 107 | |
d9a64523 A |
108 | static user_addr_t |
109 | proc_get_user_stack(struct proc *p) | |
110 | { | |
111 | return p->user_stack; | |
112 | } | |
39037602 | 113 | |
d9a64523 A |
114 | static void |
115 | uthread_set_returnval(struct uthread *uth, int retval) | |
116 | { | |
117 | uth->uu_rval[0] = retval; | |
39037602 A |
118 | } |
119 | ||
120 | __attribute__((noreturn)) | |
39236c6e A |
121 | static void |
122 | pthread_returning_to_userspace(void) | |
123 | { | |
124 | thread_exception_return(); | |
125 | } | |
126 | ||
a39ff7e2 A |
127 | __attribute__((noreturn)) |
128 | static void | |
129 | pthread_bootstrap_return(void) | |
130 | { | |
131 | thread_bootstrap_return(); | |
132 | } | |
133 | ||
39236c6e A |
134 | static uint32_t |
135 | get_task_threadmax(void) { | |
136 | return task_threadmax; | |
137 | } | |
138 | ||
39236c6e A |
139 | static uint64_t |
140 | proc_get_register(struct proc *p) { | |
141 | return (p->p_lflag & P_LREGISTER); | |
142 | } | |
143 | ||
144 | static void | |
145 | proc_set_register(struct proc *p) { | |
146 | proc_setregister(p); | |
147 | } | |
148 | ||
149 | static void* | |
150 | uthread_get_uukwe(struct uthread *t) | |
151 | { | |
d9a64523 | 152 | return &t->uu_save.uus_kwe; |
39236c6e A |
153 | } |
154 | ||
155 | static int | |
156 | uthread_is_cancelled(struct uthread *t) | |
157 | { | |
158 | return (t->uu_flag & (UT_CANCELDISABLE | UT_CANCEL | UT_CANCELED)) == UT_CANCEL; | |
159 | } | |
160 | ||
161 | static vm_map_t | |
162 | _current_map(void) | |
163 | { | |
164 | return current_map(); | |
165 | } | |
166 | ||
fe8ab488 A |
167 | static boolean_t |
168 | qos_main_thread_active(void) | |
169 | { | |
170 | return TRUE; | |
171 | } | |
172 | ||
fe8ab488 A |
173 | static int proc_usynch_get_requested_thread_qos(struct uthread *uth) |
174 | { | |
fe8ab488 A |
175 | thread_t thread = uth ? uth->uu_thread : current_thread(); |
176 | int requested_qos; | |
177 | ||
39037602 | 178 | requested_qos = proc_get_thread_policy(thread, TASK_POLICY_ATTRIBUTE, TASK_POLICY_QOS); |
fe8ab488 A |
179 | |
180 | /* | |
39037602 A |
181 | * For the purposes of userspace synchronization, it doesn't make sense to |
182 | * place an override of UNSPECIFIED on another thread, if the current thread | |
183 | * doesn't have any QoS set. In these cases, upgrade to | |
fe8ab488 A |
184 | * THREAD_QOS_USER_INTERACTIVE. |
185 | */ | |
186 | if (requested_qos == THREAD_QOS_UNSPECIFIED) { | |
187 | requested_qos = THREAD_QOS_USER_INTERACTIVE; | |
188 | } | |
189 | ||
190 | return requested_qos; | |
191 | } | |
192 | ||
39037602 A |
193 | static boolean_t |
194 | proc_usynch_thread_qos_add_override_for_resource(task_t task, struct uthread *uth, | |
195 | uint64_t tid, int override_qos, boolean_t first_override_for_resource, | |
196 | user_addr_t resource, int resource_type) | |
fe8ab488 | 197 | { |
fe8ab488 A |
198 | thread_t thread = uth ? uth->uu_thread : THREAD_NULL; |
199 | ||
39037602 | 200 | return proc_thread_qos_add_override(task, thread, tid, override_qos, |
d9a64523 | 201 | first_override_for_resource, resource, resource_type) == 0; |
a1c7dba1 A |
202 | } |
203 | ||
39037602 A |
204 | static boolean_t |
205 | proc_usynch_thread_qos_remove_override_for_resource(task_t task, | |
206 | struct uthread *uth, uint64_t tid, user_addr_t resource, int resource_type) | |
a1c7dba1 A |
207 | { |
208 | thread_t thread = uth ? uth->uu_thread : THREAD_NULL; | |
209 | ||
d9a64523 A |
210 | return proc_thread_qos_remove_override(task, thread, tid, resource, |
211 | resource_type) == 0; | |
a1c7dba1 A |
212 | } |
213 | ||
d9a64523 A |
214 | |
215 | static wait_result_t | |
216 | psynch_wait_prepare(uintptr_t kwq, struct turnstile **tstore, | |
217 | thread_t owner, block_hint_t block_hint, uint64_t deadline) | |
a1c7dba1 | 218 | { |
d9a64523 A |
219 | struct turnstile *ts; |
220 | wait_result_t wr; | |
a1c7dba1 | 221 | |
d9a64523 A |
222 | if (tstore) { |
223 | ts = turnstile_prepare(kwq, tstore, TURNSTILE_NULL, | |
224 | TURNSTILE_PTHREAD_MUTEX); | |
39236c6e | 225 | |
d9a64523 A |
226 | turnstile_update_inheritor(ts, owner, |
227 | (TURNSTILE_DELAYED_UPDATE | TURNSTILE_INHERITOR_THREAD)); | |
39037602 | 228 | |
d9a64523 | 229 | thread_set_pending_block_hint(current_thread(), block_hint); |
39236c6e | 230 | |
d9a64523 A |
231 | wr = waitq_assert_wait64_leeway(&ts->ts_waitq, (event64_t)kwq, |
232 | THREAD_ABORTSAFE, TIMEOUT_URGENCY_USER_NORMAL, deadline, 0); | |
233 | } else { | |
234 | thread_set_pending_block_hint(current_thread(), block_hint); | |
235 | ||
236 | wr = assert_wait_deadline_with_leeway((event_t)kwq, THREAD_ABORTSAFE, | |
237 | TIMEOUT_URGENCY_USER_NORMAL, deadline, 0); | |
39236c6e | 238 | } |
d9a64523 A |
239 | |
240 | return wr; | |
39236c6e A |
241 | } |
242 | ||
d9a64523 A |
243 | static void |
244 | psynch_wait_update_complete(struct turnstile *ts) | |
39236c6e | 245 | { |
d9a64523 A |
246 | assert(ts); |
247 | turnstile_update_inheritor_complete(ts, TURNSTILE_INTERLOCK_NOT_HELD); | |
39236c6e A |
248 | } |
249 | ||
d9a64523 A |
250 | static void |
251 | psynch_wait_complete(uintptr_t kwq, struct turnstile **tstore) | |
39236c6e | 252 | { |
d9a64523 A |
253 | assert(tstore); |
254 | turnstile_complete(kwq, tstore, NULL); | |
255 | } | |
39037602 | 256 | |
d9a64523 A |
257 | static void |
258 | psynch_wait_update_owner(uintptr_t kwq, thread_t owner, | |
259 | struct turnstile **tstore) | |
260 | { | |
261 | struct turnstile *ts; | |
39037602 | 262 | |
d9a64523 A |
263 | ts = turnstile_prepare(kwq, tstore, TURNSTILE_NULL, |
264 | TURNSTILE_PTHREAD_MUTEX); | |
39037602 | 265 | |
d9a64523 A |
266 | turnstile_update_inheritor(ts, owner, |
267 | (TURNSTILE_IMMEDIATE_UPDATE | TURNSTILE_INHERITOR_THREAD)); | |
268 | turnstile_update_inheritor_complete(ts, TURNSTILE_INTERLOCK_HELD); | |
269 | turnstile_complete(kwq, tstore, NULL); | |
39236c6e A |
270 | } |
271 | ||
d9a64523 A |
272 | static void |
273 | psynch_wait_cleanup(void) | |
39236c6e | 274 | { |
d9a64523 | 275 | turnstile_cleanup(); |
39236c6e A |
276 | } |
277 | ||
d9a64523 A |
278 | static kern_return_t |
279 | psynch_wait_wakeup(uintptr_t kwq, struct ksyn_waitq_element *kwe, | |
280 | struct turnstile **tstore) | |
39236c6e | 281 | { |
d9a64523 A |
282 | struct uthread *uth; |
283 | struct turnstile *ts; | |
284 | kern_return_t kr; | |
39236c6e | 285 | |
d9a64523 A |
286 | uth = __container_of(kwe, struct uthread, uu_save.uus_kwe); |
287 | assert(uth); | |
39236c6e | 288 | |
d9a64523 A |
289 | if (tstore) { |
290 | ts = turnstile_prepare(kwq, tstore, TURNSTILE_NULL, | |
291 | TURNSTILE_PTHREAD_MUTEX); | |
292 | turnstile_update_inheritor(ts, uth->uu_thread, | |
293 | (TURNSTILE_IMMEDIATE_UPDATE | TURNSTILE_INHERITOR_THREAD)); | |
294 | ||
295 | kr = waitq_wakeup64_thread(&ts->ts_waitq, (event64_t)kwq, | |
296 | uth->uu_thread, THREAD_AWAKENED); | |
297 | ||
298 | turnstile_update_inheritor_complete(ts, TURNSTILE_INTERLOCK_HELD); | |
299 | turnstile_complete(kwq, tstore, NULL); | |
300 | } else { | |
301 | kr = thread_wakeup_thread((event_t)kwq, uth->uu_thread); | |
302 | } | |
303 | ||
304 | return kr; | |
39236c6e A |
305 | } |
306 | ||
d9a64523 A |
307 | /* kernel (core) to kext shims */ |
308 | ||
309 | void | |
310 | pthread_init(void) | |
39236c6e | 311 | { |
d9a64523 A |
312 | if (!pthread_functions) { |
313 | panic("pthread kernel extension not loaded (function table is NULL)."); | |
39236c6e | 314 | } |
d9a64523 | 315 | pthread_functions->pthread_init(); |
39236c6e A |
316 | } |
317 | ||
318 | void | |
319 | pth_proc_hashinit(proc_t p) | |
320 | { | |
321 | pthread_functions->pth_proc_hashinit(p); | |
322 | } | |
323 | ||
324 | void | |
325 | pth_proc_hashdelete(proc_t p) | |
326 | { | |
327 | pthread_functions->pth_proc_hashdelete(p); | |
328 | } | |
329 | ||
330 | /* syscall shims */ | |
331 | int | |
332 | bsdthread_create(struct proc *p, struct bsdthread_create_args *uap, user_addr_t *retval) | |
333 | { | |
334 | return pthread_functions->bsdthread_create(p, uap->func, uap->func_arg, uap->stack, uap->pthread, uap->flags, retval); | |
335 | } | |
336 | ||
337 | int | |
338 | bsdthread_register(struct proc *p, struct bsdthread_register_args *uap, __unused int32_t *retval) | |
339 | { | |
d9a64523 A |
340 | kern_return_t kr; |
341 | static_assert(offsetof(struct bsdthread_register_args, threadstart) + sizeof(user_addr_t) == | |
342 | offsetof(struct bsdthread_register_args, wqthread)); | |
343 | kr = machine_thread_function_pointers_convert_from_user(current_thread(), &uap->threadstart, 2); | |
344 | assert(kr == KERN_SUCCESS); | |
345 | ||
fe8ab488 | 346 | if (pthread_functions->version >= 1) { |
d9a64523 A |
347 | return pthread_functions->bsdthread_register2(p, uap->threadstart, |
348 | uap->wqthread, uap->flags, uap->stack_addr_hint, | |
349 | uap->targetconc_ptr, uap->dispatchqueue_offset, | |
350 | uap->tsd_offset, retval); | |
fe8ab488 | 351 | } else { |
d9a64523 A |
352 | return pthread_functions->bsdthread_register(p, uap->threadstart, |
353 | uap->wqthread, uap->flags, uap->stack_addr_hint, | |
354 | uap->targetconc_ptr, uap->dispatchqueue_offset, | |
355 | retval); | |
fe8ab488 | 356 | } |
39236c6e A |
357 | } |
358 | ||
359 | int | |
360 | bsdthread_terminate(struct proc *p, struct bsdthread_terminate_args *uap, int32_t *retval) | |
361 | { | |
d9a64523 A |
362 | thread_t th = current_thread(); |
363 | if (thread_get_tag(th) & THREAD_TAG_WORKQUEUE) { | |
364 | workq_thread_terminate(p, get_bsdthread_info(th)); | |
365 | } | |
39236c6e A |
366 | return pthread_functions->bsdthread_terminate(p, uap->stackaddr, uap->freesize, uap->port, uap->sem, retval); |
367 | } | |
368 | ||
369 | int | |
370 | thread_selfid(struct proc *p, __unused struct thread_selfid_args *uap, uint64_t *retval) | |
371 | { | |
372 | return pthread_functions->thread_selfid(p, retval); | |
373 | } | |
374 | ||
39236c6e A |
375 | /* pthread synchroniser syscalls */ |
376 | ||
377 | int | |
378 | psynch_mutexwait(proc_t p, struct psynch_mutexwait_args *uap, uint32_t *retval) | |
379 | { | |
380 | return pthread_functions->psynch_mutexwait(p, uap->mutex, uap->mgen, uap->ugen, uap->tid, uap->flags, retval); | |
381 | } | |
382 | ||
383 | int | |
384 | psynch_mutexdrop(proc_t p, struct psynch_mutexdrop_args *uap, uint32_t *retval) | |
385 | { | |
386 | return pthread_functions->psynch_mutexdrop(p, uap->mutex, uap->mgen, uap->ugen, uap->tid, uap->flags, retval); | |
387 | } | |
388 | ||
389 | int | |
390 | psynch_cvbroad(proc_t p, struct psynch_cvbroad_args *uap, uint32_t *retval) | |
391 | { | |
392 | return pthread_functions->psynch_cvbroad(p, uap->cv, uap->cvlsgen, uap->cvudgen, uap->flags, uap->mutex, uap->mugen, uap->tid, retval); | |
393 | } | |
394 | ||
395 | int | |
396 | psynch_cvsignal(proc_t p, struct psynch_cvsignal_args *uap, uint32_t *retval) | |
397 | { | |
398 | return pthread_functions->psynch_cvsignal(p, uap->cv, uap->cvlsgen, uap->cvugen, uap->thread_port, uap->mutex, uap->mugen, uap->tid, uap->flags, retval); | |
399 | } | |
400 | ||
401 | int | |
402 | psynch_cvwait(proc_t p, struct psynch_cvwait_args * uap, uint32_t * retval) | |
403 | { | |
404 | return pthread_functions->psynch_cvwait(p, uap->cv, uap->cvlsgen, uap->cvugen, uap->mutex, uap->mugen, uap->flags, uap->sec, uap->nsec, retval); | |
405 | } | |
406 | ||
407 | int | |
408 | psynch_cvclrprepost(proc_t p, struct psynch_cvclrprepost_args * uap, int *retval) | |
409 | { | |
410 | return pthread_functions->psynch_cvclrprepost(p, uap->cv, uap->cvgen, uap->cvugen, uap->cvsgen, uap->prepocnt, uap->preposeq, uap->flags, retval); | |
411 | } | |
412 | ||
413 | int | |
414 | psynch_rw_longrdlock(proc_t p, struct psynch_rw_longrdlock_args * uap, uint32_t *retval) | |
415 | { | |
416 | return pthread_functions->psynch_rw_longrdlock(p, uap->rwlock, uap->lgenval, uap->ugenval, uap->rw_wc, uap->flags, retval); | |
417 | } | |
418 | ||
419 | int | |
420 | psynch_rw_rdlock(proc_t p, struct psynch_rw_rdlock_args * uap, uint32_t * retval) | |
421 | { | |
422 | return pthread_functions->psynch_rw_rdlock(p, uap->rwlock, uap->lgenval, uap->ugenval, uap->rw_wc, uap->flags, retval); | |
423 | } | |
424 | ||
425 | int | |
426 | psynch_rw_unlock(proc_t p, struct psynch_rw_unlock_args *uap, uint32_t *retval) | |
427 | { | |
428 | return pthread_functions->psynch_rw_unlock(p, uap->rwlock, uap->lgenval, uap->ugenval, uap->rw_wc, uap->flags, retval); | |
429 | } | |
430 | ||
431 | int | |
432 | psynch_rw_unlock2(__unused proc_t p, __unused struct psynch_rw_unlock2_args *uap, __unused uint32_t *retval) | |
433 | { | |
434 | return ENOTSUP; | |
435 | } | |
436 | ||
437 | int | |
438 | psynch_rw_wrlock(proc_t p, struct psynch_rw_wrlock_args *uap, uint32_t *retval) | |
439 | { | |
440 | return pthread_functions->psynch_rw_wrlock(p, uap->rwlock, uap->lgenval, uap->ugenval, uap->rw_wc, uap->flags, retval); | |
441 | } | |
442 | ||
443 | int | |
444 | psynch_rw_yieldwrlock(proc_t p, struct psynch_rw_yieldwrlock_args *uap, uint32_t *retval) | |
445 | { | |
446 | return pthread_functions->psynch_rw_yieldwrlock(p, uap->rwlock, uap->lgenval, uap->ugenval, uap->rw_wc, uap->flags, retval); | |
447 | } | |
448 | ||
449 | int | |
450 | psynch_rw_upgrade(__unused proc_t p, __unused struct psynch_rw_upgrade_args * uap, __unused uint32_t *retval) | |
451 | { | |
452 | return 0; | |
453 | } | |
454 | ||
455 | int | |
456 | psynch_rw_downgrade(__unused proc_t p, __unused struct psynch_rw_downgrade_args * uap, __unused int *retval) | |
457 | { | |
458 | return 0; | |
459 | } | |
460 | ||
813fb2f6 A |
461 | void |
462 | kdp_pthread_find_owner(thread_t thread, struct stackshot_thread_waitinfo *waitinfo) | |
463 | { | |
464 | if (pthread_functions->pthread_find_owner) | |
465 | pthread_functions->pthread_find_owner(thread, waitinfo); | |
466 | } | |
467 | ||
468 | void * | |
469 | kdp_pthread_get_thread_kwq(thread_t thread) | |
470 | { | |
471 | if (pthread_functions->pthread_get_thread_kwq) | |
472 | return pthread_functions->pthread_get_thread_kwq(thread); | |
473 | ||
474 | return NULL; | |
475 | } | |
476 | ||
d9a64523 | 477 | void |
5ba3f43e A |
478 | thread_will_park_or_terminate(thread_t thread) |
479 | { | |
480 | if (thread_owned_workloops_count(thread)) { | |
481 | (void)kevent_exit_on_workloop_ownership_leak(thread); | |
482 | } | |
483 | } | |
484 | ||
39236c6e A |
485 | /* |
486 | * The callbacks structure (defined in pthread_shims.h) contains a collection | |
487 | * of kernel functions that were not deemed sensible to expose as a KPI to all | |
488 | * kernel extensions. So the kext is given them in the form of a structure of | |
489 | * function pointers. | |
490 | */ | |
39037602 | 491 | static const struct pthread_callbacks_s pthread_callbacks = { |
39236c6e A |
492 | .version = PTHREAD_SHIMS_VERSION, |
493 | .config_thread_max = CONFIG_THREAD_MAX, | |
494 | .get_task_threadmax = get_task_threadmax, | |
495 | ||
496 | .proc_get_threadstart = proc_get_threadstart, | |
497 | .proc_set_threadstart = proc_set_threadstart, | |
498 | .proc_get_pthsize = proc_get_pthsize, | |
499 | .proc_set_pthsize = proc_set_pthsize, | |
500 | .proc_get_wqthread = proc_get_wqthread, | |
501 | .proc_set_wqthread = proc_set_wqthread, | |
39236c6e | 502 | .proc_set_dispatchqueue_offset = proc_set_dispatchqueue_offset, |
39037602 | 503 | .proc_get_pthhash = proc_get_pthhash, |
39236c6e | 504 | .proc_set_pthhash = proc_set_pthhash, |
39236c6e A |
505 | .proc_get_register = proc_get_register, |
506 | .proc_set_register = proc_set_register, | |
507 | ||
508 | /* kernel IPI interfaces */ | |
509 | .ipc_port_copyout_send = ipc_port_copyout_send, | |
510 | .task_get_ipcspace = get_task_ipcspace, | |
511 | .vm_map_page_info = vm_map_page_info, | |
39236c6e | 512 | .thread_set_wq_state32 = thread_set_wq_state32, |
5ba3f43e | 513 | #if !defined(__arm__) |
39236c6e | 514 | .thread_set_wq_state64 = thread_set_wq_state64, |
5ba3f43e | 515 | #endif |
39236c6e | 516 | |
39236c6e | 517 | .uthread_get_uukwe = uthread_get_uukwe, |
39236c6e A |
518 | .uthread_set_returnval = uthread_set_returnval, |
519 | .uthread_is_cancelled = uthread_is_cancelled, | |
39037602 | 520 | |
39236c6e | 521 | .thread_exception_return = pthread_returning_to_userspace, |
a39ff7e2 | 522 | .thread_bootstrap_return = pthread_bootstrap_return, |
39236c6e A |
523 | .unix_syscall_return = unix_syscall_return, |
524 | ||
39236c6e | 525 | .get_bsdthread_info = (void*)get_bsdthread_info, |
39236c6e | 526 | .thread_policy_set_internal = thread_policy_set_internal, |
fe8ab488 | 527 | .thread_policy_get = thread_policy_get, |
5ba3f43e | 528 | |
39236c6e A |
529 | .__pthread_testcancel = __pthread_testcancel, |
530 | ||
531 | .mach_port_deallocate = mach_port_deallocate, | |
532 | .semaphore_signal_internal_trap = semaphore_signal_internal_trap, | |
533 | .current_map = _current_map, | |
534 | .thread_create = thread_create, | |
535 | .thread_resume = thread_resume, | |
39037602 | 536 | |
39236c6e | 537 | .convert_thread_to_port = convert_thread_to_port, |
fe8ab488 A |
538 | |
539 | .proc_get_stack_addr_hint = proc_get_stack_addr_hint, | |
540 | .proc_set_stack_addr_hint = proc_set_stack_addr_hint, | |
541 | .proc_get_pthread_tsd_offset = proc_get_pthread_tsd_offset, | |
542 | .proc_set_pthread_tsd_offset = proc_set_pthread_tsd_offset, | |
5ba3f43e A |
543 | .proc_get_mach_thread_self_tsd_offset = proc_get_mach_thread_self_tsd_offset, |
544 | .proc_set_mach_thread_self_tsd_offset = proc_set_mach_thread_self_tsd_offset, | |
fe8ab488 A |
545 | |
546 | .thread_set_tsd_base = thread_set_tsd_base, | |
547 | ||
548 | .proc_usynch_get_requested_thread_qos = proc_usynch_get_requested_thread_qos, | |
fe8ab488 A |
549 | |
550 | .qos_main_thread_active = qos_main_thread_active, | |
d9a64523 | 551 | .thread_set_voucher_name = thread_set_voucher_name, |
a1c7dba1 A |
552 | |
553 | .proc_usynch_thread_qos_add_override_for_resource = proc_usynch_thread_qos_add_override_for_resource, | |
554 | .proc_usynch_thread_qos_remove_override_for_resource = proc_usynch_thread_qos_remove_override_for_resource, | |
39037602 A |
555 | |
556 | .thread_set_tag = thread_set_tag, | |
557 | .thread_get_tag = thread_get_tag, | |
558 | ||
5ba3f43e A |
559 | .proc_set_return_to_kernel_offset = proc_set_return_to_kernel_offset, |
560 | .thread_will_park_or_terminate = thread_will_park_or_terminate, | |
561 | ||
5ba3f43e | 562 | .proc_get_user_stack = proc_get_user_stack, |
d9a64523 A |
563 | .task_findtid = task_findtid, |
564 | .thread_deallocate_safe = thread_deallocate_safe, | |
565 | ||
566 | .psynch_wait_prepare = psynch_wait_prepare, | |
567 | .psynch_wait_update_complete = psynch_wait_update_complete, | |
568 | .psynch_wait_complete = psynch_wait_complete, | |
569 | .psynch_wait_cleanup = psynch_wait_cleanup, | |
570 | .psynch_wait_wakeup = psynch_wait_wakeup, | |
571 | .psynch_wait_update_owner = psynch_wait_update_owner, | |
39236c6e A |
572 | }; |
573 | ||
574 | pthread_callbacks_t pthread_kern = &pthread_callbacks; | |
575 | pthread_functions_t pthread_functions = NULL; | |
576 | ||
577 | /* | |
578 | * pthread_kext_register is called by pthread.kext upon load, it has to provide | |
579 | * us with a function pointer table of pthread internal calls. In return, this | |
580 | * file provides it with a table of function pointers it needs. | |
581 | */ | |
582 | ||
583 | void | |
584 | pthread_kext_register(pthread_functions_t fns, pthread_callbacks_t *callbacks) | |
585 | { | |
586 | if (pthread_functions != NULL) { | |
587 | panic("Re-initialisation of pthread kext callbacks."); | |
588 | } | |
39037602 | 589 | |
39236c6e A |
590 | if (callbacks != NULL) { |
591 | *callbacks = &pthread_callbacks; | |
592 | } else { | |
593 | panic("pthread_kext_register called without callbacks pointer."); | |
594 | } | |
39037602 | 595 | |
39236c6e A |
596 | if (fns) { |
597 | pthread_functions = fns; | |
598 | } | |
599 | } |