2 * Copyright (c) 2013 Apple Inc. All rights reserved.
4 * @APPLE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
21 * @APPLE_LICENSE_HEADER_END@
27 #include <mach/mach_vm.h>
30 #include <spawn_private.h>
31 #include <sys/spawn_internal.h>
32 #include <sys/ulock.h>
34 // TODO: remove me when internal.h can include *_private.h itself
35 #include "workqueue_private.h"
36 #include "qos_private.h"
38 static pthread_priority_t _main_qos
= QOS_CLASS_UNSPECIFIED
;
40 #define PTHREAD_OVERRIDE_SIGNATURE (0x6f766572)
41 #define PTHREAD_OVERRIDE_SIG_DEAD (0x7265766f)
43 struct pthread_override_s
48 pthread_priority_t priority
;
53 _pthread_set_main_qos(pthread_priority_t qos
)
59 pthread_attr_set_qos_class_np(pthread_attr_t
*__attr
,
60 qos_class_t __qos_class
,
61 int __relative_priority
)
63 if (!(__pthread_supported_features
& PTHREAD_FEATURE_BSDTHREADCTL
)) {
67 if (__relative_priority
> 0 || __relative_priority
< QOS_MIN_RELATIVE_PRIORITY
) {
72 if (__attr
->sig
== _PTHREAD_ATTR_SIG
) {
73 if (!__attr
->schedset
) {
74 __attr
->qosclass
= _pthread_priority_make_newest(__qos_class
, __relative_priority
, 0);
84 pthread_attr_get_qos_class_np(pthread_attr_t
* __restrict __attr
,
85 qos_class_t
* __restrict __qos_class
,
86 int * __restrict __relative_priority
)
88 if (!(__pthread_supported_features
& PTHREAD_FEATURE_BSDTHREADCTL
)) {
93 if (__attr
->sig
== _PTHREAD_ATTR_SIG
) {
95 qos_class_t qos
; int relpri
;
96 _pthread_priority_split_newest(__attr
->qosclass
, qos
, relpri
);
98 if (__qos_class
) { *__qos_class
= qos
; }
99 if (__relative_priority
) { *__relative_priority
= relpri
; }
101 if (__qos_class
) { *__qos_class
= 0; }
102 if (__relative_priority
) { *__relative_priority
= 0; }
111 pthread_set_qos_class_self_np(qos_class_t __qos_class
,
112 int __relative_priority
)
114 if (!(__pthread_supported_features
& PTHREAD_FEATURE_BSDTHREADCTL
)) {
118 if (__relative_priority
> 0 || __relative_priority
< QOS_MIN_RELATIVE_PRIORITY
) {
122 pthread_priority_t priority
= _pthread_priority_make_newest(__qos_class
, __relative_priority
, 0);
124 if (__pthread_supported_features
& PTHREAD_FEATURE_SETSELF
) {
125 return _pthread_set_properties_self(_PTHREAD_SET_SELF_QOS_FLAG
, priority
, 0);
127 /* We set the thread QoS class in the TSD and then call into the kernel to
128 * read the value out of it and set the QoS class.
130 _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
, priority
);
132 mach_port_t kport
= pthread_mach_thread_np(pthread_self());
133 int res
= __bsdthread_ctl(BSDTHREAD_CTL_SET_QOS
, kport
, &pthread_self()->tsd
[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
], 0);
144 pthread_set_qos_class_np(pthread_t __pthread
,
145 qos_class_t __qos_class
,
146 int __relative_priority
)
148 if (!(__pthread_supported_features
& PTHREAD_FEATURE_BSDTHREADCTL
)) {
152 if (__relative_priority
> 0 || __relative_priority
< QOS_MIN_RELATIVE_PRIORITY
) {
156 if (__pthread
!= pthread_self()) {
157 /* The kext now enforces this anyway, if we check here too, it allows us to call
158 * _pthread_set_properties_self later if we can.
163 pthread_priority_t priority
= _pthread_priority_make_newest(__qos_class
, __relative_priority
, 0);
165 if (__pthread_supported_features
& PTHREAD_FEATURE_SETSELF
) {
166 /* If we have _pthread_set_properties_self, then we can easily set this using that. */
167 return _pthread_set_properties_self(_PTHREAD_SET_SELF_QOS_FLAG
, priority
, 0);
169 /* We set the thread QoS class in the TSD and then call into the kernel to
170 * read the value out of it and set the QoS class.
172 if (__pthread
== pthread_self()) {
173 _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
, priority
);
175 __pthread
->tsd
[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
] = priority
;
178 mach_port_t kport
= pthread_mach_thread_np(__pthread
);
179 int res
= __bsdthread_ctl(BSDTHREAD_CTL_SET_QOS
, kport
, &__pthread
->tsd
[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
], 0);
190 pthread_get_qos_class_np(pthread_t __pthread
,
191 qos_class_t
* __restrict __qos_class
,
192 int * __restrict __relative_priority
)
194 if (!(__pthread_supported_features
& PTHREAD_FEATURE_BSDTHREADCTL
)) {
198 pthread_priority_t priority
;
200 if (__pthread
== pthread_self()) {
201 priority
= _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
);
203 priority
= __pthread
->tsd
[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
];
206 qos_class_t qos
; int relpri
;
207 _pthread_priority_split_newest(priority
, qos
, relpri
);
209 if (__qos_class
) { *__qos_class
= qos
; }
210 if (__relative_priority
) { *__relative_priority
= relpri
; }
218 if (!(__pthread_supported_features
& PTHREAD_FEATURE_BSDTHREADCTL
)) {
219 return QOS_CLASS_UNSPECIFIED
;
222 pthread_priority_t p
= _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
);
223 qos_class_t c
= _pthread_priority_get_qos_newest(p
);
231 return _pthread_priority_get_qos_newest(_main_qos
);
235 _pthread_qos_class_encode(qos_class_t qos_class
, int relative_priority
, unsigned long flags
)
237 if ((__pthread_supported_features
& PTHREAD_FEATURE_QOS_MAINTENANCE
) == 0) {
238 return _pthread_priority_make_version2(qos_class
, relative_priority
, flags
);
240 return _pthread_priority_make_newest(qos_class
, relative_priority
, flags
);
245 _pthread_qos_class_decode(pthread_priority_t priority
, int *relative_priority
, unsigned long *flags
)
247 qos_class_t qos
; int relpri
;
249 if ((__pthread_supported_features
& PTHREAD_FEATURE_QOS_MAINTENANCE
) == 0) {
250 _pthread_priority_split_version2(priority
, qos
, relpri
);
252 _pthread_priority_split_newest(priority
, qos
, relpri
);
255 if (relative_priority
) { *relative_priority
= relpri
; }
256 if (flags
) { *flags
= _pthread_priority_get_flags(priority
); }
261 _pthread_qos_class_encode_workqueue(int queue_priority
, unsigned long flags
)
263 if ((__pthread_supported_features
& PTHREAD_FEATURE_QOS_DEFAULT
) == 0) {
264 switch (queue_priority
) {
265 case WORKQ_HIGH_PRIOQUEUE
:
266 return _pthread_priority_make_version1(QOS_CLASS_USER_INTERACTIVE
, 0, flags
);
267 case WORKQ_DEFAULT_PRIOQUEUE
:
268 return _pthread_priority_make_version1(QOS_CLASS_USER_INITIATED
, 0, flags
);
269 case WORKQ_LOW_PRIOQUEUE
:
270 case WORKQ_NON_INTERACTIVE_PRIOQUEUE
:
271 return _pthread_priority_make_version1(QOS_CLASS_UTILITY
, 0, flags
);
272 case WORKQ_BG_PRIOQUEUE
:
273 return _pthread_priority_make_version1(QOS_CLASS_BACKGROUND
, 0, flags
);
279 if ((__pthread_supported_features
& PTHREAD_FEATURE_QOS_MAINTENANCE
) == 0) {
280 switch (queue_priority
) {
281 case WORKQ_HIGH_PRIOQUEUE
:
282 return _pthread_priority_make_version2(QOS_CLASS_USER_INITIATED
, 0, flags
);
283 case WORKQ_DEFAULT_PRIOQUEUE
:
284 return _pthread_priority_make_version2(QOS_CLASS_DEFAULT
, 0, flags
);
285 case WORKQ_LOW_PRIOQUEUE
:
286 case WORKQ_NON_INTERACTIVE_PRIOQUEUE
:
287 return _pthread_priority_make_version2(QOS_CLASS_UTILITY
, 0, flags
);
288 case WORKQ_BG_PRIOQUEUE
:
289 return _pthread_priority_make_version2(QOS_CLASS_BACKGROUND
, 0, flags
);
290 /* Legacy dispatch does not use QOS_CLASS_MAINTENANCE, so no need to handle it here */
296 switch (queue_priority
) {
297 case WORKQ_HIGH_PRIOQUEUE
:
298 return _pthread_priority_make_newest(QOS_CLASS_USER_INITIATED
, 0, flags
);
299 case WORKQ_DEFAULT_PRIOQUEUE
:
300 return _pthread_priority_make_newest(QOS_CLASS_DEFAULT
, 0, flags
);
301 case WORKQ_LOW_PRIOQUEUE
:
302 case WORKQ_NON_INTERACTIVE_PRIOQUEUE
:
303 return _pthread_priority_make_newest(QOS_CLASS_UTILITY
, 0, flags
);
304 case WORKQ_BG_PRIOQUEUE
:
305 return _pthread_priority_make_newest(QOS_CLASS_BACKGROUND
, 0, flags
);
306 /* Legacy dispatch does not use QOS_CLASS_MAINTENANCE, so no need to handle it here */
313 _pthread_set_properties_self(_pthread_set_flags_t flags
, pthread_priority_t priority
, mach_port_t voucher
)
315 if (!(__pthread_supported_features
& PTHREAD_FEATURE_SETSELF
)) {
319 int rv
= __bsdthread_ctl(BSDTHREAD_CTL_SET_SELF
, priority
, voucher
, flags
);
321 /* Set QoS TSD if we succeeded or only failed the voucher half. */
322 if ((flags
& _PTHREAD_SET_SELF_QOS_FLAG
) != 0) {
323 if (rv
== 0 || errno
== ENOENT
) {
324 _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
, priority
);
335 pthread_set_fixedpriority_self(void)
337 if (!(__pthread_supported_features
& PTHREAD_FEATURE_BSDTHREADCTL
)) {
341 if (__pthread_supported_features
& PTHREAD_FEATURE_SETSELF
) {
342 return _pthread_set_properties_self(_PTHREAD_SET_SELF_FIXEDPRIORITY_FLAG
, 0, 0);
349 pthread_set_timeshare_self(void)
351 if (!(__pthread_supported_features
& PTHREAD_FEATURE_BSDTHREADCTL
)) {
355 if (__pthread_supported_features
& PTHREAD_FEATURE_SETSELF
) {
356 return _pthread_set_properties_self(_PTHREAD_SET_SELF_TIMESHARE_FLAG
, 0, 0);
364 pthread_override_qos_class_start_np(pthread_t __pthread
, qos_class_t __qos_class
, int __relative_priority
)
366 pthread_override_t rv
;
370 /* For now, we don't have access to malloc. So we'll have to vm_allocate this, which means the tiny struct is going
371 * to use an entire page.
373 bool did_malloc
= true;
375 mach_vm_address_t vm_addr
= malloc(sizeof(struct pthread_override_s
));
377 vm_addr
= vm_page_size
;
380 kr
= mach_vm_allocate(mach_task_self(), &vm_addr
, round_page(sizeof(struct pthread_override_s
)), VM_MAKE_TAG(VM_MEMORY_LIBDISPATCH
) | VM_FLAGS_ANYWHERE
);
381 if (kr
!= KERN_SUCCESS
) {
383 return (_Nonnull pthread_override_t
) NULL
;
387 rv
= (pthread_override_t
)vm_addr
;
388 rv
->sig
= PTHREAD_OVERRIDE_SIGNATURE
;
389 rv
->pthread
= __pthread
;
390 rv
->kthread
= pthread_mach_thread_np(__pthread
);
391 rv
->priority
= _pthread_priority_make_newest(__qos_class
, __relative_priority
, 0);
392 rv
->malloced
= did_malloc
;
394 /* To ensure that the kernel port that we keep stays valid, we retain it here. */
395 kr
= mach_port_mod_refs(mach_task_self(), rv
->kthread
, MACH_PORT_RIGHT_SEND
, 1);
396 if (kr
!= KERN_SUCCESS
) {
401 res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_START
, rv
->kthread
, rv
->priority
, (uintptr_t)rv
);
404 mach_port_mod_refs(mach_task_self(), rv
->kthread
, MACH_PORT_RIGHT_SEND
, -1);
412 mach_vm_deallocate(mach_task_self(), vm_addr
, round_page(sizeof(struct pthread_override_s
)));
416 return (_Nonnull pthread_override_t
) rv
;
420 pthread_override_qos_class_end_np(pthread_override_t override
)
425 /* Double-free is a fault. Swap the signature and check the old one. */
426 if (__sync_swap(&override
->sig
, PTHREAD_OVERRIDE_SIG_DEAD
) != PTHREAD_OVERRIDE_SIGNATURE
) {
430 override
->sig
= PTHREAD_OVERRIDE_SIG_DEAD
;
432 /* Always consumes (and deallocates) the pthread_override_t object given. */
433 res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_END
, override
->kthread
, (uintptr_t)override
, 0);
434 if (res
== -1) { res
= errno
; }
436 /* EFAULT from the syscall means we underflowed. Crash here. */
438 // <rdar://problem/17645082> Disable the trap-on-underflow, it doesn't co-exist
439 // with dispatch resetting override counts on threads.
444 kr
= mach_port_mod_refs(mach_task_self(), override
->kthread
, MACH_PORT_RIGHT_SEND
, -1);
445 if (kr
!= KERN_SUCCESS
) {
449 if (override
->malloced
) {
452 kr
= mach_vm_deallocate(mach_task_self(), (mach_vm_address_t
)override
, round_page(sizeof(struct pthread_override_s
)));
453 if (kr
!= KERN_SUCCESS
) {
462 _pthread_qos_override_start_direct(mach_port_t thread
, pthread_priority_t priority
, void *resource
)
464 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_START
, thread
, priority
, (uintptr_t)resource
);
465 if (res
== -1) { res
= errno
; }
470 _pthread_qos_override_end_direct(mach_port_t thread
, void *resource
)
472 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_END
, thread
, (uintptr_t)resource
, 0);
473 if (res
== -1) { res
= errno
; }
478 _pthread_override_qos_class_start_direct(mach_port_t thread
, pthread_priority_t priority
)
480 // use pthread_self as the default per-thread memory allocation to track the override in the kernel
481 return _pthread_qos_override_start_direct(thread
, priority
, pthread_self());
485 _pthread_override_qos_class_end_direct(mach_port_t thread
)
487 // use pthread_self as the default per-thread memory allocation to track the override in the kernel
488 return _pthread_qos_override_end_direct(thread
, pthread_self());
492 _pthread_workqueue_override_start_direct(mach_port_t thread
, pthread_priority_t priority
)
494 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_DISPATCH
, thread
, priority
, 0);
495 if (res
== -1) { res
= errno
; }
500 _pthread_workqueue_override_start_direct_check_owner(mach_port_t thread
, pthread_priority_t priority
, mach_port_t
*ulock_addr
)
502 #if !TARGET_OS_IPHONE
503 static boolean_t kernel_supports_owner_check
= TRUE
;
504 if (!kernel_supports_owner_check
) {
510 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_DISPATCH
, thread
, priority
, ulock_addr
);
511 if (res
== -1) { res
= errno
; }
512 #if !TARGET_OS_IPHONE
513 if (ulock_addr
&& res
== EINVAL
) {
514 if ((uintptr_t)ulock_addr
% _Alignof(_Atomic
uint32_t)) {
515 // do not mute bad ulock addresses related errors
518 // backward compatibility for the XBS chroot
519 // BSDTHREAD_CTL_QOS_OVERRIDE_DISPATCH used to return EINVAL if
520 // arg3 was non NULL.
521 kernel_supports_owner_check
= FALSE
;
526 if (ulock_addr
&& res
== EFAULT
) {
527 // kernel wants us to redrive the call, so while we refault the
528 // memory, also revalidate the owner
529 uint32_t uval
= os_atomic_load(ulock_addr
, relaxed
);
530 if (ulock_owner_value_to_port_name(uval
) != thread
) {
541 _pthread_workqueue_override_reset(void)
543 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_RESET
, 0, 0, 0);
544 if (res
== -1) { res
= errno
; }
549 _pthread_workqueue_asynchronous_override_add(mach_port_t thread
, pthread_priority_t priority
, void *resource
)
551 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_ADD
, thread
, priority
, (uintptr_t)resource
);
552 if (res
== -1) { res
= errno
; }
557 _pthread_workqueue_asynchronous_override_reset_self(void *resource
)
559 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_RESET
,
563 if (res
== -1) { res
= errno
; }
568 _pthread_workqueue_asynchronous_override_reset_all_self(void)
570 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_RESET
,
574 if (res
== -1) { res
= errno
; }
579 posix_spawnattr_set_qos_class_np(posix_spawnattr_t
* __restrict __attr
, qos_class_t __qos_class
)
581 switch (__qos_class
) {
582 case QOS_CLASS_UTILITY
:
583 return posix_spawnattr_set_qos_clamp_np(__attr
, POSIX_SPAWN_PROC_CLAMP_UTILITY
);
584 case QOS_CLASS_BACKGROUND
:
585 return posix_spawnattr_set_qos_clamp_np(__attr
, POSIX_SPAWN_PROC_CLAMP_BACKGROUND
);
586 case QOS_CLASS_MAINTENANCE
:
587 return posix_spawnattr_set_qos_clamp_np(__attr
, POSIX_SPAWN_PROC_CLAMP_MAINTENANCE
);
594 posix_spawnattr_get_qos_class_np(const posix_spawnattr_t
*__restrict __attr
, qos_class_t
* __restrict __qos_class
)
602 int rv
= posix_spawnattr_get_qos_clamp_np(__attr
, &clamp
);
608 case POSIX_SPAWN_PROC_CLAMP_UTILITY
:
609 *__qos_class
= QOS_CLASS_UTILITY
;
611 case POSIX_SPAWN_PROC_CLAMP_BACKGROUND
:
612 *__qos_class
= QOS_CLASS_BACKGROUND
;
614 case POSIX_SPAWN_PROC_CLAMP_MAINTENANCE
:
615 *__qos_class
= QOS_CLASS_MAINTENANCE
;
618 *__qos_class
= QOS_CLASS_UNSPECIFIED
;