2 * Copyright (c) 2013 Apple Inc. All rights reserved.
4 * @APPLE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
21 * @APPLE_LICENSE_HEADER_END@
27 #include <mach/mach_vm.h>
30 #include <spawn_private.h>
31 #include <sys/spawn_internal.h>
32 #include <sys/ulock.h>
34 // TODO: remove me when internal.h can include *_private.h itself
35 #include "workqueue_private.h"
36 #include "qos_private.h"
38 static pthread_priority_t _main_qos
= QOS_CLASS_UNSPECIFIED
;
40 #define PTHREAD_OVERRIDE_SIGNATURE (0x6f766572)
41 #define PTHREAD_OVERRIDE_SIG_DEAD (0x7265766f)
43 struct pthread_override_s
48 pthread_priority_t priority
;
53 _pthread_set_main_qos(pthread_priority_t qos
)
59 pthread_attr_set_qos_class_np(pthread_attr_t
*__attr
,
60 qos_class_t __qos_class
,
61 int __relative_priority
)
63 if (!(__pthread_supported_features
& PTHREAD_FEATURE_BSDTHREADCTL
)) {
67 if (__relative_priority
> 0 || __relative_priority
< QOS_MIN_RELATIVE_PRIORITY
) {
72 if (__attr
->sig
== _PTHREAD_ATTR_SIG
) {
73 if (!__attr
->schedset
) {
74 __attr
->qosclass
= _pthread_priority_make_newest(__qos_class
, __relative_priority
, 0);
84 pthread_attr_get_qos_class_np(pthread_attr_t
* __restrict __attr
,
85 qos_class_t
* __restrict __qos_class
,
86 int * __restrict __relative_priority
)
88 if (!(__pthread_supported_features
& PTHREAD_FEATURE_BSDTHREADCTL
)) {
93 if (__attr
->sig
== _PTHREAD_ATTR_SIG
) {
95 qos_class_t qos
; int relpri
;
96 _pthread_priority_split_newest(__attr
->qosclass
, qos
, relpri
);
98 if (__qos_class
) { *__qos_class
= qos
; }
99 if (__relative_priority
) { *__relative_priority
= relpri
; }
101 if (__qos_class
) { *__qos_class
= 0; }
102 if (__relative_priority
) { *__relative_priority
= 0; }
111 pthread_set_qos_class_self_np(qos_class_t __qos_class
,
112 int __relative_priority
)
114 if (!(__pthread_supported_features
& PTHREAD_FEATURE_BSDTHREADCTL
)) {
118 if (__relative_priority
> 0 || __relative_priority
< QOS_MIN_RELATIVE_PRIORITY
) {
122 pthread_priority_t priority
= _pthread_priority_make_newest(__qos_class
, __relative_priority
, 0);
124 if (__pthread_supported_features
& PTHREAD_FEATURE_SETSELF
) {
125 return _pthread_set_properties_self(_PTHREAD_SET_SELF_QOS_FLAG
, priority
, 0);
127 /* We set the thread QoS class in the TSD and then call into the kernel to
128 * read the value out of it and set the QoS class.
130 _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
, priority
);
131 mach_port_t kport
= _pthread_kernel_thread(pthread_self());
132 int res
= __bsdthread_ctl(BSDTHREAD_CTL_SET_QOS
, kport
, &pthread_self()->tsd
[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
], 0);
143 pthread_set_qos_class_np(pthread_t __pthread
,
144 qos_class_t __qos_class
,
145 int __relative_priority
)
147 if (__pthread
!= pthread_self()) {
148 /* The kext now enforces this anyway, if we check here too, it allows us to call
149 * _pthread_set_properties_self later if we can.
154 return pthread_set_qos_class_self_np(__qos_class
, __relative_priority
);
158 pthread_get_qos_class_np(pthread_t __pthread
,
159 qos_class_t
* __restrict __qos_class
,
160 int * __restrict __relative_priority
)
162 if (!(__pthread_supported_features
& PTHREAD_FEATURE_BSDTHREADCTL
)) {
166 pthread_priority_t priority
;
168 if (__pthread
== pthread_self()) {
169 priority
= _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
);
171 priority
= __pthread
->tsd
[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
];
174 qos_class_t qos
; int relpri
;
175 _pthread_priority_split_newest(priority
, qos
, relpri
);
177 if (__qos_class
) { *__qos_class
= qos
; }
178 if (__relative_priority
) { *__relative_priority
= relpri
; }
186 if (!(__pthread_supported_features
& PTHREAD_FEATURE_BSDTHREADCTL
)) {
187 return QOS_CLASS_UNSPECIFIED
;
190 pthread_priority_t p
= _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
);
191 qos_class_t c
= _pthread_priority_get_qos_newest(p
);
199 return _pthread_priority_get_qos_newest(_main_qos
);
203 _pthread_qos_class_encode(qos_class_t qos_class
, int relative_priority
, unsigned long flags
)
205 return _pthread_priority_make_newest(qos_class
, relative_priority
, flags
);
209 _pthread_qos_class_decode(pthread_priority_t priority
, int *relative_priority
, unsigned long *flags
)
211 qos_class_t qos
; int relpri
;
213 _pthread_priority_split_newest(priority
, qos
, relpri
);
215 if (relative_priority
) { *relative_priority
= relpri
; }
216 if (flags
) { *flags
= _pthread_priority_get_flags(priority
); }
220 // Encode a legacy workqueue API priority into a pthread_priority_t. This API
221 // is deprecated and can be removed when the simulator no longer uses it.
223 _pthread_qos_class_encode_workqueue(int queue_priority
, unsigned long flags
)
225 switch (queue_priority
) {
226 case WORKQ_HIGH_PRIOQUEUE
:
227 return _pthread_priority_make_newest(QOS_CLASS_USER_INITIATED
, 0, flags
);
228 case WORKQ_DEFAULT_PRIOQUEUE
:
229 return _pthread_priority_make_newest(QOS_CLASS_DEFAULT
, 0, flags
);
230 case WORKQ_LOW_PRIOQUEUE
:
231 case WORKQ_NON_INTERACTIVE_PRIOQUEUE
:
232 return _pthread_priority_make_newest(QOS_CLASS_UTILITY
, 0, flags
);
233 case WORKQ_BG_PRIOQUEUE
:
234 return _pthread_priority_make_newest(QOS_CLASS_BACKGROUND
, 0, flags
);
235 /* Legacy dispatch does not use QOS_CLASS_MAINTENANCE, so no need to handle it here */
242 _pthread_set_properties_self(_pthread_set_flags_t flags
, pthread_priority_t priority
, mach_port_t voucher
)
244 if (!(__pthread_supported_features
& PTHREAD_FEATURE_SETSELF
)) {
248 int rv
= __bsdthread_ctl(BSDTHREAD_CTL_SET_SELF
, priority
, voucher
, flags
);
250 /* Set QoS TSD if we succeeded or only failed the voucher half. */
251 if ((flags
& _PTHREAD_SET_SELF_QOS_FLAG
) != 0) {
252 if (rv
== 0 || errno
== ENOENT
) {
253 _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
, priority
);
264 pthread_set_fixedpriority_self(void)
266 if (!(__pthread_supported_features
& PTHREAD_FEATURE_BSDTHREADCTL
)) {
270 if (__pthread_supported_features
& PTHREAD_FEATURE_SETSELF
) {
271 return _pthread_set_properties_self(_PTHREAD_SET_SELF_FIXEDPRIORITY_FLAG
, 0, 0);
278 pthread_set_timeshare_self(void)
280 if (!(__pthread_supported_features
& PTHREAD_FEATURE_BSDTHREADCTL
)) {
284 if (__pthread_supported_features
& PTHREAD_FEATURE_SETSELF
) {
285 return _pthread_set_properties_self(_PTHREAD_SET_SELF_TIMESHARE_FLAG
, 0, 0);
293 pthread_override_qos_class_start_np(pthread_t __pthread
, qos_class_t __qos_class
, int __relative_priority
)
295 pthread_override_t rv
;
299 /* For now, we don't have access to malloc. So we'll have to vm_allocate this, which means the tiny struct is going
300 * to use an entire page.
302 bool did_malloc
= true;
304 mach_vm_address_t vm_addr
= malloc(sizeof(struct pthread_override_s
));
306 vm_addr
= vm_page_size
;
309 kr
= mach_vm_allocate(mach_task_self(), &vm_addr
, round_page(sizeof(struct pthread_override_s
)), VM_MAKE_TAG(VM_MEMORY_LIBDISPATCH
) | VM_FLAGS_ANYWHERE
);
310 if (kr
!= KERN_SUCCESS
) {
312 return (_Nonnull pthread_override_t
) NULL
;
316 rv
= (pthread_override_t
)vm_addr
;
317 rv
->sig
= PTHREAD_OVERRIDE_SIGNATURE
;
318 rv
->pthread
= __pthread
;
319 rv
->kthread
= pthread_mach_thread_np(__pthread
);
320 rv
->priority
= _pthread_priority_make_newest(__qos_class
, __relative_priority
, 0);
321 rv
->malloced
= did_malloc
;
323 /* To ensure that the kernel port that we keep stays valid, we retain it here. */
324 kr
= mach_port_mod_refs(mach_task_self(), rv
->kthread
, MACH_PORT_RIGHT_SEND
, 1);
325 if (kr
!= KERN_SUCCESS
) {
330 res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_START
, rv
->kthread
, rv
->priority
, (uintptr_t)rv
);
333 mach_port_mod_refs(mach_task_self(), rv
->kthread
, MACH_PORT_RIGHT_SEND
, -1);
341 mach_vm_deallocate(mach_task_self(), vm_addr
, round_page(sizeof(struct pthread_override_s
)));
345 return (_Nonnull pthread_override_t
) rv
;
349 pthread_override_qos_class_end_np(pthread_override_t override
)
354 /* Double-free is a fault. Swap the signature and check the old one. */
355 if (_pthread_atomic_xchg_uint32_relaxed(&override
->sig
, PTHREAD_OVERRIDE_SIG_DEAD
) != PTHREAD_OVERRIDE_SIGNATURE
) {
359 /* Always consumes (and deallocates) the pthread_override_t object given. */
360 res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_END
, override
->kthread
, (uintptr_t)override
, 0);
361 if (res
== -1) { res
= errno
; }
363 /* EFAULT from the syscall means we underflowed. Crash here. */
365 // <rdar://problem/17645082> Disable the trap-on-underflow, it doesn't co-exist
366 // with dispatch resetting override counts on threads.
371 kr
= mach_port_mod_refs(mach_task_self(), override
->kthread
, MACH_PORT_RIGHT_SEND
, -1);
372 if (kr
!= KERN_SUCCESS
) {
376 if (override
->malloced
) {
379 kr
= mach_vm_deallocate(mach_task_self(), (mach_vm_address_t
)override
, round_page(sizeof(struct pthread_override_s
)));
380 if (kr
!= KERN_SUCCESS
) {
389 _pthread_qos_override_start_direct(mach_port_t thread
, pthread_priority_t priority
, void *resource
)
391 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_START
, thread
, priority
, (uintptr_t)resource
);
392 if (res
== -1) { res
= errno
; }
397 _pthread_qos_override_end_direct(mach_port_t thread
, void *resource
)
399 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_END
, thread
, (uintptr_t)resource
, 0);
400 if (res
== -1) { res
= errno
; }
405 _pthread_override_qos_class_start_direct(mach_port_t thread
, pthread_priority_t priority
)
407 // use pthread_self as the default per-thread memory allocation to track the override in the kernel
408 return _pthread_qos_override_start_direct(thread
, priority
, pthread_self());
412 _pthread_override_qos_class_end_direct(mach_port_t thread
)
414 // use pthread_self as the default per-thread memory allocation to track the override in the kernel
415 return _pthread_qos_override_end_direct(thread
, pthread_self());
419 _pthread_workqueue_override_start_direct(mach_port_t thread
, pthread_priority_t priority
)
421 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_DISPATCH
, thread
, priority
, 0);
422 if (res
== -1) { res
= errno
; }
427 _pthread_workqueue_override_start_direct_check_owner(mach_port_t thread
, pthread_priority_t priority
, mach_port_t
*ulock_addr
)
429 #if !TARGET_OS_IPHONE
430 static boolean_t kernel_supports_owner_check
= TRUE
;
431 if (!kernel_supports_owner_check
) {
437 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_DISPATCH
, thread
, priority
, ulock_addr
);
438 if (res
== -1) { res
= errno
; }
439 #if !TARGET_OS_IPHONE
440 if (ulock_addr
&& res
== EINVAL
) {
441 if ((uintptr_t)ulock_addr
% _Alignof(_Atomic
uint32_t)) {
442 // do not mute bad ulock addresses related errors
445 // backward compatibility for the XBS chroot
446 // BSDTHREAD_CTL_QOS_OVERRIDE_DISPATCH used to return EINVAL if
447 // arg3 was non NULL.
448 kernel_supports_owner_check
= FALSE
;
453 if (ulock_addr
&& res
== EFAULT
) {
454 // kernel wants us to redrive the call, so while we refault the
455 // memory, also revalidate the owner
456 uint32_t uval
= *(uint32_t volatile *)ulock_addr
;
457 if (ulock_owner_value_to_port_name(uval
) != thread
) {
468 _pthread_workqueue_override_reset(void)
470 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_RESET
, 0, 0, 0);
471 if (res
== -1) { res
= errno
; }
476 _pthread_workqueue_asynchronous_override_add(mach_port_t thread
, pthread_priority_t priority
, void *resource
)
478 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_ADD
, thread
, priority
, (uintptr_t)resource
);
479 if (res
== -1) { res
= errno
; }
484 _pthread_workqueue_asynchronous_override_reset_self(void *resource
)
486 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_RESET
,
490 if (res
== -1) { res
= errno
; }
495 _pthread_workqueue_asynchronous_override_reset_all_self(void)
497 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_RESET
,
501 if (res
== -1) { res
= errno
; }
505 static inline uint16_t
506 _pthread_workqueue_parallelism_for_priority(int qos
, unsigned long flags
)
508 int rc
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_MAX_PARALLELISM
, qos
, flags
, 0);
509 if (os_unlikely(rc
== -1)) {
512 PTHREAD_INTERNAL_CRASH(rc
, "qos_max_parallelism failed");
514 if (flags
& _PTHREAD_QOS_PARALLELISM_COUNT_LOGICAL
) {
515 return *(uint8_t *)_COMM_PAGE_LOGICAL_CPUS
;
517 return *(uint8_t *)_COMM_PAGE_PHYSICAL_CPUS
;
524 pthread_qos_max_parallelism(qos_class_t qos
, unsigned long flags
)
526 int thread_qos
= _pthread_qos_class_to_thread_qos(qos
);
527 if (thread_qos
== THREAD_QOS_UNSPECIFIED
) {
532 unsigned long syscall_flags
= _PTHREAD_QOS_PARALLELISM_COUNT_LOGICAL
;
533 uint16_t *ptr
= &_pthread_globals()->qmp_logical
[thread_qos
];
535 if (flags
& PTHREAD_MAX_PARALLELISM_PHYSICAL
) {
537 ptr
= &_pthread_globals()->qmp_physical
[thread_qos
];
540 *ptr
= _pthread_workqueue_parallelism_for_priority(thread_qos
, syscall_flags
);
546 pthread_time_constraint_max_parallelism(unsigned long flags
)
548 unsigned long syscall_flags
= _PTHREAD_QOS_PARALLELISM_COUNT_LOGICAL
;
549 uint16_t *ptr
= &_pthread_globals()->qmp_logical
[0];
551 if (flags
& PTHREAD_MAX_PARALLELISM_PHYSICAL
) {
553 ptr
= &_pthread_globals()->qmp_physical
[0];
556 *ptr
= _pthread_workqueue_parallelism_for_priority(0,
557 syscall_flags
| _PTHREAD_QOS_PARALLELISM_REALTIME
);
563 posix_spawnattr_set_qos_class_np(posix_spawnattr_t
* __restrict __attr
, qos_class_t __qos_class
)
565 switch (__qos_class
) {
566 case QOS_CLASS_UTILITY
:
567 return posix_spawnattr_set_qos_clamp_np(__attr
, POSIX_SPAWN_PROC_CLAMP_UTILITY
);
568 case QOS_CLASS_BACKGROUND
:
569 return posix_spawnattr_set_qos_clamp_np(__attr
, POSIX_SPAWN_PROC_CLAMP_BACKGROUND
);
570 case QOS_CLASS_MAINTENANCE
:
571 return posix_spawnattr_set_qos_clamp_np(__attr
, POSIX_SPAWN_PROC_CLAMP_MAINTENANCE
);
578 posix_spawnattr_get_qos_class_np(const posix_spawnattr_t
*__restrict __attr
, qos_class_t
* __restrict __qos_class
)
586 int rv
= posix_spawnattr_get_qos_clamp_np(__attr
, &clamp
);
592 case POSIX_SPAWN_PROC_CLAMP_UTILITY
:
593 *__qos_class
= QOS_CLASS_UTILITY
;
595 case POSIX_SPAWN_PROC_CLAMP_BACKGROUND
:
596 *__qos_class
= QOS_CLASS_BACKGROUND
;
598 case POSIX_SPAWN_PROC_CLAMP_MAINTENANCE
:
599 *__qos_class
= QOS_CLASS_MAINTENANCE
;
602 *__qos_class
= QOS_CLASS_UNSPECIFIED
;