2 * Copyright (c) 2013 Apple Inc. All rights reserved.
4 * @APPLE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
21 * @APPLE_LICENSE_HEADER_END@
27 #include <mach/mach_vm.h>
30 #include <spawn_private.h>
31 #include <sys/spawn_internal.h>
32 #include <sys/ulock.h>
34 // TODO: remove me when internal.h can include *_private.h itself
35 #include "workqueue_private.h"
36 #include "qos_private.h"
38 #define PTHREAD_OVERRIDE_SIGNATURE (0x6f766572)
39 #define PTHREAD_OVERRIDE_SIG_DEAD (0x7265766f)
41 struct pthread_override_s
46 pthread_priority_t priority
;
51 _pthread_qos_class_to_thread_qos(qos_class_t qos
)
54 case QOS_CLASS_USER_INTERACTIVE
: return THREAD_QOS_USER_INTERACTIVE
;
55 case QOS_CLASS_USER_INITIATED
: return THREAD_QOS_USER_INITIATED
;
56 case QOS_CLASS_DEFAULT
: return THREAD_QOS_LEGACY
;
57 case QOS_CLASS_UTILITY
: return THREAD_QOS_UTILITY
;
58 case QOS_CLASS_BACKGROUND
: return THREAD_QOS_BACKGROUND
;
59 case QOS_CLASS_MAINTENANCE
: return THREAD_QOS_MAINTENANCE
;
60 default: return THREAD_QOS_UNSPECIFIED
;
64 static inline qos_class_t
65 _pthread_qos_class_from_thread_qos(thread_qos_t tqos
)
67 static const qos_class_t thread_qos_to_qos_class
[THREAD_QOS_LAST
] = {
68 [THREAD_QOS_UNSPECIFIED
] = QOS_CLASS_UNSPECIFIED
,
69 [THREAD_QOS_MAINTENANCE
] = QOS_CLASS_MAINTENANCE
,
70 [THREAD_QOS_BACKGROUND
] = QOS_CLASS_BACKGROUND
,
71 [THREAD_QOS_UTILITY
] = QOS_CLASS_UTILITY
,
72 [THREAD_QOS_LEGACY
] = QOS_CLASS_DEFAULT
,
73 [THREAD_QOS_USER_INITIATED
] = QOS_CLASS_USER_INITIATED
,
74 [THREAD_QOS_USER_INTERACTIVE
] = QOS_CLASS_USER_INTERACTIVE
,
76 if (os_unlikely(tqos
>= THREAD_QOS_LAST
)) return QOS_CLASS_UNSPECIFIED
;
77 return thread_qos_to_qos_class
[tqos
];
80 static inline thread_qos_t
81 _pthread_validate_qos_class_and_relpri(qos_class_t qc
, int relpri
)
83 if (relpri
> 0 || relpri
< QOS_MIN_RELATIVE_PRIORITY
) {
84 return THREAD_QOS_UNSPECIFIED
;
86 return _pthread_qos_class_to_thread_qos(qc
);
90 _pthread_priority_split(pthread_priority_t pp
, qos_class_t
*qc
, int *relpri
)
92 thread_qos_t qos
= _pthread_priority_thread_qos(pp
);
93 if (qc
) *qc
= _pthread_qos_class_from_thread_qos(qos
);
94 if (relpri
) *relpri
= _pthread_priority_relpri(pp
);
98 _pthread_set_main_qos(pthread_priority_t qos
)
100 _main_qos
= (uint32_t)qos
;
104 pthread_attr_set_qos_class_np(pthread_attr_t
*attr
, qos_class_t qc
, int relpri
)
106 thread_qos_t qos
= _pthread_validate_qos_class_and_relpri(qc
, relpri
);
107 if (attr
->sig
!= _PTHREAD_ATTR_SIG
|| attr
->schedset
) {
111 attr
->qosclass
= _pthread_priority_make_from_thread_qos(qos
, relpri
, 0);
118 pthread_attr_get_qos_class_np(pthread_attr_t
*attr
, qos_class_t
*qc
, int *relpri
)
120 if (attr
->sig
!= _PTHREAD_ATTR_SIG
) {
124 _pthread_priority_split(attr
->qosset
? attr
->qosclass
: 0, qc
, relpri
);
129 pthread_set_qos_class_self_np(qos_class_t qc
, int relpri
)
131 thread_qos_t qos
= _pthread_validate_qos_class_and_relpri(qc
, relpri
);
136 pthread_priority_t pp
= _pthread_priority_make_from_thread_qos(qos
, relpri
, 0);
137 return _pthread_set_properties_self(_PTHREAD_SET_SELF_QOS_FLAG
, pp
, 0);
141 pthread_set_qos_class_np(pthread_t thread
, qos_class_t qc
, int relpri
)
143 if (thread
!= pthread_self()) {
144 /* The kext now enforces this anyway, if we check here too, it allows us to call
145 * _pthread_set_properties_self later if we can.
149 _pthread_validate_signature(thread
);
150 return pthread_set_qos_class_self_np(qc
, relpri
);
154 pthread_get_qos_class_np(pthread_t thread
, qos_class_t
*qc
, int *relpri
)
156 pthread_priority_t pp
= thread
->tsd
[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
];
157 _pthread_priority_split(pp
, qc
, relpri
);
164 pthread_priority_t pp
;
165 pp
= _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
);
166 return _pthread_qos_class_from_thread_qos(_pthread_priority_thread_qos(pp
));
172 pthread_priority_t pp
= _main_qos
;
173 return _pthread_qos_class_from_thread_qos(_pthread_priority_thread_qos(pp
));
177 _pthread_qos_class_encode(qos_class_t qc
, int relpri
, unsigned long flags
)
179 thread_qos_t qos
= _pthread_qos_class_to_thread_qos(qc
);
180 return _pthread_priority_make_from_thread_qos(qos
, relpri
, flags
);
184 _pthread_qos_class_decode(pthread_priority_t pp
, int *relpri
, unsigned long *flags
)
187 _pthread_priority_split(pp
, &qc
, relpri
);
188 if (flags
) *flags
= (pp
& _PTHREAD_PRIORITY_FLAGS_MASK
);
192 // Encode a legacy workqueue API priority into a pthread_priority_t. This API
193 // is deprecated and can be removed when the simulator no longer uses it.
195 _pthread_qos_class_encode_workqueue(int queue_priority
, unsigned long flags
)
198 switch (queue_priority
) {
199 case WORKQ_HIGH_PRIOQUEUE
: qos
= THREAD_QOS_USER_INTERACTIVE
; break;
200 case WORKQ_DEFAULT_PRIOQUEUE
: qos
= THREAD_QOS_LEGACY
; break;
201 case WORKQ_NON_INTERACTIVE_PRIOQUEUE
:
202 case WORKQ_LOW_PRIOQUEUE
: qos
= THREAD_QOS_UTILITY
; break;
203 case WORKQ_BG_PRIOQUEUE
: qos
= THREAD_QOS_BACKGROUND
; break;
205 PTHREAD_CLIENT_CRASH(queue_priority
, "Invalid priority");
207 return _pthread_priority_make_from_thread_qos(qos
, 0, flags
);
210 #define _PTHREAD_SET_SELF_OUTSIDE_QOS_SKIP \
211 (_PTHREAD_SET_SELF_QOS_FLAG | _PTHREAD_SET_SELF_FIXEDPRIORITY_FLAG | \
212 _PTHREAD_SET_SELF_TIMESHARE_FLAG | \
213 _PTHREAD_SET_SELF_ALTERNATE_AMX)
216 _pthread_set_properties_self(_pthread_set_flags_t flags
,
217 pthread_priority_t priority
, mach_port_t voucher
)
219 pthread_t self
= pthread_self();
220 _pthread_set_flags_t kflags
= flags
;
223 _pthread_validate_signature(self
);
225 if (self
->wq_outsideqos
&& (flags
& _PTHREAD_SET_SELF_OUTSIDE_QOS_SKIP
)) {
226 // A number of properties cannot be altered if we are a workloop
227 // thread that has outside of QoS properties applied to it.
228 kflags
&= ~_PTHREAD_SET_SELF_OUTSIDE_QOS_SKIP
;
229 if (kflags
== 0) goto skip
;
232 rv
= __bsdthread_ctl(BSDTHREAD_CTL_SET_SELF
, priority
, voucher
, kflags
);
235 // Set QoS TSD if we succeeded, or only failed the voucher portion of the
236 // call. Additionally, if we skipped setting QoS because of outside-of-QoS
237 // attributes then we still want to set the TSD in userspace.
238 if ((flags
& _PTHREAD_SET_SELF_QOS_FLAG
) != 0) {
239 if (rv
== 0 || errno
== ENOENT
) {
240 _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
,
252 pthread_set_fixedpriority_self(void)
254 return _pthread_set_properties_self(_PTHREAD_SET_SELF_FIXEDPRIORITY_FLAG
, 0, 0);
258 pthread_set_timeshare_self(void)
260 return _pthread_set_properties_self(_PTHREAD_SET_SELF_TIMESHARE_FLAG
, 0, 0);
264 pthread_prefer_alternate_amx_self(void)
266 return _pthread_set_properties_self(_PTHREAD_SET_SELF_ALTERNATE_AMX
, 0, 0);
271 pthread_override_qos_class_start_np(pthread_t thread
, qos_class_t qc
, int relpri
)
273 pthread_override_t rv
;
278 /* For now, we don't have access to malloc. So we'll have to vm_allocate this, which means the tiny struct is going
279 * to use an entire page.
281 bool did_malloc
= true;
283 qos
= _pthread_validate_qos_class_and_relpri(qc
, relpri
);
284 if (qos
== THREAD_QOS_UNSPECIFIED
) {
285 return (_Nonnull pthread_override_t
)NULL
;
288 mach_vm_address_t vm_addr
= malloc(sizeof(struct pthread_override_s
));
290 vm_addr
= vm_page_size
;
293 kr
= mach_vm_allocate(mach_task_self(), &vm_addr
,
294 round_page(sizeof(struct pthread_override_s
)),
295 VM_MAKE_TAG(VM_MEMORY_LIBDISPATCH
) | VM_FLAGS_ANYWHERE
);
296 if (kr
!= KERN_SUCCESS
) {
298 return (_Nonnull pthread_override_t
)NULL
;
302 rv
= (pthread_override_t
)vm_addr
;
303 rv
->sig
= PTHREAD_OVERRIDE_SIGNATURE
;
304 rv
->pthread
= thread
;
305 rv
->kthread
= pthread_mach_thread_np(thread
);
306 rv
->priority
= _pthread_priority_make_from_thread_qos(qos
, relpri
, 0);
307 rv
->malloced
= did_malloc
;
309 /* To ensure that the kernel port that we keep stays valid, we retain it here. */
310 kr
= mach_port_mod_refs(mach_task_self(), rv
->kthread
, MACH_PORT_RIGHT_SEND
, 1);
311 if (kr
!= KERN_SUCCESS
) {
316 res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_START
, rv
->kthread
, rv
->priority
, (uintptr_t)rv
);
319 mach_port_mod_refs(mach_task_self(), rv
->kthread
, MACH_PORT_RIGHT_SEND
, -1);
327 mach_vm_deallocate(mach_task_self(), vm_addr
, round_page(sizeof(struct pthread_override_s
)));
331 return (_Nonnull pthread_override_t
)rv
;
335 pthread_override_qos_class_end_np(pthread_override_t override
)
340 /* Double-free is a fault. Swap the signature and check the old one. */
341 if (_pthread_atomic_xchg_uint32_relaxed(&override
->sig
, PTHREAD_OVERRIDE_SIG_DEAD
) != PTHREAD_OVERRIDE_SIGNATURE
) {
345 /* Always consumes (and deallocates) the pthread_override_t object given. */
346 res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_END
, override
->kthread
, (uintptr_t)override
, 0);
347 if (res
== -1) { res
= errno
; }
349 /* EFAULT from the syscall means we underflowed. Crash here. */
351 // <rdar://problem/17645082> Disable the trap-on-underflow, it doesn't co-exist
352 // with dispatch resetting override counts on threads.
357 kr
= mach_port_mod_refs(mach_task_self(), override
->kthread
, MACH_PORT_RIGHT_SEND
, -1);
358 if (kr
!= KERN_SUCCESS
) {
362 if (override
->malloced
) {
365 kr
= mach_vm_deallocate(mach_task_self(), (mach_vm_address_t
)override
, round_page(sizeof(struct pthread_override_s
)));
366 if (kr
!= KERN_SUCCESS
) {
375 _pthread_qos_override_start_direct(mach_port_t thread
, pthread_priority_t priority
, void *resource
)
377 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_START
, thread
, priority
, (uintptr_t)resource
);
378 if (res
== -1) { res
= errno
; }
383 _pthread_qos_override_end_direct(mach_port_t thread
, void *resource
)
385 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_END
, thread
, (uintptr_t)resource
, 0);
386 if (res
== -1) { res
= errno
; }
391 _pthread_override_qos_class_start_direct(mach_port_t thread
, pthread_priority_t priority
)
393 // use pthread_self as the default per-thread memory allocation to track the override in the kernel
394 return _pthread_qos_override_start_direct(thread
, priority
, pthread_self());
398 _pthread_override_qos_class_end_direct(mach_port_t thread
)
400 // use pthread_self as the default per-thread memory allocation to track the override in the kernel
401 return _pthread_qos_override_end_direct(thread
, pthread_self());
405 _pthread_workqueue_override_start_direct(mach_port_t thread
, pthread_priority_t priority
)
407 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_DISPATCH
, thread
, priority
, 0);
408 if (res
== -1) { res
= errno
; }
413 _pthread_workqueue_override_start_direct_check_owner(mach_port_t thread
, pthread_priority_t priority
, mach_port_t
*ulock_addr
)
415 #if !TARGET_OS_IPHONE
416 static boolean_t kernel_supports_owner_check
= TRUE
;
417 if (!kernel_supports_owner_check
) {
423 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_DISPATCH
, thread
, priority
, ulock_addr
);
424 if (res
== -1) { res
= errno
; }
425 #if !TARGET_OS_IPHONE
426 if (ulock_addr
&& res
== EINVAL
) {
427 if ((uintptr_t)ulock_addr
% _Alignof(_Atomic
uint32_t)) {
428 // do not mute bad ulock addresses related errors
431 // backward compatibility for the XBS chroot
432 // BSDTHREAD_CTL_QOS_OVERRIDE_DISPATCH used to return EINVAL if
433 // arg3 was non NULL.
434 kernel_supports_owner_check
= FALSE
;
439 if (ulock_addr
&& res
== EFAULT
) {
440 // kernel wants us to redrive the call, so while we refault the
441 // memory, also revalidate the owner
442 uint32_t uval
= *(uint32_t volatile *)ulock_addr
;
443 if (ulock_owner_value_to_port_name(uval
) != thread
) {
454 _pthread_workqueue_override_reset(void)
456 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_RESET
, 0, 0, 0);
457 if (res
== -1) { res
= errno
; }
462 _pthread_workqueue_asynchronous_override_add(mach_port_t thread
, pthread_priority_t priority
, void *resource
)
464 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_ADD
, thread
, priority
, (uintptr_t)resource
);
465 if (res
== -1) { res
= errno
; }
470 _pthread_workqueue_asynchronous_override_reset_self(void *resource
)
472 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_RESET
,
476 if (res
== -1) { res
= errno
; }
481 _pthread_workqueue_asynchronous_override_reset_all_self(void)
483 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_RESET
,
487 if (res
== -1) { res
= errno
; }
491 static inline uint16_t
492 _pthread_workqueue_parallelism_for_priority(int qos
, unsigned long flags
)
494 int rc
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_MAX_PARALLELISM
, qos
, flags
, 0);
495 if (os_unlikely(rc
== -1)) {
498 PTHREAD_INTERNAL_CRASH(rc
, "qos_max_parallelism failed");
500 if (flags
& _PTHREAD_QOS_PARALLELISM_COUNT_LOGICAL
) {
501 return *(uint8_t *)_COMM_PAGE_LOGICAL_CPUS
;
503 return *(uint8_t *)_COMM_PAGE_PHYSICAL_CPUS
;
510 pthread_qos_max_parallelism(qos_class_t qos
, unsigned long flags
)
512 thread_qos_t thread_qos
;
513 if (qos
== QOS_CLASS_UNSPECIFIED
) {
514 qos
= QOS_CLASS_DEFAULT
; // <rdar://problem/35080198>
516 thread_qos
= _pthread_qos_class_to_thread_qos(qos
);
517 if (thread_qos
== THREAD_QOS_UNSPECIFIED
) {
522 unsigned long syscall_flags
= _PTHREAD_QOS_PARALLELISM_COUNT_LOGICAL
;
523 uint16_t *ptr
= &_pthread_globals()->qmp_logical
[thread_qos
];
525 if (flags
& PTHREAD_MAX_PARALLELISM_PHYSICAL
) {
527 ptr
= &_pthread_globals()->qmp_physical
[thread_qos
];
530 *ptr
= _pthread_workqueue_parallelism_for_priority(thread_qos
, syscall_flags
);
536 pthread_time_constraint_max_parallelism(unsigned long flags
)
538 unsigned long syscall_flags
= _PTHREAD_QOS_PARALLELISM_COUNT_LOGICAL
;
539 uint16_t *ptr
= &_pthread_globals()->qmp_logical
[0];
541 if (flags
& PTHREAD_MAX_PARALLELISM_PHYSICAL
) {
543 ptr
= &_pthread_globals()->qmp_physical
[0];
546 *ptr
= _pthread_workqueue_parallelism_for_priority(0,
547 syscall_flags
| _PTHREAD_QOS_PARALLELISM_REALTIME
);
553 posix_spawnattr_set_qos_class_np(posix_spawnattr_t
* __restrict __attr
, qos_class_t __qos_class
)
555 switch (__qos_class
) {
556 case QOS_CLASS_UTILITY
:
557 return posix_spawnattr_set_qos_clamp_np(__attr
, POSIX_SPAWN_PROC_CLAMP_UTILITY
);
558 case QOS_CLASS_BACKGROUND
:
559 return posix_spawnattr_set_qos_clamp_np(__attr
, POSIX_SPAWN_PROC_CLAMP_BACKGROUND
);
560 case QOS_CLASS_MAINTENANCE
:
561 return posix_spawnattr_set_qos_clamp_np(__attr
, POSIX_SPAWN_PROC_CLAMP_MAINTENANCE
);
568 posix_spawnattr_get_qos_class_np(const posix_spawnattr_t
*__restrict __attr
, qos_class_t
* __restrict __qos_class
)
576 int rv
= posix_spawnattr_get_qos_clamp_np(__attr
, &clamp
);
582 case POSIX_SPAWN_PROC_CLAMP_UTILITY
:
583 *__qos_class
= QOS_CLASS_UTILITY
;
585 case POSIX_SPAWN_PROC_CLAMP_BACKGROUND
:
586 *__qos_class
= QOS_CLASS_BACKGROUND
;
588 case POSIX_SPAWN_PROC_CLAMP_MAINTENANCE
:
589 *__qos_class
= QOS_CLASS_MAINTENANCE
;
592 *__qos_class
= QOS_CLASS_UNSPECIFIED
;