2 * Copyright (c) 2013 Apple Inc. All rights reserved.
4 * @APPLE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
21 * @APPLE_LICENSE_HEADER_END@
27 #include <mach/mach_vm.h>
30 #include <spawn_private.h>
31 #include <sys/spawn_internal.h>
32 #include <sys/ulock.h>
34 // TODO: remove me when internal.h can include *_private.h itself
35 #include "workqueue_private.h"
36 #include "qos_private.h"
38 #define PTHREAD_OVERRIDE_SIGNATURE (0x6f766572)
39 #define PTHREAD_OVERRIDE_SIG_DEAD (0x7265766f)
41 struct pthread_override_s
46 pthread_priority_t priority
;
51 _pthread_qos_class_to_thread_qos(qos_class_t qos
)
54 case QOS_CLASS_USER_INTERACTIVE
: return THREAD_QOS_USER_INTERACTIVE
;
55 case QOS_CLASS_USER_INITIATED
: return THREAD_QOS_USER_INITIATED
;
56 case QOS_CLASS_DEFAULT
: return THREAD_QOS_LEGACY
;
57 case QOS_CLASS_UTILITY
: return THREAD_QOS_UTILITY
;
58 case QOS_CLASS_BACKGROUND
: return THREAD_QOS_BACKGROUND
;
59 case QOS_CLASS_MAINTENANCE
: return THREAD_QOS_MAINTENANCE
;
60 default: return THREAD_QOS_UNSPECIFIED
;
64 static inline qos_class_t
65 _pthread_qos_class_from_thread_qos(thread_qos_t tqos
)
67 static const qos_class_t thread_qos_to_qos_class
[THREAD_QOS_LAST
] = {
68 [THREAD_QOS_UNSPECIFIED
] = QOS_CLASS_UNSPECIFIED
,
69 [THREAD_QOS_MAINTENANCE
] = QOS_CLASS_MAINTENANCE
,
70 [THREAD_QOS_BACKGROUND
] = QOS_CLASS_BACKGROUND
,
71 [THREAD_QOS_UTILITY
] = QOS_CLASS_UTILITY
,
72 [THREAD_QOS_LEGACY
] = QOS_CLASS_DEFAULT
,
73 [THREAD_QOS_USER_INITIATED
] = QOS_CLASS_USER_INITIATED
,
74 [THREAD_QOS_USER_INTERACTIVE
] = QOS_CLASS_USER_INTERACTIVE
,
76 if (os_unlikely(tqos
>= THREAD_QOS_LAST
)) return QOS_CLASS_UNSPECIFIED
;
77 return thread_qos_to_qos_class
[tqos
];
80 static inline thread_qos_t
81 _pthread_validate_qos_class_and_relpri(qos_class_t qc
, int relpri
)
83 if (relpri
> 0 || relpri
< QOS_MIN_RELATIVE_PRIORITY
) {
84 return THREAD_QOS_UNSPECIFIED
;
86 return _pthread_qos_class_to_thread_qos(qc
);
90 _pthread_priority_split(pthread_priority_t pp
, qos_class_t
*qc
, int *relpri
)
92 thread_qos_t qos
= _pthread_priority_thread_qos(pp
);
93 if (qc
) *qc
= _pthread_qos_class_from_thread_qos(qos
);
94 if (relpri
) *relpri
= _pthread_priority_relpri(pp
);
98 _pthread_set_main_qos(pthread_priority_t qos
)
100 _main_qos
= (uint32_t)qos
;
104 pthread_attr_set_qos_class_np(pthread_attr_t
*attr
, qos_class_t qc
, int relpri
)
106 thread_qos_t qos
= _pthread_validate_qos_class_and_relpri(qc
, relpri
);
107 if (attr
->sig
!= _PTHREAD_ATTR_SIG
|| attr
->schedset
) {
111 attr
->qosclass
= _pthread_priority_make_from_thread_qos(qos
, relpri
, 0);
118 pthread_attr_get_qos_class_np(pthread_attr_t
*attr
, qos_class_t
*qc
, int *relpri
)
120 if (attr
->sig
!= _PTHREAD_ATTR_SIG
) {
124 _pthread_priority_split(attr
->qosset
? attr
->qosclass
: 0, qc
, relpri
);
129 pthread_set_qos_class_self_np(qos_class_t qc
, int relpri
)
131 thread_qos_t qos
= _pthread_validate_qos_class_and_relpri(qc
, relpri
);
136 pthread_priority_t pp
= _pthread_priority_make_from_thread_qos(qos
, relpri
, 0);
137 return _pthread_set_properties_self(_PTHREAD_SET_SELF_QOS_FLAG
, pp
, 0);
141 pthread_set_qos_class_np(pthread_t thread
, qos_class_t qc
, int relpri
)
143 if (thread
!= pthread_self()) {
144 /* The kext now enforces this anyway, if we check here too, it allows us to call
145 * _pthread_set_properties_self later if we can.
149 return pthread_set_qos_class_self_np(qc
, relpri
);
153 pthread_get_qos_class_np(pthread_t thread
, qos_class_t
*qc
, int *relpri
)
155 pthread_priority_t pp
= thread
->tsd
[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
];
156 _pthread_priority_split(pp
, qc
, relpri
);
163 pthread_priority_t pp
;
164 pp
= _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
);
165 return _pthread_qos_class_from_thread_qos(_pthread_priority_thread_qos(pp
));
171 pthread_priority_t pp
= _main_qos
;
172 return _pthread_qos_class_from_thread_qos(_pthread_priority_thread_qos(pp
));
176 _pthread_qos_class_encode(qos_class_t qc
, int relpri
, unsigned long flags
)
178 thread_qos_t qos
= _pthread_qos_class_to_thread_qos(qc
);
179 return _pthread_priority_make_from_thread_qos(qos
, relpri
, flags
);
183 _pthread_qos_class_decode(pthread_priority_t pp
, int *relpri
, unsigned long *flags
)
186 _pthread_priority_split(pp
, &qc
, relpri
);
187 if (flags
) *flags
= (pp
& _PTHREAD_PRIORITY_FLAGS_MASK
);
191 // Encode a legacy workqueue API priority into a pthread_priority_t. This API
192 // is deprecated and can be removed when the simulator no longer uses it.
194 _pthread_qos_class_encode_workqueue(int queue_priority
, unsigned long flags
)
197 switch (queue_priority
) {
198 case WORKQ_HIGH_PRIOQUEUE
: qos
= THREAD_QOS_USER_INTERACTIVE
; break;
199 case WORKQ_DEFAULT_PRIOQUEUE
: qos
= THREAD_QOS_LEGACY
; break;
200 case WORKQ_NON_INTERACTIVE_PRIOQUEUE
:
201 case WORKQ_LOW_PRIOQUEUE
: qos
= THREAD_QOS_UTILITY
; break;
202 case WORKQ_BG_PRIOQUEUE
: qos
= THREAD_QOS_BACKGROUND
; break;
204 PTHREAD_CLIENT_CRASH(queue_priority
, "Invalid priority");
206 return _pthread_priority_make_from_thread_qos(qos
, 0, flags
);
209 #define _PTHREAD_SET_SELF_OUTSIDE_QOS_SKIP \
210 (_PTHREAD_SET_SELF_QOS_FLAG | _PTHREAD_SET_SELF_FIXEDPRIORITY_FLAG | \
211 _PTHREAD_SET_SELF_TIMESHARE_FLAG | \
212 _PTHREAD_SET_SELF_ALTERNATE_AMX)
215 _pthread_set_properties_self(_pthread_set_flags_t flags
,
216 pthread_priority_t priority
, mach_port_t voucher
)
218 pthread_t self
= pthread_self();
219 _pthread_set_flags_t kflags
= flags
;
222 if (self
->wq_outsideqos
&& (flags
& _PTHREAD_SET_SELF_OUTSIDE_QOS_SKIP
)) {
223 // A number of properties cannot be altered if we are a workloop
224 // thread that has outside of QoS properties applied to it.
225 kflags
&= ~_PTHREAD_SET_SELF_OUTSIDE_QOS_SKIP
;
226 if (kflags
== 0) goto skip
;
229 rv
= __bsdthread_ctl(BSDTHREAD_CTL_SET_SELF
, priority
, voucher
, kflags
);
232 // Set QoS TSD if we succeeded, or only failed the voucher portion of the
233 // call. Additionally, if we skipped setting QoS because of outside-of-QoS
234 // attributes then we still want to set the TSD in userspace.
235 if ((flags
& _PTHREAD_SET_SELF_QOS_FLAG
) != 0) {
236 if (rv
== 0 || errno
== ENOENT
) {
237 _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
,
249 pthread_set_fixedpriority_self(void)
251 return _pthread_set_properties_self(_PTHREAD_SET_SELF_FIXEDPRIORITY_FLAG
, 0, 0);
255 pthread_set_timeshare_self(void)
257 return _pthread_set_properties_self(_PTHREAD_SET_SELF_TIMESHARE_FLAG
, 0, 0);
261 pthread_prefer_alternate_amx_self(void)
263 return _pthread_set_properties_self(_PTHREAD_SET_SELF_ALTERNATE_AMX
, 0, 0);
268 pthread_override_qos_class_start_np(pthread_t thread
, qos_class_t qc
, int relpri
)
270 pthread_override_t rv
;
275 /* For now, we don't have access to malloc. So we'll have to vm_allocate this, which means the tiny struct is going
276 * to use an entire page.
278 bool did_malloc
= true;
280 qos
= _pthread_validate_qos_class_and_relpri(qc
, relpri
);
281 if (qos
== THREAD_QOS_UNSPECIFIED
) {
282 return (_Nonnull pthread_override_t
)NULL
;
285 mach_vm_address_t vm_addr
= malloc(sizeof(struct pthread_override_s
));
287 vm_addr
= vm_page_size
;
290 kr
= mach_vm_allocate(mach_task_self(), &vm_addr
,
291 round_page(sizeof(struct pthread_override_s
)),
292 VM_MAKE_TAG(VM_MEMORY_LIBDISPATCH
) | VM_FLAGS_ANYWHERE
);
293 if (kr
!= KERN_SUCCESS
) {
295 return (_Nonnull pthread_override_t
)NULL
;
299 rv
= (pthread_override_t
)vm_addr
;
300 rv
->sig
= PTHREAD_OVERRIDE_SIGNATURE
;
301 rv
->pthread
= thread
;
302 rv
->kthread
= pthread_mach_thread_np(thread
);
303 rv
->priority
= _pthread_priority_make_from_thread_qos(qos
, relpri
, 0);
304 rv
->malloced
= did_malloc
;
306 /* To ensure that the kernel port that we keep stays valid, we retain it here. */
307 kr
= mach_port_mod_refs(mach_task_self(), rv
->kthread
, MACH_PORT_RIGHT_SEND
, 1);
308 if (kr
!= KERN_SUCCESS
) {
313 res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_START
, rv
->kthread
, rv
->priority
, (uintptr_t)rv
);
316 mach_port_mod_refs(mach_task_self(), rv
->kthread
, MACH_PORT_RIGHT_SEND
, -1);
324 mach_vm_deallocate(mach_task_self(), vm_addr
, round_page(sizeof(struct pthread_override_s
)));
328 return (_Nonnull pthread_override_t
)rv
;
332 pthread_override_qos_class_end_np(pthread_override_t override
)
337 /* Double-free is a fault. Swap the signature and check the old one. */
338 if (_pthread_atomic_xchg_uint32_relaxed(&override
->sig
, PTHREAD_OVERRIDE_SIG_DEAD
) != PTHREAD_OVERRIDE_SIGNATURE
) {
342 /* Always consumes (and deallocates) the pthread_override_t object given. */
343 res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_END
, override
->kthread
, (uintptr_t)override
, 0);
344 if (res
== -1) { res
= errno
; }
346 /* EFAULT from the syscall means we underflowed. Crash here. */
348 // <rdar://problem/17645082> Disable the trap-on-underflow, it doesn't co-exist
349 // with dispatch resetting override counts on threads.
354 kr
= mach_port_mod_refs(mach_task_self(), override
->kthread
, MACH_PORT_RIGHT_SEND
, -1);
355 if (kr
!= KERN_SUCCESS
) {
359 if (override
->malloced
) {
362 kr
= mach_vm_deallocate(mach_task_self(), (mach_vm_address_t
)override
, round_page(sizeof(struct pthread_override_s
)));
363 if (kr
!= KERN_SUCCESS
) {
372 _pthread_qos_override_start_direct(mach_port_t thread
, pthread_priority_t priority
, void *resource
)
374 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_START
, thread
, priority
, (uintptr_t)resource
);
375 if (res
== -1) { res
= errno
; }
380 _pthread_qos_override_end_direct(mach_port_t thread
, void *resource
)
382 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_END
, thread
, (uintptr_t)resource
, 0);
383 if (res
== -1) { res
= errno
; }
388 _pthread_override_qos_class_start_direct(mach_port_t thread
, pthread_priority_t priority
)
390 // use pthread_self as the default per-thread memory allocation to track the override in the kernel
391 return _pthread_qos_override_start_direct(thread
, priority
, pthread_self());
395 _pthread_override_qos_class_end_direct(mach_port_t thread
)
397 // use pthread_self as the default per-thread memory allocation to track the override in the kernel
398 return _pthread_qos_override_end_direct(thread
, pthread_self());
402 _pthread_workqueue_override_start_direct(mach_port_t thread
, pthread_priority_t priority
)
404 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_DISPATCH
, thread
, priority
, 0);
405 if (res
== -1) { res
= errno
; }
410 _pthread_workqueue_override_start_direct_check_owner(mach_port_t thread
, pthread_priority_t priority
, mach_port_t
*ulock_addr
)
412 #if !TARGET_OS_IPHONE
413 static boolean_t kernel_supports_owner_check
= TRUE
;
414 if (!kernel_supports_owner_check
) {
420 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_DISPATCH
, thread
, priority
, ulock_addr
);
421 if (res
== -1) { res
= errno
; }
422 #if !TARGET_OS_IPHONE
423 if (ulock_addr
&& res
== EINVAL
) {
424 if ((uintptr_t)ulock_addr
% _Alignof(_Atomic
uint32_t)) {
425 // do not mute bad ulock addresses related errors
428 // backward compatibility for the XBS chroot
429 // BSDTHREAD_CTL_QOS_OVERRIDE_DISPATCH used to return EINVAL if
430 // arg3 was non NULL.
431 kernel_supports_owner_check
= FALSE
;
436 if (ulock_addr
&& res
== EFAULT
) {
437 // kernel wants us to redrive the call, so while we refault the
438 // memory, also revalidate the owner
439 uint32_t uval
= *(uint32_t volatile *)ulock_addr
;
440 if (ulock_owner_value_to_port_name(uval
) != thread
) {
451 _pthread_workqueue_override_reset(void)
453 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_RESET
, 0, 0, 0);
454 if (res
== -1) { res
= errno
; }
459 _pthread_workqueue_asynchronous_override_add(mach_port_t thread
, pthread_priority_t priority
, void *resource
)
461 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_ADD
, thread
, priority
, (uintptr_t)resource
);
462 if (res
== -1) { res
= errno
; }
467 _pthread_workqueue_asynchronous_override_reset_self(void *resource
)
469 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_RESET
,
473 if (res
== -1) { res
= errno
; }
478 _pthread_workqueue_asynchronous_override_reset_all_self(void)
480 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_RESET
,
484 if (res
== -1) { res
= errno
; }
488 static inline uint16_t
489 _pthread_workqueue_parallelism_for_priority(int qos
, unsigned long flags
)
491 int rc
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_MAX_PARALLELISM
, qos
, flags
, 0);
492 if (os_unlikely(rc
== -1)) {
495 PTHREAD_INTERNAL_CRASH(rc
, "qos_max_parallelism failed");
497 if (flags
& _PTHREAD_QOS_PARALLELISM_COUNT_LOGICAL
) {
498 return *(uint8_t *)_COMM_PAGE_LOGICAL_CPUS
;
500 return *(uint8_t *)_COMM_PAGE_PHYSICAL_CPUS
;
507 pthread_qos_max_parallelism(qos_class_t qos
, unsigned long flags
)
509 thread_qos_t thread_qos
;
510 if (qos
== QOS_CLASS_UNSPECIFIED
) {
511 qos
= QOS_CLASS_DEFAULT
; // <rdar://problem/35080198>
513 thread_qos
= _pthread_qos_class_to_thread_qos(qos
);
514 if (thread_qos
== THREAD_QOS_UNSPECIFIED
) {
519 unsigned long syscall_flags
= _PTHREAD_QOS_PARALLELISM_COUNT_LOGICAL
;
520 uint16_t *ptr
= &_pthread_globals()->qmp_logical
[thread_qos
];
522 if (flags
& PTHREAD_MAX_PARALLELISM_PHYSICAL
) {
524 ptr
= &_pthread_globals()->qmp_physical
[thread_qos
];
527 *ptr
= _pthread_workqueue_parallelism_for_priority(thread_qos
, syscall_flags
);
533 pthread_time_constraint_max_parallelism(unsigned long flags
)
535 unsigned long syscall_flags
= _PTHREAD_QOS_PARALLELISM_COUNT_LOGICAL
;
536 uint16_t *ptr
= &_pthread_globals()->qmp_logical
[0];
538 if (flags
& PTHREAD_MAX_PARALLELISM_PHYSICAL
) {
540 ptr
= &_pthread_globals()->qmp_physical
[0];
543 *ptr
= _pthread_workqueue_parallelism_for_priority(0,
544 syscall_flags
| _PTHREAD_QOS_PARALLELISM_REALTIME
);
550 posix_spawnattr_set_qos_class_np(posix_spawnattr_t
* __restrict __attr
, qos_class_t __qos_class
)
552 switch (__qos_class
) {
553 case QOS_CLASS_UTILITY
:
554 return posix_spawnattr_set_qos_clamp_np(__attr
, POSIX_SPAWN_PROC_CLAMP_UTILITY
);
555 case QOS_CLASS_BACKGROUND
:
556 return posix_spawnattr_set_qos_clamp_np(__attr
, POSIX_SPAWN_PROC_CLAMP_BACKGROUND
);
557 case QOS_CLASS_MAINTENANCE
:
558 return posix_spawnattr_set_qos_clamp_np(__attr
, POSIX_SPAWN_PROC_CLAMP_MAINTENANCE
);
565 posix_spawnattr_get_qos_class_np(const posix_spawnattr_t
*__restrict __attr
, qos_class_t
* __restrict __qos_class
)
573 int rv
= posix_spawnattr_get_qos_clamp_np(__attr
, &clamp
);
579 case POSIX_SPAWN_PROC_CLAMP_UTILITY
:
580 *__qos_class
= QOS_CLASS_UTILITY
;
582 case POSIX_SPAWN_PROC_CLAMP_BACKGROUND
:
583 *__qos_class
= QOS_CLASS_BACKGROUND
;
585 case POSIX_SPAWN_PROC_CLAMP_MAINTENANCE
:
586 *__qos_class
= QOS_CLASS_MAINTENANCE
;
589 *__qos_class
= QOS_CLASS_UNSPECIFIED
;