2 * Copyright (c) 2013 Apple Inc. All rights reserved.
4 * @APPLE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
21 * @APPLE_LICENSE_HEADER_END@
27 #include <mach/mach_vm.h>
30 #include <spawn_private.h>
31 #include <sys/spawn_internal.h>
32 #include <sys/ulock.h>
34 // TODO: remove me when internal.h can include *_private.h itself
35 #include "workqueue_private.h"
36 #include "qos_private.h"
38 #define PTHREAD_OVERRIDE_SIGNATURE (0x6f766572)
39 #define PTHREAD_OVERRIDE_SIG_DEAD (0x7265766f)
41 struct pthread_override_s
46 pthread_priority_t priority
;
51 _pthread_qos_class_to_thread_qos(qos_class_t qos
)
54 case QOS_CLASS_USER_INTERACTIVE
: return THREAD_QOS_USER_INTERACTIVE
;
55 case QOS_CLASS_USER_INITIATED
: return THREAD_QOS_USER_INITIATED
;
56 case QOS_CLASS_DEFAULT
: return THREAD_QOS_LEGACY
;
57 case QOS_CLASS_UTILITY
: return THREAD_QOS_UTILITY
;
58 case QOS_CLASS_BACKGROUND
: return THREAD_QOS_BACKGROUND
;
59 case QOS_CLASS_MAINTENANCE
: return THREAD_QOS_MAINTENANCE
;
60 default: return THREAD_QOS_UNSPECIFIED
;
64 static inline qos_class_t
65 _pthread_qos_class_from_thread_qos(thread_qos_t tqos
)
67 static const qos_class_t thread_qos_to_qos_class
[THREAD_QOS_LAST
] = {
68 [THREAD_QOS_UNSPECIFIED
] = QOS_CLASS_UNSPECIFIED
,
69 [THREAD_QOS_MAINTENANCE
] = QOS_CLASS_MAINTENANCE
,
70 [THREAD_QOS_BACKGROUND
] = QOS_CLASS_BACKGROUND
,
71 [THREAD_QOS_UTILITY
] = QOS_CLASS_UTILITY
,
72 [THREAD_QOS_LEGACY
] = QOS_CLASS_DEFAULT
,
73 [THREAD_QOS_USER_INITIATED
] = QOS_CLASS_USER_INITIATED
,
74 [THREAD_QOS_USER_INTERACTIVE
] = QOS_CLASS_USER_INTERACTIVE
,
76 if (os_unlikely(tqos
>= THREAD_QOS_LAST
)) return QOS_CLASS_UNSPECIFIED
;
77 return thread_qos_to_qos_class
[tqos
];
80 static inline thread_qos_t
81 _pthread_validate_qos_class_and_relpri(qos_class_t qc
, int relpri
)
83 if (relpri
> 0 || relpri
< QOS_MIN_RELATIVE_PRIORITY
) {
84 return THREAD_QOS_UNSPECIFIED
;
86 return _pthread_qos_class_to_thread_qos(qc
);
90 _pthread_priority_split(pthread_priority_t pp
, qos_class_t
*qc
, int *relpri
)
92 thread_qos_t qos
= _pthread_priority_thread_qos(pp
);
93 if (qc
) *qc
= _pthread_qos_class_from_thread_qos(qos
);
94 if (relpri
) *relpri
= _pthread_priority_relpri(pp
);
98 _pthread_set_main_qos(pthread_priority_t qos
)
100 _main_qos
= (uint32_t)qos
;
104 pthread_attr_set_qos_class_np(pthread_attr_t
*attr
, qos_class_t qc
, int relpri
)
106 thread_qos_t qos
= _pthread_validate_qos_class_and_relpri(qc
, relpri
);
107 if (attr
->sig
!= _PTHREAD_ATTR_SIG
|| attr
->schedset
) {
111 attr
->qosclass
= _pthread_priority_make_from_thread_qos(qos
, relpri
, 0);
118 pthread_attr_get_qos_class_np(pthread_attr_t
*attr
, qos_class_t
*qc
, int *relpri
)
120 if (attr
->sig
!= _PTHREAD_ATTR_SIG
) {
124 _pthread_priority_split(attr
->qosset
? attr
->qosclass
: 0, qc
, relpri
);
129 pthread_set_qos_class_self_np(qos_class_t qc
, int relpri
)
131 thread_qos_t qos
= _pthread_validate_qos_class_and_relpri(qc
, relpri
);
136 pthread_priority_t pp
= _pthread_priority_make_from_thread_qos(qos
, relpri
, 0);
137 return _pthread_set_properties_self(_PTHREAD_SET_SELF_QOS_FLAG
, pp
, 0);
141 pthread_set_qos_class_np(pthread_t thread
, qos_class_t qc
, int relpri
)
143 if (thread
!= pthread_self()) {
144 /* The kext now enforces this anyway, if we check here too, it allows us to call
145 * _pthread_set_properties_self later if we can.
149 return pthread_set_qos_class_self_np(qc
, relpri
);
153 pthread_get_qos_class_np(pthread_t thread
, qos_class_t
*qc
, int *relpri
)
155 pthread_priority_t pp
= thread
->tsd
[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
];
156 _pthread_priority_split(pp
, qc
, relpri
);
163 pthread_priority_t pp
;
164 pp
= _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
);
165 return _pthread_qos_class_from_thread_qos(_pthread_priority_thread_qos(pp
));
171 pthread_priority_t pp
= _main_qos
;
172 return _pthread_qos_class_from_thread_qos(_pthread_priority_thread_qos(pp
));
176 _pthread_qos_class_encode(qos_class_t qc
, int relpri
, unsigned long flags
)
178 thread_qos_t qos
= _pthread_qos_class_to_thread_qos(qc
);
179 return _pthread_priority_make_from_thread_qos(qos
, relpri
, flags
);
183 _pthread_qos_class_decode(pthread_priority_t pp
, int *relpri
, unsigned long *flags
)
186 _pthread_priority_split(pp
, &qc
, relpri
);
187 if (flags
) *flags
= (pp
& _PTHREAD_PRIORITY_FLAGS_MASK
);
191 // Encode a legacy workqueue API priority into a pthread_priority_t. This API
192 // is deprecated and can be removed when the simulator no longer uses it.
194 _pthread_qos_class_encode_workqueue(int queue_priority
, unsigned long flags
)
197 switch (queue_priority
) {
198 case WORKQ_HIGH_PRIOQUEUE
: qos
= THREAD_QOS_USER_INTERACTIVE
; break;
199 case WORKQ_DEFAULT_PRIOQUEUE
: qos
= THREAD_QOS_LEGACY
; break;
200 case WORKQ_NON_INTERACTIVE_PRIOQUEUE
:
201 case WORKQ_LOW_PRIOQUEUE
: qos
= THREAD_QOS_UTILITY
; break;
202 case WORKQ_BG_PRIOQUEUE
: qos
= THREAD_QOS_BACKGROUND
; break;
206 return _pthread_priority_make_from_thread_qos(qos
, 0, flags
);
209 #define _PTHREAD_SET_SELF_OUTSIDE_QOS_SKIP \
210 (_PTHREAD_SET_SELF_QOS_FLAG | _PTHREAD_SET_SELF_FIXEDPRIORITY_FLAG | \
211 _PTHREAD_SET_SELF_TIMESHARE_FLAG)
214 _pthread_set_properties_self(_pthread_set_flags_t flags
,
215 pthread_priority_t priority
, mach_port_t voucher
)
217 pthread_t self
= pthread_self();
218 _pthread_set_flags_t kflags
= flags
;
221 if (self
->wqoutsideqos
&& (flags
& _PTHREAD_SET_SELF_OUTSIDE_QOS_SKIP
)) {
222 // A number of properties cannot be altered if we are a workloop
223 // thread that has outside of QoS properties applied to it.
224 kflags
&= ~_PTHREAD_SET_SELF_OUTSIDE_QOS_SKIP
;
225 if (kflags
== 0) goto skip
;
228 rv
= __bsdthread_ctl(BSDTHREAD_CTL_SET_SELF
, priority
, voucher
, kflags
);
231 // Set QoS TSD if we succeeded, or only failed the voucher portion of the
232 // call. Additionally, if we skipped setting QoS because of outside-of-QoS
233 // attributes then we still want to set the TSD in userspace.
234 if ((flags
& _PTHREAD_SET_SELF_QOS_FLAG
) != 0) {
235 if (rv
== 0 || errno
== ENOENT
) {
236 _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
,
248 pthread_set_fixedpriority_self(void)
250 return _pthread_set_properties_self(_PTHREAD_SET_SELF_FIXEDPRIORITY_FLAG
, 0, 0);
254 pthread_set_timeshare_self(void)
256 return _pthread_set_properties_self(_PTHREAD_SET_SELF_TIMESHARE_FLAG
, 0, 0);
260 pthread_override_qos_class_start_np(pthread_t thread
, qos_class_t qc
, int relpri
)
262 pthread_override_t rv
;
267 /* For now, we don't have access to malloc. So we'll have to vm_allocate this, which means the tiny struct is going
268 * to use an entire page.
270 bool did_malloc
= true;
272 qos
= _pthread_validate_qos_class_and_relpri(qc
, relpri
);
273 if (qos
== THREAD_QOS_UNSPECIFIED
) {
274 return (_Nonnull pthread_override_t
)NULL
;
277 mach_vm_address_t vm_addr
= malloc(sizeof(struct pthread_override_s
));
279 vm_addr
= vm_page_size
;
282 kr
= mach_vm_allocate(mach_task_self(), &vm_addr
,
283 round_page(sizeof(struct pthread_override_s
)),
284 VM_MAKE_TAG(VM_MEMORY_LIBDISPATCH
) | VM_FLAGS_ANYWHERE
);
285 if (kr
!= KERN_SUCCESS
) {
287 return (_Nonnull pthread_override_t
)NULL
;
291 rv
= (pthread_override_t
)vm_addr
;
292 rv
->sig
= PTHREAD_OVERRIDE_SIGNATURE
;
293 rv
->pthread
= thread
;
294 rv
->kthread
= pthread_mach_thread_np(thread
);
295 rv
->priority
= _pthread_priority_make_from_thread_qos(qos
, relpri
, 0);
296 rv
->malloced
= did_malloc
;
298 /* To ensure that the kernel port that we keep stays valid, we retain it here. */
299 kr
= mach_port_mod_refs(mach_task_self(), rv
->kthread
, MACH_PORT_RIGHT_SEND
, 1);
300 if (kr
!= KERN_SUCCESS
) {
305 res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_START
, rv
->kthread
, rv
->priority
, (uintptr_t)rv
);
308 mach_port_mod_refs(mach_task_self(), rv
->kthread
, MACH_PORT_RIGHT_SEND
, -1);
316 mach_vm_deallocate(mach_task_self(), vm_addr
, round_page(sizeof(struct pthread_override_s
)));
320 return (_Nonnull pthread_override_t
)rv
;
324 pthread_override_qos_class_end_np(pthread_override_t override
)
329 /* Double-free is a fault. Swap the signature and check the old one. */
330 if (_pthread_atomic_xchg_uint32_relaxed(&override
->sig
, PTHREAD_OVERRIDE_SIG_DEAD
) != PTHREAD_OVERRIDE_SIGNATURE
) {
334 /* Always consumes (and deallocates) the pthread_override_t object given. */
335 res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_END
, override
->kthread
, (uintptr_t)override
, 0);
336 if (res
== -1) { res
= errno
; }
338 /* EFAULT from the syscall means we underflowed. Crash here. */
340 // <rdar://problem/17645082> Disable the trap-on-underflow, it doesn't co-exist
341 // with dispatch resetting override counts on threads.
346 kr
= mach_port_mod_refs(mach_task_self(), override
->kthread
, MACH_PORT_RIGHT_SEND
, -1);
347 if (kr
!= KERN_SUCCESS
) {
351 if (override
->malloced
) {
354 kr
= mach_vm_deallocate(mach_task_self(), (mach_vm_address_t
)override
, round_page(sizeof(struct pthread_override_s
)));
355 if (kr
!= KERN_SUCCESS
) {
364 _pthread_qos_override_start_direct(mach_port_t thread
, pthread_priority_t priority
, void *resource
)
366 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_START
, thread
, priority
, (uintptr_t)resource
);
367 if (res
== -1) { res
= errno
; }
372 _pthread_qos_override_end_direct(mach_port_t thread
, void *resource
)
374 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_END
, thread
, (uintptr_t)resource
, 0);
375 if (res
== -1) { res
= errno
; }
380 _pthread_override_qos_class_start_direct(mach_port_t thread
, pthread_priority_t priority
)
382 // use pthread_self as the default per-thread memory allocation to track the override in the kernel
383 return _pthread_qos_override_start_direct(thread
, priority
, pthread_self());
387 _pthread_override_qos_class_end_direct(mach_port_t thread
)
389 // use pthread_self as the default per-thread memory allocation to track the override in the kernel
390 return _pthread_qos_override_end_direct(thread
, pthread_self());
394 _pthread_workqueue_override_start_direct(mach_port_t thread
, pthread_priority_t priority
)
396 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_DISPATCH
, thread
, priority
, 0);
397 if (res
== -1) { res
= errno
; }
402 _pthread_workqueue_override_start_direct_check_owner(mach_port_t thread
, pthread_priority_t priority
, mach_port_t
*ulock_addr
)
404 #if !TARGET_OS_IPHONE
405 static boolean_t kernel_supports_owner_check
= TRUE
;
406 if (!kernel_supports_owner_check
) {
412 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_DISPATCH
, thread
, priority
, ulock_addr
);
413 if (res
== -1) { res
= errno
; }
414 #if !TARGET_OS_IPHONE
415 if (ulock_addr
&& res
== EINVAL
) {
416 if ((uintptr_t)ulock_addr
% _Alignof(_Atomic
uint32_t)) {
417 // do not mute bad ulock addresses related errors
420 // backward compatibility for the XBS chroot
421 // BSDTHREAD_CTL_QOS_OVERRIDE_DISPATCH used to return EINVAL if
422 // arg3 was non NULL.
423 kernel_supports_owner_check
= FALSE
;
428 if (ulock_addr
&& res
== EFAULT
) {
429 // kernel wants us to redrive the call, so while we refault the
430 // memory, also revalidate the owner
431 uint32_t uval
= *(uint32_t volatile *)ulock_addr
;
432 if (ulock_owner_value_to_port_name(uval
) != thread
) {
443 _pthread_workqueue_override_reset(void)
445 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_RESET
, 0, 0, 0);
446 if (res
== -1) { res
= errno
; }
451 _pthread_workqueue_asynchronous_override_add(mach_port_t thread
, pthread_priority_t priority
, void *resource
)
453 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_ADD
, thread
, priority
, (uintptr_t)resource
);
454 if (res
== -1) { res
= errno
; }
459 _pthread_workqueue_asynchronous_override_reset_self(void *resource
)
461 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_RESET
,
465 if (res
== -1) { res
= errno
; }
470 _pthread_workqueue_asynchronous_override_reset_all_self(void)
472 int res
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_RESET
,
476 if (res
== -1) { res
= errno
; }
480 static inline uint16_t
481 _pthread_workqueue_parallelism_for_priority(int qos
, unsigned long flags
)
483 int rc
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_MAX_PARALLELISM
, qos
, flags
, 0);
484 if (os_unlikely(rc
== -1)) {
487 PTHREAD_INTERNAL_CRASH(rc
, "qos_max_parallelism failed");
489 if (flags
& _PTHREAD_QOS_PARALLELISM_COUNT_LOGICAL
) {
490 return *(uint8_t *)_COMM_PAGE_LOGICAL_CPUS
;
492 return *(uint8_t *)_COMM_PAGE_PHYSICAL_CPUS
;
499 pthread_qos_max_parallelism(qos_class_t qos
, unsigned long flags
)
501 thread_qos_t thread_qos
;
502 if (qos
== QOS_CLASS_UNSPECIFIED
) {
503 qos
= QOS_CLASS_DEFAULT
; // <rdar://problem/35080198>
505 thread_qos
= _pthread_qos_class_to_thread_qos(qos
);
506 if (thread_qos
== THREAD_QOS_UNSPECIFIED
) {
511 unsigned long syscall_flags
= _PTHREAD_QOS_PARALLELISM_COUNT_LOGICAL
;
512 uint16_t *ptr
= &_pthread_globals()->qmp_logical
[thread_qos
];
514 if (flags
& PTHREAD_MAX_PARALLELISM_PHYSICAL
) {
516 ptr
= &_pthread_globals()->qmp_physical
[thread_qos
];
519 *ptr
= _pthread_workqueue_parallelism_for_priority(thread_qos
, syscall_flags
);
525 pthread_time_constraint_max_parallelism(unsigned long flags
)
527 unsigned long syscall_flags
= _PTHREAD_QOS_PARALLELISM_COUNT_LOGICAL
;
528 uint16_t *ptr
= &_pthread_globals()->qmp_logical
[0];
530 if (flags
& PTHREAD_MAX_PARALLELISM_PHYSICAL
) {
532 ptr
= &_pthread_globals()->qmp_physical
[0];
535 *ptr
= _pthread_workqueue_parallelism_for_priority(0,
536 syscall_flags
| _PTHREAD_QOS_PARALLELISM_REALTIME
);
542 posix_spawnattr_set_qos_class_np(posix_spawnattr_t
* __restrict __attr
, qos_class_t __qos_class
)
544 switch (__qos_class
) {
545 case QOS_CLASS_UTILITY
:
546 return posix_spawnattr_set_qos_clamp_np(__attr
, POSIX_SPAWN_PROC_CLAMP_UTILITY
);
547 case QOS_CLASS_BACKGROUND
:
548 return posix_spawnattr_set_qos_clamp_np(__attr
, POSIX_SPAWN_PROC_CLAMP_BACKGROUND
);
549 case QOS_CLASS_MAINTENANCE
:
550 return posix_spawnattr_set_qos_clamp_np(__attr
, POSIX_SPAWN_PROC_CLAMP_MAINTENANCE
);
557 posix_spawnattr_get_qos_class_np(const posix_spawnattr_t
*__restrict __attr
, qos_class_t
* __restrict __qos_class
)
565 int rv
= posix_spawnattr_get_qos_clamp_np(__attr
, &clamp
);
571 case POSIX_SPAWN_PROC_CLAMP_UTILITY
:
572 *__qos_class
= QOS_CLASS_UTILITY
;
574 case POSIX_SPAWN_PROC_CLAMP_BACKGROUND
:
575 *__qos_class
= QOS_CLASS_BACKGROUND
;
577 case POSIX_SPAWN_PROC_CLAMP_MAINTENANCE
:
578 *__qos_class
= QOS_CLASS_MAINTENANCE
;
581 *__qos_class
= QOS_CLASS_UNSPECIFIED
;