]> git.saurik.com Git - apple/libpthread.git/blame - src/qos.c
libpthread-105.40.1.tar.gz
[apple/libpthread.git] / src / qos.c
CommitLineData
f1a1da6c
A
1/*
2 * Copyright (c) 2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
11 * file.
12 *
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
20 *
21 * @APPLE_LICENSE_HEADER_END@
22 */
23
24#include "internal.h"
25
26#include <_simple.h>
27#include <mach/mach_vm.h>
28#include <unistd.h>
29#include <spawn.h>
30#include <spawn_private.h>
31#include <sys/spawn_internal.h>
32
33// TODO: remove me when internal.h can include *_private.h itself
34#include "workqueue_private.h"
35#include "qos_private.h"
36
37static pthread_priority_t _main_qos = QOS_CLASS_UNSPECIFIED;
38
39#define PTHREAD_OVERRIDE_SIGNATURE (0x6f766572)
40#define PTHREAD_OVERRIDE_SIG_DEAD (0x7265766f)
41
42struct pthread_override_s
43{
44 uint32_t sig;
45 pthread_t pthread;
46 mach_port_t kthread;
47 pthread_priority_t priority;
48 bool malloced;
49};
50
51void
52_pthread_set_main_qos(pthread_priority_t qos)
53{
54 _main_qos = qos;
55}
56
57int
58pthread_attr_set_qos_class_np(pthread_attr_t *__attr,
59 qos_class_t __qos_class,
60 int __relative_priority)
61{
62 if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) {
63 return ENOTSUP;
64 }
65
66 if (__relative_priority > 0 || __relative_priority < QOS_MIN_RELATIVE_PRIORITY) {
67 return EINVAL;
68 }
69
70 int ret = EINVAL;
71 if (__attr->sig == _PTHREAD_ATTR_SIG) {
72 if (!__attr->schedset) {
73 __attr->qosclass = _pthread_priority_make_newest(__qos_class, __relative_priority, 0);
74 __attr->qosset = 1;
75 ret = 0;
76 }
77 }
78
79 return ret;
80}
81
82int
83pthread_attr_get_qos_class_np(pthread_attr_t * __restrict __attr,
84 qos_class_t * __restrict __qos_class,
85 int * __restrict __relative_priority)
86{
87 if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) {
88 return ENOTSUP;
89 }
90
91 int ret = EINVAL;
92 if (__attr->sig == _PTHREAD_ATTR_SIG) {
93 if (__attr->qosset) {
94 qos_class_t qos; int relpri;
95 _pthread_priority_split_newest(__attr->qosclass, qos, relpri);
96
97 if (__qos_class) { *__qos_class = qos; }
98 if (__relative_priority) { *__relative_priority = relpri; }
99 } else {
100 if (__qos_class) { *__qos_class = 0; }
101 if (__relative_priority) { *__relative_priority = 0; }
102 }
103 ret = 0;
104 }
105
106 return ret;
107}
108
109int
110pthread_set_qos_class_self_np(qos_class_t __qos_class,
111 int __relative_priority)
112{
113 if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) {
114 return ENOTSUP;
115 }
116
117 if (__relative_priority > 0 || __relative_priority < QOS_MIN_RELATIVE_PRIORITY) {
118 return EINVAL;
119 }
120
121 pthread_priority_t priority = _pthread_priority_make_newest(__qos_class, __relative_priority, 0);
122
123 if (__pthread_supported_features & PTHREAD_FEATURE_SETSELF) {
124 return _pthread_set_properties_self(_PTHREAD_SET_SELF_QOS_FLAG, priority, 0);
125 } else {
126 /* We set the thread QoS class in the TSD and then call into the kernel to
127 * read the value out of it and set the QoS class.
128 */
129 _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS, priority);
130
131 mach_port_t kport = pthread_mach_thread_np(pthread_self());
132 int res = __bsdthread_ctl(BSDTHREAD_CTL_SET_QOS, kport, &pthread_self()->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS], 0);
133
134 if (res == -1) {
135 res = errno;
136 }
137
138 return res;
139 }
140}
141
142int
143pthread_set_qos_class_np(pthread_t __pthread,
144 qos_class_t __qos_class,
145 int __relative_priority)
146{
147 if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) {
148 return ENOTSUP;
149 }
150
151 if (__relative_priority > 0 || __relative_priority < QOS_MIN_RELATIVE_PRIORITY) {
152 return EINVAL;
153 }
154
155 if (__pthread != pthread_self()) {
156 /* The kext now enforces this anyway, if we check here too, it allows us to call
157 * _pthread_set_properties_self later if we can.
158 */
159 return EPERM;
160 }
161
162 pthread_priority_t priority = _pthread_priority_make_newest(__qos_class, __relative_priority, 0);
163
164 if (__pthread_supported_features & PTHREAD_FEATURE_SETSELF) {
165 /* If we have _pthread_set_properties_self, then we can easily set this using that. */
166 return _pthread_set_properties_self(_PTHREAD_SET_SELF_QOS_FLAG, priority, 0);
167 } else {
168 /* We set the thread QoS class in the TSD and then call into the kernel to
169 * read the value out of it and set the QoS class.
170 */
171 if (__pthread == pthread_self()) {
172 _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS, priority);
173 } else {
174 __pthread->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS] = priority;
175 }
176
177 mach_port_t kport = pthread_mach_thread_np(__pthread);
178 int res = __bsdthread_ctl(BSDTHREAD_CTL_SET_QOS, kport, &__pthread->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS], 0);
179
180 if (res == -1) {
181 res = errno;
182 }
183
184 return res;
185 }
186}
187
188int
189pthread_get_qos_class_np(pthread_t __pthread,
190 qos_class_t * __restrict __qos_class,
191 int * __restrict __relative_priority)
192{
193 if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) {
194 return ENOTSUP;
195 }
196
197 pthread_priority_t priority;
198
199 if (__pthread == pthread_self()) {
200 priority = _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS);
201 } else {
202 priority = __pthread->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS];
203 }
204
205 qos_class_t qos; int relpri;
206 _pthread_priority_split_newest(priority, qos, relpri);
207
208 if (__qos_class) { *__qos_class = qos; }
209 if (__relative_priority) { *__relative_priority = relpri; }
210
211 return 0;
212}
213
214qos_class_t
215qos_class_self(void)
216{
217 if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) {
218 return QOS_CLASS_UNSPECIFIED;
219 }
220
221 pthread_priority_t p = _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS);
222 qos_class_t c = _pthread_priority_get_qos_newest(p);
223
224 return c;
225}
226
227qos_class_t
228qos_class_main(void)
229{
230 return _pthread_priority_get_qos_newest(_main_qos);
231}
232
233pthread_priority_t
234_pthread_qos_class_encode(qos_class_t qos_class, int relative_priority, unsigned long flags)
235{
236 if ((__pthread_supported_features & PTHREAD_FEATURE_QOS_MAINTENANCE) == 0) {
237 return _pthread_priority_make_version2(qos_class, relative_priority, flags);
238 } else {
239 return _pthread_priority_make_newest(qos_class, relative_priority, flags);
240 }
241}
242
243qos_class_t
244_pthread_qos_class_decode(pthread_priority_t priority, int *relative_priority, unsigned long *flags)
245{
246 qos_class_t qos; int relpri;
247
248 if ((__pthread_supported_features & PTHREAD_FEATURE_QOS_MAINTENANCE) == 0) {
249 _pthread_priority_split_version2(priority, qos, relpri);
250 } else {
251 _pthread_priority_split_newest(priority, qos, relpri);
252 }
253
254 if (relative_priority) { *relative_priority = relpri; }
255 if (flags) { *flags = _pthread_priority_get_flags(priority); }
256 return qos;
257}
258
259pthread_priority_t
260_pthread_qos_class_encode_workqueue(int queue_priority, unsigned long flags)
261{
262 if ((__pthread_supported_features & PTHREAD_FEATURE_QOS_DEFAULT) == 0) {
263 switch (queue_priority) {
264 case WORKQ_HIGH_PRIOQUEUE:
265 return _pthread_priority_make_version1(QOS_CLASS_USER_INTERACTIVE, 0, flags);
266 case WORKQ_DEFAULT_PRIOQUEUE:
267 return _pthread_priority_make_version1(QOS_CLASS_USER_INITIATED, 0, flags);
268 case WORKQ_LOW_PRIOQUEUE:
269 case WORKQ_NON_INTERACTIVE_PRIOQUEUE:
270 return _pthread_priority_make_version1(QOS_CLASS_UTILITY, 0, flags);
271 case WORKQ_BG_PRIOQUEUE:
272 return _pthread_priority_make_version1(QOS_CLASS_BACKGROUND, 0, flags);
273 default:
274 __pthread_abort();
275 }
276 }
277
278 if ((__pthread_supported_features & PTHREAD_FEATURE_QOS_MAINTENANCE) == 0) {
279 switch (queue_priority) {
280 case WORKQ_HIGH_PRIOQUEUE:
281 return _pthread_priority_make_version2(QOS_CLASS_USER_INITIATED, 0, flags);
282 case WORKQ_DEFAULT_PRIOQUEUE:
283 return _pthread_priority_make_version2(QOS_CLASS_DEFAULT, 0, flags);
284 case WORKQ_LOW_PRIOQUEUE:
285 case WORKQ_NON_INTERACTIVE_PRIOQUEUE:
286 return _pthread_priority_make_version2(QOS_CLASS_UTILITY, 0, flags);
287 case WORKQ_BG_PRIOQUEUE:
288 return _pthread_priority_make_version2(QOS_CLASS_BACKGROUND, 0, flags);
289 /* Legacy dispatch does not use QOS_CLASS_MAINTENANCE, so no need to handle it here */
290 default:
291 __pthread_abort();
292 }
293 }
294
295 switch (queue_priority) {
296 case WORKQ_HIGH_PRIOQUEUE:
297 return _pthread_priority_make_newest(QOS_CLASS_USER_INITIATED, 0, flags);
298 case WORKQ_DEFAULT_PRIOQUEUE:
299 return _pthread_priority_make_newest(QOS_CLASS_DEFAULT, 0, flags);
300 case WORKQ_LOW_PRIOQUEUE:
301 case WORKQ_NON_INTERACTIVE_PRIOQUEUE:
302 return _pthread_priority_make_newest(QOS_CLASS_UTILITY, 0, flags);
303 case WORKQ_BG_PRIOQUEUE:
304 return _pthread_priority_make_newest(QOS_CLASS_BACKGROUND, 0, flags);
305 /* Legacy dispatch does not use QOS_CLASS_MAINTENANCE, so no need to handle it here */
306 default:
307 __pthread_abort();
308 }
309}
310
311int
312_pthread_set_properties_self(_pthread_set_flags_t flags, pthread_priority_t priority, mach_port_t voucher)
313{
314 if (!(__pthread_supported_features & PTHREAD_FEATURE_SETSELF)) {
315 return ENOTSUP;
316 }
317
318 int rv = __bsdthread_ctl(BSDTHREAD_CTL_SET_SELF, priority, voucher, flags);
319
320 /* Set QoS TSD if we succeeded or only failed the voucher half. */
321 if ((flags & _PTHREAD_SET_SELF_QOS_FLAG) != 0) {
322 if (rv == 0 || errno == ENOENT) {
323 _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS, priority);
324 }
325 }
326
327 if (rv) {
328 rv = errno;
329 }
330 return rv;
331}
332
333int
334pthread_set_fixedpriority_self(void)
335{
336 if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) {
337 return ENOTSUP;
338 }
339
340 if (__pthread_supported_features & PTHREAD_FEATURE_SETSELF) {
341 return _pthread_set_properties_self(_PTHREAD_SET_SELF_FIXEDPRIORITY_FLAG, 0, 0);
342 } else {
343 return ENOTSUP;
344 }
345}
346
347
348pthread_override_t
349pthread_override_qos_class_start_np(pthread_t __pthread, qos_class_t __qos_class, int __relative_priority)
350{
351 pthread_override_t rv;
352 kern_return_t kr;
353 int res = 0;
354
355 /* For now, we don't have access to malloc. So we'll have to vm_allocate this, which means the tiny struct is going
356 * to use an entire page.
357 */
358 bool did_malloc = true;
359
360 mach_vm_address_t vm_addr = malloc(sizeof(struct pthread_override_s));
361 if (!vm_addr) {
362 vm_addr = vm_page_size;
363 did_malloc = false;
364
365 kr = mach_vm_allocate(mach_task_self(), &vm_addr, round_page(sizeof(struct pthread_override_s)), VM_MAKE_TAG(VM_MEMORY_LIBDISPATCH) | VM_FLAGS_ANYWHERE);
366 if (kr != KERN_SUCCESS) {
367 errno = ENOMEM;
368 return NULL;
369 }
370 }
371
372 rv = (pthread_override_t)vm_addr;
373 rv->sig = PTHREAD_OVERRIDE_SIGNATURE;
374 rv->pthread = __pthread;
375 rv->kthread = pthread_mach_thread_np(__pthread);
376 rv->priority = _pthread_priority_make_newest(__qos_class, __relative_priority, 0);
377 rv->malloced = did_malloc;
378
379 /* To ensure that the kernel port that we keep stays valid, we retain it here. */
380 kr = mach_port_mod_refs(mach_task_self(), rv->kthread, MACH_PORT_RIGHT_SEND, 1);
381 if (kr != KERN_SUCCESS) {
382 res = EINVAL;
383 }
384
385 if (res == 0) {
215aeb03 386 res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_START, rv->kthread, rv->priority, (uintptr_t)rv);
f1a1da6c
A
387
388 if (res != 0) {
389 mach_port_mod_refs(mach_task_self(), rv->kthread, MACH_PORT_RIGHT_SEND, -1);
390 }
391 }
392
393 if (res != 0) {
394 if (did_malloc) {
395 free(rv);
396 } else {
397 mach_vm_deallocate(mach_task_self(), vm_addr, round_page(sizeof(struct pthread_override_s)));
398 }
399 rv = NULL;
400 }
401 return rv;
402}
403
404int
405pthread_override_qos_class_end_np(pthread_override_t override)
406{
407 kern_return_t kr;
408 int res = 0;
409
410 /* Double-free is a fault. Swap the signature and check the old one. */
411 if (__sync_swap(&override->sig, PTHREAD_OVERRIDE_SIG_DEAD) != PTHREAD_OVERRIDE_SIGNATURE) {
412 __builtin_trap();
413 }
414
415 override->sig = PTHREAD_OVERRIDE_SIG_DEAD;
416
417 /* Always consumes (and deallocates) the pthread_override_t object given. */
215aeb03 418 res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_END, override->kthread, (uintptr_t)override, 0);
f1a1da6c
A
419 if (res == -1) { res = errno; }
420
421 /* EFAULT from the syscall means we underflowed. Crash here. */
422 if (res == EFAULT) {
423 // <rdar://problem/17645082> Disable the trap-on-underflow, it doesn't co-exist
424 // with dispatch resetting override counts on threads.
425 //__builtin_trap();
426 res = 0;
427 }
428
429 kr = mach_port_mod_refs(mach_task_self(), override->kthread, MACH_PORT_RIGHT_SEND, -1);
430 if (kr != KERN_SUCCESS) {
431 res = EINVAL;
432 }
433
434 if (override->malloced) {
435 free(override);
436 } else {
437 kr = mach_vm_deallocate(mach_task_self(), (mach_vm_address_t)override, round_page(sizeof(struct pthread_override_s)));
438 if (kr != KERN_SUCCESS) {
439 res = EINVAL;
440 }
441 }
442
443 return res;
444}
445
446int
447_pthread_override_qos_class_start_direct(mach_port_t thread, pthread_priority_t priority)
448{
215aeb03
A
449 // use pthread_self as the default per-thread memory allocation to track the override in the kernel
450 int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_START, thread, priority, (uintptr_t)pthread_self());
f1a1da6c
A
451 if (res == -1) { res = errno; }
452 return res;
453}
454
455int
456_pthread_override_qos_class_end_direct(mach_port_t thread)
457{
215aeb03
A
458 // use pthread_self as the default per-thread memory allocation to track the override in the kernel
459 int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_END, thread, (uintptr_t)pthread_self(), 0);
f1a1da6c
A
460 if (res == -1) { res = errno; }
461 return res;
462}
463
464int
465_pthread_workqueue_override_start_direct(mach_port_t thread, pthread_priority_t priority)
466{
467 int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_DISPATCH, thread, priority, 0);
468 if (res == -1) { res = errno; }
469 return res;
470}
471
472int
473_pthread_workqueue_override_reset(void)
474{
475 int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_RESET, 0, 0, 0);
476 if (res == -1) { res = errno; }
477 return res;
478}
479
215aeb03
A
480int
481_pthread_workqueue_asynchronous_override_add(mach_port_t thread, pthread_priority_t priority, void *resource)
482{
483 int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_ADD, thread, priority, (uintptr_t)resource);
484 if (res == -1) { res = errno; }
485 return res;
486}
487
488int
489_pthread_workqueue_asynchronous_override_reset_self(void *resource)
490{
491 int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_RESET,
492 0 /* !reset_all */,
493 (uintptr_t)resource,
494 0);
495 if (res == -1) { res = errno; }
496 return res;
497}
498
499int
500_pthread_workqueue_asynchronous_override_reset_all_self(void)
501{
502 int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_RESET,
503 1 /* reset_all */,
504 0,
505 0);
506 if (res == -1) { res = errno; }
507 return res;
508}
509
f1a1da6c
A
510int
511posix_spawnattr_set_qos_class_np(posix_spawnattr_t * __restrict __attr, qos_class_t __qos_class)
512{
513 switch (__qos_class) {
514 case QOS_CLASS_UTILITY:
515 return posix_spawnattr_set_qos_clamp_np(__attr, POSIX_SPAWN_PROC_CLAMP_UTILITY);
516 case QOS_CLASS_BACKGROUND:
517 return posix_spawnattr_set_qos_clamp_np(__attr, POSIX_SPAWN_PROC_CLAMP_BACKGROUND);
518 case QOS_CLASS_MAINTENANCE:
519 return posix_spawnattr_set_qos_clamp_np(__attr, POSIX_SPAWN_PROC_CLAMP_MAINTENANCE);
520 default:
521 return EINVAL;
522 }
523}
524
525int
526posix_spawnattr_get_qos_class_np(const posix_spawnattr_t *__restrict __attr, qos_class_t * __restrict __qos_class)
527{
528 uint64_t clamp;
529
530 if (!__qos_class) {
531 return EINVAL;
532 }
533
534 int rv = posix_spawnattr_get_qos_clamp_np(__attr, &clamp);
535 if (rv != 0) {
536 return rv;
537 }
538
539 switch (clamp) {
540 case POSIX_SPAWN_PROC_CLAMP_UTILITY:
541 *__qos_class = QOS_CLASS_UTILITY;
542 break;
543 case POSIX_SPAWN_PROC_CLAMP_BACKGROUND:
544 *__qos_class = QOS_CLASS_BACKGROUND;
545 break;
546 case POSIX_SPAWN_PROC_CLAMP_MAINTENANCE:
547 *__qos_class = QOS_CLASS_MAINTENANCE;
548 break;
549 default:
550 *__qos_class = QOS_CLASS_UNSPECIFIED;
551 break;
552 }
553
554 return 0;
555}