]> git.saurik.com Git - apple/libpthread.git/blob - src/qos.c
ef360896e694091bf3b2d021a3132c6d26d53d7e
[apple/libpthread.git] / src / qos.c
1 /*
2 * Copyright (c) 2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
11 * file.
12 *
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
20 *
21 * @APPLE_LICENSE_HEADER_END@
22 */
23
24 #include "internal.h"
25
26 #include <_simple.h>
27 #include <mach/mach_vm.h>
28 #include <unistd.h>
29 #include <spawn.h>
30 #include <spawn_private.h>
31 #include <sys/spawn_internal.h>
32 #include <sys/ulock.h>
33
34 // TODO: remove me when internal.h can include *_private.h itself
35 #include "workqueue_private.h"
36 #include "qos_private.h"
37
38 #define PTHREAD_OVERRIDE_SIGNATURE (0x6f766572)
39 #define PTHREAD_OVERRIDE_SIG_DEAD (0x7265766f)
40
41 struct pthread_override_s
42 {
43 uint32_t sig;
44 mach_port_t kthread;
45 pthread_t pthread;
46 pthread_priority_t priority;
47 bool malloced;
48 };
49
50 thread_qos_t
51 _pthread_qos_class_to_thread_qos(qos_class_t qos)
52 {
53 switch (qos) {
54 case QOS_CLASS_USER_INTERACTIVE: return THREAD_QOS_USER_INTERACTIVE;
55 case QOS_CLASS_USER_INITIATED: return THREAD_QOS_USER_INITIATED;
56 case QOS_CLASS_DEFAULT: return THREAD_QOS_LEGACY;
57 case QOS_CLASS_UTILITY: return THREAD_QOS_UTILITY;
58 case QOS_CLASS_BACKGROUND: return THREAD_QOS_BACKGROUND;
59 case QOS_CLASS_MAINTENANCE: return THREAD_QOS_MAINTENANCE;
60 default: return THREAD_QOS_UNSPECIFIED;
61 }
62 }
63
64 static inline qos_class_t
65 _pthread_qos_class_from_thread_qos(thread_qos_t tqos)
66 {
67 static const qos_class_t thread_qos_to_qos_class[THREAD_QOS_LAST] = {
68 [THREAD_QOS_UNSPECIFIED] = QOS_CLASS_UNSPECIFIED,
69 [THREAD_QOS_MAINTENANCE] = QOS_CLASS_MAINTENANCE,
70 [THREAD_QOS_BACKGROUND] = QOS_CLASS_BACKGROUND,
71 [THREAD_QOS_UTILITY] = QOS_CLASS_UTILITY,
72 [THREAD_QOS_LEGACY] = QOS_CLASS_DEFAULT,
73 [THREAD_QOS_USER_INITIATED] = QOS_CLASS_USER_INITIATED,
74 [THREAD_QOS_USER_INTERACTIVE] = QOS_CLASS_USER_INTERACTIVE,
75 };
76 if (os_unlikely(tqos >= THREAD_QOS_LAST)) return QOS_CLASS_UNSPECIFIED;
77 return thread_qos_to_qos_class[tqos];
78 }
79
80 static inline thread_qos_t
81 _pthread_validate_qos_class_and_relpri(qos_class_t qc, int relpri)
82 {
83 if (relpri > 0 || relpri < QOS_MIN_RELATIVE_PRIORITY) {
84 return THREAD_QOS_UNSPECIFIED;
85 }
86 return _pthread_qos_class_to_thread_qos(qc);
87 }
88
89 static inline void
90 _pthread_priority_split(pthread_priority_t pp, qos_class_t *qc, int *relpri)
91 {
92 thread_qos_t qos = _pthread_priority_thread_qos(pp);
93 if (qc) *qc = _pthread_qos_class_from_thread_qos(qos);
94 if (relpri) *relpri = _pthread_priority_relpri(pp);
95 }
96
97 void
98 _pthread_set_main_qos(pthread_priority_t qos)
99 {
100 _main_qos = (uint32_t)qos;
101 }
102
103 int
104 pthread_attr_set_qos_class_np(pthread_attr_t *attr, qos_class_t qc, int relpri)
105 {
106 thread_qos_t qos = _pthread_validate_qos_class_and_relpri(qc, relpri);
107 if (attr->sig != _PTHREAD_ATTR_SIG || attr->schedset) {
108 return EINVAL;
109 }
110
111 attr->qosclass = _pthread_priority_make_from_thread_qos(qos, relpri, 0);
112 attr->qosset = 1;
113 attr->schedset = 0;
114 return 0;
115 }
116
117 int
118 pthread_attr_get_qos_class_np(pthread_attr_t *attr, qos_class_t *qc, int *relpri)
119 {
120 if (attr->sig != _PTHREAD_ATTR_SIG) {
121 return EINVAL;
122 }
123
124 _pthread_priority_split(attr->qosset ? attr->qosclass : 0, qc, relpri);
125 return 0;
126 }
127
128 int
129 pthread_set_qos_class_self_np(qos_class_t qc, int relpri)
130 {
131 thread_qos_t qos = _pthread_validate_qos_class_and_relpri(qc, relpri);
132 if (!qos) {
133 return EINVAL;
134 }
135
136 pthread_priority_t pp = _pthread_priority_make_from_thread_qos(qos, relpri, 0);
137 return _pthread_set_properties_self(_PTHREAD_SET_SELF_QOS_FLAG, pp, 0);
138 }
139
140 int
141 pthread_set_qos_class_np(pthread_t thread, qos_class_t qc, int relpri)
142 {
143 if (thread != pthread_self()) {
144 /* The kext now enforces this anyway, if we check here too, it allows us to call
145 * _pthread_set_properties_self later if we can.
146 */
147 return EPERM;
148 }
149 return pthread_set_qos_class_self_np(qc, relpri);
150 }
151
152 int
153 pthread_get_qos_class_np(pthread_t thread, qos_class_t *qc, int *relpri)
154 {
155 pthread_priority_t pp = thread->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS];
156 _pthread_priority_split(pp, qc, relpri);
157 return 0;
158 }
159
160 qos_class_t
161 qos_class_self(void)
162 {
163 pthread_priority_t pp;
164 pp = _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS);
165 return _pthread_qos_class_from_thread_qos(_pthread_priority_thread_qos(pp));
166 }
167
168 qos_class_t
169 qos_class_main(void)
170 {
171 pthread_priority_t pp = _main_qos;
172 return _pthread_qos_class_from_thread_qos(_pthread_priority_thread_qos(pp));
173 }
174
175 pthread_priority_t
176 _pthread_qos_class_encode(qos_class_t qc, int relpri, unsigned long flags)
177 {
178 thread_qos_t qos = _pthread_qos_class_to_thread_qos(qc);
179 return _pthread_priority_make_from_thread_qos(qos, relpri, flags);
180 }
181
182 qos_class_t
183 _pthread_qos_class_decode(pthread_priority_t pp, int *relpri, unsigned long *flags)
184 {
185 qos_class_t qc;
186 _pthread_priority_split(pp, &qc, relpri);
187 if (flags) *flags = (pp & _PTHREAD_PRIORITY_FLAGS_MASK);
188 return qc;
189 }
190
191 // Encode a legacy workqueue API priority into a pthread_priority_t. This API
192 // is deprecated and can be removed when the simulator no longer uses it.
193 pthread_priority_t
194 _pthread_qos_class_encode_workqueue(int queue_priority, unsigned long flags)
195 {
196 thread_qos_t qos;
197 switch (queue_priority) {
198 case WORKQ_HIGH_PRIOQUEUE: qos = THREAD_QOS_USER_INTERACTIVE; break;
199 case WORKQ_DEFAULT_PRIOQUEUE: qos = THREAD_QOS_LEGACY; break;
200 case WORKQ_NON_INTERACTIVE_PRIOQUEUE:
201 case WORKQ_LOW_PRIOQUEUE: qos = THREAD_QOS_UTILITY; break;
202 case WORKQ_BG_PRIOQUEUE: qos = THREAD_QOS_BACKGROUND; break;
203 default:
204 __pthread_abort();
205 }
206 return _pthread_priority_make_from_thread_qos(qos, 0, flags);
207 }
208
209 #define _PTHREAD_SET_SELF_OUTSIDE_QOS_SKIP \
210 (_PTHREAD_SET_SELF_QOS_FLAG | _PTHREAD_SET_SELF_FIXEDPRIORITY_FLAG | \
211 _PTHREAD_SET_SELF_TIMESHARE_FLAG)
212
213 int
214 _pthread_set_properties_self(_pthread_set_flags_t flags,
215 pthread_priority_t priority, mach_port_t voucher)
216 {
217 pthread_t self = pthread_self();
218 _pthread_set_flags_t kflags = flags;
219 int rv = 0;
220
221 if (self->wqoutsideqos && (flags & _PTHREAD_SET_SELF_OUTSIDE_QOS_SKIP)) {
222 // A number of properties cannot be altered if we are a workloop
223 // thread that has outside of QoS properties applied to it.
224 kflags &= ~_PTHREAD_SET_SELF_OUTSIDE_QOS_SKIP;
225 if (kflags == 0) goto skip;
226 }
227
228 rv = __bsdthread_ctl(BSDTHREAD_CTL_SET_SELF, priority, voucher, kflags);
229
230 skip:
231 // Set QoS TSD if we succeeded, or only failed the voucher portion of the
232 // call. Additionally, if we skipped setting QoS because of outside-of-QoS
233 // attributes then we still want to set the TSD in userspace.
234 if ((flags & _PTHREAD_SET_SELF_QOS_FLAG) != 0) {
235 if (rv == 0 || errno == ENOENT) {
236 _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS,
237 priority);
238 }
239 }
240
241 if (rv) {
242 rv = errno;
243 }
244 return rv;
245 }
246
247 int
248 pthread_set_fixedpriority_self(void)
249 {
250 return _pthread_set_properties_self(_PTHREAD_SET_SELF_FIXEDPRIORITY_FLAG, 0, 0);
251 }
252
253 int
254 pthread_set_timeshare_self(void)
255 {
256 return _pthread_set_properties_self(_PTHREAD_SET_SELF_TIMESHARE_FLAG, 0, 0);
257 }
258
259 pthread_override_t
260 pthread_override_qos_class_start_np(pthread_t thread, qos_class_t qc, int relpri)
261 {
262 pthread_override_t rv;
263 kern_return_t kr;
264 thread_qos_t qos;
265 int res = 0;
266
267 /* For now, we don't have access to malloc. So we'll have to vm_allocate this, which means the tiny struct is going
268 * to use an entire page.
269 */
270 bool did_malloc = true;
271
272 qos = _pthread_validate_qos_class_and_relpri(qc, relpri);
273 if (qos == THREAD_QOS_UNSPECIFIED) {
274 return (_Nonnull pthread_override_t)NULL;
275 }
276
277 mach_vm_address_t vm_addr = malloc(sizeof(struct pthread_override_s));
278 if (!vm_addr) {
279 vm_addr = vm_page_size;
280 did_malloc = false;
281
282 kr = mach_vm_allocate(mach_task_self(), &vm_addr,
283 round_page(sizeof(struct pthread_override_s)),
284 VM_MAKE_TAG(VM_MEMORY_LIBDISPATCH) | VM_FLAGS_ANYWHERE);
285 if (kr != KERN_SUCCESS) {
286 errno = ENOMEM;
287 return (_Nonnull pthread_override_t)NULL;
288 }
289 }
290
291 rv = (pthread_override_t)vm_addr;
292 rv->sig = PTHREAD_OVERRIDE_SIGNATURE;
293 rv->pthread = thread;
294 rv->kthread = pthread_mach_thread_np(thread);
295 rv->priority = _pthread_priority_make_from_thread_qos(qos, relpri, 0);
296 rv->malloced = did_malloc;
297
298 /* To ensure that the kernel port that we keep stays valid, we retain it here. */
299 kr = mach_port_mod_refs(mach_task_self(), rv->kthread, MACH_PORT_RIGHT_SEND, 1);
300 if (kr != KERN_SUCCESS) {
301 res = EINVAL;
302 }
303
304 if (res == 0) {
305 res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_START, rv->kthread, rv->priority, (uintptr_t)rv);
306
307 if (res != 0) {
308 mach_port_mod_refs(mach_task_self(), rv->kthread, MACH_PORT_RIGHT_SEND, -1);
309 }
310 }
311
312 if (res != 0) {
313 if (did_malloc) {
314 free(rv);
315 } else {
316 mach_vm_deallocate(mach_task_self(), vm_addr, round_page(sizeof(struct pthread_override_s)));
317 }
318 rv = NULL;
319 }
320 return (_Nonnull pthread_override_t)rv;
321 }
322
323 int
324 pthread_override_qos_class_end_np(pthread_override_t override)
325 {
326 kern_return_t kr;
327 int res = 0;
328
329 /* Double-free is a fault. Swap the signature and check the old one. */
330 if (_pthread_atomic_xchg_uint32_relaxed(&override->sig, PTHREAD_OVERRIDE_SIG_DEAD) != PTHREAD_OVERRIDE_SIGNATURE) {
331 __builtin_trap();
332 }
333
334 /* Always consumes (and deallocates) the pthread_override_t object given. */
335 res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_END, override->kthread, (uintptr_t)override, 0);
336 if (res == -1) { res = errno; }
337
338 /* EFAULT from the syscall means we underflowed. Crash here. */
339 if (res == EFAULT) {
340 // <rdar://problem/17645082> Disable the trap-on-underflow, it doesn't co-exist
341 // with dispatch resetting override counts on threads.
342 //__builtin_trap();
343 res = 0;
344 }
345
346 kr = mach_port_mod_refs(mach_task_self(), override->kthread, MACH_PORT_RIGHT_SEND, -1);
347 if (kr != KERN_SUCCESS) {
348 res = EINVAL;
349 }
350
351 if (override->malloced) {
352 free(override);
353 } else {
354 kr = mach_vm_deallocate(mach_task_self(), (mach_vm_address_t)override, round_page(sizeof(struct pthread_override_s)));
355 if (kr != KERN_SUCCESS) {
356 res = EINVAL;
357 }
358 }
359
360 return res;
361 }
362
363 int
364 _pthread_qos_override_start_direct(mach_port_t thread, pthread_priority_t priority, void *resource)
365 {
366 int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_START, thread, priority, (uintptr_t)resource);
367 if (res == -1) { res = errno; }
368 return res;
369 }
370
371 int
372 _pthread_qos_override_end_direct(mach_port_t thread, void *resource)
373 {
374 int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_END, thread, (uintptr_t)resource, 0);
375 if (res == -1) { res = errno; }
376 return res;
377 }
378
379 int
380 _pthread_override_qos_class_start_direct(mach_port_t thread, pthread_priority_t priority)
381 {
382 // use pthread_self as the default per-thread memory allocation to track the override in the kernel
383 return _pthread_qos_override_start_direct(thread, priority, pthread_self());
384 }
385
386 int
387 _pthread_override_qos_class_end_direct(mach_port_t thread)
388 {
389 // use pthread_self as the default per-thread memory allocation to track the override in the kernel
390 return _pthread_qos_override_end_direct(thread, pthread_self());
391 }
392
393 int
394 _pthread_workqueue_override_start_direct(mach_port_t thread, pthread_priority_t priority)
395 {
396 int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_DISPATCH, thread, priority, 0);
397 if (res == -1) { res = errno; }
398 return res;
399 }
400
401 int
402 _pthread_workqueue_override_start_direct_check_owner(mach_port_t thread, pthread_priority_t priority, mach_port_t *ulock_addr)
403 {
404 #if !TARGET_OS_IPHONE
405 static boolean_t kernel_supports_owner_check = TRUE;
406 if (!kernel_supports_owner_check) {
407 ulock_addr = NULL;
408 }
409 #endif
410
411 for (;;) {
412 int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_DISPATCH, thread, priority, ulock_addr);
413 if (res == -1) { res = errno; }
414 #if !TARGET_OS_IPHONE
415 if (ulock_addr && res == EINVAL) {
416 if ((uintptr_t)ulock_addr % _Alignof(_Atomic uint32_t)) {
417 // do not mute bad ulock addresses related errors
418 return EINVAL;
419 }
420 // backward compatibility for the XBS chroot
421 // BSDTHREAD_CTL_QOS_OVERRIDE_DISPATCH used to return EINVAL if
422 // arg3 was non NULL.
423 kernel_supports_owner_check = FALSE;
424 ulock_addr = NULL;
425 continue;
426 }
427 #endif
428 if (ulock_addr && res == EFAULT) {
429 // kernel wants us to redrive the call, so while we refault the
430 // memory, also revalidate the owner
431 uint32_t uval = *(uint32_t volatile *)ulock_addr;
432 if (ulock_owner_value_to_port_name(uval) != thread) {
433 return ESTALE;
434 }
435 continue;
436 }
437
438 return res;
439 }
440 }
441
442 int
443 _pthread_workqueue_override_reset(void)
444 {
445 int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_RESET, 0, 0, 0);
446 if (res == -1) { res = errno; }
447 return res;
448 }
449
450 int
451 _pthread_workqueue_asynchronous_override_add(mach_port_t thread, pthread_priority_t priority, void *resource)
452 {
453 int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_ADD, thread, priority, (uintptr_t)resource);
454 if (res == -1) { res = errno; }
455 return res;
456 }
457
458 int
459 _pthread_workqueue_asynchronous_override_reset_self(void *resource)
460 {
461 int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_RESET,
462 0 /* !reset_all */,
463 (uintptr_t)resource,
464 0);
465 if (res == -1) { res = errno; }
466 return res;
467 }
468
469 int
470 _pthread_workqueue_asynchronous_override_reset_all_self(void)
471 {
472 int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_RESET,
473 1 /* reset_all */,
474 0,
475 0);
476 if (res == -1) { res = errno; }
477 return res;
478 }
479
480 static inline uint16_t
481 _pthread_workqueue_parallelism_for_priority(int qos, unsigned long flags)
482 {
483 int rc = __bsdthread_ctl(BSDTHREAD_CTL_QOS_MAX_PARALLELISM, qos, flags, 0);
484 if (os_unlikely(rc == -1)) {
485 rc = errno;
486 if (rc != EINVAL) {
487 PTHREAD_INTERNAL_CRASH(rc, "qos_max_parallelism failed");
488 }
489 if (flags & _PTHREAD_QOS_PARALLELISM_COUNT_LOGICAL) {
490 return *(uint8_t *)_COMM_PAGE_LOGICAL_CPUS;
491 } else {
492 return *(uint8_t *)_COMM_PAGE_PHYSICAL_CPUS;
493 }
494 }
495 return (uint16_t)rc;
496 }
497
498 int
499 pthread_qos_max_parallelism(qos_class_t qos, unsigned long flags)
500 {
501 thread_qos_t thread_qos;
502 if (qos == QOS_CLASS_UNSPECIFIED) {
503 qos = QOS_CLASS_DEFAULT; // <rdar://problem/35080198>
504 }
505 thread_qos = _pthread_qos_class_to_thread_qos(qos);
506 if (thread_qos == THREAD_QOS_UNSPECIFIED) {
507 errno = EINVAL;
508 return -1;
509 }
510
511 unsigned long syscall_flags = _PTHREAD_QOS_PARALLELISM_COUNT_LOGICAL;
512 uint16_t *ptr = &_pthread_globals()->qmp_logical[thread_qos];
513
514 if (flags & PTHREAD_MAX_PARALLELISM_PHYSICAL) {
515 syscall_flags = 0;
516 ptr = &_pthread_globals()->qmp_physical[thread_qos];
517 }
518 if (*ptr == 0) {
519 *ptr = _pthread_workqueue_parallelism_for_priority(thread_qos, syscall_flags);
520 }
521 return *ptr;
522 }
523
524 int
525 pthread_time_constraint_max_parallelism(unsigned long flags)
526 {
527 unsigned long syscall_flags = _PTHREAD_QOS_PARALLELISM_COUNT_LOGICAL;
528 uint16_t *ptr = &_pthread_globals()->qmp_logical[0];
529
530 if (flags & PTHREAD_MAX_PARALLELISM_PHYSICAL) {
531 syscall_flags = 0;
532 ptr = &_pthread_globals()->qmp_physical[0];
533 }
534 if (*ptr == 0) {
535 *ptr = _pthread_workqueue_parallelism_for_priority(0,
536 syscall_flags | _PTHREAD_QOS_PARALLELISM_REALTIME);
537 }
538 return *ptr;
539 }
540
541 int
542 posix_spawnattr_set_qos_class_np(posix_spawnattr_t * __restrict __attr, qos_class_t __qos_class)
543 {
544 switch (__qos_class) {
545 case QOS_CLASS_UTILITY:
546 return posix_spawnattr_set_qos_clamp_np(__attr, POSIX_SPAWN_PROC_CLAMP_UTILITY);
547 case QOS_CLASS_BACKGROUND:
548 return posix_spawnattr_set_qos_clamp_np(__attr, POSIX_SPAWN_PROC_CLAMP_BACKGROUND);
549 case QOS_CLASS_MAINTENANCE:
550 return posix_spawnattr_set_qos_clamp_np(__attr, POSIX_SPAWN_PROC_CLAMP_MAINTENANCE);
551 default:
552 return EINVAL;
553 }
554 }
555
556 int
557 posix_spawnattr_get_qos_class_np(const posix_spawnattr_t *__restrict __attr, qos_class_t * __restrict __qos_class)
558 {
559 uint64_t clamp;
560
561 if (!__qos_class) {
562 return EINVAL;
563 }
564
565 int rv = posix_spawnattr_get_qos_clamp_np(__attr, &clamp);
566 if (rv != 0) {
567 return rv;
568 }
569
570 switch (clamp) {
571 case POSIX_SPAWN_PROC_CLAMP_UTILITY:
572 *__qos_class = QOS_CLASS_UTILITY;
573 break;
574 case POSIX_SPAWN_PROC_CLAMP_BACKGROUND:
575 *__qos_class = QOS_CLASS_BACKGROUND;
576 break;
577 case POSIX_SPAWN_PROC_CLAMP_MAINTENANCE:
578 *__qos_class = QOS_CLASS_MAINTENANCE;
579 break;
580 default:
581 *__qos_class = QOS_CLASS_UNSPECIFIED;
582 break;
583 }
584
585 return 0;
586 }