]> git.saurik.com Git - apple/libpthread.git/blob - src/qos.c
039b06baa73c5afffa0d215eb2ad5f05c241bdc4
[apple/libpthread.git] / src / qos.c
1 /*
2 * Copyright (c) 2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
11 * file.
12 *
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
20 *
21 * @APPLE_LICENSE_HEADER_END@
22 */
23
24 #include "internal.h"
25
26 #include <_simple.h>
27 #include <mach/mach_vm.h>
28 #include <unistd.h>
29 #include <spawn.h>
30 #include <spawn_private.h>
31 #include <sys/spawn_internal.h>
32 #include <sys/ulock.h>
33
34 // TODO: remove me when internal.h can include *_private.h itself
35 #include "workqueue_private.h"
36 #include "qos_private.h"
37
38 #define PTHREAD_OVERRIDE_SIGNATURE (0x6f766572)
39 #define PTHREAD_OVERRIDE_SIG_DEAD (0x7265766f)
40
41 struct pthread_override_s
42 {
43 uint32_t sig;
44 mach_port_t kthread;
45 pthread_t pthread;
46 pthread_priority_t priority;
47 bool malloced;
48 };
49
50 thread_qos_t
51 _pthread_qos_class_to_thread_qos(qos_class_t qos)
52 {
53 switch (qos) {
54 case QOS_CLASS_USER_INTERACTIVE: return THREAD_QOS_USER_INTERACTIVE;
55 case QOS_CLASS_USER_INITIATED: return THREAD_QOS_USER_INITIATED;
56 case QOS_CLASS_DEFAULT: return THREAD_QOS_LEGACY;
57 case QOS_CLASS_UTILITY: return THREAD_QOS_UTILITY;
58 case QOS_CLASS_BACKGROUND: return THREAD_QOS_BACKGROUND;
59 case QOS_CLASS_MAINTENANCE: return THREAD_QOS_MAINTENANCE;
60 default: return THREAD_QOS_UNSPECIFIED;
61 }
62 }
63
64 static inline qos_class_t
65 _pthread_qos_class_from_thread_qos(thread_qos_t tqos)
66 {
67 static const qos_class_t thread_qos_to_qos_class[THREAD_QOS_LAST] = {
68 [THREAD_QOS_UNSPECIFIED] = QOS_CLASS_UNSPECIFIED,
69 [THREAD_QOS_MAINTENANCE] = QOS_CLASS_MAINTENANCE,
70 [THREAD_QOS_BACKGROUND] = QOS_CLASS_BACKGROUND,
71 [THREAD_QOS_UTILITY] = QOS_CLASS_UTILITY,
72 [THREAD_QOS_LEGACY] = QOS_CLASS_DEFAULT,
73 [THREAD_QOS_USER_INITIATED] = QOS_CLASS_USER_INITIATED,
74 [THREAD_QOS_USER_INTERACTIVE] = QOS_CLASS_USER_INTERACTIVE,
75 };
76 if (os_unlikely(tqos >= THREAD_QOS_LAST)) return QOS_CLASS_UNSPECIFIED;
77 return thread_qos_to_qos_class[tqos];
78 }
79
80 static inline thread_qos_t
81 _pthread_validate_qos_class_and_relpri(qos_class_t qc, int relpri)
82 {
83 if (relpri > 0 || relpri < QOS_MIN_RELATIVE_PRIORITY) {
84 return THREAD_QOS_UNSPECIFIED;
85 }
86 return _pthread_qos_class_to_thread_qos(qc);
87 }
88
89 static inline void
90 _pthread_priority_split(pthread_priority_t pp, qos_class_t *qc, int *relpri)
91 {
92 thread_qos_t qos = _pthread_priority_thread_qos(pp);
93 if (qc) *qc = _pthread_qos_class_from_thread_qos(qos);
94 if (relpri) *relpri = _pthread_priority_relpri(pp);
95 }
96
97 void
98 _pthread_set_main_qos(pthread_priority_t qos)
99 {
100 _main_qos = (uint32_t)qos;
101 }
102
103 int
104 pthread_attr_set_qos_class_np(pthread_attr_t *attr, qos_class_t qc, int relpri)
105 {
106 thread_qos_t qos = _pthread_validate_qos_class_and_relpri(qc, relpri);
107 if (attr->sig != _PTHREAD_ATTR_SIG || attr->schedset) {
108 return EINVAL;
109 }
110
111 attr->qosclass = _pthread_priority_make_from_thread_qos(qos, relpri, 0);
112 attr->qosset = 1;
113 attr->schedset = 0;
114 return 0;
115 }
116
117 int
118 pthread_attr_get_qos_class_np(pthread_attr_t *attr, qos_class_t *qc, int *relpri)
119 {
120 if (attr->sig != _PTHREAD_ATTR_SIG) {
121 return EINVAL;
122 }
123
124 _pthread_priority_split(attr->qosset ? attr->qosclass : 0, qc, relpri);
125 return 0;
126 }
127
128 int
129 pthread_set_qos_class_self_np(qos_class_t qc, int relpri)
130 {
131 thread_qos_t qos = _pthread_validate_qos_class_and_relpri(qc, relpri);
132 if (!qos) {
133 return EINVAL;
134 }
135
136 pthread_priority_t pp = _pthread_priority_make_from_thread_qos(qos, relpri, 0);
137 return _pthread_set_properties_self(_PTHREAD_SET_SELF_QOS_FLAG, pp, 0);
138 }
139
140 int
141 pthread_set_qos_class_np(pthread_t thread, qos_class_t qc, int relpri)
142 {
143 if (thread != pthread_self()) {
144 /* The kext now enforces this anyway, if we check here too, it allows us to call
145 * _pthread_set_properties_self later if we can.
146 */
147 return EPERM;
148 }
149 return pthread_set_qos_class_self_np(qc, relpri);
150 }
151
152 int
153 pthread_get_qos_class_np(pthread_t thread, qos_class_t *qc, int *relpri)
154 {
155 pthread_priority_t pp = thread->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS];
156 _pthread_priority_split(pp, qc, relpri);
157 return 0;
158 }
159
160 qos_class_t
161 qos_class_self(void)
162 {
163 pthread_priority_t pp;
164 pp = _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS);
165 return _pthread_qos_class_from_thread_qos(_pthread_priority_thread_qos(pp));
166 }
167
168 qos_class_t
169 qos_class_main(void)
170 {
171 pthread_priority_t pp = _main_qos;
172 return _pthread_qos_class_from_thread_qos(_pthread_priority_thread_qos(pp));
173 }
174
175 pthread_priority_t
176 _pthread_qos_class_encode(qos_class_t qc, int relpri, unsigned long flags)
177 {
178 thread_qos_t qos = _pthread_qos_class_to_thread_qos(qc);
179 return _pthread_priority_make_from_thread_qos(qos, relpri, flags);
180 }
181
182 qos_class_t
183 _pthread_qos_class_decode(pthread_priority_t pp, int *relpri, unsigned long *flags)
184 {
185 qos_class_t qc;
186 _pthread_priority_split(pp, &qc, relpri);
187 if (flags) *flags = (pp & _PTHREAD_PRIORITY_FLAGS_MASK);
188 return qc;
189 }
190
191 // Encode a legacy workqueue API priority into a pthread_priority_t. This API
192 // is deprecated and can be removed when the simulator no longer uses it.
193 pthread_priority_t
194 _pthread_qos_class_encode_workqueue(int queue_priority, unsigned long flags)
195 {
196 thread_qos_t qos;
197 switch (queue_priority) {
198 case WORKQ_HIGH_PRIOQUEUE: qos = THREAD_QOS_USER_INTERACTIVE; break;
199 case WORKQ_DEFAULT_PRIOQUEUE: qos = THREAD_QOS_LEGACY; break;
200 case WORKQ_NON_INTERACTIVE_PRIOQUEUE:
201 case WORKQ_LOW_PRIOQUEUE: qos = THREAD_QOS_UTILITY; break;
202 case WORKQ_BG_PRIOQUEUE: qos = THREAD_QOS_BACKGROUND; break;
203 default:
204 PTHREAD_CLIENT_CRASH(queue_priority, "Invalid priority");
205 }
206 return _pthread_priority_make_from_thread_qos(qos, 0, flags);
207 }
208
209 #define _PTHREAD_SET_SELF_OUTSIDE_QOS_SKIP \
210 (_PTHREAD_SET_SELF_QOS_FLAG | _PTHREAD_SET_SELF_FIXEDPRIORITY_FLAG | \
211 _PTHREAD_SET_SELF_TIMESHARE_FLAG | \
212 _PTHREAD_SET_SELF_ALTERNATE_AMX)
213
214 int
215 _pthread_set_properties_self(_pthread_set_flags_t flags,
216 pthread_priority_t priority, mach_port_t voucher)
217 {
218 pthread_t self = pthread_self();
219 _pthread_set_flags_t kflags = flags;
220 int rv = 0;
221
222 if (self->wq_outsideqos && (flags & _PTHREAD_SET_SELF_OUTSIDE_QOS_SKIP)) {
223 // A number of properties cannot be altered if we are a workloop
224 // thread that has outside of QoS properties applied to it.
225 kflags &= ~_PTHREAD_SET_SELF_OUTSIDE_QOS_SKIP;
226 if (kflags == 0) goto skip;
227 }
228
229 rv = __bsdthread_ctl(BSDTHREAD_CTL_SET_SELF, priority, voucher, kflags);
230
231 skip:
232 // Set QoS TSD if we succeeded, or only failed the voucher portion of the
233 // call. Additionally, if we skipped setting QoS because of outside-of-QoS
234 // attributes then we still want to set the TSD in userspace.
235 if ((flags & _PTHREAD_SET_SELF_QOS_FLAG) != 0) {
236 if (rv == 0 || errno == ENOENT) {
237 _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS,
238 priority);
239 }
240 }
241
242 if (rv) {
243 rv = errno;
244 }
245 return rv;
246 }
247
248 int
249 pthread_set_fixedpriority_self(void)
250 {
251 return _pthread_set_properties_self(_PTHREAD_SET_SELF_FIXEDPRIORITY_FLAG, 0, 0);
252 }
253
254 int
255 pthread_set_timeshare_self(void)
256 {
257 return _pthread_set_properties_self(_PTHREAD_SET_SELF_TIMESHARE_FLAG, 0, 0);
258 }
259
260 int
261 pthread_prefer_alternate_amx_self(void)
262 {
263 return _pthread_set_properties_self(_PTHREAD_SET_SELF_ALTERNATE_AMX, 0, 0);
264 }
265
266
267 pthread_override_t
268 pthread_override_qos_class_start_np(pthread_t thread, qos_class_t qc, int relpri)
269 {
270 pthread_override_t rv;
271 kern_return_t kr;
272 thread_qos_t qos;
273 int res = 0;
274
275 /* For now, we don't have access to malloc. So we'll have to vm_allocate this, which means the tiny struct is going
276 * to use an entire page.
277 */
278 bool did_malloc = true;
279
280 qos = _pthread_validate_qos_class_and_relpri(qc, relpri);
281 if (qos == THREAD_QOS_UNSPECIFIED) {
282 return (_Nonnull pthread_override_t)NULL;
283 }
284
285 mach_vm_address_t vm_addr = malloc(sizeof(struct pthread_override_s));
286 if (!vm_addr) {
287 vm_addr = vm_page_size;
288 did_malloc = false;
289
290 kr = mach_vm_allocate(mach_task_self(), &vm_addr,
291 round_page(sizeof(struct pthread_override_s)),
292 VM_MAKE_TAG(VM_MEMORY_LIBDISPATCH) | VM_FLAGS_ANYWHERE);
293 if (kr != KERN_SUCCESS) {
294 errno = ENOMEM;
295 return (_Nonnull pthread_override_t)NULL;
296 }
297 }
298
299 rv = (pthread_override_t)vm_addr;
300 rv->sig = PTHREAD_OVERRIDE_SIGNATURE;
301 rv->pthread = thread;
302 rv->kthread = pthread_mach_thread_np(thread);
303 rv->priority = _pthread_priority_make_from_thread_qos(qos, relpri, 0);
304 rv->malloced = did_malloc;
305
306 /* To ensure that the kernel port that we keep stays valid, we retain it here. */
307 kr = mach_port_mod_refs(mach_task_self(), rv->kthread, MACH_PORT_RIGHT_SEND, 1);
308 if (kr != KERN_SUCCESS) {
309 res = EINVAL;
310 }
311
312 if (res == 0) {
313 res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_START, rv->kthread, rv->priority, (uintptr_t)rv);
314
315 if (res != 0) {
316 mach_port_mod_refs(mach_task_self(), rv->kthread, MACH_PORT_RIGHT_SEND, -1);
317 }
318 }
319
320 if (res != 0) {
321 if (did_malloc) {
322 free(rv);
323 } else {
324 mach_vm_deallocate(mach_task_self(), vm_addr, round_page(sizeof(struct pthread_override_s)));
325 }
326 rv = NULL;
327 }
328 return (_Nonnull pthread_override_t)rv;
329 }
330
331 int
332 pthread_override_qos_class_end_np(pthread_override_t override)
333 {
334 kern_return_t kr;
335 int res = 0;
336
337 /* Double-free is a fault. Swap the signature and check the old one. */
338 if (_pthread_atomic_xchg_uint32_relaxed(&override->sig, PTHREAD_OVERRIDE_SIG_DEAD) != PTHREAD_OVERRIDE_SIGNATURE) {
339 __builtin_trap();
340 }
341
342 /* Always consumes (and deallocates) the pthread_override_t object given. */
343 res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_END, override->kthread, (uintptr_t)override, 0);
344 if (res == -1) { res = errno; }
345
346 /* EFAULT from the syscall means we underflowed. Crash here. */
347 if (res == EFAULT) {
348 // <rdar://problem/17645082> Disable the trap-on-underflow, it doesn't co-exist
349 // with dispatch resetting override counts on threads.
350 //__builtin_trap();
351 res = 0;
352 }
353
354 kr = mach_port_mod_refs(mach_task_self(), override->kthread, MACH_PORT_RIGHT_SEND, -1);
355 if (kr != KERN_SUCCESS) {
356 res = EINVAL;
357 }
358
359 if (override->malloced) {
360 free(override);
361 } else {
362 kr = mach_vm_deallocate(mach_task_self(), (mach_vm_address_t)override, round_page(sizeof(struct pthread_override_s)));
363 if (kr != KERN_SUCCESS) {
364 res = EINVAL;
365 }
366 }
367
368 return res;
369 }
370
371 int
372 _pthread_qos_override_start_direct(mach_port_t thread, pthread_priority_t priority, void *resource)
373 {
374 int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_START, thread, priority, (uintptr_t)resource);
375 if (res == -1) { res = errno; }
376 return res;
377 }
378
379 int
380 _pthread_qos_override_end_direct(mach_port_t thread, void *resource)
381 {
382 int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_END, thread, (uintptr_t)resource, 0);
383 if (res == -1) { res = errno; }
384 return res;
385 }
386
387 int
388 _pthread_override_qos_class_start_direct(mach_port_t thread, pthread_priority_t priority)
389 {
390 // use pthread_self as the default per-thread memory allocation to track the override in the kernel
391 return _pthread_qos_override_start_direct(thread, priority, pthread_self());
392 }
393
394 int
395 _pthread_override_qos_class_end_direct(mach_port_t thread)
396 {
397 // use pthread_self as the default per-thread memory allocation to track the override in the kernel
398 return _pthread_qos_override_end_direct(thread, pthread_self());
399 }
400
401 int
402 _pthread_workqueue_override_start_direct(mach_port_t thread, pthread_priority_t priority)
403 {
404 int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_DISPATCH, thread, priority, 0);
405 if (res == -1) { res = errno; }
406 return res;
407 }
408
409 int
410 _pthread_workqueue_override_start_direct_check_owner(mach_port_t thread, pthread_priority_t priority, mach_port_t *ulock_addr)
411 {
412 #if !TARGET_OS_IPHONE
413 static boolean_t kernel_supports_owner_check = TRUE;
414 if (!kernel_supports_owner_check) {
415 ulock_addr = NULL;
416 }
417 #endif
418
419 for (;;) {
420 int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_DISPATCH, thread, priority, ulock_addr);
421 if (res == -1) { res = errno; }
422 #if !TARGET_OS_IPHONE
423 if (ulock_addr && res == EINVAL) {
424 if ((uintptr_t)ulock_addr % _Alignof(_Atomic uint32_t)) {
425 // do not mute bad ulock addresses related errors
426 return EINVAL;
427 }
428 // backward compatibility for the XBS chroot
429 // BSDTHREAD_CTL_QOS_OVERRIDE_DISPATCH used to return EINVAL if
430 // arg3 was non NULL.
431 kernel_supports_owner_check = FALSE;
432 ulock_addr = NULL;
433 continue;
434 }
435 #endif
436 if (ulock_addr && res == EFAULT) {
437 // kernel wants us to redrive the call, so while we refault the
438 // memory, also revalidate the owner
439 uint32_t uval = *(uint32_t volatile *)ulock_addr;
440 if (ulock_owner_value_to_port_name(uval) != thread) {
441 return ESTALE;
442 }
443 continue;
444 }
445
446 return res;
447 }
448 }
449
450 int
451 _pthread_workqueue_override_reset(void)
452 {
453 int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_RESET, 0, 0, 0);
454 if (res == -1) { res = errno; }
455 return res;
456 }
457
458 int
459 _pthread_workqueue_asynchronous_override_add(mach_port_t thread, pthread_priority_t priority, void *resource)
460 {
461 int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_ADD, thread, priority, (uintptr_t)resource);
462 if (res == -1) { res = errno; }
463 return res;
464 }
465
466 int
467 _pthread_workqueue_asynchronous_override_reset_self(void *resource)
468 {
469 int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_RESET,
470 0 /* !reset_all */,
471 (uintptr_t)resource,
472 0);
473 if (res == -1) { res = errno; }
474 return res;
475 }
476
477 int
478 _pthread_workqueue_asynchronous_override_reset_all_self(void)
479 {
480 int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_RESET,
481 1 /* reset_all */,
482 0,
483 0);
484 if (res == -1) { res = errno; }
485 return res;
486 }
487
488 static inline uint16_t
489 _pthread_workqueue_parallelism_for_priority(int qos, unsigned long flags)
490 {
491 int rc = __bsdthread_ctl(BSDTHREAD_CTL_QOS_MAX_PARALLELISM, qos, flags, 0);
492 if (os_unlikely(rc == -1)) {
493 rc = errno;
494 if (rc != EINVAL) {
495 PTHREAD_INTERNAL_CRASH(rc, "qos_max_parallelism failed");
496 }
497 if (flags & _PTHREAD_QOS_PARALLELISM_COUNT_LOGICAL) {
498 return *(uint8_t *)_COMM_PAGE_LOGICAL_CPUS;
499 } else {
500 return *(uint8_t *)_COMM_PAGE_PHYSICAL_CPUS;
501 }
502 }
503 return (uint16_t)rc;
504 }
505
506 int
507 pthread_qos_max_parallelism(qos_class_t qos, unsigned long flags)
508 {
509 thread_qos_t thread_qos;
510 if (qos == QOS_CLASS_UNSPECIFIED) {
511 qos = QOS_CLASS_DEFAULT; // <rdar://problem/35080198>
512 }
513 thread_qos = _pthread_qos_class_to_thread_qos(qos);
514 if (thread_qos == THREAD_QOS_UNSPECIFIED) {
515 errno = EINVAL;
516 return -1;
517 }
518
519 unsigned long syscall_flags = _PTHREAD_QOS_PARALLELISM_COUNT_LOGICAL;
520 uint16_t *ptr = &_pthread_globals()->qmp_logical[thread_qos];
521
522 if (flags & PTHREAD_MAX_PARALLELISM_PHYSICAL) {
523 syscall_flags = 0;
524 ptr = &_pthread_globals()->qmp_physical[thread_qos];
525 }
526 if (*ptr == 0) {
527 *ptr = _pthread_workqueue_parallelism_for_priority(thread_qos, syscall_flags);
528 }
529 return *ptr;
530 }
531
532 int
533 pthread_time_constraint_max_parallelism(unsigned long flags)
534 {
535 unsigned long syscall_flags = _PTHREAD_QOS_PARALLELISM_COUNT_LOGICAL;
536 uint16_t *ptr = &_pthread_globals()->qmp_logical[0];
537
538 if (flags & PTHREAD_MAX_PARALLELISM_PHYSICAL) {
539 syscall_flags = 0;
540 ptr = &_pthread_globals()->qmp_physical[0];
541 }
542 if (*ptr == 0) {
543 *ptr = _pthread_workqueue_parallelism_for_priority(0,
544 syscall_flags | _PTHREAD_QOS_PARALLELISM_REALTIME);
545 }
546 return *ptr;
547 }
548
549 int
550 posix_spawnattr_set_qos_class_np(posix_spawnattr_t * __restrict __attr, qos_class_t __qos_class)
551 {
552 switch (__qos_class) {
553 case QOS_CLASS_UTILITY:
554 return posix_spawnattr_set_qos_clamp_np(__attr, POSIX_SPAWN_PROC_CLAMP_UTILITY);
555 case QOS_CLASS_BACKGROUND:
556 return posix_spawnattr_set_qos_clamp_np(__attr, POSIX_SPAWN_PROC_CLAMP_BACKGROUND);
557 case QOS_CLASS_MAINTENANCE:
558 return posix_spawnattr_set_qos_clamp_np(__attr, POSIX_SPAWN_PROC_CLAMP_MAINTENANCE);
559 default:
560 return EINVAL;
561 }
562 }
563
564 int
565 posix_spawnattr_get_qos_class_np(const posix_spawnattr_t *__restrict __attr, qos_class_t * __restrict __qos_class)
566 {
567 uint64_t clamp;
568
569 if (!__qos_class) {
570 return EINVAL;
571 }
572
573 int rv = posix_spawnattr_get_qos_clamp_np(__attr, &clamp);
574 if (rv != 0) {
575 return rv;
576 }
577
578 switch (clamp) {
579 case POSIX_SPAWN_PROC_CLAMP_UTILITY:
580 *__qos_class = QOS_CLASS_UTILITY;
581 break;
582 case POSIX_SPAWN_PROC_CLAMP_BACKGROUND:
583 *__qos_class = QOS_CLASS_BACKGROUND;
584 break;
585 case POSIX_SPAWN_PROC_CLAMP_MAINTENANCE:
586 *__qos_class = QOS_CLASS_MAINTENANCE;
587 break;
588 default:
589 *__qos_class = QOS_CLASS_UNSPECIFIED;
590 break;
591 }
592
593 return 0;
594 }