src/qos.c

   1 /*
   2  * Copyright (c) 2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 #include "internal.h"
  25
  26 #include <_simple.h>
  27 #include <mach/mach_vm.h>
  28 #include <unistd.h>
  29 #include <spawn.h>
  30 #include <spawn_private.h>
  31 #include <sys/spawn_internal.h>
  32
  33 // TODO: remove me when internal.h can include *_private.h itself
  34 #include "workqueue_private.h"
  35 #include "qos_private.h"
  36
  37 static pthread_priority_t _main_qos = QOS_CLASS_UNSPECIFIED;
  38
  39 #define PTHREAD_OVERRIDE_SIGNATURE      (0x6f766572)
  40 #define PTHREAD_OVERRIDE_SIG_DEAD       (0x7265766f)
  41
  42 struct pthread_override_s
  43 {
  44         uint32_t sig;
  45         mach_port_t kthread;
  46         pthread_t pthread;
  47         pthread_priority_t priority;
  48         bool malloced;
  49 };
  50
  51 void
  52 _pthread_set_main_qos(pthread_priority_t qos)
  53 {
  54         _main_qos = qos;
  55 }
  56
  57 int
  58 pthread_attr_set_qos_class_np(pthread_attr_t *__attr,
  59                                                           qos_class_t __qos_class,
  60                                                           int __relative_priority)
  61 {
  62         if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) {
  63                 return ENOTSUP;
  64         }
  65
  66         if (__relative_priority > 0 || __relative_priority < QOS_MIN_RELATIVE_PRIORITY) {
  67                 return EINVAL;
  68         }
  69
  70         int ret = EINVAL;
  71         if (__attr->sig == _PTHREAD_ATTR_SIG) {
  72                 if (!__attr->schedset) {
  73                         __attr->qosclass = _pthread_priority_make_newest(__qos_class, __relative_priority, 0);
  74                         __attr->qosset = 1;
  75                         ret = 0;
  76                 }
  77         }
  78
  79         return ret;
  80 }
  81
  82 int
  83 pthread_attr_get_qos_class_np(pthread_attr_t * __restrict __attr,
  84                                                           qos_class_t * __restrict __qos_class,
  85                                                           int * __restrict __relative_priority)
  86 {
  87         if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) {
  88                 return ENOTSUP;
  89         }
  90
  91         int ret = EINVAL;
  92         if (__attr->sig == _PTHREAD_ATTR_SIG) {
  93                 if (__attr->qosset) {
  94                         qos_class_t qos; int relpri;
  95                         _pthread_priority_split_newest(__attr->qosclass, qos, relpri);
  96
  97                         if (__qos_class) { *__qos_class = qos; }
  98                         if (__relative_priority) { *__relative_priority = relpri; }
  99                 } else {
 100                         if (__qos_class) { *__qos_class = 0; }
 101                         if (__relative_priority) { *__relative_priority = 0; }
 102                 }
 103                 ret = 0;
 104         }
 105
 106         return ret;
 107 }
 108
 109 int
 110 pthread_set_qos_class_self_np(qos_class_t __qos_class,
 111                                                           int __relative_priority)
 112 {
 113         if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) {
 114                 return ENOTSUP;
 115         }
 116
 117         if (__relative_priority > 0 || __relative_priority < QOS_MIN_RELATIVE_PRIORITY) {
 118                 return EINVAL;
 119         }
 120
 121         pthread_priority_t priority = _pthread_priority_make_newest(__qos_class, __relative_priority, 0);
 122
 123         if (__pthread_supported_features & PTHREAD_FEATURE_SETSELF) {
 124                 return _pthread_set_properties_self(_PTHREAD_SET_SELF_QOS_FLAG, priority, 0);
 125         } else {
 126                 /* We set the thread QoS class in the TSD and then call into the kernel to
 127                  * read the value out of it and set the QoS class.
 128                  */
 129                 _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS, priority);
 130
 131                 mach_port_t kport = pthread_mach_thread_np(pthread_self());
 132                 int res = __bsdthread_ctl(BSDTHREAD_CTL_SET_QOS, kport, &pthread_self()->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS], 0);
 133
 134                 if (res == -1) {
 135                         res = errno;
 136                 }
 137
 138                 return res;
 139         }
 140 }
 141
 142 int
 143 pthread_set_qos_class_np(pthread_t __pthread,
 144                                                  qos_class_t __qos_class,
 145                                                  int __relative_priority)
 146 {
 147         if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) {
 148                 return ENOTSUP;
 149         }
 150
 151         if (__relative_priority > 0 || __relative_priority < QOS_MIN_RELATIVE_PRIORITY) {
 152                 return EINVAL;
 153         }
 154
 155         if (__pthread != pthread_self()) {
 156                 /* The kext now enforces this anyway, if we check here too, it allows us to call
 157                  * _pthread_set_properties_self later if we can.
 158                  */
 159                 return EPERM;
 160         }
 161
 162         pthread_priority_t priority = _pthread_priority_make_newest(__qos_class, __relative_priority, 0);
 163
 164         if (__pthread_supported_features & PTHREAD_FEATURE_SETSELF) {
 165                 /* If we have _pthread_set_properties_self, then we can easily set this using that. */
 166                 return _pthread_set_properties_self(_PTHREAD_SET_SELF_QOS_FLAG, priority, 0);
 167         } else {
 168                 /* We set the thread QoS class in the TSD and then call into the kernel to
 169                  * read the value out of it and set the QoS class.
 170                  */
 171                 if (__pthread == pthread_self()) {
 172                         _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS, priority);
 173                 } else {
 174                         __pthread->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS] = priority;
 175                 }
 176
 177                 mach_port_t kport = pthread_mach_thread_np(__pthread);
 178                 int res = __bsdthread_ctl(BSDTHREAD_CTL_SET_QOS, kport, &__pthread->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS], 0);
 179
 180                 if (res == -1) {
 181                         res = errno;
 182                 }
 183
 184                 return res;
 185         }
 186 }
 187
 188 int
 189 pthread_get_qos_class_np(pthread_t __pthread,
 190                                                  qos_class_t * __restrict __qos_class,
 191                                                  int * __restrict __relative_priority)
 192 {
 193         if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) {
 194                 return ENOTSUP;
 195         }
 196
 197         pthread_priority_t priority;
 198
 199         if (__pthread == pthread_self()) {
 200                 priority = _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS);
 201         } else {
 202                 priority = __pthread->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS];
 203         }
 204
 205         qos_class_t qos; int relpri;
 206         _pthread_priority_split_newest(priority, qos, relpri);
 207
 208         if (__qos_class) { *__qos_class = qos; }
 209         if (__relative_priority) { *__relative_priority = relpri; }
 210
 211         return 0;
 212 }
 213
 214 qos_class_t
 215 qos_class_self(void)
 216 {
 217         if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) {
 218                 return QOS_CLASS_UNSPECIFIED;
 219         }
 220
 221         pthread_priority_t p = _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS);
 222         qos_class_t c = _pthread_priority_get_qos_newest(p);
 223
 224         return c;
 225 }
 226
 227 qos_class_t
 228 qos_class_main(void)
 229 {
 230         return _pthread_priority_get_qos_newest(_main_qos);
 231 }
 232
 233 pthread_priority_t
 234 _pthread_qos_class_encode(qos_class_t qos_class, int relative_priority, unsigned long flags)
 235 {
 236         if ((__pthread_supported_features & PTHREAD_FEATURE_QOS_MAINTENANCE) == 0) {
 237                 return _pthread_priority_make_version2(qos_class, relative_priority, flags);
 238         } else {
 239                 return _pthread_priority_make_newest(qos_class, relative_priority, flags);
 240         }
 241 }
 242
 243 qos_class_t
 244 _pthread_qos_class_decode(pthread_priority_t priority, int *relative_priority, unsigned long *flags)
 245 {
 246         qos_class_t qos; int relpri;
 247
 248         if ((__pthread_supported_features & PTHREAD_FEATURE_QOS_MAINTENANCE) == 0) {
 249                 _pthread_priority_split_version2(priority, qos, relpri);
 250         } else {
 251                 _pthread_priority_split_newest(priority, qos, relpri);
 252         }
 253
 254         if (relative_priority) { *relative_priority = relpri; }
 255         if (flags) { *flags = _pthread_priority_get_flags(priority); }
 256         return qos;
 257 }
 258
 259 pthread_priority_t
 260 _pthread_qos_class_encode_workqueue(int queue_priority, unsigned long flags)
 261 {
 262         if ((__pthread_supported_features & PTHREAD_FEATURE_QOS_DEFAULT) == 0) {
 263                 switch (queue_priority) {
 264                         case WORKQ_HIGH_PRIOQUEUE:
 265                                 return _pthread_priority_make_version1(QOS_CLASS_USER_INTERACTIVE, 0, flags);
 266                         case WORKQ_DEFAULT_PRIOQUEUE:
 267                                 return _pthread_priority_make_version1(QOS_CLASS_USER_INITIATED, 0, flags);
 268                         case WORKQ_LOW_PRIOQUEUE:
 269                         case WORKQ_NON_INTERACTIVE_PRIOQUEUE:
 270                                 return _pthread_priority_make_version1(QOS_CLASS_UTILITY, 0, flags);
 271                         case WORKQ_BG_PRIOQUEUE:
 272                                 return _pthread_priority_make_version1(QOS_CLASS_BACKGROUND, 0, flags);
 273                         default:
 274                                 __pthread_abort();
 275                 }
 276         }
 277
 278         if ((__pthread_supported_features & PTHREAD_FEATURE_QOS_MAINTENANCE) == 0) {
 279                         switch (queue_priority) {
 280                                 case WORKQ_HIGH_PRIOQUEUE:
 281                                         return _pthread_priority_make_version2(QOS_CLASS_USER_INITIATED, 0, flags);
 282                                 case WORKQ_DEFAULT_PRIOQUEUE:
 283                                         return _pthread_priority_make_version2(QOS_CLASS_DEFAULT, 0, flags);
 284                                 case WORKQ_LOW_PRIOQUEUE:
 285                                 case WORKQ_NON_INTERACTIVE_PRIOQUEUE:
 286                                         return _pthread_priority_make_version2(QOS_CLASS_UTILITY, 0, flags);
 287                                 case WORKQ_BG_PRIOQUEUE:
 288                                         return _pthread_priority_make_version2(QOS_CLASS_BACKGROUND, 0, flags);
 289                                 /* Legacy dispatch does not use QOS_CLASS_MAINTENANCE, so no need to handle it here */
 290                                 default:
 291                                         __pthread_abort();
 292                         }
 293         }
 294
 295         switch (queue_priority) {
 296                 case WORKQ_HIGH_PRIOQUEUE:
 297                         return _pthread_priority_make_newest(QOS_CLASS_USER_INITIATED, 0, flags);
 298                 case WORKQ_DEFAULT_PRIOQUEUE:
 299                         return _pthread_priority_make_newest(QOS_CLASS_DEFAULT, 0, flags);
 300                 case WORKQ_LOW_PRIOQUEUE:
 301                 case WORKQ_NON_INTERACTIVE_PRIOQUEUE:
 302                         return _pthread_priority_make_newest(QOS_CLASS_UTILITY, 0, flags);
 303                 case WORKQ_BG_PRIOQUEUE:
 304                         return _pthread_priority_make_newest(QOS_CLASS_BACKGROUND, 0, flags);
 305                 /* Legacy dispatch does not use QOS_CLASS_MAINTENANCE, so no need to handle it here */
 306                 default:
 307                         __pthread_abort();
 308         }
 309 }
 310
 311 int
 312 _pthread_set_properties_self(_pthread_set_flags_t flags, pthread_priority_t priority, mach_port_t voucher)
 313 {
 314         if (!(__pthread_supported_features & PTHREAD_FEATURE_SETSELF)) {
 315                 return ENOTSUP;
 316         }
 317
 318         int rv = __bsdthread_ctl(BSDTHREAD_CTL_SET_SELF, priority, voucher, flags);
 319
 320         /* Set QoS TSD if we succeeded or only failed the voucher half. */
 321         if ((flags & _PTHREAD_SET_SELF_QOS_FLAG) != 0) {
 322                 if (rv == 0 || errno == ENOENT) {
 323                         _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS, priority);
 324                 }
 325         }
 326
 327         if (rv) {
 328                 rv = errno;
 329         }
 330         return rv;
 331 }
 332
 333 int
 334 pthread_set_fixedpriority_self(void)
 335 {
 336         if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) {
 337                 return ENOTSUP;
 338         }
 339
 340         if (__pthread_supported_features & PTHREAD_FEATURE_SETSELF) {
 341                 return _pthread_set_properties_self(_PTHREAD_SET_SELF_FIXEDPRIORITY_FLAG, 0, 0);
 342         } else {
 343                 return ENOTSUP;
 344         }
 345 }
 346
 347 int
 348 pthread_set_timeshare_self(void)
 349 {
 350         if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) {
 351                 return ENOTSUP;
 352         }
 353
 354         if (__pthread_supported_features & PTHREAD_FEATURE_SETSELF) {
 355                 return _pthread_set_properties_self(_PTHREAD_SET_SELF_TIMESHARE_FLAG, 0, 0);
 356         } else {
 357                 return ENOTSUP;
 358         }
 359 }
 360
 361
 362 pthread_override_t
 363 pthread_override_qos_class_start_np(pthread_t __pthread,  qos_class_t __qos_class, int __relative_priority)
 364 {
 365         pthread_override_t rv;
 366         kern_return_t kr;
 367         int res = 0;
 368
 369         /* For now, we don't have access to malloc. So we'll have to vm_allocate this, which means the tiny struct is going
 370          * to use an entire page.
 371          */
 372         bool did_malloc = true;
 373
 374         mach_vm_address_t vm_addr = malloc(sizeof(struct pthread_override_s));
 375         if (!vm_addr) {
 376                 vm_addr = vm_page_size;
 377                 did_malloc = false;
 378
 379                 kr = mach_vm_allocate(mach_task_self(), &vm_addr, round_page(sizeof(struct pthread_override_s)), VM_MAKE_TAG(VM_MEMORY_LIBDISPATCH) | VM_FLAGS_ANYWHERE);
 380                 if (kr != KERN_SUCCESS) {
 381                         errno = ENOMEM;
 382                         return NULL;
 383                 }
 384         }
 385
 386         rv = (pthread_override_t)vm_addr;
 387         rv->sig = PTHREAD_OVERRIDE_SIGNATURE;
 388         rv->pthread = __pthread;
 389         rv->kthread = pthread_mach_thread_np(__pthread);
 390         rv->priority = _pthread_priority_make_newest(__qos_class, __relative_priority, 0);
 391         rv->malloced = did_malloc;
 392
 393         /* To ensure that the kernel port that we keep stays valid, we retain it here. */
 394         kr = mach_port_mod_refs(mach_task_self(), rv->kthread, MACH_PORT_RIGHT_SEND, 1);
 395         if (kr != KERN_SUCCESS) {
 396                 res = EINVAL;
 397         }
 398
 399         if (res == 0) {
 400                 res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_START, rv->kthread, rv->priority, (uintptr_t)rv);
 401
 402                 if (res != 0) {
 403                         mach_port_mod_refs(mach_task_self(), rv->kthread, MACH_PORT_RIGHT_SEND, -1);
 404                 }
 405         }
 406
 407         if (res != 0) {
 408                 if (did_malloc) {
 409                         free(rv);
 410                 } else {
 411                         mach_vm_deallocate(mach_task_self(), vm_addr, round_page(sizeof(struct pthread_override_s)));
 412                 }
 413                 rv = NULL;
 414         }
 415         return rv;
 416 }
 417
 418 int
 419 pthread_override_qos_class_end_np(pthread_override_t override)
 420 {
 421         kern_return_t kr;
 422         int res = 0;
 423
 424         /* Double-free is a fault. Swap the signature and check the old one. */
 425         if (__sync_swap(&override->sig, PTHREAD_OVERRIDE_SIG_DEAD) != PTHREAD_OVERRIDE_SIGNATURE) {
 426                 __builtin_trap();
 427         }
 428
 429         override->sig = PTHREAD_OVERRIDE_SIG_DEAD;
 430
 431         /* Always consumes (and deallocates) the pthread_override_t object given. */
 432         res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_END, override->kthread, (uintptr_t)override, 0);
 433         if (res == -1) { res = errno; }
 434
 435         /* EFAULT from the syscall means we underflowed. Crash here. */
 436         if (res == EFAULT) {
 437                 // <rdar://problem/17645082> Disable the trap-on-underflow, it doesn't co-exist
 438                 // with dispatch resetting override counts on threads.
 439                 //__builtin_trap();
 440                 res = 0;
 441         }
 442
 443         kr = mach_port_mod_refs(mach_task_self(), override->kthread, MACH_PORT_RIGHT_SEND, -1);
 444         if (kr != KERN_SUCCESS) {
 445                 res = EINVAL;
 446         }
 447
 448         if (override->malloced) {
 449                 free(override);
 450         } else {
 451                 kr = mach_vm_deallocate(mach_task_self(), (mach_vm_address_t)override, round_page(sizeof(struct pthread_override_s)));
 452                 if (kr != KERN_SUCCESS) {
 453                         res = EINVAL;
 454                 }
 455         }
 456
 457         return res;
 458 }
 459
 460 int
 461 _pthread_override_qos_class_start_direct(mach_port_t thread, pthread_priority_t priority)
 462 {
 463         // use pthread_self as the default per-thread memory allocation to track the override in the kernel
 464         int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_START, thread, priority, (uintptr_t)pthread_self());
 465         if (res == -1) { res = errno; }
 466         return res;
 467 }
 468
 469 int
 470 _pthread_override_qos_class_end_direct(mach_port_t thread)
 471 {
 472         // use pthread_self as the default per-thread memory allocation to track the override in the kernel
 473         int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_END, thread, (uintptr_t)pthread_self(), 0);
 474         if (res == -1) { res = errno; }
 475         return res;
 476 }
 477
 478 int
 479 _pthread_workqueue_override_start_direct(mach_port_t thread, pthread_priority_t priority)
 480 {
 481         int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_DISPATCH, thread, priority, 0);
 482         if (res == -1) { res = errno; }
 483         return res;
 484 }
 485
 486 int
 487 _pthread_workqueue_override_reset(void)
 488 {
 489         int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_RESET, 0, 0, 0);
 490         if (res == -1) { res = errno; }
 491         return res;
 492 }
 493
 494 int
 495 _pthread_workqueue_asynchronous_override_add(mach_port_t thread, pthread_priority_t priority, void *resource)
 496 {
 497         int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_ADD, thread, priority, (uintptr_t)resource);
 498         if (res == -1) { res = errno; }
 499         return res;
 500 }
 501
 502 int
 503 _pthread_workqueue_asynchronous_override_reset_self(void *resource)
 504 {
 505         int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_RESET,
 506                                                           0 /* !reset_all */,
 507                                                           (uintptr_t)resource,
 508                                                           0);
 509         if (res == -1) { res = errno; }
 510         return res;
 511 }
 512
 513 int
 514 _pthread_workqueue_asynchronous_override_reset_all_self(void)
 515 {
 516         int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_RESET,
 517                                                           1 /* reset_all */,
 518                                                           0,
 519                                                           0);
 520         if (res == -1) { res = errno; }
 521         return res;
 522 }
 523
 524 int
 525 posix_spawnattr_set_qos_class_np(posix_spawnattr_t * __restrict __attr, qos_class_t __qos_class)
 526 {
 527         switch (__qos_class) {
 528                 case QOS_CLASS_UTILITY:
 529                         return posix_spawnattr_set_qos_clamp_np(__attr, POSIX_SPAWN_PROC_CLAMP_UTILITY);
 530                 case QOS_CLASS_BACKGROUND:
 531                         return posix_spawnattr_set_qos_clamp_np(__attr, POSIX_SPAWN_PROC_CLAMP_BACKGROUND);
 532                 case QOS_CLASS_MAINTENANCE:
 533                         return posix_spawnattr_set_qos_clamp_np(__attr, POSIX_SPAWN_PROC_CLAMP_MAINTENANCE);
 534                 default:
 535                         return EINVAL;
 536         }
 537 }
 538
 539 int
 540 posix_spawnattr_get_qos_class_np(const posix_spawnattr_t *__restrict __attr, qos_class_t * __restrict __qos_class)
 541 {
 542         uint64_t clamp;
 543
 544         if (!__qos_class) {
 545                 return EINVAL;
 546         }
 547
 548         int rv = posix_spawnattr_get_qos_clamp_np(__attr, &clamp);
 549         if (rv != 0) {
 550                 return rv;
 551         }
 552
 553         switch (clamp) {
 554                 case POSIX_SPAWN_PROC_CLAMP_UTILITY:
 555                         *__qos_class = QOS_CLASS_UTILITY;
 556                         break;
 557                 case POSIX_SPAWN_PROC_CLAMP_BACKGROUND:
 558                         *__qos_class = QOS_CLASS_BACKGROUND;
 559                         break;
 560                 case POSIX_SPAWN_PROC_CLAMP_MAINTENANCE:
 561                         *__qos_class = QOS_CLASS_MAINTENANCE;
 562                         break;
 563                 default:
 564                         *__qos_class = QOS_CLASS_UNSPECIFIED;
 565                         break;
 566         }
 567
 568         return 0;
 569 }