src/qos.c

   1 /*
   2  * Copyright (c) 2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 #include "internal.h"
  25
  26 #include <_simple.h>
  27 #include <mach/mach_vm.h>
  28 #include <unistd.h>
  29 #include <spawn.h>
  30 #include <spawn_private.h>
  31 #include <sys/spawn_internal.h>
  32
  33 // TODO: remove me when internal.h can include *_private.h itself
  34 #include "workqueue_private.h"
  35 #include "qos_private.h"
  36
  37 static pthread_priority_t _main_qos = QOS_CLASS_UNSPECIFIED;
  38
  39 #define PTHREAD_OVERRIDE_SIGNATURE      (0x6f766572)
  40 #define PTHREAD_OVERRIDE_SIG_DEAD       (0x7265766f)
  41
  42 struct pthread_override_s
  43 {
  44         uint32_t sig;
  45         pthread_t pthread;
  46         mach_port_t kthread;
  47         pthread_priority_t priority;
  48         bool malloced;
  49 };
  50
  51 void
  52 _pthread_set_main_qos(pthread_priority_t qos)
  53 {
  54         _main_qos = qos;
  55 }
  56
  57 int
  58 pthread_attr_set_qos_class_np(pthread_attr_t *__attr,
  59                                                           qos_class_t __qos_class,
  60                                                           int __relative_priority)
  61 {
  62         if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) {
  63                 return ENOTSUP;
  64         }
  65
  66         if (__relative_priority > 0 || __relative_priority < QOS_MIN_RELATIVE_PRIORITY) {
  67                 return EINVAL;
  68         }
  69
  70         int ret = EINVAL;
  71         if (__attr->sig == _PTHREAD_ATTR_SIG) {
  72                 if (!__attr->schedset) {
  73                         __attr->qosclass = _pthread_priority_make_newest(__qos_class, __relative_priority, 0);
  74                         __attr->qosset = 1;
  75                         ret = 0;
  76                 }
  77         }
  78
  79         return ret;
  80 }
  81
  82 int
  83 pthread_attr_get_qos_class_np(pthread_attr_t * __restrict __attr,
  84                                                           qos_class_t * __restrict __qos_class,
  85                                                           int * __restrict __relative_priority)
  86 {
  87         if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) {
  88                 return ENOTSUP;
  89         }
  90
  91         int ret = EINVAL;
  92         if (__attr->sig == _PTHREAD_ATTR_SIG) {
  93                 if (__attr->qosset) {
  94                         qos_class_t qos; int relpri;
  95                         _pthread_priority_split_newest(__attr->qosclass, qos, relpri);
  96
  97                         if (__qos_class) { *__qos_class = qos; }
  98                         if (__relative_priority) { *__relative_priority = relpri; }
  99                 } else {
 100                         if (__qos_class) { *__qos_class = 0; }
 101                         if (__relative_priority) { *__relative_priority = 0; }
 102                 }
 103                 ret = 0;
 104         }
 105
 106         return ret;
 107 }
 108
 109 int
 110 pthread_set_qos_class_self_np(qos_class_t __qos_class,
 111                                                           int __relative_priority)
 112 {
 113         if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) {
 114                 return ENOTSUP;
 115         }
 116
 117         if (__relative_priority > 0 || __relative_priority < QOS_MIN_RELATIVE_PRIORITY) {
 118                 return EINVAL;
 119         }
 120
 121         pthread_priority_t priority = _pthread_priority_make_newest(__qos_class, __relative_priority, 0);
 122
 123         if (__pthread_supported_features & PTHREAD_FEATURE_SETSELF) {
 124                 return _pthread_set_properties_self(_PTHREAD_SET_SELF_QOS_FLAG, priority, 0);
 125         } else {
 126                 /* We set the thread QoS class in the TSD and then call into the kernel to
 127                  * read the value out of it and set the QoS class.
 128                  */
 129                 _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS, priority);
 130
 131                 mach_port_t kport = pthread_mach_thread_np(pthread_self());
 132                 int res = __bsdthread_ctl(BSDTHREAD_CTL_SET_QOS, kport, &pthread_self()->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS], 0);
 133
 134                 if (res == -1) {
 135                         res = errno;
 136                 }
 137
 138                 return res;
 139         }
 140 }
 141
 142 int
 143 pthread_set_qos_class_np(pthread_t __pthread,
 144                                                  qos_class_t __qos_class,
 145                                                  int __relative_priority)
 146 {
 147         if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) {
 148                 return ENOTSUP;
 149         }
 150
 151         if (__relative_priority > 0 || __relative_priority < QOS_MIN_RELATIVE_PRIORITY) {
 152                 return EINVAL;
 153         }
 154
 155         if (__pthread != pthread_self()) {
 156                 /* The kext now enforces this anyway, if we check here too, it allows us to call
 157                  * _pthread_set_properties_self later if we can.
 158                  */
 159                 return EPERM;
 160         }
 161
 162         pthread_priority_t priority = _pthread_priority_make_newest(__qos_class, __relative_priority, 0);
 163
 164         if (__pthread_supported_features & PTHREAD_FEATURE_SETSELF) {
 165                 /* If we have _pthread_set_properties_self, then we can easily set this using that. */
 166                 return _pthread_set_properties_self(_PTHREAD_SET_SELF_QOS_FLAG, priority, 0);
 167         } else {
 168                 /* We set the thread QoS class in the TSD and then call into the kernel to
 169                  * read the value out of it and set the QoS class.
 170                  */
 171                 if (__pthread == pthread_self()) {
 172                         _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS, priority);
 173                 } else {
 174                         __pthread->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS] = priority;
 175                 }
 176
 177                 mach_port_t kport = pthread_mach_thread_np(__pthread);
 178                 int res = __bsdthread_ctl(BSDTHREAD_CTL_SET_QOS, kport, &__pthread->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS], 0);
 179
 180                 if (res == -1) {
 181                         res = errno;
 182                 }
 183
 184                 return res;
 185         }
 186 }
 187
 188 int
 189 pthread_get_qos_class_np(pthread_t __pthread,
 190                                                  qos_class_t * __restrict __qos_class,
 191                                                  int * __restrict __relative_priority)
 192 {
 193         if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) {
 194                 return ENOTSUP;
 195         }
 196
 197         pthread_priority_t priority;
 198
 199         if (__pthread == pthread_self()) {
 200                 priority = _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS);
 201         } else {
 202                 priority = __pthread->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS];
 203         }
 204
 205         qos_class_t qos; int relpri;
 206         _pthread_priority_split_newest(priority, qos, relpri);
 207
 208         if (__qos_class) { *__qos_class = qos; }
 209         if (__relative_priority) { *__relative_priority = relpri; }
 210
 211         return 0;
 212 }
 213
 214 qos_class_t
 215 qos_class_self(void)
 216 {
 217         if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) {
 218                 return QOS_CLASS_UNSPECIFIED;
 219         }
 220
 221         pthread_priority_t p = _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS);
 222         qos_class_t c = _pthread_priority_get_qos_newest(p);
 223
 224         return c;
 225 }
 226
 227 qos_class_t
 228 qos_class_main(void)
 229 {
 230         return _pthread_priority_get_qos_newest(_main_qos);
 231 }
 232
 233 pthread_priority_t
 234 _pthread_qos_class_encode(qos_class_t qos_class, int relative_priority, unsigned long flags)
 235 {
 236         if ((__pthread_supported_features & PTHREAD_FEATURE_QOS_MAINTENANCE) == 0) {
 237                 return _pthread_priority_make_version2(qos_class, relative_priority, flags);
 238         } else {
 239                 return _pthread_priority_make_newest(qos_class, relative_priority, flags);
 240         }
 241 }
 242
 243 qos_class_t
 244 _pthread_qos_class_decode(pthread_priority_t priority, int *relative_priority, unsigned long *flags)
 245 {
 246         qos_class_t qos; int relpri;
 247
 248         if ((__pthread_supported_features & PTHREAD_FEATURE_QOS_MAINTENANCE) == 0) {
 249                 _pthread_priority_split_version2(priority, qos, relpri);
 250         } else {
 251                 _pthread_priority_split_newest(priority, qos, relpri);
 252         }
 253
 254         if (relative_priority) { *relative_priority = relpri; }
 255         if (flags) { *flags = _pthread_priority_get_flags(priority); }
 256         return qos;
 257 }
 258
 259 pthread_priority_t
 260 _pthread_qos_class_encode_workqueue(int queue_priority, unsigned long flags)
 261 {
 262         if ((__pthread_supported_features & PTHREAD_FEATURE_QOS_DEFAULT) == 0) {
 263                 switch (queue_priority) {
 264                         case WORKQ_HIGH_PRIOQUEUE:
 265                                 return _pthread_priority_make_version1(QOS_CLASS_USER_INTERACTIVE, 0, flags);
 266                         case WORKQ_DEFAULT_PRIOQUEUE:
 267                                 return _pthread_priority_make_version1(QOS_CLASS_USER_INITIATED, 0, flags);
 268                         case WORKQ_LOW_PRIOQUEUE:
 269                         case WORKQ_NON_INTERACTIVE_PRIOQUEUE:
 270                                 return _pthread_priority_make_version1(QOS_CLASS_UTILITY, 0, flags);
 271                         case WORKQ_BG_PRIOQUEUE:
 272                                 return _pthread_priority_make_version1(QOS_CLASS_BACKGROUND, 0, flags);
 273                         default:
 274                                 __pthread_abort();
 275                 }
 276         }
 277
 278         if ((__pthread_supported_features & PTHREAD_FEATURE_QOS_MAINTENANCE) == 0) {
 279                         switch (queue_priority) {
 280                                 case WORKQ_HIGH_PRIOQUEUE:
 281                                         return _pthread_priority_make_version2(QOS_CLASS_USER_INITIATED, 0, flags);
 282                                 case WORKQ_DEFAULT_PRIOQUEUE:
 283                                         return _pthread_priority_make_version2(QOS_CLASS_DEFAULT, 0, flags);
 284                                 case WORKQ_LOW_PRIOQUEUE:
 285                                 case WORKQ_NON_INTERACTIVE_PRIOQUEUE:
 286                                         return _pthread_priority_make_version2(QOS_CLASS_UTILITY, 0, flags);
 287                                 case WORKQ_BG_PRIOQUEUE:
 288                                         return _pthread_priority_make_version2(QOS_CLASS_BACKGROUND, 0, flags);
 289                                 /* Legacy dispatch does not use QOS_CLASS_MAINTENANCE, so no need to handle it here */
 290                                 default:
 291                                         __pthread_abort();
 292                         }
 293         }
 294
 295         switch (queue_priority) {
 296                 case WORKQ_HIGH_PRIOQUEUE:
 297                         return _pthread_priority_make_newest(QOS_CLASS_USER_INITIATED, 0, flags);
 298                 case WORKQ_DEFAULT_PRIOQUEUE:
 299                         return _pthread_priority_make_newest(QOS_CLASS_DEFAULT, 0, flags);
 300                 case WORKQ_LOW_PRIOQUEUE:
 301                 case WORKQ_NON_INTERACTIVE_PRIOQUEUE:
 302                         return _pthread_priority_make_newest(QOS_CLASS_UTILITY, 0, flags);
 303                 case WORKQ_BG_PRIOQUEUE:
 304                         return _pthread_priority_make_newest(QOS_CLASS_BACKGROUND, 0, flags);
 305                 /* Legacy dispatch does not use QOS_CLASS_MAINTENANCE, so no need to handle it here */
 306                 default:
 307                         __pthread_abort();
 308         }
 309 }
 310
 311 int
 312 _pthread_set_properties_self(_pthread_set_flags_t flags, pthread_priority_t priority, mach_port_t voucher)
 313 {
 314         if (!(__pthread_supported_features & PTHREAD_FEATURE_SETSELF)) {
 315                 return ENOTSUP;
 316         }
 317
 318         int rv = __bsdthread_ctl(BSDTHREAD_CTL_SET_SELF, priority, voucher, flags);
 319
 320         /* Set QoS TSD if we succeeded or only failed the voucher half. */
 321         if ((flags & _PTHREAD_SET_SELF_QOS_FLAG) != 0) {
 322                 if (rv == 0 || errno == ENOENT) {
 323                         _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS, priority);
 324                 }
 325         }
 326
 327         if (rv) {
 328                 rv = errno;
 329         }
 330         return rv;
 331 }
 332
 333 int
 334 pthread_set_fixedpriority_self(void)
 335 {
 336         if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) {
 337                 return ENOTSUP;
 338         }
 339
 340         if (__pthread_supported_features & PTHREAD_FEATURE_SETSELF) {
 341                 return _pthread_set_properties_self(_PTHREAD_SET_SELF_FIXEDPRIORITY_FLAG, 0, 0);
 342         } else {
 343                 return ENOTSUP;
 344         }
 345 }
 346
 347
 348 pthread_override_t
 349 pthread_override_qos_class_start_np(pthread_t __pthread,  qos_class_t __qos_class, int __relative_priority)
 350 {
 351         pthread_override_t rv;
 352         kern_return_t kr;
 353         int res = 0;
 354
 355         /* For now, we don't have access to malloc. So we'll have to vm_allocate this, which means the tiny struct is going
 356          * to use an entire page.
 357          */
 358         bool did_malloc = true;
 359
 360         mach_vm_address_t vm_addr = malloc(sizeof(struct pthread_override_s));
 361         if (!vm_addr) {
 362                 vm_addr = vm_page_size;
 363                 did_malloc = false;
 364
 365                 kr = mach_vm_allocate(mach_task_self(), &vm_addr, round_page(sizeof(struct pthread_override_s)), VM_MAKE_TAG(VM_MEMORY_LIBDISPATCH) | VM_FLAGS_ANYWHERE);
 366                 if (kr != KERN_SUCCESS) {
 367                         errno = ENOMEM;
 368                         return NULL;
 369                 }
 370         }
 371
 372         rv = (pthread_override_t)vm_addr;
 373         rv->sig = PTHREAD_OVERRIDE_SIGNATURE;
 374         rv->pthread = __pthread;
 375         rv->kthread = pthread_mach_thread_np(__pthread);
 376         rv->priority = _pthread_priority_make_newest(__qos_class, __relative_priority, 0);
 377         rv->malloced = did_malloc;
 378
 379         /* To ensure that the kernel port that we keep stays valid, we retain it here. */
 380         kr = mach_port_mod_refs(mach_task_self(), rv->kthread, MACH_PORT_RIGHT_SEND, 1);
 381         if (kr != KERN_SUCCESS) {
 382                 res = EINVAL;
 383         }
 384
 385         if (res == 0) {
 386                 res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_START, rv->kthread, rv->priority, 0);
 387
 388                 if (res != 0) {
 389                         mach_port_mod_refs(mach_task_self(), rv->kthread, MACH_PORT_RIGHT_SEND, -1);
 390                 }
 391         }
 392
 393         if (res != 0) {
 394                 if (did_malloc) {
 395                         free(rv);
 396                 } else {
 397                         mach_vm_deallocate(mach_task_self(), vm_addr, round_page(sizeof(struct pthread_override_s)));
 398                 }
 399                 rv = NULL;
 400         }
 401         return rv;
 402 }
 403
 404 int
 405 pthread_override_qos_class_end_np(pthread_override_t override)
 406 {
 407         kern_return_t kr;
 408         int res = 0;
 409
 410         /* Double-free is a fault. Swap the signature and check the old one. */
 411         if (__sync_swap(&override->sig, PTHREAD_OVERRIDE_SIG_DEAD) != PTHREAD_OVERRIDE_SIGNATURE) {
 412                 __builtin_trap();
 413         }
 414
 415         override->sig = PTHREAD_OVERRIDE_SIG_DEAD;
 416
 417         /* Always consumes (and deallocates) the pthread_override_t object given. */
 418         res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_END, override->kthread, 0, 0);
 419         if (res == -1) { res = errno; }
 420
 421         /* EFAULT from the syscall means we underflowed. Crash here. */
 422         if (res == EFAULT) {
 423                 // <rdar://problem/17645082> Disable the trap-on-underflow, it doesn't co-exist
 424                 // with dispatch resetting override counts on threads.
 425                 //__builtin_trap();
 426                 res = 0;
 427         }
 428
 429         kr = mach_port_mod_refs(mach_task_self(), override->kthread, MACH_PORT_RIGHT_SEND, -1);
 430         if (kr != KERN_SUCCESS) {
 431                 res = EINVAL;
 432         }
 433
 434         if (override->malloced) {
 435                 free(override);
 436         } else {
 437                 kr = mach_vm_deallocate(mach_task_self(), (mach_vm_address_t)override, round_page(sizeof(struct pthread_override_s)));
 438                 if (kr != KERN_SUCCESS) {
 439                         res = EINVAL;
 440                 }
 441         }
 442
 443         return res;
 444 }
 445
 446 int
 447 _pthread_override_qos_class_start_direct(mach_port_t thread, pthread_priority_t priority)
 448 {
 449         int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_START, thread, priority, 0);
 450         if (res == -1) { res = errno; }
 451         return res;
 452 }
 453
 454 int
 455 _pthread_override_qos_class_end_direct(mach_port_t thread)
 456 {
 457         int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_END, thread, 0, 0);
 458         if (res == -1) { res = errno; }
 459         return res;
 460 }
 461
 462 int
 463 _pthread_workqueue_override_start_direct(mach_port_t thread, pthread_priority_t priority)
 464 {
 465         int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_DISPATCH, thread, priority, 0);
 466         if (res == -1) { res = errno; }
 467         return res;
 468 }
 469
 470 int
 471 _pthread_workqueue_override_reset(void)
 472 {
 473         int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_RESET, 0, 0, 0);
 474         if (res == -1) { res = errno; }
 475         return res;
 476 }
 477
 478 int
 479 posix_spawnattr_set_qos_class_np(posix_spawnattr_t * __restrict __attr, qos_class_t __qos_class)
 480 {
 481         switch (__qos_class) {
 482                 case QOS_CLASS_UTILITY:
 483                         return posix_spawnattr_set_qos_clamp_np(__attr, POSIX_SPAWN_PROC_CLAMP_UTILITY);
 484                 case QOS_CLASS_BACKGROUND:
 485                         return posix_spawnattr_set_qos_clamp_np(__attr, POSIX_SPAWN_PROC_CLAMP_BACKGROUND);
 486                 case QOS_CLASS_MAINTENANCE:
 487                         return posix_spawnattr_set_qos_clamp_np(__attr, POSIX_SPAWN_PROC_CLAMP_MAINTENANCE);
 488                 default:
 489                         return EINVAL;
 490         }
 491 }
 492
 493 int
 494 posix_spawnattr_get_qos_class_np(const posix_spawnattr_t *__restrict __attr, qos_class_t * __restrict __qos_class)
 495 {
 496         uint64_t clamp;
 497
 498         if (!__qos_class) {
 499                 return EINVAL;
 500         }
 501
 502         int rv = posix_spawnattr_get_qos_clamp_np(__attr, &clamp);
 503         if (rv != 0) {
 504                 return rv;
 505         }
 506
 507         switch (clamp) {
 508                 case POSIX_SPAWN_PROC_CLAMP_UTILITY:
 509                         *__qos_class = QOS_CLASS_UTILITY;
 510                         break;
 511                 case POSIX_SPAWN_PROC_CLAMP_BACKGROUND:
 512                         *__qos_class = QOS_CLASS_BACKGROUND;
 513                         break;
 514                 case POSIX_SPAWN_PROC_CLAMP_MAINTENANCE:
 515                         *__qos_class = QOS_CLASS_MAINTENANCE;
 516                         break;
 517                 default:
 518                         *__qos_class = QOS_CLASS_UNSPECIFIED;
 519                         break;
 520         }
 521
 522         return 0;
 523 }