2  * Copyright (c) 2013 Apple Inc. All rights reserved. 
   4  * @APPLE_LICENSE_HEADER_START@ 
   6  * This file contains Original Code and/or Modifications of Original Code 
   7  * as defined in and that are subject to the Apple Public Source License 
   8  * Version 2.0 (the 'License'). You may not use this file except in 
   9  * compliance with the License. Please obtain a copy of the License at 
  10  * http://www.opensource.apple.com/apsl/ and read it before using this 
  13  * The Original Code and all software distributed under the License are 
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 
  18  * Please see the License for the specific language governing rights and 
  19  * limitations under the License. 
  21  * @APPLE_LICENSE_HEADER_END@ 
  27 #include <mach/mach_vm.h> 
  30 #include <spawn_private.h> 
  31 #include <sys/spawn_internal.h> 
  32 #include <sys/ulock.h> 
  34 // TODO: remove me when internal.h can include *_private.h itself 
  35 #include "workqueue_private.h" 
  36 #include "qos_private.h" 
  38 static pthread_priority_t _main_qos 
= QOS_CLASS_UNSPECIFIED
; 
  40 #define PTHREAD_OVERRIDE_SIGNATURE      (0x6f766572) 
  41 #define PTHREAD_OVERRIDE_SIG_DEAD       (0x7265766f) 
  43 struct pthread_override_s
 
  48         pthread_priority_t priority
; 
  53 _pthread_set_main_qos(pthread_priority_t qos
) 
  59 pthread_attr_set_qos_class_np(pthread_attr_t 
*__attr
, 
  60                                                           qos_class_t __qos_class
, 
  61                                                           int __relative_priority
) 
  63         if (!(__pthread_supported_features 
& PTHREAD_FEATURE_BSDTHREADCTL
)) { 
  67         if (__relative_priority 
> 0 || __relative_priority 
< QOS_MIN_RELATIVE_PRIORITY
) { 
  72         if (__attr
->sig 
== _PTHREAD_ATTR_SIG
) { 
  73                 if (!__attr
->schedset
) { 
  74                         __attr
->qosclass 
= _pthread_priority_make_newest(__qos_class
, __relative_priority
, 0); 
  84 pthread_attr_get_qos_class_np(pthread_attr_t 
* __restrict __attr
, 
  85                                                           qos_class_t 
* __restrict __qos_class
, 
  86                                                           int * __restrict __relative_priority
) 
  88         if (!(__pthread_supported_features 
& PTHREAD_FEATURE_BSDTHREADCTL
)) { 
  93         if (__attr
->sig 
== _PTHREAD_ATTR_SIG
) { 
  95                         qos_class_t qos
; int relpri
; 
  96                         _pthread_priority_split_newest(__attr
->qosclass
, qos
, relpri
); 
  98                         if (__qos_class
) { *__qos_class 
= qos
; } 
  99                         if (__relative_priority
) { *__relative_priority 
= relpri
; } 
 101                         if (__qos_class
) { *__qos_class 
= 0; } 
 102                         if (__relative_priority
) { *__relative_priority 
= 0; } 
 111 pthread_set_qos_class_self_np(qos_class_t __qos_class
, 
 112                                                           int __relative_priority
) 
 114         if (!(__pthread_supported_features 
& PTHREAD_FEATURE_BSDTHREADCTL
)) { 
 118         if (__relative_priority 
> 0 || __relative_priority 
< QOS_MIN_RELATIVE_PRIORITY
) { 
 122         pthread_priority_t priority 
= _pthread_priority_make_newest(__qos_class
, __relative_priority
, 0); 
 124         if (__pthread_supported_features 
& PTHREAD_FEATURE_SETSELF
) { 
 125                 return _pthread_set_properties_self(_PTHREAD_SET_SELF_QOS_FLAG
, priority
, 0); 
 127                 /* We set the thread QoS class in the TSD and then call into the kernel to 
 128                  * read the value out of it and set the QoS class. 
 130                 _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
, priority
); 
 132                 mach_port_t kport 
= pthread_mach_thread_np(pthread_self()); 
 133                 int res 
= __bsdthread_ctl(BSDTHREAD_CTL_SET_QOS
, kport
, &pthread_self()->tsd
[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
], 0); 
 144 pthread_set_qos_class_np(pthread_t __pthread
, 
 145                                                  qos_class_t __qos_class
, 
 146                                                  int __relative_priority
) 
 148         if (!(__pthread_supported_features 
& PTHREAD_FEATURE_BSDTHREADCTL
)) { 
 152         if (__relative_priority 
> 0 || __relative_priority 
< QOS_MIN_RELATIVE_PRIORITY
) { 
 156         if (__pthread 
!= pthread_self()) { 
 157                 /* The kext now enforces this anyway, if we check here too, it allows us to call 
 158                  * _pthread_set_properties_self later if we can. 
 163         pthread_priority_t priority 
= _pthread_priority_make_newest(__qos_class
, __relative_priority
, 0); 
 165         if (__pthread_supported_features 
& PTHREAD_FEATURE_SETSELF
) { 
 166                 /* If we have _pthread_set_properties_self, then we can easily set this using that. */ 
 167                 return _pthread_set_properties_self(_PTHREAD_SET_SELF_QOS_FLAG
, priority
, 0); 
 169                 /* We set the thread QoS class in the TSD and then call into the kernel to 
 170                  * read the value out of it and set the QoS class. 
 172                 if (__pthread 
== pthread_self()) { 
 173                         _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
, priority
); 
 175                         __pthread
->tsd
[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
] = priority
; 
 178                 mach_port_t kport 
= pthread_mach_thread_np(__pthread
); 
 179                 int res 
= __bsdthread_ctl(BSDTHREAD_CTL_SET_QOS
, kport
, &__pthread
->tsd
[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
], 0); 
 190 pthread_get_qos_class_np(pthread_t __pthread
, 
 191                                                  qos_class_t 
* __restrict __qos_class
, 
 192                                                  int * __restrict __relative_priority
) 
 194         if (!(__pthread_supported_features 
& PTHREAD_FEATURE_BSDTHREADCTL
)) { 
 198         pthread_priority_t priority
; 
 200         if (__pthread 
== pthread_self()) { 
 201                 priority 
= _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
); 
 203                 priority 
= __pthread
->tsd
[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
]; 
 206         qos_class_t qos
; int relpri
; 
 207         _pthread_priority_split_newest(priority
, qos
, relpri
); 
 209         if (__qos_class
) { *__qos_class 
= qos
; } 
 210         if (__relative_priority
) { *__relative_priority 
= relpri
; } 
 218         if (!(__pthread_supported_features 
& PTHREAD_FEATURE_BSDTHREADCTL
)) { 
 219                 return QOS_CLASS_UNSPECIFIED
; 
 222         pthread_priority_t p 
= _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
); 
 223         qos_class_t c 
= _pthread_priority_get_qos_newest(p
); 
 231         return _pthread_priority_get_qos_newest(_main_qos
); 
 235 _pthread_qos_class_encode(qos_class_t qos_class
, int relative_priority
, unsigned long flags
) 
 237         if ((__pthread_supported_features 
& PTHREAD_FEATURE_QOS_MAINTENANCE
) == 0) { 
 238                 return _pthread_priority_make_version2(qos_class
, relative_priority
, flags
); 
 240                 return _pthread_priority_make_newest(qos_class
, relative_priority
, flags
); 
 245 _pthread_qos_class_decode(pthread_priority_t priority
, int *relative_priority
, unsigned long *flags
) 
 247         qos_class_t qos
; int relpri
; 
 249         if ((__pthread_supported_features 
& PTHREAD_FEATURE_QOS_MAINTENANCE
) == 0) { 
 250                 _pthread_priority_split_version2(priority
, qos
, relpri
); 
 252                 _pthread_priority_split_newest(priority
, qos
, relpri
); 
 255         if (relative_priority
) { *relative_priority 
= relpri
; } 
 256         if (flags
) { *flags 
= _pthread_priority_get_flags(priority
); } 
 261 _pthread_qos_class_encode_workqueue(int queue_priority
, unsigned long flags
) 
 263         if ((__pthread_supported_features 
& PTHREAD_FEATURE_QOS_DEFAULT
) == 0) { 
 264                 switch (queue_priority
) { 
 265                         case WORKQ_HIGH_PRIOQUEUE
: 
 266                                 return _pthread_priority_make_version1(QOS_CLASS_USER_INTERACTIVE
, 0, flags
); 
 267                         case WORKQ_DEFAULT_PRIOQUEUE
: 
 268                                 return _pthread_priority_make_version1(QOS_CLASS_USER_INITIATED
, 0, flags
); 
 269                         case WORKQ_LOW_PRIOQUEUE
: 
 270                         case WORKQ_NON_INTERACTIVE_PRIOQUEUE
: 
 271                                 return _pthread_priority_make_version1(QOS_CLASS_UTILITY
, 0, flags
); 
 272                         case WORKQ_BG_PRIOQUEUE
: 
 273                                 return _pthread_priority_make_version1(QOS_CLASS_BACKGROUND
, 0, flags
); 
 279         if ((__pthread_supported_features 
& PTHREAD_FEATURE_QOS_MAINTENANCE
) == 0) { 
 280                         switch (queue_priority
) { 
 281                                 case WORKQ_HIGH_PRIOQUEUE
: 
 282                                         return _pthread_priority_make_version2(QOS_CLASS_USER_INITIATED
, 0, flags
); 
 283                                 case WORKQ_DEFAULT_PRIOQUEUE
: 
 284                                         return _pthread_priority_make_version2(QOS_CLASS_DEFAULT
, 0, flags
); 
 285                                 case WORKQ_LOW_PRIOQUEUE
: 
 286                                 case WORKQ_NON_INTERACTIVE_PRIOQUEUE
: 
 287                                         return _pthread_priority_make_version2(QOS_CLASS_UTILITY
, 0, flags
); 
 288                                 case WORKQ_BG_PRIOQUEUE
: 
 289                                         return _pthread_priority_make_version2(QOS_CLASS_BACKGROUND
, 0, flags
); 
 290                                 /* Legacy dispatch does not use QOS_CLASS_MAINTENANCE, so no need to handle it here */ 
 296         switch (queue_priority
) { 
 297                 case WORKQ_HIGH_PRIOQUEUE
: 
 298                         return _pthread_priority_make_newest(QOS_CLASS_USER_INITIATED
, 0, flags
); 
 299                 case WORKQ_DEFAULT_PRIOQUEUE
: 
 300                         return _pthread_priority_make_newest(QOS_CLASS_DEFAULT
, 0, flags
); 
 301                 case WORKQ_LOW_PRIOQUEUE
: 
 302                 case WORKQ_NON_INTERACTIVE_PRIOQUEUE
: 
 303                         return _pthread_priority_make_newest(QOS_CLASS_UTILITY
, 0, flags
); 
 304                 case WORKQ_BG_PRIOQUEUE
: 
 305                         return _pthread_priority_make_newest(QOS_CLASS_BACKGROUND
, 0, flags
); 
 306                 /* Legacy dispatch does not use QOS_CLASS_MAINTENANCE, so no need to handle it here */ 
 313 _pthread_set_properties_self(_pthread_set_flags_t flags
, pthread_priority_t priority
, mach_port_t voucher
) 
 315         if (!(__pthread_supported_features 
& PTHREAD_FEATURE_SETSELF
)) { 
 319         int rv 
= __bsdthread_ctl(BSDTHREAD_CTL_SET_SELF
, priority
, voucher
, flags
); 
 321         /* Set QoS TSD if we succeeded or only failed the voucher half. */ 
 322         if ((flags 
& _PTHREAD_SET_SELF_QOS_FLAG
) != 0) { 
 323                 if (rv 
== 0 || errno 
== ENOENT
) { 
 324                         _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS
, priority
); 
 335 pthread_set_fixedpriority_self(void) 
 337         if (!(__pthread_supported_features 
& PTHREAD_FEATURE_BSDTHREADCTL
)) { 
 341         if (__pthread_supported_features 
& PTHREAD_FEATURE_SETSELF
) { 
 342                 return _pthread_set_properties_self(_PTHREAD_SET_SELF_FIXEDPRIORITY_FLAG
, 0, 0); 
 349 pthread_set_timeshare_self(void) 
 351         if (!(__pthread_supported_features 
& PTHREAD_FEATURE_BSDTHREADCTL
)) { 
 355         if (__pthread_supported_features 
& PTHREAD_FEATURE_SETSELF
) { 
 356                 return _pthread_set_properties_self(_PTHREAD_SET_SELF_TIMESHARE_FLAG
, 0, 0); 
 364 pthread_override_qos_class_start_np(pthread_t __pthread
,  qos_class_t __qos_class
, int __relative_priority
) 
 366         pthread_override_t rv
; 
 370         /* For now, we don't have access to malloc. So we'll have to vm_allocate this, which means the tiny struct is going 
 371          * to use an entire page. 
 373         bool did_malloc 
= true; 
 375         mach_vm_address_t vm_addr 
= malloc(sizeof(struct pthread_override_s
)); 
 377                 vm_addr 
= vm_page_size
; 
 380                 kr 
= mach_vm_allocate(mach_task_self(), &vm_addr
, round_page(sizeof(struct pthread_override_s
)), VM_MAKE_TAG(VM_MEMORY_LIBDISPATCH
) | VM_FLAGS_ANYWHERE
); 
 381                 if (kr 
!= KERN_SUCCESS
) { 
 383                         return (_Nonnull pthread_override_t
) NULL
; 
 387         rv 
= (pthread_override_t
)vm_addr
; 
 388         rv
->sig 
= PTHREAD_OVERRIDE_SIGNATURE
; 
 389         rv
->pthread 
= __pthread
; 
 390         rv
->kthread 
= pthread_mach_thread_np(__pthread
); 
 391         rv
->priority 
= _pthread_priority_make_newest(__qos_class
, __relative_priority
, 0); 
 392         rv
->malloced 
= did_malloc
; 
 394         /* To ensure that the kernel port that we keep stays valid, we retain it here. */ 
 395         kr 
= mach_port_mod_refs(mach_task_self(), rv
->kthread
, MACH_PORT_RIGHT_SEND
, 1); 
 396         if (kr 
!= KERN_SUCCESS
) { 
 401                 res 
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_START
, rv
->kthread
, rv
->priority
, (uintptr_t)rv
); 
 404                         mach_port_mod_refs(mach_task_self(), rv
->kthread
, MACH_PORT_RIGHT_SEND
, -1); 
 412                         mach_vm_deallocate(mach_task_self(), vm_addr
, round_page(sizeof(struct pthread_override_s
))); 
 416         return (_Nonnull pthread_override_t
) rv
; 
 420 pthread_override_qos_class_end_np(pthread_override_t override
) 
 425         /* Double-free is a fault. Swap the signature and check the old one. */ 
 426         if (__sync_swap(&override
->sig
, PTHREAD_OVERRIDE_SIG_DEAD
) != PTHREAD_OVERRIDE_SIGNATURE
) { 
 430         override
->sig 
= PTHREAD_OVERRIDE_SIG_DEAD
; 
 432         /* Always consumes (and deallocates) the pthread_override_t object given. */ 
 433         res 
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_END
, override
->kthread
, (uintptr_t)override
, 0); 
 434         if (res 
== -1) { res 
= errno
; } 
 436         /* EFAULT from the syscall means we underflowed. Crash here. */ 
 438                 // <rdar://problem/17645082> Disable the trap-on-underflow, it doesn't co-exist 
 439                 // with dispatch resetting override counts on threads. 
 444         kr 
= mach_port_mod_refs(mach_task_self(), override
->kthread
, MACH_PORT_RIGHT_SEND
, -1); 
 445         if (kr 
!= KERN_SUCCESS
) { 
 449         if (override
->malloced
) { 
 452                 kr 
= mach_vm_deallocate(mach_task_self(), (mach_vm_address_t
)override
, round_page(sizeof(struct pthread_override_s
))); 
 453                 if (kr 
!= KERN_SUCCESS
) { 
 462 _pthread_qos_override_start_direct(mach_port_t thread
, pthread_priority_t priority
, void *resource
) 
 464         int res 
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_START
, thread
, priority
, (uintptr_t)resource
); 
 465         if (res 
== -1) { res 
= errno
; } 
 470 _pthread_qos_override_end_direct(mach_port_t thread
, void *resource
) 
 472         int res 
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_END
, thread
, (uintptr_t)resource
, 0); 
 473         if (res 
== -1) { res 
= errno
; } 
 478 _pthread_override_qos_class_start_direct(mach_port_t thread
, pthread_priority_t priority
) 
 480         // use pthread_self as the default per-thread memory allocation to track the override in the kernel 
 481         return _pthread_qos_override_start_direct(thread
, priority
, pthread_self()); 
 485 _pthread_override_qos_class_end_direct(mach_port_t thread
) 
 487         // use pthread_self as the default per-thread memory allocation to track the override in the kernel 
 488         return _pthread_qos_override_end_direct(thread
, pthread_self()); 
 492 _pthread_workqueue_override_start_direct(mach_port_t thread
, pthread_priority_t priority
) 
 494         int res 
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_DISPATCH
, thread
, priority
, 0); 
 495         if (res 
== -1) { res 
= errno
; } 
 500 _pthread_workqueue_override_start_direct_check_owner(mach_port_t thread
, pthread_priority_t priority
, mach_port_t 
*ulock_addr
) 
 502 #if !TARGET_OS_IPHONE 
 503         static boolean_t kernel_supports_owner_check 
= TRUE
; 
 504         if (!kernel_supports_owner_check
) { 
 510                 int res 
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_DISPATCH
, thread
, priority
, ulock_addr
); 
 511                 if (res 
== -1) { res 
= errno
; } 
 512 #if !TARGET_OS_IPHONE 
 513                 if (ulock_addr 
&& res 
== EINVAL
) { 
 514                         if ((uintptr_t)ulock_addr 
% _Alignof(_Atomic 
uint32_t)) { 
 515                                 // do not mute bad ulock addresses related errors 
 518                         // backward compatibility for the XBS chroot 
 519                         // BSDTHREAD_CTL_QOS_OVERRIDE_DISPATCH used to return EINVAL if 
 520                         // arg3 was non NULL. 
 521                         kernel_supports_owner_check 
= FALSE
; 
 526                 if (ulock_addr 
&& res 
== EFAULT
) { 
 527                         // kernel wants us to redrive the call, so while we refault the 
 528                         // memory, also revalidate the owner 
 529                         uint32_t uval 
= os_atomic_load(ulock_addr
, relaxed
); 
 530                         if (ulock_owner_value_to_port_name(uval
) != thread
) { 
 541 _pthread_workqueue_override_reset(void) 
 543         int res 
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_RESET
, 0, 0, 0); 
 544         if (res 
== -1) { res 
= errno
; } 
 549 _pthread_workqueue_asynchronous_override_add(mach_port_t thread
, pthread_priority_t priority
, void *resource
) 
 551         int res 
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_ADD
, thread
, priority
, (uintptr_t)resource
); 
 552         if (res 
== -1) { res 
= errno
; } 
 557 _pthread_workqueue_asynchronous_override_reset_self(void *resource
) 
 559         int res 
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_RESET
, 
 563         if (res 
== -1) { res 
= errno
; } 
 568 _pthread_workqueue_asynchronous_override_reset_all_self(void) 
 570         int res 
= __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_RESET
, 
 574         if (res 
== -1) { res 
= errno
; } 
 579 posix_spawnattr_set_qos_class_np(posix_spawnattr_t 
* __restrict __attr
, qos_class_t __qos_class
) 
 581         switch (__qos_class
) { 
 582                 case QOS_CLASS_UTILITY
: 
 583                         return posix_spawnattr_set_qos_clamp_np(__attr
, POSIX_SPAWN_PROC_CLAMP_UTILITY
); 
 584                 case QOS_CLASS_BACKGROUND
: 
 585                         return posix_spawnattr_set_qos_clamp_np(__attr
, POSIX_SPAWN_PROC_CLAMP_BACKGROUND
); 
 586                 case QOS_CLASS_MAINTENANCE
: 
 587                         return posix_spawnattr_set_qos_clamp_np(__attr
, POSIX_SPAWN_PROC_CLAMP_MAINTENANCE
); 
 594 posix_spawnattr_get_qos_class_np(const posix_spawnattr_t 
*__restrict __attr
, qos_class_t 
* __restrict __qos_class
) 
 602         int rv 
= posix_spawnattr_get_qos_clamp_np(__attr
, &clamp
); 
 608                 case POSIX_SPAWN_PROC_CLAMP_UTILITY
: 
 609                         *__qos_class 
= QOS_CLASS_UTILITY
; 
 611                 case POSIX_SPAWN_PROC_CLAMP_BACKGROUND
: 
 612                         *__qos_class 
= QOS_CLASS_BACKGROUND
; 
 614                 case POSIX_SPAWN_PROC_CLAMP_MAINTENANCE
: 
 615                         *__qos_class 
= QOS_CLASS_MAINTENANCE
; 
 618                         *__qos_class 
= QOS_CLASS_UNSPECIFIED
;