]> git.saurik.com Git - apple/libpthread.git/blobdiff - src/qos.c
projects / apple / libpthread.git / blobdiff
? search:
summary | shortlog | log | commit | commitdiff | tree
raw | inline | side by side
libpthread-454.60.1.tar.gz
[apple/libpthread.git] / src / qos.c
diff --git a/src/qos.c b/src/qos.c
index e9c999398efa9604518351259b0c7c46190b94dd..4985d9e69cacd6d6c78daa21eb386f24694fd619 100644 (file)
--- a/src/qos.c
+++ b/src/qos.c
@@ -23,304 +23,240 @@
 
 #include "internal.h"
 
 
 #include "internal.h"
 
-#include <_simple.h>
 #include <mach/mach_vm.h>
 #include <unistd.h>
 #include <spawn.h>
 #include <spawn_private.h>
 #include <mach/mach_vm.h>
 #include <unistd.h>
 #include <spawn.h>
 #include <spawn_private.h>
+#include <pthread/spawn.h>
 #include <sys/spawn_internal.h>
 #include <sys/spawn_internal.h>
-
-// TODO: remove me when internal.h can include *_private.h itself
-#include "workqueue_private.h"
-#include "qos_private.h"
-
-static pthread_priority_t _main_qos = QOS_CLASS_UNSPECIFIED;
+#include <sys/ulock.h>
 
 #define PTHREAD_OVERRIDE_SIGNATURE     (0x6f766572)
 #define PTHREAD_OVERRIDE_SIG_DEAD      (0x7265766f)
 
 
 #define PTHREAD_OVERRIDE_SIGNATURE     (0x6f766572)
 #define PTHREAD_OVERRIDE_SIG_DEAD      (0x7265766f)
 
+#if !defined(VARIANT_STATIC)
+// internally redirected upcalls in case qos overrides are used
+// before __pthread_init has run
+PTHREAD_NOEXPORT void *
+malloc(size_t sz)
+{
+       if (os_likely(_pthread_malloc)) {
+               return _pthread_malloc(sz);
+       } else {
+               return NULL;
+       }
+}
+
+PTHREAD_NOEXPORT void
+free(void *p)
+{
+       if (os_likely(_pthread_free)) {
+               _pthread_free(p);
+       }
+}
+#endif // VARIANT_STATIC
+
 struct pthread_override_s
 {
        uint32_t sig;
 struct pthread_override_s
 {
        uint32_t sig;
-       pthread_t pthread;
        mach_port_t kthread;
        mach_port_t kthread;
+       pthread_t pthread;
        pthread_priority_t priority;
        bool malloced;
 };
 
        pthread_priority_t priority;
        bool malloced;
 };
 
-void
-_pthread_set_main_qos(pthread_priority_t qos)
+thread_qos_t
+_pthread_qos_class_to_thread_qos(qos_class_t qos)
 {
 {
-       _main_qos = qos;
+       switch (qos) {
+       case QOS_CLASS_USER_INTERACTIVE: return THREAD_QOS_USER_INTERACTIVE;
+       case QOS_CLASS_USER_INITIATED: return THREAD_QOS_USER_INITIATED;
+       case QOS_CLASS_DEFAULT: return THREAD_QOS_LEGACY;
+       case QOS_CLASS_UTILITY: return THREAD_QOS_UTILITY;
+       case QOS_CLASS_BACKGROUND: return THREAD_QOS_BACKGROUND;
+       case QOS_CLASS_MAINTENANCE: return THREAD_QOS_MAINTENANCE;
+       default: return THREAD_QOS_UNSPECIFIED;
+       }
 }
 
 }
 
-int
-pthread_attr_set_qos_class_np(pthread_attr_t *__attr,
-                                                         qos_class_t __qos_class,
-                                                         int __relative_priority)
+static inline qos_class_t
+_pthread_qos_class_from_thread_qos(thread_qos_t tqos)
 {
 {
-       if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) {
-               return ENOTSUP;
-       }
+       static const qos_class_t thread_qos_to_qos_class[THREAD_QOS_LAST] = {
+               [THREAD_QOS_UNSPECIFIED]      = QOS_CLASS_UNSPECIFIED,
+               [THREAD_QOS_MAINTENANCE]      = QOS_CLASS_MAINTENANCE,
+               [THREAD_QOS_BACKGROUND]       = QOS_CLASS_BACKGROUND,
+               [THREAD_QOS_UTILITY]          = QOS_CLASS_UTILITY,
+               [THREAD_QOS_LEGACY]           = QOS_CLASS_DEFAULT,
+               [THREAD_QOS_USER_INITIATED]   = QOS_CLASS_USER_INITIATED,
+               [THREAD_QOS_USER_INTERACTIVE] = QOS_CLASS_USER_INTERACTIVE,
+       };
+       if (os_unlikely(tqos >= THREAD_QOS_LAST)) return QOS_CLASS_UNSPECIFIED;
+       return thread_qos_to_qos_class[tqos];
+}
 
 
-       if (__relative_priority > 0 || __relative_priority < QOS_MIN_RELATIVE_PRIORITY) {
-               return EINVAL;
+static inline thread_qos_t
+_pthread_validate_qos_class_and_relpri(qos_class_t qc, int relpri)
+{
+       if (relpri > 0 || relpri < QOS_MIN_RELATIVE_PRIORITY) {
+               return THREAD_QOS_UNSPECIFIED;
        }
        }
+       return _pthread_qos_class_to_thread_qos(qc);
+}
 
 
-       int ret = EINVAL;
-       if (__attr->sig == _PTHREAD_ATTR_SIG) {
-               if (!__attr->schedset) {
-                       __attr->qosclass = _pthread_priority_make_newest(__qos_class, __relative_priority, 0);
-                       __attr->qosset = 1;
-                       ret = 0;
-               }
-       }
+static inline void
+_pthread_priority_split(pthread_priority_t pp, qos_class_t *qc, int *relpri)
+{
+       thread_qos_t qos = _pthread_priority_thread_qos(pp);
+       if (qc) *qc = _pthread_qos_class_from_thread_qos(qos);
+       if (relpri) *relpri = _pthread_priority_relpri(pp);
+}
 
 
-       return ret;
+void
+_pthread_set_main_qos(pthread_priority_t qos)
+{
+       _main_qos = (uint32_t)qos;
 }
 
 int
 }
 
 int
-pthread_attr_get_qos_class_np(pthread_attr_t * __restrict __attr,
-                                                         qos_class_t * __restrict __qos_class,
-                                                         int * __restrict __relative_priority)
+pthread_attr_set_qos_class_np(pthread_attr_t *attr, qos_class_t qc, int relpri)
 {
 {
-       if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) {
-               return ENOTSUP;
-       }
-
-       int ret = EINVAL;
-       if (__attr->sig == _PTHREAD_ATTR_SIG) {
-               if (__attr->qosset) {
-                       qos_class_t qos; int relpri;
-                       _pthread_priority_split_newest(__attr->qosclass, qos, relpri);
-
-                       if (__qos_class) { *__qos_class = qos; }
-                       if (__relative_priority) { *__relative_priority = relpri; }
-               } else {
-                       if (__qos_class) { *__qos_class = 0; }
-                       if (__relative_priority) { *__relative_priority = 0; }
-               }
-               ret = 0;
+       thread_qos_t qos = _pthread_validate_qos_class_and_relpri(qc, relpri);
+       if (attr->sig != _PTHREAD_ATTR_SIG || attr->schedset) {
+               return EINVAL;
        }
 
        }
 
-       return ret;
+       attr->qosclass = _pthread_priority_make_from_thread_qos(qos, relpri, 0);
+       attr->qosset = 1;
+       attr->schedset = 0;
+       return 0;
 }
 
 int
 }
 
 int
-pthread_set_qos_class_self_np(qos_class_t __qos_class,
-                                                         int __relative_priority)
+pthread_attr_get_qos_class_np(pthread_attr_t *attr, qos_class_t *qc, int *relpri)
 {
 {
-       if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) {
-               return ENOTSUP;
-       }
-
-       if (__relative_priority > 0 || __relative_priority < QOS_MIN_RELATIVE_PRIORITY) {
+       if (attr->sig != _PTHREAD_ATTR_SIG) {
                return EINVAL;
        }
 
                return EINVAL;
        }
 
-       pthread_priority_t priority = _pthread_priority_make_newest(__qos_class, __relative_priority, 0);
-
-       if (__pthread_supported_features & PTHREAD_FEATURE_SETSELF) {
-               return _pthread_set_properties_self(_PTHREAD_SET_SELF_QOS_FLAG, priority, 0);
-       } else {
-               /* We set the thread QoS class in the TSD and then call into the kernel to
-                * read the value out of it and set the QoS class.
-                */
-               _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS, priority);
-
-               mach_port_t kport = pthread_mach_thread_np(pthread_self());
-               int res = __bsdthread_ctl(BSDTHREAD_CTL_SET_QOS, kport, &pthread_self()->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS], 0);
-
-               if (res == -1) {
-                       res = errno;
-               }
-
-               return res;
-       }
+       _pthread_priority_split(attr->qosset ? attr->qosclass : 0, qc, relpri);
+       return 0;
 }
 
 int
 }
 
 int
-pthread_set_qos_class_np(pthread_t __pthread,
-                                                qos_class_t __qos_class,
-                                                int __relative_priority)
+pthread_set_qos_class_self_np(qos_class_t qc, int relpri)
 {
 {
-       if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) {
-               return ENOTSUP;
-       }
-
-       if (__relative_priority > 0 || __relative_priority < QOS_MIN_RELATIVE_PRIORITY) {
+       thread_qos_t qos = _pthread_validate_qos_class_and_relpri(qc, relpri);
+       if (!qos) {
                return EINVAL;
        }
 
                return EINVAL;
        }
 
-       if (__pthread != pthread_self()) {
+       pthread_priority_t pp = _pthread_priority_make_from_thread_qos(qos, relpri, 0);
+       return _pthread_set_properties_self(_PTHREAD_SET_SELF_QOS_FLAG, pp, 0);
+}
+
+int
+pthread_set_qos_class_np(pthread_t thread, qos_class_t qc, int relpri)
+{
+       if (thread != pthread_self()) {
                /* The kext now enforces this anyway, if we check here too, it allows us to call
                 * _pthread_set_properties_self later if we can.
                 */
                return EPERM;
        }
                /* The kext now enforces this anyway, if we check here too, it allows us to call
                 * _pthread_set_properties_self later if we can.
                 */
                return EPERM;
        }
-
-       pthread_priority_t priority = _pthread_priority_make_newest(__qos_class, __relative_priority, 0);
-
-       if (__pthread_supported_features & PTHREAD_FEATURE_SETSELF) {
-               /* If we have _pthread_set_properties_self, then we can easily set this using that. */
-               return _pthread_set_properties_self(_PTHREAD_SET_SELF_QOS_FLAG, priority, 0);
-       } else {
-               /* We set the thread QoS class in the TSD and then call into the kernel to
-                * read the value out of it and set the QoS class.
-                */
-               if (__pthread == pthread_self()) {
-                       _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS, priority);
-               } else {
-                       __pthread->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS] = priority;
-               }
-
-               mach_port_t kport = pthread_mach_thread_np(__pthread);
-               int res = __bsdthread_ctl(BSDTHREAD_CTL_SET_QOS, kport, &__pthread->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS], 0);
-
-               if (res == -1) {
-                       res = errno;
-               }
-
-               return res;
-       }
+       _pthread_validate_signature(thread);
+       return pthread_set_qos_class_self_np(qc, relpri);
 }
 
 int
 }
 
 int
-pthread_get_qos_class_np(pthread_t __pthread,
-                                                qos_class_t * __restrict __qos_class,
-                                                int * __restrict __relative_priority)
+pthread_get_qos_class_np(pthread_t thread, qos_class_t *qc, int *relpri)
 {
 {
-       if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) {
-               return ENOTSUP;
-       }
-
-       pthread_priority_t priority;
-
-       if (__pthread == pthread_self()) {
-               priority = _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS);
-       } else {
-               priority = __pthread->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS];
-       }
-
-       qos_class_t qos; int relpri;
-       _pthread_priority_split_newest(priority, qos, relpri);
-
-       if (__qos_class) { *__qos_class = qos; }
-       if (__relative_priority) { *__relative_priority = relpri; }
-
+       pthread_priority_t pp = _pthread_tsd_slot(thread, PTHREAD_QOS_CLASS);
+       _pthread_priority_split(pp, qc, relpri);
        return 0;
 }
 
 qos_class_t
 qos_class_self(void)
 {
        return 0;
 }
 
 qos_class_t
 qos_class_self(void)
 {
-       if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) {
-               return QOS_CLASS_UNSPECIFIED;
-       }
-
-       pthread_priority_t p = _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS);
-       qos_class_t c = _pthread_priority_get_qos_newest(p);
-
-       return c;
+       pthread_priority_t pp;
+       pp = _pthread_getspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS);
+       return _pthread_qos_class_from_thread_qos(_pthread_priority_thread_qos(pp));
 }
 
 qos_class_t
 qos_class_main(void)
 {
 }
 
 qos_class_t
 qos_class_main(void)
 {
-       return _pthread_priority_get_qos_newest(_main_qos);
+       pthread_priority_t pp = _main_qos;
+       return _pthread_qos_class_from_thread_qos(_pthread_priority_thread_qos(pp));
 }
 
 pthread_priority_t
 }
 
 pthread_priority_t
-_pthread_qos_class_encode(qos_class_t qos_class, int relative_priority, unsigned long flags)
+_pthread_qos_class_encode(qos_class_t qc, int relpri, unsigned long flags)
 {
 {
-       if ((__pthread_supported_features & PTHREAD_FEATURE_QOS_MAINTENANCE) == 0) {
-               return _pthread_priority_make_version2(qos_class, relative_priority, flags);
-       } else {
-               return _pthread_priority_make_newest(qos_class, relative_priority, flags);
-       }
+       thread_qos_t qos = _pthread_qos_class_to_thread_qos(qc);
+       return _pthread_priority_make_from_thread_qos(qos, relpri, flags);
 }
 
 qos_class_t
 }
 
 qos_class_t
-_pthread_qos_class_decode(pthread_priority_t priority, int *relative_priority, unsigned long *flags)
+_pthread_qos_class_decode(pthread_priority_t pp, int *relpri, unsigned long *flags)
 {
 {
-       qos_class_t qos; int relpri;
-
-       if ((__pthread_supported_features & PTHREAD_FEATURE_QOS_MAINTENANCE) == 0) {
-               _pthread_priority_split_version2(priority, qos, relpri);
-       } else {
-               _pthread_priority_split_newest(priority, qos, relpri);
-       }
-
-       if (relative_priority) { *relative_priority = relpri; }
-       if (flags) { *flags = _pthread_priority_get_flags(priority); }
-       return qos;
+       qos_class_t qc;
+       _pthread_priority_split(pp, &qc, relpri);
+       if (flags) *flags = (pp & _PTHREAD_PRIORITY_FLAGS_MASK);
+       return qc;
 }
 
 }
 
+// Encode a legacy workqueue API priority into a pthread_priority_t. This API
+// is deprecated and can be removed when the simulator no longer uses it.
 pthread_priority_t
 _pthread_qos_class_encode_workqueue(int queue_priority, unsigned long flags)
 {
 pthread_priority_t
 _pthread_qos_class_encode_workqueue(int queue_priority, unsigned long flags)
 {
-       if ((__pthread_supported_features & PTHREAD_FEATURE_QOS_DEFAULT) == 0) {
-               switch (queue_priority) {
-                       case WORKQ_HIGH_PRIOQUEUE:
-                               return _pthread_priority_make_version1(QOS_CLASS_USER_INTERACTIVE, 0, flags);
-                       case WORKQ_DEFAULT_PRIOQUEUE:
-                               return _pthread_priority_make_version1(QOS_CLASS_USER_INITIATED, 0, flags);
-                       case WORKQ_LOW_PRIOQUEUE:
-                       case WORKQ_NON_INTERACTIVE_PRIOQUEUE:
-                               return _pthread_priority_make_version1(QOS_CLASS_UTILITY, 0, flags);
-                       case WORKQ_BG_PRIOQUEUE:
-                               return _pthread_priority_make_version1(QOS_CLASS_BACKGROUND, 0, flags);
-                       default:
-                               __pthread_abort();
-               }
-       }
-
-       if ((__pthread_supported_features & PTHREAD_FEATURE_QOS_MAINTENANCE) == 0) {
-                       switch (queue_priority) {
-                               case WORKQ_HIGH_PRIOQUEUE:
-                                       return _pthread_priority_make_version2(QOS_CLASS_USER_INITIATED, 0, flags);
-                               case WORKQ_DEFAULT_PRIOQUEUE:
-                                       return _pthread_priority_make_version2(QOS_CLASS_DEFAULT, 0, flags);
-                               case WORKQ_LOW_PRIOQUEUE:
-                               case WORKQ_NON_INTERACTIVE_PRIOQUEUE:
-                                       return _pthread_priority_make_version2(QOS_CLASS_UTILITY, 0, flags);
-                               case WORKQ_BG_PRIOQUEUE:
-                                       return _pthread_priority_make_version2(QOS_CLASS_BACKGROUND, 0, flags);
-                               /* Legacy dispatch does not use QOS_CLASS_MAINTENANCE, so no need to handle it here */
-                               default:
-                                       __pthread_abort();
-                       }
-       }
-
+       thread_qos_t qos;
        switch (queue_priority) {
        switch (queue_priority) {
-               case WORKQ_HIGH_PRIOQUEUE:
-                       return _pthread_priority_make_newest(QOS_CLASS_USER_INITIATED, 0, flags);
-               case WORKQ_DEFAULT_PRIOQUEUE:
-                       return _pthread_priority_make_newest(QOS_CLASS_DEFAULT, 0, flags);
-               case WORKQ_LOW_PRIOQUEUE:
-               case WORKQ_NON_INTERACTIVE_PRIOQUEUE:
-                       return _pthread_priority_make_newest(QOS_CLASS_UTILITY, 0, flags);
-               case WORKQ_BG_PRIOQUEUE:
-                       return _pthread_priority_make_newest(QOS_CLASS_BACKGROUND, 0, flags);
-               /* Legacy dispatch does not use QOS_CLASS_MAINTENANCE, so no need to handle it here */
-               default:
-                       __pthread_abort();
-       }
+       case WORKQ_HIGH_PRIOQUEUE:      qos = THREAD_QOS_USER_INTERACTIVE; break;
+       case WORKQ_DEFAULT_PRIOQUEUE:   qos = THREAD_QOS_LEGACY; break;
+       case WORKQ_NON_INTERACTIVE_PRIOQUEUE:
+       case WORKQ_LOW_PRIOQUEUE:       qos = THREAD_QOS_UTILITY; break;
+       case WORKQ_BG_PRIOQUEUE:        qos = THREAD_QOS_BACKGROUND; break;
+       default:
+               PTHREAD_CLIENT_CRASH(queue_priority, "Invalid priority");
+       }
+       return _pthread_priority_make_from_thread_qos(qos, 0, flags);
 }
 
 }
 
+#define _PTHREAD_SET_SELF_OUTSIDE_QOS_SKIP \
+               (_PTHREAD_SET_SELF_QOS_FLAG | _PTHREAD_SET_SELF_FIXEDPRIORITY_FLAG | \
+                _PTHREAD_SET_SELF_TIMESHARE_FLAG | \
+                _PTHREAD_SET_SELF_ALTERNATE_AMX)
+
 int
 int
-_pthread_set_properties_self(_pthread_set_flags_t flags, pthread_priority_t priority, mach_port_t voucher)
+_pthread_set_properties_self(_pthread_set_flags_t flags,
+               pthread_priority_t priority, mach_port_t voucher)
 {
 {
-       if (!(__pthread_supported_features & PTHREAD_FEATURE_SETSELF)) {
-               return ENOTSUP;
+       pthread_t self = pthread_self();
+       _pthread_set_flags_t kflags = flags;
+       int rv = 0;
+
+       _pthread_validate_signature(self);
+
+       if (self->wq_outsideqos && (flags & _PTHREAD_SET_SELF_OUTSIDE_QOS_SKIP)) {
+               // A number of properties cannot be altered if we are a workloop
+               // thread that has outside of QoS properties applied to it.
+               kflags &= ~_PTHREAD_SET_SELF_OUTSIDE_QOS_SKIP;
+               if (kflags == 0) goto skip;
        }
 
        }
 
-       int rv = __bsdthread_ctl(BSDTHREAD_CTL_SET_SELF, priority, voucher, flags);
+       rv = __bsdthread_ctl(BSDTHREAD_CTL_SET_SELF, priority, voucher, kflags);
 
 
-       /* Set QoS TSD if we succeeded or only failed the voucher half. */
+skip:
+        // Set QoS TSD if we succeeded, or only failed the voucher portion of the
+        // call. Additionally, if we skipped setting QoS because of outside-of-QoS
+        // attributes then we still want to set the TSD in userspace.
        if ((flags & _PTHREAD_SET_SELF_QOS_FLAG) != 0) {
                if (rv == 0 || errno == ENOENT) {
        if ((flags & _PTHREAD_SET_SELF_QOS_FLAG) != 0) {
                if (rv == 0 || errno == ENOENT) {
-                       _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS, priority);
+                       _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS, 
+                                       priority);
                }
        }
 
                }
        }
 
@@ -333,23 +269,28 @@ _pthread_set_properties_self(_pthread_set_flags_t flags, pthread_priority_t prio
 int
 pthread_set_fixedpriority_self(void)
 {
 int
 pthread_set_fixedpriority_self(void)
 {
-       if (!(__pthread_supported_features & PTHREAD_FEATURE_BSDTHREADCTL)) {
-               return ENOTSUP;
-       }
-       
-       if (__pthread_supported_features & PTHREAD_FEATURE_SETSELF) {
-               return _pthread_set_properties_self(_PTHREAD_SET_SELF_FIXEDPRIORITY_FLAG, 0, 0);
-       } else {
-               return ENOTSUP;
-       }
+       return _pthread_set_properties_self(_PTHREAD_SET_SELF_FIXEDPRIORITY_FLAG, 0, 0);
+}
+
+int
+pthread_set_timeshare_self(void)
+{
+       return _pthread_set_properties_self(_PTHREAD_SET_SELF_TIMESHARE_FLAG, 0, 0);
+}
+
+int
+pthread_prefer_alternate_amx_self(void)
+{
+       return _pthread_set_properties_self(_PTHREAD_SET_SELF_ALTERNATE_AMX, 0, 0);
 }
 
 
 pthread_override_t
 }
 
 
 pthread_override_t
-pthread_override_qos_class_start_np(pthread_t __pthread,  qos_class_t __qos_class, int __relative_priority)
+pthread_override_qos_class_start_np(pthread_t thread,  qos_class_t qc, int relpri)
 {
        pthread_override_t rv;
        kern_return_t kr;
 {
        pthread_override_t rv;
        kern_return_t kr;
+       thread_qos_t qos;
        int res = 0;
 
        /* For now, we don't have access to malloc. So we'll have to vm_allocate this, which means the tiny struct is going
        int res = 0;
 
        /* For now, we don't have access to malloc. So we'll have to vm_allocate this, which means the tiny struct is going
@@ -357,23 +298,30 @@ pthread_override_qos_class_start_np(pthread_t __pthread,  qos_class_t __qos_clas
         */
        bool did_malloc = true;
 
         */
        bool did_malloc = true;
 
+       qos = _pthread_validate_qos_class_and_relpri(qc, relpri);
+       if (qos == THREAD_QOS_UNSPECIFIED) {
+               return (_Nonnull pthread_override_t)NULL;
+       }
+
        mach_vm_address_t vm_addr = malloc(sizeof(struct pthread_override_s));
        if (!vm_addr) {
                vm_addr = vm_page_size;
                did_malloc = false;
 
        mach_vm_address_t vm_addr = malloc(sizeof(struct pthread_override_s));
        if (!vm_addr) {
                vm_addr = vm_page_size;
                did_malloc = false;
 
-               kr = mach_vm_allocate(mach_task_self(), &vm_addr, round_page(sizeof(struct pthread_override_s)), VM_MAKE_TAG(VM_MEMORY_LIBDISPATCH) | VM_FLAGS_ANYWHERE);
+               kr = mach_vm_allocate(mach_task_self(), &vm_addr,
+                               round_page(sizeof(struct pthread_override_s)),
+                               VM_MAKE_TAG(VM_MEMORY_LIBDISPATCH) | VM_FLAGS_ANYWHERE);
                if (kr != KERN_SUCCESS) {
                        errno = ENOMEM;
                if (kr != KERN_SUCCESS) {
                        errno = ENOMEM;
-                       return NULL;
+                       return (_Nonnull pthread_override_t)NULL;
                }
        }
 
        rv = (pthread_override_t)vm_addr;
        rv->sig = PTHREAD_OVERRIDE_SIGNATURE;
                }
        }
 
        rv = (pthread_override_t)vm_addr;
        rv->sig = PTHREAD_OVERRIDE_SIGNATURE;
-       rv->pthread = __pthread;
-       rv->kthread = pthread_mach_thread_np(__pthread);
-       rv->priority = _pthread_priority_make_newest(__qos_class, __relative_priority, 0);
+       rv->pthread = thread;
+       rv->kthread = pthread_mach_thread_np(thread);
+       rv->priority = _pthread_priority_make_from_thread_qos(qos, relpri, 0);
        rv->malloced = did_malloc;
 
        /* To ensure that the kernel port that we keep stays valid, we retain it here. */
        rv->malloced = did_malloc;
 
        /* To ensure that the kernel port that we keep stays valid, we retain it here. */
@@ -383,7 +331,7 @@ pthread_override_qos_class_start_np(pthread_t __pthread,  qos_class_t __qos_clas
        }
 
        if (res == 0) {
        }
 
        if (res == 0) {
-               res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_START, rv->kthread, rv->priority, 0);
+               res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_START, rv->kthread, rv->priority, (uintptr_t)rv);
 
                if (res != 0) {
                        mach_port_mod_refs(mach_task_self(), rv->kthread, MACH_PORT_RIGHT_SEND, -1);
 
                if (res != 0) {
                        mach_port_mod_refs(mach_task_self(), rv->kthread, MACH_PORT_RIGHT_SEND, -1);
@@ -398,7 +346,7 @@ pthread_override_qos_class_start_np(pthread_t __pthread,  qos_class_t __qos_clas
                }
                rv = NULL;
        }
                }
                rv = NULL;
        }
-       return rv;
+       return (_Nonnull pthread_override_t)rv;
 }
 
 int
 }
 
 int
@@ -408,14 +356,12 @@ pthread_override_qos_class_end_np(pthread_override_t override)
        int res = 0;
 
        /* Double-free is a fault. Swap the signature and check the old one. */
        int res = 0;
 
        /* Double-free is a fault. Swap the signature and check the old one. */
-       if (__sync_swap(&override->sig, PTHREAD_OVERRIDE_SIG_DEAD) != PTHREAD_OVERRIDE_SIGNATURE) {
+       if (_pthread_atomic_xchg_uint32_relaxed(&override->sig, PTHREAD_OVERRIDE_SIG_DEAD) != PTHREAD_OVERRIDE_SIGNATURE) {
                __builtin_trap();
        }
 
                __builtin_trap();
        }
 
-       override->sig = PTHREAD_OVERRIDE_SIG_DEAD;
-
        /* Always consumes (and deallocates) the pthread_override_t object given. */
        /* Always consumes (and deallocates) the pthread_override_t object given. */
-       res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_END, override->kthread, 0, 0);
+       res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_END, override->kthread, (uintptr_t)override, 0);
        if (res == -1) { res = errno; }
 
        /* EFAULT from the syscall means we underflowed. Crash here. */
        if (res == -1) { res = errno; }
 
        /* EFAULT from the syscall means we underflowed. Crash here. */
@@ -444,21 +390,35 @@ pthread_override_qos_class_end_np(pthread_override_t override)
 }
 
 int
 }
 
 int
-_pthread_override_qos_class_start_direct(mach_port_t thread, pthread_priority_t priority)
+_pthread_qos_override_start_direct(mach_port_t thread, pthread_priority_t priority, void *resource)
 {
 {
-       int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_START, thread, priority, 0);
+       int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_START, thread, priority, (uintptr_t)resource);
        if (res == -1) { res = errno; }
        return res;
 }
 
 int
        if (res == -1) { res = errno; }
        return res;
 }
 
 int
-_pthread_override_qos_class_end_direct(mach_port_t thread)
+_pthread_qos_override_end_direct(mach_port_t thread, void *resource)
 {
 {
-       int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_END, thread, 0, 0);
+       int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_END, thread, (uintptr_t)resource, 0);
        if (res == -1) { res = errno; }
        return res;
 }
 
        if (res == -1) { res = errno; }
        return res;
 }
 
+int
+_pthread_override_qos_class_start_direct(mach_port_t thread, pthread_priority_t priority)
+{
+       // use pthread_self as the default per-thread memory allocation to track the override in the kernel
+       return _pthread_qos_override_start_direct(thread, priority, pthread_self());
+}
+
+int
+_pthread_override_qos_class_end_direct(mach_port_t thread)
+{
+       // use pthread_self as the default per-thread memory allocation to track the override in the kernel
+       return _pthread_qos_override_end_direct(thread, pthread_self());
+}
+
 int
 _pthread_workqueue_override_start_direct(mach_port_t thread, pthread_priority_t priority)
 {
 int
 _pthread_workqueue_override_start_direct(mach_port_t thread, pthread_priority_t priority)
 {
@@ -467,6 +427,47 @@ _pthread_workqueue_override_start_direct(mach_port_t thread, pthread_priority_t
        return res;
 }
 
        return res;
 }
 
+int
+_pthread_workqueue_override_start_direct_check_owner(mach_port_t thread, pthread_priority_t priority, mach_port_t *ulock_addr)
+{
+#if !TARGET_OS_IPHONE
+       static boolean_t kernel_supports_owner_check = TRUE;
+       if (!kernel_supports_owner_check) {
+               ulock_addr = NULL;
+       }
+#endif
+
+       for (;;) {
+               int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_OVERRIDE_DISPATCH, thread, priority, ulock_addr);
+               if (res == -1) { res = errno; }
+#if !TARGET_OS_IPHONE
+               if (ulock_addr && res == EINVAL) {
+                       if ((uintptr_t)ulock_addr % _Alignof(_Atomic uint32_t)) {
+                               // do not mute bad ulock addresses related errors
+                               return EINVAL;
+                       }
+                       // backward compatibility for the XBS chroot
+                       // BSDTHREAD_CTL_QOS_OVERRIDE_DISPATCH used to return EINVAL if
+                       // arg3 was non NULL.
+                       kernel_supports_owner_check = FALSE;
+                       ulock_addr = NULL;
+                       continue;
+               }
+#endif
+               if (ulock_addr && res == EFAULT) {
+                       // kernel wants us to redrive the call, so while we refault the
+                       // memory, also revalidate the owner
+                       uint32_t uval = *(uint32_t volatile *)ulock_addr;
+                       if (ulock_owner_value_to_port_name(uval) != thread) {
+                               return ESTALE;
+                       }
+                       continue;
+               }
+
+               return res;
+       }
+}
+
 int
 _pthread_workqueue_override_reset(void)
 {
 int
 _pthread_workqueue_override_reset(void)
 {
@@ -475,18 +476,109 @@ _pthread_workqueue_override_reset(void)
        return res;
 }
 
        return res;
 }
 
+int
+_pthread_workqueue_asynchronous_override_add(mach_port_t thread, pthread_priority_t priority, void *resource)
+{
+       int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_ADD, thread, priority, (uintptr_t)resource);
+       if (res == -1) { res = errno; }
+       return res;
+}
+
+int
+_pthread_workqueue_asynchronous_override_reset_self(void *resource)
+{
+       int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_RESET,
+                                                         0 /* !reset_all */,
+                                                         (uintptr_t)resource,
+                                                         0);
+       if (res == -1) { res = errno; }
+       return res;
+}
+
+int
+_pthread_workqueue_asynchronous_override_reset_all_self(void)
+{
+       int res = __bsdthread_ctl(BSDTHREAD_CTL_QOS_DISPATCH_ASYNCHRONOUS_OVERRIDE_RESET,
+                                                         1 /* reset_all */,
+                                                         0,
+                                                         0);
+       if (res == -1) { res = errno; }
+       return res;
+}
+
+static inline uint16_t
+_pthread_workqueue_parallelism_for_priority(int qos, unsigned long flags)
+{
+       int rc = __bsdthread_ctl(BSDTHREAD_CTL_QOS_MAX_PARALLELISM, qos, flags, 0);
+       if (os_unlikely(rc == -1)) {
+               rc = errno;
+               if (rc != EINVAL) {
+                       PTHREAD_INTERNAL_CRASH(rc, "qos_max_parallelism failed");
+               }
+               if (flags & _PTHREAD_QOS_PARALLELISM_COUNT_LOGICAL) {
+                       return *(uint8_t *)_COMM_PAGE_LOGICAL_CPUS;
+               } else {
+                       return *(uint8_t *)_COMM_PAGE_PHYSICAL_CPUS;
+               }
+       }
+       return (uint16_t)rc;
+}
+
+int
+pthread_qos_max_parallelism(qos_class_t qos, unsigned long flags)
+{
+       thread_qos_t thread_qos;
+       if (qos == QOS_CLASS_UNSPECIFIED) {
+               qos = QOS_CLASS_DEFAULT; // <rdar://problem/35080198>
+       }
+       thread_qos = _pthread_qos_class_to_thread_qos(qos);
+       if (thread_qos == THREAD_QOS_UNSPECIFIED) {
+               errno = EINVAL;
+               return -1;
+       }
+
+       unsigned long syscall_flags = _PTHREAD_QOS_PARALLELISM_COUNT_LOGICAL;
+       uint16_t *ptr = &_pthread_globals()->qmp_logical[thread_qos];
+
+       if (flags & PTHREAD_MAX_PARALLELISM_PHYSICAL) {
+               syscall_flags = 0;
+               ptr = &_pthread_globals()->qmp_physical[thread_qos];
+       }
+       if (*ptr == 0) {
+               *ptr = _pthread_workqueue_parallelism_for_priority(thread_qos, syscall_flags);
+       }
+       return *ptr;
+}
+
+int
+pthread_time_constraint_max_parallelism(unsigned long flags)
+{
+       unsigned long syscall_flags = _PTHREAD_QOS_PARALLELISM_COUNT_LOGICAL;
+       uint16_t *ptr = &_pthread_globals()->qmp_logical[0];
+
+       if (flags & PTHREAD_MAX_PARALLELISM_PHYSICAL) {
+               syscall_flags = 0;
+               ptr = &_pthread_globals()->qmp_physical[0];
+       }
+       if (*ptr == 0) {
+               *ptr = _pthread_workqueue_parallelism_for_priority(0,
+                               syscall_flags | _PTHREAD_QOS_PARALLELISM_REALTIME);
+       }
+       return *ptr;
+}
+
 int
 posix_spawnattr_set_qos_class_np(posix_spawnattr_t * __restrict __attr, qos_class_t __qos_class)
 {
        switch (__qos_class) {
 int
 posix_spawnattr_set_qos_class_np(posix_spawnattr_t * __restrict __attr, qos_class_t __qos_class)
 {
        switch (__qos_class) {
-               case QOS_CLASS_UTILITY:
-                       return posix_spawnattr_set_qos_clamp_np(__attr, POSIX_SPAWN_PROC_CLAMP_UTILITY);
-               case QOS_CLASS_BACKGROUND:
-                       return posix_spawnattr_set_qos_clamp_np(__attr, POSIX_SPAWN_PROC_CLAMP_BACKGROUND);
-               case QOS_CLASS_MAINTENANCE:
-                       return posix_spawnattr_set_qos_clamp_np(__attr, POSIX_SPAWN_PROC_CLAMP_MAINTENANCE);
-               default:
-                       return EINVAL;
+       case QOS_CLASS_UTILITY:
+               return posix_spawnattr_set_qos_clamp_np(__attr, POSIX_SPAWN_PROC_CLAMP_UTILITY);
+       case QOS_CLASS_BACKGROUND:
+               return posix_spawnattr_set_qos_clamp_np(__attr, POSIX_SPAWN_PROC_CLAMP_BACKGROUND);
+       case QOS_CLASS_MAINTENANCE:
+               return posix_spawnattr_set_qos_clamp_np(__attr, POSIX_SPAWN_PROC_CLAMP_MAINTENANCE);
+       default:
+               return EINVAL;
        }
 }
 
        }
 }
 
@@ -505,18 +597,18 @@ posix_spawnattr_get_qos_class_np(const posix_spawnattr_t *__restrict __attr, qos
        }
 
        switch (clamp) {
        }
 
        switch (clamp) {
-               case POSIX_SPAWN_PROC_CLAMP_UTILITY:
-                       *__qos_class = QOS_CLASS_UTILITY;
-                       break;
-               case POSIX_SPAWN_PROC_CLAMP_BACKGROUND:
-                       *__qos_class = QOS_CLASS_BACKGROUND;
-                       break;
-               case POSIX_SPAWN_PROC_CLAMP_MAINTENANCE:
-                       *__qos_class = QOS_CLASS_MAINTENANCE;
-                       break;
-               default:
-                       *__qos_class = QOS_CLASS_UNSPECIFIED;
-                       break;
+       case POSIX_SPAWN_PROC_CLAMP_UTILITY:
+               *__qos_class = QOS_CLASS_UTILITY;
+               break;
+       case POSIX_SPAWN_PROC_CLAMP_BACKGROUND:
+               *__qos_class = QOS_CLASS_BACKGROUND;
+               break;
+       case POSIX_SPAWN_PROC_CLAMP_MAINTENANCE:
+               *__qos_class = QOS_CLASS_MAINTENANCE;
+               break;
+       default:
+               *__qos_class = QOS_CLASS_UNSPECIFIED;
+               break;
        }
 
        return 0;
        }
 
        return 0;
mirror of libpthread