libpthread-301.50.1.tar.gz

diff --git a/kern/kern_internal.h b/kern/kern_internal.h
index 80533c35bceb8383975b18de184f3e9b98186bdd..fa2c27b926f3c89f65312bbe8a6123db96c69380 100644 (file)
--- a/kern/kern_internal.h
+++ b/kern/kern_internal.h
@@ -177,6 +177,7 @@ struct _pthread_registration_data {
        uint32_t tsd_offset; /* copy-in */
        uint32_t return_to_kernel_offset; /* copy-in */
        uint32_t mach_thread_self_offset; /* copy-in */
        uint32_t tsd_offset; /* copy-in */
        uint32_t return_to_kernel_offset; /* copy-in */
        uint32_t mach_thread_self_offset; /* copy-in */

+       uint32_t mutex_default_policy; /* copy-out */

 } __attribute__ ((packed));
 
 #ifdef KERNEL
 } __attribute__ ((packed));
 
 #ifdef KERNEL

diff --git a/kern/kern_support.c b/kern/kern_support.c
index ce5d07e36a235a7fdd4f846e496d76b1d34f05bf..280a18b7565eb8c47b0b1b5e47d94a568298c820 100644 (file)
--- a/kern/kern_support.c
+++ b/kern/kern_support.c
@@ -229,6 +229,11 @@ uint32_t pthread_debug_tracing = 1;
 SYSCTL_INT(_kern, OID_AUTO, pthread_debug_tracing, CTLFLAG_RW | CTLFLAG_LOCKED,
                   &pthread_debug_tracing, 0, "")
 
 SYSCTL_INT(_kern, OID_AUTO, pthread_debug_tracing, CTLFLAG_RW | CTLFLAG_LOCKED,
                   &pthread_debug_tracing, 0, "")
 

+static uint32_t pthread_mutex_default_policy;
+
+SYSCTL_INT(_kern, OID_AUTO, pthread_mutex_default_policy, CTLFLAG_RW | CTLFLAG_LOCKED,
+          &pthread_mutex_default_policy, 0, "");
+

 /*
  *       +-----+-----+-----+-----+-----+-----+-----+
  *       | MT  | BG  | UT  | DE  | IN  | UN  | mgr |
 /*
  *       +-----+-----+-----+-----+-----+-----+-----+
  *       | MT  | BG  | UT  | DE  | IN  | UN  | mgr |


@@ -891,6 +896,8 @@ _bsdthread_register(struct proc *p,
                        data.main_qos = _pthread_priority_make_newest(QOS_CLASS_UNSPECIFIED, 0, 0);
                }
 
                        data.main_qos = _pthread_priority_make_newest(QOS_CLASS_UNSPECIFIED, 0, 0);
                }
 

+               data.mutex_default_policy = pthread_mutex_default_policy;
+

                kr = copyout(&data, pthread_init_data, pthread_init_sz);
                if (kr != KERN_SUCCESS) {
                        return EINVAL;
                kr = copyout(&data, pthread_init_data, pthread_init_sz);
                if (kr != KERN_SUCCESS) {
                        return EINVAL;


@@ -4090,6 +4097,11 @@ _pthread_init(void)
        pthread_zone_threadreq = zinit(sizeof(struct threadreq),
                        1024 * sizeof(struct threadreq), 8192, "pthread.threadreq");
 
        pthread_zone_threadreq = zinit(sizeof(struct threadreq),
                        1024 * sizeof(struct threadreq), 8192, "pthread.threadreq");
 

+       int policy_bootarg;
+       if (PE_parse_boot_argn("pthread_mutex_default_policy", &policy_bootarg, sizeof(policy_bootarg))) {
+               pthread_mutex_default_policy = policy_bootarg;
+       }
+

        /*
         * register sysctls
         */
        /*
         * register sysctls
         */


@@ -4099,6 +4111,7 @@ _pthread_init(void)
        sysctl_register_oid(&sysctl__kern_wq_max_threads);
        sysctl_register_oid(&sysctl__kern_wq_max_constrained_threads);
        sysctl_register_oid(&sysctl__kern_pthread_debug_tracing);
        sysctl_register_oid(&sysctl__kern_wq_max_threads);
        sysctl_register_oid(&sysctl__kern_wq_max_constrained_threads);
        sysctl_register_oid(&sysctl__kern_pthread_debug_tracing);

+       sysctl_register_oid(&sysctl__kern_pthread_mutex_default_policy);

 
 #if DEBUG
        sysctl_register_oid(&sysctl__debug_wq_kevent_test);
 
 #if DEBUG
        sysctl_register_oid(&sysctl__debug_wq_kevent_test);

diff --git a/kern/kern_trace.h b/kern/kern_trace.h
index 57ebb4c98b46446eeab5ab0b9d8c6e2dc4f42329..e65e7b94d132895e92daa8636cb904fba760f6b9 100644 (file)
--- a/kern/kern_trace.h
+++ b/kern/kern_trace.h
@@ -33,6 +33,9 @@
  * /usr/share/misc/pthread.codes during build.
  */
 
  * /usr/share/misc/pthread.codes during build.
  */
 

+// userspace trace points force slow-paths, so must be compiled in
+#define ENABLE_USERSPACE_TRACE 0
+

 // pthread tracing subclasses
 # define _TRACE_SUB_DEFAULT 0
 # define _TRACE_SUB_WORKQUEUE 1
 // pthread tracing subclasses
 # define _TRACE_SUB_DEFAULT 0
 # define _TRACE_SUB_WORKQUEUE 1


@@ -68,12 +71,21 @@ VM_UNSLIDE(void* ptr)
 # define PTHREAD_TRACE_WQ_REQ(x,a,b,c,d,e) \
        { if (pthread_debug_tracing) { KERNEL_DEBUG_CONSTANT(x, VM_UNSLIDE(a), VM_UNSLIDE(b), c, d, e); } }
 
 # define PTHREAD_TRACE_WQ_REQ(x,a,b,c,d,e) \
        { if (pthread_debug_tracing) { KERNEL_DEBUG_CONSTANT(x, VM_UNSLIDE(a), VM_UNSLIDE(b), c, d, e); } }
 

-#endif
+#else // KERNEL
+
+#if ENABLE_USERSPACE_TRACE
+# include <sys/kdebug.h>
+# define PTHREAD_TRACE(x, a, b, c, d) kdebug_trace(TRACE_##x, a, b, c, d)
+#else // ENABLE_USERSPACE_TRACE
+# define PTHREAD_TRACE(x, a, b, c, d) do { } while(0)
+#endif // ENABLE_USERSPACE_TRACE
+
+#endif // KERNEL

 
 # define TRACE_CODE(name, subclass, code) \
        static const int TRACE_##name = KDBG_CODE(DBG_PTHREAD, subclass, code)
 
 
 # define TRACE_CODE(name, subclass, code) \
        static const int TRACE_##name = KDBG_CODE(DBG_PTHREAD, subclass, code)
 

-#else
+#else // _PTHREAD_BUILDING_CODES_

 /* When not included as a header, this file is pre-processed into perl source to generate
  * the pthread.codes file during build.
  */
 /* When not included as a header, this file is pre-processed into perl source to generate
  * the pthread.codes file during build.
  */


@@ -82,7 +94,7 @@ VM_UNSLIDE(void* ptr)
 
 # define TRACE_CODE(name, subclass, code) \
        printf("0x%x\t%s\n", ((DBG_PTHREAD << 24) | ((subclass & 0xff) << 16) | ((code & 0x3fff) << 2)), STR(name))
 
 # define TRACE_CODE(name, subclass, code) \
        printf("0x%x\t%s\n", ((DBG_PTHREAD << 24) | ((subclass & 0xff) << 16) | ((code & 0x3fff) << 2)), STR(name))

-#endif
+#endif // _PTHREAD_BUILDING_CODES_

 
 /* These defines translate into TRACE_<name> when used in source code, and are
  * pre-processed out to a codes file by the build system.
 
 /* These defines translate into TRACE_<name> when used in source code, and are
  * pre-processed out to a codes file by the build system.


@@ -124,5 +136,7 @@ TRACE_CODE(psynch_mutex_ulock, _TRACE_SUB_MUTEX, 0x0);
 TRACE_CODE(psynch_mutex_utrylock_failed, _TRACE_SUB_MUTEX, 0x1);
 TRACE_CODE(psynch_mutex_uunlock, _TRACE_SUB_MUTEX, 0x2);
 TRACE_CODE(psynch_ksyn_incorrect_owner, _TRACE_SUB_MUTEX, 0x3);
 TRACE_CODE(psynch_mutex_utrylock_failed, _TRACE_SUB_MUTEX, 0x1);
 TRACE_CODE(psynch_mutex_uunlock, _TRACE_SUB_MUTEX, 0x2);
 TRACE_CODE(psynch_ksyn_incorrect_owner, _TRACE_SUB_MUTEX, 0x3);

+TRACE_CODE(psynch_mutex_lock_updatebits, _TRACE_SUB_MUTEX, 0x4);
+TRACE_CODE(psynch_mutex_unlock_updatebits, _TRACE_SUB_MUTEX, 0x5);

 
 #endif // _KERN_TRACE_H_
 
 #endif // _KERN_TRACE_H_

diff --git a/kern/pthread-Info.plist b/kern/pthread-Info.plist
index 32869264c60fbd613ad5b6ae17feb8a04d1d9612..9c5d9c1f15a4d7b7b39d6df13192a98622bc14ca 100644 (file)
--- a/kern/pthread-Info.plist
+++ b/kern/pthread-Info.plist
@@ -34,6 +34,8 @@
        <dict>
                <key>com.apple.kpi.bsd</key>
                <string>12.0</string>
        <dict>
                <key>com.apple.kpi.bsd</key>
                <string>12.0</string>

+               <key>com.apple.kpi.iokit</key>
+               <string>13.0.0</string>

                <key>com.apple.kpi.libkern</key>
                <string>11.2</string>
                <key>com.apple.kpi.mach</key>
                <key>com.apple.kpi.libkern</key>
                <string>11.2</string>
                <key>com.apple.kpi.mach</key>

diff --git a/pthread/pthread_spis.h b/pthread/pthread_spis.h
index c1f464d67e3af14b211859a6116656cc824bb816..a0ba75430641bb3f5e143aa7470a0e496966640d 100644 (file)
--- a/pthread/pthread_spis.h
+++ b/pthread/pthread_spis.h
@@ -70,10 +70,13 @@ __BEGIN_DECLS
 #define _PTHREAD_MUTEX_POLICY_FAIRSHARE                1
 #define _PTHREAD_MUTEX_POLICY_FIRSTFIT         2
 
 #define _PTHREAD_MUTEX_POLICY_FAIRSHARE                1
 #define _PTHREAD_MUTEX_POLICY_FIRSTFIT         2
 

-/* sets the mutex policy attributes */
+/* manipulate the mutex policy attributes */

 __API_AVAILABLE(macos(10.7), ios(5.0))
 int pthread_mutexattr_setpolicy_np(pthread_mutexattr_t *, int );
 
 __API_AVAILABLE(macos(10.7), ios(5.0))
 int pthread_mutexattr_setpolicy_np(pthread_mutexattr_t *, int );
 

+__API_AVAILABLE(macos(10.13.4), ios(11.3))
+int pthread_mutexattr_getpolicy_np(const pthread_mutexattr_t *, int * );
+

 #endif /* (!_POSIX_C_SOURCE && !_XOPEN_SOURCE) || _DARWIN_C_SOURCE */
 
 __API_AVAILABLE(macos(10.11))
 #endif /* (!_POSIX_C_SOURCE && !_XOPEN_SOURCE) || _DARWIN_C_SOURCE */
 
 __API_AVAILABLE(macos(10.11))

diff --git a/src/internal.h b/src/internal.h
index b50e608593776a6decce19f61571560d0ac4c696..9f2e127f37789a1e3c1e48ca3529bed25802476c 100644 (file)
--- a/src/internal.h
+++ b/src/internal.h
@@ -507,6 +507,10 @@ PTHREAD_NOEXPORT
 void
 _pthread_key_global_init(const char *envp[]);
 
 void
 _pthread_key_global_init(const char *envp[]);
 

+PTHREAD_NOEXPORT
+void
+_pthread_mutex_global_init(const char *envp[], struct _pthread_registration_data *registration_data);
+

 PTHREAD_EXPORT
 void
 _pthread_start(pthread_t self, mach_port_t kport, void *(*fun)(void *), void * funarg, size_t stacksize, unsigned int flags);
 PTHREAD_EXPORT
 void
 _pthread_start(pthread_t self, mach_port_t kport, void *(*fun)(void *), void * funarg, size_t stacksize, unsigned int flags);


@@ -521,7 +525,7 @@ _pthread_main_thread_init(pthread_t p);
 
 PTHREAD_NOEXPORT
 void
 
 PTHREAD_NOEXPORT
 void

-_pthread_bsdthread_init(void);
+_pthread_bsdthread_init(struct _pthread_registration_data *data);

 
 PTHREAD_NOEXPORT_VARIANT
 void
 
 PTHREAD_NOEXPORT_VARIANT
 void

diff --git a/src/pthread.c b/src/pthread.c
index 4922a1c5b32bd0cccf88b3215764c691a5ad054d..8e63bd3a06e5321b0fd46f22cb5df93b62f7e3cb 100644 (file)
--- a/src/pthread.c
+++ b/src/pthread.c
@@ -1878,11 +1878,13 @@ __pthread_init(const struct _libpthread_functions *pthread_funcs,
        // Calls _pthread_set_self() to prepare the main thread for execution.
        _pthread_main_thread_init(thread);
 
        // Calls _pthread_set_self() to prepare the main thread for execution.
        _pthread_main_thread_init(thread);
 

+       struct _pthread_registration_data registration_data;

        // Set up kernel entry points with __bsdthread_register.
        // Set up kernel entry points with __bsdthread_register.

-       _pthread_bsdthread_init();
+       _pthread_bsdthread_init(&registration_data);

 
 

-       // Have pthread_key do its init envvar checks.
+       // Have pthread_key and pthread_mutex do their init envvar checks.

        _pthread_key_global_init(envp);
        _pthread_key_global_init(envp);

+       _pthread_mutex_global_init(envp, &registration_data);

 
 #if PTHREAD_DEBUG_LOG
        _SIMPLE_STRING path = _simple_salloc();
 
 #if PTHREAD_DEBUG_LOG
        _SIMPLE_STRING path = _simple_salloc();


@@ -2000,19 +2002,19 @@ _pthread_clear_qos_tsd(mach_port_t thread_port)
 /***** pthread workqueue support routines *****/
 
 PTHREAD_NOEXPORT void
 /***** pthread workqueue support routines *****/
 
 PTHREAD_NOEXPORT void

-_pthread_bsdthread_init(void)
+_pthread_bsdthread_init(struct _pthread_registration_data *data)

 {
 {

-       struct _pthread_registration_data data = {};
-       data.version = sizeof(struct _pthread_registration_data);
-       data.dispatch_queue_offset = __PTK_LIBDISPATCH_KEY0 * sizeof(void *);
-       data.return_to_kernel_offset = __TSD_RETURN_TO_KERNEL * sizeof(void *);
-       data.tsd_offset = offsetof(struct _pthread, tsd);
-       data.mach_thread_self_offset = __TSD_MACH_THREAD_SELF * sizeof(void *);
+       bzero(data, sizeof(*data));
+       data->version = sizeof(struct _pthread_registration_data);
+       data->dispatch_queue_offset = __PTK_LIBDISPATCH_KEY0 * sizeof(void *);
+       data->return_to_kernel_offset = __TSD_RETURN_TO_KERNEL * sizeof(void *);
+       data->tsd_offset = offsetof(struct _pthread, tsd);
+       data->mach_thread_self_offset = __TSD_MACH_THREAD_SELF * sizeof(void *);

 
        int rv = __bsdthread_register(thread_start,
                        start_wqthread, (int)PTHREAD_SIZE,
 
        int rv = __bsdthread_register(thread_start,
                        start_wqthread, (int)PTHREAD_SIZE,

-                       (void*)&data, (uintptr_t)sizeof(data),
-                       data.dispatch_queue_offset);
+                       (void*)data, (uintptr_t)sizeof(*data),
+                       data->dispatch_queue_offset);

 
        if (rv > 0) {
                if ((rv & PTHREAD_FEATURE_QOS_DEFAULT) == 0) {
 
        if (rv > 0) {
                if ((rv & PTHREAD_FEATURE_QOS_DEFAULT) == 0) {


@@ -2026,7 +2028,16 @@ _pthread_bsdthread_init(void)
                __pthread_supported_features = rv;
        }
 
                __pthread_supported_features = rv;
        }
 

-       pthread_priority_t main_qos = (pthread_priority_t)data.main_qos;
+       /*
+        * TODO: differentiate between (-1, EINVAL) after fork (which has the side
+        * effect of resetting the child's stack_addr_hint before bailing out) and
+        * (-1, EINVAL) because of invalid arguments.  We'd probably like to treat
+        * the latter as fatal.
+        *
+        * <rdar://problem/36451838>
+        */
+
+       pthread_priority_t main_qos = (pthread_priority_t)data->main_qos;

 
        if (_pthread_priority_get_qos_newest(main_qos) != QOS_CLASS_UNSPECIFIED) {
                _pthread_set_main_qos(main_qos);
 
        if (_pthread_priority_get_qos_newest(main_qos) != QOS_CLASS_UNSPECIFIED) {
                _pthread_set_main_qos(main_qos);

diff --git a/src/pthread_atfork.c b/src/pthread_atfork.c
index add7e48c8152d2b0637296fca91a11556913ccc3..d489c24118becf9791a71566e7a884910f73b3a1 100644 (file)
--- a/src/pthread_atfork.c
+++ b/src/pthread_atfork.c
@@ -157,7 +157,9 @@ _pthread_atfork_child(void)
        _PTHREAD_LOCK_INIT(globals->psaved_self_global_lock);
        __is_threaded = 0;
        _pthread_main_thread_init(globals->psaved_self);
        _PTHREAD_LOCK_INIT(globals->psaved_self_global_lock);
        __is_threaded = 0;
        _pthread_main_thread_init(globals->psaved_self);

-       _pthread_bsdthread_init();
+
+       struct _pthread_registration_data registration_data;
+       _pthread_bsdthread_init(&registration_data);

 }
 
 // Iterate pthread_atfork child handlers.
 }
 
 // Iterate pthread_atfork child handlers.

diff --git a/src/pthread_mutex.c b/src/pthread_mutex.c
index 4f1d06fd534a811f290fa184010db46b303b2b14..a68503c09b2ae4414ea4eb7e65198f00f93c56dc 100644 (file)
--- a/src/pthread_mutex.c
+++ b/src/pthread_mutex.c
@@ -94,20 +94,31 @@ int _pthread_mutex_unlock_slow(pthread_mutex_t *omutex);
 PTHREAD_NOEXPORT PTHREAD_WEAK // prevent inlining of return value into callers
 int _pthread_mutex_corruption_abort(_pthread_mutex *mutex);
 
 PTHREAD_NOEXPORT PTHREAD_WEAK // prevent inlining of return value into callers
 int _pthread_mutex_corruption_abort(_pthread_mutex *mutex);
 

+extern int __pthread_mutex_default_policy PTHREAD_NOEXPORT;
+
+
+int __pthread_mutex_default_policy PTHREAD_NOEXPORT =
+               _PTHREAD_MUTEX_POLICY_FAIRSHARE;
+
+PTHREAD_NOEXPORT
+void
+_pthread_mutex_global_init(const char *envp[],
+               struct _pthread_registration_data *registration_data)
+{
+       const char *envvar = _simple_getenv(envp, "PTHREAD_MUTEX_DEFAULT_POLICY");
+       if ((envvar && (envvar[0] - '0') == _PTHREAD_MUTEX_POLICY_FIRSTFIT) ||
+                       (registration_data->mutex_default_policy ==
+                               _PTHREAD_MUTEX_POLICY_FIRSTFIT)) {
+               __pthread_mutex_default_policy = _PTHREAD_MUTEX_POLICY_FIRSTFIT;
+       }
+}
+
+

 
 PTHREAD_ALWAYS_INLINE
 static inline int _pthread_mutex_init(_pthread_mutex *mutex,
                const pthread_mutexattr_t *attr, uint32_t static_type);
 
 
 PTHREAD_ALWAYS_INLINE
 static inline int _pthread_mutex_init(_pthread_mutex *mutex,
                const pthread_mutexattr_t *attr, uint32_t static_type);
 

-#define DEBUG_TRACE_POINTS 0
-
-#if DEBUG_TRACE_POINTS
-#include <sys/kdebug.h>
-#define DEBUG_TRACE(x, a, b, c, d) kdebug_trace(TRACE_##x, a, b, c, d)
-#else
-#define DEBUG_TRACE(x, a, b, c, d) do { } while(0)
-#endif
-

 typedef union mutex_seq {
        uint32_t seq[2];
        struct { uint32_t lgenval; uint32_t ugenval; };
 typedef union mutex_seq {
        uint32_t seq[2];
        struct { uint32_t lgenval; uint32_t ugenval; };


@@ -148,13 +159,6 @@ mutex_seq_load(mutex_seq *seqaddr, mutex_seq *oldseqval)
        oldseqval->seq_LU = seqaddr->seq_LU;
 }
 
        oldseqval->seq_LU = seqaddr->seq_LU;
 }
 

-PTHREAD_ALWAYS_INLINE
-static inline void
-mutex_seq_atomic_load_relaxed(mutex_seq *seqaddr, mutex_seq *oldseqval)
-{
-       oldseqval->seq_LU = os_atomic_load(&seqaddr->atomic_seq_LU, relaxed);
-}
-

 #define mutex_seq_atomic_load(seqaddr, oldseqval, m) \
                mutex_seq_atomic_load_##m(seqaddr, oldseqval)
 
 #define mutex_seq_atomic_load(seqaddr, oldseqval, m) \
                mutex_seq_atomic_load_##m(seqaddr, oldseqval)
 


@@ -265,6 +269,17 @@ pthread_mutexattr_getprotocol(const pthread_mutexattr_t *attr, int *protocol)
        return res;
 }
 
        return res;
 }
 

+int
+pthread_mutexattr_getpolicy_np(const pthread_mutexattr_t *attr, int *policy)
+{
+       int res = EINVAL;
+       if (attr->sig == _PTHREAD_MUTEX_ATTR_SIG) {
+               *policy = attr->policy;
+               res = 0;
+       }
+       return res;
+}
+

 int
 pthread_mutexattr_gettype(const pthread_mutexattr_t *attr, int *type)
 {
 int
 pthread_mutexattr_gettype(const pthread_mutexattr_t *attr, int *type)
 {


@@ -292,7 +307,7 @@ pthread_mutexattr_init(pthread_mutexattr_t *attr)
 {
        attr->prioceiling = _PTHREAD_DEFAULT_PRIOCEILING;
        attr->protocol = _PTHREAD_DEFAULT_PROTOCOL;
 {
        attr->prioceiling = _PTHREAD_DEFAULT_PRIOCEILING;
        attr->protocol = _PTHREAD_DEFAULT_PROTOCOL;

-       attr->policy = _PTHREAD_MUTEX_POLICY_FAIRSHARE;
+       attr->policy = __pthread_mutex_default_policy;

        attr->type = PTHREAD_MUTEX_DEFAULT;
        attr->sig = _PTHREAD_MUTEX_ATTR_SIG;
        attr->pshared = _PTHREAD_DEFAULT_PSHARED;
        attr->type = PTHREAD_MUTEX_DEFAULT;
        attr->sig = _PTHREAD_MUTEX_ATTR_SIG;
        attr->pshared = _PTHREAD_DEFAULT_PSHARED;


@@ -522,6 +537,9 @@ _pthread_mutex_unlock_updatebits(_pthread_mutex *mutex, uint32_t *flagsp,
                }
        } while (!mutex_seq_atomic_cmpxchgv(seqaddr, &oldseq, &newseq, release));
 
                }
        } while (!mutex_seq_atomic_cmpxchgv(seqaddr, &oldseq, &newseq, release));
 

+       PTHREAD_TRACE(psynch_mutex_unlock_updatebits, mutex, oldseq.lgenval,
+                       newseq.lgenval, oldtid);
+

        if (clearprepost) {
                __psynch_cvclrprepost(mutex, newseq.lgenval, newseq.ugenval, 0, 0,
                                newseq.lgenval, flags | _PTHREAD_MTX_OPT_MUTEX);
        if (clearprepost) {
                __psynch_cvclrprepost(mutex, newseq.lgenval, newseq.ugenval, 0, 0,
                                newseq.lgenval, flags | _PTHREAD_MTX_OPT_MUTEX);


@@ -555,10 +573,9 @@ PTHREAD_ALWAYS_INLINE
 static inline int
 _pthread_mutex_lock_updatebits(_pthread_mutex *mutex, uint64_t selfid)
 {
 static inline int
 _pthread_mutex_lock_updatebits(_pthread_mutex *mutex, uint64_t selfid)
 {

-       int res = 0;

        bool firstfit = (mutex->mtxopts.options.policy ==
                        _PTHREAD_MUTEX_POLICY_FIRSTFIT);
        bool firstfit = (mutex->mtxopts.options.policy ==
                        _PTHREAD_MUTEX_POLICY_FIRSTFIT);

-       bool isebit = false, updated = false;
+       bool gotlock = true;

 
        mutex_seq *seqaddr;
        MUTEX_GETSEQ_ADDR(mutex, &seqaddr);
 
        mutex_seq *seqaddr;
        MUTEX_GETSEQ_ADDR(mutex, &seqaddr);


@@ -571,50 +588,38 @@ _pthread_mutex_lock_updatebits(_pthread_mutex *mutex, uint64_t selfid)
        uint64_t oldtid;
 
        do {
        uint64_t oldtid;
 
        do {

-               if (firstfit && isebit && updated) {
-                       mutex_seq_atomic_load(seqaddr, &oldseq, relaxed);
-               }

                newseq = oldseq;
                oldtid = os_atomic_load(tidaddr, relaxed);
 
                newseq = oldseq;
                oldtid = os_atomic_load(tidaddr, relaxed);
 

-               if (isebit && !(oldseq.lgenval & PTH_RWL_EBIT)) {
-                       // E bit was set on first pass through the loop but is no longer
-                       // set. Apparently we spin until it arrives.
-                       // XXX: verify this is desired behavior.
-                       continue;
-               }
-
-               if (isebit) {
-                       // first fit mutex now has the E bit set. Return 1.
-                       res = 1;
-                       break;
-               }
-

                if (firstfit) {
                if (firstfit) {

-                       isebit = (oldseq.lgenval & PTH_RWL_EBIT);
-               } else if ((oldseq.lgenval & (PTH_RWL_KBIT|PTH_RWL_EBIT)) ==
-                               (PTH_RWL_KBIT|PTH_RWL_EBIT)) {
-                       // fairshare mutex and the bits are already set, just update tid
+                       // firstfit locks can have the lock stolen out from under a locker
+                       // between the unlock from the kernel and this lock path. When this
+                       // happens, we still want to set the K bit before leaving the loop
+                       // (or notice if the lock unlocks while we try to update).
+                       gotlock = !is_rwl_ebit_set(oldseq.lgenval);
+               } else if ((oldseq.lgenval & (PTH_RWL_KBIT | PTH_RWL_EBIT)) == 
+                               (PTH_RWL_KBIT | PTH_RWL_EBIT)) {
+                       // bit are already set, just update the owner tidaddr

                        break;
                }
 
                        break;
                }
 

-               // either first fit or no E bit set
-               // update the bits

                newseq.lgenval |= PTH_RWL_KBIT | PTH_RWL_EBIT;
                newseq.lgenval |= PTH_RWL_KBIT | PTH_RWL_EBIT;

+       } while (!mutex_seq_atomic_cmpxchgv(seqaddr, &oldseq, &newseq,
+                       relaxed));

 
 

-               // Retry if CAS fails, or if it succeeds with firstfit and E bit
-               // already set
-       } while (!(updated = mutex_seq_atomic_cmpxchgv(seqaddr, &oldseq, &newseq,
-                       relaxed)) || (firstfit && isebit));
-
-       if (res == 0) {
+       if (gotlock) {

                if (!os_atomic_cmpxchg(tidaddr, oldtid, selfid, relaxed)) {
                        // we own this mutex, nobody should be updating it except us
                        return _pthread_mutex_corruption_abort(mutex);
                }
        }
 
                if (!os_atomic_cmpxchg(tidaddr, oldtid, selfid, relaxed)) {
                        // we own this mutex, nobody should be updating it except us
                        return _pthread_mutex_corruption_abort(mutex);
                }
        }
 

-       return res;
+       PTHREAD_TRACE(psynch_mutex_lock_updatebits, mutex, oldseq.lgenval,
+                       newseq.lgenval, oldtid);
+
+       // failing to take the lock in firstfit returns 1 to force the caller
+       // to wait in the kernel
+       return gotlock ? 0 : 1;

 }
 
 PTHREAD_NOINLINE
 }
 
 PTHREAD_NOINLINE


@@ -777,16 +782,24 @@ _pthread_mutex_lock_slow(pthread_mutex_t *omutex, bool trylock)
                }
        } while (!mutex_seq_atomic_cmpxchgv(seqaddr, &oldseq, &newseq, acquire));
 
                }
        } while (!mutex_seq_atomic_cmpxchgv(seqaddr, &oldseq, &newseq, acquire));
 

+       PTHREAD_TRACE(psynch_mutex_lock_updatebits, omutex, oldseq.lgenval,
+                       newseq.lgenval, 0);
+

        if (gotlock) {
                os_atomic_store(tidaddr, selfid, relaxed);
                res = 0;
        if (gotlock) {
                os_atomic_store(tidaddr, selfid, relaxed);
                res = 0;

-               DEBUG_TRACE(psynch_mutex_ulock, omutex, lgenval, ugenval, selfid);
+               PTHREAD_TRACE(psynch_mutex_ulock, omutex, newseq.lgenval,
+                               newseq.ugenval, selfid);

        } else if (trylock) {
                res = EBUSY;
        } else if (trylock) {
                res = EBUSY;

-               DEBUG_TRACE(psynch_mutex_utrylock_failed, omutex, lgenval, ugenval,
-                               oldtid);
+               PTHREAD_TRACE(psynch_mutex_utrylock_failed, omutex, newseq.lgenval,
+                               newseq.ugenval, oldtid);

        } else {
        } else {

+               PTHREAD_TRACE(psynch_mutex_ulock | DBG_FUNC_START, omutex,
+                               newseq.lgenval, newseq.ugenval, oldtid);

                res = _pthread_mutex_lock_wait(omutex, newseq, oldtid);
                res = _pthread_mutex_lock_wait(omutex, newseq, oldtid);

+               PTHREAD_TRACE(psynch_mutex_ulock | DBG_FUNC_END, omutex,
+                               newseq.lgenval, newseq.ugenval, oldtid);

        }
 
        if (res == 0 && mutex->mtxopts.options.type == PTHREAD_MUTEX_RECURSIVE) {
        }
 
        if (res == 0 && mutex->mtxopts.options.type == PTHREAD_MUTEX_RECURSIVE) {


@@ -809,12 +822,14 @@ PTHREAD_ALWAYS_INLINE
 static inline int
 _pthread_mutex_lock(pthread_mutex_t *omutex, bool trylock)
 {
 static inline int
 _pthread_mutex_lock(pthread_mutex_t *omutex, bool trylock)
 {

-#if PLOCKSTAT || DEBUG_TRACE_POINTS
-       if (PLOCKSTAT_MUTEX_ACQUIRE_ENABLED() || PLOCKSTAT_MUTEX_ERROR_ENABLED() ||
-                       DEBUG_TRACE_POINTS) {
+#if ENABLE_USERSPACE_TRACE
+       return _pthread_mutex_lock_slow(omutex, trylock);
+#elif PLOCKSTAT
+       if (PLOCKSTAT_MUTEX_ACQUIRE_ENABLED() || PLOCKSTAT_MUTEX_ERROR_ENABLED()) {

                return _pthread_mutex_lock_slow(omutex, trylock);
        }
 #endif
                return _pthread_mutex_lock_slow(omutex, trylock);
        }
 #endif

+

        _pthread_mutex *mutex = (_pthread_mutex *)omutex;
        if (os_unlikely(!_pthread_mutex_check_signature_fast(mutex))) {
                return _pthread_mutex_lock_slow(omutex, trylock);
        _pthread_mutex *mutex = (_pthread_mutex *)omutex;
        if (os_unlikely(!_pthread_mutex_check_signature_fast(mutex))) {
                return _pthread_mutex_lock_slow(omutex, trylock);


@@ -897,9 +912,14 @@ _pthread_mutex_unlock_drop(pthread_mutex_t *omutex, mutex_seq newseq,
        uint64_t *tidaddr;
        MUTEX_GETTID_ADDR(mutex, &tidaddr);
 
        uint64_t *tidaddr;
        MUTEX_GETTID_ADDR(mutex, &tidaddr);
 

+       PTHREAD_TRACE(psynch_mutex_uunlock | DBG_FUNC_START, omutex, newseq.lgenval,
+                       newseq.ugenval, os_atomic_load(tidaddr, relaxed));
+

        updateval = __psynch_mutexdrop(omutex, newseq.lgenval, newseq.ugenval,
                        os_atomic_load(tidaddr, relaxed), flags);
 
        updateval = __psynch_mutexdrop(omutex, newseq.lgenval, newseq.ugenval,
                        os_atomic_load(tidaddr, relaxed), flags);
 

+       PTHREAD_TRACE(psynch_mutex_uunlock | DBG_FUNC_END, omutex, updateval, 0, 0);
+

        if (updateval == (uint32_t)-1) {
                res = errno;
 
        if (updateval == (uint32_t)-1) {
                res = errno;
 


@@ -941,8 +961,8 @@ _pthread_mutex_unlock_slow(pthread_mutex_t *omutex)
        } else {
                uint64_t *tidaddr;
                MUTEX_GETTID_ADDR(mutex, &tidaddr);
        } else {
                uint64_t *tidaddr;
                MUTEX_GETTID_ADDR(mutex, &tidaddr);

-               DEBUG_TRACE(psynch_mutex_uunlock, omutex, mtxgen, mtxugen,
-                               os_atomic_load(tidaddr, relaxed));
+               PTHREAD_TRACE(psynch_mutex_uunlock, omutex, newseq.lgenval,
+                               newseq.ugenval, os_atomic_load(tidaddr, relaxed));

        }
 
        return 0;
        }
 
        return 0;


@@ -952,9 +972,10 @@ PTHREAD_NOEXPORT_VARIANT
 int
 pthread_mutex_unlock(pthread_mutex_t *omutex)
 {
 int
 pthread_mutex_unlock(pthread_mutex_t *omutex)
 {

-#if PLOCKSTAT || DEBUG_TRACE_POINTS
-       if (PLOCKSTAT_MUTEX_RELEASE_ENABLED() || PLOCKSTAT_MUTEX_ERROR_ENABLED() ||
-                       DEBUG_TRACE_POINTS) {
+#if ENABLE_USERSPACE_TRACE
+       return _pthread_mutex_unlock_slow(omutex);
+#elif PLOCKSTAT
+       if (PLOCKSTAT_MUTEX_RELEASE_ENABLED() || PLOCKSTAT_MUTEX_ERROR_ENABLED()) {

                return _pthread_mutex_unlock_slow(omutex);
        }
 #endif
                return _pthread_mutex_unlock_slow(omutex);
        }
 #endif


@@ -1042,7 +1063,7 @@ _pthread_mutex_init(_pthread_mutex *mutex, const pthread_mutexattr_t *attr,
                mutex->prioceiling = _PTHREAD_DEFAULT_PRIOCEILING;
                mutex->mtxopts.options.protocol = _PTHREAD_DEFAULT_PROTOCOL;
                if (static_type != 3) {
                mutex->prioceiling = _PTHREAD_DEFAULT_PRIOCEILING;
                mutex->mtxopts.options.protocol = _PTHREAD_DEFAULT_PROTOCOL;
                if (static_type != 3) {

-                       mutex->mtxopts.options.policy = _PTHREAD_MUTEX_POLICY_FAIRSHARE;
+                       mutex->mtxopts.options.policy = __pthread_mutex_default_policy;

                } else {
                        mutex->mtxopts.options.policy = _PTHREAD_MUTEX_POLICY_FIRSTFIT;
                }
                } else {
                        mutex->mtxopts.options.policy = _PTHREAD_MUTEX_POLICY_FIRSTFIT;
                }

diff --git a/tests/mutex.c b/tests/mutex.c
index 7fca3252ce44c44549847c23980b0d60e7715134..0b1e1d47444e47d3c380b10466cc9f34b7f93e24 100644 (file)
--- a/tests/mutex.c
+++ b/tests/mutex.c
@@ -5,7 +5,12 @@
 #include <stdbool.h>
 #include <errno.h>
 
 #include <stdbool.h>
 #include <errno.h>
 

+#include <pthread/pthread_spis.h>
+
+#include <sys/sysctl.h>
+

 #include "darwintest_defaults.h"
 #include "darwintest_defaults.h"

+#include <darwintest_multiprocess.h>

 
 struct context {
        pthread_mutex_t mutex;
 
 struct context {
        pthread_mutex_t mutex;


@@ -80,3 +85,55 @@ T_DECL(mutex, "pthread_mutex",
                T_ASSERT_POSIX_ZERO(res, "pthread_join()");
        }
 }
                T_ASSERT_POSIX_ZERO(res, "pthread_join()");
        }
 }

+
+static void
+check_process_default_mutex_policy(int expected_policy)
+{
+       pthread_mutexattr_t mattr;
+       T_EXPECT_POSIX_ZERO(pthread_mutexattr_init(&mattr), "pthread_mutexattr_init()");
+
+       int policy;
+       T_EXPECT_POSIX_ZERO(pthread_mutexattr_getpolicy_np(&mattr, &policy),
+                       "pthread_mutexattr_getpolicy_np()");
+       T_LOG("policy was %d", policy);
+       T_EXPECT_EQ(policy, expected_policy, "Saw the expected default policy");
+
+       T_EXPECT_POSIX_ZERO(pthread_mutexattr_destroy(&mattr), "pthread_mutexattr_destroy()");
+}
+
+T_DECL(mutex_default_policy,
+               "Tests that the default mutex policy is fairshare")
+{
+       check_process_default_mutex_policy(_PTHREAD_MUTEX_POLICY_FAIRSHARE);
+}
+
+T_DECL(mutex_default_policy_sysctl,
+               "Tests that setting the policy sysctl changes the default policy")
+{
+       int firstfit_default = _PTHREAD_MUTEX_POLICY_FIRSTFIT;
+       T_EXPECT_POSIX_ZERO(
+                       sysctlbyname("kern.pthread_mutex_default_policy", NULL, NULL, &firstfit_default, sizeof(firstfit_default)),
+                       "Changed the default policy sysctl to firstfit");
+
+       dt_helper_t helper = dt_child_helper("mutex_default_policy_sysctl_helper");
+       dt_run_helpers(&helper, 1, 5);
+}
+
+T_HELPER_DECL(mutex_default_policy_sysctl_helper, "sysctl helper")
+{
+       check_process_default_mutex_policy(_PTHREAD_MUTEX_POLICY_FIRSTFIT);
+
+       int default_default = _PTHREAD_MUTEX_POLICY_FAIRSHARE;
+       T_EXPECT_POSIX_ZERO(
+                       sysctlbyname("kern.pthread_mutex_default_policy", NULL, NULL, &default_default, sizeof(default_default)),
+                       "Restored the default policy to fairshare");
+
+       T_END;
+}
+
+T_DECL(mutex_default_policy_envvar,
+               "Tests that setting the policy environment variable changes the default policy",
+               T_META_ENVVAR("PTHREAD_MUTEX_DEFAULT_POLICY=2"))
+{
+       check_process_default_mutex_policy(_PTHREAD_MUTEX_POLICY_FIRSTFIT);
+}

diff --git a/tools/locktrace.lua b/tools/locktrace.lua
new file mode 100755 (executable)
index 0000000..ecc64bc
--- /dev/null
+++ b/tools/locktrace.lua
@@ -0,0 +1,121 @@
+#!/usr/local/bin/luatrace -s
+
+trace_codename = function(codename, callback)
+       local debugid = trace.debugid(codename)
+       if debugid ~= 0 then
+               trace.single(debugid,callback)
+       else
+               printf("WARNING: Cannot locate debugid for '%s'\n", codename)
+       end
+end
+
+initial_timestamp = 0
+get_prefix = function(buf)
+       if initial_timestamp == 0 then
+               initial_timestamp = buf.timestamp
+       end
+       local secs = trace.convert_timestamp_to_nanoseconds(buf.timestamp - initial_timestamp) / 1000000000
+
+       local prefix
+       if trace.debugid_is_start(buf.debugid) then
+               prefix = "→"
+       elseif trace.debugid_is_end(buf.debugid) then
+               prefix = "←"
+       else
+               prefix = "↔"
+       end
+
+       local proc
+       proc = buf.command
+
+       return string.format("%s %6.9f %-17s [%05d.%06x] %-24s",
+               prefix, secs, proc, buf.pid, buf.threadid, buf.debugname)
+end
+
+decode_lval = function(lval)
+       local kbit = " "
+       if lval & 0x1 ~= 0 then
+               kbit = "K"
+       end
+       local ebit = " "
+       if lval & 0x2 ~= 0 then
+               ebit = "E"
+       end
+       local wbit = " "
+       if lval & 0x4 ~= 0 then
+               wbit = "W"
+       end
+
+       local count = lval >> 8
+       return string.format("[0x%06x, %s%s%s]", count, wbit, ebit, kbit)
+end
+
+decode_sval = function(sval)
+       local sbit = " "
+       if sval & 0x1 ~= 0 then
+               sbit = "S"
+       end
+       local ibit = " "
+       if sval & 0x2 ~= 0 then
+               ibit = "I"
+       end
+
+       local count = sval >> 8
+       return string.format("[0x%06x, %s%s]", count, ibit, sbit)
+end
+
+trace_codename("psynch_mutex_lock_updatebits", function(buf)
+       local prefix = get_prefix(buf)
+       if buf[4] == 0 then
+               printf("%s\tupdated lock bits, pre-kernel (addr: 0x%016x, oldlval: %s, newlval: %s)\n", prefix, buf[1], decode_lval(buf[2]), decode_lval(buf[3]))
+       else
+               printf("%s\tupdated lock bits, post-kernel (addr: 0x%016x, oldlval: %s, newlval: %s)\n", prefix, buf[1], decode_lval(buf[2]), decode_lval(buf[3]))
+       end
+end)
+
+trace_codename("psynch_mutex_unlock_updatebits", function(buf)
+       local prefix = get_prefix(buf)
+       printf("%s\tupdated unlock bits (addr: 0x%016x, oldlval: %s, newlval: %s)\n", prefix, buf[1], decode_lval(buf[2]), decode_lval(buf[3]))
+end)
+
+trace_codename("psynch_mutex_ulock", function(buf)
+       local prefix = get_prefix(buf)
+
+       if trace.debugid_is_start(buf.debugid) then
+               printf("%s\tlock busy, waiting in kernel (addr: 0x%016x, lval: %s, sval: %s, owner_tid: 0x%x)\n",
+                       prefix, buf[1], decode_lval(buf[2]), decode_sval(buf[3]), buf[4])
+       elseif trace.debugid_is_end(buf.debugid) then
+               printf("%s\tlock acquired from kernel (addr: 0x%016x, updated bits: %s)\n",
+                       prefix, buf[1], decode_lval(buf[2]))
+       else
+               printf("%s\tlock taken, uncontended (addr: 0x%016x, lval: %s, sval: %s)\n",
+                       prefix, buf[1], decode_lval(buf[2]), decode_sval(buf[3]))
+       end
+end)
+
+trace_codename("psynch_mutex_utrylock_failed", function(buf)
+       local prefix = get_prefix(buf)
+       printf("%s\tmutex trybusy addr: 0x%016x lval: %s sval: %s owner: 0x%x\n", prefix, buf[1], decode_lval(buf[2]), decode_sval(buf[3]), buf[4])
+end)
+
+trace_codename("psynch_mutex_uunlock", function(buf)
+       local prefix = get_prefix(buf)
+
+       if trace.debugid_is_start(buf.debugid) then
+               printf("%s\tunlock, signalling kernel waiters (addr: 0x%016x, lval: %s, sval: %s, owner_tid: 0x%x)\n",
+                       prefix, buf[1], decode_lval(buf[2]), decode_sval(buf[3]), buf[4])
+       elseif trace.debugid_is_end(buf.debugid) then
+               printf("%s\tunlock, waiters signalled (addr: 0x%016x, updated bits: %s)\n",
+                       prefix, buf[1], decode_lval(buf[2]))
+       else
+               printf("%s\tunlock, no kernel waiters (addr: 0x%016x, lval: %s, sval: %s)\n",
+                       prefix, buf[1], decode_lval(buf[2]), decode_sval(buf[3]))
+       end
+end)
+
+-- The trace codes we need aren't enabled by default
+darwin.sysctlbyname("kern.pthread_debug_tracing", 1)
+completion_handler = function()
+       darwin.sysctlbyname("kern.pthread_debug_tracing", 0)
+end
+trace.set_completion_handler(completion_handler)