+#define PTHREAD_RWLOCK_INIT_UNUSED 1
+
+// maximum number of times a read lock may be obtained
+#define MAX_READ_LOCKS (INT_MAX - 1)
+
+
+#if defined(__LP64__)
+#define RWLOCK_USE_INT128 1
+#endif
+
+typedef union rwlock_seq {
+ uint32_t seq[4];
+ struct { uint32_t lcntval; uint32_t rw_seq; uint32_t ucntval; };
+ struct { uint32_t lgen; uint32_t rw_wc; uint32_t ugen; };
+#if RWLOCK_USE_INT128
+ unsigned __int128 seq_LSU;
+ unsigned __int128 _Atomic atomic_seq_LSU;
+#endif
+ struct {
+ uint64_t seq_LS;
+ uint32_t seq_U;
+ uint32_t _pad;
+ };
+ struct {
+ uint64_t _Atomic atomic_seq_LS;
+ uint32_t _Atomic atomic_seq_U;
+ uint32_t _Atomic _atomic_pad;
+ };
+} rwlock_seq;
+
+_Static_assert(sizeof(rwlock_seq) == 4 * sizeof(uint32_t),
+ "Incorrect rwlock_seq size");
+
+typedef enum rwlock_seqfields {
+ RWLOCK_SEQ_NONE = 0,
+ RWLOCK_SEQ_LS = 1,
+ RWLOCK_SEQ_U = 2,
+ RWLOCK_SEQ_LSU = RWLOCK_SEQ_LS | RWLOCK_SEQ_U,
+} rwlock_seqfields;
+
+#if PTHREAD_DEBUG_LOG
+#define RWLOCK_DEBUG_SEQ(op, rwlock, oldseq, newseq, updateval, f) \
+ if (_pthread_debuglog >= 0) { \
+ _simple_dprintf(_pthread_debuglog, "rw_" #op " %p tck %7llu thr %llx " \
+ "L %x -> %x S %x -> %x U %x -> %x updt %x\n", rwlock, \
+ mach_absolute_time() - _pthread_debugstart, _pthread_threadid_self_np_direct(), \
+ (f) & RWLOCK_SEQ_LS ? (oldseq).lcntval : 0, \
+ (f) & RWLOCK_SEQ_LS ? (newseq).lcntval : 0, \
+ (f) & RWLOCK_SEQ_LS ? (oldseq).rw_seq : 0, \
+ (f) & RWLOCK_SEQ_LS ? (newseq).rw_seq : 0, \
+ (f) & RWLOCK_SEQ_U ? (oldseq).ucntval : 0, \
+ (f) & RWLOCK_SEQ_U ? (newseq).ucntval : 0, updateval); }
+#else
+#define RWLOCK_DEBUG_SEQ(m, rwlock, oldseq, newseq, updateval, f)
+#endif
+
+#if !__LITTLE_ENDIAN__
+#error RWLOCK_GETSEQ_ADDR assumes little endian layout of sequence words
+#endif
+
+OS_ALWAYS_INLINE
+static inline void
+RWLOCK_GETSEQ_ADDR(pthread_rwlock_t *rwlock, rwlock_seq **seqaddr)
+{
+ // 128-bit aligned address inside rw_seq & rw_mis arrays
+ *seqaddr = (void*)(((uintptr_t)rwlock->rw_seq + 0xful) & ~0xful);
+}
+
+OS_ALWAYS_INLINE
+static inline void
+RWLOCK_GETTID_ADDR(pthread_rwlock_t *rwlock, uint64_t **tidaddr)
+{
+ // 64-bit aligned address inside rw_tid array (&rw_tid[0] for aligned lock)
+ *tidaddr = (void*)(((uintptr_t)rwlock->rw_tid + 0x7ul) & ~0x7ul);
+}
+
+OS_ALWAYS_INLINE
+static inline void
+rwlock_seq_load(rwlock_seq *seqaddr, rwlock_seq *oldseqval,
+ const rwlock_seqfields seqfields)
+{
+ switch (seqfields) {
+ case RWLOCK_SEQ_LSU:
+#if RWLOCK_USE_INT128
+ oldseqval->seq_LSU = seqaddr->seq_LSU;
+#else
+ oldseqval->seq_LS = seqaddr->seq_LS;
+ oldseqval->seq_U = seqaddr->seq_U;
+#endif
+ break;
+ case RWLOCK_SEQ_LS:
+ oldseqval->seq_LS = seqaddr->seq_LS;
+ break;
+#if DEBUG // unused
+ case RWLOCK_SEQ_U:
+ oldseqval->seq_U = seqaddr->seq_U;
+ break;
+#endif // unused
+ default:
+ __builtin_trap();
+ }
+}
+
+OS_ALWAYS_INLINE
+static inline void
+rwlock_seq_atomic_load_relaxed(rwlock_seq *seqaddr, rwlock_seq *oldseqval,
+ const rwlock_seqfields seqfields)
+{
+ switch (seqfields) {
+ case RWLOCK_SEQ_LSU:
+#if RWLOCK_USE_INT128
+#if defined(__arm64__) && defined(__ARM_ARCH_8_2__)
+ // Workaround clang armv81 codegen bug for 128bit os_atomic_load
+ // rdar://problem/31213932
+ oldseqval->seq_LSU = seqaddr->seq_LSU;
+ while (!os_atomic_cmpxchgv(&seqaddr->atomic_seq_LSU,
+ oldseqval->seq_LSU, oldseqval->seq_LSU, &oldseqval->seq_LSU,
+ relaxed));
+#else
+ oldseqval->seq_LSU = os_atomic_load_wide(&seqaddr->atomic_seq_LSU, relaxed);
+#endif
+#else
+ oldseqval->seq_LS = os_atomic_load_wide(&seqaddr->atomic_seq_LS, relaxed);
+ oldseqval->seq_U = os_atomic_load(&seqaddr->atomic_seq_U, relaxed);
+#endif
+ break;
+ case RWLOCK_SEQ_LS:
+ oldseqval->seq_LS = os_atomic_load_wide(&seqaddr->atomic_seq_LS, relaxed);
+ break;
+#if DEBUG // unused
+ case RWLOCK_SEQ_U:
+ oldseqval->seq_U = os_atomic_load(&seqaddr->atomic_seq_U, relaxed);
+ break;
+#endif // unused
+ default:
+ __builtin_trap();
+ }
+}
+
+#define rwlock_seq_atomic_load(seqaddr, oldseqval, seqfields, m) \
+ rwlock_seq_atomic_load_##m(seqaddr, oldseqval, seqfields)
projects / apple / libpthread.git / blobdiff