+ if (wake_one) {
+ if (action == LCK_WAKE_DEFAULT) {
+ priority = WAITQ_PROMOTE_ON_WAKE;
+ } else {
+ assert(action == LCK_WAKE_DO_NOT_TRANSFER_PUSH);
+ priority = WAITQ_ALL_PRIORITIES;
+ }
+
+ /*
+ * WAITQ_PROMOTE_ON_WAKE will call turnstile_update_inheritor
+ * if it finds a thread
+ */
+ wokeup = waitq_wakeup64_identify(&ts->ts_waitq, CAST_EVENT64_T(event), result, priority);
+ if (wokeup != NULL) {
+ if (thread_wokenup != NULL) {
+ *thread_wokenup = wokeup;
+ } else {
+ thread_deallocate_safe(wokeup);
+ }
+ ret = KERN_SUCCESS;
+ if (action == LCK_WAKE_DO_NOT_TRANSFER_PUSH) {
+ goto complete;
+ }
+ } else {
+ if (thread_wokenup != NULL) {
+ *thread_wokenup = NULL;
+ }
+ turnstile_update_inheritor(ts, TURNSTILE_INHERITOR_NULL, TURNSTILE_IMMEDIATE_UPDATE);
+ ret = KERN_NOT_WAITING;
+ }
+ } else {
+ ret = waitq_wakeup64_all(&ts->ts_waitq, CAST_EVENT64_T(event), result, WAITQ_ALL_PRIORITIES);
+ turnstile_update_inheritor(ts, TURNSTILE_INHERITOR_NULL, TURNSTILE_IMMEDIATE_UPDATE);
+ }
+
+ /*
+ * turnstile_update_inheritor_complete could be called while holding the interlock.
+ * In this case the new inheritor or is null, or is a thread that is just been woken up
+ * and have not blocked because it is racing with the same interlock used here
+ * after the wait.
+ * So there is no chain to update for the new inheritor.
+ *
+ * However unless the current thread is the old inheritor,
+ * old inheritor can be blocked and requires a chain update.
+ *
+ * The chain should be short because kernel turnstiles cannot have user turnstiles
+ * chained after them.
+ *
+ * We can anyway optimize this by asking turnstile to tell us
+ * if old inheritor needs an update and drop the lock
+ * just in that case.
+ */
+ turnstile_hash_bucket_unlock((uintptr_t)NULL, &index, type, 0);
+
+ turnstile_update_inheritor_complete(ts, TURNSTILE_INTERLOCK_NOT_HELD);
+
+ turnstile_hash_bucket_lock((uintptr_t)NULL, &index, type);
+
+complete:
+ turnstile_complete((uintptr_t)event, NULL, NULL, type);
+
+ turnstile_hash_bucket_unlock((uintptr_t)NULL, &index, type, 0);
+
+ turnstile_cleanup();
+
+ return ret;
+}
+
+static wait_result_t
+sleep_with_inheritor_and_turnstile_type(event_t event,
+ thread_t inheritor,
+ wait_interrupt_t interruptible,
+ uint64_t deadline,
+ turnstile_type_t type,
+ void (^primitive_lock)(void),
+ void (^primitive_unlock)(void))
+{
+ wait_result_t ret;
+ uint32_t index;
+ struct turnstile *ts = NULL;
+
+ /*
+ * the hash bucket spinlock is used as turnstile interlock,
+ * lock it before releasing the primitive lock
+ */
+ turnstile_hash_bucket_lock((uintptr_t)event, &index, type);
+
+ primitive_unlock();
+
+ ts = turnstile_prepare((uintptr_t)event, NULL, TURNSTILE_NULL, type);
+
+ thread_set_pending_block_hint(current_thread(), kThreadWaitSleepWithInheritor);
+ /*
+ * We need TURNSTILE_DELAYED_UPDATE because we will call
+ * waitq_assert_wait64 after.
+ */
+ turnstile_update_inheritor(ts, inheritor, (TURNSTILE_DELAYED_UPDATE | TURNSTILE_INHERITOR_THREAD));
+
+ ret = waitq_assert_wait64(&ts->ts_waitq, CAST_EVENT64_T(event), interruptible, deadline);
+
+ turnstile_hash_bucket_unlock((uintptr_t)NULL, &index, type, 0);
+
+ /*
+ * Update new and old inheritor chains outside the interlock;
+ */
+ turnstile_update_inheritor_complete(ts, TURNSTILE_INTERLOCK_NOT_HELD);
+
+ if (ret == THREAD_WAITING) {
+ ret = thread_block(THREAD_CONTINUE_NULL);
+ }
+
+ turnstile_hash_bucket_lock((uintptr_t)NULL, &index, type);
+
+ turnstile_complete((uintptr_t)event, NULL, NULL, type);
+
+ turnstile_hash_bucket_unlock((uintptr_t)NULL, &index, type, 0);
+
+ turnstile_cleanup();
+
+ primitive_lock();
+
+ return ret;
+}
+
+kern_return_t
+change_sleep_inheritor_and_turnstile_type(event_t event,
+ thread_t inheritor,
+ turnstile_type_t type)
+{
+ uint32_t index;
+ struct turnstile *ts = NULL;
+ kern_return_t ret = KERN_SUCCESS;
+ /*
+ * the hash bucket spinlock is used as turnstile interlock
+ */
+ turnstile_hash_bucket_lock((uintptr_t)event, &index, type);
+
+ ts = turnstile_prepare((uintptr_t)event, NULL, TURNSTILE_NULL, type);
+
+ if (!turnstile_has_waiters(ts)) {
+ ret = KERN_NOT_WAITING;
+ }
+
+ /*
+ * We will not call an assert_wait later so use TURNSTILE_IMMEDIATE_UPDATE
+ */
+ turnstile_update_inheritor(ts, inheritor, (TURNSTILE_IMMEDIATE_UPDATE | TURNSTILE_INHERITOR_THREAD));
+
+ turnstile_hash_bucket_unlock((uintptr_t)NULL, &index, type, 0);
+
+ /*
+ * update the chains outside the interlock
+ */
+ turnstile_update_inheritor_complete(ts, TURNSTILE_INTERLOCK_NOT_HELD);
+
+ turnstile_hash_bucket_lock((uintptr_t)NULL, &index, type);
+
+ turnstile_complete((uintptr_t)event, NULL, NULL, type);
+
+ turnstile_hash_bucket_unlock((uintptr_t)NULL, &index, type, 0);
+
+ turnstile_cleanup();
+
+ return ret;
+}
+
+typedef void (^void_block_void)(void);
+
+/*
+ * sleep_with_inheritor functions with lck_mtx_t as locking primitive.
+ */
+
+wait_result_t
+lck_mtx_sleep_with_inheritor_and_turnstile_type(lck_mtx_t *lock, lck_sleep_action_t lck_sleep_action, event_t event, thread_t inheritor, wait_interrupt_t interruptible, uint64_t deadline, turnstile_type_t type)
+{
+ LCK_MTX_ASSERT(lock, LCK_MTX_ASSERT_OWNED);
+
+ if (lck_sleep_action & LCK_SLEEP_UNLOCK) {
+ return sleep_with_inheritor_and_turnstile_type(event,
+ inheritor,
+ interruptible,
+ deadline,
+ type,
+ ^{;},
+ ^{lck_mtx_unlock(lock);});
+ } else if (lck_sleep_action & LCK_SLEEP_SPIN) {
+ return sleep_with_inheritor_and_turnstile_type(event,
+ inheritor,
+ interruptible,
+ deadline,
+ type,
+ ^{lck_mtx_lock_spin(lock);},
+ ^{lck_mtx_unlock(lock);});
+ } else if (lck_sleep_action & LCK_SLEEP_SPIN_ALWAYS) {
+ return sleep_with_inheritor_and_turnstile_type(event,
+ inheritor,
+ interruptible,
+ deadline,
+ type,
+ ^{lck_mtx_lock_spin_always(lock);},
+ ^{lck_mtx_unlock(lock);});
+ } else {
+ return sleep_with_inheritor_and_turnstile_type(event,
+ inheritor,
+ interruptible,
+ deadline,
+ type,
+ ^{lck_mtx_lock(lock);},
+ ^{lck_mtx_unlock(lock);});
+ }
+}
+
+/*
+ * Name: lck_spin_sleep_with_inheritor
+ *
+ * Description: deschedule the current thread and wait on the waitq associated with event to be woken up.
+ * While waiting, the sched priority of the waiting thread will contribute to the push of the event that will
+ * be directed to the inheritor specified.
+ * An interruptible mode and deadline can be specified to return earlier from the wait.
+ *
+ * Args:
+ * Arg1: lck_spin_t lock used to protect the sleep. The lock will be dropped while sleeping and reaquired before returning according to the sleep action specified.
+ * Arg2: sleep action. LCK_SLEEP_DEFAULT, LCK_SLEEP_UNLOCK.
+ * Arg3: event to wait on.
+ * Arg4: thread to propagate the event push to.
+ * Arg5: interruptible flag for wait.
+ * Arg6: deadline for wait.
+ *
+ * Conditions: Lock must be held. Returns with the lock held according to the sleep action specified.
+ * Lock will be dropped while waiting.
+ * The inheritor specified cannot run in user space until another inheritor is specified for the event or a
+ * wakeup for the event is called.
+ *
+ * Returns: result of the wait.
+ */
+wait_result_t
+lck_spin_sleep_with_inheritor(
+ lck_spin_t *lock,
+ lck_sleep_action_t lck_sleep_action,
+ event_t event,
+ thread_t inheritor,
+ wait_interrupt_t interruptible,
+ uint64_t deadline)
+{
+ if (lck_sleep_action & LCK_SLEEP_UNLOCK) {
+ return sleep_with_inheritor_and_turnstile_type(event, inheritor,
+ interruptible, deadline, TURNSTILE_SLEEP_INHERITOR,
+ ^{}, ^{ lck_spin_unlock(lock); });
+ } else {
+ return sleep_with_inheritor_and_turnstile_type(event, inheritor,
+ interruptible, deadline, TURNSTILE_SLEEP_INHERITOR,
+ ^{ lck_spin_lock(lock); }, ^{ lck_spin_unlock(lock); });
+ }
+}
+
+/*
+ * Name: lck_mtx_sleep_with_inheritor
+ *
+ * Description: deschedule the current thread and wait on the waitq associated with event to be woken up.
+ * While waiting, the sched priority of the waiting thread will contribute to the push of the event that will
+ * be directed to the inheritor specified.
+ * An interruptible mode and deadline can be specified to return earlier from the wait.
+ *
+ * Args:
+ * Arg1: lck_mtx_t lock used to protect the sleep. The lock will be dropped while sleeping and reaquired before returning according to the sleep action specified.
+ * Arg2: sleep action. LCK_SLEEP_DEFAULT, LCK_SLEEP_UNLOCK, LCK_SLEEP_SPIN, LCK_SLEEP_SPIN_ALWAYS.
+ * Arg3: event to wait on.
+ * Arg4: thread to propagate the event push to.
+ * Arg5: interruptible flag for wait.
+ * Arg6: deadline for wait.
+ *
+ * Conditions: Lock must be held. Returns with the lock held according to the sleep action specified.
+ * Lock will be dropped while waiting.
+ * The inheritor specified cannot run in user space until another inheritor is specified for the event or a
+ * wakeup for the event is called.
+ *
+ * Returns: result of the wait.
+ */
+wait_result_t
+lck_mtx_sleep_with_inheritor(lck_mtx_t *lock, lck_sleep_action_t lck_sleep_action, event_t event, thread_t inheritor, wait_interrupt_t interruptible, uint64_t deadline)
+{
+ return lck_mtx_sleep_with_inheritor_and_turnstile_type(lock, lck_sleep_action, event, inheritor, interruptible, deadline, TURNSTILE_SLEEP_INHERITOR);
+}
+
+/*
+ * sleep_with_inheritor functions with lck_rw_t as locking primitive.
+ */
+
+wait_result_t
+lck_rw_sleep_with_inheritor_and_turnstile_type(lck_rw_t *lock, lck_sleep_action_t lck_sleep_action, event_t event, thread_t inheritor, wait_interrupt_t interruptible, uint64_t deadline, turnstile_type_t type)
+{
+ __block lck_rw_type_t lck_rw_type = LCK_RW_TYPE_EXCLUSIVE;
+
+ LCK_RW_ASSERT(lock, LCK_RW_ASSERT_HELD);
+
+ if (lck_sleep_action & LCK_SLEEP_UNLOCK) {
+ return sleep_with_inheritor_and_turnstile_type(event,
+ inheritor,
+ interruptible,
+ deadline,
+ type,
+ ^{;},
+ ^{lck_rw_type = lck_rw_done(lock);});
+ } else if (!(lck_sleep_action & (LCK_SLEEP_SHARED | LCK_SLEEP_EXCLUSIVE))) {
+ return sleep_with_inheritor_and_turnstile_type(event,
+ inheritor,
+ interruptible,
+ deadline,
+ type,
+ ^{lck_rw_lock(lock, lck_rw_type);},
+ ^{lck_rw_type = lck_rw_done(lock);});
+ } else if (lck_sleep_action & LCK_SLEEP_EXCLUSIVE) {
+ return sleep_with_inheritor_and_turnstile_type(event,
+ inheritor,
+ interruptible,
+ deadline,
+ type,
+ ^{lck_rw_lock_exclusive(lock);},
+ ^{lck_rw_type = lck_rw_done(lock);});
+ } else {
+ return sleep_with_inheritor_and_turnstile_type(event,
+ inheritor,
+ interruptible,
+ deadline,
+ type,
+ ^{lck_rw_lock_shared(lock);},
+ ^{lck_rw_type = lck_rw_done(lock);});
+ }
+}
+
+/*
+ * Name: lck_rw_sleep_with_inheritor
+ *
+ * Description: deschedule the current thread and wait on the waitq associated with event to be woken up.
+ * While waiting, the sched priority of the waiting thread will contribute to the push of the event that will
+ * be directed to the inheritor specified.
+ * An interruptible mode and deadline can be specified to return earlier from the wait.
+ *
+ * Args:
+ * Arg1: lck_rw_t lock used to protect the sleep. The lock will be dropped while sleeping and reaquired before returning according to the sleep action specified.
+ * Arg2: sleep action. LCK_SLEEP_DEFAULT, LCK_SLEEP_SHARED, LCK_SLEEP_EXCLUSIVE.
+ * Arg3: event to wait on.
+ * Arg4: thread to propagate the event push to.
+ * Arg5: interruptible flag for wait.
+ * Arg6: deadline for wait.
+ *
+ * Conditions: Lock must be held. Returns with the lock held according to the sleep action specified.
+ * Lock will be dropped while waiting.
+ * The inheritor specified cannot run in user space until another inheritor is specified for the event or a
+ * wakeup for the event is called.
+ *
+ * Returns: result of the wait.
+ */
+wait_result_t
+lck_rw_sleep_with_inheritor(lck_rw_t *lock, lck_sleep_action_t lck_sleep_action, event_t event, thread_t inheritor, wait_interrupt_t interruptible, uint64_t deadline)
+{
+ return lck_rw_sleep_with_inheritor_and_turnstile_type(lock, lck_sleep_action, event, inheritor, interruptible, deadline, TURNSTILE_SLEEP_INHERITOR);
+}
+
+/*
+ * wakeup_with_inheritor functions are independent from the locking primitive.
+ */
+
+/*
+ * Name: wakeup_one_with_inheritor
+ *
+ * Description: wake up one waiter for event if any. The thread woken up will be the one with the higher sched priority waiting on event.
+ * The push for the event will be transferred from the last inheritor to the woken up thread if LCK_WAKE_DEFAULT is specified.
+ * If LCK_WAKE_DO_NOT_TRANSFER_PUSH is specified the push will not be transferred.
+ *
+ * Args:
+ * Arg1: event to wake from.
+ * Arg2: wait result to pass to the woken up thread.
+ * Arg3: wake flag. LCK_WAKE_DEFAULT or LCK_WAKE_DO_NOT_TRANSFER_PUSH.
+ * Arg4: pointer for storing the thread wokenup.
+ *
+ * Returns: KERN_NOT_WAITING if no threads were waiting, KERN_SUCCESS otherwise.
+ *
+ * Conditions: The new inheritor wokenup cannot run in user space until another inheritor is specified for the event or a
+ * wakeup for the event is called.
+ * A reference for the wokenup thread is acquired.
+ * NOTE: this cannot be called from interrupt context.
+ */
+kern_return_t
+wakeup_one_with_inheritor(event_t event, wait_result_t result, lck_wake_action_t action, thread_t *thread_wokenup)
+{
+ return wakeup_with_inheritor_and_turnstile_type(event,
+ TURNSTILE_SLEEP_INHERITOR,
+ result,
+ TRUE,
+ action,
+ thread_wokenup);
+}
+
+/*
+ * Name: wakeup_all_with_inheritor
+ *
+ * Description: wake up all waiters waiting for event. The old inheritor will lose the push.
+ *
+ * Args:
+ * Arg1: event to wake from.
+ * Arg2: wait result to pass to the woken up threads.
+ *
+ * Returns: KERN_NOT_WAITING if no threads were waiting, KERN_SUCCESS otherwise.
+ *
+ * Conditions: NOTE: this cannot be called from interrupt context.
+ */
+kern_return_t
+wakeup_all_with_inheritor(event_t event, wait_result_t result)
+{
+ return wakeup_with_inheritor_and_turnstile_type(event,
+ TURNSTILE_SLEEP_INHERITOR,
+ result,
+ FALSE,
+ 0,
+ NULL);
+}
+
+/*
+ * change_sleep_inheritor is independent from the locking primitive.
+ */
+
+/*
+ * Name: change_sleep_inheritor
+ *
+ * Description: Redirect the push of the waiting threads of event to the new inheritor specified.
+ *
+ * Args:
+ * Arg1: event to redirect the push.
+ * Arg2: new inheritor for event.
+ *
+ * Returns: KERN_NOT_WAITING if no threads were waiting, KERN_SUCCESS otherwise.
+ *
+ * Conditions: In case of success, the new inheritor cannot run in user space until another inheritor is specified for the event or a
+ * wakeup for the event is called.
+ * NOTE: this cannot be called from interrupt context.
+ */
+kern_return_t
+change_sleep_inheritor(event_t event, thread_t inheritor)
+{
+ return change_sleep_inheritor_and_turnstile_type(event,
+ inheritor,
+ TURNSTILE_SLEEP_INHERITOR);
+}
+
+void
+kdp_sleep_with_inheritor_find_owner(struct waitq * waitq, __unused event64_t event, thread_waitinfo_t * waitinfo)
+{
+ assert(waitinfo->wait_type == kThreadWaitSleepWithInheritor);
+ assert(waitq_is_turnstile_queue(waitq));
+ waitinfo->owner = 0;
+ waitinfo->context = 0;
+
+ if (waitq_held(waitq)) {
+ return;
+ }
+
+ struct turnstile *turnstile = waitq_to_turnstile(waitq);
+ assert(turnstile->ts_inheritor_flags & TURNSTILE_INHERITOR_THREAD);
+ waitinfo->owner = thread_tid(turnstile->ts_inheritor);
+}
+
+typedef void (*void_func_void)(void);
+
+static kern_return_t
+gate_try_close(gate_t *gate)
+{
+ uintptr_t state;
+ thread_t holder;
+ kern_return_t ret;
+ __assert_only bool waiters;
+ thread_t thread = current_thread();
+
+ if (os_atomic_cmpxchg(&gate->gate_data, 0, GATE_THREAD_TO_STATE(thread), acquire)) {
+ return KERN_SUCCESS;
+ }
+
+ gate_ilock(gate);
+ state = ordered_load_gate(gate);
+ holder = GATE_STATE_TO_THREAD(state);
+
+ if (holder == NULL) {
+ waiters = gate_has_waiters(state);
+ assert(waiters == FALSE);
+
+ state = GATE_THREAD_TO_STATE(current_thread());
+ state |= GATE_ILOCK;
+ ordered_store_gate(gate, state);
+ ret = KERN_SUCCESS;
+ } else {
+ if (holder == current_thread()) {
+ panic("Trying to close a gate already owned by current thread %p", current_thread());
+ }
+ ret = KERN_FAILURE;
+ }
+
+ gate_iunlock(gate);
+ return ret;
+}
+
+static void
+gate_close(gate_t* gate)
+{
+ uintptr_t state;
+ thread_t holder;
+ __assert_only bool waiters;
+ thread_t thread = current_thread();
+
+ if (os_atomic_cmpxchg(&gate->gate_data, 0, GATE_THREAD_TO_STATE(thread), acquire)) {
+ return;
+ }
+
+ gate_ilock(gate);
+ state = ordered_load_gate(gate);
+ holder = GATE_STATE_TO_THREAD(state);
+
+ if (holder != NULL) {
+ panic("Closing a gate already owned by %p from current thread %p", holder, current_thread());
+ }
+
+ waiters = gate_has_waiters(state);
+ assert(waiters == FALSE);
+
+ state = GATE_THREAD_TO_STATE(thread);
+ state |= GATE_ILOCK;
+ ordered_store_gate(gate, state);
+
+ gate_iunlock(gate);
+}
+
+static void
+gate_open_turnstile(gate_t *gate)
+{
+ struct turnstile *ts = NULL;
+
+ ts = turnstile_prepare((uintptr_t)gate, &gate->turnstile, TURNSTILE_NULL, TURNSTILE_KERNEL_MUTEX);
+ waitq_wakeup64_all(&ts->ts_waitq, CAST_EVENT64_T(GATE_EVENT(gate)), THREAD_AWAKENED, WAITQ_ALL_PRIORITIES);
+ turnstile_update_inheritor(ts, TURNSTILE_INHERITOR_NULL, TURNSTILE_IMMEDIATE_UPDATE);
+ turnstile_update_inheritor_complete(ts, TURNSTILE_INTERLOCK_HELD);
+ turnstile_complete((uintptr_t)gate, &gate->turnstile, NULL, TURNSTILE_KERNEL_MUTEX);
+ /*
+ * We can do the cleanup while holding the interlock.
+ * It is ok because:
+ * 1. current_thread is the previous inheritor and it is running
+ * 2. new inheritor is NULL.
+ * => No chain of turnstiles needs to be updated.
+ */
+ turnstile_cleanup();
+}
+
+static void
+gate_open(gate_t *gate)
+{
+ uintptr_t state;
+ thread_t holder;
+ bool waiters;
+ thread_t thread = current_thread();
+
+ if (os_atomic_cmpxchg(&gate->gate_data, GATE_THREAD_TO_STATE(thread), 0, release)) {
+ return;
+ }
+
+ gate_ilock(gate);
+ state = ordered_load_gate(gate);
+ holder = GATE_STATE_TO_THREAD(state);
+ waiters = gate_has_waiters(state);
+
+ if (holder != thread) {
+ panic("Opening gate owned by %p from current thread %p", holder, thread);
+ }
+
+ if (waiters) {
+ gate_open_turnstile(gate);
+ }
+
+ state = GATE_ILOCK;
+ ordered_store_gate(gate, state);
+
+ gate_iunlock(gate);
+}
+
+static kern_return_t
+gate_handoff_turnstile(gate_t *gate,
+ int flags,
+ thread_t *thread_woken_up,
+ bool *waiters)
+{
+ struct turnstile *ts = NULL;
+ kern_return_t ret = KERN_FAILURE;
+ thread_t hp_thread;
+
+ ts = turnstile_prepare((uintptr_t)gate, &gate->turnstile, TURNSTILE_NULL, TURNSTILE_KERNEL_MUTEX);
+ /*
+ * Wake up the higest priority thread waiting on the gate
+ */
+ hp_thread = waitq_wakeup64_identify(&ts->ts_waitq, CAST_EVENT64_T(GATE_EVENT(gate)), THREAD_AWAKENED, WAITQ_PROMOTE_ON_WAKE);
+
+ if (hp_thread != NULL) {
+ /*
+ * In this case waitq_wakeup64_identify has called turnstile_update_inheritor for us
+ */
+ turnstile_update_inheritor_complete(ts, TURNSTILE_INTERLOCK_HELD);
+ *thread_woken_up = hp_thread;
+ *waiters = turnstile_has_waiters(ts);
+ /*
+ * Note: hp_thread is the new holder and the new inheritor.
+ * In case there are no more waiters, it doesn't need to be the inheritor
+ * and it shouldn't be it by the time it finishes the wait, so that its next open or
+ * handoff can go through the fast path.
+ * We could set the inheritor to NULL here, or the new holder itself can set it
+ * on its way back from the sleep. In the latter case there are more chanses that
+ * new waiters will come by, avoiding to do the opearation at all.
+ */
+ ret = KERN_SUCCESS;
+ } else {
+ /*
+ * waiters can have been woken up by an interrupt and still not
+ * have updated gate->waiters, so we couldn't find them on the waitq.
+ * Update the inheritor to NULL here, so that the current thread can return to userspace
+ * indipendently from when the interrupted waiters will finish the wait.
+ */
+ if (flags == GATE_HANDOFF_OPEN_IF_NO_WAITERS) {
+ turnstile_update_inheritor(ts, TURNSTILE_INHERITOR_NULL, TURNSTILE_IMMEDIATE_UPDATE);
+ turnstile_update_inheritor_complete(ts, TURNSTILE_INTERLOCK_HELD);
+ }
+ // there are no waiters.
+ ret = KERN_NOT_WAITING;
+ }
+
+ turnstile_complete((uintptr_t)gate, &gate->turnstile, NULL, TURNSTILE_KERNEL_MUTEX);
+
+ /*
+ * We can do the cleanup while holding the interlock.
+ * It is ok because:
+ * 1. current_thread is the previous inheritor and it is running
+ * 2. new inheritor is NULL or it is a just wokenup thread that will race acquiring the lock
+ * of the gate before trying to sleep.
+ * => No chain of turnstiles needs to be updated.
+ */
+ turnstile_cleanup();
+
+ return ret;
+}
+
+static kern_return_t
+gate_handoff(gate_t *gate,
+ int flags)
+{
+ kern_return_t ret;
+ thread_t new_holder = NULL;
+ uintptr_t state;
+ thread_t holder;
+ bool waiters;
+ thread_t thread = current_thread();
+
+ assert(flags == GATE_HANDOFF_OPEN_IF_NO_WAITERS || flags == GATE_HANDOFF_DEFAULT);
+
+ if (flags == GATE_HANDOFF_OPEN_IF_NO_WAITERS) {
+ if (os_atomic_cmpxchg(&gate->gate_data, GATE_THREAD_TO_STATE(thread), 0, release)) {
+ //gate opened but there were no waiters, so return KERN_NOT_WAITING.
+ return KERN_NOT_WAITING;
+ }
+ }
+
+ gate_ilock(gate);
+ state = ordered_load_gate(gate);
+ holder = GATE_STATE_TO_THREAD(state);
+ waiters = gate_has_waiters(state);
+
+ if (holder != current_thread()) {
+ panic("Handing off gate owned by %p from current thread %p", holder, current_thread());
+ }
+
+ if (waiters) {
+ ret = gate_handoff_turnstile(gate, flags, &new_holder, &waiters);
+ if (ret == KERN_SUCCESS) {
+ state = GATE_THREAD_TO_STATE(new_holder);
+ if (waiters) {
+ state |= GATE_WAITERS;
+ }
+ } else {
+ if (flags == GATE_HANDOFF_OPEN_IF_NO_WAITERS) {
+ state = 0;
+ }
+ }
+ } else {
+ if (flags == GATE_HANDOFF_OPEN_IF_NO_WAITERS) {
+ state = 0;
+ }
+ ret = KERN_NOT_WAITING;
+ }
+ state |= GATE_ILOCK;
+ ordered_store_gate(gate, state);
+
+ gate_iunlock(gate);
+
+ if (new_holder) {
+ thread_deallocate(new_holder);
+ }
+ return ret;
+}
+
+static void_func_void
+gate_steal_turnstile(gate_t *gate,
+ thread_t new_inheritor)
+{
+ struct turnstile *ts = NULL;
+
+ ts = turnstile_prepare((uintptr_t)gate, &gate->turnstile, TURNSTILE_NULL, TURNSTILE_KERNEL_MUTEX);
+
+ turnstile_update_inheritor(ts, new_inheritor, (TURNSTILE_IMMEDIATE_UPDATE | TURNSTILE_INHERITOR_THREAD));
+ turnstile_update_inheritor_complete(ts, TURNSTILE_INTERLOCK_HELD);
+ turnstile_complete((uintptr_t)gate, &gate->turnstile, NULL, TURNSTILE_KERNEL_MUTEX);
+
+ /*
+ * turnstile_cleanup might need to update the chain of the old holder.
+ * This operation should happen without the turnstile interlock held.
+ */
+ return turnstile_cleanup;
+}
+
+static void
+gate_steal(gate_t *gate)
+{
+ uintptr_t state;
+ thread_t holder;
+ thread_t thread = current_thread();
+ bool waiters;
+
+ void_func_void func_after_interlock_unlock;
+
+ gate_ilock(gate);
+ state = ordered_load_gate(gate);
+ holder = GATE_STATE_TO_THREAD(state);
+ waiters = gate_has_waiters(state);
+
+ assert(holder != NULL);
+ state = GATE_THREAD_TO_STATE(thread) | GATE_ILOCK;
+ if (waiters) {
+ state |= GATE_WAITERS;
+ ordered_store_gate(gate, state);
+ func_after_interlock_unlock = gate_steal_turnstile(gate, thread);
+ gate_iunlock(gate);
+
+ func_after_interlock_unlock();
+ } else {
+ ordered_store_gate(gate, state);
+ gate_iunlock(gate);
+ }
+}
+
+static void_func_void
+gate_wait_turnstile(gate_t *gate,
+ wait_interrupt_t interruptible,
+ uint64_t deadline,
+ thread_t holder,
+ wait_result_t* wait,
+ bool* waiters)
+{
+ struct turnstile *ts;
+ uintptr_t state;
+
+ ts = turnstile_prepare((uintptr_t)gate, &gate->turnstile, TURNSTILE_NULL, TURNSTILE_KERNEL_MUTEX);
+
+ turnstile_update_inheritor(ts, holder, (TURNSTILE_DELAYED_UPDATE | TURNSTILE_INHERITOR_THREAD));
+ waitq_assert_wait64(&ts->ts_waitq, CAST_EVENT64_T(GATE_EVENT(gate)), interruptible, deadline);
+
+ gate_iunlock(gate);
+
+ turnstile_update_inheritor_complete(ts, TURNSTILE_INTERLOCK_NOT_HELD);
+
+ *wait = thread_block(THREAD_CONTINUE_NULL);
+
+ gate_ilock(gate);
+
+ *waiters = turnstile_has_waiters(ts);
+
+ if (!*waiters) {
+ /*
+ * We want to enable the fast path as soon as we see that there are no more waiters.
+ * On the fast path the holder will not do any turnstile operations.
+ * Set the inheritor as NULL here.
+ *
+ * NOTE: if it was an open operation that woke this thread up, the inheritor has
+ * already been set to NULL.
+ */
+ state = ordered_load_gate(gate);
+ holder = GATE_STATE_TO_THREAD(state);
+ if (holder &&
+ ((*wait != THREAD_AWAKENED) || // thread interrupted or timedout
+ holder == current_thread())) { // thread was woken up and it is the new holder
+ turnstile_update_inheritor(ts, TURNSTILE_INHERITOR_NULL, TURNSTILE_IMMEDIATE_UPDATE);
+ turnstile_update_inheritor_complete(ts, TURNSTILE_INTERLOCK_NOT_HELD);
+ }
+ }
+
+ turnstile_complete((uintptr_t)gate, &gate->turnstile, NULL, TURNSTILE_KERNEL_MUTEX);
+
+ /*
+ * turnstile_cleanup might need to update the chain of the old holder.
+ * This operation should happen without the turnstile primitive interlock held.
+ */
+ return turnstile_cleanup;
+}
+
+static gate_wait_result_t
+gate_wait(gate_t* gate,
+ wait_interrupt_t interruptible,
+ uint64_t deadline,
+ void (^primitive_unlock)(void),
+ void (^primitive_lock)(void))
+{
+ gate_wait_result_t ret;
+ void_func_void func_after_interlock_unlock;
+ wait_result_t wait_result;
+ uintptr_t state;
+ thread_t holder;
+ bool waiters;
+
+
+ gate_ilock(gate);
+ state = ordered_load_gate(gate);
+ holder = GATE_STATE_TO_THREAD(state);
+
+ if (holder == NULL) {
+ panic("Trying to wait on open gate thread %p gate %p", current_thread(), gate);
+ }
+
+ state |= GATE_WAITERS;
+ ordered_store_gate(gate, state);
+
+ /*
+ * Release the primitive lock before any
+ * turnstile operation. Turnstile
+ * does not support a blocking primitive as
+ * interlock.
+ *
+ * In this way, concurrent threads will be
+ * able to acquire the primitive lock
+ * but still will wait for me through the
+ * gate interlock.
+ */
+ primitive_unlock();
+
+ func_after_interlock_unlock = gate_wait_turnstile( gate,
+ interruptible,
+ deadline,
+ holder,
+ &wait_result,
+ &waiters);
+
+ state = ordered_load_gate(gate);
+ holder = GATE_STATE_TO_THREAD(state);
+
+ switch (wait_result) {
+ case THREAD_INTERRUPTED:
+ case THREAD_TIMED_OUT:
+ assert(holder != current_thread());
+
+ if (waiters) {
+ state |= GATE_WAITERS;
+ } else {
+ state &= ~GATE_WAITERS;
+ }
+ ordered_store_gate(gate, state);
+
+ if (wait_result == THREAD_INTERRUPTED) {
+ ret = GATE_INTERRUPTED;
+ } else {
+ ret = GATE_TIMED_OUT;
+ }
+ break;
+ default:
+ /*
+ * Note it is possible that even if the gate was handed off to
+ * me, someone called gate_steal() before I woke up.
+ *
+ * As well as it is possible that the gate was opened, but someone
+ * closed it while I was waking up.
+ *
+ * In both cases we return GATE_OPENED, as the gate was opened to me
+ * at one point, it is the caller responsibility to check again if
+ * the gate is open.
+ */
+ if (holder == current_thread()) {
+ ret = GATE_HANDOFF;
+ } else {
+ ret = GATE_OPENED;
+ }
+ break;
+ }
+
+ gate_iunlock(gate);
+
+ /*
+ * turnstile func that needs to be executed without
+ * holding the primitive interlock
+ */
+ func_after_interlock_unlock();
+
+ primitive_lock();
+
+ return ret;
+}
+static void
+gate_assert(gate_t *gate, int flags)
+{
+ uintptr_t state;
+ thread_t holder;
+
+ gate_ilock(gate);
+ state = ordered_load_gate(gate);
+ holder = GATE_STATE_TO_THREAD(state);
+
+ switch (flags) {
+ case GATE_ASSERT_CLOSED:
+ assert(holder != NULL);
+ break;
+ case GATE_ASSERT_OPEN:
+ assert(holder == NULL);
+ break;
+ case GATE_ASSERT_HELD:
+ assert(holder == current_thread());
+ break;
+ default:
+ panic("invalid %s flag %d", __func__, flags);
+ }
+
+ gate_iunlock(gate);
+}
+
+static void
+gate_init(gate_t *gate)
+{
+ gate->gate_data = 0;
+ gate->turnstile = NULL;
+}
+
+static void
+gate_destroy(__assert_only gate_t *gate)
+{
+ assert(gate->gate_data == 0);
+ assert(gate->turnstile == NULL);
+}
+
+/*
+ * Name: lck_rw_gate_init
+ *
+ * Description: initializes a variable declared with decl_lck_rw_gate_data.
+ *
+ * Args:
+ * Arg1: lck_rw_t lock used to protect the gate.
+ * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data.
+ */
+void
+lck_rw_gate_init(lck_rw_t *lock, gate_t *gate)
+{
+ (void) lock;
+ gate_init(gate);
+}
+
+/*
+ * Name: lck_rw_gate_destroy
+ *
+ * Description: destroys a variable previously initialized.
+ *
+ * Args:
+ * Arg1: lck_rw_t lock used to protect the gate.
+ * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data.
+ */
+void
+lck_rw_gate_destroy(lck_rw_t *lock, gate_t *gate)
+{
+ (void) lock;
+ gate_destroy(gate);
+}
+
+/*
+ * Name: lck_rw_gate_try_close
+ *
+ * Description: Tries to close the gate.
+ * In case of success the current thread will be set as
+ * the holder of the gate.
+ *
+ * Args:
+ * Arg1: lck_rw_t lock used to protect the gate.
+ * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data.
+ *
+ * Conditions: Lock must be held. Returns with the lock held.
+ *
+ * Returns:
+ * KERN_SUCCESS in case the gate was successfully closed. The current thread is the new holder
+ * of the gate.
+ * A matching lck_rw_gate_open() or lck_rw_gate_handoff() needs to be called later on
+ * to wake up possible waiters on the gate before returning to userspace.
+ * If the intent is to conditionally probe the gate before waiting, the lock must not be dropped
+ * between the calls to lck_rw_gate_try_close() and lck_rw_gate_wait().
+ *
+ * KERN_FAILURE in case the gate was already closed. Will panic if the current thread was already the holder of the gate.
+ * lck_rw_gate_wait() should be called instead if the intent is to unconditionally wait on this gate.
+ * The calls to lck_rw_gate_try_close() and lck_rw_gate_wait() should
+ * be done without dropping the lock that is protecting the gate in between.
+ */
+int
+lck_rw_gate_try_close(__assert_only lck_rw_t *lock, gate_t *gate)
+{
+ LCK_RW_ASSERT(lock, LCK_RW_ASSERT_HELD);
+
+ return gate_try_close(gate);
+}
+
+/*
+ * Name: lck_rw_gate_close
+ *
+ * Description: Closes the gate. The current thread will be set as
+ * the holder of the gate. Will panic if the gate is already closed.
+ * A matching lck_rw_gate_open() or lck_rw_gate_handoff() needs to be called later on
+ * to wake up possible waiters on the gate before returning to userspace.
+ *
+ * Args:
+ * Arg1: lck_rw_t lock used to protect the gate.
+ * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data.
+ *
+ * Conditions: Lock must be held. Returns with the lock held.
+ * The gate must be open.
+ *
+ */
+void
+lck_rw_gate_close(__assert_only lck_rw_t *lock, gate_t *gate)
+{
+ LCK_RW_ASSERT(lock, LCK_RW_ASSERT_HELD);
+
+ return gate_close(gate);
+}
+
+/*
+ * Name: lck_rw_gate_open
+ *
+ * Description: Opens the gate and wakes up possible waiters.
+ *
+ * Args:
+ * Arg1: lck_rw_t lock used to protect the gate.
+ * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data.
+ *
+ * Conditions: Lock must be held. Returns with the lock held.
+ * The current thread must be the holder of the gate.
+ *
+ */
+void
+lck_rw_gate_open(__assert_only lck_rw_t *lock, gate_t *gate)
+{
+ LCK_RW_ASSERT(lock, LCK_RW_ASSERT_HELD);
+
+ gate_open(gate);
+}
+
+/*
+ * Name: lck_rw_gate_handoff
+ *
+ * Description: Tries to transfer the ownership of the gate. The waiter with highest sched
+ * priority will be selected as the new holder of the gate, and woken up,
+ * with the gate remaining in the closed state throughout.
+ * If no waiters are present, the gate will be kept closed and KERN_NOT_WAITING
+ * will be returned.
+ * GATE_HANDOFF_OPEN_IF_NO_WAITERS flag can be used to specify if the gate should be opened in
+ * case no waiters were found.
+ *
+ *
+ * Args:
+ * Arg1: lck_rw_t lock used to protect the gate.
+ * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data.
+ * Arg3: flags - GATE_HANDOFF_DEFAULT or GATE_HANDOFF_OPEN_IF_NO_WAITERS
+ *
+ * Conditions: Lock must be held. Returns with the lock held.
+ * The current thread must be the holder of the gate.
+ *
+ * Returns:
+ * KERN_SUCCESS in case one of the waiters became the new holder.
+ * KERN_NOT_WAITING in case there were no waiters.
+ *
+ */
+kern_return_t
+lck_rw_gate_handoff(__assert_only lck_rw_t *lock, gate_t *gate, int flags)
+{
+ LCK_RW_ASSERT(lock, LCK_RW_ASSERT_HELD);
+
+ return gate_handoff(gate, flags);
+}
+
+/*
+ * Name: lck_rw_gate_steal
+ *
+ * Description: Set the current ownership of the gate. It sets the current thread as the
+ * new holder of the gate.
+ * A matching lck_rw_gate_open() or lck_rw_gate_handoff() needs to be called later on
+ * to wake up possible waiters on the gate before returning to userspace.
+ * NOTE: the previous holder should not call lck_rw_gate_open() or lck_rw_gate_handoff()
+ * anymore.
+ *
+ *
+ * Args:
+ * Arg1: lck_rw_t lock used to protect the gate.
+ * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data.
+ *
+ * Conditions: Lock must be held. Returns with the lock held.
+ * The gate must be closed and the current thread must not already be the holder.
+ *
+ */
+void
+lck_rw_gate_steal(__assert_only lck_rw_t *lock, gate_t *gate)
+{
+ LCK_RW_ASSERT(lock, LCK_RW_ASSERT_HELD);
+
+ gate_steal(gate);
+}
+
+/*
+ * Name: lck_rw_gate_wait
+ *
+ * Description: Waits for the current thread to become the holder of the gate or for the
+ * gate to become open. An interruptible mode and deadline can be specified
+ * to return earlier from the wait.
+ *
+ * Args:
+ * Arg1: lck_rw_t lock used to protect the gate.
+ * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data.
+ * Arg3: sleep action. LCK_SLEEP_DEFAULT, LCK_SLEEP_SHARED, LCK_SLEEP_EXCLUSIVE.
+ * Arg3: interruptible flag for wait.
+ * Arg4: deadline
+ *
+ * Conditions: Lock must be held. Returns with the lock held according to the sleep action specified.
+ * Lock will be dropped while waiting.
+ * The gate must be closed.
+ *
+ * Returns: Reason why the thread was woken up.
+ * GATE_HANDOFF - the current thread was handed off the ownership of the gate.
+ * A matching lck_rw_gate_open() or lck_rw_gate_handoff() needs to be called later on
+ * to wake up possible waiters on the gate before returning to userspace.
+ * GATE_OPENED - the gate was opened by the holder.
+ * GATE_TIMED_OUT - the thread was woken up by a timeout.
+ * GATE_INTERRUPTED - the thread was interrupted while sleeping.
+ *
+ */
+gate_wait_result_t
+lck_rw_gate_wait(lck_rw_t *lock, gate_t *gate, lck_sleep_action_t lck_sleep_action, wait_interrupt_t interruptible, uint64_t deadline)
+{
+ __block lck_rw_type_t lck_rw_type = LCK_RW_TYPE_EXCLUSIVE;
+
+ LCK_RW_ASSERT(lock, LCK_RW_ASSERT_HELD);
+
+ if (lck_sleep_action & LCK_SLEEP_UNLOCK) {
+ return gate_wait(gate,
+ interruptible,
+ deadline,
+ ^{lck_rw_type = lck_rw_done(lock);},
+ ^{;});
+ } else if (!(lck_sleep_action & (LCK_SLEEP_SHARED | LCK_SLEEP_EXCLUSIVE))) {
+ return gate_wait(gate,
+ interruptible,
+ deadline,
+ ^{lck_rw_type = lck_rw_done(lock);},
+ ^{lck_rw_lock(lock, lck_rw_type);});
+ } else if (lck_sleep_action & LCK_SLEEP_EXCLUSIVE) {
+ return gate_wait(gate,
+ interruptible,
+ deadline,
+ ^{lck_rw_type = lck_rw_done(lock);},
+ ^{lck_rw_lock_exclusive(lock);});
+ } else {
+ return gate_wait(gate,
+ interruptible,
+ deadline,
+ ^{lck_rw_type = lck_rw_done(lock);},
+ ^{lck_rw_lock_shared(lock);});
+ }
+}
+
+/*
+ * Name: lck_rw_gate_assert
+ *
+ * Description: asserts that the gate is in the specified state.
+ *
+ * Args:
+ * Arg1: lck_rw_t lock used to protect the gate.
+ * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data.
+ * Arg3: flags to specified assert type.
+ * GATE_ASSERT_CLOSED - the gate is currently closed
+ * GATE_ASSERT_OPEN - the gate is currently opened
+ * GATE_ASSERT_HELD - the gate is currently closed and the current thread is the holder
+ */
+void
+lck_rw_gate_assert(__assert_only lck_rw_t *lock, gate_t *gate, int flags)
+{
+ LCK_RW_ASSERT(lock, LCK_RW_ASSERT_HELD);
+
+ gate_assert(gate, flags);
+ return;
+}
+
+/*
+ * Name: lck_mtx_gate_init
+ *
+ * Description: initializes a variable declared with decl_lck_mtx_gate_data.
+ *
+ * Args:
+ * Arg1: lck_mtx_t lock used to protect the gate.
+ * Arg2: pointer to the gate data declared with decl_lck_mtx_gate_data.
+ */
+void
+lck_mtx_gate_init(lck_mtx_t *lock, gate_t *gate)
+{
+ (void) lock;
+ gate_init(gate);
+}
+
+/*
+ * Name: lck_mtx_gate_destroy
+ *
+ * Description: destroys a variable previously initialized
+ *
+ * Args:
+ * Arg1: lck_mtx_t lock used to protect the gate.
+ * Arg2: pointer to the gate data declared with decl_lck_mtx_gate_data.
+ */
+void
+lck_mtx_gate_destroy(lck_mtx_t *lock, gate_t *gate)
+{
+ (void) lock;
+ gate_destroy(gate);
+}
+
+/*
+ * Name: lck_mtx_gate_try_close
+ *
+ * Description: Tries to close the gate.
+ * In case of success the current thread will be set as
+ * the holder of the gate.
+ *
+ * Args:
+ * Arg1: lck_mtx_t lock used to protect the gate.
+ * Arg2: pointer to the gate data declared with decl_lck_mtx_gate_data.
+ *
+ * Conditions: Lock must be held. Returns with the lock held.
+ *
+ * Returns:
+ * KERN_SUCCESS in case the gate was successfully closed. The current thread is the new holder
+ * of the gate.
+ * A matching lck_mtx_gate_open() or lck_mtx_gate_handoff() needs to be called later on
+ * to wake up possible waiters on the gate before returning to userspace.
+ * If the intent is to conditionally probe the gate before waiting, the lock must not be dropped
+ * between the calls to lck_mtx_gate_try_close() and lck_mtx_gate_wait().
+ *
+ * KERN_FAILURE in case the gate was already closed. Will panic if the current thread was already the holder of the gate.
+ * lck_mtx_gate_wait() should be called instead if the intent is to unconditionally wait on this gate.
+ * The calls to lck_mtx_gate_try_close() and lck_mtx_gate_wait() should
+ * be done without dropping the lock that is protecting the gate in between.
+ */
+int
+lck_mtx_gate_try_close(__assert_only lck_mtx_t *lock, gate_t *gate)
+{
+ LCK_MTX_ASSERT(lock, LCK_MTX_ASSERT_OWNED);
+
+ return gate_try_close(gate);
+}
+
+/*
+ * Name: lck_mtx_gate_close
+ *
+ * Description: Closes the gate. The current thread will be set as
+ * the holder of the gate. Will panic if the gate is already closed.
+ * A matching lck_mtx_gate_open() or lck_mtx_gate_handoff() needs to be called later on
+ * to wake up possible waiters on the gate before returning to userspace.
+ *
+ * Args:
+ * Arg1: lck_mtx_t lock used to protect the gate.
+ * Arg2: pointer to the gate data declared with decl_lck_mtx_gate_data.
+ *
+ * Conditions: Lock must be held. Returns with the lock held.
+ * The gate must be open.
+ *
+ */
projects / apple / xnu.git / blobdiff