+#ifdef KERNEL_PRIVATE
+/*
+ * Name: lck_spin_sleep_with_inheritor
+ *
+ * Description: deschedule the current thread and wait on the waitq associated with event to be woken up.
+ * While waiting, the sched priority of the waiting thread will contribute to the push of the event that will
+ * be directed to the inheritor specified.
+ * An interruptible mode and deadline can be specified to return earlier from the wait.
+ *
+ * Args:
+ * Arg1: lck_spin_t lock used to protect the sleep. The lock will be dropped while sleeping and reaquired before returning according to the sleep action specified.
+ * Arg2: sleep action. LCK_SLEEP_DEFAULT, LCK_SLEEP_UNLOCK.
+ * Arg3: event to wait on.
+ * Arg4: thread to propagate the event push to.
+ * Arg5: interruptible flag for wait.
+ * Arg6: deadline for wait.
+ *
+ * Conditions: Lock must be held. Returns with the lock held according to the sleep action specified.
+ * Lock will be dropped while waiting.
+ * The inheritor specified cannot run in user space until another inheritor is specified for the event or a
+ * wakeup for the event is called.
+ *
+ * Returns: result of the wait.
+ */
+extern wait_result_t lck_spin_sleep_with_inheritor(lck_spin_t *lock, lck_sleep_action_t lck_sleep_action, event_t event, thread_t inheritor, wait_interrupt_t interruptible, uint64_t deadline);
+
+/*
+ * Name: lck_mtx_sleep_with_inheritor
+ *
+ * Description: deschedule the current thread and wait on the waitq associated with event to be woken up.
+ * While waiting, the sched priority of the waiting thread will contribute to the push of the event that will
+ * be directed to the inheritor specified.
+ * An interruptible mode and deadline can be specified to return earlier from the wait.
+ *
+ * Args:
+ * Arg1: lck_mtx_t lock used to protect the sleep. The lock will be dropped while sleeping and reaquired before returning according to the sleep action specified.
+ * Arg2: sleep action. LCK_SLEEP_DEFAULT, LCK_SLEEP_UNLOCK, LCK_SLEEP_SPIN, LCK_SLEEP_SPIN_ALWAYS.
+ * Arg3: event to wait on.
+ * Arg4: thread to propagate the event push to.
+ * Arg5: interruptible flag for wait.
+ * Arg6: deadline for wait.
+ *
+ * Conditions: Lock must be held. Returns with the lock held according to the sleep action specified.
+ * Lock will be dropped while waiting.
+ * The inheritor specified cannot run in user space until another inheritor is specified for the event or a
+ * wakeup for the event is called.
+ *
+ * Returns: result of the wait.
+ */
+extern wait_result_t lck_mtx_sleep_with_inheritor(lck_mtx_t *lock, lck_sleep_action_t lck_sleep_action, event_t event, thread_t inheritor, wait_interrupt_t interruptible, uint64_t deadline);
+
+/*
+ * Name: lck_mtx_sleep_with_inheritor
+ *
+ * Description: deschedule the current thread and wait on the waitq associated with event to be woken up.
+ * While waiting, the sched priority of the waiting thread will contribute to the push of the event that will
+ * be directed to the inheritor specified.
+ * An interruptible mode and deadline can be specified to return earlier from the wait.
+ *
+ * Args:
+ * Arg1: lck_rw_t lock used to protect the sleep. The lock will be dropped while sleeping and reaquired before returning according to the sleep action specified.
+ * Arg2: sleep action. LCK_SLEEP_DEFAULT, LCK_SLEEP_SHARED, LCK_SLEEP_EXCLUSIVE.
+ * Arg3: event to wait on.
+ * Arg4: thread to propagate the event push to.
+ * Arg5: interruptible flag for wait.
+ * Arg6: deadline for wait.
+ *
+ * Conditions: Lock must be held. Returns with the lock held according to the sleep action specified.
+ * Lock will be dropped while waiting.
+ * The inheritor specified cannot run in user space until another inheritor is specified for the event or a
+ * wakeup for the event is called.
+ *
+ * Returns: result of the wait.
+ */
+extern wait_result_t lck_rw_sleep_with_inheritor(lck_rw_t *lock, lck_sleep_action_t lck_sleep_action, event_t event, thread_t inheritor, wait_interrupt_t interruptible, uint64_t deadline);
+
+/*
+ * Name: wakeup_one_with_inheritor
+ *
+ * Description: wake up one waiter for event if any. The thread woken up will be the one with the higher sched priority waiting on event.
+ * The push for the event will be transferred from the last inheritor to the woken up thread.
+ *
+ * Args:
+ * Arg1: event to wake from.
+ * Arg2: wait result to pass to the woken up thread.
+ * Arg3: pointer for storing the thread wokenup.
+ *
+ * Returns: KERN_NOT_WAITING if no threads were waiting, KERN_SUCCESS otherwise.
+ *
+ * Conditions: The new inheritor wokenup cannot run in user space until another inheritor is specified for the event or a
+ * wakeup for the event is called.
+ * A reference for the wokenup thread is acquired.
+ * NOTE: this cannot be called from interrupt context.
+ */
+extern kern_return_t wakeup_one_with_inheritor(event_t event, wait_result_t result, lck_wake_action_t action, thread_t *thread_wokenup);
+
+/*
+ * Name: wakeup_all_with_inheritor
+ *
+ * Description: wake up all waiters waiting for event. The old inheritor will lose the push.
+ *
+ * Args:
+ * Arg1: event to wake from.
+ * Arg2: wait result to pass to the woken up threads.
+ *
+ * Returns: KERN_NOT_WAITING if no threads were waiting, KERN_SUCCESS otherwise.
+ *
+ * Conditions: NOTE: this cannot be called from interrupt context.
+ */
+extern kern_return_t wakeup_all_with_inheritor(event_t event, wait_result_t result);
+
+/*
+ * Name: change_sleep_inheritor
+ *
+ * Description: Redirect the push of the waiting threads of event to the new inheritor specified.
+ *
+ * Args:
+ * Arg1: event to redirect the push.
+ * Arg2: new inheritor for event.
+ *
+ * Returns: KERN_NOT_WAITING if no threads were waiting, KERN_SUCCESS otherwise.
+ *
+ * Conditions: In case of success, the new inheritor cannot run in user space until another inheritor is specified for the event or a
+ * wakeup for the event is called.
+ * NOTE: this cannot be called from interrupt context.
+ */
+extern kern_return_t change_sleep_inheritor(event_t event, thread_t inheritor);
+
+/*
+ * gate structure
+ */
+typedef struct gate {
+ uintptr_t gate_data; // thread holder, interlock bit and waiter bit
+ struct turnstile *turnstile; // protected by the interlock bit
+} gate_t;
+
+#define GATE_ILOCK_BIT 0
+#define GATE_WAITERS_BIT 1
+
+#define GATE_ILOCK (1 << GATE_ILOCK_BIT)
+#define GATE_WAITERS (1 << GATE_WAITERS_BIT)
+
+#define gate_ilock(gate) hw_lock_bit((hw_lock_bit_t*)(&(gate)->gate_data), GATE_ILOCK_BIT, LCK_GRP_NULL)
+#define gate_iunlock(gate) hw_unlock_bit((hw_lock_bit_t*)(&(gate)->gate_data), GATE_ILOCK_BIT)
+#define gate_has_waiters(state) ((state & GATE_WAITERS) != 0)
+#define ordered_load_gate(gate) os_atomic_load(&(gate)->gate_data, compiler_acq_rel)
+#define ordered_store_gate(gate, value) os_atomic_store(&(gate)->gate_data, value, compiler_acq_rel)
+
+#define GATE_THREAD_MASK (~(uintptr_t)(GATE_ILOCK | GATE_WAITERS))
+#define GATE_STATE_TO_THREAD(state) (thread_t)(state & GATE_THREAD_MASK)
+#define GATE_THREAD_TO_STATE(thread) ((uintptr_t)thread)
+
+/*
+ * Possible gate_wait_result_t values.
+ */
+typedef int gate_wait_result_t;
+#define GATE_HANDOFF 0
+#define GATE_OPENED 1
+#define GATE_TIMED_OUT 2
+#define GATE_INTERRUPTED 3
+
+/*
+ * Gate flags used by gate_assert
+ */
+#define GATE_ASSERT_CLOSED 0
+#define GATE_ASSERT_OPEN 1
+#define GATE_ASSERT_HELD 2
+
+/*
+ * Gate flags used by gate_handoff
+ */
+#define GATE_HANDOFF_DEFAULT 0
+#define GATE_HANDOFF_OPEN_IF_NO_WAITERS 1
+
+#define GATE_EVENT(gate) ((event_t) gate)
+#define EVENT_TO_GATE(event) ((gate_t *) event)
+
+/*
+ * Name: decl_lck_rw_gate_data
+ *
+ * Description: declares a gate variable with specified storage class.
+ * The gate itself will be stored in this variable and it is the caller's responsibility
+ * to ensure that this variable's memory is going to be accessible by all threads that will use
+ * the gate.
+ * Every gate function will require a pointer to this variable as parameter. The same pointer should
+ * be used in every thread.
+ *
+ * The variable needs to be initialized once with lck_rw_gate_init() and destroyed once with
+ * lck_rw_gate_destroy() when not needed anymore.
+ *
+ * The gate will be used in conjunction with a lck_rw_t.
+ *
+ * Args:
+ * Arg1: storage class.
+ * Arg2: variable name.
+ */
+#define decl_lck_rw_gate_data(class, name) class gate_t name
+
+/*
+ * Name: lck_rw_gate_init
+ *
+ * Description: initializes a variable declared with decl_lck_rw_gate_data.
+ *
+ * Args:
+ * Arg1: lck_rw_t lock used to protect the gate.
+ * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data.
+ */
+extern void lck_rw_gate_init(lck_rw_t *lock, gate_t *gate);
+
+/*
+ * Name: lck_rw_gate_destroy
+ *
+ * Description: destroys a variable previously initialized.
+ *
+ * Args:
+ * Arg1: lck_rw_t lock used to protect the gate.
+ * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data.
+ */
+extern void lck_rw_gate_destroy(lck_rw_t *lock, gate_t *gate);
+
+/*
+ * Name: lck_rw_gate_try_close
+ *
+ * Description: Tries to close the gate.
+ * In case of success the current thread will be set as
+ * the holder of the gate.
+ *
+ * Args:
+ * Arg1: lck_rw_t lock used to protect the gate.
+ * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data.
+ *
+ * Conditions: Lock must be held. Returns with the lock held.
+ *
+ * Returns:
+ * KERN_SUCCESS in case the gate was successfully closed. The current thread is the new holder
+ * of the gate.
+ * A matching lck_rw_gate_open() or lck_rw_gate_handoff() needs to be called later on
+ * to wake up possible waiters on the gate before returning to userspace.
+ * If the intent is to conditionally probe the gate before waiting, the lock must not be dropped
+ * between the calls to lck_rw_gate_try_close() and lck_rw_gate_wait().
+ *
+ * KERN_FAILURE in case the gate was already closed. Will panic if the current thread was already the holder of the gate.
+ * lck_rw_gate_wait() should be called instead if the intent is to unconditionally wait on this gate.
+ * The calls to lck_rw_gate_try_close() and lck_rw_gate_wait() should
+ * be done without dropping the lock that is protecting the gate in between.
+ */
+extern kern_return_t lck_rw_gate_try_close(lck_rw_t *lock, gate_t *gate);
+
+/*
+ * Name: lck_rw_gate_close
+ *
+ * Description: Closes the gate. The current thread will be set as
+ * the holder of the gate. Will panic if the gate is already closed.
+ * A matching lck_rw_gate_open() or lck_rw_gate_handoff() needs to be called later on
+ * to wake up possible waiters on the gate before returning to userspace.
+ *
+ * Args:
+ * Arg1: lck_rw_t lock used to protect the gate.
+ * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data.
+ *
+ * Conditions: Lock must be held. Returns with the lock held.
+ * The gate must be open.
+ *
+ */
+extern void lck_rw_gate_close(lck_rw_t *lock, gate_t *gate);
projects / apple / xnu.git / blobdiff