X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/5d5c5d0d5b79ade9a973d55186ffda2638ba2b6e..HEAD:/osfmk/kern/locks.h diff --git a/osfmk/kern/locks.h b/osfmk/kern/locks.h index f7b65df47..e8475383c 100644 --- a/osfmk/kern/locks.h +++ b/osfmk/kern/locks.h @@ -1,423 +1,1167 @@ /* - * Copyright (c) 2003-2005 Apple Computer, Inc. All rights reserved. - * - * @APPLE_LICENSE_OSREFERENCE_HEADER_START@ - * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the - * License may not be used to create, or enable the creation or - * redistribution of, unlawful or unlicensed copies of an Apple operating - * system, or to circumvent, violate, or enable the circumvention or - * violation of, any terms of an Apple operating system software license - * agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this - * file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and + * Copyright (c) 2003-2019 Apple Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and * limitations under the License. * - * @APPLE_LICENSE_OSREFERENCE_HEADER_END@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #ifndef _KERN_LOCKS_H_ #define _KERN_LOCKS_H_ -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include -#ifdef MACH_KERNEL_PRIVATE -#include +__BEGIN_DECLS -extern void lck_mod_init( - void); +typedef unsigned int lck_sleep_action_t; -typedef unsigned int lck_type_t; +#define LCK_SLEEP_DEFAULT 0x00 /* Release the lock while waiting for the event, then reclaim */ +/* RW locks are returned in the same mode */ +#define LCK_SLEEP_UNLOCK 0x01 /* Release the lock and return unheld */ +#define LCK_SLEEP_SHARED 0x02 /* Reclaim the lock in shared mode (RW only) */ +#define LCK_SLEEP_EXCLUSIVE 0x04 /* Reclaim the lock in exclusive mode (RW only) */ +#define LCK_SLEEP_SPIN 0x08 /* Reclaim the lock in spin mode (mutex only) */ +#define LCK_SLEEP_PROMOTED_PRI 0x10 /* Sleep at a promoted priority */ +#define LCK_SLEEP_SPIN_ALWAYS 0x20 /* Reclaim the lock in spin-always mode (mutex only) */ -#define LCK_TYPE_SPIN 1 -#define LCK_TYPE_MTX 2 -#define LCK_TYPE_RW 3 +#define LCK_SLEEP_MASK 0x3f /* Valid actions */ -#endif +typedef unsigned int lck_wake_action_t; -typedef unsigned int lck_sleep_action_t; +#define LCK_WAKE_DEFAULT 0x00 /* If waiters are present, transfer their push to the wokenup thread */ +#define LCK_WAKE_DO_NOT_TRANSFER_PUSH 0x01 /* Do not transfer waiters push when waking up */ -#define LCK_SLEEP_DEFAULT 0x00 /* Release the lock while waiting for the event, then reclaim */ - /* RW locks are returned in the same mode */ -#define LCK_SLEEP_UNLOCK 0x01 /* Release the lock and return unheld */ -#define LCK_SLEEP_SHARED 0x02 /* Reclaim the lock in shared mode (RW only) */ -#define LCK_SLEEP_EXCLUSIVE 0x04 /* Reclaim the lock in exclusive mode (RW only) */ +#ifdef XNU_KERNEL_PRIVATE +#include -#define LCK_SLEEP_MASK 0x07 /* Valid actions */ +typedef struct _lck_attr_ { + unsigned int lck_attr_val; +} lck_attr_t; -#ifdef MACH_KERNEL_PRIVATE +extern lck_attr_t LockDefaultLckAttr; -typedef struct { - uint64_t lck_grp_spin_util_cnt; - uint64_t lck_grp_spin_held_cnt; - uint64_t lck_grp_spin_miss_cnt; - uint64_t lck_grp_spin_held_max; - uint64_t lck_grp_spin_held_cum; -} lck_grp_spin_stat_t; +#define LCK_ATTR_NONE 0 +#define LCK_ATTR_DEBUG 0x00000001 +#define LCK_ATTR_RW_SHARED_PRIORITY 0x00010000 +#else /* !XNU_KERNEL_PRIVATE */ +typedef struct __lck_attr__ lck_attr_t; +#endif /* !XNU_KERNEL_PRIVATE */ -typedef struct { - uint64_t lck_grp_mtx_util_cnt; - uint64_t lck_grp_mtx_held_cnt; - uint64_t lck_grp_mtx_miss_cnt; - uint64_t lck_grp_mtx_wait_cnt; - uint64_t lck_grp_mtx_held_max; - uint64_t lck_grp_mtx_held_cum; - uint64_t lck_grp_mtx_wait_max; - uint64_t lck_grp_mtx_wait_cum; -} lck_grp_mtx_stat_t; +#define LCK_ATTR_NULL (lck_attr_t *)NULL + +extern lck_attr_t *lck_attr_alloc_init(void); + +extern void lck_attr_setdefault( + lck_attr_t *attr); + +extern void lck_attr_setdebug( + lck_attr_t *attr); + +extern void lck_attr_cleardebug( + lck_attr_t *attr); + +#ifdef XNU_KERNEL_PRIVATE + +#if __x86_64__ +/* + * Extended mutexes are only implemented on x86_64 + */ +#define HAS_EXT_MUTEXES 1 +#endif /* __x86_64__ */ + +typedef union { + uint16_t tcurnext; + struct { + uint8_t cticket; + uint8_t nticket; + }; +} lck_ticket_internal; typedef struct { - uint64_t lck_grp_rw_util_cnt; - uint64_t lck_grp_rw_held_cnt; - uint64_t lck_grp_rw_miss_cnt; - uint64_t lck_grp_rw_wait_cnt; - uint64_t lck_grp_rw_held_max; - uint64_t lck_grp_rw_held_cum; - uint64_t lck_grp_rw_wait_max; - uint64_t lck_grp_rw_wait_cum; -} lck_grp_rw_stat_t; - -typedef struct _lck_grp_stat_ { - lck_grp_spin_stat_t lck_grp_spin_stat; - lck_grp_mtx_stat_t lck_grp_mtx_stat; - lck_grp_rw_stat_t lck_grp_rw_stat; -} lck_grp_stat_t; - -#define LCK_GRP_MAX_NAME 64 - -typedef struct _lck_grp_ { - queue_chain_t lck_grp_link; - unsigned int lck_grp_refcnt; - unsigned int lck_grp_spincnt; - unsigned int lck_grp_mtxcnt; - unsigned int lck_grp_rwcnt; - unsigned int lck_grp_attr; - char lck_grp_name[LCK_GRP_MAX_NAME]; - lck_grp_stat_t lck_grp_stat; -} lck_grp_t; - -#define LCK_GRP_NULL (lck_grp_t *)0 + lck_ticket_internal tu; + uintptr_t lck_owner; +} lck_ticket_t; + +void lck_ticket_init(lck_ticket_t *tlock, lck_grp_t *grp); +#if LOCK_STATS +void lck_ticket_lock(lck_ticket_t *tlock, lck_grp_t *grp); #else -typedef struct __lck_grp__ lck_grp_t; +void lck_ticket_lock(lck_ticket_t *tlock); +#define lck_ticket_lock(tlock, grp) lck_ticket_lock(tlock) +#endif /* LOCK_STATS */ + +void lck_ticket_unlock(lck_ticket_t *tlock); +void lck_ticket_assert_owned(lck_ticket_t *tlock); + +extern void lck_attr_rw_shared_priority( + lck_attr_t *attr); #endif -#ifdef MACH_KERNEL_PRIVATE -typedef struct _lck_grp_attr_ { - unsigned int grp_attr_val; -} lck_grp_attr_t; +extern void lck_attr_free( + lck_attr_t *attr); -extern lck_grp_attr_t LockDefaultGroupAttr; +#define decl_lck_spin_data(class, name) class lck_spin_t name -#define LCK_GRP_ATTR_STAT 0x1 +extern lck_spin_t *lck_spin_alloc_init( + lck_grp_t *grp, + lck_attr_t *attr); -#else -typedef struct __lck_grp_attr__ lck_grp_attr_t; -#endif +extern void lck_spin_init( + lck_spin_t *lck, + lck_grp_t *grp, + lck_attr_t *attr); -#define LCK_GRP_ATTR_NULL (lck_grp_attr_t *)0 +extern void lck_spin_lock( + lck_spin_t *lck); -__BEGIN_DECLS +extern void lck_spin_lock_grp( + lck_spin_t *lck, + lck_grp_t *grp); -extern lck_grp_attr_t *lck_grp_attr_alloc_init( - void); +extern void lck_spin_unlock( + lck_spin_t *lck); -extern void lck_grp_attr_setdefault( - lck_grp_attr_t *attr); +extern void lck_spin_destroy( + lck_spin_t *lck, + lck_grp_t *grp); -extern void lck_grp_attr_setstat( - lck_grp_attr_t *attr); +extern void lck_spin_free( + lck_spin_t *lck, + lck_grp_t *grp); -extern void lck_grp_attr_free( - lck_grp_attr_t *attr); +extern wait_result_t lck_spin_sleep( + lck_spin_t *lck, + lck_sleep_action_t lck_sleep_action, + event_t event, + wait_interrupt_t interruptible); -extern lck_grp_t *lck_grp_alloc_init( - const char* grp_name, - lck_grp_attr_t *attr); +extern wait_result_t lck_spin_sleep_grp( + lck_spin_t *lck, + lck_sleep_action_t lck_sleep_action, + event_t event, + wait_interrupt_t interruptible, + lck_grp_t *grp); -__END_DECLS +extern wait_result_t lck_spin_sleep_deadline( + lck_spin_t *lck, + lck_sleep_action_t lck_sleep_action, + event_t event, + wait_interrupt_t interruptible, + uint64_t deadline); + +#ifdef KERNEL_PRIVATE + +extern void lck_spin_lock_nopreempt( + lck_spin_t *lck); + +extern void lck_spin_lock_nopreempt_grp( + lck_spin_t *lck, lck_grp_t *grp); + +extern void lck_spin_unlock_nopreempt( + lck_spin_t *lck); + +extern boolean_t lck_spin_try_lock_grp( + lck_spin_t *lck, + lck_grp_t *grp); + +extern boolean_t lck_spin_try_lock( + lck_spin_t *lck); -#ifdef MACH_KERNEL_PRIVATE -extern void lck_grp_init( - lck_grp_t *grp, - const char* grp_name, - lck_grp_attr_t *attr); +extern boolean_t lck_spin_try_lock_nopreempt( + lck_spin_t *lck); -extern void lck_grp_reference( - lck_grp_t *grp); +extern boolean_t lck_spin_try_lock_nopreempt_grp( + lck_spin_t *lck, + lck_grp_t *grp); -extern void lck_grp_deallocate( - lck_grp_t *grp); +/* NOT SAFE: To be used only by kernel debugger to avoid deadlock. */ +extern boolean_t kdp_lck_spin_is_acquired( + lck_spin_t *lck); -extern void lck_grp_lckcnt_incr( - lck_grp_t *grp, - lck_type_t lck_type); +struct _lck_mtx_ext_; +extern void lck_mtx_init_ext( + lck_mtx_t *lck, + struct _lck_mtx_ext_ *lck_ext, + lck_grp_t *grp, + lck_attr_t *attr); -extern void lck_grp_lckcnt_decr( - lck_grp_t *grp, - lck_type_t lck_type); #endif -__BEGIN_DECLS +#define decl_lck_mtx_data(class, name) class lck_mtx_t name -extern void lck_grp_free( - lck_grp_t *grp); +extern lck_mtx_t *lck_mtx_alloc_init( + lck_grp_t *grp, + lck_attr_t *attr); -__END_DECLS +extern void lck_mtx_init( + lck_mtx_t *lck, + lck_grp_t *grp, + lck_attr_t *attr); +extern void lck_mtx_lock( + lck_mtx_t *lck); -#ifdef MACH_KERNEL_PRIVATE -typedef struct _lck_attr_ { - unsigned int lck_attr_val; -} lck_attr_t; +extern void lck_mtx_unlock( + lck_mtx_t *lck); -extern lck_attr_t LockDefaultLckAttr; +extern void lck_mtx_destroy( + lck_mtx_t *lck, + lck_grp_t *grp); -#define LCK_ATTR_NONE 0 +extern void lck_mtx_free( + lck_mtx_t *lck, + lck_grp_t *grp); -#define LCK_ATTR_DEBUG 0x00000001 -#define LCK_ATTR_RW_SHARED_PRIORITY 0x00010000 +extern wait_result_t lck_mtx_sleep( + lck_mtx_t *lck, + lck_sleep_action_t lck_sleep_action, + event_t event, + wait_interrupt_t interruptible); -#else -typedef struct __lck_attr__ lck_attr_t; -#endif +extern wait_result_t lck_mtx_sleep_deadline( + lck_mtx_t *lck, + lck_sleep_action_t lck_sleep_action, + event_t event, + wait_interrupt_t interruptible, + uint64_t deadline); -#define LCK_ATTR_NULL (lck_attr_t *)0 +#ifdef KERNEL_PRIVATE +/* + * Name: lck_spin_sleep_with_inheritor + * + * Description: deschedule the current thread and wait on the waitq associated with event to be woken up. + * While waiting, the sched priority of the waiting thread will contribute to the push of the event that will + * be directed to the inheritor specified. + * An interruptible mode and deadline can be specified to return earlier from the wait. + * + * Args: + * Arg1: lck_spin_t lock used to protect the sleep. The lock will be dropped while sleeping and reaquired before returning according to the sleep action specified. + * Arg2: sleep action. LCK_SLEEP_DEFAULT, LCK_SLEEP_UNLOCK. + * Arg3: event to wait on. + * Arg4: thread to propagate the event push to. + * Arg5: interruptible flag for wait. + * Arg6: deadline for wait. + * + * Conditions: Lock must be held. Returns with the lock held according to the sleep action specified. + * Lock will be dropped while waiting. + * The inheritor specified cannot run in user space until another inheritor is specified for the event or a + * wakeup for the event is called. + * + * Returns: result of the wait. + */ +extern wait_result_t lck_spin_sleep_with_inheritor(lck_spin_t *lock, lck_sleep_action_t lck_sleep_action, event_t event, thread_t inheritor, wait_interrupt_t interruptible, uint64_t deadline); -__BEGIN_DECLS +/* + * Name: lck_mtx_sleep_with_inheritor + * + * Description: deschedule the current thread and wait on the waitq associated with event to be woken up. + * While waiting, the sched priority of the waiting thread will contribute to the push of the event that will + * be directed to the inheritor specified. + * An interruptible mode and deadline can be specified to return earlier from the wait. + * + * Args: + * Arg1: lck_mtx_t lock used to protect the sleep. The lock will be dropped while sleeping and reaquired before returning according to the sleep action specified. + * Arg2: sleep action. LCK_SLEEP_DEFAULT, LCK_SLEEP_UNLOCK, LCK_SLEEP_SPIN, LCK_SLEEP_SPIN_ALWAYS. + * Arg3: event to wait on. + * Arg4: thread to propagate the event push to. + * Arg5: interruptible flag for wait. + * Arg6: deadline for wait. + * + * Conditions: Lock must be held. Returns with the lock held according to the sleep action specified. + * Lock will be dropped while waiting. + * The inheritor specified cannot run in user space until another inheritor is specified for the event or a + * wakeup for the event is called. + * + * Returns: result of the wait. + */ +extern wait_result_t lck_mtx_sleep_with_inheritor(lck_mtx_t *lock, lck_sleep_action_t lck_sleep_action, event_t event, thread_t inheritor, wait_interrupt_t interruptible, uint64_t deadline); -extern lck_attr_t *lck_attr_alloc_init( - void); +/* + * Name: lck_mtx_sleep_with_inheritor + * + * Description: deschedule the current thread and wait on the waitq associated with event to be woken up. + * While waiting, the sched priority of the waiting thread will contribute to the push of the event that will + * be directed to the inheritor specified. + * An interruptible mode and deadline can be specified to return earlier from the wait. + * + * Args: + * Arg1: lck_rw_t lock used to protect the sleep. The lock will be dropped while sleeping and reaquired before returning according to the sleep action specified. + * Arg2: sleep action. LCK_SLEEP_DEFAULT, LCK_SLEEP_SHARED, LCK_SLEEP_EXCLUSIVE. + * Arg3: event to wait on. + * Arg4: thread to propagate the event push to. + * Arg5: interruptible flag for wait. + * Arg6: deadline for wait. + * + * Conditions: Lock must be held. Returns with the lock held according to the sleep action specified. + * Lock will be dropped while waiting. + * The inheritor specified cannot run in user space until another inheritor is specified for the event or a + * wakeup for the event is called. + * + * Returns: result of the wait. + */ +extern wait_result_t lck_rw_sleep_with_inheritor(lck_rw_t *lock, lck_sleep_action_t lck_sleep_action, event_t event, thread_t inheritor, wait_interrupt_t interruptible, uint64_t deadline); -extern void lck_attr_setdefault( - lck_attr_t *attr); +/* + * Name: wakeup_one_with_inheritor + * + * Description: wake up one waiter for event if any. The thread woken up will be the one with the higher sched priority waiting on event. + * The push for the event will be transferred from the last inheritor to the woken up thread. + * + * Args: + * Arg1: event to wake from. + * Arg2: wait result to pass to the woken up thread. + * Arg3: pointer for storing the thread wokenup. + * + * Returns: KERN_NOT_WAITING if no threads were waiting, KERN_SUCCESS otherwise. + * + * Conditions: The new inheritor wokenup cannot run in user space until another inheritor is specified for the event or a + * wakeup for the event is called. + * A reference for the wokenup thread is acquired. + * NOTE: this cannot be called from interrupt context. + */ +extern kern_return_t wakeup_one_with_inheritor(event_t event, wait_result_t result, lck_wake_action_t action, thread_t *thread_wokenup); -extern void lck_attr_setdebug( - lck_attr_t *attr); +/* + * Name: wakeup_all_with_inheritor + * + * Description: wake up all waiters waiting for event. The old inheritor will lose the push. + * + * Args: + * Arg1: event to wake from. + * Arg2: wait result to pass to the woken up threads. + * + * Returns: KERN_NOT_WAITING if no threads were waiting, KERN_SUCCESS otherwise. + * + * Conditions: NOTE: this cannot be called from interrupt context. + */ +extern kern_return_t wakeup_all_with_inheritor(event_t event, wait_result_t result); -#ifdef XNU_KERNEL_PRIVATE -extern void lck_attr_rw_shared_priority( - lck_attr_t *attr); -#endif +/* + * Name: change_sleep_inheritor + * + * Description: Redirect the push of the waiting threads of event to the new inheritor specified. + * + * Args: + * Arg1: event to redirect the push. + * Arg2: new inheritor for event. + * + * Returns: KERN_NOT_WAITING if no threads were waiting, KERN_SUCCESS otherwise. + * + * Conditions: In case of success, the new inheritor cannot run in user space until another inheritor is specified for the event or a + * wakeup for the event is called. + * NOTE: this cannot be called from interrupt context. + */ +extern kern_return_t change_sleep_inheritor(event_t event, thread_t inheritor); + +/* + * gate structure + */ +typedef struct gate { + uintptr_t gate_data; // thread holder, interlock bit and waiter bit + struct turnstile *turnstile; // protected by the interlock bit +} gate_t; + +#define GATE_ILOCK_BIT 0 +#define GATE_WAITERS_BIT 1 + +#define GATE_ILOCK (1 << GATE_ILOCK_BIT) +#define GATE_WAITERS (1 << GATE_WAITERS_BIT) + +#define gate_ilock(gate) hw_lock_bit((hw_lock_bit_t*)(&(gate)->gate_data), GATE_ILOCK_BIT, LCK_GRP_NULL) +#define gate_iunlock(gate) hw_unlock_bit((hw_lock_bit_t*)(&(gate)->gate_data), GATE_ILOCK_BIT) +#define gate_has_waiters(state) ((state & GATE_WAITERS) != 0) +#define ordered_load_gate(gate) os_atomic_load(&(gate)->gate_data, compiler_acq_rel) +#define ordered_store_gate(gate, value) os_atomic_store(&(gate)->gate_data, value, compiler_acq_rel) + +#define GATE_THREAD_MASK (~(uintptr_t)(GATE_ILOCK | GATE_WAITERS)) +#define GATE_STATE_TO_THREAD(state) (thread_t)(state & GATE_THREAD_MASK) +#define GATE_THREAD_TO_STATE(thread) ((uintptr_t)thread) + +/* + * Possible gate_wait_result_t values. + */ +typedef int gate_wait_result_t; +#define GATE_HANDOFF 0 +#define GATE_OPENED 1 +#define GATE_TIMED_OUT 2 +#define GATE_INTERRUPTED 3 + +/* + * Gate flags used by gate_assert + */ +#define GATE_ASSERT_CLOSED 0 +#define GATE_ASSERT_OPEN 1 +#define GATE_ASSERT_HELD 2 + +/* + * Gate flags used by gate_handoff + */ +#define GATE_HANDOFF_DEFAULT 0 +#define GATE_HANDOFF_OPEN_IF_NO_WAITERS 1 + +#define GATE_EVENT(gate) ((event_t) gate) +#define EVENT_TO_GATE(event) ((gate_t *) event) + +/* + * Name: decl_lck_rw_gate_data + * + * Description: declares a gate variable with specified storage class. + * The gate itself will be stored in this variable and it is the caller's responsibility + * to ensure that this variable's memory is going to be accessible by all threads that will use + * the gate. + * Every gate function will require a pointer to this variable as parameter. The same pointer should + * be used in every thread. + * + * The variable needs to be initialized once with lck_rw_gate_init() and destroyed once with + * lck_rw_gate_destroy() when not needed anymore. + * + * The gate will be used in conjunction with a lck_rw_t. + * + * Args: + * Arg1: storage class. + * Arg2: variable name. + */ +#define decl_lck_rw_gate_data(class, name) class gate_t name + +/* + * Name: lck_rw_gate_init + * + * Description: initializes a variable declared with decl_lck_rw_gate_data. + * + * Args: + * Arg1: lck_rw_t lock used to protect the gate. + * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data. + */ +extern void lck_rw_gate_init(lck_rw_t *lock, gate_t *gate); + +/* + * Name: lck_rw_gate_destroy + * + * Description: destroys a variable previously initialized. + * + * Args: + * Arg1: lck_rw_t lock used to protect the gate. + * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data. + */ +extern void lck_rw_gate_destroy(lck_rw_t *lock, gate_t *gate); + +/* + * Name: lck_rw_gate_try_close + * + * Description: Tries to close the gate. + * In case of success the current thread will be set as + * the holder of the gate. + * + * Args: + * Arg1: lck_rw_t lock used to protect the gate. + * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data. + * + * Conditions: Lock must be held. Returns with the lock held. + * + * Returns: + * KERN_SUCCESS in case the gate was successfully closed. The current thread is the new holder + * of the gate. + * A matching lck_rw_gate_open() or lck_rw_gate_handoff() needs to be called later on + * to wake up possible waiters on the gate before returning to userspace. + * If the intent is to conditionally probe the gate before waiting, the lock must not be dropped + * between the calls to lck_rw_gate_try_close() and lck_rw_gate_wait(). + * + * KERN_FAILURE in case the gate was already closed. Will panic if the current thread was already the holder of the gate. + * lck_rw_gate_wait() should be called instead if the intent is to unconditionally wait on this gate. + * The calls to lck_rw_gate_try_close() and lck_rw_gate_wait() should + * be done without dropping the lock that is protecting the gate in between. + */ +extern kern_return_t lck_rw_gate_try_close(lck_rw_t *lock, gate_t *gate); + +/* + * Name: lck_rw_gate_close + * + * Description: Closes the gate. The current thread will be set as + * the holder of the gate. Will panic if the gate is already closed. + * A matching lck_rw_gate_open() or lck_rw_gate_handoff() needs to be called later on + * to wake up possible waiters on the gate before returning to userspace. + * + * Args: + * Arg1: lck_rw_t lock used to protect the gate. + * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data. + * + * Conditions: Lock must be held. Returns with the lock held. + * The gate must be open. + * + */ +extern void lck_rw_gate_close(lck_rw_t *lock, gate_t *gate); + + +/* + * Name: lck_rw_gate_open + * + * Description: Opens the gate and wakes up possible waiters. + * + * Args: + * Arg1: lck_rw_t lock used to protect the gate. + * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data. + * + * Conditions: Lock must be held. Returns with the lock held. + * The current thread must be the holder of the gate. + * + */ +extern void lck_rw_gate_open(lck_rw_t *lock, gate_t *gate); -extern void lck_attr_free( - lck_attr_t *attr); +/* + * Name: lck_rw_gate_handoff + * + * Description: Tries to transfer the ownership of the gate. The waiter with highest sched + * priority will be selected as the new holder of the gate, and woken up, + * with the gate remaining in the closed state throughout. + * If no waiters are present, the gate will be kept closed and KERN_NOT_WAITING + * will be returned. + * GATE_HANDOFF_OPEN_IF_NO_WAITERS flag can be used to specify if the gate should be opened in + * case no waiters were found. + * + * + * Args: + * Arg1: lck_rw_t lock used to protect the gate. + * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data. + * Arg3: flags - GATE_HANDOFF_DEFAULT or GATE_HANDOFF_OPEN_IF_NO_WAITERS + * + * Conditions: Lock must be held. Returns with the lock held. + * The current thread must be the holder of the gate. + * + * Returns: + * KERN_SUCCESS in case one of the waiters became the new holder. + * KERN_NOT_WAITING in case there were no waiters. + * + */ +extern kern_return_t lck_rw_gate_handoff(lck_rw_t *lock, gate_t *gate, int flags); -#define decl_lck_spin_data(class,name) class lck_spin_t name; +/* + * Name: lck_rw_gate_steal + * + * Description: Set the current ownership of the gate. It sets the current thread as the + * new holder of the gate. + * A matching lck_rw_gate_open() or lck_rw_gate_handoff() needs to be called later on + * to wake up possible waiters on the gate before returning to userspace. + * NOTE: the previous holder should not call lck_rw_gate_open() or lck_rw_gate_handoff() + * anymore. + * + * + * Args: + * Arg1: lck_rw_t lock used to protect the gate. + * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data. + * + * Conditions: Lock must be held. Returns with the lock held. + * The gate must be closed and the current thread must not already be the holder. + * + */ +extern void lck_rw_gate_steal(lck_rw_t *lock, gate_t *gate); -extern lck_spin_t *lck_spin_alloc_init( - lck_grp_t *grp, - lck_attr_t *attr); +/* + * Name: lck_rw_gate_wait + * + * Description: Waits for the current thread to become the holder of the gate or for the + * gate to become open. An interruptible mode and deadline can be specified + * to return earlier from the wait. + * + * Args: + * Arg1: lck_rw_t lock used to protect the gate. + * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data. + * Arg3: sleep action. LCK_SLEEP_DEFAULT, LCK_SLEEP_SHARED, LCK_SLEEP_EXCLUSIVE. + * Arg3: interruptible flag for wait. + * Arg4: deadline + * + * Conditions: Lock must be held. Returns with the lock held according to the sleep action specified. + * Lock will be dropped while waiting. + * The gate must be closed. + * + * Returns: Reason why the thread was woken up. + * GATE_HANDOFF - the current thread was handed off the ownership of the gate. + * A matching lck_rw_gate_open() or lck_rw_gate_handoff() needs to be called later on + * to wake up possible waiters on the gate before returning to userspace. + * GATE_OPENED - the gate was opened by the holder. + * GATE_TIMED_OUT - the thread was woken up by a timeout. + * GATE_INTERRUPTED - the thread was interrupted while sleeping. + * + */ +extern gate_wait_result_t lck_rw_gate_wait(lck_rw_t *lock, gate_t *gate, lck_sleep_action_t lck_sleep_action, wait_interrupt_t interruptible, uint64_t deadline); -extern void lck_spin_init( - lck_spin_t *lck, - lck_grp_t *grp, - lck_attr_t *attr); +/* + * Name: lck_rw_gate_assert + * + * Description: asserts that the gate is in the specified state. + * + * Args: + * Arg1: lck_rw_t lock used to protect the gate. + * Arg2: pointer to the gate data declared with decl_lck_rw_gate_data. + * Arg3: flags to specified assert type. + * GATE_ASSERT_CLOSED - the gate is currently closed + * GATE_ASSERT_OPEN - the gate is currently opened + * GATE_ASSERT_HELD - the gate is currently closed and the current thread is the holder + */ +extern void lck_rw_gate_assert(lck_rw_t *lock, gate_t *gate, int flags); -extern void lck_spin_lock( - lck_spin_t *lck); +/* + * Name: decl_lck_mtx_gate_data + * + * Description: declares a gate variable with specified storage class. + * The gate itself will be stored in this variable and it is the caller's responsibility + * to ensure that this variable's memory is going to be accessible by all threads that will use + * the gate. + * Every gate function will require a pointer to this variable as parameter. The same pointer should + * be used in every thread. + * + * The variable needs to be initialized once with lck_mtx_gate_init() and destroyed once with + * lck_mtx_gate_destroy() when not needed anymore. + * + * The gate will be used in conjunction with a lck_mtx_t. + * + * Args: + * Arg1: storage class. + * Arg2: variable name. + */ +#define decl_lck_mtx_gate_data(class, name) class gate_t name -extern void lck_spin_unlock( - lck_spin_t *lck); +/* + * Name: lck_mtx_gate_init + * + * Description: initializes a variable declared with decl_lck_mtx_gate_data. + * + * Args: + * Arg1: lck_mtx_t lock used to protect the gate. + * Arg2: pointer to the gate data declared with decl_lck_mtx_gate_data. + */ +extern void lck_mtx_gate_init(lck_mtx_t *lock, gate_t *gate); -extern void lck_spin_destroy( - lck_spin_t *lck, - lck_grp_t *grp); +/* + * Name: lck_mtx_gate_destroy + * + * Description: destroys a variable previously initialized + * + * Args: + * Arg1: lck_mtx_t lock used to protect the gate. + * Arg2: pointer to the gate data declared with decl_lck_mtx_gate_data. + */ +extern void lck_mtx_gate_destroy(lck_mtx_t *lock, gate_t *gate); -extern void lck_spin_free( - lck_spin_t *lck, - lck_grp_t *grp); +/* + * Name: lck_mtx_gate_try_close + * + * Description: Tries to close the gate. + * In case of success the current thread will be set as + * the holder of the gate. + * + * Args: + * Arg1: lck_mtx_t lock used to protect the gate. + * Arg2: pointer to the gate data declared with decl_lck_mtx_gate_data. + * + * Conditions: Lock must be held. Returns with the lock held. + * + * Returns: + * KERN_SUCCESS in case the gate was successfully closed. The current thread is the new holder + * of the gate. + * A matching lck_mtx_gate_open() or lck_mtx_gate_handoff() needs to be called later on + * to wake up possible waiters on the gate before returning to userspace. + * If the intent is to conditionally probe the gate before waiting, the lock must not be dropped + * between the calls to lck_mtx_gate_try_close() and lck_mtx_gate_wait(). + * + * KERN_FAILURE in case the gate was already closed. Will panic if the current thread was already the holder of the gate. + * lck_mtx_gate_wait() should be called instead if the intent is to unconditionally wait on this gate. + * The calls to lck_mtx_gate_try_close() and lck_mtx_gate_wait() should + * be done without dropping the lock that is protecting the gate in between. + */ +extern kern_return_t lck_mtx_gate_try_close(lck_mtx_t *lock, gate_t *gate); -extern wait_result_t lck_spin_sleep( - lck_spin_t *lck, - lck_sleep_action_t lck_sleep_action, - event_t event, - wait_interrupt_t interruptible); +/* + * Name: lck_mtx_gate_close + * + * Description: Closes the gate. The current thread will be set as + * the holder of the gate. Will panic if the gate is already closed. + * A matching lck_mtx_gate_open() or lck_mtx_gate_handoff() needs to be called later on + * to wake up possible waiters on the gate before returning to userspace. + * + * Args: + * Arg1: lck_mtx_t lock used to protect the gate. + * Arg2: pointer to the gate data declared with decl_lck_mtx_gate_data. + * + * Conditions: Lock must be held. Returns with the lock held. + * The gate must be open. + * + */ +extern void lck_mtx_gate_close(lck_mtx_t *lock, gate_t *gate); -extern wait_result_t lck_spin_sleep_deadline( - lck_spin_t *lck, - lck_sleep_action_t lck_sleep_action, - event_t event, - wait_interrupt_t interruptible, - uint64_t deadline); +/* + * Name: lck_mtx_gate_open + * + * Description: Opens of the gate and wakes up possible waiters. + * + * Args: + * Arg1: lck_mtx_t lock used to protect the gate. + * Arg2: pointer to the gate data declared with decl_lck_mtx_gate_data. + * + * Conditions: Lock must be held. Returns with the lock held. + * The current thread must be the holder of the gate. + * + */ +extern void lck_mtx_gate_open(lck_mtx_t *lock, gate_t *gate); -#ifdef KERNEL_PRIVATE +/* + * Name: lck_mtx_gate_handoff + * + * Description: Set the current ownership of the gate. The waiter with highest sched + * priority will be selected as the new holder of the gate, and woken up, + * with the gate remaining in the closed state throughout. + * If no waiters are present, the gate will be kept closed and KERN_NOT_WAITING + * will be returned. + * OPEN_ON_FAILURE flag can be used to specify if the gate should be opened in + * case no waiters were found. + * + * + * Args: + * Arg1: lck_mtx_t lock used to protect the gate. + * Arg2: pointer to the gate data declared with decl_lck_mtx_gate_data. + * Arg3: flags - GATE_NO_FALGS or OPEN_ON_FAILURE + * + * Conditions: Lock must be held. Returns with the lock held. + * The current thread must be the holder of the gate. + * + * Returns: + * KERN_SUCCESS in case one of the waiters became the new holder. + * KERN_NOT_WAITING in case there were no waiters. + * + */ +extern kern_return_t lck_mtx_gate_handoff(lck_mtx_t *lock, gate_t *gate, int flags); + +/* + * Name: lck_mtx_gate_steal + * + * Description: Steals the ownership of the gate. It sets the current thread as the + * new holder of the gate. + * A matching lck_mtx_gate_open() or lck_mtx_gate_handoff() needs to be called later on + * to wake up possible waiters on the gate before returning to userspace. + * NOTE: the previous holder should not call lck_mtx_gate_open() or lck_mtx_gate_handoff() + * anymore. + * + * + * Args: + * Arg1: lck_mtx_t lock used to protect the gate. + * Arg2: pointer to the gate data declared with decl_lck_mtx_gate_data. + * + * Conditions: Lock must be held. Returns with the lock held. + * The gate must be closed and the current thread must not already be the holder. + * + */ +extern void lck_mtx_gate_steal(lck_mtx_t *lock, gate_t *gate); -extern boolean_t lck_spin_try_lock( - lck_spin_t *lck); +/* + * Name: lck_mtx_gate_wait + * + * Description: Waits for the current thread to become the holder of the gate or for the + * gate to become open. An interruptible mode and deadline can be specified + * to return earlier from the wait. + * + * Args: + * Arg1: lck_mtx_t lock used to protect the gate. + * Arg2: pointer to the gate data declared with decl_lck_mtx_gate_data. + * Arg3: sleep action. LCK_SLEEP_DEFAULT, LCK_SLEEP_UNLOCK, LCK_SLEEP_SPIN, LCK_SLEEP_SPIN_ALWAYS. + * Arg3: interruptible flag for wait. + * Arg4: deadline + * + * Conditions: Lock must be held. Returns with the lock held according to the sleep action specified. + * Lock will be dropped while waiting. + * The gate must be closed. + * + * Returns: Reason why the thread was woken up. + * GATE_HANDOFF - the current thread was handed off the ownership of the gate. + * A matching lck_mtx_gate_open() or lck_mtx_gate_handoff() needs to be called later on + * to wake up possible waiters on the gate before returning to userspace. + * GATE_OPENED - the gate was opened by the holder. + * GATE_TIMED_OUT - the thread was woken up by a timeout. + * GATE_INTERRUPTED - the thread was interrupted while sleeping. + * + */ +extern gate_wait_result_t lck_mtx_gate_wait(lck_mtx_t *lock, gate_t *gate, lck_sleep_action_t lck_sleep_action, wait_interrupt_t interruptible, uint64_t deadline); + +/* + * Name: lck_mtx_gate_assert + * + * Description: asserts that the gate is in the specified state. + * + * Args: + * Arg1: lck_mtx_t lock used to protect the gate. + * Arg2: pointer to the gate data declared with decl_lck_mtx_gate_data. + * Arg3: flags to specified assert type. + * GATE_ASSERT_CLOSED - the gate is currently closed + * GATE_ASSERT_OPEN - the gate is currently opened + * GATE_ASSERT_HELD - the gate is currently closed and the current thread is the holder + */ +extern void lck_mtx_gate_assert(lck_mtx_t *lock, gate_t *gate, int flags); + +#endif //KERNEL_PRIVATE + +#if DEVELOPMENT || DEBUG +#define FULL_CONTENDED 0 +#define HALF_CONTENDED 1 +#define MAX_CONDENDED 2 + +extern void erase_all_test_mtx_stats(void); +extern int get_test_mtx_stats_string(char* buffer, int buffer_size); +extern void lck_mtx_test_init(void); +extern void lck_mtx_test_lock(void); +extern void lck_mtx_test_unlock(void); +extern int lck_mtx_test_mtx_uncontended(int iter, char* buffer, int buffer_size); +extern int lck_mtx_test_mtx_contended(int iter, char* buffer, int buffer_size, int type); +extern int lck_mtx_test_mtx_uncontended_loop_time(int iter, char* buffer, int buffer_size); +extern int lck_mtx_test_mtx_contended_loop_time(int iter, char* buffer, int buffer_size, int type); #endif +#ifdef KERNEL_PRIVATE +extern boolean_t lck_mtx_try_lock( + lck_mtx_t *lck); -#define decl_lck_mtx_data(class,name) class lck_mtx_t name; +extern void mutex_pause(uint32_t); -extern lck_mtx_t *lck_mtx_alloc_init( - lck_grp_t *grp, - lck_attr_t *attr); +extern void lck_mtx_yield( + lck_mtx_t *lck); -extern void lck_mtx_init( - lck_mtx_t *lck, - lck_grp_t *grp, - lck_attr_t *attr); +extern boolean_t lck_mtx_try_lock_spin( + lck_mtx_t *lck); -extern void lck_mtx_lock( - lck_mtx_t *lck); +extern void lck_mtx_lock_spin( + lck_mtx_t *lck); -extern void lck_mtx_unlock( - lck_mtx_t *lck); +extern boolean_t kdp_lck_mtx_lock_spin_is_acquired( + lck_mtx_t *lck); -extern void lck_mtx_destroy( - lck_mtx_t *lck, - lck_grp_t *grp); +extern void lck_mtx_convert_spin( + lck_mtx_t *lck); -extern void lck_mtx_free( - lck_mtx_t *lck, - lck_grp_t *grp); +extern void lck_mtx_lock_spin_always( + lck_mtx_t *lck); -extern wait_result_t lck_mtx_sleep( - lck_mtx_t *lck, - lck_sleep_action_t lck_sleep_action, - event_t event, - wait_interrupt_t interruptible); +extern boolean_t lck_mtx_try_lock_spin_always( + lck_mtx_t *lck); -extern wait_result_t lck_mtx_sleep_deadline( - lck_mtx_t *lck, - lck_sleep_action_t lck_sleep_action, - event_t event, - wait_interrupt_t interruptible, - uint64_t deadline); +#define lck_mtx_unlock_always(l) lck_mtx_unlock(l) -#ifdef KERNEL_PRIVATE +extern void lck_spin_assert( + lck_spin_t *lck, + unsigned int type); -extern boolean_t lck_mtx_try_lock( - lck_mtx_t *lck); +extern boolean_t kdp_lck_rw_lock_is_acquired_exclusive( + lck_rw_t *lck); -#endif /* KERNEL_PRIVATE */ +#endif /* KERNEL_PRIVATE */ -extern void lck_mtx_assert( - lck_mtx_t *lck, - unsigned int type); +extern void lck_mtx_assert( + lck_mtx_t *lck, + unsigned int type); -__END_DECLS +#if MACH_ASSERT +#define LCK_MTX_ASSERT(lck, type) lck_mtx_assert((lck),(type)) +#define LCK_SPIN_ASSERT(lck, type) lck_spin_assert((lck),(type)) +#define LCK_RW_ASSERT(lck, type) lck_rw_assert((lck),(type)) +#else /* MACH_ASSERT */ +#define LCK_MTX_ASSERT(lck, type) +#define LCK_SPIN_ASSERT(lck, type) +#define LCK_RW_ASSERT(lck, type) +#endif /* MACH_ASSERT */ -#define LCK_MTX_ASSERT_OWNED 0x01 -#define LCK_MTX_ASSERT_NOTOWNED 0x02 +#if DEBUG +#define LCK_MTX_ASSERT_DEBUG(lck, type) lck_mtx_assert((lck),(type)) +#define LCK_SPIN_ASSERT_DEBUG(lck, type) lck_spin_assert((lck),(type)) +#define LCK_RW_ASSERT_DEBUG(lck, type) lck_rw_assert((lck),(type)) +#else /* DEBUG */ +#define LCK_MTX_ASSERT_DEBUG(lck, type) +#define LCK_SPIN_ASSERT_DEBUG(lck, type) +#define LCK_RW_ASSERT_DEBUG(lck, type) +#endif /* DEBUG */ -#ifdef MACH_KERNEL_PRIVATE -extern void lck_mtx_lock_wait( - lck_mtx_t *lck, - thread_t holder); +#define LCK_ASSERT_OWNED 1 +#define LCK_ASSERT_NOTOWNED 2 -extern int lck_mtx_lock_acquire( - lck_mtx_t *lck); +#define LCK_MTX_ASSERT_OWNED LCK_ASSERT_OWNED +#define LCK_MTX_ASSERT_NOTOWNED LCK_ASSERT_NOTOWNED -extern void lck_mtx_unlock_wakeup( - lck_mtx_t *lck, - thread_t holder); +#ifdef MACH_KERNEL_PRIVATE +struct turnstile; +extern void lck_mtx_lock_wait( + lck_mtx_t *lck, + thread_t holder, + struct turnstile **ts); + +extern int lck_mtx_lock_acquire( + lck_mtx_t *lck, + struct turnstile *ts); + +extern boolean_t lck_mtx_unlock_wakeup( + lck_mtx_t *lck, + thread_t holder); + +extern boolean_t lck_mtx_ilk_unlock( + lck_mtx_t *lck); + +extern boolean_t lck_mtx_ilk_try_lock( + lck_mtx_t *lck); + +extern void lck_mtx_wakeup_adjust_pri(thread_t thread, integer_t priority); -extern boolean_t lck_mtx_ilk_unlock( - lck_mtx_t *lck); #endif -#define decl_lck_rw_data(class,name) class lck_rw_t name; +#define decl_lck_rw_data(class, name) class lck_rw_t name -typedef unsigned int lck_rw_type_t; +typedef unsigned int lck_rw_type_t; -#define LCK_RW_TYPE_SHARED 0x01 -#define LCK_RW_TYPE_EXCLUSIVE 0x02 +#define LCK_RW_TYPE_SHARED 0x01 +#define LCK_RW_TYPE_EXCLUSIVE 0x02 -__BEGIN_DECLS +#ifdef XNU_KERNEL_PRIVATE +#define LCK_RW_ASSERT_SHARED 0x01 +#define LCK_RW_ASSERT_EXCLUSIVE 0x02 +#define LCK_RW_ASSERT_HELD 0x03 +#define LCK_RW_ASSERT_NOTHELD 0x04 +#endif + +extern lck_rw_t *lck_rw_alloc_init( + lck_grp_t *grp, + lck_attr_t *attr); -extern lck_rw_t *lck_rw_alloc_init( - lck_grp_t *grp, - lck_attr_t *attr); +extern void lck_rw_init( + lck_rw_t *lck, + lck_grp_t *grp, + lck_attr_t *attr); -extern void lck_rw_init( - lck_rw_t *lck, - lck_grp_t *grp, - lck_attr_t *attr); +extern void lck_rw_lock( + lck_rw_t *lck, + lck_rw_type_t lck_rw_type); -extern void lck_rw_lock( - lck_rw_t *lck, - lck_rw_type_t lck_rw_type); +extern void lck_rw_unlock( + lck_rw_t *lck, + lck_rw_type_t lck_rw_type); -extern void lck_rw_unlock( - lck_rw_t *lck, - lck_rw_type_t lck_rw_type); +extern void lck_rw_lock_shared( + lck_rw_t *lck); -extern void lck_rw_lock_shared( - lck_rw_t *lck); +extern void lck_rw_unlock_shared( + lck_rw_t *lck); -extern void lck_rw_unlock_shared( - lck_rw_t *lck); +extern boolean_t lck_rw_lock_yield_shared( + lck_rw_t *lck, + boolean_t force_yield); + +extern void lck_rw_lock_exclusive( + lck_rw_t *lck); +/* + * Grabs the lock exclusive. + * Returns true iff the thread spun or blocked while attempting to + * acquire the lock. + * + * Note that the return value is ONLY A HEURISTIC w.r.t. the lock's + * contention. + * + * This routine IS EXPERIMENTAL. + * It's only used for the vm object lock, and use for other subsystems + * is UNSUPPORTED. + */ +extern bool lck_rw_lock_exclusive_check_contended( + lck_rw_t *lck); + +extern void lck_rw_unlock_exclusive( + lck_rw_t *lck); + +#ifdef XNU_KERNEL_PRIVATE +/* + * CAUTION + * read-write locks do not have a concept of ownership, so lck_rw_assert() + * merely asserts that someone is holding the lock, not necessarily the caller. + */ +extern void lck_rw_assert( + lck_rw_t *lck, + unsigned int type); -extern void lck_rw_lock_exclusive( - lck_rw_t *lck); +extern void lck_rw_clear_promotion(thread_t thread, uintptr_t trace_obj); +extern void lck_rw_set_promotion_locked(thread_t thread); -extern void lck_rw_unlock_exclusive( - lck_rw_t *lck); +uintptr_t unslide_for_kdebug(void* object); +#endif /* XNU_KERNEL_PRIVATE */ -#ifdef KERNEL_PRIVATE +#ifdef KERNEL_PRIVATE -extern lck_rw_type_t lck_rw_done( - lck_rw_t *lck); +extern lck_rw_type_t lck_rw_done( + lck_rw_t *lck); #endif -extern void lck_rw_destroy( - lck_rw_t *lck, - lck_grp_t *grp); +extern void lck_rw_destroy( + lck_rw_t *lck, + lck_grp_t *grp); + +extern void lck_rw_free( + lck_rw_t *lck, + lck_grp_t *grp); -extern void lck_rw_free( - lck_rw_t *lck, - lck_grp_t *grp); +extern wait_result_t lck_rw_sleep( + lck_rw_t *lck, + lck_sleep_action_t lck_sleep_action, + event_t event, + wait_interrupt_t interruptible); -extern wait_result_t lck_rw_sleep( - lck_rw_t *lck, - lck_sleep_action_t lck_sleep_action, - event_t event, - wait_interrupt_t interruptible); +extern wait_result_t lck_rw_sleep_deadline( + lck_rw_t *lck, + lck_sleep_action_t lck_sleep_action, + event_t event, + wait_interrupt_t interruptible, + uint64_t deadline); -extern wait_result_t lck_rw_sleep_deadline( - lck_rw_t *lck, - lck_sleep_action_t lck_sleep_action, - event_t event, - wait_interrupt_t interruptible, - uint64_t deadline); +extern boolean_t lck_rw_lock_shared_to_exclusive( + lck_rw_t *lck); -#ifdef KERNEL_PRIVATE +extern void lck_rw_lock_exclusive_to_shared( + lck_rw_t *lck); -extern boolean_t lck_rw_lock_shared_to_exclusive( - lck_rw_t *lck); +extern boolean_t lck_rw_try_lock( + lck_rw_t *lck, + lck_rw_type_t lck_rw_type); -extern void lck_rw_lock_exclusive_to_shared( - lck_rw_t *lck); +#ifdef KERNEL_PRIVATE -extern boolean_t lck_rw_try_lock( - lck_rw_t *lck, - lck_rw_type_t lck_rw_type); +extern boolean_t lck_rw_try_lock_shared( + lck_rw_t *lck); -extern boolean_t lck_rw_try_lock_shared( - lck_rw_t *lck); +extern boolean_t lck_rw_try_lock_exclusive( + lck_rw_t *lck); -extern boolean_t lck_rw_try_lock_exclusive( - lck_rw_t *lck); #endif +#if XNU_KERNEL_PRIVATE + +struct lck_attr_startup_spec { + lck_attr_t *lck_attr; + uint32_t lck_attr_set_flags; + uint32_t lck_attr_clear_flags; +}; + +struct lck_spin_startup_spec { + lck_spin_t *lck; + lck_grp_t *lck_grp; + lck_attr_t *lck_attr; +}; + +struct lck_mtx_startup_spec { + lck_mtx_t *lck; + struct _lck_mtx_ext_ *lck_ext; + lck_grp_t *lck_grp; + lck_attr_t *lck_attr; +}; + +struct lck_rw_startup_spec { + lck_rw_t *lck; + lck_grp_t *lck_grp; + lck_attr_t *lck_attr; +}; + +extern void lck_attr_startup_init( + struct lck_attr_startup_spec *spec); + +extern void lck_spin_startup_init( + struct lck_spin_startup_spec *spec); + +extern void lck_mtx_startup_init( + struct lck_mtx_startup_spec *spec); + +extern void lck_rw_startup_init( + struct lck_rw_startup_spec *spec); + +/* + * Auto-initializing locks declarations + * ------------------------------------ + * + * Unless you need to configure your locks in very specific ways, + * there is no point creating explicit lock attributes. For most + * static locks, these declaration macros can be used: + * + * - LCK_SPIN_DECLARE for spinlocks, + * - LCK_MTX_EARLY_DECLARE for mutexes initialized before memory + * allocations are possible, + * - LCK_MTX_DECLARE for mutexes, + * - LCK_RW_DECLARE for reader writer locks. + * + * For cases when some particular attributes need to be used, + * these come in *_ATTR variants that take a variable declared with + * LCK_ATTR_DECLARE as an argument. + */ +#define LCK_ATTR_DECLARE(var, set_flags, clear_flags) \ + SECURITY_READ_ONLY_LATE(lck_attr_t) var; \ + static __startup_data struct lck_attr_startup_spec \ + __startup_lck_attr_spec_ ## var = { &var, set_flags, clear_flags }; \ + STARTUP_ARG(LOCKS_EARLY, STARTUP_RANK_SECOND, lck_attr_startup_init, \ + &__startup_lck_attr_spec_ ## var) + +#define LCK_SPIN_DECLARE_ATTR(var, grp, attr) \ + lck_spin_t var; \ + static __startup_data struct lck_spin_startup_spec \ + __startup_lck_spin_spec_ ## var = { &var, grp, attr }; \ + STARTUP_ARG(LOCKS_EARLY, STARTUP_RANK_FOURTH, lck_spin_startup_init, \ + &__startup_lck_spin_spec_ ## var) + +#define LCK_SPIN_DECLARE(var, grp) \ + LCK_SPIN_DECLARE_ATTR(var, grp, LCK_ATTR_NULL) + +#define LCK_MTX_DECLARE_ATTR(var, grp, attr) \ + lck_mtx_t var; \ + static __startup_data struct lck_mtx_startup_spec \ + __startup_lck_mtx_spec_ ## var = { &var, NULL, grp, attr }; \ + STARTUP_ARG(LOCKS, STARTUP_RANK_FIRST, lck_mtx_startup_init, \ + &__startup_lck_mtx_spec_ ## var) + +#define LCK_MTX_DECLARE(var, grp) \ + LCK_MTX_DECLARE_ATTR(var, grp, LCK_ATTR_NULL) + +#define LCK_MTX_EARLY_DECLARE_ATTR(var, grp, attr) \ + lck_mtx_ext_t var ## _ext; \ + lck_mtx_t var; \ + static __startup_data struct lck_mtx_startup_spec \ + __startup_lck_mtx_spec_ ## var = { &var, &var ## _ext, grp, attr }; \ + STARTUP_ARG(LOCKS_EARLY, STARTUP_RANK_FOURTH, lck_mtx_startup_init, \ + &__startup_lck_mtx_spec_ ## var) + +#define LCK_MTX_EARLY_DECLARE(var, grp) \ + LCK_MTX_EARLY_DECLARE_ATTR(var, grp, LCK_ATTR_NULL) + +#define LCK_RW_DECLARE_ATTR(var, grp, attr) \ + lck_rw_t var; \ + static __startup_data struct lck_rw_startup_spec \ + __startup_lck_rw_spec_ ## var = { &var, grp, attr }; \ + STARTUP_ARG(LOCKS_EARLY, STARTUP_RANK_FOURTH, lck_rw_startup_init, \ + &__startup_lck_rw_spec_ ## var) + +#define LCK_RW_DECLARE(var, grp) \ + LCK_RW_DECLARE_ATTR(var, grp, LCK_ATTR_NULL) + +#endif /* XNU_KERNEL_PRIVATE */ __END_DECLS