/*
- * Copyright (c) 2003-2005 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2003-2012 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
#define LCK_TYPE_SPIN 1
#define LCK_TYPE_MTX 2
-#define LCK_TYPE_RW 3
+#define LCK_TYPE_RW 3
#endif
#define LCK_SLEEP_UNLOCK 0x01 /* Release the lock and return unheld */
#define LCK_SLEEP_SHARED 0x02 /* Reclaim the lock in shared mode (RW only) */
#define LCK_SLEEP_EXCLUSIVE 0x04 /* Reclaim the lock in exclusive mode (RW only) */
+#define LCK_SLEEP_SPIN 0x08 /* Reclaim the lock in spin mode (mutex only) */
+#define LCK_SLEEP_PROMOTED_PRI 0x10 /* Sleep at a promoted priority */
+#define LCK_SLEEP_SPIN_ALWAYS 0x20 /* Reclaim the lock in spin-always mode (mutex only) */
-#define LCK_SLEEP_MASK 0x07 /* Valid actions */
+#define LCK_SLEEP_MASK 0x3f /* Valid actions */
#ifdef MACH_KERNEL_PRIVATE
typedef struct {
uint64_t lck_grp_mtx_util_cnt;
+ /* On x86, this is used as the "direct wait" count */
uint64_t lck_grp_mtx_held_cnt;
uint64_t lck_grp_mtx_miss_cnt;
uint64_t lck_grp_mtx_wait_cnt;
+ /* Rest currently unused */
uint64_t lck_grp_mtx_held_max;
uint64_t lck_grp_mtx_held_cum;
uint64_t lck_grp_mtx_wait_max;
typedef struct _lck_grp_ {
queue_chain_t lck_grp_link;
- unsigned int lck_grp_refcnt;
- unsigned int lck_grp_spincnt;
- unsigned int lck_grp_mtxcnt;
- unsigned int lck_grp_rwcnt;
- unsigned int lck_grp_attr;
- char lck_grp_name[LCK_GRP_MAX_NAME];
+ uint32_t lck_grp_refcnt;
+ uint32_t lck_grp_spincnt;
+ uint32_t lck_grp_mtxcnt;
+ uint32_t lck_grp_rwcnt;
+ uint32_t lck_grp_attr;
+ char lck_grp_name[LCK_GRP_MAX_NAME];
lck_grp_stat_t lck_grp_stat;
} lck_grp_t;
+#define lck_grp_miss lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_miss_cnt
+#define lck_grp_held lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_held_cnt
+#define lck_grp_util lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_util_cnt
+#define lck_grp_wait lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_wait_cnt
+#define lck_grp_direct_wait lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_held_cnt
+
#define LCK_GRP_NULL (lck_grp_t *)0
#else
#ifdef MACH_KERNEL_PRIVATE
typedef struct _lck_grp_attr_ {
- unsigned int grp_attr_val;
+ uint32_t grp_attr_val;
} lck_grp_attr_t;
extern lck_grp_attr_t LockDefaultGroupAttr;
extern lck_attr_t LockDefaultLckAttr;
-#define LCK_ATTR_NONE 0
-#define LCK_ATTR_DEBUG 0x1
+#define LCK_ATTR_NONE 0
+
+#define LCK_ATTR_DEBUG 0x00000001
+#define LCK_ATTR_RW_SHARED_PRIORITY 0x00010000
#else
typedef struct __lck_attr__ lck_attr_t;
extern void lck_attr_setdebug(
lck_attr_t *attr);
+extern void lck_attr_cleardebug(
+ lck_attr_t *attr);
+
+#ifdef XNU_KERNEL_PRIVATE
+extern void lck_attr_rw_shared_priority(
+ lck_attr_t *attr);
+#endif
+
extern void lck_attr_free(
lck_attr_t *attr);
#ifdef KERNEL_PRIVATE
-extern boolean_t lck_spin_try_lock(
- lck_spin_t *lck);
+extern void lck_spin_lock_nopreempt( lck_spin_t *lck);
+
+extern void lck_spin_unlock_nopreempt( lck_spin_t *lck);
+
+extern boolean_t lck_spin_try_lock( lck_spin_t *lck);
+
+extern boolean_t lck_spin_try_lock_nopreempt( lck_spin_t *lck);
+
+/* NOT SAFE: To be used only by kernel debugger to avoid deadlock. */
+extern boolean_t kdp_lck_spin_is_acquired( lck_spin_t *lck);
+
+struct _lck_mtx_ext_;
+extern void lck_mtx_init_ext(lck_mtx_t *lck, struct _lck_mtx_ext_ *lck_ext,
+ lck_grp_t *grp, lck_attr_t *attr);
#endif
lck_mtx_t *lck,
lck_grp_t *grp,
lck_attr_t *attr);
-
extern void lck_mtx_lock(
lck_mtx_t *lck);
event_t event,
wait_interrupt_t interruptible,
uint64_t deadline);
+#if DEVELOPMENT || DEBUG
+extern void erase_all_test_mtx_stats(void);
+extern int get_test_mtx_stats_string(char* buffer, int buffer_size);
+extern void lck_mtx_test_init(void);
+extern void lck_mtx_test_lock(void);
+extern void lck_mtx_test_unlock(void);
+extern int lck_mtx_test_mtx_uncontended(int iter, char* buffer, int buffer_size);
+extern int lck_mtx_test_mtx_contended(int iter, char* buffer, int buffer_size);
+extern int lck_mtx_test_mtx_uncontended_loop_time(int iter, char* buffer, int buffer_size);
+extern int lck_mtx_test_mtx_contended_loop_time(int iter, char* buffer, int buffer_size);
+#endif
#ifdef KERNEL_PRIVATE
extern boolean_t lck_mtx_try_lock(
lck_mtx_t *lck);
+extern void mutex_pause(uint32_t);
+
+extern void lck_mtx_yield (
+ lck_mtx_t *lck);
+
+extern boolean_t lck_mtx_try_lock_spin(
+ lck_mtx_t *lck);
+
+extern void lck_mtx_lock_spin(
+ lck_mtx_t *lck);
+
+extern boolean_t kdp_lck_mtx_lock_spin_is_acquired(
+ lck_mtx_t *lck);
+
+extern void lck_mtx_convert_spin(
+ lck_mtx_t *lck);
+
+extern void lck_mtx_lock_spin_always(
+ lck_mtx_t *lck);
+
+extern boolean_t lck_mtx_try_lock_spin_always(
+ lck_mtx_t *lck);
+
+#define lck_mtx_unlock_always(l) lck_mtx_unlock(l)
+
+extern void lck_spin_assert(
+ lck_spin_t *lck,
+ unsigned int type);
+
+extern boolean_t kdp_lck_rw_lock_is_acquired_exclusive(
+ lck_rw_t *lck);
+
#endif /* KERNEL_PRIVATE */
extern void lck_mtx_assert(
lck_mtx_t *lck,
unsigned int type);
+#if MACH_ASSERT
+#define LCK_MTX_ASSERT(lck,type) lck_mtx_assert((lck),(type))
+#define LCK_SPIN_ASSERT(lck,type) lck_spin_assert((lck),(type))
+#define LCK_RW_ASSERT(lck,type) lck_rw_assert((lck),(type))
+#else /* MACH_ASSERT */
+#define LCK_MTX_ASSERT(lck,type)
+#define LCK_SPIN_ASSERT(lck,type)
+#define LCK_RW_ASSERT(lck,type)
+#endif /* MACH_ASSERT */
+
+#if DEBUG
+#define LCK_MTX_ASSERT_DEBUG(lck,type) lck_mtx_assert((lck),(type))
+#define LCK_SPIN_ASSERT_DEBUG(lck,type) lck_spin_assert((lck),(type))
+#define LCK_RW_ASSERT_DEBUG(lck,type) lck_rw_assert((lck),(type))
+#else /* DEBUG */
+#define LCK_MTX_ASSERT_DEBUG(lck,type)
+#define LCK_SPIN_ASSERT_DEBUG(lck,type)
+#define LCK_RW_ASSERT_DEBUG(lck,type)
+#endif /* DEBUG */
+
__END_DECLS
-#define LCK_MTX_ASSERT_OWNED 0x01
-#define LCK_MTX_ASSERT_NOTOWNED 0x02
+#define LCK_ASSERT_OWNED 1
+#define LCK_ASSERT_NOTOWNED 2
+
+#define LCK_MTX_ASSERT_OWNED LCK_ASSERT_OWNED
+#define LCK_MTX_ASSERT_NOTOWNED LCK_ASSERT_NOTOWNED
#ifdef MACH_KERNEL_PRIVATE
extern void lck_mtx_lock_wait(
extern boolean_t lck_mtx_ilk_unlock(
lck_mtx_t *lck);
+
+extern boolean_t lck_mtx_ilk_try_lock(
+ lck_mtx_t *lck);
+
+extern void lck_mtx_wakeup_adjust_pri(thread_t thread, integer_t priority);
+
#endif
#define decl_lck_rw_data(class,name) class lck_rw_t name;
#define LCK_RW_TYPE_SHARED 0x01
#define LCK_RW_TYPE_EXCLUSIVE 0x02
+#ifdef XNU_KERNEL_PRIVATE
+#define LCK_RW_ASSERT_SHARED 0x01
+#define LCK_RW_ASSERT_EXCLUSIVE 0x02
+#define LCK_RW_ASSERT_HELD 0x03
+#define LCK_RW_ASSERT_NOTHELD 0x04
+#endif
+
__BEGIN_DECLS
extern lck_rw_t *lck_rw_alloc_init(
extern void lck_rw_unlock_shared(
lck_rw_t *lck);
+extern boolean_t lck_rw_lock_yield_shared(
+ lck_rw_t *lck,
+ boolean_t force_yield);
+
extern void lck_rw_lock_exclusive(
lck_rw_t *lck);
extern void lck_rw_unlock_exclusive(
lck_rw_t *lck);
+#ifdef XNU_KERNEL_PRIVATE
+/*
+ * CAUTION
+ * read-write locks do not have a concept of ownership, so lck_rw_assert()
+ * merely asserts that someone is holding the lock, not necessarily the caller.
+ */
+extern void lck_rw_assert(
+ lck_rw_t *lck,
+ unsigned int type);
+
+extern void lck_rw_clear_promotion(thread_t thread, uintptr_t trace_obj);
+extern void lck_rw_set_promotion_locked(thread_t thread);
+
+uintptr_t unslide_for_kdebug(void* object);
+#endif
+
#ifdef KERNEL_PRIVATE
extern lck_rw_type_t lck_rw_done(
wait_interrupt_t interruptible,
uint64_t deadline);
-#ifdef KERNEL_PRIVATE
-
extern boolean_t lck_rw_lock_shared_to_exclusive(
lck_rw_t *lck);
lck_rw_t *lck,
lck_rw_type_t lck_rw_type);
+#ifdef KERNEL_PRIVATE
+
extern boolean_t lck_rw_try_lock_shared(
lck_rw_t *lck);
projects / apple / xnu.git / blobdiff