X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/8ad349bb6ed4a0be06e34c92be0d98b92e078db4..b226f5e54a60dc81db17b1260381d7dbfea3cdf1:/osfmk/kern/locks.h diff --git a/osfmk/kern/locks.h b/osfmk/kern/locks.h index cd4e9a178..4db3c40f5 100644 --- a/osfmk/kern/locks.h +++ b/osfmk/kern/locks.h @@ -1,31 +1,29 @@ /* - * Copyright (c) 2003-2005 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2003-2012 Apple Inc. All rights reserved. * - * @APPLE_LICENSE_OSREFERENCE_HEADER_START@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the - * License may not be used to create, or enable the creation or - * redistribution of, unlawful or unlicensed copies of an Apple operating - * system, or to circumvent, violate, or enable the circumvention or - * violation of, any terms of an Apple operating system software license - * agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this - * file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and * limitations under the License. - * - * @APPLE_LICENSE_OSREFERENCE_HEADER_END@ + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #ifndef _KERN_LOCKS_H_ @@ -48,7 +46,7 @@ typedef unsigned int lck_type_t; #define LCK_TYPE_SPIN 1 #define LCK_TYPE_MTX 2 -#define LCK_TYPE_RW 3 +#define LCK_TYPE_RW 3 #endif @@ -59,8 +57,11 @@ typedef unsigned int lck_sleep_action_t; #define LCK_SLEEP_UNLOCK 0x01 /* Release the lock and return unheld */ #define LCK_SLEEP_SHARED 0x02 /* Reclaim the lock in shared mode (RW only) */ #define LCK_SLEEP_EXCLUSIVE 0x04 /* Reclaim the lock in exclusive mode (RW only) */ +#define LCK_SLEEP_SPIN 0x08 /* Reclaim the lock in spin mode (mutex only) */ +#define LCK_SLEEP_PROMOTED_PRI 0x10 /* Sleep at a promoted priority */ +#define LCK_SLEEP_SPIN_ALWAYS 0x20 /* Reclaim the lock in spin-always mode (mutex only) */ -#define LCK_SLEEP_MASK 0x07 /* Valid actions */ +#define LCK_SLEEP_MASK 0x3f /* Valid actions */ #ifdef MACH_KERNEL_PRIVATE @@ -74,9 +75,11 @@ typedef struct { typedef struct { uint64_t lck_grp_mtx_util_cnt; + /* On x86, this is used as the "direct wait" count */ uint64_t lck_grp_mtx_held_cnt; uint64_t lck_grp_mtx_miss_cnt; uint64_t lck_grp_mtx_wait_cnt; + /* Rest currently unused */ uint64_t lck_grp_mtx_held_max; uint64_t lck_grp_mtx_held_cum; uint64_t lck_grp_mtx_wait_max; @@ -104,15 +107,21 @@ typedef struct _lck_grp_stat_ { typedef struct _lck_grp_ { queue_chain_t lck_grp_link; - unsigned int lck_grp_refcnt; - unsigned int lck_grp_spincnt; - unsigned int lck_grp_mtxcnt; - unsigned int lck_grp_rwcnt; - unsigned int lck_grp_attr; - char lck_grp_name[LCK_GRP_MAX_NAME]; + uint32_t lck_grp_refcnt; + uint32_t lck_grp_spincnt; + uint32_t lck_grp_mtxcnt; + uint32_t lck_grp_rwcnt; + uint32_t lck_grp_attr; + char lck_grp_name[LCK_GRP_MAX_NAME]; lck_grp_stat_t lck_grp_stat; } lck_grp_t; +#define lck_grp_miss lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_miss_cnt +#define lck_grp_held lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_held_cnt +#define lck_grp_util lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_util_cnt +#define lck_grp_wait lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_wait_cnt +#define lck_grp_direct_wait lck_grp_stat.lck_grp_mtx_stat.lck_grp_mtx_held_cnt + #define LCK_GRP_NULL (lck_grp_t *)0 #else @@ -121,7 +130,7 @@ typedef struct __lck_grp__ lck_grp_t; #ifdef MACH_KERNEL_PRIVATE typedef struct _lck_grp_attr_ { - unsigned int grp_attr_val; + uint32_t grp_attr_val; } lck_grp_attr_t; extern lck_grp_attr_t LockDefaultGroupAttr; @@ -189,8 +198,10 @@ typedef struct _lck_attr_ { extern lck_attr_t LockDefaultLckAttr; -#define LCK_ATTR_NONE 0 -#define LCK_ATTR_DEBUG 0x1 +#define LCK_ATTR_NONE 0 + +#define LCK_ATTR_DEBUG 0x00000001 +#define LCK_ATTR_RW_SHARED_PRIORITY 0x00010000 #else typedef struct __lck_attr__ lck_attr_t; @@ -209,6 +220,14 @@ extern void lck_attr_setdefault( extern void lck_attr_setdebug( lck_attr_t *attr); +extern void lck_attr_cleardebug( + lck_attr_t *attr); + +#ifdef XNU_KERNEL_PRIVATE +extern void lck_attr_rw_shared_priority( + lck_attr_t *attr); +#endif + extern void lck_attr_free( lck_attr_t *attr); @@ -252,8 +271,20 @@ extern wait_result_t lck_spin_sleep_deadline( #ifdef KERNEL_PRIVATE -extern boolean_t lck_spin_try_lock( - lck_spin_t *lck); +extern void lck_spin_lock_nopreempt( lck_spin_t *lck); + +extern void lck_spin_unlock_nopreempt( lck_spin_t *lck); + +extern boolean_t lck_spin_try_lock( lck_spin_t *lck); + +extern boolean_t lck_spin_try_lock_nopreempt( lck_spin_t *lck); + +/* NOT SAFE: To be used only by kernel debugger to avoid deadlock. */ +extern boolean_t kdp_lck_spin_is_acquired( lck_spin_t *lck); + +struct _lck_mtx_ext_; +extern void lck_mtx_init_ext(lck_mtx_t *lck, struct _lck_mtx_ext_ *lck_ext, + lck_grp_t *grp, lck_attr_t *attr); #endif @@ -268,7 +299,6 @@ extern void lck_mtx_init( lck_mtx_t *lck, lck_grp_t *grp, lck_attr_t *attr); - extern void lck_mtx_lock( lck_mtx_t *lck); @@ -295,22 +325,88 @@ extern wait_result_t lck_mtx_sleep_deadline( event_t event, wait_interrupt_t interruptible, uint64_t deadline); +#if DEVELOPMENT || DEBUG +extern void erase_all_test_mtx_stats(void); +extern int get_test_mtx_stats_string(char* buffer, int buffer_size); +extern void lck_mtx_test_init(void); +extern void lck_mtx_test_lock(void); +extern void lck_mtx_test_unlock(void); +extern int lck_mtx_test_mtx_uncontended(int iter, char* buffer, int buffer_size); +extern int lck_mtx_test_mtx_contended(int iter, char* buffer, int buffer_size); +extern int lck_mtx_test_mtx_uncontended_loop_time(int iter, char* buffer, int buffer_size); +extern int lck_mtx_test_mtx_contended_loop_time(int iter, char* buffer, int buffer_size); +#endif #ifdef KERNEL_PRIVATE extern boolean_t lck_mtx_try_lock( lck_mtx_t *lck); +extern void mutex_pause(uint32_t); + +extern void lck_mtx_yield ( + lck_mtx_t *lck); + +extern boolean_t lck_mtx_try_lock_spin( + lck_mtx_t *lck); + +extern void lck_mtx_lock_spin( + lck_mtx_t *lck); + +extern boolean_t kdp_lck_mtx_lock_spin_is_acquired( + lck_mtx_t *lck); + +extern void lck_mtx_convert_spin( + lck_mtx_t *lck); + +extern void lck_mtx_lock_spin_always( + lck_mtx_t *lck); + +extern boolean_t lck_mtx_try_lock_spin_always( + lck_mtx_t *lck); + +#define lck_mtx_unlock_always(l) lck_mtx_unlock(l) + +extern void lck_spin_assert( + lck_spin_t *lck, + unsigned int type); + +extern boolean_t kdp_lck_rw_lock_is_acquired_exclusive( + lck_rw_t *lck); + #endif /* KERNEL_PRIVATE */ extern void lck_mtx_assert( lck_mtx_t *lck, unsigned int type); +#if MACH_ASSERT +#define LCK_MTX_ASSERT(lck,type) lck_mtx_assert((lck),(type)) +#define LCK_SPIN_ASSERT(lck,type) lck_spin_assert((lck),(type)) +#define LCK_RW_ASSERT(lck,type) lck_rw_assert((lck),(type)) +#else /* MACH_ASSERT */ +#define LCK_MTX_ASSERT(lck,type) +#define LCK_SPIN_ASSERT(lck,type) +#define LCK_RW_ASSERT(lck,type) +#endif /* MACH_ASSERT */ + +#if DEBUG +#define LCK_MTX_ASSERT_DEBUG(lck,type) lck_mtx_assert((lck),(type)) +#define LCK_SPIN_ASSERT_DEBUG(lck,type) lck_spin_assert((lck),(type)) +#define LCK_RW_ASSERT_DEBUG(lck,type) lck_rw_assert((lck),(type)) +#else /* DEBUG */ +#define LCK_MTX_ASSERT_DEBUG(lck,type) +#define LCK_SPIN_ASSERT_DEBUG(lck,type) +#define LCK_RW_ASSERT_DEBUG(lck,type) +#endif /* DEBUG */ + __END_DECLS -#define LCK_MTX_ASSERT_OWNED 0x01 -#define LCK_MTX_ASSERT_NOTOWNED 0x02 +#define LCK_ASSERT_OWNED 1 +#define LCK_ASSERT_NOTOWNED 2 + +#define LCK_MTX_ASSERT_OWNED LCK_ASSERT_OWNED +#define LCK_MTX_ASSERT_NOTOWNED LCK_ASSERT_NOTOWNED #ifdef MACH_KERNEL_PRIVATE extern void lck_mtx_lock_wait( @@ -326,6 +422,12 @@ extern void lck_mtx_unlock_wakeup( extern boolean_t lck_mtx_ilk_unlock( lck_mtx_t *lck); + +extern boolean_t lck_mtx_ilk_try_lock( + lck_mtx_t *lck); + +extern void lck_mtx_wakeup_adjust_pri(thread_t thread, integer_t priority); + #endif #define decl_lck_rw_data(class,name) class lck_rw_t name; @@ -335,6 +437,13 @@ typedef unsigned int lck_rw_type_t; #define LCK_RW_TYPE_SHARED 0x01 #define LCK_RW_TYPE_EXCLUSIVE 0x02 +#ifdef XNU_KERNEL_PRIVATE +#define LCK_RW_ASSERT_SHARED 0x01 +#define LCK_RW_ASSERT_EXCLUSIVE 0x02 +#define LCK_RW_ASSERT_HELD 0x03 +#define LCK_RW_ASSERT_NOTHELD 0x04 +#endif + __BEGIN_DECLS extern lck_rw_t *lck_rw_alloc_init( @@ -360,12 +469,32 @@ extern void lck_rw_lock_shared( extern void lck_rw_unlock_shared( lck_rw_t *lck); +extern boolean_t lck_rw_lock_yield_shared( + lck_rw_t *lck, + boolean_t force_yield); + extern void lck_rw_lock_exclusive( lck_rw_t *lck); extern void lck_rw_unlock_exclusive( lck_rw_t *lck); +#ifdef XNU_KERNEL_PRIVATE +/* + * CAUTION + * read-write locks do not have a concept of ownership, so lck_rw_assert() + * merely asserts that someone is holding the lock, not necessarily the caller. + */ +extern void lck_rw_assert( + lck_rw_t *lck, + unsigned int type); + +extern void lck_rw_clear_promotion(thread_t thread, uintptr_t trace_obj); +extern void lck_rw_set_promotion_locked(thread_t thread); + +uintptr_t unslide_for_kdebug(void* object); +#endif + #ifdef KERNEL_PRIVATE extern lck_rw_type_t lck_rw_done( @@ -393,8 +522,6 @@ extern wait_result_t lck_rw_sleep_deadline( wait_interrupt_t interruptible, uint64_t deadline); -#ifdef KERNEL_PRIVATE - extern boolean_t lck_rw_lock_shared_to_exclusive( lck_rw_t *lck); @@ -405,6 +532,8 @@ extern boolean_t lck_rw_try_lock( lck_rw_t *lck, lck_rw_type_t lck_rw_type); +#ifdef KERNEL_PRIVATE + extern boolean_t lck_rw_try_lock_shared( lck_rw_t *lck);