X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/b0d623f7f2ae71ed96e60569f61f9a9a27016e80..4d15aeb193b2c68f1d38666c317f8d3734f5f083:/osfmk/i386/locks.h diff --git a/osfmk/i386/locks.h b/osfmk/i386/locks.h index d74e94156..053de1577 100644 --- a/osfmk/i386/locks.h +++ b/osfmk/i386/locks.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004-2007 Apple Inc. All rights reserved. + * Copyright (c) 2004-2012 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -40,90 +40,84 @@ extern unsigned int LcksOpts; #define enaLkDeb 0x00000001 /* Request debug in default attribute */ #define enaLkStat 0x00000002 /* Request statistic in default attribute */ +#define disLkRWPrio 0x00000004 /* Disable RW lock priority promotion */ -#endif +#endif /* MACH_KERNEL_PRIVATE */ -#ifdef MACH_KERNEL_PRIVATE +#if defined(MACH_KERNEL_PRIVATE) typedef struct { - unsigned long interlock; - unsigned long lck_spin_pad[9]; /* XXX - usimple_lock_data_t */ + volatile uintptr_t interlock; +#if MACH_LDEBUG + unsigned long lck_spin_pad[9]; /* XXX - usimple_lock_data_t */ +#endif } lck_spin_t; #define LCK_SPIN_TAG_DESTROYED 0x00002007 /* lock marked as Destroyed */ -#else +#else /* MACH_KERNEL_PRIVATE */ #ifdef KERNEL_PRIVATE typedef struct { unsigned long opaque[10]; } lck_spin_t; -#else +#else /* KERNEL_PRIVATE */ typedef struct __lck_spin_t__ lck_spin_t; #endif #endif #ifdef MACH_KERNEL_PRIVATE +/* The definition of this structure, including the layout of the + * state bitfield, is tailored to the asm implementation in i386_lock.s + */ typedef struct _lck_mtx_ { union { struct { - volatile uintptr_t lck_mtxd_owner; - unsigned long lck_mtxd_ptr; - volatile uint32_t lck_mtxd_waiters:16, - lck_mtxd_pri:8, - lck_mtxd_ilocked:1, - lck_mtxd_mlocked:1, - lck_mtxd_promoted:1, - lck_mtxd_spin:1, - lck_mtxd_pad4:4; /* padding */ -#ifdef __x86_64__ - unsigned int lck_mtxd_pad; -#endif - } lck_mtxd; + volatile uintptr_t lck_mtx_owner; + union { + struct { + volatile uint32_t + lck_mtx_waiters:16, + lck_mtx_pri:8, + lck_mtx_ilocked:1, + lck_mtx_mlocked:1, + lck_mtx_promoted:1, + lck_mtx_spin:1, + lck_mtx_is_ext:1, + lck_mtx_pad3:3; + }; + uint32_t lck_mtx_state; + }; + /* Pad field used as a canary, initialized to ~0 */ + uint32_t lck_mtx_pad32; + }; struct { - unsigned long lck_mtxi_tag; - struct _lck_mtx_ext_ *lck_mtxi_ptr; - unsigned long lck_mtxi_pad; - } lck_mtxi; - } lck_mtx_sw; + struct _lck_mtx_ext_ *lck_mtx_ptr; + uint32_t lck_mtx_tag; + uint32_t lck_mtx_pad32_2; + }; + }; } lck_mtx_t; -#define lck_mtx_owner lck_mtx_sw.lck_mtxd.lck_mtxd_owner -#define lck_mtx_waiters lck_mtx_sw.lck_mtxd.lck_mtxd_waiters -#define lck_mtx_pri lck_mtx_sw.lck_mtxd.lck_mtxd_pri -#define lck_mtx_ilocked lck_mtx_sw.lck_mtxd.lck_mtxd_ilocked -#define lck_mtx_mlocked lck_mtx_sw.lck_mtxd.lck_mtxd_mlocked -#define lck_mtx_promoted lck_mtx_sw.lck_mtxd.lck_mtxd_promoted -#define lck_mtx_spin lck_mtx_sw.lck_mtxd.lck_mtxd_spin - -#define lck_mtx_tag lck_mtx_sw.lck_mtxi.lck_mtxi_tag -#define lck_mtx_ptr lck_mtx_sw.lck_mtxi.lck_mtxi_ptr -#define lck_mtx_state lck_mtx_sw.lck_mtxi.lck_mtxi_pad - -#define LCK_MTX_TAG_INDIRECT 0x00001007 /* lock marked as Indirect */ -#define LCK_MTX_TAG_DESTROYED 0x00002007 /* lock marked as Destroyed */ -#define LCK_MTX_PTR_EXTENDED 0x00003007 /* lock is extended version */ +/* This pattern must subsume the interlocked, mlocked and spin bits */ +#define LCK_MTX_TAG_INDIRECT 0x07ff1007 /* lock marked as Indirect */ +#define LCK_MTX_TAG_DESTROYED 0x07fe2007 /* lock marked as Destroyed */ /* Adaptive spin before blocking */ -extern unsigned int MutexSpin; +extern uint64_t MutexSpin; extern int lck_mtx_lock_spinwait_x86(lck_mtx_t *mutex); extern void lck_mtx_lock_wait_x86(lck_mtx_t *mutex); extern void lck_mtx_lock_acquire_x86(lck_mtx_t *mutex); -extern void lck_mtx_unlock_wakeup_x86(lck_mtx_t *mutex, int owner_was_promoted); +extern void lck_mtx_unlock_wakeup_x86(lck_mtx_t *mutex, int prior_lock_state); extern void lck_mtx_lock_mark_destroyed(lck_mtx_t *mutex); -extern int lck_mtx_lock_mark_promoted(lck_mtx_t *mutex); -extern int lck_mtx_lock_decr_waiter(lck_mtx_t *mutex); extern int lck_mtx_lock_grab_mutex(lck_mtx_t *mutex); -extern integer_t lck_mtx_lock_get_pri(lck_mtx_t *mutex); -extern void hw_lock_byte_init(uint8_t *lock_byte); -extern void hw_lock_byte_lock(uint8_t *lock_byte); -extern void hw_lock_byte_unlock(uint8_t *lock_byte); +extern void hw_lock_byte_init(volatile uint8_t *lock_byte); +extern void hw_lock_byte_lock(volatile uint8_t *lock_byte); +extern void hw_lock_byte_unlock(volatile uint8_t *lock_byte); typedef struct { unsigned int type; -#ifdef __x86_64__ unsigned int pad4; -#endif vm_offset_t pc; vm_offset_t thread; } lck_mtx_deb_t; @@ -138,14 +132,10 @@ typedef struct _lck_mtx_ext_ { lck_mtx_t lck_mtx; struct _lck_grp_ *lck_mtx_grp; unsigned int lck_mtx_attr; -#ifdef __x86_64__ unsigned int lck_mtx_pad1; -#endif lck_mtx_deb_t lck_mtx_deb; uint64_t lck_mtx_stat; -#ifdef __x86_64__ unsigned int lck_mtx_pad2[2]; -#endif } lck_mtx_ext_t; #define LCK_MTX_ATTR_DEBUG 0x1 @@ -153,10 +143,22 @@ typedef struct _lck_mtx_ext_ { #define LCK_MTX_ATTR_STAT 0x2 #define LCK_MTX_ATTR_STATb 1 +#define LCK_MTX_EVENT(lck) ((event_t)(((unsigned int*)(lck))+(sizeof(lck_mtx_t)-1)/sizeof(unsigned int))) +#define LCK_EVENT_TO_MUTEX(event) ((lck_mtx_t *)(uintptr_t)(((unsigned int *)(event)) - ((sizeof(lck_mtx_t)-1)/sizeof(unsigned int)))) + +#else /* MACH_KERNEL_PRIVATE */ +#ifdef XNU_KERNEL_PRIVATE +typedef struct { + unsigned long opaque[2]; +} lck_mtx_t; + +typedef struct { + unsigned long opaque[10]; +} lck_mtx_ext_t; #else #ifdef KERNEL_PRIVATE typedef struct { - unsigned long opaque[3]; + unsigned long opaque[2]; } lck_mtx_t; typedef struct { @@ -168,12 +170,13 @@ typedef struct __lck_mtx_t__ lck_mtx_t; typedef struct __lck_mtx_ext_t__ lck_mtx_ext_t; #endif #endif +#endif #ifdef MACH_KERNEL_PRIVATE #pragma pack(1) /* Make sure the structure stays as we defined it */ typedef struct _lck_rw_t_internal_ { volatile uint16_t lck_rw_shared_count; /* No. of accepted readers */ - uint8_t lck_rw_interlock; /* Interlock byte */ + volatile uint8_t lck_rw_interlock; /* Interlock byte */ volatile uint8_t lck_rw_priv_excl:1, /* Writers prioritized if set */ lck_rw_want_upgrade:1, /* Read-to-write upgrade waiting */ @@ -187,9 +190,7 @@ typedef struct _lck_rw_t_internal_ { * are in */ uint32_t lck_rw_pad8; -#ifdef __x86_64__ uint32_t lck_rw_pad12; -#endif } lck_rw_t; #pragma pack() @@ -206,14 +207,32 @@ typedef struct _lck_rw_t_internal_ { #define LCK_RW_TAG_DESTROYED 0x00002007 /* lock marked as Destroyed */ +#define RW_LOCK_READER_EVENT(x) ((event_t) (((unsigned char*) (x)) + (offsetof(lck_rw_t, lck_rw_tag)))) +#define RW_LOCK_WRITER_EVENT(x) ((event_t) (((unsigned char*) (x)) + (offsetof(lck_rw_t, lck_rw_pad8)))) +#define READ_EVENT_TO_RWLOCK(x) ((lck_rw_t *)(((unsigned char*)(x) - (offsetof(lck_rw_t, lck_rw_tag))))) +#define WRITE_EVENT_TO_RWLOCK(x) ((lck_rw_t *)(((unsigned char*)(x) - (offsetof(lck_rw_t, lck_rw_pad8))))) + +#if LOCK_PRIVATE + +#define disable_preemption_for_thread(t) ((cpu_data_t GS_RELATIVE *)0UL)->cpu_preemption_level++ + +#define LCK_MTX_THREAD_TO_STATE(t) ((uintptr_t)t) +#define PLATFORM_LCK_ILOCK 0 + +#define LOCK_SNOOP_SPINS 1000 +#define LOCK_PRETEST 1 + +/* Spinlock panic deadline, in mach_absolute_time units (ns on i386) */ +#define LOCK_PANIC_TIMEOUT 0xf00000 /* 250 ms (huge) */ + +#endif // LOCK_PRIVATE + #else #ifdef KERNEL_PRIVATE #pragma pack(1) typedef struct { uint32_t opaque[3]; -#ifdef __x86_64__ uint32_t opaque4; -#endif } lck_rw_t; #pragma pack() #else @@ -221,4 +240,10 @@ typedef struct __lck_rw_t__ lck_rw_t; #endif #endif +#ifdef MACH_KERNEL_PRIVATE + +extern void kernel_preempt_check (void); + +#endif /* MACH_KERNEL_PRIVATE */ + #endif /* _I386_LOCKS_H_ */