X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/4452a7af2eac33dbad800bcc91f2399d62c18f53..99c3a10404e5d1ef94397ab4df5a8b74711fc4d3:/osfmk/i386/locks.h diff --git a/osfmk/i386/locks.h b/osfmk/i386/locks.h index 3129d79bd..065dfecea 100644 --- a/osfmk/i386/locks.h +++ b/osfmk/i386/locks.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2004-2007 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -41,61 +41,97 @@ extern unsigned int LcksOpts; #define enaLkDeb 0x00000001 /* Request debug in default attribute */ #define enaLkStat 0x00000002 /* Request statistic in default attribute */ -#endif +#endif /* MACH_KERNEL_PRIVATE */ -#ifdef MACH_KERNEL_PRIVATE +#if defined(MACH_KERNEL_PRIVATE) typedef struct { - unsigned int lck_spin_data[10]; /* XXX - usimple_lock_data_t */ + volatile uintptr_t interlock; +#if MACH_LDEBUG + unsigned long lck_spin_pad[9]; /* XXX - usimple_lock_data_t */ +#endif } lck_spin_t; #define LCK_SPIN_TAG_DESTROYED 0x00002007 /* lock marked as Destroyed */ -#else +#else /* MACH_KERNEL_PRIVATE */ #ifdef KERNEL_PRIVATE typedef struct { - unsigned int opaque[10]; + unsigned long opaque[10]; } lck_spin_t; -#else +#else /* KERNEL_PRIVATE */ typedef struct __lck_spin_t__ lck_spin_t; #endif #endif #ifdef MACH_KERNEL_PRIVATE +/* The definition of this structure, including the layout of the + * state bitfield, is tailored to the asm implementation in i386_lock.s + */ typedef struct _lck_mtx_ { union { struct { - unsigned int lck_mtxd_ilk; - unsigned int lck_mtxd_locked; - unsigned short lck_mtxd_waiters; - unsigned short lck_mtxd_pri; + volatile uintptr_t lck_mtxd_owner; + union { + struct { + volatile uint32_t + lck_mtxd_waiters:16, + lck_mtxd_pri:8, + lck_mtxd_ilocked:1, + lck_mtxd_mlocked:1, + lck_mtxd_promoted:1, + lck_mtxd_spin:1, + lck_mtxd_is_ext:1, + lck_mtxd_pad3:3; + }; + uint32_t lck_mtxd_state; + }; +#if defined(__x86_64__) + /* Pad field used as a canary, initialized to ~0 */ + uint32_t lck_mtxd_pad32; +#endif } lck_mtxd; struct { - unsigned int lck_mtxi_tag; - struct _lck_mtx_ext_ *lck_mtxi_ptr; - unsigned int lck_mtxi_pad8; + struct _lck_mtx_ext_ *lck_mtxi_ptr; + uint32_t lck_mtxi_tag; +#if defined(__x86_64__) + uint32_t lck_mtxi_pad32; +#endif } lck_mtxi; } lck_mtx_sw; } lck_mtx_t; -#define lck_mtx_ilk lck_mtx_sw.lck_mtxd.lck_mtxd_ilk -#define lck_mtx_locked lck_mtx_sw.lck_mtxd.lck_mtxd_locked +#define lck_mtx_owner lck_mtx_sw.lck_mtxd.lck_mtxd_owner #define lck_mtx_waiters lck_mtx_sw.lck_mtxd.lck_mtxd_waiters #define lck_mtx_pri lck_mtx_sw.lck_mtxd.lck_mtxd_pri +#define lck_mtx_promoted lck_mtx_sw.lck_mtxd.lck_mtxd_promoted +#define lck_mtx_is_ext lck_mtx_sw.lck_mtxd.lck_mtxd_is_ext #define lck_mtx_tag lck_mtx_sw.lck_mtxi.lck_mtxi_tag #define lck_mtx_ptr lck_mtx_sw.lck_mtxi.lck_mtxi_ptr - -#define LCK_MTX_TAG_INDIRECT 0x00001007 /* lock marked as Indirect */ -#define LCK_MTX_TAG_DESTROYED 0x00002007 /* lock marked as Destroyed */ +#define lck_mtx_state lck_mtx_sw.lck_mtxd.lck_mtxd_state +/* This pattern must subsume the interlocked, mlocked and spin bits */ +#define LCK_MTX_TAG_INDIRECT 0x07ff1007 /* lock marked as Indirect */ +#define LCK_MTX_TAG_DESTROYED 0x07fe2007 /* lock marked as Destroyed */ /* Adaptive spin before blocking */ extern unsigned int MutexSpin; -extern void lck_mtx_lock_spin(lck_mtx_t *lck); +extern int lck_mtx_lock_spinwait_x86(lck_mtx_t *mutex); +extern void lck_mtx_lock_wait_x86(lck_mtx_t *mutex); +extern void lck_mtx_lock_acquire_x86(lck_mtx_t *mutex); +extern void lck_mtx_unlock_wakeup_x86(lck_mtx_t *mutex, int prior_lock_state); + +extern void lck_mtx_lock_mark_destroyed(lck_mtx_t *mutex); +extern int lck_mtx_lock_grab_mutex(lck_mtx_t *mutex); -extern void lck_mtx_interlock_spin(lck_mtx_t *lck); +extern void hw_lock_byte_init(volatile uint8_t *lock_byte); +extern void hw_lock_byte_lock(volatile uint8_t *lock_byte); +extern void hw_lock_byte_unlock(volatile uint8_t *lock_byte); typedef struct { unsigned int type; +#ifdef __x86_64__ + unsigned int pad4; +#endif vm_offset_t pc; vm_offset_t thread; } lck_mtx_deb_t; @@ -110,8 +146,14 @@ typedef struct _lck_mtx_ext_ { lck_mtx_t lck_mtx; struct _lck_grp_ *lck_mtx_grp; unsigned int lck_mtx_attr; +#ifdef __x86_64__ + unsigned int lck_mtx_pad1; +#endif lck_mtx_deb_t lck_mtx_deb; - lck_mtx_stat_t lck_mtx_stat; + uint64_t lck_mtx_stat; +#ifdef __x86_64__ + unsigned int lck_mtx_pad2[2]; +#endif } lck_mtx_ext_t; #define LCK_MTX_ATTR_DEBUG 0x1 @@ -119,28 +161,55 @@ typedef struct _lck_mtx_ext_ { #define LCK_MTX_ATTR_STAT 0x2 #define LCK_MTX_ATTR_STATb 1 +#else /* MACH_KERNEL_PRIVATE */ +#ifdef XNU_KERNEL_PRIVATE +typedef struct { + unsigned long opaque[2]; +} lck_mtx_t; + +typedef struct { + unsigned long opaque[10]; +} lck_mtx_ext_t; #else #ifdef KERNEL_PRIVATE typedef struct { - unsigned int opaque[3]; + unsigned long opaque[2]; } lck_mtx_t; + +typedef struct { + unsigned long opaque[10]; +} lck_mtx_ext_t; + #else -typedef struct __lck_mtx_t__ lck_mtx_t; +typedef struct __lck_mtx_t__ lck_mtx_t; +typedef struct __lck_mtx_ext_t__ lck_mtx_ext_t; +#endif #endif #endif #ifdef MACH_KERNEL_PRIVATE -typedef struct { - hw_lock_data_t interlock; - volatile unsigned int - read_count:16, /* No. of accepted readers */ - want_upgrade:1, /* Read-to-write upgrade waiting */ - want_write:1, /* Writer waiting or locked for write */ - waiting:1, /* Someone is sleeping on lock */ - can_sleep:1, /* Can attempts to lock go to sleep? */ - read_priority:1;/* New read takes piority over write */ - unsigned int lck_rw_tag; +#pragma pack(1) /* Make sure the structure stays as we defined it */ +typedef struct _lck_rw_t_internal_ { + volatile uint16_t lck_rw_shared_count; /* No. of accepted readers */ + volatile uint8_t lck_rw_interlock; /* Interlock byte */ + volatile uint8_t + lck_rw_priv_excl:1, /* Writers prioritized if set */ + lck_rw_want_upgrade:1, /* Read-to-write upgrade waiting */ + lck_rw_want_write:1, /* Writer waiting or locked for write */ + lck_r_waiting:1, /* Reader is sleeping on lock */ + lck_w_waiting:1, /* Writer is sleeping on lock */ + lck_rw_can_sleep:1, /* Can attempts to lock go to sleep? */ + lck_rw_padb6:2; /* padding */ + + uint32_t lck_rw_tag; /* This can be obsoleted when stats + * are in + */ + uint32_t lck_rw_pad8; +#ifdef __x86_64__ + uint32_t lck_rw_pad12; +#endif } lck_rw_t; +#pragma pack() #define LCK_RW_ATTR_DEBUG 0x1 #define LCK_RW_ATTR_DEBUGb 0 @@ -157,9 +226,14 @@ typedef struct { #else #ifdef KERNEL_PRIVATE +#pragma pack(1) typedef struct { - unsigned int opaque[3]; + uint32_t opaque[3]; +#ifdef __x86_64__ + uint32_t opaque4; +#endif } lck_rw_t; +#pragma pack() #else typedef struct __lck_rw_t__ lck_rw_t; #endif