[apple/xnu.git] / osfmk / i386 / locks.h

/*
 * Copyright (c) 2004-2012 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#ifndef _I386_LOCKS_H_
#define _I386_LOCKS_H_

#include <sys/appleapiopts.h>
#include <kern/kern_types.h>
#include <kern/assert.h>

#ifdef  MACH_KERNEL_PRIVATE

#include <i386/hw_lock_types.h>

extern  unsigned int    LcksOpts;
#if DEVELOPMENT || DEBUG
extern  unsigned int    LckDisablePreemptCheck;
#endif

#define enaLkDeb                0x00000001      /* Request debug in default attribute */
#define enaLkStat               0x00000002      /* Request statistic in default attribute */
#define disLkRWPrio             0x00000004      /* Disable RW lock priority promotion */
#define enaLkTimeStat           0x00000008      /* Request time statistics in default attribute */

#endif /* MACH_KERNEL_PRIVATE */

#if     defined(MACH_KERNEL_PRIVATE)
typedef struct {
	volatile uintptr_t      interlock;
#if     MACH_LDEBUG
	unsigned long   lck_spin_pad[9];        /* XXX - usimple_lock_data_t */
#endif
} lck_spin_t;

#define LCK_SPIN_TAG_DESTROYED          0x00002007      /* lock marked as Destroyed */

#else /* MACH_KERNEL_PRIVATE */
#ifdef  KERNEL_PRIVATE
typedef struct {
	unsigned long    opaque[10];
} lck_spin_t;
#else /* KERNEL_PRIVATE */
typedef struct __lck_spin_t__   lck_spin_t;
#endif
#endif

#ifdef  MACH_KERNEL_PRIVATE
/* The definition of this structure, including the layout of the
 * state bitfield, is tailored to the asm implementation in i386_lock.s
 */
typedef struct _lck_mtx_ {
	union {
		struct {
			volatile uintptr_t              lck_mtx_owner;
			union {
				struct {
					volatile uint32_t
					    lck_mtx_waiters:16,
					    lck_mtx_pri:8, // unused
					    lck_mtx_ilocked:1,
					    lck_mtx_mlocked:1,
					    lck_mtx_promoted:1, // unused
					    lck_mtx_spin:1,
					    lck_mtx_is_ext:1,
					    lck_mtx_pad3:3;
				};
				uint32_t        lck_mtx_state;
			};
			/* Pad field used as a canary, initialized to ~0 */
			uint32_t                        lck_mtx_pad32;
		};
		struct {
			struct _lck_mtx_ext_            *lck_mtx_ptr;
			uint32_t                        lck_mtx_tag;
			uint32_t                        lck_mtx_pad32_2;
		};
	};
} lck_mtx_t;

#define LCK_MTX_WAITERS_MSK             0x0000ffff
#define LCK_MTX_WAITER                  0x00000001
#define LCK_MTX_PRIORITY_MSK            0x00ff0000
#define LCK_MTX_ILOCKED_MSK             0x01000000
#define LCK_MTX_MLOCKED_MSK             0x02000000
#define LCK_MTX_SPIN_MSK                0x08000000

/* This pattern must subsume the interlocked, mlocked and spin bits */
#define LCK_MTX_TAG_INDIRECT                    0x07ff1007      /* lock marked as Indirect  */
#define LCK_MTX_TAG_DESTROYED                   0x07fe2007      /* lock marked as Destroyed */

/* Adaptive spin before blocking */
extern uint64_t         MutexSpin;
extern uint64_t         low_MutexSpin;
extern int64_t         high_MutexSpin;

typedef enum lck_mtx_spinwait_ret_type {
	LCK_MTX_SPINWAIT_ACQUIRED = 0,

	LCK_MTX_SPINWAIT_SPUN_HIGH_THR = 1,
	LCK_MTX_SPINWAIT_SPUN_OWNER_NOT_CORE = 2,
	LCK_MTX_SPINWAIT_SPUN_NO_WINDOW_CONTENTION = 3,
	LCK_MTX_SPINWAIT_SPUN_SLIDING_THR = 4,

	LCK_MTX_SPINWAIT_NO_SPIN = 5,
} lck_mtx_spinwait_ret_type_t;

extern lck_mtx_spinwait_ret_type_t              lck_mtx_lock_spinwait_x86(lck_mtx_t *mutex);
struct turnstile;
extern void                                     lck_mtx_lock_wait_x86(lck_mtx_t *mutex, struct turnstile **ts);
extern void                                     lck_mtx_lock_acquire_x86(lck_mtx_t *mutex);

extern void                                     lck_mtx_lock_slow(lck_mtx_t *lock);
extern boolean_t                                lck_mtx_try_lock_slow(lck_mtx_t *lock);
extern void                                     lck_mtx_unlock_slow(lck_mtx_t *lock);
extern void                                     lck_mtx_lock_spin_slow(lck_mtx_t *lock);
extern boolean_t                                lck_mtx_try_lock_spin_slow(lck_mtx_t *lock);
extern void                                     hw_lock_byte_init(volatile uint8_t *lock_byte);
extern void                                     hw_lock_byte_lock(volatile uint8_t *lock_byte);
extern void                                     hw_lock_byte_unlock(volatile uint8_t *lock_byte);

typedef struct {
	unsigned int            type;
	unsigned int            pad4;
	vm_offset_t             pc;
	vm_offset_t             thread;
} lck_mtx_deb_t;

#define MUTEX_TAG       0x4d4d

typedef struct {
	unsigned int            lck_mtx_stat_data;
} lck_mtx_stat_t;

typedef struct _lck_mtx_ext_ {
	lck_mtx_t               lck_mtx;
	struct _lck_grp_        *lck_mtx_grp;
	unsigned int            lck_mtx_attr;
	unsigned int            lck_mtx_pad1;
	lck_mtx_deb_t           lck_mtx_deb;
	uint64_t                lck_mtx_stat;
	unsigned int            lck_mtx_pad2[2];
} lck_mtx_ext_t;

#define LCK_MTX_ATTR_DEBUG      0x1
#define LCK_MTX_ATTR_DEBUGb     0
#define LCK_MTX_ATTR_STAT       0x2
#define LCK_MTX_ATTR_STATb      1

#define LCK_MTX_EVENT(lck)        ((event_t)(((unsigned int*)(lck))+(sizeof(lck_mtx_t)-1)/sizeof(unsigned int)))
#define LCK_EVENT_TO_MUTEX(event) ((lck_mtx_t *)(uintptr_t)(((unsigned int *)(event)) - ((sizeof(lck_mtx_t)-1)/sizeof(unsigned int))))

#else /* MACH_KERNEL_PRIVATE */
#ifdef  XNU_KERNEL_PRIVATE
typedef struct {
	unsigned long           opaque[2];
} lck_mtx_t;

typedef struct {
	unsigned long           opaque[10];
} lck_mtx_ext_t;
#else
#ifdef  KERNEL_PRIVATE
typedef struct {
	unsigned long           opaque[2];
} lck_mtx_t;

typedef struct {
	unsigned long           opaque[10];
} lck_mtx_ext_t;

#else
typedef struct __lck_mtx_t__            lck_mtx_t;
typedef struct __lck_mtx_ext_t__        lck_mtx_ext_t;
#endif
#endif
#endif

#ifdef  MACH_KERNEL_PRIVATE
typedef union _lck_rw_t_internal_ {
	struct {
		volatile uint16_t       lck_rw_shared_count;    /* No. of accepted readers */
		volatile uint8_t        lck_rw_interlock;       /* Interlock byte */
		volatile uint8_t
		    lck_rw_priv_excl:1,                         /* Writers prioritized if set */
		    lck_rw_want_upgrade:1,                      /* Read-to-write upgrade waiting */
		    lck_rw_want_write:1,                        /* Writer waiting or locked for write */
		    lck_r_waiting:1,                            /* Reader is sleeping on lock */
		    lck_w_waiting:1,                            /* Writer is sleeping on lock */
		    lck_rw_can_sleep:1,                         /* Can attempts to lock go to sleep? */
		    lck_rw_padb6:2;                             /* padding */
		uint32_t                lck_rw_tag;             /* This can be obsoleted when stats are in */
		thread_t                lck_rw_owner;           /* Unused */
	};
	struct {
		uint32_t                data;                   /* Single word for count, ilk, and bitfields */
		uint32_t                lck_rw_pad4;
		uint32_t                lck_rw_pad8;
		uint32_t                lck_rw_pad12;
	};
} lck_rw_t;
#define LCK_RW_T_SIZE           16

static_assert(sizeof(lck_rw_t) == LCK_RW_T_SIZE);

#define LCK_RW_SHARED_SHIFT      0
#define LCK_RW_INTERLOCK_BIT    16
#define LCK_RW_PRIV_EXCL_BIT    24
#define LCK_RW_WANT_UPGRADE_BIT 25
#define LCK_RW_WANT_EXCL_BIT    26
#define LCK_RW_R_WAITING_BIT    27
#define LCK_RW_W_WAITING_BIT    28
#define LCK_RW_CAN_SLEEP_BIT    29

#define LCK_RW_INTERLOCK        (1 << LCK_RW_INTERLOCK_BIT)
#define LCK_RW_WANT_UPGRADE     (1 << LCK_RW_WANT_UPGRADE_BIT)
#define LCK_RW_WANT_EXCL        (1 << LCK_RW_WANT_EXCL_BIT)
#define LCK_RW_R_WAITING        (1 << LCK_RW_R_WAITING_BIT)
#define LCK_RW_W_WAITING        (1 << LCK_RW_W_WAITING_BIT)
#define LCK_RW_PRIV_EXCL        (1 << LCK_RW_PRIV_EXCL_BIT)
#define LCK_RW_TAG_VALID        (1 << LCK_RW_TAG_VALID_BIT)
#define LCK_RW_SHARED_MASK      (0xffff << LCK_RW_SHARED_SHIFT)
#define LCK_RW_SHARED_READER    (1 << LCK_RW_SHARED_SHIFT)

#define LCK_RW_WANT_WRITE       LCK_RW_WANT_EXCL


#define LCK_RW_ATTR_DEBUG       0x1
#define LCK_RW_ATTR_DEBUGb      0
#define LCK_RW_ATTR_STAT        0x2
#define LCK_RW_ATTR_STATb       1
#define LCK_RW_ATTR_READ_PRI    0x3
#define LCK_RW_ATTR_READ_PRIb   2
#define LCK_RW_ATTR_DIS_THREAD  0x40000000
#define LCK_RW_ATTR_DIS_THREADb 30
#define LCK_RW_ATTR_DIS_MYLOCK  0x10000000
#define LCK_RW_ATTR_DIS_MYLOCKb 28

#define LCK_RW_TAG_DESTROYED            0x00002007      /* lock marked as Destroyed */

#define RW_LOCK_READER_EVENT(x)   ((event_t) (((unsigned char*) (x)) + (offsetof(lck_rw_t, lck_rw_tag))))
#define RW_LOCK_WRITER_EVENT(x)   ((event_t) (((unsigned char*) (x)) + (offsetof(lck_rw_t, lck_rw_pad8))))
#define READ_EVENT_TO_RWLOCK(x)   ((lck_rw_t *)(((unsigned char*)(x) - (offsetof(lck_rw_t, lck_rw_tag)))))
#define WRITE_EVENT_TO_RWLOCK(x)  ((lck_rw_t *)(((unsigned char*)(x) - (offsetof(lck_rw_t, lck_rw_pad8)))))

#if LOCK_PRIVATE

#define disable_preemption_for_thread(t) ((cpu_data_t GS_RELATIVE *)0UL)->cpu_preemption_level++
#define preemption_disabled_for_thread(t) (((cpu_data_t GS_RELATIVE *)0UL)->cpu_preemption_level > 0)

#define LCK_MTX_THREAD_TO_STATE(t)      ((uintptr_t)t)
#define PLATFORM_LCK_ILOCK              0

#define LOCK_SNOOP_SPINS        1000
#define LOCK_PRETEST            1

/* hw_lock_lock static panic deadline, in timebase units. hw_lock_to() uses
 * LockTimeoutTSC computed at startup
 */
#define LOCK_PANIC_TIMEOUT      0xf000000  /* 251e6 TSC ticks */

#endif  // LOCK_PRIVATE

#else
#ifdef  KERNEL_PRIVATE
#pragma pack(1)
typedef struct {
	uint32_t                opaque[3];
	uint32_t                opaque4;
} lck_rw_t;
#pragma pack()
#else
typedef struct __lck_rw_t__     lck_rw_t;
#endif
#endif

#ifdef MACH_KERNEL_PRIVATE

extern void             kernel_preempt_check(void);

#endif /* MACH_KERNEL_PRIVATE */
#endif  /* _I386_LOCKS_H_ */
Commit	Line	Data
91447636	1	/*
39236c6e	2	* Copyright (c) 2004-2012 Apple Inc. All rights reserved.
91447636	3	*
2d21ac55	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
0a7de745	5	*
2d21ac55 A	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
0a7de745	14	*
2d21ac55 A	15	* Please obtain a copy of the License at
2d21ac55 A	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
0a7de745	17	*
2d21ac55 A	18	* The Original Code and all software distributed under the License are
2d21ac55 A	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5 A	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
8f6c56a5 A	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55 A	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
0a7de745	25	*
2d21ac55	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
91447636 A	27	*/
91447636 A	28
0a7de745 A	29	#ifndef _I386_LOCKS_H_
0a7de745 A	30	#define _I386_LOCKS_H_
91447636 A	31
	32	#include <sys/appleapiopts.h>
	33	#include <kern/kern_types.h>
d9a64523	34	#include <kern/assert.h>
91447636	35
0a7de745	36	#ifdef MACH_KERNEL_PRIVATE
91447636 A	37
	38	#include <i386/hw_lock_types.h>
	39
0a7de745	40	extern unsigned int LcksOpts;
5ba3f43e	41	#if DEVELOPMENT \|\| DEBUG
0a7de745	42	extern unsigned int LckDisablePreemptCheck;
5ba3f43e	43	#endif
91447636	44
0a7de745 A	45	#define enaLkDeb 0x00000001 /* Request debug in default attribute */
	46	#define enaLkStat 0x00000002 /* Request statistic in default attribute */
	47	#define disLkRWPrio 0x00000004 /* Disable RW lock priority promotion */
	48	#define enaLkTimeStat 0x00000008 /* Request time statistics in default attribute */
91447636	49
6d2010ae	50	#endif /* MACH_KERNEL_PRIVATE */
91447636	51
0a7de745	52	#if defined(MACH_KERNEL_PRIVATE)
91447636	53	typedef struct {
0a7de745 A	54	volatile uintptr_t interlock;
	55	#if MACH_LDEBUG
	56	unsigned long lck_spin_pad[9]; /* XXX - usimple_lock_data_t */
6d2010ae	57	#endif
91447636 A	58	} lck_spin_t;
91447636 A	59
0a7de745	60	#define LCK_SPIN_TAG_DESTROYED 0x00002007 /* lock marked as Destroyed */
91447636	61
6d2010ae	62	#else /* MACH_KERNEL_PRIVATE */
0a7de745	63	#ifdef KERNEL_PRIVATE
91447636	64	typedef struct {
b0d623f7	65	unsigned long opaque[10];
91447636	66	} lck_spin_t;
6d2010ae	67	#else /* KERNEL_PRIVATE */
0a7de745	68	typedef struct __lck_spin_t__ lck_spin_t;
91447636 A	69	#endif
	70	#endif
	71
0a7de745	72	#ifdef MACH_KERNEL_PRIVATE
6d2010ae A	73	/* The definition of this structure, including the layout of the
	74	* state bitfield, is tailored to the asm implementation in i386_lock.s
	75	*/
91447636 A	76	typedef struct _lck_mtx_ {
	77	union {
	78	struct {
0a7de745	79	volatile uintptr_t lck_mtx_owner;
6d2010ae A	80	union {
	81	struct {
	82	volatile uint32_t
0a7de745	83	lck_mtx_waiters:16,
cb323159	84	lck_mtx_pri:8, // unused
0a7de745 A	85	lck_mtx_ilocked:1,
0a7de745 A	86	lck_mtx_mlocked:1,
cb323159	87	lck_mtx_promoted:1, // unused
0a7de745 A	88	lck_mtx_spin:1,
	89	lck_mtx_is_ext:1,
	90	lck_mtx_pad3:3;
6d2010ae	91	};
0a7de745	92	uint32_t lck_mtx_state;
6d2010ae	93	};
6d2010ae	94	/* Pad field used as a canary, initialized to ~0 */
0a7de745	95	uint32_t lck_mtx_pad32;
39037602	96	};
91447636	97	struct {
0a7de745 A	98	struct _lck_mtx_ext_ *lck_mtx_ptr;
	99	uint32_t lck_mtx_tag;
	100	uint32_t lck_mtx_pad32_2;
39037602 A	101	};
39037602 A	102	};
91447636 A	103	} lck_mtx_t;
91447636 A	104
0a7de745 A	105	#define LCK_MTX_WAITERS_MSK 0x0000ffff
	106	#define LCK_MTX_WAITER 0x00000001
	107	#define LCK_MTX_PRIORITY_MSK 0x00ff0000
	108	#define LCK_MTX_ILOCKED_MSK 0x01000000
	109	#define LCK_MTX_MLOCKED_MSK 0x02000000
0a7de745	110	#define LCK_MTX_SPIN_MSK 0x08000000
d9a64523	111
6d2010ae	112	/* This pattern must subsume the interlocked, mlocked and spin bits */
0a7de745 A	113	#define LCK_MTX_TAG_INDIRECT 0x07ff1007 /* lock marked as Indirect */
0a7de745 A	114	#define LCK_MTX_TAG_DESTROYED 0x07fe2007 /* lock marked as Destroyed */
91447636	115
0c530ab8	116	/* Adaptive spin before blocking */
0a7de745	117	extern uint64_t MutexSpin;
ea3f0419 A	118	extern uint64_t low_MutexSpin;
ea3f0419 A	119	extern int64_t high_MutexSpin;
0c530ab8	120
d9a64523 A	121	typedef enum lck_mtx_spinwait_ret_type {
d9a64523 A	122	LCK_MTX_SPINWAIT_ACQUIRED = 0,
ea3f0419 A	123
	124	LCK_MTX_SPINWAIT_SPUN_HIGH_THR = 1,
	125	LCK_MTX_SPINWAIT_SPUN_OWNER_NOT_CORE = 2,
	126	LCK_MTX_SPINWAIT_SPUN_NO_WINDOW_CONTENTION = 3,
	127	LCK_MTX_SPINWAIT_SPUN_SLIDING_THR = 4,
	128
	129	LCK_MTX_SPINWAIT_NO_SPIN = 5,
d9a64523 A	130	} lck_mtx_spinwait_ret_type_t;
d9a64523 A	131
0a7de745	132	extern lck_mtx_spinwait_ret_type_t lck_mtx_lock_spinwait_x86(lck_mtx_t *mutex);
cb323159 A	133	struct turnstile;
cb323159 A	134	extern void lck_mtx_lock_wait_x86(lck_mtx_t mutex, struct turnstile *ts);
0a7de745	135	extern void lck_mtx_lock_acquire_x86(lck_mtx_t *mutex);
d9a64523	136
0a7de745 A	137	extern void lck_mtx_lock_slow(lck_mtx_t *lock);
	138	extern boolean_t lck_mtx_try_lock_slow(lck_mtx_t *lock);
	139	extern void lck_mtx_unlock_slow(lck_mtx_t *lock);
	140	extern void lck_mtx_lock_spin_slow(lck_mtx_t *lock);
	141	extern boolean_t lck_mtx_try_lock_spin_slow(lck_mtx_t *lock);
	142	extern void hw_lock_byte_init(volatile uint8_t *lock_byte);
	143	extern void hw_lock_byte_lock(volatile uint8_t *lock_byte);
	144	extern void hw_lock_byte_unlock(volatile uint8_t *lock_byte);
0c530ab8	145
91447636	146	typedef struct {
0a7de745 A	147	unsigned int type;
	148	unsigned int pad4;
	149	vm_offset_t pc;
	150	vm_offset_t thread;
91447636 A	151	} lck_mtx_deb_t;
	152
	153	#define MUTEX_TAG 0x4d4d
	154
	155	typedef struct {
0a7de745	156	unsigned int lck_mtx_stat_data;
91447636 A	157	} lck_mtx_stat_t;
	158
	159	typedef struct _lck_mtx_ext_ {
0a7de745 A	160	lck_mtx_t lck_mtx;
	161	struct _lck_grp_ *lck_mtx_grp;
	162	unsigned int lck_mtx_attr;
	163	unsigned int lck_mtx_pad1;
	164	lck_mtx_deb_t lck_mtx_deb;
	165	uint64_t lck_mtx_stat;
	166	unsigned int lck_mtx_pad2[2];
91447636 A	167	} lck_mtx_ext_t;
91447636 A	168
0a7de745 A	169	#define LCK_MTX_ATTR_DEBUG 0x1
	170	#define LCK_MTX_ATTR_DEBUGb 0
	171	#define LCK_MTX_ATTR_STAT 0x2
	172	#define LCK_MTX_ATTR_STATb 1
91447636	173
813fb2f6 A	174	#define LCK_MTX_EVENT(lck) ((event_t)(((unsigned int*)(lck))+(sizeof(lck_mtx_t)-1)/sizeof(unsigned int)))
813fb2f6 A	175	#define LCK_EVENT_TO_MUTEX(event) ((lck_mtx_t )(uintptr_t)(((unsigned int )(event)) - ((sizeof(lck_mtx_t)-1)/sizeof(unsigned int))))
3e170ce0	176
6d2010ae	177	#else /* MACH_KERNEL_PRIVATE */
0a7de745	178	#ifdef XNU_KERNEL_PRIVATE
6d2010ae	179	typedef struct {
0a7de745	180	unsigned long opaque[2];
6d2010ae A	181	} lck_mtx_t;
	182
	183	typedef struct {
0a7de745	184	unsigned long opaque[10];
6d2010ae	185	} lck_mtx_ext_t;
91447636	186	#else
0a7de745	187	#ifdef KERNEL_PRIVATE
91447636	188	typedef struct {
0a7de745	189	unsigned long opaque[2];
91447636	190	} lck_mtx_t;
b0d623f7 A	191
b0d623f7 A	192	typedef struct {
0a7de745	193	unsigned long opaque[10];
b0d623f7 A	194	} lck_mtx_ext_t;
b0d623f7 A	195
91447636	196	#else
0a7de745 A	197	typedef struct __lck_mtx_t__ lck_mtx_t;
0a7de745 A	198	typedef struct __lck_mtx_ext_t__ lck_mtx_ext_t;
91447636 A	199	#endif
91447636 A	200	#endif
6d2010ae	201	#endif
91447636	202
0a7de745	203	#ifdef MACH_KERNEL_PRIVATE
5ba3f43e A	204	typedef union _lck_rw_t_internal_ {
5ba3f43e A	205	struct {
0a7de745 A	206	volatile uint16_t lck_rw_shared_count; /* No. of accepted readers */
0a7de745 A	207	volatile uint8_t lck_rw_interlock; /* Interlock byte */
5ba3f43e	208	volatile uint8_t
0a7de745 A	209	lck_rw_priv_excl:1, /* Writers prioritized if set */
	210	lck_rw_want_upgrade:1, /* Read-to-write upgrade waiting */
	211	lck_rw_want_write:1, /* Writer waiting or locked for write */
	212	lck_r_waiting:1, /* Reader is sleeping on lock */
	213	lck_w_waiting:1, /* Writer is sleeping on lock */
	214	lck_rw_can_sleep:1, /* Can attempts to lock go to sleep? */
	215	lck_rw_padb6:2; /* padding */
	216	uint32_t lck_rw_tag; /* This can be obsoleted when stats are in */
	217	thread_t lck_rw_owner; /* Unused */
5ba3f43e A	218	};
5ba3f43e A	219	struct {
0a7de745 A	220	uint32_t data; /* Single word for count, ilk, and bitfields */
	221	uint32_t lck_rw_pad4;
	222	uint32_t lck_rw_pad8;
	223	uint32_t lck_rw_pad12;
5ba3f43e	224	};
91447636	225	} lck_rw_t;
0a7de745	226	#define LCK_RW_T_SIZE 16
d9a64523 A	227
d9a64523 A	228	static_assert(sizeof(lck_rw_t) == LCK_RW_T_SIZE);
91447636	229
0a7de745 A	230	#define LCK_RW_SHARED_SHIFT 0
	231	#define LCK_RW_INTERLOCK_BIT 16
	232	#define LCK_RW_PRIV_EXCL_BIT 24
	233	#define LCK_RW_WANT_UPGRADE_BIT 25
	234	#define LCK_RW_WANT_EXCL_BIT 26
	235	#define LCK_RW_R_WAITING_BIT 27
	236	#define LCK_RW_W_WAITING_BIT 28
	237	#define LCK_RW_CAN_SLEEP_BIT 29
	238
	239	#define LCK_RW_INTERLOCK (1 << LCK_RW_INTERLOCK_BIT)
	240	#define LCK_RW_WANT_UPGRADE (1 << LCK_RW_WANT_UPGRADE_BIT)
	241	#define LCK_RW_WANT_EXCL (1 << LCK_RW_WANT_EXCL_BIT)
	242	#define LCK_RW_R_WAITING (1 << LCK_RW_R_WAITING_BIT)
	243	#define LCK_RW_W_WAITING (1 << LCK_RW_W_WAITING_BIT)
	244	#define LCK_RW_PRIV_EXCL (1 << LCK_RW_PRIV_EXCL_BIT)
	245	#define LCK_RW_TAG_VALID (1 << LCK_RW_TAG_VALID_BIT)
	246	#define LCK_RW_SHARED_MASK (0xffff << LCK_RW_SHARED_SHIFT)
	247	#define LCK_RW_SHARED_READER (1 << LCK_RW_SHARED_SHIFT)
	248
	249	#define LCK_RW_WANT_WRITE LCK_RW_WANT_EXCL
	250
	251
	252	#define LCK_RW_ATTR_DEBUG 0x1
	253	#define LCK_RW_ATTR_DEBUGb 0
	254	#define LCK_RW_ATTR_STAT 0x2
	255	#define LCK_RW_ATTR_STATb 1
	256	#define LCK_RW_ATTR_READ_PRI 0x3
	257	#define LCK_RW_ATTR_READ_PRIb 2
	258	#define LCK_RW_ATTR_DIS_THREAD 0x40000000
	259	#define LCK_RW_ATTR_DIS_THREADb 30
	260	#define LCK_RW_ATTR_DIS_MYLOCK 0x10000000
	261	#define LCK_RW_ATTR_DIS_MYLOCKb 28
	262
	263	#define LCK_RW_TAG_DESTROYED 0x00002007 /* lock marked as Destroyed */
91447636	264
813fb2f6 A	265	#define RW_LOCK_READER_EVENT(x) ((event_t) (((unsigned char*) (x)) + (offsetof(lck_rw_t, lck_rw_tag))))
	266	#define RW_LOCK_WRITER_EVENT(x) ((event_t) (((unsigned char*) (x)) + (offsetof(lck_rw_t, lck_rw_pad8))))
	267	#define READ_EVENT_TO_RWLOCK(x) ((lck_rw_t )(((unsigned char)(x) - (offsetof(lck_rw_t, lck_rw_tag)))))
	268	#define WRITE_EVENT_TO_RWLOCK(x) ((lck_rw_t )(((unsigned char)(x) - (offsetof(lck_rw_t, lck_rw_pad8)))))
	269
39037602 A	270	#if LOCK_PRIVATE
	271
	272	#define disable_preemption_for_thread(t) ((cpu_data_t GS_RELATIVE *)0UL)->cpu_preemption_level++
d9a64523	273	#define preemption_disabled_for_thread(t) (((cpu_data_t GS_RELATIVE *)0UL)->cpu_preemption_level > 0)
39037602	274
0a7de745 A	275	#define LCK_MTX_THREAD_TO_STATE(t) ((uintptr_t)t)
0a7de745 A	276	#define PLATFORM_LCK_ILOCK 0
39037602	277
0a7de745 A	278	#define LOCK_SNOOP_SPINS 1000
0a7de745 A	279	#define LOCK_PRETEST 1
39037602	280
0a7de745 A	281	/* hw_lock_lock static panic deadline, in timebase units. hw_lock_to() uses
	282	* LockTimeoutTSC computed at startup
	283	*/
	284	#define LOCK_PANIC_TIMEOUT 0xf000000 /* 251e6 TSC ticks */
39037602	285
0a7de745	286	#endif // LOCK_PRIVATE
39037602	287
91447636	288	#else
0a7de745	289	#ifdef KERNEL_PRIVATE
b0d623f7	290	#pragma pack(1)
91447636	291	typedef struct {
0a7de745 A	292	uint32_t opaque[3];
0a7de745 A	293	uint32_t opaque4;
91447636	294	} lck_rw_t;
b0d623f7	295	#pragma pack()
91447636	296	#else
0a7de745	297	typedef struct __lck_rw_t__ lck_rw_t;
91447636 A	298	#endif
	299	#endif
	300
fe8ab488 A	301	#ifdef MACH_KERNEL_PRIVATE
fe8ab488 A	302
0a7de745	303	extern void kernel_preempt_check(void);
fe8ab488 A	304
fe8ab488 A	305	#endif /* MACH_KERNEL_PRIVATE */
0a7de745	306	#endif /* _I386_LOCKS_H_ */