[apple/xnu.git] / osfmk / i386 / lock.h

/*
 * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
/*
 * Copyright (C) 1998 Apple Computer
 * All Rights Reserved
 */
/*
 * @OSF_COPYRIGHT@
 */
/* 
 * Mach Operating System
 * Copyright (c) 1991,1990 Carnegie Mellon University
 * All Rights Reserved.
 * 
 * Permission to use, copy, modify and distribute this software and its
 * documentation is hereby granted, provided that both the copyright
 * notice and this permission notice appear in all copies of the
 * software, derivative works or modified versions, and any portions
 * thereof, and that both notices appear in supporting documentation.
 * 
 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
 * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
 * 
 * Carnegie Mellon requests users of this software to return to
 * 
 *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
 *  School of Computer Science
 *  Carnegie Mellon University
 *  Pittsburgh PA 15213-3890
 * 
 * any improvements or extensions that they make and grant Carnegie Mellon
 * the rights to redistribute these changes.
 */

/*
 */

/*
 * Machine-dependent simple locks for the i386.
 */
#ifdef	KERNEL_PRIVATE

#ifndef	_I386_LOCK_H_
#define	_I386_LOCK_H_

#include <sys/appleapiopts.h>

#ifdef __APPLE_API_PRIVATE

#ifdef MACH_KERNEL_PRIVATE

#include <kern/macro_help.h>
#include <kern/assert.h>
#include <i386/hw_lock_types.h>
#include <i386/locks.h>

#include <mach_rt.h>
#include <mach_ldebug.h>

typedef struct {
	lck_mtx_t	lck_mtx;	/* inlined lck_mtx, need to be first */
#if     MACH_LDEBUG     
	int				type;
#define MUTEX_TAG       0x4d4d
	vm_offset_t		pc;
	vm_offset_t		thread;
#endif  /* MACH_LDEBUG */
} mutex_t;

typedef struct {
	decl_simple_lock_data(,interlock) /* "hardware" interlock field */
	volatile unsigned int
		read_count:16,	/* No. of accepted readers */
		want_upgrade:1,	/* Read-to-write upgrade waiting */
		want_write:1,	/* Writer is waiting, or locked for write */ 
		waiting:1,		/* Someone is sleeping on lock */
	can_sleep:1;		/* Can attempts to lock go to sleep? */
} lock_t;

extern unsigned int LockTimeOut;			/* Number of hardware ticks of a lock timeout */


#if defined(__GNUC__)

/*
 *	General bit-lock routines.
 */

#define	bit_lock(bit,l)							\
	__asm__ volatile("	jmp	1f	\n			\
		 	0:	btl	%0, %1	\n			\
				jb	0b	\n			\
			1:	lock		\n			\
				btsl	%0,%1	\n			\
				jb	0b"			:	\
								:	\
			"r" (bit), "m" (*(volatile int *)(l))	:	\
			"memory");

#define	bit_unlock(bit,l)						\
	__asm__ volatile("	lock		\n			\
				btrl	%0,%1"			:	\
								:	\
			"r" (bit), "m" (*(volatile int *)(l)));

/*
 *      Set or clear individual bits in a long word.
 *      The locked access is needed only to lock access
 *      to the word, not to individual bits.
 */

#define	i_bit_set(bit,l)						\
	__asm__ volatile("	lock		\n			\
				btsl	%0,%1"			:	\
								:	\
			"r" (bit), "m" (*(volatile int *)(l)));

#define	i_bit_clear(bit,l)						\
	__asm__ volatile("	lock		\n			\
				btrl	%0,%1"			:	\
								:	\
			"r" (bit), "m" (*(volatile int *)(l)));

static inline unsigned long i_bit_isset(unsigned int test, volatile unsigned long *word)
{
	int	bit;

	__asm__ volatile("btl %2,%1\n\tsbbl %0,%0" : "=r" (bit)
		: "m" (word), "ir" (test));
	return bit;
}

static inline char	xchgb(volatile char * cp, char new);

static inline void	atomic_incl(long * p, long delta);
static inline void	atomic_incs(short * p, short delta);
static inline void	atomic_incb(char * p, char delta);

static inline void	atomic_decl(long * p, long delta);
static inline void	atomic_decs(short * p, short delta);
static inline void	atomic_decb(char * p, char delta);

static inline long	atomic_getl(long * p);
static inline short	atomic_gets(short * p);
static inline char	atomic_getb(char * p);

static inline void	atomic_setl(long * p, long value);
static inline void	atomic_sets(short * p, short value);
static inline void	atomic_setb(char * p, char value);

static inline char	xchgb(volatile char * cp, char new)
{
	register char	old = new;

	__asm__ volatile ("	xchgb	%0,%2"			:
			"=q" (old)				:
			"0" (new), "m" (*(volatile char *)cp) : "memory");
	return (old);
}

/*
 * Compare and exchange:
 * - returns failure (0) if the location did not contain the old value,
 * - returns success (1) if the location was set to the new value.
 */
static inline uint32_t
atomic_cmpxchg(uint32_t *p, uint32_t old, uint32_t new)
{
	uint32_t res = old;

	asm volatile(
		"lock;	cmpxchgl	%1,%2;	\n\t"
		"	setz		%%al;	\n\t"
		"	movzbl		%%al,%0"
		: "+a" (res)	/* %0: old value to compare, returns success */
		: "r" (new),	/* %1: new value to set */
		  "m" (*(p))	/* %2: memory address */
		: "memory");
	return (res);
}

static inline uint64_t
atomic_load64(uint64_t *quadp)
{
	uint64_t ret;

	asm volatile(
		"	lock; cmpxchg8b	%1"
		: "=A" (ret)
		: "m" (*quadp), "a" (0), "d" (0), "b" (0), "c" (0));
	return (ret);
}

static inline uint64_t
atomic_loadstore64(uint64_t *quadp, uint64_t new)
{
	uint64_t ret;

	ret = *quadp;
	asm volatile(
		"1:				\n\t"
		"	lock; cmpxchg8b	%1	\n\t"
		"	jnz	1b"
		: "+A" (ret)
		: "m" (*quadp),
		  "b" ((uint32_t)new), "c" ((uint32_t)(new >> 32)));
	return (ret);
}

static inline void	atomic_incl(long * p, long delta)
{
	__asm__ volatile ("	lock		\n		\
				addl    %0,%1"		:	\
							:	\
				"r" (delta), "m" (*(volatile long *)p));
}

static inline void	atomic_incs(short * p, short delta)
{
	__asm__ volatile ("	lock		\n		\
				addw    %0,%1"		:	\
							:	\
				"q" (delta), "m" (*(volatile short *)p));
}

static inline void	atomic_incb(char * p, char delta)
{
	__asm__ volatile ("	lock		\n		\
				addb    %0,%1"		:	\
							:	\
				"q" (delta), "m" (*(volatile char *)p));
}

static inline void	atomic_decl(long * p, long delta)
{
	__asm__ volatile ("	lock		\n		\
				subl	%0,%1"		:	\
							:	\
				"r" (delta), "m" (*(volatile long *)p));
}

static inline int	atomic_decl_and_test(long * p, long delta)
{
	uint8_t	ret;
	asm volatile (
		"	lock		\n\t"
		"	subl	%1,%2	\n\t"
		"	sete	%0"
		: "=qm" (ret)
		: "r" (delta), "m" (*(volatile long *)p));
	return ret;
}

static inline void	atomic_decs(short * p, short delta)
{
	__asm__ volatile ("	lock		\n		\
				subw    %0,%1"		:	\
							:	\
				"q" (delta), "m" (*(volatile short *)p));
}

static inline void	atomic_decb(char * p, char delta)
{
	__asm__ volatile ("	lock		\n		\
				subb    %0,%1"		:	\
							:	\
				"q" (delta), "m" (*(volatile char *)p));
}

static inline long	atomic_getl(long * p)
{
	return (*p);
}

static inline short	atomic_gets(short * p)
{
	return (*p);
}

static inline char	atomic_getb(char * p)
{
	return (*p);
}

static inline void	atomic_setl(long * p, long value)
{
	*p = value;
}

static inline void	atomic_sets(short * p, short value)
{
	*p = value;
}

static inline void	atomic_setb(char * p, char value)
{
	*p = value;
}


#else	/* !defined(__GNUC__) */

extern void	i_bit_set(
	int index,
	void *addr);

extern void	i_bit_clear(
	int index,
	void *addr);

extern void bit_lock(
	int index,
	void *addr);

extern void bit_unlock(
	int index,
	void *addr);

/*
 * All other routines defined in __GNUC__ case lack
 * definitions otherwise. - XXX
 */

#endif	/* !defined(__GNUC__) */

extern void		kernel_preempt_check (void);

#endif /* MACH_KERNEL_PRIVATE */

#endif /* __APLE_API_PRIVATE */

#endif	/* _I386_LOCK_H_ */

#endif	/* KERNEL_PRIVATE */
Commit	Line	Data
1c79356b	1	/*
91447636	2	* Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
1c79356b	3	*
8f6c56a5	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
1c79356b	5	*
8f6c56a5 A	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
8ad349bb	24	* limitations under the License.
8f6c56a5 A	25	*
8f6c56a5 A	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
1c79356b A	27	*/
	28	/*
	29	* Copyright (C) 1998 Apple Computer
	30	* All Rights Reserved
	31	*/
	32	/*
	33	* @OSF_COPYRIGHT@
	34	*/
	35	/*
	36	* Mach Operating System
	37	* Copyright (c) 1991,1990 Carnegie Mellon University
	38	* All Rights Reserved.
	39	*
	40	* Permission to use, copy, modify and distribute this software and its
	41	* documentation is hereby granted, provided that both the copyright
	42	* notice and this permission notice appear in all copies of the
	43	* software, derivative works or modified versions, and any portions
	44	* thereof, and that both notices appear in supporting documentation.
	45	*
	46	* CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
	47	* CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
	48	* ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
	49	*
	50	* Carnegie Mellon requests users of this software to return to
	51	*
	52	* Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
	53	* School of Computer Science
	54	* Carnegie Mellon University
	55	* Pittsburgh PA 15213-3890
	56	*
	57	* any improvements or extensions that they make and grant Carnegie Mellon
	58	* the rights to redistribute these changes.
	59	*/
	60
	61	/*
	62	*/
	63
	64	/*
	65	* Machine-dependent simple locks for the i386.
	66	*/
91447636	67	#ifdef KERNEL_PRIVATE
1c79356b A	68
	69	#ifndef _I386_LOCK_H_
	70	#define _I386_LOCK_H_
	71
9bccf70c A	72	#include <sys/appleapiopts.h>
	73
	74	#ifdef __APPLE_API_PRIVATE
	75
	76	#ifdef MACH_KERNEL_PRIVATE
	77
1c79356b A	78	#include <kern/macro_help.h>
	79	#include <kern/assert.h>
	80	#include <i386/hw_lock_types.h>
91447636	81	#include <i386/locks.h>
1c79356b	82
1c79356b A	83	#include <mach_rt.h>
1c79356b A	84	#include <mach_ldebug.h>
91447636 A	85
	86	typedef struct {
	87	lck_mtx_t lck_mtx; /* inlined lck_mtx, need to be first */
	88	#if MACH_LDEBUG
	89	int type;
	90	#define MUTEX_TAG 0x4d4d
	91	vm_offset_t pc;
	92	vm_offset_t thread;
	93	#endif /* MACH_LDEBUG */
	94	} mutex_t;
	95
21362eb3 A	96	typedef struct {
	97	decl_simple_lock_data(,interlock) /* "hardware" interlock field */
	98	volatile unsigned int
	99	read_count:16, /* No. of accepted readers */
	100	want_upgrade:1, /* Read-to-write upgrade waiting */
	101	want_write:1, /* Writer is waiting, or locked for write */
	102	waiting:1, /* Someone is sleeping on lock */
	103	can_sleep:1; /* Can attempts to lock go to sleep? */
	104	} lock_t;
91447636 A	105
91447636 A	106	extern unsigned int LockTimeOut; /* Number of hardware ticks of a lock timeout */
1c79356b A	107
	108
	109	#if defined(__GNUC__)
	110
	111	/*
	112	* General bit-lock routines.
	113	*/
	114
	115	#define bit_lock(bit,l) \
	116	__asm__ volatile(" jmp 1f \n \
	117	0: btl %0, %1 \n \
	118	jb 0b \n \
	119	1: lock \n \
	120	btsl %0,%1 \n \
	121	jb 0b" : \
	122	: \
	123	"r" (bit), "m" ((volatile int )(l)) : \
	124	"memory");
	125
	126	#define bit_unlock(bit,l) \
	127	__asm__ volatile(" lock \n \
	128	btrl %0,%1" : \
	129	: \
	130	"r" (bit), "m" ((volatile int )(l)));
	131
	132	/*
	133	* Set or clear individual bits in a long word.
	134	* The locked access is needed only to lock access
	135	* to the word, not to individual bits.
	136	*/
	137
	138	#define i_bit_set(bit,l) \
	139	__asm__ volatile(" lock \n \
	140	btsl %0,%1" : \
	141	: \
	142	"r" (bit), "m" ((volatile int )(l)));
	143
	144	#define i_bit_clear(bit,l) \
	145	__asm__ volatile(" lock \n \
	146	btrl %0,%1" : \
	147	: \
	148	"r" (bit), "m" ((volatile int )(l)));
	149
91447636	150	static inline unsigned long i_bit_isset(unsigned int test, volatile unsigned long *word)
1c79356b A	151	{
	152	int bit;
	153
	154	__asm__ volatile("btl %2,%1\n\tsbbl %0,%0" : "=r" (bit)
91447636	155	: "m" (word), "ir" (test));
1c79356b A	156	return bit;
	157	}
	158
91447636	159	static inline char xchgb(volatile char * cp, char new);
1c79356b	160
21362eb3 A	161	static inline void atomic_incl(long * p, long delta);
	162	static inline void atomic_incs(short * p, short delta);
	163	static inline void atomic_incb(char * p, char delta);
1c79356b	164
21362eb3 A	165	static inline void atomic_decl(long * p, long delta);
	166	static inline void atomic_decs(short * p, short delta);
	167	static inline void atomic_decb(char * p, char delta);
1c79356b	168
21362eb3 A	169	static inline long atomic_getl(long * p);
	170	static inline short atomic_gets(short * p);
	171	static inline char atomic_getb(char * p);
1c79356b	172
21362eb3 A	173	static inline void atomic_setl(long * p, long value);
	174	static inline void atomic_sets(short * p, short value);
	175	static inline void atomic_setb(char * p, char value);
1c79356b	176
91447636	177	static inline char xchgb(volatile char * cp, char new)
1c79356b A	178	{
	179	register char old = new;
	180
	181	__asm__ volatile (" xchgb %0,%2" :
	182	"=q" (old) :
	183	"0" (new), "m" ((volatile char )cp) : "memory");
	184	return (old);
	185	}
	186
91447636 A	187	/*
	188	* Compare and exchange:
	189	* - returns failure (0) if the location did not contain the old value,
	190	* - returns success (1) if the location was set to the new value.
	191	*/
	192	static inline uint32_t
	193	atomic_cmpxchg(uint32_t *p, uint32_t old, uint32_t new)
	194	{
	195	uint32_t res = old;
	196
	197	asm volatile(
	198	"lock; cmpxchgl %1,%2; \n\t"
	199	" setz %%al; \n\t"
	200	" movzbl %%al,%0"
	201	: "+a" (res) /* %0: old value to compare, returns success */
	202	: "r" (new), /* %1: new value to set */
	203	"m" ((p)) / %2: memory address */
	204	: "memory");
	205	return (res);
	206	}
	207
21362eb3 A	208	static inline uint64_t
	209	atomic_load64(uint64_t *quadp)
	210	{
	211	uint64_t ret;
	212
	213	asm volatile(
	214	" lock; cmpxchg8b %1"
	215	: "=A" (ret)
	216	: "m" (*quadp), "a" (0), "d" (0), "b" (0), "c" (0));
	217	return (ret);
	218	}
	219
	220	static inline uint64_t
	221	atomic_loadstore64(uint64_t *quadp, uint64_t new)
	222	{
	223	uint64_t ret;
	224
	225	ret = *quadp;
	226	asm volatile(
	227	"1: \n\t"
	228	" lock; cmpxchg8b %1 \n\t"
	229	" jnz 1b"
	230	: "+A" (ret)
	231	: "m" (*quadp),
	232	"b" ((uint32_t)new), "c" ((uint32_t)(new >> 32)));
	233	return (ret);
	234	}
	235
	236	static inline void atomic_incl(long * p, long delta)
1c79356b	237	{
1c79356b A	238	__asm__ volatile (" lock \n \
	239	addl %0,%1" : \
	240	: \
	241	"r" (delta), "m" ((volatile long )p));
1c79356b A	242	}
1c79356b A	243
21362eb3	244	static inline void atomic_incs(short * p, short delta)
1c79356b	245	{
1c79356b A	246	__asm__ volatile (" lock \n \
	247	addw %0,%1" : \
	248	: \
	249	"q" (delta), "m" ((volatile short )p));
1c79356b A	250	}
1c79356b A	251
21362eb3	252	static inline void atomic_incb(char * p, char delta)
1c79356b	253	{
1c79356b A	254	__asm__ volatile (" lock \n \
	255	addb %0,%1" : \
	256	: \
	257	"q" (delta), "m" ((volatile char )p));
1c79356b A	258	}
1c79356b A	259
21362eb3	260	static inline void atomic_decl(long * p, long delta)
1c79356b	261	{
1c79356b A	262	__asm__ volatile (" lock \n \
	263	subl %0,%1" : \
	264	: \
	265	"r" (delta), "m" ((volatile long )p));
1c79356b A	266	}
1c79356b A	267
21362eb3	268	static inline int atomic_decl_and_test(long * p, long delta)
91447636 A	269	{
	270	uint8_t ret;
	271	asm volatile (
	272	" lock \n\t"
	273	" subl %1,%2 \n\t"
	274	" sete %0"
	275	: "=qm" (ret)
	276	: "r" (delta), "m" ((volatile long )p));
	277	return ret;
	278	}
	279
21362eb3	280	static inline void atomic_decs(short * p, short delta)
1c79356b	281	{
1c79356b A	282	__asm__ volatile (" lock \n \
	283	subw %0,%1" : \
	284	: \
	285	"q" (delta), "m" ((volatile short )p));
1c79356b A	286	}
1c79356b A	287
21362eb3	288	static inline void atomic_decb(char * p, char delta)
1c79356b	289	{
1c79356b A	290	__asm__ volatile (" lock \n \
	291	subb %0,%1" : \
	292	: \
	293	"q" (delta), "m" ((volatile char )p));
1c79356b A	294	}
1c79356b A	295
21362eb3	296	static inline long atomic_getl(long * p)
1c79356b A	297	{
	298	return (*p);
	299	}
	300
21362eb3	301	static inline short atomic_gets(short * p)
1c79356b A	302	{
	303	return (*p);
	304	}
	305
21362eb3	306	static inline char atomic_getb(char * p)
1c79356b A	307	{
	308	return (*p);
	309	}
	310
21362eb3	311	static inline void atomic_setl(long * p, long value)
1c79356b A	312	{
	313	*p = value;
	314	}
	315
21362eb3	316	static inline void atomic_sets(short * p, short value)
1c79356b A	317	{
	318	*p = value;
	319	}
	320
21362eb3	321	static inline void atomic_setb(char * p, char value)
1c79356b A	322	{
	323	*p = value;
	324	}
	325
	326
	327	#else /* !defined(__GNUC__) */
	328
	329	extern void i_bit_set(
	330	int index,
	331	void *addr);
	332
	333	extern void i_bit_clear(
	334	int index,
	335	void *addr);
	336
	337	extern void bit_lock(
	338	int index,
	339	void *addr);
	340
	341	extern void bit_unlock(
	342	int index,
	343	void *addr);
	344
	345	/*
	346	* All other routines defined in __GNUC__ case lack
	347	* definitions otherwise. - XXX
	348	*/
	349
	350	#endif /* !defined(__GNUC__) */
	351
9bccf70c A	352	extern void kernel_preempt_check (void);
9bccf70c A	353
0b4e3aa0	354	#endif /* MACH_KERNEL_PRIVATE */
1c79356b	355
9bccf70c	356	#endif /* __APLE_API_PRIVATE */
1c79356b A	357
1c79356b A	358	#endif /* _I386_LOCK_H_ */
91447636 A	359
91447636 A	360	#endif /* KERNEL_PRIVATE */