[apple/xnu.git] / osfmk / i386 / lock.h

/*
 * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_OSREFERENCE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code 
 * as defined in and that are subject to the Apple Public Source License 
 * Version 2.0 (the 'License'). You may not use this file except in 
 * compliance with the License.  The rights granted to you under the 
 * License may not be used to create, or enable the creation or 
 * redistribution of, unlawful or unlicensed copies of an Apple operating 
 * system, or to circumvent, violate, or enable the circumvention or 
 * violation of, any terms of an Apple operating system software license 
 * agreement.
 *
 * Please obtain a copy of the License at 
 * http://www.opensource.apple.com/apsl/ and read it before using this 
 * file.
 *
 * The Original Code and all software distributed under the License are 
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 
 * Please see the License for the specific language governing rights and 
 * limitations under the License.
 *
 * @APPLE_LICENSE_OSREFERENCE_HEADER_END@
 */
/*
 * Copyright (C) 1998 Apple Computer
 * All Rights Reserved
 */
/*
 * @OSF_COPYRIGHT@
 */
/* 
 * Mach Operating System
 * Copyright (c) 1991,1990 Carnegie Mellon University
 * All Rights Reserved.
 * 
 * Permission to use, copy, modify and distribute this software and its
 * documentation is hereby granted, provided that both the copyright
 * notice and this permission notice appear in all copies of the
 * software, derivative works or modified versions, and any portions
 * thereof, and that both notices appear in supporting documentation.
 * 
 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
 * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
 * 
 * Carnegie Mellon requests users of this software to return to
 * 
 *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
 *  School of Computer Science
 *  Carnegie Mellon University
 *  Pittsburgh PA 15213-3890
 * 
 * any improvements or extensions that they make and grant Carnegie Mellon
 * the rights to redistribute these changes.
 */

/*
 */

/*
 * Machine-dependent simple locks for the i386.
 */
#ifdef	KERNEL_PRIVATE

#ifndef	_I386_LOCK_H_
#define	_I386_LOCK_H_

#include <sys/appleapiopts.h>

#ifdef __APPLE_API_PRIVATE

#ifdef MACH_KERNEL_PRIVATE

#include <kern/macro_help.h>
#include <kern/assert.h>
#include <i386/hw_lock_types.h>
#include <i386/locks.h>

#include <mach_rt.h>
#include <mach_ldebug.h>

typedef struct {
	lck_mtx_t	lck_mtx;	/* inlined lck_mtx, need to be first */
#if     MACH_LDEBUG     
	int				type;
#define MUTEX_TAG       0x4d4d
	vm_offset_t		pc;
	vm_offset_t		thread;
#endif  /* MACH_LDEBUG */
} mutex_t;

typedef lck_rw_t lock_t;

extern unsigned int LockTimeOut;			/* Number of hardware ticks of a lock timeout */


#if defined(__GNUC__)

/*
 *	General bit-lock routines.
 */

#define	bit_lock(bit,l)							\
	__asm__ volatile("	jmp	1f	\n			\
		 	0:	btl	%0, %1	\n			\
				jb	0b	\n			\
			1:	lock		\n			\
				btsl	%0,%1	\n			\
				jb	0b"			:	\
								:	\
			"r" (bit), "m" (*(volatile int *)(l))	:	\
			"memory");

#define	bit_unlock(bit,l)						\
	__asm__ volatile("	lock		\n			\
				btrl	%0,%1"			:	\
								:	\
			"r" (bit), "m" (*(volatile int *)(l)));

/*
 *      Set or clear individual bits in a long word.
 *      The locked access is needed only to lock access
 *      to the word, not to individual bits.
 */

#define	i_bit_set(bit,l)						\
	__asm__ volatile("	lock		\n			\
				btsl	%0,%1"			:	\
								:	\
			"r" (bit), "m" (*(volatile int *)(l)));

#define	i_bit_clear(bit,l)						\
	__asm__ volatile("	lock		\n			\
				btrl	%0,%1"			:	\
								:	\
			"r" (bit), "m" (*(volatile int *)(l)));

static inline unsigned long i_bit_isset(unsigned int test, volatile unsigned long *word)
{
	int	bit;

	__asm__ volatile("btl %2,%1\n\tsbbl %0,%0" : "=r" (bit)
		: "m" (word), "ir" (test));
	return bit;
}

static inline char	xchgb(volatile char * cp, char new);

static inline void	atomic_incl(volatile long * p, long delta);
static inline void	atomic_incs(volatile short * p, short delta);
static inline void	atomic_incb(volatile char * p, char delta);

static inline void	atomic_decl(volatile long * p, long delta);
static inline void	atomic_decs(volatile short * p, short delta);
static inline void	atomic_decb(volatile char * p, char delta);

static inline long	atomic_getl(const volatile long * p);
static inline short	atomic_gets(const volatile short * p);
static inline char	atomic_getb(const volatile char * p);

static inline void	atomic_setl(volatile long * p, long value);
static inline void	atomic_sets(volatile short * p, short value);
static inline void	atomic_setb(volatile char * p, char value);

static inline char	xchgb(volatile char * cp, char new)
{
	register char	old = new;

	__asm__ volatile ("	xchgb	%0,%2"			:
			"=q" (old)				:
			"0" (new), "m" (*(volatile char *)cp) : "memory");
	return (old);
}

/*
 * Compare and exchange:
 * - returns failure (0) if the location did not contain the old value,
 * - returns success (1) if the location was set to the new value.
 */
static inline uint32_t
atomic_cmpxchg(uint32_t *p, uint32_t old, uint32_t new)
{
	uint32_t res = old;

	asm volatile(
		"lock;	cmpxchgl	%1,%2;	\n\t"
		"	setz		%%al;	\n\t"
		"	movzbl		%%al,%0"
		: "+a" (res)	/* %0: old value to compare, returns success */
		: "r" (new),	/* %1: new value to set */
		  "m" (*(p))	/* %2: memory address */
		: "memory");
	return (res);
}

static inline void	atomic_incl(volatile long * p, long delta)
{
	__asm__ volatile ("	lock		\n		\
				addl    %0,%1"		:	\
							:	\
				"r" (delta), "m" (*(volatile long *)p));
}

static inline void	atomic_incs(volatile short * p, short delta)
{
	__asm__ volatile ("	lock		\n		\
				addw    %0,%1"		:	\
							:	\
				"q" (delta), "m" (*(volatile short *)p));
}

static inline void	atomic_incb(volatile char * p, char delta)
{
	__asm__ volatile ("	lock		\n		\
				addb    %0,%1"		:	\
							:	\
				"q" (delta), "m" (*(volatile char *)p));
}

static inline void	atomic_decl(volatile long * p, long delta)
{
	__asm__ volatile ("	lock		\n		\
				subl	%0,%1"		:	\
							:	\
				"r" (delta), "m" (*(volatile long *)p));
}

static inline int	atomic_decl_and_test(volatile long * p, long delta)
{
	uint8_t	ret;
	asm volatile (
		"	lock		\n\t"
		"	subl	%1,%2	\n\t"
		"	sete	%0"
		: "=qm" (ret)
		: "r" (delta), "m" (*(volatile long *)p));
	return ret;
}

static inline void	atomic_decs(volatile short * p, short delta)
{
	__asm__ volatile ("	lock		\n		\
				subw    %0,%1"		:	\
							:	\
				"q" (delta), "m" (*(volatile short *)p));
}

static inline void	atomic_decb(volatile char * p, char delta)
{
	__asm__ volatile ("	lock		\n		\
				subb    %0,%1"		:	\
							:	\
				"q" (delta), "m" (*(volatile char *)p));
}

static inline long	atomic_getl(const volatile long * p)
{
	return (*p);
}

static inline short	atomic_gets(const volatile short * p)
{
	return (*p);
}

static inline char	atomic_getb(const volatile char * p)
{
	return (*p);
}

static inline void	atomic_setl(volatile long * p, long value)
{
	*p = value;
}

static inline void	atomic_sets(volatile short * p, short value)
{
	*p = value;
}

static inline void	atomic_setb(volatile char * p, char value)
{
	*p = value;
}


#else	/* !defined(__GNUC__) */

extern void	i_bit_set(
	int index,
	void *addr);

extern void	i_bit_clear(
	int index,
	void *addr);

extern void bit_lock(
	int index,
	void *addr);

extern void bit_unlock(
	int index,
	void *addr);

/*
 * All other routines defined in __GNUC__ case lack
 * definitions otherwise. - XXX
 */

#endif	/* !defined(__GNUC__) */

extern void		kernel_preempt_check (void);

#endif /* MACH_KERNEL_PRIVATE */

#endif /* __APLE_API_PRIVATE */

#endif	/* _I386_LOCK_H_ */

#endif	/* KERNEL_PRIVATE */
Commit	Line	Data
1c79356b	1	/*
91447636	2	* Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
1c79356b	3	*
8ad349bb	4	* @APPLE_LICENSE_OSREFERENCE_HEADER_START@
1c79356b	5	*
8ad349bb A	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the
	10	* License may not be used to create, or enable the creation or
	11	* redistribution of, unlawful or unlicensed copies of an Apple operating
	12	* system, or to circumvent, violate, or enable the circumvention or
	13	* violation of, any terms of an Apple operating system software license
	14	* agreement.
	15	*
	16	* Please obtain a copy of the License at
	17	* http://www.opensource.apple.com/apsl/ and read it before using this
	18	* file.
	19	*
	20	* The Original Code and all software distributed under the License are
	21	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	22	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	23	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	24	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	25	* Please see the License for the specific language governing rights and
	26	* limitations under the License.
	27	*
	28	* @APPLE_LICENSE_OSREFERENCE_HEADER_END@
1c79356b A	29	*/
	30	/*
	31	* Copyright (C) 1998 Apple Computer
	32	* All Rights Reserved
	33	*/
	34	/*
	35	* @OSF_COPYRIGHT@
	36	*/
	37	/*
	38	* Mach Operating System
	39	* Copyright (c) 1991,1990 Carnegie Mellon University
	40	* All Rights Reserved.
	41	*
	42	* Permission to use, copy, modify and distribute this software and its
	43	* documentation is hereby granted, provided that both the copyright
	44	* notice and this permission notice appear in all copies of the
	45	* software, derivative works or modified versions, and any portions
	46	* thereof, and that both notices appear in supporting documentation.
	47	*
	48	* CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
	49	* CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
	50	* ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
	51	*
	52	* Carnegie Mellon requests users of this software to return to
	53	*
	54	* Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
	55	* School of Computer Science
	56	* Carnegie Mellon University
	57	* Pittsburgh PA 15213-3890
	58	*
	59	* any improvements or extensions that they make and grant Carnegie Mellon
	60	* the rights to redistribute these changes.
	61	*/
	62
	63	/*
	64	*/
	65
	66	/*
	67	* Machine-dependent simple locks for the i386.
	68	*/
91447636	69	#ifdef KERNEL_PRIVATE
1c79356b A	70
	71	#ifndef _I386_LOCK_H_
	72	#define _I386_LOCK_H_
	73
9bccf70c A	74	#include <sys/appleapiopts.h>
	75
	76	#ifdef __APPLE_API_PRIVATE
	77
	78	#ifdef MACH_KERNEL_PRIVATE
	79
1c79356b A	80	#include <kern/macro_help.h>
	81	#include <kern/assert.h>
	82	#include <i386/hw_lock_types.h>
91447636	83	#include <i386/locks.h>
1c79356b	84
1c79356b A	85	#include <mach_rt.h>
1c79356b A	86	#include <mach_ldebug.h>
91447636 A	87
	88	typedef struct {
	89	lck_mtx_t lck_mtx; /* inlined lck_mtx, need to be first */
	90	#if MACH_LDEBUG
	91	int type;
	92	#define MUTEX_TAG 0x4d4d
	93	vm_offset_t pc;
	94	vm_offset_t thread;
	95	#endif /* MACH_LDEBUG */
	96	} mutex_t;
	97
5d5c5d0d	98	typedef lck_rw_t lock_t;
91447636 A	99
91447636 A	100	extern unsigned int LockTimeOut; /* Number of hardware ticks of a lock timeout */
1c79356b A	101
	102
	103	#if defined(__GNUC__)
	104
	105	/*
	106	* General bit-lock routines.
	107	*/
	108
	109	#define bit_lock(bit,l) \
	110	__asm__ volatile(" jmp 1f \n \
	111	0: btl %0, %1 \n \
	112	jb 0b \n \
	113	1: lock \n \
	114	btsl %0,%1 \n \
	115	jb 0b" : \
	116	: \
	117	"r" (bit), "m" ((volatile int )(l)) : \
	118	"memory");
	119
	120	#define bit_unlock(bit,l) \
	121	__asm__ volatile(" lock \n \
	122	btrl %0,%1" : \
	123	: \
	124	"r" (bit), "m" ((volatile int )(l)));
	125
	126	/*
	127	* Set or clear individual bits in a long word.
	128	* The locked access is needed only to lock access
	129	* to the word, not to individual bits.
	130	*/
	131
	132	#define i_bit_set(bit,l) \
	133	__asm__ volatile(" lock \n \
	134	btsl %0,%1" : \
	135	: \
	136	"r" (bit), "m" ((volatile int )(l)));
	137
	138	#define i_bit_clear(bit,l) \
	139	__asm__ volatile(" lock \n \
	140	btrl %0,%1" : \
	141	: \
	142	"r" (bit), "m" ((volatile int )(l)));
	143
91447636	144	static inline unsigned long i_bit_isset(unsigned int test, volatile unsigned long *word)
1c79356b A	145	{
	146	int bit;
	147
	148	__asm__ volatile("btl %2,%1\n\tsbbl %0,%0" : "=r" (bit)
91447636	149	: "m" (word), "ir" (test));
1c79356b A	150	return bit;
	151	}
	152
91447636	153	static inline char xchgb(volatile char * cp, char new);
1c79356b	154
5d5c5d0d A	155	static inline void atomic_incl(volatile long * p, long delta);
	156	static inline void atomic_incs(volatile short * p, short delta);
	157	static inline void atomic_incb(volatile char * p, char delta);
1c79356b	158
5d5c5d0d A	159	static inline void atomic_decl(volatile long * p, long delta);
	160	static inline void atomic_decs(volatile short * p, short delta);
	161	static inline void atomic_decb(volatile char * p, char delta);
1c79356b	162
5d5c5d0d A	163	static inline long atomic_getl(const volatile long * p);
	164	static inline short atomic_gets(const volatile short * p);
	165	static inline char atomic_getb(const volatile char * p);
1c79356b	166
5d5c5d0d A	167	static inline void atomic_setl(volatile long * p, long value);
	168	static inline void atomic_sets(volatile short * p, short value);
	169	static inline void atomic_setb(volatile char * p, char value);
1c79356b	170
91447636	171	static inline char xchgb(volatile char * cp, char new)
1c79356b A	172	{
	173	register char old = new;
	174
	175	__asm__ volatile (" xchgb %0,%2" :
	176	"=q" (old) :
	177	"0" (new), "m" ((volatile char )cp) : "memory");
	178	return (old);
	179	}
	180
91447636 A	181	/*
	182	* Compare and exchange:
	183	* - returns failure (0) if the location did not contain the old value,
	184	* - returns success (1) if the location was set to the new value.
	185	*/
	186	static inline uint32_t
	187	atomic_cmpxchg(uint32_t *p, uint32_t old, uint32_t new)
	188	{
	189	uint32_t res = old;
	190
	191	asm volatile(
	192	"lock; cmpxchgl %1,%2; \n\t"
	193	" setz %%al; \n\t"
	194	" movzbl %%al,%0"
	195	: "+a" (res) /* %0: old value to compare, returns success */
	196	: "r" (new), /* %1: new value to set */
	197	"m" ((p)) / %2: memory address */
	198	: "memory");
	199	return (res);
	200	}
	201
5d5c5d0d	202	static inline void atomic_incl(volatile long * p, long delta)
1c79356b	203	{
1c79356b A	204	__asm__ volatile (" lock \n \
	205	addl %0,%1" : \
	206	: \
	207	"r" (delta), "m" ((volatile long )p));
1c79356b A	208	}
1c79356b A	209
5d5c5d0d	210	static inline void atomic_incs(volatile short * p, short delta)
1c79356b	211	{
1c79356b A	212	__asm__ volatile (" lock \n \
	213	addw %0,%1" : \
	214	: \
	215	"q" (delta), "m" ((volatile short )p));
1c79356b A	216	}
1c79356b A	217
5d5c5d0d	218	static inline void atomic_incb(volatile char * p, char delta)
1c79356b	219	{
1c79356b A	220	__asm__ volatile (" lock \n \
	221	addb %0,%1" : \
	222	: \
	223	"q" (delta), "m" ((volatile char )p));
1c79356b A	224	}
1c79356b A	225
5d5c5d0d	226	static inline void atomic_decl(volatile long * p, long delta)
1c79356b	227	{
1c79356b A	228	__asm__ volatile (" lock \n \
	229	subl %0,%1" : \
	230	: \
	231	"r" (delta), "m" ((volatile long )p));
1c79356b A	232	}
1c79356b A	233
5d5c5d0d	234	static inline int atomic_decl_and_test(volatile long * p, long delta)
91447636 A	235	{
	236	uint8_t ret;
	237	asm volatile (
	238	" lock \n\t"
	239	" subl %1,%2 \n\t"
	240	" sete %0"
	241	: "=qm" (ret)
	242	: "r" (delta), "m" ((volatile long )p));
	243	return ret;
	244	}
	245
5d5c5d0d	246	static inline void atomic_decs(volatile short * p, short delta)
1c79356b	247	{
1c79356b A	248	__asm__ volatile (" lock \n \
	249	subw %0,%1" : \
	250	: \
	251	"q" (delta), "m" ((volatile short )p));
1c79356b A	252	}
1c79356b A	253
5d5c5d0d	254	static inline void atomic_decb(volatile char * p, char delta)
1c79356b	255	{
1c79356b A	256	__asm__ volatile (" lock \n \
	257	subb %0,%1" : \
	258	: \
	259	"q" (delta), "m" ((volatile char )p));
1c79356b A	260	}
1c79356b A	261
5d5c5d0d	262	static inline long atomic_getl(const volatile long * p)
1c79356b A	263	{
	264	return (*p);
	265	}
	266
5d5c5d0d	267	static inline short atomic_gets(const volatile short * p)
1c79356b A	268	{
	269	return (*p);
	270	}
	271
5d5c5d0d	272	static inline char atomic_getb(const volatile char * p)
1c79356b A	273	{
	274	return (*p);
	275	}
	276
5d5c5d0d	277	static inline void atomic_setl(volatile long * p, long value)
1c79356b A	278	{
	279	*p = value;
	280	}
	281
5d5c5d0d	282	static inline void atomic_sets(volatile short * p, short value)
1c79356b A	283	{
	284	*p = value;
	285	}
	286
5d5c5d0d	287	static inline void atomic_setb(volatile char * p, char value)
1c79356b A	288	{
	289	*p = value;
	290	}
	291
	292
	293	#else /* !defined(__GNUC__) */
	294
	295	extern void i_bit_set(
	296	int index,
	297	void *addr);
	298
	299	extern void i_bit_clear(
	300	int index,
	301	void *addr);
	302
	303	extern void bit_lock(
	304	int index,
	305	void *addr);
	306
	307	extern void bit_unlock(
	308	int index,
	309	void *addr);
	310
	311	/*
	312	* All other routines defined in __GNUC__ case lack
	313	* definitions otherwise. - XXX
	314	*/
	315
	316	#endif /* !defined(__GNUC__) */
	317
9bccf70c A	318	extern void kernel_preempt_check (void);
9bccf70c A	319
0b4e3aa0	320	#endif /* MACH_KERNEL_PRIVATE */
1c79356b	321
9bccf70c	322	#endif /* __APLE_API_PRIVATE */
1c79356b A	323
1c79356b A	324	#endif /* _I386_LOCK_H_ */
91447636 A	325
91447636 A	326	#endif /* KERNEL_PRIVATE */