[apple/xnu.git] / osfmk / i386 / lock.h

/*
 * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 * 
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */
/*
 * Copyright (C) 1998 Apple Computer
 * All Rights Reserved
 */
/*
 * @OSF_COPYRIGHT@
 */
/* 
 * Mach Operating System
 * Copyright (c) 1991,1990 Carnegie Mellon University
 * All Rights Reserved.
 * 
 * Permission to use, copy, modify and distribute this software and its
 * documentation is hereby granted, provided that both the copyright
 * notice and this permission notice appear in all copies of the
 * software, derivative works or modified versions, and any portions
 * thereof, and that both notices appear in supporting documentation.
 * 
 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
 * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
 * 
 * Carnegie Mellon requests users of this software to return to
 * 
 *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
 *  School of Computer Science
 *  Carnegie Mellon University
 *  Pittsburgh PA 15213-3890
 * 
 * any improvements or extensions that they make and grant Carnegie Mellon
 * the rights to redistribute these changes.
 */

/*
 */

/*
 * Machine-dependent simple locks for the i386.
 */
#ifdef	KERNEL_PRIVATE

#ifndef	_I386_LOCK_H_
#define	_I386_LOCK_H_

#include <sys/appleapiopts.h>

#ifdef __APPLE_API_PRIVATE

#ifdef MACH_KERNEL_PRIVATE

#include <kern/macro_help.h>
#include <kern/assert.h>
#include <i386/hw_lock_types.h>
#include <i386/locks.h>

#include <mach_rt.h>
#include <mach_ldebug.h>

typedef struct {
	lck_mtx_t	lck_mtx;	/* inlined lck_mtx, need to be first */
#if     MACH_LDEBUG     
	int				type;
#define MUTEX_TAG       0x4d4d
	vm_offset_t		pc;
	vm_offset_t		thread;
#endif  /* MACH_LDEBUG */
} mutex_t;

typedef lck_rw_t lock_t;

extern unsigned int LockTimeOut;			/* Number of hardware ticks of a lock timeout */


#if defined(__GNUC__)

/*
 *	General bit-lock routines.
 */

#define	bit_lock(bit,l)							\
	__asm__ volatile("	jmp	1f	\n			\
		 	0:	btl	%0, %1	\n			\
				jb	0b	\n			\
			1:	lock		\n			\
				btsl	%0,%1	\n			\
				jb	0b"			:	\
								:	\
			"r" (bit), "m" (*(volatile int *)(l))	:	\
			"memory");

#define	bit_unlock(bit,l)						\
	__asm__ volatile("	lock		\n			\
				btrl	%0,%1"			:	\
								:	\
			"r" (bit), "m" (*(volatile int *)(l)));

/*
 *      Set or clear individual bits in a long word.
 *      The locked access is needed only to lock access
 *      to the word, not to individual bits.
 */

#define	i_bit_set(bit,l)						\
	__asm__ volatile("	lock		\n			\
				btsl	%0,%1"			:	\
								:	\
			"r" (bit), "m" (*(volatile int *)(l)));

#define	i_bit_clear(bit,l)						\
	__asm__ volatile("	lock		\n			\
				btrl	%0,%1"			:	\
								:	\
			"r" (bit), "m" (*(volatile int *)(l)));

static inline unsigned long i_bit_isset(unsigned int test, volatile unsigned long *word)
{
	int	bit;

	__asm__ volatile("btl %2,%1\n\tsbbl %0,%0" : "=r" (bit)
		: "m" (word), "ir" (test));
	return bit;
}

static inline char	xchgb(volatile char * cp, char new);

static inline void	atomic_incl(volatile long * p, long delta);
static inline void	atomic_incs(volatile short * p, short delta);
static inline void	atomic_incb(volatile char * p, char delta);

static inline void	atomic_decl(volatile long * p, long delta);
static inline void	atomic_decs(volatile short * p, short delta);
static inline void	atomic_decb(volatile char * p, char delta);

static inline long	atomic_getl(const volatile long * p);
static inline short	atomic_gets(const volatile short * p);
static inline char	atomic_getb(const volatile char * p);

static inline void	atomic_setl(volatile long * p, long value);
static inline void	atomic_sets(volatile short * p, short value);
static inline void	atomic_setb(volatile char * p, char value);

static inline char	xchgb(volatile char * cp, char new)
{
	register char	old = new;

	__asm__ volatile ("	xchgb	%0,%2"			:
			"=q" (old)				:
			"0" (new), "m" (*(volatile char *)cp) : "memory");
	return (old);
}

/*
 * Compare and exchange:
 * - returns failure (0) if the location did not contain the old value,
 * - returns success (1) if the location was set to the new value.
 */
static inline uint32_t
atomic_cmpxchg(uint32_t *p, uint32_t old, uint32_t new)
{
	uint32_t res = old;

	asm volatile(
		"lock;	cmpxchgl	%1,%2;	\n\t"
		"	setz		%%al;	\n\t"
		"	movzbl		%%al,%0"
		: "+a" (res)	/* %0: old value to compare, returns success */
		: "r" (new),	/* %1: new value to set */
		  "m" (*(p))	/* %2: memory address */
		: "memory");
	return (res);
}

static inline void	atomic_incl(volatile long * p, long delta)
{
	__asm__ volatile ("	lock		\n		\
				addl    %0,%1"		:	\
							:	\
				"r" (delta), "m" (*(volatile long *)p));
}

static inline void	atomic_incs(volatile short * p, short delta)
{
	__asm__ volatile ("	lock		\n		\
				addw    %0,%1"		:	\
							:	\
				"q" (delta), "m" (*(volatile short *)p));
}

static inline void	atomic_incb(volatile char * p, char delta)
{
	__asm__ volatile ("	lock		\n		\
				addb    %0,%1"		:	\
							:	\
				"q" (delta), "m" (*(volatile char *)p));
}

static inline void	atomic_decl(volatile long * p, long delta)
{
	__asm__ volatile ("	lock		\n		\
				subl	%0,%1"		:	\
							:	\
				"r" (delta), "m" (*(volatile long *)p));
}

static inline int	atomic_decl_and_test(volatile long * p, long delta)
{
	uint8_t	ret;
	asm volatile (
		"	lock		\n\t"
		"	subl	%1,%2	\n\t"
		"	sete	%0"
		: "=qm" (ret)
		: "r" (delta), "m" (*(volatile long *)p));
	return ret;
}

static inline void	atomic_decs(volatile short * p, short delta)
{
	__asm__ volatile ("	lock		\n		\
				subw    %0,%1"		:	\
							:	\
				"q" (delta), "m" (*(volatile short *)p));
}

static inline void	atomic_decb(volatile char * p, char delta)
{
	__asm__ volatile ("	lock		\n		\
				subb    %0,%1"		:	\
							:	\
				"q" (delta), "m" (*(volatile char *)p));
}

static inline long	atomic_getl(const volatile long * p)
{
	return (*p);
}

static inline short	atomic_gets(const volatile short * p)
{
	return (*p);
}

static inline char	atomic_getb(const volatile char * p)
{
	return (*p);
}

static inline void	atomic_setl(volatile long * p, long value)
{
	*p = value;
}

static inline void	atomic_sets(volatile short * p, short value)
{
	*p = value;
}

static inline void	atomic_setb(volatile char * p, char value)
{
	*p = value;
}


#else	/* !defined(__GNUC__) */

extern void	i_bit_set(
	int index,
	void *addr);

extern void	i_bit_clear(
	int index,
	void *addr);

extern void bit_lock(
	int index,
	void *addr);

extern void bit_unlock(
	int index,
	void *addr);

/*
 * All other routines defined in __GNUC__ case lack
 * definitions otherwise. - XXX
 */

#endif	/* !defined(__GNUC__) */

extern void		kernel_preempt_check (void);

#endif /* MACH_KERNEL_PRIVATE */

#endif /* __APLE_API_PRIVATE */

#endif	/* _I386_LOCK_H_ */

#endif	/* KERNEL_PRIVATE */
Commit	Line	Data
1c79356b	1	/*
91447636	2	* Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
1c79356b	3	*
8f6c56a5	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
1c79356b	5	*
8f6c56a5 A	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
8ad349bb	24	* limitations under the License.
8f6c56a5 A	25	*
8f6c56a5 A	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
1c79356b A	27	*/
	28	/*
	29	* Copyright (C) 1998 Apple Computer
	30	* All Rights Reserved
	31	*/
	32	/*
	33	* @OSF_COPYRIGHT@
	34	*/
	35	/*
	36	* Mach Operating System
	37	* Copyright (c) 1991,1990 Carnegie Mellon University
	38	* All Rights Reserved.
	39	*
	40	* Permission to use, copy, modify and distribute this software and its
	41	* documentation is hereby granted, provided that both the copyright
	42	* notice and this permission notice appear in all copies of the
	43	* software, derivative works or modified versions, and any portions
	44	* thereof, and that both notices appear in supporting documentation.
	45	*
	46	* CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
	47	* CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
	48	* ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
	49	*
	50	* Carnegie Mellon requests users of this software to return to
	51	*
	52	* Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
	53	* School of Computer Science
	54	* Carnegie Mellon University
	55	* Pittsburgh PA 15213-3890
	56	*
	57	* any improvements or extensions that they make and grant Carnegie Mellon
	58	* the rights to redistribute these changes.
	59	*/
	60
	61	/*
	62	*/
	63
	64	/*
	65	* Machine-dependent simple locks for the i386.
	66	*/
91447636	67	#ifdef KERNEL_PRIVATE
1c79356b A	68
	69	#ifndef _I386_LOCK_H_
	70	#define _I386_LOCK_H_
	71
9bccf70c A	72	#include <sys/appleapiopts.h>
	73
	74	#ifdef __APPLE_API_PRIVATE
	75
	76	#ifdef MACH_KERNEL_PRIVATE
	77
1c79356b A	78	#include <kern/macro_help.h>
	79	#include <kern/assert.h>
	80	#include <i386/hw_lock_types.h>
91447636	81	#include <i386/locks.h>
1c79356b	82
1c79356b A	83	#include <mach_rt.h>
1c79356b A	84	#include <mach_ldebug.h>
91447636 A	85
	86	typedef struct {
	87	lck_mtx_t lck_mtx; /* inlined lck_mtx, need to be first */
	88	#if MACH_LDEBUG
	89	int type;
	90	#define MUTEX_TAG 0x4d4d
	91	vm_offset_t pc;
	92	vm_offset_t thread;
	93	#endif /* MACH_LDEBUG */
	94	} mutex_t;
	95
4452a7af	96	typedef lck_rw_t lock_t;
91447636 A	97
91447636 A	98	extern unsigned int LockTimeOut; /* Number of hardware ticks of a lock timeout */
1c79356b A	99
	100
	101	#if defined(__GNUC__)
	102
	103	/*
	104	* General bit-lock routines.
	105	*/
	106
	107	#define bit_lock(bit,l) \
	108	__asm__ volatile(" jmp 1f \n \
	109	0: btl %0, %1 \n \
	110	jb 0b \n \
	111	1: lock \n \
	112	btsl %0,%1 \n \
	113	jb 0b" : \
	114	: \
	115	"r" (bit), "m" ((volatile int )(l)) : \
	116	"memory");
	117
	118	#define bit_unlock(bit,l) \
	119	__asm__ volatile(" lock \n \
	120	btrl %0,%1" : \
	121	: \
	122	"r" (bit), "m" ((volatile int )(l)));
	123
	124	/*
	125	* Set or clear individual bits in a long word.
	126	* The locked access is needed only to lock access
	127	* to the word, not to individual bits.
	128	*/
	129
	130	#define i_bit_set(bit,l) \
	131	__asm__ volatile(" lock \n \
	132	btsl %0,%1" : \
	133	: \
	134	"r" (bit), "m" ((volatile int )(l)));
	135
	136	#define i_bit_clear(bit,l) \
	137	__asm__ volatile(" lock \n \
	138	btrl %0,%1" : \
	139	: \
	140	"r" (bit), "m" ((volatile int )(l)));
	141
91447636	142	static inline unsigned long i_bit_isset(unsigned int test, volatile unsigned long *word)
1c79356b A	143	{
	144	int bit;
	145
	146	__asm__ volatile("btl %2,%1\n\tsbbl %0,%0" : "=r" (bit)
91447636	147	: "m" (word), "ir" (test));
1c79356b A	148	return bit;
	149	}
	150
91447636	151	static inline char xchgb(volatile char * cp, char new);
1c79356b	152
4452a7af A	153	static inline void atomic_incl(volatile long * p, long delta);
	154	static inline void atomic_incs(volatile short * p, short delta);
	155	static inline void atomic_incb(volatile char * p, char delta);
1c79356b	156
4452a7af A	157	static inline void atomic_decl(volatile long * p, long delta);
	158	static inline void atomic_decs(volatile short * p, short delta);
	159	static inline void atomic_decb(volatile char * p, char delta);
1c79356b	160
4452a7af A	161	static inline long atomic_getl(const volatile long * p);
	162	static inline short atomic_gets(const volatile short * p);
	163	static inline char atomic_getb(const volatile char * p);
1c79356b	164
4452a7af A	165	static inline void atomic_setl(volatile long * p, long value);
	166	static inline void atomic_sets(volatile short * p, short value);
	167	static inline void atomic_setb(volatile char * p, char value);
1c79356b	168
91447636	169	static inline char xchgb(volatile char * cp, char new)
1c79356b A	170	{
	171	register char old = new;
	172
	173	__asm__ volatile (" xchgb %0,%2" :
	174	"=q" (old) :
	175	"0" (new), "m" ((volatile char )cp) : "memory");
	176	return (old);
	177	}
	178
91447636 A	179	/*
	180	* Compare and exchange:
	181	* - returns failure (0) if the location did not contain the old value,
	182	* - returns success (1) if the location was set to the new value.
	183	*/
	184	static inline uint32_t
	185	atomic_cmpxchg(uint32_t *p, uint32_t old, uint32_t new)
	186	{
	187	uint32_t res = old;
	188
	189	asm volatile(
	190	"lock; cmpxchgl %1,%2; \n\t"
	191	" setz %%al; \n\t"
	192	" movzbl %%al,%0"
	193	: "+a" (res) /* %0: old value to compare, returns success */
	194	: "r" (new), /* %1: new value to set */
	195	"m" ((p)) / %2: memory address */
	196	: "memory");
	197	return (res);
	198	}
	199
4452a7af	200	static inline void atomic_incl(volatile long * p, long delta)
1c79356b	201	{
1c79356b A	202	__asm__ volatile (" lock \n \
	203	addl %0,%1" : \
	204	: \
	205	"r" (delta), "m" ((volatile long )p));
1c79356b A	206	}
1c79356b A	207
4452a7af	208	static inline void atomic_incs(volatile short * p, short delta)
1c79356b	209	{
1c79356b A	210	__asm__ volatile (" lock \n \
	211	addw %0,%1" : \
	212	: \
	213	"q" (delta), "m" ((volatile short )p));
1c79356b A	214	}
1c79356b A	215
4452a7af	216	static inline void atomic_incb(volatile char * p, char delta)
1c79356b	217	{
1c79356b A	218	__asm__ volatile (" lock \n \
	219	addb %0,%1" : \
	220	: \
	221	"q" (delta), "m" ((volatile char )p));
1c79356b A	222	}
1c79356b A	223
4452a7af	224	static inline void atomic_decl(volatile long * p, long delta)
1c79356b	225	{
1c79356b A	226	__asm__ volatile (" lock \n \
	227	subl %0,%1" : \
	228	: \
	229	"r" (delta), "m" ((volatile long )p));
1c79356b A	230	}
1c79356b A	231
4452a7af	232	static inline int atomic_decl_and_test(volatile long * p, long delta)
91447636 A	233	{
	234	uint8_t ret;
	235	asm volatile (
	236	" lock \n\t"
	237	" subl %1,%2 \n\t"
	238	" sete %0"
	239	: "=qm" (ret)
	240	: "r" (delta), "m" ((volatile long )p));
	241	return ret;
	242	}
	243
4452a7af	244	static inline void atomic_decs(volatile short * p, short delta)
1c79356b	245	{
1c79356b A	246	__asm__ volatile (" lock \n \
	247	subw %0,%1" : \
	248	: \
	249	"q" (delta), "m" ((volatile short )p));
1c79356b A	250	}
1c79356b A	251
4452a7af	252	static inline void atomic_decb(volatile char * p, char delta)
1c79356b	253	{
1c79356b A	254	__asm__ volatile (" lock \n \
	255	subb %0,%1" : \
	256	: \
	257	"q" (delta), "m" ((volatile char )p));
1c79356b A	258	}
1c79356b A	259
4452a7af	260	static inline long atomic_getl(const volatile long * p)
1c79356b A	261	{
	262	return (*p);
	263	}
	264
4452a7af	265	static inline short atomic_gets(const volatile short * p)
1c79356b A	266	{
	267	return (*p);
	268	}
	269
4452a7af	270	static inline char atomic_getb(const volatile char * p)
1c79356b A	271	{
	272	return (*p);
	273	}
	274
4452a7af	275	static inline void atomic_setl(volatile long * p, long value)
1c79356b A	276	{
	277	*p = value;
	278	}
	279
4452a7af	280	static inline void atomic_sets(volatile short * p, short value)
1c79356b A	281	{
	282	*p = value;
	283	}
	284
4452a7af	285	static inline void atomic_setb(volatile char * p, char value)
1c79356b A	286	{
	287	*p = value;
	288	}
	289
	290
	291	#else /* !defined(__GNUC__) */
	292
	293	extern void i_bit_set(
	294	int index,
	295	void *addr);
	296
	297	extern void i_bit_clear(
	298	int index,
	299	void *addr);
	300
	301	extern void bit_lock(
	302	int index,
	303	void *addr);
	304
	305	extern void bit_unlock(
	306	int index,
	307	void *addr);
	308
	309	/*
	310	* All other routines defined in __GNUC__ case lack
	311	* definitions otherwise. - XXX
	312	*/
	313
	314	#endif /* !defined(__GNUC__) */
	315
9bccf70c A	316	extern void kernel_preempt_check (void);
9bccf70c A	317
0b4e3aa0	318	#endif /* MACH_KERNEL_PRIVATE */
1c79356b	319
9bccf70c	320	#endif /* __APLE_API_PRIVATE */
1c79356b A	321
1c79356b A	322	#endif /* _I386_LOCK_H_ */
91447636 A	323
91447636 A	324	#endif /* KERNEL_PRIVATE */