[apple/xnu.git] / osfmk / ppc / commpage / commpage_asm.s

/*
 * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * Copyright (c) 1999-2003 Apple Computer, Inc.  All Rights Reserved.
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this
 * file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */

#include <sys/appleapiopts.h>
#include <ppc/asm.h>
#include <ppc/proc_reg.h>
#include <machine/cpu_capabilities.h>
#include <machine/commpage.h>


// commpage_time_dcba() uses a stack frame as follows:

#define	kBufSiz		1024				// Size of the buffer we use to do DCBA timing on G4
#define	kSFSize		(kBufSiz+128+16)	// Stack frame size, which contains the 128-byte-aligned buffer
#define	kLoopCnt	5					// Iterations of the timing loop
#define	kDCBA		22					// Bit in cr5 used as a flag in timing loop


// commpage_set_timestamp() uses the red zone for temporary storage:

#define	rzSaveF1			-8		// caller's FPR1
#define	rzSaveF2			-16		// caller's FPR2
#define	rzSaveF3			-24		// caller's FPR3
#define	rzSaveF4			-32		// caller's FPR4
#define	rzSaveF5			-40		// caller's FPR5
#define	rzNewTimeBase		-48		// used to load 64-bit TBR into a FPR


// commpage_set_timestamp() uses the following data.  kkTicksPerSec remembers
// the number used to compute _COMM_PAGE_SEC_PER_TICK.  Since this constant
// rarely changes, we use it to avoid needless recomputation.  It is a double
// value, pre-initialize with an exponent of 2**52.

#define	kkBinary0		0					// offset in data to long long 0 (a constant)
#define	kkDouble1		8					// offset in data to double 1.0 (a constant)
#define	kkTicksPerSec	16					// offset in data to double(ticks_per_sec)

        .data
        .align	3							// three doubleword fields
Ldata:
        .long	0							// kkBinary0
        .long	0
        .double	1.0e0						// kkDouble1        
        .long	0x43300000					// kkTicksPerSec (plus 2**52)
        .long	0							// this is where we store ticks_per_sec, to float

        .text
        .align	2
        .globl	EXT(commpage_time_dcba)
        .globl	EXT(commpage_set_timestamp)


/*	***********************************************
 *	* C O M M P A G E _ S E T _ T I M E S T A M P *
 *	***********************************************
 *
 *	Update the gettimeofday() shared data on the commpage, as follows:
 *		_COMM_PAGE_TIMESTAMP = a BSD-style pair of uint_32's for secs and usecs
 *		_COMM_PAGE_TIMEBASE = the timebase at which the timestamp was valid
 *		_COMM_PAGE_SEC_PER_TICK = multiply timebase ticks by this to get seconds (double)
 *	The convention is that if the timebase is 0, the data is invalid.  Because other
 *	CPUs are reading the three values asynchronously and must get a consistent set, 
 *	it is critical that we update them with the following protocol:
 *		1. set timebase to 0 (atomically), to invalidate all three values
 *		2. eieio (to create a barrier in stores to cacheable memory)
 *		3. change timestamp and "secs per tick"
 *		4. eieio
 *		5. set timebase nonzero (atomically)
 *	This works because readers read the timebase, then the timestamp and divisor, sync
 *	if MP, then read the timebase a second time and check to be sure it is equal to the first.
 *
 *	We could save a few cycles on 64-bit machines by special casing them, but it probably
 *	isn't necessary because this routine shouldn't be called very often.
 *
 *	When called:
 *		r3 = upper half of timebase (timebase is disabled if 0)
 *		r4 = lower half of timebase
 *		r5 = seconds part of timestamp
 *		r6 = useconds part of timestamp
 *		r7 = divisor (ie, timebase ticks per sec)
 *	We set up:
 *		r8 = ptr to our static data (kkBinary0, kkDouble1, kkTicksPerSec)
 *		r9 = ptr to comm page in kernel map
 *
 *	--> Interrupts must be disabled and rtclock locked when called.  <--
 */
 
        .align	5
LEXT(commpage_set_timestamp)				// void commpage_set_timestamp(tbr,secs,usecs,divisor)
        mfmsr	r11							// get MSR
        ori		r2,r11,MASK(MSR_FP)			// turn FP on
        mtmsr	r2
        isync								// wait until MSR changes take effect
        
        or.		r0,r3,r4					// is timebase 0? (thus disabled)
        lis		r8,hi16(Ldata)				// point to our data
        lis		r9,ha16(EXT(commPagePtr))	// get ptr to address of commpage in kernel map
        stfd	f1,rzSaveF1(r1)				// save a FPR in the red zone
        ori		r8,r8,lo16(Ldata)
        lwz		r9,lo16(EXT(commPagePtr))(r9)	// r9 <- commPagePtr
        lfd		f1,kkBinary0(r8)			// get fixed 0s
        li		r0,_COMM_PAGE_BASE_ADDRESS	// get va in user space of commpage
        cmpwi	cr1,r9,0					// is commpage allocated yet?
        sub		r9,r9,r0					// r9 <- commpage address, biased by user va
        beq--	cr1,3f						// skip if not allocated
        stfd	f1,_COMM_PAGE_TIMEBASE(r9)	// turn off the timestamp (atomically)
        eieio								// make sure all CPUs see it is off
        beq		3f							// all we had to do is turn off timestamp
        
        lwz		r0,kkTicksPerSec+4(r8)		// get last ticks_per_sec (or 0 if first)
        stw		r3,rzNewTimeBase(r1)		// store new timebase so we can lfd
        stw		r4,rzNewTimeBase+4(r1)
        cmpw	r0,r7						// do we need to recompute _COMM_PAGE_SEC_PER_TICK?
        stw		r5,_COMM_PAGE_TIMESTAMP(r9)	// store the new timestamp
        stw		r6,_COMM_PAGE_TIMESTAMP+4(r9)
        lfd		f1,rzNewTimeBase(r1)		// get timebase in a FPR so we can store atomically
        beq++	2f							// same ticks_per_sec, no need to recompute
        
        stw		r7,kkTicksPerSec+4(r8)		// must recompute SEC_PER_TICK
        stfd	f2,rzSaveF2(r1)				// we'll need a few more temp FPRs
        stfd	f3,rzSaveF3(r1)
        stfd	f4,rzSaveF4(r1)
        stfd	f5,rzSaveF5(r1)
        lfd		f2,_COMM_PAGE_2_TO_52(r9)	// f2 <- double(2**52)
        lfd		f3,kkTicksPerSec(r8)		// float new ticks_per_sec + 2**52
        lfd		f4,kkDouble1(r8)			// f4 <- double(1.0)
        mffs	f5							// save caller's FPSCR
        mtfsfi	7,0							// clear Inexeact Exception bit, set round-to-nearest
        fsub	f3,f3,f2					// get ticks_per_sec
        fdiv	f3,f4,f3					// divide 1 by ticks_per_sec to get SEC_PER_TICK
        stfd	f3,_COMM_PAGE_SEC_PER_TICK(r9)
        mtfsf	0xFF,f5						// restore FPSCR
        lfd		f2,rzSaveF2(r1)				// restore FPRs
        lfd		f3,rzSaveF3(r1)
        lfd		f4,rzSaveF4(r1)
        lfd		f5,rzSaveF5(r1)
2:											// f1 == new timestamp
        eieio								// wait until the stores take
        stfd	f1,_COMM_PAGE_TIMEBASE(r9)	// then turn the timestamp back on (atomically)
3:											// here once all fields updated
        lfd		f1,rzSaveF1(r1)				// restore last FPR
        mtmsr	r11							// turn FP back off
        isync
        blr


/*	***************************************
 *	* C O M M P A G E _ T I M E _ D C B A *
 *	***************************************
 *
 *	Not all processors that support the DCBA opcode actually benefit from it.
 *	Some store-gather and read-cancel well enough that there is no need to use
 *	DCBA to avoid fetching cache lines that will be completely overwritten, while
 *	others have this feature disabled (to work around errata etc), and so benefit
 *	from DCBA.  Since it is hard to tell the one group from the other, we just
 *	time loops with and without DCBA, and pick the fastest.  Thus we avoid
 *	delicate dependence on processor and/or platform revisions.
 *
 *	We return either kDcbaRecommended or zero.
 *
 *		int commpage_time_dcba( void );
 */
 
LEXT(commpage_time_dcba)
        mflr	r12					// get return
        stw		r12,8(r1)			// save
        stwu	r1,-kSFSize(r1)		// carve our temp buffer from the stack
        addi	r11,r1,127+16		// get base address...
        rlwinm	r11,r11,0,0,24		// ...of our buffer, 128-byte aligned
        crset	kDCBA				// first, use DCBA
        bl		LTest				// time it with DCBA
        srwi	r0,r3,3				// bias 12 pct in favor of not using DCBA...
        add		r10,r3,r0			// ...because DCBA is always slower with warm cache
        crclr	kDCBA
        bl		LTest				// time without DCBA
        cmplw	r10,r3				// which is better?
        mtlr	r12					// restore return
        lwz		r1,0(r1)			// pop off our stack frame
        li		r3,kDcbaRecommended		// assume using DCBA is faster
        bltlr
        li		r3,0			// no DCBA is faster
        blr
                
        
// Subroutine to time a loop with or without DCBA.
//		kDCBA = set if we should use DCBA
//		r11 = base of buffer to use for test (kBufSiz bytes)
//
//		We return TBR ticks in r3.
//		We use r0,r3-r9.

LTest:
        li		r4,kLoopCnt			// number of times to loop
        li		r3,-1				// initialize fastest time
1:
        mr		r6,r11				// initialize buffer ptr
        li		r0,kBufSiz/32		// r0 <- cache blocks to test
        mtctr	r0
2:
        dcbf	0,r6				// first, force the blocks out of the cache
        addi	r6,r6,32
        bdnz	2b
        sync						// make sure all the flushes take
        mr		r6,r11				// re-initialize buffer ptr
        mtctr	r0					// reset cache-block count
        mftbu	r7					// remember upper half so we can check for carry
        mftb	r8					// start the timer
3:									// loop over cache blocks
        bf		kDCBA,4f			// should we DCBA?
        dcba	0,r6
4:
        stw		r0,0(r6)			// store the entire cache block
        stw		r0,4(r6)
        stw		r0,8(r6)
        stw		r0,12(r6)
        stw		r0,16(r6)
        stw		r0,20(r6)
        stw		r0,24(r6)
        stw		r0,28(r6)
        addi	r6,r6,32
        bdnz	3b
        mftb	r9
        mftbu	r0
        cmpw	r0,r7				// did timebase carry?
        bne		1b					// yes, retest rather than fuss
        sub		r9,r9,r8			// r9 <- time for this loop
        cmplw	r9,r3				// faster than current best?
        bge		5f					// no
        mr		r3,r9				// remember fastest time through loop
5:
        subi	r4,r4,1				// decrement outer loop count
        cmpwi	r4,0				// more to go?
        bne		1b					// loop if so
        blr							// return fastest time in r3
Commit	Line	Data
d7e50217 A	1	/*
	2	* Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_LICENSE_HEADER_START@
	5	*
	6	* Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved.
	7	*
	8	* This file contains Original Code and/or Modifications of Original Code
	9	* as defined in and that are subject to the Apple Public Source License
	10	* Version 2.0 (the 'License'). You may not use this file except in
	11	* compliance with the License. Please obtain a copy of the License at
	12	* http://www.opensource.apple.com/apsl/ and read it before using this
	13	* file.
	14	*
	15	* The Original Code and all software distributed under the License are
	16	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	17	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	18	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	19	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	20	* Please see the License for the specific language governing rights and
	21	* limitations under the License.
	22	*
	23	* @APPLE_LICENSE_HEADER_END@
	24	*/
	25
	26	#include <sys/appleapiopts.h>
	27	#include <ppc/asm.h>
	28	#include <ppc/proc_reg.h>
	29	#include <machine/cpu_capabilities.h>
	30	#include <machine/commpage.h>
	31
	32
	33	// commpage_time_dcba() uses a stack frame as follows:
	34
	35	#define kBufSiz 1024 // Size of the buffer we use to do DCBA timing on G4
	36	#define kSFSize (kBufSiz+128+16) // Stack frame size, which contains the 128-byte-aligned buffer
	37	#define kLoopCnt 5 // Iterations of the timing loop
	38	#define kDCBA 22 // Bit in cr5 used as a flag in timing loop
	39
	40
	41	// commpage_set_timestamp() uses the red zone for temporary storage:
	42
	43	#define rzSaveF1 -8 // caller's FPR1
	44	#define rzSaveF2 -16 // caller's FPR2
	45	#define rzSaveF3 -24 // caller's FPR3
	46	#define rzSaveF4 -32 // caller's FPR4
	47	#define rzSaveF5 -40 // caller's FPR5
	48	#define rzNewTimeBase -48 // used to load 64-bit TBR into a FPR
	49
	50
	51	// commpage_set_timestamp() uses the following data. kkTicksPerSec remembers
	52	// the number used to compute _COMM_PAGE_SEC_PER_TICK. Since this constant
	53	// rarely changes, we use it to avoid needless recomputation. It is a double
	54	// value, pre-initialize with an exponent of 2**52.
	55
	56	#define kkBinary0 0 // offset in data to long long 0 (a constant)
	57	#define kkDouble1 8 // offset in data to double 1.0 (a constant)
	58	#define kkTicksPerSec 16 // offset in data to double(ticks_per_sec)
	59
	60	.data
	61	.align 3 // three doubleword fields
	62	Ldata:
	63	.long 0 // kkBinary0
	64	.long 0
65	.double 1.0e0 // kkDouble1
66	.long 0x43300000 // kkTicksPerSec (plus 2**52)
67	.long 0 // this is where we store ticks_per_sec, to float
68
69	.text
70	.align 2
71	.globl EXT(commpage_time_dcba)
72	.globl EXT(commpage_set_timestamp)
73
74
75	/* ***********************************************
76	* * C O M M P A G E _ S E T _ T I M E S T A M P *
77	* ***********************************************
78	*
79	* Update the gettimeofday() shared data on the commpage, as follows:
80	* _COMM_PAGE_TIMESTAMP = a BSD-style pair of uint_32's for secs and usecs
81	* _COMM_PAGE_TIMEBASE = the timebase at which the timestamp was valid
82	* _COMM_PAGE_SEC_PER_TICK = multiply timebase ticks by this to get seconds (double)
83	* The convention is that if the timebase is 0, the data is invalid. Because other
84	* CPUs are reading the three values asynchronously and must get a consistent set,
85	* it is critical that we update them with the following protocol:
86	* 1. set timebase to 0 (atomically), to invalidate all three values
87	* 2. eieio (to create a barrier in stores to cacheable memory)
88	* 3. change timestamp and "secs per tick"
89	* 4. eieio
90	* 5. set timebase nonzero (atomically)
91	* This works because readers read the timebase, then the timestamp and divisor, sync
92	* if MP, then read the timebase a second time and check to be sure it is equal to the first.
93	*
94	* We could save a few cycles on 64-bit machines by special casing them, but it probably
95	* isn't necessary because this routine shouldn't be called very often.
96	*
97	* When called:
98	* r3 = upper half of timebase (timebase is disabled if 0)
99	* r4 = lower half of timebase
100	* r5 = seconds part of timestamp
101	* r6 = useconds part of timestamp
102	* r7 = divisor (ie, timebase ticks per sec)
103	* We set up:
104	* r8 = ptr to our static data (kkBinary0, kkDouble1, kkTicksPerSec)
105	* r9 = ptr to comm page in kernel map
106	*
107	* --> Interrupts must be disabled and rtclock locked when called. <--
108	*/
109
110	.align 5
111	LEXT(commpage_set_timestamp) // void commpage_set_timestamp(tbr,secs,usecs,divisor)
112	mfmsr r11 // get MSR
113	ori r2,r11,MASK(MSR_FP) // turn FP on
114	mtmsr r2
115	isync // wait until MSR changes take effect
116
117	or. r0,r3,r4 // is timebase 0? (thus disabled)
118	lis r8,hi16(Ldata) // point to our data
119	lis r9,ha16(EXT(commPagePtr)) // get ptr to address of commpage in kernel map
120	stfd f1,rzSaveF1(r1) // save a FPR in the red zone
121	ori r8,r8,lo16(Ldata)
122	lwz r9,lo16(EXT(commPagePtr))(r9) // r9 <- commPagePtr
123	lfd f1,kkBinary0(r8) // get fixed 0s
124	li r0,_COMM_PAGE_BASE_ADDRESS // get va in user space of commpage
125	cmpwi cr1,r9,0 // is commpage allocated yet?
126	sub r9,r9,r0 // r9 <- commpage address, biased by user va
127	beq-- cr1,3f // skip if not allocated
128	stfd f1,_COMM_PAGE_TIMEBASE(r9) // turn off the timestamp (atomically)
129	eieio // make sure all CPUs see it is off
130	beq 3f // all we had to do is turn off timestamp
131
132	lwz r0,kkTicksPerSec+4(r8) // get last ticks_per_sec (or 0 if first)
133	stw r3,rzNewTimeBase(r1) // store new timebase so we can lfd
134	stw r4,rzNewTimeBase+4(r1)
135	cmpw r0,r7 // do we need to recompute _COMM_PAGE_SEC_PER_TICK?
136	stw r5,_COMM_PAGE_TIMESTAMP(r9) // store the new timestamp
137	stw r6,_COMM_PAGE_TIMESTAMP+4(r9)
138	lfd f1,rzNewTimeBase(r1) // get timebase in a FPR so we can store atomically
139	beq++ 2f // same ticks_per_sec, no need to recompute
140
141	stw r7,kkTicksPerSec+4(r8) // must recompute SEC_PER_TICK
142	stfd f2,rzSaveF2(r1) // we'll need a few more temp FPRs
143	stfd f3,rzSaveF3(r1)
144	stfd f4,rzSaveF4(r1)
145	stfd f5,rzSaveF5(r1)
146	lfd f2,_COMM_PAGE_2_TO_52(r9) // f2 <- double(2**52)
147	lfd f3,kkTicksPerSec(r8) // float new ticks_per_sec + 2**52
148	lfd f4,kkDouble1(r8) // f4 <- double(1.0)
149	mffs f5 // save caller's FPSCR
150	mtfsfi 7,0 // clear Inexeact Exception bit, set round-to-nearest
151	fsub f3,f3,f2 // get ticks_per_sec
152	fdiv f3,f4,f3 // divide 1 by ticks_per_sec to get SEC_PER_TICK
153	stfd f3,_COMM_PAGE_SEC_PER_TICK(r9)
154	mtfsf 0xFF,f5 // restore FPSCR
155	lfd f2,rzSaveF2(r1) // restore FPRs
156	lfd f3,rzSaveF3(r1)
157	lfd f4,rzSaveF4(r1)
158	lfd f5,rzSaveF5(r1)
159	2: // f1 == new timestamp
160	eieio // wait until the stores take
161	stfd f1,_COMM_PAGE_TIMEBASE(r9) // then turn the timestamp back on (atomically)
162	3: // here once all fields updated
163	lfd f1,rzSaveF1(r1) // restore last FPR
164	mtmsr r11 // turn FP back off
165	isync
166	blr
167
168
169	/* ***************************************
170	* * C O M M P A G E _ T I M E _ D C B A *
171	* ***************************************
172	*
173	* Not all processors that support the DCBA opcode actually benefit from it.
174	* Some store-gather and read-cancel well enough that there is no need to use
175	* DCBA to avoid fetching cache lines that will be completely overwritten, while
176	* others have this feature disabled (to work around errata etc), and so benefit
177	* from DCBA. Since it is hard to tell the one group from the other, we just
178	* time loops with and without DCBA, and pick the fastest. Thus we avoid
179	* delicate dependence on processor and/or platform revisions.
180	*
181	* We return either kDcbaRecommended or zero.
182	*
183	* int commpage_time_dcba( void );
184	*/
185
186	LEXT(commpage_time_dcba)
187	mflr r12 // get return
188	stw r12,8(r1) // save
189	stwu r1,-kSFSize(r1) // carve our temp buffer from the stack
190	addi r11,r1,127+16 // get base address...
191	rlwinm r11,r11,0,0,24 // ...of our buffer, 128-byte aligned
192	crset kDCBA // first, use DCBA
193	bl LTest // time it with DCBA
194	srwi r0,r3,3 // bias 12 pct in favor of not using DCBA...
195	add r10,r3,r0 // ...because DCBA is always slower with warm cache
196	crclr kDCBA
197	bl LTest // time without DCBA
198	cmplw r10,r3 // which is better?
199	mtlr r12 // restore return
200	lwz r1,0(r1) // pop off our stack frame
201	li r3,kDcbaRecommended // assume using DCBA is faster
202	bltlr
203	li r3,0 // no DCBA is faster
204	blr
205
206
207	// Subroutine to time a loop with or without DCBA.
208	// kDCBA = set if we should use DCBA
209	// r11 = base of buffer to use for test (kBufSiz bytes)
210	//
211	// We return TBR ticks in r3.
212	// We use r0,r3-r9.
213
214	LTest:
215	li r4,kLoopCnt // number of times to loop
216	li r3,-1 // initialize fastest time
217	1:
218	mr r6,r11 // initialize buffer ptr
219	li r0,kBufSiz/32 // r0 <- cache blocks to test
220	mtctr r0
221	2:
222	dcbf 0,r6 // first, force the blocks out of the cache
223	addi r6,r6,32
224	bdnz 2b
225	sync // make sure all the flushes take
226	mr r6,r11 // re-initialize buffer ptr
227	mtctr r0 // reset cache-block count
228	mftbu r7 // remember upper half so we can check for carry
229	mftb r8 // start the timer
230	3: // loop over cache blocks
231	bf kDCBA,4f // should we DCBA?
232	dcba 0,r6
233	4:
234	stw r0,0(r6) // store the entire cache block
235	stw r0,4(r6)
236	stw r0,8(r6)
237	stw r0,12(r6)
238	stw r0,16(r6)
239	stw r0,20(r6)
240	stw r0,24(r6)
241	stw r0,28(r6)
242	addi r6,r6,32
243	bdnz 3b
244	mftb r9
245	mftbu r0
246	cmpw r0,r7 // did timebase carry?
247	bne 1b // yes, retest rather than fuss
248	sub r9,r9,r8 // r9 <- time for this loop
249	cmplw r9,r3 // faster than current best?
250	bge 5f // no
251	mr r3,r9 // remember fastest time through loop
252	5:
253	subi r4,r4,1 // decrement outer loop count
254	cmpwi r4,0 // more to go?
255	bne 1b // loop if so
256	blr // return fastest time in r3