[apple/xnu.git] / osfmk / ppc / commpage / commpage_asm.s

/*
 * Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * The contents of this file constitute Original Code as defined in and
 * are subject to the Apple Public Source License Version 1.1 (the
 * "License").  You may not use this file except in compliance with the
 * License.  Please obtain a copy of the License at
 * http://www.apple.com/publicsource and read it before using this file.
 * 
 * This Original Code and all software distributed under the License are
 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
 * License for the specific language governing rights and limitations
 * under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */

#include <sys/appleapiopts.h>
#include <ppc/asm.h>
#include <ppc/proc_reg.h>
#include <machine/cpu_capabilities.h>
#include <machine/commpage.h>


// commpage_time_dcba() uses a stack frame as follows:

#define	kBufSiz		1024				// Size of the buffer we use to do DCBA timing on G4
#define	kSFSize		(kBufSiz+128+16)	// Stack frame size, which contains the 128-byte-aligned buffer
#define	kLoopCnt	5					// Iterations of the timing loop
#define	kDCBA		22					// Bit in cr5 used as a flag in timing loop


// commpage_set_timestamp() uses the red zone for temporary storage:

#define	rzSaveF1			-8		// caller's FPR1
#define	rzSaveF2			-16		// caller's FPR2
#define	rzSaveF3			-24		// caller's FPR3
#define	rzSaveF4			-32		// caller's FPR4
#define	rzSaveF5			-40		// caller's FPR5
#define	rzNewTimeBase		-48		// used to load 64-bit TBR into a FPR


// commpage_set_timestamp() uses the following data.  kkTicksPerSec remembers
// the number used to compute _COMM_PAGE_SEC_PER_TICK.  Since this constant
// rarely changes, we use it to avoid needless recomputation.  It is a double
// value, pre-initialize with an exponent of 2**52.

#define	kkBinary0		0					// offset in data to long long 0 (a constant)
#define	kkDouble1		8					// offset in data to double 1.0 (a constant)
#define	kkTicksPerSec	16					// offset in data to double(ticks_per_sec)

        .data
        .align	3							// three doubleword fields
Ldata:
        .long	0							// kkBinary0
        .long	0
        .double	1.0e0						// kkDouble1        
        .long	0x43300000					// kkTicksPerSec (plus 2**52)
        .long	0							// this is where we store ticks_per_sec, to float

        .text
        .align	2
        .globl	EXT(commpage_time_dcba)
        .globl	EXT(commpage_set_timestamp)


/*	***********************************************
 *	* C O M M P A G E _ S E T _ T I M E S T A M P *
 *	***********************************************
 *
 *	Update the gettimeofday() shared data on the commpage, as follows:
 *		_COMM_PAGE_TIMESTAMP = a BSD-style pair of uint_32's for secs and usecs
 *		_COMM_PAGE_TIMEBASE = the timebase at which the timestamp was valid
 *		_COMM_PAGE_SEC_PER_TICK = multiply timebase ticks by this to get seconds (double)
 *	The convention is that if the timebase is 0, the data is invalid.  Because other
 *	CPUs are reading the three values asynchronously and must get a consistent set, 
 *	it is critical that we update them with the following protocol:
 *		1. set timebase to 0 (atomically), to invalidate all three values
 *		2. eieio (to create a barrier in stores to cacheable memory)
 *		3. change timestamp and "secs per tick"
 *		4. eieio
 *		5. set timebase nonzero (atomically)
 *	This works because readers read the timebase, then the timestamp and divisor, sync
 *	if MP, then read the timebase a second time and check to be sure it is equal to the first.
 *
 *	We could save a few cycles on 64-bit machines by special casing them, but it probably
 *	isn't necessary because this routine shouldn't be called very often.
 *
 *	When called:
 *		r3 = upper half of timebase (timebase is disabled if 0)
 *		r4 = lower half of timebase
 *		r5 = seconds part of timestamp
 *		r6 = useconds part of timestamp
 *		r7 = divisor (ie, timebase ticks per sec)
 *	We set up:
 *		r8 = ptr to our static data (kkBinary0, kkDouble1, kkTicksPerSec)
 *		r9 = ptr to comm page in kernel map
 *
 *	--> Interrupts must be disabled and rtclock locked when called.  <--
 */
 
        .align	5
LEXT(commpage_set_timestamp)				// void commpage_set_timestamp(tbr,secs,usecs,divisor)
        mfmsr	r11							// get MSR
        ori		r2,r11,MASK(MSR_FP)			// turn FP on
        mtmsr	r2
        isync								// wait until MSR changes take effect
        
        or.		r0,r3,r4					// is timebase 0? (thus disabled)
        lis		r8,hi16(Ldata)				// point to our data
        lis		r9,ha16(EXT(commPagePtr))	// get ptr to address of commpage in kernel map
        stfd	f1,rzSaveF1(r1)				// save a FPR in the red zone
        ori		r8,r8,lo16(Ldata)
        lwz		r9,lo16(EXT(commPagePtr))(r9)	// r9 <- commPagePtr
        lfd		f1,kkBinary0(r8)			// get fixed 0s
        li		r0,_COMM_PAGE_BASE_ADDRESS	// get va in user space of commpage
        cmpwi	cr1,r9,0					// is commpage allocated yet?
        sub		r9,r9,r0					// r9 <- commpage address, biased by user va
        beq--	cr1,3f						// skip if not allocated
        stfd	f1,_COMM_PAGE_TIMEBASE(r9)	// turn off the timestamp (atomically)
        eieio								// make sure all CPUs see it is off
        beq		3f							// all we had to do is turn off timestamp
        
        lwz		r0,kkTicksPerSec+4(r8)		// get last ticks_per_sec (or 0 if first)
        stw		r3,rzNewTimeBase(r1)		// store new timebase so we can lfd
        stw		r4,rzNewTimeBase+4(r1)
        cmpw	r0,r7						// do we need to recompute _COMM_PAGE_SEC_PER_TICK?
        stw		r5,_COMM_PAGE_TIMESTAMP(r9)	// store the new timestamp
        stw		r6,_COMM_PAGE_TIMESTAMP+4(r9)
        lfd		f1,rzNewTimeBase(r1)		// get timebase in a FPR so we can store atomically
        beq++	2f							// same ticks_per_sec, no need to recompute
        
        stw		r7,kkTicksPerSec+4(r8)		// must recompute SEC_PER_TICK
        stfd	f2,rzSaveF2(r1)				// we'll need a few more temp FPRs
        stfd	f3,rzSaveF3(r1)
        stfd	f4,rzSaveF4(r1)
        stfd	f5,rzSaveF5(r1)
        lfd		f2,_COMM_PAGE_2_TO_52(r9)	// f2 <- double(2**52)
        lfd		f3,kkTicksPerSec(r8)		// float new ticks_per_sec + 2**52
        lfd		f4,kkDouble1(r8)			// f4 <- double(1.0)
        mffs	f5							// save caller's FPSCR
        mtfsfi	7,0							// clear Inexeact Exception bit, set round-to-nearest
        fsub	f3,f3,f2					// get ticks_per_sec
        fdiv	f3,f4,f3					// divide 1 by ticks_per_sec to get SEC_PER_TICK
        stfd	f3,_COMM_PAGE_SEC_PER_TICK(r9)
        mtfsf	0xFF,f5						// restore FPSCR
        lfd		f2,rzSaveF2(r1)				// restore FPRs
        lfd		f3,rzSaveF3(r1)
        lfd		f4,rzSaveF4(r1)
        lfd		f5,rzSaveF5(r1)
2:											// f1 == new timestamp
        eieio								// wait until the stores take
        stfd	f1,_COMM_PAGE_TIMEBASE(r9)	// then turn the timestamp back on (atomically)
3:											// here once all fields updated
        lfd		f1,rzSaveF1(r1)				// restore last FPR
        mtmsr	r11							// turn FP back off
        isync
        blr


/*	***************************************
 *	* C O M M P A G E _ T I M E _ D C B A *
 *	***************************************
 *
 *	Not all processors that support the DCBA opcode actually benefit from it.
 *	Some store-gather and read-cancel well enough that there is no need to use
 *	DCBA to avoid fetching cache lines that will be completely overwritten, while
 *	others have this feature disabled (to work around errata etc), and so benefit
 *	from DCBA.  Since it is hard to tell the one group from the other, we just
 *	time loops with and without DCBA, and pick the fastest.  Thus we avoid
 *	delicate dependence on processor and/or platform revisions.
 *
 *	We return either kDcbaRecommended or zero.
 *
 *		int commpage_time_dcba( void );
 */
 
LEXT(commpage_time_dcba)
        mflr	r12					// get return
        stw		r12,8(r1)			// save
        stwu	r1,-kSFSize(r1)		// carve our temp buffer from the stack
        addi	r11,r1,127+16		// get base address...
        rlwinm	r11,r11,0,0,24		// ...of our buffer, 128-byte aligned
        crset	kDCBA				// first, use DCBA
        bl		LTest				// time it with DCBA
        srwi	r0,r3,3				// bias 12 pct in favor of not using DCBA...
        add		r10,r3,r0			// ...because DCBA is always slower with warm cache
        crclr	kDCBA
        bl		LTest				// time without DCBA
        cmplw	r10,r3				// which is better?
        mtlr	r12					// restore return
        lwz		r1,0(r1)			// pop off our stack frame
        li		r3,kDcbaRecommended		// assume using DCBA is faster
        bltlr
        li		r3,0			// no DCBA is faster
        blr
                
        
// Subroutine to time a loop with or without DCBA.
//		kDCBA = set if we should use DCBA
//		r11 = base of buffer to use for test (kBufSiz bytes)
//
//		We return TBR ticks in r3.
//		We use r0,r3-r9.

LTest:
        li		r4,kLoopCnt			// number of times to loop
        li		r3,-1				// initialize fastest time
1:
        mr		r6,r11				// initialize buffer ptr
        li		r0,kBufSiz/32		// r0 <- cache blocks to test
        mtctr	r0
2:
        dcbf	0,r6				// first, force the blocks out of the cache
        addi	r6,r6,32
        bdnz	2b
        sync						// make sure all the flushes take
        mr		r6,r11				// re-initialize buffer ptr
        mtctr	r0					// reset cache-block count
        mftbu	r7					// remember upper half so we can check for carry
        mftb	r8					// start the timer
3:									// loop over cache blocks
        bf		kDCBA,4f			// should we DCBA?
        dcba	0,r6
4:
        stw		r0,0(r6)			// store the entire cache block
        stw		r0,4(r6)
        stw		r0,8(r6)
        stw		r0,12(r6)
        stw		r0,16(r6)
        stw		r0,20(r6)
        stw		r0,24(r6)
        stw		r0,28(r6)
        addi	r6,r6,32
        bdnz	3b
        mftb	r9
        mftbu	r0
        cmpw	r0,r7				// did timebase carry?
        bne		1b					// yes, retest rather than fuss
        sub		r9,r9,r8			// r9 <- time for this loop
        cmplw	r9,r3				// faster than current best?
        bge		5f					// no
        mr		r3,r9				// remember fastest time through loop
5:
        subi	r4,r4,1				// decrement outer loop count
        cmpwi	r4,0				// more to go?
        bne		1b					// loop if so
        blr							// return fastest time in r3
Commit	Line	Data
43866e37 A	1	/*
	2	* Copyright (c) 2003 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_LICENSE_HEADER_START@
	5	*
e5568f75 A	6	* The contents of this file constitute Original Code as defined in and
	7	* are subject to the Apple Public Source License Version 1.1 (the
	8	* "License"). You may not use this file except in compliance with the
	9	* License. Please obtain a copy of the License at
	10	* http://www.apple.com/publicsource and read it before using this file.
43866e37	11	*
e5568f75 A	12	* This Original Code and all software distributed under the License are
e5568f75 A	13	* distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
43866e37 A	14	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
43866e37 A	15	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
e5568f75 A	16	* FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
	17	* License for the specific language governing rights and limitations
	18	* under the License.
43866e37 A	19	*
	20	* @APPLE_LICENSE_HEADER_END@
	21	*/
	22
	23	#include <sys/appleapiopts.h>
	24	#include <ppc/asm.h>
	25	#include <ppc/proc_reg.h>
	26	#include <machine/cpu_capabilities.h>
	27	#include <machine/commpage.h>
	28
	29
	30	// commpage_time_dcba() uses a stack frame as follows:
	31
	32	#define kBufSiz 1024 // Size of the buffer we use to do DCBA timing on G4
	33	#define kSFSize (kBufSiz+128+16) // Stack frame size, which contains the 128-byte-aligned buffer
	34	#define kLoopCnt 5 // Iterations of the timing loop
	35	#define kDCBA 22 // Bit in cr5 used as a flag in timing loop
	36
55e303ae A	37
	38	// commpage_set_timestamp() uses the red zone for temporary storage:
	39
	40	#define rzSaveF1 -8 // caller's FPR1
	41	#define rzSaveF2 -16 // caller's FPR2
	42	#define rzSaveF3 -24 // caller's FPR3
	43	#define rzSaveF4 -32 // caller's FPR4
	44	#define rzSaveF5 -40 // caller's FPR5
	45	#define rzNewTimeBase -48 // used to load 64-bit TBR into a FPR
	46
	47
	48	// commpage_set_timestamp() uses the following data. kkTicksPerSec remembers
	49	// the number used to compute _COMM_PAGE_SEC_PER_TICK. Since this constant
	50	// rarely changes, we use it to avoid needless recomputation. It is a double
	51	// value, pre-initialize with an exponent of 2**52.
	52
	53	#define kkBinary0 0 // offset in data to long long 0 (a constant)
	54	#define kkDouble1 8 // offset in data to double 1.0 (a constant)
	55	#define kkTicksPerSec 16 // offset in data to double(ticks_per_sec)
	56
43866e37 A	57	.data
	58	.align 3 // three doubleword fields
	59	Ldata:
	60	.long 0 // kkBinary0
	61	.long 0
	62	.double 1.0e0 // kkDouble1
	63	.long 0x43300000 // kkTicksPerSec (plus 2**52)
	64	.long 0 // this is where we store ticks_per_sec, to float
	65
	66	.text
	67	.align 2
	68	.globl EXT(commpage_time_dcba)
55e303ae A	69	.globl EXT(commpage_set_timestamp)
	70
	71
	72	/* ***********************************************
	73	* * C O M M P A G E _ S E T _ T I M E S T A M P *
	74	* ***********************************************
	75	*
	76	* Update the gettimeofday() shared data on the commpage, as follows:
	77	* _COMM_PAGE_TIMESTAMP = a BSD-style pair of uint_32's for secs and usecs
	78	* _COMM_PAGE_TIMEBASE = the timebase at which the timestamp was valid
	79	* _COMM_PAGE_SEC_PER_TICK = multiply timebase ticks by this to get seconds (double)
	80	* The convention is that if the timebase is 0, the data is invalid. Because other
	81	* CPUs are reading the three values asynchronously and must get a consistent set,
	82	* it is critical that we update them with the following protocol:
	83	* 1. set timebase to 0 (atomically), to invalidate all three values
	84	* 2. eieio (to create a barrier in stores to cacheable memory)
	85	* 3. change timestamp and "secs per tick"
	86	* 4. eieio
	87	* 5. set timebase nonzero (atomically)
	88	* This works because readers read the timebase, then the timestamp and divisor, sync
	89	* if MP, then read the timebase a second time and check to be sure it is equal to the first.
	90	*
	91	* We could save a few cycles on 64-bit machines by special casing them, but it probably
	92	* isn't necessary because this routine shouldn't be called very often.
	93	*
	94	* When called:
	95	* r3 = upper half of timebase (timebase is disabled if 0)
	96	* r4 = lower half of timebase
	97	* r5 = seconds part of timestamp
	98	* r6 = useconds part of timestamp
	99	* r7 = divisor (ie, timebase ticks per sec)
	100	* We set up:
	101	* r8 = ptr to our static data (kkBinary0, kkDouble1, kkTicksPerSec)
	102	* r9 = ptr to comm page in kernel map
	103	*
	104	* --> Interrupts must be disabled and rtclock locked when called. <--
	105	*/
	106
	107	.align 5
	108	LEXT(commpage_set_timestamp) // void commpage_set_timestamp(tbr,secs,usecs,divisor)
	109	mfmsr r11 // get MSR
	110	ori r2,r11,MASK(MSR_FP) // turn FP on
	111	mtmsr r2
	112	isync // wait until MSR changes take effect
	113
	114	or. r0,r3,r4 // is timebase 0? (thus disabled)
	115	lis r8,hi16(Ldata) // point to our data
	116	lis r9,ha16(EXT(commPagePtr)) // get ptr to address of commpage in kernel map
	117	stfd f1,rzSaveF1(r1) // save a FPR in the red zone
	118	ori r8,r8,lo16(Ldata)
	119	lwz r9,lo16(EXT(commPagePtr))(r9) // r9 <- commPagePtr
	120	lfd f1,kkBinary0(r8) // get fixed 0s
	121	li r0,_COMM_PAGE_BASE_ADDRESS // get va in user space of commpage
	122	cmpwi cr1,r9,0 // is commpage allocated yet?
	123	sub r9,r9,r0 // r9 <- commpage address, biased by user va
	124	beq-- cr1,3f // skip if not allocated
	125	stfd f1,_COMM_PAGE_TIMEBASE(r9) // turn off the timestamp (atomically)
	126	eieio // make sure all CPUs see it is off
	127	beq 3f // all we had to do is turn off timestamp
	128
	129	lwz r0,kkTicksPerSec+4(r8) // get last ticks_per_sec (or 0 if first)
	130	stw r3,rzNewTimeBase(r1) // store new timebase so we can lfd
	131	stw r4,rzNewTimeBase+4(r1)
	132	cmpw r0,r7 // do we need to recompute _COMM_PAGE_SEC_PER_TICK?
133	stw r5,_COMM_PAGE_TIMESTAMP(r9) // store the new timestamp
134	stw r6,_COMM_PAGE_TIMESTAMP+4(r9)
135	lfd f1,rzNewTimeBase(r1) // get timebase in a FPR so we can store atomically
136	beq++ 2f // same ticks_per_sec, no need to recompute
137
138	stw r7,kkTicksPerSec+4(r8) // must recompute SEC_PER_TICK
139	stfd f2,rzSaveF2(r1) // we'll need a few more temp FPRs
140	stfd f3,rzSaveF3(r1)
141	stfd f4,rzSaveF4(r1)
142	stfd f5,rzSaveF5(r1)
143	lfd f2,_COMM_PAGE_2_TO_52(r9) // f2 <- double(2**52)
144	lfd f3,kkTicksPerSec(r8) // float new ticks_per_sec + 2**52
145	lfd f4,kkDouble1(r8) // f4 <- double(1.0)
146	mffs f5 // save caller's FPSCR
147	mtfsfi 7,0 // clear Inexeact Exception bit, set round-to-nearest
148	fsub f3,f3,f2 // get ticks_per_sec
149	fdiv f3,f4,f3 // divide 1 by ticks_per_sec to get SEC_PER_TICK
150	stfd f3,_COMM_PAGE_SEC_PER_TICK(r9)
151	mtfsf 0xFF,f5 // restore FPSCR
152	lfd f2,rzSaveF2(r1) // restore FPRs
153	lfd f3,rzSaveF3(r1)
154	lfd f4,rzSaveF4(r1)
155	lfd f5,rzSaveF5(r1)
156	2: // f1 == new timestamp
157	eieio // wait until the stores take
158	stfd f1,_COMM_PAGE_TIMEBASE(r9) // then turn the timestamp back on (atomically)
159	3: // here once all fields updated
160	lfd f1,rzSaveF1(r1) // restore last FPR
161	mtmsr r11 // turn FP back off
162	isync
163	blr
164
43866e37 A	165
	166	/* ***************************************
	167	* * C O M M P A G E _ T I M E _ D C B A *
	168	* ***************************************
	169	*
	170	* Not all processors that support the DCBA opcode actually benefit from it.
	171	* Some store-gather and read-cancel well enough that there is no need to use
	172	* DCBA to avoid fetching cache lines that will be completely overwritten, while
	173	* others have this feature disabled (to work around errata etc), and so benefit
	174	* from DCBA. Since it is hard to tell the one group from the other, we just
	175	* time loops with and without DCBA, and pick the fastest. Thus we avoid
	176	* delicate dependence on processor and/or platform revisions.
	177	*
	178	* We return either kDcbaRecommended or zero.
	179	*
	180	* int commpage_time_dcba( void );
	181	*/
	182
	183	LEXT(commpage_time_dcba)
	184	mflr r12 // get return
	185	stw r12,8(r1) // save
	186	stwu r1,-kSFSize(r1) // carve our temp buffer from the stack
	187	addi r11,r1,127+16 // get base address...
	188	rlwinm r11,r11,0,0,24 // ...of our buffer, 128-byte aligned
	189	crset kDCBA // first, use DCBA
	190	bl LTest // time it with DCBA
	191	srwi r0,r3,3 // bias 12 pct in favor of not using DCBA...
	192	add r10,r3,r0 // ...because DCBA is always slower with warm cache
	193	crclr kDCBA
	194	bl LTest // time without DCBA
	195	cmplw r10,r3 // which is better?
	196	mtlr r12 // restore return
	197	lwz r1,0(r1) // pop off our stack frame
	198	li r3,kDcbaRecommended // assume using DCBA is faster
	199	bltlr
	200	li r3,0 // no DCBA is faster
	201	blr
	202
	203
	204	// Subroutine to time a loop with or without DCBA.
	205	// kDCBA = set if we should use DCBA
	206	// r11 = base of buffer to use for test (kBufSiz bytes)
	207	//
	208	// We return TBR ticks in r3.
	209	// We use r0,r3-r9.
	210
	211	LTest:
	212	li r4,kLoopCnt // number of times to loop
	213	li r3,-1 // initialize fastest time
	214	1:
	215	mr r6,r11 // initialize buffer ptr
	216	li r0,kBufSiz/32 // r0 <- cache blocks to test
	217	mtctr r0
	218	2:
	219	dcbf 0,r6 // first, force the blocks out of the cache
	220	addi r6,r6,32
	221	bdnz 2b
	222	sync // make sure all the flushes take
	223	mr r6,r11 // re-initialize buffer ptr
	224	mtctr r0 // reset cache-block count
	225	mftbu r7 // remember upper half so we can check for carry
	226	mftb r8 // start the timer
	227	3: // loop over cache blocks
	228	bf kDCBA,4f // should we DCBA?
229	dcba 0,r6
230	4:
231	stw r0,0(r6) // store the entire cache block
232	stw r0,4(r6)
233	stw r0,8(r6)
234	stw r0,12(r6)
235	stw r0,16(r6)
236	stw r0,20(r6)
237	stw r0,24(r6)
238	stw r0,28(r6)
239	addi r6,r6,32
240	bdnz 3b
241	mftb r9
242	mftbu r0
243	cmpw r0,r7 // did timebase carry?
244	bne 1b // yes, retest rather than fuss
245	sub r9,r9,r8 // r9 <- time for this loop
246	cmplw r9,r3 // faster than current best?
247	bge 5f // no
248	mr r3,r9 // remember fastest time through loop
249	5:
250	subi r4,r4,1 // decrement outer loop count
251	cmpwi r4,0 // more to go?
252	bne 1b // loop if so
253	blr // return fastest time in r3