[apple/xnu.git] / osfmk / ppc / cache.s

/*
 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this
 * file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */
/*
 * @OSF_COPYRIGHT@
 */

#include <ppc/asm.h>
#include <ppc/proc_reg.h>
#include <ppc/exception.h>
#include <assym.s>

/* These routines run in 32 or 64-bit addressing, and handle
 * 32 and 128 byte caches.  They do not use compare instructions
 * on addresses, since compares are 32/64-bit-mode-specific.
 */

#define	kDcbf			0x1
#define	kDcbfb			31
#define	kDcbi			0x2
#define	kDcbib			30
#define	kIcbi			0x4
#define	kIcbib			29


/*
 * extern void flush_dcache(vm_offset_t addr, unsigned count, boolean phys);
 * extern void flush_dcache64(addr64_t addr, unsigned count, boolean phys);
 *
 * flush_dcache takes a virtual or physical address and count to flush
 * and (can be called for multiple virtual pages).
 *
 * it flushes the data cache
 * cache for the address range in question
 *
 * if 'phys' is non-zero then physical addresses will be used
 */


        .text
        .align	5
        .globl	_flush_dcache
_flush_dcache:
        li		r0,kDcbf					// use DCBF instruction
        rlwinm	r3,r3,0,0,31				// truncate address in case this is a 64-bit machine
        b		cache_op_join				// join common code

        .align	5
        .globl	_flush_dcache64
_flush_dcache64:
		rlwinm	r3,r3,0,1,0					; Duplicate high half of long long paddr into top of reg
		li		r0,kDcbf					// use DCBF instruction
		rlwimi	r3,r4,0,0,31				; Combine bottom of long long to full 64-bits
		mr		r4,r5						; Move count
		mr		r5,r6						; Move physical flag
        b		cache_op_join				// join common code


/*
 * extern void invalidate_dcache(vm_offset_t va, unsigned count, boolean phys);
 * extern void invalidate_dcache64(addr64_t va, unsigned count, boolean phys);
 *
 * invalidate_dcache takes a virtual or physical address and count to
 * invalidate and (can be called for multiple virtual pages).
 *
 * it invalidates the data cache for the address range in question
 */
 
        .globl	_invalidate_dcache
_invalidate_dcache:
        li		r0,kDcbi					// use DCBI instruction
        rlwinm	r3,r3,0,0,31				// truncate address in case this is a 64-bit machine
        b		cache_op_join				// join common code


        .align	5
        .globl	_invalidate_dcache64
_invalidate_dcache64:
		rlwinm	r3,r3,0,1,0					; Duplicate high half of long long paddr into top of reg
        li		r0,kDcbi					// use DCBI instruction
		rlwimi	r3,r4,0,0,31				; Combine bottom of long long to full 64-bits
		mr		r4,r5						; Move count
		mr		r5,r6						; Move physical flag
        b		cache_op_join				// join common code

/*
 * extern void invalidate_icache(vm_offset_t addr, unsigned cnt, boolean phys);
 * extern void invalidate_icache64(addr64_t addr, unsigned cnt, boolean phys);
 *
 * invalidate_icache takes a virtual or physical address and
 * count to invalidate, (can be called for multiple virtual pages).
 *
 * it invalidates the instruction cache for the address range in question.
 */
 
        .globl	_invalidate_icache
_invalidate_icache:
        li		r0,kIcbi					// use ICBI instruction
        rlwinm	r3,r3,0,0,31				// truncate address in case this is a 64-bit machine
        b		cache_op_join				// join common code
        

        .align	5
        .globl	_invalidate_icache64
_invalidate_icache64:
		rlwinm	r3,r3,0,1,0					; Duplicate high half of long long paddr into top of reg
        li		r0,kIcbi					// use ICBI instruction
		rlwimi	r3,r4,0,0,31				; Combine bottom of long long to full 64-bits
		mr		r4,r5						; Move count
		mr		r5,r6						; Move physical flag
        b		cache_op_join				// join common code
                        
/*
 * extern void sync_ppage(ppnum_t pa);
 *
 * sync_ppage takes a physical page number
 *
 * it writes out the data cache and invalidates the instruction
 * cache for the address range in question
 */

        .globl	_sync_ppage
        .align	5
_sync_ppage:								// Should be the most commonly called routine, by far 
		mfsprg	r2,2
        li		r0,kDcbf+kIcbi				// we need to dcbf and then icbi
		mtcrf	0x02,r2						; Move pf64Bit to cr6
        li		r5,1						// set flag for physical addresses
		li		r4,4096						; Set page size
		bt++	pf64Bitb,spp64				; Skip if 64-bit (only they take the hint)
        rlwinm	r3,r3,12,0,19				; Convert to physical address - 32-bit
        b		cache_op_join				; Join up....
        
spp64:	sldi	r3,r3,12					; Convert to physical address - 64-bit        
        b		cache_op_join				; Join up....
                        

/*
 * extern void sync_cache_virtual(vm_offset_t addr, unsigned count);
 *
 * Like "sync_cache", except it takes a virtual address and byte count.
 * It flushes the data cache, invalidates the I cache, and sync's.
 */
 
        .globl	_sync_cache_virtual
        .align	5
_sync_cache_virtual:
        li		r0,kDcbf+kIcbi				// we need to dcbf and then icbi
        li		r5,0						// set flag for virtual addresses
        b		cache_op_join				// join common code
        
                        
/*
 * extern void sync_cache(vm_offset_t pa, unsigned count);
 * extern void sync_cache64(addr64_t pa, unsigned count);
 *
 * sync_cache takes a physical address and count to sync, thus
 * must not be called for multiple virtual pages.
 *
 * it writes out the data cache and invalidates the instruction
 * cache for the address range in question
 */

        .globl	_sync_cache
        .align	5
_sync_cache:
        li		r0,kDcbf+kIcbi				// we need to dcbf and then icbi
        li		r5,1						// set flag for physical addresses
        rlwinm	r3,r3,0,0,31				// truncate address in case this is a 64-bit machine
        b		cache_op_join				// join common code

        .globl	_sync_cache64
        .align	5
_sync_cache64: 
		rlwinm	r3,r3,0,1,0					; Duplicate high half of long long paddr into top of reg
        li		r0,kDcbf+kIcbi				// we need to dcbf and then icbi
		rlwimi	r3,r4,0,0,31				; Combine bottom of long long to full 64-bits
       	mr		r4,r5						; Copy over the length
        li		r5,1						// set flag for physical addresses

        
        // Common code to handle the cache operations.

cache_op_join:								// here with r3=addr, r4=count, r5=phys flag, r0=bits
        mfsprg	r10,2						// r10 <- processor feature flags
        cmpwi	cr5,r5,0					// using physical addresses?
        mtcrf	0x01,r0						// move kDcbf, kDcbi, and kIcbi bits to CR7
        andi.	r9,r10,pf32Byte+pf128Byte	// r9 <- cache line size
        mtcrf	0x02,r10					// move pf64Bit bit to CR6
        subi	r8,r9,1						// r8 <- (linesize-1)
        beq--	cr5,cache_op_2				// skip if using virtual addresses
        
        bf--	pf64Bitb,cache_op_not64		// This is not a 64-bit machine
       
        srdi	r12,r3,31					// Slide bit 32 to bit 63
        cmpldi	r12,1						// Are we in the I/O mapped area?
        beqlr--								// No cache ops allowed here...
        
cache_op_not64:
        mflr	r12							// save return address
        bl		EXT(ml_set_physical)		// turn on physical addressing
        mtlr	r12							// restore return address

        // get r3=first cache line, r4=first line not in set, r6=byte count
        
cache_op_2:        
        add		r7,r3,r4					// point to 1st byte not to operate on
        andc	r3,r3,r8					// r3 <- 1st line to operate on
        add		r4,r7,r8					// round up
        andc	r4,r4,r8					// r4 <- 1st line not to operate on
        sub.	r6,r4,r3					// r6 <- byte count to operate on
        beq--	cache_op_exit				// nothing to do
        bf--	kDcbfb,cache_op_6			// no need to dcbf
        
        
        // DCBF loop
        
cache_op_5:
        sub.	r6,r6,r9					// more to go?
        dcbf	r6,r3						// flush next line to RAM
        bne		cache_op_5					// loop if more to go
        sync								// make sure the data reaches RAM
        sub		r6,r4,r3					// reset count


        // ICBI loop
        
cache_op_6:
        bf--	kIcbib,cache_op_8			// no need to icbi
cache_op_7:
        sub.	r6,r6,r9					// more to go?
        icbi	r6,r3						// invalidate next line
        bne		cache_op_7
        sub		r6,r4,r3					// reset count
        isync
        sync
        
        
        // DCBI loop
        
cache_op_8:
        bf++	kDcbib,cache_op_exit		// no need to dcbi
cache_op_9:
        sub.	r6,r6,r9					// more to go?
        dcbi	r6,r3						// invalidate next line
        bne		cache_op_9
        sync
        
        
        // restore MSR iff necessary and done
        
cache_op_exit:
        beqlr--	cr5							// if using virtual addresses, no need to restore MSR
        b		EXT(ml_restore)				// restore MSR and return


////////////////////////////////////////////////////

        .align	5
        .globl	_dcache_incoherent_io_store64
_dcache_incoherent_io_store64:
		rlwinm	r3,r3,0,1,0					; Duplicate high half of long long paddr into top of reg
		rlwimi	r3,r4,0,0,31				; Combine bottom of long long to full 64-bits
		mr		r4,r5						; Move count

											// here with r3=addr, r4=count
        mfsprg	r10,2						// r10 <- processor feature flags
        andi.	r9,r10,pf32Byte+pf128Byte	// r9 <- cache line size
        mtcrf	0x02,r10					// move pf64Bit bit to CR6
        subi	r8,r9,1						// r8 <- (linesize-1)
        
        bf--	pf64Bitb,cache_ios_not64	// This is not a 64-bit machine
       
        srdi	r12,r3,31					// Slide bit 32 to bit 63
        cmpldi	r12,1						// Are we in the I/O mapped area?
        beqlr--								// No cache ops allowed here...
        
cache_ios_not64:
        mflr	r12							// save return address
        bl		EXT(ml_set_physical)		// turn on physical addressing
        mtlr	r12							// restore return address

        // get r3=first cache line, r4=first line not in set, r6=byte count
        add		r7,r3,r4					// point to 1st byte not to operate on
        andc	r3,r3,r8					// r3 <- 1st line to operate on
        add		r4,r7,r8					// round up
        andc	r4,r4,r8					// r4 <- 1st line not to operate on
        sub.	r6,r4,r3					// r6 <- byte count to operate on
        beq--	cache_ios_exit				// nothing to do
        
        sub.	r6,r6,r9					// >1 line?
        beq		cache_ios_last_line			// use dcbst on all lines but last
        
        // DCBST loop
cache_ios_5:
        sub.	r6,r6,r9					// more to go?
        dcbst	r6,r3						// store next line
        bne		cache_ios_5					// loop if more to go

cache_ios_last_line:
        sync								// flush last line
        isync
        dcbf	r6,r3
        sync
        isync
        add		r6,r6,r3
        lwz		r0,0(r6)					// make sure the data reaches RAM (not just the memory controller)
        isync

        // restore MSR
cache_ios_exit:
        b		EXT(ml_restore)				// restore MSR and return


////////////////////////////////////////////////////

        .align	5
        .globl	_dcache_incoherent_io_flush64
_dcache_incoherent_io_flush64:
		rlwinm	r3,r3,0,1,0					; Duplicate high half of long long paddr into top of reg
		rlwimi	r3,r4,0,0,31				; Combine bottom of long long to full 64-bits
		mr		r4,r5						; Move count

											// here with r3=addr, r4=count
        mfsprg	r10,2						// r10 <- processor feature flags
        andi.	r9,r10,pf32Byte+pf128Byte	// r9 <- cache line size
        mtcrf	0x02,r10					// move pf64Bit bit to CR6
        subi	r8,r9,1						// r8 <- (linesize-1)
        
        bf--	pf64Bitb,cache_iof_not64	// This is not a 64-bit machine
       
        srdi	r12,r3,31					// Slide bit 32 to bit 63
        cmpldi	r12,1						// Are we in the I/O mapped area?
        beqlr--								// No cache ops allowed here...
        
cache_iof_not64:
        mflr	r12							// save return address
        bl		EXT(ml_set_physical)		// turn on physical addressing
        mtlr	r12							// restore return address

        // get r3=first cache line, r4=first line not in set, r6=byte count
        add		r7,r3,r4					// point to 1st byte not to operate on
        andc	r3,r3,r8					// r3 <- 1st line to operate on
        add		r4,r7,r8					// round up
        andc	r4,r4,r8					// r4 <- 1st line not to operate on
        sub.	r6,r4,r3					// r6 <- byte count to operate on
        beq--	cache_iof_exit				// nothing to do
        
        // DCBF loop
cache_iof_5:
        sub.	r6,r6,r9					// more to go?
        dcbf	r6,r3						// store next line
        bne		cache_iof_5					// loop if more to go

cache_iof_last_line:
        sync								// flush last line
        isync

        // restore MSR
cache_iof_exit:
        b		EXT(ml_restore)				// restore MSR and return
Commit	Line	Data
1c79356b A	1	/*
	2	* Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_LICENSE_HEADER_START@
	5	*
ff6e181a A	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. Please obtain a copy of the License at
	10	* http://www.opensource.apple.com/apsl/ and read it before using this
	11	* file.
1c79356b	12	*
ff6e181a A	13	* The Original Code and all software distributed under the License are
ff6e181a A	14	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
1c79356b A	15	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
1c79356b A	16	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
ff6e181a A	17	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	18	* Please see the License for the specific language governing rights and
	19	* limitations under the License.
1c79356b A	20	*
	21	* @APPLE_LICENSE_HEADER_END@
	22	*/
	23	/*
	24	* @OSF_COPYRIGHT@
	25	*/
	26
1c79356b A	27	#include <ppc/asm.h>
1c79356b A	28	#include <ppc/proc_reg.h>
55e303ae	29	#include <ppc/exception.h>
1c79356b	30	#include <assym.s>
1c79356b	31
55e303ae A	32	/* These routines run in 32 or 64-bit addressing, and handle
	33	* 32 and 128 byte caches. They do not use compare instructions
	34	* on addresses, since compares are 32/64-bit-mode-specific.
1c79356b A	35	*/
1c79356b A	36
55e303ae A	37	#define kDcbf 0x1
	38	#define kDcbfb 31
	39	#define kDcbi 0x2
	40	#define kDcbib 30
	41	#define kIcbi 0x4
	42	#define kIcbib 29
1c79356b	43
1c79356b A	44
	45	/*
	46	* extern void flush_dcache(vm_offset_t addr, unsigned count, boolean phys);
55e303ae	47	* extern void flush_dcache64(addr64_t addr, unsigned count, boolean phys);
1c79356b A	48	*
	49	* flush_dcache takes a virtual or physical address and count to flush
	50	* and (can be called for multiple virtual pages).
	51	*
	52	* it flushes the data cache
	53	* cache for the address range in question
	54	*
	55	* if 'phys' is non-zero then physical addresses will be used
	56	*/
	57
1c79356b	58
55e303ae A	59
	60	.text
	61	.align 5
	62	.globl _flush_dcache
	63	_flush_dcache:
	64	li r0,kDcbf // use DCBF instruction
	65	rlwinm r3,r3,0,0,31 // truncate address in case this is a 64-bit machine
	66	b cache_op_join // join common code
1c79356b	67
55e303ae A	68	.align 5
	69	.globl _flush_dcache64
	70	_flush_dcache64:
	71	rlwinm r3,r3,0,1,0 ; Duplicate high half of long long paddr into top of reg
	72	li r0,kDcbf // use DCBF instruction
	73	rlwimi r3,r4,0,0,31 ; Combine bottom of long long to full 64-bits
	74	mr r4,r5 ; Move count
	75	mr r5,r6 ; Move physical flag
	76	b cache_op_join // join common code
1c79356b A	77
	78
	79	/*
	80	* extern void invalidate_dcache(vm_offset_t va, unsigned count, boolean phys);
55e303ae	81	* extern void invalidate_dcache64(addr64_t va, unsigned count, boolean phys);
1c79356b A	82	*
	83	* invalidate_dcache takes a virtual or physical address and count to
	84	* invalidate and (can be called for multiple virtual pages).
	85	*
	86	* it invalidates the data cache for the address range in question
	87	*/
55e303ae A	88
	89	.globl _invalidate_dcache
	90	_invalidate_dcache:
	91	li r0,kDcbi // use DCBI instruction
	92	rlwinm r3,r3,0,0,31 // truncate address in case this is a 64-bit machine
	93	b cache_op_join // join common code
	94
	95
	96	.align 5
	97	.globl _invalidate_dcache64
	98	_invalidate_dcache64:
	99	rlwinm r3,r3,0,1,0 ; Duplicate high half of long long paddr into top of reg
	100	li r0,kDcbi // use DCBI instruction
	101	rlwimi r3,r4,0,0,31 ; Combine bottom of long long to full 64-bits
	102	mr r4,r5 ; Move count
	103	mr r5,r6 ; Move physical flag
	104	b cache_op_join // join common code
1c79356b A	105
	106	/*
	107	* extern void invalidate_icache(vm_offset_t addr, unsigned cnt, boolean phys);
55e303ae	108	* extern void invalidate_icache64(addr64_t addr, unsigned cnt, boolean phys);
1c79356b A	109	*
	110	* invalidate_icache takes a virtual or physical address and
	111	* count to invalidate, (can be called for multiple virtual pages).
	112	*
	113	* it invalidates the instruction cache for the address range in question.
55e303ae A	114	*/
	115
	116	.globl _invalidate_icache
	117	_invalidate_icache:
	118	li r0,kIcbi // use ICBI instruction
	119	rlwinm r3,r3,0,0,31 // truncate address in case this is a 64-bit machine
	120	b cache_op_join // join common code
	121
	122
	123	.align 5
	124	.globl _invalidate_icache64
	125	_invalidate_icache64:
	126	rlwinm r3,r3,0,1,0 ; Duplicate high half of long long paddr into top of reg
	127	li r0,kIcbi // use ICBI instruction
	128	rlwimi r3,r4,0,0,31 ; Combine bottom of long long to full 64-bits
	129	mr r4,r5 ; Move count
	130	mr r5,r6 ; Move physical flag
	131	b cache_op_join // join common code
	132
	133	/*
	134	* extern void sync_ppage(ppnum_t pa);
	135	*
	136	* sync_ppage takes a physical page number
	137	*
	138	* it writes out the data cache and invalidates the instruction
	139	* cache for the address range in question
1c79356b A	140	*/
1c79356b A	141
55e303ae A	142	.globl _sync_ppage
	143	.align 5
	144	_sync_ppage: // Should be the most commonly called routine, by far
	145	mfsprg r2,2
	146	li r0,kDcbf+kIcbi // we need to dcbf and then icbi
	147	mtcrf 0x02,r2 ; Move pf64Bit to cr6
	148	li r5,1 // set flag for physical addresses
	149	li r4,4096 ; Set page size
	150	bt++ pf64Bitb,spp64 ; Skip if 64-bit (only they take the hint)
	151	rlwinm r3,r3,12,0,19 ; Convert to physical address - 32-bit
	152	b cache_op_join ; Join up....
	153
	154	spp64: sldi r3,r3,12 ; Convert to physical address - 64-bit
	155	b cache_op_join ; Join up....
	156
de355530	157
de355530	158
55e303ae A	159	/*
	160	* extern void sync_cache_virtual(vm_offset_t addr, unsigned count);
	161	*
	162	* Like "sync_cache", except it takes a virtual address and byte count.
	163	* It flushes the data cache, invalidates the I cache, and sync's.
	164	*/
	165
	166	.globl _sync_cache_virtual
	167	.align 5
	168	_sync_cache_virtual:
	169	li r0,kDcbf+kIcbi // we need to dcbf and then icbi
	170	li r5,0 // set flag for virtual addresses
	171	b cache_op_join // join common code
	172
	173
	174	/*
	175	* extern void sync_cache(vm_offset_t pa, unsigned count);
	176	* extern void sync_cache64(addr64_t pa, unsigned count);
	177	*
	178	* sync_cache takes a physical address and count to sync, thus
	179	* must not be called for multiple virtual pages.
	180	*
	181	* it writes out the data cache and invalidates the instruction
	182	* cache for the address range in question
	183	*/
1c79356b	184
55e303ae A	185	.globl _sync_cache
	186	.align 5
	187	_sync_cache:
	188	li r0,kDcbf+kIcbi // we need to dcbf and then icbi
	189	li r5,1 // set flag for physical addresses
	190	rlwinm r3,r3,0,0,31 // truncate address in case this is a 64-bit machine
	191	b cache_op_join // join common code
	192
	193	.globl _sync_cache64
	194	.align 5
	195	_sync_cache64:
	196	rlwinm r3,r3,0,1,0 ; Duplicate high half of long long paddr into top of reg
	197	li r0,kDcbf+kIcbi // we need to dcbf and then icbi
	198	rlwimi r3,r4,0,0,31 ; Combine bottom of long long to full 64-bits
	199	mr r4,r5 ; Copy over the length
	200	li r5,1 // set flag for physical addresses
	201
	202
	203	// Common code to handle the cache operations.
	204
	205	cache_op_join: // here with r3=addr, r4=count, r5=phys flag, r0=bits
	206	mfsprg r10,2 // r10 <- processor feature flags
	207	cmpwi cr5,r5,0 // using physical addresses?
	208	mtcrf 0x01,r0 // move kDcbf, kDcbi, and kIcbi bits to CR7
	209	andi. r9,r10,pf32Byte+pf128Byte // r9 <- cache line size
	210	mtcrf 0x02,r10 // move pf64Bit bit to CR6
	211	subi r8,r9,1 // r8 <- (linesize-1)
	212	beq-- cr5,cache_op_2 // skip if using virtual addresses
	213
	214	bf-- pf64Bitb,cache_op_not64 // This is not a 64-bit machine
	215
	216	srdi r12,r3,31 // Slide bit 32 to bit 63
	217	cmpldi r12,1 // Are we in the I/O mapped area?
	218	beqlr-- // No cache ops allowed here...
	219
	220	cache_op_not64:
	221	mflr r12 // save return address
	222	bl EXT(ml_set_physical) // turn on physical addressing
	223	mtlr r12 // restore return address
	224
	225	// get r3=first cache line, r4=first line not in set, r6=byte count
	226
	227	cache_op_2:
	228	add r7,r3,r4 // point to 1st byte not to operate on
	229	andc r3,r3,r8 // r3 <- 1st line to operate on
	230	add r4,r7,r8 // round up
	231	andc r4,r4,r8 // r4 <- 1st line not to operate on
	232	sub. r6,r4,r3 // r6 <- byte count to operate on
	233	beq-- cache_op_exit // nothing to do
	234	bf-- kDcbfb,cache_op_6 // no need to dcbf
	235
	236
	237	// DCBF loop
	238
	239	cache_op_5:
	240	sub. r6,r6,r9 // more to go?
	241	dcbf r6,r3 // flush next line to RAM
	242	bne cache_op_5 // loop if more to go
	243	sync // make sure the data reaches RAM
	244	sub r6,r4,r3 // reset count
	245
	246
	247	// ICBI loop
	248
249	cache_op_6:
250	bf-- kIcbib,cache_op_8 // no need to icbi
251	cache_op_7:
252	sub. r6,r6,r9 // more to go?
253	icbi r6,r3 // invalidate next line
254	bne cache_op_7
255	sub r6,r4,r3 // reset count
256	isync
257	sync
258
259
260	// DCBI loop
261
262	cache_op_8:
263	bf++ kDcbib,cache_op_exit // no need to dcbi
264	cache_op_9:
265	sub. r6,r6,r9 // more to go?
266	dcbi r6,r3 // invalidate next line
267	bne cache_op_9
268	sync
269
270
271	// restore MSR iff necessary and done
272
273	cache_op_exit:
274	beqlr-- cr5 // if using virtual addresses, no need to restore MSR
275	b EXT(ml_restore) // restore MSR and return
1c79356b	276
91447636 A	277
	278	////////////////////////////////////////////////////
	279
	280	.align 5
	281	.globl _dcache_incoherent_io_store64
	282	_dcache_incoherent_io_store64:
	283	rlwinm r3,r3,0,1,0 ; Duplicate high half of long long paddr into top of reg
	284	rlwimi r3,r4,0,0,31 ; Combine bottom of long long to full 64-bits
	285	mr r4,r5 ; Move count
	286
	287	// here with r3=addr, r4=count
	288	mfsprg r10,2 // r10 <- processor feature flags
	289	andi. r9,r10,pf32Byte+pf128Byte // r9 <- cache line size
	290	mtcrf 0x02,r10 // move pf64Bit bit to CR6
	291	subi r8,r9,1 // r8 <- (linesize-1)
	292
	293	bf-- pf64Bitb,cache_ios_not64 // This is not a 64-bit machine
	294
	295	srdi r12,r3,31 // Slide bit 32 to bit 63
	296	cmpldi r12,1 // Are we in the I/O mapped area?
	297	beqlr-- // No cache ops allowed here...
	298
	299	cache_ios_not64:
	300	mflr r12 // save return address
	301	bl EXT(ml_set_physical) // turn on physical addressing
	302	mtlr r12 // restore return address
	303
	304	// get r3=first cache line, r4=first line not in set, r6=byte count
	305	add r7,r3,r4 // point to 1st byte not to operate on
	306	andc r3,r3,r8 // r3 <- 1st line to operate on
	307	add r4,r7,r8 // round up
	308	andc r4,r4,r8 // r4 <- 1st line not to operate on
	309	sub. r6,r4,r3 // r6 <- byte count to operate on
	310	beq-- cache_ios_exit // nothing to do
	311
	312	sub. r6,r6,r9 // >1 line?
	313	beq cache_ios_last_line // use dcbst on all lines but last
	314
	315	// DCBST loop
	316	cache_ios_5:
	317	sub. r6,r6,r9 // more to go?
	318	dcbst r6,r3 // store next line
	319	bne cache_ios_5 // loop if more to go
	320
	321	cache_ios_last_line:
	322	sync // flush last line
	323	isync
	324	dcbf r6,r3
	325	sync
	326	isync
	327	add r6,r6,r3
	328	lwz r0,0(r6) // make sure the data reaches RAM (not just the memory controller)
	329	isync
	330
	331	// restore MSR
	332	cache_ios_exit:
	333	b EXT(ml_restore) // restore MSR and return
	334
	335
	336	////////////////////////////////////////////////////
	337
	338	.align 5
	339	.globl _dcache_incoherent_io_flush64
	340	_dcache_incoherent_io_flush64:
341	rlwinm r3,r3,0,1,0 ; Duplicate high half of long long paddr into top of reg
342	rlwimi r3,r4,0,0,31 ; Combine bottom of long long to full 64-bits
343	mr r4,r5 ; Move count
344
345	// here with r3=addr, r4=count
346	mfsprg r10,2 // r10 <- processor feature flags
347	andi. r9,r10,pf32Byte+pf128Byte // r9 <- cache line size
348	mtcrf 0x02,r10 // move pf64Bit bit to CR6
349	subi r8,r9,1 // r8 <- (linesize-1)
350
351	bf-- pf64Bitb,cache_iof_not64 // This is not a 64-bit machine
352
353	srdi r12,r3,31 // Slide bit 32 to bit 63
354	cmpldi r12,1 // Are we in the I/O mapped area?
355	beqlr-- // No cache ops allowed here...
356
357	cache_iof_not64:
358	mflr r12 // save return address
359	bl EXT(ml_set_physical) // turn on physical addressing
360	mtlr r12 // restore return address
361
362	// get r3=first cache line, r4=first line not in set, r6=byte count
363	add r7,r3,r4 // point to 1st byte not to operate on
364	andc r3,r3,r8 // r3 <- 1st line to operate on
365	add r4,r7,r8 // round up
366	andc r4,r4,r8 // r4 <- 1st line not to operate on
367	sub. r6,r4,r3 // r6 <- byte count to operate on
368	beq-- cache_iof_exit // nothing to do
369
370	// DCBF loop
371	cache_iof_5:
372	sub. r6,r6,r9 // more to go?
373	dcbf r6,r3 // store next line
374	bne cache_iof_5 // loop if more to go
375
376	cache_iof_last_line:
377	sync // flush last line
378	isync
379
380	// restore MSR
381	cache_iof_exit:
382	b EXT(ml_restore) // restore MSR and return
383
384