[apple/xnu.git] / osfmk / ppc / cache.s

/*
 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 * 
 * Copyright (c) 1999-2003 Apple Computer, Inc.  All Rights Reserved.
 * 
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this
 * file.
 * 
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 * 
 * @APPLE_LICENSE_HEADER_END@
 */
/*
 * @OSF_COPYRIGHT@
 */

#include <cpus.h>

#include <ppc/asm.h>
#include <ppc/proc_reg.h>
#include <cpus.h>
#include <assym.s>
#include <mach_debug.h>
#include <mach/ppc/vm_param.h>

/*
 * extern void sync_cache(vm_offset_t pa, unsigned count);
 *
 * sync_cache takes a physical address and count to sync, thus
 * must not be called for multiple virtual pages.
 *
 * it writes out the data cache and invalidates the instruction
 * cache for the address range in question
 */

ENTRY(sync_cache, TAG_NO_FRAME_USED)

	/* Switch off data translations */
	mfmsr	r6
	rlwinm	r6,r6,0,MSR_FP_BIT+1,MSR_FP_BIT-1	; Force floating point off
	rlwinm	r6,r6,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1	; Force vectors off
	rlwinm	r7,	r6,	0,	MSR_DR_BIT+1,	MSR_DR_BIT-1
	mtmsr	r7
	isync

	/* Check to see if the address is aligned. */
	add	r8, r3,r4
	andi.	r8,r8,(CACHE_LINE_SIZE-1)
	beq-	.L_sync_check
	addi	r4,r4,CACHE_LINE_SIZE
	li	r7,(CACHE_LINE_SIZE-1)	/* Align buffer & count - avoid overflow problems */
	andc	r4,r4,r7
	andc	r3,r3,r7

.L_sync_check:
	cmpwi	r4,	CACHE_LINE_SIZE
	ble	.L_sync_one_line
	
	/* Make ctr hold count of how many times we should loop */
	addi	r8,	r4,	(CACHE_LINE_SIZE-1)
	srwi	r8,	r8,	CACHE_LINE_POW2
	mtctr	r8

	/* loop to flush the data cache */
.L_sync_data_loop:
	subic	r4,	r4,	CACHE_LINE_SIZE
	dcbf	r3,	r4
	bdnz	.L_sync_data_loop
	
	sync
	mtctr	r8

	/* loop to invalidate the instruction cache */
.L_sync_inval_loop:
	icbi	r3,	r4
	addic	r4,	r4,	CACHE_LINE_SIZE
	bdnz	.L_sync_inval_loop

.L_sync_cache_done:
	sync			/* Finish physical writes */
	mtmsr	r6		/* Restore original translations */
	isync			/* Ensure data translations are on */
	blr

.L_sync_one_line:
	dcbf	0,r3
	sync
	icbi	0,r3
	b	.L_sync_cache_done

/*
 * extern void flush_dcache(vm_offset_t addr, unsigned count, boolean phys);
 *
 * flush_dcache takes a virtual or physical address and count to flush
 * and (can be called for multiple virtual pages).
 *
 * it flushes the data cache
 * cache for the address range in question
 *
 * if 'phys' is non-zero then physical addresses will be used
 */

ENTRY(flush_dcache, TAG_NO_FRAME_USED)

	/* optionally switch off data translations */

	cmpwi	r5,	0
	mfmsr	r6
	beq+	0f
	rlwinm	r6,r6,0,MSR_FP_BIT+1,MSR_FP_BIT-1	; Force floating point off
	rlwinm	r6,r6,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1	; Force vectors off
	rlwinm	r7,	r6,	0,	MSR_DR_BIT+1,	MSR_DR_BIT-1
	mtmsr	r7
	isync
0:	

	/* Check to see if the address is aligned. */
	add	r8, r3,r4
	andi.	r8,r8,(CACHE_LINE_SIZE-1)
	beq-	.L_flush_dcache_check
	addi	r4,r4,CACHE_LINE_SIZE
	li	r7,(CACHE_LINE_SIZE-1)	/* Align buffer & count - avoid overflow problems */
	andc	r4,r4,r7
	andc	r3,r3,r7

.L_flush_dcache_check:
	cmpwi	r4,	CACHE_LINE_SIZE
	ble	.L_flush_dcache_one_line
	
	/* Make ctr hold count of how many times we should loop */
	addi	r8,	r4,	(CACHE_LINE_SIZE-1)
	srwi	r8,	r8,	CACHE_LINE_POW2
	mtctr	r8

.L_flush_dcache_flush_loop:
	subic	r4,	r4,	CACHE_LINE_SIZE
	dcbf	r3,	r4
	bdnz	.L_flush_dcache_flush_loop

.L_flush_dcache_done:
	/* Sync restore msr if it was modified */
	cmpwi	r5,	0
	sync			/* make sure invalidates have completed */
	beq+	0f
	mtmsr	r6		/* Restore original translations */
	isync			/* Ensure data translations are on */
0:
	blr

.L_flush_dcache_one_line:
	xor	r4,r4,r4
	dcbf	0,r3
	b	.L_flush_dcache_done


/*
 * extern void invalidate_dcache(vm_offset_t va, unsigned count, boolean phys);
 *
 * invalidate_dcache takes a virtual or physical address and count to
 * invalidate and (can be called for multiple virtual pages).
 *
 * it invalidates the data cache for the address range in question
 */

ENTRY(invalidate_dcache, TAG_NO_FRAME_USED)

	/* optionally switch off data translations */

	cmpwi	r5,	0
	mfmsr	r6
	beq+	0f
	rlwinm	r6,r6,0,MSR_FP_BIT+1,MSR_FP_BIT-1	; Force floating point off
	rlwinm	r6,r6,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1	; Force vectors off
	rlwinm	r7,	r6,	0,	MSR_DR_BIT+1,	MSR_DR_BIT-1
	mtmsr	r7
	isync
0:	

	/* Check to see if the address is aligned. */
	add	r8, r3,r4
	andi.	r8,r8,(CACHE_LINE_SIZE-1)
	beq-	.L_invalidate_dcache_check
	addi	r4,r4,CACHE_LINE_SIZE
	li	r7,(CACHE_LINE_SIZE-1)	/* Align buffer & count - avoid overflow problems */
	andc	r4,r4,r7
	andc	r3,r3,r7

.L_invalidate_dcache_check:
	cmpwi	r4,	CACHE_LINE_SIZE
	ble	.L_invalidate_dcache_one_line
	
	/* Make ctr hold count of how many times we should loop */
	addi	r8,	r4,	(CACHE_LINE_SIZE-1)
	srwi	r8,	r8,	CACHE_LINE_POW2
	mtctr	r8

.L_invalidate_dcache_invalidate_loop:
	subic	r4,	r4,	CACHE_LINE_SIZE
	dcbi	r3,	r4
	bdnz	.L_invalidate_dcache_invalidate_loop

.L_invalidate_dcache_done:
	/* Sync restore msr if it was modified */
	cmpwi	r5,	0
	sync			/* make sure invalidates have completed */
	beq+	0f
	mtmsr	r6		/* Restore original translations */
	isync			/* Ensure data translations are on */
0:
	blr

.L_invalidate_dcache_one_line:
	xor	r4,r4,r4
	dcbi	0,r3
	b	.L_invalidate_dcache_done

/*
 * extern void invalidate_icache(vm_offset_t addr, unsigned cnt, boolean phys);
 *
 * invalidate_icache takes a virtual or physical address and
 * count to invalidate, (can be called for multiple virtual pages).
 *
 * it invalidates the instruction cache for the address range in question.
 */

ENTRY(invalidate_icache, TAG_NO_FRAME_USED)

	/* optionally switch off data translations */
	cmpwi	r5,	0
	mfmsr	r6
	beq+	0f
	rlwinm	r6,r6,0,MSR_FP_BIT+1,MSR_FP_BIT-1	; Force floating point off
	rlwinm	r6,r6,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1	; Force vectors off
	rlwinm	r7,	r6,	0,	MSR_DR_BIT+1,	MSR_DR_BIT-1
	mtmsr	r7
	isync
0:	

	/* Check to see if the address is aligned. */
	add	r8, r3,r4
	andi.	r8,r8,(CACHE_LINE_SIZE-1)
	beq-	.L_invalidate_icache_check
	addi	r4,r4,CACHE_LINE_SIZE
	li	r7,(CACHE_LINE_SIZE-1)	/* Align buffer & count - avoid overflow problems */
	andc	r4,r4,r7
	andc	r3,r3,r7

.L_invalidate_icache_check:
	cmpwi	r4,	CACHE_LINE_SIZE
	ble	.L_invalidate_icache_one_line
	
	/* Make ctr hold count of how many times we should loop */
	addi	r8,	r4,	(CACHE_LINE_SIZE-1)
	srwi	r8,	r8,	CACHE_LINE_POW2
	mtctr	r8

.L_invalidate_icache_invalidate_loop:
	subic	r4,	r4,	CACHE_LINE_SIZE
	icbi	r3,	r4
	bdnz	.L_invalidate_icache_invalidate_loop

.L_invalidate_icache_done:
	sync			/* make sure invalidates have completed */
	mtmsr	r6		/* Restore original translations */
	isync			/* Ensure data translations are on */
	blr

.L_invalidate_icache_one_line:
	xor	r4,r4,r4
	icbi	0,r3
	b	.L_invalidate_icache_done
Commit	Line	Data
1c79356b A	1	/*
	2	* Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_LICENSE_HEADER_START@
	5	*
43866e37	6	* Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved.
1c79356b	7	*
43866e37 A	8	* This file contains Original Code and/or Modifications of Original Code
	9	* as defined in and that are subject to the Apple Public Source License
	10	* Version 2.0 (the 'License'). You may not use this file except in
	11	* compliance with the License. Please obtain a copy of the License at
	12	* http://www.opensource.apple.com/apsl/ and read it before using this
	13	* file.
	14	*
	15	* The Original Code and all software distributed under the License are
	16	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
1c79356b A	17	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
1c79356b A	18	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
43866e37 A	19	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	20	* Please see the License for the specific language governing rights and
	21	* limitations under the License.
1c79356b A	22	*
	23	* @APPLE_LICENSE_HEADER_END@
	24	*/
	25	/*
	26	* @OSF_COPYRIGHT@
	27	*/
	28
de355530 A	29	#include <cpus.h>
de355530 A	30
1c79356b A	31	#include <ppc/asm.h>
1c79356b A	32	#include <ppc/proc_reg.h>
de355530	33	#include <cpus.h>
1c79356b	34	#include <assym.s>
de355530 A	35	#include <mach_debug.h>
de355530 A	36	#include <mach/ppc/vm_param.h>
1c79356b	37
de355530 A	38	/*
	39	* extern void sync_cache(vm_offset_t pa, unsigned count);
	40	*
	41	* sync_cache takes a physical address and count to sync, thus
	42	* must not be called for multiple virtual pages.
	43	*
	44	* it writes out the data cache and invalidates the instruction
	45	* cache for the address range in question
1c79356b A	46	*/
1c79356b A	47
de355530 A	48	ENTRY(sync_cache, TAG_NO_FRAME_USED)
	49
	50	/* Switch off data translations */
	51	mfmsr r6
	52	rlwinm r6,r6,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
	53	rlwinm r6,r6,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
	54	rlwinm r7, r6, 0, MSR_DR_BIT+1, MSR_DR_BIT-1
	55	mtmsr r7
	56	isync
	57
	58	/* Check to see if the address is aligned. */
	59	add r8, r3,r4
	60	andi. r8,r8,(CACHE_LINE_SIZE-1)
	61	beq- .L_sync_check
	62	addi r4,r4,CACHE_LINE_SIZE
	63	li r7,(CACHE_LINE_SIZE-1) /* Align buffer & count - avoid overflow problems */
	64	andc r4,r4,r7
	65	andc r3,r3,r7
	66
	67	.L_sync_check:
	68	cmpwi r4, CACHE_LINE_SIZE
	69	ble .L_sync_one_line
	70
	71	/* Make ctr hold count of how many times we should loop */
	72	addi r8, r4, (CACHE_LINE_SIZE-1)
	73	srwi r8, r8, CACHE_LINE_POW2
	74	mtctr r8
1c79356b	75
de355530 A	76	/* loop to flush the data cache */
	77	.L_sync_data_loop:
	78	subic r4, r4, CACHE_LINE_SIZE
	79	dcbf r3, r4
	80	bdnz .L_sync_data_loop
	81
	82	sync
	83	mtctr r8
	84
	85	/* loop to invalidate the instruction cache */
	86	.L_sync_inval_loop:
	87	icbi r3, r4
	88	addic r4, r4, CACHE_LINE_SIZE
	89	bdnz .L_sync_inval_loop
	90
	91	.L_sync_cache_done:
	92	sync /* Finish physical writes */
	93	mtmsr r6 /* Restore original translations */
	94	isync /* Ensure data translations are on */
	95	blr
	96
	97	.L_sync_one_line:
	98	dcbf 0,r3
	99	sync
	100	icbi 0,r3
	101	b .L_sync_cache_done
1c79356b A	102
	103	/*
	104	* extern void flush_dcache(vm_offset_t addr, unsigned count, boolean phys);
	105	*
	106	* flush_dcache takes a virtual or physical address and count to flush
	107	* and (can be called for multiple virtual pages).
	108	*
	109	* it flushes the data cache
	110	* cache for the address range in question
	111	*
	112	* if 'phys' is non-zero then physical addresses will be used
	113	*/
	114
de355530 A	115	ENTRY(flush_dcache, TAG_NO_FRAME_USED)
	116
	117	/* optionally switch off data translations */
	118
	119	cmpwi r5, 0
	120	mfmsr r6
	121	beq+ 0f
	122	rlwinm r6,r6,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
	123	rlwinm r6,r6,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
	124	rlwinm r7, r6, 0, MSR_DR_BIT+1, MSR_DR_BIT-1
	125	mtmsr r7
	126	isync
	127	0:
	128
	129	/* Check to see if the address is aligned. */
	130	add r8, r3,r4
	131	andi. r8,r8,(CACHE_LINE_SIZE-1)
	132	beq- .L_flush_dcache_check
	133	addi r4,r4,CACHE_LINE_SIZE
	134	li r7,(CACHE_LINE_SIZE-1) /* Align buffer & count - avoid overflow problems */
	135	andc r4,r4,r7
	136	andc r3,r3,r7
1c79356b	137
de355530 A	138	.L_flush_dcache_check:
	139	cmpwi r4, CACHE_LINE_SIZE
	140	ble .L_flush_dcache_one_line
	141
	142	/* Make ctr hold count of how many times we should loop */
	143	addi r8, r4, (CACHE_LINE_SIZE-1)
	144	srwi r8, r8, CACHE_LINE_POW2
	145	mtctr r8
1c79356b	146
de355530 A	147	.L_flush_dcache_flush_loop:
	148	subic r4, r4, CACHE_LINE_SIZE
	149	dcbf r3, r4
	150	bdnz .L_flush_dcache_flush_loop
	151
	152	.L_flush_dcache_done:
	153	/* Sync restore msr if it was modified */
	154	cmpwi r5, 0
	155	sync /* make sure invalidates have completed */
	156	beq+ 0f
	157	mtmsr r6 /* Restore original translations */
	158	isync /* Ensure data translations are on */
	159	0:
	160	blr
	161
	162	.L_flush_dcache_one_line:
	163	xor r4,r4,r4
	164	dcbf 0,r3
	165	b .L_flush_dcache_done
1c79356b A	166
	167
	168	/*
	169	* extern void invalidate_dcache(vm_offset_t va, unsigned count, boolean phys);
	170	*
	171	* invalidate_dcache takes a virtual or physical address and count to
	172	* invalidate and (can be called for multiple virtual pages).
	173	*
	174	* it invalidates the data cache for the address range in question
	175	*/
de355530 A	176
	177	ENTRY(invalidate_dcache, TAG_NO_FRAME_USED)
	178
	179	/* optionally switch off data translations */
	180
	181	cmpwi r5, 0
	182	mfmsr r6
	183	beq+ 0f
	184	rlwinm r6,r6,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
	185	rlwinm r6,r6,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
	186	rlwinm r7, r6, 0, MSR_DR_BIT+1, MSR_DR_BIT-1
	187	mtmsr r7
	188	isync
	189	0:
	190
	191	/* Check to see if the address is aligned. */
	192	add r8, r3,r4
	193	andi. r8,r8,(CACHE_LINE_SIZE-1)
	194	beq- .L_invalidate_dcache_check
	195	addi r4,r4,CACHE_LINE_SIZE
	196	li r7,(CACHE_LINE_SIZE-1) /* Align buffer & count - avoid overflow problems */
	197	andc r4,r4,r7
	198	andc r3,r3,r7
	199
	200	.L_invalidate_dcache_check:
	201	cmpwi r4, CACHE_LINE_SIZE
	202	ble .L_invalidate_dcache_one_line
	203
	204	/* Make ctr hold count of how many times we should loop */
	205	addi r8, r4, (CACHE_LINE_SIZE-1)
	206	srwi r8, r8, CACHE_LINE_POW2
	207	mtctr r8
	208
	209	.L_invalidate_dcache_invalidate_loop:
	210	subic r4, r4, CACHE_LINE_SIZE
	211	dcbi r3, r4
	212	bdnz .L_invalidate_dcache_invalidate_loop
	213
	214	.L_invalidate_dcache_done:
	215	/* Sync restore msr if it was modified */
	216	cmpwi r5, 0
	217	sync /* make sure invalidates have completed */
	218	beq+ 0f
	219	mtmsr r6 /* Restore original translations */
	220	isync /* Ensure data translations are on */
	221	0:
	222	blr
	223
	224	.L_invalidate_dcache_one_line:
	225	xor r4,r4,r4
	226	dcbi 0,r3
	227	b .L_invalidate_dcache_done
1c79356b A	228
	229	/*
	230	* extern void invalidate_icache(vm_offset_t addr, unsigned cnt, boolean phys);
	231	*
	232	* invalidate_icache takes a virtual or physical address and
	233	* count to invalidate, (can be called for multiple virtual pages).
	234	*
	235	* it invalidates the instruction cache for the address range in question.
	236	*/
	237
de355530	238	ENTRY(invalidate_icache, TAG_NO_FRAME_USED)
1c79356b	239
de355530 A	240	/* optionally switch off data translations */
	241	cmpwi r5, 0
	242	mfmsr r6
	243	beq+ 0f
	244	rlwinm r6,r6,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
	245	rlwinm r6,r6,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
	246	rlwinm r7, r6, 0, MSR_DR_BIT+1, MSR_DR_BIT-1
	247	mtmsr r7
	248	isync
	249	0:
1c79356b	250
de355530 A	251	/* Check to see if the address is aligned. */
	252	add r8, r3,r4
	253	andi. r8,r8,(CACHE_LINE_SIZE-1)
	254	beq- .L_invalidate_icache_check
	255	addi r4,r4,CACHE_LINE_SIZE
	256	li r7,(CACHE_LINE_SIZE-1) /* Align buffer & count - avoid overflow problems */
	257	andc r4,r4,r7
	258	andc r3,r3,r7
	259
	260	.L_invalidate_icache_check:
	261	cmpwi r4, CACHE_LINE_SIZE
	262	ble .L_invalidate_icache_one_line
	263
	264	/* Make ctr hold count of how many times we should loop */
	265	addi r8, r4, (CACHE_LINE_SIZE-1)
	266	srwi r8, r8, CACHE_LINE_POW2
	267	mtctr r8
	268
	269	.L_invalidate_icache_invalidate_loop:
	270	subic r4, r4, CACHE_LINE_SIZE
	271	icbi r3, r4
	272	bdnz .L_invalidate_icache_invalidate_loop
1c79356b	273
de355530 A	274	.L_invalidate_icache_done:
	275	sync /* make sure invalidates have completed */
	276	mtmsr r6 /* Restore original translations */
	277	isync /* Ensure data translations are on */
	278	blr
1c79356b	279
de355530 A	280	.L_invalidate_icache_one_line:
	281	xor r4,r4,r4
	282	icbi 0,r3
	283	b .L_invalidate_icache_done