]> git.saurik.com Git - apple/xnu.git/blob - osfmk/ppc/cache.s
projects / apple / xnu.git / blob
? search:


a9afd07cb934597c45564b52ad1bb0d16ee9bab4
[apple/xnu.git] / osfmk / ppc / cache.s
1 /*
2 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
11 * file.
12 *
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
20 *
21 * @APPLE_LICENSE_HEADER_END@
22 */
23 /*
24 * @OSF_COPYRIGHT@
25 */
26
27 #include <ppc/asm.h>
28 #include <ppc/proc_reg.h>
29 #include <ppc/exception.h>
30 #include <assym.s>
31
32 /* These routines run in 32 or 64-bit addressing, and handle
33 * 32 and 128 byte caches. They do not use compare instructions
34 * on addresses, since compares are 32/64-bit-mode-specific.
35 */
36
37 #define kDcbf 0x1
38 #define kDcbfb 31
39 #define kDcbi 0x2
40 #define kDcbib 30
41 #define kIcbi 0x4
42 #define kIcbib 29
43
44
45 /*
46 * extern void flush_dcache(vm_offset_t addr, unsigned count, boolean phys);
47 * extern void flush_dcache64(addr64_t addr, unsigned count, boolean phys);
48 *
49 * flush_dcache takes a virtual or physical address and count to flush
50 * and (can be called for multiple virtual pages).
51 *
52 * it flushes the data cache
53 * cache for the address range in question
54 *
55 * if 'phys' is non-zero then physical addresses will be used
56 */
57
58
59
60 .text
61 .align 5
62 .globl _flush_dcache
63 _flush_dcache:
64 li r0,kDcbf // use DCBF instruction
65 rlwinm r3,r3,0,0,31 // truncate address in case this is a 64-bit machine
66 b cache_op_join // join common code
67
68 .align 5
69 .globl _flush_dcache64
70 _flush_dcache64:
71 rlwinm r3,r3,0,1,0 ; Duplicate high half of long long paddr into top of reg
72 li r0,kDcbf // use DCBF instruction
73 rlwimi r3,r4,0,0,31 ; Combine bottom of long long to full 64-bits
74 mr r4,r5 ; Move count
75 mr r5,r6 ; Move physical flag
76 b cache_op_join // join common code
77
78
79 /*
80 * extern void invalidate_dcache(vm_offset_t va, unsigned count, boolean phys);
81 * extern void invalidate_dcache64(addr64_t va, unsigned count, boolean phys);
82 *
83 * invalidate_dcache takes a virtual or physical address and count to
84 * invalidate and (can be called for multiple virtual pages).
85 *
86 * it invalidates the data cache for the address range in question
87 */
88
89 .globl _invalidate_dcache
90 _invalidate_dcache:
91 li r0,kDcbi // use DCBI instruction
92 rlwinm r3,r3,0,0,31 // truncate address in case this is a 64-bit machine
93 b cache_op_join // join common code
94
95
96 .align 5
97 .globl _invalidate_dcache64
98 _invalidate_dcache64:
99 rlwinm r3,r3,0,1,0 ; Duplicate high half of long long paddr into top of reg
100 li r0,kDcbi // use DCBI instruction
101 rlwimi r3,r4,0,0,31 ; Combine bottom of long long to full 64-bits
102 mr r4,r5 ; Move count
103 mr r5,r6 ; Move physical flag
104 b cache_op_join // join common code
105
106 /*
107 * extern void invalidate_icache(vm_offset_t addr, unsigned cnt, boolean phys);
108 * extern void invalidate_icache64(addr64_t addr, unsigned cnt, boolean phys);
109 *
110 * invalidate_icache takes a virtual or physical address and
111 * count to invalidate, (can be called for multiple virtual pages).
112 *
113 * it invalidates the instruction cache for the address range in question.
114 */
115
116 .globl _invalidate_icache
117 _invalidate_icache:
118 li r0,kIcbi // use ICBI instruction
119 rlwinm r3,r3,0,0,31 // truncate address in case this is a 64-bit machine
120 b cache_op_join // join common code
121
122
123 .align 5
124 .globl _invalidate_icache64
125 _invalidate_icache64:
126 rlwinm r3,r3,0,1,0 ; Duplicate high half of long long paddr into top of reg
127 li r0,kIcbi // use ICBI instruction
128 rlwimi r3,r4,0,0,31 ; Combine bottom of long long to full 64-bits
129 mr r4,r5 ; Move count
130 mr r5,r6 ; Move physical flag
131 b cache_op_join // join common code
132
133 /*
134 * extern void sync_ppage(ppnum_t pa);
135 *
136 * sync_ppage takes a physical page number
137 *
138 * it writes out the data cache and invalidates the instruction
139 * cache for the address range in question
140 */
141
142 .globl _sync_ppage
143 .align 5
144 _sync_ppage: // Should be the most commonly called routine, by far
145 mfsprg r2,2
146 li r0,kDcbf+kIcbi // we need to dcbf and then icbi
147 mtcrf 0x02,r2 ; Move pf64Bit to cr6
148 li r5,1 // set flag for physical addresses
149 li r4,4096 ; Set page size
150 bt++ pf64Bitb,spp64 ; Skip if 64-bit (only they take the hint)
151 rlwinm r3,r3,12,0,19 ; Convert to physical address - 32-bit
152 b cache_op_join ; Join up....
153
154 spp64: sldi r3,r3,12 ; Convert to physical address - 64-bit
155 b cache_op_join ; Join up....
156
157
158
159 /*
160 * extern void sync_cache_virtual(vm_offset_t addr, unsigned count);
161 *
162 * Like "sync_cache", except it takes a virtual address and byte count.
163 * It flushes the data cache, invalidates the I cache, and sync's.
164 */
165
166 .globl _sync_cache_virtual
167 .align 5
168 _sync_cache_virtual:
169 li r0,kDcbf+kIcbi // we need to dcbf and then icbi
170 li r5,0 // set flag for virtual addresses
171 b cache_op_join // join common code
172
173
174 /*
175 * extern void sync_cache(vm_offset_t pa, unsigned count);
176 * extern void sync_cache64(addr64_t pa, unsigned count);
177 *
178 * sync_cache takes a physical address and count to sync, thus
179 * must not be called for multiple virtual pages.
180 *
181 * it writes out the data cache and invalidates the instruction
182 * cache for the address range in question
183 */
184
185 .globl _sync_cache
186 .align 5
187 _sync_cache:
188 li r0,kDcbf+kIcbi // we need to dcbf and then icbi
189 li r5,1 // set flag for physical addresses
190 rlwinm r3,r3,0,0,31 // truncate address in case this is a 64-bit machine
191 b cache_op_join // join common code
192
193 .globl _sync_cache64
194 .align 5
195 _sync_cache64:
196 rlwinm r3,r3,0,1,0 ; Duplicate high half of long long paddr into top of reg
197 li r0,kDcbf+kIcbi // we need to dcbf and then icbi
198 rlwimi r3,r4,0,0,31 ; Combine bottom of long long to full 64-bits
199 mr r4,r5 ; Copy over the length
200 li r5,1 // set flag for physical addresses
201
202
203 // Common code to handle the cache operations.
204
205 cache_op_join: // here with r3=addr, r4=count, r5=phys flag, r0=bits
206 mfsprg r10,2 // r10 <- processor feature flags
207 cmpwi cr5,r5,0 // using physical addresses?
208 mtcrf 0x01,r0 // move kDcbf, kDcbi, and kIcbi bits to CR7
209 andi. r9,r10,pf32Byte+pf128Byte // r9 <- cache line size
210 mtcrf 0x02,r10 // move pf64Bit bit to CR6
211 subi r8,r9,1 // r8 <- (linesize-1)
212 beq-- cr5,cache_op_2 // skip if using virtual addresses
213
214 bf-- pf64Bitb,cache_op_not64 // This is not a 64-bit machine
215
216 srdi r12,r3,31 // Slide bit 32 to bit 63
217 cmpldi r12,1 // Are we in the I/O mapped area?
218 beqlr-- // No cache ops allowed here...
219
220 cache_op_not64:
221 mflr r12 // save return address
222 bl EXT(ml_set_physical) // turn on physical addressing
223 mtlr r12 // restore return address
224
225 // get r3=first cache line, r4=first line not in set, r6=byte count
226
227 cache_op_2:
228 add r7,r3,r4 // point to 1st byte not to operate on
229 andc r3,r3,r8 // r3 <- 1st line to operate on
230 add r4,r7,r8 // round up
231 andc r4,r4,r8 // r4 <- 1st line not to operate on
232 sub. r6,r4,r3 // r6 <- byte count to operate on
233 beq-- cache_op_exit // nothing to do
234 bf-- kDcbfb,cache_op_6 // no need to dcbf
235
236
237 // DCBF loop
238
239 cache_op_5:
240 sub. r6,r6,r9 // more to go?
241 dcbf r6,r3 // flush next line to RAM
242 bne cache_op_5 // loop if more to go
243 sync // make sure the data reaches RAM
244 sub r6,r4,r3 // reset count
245
246
247 // ICBI loop
248
249 cache_op_6:
250 bf-- kIcbib,cache_op_8 // no need to icbi
251 cache_op_7:
252 sub. r6,r6,r9 // more to go?
253 icbi r6,r3 // invalidate next line
254 bne cache_op_7
255 sub r6,r4,r3 // reset count
256 isync
257 sync
258
259
260 // DCBI loop
261
262 cache_op_8:
263 bf++ kDcbib,cache_op_exit // no need to dcbi
264 cache_op_9:
265 sub. r6,r6,r9 // more to go?
266 dcbi r6,r3 // invalidate next line
267 bne cache_op_9
268 sync
269
270
271 // restore MSR iff necessary and done
272
273 cache_op_exit:
274 beqlr-- cr5 // if using virtual addresses, no need to restore MSR
275 b EXT(ml_restore) // restore MSR and return
276
277
278 ////////////////////////////////////////////////////
279
280 .align 5
281 .globl _dcache_incoherent_io_store64
282 _dcache_incoherent_io_store64:
283 rlwinm r3,r3,0,1,0 ; Duplicate high half of long long paddr into top of reg
284 rlwimi r3,r4,0,0,31 ; Combine bottom of long long to full 64-bits
285 mr r4,r5 ; Move count
286
287 // here with r3=addr, r4=count
288 mfsprg r10,2 // r10 <- processor feature flags
289 andi. r9,r10,pf32Byte+pf128Byte // r9 <- cache line size
290 mtcrf 0x02,r10 // move pf64Bit bit to CR6
291 subi r8,r9,1 // r8 <- (linesize-1)
292
293 bf-- pf64Bitb,cache_ios_not64 // This is not a 64-bit machine
294
295 srdi r12,r3,31 // Slide bit 32 to bit 63
296 cmpldi r12,1 // Are we in the I/O mapped area?
297 beqlr-- // No cache ops allowed here...
298
299 cache_ios_not64:
300 mflr r12 // save return address
301 bl EXT(ml_set_physical) // turn on physical addressing
302 mtlr r12 // restore return address
303
304 // get r3=first cache line, r4=first line not in set, r6=byte count
305 add r7,r3,r4 // point to 1st byte not to operate on
306 andc r3,r3,r8 // r3 <- 1st line to operate on
307 add r4,r7,r8 // round up
308 andc r4,r4,r8 // r4 <- 1st line not to operate on
309 sub. r6,r4,r3 // r6 <- byte count to operate on
310 beq-- cache_ios_exit // nothing to do
311
312 sub. r6,r6,r9 // >1 line?
313 beq cache_ios_last_line // use dcbst on all lines but last
314
315 // DCBST loop
316 cache_ios_5:
317 sub. r6,r6,r9 // more to go?
318 dcbst r6,r3 // store next line
319 bne cache_ios_5 // loop if more to go
320
321 cache_ios_last_line:
322 sync // flush last line
323 isync
324 dcbf r6,r3
325 sync
326 isync
327 add r6,r6,r3
328 lwz r0,0(r6) // make sure the data reaches RAM (not just the memory controller)
329 isync
330
331 // restore MSR
332 cache_ios_exit:
333 b EXT(ml_restore) // restore MSR and return
334
335
336 ////////////////////////////////////////////////////
337
338 .align 5
339 .globl _dcache_incoherent_io_flush64
340 _dcache_incoherent_io_flush64:
341 rlwinm r3,r3,0,1,0 ; Duplicate high half of long long paddr into top of reg
342 rlwimi r3,r4,0,0,31 ; Combine bottom of long long to full 64-bits
343 mr r4,r5 ; Move count
344
345 // here with r3=addr, r4=count
346 mfsprg r10,2 // r10 <- processor feature flags
347 andi. r9,r10,pf32Byte+pf128Byte // r9 <- cache line size
348 mtcrf 0x02,r10 // move pf64Bit bit to CR6
349 subi r8,r9,1 // r8 <- (linesize-1)
350
351 bf-- pf64Bitb,cache_iof_not64 // This is not a 64-bit machine
352
353 srdi r12,r3,31 // Slide bit 32 to bit 63
354 cmpldi r12,1 // Are we in the I/O mapped area?
355 beqlr-- // No cache ops allowed here...
356
357 cache_iof_not64:
358 mflr r12 // save return address
359 bl EXT(ml_set_physical) // turn on physical addressing
360 mtlr r12 // restore return address
361
362 // get r3=first cache line, r4=first line not in set, r6=byte count
363 add r7,r3,r4 // point to 1st byte not to operate on
364 andc r3,r3,r8 // r3 <- 1st line to operate on
365 add r4,r7,r8 // round up
366 andc r4,r4,r8 // r4 <- 1st line not to operate on
367 sub. r6,r4,r3 // r6 <- byte count to operate on
368 beq-- cache_iof_exit // nothing to do
369
370 // DCBF loop
371 cache_iof_5:
372 sub. r6,r6,r9 // more to go?
373 dcbf r6,r3 // store next line
374 bne cache_iof_5 // loop if more to go
375
376 cache_iof_last_line:
377 sync // flush last line
378 isync
379
380 // restore MSR
381 cache_iof_exit:
382 b EXT(ml_restore) // restore MSR and return
383
384
mirror of XNU