]> git.saurik.com Git - apple/xnu.git/blob - osfmk/ppc/machine_routines_asm.s
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-792.tar.gz
[apple/xnu.git] / osfmk / ppc / machine_routines_asm.s
1 /*
2 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * The contents of this file constitute Original Code as defined in and
7 * are subject to the Apple Public Source License Version 1.1 (the
8 * "License"). You may not use this file except in compliance with the
9 * License. Please obtain a copy of the License at
10 * http://www.apple.com/publicsource and read it before using this file.
11 *
12 * This Original Code and all software distributed under the License are
13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17 * License for the specific language governing rights and limitations
18 * under the License.
19 *
20 * @APPLE_LICENSE_HEADER_END@
21 */
22 #include <ppc/asm.h>
23 #include <ppc/proc_reg.h>
24 #include <assym.s>
25 #include <debug.h>
26 #include <mach/ppc/vm_param.h>
27 #include <ppc/exception.h>
28
29
30 /*
31 * ml_set_physical() -- turn off DR and (if 64-bit) turn SF on
32 * it is assumed that pf64Bit is already in cr6
33 * ml_set_physical_get_ffs() -- turn DR off, SF on, and get feature flags
34 * ml_set_physical_disabled() -- turn DR and EE off, SF on, get feature flags
35 * ml_set_translation_off() -- turn DR, IR, and EE off, SF on, get feature flags
36 *
37 * Callable only from assembler, these return:
38 * r2 -- new MSR
39 * r11 -- old MSR
40 * r10 -- feature flags (pf64Bit etc, ie SPRG 2)
41 * cr6 -- feature flags 24-27, ie pf64Bit, pf128Byte, and pf32Byte
42 *
43 * Uses r0 and r2. ml_set_translation_off also uses r3 and cr5.
44 */
45
46 .align 4
47 .globl EXT(ml_set_translation_off)
48 LEXT(ml_set_translation_off)
49 mfsprg r10,2 // get feature flags
50 li r0,0 ; Clear this
51 mtcrf 0x02,r10 // move pf64Bit etc to cr6
52 ori r0,r0,lo16(MASK(MSR_EE)+MASK(MSR_FP)+MASK(MSR_IR)+MASK(MSR_DR)) // turn off all 4
53 mfmsr r11 // get MSR
54 oris r0,r0,hi16(MASK(MSR_VEC)) // Turn off vector too
55 mtcrf 0x04,r10 // move pfNoMSRir etc to cr5
56 andc r2,r11,r0 // turn off EE, IR, and DR
57 bt++ pf64Bitb,ml_set_physical_64 // skip if 64-bit (only they take the hint)
58 bf pfNoMSRirb,ml_set_physical_32 // skip if we can load MSR directly
59 li r0,loadMSR // Get the MSR setter SC
60 mr r3,r2 // copy new MSR to r2
61 sc // Set it
62 blr
63
64 .align 4
65 .globl EXT(ml_set_physical_disabled)
66
67 LEXT(ml_set_physical_disabled)
68 li r0,0 ; Clear
69 mfsprg r10,2 // get feature flags
70 ori r0,r0,lo16(MASK(MSR_EE)) // turn EE and fp off
71 mtcrf 0x02,r10 // move pf64Bit etc to cr6
72 b ml_set_physical_join
73
74 .align 5
75 .globl EXT(ml_set_physical_get_ffs)
76
77 LEXT(ml_set_physical_get_ffs)
78 mfsprg r10,2 // get feature flags
79 mtcrf 0x02,r10 // move pf64Bit etc to cr6
80
81 .globl EXT(ml_set_physical)
82 LEXT(ml_set_physical)
83
84 li r0,0 // do not turn off interrupts
85
86 ml_set_physical_join:
87 oris r0,r0,hi16(MASK(MSR_VEC)) // Always gonna turn of vectors
88 mfmsr r11 // get MSR
89 ori r0,r0,lo16(MASK(MSR_DR)+MASK(MSR_FP)) // always turn off DR and FP bit
90 andc r2,r11,r0 // turn off DR and maybe EE
91 bt++ pf64Bitb,ml_set_physical_64 // skip if 64-bit (only they take the hint)
92 ml_set_physical_32:
93 mtmsr r2 // turn off translation
94 isync
95 blr
96
97 ml_set_physical_64:
98 li r0,1 // get a 1 to slam into SF
99 rldimi r2,r0,63,MSR_SF_BIT // set SF bit (bit 0)
100 mtmsrd r2 // set 64-bit mode, turn off data relocation
101 isync // synchronize
102 blr
103
104
105 /*
106 * ml_restore(old_MSR)
107 *
108 * Callable only from assembler, restores the MSR in r11 saved by ml_set_physical.
109 * We assume cr6 and r11 are as set by ml_set_physical, ie:
110 * cr6 - pf64Bit flag (feature flags 24-27)
111 * r11 - old MSR
112 */
113
114 .align 5
115 .globl EXT(ml_restore)
116
117 LEXT(ml_restore)
118 bt++ pf64Bitb,ml_restore_64 // handle 64-bit cpus (only they take the hint)
119 mtmsr r11 // restore a 32-bit MSR
120 isync
121 blr
122
123 ml_restore_64:
124 mtmsrd r11 // restore a 64-bit MSR
125 isync
126 blr
127
128
129 /* PCI config cycle probing
130 *
131 * boolean_t ml_probe_read(vm_offset_t paddr, unsigned int *val)
132 *
133 * Read the memory location at physical address paddr.
134 * This is a part of a device probe, so there is a good chance we will
135 * have a machine check here. So we have to be able to handle that.
136 * We assume that machine checks are enabled both in MSR and HIDs
137 */
138
139 ; Force a line boundry here
140 .align 5
141 .globl EXT(ml_probe_read)
142
143 LEXT(ml_probe_read)
144
145 mfsprg r9,2 ; Get feature flags
146
147 rlwinm. r0,r9,0,pf64Bitb,pf64Bitb ; Are we on a 64-bit machine?
148 rlwinm r3,r3,0,0,31 ; Clean up for 64-bit machines
149 bne++ mpr64bit ; Go do this the 64-bit way...
150
151 mpr32bit: lis r8,hi16(MASK(MSR_VEC)) ; Get the vector flag
152 mfmsr r0 ; Save the current MSR
153 ori r8,r8,lo16(MASK(MSR_FP)) ; Add the FP flag
154
155 neg r10,r3 ; Number of bytes to end of page
156 andc r0,r0,r8 ; Clear VEC and FP
157 rlwinm. r10,r10,0,20,31 ; Clear excess junk and test for page bndry
158 ori r8,r8,lo16(MASK(MSR_EE)|MASK(MSR_IR)|MASK(MSR_DR)) ; Drop EE, IR, and DR
159 mr r12,r3 ; Save the load address
160 andc r2,r0,r8 ; Clear VEC, FP, and EE
161 mtcrf 0x04,r9 ; Set the features
162 cmplwi cr1,r10,4 ; At least 4 bytes left in page?
163 beq- mprdoit ; We are right on the boundary...
164 li r3,0
165 bltlr- cr1 ; No, just return failure...
166
167 mprdoit:
168
169 bt pfNoMSRirb,mprNoMSR ; No MSR...
170
171 mtmsr r2 ; Translation and all off
172 isync ; Toss prefetch
173 b mprNoMSRx
174
175 mprNoMSR:
176 mr r5,r0
177 li r0,loadMSR ; Get the MSR setter SC
178 mr r3,r2 ; Get new MSR
179 sc ; Set it
180 mr r0,r5
181 li r3,0
182 mprNoMSRx:
183
184 mfspr r6, hid0 ; Get a copy of hid0
185
186 rlwinm. r5, r9, 0, pfNoMuMMCKb, pfNoMuMMCKb ; Check for NoMuMMCK
187 bne mprNoMuM
188
189 rlwinm r5, r6, 0, ice+1, ice-1 ; Turn off L1 I-Cache
190 mtspr hid0, r5
191 isync ; Wait for I-Cache off
192 rlwinm r5, r6, 0, mum+1, mum-1 ; Turn off MuM w/ I-Cache on
193 mtspr hid0, r5
194 mprNoMuM:
195
196 ;
197 ; We need to insure that there is no more than 1 BAT register that
198 ; can get a hit. There could be repercussions beyond the ken
199 ; of mortal man. It is best not to tempt fate.
200 ;
201
202 ; Note: we will reload these from the shadow BATs later
203
204 li r10,0 ; Clear a register
205
206 sync ; Make sure all is well
207
208 mtdbatu 1,r10 ; Invalidate DBAT 1
209 mtdbatu 2,r10 ; Invalidate DBAT 2
210 mtdbatu 3,r10 ; Invalidate DBAT 3
211
212 rlwinm r10,r12,0,0,14 ; Round down to a 128k boundary
213 ori r11,r10,0x32 ; Set uncached, coherent, R/W
214 ori r10,r10,2 ; Make the upper half (128k, valid supervisor)
215 mtdbatl 0,r11 ; Set lower BAT first
216 mtdbatu 0,r10 ; Now the upper
217 sync ; Just make sure
218
219 dcbf 0,r12 ; Make sure we kill the cache to avoid paradoxes
220 sync
221
222 ori r11,r2,lo16(MASK(MSR_DR)) ; Turn on data translation
223 mtmsr r11 ; Do it for real
224 isync ; Make sure of it
225
226 eieio ; Make sure of all previous accesses
227 sync ; Make sure it is all caught up
228
229 lwz r11,0(r12) ; Get it and maybe machine check here
230
231 eieio ; Make sure of ordering again
232 sync ; Get caught up yet again
233 isync ; Do not go further till we are here
234
235 mtmsr r2 ; Turn translation back off
236 isync
237
238 lis r10,hi16(EXT(shadow_BAT)+shdDBAT) ; Get shadow address
239 ori r10,r10,lo16(EXT(shadow_BAT)+shdDBAT) ; Get shadow address
240
241 lwz r5,0(r10) ; Pick up DBAT 0 high
242 lwz r6,4(r10) ; Pick up DBAT 0 low
243 lwz r7,8(r10) ; Pick up DBAT 1 high
244 lwz r8,16(r10) ; Pick up DBAT 2 high
245 lwz r9,24(r10) ; Pick up DBAT 3 high
246
247 mtdbatu 0,r5 ; Restore DBAT 0 high
248 mtdbatl 0,r6 ; Restore DBAT 0 low
249 mtdbatu 1,r7 ; Restore DBAT 1 high
250 mtdbatu 2,r8 ; Restore DBAT 2 high
251 mtdbatu 3,r9 ; Restore DBAT 3 high
252 sync
253
254 li r3,1 ; We made it
255
256 mtmsr r0 ; Restore translation and exceptions
257 isync ; Toss speculations
258
259 stw r11,0(r4) ; Save the loaded value
260 blr ; Return...
261
262 ; Force a line boundry here. This means we will be able to check addresses better
263 .align 5
264 .globl EXT(ml_probe_read_mck)
265 LEXT(ml_probe_read_mck)
266
267
268 /* PCI config cycle probing - 64-bit
269 *
270 * boolean_t ml_probe_read_64(addr64_t paddr, unsigned int *val)
271 *
272 * Read the memory location at physical address paddr.
273 * This is a part of a device probe, so there is a good chance we will
274 * have a machine check here. So we have to be able to handle that.
275 * We assume that machine checks are enabled both in MSR and HIDs
276 */
277
278 ; Force a line boundry here
279 .align 6
280 .globl EXT(ml_probe_read_64)
281
282 LEXT(ml_probe_read_64)
283
284 mfsprg r9,2 ; Get feature flags
285 rlwinm r3,r3,0,1,0 ; Copy low 32 bits to top 32
286 rlwinm. r0,r9,0,pf64Bitb,pf64Bitb ; Are we on a 64-bit machine?
287 rlwimi r3,r4,0,0,31 ; Insert low part of 64-bit address in bottom 32 bits
288
289 mr r4,r5 ; Move result to common register
290 beq-- mpr32bit ; Go do this the 32-bit way...
291
292 mpr64bit: andi. r0,r3,3 ; Check if we are on a word boundary
293 li r0,0 ; Clear the EE bit (and everything else for that matter)
294 bne-- mprFail ; Boundary not good...
295 mfmsr r11 ; Get the MSR
296 mtmsrd r0,1 ; Set the EE bit only (do not care about RI)
297 rlwinm r11,r11,0,MSR_EE_BIT,MSR_EE_BIT ; Isolate just the EE bit
298 mfmsr r10 ; Refresh our view of the MSR (VMX/FP may have changed)
299 or r12,r10,r11 ; Turn on EE if on before we turned it off
300 ori r0,r0,lo16(MASK(MSR_IR)|MASK(MSR_DR)) ; Get the IR and DR bits
301 li r2,1 ; Get a 1
302 sldi r2,r2,63 ; Get the 64-bit bit
303 andc r10,r10,r0 ; Clear IR and DR
304 or r10,r10,r2 ; Set 64-bit
305
306 li r0,1 ; Get a 1
307 mtmsrd r10 ; Translation and EE off, 64-bit on
308 isync
309
310 sldi r0,r0,32+8 ; Get the right bit to inhibit caching
311
312 mfspr r8,hid4 ; Get HID4
313 or r2,r8,r0 ; Set bit to make real accesses cache-inhibited
314 sync ; Sync up
315 mtspr hid4,r2 ; Make real accesses cache-inhibited
316 isync ; Toss prefetches
317
318 lis r7,0xE000 ; Get the unlikeliest ESID possible
319 srdi r7,r7,1 ; Make 0x7FFFFFFFF0000000
320 slbie r7 ; Make sure the ERAT is cleared
321
322 sync
323 isync
324
325 eieio ; Make sure of all previous accesses
326
327 lwz r11,0(r3) ; Get it and maybe machine check here
328
329 eieio ; Make sure of ordering again
330 sync ; Get caught up yet again
331 isync ; Do not go further till we are here
332
333 sync ; Sync up
334 mtspr hid4,r8 ; Make real accesses not cache-inhibited
335 isync ; Toss prefetches
336
337 lis r7,0xE000 ; Get the unlikeliest ESID possible
338 srdi r7,r7,1 ; Make 0x7FFFFFFFF0000000
339 slbie r7 ; Make sure the ERAT is cleared
340
341 mtmsrd r12 ; Restore entry MSR
342 isync
343
344 stw r11,0(r4) ; Pass back the result
345 li r3,1 ; Indicate success
346 blr ; Leave...
347
348 mprFail: li r3,0 ; Set failure
349 blr ; Leave...
350
351 ; Force a line boundry here. This means we will be able to check addresses better
352 .align 6
353 .globl EXT(ml_probe_read_mck_64)
354 LEXT(ml_probe_read_mck_64)
355
356
357 /* Read physical address byte
358 *
359 * unsigned int ml_phys_read_byte(vm_offset_t paddr)
360 * unsigned int ml_phys_read_byte_64(addr64_t paddr)
361 *
362 * Read the byte at physical address paddr. Memory should not be cache inhibited.
363 */
364
365 ; Force a line boundry here
366
367 .align 5
368 .globl EXT(ml_phys_read_byte_64)
369
370 LEXT(ml_phys_read_byte_64)
371
372 rlwinm r3,r3,0,1,0 ; Copy low 32 bits to top 32
373 rlwimi r3,r4,0,0,31 ; Insert low part of 64-bit address in bottom 32 bits
374 b ml_phys_read_byte_join
375
376 .globl EXT(ml_phys_read_byte)
377
378 LEXT(ml_phys_read_byte)
379 rlwinm r3,r3,0,0,31 ; truncate address to 32-bits
380 ml_phys_read_byte_join: ; r3 = address to read (reg64_t)
381 mflr r11 ; Save the return
382 bl rdwrpre ; Get set up, translation/interrupts off, 64-bit on, etc.
383
384 lbz r3,0(r3) ; Get the byte
385 b rdwrpost ; Clean up and leave...
386
387
388 /* Read physical address half word
389 *
390 * unsigned int ml_phys_read_half(vm_offset_t paddr)
391 * unsigned int ml_phys_read_half_64(addr64_t paddr)
392 *
393 * Read the half word at physical address paddr. Memory should not be cache inhibited.
394 */
395
396 ; Force a line boundry here
397
398 .align 5
399 .globl EXT(ml_phys_read_half_64)
400
401 LEXT(ml_phys_read_half_64)
402
403 rlwinm r3,r3,0,1,0 ; Copy low 32 bits to top 32
404 rlwimi r3,r4,0,0,31 ; Insert low part of 64-bit address in bottom 32 bits
405 b ml_phys_read_half_join
406
407 .globl EXT(ml_phys_read_half)
408
409 LEXT(ml_phys_read_half)
410 rlwinm r3,r3,0,0,31 ; truncate address to 32-bits
411 ml_phys_read_half_join: ; r3 = address to read (reg64_t)
412 mflr r11 ; Save the return
413 bl rdwrpre ; Get set up, translation/interrupts off, 64-bit on, etc.
414
415 lhz r3,0(r3) ; Get the half word
416 b rdwrpost ; Clean up and leave...
417
418
419 /* Read physical address word
420 *
421 * unsigned int ml_phys_read(vm_offset_t paddr)
422 * unsigned int ml_phys_read_64(addr64_t paddr)
423 * unsigned int ml_phys_read_word(vm_offset_t paddr)
424 * unsigned int ml_phys_read_word_64(addr64_t paddr)
425 *
426 * Read the word at physical address paddr. Memory should not be cache inhibited.
427 */
428
429 ; Force a line boundry here
430
431 .align 5
432 .globl EXT(ml_phys_read_64)
433 .globl EXT(ml_phys_read_word_64)
434
435 LEXT(ml_phys_read_64)
436 LEXT(ml_phys_read_word_64)
437
438 rlwinm r3,r3,0,1,0 ; Copy low 32 bits to top 32
439 rlwimi r3,r4,0,0,31 ; Insert low part of 64-bit address in bottom 32 bits
440 b ml_phys_read_word_join
441
442 .globl EXT(ml_phys_read)
443 .globl EXT(ml_phys_read_word)
444
445 LEXT(ml_phys_read)
446 LEXT(ml_phys_read_word)
447 rlwinm r3,r3,0,0,31 ; truncate address to 32-bits
448 ml_phys_read_word_join: ; r3 = address to read (reg64_t)
449 mflr r11 ; Save the return
450 bl rdwrpre ; Get set up, translation/interrupts off, 64-bit on, etc.
451
452 lwz r3,0(r3) ; Get the word
453 b rdwrpost ; Clean up and leave...
454
455
456 /* Read physical address double word
457 *
458 * unsigned long long ml_phys_read_double(vm_offset_t paddr)
459 * unsigned long long ml_phys_read_double_64(addr64_t paddr)
460 *
461 * Read the double word at physical address paddr. Memory should not be cache inhibited.
462 */
463
464 ; Force a line boundry here
465
466 .align 5
467 .globl EXT(ml_phys_read_double_64)
468
469 LEXT(ml_phys_read_double_64)
470
471 rlwinm r3,r3,0,1,0 ; Copy low 32 bits to top 32
472 rlwimi r3,r4,0,0,31 ; Insert low part of 64-bit address in bottom 32 bits
473 b ml_phys_read_double_join
474
475 .globl EXT(ml_phys_read_double)
476
477 LEXT(ml_phys_read_double)
478 rlwinm r3,r3,0,0,31 ; truncate address to 32-bits
479 ml_phys_read_double_join: ; r3 = address to read (reg64_t)
480 mflr r11 ; Save the return
481 bl rdwrpre ; Get set up, translation/interrupts off, 64-bit on, etc.
482
483 lwz r4,4(r3) ; Get the low word
484 lwz r3,0(r3) ; Get the high word
485 b rdwrpost ; Clean up and leave...
486
487
488 /* Write physical address byte
489 *
490 * void ml_phys_write_byte(vm_offset_t paddr, unsigned int data)
491 * void ml_phys_write_byte_64(addr64_t paddr, unsigned int data)
492 *
493 * Write the byte at physical address paddr. Memory should not be cache inhibited.
494 */
495
496 .align 5
497 .globl EXT(ml_phys_write_byte_64)
498
499 LEXT(ml_phys_write_byte_64)
500
501 rlwinm r3,r3,0,1,0 ; Copy low 32 bits to top 32
502 rlwimi r3,r4,0,0,31 ; Insert low part of 64-bit address in bottom 32 bits
503 mr r4,r5 ; Copy over the data
504 b ml_phys_write_byte_join
505
506 .globl EXT(ml_phys_write_byte)
507
508 LEXT(ml_phys_write_byte)
509 rlwinm r3,r3,0,0,31 ; truncate address to 32-bits
510 ml_phys_write_byte_join: ; r3 = address to write (reg64_t), r4 = data
511 mflr r11 ; Save the return
512 bl rdwrpre ; Get set up, translation/interrupts off, 64-bit on, etc.
513
514 stb r4,0(r3) ; Set the byte
515 b rdwrpost ; Clean up and leave...
516
517
518 /* Write physical address half word
519 *
520 * void ml_phys_write_half(vm_offset_t paddr, unsigned int data)
521 * void ml_phys_write_half_64(addr64_t paddr, unsigned int data)
522 *
523 * Write the half word at physical address paddr. Memory should not be cache inhibited.
524 */
525
526 .align 5
527 .globl EXT(ml_phys_write_half_64)
528
529 LEXT(ml_phys_write_half_64)
530
531 rlwinm r3,r3,0,1,0 ; Copy low 32 bits to top 32
532 rlwimi r3,r4,0,0,31 ; Insert low part of 64-bit address in bottom 32 bits
533 mr r4,r5 ; Copy over the data
534 b ml_phys_write_half_join
535
536 .globl EXT(ml_phys_write_half)
537
538 LEXT(ml_phys_write_half)
539 rlwinm r3,r3,0,0,31 ; truncate address to 32-bits
540 ml_phys_write_half_join: ; r3 = address to write (reg64_t), r4 = data
541 mflr r11 ; Save the return
542 bl rdwrpre ; Get set up, translation/interrupts off, 64-bit on, etc.
543
544 sth r4,0(r3) ; Set the half word
545 b rdwrpost ; Clean up and leave...
546
547
548 /* Write physical address word
549 *
550 * void ml_phys_write(vm_offset_t paddr, unsigned int data)
551 * void ml_phys_write_64(addr64_t paddr, unsigned int data)
552 * void ml_phys_write_word(vm_offset_t paddr, unsigned int data)
553 * void ml_phys_write_word_64(addr64_t paddr, unsigned int data)
554 *
555 * Write the word at physical address paddr. Memory should not be cache inhibited.
556 */
557
558 .align 5
559 .globl EXT(ml_phys_write_64)
560 .globl EXT(ml_phys_write_word_64)
561
562 LEXT(ml_phys_write_64)
563 LEXT(ml_phys_write_word_64)
564
565 rlwinm r3,r3,0,1,0 ; Copy low 32 bits to top 32
566 rlwimi r3,r4,0,0,31 ; Insert low part of 64-bit address in bottom 32 bits
567 mr r4,r5 ; Copy over the data
568 b ml_phys_write_word_join
569
570 .globl EXT(ml_phys_write)
571 .globl EXT(ml_phys_write_word)
572
573 LEXT(ml_phys_write)
574 LEXT(ml_phys_write_word)
575 rlwinm r3,r3,0,0,31 ; truncate address to 32-bits
576 ml_phys_write_word_join: ; r3 = address to write (reg64_t), r4 = data
577 mflr r11 ; Save the return
578 bl rdwrpre ; Get set up, translation/interrupts off, 64-bit on, etc.
579
580 stw r4,0(r3) ; Set the word
581 b rdwrpost ; Clean up and leave...
582
583
584 /* Write physical address double word
585 *
586 * void ml_phys_write_double(vm_offset_t paddr, unsigned long long data)
587 * void ml_phys_write_double_64(addr64_t paddr, unsigned long long data)
588 *
589 * Write the double word at physical address paddr. Memory should not be cache inhibited.
590 */
591
592 .align 5
593 .globl EXT(ml_phys_write_double_64)
594
595 LEXT(ml_phys_write_double_64)
596
597 rlwinm r3,r3,0,1,0 ; Copy low 32 bits to top 32
598 rlwimi r3,r4,0,0,31 ; Insert low part of 64-bit address in bottom 32 bits
599 mr r4,r5 ; Copy over the high data
600 mr r5,r6 ; Copy over the low data
601 b ml_phys_write_double_join
602
603 .globl EXT(ml_phys_write_double)
604
605 LEXT(ml_phys_write_double)
606 rlwinm r3,r3,0,0,31 ; truncate address to 32-bits
607 ml_phys_write_double_join: ; r3 = address to write (reg64_t), r4,r5 = data (long long)
608 mflr r11 ; Save the return
609 bl rdwrpre ; Get set up, translation/interrupts off, 64-bit on, etc.
610
611 stw r4,0(r3) ; Set the high word
612 stw r5,4(r3) ; Set the low word
613 b rdwrpost ; Clean up and leave...
614
615
616 .align 5
617
618 rdwrpre: mfsprg r12,2 ; Get feature flags
619 lis r8,hi16(MASK(MSR_VEC)) ; Get the vector flag
620 mfmsr r10 ; Save the MSR
621 ori r8,r8,lo16(MASK(MSR_FP)) ; Add the FP flag
622 mtcrf 0x02,r12 ; move pf64Bit
623 andc r10,r10,r8 ; Clear VEC and FP
624 ori r9,r8,lo16(MASK(MSR_EE)|MASK(MSR_IR)|MASK(MSR_DR)) ; Drop EE, DR, and IR
625 li r2,1 ; Prepare for 64 bit
626 andc r9,r10,r9 ; Clear VEC, FP, DR, and EE
627 bf-- pf64Bitb,rdwrpre32 ; Join 32-bit code...
628
629 srdi r7,r3,31 ; Get a 1 if address is in I/O memory
630 rldimi r9,r2,63,MSR_SF_BIT ; set SF bit (bit 0)
631 cmpldi cr7,r7,1 ; Is source in I/O memory?
632 mtmsrd r9 ; set 64-bit mode, turn off EE, DR, and IR
633 isync ; synchronize
634
635 sldi r0,r2,32+8 ; Get the right bit to turn off caching
636
637 bnelr++ cr7 ; We are not in the I/O area, all ready...
638
639 mfspr r8,hid4 ; Get HID4
640 or r2,r8,r0 ; Set bit to make real accesses cache-inhibited
641 sync ; Sync up
642 mtspr hid4,r2 ; Make real accesses cache-inhibited
643 isync ; Toss prefetches
644
645 lis r7,0xE000 ; Get the unlikeliest ESID possible
646 srdi r7,r7,1 ; Make 0x7FFFFFFFF0000000
647 slbie r7 ; Make sure the ERAT is cleared
648
649 sync
650 isync
651 blr ; Finally, all ready...
652
653 .align 5
654
655 rdwrpre32: rlwimi r9,r10,0,MSR_IR_BIT,MSR_IR_BIT ; Leave the IR bit unchanged
656 mtmsr r9 ; Drop EE, DR, and leave IR unchanged
657 isync
658 blr ; All set up, leave...
659
660 .align 5
661
662 rdwrpost: mtlr r11 ; Restore the return
663 bt++ pf64Bitb,rdwrpost64 ; Join 64-bit code...
664
665 mtmsr r10 ; Restore entry MSR (sans FP and VEC)
666 isync
667 blr ; Leave...
668
669 rdwrpost64: bne++ cr7,rdwrpcok ; Skip enabling real mode caching if we did not change it...
670
671 sync ; Sync up
672 mtspr hid4,r8 ; Make real accesses not cache-inhibited
673 isync ; Toss prefetches
674
675 lis r7,0xE000 ; Get the unlikeliest ESID possible
676 srdi r7,r7,1 ; Make 0x7FFFFFFFF0000000
677 slbie r7 ; Make sure the ERAT is cleared
678
679 rdwrpcok: mtmsrd r10 ; Restore entry MSR (sans FP and VEC)
680 isync
681 blr ; Leave...
682
683
684 /* set interrupts enabled or disabled
685 *
686 * boolean_t set_interrupts_enabled(boolean_t enable)
687 *
688 * Set EE bit to "enable" and return old value as boolean
689 */
690
691 ; Force a line boundry here
692 .align 5
693 .globl EXT(ml_set_interrupts_enabled)
694
695 LEXT(ml_set_interrupts_enabled)
696
697 andi. r4,r3,1 ; Are we turning interruptions on?
698 lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable
699 mfmsr r5 ; Get the current MSR
700 ori r0,r0,lo16(MASK(MSR_EE)|MASK(MSR_FP)) ; Get float enable and EE enable
701 rlwinm r3,r5,17,31,31 ; Set return value
702 andc r5,r5,r0 ; Force VEC and FP off
703 bne CheckPreemption ; Interrupts going on, check ASTs...
704
705 mtmsr r5 ; Slam diable (always going disabled here)
706 isync ; Need this because FP/Vec might go off
707 blr
708
709 .align 5
710
711 CheckPreemption:
712 mfsprg r9,1 ; Get current activation
713 lwz r7,ACT_PER_PROC(r9) ; Get the per_proc block
714 ori r5,r5,lo16(MASK(MSR_EE)) ; Turn on the enable
715 lwz r8,PP_PENDING_AST(r7) ; Get pending AST mask
716 li r6,AST_URGENT ; Get the type we will preempt for
717 lwz r7,ACT_PREEMPT_CNT(r9) ; Get preemption count
718 lis r0,hi16(DoPreemptCall) ; High part of Preempt FW call
719 cmpwi cr1,r7,0 ; Are preemptions masked off?
720 and. r8,r8,r6 ; Are we urgent?
721 crorc cr1_eq,cr0_eq,cr1_eq ; Remember if preemptions are masked or not urgent
722 ori r0,r0,lo16(DoPreemptCall) ; Bottome of FW call
723
724 mtmsr r5 ; Restore the MSR now, before we can preempt
725 isync ; Need this because FP/Vec might go off
726
727 beqlr++ cr1 ; Return if no premption...
728 sc ; Preempt
729 blr
730
731 ; Force a line boundry here
732 .align 5
733 .globl EXT(timer_update)
734
735 LEXT(timer_update)
736 stw r4,TIMER_HIGHCHK(r3)
737 eieio
738 stw r5,TIMER_LOW(r3)
739 eieio
740 stw r4,TIMER_HIGH(r3)
741 blr
742
743 ; Force a line boundry here
744 .align 5
745 .globl EXT(timer_grab)
746
747 LEXT(timer_grab)
748 0: lwz r11,TIMER_HIGH(r3)
749 lwz r4,TIMER_LOW(r3)
750 isync
751 lwz r9,TIMER_HIGHCHK(r3)
752 cmpw r11,r9
753 bne-- 0b
754 mr r3,r11
755 blr
756
757 ; Force a line boundry here
758 .align 5
759 .globl EXT(timer_event)
760
761 LEXT(timer_event)
762 mfsprg r10,1 ; Get the current activation
763 lwz r10,ACT_PER_PROC(r10) ; Get the per_proc block
764 addi r10,r10,PP_PROCESSOR
765 lwz r11,CURRENT_TIMER(r10)
766
767 lwz r9,TIMER_LOW(r11)
768 lwz r2,TIMER_TSTAMP(r11)
769 add r0,r9,r3
770 subf r5,r2,r0
771 cmplw r5,r9
772 bge++ 0f
773
774 lwz r6,TIMER_HIGH(r11)
775 addi r6,r6,1
776 stw r6,TIMER_HIGHCHK(r11)
777 eieio
778 stw r5,TIMER_LOW(r11)
779 eieio
780 stw r6,TIMER_HIGH(r11)
781 b 1f
782
783 0: stw r5,TIMER_LOW(r11)
784
785 1: stw r4,CURRENT_TIMER(r10)
786 stw r3,TIMER_TSTAMP(r4)
787 blr
788
789 /* Set machine into idle power-saving mode.
790 *
791 * void machine_idle(void)
792 *
793 * We will use the PPC NAP or DOZE for this.
794 * This call always returns. Must be called with spllo (i.e., interruptions
795 * enabled).
796 *
797 */
798
799 ; Force a line boundry here
800 .align 5
801 .globl EXT(machine_idle)
802
803 LEXT(machine_idle)
804
805 mfsprg r12,1 ; Get the current activation
806 lwz r12,ACT_PER_PROC(r12) ; Get the per_proc block
807 lhz r10,PP_CPU_FLAGS(r12) ; Get the flags
808 lwz r11,PP_INTS_ENABLED(r12) ; Get interrupt enabled state
809 andi. r10,r10,SignalReady ; Are Signal ready?
810 cmpwi cr1,r11,0 ; Are interrupt disabled?
811 cror cr0_eq, cr1_eq, cr0_eq ; Interrupt disabled or Signal not ready?
812 mfmsr r3 ; Save the MSR
813
814 beq-- nonap ; Yes, return after re-enabling interrupts
815 lis r0,hi16(MASK(MSR_VEC)) ; Get the vector flag
816 ori r0,r0,lo16(MASK(MSR_FP)) ; Add the FP flag
817 andc r3,r3,r0 ; Clear VEC and FP
818 ori r0,r0,lo16(MASK(MSR_EE)) ; Drop EE also
819 andc r5,r3,r0 ; Clear VEC, FP, DR, and EE
820
821 mtmsr r5 ; Hold up interruptions for now
822 isync ; May have messed with fp/vec
823 mfsprg r11,2 ; Get CPU specific features
824 mfspr r6,hid0 ; Get the current power-saving mode
825 mtcrf 0xC7,r11 ; Get the facility flags
826
827 lis r4,hi16(napm) ; Assume we can nap
828 bt pfWillNapb,yesnap ; Yeah, nap is ok...
829
830 lis r4,hi16(dozem) ; Assume we can doze
831 bt pfCanDozeb,yesnap ; We can sleep or doze one this machine...
832
833 nonap: ori r3,r3,lo16(MASK(MSR_EE)) ; Flip on EE
834
835 mtmsr r3 ; Turn interruptions back on
836 blr ; Leave...
837
838 yesnap: mftbu r9 ; Get the upper timebase
839 mftb r7 ; Get the lower timebase
840 mftbu r8 ; Get the upper one again
841 cmplw r9,r8 ; Did the top tick?
842 bne-- yesnap ; Yeah, need to get it again...
843 stw r8,napStamp(r12) ; Set high order time stamp
844 stw r7,napStamp+4(r12) ; Set low order nap stamp
845
846 rlwinm. r0,r11,0,pfAltivecb,pfAltivecb ; Do we have altivec?
847 beq-- minovec ; No...
848 dssall ; Stop the streams before we nap/doze
849 sync
850 lwz r8,napStamp(r12) ; Reload high order time stamp
851 clearpipe:
852 cmplw r8,r8
853 bne- clearpipe
854 isync
855
856 minovec: rlwinm. r7,r11,0,pfNoL2PFNapb,pfNoL2PFNapb ; Turn off L2 Prefetch before nap?
857 beq++ miL2PFok
858
859 mfspr r7,msscr0 ; Get currect MSSCR0 value
860 rlwinm r7,r7,0,0,l2pfes-1 ; Disable L2 Prefetch
861 mtspr msscr0,r7 ; Updates MSSCR0 value
862 sync
863 isync
864
865 miL2PFok:
866 rlwinm. r7,r11,0,pfSlowNapb,pfSlowNapb ; Should nap at slow speed?
867 beq minoslownap
868
869 mfspr r7,hid1 ; Get current HID1 value
870 oris r7,r7,hi16(hid1psm) ; Select PLL1
871 mtspr hid1,r7 ; Update HID1 value
872
873
874 ;
875 ; We have to open up interruptions here because book 4 says that we should
876 ; turn on only the POW bit and that we should have interrupts enabled.
877 ; The interrupt handler will detect that nap or doze is set if an interrupt
878 ; is taken and set everything up to return directly to machine_idle_ret.
879 ; So, make sure everything we need there is already set up...
880 ;
881
882 minoslownap:
883 lis r10,hi16(dozem|napm|sleepm) ; Mask of power management bits
884
885 bf-- pf64Bitb,mipNSF1 ; skip if 32-bit...
886
887 sldi r4,r4,32 ; Position the flags
888 sldi r10,r10,32 ; Position the masks
889
890 mipNSF1: li r2,lo16(MASK(MSR_DR)|MASK(MSR_IR)) ; Get the translation mask
891 andc r6,r6,r10 ; Clean up the old power bits
892 ori r7,r5,lo16(MASK(MSR_EE)) ; Flip on EE to make exit msr
893 andc r5,r5,r2 ; Clear IR and DR from current MSR
894 or r6,r6,r4 ; Set nap or doze
895 ori r5,r5,lo16(MASK(MSR_EE)) ; Flip on EE to make nap msr
896 oris r2,r5,hi16(MASK(MSR_POW)) ; Turn on power management in next MSR
897
898 sync
899 mtspr hid0,r6 ; Set up the HID for nap/doze
900 mfspr r6,hid0 ; Yes, this is silly, keep it here
901 mfspr r6,hid0 ; Yes, this is a duplicate, keep it here
902 mfspr r6,hid0 ; Yes, this is a duplicate, keep it here
903 mfspr r6,hid0 ; Yes, this is a duplicate, keep it here
904 mfspr r6,hid0 ; Yes, this is a duplicate, keep it here
905 mfspr r6,hid0 ; Yes, this is a duplicate, keep it here
906 isync ; Make sure it is set
907
908
909 ;
910 ; Turn translation off to nap
911 ;
912
913 bt pfNoMSRirb,miNoMSR ; Jump if we need to use SC for this...
914 mtmsr r5 ; Turn translation off, interrupts on
915 isync ; Wait for it
916 b miNoMSRx ; Jump back in line...
917
918 miNoMSR: mr r3,r5 ; Pass in the new MSR value
919 li r0,loadMSR ; MSR setter ultrafast
920 sc ; Do it to it like you never done before...
921
922 miNoMSRx: bf-- pf64Bitb,mipowloop ; skip if 32-bit...
923
924 li r3,0x10 ; Fancy nap threshold is 0x10 ticks
925 mftb r8 ; Get the low half of the time base
926 mfdec r4 ; Get the decrementer ticks
927 cmplw r4,r3 ; Less than threshold?
928 blt mipowloop
929
930 mtdec r3 ; Load decrementer with threshold
931 isync ; and make sure,
932 mfdec r3 ; really sure, it gets there
933
934 rlwinm r6,r2,0,MSR_EE_BIT+1,MSR_EE_BIT-1 ; Clear out the EE bit
935 sync ; Make sure queues are clear
936 mtmsr r6 ; Set MSR with EE off but POW on
937 isync ; Make sure this takes before we proceed
938
939 mftb r9 ; Get the low half of the time base
940 sub r9,r9,r8 ; Get the number of ticks spent waiting
941 sub r4,r4,r9 ; Adjust the decrementer value
942
943 mtdec r4 ; Load decrementer with the rest of the timeout
944 isync ; and make sure,
945 mfdec r4 ; really sure, it gets there
946
947 mipowloop:
948 sync ; Make sure queues are clear
949 mtmsr r2 ; Nap or doze, MSR with POW, EE set, translation off
950 isync ; Make sure this takes before we proceed
951 b mipowloop ; loop if POW does not take
952
953 ;
954 ; Note that the interrupt handler will turn off the nap/doze bits in the hid.
955 ; Also remember that the interrupt handler will force return to here whenever
956 ; the nap/doze bits are set.
957 ;
958 .globl EXT(machine_idle_ret)
959 LEXT(machine_idle_ret)
960 mtmsr r7 ; Make sure the MSR is what we want
961 isync ; In case we turn on translation
962 ;
963 ; Protect against a lost decrementer trap if the current decrementer value is negative
964 ; by more than 10 ticks, re-arm it since it is unlikely to fire at this point...
965 ; A hardware interrupt got us out of machine_idle and may also be contributing to this state
966 ;
967 mfdec r6 ; Get decrementer
968 cmpwi cr0,r6,-10 ; Compare decrementer with -10
969 bgelr++ ; Return if greater
970 li r0,1 ; Load 1
971 mtdec r0 ; Set decrementer to 1
972 blr ; Return...
973
974 /* Put machine to sleep.
975 * This call never returns. We always exit sleep via a soft reset.
976 * All external interruptions must be drained at this point and disabled.
977 *
978 * void ml_ppc_do_sleep(void)
979 *
980 * We will use the PPC SLEEP for this.
981 *
982 * There is one bit of hackery in here: we need to enable for
983 * interruptions when we go to sleep and there may be a pending
984 * decrimenter rupt. So we make the decrimenter 0x7FFFFFFF and enable for
985 * interruptions. The decrimenter rupt vector recognizes this and returns
986 * directly back here.
987 *
988 */
989
990 ; Force a line boundry here
991 .align 5
992 .globl EXT(ml_ppc_sleep)
993 LEXT(ml_ppc_sleep)
994
995 .globl EXT(ml_ppc_do_sleep)
996 LEXT(ml_ppc_do_sleep)
997
998 #if 0
999 mfmsr r5 ; Hack to spin instead of sleep
1000 rlwinm r5,r5,0,MSR_DR_BIT+1,MSR_IR_BIT-1 ; Turn off translation
1001 rlwinm r5,r5,0,MSR_EE_BIT+1,MSR_EE_BIT-1 ; Turn off interruptions
1002 mtmsr r5 ; No talking
1003 isync
1004
1005 deadsleep: addi r3,r3,1 ; Make analyzer happy
1006 addi r3,r3,1
1007 addi r3,r3,1
1008 b deadsleep ; Die the death of 1000 joys...
1009 #endif
1010
1011 mfsprg r12,1 ; Get the current activation
1012 lwz r12,ACT_PER_PROC(r12) ; Get the per_proc block
1013 mfsprg r11,2 ; Get CPU specific features
1014 eqv r10,r10,r10 ; Get all foxes
1015 mtcrf 0x04,r11 ; move pfNoMSRirb to cr5
1016 mfspr r4,hid0 ; Get the current power-saving mode
1017 mtcrf 0x02,r11 ; move pf64Bit to cr6
1018
1019 rlwinm. r5,r11,0,pfNoL2PFNapb,pfNoL2PFNapb ; Turn off L2 Prefetch before sleep?
1020 beq mpsL2PFok
1021
1022 mfspr r5,msscr0 ; Get currect MSSCR0 value
1023 rlwinm r5,r5,0,0,l2pfes-1 ; Disable L2 Prefetch
1024 mtspr msscr0,r5 ; Updates MSSCR0 value
1025 sync
1026 isync
1027
1028 mpsL2PFok:
1029 bt++ pf64Bitb,mpsPF64bit ; PM bits are shifted on 64bit systems.
1030
1031 rlwinm r4,r4,0,sleep+1,doze-1 ; Clear all possible power-saving modes (not DPM though)
1032 oris r4,r4,hi16(sleepm) ; Set sleep
1033 b mpsClearDEC
1034
1035 mpsPF64bit:
1036 lis r5, hi16(dozem|napm|sleepm) ; Clear all possible power-saving modes (not DPM though)
1037 sldi r5, r5, 32
1038 andc r4, r4, r5
1039 lis r5, hi16(napm) ; Set sleep
1040 sldi r5, r5, 32
1041 or r4, r4, r5
1042
1043 mpsClearDEC:
1044 mfmsr r5 ; Get the current MSR
1045 rlwinm r10,r10,0,1,31 ; Make 0x7FFFFFFF
1046 mtdec r10 ; Load decrimenter with 0x7FFFFFFF
1047 isync ; and make sure,
1048 mfdec r9 ; really sure, it gets there
1049
1050 li r2,1 ; Prepare for 64 bit
1051 rlwinm r5,r5,0,MSR_DR_BIT+1,MSR_IR_BIT-1 ; Turn off translation
1052 ;
1053 ; Note that we need translation off before we set the HID to sleep. Otherwise
1054 ; we will ignore any PTE misses that occur and cause an infinite loop.
1055 ;
1056 bf++ pf64Bitb,mpsCheckMSR ; check 64-bit processor
1057 rldimi r5,r2,63,MSR_SF_BIT ; set SF bit (bit 0)
1058 mtmsrd r5 ; set 64-bit mode, turn off EE, DR, and IR
1059 isync ; Toss prefetch
1060 b mpsNoMSRx
1061
1062 mpsCheckMSR:
1063 bt pfNoMSRirb,mpsNoMSR ; No MSR...
1064
1065 mtmsr r5 ; Translation off
1066 isync ; Toss prefetch
1067 b mpsNoMSRx
1068
1069 mpsNoMSR:
1070 li r0,loadMSR ; Get the MSR setter SC
1071 mr r3,r5 ; Get new MSR
1072 sc ; Set it
1073 mpsNoMSRx:
1074
1075 ori r3,r5,lo16(MASK(MSR_EE)) ; Flip on EE
1076 sync
1077 mtspr hid0,r4 ; Set up the HID to sleep
1078 mfspr r4,hid0 ; Yes, this is silly, keep it here
1079 mfspr r4,hid0 ; Yes, this is a duplicate, keep it here
1080 mfspr r4,hid0 ; Yes, this is a duplicate, keep it here
1081 mfspr r4,hid0 ; Yes, this is a duplicate, keep it here
1082 mfspr r4,hid0 ; Yes, this is a duplicate, keep it here
1083 mfspr r4,hid0 ; Yes, this is a duplicate, keep it here
1084
1085 mtmsr r3 ; Enable for interrupts to drain decrimenter
1086
1087 add r6,r4,r5 ; Just waste time
1088 add r6,r6,r4 ; A bit more
1089 add r6,r6,r5 ; A bit more
1090
1091 mtmsr r5 ; Interruptions back off
1092 isync ; Toss prefetch
1093
1094 ;
1095 ; We are here with translation off, interrupts off, all possible
1096 ; interruptions drained off, and a decrimenter that will not pop.
1097 ;
1098
1099 bl EXT(cacheInit) ; Clear out the caches. This will leave them on
1100 bl EXT(cacheDisable) ; Turn off all caches
1101
1102 mfmsr r5 ; Get the current MSR
1103 oris r5,r5,hi16(MASK(MSR_POW)) ; Turn on power management in next MSR
1104 ; Leave EE off because power goes off shortly
1105 mfsprg r12,0 ; Get the per_proc_info
1106 li r10,PP_CPU_FLAGS
1107 lhz r11,PP_CPU_FLAGS(r12) ; Get the flags
1108 ori r11,r11,SleepState ; Marked SleepState
1109 sth r11,PP_CPU_FLAGS(r12) ; Set the flags
1110 dcbf r10,r12
1111
1112 mfsprg r11,2 ; Get CPU specific features
1113 rlwinm. r0,r11,0,pf64Bitb,pf64Bitb ; Test for 64 bit processor
1114 eqv r4,r4,r4 ; Get all foxes
1115 rlwinm r4,r4,0,1,31 ; Make 0x7FFFFFFF
1116 beq slSleepNow ; skip if 32-bit...
1117 li r3, 0x4000 ; Cause decrimenter to roll over soon
1118 mtdec r3 ; Load decrimenter with 0x00004000
1119 isync ; and make sure,
1120 mfdec r3 ; really sure, it gets there
1121
1122 slSleepNow:
1123 sync ; Sync it all up
1124 mtmsr r5 ; Do sleep with interruptions enabled
1125 isync ; Take a pill
1126 mtdec r4 ; Load decrimenter with 0x7FFFFFFF
1127 isync ; and make sure,
1128 mfdec r3 ; really sure, it gets there
1129 b slSleepNow ; Go back to sleep if we wake up...
1130
1131
1132
1133 /* Initialize all caches including the TLBs
1134 *
1135 * void cacheInit(void)
1136 *
1137 * This is used to force the caches to an initial clean state. First, we
1138 * check if the cache is on, if so, we need to flush the contents to memory.
1139 * Then we invalidate the L1. Next, we configure and invalidate the L2 etc.
1140 * Finally we turn on all of the caches
1141 *
1142 * Note that if translation is not disabled when this is called, the TLB will not
1143 * be completely clear after return.
1144 *
1145 */
1146
1147 ; Force a line boundry here
1148 .align 5
1149 .globl EXT(cacheInit)
1150
1151 LEXT(cacheInit)
1152
1153 mfsprg r12,0 ; Get the per_proc_info
1154 mfspr r9,hid0 ; Get the current power-saving mode
1155
1156 mfsprg r11,2 ; Get CPU specific features
1157 mfmsr r7 ; Get the current MSR
1158 rlwinm r7,r7,0,MSR_FP_BIT+1,MSR_FP_BIT-1 ; Force floating point off
1159 rlwinm r7,r7,0,MSR_VEC_BIT+1,MSR_VEC_BIT-1 ; Force vectors off
1160 rlwimi r11,r11,pfLClckb+1,31,31 ; Move pfLClck to another position (to keep from using non-volatile CRs)
1161 rlwinm r5,r7,0,MSR_DR_BIT+1,MSR_IR_BIT-1 ; Turn off translation
1162 rlwinm r5,r5,0,MSR_EE_BIT+1,MSR_EE_BIT-1 ; Turn off interruptions
1163 mtcrf 0x87,r11 ; Get the feature flags
1164 lis r10,hi16(dozem|napm|sleepm|dpmm) ; Mask of power management bits
1165 bf-- pf64Bitb,cIniNSF1 ; Skip if 32-bit...
1166
1167 sldi r10,r10,32 ; Position the masks
1168
1169 cIniNSF1: andc r4,r9,r10 ; Clean up the old power bits
1170 mtspr hid0,r4 ; Set up the HID
1171 mfspr r4,hid0 ; Yes, this is silly, keep it here
1172 mfspr r4,hid0 ; Yes, this is a duplicate, keep it here
1173 mfspr r4,hid0 ; Yes, this is a duplicate, keep it here
1174 mfspr r4,hid0 ; Yes, this is a duplicate, keep it here
1175 mfspr r4,hid0 ; Yes, this is a duplicate, keep it here
1176 mfspr r4,hid0 ; Yes, this is a duplicate, keep it here
1177
1178 bt pfNoMSRirb,ciNoMSR ; No MSR...
1179
1180 mtmsr r5 ; Translation and all off
1181 isync ; Toss prefetch
1182 b ciNoMSRx
1183
1184 ciNoMSR:
1185 li r0,loadMSR ; Get the MSR setter SC
1186 mr r3,r5 ; Get new MSR
1187 sc ; Set it
1188 ciNoMSRx:
1189
1190 bf pfAltivecb,cinoDSS ; No Altivec here...
1191
1192 dssall ; Stop streams
1193 sync
1194
1195 cinoDSS: li r5,tlbieLock ; Get the TLBIE lock
1196 li r0,128 ; Get number of TLB entries
1197
1198 li r6,0 ; Start at 0
1199 bf-- pf64Bitb,citlbhang ; Skip if 32-bit...
1200 li r0,1024 ; Get the number of TLB entries
1201
1202 citlbhang: lwarx r2,0,r5 ; Get the TLBIE lock
1203 mr. r2,r2 ; Is it locked?
1204 bne- citlbhang ; It is locked, go wait...
1205 stwcx. r0,0,r5 ; Try to get it
1206 bne- citlbhang ; We was beat...
1207
1208 mtctr r0 ; Set the CTR
1209
1210 cipurgeTLB: tlbie r6 ; Purge this entry
1211 addi r6,r6,4096 ; Next page
1212 bdnz cipurgeTLB ; Do them all...
1213
1214 mtcrf 0x80,r11 ; Set SMP capability
1215 sync ; Make sure all TLB purges are done
1216 eieio ; Order, order in the court
1217
1218 bf pfSMPcapb,cinoSMP ; SMP incapable...
1219
1220 tlbsync ; Sync all TLBs
1221 sync
1222 isync
1223
1224 bf-- pf64Bitb,cinoSMP ; Skip if 32-bit...
1225 ptesync ; Wait for quiet again
1226 sync
1227
1228 cinoSMP: stw r2,tlbieLock(0) ; Unlock TLBIE lock
1229
1230 bt++ pf64Bitb,cin64 ; Skip if 64-bit...
1231
1232 rlwinm. r0,r9,0,ice,dce ; Were either of the level 1s on?
1233 beq- cinoL1 ; No, no need to flush...
1234
1235 rlwinm. r0,r11,0,pfL1fab,pfL1fab ; do we have L1 flush assist?
1236 beq ciswdl1 ; If no hw flush assist, go do by software...
1237
1238 mfspr r8,msscr0 ; Get the memory system control register
1239 oris r8,r8,hi16(dl1hwfm) ; Turn on the hardware flush request
1240
1241 mtspr msscr0,r8 ; Start the flush operation
1242
1243 ciwdl1f: mfspr r8,msscr0 ; Get the control register again
1244
1245 rlwinm. r8,r8,0,dl1hwf,dl1hwf ; Has the flush request been reset yet?
1246 bne ciwdl1f ; No, flush is still in progress...
1247 b ciinvdl1 ; Go invalidate l1...
1248
1249 ;
1250 ; We need to either make this very complicated or to use ROM for
1251 ; the flush. The problem is that if during the following sequence a
1252 ; snoop occurs that invalidates one of the lines in the cache, the
1253 ; PLRU sequence will be altered making it possible to miss lines
1254 ; during the flush. So, we either need to dedicate an area of RAM
1255 ; to each processor, lock use of a RAM area, or use ROM. ROM is
1256 ; by far the easiest. Note that this is not an issue for machines
1257 ; that have harware flush assists.
1258 ;
1259
1260 ciswdl1: lwz r0,pfl1dSize(r12) ; Get the level 1 cache size
1261
1262 bf 31,cisnlck ; Skip if pfLClck not set...
1263
1264 mfspr r4,msscr0 ; ?
1265 rlwinm r6,r4,0,0,l2pfes-1 ; ?
1266 mtspr msscr0,r6 ; Set it
1267 sync
1268 isync
1269
1270 mfspr r8,ldstcr ; Save the LDSTCR
1271 li r2,1 ; Get a mask of 0x01
1272 lis r3,0xFFF0 ; Point to ROM
1273 rlwinm r11,r0,29,3,31 ; Get the amount of memory to handle all indexes
1274
1275 li r6,0 ; Start here
1276
1277 cisiniflsh: dcbf r6,r3 ; Flush each line of the range we use
1278 addi r6,r6,32 ; Bump to the next
1279 cmplw r6,r0 ; Have we reached the end?
1280 blt+ cisiniflsh ; Nope, continue initial flush...
1281
1282 sync ; Make sure it is done
1283
1284 addi r11,r11,-1 ; Get mask for index wrap
1285 li r6,0 ; Get starting offset
1286
1287 cislckit: not r5,r2 ; Lock all but 1 way
1288 rlwimi r5,r8,0,0,23 ; Build LDSTCR
1289 mtspr ldstcr,r5 ; Lock a way
1290 sync ; Clear out memory accesses
1291 isync ; Wait for all
1292
1293
1294 cistouch: lwzx r10,r3,r6 ; Pick up some trash
1295 addi r6,r6,32 ; Go to the next index
1296 and. r0,r6,r11 ; See if we are about to do next index
1297 bne+ cistouch ; Nope, do more...
1298
1299 sync ; Make sure it is all done
1300 isync
1301
1302 sub r6,r6,r11 ; Back up to start + 1
1303 addi r6,r6,-1 ; Get it right
1304
1305 cisflush: dcbf r3,r6 ; Flush everything out
1306 addi r6,r6,32 ; Go to the next index
1307 and. r0,r6,r11 ; See if we are about to do next index
1308 bne+ cisflush ; Nope, do more...
1309
1310 sync ; Make sure it is all done
1311 isync
1312
1313
1314 rlwinm. r2,r2,1,24,31 ; Shift to next way
1315 bne+ cislckit ; Do this for all ways...
1316
1317 mtspr ldstcr,r8 ; Slam back to original
1318 sync
1319 isync
1320
1321 mtspr msscr0,r4 ; ?
1322 sync
1323 isync
1324
1325 b cinoL1 ; Go on to level 2...
1326
1327
1328 cisnlck: rlwinm r2,r0,0,1,30 ; Double cache size
1329 add r0,r0,r2 ; Get 3 times cache size
1330 rlwinm r0,r0,26,6,31 ; Get 3/2 number of cache lines
1331 lis r3,0xFFF0 ; Dead recon ROM address for now
1332 mtctr r0 ; Number of lines to flush
1333
1334 ciswfldl1a: lwz r2,0(r3) ; Flush anything else
1335 addi r3,r3,32 ; Next line
1336 bdnz ciswfldl1a ; Flush the lot...
1337
1338 ciinvdl1: sync ; Make sure all flushes have been committed
1339
1340 mfspr r8,hid0 ; Get the HID0 bits
1341 rlwinm r8,r8,0,dce+1,ice-1 ; Clear cache enables
1342 mtspr hid0,r8 ; and turn off L1 cache
1343 sync ; Make sure all is done
1344 isync
1345
1346 ori r8,r8,lo16(icem|dcem|icfim|dcfim) ; Set the HID0 bits for enable, and invalidate
1347 sync
1348 isync
1349
1350 mtspr hid0,r8 ; Start the invalidate and turn on cache
1351 rlwinm r8,r8,0,dcfi+1,icfi-1 ; Turn off the invalidate bits
1352 mtspr hid0,r8 ; Turn off the invalidate (needed for some older machines)
1353 sync
1354
1355
1356 cinoL1:
1357 ;
1358 ; Flush and disable the level 2
1359 ;
1360 mfsprg r10,2 ; need to check 2 features we did not put in CR
1361 rlwinm. r0,r10,0,pfL2b,pfL2b ; do we have L2?
1362 beq cinol2 ; No level 2 cache to flush
1363
1364 mfspr r8,l2cr ; Get the L2CR
1365 lwz r3,pfl2cr(r12) ; Get the L2CR value
1366 rlwinm. r0,r8,0,l2e,l2e ; Was the L2 enabled?
1367 bne ciflushl2 ; Yes, force flush
1368 cmplwi r8, 0 ; Was the L2 all the way off?
1369 beq ciinvdl2 ; Yes, force invalidate
1370 lis r0,hi16(l2sizm|l2clkm|l2ramm|l2ohm) ; Get confiuration bits
1371 xor r2,r8,r3 ; Get changing bits?
1372 ori r0,r0,lo16(l2slm|l2dfm|l2bypm) ; More config bits
1373 and. r0,r0,r2 ; Did any change?
1374 bne- ciinvdl2 ; Yes, just invalidate and get PLL synced...
1375
1376 ciflushl2:
1377 rlwinm. r0,r10,0,pfL2fab,pfL2fab ; hardware-assisted L2 flush?
1378 beq ciswfl2 ; Flush not in hardware...
1379
1380 mr r10,r8 ; Take a copy now
1381
1382 bf 31,cinol2lck ; Skip if pfLClck not set...
1383
1384 oris r10,r10,hi16(l2ionlym|l2donlym) ; Set both instruction- and data-only
1385 sync
1386 mtspr l2cr,r10 ; Lock out the cache
1387 sync
1388 isync
1389
1390 cinol2lck: ori r10,r10,lo16(l2hwfm) ; Request flush
1391 sync ; Make sure everything is done
1392
1393 mtspr l2cr,r10 ; Request flush
1394
1395 cihwfl2: mfspr r10,l2cr ; Get back the L2CR
1396 rlwinm. r10,r10,0,l2hwf,l2hwf ; Is the flush over?
1397 bne+ cihwfl2 ; Nope, keep going...
1398 b ciinvdl2 ; Flush done, go invalidate L2...
1399
1400 ciswfl2:
1401 lwz r0,pfl2Size(r12) ; Get the L2 size
1402 oris r2,r8,hi16(l2dom) ; Set L2 to data only mode
1403
1404 b ciswfl2doa ; Branch to next line...
1405
1406 .align 5
1407 ciswfl2doc:
1408 mtspr l2cr,r2 ; Disable L2
1409 sync
1410 isync
1411 b ciswfl2dod ; It is off, go invalidate it...
1412
1413 ciswfl2doa:
1414 b ciswfl2dob ; Branch to next...
1415
1416 ciswfl2dob:
1417 sync ; Finish memory stuff
1418 isync ; Stop speculation
1419 b ciswfl2doc ; Jump back up and turn on data only...
1420 ciswfl2dod:
1421 rlwinm r0,r0,27,5,31 ; Get the number of lines
1422 lis r10,0xFFF0 ; Dead recon ROM for now
1423 mtctr r0 ; Set the number of lines
1424
1425 ciswfldl2a: lwz r0,0(r10) ; Load something to flush something
1426 addi r10,r10,32 ; Next line
1427 bdnz ciswfldl2a ; Do the lot...
1428
1429 ciinvdl2: rlwinm r8,r3,0,l2e+1,31 ; Clear the enable bit
1430 b cinla ; Branch to next line...
1431
1432 .align 5
1433 cinlc: mtspr l2cr,r8 ; Disable L2
1434 sync
1435 isync
1436 b ciinvl2 ; It is off, go invalidate it...
1437
1438 cinla: b cinlb ; Branch to next...
1439
1440 cinlb: sync ; Finish memory stuff
1441 isync ; Stop speculation
1442 b cinlc ; Jump back up and turn off cache...
1443
1444 ciinvl2: sync
1445 isync
1446
1447 cmplwi r3, 0 ; Should the L2 be all the way off?
1448 beq cinol2 ; Yes, done with L2
1449
1450 oris r2,r8,hi16(l2im) ; Get the invalidate flag set
1451
1452 mtspr l2cr,r2 ; Start the invalidate
1453 sync
1454 isync
1455 ciinvdl2a: mfspr r2,l2cr ; Get the L2CR
1456 mfsprg r0,2 ; need to check a feature in "non-volatile" set
1457 rlwinm. r0,r0,0,pfL2ib,pfL2ib ; flush in HW?
1458 beq ciinvdl2b ; Flush not in hardware...
1459 rlwinm. r2,r2,0,l2i,l2i ; Is the invalidate still going?
1460 bne+ ciinvdl2a ; Assume so, this will take a looong time...
1461 sync
1462 b cinol2 ; No level 2 cache to flush
1463 ciinvdl2b:
1464 rlwinm. r2,r2,0,l2ip,l2ip ; Is the invalidate still going?
1465 bne+ ciinvdl2a ; Assume so, this will take a looong time...
1466 sync
1467 mtspr l2cr,r8 ; Turn off the invalidate request
1468
1469 cinol2:
1470
1471 ;
1472 ; Flush and enable the level 3
1473 ;
1474 bf pfL3b,cinol3 ; No level 3 cache to flush
1475
1476 mfspr r8,l3cr ; Get the L3CR
1477 lwz r3,pfl3cr(r12) ; Get the L3CR value
1478 rlwinm. r0,r8,0,l3e,l3e ; Was the L3 enabled?
1479 bne ciflushl3 ; Yes, force flush
1480 cmplwi r8, 0 ; Was the L3 all the way off?
1481 beq ciinvdl3 ; Yes, force invalidate
1482 lis r0,hi16(l3pem|l3sizm|l3dxm|l3clkm|l3spom|l3ckspm) ; Get configuration bits
1483 xor r2,r8,r3 ; Get changing bits?
1484 ori r0,r0,lo16(l3pspm|l3repm|l3rtm|l3cyam|l3dmemm|l3dmsizm) ; More config bits
1485 and. r0,r0,r2 ; Did any change?
1486 bne- ciinvdl3 ; Yes, just invalidate and get PLL synced...
1487
1488 ciflushl3:
1489 sync ; 7450 book says do this even though not needed
1490 mr r10,r8 ; Take a copy now
1491
1492 bf 31,cinol3lck ; Skip if pfL23lck not set...
1493
1494 oris r10,r10,hi16(l3iom) ; Set instruction-only
1495 ori r10,r10,lo16(l3donlym) ; Set data-only
1496 sync
1497 mtspr l3cr,r10 ; Lock out the cache
1498 sync
1499 isync
1500
1501 cinol3lck: ori r10,r10,lo16(l3hwfm) ; Request flush
1502 sync ; Make sure everything is done
1503
1504 mtspr l3cr,r10 ; Request flush
1505
1506 cihwfl3: mfspr r10,l3cr ; Get back the L3CR
1507 rlwinm. r10,r10,0,l3hwf,l3hwf ; Is the flush over?
1508 bne+ cihwfl3 ; Nope, keep going...
1509
1510 ciinvdl3: rlwinm r8,r3,0,l3e+1,31 ; Clear the enable bit
1511 sync ; Make sure of life, liberty, and justice
1512 mtspr l3cr,r8 ; Disable L3
1513 sync
1514
1515 cmplwi r3, 0 ; Should the L3 be all the way off?
1516 beq cinol3 ; Yes, done with L3
1517
1518 ori r8,r8,lo16(l3im) ; Get the invalidate flag set
1519
1520 mtspr l3cr,r8 ; Start the invalidate
1521
1522 ciinvdl3b: mfspr r8,l3cr ; Get the L3CR
1523 rlwinm. r8,r8,0,l3i,l3i ; Is the invalidate still going?
1524 bne+ ciinvdl3b ; Assume so...
1525 sync
1526
1527 lwz r10, pfBootConfig(r12) ; ?
1528 rlwinm. r10, r10, 24, 28, 31 ; ?
1529 beq ciinvdl3nopdet ; ?
1530
1531 mfspr r8,l3pdet ; ?
1532 srw r2, r8, r10 ; ?
1533 rlwimi r2, r8, 0, 24, 31 ; ?
1534 subfic r10, r10, 32 ; ?
1535 li r8, -1 ; ?
1536 ori r2, r2, 0x0080 ; ?
1537 slw r8, r8, r10 ; ?
1538 or r8, r2, r8 ; ?
1539 mtspr l3pdet, r8 ; ?
1540 isync
1541
1542 ciinvdl3nopdet:
1543 mfspr r8,l3cr ; Get the L3CR
1544 rlwinm r8,r8,0,l3clken+1,l3clken-1 ; Clear the clock enable bit
1545 mtspr l3cr,r8 ; Disable the clock
1546
1547 li r2,128 ; ?
1548 ciinvdl3c: addi r2,r2,-1 ; ?
1549 cmplwi r2,0 ; ?
1550 bne+ ciinvdl3c
1551
1552 mfspr r10,msssr0 ; ?
1553 rlwinm r10,r10,0,vgL3TAG+1,vgL3TAG-1 ; ?
1554 mtspr msssr0,r10 ; ?
1555 sync
1556
1557 mtspr l3cr,r3 ; Enable it as desired
1558 sync
1559 cinol3:
1560 mfsprg r0,2 ; need to check a feature in "non-volatile" set
1561 rlwinm. r0,r0,0,pfL2b,pfL2b ; is there an L2 cache?
1562 beq cinol2a ; No level 2 cache to enable
1563
1564 lwz r3,pfl2cr(r12) ; Get the L2CR value
1565 cmplwi r3, 0 ; Should the L2 be all the way off?
1566 beq cinol2a : Yes, done with L2
1567 mtspr l2cr,r3 ; Enable it as desired
1568 sync
1569
1570 ;
1571 ; Invalidate and turn on L1s
1572 ;
1573
1574 cinol2a:
1575 bt 31,cinoexit ; Skip if pfLClck set...
1576
1577 rlwinm r8,r9,0,dce+1,ice-1 ; Clear the I- and D- cache enables
1578 mtspr hid0,r8 ; Turn off dem caches
1579 sync
1580
1581 ori r8,r9,lo16(icem|dcem|icfim|dcfim) ; Set the HID0 bits for enable, and invalidate
1582 rlwinm r9,r8,0,dcfi+1,icfi-1 ; Turn off the invalidate bits
1583 sync
1584 isync
1585
1586 mtspr hid0,r8 ; Start the invalidate and turn on L1 cache
1587
1588 cinoexit: mtspr hid0,r9 ; Turn off the invalidate (needed for some older machines) and restore entry conditions
1589 sync
1590 mtmsr r7 ; Restore MSR to entry
1591 isync
1592 blr ; Return...
1593
1594
1595 ;
1596 ; Handle 64-bit architecture
1597 ; This processor can not run without caches, so we just push everything out
1598 ; and flush. It will be relativily clean afterwards
1599 ;
1600
1601 .align 5
1602
1603 cin64:
1604 mfspr r10,hid1 ; Save hid1
1605 mfspr r4,hid4 ; Save hid4
1606 mr r12,r10 ; Really save hid1
1607 mr r11,r4 ; Get a working copy of hid4
1608
1609 li r0,0 ; Get a 0
1610 eqv r2,r2,r2 ; Get all foxes
1611
1612 rldimi r10,r0,55,7 ; Clear I$ prefetch bits (7:8)
1613
1614 isync
1615 mtspr hid1,r10 ; Stick it
1616 mtspr hid1,r10 ; Stick it again
1617 isync
1618
1619 rldimi r11,r2,38,25 ; Disable D$ prefetch (25:25)
1620
1621 sync
1622 mtspr hid4,r11 ; Stick it
1623 isync
1624
1625 li r3,8 ; Set bit 28+32
1626 sldi r3,r3,32 ; Make it bit 28
1627 or r3,r3,r11 ; Turn on the flash invalidate L1D$
1628
1629 oris r5,r11,0x0600 ; Set disable L1D$ bits
1630 sync
1631 mtspr hid4,r3 ; Invalidate
1632 isync
1633
1634 mtspr hid4,r5 ; Un-invalidate and disable L1D$
1635 isync
1636
1637 lis r8,GUSModeReg ; Get the GUS mode ring address
1638 mfsprg r0,2 ; Get the feature flags
1639 ori r8,r8,0x8000 ; Set to read data
1640 rlwinm. r0,r0,pfSCOMFixUpb+1,31,31 ; Set shift if we need a fix me up
1641
1642 sync
1643
1644 mtspr scomc,r8 ; Request the GUS mode
1645 mfspr r11,scomd ; Get the GUS mode
1646 mfspr r8,scomc ; Get back the status (we just ignore it)
1647 sync
1648 isync
1649
1650 sld r11,r11,r0 ; Fix up if needed
1651
1652 ori r6,r11,lo16(GUSMdmapen) ; Set the bit that means direct L2 cache address
1653 lis r8,GUSModeReg ; Get GUS mode register address
1654
1655 sync
1656
1657 mtspr scomd,r6 ; Set that we want direct L2 mode
1658 mtspr scomc,r8 ; Tell GUS we want direct L2 mode
1659 mfspr r3,scomc ; Get back the status
1660 sync
1661 isync
1662
1663 li r3,0 ; Clear start point
1664
1665 cflushlp: lis r6,0x0040 ; Pick 4MB line as our target
1666 or r6,r6,r3 ; Put in the line offset
1667 lwz r5,0(r6) ; Load a line
1668 addis r6,r6,8 ; Roll bit 42:44
1669 lwz r5,0(r6) ; Load a line
1670 addis r6,r6,8 ; Roll bit 42:44
1671 lwz r5,0(r6) ; Load a line
1672 addis r6,r6,8 ; Roll bit 42:44
1673 lwz r5,0(r6) ; Load a line
1674 addis r6,r6,8 ; Roll bit 42:44
1675 lwz r5,0(r6) ; Load a line
1676 addis r6,r6,8 ; Roll bit 42:44
1677 lwz r5,0(r6) ; Load a line
1678 addis r6,r6,8 ; Roll bit 42:44
1679 lwz r5,0(r6) ; Load a line
1680 addis r6,r6,8 ; Roll bit 42:44
1681 lwz r5,0(r6) ; Load a line
1682
1683 addi r3,r3,128 ; Next line
1684 andis. r5,r3,8 ; Have we done enough?
1685 beq++ cflushlp ; Not yet...
1686
1687 sync
1688
1689 lis r6,0x0040 ; Pick 4MB line as our target
1690
1691 cflushx: dcbf 0,r6 ; Flush line and invalidate
1692 addi r6,r6,128 ; Next line
1693 andis. r5,r6,0x0080 ; Have we done enough?
1694 beq++ cflushx ; Keep on flushing...
1695
1696 mr r3,r10 ; Copy current hid1
1697 rldimi r3,r2,54,9 ; Set force icbi match mode
1698
1699 li r6,0 ; Set start if ICBI range
1700 isync
1701 mtspr hid1,r3 ; Stick it
1702 mtspr hid1,r3 ; Stick it again
1703 isync
1704
1705 cflicbi: icbi 0,r6 ; Kill I$
1706 addi r6,r6,128 ; Next line
1707 andis. r5,r6,1 ; Have we done them all?
1708 beq++ cflicbi ; Not yet...
1709
1710 lis r8,GUSModeReg ; Get GUS mode register address
1711
1712 sync
1713
1714 mtspr scomd,r11 ; Set that we do not want direct mode
1715 mtspr scomc,r8 ; Tell GUS we do not want direct mode
1716 mfspr r3,scomc ; Get back the status
1717 sync
1718 isync
1719
1720 isync
1721 mtspr hid0,r9 ; Restore entry hid0
1722 mfspr r9,hid0 ; Yes, this is silly, keep it here
1723 mfspr r9,hid0 ; Yes, this is a duplicate, keep it here
1724 mfspr r9,hid0 ; Yes, this is a duplicate, keep it here
1725 mfspr r9,hid0 ; Yes, this is a duplicate, keep it here
1726 mfspr r9,hid0 ; Yes, this is a duplicate, keep it here
1727 mfspr r9,hid0 ; Yes, this is a duplicate, keep it here
1728 isync
1729
1730 isync
1731 mtspr hid1,r12 ; Restore entry hid1
1732 mtspr hid1,r12 ; Stick it again
1733 isync
1734
1735 sync
1736 mtspr hid4,r4 ; Restore entry hid4
1737 isync
1738
1739 sync
1740 mtmsr r7 ; Restore MSR to entry
1741 isync
1742 blr ; Return...
1743
1744
1745
1746 /* Disables all caches
1747 *
1748 * void cacheDisable(void)
1749 *
1750 * Turns off all caches on the processor. They are not flushed.
1751 *
1752 */
1753
1754 ; Force a line boundry here
1755 .align 5
1756 .globl EXT(cacheDisable)
1757
1758 LEXT(cacheDisable)
1759
1760 mfsprg r11,2 ; Get CPU specific features
1761 mtcrf 0x83,r11 ; Set feature flags
1762
1763 bf pfAltivecb,cdNoAlt ; No vectors...
1764
1765 dssall ; Stop streams
1766
1767 cdNoAlt: sync
1768
1769 btlr pf64Bitb ; No way to disable a 64-bit machine...
1770
1771 mfspr r5,hid0 ; Get the hid
1772 rlwinm r5,r5,0,dce+1,ice-1 ; Clear the I- and D- cache enables
1773 mtspr hid0,r5 ; Turn off dem caches
1774 sync
1775
1776 rlwinm. r0,r11,0,pfL2b,pfL2b ; is there an L2?
1777 beq cdNoL2 ; Skip if no L2...
1778
1779 mfspr r5,l2cr ; Get the L2
1780 rlwinm r5,r5,0,l2e+1,31 ; Turn off enable bit
1781
1782 b cinlaa ; Branch to next line...
1783
1784 .align 5
1785 cinlcc: mtspr l2cr,r5 ; Disable L2
1786 sync
1787 isync
1788 b cdNoL2 ; It is off, we are done...
1789
1790 cinlaa: b cinlbb ; Branch to next...
1791
1792 cinlbb: sync ; Finish memory stuff
1793 isync ; Stop speculation
1794 b cinlcc ; Jump back up and turn off cache...
1795
1796 cdNoL2:
1797
1798 bf pfL3b,cdNoL3 ; Skip down if no L3...
1799
1800 mfspr r5,l3cr ; Get the L3
1801 rlwinm r5,r5,0,l3e+1,31 ; Turn off enable bit
1802 rlwinm r5,r5,0,l3clken+1,l3clken-1 ; Turn off cache enable bit
1803 mtspr l3cr,r5 ; Disable the caches
1804 sync
1805
1806 cdNoL3:
1807 blr ; Leave...
1808
1809
1810 /* Initialize processor thermal monitoring
1811 * void ml_thrm_init(void)
1812 *
1813 * Obsolete, deprecated and will be removed.
1814 */
1815
1816 ; Force a line boundry here
1817 .align 5
1818 .globl EXT(ml_thrm_init)
1819
1820 LEXT(ml_thrm_init)
1821 blr
1822
1823 /* Set thermal monitor bounds
1824 * void ml_thrm_set(unsigned int low, unsigned int high)
1825 *
1826 * Obsolete, deprecated and will be removed.
1827 */
1828
1829 ; Force a line boundry here
1830 .align 5
1831 .globl EXT(ml_thrm_set)
1832
1833 LEXT(ml_thrm_set)
1834 blr
1835
1836 /* Read processor temprature
1837 * unsigned int ml_read_temp(void)
1838 *
1839 * Obsolete, deprecated and will be removed.
1840 */
1841
1842 ; Force a line boundry here
1843 .align 5
1844 .globl EXT(ml_read_temp)
1845
1846 LEXT(ml_read_temp)
1847 li r3,-1
1848 blr
1849
1850 /* Throttle processor speed up or down
1851 * unsigned int ml_throttle(unsigned int step)
1852 *
1853 * Returns old speed and sets new. Both step and return are values from 0 to
1854 * 255 that define number of throttle steps, 0 being off and "ictcfim" is max * 2.
1855 *
1856 * Obsolete, deprecated and will be removed.
1857 */
1858
1859 ; Force a line boundry here
1860 .align 5
1861 .globl EXT(ml_throttle)
1862
1863 LEXT(ml_throttle)
1864 li r3,0
1865 blr
1866
1867 /*
1868 ** ml_get_timebase()
1869 **
1870 ** Entry - R3 contains pointer to 64 bit structure.
1871 **
1872 ** Exit - 64 bit structure filled in.
1873 **
1874 */
1875 ; Force a line boundry here
1876 .align 5
1877 .globl EXT(ml_get_timebase)
1878
1879 LEXT(ml_get_timebase)
1880
1881 loop:
1882 mftbu r4
1883 mftb r5
1884 mftbu r6
1885 cmpw r6, r4
1886 bne- loop
1887
1888 stw r4, 0(r3)
1889 stw r5, 4(r3)
1890
1891 blr
1892
1893 /*
1894 * unsigned int cpu_number(void)
1895 *
1896 * Returns the current cpu number.
1897 */
1898
1899 .align 5
1900 .globl EXT(cpu_number)
1901
1902 LEXT(cpu_number)
1903 mfsprg r4,1 ; Get the current activation
1904 lwz r4,ACT_PER_PROC(r4) ; Get the per_proc block
1905 lhz r3,PP_CPU_NUMBER(r4) ; Get CPU number
1906 blr ; Return...
1907
1908 /*
1909 * processor_t current_processor(void)
1910 *
1911 * Returns the current processor.
1912 */
1913
1914 .align 5
1915 .globl EXT(current_processor)
1916
1917 LEXT(current_processor)
1918 mfsprg r3,1 ; Get the current activation
1919 lwz r3,ACT_PER_PROC(r3) ; Get the per_proc block
1920 addi r3,r3,PP_PROCESSOR
1921 blr
1922
1923 #if PROCESSOR_SIZE > PP_PROCESSOR_SIZE
1924 #error processor overflows per_proc
1925 #endif
1926
1927 /*
1928 * ast_t *ast_pending(void)
1929 *
1930 * Returns the address of the pending AST mask for the current processor.
1931 */
1932
1933 .align 5
1934 .globl EXT(ast_pending)
1935
1936 LEXT(ast_pending)
1937 mfsprg r3,1 ; Get the current activation
1938 lwz r3,ACT_PER_PROC(r3) ; Get the per_proc block
1939 addi r3,r3,PP_PENDING_AST
1940 blr ; Return...
1941
1942 /*
1943 * void machine_set_current_thread(thread_t)
1944 *
1945 * Set the current thread
1946 */
1947 .align 5
1948 .globl EXT(machine_set_current_thread)
1949
1950 LEXT(machine_set_current_thread)
1951
1952 mfsprg r4,1 ; Get spr1
1953 lwz r5,ACT_PER_PROC(r4) ; Get the PerProc from the previous active thread
1954 stw r5,ACT_PER_PROC(r3) ; Set the PerProc in the active thread
1955 mtsprg 1,r3 ; Set spr1 with the active thread
1956 blr ; Return...
1957
1958 /*
1959 * thread_t current_thread(void)
1960 * thread_t current_act(void)
1961 *
1962 *
1963 * Return the current thread for outside components.
1964 */
1965 .align 5
1966 .globl EXT(current_thread)
1967 .globl EXT(current_act)
1968
1969 LEXT(current_thread)
1970 LEXT(current_act)
1971
1972 mfsprg r3,1
1973 blr
1974
1975 .align 5
1976 .globl EXT(clock_get_uptime)
1977 LEXT(clock_get_uptime)
1978 1: mftbu r9
1979 mftb r0
1980 mftbu r11
1981 cmpw r11,r9
1982 bne-- 1b
1983 stw r0,4(r3)
1984 stw r9,0(r3)
1985 blr
1986
1987
1988 .align 5
1989 .globl EXT(mach_absolute_time)
1990 LEXT(mach_absolute_time)
1991 1: mftbu r3
1992 mftb r4
1993 mftbu r0
1994 cmpw r0,r3
1995 bne-- 1b
1996 blr
1997
1998 /*
1999 ** ml_sense_nmi()
2000 **
2001 */
2002 ; Force a line boundry here
2003 .align 5
2004 .globl EXT(ml_sense_nmi)
2005
2006 LEXT(ml_sense_nmi)
2007
2008 blr ; Leave...
2009
2010 /*
2011 ** ml_set_processor_speed_powertune()
2012 **
2013 */
2014 ; Force a line boundry here
2015 .align 5
2016 .globl EXT(ml_set_processor_speed_powertune)
2017
2018 LEXT(ml_set_processor_speed_powertune)
2019 mflr r0 ; Save the link register
2020 stwu r1, -(FM_ALIGN(4*4)+FM_SIZE)(r1) ; Make some space on the stack
2021 stw r28, FM_ARG0+0x00(r1) ; Save a register
2022 stw r29, FM_ARG0+0x04(r1) ; Save a register
2023 stw r30, FM_ARG0+0x08(r1) ; Save a register
2024 stw r31, FM_ARG0+0x0C(r1) ; Save a register
2025 stw r0, (FM_ALIGN(4*4)+FM_SIZE+FM_LR_SAVE)(r1) ; Save the return
2026
2027 mfsprg r31,1 ; Get the current activation
2028 lwz r31,ACT_PER_PROC(r31) ; Get the per_proc block
2029
2030 lwz r30, pfPowerModes(r31) ; Get the supported power modes
2031
2032 rlwinm r28, r3, 31-dnap, dnap, dnap ; Shift the 1 bit to the dnap+32 bit
2033 rlwinm r3, r3, 2, 29, 29 ; Shift the 1 to a 4 and mask
2034 addi r3, r3, pfPowerTune0 ; Add in the pfPowerTune0 offset
2035 lwzx r29, r31, r3 ; Load the PowerTune number 0 or 1
2036
2037 sldi r28, r28, 32 ; Shift to the top half
2038 ld r3, pfHID0(r31) ; Load the saved hid0 value
2039 and r28, r28, r3 ; Save the dnap bit
2040 lis r4, hi16(dnapm) ; Make a mask for the dnap bit
2041 sldi r4, r4, 32 ; Shift to the top half
2042 andc r3, r3, r4 ; Clear the dnap bit
2043 or r28, r28, r3 ; Insert the dnap bit as needed for later
2044
2045 sync
2046 mtspr hid0, r3 ; Turn off dnap in hid0
2047 mfspr r3, hid0 ; Yes, this is silly, keep it here
2048 mfspr r3, hid0 ; Yes, this is a duplicate, keep it here
2049 mfspr r3, hid0 ; Yes, this is a duplicate, keep it here
2050 mfspr r3, hid0 ; Yes, this is a duplicate, keep it here
2051 mfspr r3, hid0 ; Yes, this is a duplicate, keep it here
2052 mfspr r3, hid0 ; Yes, this is a duplicate, keep it here
2053 isync ; Make sure it is set
2054
2055 lis r3, hi16(PowerTuneControlReg) ; Write zero to the PCR
2056 ori r3, r3, lo16(PowerTuneControlReg)
2057 li r4, 0
2058 li r5, 0
2059 bl _ml_scom_write
2060
2061 lis r3, hi16(PowerTuneControlReg) ; Write the PowerTune value to the PCR
2062 ori r3, r3, lo16(PowerTuneControlReg)
2063 li r4, 0
2064 mr r5, r29
2065 bl _ml_scom_write
2066
2067 rlwinm r29, r29, 13-6, 6, 7 ; Move to PSR speed location and isolate the requested speed
2068 spsPowerTuneLoop:
2069 lis r3, hi16(PowerTuneStatusReg) ; Read the status from the PSR
2070 ori r3, r3, lo16(PowerTuneStatusReg)
2071 li r4, 0
2072 bl _ml_scom_read
2073 srdi r5, r5, 32
2074 rlwinm r0, r5, 0, 6, 7 ; Isolate the current speed
2075 rlwimi r0, r5, 0, 2, 2 ; Copy in the change in progress bit
2076 cmpw r0, r29 ; Compare the requested and current speeds
2077 beq spsPowerTuneDone
2078 rlwinm. r0, r5, 0, 3, 3
2079 beq spsPowerTuneLoop
2080
2081 spsPowerTuneDone:
2082 sync
2083 mtspr hid0, r28 ; Turn on dnap in hid0 if needed
2084 mfspr r28, hid0 ; Yes, this is silly, keep it here
2085 mfspr r28, hid0 ; Yes, this is a duplicate, keep it here
2086 mfspr r28, hid0 ; Yes, this is a duplicate, keep it here
2087 mfspr r28, hid0 ; Yes, this is a duplicate, keep it here
2088 mfspr r28, hid0 ; Yes, this is a duplicate, keep it here
2089 mfspr r28, hid0 ; Yes, this is a duplicate, keep it here
2090 isync ; Make sure it is set
2091
2092 lwz r0, (FM_ALIGN(4*4)+FM_SIZE+FM_LR_SAVE)(r1) ; Get the return
2093 lwz r28, FM_ARG0+0x00(r1) ; Restore a register
2094 lwz r29, FM_ARG0+0x04(r1) ; Restore a register
2095 lwz r30, FM_ARG0+0x08(r1) ; Restore a register
2096 lwz r31, FM_ARG0+0x0C(r1) ; Restore a register
2097 lwz r1, FM_BACKPTR(r1) ; Pop the stack
2098 mtlr r0
2099 blr
2100
2101 /*
2102 ** ml_set_processor_speed_dpll()
2103 **
2104 */
2105 ; Force a line boundry here
2106 .align 5
2107 .globl EXT(ml_set_processor_speed_dpll)
2108
2109 LEXT(ml_set_processor_speed_dpll)
2110 mfsprg r5,1 ; Get the current activation
2111 lwz r5,ACT_PER_PROC(r5) ; Get the per_proc block
2112
2113 cmplwi r3, 0 ; Turn off BTIC before low speed
2114 beq spsDPLL1
2115 mfspr r4, hid0 ; Get the current hid0 value
2116 rlwinm r4, r4, 0, btic+1, btic-1 ; Clear the BTIC bit
2117 sync
2118 mtspr hid0, r4 ; Set the new hid0 value
2119 isync
2120 sync
2121
2122 spsDPLL1:
2123 mfspr r4, hid1 ; Get the current PLL settings
2124 rlwimi r4, r3, 31-hid1ps, hid1ps, hid1ps ; Copy the PLL Select bit
2125 stw r4, pfHID1(r5) ; Save the new hid1 value
2126 mtspr hid1, r4 ; Select desired PLL
2127
2128 cmplwi r3, 0 ; Restore BTIC after high speed
2129 bne spsDPLL2
2130 lwz r4, pfHID0(r5) ; Load the hid0 value
2131 sync
2132 mtspr hid0, r4 ; Set the hid0 value
2133 isync
2134 sync
2135 spsDPLL2:
2136 blr
2137
2138
2139 /*
2140 ** ml_set_processor_speed_dfs()
2141 **
2142 */
2143 ; Force a line boundry here
2144 .align 5
2145 .globl EXT(ml_set_processor_speed_dfs)
2146
2147 LEXT(ml_set_processor_speed_dfs)
2148 mfsprg r5,1 ; Get the current activation
2149 lwz r5,ACT_PER_PROC(r5) ; Get the per_proc block
2150
2151 cmplwi r3, 0 ; full speed?
2152 mfspr r3, hid1 ; Get the current HID1
2153 rlwinm r3, r3, 0, hid1dfs1+1, hid1dfs0-1 ; assume full speed, clear dfs bits
2154 beq spsDFS
2155 oris r3, r3, hi16(hid1dfs1m) ; slow, set half speed dfs1 bit
2156
2157 spsDFS:
2158 stw r3, pfHID1(r5) ; Save the new hid1 value
2159 sync
2160 mtspr hid1, r3 ; Set the new HID1
2161 sync
2162 isync
2163 blr
2164
2165
2166 /*
2167 ** ml_set_processor_voltage()
2168 **
2169 */
2170 ; Force a line boundry here
2171 .align 5
2172 .globl EXT(ml_set_processor_voltage)
2173
2174 LEXT(ml_set_processor_voltage)
2175 mfsprg r5,1 ; Get the current activation
2176 lwz r5,ACT_PER_PROC(r5) ; Get the per_proc block
2177
2178 lwz r6, pfPowerModes(r5) ; Get the supported power modes
2179
2180 rlwinm. r0, r6, 0, pmDPLLVminb, pmDPLLVminb ; Is DPLL Vmin supported
2181 beq spvDone
2182
2183 mfspr r4, hid2 ; Get HID2 value
2184 rlwimi r4, r3, 31-hid2vmin, hid2vmin, hid2vmin ; Insert the voltage mode bit
2185 mtspr hid2, r4 ; Set the voltage mode
2186 sync ; Make sure it is done
2187
2188 spvDone:
2189 blr
2190
2191
2192 ;
2193 ; unsigned int ml_scom_write(unsigned int reg, unsigned long long data)
2194 ; 64-bit machines only
2195 ; returns status
2196 ;
2197
2198 .align 5
2199 .globl EXT(ml_scom_write)
2200
2201 LEXT(ml_scom_write)
2202
2203 rldicr r3,r3,8,47 ; Align register it correctly
2204 rldimi r5,r4,32,0 ; Merge the high part of data
2205 sync ; Clean up everything
2206
2207 mtspr scomd,r5 ; Stick in the data
2208 mtspr scomc,r3 ; Set write to register
2209 sync
2210 isync
2211
2212 mfspr r3,scomc ; Read back status
2213 blr ; leave....
2214
2215 ;
2216 ; unsigned int ml_read_scom(unsigned int reg, unsigned long long *data)
2217 ; 64-bit machines only
2218 ; returns status
2219 ; ASM Callers: data (r4) can be zero and the 64 bit data will be returned in r5
2220 ;
2221
2222 .align 5
2223 .globl EXT(ml_scom_read)
2224
2225 LEXT(ml_scom_read)
2226
2227 mfsprg r0,2 ; Get the feature flags
2228 rldicr r3,r3,8,47 ; Align register it correctly
2229 rlwinm r0,r0,pfSCOMFixUpb+1,31,31 ; Set shift if we need a fix me up
2230
2231 ori r3,r3,0x8000 ; Set to read data
2232 sync
2233
2234 mtspr scomc,r3 ; Request the register
2235 mfspr r5,scomd ; Get the register contents
2236 mfspr r3,scomc ; Get back the status
2237 sync
2238 isync
2239
2240 sld r5,r5,r0 ; Fix up if needed
2241
2242 cmplwi r4, 0 ; If data pointer is null, just return
2243 beqlr ; the received data in r5
2244 std r5,0(r4) ; Pass back the received data
2245 blr ; Leave...
2246
2247 ;
2248 ; Calculates the hdec to dec ratio
2249 ;
2250
2251 .align 5
2252 .globl EXT(ml_hdec_ratio)
2253
2254 LEXT(ml_hdec_ratio)
2255
2256 li r0,0 ; Clear the EE bit (and everything else for that matter)
2257 mfmsr r11 ; Get the MSR
2258 mtmsrd r0,1 ; Set the EE bit only (do not care about RI)
2259 rlwinm r11,r11,0,MSR_EE_BIT,MSR_EE_BIT ; Isolate just the EE bit
2260 mfmsr r10 ; Refresh our view of the MSR (VMX/FP may have changed)
2261 or r12,r10,r11 ; Turn on EE if on before we turned it off
2262
2263 mftb r9 ; Get time now
2264 mfspr r2,hdec ; Save hdec
2265
2266 mhrcalc: mftb r8 ; Get time now
2267 sub r8,r8,r9 ; How many ticks?
2268 cmplwi r8,10000 ; 10000 yet?
2269 blt mhrcalc ; Nope...
2270
2271 mfspr r9,hdec ; Get hdec now
2272 sub r3,r2,r9 ; How many ticks?
2273 mtmsrd r12,1 ; Flip EE on if needed
2274 blr ; Leave...
mirror of XNU