]> git.saurik.com Git - apple/xnu.git/blame - osfmk/ppc/hw_lock.s
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-1486.2.11.tar.gz
[apple/xnu.git] / osfmk / ppc / hw_lock.s
CommitLineData
1c79356b 1/*
2d21ac55 2 * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
1c79356b 3 *
2d21ac55 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
1c79356b 5 *
2d21ac55
A		 6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
8f6c56a5 14 *
2d21ac55
A		 15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5
A		 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55
A		 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
8f6c56a5 25 *
2d21ac55 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
1c79356b
A		 27 */
28
1c79356b
A		 29#include <mach_assert.h>
30#include <mach_ldebug.h>
1c79356b
A		 31#include <ppc/asm.h>
32#include <ppc/proc_reg.h>
33#include <assym.s>
34
2d21ac55
A		 35
36#include <config_dtrace.h>
37#if CONFIG_DTRACE
38 #define LOCKSTAT_LABEL(lab) \
39 .data __ASMNL__ \
40 .globl lab __ASMNL__ \
41 lab: __ASMNL__ \
42 .long 9f __ASMNL__ \
43 .text __ASMNL__ \
44 9: __ASMNL__ \
45
46 .globl _dtrace_probe, _lockstat_probemap
47#define LOCKSTAT_RECORD(id) \
48 lis r6,hi16(_lockstat_probemap) __ASMNL__ \
49 ori r6,r6,lo16(_lockstat_probemap) __ASMNL__ \
50 lwz r5,4*id(r6) __ASMNL__ \
51 mr. r5,r5 __ASMNL__ \
52 beqlr-- __ASMNL__ \
53 mr r4,r3 __ASMNL__ \
54 mr r3,r5 __ASMNL__ \
55 li r5,0 __ASMNL__ \
56 li r6,0 __ASMNL__ \
57 li r7,0 __ASMNL__ \
58 li r8,0 __ASMNL__ \
59 PROLOG(0) __ASMNL__ \
60 bl _dtrace_probe __ASMNL__ \
61 EPILOG
62#endif
63
64
65
1c79356b
A		 66#define STRING ascii
67
0b4e3aa0 68#define ILK_LOCKED 0x01
9bccf70c 69#define WAIT_FLAG 0x02
91447636
A		 70#define WANT_UPGRADE 0x04
71#define WANT_EXCL 0x08
0c530ab8 72#define PRIV_EXCL 0x8000
1c79356b 73
91447636 74#define TH_FN_OWNED 0x01
1c79356b 75
91447636
A		 76# volatile CR bits
77#define hwtimeout 20
78#define mlckmiss 21
1c79356b 79
91447636 80#define RW_DATA 0
1c79356b 81
91447636
A		 82#define PROLOG(space) \
83 stwu r1,-(FM_ALIGN(space)+FM_SIZE)(r1) __ASMNL__ \
84 mfcr r2 __ASMNL__ \
85 mflr r0 __ASMNL__ \
86 stw r3,FM_ARG0(r1) __ASMNL__ \
87 stw r11,FM_ARG0+0x04(r1) __ASMNL__ \
88 stw r2,(FM_ALIGN(space)+FM_SIZE+FM_CR_SAVE)(r1) __ASMNL__ \
89 stw r0,(FM_ALIGN(space)+FM_SIZE+FM_LR_SAVE)(r1) __ASMNL__
1c79356b 90
91447636
A		 91#define EPILOG \
92 lwz r1,0(r1) __ASMNL__ \
93 lwz r0,FM_LR_SAVE(r1) __ASMNL__ \
94 mtlr r0 __ASMNL__
1c79356b 95
1c79356b 96/*
55e303ae 97 * void hw_lock_init(hw_lock_t)
1c79356b 98 *
55e303ae 99 * Initialize a hardware lock.
1c79356b 100 */
55e303ae
A		 101 .align 5
102 .globl EXT(hw_lock_init)
1c79356b 103
55e303ae 104LEXT(hw_lock_init)
1c79356b 105
55e303ae
A		 106 li r0, 0 ; set lock to free == 0
107 stw r0, 0(r3) ; Initialize the lock
1c79356b
A		 108 blr
109
110/*
91447636 111 * unsigned int hw_lock_bit(hw_lock_t, unsigned int bit, unsigned int timeout)
1c79356b 112 *
91447636
A		 113 * Try to acquire spin-lock. The second parameter is the bit mask to test and set.
114 * multiple bits may be set. Return success (1) or failure (0).
115 * Attempt will fail after timeout ticks of the timebase.
1c79356b 116 */
1c79356b 117 .align 5
91447636 118 .globl EXT(hw_lock_bit)
1c79356b 119
91447636 120LEXT(hw_lock_bit)
1c79356b 121
91447636
A		 122 crset hwtimeout ; timeout option
123 mr r12,r4 ; Load bit mask
124 mr r4,r5 ; Load timeout value
125 b lckcomm ; Join on up...
1c79356b 126
1c79356b
A		 127/*
128 * void hw_lock_lock(hw_lock_t)
129 *
55e303ae
A		 130 * Acquire lock, spinning until it becomes available.
131 * Return with preemption disabled.
132 * We will just set a default timeout and jump into the NORMAL timeout lock.
1c79356b 133 */
1c79356b
A		 134 .align 5
135 .globl EXT(hw_lock_lock)
136
137LEXT(hw_lock_lock)
91447636
A		 138 crclr hwtimeout ; no timeout option
139 li r4,0 ; request default timeout value
140 li r12,ILK_LOCKED ; Load bit mask
141 b lckcomm ; Join on up...
142
55e303ae 143lockDisa:
91447636
A		 144 crset hwtimeout ; timeout option
145 li r4,0 ; request default timeout value
146 li r12,ILK_LOCKED ; Load bit mask
55e303ae 147 b lckcomm ; Join on up...
1c79356b
A		 148
149/*
55e303ae 150 * unsigned int hw_lock_to(hw_lock_t, unsigned int timeout)
1c79356b 151 *
55e303ae
A		 152 * Try to acquire spin-lock. Return success (1) or failure (0).
153 * Attempt will fail after timeout ticks of the timebase.
154 * We try fairly hard to get this lock. We disable for interruptions, but
155 * reenable after a "short" timeout (128 ticks, we may want to change this).
156 * After checking to see if the large timeout value (passed in) has expired and a
157 * sufficient number of cycles have gone by (to insure pending 'rupts are taken),
158 * we return either in abject failure, or disable and go back to the lock sniff routine.
159 * If the sniffer finds the lock free, it jumps right up and tries to grab it.
1c79356b
A		 160 */
161 .align 5
162 .globl EXT(hw_lock_to)
163
164LEXT(hw_lock_to)
91447636
A		 165 crset hwtimeout ; timeout option
166 li r12,ILK_LOCKED ; Load bit mask
55e303ae
A		 167lckcomm:
168 mfsprg r6,1 ; Get the current activation
169 lwz r5,ACT_PREEMPT_CNT(r6) ; Get the preemption level
170 addi r5,r5,1 ; Bring up the disable count
171 stw r5,ACT_PREEMPT_CNT(r6) ; Save it back
172 mr r5,r3 ; Get the address of the lock
173 li r8,0 ; Set r8 to zero
174
175lcktry: lwarx r6,0,r5 ; Grab the lock value
91447636
A		 176 and. r3,r6,r12 ; Is it locked?
177 or r6,r6,r12 ; Set interlock
55e303ae
A		 178 bne-- lckspin ; Yeah, wait for it to clear...
179 stwcx. r6,0,r5 ; Try to seize that there durn lock
180 bne-- lcktry ; Couldn't get it...
181 li r3,1 ; return true
91447636
A		 182 .globl EXT(hwllckPatch_isync)
183LEXT(hwllckPatch_isync)
55e303ae
A		 184 isync ; Make sure we don't use a speculativily loaded value
185 blr ; Go on home...
186
187lckspin: li r6,lgKillResv ; Get killing field
188 stwcx. r6,0,r6 ; Kill reservation
1c79356b 189
55e303ae
A		 190 mr. r4,r4 ; Test timeout value
191 bne++ lockspin0
192 lis r4,hi16(EXT(LockTimeOut)) ; Get the high part
193 ori r4,r4,lo16(EXT(LockTimeOut)) ; And the low part
194 lwz r4,0(r4) ; Get the timeout value
195lockspin0:
196 mr. r8,r8 ; Is r8 set to zero
197 bne++ lockspin1 ; If yes, first spin attempt
198 lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable
199 mfmsr r9 ; Get the MSR value
200 ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable
201 ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too
202 andc r9,r9,r0 ; Clear FP and VEC
203 andc r7,r9,r7 ; Clear EE as well
204 mtmsr r7 ; Turn off interruptions
205 isync ; May have turned off vec and fp here
206 mftb r8 ; Get timestamp on entry
207 b lcksniff
208
209lockspin1: mtmsr r7 ; Turn off interruptions
210 mftb r8 ; Get timestamp on entry
211
212lcksniff: lwz r3,0(r5) ; Get that lock in here
91447636 213 and. r3,r3,r12 ; Is it free yet?
55e303ae 214 beq++ lckretry ; Yeah, try for it again...
1c79356b 215
55e303ae
A		 216 mftb r10 ; Time stamp us now
217 sub r10,r10,r8 ; Get the elapsed time
218 cmplwi r10,128 ; Have we been spinning for 128 tb ticks?
219 blt++ lcksniff ; Not yet...
de355530 220
55e303ae 221 mtmsr r9 ; Say, any interrupts pending?
1c79356b 222
55e303ae
A		 223; The following instructions force the pipeline to be interlocked to that only one
224; instruction is issued per cycle. The insures that we stay enabled for a long enough
225; time; if it's too short, pending interruptions will not have a chance to be taken
de355530 226
55e303ae
A		 227 subi r4,r4,128 ; Back off elapsed time from timeout value
228 or r4,r4,r4 ; Do nothing here but force a single cycle delay
229 mr. r4,r4 ; See if we used the whole timeout
230 li r3,0 ; Assume a timeout return code
231 or r4,r4,r4 ; Do nothing here but force a single cycle delay
232
233 ble-- lckfail ; We failed
234 b lockspin1 ; Now that we've opened an enable window, keep trying...
235lckretry:
236 mtmsr r9 ; Restore interrupt state
237 li r8,1 ; Insure that R8 is not 0
238 b lcktry
239lckfail: ; We couldn't get the lock
91447636 240 bf hwtimeout,lckpanic
55e303ae
A		 241 li r3,0 ; Set failure return code
242 blr ; Return, head hanging low...
91447636
A		 243lckpanic:
244 mr r4,r5
245 mr r5,r3
246 lis r3,hi16(lckpanic_str) ; Get the failed lck message
247 ori r3,r3,lo16(lckpanic_str) ; Get the failed lck message
248 bl EXT(panic)
249 BREAKPOINT_TRAP ; We die here anyway
250 .data
251lckpanic_str:
252 STRINGD "timeout on attempt to acquire lock (0x%08X), value = 0x%08X\n\000"
253 .text
1c79356b
A		 254
255/*
91447636 256 * void hw_lock_unlock(hw_lock_t)
1c79356b 257 *
91447636
A		 258 * Unconditionally release lock.
259 * Release preemption level.
1c79356b 260 */
1c79356b 261 .align 5
91447636 262 .globl EXT(hw_lock_unlock)
55e303ae 263
91447636 264LEXT(hw_lock_unlock)
1c79356b 265
91447636
A		 266 .globl EXT(hwulckPatch_isync)
267LEXT(hwulckPatch_isync)
268 isync
269 .globl EXT(hwulckPatch_eieio)
270LEXT(hwulckPatch_eieio)
271 eieio
272 li r0, 0 ; set lock to free
273 stw r0, 0(r3)
1c79356b 274
91447636 275 b epStart ; Go enable preemption...
1c79356b
A		 276
277/*
55e303ae 278 * unsigned int hw_unlock_bit(hw_lock_t, unsigned int bit)
1c79356b 279 *
55e303ae
A		 280 * Release bit based spin-lock. The second parameter is the bit mask to clear.
281 * Multiple bits may be cleared.
1c79356b 282 *
1c79356b 283 */
1c79356b
A		 284 .align 5
285 .globl EXT(hw_unlock_bit)
286
287LEXT(hw_unlock_bit)
288
55e303ae
A		 289 .globl EXT(hwulckbPatch_isync)
290LEXT(hwulckbPatch_isync)
291 isync
292 .globl EXT(hwulckbPatch_eieio)
293LEXT(hwulckbPatch_eieio)
294 eieio
295ubittry: lwarx r0,0,r3 ; Grab the lock value
296 andc r0,r0,r4 ; Clear the lock bits
297 stwcx. r0,0,r3 ; Try to clear that there durn lock
298 bne- ubittry ; Try again, couldn't save it...
1c79356b 299
91447636 300 b epStart ; Go enable preemption...
1c79356b
A		 301
302/*
55e303ae 303 * unsigned int hw_lock_mbits(hw_lock_t, unsigned int bits, unsigned int value,
1c79356b
A		 304 * unsigned int newb, unsigned int timeout)
305 *
55e303ae
A		 306 * Try to acquire spin-lock. The second parameter is the bit mask to check.
307 * The third is the value of those bits and the 4th is what to set them to.
308 * Return success (1) or failure (0).
309 * Attempt will fail after timeout ticks of the timebase.
310 * We try fairly hard to get this lock. We disable for interruptions, but
311 * reenable after a "short" timeout (128 ticks, we may want to shorten this).
312 * After checking to see if the large timeout value (passed in) has expired and a
313 * sufficient number of cycles have gone by (to insure pending 'rupts are taken),
314 * we return either in abject failure, or disable and go back to the lock sniff routine.
315 * If the sniffer finds the lock free, it jumps right up and tries to grab it.
1c79356b 316 */
1c79356b 317 .align 5
1c79356b
A		 318 .globl EXT(hw_lock_mbits)
319
320LEXT(hw_lock_mbits)
321
55e303ae 322 li r10,0
1c79356b 323
55e303ae
A		 324mbittry: lwarx r12,0,r3 ; Grab the lock value
325 and r0,r12,r4 ; Clear extra bits
326 andc r12,r12,r4 ; Clear all bits in the bit mask
327 or r12,r12,r6 ; Turn on the lock bits
328 cmplw r0,r5 ; Are these the right bits?
329 bne-- mbitspin ; Nope, wait for it to clear...
330 stwcx. r12,0,r3 ; Try to seize that there durn lock
331 beq++ mbitgot ; We got it, yahoo...
332 b mbittry ; Just start up again if the store failed...
1c79356b
A		 333
334 .align 5
55e303ae
A		 335mbitspin: li r11,lgKillResv ; Point to killing field
336 stwcx. r11,0,r11 ; Kill it
1c79356b 337
55e303ae
A		 338 mr. r10,r10 ; Is r10 set to zero
339 bne++ mbitspin0 ; If yes, first spin attempt
340 lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable
341 mfmsr r9 ; Get the MSR value
342 ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable
343 ori r8,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too
344 andc r9,r9,r0 ; Clear FP and VEC
345 andc r8,r9,r8 ; Clear EE as well
346 mtmsr r8 ; Turn off interruptions
347 isync ; May have turned off vectors or float here
348 mftb r10 ; Get the low part of the time base
349 b mbitsniff
350mbitspin0:
351 mtmsr r8 ; Turn off interruptions
352 mftb r10 ; Get the low part of the time base
353mbitsniff:
354 lwz r12,0(r3) ; Get that lock in here
355 and r0,r12,r4 ; Clear extra bits
356 cmplw r0,r5 ; Are these the right bits?
357 beq++ mbitretry ; Yeah, try for it again...
1c79356b 358
55e303ae
A		 359 mftb r11 ; Time stamp us now
360 sub r11,r11,r10 ; Get the elapsed time
361 cmplwi r11,128 ; Have we been spinning for 128 tb ticks?
362 blt++ mbitsniff ; Not yet...
1c79356b 363
55e303ae 364 mtmsr r9 ; Say, any interrupts pending?
1c79356b
A		 365
366; The following instructions force the pipeline to be interlocked to that only one
367; instruction is issued per cycle. The insures that we stay enabled for a long enough
368; time. If it is too short, pending interruptions will not have a chance to be taken
369
55e303ae
A		 370 subi r7,r7,128 ; Back off elapsed time from timeout value
371 or r7,r7,r7 ; Do nothing here but force a single cycle delay
372 mr. r7,r7 ; See if we used the whole timeout
373 or r7,r7,r7 ; Do nothing here but force a single cycle delay
1c79356b 374
55e303ae
A		 375 ble-- mbitfail ; We failed
376 b mbitspin0 ; Now that we have opened an enable window, keep trying...
377mbitretry:
378 mtmsr r9 ; Enable for interruptions
379 li r10,1 ; Make sure this is non-zero
380 b mbittry
1c79356b
A		 381
382 .align 5
55e303ae
A		 383mbitgot:
384 li r3,1 ; Set good return code
91447636
A		 385 .globl EXT(hwlmlckPatch_isync)
386LEXT(hwlmlckPatch_isync)
55e303ae 387 isync ; Make sure we do not use a speculativily loaded value
1c79356b
A		 388 blr
389
55e303ae
A		 390mbitfail: li r3,0 ; Set failure return code
391 blr ; Return, head hanging low...
1c79356b
A		 392
393/*
394 * unsigned int hw_cpu_sync(unsigned int *, unsigned int timeout)
395 *
55e303ae
A		 396 * Spin until word hits 0 or timeout.
397 * Return success (1) or failure (0).
398 * Attempt will fail after timeout ticks of the timebase.
de355530 399 *
55e303ae
A		 400 * The theory is that a processor will bump a counter as it signals
401 * other processors. Then it will spin untl the counter hits 0 (or
402 * times out). The other processors, as it receives the signal will
403 * decrement the counter.
1c79356b 404 *
55e303ae
A		 405 * The other processors use interlocked update to decrement, this one
406 * does not need to interlock.
1c79356b 407 */
1c79356b 408 .align 5
1c79356b
A		 409 .globl EXT(hw_cpu_sync)
410
411LEXT(hw_cpu_sync)
412
55e303ae
A		 413 mftb r10 ; Get the low part of the time base
414 mr r9,r3 ; Save the sync word address
415 li r3,1 ; Assume we work
1c79356b 416
55e303ae
A		 417csynctry: lwz r11,0(r9) ; Grab the sync value
418 mr. r11,r11 ; Counter hit 0?
419 beqlr- ; Yeah, we are sunk...
420 mftb r12 ; Time stamp us now
1c79356b 421
55e303ae
A		 422 sub r12,r12,r10 ; Get the elapsed time
423 cmplw r4,r12 ; Have we gone too long?
424 bge+ csynctry ; Not yet...
1c79356b 425
55e303ae
A		 426 li r3,0 ; Set failure...
427 blr ; Return, head hanging low...
1c79356b
A		 428
429/*
430 * unsigned int hw_cpu_wcng(unsigned int *, unsigned int, unsigned int timeout)
431 *
55e303ae
A		 432 * Spin until word changes or timeout.
433 * Return success (1) or failure (0).
434 * Attempt will fail after timeout ticks of the timebase.
1c79356b 435 *
55e303ae
A		 436 * This is used to insure that a processor passes a certain point.
437 * An example of use is to monitor the last interrupt time in the
438 * per_proc block. This can be used to insure that the other processor
439 * has seen at least one interrupt since a specific time.
1c79356b 440 */
1c79356b 441 .align 5
0b4e3aa0 442 .globl EXT(hw_cpu_wcng)
1c79356b
A		 443
444LEXT(hw_cpu_wcng)
445
55e303ae
A		 446 mftb r10 ; Get the low part of the time base
447 mr r9,r3 ; Save the sync word address
448 li r3,1 ; Assume we work
1c79356b 449
55e303ae
A		 450wcngtry: lwz r11,0(r9) ; Grab the value
451 cmplw r11,r4 ; Do they still match?
452 bnelr- ; Nope, cool...
453 mftb r12 ; Time stamp us now
1c79356b 454
55e303ae
A		 455 sub r12,r12,r10 ; Get the elapsed time
456 cmplw r5,r12 ; Have we gone too long?
457 bge+ wcngtry ; Not yet...
1c79356b 458
55e303ae
A		 459 li r3,0 ; Set failure...
460 blr ; Return, head hanging low...
1c79356b
A		 461
462
463/*
55e303ae 464 * unsigned int hw_lock_try(hw_lock_t)
1c79356b 465 *
55e303ae
A		 466 * Try to acquire spin-lock. Return success (1) or failure (0)
467 * Returns with preemption disabled on success.
1c79356b
A		 468 *
469 */
470 .align 5
471 .globl EXT(hw_lock_try)
472
473LEXT(hw_lock_try)
474
55e303ae
A		 475 lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable
476 mfmsr r9 ; Get the MSR value
477 ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable
478 ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too
479 andc r9,r9,r0 ; Clear FP and VEC
480 andc r7,r9,r7 ; Clear EE as well
1c79356b 481
55e303ae 482 mtmsr r7 ; Disable interruptions and thus, preemption
1c79356b 483
55e303ae
A		 484 lwz r5,0(r3) ; Quick load
485 andi. r6,r5,ILK_LOCKED ; TEST...
486 bne-- .L_lock_try_failed ; No go...
1c79356b 487
55e303ae
A		 488.L_lock_try_loop:
489 lwarx r5,0,r3 ; Ld from addr of arg and reserve
1c79356b 490
55e303ae 491 andi. r6,r5,ILK_LOCKED ; TEST...
0b4e3aa0 492 ori r5,r5,ILK_LOCKED
55e303ae 493 bne-- .L_lock_try_failedX ; branch if taken. Predict free
1c79356b 494
55e303ae
A		 495 stwcx. r5,0,r3 ; And SET (if still reserved)
496 bne-- .L_lock_try_loop ; If set failed, loop back
1c79356b 497
91447636
A		 498 .globl EXT(hwltlckPatch_isync)
499LEXT(hwltlckPatch_isync)
1c79356b
A		 500 isync
501
55e303ae
A		 502 mfsprg r6,1 ; Get current activation
503 lwz r5,ACT_PREEMPT_CNT(r6) ; Get the preemption level
504 addi r5,r5,1 ; Bring up the disable count
505 stw r5,ACT_PREEMPT_CNT(r6) ; Save it back
1c79356b 506
55e303ae
A		 507 mtmsr r9 ; Allow interruptions now
508 li r3,1 ; Set that the lock was free
1c79356b
A		 509 blr
510
55e303ae
A		 511.L_lock_try_failedX:
512 li r6,lgKillResv ; Killing field
513 stwcx. r6,0,r6 ; Kill reservation
514
1c79356b 515.L_lock_try_failed:
55e303ae
A		 516 mtmsr r9 ; Allow interruptions now
517 li r3,0 ; FAILURE - lock was taken
1c79356b
A		 518 blr
519
520/*
55e303ae 521 * unsigned int hw_lock_held(hw_lock_t)
1c79356b 522 *
55e303ae
A		 523 * Return 1 if lock is held
524 * Doesn't change preemption state.
525 * N.B. Racy, of course.
1c79356b
A		 526 */
527 .align 5
528 .globl EXT(hw_lock_held)
529
530LEXT(hw_lock_held)
531
55e303ae
A		 532 isync ; Make sure we don't use a speculativily fetched lock
533 lwz r3, 0(r3) ; Get lock value
534 andi. r6,r3,ILK_LOCKED ; Extract the ILK_LOCKED bit
1c79356b
A		 535 blr
536
537/*
9bccf70c 538 * uint32_t hw_compare_and_store(uint32_t oldval, uint32_t newval, uint32_t *dest)
1c79356b 539 *
55e303ae
A		 540 * Compare old to area if equal, store new, and return true
541 * else return false and no store
542 * This is an atomic operation
1c79356b
A		 543 */
544 .align 5
545 .globl EXT(hw_compare_and_store)
b0d623f7
A		 546 .globl EXT(OSCompareAndSwap)
547 .globl EXT(OSCompareAndSwapPtr)
1c79356b
A		 548
549LEXT(hw_compare_and_store)
b0d623f7
A		 550LEXT(OSCompareAndSwap)
551LEXT(OSCompareAndSwapPtr)
1c79356b 552
55e303ae 553 mr r6,r3 ; Save the old value
1c79356b 554
55e303ae
A		 555cstry: lwarx r9,0,r5 ; Grab the area value
556 li r3,1 ; Assume it works
557 cmplw cr0,r9,r6 ; Does it match the old value?
558 bne-- csfail ; No, it must have changed...
559 stwcx. r4,0,r5 ; Try to save the new value
560 bne-- cstry ; Didn't get it, try again...
91447636
A		 561 .globl EXT(hwcsatomicPatch_isync)
562LEXT(hwcsatomicPatch_isync)
55e303ae
A		 563 isync ; Just hold up prefetch
564 blr ; Return...
1c79356b 565
55e303ae
A		 566csfail: li r3,lgKillResv ; Killing field
567 stwcx. r3,0,r3 ; Blow reservation
568
569 li r3,0 ; Set failure
570 blr ; Better luck next time...
1c79356b
A		 571
572
573/*
9bccf70c 574 * uint32_t hw_atomic_add(uint32_t *dest, uint32_t delt)
1c79356b 575 *
55e303ae
A		 576 * Atomically add the second parameter to the first.
577 * Returns the result.
1c79356b
A		 578 *
579 */
580 .align 5
581 .globl EXT(hw_atomic_add)
582
583LEXT(hw_atomic_add)
584
55e303ae 585 mr r6,r3 ; Save the area
1c79356b 586
55e303ae
A		 587addtry: lwarx r3,0,r6 ; Grab the area value
588 add r3,r3,r4 ; Add the value
589 stwcx. r3,0,r6 ; Try to save the new value
590 bne-- addtry ; Didn't get it, try again...
591 blr ; Return...
1c79356b
A		 592
593
594/*
9bccf70c 595 * uint32_t hw_atomic_sub(uint32_t *dest, uint32_t delt)
1c79356b 596 *
55e303ae
A		 597 * Atomically subtract the second parameter from the first.
598 * Returns the result.
1c79356b
A		 599 *
600 */
601 .align 5
602 .globl EXT(hw_atomic_sub)
603
604LEXT(hw_atomic_sub)
605
55e303ae 606 mr r6,r3 ; Save the area
1c79356b 607
55e303ae
A		 608subtry: lwarx r3,0,r6 ; Grab the area value
609 sub r3,r3,r4 ; Subtract the value
610 stwcx. r3,0,r6 ; Try to save the new value
611 bne-- subtry ; Didn't get it, try again...
612 blr ; Return...
1c79356b
A		 613
614
0b4e3aa0 615/*
9bccf70c 616 * uint32_t hw_atomic_or(uint32_t *dest, uint32_t mask)
0b4e3aa0 617 *
55e303ae
A		 618 * Atomically ORs the second parameter into the first.
619 * Returns the result.
0b4e3aa0
A		 620 */
621 .align 5
622 .globl EXT(hw_atomic_or)
0b4e3aa0 623LEXT(hw_atomic_or)
2d21ac55
A		 624 .globl EXT(hw_atomic_or_noret)
625LEXT(hw_atomic_or_noret)
55e303ae 626 mr r6,r3 ; Save the area
0b4e3aa0 627
55e303ae
A		 628ortry: lwarx r3,0,r6 ; Grab the area value
629 or r3,r3,r4 ; OR the value
630 stwcx. r3,0,r6 ; Try to save the new value
631 bne-- ortry ; Did not get it, try again...
632 blr ; Return...
0b4e3aa0
A		 633
634
635/*
9bccf70c 636 * uint32_t hw_atomic_and(uint32_t *dest, uint32_t mask)
0b4e3aa0 637 *
55e303ae
A		 638 * Atomically ANDs the second parameter with the first.
639 * Returns the result.
0b4e3aa0
A		 640 *
641 */
642 .align 5
643 .globl EXT(hw_atomic_and)
0b4e3aa0 644LEXT(hw_atomic_and)
2d21ac55
A		 645 .globl EXT(hw_atomic_and_noret)
646LEXT(hw_atomic_and_noret)
55e303ae 647 mr r6,r3 ; Save the area
0b4e3aa0 648
55e303ae
A		 649andtry: lwarx r3,0,r6 ; Grab the area value
650 and r3,r3,r4 ; AND the value
651 stwcx. r3,0,r6 ; Try to save the new value
652 bne-- andtry ; Did not get it, try again...
653 blr ; Return...
0b4e3aa0
A		 654
655
1c79356b
A		 656/*
657 * void hw_queue_atomic(unsigned int * anchor, unsigned int * elem, unsigned int disp)
658 *
55e303ae
A		 659 * Atomically inserts the element at the head of the list
660 * anchor is the pointer to the first element
661 * element is the pointer to the element to insert
662 * disp is the displacement into the element to the chain pointer
1c79356b
A		 663 */
664 .align 5
665 .globl EXT(hw_queue_atomic)
b0d623f7 666 .globl EXT(OSEnqueueAtomic)
1c79356b
A		 667
668LEXT(hw_queue_atomic)
b0d623f7 669LEXT(OSEnqueueAtomic)
1c79356b 670
55e303ae
A		 671 mr r7,r4 ; Make end point the same as start
672 mr r8,r5 ; Copy the displacement also
673 b hw_queue_comm ; Join common code...
1c79356b
A		 674
675/*
676 * void hw_queue_atomic_list(unsigned int * anchor, unsigned int * first, unsigned int * last, unsigned int disp)
677 *
55e303ae
A		 678 * Atomically inserts the list of elements at the head of the list
679 * anchor is the pointer to the first element
680 * first is the pointer to the first element to insert
681 * last is the pointer to the last element to insert
682 * disp is the displacement into the element to the chain pointer
1c79356b
A		 683 */
684 .align 5
685 .globl EXT(hw_queue_atomic_list)
686
687LEXT(hw_queue_atomic_list)
688
55e303ae
A		 689 mr r7,r5 ; Make end point the same as start
690 mr r8,r6 ; Copy the displacement also
1c79356b
A		 691
692hw_queue_comm:
55e303ae
A		 693 lwarx r9,0,r3 ; Pick up the anchor
694 stwx r9,r8,r7 ; Chain that to the end of the new stuff
695 eieio ; Make sure this store makes it before the anchor update
696 stwcx. r4,0,r3 ; Try to chain into the front
697 bne-- hw_queue_comm ; Didn't make it, try again...
698
699 blr ; Return...
1c79356b
A		 700
701/*
702 * unsigned int *hw_dequeue_atomic(unsigned int *anchor, unsigned int disp)
703 *
55e303ae
A		 704 * Atomically removes the first element in a list and returns it.
705 * anchor is the pointer to the first element
706 * disp is the displacement into the element to the chain pointer
707 * Returns element if found, 0 if empty.
1c79356b
A		 708 */
709 .align 5
710 .globl EXT(hw_dequeue_atomic)
b0d623f7 711 .globl EXT(OSDequeueAtomic)
1c79356b
A		 712
713LEXT(hw_dequeue_atomic)
b0d623f7 714LEXT(OSDequeueAtomic)
1c79356b 715
55e303ae 716 mr r5,r3 ; Save the anchor
1c79356b
A		 717
718hw_dequeue_comm:
55e303ae
A		 719 lwarx r3,0,r5 ; Pick up the anchor
720 mr. r3,r3 ; Is the list empty?
721 beq-- hdcFail ; Leave it list empty...
722 lwzx r9,r4,r3 ; Get the next in line
723 stwcx. r9,0,r5 ; Try to chain into the front
724 beqlr++ ; Got the thing, go away with it...
725 b hw_dequeue_comm ; Did not make it, try again...
726
727hdcFail: li r4,lgKillResv ; Killing field
728 stwcx. r4,0,r4 ; Dump reservation
729 blr ; Leave...
730
1c79356b 731
91447636
A		 732/*
733 * Routines for mutex lock debugging.
734 */
735
736/*
737 * Gets lock check flags in CR6: CR bits 24-27
738 */
739#define CHECK_SETUP(rg) \
740 lbz rg,lglcksWork(0) __ASMNL__ \
741 mtcrf 2,rg __ASMNL__
742
743
744/*
745 * Checks for expected lock type.
746 */
747#define CHECK_MUTEX_TYPE() \
748 bf MUTEX_ATTR_DEBUGb,1f __ASMNL__ \
749 bt 24+disLktypeb,1f __ASMNL__ \
750 lwz r10,MUTEX_TYPE(r3) __ASMNL__ \
751 cmpwi r10,MUTEX_TAG __ASMNL__ \
752 beq++ 1f __ASMNL__ \
753 PROLOG(0) __ASMNL__ \
754 mr r4,r11 __ASMNL__ \
755 mr r5,r10 __ASMNL__ \
756 lis r3,hi16(not_a_mutex) __ASMNL__ \
757 ori r3,r3,lo16(not_a_mutex) __ASMNL__ \
758 bl EXT(panic) __ASMNL__ \
759 BREAKPOINT_TRAP __ASMNL__ \
7601:
761
762 .data
763not_a_mutex:
764 STRINGD "mutex (0x%08X) not a mutex type (0x%08X)\n\000"
765 .text
766
767/*
768 * Verifies return to the correct thread in "unlock" situations.
769 */
770#define CHECK_THREAD(thread_offset) \
771 bf MUTEX_ATTR_DEBUGb,3f __ASMNL__ \
772 bt 24+disLkThreadb,3f __ASMNL__ \
773 mfsprg r10,1 __ASMNL__ \
774 lwz r5,MUTEX_DATA(r3) __ASMNL__ \
775 rlwinm. r9,r5,0,0,29 __ASMNL__ \
776 bne++ 1f __ASMNL__ \
777 lis r3,hi16(not_held) __ASMNL__ \
778 ori r3,r3,lo16(not_held) __ASMNL__ \
779 b 2f __ASMNL__ \
7801: __ASMNL__ \
781 cmpw r9,r10 __ASMNL__ \
782 beq++ 3f __ASMNL__ \
783 mr r5,r10 __ASMNL__ \
784 mr r6,r9 __ASMNL__ \
785 lis r3,hi16(wrong_thread) __ASMNL__ \
786 ori r3,r3,lo16(wrong_thread) __ASMNL__ \
7872: __ASMNL__ \
788 mr r4,r11 __ASMNL__ \
789 PROLOG(0) __ASMNL__ \
790 bl EXT(panic) __ASMNL__ \
791 BREAKPOINT_TRAP __ASMNL__ \
7923:
793
794 .data
795not_held:
796 STRINGD "mutex (0x%08X) not held\n\000"
797wrong_thread:
798 STRINGD "mutex (0x%08X) unlocked by non-owner(0x%08X), current owner(0x%08X)\n\000"
799 .text
800
801#define CHECK_MYLOCK() \
802 bf MUTEX_ATTR_DEBUGb,1f __ASMNL__ \
803 bt 24+disLkMyLckb,1f __ASMNL__ \
804 mfsprg r10,1 __ASMNL__ \
805 lwz r9,MUTEX_DATA(r3) __ASMNL__ \
806 rlwinm r9,r9,0,0,29 __ASMNL__ \
807 cmpw r9,r10 __ASMNL__ \
808 bne++ 1f __ASMNL__ \
809 mr r4,r11 __ASMNL__ \
810 lis r3, hi16(mylock_attempt) __ASMNL__ \
811 ori r3,r3,lo16(mylock_attempt) __ASMNL__ \
812 bl EXT(panic) __ASMNL__ \
813 BREAKPOINT_TRAP __ASMNL__ \
8141:
815
816 .data
817mylock_attempt:
818 STRINGD "mutex (0x%08X) recursive lock attempt\n\000"
819 .text
820
821#define LCK_STACK(lck, stack, lck_stack, frame_cnt, lr_save, tmp) \
822 bf 24+enaLkExtStckb,3f __ASMNL__ \
823 addi lck_stack,lck,MUTEX_STACK __ASMNL__ \
824 li frame_cnt,MUTEX_FRAMES-1 __ASMNL__ \
8251: __ASMNL__ \
826 mr tmp,stack __ASMNL__ \
827 lwz stack,0(stack) __ASMNL__ \
828 xor tmp,stack,tmp __ASMNL__ \
829 cmplwi tmp,8192 __ASMNL__ \
830 bge-- 2f __ASMNL__ \
831 lwz lr_save,FM_LR_SAVE(stack) __ASMNL__ \
832 stwu lr_save,4(lck_stack) __ASMNL__ \
833 subi frame_cnt,frame_cnt,1 __ASMNL__ \
834 cmpi cr0,frame_cnt,0 __ASMNL__ \
835 bne 1b __ASMNL__ \
836 b 3f __ASMNL__ \
8372: __ASMNL__ \
838 li tmp,0 __ASMNL__ \
839 stwu tmp,4(lck_stack) __ASMNL__ \
840 subi frame_cnt,frame_cnt,1 __ASMNL__ \
841 cmpi cr0,frame_cnt,0 __ASMNL__ \
842 bne 2b __ASMNL__ \
8433:
844
55e303ae 845 .align 5
91447636
A		 846 mr r11,r3 ; Save lock addr
847mlckeEnter:
848 lwz r0,MUTEX_ATTR(r3)
849 mtcrf 1,r0 ; Set cr7
850 CHECK_SETUP(r12)
851 CHECK_MUTEX_TYPE()
852
b0d623f7 853 bf MUTEX_ATTR_DEBUGb,L_mtx_lock_assert_wait_2
91447636
A		 854 PROLOG(0)
855 bl EXT(assert_wait_possible)
856 mr. r3,r3
b0d623f7
A		 857 bne L_mtx_lock_assert_wait_1
858 lis r3,hi16(L_mtx_lock_assert_wait_panic_str)
859 ori r3,r3,lo16(L_mtx_lock_assert_wait_panic_str)
91447636
A		 860 bl EXT(panic)
861 BREAKPOINT_TRAP ; We die here anyway
862
863 .data
b0d623f7 864L_mtx_lock_assert_wait_panic_str:
91447636
A		 865 STRINGD "mutex lock attempt with assert_wait_possible false\n\000"
866 .text
867
b0d623f7 868L_mtx_lock_assert_wait_1:
91447636
A		 869 lwz r3,FM_ARG0(r1)
870 lwz r11,FM_ARG0+0x04(r1)
871 lwz r2,(FM_ALIGN(0)+FM_SIZE+FM_CR_SAVE)(r1)
872 mtcr r2
873 EPILOG
b0d623f7 874L_mtx_lock_assert_wait_2:
1c79356b 875
55e303ae 876 mfsprg r6,1 ; load the current thread
91447636
A		 877 bf MUTEX_ATTR_STATb,mlckestatskip ; Branch if no stat
878 lwz r5,MUTEX_GRP(r3) ; Load lock group
879 li r7,GRP_MTX_STAT_UTIL+4 ; Set stat util offset
880mlckestatloop:
881 lwarx r8,r7,r5 ; Load stat util cnt
882 addi r8,r8,1 ; Increment stat util cnt
883 stwcx. r8,r7,r5 ; Store stat util cnt
884 bne-- mlckestatloop ; Retry if failed
885 mr. r8,r8 ; Test for zero
886 bne++ mlckestatskip ; Did stat util cnt wrapped?
887 lwz r8,GRP_MTX_STAT_UTIL(r5) ; Load upper stat util cnt
888 addi r8,r8,1 ; Increment upper stat util cnt
889 stw r8,GRP_MTX_STAT_UTIL(r5) ; Store upper stat util cnt
890mlckestatskip:
891 lwz r5,MUTEX_DATA(r3) ; Get the lock quickly
ab86ba33
A		 892 li r4,0
893 li r8,0
91447636
A		 894 lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable
895 mfmsr r9 ; Get the MSR value
896 ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable
897 ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too
898 andc r9,r9,r0 ; Clear FP and VEC
899 andc r7,r9,r7 ; Clear EE as well
900 mtmsr r7 ; Turn off interruptions
901 isync ; May have turned off vec and fp here
55e303ae 902 mr. r5,r5 ; Quick check
91447636 903 bne-- mlckespin01 ; Can not get it right now...
55e303ae 904
91447636
A		 905mlcketry:
906 lwarx r5,MUTEX_DATA,r3 ; load the mutex lock
9bccf70c 907 mr. r5,r5
91447636
A		 908 bne-- mlckespin0 ; Can not get it right now...
909 stwcx. r6,MUTEX_DATA,r3 ; grab the lock
910 bne-- mlcketry ; loop back if failed
911 .globl EXT(mlckePatch_isync)
912LEXT(mlckePatch_isync)
55e303ae 913 isync ; stop prefeteching
91447636
A		 914 mflr r12
915 bf MUTEX_ATTR_DEBUGb,mlckedebskip
916 mr r8,r6 ; Get the active thread
917 stw r12,MUTEX_STACK(r3) ; Save our caller
918 stw r8,MUTEX_THREAD(r3) ; Set the mutex's holding thread
919 mr r5,r1
920 LCK_STACK(r3,r5,r6,r7,r8,r10)
921mlckedebskip:
922 mtmsr r9 ; Say, any interrupts pending?
0b4e3aa0 923 blr
55e303ae 924
91447636 925mlckespin0:
55e303ae
A		 926 li r5,lgKillResv ; Killing field
927 stwcx. r5,0,r5 ; Kill reservation
91447636
A		 928mlckespin01:
929 mflr r12
930 mtmsr r9 ; Say, any interrupts pending?
931 bl mlckspin1
932 mtmsr r7 ; Turn off interruptions, vec and fp off already
933 mtlr r12
934 b mlcketry
935
936/*
937 * void lck_mtx_lock(lck_mtx_t*)
938 *
939 */
940 .align 5
941 .globl EXT(lck_mtx_lock)
942LEXT(lck_mtx_lock)
943
91447636
A		 944 mfsprg r6,1 ; load the current thread
945 lwz r5,MUTEX_DATA(r3) ; Get the lock quickly
946 mr r11,r3 ; Save lock addr
947 li r4,0
948 li r8,0
949 li r9,0
950 mr. r5,r5 ; Quick check
951 bne-- mlckspin00 ; Indirect or Can not get it right now...
952
953mlcktry:
954 lwarx r5,MUTEX_DATA,r3 ; load the mutex lock
955 mr. r5,r5
956 bne-- mlckspin01 ; Can not get it right now...
957 stwcx. r6,MUTEX_DATA,r3 ; grab the lock
958 bne-- mlcktry ; loop back if failed
959 .globl EXT(mlckPatch_isync)
960LEXT(mlckPatch_isync)
961 isync ; stop prefeteching
962 blr
2d21ac55
A		 963; Need to debug making blr above a patch point and record:
964; LOCKSTAT_RECORD(LS_LCK_MTX_LOCK_ACQUIRE)
91447636
A		 965
966mlckspin00:
967 cmpli cr0,r5,MUTEX_IND ; Is it a mutex indirect
968 bne-- mlckspin02 ; No, go handle contention
969 lwz r3,MUTEX_PTR(r3) ; load mutex ext pointer
970 b mlckeEnter
971mlckspin01:
972 li r5,lgKillResv ; Killing field
973 stwcx. r5,0,r5 ; Kill reservation
974mlckspin02:
975 mflr r12
976 li r0,0
977 mtcrf 1,r0 ; Set cr7 to zero
978 bl mlckspin1
979 mtlr r12
980 b mlcktry
981
982
983mlckspin1:
984 mr. r4,r4 ; Test timeout value
985 bne++ mlckspin2
986 lis r4,hi16(EXT(MutexSpin)) ; Get the high part
ab86ba33
A		 987 ori r4,r4,lo16(EXT(MutexSpin) ) ; And the low part
988 lwz r4,0(r4) ; Get spin timerout value
989 mr. r4,r4 ; Test spin timeout value
91447636
A		 990 bne++ mlckspin2 ; Is spin timeout requested
991 crclr mlckmiss ; Clear miss test
992 b mlckslow1 ; Don't try to spin
ab86ba33
A		 993
994mlckspin2: mr. r8,r8 ; Is r8 set to zero
995 bne++ mlckspin3 ; If yes, first spin attempt
91447636
A		 996 crclr mlckmiss ; Clear miss test
997 mr. r9,r9 ; Is r9 set to zero
998 bne++ mlckspin3 ; If yes, r9 set with msr value
ab86ba33
A		 999 lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable
1000 mfmsr r9 ; Get the MSR value
1001 ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable
1002 ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too
1003 andc r9,r9,r0 ; Clear FP and VEC
1004 andc r7,r9,r7 ; Clear EE as well
1005 mtmsr r7 ; Turn off interruptions
1006 isync ; May have turned off vec and fp here
1007 mftb r8 ; Get timestamp on entry
1008 b mlcksniff
1009
1010mlckspin3: mtmsr r7 ; Turn off interruptions
1011 mftb r8 ; Get timestamp on entry
1012
91447636 1013mlcksniff: lwz r5,MUTEX_DATA(r3) ; Get that lock in here
ab86ba33
A		 1014 mr. r5,r5 ; Is the lock held
1015 beq++ mlckretry ; No, try for it again...
91447636 1016 rlwinm. r10,r5,0,0,29 ; Extract the lock owner
ab86ba33 1017 beq++ mlckslow0 ; InterLock is held
91447636
A		 1018 bf MUTEX_ATTR_STATb,mlStatSkip ; Branch if no stat
1019 andi. r5,r5,ILK_LOCKED ; extract interlocked?
1020 bne mlStatSkip ; yes, skip
1021 bt mlckmiss,mlStatSkip ; miss already counted
1022 crset mlckmiss ; Remember miss recorded
1023 lwz r5,MUTEX_GRP(r3) ; Load lock group
1024 addi r5,r5,GRP_MTX_STAT_MISS+4 ; Add stat miss offset
1025mlStatLoop:
1026 lwarx r6,0,r5 ; Load stat miss cnt
1027 addi r6,r6,1 ; Increment stat miss cnt
1028 stwcx. r6,0,r5 ; Update stat miss cnt
1029 bne-- mlStatLoop ; Retry if failed
1030 mfsprg r6,1 ; Reload current thread
1031mlStatSkip:
1032 lwz r2,ACT_MACT_SPF(r10) ; Get the special flags
1033 rlwinm. r2,r2,0,OnProcbit,OnProcbit ; Is OnProcbit set?
ab86ba33 1034 beq mlckslow0 ; Lock owner isn't running
2d21ac55
A		 1035 lis r2,hi16(TH_IDLE) ; Get thread idle state
1036 ori r2,r2,lo16(TH_IDLE) ; Get thread idle state
1037 lwz r10,THREAD_STATE(r10) ; Get the thread state
1038 and. r10,r10,r2 ; Is idle set?
1039 bne mlckslow0 ; Lock owner is idling
ab86ba33
A		 1040
1041 mftb r10 ; Time stamp us now
1042 sub r10,r10,r8 ; Get the elapsed time
1043 cmplwi r10,128 ; Have we been spinning for 128 tb ticks?
1044 blt++ mlcksniff ; Not yet...
1045
1046 mtmsr r9 ; Say, any interrupts pending?
1047
1048; The following instructions force the pipeline to be interlocked to that only one
1049; instruction is issued per cycle. The insures that we stay enabled for a long enough
1050; time; if it's too short, pending interruptions will not have a chance to be taken
1051
1052 subi r4,r4,128 ; Back off elapsed time from timeout value
1053 or r4,r4,r4 ; Do nothing here but force a single cycle delay
1054 mr. r4,r4 ; See if we used the whole timeout
1055 or r4,r4,r4 ; Do nothing here but force a single cycle delay
1056
1057 ble-- mlckslow1 ; We failed
91447636 1058 b mlckspin3 ; Now that we've opened an enable window, keep trying...
ab86ba33
A		 1059mlckretry:
1060 mtmsr r9 ; Restore interrupt state
1061 li r8,1 ; Show already through once
91447636
A		 1062 blr
1063
ab86ba33
A		 1064mlckslow0: ; We couldn't get the lock
1065 mtmsr r9 ; Restore interrupt state
55e303ae 1066
ab86ba33 1067mlckslow1:
91447636 1068 mtlr r12
1c79356b 1069
55e303ae 1070 PROLOG(0)
1c79356b 1071.L_ml_retry:
55e303ae
A		 1072 bl lockDisa ; Go get a lock on the mutex's interlock lock
1073 mr. r4,r3 ; Did we get it?
1074 lwz r3,FM_ARG0(r1) ; Restore the lock address
3a60a9f5 1075 bne++ mlGotInt ; We got it just fine...
91447636 1076 mr r4,r11 ; Saved lock addr
55e303ae
A		 1077 lis r3,hi16(mutex_failed1) ; Get the failed mutex message
1078 ori r3,r3,lo16(mutex_failed1) ; Get the failed mutex message
55e303ae
A		 1079 bl EXT(panic) ; Call panic
1080 BREAKPOINT_TRAP ; We die here anyway, can not get the lock
1c79356b
A		 1081
1082 .data
1083mutex_failed1:
91447636 1084 STRINGD "attempt to interlock mutex (0x%08X) failed on mutex lock\n\000"
1c79356b
A		 1085 .text
1086
1087mlGotInt:
1088
55e303ae
A		 1089; Note that there is no reason to do a load and reserve here. We already
1090; hold the interlock lock and no one can touch this field unless they
1091; have that, so, we're free to play
1092
91447636 1093 lwz r4,MUTEX_DATA(r3) ; Get the mutex's lock field
55e303ae
A		 1094 rlwinm. r9,r4,30,2,31 ; So, can we have it?
1095 bne- mlInUse ; Nope, sombody's playing already...
1c79356b 1096
91447636
A		 1097 bf++ MUTEX_ATTR_DEBUGb,mlDebSkip
1098 CHECK_SETUP(r5)
55e303ae
A		 1099 mfsprg r9,1 ; Get the current activation
1100 lwz r5,0(r1) ; Get previous save frame
91447636
A		 1101 lwz r6,FM_LR_SAVE(r5) ; Get our caller's address
1102 mr r8,r9 ; Get the active thread
1103 stw r6,MUTEX_STACK(r3) ; Save our caller
55e303ae 1104 stw r8,MUTEX_THREAD(r3) ; Set the mutex's holding thread
91447636
A		 1105 LCK_STACK(r3,r5,r6,r7,r8,r10)
1106mlDebSkip:
1107 mr r3,r11 ; Get the based lock address
1108 bl EXT(lck_mtx_lock_acquire)
1109 lwz r2,(FM_ALIGN(0)+FM_SIZE+FM_CR_SAVE)(r1)
9bccf70c 1110 mfsprg r5,1
91447636 1111 mtcr r2
9bccf70c 1112 mr. r4,r3
91447636
A		 1113 lwz r3,FM_ARG0(r1) ; restore r3 (saved in prolog)
1114 lwz r11,FM_ARG0+0x04(r1) ; restore r11 (saved in prolog)
9bccf70c
A		 1115 beq mlUnlock
1116 ori r5,r5,WAIT_FLAG
1c79356b 1117
55e303ae 1118mlUnlock: eieio
91447636 1119 stw r5,MUTEX_DATA(r3) ; grab the mutexlock and free the interlock
1c79356b 1120
55e303ae
A		 1121 EPILOG ; Restore all saved registers
1122 b epStart ; Go enable preemption...
de355530 1123
55e303ae
A		 1124; We come to here when we have a resource conflict. In other words,
1125; the mutex is held.
1c79356b
A		 1126
1127mlInUse:
1128
de355530 1129 CHECK_SETUP(r12)
91447636 1130 CHECK_MYLOCK() ; Assert we don't own the lock already */
d7e50217 1131
55e303ae
A		 1132; Note that we come in here with the interlock set. The wait routine
1133; will unlock it before waiting.
1134
91447636
A		 1135 bf MUTEX_ATTR_STATb,mlStatSkip2 ; Branch if no stat
1136 lwz r5,MUTEX_GRP(r3) ; Load lck group
1137 bt mlckmiss,mlStatSkip1 ; Skip miss already counted
1138 crset mlckmiss ; Remember miss recorded
1139 li r9,GRP_MTX_STAT_MISS+4 ; Get stat miss offset
1140mlStatLoop1:
1141 lwarx r8,r9,r5 ; Load stat miss cnt
1142 addi r8,r8,1 ; Increment stat miss cnt
1143 stwcx. r8,r9,r5 ; Store stat miss cnt
1144 bne-- mlStatLoop1 ; Retry if failed
1145mlStatSkip1:
1146 lwz r9,GRP_MTX_STAT_WAIT+4(r5) ; Load wait cnt
1147 addi r9,r9,1 ; Increment wait cnt
1148 stw r9,GRP_MTX_STAT_WAIT+4(r5) ; Update miss cnt
1149mlStatSkip2:
55e303ae 1150 ori r4,r4,WAIT_FLAG ; Set the wait flag
91447636 1151 stw r4,MUTEX_DATA(r3)
55e303ae 1152 rlwinm r4,r4,0,0,29 ; Extract the lock owner
91447636
A		 1153 mfcr r2
1154 stw r2,(FM_ALIGN(0)+FM_SIZE+FM_CR_SAVE)(r1)
1155 mr r3,r11 ; Get the based lock address
1156 bl EXT(lck_mtx_lock_wait) ; Wait for our turn at the lock
1c79356b 1157
55e303ae 1158 lwz r3,FM_ARG0(r1) ; restore r3 (saved in prolog)
91447636
A		 1159 lwz r11,FM_ARG0+0x04(r1) ; restore r11 (saved in prolog)
1160 lwz r2,(FM_ALIGN(0)+FM_SIZE+FM_CR_SAVE)(r1)
1161 mtcr r2
55e303ae 1162 b .L_ml_retry ; and try again...
1c79356b
A		 1163
1164
1165/*
91447636 1166 * void lck_mtx_try_lock(_extlck_mtx_ext_t*)
1c79356b
A		 1167 *
1168 */
1c79356b 1169 .align 5
91447636
A		 1170 .globl EXT(lck_mtx_try_lock_ext)
1171LEXT(lck_mtx_try_lock_ext)
91447636
A		 1172 mr r11,r3 ; Save lock addr
1173mlteEnter:
1174 lwz r0,MUTEX_ATTR(r3)
1175 mtcrf 1,r0 ; Set cr7
1176 CHECK_SETUP(r12)
1177 CHECK_MUTEX_TYPE()
1178
1179 bf MUTEX_ATTR_STATb,mlteStatSkip ; Branch if no stat
1180 lwz r5,MUTEX_GRP(r3) ; Load lock group
1181 li r7,GRP_MTX_STAT_UTIL+4 ; Set stat util offset
1182mlteStatLoop:
1183 lwarx r8,r7,r5 ; Load stat util cnt
1184 addi r8,r8,1 ; Increment stat util cnt
1185 stwcx. r8,r7,r5 ; Store stat util cnt
1186 bne-- mlteStatLoop ; Retry if failed
1187 mr. r8,r8 ; Test for zero
1188 bne++ mlteStatSkip ; Did stat util cnt wrapped?
1189 lwz r8,GRP_MTX_STAT_UTIL(r5) ; Load upper stat util cnt
1190 addi r8,r8,1 ; Increment upper stat util cnt
1191 stw r8,GRP_MTX_STAT_UTIL(r5) ; Store upper stat util cnt
1192mlteStatSkip:
55e303ae 1193 mfsprg r6,1 ; load the current thread
91447636 1194 lwz r5,MUTEX_DATA(r3) ; Get the lock value
55e303ae 1195 mr. r5,r5 ; Quick check
b0d623f7 1196 bne-- L_mtx_try_slow ; Can not get it now...
91447636
A		 1197 mfmsr r9 ; Get the MSR value
1198 lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable
1199 ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable
1200 ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too
1201 andc r9,r9,r0 ; Clear FP and VEC
1202 andc r7,r9,r7 ; Clear EE as well
1203 mtmsr r7 ; Turn off interruptions
1204 isync ; May have turned off vec and fp here
55e303ae 1205
91447636
A		 1206mlteLoopTry:
1207 lwarx r5,MUTEX_DATA,r3 ; load the lock value
9bccf70c 1208 mr. r5,r5
91447636
A		 1209 bne-- mlteSlowX ; branch to the slow path
1210 stwcx. r6,MUTEX_DATA,r3 ; grab the lock
1211 bne-- mlteLoopTry ; retry if failed
1212 .globl EXT(mltelckPatch_isync)
1213LEXT(mltelckPatch_isync)
55e303ae 1214 isync ; stop prefetching
91447636
A		 1215 mflr r12
1216 bf MUTEX_ATTR_DEBUGb,mlteDebSkip
1217 mr r8,r6 ; Get the active thread
1218 stw r12,MUTEX_STACK(r3) ; Save our caller
1219 stw r8,MUTEX_THREAD(r3) ; Set the mutex's holding thread
1220 mr r5,r1
1221 LCK_STACK(r3,r5,r6,r7,r8,r10)
1222mlteDebSkip:
0b4e3aa0 1223 li r3, 1
91447636 1224 mtmsr r9 ; Say, any interrupts pending?
0b4e3aa0 1225 blr
91447636 1226mlteSlowX:
55e303ae
A		 1227 li r5,lgKillResv ; Killing field
1228 stwcx. r5,0,r5 ; Kill reservation
91447636 1229 mtmsr r9 ; Say, any interrupts pending?
b0d623f7 1230 b L_mtx_try_slow
55e303ae 1231
55e303ae 1232
91447636
A		 1233/*
1234 * void lck_mtx_try_lock(lck_mtx_t*)
1235 *
1236 */
1237 .align 5
1238 .globl EXT(lck_mtx_try_lock)
1239LEXT(lck_mtx_try_lock)
1c79356b 1240
91447636
A		 1241 mfsprg r6,1 ; load the current thread
1242 lwz r5,MUTEX_DATA(r3) ; Get the lock value
1243 mr r11,r3 ; Save lock addr
1244 mr. r5,r5 ; Quick check
1245 bne-- mltSlow00 ; Indirect or Can not get it now...
1246
1247mltLoopTry:
1248 lwarx r5,MUTEX_DATA,r3 ; load the lock value
1249 mr. r5,r5
1250 bne-- mltSlow01 ; branch to the slow path
1251 stwcx. r6,MUTEX_DATA,r3 ; grab the lock
1252 bne-- mltLoopTry ; retry if failed
1253 .globl EXT(mltlckPatch_isync)
1254LEXT(mltlckPatch_isync)
1255 isync ; stop prefetching
1256 li r3, 1
1257 blr
1258
1259mltSlow00:
1260 cmpli cr0,r5,MUTEX_IND ; Is it a mutex indirect
1261 bne-- mltSlow02 ; No, go handle contention
1262 lwz r3,MUTEX_PTR(r3) ; load mutex ext pointer
1263 b mlteEnter
1264mltSlow01:
1265 li r5,lgKillResv ; Killing field
1266 stwcx. r5,0,r5 ; Kill reservation
1267
1268mltSlow02:
1269 li r0,0
1270 mtcrf 1,r0 ; Set cr7 to zero
1271
b0d623f7 1272L_mtx_try_slow:
91447636 1273 PROLOG(0)
1c79356b 1274
91447636 1275 lwz r6,MUTEX_DATA(r3) ; Quick check
55e303ae
A		 1276 rlwinm. r6,r6,30,2,31 ; to see if someone has this lock already
1277 bne- mtFail ; Someone's got it already...
1278
1279 bl lockDisa ; Go get a lock on the mutex's interlock lock
3a60a9f5 1280 mr. r4,r3 ; Did we get it?
55e303ae 1281 lwz r3,FM_ARG0(r1) ; Restore the lock address
3a60a9f5 1282 bne++ mtGotInt ; We got it just fine...
91447636 1283 mr r4,r11 ; Saved lock addr
55e303ae
A		 1284 lis r3,hi16(mutex_failed2) ; Get the failed mutex message
1285 ori r3,r3,lo16(mutex_failed2) ; Get the failed mutex message
55e303ae
A		 1286 bl EXT(panic) ; Call panic
1287 BREAKPOINT_TRAP ; We die here anyway, can not get the lock
1c79356b
A		 1288
1289 .data
1290mutex_failed2:
91447636 1291 STRINGD "attempt to interlock mutex (0x%08X) failed on mutex lock try\n\000"
1c79356b
A		 1292 .text
1293
1294mtGotInt:
1295
55e303ae
A		 1296; Note that there is no reason to do a load and reserve here. We already
1297; hold the interlock and no one can touch at this field unless they
1298; have that, so, we're free to play
1c79356b 1299
91447636 1300 lwz r4,MUTEX_DATA(r3) ; Get the mutex's lock field
55e303ae
A		 1301 rlwinm. r9,r4,30,2,31 ; So, can we have it?
1302 bne- mtInUse ; Nope, sombody's playing already...
1c79356b 1303
91447636
A		 1304 bf++ MUTEX_ATTR_DEBUGb,mtDebSkip
1305 CHECK_SETUP(r5)
55e303ae
A		 1306 mfsprg r9,1 ; Get the current activation
1307 lwz r5,0(r1) ; Get previous save frame
91447636
A		 1308 lwz r6,FM_LR_SAVE(r5) ; Get our caller's address
1309 mr r8,r9 ; Get the active thread
1310 stw r6,MUTEX_STACK(r3) ; Save our caller
55e303ae 1311 stw r8,MUTEX_THREAD(r3) ; Set the mutex's holding thread
91447636
A		 1312 LCK_STACK(r3,r5,r6,r7,r8,r10)
1313mtDebSkip:
1314 mr r3,r11 ; Get the based lock address
1315 bl EXT(lck_mtx_lock_acquire)
9bccf70c
A		 1316 mfsprg r5,1
1317 mr. r4,r3
91447636
A		 1318 lwz r3,FM_ARG0(r1) ; restore r3 (saved in prolog)
1319 lwz r11,FM_ARG0+0x04(r1) ; restore r11 (saved in prolog)
9bccf70c
A		 1320 beq mtUnlock
1321 ori r5,r5,WAIT_FLAG
1c79356b 1322
55e303ae 1323mtUnlock: eieio
91447636 1324 stw r5,MUTEX_DATA(r3) ; grab the mutexlock and free the interlock
1c79356b 1325
55e303ae 1326 bl epStart ; Go enable preemption...
0b4e3aa0 1327
1c79356b 1328 li r3, 1
55e303ae
A		 1329 EPILOG ; Restore all saved registers
1330 blr ; Return...
1c79356b 1331
55e303ae
A		 1332; We come to here when we have a resource conflict. In other words,
1333; the mutex is held.
1c79356b 1334
0b4e3aa0 1335mtInUse:
91447636
A		 1336 bf++ MUTEX_ATTR_STATb,mtStatSkip ; Branch if no stat
1337 lwz r5,MUTEX_GRP(r3) ; Load lock group
1338 li r9,GRP_MTX_STAT_MISS+4 ; Get stat miss offset
1339mtStatLoop:
1340 lwarx r8,r9,r5 ; Load stat miss cnt
1341 addi r8,r8,1 ; Increment stat miss cnt
1342 stwcx. r8,r9,r5 ; Store stat miss cnt
1343 bne-- mtStatLoop ; Retry if failed
1344mtStatSkip:
55e303ae 1345 rlwinm r4,r4,0,0,30 ; Get the unlock value
91447636 1346 stw r4,MUTEX_DATA(r3) ; free the interlock
55e303ae 1347 bl epStart ; Go enable preemption...
1c79356b 1348
55e303ae
A		 1349mtFail: li r3,0 ; Set failure code
1350 EPILOG ; Restore all saved registers
1351 blr ; Return...
1c79356b
A		 1352
1353
55e303ae 1354
91447636
A		 1355/*
1356 * void lck_mtx_ext_unlock(lck_mtx_ext_t* l)
1357 *
1358 */
1359 .align 5
1360 .globl EXT(lck_mtx_ext_unlock)
1361LEXT(lck_mtx_ext_unlock)
91447636
A		 1362mlueEnter:
1363 .globl EXT(mulckePatch_isync)
1364LEXT(mulckePatch_isync)
55e303ae 1365 isync
91447636
A		 1366 .globl EXT(mulckePatch_eieio)
1367LEXT(mulckePatch_eieio)
55e303ae 1368 eieio
91447636
A		 1369 mr r11,r3 ; Save lock addr
1370mlueEnter1:
1371 lwz r0,MUTEX_ATTR(r3)
1372 mtcrf 1,r0 ; Set cr7
1373 CHECK_SETUP(r12)
1374 CHECK_MUTEX_TYPE()
1375 CHECK_THREAD(MUTEX_THREAD)
1c79356b 1376
91447636 1377 lwz r5,MUTEX_DATA(r3) ; Get the lock
55e303ae 1378 rlwinm. r4,r5,0,30,31 ; Quick check
b0d623f7 1379 bne-- L_mtx_unlock_slow ; Can not get it now...
91447636
A		 1380 mfmsr r9 ; Get the MSR value
1381 lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable
1382 ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable
1383 ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too
1384 andc r9,r9,r0 ; Clear FP and VEC
1385 andc r7,r9,r7 ; Clear EE as well
1386 mtmsr r7 ; Turn off interruptions
1387 isync ; May have turned off vec and fp here
55e303ae 1388
91447636
A		 1389mlueLoop:
1390 lwarx r5,MUTEX_DATA,r3
55e303ae
A		 1391 rlwinm. r4,r5,0,30,31 ; Bail if pending waiter or interlock set
1392 li r5,0 ; Clear the mutexlock
91447636
A		 1393 bne-- mlueSlowX
1394 stwcx. r5,MUTEX_DATA,r3
1395 bne-- mlueLoop
1396 mtmsr r9 ; Say, any interrupts pending?
55e303ae
A		 1397 blr
1398
91447636 1399mlueSlowX:
55e303ae
A		 1400 li r5,lgKillResv ; Killing field
1401 stwcx. r5,0,r5 ; Dump reservation
91447636 1402 mtmsr r9 ; Say, any interrupts pending?
b0d623f7 1403 b L_mtx_unlock_slow ; Join slow path...
55e303ae
A		 1404
1405/*
91447636 1406 * void lck_mtx_unlock(lck_mtx_t* l)
55e303ae
A		 1407 *
1408 */
1c79356b 1409 .align 5
91447636
A		 1410 .globl EXT(lck_mtx_unlock)
1411LEXT(lck_mtx_unlock)
91447636
A		 1412mluEnter:
1413 .globl EXT(mulckPatch_isync)
1414LEXT(mulckPatch_isync)
1415 isync
1416 .globl EXT(mulckPatch_eieio)
1417LEXT(mulckPatch_eieio)
1418 eieio
1419 mr r11,r3 ; Save lock addr
1420mluEnter1:
1421 lwz r5,MUTEX_DATA(r3) ; Get the lock
55e303ae 1422 rlwinm. r4,r5,0,30,31 ; Quick check
91447636 1423 bne-- mluSlow0 ; Indirect or Can not get it now...
55e303ae 1424
91447636
A		 1425mluLoop:
1426 lwarx r5,MUTEX_DATA,r3
55e303ae
A		 1427 rlwinm. r4,r5,0,30,31 ; Bail if pending waiter or interlock set
1428 li r5,0 ; Clear the mutexlock
91447636
A		 1429 bne-- mluSlowX
1430 stwcx. r5,MUTEX_DATA,r3
1431 bne-- mluLoop
2d21ac55
A		 1432#if CONFIG_DTRACE
1433/* lock released - LS_LCK_MTX_UNLOCK_RELEASE */
1434 LOCKSTAT_LABEL(_lck_mtx_unlock_lockstat_patch_point)
0b4e3aa0 1435 blr
91447636 1436
2d21ac55
A		 1437 LOCKSTAT_RECORD(LS_LCK_MTX_UNLOCK_RELEASE)
1438#endif
1439 blr
1440
1441
91447636
A		 1442mluSlow0:
1443 cmpli cr0,r5,MUTEX_IND ; Is it a mutex indirect
b0d623f7 1444 bne-- L_mtx_unlock_slow ; No, go handle contention
91447636
A		 1445 lwz r3,MUTEX_PTR(r3) ; load mutex ext pointer
1446 b mlueEnter1
1447mluSlowX:
55e303ae
A		 1448 li r5,lgKillResv ; Killing field
1449 stwcx. r5,0,r5 ; Dump reservation
1450
b0d623f7 1451L_mtx_unlock_slow:
55e303ae 1452
1c79356b
A		 1453 PROLOG(0)
1454
55e303ae
A		 1455 bl lockDisa ; Go get a lock on the mutex's interlock lock
1456 mr. r4,r3 ; Did we get it?
1457 lwz r3,FM_ARG0(r1) ; Restore the lock address
3a60a9f5 1458 bne++ muGotInt ; We got it just fine...
91447636 1459 mr r4,r11 ; Saved lock addr
55e303ae
A		 1460 lis r3,hi16(mutex_failed3) ; Get the failed mutex message
1461 ori r3,r3,lo16(mutex_failed3) ; Get the failed mutex message
55e303ae
A		 1462 bl EXT(panic) ; Call panic
1463 BREAKPOINT_TRAP ; We die here anyway, can not get the lock
1c79356b
A		 1464
1465 .data
1466mutex_failed3:
91447636 1467 STRINGD "attempt to interlock mutex (0x%08X) failed on mutex unlock\n\000"
1c79356b
A		 1468 .text
1469
1470
1471muGotInt:
91447636 1472 lwz r4,MUTEX_DATA(r3)
55e303ae 1473 andi. r5,r4,WAIT_FLAG ; are there any waiters ?
9bccf70c 1474 rlwinm r4,r4,0,0,29
55e303ae 1475 beq+ muUnlock ; Nope, we're done...
1c79356b 1476
91447636
A		 1477 mr r3,r11 ; Get the based lock address
1478 bl EXT(lck_mtx_unlock_wakeup) ; yes, wake a thread
55e303ae 1479 lwz r3,FM_ARG0(r1) ; restore r3 (saved in prolog)
91447636
A		 1480 lwz r11,FM_ARG0+0x04(r1) ; restore r11 (saved in prolog)
1481 lwz r5,MUTEX_DATA(r3) ; load the lock
1c79356b
A		 1482
1483muUnlock:
55e303ae
A		 1484 andi. r5,r5,WAIT_FLAG ; Get the unlock value
1485 eieio
91447636 1486 stw r5,MUTEX_DATA(r3) ; unlock the interlock and lock
55e303ae
A		 1487
1488 EPILOG ; Deal with the stack now, enable_preemption doesn't always want one
1489 b epStart ; Go enable preemption...
1c79356b
A		 1490
1491/*
91447636
A		 1492 * void lck_mtx_assert(lck_mtx_t* l, unsigned int)
1493 *
1c79356b 1494 */
91447636
A		 1495 .align 5
1496 .globl EXT(lck_mtx_assert)
1497LEXT(lck_mtx_assert)
91447636
A		 1498 mr r11,r3
1499maEnter:
1500 lwz r5,MUTEX_DATA(r3)
1501 cmpli cr0,r5,MUTEX_IND ; Is it a mutex indirect
1502 bne-- maCheck ; No, go check the assertion
1503 lwz r3,MUTEX_PTR(r3) ; load mutex ext pointer
1504 b maEnter
1505maCheck:
1506 mfsprg r6,1 ; load the current thread
1507 rlwinm r5,r5,0,0,29 ; Extract the lock owner
1508 cmpwi r4,MUTEX_ASSERT_OWNED
1509 cmplw cr1,r6,r5 ; Is the lock held by current act
1510 crandc cr0_eq,cr0_eq,cr1_eq ; Check owned assertion
1511 bne-- maNext
1512 mr r4,r11
1513 lis r3,hi16(mutex_assert1) ; Get the failed mutex message
1514 ori r3,r3,lo16(mutex_assert1) ; Get the failed mutex message
1515 b maPanic ; Panic path
1516maNext:
1517 cmpwi r4,MUTEX_ASSERT_NOTOWNED ; Check not owned assertion
1518 crand cr0_eq,cr0_eq,cr1_eq ;
1519 bnelr++
1520maPanic:
55e303ae 1521 PROLOG(0)
91447636
A		 1522 mr r4,r11
1523 lis r3,hi16(mutex_assert2) ; Get the failed mutex message
1524 ori r3,r3,lo16(mutex_assert2) ; Get the failed mutex message
1525 bl EXT(panic) ; Call panic
1526 BREAKPOINT_TRAP ; We die here anyway
55e303ae 1527
91447636
A		 1528 .data
1529mutex_assert1:
1530 STRINGD "mutex (0x%08X) not owned\n\000"
1531mutex_assert2:
1532 STRINGD "mutex (0x%08X) owned\n\000"
1533 .text
1534
1535
55e303ae 1536/*
91447636 1537 * void lck_mtx_ilk_unlock(lck_mtx *lock)
55e303ae 1538 */
91447636
A		 1539 .globl EXT(lck_mtx_ilk_unlock)
1540LEXT(lck_mtx_ilk_unlock)
1c79356b 1541
91447636 1542 lwz r10,MUTEX_DATA(r3)
0b4e3aa0 1543 rlwinm r10,r10,0,0,30
55e303ae 1544 eieio
91447636 1545 stw r10,MUTEX_DATA(r3)
1c79356b 1546
55e303ae 1547 b epStart ; Go enable preemption...
1c79356b 1548
55e303ae
A		 1549/*
1550 * void _enable_preemption_no_check(void)
1c79356b 1551 *
55e303ae 1552 * This version does not check if we get preempted or not
1c79356b 1553 */
1c79356b
A		 1554 .align 4
1555 .globl EXT(_enable_preemption_no_check)
1556
1557LEXT(_enable_preemption_no_check)
1c79356b 1558
55e303ae
A		 1559 cmplw cr1,r1,r1 ; Force zero cr so we know not to check if preempted
1560 b epCommn ; Join up with the other enable code...
1c79356b 1561
55e303ae
A		 1562/*
1563 * void _enable_preemption(void)
1564 *
1565 * This version checks if we get preempted or not
1566 */
1c79356b
A		 1567 .align 5
1568 .globl EXT(_enable_preemption)
1569
1570LEXT(_enable_preemption)
1571
91447636 1572; Here is where we enable preemption.
55e303ae
A		 1573
1574epStart:
1575 cmplwi cr1,r1,0 ; Force non-zero cr so we know to check if preempted
1576
1577epCommn:
1578 mfsprg r3,1 ; Get current activation
1579 li r8,-1 ; Get a decrementer
1580 lwz r5,ACT_PREEMPT_CNT(r3) ; Get the preemption level
1581 add. r5,r5,r8 ; Bring down the disable count
1582 blt- epTooFar ; Yeah, we did...
1583 stw r5,ACT_PREEMPT_CNT(r3) ; Save it back
1584 crandc cr0_eq,cr0_eq,cr1_eq
1585 beq+ epCheckPreempt ; Go check if we need to be preempted...
1586 blr ; Leave...
1c79356b 1587epTooFar:
55e303ae
A		 1588 mr r4,r5
1589 lis r3,hi16(epTooFarStr) ; First half of panic string
1590 ori r3,r3,lo16(epTooFarStr) ; Second half of panic string
1591 PROLOG(0)
1592 bl EXT(panic)
1593 BREAKPOINT_TRAP ; We die here anyway
1c79356b
A		 1594
1595 .data
1596epTooFarStr:
91447636 1597 STRINGD "enable_preemption: preemption_level %d\n\000"
de355530 1598
55e303ae 1599 .text
1c79356b 1600 .align 5
1c79356b 1601epCheckPreempt:
55e303ae
A		 1602 lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable
1603 mfmsr r9 ; Get the MSR value
1604 ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable
91447636 1605 andi. r4,r9,lo16(MASK(MSR_EE)) ; We cannot preempt if interruptions are off
55e303ae
A		 1606 beq+ epCPno ; No preemption here...
1607 ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too
1608 andc r9,r9,r0 ; Clear FP and VEC
1609 andc r7,r9,r7 ; Clear EE as well
1610 mtmsr r7 ; Turn off interruptions
1611 isync ; May have turned off vec and fp here
91447636
A		 1612 lwz r3,ACT_PER_PROC(r3) ; Get the per_proc block
1613 lwz r7,PP_PENDING_AST(r3) ; Get pending AST mask
55e303ae 1614 li r5,AST_URGENT ; Get the requests we do honor
55e303ae
A		 1615 lis r0,hi16(DoPreemptCall) ; Just in case, get the top of firmware call
1616 and. r7,r7,r5 ; Should we preempt?
1617 ori r0,r0,lo16(DoPreemptCall) ; Merge in bottom part
1618 mtmsr r9 ; Allow interrupts if we can
1619epCPno:
1620 beqlr+ ; We probably will not preempt...
1621 sc ; Do the preemption
1622 blr ; Now, go away now...
1c79356b
A		 1623
1624/*
55e303ae
A		 1625 * void disable_preemption(void)
1626 *
91447636 1627 * Here is where we disable preemption.
1c79356b 1628 */
1c79356b 1629 .align 5
1c79356b
A		 1630 .globl EXT(_disable_preemption)
1631
1632LEXT(_disable_preemption)
1633
55e303ae
A		 1634 mfsprg r6,1 ; Get the current activation
1635 lwz r5,ACT_PREEMPT_CNT(r6) ; Get the preemption level
1636 addi r5,r5,1 ; Bring up the disable count
1637 stw r5,ACT_PREEMPT_CNT(r6) ; Save it back
1638 blr ; Return...
1c79356b
A		 1639
1640/*
55e303ae
A		 1641 * int get_preemption_level(void)
1642 *
1643 * Return the current preemption level
1c79356b 1644 */
1c79356b 1645 .align 5
55e303ae 1646 .globl EXT(get_preemption_level)
de355530 1647
55e303ae
A		 1648LEXT(get_preemption_level)
1649
1650 mfsprg r6,1 ; Get current activation
1651 lwz r3,ACT_PREEMPT_CNT(r6) ; Get the preemption level
1652 blr ; Return...
9bccf70c 1653
9bccf70c 1654/*
55e303ae
A		 1655 * void ppc_usimple_lock_init(simple_lock_t, etap_event_t)
1656 *
1657 * Initialize a simple lock.
9bccf70c
A		 1658 */
1659 .align 5
55e303ae 1660 .globl EXT(ppc_usimple_lock_init)
9bccf70c 1661
55e303ae 1662LEXT(ppc_usimple_lock_init)
1c79356b 1663
55e303ae
A		 1664 li r0, 0 ; set lock to free == 0
1665 stw r0, 0(r3) ; Initialize the lock
1666 blr
1667
1c79356b 1668/*
91447636
A		 1669 * void lck_spin_lock(lck_spin_t *)
1670 * void ppc_usimple_lock(simple_lock_t *)
55e303ae 1671 *
1c79356b 1672 */
1c79356b 1673 .align 5
91447636
A		 1674 .globl EXT(lck_spin_lock)
1675LEXT(lck_spin_lock)
55e303ae 1676 .globl EXT(ppc_usimple_lock)
55e303ae 1677LEXT(ppc_usimple_lock)
1c79356b 1678
55e303ae
A		 1679 mfsprg r6,1 ; Get the current activation
1680 lwz r5,ACT_PREEMPT_CNT(r6) ; Get the preemption level
1681 addi r5,r5,1 ; Bring up the disable count
1682 stw r5,ACT_PREEMPT_CNT(r6) ; Save it back
1683 mr r5,r3 ; Get the address of the lock
1684 li r8,0 ; Set r8 to zero
1685 li r4,0 ; Set r4 to zero
1686
91447636 1687slcktry: lwarx r11,SLOCK_ILK,r5 ; Grab the lock value
55e303ae
A		 1688 andi. r3,r11,ILK_LOCKED ; Is it locked?
1689 ori r11,r6,ILK_LOCKED ; Set interlock
1690 bne-- slckspin ; Yeah, wait for it to clear...
91447636 1691 stwcx. r11,SLOCK_ILK,r5 ; Try to seize that there durn lock
55e303ae 1692 bne-- slcktry ; Couldn't get it...
91447636
A		 1693 .globl EXT(slckPatch_isync)
1694LEXT(slckPatch_isync)
55e303ae
A		 1695 isync ; Make sure we don't use a speculativily loaded value
1696 blr ; Go on home...
1697
1698slckspin: li r11,lgKillResv ; Killing field
1699 stwcx. r11,0,r11 ; Kill reservation
1700
1701 mr. r4,r4 ; Test timeout value
1702 bne++ slockspin0
1703 lis r4,hi16(EXT(LockTimeOut)) ; Get the high part
1704 ori r4,r4,lo16(EXT(LockTimeOut)) ; And the low part
1705 lwz r4,0(r4) ; Get the timerout value
1706
1707slockspin0: mr. r8,r8 ; Is r8 set to zero
1708 bne++ slockspin1 ; If yes, first spin attempt
1709 lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable
1710 mfmsr r9 ; Get the MSR value
1711 ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable
1712 ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too
1713 andc r9,r9,r0 ; Clear FP and VEC
1714 andc r7,r9,r7 ; Clear EE as well
1715 mtmsr r7 ; Turn off interruptions
1716 isync ; May have turned off vec and fp here
1717 mftb r8 ; Get timestamp on entry
1718 b slcksniff
1719
1720slockspin1: mtmsr r7 ; Turn off interruptions
1721 mftb r8 ; Get timestamp on entry
1722
91447636 1723slcksniff: lwz r3,SLOCK_ILK(r5) ; Get that lock in here
55e303ae
A		 1724 andi. r3,r3,ILK_LOCKED ; Is it free yet?
1725 beq++ slckretry ; Yeah, try for it again...
1726
1727 mftb r10 ; Time stamp us now
1728 sub r10,r10,r8 ; Get the elapsed time
1729 cmplwi r10,128 ; Have we been spinning for 128 tb ticks?
1730 blt++ slcksniff ; Not yet...
1731
1732 mtmsr r9 ; Say, any interrupts pending?
9bccf70c 1733
55e303ae
A		 1734; The following instructions force the pipeline to be interlocked to that only one
1735; instruction is issued per cycle. The insures that we stay enabled for a long enough
1736; time; if it's too short, pending interruptions will not have a chance to be taken
1c79356b 1737
55e303ae
A		 1738 subi r4,r4,128 ; Back off elapsed time from timeout value
1739 or r4,r4,r4 ; Do nothing here but force a single cycle delay
1740 mr. r4,r4 ; See if we used the whole timeout
1741 li r3,0 ; Assume a timeout return code
1742 or r4,r4,r4 ; Do nothing here but force a single cycle delay
1743
1744 ble-- slckfail ; We failed
1745 b slockspin1 ; Now that we've opened an enable window, keep trying...
1746slckretry:
1747 mtmsr r9 ; Restore interrupt state
1748 li r8,1 ; Show already through once
1749 b slcktry
1750slckfail: ; We couldn't get the lock
1751 lis r3,hi16(slckpanic_str)
1752 ori r3,r3,lo16(slckpanic_str)
1753 mr r4,r5
1754 mflr r5
1755 PROLOG(0)
1756 bl EXT(panic)
1757 BREAKPOINT_TRAP ; We die here anyway
de355530 1758
55e303ae
A		 1759 .data
1760slckpanic_str:
91447636 1761 STRINGD "simple lock (0x%08X) deadlock detection, pc=0x%08X\n\000"
55e303ae 1762 .text
1c79356b 1763
0b4e3aa0 1764/*
91447636
A		 1765 * boolean_t lck_spin_try_lock(lck_spin_t *)
1766 * unsigned int ppc_usimple_lock_try(simple_lock_t *)
0b4e3aa0 1767 *
0b4e3aa0
A		 1768 */
1769 .align 5
91447636
A		 1770 .globl EXT(lck_spin_try_lock)
1771LEXT(lck_spin_try_lock)
55e303ae 1772 .globl EXT(ppc_usimple_lock_try)
55e303ae 1773LEXT(ppc_usimple_lock_try)
0b4e3aa0 1774
55e303ae
A		 1775 lis r0,hi16(MASK(MSR_VEC)) ; Get vector enable
1776 mfmsr r9 ; Get the MSR value
1777 ori r0,r0,lo16(MASK(MSR_FP)) ; Get FP enable
1778 ori r7,r0,lo16(MASK(MSR_EE)) ; Get EE bit on too
1779 andc r9,r9,r0 ; Clear FP and VEC
1780 andc r7,r9,r7 ; Clear EE as well
1781 mtmsr r7 ; Disable interruptions and thus, preemption
1782 mfsprg r6,1 ; Get current activation
1783
91447636 1784 lwz r11,SLOCK_ILK(r3) ; Get the lock
55e303ae
A		 1785 andi. r5,r11,ILK_LOCKED ; Check it...
1786 bne-- slcktryfail ; Quickly fail...
1787
1788slcktryloop:
91447636 1789 lwarx r11,SLOCK_ILK,r3 ; Ld from addr of arg and reserve
55e303ae
A		 1790
1791 andi. r5,r11,ILK_LOCKED ; TEST...
1792 ori r5,r6,ILK_LOCKED
1793 bne-- slcktryfailX ; branch if taken. Predict free
1794
91447636 1795 stwcx. r5,SLOCK_ILK,r3 ; And SET (if still reserved)
55e303ae
A		 1796 bne-- slcktryloop ; If set failed, loop back
1797
91447636
A		 1798 .globl EXT(stlckPatch_isync)
1799LEXT(stlckPatch_isync)
55e303ae
A		 1800 isync
1801
1802 lwz r5,ACT_PREEMPT_CNT(r6) ; Get the preemption level
1803 addi r5,r5,1 ; Bring up the disable count
1804 stw r5,ACT_PREEMPT_CNT(r6) ; Save it back
1805
1806 mtmsr r9 ; Allow interruptions now
1807 li r3,1 ; Set that the lock was free
1808 blr
1809
1810slcktryfailX:
1811 li r5,lgKillResv ; Killing field
1812 stwcx. r5,0,r5 ; Kill reservation
1813
1814slcktryfail:
1815 mtmsr r9 ; Allow interruptions now
1816 li r3,0 ; FAILURE - lock was taken
1817 blr
1818
0b4e3aa0
A		 1819
1820/*
91447636
A		 1821 * void lck_spin_unlock(lck_spin_t *)
1822 * void ppc_usimple_unlock_rwcmb(simple_lock_t *)
0b4e3aa0 1823 *
0b4e3aa0
A		 1824 */
1825 .align 5
91447636
A		 1826 .globl EXT(lck_spin_unlock)
1827LEXT(lck_spin_unlock)
55e303ae 1828 .globl EXT(ppc_usimple_unlock_rwcmb)
55e303ae 1829LEXT(ppc_usimple_unlock_rwcmb)
0b4e3aa0 1830
55e303ae
A		 1831 li r0,0
1832 .globl EXT(sulckPatch_isync)
1833LEXT(sulckPatch_isync)
1834 isync
1835 .globl EXT(sulckPatch_eieio)
1836LEXT(sulckPatch_eieio)
1837 eieio
91447636 1838 stw r0, SLOCK_ILK(r3)
55e303ae
A		 1839
1840 b epStart ; Go enable preemption...
0b4e3aa0
A		 1841
1842/*
91447636 1843 * void ppc_usimple_unlock_rwmb(simple_lock_t *)
0b4e3aa0 1844 *
0b4e3aa0
A		 1845 */
1846 .align 5
55e303ae 1847 .globl EXT(ppc_usimple_unlock_rwmb)
0b4e3aa0 1848
55e303ae 1849LEXT(ppc_usimple_unlock_rwmb)
0b4e3aa0 1850
55e303ae
A		 1851 li r0,0
1852 sync
91447636 1853 stw r0, SLOCK_ILK(r3)
0b4e3aa0 1854
55e303ae 1855 b epStart ; Go enable preemption...
0b4e3aa0 1856
91447636
A		 1857/*
1858 * void lck_rw_lock_exclusive(lck_rw_t*)
1859 *
1860 */
1861 .align 5
1862 .globl EXT(lck_rw_lock_exclusive)
1863LEXT(lck_rw_lock_exclusive)
1864#if !MACH_LDEBUG
1865 .globl EXT(lock_write)
1866LEXT(lock_write)
1867#endif
0c530ab8
A		 1868 lis r7,0xFFFF
1869 ori r7,r7,(WANT_EXCL|WANT_UPGRADE|ILK_LOCKED)
91447636 1870rwleloop: lwarx r5,RW_DATA,r3 ; Grab the lock value
0c530ab8 1871 and. r8,r5,r7 ; Can we have it?
91447636
A		 1872 ori r6,r5,WANT_EXCL ; Mark Exclusive
1873 bne-- rwlespin ; Branch if cannot be held
1874 stwcx. r6,RW_DATA,r3 ; Update lock word
1875 bne-- rwleloop
1876 .globl EXT(rwlePatch_isync)
1877LEXT(rwlePatch_isync)
1878 isync
1879 blr
1880rwlespin:
1881 li r4,lgKillResv ; Killing field
1882 stwcx. r4,0,r4 ; Kill it
1883 cmpli cr0,r5,RW_IND ; Is it a lock indirect
1884 bne-- rwlespin1 ; No, go handle contention
1885 mr r4,r3 ; pass lock pointer
1886 lwz r3,RW_PTR(r3) ; load lock ext pointer
1887 b EXT(lck_rw_lock_exclusive_ext)
1888rwlespin1:
1889 b EXT(lck_rw_lock_exclusive_gen)
1890
1891/*
1892 * void lck_rw_lock_shared(lck_rw_t*)
1893 *
1894 */
1895 .align 5
1896 .globl EXT(lck_rw_lock_shared)
1897LEXT(lck_rw_lock_shared)
1898#if !MACH_LDEBUG
1899 .globl EXT(lock_read)
1900LEXT(lock_read)
1901#endif
1902rwlsloop: lwarx r5,RW_DATA,r3 ; Grab the lock value
1903 andi. r7,r5,WANT_EXCL|WANT_UPGRADE|ILK_LOCKED ; Can we have it?
0c530ab8
A		 1904 bne-- rwlsopt ; Branch if cannot be held
1905rwlsloopres:
91447636 1906 addis r6,r5,1 ; Increment read cnt
91447636
A		 1907 stwcx. r6,RW_DATA,r3 ; Update lock word
1908 bne-- rwlsloop
1909 .globl EXT(rwlsPatch_isync)
1910LEXT(rwlsPatch_isync)
1911 isync
1912 blr
0c530ab8
A		 1913rwlsopt:
1914 andi. r7,r5,PRIV_EXCL|ILK_LOCKED ; Can we have it?
1915 bne-- rwlsspin ; Branch if cannot be held
1916 lis r7,0xFFFF ; Get read cnt mask
1917 and. r8,r5,r7 ; Is it shared
1918 bne rwlsloopres ; Branch if can be held
91447636
A		 1919rwlsspin:
1920 li r4,lgKillResv ; Killing field
1921 stwcx. r4,0,r4 ; Kill it
1922 cmpli cr0,r5,RW_IND ; Is it a lock indirect
1923 bne-- rwlsspin1 ; No, go handle contention
1924 mr r4,r3 ; pass lock pointer
1925 lwz r3,RW_PTR(r3) ; load lock ext pointer
1926 b EXT(lck_rw_lock_shared_ext)
1927rwlsspin1:
1928 b EXT(lck_rw_lock_shared_gen)
1929
1930/*
1931 * boolean_t lck_rw_lock_shared_to_exclusive(lck_rw_t*)
1932 *
1933 */
1934 .align 5
1935 .globl EXT(lck_rw_lock_shared_to_exclusive)
1936LEXT(lck_rw_lock_shared_to_exclusive)
1937#if !MACH_LDEBUG
1938 .globl EXT(lock_read_to_write)
1939LEXT(lock_read_to_write)
1940#endif
1941rwlseloop: lwarx r5,RW_DATA,r3 ; Grab the lock value
1942 addis r6,r5,0xFFFF ; Decrement read cnt
1943 lis r8,0xFFFF ; Get read count mask
1944 ori r8,r8,WANT_UPGRADE|ILK_LOCKED ; Include Interlock and upgrade flags
1945 and. r7,r6,r8 ; Can we have it?
1946 ori r9,r6,WANT_UPGRADE ; Mark Exclusive
1947 bne-- rwlsespin ; Branch if cannot be held
1948 stwcx. r9,RW_DATA,r3 ; Update lock word
1949 bne-- rwlseloop
1950 .globl EXT(rwlsePatch_isync)
1951LEXT(rwlsePatch_isync)
1952 isync
2d21ac55 1953 li r3,1 ; Succeed, return TRUE...
91447636
A		 1954 blr
1955rwlsespin:
1956 li r4,lgKillResv ; Killing field
1957 stwcx. r4,0,r4 ; Kill it
1958 cmpli cr0,r5,RW_IND ; Is it a lock indirect
1959 bne-- rwlsespin1 ; No, go handle contention
1960 mr r4,r3 ; pass lock pointer
1961 lwz r3,RW_PTR(r3) ; load lock ext pointer
1962 b EXT(lck_rw_lock_shared_to_exclusive_ext)
1963rwlsespin1:
1964 b EXT(lck_rw_lock_shared_to_exclusive_gen)
1965
1966
1967
1968/*
1969 * void lck_rw_lock_exclusive_to_shared(lck_rw_t*)
1970 *
1971 */
1972 .align 5
1973 .globl EXT(lck_rw_lock_exclusive_to_shared)
1974LEXT(lck_rw_lock_exclusive_to_shared)
1975#if !MACH_LDEBUG
1976 .globl EXT(lock_write_to_read)
1977LEXT(lock_write_to_read)
1978#endif
1979 .globl EXT(rwlesPatch_isync)
1980LEXT(rwlesPatch_isync)
1981 isync
1982 .globl EXT(rwlesPatch_eieio)
1983LEXT(rwlesPatch_eieio)
1984 eieio
1985rwlesloop: lwarx r5,RW_DATA,r3 ; Grab the lock value
1986 andi. r7,r5,ILK_LOCKED ; Test interlock flag
1987 bne-- rwlesspin ; Branch if interlocked
1988 lis r6,1 ; Get 1 for read count
1989 andi. r10,r5,WANT_UPGRADE ; Is it held with upgrade
1990 li r9,WANT_UPGRADE|WAIT_FLAG ; Get upgrade and wait flags mask
1991 bne rwlesexcl1 ; Skip if held with upgrade
1992 li r9,WANT_EXCL|WAIT_FLAG ; Get exclusive and wait flags mask
1993rwlesexcl1:
1994 andc r7,r5,r9 ; Marked free
1995 rlwimi r6,r7,0,16,31 ; Set shared cnt to one
1996 stwcx. r6,RW_DATA,r3 ; Update lock word
1997 bne-- rwlesloop
1998 andi. r7,r5,WAIT_FLAG ; Test wait flag
1999 beqlr++ ; Return of no waiters
2000 addi r3,r3,RW_EVENT ; Get lock event address
2001 b EXT(thread_wakeup) ; wakeup waiters
2002rwlesspin:
2003 li r4,lgKillResv ; Killing field
2004 stwcx. r4,0,r4 ; Kill it
2005 cmpli cr0,r5,RW_IND ; Is it a lock indirect
2006 bne-- rwlesspin1 ; No, go handle contention
2007 mr r4,r3 ; pass lock pointer
2008 lwz r3,RW_PTR(r3) ; load lock ext pointer
2009 b EXT(lck_rw_lock_exclusive_to_shared_ext)
2010rwlesspin1:
2011 b EXT(lck_rw_lock_exclusive_to_shared_gen)
2012
2013
2014
2015/*
2016 * boolean_t lck_rw_try_lock_exclusive(lck_rw_t*)
2017 *
2018 */
2019 .align 5
2020 .globl EXT(lck_rw_try_lock_exclusive)
2021LEXT(lck_rw_try_lock_exclusive)
2022 lis r10,0xFFFF ; Load read count mask
2023 ori r10,r10,WANT_EXCL|WANT_UPGRADE ; Include exclusive and upgrade flags
2024rwtleloop: lwarx r5,RW_DATA,r3 ; Grab the lock value
2025 andi. r7,r5,ILK_LOCKED ; Test interlock flag
2026 bne-- rwtlespin ; Branch if interlocked
2027 and. r7,r5,r10 ; Can we have it
2028 ori r6,r5,WANT_EXCL ; Mark Exclusive
2029 bne-- rwtlefail ;
2030 stwcx. r6,RW_DATA,r3 ; Update lock word
2031 bne-- rwtleloop
2032 .globl EXT(rwtlePatch_isync)
2033LEXT(rwtlePatch_isync)
2034 isync
2035 li r3,1 ; Return TRUE
2036 blr
2037rwtlefail:
2038 li r4,lgKillResv ; Killing field
2039 stwcx. r4,0,r4 ; Kill it
2040 li r3,0 ; Return FALSE
2041 blr
2042rwtlespin:
2043 li r4,lgKillResv ; Killing field
2044 stwcx. r4,0,r4 ; Kill it
2045 cmpli cr0,r5,RW_IND ; Is it a lock indirect
2046 bne-- rwtlespin1 ; No, go handle contention
2047 mr r4,r3 ; pass lock pointer
2048 lwz r3,RW_PTR(r3) ; load lock ext pointer
2049 b EXT(lck_rw_try_lock_exclusive_ext)
2050rwtlespin1:
2051 b EXT(lck_rw_try_lock_exclusive_gen)
2052
2053
2054/*
2055 * boolean_t lck_rw_try_lock_shared(lck_rw_t*)
2056 *
2057 */
2058 .align 5
2059 .globl EXT(lck_rw_try_lock_shared)
2060LEXT(lck_rw_try_lock_shared)
2061rwtlsloop: lwarx r5,RW_DATA,r3 ; Grab the lock value
2062 andi. r7,r5,ILK_LOCKED ; Test interlock flag
2063 bne-- rwtlsspin ; Branch if interlocked
2064 andi. r7,r5,WANT_EXCL|WANT_UPGRADE ; So, can we have it?
0c530ab8
A		 2065 bne-- rwtlsopt ; Branch if held exclusive
2066rwtlsloopres:
91447636 2067 addis r6,r5,1 ; Increment read cnt
91447636
A		 2068 stwcx. r6,RW_DATA,r3 ; Update lock word
2069 bne-- rwtlsloop
2070 .globl EXT(rwtlsPatch_isync)
2071LEXT(rwtlsPatch_isync)
2072 isync
2073 li r3,1 ; Return TRUE
2074 blr
0c530ab8
A		 2075rwtlsopt:
2076 andi. r7,r5,PRIV_EXCL ; Can we have it?
2077 bne-- rwtlsfail ; Branch if cannot be held
2078 lis r7,0xFFFF ; Get read cnt mask
2079 and. r8,r5,r7 ; Is it shared
2080 bne rwtlsloopres ; Branch if can be held
91447636
A		 2081rwtlsfail:
2082 li r3,0 ; Return FALSE
2083 blr
2084rwtlsspin:
2085 li r4,lgKillResv ; Killing field
2086 stwcx. r4,0,r4 ; Kill it
2087 cmpli cr0,r5,RW_IND ; Is it a lock indirect
2088 bne-- rwtlsspin1 ; No, go handle contention
2089 mr r4,r3 ; pass lock pointer
2090 lwz r3,RW_PTR(r3) ; load lock ext pointer
2091 b EXT(lck_rw_try_lock_shared_ext)
2092rwtlsspin1:
2093 b EXT(lck_rw_try_lock_shared_gen)
55e303ae 2094
55e303ae 2095
9bccf70c 2096
91447636
A		 2097/*
2098 * lck_rw_type_t lck_rw_done(lck_rw_t*)
2099 *
2100 */
2101 .align 5
2102 .globl EXT(lck_rw_done)
2103LEXT(lck_rw_done)
2104#if !MACH_LDEBUG
2105 .globl EXT(lock_done)
2106LEXT(lock_done)
2107#endif
2108 .globl EXT(rwldPatch_isync)
2109LEXT(rwldPatch_isync)
2110 isync
2111 .globl EXT(rwldPatch_eieio)
2112LEXT(rwldPatch_eieio)
2113 eieio
2114 li r10,WAIT_FLAG ; Get wait flag
2115 lis r7,0xFFFF ; Get read cnt mask
2116 mr r12,r3 ; Save lock addr
2117rwldloop: lwarx r5,RW_DATA,r3 ; Grab the lock value
2118 andi. r8,r5,ILK_LOCKED ; Test interlock flag
2119 bne-- rwldspin ; Branch if interlocked
2120 and. r8,r5,r7 ; Is it shared
2121 cmpi cr1,r8,0 ; Is it shared
2122 beq cr1,rwldexcl ; No, check exclusive
2123 li r11,RW_SHARED ; Set return value
2124 addis r6,r5,0xFFFF ; Decrement read count
2125 and. r8,r6,r7 ; Is it still shared
2126 li r8,0 ; Assume no wakeup
2127 bne rwldshared1 ; Skip if still held shared
2128 and r8,r6,r10 ; Extract wait flag
2129 andc r6,r6,r10 ; Clear wait flag
2130rwldshared1:
2131 b rwldstore
2132rwldexcl:
2133 li r11,RW_EXCL ; Set return value
2134 li r9,WANT_UPGRADE ; Get upgrade flag
2135 and. r6,r5,r9 ; Is it held with upgrade
2136 li r9,WANT_UPGRADE|WAIT_FLAG ; Mask upgrade abd wait flags
2137 bne rwldexcl1 ; Skip if held with upgrade
2138 li r9,WANT_EXCL|WAIT_FLAG ; Mask exclusive and wait flags
2139rwldexcl1:
2140 andc r6,r5,r9 ; Marked free
2141 and r8,r5,r10 ; Null if no waiter
2142rwldstore:
2143 stwcx. r6,RW_DATA,r3 ; Update lock word
2144 bne-- rwldloop
2145 mr. r8,r8 ; wakeup needed?
2146 mr r3,r11 ; Return lock held type
2147 beqlr++
2148 mr r3,r12 ; Restore lock address
2149 PROLOG(0)
2150 addi r3,r3,RW_EVENT ; Get lock event address
2151 bl EXT(thread_wakeup) ; wakeup threads
2152 lwz r2,(FM_ALIGN(0)+FM_SIZE+FM_CR_SAVE)(r1)
2153 mtcr r2
2154 EPILOG
2155 li r3,RW_SHARED ; Assume lock type shared
2156 bne cr1,rwldret ; Branch if was held exclusive
2157 li r3,RW_EXCL ; Return lock type exclusive
2158rwldret:
2159 blr
2160rwldspin:
2161 li r4,lgKillResv ; Killing field
2162 stwcx. r4,0,r4 ; Kill it
2163 cmpli cr0,r5,RW_IND ; Is it a lock indirect
2164 bne-- rwldspin1 ; No, go handle contention
2165 mr r4,r3 ; pass lock pointer
2166 lwz r3,RW_PTR(r3) ; load lock ext pointer
2167 b EXT(lck_rw_done_ext)
2168rwldspin1:
2169 b EXT(lck_rw_done_gen)
2170
2171/*
2172 * void lck_rw_ilk_lock(lck_rw_t *lock)
2173 */
2174 .globl EXT(lck_rw_ilk_lock)
2175LEXT(lck_rw_ilk_lock)
2176 crclr hwtimeout ; no timeout option
2177 li r4,0 ; request default timeout value
2178 li r12,ILK_LOCKED ; Load bit mask
2179 b lckcomm ; Join on up...
2180
2181/*
2182 * void lck_rw_ilk_unlock(lck_rw_t *lock)
2183 */
2184 .globl EXT(lck_rw_ilk_unlock)
2185LEXT(lck_rw_ilk_unlock)
2186 li r4,1
2187 b EXT(hw_unlock_bit)
mirror of XNU