]>
Commit | Line | Data |
---|---|---|
0c530ab8 | 1 | /* |
7ddcb079 | 2 | * Copyright (c) 2004-2011 Apple Inc. All rights reserved. |
0c530ab8 | 3 | * |
2d21ac55 | 4 | * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ |
0c530ab8 | 5 | * |
2d21ac55 A |
6 | * This file contains Original Code and/or Modifications of Original Code |
7 | * as defined in and that are subject to the Apple Public Source License | |
8 | * Version 2.0 (the 'License'). You may not use this file except in | |
9 | * compliance with the License. The rights granted to you under the License | |
10 | * may not be used to create, or enable the creation or redistribution of, | |
11 | * unlawful or unlicensed copies of an Apple operating system, or to | |
12 | * circumvent, violate, or enable the circumvention or violation of, any | |
13 | * terms of an Apple operating system software license agreement. | |
0c530ab8 | 14 | * |
2d21ac55 A |
15 | * Please obtain a copy of the License at |
16 | * http://www.opensource.apple.com/apsl/ and read it before using this file. | |
17 | * | |
18 | * The Original Code and all software distributed under the License are | |
19 | * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER | |
0c530ab8 A |
20 | * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, |
21 | * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, | |
2d21ac55 A |
22 | * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. |
23 | * Please see the License for the specific language governing rights and | |
24 | * limitations under the License. | |
0c530ab8 | 25 | * |
2d21ac55 | 26 | * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ |
0c530ab8 A |
27 | */ |
28 | ||
29 | /* | |
30 | * CPU-specific power management support. | |
31 | * | |
32 | * Implements the "wrappers" to the KEXT. | |
33 | */ | |
0c530ab8 | 34 | #include <i386/asm.h> |
b0d623f7 | 35 | #include <i386/machine_cpu.h> |
0c530ab8 | 36 | #include <i386/mp.h> |
b0d623f7 | 37 | #include <i386/machine_routines.h> |
0c530ab8 | 38 | #include <i386/proc_reg.h> |
b0d623f7 A |
39 | #include <i386/pmap.h> |
40 | #include <i386/misc_protos.h> | |
41 | #include <kern/machine.h> | |
0c530ab8 A |
42 | #include <kern/pms.h> |
43 | #include <kern/processor.h> | |
7e4a7d39 | 44 | #include <kern/etimer.h> |
2d21ac55 | 45 | #include <i386/cpu_threads.h> |
0c530ab8 A |
46 | #include <i386/pmCPU.h> |
47 | #include <i386/cpuid.h> | |
6d2010ae | 48 | #include <i386/rtclock_protos.h> |
593a1d5f | 49 | #include <kern/sched_prim.h> |
b0d623f7 | 50 | #include <i386/lapic.h> |
6d2010ae A |
51 | #include <i386/pal_routines.h> |
52 | ||
53 | #include <sys/kdebug.h> | |
593a1d5f | 54 | |
0c530ab8 A |
55 | extern int disableConsoleOutput; |
56 | ||
7ddcb079 | 57 | #define DELAY_UNSET 0xFFFFFFFFFFFFFFFFULL |
0c530ab8 A |
58 | |
59 | /* | |
60 | * The following is set when the KEXT loads and initializes. | |
61 | */ | |
6d2010ae | 62 | pmDispatch_t *pmDispatch = NULL; |
0c530ab8 | 63 | |
7ddcb079 A |
64 | static uint32_t pmInitDone = 0; |
65 | static boolean_t earlyTopology = FALSE; | |
66 | static uint64_t earlyMaxBusDelay = DELAY_UNSET; | |
67 | static uint64_t earlyMaxIntDelay = DELAY_UNSET; | |
2d21ac55 | 68 | |
0c530ab8 A |
69 | /* |
70 | * Initialize the Cstate change code. | |
71 | */ | |
72 | void | |
73 | power_management_init(void) | |
74 | { | |
0c530ab8 A |
75 | if (pmDispatch != NULL && pmDispatch->cstateInit != NULL) |
76 | (*pmDispatch->cstateInit)(); | |
77 | } | |
78 | ||
0c530ab8 | 79 | /* |
593a1d5f A |
80 | * Called when the CPU is idle. It calls into the power management kext |
81 | * to determine the best way to idle the CPU. | |
0c530ab8 | 82 | */ |
593a1d5f A |
83 | void |
84 | machine_idle(void) | |
0c530ab8 | 85 | { |
593a1d5f | 86 | cpu_data_t *my_cpu = current_cpu_datap(); |
0c530ab8 | 87 | |
593a1d5f A |
88 | if (my_cpu == NULL) |
89 | goto out; | |
0c530ab8 | 90 | |
593a1d5f A |
91 | my_cpu->lcpu.state = LCPU_IDLE; |
92 | DBGLOG(cpu_handle, cpu_number(), MP_IDLE); | |
93 | MARK_CPU_IDLE(cpu_number()); | |
0c530ab8 | 94 | |
7ddcb079 A |
95 | if (pmInitDone) { |
96 | /* | |
97 | * Handle case where ml_set_maxbusdelay() or ml_set_maxintdelay() | |
98 | * were called prior to the CPU PM kext being registered. We do | |
316670eb A |
99 | * this here since we know at this point the values will be first |
100 | * used since idle is where the decisions using these values is made. | |
7ddcb079 A |
101 | */ |
102 | if (earlyMaxBusDelay != DELAY_UNSET) | |
103 | ml_set_maxbusdelay((uint32_t)(earlyMaxBusDelay & 0xFFFFFFFF)); | |
104 | ||
105 | if (earlyMaxIntDelay != DELAY_UNSET) | |
106 | ml_set_maxintdelay(earlyMaxIntDelay); | |
107 | } | |
108 | ||
593a1d5f A |
109 | if (pmInitDone |
110 | && pmDispatch != NULL | |
b7266188 A |
111 | && pmDispatch->MachineIdle != NULL) |
112 | (*pmDispatch->MachineIdle)(0x7FFFFFFFFFFFFFFFULL); | |
593a1d5f A |
113 | else { |
114 | /* | |
115 | * If no power management, re-enable interrupts and halt. | |
116 | * This will keep the CPU from spinning through the scheduler | |
117 | * and will allow at least some minimal power savings (but it | |
118 | * cause problems in some MP configurations w.r.t. the APIC | |
119 | * stopping during a GV3 transition). | |
120 | */ | |
6d2010ae A |
121 | pal_hlt(); |
122 | ||
123 | /* Once woken, re-disable interrupts. */ | |
124 | pal_cli(); | |
0c530ab8 | 125 | } |
0c530ab8 | 126 | |
593a1d5f A |
127 | /* |
128 | * Mark the CPU as running again. | |
129 | */ | |
130 | MARK_CPU_ACTIVE(cpu_number()); | |
131 | DBGLOG(cpu_handle, cpu_number(), MP_UNIDLE); | |
132 | my_cpu->lcpu.state = LCPU_RUN; | |
133 | ||
134 | /* | |
135 | * Re-enable interrupts. | |
136 | */ | |
137 | out: | |
6d2010ae | 138 | pal_sti(); |
2d21ac55 A |
139 | } |
140 | ||
141 | /* | |
142 | * Called when the CPU is to be halted. It will choose the best C-State | |
143 | * to be in. | |
144 | */ | |
145 | void | |
146 | pmCPUHalt(uint32_t reason) | |
147 | { | |
593a1d5f | 148 | cpu_data_t *cpup = current_cpu_datap(); |
2d21ac55 A |
149 | |
150 | switch (reason) { | |
151 | case PM_HALT_DEBUG: | |
593a1d5f | 152 | cpup->lcpu.state = LCPU_PAUSE; |
6d2010ae | 153 | pal_stop_cpu(FALSE); |
2d21ac55 A |
154 | break; |
155 | ||
156 | case PM_HALT_PANIC: | |
593a1d5f | 157 | cpup->lcpu.state = LCPU_PAUSE; |
6d2010ae | 158 | pal_stop_cpu(TRUE); |
2d21ac55 A |
159 | break; |
160 | ||
161 | case PM_HALT_NORMAL: | |
162 | default: | |
6d2010ae | 163 | pal_cli(); |
2d21ac55 | 164 | |
6d2010ae | 165 | if (pmInitDone |
2d21ac55 A |
166 | && pmDispatch != NULL |
167 | && pmDispatch->pmCPUHalt != NULL) { | |
593a1d5f A |
168 | /* |
169 | * Halt the CPU (and put it in a low power state. | |
170 | */ | |
2d21ac55 | 171 | (*pmDispatch->pmCPUHalt)(); |
2d21ac55 | 172 | |
593a1d5f A |
173 | /* |
174 | * We've exited halt, so get the the CPU schedulable again. | |
175 | */ | |
176 | i386_init_slave_fast(); | |
177 | ||
178 | panic("init_slave_fast returned"); | |
6d2010ae A |
179 | } else |
180 | { | |
2d21ac55 A |
181 | /* |
182 | * If no power managment and a processor is taken off-line, | |
183 | * then invalidate the cache and halt it (it will not be able | |
184 | * to be brought back on-line without resetting the CPU). | |
185 | */ | |
186 | __asm__ volatile ("wbinvd"); | |
593a1d5f | 187 | cpup->lcpu.state = LCPU_HALT; |
6d2010ae | 188 | pal_stop_cpu(FALSE); |
593a1d5f A |
189 | |
190 | panic("back from Halt"); | |
2d21ac55 | 191 | } |
6d2010ae | 192 | |
2d21ac55 | 193 | break; |
0c530ab8 A |
194 | } |
195 | } | |
196 | ||
2d21ac55 | 197 | void |
593a1d5f | 198 | pmMarkAllCPUsOff(void) |
2d21ac55 | 199 | { |
593a1d5f A |
200 | if (pmInitDone |
201 | && pmDispatch != NULL | |
202 | && pmDispatch->markAllCPUsOff != NULL) | |
203 | (*pmDispatch->markAllCPUsOff)(); | |
2d21ac55 A |
204 | } |
205 | ||
206 | static void | |
207 | pmInitComplete(void) | |
208 | { | |
7ddcb079 A |
209 | if (earlyTopology |
210 | && pmDispatch != NULL | |
211 | && pmDispatch->pmCPUStateInit != NULL) { | |
060df5ea | 212 | (*pmDispatch->pmCPUStateInit)(); |
7ddcb079 A |
213 | earlyTopology = FALSE; |
214 | } | |
060df5ea | 215 | |
2d21ac55 A |
216 | pmInitDone = 1; |
217 | } | |
218 | ||
219 | static x86_lcpu_t * | |
220 | pmGetLogicalCPU(int cpu) | |
0c530ab8 | 221 | { |
2d21ac55 A |
222 | return(cpu_to_lcpu(cpu)); |
223 | } | |
224 | ||
225 | static x86_lcpu_t * | |
226 | pmGetMyLogicalCPU(void) | |
227 | { | |
228 | cpu_data_t *cpup = current_cpu_datap(); | |
0c530ab8 | 229 | |
2d21ac55 A |
230 | return(&cpup->lcpu); |
231 | } | |
232 | ||
233 | static x86_core_t * | |
234 | pmGetCore(int cpu) | |
235 | { | |
236 | return(cpu_to_core(cpu)); | |
0c530ab8 A |
237 | } |
238 | ||
2d21ac55 A |
239 | static x86_core_t * |
240 | pmGetMyCore(void) | |
0c530ab8 | 241 | { |
2d21ac55 | 242 | cpu_data_t *cpup = current_cpu_datap(); |
0c530ab8 | 243 | |
2d21ac55 | 244 | return(cpup->lcpu.core); |
0c530ab8 A |
245 | } |
246 | ||
593a1d5f A |
247 | static x86_die_t * |
248 | pmGetDie(int cpu) | |
249 | { | |
250 | return(cpu_to_die(cpu)); | |
251 | } | |
252 | ||
253 | static x86_die_t * | |
254 | pmGetMyDie(void) | |
255 | { | |
256 | cpu_data_t *cpup = current_cpu_datap(); | |
257 | ||
258 | return(cpup->lcpu.die); | |
259 | } | |
260 | ||
2d21ac55 A |
261 | static x86_pkg_t * |
262 | pmGetPackage(int cpu) | |
0c530ab8 | 263 | { |
2d21ac55 A |
264 | return(cpu_to_package(cpu)); |
265 | } | |
266 | ||
267 | static x86_pkg_t * | |
268 | pmGetMyPackage(void) | |
269 | { | |
270 | cpu_data_t *cpup = current_cpu_datap(); | |
271 | ||
593a1d5f | 272 | return(cpup->lcpu.package); |
2d21ac55 A |
273 | } |
274 | ||
275 | static void | |
276 | pmLockCPUTopology(int lock) | |
277 | { | |
278 | if (lock) { | |
279 | simple_lock(&x86_topo_lock); | |
280 | } else { | |
281 | simple_unlock(&x86_topo_lock); | |
282 | } | |
0c530ab8 A |
283 | } |
284 | ||
285 | /* | |
2d21ac55 A |
286 | * Called to get the next deadline that has been set by the |
287 | * power management code. | |
6d2010ae A |
288 | * Note: a return of 0 from AICPM and this routine signifies |
289 | * that no deadline is set. | |
0c530ab8 | 290 | */ |
2d21ac55 A |
291 | uint64_t |
292 | pmCPUGetDeadline(cpu_data_t *cpu) | |
293 | { | |
060df5ea | 294 | uint64_t deadline = 0; |
0c530ab8 | 295 | |
7ddcb079 | 296 | if (pmInitDone |
2d21ac55 A |
297 | && pmDispatch != NULL |
298 | && pmDispatch->GetDeadline != NULL) | |
299 | deadline = (*pmDispatch->GetDeadline)(&cpu->lcpu); | |
300 | ||
301 | return(deadline); | |
0c530ab8 A |
302 | } |
303 | ||
304 | /* | |
2d21ac55 A |
305 | * Called to determine if the supplied deadline or the power management |
306 | * deadline is sooner. Returns which ever one is first. | |
0c530ab8 | 307 | */ |
2d21ac55 A |
308 | uint64_t |
309 | pmCPUSetDeadline(cpu_data_t *cpu, uint64_t deadline) | |
0c530ab8 | 310 | { |
b0d623f7 | 311 | if (pmInitDone |
2d21ac55 A |
312 | && pmDispatch != NULL |
313 | && pmDispatch->SetDeadline != NULL) | |
314 | deadline = (*pmDispatch->SetDeadline)(&cpu->lcpu, deadline); | |
315 | ||
316 | return(deadline); | |
0c530ab8 A |
317 | } |
318 | ||
0c530ab8 | 319 | /* |
2d21ac55 | 320 | * Called when a power management deadline expires. |
0c530ab8 A |
321 | */ |
322 | void | |
2d21ac55 | 323 | pmCPUDeadline(cpu_data_t *cpu) |
0c530ab8 | 324 | { |
2d21ac55 A |
325 | if (pmInitDone |
326 | && pmDispatch != NULL | |
327 | && pmDispatch->Deadline != NULL) | |
328 | (*pmDispatch->Deadline)(&cpu->lcpu); | |
329 | } | |
330 | ||
331 | /* | |
332 | * Called to get a CPU out of idle. | |
333 | */ | |
334 | boolean_t | |
335 | pmCPUExitIdle(cpu_data_t *cpu) | |
336 | { | |
337 | boolean_t do_ipi; | |
338 | ||
339 | if (pmInitDone | |
340 | && pmDispatch != NULL | |
341 | && pmDispatch->exitIdle != NULL) | |
342 | do_ipi = (*pmDispatch->exitIdle)(&cpu->lcpu); | |
343 | else | |
344 | do_ipi = TRUE; | |
345 | ||
346 | return(do_ipi); | |
0c530ab8 A |
347 | } |
348 | ||
593a1d5f A |
349 | kern_return_t |
350 | pmCPUExitHalt(int cpu) | |
351 | { | |
352 | kern_return_t rc = KERN_INVALID_ARGUMENT; | |
353 | ||
354 | if (pmInitDone | |
355 | && pmDispatch != NULL | |
356 | && pmDispatch->exitHalt != NULL) | |
357 | rc = pmDispatch->exitHalt(cpu_to_lcpu(cpu)); | |
358 | ||
359 | return(rc); | |
360 | } | |
361 | ||
e2fac8b1 A |
362 | kern_return_t |
363 | pmCPUExitHaltToOff(int cpu) | |
364 | { | |
365 | kern_return_t rc = KERN_INVALID_ARGUMENT; | |
366 | ||
367 | if (pmInitDone | |
368 | && pmDispatch != NULL | |
369 | && pmDispatch->exitHaltToOff != NULL) | |
370 | rc = pmDispatch->exitHaltToOff(cpu_to_lcpu(cpu)); | |
371 | ||
372 | return(rc); | |
373 | } | |
374 | ||
2d21ac55 | 375 | /* |
593a1d5f | 376 | * Called to initialize the power management structures for the CPUs. |
2d21ac55 | 377 | */ |
0c530ab8 | 378 | void |
593a1d5f | 379 | pmCPUStateInit(void) |
0c530ab8 | 380 | { |
593a1d5f A |
381 | if (pmDispatch != NULL && pmDispatch->pmCPUStateInit != NULL) |
382 | (*pmDispatch->pmCPUStateInit)(); | |
060df5ea A |
383 | else |
384 | earlyTopology = TRUE; | |
0c530ab8 A |
385 | } |
386 | ||
2d21ac55 | 387 | /* |
593a1d5f | 388 | * Called when a CPU is being restarted after being powered off (as in S3). |
2d21ac55 | 389 | */ |
0c530ab8 | 390 | void |
593a1d5f | 391 | pmCPUMarkRunning(cpu_data_t *cpu) |
0c530ab8 | 392 | { |
593a1d5f A |
393 | cpu_data_t *cpup = current_cpu_datap(); |
394 | ||
2d21ac55 A |
395 | if (pmInitDone |
396 | && pmDispatch != NULL | |
593a1d5f A |
397 | && pmDispatch->markCPURunning != NULL) |
398 | (*pmDispatch->markCPURunning)(&cpu->lcpu); | |
399 | else | |
400 | cpup->lcpu.state = LCPU_RUN; | |
2d21ac55 A |
401 | } |
402 | ||
403 | /* | |
404 | * Called to get/set CPU power management state. | |
405 | */ | |
406 | int | |
407 | pmCPUControl(uint32_t cmd, void *datap) | |
408 | { | |
409 | int rc = -1; | |
410 | ||
411 | if (pmDispatch != NULL | |
412 | && pmDispatch->pmCPUControl != NULL) | |
413 | rc = (*pmDispatch->pmCPUControl)(cmd, datap); | |
414 | ||
415 | return(rc); | |
0c530ab8 A |
416 | } |
417 | ||
593a1d5f A |
418 | /* |
419 | * Called to save the timer state used by power management prior | |
420 | * to "sleeping". | |
421 | */ | |
422 | void | |
423 | pmTimerSave(void) | |
424 | { | |
425 | if (pmDispatch != NULL | |
426 | && pmDispatch->pmTimerStateSave != NULL) | |
427 | (*pmDispatch->pmTimerStateSave)(); | |
428 | } | |
429 | ||
430 | /* | |
431 | * Called to restore the timer state used by power management after | |
432 | * waking from "sleep". | |
433 | */ | |
434 | void | |
435 | pmTimerRestore(void) | |
436 | { | |
437 | if (pmDispatch != NULL | |
438 | && pmDispatch->pmTimerStateRestore != NULL) | |
439 | (*pmDispatch->pmTimerStateRestore)(); | |
440 | } | |
441 | ||
2d21ac55 A |
442 | /* |
443 | * Set the worst-case time for the C4 to C2 transition. | |
444 | * No longer does anything. | |
445 | */ | |
0c530ab8 | 446 | void |
2d21ac55 | 447 | ml_set_maxsnoop(__unused uint32_t maxdelay) |
0c530ab8 | 448 | { |
0c530ab8 A |
449 | } |
450 | ||
2d21ac55 A |
451 | |
452 | /* | |
453 | * Get the worst-case time for the C4 to C2 transition. Returns nanoseconds. | |
454 | */ | |
455 | unsigned | |
456 | ml_get_maxsnoop(void) | |
457 | { | |
458 | uint64_t max_snoop = 0; | |
459 | ||
7ddcb079 A |
460 | if (pmInitDone |
461 | && pmDispatch != NULL | |
2d21ac55 A |
462 | && pmDispatch->getMaxSnoop != NULL) |
463 | max_snoop = pmDispatch->getMaxSnoop(); | |
464 | ||
465 | return((unsigned)(max_snoop & 0xffffffff)); | |
466 | } | |
467 | ||
468 | ||
469 | uint32_t | |
470 | ml_get_maxbusdelay(void) | |
471 | { | |
472 | uint64_t max_delay = 0; | |
473 | ||
7ddcb079 A |
474 | if (pmInitDone |
475 | && pmDispatch != NULL | |
2d21ac55 A |
476 | && pmDispatch->getMaxBusDelay != NULL) |
477 | max_delay = pmDispatch->getMaxBusDelay(); | |
478 | ||
479 | return((uint32_t)(max_delay & 0xffffffff)); | |
480 | } | |
481 | ||
482 | /* | |
483 | * Set the maximum delay time allowed for snoop on the bus. | |
484 | * | |
485 | * Note that this value will be compared to the amount of time that it takes | |
486 | * to transition from a non-snooping power state (C4) to a snooping state (C2). | |
487 | * If maxBusDelay is less than C4C2SnoopDelay, | |
488 | * we will not enter the lowest power state. | |
489 | */ | |
0c530ab8 | 490 | void |
2d21ac55 | 491 | ml_set_maxbusdelay(uint32_t mdelay) |
0c530ab8 | 492 | { |
2d21ac55 A |
493 | uint64_t maxdelay = mdelay; |
494 | ||
495 | if (pmDispatch != NULL | |
7ddcb079 A |
496 | && pmDispatch->setMaxBusDelay != NULL) { |
497 | earlyMaxBusDelay = DELAY_UNSET; | |
2d21ac55 | 498 | pmDispatch->setMaxBusDelay(maxdelay); |
7ddcb079 A |
499 | } else |
500 | earlyMaxBusDelay = maxdelay; | |
593a1d5f A |
501 | } |
502 | ||
503 | uint64_t | |
504 | ml_get_maxintdelay(void) | |
505 | { | |
506 | uint64_t max_delay = 0; | |
507 | ||
508 | if (pmDispatch != NULL | |
509 | && pmDispatch->getMaxIntDelay != NULL) | |
510 | max_delay = pmDispatch->getMaxIntDelay(); | |
511 | ||
512 | return(max_delay); | |
513 | } | |
514 | ||
515 | /* | |
516 | * Set the maximum delay allowed for an interrupt. | |
517 | */ | |
518 | void | |
519 | ml_set_maxintdelay(uint64_t mdelay) | |
520 | { | |
521 | if (pmDispatch != NULL | |
7ddcb079 A |
522 | && pmDispatch->setMaxIntDelay != NULL) { |
523 | earlyMaxIntDelay = DELAY_UNSET; | |
593a1d5f | 524 | pmDispatch->setMaxIntDelay(mdelay); |
7ddcb079 A |
525 | } else |
526 | earlyMaxIntDelay = mdelay; | |
0c530ab8 A |
527 | } |
528 | ||
6d2010ae A |
529 | boolean_t |
530 | ml_get_interrupt_prewake_applicable() | |
531 | { | |
532 | boolean_t applicable = FALSE; | |
533 | ||
534 | if (pmInitDone | |
535 | && pmDispatch != NULL | |
536 | && pmDispatch->pmInterruptPrewakeApplicable != NULL) | |
537 | applicable = pmDispatch->pmInterruptPrewakeApplicable(); | |
538 | ||
539 | return applicable; | |
540 | } | |
541 | ||
2d21ac55 A |
542 | /* |
543 | * Put a CPU into "safe" mode with respect to power. | |
544 | * | |
545 | * Some systems cannot operate at a continuous "normal" speed without | |
546 | * exceeding the thermal design. This is called per-CPU to place the | |
547 | * CPUs into a "safe" operating mode. | |
548 | */ | |
0c530ab8 | 549 | void |
2d21ac55 A |
550 | pmSafeMode(x86_lcpu_t *lcpu, uint32_t flags) |
551 | { | |
552 | if (pmDispatch != NULL | |
553 | && pmDispatch->pmCPUSafeMode != NULL) | |
554 | pmDispatch->pmCPUSafeMode(lcpu, flags); | |
555 | else { | |
556 | /* | |
557 | * Do something reasonable if the KEXT isn't present. | |
558 | * | |
559 | * We only look at the PAUSE and RESUME flags. The other flag(s) | |
560 | * will not make any sense without the KEXT, so just ignore them. | |
561 | * | |
593a1d5f A |
562 | * We set the CPU's state to indicate that it's halted. If this |
563 | * is the CPU we're currently running on, then spin until the | |
564 | * state becomes non-halted. | |
2d21ac55 A |
565 | */ |
566 | if (flags & PM_SAFE_FL_PAUSE) { | |
593a1d5f | 567 | lcpu->state = LCPU_PAUSE; |
2d21ac55 | 568 | if (lcpu == x86_lcpu()) { |
593a1d5f | 569 | while (lcpu->state == LCPU_PAUSE) |
2d21ac55 A |
570 | cpu_pause(); |
571 | } | |
572 | } | |
573 | ||
574 | /* | |
575 | * Clear the halted flag for the specified CPU, that will | |
576 | * get it out of it's spin loop. | |
577 | */ | |
578 | if (flags & PM_SAFE_FL_RESUME) { | |
593a1d5f | 579 | lcpu->state = LCPU_RUN; |
2d21ac55 A |
580 | } |
581 | } | |
582 | } | |
583 | ||
c910b4d9 A |
584 | static uint32_t saved_run_count = 0; |
585 | ||
586 | void | |
587 | machine_run_count(uint32_t count) | |
588 | { | |
589 | if (pmDispatch != NULL | |
590 | && pmDispatch->pmSetRunCount != NULL) | |
591 | pmDispatch->pmSetRunCount(count); | |
592 | else | |
593 | saved_run_count = count; | |
594 | } | |
595 | ||
596 | boolean_t | |
b7266188 | 597 | machine_processor_is_inactive(processor_t processor) |
c910b4d9 | 598 | { |
b7266188 A |
599 | int cpu = processor->cpu_id; |
600 | ||
c910b4d9 A |
601 | if (pmDispatch != NULL |
602 | && pmDispatch->pmIsCPUUnAvailable != NULL) | |
603 | return(pmDispatch->pmIsCPUUnAvailable(cpu_to_lcpu(cpu))); | |
604 | else | |
605 | return(FALSE); | |
606 | } | |
607 | ||
b7266188 A |
608 | processor_t |
609 | machine_choose_processor(processor_set_t pset, | |
610 | processor_t preferred) | |
611 | { | |
612 | int startCPU; | |
613 | int endCPU; | |
614 | int preferredCPU; | |
615 | int chosenCPU; | |
616 | ||
617 | if (!pmInitDone) | |
618 | return(preferred); | |
619 | ||
620 | if (pset == NULL) { | |
621 | startCPU = -1; | |
622 | endCPU = -1; | |
623 | } else { | |
624 | startCPU = pset->cpu_set_low; | |
625 | endCPU = pset->cpu_set_hi; | |
626 | } | |
627 | ||
628 | if (preferred == NULL) | |
629 | preferredCPU = -1; | |
630 | else | |
631 | preferredCPU = preferred->cpu_id; | |
632 | ||
633 | if (pmDispatch != NULL | |
634 | && pmDispatch->pmChooseCPU != NULL) { | |
635 | chosenCPU = pmDispatch->pmChooseCPU(startCPU, endCPU, preferredCPU); | |
636 | ||
637 | if (chosenCPU == -1) | |
638 | return(NULL); | |
639 | return(cpu_datap(chosenCPU)->cpu_processor); | |
640 | } | |
641 | ||
642 | return(preferred); | |
643 | } | |
644 | ||
060df5ea | 645 | static int |
6d2010ae | 646 | pmThreadGetUrgency(uint64_t *rt_period, uint64_t *rt_deadline) |
060df5ea A |
647 | { |
648 | ||
6d2010ae | 649 | return(thread_get_urgency(rt_period, rt_deadline)); |
060df5ea A |
650 | } |
651 | ||
6d2010ae A |
652 | #if DEBUG |
653 | uint32_t urgency_stats[64][THREAD_URGENCY_MAX]; | |
654 | #endif | |
655 | ||
656 | #define URGENCY_NOTIFICATION_ASSERT_NS (5 * 1000 * 1000) | |
657 | uint64_t urgency_notification_assert_abstime_threshold, urgency_notification_max_recorded; | |
658 | ||
060df5ea A |
659 | void |
660 | thread_tell_urgency(int urgency, | |
6d2010ae A |
661 | uint64_t rt_period, |
662 | uint64_t rt_deadline) | |
663 | { | |
664 | uint64_t urgency_notification_time_start, delta; | |
665 | boolean_t urgency_assert = (urgency_notification_assert_abstime_threshold != 0); | |
666 | assert(get_preemption_level() > 0 || ml_get_interrupts_enabled() == FALSE); | |
667 | #if DEBUG | |
668 | urgency_stats[cpu_number() % 64][urgency]++; | |
669 | #endif | |
670 | if (!pmInitDone | |
671 | || pmDispatch == NULL | |
672 | || pmDispatch->pmThreadTellUrgency == NULL) | |
673 | return; | |
674 | ||
675 | KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED,MACH_URGENCY) | DBG_FUNC_START, urgency, rt_period, (rt_deadline >> 32), rt_deadline, 0); | |
676 | ||
677 | if (__improbable((urgency_assert == TRUE))) | |
678 | urgency_notification_time_start = mach_absolute_time(); | |
679 | ||
680 | pmDispatch->pmThreadTellUrgency(urgency, rt_period, rt_deadline); | |
681 | ||
682 | if (__improbable((urgency_assert == TRUE))) { | |
683 | delta = mach_absolute_time() - urgency_notification_time_start; | |
684 | ||
685 | if (__improbable(delta > urgency_notification_max_recorded)) { | |
686 | /* This is not synchronized, but it doesn't matter | |
687 | * if we (rarely) miss an event, as it is statistically | |
688 | * unlikely that it will never recur. | |
689 | */ | |
690 | urgency_notification_max_recorded = delta; | |
691 | ||
692 | if (__improbable((delta > urgency_notification_assert_abstime_threshold) && !machine_timeout_suspended())) | |
693 | panic("Urgency notification callout %p exceeded threshold, 0x%llx abstime units", pmDispatch->pmThreadTellUrgency, delta); | |
694 | } | |
695 | } | |
060df5ea | 696 | |
6d2010ae | 697 | KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED,MACH_URGENCY) | DBG_FUNC_END, urgency, rt_period, (rt_deadline >> 32), rt_deadline, 0); |
060df5ea A |
698 | } |
699 | ||
700 | void | |
701 | active_rt_threads(boolean_t active) | |
702 | { | |
703 | if (!pmInitDone | |
704 | || pmDispatch == NULL | |
705 | || pmDispatch->pmActiveRTThreads == NULL) | |
706 | return; | |
707 | ||
708 | pmDispatch->pmActiveRTThreads(active); | |
709 | } | |
710 | ||
c910b4d9 A |
711 | static uint32_t |
712 | pmGetSavedRunCount(void) | |
713 | { | |
714 | return(saved_run_count); | |
715 | } | |
716 | ||
2d21ac55 A |
717 | /* |
718 | * Returns the root of the package tree. | |
719 | */ | |
720 | static x86_pkg_t * | |
721 | pmGetPkgRoot(void) | |
722 | { | |
723 | return(x86_pkgs); | |
724 | } | |
725 | ||
726 | static boolean_t | |
727 | pmCPUGetHibernate(int cpu) | |
0c530ab8 | 728 | { |
2d21ac55 | 729 | return(cpu_datap(cpu)->cpu_hibernate); |
0c530ab8 A |
730 | } |
731 | ||
2d21ac55 A |
732 | static processor_t |
733 | pmLCPUtoProcessor(int lcpu) | |
734 | { | |
735 | return(cpu_datap(lcpu)->cpu_processor); | |
736 | } | |
737 | ||
c910b4d9 A |
738 | static void |
739 | pmReSyncDeadlines(int cpu) | |
740 | { | |
741 | static boolean_t registered = FALSE; | |
742 | ||
743 | if (!registered) { | |
744 | PM_interrupt_register(&etimer_resync_deadlines); | |
745 | registered = TRUE; | |
746 | } | |
747 | ||
748 | if ((uint32_t)cpu == current_cpu_datap()->lcpu.cpu_num) | |
749 | etimer_resync_deadlines(); | |
750 | else | |
751 | cpu_PM_interrupt(cpu); | |
752 | } | |
753 | ||
b0d623f7 A |
754 | static void |
755 | pmSendIPI(int cpu) | |
756 | { | |
757 | lapic_send_ipi(cpu, LAPIC_PM_INTERRUPT); | |
758 | } | |
759 | ||
060df5ea A |
760 | static void |
761 | pmGetNanotimeInfo(pm_rtc_nanotime_t *rtc_nanotime) | |
762 | { | |
763 | /* | |
764 | * Make sure that nanotime didn't change while we were reading it. | |
765 | */ | |
766 | do { | |
6d2010ae A |
767 | rtc_nanotime->generation = pal_rtc_nanotime_info.generation; /* must be first */ |
768 | rtc_nanotime->tsc_base = pal_rtc_nanotime_info.tsc_base; | |
769 | rtc_nanotime->ns_base = pal_rtc_nanotime_info.ns_base; | |
770 | rtc_nanotime->scale = pal_rtc_nanotime_info.scale; | |
771 | rtc_nanotime->shift = pal_rtc_nanotime_info.shift; | |
772 | } while(pal_rtc_nanotime_info.generation != 0 | |
773 | && rtc_nanotime->generation != pal_rtc_nanotime_info.generation); | |
060df5ea A |
774 | } |
775 | ||
776 | static uint32_t | |
6d2010ae | 777 | pmTimerQueueMigrate(int target_cpu) |
7e4a7d39 | 778 | { |
6d2010ae A |
779 | /* Call the etimer code to do this. */ |
780 | return (target_cpu != cpu_number()) | |
781 | ? etimer_queue_migrate(target_cpu) | |
782 | : 0; | |
7e4a7d39 A |
783 | } |
784 | ||
6d2010ae | 785 | |
2d21ac55 A |
786 | /* |
787 | * Called by the power management kext to register itself and to get the | |
788 | * callbacks it might need into other kernel functions. This interface | |
789 | * is versioned to allow for slight mis-matches between the kext and the | |
790 | * kernel. | |
791 | */ | |
0c530ab8 | 792 | void |
2d21ac55 A |
793 | pmKextRegister(uint32_t version, pmDispatch_t *cpuFuncs, |
794 | pmCallBacks_t *callbacks) | |
0c530ab8 | 795 | { |
2d21ac55 | 796 | if (callbacks != NULL && version == PM_DISPATCH_VERSION) { |
c910b4d9 A |
797 | callbacks->setRTCPop = setPop; |
798 | callbacks->resyncDeadlines = pmReSyncDeadlines; | |
799 | callbacks->initComplete = pmInitComplete; | |
800 | callbacks->GetLCPU = pmGetLogicalCPU; | |
801 | callbacks->GetCore = pmGetCore; | |
802 | callbacks->GetDie = pmGetDie; | |
803 | callbacks->GetPackage = pmGetPackage; | |
804 | callbacks->GetMyLCPU = pmGetMyLogicalCPU; | |
805 | callbacks->GetMyCore = pmGetMyCore; | |
806 | callbacks->GetMyDie = pmGetMyDie; | |
807 | callbacks->GetMyPackage = pmGetMyPackage; | |
808 | callbacks->GetPkgRoot = pmGetPkgRoot; | |
809 | callbacks->LockCPUTopology = pmLockCPUTopology; | |
810 | callbacks->GetHibernate = pmCPUGetHibernate; | |
811 | callbacks->LCPUtoProcessor = pmLCPUtoProcessor; | |
812 | callbacks->ThreadBind = thread_bind; | |
813 | callbacks->GetSavedRunCount = pmGetSavedRunCount; | |
7e4a7d39 | 814 | callbacks->GetNanotimeInfo = pmGetNanotimeInfo; |
060df5ea A |
815 | callbacks->ThreadGetUrgency = pmThreadGetUrgency; |
816 | callbacks->RTCClockAdjust = rtc_clock_adjust; | |
817 | callbacks->timerQueueMigrate = pmTimerQueueMigrate; | |
c910b4d9 | 818 | callbacks->topoParms = &topoParms; |
6d2010ae | 819 | callbacks->pmSendIPI = pmSendIPI; |
060df5ea A |
820 | callbacks->InterruptPending = lapic_is_interrupt_pending; |
821 | callbacks->IsInterrupting = lapic_is_interrupting; | |
822 | callbacks->InterruptStats = lapic_interrupt_counts; | |
823 | callbacks->DisableApicTimer = lapic_disable_timer; | |
c910b4d9 | 824 | } else { |
6d2010ae | 825 | panic("Version mis-match between Kernel and CPU PM"); |
2d21ac55 A |
826 | } |
827 | ||
828 | if (cpuFuncs != NULL) { | |
829 | pmDispatch = cpuFuncs; | |
b0d623f7 | 830 | |
7ddcb079 A |
831 | if (earlyTopology |
832 | && pmDispatch->pmCPUStateInit != NULL) { | |
833 | (*pmDispatch->pmCPUStateInit)(); | |
834 | earlyTopology = FALSE; | |
835 | } | |
836 | ||
b0d623f7 A |
837 | if (pmDispatch->pmIPIHandler != NULL) { |
838 | lapic_set_pm_func((i386_intr_func_t)pmDispatch->pmIPIHandler); | |
839 | } | |
2d21ac55 | 840 | } |
0c530ab8 A |
841 | } |
842 | ||
2d21ac55 A |
843 | /* |
844 | * Unregisters the power management functions from the kext. | |
845 | */ | |
0c530ab8 | 846 | void |
2d21ac55 | 847 | pmUnRegister(pmDispatch_t *cpuFuncs) |
0c530ab8 | 848 | { |
2d21ac55 A |
849 | if (cpuFuncs != NULL && pmDispatch == cpuFuncs) { |
850 | pmDispatch = NULL; | |
851 | } | |
0c530ab8 | 852 | } |
2d21ac55 | 853 | |
593a1d5f A |
854 | /****************************************************************************** |
855 | * | |
856 | * All of the following are deprecated interfaces and no longer used. | |
857 | * | |
858 | ******************************************************************************/ | |
859 | kern_return_t | |
860 | pmsControl(__unused uint32_t request, __unused user_addr_t reqaddr, | |
861 | __unused uint32_t reqsize) | |
862 | { | |
863 | return(KERN_SUCCESS); | |
864 | } | |
865 | ||
866 | void | |
867 | pmsInit(void) | |
868 | { | |
869 | } | |
870 | ||
871 | void | |
872 | pmsStart(void) | |
873 | { | |
874 | } | |
875 | ||
876 | void | |
877 | pmsPark(void) | |
878 | { | |
879 | } | |
880 | ||
881 | void | |
882 | pmsRun(__unused uint32_t nstep) | |
883 | { | |
884 | } | |
885 | ||
886 | kern_return_t | |
887 | pmsBuild(__unused pmsDef *pd, __unused uint32_t pdsize, | |
888 | __unused pmsSetFunc_t *functab, | |
889 | __unused uint32_t platformData, __unused pmsQueryFunc_t queryFunc) | |
890 | { | |
891 | return(KERN_SUCCESS); | |
892 | } |