]> git.saurik.com Git - apple/xnu.git/blame - osfmk/i386/pmCPU.c
xnu-2050.7.9.tar.gz
[apple/xnu.git] / osfmk / i386 / pmCPU.c
CommitLineData
0c530ab8 1/*
7ddcb079 2 * Copyright (c) 2004-2011 Apple Inc. All rights reserved.
0c530ab8 3 *
2d21ac55 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
0c530ab8 5 *
2d21ac55
A
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
0c530ab8 14 *
2d21ac55
A
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
0c530ab8
A
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55
A
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
0c530ab8 25 *
2d21ac55 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
0c530ab8
A
27 */
28
29/*
30 * CPU-specific power management support.
31 *
32 * Implements the "wrappers" to the KEXT.
33 */
0c530ab8 34#include <i386/asm.h>
b0d623f7 35#include <i386/machine_cpu.h>
0c530ab8 36#include <i386/mp.h>
b0d623f7 37#include <i386/machine_routines.h>
0c530ab8 38#include <i386/proc_reg.h>
b0d623f7
A
39#include <i386/pmap.h>
40#include <i386/misc_protos.h>
41#include <kern/machine.h>
0c530ab8
A
42#include <kern/pms.h>
43#include <kern/processor.h>
7e4a7d39 44#include <kern/etimer.h>
2d21ac55 45#include <i386/cpu_threads.h>
0c530ab8
A
46#include <i386/pmCPU.h>
47#include <i386/cpuid.h>
6d2010ae 48#include <i386/rtclock_protos.h>
593a1d5f 49#include <kern/sched_prim.h>
b0d623f7 50#include <i386/lapic.h>
6d2010ae
A
51#include <i386/pal_routines.h>
52
53#include <sys/kdebug.h>
593a1d5f 54
0c530ab8
A
55extern int disableConsoleOutput;
56
7ddcb079 57#define DELAY_UNSET 0xFFFFFFFFFFFFFFFFULL
0c530ab8
A
58
59/*
60 * The following is set when the KEXT loads and initializes.
61 */
6d2010ae 62pmDispatch_t *pmDispatch = NULL;
0c530ab8 63
7ddcb079
A
64static uint32_t pmInitDone = 0;
65static boolean_t earlyTopology = FALSE;
66static uint64_t earlyMaxBusDelay = DELAY_UNSET;
67static uint64_t earlyMaxIntDelay = DELAY_UNSET;
2d21ac55 68
0c530ab8
A
69/*
70 * Initialize the Cstate change code.
71 */
72void
73power_management_init(void)
74{
0c530ab8
A
75 if (pmDispatch != NULL && pmDispatch->cstateInit != NULL)
76 (*pmDispatch->cstateInit)();
77}
78
0c530ab8 79/*
593a1d5f
A
80 * Called when the CPU is idle. It calls into the power management kext
81 * to determine the best way to idle the CPU.
0c530ab8 82 */
593a1d5f
A
83void
84machine_idle(void)
0c530ab8 85{
593a1d5f 86 cpu_data_t *my_cpu = current_cpu_datap();
0c530ab8 87
593a1d5f
A
88 if (my_cpu == NULL)
89 goto out;
0c530ab8 90
593a1d5f
A
91 my_cpu->lcpu.state = LCPU_IDLE;
92 DBGLOG(cpu_handle, cpu_number(), MP_IDLE);
93 MARK_CPU_IDLE(cpu_number());
0c530ab8 94
7ddcb079
A
95 if (pmInitDone) {
96 /*
97 * Handle case where ml_set_maxbusdelay() or ml_set_maxintdelay()
98 * were called prior to the CPU PM kext being registered. We do
316670eb
A
99 * this here since we know at this point the values will be first
100 * used since idle is where the decisions using these values is made.
7ddcb079
A
101 */
102 if (earlyMaxBusDelay != DELAY_UNSET)
103 ml_set_maxbusdelay((uint32_t)(earlyMaxBusDelay & 0xFFFFFFFF));
104
105 if (earlyMaxIntDelay != DELAY_UNSET)
106 ml_set_maxintdelay(earlyMaxIntDelay);
107 }
108
593a1d5f
A
109 if (pmInitDone
110 && pmDispatch != NULL
b7266188
A
111 && pmDispatch->MachineIdle != NULL)
112 (*pmDispatch->MachineIdle)(0x7FFFFFFFFFFFFFFFULL);
593a1d5f
A
113 else {
114 /*
115 * If no power management, re-enable interrupts and halt.
116 * This will keep the CPU from spinning through the scheduler
117 * and will allow at least some minimal power savings (but it
118 * cause problems in some MP configurations w.r.t. the APIC
119 * stopping during a GV3 transition).
120 */
6d2010ae
A
121 pal_hlt();
122
123 /* Once woken, re-disable interrupts. */
124 pal_cli();
0c530ab8 125 }
0c530ab8 126
593a1d5f
A
127 /*
128 * Mark the CPU as running again.
129 */
130 MARK_CPU_ACTIVE(cpu_number());
131 DBGLOG(cpu_handle, cpu_number(), MP_UNIDLE);
132 my_cpu->lcpu.state = LCPU_RUN;
133
134 /*
135 * Re-enable interrupts.
136 */
137 out:
6d2010ae 138 pal_sti();
2d21ac55
A
139}
140
141/*
142 * Called when the CPU is to be halted. It will choose the best C-State
143 * to be in.
144 */
145void
146pmCPUHalt(uint32_t reason)
147{
593a1d5f 148 cpu_data_t *cpup = current_cpu_datap();
2d21ac55
A
149
150 switch (reason) {
151 case PM_HALT_DEBUG:
593a1d5f 152 cpup->lcpu.state = LCPU_PAUSE;
6d2010ae 153 pal_stop_cpu(FALSE);
2d21ac55
A
154 break;
155
156 case PM_HALT_PANIC:
593a1d5f 157 cpup->lcpu.state = LCPU_PAUSE;
6d2010ae 158 pal_stop_cpu(TRUE);
2d21ac55
A
159 break;
160
161 case PM_HALT_NORMAL:
162 default:
6d2010ae 163 pal_cli();
2d21ac55 164
6d2010ae 165 if (pmInitDone
2d21ac55
A
166 && pmDispatch != NULL
167 && pmDispatch->pmCPUHalt != NULL) {
593a1d5f
A
168 /*
169 * Halt the CPU (and put it in a low power state.
170 */
2d21ac55 171 (*pmDispatch->pmCPUHalt)();
2d21ac55 172
593a1d5f
A
173 /*
174 * We've exited halt, so get the the CPU schedulable again.
175 */
176 i386_init_slave_fast();
177
178 panic("init_slave_fast returned");
6d2010ae
A
179 } else
180 {
2d21ac55
A
181 /*
182 * If no power managment and a processor is taken off-line,
183 * then invalidate the cache and halt it (it will not be able
184 * to be brought back on-line without resetting the CPU).
185 */
186 __asm__ volatile ("wbinvd");
593a1d5f 187 cpup->lcpu.state = LCPU_HALT;
6d2010ae 188 pal_stop_cpu(FALSE);
593a1d5f
A
189
190 panic("back from Halt");
2d21ac55 191 }
6d2010ae 192
2d21ac55 193 break;
0c530ab8
A
194 }
195}
196
2d21ac55 197void
593a1d5f 198pmMarkAllCPUsOff(void)
2d21ac55 199{
593a1d5f
A
200 if (pmInitDone
201 && pmDispatch != NULL
202 && pmDispatch->markAllCPUsOff != NULL)
203 (*pmDispatch->markAllCPUsOff)();
2d21ac55
A
204}
205
206static void
207pmInitComplete(void)
208{
7ddcb079
A
209 if (earlyTopology
210 && pmDispatch != NULL
211 && pmDispatch->pmCPUStateInit != NULL) {
060df5ea 212 (*pmDispatch->pmCPUStateInit)();
7ddcb079
A
213 earlyTopology = FALSE;
214 }
060df5ea 215
2d21ac55
A
216 pmInitDone = 1;
217}
218
219static x86_lcpu_t *
220pmGetLogicalCPU(int cpu)
0c530ab8 221{
2d21ac55
A
222 return(cpu_to_lcpu(cpu));
223}
224
225static x86_lcpu_t *
226pmGetMyLogicalCPU(void)
227{
228 cpu_data_t *cpup = current_cpu_datap();
0c530ab8 229
2d21ac55
A
230 return(&cpup->lcpu);
231}
232
233static x86_core_t *
234pmGetCore(int cpu)
235{
236 return(cpu_to_core(cpu));
0c530ab8
A
237}
238
2d21ac55
A
239static x86_core_t *
240pmGetMyCore(void)
0c530ab8 241{
2d21ac55 242 cpu_data_t *cpup = current_cpu_datap();
0c530ab8 243
2d21ac55 244 return(cpup->lcpu.core);
0c530ab8
A
245}
246
593a1d5f
A
247static x86_die_t *
248pmGetDie(int cpu)
249{
250 return(cpu_to_die(cpu));
251}
252
253static x86_die_t *
254pmGetMyDie(void)
255{
256 cpu_data_t *cpup = current_cpu_datap();
257
258 return(cpup->lcpu.die);
259}
260
2d21ac55
A
261static x86_pkg_t *
262pmGetPackage(int cpu)
0c530ab8 263{
2d21ac55
A
264 return(cpu_to_package(cpu));
265}
266
267static x86_pkg_t *
268pmGetMyPackage(void)
269{
270 cpu_data_t *cpup = current_cpu_datap();
271
593a1d5f 272 return(cpup->lcpu.package);
2d21ac55
A
273}
274
275static void
276pmLockCPUTopology(int lock)
277{
278 if (lock) {
279 simple_lock(&x86_topo_lock);
280 } else {
281 simple_unlock(&x86_topo_lock);
282 }
0c530ab8
A
283}
284
285/*
2d21ac55
A
286 * Called to get the next deadline that has been set by the
287 * power management code.
6d2010ae
A
288 * Note: a return of 0 from AICPM and this routine signifies
289 * that no deadline is set.
0c530ab8 290 */
2d21ac55
A
291uint64_t
292pmCPUGetDeadline(cpu_data_t *cpu)
293{
060df5ea 294 uint64_t deadline = 0;
0c530ab8 295
7ddcb079 296 if (pmInitDone
2d21ac55
A
297 && pmDispatch != NULL
298 && pmDispatch->GetDeadline != NULL)
299 deadline = (*pmDispatch->GetDeadline)(&cpu->lcpu);
300
301 return(deadline);
0c530ab8
A
302}
303
304/*
2d21ac55
A
305 * Called to determine if the supplied deadline or the power management
306 * deadline is sooner. Returns which ever one is first.
0c530ab8 307 */
2d21ac55
A
308uint64_t
309pmCPUSetDeadline(cpu_data_t *cpu, uint64_t deadline)
0c530ab8 310{
b0d623f7 311 if (pmInitDone
2d21ac55
A
312 && pmDispatch != NULL
313 && pmDispatch->SetDeadline != NULL)
314 deadline = (*pmDispatch->SetDeadline)(&cpu->lcpu, deadline);
315
316 return(deadline);
0c530ab8
A
317}
318
0c530ab8 319/*
2d21ac55 320 * Called when a power management deadline expires.
0c530ab8
A
321 */
322void
2d21ac55 323pmCPUDeadline(cpu_data_t *cpu)
0c530ab8 324{
2d21ac55
A
325 if (pmInitDone
326 && pmDispatch != NULL
327 && pmDispatch->Deadline != NULL)
328 (*pmDispatch->Deadline)(&cpu->lcpu);
329}
330
331/*
332 * Called to get a CPU out of idle.
333 */
334boolean_t
335pmCPUExitIdle(cpu_data_t *cpu)
336{
337 boolean_t do_ipi;
338
339 if (pmInitDone
340 && pmDispatch != NULL
341 && pmDispatch->exitIdle != NULL)
342 do_ipi = (*pmDispatch->exitIdle)(&cpu->lcpu);
343 else
344 do_ipi = TRUE;
345
346 return(do_ipi);
0c530ab8
A
347}
348
593a1d5f
A
349kern_return_t
350pmCPUExitHalt(int cpu)
351{
352 kern_return_t rc = KERN_INVALID_ARGUMENT;
353
354 if (pmInitDone
355 && pmDispatch != NULL
356 && pmDispatch->exitHalt != NULL)
357 rc = pmDispatch->exitHalt(cpu_to_lcpu(cpu));
358
359 return(rc);
360}
361
e2fac8b1
A
362kern_return_t
363pmCPUExitHaltToOff(int cpu)
364{
365 kern_return_t rc = KERN_INVALID_ARGUMENT;
366
367 if (pmInitDone
368 && pmDispatch != NULL
369 && pmDispatch->exitHaltToOff != NULL)
370 rc = pmDispatch->exitHaltToOff(cpu_to_lcpu(cpu));
371
372 return(rc);
373}
374
2d21ac55 375/*
593a1d5f 376 * Called to initialize the power management structures for the CPUs.
2d21ac55 377 */
0c530ab8 378void
593a1d5f 379pmCPUStateInit(void)
0c530ab8 380{
593a1d5f
A
381 if (pmDispatch != NULL && pmDispatch->pmCPUStateInit != NULL)
382 (*pmDispatch->pmCPUStateInit)();
060df5ea
A
383 else
384 earlyTopology = TRUE;
0c530ab8
A
385}
386
2d21ac55 387/*
593a1d5f 388 * Called when a CPU is being restarted after being powered off (as in S3).
2d21ac55 389 */
0c530ab8 390void
593a1d5f 391pmCPUMarkRunning(cpu_data_t *cpu)
0c530ab8 392{
593a1d5f
A
393 cpu_data_t *cpup = current_cpu_datap();
394
2d21ac55
A
395 if (pmInitDone
396 && pmDispatch != NULL
593a1d5f
A
397 && pmDispatch->markCPURunning != NULL)
398 (*pmDispatch->markCPURunning)(&cpu->lcpu);
399 else
400 cpup->lcpu.state = LCPU_RUN;
2d21ac55
A
401}
402
403/*
404 * Called to get/set CPU power management state.
405 */
406int
407pmCPUControl(uint32_t cmd, void *datap)
408{
409 int rc = -1;
410
411 if (pmDispatch != NULL
412 && pmDispatch->pmCPUControl != NULL)
413 rc = (*pmDispatch->pmCPUControl)(cmd, datap);
414
415 return(rc);
0c530ab8
A
416}
417
593a1d5f
A
418/*
419 * Called to save the timer state used by power management prior
420 * to "sleeping".
421 */
422void
423pmTimerSave(void)
424{
425 if (pmDispatch != NULL
426 && pmDispatch->pmTimerStateSave != NULL)
427 (*pmDispatch->pmTimerStateSave)();
428}
429
430/*
431 * Called to restore the timer state used by power management after
432 * waking from "sleep".
433 */
434void
435pmTimerRestore(void)
436{
437 if (pmDispatch != NULL
438 && pmDispatch->pmTimerStateRestore != NULL)
439 (*pmDispatch->pmTimerStateRestore)();
440}
441
2d21ac55
A
442/*
443 * Set the worst-case time for the C4 to C2 transition.
444 * No longer does anything.
445 */
0c530ab8 446void
2d21ac55 447ml_set_maxsnoop(__unused uint32_t maxdelay)
0c530ab8 448{
0c530ab8
A
449}
450
2d21ac55
A
451
452/*
453 * Get the worst-case time for the C4 to C2 transition. Returns nanoseconds.
454 */
455unsigned
456ml_get_maxsnoop(void)
457{
458 uint64_t max_snoop = 0;
459
7ddcb079
A
460 if (pmInitDone
461 && pmDispatch != NULL
2d21ac55
A
462 && pmDispatch->getMaxSnoop != NULL)
463 max_snoop = pmDispatch->getMaxSnoop();
464
465 return((unsigned)(max_snoop & 0xffffffff));
466}
467
468
469uint32_t
470ml_get_maxbusdelay(void)
471{
472 uint64_t max_delay = 0;
473
7ddcb079
A
474 if (pmInitDone
475 && pmDispatch != NULL
2d21ac55
A
476 && pmDispatch->getMaxBusDelay != NULL)
477 max_delay = pmDispatch->getMaxBusDelay();
478
479 return((uint32_t)(max_delay & 0xffffffff));
480}
481
482/*
483 * Set the maximum delay time allowed for snoop on the bus.
484 *
485 * Note that this value will be compared to the amount of time that it takes
486 * to transition from a non-snooping power state (C4) to a snooping state (C2).
487 * If maxBusDelay is less than C4C2SnoopDelay,
488 * we will not enter the lowest power state.
489 */
0c530ab8 490void
2d21ac55 491ml_set_maxbusdelay(uint32_t mdelay)
0c530ab8 492{
2d21ac55
A
493 uint64_t maxdelay = mdelay;
494
495 if (pmDispatch != NULL
7ddcb079
A
496 && pmDispatch->setMaxBusDelay != NULL) {
497 earlyMaxBusDelay = DELAY_UNSET;
2d21ac55 498 pmDispatch->setMaxBusDelay(maxdelay);
7ddcb079
A
499 } else
500 earlyMaxBusDelay = maxdelay;
593a1d5f
A
501}
502
503uint64_t
504ml_get_maxintdelay(void)
505{
506 uint64_t max_delay = 0;
507
508 if (pmDispatch != NULL
509 && pmDispatch->getMaxIntDelay != NULL)
510 max_delay = pmDispatch->getMaxIntDelay();
511
512 return(max_delay);
513}
514
515/*
516 * Set the maximum delay allowed for an interrupt.
517 */
518void
519ml_set_maxintdelay(uint64_t mdelay)
520{
521 if (pmDispatch != NULL
7ddcb079
A
522 && pmDispatch->setMaxIntDelay != NULL) {
523 earlyMaxIntDelay = DELAY_UNSET;
593a1d5f 524 pmDispatch->setMaxIntDelay(mdelay);
7ddcb079
A
525 } else
526 earlyMaxIntDelay = mdelay;
0c530ab8
A
527}
528
6d2010ae
A
529boolean_t
530ml_get_interrupt_prewake_applicable()
531{
532 boolean_t applicable = FALSE;
533
534 if (pmInitDone
535 && pmDispatch != NULL
536 && pmDispatch->pmInterruptPrewakeApplicable != NULL)
537 applicable = pmDispatch->pmInterruptPrewakeApplicable();
538
539 return applicable;
540}
541
2d21ac55
A
542/*
543 * Put a CPU into "safe" mode with respect to power.
544 *
545 * Some systems cannot operate at a continuous "normal" speed without
546 * exceeding the thermal design. This is called per-CPU to place the
547 * CPUs into a "safe" operating mode.
548 */
0c530ab8 549void
2d21ac55
A
550pmSafeMode(x86_lcpu_t *lcpu, uint32_t flags)
551{
552 if (pmDispatch != NULL
553 && pmDispatch->pmCPUSafeMode != NULL)
554 pmDispatch->pmCPUSafeMode(lcpu, flags);
555 else {
556 /*
557 * Do something reasonable if the KEXT isn't present.
558 *
559 * We only look at the PAUSE and RESUME flags. The other flag(s)
560 * will not make any sense without the KEXT, so just ignore them.
561 *
593a1d5f
A
562 * We set the CPU's state to indicate that it's halted. If this
563 * is the CPU we're currently running on, then spin until the
564 * state becomes non-halted.
2d21ac55
A
565 */
566 if (flags & PM_SAFE_FL_PAUSE) {
593a1d5f 567 lcpu->state = LCPU_PAUSE;
2d21ac55 568 if (lcpu == x86_lcpu()) {
593a1d5f 569 while (lcpu->state == LCPU_PAUSE)
2d21ac55
A
570 cpu_pause();
571 }
572 }
573
574 /*
575 * Clear the halted flag for the specified CPU, that will
576 * get it out of it's spin loop.
577 */
578 if (flags & PM_SAFE_FL_RESUME) {
593a1d5f 579 lcpu->state = LCPU_RUN;
2d21ac55
A
580 }
581 }
582}
583
c910b4d9
A
584static uint32_t saved_run_count = 0;
585
586void
587machine_run_count(uint32_t count)
588{
589 if (pmDispatch != NULL
590 && pmDispatch->pmSetRunCount != NULL)
591 pmDispatch->pmSetRunCount(count);
592 else
593 saved_run_count = count;
594}
595
596boolean_t
b7266188 597machine_processor_is_inactive(processor_t processor)
c910b4d9 598{
b7266188
A
599 int cpu = processor->cpu_id;
600
c910b4d9
A
601 if (pmDispatch != NULL
602 && pmDispatch->pmIsCPUUnAvailable != NULL)
603 return(pmDispatch->pmIsCPUUnAvailable(cpu_to_lcpu(cpu)));
604 else
605 return(FALSE);
606}
607
b7266188
A
608processor_t
609machine_choose_processor(processor_set_t pset,
610 processor_t preferred)
611{
612 int startCPU;
613 int endCPU;
614 int preferredCPU;
615 int chosenCPU;
616
617 if (!pmInitDone)
618 return(preferred);
619
620 if (pset == NULL) {
621 startCPU = -1;
622 endCPU = -1;
623 } else {
624 startCPU = pset->cpu_set_low;
625 endCPU = pset->cpu_set_hi;
626 }
627
628 if (preferred == NULL)
629 preferredCPU = -1;
630 else
631 preferredCPU = preferred->cpu_id;
632
633 if (pmDispatch != NULL
634 && pmDispatch->pmChooseCPU != NULL) {
635 chosenCPU = pmDispatch->pmChooseCPU(startCPU, endCPU, preferredCPU);
636
637 if (chosenCPU == -1)
638 return(NULL);
639 return(cpu_datap(chosenCPU)->cpu_processor);
640 }
641
642 return(preferred);
643}
644
060df5ea 645static int
6d2010ae 646pmThreadGetUrgency(uint64_t *rt_period, uint64_t *rt_deadline)
060df5ea
A
647{
648
6d2010ae 649 return(thread_get_urgency(rt_period, rt_deadline));
060df5ea
A
650}
651
6d2010ae
A
652#if DEBUG
653uint32_t urgency_stats[64][THREAD_URGENCY_MAX];
654#endif
655
656#define URGENCY_NOTIFICATION_ASSERT_NS (5 * 1000 * 1000)
657uint64_t urgency_notification_assert_abstime_threshold, urgency_notification_max_recorded;
658
060df5ea
A
659void
660thread_tell_urgency(int urgency,
6d2010ae
A
661 uint64_t rt_period,
662 uint64_t rt_deadline)
663{
664 uint64_t urgency_notification_time_start, delta;
665 boolean_t urgency_assert = (urgency_notification_assert_abstime_threshold != 0);
666 assert(get_preemption_level() > 0 || ml_get_interrupts_enabled() == FALSE);
667#if DEBUG
668 urgency_stats[cpu_number() % 64][urgency]++;
669#endif
670 if (!pmInitDone
671 || pmDispatch == NULL
672 || pmDispatch->pmThreadTellUrgency == NULL)
673 return;
674
675 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED,MACH_URGENCY) | DBG_FUNC_START, urgency, rt_period, (rt_deadline >> 32), rt_deadline, 0);
676
677 if (__improbable((urgency_assert == TRUE)))
678 urgency_notification_time_start = mach_absolute_time();
679
680 pmDispatch->pmThreadTellUrgency(urgency, rt_period, rt_deadline);
681
682 if (__improbable((urgency_assert == TRUE))) {
683 delta = mach_absolute_time() - urgency_notification_time_start;
684
685 if (__improbable(delta > urgency_notification_max_recorded)) {
686 /* This is not synchronized, but it doesn't matter
687 * if we (rarely) miss an event, as it is statistically
688 * unlikely that it will never recur.
689 */
690 urgency_notification_max_recorded = delta;
691
692 if (__improbable((delta > urgency_notification_assert_abstime_threshold) && !machine_timeout_suspended()))
693 panic("Urgency notification callout %p exceeded threshold, 0x%llx abstime units", pmDispatch->pmThreadTellUrgency, delta);
694 }
695 }
060df5ea 696
6d2010ae 697 KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED,MACH_URGENCY) | DBG_FUNC_END, urgency, rt_period, (rt_deadline >> 32), rt_deadline, 0);
060df5ea
A
698}
699
700void
701active_rt_threads(boolean_t active)
702{
703 if (!pmInitDone
704 || pmDispatch == NULL
705 || pmDispatch->pmActiveRTThreads == NULL)
706 return;
707
708 pmDispatch->pmActiveRTThreads(active);
709}
710
c910b4d9
A
711static uint32_t
712pmGetSavedRunCount(void)
713{
714 return(saved_run_count);
715}
716
2d21ac55
A
717/*
718 * Returns the root of the package tree.
719 */
720static x86_pkg_t *
721pmGetPkgRoot(void)
722{
723 return(x86_pkgs);
724}
725
726static boolean_t
727pmCPUGetHibernate(int cpu)
0c530ab8 728{
2d21ac55 729 return(cpu_datap(cpu)->cpu_hibernate);
0c530ab8
A
730}
731
2d21ac55
A
732static processor_t
733pmLCPUtoProcessor(int lcpu)
734{
735 return(cpu_datap(lcpu)->cpu_processor);
736}
737
c910b4d9
A
738static void
739pmReSyncDeadlines(int cpu)
740{
741 static boolean_t registered = FALSE;
742
743 if (!registered) {
744 PM_interrupt_register(&etimer_resync_deadlines);
745 registered = TRUE;
746 }
747
748 if ((uint32_t)cpu == current_cpu_datap()->lcpu.cpu_num)
749 etimer_resync_deadlines();
750 else
751 cpu_PM_interrupt(cpu);
752}
753
b0d623f7
A
754static void
755pmSendIPI(int cpu)
756{
757 lapic_send_ipi(cpu, LAPIC_PM_INTERRUPT);
758}
759
060df5ea
A
760static void
761pmGetNanotimeInfo(pm_rtc_nanotime_t *rtc_nanotime)
762{
763 /*
764 * Make sure that nanotime didn't change while we were reading it.
765 */
766 do {
6d2010ae
A
767 rtc_nanotime->generation = pal_rtc_nanotime_info.generation; /* must be first */
768 rtc_nanotime->tsc_base = pal_rtc_nanotime_info.tsc_base;
769 rtc_nanotime->ns_base = pal_rtc_nanotime_info.ns_base;
770 rtc_nanotime->scale = pal_rtc_nanotime_info.scale;
771 rtc_nanotime->shift = pal_rtc_nanotime_info.shift;
772 } while(pal_rtc_nanotime_info.generation != 0
773 && rtc_nanotime->generation != pal_rtc_nanotime_info.generation);
060df5ea
A
774}
775
776static uint32_t
6d2010ae 777pmTimerQueueMigrate(int target_cpu)
7e4a7d39 778{
6d2010ae
A
779 /* Call the etimer code to do this. */
780 return (target_cpu != cpu_number())
781 ? etimer_queue_migrate(target_cpu)
782 : 0;
7e4a7d39
A
783}
784
6d2010ae 785
2d21ac55
A
786/*
787 * Called by the power management kext to register itself and to get the
788 * callbacks it might need into other kernel functions. This interface
789 * is versioned to allow for slight mis-matches between the kext and the
790 * kernel.
791 */
0c530ab8 792void
2d21ac55
A
793pmKextRegister(uint32_t version, pmDispatch_t *cpuFuncs,
794 pmCallBacks_t *callbacks)
0c530ab8 795{
2d21ac55 796 if (callbacks != NULL && version == PM_DISPATCH_VERSION) {
c910b4d9
A
797 callbacks->setRTCPop = setPop;
798 callbacks->resyncDeadlines = pmReSyncDeadlines;
799 callbacks->initComplete = pmInitComplete;
800 callbacks->GetLCPU = pmGetLogicalCPU;
801 callbacks->GetCore = pmGetCore;
802 callbacks->GetDie = pmGetDie;
803 callbacks->GetPackage = pmGetPackage;
804 callbacks->GetMyLCPU = pmGetMyLogicalCPU;
805 callbacks->GetMyCore = pmGetMyCore;
806 callbacks->GetMyDie = pmGetMyDie;
807 callbacks->GetMyPackage = pmGetMyPackage;
808 callbacks->GetPkgRoot = pmGetPkgRoot;
809 callbacks->LockCPUTopology = pmLockCPUTopology;
810 callbacks->GetHibernate = pmCPUGetHibernate;
811 callbacks->LCPUtoProcessor = pmLCPUtoProcessor;
812 callbacks->ThreadBind = thread_bind;
813 callbacks->GetSavedRunCount = pmGetSavedRunCount;
7e4a7d39 814 callbacks->GetNanotimeInfo = pmGetNanotimeInfo;
060df5ea
A
815 callbacks->ThreadGetUrgency = pmThreadGetUrgency;
816 callbacks->RTCClockAdjust = rtc_clock_adjust;
817 callbacks->timerQueueMigrate = pmTimerQueueMigrate;
c910b4d9 818 callbacks->topoParms = &topoParms;
6d2010ae 819 callbacks->pmSendIPI = pmSendIPI;
060df5ea
A
820 callbacks->InterruptPending = lapic_is_interrupt_pending;
821 callbacks->IsInterrupting = lapic_is_interrupting;
822 callbacks->InterruptStats = lapic_interrupt_counts;
823 callbacks->DisableApicTimer = lapic_disable_timer;
c910b4d9 824 } else {
6d2010ae 825 panic("Version mis-match between Kernel and CPU PM");
2d21ac55
A
826 }
827
828 if (cpuFuncs != NULL) {
829 pmDispatch = cpuFuncs;
b0d623f7 830
7ddcb079
A
831 if (earlyTopology
832 && pmDispatch->pmCPUStateInit != NULL) {
833 (*pmDispatch->pmCPUStateInit)();
834 earlyTopology = FALSE;
835 }
836
b0d623f7
A
837 if (pmDispatch->pmIPIHandler != NULL) {
838 lapic_set_pm_func((i386_intr_func_t)pmDispatch->pmIPIHandler);
839 }
2d21ac55 840 }
0c530ab8
A
841}
842
2d21ac55
A
843/*
844 * Unregisters the power management functions from the kext.
845 */
0c530ab8 846void
2d21ac55 847pmUnRegister(pmDispatch_t *cpuFuncs)
0c530ab8 848{
2d21ac55
A
849 if (cpuFuncs != NULL && pmDispatch == cpuFuncs) {
850 pmDispatch = NULL;
851 }
0c530ab8 852}
2d21ac55 853
593a1d5f
A
854/******************************************************************************
855 *
856 * All of the following are deprecated interfaces and no longer used.
857 *
858 ******************************************************************************/
859kern_return_t
860pmsControl(__unused uint32_t request, __unused user_addr_t reqaddr,
861 __unused uint32_t reqsize)
862{
863 return(KERN_SUCCESS);
864}
865
866void
867pmsInit(void)
868{
869}
870
871void
872pmsStart(void)
873{
874}
875
876void
877pmsPark(void)
878{
879}
880
881void
882pmsRun(__unused uint32_t nstep)
883{
884}
885
886kern_return_t
887pmsBuild(__unused pmsDef *pd, __unused uint32_t pdsize,
888 __unused pmsSetFunc_t *functab,
889 __unused uint32_t platformData, __unused pmsQueryFunc_t queryFunc)
890{
891 return(KERN_SUCCESS);
892}