osfmk/i386/pmCPU.c

   1 /*
   2  * Copyright (c) 2004-2011 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 /*
  30  * CPU-specific power management support.
  31  *
  32  * Implements the "wrappers" to the KEXT.
  33  */
  34 #include <i386/asm.h>
  35 #include <i386/machine_cpu.h>
  36 #include <i386/mp.h>
  37 #include <i386/machine_routines.h>
  38 #include <i386/proc_reg.h>
  39 #include <i386/pmap.h>
  40 #include <i386/misc_protos.h>
  41 #include <kern/machine.h>
  42 #include <kern/pms.h>
  43 #include <kern/processor.h>
  44 #include <kern/etimer.h>
  45 #include <i386/cpu_threads.h>
  46 #include <i386/pmCPU.h>
  47 #include <i386/cpuid.h>
  48 #include <i386/rtclock_protos.h>
  49 #include <kern/sched_prim.h>
  50 #include <i386/lapic.h>
  51 #include <i386/pal_routines.h>
  52 #include <sys/kdebug.h>
  53
  54 extern int disableConsoleOutput;
  55
  56 #define DELAY_UNSET             0xFFFFFFFFFFFFFFFFULL
  57
  58 uint64_t cpu_itime_bins[CPU_ITIME_BINS] = {16* NSEC_PER_USEC, 32* NSEC_PER_USEC, 64* NSEC_PER_USEC, 128* NSEC_PER_USEC, 256* NSEC_PER_USEC, 512* NSEC_PER_USEC, 1024* NSEC_PER_USEC, 2048* NSEC_PER_USEC, 4096* NSEC_PER_USEC, 8192* NSEC_PER_USEC, 16384* NSEC_PER_USEC, 32768* NSEC_PER_USEC};
  59 uint64_t *cpu_rtime_bins = &cpu_itime_bins[0];
  60
  61 /*
  62  * The following is set when the KEXT loads and initializes.
  63  */
  64 pmDispatch_t    *pmDispatch     = NULL;
  65
  66 uint32_t                pmInitDone              = 0;
  67 static boolean_t        earlyTopology           = FALSE;
  68 static uint64_t         earlyMaxBusDelay        = DELAY_UNSET;
  69 static uint64_t         earlyMaxIntDelay        = DELAY_UNSET;
  70
  71 /*
  72  * Initialize the Cstate change code.
  73  */
  74 void
  75 power_management_init(void)
  76 {
  77     if (pmDispatch != NULL && pmDispatch->cstateInit != NULL)
  78         (*pmDispatch->cstateInit)();
  79 }
  80
  81 static inline void machine_classify_interval(uint64_t interval, uint64_t *bins, uint64_t *binvals, uint32_t nbins) {
  82         uint32_t i;
  83         for (i = 0; i < nbins; i++) {
  84                 if (interval < binvals[i]) {
  85                         bins[i]++;
  86                         break;
  87                 }
  88         }
  89 }
  90
  91 /*
  92  * Called when the CPU is idle.  It calls into the power management kext
  93  * to determine the best way to idle the CPU.
  94  */
  95 void
  96 machine_idle(void)
  97 {
  98         cpu_data_t              *my_cpu         = current_cpu_datap();
  99         uint64_t                ctime, rtime, itime;
 100
 101         if (my_cpu == NULL)
 102                 goto out;
 103
 104         ctime = mach_absolute_time();
 105
 106         my_cpu->lcpu.state = LCPU_IDLE;
 107         DBGLOG(cpu_handle, cpu_number(), MP_IDLE);
 108         MARK_CPU_IDLE(cpu_number());
 109
 110         rtime = ctime - my_cpu->cpu_ixtime;
 111
 112         my_cpu->cpu_rtime_total += rtime;
 113         machine_classify_interval(rtime, &my_cpu->cpu_rtimes[0], &cpu_rtime_bins[0], CPU_RTIME_BINS);
 114
 115         if (pmInitDone) {
 116                 /*
 117                  * Handle case where ml_set_maxbusdelay() or ml_set_maxintdelay()
 118                  * were called prior to the CPU PM kext being registered.  We do
 119                  * this here since we know at this point the values will be first
 120                  * used since idle is where the decisions using these values is made.
 121                  */
 122                 if (earlyMaxBusDelay != DELAY_UNSET)
 123                         ml_set_maxbusdelay((uint32_t)(earlyMaxBusDelay & 0xFFFFFFFF));
 124
 125                 if (earlyMaxIntDelay != DELAY_UNSET)
 126                         ml_set_maxintdelay(earlyMaxIntDelay);
 127         }
 128
 129         if (pmInitDone
 130             && pmDispatch != NULL
 131             && pmDispatch->MachineIdle != NULL)
 132                 (*pmDispatch->MachineIdle)(0x7FFFFFFFFFFFFFFFULL);
 133         else {
 134                 /*
 135                  * If no power management, re-enable interrupts and halt.
 136                  * This will keep the CPU from spinning through the scheduler
 137                  * and will allow at least some minimal power savings (but it
 138                  * cause problems in some MP configurations w.r.t. the APIC
 139                  * stopping during a GV3 transition).
 140                  */
 141                 pal_hlt();
 142
 143                 /* Once woken, re-disable interrupts. */
 144                 pal_cli();
 145         }
 146
 147         /*
 148          * Mark the CPU as running again.
 149          */
 150         MARK_CPU_ACTIVE(cpu_number());
 151         DBGLOG(cpu_handle, cpu_number(), MP_UNIDLE);
 152
 153         uint64_t ixtime = my_cpu->cpu_ixtime = mach_absolute_time();
 154         my_cpu->cpu_idle_exits++;
 155
 156         itime = ixtime - ctime;
 157
 158         my_cpu->lcpu.state = LCPU_RUN;
 159
 160         machine_classify_interval(itime, &my_cpu->cpu_itimes[0], &cpu_itime_bins[0], CPU_ITIME_BINS);
 161         my_cpu->cpu_itime_total += itime;
 162
 163
 164         /*
 165          * Re-enable interrupts.
 166          */
 167 out:
 168         pal_sti();
 169 }
 170
 171 /*
 172  * Called when the CPU is to be halted.  It will choose the best C-State
 173  * to be in.
 174  */
 175 void
 176 pmCPUHalt(uint32_t reason)
 177 {
 178     cpu_data_t  *cpup   = current_cpu_datap();
 179
 180     switch (reason) {
 181     case PM_HALT_DEBUG:
 182         cpup->lcpu.state = LCPU_PAUSE;
 183         pal_stop_cpu(FALSE);
 184         break;
 185
 186     case PM_HALT_PANIC:
 187         cpup->lcpu.state = LCPU_PAUSE;
 188         pal_stop_cpu(TRUE);
 189         break;
 190
 191     case PM_HALT_NORMAL:
 192     case PM_HALT_SLEEP:
 193     default:
 194         pal_cli();
 195
 196         if (pmInitDone
 197             && pmDispatch != NULL
 198             && pmDispatch->pmCPUHalt != NULL) {
 199             /*
 200              * Halt the CPU (and put it in a low power state.
 201              */
 202             (*pmDispatch->pmCPUHalt)();
 203
 204             /*
 205              * We've exited halt, so get the CPU schedulable again.
 206              * - by calling the fast init routine for a slave, or
 207              * - by returning if we're the master processor.
 208              */
 209             if (cpup->cpu_number != master_cpu) {
 210                 i386_init_slave_fast();
 211                 panic("init_slave_fast returned");
 212             }
 213         } else
 214         {
 215             /*
 216              * If no power managment and a processor is taken off-line,
 217              * then invalidate the cache and halt it (it will not be able
 218              * to be brought back on-line without resetting the CPU).
 219              */
 220             __asm__ volatile ("wbinvd");
 221             cpup->lcpu.state = LCPU_HALT;
 222             pal_stop_cpu(FALSE);
 223
 224             panic("back from Halt");
 225         }
 226
 227         break;
 228     }
 229 }
 230
 231 void
 232 pmMarkAllCPUsOff(void)
 233 {
 234     if (pmInitDone
 235         && pmDispatch != NULL
 236         && pmDispatch->markAllCPUsOff != NULL)
 237         (*pmDispatch->markAllCPUsOff)();
 238 }
 239
 240 static void
 241 pmInitComplete(void)
 242 {
 243     if (earlyTopology
 244         && pmDispatch != NULL
 245         && pmDispatch->pmCPUStateInit != NULL) {
 246         (*pmDispatch->pmCPUStateInit)();
 247         earlyTopology = FALSE;
 248     }
 249
 250     pmInitDone = 1;
 251 }
 252
 253 x86_lcpu_t *
 254 pmGetLogicalCPU(int cpu)
 255 {
 256     return(cpu_to_lcpu(cpu));
 257 }
 258
 259 x86_lcpu_t *
 260 pmGetMyLogicalCPU(void)
 261 {
 262     cpu_data_t  *cpup   = current_cpu_datap();
 263
 264     return(&cpup->lcpu);
 265 }
 266
 267 static x86_core_t *
 268 pmGetCore(int cpu)
 269 {
 270     return(cpu_to_core(cpu));
 271 }
 272
 273 static x86_core_t *
 274 pmGetMyCore(void)
 275 {
 276     cpu_data_t  *cpup   = current_cpu_datap();
 277
 278     return(cpup->lcpu.core);
 279 }
 280
 281 static x86_die_t *
 282 pmGetDie(int cpu)
 283 {
 284     return(cpu_to_die(cpu));
 285 }
 286
 287 static x86_die_t *
 288 pmGetMyDie(void)
 289 {
 290     cpu_data_t  *cpup   = current_cpu_datap();
 291
 292     return(cpup->lcpu.die);
 293 }
 294
 295 static x86_pkg_t *
 296 pmGetPackage(int cpu)
 297 {
 298     return(cpu_to_package(cpu));
 299 }
 300
 301 static x86_pkg_t *
 302 pmGetMyPackage(void)
 303 {
 304     cpu_data_t  *cpup   = current_cpu_datap();
 305
 306     return(cpup->lcpu.package);
 307 }
 308
 309 static void
 310 pmLockCPUTopology(int lock)
 311 {
 312     if (lock) {
 313         simple_lock(&x86_topo_lock);
 314     } else {
 315         simple_unlock(&x86_topo_lock);
 316     }
 317 }
 318
 319 /*
 320  * Called to get the next deadline that has been set by the
 321  * power management code.
 322  * Note: a return of 0 from AICPM and this routine signifies
 323  * that no deadline is set.
 324  */
 325 uint64_t
 326 pmCPUGetDeadline(cpu_data_t *cpu)
 327 {
 328     uint64_t    deadline        = 0;
 329
 330     if (pmInitDone
 331         && pmDispatch != NULL
 332         && pmDispatch->GetDeadline != NULL)
 333         deadline = (*pmDispatch->GetDeadline)(&cpu->lcpu);
 334
 335     return(deadline);
 336 }
 337
 338 /*
 339  * Called to determine if the supplied deadline or the power management
 340  * deadline is sooner.  Returns which ever one is first.
 341  */
 342 uint64_t
 343 pmCPUSetDeadline(cpu_data_t *cpu, uint64_t deadline)
 344 {
 345    if (pmInitDone
 346         && pmDispatch != NULL
 347         && pmDispatch->SetDeadline != NULL)
 348         deadline = (*pmDispatch->SetDeadline)(&cpu->lcpu, deadline);
 349
 350     return(deadline);
 351 }
 352
 353 /*
 354  * Called when a power management deadline expires.
 355  */
 356 void
 357 pmCPUDeadline(cpu_data_t *cpu)
 358 {
 359     if (pmInitDone
 360         && pmDispatch != NULL
 361         && pmDispatch->Deadline != NULL)
 362         (*pmDispatch->Deadline)(&cpu->lcpu);
 363 }
 364
 365 /*
 366  * Called to get a CPU out of idle.
 367  */
 368 boolean_t
 369 pmCPUExitIdle(cpu_data_t *cpu)
 370 {
 371     boolean_t           do_ipi;
 372
 373     if (pmInitDone
 374         && pmDispatch != NULL
 375         && pmDispatch->exitIdle != NULL)
 376         do_ipi = (*pmDispatch->exitIdle)(&cpu->lcpu);
 377     else
 378         do_ipi = TRUE;
 379
 380     return(do_ipi);
 381 }
 382
 383 kern_return_t
 384 pmCPUExitHalt(int cpu)
 385 {
 386     kern_return_t       rc      = KERN_INVALID_ARGUMENT;
 387
 388     if (pmInitDone
 389         && pmDispatch != NULL
 390         && pmDispatch->exitHalt != NULL)
 391         rc = pmDispatch->exitHalt(cpu_to_lcpu(cpu));
 392
 393     return(rc);
 394 }
 395
 396 kern_return_t
 397 pmCPUExitHaltToOff(int cpu)
 398 {
 399     kern_return_t       rc      = KERN_SUCCESS;
 400
 401     if (pmInitDone
 402         && pmDispatch != NULL
 403         && pmDispatch->exitHaltToOff != NULL)
 404         rc = pmDispatch->exitHaltToOff(cpu_to_lcpu(cpu));
 405
 406     return(rc);
 407 }
 408
 409 /*
 410  * Called to initialize the power management structures for the CPUs.
 411  */
 412 void
 413 pmCPUStateInit(void)
 414 {
 415     if (pmDispatch != NULL && pmDispatch->pmCPUStateInit != NULL)
 416         (*pmDispatch->pmCPUStateInit)();
 417     else
 418         earlyTopology = TRUE;
 419 }
 420
 421 /*
 422  * Called when a CPU is being restarted after being powered off (as in S3).
 423  */
 424 void
 425 pmCPUMarkRunning(cpu_data_t *cpu)
 426 {
 427     cpu_data_t  *cpup   = current_cpu_datap();
 428
 429     if (pmInitDone
 430         && pmDispatch != NULL
 431         && pmDispatch->markCPURunning != NULL)
 432         (*pmDispatch->markCPURunning)(&cpu->lcpu);
 433     else
 434         cpup->lcpu.state = LCPU_RUN;
 435 }
 436
 437 /*
 438  * Called to get/set CPU power management state.
 439  */
 440 int
 441 pmCPUControl(uint32_t cmd, void *datap)
 442 {
 443     int         rc      = -1;
 444
 445     if (pmDispatch != NULL
 446         && pmDispatch->pmCPUControl != NULL)
 447         rc = (*pmDispatch->pmCPUControl)(cmd, datap);
 448
 449     return(rc);
 450 }
 451
 452 /*
 453  * Called to save the timer state used by power management prior
 454  * to "sleeping".
 455  */
 456 void
 457 pmTimerSave(void)
 458 {
 459     if (pmDispatch != NULL
 460         && pmDispatch->pmTimerStateSave != NULL)
 461         (*pmDispatch->pmTimerStateSave)();
 462 }
 463
 464 /*
 465  * Called to restore the timer state used by power management after
 466  * waking from "sleep".
 467  */
 468 void
 469 pmTimerRestore(void)
 470 {
 471     if (pmDispatch != NULL
 472         && pmDispatch->pmTimerStateRestore != NULL)
 473         (*pmDispatch->pmTimerStateRestore)();
 474 }
 475
 476 /*
 477  * Set the worst-case time for the C4 to C2 transition.
 478  * No longer does anything.
 479  */
 480 void
 481 ml_set_maxsnoop(__unused uint32_t maxdelay)
 482 {
 483 }
 484
 485
 486 /*
 487  * Get the worst-case time for the C4 to C2 transition.  Returns nanoseconds.
 488  */
 489 unsigned
 490 ml_get_maxsnoop(void)
 491 {
 492     uint64_t    max_snoop       = 0;
 493
 494     if (pmInitDone
 495         && pmDispatch != NULL
 496         && pmDispatch->getMaxSnoop != NULL)
 497         max_snoop = pmDispatch->getMaxSnoop();
 498
 499     return((unsigned)(max_snoop & 0xffffffff));
 500 }
 501
 502
 503 uint32_t
 504 ml_get_maxbusdelay(void)
 505 {
 506     uint64_t    max_delay       = 0;
 507
 508     if (pmInitDone
 509         && pmDispatch != NULL
 510         && pmDispatch->getMaxBusDelay != NULL)
 511         max_delay = pmDispatch->getMaxBusDelay();
 512
 513     return((uint32_t)(max_delay & 0xffffffff));
 514 }
 515
 516 /*
 517  * Set the maximum delay time allowed for snoop on the bus.
 518  *
 519  * Note that this value will be compared to the amount of time that it takes
 520  * to transition from a non-snooping power state (C4) to a snooping state (C2).
 521  * If maxBusDelay is less than C4C2SnoopDelay,
 522  * we will not enter the lowest power state.
 523  */
 524 void
 525 ml_set_maxbusdelay(uint32_t mdelay)
 526 {
 527     uint64_t    maxdelay        = mdelay;
 528
 529     if (pmDispatch != NULL
 530         && pmDispatch->setMaxBusDelay != NULL) {
 531         earlyMaxBusDelay = DELAY_UNSET;
 532         pmDispatch->setMaxBusDelay(maxdelay);
 533     } else
 534         earlyMaxBusDelay = maxdelay;
 535 }
 536
 537 uint64_t
 538 ml_get_maxintdelay(void)
 539 {
 540     uint64_t    max_delay       = 0;
 541
 542     if (pmDispatch != NULL
 543         && pmDispatch->getMaxIntDelay != NULL)
 544         max_delay = pmDispatch->getMaxIntDelay();
 545
 546     return(max_delay);
 547 }
 548
 549 /*
 550  * Set the maximum delay allowed for an interrupt.
 551  */
 552 void
 553 ml_set_maxintdelay(uint64_t mdelay)
 554 {
 555     if (pmDispatch != NULL
 556         && pmDispatch->setMaxIntDelay != NULL) {
 557         earlyMaxIntDelay = DELAY_UNSET;
 558         pmDispatch->setMaxIntDelay(mdelay);
 559     } else
 560         earlyMaxIntDelay = mdelay;
 561 }
 562
 563 boolean_t
 564 ml_get_interrupt_prewake_applicable()
 565 {
 566     boolean_t applicable = FALSE;
 567
 568     if (pmInitDone
 569         && pmDispatch != NULL
 570         && pmDispatch->pmInterruptPrewakeApplicable != NULL)
 571         applicable = pmDispatch->pmInterruptPrewakeApplicable();
 572
 573     return applicable;
 574 }
 575
 576 /*
 577  * Put a CPU into "safe" mode with respect to power.
 578  *
 579  * Some systems cannot operate at a continuous "normal" speed without
 580  * exceeding the thermal design.  This is called per-CPU to place the
 581  * CPUs into a "safe" operating mode.
 582  */
 583 void
 584 pmSafeMode(x86_lcpu_t *lcpu, uint32_t flags)
 585 {
 586     if (pmDispatch != NULL
 587         && pmDispatch->pmCPUSafeMode != NULL)
 588         pmDispatch->pmCPUSafeMode(lcpu, flags);
 589     else {
 590         /*
 591          * Do something reasonable if the KEXT isn't present.
 592          *
 593          * We only look at the PAUSE and RESUME flags.  The other flag(s)
 594          * will not make any sense without the KEXT, so just ignore them.
 595          *
 596          * We set the CPU's state to indicate that it's halted.  If this
 597          * is the CPU we're currently running on, then spin until the
 598          * state becomes non-halted.
 599          */
 600         if (flags & PM_SAFE_FL_PAUSE) {
 601             lcpu->state = LCPU_PAUSE;
 602             if (lcpu == x86_lcpu()) {
 603                 while (lcpu->state == LCPU_PAUSE)
 604                     cpu_pause();
 605             }
 606         }
 607
 608         /*
 609          * Clear the halted flag for the specified CPU, that will
 610          * get it out of it's spin loop.
 611          */
 612         if (flags & PM_SAFE_FL_RESUME) {
 613             lcpu->state = LCPU_RUN;
 614         }
 615     }
 616 }
 617
 618 static uint32_t         saved_run_count = 0;
 619
 620 void
 621 machine_run_count(uint32_t count)
 622 {
 623     if (pmDispatch != NULL
 624         && pmDispatch->pmSetRunCount != NULL)
 625         pmDispatch->pmSetRunCount(count);
 626     else
 627         saved_run_count = count;
 628 }
 629
 630 boolean_t
 631 machine_processor_is_inactive(processor_t processor)
 632 {
 633     int         cpu = processor->cpu_id;
 634
 635     if (pmDispatch != NULL
 636         && pmDispatch->pmIsCPUUnAvailable != NULL)
 637         return(pmDispatch->pmIsCPUUnAvailable(cpu_to_lcpu(cpu)));
 638     else
 639         return(FALSE);
 640 }
 641
 642 processor_t
 643 machine_choose_processor(processor_set_t pset,
 644                          processor_t preferred)
 645 {
 646     int         startCPU;
 647     int         endCPU;
 648     int         preferredCPU;
 649     int         chosenCPU;
 650
 651     if (!pmInitDone)
 652         return(preferred);
 653
 654     if (pset == NULL) {
 655         startCPU = -1;
 656         endCPU = -1;
 657     } else {
 658         startCPU = pset->cpu_set_low;
 659         endCPU = pset->cpu_set_hi;
 660     }
 661
 662     if (preferred == NULL)
 663         preferredCPU = -1;
 664     else
 665         preferredCPU = preferred->cpu_id;
 666
 667     if (pmDispatch != NULL
 668         && pmDispatch->pmChooseCPU != NULL) {
 669         chosenCPU = pmDispatch->pmChooseCPU(startCPU, endCPU, preferredCPU);
 670
 671         if (chosenCPU == -1)
 672             return(NULL);
 673         return(cpu_datap(chosenCPU)->cpu_processor);
 674     }
 675
 676     return(preferred);
 677 }
 678
 679 static int
 680 pmThreadGetUrgency(uint64_t *rt_period, uint64_t *rt_deadline)
 681 {
 682
 683     return(thread_get_urgency(rt_period, rt_deadline));
 684 }
 685
 686 #if     DEBUG
 687 uint32_t        urgency_stats[64][THREAD_URGENCY_MAX];
 688 #endif
 689
 690 #define         URGENCY_NOTIFICATION_ASSERT_NS (5 * 1000 * 1000)
 691 uint64_t        urgency_notification_assert_abstime_threshold, urgency_notification_max_recorded;
 692
 693 void
 694 thread_tell_urgency(int urgency,
 695     uint64_t rt_period,
 696     uint64_t rt_deadline)
 697 {
 698         uint64_t        urgency_notification_time_start, delta;
 699         boolean_t       urgency_assert = (urgency_notification_assert_abstime_threshold != 0);
 700         assert(get_preemption_level() > 0 || ml_get_interrupts_enabled() == FALSE);
 701 #if     DEBUG
 702         urgency_stats[cpu_number() % 64][urgency]++;
 703 #endif
 704         if (!pmInitDone
 705             || pmDispatch == NULL
 706             || pmDispatch->pmThreadTellUrgency == NULL)
 707                 return;
 708
 709         KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED,MACH_URGENCY) | DBG_FUNC_START, urgency, rt_period, (rt_deadline >> 32), rt_deadline, 0);
 710
 711         if (__improbable((urgency_assert == TRUE)))
 712                 urgency_notification_time_start = mach_absolute_time();
 713
 714         pmDispatch->pmThreadTellUrgency(urgency, rt_period, rt_deadline);
 715
 716         if (__improbable((urgency_assert == TRUE))) {
 717                 delta = mach_absolute_time() - urgency_notification_time_start;
 718
 719                 if (__improbable(delta > urgency_notification_max_recorded)) {
 720                         /* This is not synchronized, but it doesn't matter
 721                          * if we (rarely) miss an event, as it is statistically
 722                          * unlikely that it will never recur.
 723                          */
 724                         urgency_notification_max_recorded = delta;
 725
 726                         if (__improbable((delta > urgency_notification_assert_abstime_threshold) && !machine_timeout_suspended()))
 727                                 panic("Urgency notification callout %p exceeded threshold, 0x%llx abstime units", pmDispatch->pmThreadTellUrgency, delta);
 728                 }
 729         }
 730
 731         KERNEL_DEBUG_CONSTANT(MACHDBG_CODE(DBG_MACH_SCHED,MACH_URGENCY) | DBG_FUNC_END, urgency, rt_period, (rt_deadline >> 32), rt_deadline, 0);
 732 }
 733
 734 void
 735 active_rt_threads(boolean_t active)
 736 {
 737     if (!pmInitDone
 738         || pmDispatch == NULL
 739         || pmDispatch->pmActiveRTThreads == NULL)
 740         return;
 741
 742     pmDispatch->pmActiveRTThreads(active);
 743 }
 744
 745 static uint32_t
 746 pmGetSavedRunCount(void)
 747 {
 748     return(saved_run_count);
 749 }
 750
 751 /*
 752  * Returns the root of the package tree.
 753  */
 754 x86_pkg_t *
 755 pmGetPkgRoot(void)
 756 {
 757     return(x86_pkgs);
 758 }
 759
 760 static boolean_t
 761 pmCPUGetHibernate(int cpu)
 762 {
 763     return(cpu_datap(cpu)->cpu_hibernate);
 764 }
 765
 766 processor_t
 767 pmLCPUtoProcessor(int lcpu)
 768 {
 769     return(cpu_datap(lcpu)->cpu_processor);
 770 }
 771
 772 static void
 773 pmReSyncDeadlines(int cpu)
 774 {
 775     static boolean_t    registered      = FALSE;
 776
 777     if (!registered) {
 778         PM_interrupt_register(&etimer_resync_deadlines);
 779         registered = TRUE;
 780     }
 781
 782     if ((uint32_t)cpu == current_cpu_datap()->lcpu.cpu_num)
 783         etimer_resync_deadlines();
 784     else
 785         cpu_PM_interrupt(cpu);
 786 }
 787
 788 static void
 789 pmSendIPI(int cpu)
 790 {
 791     lapic_send_ipi(cpu, LAPIC_PM_INTERRUPT);
 792 }
 793
 794 static void
 795 pmGetNanotimeInfo(pm_rtc_nanotime_t *rtc_nanotime)
 796 {
 797         /*
 798          * Make sure that nanotime didn't change while we were reading it.
 799          */
 800         do {
 801                 rtc_nanotime->generation = pal_rtc_nanotime_info.generation; /* must be first */
 802                 rtc_nanotime->tsc_base = pal_rtc_nanotime_info.tsc_base;
 803                 rtc_nanotime->ns_base = pal_rtc_nanotime_info.ns_base;
 804                 rtc_nanotime->scale = pal_rtc_nanotime_info.scale;
 805                 rtc_nanotime->shift = pal_rtc_nanotime_info.shift;
 806         } while(pal_rtc_nanotime_info.generation != 0
 807                 && rtc_nanotime->generation != pal_rtc_nanotime_info.generation);
 808 }
 809
 810 uint32_t
 811 pmTimerQueueMigrate(int target_cpu)
 812 {
 813     /* Call the etimer code to do this. */
 814     return (target_cpu != cpu_number())
 815                 ? etimer_queue_migrate(target_cpu)
 816                 : 0;
 817 }
 818
 819
 820 /*
 821  * Called by the power management kext to register itself and to get the
 822  * callbacks it might need into other kernel functions.  This interface
 823  * is versioned to allow for slight mis-matches between the kext and the
 824  * kernel.
 825  */
 826 void
 827 pmKextRegister(uint32_t version, pmDispatch_t *cpuFuncs,
 828                pmCallBacks_t *callbacks)
 829 {
 830     if (callbacks != NULL && version == PM_DISPATCH_VERSION) {
 831         callbacks->setRTCPop            = setPop;
 832         callbacks->resyncDeadlines      = pmReSyncDeadlines;
 833         callbacks->initComplete         = pmInitComplete;
 834         callbacks->GetLCPU              = pmGetLogicalCPU;
 835         callbacks->GetCore              = pmGetCore;
 836         callbacks->GetDie               = pmGetDie;
 837         callbacks->GetPackage           = pmGetPackage;
 838         callbacks->GetMyLCPU            = pmGetMyLogicalCPU;
 839         callbacks->GetMyCore            = pmGetMyCore;
 840         callbacks->GetMyDie             = pmGetMyDie;
 841         callbacks->GetMyPackage         = pmGetMyPackage;
 842         callbacks->GetPkgRoot           = pmGetPkgRoot;
 843         callbacks->LockCPUTopology      = pmLockCPUTopology;
 844         callbacks->GetHibernate         = pmCPUGetHibernate;
 845         callbacks->LCPUtoProcessor      = pmLCPUtoProcessor;
 846         callbacks->ThreadBind           = thread_bind;
 847         callbacks->GetSavedRunCount     = pmGetSavedRunCount;
 848         callbacks->GetNanotimeInfo      = pmGetNanotimeInfo;
 849         callbacks->ThreadGetUrgency     = pmThreadGetUrgency;
 850         callbacks->RTCClockAdjust       = rtc_clock_adjust;
 851         callbacks->timerQueueMigrate    = pmTimerQueueMigrate;
 852         callbacks->topoParms            = &topoParms;
 853         callbacks->pmSendIPI            = pmSendIPI;
 854         callbacks->InterruptPending     = lapic_is_interrupt_pending;
 855         callbacks->IsInterrupting       = lapic_is_interrupting;
 856         callbacks->InterruptStats       = lapic_interrupt_counts;
 857         callbacks->DisableApicTimer     = lapic_disable_timer;
 858     } else {
 859         panic("Version mis-match between Kernel and CPU PM");
 860     }
 861
 862     if (cpuFuncs != NULL) {
 863         if (pmDispatch) {
 864             panic("Attempt to re-register power management interface--AICPM present in xcpm mode? %p->%p", pmDispatch, cpuFuncs);
 865         }
 866
 867         pmDispatch = cpuFuncs;
 868
 869         if (earlyTopology
 870             && pmDispatch->pmCPUStateInit != NULL) {
 871             (*pmDispatch->pmCPUStateInit)();
 872             earlyTopology = FALSE;
 873         }
 874
 875         if (pmDispatch->pmIPIHandler != NULL) {
 876             lapic_set_pm_func((i386_intr_func_t)pmDispatch->pmIPIHandler);
 877         }
 878     }
 879 }
 880
 881 /*
 882  * Unregisters the power management functions from the kext.
 883  */
 884 void
 885 pmUnRegister(pmDispatch_t *cpuFuncs)
 886 {
 887     if (cpuFuncs != NULL && pmDispatch == cpuFuncs) {
 888         pmDispatch = NULL;
 889     }
 890 }
 891
 892 /******************************************************************************
 893  *
 894  * All of the following are deprecated interfaces and no longer used.
 895  *
 896  ******************************************************************************/
 897 kern_return_t
 898 pmsControl(__unused uint32_t request, __unused user_addr_t reqaddr,
 899            __unused uint32_t reqsize)
 900 {
 901     return(KERN_SUCCESS);
 902 }
 903
 904 void
 905 pmsInit(void)
 906 {
 907 }
 908
 909 void
 910 pmsStart(void)
 911 {
 912 }
 913
 914 void
 915 pmsPark(void)
 916 {
 917 }
 918
 919 void
 920 pmsRun(__unused uint32_t nstep)
 921 {
 922 }
 923
 924 kern_return_t
 925 pmsBuild(__unused pmsDef *pd, __unused uint32_t pdsize,
 926          __unused pmsSetFunc_t *functab,
 927          __unused uint32_t platformData, __unused pmsQueryFunc_t queryFunc)
 928 {
 929     return(KERN_SUCCESS);
 930 }
 931
 932 void machine_track_platform_idle(boolean_t entry) {
 933         cpu_data_t              *my_cpu         = current_cpu_datap();
 934
 935         if (entry) {
 936                 (void)__sync_fetch_and_add(&my_cpu->lcpu.package->num_idle, 1);
 937         }
 938         else {
 939                 uint32_t nidle = __sync_fetch_and_sub(&my_cpu->lcpu.package->num_idle, 1);
 940                 if (nidle == topoParms.nLThreadsPerPackage) {
 941                         my_cpu->lcpu.package->package_idle_exits++;
 942                 }
 943         }
 944 }