osfmk/arm/machine_routines_common.c

   1 /*
   2  * Copyright (c) 2007-2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <arm/machine_cpu.h>
  30 #include <arm/cpu_internal.h>
  31 #include <arm/cpuid.h>
  32 #include <arm/cpu_data.h>
  33 #include <arm/cpu_data_internal.h>
  34 #include <arm/misc_protos.h>
  35 #include <arm/machdep_call.h>
  36 #include <arm/machine_routines.h>
  37 #include <arm/rtclock.h>
  38 #include <kern/machine.h>
  39 #include <kern/thread.h>
  40 #include <kern/thread_group.h>
  41 #include <kern/policy_internal.h>
  42 #include <machine/config.h>
  43
  44 #if MONOTONIC
  45 #include <kern/monotonic.h>
  46 #include <machine/monotonic.h>
  47 #endif /* MONOTONIC */
  48
  49 #include <mach/machine.h>
  50
  51 #if INTERRUPT_MASKED_DEBUG
  52 extern boolean_t interrupt_masked_debug;
  53 extern uint64_t interrupt_masked_timeout;
  54 #endif
  55
  56 extern uint64_t mach_absolutetime_asleep;
  57
  58 static void
  59 sched_perfcontrol_oncore_default(perfcontrol_state_t new_thread_state __unused, going_on_core_t on __unused)
  60 {
  61 }
  62
  63 static void
  64 sched_perfcontrol_switch_default(perfcontrol_state_t old_thread_state __unused, perfcontrol_state_t new_thread_state __unused)
  65 {
  66 }
  67
  68 static void
  69 sched_perfcontrol_offcore_default(perfcontrol_state_t old_thread_state __unused, going_off_core_t off __unused, boolean_t thread_terminating __unused)
  70 {
  71 }
  72
  73 static void
  74 sched_perfcontrol_thread_group_default(thread_group_data_t data __unused)
  75 {
  76 }
  77
  78 static void
  79 sched_perfcontrol_max_runnable_latency_default(perfcontrol_max_runnable_latency_t latencies __unused)
  80 {
  81 }
  82
  83 static void
  84 sched_perfcontrol_work_interval_notify_default(perfcontrol_state_t thread_state __unused,
  85                                                perfcontrol_work_interval_t work_interval __unused)
  86 {
  87 }
  88
  89 static void
  90 sched_perfcontrol_work_interval_ctl_default(perfcontrol_state_t thread_state __unused,
  91                                             perfcontrol_work_interval_instance_t instance __unused)
  92 {
  93 }
  94
  95 static void
  96 sched_perfcontrol_deadline_passed_default(__unused uint64_t deadline)
  97 {
  98 }
  99
 100 static void
 101 sched_perfcontrol_csw_default(
 102         __unused perfcontrol_event event, __unused uint32_t cpu_id, __unused uint64_t timestamp,
 103         __unused uint32_t flags, __unused struct perfcontrol_thread_data *offcore,
 104         __unused struct perfcontrol_thread_data *oncore,
 105         __unused struct perfcontrol_cpu_counters *cpu_counters, __unused void *unused)
 106 {
 107 }
 108
 109 static void
 110 sched_perfcontrol_state_update_default(
 111         __unused perfcontrol_event event, __unused uint32_t cpu_id, __unused uint64_t timestamp,
 112         __unused uint32_t flags, __unused struct perfcontrol_thread_data *thr_data,
 113         __unused void *unused)
 114 {
 115 }
 116
 117 sched_perfcontrol_offcore_t                     sched_perfcontrol_offcore = sched_perfcontrol_offcore_default;
 118 sched_perfcontrol_context_switch_t              sched_perfcontrol_switch = sched_perfcontrol_switch_default;
 119 sched_perfcontrol_oncore_t                      sched_perfcontrol_oncore = sched_perfcontrol_oncore_default;
 120 sched_perfcontrol_thread_group_init_t           sched_perfcontrol_thread_group_init = sched_perfcontrol_thread_group_default;
 121 sched_perfcontrol_thread_group_deinit_t         sched_perfcontrol_thread_group_deinit = sched_perfcontrol_thread_group_default;
 122 sched_perfcontrol_thread_group_flags_update_t   sched_perfcontrol_thread_group_flags_update = sched_perfcontrol_thread_group_default;
 123 sched_perfcontrol_max_runnable_latency_t        sched_perfcontrol_max_runnable_latency = sched_perfcontrol_max_runnable_latency_default;
 124 sched_perfcontrol_work_interval_notify_t        sched_perfcontrol_work_interval_notify = sched_perfcontrol_work_interval_notify_default;
 125 sched_perfcontrol_work_interval_ctl_t           sched_perfcontrol_work_interval_ctl = sched_perfcontrol_work_interval_ctl_default;
 126 sched_perfcontrol_deadline_passed_t             sched_perfcontrol_deadline_passed = sched_perfcontrol_deadline_passed_default;
 127 sched_perfcontrol_csw_t                         sched_perfcontrol_csw = sched_perfcontrol_csw_default;
 128 sched_perfcontrol_state_update_t                sched_perfcontrol_state_update = sched_perfcontrol_state_update_default;
 129
 130 void
 131 sched_perfcontrol_register_callbacks(sched_perfcontrol_callbacks_t callbacks, unsigned long size_of_state)
 132 {
 133         assert(callbacks == NULL || callbacks->version >= SCHED_PERFCONTROL_CALLBACKS_VERSION_2);
 134
 135         if (size_of_state > sizeof(struct perfcontrol_state)) {
 136                 panic("%s: Invalid required state size %lu", __FUNCTION__, size_of_state);
 137         }
 138
 139         if (callbacks) {
 140
 141
 142                 if (callbacks->version >= SCHED_PERFCONTROL_CALLBACKS_VERSION_7) {
 143                         if (callbacks->work_interval_ctl != NULL) {
 144                                 sched_perfcontrol_work_interval_ctl = callbacks->work_interval_ctl;
 145                         } else {
 146                                 sched_perfcontrol_work_interval_ctl = sched_perfcontrol_work_interval_ctl_default;
 147                         }
 148                 }
 149
 150                 if (callbacks->version >= SCHED_PERFCONTROL_CALLBACKS_VERSION_5) {
 151                         if (callbacks->csw != NULL) {
 152                                 sched_perfcontrol_csw = callbacks->csw;
 153                         } else {
 154                                 sched_perfcontrol_csw = sched_perfcontrol_csw_default;
 155                         }
 156
 157                         if (callbacks->state_update != NULL) {
 158                                 sched_perfcontrol_state_update = callbacks->state_update;
 159                         } else {
 160                                 sched_perfcontrol_state_update = sched_perfcontrol_state_update_default;
 161                         }
 162                 }
 163
 164                 if (callbacks->version >= SCHED_PERFCONTROL_CALLBACKS_VERSION_4) {
 165                         if (callbacks->deadline_passed != NULL) {
 166                                 sched_perfcontrol_deadline_passed = callbacks->deadline_passed;
 167                         } else {
 168                                 sched_perfcontrol_deadline_passed = sched_perfcontrol_deadline_passed_default;
 169                         }
 170                 }
 171
 172                 if (callbacks->offcore != NULL) {
 173                         sched_perfcontrol_offcore = callbacks->offcore;
 174                 } else {
 175                         sched_perfcontrol_offcore = sched_perfcontrol_offcore_default;
 176                 }
 177
 178                 if (callbacks->context_switch != NULL) {
 179                         sched_perfcontrol_switch = callbacks->context_switch;
 180                 } else {
 181                         sched_perfcontrol_switch = sched_perfcontrol_switch_default;
 182                 }
 183
 184                 if (callbacks->oncore != NULL) {
 185                         sched_perfcontrol_oncore = callbacks->oncore;
 186                 } else {
 187                         sched_perfcontrol_oncore = sched_perfcontrol_oncore_default;
 188                 }
 189
 190                 if (callbacks->max_runnable_latency != NULL) {
 191                         sched_perfcontrol_max_runnable_latency = callbacks->max_runnable_latency;
 192                 } else {
 193                         sched_perfcontrol_max_runnable_latency = sched_perfcontrol_max_runnable_latency_default;
 194                 }
 195
 196                 if (callbacks->work_interval_notify != NULL) {
 197                         sched_perfcontrol_work_interval_notify = callbacks->work_interval_notify;
 198                 } else {
 199                         sched_perfcontrol_work_interval_notify = sched_perfcontrol_work_interval_notify_default;
 200                 }
 201         } else {
 202                 /* reset to defaults */
 203                 sched_perfcontrol_offcore = sched_perfcontrol_offcore_default;
 204                 sched_perfcontrol_switch = sched_perfcontrol_switch_default;
 205                 sched_perfcontrol_oncore = sched_perfcontrol_oncore_default;
 206                 sched_perfcontrol_thread_group_init = sched_perfcontrol_thread_group_default;
 207                 sched_perfcontrol_thread_group_deinit = sched_perfcontrol_thread_group_default;
 208                 sched_perfcontrol_thread_group_flags_update = sched_perfcontrol_thread_group_default;
 209                 sched_perfcontrol_max_runnable_latency = sched_perfcontrol_max_runnable_latency_default;
 210                 sched_perfcontrol_work_interval_notify = sched_perfcontrol_work_interval_notify_default;
 211                 sched_perfcontrol_work_interval_ctl = sched_perfcontrol_work_interval_ctl_default;
 212                 sched_perfcontrol_csw = sched_perfcontrol_csw_default;
 213                 sched_perfcontrol_state_update = sched_perfcontrol_state_update_default;
 214         }
 215 }
 216
 217
 218 static void
 219 machine_switch_populate_perfcontrol_thread_data(struct perfcontrol_thread_data *data,
 220                                                 thread_t thread,
 221                                                 uint64_t same_pri_latency)
 222 {
 223         bzero(data, sizeof(struct perfcontrol_thread_data));
 224         data->perfctl_class = thread_get_perfcontrol_class(thread);
 225         data->energy_estimate_nj = 0;
 226         data->thread_id = thread->thread_id;
 227         data->scheduling_latency_at_same_basepri = same_pri_latency;
 228         data->perfctl_state = FIND_PERFCONTROL_STATE(thread);
 229 }
 230
 231 static void
 232 machine_switch_populate_perfcontrol_cpu_counters(struct perfcontrol_cpu_counters *cpu_counters)
 233 {
 234 #if MONOTONIC
 235         mt_perfcontrol(&cpu_counters->instructions, &cpu_counters->cycles);
 236 #else /* MONOTONIC */
 237         cpu_counters->instructions = 0;
 238         cpu_counters->cycles = 0;
 239 #endif /* !MONOTONIC */
 240 }
 241
 242 int perfcontrol_callout_stats_enabled = 0;
 243 static _Atomic uint64_t perfcontrol_callout_stats[PERFCONTROL_CALLOUT_MAX][PERFCONTROL_STAT_MAX];
 244 static _Atomic uint64_t perfcontrol_callout_count[PERFCONTROL_CALLOUT_MAX];
 245
 246 #if MONOTONIC
 247 static inline
 248 bool perfcontrol_callout_counters_begin(uint64_t *counters)
 249 {
 250     if (!perfcontrol_callout_stats_enabled)
 251         return false;
 252     mt_fixed_counts(counters);
 253     return true;
 254 }
 255
 256 static inline
 257 void perfcontrol_callout_counters_end(uint64_t *start_counters,
 258         perfcontrol_callout_type_t type)
 259 {
 260     uint64_t end_counters[MT_CORE_NFIXED];
 261     mt_fixed_counts(end_counters);
 262     atomic_fetch_add_explicit(&perfcontrol_callout_stats[type][PERFCONTROL_STAT_CYCLES],
 263             end_counters[MT_CORE_CYCLES] - start_counters[MT_CORE_CYCLES], memory_order_relaxed);
 264 #ifdef MT_CORE_INSTRS
 265     atomic_fetch_add_explicit(&perfcontrol_callout_stats[type][PERFCONTROL_STAT_INSTRS],
 266             end_counters[MT_CORE_INSTRS] - start_counters[MT_CORE_INSTRS], memory_order_relaxed);
 267 #endif /* defined(MT_CORE_INSTRS) */
 268     atomic_fetch_add_explicit(&perfcontrol_callout_count[type], 1, memory_order_relaxed);
 269 }
 270 #endif /* MONOTONIC */
 271
 272 uint64_t perfcontrol_callout_stat_avg(perfcontrol_callout_type_t type,
 273         perfcontrol_callout_stat_t stat)
 274 {
 275     if (!perfcontrol_callout_stats_enabled)
 276         return 0;
 277     return (perfcontrol_callout_stats[type][stat] / perfcontrol_callout_count[type]);
 278 }
 279
 280 void
 281 machine_switch_perfcontrol_context(perfcontrol_event event,
 282                                    uint64_t timestamp,
 283                                    uint32_t flags,
 284                                    uint64_t new_thread_same_pri_latency,
 285                                    thread_t old,
 286                                    thread_t new)
 287 {
 288         if (sched_perfcontrol_switch != sched_perfcontrol_switch_default) {
 289                 perfcontrol_state_t old_perfcontrol_state = FIND_PERFCONTROL_STATE(old);
 290                 perfcontrol_state_t new_perfcontrol_state = FIND_PERFCONTROL_STATE(new);
 291                 sched_perfcontrol_switch(old_perfcontrol_state, new_perfcontrol_state);
 292         }
 293
 294         if (sched_perfcontrol_csw != sched_perfcontrol_csw_default) {
 295                 uint32_t cpu_id = (uint32_t)cpu_number();
 296                 struct perfcontrol_cpu_counters cpu_counters;
 297                 struct perfcontrol_thread_data offcore, oncore;
 298                 machine_switch_populate_perfcontrol_thread_data(&offcore, old, 0);
 299                 machine_switch_populate_perfcontrol_thread_data(&oncore, new,
 300                         new_thread_same_pri_latency);
 301                 machine_switch_populate_perfcontrol_cpu_counters(&cpu_counters);
 302
 303 #if MONOTONIC
 304                 uint64_t counters[MT_CORE_NFIXED];
 305                 bool ctrs_enabled = perfcontrol_callout_counters_begin(counters);
 306 #endif /* MONOTONIC */
 307                 sched_perfcontrol_csw(event, cpu_id, timestamp, flags,
 308                         &offcore, &oncore, &cpu_counters, NULL);
 309 #if MONOTONIC
 310                 if (ctrs_enabled) perfcontrol_callout_counters_end(counters, PERFCONTROL_CALLOUT_CONTEXT);
 311 #endif /* MONOTONIC */
 312
 313 #if __arm64__
 314                 old->machine.energy_estimate_nj += offcore.energy_estimate_nj;
 315                 new->machine.energy_estimate_nj += oncore.energy_estimate_nj;
 316 #endif
 317         }
 318 }
 319
 320 void
 321 machine_switch_perfcontrol_state_update(perfcontrol_event event,
 322                                         uint64_t timestamp,
 323                                         uint32_t flags,
 324                                         thread_t thread)
 325 {
 326         if (sched_perfcontrol_state_update == sched_perfcontrol_state_update_default)
 327                 return;
 328         uint32_t cpu_id = (uint32_t)cpu_number();
 329         struct perfcontrol_thread_data data;
 330         machine_switch_populate_perfcontrol_thread_data(&data, thread, 0);
 331
 332 #if MONOTONIC
 333         uint64_t counters[MT_CORE_NFIXED];
 334         bool ctrs_enabled = perfcontrol_callout_counters_begin(counters);
 335 #endif /* MONOTONIC */
 336         sched_perfcontrol_state_update(event, cpu_id, timestamp, flags,
 337                 &data, NULL);
 338 #if MONOTONIC
 339         if (ctrs_enabled) perfcontrol_callout_counters_end(counters, PERFCONTROL_CALLOUT_STATE_UPDATE);
 340 #endif /* MONOTONIC */
 341
 342 #if __arm64__
 343         thread->machine.energy_estimate_nj += data.energy_estimate_nj;
 344 #endif
 345 }
 346
 347 void
 348 machine_thread_going_on_core(thread_t   new_thread,
 349                              int        urgency,
 350                              uint64_t   sched_latency,
 351                              uint64_t   same_pri_latency,
 352                              uint64_t   timestamp)
 353 {
 354
 355         if (sched_perfcontrol_oncore == sched_perfcontrol_oncore_default)
 356                 return;
 357         struct going_on_core on_core;
 358         perfcontrol_state_t state = FIND_PERFCONTROL_STATE(new_thread);
 359
 360         on_core.thread_id = new_thread->thread_id;
 361         on_core.energy_estimate_nj = 0;
 362         on_core.qos_class = proc_get_effective_thread_policy(new_thread, TASK_POLICY_QOS);
 363         on_core.urgency = urgency;
 364         on_core.is_32_bit = thread_is_64bit(new_thread) ? FALSE : TRUE;
 365         on_core.is_kernel_thread = new_thread->task == kernel_task;
 366         on_core.scheduling_latency = sched_latency;
 367         on_core.start_time = timestamp;
 368         on_core.scheduling_latency_at_same_basepri = same_pri_latency;
 369
 370 #if MONOTONIC
 371         uint64_t counters[MT_CORE_NFIXED];
 372         bool ctrs_enabled = perfcontrol_callout_counters_begin(counters);
 373 #endif /* MONOTONIC */
 374         sched_perfcontrol_oncore(state, &on_core);
 375 #if MONOTONIC
 376         if (ctrs_enabled) perfcontrol_callout_counters_end(counters, PERFCONTROL_CALLOUT_ON_CORE);
 377 #endif /* MONOTONIC */
 378
 379 #if __arm64__
 380         new_thread->machine.energy_estimate_nj += on_core.energy_estimate_nj;
 381 #endif
 382 }
 383
 384 void
 385 machine_thread_going_off_core(thread_t old_thread, boolean_t thread_terminating, uint64_t last_dispatch)
 386 {
 387         if (sched_perfcontrol_offcore == sched_perfcontrol_offcore_default)
 388                 return;
 389         struct going_off_core off_core;
 390         perfcontrol_state_t state = FIND_PERFCONTROL_STATE(old_thread);
 391
 392         off_core.thread_id = old_thread->thread_id;
 393         off_core.energy_estimate_nj = 0;
 394         off_core.end_time = last_dispatch;
 395
 396 #if MONOTONIC
 397         uint64_t counters[MT_CORE_NFIXED];
 398         bool ctrs_enabled = perfcontrol_callout_counters_begin(counters);
 399 #endif /* MONOTONIC */
 400         sched_perfcontrol_offcore(state, &off_core, thread_terminating);
 401 #if MONOTONIC
 402         if (ctrs_enabled) perfcontrol_callout_counters_end(counters, PERFCONTROL_CALLOUT_OFF_CORE);
 403 #endif /* MONOTONIC */
 404
 405 #if __arm64__
 406         old_thread->machine.energy_estimate_nj += off_core.energy_estimate_nj;
 407 #endif
 408 }
 409
 410
 411 void
 412 machine_max_runnable_latency(uint64_t bg_max_latency,
 413                                                          uint64_t default_max_latency,
 414                                                          uint64_t realtime_max_latency)
 415 {
 416         if (sched_perfcontrol_max_runnable_latency == sched_perfcontrol_max_runnable_latency_default)
 417                 return;
 418         struct perfcontrol_max_runnable_latency latencies = {
 419                 .max_scheduling_latencies = {
 420                         [THREAD_URGENCY_NONE] = 0,
 421                         [THREAD_URGENCY_BACKGROUND] = bg_max_latency,
 422                         [THREAD_URGENCY_NORMAL] = default_max_latency,
 423                         [THREAD_URGENCY_REAL_TIME] = realtime_max_latency
 424                 }
 425         };
 426
 427         sched_perfcontrol_max_runnable_latency(&latencies);
 428 }
 429
 430 void
 431 machine_work_interval_notify(thread_t thread,
 432                              struct kern_work_interval_args* kwi_args)
 433 {
 434         if (sched_perfcontrol_work_interval_notify == sched_perfcontrol_work_interval_notify_default)
 435                 return;
 436         perfcontrol_state_t state = FIND_PERFCONTROL_STATE(thread);
 437         struct perfcontrol_work_interval work_interval = {
 438                 .thread_id      = thread->thread_id,
 439                 .qos_class      = proc_get_effective_thread_policy(thread, TASK_POLICY_QOS),
 440                 .urgency        = kwi_args->urgency,
 441                 .flags          = kwi_args->notify_flags,
 442                 .work_interval_id = kwi_args->work_interval_id,
 443                 .start          = kwi_args->start,
 444                 .finish         = kwi_args->finish,
 445                 .deadline       = kwi_args->deadline,
 446                 .next_start     = kwi_args->next_start,
 447                 .create_flags   = kwi_args->create_flags,
 448         };
 449         sched_perfcontrol_work_interval_notify(state, &work_interval);
 450 }
 451
 452
 453 void
 454 machine_perfcontrol_deadline_passed(uint64_t deadline)
 455 {
 456         if (sched_perfcontrol_deadline_passed != sched_perfcontrol_deadline_passed_default)
 457                 sched_perfcontrol_deadline_passed(deadline);
 458 }
 459
 460 #if INTERRUPT_MASKED_DEBUG
 461 /*
 462  * ml_spin_debug_reset()
 463  * Reset the timestamp on a thread that has been unscheduled
 464  * to avoid false alarms.    Alarm will go off if interrupts are held
 465  * disabled for too long, starting from now.
 466  */
 467 void
 468 ml_spin_debug_reset(thread_t thread)
 469 {
 470     thread->machine.intmask_timestamp = mach_absolute_time();
 471 }
 472
 473 /*
 474  * ml_spin_debug_clear()
 475  * Clear the timestamp on a thread that has been unscheduled
 476  * to avoid false alarms
 477  */
 478 void
 479 ml_spin_debug_clear(thread_t thread)
 480 {
 481     thread->machine.intmask_timestamp = 0;
 482 }
 483
 484 /*
 485  * ml_spin_debug_clear_self()
 486  * Clear the timestamp on the current thread to prevent
 487  * false alarms
 488  */
 489 void
 490 ml_spin_debug_clear_self()
 491 {
 492         ml_spin_debug_clear(current_thread());
 493 }
 494
 495 void
 496 ml_check_interrupts_disabled_duration(thread_t thread)
 497 {
 498     uint64_t start;
 499     uint64_t now;
 500
 501     start = thread->machine.intmask_timestamp;
 502     if (start != 0) {
 503         now = mach_absolute_time();
 504
 505         if ((now - start) > interrupt_masked_timeout) {
 506             mach_timebase_info_data_t timebase;
 507             clock_timebase_info(&timebase);
 508
 509 #ifndef KASAN
 510             /*
 511              * Disable the actual panic for KASAN due to the overhead of KASAN itself, leave the rest of the
 512              * mechanism enabled so that KASAN can catch any bugs in the mechanism itself.
 513              */
 514             panic("Interrupts held disabled for %llu nanoseconds", (((now - start) * timebase.numer)/timebase.denom));
 515 #endif
 516         }
 517     }
 518
 519     return;
 520 }
 521 #endif // INTERRUPT_MASKED_DEBUG
 522
 523
 524 boolean_t
 525 ml_set_interrupts_enabled(boolean_t enable)
 526 {
 527     thread_t    thread;
 528     uint64_t    state;
 529
 530 #if __arm__
 531 #define INTERRUPT_MASK PSR_IRQF
 532     state = __builtin_arm_rsr("cpsr");
 533 #else
 534 #define INTERRUPT_MASK DAIF_IRQF
 535     state = __builtin_arm_rsr("DAIF");
 536 #endif
 537     if (enable) {
 538 #if INTERRUPT_MASKED_DEBUG
 539         if (interrupt_masked_debug && (state & INTERRUPT_MASK)) {
 540             // Interrupts are currently masked, we will enable them (after finishing this check)
 541             thread = current_thread();
 542             ml_check_interrupts_disabled_duration(thread);
 543             thread->machine.intmask_timestamp = 0;
 544         }
 545 #endif  // INTERRUPT_MASKED_DEBUG
 546         if (get_preemption_level() == 0) {
 547             thread = current_thread();
 548             while (thread->machine.CpuDatap->cpu_pending_ast & AST_URGENT) {
 549 #if __ARM_USER_PROTECT__
 550                 uintptr_t up = arm_user_protect_begin(thread);
 551 #endif
 552                 ast_taken_kernel();
 553 #if __ARM_USER_PROTECT__
 554                 arm_user_protect_end(thread, up, FALSE);
 555 #endif
 556             }
 557         }
 558 #if __arm__
 559         __asm__ volatile ("cpsie if" ::: "memory"); // Enable IRQ FIQ
 560 #else
 561         __builtin_arm_wsr("DAIFClr", (DAIFSC_IRQF | DAIFSC_FIQF));
 562 #endif
 563     } else {
 564 #if __arm__
 565         __asm__ volatile ("cpsid if" ::: "memory"); // Mask IRQ FIQ
 566 #else
 567         __builtin_arm_wsr("DAIFSet", (DAIFSC_IRQF | DAIFSC_FIQF));
 568 #endif
 569 #if INTERRUPT_MASKED_DEBUG
 570         if (interrupt_masked_debug && ((state & INTERRUPT_MASK) == 0)) {
 571             // Interrupts were enabled, we just masked them
 572             current_thread()->machine.intmask_timestamp = mach_absolute_time();
 573         }
 574 #endif
 575     }
 576     return ((state & INTERRUPT_MASK) == 0);
 577 }
 578
 579 static boolean_t ml_quiescing;
 580
 581 void ml_set_is_quiescing(boolean_t quiescing)
 582 {
 583     assert(FALSE == ml_get_interrupts_enabled());
 584     ml_quiescing = quiescing;
 585 }
 586
 587 boolean_t ml_is_quiescing(void)
 588 {
 589     assert(FALSE == ml_get_interrupts_enabled());
 590     return (ml_quiescing);
 591 }
 592
 593 uint64_t ml_get_booter_memory_size(void)
 594 {
 595     enum { kRoundSize = 512*1024*1024ULL };
 596         uint64_t size;
 597         size = BootArgs->memSizeActual;
 598     if (!size)
 599     {
 600                 size  = BootArgs->memSize;
 601                 size  = (size + kRoundSize - 1) & ~(kRoundSize - 1);
 602                 size -= BootArgs->memSize;
 603     }
 604     return (size);
 605 }
 606
 607 uint64_t
 608 ml_get_abstime_offset(void)
 609 {
 610         return rtclock_base_abstime;
 611 }
 612
 613 uint64_t
 614 ml_get_conttime_offset(void)
 615 {
 616         return (rtclock_base_abstime + mach_absolutetime_asleep);
 617 }
 618
 619 uint64_t
 620 ml_get_time_since_reset(void)
 621 {
 622         /* The timebase resets across S2R, so just return the raw value. */
 623         return ml_get_hwclock();
 624 }
 625
 626 uint64_t
 627 ml_get_conttime_wake_time(void)
 628 {
 629         /* The wake time is simply our continuous time offset. */
 630         return ml_get_conttime_offset();
 631 }
 632