osfmk/arm/machine_routines_common.c

   1 /*
   2  * Copyright (c) 2007-2013 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <arm/machine_cpu.h>
  30 #include <arm/cpu_internal.h>
  31 #include <arm/cpuid.h>
  32 #include <arm/cpu_data.h>
  33 #include <arm/cpu_data_internal.h>
  34 #include <arm/misc_protos.h>
  35 #include <arm/machdep_call.h>
  36 #include <arm/machine_routines.h>
  37 #include <arm/rtclock.h>
  38 #include <kern/machine.h>
  39 #include <kern/thread.h>
  40 #include <kern/thread_group.h>
  41 #include <kern/policy_internal.h>
  42 #include <machine/config.h>
  43 #include <pexpert/pexpert.h>
  44
  45 #if MONOTONIC
  46 #include <kern/monotonic.h>
  47 #include <machine/monotonic.h>
  48 #endif /* MONOTONIC */
  49
  50 #include <mach/machine.h>
  51
  52 #if INTERRUPT_MASKED_DEBUG
  53 extern boolean_t interrupt_masked_debug;
  54 extern uint64_t interrupt_masked_timeout;
  55 #endif
  56
  57 extern uint64_t mach_absolutetime_asleep;
  58
  59 static void
  60 sched_perfcontrol_oncore_default(perfcontrol_state_t new_thread_state __unused, going_on_core_t on __unused)
  61 {
  62 }
  63
  64 static void
  65 sched_perfcontrol_switch_default(perfcontrol_state_t old_thread_state __unused, perfcontrol_state_t new_thread_state __unused)
  66 {
  67 }
  68
  69 static void
  70 sched_perfcontrol_offcore_default(perfcontrol_state_t old_thread_state __unused, going_off_core_t off __unused, boolean_t thread_terminating __unused)
  71 {
  72 }
  73
  74 static void
  75 sched_perfcontrol_thread_group_default(thread_group_data_t data __unused)
  76 {
  77 }
  78
  79 static void
  80 sched_perfcontrol_max_runnable_latency_default(perfcontrol_max_runnable_latency_t latencies __unused)
  81 {
  82 }
  83
  84 static void
  85 sched_perfcontrol_work_interval_notify_default(perfcontrol_state_t thread_state __unused,
  86                                                perfcontrol_work_interval_t work_interval __unused)
  87 {
  88 }
  89
  90 static void
  91 sched_perfcontrol_work_interval_ctl_default(perfcontrol_state_t thread_state __unused,
  92                                             perfcontrol_work_interval_instance_t instance __unused)
  93 {
  94 }
  95
  96 static void
  97 sched_perfcontrol_deadline_passed_default(__unused uint64_t deadline)
  98 {
  99 }
 100
 101 static void
 102 sched_perfcontrol_csw_default(
 103         __unused perfcontrol_event event, __unused uint32_t cpu_id, __unused uint64_t timestamp,
 104         __unused uint32_t flags, __unused struct perfcontrol_thread_data *offcore,
 105         __unused struct perfcontrol_thread_data *oncore,
 106         __unused struct perfcontrol_cpu_counters *cpu_counters, __unused void *unused)
 107 {
 108 }
 109
 110 static void
 111 sched_perfcontrol_state_update_default(
 112         __unused perfcontrol_event event, __unused uint32_t cpu_id, __unused uint64_t timestamp,
 113         __unused uint32_t flags, __unused struct perfcontrol_thread_data *thr_data,
 114         __unused void *unused)
 115 {
 116 }
 117
 118 sched_perfcontrol_offcore_t                     sched_perfcontrol_offcore = sched_perfcontrol_offcore_default;
 119 sched_perfcontrol_context_switch_t              sched_perfcontrol_switch = sched_perfcontrol_switch_default;
 120 sched_perfcontrol_oncore_t                      sched_perfcontrol_oncore = sched_perfcontrol_oncore_default;
 121 sched_perfcontrol_thread_group_init_t           sched_perfcontrol_thread_group_init = sched_perfcontrol_thread_group_default;
 122 sched_perfcontrol_thread_group_deinit_t         sched_perfcontrol_thread_group_deinit = sched_perfcontrol_thread_group_default;
 123 sched_perfcontrol_thread_group_flags_update_t   sched_perfcontrol_thread_group_flags_update = sched_perfcontrol_thread_group_default;
 124 sched_perfcontrol_max_runnable_latency_t        sched_perfcontrol_max_runnable_latency = sched_perfcontrol_max_runnable_latency_default;
 125 sched_perfcontrol_work_interval_notify_t        sched_perfcontrol_work_interval_notify = sched_perfcontrol_work_interval_notify_default;
 126 sched_perfcontrol_work_interval_ctl_t           sched_perfcontrol_work_interval_ctl = sched_perfcontrol_work_interval_ctl_default;
 127 sched_perfcontrol_deadline_passed_t             sched_perfcontrol_deadline_passed = sched_perfcontrol_deadline_passed_default;
 128 sched_perfcontrol_csw_t                         sched_perfcontrol_csw = sched_perfcontrol_csw_default;
 129 sched_perfcontrol_state_update_t                sched_perfcontrol_state_update = sched_perfcontrol_state_update_default;
 130
 131 void
 132 sched_perfcontrol_register_callbacks(sched_perfcontrol_callbacks_t callbacks, unsigned long size_of_state)
 133 {
 134         assert(callbacks == NULL || callbacks->version >= SCHED_PERFCONTROL_CALLBACKS_VERSION_2);
 135
 136         if (size_of_state > sizeof(struct perfcontrol_state)) {
 137                 panic("%s: Invalid required state size %lu", __FUNCTION__, size_of_state);
 138         }
 139
 140         if (callbacks) {
 141
 142
 143                 if (callbacks->version >= SCHED_PERFCONTROL_CALLBACKS_VERSION_7) {
 144                         if (callbacks->work_interval_ctl != NULL) {
 145                                 sched_perfcontrol_work_interval_ctl = callbacks->work_interval_ctl;
 146                         } else {
 147                                 sched_perfcontrol_work_interval_ctl = sched_perfcontrol_work_interval_ctl_default;
 148                         }
 149                 }
 150
 151                 if (callbacks->version >= SCHED_PERFCONTROL_CALLBACKS_VERSION_5) {
 152                         if (callbacks->csw != NULL) {
 153                                 sched_perfcontrol_csw = callbacks->csw;
 154                         } else {
 155                                 sched_perfcontrol_csw = sched_perfcontrol_csw_default;
 156                         }
 157
 158                         if (callbacks->state_update != NULL) {
 159                                 sched_perfcontrol_state_update = callbacks->state_update;
 160                         } else {
 161                                 sched_perfcontrol_state_update = sched_perfcontrol_state_update_default;
 162                         }
 163                 }
 164
 165                 if (callbacks->version >= SCHED_PERFCONTROL_CALLBACKS_VERSION_4) {
 166                         if (callbacks->deadline_passed != NULL) {
 167                                 sched_perfcontrol_deadline_passed = callbacks->deadline_passed;
 168                         } else {
 169                                 sched_perfcontrol_deadline_passed = sched_perfcontrol_deadline_passed_default;
 170                         }
 171                 }
 172
 173                 if (callbacks->offcore != NULL) {
 174                         sched_perfcontrol_offcore = callbacks->offcore;
 175                 } else {
 176                         sched_perfcontrol_offcore = sched_perfcontrol_offcore_default;
 177                 }
 178
 179                 if (callbacks->context_switch != NULL) {
 180                         sched_perfcontrol_switch = callbacks->context_switch;
 181                 } else {
 182                         sched_perfcontrol_switch = sched_perfcontrol_switch_default;
 183                 }
 184
 185                 if (callbacks->oncore != NULL) {
 186                         sched_perfcontrol_oncore = callbacks->oncore;
 187                 } else {
 188                         sched_perfcontrol_oncore = sched_perfcontrol_oncore_default;
 189                 }
 190
 191                 if (callbacks->max_runnable_latency != NULL) {
 192                         sched_perfcontrol_max_runnable_latency = callbacks->max_runnable_latency;
 193                 } else {
 194                         sched_perfcontrol_max_runnable_latency = sched_perfcontrol_max_runnable_latency_default;
 195                 }
 196
 197                 if (callbacks->work_interval_notify != NULL) {
 198                         sched_perfcontrol_work_interval_notify = callbacks->work_interval_notify;
 199                 } else {
 200                         sched_perfcontrol_work_interval_notify = sched_perfcontrol_work_interval_notify_default;
 201                 }
 202         } else {
 203                 /* reset to defaults */
 204                 sched_perfcontrol_offcore = sched_perfcontrol_offcore_default;
 205                 sched_perfcontrol_switch = sched_perfcontrol_switch_default;
 206                 sched_perfcontrol_oncore = sched_perfcontrol_oncore_default;
 207                 sched_perfcontrol_thread_group_init = sched_perfcontrol_thread_group_default;
 208                 sched_perfcontrol_thread_group_deinit = sched_perfcontrol_thread_group_default;
 209                 sched_perfcontrol_thread_group_flags_update = sched_perfcontrol_thread_group_default;
 210                 sched_perfcontrol_max_runnable_latency = sched_perfcontrol_max_runnable_latency_default;
 211                 sched_perfcontrol_work_interval_notify = sched_perfcontrol_work_interval_notify_default;
 212                 sched_perfcontrol_work_interval_ctl = sched_perfcontrol_work_interval_ctl_default;
 213                 sched_perfcontrol_csw = sched_perfcontrol_csw_default;
 214                 sched_perfcontrol_state_update = sched_perfcontrol_state_update_default;
 215         }
 216 }
 217
 218
 219 static void
 220 machine_switch_populate_perfcontrol_thread_data(struct perfcontrol_thread_data *data,
 221                                                 thread_t thread,
 222                                                 uint64_t same_pri_latency)
 223 {
 224         bzero(data, sizeof(struct perfcontrol_thread_data));
 225         data->perfctl_class = thread_get_perfcontrol_class(thread);
 226         data->energy_estimate_nj = 0;
 227         data->thread_id = thread->thread_id;
 228         data->scheduling_latency_at_same_basepri = same_pri_latency;
 229         data->perfctl_state = FIND_PERFCONTROL_STATE(thread);
 230 }
 231
 232 static void
 233 machine_switch_populate_perfcontrol_cpu_counters(struct perfcontrol_cpu_counters *cpu_counters)
 234 {
 235 #if MONOTONIC
 236         mt_perfcontrol(&cpu_counters->instructions, &cpu_counters->cycles);
 237 #else /* MONOTONIC */
 238         cpu_counters->instructions = 0;
 239         cpu_counters->cycles = 0;
 240 #endif /* !MONOTONIC */
 241 }
 242
 243 int perfcontrol_callout_stats_enabled = 0;
 244 static _Atomic uint64_t perfcontrol_callout_stats[PERFCONTROL_CALLOUT_MAX][PERFCONTROL_STAT_MAX];
 245 static _Atomic uint64_t perfcontrol_callout_count[PERFCONTROL_CALLOUT_MAX];
 246
 247 #if MONOTONIC
 248 static inline
 249 bool perfcontrol_callout_counters_begin(uint64_t *counters)
 250 {
 251     if (!perfcontrol_callout_stats_enabled)
 252         return false;
 253     mt_fixed_counts(counters);
 254     return true;
 255 }
 256
 257 static inline
 258 void perfcontrol_callout_counters_end(uint64_t *start_counters,
 259         perfcontrol_callout_type_t type)
 260 {
 261     uint64_t end_counters[MT_CORE_NFIXED];
 262     mt_fixed_counts(end_counters);
 263     atomic_fetch_add_explicit(&perfcontrol_callout_stats[type][PERFCONTROL_STAT_CYCLES],
 264             end_counters[MT_CORE_CYCLES] - start_counters[MT_CORE_CYCLES], memory_order_relaxed);
 265 #ifdef MT_CORE_INSTRS
 266     atomic_fetch_add_explicit(&perfcontrol_callout_stats[type][PERFCONTROL_STAT_INSTRS],
 267             end_counters[MT_CORE_INSTRS] - start_counters[MT_CORE_INSTRS], memory_order_relaxed);
 268 #endif /* defined(MT_CORE_INSTRS) */
 269     atomic_fetch_add_explicit(&perfcontrol_callout_count[type], 1, memory_order_relaxed);
 270 }
 271 #endif /* MONOTONIC */
 272
 273 uint64_t perfcontrol_callout_stat_avg(perfcontrol_callout_type_t type,
 274         perfcontrol_callout_stat_t stat)
 275 {
 276     if (!perfcontrol_callout_stats_enabled)
 277         return 0;
 278     return (perfcontrol_callout_stats[type][stat] / perfcontrol_callout_count[type]);
 279 }
 280
 281 void
 282 machine_switch_perfcontrol_context(perfcontrol_event event,
 283                                    uint64_t timestamp,
 284                                    uint32_t flags,
 285                                    uint64_t new_thread_same_pri_latency,
 286                                    thread_t old,
 287                                    thread_t new)
 288 {
 289         if (sched_perfcontrol_switch != sched_perfcontrol_switch_default) {
 290                 perfcontrol_state_t old_perfcontrol_state = FIND_PERFCONTROL_STATE(old);
 291                 perfcontrol_state_t new_perfcontrol_state = FIND_PERFCONTROL_STATE(new);
 292                 sched_perfcontrol_switch(old_perfcontrol_state, new_perfcontrol_state);
 293         }
 294
 295         if (sched_perfcontrol_csw != sched_perfcontrol_csw_default) {
 296                 uint32_t cpu_id = (uint32_t)cpu_number();
 297                 struct perfcontrol_cpu_counters cpu_counters;
 298                 struct perfcontrol_thread_data offcore, oncore;
 299                 machine_switch_populate_perfcontrol_thread_data(&offcore, old, 0);
 300                 machine_switch_populate_perfcontrol_thread_data(&oncore, new,
 301                         new_thread_same_pri_latency);
 302                 machine_switch_populate_perfcontrol_cpu_counters(&cpu_counters);
 303
 304 #if MONOTONIC
 305                 uint64_t counters[MT_CORE_NFIXED];
 306                 bool ctrs_enabled = perfcontrol_callout_counters_begin(counters);
 307 #endif /* MONOTONIC */
 308                 sched_perfcontrol_csw(event, cpu_id, timestamp, flags,
 309                         &offcore, &oncore, &cpu_counters, NULL);
 310 #if MONOTONIC
 311                 if (ctrs_enabled) perfcontrol_callout_counters_end(counters, PERFCONTROL_CALLOUT_CONTEXT);
 312 #endif /* MONOTONIC */
 313
 314 #if __arm64__
 315                 old->machine.energy_estimate_nj += offcore.energy_estimate_nj;
 316                 new->machine.energy_estimate_nj += oncore.energy_estimate_nj;
 317 #endif
 318         }
 319 }
 320
 321 void
 322 machine_switch_perfcontrol_state_update(perfcontrol_event event,
 323                                         uint64_t timestamp,
 324                                         uint32_t flags,
 325                                         thread_t thread)
 326 {
 327         if (sched_perfcontrol_state_update == sched_perfcontrol_state_update_default)
 328                 return;
 329         uint32_t cpu_id = (uint32_t)cpu_number();
 330         struct perfcontrol_thread_data data;
 331         machine_switch_populate_perfcontrol_thread_data(&data, thread, 0);
 332
 333 #if MONOTONIC
 334         uint64_t counters[MT_CORE_NFIXED];
 335         bool ctrs_enabled = perfcontrol_callout_counters_begin(counters);
 336 #endif /* MONOTONIC */
 337         sched_perfcontrol_state_update(event, cpu_id, timestamp, flags,
 338                 &data, NULL);
 339 #if MONOTONIC
 340         if (ctrs_enabled) perfcontrol_callout_counters_end(counters, PERFCONTROL_CALLOUT_STATE_UPDATE);
 341 #endif /* MONOTONIC */
 342
 343 #if __arm64__
 344         thread->machine.energy_estimate_nj += data.energy_estimate_nj;
 345 #endif
 346 }
 347
 348 void
 349 machine_thread_going_on_core(thread_t   new_thread,
 350                              int        urgency,
 351                              uint64_t   sched_latency,
 352                              uint64_t   same_pri_latency,
 353                              uint64_t   timestamp)
 354 {
 355
 356         if (sched_perfcontrol_oncore == sched_perfcontrol_oncore_default)
 357                 return;
 358         struct going_on_core on_core;
 359         perfcontrol_state_t state = FIND_PERFCONTROL_STATE(new_thread);
 360
 361         on_core.thread_id = new_thread->thread_id;
 362         on_core.energy_estimate_nj = 0;
 363         on_core.qos_class = proc_get_effective_thread_policy(new_thread, TASK_POLICY_QOS);
 364         on_core.urgency = urgency;
 365         on_core.is_32_bit = thread_is_64bit_data(new_thread) ? FALSE : TRUE;
 366         on_core.is_kernel_thread = new_thread->task == kernel_task;
 367         on_core.scheduling_latency = sched_latency;
 368         on_core.start_time = timestamp;
 369         on_core.scheduling_latency_at_same_basepri = same_pri_latency;
 370
 371 #if MONOTONIC
 372         uint64_t counters[MT_CORE_NFIXED];
 373         bool ctrs_enabled = perfcontrol_callout_counters_begin(counters);
 374 #endif /* MONOTONIC */
 375         sched_perfcontrol_oncore(state, &on_core);
 376 #if MONOTONIC
 377         if (ctrs_enabled) perfcontrol_callout_counters_end(counters, PERFCONTROL_CALLOUT_ON_CORE);
 378 #endif /* MONOTONIC */
 379
 380 #if __arm64__
 381         new_thread->machine.energy_estimate_nj += on_core.energy_estimate_nj;
 382 #endif
 383 }
 384
 385 void
 386 machine_thread_going_off_core(thread_t old_thread, boolean_t thread_terminating, uint64_t last_dispatch)
 387 {
 388         if (sched_perfcontrol_offcore == sched_perfcontrol_offcore_default)
 389                 return;
 390         struct going_off_core off_core;
 391         perfcontrol_state_t state = FIND_PERFCONTROL_STATE(old_thread);
 392
 393         off_core.thread_id = old_thread->thread_id;
 394         off_core.energy_estimate_nj = 0;
 395         off_core.end_time = last_dispatch;
 396
 397 #if MONOTONIC
 398         uint64_t counters[MT_CORE_NFIXED];
 399         bool ctrs_enabled = perfcontrol_callout_counters_begin(counters);
 400 #endif /* MONOTONIC */
 401         sched_perfcontrol_offcore(state, &off_core, thread_terminating);
 402 #if MONOTONIC
 403         if (ctrs_enabled) perfcontrol_callout_counters_end(counters, PERFCONTROL_CALLOUT_OFF_CORE);
 404 #endif /* MONOTONIC */
 405
 406 #if __arm64__
 407         old_thread->machine.energy_estimate_nj += off_core.energy_estimate_nj;
 408 #endif
 409 }
 410
 411
 412 void
 413 machine_max_runnable_latency(uint64_t bg_max_latency,
 414                                                          uint64_t default_max_latency,
 415                                                          uint64_t realtime_max_latency)
 416 {
 417         if (sched_perfcontrol_max_runnable_latency == sched_perfcontrol_max_runnable_latency_default)
 418                 return;
 419         struct perfcontrol_max_runnable_latency latencies = {
 420                 .max_scheduling_latencies = {
 421                         [THREAD_URGENCY_NONE] = 0,
 422                         [THREAD_URGENCY_BACKGROUND] = bg_max_latency,
 423                         [THREAD_URGENCY_NORMAL] = default_max_latency,
 424                         [THREAD_URGENCY_REAL_TIME] = realtime_max_latency
 425                 }
 426         };
 427
 428         sched_perfcontrol_max_runnable_latency(&latencies);
 429 }
 430
 431 void
 432 machine_work_interval_notify(thread_t thread,
 433                              struct kern_work_interval_args* kwi_args)
 434 {
 435         if (sched_perfcontrol_work_interval_notify == sched_perfcontrol_work_interval_notify_default)
 436                 return;
 437         perfcontrol_state_t state = FIND_PERFCONTROL_STATE(thread);
 438         struct perfcontrol_work_interval work_interval = {
 439                 .thread_id      = thread->thread_id,
 440                 .qos_class      = proc_get_effective_thread_policy(thread, TASK_POLICY_QOS),
 441                 .urgency        = kwi_args->urgency,
 442                 .flags          = kwi_args->notify_flags,
 443                 .work_interval_id = kwi_args->work_interval_id,
 444                 .start          = kwi_args->start,
 445                 .finish         = kwi_args->finish,
 446                 .deadline       = kwi_args->deadline,
 447                 .next_start     = kwi_args->next_start,
 448                 .create_flags   = kwi_args->create_flags,
 449         };
 450         sched_perfcontrol_work_interval_notify(state, &work_interval);
 451 }
 452
 453
 454 void
 455 machine_perfcontrol_deadline_passed(uint64_t deadline)
 456 {
 457         if (sched_perfcontrol_deadline_passed != sched_perfcontrol_deadline_passed_default)
 458                 sched_perfcontrol_deadline_passed(deadline);
 459 }
 460
 461 #if INTERRUPT_MASKED_DEBUG
 462 /*
 463  * ml_spin_debug_reset()
 464  * Reset the timestamp on a thread that has been unscheduled
 465  * to avoid false alarms.    Alarm will go off if interrupts are held
 466  * disabled for too long, starting from now.
 467  */
 468 void
 469 ml_spin_debug_reset(thread_t thread)
 470 {
 471         thread->machine.intmask_timestamp = mach_absolute_time();
 472 }
 473
 474 /*
 475  * ml_spin_debug_clear()
 476  * Clear the timestamp on a thread that has been unscheduled
 477  * to avoid false alarms
 478  */
 479 void
 480 ml_spin_debug_clear(thread_t thread)
 481 {
 482         thread->machine.intmask_timestamp = 0;
 483 }
 484
 485 /*
 486  * ml_spin_debug_clear_self()
 487  * Clear the timestamp on the current thread to prevent
 488  * false alarms
 489  */
 490 void
 491 ml_spin_debug_clear_self()
 492 {
 493         ml_spin_debug_clear(current_thread());
 494 }
 495
 496 void
 497 ml_check_interrupts_disabled_duration(thread_t thread)
 498 {
 499         uint64_t start;
 500         uint64_t now;
 501
 502         start = thread->machine.intmask_timestamp;
 503         if (start != 0) {
 504                 now = mach_absolute_time();
 505
 506                 if ((now - start) > interrupt_masked_timeout * debug_cpu_performance_degradation_factor) {
 507                         mach_timebase_info_data_t timebase;
 508                         clock_timebase_info(&timebase);
 509
 510 #ifndef KASAN
 511                         /*
 512                         * Disable the actual panic for KASAN due to the overhead of KASAN itself, leave the rest of the
 513                         * mechanism enabled so that KASAN can catch any bugs in the mechanism itself.
 514                         */
 515                         panic("Interrupts held disabled for %llu nanoseconds", (((now - start) * timebase.numer)/timebase.denom));
 516 #endif
 517                 }
 518         }
 519
 520         return;
 521 }
 522 #endif // INTERRUPT_MASKED_DEBUG
 523
 524
 525 boolean_t
 526 ml_set_interrupts_enabled(boolean_t enable)
 527 {
 528         thread_t        thread;
 529         uint64_t        state;
 530
 531 #if __arm__
 532 #define INTERRUPT_MASK PSR_IRQF
 533         state = __builtin_arm_rsr("cpsr");
 534 #else
 535 #define INTERRUPT_MASK DAIF_IRQF
 536         state = __builtin_arm_rsr("DAIF");
 537 #endif
 538         if (enable && (state & INTERRUPT_MASK)) {
 539 #if INTERRUPT_MASKED_DEBUG
 540                 if (interrupt_masked_debug) {
 541                         // Interrupts are currently masked, we will enable them (after finishing this check)
 542                         thread = current_thread();
 543                         ml_check_interrupts_disabled_duration(thread);
 544                         thread->machine.intmask_timestamp = 0;
 545                 }
 546 #endif  // INTERRUPT_MASKED_DEBUG
 547                 if (get_preemption_level() == 0) {
 548                         thread = current_thread();
 549                         while (thread->machine.CpuDatap->cpu_pending_ast & AST_URGENT) {
 550 #if __ARM_USER_PROTECT__
 551                                 uintptr_t up = arm_user_protect_begin(thread);
 552 #endif
 553                                 ast_taken_kernel();
 554 #if __ARM_USER_PROTECT__
 555                                 arm_user_protect_end(thread, up, FALSE);
 556 #endif
 557                         }
 558                 }
 559 #if __arm__
 560                 __asm__ volatile ("cpsie if" ::: "memory"); // Enable IRQ FIQ
 561 #else
 562                 __builtin_arm_wsr("DAIFClr", (DAIFSC_IRQF | DAIFSC_FIQF));
 563 #endif
 564         } else if (!enable && ((state & INTERRUPT_MASK) == 0)) {
 565 #if __arm__
 566                 __asm__ volatile ("cpsid if" ::: "memory"); // Mask IRQ FIQ
 567 #else
 568                 __builtin_arm_wsr("DAIFSet", (DAIFSC_IRQF | DAIFSC_FIQF));
 569 #endif
 570 #if INTERRUPT_MASKED_DEBUG
 571                 if (interrupt_masked_debug) {
 572                         // Interrupts were enabled, we just masked them
 573                         current_thread()->machine.intmask_timestamp = mach_absolute_time();
 574                 }
 575 #endif
 576         }
 577         return ((state & INTERRUPT_MASK) == 0);
 578 }
 579
 580 /*
 581  *      Routine:        ml_at_interrupt_context
 582  *      Function:       Check if running at interrupt context
 583  */
 584 boolean_t
 585 ml_at_interrupt_context(void)
 586 {
 587         /* Do not use a stack-based check here, as the top-level exception handler
 588          * is free to use some other stack besides the per-CPU interrupt stack.
 589          * Interrupts should always be disabled if we're at interrupt context.
 590          * Check that first, as we may be in a preemptible non-interrupt context, in
 591          * which case we could be migrated to a different CPU between obtaining
 592          * the per-cpu data pointer and loading cpu_int_state.  We then might end
 593          * up checking the interrupt state of a different CPU, resulting in a false
 594          * positive.  But if interrupts are disabled, we also know we cannot be
 595          * preempted. */
 596         return (!ml_get_interrupts_enabled() && (getCpuDatap()->cpu_int_state != NULL));
 597 }
 598
 599 vm_offset_t
 600 ml_stack_remaining(void)
 601 {
 602         uintptr_t local = (uintptr_t) &local;
 603         vm_offset_t     intstack_top_ptr;
 604
 605         /* Since this is a stack-based check, we don't need to worry about
 606          * preemption as we do in ml_at_interrupt_context().  If we are preemptible,
 607          * then the sp should never be within any CPU's interrupt stack unless
 608          * something has gone horribly wrong. */
 609         intstack_top_ptr = getCpuDatap()->intstack_top;
 610         if ((local < intstack_top_ptr) && (local > intstack_top_ptr - INTSTACK_SIZE)) {
 611                 return (local - (getCpuDatap()->intstack_top - INTSTACK_SIZE));
 612         } else {
 613                 return (local - current_thread()->kernel_stack);
 614         }
 615 }
 616
 617 static boolean_t ml_quiescing;
 618
 619 void ml_set_is_quiescing(boolean_t quiescing)
 620 {
 621         assert(FALSE == ml_get_interrupts_enabled());
 622         ml_quiescing = quiescing;
 623 }
 624
 625 boolean_t ml_is_quiescing(void)
 626 {
 627         assert(FALSE == ml_get_interrupts_enabled());
 628         return (ml_quiescing);
 629 }
 630
 631 uint64_t ml_get_booter_memory_size(void)
 632 {
 633         uint64_t size;
 634         uint64_t roundsize = 512*1024*1024ULL;
 635         size = BootArgs->memSizeActual;
 636         if (!size) {
 637                 size  = BootArgs->memSize;
 638                 if (size < (2 * roundsize)) roundsize >>= 1;
 639                 size  = (size + roundsize - 1) & ~(roundsize - 1);
 640                 size -= BootArgs->memSize;
 641         }
 642         return (size);
 643 }
 644
 645 uint64_t
 646 ml_get_abstime_offset(void)
 647 {
 648         return rtclock_base_abstime;
 649 }
 650
 651 uint64_t
 652 ml_get_conttime_offset(void)
 653 {
 654         return (rtclock_base_abstime + mach_absolutetime_asleep);
 655 }
 656
 657 uint64_t
 658 ml_get_time_since_reset(void)
 659 {
 660         /* The timebase resets across S2R, so just return the raw value. */
 661         return ml_get_hwclock();
 662 }
 663
 664 uint64_t
 665 ml_get_conttime_wake_time(void)
 666 {
 667         /* The wake time is simply our continuous time offset. */
 668         return ml_get_conttime_offset();
 669 }
 670