osfmk/arm/commpage/commpage.c

   1 /*
   2  * Copyright (c) 2007 Apple Inc. All rights reserved.
   3  * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved.
   4  *
   5  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   6  *
   7  * This file contains Original Code and/or Modifications of Original Code
   8  * as defined in and that are subject to the Apple Public Source License
   9  * Version 2.0 (the 'License'). You may not use this file except in
  10  * compliance with the License. The rights granted to you under the License
  11  * may not be used to create, or enable the creation or redistribution of,
  12  * unlawful or unlicensed copies of an Apple operating system, or to
  13  * circumvent, violate, or enable the circumvention or violation of, any
  14  * terms of an Apple operating system software license agreement.
  15  *
  16  * Please obtain a copy of the License at
  17  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  18  *
  19  * The Original Code and all software distributed under the License are
  20  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  21  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  22  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  23  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  24  * Please see the License for the specific language governing rights and
  25  * limitations under the License.
  26  *
  27  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  28  */
  29 /*
  30  * @OSF_COPYRIGHT@
  31  */
  32 /*
  33  * @APPLE_FREE_COPYRIGHT@
  34  */
  35 /*
  36  *      File:           arm/commpage/commpage.c
  37  *      Purpose:        Set up and export a RO/RW page
  38  */
  39 #include <libkern/section_keywords.h>
  40 #include <mach/mach_types.h>
  41 #include <mach/machine.h>
  42 #include <mach/vm_map.h>
  43 #include <machine/cpu_capabilities.h>
  44 #include <machine/commpage.h>
  45 #include <machine/config.h>
  46 #include <machine/pmap.h>
  47 #include <vm/vm_kern.h>
  48 #include <vm/vm_map.h>
  49 #include <vm/vm_protos.h>
  50 #include <ipc/ipc_port.h>
  51 #include <arm/cpuid.h>          /* for cpuid_info() & cache_info() */
  52 #include <arm/rtclock.h>
  53 #include <libkern/OSAtomic.h>
  54 #include <stdatomic.h>
  55 #include <kern/remote_time.h>
  56 #include <machine/machine_remote_time.h>
  57 #include <machine/machine_routines.h>
  58
  59 #include <sys/kdebug.h>
  60
  61 #if CONFIG_ATM
  62 #include <atm/atm_internal.h>
  63 #endif
  64
  65 static int commpage_cpus( void );
  66
  67
  68 static void commpage_init_cpu_capabilities( void );
  69
  70 SECURITY_READ_ONLY_LATE(vm_address_t)   commPagePtr = 0;
  71 SECURITY_READ_ONLY_LATE(vm_address_t)   sharedpage_rw_addr = 0;
  72 SECURITY_READ_ONLY_LATE(uint64_t)       _cpu_capabilities = 0;
  73 SECURITY_READ_ONLY_LATE(vm_address_t)   sharedpage_rw_text_addr = 0;
  74
  75 extern user64_addr_t commpage_text64_location;
  76 extern user32_addr_t commpage_text32_location;
  77
  78 /* For sysctl access from BSD side */
  79 extern int      gARMv81Atomics;
  80 extern int      gARMv8Crc32;
  81 extern int      gARMv82FHM;
  82 extern int      gARMv82SHA512;
  83 extern int      gARMv82SHA3;
  84
  85 void
  86 commpage_populate(void)
  87 {
  88         uint16_t        c2;
  89         int cpufamily;
  90
  91         // Create the data and the text commpage
  92         vm_map_address_t kernel_data_addr, kernel_text_addr, user_text_addr;
  93         pmap_create_sharedpages(&kernel_data_addr, &kernel_text_addr, &user_text_addr);
  94
  95         sharedpage_rw_addr = kernel_data_addr;
  96         sharedpage_rw_text_addr = kernel_text_addr;
  97         commPagePtr = (vm_address_t) _COMM_PAGE_BASE_ADDRESS;
  98
  99 #if __arm64__
 100         commpage_text64_location = user_text_addr;
 101         bcopy(_COMM_PAGE64_SIGNATURE_STRING, (void *)(_COMM_PAGE_SIGNATURE + _COMM_PAGE_RW_OFFSET),
 102             MIN(_COMM_PAGE_SIGNATURELEN, strlen(_COMM_PAGE64_SIGNATURE_STRING)));
 103 #else
 104         commpage_text32_location = user_text_addr;
 105         bcopy(_COMM_PAGE32_SIGNATURE_STRING, (void *)(_COMM_PAGE_SIGNATURE + _COMM_PAGE_RW_OFFSET),
 106             MIN(_COMM_PAGE_SIGNATURELEN, strlen(_COMM_PAGE32_SIGNATURE_STRING)));
 107 #endif
 108
 109         *((uint16_t*)(_COMM_PAGE_VERSION + _COMM_PAGE_RW_OFFSET)) = (uint16_t) _COMM_PAGE_THIS_VERSION;
 110
 111         commpage_init_cpu_capabilities();
 112         commpage_set_timestamp(0, 0, 0, 0, 0);
 113
 114         if (_cpu_capabilities & kCache32) {
 115                 c2 = 32;
 116         } else if (_cpu_capabilities & kCache64) {
 117                 c2 = 64;
 118         } else if (_cpu_capabilities & kCache128) {
 119                 c2 = 128;
 120         } else {
 121                 c2 = 0;
 122         }
 123
 124         *((uint16_t*)(_COMM_PAGE_CACHE_LINESIZE + _COMM_PAGE_RW_OFFSET)) = c2;
 125
 126         commpage_update_active_cpus();
 127         cpufamily = cpuid_get_cpufamily();
 128
 129         *((uint8_t*)(_COMM_PAGE_PHYSICAL_CPUS + _COMM_PAGE_RW_OFFSET)) = (uint8_t) machine_info.physical_cpu_max;
 130         *((uint8_t*)(_COMM_PAGE_LOGICAL_CPUS + _COMM_PAGE_RW_OFFSET)) = (uint8_t) machine_info.logical_cpu_max;
 131         *((uint64_t*)(_COMM_PAGE_MEMORY_SIZE + _COMM_PAGE_RW_OFFSET)) = machine_info.max_mem;
 132         *((uint32_t*)(_COMM_PAGE_CPUFAMILY + _COMM_PAGE_RW_OFFSET)) = (uint32_t)cpufamily;
 133         *((uint32_t*)(_COMM_PAGE_DEV_FIRM + _COMM_PAGE_RW_OFFSET)) = (uint32_t)PE_i_can_has_debugger(NULL);
 134         *((uint8_t*)(_COMM_PAGE_USER_TIMEBASE + _COMM_PAGE_RW_OFFSET)) = user_timebase_type();
 135         *((uint8_t*)(_COMM_PAGE_CONT_HWCLOCK + _COMM_PAGE_RW_OFFSET)) = (uint8_t)user_cont_hwclock_allowed();
 136         *((uint8_t*)(_COMM_PAGE_KERNEL_PAGE_SHIFT + _COMM_PAGE_RW_OFFSET)) = (uint8_t) page_shift;
 137
 138 #if __arm64__
 139         *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_32 + _COMM_PAGE_RW_OFFSET)) = (uint8_t) page_shift_user32;
 140         *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_64 + _COMM_PAGE_RW_OFFSET)) = (uint8_t) SIXTEENK_PAGE_SHIFT;
 141 #elif (__ARM_ARCH_7K__ >= 2)
 142         /* enforce 16KB alignment for watch targets with new ABI */
 143         *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_32 + _COMM_PAGE_RW_OFFSET)) = (uint8_t) SIXTEENK_PAGE_SHIFT;
 144         *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_64 + _COMM_PAGE_RW_OFFSET)) = (uint8_t) SIXTEENK_PAGE_SHIFT;
 145 #else /* __arm64__ */
 146         *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_32 + _COMM_PAGE_RW_OFFSET)) = (uint8_t) PAGE_SHIFT;
 147         *((uint8_t*)(_COMM_PAGE_USER_PAGE_SHIFT_64 + _COMM_PAGE_RW_OFFSET)) = (uint8_t) PAGE_SHIFT;
 148 #endif /* __arm64__ */
 149
 150         commpage_update_timebase();
 151         commpage_update_mach_continuous_time(0);
 152
 153         clock_sec_t secs;
 154         clock_usec_t microsecs;
 155         clock_get_boottime_microtime(&secs, &microsecs);
 156         commpage_update_boottime(secs * USEC_PER_SEC + microsecs);
 157
 158         /*
 159          * set commpage approximate time to zero for initialization.
 160          * scheduler shall populate correct value before running user thread
 161          */
 162         *((uint64_t *)(_COMM_PAGE_APPROX_TIME + _COMM_PAGE_RW_OFFSET)) = 0;
 163 #ifdef CONFIG_MACH_APPROXIMATE_TIME
 164         *((uint8_t *)(_COMM_PAGE_APPROX_TIME_SUPPORTED + _COMM_PAGE_RW_OFFSET)) = 1;
 165 #else
 166         *((uint8_t *)(_COMM_PAGE_APPROX_TIME_SUPPORTED + _COMM_PAGE_RW_OFFSET)) = 0;
 167 #endif
 168
 169         commpage_update_kdebug_state();
 170
 171 #if CONFIG_ATM
 172         commpage_update_atm_diagnostic_config(atm_get_diagnostic_config());
 173 #endif
 174
 175
 176         *((uint64_t*)(_COMM_PAGE_REMOTETIME_PARAMS + _COMM_PAGE_RW_OFFSET)) = BT_RESET_SENTINEL_TS;
 177 }
 178
 179 #define COMMPAGE_TEXT_SEGMENT "__TEXT_EXEC"
 180 #define COMMPAGE_TEXT_SECTION "__commpage_text"
 181
 182 /* Get a pointer to the start of the ARM PFZ code section. This macro tell the
 183  * linker that the storage for the variable here is at the start of the section */
 184 extern char commpage_text_start[]
 185 __SECTION_START_SYM(COMMPAGE_TEXT_SEGMENT, COMMPAGE_TEXT_SECTION);
 186
 187 /* Get a pointer to the end of the ARM PFZ code section. This macro tell the
 188  * linker that the storage for the variable here is at the end of the section */
 189 extern char commpage_text_end[]
 190 __SECTION_END_SYM(COMMPAGE_TEXT_SEGMENT, COMMPAGE_TEXT_SECTION);
 191
 192 /* This is defined in the commpage text section as a symbol at the start of the preemptible
 193  * functions */
 194 extern char commpage_text_preemptible_functions;
 195
 196 #if CONFIG_ARM_PFZ
 197 static size_t size_of_pfz = 0;
 198 #endif
 199
 200 /* This is the opcode for brk #666 */
 201 #define BRK_666_OPCODE 0xD4205340
 202
 203 void
 204 commpage_text_populate(void)
 205 {
 206 #if CONFIG_ARM_PFZ
 207         size_t size_of_commpage_text = commpage_text_end - commpage_text_start;
 208         if (size_of_commpage_text == 0) {
 209                 panic("ARM comm page text section %s,%s missing", COMMPAGE_TEXT_SEGMENT, COMMPAGE_TEXT_SECTION);
 210         }
 211         assert(size_of_commpage_text <= PAGE_SIZE);
 212         assert(size_of_commpage_text > 0);
 213
 214         /* Get the size of the PFZ half of the comm page text section. */
 215         size_of_pfz = &commpage_text_preemptible_functions - commpage_text_start;
 216
 217         // Copy the code segment of comm page text section into the PFZ
 218         memcpy((void *) _COMM_PAGE64_TEXT_START_ADDRESS, (void *) commpage_text_start, size_of_commpage_text);
 219
 220         // Make sure to populate the rest of it with brk 666 so that undefined code
 221         // doesn't get  run
 222         memset((char *) _COMM_PAGE64_TEXT_START_ADDRESS + size_of_commpage_text, BRK_666_OPCODE,
 223             PAGE_SIZE - size_of_commpage_text);
 224 #endif
 225 }
 226
 227 uint32_t
 228 commpage_is_in_pfz64(addr64_t addr64)
 229 {
 230 #if CONFIG_ARM_PFZ
 231         if ((addr64 >= commpage_text64_location) &&
 232             (addr64 < (commpage_text64_location + size_of_pfz))) {
 233                 return 1;
 234         } else {
 235                 return 0;
 236         }
 237 #else
 238 #pragma unused (addr64)
 239         return 0;
 240 #endif
 241 }
 242
 243
 244 void
 245 commpage_set_timestamp(
 246         uint64_t        tbr,
 247         uint64_t        secs,
 248         uint64_t        frac,
 249         uint64_t        scale,
 250         uint64_t        tick_per_sec)
 251 {
 252         new_commpage_timeofday_data_t *commpage_timeofday_datap;
 253
 254         if (commPagePtr == 0) {
 255                 return;
 256         }
 257
 258         commpage_timeofday_datap =  (new_commpage_timeofday_data_t *)(_COMM_PAGE_NEWTIMEOFDAY_DATA + _COMM_PAGE_RW_OFFSET);
 259
 260         commpage_timeofday_datap->TimeStamp_tick = 0x0ULL;
 261
 262 #if     (__ARM_ARCH__ >= 7)
 263         __asm__ volatile ("dmb ish");
 264 #endif
 265         commpage_timeofday_datap->TimeStamp_sec = secs;
 266         commpage_timeofday_datap->TimeStamp_frac = frac;
 267         commpage_timeofday_datap->Ticks_scale = scale;
 268         commpage_timeofday_datap->Ticks_per_sec = tick_per_sec;
 269
 270 #if     (__ARM_ARCH__ >= 7)
 271         __asm__ volatile ("dmb ish");
 272 #endif
 273         commpage_timeofday_datap->TimeStamp_tick = tbr;
 274
 275 }
 276
 277 /*
 278  * Update _COMM_PAGE_MEMORY_PRESSURE.  Called periodically from vm's compute_memory_pressure()
 279  */
 280
 281 void
 282 commpage_set_memory_pressure(
 283         unsigned int    pressure )
 284 {
 285         if (commPagePtr == 0) {
 286                 return;
 287         }
 288         *((uint32_t *)(_COMM_PAGE_MEMORY_PRESSURE + _COMM_PAGE_RW_OFFSET)) = pressure;
 289 }
 290
 291 /*
 292  * Determine number of CPUs on this system.
 293  */
 294 static int
 295 commpage_cpus( void )
 296 {
 297         int cpus;
 298
 299         cpus = machine_info.max_cpus;
 300
 301         if (cpus == 0) {
 302                 panic("commpage cpus==0");
 303         }
 304         if (cpus > 0xFF) {
 305                 cpus = 0xFF;
 306         }
 307
 308         return cpus;
 309 }
 310
 311 uint64_t
 312 _get_cpu_capabilities(void)
 313 {
 314         return _cpu_capabilities;
 315 }
 316
 317 vm_address_t
 318 _get_commpage_priv_address(void)
 319 {
 320         return sharedpage_rw_addr;
 321 }
 322
 323 vm_address_t
 324 _get_commpage_text_priv_address(void)
 325 {
 326         return sharedpage_rw_text_addr;
 327 }
 328
 329 /*
 330  * Initialize _cpu_capabilities vector
 331  */
 332 static void
 333 commpage_init_cpu_capabilities( void )
 334 {
 335         uint64_t bits;
 336         int cpus;
 337         ml_cpu_info_t cpu_info;
 338
 339         bits = 0;
 340         ml_cpu_get_info(&cpu_info);
 341
 342         switch (cpu_info.cache_line_size) {
 343         case 128:
 344                 bits |= kCache128;
 345                 break;
 346         case 64:
 347                 bits |= kCache64;
 348                 break;
 349         case 32:
 350                 bits |= kCache32;
 351                 break;
 352         default:
 353                 break;
 354         }
 355         cpus = commpage_cpus();
 356
 357         if (cpus == 1) {
 358                 bits |= kUP;
 359         }
 360
 361         bits |= (cpus << kNumCPUsShift);
 362
 363         bits |= kFastThreadLocalStorage;        // TPIDRURO for TLS
 364
 365 #if     __ARM_VFP__
 366         bits |= kHasVfp;
 367         arm_mvfp_info_t *mvfp_info = arm_mvfp_info();
 368         if (mvfp_info->neon) {
 369                 bits |= kHasNeon;
 370         }
 371         if (mvfp_info->neon_hpfp) {
 372                 bits |= kHasNeonHPFP;
 373         }
 374         if (mvfp_info->neon_fp16) {
 375                 bits |= kHasNeonFP16;
 376         }
 377 #endif
 378 #if defined(__arm64__)
 379         bits |= kHasFMA;
 380 #endif
 381 #if     __ARM_ENABLE_WFE_
 382         bits |= kHasEvent;
 383 #endif
 384 #if __ARM_V8_CRYPTO_EXTENSIONS__
 385         bits |= kHasARMv8Crypto;
 386 #endif
 387 #ifdef __arm64__
 388         uint64_t isar0 = __builtin_arm_rsr64("ID_AA64ISAR0_EL1");
 389         if ((isar0 & ID_AA64ISAR0_EL1_ATOMIC_MASK) == ID_AA64ISAR0_EL1_ATOMIC_8_1) {
 390                 bits |= kHasARMv81Atomics;
 391                 gARMv81Atomics = 1;
 392         }
 393         if ((isar0 & ID_AA64ISAR0_EL1_CRC32_MASK) == ID_AA64ISAR0_EL1_CRC32_EN) {
 394                 bits |= kHasARMv8Crc32;
 395                 gARMv8Crc32 = 1;
 396         }
 397         if ((isar0 & ID_AA64ISAR0_EL1_FHM_MASK) >= ID_AA64ISAR0_EL1_FHM_8_2) {
 398                 bits |= kHasARMv82FHM;
 399                 gARMv82FHM = 1;
 400         }
 401
 402         if ((isar0 & ID_AA64ISAR0_EL1_SHA2_MASK) > ID_AA64ISAR0_EL1_SHA2_EN) {
 403                 bits |= kHasARMv82SHA512;
 404                 gARMv82SHA512 = 1;
 405         }
 406         if ((isar0 & ID_AA64ISAR0_EL1_SHA3_MASK) >= ID_AA64ISAR0_EL1_SHA3_EN) {
 407                 bits |= kHasARMv82SHA3;
 408                 gARMv82SHA3 = 1;
 409         }
 410
 411 #endif
 412
 413
 414
 415
 416         _cpu_capabilities = bits;
 417
 418         *((uint32_t *)(_COMM_PAGE_CPU_CAPABILITIES + _COMM_PAGE_RW_OFFSET)) = (uint32_t)_cpu_capabilities;
 419         *((uint64_t *)(_COMM_PAGE_CPU_CAPABILITIES64 + _COMM_PAGE_RW_OFFSET)) = _cpu_capabilities;
 420 }
 421
 422 /*
 423  * Updated every time a logical CPU goes offline/online
 424  */
 425 void
 426 commpage_update_active_cpus(void)
 427 {
 428         if (!commPagePtr) {
 429                 return;
 430         }
 431         *((uint8_t *)(_COMM_PAGE_ACTIVE_CPUS + _COMM_PAGE_RW_OFFSET)) = (uint8_t)processor_avail_count;
 432
 433 }
 434
 435 /*
 436  * Update the commpage bits for mach_absolute_time and mach_continuous_time (for userspace)
 437  */
 438 void
 439 commpage_update_timebase(void)
 440 {
 441         if (commPagePtr) {
 442                 *((uint64_t*)(_COMM_PAGE_TIMEBASE_OFFSET + _COMM_PAGE_RW_OFFSET)) = rtclock_base_abstime;
 443         }
 444 }
 445
 446 /*
 447  * Update the commpage with current kdebug state. This currently has bits for
 448  * global trace state, and typefilter enablement. It is likely additional state
 449  * will be tracked in the future.
 450  *
 451  * INVARIANT: This value will always be 0 if global tracing is disabled. This
 452  * allows simple guard tests of "if (*_COMM_PAGE_KDEBUG_ENABLE) { ... }"
 453  */
 454 void
 455 commpage_update_kdebug_state(void)
 456 {
 457         if (commPagePtr) {
 458                 *((volatile uint32_t*)(_COMM_PAGE_KDEBUG_ENABLE + _COMM_PAGE_RW_OFFSET)) = kdebug_commpage_state();
 459         }
 460 }
 461
 462 /* Ditto for atm_diagnostic_config */
 463 void
 464 commpage_update_atm_diagnostic_config(uint32_t diagnostic_config)
 465 {
 466         if (commPagePtr) {
 467                 *((volatile uint32_t*)(_COMM_PAGE_ATM_DIAGNOSTIC_CONFIG + _COMM_PAGE_RW_OFFSET)) = diagnostic_config;
 468         }
 469 }
 470
 471 /*
 472  * Update the commpage data with the state of multiuser mode for
 473  * this device. Allowing various services in userspace to avoid
 474  * IPC in the (more common) non-multiuser environment.
 475  */
 476 void
 477 commpage_update_multiuser_config(uint32_t multiuser_config)
 478 {
 479         if (commPagePtr) {
 480                 *((volatile uint32_t *)(_COMM_PAGE_MULTIUSER_CONFIG + _COMM_PAGE_RW_OFFSET)) = multiuser_config;
 481         }
 482 }
 483
 484 /*
 485  * update the commpage data for
 486  * last known value of mach_absolute_time()
 487  */
 488
 489 void
 490 commpage_update_mach_approximate_time(uint64_t abstime)
 491 {
 492 #ifdef CONFIG_MACH_APPROXIMATE_TIME
 493         uintptr_t approx_time_base = (uintptr_t)(_COMM_PAGE_APPROX_TIME + _COMM_PAGE_RW_OFFSET);
 494         uint64_t saved_data;
 495
 496         if (commPagePtr) {
 497                 saved_data = atomic_load_explicit((_Atomic uint64_t *)approx_time_base,
 498                     memory_order_relaxed);
 499                 if (saved_data < abstime) {
 500                         /* ignoring the success/fail return value assuming that
 501                          * if the value has been updated since we last read it,
 502                          * "someone" has a newer timestamp than us and ours is
 503                          * now invalid. */
 504                         atomic_compare_exchange_strong_explicit((_Atomic uint64_t *)approx_time_base,
 505                             &saved_data, abstime, memory_order_relaxed, memory_order_relaxed);
 506                 }
 507         }
 508 #else
 509 #pragma unused (abstime)
 510 #endif
 511 }
 512
 513 /*
 514  * update the commpage data's total system sleep time for
 515  * userspace call to mach_continuous_time()
 516  */
 517 void
 518 commpage_update_mach_continuous_time(uint64_t sleeptime)
 519 {
 520         if (commPagePtr) {
 521 #ifdef __arm64__
 522                 *((uint64_t *)(_COMM_PAGE_CONT_TIMEBASE + _COMM_PAGE_RW_OFFSET)) = sleeptime;
 523 #else
 524                 uint64_t *c_time_base = (uint64_t *)(_COMM_PAGE_CONT_TIMEBASE + _COMM_PAGE_RW_OFFSET);
 525                 uint64_t old;
 526                 do {
 527                         old = *c_time_base;
 528                 } while (!OSCompareAndSwap64(old, sleeptime, c_time_base));
 529 #endif /* __arm64__ */
 530         }
 531 }
 532
 533 void
 534 commpage_update_mach_continuous_time_hw_offset(uint64_t offset)
 535 {
 536         *((uint64_t *)(_COMM_PAGE_CONT_HW_TIMEBASE + _COMM_PAGE_RW_OFFSET)) = offset;
 537 }
 538
 539 /*
 540  * update the commpage's value for the boot time
 541  */
 542 void
 543 commpage_update_boottime(uint64_t value)
 544 {
 545         if (commPagePtr) {
 546 #ifdef __arm64__
 547                 *((uint64_t *)(_COMM_PAGE_BOOTTIME_USEC + _COMM_PAGE_RW_OFFSET)) = value;
 548 #else
 549                 uint64_t *cp = (uint64_t *)(_COMM_PAGE_BOOTTIME_USEC + _COMM_PAGE_RW_OFFSET);
 550                 uint64_t old_value;
 551                 do {
 552                         old_value = *cp;
 553                 } while (!OSCompareAndSwap64(old_value, value, cp));
 554 #endif /* __arm64__ */
 555         }
 556 }
 557
 558 /*
 559  * set the commpage's remote time params for
 560  * userspace call to mach_bridge_remote_time()
 561  */
 562 void
 563 commpage_set_remotetime_params(double rate, uint64_t base_local_ts, uint64_t base_remote_ts)
 564 {
 565         if (commPagePtr) {
 566 #ifdef __arm64__
 567                 struct bt_params *paramsp = (struct bt_params *)(_COMM_PAGE_REMOTETIME_PARAMS + _COMM_PAGE_RW_OFFSET);
 568                 paramsp->base_local_ts = 0;
 569                 __asm__ volatile ("dmb ish" ::: "memory");
 570                 paramsp->rate = rate;
 571                 paramsp->base_remote_ts = base_remote_ts;
 572                 __asm__ volatile ("dmb ish" ::: "memory");
 573                 paramsp->base_local_ts = base_local_ts;  //This will act as a generation count
 574 #else
 575                 (void)rate;
 576                 (void)base_local_ts;
 577                 (void)base_remote_ts;
 578 #endif /* __arm64__ */
 579         }
 580 }
 581
 582
 583 /*
 584  * After this counter has incremented, all running CPUs are guaranteed to
 585  * have quiesced, i.e. executed serially dependent memory barriers.
 586  * This is only tracked for CPUs running in userspace, therefore only useful
 587  * outside the kernel.
 588  *
 589  * Note that you can't know which side of those barriers your read was from,
 590  * so you have to observe 2 increments in order to ensure that you saw a
 591  * serially dependent barrier chain across all running CPUs.
 592  */
 593 uint64_t
 594 commpage_increment_cpu_quiescent_counter(void)
 595 {
 596         if (!commPagePtr) {
 597                 return 0;
 598         }
 599
 600         uint64_t old_gen;
 601
 602         _Atomic uint64_t *sched_gen = (_Atomic uint64_t *)(_COMM_PAGE_CPU_QUIESCENT_COUNTER +
 603             _COMM_PAGE_RW_OFFSET);
 604         /*
 605          * On 32bit architectures, double-wide atomic load or stores are a CAS,
 606          * so the atomic increment is the most efficient way to increment the
 607          * counter.
 608          *
 609          * On 64bit architectures however, because the update is synchronized by
 610          * the cpu mask, relaxed loads and stores is more efficient.
 611          */
 612 #if __LP64__
 613         old_gen = os_atomic_load(sched_gen, relaxed);
 614         os_atomic_store(sched_gen, old_gen + 1, relaxed);
 615 #else
 616         old_gen = atomic_fetch_add_explicit(sched_gen, 1, memory_order_relaxed);
 617 #endif
 618         return old_gen;
 619 }
 620
 621 /*
 622  * update the commpage with if dtrace user land probes are enabled
 623  */
 624 void
 625 commpage_update_dof(boolean_t enabled)
 626 {
 627 #if CONFIG_DTRACE
 628         *((uint8_t*)(_COMM_PAGE_DTRACE_DOF_ENABLED + _COMM_PAGE_RW_OFFSET)) = (enabled ? 1 : 0);
 629 #else
 630         (void)enabled;
 631 #endif
 632 }
 633
 634 /*
 635  * update the dyld global config flags
 636  */
 637 void
 638 commpage_update_dyld_flags(uint64_t value)
 639 {
 640         *((uint64_t*)(_COMM_PAGE_DYLD_FLAGS + _COMM_PAGE_RW_OFFSET)) = value;
 641
 642 }