pexpert/arm/pe_init.c

   1 /*
   2  * Copyright (c) 2000-2017 Apple Inc. All rights reserved.
   3  *
   4  *    arm platform expert initialization.
   5  */
   6 #include <sys/types.h>
   7 #include <sys/kdebug.h>
   8 #include <mach/vm_param.h>
   9 #include <pexpert/protos.h>
  10 #include <pexpert/pexpert.h>
  11 #include <pexpert/boot.h>
  12 #include <pexpert/device_tree.h>
  13 #include <pexpert/pe_images.h>
  14 #include <kern/sched_prim.h>
  15 #include <machine/machine_routines.h>
  16 #include <arm/caches_internal.h>
  17 #include <kern/debug.h>
  18 #include <libkern/section_keywords.h>
  19
  20 /* extern references */
  21 extern void     pe_identify_machine(boot_args *bootArgs);
  22
  23 /* static references */
  24 static void     pe_prepare_images(void);
  25
  26 /* private globals */
  27 PE_state_t      PE_state;
  28 #define FW_VERS_LEN 128
  29 char            firmware_version[FW_VERS_LEN];
  30
  31 /*
  32  * This variable is only modified once, when the BSP starts executing. We put it in __TEXT
  33  * as page protections on kernel text early in startup are read-write. The kernel is
  34  * locked down later in start-up, said mappings become RO and thus this
  35  * variable becomes immutable.
  36  *
  37  * See osfmk/arm/arm_vm_init.c for more information.
  38  */
  39 SECURITY_READ_ONLY_SPECIAL_SECTION(volatile uint32_t, "__TEXT,__const") debug_enabled = FALSE;
  40
  41 uint8_t         gPlatformECID[8];
  42 uint32_t        gPlatformMemoryID;
  43 static boolean_t vc_progress_initialized = FALSE;
  44 uint64_t    last_hwaccess_thread = 0;
  45 char     gTargetTypeBuffer[8];
  46 char     gModelTypeBuffer[32];
  47
  48 /* Clock Frequency Info */
  49 clock_frequency_info_t gPEClockFrequencyInfo;
  50
  51 vm_offset_t gPanicBase;
  52 unsigned int gPanicSize;
  53 struct embedded_panic_header *panic_info = NULL;
  54
  55 /* Maximum size of panic log excluding headers, in bytes */
  56 static unsigned int panic_text_len;
  57
  58 /* Whether a console is standing by for panic logging */
  59 static boolean_t panic_console_available = FALSE;
  60
  61 extern uint32_t crc32(uint32_t crc, const void *buf, size_t size);
  62
  63 static void
  64 check_for_panic_log(void)
  65 {
  66         DTEntry entry, chosen;
  67         unsigned int size;
  68         uintptr_t *reg_prop;
  69         uint32_t *panic_region_length;
  70
  71         /*
  72          * Find the vram node in the device tree
  73          */
  74         if (kSuccess != DTLookupEntry(0, "pram", &entry))
  75                 return;
  76
  77         if (kSuccess != DTGetProperty(entry, "reg", (void **)&reg_prop, &size))
  78                 return;
  79
  80         if (kSuccess != DTLookupEntry(0, "/chosen", &chosen))
  81                 return;
  82
  83         if (kSuccess != DTGetProperty(chosen, "embedded-panic-log-size", (void **) &panic_region_length, &size))
  84                 return;
  85
  86         /*
  87          * Map the first page of VRAM into the kernel for use in case of
  88          * panic
  89          */
  90         /* Note: map as normal memory. */
  91         gPanicBase = ml_io_map_wcomb(reg_prop[0], panic_region_length[0]);
  92
  93         /* Deduct the size of the panic header from the panic region size */
  94         panic_text_len = panic_region_length[0] - sizeof(struct embedded_panic_header);
  95         gPanicSize = panic_region_length[0];
  96         panic_info = (struct embedded_panic_header *)gPanicBase;
  97
  98         /* Check if a shared memory console is running in the panic buffer */
  99         if (panic_info->eph_magic == 'SHMC') {
 100                 panic_console_available = TRUE;
 101                 return;
 102         }
 103
 104         /* Check if there's a boot profile in the panic buffer */
 105         if (panic_info->eph_magic == 'BTRC') {
 106                 return;
 107         }
 108
 109         /*
 110          * Check to see if a panic (FUNK) is in VRAM from the last time
 111          */
 112         if (panic_info->eph_magic == EMBEDDED_PANIC_MAGIC) {
 113                 printf("iBoot didn't extract panic log from previous session crash, this is bad\n");
 114         }
 115
 116         /* Clear panic region */
 117         bzero((void *)gPanicBase, gPanicSize);
 118 }
 119
 120 int
 121 PE_initialize_console(PE_Video * info, int op)
 122 {
 123         static int last_console = -1;
 124
 125         if (info && (info != &PE_state.video)) info->v_scale = PE_state.video.v_scale;
 126
 127         switch (op) {
 128
 129         case kPEDisableScreen:
 130                 initialize_screen(info, op);
 131                 last_console = switch_to_serial_console();
 132                 kprintf("kPEDisableScreen %d\n", last_console);
 133                 break;
 134
 135         case kPEEnableScreen:
 136                 initialize_screen(info, op);
 137                 if (info)
 138                         PE_state.video = *info;
 139                 kprintf("kPEEnableScreen %d\n", last_console);
 140                 if (last_console != -1)
 141                         switch_to_old_console(last_console);
 142                 break;
 143
 144         case kPEReleaseScreen:
 145                 /*
 146                  * we don't show the progress indicator on boot, but want to
 147                  * show it afterwards.
 148                  */
 149                 if (!vc_progress_initialized) {
 150                         default_progress.dx = 0;
 151                         default_progress.dy = 0;
 152                         vc_progress_initialize(&default_progress,
 153                                                default_progress_data1x,
 154                                                default_progress_data2x,
 155                                                default_progress_data3x,
 156                                                (unsigned char *) appleClut8);
 157                         vc_progress_initialized = TRUE;
 158                 }
 159                 initialize_screen(info, op);
 160                 break;
 161
 162         default:
 163                 initialize_screen(info, op);
 164                 break;
 165         }
 166
 167         return 0;
 168 }
 169
 170 void
 171 PE_init_iokit(void)
 172 {
 173         DTEntry         entry;
 174         unsigned int    size, scale;
 175         unsigned long   display_size;
 176         void            **map;
 177         unsigned int    show_progress;
 178         int             *delta, image_size, flip;
 179         uint32_t        start_time_value = 0;
 180         uint32_t        debug_wait_start_value = 0;
 181         uint32_t        load_kernel_start_value = 0;
 182         uint32_t        populate_registry_time_value = 0;
 183
 184         PE_init_kprintf(TRUE);
 185         PE_init_printf(TRUE);
 186
 187         printf("iBoot version: %s\n", firmware_version);
 188
 189         if (kSuccess == DTLookupEntry(0, "/chosen/memory-map", &entry)) {
 190
 191                 boot_progress_element *bootPict;
 192
 193                 if (kSuccess == DTGetProperty(entry, "BootCLUT", (void **) &map, &size))
 194                         bcopy(map[0], appleClut8, sizeof(appleClut8));
 195
 196                 if (kSuccess == DTGetProperty(entry, "Pict-FailedBoot", (void **) &map, &size)) {
 197
 198                         bootPict = (boot_progress_element *) map[0];
 199                         default_noroot.width = bootPict->width;
 200                         default_noroot.height = bootPict->height;
 201                         default_noroot.dx = 0;
 202                         default_noroot.dy = bootPict->yOffset;
 203                         default_noroot_data = &bootPict->data[0];
 204                 }
 205         }
 206
 207         pe_prepare_images();
 208
 209         scale = PE_state.video.v_scale;
 210         flip = 1;
 211
 212         if (PE_parse_boot_argn("-progress", &show_progress, sizeof (show_progress)) && show_progress) {
 213                 /* Rotation: 0:normal, 1:right 90, 2:left 180, 3:left 90 */
 214                 switch (PE_state.video.v_rotate) {
 215                 case 2:
 216                         flip = -1;
 217                         /* fall through */
 218                 case 0:
 219                         display_size = PE_state.video.v_height;
 220                         image_size = default_progress.height;
 221                         delta = &default_progress.dy;
 222                         break;
 223                 case 1:
 224                         flip = -1;
 225                         /* fall through */
 226                 case 3:
 227                 default:
 228                         display_size = PE_state.video.v_width;
 229                         image_size = default_progress.width;
 230                         delta = &default_progress.dx;
 231                 }
 232                 assert(*delta >= 0);
 233                 while (((unsigned)(*delta + image_size)) >= (display_size / 2)) {
 234                         *delta -= 50 * scale;
 235                         assert(*delta >= 0);
 236                 }
 237                 *delta *= flip;
 238
 239                 /* Check for DT-defined progress y delta */
 240                 PE_get_default("progress-dy", &default_progress.dy, sizeof(default_progress.dy));
 241
 242                 vc_progress_initialize(&default_progress,
 243                                        default_progress_data1x,
 244                                        default_progress_data2x,
 245                                        default_progress_data3x,
 246                                        (unsigned char *) appleClut8);
 247                 vc_progress_initialized = TRUE;
 248         }
 249
 250         if (kdebug_enable && kdebug_debugid_enabled(IOKDBG_CODE(DBG_BOOTER, 0))) {
 251                 /* Trace iBoot-provided timing information. */
 252                 if (kSuccess == DTLookupEntry(0, "/chosen/iBoot", &entry)) {
 253                         uint32_t * value_ptr;
 254
 255                         if (kSuccess == DTGetProperty(entry, "start-time", (void **)&value_ptr, &size)) {
 256                                 if (size == sizeof(start_time_value))
 257                                         start_time_value = *value_ptr;
 258                         }
 259
 260                         if (kSuccess == DTGetProperty(entry, "debug-wait-start", (void **)&value_ptr, &size)) {
 261                                 if (size == sizeof(debug_wait_start_value))
 262                                         debug_wait_start_value = *value_ptr;
 263                         }
 264
 265                         if (kSuccess == DTGetProperty(entry, "load-kernel-start", (void **)&value_ptr, &size)) {
 266                                 if (size == sizeof(load_kernel_start_value))
 267                                         load_kernel_start_value = *value_ptr;
 268                         }
 269
 270                         if (kSuccess == DTGetProperty(entry, "populate-registry-time", (void **)&value_ptr, &size)) {
 271                                 if (size == sizeof(populate_registry_time_value))
 272                                         populate_registry_time_value = *value_ptr;
 273                         }
 274                 }
 275
 276                 KDBG_RELEASE(IOKDBG_CODE(DBG_BOOTER, 0), start_time_value, debug_wait_start_value, load_kernel_start_value, populate_registry_time_value);
 277         }
 278
 279         StartIOKit(PE_state.deviceTreeHead, PE_state.bootArgs, (void *) 0, (void *) 0);
 280 }
 281
 282 void
 283 PE_init_platform(boolean_t vm_initialized, void *args)
 284 {
 285         DTEntry         entry;
 286         unsigned int    size;
 287         void          **prop;
 288         boot_args      *boot_args_ptr = (boot_args *) args;
 289
 290         if (PE_state.initialized == FALSE) {
 291                 PE_state.initialized = TRUE;
 292                 PE_state.bootArgs = boot_args_ptr;
 293                 PE_state.deviceTreeHead = boot_args_ptr->deviceTreeP;
 294                 PE_state.video.v_baseAddr = boot_args_ptr->Video.v_baseAddr;
 295                 PE_state.video.v_rowBytes = boot_args_ptr->Video.v_rowBytes;
 296                 PE_state.video.v_width = boot_args_ptr->Video.v_width;
 297                 PE_state.video.v_height = boot_args_ptr->Video.v_height;
 298                 PE_state.video.v_depth = (boot_args_ptr->Video.v_depth >> kBootVideoDepthDepthShift) & kBootVideoDepthMask;
 299                 PE_state.video.v_rotate = (boot_args_ptr->Video.v_depth >> kBootVideoDepthRotateShift) & kBootVideoDepthMask;
 300                 PE_state.video.v_scale = ((boot_args_ptr->Video.v_depth >> kBootVideoDepthScaleShift) & kBootVideoDepthMask) + 1;
 301                 PE_state.video.v_display = boot_args_ptr->Video.v_display;
 302                 strlcpy(PE_state.video.v_pixelFormat, "BBBBBBBBGGGGGGGGRRRRRRRR", sizeof(PE_state.video.v_pixelFormat));
 303         }
 304         if (!vm_initialized) {
 305                 /*
 306                  * Setup the Device Tree routines
 307                  * so the console can be found and the right I/O space
 308                  * can be used..
 309                  */
 310                 DTInit(PE_state.deviceTreeHead);
 311                 pe_identify_machine(boot_args_ptr);
 312         } else {
 313                 pe_arm_init_interrupts(args);
 314                 pe_arm_init_debug(args);
 315         }
 316
 317         if (!vm_initialized) {
 318                 if (kSuccess == (DTFindEntry("name", "device-tree", &entry))) {
 319                         if (kSuccess == DTGetProperty(entry, "target-type",
 320                                 (void **)&prop, &size)) {
 321                                 if (size > sizeof(gTargetTypeBuffer))
 322                                         size = sizeof(gTargetTypeBuffer);
 323                                 bcopy(prop,gTargetTypeBuffer,size);
 324                                 gTargetTypeBuffer[size-1]='\0';
 325                         }
 326                 }
 327                 if (kSuccess == (DTFindEntry("name", "device-tree", &entry))) {
 328                         if (kSuccess == DTGetProperty(entry, "model",
 329                                 (void **)&prop, &size)) {
 330                                 if (size > sizeof(gModelTypeBuffer))
 331                                         size = sizeof(gModelTypeBuffer);
 332                                 bcopy(prop,gModelTypeBuffer,size);
 333                                 gModelTypeBuffer[size-1]='\0';
 334                         }
 335                 }
 336                 if (kSuccess == DTLookupEntry(NULL, "/chosen", &entry)) {
 337                         if (kSuccess == DTGetProperty(entry, "debug-enabled",
 338                                                       (void **) &prop, &size)) {
 339                                 /*
 340                                  * We purposefully modify a constified variable as
 341                                  * it will get locked down by a trusted monitor or
 342                                  * via page table mappings. We don't want people easily
 343                                  * modifying this variable...
 344                                  */
 345 #pragma clang diagnostic push
 346 #pragma clang diagnostic ignored "-Wcast-qual"
 347                                 boolean_t *modify_debug_enabled = (boolean_t *) &debug_enabled;
 348                                 if (size > sizeof(uint32_t))
 349                                         size = sizeof(uint32_t);
 350                                 bcopy(prop, modify_debug_enabled, size);
 351 #pragma clang diagnostic pop
 352                         }
 353                         if (kSuccess == DTGetProperty(entry, "firmware-version",
 354                                                       (void **) &prop, &size)) {
 355                                 if (size > sizeof(firmware_version))
 356                                         size = sizeof(firmware_version);
 357                                 bcopy(prop, firmware_version, size);
 358                                 firmware_version[size - 1] = '\0';
 359                         }
 360                         if (kSuccess == DTGetProperty(entry, "unique-chip-id",
 361                                                       (void **) &prop, &size)) {
 362                                 if (size > sizeof(gPlatformECID))
 363                                         size = sizeof(gPlatformECID);
 364                                 bcopy(prop,gPlatformECID,size);
 365                         }
 366                         if (kSuccess == DTGetProperty(entry, "dram-vendor-id",
 367                                                       (void **) &prop, &size)) {
 368                                 if (size > sizeof(gPlatformMemoryID))
 369                                         size = sizeof(gPlatformMemoryID);
 370                                 bcopy(prop,&gPlatformMemoryID,size);
 371                         }
 372                 }
 373                 pe_init_debug();
 374         }
 375 }
 376
 377 void
 378 PE_create_console(void)
 379 {
 380         /*
 381          * Check the head of VRAM for a panic log saved on last panic.
 382          * Do this before the VRAM is trashed.
 383          */
 384         check_for_panic_log();
 385
 386         if (PE_state.video.v_display)
 387                 PE_initialize_console(&PE_state.video, kPEGraphicsMode);
 388         else
 389                 PE_initialize_console(&PE_state.video, kPETextMode);
 390 }
 391
 392 int
 393 PE_current_console(PE_Video * info)
 394 {
 395         *info = PE_state.video;
 396         return (0);
 397 }
 398
 399 void
 400 PE_display_icon(__unused unsigned int flags, __unused const char *name)
 401 {
 402         if (default_noroot_data)
 403                 vc_display_icon(&default_noroot, default_noroot_data);
 404 }
 405
 406 extern          boolean_t
 407 PE_get_hotkey(__unused unsigned char key)
 408 {
 409         return (FALSE);
 410 }
 411
 412 static timebase_callback_func gTimebaseCallback;
 413
 414 void
 415 PE_register_timebase_callback(timebase_callback_func callback)
 416 {
 417         gTimebaseCallback = callback;
 418
 419         PE_call_timebase_callback();
 420 }
 421
 422 void
 423 PE_call_timebase_callback(void)
 424 {
 425         struct timebase_freq_t timebase_freq;
 426
 427         timebase_freq.timebase_num = gPEClockFrequencyInfo.timebase_frequency_hz;
 428         timebase_freq.timebase_den = 1;
 429
 430         if (gTimebaseCallback)
 431                 gTimebaseCallback(&timebase_freq);
 432 }
 433
 434 /*
 435  * The default PE_poll_input handler.
 436  */
 437 static int
 438 PE_stub_poll_input(__unused unsigned int options, char *c)
 439 {
 440         *c = uart_getc();
 441         return 0;               /* 0 for success, 1 for unsupported */
 442 }
 443
 444 /*
 445  * Called by the kernel debugger to poll for keyboard input.
 446  * Keyboard drivers may replace the default stub function
 447  * with their polled-mode input function.
 448  */
 449 int             (*PE_poll_input) (unsigned int options, char *c) = PE_stub_poll_input;
 450
 451 /*
 452  * This routine will return 1 if you are running on a device with a variant
 453  * of iBoot that allows debugging. This is typically not the case on production
 454  * fused parts (even when running development variants of iBoot).
 455  *
 456  * The routine takes an optional argument of the flags passed to debug="" so
 457  * kexts don't have to parse the boot arg themselves.
 458  */
 459 uint32_t
 460 PE_i_can_has_debugger(uint32_t *debug_flags)
 461 {
 462         if (debug_flags) {
 463                 if (debug_enabled)
 464                         *debug_flags = debug_boot_arg;
 465                 else
 466                         *debug_flags = 0;
 467         }
 468         return (debug_enabled);
 469 }
 470
 471 void
 472 PE_save_buffer_to_vram(unsigned char *buf, unsigned int *size)
 473 {
 474         if (!panic_info || !size) {
 475                 return;
 476         }
 477
 478         if (!buf) {
 479                 *size = panic_text_len;
 480                 return;
 481         }
 482
 483         if (*size == 0) {
 484                 return;
 485         }
 486
 487         *size = *size > panic_text_len ? panic_text_len : *size;
 488         if (panic_info->eph_magic != EMBEDDED_PANIC_MAGIC)
 489                 printf("Error!! Current Magic 0x%X, expected value 0x%x", panic_info->eph_magic, EMBEDDED_PANIC_MAGIC);
 490
 491         /* CRC everything after the CRC itself - starting with the panic header version */
 492         panic_info->eph_crc = crc32(0L, &panic_info->eph_version, (panic_text_len +
 493                                 sizeof(struct embedded_panic_header) - offsetof(struct embedded_panic_header, eph_version)));
 494 }
 495
 496 uint32_t
 497 PE_get_offset_into_panic_region(char *location)
 498 {
 499         assert(panic_info != NULL);
 500         assert(location > (char *) panic_info);
 501         assert((unsigned int)(location - (char *) panic_info) < panic_text_len);
 502
 503         return (uint32_t) (location - gPanicBase);
 504 }
 505
 506 void
 507 PE_init_panicheader()
 508 {
 509         if (!panic_info)
 510                 return;
 511
 512         bzero(panic_info, sizeof(struct embedded_panic_header));
 513
 514         /*
 515          * The panic log begins immediately after the panic header -- debugger synchronization and other functions
 516          * may log into this region before we've become the exclusive panicking CPU and initialize the header here.
 517          */
 518         panic_info->eph_panic_log_offset = PE_get_offset_into_panic_region(debug_buf_base);
 519
 520         panic_info->eph_magic = EMBEDDED_PANIC_MAGIC;
 521         panic_info->eph_version = EMBEDDED_PANIC_HEADER_CURRENT_VERSION;
 522
 523         return;
 524 }
 525
 526 /*
 527  * Tries to update the panic header to keep it consistent on nested panics.
 528  *
 529  * NOTE: The purpose of this function is NOT to detect/correct corruption in the panic region,
 530  *       it is to update the panic header to make it consistent when we nest panics.
 531  */
 532 void
 533 PE_update_panicheader_nestedpanic()
 534 {
 535         if (!panic_info)
 536                 return;
 537
 538         /*
 539          * If the panic log offset is not set, re-init the panic header
 540          */
 541         if (panic_info->eph_panic_log_offset == 0) {
 542                 PE_init_panicheader();
 543                 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_NESTED_PANIC;
 544                 return;
 545         }
 546
 547         panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_NESTED_PANIC;
 548
 549         /*
 550          * If the panic log length is not set, set the end to
 551          * the current location of the debug_buf_ptr to close it.
 552          */
 553         if (panic_info->eph_panic_log_len == 0) {
 554                 panic_info->eph_panic_log_len = PE_get_offset_into_panic_region(debug_buf_ptr);
 555
 556                 /* If this assert fires, it's indicative of corruption in the panic region */
 557                 assert(panic_info->eph_other_log_offset == panic_info->eph_other_log_len == 0);
 558         }
 559
 560         /* If this assert fires, it's likely indicative of corruption in the panic region */
 561         assert(((panic_info->eph_stackshot_offset == 0) && (panic_info->eph_stackshot_len == 0)) ||
 562                         ((panic_info->eph_stackshot_offset != 0) && (panic_info->eph_stackshot_len != 0)));
 563
 564         /*
 565          * If we haven't set up the other log yet, set the beginning of the other log
 566          * to the current location of the debug_buf_ptr
 567          */
 568         if (panic_info->eph_other_log_offset == 0) {
 569                 panic_info->eph_other_log_offset = PE_get_offset_into_panic_region(debug_buf_ptr);
 570
 571                 /* If this assert fires, it's indicative of corruption in the panic region */
 572                 assert(panic_info->eph_other_log_len == 0);
 573         }
 574
 575         return;
 576 }
 577
 578 boolean_t
 579 PE_reboot_on_panic(void)
 580 {
 581         uint32_t debug_flags;
 582
 583         if (PE_i_can_has_debugger(&debug_flags)
 584                 && (debug_flags & DB_NMI)) {
 585                 /* kernel debugging is active */
 586                 return FALSE;
 587         } else {
 588                 return TRUE;
 589         }
 590 }
 591
 592 void
 593 PE_sync_panic_buffers(void)
 594 {
 595         /*
 596          * rdar://problem/26453070:
 597          * The iBoot panic region is write-combined on arm64.  We must flush dirty lines
 598          * from L1/L2 as late as possible before reset, with no further reads of the panic
 599          * region between the flush and the reset.  Some targets have an additional memcache (L3),
 600          * and a read may bring dirty lines out of L3 and back into L1/L2, causing the lines to
 601          * be discarded on reset.  If we can make sure the lines are flushed to L3/DRAM,
 602          * the platform reset handler will flush any L3.
 603          */
 604         if (gPanicBase)
 605                 CleanPoC_DcacheRegion_Force(gPanicBase, gPanicSize);
 606 }
 607
 608 static void
 609 pe_prepare_images(void)
 610 {
 611         if ((1 & PE_state.video.v_rotate) != 0) {
 612                 // Only square square images with radial symmetry are supported
 613                 // No need to actually rotate the data
 614
 615                 // Swap the dx and dy offsets
 616                 uint32_t tmp = default_progress.dx;
 617                 default_progress.dx = default_progress.dy;
 618                 default_progress.dy = tmp;
 619         }
 620 #if 0
 621         uint32_t cnt, cnt2, cnt3, cnt4;
 622         uint32_t tmp, width, height;
 623         uint8_t  data, *new_data;
 624         const uint8_t *old_data;
 625
 626         width  = default_progress.width;
 627         height = default_progress.height * default_progress.count;
 628
 629         // Scale images if the UI is being scaled
 630         if (PE_state.video.v_scale > 1) {
 631                 new_data = kalloc(width * height * scale * scale);
 632                 if (new_data != 0) {
 633                         old_data = default_progress_data;
 634                         default_progress_data = new_data;
 635                         for (cnt = 0; cnt < height; cnt++) {
 636                                 for (cnt2 = 0; cnt2 < width; cnt2++) {
 637                                         data = *(old_data++);
 638                                         for (cnt3 = 0; cnt3 < scale; cnt3++) {
 639                                                 for (cnt4 = 0; cnt4 < scale; cnt4++) {
 640                                                         new_data[width * scale * cnt3 + cnt4] = data;
 641                                                 }
 642                                         }
 643                                         new_data += scale;
 644                                 }
 645                                 new_data += width * scale * (scale - 1);
 646                         }
 647                         default_progress.width  *= scale;
 648                         default_progress.height *= scale;
 649                         default_progress.dx     *= scale;
 650                         default_progress.dy     *= scale;
 651                 }
 652         }
 653 #endif
 654 }
 655
 656 void
 657 PE_mark_hwaccess(uint64_t thread)
 658 {
 659         last_hwaccess_thread = thread;
 660         asm volatile("dmb ish");
 661 }