osfmk/i386/hibernate_i386.c

   1 /*
   2  * Copyright (c) 2004-2020 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28 /*!
  29  * i386/x86_64-specific functions required to support hibernation entry, and also to
  30  * support hibernation exit after wired pages have already been restored.
  31  */
  32
  33 #include <kern/machine.h>
  34 #include <kern/misc_protos.h>
  35 #include <kern/thread.h>
  36 #include <kern/processor.h>
  37 #include <kern/kalloc.h>
  38 #include <mach/machine.h>
  39 #include <mach/processor_info.h>
  40 #include <mach/mach_types.h>
  41 #include <i386/pmap.h>
  42 #include <kern/cpu_data.h>
  43 #include <IOKit/IOPlatformExpert.h>
  44
  45 #include <pexpert/i386/efi.h>
  46
  47 #include <IOKit/IOHibernatePrivate.h>
  48 #include <machine/pal_hibernate.h>
  49 #include <vm/vm_page.h>
  50 #include <i386/i386_lowmem.h>
  51 #include <san/kasan.h>
  52
  53 extern ppnum_t max_ppnum;
  54
  55 #define MAX_BANKS       32
  56
  57 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  58
  59 hibernate_page_list_t *
  60 hibernate_page_list_allocate(boolean_t log)
  61 {
  62         ppnum_t                 base, num;
  63         vm_size_t               size;
  64         uint32_t                bank, num_banks;
  65         uint32_t                pages, page_count;
  66         hibernate_page_list_t * list;
  67         hibernate_bitmap_t *    bitmap;
  68
  69         EfiMemoryRange *        mptr;
  70         uint32_t                mcount, msize, i;
  71         hibernate_bitmap_t      dram_ranges[MAX_BANKS];
  72         boot_args *             args = (boot_args *) PE_state.bootArgs;
  73         uint32_t                non_os_pagecount;
  74         ppnum_t                 pnmax = max_ppnum;
  75
  76         mptr = (EfiMemoryRange *)ml_static_ptovirt(args->MemoryMap);
  77         if (args->MemoryMapDescriptorSize == 0) {
  78                 panic("Invalid memory map descriptor size");
  79         }
  80         msize = args->MemoryMapDescriptorSize;
  81         mcount = args->MemoryMapSize / msize;
  82
  83 #if KASAN
  84         /* adjust max page number to include stolen memory */
  85         if (atop(shadow_ptop) > pnmax) {
  86                 pnmax = (ppnum_t)atop(shadow_ptop);
  87         }
  88 #endif
  89
  90         num_banks = 0;
  91         non_os_pagecount = 0;
  92         for (i = 0; i < mcount; i++, mptr = (EfiMemoryRange *)(((vm_offset_t)mptr) + msize)) {
  93                 base = (ppnum_t) (mptr->PhysicalStart >> I386_PGSHIFT);
  94                 num = (ppnum_t) mptr->NumberOfPages;
  95
  96 #if KASAN
  97                 if (i == shadow_stolen_idx) {
  98                         /*
  99                          * Add all stolen pages to the bitmap. Later we will prune the unused
 100                          * pages.
 101                          */
 102                         num += shadow_pages_total;
 103                 }
 104 #endif
 105
 106                 if (base > pnmax) {
 107                         continue;
 108                 }
 109                 if ((base + num - 1) > pnmax) {
 110                         num = pnmax - base + 1;
 111                 }
 112                 if (!num) {
 113                         continue;
 114                 }
 115
 116                 switch (mptr->Type) {
 117                 // any kind of dram
 118                 case kEfiACPIMemoryNVS:
 119                 case kEfiPalCode:
 120                         non_os_pagecount += num;
 121                         OS_FALLTHROUGH;
 122
 123                 // OS used dram
 124                 case kEfiLoaderCode:
 125                 case kEfiLoaderData:
 126                 case kEfiBootServicesCode:
 127                 case kEfiBootServicesData:
 128                 case kEfiConventionalMemory:
 129
 130                         for (bank = 0; bank < num_banks; bank++) {
 131                                 if (dram_ranges[bank].first_page <= base) {
 132                                         continue;
 133                                 }
 134                                 if ((base + num) == dram_ranges[bank].first_page) {
 135                                         dram_ranges[bank].first_page = base;
 136                                         num = 0;
 137                                 }
 138                                 break;
 139                         }
 140                         if (!num) {
 141                                 break;
 142                         }
 143
 144                         if (bank && (base == (1 + dram_ranges[bank - 1].last_page))) {
 145                                 bank--;
 146                         } else {
 147                                 num_banks++;
 148                                 if (num_banks >= MAX_BANKS) {
 149                                         break;
 150                                 }
 151                                 bcopy(&dram_ranges[bank],
 152                                     &dram_ranges[bank + 1],
 153                                     (num_banks - bank - 1) * sizeof(hibernate_bitmap_t));
 154                                 dram_ranges[bank].first_page = base;
 155                         }
 156                         dram_ranges[bank].last_page = base + num - 1;
 157                         break;
 158
 159                 // runtime services will be restarted, so no save
 160                 case kEfiRuntimeServicesCode:
 161                 case kEfiRuntimeServicesData:
 162                 // contents are volatile once the platform expert starts
 163                 case kEfiACPIReclaimMemory:
 164                 // non dram
 165                 case kEfiReservedMemoryType:
 166                 case kEfiUnusableMemory:
 167                 case kEfiMemoryMappedIO:
 168                 case kEfiMemoryMappedIOPortSpace:
 169                 default:
 170                         break;
 171                 }
 172         }
 173
 174         if (num_banks >= MAX_BANKS) {
 175                 HIBLOG("%s error, num_banks exceed MAX_BANKS(0x%x)\n", __FUNCTION__, MAX_BANKS);
 176                 return NULL;
 177         }
 178
 179         // size the hibernation bitmap
 180
 181         size = sizeof(hibernate_page_list_t);
 182         page_count = 0;
 183         for (bank = 0; bank < num_banks; bank++) {
 184                 pages = dram_ranges[bank].last_page + 1 - dram_ranges[bank].first_page;
 185                 page_count += pages;
 186                 size += sizeof(hibernate_bitmap_t) + ((pages + 31) >> 5) * sizeof(uint32_t);
 187         }
 188
 189         list = (hibernate_page_list_t *)kalloc(size);
 190         if (!list) {
 191                 return list;
 192         }
 193
 194         list->list_size  = (uint32_t)size;
 195         list->page_count = page_count;
 196         list->bank_count = num_banks;
 197
 198         // convert to hibernation bitmap.
 199
 200         bitmap = &list->bank_bitmap[0];
 201         for (bank = 0; bank < num_banks; bank++) {
 202                 bitmap->first_page = dram_ranges[bank].first_page;
 203                 bitmap->last_page  = dram_ranges[bank].last_page;
 204                 bitmap->bitmapwords = (bitmap->last_page + 1
 205                     - bitmap->first_page + 31) >> 5;
 206                 if (log) {
 207                         kprintf("hib bank[%d]: 0x%x000 end 0x%xfff\n",
 208                             bank, bitmap->first_page, bitmap->last_page);
 209                 }
 210                 bitmap = (hibernate_bitmap_t *) &bitmap->bitmap[bitmap->bitmapwords];
 211         }
 212         if (log) {
 213                 printf("efi pagecount %d\n", non_os_pagecount);
 214         }
 215
 216         return list;
 217 }
 218
 219 // mark pages not to be saved, but available for scratch usage during restore
 220
 221 void
 222 hibernate_page_list_setall_machine( __unused hibernate_page_list_t * page_list,
 223     __unused hibernate_page_list_t * page_list_wired,
 224     __unused boolean_t preflight,
 225     __unused uint32_t * pagesOut)
 226 {
 227 }
 228
 229 // mark pages not to be saved and not for scratch usage during restore
 230 void
 231 hibernate_page_list_set_volatile( hibernate_page_list_t * page_list,
 232     hibernate_page_list_t * page_list_wired,
 233     uint32_t * pagesOut)
 234 {
 235         boot_args * args = (boot_args *) PE_state.bootArgs;
 236
 237         if (args->efiRuntimeServicesPageStart) {
 238                 hibernate_set_page_state(page_list, page_list_wired,
 239                     args->efiRuntimeServicesPageStart, args->efiRuntimeServicesPageCount,
 240                     kIOHibernatePageStateFree);
 241                 *pagesOut -= args->efiRuntimeServicesPageCount;
 242         }
 243 }
 244
 245 kern_return_t
 246 hibernate_processor_setup(IOHibernateImageHeader * header)
 247 {
 248         boot_args * args = (boot_args *) PE_state.bootArgs;
 249
 250         cpu_datap(0)->cpu_hibernate = 1;
 251         header->processorFlags = 0;
 252
 253         header->runtimePages     = args->efiRuntimeServicesPageStart;
 254         header->runtimePageCount = args->efiRuntimeServicesPageCount;
 255         header->runtimeVirtualPages = args->efiRuntimeServicesVirtualPageStart;
 256         header->performanceDataStart = args->performanceDataStart;
 257         header->performanceDataSize = args->performanceDataSize;
 258
 259         return KERN_SUCCESS;
 260 }
 261
 262 static boolean_t hibernate_vm_locks_safe;
 263
 264 void
 265 hibernate_vm_lock(void)
 266 {
 267         if (current_cpu_datap()->cpu_hibernate) {
 268                 hibernate_vm_lock_queues();
 269                 hibernate_vm_locks_safe = TRUE;
 270         }
 271 }
 272
 273 void
 274 hibernate_vm_unlock(void)
 275 {
 276         assert(FALSE == ml_get_interrupts_enabled());
 277         if (current_cpu_datap()->cpu_hibernate) {
 278                 hibernate_vm_unlock_queues();
 279         }
 280         ml_set_is_quiescing(TRUE);
 281 }
 282
 283 // ACPI calls hibernate_vm_lock(), interrupt disable, hibernate_vm_unlock() on sleep,
 284 // hibernate_vm_lock_end() and interrupt enable on wake.
 285 // VM locks are safely single threaded between hibernate_vm_lock() and hibernate_vm_lock_end().
 286
 287 void
 288 hibernate_vm_lock_end(void)
 289 {
 290         assert(FALSE == ml_get_interrupts_enabled());
 291         hibernate_vm_locks_safe = FALSE;
 292         ml_set_is_quiescing(FALSE);
 293 }
 294
 295 boolean_t
 296 hibernate_vm_locks_are_safe(void)
 297 {
 298         assert(FALSE == ml_get_interrupts_enabled());
 299         return hibernate_vm_locks_safe;
 300 }
 301
 302 void
 303 pal_hib_write_hook(void)
 304 {
 305 }