/*
- * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 2004-2016 Apple Inc. All rights reserved.
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
- * @APPLE_LICENSE_HEADER_START@
- *
* This file contains Original Code and/or Modifications of Original Code
* as defined in and that are subject to the Apple Public Source License
* Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this
- * file.
- *
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ *
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ *
* The Original Code and all software distributed under the License are
* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
* Please see the License for the specific language governing rights and
* limitations under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
+ *
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
-
/*
-
-Sleep:
-
-- PMRootDomain calls IOHibernateSystemSleep() before system sleep
-(devices awake, normal execution context)
-- IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
- grabs its extents and searches for a polling driver willing to work with that IOMedia.
- The BSD code makes an ioctl to the storage driver to get the partition base offset to
- the disk, and other ioctls to get the transfer constraints
- If successful, the file is written to make sure its initially not bootable (in case of
- later failure) and nvram set to point to the first block of the file. (Has to be done
- here so blocking is possible in nvram support).
- hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
- page out any pages it wants to (currently zero, but probably some percentage of memory).
- Its assumed just allocating pages will cause the VM system to naturally select the best
- pages for eviction. It also copies processor flags needed for the restore path and sets
- a flag in the boot processor proc info.
- gIOHibernateState = kIOHibernateStateHibernating.
-- Regular sleep progresses - some drivers may inspect the root domain property
- kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
- as usual but leaves motherboard I/O on.
-- Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
- at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
- all ppc RC bits out of the hash table and caches into the mapping structures.
-- hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
- hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
- All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
- by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
- The image header and block list are written. The header includes the second file extent so
- only the header block is needed to read the file, regardless of filesystem.
- The kernel section "__HIB" is written uncompressed to the image. This section of code and data
- (only) is used to decompress the image during wake/boot.
- Some additional pages are removed from the bitmaps - the buffers used for hibernation.
- The bitmaps are written to the image.
- More areas are removed from the bitmaps (after they have been written to the image) - the
- section "__HIB" pages and interrupt stack.
- Each wired page is compressed and written and then each non-wired page. Compression and
- disk writes are in parallel.
- The image header is written to the start of the file and the polling driver closed.
- The machine powers down (or sleeps).
-
-Boot/Wake:
-
-- BootX sees the boot-image nvram variable containing the device and block number of the image,
- reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
-- BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
- in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
- that is in the kernel's __HIB section.
-- BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
- hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
- only code & data in that section is safe to call since all the other wired pages are still
- compressed in the image.
-- hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
- It uses the bitmaps to work out which pages can be uncompressed from the image to their final
- location directly, and copies those that can't to interim free pages. When the image has been
- completed, the copies are uncompressed, overwriting the wired image pages.
- hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
- is used to get pages into place for 64bit.
-- the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
- changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
- are removed from the software strutures, and the hash table is reinitialized.
-- After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
- of memory, using the polled mode driver, before other threads can run or any devices are turned on.
- This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
- for the remaining non wired pages.
-- The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
- (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
- is closed via bsd.
-
-Polled Mode I/O:
-
-IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
-registry, specifying an object of calls IOPolledInterface.
-
-Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
-partition) that the image is going to live, looking for polled interface properties. If it finds
-one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
-interfaces found are kept in an ordered list.
-
-There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
-few different contexts things happen in:
-
-- there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
-up and running) and after wake - this is safe to allocate memory and do anything. The device
-ignores sleep requests from that point since its a waste of time if it goes to sleep and
-immediately wakes back up for the image write.
-
-- there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
-immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
-from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
-used to flush and set the disk to sleep.
-
-- there is an Open/Close (AfterSleep) pair made around the image read operations that happen
-immediately after sleep. These can't block or allocate memory. This is happening after the platform
-expert has woken the low level bits of the system, but most of the I/O system has not. There is only
-one thread running.
-
-For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
-(prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
-the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
-Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
-that is called for the hardware to check for events, and complete the I/O via the callback.
-The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
-to restrict I/O ops.
-*/
+ *
+ * Sleep:
+ *
+ * - PMRootDomain calls IOHibernateSystemSleep() before system sleep
+ * (devices awake, normal execution context)
+ * - IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
+ * grabs its extents and searches for a polling driver willing to work with that IOMedia.
+ * The BSD code makes an ioctl to the storage driver to get the partition base offset to
+ * the disk, and other ioctls to get the transfer constraints
+ * If successful, the file is written to make sure its initially not bootable (in case of
+ * later failure) and nvram set to point to the first block of the file. (Has to be done
+ * here so blocking is possible in nvram support).
+ * hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
+ * page out any pages it wants to (currently zero, but probably some percentage of memory).
+ * Its assumed just allocating pages will cause the VM system to naturally select the best
+ * pages for eviction. It also copies processor flags needed for the restore path and sets
+ * a flag in the boot processor proc info.
+ * gIOHibernateState = kIOHibernateStateHibernating.
+ * - Regular sleep progresses - some drivers may inspect the root domain property
+ * kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
+ * as usual but leaves motherboard I/O on.
+ * - Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
+ * at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
+ * all ppc RC bits out of the hash table and caches into the mapping structures.
+ * - hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
+ * hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
+ * All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
+ * by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
+ * The image header and block list are written. The header includes the second file extent so
+ * only the header block is needed to read the file, regardless of filesystem.
+ * The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data
+ * (only) is used to decompress the image during wake/boot.
+ * Some additional pages are removed from the bitmaps - the buffers used for hibernation.
+ * The bitmaps are written to the image.
+ * More areas are removed from the bitmaps (after they have been written to the image) - the
+ * segment "__HIB" pages and interrupt stack.
+ * Each wired page is compressed and written and then each non-wired page. Compression and
+ * disk writes are in parallel.
+ * The image header is written to the start of the file and the polling driver closed.
+ * The machine powers down (or sleeps).
+ *
+ * Boot/Wake:
+ *
+ * - BootX sees the boot-image nvram variable containing the device and block number of the image,
+ * reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
+ * - BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
+ * in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
+ * that is in the kernel's __HIB section.
+ * - BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
+ * hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
+ * only code & data in that section is safe to call since all the other wired pages are still
+ * compressed in the image.
+ * - hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
+ * It uses the bitmaps to work out which pages can be uncompressed from the image to their final
+ * location directly, and copies those that can't to interim free pages. When the image has been
+ * completed, the copies are uncompressed, overwriting the wired image pages.
+ * hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
+ * is used to get pages into place for 64bit.
+ * - the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
+ * changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
+ * are removed from the software strutures, and the hash table is reinitialized.
+ * - After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
+ * of memory, using the polled mode driver, before other threads can run or any devices are turned on.
+ * This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
+ * for the remaining non wired pages.
+ * - The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
+ * (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
+ * is closed via bsd.
+ *
+ * Polled Mode I/O:
+ *
+ * IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
+ * registry, specifying an object of calls IOPolledInterface.
+ *
+ * Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
+ * partition) that the image is going to live, looking for polled interface properties. If it finds
+ * one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
+ * interfaces found are kept in an ordered list.
+ *
+ * There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
+ * few different contexts things happen in:
+ *
+ * - there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
+ * up and running) and after wake - this is safe to allocate memory and do anything. The device
+ * ignores sleep requests from that point since its a waste of time if it goes to sleep and
+ * immediately wakes back up for the image write.
+ *
+ * - there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
+ * immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
+ * from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
+ * used to flush and set the disk to sleep.
+ *
+ * - there is an Open/Close (AfterSleep) pair made around the image read operations that happen
+ * immediately after sleep. These can't block or allocate memory. This is happening after the platform
+ * expert has woken the low level bits of the system, but most of the I/O system has not. There is only
+ * one thread running.
+ *
+ * For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
+ * (prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
+ * the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
+ * Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
+ * that is called for the hardware to check for events, and complete the I/O via the callback.
+ * The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
+ * to restrict I/O ops.
+ */
#include <sys/systm.h>
#include <IOKit/IOMessage.h>
#include <IOKit/IODeviceTreeSupport.h>
#include <IOKit/IOBSD.h>
+#include <IOKit/IOKitKeysPrivate.h>
#include "RootDomainUserClient.h"
#include <IOKit/pwr_mgt/IOPowerConnection.h>
#include "IOPMPowerStateQueue.h"
#include <IOKit/IOBufferMemoryDescriptor.h>
-#include <crypto/aes.h>
+#include <IOKit/AppleKeyStoreInterface.h>
+#include <libkern/crypto/aes.h>
#include <sys/uio.h>
#include <sys/conf.h>
#include <sys/stat.h>
#include <sys/fcntl.h> // (FWRITE, ...)
-extern "C" {
#include <sys/sysctl.h>
-}
+#include <sys/kdebug.h>
+#include <stdint.h>
#include <IOKit/IOHibernatePrivate.h>
#include <IOKit/IOPolledInterface.h>
#include <IOKit/IONVRAM.h>
#include "IOHibernateInternal.h"
-#include "WKdm.h"
+#include <vm/vm_protos.h>
#include "IOKitKernelInternal.h"
+#include <pexpert/device_tree.h>
+
+#include <machine/pal_routines.h>
+#include <machine/pal_hibernate.h>
+#if defined(__i386__) || defined(__x86_64__)
+#include <i386/tsc.h>
+#include <i386/cpuid.h>
+#include <vm/WKdm_new.h>
+#elif defined(__arm64__)
+#include <arm64/amcc_rorgn.h>
+#endif /* defined(__i386__) || defined(__x86_64__) */
+#include <san/kasan.h>
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject);
-
-OSMetaClassDefineReservedUnused(IOPolledInterface, 0);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 1);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 2);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 3);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 4);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 5);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 6);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 7);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 8);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 9);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 10);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 11);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 12);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 13);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 14);
-OSMetaClassDefineReservedUnused(IOPolledInterface, 15);
+extern "C" addr64_t kvtophys(vm_offset_t va);
+extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-extern uint32_t gIOHibernateState;
-uint32_t gIOHibernateMode;
-static char gIOHibernateBootSignature[256+1];
-static char gIOHibernateFilename[MAXPATHLEN+1];
-static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
-uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms)
+#define DISABLE_TRIM 0
+#define TRIM_DELAY 25000
-static IODTNVRAM * gIOOptionsEntry;
-static IORegistryEntry * gIOChosenEntry;
+extern unsigned int save_kdebug_enable;
+extern uint32_t gIOHibernateState;
+uint32_t gIOHibernateMode;
+static char gIOHibernateBootSignature[256 + 1];
+static char gIOHibernateFilename[MAXPATHLEN + 1];
+uint32_t gIOHibernateCount;
-static IOPolledFileIOVars gFileVars;
-static IOHibernateVars gIOHibernateVars;
-static struct kern_direct_file_io_ref_t * gIOHibernateFileRef;
-static hibernate_cryptvars_t gIOHibernateCryptWakeContext;
+static uuid_string_t gIOHibernateBridgeBootSessionUUIDString;
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
+uint32_t gIOHibernateFreeTime = 0 * 1000; // max time to spend freeing pages (ms)
-enum { kXPRamAudioVolume = 8 };
-enum { kDefaultIOSize = 128 * 1024 };
-enum { kVideoMapSize = 32 * 1024 * 1024 };
+enum {
+ HIB_COMPR_RATIO_ARM64 = (0xa5), // compression ~65%. Since we don't support retries we start higher.
+ HIB_COMPR_RATIO_INTEL = (0x80) // compression 50%
+};
-#ifndef kIOMediaPreferredBlockSizeKey
-#define kIOMediaPreferredBlockSizeKey "Preferred Block Size"
-#endif
+#if defined(__arm64__)
+static uint64_t gIOHibernateCompression = HIB_COMPR_RATIO_ARM64;
+#else
+static uint64_t gIOHibernateCompression = HIB_COMPR_RATIO_INTEL;
+#endif /* __arm64__ */
+boolean_t gIOHibernateStandbyDisabled;
+
+static IODTNVRAM * gIOOptionsEntry;
+static IORegistryEntry * gIOChosenEntry;
+
+static const OSSymbol * gIOHibernateBootImageKey;
+static const OSSymbol * gIOHibernateBootSignatureKey;
+static const OSSymbol * gIOBridgeBootSessionUUIDKey;
+
+#if defined(__i386__) || defined(__x86_64__)
+
+static const OSSymbol * gIOHibernateRTCVariablesKey;
+static const OSSymbol * gIOHibernateBoot0082Key;
+static const OSSymbol * gIOHibernateBootNextKey;
+static OSData * gIOHibernateBoot0082Data;
+static OSData * gIOHibernateBootNextData;
+static OSObject * gIOHibernateBootNextSave;
+
+#endif /* defined(__i386__) || defined(__x86_64__) */
+
+static IOLock * gFSLock;
+uint32_t gFSState;
+static thread_call_t gIOHibernateTrimCalloutEntry;
+static IOPolledFileIOVars gFileVars;
+static IOHibernateVars gIOHibernateVars;
+static IOPolledFileCryptVars gIOHibernateCryptWakeContext;
+static hibernate_graphics_t _hibernateGraphics;
+static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
+static hibernate_statistics_t _hibernateStats;
+static hibernate_statistics_t * gIOHibernateStats = &_hibernateStats;
+
+enum{
+ kFSIdle = 0,
+ kFSOpening = 2,
+ kFSOpened = 3,
+ kFSTimedOut = 4,
+ kFSTrimDelay = 5
+};
-#ifndef kIOBootPathKey
-#define kIOBootPathKey "bootpath"
-#endif
-#ifndef kIOSelectedBootDeviceKey
-#define kIOSelectedBootDeviceKey "boot-device"
-#endif
+static IOReturn IOHibernateDone(IOHibernateVars * vars);
+static IOReturn IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature);
+static void IOSetBootImageNVRAM(OSData * data);
+static void IOHibernateSystemPostWakeTrim(void * p1, void * p2);
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-enum { kIOHibernateMinPollersNeeded = 2 };
+enum { kDefaultIOSize = 128 * 1024 };
+enum { kVideoMapSize = 80 * 1024 * 1024 };
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
static IOReturn
IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
- IOByteCount offset, addr64_t bytes, IOByteCount length)
+ IOByteCount offset, addr64_t bytes, IOByteCount length)
{
- addr64_t srcAddr = bytes;
- IOByteCount remaining;
+ addr64_t srcAddr = bytes;
+ IOByteCount remaining;
- remaining = length = min(length, md->getLength() - offset);
- while (remaining) { // (process another target segment?)
- addr64_t dstAddr64;
- IOByteCount dstLen;
+ remaining = length = min(length, md->getLength() - offset);
+ while (remaining) { // (process another target segment?)
+ addr64_t dstAddr64;
+ IOByteCount dstLen;
- dstAddr64 = md->getPhysicalSegment64(offset, &dstLen);
- if (!dstAddr64)
- break;
+ dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
+ if (!dstAddr64) {
+ break;
+ }
- // Clip segment length to remaining
- if (dstLen > remaining)
- dstLen = remaining;
+ // Clip segment length to remaining
+ if (dstLen > remaining) {
+ dstLen = remaining;
+ }
#if 1
- bcopy_phys(srcAddr, dstAddr64, dstLen);
+ bcopy_phys(srcAddr, dstAddr64, dstLen);
#else
- copypv(srcAddr, dstAddr64, dstLen,
- cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
+ copypv(srcAddr, dstAddr64, dstLen,
+ cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
#endif
- srcAddr += dstLen;
- offset += dstLen;
- remaining -= dstLen;
- }
+ srcAddr += dstLen;
+ offset += dstLen;
+ remaining -= dstLen;
+ }
- assert(!remaining);
+ assert(!remaining);
- return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
+ return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
}
// copy from MD to phys addr
static IOReturn
IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
- IOByteCount offset, addr64_t bytes, IOByteCount length)
+ IOByteCount offset, addr64_t bytes, IOByteCount length)
{
- addr64_t dstAddr = bytes;
- IOByteCount remaining;
+ addr64_t dstAddr = bytes;
+ IOByteCount remaining;
- remaining = length = min(length, md->getLength() - offset);
- while (remaining) { // (process another target segment?)
- addr64_t srcAddr64;
- IOByteCount dstLen;
+ remaining = length = min(length, md->getLength() - offset);
+ while (remaining) { // (process another target segment?)
+ addr64_t srcAddr64;
+ IOByteCount dstLen;
- srcAddr64 = md->getPhysicalSegment64(offset, &dstLen);
- if (!srcAddr64)
- break;
+ srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
+ if (!srcAddr64) {
+ break;
+ }
- // Clip segment length to remaining
- if (dstLen > remaining)
- dstLen = remaining;
+ // Clip segment length to remaining
+ if (dstLen > remaining) {
+ dstLen = remaining;
+ }
#if 1
- bcopy_phys(srcAddr64, dstAddr, dstLen);
+ bcopy_phys(srcAddr64, dstAddr, dstLen);
#else
- copypv(srcAddr, dstAddr64, dstLen,
- cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
+ copypv(srcAddr, dstAddr64, dstLen,
+ cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
#endif
- dstAddr += dstLen;
- offset += dstLen;
- remaining -= dstLen;
- }
+ dstAddr += dstLen;
+ offset += dstLen;
+ remaining -= dstLen;
+ }
- assert(!remaining);
+ assert(!remaining);
- return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
+ return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
void
hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
- vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
+ vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
+{
+ count += ppnum;
+
+ if (count > UINT_MAX) {
+ panic("hibernate_set_page_state ppnum");
+ }
+
+ switch (kind) {
+ case kIOHibernatePageStateUnwiredSave:
+ // unwired save
+ for (; ppnum < count; ppnum++) {
+ hibernate_page_bitset(page_list, FALSE, (uint32_t) ppnum);
+ hibernate_page_bitset(page_list_wired, TRUE, (uint32_t) ppnum);
+ }
+ break;
+ case kIOHibernatePageStateWiredSave:
+ // wired save
+ for (; ppnum < count; ppnum++) {
+ hibernate_page_bitset(page_list, FALSE, (uint32_t) ppnum);
+ hibernate_page_bitset(page_list_wired, FALSE, (uint32_t) ppnum);
+ }
+ break;
+ case kIOHibernatePageStateFree:
+ // free page
+ for (; ppnum < count; ppnum++) {
+ hibernate_page_bitset(page_list, TRUE, (uint32_t) ppnum);
+ hibernate_page_bitset(page_list_wired, TRUE, (uint32_t) ppnum);
+ }
+ break;
+ default:
+ panic("hibernate_set_page_state");
+ }
+}
+
+static void
+hibernate_set_descriptor_page_state(IOHibernateVars *vars,
+ IOMemoryDescriptor *descriptor,
+ uint32_t kind,
+ uint32_t *pageCount)
{
- count += ppnum;
- switch (kind)
- {
- case kIOHibernatePageStateUnwiredSave:
- // unwired save
- for (; ppnum < count; ppnum++)
- {
- hibernate_page_bitset(page_list, FALSE, ppnum);
- hibernate_page_bitset(page_list_wired, TRUE, ppnum);
- }
- break;
- case kIOHibernatePageStateWiredSave:
- // wired save
- for (; ppnum < count; ppnum++)
- {
- hibernate_page_bitset(page_list, FALSE, ppnum);
- hibernate_page_bitset(page_list_wired, FALSE, ppnum);
- }
- break;
- case kIOHibernatePageStateFree:
- // free page
- for (; ppnum < count; ppnum++)
- {
- hibernate_page_bitset(page_list, TRUE, ppnum);
- hibernate_page_bitset(page_list_wired, TRUE, ppnum);
- }
- break;
- default:
- panic("hibernate_set_page_state");
- }
+ IOItemCount count;
+ addr64_t phys64;
+ IOByteCount segLen;
+ if (descriptor) {
+ for (count = 0;
+ (phys64 = descriptor->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+ count += segLen) {
+ hibernate_set_page_state(vars->page_list, vars->page_list_wired,
+ atop_64(phys64), atop_32(segLen),
+ kind);
+ *pageCount -= atop_32(segLen);
+ }
+ }
}
static vm_offset_t
-hibernate_page_list_iterate(hibernate_page_list_t * list,
- void ** iterator, vm_offset_t * ppnum)
+hibernate_page_list_iterate(hibernate_page_list_t * list, ppnum_t * pPage)
{
- uint32_t count, idx;
-
- idx = (uint32_t) *iterator;
-
- if (!idx)
- idx = hibernate_page_list_count(list, TRUE, idx);
+ uint32_t page = ((typeof(page)) * pPage);
+ uint32_t count;
+ hibernate_bitmap_t * bitmap;
+
+ while ((bitmap = hibernate_page_bitmap_pin(list, &page))) {
+ count = hibernate_page_bitmap_count(bitmap, TRUE, page);
+ if (!count) {
+ break;
+ }
+ page += count;
+ if (page <= bitmap->last_page) {
+ break;
+ }
+ }
- *ppnum = idx;
- count = hibernate_page_list_count(list, FALSE, idx);
- idx += count;
- idx += hibernate_page_list_count(list, TRUE, idx);
- *iterator = (void *) idx;
+ *pPage = page;
+ if (bitmap) {
+ count = hibernate_page_bitmap_count(bitmap, FALSE, page);
+ } else {
+ count = 0;
+ }
- return (count);
+ return count;
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-static IOReturn
-IOHibernatePollerProbe(IOPolledFileIOVars * vars, IOService * target)
+IOReturn
+IOHibernateSystemSleep(void)
{
- IOReturn err = kIOReturnError;
- int32_t idx;
- IOPolledInterface * poller;
-
- for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
- {
- poller = (IOPolledInterface *) vars->pollers->getObject(idx);
- err = poller->probe(target);
- if (err)
- {
- HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
- break;
- }
- }
-
- return (err);
-}
+ IOReturn err;
+ OSData * nvramData;
+ OSObject * obj;
+ OSString * str;
+ OSNumber * num;
+ bool dsSSD, vmflush, swapPinned;
+ IOHibernateVars * vars;
+ uint64_t setFileSize = 0;
+
+ gIOHibernateState = kIOHibernateStateInactive;
+
+ gIOHibernateDebugFlags = 0;
+ if (kIOLogHibernate & gIOKitDebug) {
+ gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
+ }
-static IOReturn
-IOHibernatePollerOpen(IOPolledFileIOVars * vars, uint32_t state, IOMemoryDescriptor * md)
-{
- IOReturn err = kIOReturnError;
- int32_t idx;
- IOPolledInterface * poller;
-
- for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
- {
- poller = (IOPolledInterface *) vars->pollers->getObject(idx);
- err = poller->open(state, md);
- if (err)
- {
- HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
- break;
- }
- }
-
- return (err);
-}
+ if (IOService::getPMRootDomain()->getHibernateSettings(
+ &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime)) {
+ if (kIOHibernateModeSleep & gIOHibernateMode) {
+ // default to discard clean for safe sleep
+ gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
+ | kIOHibernateModeDiscardCleanActive);
+ }
+ }
-static IOReturn
-IOHibernatePollerClose(IOPolledFileIOVars * vars, uint32_t state)
-{
- IOReturn err = kIOReturnError;
- int32_t idx;
- IOPolledInterface * poller;
-
- for (idx = 0;
- (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
- idx++)
- {
- err = poller->close(state);
- if (err)
- HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
- }
-
- return (err);
+ if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey))) {
+ if ((str = OSDynamicCast(OSString, obj))) {
+ strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
+ sizeof(gIOHibernateFilename));
+ }
+ obj->release();
+ }
+
+ if (!gIOHibernateMode || !gIOHibernateFilename[0]) {
+ return kIOReturnUnsupported;
+ }
+
+ HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
+
+ vars = IONew(IOHibernateVars, 1);
+ if (!vars) {
+ return kIOReturnNoMemory;
+ }
+ bzero(vars, sizeof(*vars));
+
+ IOLockLock(gFSLock);
+ if (!gIOHibernateTrimCalloutEntry) {
+ gIOHibernateTrimCalloutEntry = thread_call_allocate(&IOHibernateSystemPostWakeTrim, &gFSLock);
+ }
+ IOHibernateSystemPostWakeTrim(NULL, NULL);
+ thread_call_cancel(gIOHibernateTrimCalloutEntry);
+ if (kFSIdle != gFSState) {
+ HIBLOG("hibernate file busy\n");
+ IOLockUnlock(gFSLock);
+ IODelete(vars, IOHibernateVars, 1);
+ return kIOReturnBusy;
+ }
+ gFSState = kFSOpening;
+ IOLockUnlock(gFSLock);
+
+ swapPinned = false;
+ do{
+ vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
+ HIBERNATION_SRC_BUFFER_SIZE, page_size);
+
+ vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
+ ptoa_64(gIOHibernateHandoffPageCount), page_size);
+
+ if (!vars->srcBuffer || !vars->handoffBuffer) {
+ err = kIOReturnNoMemory;
+ break;
+ }
+
+ if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey))) {
+ if ((num = OSDynamicCast(OSNumber, obj))) {
+ vars->fileMinSize = num->unsigned64BitValue();
+ }
+ obj->release();
+ }
+ if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey))) {
+ if ((num = OSDynamicCast(OSNumber, obj))) {
+ vars->fileMaxSize = num->unsigned64BitValue();
+ }
+ obj->release();
+ }
+
+ boolean_t encryptedswap = true;
+ uint32_t pageCount;
+ AbsoluteTime startTime, endTime;
+ uint64_t nsec;
+
+ bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
+ gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
+ gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
+
+ vmflush = ((kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
+ err = hibernate_alloc_page_lists(&vars->page_list,
+ &vars->page_list_wired,
+ &vars->page_list_pal);
+ if (KERN_SUCCESS != err) {
+ HIBLOG("%s err, hibernate_alloc_page_lists return 0x%x\n", __FUNCTION__, err);
+ break;
+ }
+
+ err = hibernate_pin_swap(TRUE);
+ if (KERN_SUCCESS != err) {
+ HIBLOG("%s error, hibernate_pin_swap return 0x%x\n", __FUNCTION__, err);
+ break;
+ }
+ swapPinned = true;
+
+ if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode)) {
+ hibernate_page_list_setall(vars->page_list,
+ vars->page_list_wired,
+ vars->page_list_pal,
+ true /* preflight */,
+ vmflush /* discard */,
+ &pageCount);
+ PE_Video consoleInfo;
+ bzero(&consoleInfo, sizeof(consoleInfo));
+ IOService::getPlatform()->getConsoleInfo(&consoleInfo);
+
+ // estimate: 6% increase in pages compressed
+ // screen preview 2 images compressed 0%
+ setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8)
+ + vars->page_list->list_size
+ + (consoleInfo.v_width * consoleInfo.v_height * 8);
+ enum { setFileRound = 1024 * 1024ULL };
+ setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1));
+
+ HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n",
+ pageCount, (100ULL * gIOHibernateCompression) >> 8,
+ setFileSize, vars->fileMinSize);
+
+ if (!(kIOHibernateModeFileResize & gIOHibernateMode)
+ && (setFileSize < vars->fileMinSize)) {
+ setFileSize = vars->fileMinSize;
+ }
+ }
+
+ vars->volumeCryptKeySize = sizeof(vars->volumeCryptKey);
+ err = IOPolledFileOpen(gIOHibernateFilename,
+ (kIOPolledFileCreate | kIOPolledFileHibernate),
+ setFileSize, 0,
+ gIOHibernateCurrentHeader, sizeof(gIOHibernateCurrentHeader),
+ &vars->fileVars, &nvramData,
+ &vars->volumeCryptKey[0], &vars->volumeCryptKeySize);
+
+ if (KERN_SUCCESS != err) {
+ IOLockLock(gFSLock);
+ if (kFSOpening != gFSState) {
+ err = kIOReturnTimeout;
+ }
+ IOLockUnlock(gFSLock);
+ }
+
+ if (KERN_SUCCESS != err) {
+ HIBLOG("IOPolledFileOpen(%x)\n", err);
+ break;
+ }
+
+ // write extents for debug data usage in EFI
+ IOWriteExtentsToFile(vars->fileVars, kIOHibernateHeaderOpenSignature);
+
+ err = IOPolledFilePollersSetup(vars->fileVars, kIOPolledPreflightState);
+ if (KERN_SUCCESS != err) {
+ break;
+ }
+
+ clock_get_uptime(&startTime);
+ err = hibernate_setup(gIOHibernateCurrentHeader,
+ vmflush,
+ vars->page_list, vars->page_list_wired, vars->page_list_pal);
+ clock_get_uptime(&endTime);
+ SUB_ABSOLUTETIME(&endTime, &startTime);
+ absolutetime_to_nanoseconds(endTime, &nsec);
+
+ boolean_t haveSwapPin, hibFileSSD;
+ haveSwapPin = vm_swap_files_pinned();
+
+ hibFileSSD = (kIOPolledFileSSD & vars->fileVars->flags);
+
+ HIBLOG("hibernate_setup(%d) took %qd ms, swapPin(%d) ssd(%d)\n",
+ err, nsec / 1000000ULL,
+ haveSwapPin, hibFileSSD);
+ if (KERN_SUCCESS != err) {
+ break;
+ }
+
+ gIOHibernateStandbyDisabled = ((!haveSwapPin || !hibFileSSD));
+
+ dsSSD = ((0 != (kIOPolledFileSSD & vars->fileVars->flags))
+ && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
+
+ if (dsSSD) {
+ gIOHibernateCurrentHeader->options |= kIOHibernateOptionSSD | kIOHibernateOptionColor;
+ } else {
+ gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
+ }
+
+
+#if defined(__i386__) || defined(__x86_64__)
+ if (vars->volumeCryptKeySize &&
+ (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey))) {
+ uintptr_t smcVars[2];
+ smcVars[0] = vars->volumeCryptKeySize;
+ smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
+
+ IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
+ bzero(smcVars, sizeof(smcVars));
+ }
+#endif
+
+
+ if (encryptedswap || vars->volumeCryptKeySize) {
+ gIOHibernateMode ^= kIOHibernateModeEncrypt;
+ }
+
+ if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options) {
+ vars->videoAllocSize = kVideoMapSize;
+ if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize, VM_KERN_MEMORY_IOKIT)) {
+ vars->videoMapping = 0;
+ }
+ }
+
+ // generate crypt keys
+ for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++) {
+ vars->wiredCryptKey[i] = ((uint8_t) random());
+ }
+ for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++) {
+ vars->cryptKey[i] = ((uint8_t) random());
+ }
+
+ // set nvram
+
+ IOSetBootImageNVRAM(nvramData);
+ nvramData->release();
+
+#if defined(__i386__) || defined(__x86_64__)
+ {
+ struct AppleRTCHibernateVars {
+ uint8_t signature[4];
+ uint32_t revision;
+ uint8_t booterSignature[20];
+ uint8_t wiredCryptKey[16];
+ };
+ AppleRTCHibernateVars rtcVars;
+ OSData * data;
+
+ rtcVars.signature[0] = 'A';
+ rtcVars.signature[1] = 'A';
+ rtcVars.signature[2] = 'P';
+ rtcVars.signature[3] = 'L';
+ rtcVars.revision = 1;
+ bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
+
+ if (gIOChosenEntry
+ && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOHibernateBootSignatureKey)))
+ && (sizeof(rtcVars.booterSignature) <= data->getLength())) {
+ bcopy(data->getBytesNoCopy(), &rtcVars.booterSignature[0], sizeof(rtcVars.booterSignature));
+ } else if (gIOHibernateBootSignature[0]) {
+ char c;
+ uint8_t value = 0;
+ uint32_t in, out, digits;
+ for (in = out = digits = 0;
+ (c = gIOHibernateBootSignature[in]) && (in < sizeof(gIOHibernateBootSignature));
+ in++) {
+ if ((c >= 'a') && (c <= 'f')) {
+ c -= 'a' - 10;
+ } else if ((c >= 'A') && (c <= 'F')) {
+ c -= 'A' - 10;
+ } else if ((c >= '0') && (c <= '9')) {
+ c -= '0';
+ } else {
+ if (c == '=') {
+ out = digits = value = 0;
+ }
+ continue;
+ }
+ value = ((uint8_t) ((value << 4) | c));
+ if (digits & 1) {
+ rtcVars.booterSignature[out++] = value;
+ if (out >= sizeof(rtcVars.booterSignature)) {
+ break;
+ }
+ }
+ digits++;
+ }
+ }
+#if DEBUG || DEVELOPMENT
+ if (kIOLogHibernate & gIOKitDebug) {
+ IOKitKernelLogBuffer("H> rtc:",
+ &rtcVars, sizeof(rtcVars), &kprintf);
+ }
+#endif /* DEBUG || DEVELOPMENT */
+
+ data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
+ if (data) {
+ if (gIOHibernateRTCVariablesKey) {
+ IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
+ }
+ data->release();
+ }
+ if (gIOChosenEntry && gIOOptionsEntry) {
+ data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
+ if (data) {
+ gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
+ }
+ // set BootNext
+ if (!gIOHibernateBoot0082Data) {
+ OSData * fileData = NULL;
+ data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
+ if (data && data->getLength() >= 4) {
+ fileData = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-file-path"));
+ }
+ if (data && (data->getLength() <= UINT16_MAX)) {
+ // AppleNVRAM_EFI_LOAD_OPTION
+ struct {
+ uint32_t Attributes;
+ uint16_t FilePathLength;
+ uint16_t Desc;
+ } loadOptionHeader;
+ loadOptionHeader.Attributes = 1;
+ loadOptionHeader.FilePathLength = ((uint16_t) data->getLength());
+ loadOptionHeader.Desc = 0;
+ if (fileData) {
+ loadOptionHeader.FilePathLength -= 4;
+ loadOptionHeader.FilePathLength += fileData->getLength();
+ }
+ gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
+ if (gIOHibernateBoot0082Data) {
+ gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
+ if (fileData) {
+ gIOHibernateBoot0082Data->appendBytes(data->getBytesNoCopy(), data->getLength() - 4);
+ gIOHibernateBoot0082Data->appendBytes(fileData);
+ } else {
+ gIOHibernateBoot0082Data->appendBytes(data);
+ }
+ }
+ }
+ }
+ if (!gIOHibernateBootNextData) {
+ uint16_t bits = 0x0082;
+ gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
+ }
+
+#if DEBUG || DEVELOPMENT
+ if (kIOLogHibernate & gIOKitDebug) {
+ IOKitKernelLogBuffer("H> bootnext:",
+ gIOHibernateBoot0082Data->getBytesNoCopy(), gIOHibernateBoot0082Data->getLength(), &kprintf);
+ }
+#endif /* DEBUG || DEVELOPMENT */
+ if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData) {
+ gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
+ gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
+ gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
+ }
+ // BootNext
+ }
+ }
+#endif /* !i386 && !x86_64 */
+ }while (false);
+
+ if (swapPinned) {
+ hibernate_pin_swap(FALSE);
+ }
+
+ IOLockLock(gFSLock);
+ if ((kIOReturnSuccess == err) && (kFSOpening != gFSState)) {
+ HIBLOG("hibernate file close due timeout\n");
+ err = kIOReturnTimeout;
+ }
+ if (kIOReturnSuccess == err) {
+ gFSState = kFSOpened;
+ gIOHibernateVars = *vars;
+ gFileVars = *vars->fileVars;
+ gFileVars.allocated = false;
+ gIOHibernateVars.fileVars = &gFileVars;
+ gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
+ gIOHibernateCurrentHeader->kernVirtSlide = vm_kernel_slide;
+ gIOHibernateState = kIOHibernateStateHibernating;
+
+#if DEBUG || DEVELOPMENT
+ if (kIOLogHibernate & gIOKitDebug) {
+ OSData * data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
+ if (data) {
+ uintptr_t * smcVars = (typeof(smcVars))data->getBytesNoCopy();
+ IOKitKernelLogBuffer("H> smc:",
+ (const void *)smcVars[1], smcVars[0], &kprintf);
+ }
+ }
+#endif /* DEBUG || DEVELOPMENT */
+ } else {
+ IOPolledFileIOVars * fileVars = vars->fileVars;
+ IOHibernateDone(vars);
+ IOPolledFileClose(&fileVars,
+#if DISABLE_TRIM
+ 0, NULL, 0, 0, 0);
+#else
+ 0, NULL, 0, sizeof(IOHibernateImageHeader), setFileSize);
+#endif
+ gFSState = kFSIdle;
+ }
+ IOLockUnlock(gFSLock);
+
+ if (vars->fileVars) {
+ IODelete(vars->fileVars, IOPolledFileIOVars, 1);
+ }
+ IODelete(vars, IOHibernateVars, 1);
+
+ return err;
}
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
static void
-IOHibernatePollerIOComplete(void * target,
- void * parameter,
- IOReturn status,
- UInt64 actualByteCount)
+IOSetBootImageNVRAM(OSData * data)
{
- IOPolledFileIOVars * vars = (IOPolledFileIOVars *) parameter;
+ IORegistryEntry * regEntry;
- vars->ioStatus = status;
+ if (!gIOOptionsEntry) {
+ regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
+ gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
+ if (regEntry && !gIOOptionsEntry) {
+ regEntry->release();
+ }
+ }
+ if (gIOOptionsEntry && gIOHibernateBootImageKey) {
+ if (data) {
+ gIOOptionsEntry->setProperty(gIOHibernateBootImageKey, data);
+#if DEBUG || DEVELOPMENT
+ if (kIOLogHibernate & gIOKitDebug) {
+ IOKitKernelLogBuffer("H> boot-image:",
+ data->getBytesNoCopy(), data->getLength(), &kprintf);
+ }
+#endif /* DEBUG || DEVELOPMENT */
+ } else {
+ gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
+#if __x86_64__
+ gIOOptionsEntry->sync();
+#else
+ if (gIOHibernateState == kIOHibernateStateWakingFromHibernate) {
+ // if we woke from hibernation, the booter may have changed the state of NVRAM, so force a sync
+ gIOOptionsEntry->sync();
+ }
+#endif
+ }
+ }
}
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+/*
+ * Writes header to disk with signature, block size and file extents data.
+ * If there are more than 2 extents, then they are written on second block.
+ */
static IOReturn
-IOHibernatePollerIO(IOPolledFileIOVars * vars,
- uint32_t operation, uint32_t bufferOffset,
- uint64_t deviceOffset, uint64_t length)
+IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature)
{
+ IOHibernateImageHeader hdr;
+ IOItemCount count;
+ IOReturn err = kIOReturnSuccess;
+ int rc;
+ IOPolledFileExtent * fileExtents;
+
+ fileExtents = (typeof(fileExtents))vars->fileExtents->getBytesNoCopy();
+
+ memset(&hdr, 0, sizeof(IOHibernateImageHeader));
+ count = vars->fileExtents->getLength();
+ if (count > sizeof(hdr.fileExtentMap)) {
+ hdr.fileExtentMapSize = count;
+ count = sizeof(hdr.fileExtentMap);
+ } else {
+ hdr.fileExtentMapSize = sizeof(hdr.fileExtentMap);
+ }
- IOReturn err = kIOReturnError;
- IOPolledInterface * poller;
- IOPolledCompletion completion;
+ bcopy(fileExtents, &hdr.fileExtentMap[0], count);
+
+ // copy file block extent list if larger than header
+ if (hdr.fileExtentMapSize > sizeof(hdr.fileExtentMap)) {
+ count = hdr.fileExtentMapSize - sizeof(hdr.fileExtentMap);
+ rc = kern_write_file(vars->fileRef, vars->blockSize,
+ (caddr_t)(((uint8_t *)fileExtents) + sizeof(hdr.fileExtentMap)),
+ count, IO_SKIP_ENCRYPTION);
+ if (rc != 0) {
+ HIBLOG("kern_write_file returned %d\n", rc);
+ err = kIOReturnIOError;
+ goto exit;
+ }
+ }
+ hdr.signature = signature;
+ hdr.deviceBlockSize = vars->blockSize;
+
+ rc = kern_write_file(vars->fileRef, 0, (char *)&hdr, sizeof(hdr), IO_SKIP_ENCRYPTION);
+ if (rc != 0) {
+ HIBLOG("kern_write_file returned %d\n", rc);
+ err = kIOReturnIOError;
+ goto exit;
+ }
- completion.target = 0;
- completion.action = &IOHibernatePollerIOComplete;
- completion.parameter = vars;
+exit:
+ return err;
+}
- vars->ioStatus = -1;
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+DECLARE_IOHIBERNATEPROGRESSALPHA
- poller = (IOPolledInterface *) vars->pollers->getObject(0);
- err = poller->startIO(operation, bufferOffset, deviceOffset + vars->block0, length, completion);
- if (err)
- HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
+static void
+ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
+{
+ uint32_t rowBytes, pixelShift;
+ uint32_t x, y;
+ int32_t blob;
+ uint32_t alpha, color, result;
+ uint8_t * out, in;
+ uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
+
+ rowBytes = display->rowBytes;
+ pixelShift = display->depth >> 4;
+ if (pixelShift < 1) {
+ return;
+ }
- return (err);
+ screen += ((display->width
+ - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
+ + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
+
+ for (y = 0; y < kIOHibernateProgressHeight; y++) {
+ out = screen + y * rowBytes;
+ for (blob = 0; blob < kIOHibernateProgressCount; blob++) {
+ color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
+ for (x = 0; x < kIOHibernateProgressWidth; x++) {
+ alpha = gIOHibernateProgressAlpha[y][x];
+ result = color;
+ if (alpha) {
+ if (0xff != alpha) {
+ if (1 == pixelShift) {
+ in = *((uint16_t *)out) & 0x1f; // 16
+ in = ((uint8_t)(in << 3)) | ((uint8_t)(in >> 2));
+ } else {
+ in = *((uint32_t *)out) & 0xff; // 32
+ }
+ saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
+ result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
+ }
+ if (1 == pixelShift) {
+ result >>= 3;
+ *((uint16_t *)out) = ((uint16_t)((result << 10) | (result << 5) | result)); // 16
+ } else {
+ *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
+ }
+ }
+ out += (1 << pixelShift);
+ }
+ out += (kIOHibernateProgressSpacing << pixelShift);
+ }
+ }
}
-static IOReturn
-IOHibernatePollerIODone(IOPolledFileIOVars * vars)
+
+static void
+ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
{
- IOReturn err = kIOReturnError;
- int32_t idx;
- IOPolledInterface * poller;
-
- while (-1 == vars->ioStatus)
- {
- for (idx = 0;
- (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
- idx++)
- {
- err = poller->checkForWork();
- if (err)
- HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
- }
- }
-
- if (kIOReturnSuccess != vars->ioStatus)
- HIBLOG("IOPolledInterface::ioStatus 0x%x\n", vars->ioStatus);
-
- return (vars->ioStatus);
+ uint32_t rowBytes, pixelShift;
+ uint32_t x, y;
+ int32_t blob, lastBlob;
+ uint32_t alpha, in, color, result;
+ uint8_t * out;
+ uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
+
+ pixelShift = display->depth >> 4;
+ if (pixelShift < 1) {
+ return;
+ }
+
+ rowBytes = display->rowBytes;
+
+ screen += ((display->width
+ - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
+ + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
+
+ lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
+
+ screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
+
+ for (y = 0; y < kIOHibernateProgressHeight; y++) {
+ out = screen + y * rowBytes;
+ for (blob = firstBlob; blob <= lastBlob; blob++) {
+ color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
+ for (x = 0; x < kIOHibernateProgressWidth; x++) {
+ alpha = gIOHibernateProgressAlpha[y][x];
+ result = color;
+ if (alpha) {
+ if (0xff != alpha) {
+ in = display->progressSaveUnder[blob][saveindex[blob]++];
+ result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
+ }
+ if (1 == pixelShift) {
+ result >>= 3;
+ *((uint16_t *)out) = ((uint16_t)((result << 10) | (result << 5) | result)); // 16
+ } else {
+ *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
+ }
+ }
+ out += (1 << pixelShift);
+ }
+ out += (kIOHibernateProgressSpacing << pixelShift);
+ }
+ }
}
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+IOReturn
+IOHibernateIOKitSleep(void)
+{
+ IOReturn ret = kIOReturnSuccess;
+ IOLockLock(gFSLock);
+ if (kFSOpening == gFSState) {
+ gFSState = kFSTimedOut;
+ HIBLOG("hibernate file open timed out\n");
+ ret = kIOReturnTimeout;
+ }
+ IOLockUnlock(gFSLock);
+ return ret;
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
IOReturn
-IOPolledInterface::checkAllForWork(void)
+IOHibernateSystemHasSlept(void)
{
- IOReturn err = kIOReturnNotReady;
- int32_t idx;
- IOPolledInterface * poller;
+ IOReturn ret = kIOReturnSuccess;
+ IOHibernateVars * vars = &gIOHibernateVars;
+ OSObject * obj = NULL;
+ OSData * data;
+
+ IOLockLock(gFSLock);
+ if ((kFSOpened != gFSState) && gIOHibernateMode) {
+ ret = kIOReturnTimeout;
+ }
+ IOLockUnlock(gFSLock);
+ if (kIOReturnSuccess != ret) {
+ return ret;
+ }
+
+ if (gIOHibernateMode) {
+ obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
+ }
+ vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
+ if (obj && !vars->previewBuffer) {
+ obj->release();
+ }
+ if (vars->previewBuffer && (vars->previewBuffer->getLength() > UINT_MAX)) {
+ OSSafeReleaseNULL(vars->previewBuffer);
+ }
- IOHibernateVars * vars = &gIOHibernateVars;
+ vars->consoleMapping = NULL;
+ if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare())) {
+ vars->previewBuffer->release();
+ vars->previewBuffer = NULL;
+ }
- if (!vars->fileVars || !vars->fileVars->pollers)
- return (err);
+ if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
+ && vars->previewBuffer
+ && (data = OSDynamicCast(OSData,
+ IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey)))) {
+ UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
+ HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
+
+ IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
+
+ if (kIOHibernatePreviewUpdates & flags) {
+ PE_Video consoleInfo;
+ hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
+
+ IOService::getPlatform()->getConsoleInfo(&consoleInfo);
+
+ graphicsInfo->width = (uint32_t) consoleInfo.v_width;
+ graphicsInfo->height = (uint32_t) consoleInfo.v_height;
+ graphicsInfo->rowBytes = (uint32_t) consoleInfo.v_rowBytes;
+ graphicsInfo->depth = (uint32_t) consoleInfo.v_depth;
+ vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
+
+ HIBPRINT("video %p %d %d %d\n",
+ vars->consoleMapping, graphicsInfo->depth,
+ graphicsInfo->width, graphicsInfo->height);
+ if (vars->consoleMapping) {
+ ProgressInit(graphicsInfo, vars->consoleMapping,
+ &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
+ }
+ }
+ }
- for (idx = 0;
- (poller = (IOPolledInterface *) vars->fileVars->pollers->getObject(idx));
- idx++)
- {
- err = poller->checkForWork();
- if (err)
- HIBLOG("IOPolledInterface::checkAllForWork[%d] 0x%x\n", idx, err);
- }
+ if (gIOOptionsEntry) {
+#if __x86_64__
+ gIOOptionsEntry->sync();
+#else
+ if (gIOHibernateMode) {
+ gIOOptionsEntry->sync();
+ }
+#endif
+ }
- return (err);
+ return ret;
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-struct _OpenFileContext
+static const DeviceTreeNode *
+MergeDeviceTree(const DeviceTreeNode * entry, IORegistryEntry * regEntry, OSSet * entriesToUpdate, vm_offset_t region_start, vm_size_t region_size)
{
- OSData * extents;
- uint64_t size;
-};
+ DeviceTreeNodeProperty * prop;
+ const DeviceTreeNode * child;
+ IORegistryEntry * childRegEntry;
+ const char * nameProp;
+ unsigned int propLen, idx;
+
+ bool updateEntry = true;
+ if (!regEntry) {
+ updateEntry = false;
+ } else if (entriesToUpdate && !entriesToUpdate->containsObject(regEntry)) {
+ updateEntry = false;
+ }
-static void
-file_extent_callback(void * ref, uint64_t start, uint64_t length)
-{
- _OpenFileContext * ctx = (_OpenFileContext *) ref;
- IOPolledFileExtent extent;
+ prop = (DeviceTreeNodeProperty *) (entry + 1);
+ for (idx = 0; idx < entry->nProperties; idx++) {
+ if (updateEntry && (0 != strcmp("name", prop->name))) {
+ regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
+// HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
+ }
+ prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
+ }
- extent.start = start;
- extent.length = length;
+ if (entriesToUpdate) {
+ entriesToUpdate->removeObject(regEntry);
+ if (entriesToUpdate->getCount() == 0) {
+ // we've updated all the entries we care about so we can stop
+ return NULL;
+ }
+ }
- ctx->extents->appendBytes(&extent, sizeof(extent));
- ctx->size += length;
+ child = (const DeviceTreeNode *) prop;
+ for (idx = 0; idx < entry->nChildren; idx++) {
+ if (kSuccess != SecureDTGetPropertyRegion(child, "name", (void const **) &nameProp, &propLen,
+ region_start, region_size)) {
+ panic("no name");
+ }
+ childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
+// HIBPRINT("%s == %p\n", nameProp, childRegEntry);
+ child = MergeDeviceTree(child, childRegEntry, entriesToUpdate, region_start, region_size);
+ if (!child) {
+ // the recursive call updated the last entry we cared about, so we can stop
+ break;
+ }
+ }
+ return child;
}
IOReturn
-IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer,
- IOPolledFileIOVars ** fileVars, OSData ** fileExtents,
- OSData ** imagePath)
+IOHibernateSystemWake(void)
{
- IOReturn err = kIOReturnError;
- IOPolledFileIOVars * vars;
- _OpenFileContext ctx;
- OSData * extentsData;
- OSNumber * num;
- IORegistryEntry * part = 0;
- OSDictionary * matching;
- OSIterator * iter;
- dev_t hibernate_image_dev;
- uint64_t maxiobytes;
-
- vars = &gFileVars;
- do
- {
- HIBLOG("sizeof(IOHibernateImageHeader) == %ld\n", sizeof(IOHibernateImageHeader));
- if (sizeof(IOHibernateImageHeader) != 512)
- continue;
-
- vars->io = false;
- vars->buffer = (uint8_t *) ioBuffer->getBytesNoCopy();
- vars->bufferHalf = 0;
- vars->bufferOffset = 0;
- vars->bufferSize = ioBuffer->getLength() >> 1;
-
- extentsData = OSData::withCapacity(32);
-
- ctx.extents = extentsData;
- ctx.size = 0;
- vars->fileRef = kern_open_file_for_direct_io(filename,
- &file_extent_callback, &ctx,
- &hibernate_image_dev,
- &vars->block0,
- &maxiobytes);
- if (!vars->fileRef)
- {
- err = kIOReturnNoSpace;
- break;
- }
- HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx\n", filename, ctx.size,
- vars->block0, maxiobytes);
- if (ctx.size < 1*1024*1024) // check against image size estimate!
- {
- err = kIOReturnNoSpace;
- break;
- }
-
- if (maxiobytes < vars->bufferSize)
- vars->bufferSize = maxiobytes;
-
- vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
-
- matching = IOService::serviceMatching("IOMedia");
- num = OSNumber::withNumber(major(hibernate_image_dev), 32);
- matching->setObject(kIOBSDMajorKey, num);
- num->release();
- num = OSNumber::withNumber(minor(hibernate_image_dev), 32);
- matching->setObject(kIOBSDMinorKey, num);
- num->release();
- iter = IOService::getMatchingServices(matching);
- matching->release();
- if (iter)
- {
- part = (IORegistryEntry *) iter->getNextObject();
- part->retain();
- iter->release();
- }
-
- int minor, major;
- IORegistryEntry * next;
- IORegistryEntry * child;
- OSData * data;
+ if (kFSOpened == gFSState) {
+ IOPolledFilePollersClose(gIOHibernateVars.fileVars, kIOPolledPostflightState);
+ IOHibernateDone(&gIOHibernateVars);
+ } else {
+ IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
+ IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
+ }
+
+ if (gIOOptionsEntry && gIOHibernateBootImageKey) {
+ // if we got this far, clear boot-image
+ // we don't need to sync immediately; the booter should have already removed this entry
+ // we just want to make sure that if anyone syncs nvram after this point, we don't re-write
+ // a stale boot-image value
+ gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
+ }
- num = (OSNumber *) part->getProperty(kIOBSDMajorKey);
- if (!num)
- break;
- major = num->unsigned32BitValue();
- num = (OSNumber *) part->getProperty(kIOBSDMinorKey);
- if (!num)
- break;
- minor = num->unsigned32BitValue();
-
- hibernate_image_dev = makedev(major, minor);
-
- vars->pollers = OSArray::withCapacity(4);
- if (!vars->pollers)
- break;
-
- vars->blockSize = 512;
- next = part;
- do
- {
- IOPolledInterface * poller;
- if ((poller = OSDynamicCast(IOPolledInterface, next->getProperty(kIOPolledInterfaceSupportKey))))
- vars->pollers->setObject(poller);
- if ((num = OSDynamicCast(OSNumber, next->getProperty(kIOMediaPreferredBlockSizeKey))))
- vars->blockSize = num->unsigned32BitValue();
- child = next;
- }
- while ((next = child->getParentEntry(gIOServicePlane))
- && child->isParent(next, gIOServicePlane, true));
-
- HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n",
- major, minor, vars->blockSize, vars->pollers->getCount());
- if (vars->pollers->getCount() < kIOHibernateMinPollersNeeded)
- continue;
-
- err = IOHibernatePollerProbe(vars, (IOService *) part);
- if (kIOReturnSuccess != err)
- break;
-
- err = IOHibernatePollerOpen(vars, kIOPolledPreflightState, ioBuffer);
- if (kIOReturnSuccess != err)
- break;
-
- *fileVars = vars;
- *fileExtents = extentsData;
-
- // make imagePath
- char str1[256];
- char str2[24];
- int len = sizeof(str1);
-
- if ((extentsData->getLength() >= sizeof(IOPolledFileExtent))
- && part->getPath(str1, &len, gIODTPlane))
- {
- // (strip the plane name)
- char * tail = strchr(str1, ':');
- if (!tail)
- tail = str1 - 1;
- data = OSData::withBytes(tail + 1, strlen(tail + 1));
- sprintf(str2, ",%qx", vars->extentMap[0]);
- data->appendBytes(str2, strlen(str2));
- *imagePath = data;
- }
- }
- while (false);
-
- if (kIOReturnSuccess != err)
- {
- HIBLOG("error 0x%x opening hibernation file\n", err);
- if (vars->fileRef)
- kern_close_file_for_direct_io(vars->fileRef);
- }
-
- if (part)
- part->release();
-
- return (err);
+ return kIOReturnSuccess;
}
-IOReturn
-IOPolledFileClose( IOPolledFileIOVars * vars )
+static IOReturn
+IOHibernateDone(IOHibernateVars * vars)
{
- if (vars->pollers)
- {
- IOHibernatePollerClose(vars, kIOPolledPostflightState);
- vars->pollers->release();
- }
+ IOReturn err;
+ OSData * data;
- gIOHibernateFileRef = vars->fileRef;
+ hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);
- bzero(vars, sizeof(IOPolledFileIOVars));
+ if (vars->videoMapping) {
+ if (vars->videoMapSize) {
+ // remove mappings
+ IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
+ }
+ if (vars->videoAllocSize) {
+ // dealloc range
+ kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
+ }
+ }
- return (kIOReturnSuccess);
-}
+ if (vars->previewBuffer) {
+ vars->previewBuffer->release();
+ vars->previewBuffer = NULL;
+ }
-static IOReturn
-IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
-{
- IOPolledFileExtent * extentMap;
+ if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
+ IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey,
+ gIOHibernateCurrentHeader->options, 32);
+ } else {
+ IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
+ }
- extentMap = vars->extentMap;
+ if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
+ && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus)) {
+ IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey,
+ &gIOHibernateGraphicsInfo->gfxStatus,
+ sizeof(gIOHibernateGraphicsInfo->gfxStatus));
+ } else {
+ IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
+ }
- vars->position = position;
+ // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
+
+#if defined(__i386__) || defined(__x86_64__)
+ IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
+ IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
+
+ /*
+ * Hibernate variable is written to NVRAM on platforms in which RtcRam
+ * is not backed by coin cell. Remove Hibernate data from NVRAM.
+ */
+ if (gIOOptionsEntry) {
+ if (gIOHibernateRTCVariablesKey) {
+ if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
+ gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
+ }
+ }
- while (position >= extentMap->length)
- {
- position -= extentMap->length;
- extentMap++;
- }
+ if (gIOHibernateBootNextKey) {
+ if (gIOHibernateBootNextSave) {
+ gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
+ gIOHibernateBootNextSave->release();
+ gIOHibernateBootNextSave = NULL;
+ } else {
+ gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
+ }
+ }
+ if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) {
+ gIOOptionsEntry->sync();
+ }
+ }
+#endif
- vars->currentExtent = extentMap;
- vars->extentRemaining = extentMap->length - position;
- vars->extentPosition = vars->position - position;
+ if (vars->srcBuffer) {
+ vars->srcBuffer->release();
+ }
- if (vars->bufferSize <= vars->extentRemaining)
- vars->bufferLimit = vars->bufferSize;
- else
- vars->bufferLimit = vars->extentRemaining;
- return (kIOReturnSuccess);
-}
+ bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
+ if (vars->handoffBuffer) {
+ if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
+ IOHibernateHandoff * handoff;
+ bool done = false;
+ for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
+ !done;
+ handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount]) {
+ HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
+ uint8_t * __unused data = &handoff->data[0];
+ switch (handoff->type) {
+ case kIOHibernateHandoffTypeEnd:
+ done = true;
+ break;
+
+ case kIOHibernateHandoffTypeDeviceTree:
+ {
+#if defined(__i386__) || defined(__x86_64__)
+ // On Intel, process the entirety of the passed in device tree
+ OSSet * entriesToUpdate = NULL;
+#elif defined(__arm64__)
+ // On ARM, only allow hibernation to update specific entries
+ const char *mergePaths[] = {
+ kIODeviceTreePlane ":/chosen/boot-object-manifests",
+ kIODeviceTreePlane ":/chosen/secure-boot-hashes",
+ };
+ const size_t mergePathCount = sizeof(mergePaths) / sizeof(mergePaths[0]);
+ OSSet * entriesToUpdate = OSSet::withCapacity(mergePathCount);
+ for (size_t i = 0; i < mergePathCount; i++) {
+ IORegistryEntry *entry = IORegistryEntry::fromPath(mergePaths[i]);
+ if (!entry) {
+ panic("failed to find %s in IORegistry", mergePaths[i]);
+ }
+ entriesToUpdate->setObject(entry);
+ OSSafeReleaseNULL(entry);
+ }
+#endif
+ MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot(), entriesToUpdate,
+ (vm_offset_t)data, (vm_size_t)handoff->bytecount);
+ OSSafeReleaseNULL(entriesToUpdate);
+ break;
+ }
+
+ case kIOHibernateHandoffTypeKeyStore:
+#if defined(__i386__) || defined(__x86_64__)
+ {
+ IOBufferMemoryDescriptor *
+ md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
+ if (md) {
+ IOSetKeyStoreData(md);
+ }
+ }
+#endif
+ break;
+
+ default:
+ done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
+ break;
+ }
+ }
+#if defined(__i386__) || defined(__x86_64__)
+ if (vars->volumeCryptKeySize) {
+ IOBufferMemoryDescriptor *
+ bmd = IOBufferMemoryDescriptor::withBytes(&vars->volumeCryptKey[0],
+ vars->volumeCryptKeySize, kIODirectionOutIn);
+ if (!bmd) {
+ panic("IOBufferMemoryDescriptor");
+ }
+ IOSetAPFSKeyStoreData(bmd);
+ bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
+ }
+#endif
+ }
+ vars->handoffBuffer->release();
+ }
-static IOReturn
-IOPolledFileWrite(IOPolledFileIOVars * vars,
- const uint8_t * bytes, IOByteCount size,
- hibernate_cryptvars_t * cryptvars)
-{
- IOReturn err = kIOReturnSuccess;
- IOByteCount copy;
- bool flush = false;
-
- do
- {
- if (!bytes && !size)
- {
- // seek to end of block & flush
- size = vars->position & (vars->blockSize - 1);
- if (size)
- size = vars->blockSize - size;
- flush = true;
- // use some garbage for the fill
- bytes = vars->buffer + vars->bufferOffset;
- }
-
- copy = vars->bufferLimit - vars->bufferOffset;
- if (copy > size)
- copy = size;
- else
- flush = true;
-
- if (bytes)
- {
- bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
- bytes += copy;
- }
- else
- bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
-
- size -= copy;
- vars->bufferOffset += copy;
- vars->position += copy;
-
- if (flush && vars->bufferOffset)
- {
- uint64_t offset = (vars->position - vars->bufferOffset
- - vars->extentPosition + vars->currentExtent->start);
- uint32_t length = (vars->bufferOffset);
-
- if (cryptvars && vars->encryptStart && (vars->position > vars->encryptStart))
- {
- uint32_t encryptLen, encryptStart;
- encryptLen = vars->position - vars->encryptStart;
- if (encryptLen > length)
- encryptLen = length;
- encryptStart = length - encryptLen;
-
- // encrypt the buffer
- aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
- &cryptvars->aes_iv[0],
- encryptLen / AES_BLOCK_SIZE,
- vars->buffer + vars->bufferHalf + encryptStart,
- &cryptvars->ctx.encrypt);
- // save initial vector for following encrypts
- bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
- &cryptvars->aes_iv[0],
- AES_BLOCK_SIZE);
- }
-
- if (vars->io)
- {
- err = IOHibernatePollerIODone(vars);
- if (kIOReturnSuccess != err)
- break;
- }
-
-if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
-//if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
-
- err = IOHibernatePollerIO(vars, kIOPolledWrite, vars->bufferHalf, offset, length);
- if (kIOReturnSuccess != err)
- break;
- vars->io = true;
-
- vars->extentRemaining -= vars->bufferOffset;
- if (!vars->extentRemaining)
- {
- vars->currentExtent++;
- vars->extentRemaining = vars->currentExtent->length;
- vars->extentPosition = vars->position;
- if (!vars->extentRemaining)
- {
- err = kIOReturnOverrun;
- break;
- }
- }
-
- vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
- vars->bufferOffset = 0;
- if (vars->bufferSize <= vars->extentRemaining)
- vars->bufferLimit = vars->bufferSize;
- else
- vars->bufferLimit = vars->extentRemaining;
-
- flush = false;
- }
- }
- while (size);
-
- return (err);
+ if (gIOChosenEntry
+ && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOBridgeBootSessionUUIDKey)))
+ && (sizeof(gIOHibernateBridgeBootSessionUUIDString) <= data->getLength())) {
+ bcopy(data->getBytesNoCopy(), &gIOHibernateBridgeBootSessionUUIDString[0],
+ sizeof(gIOHibernateBridgeBootSessionUUIDString));
+ }
+
+ if (vars->hwEncrypt) {
+ err = IOPolledFilePollersSetEncryptionKey(vars->fileVars, NULL, 0);
+ HIBLOG("IOPolledFilePollersSetEncryptionKey(0,%x)\n", err);
+ }
+
+ bzero(vars, sizeof(*vars));
+
+// gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
+ gIOHibernateCount++;
+
+ return kIOReturnSuccess;
}
-static IOReturn
-IOPolledFileRead(IOPolledFileIOVars * vars,
- uint8_t * bytes, IOByteCount size,
- hibernate_cryptvars_t * cryptvars)
+static void
+IOHibernateSystemPostWakeTrim(void * p1, void * p2)
{
- IOReturn err = kIOReturnSuccess;
- IOByteCount copy;
-
-// bytesWritten += size;
-
- do
- {
- copy = vars->bufferLimit - vars->bufferOffset;
- if (copy > size)
- copy = size;
-
- if (bytes)
- {
- bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
- bytes += copy;
- }
- size -= copy;
- vars->bufferOffset += copy;
-// vars->position += copy;
-
- if (vars->bufferOffset == vars->bufferLimit)
- {
- if (vars->io)
- {
- err = IOHibernatePollerIODone(vars);
- if (kIOReturnSuccess != err)
- break;
- }
- else
- cryptvars = 0;
-
-if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
-
- vars->position += vars->lastRead;
- vars->extentRemaining -= vars->lastRead;
- vars->bufferLimit = vars->lastRead;
-
- if (!vars->extentRemaining)
- {
- vars->currentExtent++;
- vars->extentRemaining = vars->currentExtent->length;
- vars->extentPosition = vars->position;
- if (!vars->extentRemaining)
- {
- err = kIOReturnOverrun;
- break;
- }
- }
-
- if (vars->extentRemaining <= vars->bufferSize)
- vars->lastRead = vars->extentRemaining;
- else
- vars->lastRead = vars->bufferSize;
-
- uint64_t offset = (vars->position
- - vars->extentPosition + vars->currentExtent->start);
- uint64_t length = (vars->lastRead);
-
-//if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
-
- err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length);
- if (kIOReturnSuccess != err)
- break;
- vars->io = true;
-
- vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
- vars->bufferOffset = 0;
-
- if (cryptvars)
- {
- uint8_t thisVector[AES_BLOCK_SIZE];
- // save initial vector for following decrypts
- bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
- bcopy(vars->buffer + vars->bufferHalf + vars->lastRead - AES_BLOCK_SIZE,
- &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
- // decrypt the buffer
- aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
- &thisVector[0],
- vars->lastRead / AES_BLOCK_SIZE,
- vars->buffer + vars->bufferHalf,
- &cryptvars->ctx.decrypt);
- }
- }
- }
- while (size);
-
- return (err);
+ // invalidate & close the image file
+ if (p1) {
+ IOLockLock(gFSLock);
+ }
+ if (kFSTrimDelay == gFSState) {
+ IOPolledFileIOVars * vars = &gFileVars;
+ IOPolledFileClose(&vars,
+#if DISABLE_TRIM
+ 0, NULL, 0, 0, 0);
+#else
+ 0, (caddr_t)gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader),
+ sizeof(IOHibernateImageHeader), gIOHibernateCurrentHeader->imageSize);
+#endif
+ gFSState = kFSIdle;
+ }
+ if (p1) {
+ IOLockUnlock(gFSLock);
+ }
}
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
IOReturn
-IOHibernateSystemSleep(void)
+IOHibernateSystemPostWake(bool now)
{
- IOReturn err;
- OSData * data;
- OSObject * obj;
- OSString * str;
- OSNumber * num;
-
- IOHibernateVars * vars = &gIOHibernateVars;
-
- if (vars->fileVars && vars->fileVars->fileRef)
- // already on the way down
- return (kIOReturnSuccess);
-
- gIOHibernateState = kIOHibernateStateInactive;
-
- if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateModeKey)))
- {
- if ((num = OSDynamicCast(OSNumber, obj)))
- gIOHibernateMode = num->unsigned32BitValue();
- if (kIOHibernateModeSleep & gIOHibernateMode)
- // default to discard clean for safe sleep
- gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
- | kIOHibernateModeDiscardCleanActive);
-
- obj->release();
- }
- if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeRatioKey)))
- {
- if ((num = OSDynamicCast(OSNumber, obj)))
- gIOHibernateFreeRatio = num->unsigned32BitValue();
- obj->release();
- }
- if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeTimeKey)))
- {
- if ((num = OSDynamicCast(OSNumber, obj)))
- gIOHibernateFreeTime = num->unsigned32BitValue();
- obj->release();
- }
- if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
- {
- if ((str = OSDynamicCast(OSString, obj)))
- strcpy(gIOHibernateFilename, str->getCStringNoCopy());
- obj->release();
- }
-
- if (!gIOHibernateMode || !gIOHibernateFilename[0])
- return (kIOReturnUnsupported);
-
- HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
-
- do
- {
- vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(0, 4 * page_size, page_size);
- vars->ioBuffer = IOBufferMemoryDescriptor::withOptions(0, 2 * kDefaultIOSize, page_size);
-
- if (!vars->srcBuffer || !vars->ioBuffer)
- {
- err = kIOReturnNoMemory;
- break;
- }
-
- err = IOPolledFileOpen(gIOHibernateFilename, vars->ioBuffer,
- &vars->fileVars, &vars->fileExtents, &data);
- if (KERN_SUCCESS != err)
- {
- HIBLOG("IOPolledFileOpen(%x)\n", err);
- break;
- }
- if (vars->fileVars->fileRef)
- {
- // invalidate the image file
- gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
- int err = kern_write_file(vars->fileVars->fileRef, 0,
- (caddr_t) gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
- if (KERN_SUCCESS != err)
- HIBLOG("kern_write_file(%d)\n", err);
- }
-
- bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
-
- boolean_t encryptedswap;
- err = hibernate_setup(gIOHibernateCurrentHeader,
- gIOHibernateFreeRatio, gIOHibernateFreeTime,
- &vars->page_list, &vars->page_list_wired, &encryptedswap);
- if (KERN_SUCCESS != err)
- {
- HIBLOG("hibernate_setup(%d)\n", err);
- break;
- }
-
- if (encryptedswap)
- gIOHibernateMode ^= kIOHibernateModeEncrypt;
-
- vars->videoAllocSize = kVideoMapSize;
- if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize))
- vars->videoMapping = 0;
-
- // generate crypt keys
- for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
- vars->wiredCryptKey[i] = random();
- for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
- vars->cryptKey[i] = random();
-
- // set nvram
-
- IORegistryEntry * regEntry;
- if (!gIOOptionsEntry)
- {
- regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
- gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
- if (regEntry && !gIOOptionsEntry)
- regEntry->release();
- }
- if (!gIOChosenEntry)
- gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
-
- if (gIOOptionsEntry)
- {
- const OSSymbol * sym;
- size_t len;
- char valueString[16];
-
- sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
- if (sym)
- {
- gIOOptionsEntry->setProperty(sym, data);
- sym->release();
- }
- data->release();
-
- vars->saveBootDevice = gIOOptionsEntry->copyProperty(kIOSelectedBootDeviceKey);
- if (gIOChosenEntry)
- {
- OSData * bootDevice = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOBootPathKey));
- if (bootDevice)
- {
- sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey);
- OSString * str2 = OSString::withCStringNoCopy((const char *) bootDevice->getBytesNoCopy());
- if (sym && str2)
- gIOOptionsEntry->setProperty(sym, str2);
- if (sym)
- sym->release();
- if (str2)
- str2->release();
- }
-
- data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMemorySignatureKey));
- if (data)
- {
- vars->haveFastBoot = true;
-
- len = sprintf(valueString, "0x%lx", *((UInt32 *)data->getBytesNoCopy()));
- data = OSData::withBytes(valueString, len + 1);
- sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey);
- if (sym && data)
- gIOOptionsEntry->setProperty(sym, data);
- if (sym)
- sym->release();
- if (data)
- data->release();
- }
- data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
- if (data)
- gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
- }
-
- if (kIOHibernateModeEncrypt & gIOHibernateMode)
- {
- data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
- sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
- if (sym && data)
- gIOOptionsEntry->setProperty(sym, data);
- if (sym)
- sym->release();
- if (data)
- data->release();
- if (gIOHibernateBootSignature[0])
- {
- data = OSData::withCapacity(16);
- sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
- if (sym && data)
- {
- char c;
- uint8_t value;
- for (uint32_t i = 0; (c = gIOHibernateBootSignature[i]); i++)
- {
- if (c >= 'a')
- c -= 'a' - 10;
- else if (c >= 'A')
- c -= 'A' - 10;
- else if (c >= '0')
- c -= '0';
- else
- continue;
- value = (value << 4) | c;
- if (i & 1)
- data->appendBytes(&value, sizeof(value));
- }
- gIOOptionsEntry->setProperty(sym, data);
- }
- if (sym)
- sym->release();
- if (data)
- data->release();
- }
- }
-
- if (!vars->haveFastBoot)
- {
- // set boot volume to zero
- IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
- if (platform && (kIOReturnSuccess == platform->readXPRAM(kXPRamAudioVolume,
- &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume))))
- {
- uint8_t newVolume;
- newVolume = vars->saveBootAudioVolume & 0xf8;
- platform->writeXPRAM(kXPRamAudioVolume,
- &newVolume, sizeof(newVolume));
- }
- }
+ gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
+ IOSetBootImageNVRAM(NULL);
+
+ IOLockLock(gFSLock);
+ if (kFSTrimDelay == gFSState) {
+ thread_call_cancel(gIOHibernateTrimCalloutEntry);
+ IOHibernateSystemPostWakeTrim(NULL, NULL);
+ } else if (kFSOpened != gFSState) {
+ gFSState = kFSIdle;
+ } else {
+ gFSState = kFSTrimDelay;
+ if (now) {
+ thread_call_cancel(gIOHibernateTrimCalloutEntry);
+ IOHibernateSystemPostWakeTrim(NULL, NULL);
+ } else {
+ AbsoluteTime deadline;
+ clock_interval_to_deadline(TRIM_DELAY, kMillisecondScale, &deadline );
+ thread_call_enter1_delayed(gIOHibernateTrimCalloutEntry, NULL, deadline);
+ }
}
- // --
+ IOLockUnlock(gFSLock);
- gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
- gIOHibernateState = kIOHibernateStateHibernating;
- }
- while (false);
+ return kIOReturnSuccess;
+}
- return (err);
+uint32_t
+IOHibernateWasScreenLocked(void)
+{
+ uint32_t ret = 0;
+ if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry) {
+ OSData *
+ data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
+ if (data) {
+ ret = ((uint32_t *)data->getBytesNoCopy())[0];
+ gIOChosenEntry->setProperty(kIOBooterScreenLockStateKey, data);
+ }
+ } else {
+ gIOChosenEntry->removeProperty(kIOBooterScreenLockStateKey);
+ }
+
+ return ret;
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-IOReturn
-IOHibernateSystemHasSlept(void)
-{
- IOHibernateVars * vars = &gIOHibernateVars;
-
- if ((vars->previewData = OSDynamicCast(OSData,
- IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewBufferKey))))
- {
- vars->previewBuffer = IOMemoryDescriptor::withAddress(
- (void *) vars->previewData->getBytesNoCopy(),
- vars->previewData->getLength(),
- kIODirectionInOut);
-
- if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
- {
- vars->previewBuffer->release();
- vars->previewBuffer = 0;
- }
- if (!vars->previewBuffer)
- vars->previewData = 0;
- }
- if (gIOOptionsEntry)
- gIOOptionsEntry->sync();
-
- return (kIOReturnSuccess);
-}
+SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
+ CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_LOCKED,
+ gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
+SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
+ CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_LOCKED,
+ gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
+SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
+ CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_LOCKED,
+ &gIOHibernateMode, 0, "");
+SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
+ CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_LOCKED,
+ &_hibernateStats, hibernate_statistics_t, "");
+SYSCTL_OID_MANUAL(_kern_bridge, OID_AUTO, bootsessionuuid,
+ CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
+ gIOHibernateBridgeBootSessionUUIDString, sizeof(gIOHibernateBridgeBootSessionUUIDString),
+ sysctl_handle_string, "A", "");
+
+SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
+ CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_ANYBODY,
+ &_hibernateStats.graphicsReadyTime, 0, "");
+SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
+ CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_ANYBODY,
+ &_hibernateStats.wakeNotificationTime, 0, "");
+SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
+ CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_ANYBODY,
+ &_hibernateStats.lockScreenReadyTime, 0, "");
+SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
+ CTLFLAG_RW | CTLFLAG_KERN | CTLFLAG_ANYBODY,
+ &_hibernateStats.hidReadyTime, 0, "");
+
+SYSCTL_UINT(_kern, OID_AUTO, hibernatecount,
+ CTLFLAG_RD | CTLFLAG_KERN | CTLFLAG_ANYBODY,
+ &gIOHibernateCount, 0, "");
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-IOReturn
-IOHibernateSystemWake(void)
+static int
+hibernate_set_preview SYSCTL_HANDLER_ARGS
{
- IOHibernateVars * vars = &gIOHibernateVars;
-
- hibernate_teardown(vars->page_list, vars->page_list_wired);
-
- if (vars->videoMapping)
- {
- if (vars->videoMapSize)
- // remove mappings
- IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
- if (vars->videoAllocSize)
- // dealloc range
- kmem_free(kernel_map, trunc_page_32(vars->videoMapping), vars->videoAllocSize);
- }
-
- if (vars->previewBuffer)
- {
- vars->previewBuffer->release();
- vars->previewBuffer = 0;
- }
-
- if (vars->fileVars)
- {
- IOPolledFileClose(vars->fileVars);
- }
-
- // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
-
- OSData * data = OSData::withCapacity(4);
- if (gIOOptionsEntry && data)
- {
- const OSSymbol * sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
- if (sym)
- {
- gIOOptionsEntry->setProperty(sym, data);
- sym->release();
- }
- sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey);
- if (sym)
- {
- if (vars->saveBootDevice)
- {
- gIOOptionsEntry->setProperty(sym, vars->saveBootDevice);
- vars->saveBootDevice->release();
- }
- sym->release();
- }
- sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
- if (sym)
- {
- gIOOptionsEntry->setProperty(sym, data);
- sym->release();
- }
- sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey);
- if (sym)
- {
- gIOOptionsEntry->removeProperty(sym);
- sym->release();
- }
- }
- if (data)
- data->release();
-
- if (gIOOptionsEntry)
- {
- if (!vars->haveFastBoot)
- {
- // reset boot audio volume
- IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
- if (platform)
- platform->writeXPRAM(kXPRamAudioVolume,
- &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume));
- }
-
- // sync now to hardware if the booter has not
- if (kIOHibernateStateInactive == gIOHibernateState)
- gIOOptionsEntry->sync();
- else
- // just sync the variables in case a later panic syncs nvram (it won't sync variables)
- gIOOptionsEntry->syncOFVariables();
- }
-
- if (vars->srcBuffer)
- vars->srcBuffer->release();
- if (vars->ioBuffer)
- vars->ioBuffer->release();
- if (vars->fileExtents)
- vars->fileExtents->release();
-
- bzero(vars, sizeof(*vars));
+#pragma unused(oidp, arg1, arg2)
-// gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
+ if (!IOTaskHasEntitlement(current_task(), kIOHibernateSetPreviewEntitlementKey)) {
+ return EPERM;
+ }
- return (kIOReturnSuccess);
-}
+ if ((req->newptr == USER_ADDR_NULL) || (!req->newlen)) {
+ IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewBufferKey);
+ return 0;
+ }
-IOReturn
-IOHibernateSystemPostWake(void)
-{
- if (gIOHibernateFileRef)
- {
- kern_close_file_for_direct_io(gIOHibernateFileRef);
- gIOHibernateFileRef = 0;
- }
- return (kIOReturnSuccess);
+ size_t rounded_size = round_page(req->newlen);
+ IOBufferMemoryDescriptor *md = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn, rounded_size, page_size);
+ if (!md) {
+ return ENOMEM;
+ }
+
+ uint8_t *bytes = (uint8_t *)md->getBytesNoCopy();
+ int error = SYSCTL_IN(req, bytes, req->newlen);
+ if (error) {
+ md->release();
+ return error;
+ }
+
+ IOService::getPMRootDomain()->setProperty(kIOHibernatePreviewBufferKey, md);
+ md->release();
+
+ return 0;
}
+SYSCTL_PROC(_kern, OID_AUTO, hibernatepreview,
+ CTLTYPE_OPAQUE | CTLFLAG_WR | CTLFLAG_LOCKED | CTLFLAG_ANYBODY, NULL, 0,
+ hibernate_set_preview, "S", "");
+
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
void
IOHibernateSystemInit(IOPMrootDomain * rootDomain)
{
- OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
- if (data)
- {
- rootDomain->setProperty(kIOHibernateStateKey, data);
- data->release();
- }
-
- if (PE_parse_boot_arg("hfile", gIOHibernateFilename))
- gIOHibernateMode = kIOHibernateModeOn;
- else
- gIOHibernateFilename[0] = 0;
-
- static SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
- CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
- gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
- sysctl_register_oid(&sysctl__kern_hibernatefile);
-
- static SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
- CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
- gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
- sysctl_register_oid(&sysctl__kern_bootsignature);
-
- static SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
- CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
- &gIOHibernateMode, 0, "");
- sysctl_register_oid(&sysctl__kern_hibernatemode);
+ gIOHibernateBootImageKey = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
+ gIOHibernateBootSignatureKey = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
+ gIOBridgeBootSessionUUIDKey = OSSymbol::withCStringNoCopy(kIOBridgeBootSessionUUIDKey);
+
+#if defined(__i386__) || defined(__x86_64__)
+ gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
+ gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
+ gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
+ gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
+#endif /* defined(__i386__) || defined(__x86_64__) */
+
+ OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
+ if (data) {
+ rootDomain->setProperty(kIOHibernateStateKey, data);
+ data->release();
+ }
+
+ if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename))) {
+ gIOHibernateMode = kIOHibernateModeOn;
+ } else {
+ gIOHibernateFilename[0] = 0;
+ }
+
+ gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
+
+ if (gIOChosenEntry
+ && (data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(gIOBridgeBootSessionUUIDKey)))
+ && (sizeof(gIOHibernateBridgeBootSessionUUIDString) <= data->getLength())) {
+ sysctl_register_oid(&sysctl__kern_bridge_bootsessionuuid);
+ bcopy(data->getBytesNoCopy(), &gIOHibernateBridgeBootSessionUUIDString[0], sizeof(gIOHibernateBridgeBootSessionUUIDString));
+ }
+
+ gFSLock = IOLockAlloc();
+ gIOHibernateCount = 0;
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-static void
-hibernate_setup_for_wake(void)
+static IOReturn
+IOHibernatePolledFileWrite(IOHibernateVars * vars,
+ const uint8_t * bytes, IOByteCount size,
+ IOPolledFileCryptVars * cryptvars)
{
-#if __ppc__
- // go slow (state needed for wake)
- ml_set_processor_speed(1);
-#endif
+ IOReturn err;
+
+
+ err = IOPolledFileWrite(vars->fileVars, bytes, size, cryptvars);
+ if ((kIOReturnSuccess == err) && hibernate_should_abort()) {
+ err = kIOReturnAborted;
+ }
+
+
+ return err;
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-extern "C" boolean_t
+extern "C" uint32_t
hibernate_write_image(void)
{
- IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
- IOHibernateVars * vars = &gIOHibernateVars;
- IOPolledFileExtent * fileExtents;
-
- uint32_t pageCount, pagesDone;
- IOReturn err;
- vm_offset_t ppnum;
- IOItemCount page, count;
- uint8_t * src;
- uint8_t * data;
- IOByteCount pageCompressedSize;
- uint64_t compressedSize, uncompressedSize;
- uint64_t image1Size = 0;
- uint32_t bitmap_size;
- bool iterDone, pollerOpen, needEncryptStart;
- uint32_t restore1Sum, sum, sum1, sum2;
- uint32_t tag;
- uint32_t pageType;
- uint32_t pageAndCount[2];
-
- AbsoluteTime startTime, endTime;
- AbsoluteTime allTime, compTime, decoTime;
- uint64_t nsec;
- uint32_t lastProgressStamp = 0;
- uint32_t progressStamp;
-
- hibernate_cryptvars_t _cryptvars;
- hibernate_cryptvars_t * cryptvars = 0;
-
- if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
- return (false /* sleep */ );
-
- restore1Sum = sum1 = sum2 = 0;
-
- // encryption data. "iv" is the "initial vector".
- if (kIOHibernateModeEncrypt & gIOHibernateMode)
- {
- static const unsigned char first_iv[AES_BLOCK_SIZE]
- = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
- 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
-
- cryptvars = &gIOHibernateCryptWakeContext;
- bzero(cryptvars, sizeof(hibernate_cryptvars_t));
- aes_encrypt_key(vars->cryptKey,
- kIOHibernateAESKeySize,
- &cryptvars->ctx.encrypt);
- aes_decrypt_key(vars->cryptKey,
- kIOHibernateAESKeySize,
- &cryptvars->ctx.decrypt);
-
- cryptvars = &_cryptvars;
- bzero(cryptvars, sizeof(hibernate_cryptvars_t));
- aes_encrypt_key(vars->wiredCryptKey,
- kIOHibernateAESKeySize,
- &cryptvars->ctx.encrypt);
-
- bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
- bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
- bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
- bzero(gIOHibernateCryptWakeVars, sizeof(hibernate_cryptwakevars_t));
- }
-
- hibernate_setup_for_wake();
-
- hibernate_page_list_setall(vars->page_list,
- vars->page_list_wired,
- &pageCount);
-
- HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
-
- fileExtents = (IOPolledFileExtent *) vars->fileExtents->getBytesNoCopy();
+ IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
+ IOHibernateVars * vars = &gIOHibernateVars;
+ IOPolledFileExtent * fileExtents;
+
+#if !defined(__arm64__)
+ _static_assert_1_arg(sizeof(IOHibernateImageHeader) == 512);
+#endif /* !defined(__arm64__) */
+
+ uint32_t pageCount, pagesDone;
+ IOReturn err;
+ ppnum_t ppnum, page;
+ vm_offset_t count;
+ uint8_t * src;
+ uint8_t * data;
+ uint8_t * compressed;
+ uint8_t * scratch;
+ IOByteCount pageCompressedSize;
+ uint64_t compressedSize, uncompressedSize;
+ uint64_t image1Size = 0;
+ uint32_t bitmap_size;
+ bool iterDone, pollerOpen, needEncrypt;
+ int wkresult;
+ uint32_t tag;
+ uint32_t pageType;
+ uint32_t pageAndCount[2];
+ addr64_t phys64;
+ IOByteCount segLen;
+ uint32_t restore1Sum = 0, sum = 0, sum1 = 0, sum2 = 0;
+ uintptr_t hibernateBase;
+ uintptr_t hibernateEnd;
+
+ AbsoluteTime startTime, endTime;
+ AbsoluteTime allTime, compTime;
+ uint64_t compBytes;
+ uint64_t nsec;
+ uint64_t lastProgressStamp = 0;
+ uint64_t progressStamp;
+ uint32_t blob, lastBlob = (uint32_t) -1L;
+
+ uint32_t wiredPagesEncrypted;
+ uint32_t dirtyPagesEncrypted;
+ uint32_t wiredPagesClear;
+ uint32_t svPageCount;
+ uint32_t zvPageCount;
+
+ IOPolledFileCryptVars _cryptvars;
+ IOPolledFileCryptVars * cryptvars = NULL;
+
+ wiredPagesEncrypted = 0;
+ dirtyPagesEncrypted = 0;
+ wiredPagesClear = 0;
+ svPageCount = 0;
+ zvPageCount = 0;
+
+ if (!vars->fileVars
+ || !vars->fileVars->pollers
+ || !(kIOHibernateModeOn & gIOHibernateMode)) {
+ return kIOHibernatePostWriteSleep;
+ }
+
+
+#if !defined(__arm64__)
+ if (kIOHibernateModeSleep & gIOHibernateMode) {
+ kdebug_enable = save_kdebug_enable;
+ }
+#endif /* !defined(__arm64__) */
+
+ pal_hib_write_hook();
+
+ KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START);
+ IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
+
+#if CRYPTO
+ // encryption data. "iv" is the "initial vector".
+ if (kIOHibernateModeEncrypt & gIOHibernateMode) {
+ static const unsigned char first_iv[AES_BLOCK_SIZE]
+ = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
+ 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
+
+ cryptvars = &gIOHibernateCryptWakeContext;
+ bzero(cryptvars, sizeof(IOPolledFileCryptVars));
+ aes_encrypt_key(vars->cryptKey,
+ kIOHibernateAESKeySize,
+ &cryptvars->ctx.encrypt);
+ aes_decrypt_key(vars->cryptKey,
+ kIOHibernateAESKeySize,
+ &cryptvars->ctx.decrypt);
+
+ cryptvars = &_cryptvars;
+ bzero(cryptvars, sizeof(IOPolledFileCryptVars));
+ for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++) {
+ vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
+ }
+ aes_encrypt_key(vars->wiredCryptKey,
+ kIOHibernateAESKeySize,
+ &cryptvars->ctx.encrypt);
+
+ bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
+ bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
+ bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
+ }
+#endif /* CRYPTO */
+
+ hibernate_page_list_setall(vars->page_list,
+ vars->page_list_wired,
+ vars->page_list_pal,
+ false /* !preflight */,
+ /* discard_all */
+ ((0 == (kIOHibernateModeSleep & gIOHibernateMode))
+ && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))),
+ &pageCount);
+
+ HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
+
+ fileExtents = (IOPolledFileExtent *) vars->fileVars->fileExtents->getBytesNoCopy();
#if 0
- count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
- for (page = 0; page < count; page++)
- {
- HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
- fileExtents[page].start, fileExtents[page].length,
- fileExtents[page].start + fileExtents[page].length);
- }
+ count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
+ for (page = 0; page < count; page++) {
+ HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
+ fileExtents[page].start, fileExtents[page].length,
+ fileExtents[page].start + fileExtents[page].length);
+ }
#endif
- needEncryptStart = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
-
- AbsoluteTime_to_scalar(&compTime) = 0;
- AbsoluteTime_to_scalar(&decoTime) = 0;
-
- clock_get_uptime(&allTime);
-
- do
- {
- compressedSize = 0;
- uncompressedSize = 0;
- iterDone = false;
- pageType = 0; // wired pages first
-
- IOPolledFileSeek(vars->fileVars, sizeof(IOHibernateImageHeader));
-
- HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
- ml_get_interrupts_enabled());
- err = IOHibernatePollerOpen(vars->fileVars, kIOPolledBeforeSleepState, vars->ioBuffer);
- HIBLOG("IOHibernatePollerOpen(%x)\n", err);
- pollerOpen = (kIOReturnSuccess == err);
- if (!pollerOpen)
- break;
-
- // copy file block extent list if larger than header
-
- count = vars->fileExtents->getLength();
- if (count > sizeof(header->fileExtentMap))
- {
- count -= sizeof(header->fileExtentMap);
- err = IOPolledFileWrite(vars->fileVars,
- ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
- if (kIOReturnSuccess != err)
- break;
- }
-
- // copy out restore1 code
-
- page = atop_32(sectHIBB);
- count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page;
- header->restore1CodePage = page;
- header->restore1PageCount = count;
- header->restore1CodeOffset = ((uint32_t) &hibernate_machine_entrypoint) - sectHIBB;
- header->restore1StackOffset = ((uint32_t) &gIOHibernateRestoreStackEnd[0]) - 64 - sectHIBB;
-
- // sum __HIB sect, with zeros for the stack
- src = (uint8_t *) trunc_page(sectHIBB);
- for (page = 0; page < count; page++)
- {
- if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
- restore1Sum += hibernate_sum(src, page_size);
- else
- restore1Sum += 0x10000001;
- src += page_size;
- }
- sum1 = restore1Sum;
-
- // write the __HIB sect, with zeros for the stack
-
- src = (uint8_t *) trunc_page(sectHIBB);
- count = ((uint32_t) &gIOHibernateRestoreStack[0]) - trunc_page(sectHIBB);
- if (count)
- {
- err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
- if (kIOReturnSuccess != err)
- break;
- }
- err = IOPolledFileWrite(vars->fileVars,
- (uint8_t *) 0,
- &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
- cryptvars);
- if (kIOReturnSuccess != err)
- break;
- src = &gIOHibernateRestoreStackEnd[0];
- count = round_page(sectHIBB + sectSizeHIB) - ((uint32_t) src);
- if (count)
- {
- err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
- if (kIOReturnSuccess != err)
- break;
- }
-
- // write the preview buffer
-
- addr64_t phys64;
- IOByteCount segLen;
-
- if (vars->previewData)
- {
- ppnum = 0;
- count = 0;
- do
- {
- phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen);
- pageAndCount[0] = atop_64(phys64);
- pageAndCount[1] = atop_32(segLen);
- err = IOPolledFileWrite(vars->fileVars,
- (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
- cryptvars);
- if (kIOReturnSuccess != err)
- break;
- count += segLen;
- ppnum += sizeof(pageAndCount);
- }
- while (phys64);
- if (kIOReturnSuccess != err)
- break;
-
- src = (uint8_t *) vars->previewData->getBytesNoCopy();
- count = vars->previewData->getLength();
-
- header->previewPageListSize = ppnum;
- header->previewSize = count + ppnum;
-
- for (page = 0; page < count; page += page_size)
- sum1 += hibernate_sum(src + page, page_size);
-
- err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
- if (kIOReturnSuccess != err)
- break;
- }
-
- // mark areas for no save
-
- for (count = 0;
- (phys64 = vars->ioBuffer->getPhysicalSegment64(count, &segLen));
- count += segLen)
- {
- hibernate_set_page_state(vars->page_list, vars->page_list_wired,
- atop_64(phys64), atop_32(segLen),
- kIOHibernatePageStateFree);
- pageCount -= atop_32(segLen);
- }
-
- for (count = 0;
- (phys64 = vars->srcBuffer->getPhysicalSegment64(count, &segLen));
- count += segLen)
- {
- hibernate_set_page_state(vars->page_list, vars->page_list_wired,
- atop_64(phys64), atop_32(segLen),
- kIOHibernatePageStateFree);
- pageCount -= atop_32(segLen);
- }
-
- // copy out bitmap of pages available for trashing during restore
-
- bitmap_size = vars->page_list_wired->list_size;
- src = (uint8_t *) vars->page_list_wired;
- err = IOPolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
- if (kIOReturnSuccess != err)
- break;
-
- // mark more areas for no save, but these are not available
- // for trashing during restore
-
-#if !__i386__
- page = atop_32(sectHIBB);
- count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page;
+ needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
+ AbsoluteTime_to_scalar(&compTime) = 0;
+ compBytes = 0;
+
+ clock_get_uptime(&allTime);
+ IOService::getPMRootDomain()->pmStatsRecordEvent(
+ kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
+ do{
+ compressedSize = 0;
+ uncompressedSize = 0;
+ svPageCount = 0;
+ zvPageCount = 0;
+
+ IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);
+
+ HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
+ ml_get_interrupts_enabled());
+ err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledBeforeSleepState,
+ // abortable if not low battery
+ !IOService::getPMRootDomain()->mustHibernate());
+ HIBLOG("IOHibernatePollerOpen(%x)\n", err);
+ pollerOpen = (kIOReturnSuccess == err);
+ if (!pollerOpen) {
+ break;
+ }
+
+ if (vars->volumeCryptKeySize) {
+ err = IOPolledFilePollersSetEncryptionKey(vars->fileVars, &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
+ HIBLOG("IOPolledFilePollersSetEncryptionKey(%x)\n", err);
+ vars->hwEncrypt = (kIOReturnSuccess == err);
+ bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
+ if (vars->hwEncrypt) {
+ header->options |= kIOHibernateOptionHWEncrypt;
+ }
+ }
+
+ // copy file block extent list if larger than header
+
+ count = vars->fileVars->fileExtents->getLength();
+ if (count > sizeof(header->fileExtentMap)) {
+ count -= sizeof(header->fileExtentMap);
+ err = IOHibernatePolledFileWrite(vars,
+ ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ }
+
+ // copy out restore1 code
+
+ for (count = 0;
+ (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
+ count += segLen) {
+ for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++) {
+ gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64_ppnum(phys64) + pagesDone;
+ }
+ }
+
+ hibernateBase = HIB_BASE; /* Defined in PAL headers */
+ hibernateEnd = (segHIBB + segSizeHIB);
+
+ page = atop_32(kvtophys(hibernateBase));
+ count = atop_32(round_page(hibernateEnd) - hibernateBase);
+ uintptr_t entrypoint = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
+ uintptr_t stack = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
+ if ((count > UINT_MAX) || (entrypoint > UINT_MAX) || (stack > UINT_MAX)) {
+ panic("malformed kernel layout");
+ }
+ header->restore1CodePhysPage = (ppnum_t) page;
+ header->restore1CodeVirt = hibernateBase;
+ header->restore1PageCount = (uint32_t) count;
+ header->restore1CodeOffset = (uint32_t) entrypoint;
+ header->restore1StackOffset = (uint32_t) stack;
+
+ if (uuid_parse(&gIOHibernateBridgeBootSessionUUIDString[0], &header->bridgeBootSessionUUID[0])) {
+ bzero(&header->bridgeBootSessionUUID[0], sizeof(header->bridgeBootSessionUUID));
+ }
+
+ // sum __HIB seg, with zeros for the stack
+ src = (uint8_t *) trunc_page(hibernateBase);
+ for (page = 0; page < count; page++) {
+ if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0])) {
+ restore1Sum += hibernate_sum_page(src, (uint32_t) (header->restore1CodeVirt + page));
+ } else {
+ restore1Sum += 0x00000000;
+ }
+ src += page_size;
+ }
+ sum1 = restore1Sum;
+
+ // write the __HIB seg, with zeros for the stack
+
+ src = (uint8_t *) trunc_page(hibernateBase);
+ count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
+ if (count) {
+ err = IOHibernatePolledFileWrite(vars, src, count, cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ }
+ err = IOHibernatePolledFileWrite(vars,
+ (uint8_t *) NULL,
+ &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
+ cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ src = &gIOHibernateRestoreStackEnd[0];
+ count = round_page(hibernateEnd) - ((uintptr_t) src);
+ if (count) {
+ err = IOHibernatePolledFileWrite(vars, src, count, cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ }
+
+ if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode)) {
+ vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
+ vars->fileVars->encryptEnd = UINT64_MAX;
+ HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
+ }
+
+ // write the preview buffer
+
+ if (vars->previewBuffer) {
+ ppnum = 0;
+ count = 0;
+ do{
+ phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
+ pageAndCount[0] = atop_64_ppnum(phys64);
+ pageAndCount[1] = atop_64_ppnum(segLen);
+ err = IOHibernatePolledFileWrite(vars,
+ (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
+ cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ count += segLen;
+ ppnum += sizeof(pageAndCount);
+ }while (phys64);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+
+ src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
+
+ ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
+
+ count = (uint32_t) vars->previewBuffer->getLength();
+
+ header->previewPageListSize = ((uint32_t) ppnum);
+ header->previewSize = ((uint32_t) (count + ppnum));
+
+ for (page = 0; page < count; page += page_size) {
+ phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
+ sum1 += hibernate_sum_page(src + page, atop_64_ppnum(phys64));
+ }
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ err = IOHibernatePolledFileWrite(vars, src, count, cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ }
+
+ // mark areas for no save
+ hibernate_set_descriptor_page_state(vars, IOPolledFileGetIOBuffer(vars->fileVars),
+ kIOHibernatePageStateFree, &pageCount);
+ hibernate_set_descriptor_page_state(vars, vars->srcBuffer,
+ kIOHibernatePageStateFree, &pageCount);
+
+ // copy out bitmap of pages available for trashing during restore
+
+ bitmap_size = vars->page_list_wired->list_size;
+ src = (uint8_t *) vars->page_list_wired;
+ err = IOHibernatePolledFileWrite(vars, src, bitmap_size, cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+
+ // mark more areas for no save, but these are not available
+ // for trashing during restore
+
+ hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
+
+#if defined(__i386__) || defined(__x86_64__)
+ // __HIB is explicitly saved above so we don't have to save it again
+ page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
+ count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
+ hibernate_set_page_state(vars->page_list, vars->page_list_wired,
+ page, count,
+ kIOHibernatePageStateFree);
+ pageCount -= count;
+#elif defined(__arm64__)
+ // the segments described in IOHibernateHibSegInfo are stored directly in the
+ // hibernation file, so they don't need to be saved again
+ extern unsigned long gPhysBase, gPhysSize;
+ for (size_t i = 0; i < NUM_HIBSEGINFO_SEGMENTS; i++) {
+ page = segInfo->segments[i].physPage;
+ count = segInfo->segments[i].pageCount;
+ uint64_t physAddr = ptoa_64(page);
+ uint64_t size = ptoa_64(count);
+ if (size &&
+ (physAddr >= gPhysBase) &&
+ (physAddr + size <= gPhysBase + gPhysSize)) {
+ hibernate_set_page_state(vars->page_list, vars->page_list_wired,
+ page, count,
+ kIOHibernatePageStateFree);
+ pageCount -= count;
+ }
+ }
#else
- // XXX
- page = atop_32(sectHIBB & 0x3FFFFFFF);
- count = atop_32(round_page((sectHIBB + sectSizeHIB) & 0x3FFFFFFF)) - page;
+#error unimplemented
#endif
- hibernate_set_page_state(vars->page_list, vars->page_list_wired,
- page, count,
- kIOHibernatePageStateFree);
- pageCount -= count;
-
-
-
- if (vars->previewBuffer) for (count = 0;
- (phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen));
- count += segLen)
- {
- hibernate_set_page_state(vars->page_list, vars->page_list_wired,
- atop_64(phys64), atop_32(segLen),
- kIOHibernatePageStateFree);
- pageCount -= atop_32(segLen);
- }
-
- src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
-
- void * iter = 0;
- pagesDone = 0;
-
- HIBLOG("writing %d pages\n", pageCount);
-
- do
- {
- count = hibernate_page_list_iterate(pageType ? vars->page_list : vars->page_list_wired,
- &iter, &ppnum);
-// kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
-
- iterDone = !count;
-
- pageAndCount[0] = ppnum;
- pageAndCount[1] = count;
- err = IOPolledFileWrite(vars->fileVars,
- (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
- cryptvars);
- if (kIOReturnSuccess != err)
- break;
-
- for (page = 0; page < count; page++)
- {
- err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(ppnum), page_size);
- if (err)
- {
- HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%d] %x\n", __LINE__, ppnum, err);
- break;
- }
-
- sum = hibernate_sum(src, page_size);
-
- clock_get_uptime(&startTime);
-
- pageCompressedSize = WKdm_compress ((WK_word*) src, (WK_word*) (src + page_size), PAGE_SIZE_IN_WORDS);
-
- clock_get_uptime(&endTime);
- ADD_ABSOLUTETIME(&compTime, &endTime);
- SUB_ABSOLUTETIME(&compTime, &startTime);
-
- if (kIOHibernateModeEncrypt & gIOHibernateMode)
- pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1);
-
- if (pageCompressedSize > page_size)
- {
-// HIBLOG("------------lose: %d\n", pageCompressedSize);
- pageCompressedSize = page_size;
- }
-
- if (pageCompressedSize != page_size)
- data = (src + page_size);
- else
- data = src;
-
- tag = pageCompressedSize | kIOHibernateTagSignature;
-
- if (pageType)
- sum2 += sum;
- else
- sum1 += sum;
-
- if (needEncryptStart && (ppnum >= atop_32(sectDATAB)))
- {
- // start encrypting partway into the data about to be written
- vars->fileVars->encryptStart = (vars->fileVars->position + AES_BLOCK_SIZE - 1)
- & ~(AES_BLOCK_SIZE - 1);
- needEncryptStart = false;
- }
-
- err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
- if (kIOReturnSuccess != err)
- break;
-
- err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
- if (kIOReturnSuccess != err)
- break;
-
- compressedSize += pageCompressedSize;
- if (pageCompressedSize)
- uncompressedSize += page_size;
- ppnum++;
- pagesDone++;
-
- if (0 == (8191 & pagesDone))
- {
- clock_get_uptime(&endTime);
- SUB_ABSOLUTETIME(&endTime, &allTime);
- absolutetime_to_nanoseconds(endTime, &nsec);
- progressStamp = nsec / 750000000ULL;
- if (progressStamp != lastProgressStamp)
- {
- lastProgressStamp = progressStamp;
- HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
- }
- }
- }
- if (kIOReturnSuccess != err)
- break;
- if (iterDone && !pageType)
- {
- err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
- if (kIOReturnSuccess != err)
- break;
-
- iterDone = false;
- pageType = 1;
- iter = 0;
- image1Size = vars->fileVars->position;
- if (cryptvars)
- {
- bcopy(&cryptvars->aes_iv[0],
- &gIOHibernateCryptWakeContext.aes_iv[0],
- sizeof(cryptvars->aes_iv));
- cryptvars = &gIOHibernateCryptWakeContext;
- }
- HIBLOG("image1Size %qd\n", image1Size);
- }
- }
- while (!iterDone);
- if (kIOReturnSuccess != err)
- break;
- err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
- if (kIOReturnSuccess != err)
- break;
-
- // Header:
-
- header->imageSize = vars->fileVars->position;
- header->image1Size = image1Size;
- header->bitmapSize = bitmap_size;
- header->pageCount = pageCount;
- header->encryptStart = vars->fileVars->encryptStart;
-
- header->restore1Sum = restore1Sum;
- header->image1Sum = sum1;
- header->image2Sum = sum2;
-
- count = vars->fileExtents->getLength();
- if (count > sizeof(header->fileExtentMap))
- {
- header->fileExtentMapSize = count;
- count = sizeof(header->fileExtentMap);
- }
- else
- header->fileExtentMapSize = sizeof(header->fileExtentMap);
- bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
-
- IOPolledFileSeek(vars->fileVars, 0);
- err = IOPolledFileWrite(vars->fileVars,
- (uint8_t *) header, sizeof(IOHibernateImageHeader),
- cryptvars);
- if (kIOReturnSuccess != err)
- break;
- err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
- if (kIOReturnSuccess != err)
- break;
- err = IOHibernatePollerIODone(vars->fileVars);
- if (kIOReturnSuccess != err)
- break;
- }
- while (false);
-
- clock_get_uptime(&endTime);
- SUB_ABSOLUTETIME(&endTime, &allTime);
- absolutetime_to_nanoseconds(endTime, &nsec);
- HIBLOG("all time: %qd ms, ",
- nsec / 1000000ULL);
-
- absolutetime_to_nanoseconds(compTime, &nsec);
- HIBLOG("comp time: %qd ms, ",
- nsec / 1000000ULL);
-
- absolutetime_to_nanoseconds(decoTime, &nsec);
- HIBLOG("deco time: %qd ms, ",
- nsec / 1000000ULL);
-
- HIBLOG("\nimage %qd, uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
- header->imageSize,
- uncompressedSize, atop_32(uncompressedSize), compressedSize,
- (int) ((compressedSize * 100ULL) / uncompressedSize),
- sum1, sum2);
-
- if (pollerOpen)
- IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState);
-
- HIBLOG("hibernate_write_image done(%x)\n", err);
-
- // should we come back via regular wake, set the state in memory.
- gIOHibernateState = kIOHibernateStateInactive;
-
- if ((kIOReturnSuccess == err) && !(kIOHibernateModeSleep & gIOHibernateMode))
- return (true /* power down */ );
- else
- return (false /* sleep */ );
-}
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+ hibernate_set_descriptor_page_state(vars, vars->previewBuffer,
+ kIOHibernatePageStateFree, &pageCount);
+ hibernate_set_descriptor_page_state(vars, vars->handoffBuffer,
+ kIOHibernatePageStateFree, &pageCount);
-DECLARE_IOHIBERNATEPROGRESSALPHA
+#if KASAN
+ vm_size_t shadow_pages_free = atop_64(shadow_ptop) - atop_64(shadow_pnext);
-static void
-ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
-{
- uint32_t rowBytes, pixelShift;
- uint32_t x, y;
- int32_t blob, lastBlob;
- uint32_t alpha, in, color, result;
- uint8_t * out;
- uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
-
- pixelShift = display->depth >> 4;
- if (pixelShift < 1)
- return;
-
- rowBytes = display->rowBytes;
-
- screen += ((display->width
- - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
- + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
-
- lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
-
- screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
-
- for (y = 0; y < kIOHibernateProgressHeight; y++)
- {
- out = screen + y * rowBytes;
- for (blob = firstBlob; blob <= lastBlob; blob++)
- {
- color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
- for (x = 0; x < kIOHibernateProgressWidth; x++)
- {
- alpha = gIOHibernateProgressAlpha[y][x];
- result = color;
- if (alpha)
- {
- if (0xff != alpha)
- {
- in = display->progressSaveUnder[blob][saveindex[blob]++];
- result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
- }
- if (1 == pixelShift)
- {
- result >>= 3;
- *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
- }
- else
- *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
- }
- out += (1 << pixelShift);
- }
- out += (kIOHibernateProgressSpacing << pixelShift);
- }
- }
+ /* no need to save unused shadow pages */
+ hibernate_set_page_state(vars->page_list, vars->page_list_wired,
+ atop_64(shadow_pnext),
+ shadow_pages_free,
+ kIOHibernatePageStateFree);
+#endif
+
+ src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
+ compressed = src + page_size;
+ scratch = compressed + page_size;
+
+ pagesDone = 0;
+ lastBlob = 0;
+
+ HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
+ bitmap_size, header->previewSize,
+ pageCount, vars->fileVars->position);
+
+
+ enum
+ // pageType
+ {
+ kWired = 0x02,
+ kEncrypt = 0x01,
+ kWiredEncrypt = kWired | kEncrypt,
+ kWiredClear = kWired,
+ kUnwiredEncrypt = kEncrypt
+ };
+
+#if defined(__i386__) || defined(__x86_64__)
+ bool cpuAES = (0 != (CPUID_FEATURE_AES & cpuid_features()));
+#else /* defined(__i386__) || defined(__x86_64__) */
+ static const bool cpuAES = true;
+#endif /* defined(__i386__) || defined(__x86_64__) */
+
+ for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--) {
+ if (kUnwiredEncrypt == pageType) {
+ // start unwired image
+ if (!vars->hwEncrypt && (kIOHibernateModeEncrypt & gIOHibernateMode)) {
+ vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
+ vars->fileVars->encryptEnd = UINT64_MAX;
+ HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
+ }
+ bcopy(&cryptvars->aes_iv[0],
+ &gIOHibernateCryptWakeContext.aes_iv[0],
+ sizeof(cryptvars->aes_iv));
+ cryptvars = &gIOHibernateCryptWakeContext;
+ }
+ for (iterDone = false, ppnum = 0; !iterDone;) {
+ if (cpuAES && (pageType == kWiredClear)) {
+ count = 0;
+ } else {
+ count = hibernate_page_list_iterate((kWired & pageType) ? vars->page_list_wired : vars->page_list,
+ &ppnum);
+ if (count > UINT_MAX) {
+ count = UINT_MAX;
+ }
+ }
+// kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
+ iterDone = !count;
+
+ if (!cpuAES) {
+ if (count && (kWired & pageType) && needEncrypt) {
+ uint32_t checkIndex;
+ for (checkIndex = 0;
+ (checkIndex < count)
+ && (((kEncrypt & pageType) == 0) == pmap_is_noencrypt(((ppnum_t)(ppnum + checkIndex))));
+ checkIndex++) {
+ }
+ if (!checkIndex) {
+ ppnum++;
+ continue;
+ }
+ count = checkIndex;
+ }
+ }
+
+ switch (pageType) {
+ case kWiredEncrypt: wiredPagesEncrypted += count; break;
+ case kWiredClear: wiredPagesClear += count; break;
+ case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
+ }
+
+ if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */
+ } else {
+ pageAndCount[0] = (uint32_t) ppnum;
+ pageAndCount[1] = (uint32_t) count;
+ err = IOHibernatePolledFileWrite(vars,
+ (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
+ cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ }
+
+ for (page = ppnum; page < (ppnum + count); page++) {
+ err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
+ if (err) {
+ HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
+ break;
+ }
+
+ sum = hibernate_sum_page(src, (uint32_t) page);
+ if (kWired & pageType) {
+ sum1 += sum;
+ } else {
+ sum2 += sum;
+ }
+
+ clock_get_uptime(&startTime);
+ wkresult = WKdm_compress_new((const WK_word*) src,
+ (WK_word*) compressed,
+ (WK_word*) scratch,
+ (uint32_t) (page_size - 4));
+
+ clock_get_uptime(&endTime);
+ ADD_ABSOLUTETIME(&compTime, &endTime);
+ SUB_ABSOLUTETIME(&compTime, &startTime);
+
+ compBytes += page_size;
+ pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
+
+ if (pageCompressedSize == 0) {
+ pageCompressedSize = 4;
+ data = src;
+
+ if (*(uint32_t *)src) {
+ svPageCount++;
+ } else {
+ zvPageCount++;
+ }
+ } else {
+ if (pageCompressedSize != page_size) {
+ data = compressed;
+ } else {
+ data = src;
+ }
+ }
+
+ assert(pageCompressedSize <= page_size);
+ tag = ((uint32_t) pageCompressedSize) | kIOHibernateTagSignature;
+ err = IOHibernatePolledFileWrite(vars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+
+ err = IOHibernatePolledFileWrite(vars, data, (pageCompressedSize + 3) & ~3, cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+
+ compressedSize += pageCompressedSize;
+ uncompressedSize += page_size;
+ pagesDone++;
+
+ if (vars->consoleMapping && (0 == (1023 & pagesDone))) {
+ blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
+ if (blob != lastBlob) {
+ ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
+ lastBlob = blob;
+ }
+ }
+ if (0 == (8191 & pagesDone)) {
+ clock_get_uptime(&endTime);
+ SUB_ABSOLUTETIME(&endTime, &allTime);
+ absolutetime_to_nanoseconds(endTime, &nsec);
+ progressStamp = nsec / 750000000ULL;
+ if (progressStamp != lastProgressStamp) {
+ lastProgressStamp = progressStamp;
+ HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
+ }
+ }
+ }
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ ppnum = page;
+ }
+
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+
+ if ((kEncrypt & pageType) && vars->fileVars->encryptStart) {
+ vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
+ HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
+ }
+
+ if (kWiredEncrypt != pageType) {
+ // end of image1/2 - fill to next block
+ err = IOHibernatePolledFileWrite(vars, NULL, 0, cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+ }
+ if (kWiredClear == pageType) {
+ // enlarge wired image for test
+// err = IOHibernatePolledFileWrite(vars, 0, 0x60000000, cryptvars);
+
+ // end wired image
+ header->encryptStart = vars->fileVars->encryptStart;
+ header->encryptEnd = vars->fileVars->encryptEnd;
+ image1Size = vars->fileVars->position;
+ HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
+ image1Size, header->encryptStart, header->encryptEnd);
+ }
+ }
+ if (kIOReturnSuccess != err) {
+ if (kIOReturnOverrun == err) {
+ // update actual compression ratio on not enough space (for retry)
+ gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
+ }
+
+ // update partial amount written (for IOPolledFileClose cleanup/unmap)
+ header->imageSize = vars->fileVars->position;
+ break;
+ }
+
+
+ // Header:
+
+ header->imageSize = vars->fileVars->position;
+ header->image1Size = image1Size;
+ header->bitmapSize = bitmap_size;
+ header->pageCount = pageCount;
+
+ header->restore1Sum = restore1Sum;
+ header->image1Sum = sum1;
+ header->image2Sum = sum2;
+ header->sleepTime = gIOLastSleepTime.tv_sec;
+
+ header->compression = ((uint32_t)((compressedSize << 8) / uncompressedSize));
+#if defined(__arm64__)
+ /*
+ * We don't support retry on hibernation failure and so
+ * we don't want to set this value to anything smaller
+ * just because we may have been lucky this time around.
+ * Though we'll let it go higher.
+ */
+ if (header->compression < HIB_COMPR_RATIO_ARM64) {
+ header->compression = HIB_COMPR_RATIO_ARM64;
+ }
+#endif /* __arm64__ */
+
+ gIOHibernateCompression = header->compression;
+
+ count = vars->fileVars->fileExtents->getLength();
+ if (count > sizeof(header->fileExtentMap)) {
+ header->fileExtentMapSize = ((uint32_t) count);
+ count = sizeof(header->fileExtentMap);
+ } else {
+ header->fileExtentMapSize = sizeof(header->fileExtentMap);
+ }
+ bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
+
+ header->deviceBase = vars->fileVars->block0;
+ header->deviceBlockSize = vars->fileVars->blockSize;
+ header->lastHibAbsTime = mach_absolute_time();
+ header->lastHibContTime = mach_continuous_time();
+
+
+ IOPolledFileSeek(vars->fileVars, 0);
+ err = IOHibernatePolledFileWrite(vars,
+ (uint8_t *) header, sizeof(IOHibernateImageHeader),
+ cryptvars);
+ if (kIOReturnSuccess != err) {
+ break;
+ }
+
+ err = IOHibernatePolledFileWrite(vars, NULL, 0, cryptvars);
+ }while (false);
+
+ clock_get_uptime(&endTime);
+
+ IOService::getPMRootDomain()->pmStatsRecordEvent(
+ kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
+
+ SUB_ABSOLUTETIME(&endTime, &allTime);
+ absolutetime_to_nanoseconds(endTime, &nsec);
+ HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
+
+ absolutetime_to_nanoseconds(compTime, &nsec);
+ HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
+ compBytes,
+ nsec / 1000000ULL,
+ nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+ absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
+ HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
+ vars->fileVars->cryptBytes,
+ nsec / 1000000ULL,
+ nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+ HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%)\n",
+ header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
+ uncompressedSize, atop_32(uncompressedSize), compressedSize,
+ uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0);
+
+ HIBLOG("\nsum1 %x, sum2 %x\n", sum1, sum2);
+
+ HIBLOG("svPageCount %d, zvPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n",
+ svPageCount, zvPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
+
+ if (pollerOpen) {
+ IOPolledFilePollersClose(vars->fileVars, (kIOReturnSuccess == err) ? kIOPolledBeforeSleepState : kIOPolledBeforeSleepStateAborted );
+ }
+
+ if (vars->consoleMapping) {
+ ProgressUpdate(gIOHibernateGraphicsInfo,
+ vars->consoleMapping, 0, kIOHibernateProgressCount);
+ }
+
+ HIBLOG("hibernate_write_image done(%x)\n", err);
+
+ // should we come back via regular wake, set the state in memory.
+ gIOHibernateState = kIOHibernateStateInactive;
+
+ KDBG(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END, wiredPagesEncrypted,
+ wiredPagesClear, dirtyPagesEncrypted);
+
+#if defined(__arm64__)
+ if (kIOReturnSuccess == err) {
+ return kIOHibernatePostWriteHalt;
+ } else {
+ // on ARM, once ApplePMGR decides we're hibernating, we can't turn back
+ // see: <rdar://problem/63848862> Tonga ApplePMGR diff quiesce path support
+ panic("hibernate_write_image encountered error 0x%x", err);
+ }
+#else
+ if (kIOReturnSuccess == err) {
+ if (kIOHibernateModeSleep & gIOHibernateMode) {
+ return kIOHibernatePostWriteSleep;
+ } else if (kIOHibernateModeRestart & gIOHibernateMode) {
+ return kIOHibernatePostWriteRestart;
+ } else {
+ /* by default, power down */
+ return kIOHibernatePostWriteHalt;
+ }
+ } else if (kIOReturnAborted == err) {
+ return kIOHibernatePostWriteWake;
+ } else {
+ /* on error, sleep */
+ return kIOHibernatePostWriteSleep;
+ }
+#endif
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-extern "C" void
+extern "C" void
hibernate_machine_init(void)
{
- IOReturn err;
- uint32_t sum;
- uint32_t pagesDone;
- AbsoluteTime allTime, endTime;
- uint64_t nsec;
- uint32_t lastProgressStamp = 0;
- uint32_t progressStamp;
- uint64_t progressZeroPosition = 0;
- uint32_t blob, lastBlob = (uint32_t) -1L;
- hibernate_cryptvars_t * cryptvars = 0;
+ IOReturn err;
+ uint32_t sum;
+ uint32_t pagesDone;
+ uint32_t pagesRead = 0;
+ AbsoluteTime startTime, compTime;
+ AbsoluteTime allTime, endTime;
+ AbsoluteTime startIOTime, endIOTime;
+ uint64_t nsec, nsecIO;
+ uint64_t compBytes;
+ uint64_t lastProgressStamp = 0;
+ uint64_t progressStamp;
+ IOPolledFileCryptVars * cryptvars = NULL;
+
+ IOHibernateVars * vars = &gIOHibernateVars;
+ bzero(gIOHibernateStats, sizeof(hibernate_statistics_t));
+
+ if (!vars->fileVars || !vars->fileVars->pollers) {
+ return;
+ }
+
+ sum = gIOHibernateCurrentHeader->actualImage1Sum;
+ pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
+
+ if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) {
+ HIBLOG("regular wake\n");
+ return;
+ }
- IOHibernateVars * vars = &gIOHibernateVars;
+ HIBPRINT("diag %x %x %x %x\n",
+ gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
+ gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
+
+#if defined(__i386__) || defined(__x86_64__)
+#define t40ms(x) ((uint32_t)((tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000)))
+#else /* defined(__i386__) || defined(__x86_64__) */
+#define t40ms(x) x
+#endif /* defined(__i386__) || defined(__x86_64__) */
+#define tStat(x, y) gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y);
+ tStat(booterStart, booterStart);
+ gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart;
+ tStat(booterDuration0, booterTime0);
+ tStat(booterDuration1, booterTime1);
+ tStat(booterDuration2, booterTime2);
+ tStat(booterDuration, booterTime);
+ tStat(booterConnectDisplayDuration, connectDisplayTime);
+ tStat(booterSplashDuration, splashTime);
+ tStat(trampolineDuration, trampolineTime);
+
+ gIOHibernateStats->image1Size = gIOHibernateCurrentHeader->image1Size;
+ gIOHibernateStats->imageSize = gIOHibernateCurrentHeader->imageSize;
+ gIOHibernateStats->image1Pages = pagesDone;
+
+ /* HIBERNATE_stats */
+ KDBG(IOKDBG_CODE(DBG_HIBERNATE, 14), gIOHibernateStats->smcStart,
+ gIOHibernateStats->booterStart, gIOHibernateStats->booterDuration,
+ gIOHibernateStats->trampolineDuration);
+
+ HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n",
+ gIOHibernateStats->booterStart,
+ gIOHibernateStats->smcStart,
+ gIOHibernateStats->booterDuration0,
+ gIOHibernateStats->booterDuration1,
+ gIOHibernateStats->booterDuration2,
+ gIOHibernateStats->booterDuration,
+ gIOHibernateStats->booterConnectDisplayDuration,
+ gIOHibernateStats->booterSplashDuration,
+ gIOHibernateStats->trampolineDuration);
+
+ HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, imageSize 0x%qx, image1Size 0x%qx, conflictCount %d, nextFree %x\n",
+ gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size,
+ gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
- if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
- return;
+ if ((0 != (kIOHibernateModeSleep & gIOHibernateMode))
+ && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))) {
+ hibernate_page_list_discard(vars->page_list);
+ }
- if ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)
- hibernate_page_list_discard(vars->page_list);
+ if (vars->hwEncrypt) {
+ // if vars->hwEncrypt is true, we don't need cryptvars since we supply the
+ // decryption key via IOPolledFilePollersSetEncryptionKey
+ cryptvars = NULL;
+ } else {
+ cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : NULL;
+ }
+ if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount) {
+ panic("handoff overflow");
+ }
- sum = gIOHibernateCurrentHeader->actualImage1Sum;
- pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
+ IOHibernateHandoff * handoff;
+ bool done = false;
+ bool foundCryptData = false;
+ bool foundVolumeEncryptData = false;
+ const uint8_t * handoffStart = (const uint8_t*)vars->handoffBuffer->getBytesNoCopy();
+ const uint8_t * handoffEnd = handoffStart + vars->handoffBuffer->getLength();
+
+ for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
+ !done;
+ handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount]) {
+ if (((uint8_t*)handoff < handoffStart) ||
+ (&handoff->data[handoff->bytecount] > handoffEnd)) {
+ panic("handoff out of range");
+ }
+// HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
+ uint8_t * data = &handoff->data[0];
+ switch (handoff->type) {
+ case kIOHibernateHandoffTypeEnd:
+ done = true;
+ break;
+
+ case kIOHibernateHandoffTypeGraphicsInfo:
+ if (handoff->bytecount == sizeof(*gIOHibernateGraphicsInfo)) {
+ bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
+ }
+ break;
+
+ case kIOHibernateHandoffTypeCryptVars:
+ if (cryptvars) {
+ hibernate_cryptwakevars_t *
+ wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
+ if (handoff->bytecount == sizeof(*wakevars)) {
+ bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
+ } else {
+ panic("kIOHibernateHandoffTypeCryptVars(%d)", handoff->bytecount);
+ }
+ }
+ foundCryptData = true;
+ bzero(data, handoff->bytecount);
+ break;
+
+ case kIOHibernateHandoffTypeVolumeCryptKey:
+ if (handoff->bytecount == vars->volumeCryptKeySize) {
+ bcopy(data, &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
+ foundVolumeEncryptData = true;
+ } else {
+ panic("kIOHibernateHandoffTypeVolumeCryptKey(%d)", handoff->bytecount);
+ }
+ break;
+
+#if defined(__i386__) || defined(__x86_64__)
+ case kIOHibernateHandoffTypeMemoryMap:
+
+ clock_get_uptime(&allTime);
+
+ hibernate_newruntime_map(data, handoff->bytecount,
+ gIOHibernateCurrentHeader->systemTableOffset);
+
+ clock_get_uptime(&endTime);
+
+ SUB_ABSOLUTETIME(&endTime, &allTime);
+ absolutetime_to_nanoseconds(endTime, &nsec);
+
+ HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL);
+
+ break;
+
+ case kIOHibernateHandoffTypeDeviceTree:
+ {
+// DTEntry chosen = NULL;
+// HIBPRINT("SecureDTLookupEntry %d\n", SecureDTLookupEntry((const DTEntry) data, "/chosen", &chosen));
+ }
+ break;
+#endif /* defined(__i386__) || defined(__x86_64__) */
- HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, image1Size %qx, conflictCount %d, nextFree %x\n",
- gIOHibernateState, pagesDone, sum, gIOHibernateCurrentHeader->image1Size,
- gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
+ default:
+ done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
+ break;
+ }
+ }
- if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
- {
- HIBLOG("regular wake\n");
- return;
- }
-
- HIBPRINT("diag %x %x %x %x\n",
- gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
- gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
-
- HIBPRINT("video %x %d %d %d\n",
- gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
- gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height);
-
- if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress)
- {
- vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
- * gIOHibernateGraphicsInfo->rowBytes);
- IOMapPages(kernel_map,
- vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
- vars->videoMapSize, kIOMapInhibitCache );
- }
-
- uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();;
- uint32_t decoOffset;
-
- clock_get_uptime(&allTime);
-
- HIBLOG("IOHibernatePollerOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
- err = IOHibernatePollerOpen(vars->fileVars, kIOPolledAfterSleepState, 0);
- HIBLOG("IOHibernatePollerOpen(%x)\n", err);
-
- if (gIOHibernateCurrentHeader->previewSize)
- progressZeroPosition = gIOHibernateCurrentHeader->previewSize
- + gIOHibernateCurrentHeader->fileExtentMapSize
- - sizeof(gIOHibernateCurrentHeader->fileExtentMap)
- + ptoa_64(gIOHibernateCurrentHeader->restore1PageCount);
-
- IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
-
- if (vars->videoMapping)
- {
- lastBlob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
- / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
- ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, 0, lastBlob);
- }
-
- cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
- if (kIOHibernateModeEncrypt & gIOHibernateMode)
- {
- cryptvars = &gIOHibernateCryptWakeContext;
- bcopy(&gIOHibernateCryptWakeVars->aes_iv[0],
- &cryptvars->aes_iv[0],
- sizeof(cryptvars->aes_iv));
- }
-
- // kick off the read ahead
- vars->fileVars->io = false;
- vars->fileVars->bufferHalf = 0;
- vars->fileVars->bufferLimit = 0;
- vars->fileVars->lastRead = 0;
- vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
-
- IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
- vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
- // --
-
- HIBLOG("hibernate_machine_init reading\n");
-
- uint32_t * header = (uint32_t *) src;
- sum = 0;
-
- do
- {
- unsigned int count;
- unsigned int page;
- uint32_t tag;
- vm_offset_t ppnum, compressedSize;
-
- IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
-
- ppnum = header[0];
- count = header[1];
+ if (vars->hwEncrypt && !foundVolumeEncryptData) {
+ panic("no volumeCryptKey");
+ } else if (cryptvars && !foundCryptData) {
+ panic("hibernate handoff");
+ }
+
+ HIBPRINT("video 0x%llx %d %d %d status %x\n",
+ gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
+ gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus);
+
+ if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress) {
+ vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
+ * gIOHibernateGraphicsInfo->rowBytes);
+ if (vars->videoMapSize > vars->videoAllocSize) {
+ vars->videoMapSize = 0;
+ } else {
+ IOMapPages(kernel_map,
+ vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
+ vars->videoMapSize, kIOMapInhibitCache );
+ }
+ }
+
+ if (vars->videoMapSize) {
+ ProgressUpdate(gIOHibernateGraphicsInfo,
+ (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
+ }
+
+ uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
+ uint8_t * compressed = src + page_size;
+ uint8_t * scratch = compressed + page_size;
+ uint32_t decoOffset;
+
+ clock_get_uptime(&allTime);
+ AbsoluteTime_to_scalar(&compTime) = 0;
+ compBytes = 0;
+
+ HIBLOG("IOPolledFilePollersOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
+ err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledAfterSleepState, false);
+ clock_get_uptime(&startIOTime);
+ endTime = startIOTime;
+ SUB_ABSOLUTETIME(&endTime, &allTime);
+ absolutetime_to_nanoseconds(endTime, &nsec);
+ HIBLOG("IOPolledFilePollersOpen(%x) %qd ms\n", err, nsec / 1000000ULL);
+
+ if (vars->hwEncrypt) {
+ err = IOPolledFilePollersSetEncryptionKey(vars->fileVars,
+ &vars->volumeCryptKey[0], vars->volumeCryptKeySize);
+ HIBLOG("IOPolledFilePollersSetEncryptionKey(%x) %ld\n", err, vars->volumeCryptKeySize);
+ if (kIOReturnSuccess != err) {
+ panic("IOPolledFilePollersSetEncryptionKey(0x%x)", err);
+ }
+ cryptvars = NULL;
+ }
+
+ IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
+
+ // kick off the read ahead
+ vars->fileVars->bufferHalf = 0;
+ vars->fileVars->bufferLimit = 0;
+ vars->fileVars->lastRead = 0;
+ vars->fileVars->readEnd = gIOHibernateCurrentHeader->imageSize;
+ vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
+ vars->fileVars->cryptBytes = 0;
+ AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;
+
+ err = IOPolledFileRead(vars->fileVars, NULL, 0, cryptvars);
+ if (kIOReturnSuccess != err) {
+ panic("Hibernate restore error %x", err);
+ }
+ vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
+ // --
+
+ HIBLOG("hibernate_machine_init reading\n");
+
+
+ uint32_t * header = (uint32_t *) src;
+ sum = 0;
+
+ while (kIOReturnSuccess == err) {
+ unsigned int count;
+ unsigned int page;
+ uint32_t tag;
+ vm_offset_t compressedSize;
+ ppnum_t ppnum;
+
+ err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
+ if (kIOReturnSuccess != err) {
+ panic("Hibernate restore error %x", err);
+ }
+
+ ppnum = header[0];
+ count = header[1];
// HIBPRINT("(%x, %x)\n", ppnum, count);
- if (!count)
- break;
-
- for (page = 0; page < count; page++)
- {
- IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
-
- compressedSize = kIOHibernateTagLength & tag;
- if (!compressedSize)
- {
- ppnum++;
- pagesDone++;
- continue;
- }
-
- IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
-
- if (compressedSize != page_size)
- {
- decoOffset = page_size;
- WKdm_decompress((WK_word*) src, (WK_word*) (src + decoOffset), PAGE_SIZE_IN_WORDS);
- }
- else
- decoOffset = 0;
-
- sum += hibernate_sum((src + decoOffset), page_size);
-
- err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
- if (err)
- HIBLOG("IOMemoryDescriptorReadToPhysical [%d] %x\n", ppnum, err);
-
- ppnum++;
- pagesDone++;
-
- if (vars->videoMapping && (0 == (255 & pagesDone)))
- {
- blob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
- / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
- if (blob != lastBlob)
- {
- ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, lastBlob, blob);
- lastBlob = blob;
- }
- }
-
- if (0 == (8191 & pagesDone))
- {
- clock_get_uptime(&endTime);
- SUB_ABSOLUTETIME(&endTime, &allTime);
- absolutetime_to_nanoseconds(endTime, &nsec);
- progressStamp = nsec / 750000000ULL;
- if (progressStamp != lastProgressStamp)
- {
- lastProgressStamp = progressStamp;
- HIBPRINT("pages %d (%d%%)\n", pagesDone,
- (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
+ if (!count) {
+ break;
+ }
+
+ for (page = 0; page < count; page++) {
+ err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
+ if (kIOReturnSuccess != err) {
+ panic("Hibernate restore error %x", err);
+ }
+
+ compressedSize = kIOHibernateTagLength & tag;
+ if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength)) {
+ err = kIOReturnIPCError;
+ panic("Hibernate restore error %x", err);
+ }
+
+ err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
+ if (kIOReturnSuccess != err) {
+ panic("Hibernate restore error %x", err);
+ }
+
+ if (compressedSize < page_size) {
+ decoOffset = ((uint32_t) page_size);
+ clock_get_uptime(&startTime);
+
+ if (compressedSize == 4) {
+ int i;
+ uint32_t *s, *d;
+
+ s = (uint32_t *)src;
+ d = (uint32_t *)(uintptr_t)compressed;
+
+ for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++) {
+ *d++ = *s;
+ }
+ } else {
+ pal_hib_decompress_page(src, compressed, scratch, ((unsigned int) compressedSize));
+ }
+ clock_get_uptime(&endTime);
+ ADD_ABSOLUTETIME(&compTime, &endTime);
+ SUB_ABSOLUTETIME(&compTime, &startTime);
+ compBytes += page_size;
+ } else {
+ decoOffset = 0;
+ }
+
+ sum += hibernate_sum_page((src + decoOffset), ((uint32_t) ppnum));
+ err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
+ if (err) {
+ HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
+ panic("Hibernate restore error %x", err);
+ }
+
+
+ ppnum++;
+ pagesDone++;
+ pagesRead++;
+
+ if (0 == (8191 & pagesDone)) {
+ clock_get_uptime(&endTime);
+ SUB_ABSOLUTETIME(&endTime, &allTime);
+ absolutetime_to_nanoseconds(endTime, &nsec);
+ progressStamp = nsec / 750000000ULL;
+ if (progressStamp != lastProgressStamp) {
+ lastProgressStamp = progressStamp;
+ HIBPRINT("pages %d (%d%%)\n", pagesDone,
+ (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
+ }
+ }
}
- }
}
- }
- while (true);
+ if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages)) {
+ err = kIOReturnLockedRead;
+ }
+
+ if (kIOReturnSuccess != err) {
+ panic("Hibernate restore error %x", err);
+ }
+
+
+ gIOHibernateCurrentHeader->actualImage2Sum = sum;
+ gIOHibernateCompression = gIOHibernateCurrentHeader->compression;
- gIOHibernateCurrentHeader->actualImage2Sum = sum;
+ clock_get_uptime(&endIOTime);
- if (vars->fileVars->io)
- (void) IOHibernatePollerIODone(vars->fileVars);
+ err = IOPolledFilePollersClose(vars->fileVars, kIOPolledAfterSleepState);
- err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState);
+ clock_get_uptime(&endTime);
- if (vars->videoMapping)
- ProgressUpdate(gIOHibernateGraphicsInfo,
- (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
+ IOService::getPMRootDomain()->pmStatsRecordEvent(
+ kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
+ IOService::getPMRootDomain()->pmStatsRecordEvent(
+ kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
- clock_get_uptime(&endTime);
- SUB_ABSOLUTETIME(&endTime, &allTime);
- absolutetime_to_nanoseconds(endTime, &nsec);
+ SUB_ABSOLUTETIME(&endTime, &allTime);
+ absolutetime_to_nanoseconds(endTime, &nsec);
- HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %qd ms\n",
- pagesDone, sum, nsec / 1000000ULL);
+ SUB_ABSOLUTETIME(&endIOTime, &startIOTime);
+ absolutetime_to_nanoseconds(endIOTime, &nsecIO);
+
+ gIOHibernateStats->kernelImageReadDuration = ((uint32_t) (nsec / 1000000ULL));
+ gIOHibernateStats->imagePages = pagesDone;
+
+ HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ",
+ pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize,
+ nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0);
+
+ absolutetime_to_nanoseconds(compTime, &nsec);
+ HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
+ compBytes,
+ nsec / 1000000ULL,
+ nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+ absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
+ HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n",
+ vars->fileVars->cryptBytes,
+ nsec / 1000000ULL,
+ nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
+
+ KDBG(IOKDBG_CODE(DBG_HIBERNATE, 2), pagesRead, pagesDone);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+void
+IOHibernateSetWakeCapabilities(uint32_t capability)
+{
+ if (kIOHibernateStateWakingFromHibernate == gIOHibernateState) {
+ gIOHibernateStats->wakeCapability = capability;
+
+ if (kIOPMSystemCapabilityGraphics & capability) {
+ vm_compressor_do_warmup();
+ }
+ }
}
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+void
+IOHibernateSystemRestart(void)
+{
+ static uint8_t noteStore[32] __attribute__((aligned(32)));
+ IORegistryEntry * regEntry;
+ const OSSymbol * sym;
+ OSData * noteProp;
+ OSData * data;
+ uintptr_t * smcVars;
+ uint8_t * smcBytes;
+ size_t len;
+ addr64_t element;
+
+ data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
+ if (!data) {
+ return;
+ }
+
+ smcVars = (typeof(smcVars))data->getBytesNoCopy();
+ smcBytes = (typeof(smcBytes))smcVars[1];
+ len = smcVars[0];
+ if (len > sizeof(noteStore)) {
+ len = sizeof(noteStore);
+ }
+ noteProp = OSData::withCapacity(3 * sizeof(element));
+ if (!noteProp) {
+ return;
+ }
+ element = len;
+ noteProp->appendBytes(&element, sizeof(element));
+ element = crc32(0, smcBytes, len);
+ noteProp->appendBytes(&element, sizeof(element));
+
+ bcopy(smcBytes, noteStore, len);
+ element = (addr64_t) ¬eStore[0];
+ element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
+ noteProp->appendBytes(&element, sizeof(element));
+
+ if (!gIOOptionsEntry) {
+ regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
+ gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
+ if (regEntry && !gIOOptionsEntry) {
+ regEntry->release();
+ }
+ }
+
+ sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
+ if (gIOOptionsEntry && sym) {
+ gIOOptionsEntry->setProperty(sym, noteProp);
+ }
+ if (noteProp) {
+ noteProp->release();
+ }
+ if (sym) {
+ sym->release();
+ }
+}
projects / apple / xnu.git / blobdiff