]> git.saurik.com Git - apple/xnu.git/blame - iokit/Kernel/IOHibernateIO.cpp
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-3789.31.2.tar.gz
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
CommitLineData
3a60a9f5 1/*
39037602 2 * Copyright (c) 2004-2016 Apple Inc. All rights reserved.
3a60a9f5 3 *
2d21ac55 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
39037602 5 *
2d21ac55
A		 6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
39037602 14 *
2d21ac55
A		 15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
39037602 17 *
2d21ac55
A		 18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5
A		 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55
A		 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
39037602 25 *
2d21ac55 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
3a60a9f5
A		 27 */
28
3a60a9f5
A		 29/*
30
31Sleep:
32
33- PMRootDomain calls IOHibernateSystemSleep() before system sleep
34(devices awake, normal execution context)
35- IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
36 grabs its extents and searches for a polling driver willing to work with that IOMedia.
37 The BSD code makes an ioctl to the storage driver to get the partition base offset to
38 the disk, and other ioctls to get the transfer constraints
39 If successful, the file is written to make sure its initially not bootable (in case of
40 later failure) and nvram set to point to the first block of the file. (Has to be done
41 here so blocking is possible in nvram support).
42 hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
43 page out any pages it wants to (currently zero, but probably some percentage of memory).
44 Its assumed just allocating pages will cause the VM system to naturally select the best
45 pages for eviction. It also copies processor flags needed for the restore path and sets
46 a flag in the boot processor proc info.
47 gIOHibernateState = kIOHibernateStateHibernating.
48- Regular sleep progresses - some drivers may inspect the root domain property
49 kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
50 as usual but leaves motherboard I/O on.
51- Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
52 at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
53 all ppc RC bits out of the hash table and caches into the mapping structures.
54- hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
55 hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
56 All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
57 by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
58 The image header and block list are written. The header includes the second file extent so
59 only the header block is needed to read the file, regardless of filesystem.
316670eb 60 The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data
3a60a9f5
A		 61 (only) is used to decompress the image during wake/boot.
62 Some additional pages are removed from the bitmaps - the buffers used for hibernation.
63 The bitmaps are written to the image.
64 More areas are removed from the bitmaps (after they have been written to the image) - the
316670eb 65 segment "__HIB" pages and interrupt stack.
3a60a9f5
A		 66 Each wired page is compressed and written and then each non-wired page. Compression and
67 disk writes are in parallel.
68 The image header is written to the start of the file and the polling driver closed.
69 The machine powers down (or sleeps).
70
71Boot/Wake:
72
73- BootX sees the boot-image nvram variable containing the device and block number of the image,
74 reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
75- BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
76 in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
77 that is in the kernel's __HIB section.
78- BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
79 hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
80 only code & data in that section is safe to call since all the other wired pages are still
81 compressed in the image.
82- hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
83 It uses the bitmaps to work out which pages can be uncompressed from the image to their final
84 location directly, and copies those that can't to interim free pages. When the image has been
85 completed, the copies are uncompressed, overwriting the wired image pages.
86 hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
87 is used to get pages into place for 64bit.
88- the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
89 changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
90 are removed from the software strutures, and the hash table is reinitialized.
91- After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
92 of memory, using the polled mode driver, before other threads can run or any devices are turned on.
93 This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
94 for the remaining non wired pages.
95- The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
96 (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
97 is closed via bsd.
98
99Polled Mode I/O:
100
101IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
102registry, specifying an object of calls IOPolledInterface.
103
104Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
105partition) that the image is going to live, looking for polled interface properties. If it finds
106one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
107interfaces found are kept in an ordered list.
108
109There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
110few different contexts things happen in:
111
112- there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
113up and running) and after wake - this is safe to allocate memory and do anything. The device
114ignores sleep requests from that point since its a waste of time if it goes to sleep and
115immediately wakes back up for the image write.
116
117- there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
118immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
119from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
120used to flush and set the disk to sleep.
121
122- there is an Open/Close (AfterSleep) pair made around the image read operations that happen
123immediately after sleep. These can't block or allocate memory. This is happening after the platform
124expert has woken the low level bits of the system, but most of the I/O system has not. There is only
125one thread running.
126
127For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
128(prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
129the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
130Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
131that is called for the hardware to check for events, and complete the I/O via the callback.
132The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
133to restrict I/O ops.
134*/
135
136#include <sys/systm.h>
137
138#include <IOKit/IOWorkLoop.h>
139#include <IOKit/IOCommandGate.h>
140#include <IOKit/IOTimerEventSource.h>
141#include <IOKit/IOPlatformExpert.h>
142#include <IOKit/IOKitDebug.h>
143#include <IOKit/IOTimeStamp.h>
144#include <IOKit/pwr_mgt/RootDomain.h>
145#include <IOKit/pwr_mgt/IOPMPrivate.h>
146#include <IOKit/IOMessage.h>
147#include <IOKit/IODeviceTreeSupport.h>
148#include <IOKit/IOBSD.h>
149#include "RootDomainUserClient.h"
150#include <IOKit/pwr_mgt/IOPowerConnection.h>
151#include "IOPMPowerStateQueue.h"
152#include <IOKit/IOBufferMemoryDescriptor.h>
6d2010ae 153#include <IOKit/AppleKeyStoreInterface.h>
316670eb 154#include <libkern/crypto/aes.h>
3a60a9f5
A		 155
156#include <sys/uio.h>
157#include <sys/conf.h>
158#include <sys/stat.h>
159#include <sys/fcntl.h> // (FWRITE, ...)
3a60a9f5 160#include <sys/sysctl.h>
6d2010ae 161#include <sys/kdebug.h>
fe8ab488 162#include <stdint.h>
3a60a9f5
A		 163
164#include <IOKit/IOHibernatePrivate.h>
165#include <IOKit/IOPolledInterface.h>
166#include <IOKit/IONVRAM.h>
167#include "IOHibernateInternal.h"
39236c6e 168#include <vm/WKdm_new.h>
39037602 169#include <vm/vm_protos.h>
3a60a9f5 170#include "IOKitKernelInternal.h"
6d2010ae
A		 171#include <pexpert/device_tree.h>
172
173#include <machine/pal_routines.h>
174#include <machine/pal_hibernate.h>
39236c6e 175#include <i386/tsc.h>
39037602 176#include <i386/cpuid.h>
6d2010ae
A		 177
178extern "C" addr64_t kvtophys(vm_offset_t va);
99c3a104 179extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
3a60a9f5
A		 180
181/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
182
39236c6e 183#define DISABLE_TRIM 0
39037602 184#define TRIM_DELAY 25000
39236c6e 185
0b4c1975 186extern unsigned int save_kdebug_enable;
3a60a9f5
A		 187extern uint32_t gIOHibernateState;
188uint32_t gIOHibernateMode;
189static char gIOHibernateBootSignature[256+1];
190static char gIOHibernateFilename[MAXPATHLEN+1];
db609669
A		 191static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
192uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms)
193static uint64_t gIOHibernateCompression = 0x80; // default compression 50%
39037602 194boolean_t gIOHibernateStandbyDisabled;
3a60a9f5
A		 195
196static IODTNVRAM * gIOOptionsEntry;
197static IORegistryEntry * gIOChosenEntry;
3e170ce0
A		 198
199static const OSSymbol * gIOHibernateBootImageKey;
200
b0d623f7 201#if defined(__i386__) || defined(__x86_64__)
3e170ce0 202
060df5ea
A		 203static const OSSymbol * gIOHibernateRTCVariablesKey;
204static const OSSymbol * gIOHibernateBoot0082Key;
205static const OSSymbol * gIOHibernateBootNextKey;
206static OSData * gIOHibernateBoot0082Data;
207static OSData * gIOHibernateBootNextData;
208static OSObject * gIOHibernateBootNextSave;
3e170ce0
A		 209
210static IOPolledFileIOVars * gDebugImageFileVars;
211static IOLock * gDebugImageLock;
212
213#endif /* defined(__i386__) || defined(__x86_64__) */
3a60a9f5 214
316670eb
A		 215static IOLock * gFSLock;
216static uint32_t gFSState;
3a60a9f5
A		 217static IOPolledFileIOVars gFileVars;
218static IOHibernateVars gIOHibernateVars;
3e170ce0 219static IOPolledFileCryptVars gIOHibernateCryptWakeContext;
6d2010ae
A		 220static hibernate_graphics_t _hibernateGraphics;
221static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
39236c6e
A		 222static hibernate_statistics_t _hibernateStats;
223static hibernate_statistics_t * gIOHibernateStats = &_hibernateStats;
3a60a9f5 224
316670eb
A		 225enum
226{
227 kFSIdle = 0,
228 kFSOpening = 2,
229 kFSOpened = 3,
230 kFSTimedOut = 4,
231};
232
233static IOReturn IOHibernateDone(IOHibernateVars * vars);
3e170ce0
A		 234static IOReturn IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature);
235static void IOSetBootImageNVRAM(OSData * data);
316670eb 236
3a60a9f5
A		 237/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
238
3a60a9f5 239enum { kDefaultIOSize = 128 * 1024 };
22ba694c 240enum { kVideoMapSize = 80 * 1024 * 1024 };
3a60a9f5 241
3a60a9f5
A		 242/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
243
244// copy from phys addr to MD
245
246static IOReturn
247IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
248 IOByteCount offset, addr64_t bytes, IOByteCount length)
249{
250 addr64_t srcAddr = bytes;
251 IOByteCount remaining;
252
253 remaining = length = min(length, md->getLength() - offset);
254 while (remaining) { // (process another target segment?)
255 addr64_t dstAddr64;
256 IOByteCount dstLen;
257
b0d623f7 258 dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
3a60a9f5
A		 259 if (!dstAddr64)
260 break;
261
262 // Clip segment length to remaining
263 if (dstLen > remaining)
264 dstLen = remaining;
265
266#if 1
267 bcopy_phys(srcAddr, dstAddr64, dstLen);
268#else
269 copypv(srcAddr, dstAddr64, dstLen,
270 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
271#endif
272 srcAddr += dstLen;
273 offset += dstLen;
274 remaining -= dstLen;
275 }
276
277 assert(!remaining);
278
279 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
280}
281
282// copy from MD to phys addr
283
284static IOReturn
285IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
286 IOByteCount offset, addr64_t bytes, IOByteCount length)
287{
288 addr64_t dstAddr = bytes;
289 IOByteCount remaining;
290
291 remaining = length = min(length, md->getLength() - offset);
292 while (remaining) { // (process another target segment?)
293 addr64_t srcAddr64;
294 IOByteCount dstLen;
295
b0d623f7 296 srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
3a60a9f5
A		 297 if (!srcAddr64)
298 break;
299
300 // Clip segment length to remaining
301 if (dstLen > remaining)
302 dstLen = remaining;
303
304#if 1
305 bcopy_phys(srcAddr64, dstAddr, dstLen);
306#else
307 copypv(srcAddr, dstAddr64, dstLen,
308 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
309#endif
310 dstAddr += dstLen;
311 offset += dstLen;
312 remaining -= dstLen;
313 }
314
315 assert(!remaining);
316
317 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
318}
319
320/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
321
322void
323hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
324 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
325{
326 count += ppnum;
327 switch (kind)
328 {
329 case kIOHibernatePageStateUnwiredSave:
330 // unwired save
331 for (; ppnum < count; ppnum++)
332 {
333 hibernate_page_bitset(page_list, FALSE, ppnum);
334 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
335 }
336 break;
337 case kIOHibernatePageStateWiredSave:
338 // wired save
339 for (; ppnum < count; ppnum++)
340 {
341 hibernate_page_bitset(page_list, FALSE, ppnum);
342 hibernate_page_bitset(page_list_wired, FALSE, ppnum);
343 }
344 break;
345 case kIOHibernatePageStateFree:
346 // free page
347 for (; ppnum < count; ppnum++)
348 {
349 hibernate_page_bitset(page_list, TRUE, ppnum);
350 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
351 }
352 break;
353 default:
354 panic("hibernate_set_page_state");
355 }
356}
357
3e170ce0
A		 358static vm_offset_t
359hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
360{
361 uint32_t page = *pPage;
362 uint32_t count;
363 hibernate_bitmap_t * bitmap;
fe8ab488 364
3e170ce0
A		 365 while ((bitmap = hibernate_page_bitmap_pin(list, &page)))
366 {
367 count = hibernate_page_bitmap_count(bitmap, TRUE, page);
368 if (!count)
369 break;
370 page += count;
371 if (page <= bitmap->last_page)
372 break;
fe8ab488 373 }
fe8ab488 374
3e170ce0
A		 375 *pPage = page;
376 if (bitmap)
377 count = hibernate_page_bitmap_count(bitmap, FALSE, page);
378 else
379 count = 0;
380
381 return (count);
fe8ab488 382}
fe8ab488
A		 383
384/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
385
3a60a9f5
A		 386IOReturn
387IOHibernateSystemSleep(void)
388{
389 IOReturn err;
3e170ce0 390 OSData * nvramData;
3a60a9f5
A		 391 OSObject * obj;
392 OSString * str;
db609669 393 OSNumber * num;
39037602 394 bool dsSSD, vmflush, swapPinned;
316670eb 395 IOHibernateVars * vars;
3e170ce0 396 uint64_t setFileSize = 0;
3a60a9f5
A		 397
398 gIOHibernateState = kIOHibernateStateInactive;
399
b0d623f7
A		 400 gIOHibernateDebugFlags = 0;
401 if (kIOLogHibernate & gIOKitDebug)
402 gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
403
0b4c1975
A		 404 if (IOService::getPMRootDomain()->getHibernateSettings(
405 &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime))
6d2010ae 406 {
3a60a9f5
A		 407 if (kIOHibernateModeSleep & gIOHibernateMode)
408 // default to discard clean for safe sleep
409 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
410 | kIOHibernateModeDiscardCleanActive);
6d2010ae 411 }
3a60a9f5 412
3a60a9f5
A		 413 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
414 {
415 if ((str = OSDynamicCast(OSString, obj)))
2d21ac55
A		 416 strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
417 sizeof(gIOHibernateFilename));
3a60a9f5
A		 418 obj->release();
419 }
420
421 if (!gIOHibernateMode || !gIOHibernateFilename[0])
422 return (kIOReturnUnsupported);
423
424 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
425
316670eb
A		 426 vars = IONew(IOHibernateVars, 1);
427 if (!vars) return (kIOReturnNoMemory);
428 bzero(vars, sizeof(*vars));
429
430 IOLockLock(gFSLock);
431 if (kFSIdle != gFSState)
432 {
433 HIBLOG("hibernate file busy\n");
434 IOLockUnlock(gFSLock);
435 IODelete(vars, IOHibernateVars, 1);
436 return (kIOReturnBusy);
437 }
438 gFSState = kFSOpening;
439 IOLockUnlock(gFSLock);
2d21ac55 440
39037602 441 swapPinned = false;
3a60a9f5
A		 442 do
443 {
0c530ab8 444 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
39037602 445 2 * page_size + WKdm_SCRATCH_BUF_SIZE_INTERNAL, page_size);
3a60a9f5 446
6d2010ae
A		 447 vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
448 ptoa_64(gIOHibernateHandoffPageCount), page_size);
449
3e170ce0 450 if (!vars->srcBuffer || !vars->handoffBuffer)
3a60a9f5
A		 451 {
452 err = kIOReturnNoMemory;
453 break;
454 }
455
db609669
A		 456 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey)))
457 {
458 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMinSize = num->unsigned64BitValue();
459 obj->release();
460 }
461 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey)))
462 {
463 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMaxSize = num->unsigned64BitValue();
464 obj->release();
465 }
466
467 boolean_t encryptedswap = true;
468 uint32_t pageCount;
469 AbsoluteTime startTime, endTime;
470 uint64_t nsec;
471
472 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
473 gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
6d2010ae 474 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
db609669 475
39037602 476 vmflush = ((kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
39236c6e
A		 477 err = hibernate_alloc_page_lists(&vars->page_list,
478 &vars->page_list_wired,
479 &vars->page_list_pal);
39037602
A		 480 if (KERN_SUCCESS != err) break;
481
482 err = hibernate_pin_swap(TRUE);
483 if (KERN_SUCCESS != err) break;
484 swapPinned = true;
db609669
A		 485
486 if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode))
487 {
488 hibernate_page_list_setall(vars->page_list,
489 vars->page_list_wired,
490 vars->page_list_pal,
491 true /* preflight */,
39236c6e 492 vmflush /* discard */,
db609669
A		 493 &pageCount);
494 PE_Video consoleInfo;
495 bzero(&consoleInfo, sizeof(consoleInfo));
496 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
497
8a3053a0 498 // estimate: 6% increase in pages compressed
fe8ab488 499 // screen preview 2 images compressed 0%
8a3053a0 500 setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8)
db609669 501 + vars->page_list->list_size
8a3053a0
A		 502 + (consoleInfo.v_width * consoleInfo.v_height * 8);
503 enum { setFileRound = 1024*1024ULL };
504 setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1));
fe8ab488 505
db609669
A		 506 HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n",
507 pageCount, (100ULL * gIOHibernateCompression) >> 8,
508 setFileSize, vars->fileMinSize);
509
510 if (!(kIOHibernateModeFileResize & gIOHibernateMode)
511 && (setFileSize < vars->fileMinSize))
512 {
513 setFileSize = vars->fileMinSize;
514 }
515 }
516
3e170ce0
A		 517 // Invalidate the image file
518 if (gDebugImageLock) {
519 IOLockLock(gDebugImageLock);
520 if (gDebugImageFileVars != 0) {
521 kprintf("IOHIBSystemSleep: Closing debugdata file\n");
522 IOSetBootImageNVRAM(0);
523 IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
524 }
525 IOLockUnlock(gDebugImageLock);
526 }
db609669 527
3e170ce0
A		 528 err = IOPolledFileOpen(gIOHibernateFilename, setFileSize, 0,
529 gIOHibernateCurrentHeader, sizeof(gIOHibernateCurrentHeader),
530 &vars->fileVars, &nvramData,
531 &vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
fe8ab488 532
3e170ce0
A		 533 if (KERN_SUCCESS != err)
534 {
535 IOLockLock(gFSLock);
536 if (kFSOpening != gFSState) err = kIOReturnTimeout;
537 IOLockUnlock(gFSLock);
538 }
8a3053a0 539
3a60a9f5
A		 540 if (KERN_SUCCESS != err)
541 {
542 HIBLOG("IOPolledFileOpen(%x)\n", err);
543 break;
544 }
3a60a9f5 545
3e170ce0
A		 546 // write extents for debug data usage in EFI
547 IOWriteExtentsToFile(vars->fileVars, kIOHibernateHeaderOpenSignature);
548
549 err = IOPolledFilePollersSetup(vars->fileVars, kIOPolledPreflightState);
550 if (KERN_SUCCESS != err) break;
551
39236c6e
A		 552 clock_get_uptime(&startTime);
553 err = hibernate_setup(gIOHibernateCurrentHeader,
39236c6e
A		 554 vmflush,
555 vars->page_list, vars->page_list_wired, vars->page_list_pal);
556 clock_get_uptime(&endTime);
557 SUB_ABSOLUTETIME(&endTime, &startTime);
558 absolutetime_to_nanoseconds(endTime, &nsec);
39037602
A		 559
560 boolean_t haveSwapPin, hibFileSSD;
561 haveSwapPin = vm_swap_files_pinned();
562
563 hibFileSSD = (kIOPolledFileSSD & vars->fileVars->flags);
564
565 HIBLOG("hibernate_setup(%d) took %qd ms, swapPin(%d) ssd(%d)\n",
566 err, nsec / 1000000ULL,
567 haveSwapPin, hibFileSSD);
3e170ce0 568 if (KERN_SUCCESS != err) break;
39236c6e 569
39037602
A		 570 gIOHibernateStandbyDisabled = ((!haveSwapPin || !hibFileSSD));
571
3e170ce0 572 dsSSD = ((0 != (kIOPolledFileSSD & vars->fileVars->flags))
0b4c1975 573 && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
3e170ce0
A		 574
575 if (dsSSD) gIOHibernateCurrentHeader->options |= kIOHibernateOptionSSD | kIOHibernateOptionColor;
576 else gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
577
6d2010ae
A		 578
579#if defined(__i386__) || defined(__x86_64__)
3e170ce0
A		 580 if (!uuid_is_null(vars->volumeCryptKey) &&
581 (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey)))
582 {
583 uintptr_t smcVars[2];
584 smcVars[0] = sizeof(vars->volumeCryptKey);
585 smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
6d2010ae 586
3e170ce0
A		 587 IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
588 bzero(smcVars, sizeof(smcVars));
589 }
6d2010ae 590#endif
0b4c1975 591
0b4c1975 592
6d2010ae 593 if (encryptedswap || !uuid_is_null(vars->volumeCryptKey))
3a60a9f5
A		 594 gIOHibernateMode ^= kIOHibernateModeEncrypt;
595
0b4c1975
A		 596 if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
597 {
598 vars->videoAllocSize = kVideoMapSize;
3e170ce0 599 if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize, VM_KERN_MEMORY_IOKIT))
0b4c1975
A		 600 vars->videoMapping = 0;
601 }
3a60a9f5
A		 602
603 // generate crypt keys
604 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
605 vars->wiredCryptKey[i] = random();
606 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
607 vars->cryptKey[i] = random();
608
609 // set nvram
610
3e170ce0
A		 611 IOSetBootImageNVRAM(nvramData);
612 nvramData->release();
3a60a9f5 613
b0d623f7 614#if defined(__i386__) || defined(__x86_64__)
3e170ce0 615 {
0c530ab8
A		 616 struct AppleRTCHibernateVars
617 {
618 uint8_t signature[4];
619 uint32_t revision;
620 uint8_t booterSignature[20];
621 uint8_t wiredCryptKey[16];
622 };
623 AppleRTCHibernateVars rtcVars;
3e170ce0 624 OSData * data;
0c530ab8
A		 625
626 rtcVars.signature[0] = 'A';
627 rtcVars.signature[1] = 'A';
628 rtcVars.signature[2] = 'P';
629 rtcVars.signature[3] = 'L';
630 rtcVars.revision = 1;
631 bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
632 if (gIOHibernateBootSignature[0])
633 {
634 char c;
635 uint8_t value = 0;
636 for (uint32_t i = 0;
637 (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1));
638 i++)
639 {
3e170ce0
A		 640 if (c >= 'a') c -= 'a' - 10;
641 else if (c >= 'A') c -= 'A' - 10;
642 else if (c >= '0') c -= '0';
643 else continue;
0c530ab8 644 value = (value << 4) | c;
3e170ce0 645 if (i & 1) rtcVars.booterSignature[i >> 1] = value;
0c530ab8
A		 646 }
647 }
648 data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
649 if (data)
2d21ac55 650 {
060df5ea
A		 651 if (gIOHibernateRTCVariablesKey)
652 IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
060df5ea 653 data->release();
0c530ab8
A		 654 }
655 if (gIOChosenEntry)
656 {
657 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
3e170ce0
A		 658 if (data) gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
659 // set BootNext
660 if (!gIOHibernateBoot0082Data)
060df5ea 661 {
3e170ce0
A		 662 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
663 if (data)
060df5ea 664 {
3e170ce0
A		 665 // AppleNVRAM_EFI_LOAD_OPTION
666 struct {
667 uint32_t Attributes;
668 uint16_t FilePathLength;
669 uint16_t Desc;
670 } loadOptionHeader;
671 loadOptionHeader.Attributes = 1;
672 loadOptionHeader.FilePathLength = data->getLength();
673 loadOptionHeader.Desc = 0;
674 gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
675 if (gIOHibernateBoot0082Data)
060df5ea 676 {
3e170ce0
A		 677 gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
678 gIOHibernateBoot0082Data->appendBytes(data);
060df5ea
A		 679 }
680 }
3e170ce0
A		 681 }
682 if (!gIOHibernateBootNextData)
683 {
684 uint16_t bits = 0x0082;
685 gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
686 }
687 if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData)
688 {
689 gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
690 gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
691 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
692 }
693 // BootNext
3a60a9f5
A		 694 }
695 }
3e170ce0 696#endif /* !i386 && !x86_64 */
316670eb
A		 697 }
698 while (false);
699
39037602
A		 700 if (swapPinned) hibernate_pin_swap(FALSE);
701
316670eb 702 IOLockLock(gFSLock);
3e170ce0
A		 703 if ((kIOReturnSuccess == err) && (kFSOpening != gFSState))
704 {
705 HIBLOG("hibernate file close due timeout\n");
706 err = kIOReturnTimeout;
707 }
708 if (kIOReturnSuccess == err)
316670eb
A		 709 {
710 gFSState = kFSOpened;
711 gIOHibernateVars = *vars;
712 gFileVars = *vars->fileVars;
3e170ce0 713 gFileVars.allocated = false;
316670eb 714 gIOHibernateVars.fileVars = &gFileVars;
3a60a9f5
A		 715 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
716 gIOHibernateState = kIOHibernateStateHibernating;
717 }
316670eb
A		 718 else
719 {
3e170ce0 720 IOPolledFileIOVars * fileVars = vars->fileVars;
316670eb 721 IOHibernateDone(vars);
3e170ce0
A		 722 IOPolledFileClose(&fileVars,
723#if DISABLE_TRIM
724 0, NULL, 0, 0, 0);
725#else
726 0, NULL, 0, sizeof(IOHibernateImageHeader), setFileSize);
727#endif
316670eb
A		 728 gFSState = kFSIdle;
729 }
730 IOLockUnlock(gFSLock);
731
732 if (vars->fileVars) IODelete(vars->fileVars, IOPolledFileIOVars, 1);
733 IODelete(vars, IOHibernateVars, 1);
3a60a9f5
A		 734
735 return (err);
736}
737
738/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
739
3e170ce0
A		 740static void
741IOSetBootImageNVRAM(OSData * data)
742{
743 IORegistryEntry * regEntry;
744
745 if (!gIOOptionsEntry)
746 {
747 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
748 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
749 if (regEntry && !gIOOptionsEntry)
750 regEntry->release();
751 }
752 if (gIOOptionsEntry && gIOHibernateBootImageKey)
753 {
754 if (data) gIOOptionsEntry->setProperty(gIOHibernateBootImageKey, data);
755 else
756 {
757 gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
758 gIOOptionsEntry->sync();
759 }
760 }
761}
762
763/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
764/*
765 * Writes header to disk with signature, block size and file extents data.
766 * If there are more than 2 extents, then they are written on second block.
767 */
768static IOReturn
769IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature)
770{
771 IOHibernateImageHeader hdr;
772 IOItemCount count;
773 IOReturn err = kIOReturnSuccess;
774 int rc;
775 IOPolledFileExtent * fileExtents;
776
39037602 777 fileExtents = (typeof(fileExtents)) vars->fileExtents->getBytesNoCopy();
3e170ce0
A		 778
779 memset(&hdr, 0, sizeof(IOHibernateImageHeader));
780 count = vars->fileExtents->getLength();
781 if (count > sizeof(hdr.fileExtentMap))
782 {
783 hdr.fileExtentMapSize = count;
784 count = sizeof(hdr.fileExtentMap);
785 }
786 else
787 hdr.fileExtentMapSize = sizeof(hdr.fileExtentMap);
788
789 bcopy(fileExtents, &hdr.fileExtentMap[0], count);
790
791 // copy file block extent list if larger than header
792 if (hdr.fileExtentMapSize > sizeof(hdr.fileExtentMap))
793 {
794 count = hdr.fileExtentMapSize - sizeof(hdr.fileExtentMap);
795 rc = kern_write_file(vars->fileRef, vars->blockSize,
796 (caddr_t)(((uint8_t *)fileExtents) + sizeof(hdr.fileExtentMap)),
797 count, IO_SKIP_ENCRYPTION);
798 if (rc != 0) {
799 HIBLOG("kern_write_file returned %d\n", rc);
800 err = kIOReturnIOError;
801 goto exit;
802 }
803 }
804 hdr.signature = signature;
805 hdr.deviceBlockSize = vars->blockSize;
806
807 rc = kern_write_file(vars->fileRef, 0, (char *)&hdr, sizeof(hdr), IO_SKIP_ENCRYPTION);
808 if (rc != 0) {
809 HIBLOG("kern_write_file returned %d\n", rc);
810 err = kIOReturnIOError;
811 goto exit;
812 }
813
814exit:
815 return err;
816}
817
39037602
A		 818extern "C" boolean_t root_is_CF_drive;
819
3e170ce0
A		 820void
821IOOpenDebugDataFile(const char *fname, uint64_t size)
822{
823 IOReturn err;
824 OSData * imagePath = NULL;
825 uint64_t padding;
826
827 if (!gDebugImageLock) {
828 gDebugImageLock = IOLockAlloc();
829 }
830
39037602
A		 831 if (root_is_CF_drive) return;
832
3e170ce0
A		 833 // Try to get a lock, but don't block for getting lock
834 if (!IOLockTryLock(gDebugImageLock)) {
835 HIBLOG("IOOpenDebugDataFile: Failed to get lock\n");
836 return;
837 }
838
839 if (gDebugImageFileVars || !fname || !size) {
840 HIBLOG("IOOpenDebugDataFile: conditions failed\n");
841 goto exit;
842 }
843
844 padding = (PAGE_SIZE*2); // allocate couple more pages for header and fileextents
845 err = IOPolledFileOpen(fname, size+padding, 32ULL*1024*1024*1024,
846 NULL, 0,
847 &gDebugImageFileVars, &imagePath, NULL, 0);
848
849 if ((kIOReturnSuccess == err) && imagePath)
850 {
851 if ((gDebugImageFileVars->fileSize < (size+padding)) ||
852 (gDebugImageFileVars->fileExtents->getLength() > PAGE_SIZE)) {
853 // Can't use the file
854 IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
855 HIBLOG("IOOpenDebugDataFile: too many file extents\n");
856 goto exit;
857 }
858
859 // write extents for debug data usage in EFI
860 IOWriteExtentsToFile(gDebugImageFileVars, kIOHibernateHeaderOpenSignature);
861 IOSetBootImageNVRAM(imagePath);
862 kprintf("IOOpenDebugDataFile: opened debugdata file\n");
863 }
864
865exit:
866 IOLockUnlock(gDebugImageLock);
867
868 if (imagePath) imagePath->release();
869 return;
870}
871
39037602
A		 872void
873IOCloseDebugDataFile()
874{
875 IOSetBootImageNVRAM(0);
876
877 if (gDebugImageLock) {
878 IOLockLock(gDebugImageLock);
879 if (gDebugImageFileVars != 0) {
880 kprintf("IOHibernateSystemPostWake: Closing debugdata file\n");
881 IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
882 }
883 IOLockUnlock(gDebugImageLock);
884 }
885
886
887}
888
3e170ce0
A		 889/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
890
2d21ac55
A		 891DECLARE_IOHIBERNATEPROGRESSALPHA
892
893static void
894ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
895{
896 uint32_t rowBytes, pixelShift;
897 uint32_t x, y;
898 int32_t blob;
899 uint32_t alpha, in, color, result;
900 uint8_t * out;
901 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
902
903 rowBytes = display->rowBytes;
904 pixelShift = display->depth >> 4;
905 if (pixelShift < 1) return;
906
907 screen += ((display->width
908 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
909 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
910
911 for (y = 0; y < kIOHibernateProgressHeight; y++)
912 {
913 out = screen + y * rowBytes;
914 for (blob = 0; blob < kIOHibernateProgressCount; blob++)
915 {
916 color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
917 for (x = 0; x < kIOHibernateProgressWidth; x++)
918 {
919 alpha = gIOHibernateProgressAlpha[y][x];
920 result = color;
921 if (alpha)
922 {
923 if (0xff != alpha)
924 {
925 if (1 == pixelShift)
926 {
927 in = *((uint16_t *)out) & 0x1f; // 16
928 in = (in << 3) | (in >> 2);
929 }
930 else
931 in = *((uint32_t *)out) & 0xff; // 32
932 saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
933 result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
934 }
935 if (1 == pixelShift)
936 {
937 result >>= 3;
938 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
939 }
940 else
941 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
942 }
943 out += (1 << pixelShift);
944 }
945 out += (kIOHibernateProgressSpacing << pixelShift);
946 }
947 }
948}
949
950
951static void
952ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
953{
954 uint32_t rowBytes, pixelShift;
955 uint32_t x, y;
956 int32_t blob, lastBlob;
957 uint32_t alpha, in, color, result;
958 uint8_t * out;
959 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
960
961 pixelShift = display->depth >> 4;
962 if (pixelShift < 1)
963 return;
964
965 rowBytes = display->rowBytes;
966
967 screen += ((display->width
968 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
969 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
970
971 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
972
973 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
974
975 for (y = 0; y < kIOHibernateProgressHeight; y++)
976 {
977 out = screen + y * rowBytes;
978 for (blob = firstBlob; blob <= lastBlob; blob++)
979 {
980 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
981 for (x = 0; x < kIOHibernateProgressWidth; x++)
982 {
983 alpha = gIOHibernateProgressAlpha[y][x];
984 result = color;
985 if (alpha)
986 {
987 if (0xff != alpha)
988 {
989 in = display->progressSaveUnder[blob][saveindex[blob]++];
990 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
991 }
992 if (1 == pixelShift)
993 {
994 result >>= 3;
995 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
996 }
997 else
998 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
999 }
1000 out += (1 << pixelShift);
1001 }
1002 out += (kIOHibernateProgressSpacing << pixelShift);
1003 }
1004 }
1005}
1006
1007/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1008
316670eb
A		 1009IOReturn
1010IOHibernateIOKitSleep(void)
1011{
1012 IOReturn ret = kIOReturnSuccess;
1013 IOLockLock(gFSLock);
1014 if (kFSOpening == gFSState)
1015 {
1016 gFSState = kFSTimedOut;
1017 HIBLOG("hibernate file open timed out\n");
1018 ret = kIOReturnTimeout;
1019 }
1020 IOLockUnlock(gFSLock);
1021 return (ret);
1022}
1023
1024/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1025
3a60a9f5
A		 1026IOReturn
1027IOHibernateSystemHasSlept(void)
1028{
316670eb 1029 IOReturn ret = kIOReturnSuccess;
3a60a9f5 1030 IOHibernateVars * vars = &gIOHibernateVars;
316670eb 1031 OSObject * obj = 0;
2d21ac55
A		 1032 OSData * data;
1033
316670eb
A		 1034 IOLockLock(gFSLock);
1035 if ((kFSOpened != gFSState) && gIOHibernateMode)
1036 {
1037 ret = kIOReturnTimeout;
1038 }
1039 IOLockUnlock(gFSLock);
1040 if (kIOReturnSuccess != ret) return (ret);
1041
1042 if (gIOHibernateMode) obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
2d21ac55
A		 1043 vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
1044 if (obj && !vars->previewBuffer)
1045 obj->release();
3a60a9f5 1046
2d21ac55
A		 1047 vars->consoleMapping = NULL;
1048 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
3a60a9f5 1049 {
2d21ac55
A		 1050 vars->previewBuffer->release();
1051 vars->previewBuffer = 0;
1052 }
3a60a9f5 1053
0b4c1975
A		 1054 if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1055 && vars->previewBuffer
1056 && (data = OSDynamicCast(OSData,
2d21ac55
A		 1057 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey))))
1058 {
1059 UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
b0d623f7 1060 HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
2d21ac55
A		 1061
1062 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
1063
1064 if (kIOHibernatePreviewUpdates & flags)
1065 {
1066 PE_Video consoleInfo;
1067 hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
1068
1069 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1070
1071 graphicsInfo->width = consoleInfo.v_width;
1072 graphicsInfo->height = consoleInfo.v_height;
1073 graphicsInfo->rowBytes = consoleInfo.v_rowBytes;
1074 graphicsInfo->depth = consoleInfo.v_depth;
1075 vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
1076
1077 HIBPRINT("video %p %d %d %d\n",
0b4c1975
A		 1078 vars->consoleMapping, graphicsInfo->depth,
1079 graphicsInfo->width, graphicsInfo->height);
2d21ac55
A		 1080 if (vars->consoleMapping)
1081 ProgressInit(graphicsInfo, vars->consoleMapping,
1082 &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
1083 }
3a60a9f5 1084 }
2d21ac55 1085
3a60a9f5
A		 1086 if (gIOOptionsEntry)
1087 gIOOptionsEntry->sync();
1088
316670eb 1089 return (ret);
3a60a9f5
A		 1090}
1091
1092/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1093
6d2010ae
A		 1094static DeviceTreeNode *
1095MergeDeviceTree(DeviceTreeNode * entry, IORegistryEntry * regEntry)
1096{
1097 DeviceTreeNodeProperty * prop;
1098 DeviceTreeNode * child;
1099 IORegistryEntry * childRegEntry;
1100 const char * nameProp;
1101 unsigned int propLen, idx;
1102
1103 prop = (DeviceTreeNodeProperty *) (entry + 1);
1104 for (idx = 0; idx < entry->nProperties; idx++)
1105 {
1106 if (regEntry && (0 != strcmp("name", prop->name)))
1107 {
1108 regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
1109// HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
1110 }
1111 prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
1112 }
1113
1114 child = (DeviceTreeNode *) prop;
1115 for (idx = 0; idx < entry->nChildren; idx++)
1116 {
1117 if (kSuccess != DTGetProperty(child, "name", (void **) &nameProp, &propLen))
1118 panic("no name");
1119 childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
1120// HIBPRINT("%s == %p\n", nameProp, childRegEntry);
1121 child = MergeDeviceTree(child, childRegEntry);
1122 }
1123 return (child);
1124}
1125
3a60a9f5
A		 1126IOReturn
1127IOHibernateSystemWake(void)
1128{
316670eb
A		 1129 if (kFSOpened == gFSState)
1130 {
3e170ce0 1131 IOPolledFilePollersClose(gIOHibernateVars.fileVars, kIOPolledPostflightState);
316670eb
A		 1132 IOHibernateDone(&gIOHibernateVars);
1133 }
1134 else
1135 {
1136 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1137 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1138 }
1139 return (kIOReturnSuccess);
1140}
3a60a9f5 1141
316670eb
A		 1142static IOReturn
1143IOHibernateDone(IOHibernateVars * vars)
1144{
99c3a104 1145 hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);
3a60a9f5
A		 1146
1147 if (vars->videoMapping)
1148 {
1149 if (vars->videoMapSize)
1150 // remove mappings
1151 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1152 if (vars->videoAllocSize)
1153 // dealloc range
b0d623f7 1154 kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
3a60a9f5
A		 1155 }
1156
1157 if (vars->previewBuffer)
1158 {
1159 vars->previewBuffer->release();
1160 vars->previewBuffer = 0;
1161 }
1162
0b4c1975
A		 1163 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1164 {
1165 IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey,
1166 gIOHibernateCurrentHeader->options, 32);
1167 }
1168 else
1169 {
1170 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1171 }
1172
1173 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1174 && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus))
1175 {
1176 IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey,
1177 &gIOHibernateGraphicsInfo->gfxStatus,
1178 sizeof(gIOHibernateGraphicsInfo->gfxStatus));
1179 }
1180 else
1181 {
1182 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1183 }
1184
3a60a9f5
A		 1185 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
1186
b0d623f7 1187#if defined(__i386__) || defined(__x86_64__)
060df5ea 1188 IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
6d2010ae 1189 IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
2d21ac55
A		 1190
1191 /*
1192 * Hibernate variable is written to NVRAM on platforms in which RtcRam
1193 * is not backed by coin cell. Remove Hibernate data from NVRAM.
1194 */
1195 if (gIOOptionsEntry) {
2d21ac55 1196
060df5ea
A		 1197 if (gIOHibernateRTCVariablesKey) {
1198 if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
1199 gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
1200 }
1201 }
1202
1203 if (gIOHibernateBootNextKey)
1204 {
1205 if (gIOHibernateBootNextSave)
1206 {
1207 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
1208 gIOHibernateBootNextSave->release();
1209 gIOHibernateBootNextSave = NULL;
2d21ac55 1210 }
060df5ea
A		 1211 else
1212 gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
1213 }
39236c6e 1214 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) gIOOptionsEntry->sync();
2d21ac55 1215 }
0c530ab8 1216#endif
3a60a9f5 1217
3e170ce0 1218 if (vars->srcBuffer) vars->srcBuffer->release();
6d2010ae 1219 bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
99c3a104 1220 if (vars->handoffBuffer)
6d2010ae 1221 {
99c3a104 1222 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
6d2010ae 1223 {
99c3a104
A		 1224 IOHibernateHandoff * handoff;
1225 bool done = false;
1226 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
1227 !done;
1228 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
6d2010ae 1229 {
99c3a104
A		 1230 HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
1231 uint8_t * data = &handoff->data[0];
1232 switch (handoff->type)
1233 {
1234 case kIOHibernateHandoffTypeEnd:
1235 done = true;
1236 break;
6d2010ae 1237
99c3a104
A		 1238 case kIOHibernateHandoffTypeDeviceTree:
1239 MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot());
1240 break;
1241
1242 case kIOHibernateHandoffTypeKeyStore:
6d2010ae 1243#if defined(__i386__) || defined(__x86_64__)
6d2010ae 1244 {
99c3a104
A		 1245 IOBufferMemoryDescriptor *
1246 md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
1247 if (md)
1248 {
1249 IOSetKeyStoreData(md);
1250 }
6d2010ae 1251 }
6d2010ae 1252#endif
99c3a104
A		 1253 break;
1254
1255 default:
1256 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
1257 break;
1258 }
1259 }
6d2010ae
A		 1260 }
1261 vars->handoffBuffer->release();
1262 }
3a60a9f5
A		 1263
1264 bzero(vars, sizeof(*vars));
1265
1266// gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
1267
1268 return (kIOReturnSuccess);
1269}
1270
1271IOReturn
1272IOHibernateSystemPostWake(void)
1273{
3e170ce0 1274 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
39037602 1275 IOLockLock(gFSLock);
316670eb 1276 if (kFSOpened == gFSState)
3a60a9f5 1277 {
6d2010ae 1278 // invalidate & close the image file
3e170ce0
A		 1279 IOSleep(TRIM_DELAY);
1280 IOPolledFileIOVars * vars = &gFileVars;
1281 IOPolledFileClose(&vars,
39236c6e 1282#if DISABLE_TRIM
3e170ce0 1283 0, NULL, 0, 0, 0);
39236c6e 1284#else
3e170ce0
A		 1285 0, (caddr_t)gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader),
1286 sizeof(IOHibernateImageHeader), gIOHibernateCurrentHeader->imageSize);
39236c6e 1287#endif
fe8ab488 1288 }
3e170ce0 1289 gFSState = kFSIdle;
fe8ab488 1290
39037602 1291 IOLockUnlock(gFSLock);
fe8ab488 1292
39037602
A		 1293 // IOCloseDebugDataFile() calls IOSetBootImageNVRAM() unconditionally
1294 IOCloseDebugDataFile( );
3a60a9f5
A		 1295 return (kIOReturnSuccess);
1296}
1297
d190cdc3 1298uint32_t IOHibernateWasScreenLocked(void)
316670eb 1299{
d190cdc3 1300 uint32_t ret = 0;
316670eb
A		 1301 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry)
1302 {
1303 OSData *
1304 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
d190cdc3 1305 if (data)
316670eb 1306 {
d190cdc3
A		 1307 ret = ((uint32_t *)data->getBytesNoCopy())[0];
1308 gIOChosenEntry->setProperty(kIOBooterScreenLockStateKey, data);
1309 }
316670eb 1310 }
d190cdc3
A		 1311 else gIOChosenEntry->removeProperty(kIOBooterScreenLockStateKey);
1312
316670eb
A		 1313 return (ret);
1314}
1315
3a60a9f5
A		 1316/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1317
2d21ac55 1318SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
6d2010ae 1319 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2d21ac55
A		 1320 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
1321SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
6d2010ae 1322 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2d21ac55
A		 1323 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
1324SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
6d2010ae 1325 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2d21ac55 1326 &gIOHibernateMode, 0, "");
39236c6e
A		 1327SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
1328 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
39037602 1329 &_hibernateStats, hibernate_statistics_t, "");
39236c6e
A		 1330
1331SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
1332 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
39037602 1333 &_hibernateStats.graphicsReadyTime, 0, "");
39236c6e
A		 1334SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
1335 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
39037602 1336 &_hibernateStats.wakeNotificationTime, 0, "");
39236c6e
A		 1337SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
1338 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
39037602 1339 &_hibernateStats.lockScreenReadyTime, 0, "");
39236c6e
A		 1340SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
1341 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
39037602 1342 &_hibernateStats.hidReadyTime, 0, "");
39236c6e 1343
2d21ac55 1344
3a60a9f5
A		 1345void
1346IOHibernateSystemInit(IOPMrootDomain * rootDomain)
1347{
3e170ce0
A		 1348 gIOHibernateBootImageKey = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1349
1350#if defined(__i386__) || defined(__x86_64__)
1351 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1352 gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
1353 gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
1354 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1355#endif /* defined(__i386__) || defined(__x86_64__) */
1356
3a60a9f5
A		 1357 OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
1358 if (data)
1359 {
1360 rootDomain->setProperty(kIOHibernateStateKey, data);
1361 data->release();
1362 }
1363
2d21ac55 1364 if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename)))
3a60a9f5
A		 1365 gIOHibernateMode = kIOHibernateModeOn;
1366 else
1367 gIOHibernateFilename[0] = 0;
1368
3a60a9f5 1369 sysctl_register_oid(&sysctl__kern_hibernatefile);
3a60a9f5 1370 sysctl_register_oid(&sysctl__kern_bootsignature);
3a60a9f5 1371 sysctl_register_oid(&sysctl__kern_hibernatemode);
39236c6e
A		 1372 sysctl_register_oid(&sysctl__kern_hibernatestatistics);
1373 sysctl_register_oid(&sysctl__kern_hibernategraphicsready);
1374 sysctl_register_oid(&sysctl__kern_hibernatewakenotification);
1375 sysctl_register_oid(&sysctl__kern_hibernatelockscreenready);
1376 sysctl_register_oid(&sysctl__kern_hibernatehidready);
316670eb 1377
3e170ce0
A		 1378 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1379
316670eb 1380 gFSLock = IOLockAlloc();
3a60a9f5
A		 1381}
1382
3a60a9f5
A		 1383/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1384
3e170ce0
A		 1385static IOReturn
1386IOHibernatePolledFileWrite(IOPolledFileIOVars * vars,
1387 const uint8_t * bytes, IOByteCount size,
1388 IOPolledFileCryptVars * cryptvars)
6d2010ae 1389{
3e170ce0 1390 IOReturn err;
6d2010ae 1391
3e170ce0
A		 1392 err = IOPolledFileWrite(vars, bytes, size, cryptvars);
1393 if ((kIOReturnSuccess == err) && hibernate_should_abort()) err = kIOReturnAborted;
6d2010ae 1394
3e170ce0 1395 return (err);
6d2010ae
A		 1396}
1397
3e170ce0 1398/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
6d2010ae 1399
2d21ac55 1400extern "C" uint32_t
3a60a9f5
A		 1401hibernate_write_image(void)
1402{
1403 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
1404 IOHibernateVars * vars = &gIOHibernateVars;
1405 IOPolledFileExtent * fileExtents;
1406
39037602 1407 _static_assert_1_arg(sizeof(IOHibernateImageHeader) == 512);
0c530ab8 1408
3a60a9f5
A		 1409 uint32_t pageCount, pagesDone;
1410 IOReturn err;
0b4c1975
A		 1411 vm_offset_t ppnum, page;
1412 IOItemCount count;
3a60a9f5
A		 1413 uint8_t * src;
1414 uint8_t * data;
39236c6e
A		 1415 uint8_t * compressed;
1416 uint8_t * scratch;
3e170ce0 1417 IOByteCount pageCompressedSize;
3a60a9f5
A		 1418 uint64_t compressedSize, uncompressedSize;
1419 uint64_t image1Size = 0;
1420 uint32_t bitmap_size;
0b4c1975 1421 bool iterDone, pollerOpen, needEncrypt;
3a60a9f5 1422 uint32_t restore1Sum, sum, sum1, sum2;
39236c6e 1423 int wkresult;
3a60a9f5
A		 1424 uint32_t tag;
1425 uint32_t pageType;
1426 uint32_t pageAndCount[2];
6d2010ae
A		 1427 addr64_t phys64;
1428 IOByteCount segLen;
3e170ce0
A		 1429 uintptr_t hibernateBase;
1430 uintptr_t hibernateEnd;
3a60a9f5
A		 1431
1432 AbsoluteTime startTime, endTime;
6d2010ae 1433 AbsoluteTime allTime, compTime;
0b4c1975 1434 uint64_t compBytes;
3a60a9f5
A		 1435 uint64_t nsec;
1436 uint32_t lastProgressStamp = 0;
1437 uint32_t progressStamp;
2d21ac55 1438 uint32_t blob, lastBlob = (uint32_t) -1L;
3a60a9f5 1439
39236c6e
A		 1440 uint32_t wiredPagesEncrypted;
1441 uint32_t dirtyPagesEncrypted;
1442 uint32_t wiredPagesClear;
3e170ce0
A		 1443 uint32_t svPageCount;
1444 uint32_t zvPageCount;
39236c6e 1445
3e170ce0
A		 1446 IOPolledFileCryptVars _cryptvars;
1447 IOPolledFileCryptVars * cryptvars = 0;
3a60a9f5 1448
39236c6e
A		 1449 wiredPagesEncrypted = 0;
1450 dirtyPagesEncrypted = 0;
1451 wiredPagesClear = 0;
3e170ce0
A		 1452 svPageCount = 0;
1453 zvPageCount = 0;
0b4c1975 1454
39037602
A		 1455 if (!vars->fileVars
1456 || !vars->fileVars->pollers
1457 || !(kIOHibernateModeOn & gIOHibernateMode)) return (kIOHibernatePostWriteSleep);
3a60a9f5 1458
0b4c1975
A		 1459 if (kIOHibernateModeSleep & gIOHibernateMode)
1460 kdebug_enable = save_kdebug_enable;
1461
1462 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START, 0, 0, 0, 0, 0);
6d2010ae 1463 IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
0b4c1975 1464
3a60a9f5
A		 1465 restore1Sum = sum1 = sum2 = 0;
1466
2d21ac55 1467#if CRYPTO
3a60a9f5
A		 1468 // encryption data. "iv" is the "initial vector".
1469 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1470 {
1471 static const unsigned char first_iv[AES_BLOCK_SIZE]
1472 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
1473 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
1474
1475 cryptvars = &gIOHibernateCryptWakeContext;
3e170ce0 1476 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
3a60a9f5
A		 1477 aes_encrypt_key(vars->cryptKey,
1478 kIOHibernateAESKeySize,
1479 &cryptvars->ctx.encrypt);
1480 aes_decrypt_key(vars->cryptKey,
1481 kIOHibernateAESKeySize,
1482 &cryptvars->ctx.decrypt);
1483
1484 cryptvars = &_cryptvars;
3e170ce0 1485 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
6d2010ae
A		 1486 for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++)
1487 vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
1488 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
3a60a9f5
A		 1489 aes_encrypt_key(vars->wiredCryptKey,
1490 kIOHibernateAESKeySize,
1491 &cryptvars->ctx.encrypt);
1492
1493 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1494 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1495 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
3a60a9f5 1496 }
2d21ac55 1497#endif /* CRYPTO */
3a60a9f5 1498
3a60a9f5
A		 1499 hibernate_page_list_setall(vars->page_list,
1500 vars->page_list_wired,
db609669
A		 1501 vars->page_list_pal,
1502 false /* !preflight */,
39236c6e
A		 1503 /* discard_all */
1504 ((0 == (kIOHibernateModeSleep & gIOHibernateMode))
1505 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))),
3a60a9f5
A		 1506 &pageCount);
1507
1508 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
1509
3e170ce0 1510 fileExtents = (IOPolledFileExtent *) vars->fileVars->fileExtents->getBytesNoCopy();
3a60a9f5
A		 1511
1512#if 0
1513 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
1514 for (page = 0; page < count; page++)
1515 {
1516 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
1517 fileExtents[page].start, fileExtents[page].length,
1518 fileExtents[page].start + fileExtents[page].length);
1519 }
1520#endif
1521
0b4c1975 1522 needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
3a60a9f5 1523 AbsoluteTime_to_scalar(&compTime) = 0;
6d2010ae 1524 compBytes = 0;
3a60a9f5
A		 1525
1526 clock_get_uptime(&allTime);
b0d623f7
A		 1527 IOService::getPMRootDomain()->pmStatsRecordEvent(
1528 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
3a60a9f5
A		 1529 do
1530 {
1531 compressedSize = 0;
1532 uncompressedSize = 0;
3e170ce0
A		 1533 svPageCount = 0;
1534 zvPageCount = 0;
3a60a9f5 1535
39236c6e 1536 IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);
3a60a9f5
A		 1537
1538 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
1539 ml_get_interrupts_enabled());
3e170ce0 1540 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledBeforeSleepState, true);
3a60a9f5
A		 1541 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
1542 pollerOpen = (kIOReturnSuccess == err);
1543 if (!pollerOpen)
1544 break;
1545
1546 // copy file block extent list if larger than header
1547
3e170ce0 1548 count = vars->fileVars->fileExtents->getLength();
3a60a9f5
A		 1549 if (count > sizeof(header->fileExtentMap))
1550 {
1551 count -= sizeof(header->fileExtentMap);
3e170ce0 1552 err = IOHibernatePolledFileWrite(vars->fileVars,
3a60a9f5
A		 1553 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
1554 if (kIOReturnSuccess != err)
1555 break;
1556 }
1557
6d2010ae 1558 hibernateBase = HIB_BASE; /* Defined in PAL headers */
316670eb 1559 hibernateEnd = (segHIBB + segSizeHIB);
6d2010ae 1560
3a60a9f5 1561 // copy out restore1 code
6d2010ae
A		 1562
1563 for (count = 0;
1564 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1565 count += segLen)
1566 {
1567 for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++)
1568 {
1569 gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone;
1570 }
1571 }
1572
1573 page = atop_32(kvtophys(hibernateBase));
1574 count = atop_32(round_page(hibernateEnd) - hibernateBase);
1575 header->restore1CodePhysPage = page;
1576 header->restore1CodeVirt = hibernateBase;
3a60a9f5 1577 header->restore1PageCount = count;
b0d623f7
A		 1578 header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
1579 header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
3a60a9f5 1580
316670eb 1581 // sum __HIB seg, with zeros for the stack
b0d623f7 1582 src = (uint8_t *) trunc_page(hibernateBase);
3a60a9f5
A		 1583 for (page = 0; page < count; page++)
1584 {
1585 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
6d2010ae 1586 restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page);
3a60a9f5 1587 else
0b4c1975 1588 restore1Sum += 0x00000000;
3a60a9f5
A		 1589 src += page_size;
1590 }
1591 sum1 = restore1Sum;
1592
316670eb 1593 // write the __HIB seg, with zeros for the stack
3a60a9f5 1594
b0d623f7
A		 1595 src = (uint8_t *) trunc_page(hibernateBase);
1596 count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
3a60a9f5
A		 1597 if (count)
1598 {
3e170ce0 1599 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
3a60a9f5
A		 1600 if (kIOReturnSuccess != err)
1601 break;
1602 }
3e170ce0 1603 err = IOHibernatePolledFileWrite(vars->fileVars,
3a60a9f5
A		 1604 (uint8_t *) 0,
1605 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
1606 cryptvars);
1607 if (kIOReturnSuccess != err)
1608 break;
1609 src = &gIOHibernateRestoreStackEnd[0];
b0d623f7 1610 count = round_page(hibernateEnd) - ((uintptr_t) src);
3a60a9f5
A		 1611 if (count)
1612 {
3e170ce0 1613 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
3a60a9f5
A		 1614 if (kIOReturnSuccess != err)
1615 break;
1616 }
1617
39236c6e
A		 1618 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1619 {
1620 vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
1621 vars->fileVars->encryptEnd = UINT64_MAX;
1622 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1623 }
316670eb 1624
3a60a9f5
A		 1625 // write the preview buffer
1626
2d21ac55 1627 if (vars->previewBuffer)
3a60a9f5
A		 1628 {
1629 ppnum = 0;
1630 count = 0;
1631 do
1632 {
b0d623f7 1633 phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
3a60a9f5
A		 1634 pageAndCount[0] = atop_64(phys64);
1635 pageAndCount[1] = atop_32(segLen);
3e170ce0 1636 err = IOHibernatePolledFileWrite(vars->fileVars,
3a60a9f5
A		 1637 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1638 cryptvars);
1639 if (kIOReturnSuccess != err)
1640 break;
1641 count += segLen;
1642 ppnum += sizeof(pageAndCount);
1643 }
1644 while (phys64);
1645 if (kIOReturnSuccess != err)
1646 break;
1647
b0d623f7 1648 src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
316670eb
A		 1649
1650 ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
1651
2d21ac55 1652 count = vars->previewBuffer->getLength();
3a60a9f5
A		 1653
1654 header->previewPageListSize = ppnum;
1655 header->previewSize = count + ppnum;
1656
1657 for (page = 0; page < count; page += page_size)
0b4c1975
A		 1658 {
1659 phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
1660 sum1 += hibernate_sum_page(src + page, atop_64(phys64));
1661 }
3e170ce0 1662 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
3a60a9f5
A		 1663 if (kIOReturnSuccess != err)
1664 break;
1665 }
1666
1667 // mark areas for no save
3e170ce0
A		 1668 IOMemoryDescriptor * ioBuffer;
1669 ioBuffer = IOPolledFileGetIOBuffer(vars->fileVars);
3a60a9f5 1670 for (count = 0;
3e170ce0 1671 (phys64 = ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
3a60a9f5
A		 1672 count += segLen)
1673 {
1674 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1675 atop_64(phys64), atop_32(segLen),
1676 kIOHibernatePageStateFree);
1677 pageCount -= atop_32(segLen);
1678 }
1679
1680 for (count = 0;
b0d623f7 1681 (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
3a60a9f5
A		 1682 count += segLen)
1683 {
1684 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1685 atop_64(phys64), atop_32(segLen),
1686 kIOHibernatePageStateFree);
1687 pageCount -= atop_32(segLen);
1688 }
1689
1690 // copy out bitmap of pages available for trashing during restore
1691
1692 bitmap_size = vars->page_list_wired->list_size;
1693 src = (uint8_t *) vars->page_list_wired;
3e170ce0 1694 err = IOHibernatePolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
3a60a9f5
A		 1695 if (kIOReturnSuccess != err)
1696 break;
1697
1698 // mark more areas for no save, but these are not available
1699 // for trashing during restore
0c530ab8
A		 1700
1701 hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
3a60a9f5 1702
6d2010ae
A		 1703
1704 page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
1705 count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
3a60a9f5
A		 1706 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1707 page, count,
1708 kIOHibernatePageStateFree);
1709 pageCount -= count;
3a60a9f5
A		 1710
1711 if (vars->previewBuffer) for (count = 0;
b0d623f7 1712 (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
3a60a9f5
A		 1713 count += segLen)
1714 {
1715 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1716 atop_64(phys64), atop_32(segLen),
1717 kIOHibernatePageStateFree);
1718 pageCount -= atop_32(segLen);
1719 }
1720
6d2010ae
A		 1721 for (count = 0;
1722 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1723 count += segLen)
1724 {
1725 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1726 atop_64(phys64), atop_32(segLen),
1727 kIOHibernatePageStateFree);
1728 pageCount -= atop_32(segLen);
1729 }
1730
3a60a9f5 1731 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
39236c6e
A		 1732 compressed = src + page_size;
1733 scratch = compressed + page_size;
1734
6d2010ae
A		 1735 pagesDone = 0;
1736 lastBlob = 0;
3a60a9f5 1737
db609669
A		 1738 HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
1739 bitmap_size, header->previewSize,
1740 pageCount, vars->fileVars->position);
3a60a9f5 1741
0b4c1975
A		 1742 enum
1743 // pageType
1744 {
1745 kWired = 0x02,
1746 kEncrypt = 0x01,
1747 kWiredEncrypt = kWired | kEncrypt,
1748 kWiredClear = kWired,
1749 kUnwiredEncrypt = kEncrypt
1750 };
1751
39037602
A		 1752 bool cpuAES = (0 != (CPUID_FEATURE_AES & cpuid_features()));
1753#define _pmap_is_noencrypt(x) (cpuAES ? false : pmap_is_noencrypt((x)))
1754
0b4c1975 1755 for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--)
3a60a9f5 1756 {
316670eb
A		 1757 if (kUnwiredEncrypt == pageType)
1758 {
1759 // start unwired image
39236c6e
A		 1760 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1761 {
1762 vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
1763 vars->fileVars->encryptEnd = UINT64_MAX;
1764 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1765 }
316670eb
A		 1766 bcopy(&cryptvars->aes_iv[0],
1767 &gIOHibernateCryptWakeContext.aes_iv[0],
1768 sizeof(cryptvars->aes_iv));
1769 cryptvars = &gIOHibernateCryptWakeContext;
0b4c1975
A		 1770 }
1771 for (iterDone = false, ppnum = 0; !iterDone; )
1772 {
1773 count = hibernate_page_list_iterate((kWired & pageType)
1774 ? vars->page_list_wired : vars->page_list,
1775 &ppnum);
1776// kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
1777 iterDone = !count;
39037602 1778
0b4c1975 1779 if (count && (kWired & pageType) && needEncrypt)
3a60a9f5 1780 {
0b4c1975
A		 1781 uint32_t checkIndex;
1782 for (checkIndex = 0;
1783 (checkIndex < count)
39037602 1784 && (((kEncrypt & pageType) == 0) == _pmap_is_noencrypt(ppnum + checkIndex));
0b4c1975
A		 1785 checkIndex++)
1786 {}
1787 if (!checkIndex)
1788 {
1789 ppnum++;
1790 continue;
1791 }
1792 count = checkIndex;
3a60a9f5
A		 1793 }
1794
0b4c1975
A		 1795 switch (pageType)
1796 {
39236c6e
A		 1797 case kWiredEncrypt: wiredPagesEncrypted += count; break;
1798 case kWiredClear: wiredPagesClear += count; break;
1799 case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
0b4c1975
A		 1800 }
1801
1802 if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */}
3a60a9f5 1803 else
3a60a9f5 1804 {
0b4c1975
A		 1805 pageAndCount[0] = ppnum;
1806 pageAndCount[1] = count;
3e170ce0 1807 err = IOHibernatePolledFileWrite(vars->fileVars,
0b4c1975
A		 1808 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1809 cryptvars);
1810 if (kIOReturnSuccess != err)
1811 break;
3a60a9f5 1812 }
0b4c1975
A		 1813
1814 for (page = ppnum; page < (ppnum + count); page++)
3a60a9f5 1815 {
0b4c1975
A		 1816 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
1817 if (err)
1818 {
1819 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
1820 break;
1821 }
1822
1823 sum = hibernate_sum_page(src, page);
1824 if (kWired & pageType)
1825 sum1 += sum;
1826 else
1827 sum2 += sum;
1828
1829 clock_get_uptime(&startTime);
3e170ce0 1830 wkresult = WKdm_compress_new((const WK_word*) src,
39236c6e
A		 1831 (WK_word*) compressed,
1832 (WK_word*) scratch,
1833 page_size - 4);
0b4c1975 1834
3a60a9f5 1835 clock_get_uptime(&endTime);
0b4c1975
A		 1836 ADD_ABSOLUTETIME(&compTime, &endTime);
1837 SUB_ABSOLUTETIME(&compTime, &startTime);
39236c6e 1838
0b4c1975 1839 compBytes += page_size;
39236c6e
A		 1840 pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
1841
3e170ce0 1842 if (pageCompressedSize == 0)
39236c6e 1843 {
3e170ce0 1844 pageCompressedSize = 4;
0b4c1975 1845 data = src;
3e170ce0
A		 1846
1847 if (*(uint32_t *)src)
1848 svPageCount++;
1849 else
1850 zvPageCount++;
1851 }
1852 else
1853 {
1854 if (pageCompressedSize != page_size)
1855 data = compressed;
1856 else
1857 data = src;
1858 }
0b4c1975
A		 1859
1860 tag = pageCompressedSize | kIOHibernateTagSignature;
3e170ce0 1861 err = IOHibernatePolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
0b4c1975
A		 1862 if (kIOReturnSuccess != err)
1863 break;
1864
3e170ce0 1865 err = IOHibernatePolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
0b4c1975
A		 1866 if (kIOReturnSuccess != err)
1867 break;
1868
1869 compressedSize += pageCompressedSize;
39236c6e 1870 uncompressedSize += page_size;
0b4c1975
A		 1871 pagesDone++;
1872
1873 if (vars->consoleMapping && (0 == (1023 & pagesDone)))
1874 {
1875 blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
1876 if (blob != lastBlob)
1877 {
1878 ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
1879 lastBlob = blob;
1880 }
1881 }
1882 if (0 == (8191 & pagesDone))
1883 {
1884 clock_get_uptime(&endTime);
1885 SUB_ABSOLUTETIME(&endTime, &allTime);
1886 absolutetime_to_nanoseconds(endTime, &nsec);
1887 progressStamp = nsec / 750000000ULL;
1888 if (progressStamp != lastProgressStamp)
1889 {
1890 lastProgressStamp = progressStamp;
1891 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
1892 }
3a60a9f5
A		 1893 }
1894 }
0b4c1975
A		 1895 if (kIOReturnSuccess != err)
1896 break;
1897 ppnum = page;
3a60a9f5 1898 }
0b4c1975 1899
3a60a9f5
A		 1900 if (kIOReturnSuccess != err)
1901 break;
0b4c1975 1902
39236c6e 1903 if ((kEncrypt & pageType) && vars->fileVars->encryptStart)
3a60a9f5 1904 {
7ddcb079 1905 vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
0b4c1975
A		 1906 HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
1907 }
1908
1909 if (kWiredEncrypt != pageType)
1910 {
1911 // end of image1/2 - fill to next block
3e170ce0 1912 err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars);
3a60a9f5
A		 1913 if (kIOReturnSuccess != err)
1914 break;
0b4c1975
A		 1915 }
1916 if (kWiredClear == pageType)
1917 {
7ddcb079 1918 // enlarge wired image for test
3e170ce0 1919// err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);
7ddcb079 1920
0b4c1975
A		 1921 // end wired image
1922 header->encryptStart = vars->fileVars->encryptStart;
1923 header->encryptEnd = vars->fileVars->encryptEnd;
3a60a9f5 1924 image1Size = vars->fileVars->position;
7ddcb079 1925 HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
0b4c1975 1926 image1Size, header->encryptStart, header->encryptEnd);
3a60a9f5
A		 1927 }
1928 }
3a60a9f5 1929 if (kIOReturnSuccess != err)
8a3053a0
A		 1930 {
1931 if (kIOReturnOverrun == err)
1932 {
3e170ce0 1933 // update actual compression ratio on not enough space (for retry)
8a3053a0
A		 1934 gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
1935 }
3e170ce0
A		 1936
1937 // update partial amount written (for IOPolledFileClose cleanup/unmap)
1938 header->imageSize = vars->fileVars->position;
3a60a9f5 1939 break;
8a3053a0 1940 }
3a60a9f5
A		 1941
1942 // Header:
1943
1944 header->imageSize = vars->fileVars->position;
1945 header->image1Size = image1Size;
1946 header->bitmapSize = bitmap_size;
1947 header->pageCount = pageCount;
3a60a9f5
A		 1948
1949 header->restore1Sum = restore1Sum;
1950 header->image1Sum = sum1;
1951 header->image2Sum = sum2;
316670eb 1952 header->sleepTime = gIOLastSleepTime.tv_sec;
db609669
A		 1953
1954 header->compression = (compressedSize << 8) / uncompressedSize;
1955 gIOHibernateCompression = header->compression;
3a60a9f5 1956
3e170ce0 1957 count = vars->fileVars->fileExtents->getLength();
3a60a9f5
A		 1958 if (count > sizeof(header->fileExtentMap))
1959 {
1960 header->fileExtentMapSize = count;
1961 count = sizeof(header->fileExtentMap);
1962 }
1963 else
1964 header->fileExtentMapSize = sizeof(header->fileExtentMap);
1965 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
0b4c1975 1966
39236c6e
A		 1967 header->deviceBase = vars->fileVars->block0;
1968 header->deviceBlockSize = vars->fileVars->blockSize;
6d2010ae 1969
3a60a9f5 1970 IOPolledFileSeek(vars->fileVars, 0);
3e170ce0 1971 err = IOHibernatePolledFileWrite(vars->fileVars,
3a60a9f5
A		 1972 (uint8_t *) header, sizeof(IOHibernateImageHeader),
1973 cryptvars);
1974 if (kIOReturnSuccess != err)
1975 break;
3e170ce0 1976 err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars);
3a60a9f5
A		 1977 }
1978 while (false);
1979
1980 clock_get_uptime(&endTime);
b0d623f7
A		 1981
1982 IOService::getPMRootDomain()->pmStatsRecordEvent(
1983 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
1984
3a60a9f5
A		 1985 SUB_ABSOLUTETIME(&endTime, &allTime);
1986 absolutetime_to_nanoseconds(endTime, &nsec);
39236c6e 1987 HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
3a60a9f5
A		 1988
1989 absolutetime_to_nanoseconds(compTime, &nsec);
6d2010ae
A		 1990 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
1991 compBytes,
1992 nsec / 1000000ULL,
1993 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
3a60a9f5 1994
6d2010ae
A		 1995 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
1996 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
1997 vars->fileVars->cryptBytes,
1998 nsec / 1000000ULL,
1999 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
3a60a9f5 2000
db609669
A		 2001 HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
2002 header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
3a60a9f5 2003 uncompressedSize, atop_32(uncompressedSize), compressedSize,
8f6c56a5 2004 uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
3a60a9f5
A		 2005 sum1, sum2);
2006
3e170ce0
A		 2007 HIBLOG("svPageCount %d, zvPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n",
2008 svPageCount, zvPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
c910b4d9 2009
3a60a9f5 2010 if (pollerOpen)
ecc0ceb4 2011 IOPolledFilePollersClose(vars->fileVars, (kIOReturnSuccess == err) ? kIOPolledBeforeSleepState : kIOPolledBeforeSleepStateAborted );
3a60a9f5 2012
2d21ac55
A		 2013 if (vars->consoleMapping)
2014 ProgressUpdate(gIOHibernateGraphicsInfo,
2015 vars->consoleMapping, 0, kIOHibernateProgressCount);
2016
3a60a9f5
A		 2017 HIBLOG("hibernate_write_image done(%x)\n", err);
2018
2019 // should we come back via regular wake, set the state in memory.
2020 gIOHibernateState = kIOHibernateStateInactive;
2021
0b4c1975 2022 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END,
39236c6e 2023 wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted, 0, 0);
0b4c1975 2024
2d21ac55
A		 2025 if (kIOReturnSuccess == err)
2026 {
2027 if (kIOHibernateModeSleep & gIOHibernateMode)
2028 {
2029 return (kIOHibernatePostWriteSleep);
2030 }
2031 else if(kIOHibernateModeRestart & gIOHibernateMode)
2032 {
2033 return (kIOHibernatePostWriteRestart);
2034 }
2035 else
2036 {
2037 /* by default, power down */
2038 return (kIOHibernatePostWriteHalt);
2039 }
2040 }
2041 else if (kIOReturnAborted == err)
2042 {
2043 return (kIOHibernatePostWriteWake);
2044 }
3a60a9f5 2045 else
3a60a9f5 2046 {
2d21ac55
A		 2047 /* on error, sleep */
2048 return (kIOHibernatePostWriteSleep);
3a60a9f5
A		 2049 }
2050}
2051
2052/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2053
2054extern "C" void
2055hibernate_machine_init(void)
2056{
2057 IOReturn err;
2058 uint32_t sum;
2059 uint32_t pagesDone;
0b4c1975 2060 uint32_t pagesRead = 0;
6d2010ae 2061 AbsoluteTime startTime, compTime;
3a60a9f5 2062 AbsoluteTime allTime, endTime;
39236c6e
A		 2063 AbsoluteTime startIOTime, endIOTime;
2064 uint64_t nsec, nsecIO;
6d2010ae 2065 uint64_t compBytes;
3a60a9f5
A		 2066 uint32_t lastProgressStamp = 0;
2067 uint32_t progressStamp;
3e170ce0 2068 IOPolledFileCryptVars * cryptvars = 0;
3a60a9f5
A		 2069
2070 IOHibernateVars * vars = &gIOHibernateVars;
39236c6e 2071 bzero(gIOHibernateStats, sizeof(hibernate_statistics_t));
3a60a9f5 2072
3e170ce0 2073 if (!vars->fileVars || !vars->fileVars->pollers)
3a60a9f5
A		 2074 return;
2075
3a60a9f5
A		 2076 sum = gIOHibernateCurrentHeader->actualImage1Sum;
2077 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
2078
3a60a9f5
A		 2079 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
2080 {
2081 HIBLOG("regular wake\n");
2082 return;
2083 }
2084
2085 HIBPRINT("diag %x %x %x %x\n",
2086 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
6d2010ae 2087 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
3a60a9f5 2088
39236c6e
A		 2089#define t40ms(x) (tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000)
2090#define tStat(x, y) gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y);
2091 tStat(booterStart, booterStart);
39037602 2092 gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart;
39236c6e
A		 2093 tStat(booterDuration0, booterTime0);
2094 tStat(booterDuration1, booterTime1);
2095 tStat(booterDuration2, booterTime2);
2096 tStat(booterDuration, booterTime);
2097 tStat(booterConnectDisplayDuration, connectDisplayTime);
2098 tStat(booterSplashDuration, splashTime);
2099 tStat(trampolineDuration, trampolineTime);
2100
2101 gIOHibernateStats->image1Size = gIOHibernateCurrentHeader->image1Size;
2102 gIOHibernateStats->imageSize = gIOHibernateCurrentHeader->imageSize;
2103 gIOHibernateStats->image1Pages = pagesDone;
2104
2105 HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n",
2106 gIOHibernateStats->booterStart,
2107 gIOHibernateStats->smcStart,
2108 gIOHibernateStats->booterDuration0,
2109 gIOHibernateStats->booterDuration1,
2110 gIOHibernateStats->booterDuration2,
2111 gIOHibernateStats->booterDuration,
2112 gIOHibernateStats->booterConnectDisplayDuration,
2113 gIOHibernateStats->booterSplashDuration,
2114 gIOHibernateStats->trampolineDuration);
2115
2116 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, imageSize 0x%qx, image1Size 0x%qx, conflictCount %d, nextFree %x\n",
2117 gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size,
2118 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
bd504ef0 2119
39236c6e
A		 2120 if ((0 != (kIOHibernateModeSleep & gIOHibernateMode))
2121 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)))
2122 {
0c530ab8 2123 hibernate_page_list_discard(vars->page_list);
39236c6e 2124 }
0c530ab8 2125
6d2010ae
A		 2126 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
2127
2128 if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount)
2129 panic("handoff overflow");
2130
2131 IOHibernateHandoff * handoff;
2132 bool done = false;
2133 bool foundCryptData = false;
2134
2135 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
2136 !done;
2137 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
2138 {
2139// HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
2140 uint8_t * data = &handoff->data[0];
2141 switch (handoff->type)
2142 {
2143 case kIOHibernateHandoffTypeEnd:
2144 done = true;
2145 break;
2146
2147 case kIOHibernateHandoffTypeGraphicsInfo:
39037602
A		 2148 if (handoff->bytecount == sizeof(*gIOHibernateGraphicsInfo))
2149 {
2150 bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
2151 }
6d2010ae
A		 2152 break;
2153
2154 case kIOHibernateHandoffTypeCryptVars:
2155 if (cryptvars)
2156 {
2157 hibernate_cryptwakevars_t *
2158 wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
2159 bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
2160 }
2161 foundCryptData = true;
2162 bzero(data, handoff->bytecount);
2163 break;
2164
2165 case kIOHibernateHandoffTypeMemoryMap:
bd504ef0
A		 2166
2167 clock_get_uptime(&allTime);
2168
6d2010ae
A		 2169 hibernate_newruntime_map(data, handoff->bytecount,
2170 gIOHibernateCurrentHeader->systemTableOffset);
bd504ef0
A		 2171
2172 clock_get_uptime(&endTime);
2173
2174 SUB_ABSOLUTETIME(&endTime, &allTime);
2175 absolutetime_to_nanoseconds(endTime, &nsec);
2176
2177 HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL);
2178
6d2010ae
A		 2179 break;
2180
2181 case kIOHibernateHandoffTypeDeviceTree:
2182 {
2183// DTEntry chosen = NULL;
2184// HIBPRINT("DTLookupEntry %d\n", DTLookupEntry((const DTEntry) data, "/chosen", &chosen));
2185 }
2186 break;
2187
2188 default:
2189 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
2190 break;
2191 }
2192 }
2193 if (cryptvars && !foundCryptData)
2194 panic("hibernate handoff");
2195
39037602 2196 HIBPRINT("video 0x%llx %d %d %d status %x\n",
316670eb
A		 2197 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
2198 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus);
2199
2200 if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress)
3a60a9f5
A		 2201 {
2202 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
2203 * gIOHibernateGraphicsInfo->rowBytes);
22ba694c
A		 2204 if (vars->videoMapSize > vars->videoAllocSize) vars->videoMapSize = 0;
2205 else
2206 {
2207 IOMapPages(kernel_map,
2208 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
2209 vars->videoMapSize, kIOMapInhibitCache );
2210 }
3a60a9f5
A		 2211 }
2212
316670eb
A		 2213 if (vars->videoMapSize)
2214 ProgressUpdate(gIOHibernateGraphicsInfo,
2215 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2216
0c530ab8 2217 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
39236c6e
A		 2218 uint8_t * compressed = src + page_size;
2219 uint8_t * scratch = compressed + page_size;
2220 uint32_t decoOffset;
3a60a9f5
A		 2221
2222 clock_get_uptime(&allTime);
6d2010ae
A		 2223 AbsoluteTime_to_scalar(&compTime) = 0;
2224 compBytes = 0;
3a60a9f5 2225
3e170ce0
A		 2226 HIBLOG("IOPolledFilePollersOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
2227 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledAfterSleepState, false);
39236c6e
A		 2228 clock_get_uptime(&startIOTime);
2229 endTime = startIOTime;
2230 SUB_ABSOLUTETIME(&endTime, &allTime);
2231 absolutetime_to_nanoseconds(endTime, &nsec);
3e170ce0 2232 HIBLOG("IOPolledFilePollersOpen(%x) %qd ms\n", err, nsec / 1000000ULL);
3a60a9f5 2233
3a60a9f5
A		 2234 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
2235
3a60a9f5 2236 // kick off the read ahead
3a60a9f5
A		 2237 vars->fileVars->bufferHalf = 0;
2238 vars->fileVars->bufferLimit = 0;
2239 vars->fileVars->lastRead = 0;
6d2010ae 2240 vars->fileVars->readEnd = gIOHibernateCurrentHeader->imageSize;
3a60a9f5 2241 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
6d2010ae
A		 2242 vars->fileVars->cryptBytes = 0;
2243 AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;
3a60a9f5 2244
6d2010ae 2245 err = IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
3a60a9f5
A		 2246 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2247 // --
2248
2249 HIBLOG("hibernate_machine_init reading\n");
2250
2251 uint32_t * header = (uint32_t *) src;
2252 sum = 0;
2253
6d2010ae 2254 while (kIOReturnSuccess == err)
3a60a9f5
A		 2255 {
2256 unsigned int count;
2257 unsigned int page;
2258 uint32_t tag;
2259 vm_offset_t ppnum, compressedSize;
2260
36401178
A		 2261 err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
2262 if (kIOReturnSuccess != err)
2263 break;
3a60a9f5
A		 2264
2265 ppnum = header[0];
2266 count = header[1];
2267
2268// HIBPRINT("(%x, %x)\n", ppnum, count);
2269
2270 if (!count)
2271 break;
2272
2273 for (page = 0; page < count; page++)
2274 {
36401178
A		 2275 err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
2276 if (kIOReturnSuccess != err)
2277 break;
3a60a9f5
A		 2278
2279 compressedSize = kIOHibernateTagLength & tag;
36401178
A		 2280 if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength))
2281 {
2282 err = kIOReturnIPCError;
2283 break;
2284 }
2285
3e170ce0
A		 2286 err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
2287 if (kIOReturnSuccess != err) break;
2288
2289 if (compressedSize < page_size)
36401178 2290 {
3e170ce0
A		 2291 decoOffset = page_size;
2292 clock_get_uptime(&startTime);
2293
2294 if (compressedSize == 4) {
2295 int i;
2296 uint32_t *s, *d;
2297
2298 s = (uint32_t *)src;
2299 d = (uint32_t *)(uintptr_t)compressed;
2300
2301 for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++)
2302 *d++ = *s;
39236c6e 2303 }
3e170ce0
A		 2304 else
2305 WKdm_decompress_new((WK_word*) src, (WK_word*) compressed, (WK_word*) scratch, compressedSize);
2306 clock_get_uptime(&endTime);
2307 ADD_ABSOLUTETIME(&compTime, &endTime);
2308 SUB_ABSOLUTETIME(&compTime, &startTime);
2309 compBytes += page_size;
2310 }
2311 else decoOffset = 0;
39236c6e 2312
3e170ce0
A		 2313 sum += hibernate_sum_page((src + decoOffset), ppnum);
2314 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
2315 if (err)
2316 {
39236c6e
A		 2317 HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
2318 break;
36401178 2319 }
3a60a9f5
A		 2320
2321 ppnum++;
2322 pagesDone++;
0b4c1975 2323 pagesRead++;
3a60a9f5 2324
3a60a9f5
A		 2325 if (0 == (8191 & pagesDone))
2326 {
2327 clock_get_uptime(&endTime);
2328 SUB_ABSOLUTETIME(&endTime, &allTime);
2329 absolutetime_to_nanoseconds(endTime, &nsec);
2330 progressStamp = nsec / 750000000ULL;
2331 if (progressStamp != lastProgressStamp)
2332 {
2333 lastProgressStamp = progressStamp;
2334 HIBPRINT("pages %d (%d%%)\n", pagesDone,
2335 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
2336 }
2337 }
2338 }
2339 }
7ddcb079
A		 2340 if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages))
2341 err = kIOReturnLockedRead;
3a60a9f5 2342
36401178
A		 2343 if (kIOReturnSuccess != err)
2344 panic("Hibernate restore error %x", err);
2345
3a60a9f5 2346 gIOHibernateCurrentHeader->actualImage2Sum = sum;
db609669 2347 gIOHibernateCompression = gIOHibernateCurrentHeader->compression;
3a60a9f5 2348
39236c6e
A		 2349 clock_get_uptime(&endIOTime);
2350
3e170ce0 2351 err = IOPolledFilePollersClose(vars->fileVars, kIOPolledAfterSleepState);
3a60a9f5 2352
3a60a9f5 2353 clock_get_uptime(&endTime);
b0d623f7
A		 2354
2355 IOService::getPMRootDomain()->pmStatsRecordEvent(
2356 kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
2357 IOService::getPMRootDomain()->pmStatsRecordEvent(
2358 kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
2359
3a60a9f5
A		 2360 SUB_ABSOLUTETIME(&endTime, &allTime);
2361 absolutetime_to_nanoseconds(endTime, &nsec);
2362
39236c6e
A		 2363 SUB_ABSOLUTETIME(&endIOTime, &startIOTime);
2364 absolutetime_to_nanoseconds(endIOTime, &nsecIO);
2365
2366 gIOHibernateStats->kernelImageReadDuration = nsec / 1000000ULL;
2367 gIOHibernateStats->imagePages = pagesDone;
2368
2369 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ",
2370 pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize,
2371 nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0);
6d2010ae
A		 2372
2373 absolutetime_to_nanoseconds(compTime, &nsec);
2374 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2375 compBytes,
2376 nsec / 1000000ULL,
2377 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2378
2379 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2380 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n",
2381 vars->fileVars->cryptBytes,
2382 nsec / 1000000ULL,
2383 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2384
2385 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_NONE, pagesRead, pagesDone, 0, 0, 0);
3a60a9f5
A		 2386}
2387
2388/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
99c3a104 2389
39236c6e
A		 2390void IOHibernateSetWakeCapabilities(uint32_t capability)
2391{
2392 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
2393 {
2394 gIOHibernateStats->wakeCapability = capability;
8a3053a0
A		 2395
2396 if (kIOPMSystemCapabilityGraphics & capability)
2397 {
2398 vm_compressor_do_warmup();
2399 }
39236c6e
A		 2400 }
2401}
2402
2403/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2404
99c3a104
A		 2405void IOHibernateSystemRestart(void)
2406{
2407 static uint8_t noteStore[32] __attribute__((aligned(32)));
2408 IORegistryEntry * regEntry;
2409 const OSSymbol * sym;
2410 OSData * noteProp;
2411 OSData * data;
2412 uintptr_t * smcVars;
2413 uint8_t * smcBytes;
2414 size_t len;
2415 addr64_t element;
2416
2417 data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
2418 if (!data) return;
2419
2420 smcVars = (typeof(smcVars)) data->getBytesNoCopy();
2421 smcBytes = (typeof(smcBytes)) smcVars[1];
2422 len = smcVars[0];
2423 if (len > sizeof(noteStore)) len = sizeof(noteStore);
2424 noteProp = OSData::withCapacity(3 * sizeof(element));
2425 if (!noteProp) return;
2426 element = len;
2427 noteProp->appendBytes(&element, sizeof(element));
2428 element = crc32(0, smcBytes, len);
2429 noteProp->appendBytes(&element, sizeof(element));
2430
2431 bcopy(smcBytes, noteStore, len);
2432 element = (addr64_t) &noteStore[0];
2433 element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
2434 noteProp->appendBytes(&element, sizeof(element));
2435
2436 if (!gIOOptionsEntry)
2437 {
2438 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
2439 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
2440 if (regEntry && !gIOOptionsEntry)
2441 regEntry->release();
2442 }
2443
2444 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
2445 if (gIOOptionsEntry && sym) gIOOptionsEntry->setProperty(sym, noteProp);
2446 if (noteProp) noteProp->release();
2447 if (sym) sym->release();
2448}
2449
2450
2451
mirror of XNU