]> git.saurik.com Git - apple/xnu.git/blame - iokit/Kernel/IOHibernateIO.cpp
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-3248.60.10.tar.gz
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
CommitLineData
3a60a9f5 1/*
b0d623f7 2 * Copyright (c) 2004-2008 Apple Computer, Inc. All rights reserved.
3a60a9f5 3 *
2d21ac55 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
3a60a9f5 5 *
2d21ac55
A		 6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
8f6c56a5 14 *
2d21ac55
A		 15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5
A		 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55
A		 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
8f6c56a5 25 *
2d21ac55 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
3a60a9f5
A		 27 */
28
29
30/*
31
32Sleep:
33
34- PMRootDomain calls IOHibernateSystemSleep() before system sleep
35(devices awake, normal execution context)
36- IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
37 grabs its extents and searches for a polling driver willing to work with that IOMedia.
38 The BSD code makes an ioctl to the storage driver to get the partition base offset to
39 the disk, and other ioctls to get the transfer constraints
40 If successful, the file is written to make sure its initially not bootable (in case of
41 later failure) and nvram set to point to the first block of the file. (Has to be done
42 here so blocking is possible in nvram support).
43 hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
44 page out any pages it wants to (currently zero, but probably some percentage of memory).
45 Its assumed just allocating pages will cause the VM system to naturally select the best
46 pages for eviction. It also copies processor flags needed for the restore path and sets
47 a flag in the boot processor proc info.
48 gIOHibernateState = kIOHibernateStateHibernating.
49- Regular sleep progresses - some drivers may inspect the root domain property
50 kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
51 as usual but leaves motherboard I/O on.
52- Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
53 at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
54 all ppc RC bits out of the hash table and caches into the mapping structures.
55- hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
56 hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
57 All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
58 by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
59 The image header and block list are written. The header includes the second file extent so
60 only the header block is needed to read the file, regardless of filesystem.
316670eb 61 The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data
3a60a9f5
A		 62 (only) is used to decompress the image during wake/boot.
63 Some additional pages are removed from the bitmaps - the buffers used for hibernation.
64 The bitmaps are written to the image.
65 More areas are removed from the bitmaps (after they have been written to the image) - the
316670eb 66 segment "__HIB" pages and interrupt stack.
3a60a9f5
A		 67 Each wired page is compressed and written and then each non-wired page. Compression and
68 disk writes are in parallel.
69 The image header is written to the start of the file and the polling driver closed.
70 The machine powers down (or sleeps).
71
72Boot/Wake:
73
74- BootX sees the boot-image nvram variable containing the device and block number of the image,
75 reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
76- BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
77 in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
78 that is in the kernel's __HIB section.
79- BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
80 hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
81 only code & data in that section is safe to call since all the other wired pages are still
82 compressed in the image.
83- hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
84 It uses the bitmaps to work out which pages can be uncompressed from the image to their final
85 location directly, and copies those that can't to interim free pages. When the image has been
86 completed, the copies are uncompressed, overwriting the wired image pages.
87 hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
88 is used to get pages into place for 64bit.
89- the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
90 changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
91 are removed from the software strutures, and the hash table is reinitialized.
92- After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
93 of memory, using the polled mode driver, before other threads can run or any devices are turned on.
94 This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
95 for the remaining non wired pages.
96- The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
97 (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
98 is closed via bsd.
99
100Polled Mode I/O:
101
102IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
103registry, specifying an object of calls IOPolledInterface.
104
105Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
106partition) that the image is going to live, looking for polled interface properties. If it finds
107one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
108interfaces found are kept in an ordered list.
109
110There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
111few different contexts things happen in:
112
113- there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
114up and running) and after wake - this is safe to allocate memory and do anything. The device
115ignores sleep requests from that point since its a waste of time if it goes to sleep and
116immediately wakes back up for the image write.
117
118- there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
119immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
120from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
121used to flush and set the disk to sleep.
122
123- there is an Open/Close (AfterSleep) pair made around the image read operations that happen
124immediately after sleep. These can't block or allocate memory. This is happening after the platform
125expert has woken the low level bits of the system, but most of the I/O system has not. There is only
126one thread running.
127
128For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
129(prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
130the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
131Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
132that is called for the hardware to check for events, and complete the I/O via the callback.
133The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
134to restrict I/O ops.
135*/
136
137#include <sys/systm.h>
138
139#include <IOKit/IOWorkLoop.h>
140#include <IOKit/IOCommandGate.h>
141#include <IOKit/IOTimerEventSource.h>
142#include <IOKit/IOPlatformExpert.h>
143#include <IOKit/IOKitDebug.h>
144#include <IOKit/IOTimeStamp.h>
145#include <IOKit/pwr_mgt/RootDomain.h>
146#include <IOKit/pwr_mgt/IOPMPrivate.h>
147#include <IOKit/IOMessage.h>
148#include <IOKit/IODeviceTreeSupport.h>
149#include <IOKit/IOBSD.h>
150#include "RootDomainUserClient.h"
151#include <IOKit/pwr_mgt/IOPowerConnection.h>
152#include "IOPMPowerStateQueue.h"
153#include <IOKit/IOBufferMemoryDescriptor.h>
6d2010ae 154#include <IOKit/AppleKeyStoreInterface.h>
316670eb 155#include <libkern/crypto/aes.h>
3a60a9f5
A		 156
157#include <sys/uio.h>
158#include <sys/conf.h>
159#include <sys/stat.h>
160#include <sys/fcntl.h> // (FWRITE, ...)
3a60a9f5 161#include <sys/sysctl.h>
6d2010ae 162#include <sys/kdebug.h>
fe8ab488 163#include <stdint.h>
3a60a9f5
A		 164
165#include <IOKit/IOHibernatePrivate.h>
166#include <IOKit/IOPolledInterface.h>
167#include <IOKit/IONVRAM.h>
168#include "IOHibernateInternal.h"
39236c6e 169#include <vm/WKdm_new.h>
3a60a9f5 170#include "IOKitKernelInternal.h"
6d2010ae
A		 171#include <pexpert/device_tree.h>
172
173#include <machine/pal_routines.h>
174#include <machine/pal_hibernate.h>
39236c6e 175#include <i386/tsc.h>
6d2010ae
A		 176
177extern "C" addr64_t kvtophys(vm_offset_t va);
99c3a104 178extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
3a60a9f5
A		 179
180/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
181
39236c6e
A		 182#define DISABLE_TRIM 0
183#define TRIM_DELAY 5000
184
3e170ce0 185extern boolean_t root_is_CF_drive;
0b4c1975 186extern unsigned int save_kdebug_enable;
3a60a9f5
A		 187extern uint32_t gIOHibernateState;
188uint32_t gIOHibernateMode;
189static char gIOHibernateBootSignature[256+1];
190static char gIOHibernateFilename[MAXPATHLEN+1];
db609669
A		 191static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
192uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms)
193static uint64_t gIOHibernateCompression = 0x80; // default compression 50%
3a60a9f5
A		 194
195static IODTNVRAM * gIOOptionsEntry;
196static IORegistryEntry * gIOChosenEntry;
3e170ce0
A		 197
198static const OSSymbol * gIOHibernateBootImageKey;
199
b0d623f7 200#if defined(__i386__) || defined(__x86_64__)
3e170ce0 201
060df5ea
A		 202static const OSSymbol * gIOHibernateRTCVariablesKey;
203static const OSSymbol * gIOHibernateBoot0082Key;
204static const OSSymbol * gIOHibernateBootNextKey;
205static OSData * gIOHibernateBoot0082Data;
206static OSData * gIOHibernateBootNextData;
207static OSObject * gIOHibernateBootNextSave;
3e170ce0
A		 208
209static IOPolledFileIOVars * gDebugImageFileVars;
210static IOLock * gDebugImageLock;
211
212#endif /* defined(__i386__) || defined(__x86_64__) */
3a60a9f5 213
316670eb
A		 214static IOLock * gFSLock;
215static uint32_t gFSState;
3a60a9f5
A		 216static IOPolledFileIOVars gFileVars;
217static IOHibernateVars gIOHibernateVars;
3e170ce0 218static IOPolledFileCryptVars gIOHibernateCryptWakeContext;
6d2010ae
A		 219static hibernate_graphics_t _hibernateGraphics;
220static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
39236c6e
A		 221static hibernate_statistics_t _hibernateStats;
222static hibernate_statistics_t * gIOHibernateStats = &_hibernateStats;
3a60a9f5 223
316670eb
A		 224enum
225{
226 kFSIdle = 0,
227 kFSOpening = 2,
228 kFSOpened = 3,
229 kFSTimedOut = 4,
230};
231
232static IOReturn IOHibernateDone(IOHibernateVars * vars);
3e170ce0
A		 233static IOReturn IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature);
234static void IOSetBootImageNVRAM(OSData * data);
316670eb 235
3a60a9f5
A		 236/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
237
3a60a9f5 238enum { kDefaultIOSize = 128 * 1024 };
22ba694c 239enum { kVideoMapSize = 80 * 1024 * 1024 };
3a60a9f5 240
3a60a9f5
A		 241/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
242
243// copy from phys addr to MD
244
245static IOReturn
246IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
247 IOByteCount offset, addr64_t bytes, IOByteCount length)
248{
249 addr64_t srcAddr = bytes;
250 IOByteCount remaining;
251
252 remaining = length = min(length, md->getLength() - offset);
253 while (remaining) { // (process another target segment?)
254 addr64_t dstAddr64;
255 IOByteCount dstLen;
256
b0d623f7 257 dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
3a60a9f5
A		 258 if (!dstAddr64)
259 break;
260
261 // Clip segment length to remaining
262 if (dstLen > remaining)
263 dstLen = remaining;
264
265#if 1
266 bcopy_phys(srcAddr, dstAddr64, dstLen);
267#else
268 copypv(srcAddr, dstAddr64, dstLen,
269 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
270#endif
271 srcAddr += dstLen;
272 offset += dstLen;
273 remaining -= dstLen;
274 }
275
276 assert(!remaining);
277
278 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
279}
280
281// copy from MD to phys addr
282
283static IOReturn
284IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
285 IOByteCount offset, addr64_t bytes, IOByteCount length)
286{
287 addr64_t dstAddr = bytes;
288 IOByteCount remaining;
289
290 remaining = length = min(length, md->getLength() - offset);
291 while (remaining) { // (process another target segment?)
292 addr64_t srcAddr64;
293 IOByteCount dstLen;
294
b0d623f7 295 srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
3a60a9f5
A		 296 if (!srcAddr64)
297 break;
298
299 // Clip segment length to remaining
300 if (dstLen > remaining)
301 dstLen = remaining;
302
303#if 1
304 bcopy_phys(srcAddr64, dstAddr, dstLen);
305#else
306 copypv(srcAddr, dstAddr64, dstLen,
307 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
308#endif
309 dstAddr += dstLen;
310 offset += dstLen;
311 remaining -= dstLen;
312 }
313
314 assert(!remaining);
315
316 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
317}
318
319/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
320
321void
322hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
323 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
324{
325 count += ppnum;
326 switch (kind)
327 {
328 case kIOHibernatePageStateUnwiredSave:
329 // unwired save
330 for (; ppnum < count; ppnum++)
331 {
332 hibernate_page_bitset(page_list, FALSE, ppnum);
333 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
334 }
335 break;
336 case kIOHibernatePageStateWiredSave:
337 // wired save
338 for (; ppnum < count; ppnum++)
339 {
340 hibernate_page_bitset(page_list, FALSE, ppnum);
341 hibernate_page_bitset(page_list_wired, FALSE, ppnum);
342 }
343 break;
344 case kIOHibernatePageStateFree:
345 // free page
346 for (; ppnum < count; ppnum++)
347 {
348 hibernate_page_bitset(page_list, TRUE, ppnum);
349 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
350 }
351 break;
352 default:
353 panic("hibernate_set_page_state");
354 }
355}
356
3e170ce0
A		 357static vm_offset_t
358hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
359{
360 uint32_t page = *pPage;
361 uint32_t count;
362 hibernate_bitmap_t * bitmap;
fe8ab488 363
3e170ce0
A		 364 while ((bitmap = hibernate_page_bitmap_pin(list, &page)))
365 {
366 count = hibernate_page_bitmap_count(bitmap, TRUE, page);
367 if (!count)
368 break;
369 page += count;
370 if (page <= bitmap->last_page)
371 break;
fe8ab488 372 }
fe8ab488 373
3e170ce0
A		 374 *pPage = page;
375 if (bitmap)
376 count = hibernate_page_bitmap_count(bitmap, FALSE, page);
377 else
378 count = 0;
379
380 return (count);
fe8ab488 381}
fe8ab488
A		 382
383/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
384
3a60a9f5
A		 385IOReturn
386IOHibernateSystemSleep(void)
387{
388 IOReturn err;
3e170ce0 389 OSData * nvramData;
3a60a9f5
A		 390 OSObject * obj;
391 OSString * str;
db609669 392 OSNumber * num;
39236c6e 393 bool dsSSD, vmflush;
316670eb 394 IOHibernateVars * vars;
3e170ce0 395 uint64_t setFileSize = 0;
3a60a9f5
A		 396
397 gIOHibernateState = kIOHibernateStateInactive;
398
b0d623f7
A		 399 gIOHibernateDebugFlags = 0;
400 if (kIOLogHibernate & gIOKitDebug)
401 gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
402
0b4c1975
A		 403 if (IOService::getPMRootDomain()->getHibernateSettings(
404 &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime))
6d2010ae 405 {
3a60a9f5
A		 406 if (kIOHibernateModeSleep & gIOHibernateMode)
407 // default to discard clean for safe sleep
408 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
409 | kIOHibernateModeDiscardCleanActive);
6d2010ae 410 }
3a60a9f5 411
3a60a9f5
A		 412 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
413 {
414 if ((str = OSDynamicCast(OSString, obj)))
2d21ac55
A		 415 strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
416 sizeof(gIOHibernateFilename));
3a60a9f5
A		 417 obj->release();
418 }
419
420 if (!gIOHibernateMode || !gIOHibernateFilename[0])
421 return (kIOReturnUnsupported);
422
423 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
424
316670eb
A		 425 vars = IONew(IOHibernateVars, 1);
426 if (!vars) return (kIOReturnNoMemory);
427 bzero(vars, sizeof(*vars));
428
429 IOLockLock(gFSLock);
430 if (kFSIdle != gFSState)
431 {
432 HIBLOG("hibernate file busy\n");
433 IOLockUnlock(gFSLock);
434 IODelete(vars, IOHibernateVars, 1);
435 return (kIOReturnBusy);
436 }
437 gFSState = kFSOpening;
438 IOLockUnlock(gFSLock);
2d21ac55 439
3a60a9f5
A		 440 do
441 {
0c530ab8 442 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
39236c6e 443 2 * page_size + WKdm_SCRATCH_BUF_SIZE, page_size);
3a60a9f5 444
6d2010ae
A		 445 vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
446 ptoa_64(gIOHibernateHandoffPageCount), page_size);
447
3e170ce0 448 if (!vars->srcBuffer || !vars->handoffBuffer)
3a60a9f5
A		 449 {
450 err = kIOReturnNoMemory;
451 break;
452 }
453
db609669
A		 454 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey)))
455 {
456 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMinSize = num->unsigned64BitValue();
457 obj->release();
458 }
459 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey)))
460 {
461 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMaxSize = num->unsigned64BitValue();
462 obj->release();
463 }
464
465 boolean_t encryptedswap = true;
466 uint32_t pageCount;
467 AbsoluteTime startTime, endTime;
468 uint64_t nsec;
469
470 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
471 gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
6d2010ae 472 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
db609669 473
3e170ce0 474 vmflush = ((kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)) && root_is_CF_drive == FALSE);
39236c6e
A		 475 err = hibernate_alloc_page_lists(&vars->page_list,
476 &vars->page_list_wired,
477 &vars->page_list_pal);
478 if (KERN_SUCCESS != err)
479 break;
db609669
A		 480
481 if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode))
482 {
483 hibernate_page_list_setall(vars->page_list,
484 vars->page_list_wired,
485 vars->page_list_pal,
486 true /* preflight */,
39236c6e 487 vmflush /* discard */,
db609669
A		 488 &pageCount);
489 PE_Video consoleInfo;
490 bzero(&consoleInfo, sizeof(consoleInfo));
491 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
492
8a3053a0 493 // estimate: 6% increase in pages compressed
fe8ab488 494 // screen preview 2 images compressed 0%
8a3053a0 495 setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8)
db609669 496 + vars->page_list->list_size
8a3053a0
A		 497 + (consoleInfo.v_width * consoleInfo.v_height * 8);
498 enum { setFileRound = 1024*1024ULL };
499 setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1));
fe8ab488 500
db609669
A		 501 HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n",
502 pageCount, (100ULL * gIOHibernateCompression) >> 8,
503 setFileSize, vars->fileMinSize);
504
505 if (!(kIOHibernateModeFileResize & gIOHibernateMode)
506 && (setFileSize < vars->fileMinSize))
507 {
508 setFileSize = vars->fileMinSize;
509 }
510 }
511
3e170ce0
A		 512 // Invalidate the image file
513 if (gDebugImageLock) {
514 IOLockLock(gDebugImageLock);
515 if (gDebugImageFileVars != 0) {
516 kprintf("IOHIBSystemSleep: Closing debugdata file\n");
517 IOSetBootImageNVRAM(0);
518 IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
519 }
520 IOLockUnlock(gDebugImageLock);
521 }
db609669 522
3e170ce0
A		 523 err = IOPolledFileOpen(gIOHibernateFilename, setFileSize, 0,
524 gIOHibernateCurrentHeader, sizeof(gIOHibernateCurrentHeader),
525 &vars->fileVars, &nvramData,
526 &vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
fe8ab488 527
3e170ce0
A		 528 if (KERN_SUCCESS != err)
529 {
530 IOLockLock(gFSLock);
531 if (kFSOpening != gFSState) err = kIOReturnTimeout;
532 IOLockUnlock(gFSLock);
533 }
8a3053a0 534
3a60a9f5
A		 535 if (KERN_SUCCESS != err)
536 {
537 HIBLOG("IOPolledFileOpen(%x)\n", err);
538 break;
539 }
3a60a9f5 540
3e170ce0
A		 541 // write extents for debug data usage in EFI
542 IOWriteExtentsToFile(vars->fileVars, kIOHibernateHeaderOpenSignature);
543
544 err = IOPolledFilePollersSetup(vars->fileVars, kIOPolledPreflightState);
545 if (KERN_SUCCESS != err) break;
546
39236c6e
A		 547 clock_get_uptime(&startTime);
548 err = hibernate_setup(gIOHibernateCurrentHeader,
39236c6e
A		 549 vmflush,
550 vars->page_list, vars->page_list_wired, vars->page_list_pal);
551 clock_get_uptime(&endTime);
552 SUB_ABSOLUTETIME(&endTime, &startTime);
553 absolutetime_to_nanoseconds(endTime, &nsec);
554 HIBLOG("hibernate_setup(%d) took %qd ms\n", err, nsec / 1000000ULL);
3e170ce0 555 if (KERN_SUCCESS != err) break;
39236c6e 556
3e170ce0 557 dsSSD = ((0 != (kIOPolledFileSSD & vars->fileVars->flags))
0b4c1975 558 && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
3e170ce0
A		 559
560 if (dsSSD) gIOHibernateCurrentHeader->options |= kIOHibernateOptionSSD | kIOHibernateOptionColor;
561 else gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
562
6d2010ae
A		 563
564#if defined(__i386__) || defined(__x86_64__)
3e170ce0
A		 565 if (!uuid_is_null(vars->volumeCryptKey) &&
566 (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey)))
567 {
568 uintptr_t smcVars[2];
569 smcVars[0] = sizeof(vars->volumeCryptKey);
570 smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
6d2010ae 571
3e170ce0
A		 572 IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
573 bzero(smcVars, sizeof(smcVars));
574 }
6d2010ae 575#endif
0b4c1975 576
0b4c1975 577
6d2010ae 578 if (encryptedswap || !uuid_is_null(vars->volumeCryptKey))
3a60a9f5
A		 579 gIOHibernateMode ^= kIOHibernateModeEncrypt;
580
0b4c1975
A		 581 if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
582 {
583 vars->videoAllocSize = kVideoMapSize;
3e170ce0 584 if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize, VM_KERN_MEMORY_IOKIT))
0b4c1975
A		 585 vars->videoMapping = 0;
586 }
3a60a9f5
A		 587
588 // generate crypt keys
589 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
590 vars->wiredCryptKey[i] = random();
591 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
592 vars->cryptKey[i] = random();
593
594 // set nvram
595
3e170ce0
A		 596 IOSetBootImageNVRAM(nvramData);
597 nvramData->release();
3a60a9f5 598
b0d623f7 599#if defined(__i386__) || defined(__x86_64__)
3e170ce0 600 {
0c530ab8
A		 601 struct AppleRTCHibernateVars
602 {
603 uint8_t signature[4];
604 uint32_t revision;
605 uint8_t booterSignature[20];
606 uint8_t wiredCryptKey[16];
607 };
608 AppleRTCHibernateVars rtcVars;
3e170ce0 609 OSData * data;
0c530ab8
A		 610
611 rtcVars.signature[0] = 'A';
612 rtcVars.signature[1] = 'A';
613 rtcVars.signature[2] = 'P';
614 rtcVars.signature[3] = 'L';
615 rtcVars.revision = 1;
616 bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
617 if (gIOHibernateBootSignature[0])
618 {
619 char c;
620 uint8_t value = 0;
621 for (uint32_t i = 0;
622 (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1));
623 i++)
624 {
3e170ce0
A		 625 if (c >= 'a') c -= 'a' - 10;
626 else if (c >= 'A') c -= 'A' - 10;
627 else if (c >= '0') c -= '0';
628 else continue;
0c530ab8 629 value = (value << 4) | c;
3e170ce0 630 if (i & 1) rtcVars.booterSignature[i >> 1] = value;
0c530ab8
A		 631 }
632 }
633 data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
634 if (data)
2d21ac55 635 {
060df5ea
A		 636 if (gIOHibernateRTCVariablesKey)
637 IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
060df5ea 638 data->release();
0c530ab8
A		 639 }
640 if (gIOChosenEntry)
641 {
642 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
3e170ce0
A		 643 if (data) gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
644 // set BootNext
645 if (!gIOHibernateBoot0082Data)
060df5ea 646 {
3e170ce0
A		 647 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
648 if (data)
060df5ea 649 {
3e170ce0
A		 650 // AppleNVRAM_EFI_LOAD_OPTION
651 struct {
652 uint32_t Attributes;
653 uint16_t FilePathLength;
654 uint16_t Desc;
655 } loadOptionHeader;
656 loadOptionHeader.Attributes = 1;
657 loadOptionHeader.FilePathLength = data->getLength();
658 loadOptionHeader.Desc = 0;
659 gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
660 if (gIOHibernateBoot0082Data)
060df5ea 661 {
3e170ce0
A		 662 gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
663 gIOHibernateBoot0082Data->appendBytes(data);
060df5ea
A		 664 }
665 }
3e170ce0
A		 666 }
667 if (!gIOHibernateBootNextData)
668 {
669 uint16_t bits = 0x0082;
670 gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
671 }
672 if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData)
673 {
674 gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
675 gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
676 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
677 }
678 // BootNext
3a60a9f5
A		 679 }
680 }
3e170ce0 681#endif /* !i386 && !x86_64 */
316670eb
A		 682 }
683 while (false);
684
685 IOLockLock(gFSLock);
3e170ce0
A		 686 if ((kIOReturnSuccess == err) && (kFSOpening != gFSState))
687 {
688 HIBLOG("hibernate file close due timeout\n");
689 err = kIOReturnTimeout;
690 }
691 if (kIOReturnSuccess == err)
316670eb
A		 692 {
693 gFSState = kFSOpened;
694 gIOHibernateVars = *vars;
695 gFileVars = *vars->fileVars;
3e170ce0 696 gFileVars.allocated = false;
316670eb 697 gIOHibernateVars.fileVars = &gFileVars;
3a60a9f5
A		 698 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
699 gIOHibernateState = kIOHibernateStateHibernating;
700 }
316670eb
A		 701 else
702 {
3e170ce0 703 IOPolledFileIOVars * fileVars = vars->fileVars;
316670eb 704 IOHibernateDone(vars);
3e170ce0
A		 705 IOPolledFileClose(&fileVars,
706#if DISABLE_TRIM
707 0, NULL, 0, 0, 0);
708#else
709 0, NULL, 0, sizeof(IOHibernateImageHeader), setFileSize);
710#endif
316670eb
A		 711 gFSState = kFSIdle;
712 }
713 IOLockUnlock(gFSLock);
714
715 if (vars->fileVars) IODelete(vars->fileVars, IOPolledFileIOVars, 1);
716 IODelete(vars, IOHibernateVars, 1);
3a60a9f5
A		 717
718 return (err);
719}
720
721/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
722
3e170ce0
A		 723static void
724IOSetBootImageNVRAM(OSData * data)
725{
726 IORegistryEntry * regEntry;
727
728 if (!gIOOptionsEntry)
729 {
730 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
731 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
732 if (regEntry && !gIOOptionsEntry)
733 regEntry->release();
734 }
735 if (gIOOptionsEntry && gIOHibernateBootImageKey)
736 {
737 if (data) gIOOptionsEntry->setProperty(gIOHibernateBootImageKey, data);
738 else
739 {
740 gIOOptionsEntry->removeProperty(gIOHibernateBootImageKey);
741 gIOOptionsEntry->sync();
742 }
743 }
744}
745
746/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
747/*
748 * Writes header to disk with signature, block size and file extents data.
749 * If there are more than 2 extents, then they are written on second block.
750 */
751static IOReturn
752IOWriteExtentsToFile(IOPolledFileIOVars * vars, uint32_t signature)
753{
754 IOHibernateImageHeader hdr;
755 IOItemCount count;
756 IOReturn err = kIOReturnSuccess;
757 int rc;
758 IOPolledFileExtent * fileExtents;
759
760 fileExtents = (typeof(fileExtents)) vars->fileExtents->getBytesNoCopy(),
761
762 memset(&hdr, 0, sizeof(IOHibernateImageHeader));
763 count = vars->fileExtents->getLength();
764 if (count > sizeof(hdr.fileExtentMap))
765 {
766 hdr.fileExtentMapSize = count;
767 count = sizeof(hdr.fileExtentMap);
768 }
769 else
770 hdr.fileExtentMapSize = sizeof(hdr.fileExtentMap);
771
772 bcopy(fileExtents, &hdr.fileExtentMap[0], count);
773
774 // copy file block extent list if larger than header
775 if (hdr.fileExtentMapSize > sizeof(hdr.fileExtentMap))
776 {
777 count = hdr.fileExtentMapSize - sizeof(hdr.fileExtentMap);
778 rc = kern_write_file(vars->fileRef, vars->blockSize,
779 (caddr_t)(((uint8_t *)fileExtents) + sizeof(hdr.fileExtentMap)),
780 count, IO_SKIP_ENCRYPTION);
781 if (rc != 0) {
782 HIBLOG("kern_write_file returned %d\n", rc);
783 err = kIOReturnIOError;
784 goto exit;
785 }
786 }
787 hdr.signature = signature;
788 hdr.deviceBlockSize = vars->blockSize;
789
790 rc = kern_write_file(vars->fileRef, 0, (char *)&hdr, sizeof(hdr), IO_SKIP_ENCRYPTION);
791 if (rc != 0) {
792 HIBLOG("kern_write_file returned %d\n", rc);
793 err = kIOReturnIOError;
794 goto exit;
795 }
796
797exit:
798 return err;
799}
800
801void
802IOOpenDebugDataFile(const char *fname, uint64_t size)
803{
804 IOReturn err;
805 OSData * imagePath = NULL;
806 uint64_t padding;
807
808 if (!gDebugImageLock) {
809 gDebugImageLock = IOLockAlloc();
810 }
811
812 // Try to get a lock, but don't block for getting lock
813 if (!IOLockTryLock(gDebugImageLock)) {
814 HIBLOG("IOOpenDebugDataFile: Failed to get lock\n");
815 return;
816 }
817
818 if (gDebugImageFileVars || !fname || !size) {
819 HIBLOG("IOOpenDebugDataFile: conditions failed\n");
820 goto exit;
821 }
822
823 padding = (PAGE_SIZE*2); // allocate couple more pages for header and fileextents
824 err = IOPolledFileOpen(fname, size+padding, 32ULL*1024*1024*1024,
825 NULL, 0,
826 &gDebugImageFileVars, &imagePath, NULL, 0);
827
828 if ((kIOReturnSuccess == err) && imagePath)
829 {
830 if ((gDebugImageFileVars->fileSize < (size+padding)) ||
831 (gDebugImageFileVars->fileExtents->getLength() > PAGE_SIZE)) {
832 // Can't use the file
833 IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
834 HIBLOG("IOOpenDebugDataFile: too many file extents\n");
835 goto exit;
836 }
837
838 // write extents for debug data usage in EFI
839 IOWriteExtentsToFile(gDebugImageFileVars, kIOHibernateHeaderOpenSignature);
840 IOSetBootImageNVRAM(imagePath);
841 kprintf("IOOpenDebugDataFile: opened debugdata file\n");
842 }
843
844exit:
845 IOLockUnlock(gDebugImageLock);
846
847 if (imagePath) imagePath->release();
848 return;
849}
850
851/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
852
2d21ac55
A		 853DECLARE_IOHIBERNATEPROGRESSALPHA
854
855static void
856ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
857{
858 uint32_t rowBytes, pixelShift;
859 uint32_t x, y;
860 int32_t blob;
861 uint32_t alpha, in, color, result;
862 uint8_t * out;
863 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
864
865 rowBytes = display->rowBytes;
866 pixelShift = display->depth >> 4;
867 if (pixelShift < 1) return;
868
869 screen += ((display->width
870 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
871 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
872
873 for (y = 0; y < kIOHibernateProgressHeight; y++)
874 {
875 out = screen + y * rowBytes;
876 for (blob = 0; blob < kIOHibernateProgressCount; blob++)
877 {
878 color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
879 for (x = 0; x < kIOHibernateProgressWidth; x++)
880 {
881 alpha = gIOHibernateProgressAlpha[y][x];
882 result = color;
883 if (alpha)
884 {
885 if (0xff != alpha)
886 {
887 if (1 == pixelShift)
888 {
889 in = *((uint16_t *)out) & 0x1f; // 16
890 in = (in << 3) | (in >> 2);
891 }
892 else
893 in = *((uint32_t *)out) & 0xff; // 32
894 saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
895 result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
896 }
897 if (1 == pixelShift)
898 {
899 result >>= 3;
900 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
901 }
902 else
903 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
904 }
905 out += (1 << pixelShift);
906 }
907 out += (kIOHibernateProgressSpacing << pixelShift);
908 }
909 }
910}
911
912
913static void
914ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
915{
916 uint32_t rowBytes, pixelShift;
917 uint32_t x, y;
918 int32_t blob, lastBlob;
919 uint32_t alpha, in, color, result;
920 uint8_t * out;
921 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
922
923 pixelShift = display->depth >> 4;
924 if (pixelShift < 1)
925 return;
926
927 rowBytes = display->rowBytes;
928
929 screen += ((display->width
930 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
931 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
932
933 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
934
935 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
936
937 for (y = 0; y < kIOHibernateProgressHeight; y++)
938 {
939 out = screen + y * rowBytes;
940 for (blob = firstBlob; blob <= lastBlob; blob++)
941 {
942 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
943 for (x = 0; x < kIOHibernateProgressWidth; x++)
944 {
945 alpha = gIOHibernateProgressAlpha[y][x];
946 result = color;
947 if (alpha)
948 {
949 if (0xff != alpha)
950 {
951 in = display->progressSaveUnder[blob][saveindex[blob]++];
952 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
953 }
954 if (1 == pixelShift)
955 {
956 result >>= 3;
957 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
958 }
959 else
960 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
961 }
962 out += (1 << pixelShift);
963 }
964 out += (kIOHibernateProgressSpacing << pixelShift);
965 }
966 }
967}
968
969/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
970
316670eb
A		 971IOReturn
972IOHibernateIOKitSleep(void)
973{
974 IOReturn ret = kIOReturnSuccess;
975 IOLockLock(gFSLock);
976 if (kFSOpening == gFSState)
977 {
978 gFSState = kFSTimedOut;
979 HIBLOG("hibernate file open timed out\n");
980 ret = kIOReturnTimeout;
981 }
982 IOLockUnlock(gFSLock);
983 return (ret);
984}
985
986/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
987
3a60a9f5
A		 988IOReturn
989IOHibernateSystemHasSlept(void)
990{
316670eb 991 IOReturn ret = kIOReturnSuccess;
3a60a9f5 992 IOHibernateVars * vars = &gIOHibernateVars;
316670eb 993 OSObject * obj = 0;
2d21ac55
A		 994 OSData * data;
995
316670eb
A		 996 IOLockLock(gFSLock);
997 if ((kFSOpened != gFSState) && gIOHibernateMode)
998 {
999 ret = kIOReturnTimeout;
1000 }
1001 IOLockUnlock(gFSLock);
1002 if (kIOReturnSuccess != ret) return (ret);
1003
1004 if (gIOHibernateMode) obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
2d21ac55
A		 1005 vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
1006 if (obj && !vars->previewBuffer)
1007 obj->release();
3a60a9f5 1008
2d21ac55
A		 1009 vars->consoleMapping = NULL;
1010 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
3a60a9f5 1011 {
2d21ac55
A		 1012 vars->previewBuffer->release();
1013 vars->previewBuffer = 0;
1014 }
3a60a9f5 1015
0b4c1975
A		 1016 if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1017 && vars->previewBuffer
1018 && (data = OSDynamicCast(OSData,
2d21ac55
A		 1019 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey))))
1020 {
1021 UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
b0d623f7 1022 HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
2d21ac55
A		 1023
1024 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
1025
1026 if (kIOHibernatePreviewUpdates & flags)
1027 {
1028 PE_Video consoleInfo;
1029 hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
1030
1031 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1032
1033 graphicsInfo->width = consoleInfo.v_width;
1034 graphicsInfo->height = consoleInfo.v_height;
1035 graphicsInfo->rowBytes = consoleInfo.v_rowBytes;
1036 graphicsInfo->depth = consoleInfo.v_depth;
1037 vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
1038
1039 HIBPRINT("video %p %d %d %d\n",
0b4c1975
A		 1040 vars->consoleMapping, graphicsInfo->depth,
1041 graphicsInfo->width, graphicsInfo->height);
2d21ac55
A		 1042 if (vars->consoleMapping)
1043 ProgressInit(graphicsInfo, vars->consoleMapping,
1044 &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
1045 }
3a60a9f5 1046 }
2d21ac55 1047
3a60a9f5
A		 1048 if (gIOOptionsEntry)
1049 gIOOptionsEntry->sync();
1050
316670eb 1051 return (ret);
3a60a9f5
A		 1052}
1053
1054/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1055
6d2010ae
A		 1056static DeviceTreeNode *
1057MergeDeviceTree(DeviceTreeNode * entry, IORegistryEntry * regEntry)
1058{
1059 DeviceTreeNodeProperty * prop;
1060 DeviceTreeNode * child;
1061 IORegistryEntry * childRegEntry;
1062 const char * nameProp;
1063 unsigned int propLen, idx;
1064
1065 prop = (DeviceTreeNodeProperty *) (entry + 1);
1066 for (idx = 0; idx < entry->nProperties; idx++)
1067 {
1068 if (regEntry && (0 != strcmp("name", prop->name)))
1069 {
1070 regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
1071// HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
1072 }
1073 prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
1074 }
1075
1076 child = (DeviceTreeNode *) prop;
1077 for (idx = 0; idx < entry->nChildren; idx++)
1078 {
1079 if (kSuccess != DTGetProperty(child, "name", (void **) &nameProp, &propLen))
1080 panic("no name");
1081 childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
1082// HIBPRINT("%s == %p\n", nameProp, childRegEntry);
1083 child = MergeDeviceTree(child, childRegEntry);
1084 }
1085 return (child);
1086}
1087
3a60a9f5
A		 1088IOReturn
1089IOHibernateSystemWake(void)
1090{
316670eb
A		 1091 if (kFSOpened == gFSState)
1092 {
3e170ce0 1093 IOPolledFilePollersClose(gIOHibernateVars.fileVars, kIOPolledPostflightState);
316670eb
A		 1094 IOHibernateDone(&gIOHibernateVars);
1095 }
1096 else
1097 {
1098 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1099 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1100 }
1101 return (kIOReturnSuccess);
1102}
3a60a9f5 1103
316670eb
A		 1104static IOReturn
1105IOHibernateDone(IOHibernateVars * vars)
1106{
99c3a104 1107 hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);
3a60a9f5
A		 1108
1109 if (vars->videoMapping)
1110 {
1111 if (vars->videoMapSize)
1112 // remove mappings
1113 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1114 if (vars->videoAllocSize)
1115 // dealloc range
b0d623f7 1116 kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
3a60a9f5
A		 1117 }
1118
1119 if (vars->previewBuffer)
1120 {
1121 vars->previewBuffer->release();
1122 vars->previewBuffer = 0;
1123 }
1124
0b4c1975
A		 1125 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1126 {
1127 IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey,
1128 gIOHibernateCurrentHeader->options, 32);
1129 }
1130 else
1131 {
1132 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1133 }
1134
1135 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1136 && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus))
1137 {
1138 IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey,
1139 &gIOHibernateGraphicsInfo->gfxStatus,
1140 sizeof(gIOHibernateGraphicsInfo->gfxStatus));
1141 }
1142 else
1143 {
1144 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1145 }
1146
3a60a9f5
A		 1147 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
1148
b0d623f7 1149#if defined(__i386__) || defined(__x86_64__)
060df5ea 1150 IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
6d2010ae 1151 IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
2d21ac55
A		 1152
1153 /*
1154 * Hibernate variable is written to NVRAM on platforms in which RtcRam
1155 * is not backed by coin cell. Remove Hibernate data from NVRAM.
1156 */
1157 if (gIOOptionsEntry) {
2d21ac55 1158
060df5ea
A		 1159 if (gIOHibernateRTCVariablesKey) {
1160 if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
1161 gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
1162 }
1163 }
1164
1165 if (gIOHibernateBootNextKey)
1166 {
1167 if (gIOHibernateBootNextSave)
1168 {
1169 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
1170 gIOHibernateBootNextSave->release();
1171 gIOHibernateBootNextSave = NULL;
2d21ac55 1172 }
060df5ea
A		 1173 else
1174 gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
1175 }
39236c6e 1176 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) gIOOptionsEntry->sync();
2d21ac55 1177 }
0c530ab8 1178#endif
3a60a9f5 1179
3e170ce0 1180 if (vars->srcBuffer) vars->srcBuffer->release();
6d2010ae 1181 bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
99c3a104 1182 if (vars->handoffBuffer)
6d2010ae 1183 {
99c3a104 1184 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
6d2010ae 1185 {
99c3a104
A		 1186 IOHibernateHandoff * handoff;
1187 bool done = false;
1188 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
1189 !done;
1190 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
6d2010ae 1191 {
99c3a104
A		 1192 HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
1193 uint8_t * data = &handoff->data[0];
1194 switch (handoff->type)
1195 {
1196 case kIOHibernateHandoffTypeEnd:
1197 done = true;
1198 break;
6d2010ae 1199
99c3a104
A		 1200 case kIOHibernateHandoffTypeDeviceTree:
1201 MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot());
1202 break;
1203
1204 case kIOHibernateHandoffTypeKeyStore:
6d2010ae 1205#if defined(__i386__) || defined(__x86_64__)
6d2010ae 1206 {
99c3a104
A		 1207 IOBufferMemoryDescriptor *
1208 md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
1209 if (md)
1210 {
1211 IOSetKeyStoreData(md);
1212 }
6d2010ae 1213 }
6d2010ae 1214#endif
99c3a104
A		 1215 break;
1216
1217 default:
1218 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
1219 break;
1220 }
1221 }
6d2010ae
A		 1222 }
1223 vars->handoffBuffer->release();
1224 }
3a60a9f5
A		 1225
1226 bzero(vars, sizeof(*vars));
1227
1228// gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
1229
1230 return (kIOReturnSuccess);
1231}
1232
1233IOReturn
1234IOHibernateSystemPostWake(void)
1235{
3e170ce0 1236 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
316670eb 1237 if (kFSOpened == gFSState)
3a60a9f5 1238 {
6d2010ae 1239 // invalidate & close the image file
3e170ce0
A		 1240 IOSleep(TRIM_DELAY);
1241 IOPolledFileIOVars * vars = &gFileVars;
1242 IOPolledFileClose(&vars,
39236c6e 1243#if DISABLE_TRIM
3e170ce0 1244 0, NULL, 0, 0, 0);
39236c6e 1245#else
3e170ce0
A		 1246 0, (caddr_t)gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader),
1247 sizeof(IOHibernateImageHeader), gIOHibernateCurrentHeader->imageSize);
39236c6e 1248#endif
fe8ab488 1249 }
3e170ce0 1250 gFSState = kFSIdle;
fe8ab488 1251
3e170ce0 1252 IOSetBootImageNVRAM(0);
fe8ab488 1253
3e170ce0
A		 1254 if (gDebugImageLock) {
1255 IOLockLock(gDebugImageLock);
1256 if (gDebugImageFileVars != 0) {
1257 kprintf("IOHibernateSystemPostWake: Closing debugdata file\n");
1258 IOPolledFileClose(&gDebugImageFileVars, 0, 0, 0, 0, 0);
fe8ab488 1259 }
3e170ce0 1260 IOLockUnlock(gDebugImageLock);
fe8ab488
A		 1261 }
1262
3a60a9f5
A		 1263 return (kIOReturnSuccess);
1264}
1265
316670eb
A		 1266bool IOHibernateWasScreenLocked(void)
1267{
1268 bool ret = false;
1269 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry)
1270 {
1271 OSData *
1272 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
1273 if (data) switch (*((uint32_t *)data->getBytesNoCopy()))
1274 {
1275 case kIOScreenLockLocked:
1276 case kIOScreenLockFileVaultDialog:
1277 ret = true;
1278 break;
1279 case kIOScreenLockNoLock:
1280 case kIOScreenLockUnlocked:
1281 default:
1282 ret = false;
1283 break;
1284 }
1285 }
1286 return (ret);
1287}
1288
3a60a9f5
A		 1289/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1290
2d21ac55 1291SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
6d2010ae 1292 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2d21ac55
A		 1293 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
1294SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
6d2010ae 1295 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2d21ac55
A		 1296 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
1297SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
6d2010ae 1298 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2d21ac55 1299 &gIOHibernateMode, 0, "");
39236c6e
A		 1300SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
1301 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
1302 gIOHibernateStats, hibernate_statistics_t, "");
1303
1304SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
1305 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1306 &gIOHibernateStats->graphicsReadyTime, 0, "");
1307SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
1308 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1309 &gIOHibernateStats->wakeNotificationTime, 0, "");
1310SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
1311 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1312 &gIOHibernateStats->lockScreenReadyTime, 0, "");
1313SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
1314 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
1315 &gIOHibernateStats->hidReadyTime, 0, "");
1316
2d21ac55 1317
3a60a9f5
A		 1318void
1319IOHibernateSystemInit(IOPMrootDomain * rootDomain)
1320{
3e170ce0
A		 1321 gIOHibernateBootImageKey = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1322
1323#if defined(__i386__) || defined(__x86_64__)
1324 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1325 gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
1326 gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
1327 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1328#endif /* defined(__i386__) || defined(__x86_64__) */
1329
3a60a9f5
A		 1330 OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
1331 if (data)
1332 {
1333 rootDomain->setProperty(kIOHibernateStateKey, data);
1334 data->release();
1335 }
1336
2d21ac55 1337 if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename)))
3a60a9f5
A		 1338 gIOHibernateMode = kIOHibernateModeOn;
1339 else
1340 gIOHibernateFilename[0] = 0;
1341
3a60a9f5 1342 sysctl_register_oid(&sysctl__kern_hibernatefile);
3a60a9f5 1343 sysctl_register_oid(&sysctl__kern_bootsignature);
3a60a9f5 1344 sysctl_register_oid(&sysctl__kern_hibernatemode);
39236c6e
A		 1345 sysctl_register_oid(&sysctl__kern_hibernatestatistics);
1346 sysctl_register_oid(&sysctl__kern_hibernategraphicsready);
1347 sysctl_register_oid(&sysctl__kern_hibernatewakenotification);
1348 sysctl_register_oid(&sysctl__kern_hibernatelockscreenready);
1349 sysctl_register_oid(&sysctl__kern_hibernatehidready);
316670eb 1350
3e170ce0
A		 1351 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1352
316670eb 1353 gFSLock = IOLockAlloc();
3a60a9f5
A		 1354}
1355
2d21ac55 1356
3a60a9f5
A		 1357/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1358
1359static void
1360hibernate_setup_for_wake(void)
1361{
3a60a9f5
A		 1362}
1363
1364/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1365
3e170ce0
A		 1366static IOReturn
1367IOHibernatePolledFileWrite(IOPolledFileIOVars * vars,
1368 const uint8_t * bytes, IOByteCount size,
1369 IOPolledFileCryptVars * cryptvars)
6d2010ae 1370{
3e170ce0 1371 IOReturn err;
6d2010ae 1372
3e170ce0
A		 1373 err = IOPolledFileWrite(vars, bytes, size, cryptvars);
1374 if ((kIOReturnSuccess == err) && hibernate_should_abort()) err = kIOReturnAborted;
6d2010ae 1375
3e170ce0 1376 return (err);
6d2010ae
A		 1377}
1378
3e170ce0 1379/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
6d2010ae 1380
2d21ac55 1381extern "C" uint32_t
3a60a9f5
A		 1382hibernate_write_image(void)
1383{
1384 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
1385 IOHibernateVars * vars = &gIOHibernateVars;
1386 IOPolledFileExtent * fileExtents;
1387
3e170ce0 1388 assert_static(sizeof(IOHibernateImageHeader) == 512);
0c530ab8 1389
3a60a9f5
A		 1390 uint32_t pageCount, pagesDone;
1391 IOReturn err;
0b4c1975
A		 1392 vm_offset_t ppnum, page;
1393 IOItemCount count;
3a60a9f5
A		 1394 uint8_t * src;
1395 uint8_t * data;
39236c6e
A		 1396 uint8_t * compressed;
1397 uint8_t * scratch;
3e170ce0 1398 IOByteCount pageCompressedSize;
3a60a9f5
A		 1399 uint64_t compressedSize, uncompressedSize;
1400 uint64_t image1Size = 0;
1401 uint32_t bitmap_size;
0b4c1975 1402 bool iterDone, pollerOpen, needEncrypt;
3a60a9f5 1403 uint32_t restore1Sum, sum, sum1, sum2;
39236c6e 1404 int wkresult;
3a60a9f5
A		 1405 uint32_t tag;
1406 uint32_t pageType;
1407 uint32_t pageAndCount[2];
6d2010ae
A		 1408 addr64_t phys64;
1409 IOByteCount segLen;
3e170ce0
A		 1410 uintptr_t hibernateBase;
1411 uintptr_t hibernateEnd;
3a60a9f5
A		 1412
1413 AbsoluteTime startTime, endTime;
6d2010ae 1414 AbsoluteTime allTime, compTime;
0b4c1975 1415 uint64_t compBytes;
3a60a9f5
A		 1416 uint64_t nsec;
1417 uint32_t lastProgressStamp = 0;
1418 uint32_t progressStamp;
2d21ac55 1419 uint32_t blob, lastBlob = (uint32_t) -1L;
3a60a9f5 1420
39236c6e
A		 1421 uint32_t wiredPagesEncrypted;
1422 uint32_t dirtyPagesEncrypted;
1423 uint32_t wiredPagesClear;
3e170ce0
A		 1424 uint32_t svPageCount;
1425 uint32_t zvPageCount;
39236c6e 1426
3e170ce0
A		 1427 IOPolledFileCryptVars _cryptvars;
1428 IOPolledFileCryptVars * cryptvars = 0;
3a60a9f5 1429
39236c6e
A		 1430 wiredPagesEncrypted = 0;
1431 dirtyPagesEncrypted = 0;
1432 wiredPagesClear = 0;
3e170ce0
A		 1433 svPageCount = 0;
1434 zvPageCount = 0;
0b4c1975 1435
3e170ce0 1436 if (!vars->fileVars || !vars->fileVars->pollers)
3a60a9f5
A		 1437 return (false /* sleep */ );
1438
0b4c1975
A		 1439 if (kIOHibernateModeSleep & gIOHibernateMode)
1440 kdebug_enable = save_kdebug_enable;
1441
1442 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START, 0, 0, 0, 0, 0);
6d2010ae 1443 IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
0b4c1975 1444
3a60a9f5
A		 1445 restore1Sum = sum1 = sum2 = 0;
1446
2d21ac55 1447#if CRYPTO
3a60a9f5
A		 1448 // encryption data. "iv" is the "initial vector".
1449 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1450 {
1451 static const unsigned char first_iv[AES_BLOCK_SIZE]
1452 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
1453 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
1454
1455 cryptvars = &gIOHibernateCryptWakeContext;
3e170ce0 1456 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
3a60a9f5
A		 1457 aes_encrypt_key(vars->cryptKey,
1458 kIOHibernateAESKeySize,
1459 &cryptvars->ctx.encrypt);
1460 aes_decrypt_key(vars->cryptKey,
1461 kIOHibernateAESKeySize,
1462 &cryptvars->ctx.decrypt);
1463
1464 cryptvars = &_cryptvars;
3e170ce0 1465 bzero(cryptvars, sizeof(IOPolledFileCryptVars));
6d2010ae
A		 1466 for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++)
1467 vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
1468 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
3a60a9f5
A		 1469 aes_encrypt_key(vars->wiredCryptKey,
1470 kIOHibernateAESKeySize,
1471 &cryptvars->ctx.encrypt);
1472
1473 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1474 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1475 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
3a60a9f5 1476 }
2d21ac55 1477#endif /* CRYPTO */
3a60a9f5
A		 1478
1479 hibernate_setup_for_wake();
1480
1481 hibernate_page_list_setall(vars->page_list,
1482 vars->page_list_wired,
db609669
A		 1483 vars->page_list_pal,
1484 false /* !preflight */,
39236c6e
A		 1485 /* discard_all */
1486 ((0 == (kIOHibernateModeSleep & gIOHibernateMode))
1487 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))),
3a60a9f5
A		 1488 &pageCount);
1489
1490 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
1491
3e170ce0 1492 fileExtents = (IOPolledFileExtent *) vars->fileVars->fileExtents->getBytesNoCopy();
3a60a9f5
A		 1493
1494#if 0
1495 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
1496 for (page = 0; page < count; page++)
1497 {
1498 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
1499 fileExtents[page].start, fileExtents[page].length,
1500 fileExtents[page].start + fileExtents[page].length);
1501 }
1502#endif
1503
0b4c1975 1504 needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
3a60a9f5 1505 AbsoluteTime_to_scalar(&compTime) = 0;
6d2010ae 1506 compBytes = 0;
3a60a9f5
A		 1507
1508 clock_get_uptime(&allTime);
b0d623f7
A		 1509 IOService::getPMRootDomain()->pmStatsRecordEvent(
1510 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
3a60a9f5
A		 1511 do
1512 {
1513 compressedSize = 0;
1514 uncompressedSize = 0;
3e170ce0
A		 1515 svPageCount = 0;
1516 zvPageCount = 0;
3a60a9f5 1517
39236c6e 1518 IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);
3a60a9f5
A		 1519
1520 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
1521 ml_get_interrupts_enabled());
3e170ce0 1522 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledBeforeSleepState, true);
3a60a9f5
A		 1523 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
1524 pollerOpen = (kIOReturnSuccess == err);
1525 if (!pollerOpen)
1526 break;
1527
1528 // copy file block extent list if larger than header
1529
3e170ce0 1530 count = vars->fileVars->fileExtents->getLength();
3a60a9f5
A		 1531 if (count > sizeof(header->fileExtentMap))
1532 {
1533 count -= sizeof(header->fileExtentMap);
3e170ce0 1534 err = IOHibernatePolledFileWrite(vars->fileVars,
3a60a9f5
A		 1535 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
1536 if (kIOReturnSuccess != err)
1537 break;
1538 }
1539
6d2010ae 1540 hibernateBase = HIB_BASE; /* Defined in PAL headers */
316670eb 1541 hibernateEnd = (segHIBB + segSizeHIB);
6d2010ae 1542
3a60a9f5 1543 // copy out restore1 code
6d2010ae
A		 1544
1545 for (count = 0;
1546 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1547 count += segLen)
1548 {
1549 for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++)
1550 {
1551 gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone;
1552 }
1553 }
1554
1555 page = atop_32(kvtophys(hibernateBase));
1556 count = atop_32(round_page(hibernateEnd) - hibernateBase);
1557 header->restore1CodePhysPage = page;
1558 header->restore1CodeVirt = hibernateBase;
3a60a9f5 1559 header->restore1PageCount = count;
b0d623f7
A		 1560 header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
1561 header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
3a60a9f5 1562
316670eb 1563 // sum __HIB seg, with zeros for the stack
b0d623f7 1564 src = (uint8_t *) trunc_page(hibernateBase);
3a60a9f5
A		 1565 for (page = 0; page < count; page++)
1566 {
1567 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
6d2010ae 1568 restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page);
3a60a9f5 1569 else
0b4c1975 1570 restore1Sum += 0x00000000;
3a60a9f5
A		 1571 src += page_size;
1572 }
1573 sum1 = restore1Sum;
1574
316670eb 1575 // write the __HIB seg, with zeros for the stack
3a60a9f5 1576
b0d623f7
A		 1577 src = (uint8_t *) trunc_page(hibernateBase);
1578 count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
3a60a9f5
A		 1579 if (count)
1580 {
3e170ce0 1581 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
3a60a9f5
A		 1582 if (kIOReturnSuccess != err)
1583 break;
1584 }
3e170ce0 1585 err = IOHibernatePolledFileWrite(vars->fileVars,
3a60a9f5
A		 1586 (uint8_t *) 0,
1587 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
1588 cryptvars);
1589 if (kIOReturnSuccess != err)
1590 break;
1591 src = &gIOHibernateRestoreStackEnd[0];
b0d623f7 1592 count = round_page(hibernateEnd) - ((uintptr_t) src);
3a60a9f5
A		 1593 if (count)
1594 {
3e170ce0 1595 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
3a60a9f5
A		 1596 if (kIOReturnSuccess != err)
1597 break;
1598 }
1599
39236c6e
A		 1600 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1601 {
1602 vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
1603 vars->fileVars->encryptEnd = UINT64_MAX;
1604 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1605 }
316670eb 1606
3a60a9f5
A		 1607 // write the preview buffer
1608
2d21ac55 1609 if (vars->previewBuffer)
3a60a9f5
A		 1610 {
1611 ppnum = 0;
1612 count = 0;
1613 do
1614 {
b0d623f7 1615 phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
3a60a9f5
A		 1616 pageAndCount[0] = atop_64(phys64);
1617 pageAndCount[1] = atop_32(segLen);
3e170ce0 1618 err = IOHibernatePolledFileWrite(vars->fileVars,
3a60a9f5
A		 1619 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1620 cryptvars);
1621 if (kIOReturnSuccess != err)
1622 break;
1623 count += segLen;
1624 ppnum += sizeof(pageAndCount);
1625 }
1626 while (phys64);
1627 if (kIOReturnSuccess != err)
1628 break;
1629
b0d623f7 1630 src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
316670eb
A		 1631
1632 ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
1633
2d21ac55 1634 count = vars->previewBuffer->getLength();
3a60a9f5
A		 1635
1636 header->previewPageListSize = ppnum;
1637 header->previewSize = count + ppnum;
1638
1639 for (page = 0; page < count; page += page_size)
0b4c1975
A		 1640 {
1641 phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
1642 sum1 += hibernate_sum_page(src + page, atop_64(phys64));
1643 }
3e170ce0 1644 err = IOHibernatePolledFileWrite(vars->fileVars, src, count, cryptvars);
3a60a9f5
A		 1645 if (kIOReturnSuccess != err)
1646 break;
1647 }
1648
1649 // mark areas for no save
3e170ce0
A		 1650 IOMemoryDescriptor * ioBuffer;
1651 ioBuffer = IOPolledFileGetIOBuffer(vars->fileVars);
3a60a9f5 1652 for (count = 0;
3e170ce0 1653 (phys64 = ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
3a60a9f5
A		 1654 count += segLen)
1655 {
1656 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1657 atop_64(phys64), atop_32(segLen),
1658 kIOHibernatePageStateFree);
1659 pageCount -= atop_32(segLen);
1660 }
1661
1662 for (count = 0;
b0d623f7 1663 (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
3a60a9f5
A		 1664 count += segLen)
1665 {
1666 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1667 atop_64(phys64), atop_32(segLen),
1668 kIOHibernatePageStateFree);
1669 pageCount -= atop_32(segLen);
1670 }
1671
1672 // copy out bitmap of pages available for trashing during restore
1673
1674 bitmap_size = vars->page_list_wired->list_size;
1675 src = (uint8_t *) vars->page_list_wired;
3e170ce0 1676 err = IOHibernatePolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
3a60a9f5
A		 1677 if (kIOReturnSuccess != err)
1678 break;
1679
1680 // mark more areas for no save, but these are not available
1681 // for trashing during restore
0c530ab8
A		 1682
1683 hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
3a60a9f5 1684
6d2010ae
A		 1685
1686 page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
1687 count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
3a60a9f5
A		 1688 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1689 page, count,
1690 kIOHibernatePageStateFree);
1691 pageCount -= count;
3a60a9f5
A		 1692
1693 if (vars->previewBuffer) for (count = 0;
b0d623f7 1694 (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
3a60a9f5
A		 1695 count += segLen)
1696 {
1697 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1698 atop_64(phys64), atop_32(segLen),
1699 kIOHibernatePageStateFree);
1700 pageCount -= atop_32(segLen);
1701 }
1702
6d2010ae
A		 1703 for (count = 0;
1704 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
1705 count += segLen)
1706 {
1707 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1708 atop_64(phys64), atop_32(segLen),
1709 kIOHibernatePageStateFree);
1710 pageCount -= atop_32(segLen);
1711 }
1712
3a60a9f5 1713 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
39236c6e
A		 1714 compressed = src + page_size;
1715 scratch = compressed + page_size;
1716
6d2010ae
A		 1717 pagesDone = 0;
1718 lastBlob = 0;
3a60a9f5 1719
db609669
A		 1720 HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
1721 bitmap_size, header->previewSize,
1722 pageCount, vars->fileVars->position);
3a60a9f5 1723
0b4c1975
A		 1724 enum
1725 // pageType
1726 {
1727 kWired = 0x02,
1728 kEncrypt = 0x01,
1729 kWiredEncrypt = kWired | kEncrypt,
1730 kWiredClear = kWired,
1731 kUnwiredEncrypt = kEncrypt
1732 };
1733
1734 for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--)
3a60a9f5 1735 {
316670eb
A		 1736 if (kUnwiredEncrypt == pageType)
1737 {
1738 // start unwired image
39236c6e
A		 1739 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1740 {
1741 vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
1742 vars->fileVars->encryptEnd = UINT64_MAX;
1743 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
1744 }
316670eb
A		 1745 bcopy(&cryptvars->aes_iv[0],
1746 &gIOHibernateCryptWakeContext.aes_iv[0],
1747 sizeof(cryptvars->aes_iv));
1748 cryptvars = &gIOHibernateCryptWakeContext;
0b4c1975
A		 1749 }
1750 for (iterDone = false, ppnum = 0; !iterDone; )
1751 {
1752 count = hibernate_page_list_iterate((kWired & pageType)
1753 ? vars->page_list_wired : vars->page_list,
1754 &ppnum);
1755// kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
1756 iterDone = !count;
3a60a9f5 1757
0b4c1975 1758 if (count && (kWired & pageType) && needEncrypt)
3a60a9f5 1759 {
0b4c1975
A		 1760 uint32_t checkIndex;
1761 for (checkIndex = 0;
1762 (checkIndex < count)
3e170ce0 1763 && (((kEncrypt & pageType) == 0) == pmap_is_noencrypt(ppnum + checkIndex));
0b4c1975
A		 1764 checkIndex++)
1765 {}
1766 if (!checkIndex)
1767 {
1768 ppnum++;
1769 continue;
1770 }
1771 count = checkIndex;
3a60a9f5
A		 1772 }
1773
0b4c1975
A		 1774 switch (pageType)
1775 {
39236c6e
A		 1776 case kWiredEncrypt: wiredPagesEncrypted += count; break;
1777 case kWiredClear: wiredPagesClear += count; break;
1778 case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
0b4c1975
A		 1779 }
1780
1781 if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */}
3a60a9f5 1782 else
3a60a9f5 1783 {
0b4c1975
A		 1784 pageAndCount[0] = ppnum;
1785 pageAndCount[1] = count;
3e170ce0 1786 err = IOHibernatePolledFileWrite(vars->fileVars,
0b4c1975
A		 1787 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1788 cryptvars);
1789 if (kIOReturnSuccess != err)
1790 break;
3a60a9f5 1791 }
0b4c1975
A		 1792
1793 for (page = ppnum; page < (ppnum + count); page++)
3a60a9f5 1794 {
0b4c1975
A		 1795 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
1796 if (err)
1797 {
1798 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
1799 break;
1800 }
1801
1802 sum = hibernate_sum_page(src, page);
1803 if (kWired & pageType)
1804 sum1 += sum;
1805 else
1806 sum2 += sum;
1807
1808 clock_get_uptime(&startTime);
3e170ce0 1809 wkresult = WKdm_compress_new((const WK_word*) src,
39236c6e
A		 1810 (WK_word*) compressed,
1811 (WK_word*) scratch,
1812 page_size - 4);
0b4c1975 1813
3a60a9f5 1814 clock_get_uptime(&endTime);
0b4c1975
A		 1815 ADD_ABSOLUTETIME(&compTime, &endTime);
1816 SUB_ABSOLUTETIME(&compTime, &startTime);
39236c6e 1817
0b4c1975 1818 compBytes += page_size;
39236c6e
A		 1819 pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
1820
3e170ce0 1821 if (pageCompressedSize == 0)
39236c6e 1822 {
3e170ce0 1823 pageCompressedSize = 4;
0b4c1975 1824 data = src;
3e170ce0
A		 1825
1826 if (*(uint32_t *)src)
1827 svPageCount++;
1828 else
1829 zvPageCount++;
1830 }
1831 else
1832 {
1833 if (pageCompressedSize != page_size)
1834 data = compressed;
1835 else
1836 data = src;
1837 }
0b4c1975
A		 1838
1839 tag = pageCompressedSize | kIOHibernateTagSignature;
3e170ce0 1840 err = IOHibernatePolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
0b4c1975
A		 1841 if (kIOReturnSuccess != err)
1842 break;
1843
3e170ce0 1844 err = IOHibernatePolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
0b4c1975
A		 1845 if (kIOReturnSuccess != err)
1846 break;
1847
1848 compressedSize += pageCompressedSize;
39236c6e 1849 uncompressedSize += page_size;
0b4c1975
A		 1850 pagesDone++;
1851
1852 if (vars->consoleMapping && (0 == (1023 & pagesDone)))
1853 {
1854 blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
1855 if (blob != lastBlob)
1856 {
1857 ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
1858 lastBlob = blob;
1859 }
1860 }
1861 if (0 == (8191 & pagesDone))
1862 {
1863 clock_get_uptime(&endTime);
1864 SUB_ABSOLUTETIME(&endTime, &allTime);
1865 absolutetime_to_nanoseconds(endTime, &nsec);
1866 progressStamp = nsec / 750000000ULL;
1867 if (progressStamp != lastProgressStamp)
1868 {
1869 lastProgressStamp = progressStamp;
1870 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
1871 }
3a60a9f5
A		 1872 }
1873 }
0b4c1975
A		 1874 if (kIOReturnSuccess != err)
1875 break;
1876 ppnum = page;
3a60a9f5 1877 }
0b4c1975 1878
3a60a9f5
A		 1879 if (kIOReturnSuccess != err)
1880 break;
0b4c1975 1881
39236c6e 1882 if ((kEncrypt & pageType) && vars->fileVars->encryptStart)
3a60a9f5 1883 {
7ddcb079 1884 vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
0b4c1975
A		 1885 HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
1886 }
1887
1888 if (kWiredEncrypt != pageType)
1889 {
1890 // end of image1/2 - fill to next block
3e170ce0 1891 err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars);
3a60a9f5
A		 1892 if (kIOReturnSuccess != err)
1893 break;
0b4c1975
A		 1894 }
1895 if (kWiredClear == pageType)
1896 {
7ddcb079 1897 // enlarge wired image for test
3e170ce0 1898// err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);
7ddcb079 1899
0b4c1975
A		 1900 // end wired image
1901 header->encryptStart = vars->fileVars->encryptStart;
1902 header->encryptEnd = vars->fileVars->encryptEnd;
3a60a9f5 1903 image1Size = vars->fileVars->position;
7ddcb079 1904 HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
0b4c1975 1905 image1Size, header->encryptStart, header->encryptEnd);
3a60a9f5
A		 1906 }
1907 }
3a60a9f5 1908 if (kIOReturnSuccess != err)
8a3053a0
A		 1909 {
1910 if (kIOReturnOverrun == err)
1911 {
3e170ce0 1912 // update actual compression ratio on not enough space (for retry)
8a3053a0
A		 1913 gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
1914 }
3e170ce0
A		 1915
1916 // update partial amount written (for IOPolledFileClose cleanup/unmap)
1917 header->imageSize = vars->fileVars->position;
3a60a9f5 1918 break;
8a3053a0 1919 }
3a60a9f5
A		 1920
1921 // Header:
1922
1923 header->imageSize = vars->fileVars->position;
1924 header->image1Size = image1Size;
1925 header->bitmapSize = bitmap_size;
1926 header->pageCount = pageCount;
3a60a9f5
A		 1927
1928 header->restore1Sum = restore1Sum;
1929 header->image1Sum = sum1;
1930 header->image2Sum = sum2;
316670eb 1931 header->sleepTime = gIOLastSleepTime.tv_sec;
db609669
A		 1932
1933 header->compression = (compressedSize << 8) / uncompressedSize;
1934 gIOHibernateCompression = header->compression;
3a60a9f5 1935
3e170ce0 1936 count = vars->fileVars->fileExtents->getLength();
3a60a9f5
A		 1937 if (count > sizeof(header->fileExtentMap))
1938 {
1939 header->fileExtentMapSize = count;
1940 count = sizeof(header->fileExtentMap);
1941 }
1942 else
1943 header->fileExtentMapSize = sizeof(header->fileExtentMap);
1944 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
0b4c1975 1945
39236c6e
A		 1946 header->deviceBase = vars->fileVars->block0;
1947 header->deviceBlockSize = vars->fileVars->blockSize;
6d2010ae 1948
3a60a9f5 1949 IOPolledFileSeek(vars->fileVars, 0);
3e170ce0 1950 err = IOHibernatePolledFileWrite(vars->fileVars,
3a60a9f5
A		 1951 (uint8_t *) header, sizeof(IOHibernateImageHeader),
1952 cryptvars);
1953 if (kIOReturnSuccess != err)
1954 break;
3e170ce0 1955 err = IOHibernatePolledFileWrite(vars->fileVars, 0, 0, cryptvars);
3a60a9f5
A		 1956 }
1957 while (false);
1958
1959 clock_get_uptime(&endTime);
b0d623f7
A		 1960
1961 IOService::getPMRootDomain()->pmStatsRecordEvent(
1962 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
1963
3a60a9f5
A		 1964 SUB_ABSOLUTETIME(&endTime, &allTime);
1965 absolutetime_to_nanoseconds(endTime, &nsec);
39236c6e 1966 HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
3a60a9f5
A		 1967
1968 absolutetime_to_nanoseconds(compTime, &nsec);
6d2010ae
A		 1969 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
1970 compBytes,
1971 nsec / 1000000ULL,
1972 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
3a60a9f5 1973
6d2010ae
A		 1974 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
1975 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
1976 vars->fileVars->cryptBytes,
1977 nsec / 1000000ULL,
1978 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
3a60a9f5 1979
db609669
A		 1980 HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
1981 header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
3a60a9f5 1982 uncompressedSize, atop_32(uncompressedSize), compressedSize,
8f6c56a5 1983 uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
3a60a9f5
A		 1984 sum1, sum2);
1985
3e170ce0
A		 1986 HIBLOG("svPageCount %d, zvPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n",
1987 svPageCount, zvPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
c910b4d9 1988
3a60a9f5 1989 if (pollerOpen)
ecc0ceb4 1990 IOPolledFilePollersClose(vars->fileVars, (kIOReturnSuccess == err) ? kIOPolledBeforeSleepState : kIOPolledBeforeSleepStateAborted );
3a60a9f5 1991
2d21ac55
A		 1992 if (vars->consoleMapping)
1993 ProgressUpdate(gIOHibernateGraphicsInfo,
1994 vars->consoleMapping, 0, kIOHibernateProgressCount);
1995
3a60a9f5
A		 1996 HIBLOG("hibernate_write_image done(%x)\n", err);
1997
1998 // should we come back via regular wake, set the state in memory.
1999 gIOHibernateState = kIOHibernateStateInactive;
2000
0b4c1975 2001 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END,
39236c6e 2002 wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted, 0, 0);
0b4c1975 2003
2d21ac55
A		 2004 if (kIOReturnSuccess == err)
2005 {
2006 if (kIOHibernateModeSleep & gIOHibernateMode)
2007 {
2008 return (kIOHibernatePostWriteSleep);
2009 }
2010 else if(kIOHibernateModeRestart & gIOHibernateMode)
2011 {
2012 return (kIOHibernatePostWriteRestart);
2013 }
2014 else
2015 {
2016 /* by default, power down */
2017 return (kIOHibernatePostWriteHalt);
2018 }
2019 }
2020 else if (kIOReturnAborted == err)
2021 {
2022 return (kIOHibernatePostWriteWake);
2023 }
3a60a9f5 2024 else
3a60a9f5 2025 {
2d21ac55
A		 2026 /* on error, sleep */
2027 return (kIOHibernatePostWriteSleep);
3a60a9f5
A		 2028 }
2029}
2030
2031/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2032
2033extern "C" void
2034hibernate_machine_init(void)
2035{
2036 IOReturn err;
2037 uint32_t sum;
2038 uint32_t pagesDone;
0b4c1975 2039 uint32_t pagesRead = 0;
6d2010ae 2040 AbsoluteTime startTime, compTime;
3a60a9f5 2041 AbsoluteTime allTime, endTime;
39236c6e
A		 2042 AbsoluteTime startIOTime, endIOTime;
2043 uint64_t nsec, nsecIO;
6d2010ae 2044 uint64_t compBytes;
3a60a9f5
A		 2045 uint32_t lastProgressStamp = 0;
2046 uint32_t progressStamp;
3e170ce0 2047 IOPolledFileCryptVars * cryptvars = 0;
3a60a9f5
A		 2048
2049 IOHibernateVars * vars = &gIOHibernateVars;
39236c6e 2050 bzero(gIOHibernateStats, sizeof(hibernate_statistics_t));
3a60a9f5 2051
3e170ce0 2052 if (!vars->fileVars || !vars->fileVars->pollers)
3a60a9f5
A		 2053 return;
2054
3a60a9f5
A		 2055 sum = gIOHibernateCurrentHeader->actualImage1Sum;
2056 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
2057
3a60a9f5
A		 2058 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
2059 {
2060 HIBLOG("regular wake\n");
2061 return;
2062 }
2063
2064 HIBPRINT("diag %x %x %x %x\n",
2065 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
6d2010ae 2066 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
3a60a9f5 2067
39236c6e
A		 2068#define t40ms(x) (tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000)
2069#define tStat(x, y) gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y);
2070 tStat(booterStart, booterStart);
2071 gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart,
2072 tStat(booterDuration0, booterTime0);
2073 tStat(booterDuration1, booterTime1);
2074 tStat(booterDuration2, booterTime2);
2075 tStat(booterDuration, booterTime);
2076 tStat(booterConnectDisplayDuration, connectDisplayTime);
2077 tStat(booterSplashDuration, splashTime);
2078 tStat(trampolineDuration, trampolineTime);
2079
2080 gIOHibernateStats->image1Size = gIOHibernateCurrentHeader->image1Size;
2081 gIOHibernateStats->imageSize = gIOHibernateCurrentHeader->imageSize;
2082 gIOHibernateStats->image1Pages = pagesDone;
2083
2084 HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n",
2085 gIOHibernateStats->booterStart,
2086 gIOHibernateStats->smcStart,
2087 gIOHibernateStats->booterDuration0,
2088 gIOHibernateStats->booterDuration1,
2089 gIOHibernateStats->booterDuration2,
2090 gIOHibernateStats->booterDuration,
2091 gIOHibernateStats->booterConnectDisplayDuration,
2092 gIOHibernateStats->booterSplashDuration,
2093 gIOHibernateStats->trampolineDuration);
2094
2095 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, imageSize 0x%qx, image1Size 0x%qx, conflictCount %d, nextFree %x\n",
2096 gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size,
2097 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
bd504ef0 2098
39236c6e
A		 2099 if ((0 != (kIOHibernateModeSleep & gIOHibernateMode))
2100 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)))
2101 {
0c530ab8 2102 hibernate_page_list_discard(vars->page_list);
39236c6e 2103 }
0c530ab8 2104
6d2010ae
A		 2105 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
2106
2107 if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount)
2108 panic("handoff overflow");
2109
2110 IOHibernateHandoff * handoff;
2111 bool done = false;
2112 bool foundCryptData = false;
2113
2114 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
2115 !done;
2116 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
2117 {
2118// HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
2119 uint8_t * data = &handoff->data[0];
2120 switch (handoff->type)
2121 {
2122 case kIOHibernateHandoffTypeEnd:
2123 done = true;
2124 break;
2125
2126 case kIOHibernateHandoffTypeGraphicsInfo:
2127 bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
2128 break;
2129
2130 case kIOHibernateHandoffTypeCryptVars:
2131 if (cryptvars)
2132 {
2133 hibernate_cryptwakevars_t *
2134 wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
2135 bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
2136 }
2137 foundCryptData = true;
2138 bzero(data, handoff->bytecount);
2139 break;
2140
2141 case kIOHibernateHandoffTypeMemoryMap:
bd504ef0
A		 2142
2143 clock_get_uptime(&allTime);
2144
6d2010ae
A		 2145 hibernate_newruntime_map(data, handoff->bytecount,
2146 gIOHibernateCurrentHeader->systemTableOffset);
bd504ef0
A		 2147
2148 clock_get_uptime(&endTime);
2149
2150 SUB_ABSOLUTETIME(&endTime, &allTime);
2151 absolutetime_to_nanoseconds(endTime, &nsec);
2152
2153 HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL);
2154
6d2010ae
A		 2155 break;
2156
2157 case kIOHibernateHandoffTypeDeviceTree:
2158 {
2159// DTEntry chosen = NULL;
2160// HIBPRINT("DTLookupEntry %d\n", DTLookupEntry((const DTEntry) data, "/chosen", &chosen));
2161 }
2162 break;
2163
2164 default:
2165 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
2166 break;
2167 }
2168 }
2169 if (cryptvars && !foundCryptData)
2170 panic("hibernate handoff");
2171
316670eb
A		 2172 HIBPRINT("video %x %d %d %d status %x\n",
2173 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
2174 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus);
2175
2176 if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress)
3a60a9f5
A		 2177 {
2178 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
2179 * gIOHibernateGraphicsInfo->rowBytes);
22ba694c
A		 2180 if (vars->videoMapSize > vars->videoAllocSize) vars->videoMapSize = 0;
2181 else
2182 {
2183 IOMapPages(kernel_map,
2184 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
2185 vars->videoMapSize, kIOMapInhibitCache );
2186 }
3a60a9f5
A		 2187 }
2188
316670eb
A		 2189 if (vars->videoMapSize)
2190 ProgressUpdate(gIOHibernateGraphicsInfo,
2191 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2192
0c530ab8 2193 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
39236c6e
A		 2194 uint8_t * compressed = src + page_size;
2195 uint8_t * scratch = compressed + page_size;
2196 uint32_t decoOffset;
3a60a9f5
A		 2197
2198 clock_get_uptime(&allTime);
6d2010ae
A		 2199 AbsoluteTime_to_scalar(&compTime) = 0;
2200 compBytes = 0;
3a60a9f5 2201
3e170ce0
A		 2202 HIBLOG("IOPolledFilePollersOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
2203 err = IOPolledFilePollersOpen(vars->fileVars, kIOPolledAfterSleepState, false);
39236c6e
A		 2204 clock_get_uptime(&startIOTime);
2205 endTime = startIOTime;
2206 SUB_ABSOLUTETIME(&endTime, &allTime);
2207 absolutetime_to_nanoseconds(endTime, &nsec);
3e170ce0 2208 HIBLOG("IOPolledFilePollersOpen(%x) %qd ms\n", err, nsec / 1000000ULL);
3a60a9f5 2209
3a60a9f5
A		 2210 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
2211
3a60a9f5 2212 // kick off the read ahead
3a60a9f5
A		 2213 vars->fileVars->bufferHalf = 0;
2214 vars->fileVars->bufferLimit = 0;
2215 vars->fileVars->lastRead = 0;
6d2010ae 2216 vars->fileVars->readEnd = gIOHibernateCurrentHeader->imageSize;
3a60a9f5 2217 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
6d2010ae
A		 2218 vars->fileVars->cryptBytes = 0;
2219 AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;
3a60a9f5 2220
6d2010ae 2221 err = IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
3a60a9f5
A		 2222 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2223 // --
2224
2225 HIBLOG("hibernate_machine_init reading\n");
2226
2227 uint32_t * header = (uint32_t *) src;
2228 sum = 0;
2229
6d2010ae 2230 while (kIOReturnSuccess == err)
3a60a9f5
A		 2231 {
2232 unsigned int count;
2233 unsigned int page;
2234 uint32_t tag;
2235 vm_offset_t ppnum, compressedSize;
2236
36401178
A		 2237 err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
2238 if (kIOReturnSuccess != err)
2239 break;
3a60a9f5
A		 2240
2241 ppnum = header[0];
2242 count = header[1];
2243
2244// HIBPRINT("(%x, %x)\n", ppnum, count);
2245
2246 if (!count)
2247 break;
2248
2249 for (page = 0; page < count; page++)
2250 {
36401178
A		 2251 err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
2252 if (kIOReturnSuccess != err)
2253 break;
3a60a9f5
A		 2254
2255 compressedSize = kIOHibernateTagLength & tag;
36401178
A		 2256 if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength))
2257 {
2258 err = kIOReturnIPCError;
2259 break;
2260 }
2261
3e170ce0
A		 2262 err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
2263 if (kIOReturnSuccess != err) break;
2264
2265 if (compressedSize < page_size)
36401178 2266 {
3e170ce0
A		 2267 decoOffset = page_size;
2268 clock_get_uptime(&startTime);
2269
2270 if (compressedSize == 4) {
2271 int i;
2272 uint32_t *s, *d;
2273
2274 s = (uint32_t *)src;
2275 d = (uint32_t *)(uintptr_t)compressed;
2276
2277 for (i = 0; i < (int)(PAGE_SIZE / sizeof(int32_t)); i++)
2278 *d++ = *s;
39236c6e 2279 }
3e170ce0
A		 2280 else
2281 WKdm_decompress_new((WK_word*) src, (WK_word*) compressed, (WK_word*) scratch, compressedSize);
2282 clock_get_uptime(&endTime);
2283 ADD_ABSOLUTETIME(&compTime, &endTime);
2284 SUB_ABSOLUTETIME(&compTime, &startTime);
2285 compBytes += page_size;
2286 }
2287 else decoOffset = 0;
39236c6e 2288
3e170ce0
A		 2289 sum += hibernate_sum_page((src + decoOffset), ppnum);
2290 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
2291 if (err)
2292 {
39236c6e
A		 2293 HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
2294 break;
36401178 2295 }
3a60a9f5
A		 2296
2297 ppnum++;
2298 pagesDone++;
0b4c1975 2299 pagesRead++;
3a60a9f5 2300
3a60a9f5
A		 2301 if (0 == (8191 & pagesDone))
2302 {
2303 clock_get_uptime(&endTime);
2304 SUB_ABSOLUTETIME(&endTime, &allTime);
2305 absolutetime_to_nanoseconds(endTime, &nsec);
2306 progressStamp = nsec / 750000000ULL;
2307 if (progressStamp != lastProgressStamp)
2308 {
2309 lastProgressStamp = progressStamp;
2310 HIBPRINT("pages %d (%d%%)\n", pagesDone,
2311 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
2312 }
2313 }
2314 }
2315 }
7ddcb079
A		 2316 if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages))
2317 err = kIOReturnLockedRead;
3a60a9f5 2318
36401178
A		 2319 if (kIOReturnSuccess != err)
2320 panic("Hibernate restore error %x", err);
2321
3a60a9f5 2322 gIOHibernateCurrentHeader->actualImage2Sum = sum;
db609669 2323 gIOHibernateCompression = gIOHibernateCurrentHeader->compression;
3a60a9f5 2324
39236c6e
A		 2325 clock_get_uptime(&endIOTime);
2326
3e170ce0 2327 err = IOPolledFilePollersClose(vars->fileVars, kIOPolledAfterSleepState);
3a60a9f5 2328
3a60a9f5 2329 clock_get_uptime(&endTime);
b0d623f7
A		 2330
2331 IOService::getPMRootDomain()->pmStatsRecordEvent(
2332 kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
2333 IOService::getPMRootDomain()->pmStatsRecordEvent(
2334 kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
2335
3a60a9f5
A		 2336 SUB_ABSOLUTETIME(&endTime, &allTime);
2337 absolutetime_to_nanoseconds(endTime, &nsec);
2338
39236c6e
A		 2339 SUB_ABSOLUTETIME(&endIOTime, &startIOTime);
2340 absolutetime_to_nanoseconds(endIOTime, &nsecIO);
2341
2342 gIOHibernateStats->kernelImageReadDuration = nsec / 1000000ULL;
2343 gIOHibernateStats->imagePages = pagesDone;
2344
2345 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ",
2346 pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize,
2347 nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0);
6d2010ae
A		 2348
2349 absolutetime_to_nanoseconds(compTime, &nsec);
2350 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2351 compBytes,
2352 nsec / 1000000ULL,
2353 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2354
2355 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2356 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n",
2357 vars->fileVars->cryptBytes,
2358 nsec / 1000000ULL,
2359 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
2360
2361 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_NONE, pagesRead, pagesDone, 0, 0, 0);
3a60a9f5
A		 2362}
2363
2364/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
99c3a104 2365
39236c6e
A		 2366void IOHibernateSetWakeCapabilities(uint32_t capability)
2367{
2368 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
2369 {
2370 gIOHibernateStats->wakeCapability = capability;
8a3053a0
A		 2371
2372 if (kIOPMSystemCapabilityGraphics & capability)
2373 {
2374 vm_compressor_do_warmup();
2375 }
39236c6e
A		 2376 }
2377}
2378
2379/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2380
99c3a104
A		 2381void IOHibernateSystemRestart(void)
2382{
2383 static uint8_t noteStore[32] __attribute__((aligned(32)));
2384 IORegistryEntry * regEntry;
2385 const OSSymbol * sym;
2386 OSData * noteProp;
2387 OSData * data;
2388 uintptr_t * smcVars;
2389 uint8_t * smcBytes;
2390 size_t len;
2391 addr64_t element;
2392
2393 data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
2394 if (!data) return;
2395
2396 smcVars = (typeof(smcVars)) data->getBytesNoCopy();
2397 smcBytes = (typeof(smcBytes)) smcVars[1];
2398 len = smcVars[0];
2399 if (len > sizeof(noteStore)) len = sizeof(noteStore);
2400 noteProp = OSData::withCapacity(3 * sizeof(element));
2401 if (!noteProp) return;
2402 element = len;
2403 noteProp->appendBytes(&element, sizeof(element));
2404 element = crc32(0, smcBytes, len);
2405 noteProp->appendBytes(&element, sizeof(element));
2406
2407 bcopy(smcBytes, noteStore, len);
2408 element = (addr64_t) &noteStore[0];
2409 element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
2410 noteProp->appendBytes(&element, sizeof(element));
2411
2412 if (!gIOOptionsEntry)
2413 {
2414 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
2415 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
2416 if (regEntry && !gIOOptionsEntry)
2417 regEntry->release();
2418 }
2419
2420 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
2421 if (gIOOptionsEntry && sym) gIOOptionsEntry->setProperty(sym, noteProp);
2422 if (noteProp) noteProp->release();
2423 if (sym) sym->release();
2424}
2425
2426
2427
mirror of XNU