]> git.saurik.com Git - apple/xnu.git/blame - iokit/Kernel/IOHibernateIO.cpp
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-2782.40.9.tar.gz
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
CommitLineData
3a60a9f5 1/*
b0d623f7 2 * Copyright (c) 2004-2008 Apple Computer, Inc. All rights reserved.
3a60a9f5 3 *
2d21ac55 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
3a60a9f5 5 *
2d21ac55
A		 6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
8f6c56a5 14 *
2d21ac55
A		 15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5
A		 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55
A		 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
8f6c56a5 25 *
2d21ac55 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
3a60a9f5
A		 27 */
28
29
30/*
31
32Sleep:
33
34- PMRootDomain calls IOHibernateSystemSleep() before system sleep
35(devices awake, normal execution context)
36- IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
37 grabs its extents and searches for a polling driver willing to work with that IOMedia.
38 The BSD code makes an ioctl to the storage driver to get the partition base offset to
39 the disk, and other ioctls to get the transfer constraints
40 If successful, the file is written to make sure its initially not bootable (in case of
41 later failure) and nvram set to point to the first block of the file. (Has to be done
42 here so blocking is possible in nvram support).
43 hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
44 page out any pages it wants to (currently zero, but probably some percentage of memory).
45 Its assumed just allocating pages will cause the VM system to naturally select the best
46 pages for eviction. It also copies processor flags needed for the restore path and sets
47 a flag in the boot processor proc info.
48 gIOHibernateState = kIOHibernateStateHibernating.
49- Regular sleep progresses - some drivers may inspect the root domain property
50 kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
51 as usual but leaves motherboard I/O on.
52- Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
53 at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
54 all ppc RC bits out of the hash table and caches into the mapping structures.
55- hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
56 hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
57 All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
58 by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
59 The image header and block list are written. The header includes the second file extent so
60 only the header block is needed to read the file, regardless of filesystem.
316670eb 61 The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data
3a60a9f5
A		 62 (only) is used to decompress the image during wake/boot.
63 Some additional pages are removed from the bitmaps - the buffers used for hibernation.
64 The bitmaps are written to the image.
65 More areas are removed from the bitmaps (after they have been written to the image) - the
316670eb 66 segment "__HIB" pages and interrupt stack.
3a60a9f5
A		 67 Each wired page is compressed and written and then each non-wired page. Compression and
68 disk writes are in parallel.
69 The image header is written to the start of the file and the polling driver closed.
70 The machine powers down (or sleeps).
71
72Boot/Wake:
73
74- BootX sees the boot-image nvram variable containing the device and block number of the image,
75 reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
76- BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
77 in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
78 that is in the kernel's __HIB section.
79- BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
80 hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
81 only code & data in that section is safe to call since all the other wired pages are still
82 compressed in the image.
83- hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
84 It uses the bitmaps to work out which pages can be uncompressed from the image to their final
85 location directly, and copies those that can't to interim free pages. When the image has been
86 completed, the copies are uncompressed, overwriting the wired image pages.
87 hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
88 is used to get pages into place for 64bit.
89- the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
90 changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
91 are removed from the software strutures, and the hash table is reinitialized.
92- After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
93 of memory, using the polled mode driver, before other threads can run or any devices are turned on.
94 This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
95 for the remaining non wired pages.
96- The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
97 (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
98 is closed via bsd.
99
100Polled Mode I/O:
101
102IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
103registry, specifying an object of calls IOPolledInterface.
104
105Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
106partition) that the image is going to live, looking for polled interface properties. If it finds
107one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
108interfaces found are kept in an ordered list.
109
110There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
111few different contexts things happen in:
112
113- there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
114up and running) and after wake - this is safe to allocate memory and do anything. The device
115ignores sleep requests from that point since its a waste of time if it goes to sleep and
116immediately wakes back up for the image write.
117
118- there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
119immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
120from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
121used to flush and set the disk to sleep.
122
123- there is an Open/Close (AfterSleep) pair made around the image read operations that happen
124immediately after sleep. These can't block or allocate memory. This is happening after the platform
125expert has woken the low level bits of the system, but most of the I/O system has not. There is only
126one thread running.
127
128For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
129(prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
130the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
131Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
132that is called for the hardware to check for events, and complete the I/O via the callback.
133The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
134to restrict I/O ops.
135*/
136
137#include <sys/systm.h>
138
139#include <IOKit/IOWorkLoop.h>
140#include <IOKit/IOCommandGate.h>
141#include <IOKit/IOTimerEventSource.h>
142#include <IOKit/IOPlatformExpert.h>
143#include <IOKit/IOKitDebug.h>
144#include <IOKit/IOTimeStamp.h>
145#include <IOKit/pwr_mgt/RootDomain.h>
146#include <IOKit/pwr_mgt/IOPMPrivate.h>
147#include <IOKit/IOMessage.h>
148#include <IOKit/IODeviceTreeSupport.h>
149#include <IOKit/IOBSD.h>
150#include "RootDomainUserClient.h"
151#include <IOKit/pwr_mgt/IOPowerConnection.h>
152#include "IOPMPowerStateQueue.h"
153#include <IOKit/IOBufferMemoryDescriptor.h>
6d2010ae 154#include <IOKit/AppleKeyStoreInterface.h>
316670eb 155#include <libkern/crypto/aes.h>
3a60a9f5
A		 156
157#include <sys/uio.h>
158#include <sys/conf.h>
159#include <sys/stat.h>
160#include <sys/fcntl.h> // (FWRITE, ...)
3a60a9f5 161#include <sys/sysctl.h>
6d2010ae 162#include <sys/kdebug.h>
fe8ab488 163#include <stdint.h>
3a60a9f5
A		 164
165#include <IOKit/IOHibernatePrivate.h>
166#include <IOKit/IOPolledInterface.h>
167#include <IOKit/IONVRAM.h>
168#include "IOHibernateInternal.h"
39236c6e 169#include <vm/WKdm_new.h>
3a60a9f5 170#include "IOKitKernelInternal.h"
6d2010ae
A		 171#include <pexpert/device_tree.h>
172
173#include <machine/pal_routines.h>
174#include <machine/pal_hibernate.h>
39236c6e 175#include <i386/tsc.h>
6d2010ae
A		 176
177extern "C" addr64_t kvtophys(vm_offset_t va);
99c3a104 178extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
3a60a9f5
A		 179
180/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
181
39236c6e
A		 182#define DISABLE_TRIM 0
183#define TRIM_DELAY 5000
184
0b4c1975 185extern unsigned int save_kdebug_enable;
3a60a9f5
A		 186extern uint32_t gIOHibernateState;
187uint32_t gIOHibernateMode;
188static char gIOHibernateBootSignature[256+1];
189static char gIOHibernateFilename[MAXPATHLEN+1];
db609669
A		 190static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
191uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms)
192static uint64_t gIOHibernateCompression = 0x80; // default compression 50%
3a60a9f5
A		 193
194static IODTNVRAM * gIOOptionsEntry;
195static IORegistryEntry * gIOChosenEntry;
b0d623f7 196#if defined(__i386__) || defined(__x86_64__)
0c530ab8 197static const OSSymbol * gIOCreateEFIDevicePathSymbol;
060df5ea
A		 198static const OSSymbol * gIOHibernateRTCVariablesKey;
199static const OSSymbol * gIOHibernateBoot0082Key;
200static const OSSymbol * gIOHibernateBootNextKey;
201static OSData * gIOHibernateBoot0082Data;
202static OSData * gIOHibernateBootNextData;
203static OSObject * gIOHibernateBootNextSave;
fe8ab488 204static struct kern_direct_file_io_ref_t * gDebugImageFileRef;
0c530ab8 205#endif
3a60a9f5 206
316670eb
A		 207static IOLock * gFSLock;
208static uint32_t gFSState;
3a60a9f5
A		 209static IOPolledFileIOVars gFileVars;
210static IOHibernateVars gIOHibernateVars;
211static struct kern_direct_file_io_ref_t * gIOHibernateFileRef;
212static hibernate_cryptvars_t gIOHibernateCryptWakeContext;
6d2010ae
A		 213static hibernate_graphics_t _hibernateGraphics;
214static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
39236c6e
A		 215static hibernate_statistics_t _hibernateStats;
216static hibernate_statistics_t * gIOHibernateStats = &_hibernateStats;
3a60a9f5 217
316670eb
A		 218enum
219{
220 kFSIdle = 0,
221 kFSOpening = 2,
222 kFSOpened = 3,
223 kFSTimedOut = 4,
224};
225
226static IOReturn IOHibernateDone(IOHibernateVars * vars);
227
3a60a9f5
A		 228/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
229
230enum { kXPRamAudioVolume = 8 };
231enum { kDefaultIOSize = 128 * 1024 };
22ba694c 232enum { kVideoMapSize = 80 * 1024 * 1024 };
3a60a9f5
A		 233
234#ifndef kIOMediaPreferredBlockSizeKey
235#define kIOMediaPreferredBlockSizeKey "Preferred Block Size"
236#endif
237
238#ifndef kIOBootPathKey
239#define kIOBootPathKey "bootpath"
240#endif
241#ifndef kIOSelectedBootDeviceKey
242#define kIOSelectedBootDeviceKey "boot-device"
243#endif
244
3a60a9f5
A		 245/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
246
247// copy from phys addr to MD
248
249static IOReturn
250IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
251 IOByteCount offset, addr64_t bytes, IOByteCount length)
252{
253 addr64_t srcAddr = bytes;
254 IOByteCount remaining;
255
256 remaining = length = min(length, md->getLength() - offset);
257 while (remaining) { // (process another target segment?)
258 addr64_t dstAddr64;
259 IOByteCount dstLen;
260
b0d623f7 261 dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
3a60a9f5
A		 262 if (!dstAddr64)
263 break;
264
265 // Clip segment length to remaining
266 if (dstLen > remaining)
267 dstLen = remaining;
268
269#if 1
270 bcopy_phys(srcAddr, dstAddr64, dstLen);
271#else
272 copypv(srcAddr, dstAddr64, dstLen,
273 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
274#endif
275 srcAddr += dstLen;
276 offset += dstLen;
277 remaining -= dstLen;
278 }
279
280 assert(!remaining);
281
282 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
283}
284
285// copy from MD to phys addr
286
287static IOReturn
288IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
289 IOByteCount offset, addr64_t bytes, IOByteCount length)
290{
291 addr64_t dstAddr = bytes;
292 IOByteCount remaining;
293
294 remaining = length = min(length, md->getLength() - offset);
295 while (remaining) { // (process another target segment?)
296 addr64_t srcAddr64;
297 IOByteCount dstLen;
298
b0d623f7 299 srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
3a60a9f5
A		 300 if (!srcAddr64)
301 break;
302
303 // Clip segment length to remaining
304 if (dstLen > remaining)
305 dstLen = remaining;
306
307#if 1
308 bcopy_phys(srcAddr64, dstAddr, dstLen);
309#else
310 copypv(srcAddr, dstAddr64, dstLen,
311 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
312#endif
313 dstAddr += dstLen;
314 offset += dstLen;
315 remaining -= dstLen;
316 }
317
318 assert(!remaining);
319
320 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
321}
322
323/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
324
325void
326hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
327 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
328{
329 count += ppnum;
330 switch (kind)
331 {
332 case kIOHibernatePageStateUnwiredSave:
333 // unwired save
334 for (; ppnum < count; ppnum++)
335 {
336 hibernate_page_bitset(page_list, FALSE, ppnum);
337 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
338 }
339 break;
340 case kIOHibernatePageStateWiredSave:
341 // wired save
342 for (; ppnum < count; ppnum++)
343 {
344 hibernate_page_bitset(page_list, FALSE, ppnum);
345 hibernate_page_bitset(page_list_wired, FALSE, ppnum);
346 }
347 break;
348 case kIOHibernatePageStateFree:
349 // free page
350 for (; ppnum < count; ppnum++)
351 {
352 hibernate_page_bitset(page_list, TRUE, ppnum);
353 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
354 }
355 break;
356 default:
357 panic("hibernate_set_page_state");
358 }
359}
360
361static vm_offset_t
0c530ab8 362hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
3a60a9f5 363{
0c530ab8
A		 364 uint32_t page = *pPage;
365 uint32_t count;
366 hibernate_bitmap_t * bitmap;
21362eb3 367
0c530ab8
A		 368 while ((bitmap = hibernate_page_bitmap_pin(list, &page)))
369 {
370 count = hibernate_page_bitmap_count(bitmap, TRUE, page);
371 if (!count)
372 break;
373 page += count;
374 if (page <= bitmap->last_page)
375 break;
376 }
6601e61a 377
0c530ab8
A		 378 *pPage = page;
379 if (bitmap)
380 count = hibernate_page_bitmap_count(bitmap, FALSE, page);
381 else
382 count = 0;
3a60a9f5
A		 383
384 return (count);
385}
386
387/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
388
389static IOReturn
390IOHibernatePollerProbe(IOPolledFileIOVars * vars, IOService * target)
391{
392 IOReturn err = kIOReturnError;
393 int32_t idx;
394 IOPolledInterface * poller;
395
396 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
397 {
398 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
399 err = poller->probe(target);
400 if (err)
401 {
402 HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
403 break;
404 }
405 }
406
407 return (err);
408}
409
410static IOReturn
411IOHibernatePollerOpen(IOPolledFileIOVars * vars, uint32_t state, IOMemoryDescriptor * md)
412{
413 IOReturn err = kIOReturnError;
414 int32_t idx;
415 IOPolledInterface * poller;
416
417 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
418 {
419 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
420 err = poller->open(state, md);
421 if (err)
422 {
423 HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
424 break;
425 }
426 }
427
428 return (err);
429}
430
431static IOReturn
432IOHibernatePollerClose(IOPolledFileIOVars * vars, uint32_t state)
433{
434 IOReturn err = kIOReturnError;
435 int32_t idx;
436 IOPolledInterface * poller;
437
438 for (idx = 0;
439 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
440 idx++)
441 {
442 err = poller->close(state);
443 if (err)
444 HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
445 }
446
447 return (err);
448}
449
450static void
451IOHibernatePollerIOComplete(void * target,
452 void * parameter,
453 IOReturn status,
454 UInt64 actualByteCount)
455{
456 IOPolledFileIOVars * vars = (IOPolledFileIOVars *) parameter;
457
458 vars->ioStatus = status;
459}
460
461static IOReturn
462IOHibernatePollerIO(IOPolledFileIOVars * vars,
463 uint32_t operation, uint32_t bufferOffset,
464 uint64_t deviceOffset, uint64_t length)
465{
466
467 IOReturn err = kIOReturnError;
468 IOPolledInterface * poller;
469 IOPolledCompletion completion;
470
471 completion.target = 0;
472 completion.action = &IOHibernatePollerIOComplete;
473 completion.parameter = vars;
474
475 vars->ioStatus = -1;
476
477 poller = (IOPolledInterface *) vars->pollers->getObject(0);
478 err = poller->startIO(operation, bufferOffset, deviceOffset + vars->block0, length, completion);
479 if (err)
480 HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
481
482 return (err);
483}
484
485static IOReturn
2d21ac55 486IOHibernatePollerIODone(IOPolledFileIOVars * vars, bool abortable)
3a60a9f5 487{
2d21ac55
A		 488 IOReturn err = kIOReturnSuccess;
489 int32_t idx = 0;
3a60a9f5
A		 490 IOPolledInterface * poller;
491
492 while (-1 == vars->ioStatus)
493 {
2d21ac55
A		 494 for (idx = 0;
495 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
3a60a9f5
A		 496 idx++)
497 {
2d21ac55
A		 498 IOReturn newErr;
499 newErr = poller->checkForWork();
500 if ((newErr == kIOReturnAborted) && !abortable)
501 newErr = kIOReturnSuccess;
502 if (kIOReturnSuccess == err)
503 err = newErr;
3a60a9f5
A		 504 }
505 }
506
0b4c1975
A		 507 if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort())
508 {
509 err = kIOReturnAborted;
510 HIBLOG("IOPolledInterface::checkForWork sw abort\n");
511 }
512
2d21ac55
A		 513 if (err)
514 {
515 HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
516 }
517 else
518 {
519 err = vars->ioStatus;
520 if (kIOReturnSuccess != err)
521 HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
522 }
3a60a9f5 523
2d21ac55 524 return (err);
3a60a9f5
A		 525}
526
527IOReturn
528IOPolledInterface::checkAllForWork(void)
529{
530 IOReturn err = kIOReturnNotReady;
531 int32_t idx;
532 IOPolledInterface * poller;
533
534 IOHibernateVars * vars = &gIOHibernateVars;
535
536 if (!vars->fileVars || !vars->fileVars->pollers)
537 return (err);
538
539 for (idx = 0;
540 (poller = (IOPolledInterface *) vars->fileVars->pollers->getObject(idx));
541 idx++)
542 {
543 err = poller->checkForWork();
544 if (err)
545 HIBLOG("IOPolledInterface::checkAllForWork[%d] 0x%x\n", idx, err);
546 }
547
548 return (err);
549}
550
551/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
552
553struct _OpenFileContext
554{
555 OSData * extents;
556 uint64_t size;
557};
558
559static void
560file_extent_callback(void * ref, uint64_t start, uint64_t length)
561{
562 _OpenFileContext * ctx = (_OpenFileContext *) ref;
563 IOPolledFileExtent extent;
564
565 extent.start = start;
566 extent.length = length;
567
39236c6e
A		 568 HIBLOG("[0x%qx, 0x%qx]\n", start, length);
569
3a60a9f5
A		 570 ctx->extents->appendBytes(&extent, sizeof(extent));
571 ctx->size += length;
572}
573
6d2010ae
A		 574static IOService *
575IOCopyMediaForDev(dev_t device)
576{
577 OSDictionary * matching;
578 OSNumber * num;
579 OSIterator * iter;
580 IOService * result = 0;
581
582 matching = IOService::serviceMatching("IOMedia");
583 if (!matching)
584 return (0);
585 do
586 {
587 num = OSNumber::withNumber(major(device), 32);
588 if (!num)
589 break;
590 matching->setObject(kIOBSDMajorKey, num);
591 num->release();
592 num = OSNumber::withNumber(minor(device), 32);
593 if (!num)
594 break;
595 matching->setObject(kIOBSDMinorKey, num);
596 num->release();
597 if (!num)
598 break;
599 iter = IOService::getMatchingServices(matching);
600 if (iter)
601 {
602 result = (IOService *) iter->getNextObject();
603 result->retain();
604 iter->release();
605 }
606 }
607 while (false);
608 matching->release();
609
610 return (result);
611}
612
fe8ab488
A		 613/*
614 * Writes header to disk with signature, block size and file extents data.
615 * If there are more than 2 extents, then they are written on second block.
616 */
617static IOReturn
618WriteExtentsToFile(struct kern_direct_file_io_ref_t * fileRef,
619 uint32_t signature, uint32_t blockSize,
620 IOPolledFileExtent *fileExtents,
621 IOByteCount size)
622{
623 IOHibernateImageHeader hdr;
624 IOItemCount count;
625 IOReturn err = kIOReturnSuccess;
626 int rc;
627
628 memset(&hdr, 0, sizeof(IOHibernateImageHeader));
629 count = size;
630 if (count > sizeof(hdr.fileExtentMap))
631 {
632 hdr.fileExtentMapSize = count;
633 count = sizeof(hdr.fileExtentMap);
634 }
635 else
636 hdr.fileExtentMapSize = sizeof(hdr.fileExtentMap);
637
638 bcopy(fileExtents, &hdr.fileExtentMap[0], count);
639
640 // copy file block extent list if larger than header
641 if (hdr.fileExtentMapSize > sizeof(hdr.fileExtentMap))
642 {
643 count = hdr.fileExtentMapSize - sizeof(hdr.fileExtentMap);
644 rc = kern_write_file(fileRef, blockSize,
645 (caddr_t)(((uint8_t *)fileExtents) + sizeof(hdr.fileExtentMap)),
646 count, IO_SKIP_ENCRYPTION);
647 if (rc != 0) {
648 HIBLOG("kern_write_file returned %d\n", rc);
649 err = kIOReturnIOError;
650 goto exit;
651 }
652 }
653 hdr.signature = signature;
654 hdr.deviceBlockSize = blockSize;
655
656 rc = kern_write_file(fileRef, 0, (char *)&hdr, sizeof(hdr), IO_SKIP_ENCRYPTION);
657 if (rc != 0) {
658 HIBLOG("kern_write_file returned %d\n", rc);
659 err = kIOReturnIOError;
660 goto exit;
661 }
662
663exit:
664 return err;
665}
666
667static IOReturn
668GetImageBlockSize(IOService *part, OSArray *pollers, IOByteCount *blockSize)
669{
670 IOService * service;
671 IORegistryEntry * next;
672 IORegistryEntry * child;
673
674 IOReturn err = kIOReturnSuccess;
675
676
677 next = part;
678 do
679 {
680 IOPolledInterface * poller;
681 OSObject * obj;
682 OSNumber * num;
683
684 obj = next->getProperty(kIOPolledInterfaceSupportKey);
685 if (kOSBooleanFalse == obj)
686 {
687 pollers->flushCollection();
688 break;
689 }
690 else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
691 pollers->setObject(poller);
692
693 if ((service = OSDynamicCast(IOService, next))
694 && service->getDeviceMemory()
695 && !pollers->getCount()) break;
696
697 if ((num = OSDynamicCast(OSNumber, next->getProperty(kIOMediaPreferredBlockSizeKey))))
698 *blockSize = num->unsigned32BitValue();
699 child = next;
700 }
701 while ((next = child->getParentEntry(gIOServicePlane))
702 && child->isParent(next, gIOServicePlane, true));
703
704 if (*blockSize < 4096) *blockSize = 4096;
705
706 if (!pollers->getCount())
707 err = kIOReturnUnsupported;
708
709 return err;
710}
711
3a60a9f5 712IOReturn
db609669
A		 713IOPolledFileOpen( const char * filename, uint64_t setFileSize,
714 IOBufferMemoryDescriptor * ioBuffer,
715 IOPolledFileIOVars ** fileVars, OSData ** fileExtents,
716 OSData ** imagePath, uint8_t * volumeCryptKey)
3a60a9f5 717{
316670eb 718 IOReturn err = kIOReturnSuccess;
3a60a9f5
A		 719 IOPolledFileIOVars * vars;
720 _OpenFileContext ctx;
721 OSData * extentsData;
6d2010ae
A		 722 IOService * part = 0;
723 OSString * keyUUID = 0;
724 OSString * keyStoreUUID = 0;
725 dev_t block_dev;
3a60a9f5
A		 726 dev_t hibernate_image_dev;
727 uint64_t maxiobytes;
316670eb
A		 728 AbsoluteTime startTime, endTime;
729 uint64_t nsec;
fe8ab488
A		 730 caddr_t write_file_addr = NULL;
731 vm_size_t write_file_len = 0;
316670eb
A		 732
733 vars = IONew(IOPolledFileIOVars, 1);
734 if (!vars) return (kIOReturnNoMemory);
735 bzero(vars, sizeof(*vars));
3a60a9f5 736
3a60a9f5
A		 737 do
738 {
3a60a9f5
A		 739 vars->io = false;
740 vars->buffer = (uint8_t *) ioBuffer->getBytesNoCopy();
741 vars->bufferHalf = 0;
742 vars->bufferOffset = 0;
743 vars->bufferSize = ioBuffer->getLength() >> 1;
744
745 extentsData = OSData::withCapacity(32);
316670eb 746 ctx.extents = extentsData;
3a60a9f5 747 ctx.size = 0;
316670eb 748 clock_get_uptime(&startTime);
fe8ab488
A		 749 if (!gDebugImageFileRef)
750 {
751 // Avoid writing the header if it is written when file is prep'd for debug data
752 // Image is locked during prep for debug data. So, write may fail.
753 write_file_addr = (caddr_t)gIOHibernateCurrentHeader;
754 write_file_len = sizeof(IOHibernateImageHeader);
755 }
3a60a9f5 756 vars->fileRef = kern_open_file_for_direct_io(filename,
fe8ab488 757 true,
3a60a9f5 758 &file_extent_callback, &ctx,
db609669
A		 759 setFileSize,
760 // write file:
fe8ab488
A		 761 0, write_file_addr,
762 write_file_len,
db609669 763 // results
6d2010ae 764 &block_dev,
3a60a9f5
A		 765 &hibernate_image_dev,
766 &vars->block0,
0b4c1975 767 &maxiobytes,
db609669 768 &vars->flags);
316670eb
A		 769#if 0
770 uint32_t msDelay = (131071 & random());
771 HIBLOG("sleep %d\n", msDelay);
772 IOSleep(msDelay);
773#endif
774 clock_get_uptime(&endTime);
775 SUB_ABSOLUTETIME(&endTime, &startTime);
776 absolutetime_to_nanoseconds(endTime, &nsec);
777
778 if (!vars->fileRef) err = kIOReturnNoSpace;
779
780 IOLockLock(gFSLock);
781 if (kFSOpening != gFSState) err = kIOReturnTimeout;
782 IOLockUnlock(gFSLock);
783
784 HIBLOG("kern_open_file_for_direct_io(%d) took %qd ms\n", err, nsec / 1000000ULL);
785 if (kIOReturnSuccess != err) break;
0b4c1975
A		 786
787 if (kIOHibernateModeSSDInvert & gIOHibernateMode)
6d2010ae 788 vars->flags ^= kIOHibernateOptionSSD;
0b4c1975
A		 789
790 HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx ssd %d\n", filename, ctx.size,
6d2010ae 791 vars->block0, maxiobytes, kIOHibernateOptionSSD & vars->flags);
3a60a9f5
A		 792 if (ctx.size < 1*1024*1024) // check against image size estimate!
793 {
794 err = kIOReturnNoSpace;
795 break;
796 }
797
db609669 798 vars->fileSize = ctx.size;
39236c6e 799 if (maxiobytes < vars->bufferSize) vars->bufferSize = maxiobytes;
3a60a9f5
A		 800
801 vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
2d21ac55 802
6d2010ae
A		 803 part = IOCopyMediaForDev(block_dev);
804 if (!part)
39236c6e
A		 805 {
806 err = kIOReturnNotFound;
6d2010ae 807 break;
39236c6e 808 }
6d2010ae
A		 809 err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false,
810 (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL);
811 if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID)
812 {
813// IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy());
814 uuid_t volumeKeyUUID;
815 aks_volume_key_t vek;
816 static IOService * sKeyStore;
817 static const OSSymbol * sAKSGetKey;
818
819 if (!sAKSGetKey)
820 sAKSGetKey = OSSymbol::withCStringNoCopy(AKS_PLATFORM_FUNCTION_GETKEY);
821 if (!sKeyStore)
822 sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane);
823 if (sKeyStore)
824 err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID);
825 else
826 err = kIOReturnNoResources;
827 if (kIOReturnSuccess == err)
828 err = sKeyStore->callPlatformFunction(sAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL);
829 if (kIOReturnSuccess != err)
830 IOLog("volume key err 0x%x\n", err);
831 else
832 {
833 size_t bytes = (kIOHibernateAESKeySize / 8);
834 if (vek.key.keybytecount < bytes)
835 bytes = vek.key.keybytecount;
836 bcopy(&vek.key.keybytes[0], volumeCryptKey, bytes);
837 }
838 bzero(&vek, sizeof(vek));
839 }
840 part->release();
841
842 part = IOCopyMediaForDev(hibernate_image_dev);
843 if (!part)
39236c6e
A		 844 {
845 err = kIOReturnNotFound;
6d2010ae 846 break;
39236c6e 847 }
6d2010ae 848
3a60a9f5 849 vars->pollers = OSArray::withCapacity(4);
fe8ab488 850 if (!vars->pollers)
39236c6e
A		 851 {
852 err = kIOReturnNoMemory;
853 break;
fe8ab488 854 }
15129b1c 855
fe8ab488 856 err = GetImageBlockSize(part, vars->pollers, &vars->blockSize);
3a60a9f5 857
fe8ab488
A		 858 HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n",
859 major(hibernate_image_dev), minor(hibernate_image_dev), (long)vars->blockSize,
860 vars->pollers->getCount());
39236c6e 861
fe8ab488
A		 862 if (err != kIOReturnSuccess)
863 break;
39236c6e 864
fe8ab488
A		 865 IORegistryEntry * next;
866 OSData * data;
39236c6e
A		 867 if (vars->blockSize < sizeof(IOHibernateImageHeader))
868 {
869 err = kIOReturnError;
870 continue;
871 }
3a60a9f5
A		 872
873 err = IOHibernatePollerProbe(vars, (IOService *) part);
39236c6e 874 if (kIOReturnSuccess != err) break;
3a60a9f5
A		 875
876 err = IOHibernatePollerOpen(vars, kIOPolledPreflightState, ioBuffer);
39236c6e 877 if (kIOReturnSuccess != err) break;
3a60a9f5 878
bd504ef0
A		 879 vars->media = part;
880 next = part;
881 while (next)
882 {
883 next->setProperty(kIOPolledInterfaceActiveKey, kOSBooleanTrue);
884 next = next->getParentEntry(gIOServicePlane);
885 }
886
3a60a9f5
A		 887 *fileVars = vars;
888 *fileExtents = extentsData;
889
890 // make imagePath
3a60a9f5 891
0c530ab8 892 if ((extentsData->getLength() >= sizeof(IOPolledFileExtent)))
3a60a9f5 893 {
6d2010ae 894 char str2[24 + sizeof(uuid_string_t) + 2];
0c530ab8 895
b0d623f7 896#if defined(__i386__) || defined(__x86_64__)
0c530ab8
A		 897 if (!gIOCreateEFIDevicePathSymbol)
898 gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
899
6d2010ae
A		 900 if (keyUUID)
901 snprintf(str2, sizeof(str2), "%qx:%s",
902 vars->extentMap[0].start, keyUUID->getCStringNoCopy());
903 else
904 snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
0c530ab8
A		 905
906 err = IOService::getPlatform()->callPlatformFunction(
907 gIOCreateEFIDevicePathSymbol, false,
6d2010ae
A		 908 (void *) part, (void *) str2,
909 (void *) (uintptr_t) true, (void *) &data);
0c530ab8
A		 910#else
911 char str1[256];
912 int len = sizeof(str1);
913
914 if (!part->getPath(str1, &len, gIODTPlane))
915 err = kIOReturnNotFound;
916 else
917 {
2d21ac55 918 snprintf(str2, sizeof(str2), ",%qx", vars->extentMap[0].start);
0c530ab8
A		 919 // (strip the plane name)
920 char * tail = strchr(str1, ':');
921 if (!tail)
922 tail = str1 - 1;
923 data = OSData::withBytes(tail + 1, strlen(tail + 1));
924 data->appendBytes(str2, strlen(str2));
925 }
926#endif
927 if (kIOReturnSuccess == err)
3a60a9f5 928 *imagePath = data;
0c530ab8
A		 929 else
930 HIBLOG("error 0x%x getting path\n", err);
3a60a9f5
A		 931 }
932 }
933 while (false);
934
935 if (kIOReturnSuccess != err)
936 {
937 HIBLOG("error 0x%x opening hibernation file\n", err);
938 if (vars->fileRef)
b0d623f7 939 {
7ddcb079 940 kern_close_file_for_direct_io(vars->fileRef, 0, 0, 0, 0, 0);
316670eb 941 vars->fileRef = NULL;
b0d623f7 942 }
3a60a9f5 943 }
fe8ab488
A		 944 else
945 {
946 WriteExtentsToFile(vars->fileRef, kIOHibernateHeaderOpenSignature, vars->blockSize,
947 (IOPolledFileExtent *)extentsData->getBytesNoCopy(),
948 extentsData->getLength());
949 }
3a60a9f5
A		 950
951 if (part)
952 part->release();
953
954 return (err);
955}
956
957IOReturn
958IOPolledFileClose( IOPolledFileIOVars * vars )
959{
960 if (vars->pollers)
961 {
962 IOHibernatePollerClose(vars, kIOPolledPostflightState);
963 vars->pollers->release();
964 }
965
3a60a9f5
A		 966 bzero(vars, sizeof(IOPolledFileIOVars));
967
968 return (kIOReturnSuccess);
969}
970
971static IOReturn
972IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
973{
974 IOPolledFileExtent * extentMap;
975
976 extentMap = vars->extentMap;
977
978 vars->position = position;
979
980 while (position >= extentMap->length)
981 {
982 position -= extentMap->length;
983 extentMap++;
984 }
985
986 vars->currentExtent = extentMap;
987 vars->extentRemaining = extentMap->length - position;
988 vars->extentPosition = vars->position - position;
989
990 if (vars->bufferSize <= vars->extentRemaining)
991 vars->bufferLimit = vars->bufferSize;
992 else
993 vars->bufferLimit = vars->extentRemaining;
994
995 return (kIOReturnSuccess);
996}
997
998static IOReturn
999IOPolledFileWrite(IOPolledFileIOVars * vars,
1000 const uint8_t * bytes, IOByteCount size,
1001 hibernate_cryptvars_t * cryptvars)
1002{
1003 IOReturn err = kIOReturnSuccess;
1004 IOByteCount copy;
1005 bool flush = false;
1006
1007 do
1008 {
1009 if (!bytes && !size)
1010 {
1011 // seek to end of block & flush
1012 size = vars->position & (vars->blockSize - 1);
1013 if (size)
1014 size = vars->blockSize - size;
1015 flush = true;
1016 // use some garbage for the fill
1017 bytes = vars->buffer + vars->bufferOffset;
1018 }
1019
1020 copy = vars->bufferLimit - vars->bufferOffset;
1021 if (copy > size)
1022 copy = size;
1023 else
1024 flush = true;
1025
1026 if (bytes)
1027 {
1028 bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
1029 bytes += copy;
1030 }
1031 else
1032 bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
1033
1034 size -= copy;
1035 vars->bufferOffset += copy;
1036 vars->position += copy;
1037
1038 if (flush && vars->bufferOffset)
1039 {
1040 uint64_t offset = (vars->position - vars->bufferOffset
1041 - vars->extentPosition + vars->currentExtent->start);
1042 uint32_t length = (vars->bufferOffset);
1043
2d21ac55 1044#if CRYPTO
0b4c1975
A		 1045 if (cryptvars && vars->encryptStart
1046 && (vars->position > vars->encryptStart)
1047 && ((vars->position - length) < vars->encryptEnd))
3a60a9f5 1048 {
6d2010ae
A		 1049 AbsoluteTime startTime, endTime;
1050
7ddcb079 1051 uint64_t encryptLen, encryptStart;
3a60a9f5
A		 1052 encryptLen = vars->position - vars->encryptStart;
1053 if (encryptLen > length)
1054 encryptLen = length;
1055 encryptStart = length - encryptLen;
0b4c1975
A		 1056 if (vars->position > vars->encryptEnd)
1057 encryptLen -= (vars->position - vars->encryptEnd);
1058
6d2010ae
A		 1059 clock_get_uptime(&startTime);
1060
3a60a9f5
A		 1061 // encrypt the buffer
1062 aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
1063 &cryptvars->aes_iv[0],
1064 encryptLen / AES_BLOCK_SIZE,
1065 vars->buffer + vars->bufferHalf + encryptStart,
1066 &cryptvars->ctx.encrypt);
6d2010ae
A		 1067
1068 clock_get_uptime(&endTime);
1069 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
1070 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
1071 vars->cryptBytes += encryptLen;
1072
3a60a9f5
A		 1073 // save initial vector for following encrypts
1074 bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
1075 &cryptvars->aes_iv[0],
1076 AES_BLOCK_SIZE);
1077 }
2d21ac55 1078#endif /* CRYPTO */
3a60a9f5
A		 1079
1080 if (vars->io)
1081 {
2d21ac55 1082 err = IOHibernatePollerIODone(vars, true);
3a60a9f5
A		 1083 if (kIOReturnSuccess != err)
1084 break;
1085 }
1086
1087if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
1088//if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
1089
1090 err = IOHibernatePollerIO(vars, kIOPolledWrite, vars->bufferHalf, offset, length);
1091 if (kIOReturnSuccess != err)
1092 break;
1093 vars->io = true;
1094
1095 vars->extentRemaining -= vars->bufferOffset;
1096 if (!vars->extentRemaining)
1097 {
1098 vars->currentExtent++;
1099 vars->extentRemaining = vars->currentExtent->length;
1100 vars->extentPosition = vars->position;
1101 if (!vars->extentRemaining)
1102 {
1103 err = kIOReturnOverrun;
1104 break;
1105 }
1106 }
1107
1108 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
1109 vars->bufferOffset = 0;
1110 if (vars->bufferSize <= vars->extentRemaining)
1111 vars->bufferLimit = vars->bufferSize;
1112 else
1113 vars->bufferLimit = vars->extentRemaining;
1114
1115 flush = false;
1116 }
1117 }
1118 while (size);
1119
1120 return (err);
1121}
1122
1123static IOReturn
1124IOPolledFileRead(IOPolledFileIOVars * vars,
1125 uint8_t * bytes, IOByteCount size,
1126 hibernate_cryptvars_t * cryptvars)
1127{
1128 IOReturn err = kIOReturnSuccess;
1129 IOByteCount copy;
1130
1131// bytesWritten += size;
1132
1133 do
1134 {
1135 copy = vars->bufferLimit - vars->bufferOffset;
1136 if (copy > size)
1137 copy = size;
1138
1139 if (bytes)
1140 {
1141 bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
1142 bytes += copy;
1143 }
1144 size -= copy;
1145 vars->bufferOffset += copy;
1146// vars->position += copy;
1147
6d2010ae 1148 if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd))
3a60a9f5
A		 1149 {
1150 if (vars->io)
1151 {
2d21ac55 1152 err = IOHibernatePollerIODone(vars, false);
3a60a9f5
A		 1153 if (kIOReturnSuccess != err)
1154 break;
1155 }
1156 else
1157 cryptvars = 0;
1158
1159if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
1160
6d2010ae 1161 vars->position += vars->lastRead;
3a60a9f5 1162 vars->extentRemaining -= vars->lastRead;
6d2010ae 1163 vars->bufferLimit = vars->lastRead;
3a60a9f5
A		 1164
1165 if (!vars->extentRemaining)
1166 {
1167 vars->currentExtent++;
1168 vars->extentRemaining = vars->currentExtent->length;
1169 vars->extentPosition = vars->position;
1170 if (!vars->extentRemaining)
1171 {
1172 err = kIOReturnOverrun;
1173 break;
1174 }
1175 }
1176
36401178
A		 1177 uint64_t length;
1178 uint64_t lastReadLength = vars->lastRead;
3a60a9f5
A		 1179 uint64_t offset = (vars->position
1180 - vars->extentPosition + vars->currentExtent->start);
36401178
A		 1181 if (vars->extentRemaining <= vars->bufferSize)
1182 length = vars->extentRemaining;
1183 else
1184 length = vars->bufferSize;
6d2010ae
A		 1185 if ((length + vars->position) > vars->readEnd)
1186 length = vars->readEnd - vars->position;
3a60a9f5 1187
6d2010ae
A		 1188 vars->lastRead = length;
1189 if (length)
1190 {
3a60a9f5 1191//if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
6d2010ae
A		 1192 err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length);
1193 if (kIOReturnSuccess != err)
1194 break;
1195 vars->io = true;
1196 }
3a60a9f5
A		 1197
1198 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
1199 vars->bufferOffset = 0;
1200
2d21ac55 1201#if CRYPTO
3a60a9f5
A		 1202 if (cryptvars)
1203 {
1204 uint8_t thisVector[AES_BLOCK_SIZE];
6d2010ae
A		 1205 AbsoluteTime startTime, endTime;
1206
3a60a9f5
A		 1207 // save initial vector for following decrypts
1208 bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
36401178 1209 bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE,
3a60a9f5 1210 &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
6d2010ae 1211
3a60a9f5 1212 // decrypt the buffer
6d2010ae
A		 1213 clock_get_uptime(&startTime);
1214
3a60a9f5
A		 1215 aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
1216 &thisVector[0],
36401178 1217 lastReadLength / AES_BLOCK_SIZE,
3a60a9f5
A		 1218 vars->buffer + vars->bufferHalf,
1219 &cryptvars->ctx.decrypt);
6d2010ae
A		 1220
1221 clock_get_uptime(&endTime);
1222 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
1223 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
1224 vars->cryptBytes += lastReadLength;
3a60a9f5 1225 }
b0d623f7 1226#endif /* CRYPTO */
3a60a9f5
A		 1227 }
1228 }
1229 while (size);
1230
1231 return (err);
1232}
1233
1234/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
99c3a104 1235
fe8ab488
A		 1236#if HIBERNATION
1237IOReturn
1238IOHibernateOpenForDebugData( )
1239{
1240 dev_t image_dev;
1241 OSData *extentsData = NULL;
1242 OSObject *obj;
1243 OSString *str;
1244 IOByteCount blockSize = 0;
1245 IOByteCount size;
1246 IOService * part = 0;
1247 OSData * data = NULL;
1248
1249 IOPolledFileExtent * fileExtents;
1250 IOReturn err = kIOReturnSuccess;
1251 IORegistryEntry * regEntry;
1252 OSArray * pollers = NULL;
1253
1254 _OpenFileContext ctx;
1255
1256 if (gDebugImageFileRef != NULL)
1257 return kIOReturnError;
1258
1259 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
1260 {
1261 if ((str = OSDynamicCast(OSString, obj)))
1262 strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
1263 sizeof(gIOHibernateFilename));
1264 obj->release();
1265 }
1266
1267 if (!gIOHibernateFilename[0]) {
1268 HIBLOG("Failed to get hibernate image filename\n");
1269 return (kIOReturnUnsupported);
1270 }
1271
1272 extentsData = OSData::withCapacity(32);
1273 ctx.extents = extentsData;
1274 ctx.size = 0;
1275
1276 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
1277 gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
1278 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1279
1280 gDebugImageFileRef = kern_open_file_for_direct_io(gIOHibernateFilename,
1281 false,
1282 &file_extent_callback, &ctx,
1283 0, 0,
1284 (caddr_t)gIOHibernateCurrentHeader,
1285 sizeof(IOHibernateImageHeader),
1286 NULL, &image_dev, NULL, NULL, NULL);
1287
1288 if (gDebugImageFileRef == NULL)
1289 {
1290 HIBLOG("Failed to open the file \n");
1291 err = kIOReturnError;
1292 goto exit;
1293 }
1294 fileExtents = (IOPolledFileExtent *)extentsData->getBytesNoCopy();
1295 size = extentsData->getLength();
1296
1297 part = IOCopyMediaForDev(image_dev);
1298 if (!part)
1299 {
1300 HIBLOG("Failed to get the media device\n");
1301 err = kIOReturnNotFound;
1302 goto exit;
1303 }
1304
1305
1306 pollers = OSArray::withCapacity(4);
1307 if (!pollers)
1308 {
1309 err = kIOReturnNoMemory;
1310 goto exit;
1311 }
1312
1313 err = GetImageBlockSize(part, pollers, &blockSize);
1314 if (err != kIOReturnSuccess)
1315 {
1316 HIBLOG("Failed to get block size\n");
1317 goto exit;
1318 }
1319 if (blockSize < sizeof(IOHibernateImageHeader))
1320 {
1321 HIBLOG("block size %llu is less than the size of the header\n", blockSize);
1322 err = kIOReturnError;
1323 goto exit;
1324 }
1325
1326 WriteExtentsToFile(gDebugImageFileRef, kIOHibernateHeaderOpenSignature,
1327 blockSize, fileExtents, size);
1328
1329 char str2[24 + sizeof(uuid_string_t) + 2];
1330
1331 if (!gIOCreateEFIDevicePathSymbol)
1332 gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
1333
1334 snprintf(str2, sizeof(str2), "%qx", fileExtents[0].start);
1335
1336 err = IOService::getPlatform()->callPlatformFunction(
1337 gIOCreateEFIDevicePathSymbol, false,
1338 (void *) part, (void *) str2,
1339 (void *) (uintptr_t) true, (void *) &data);
1340
1341 if (!gIOOptionsEntry)
1342 {
1343 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
1344 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
1345 if (regEntry && !gIOOptionsEntry)
1346 regEntry->release();
1347 }
1348 if (gIOOptionsEntry)
1349 {
1350 const OSSymbol * sym;
1351
1352 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1353 if (sym)
1354 {
1355 gIOOptionsEntry->setProperty(sym, data);
1356 sym->release();
1357 }
1358 }
1359
1360
1361exit:
1362
1363 if ( (err != kIOReturnSuccess) && gDebugImageFileRef) {
1364 kern_close_file_for_direct_io(gDebugImageFileRef, 0, 0, 0, 0, 0);
1365 gDebugImageFileRef = NULL;
1366 }
1367 if (extentsData) extentsData->release();
1368 if (part) part->release();
1369 if (pollers) pollers->release();
1370 if (data) data->release();
1371
1372 return err;
1373}
1374#endif
1375
1376/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1377
3a60a9f5
A		 1378IOReturn
1379IOHibernateSystemSleep(void)
1380{
1381 IOReturn err;
1382 OSData * data;
1383 OSObject * obj;
1384 OSString * str;
db609669 1385 OSNumber * num;
39236c6e 1386 bool dsSSD, vmflush;
316670eb 1387 IOHibernateVars * vars;
3a60a9f5
A		 1388
1389 gIOHibernateState = kIOHibernateStateInactive;
1390
316670eb
A		 1391 if (!gIOChosenEntry)
1392 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1393
b0d623f7
A		 1394 gIOHibernateDebugFlags = 0;
1395 if (kIOLogHibernate & gIOKitDebug)
1396 gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
1397
0b4c1975
A		 1398 if (IOService::getPMRootDomain()->getHibernateSettings(
1399 &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime))
6d2010ae 1400 {
3a60a9f5
A		 1401 if (kIOHibernateModeSleep & gIOHibernateMode)
1402 // default to discard clean for safe sleep
1403 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
1404 | kIOHibernateModeDiscardCleanActive);
6d2010ae 1405 }
3a60a9f5 1406
3a60a9f5
A		 1407 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
1408 {
1409 if ((str = OSDynamicCast(OSString, obj)))
2d21ac55
A		 1410 strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
1411 sizeof(gIOHibernateFilename));
3a60a9f5
A		 1412 obj->release();
1413 }
1414
1415 if (!gIOHibernateMode || !gIOHibernateFilename[0])
1416 return (kIOReturnUnsupported);
1417
1418 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
1419
316670eb
A		 1420 vars = IONew(IOHibernateVars, 1);
1421 if (!vars) return (kIOReturnNoMemory);
1422 bzero(vars, sizeof(*vars));
1423
1424 IOLockLock(gFSLock);
1425 if (kFSIdle != gFSState)
1426 {
1427 HIBLOG("hibernate file busy\n");
1428 IOLockUnlock(gFSLock);
1429 IODelete(vars, IOHibernateVars, 1);
1430 return (kIOReturnBusy);
1431 }
1432 gFSState = kFSOpening;
1433 IOLockUnlock(gFSLock);
2d21ac55 1434
3a60a9f5
A		 1435 do
1436 {
0c530ab8 1437 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
39236c6e 1438 2 * page_size + WKdm_SCRATCH_BUF_SIZE, page_size);
0c530ab8
A		 1439 vars->ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1440 2 * kDefaultIOSize, page_size);
3a60a9f5 1441
6d2010ae
A		 1442 vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1443 ptoa_64(gIOHibernateHandoffPageCount), page_size);
1444
1445 if (!vars->srcBuffer || !vars->ioBuffer || !vars->handoffBuffer)
3a60a9f5
A		 1446 {
1447 err = kIOReturnNoMemory;
1448 break;
1449 }
1450
db609669
A		 1451 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey)))
1452 {
1453 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMinSize = num->unsigned64BitValue();
1454 obj->release();
1455 }
1456 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey)))
1457 {
1458 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMaxSize = num->unsigned64BitValue();
1459 obj->release();
1460 }
1461
1462 boolean_t encryptedswap = true;
1463 uint32_t pageCount;
1464 AbsoluteTime startTime, endTime;
1465 uint64_t nsec;
1466
1467 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
1468 gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
6d2010ae 1469 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
db609669 1470
39236c6e 1471 vmflush = (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey));
db609669 1472 uint64_t setFileSize = 0;
39236c6e
A		 1473 err = hibernate_alloc_page_lists(&vars->page_list,
1474 &vars->page_list_wired,
1475 &vars->page_list_pal);
1476 if (KERN_SUCCESS != err)
1477 break;
db609669
A		 1478
1479 if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode))
1480 {
1481 hibernate_page_list_setall(vars->page_list,
1482 vars->page_list_wired,
1483 vars->page_list_pal,
1484 true /* preflight */,
39236c6e 1485 vmflush /* discard */,
db609669
A		 1486 &pageCount);
1487 PE_Video consoleInfo;
1488 bzero(&consoleInfo, sizeof(consoleInfo));
1489 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1490
8a3053a0 1491 // estimate: 6% increase in pages compressed
fe8ab488 1492 // screen preview 2 images compressed 0%
8a3053a0 1493 setFileSize = ((ptoa_64((106 * pageCount) / 100) * gIOHibernateCompression) >> 8)
db609669 1494 + vars->page_list->list_size
8a3053a0
A		 1495 + (consoleInfo.v_width * consoleInfo.v_height * 8);
1496 enum { setFileRound = 1024*1024ULL };
1497 setFileSize = ((setFileSize + setFileRound) & ~(setFileRound - 1));
fe8ab488 1498
db609669
A		 1499 HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n",
1500 pageCount, (100ULL * gIOHibernateCompression) >> 8,
1501 setFileSize, vars->fileMinSize);
1502
1503 if (!(kIOHibernateModeFileResize & gIOHibernateMode)
1504 && (setFileSize < vars->fileMinSize))
1505 {
1506 setFileSize = vars->fileMinSize;
1507 }
1508 }
1509
1510 // open & invalidate the image file
1511
fe8ab488
A		 1512 if (gDebugImageFileRef) {
1513 kern_close_file_for_direct_io(gDebugImageFileRef, 0, 0, 0, 0, 0);
1514 gDebugImageFileRef = NULL;
1515 }
1516
db609669 1517 err = IOPolledFileOpen(gIOHibernateFilename, setFileSize, vars->ioBuffer,
6d2010ae
A		 1518 &vars->fileVars, &vars->fileExtents, &data,
1519 &vars->volumeCryptKey[0]);
8a3053a0 1520
3a60a9f5
A		 1521 if (KERN_SUCCESS != err)
1522 {
1523 HIBLOG("IOPolledFileOpen(%x)\n", err);
1524 break;
1525 }
3a60a9f5 1526
39236c6e
A		 1527 clock_get_uptime(&startTime);
1528 err = hibernate_setup(gIOHibernateCurrentHeader,
1529 gIOHibernateFreeRatio, gIOHibernateFreeTime,
1530 vmflush,
1531 vars->page_list, vars->page_list_wired, vars->page_list_pal);
1532 clock_get_uptime(&endTime);
1533 SUB_ABSOLUTETIME(&endTime, &startTime);
1534 absolutetime_to_nanoseconds(endTime, &nsec);
1535 HIBLOG("hibernate_setup(%d) took %qd ms\n", err, nsec / 1000000ULL);
1536
6d2010ae 1537 dsSSD = ((0 != (kIOHibernateOptionSSD & vars->fileVars->flags))
0b4c1975 1538 && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
0b4c1975
A		 1539 if (dsSSD)
1540 {
1541 gIOHibernateCurrentHeader->options |=
1542 kIOHibernateOptionSSD
1543 | kIOHibernateOptionColor;
6d2010ae
A		 1544
1545#if defined(__i386__) || defined(__x86_64__)
1546 if (!uuid_is_null(vars->volumeCryptKey) &&
1547 (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey)))
1548 {
1549 uintptr_t smcVars[2];
1550 smcVars[0] = sizeof(vars->volumeCryptKey);
316670eb 1551 smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
6d2010ae
A		 1552
1553 IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
1554 bzero(smcVars, sizeof(smcVars));
1555 }
1556#endif
0b4c1975
A		 1557 }
1558 else
1559 {
1560 gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
1561 }
1562
0b4c1975 1563
3a60a9f5 1564 if (KERN_SUCCESS != err)
3a60a9f5 1565 break;
3a60a9f5 1566
6d2010ae 1567 if (encryptedswap || !uuid_is_null(vars->volumeCryptKey))
3a60a9f5
A		 1568 gIOHibernateMode ^= kIOHibernateModeEncrypt;
1569
0b4c1975
A		 1570 if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1571 {
1572 vars->videoAllocSize = kVideoMapSize;
1573 if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize))
1574 vars->videoMapping = 0;
1575 }
3a60a9f5
A		 1576
1577 // generate crypt keys
1578 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
1579 vars->wiredCryptKey[i] = random();
1580 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
1581 vars->cryptKey[i] = random();
1582
1583 // set nvram
1584
1585 IORegistryEntry * regEntry;
1586 if (!gIOOptionsEntry)
1587 {
1588 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
1589 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
1590 if (regEntry && !gIOOptionsEntry)
1591 regEntry->release();
1592 }
3a60a9f5
A		 1593
1594 if (gIOOptionsEntry)
1595 {
1596 const OSSymbol * sym;
3a60a9f5
A		 1597
1598 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1599 if (sym)
1600 {
1601 gIOOptionsEntry->setProperty(sym, data);
1602 sym->release();
1603 }
1604 data->release();
1605
b0d623f7 1606#if defined(__i386__) || defined(__x86_64__)
0c530ab8
A		 1607 struct AppleRTCHibernateVars
1608 {
1609 uint8_t signature[4];
1610 uint32_t revision;
1611 uint8_t booterSignature[20];
1612 uint8_t wiredCryptKey[16];
1613 };
1614 AppleRTCHibernateVars rtcVars;
1615
1616 rtcVars.signature[0] = 'A';
1617 rtcVars.signature[1] = 'A';
1618 rtcVars.signature[2] = 'P';
1619 rtcVars.signature[3] = 'L';
1620 rtcVars.revision = 1;
1621 bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
1622 if (gIOHibernateBootSignature[0])
1623 {
1624 char c;
1625 uint8_t value = 0;
1626 for (uint32_t i = 0;
1627 (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1));
1628 i++)
1629 {
1630 if (c >= 'a')
1631 c -= 'a' - 10;
1632 else if (c >= 'A')
1633 c -= 'A' - 10;
1634 else if (c >= '0')
1635 c -= '0';
1636 else
1637 continue;
1638 value = (value << 4) | c;
1639 if (i & 1)
1640 rtcVars.booterSignature[i >> 1] = value;
1641 }
1642 }
1643 data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
1644 if (data)
2d21ac55 1645 {
060df5ea
A		 1646 if (!gIOHibernateRTCVariablesKey)
1647 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1648 if (gIOHibernateRTCVariablesKey)
1649 IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
1650
1651 if( gIOOptionsEntry )
1652 {
1653 if( gIOHibernateMode & kIOHibernateModeSwitch )
1654 {
1655 const OSSymbol *sym;
1656 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSwitchVarsKey);
1657 if( sym )
2d21ac55 1658 {
060df5ea
A		 1659 gIOOptionsEntry->setProperty(sym, data); /* intentional insecure backup of rtc boot vars */
1660 sym->release();
2d21ac55 1661 }
060df5ea
A		 1662 }
1663 }
2d21ac55 1664
060df5ea 1665 data->release();
0c530ab8
A		 1666 }
1667 if (gIOChosenEntry)
1668 {
1669 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
1670 if (data)
1671 gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
060df5ea
A		 1672 {
1673 // set BootNext
1674
1675 if (!gIOHibernateBoot0082Data)
1676 {
1677 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
1678 if (data)
1679 {
1680 // AppleNVRAM_EFI_LOAD_OPTION
1681 struct {
1682 uint32_t Attributes;
1683 uint16_t FilePathLength;
1684 uint16_t Desc;
1685 } loadOptionHeader;
1686 loadOptionHeader.Attributes = 1;
1687 loadOptionHeader.FilePathLength = data->getLength();
1688 loadOptionHeader.Desc = 0;
1689 gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
1690 if (gIOHibernateBoot0082Data)
1691 {
1692 gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
1693 gIOHibernateBoot0082Data->appendBytes(data);
1694 }
1695 }
1696 }
1697 if (!gIOHibernateBoot0082Key)
1698 gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
1699 if (!gIOHibernateBootNextKey)
1700 gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
1701 if (!gIOHibernateBootNextData)
1702 {
1703 uint16_t bits = 0x0082;
1704 gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
1705 }
1706 if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData)
1707 {
1708 gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
1709 gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
1710 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
1711 }
1712 }
0c530ab8 1713 }
b0d623f7 1714#else /* !i386 && !x86_64 */
3a60a9f5
A		 1715 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1716 {
1717 data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1718 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
1719 if (sym && data)
1720 gIOOptionsEntry->setProperty(sym, data);
1721 if (sym)
1722 sym->release();
1723 if (data)
1724 data->release();
2d21ac55 1725 if (false && gIOHibernateBootSignature[0])
3a60a9f5
A		 1726 {
1727 data = OSData::withCapacity(16);
1728 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
1729 if (sym && data)
1730 {
1731 char c;
0c530ab8 1732 uint8_t value = 0;
3a60a9f5
A		 1733 for (uint32_t i = 0; (c = gIOHibernateBootSignature[i]); i++)
1734 {
1735 if (c >= 'a')
1736 c -= 'a' - 10;
1737 else if (c >= 'A')
1738 c -= 'A' - 10;
1739 else if (c >= '0')
1740 c -= '0';
1741 else
1742 continue;
1743 value = (value << 4) | c;
1744 if (i & 1)
1745 data->appendBytes(&value, sizeof(value));
1746 }
1747 gIOOptionsEntry->setProperty(sym, data);
1748 }
1749 if (sym)
1750 sym->release();
1751 if (data)
1752 data->release();
1753 }
1754 }
3a60a9f5
A		 1755 if (!vars->haveFastBoot)
1756 {
1757 // set boot volume to zero
1758 IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
1759 if (platform && (kIOReturnSuccess == platform->readXPRAM(kXPRamAudioVolume,
1760 &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume))))
1761 {
1762 uint8_t newVolume;
1763 newVolume = vars->saveBootAudioVolume & 0xf8;
1764 platform->writeXPRAM(kXPRamAudioVolume,
1765 &newVolume, sizeof(newVolume));
1766 }
1767 }
b0d623f7 1768#endif /* !i386 && !x86_64 */
3a60a9f5
A		 1769 }
1770 // --
1771
316670eb
A		 1772 }
1773 while (false);
1774
1775 IOLockLock(gFSLock);
1776 if ((kIOReturnSuccess == err) && (kFSOpening == gFSState))
1777 {
1778 gFSState = kFSOpened;
1779 gIOHibernateVars = *vars;
1780 gFileVars = *vars->fileVars;
1781 gIOHibernateVars.fileVars = &gFileVars;
1782 gIOHibernateFileRef = gFileVars.fileRef;
3a60a9f5
A		 1783 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
1784 gIOHibernateState = kIOHibernateStateHibernating;
1785 }
316670eb
A		 1786 else
1787 {
1788 HIBLOG("hibernate file close due timeout\n");
1789 if (vars->fileVars && vars->fileVars->fileRef) kern_close_file_for_direct_io(vars->fileVars->fileRef, 0, 0, 0, 0, 0);
1790 IOHibernateDone(vars);
1791 gFSState = kFSIdle;
1792 }
1793 IOLockUnlock(gFSLock);
1794
1795 if (vars->fileVars) IODelete(vars->fileVars, IOPolledFileIOVars, 1);
1796 IODelete(vars, IOHibernateVars, 1);
3a60a9f5
A		 1797
1798 return (err);
1799}
1800
1801/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1802
2d21ac55
A		 1803DECLARE_IOHIBERNATEPROGRESSALPHA
1804
1805static void
1806ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
1807{
1808 uint32_t rowBytes, pixelShift;
1809 uint32_t x, y;
1810 int32_t blob;
1811 uint32_t alpha, in, color, result;
1812 uint8_t * out;
1813 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1814
1815 rowBytes = display->rowBytes;
1816 pixelShift = display->depth >> 4;
1817 if (pixelShift < 1) return;
1818
1819 screen += ((display->width
1820 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1821 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1822
1823 for (y = 0; y < kIOHibernateProgressHeight; y++)
1824 {
1825 out = screen + y * rowBytes;
1826 for (blob = 0; blob < kIOHibernateProgressCount; blob++)
1827 {
1828 color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
1829 for (x = 0; x < kIOHibernateProgressWidth; x++)
1830 {
1831 alpha = gIOHibernateProgressAlpha[y][x];
1832 result = color;
1833 if (alpha)
1834 {
1835 if (0xff != alpha)
1836 {
1837 if (1 == pixelShift)
1838 {
1839 in = *((uint16_t *)out) & 0x1f; // 16
1840 in = (in << 3) | (in >> 2);
1841 }
1842 else
1843 in = *((uint32_t *)out) & 0xff; // 32
1844 saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
1845 result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
1846 }
1847 if (1 == pixelShift)
1848 {
1849 result >>= 3;
1850 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1851 }
1852 else
1853 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1854 }
1855 out += (1 << pixelShift);
1856 }
1857 out += (kIOHibernateProgressSpacing << pixelShift);
1858 }
1859 }
1860}
1861
1862
1863static void
1864ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
1865{
1866 uint32_t rowBytes, pixelShift;
1867 uint32_t x, y;
1868 int32_t blob, lastBlob;
1869 uint32_t alpha, in, color, result;
1870 uint8_t * out;
1871 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1872
1873 pixelShift = display->depth >> 4;
1874 if (pixelShift < 1)
1875 return;
1876
1877 rowBytes = display->rowBytes;
1878
1879 screen += ((display->width
1880 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1881 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1882
1883 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
1884
1885 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
1886
1887 for (y = 0; y < kIOHibernateProgressHeight; y++)
1888 {
1889 out = screen + y * rowBytes;
1890 for (blob = firstBlob; blob <= lastBlob; blob++)
1891 {
1892 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
1893 for (x = 0; x < kIOHibernateProgressWidth; x++)
1894 {
1895 alpha = gIOHibernateProgressAlpha[y][x];
1896 result = color;
1897 if (alpha)
1898 {
1899 if (0xff != alpha)
1900 {
1901 in = display->progressSaveUnder[blob][saveindex[blob]++];
1902 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
1903 }
1904 if (1 == pixelShift)
1905 {
1906 result >>= 3;
1907 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1908 }
1909 else
1910 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1911 }
1912 out += (1 << pixelShift);
1913 }
1914 out += (kIOHibernateProgressSpacing << pixelShift);
1915 }
1916 }
1917}
1918
1919/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1920
316670eb
A		 1921IOReturn
1922IOHibernateIOKitSleep(void)
1923{
1924 IOReturn ret = kIOReturnSuccess;
1925 IOLockLock(gFSLock);
1926 if (kFSOpening == gFSState)
1927 {
1928 gFSState = kFSTimedOut;
1929 HIBLOG("hibernate file open timed out\n");
1930 ret = kIOReturnTimeout;
1931 }
1932 IOLockUnlock(gFSLock);
1933 return (ret);
1934}
1935
1936/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1937
3a60a9f5
A		 1938IOReturn
1939IOHibernateSystemHasSlept(void)
1940{
316670eb 1941 IOReturn ret = kIOReturnSuccess;
3a60a9f5 1942 IOHibernateVars * vars = &gIOHibernateVars;
316670eb 1943 OSObject * obj = 0;
2d21ac55
A		 1944 OSData * data;
1945
316670eb
A		 1946 IOLockLock(gFSLock);
1947 if ((kFSOpened != gFSState) && gIOHibernateMode)
1948 {
1949 ret = kIOReturnTimeout;
1950 }
1951 IOLockUnlock(gFSLock);
1952 if (kIOReturnSuccess != ret) return (ret);
1953
1954 if (gIOHibernateMode) obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
2d21ac55
A		 1955 vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
1956 if (obj && !vars->previewBuffer)
1957 obj->release();
3a60a9f5 1958
2d21ac55
A		 1959 vars->consoleMapping = NULL;
1960 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
3a60a9f5 1961 {
2d21ac55
A		 1962 vars->previewBuffer->release();
1963 vars->previewBuffer = 0;
1964 }
3a60a9f5 1965
0b4c1975
A		 1966 if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1967 && vars->previewBuffer
1968 && (data = OSDynamicCast(OSData,
2d21ac55
A		 1969 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey))))
1970 {
1971 UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
b0d623f7 1972 HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
2d21ac55
A		 1973
1974 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
1975
1976 if (kIOHibernatePreviewUpdates & flags)
1977 {
1978 PE_Video consoleInfo;
1979 hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
1980
1981 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1982
1983 graphicsInfo->width = consoleInfo.v_width;
1984 graphicsInfo->height = consoleInfo.v_height;
1985 graphicsInfo->rowBytes = consoleInfo.v_rowBytes;
1986 graphicsInfo->depth = consoleInfo.v_depth;
1987 vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
1988
1989 HIBPRINT("video %p %d %d %d\n",
0b4c1975
A		 1990 vars->consoleMapping, graphicsInfo->depth,
1991 graphicsInfo->width, graphicsInfo->height);
2d21ac55
A		 1992 if (vars->consoleMapping)
1993 ProgressInit(graphicsInfo, vars->consoleMapping,
1994 &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
1995 }
3a60a9f5 1996 }
2d21ac55 1997
3a60a9f5
A		 1998 if (gIOOptionsEntry)
1999 gIOOptionsEntry->sync();
2000
316670eb 2001 return (ret);
3a60a9f5
A		 2002}
2003
2004/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2005
6d2010ae
A		 2006static DeviceTreeNode *
2007MergeDeviceTree(DeviceTreeNode * entry, IORegistryEntry * regEntry)
2008{
2009 DeviceTreeNodeProperty * prop;
2010 DeviceTreeNode * child;
2011 IORegistryEntry * childRegEntry;
2012 const char * nameProp;
2013 unsigned int propLen, idx;
2014
2015 prop = (DeviceTreeNodeProperty *) (entry + 1);
2016 for (idx = 0; idx < entry->nProperties; idx++)
2017 {
2018 if (regEntry && (0 != strcmp("name", prop->name)))
2019 {
2020 regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
2021// HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
2022 }
2023 prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
2024 }
2025
2026 child = (DeviceTreeNode *) prop;
2027 for (idx = 0; idx < entry->nChildren; idx++)
2028 {
2029 if (kSuccess != DTGetProperty(child, "name", (void **) &nameProp, &propLen))
2030 panic("no name");
2031 childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
2032// HIBPRINT("%s == %p\n", nameProp, childRegEntry);
2033 child = MergeDeviceTree(child, childRegEntry);
2034 }
2035 return (child);
2036}
2037
3a60a9f5
A		 2038IOReturn
2039IOHibernateSystemWake(void)
2040{
316670eb
A		 2041 if (kFSOpened == gFSState)
2042 {
2043 IOHibernateDone(&gIOHibernateVars);
2044 }
2045 else
2046 {
2047 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
2048 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
2049 }
2050 return (kIOReturnSuccess);
2051}
3a60a9f5 2052
316670eb
A		 2053static IOReturn
2054IOHibernateDone(IOHibernateVars * vars)
2055{
bd504ef0
A		 2056 IORegistryEntry * next;
2057
99c3a104 2058 hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);
3a60a9f5
A		 2059
2060 if (vars->videoMapping)
2061 {
2062 if (vars->videoMapSize)
2063 // remove mappings
2064 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
2065 if (vars->videoAllocSize)
2066 // dealloc range
b0d623f7 2067 kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
3a60a9f5
A		 2068 }
2069
2070 if (vars->previewBuffer)
2071 {
2072 vars->previewBuffer->release();
2073 vars->previewBuffer = 0;
2074 }
2075
0b4c1975
A		 2076 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
2077 {
2078 IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey,
2079 gIOHibernateCurrentHeader->options, 32);
2080 }
2081 else
2082 {
2083 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
2084 }
2085
2086 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
2087 && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus))
2088 {
2089 IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey,
2090 &gIOHibernateGraphicsInfo->gfxStatus,
2091 sizeof(gIOHibernateGraphicsInfo->gfxStatus));
2092 }
2093 else
2094 {
2095 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
2096 }
2097
3a60a9f5
A		 2098 if (vars->fileVars)
2099 {
bd504ef0
A		 2100 if ((next = vars->fileVars->media)) do
2101 {
2102 next->removeProperty(kIOPolledInterfaceActiveKey);
2103 next = next->getParentEntry(gIOServicePlane);
2104 }
2105 while (next);
3a60a9f5
A		 2106 IOPolledFileClose(vars->fileVars);
2107 }
2108
2109 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
2110
b0d623f7 2111#if defined(__i386__) || defined(__x86_64__)
060df5ea 2112 IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
6d2010ae 2113 IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
2d21ac55
A		 2114
2115 /*
2116 * Hibernate variable is written to NVRAM on platforms in which RtcRam
2117 * is not backed by coin cell. Remove Hibernate data from NVRAM.
2118 */
2119 if (gIOOptionsEntry) {
2d21ac55 2120
060df5ea
A		 2121 if (gIOHibernateRTCVariablesKey) {
2122 if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
2123 gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
2124 }
2125 }
2126
2127 if (gIOHibernateBootNextKey)
2128 {
2129 if (gIOHibernateBootNextSave)
2130 {
2131 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
2132 gIOHibernateBootNextSave->release();
2133 gIOHibernateBootNextSave = NULL;
2d21ac55 2134 }
060df5ea
A		 2135 else
2136 gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
2137 }
39236c6e 2138 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) gIOOptionsEntry->sync();
2d21ac55 2139 }
0c530ab8 2140#endif
3a60a9f5
A		 2141
2142 if (vars->srcBuffer)
2143 vars->srcBuffer->release();
2144 if (vars->ioBuffer)
2145 vars->ioBuffer->release();
6d2010ae 2146 bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
99c3a104 2147 if (vars->handoffBuffer)
6d2010ae 2148 {
99c3a104 2149 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
6d2010ae 2150 {
99c3a104
A		 2151 IOHibernateHandoff * handoff;
2152 bool done = false;
2153 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
2154 !done;
2155 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
6d2010ae 2156 {
99c3a104
A		 2157 HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
2158 uint8_t * data = &handoff->data[0];
2159 switch (handoff->type)
2160 {
2161 case kIOHibernateHandoffTypeEnd:
2162 done = true;
2163 break;
6d2010ae 2164
99c3a104
A		 2165 case kIOHibernateHandoffTypeDeviceTree:
2166 MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot());
2167 break;
2168
2169 case kIOHibernateHandoffTypeKeyStore:
6d2010ae 2170#if defined(__i386__) || defined(__x86_64__)
6d2010ae 2171 {
99c3a104
A		 2172 IOBufferMemoryDescriptor *
2173 md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
2174 if (md)
2175 {
2176 IOSetKeyStoreData(md);
2177 }
6d2010ae 2178 }
6d2010ae 2179#endif
99c3a104
A		 2180 break;
2181
2182 default:
2183 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
2184 break;
2185 }
2186 }
6d2010ae
A		 2187 }
2188 vars->handoffBuffer->release();
2189 }
3a60a9f5
A		 2190 if (vars->fileExtents)
2191 vars->fileExtents->release();
2192
2193 bzero(vars, sizeof(*vars));
2194
2195// gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
2196
2197 return (kIOReturnSuccess);
2198}
2199
2200IOReturn
2201IOHibernateSystemPostWake(void)
2202{
316670eb
A		 2203 struct kern_direct_file_io_ref_t * fileRef;
2204
2205 if (kFSOpened == gFSState)
3a60a9f5 2206 {
6d2010ae 2207 // invalidate & close the image file
0c530ab8 2208 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
316670eb
A		 2209 if ((fileRef = gIOHibernateFileRef))
2210 {
2211 gIOHibernateFileRef = 0;
39236c6e 2212 IOSleep(TRIM_DELAY);
316670eb 2213 kern_close_file_for_direct_io(fileRef,
39236c6e
A		 2214#if DISABLE_TRIM
2215 0, 0, 0, 0, 0);
2216#else
6d2010ae 2217 0, (caddr_t) gIOHibernateCurrentHeader,
7ddcb079 2218 sizeof(IOHibernateImageHeader),
39236c6e 2219 0,
7ddcb079 2220 gIOHibernateCurrentHeader->imageSize);
39236c6e 2221#endif
316670eb
A		 2222 }
2223 gFSState = kFSIdle;
3a60a9f5 2224 }
fe8ab488
A		 2225
2226 if (gDebugImageFileRef) {
2227 kern_close_file_for_direct_io(gDebugImageFileRef, 0, 0, 0, 0, 0);
2228 gDebugImageFileRef = NULL;
2229 }
2230
2231 if (!gIOOptionsEntry)
2232 {
2233 IORegistryEntry * regEntry;
2234 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
2235 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
2236 if (regEntry && !gIOOptionsEntry)
2237 regEntry->release();
2238 }
2239 if (gIOOptionsEntry)
2240 {
2241 const OSSymbol * sym;
2242
2243 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
2244 if (sym)
2245 {
2246 gIOOptionsEntry->removeProperty(sym);
2247 gIOOptionsEntry->sync();
2248 sym->release();
2249 }
2250 }
2251
3a60a9f5
A		 2252 return (kIOReturnSuccess);
2253}
2254
316670eb
A		 2255bool IOHibernateWasScreenLocked(void)
2256{
2257 bool ret = false;
2258 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry)
2259 {
2260 OSData *
2261 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
2262 if (data) switch (*((uint32_t *)data->getBytesNoCopy()))
2263 {
2264 case kIOScreenLockLocked:
2265 case kIOScreenLockFileVaultDialog:
2266 ret = true;
2267 break;
2268 case kIOScreenLockNoLock:
2269 case kIOScreenLockUnlocked:
2270 default:
2271 ret = false;
2272 break;
2273 }
2274 }
2275 return (ret);
2276}
2277
3a60a9f5
A		 2278/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2279
2d21ac55 2280SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
6d2010ae 2281 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2d21ac55
A		 2282 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
2283SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
6d2010ae 2284 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2d21ac55
A		 2285 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
2286SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
6d2010ae 2287 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2d21ac55 2288 &gIOHibernateMode, 0, "");
39236c6e
A		 2289SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
2290 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2291 gIOHibernateStats, hibernate_statistics_t, "");
2292
2293SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
2294 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
2295 &gIOHibernateStats->graphicsReadyTime, 0, "");
2296SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
2297 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
2298 &gIOHibernateStats->wakeNotificationTime, 0, "");
2299SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
2300 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
2301 &gIOHibernateStats->lockScreenReadyTime, 0, "");
2302SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
2303 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
2304 &gIOHibernateStats->hidReadyTime, 0, "");
2305
2d21ac55 2306
3a60a9f5
A		 2307void
2308IOHibernateSystemInit(IOPMrootDomain * rootDomain)
2309{
2310 OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
2311 if (data)
2312 {
2313 rootDomain->setProperty(kIOHibernateStateKey, data);
2314 data->release();
2315 }
2316
2d21ac55 2317 if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename)))
3a60a9f5
A		 2318 gIOHibernateMode = kIOHibernateModeOn;
2319 else
2320 gIOHibernateFilename[0] = 0;
2321
3a60a9f5 2322 sysctl_register_oid(&sysctl__kern_hibernatefile);
3a60a9f5 2323 sysctl_register_oid(&sysctl__kern_bootsignature);
3a60a9f5 2324 sysctl_register_oid(&sysctl__kern_hibernatemode);
39236c6e
A		 2325 sysctl_register_oid(&sysctl__kern_hibernatestatistics);
2326 sysctl_register_oid(&sysctl__kern_hibernategraphicsready);
2327 sysctl_register_oid(&sysctl__kern_hibernatewakenotification);
2328 sysctl_register_oid(&sysctl__kern_hibernatelockscreenready);
2329 sysctl_register_oid(&sysctl__kern_hibernatehidready);
316670eb
A		 2330
2331 gFSLock = IOLockAlloc();
3a60a9f5
A		 2332}
2333
2d21ac55 2334
3a60a9f5
A		 2335/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2336
2337static void
2338hibernate_setup_for_wake(void)
2339{
3a60a9f5
A		 2340}
2341
2342/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2343
0c530ab8
A		 2344#define C_ASSERT(e) typedef char __C_ASSERT__[(e) ? 1 : -1]
2345
0b4c1975
A		 2346static bool
2347no_encrypt_page(vm_offset_t ppnum)
2348{
2349 if (pmap_is_noencrypt((ppnum_t)ppnum) == TRUE)
2350 {
2351 return true;
2352 }
2353 return false;
2354}
2355
6d2010ae
A		 2356static void
2357hibernate_pal_callback(void *vars_arg, vm_offset_t addr)
2358{
2359 IOHibernateVars *vars = (IOHibernateVars *)vars_arg;
2360 /* Make sure it's not in either of the save lists */
2361 hibernate_set_page_state(vars->page_list, vars->page_list_wired, atop_64(addr), 1, kIOHibernatePageStateFree);
2362
2363 /* Set it in the bitmap of pages owned by the PAL */
2364 hibernate_page_bitset(vars->page_list_pal, TRUE, atop_64(addr));
2365}
2366
2367static struct hibernate_cryptvars_t *local_cryptvars;
2368
2369extern "C" int
2370hibernate_pal_write(void *buffer, size_t size)
2371{
2372 IOHibernateVars * vars = &gIOHibernateVars;
2373
2374 IOReturn err = IOPolledFileWrite(vars->fileVars, (const uint8_t *)buffer, size, local_cryptvars);
2375 if (kIOReturnSuccess != err) {
2376 kprintf("epic hibernate fail! %d\n", err);
2377 return err;
2378 }
2379
2380 return 0;
2381}
2382
2383
2d21ac55 2384extern "C" uint32_t
3a60a9f5
A		 2385hibernate_write_image(void)
2386{
2387 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
2388 IOHibernateVars * vars = &gIOHibernateVars;
2389 IOPolledFileExtent * fileExtents;
2390
0c530ab8
A		 2391 C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
2392
3a60a9f5
A		 2393 uint32_t pageCount, pagesDone;
2394 IOReturn err;
0b4c1975
A		 2395 vm_offset_t ppnum, page;
2396 IOItemCount count;
3a60a9f5
A		 2397 uint8_t * src;
2398 uint8_t * data;
39236c6e
A		 2399 uint8_t * compressed;
2400 uint8_t * scratch;
2401 void * zerosCompressed;
2402 IOByteCount pageCompressedSize, zerosCompressedLen;
3a60a9f5
A		 2403 uint64_t compressedSize, uncompressedSize;
2404 uint64_t image1Size = 0;
2405 uint32_t bitmap_size;
0b4c1975 2406 bool iterDone, pollerOpen, needEncrypt;
3a60a9f5 2407 uint32_t restore1Sum, sum, sum1, sum2;
39236c6e 2408 int wkresult;
3a60a9f5
A		 2409 uint32_t tag;
2410 uint32_t pageType;
2411 uint32_t pageAndCount[2];
6d2010ae
A		 2412 addr64_t phys64;
2413 IOByteCount segLen;
3a60a9f5
A		 2414
2415 AbsoluteTime startTime, endTime;
6d2010ae 2416 AbsoluteTime allTime, compTime;
0b4c1975 2417 uint64_t compBytes;
3a60a9f5
A		 2418 uint64_t nsec;
2419 uint32_t lastProgressStamp = 0;
2420 uint32_t progressStamp;
2d21ac55 2421 uint32_t blob, lastBlob = (uint32_t) -1L;
3a60a9f5 2422
39236c6e
A		 2423 uint32_t wiredPagesEncrypted;
2424 uint32_t dirtyPagesEncrypted;
2425 uint32_t wiredPagesClear;
2426 uint32_t zeroPageCount;
2427
3a60a9f5
A		 2428 hibernate_cryptvars_t _cryptvars;
2429 hibernate_cryptvars_t * cryptvars = 0;
2430
39236c6e
A		 2431 wiredPagesEncrypted = 0;
2432 dirtyPagesEncrypted = 0;
2433 wiredPagesClear = 0;
2434 zeroPageCount = 0;
0b4c1975 2435
3a60a9f5
A		 2436 if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
2437 return (false /* sleep */ );
2438
0b4c1975
A		 2439 if (kIOHibernateModeSleep & gIOHibernateMode)
2440 kdebug_enable = save_kdebug_enable;
2441
2442 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START, 0, 0, 0, 0, 0);
6d2010ae 2443 IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
0b4c1975 2444
3a60a9f5
A		 2445 restore1Sum = sum1 = sum2 = 0;
2446
6d2010ae
A		 2447 hibernate_pal_prepare();
2448
2d21ac55 2449#if CRYPTO
3a60a9f5
A		 2450 // encryption data. "iv" is the "initial vector".
2451 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2452 {
2453 static const unsigned char first_iv[AES_BLOCK_SIZE]
2454 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
2455 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
2456
2457 cryptvars = &gIOHibernateCryptWakeContext;
2458 bzero(cryptvars, sizeof(hibernate_cryptvars_t));
2459 aes_encrypt_key(vars->cryptKey,
2460 kIOHibernateAESKeySize,
2461 &cryptvars->ctx.encrypt);
2462 aes_decrypt_key(vars->cryptKey,
2463 kIOHibernateAESKeySize,
2464 &cryptvars->ctx.decrypt);
2465
2466 cryptvars = &_cryptvars;
2467 bzero(cryptvars, sizeof(hibernate_cryptvars_t));
6d2010ae
A		 2468 for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++)
2469 vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
2470 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
3a60a9f5
A		 2471 aes_encrypt_key(vars->wiredCryptKey,
2472 kIOHibernateAESKeySize,
2473 &cryptvars->ctx.encrypt);
2474
2475 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
2476 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
2477 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
6d2010ae
A		 2478
2479 local_cryptvars = cryptvars;
3a60a9f5 2480 }
2d21ac55 2481#endif /* CRYPTO */
3a60a9f5
A		 2482
2483 hibernate_setup_for_wake();
2484
2485 hibernate_page_list_setall(vars->page_list,
2486 vars->page_list_wired,
db609669
A		 2487 vars->page_list_pal,
2488 false /* !preflight */,
39236c6e
A		 2489 /* discard_all */
2490 ((0 == (kIOHibernateModeSleep & gIOHibernateMode))
2491 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))),
3a60a9f5
A		 2492 &pageCount);
2493
2494 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
2495
2496 fileExtents = (IOPolledFileExtent *) vars->fileExtents->getBytesNoCopy();
2497
2498#if 0
2499 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
2500 for (page = 0; page < count; page++)
2501 {
2502 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
2503 fileExtents[page].start, fileExtents[page].length,
2504 fileExtents[page].start + fileExtents[page].length);
2505 }
2506#endif
2507
0b4c1975 2508 needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
3a60a9f5 2509 AbsoluteTime_to_scalar(&compTime) = 0;
6d2010ae 2510 compBytes = 0;
3a60a9f5
A		 2511
2512 clock_get_uptime(&allTime);
b0d623f7
A		 2513 IOService::getPMRootDomain()->pmStatsRecordEvent(
2514 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
3a60a9f5
A		 2515 do
2516 {
2517 compressedSize = 0;
2518 uncompressedSize = 0;
39236c6e 2519 zeroPageCount = 0;
3a60a9f5 2520
39236c6e 2521 IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);
3a60a9f5
A		 2522
2523 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
2524 ml_get_interrupts_enabled());
2525 err = IOHibernatePollerOpen(vars->fileVars, kIOPolledBeforeSleepState, vars->ioBuffer);
2526 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
2527 pollerOpen = (kIOReturnSuccess == err);
2528 if (!pollerOpen)
2529 break;
2530
2531 // copy file block extent list if larger than header
2532
2533 count = vars->fileExtents->getLength();
2534 if (count > sizeof(header->fileExtentMap))
2535 {
2536 count -= sizeof(header->fileExtentMap);
2537 err = IOPolledFileWrite(vars->fileVars,
2538 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
2539 if (kIOReturnSuccess != err)
2540 break;
2541 }
2542
b0d623f7
A		 2543 uintptr_t hibernateBase;
2544 uintptr_t hibernateEnd;
2545
6d2010ae 2546 hibernateBase = HIB_BASE; /* Defined in PAL headers */
b0d623f7 2547
316670eb 2548 hibernateEnd = (segHIBB + segSizeHIB);
6d2010ae 2549
3a60a9f5 2550 // copy out restore1 code
6d2010ae
A		 2551
2552 for (count = 0;
2553 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2554 count += segLen)
2555 {
2556 for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++)
2557 {
2558 gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone;
2559 }
2560 }
2561
2562 page = atop_32(kvtophys(hibernateBase));
2563 count = atop_32(round_page(hibernateEnd) - hibernateBase);
2564 header->restore1CodePhysPage = page;
2565 header->restore1CodeVirt = hibernateBase;
3a60a9f5 2566 header->restore1PageCount = count;
b0d623f7
A		 2567 header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
2568 header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
3a60a9f5 2569
316670eb 2570 // sum __HIB seg, with zeros for the stack
b0d623f7 2571 src = (uint8_t *) trunc_page(hibernateBase);
3a60a9f5
A		 2572 for (page = 0; page < count; page++)
2573 {
2574 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
6d2010ae 2575 restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page);
3a60a9f5 2576 else
0b4c1975 2577 restore1Sum += 0x00000000;
3a60a9f5
A		 2578 src += page_size;
2579 }
2580 sum1 = restore1Sum;
2581
316670eb 2582 // write the __HIB seg, with zeros for the stack
3a60a9f5 2583
b0d623f7
A		 2584 src = (uint8_t *) trunc_page(hibernateBase);
2585 count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
3a60a9f5
A		 2586 if (count)
2587 {
2588 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
2589 if (kIOReturnSuccess != err)
2590 break;
2591 }
2592 err = IOPolledFileWrite(vars->fileVars,
2593 (uint8_t *) 0,
2594 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
2595 cryptvars);
2596 if (kIOReturnSuccess != err)
2597 break;
2598 src = &gIOHibernateRestoreStackEnd[0];
b0d623f7 2599 count = round_page(hibernateEnd) - ((uintptr_t) src);
3a60a9f5
A		 2600 if (count)
2601 {
2602 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
2603 if (kIOReturnSuccess != err)
2604 break;
2605 }
2606
39236c6e
A		 2607 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2608 {
2609 vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
2610 vars->fileVars->encryptEnd = UINT64_MAX;
2611 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
2612 }
316670eb 2613
3a60a9f5
A		 2614 // write the preview buffer
2615
2d21ac55 2616 if (vars->previewBuffer)
3a60a9f5
A		 2617 {
2618 ppnum = 0;
2619 count = 0;
2620 do
2621 {
b0d623f7 2622 phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
3a60a9f5
A		 2623 pageAndCount[0] = atop_64(phys64);
2624 pageAndCount[1] = atop_32(segLen);
2625 err = IOPolledFileWrite(vars->fileVars,
2626 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
2627 cryptvars);
2628 if (kIOReturnSuccess != err)
2629 break;
2630 count += segLen;
2631 ppnum += sizeof(pageAndCount);
2632 }
2633 while (phys64);
2634 if (kIOReturnSuccess != err)
2635 break;
2636
b0d623f7 2637 src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
316670eb
A		 2638
2639 ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
2640
2d21ac55 2641 count = vars->previewBuffer->getLength();
3a60a9f5
A		 2642
2643 header->previewPageListSize = ppnum;
2644 header->previewSize = count + ppnum;
2645
2646 for (page = 0; page < count; page += page_size)
0b4c1975
A		 2647 {
2648 phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
2649 sum1 += hibernate_sum_page(src + page, atop_64(phys64));
2650 }
3a60a9f5
A		 2651 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
2652 if (kIOReturnSuccess != err)
2653 break;
2654 }
2655
2656 // mark areas for no save
2657
2658 for (count = 0;
b0d623f7 2659 (phys64 = vars->ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
3a60a9f5
A		 2660 count += segLen)
2661 {
2662 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2663 atop_64(phys64), atop_32(segLen),
2664 kIOHibernatePageStateFree);
2665 pageCount -= atop_32(segLen);
2666 }
2667
2668 for (count = 0;
b0d623f7 2669 (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
3a60a9f5
A		 2670 count += segLen)
2671 {
2672 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2673 atop_64(phys64), atop_32(segLen),
2674 kIOHibernatePageStateFree);
2675 pageCount -= atop_32(segLen);
2676 }
2677
2678 // copy out bitmap of pages available for trashing during restore
2679
2680 bitmap_size = vars->page_list_wired->list_size;
2681 src = (uint8_t *) vars->page_list_wired;
2682 err = IOPolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
2683 if (kIOReturnSuccess != err)
2684 break;
2685
2686 // mark more areas for no save, but these are not available
2687 // for trashing during restore
0c530ab8
A		 2688
2689 hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
3a60a9f5 2690
6d2010ae
A		 2691
2692 page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
2693 count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
3a60a9f5
A		 2694 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2695 page, count,
2696 kIOHibernatePageStateFree);
2697 pageCount -= count;
3a60a9f5
A		 2698
2699 if (vars->previewBuffer) for (count = 0;
b0d623f7 2700 (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
3a60a9f5
A		 2701 count += segLen)
2702 {
2703 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2704 atop_64(phys64), atop_32(segLen),
2705 kIOHibernatePageStateFree);
2706 pageCount -= atop_32(segLen);
2707 }
2708
6d2010ae
A		 2709 for (count = 0;
2710 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2711 count += segLen)
2712 {
2713 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2714 atop_64(phys64), atop_32(segLen),
2715 kIOHibernatePageStateFree);
2716 pageCount -= atop_32(segLen);
2717 }
2718
2719 (void)hibernate_pal_callback;
2720
3a60a9f5 2721 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
39236c6e
A		 2722 compressed = src + page_size;
2723 scratch = compressed + page_size;
2724
2725 // compress a zero page
2726 bzero(src, page_size);
2727 zerosCompressed = vars->handoffBuffer->getBytesNoCopy();
2728 zerosCompressedLen = WKdm_compress_new((WK_word*) src,
2729 (WK_word*) zerosCompressed,
2730 (WK_word*) scratch,
2731 page_size - 4);
2732
6d2010ae
A		 2733 pagesDone = 0;
2734 lastBlob = 0;
3a60a9f5 2735
db609669
A		 2736 HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
2737 bitmap_size, header->previewSize,
2738 pageCount, vars->fileVars->position);
3a60a9f5 2739
0b4c1975
A		 2740 enum
2741 // pageType
2742 {
2743 kWired = 0x02,
2744 kEncrypt = 0x01,
2745 kWiredEncrypt = kWired | kEncrypt,
2746 kWiredClear = kWired,
2747 kUnwiredEncrypt = kEncrypt
2748 };
2749
2750 for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--)
3a60a9f5 2751 {
316670eb
A		 2752 if (kUnwiredEncrypt == pageType)
2753 {
2754 // start unwired image
39236c6e
A		 2755 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2756 {
2757 vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
2758 vars->fileVars->encryptEnd = UINT64_MAX;
2759 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
2760 }
316670eb
A		 2761 bcopy(&cryptvars->aes_iv[0],
2762 &gIOHibernateCryptWakeContext.aes_iv[0],
2763 sizeof(cryptvars->aes_iv));
2764 cryptvars = &gIOHibernateCryptWakeContext;
0b4c1975
A		 2765 }
2766 for (iterDone = false, ppnum = 0; !iterDone; )
2767 {
2768 count = hibernate_page_list_iterate((kWired & pageType)
2769 ? vars->page_list_wired : vars->page_list,
2770 &ppnum);
2771// kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
2772 iterDone = !count;
3a60a9f5 2773
0b4c1975 2774 if (count && (kWired & pageType) && needEncrypt)
3a60a9f5 2775 {
0b4c1975
A		 2776 uint32_t checkIndex;
2777 for (checkIndex = 0;
2778 (checkIndex < count)
2779 && (((kEncrypt & pageType) == 0) == no_encrypt_page(ppnum + checkIndex));
2780 checkIndex++)
2781 {}
2782 if (!checkIndex)
2783 {
2784 ppnum++;
2785 continue;
2786 }
2787 count = checkIndex;
3a60a9f5
A		 2788 }
2789
0b4c1975
A		 2790 switch (pageType)
2791 {
39236c6e
A		 2792 case kWiredEncrypt: wiredPagesEncrypted += count; break;
2793 case kWiredClear: wiredPagesClear += count; break;
2794 case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
0b4c1975
A		 2795 }
2796
2797 if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */}
3a60a9f5 2798 else
3a60a9f5 2799 {
0b4c1975
A		 2800 pageAndCount[0] = ppnum;
2801 pageAndCount[1] = count;
2802 err = IOPolledFileWrite(vars->fileVars,
2803 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
2804 cryptvars);
2805 if (kIOReturnSuccess != err)
2806 break;
3a60a9f5 2807 }
0b4c1975
A		 2808
2809 for (page = ppnum; page < (ppnum + count); page++)
3a60a9f5 2810 {
0b4c1975
A		 2811 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
2812 if (err)
2813 {
2814 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
2815 break;
2816 }
2817
2818 sum = hibernate_sum_page(src, page);
2819 if (kWired & pageType)
2820 sum1 += sum;
2821 else
2822 sum2 += sum;
2823
2824 clock_get_uptime(&startTime);
39236c6e
A		 2825 wkresult = WKdm_compress_new((WK_word*) src,
2826 (WK_word*) compressed,
2827 (WK_word*) scratch,
2828 page_size - 4);
0b4c1975 2829
3a60a9f5 2830 clock_get_uptime(&endTime);
0b4c1975
A		 2831 ADD_ABSOLUTETIME(&compTime, &endTime);
2832 SUB_ABSOLUTETIME(&compTime, &startTime);
39236c6e 2833
0b4c1975 2834 compBytes += page_size;
39236c6e
A		 2835 pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
2836
2837 if ((pageCompressedSize == zerosCompressedLen)
2838 && !bcmp(compressed, zerosCompressed, zerosCompressedLen))
2839 {
2840 pageCompressedSize = 0;
2841 zeroPageCount++;
2842 }
2843
0b4c1975
A		 2844 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2845 pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1);
db609669 2846
0b4c1975 2847 if (pageCompressedSize != page_size)
39236c6e 2848 data = compressed;
0b4c1975
A		 2849 else
2850 data = src;
2851
2852 tag = pageCompressedSize | kIOHibernateTagSignature;
2853 err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
2854 if (kIOReturnSuccess != err)
2855 break;
2856
2857 err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
2858 if (kIOReturnSuccess != err)
2859 break;
2860
2861 compressedSize += pageCompressedSize;
39236c6e 2862 uncompressedSize += page_size;
0b4c1975
A		 2863 pagesDone++;
2864
2865 if (vars->consoleMapping && (0 == (1023 & pagesDone)))
2866 {
2867 blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
2868 if (blob != lastBlob)
2869 {
2870 ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
2871 lastBlob = blob;
2872 }
2873 }
2874 if (0 == (8191 & pagesDone))
2875 {
2876 clock_get_uptime(&endTime);
2877 SUB_ABSOLUTETIME(&endTime, &allTime);
2878 absolutetime_to_nanoseconds(endTime, &nsec);
2879 progressStamp = nsec / 750000000ULL;
2880 if (progressStamp != lastProgressStamp)
2881 {
2882 lastProgressStamp = progressStamp;
2883 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
2884 }
3a60a9f5
A		 2885 }
2886 }
0b4c1975
A		 2887 if (kIOReturnSuccess != err)
2888 break;
2889 ppnum = page;
3a60a9f5 2890 }
0b4c1975 2891
3a60a9f5
A		 2892 if (kIOReturnSuccess != err)
2893 break;
0b4c1975 2894
39236c6e 2895 if ((kEncrypt & pageType) && vars->fileVars->encryptStart)
3a60a9f5 2896 {
7ddcb079 2897 vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
0b4c1975
A		 2898 HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
2899 }
2900
2901 if (kWiredEncrypt != pageType)
2902 {
2903 // end of image1/2 - fill to next block
3a60a9f5
A		 2904 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
2905 if (kIOReturnSuccess != err)
2906 break;
0b4c1975
A		 2907 }
2908 if (kWiredClear == pageType)
2909 {
7ddcb079
A		 2910 // enlarge wired image for test
2911// err = IOPolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);
2912
0b4c1975
A		 2913 // end wired image
2914 header->encryptStart = vars->fileVars->encryptStart;
2915 header->encryptEnd = vars->fileVars->encryptEnd;
3a60a9f5 2916 image1Size = vars->fileVars->position;
7ddcb079 2917 HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
0b4c1975 2918 image1Size, header->encryptStart, header->encryptEnd);
3a60a9f5
A		 2919 }
2920 }
3a60a9f5 2921 if (kIOReturnSuccess != err)
8a3053a0
A		 2922 {
2923 if (kIOReturnOverrun == err)
2924 {
2925 // update actual compression ratio on not enough space
2926 gIOHibernateCompression = (compressedSize << 8) / uncompressedSize;
2927 }
3a60a9f5 2928 break;
8a3053a0 2929 }
3a60a9f5
A		 2930
2931 // Header:
2932
2933 header->imageSize = vars->fileVars->position;
2934 header->image1Size = image1Size;
2935 header->bitmapSize = bitmap_size;
2936 header->pageCount = pageCount;
3a60a9f5
A		 2937
2938 header->restore1Sum = restore1Sum;
2939 header->image1Sum = sum1;
2940 header->image2Sum = sum2;
316670eb 2941 header->sleepTime = gIOLastSleepTime.tv_sec;
db609669
A		 2942
2943 header->compression = (compressedSize << 8) / uncompressedSize;
2944 gIOHibernateCompression = header->compression;
3a60a9f5
A		 2945
2946 count = vars->fileExtents->getLength();
2947 if (count > sizeof(header->fileExtentMap))
2948 {
2949 header->fileExtentMapSize = count;
2950 count = sizeof(header->fileExtentMap);
2951 }
2952 else
2953 header->fileExtentMapSize = sizeof(header->fileExtentMap);
2954 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
0b4c1975 2955
39236c6e
A		 2956 header->deviceBase = vars->fileVars->block0;
2957 header->deviceBlockSize = vars->fileVars->blockSize;
6d2010ae 2958
3a60a9f5
A		 2959 IOPolledFileSeek(vars->fileVars, 0);
2960 err = IOPolledFileWrite(vars->fileVars,
2961 (uint8_t *) header, sizeof(IOHibernateImageHeader),
2962 cryptvars);
2963 if (kIOReturnSuccess != err)
2964 break;
2965 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
2966 if (kIOReturnSuccess != err)
2967 break;
2d21ac55 2968 err = IOHibernatePollerIODone(vars->fileVars, true);
3a60a9f5
A		 2969 if (kIOReturnSuccess != err)
2970 break;
2971 }
2972 while (false);
2973
2974 clock_get_uptime(&endTime);
b0d623f7
A		 2975
2976 IOService::getPMRootDomain()->pmStatsRecordEvent(
2977 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
2978
3a60a9f5
A		 2979 SUB_ABSOLUTETIME(&endTime, &allTime);
2980 absolutetime_to_nanoseconds(endTime, &nsec);
39236c6e 2981 HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
3a60a9f5
A		 2982
2983 absolutetime_to_nanoseconds(compTime, &nsec);
6d2010ae
A		 2984 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2985 compBytes,
2986 nsec / 1000000ULL,
2987 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
3a60a9f5 2988
6d2010ae
A		 2989 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2990 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
2991 vars->fileVars->cryptBytes,
2992 nsec / 1000000ULL,
2993 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
3a60a9f5 2994
db609669
A		 2995 HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
2996 header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
3a60a9f5 2997 uncompressedSize, atop_32(uncompressedSize), compressedSize,
8f6c56a5 2998 uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
3a60a9f5
A		 2999 sum1, sum2);
3000
39236c6e
A		 3001 HIBLOG("zeroPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n",
3002 zeroPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
0b4c1975 3003
c910b4d9
A		 3004 if (vars->fileVars->io)
3005 (void) IOHibernatePollerIODone(vars->fileVars, false);
3006
3a60a9f5
A		 3007 if (pollerOpen)
3008 IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState);
3009
2d21ac55
A		 3010 if (vars->consoleMapping)
3011 ProgressUpdate(gIOHibernateGraphicsInfo,
3012 vars->consoleMapping, 0, kIOHibernateProgressCount);
3013
3a60a9f5
A		 3014 HIBLOG("hibernate_write_image done(%x)\n", err);
3015
3016 // should we come back via regular wake, set the state in memory.
3017 gIOHibernateState = kIOHibernateStateInactive;
3018
0b4c1975 3019 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END,
39236c6e 3020 wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted, 0, 0);
0b4c1975 3021
2d21ac55
A		 3022 if (kIOReturnSuccess == err)
3023 {
3024 if (kIOHibernateModeSleep & gIOHibernateMode)
3025 {
3026 return (kIOHibernatePostWriteSleep);
3027 }
3028 else if(kIOHibernateModeRestart & gIOHibernateMode)
3029 {
3030 return (kIOHibernatePostWriteRestart);
3031 }
3032 else
3033 {
3034 /* by default, power down */
3035 return (kIOHibernatePostWriteHalt);
3036 }
3037 }
3038 else if (kIOReturnAborted == err)
3039 {
3040 return (kIOHibernatePostWriteWake);
3041 }
3a60a9f5 3042 else
3a60a9f5 3043 {
2d21ac55
A		 3044 /* on error, sleep */
3045 return (kIOHibernatePostWriteSleep);
3a60a9f5
A		 3046 }
3047}
3048
3049/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
3050
3051extern "C" void
3052hibernate_machine_init(void)
3053{
3054 IOReturn err;
3055 uint32_t sum;
3056 uint32_t pagesDone;
0b4c1975 3057 uint32_t pagesRead = 0;
6d2010ae 3058 AbsoluteTime startTime, compTime;
3a60a9f5 3059 AbsoluteTime allTime, endTime;
39236c6e
A		 3060 AbsoluteTime startIOTime, endIOTime;
3061 uint64_t nsec, nsecIO;
6d2010ae 3062 uint64_t compBytes;
3a60a9f5
A		 3063 uint32_t lastProgressStamp = 0;
3064 uint32_t progressStamp;
3a60a9f5
A		 3065 hibernate_cryptvars_t * cryptvars = 0;
3066
3067 IOHibernateVars * vars = &gIOHibernateVars;
39236c6e 3068 bzero(gIOHibernateStats, sizeof(hibernate_statistics_t));
3a60a9f5
A		 3069
3070 if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
3071 return;
3072
3a60a9f5
A		 3073 sum = gIOHibernateCurrentHeader->actualImage1Sum;
3074 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
3075
3a60a9f5
A		 3076 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
3077 {
3078 HIBLOG("regular wake\n");
3079 return;
3080 }
3081
3082 HIBPRINT("diag %x %x %x %x\n",
3083 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
6d2010ae 3084 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
3a60a9f5 3085
39236c6e
A		 3086#define t40ms(x) (tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000)
3087#define tStat(x, y) gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y);
3088 tStat(booterStart, booterStart);
3089 gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart,
3090 tStat(booterDuration0, booterTime0);
3091 tStat(booterDuration1, booterTime1);
3092 tStat(booterDuration2, booterTime2);
3093 tStat(booterDuration, booterTime);
3094 tStat(booterConnectDisplayDuration, connectDisplayTime);
3095 tStat(booterSplashDuration, splashTime);
3096 tStat(trampolineDuration, trampolineTime);
3097
3098 gIOHibernateStats->image1Size = gIOHibernateCurrentHeader->image1Size;
3099 gIOHibernateStats->imageSize = gIOHibernateCurrentHeader->imageSize;
3100 gIOHibernateStats->image1Pages = pagesDone;
3101
3102 HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n",
3103 gIOHibernateStats->booterStart,
3104 gIOHibernateStats->smcStart,
3105 gIOHibernateStats->booterDuration0,
3106 gIOHibernateStats->booterDuration1,
3107 gIOHibernateStats->booterDuration2,
3108 gIOHibernateStats->booterDuration,
3109 gIOHibernateStats->booterConnectDisplayDuration,
3110 gIOHibernateStats->booterSplashDuration,
3111 gIOHibernateStats->trampolineDuration);
3112
3113 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, imageSize 0x%qx, image1Size 0x%qx, conflictCount %d, nextFree %x\n",
3114 gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size,
3115 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
bd504ef0 3116
39236c6e
A		 3117 if ((0 != (kIOHibernateModeSleep & gIOHibernateMode))
3118 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)))
3119 {
0c530ab8 3120 hibernate_page_list_discard(vars->page_list);
39236c6e 3121 }
0c530ab8 3122
6d2010ae
A		 3123 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
3124
3125 if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount)
3126 panic("handoff overflow");
3127
3128 IOHibernateHandoff * handoff;
3129 bool done = false;
3130 bool foundCryptData = false;
3131
3132 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
3133 !done;
3134 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
3135 {
3136// HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
3137 uint8_t * data = &handoff->data[0];
3138 switch (handoff->type)
3139 {
3140 case kIOHibernateHandoffTypeEnd:
3141 done = true;
3142 break;
3143
3144 case kIOHibernateHandoffTypeGraphicsInfo:
3145 bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
3146 break;
3147
3148 case kIOHibernateHandoffTypeCryptVars:
3149 if (cryptvars)
3150 {
3151 hibernate_cryptwakevars_t *
3152 wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
3153 bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
3154 }
3155 foundCryptData = true;
3156 bzero(data, handoff->bytecount);
3157 break;
3158
3159 case kIOHibernateHandoffTypeMemoryMap:
bd504ef0
A		 3160
3161 clock_get_uptime(&allTime);
3162
6d2010ae
A		 3163 hibernate_newruntime_map(data, handoff->bytecount,
3164 gIOHibernateCurrentHeader->systemTableOffset);
bd504ef0
A		 3165
3166 clock_get_uptime(&endTime);
3167
3168 SUB_ABSOLUTETIME(&endTime, &allTime);
3169 absolutetime_to_nanoseconds(endTime, &nsec);
3170
3171 HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL);
3172
6d2010ae
A		 3173 break;
3174
3175 case kIOHibernateHandoffTypeDeviceTree:
3176 {
3177// DTEntry chosen = NULL;
3178// HIBPRINT("DTLookupEntry %d\n", DTLookupEntry((const DTEntry) data, "/chosen", &chosen));
3179 }
3180 break;
3181
3182 default:
3183 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
3184 break;
3185 }
3186 }
3187 if (cryptvars && !foundCryptData)
3188 panic("hibernate handoff");
3189
316670eb
A		 3190 HIBPRINT("video %x %d %d %d status %x\n",
3191 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
3192 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus);
3193
3194 if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress)
3a60a9f5
A		 3195 {
3196 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
3197 * gIOHibernateGraphicsInfo->rowBytes);
22ba694c
A		 3198 if (vars->videoMapSize > vars->videoAllocSize) vars->videoMapSize = 0;
3199 else
3200 {
3201 IOMapPages(kernel_map,
3202 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
3203 vars->videoMapSize, kIOMapInhibitCache );
3204 }
3a60a9f5
A		 3205 }
3206
316670eb
A		 3207 if (vars->videoMapSize)
3208 ProgressUpdate(gIOHibernateGraphicsInfo,
3209 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
3210
0c530ab8 3211 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
39236c6e
A		 3212 uint8_t * compressed = src + page_size;
3213 uint8_t * scratch = compressed + page_size;
3214 uint32_t decoOffset;
3a60a9f5
A		 3215
3216 clock_get_uptime(&allTime);
6d2010ae
A		 3217 AbsoluteTime_to_scalar(&compTime) = 0;
3218 compBytes = 0;
3a60a9f5
A		 3219
3220 HIBLOG("IOHibernatePollerOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
3221 err = IOHibernatePollerOpen(vars->fileVars, kIOPolledAfterSleepState, 0);
39236c6e
A		 3222 clock_get_uptime(&startIOTime);
3223 endTime = startIOTime;
3224 SUB_ABSOLUTETIME(&endTime, &allTime);
3225 absolutetime_to_nanoseconds(endTime, &nsec);
3226 HIBLOG("IOHibernatePollerOpen(%x) %qd ms\n", err, nsec / 1000000ULL);
3a60a9f5 3227
3a60a9f5
A		 3228 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
3229
3a60a9f5
A		 3230 // kick off the read ahead
3231 vars->fileVars->io = false;
3232 vars->fileVars->bufferHalf = 0;
3233 vars->fileVars->bufferLimit = 0;
3234 vars->fileVars->lastRead = 0;
6d2010ae 3235 vars->fileVars->readEnd = gIOHibernateCurrentHeader->imageSize;
3a60a9f5 3236 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
6d2010ae
A		 3237 vars->fileVars->cryptBytes = 0;
3238 AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;
3a60a9f5 3239
6d2010ae 3240 err = IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
3a60a9f5
A		 3241 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
3242 // --
3243
3244 HIBLOG("hibernate_machine_init reading\n");
3245
3246 uint32_t * header = (uint32_t *) src;
3247 sum = 0;
3248
6d2010ae 3249 while (kIOReturnSuccess == err)
3a60a9f5
A		 3250 {
3251 unsigned int count;
3252 unsigned int page;
3253 uint32_t tag;
3254 vm_offset_t ppnum, compressedSize;
3255
36401178
A		 3256 err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
3257 if (kIOReturnSuccess != err)
3258 break;
3a60a9f5
A		 3259
3260 ppnum = header[0];
3261 count = header[1];
3262
3263// HIBPRINT("(%x, %x)\n", ppnum, count);
3264
3265 if (!count)
3266 break;
3267
3268 for (page = 0; page < count; page++)
3269 {
36401178
A		 3270 err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
3271 if (kIOReturnSuccess != err)
3272 break;
3a60a9f5
A		 3273
3274 compressedSize = kIOHibernateTagLength & tag;
36401178
A		 3275 if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength))
3276 {
3277 err = kIOReturnIPCError;
3278 break;
3279 }
3280
39236c6e 3281 if (!compressedSize) bzero_phys(ptoa_64(ppnum), page_size);
3a60a9f5 3282 else
36401178 3283 {
39236c6e
A		 3284 err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
3285 if (kIOReturnSuccess != err) break;
3286 if (compressedSize < page_size)
3287 {
3288 decoOffset = page_size;
3289 clock_get_uptime(&startTime);
3290 WKdm_decompress_new((WK_word*) src, (WK_word*) compressed, (WK_word*) scratch, page_size);
3291 clock_get_uptime(&endTime);
3292 ADD_ABSOLUTETIME(&compTime, &endTime);
3293 SUB_ABSOLUTETIME(&compTime, &startTime);
3294 compBytes += page_size;
3295 }
3296 else decoOffset = 0;
3297
3298 sum += hibernate_sum_page((src + decoOffset), ppnum);
3299 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
3300 if (err)
3301 {
3302 HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
3303 break;
3304 }
36401178 3305 }
3a60a9f5
A		 3306
3307 ppnum++;
3308 pagesDone++;
0b4c1975 3309 pagesRead++;
3a60a9f5 3310
3a60a9f5
A		 3311 if (0 == (8191 & pagesDone))
3312 {
3313 clock_get_uptime(&endTime);
3314 SUB_ABSOLUTETIME(&endTime, &allTime);
3315 absolutetime_to_nanoseconds(endTime, &nsec);
3316 progressStamp = nsec / 750000000ULL;
3317 if (progressStamp != lastProgressStamp)
3318 {
3319 lastProgressStamp = progressStamp;
3320 HIBPRINT("pages %d (%d%%)\n", pagesDone,
3321 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
3322 }
3323 }
3324 }
3325 }
7ddcb079
A		 3326 if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages))
3327 err = kIOReturnLockedRead;
3a60a9f5 3328
36401178
A		 3329 if (kIOReturnSuccess != err)
3330 panic("Hibernate restore error %x", err);
3331
3a60a9f5 3332 gIOHibernateCurrentHeader->actualImage2Sum = sum;
db609669 3333 gIOHibernateCompression = gIOHibernateCurrentHeader->compression;
3a60a9f5
A		 3334
3335 if (vars->fileVars->io)
2d21ac55 3336 (void) IOHibernatePollerIODone(vars->fileVars, false);
3a60a9f5 3337
39236c6e
A		 3338 clock_get_uptime(&endIOTime);
3339
3a60a9f5
A		 3340 err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState);
3341
3a60a9f5 3342 clock_get_uptime(&endTime);
b0d623f7
A		 3343
3344 IOService::getPMRootDomain()->pmStatsRecordEvent(
3345 kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
3346 IOService::getPMRootDomain()->pmStatsRecordEvent(
3347 kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
3348
3a60a9f5
A		 3349 SUB_ABSOLUTETIME(&endTime, &allTime);
3350 absolutetime_to_nanoseconds(endTime, &nsec);
3351
39236c6e
A		 3352 SUB_ABSOLUTETIME(&endIOTime, &startIOTime);
3353 absolutetime_to_nanoseconds(endIOTime, &nsecIO);
3354
3355 gIOHibernateStats->kernelImageReadDuration = nsec / 1000000ULL;
3356 gIOHibernateStats->imagePages = pagesDone;
3357
3358 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ",
3359 pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize,
3360 nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0);
6d2010ae
A		 3361
3362 absolutetime_to_nanoseconds(compTime, &nsec);
3363 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
3364 compBytes,
3365 nsec / 1000000ULL,
3366 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
3367
3368 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
3369 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n",
3370 vars->fileVars->cryptBytes,
3371 nsec / 1000000ULL,
3372 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
3373
3374 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_NONE, pagesRead, pagesDone, 0, 0, 0);
3a60a9f5
A		 3375}
3376
3377/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
99c3a104 3378
39236c6e
A		 3379void IOHibernateSetWakeCapabilities(uint32_t capability)
3380{
3381 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
3382 {
3383 gIOHibernateStats->wakeCapability = capability;
8a3053a0
A		 3384
3385 if (kIOPMSystemCapabilityGraphics & capability)
3386 {
3387 vm_compressor_do_warmup();
3388 }
39236c6e
A		 3389 }
3390}
3391
3392/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
3393
99c3a104
A		 3394void IOHibernateSystemRestart(void)
3395{
3396 static uint8_t noteStore[32] __attribute__((aligned(32)));
3397 IORegistryEntry * regEntry;
3398 const OSSymbol * sym;
3399 OSData * noteProp;
3400 OSData * data;
3401 uintptr_t * smcVars;
3402 uint8_t * smcBytes;
3403 size_t len;
3404 addr64_t element;
3405
3406 data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
3407 if (!data) return;
3408
3409 smcVars = (typeof(smcVars)) data->getBytesNoCopy();
3410 smcBytes = (typeof(smcBytes)) smcVars[1];
3411 len = smcVars[0];
3412 if (len > sizeof(noteStore)) len = sizeof(noteStore);
3413 noteProp = OSData::withCapacity(3 * sizeof(element));
3414 if (!noteProp) return;
3415 element = len;
3416 noteProp->appendBytes(&element, sizeof(element));
3417 element = crc32(0, smcBytes, len);
3418 noteProp->appendBytes(&element, sizeof(element));
3419
3420 bcopy(smcBytes, noteStore, len);
3421 element = (addr64_t) &noteStore[0];
3422 element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
3423 noteProp->appendBytes(&element, sizeof(element));
3424
3425 if (!gIOOptionsEntry)
3426 {
3427 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
3428 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
3429 if (regEntry && !gIOOptionsEntry)
3430 regEntry->release();
3431 }
3432
3433 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
3434 if (gIOOptionsEntry && sym) gIOOptionsEntry->setProperty(sym, noteProp);
3435 if (noteProp) noteProp->release();
3436 if (sym) sym->release();
3437}
3438
3439
3440
mirror of XNU