]> git.saurik.com Git - apple/xnu.git/blame - iokit/Kernel/IOHibernateIO.cpp
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-2422.1.72.tar.gz
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
CommitLineData
3a60a9f5 1/*
b0d623f7 2 * Copyright (c) 2004-2008 Apple Computer, Inc. All rights reserved.
3a60a9f5 3 *
2d21ac55 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
3a60a9f5 5 *
2d21ac55
A		 6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
8f6c56a5 14 *
2d21ac55
A		 15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5
A		 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55
A		 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
8f6c56a5 25 *
2d21ac55 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
3a60a9f5
A		 27 */
28
29
30/*
31
32Sleep:
33
34- PMRootDomain calls IOHibernateSystemSleep() before system sleep
35(devices awake, normal execution context)
36- IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
37 grabs its extents and searches for a polling driver willing to work with that IOMedia.
38 The BSD code makes an ioctl to the storage driver to get the partition base offset to
39 the disk, and other ioctls to get the transfer constraints
40 If successful, the file is written to make sure its initially not bootable (in case of
41 later failure) and nvram set to point to the first block of the file. (Has to be done
42 here so blocking is possible in nvram support).
43 hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
44 page out any pages it wants to (currently zero, but probably some percentage of memory).
45 Its assumed just allocating pages will cause the VM system to naturally select the best
46 pages for eviction. It also copies processor flags needed for the restore path and sets
47 a flag in the boot processor proc info.
48 gIOHibernateState = kIOHibernateStateHibernating.
49- Regular sleep progresses - some drivers may inspect the root domain property
50 kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
51 as usual but leaves motherboard I/O on.
52- Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
53 at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
54 all ppc RC bits out of the hash table and caches into the mapping structures.
55- hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
56 hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
57 All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
58 by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
59 The image header and block list are written. The header includes the second file extent so
60 only the header block is needed to read the file, regardless of filesystem.
316670eb 61 The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data
3a60a9f5
A		 62 (only) is used to decompress the image during wake/boot.
63 Some additional pages are removed from the bitmaps - the buffers used for hibernation.
64 The bitmaps are written to the image.
65 More areas are removed from the bitmaps (after they have been written to the image) - the
316670eb 66 segment "__HIB" pages and interrupt stack.
3a60a9f5
A		 67 Each wired page is compressed and written and then each non-wired page. Compression and
68 disk writes are in parallel.
69 The image header is written to the start of the file and the polling driver closed.
70 The machine powers down (or sleeps).
71
72Boot/Wake:
73
74- BootX sees the boot-image nvram variable containing the device and block number of the image,
75 reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
76- BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
77 in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
78 that is in the kernel's __HIB section.
79- BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
80 hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
81 only code & data in that section is safe to call since all the other wired pages are still
82 compressed in the image.
83- hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
84 It uses the bitmaps to work out which pages can be uncompressed from the image to their final
85 location directly, and copies those that can't to interim free pages. When the image has been
86 completed, the copies are uncompressed, overwriting the wired image pages.
87 hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
88 is used to get pages into place for 64bit.
89- the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
90 changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
91 are removed from the software strutures, and the hash table is reinitialized.
92- After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
93 of memory, using the polled mode driver, before other threads can run or any devices are turned on.
94 This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
95 for the remaining non wired pages.
96- The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
97 (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
98 is closed via bsd.
99
100Polled Mode I/O:
101
102IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
103registry, specifying an object of calls IOPolledInterface.
104
105Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
106partition) that the image is going to live, looking for polled interface properties. If it finds
107one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
108interfaces found are kept in an ordered list.
109
110There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
111few different contexts things happen in:
112
113- there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
114up and running) and after wake - this is safe to allocate memory and do anything. The device
115ignores sleep requests from that point since its a waste of time if it goes to sleep and
116immediately wakes back up for the image write.
117
118- there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
119immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
120from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
121used to flush and set the disk to sleep.
122
123- there is an Open/Close (AfterSleep) pair made around the image read operations that happen
124immediately after sleep. These can't block or allocate memory. This is happening after the platform
125expert has woken the low level bits of the system, but most of the I/O system has not. There is only
126one thread running.
127
128For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
129(prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
130the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
131Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
132that is called for the hardware to check for events, and complete the I/O via the callback.
133The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
134to restrict I/O ops.
135*/
136
137#include <sys/systm.h>
138
139#include <IOKit/IOWorkLoop.h>
140#include <IOKit/IOCommandGate.h>
141#include <IOKit/IOTimerEventSource.h>
142#include <IOKit/IOPlatformExpert.h>
143#include <IOKit/IOKitDebug.h>
144#include <IOKit/IOTimeStamp.h>
145#include <IOKit/pwr_mgt/RootDomain.h>
146#include <IOKit/pwr_mgt/IOPMPrivate.h>
147#include <IOKit/IOMessage.h>
148#include <IOKit/IODeviceTreeSupport.h>
149#include <IOKit/IOBSD.h>
150#include "RootDomainUserClient.h"
151#include <IOKit/pwr_mgt/IOPowerConnection.h>
152#include "IOPMPowerStateQueue.h"
153#include <IOKit/IOBufferMemoryDescriptor.h>
6d2010ae 154#include <IOKit/AppleKeyStoreInterface.h>
316670eb 155#include <libkern/crypto/aes.h>
3a60a9f5
A		 156
157#include <sys/uio.h>
158#include <sys/conf.h>
159#include <sys/stat.h>
160#include <sys/fcntl.h> // (FWRITE, ...)
3a60a9f5 161#include <sys/sysctl.h>
6d2010ae 162#include <sys/kdebug.h>
3a60a9f5
A		 163
164#include <IOKit/IOHibernatePrivate.h>
165#include <IOKit/IOPolledInterface.h>
166#include <IOKit/IONVRAM.h>
167#include "IOHibernateInternal.h"
39236c6e 168#include <vm/WKdm_new.h>
3a60a9f5 169#include "IOKitKernelInternal.h"
6d2010ae
A		 170#include <pexpert/device_tree.h>
171
172#include <machine/pal_routines.h>
173#include <machine/pal_hibernate.h>
39236c6e 174#include <i386/tsc.h>
6d2010ae
A		 175
176extern "C" addr64_t kvtophys(vm_offset_t va);
99c3a104 177extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
3a60a9f5
A		 178
179/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
180
39236c6e
A		 181#define DISABLE_TRIM 0
182#define TRIM_DELAY 5000
183
0b4c1975 184extern unsigned int save_kdebug_enable;
3a60a9f5
A		 185extern uint32_t gIOHibernateState;
186uint32_t gIOHibernateMode;
187static char gIOHibernateBootSignature[256+1];
188static char gIOHibernateFilename[MAXPATHLEN+1];
db609669
A		 189static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
190uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms)
191static uint64_t gIOHibernateCompression = 0x80; // default compression 50%
3a60a9f5
A		 192
193static IODTNVRAM * gIOOptionsEntry;
194static IORegistryEntry * gIOChosenEntry;
b0d623f7 195#if defined(__i386__) || defined(__x86_64__)
0c530ab8 196static const OSSymbol * gIOCreateEFIDevicePathSymbol;
060df5ea
A		 197static const OSSymbol * gIOHibernateRTCVariablesKey;
198static const OSSymbol * gIOHibernateBoot0082Key;
199static const OSSymbol * gIOHibernateBootNextKey;
200static OSData * gIOHibernateBoot0082Data;
201static OSData * gIOHibernateBootNextData;
202static OSObject * gIOHibernateBootNextSave;
0c530ab8 203#endif
3a60a9f5 204
316670eb
A		 205static IOLock * gFSLock;
206static uint32_t gFSState;
3a60a9f5
A		 207static IOPolledFileIOVars gFileVars;
208static IOHibernateVars gIOHibernateVars;
209static struct kern_direct_file_io_ref_t * gIOHibernateFileRef;
210static hibernate_cryptvars_t gIOHibernateCryptWakeContext;
6d2010ae
A		 211static hibernate_graphics_t _hibernateGraphics;
212static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
39236c6e
A		 213static hibernate_statistics_t _hibernateStats;
214static hibernate_statistics_t * gIOHibernateStats = &_hibernateStats;
3a60a9f5 215
316670eb
A		 216enum
217{
218 kFSIdle = 0,
219 kFSOpening = 2,
220 kFSOpened = 3,
221 kFSTimedOut = 4,
222};
223
224static IOReturn IOHibernateDone(IOHibernateVars * vars);
225
3a60a9f5
A		 226/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
227
228enum { kXPRamAudioVolume = 8 };
229enum { kDefaultIOSize = 128 * 1024 };
230enum { kVideoMapSize = 32 * 1024 * 1024 };
231
232#ifndef kIOMediaPreferredBlockSizeKey
233#define kIOMediaPreferredBlockSizeKey "Preferred Block Size"
234#endif
235
236#ifndef kIOBootPathKey
237#define kIOBootPathKey "bootpath"
238#endif
239#ifndef kIOSelectedBootDeviceKey
240#define kIOSelectedBootDeviceKey "boot-device"
241#endif
242
243
244enum { kIOHibernateMinPollersNeeded = 2 };
245
246/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
247
248// copy from phys addr to MD
249
250static IOReturn
251IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
252 IOByteCount offset, addr64_t bytes, IOByteCount length)
253{
254 addr64_t srcAddr = bytes;
255 IOByteCount remaining;
256
257 remaining = length = min(length, md->getLength() - offset);
258 while (remaining) { // (process another target segment?)
259 addr64_t dstAddr64;
260 IOByteCount dstLen;
261
b0d623f7 262 dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
3a60a9f5
A		 263 if (!dstAddr64)
264 break;
265
266 // Clip segment length to remaining
267 if (dstLen > remaining)
268 dstLen = remaining;
269
270#if 1
271 bcopy_phys(srcAddr, dstAddr64, dstLen);
272#else
273 copypv(srcAddr, dstAddr64, dstLen,
274 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
275#endif
276 srcAddr += dstLen;
277 offset += dstLen;
278 remaining -= dstLen;
279 }
280
281 assert(!remaining);
282
283 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
284}
285
286// copy from MD to phys addr
287
288static IOReturn
289IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
290 IOByteCount offset, addr64_t bytes, IOByteCount length)
291{
292 addr64_t dstAddr = bytes;
293 IOByteCount remaining;
294
295 remaining = length = min(length, md->getLength() - offset);
296 while (remaining) { // (process another target segment?)
297 addr64_t srcAddr64;
298 IOByteCount dstLen;
299
b0d623f7 300 srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
3a60a9f5
A		 301 if (!srcAddr64)
302 break;
303
304 // Clip segment length to remaining
305 if (dstLen > remaining)
306 dstLen = remaining;
307
308#if 1
309 bcopy_phys(srcAddr64, dstAddr, dstLen);
310#else
311 copypv(srcAddr, dstAddr64, dstLen,
312 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
313#endif
314 dstAddr += dstLen;
315 offset += dstLen;
316 remaining -= dstLen;
317 }
318
319 assert(!remaining);
320
321 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
322}
323
324/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
325
326void
327hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
328 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
329{
330 count += ppnum;
331 switch (kind)
332 {
333 case kIOHibernatePageStateUnwiredSave:
334 // unwired save
335 for (; ppnum < count; ppnum++)
336 {
337 hibernate_page_bitset(page_list, FALSE, ppnum);
338 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
339 }
340 break;
341 case kIOHibernatePageStateWiredSave:
342 // wired save
343 for (; ppnum < count; ppnum++)
344 {
345 hibernate_page_bitset(page_list, FALSE, ppnum);
346 hibernate_page_bitset(page_list_wired, FALSE, ppnum);
347 }
348 break;
349 case kIOHibernatePageStateFree:
350 // free page
351 for (; ppnum < count; ppnum++)
352 {
353 hibernate_page_bitset(page_list, TRUE, ppnum);
354 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
355 }
356 break;
357 default:
358 panic("hibernate_set_page_state");
359 }
360}
361
362static vm_offset_t
0c530ab8 363hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
3a60a9f5 364{
0c530ab8
A		 365 uint32_t page = *pPage;
366 uint32_t count;
367 hibernate_bitmap_t * bitmap;
21362eb3 368
0c530ab8
A		 369 while ((bitmap = hibernate_page_bitmap_pin(list, &page)))
370 {
371 count = hibernate_page_bitmap_count(bitmap, TRUE, page);
372 if (!count)
373 break;
374 page += count;
375 if (page <= bitmap->last_page)
376 break;
377 }
6601e61a 378
0c530ab8
A		 379 *pPage = page;
380 if (bitmap)
381 count = hibernate_page_bitmap_count(bitmap, FALSE, page);
382 else
383 count = 0;
3a60a9f5
A		 384
385 return (count);
386}
387
388/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
389
390static IOReturn
391IOHibernatePollerProbe(IOPolledFileIOVars * vars, IOService * target)
392{
393 IOReturn err = kIOReturnError;
394 int32_t idx;
395 IOPolledInterface * poller;
396
397 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
398 {
399 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
400 err = poller->probe(target);
401 if (err)
402 {
403 HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
404 break;
405 }
406 }
407
408 return (err);
409}
410
411static IOReturn
412IOHibernatePollerOpen(IOPolledFileIOVars * vars, uint32_t state, IOMemoryDescriptor * md)
413{
414 IOReturn err = kIOReturnError;
415 int32_t idx;
416 IOPolledInterface * poller;
417
418 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
419 {
420 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
421 err = poller->open(state, md);
422 if (err)
423 {
424 HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
425 break;
426 }
427 }
428
429 return (err);
430}
431
432static IOReturn
433IOHibernatePollerClose(IOPolledFileIOVars * vars, uint32_t state)
434{
435 IOReturn err = kIOReturnError;
436 int32_t idx;
437 IOPolledInterface * poller;
438
439 for (idx = 0;
440 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
441 idx++)
442 {
443 err = poller->close(state);
444 if (err)
445 HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
446 }
447
448 return (err);
449}
450
451static void
452IOHibernatePollerIOComplete(void * target,
453 void * parameter,
454 IOReturn status,
455 UInt64 actualByteCount)
456{
457 IOPolledFileIOVars * vars = (IOPolledFileIOVars *) parameter;
458
459 vars->ioStatus = status;
460}
461
462static IOReturn
463IOHibernatePollerIO(IOPolledFileIOVars * vars,
464 uint32_t operation, uint32_t bufferOffset,
465 uint64_t deviceOffset, uint64_t length)
466{
467
468 IOReturn err = kIOReturnError;
469 IOPolledInterface * poller;
470 IOPolledCompletion completion;
471
472 completion.target = 0;
473 completion.action = &IOHibernatePollerIOComplete;
474 completion.parameter = vars;
475
476 vars->ioStatus = -1;
477
478 poller = (IOPolledInterface *) vars->pollers->getObject(0);
479 err = poller->startIO(operation, bufferOffset, deviceOffset + vars->block0, length, completion);
480 if (err)
481 HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
482
483 return (err);
484}
485
486static IOReturn
2d21ac55 487IOHibernatePollerIODone(IOPolledFileIOVars * vars, bool abortable)
3a60a9f5 488{
2d21ac55
A		 489 IOReturn err = kIOReturnSuccess;
490 int32_t idx = 0;
3a60a9f5
A		 491 IOPolledInterface * poller;
492
493 while (-1 == vars->ioStatus)
494 {
2d21ac55
A		 495 for (idx = 0;
496 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
3a60a9f5
A		 497 idx++)
498 {
2d21ac55
A		 499 IOReturn newErr;
500 newErr = poller->checkForWork();
501 if ((newErr == kIOReturnAborted) && !abortable)
502 newErr = kIOReturnSuccess;
503 if (kIOReturnSuccess == err)
504 err = newErr;
3a60a9f5
A		 505 }
506 }
507
0b4c1975
A		 508 if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort())
509 {
510 err = kIOReturnAborted;
511 HIBLOG("IOPolledInterface::checkForWork sw abort\n");
512 }
513
2d21ac55
A		 514 if (err)
515 {
516 HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
517 }
518 else
519 {
520 err = vars->ioStatus;
521 if (kIOReturnSuccess != err)
522 HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
523 }
3a60a9f5 524
2d21ac55 525 return (err);
3a60a9f5
A		 526}
527
528IOReturn
529IOPolledInterface::checkAllForWork(void)
530{
531 IOReturn err = kIOReturnNotReady;
532 int32_t idx;
533 IOPolledInterface * poller;
534
535 IOHibernateVars * vars = &gIOHibernateVars;
536
537 if (!vars->fileVars || !vars->fileVars->pollers)
538 return (err);
539
540 for (idx = 0;
541 (poller = (IOPolledInterface *) vars->fileVars->pollers->getObject(idx));
542 idx++)
543 {
544 err = poller->checkForWork();
545 if (err)
546 HIBLOG("IOPolledInterface::checkAllForWork[%d] 0x%x\n", idx, err);
547 }
548
549 return (err);
550}
551
552/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
553
554struct _OpenFileContext
555{
556 OSData * extents;
557 uint64_t size;
558};
559
560static void
561file_extent_callback(void * ref, uint64_t start, uint64_t length)
562{
563 _OpenFileContext * ctx = (_OpenFileContext *) ref;
564 IOPolledFileExtent extent;
565
566 extent.start = start;
567 extent.length = length;
568
39236c6e
A		 569 HIBLOG("[0x%qx, 0x%qx]\n", start, length);
570
3a60a9f5
A		 571 ctx->extents->appendBytes(&extent, sizeof(extent));
572 ctx->size += length;
573}
574
6d2010ae
A		 575static IOService *
576IOCopyMediaForDev(dev_t device)
577{
578 OSDictionary * matching;
579 OSNumber * num;
580 OSIterator * iter;
581 IOService * result = 0;
582
583 matching = IOService::serviceMatching("IOMedia");
584 if (!matching)
585 return (0);
586 do
587 {
588 num = OSNumber::withNumber(major(device), 32);
589 if (!num)
590 break;
591 matching->setObject(kIOBSDMajorKey, num);
592 num->release();
593 num = OSNumber::withNumber(minor(device), 32);
594 if (!num)
595 break;
596 matching->setObject(kIOBSDMinorKey, num);
597 num->release();
598 if (!num)
599 break;
600 iter = IOService::getMatchingServices(matching);
601 if (iter)
602 {
603 result = (IOService *) iter->getNextObject();
604 result->retain();
605 iter->release();
606 }
607 }
608 while (false);
609 matching->release();
610
611 return (result);
612}
613
3a60a9f5 614IOReturn
db609669
A		 615IOPolledFileOpen( const char * filename, uint64_t setFileSize,
616 IOBufferMemoryDescriptor * ioBuffer,
617 IOPolledFileIOVars ** fileVars, OSData ** fileExtents,
618 OSData ** imagePath, uint8_t * volumeCryptKey)
3a60a9f5 619{
316670eb 620 IOReturn err = kIOReturnSuccess;
3a60a9f5
A		 621 IOPolledFileIOVars * vars;
622 _OpenFileContext ctx;
623 OSData * extentsData;
624 OSNumber * num;
6d2010ae
A		 625 IOService * part = 0;
626 OSString * keyUUID = 0;
627 OSString * keyStoreUUID = 0;
628 dev_t block_dev;
3a60a9f5
A		 629 dev_t hibernate_image_dev;
630 uint64_t maxiobytes;
316670eb
A		 631 AbsoluteTime startTime, endTime;
632 uint64_t nsec;
633
634 vars = IONew(IOPolledFileIOVars, 1);
635 if (!vars) return (kIOReturnNoMemory);
636 bzero(vars, sizeof(*vars));
3a60a9f5 637
3a60a9f5
A		 638 do
639 {
3a60a9f5
A		 640 vars->io = false;
641 vars->buffer = (uint8_t *) ioBuffer->getBytesNoCopy();
642 vars->bufferHalf = 0;
643 vars->bufferOffset = 0;
644 vars->bufferSize = ioBuffer->getLength() >> 1;
645
646 extentsData = OSData::withCapacity(32);
316670eb 647 ctx.extents = extentsData;
3a60a9f5 648 ctx.size = 0;
316670eb 649 clock_get_uptime(&startTime);
3a60a9f5
A		 650 vars->fileRef = kern_open_file_for_direct_io(filename,
651 &file_extent_callback, &ctx,
db609669
A		 652 setFileSize,
653 // write file:
654 0, (caddr_t) gIOHibernateCurrentHeader,
655 sizeof(IOHibernateImageHeader),
656 // results
6d2010ae 657 &block_dev,
3a60a9f5
A		 658 &hibernate_image_dev,
659 &vars->block0,
0b4c1975 660 &maxiobytes,
db609669 661 &vars->flags);
316670eb
A		 662#if 0
663 uint32_t msDelay = (131071 & random());
664 HIBLOG("sleep %d\n", msDelay);
665 IOSleep(msDelay);
666#endif
667 clock_get_uptime(&endTime);
668 SUB_ABSOLUTETIME(&endTime, &startTime);
669 absolutetime_to_nanoseconds(endTime, &nsec);
670
671 if (!vars->fileRef) err = kIOReturnNoSpace;
672
673 IOLockLock(gFSLock);
674 if (kFSOpening != gFSState) err = kIOReturnTimeout;
675 IOLockUnlock(gFSLock);
676
677 HIBLOG("kern_open_file_for_direct_io(%d) took %qd ms\n", err, nsec / 1000000ULL);
678 if (kIOReturnSuccess != err) break;
0b4c1975
A		 679
680 if (kIOHibernateModeSSDInvert & gIOHibernateMode)
6d2010ae 681 vars->flags ^= kIOHibernateOptionSSD;
0b4c1975
A		 682
683 HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx ssd %d\n", filename, ctx.size,
6d2010ae 684 vars->block0, maxiobytes, kIOHibernateOptionSSD & vars->flags);
3a60a9f5
A		 685 if (ctx.size < 1*1024*1024) // check against image size estimate!
686 {
687 err = kIOReturnNoSpace;
688 break;
689 }
690
db609669 691 vars->fileSize = ctx.size;
39236c6e 692 if (maxiobytes < vars->bufferSize) vars->bufferSize = maxiobytes;
3a60a9f5
A		 693
694 vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
2d21ac55 695
6d2010ae
A		 696 part = IOCopyMediaForDev(block_dev);
697 if (!part)
39236c6e
A		 698 {
699 err = kIOReturnNotFound;
6d2010ae 700 break;
39236c6e 701 }
6d2010ae
A		 702 err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false,
703 (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL);
704 if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID)
705 {
706// IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy());
707 uuid_t volumeKeyUUID;
708 aks_volume_key_t vek;
709 static IOService * sKeyStore;
710 static const OSSymbol * sAKSGetKey;
711
712 if (!sAKSGetKey)
713 sAKSGetKey = OSSymbol::withCStringNoCopy(AKS_PLATFORM_FUNCTION_GETKEY);
714 if (!sKeyStore)
715 sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane);
716 if (sKeyStore)
717 err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID);
718 else
719 err = kIOReturnNoResources;
720 if (kIOReturnSuccess == err)
721 err = sKeyStore->callPlatformFunction(sAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL);
722 if (kIOReturnSuccess != err)
723 IOLog("volume key err 0x%x\n", err);
724 else
725 {
726 size_t bytes = (kIOHibernateAESKeySize / 8);
727 if (vek.key.keybytecount < bytes)
728 bytes = vek.key.keybytecount;
729 bcopy(&vek.key.keybytes[0], volumeCryptKey, bytes);
730 }
731 bzero(&vek, sizeof(vek));
732 }
733 part->release();
734
735 part = IOCopyMediaForDev(hibernate_image_dev);
736 if (!part)
39236c6e
A		 737 {
738 err = kIOReturnNotFound;
6d2010ae 739 break;
39236c6e 740 }
6d2010ae 741
3a60a9f5
A		 742 IORegistryEntry * next;
743 IORegistryEntry * child;
744 OSData * data;
745
3a60a9f5
A		 746 vars->pollers = OSArray::withCapacity(4);
747 if (!vars->pollers)
39236c6e
A		 748 {
749 err = kIOReturnNoMemory;
750 break;
751 }
3a60a9f5
A		 752
753 vars->blockSize = 512;
754 next = part;
755 do
756 {
757 IOPolledInterface * poller;
c0fea474
A		 758 OSObject * obj;
759
760 obj = next->getProperty(kIOPolledInterfaceSupportKey);
761 if (kOSBooleanFalse == obj)
762 {
763 vars->pollers->flushCollection();
764 break;
765 }
766 else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
3a60a9f5
A		 767 vars->pollers->setObject(poller);
768 if ((num = OSDynamicCast(OSNumber, next->getProperty(kIOMediaPreferredBlockSizeKey))))
769 vars->blockSize = num->unsigned32BitValue();
770 child = next;
771 }
772 while ((next = child->getParentEntry(gIOServicePlane))
773 && child->isParent(next, gIOServicePlane, true));
774
39236c6e
A		 775 if (vars->blockSize < 4096) vars->blockSize = 4096;
776
3a60a9f5 777 HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n",
6d2010ae 778 major(hibernate_image_dev), minor(hibernate_image_dev), (long)vars->blockSize, vars->pollers->getCount());
39236c6e 779
3a60a9f5 780 if (vars->pollers->getCount() < kIOHibernateMinPollersNeeded)
39236c6e
A		 781 {
782 err = kIOReturnUnsupported;
3a60a9f5 783 continue;
39236c6e
A		 784 }
785 if (vars->blockSize < sizeof(IOHibernateImageHeader))
786 {
787 err = kIOReturnError;
788 continue;
789 }
3a60a9f5
A		 790
791 err = IOHibernatePollerProbe(vars, (IOService *) part);
39236c6e 792 if (kIOReturnSuccess != err) break;
3a60a9f5
A		 793
794 err = IOHibernatePollerOpen(vars, kIOPolledPreflightState, ioBuffer);
39236c6e 795 if (kIOReturnSuccess != err) break;
3a60a9f5 796
bd504ef0
A		 797 vars->media = part;
798 next = part;
799 while (next)
800 {
801 next->setProperty(kIOPolledInterfaceActiveKey, kOSBooleanTrue);
802 next = next->getParentEntry(gIOServicePlane);
803 }
804
3a60a9f5
A		 805 *fileVars = vars;
806 *fileExtents = extentsData;
807
808 // make imagePath
3a60a9f5 809
0c530ab8 810 if ((extentsData->getLength() >= sizeof(IOPolledFileExtent)))
3a60a9f5 811 {
6d2010ae 812 char str2[24 + sizeof(uuid_string_t) + 2];
0c530ab8 813
b0d623f7 814#if defined(__i386__) || defined(__x86_64__)
0c530ab8
A		 815 if (!gIOCreateEFIDevicePathSymbol)
816 gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
817
6d2010ae
A		 818 if (keyUUID)
819 snprintf(str2, sizeof(str2), "%qx:%s",
820 vars->extentMap[0].start, keyUUID->getCStringNoCopy());
821 else
822 snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
0c530ab8
A		 823
824 err = IOService::getPlatform()->callPlatformFunction(
825 gIOCreateEFIDevicePathSymbol, false,
6d2010ae
A		 826 (void *) part, (void *) str2,
827 (void *) (uintptr_t) true, (void *) &data);
0c530ab8
A		 828#else
829 char str1[256];
830 int len = sizeof(str1);
831
832 if (!part->getPath(str1, &len, gIODTPlane))
833 err = kIOReturnNotFound;
834 else
835 {
2d21ac55 836 snprintf(str2, sizeof(str2), ",%qx", vars->extentMap[0].start);
0c530ab8
A		 837 // (strip the plane name)
838 char * tail = strchr(str1, ':');
839 if (!tail)
840 tail = str1 - 1;
841 data = OSData::withBytes(tail + 1, strlen(tail + 1));
842 data->appendBytes(str2, strlen(str2));
843 }
844#endif
845 if (kIOReturnSuccess == err)
3a60a9f5 846 *imagePath = data;
0c530ab8
A		 847 else
848 HIBLOG("error 0x%x getting path\n", err);
3a60a9f5
A		 849 }
850 }
851 while (false);
852
853 if (kIOReturnSuccess != err)
854 {
855 HIBLOG("error 0x%x opening hibernation file\n", err);
856 if (vars->fileRef)
b0d623f7 857 {
7ddcb079 858 kern_close_file_for_direct_io(vars->fileRef, 0, 0, 0, 0, 0);
316670eb 859 vars->fileRef = NULL;
b0d623f7 860 }
3a60a9f5
A		 861 }
862
863 if (part)
864 part->release();
865
866 return (err);
867}
868
869IOReturn
870IOPolledFileClose( IOPolledFileIOVars * vars )
871{
872 if (vars->pollers)
873 {
874 IOHibernatePollerClose(vars, kIOPolledPostflightState);
875 vars->pollers->release();
876 }
877
3a60a9f5
A		 878 bzero(vars, sizeof(IOPolledFileIOVars));
879
880 return (kIOReturnSuccess);
881}
882
883static IOReturn
884IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
885{
886 IOPolledFileExtent * extentMap;
887
888 extentMap = vars->extentMap;
889
890 vars->position = position;
891
892 while (position >= extentMap->length)
893 {
894 position -= extentMap->length;
895 extentMap++;
896 }
897
898 vars->currentExtent = extentMap;
899 vars->extentRemaining = extentMap->length - position;
900 vars->extentPosition = vars->position - position;
901
902 if (vars->bufferSize <= vars->extentRemaining)
903 vars->bufferLimit = vars->bufferSize;
904 else
905 vars->bufferLimit = vars->extentRemaining;
906
907 return (kIOReturnSuccess);
908}
909
910static IOReturn
911IOPolledFileWrite(IOPolledFileIOVars * vars,
912 const uint8_t * bytes, IOByteCount size,
913 hibernate_cryptvars_t * cryptvars)
914{
915 IOReturn err = kIOReturnSuccess;
916 IOByteCount copy;
917 bool flush = false;
918
919 do
920 {
921 if (!bytes && !size)
922 {
923 // seek to end of block & flush
924 size = vars->position & (vars->blockSize - 1);
925 if (size)
926 size = vars->blockSize - size;
927 flush = true;
928 // use some garbage for the fill
929 bytes = vars->buffer + vars->bufferOffset;
930 }
931
932 copy = vars->bufferLimit - vars->bufferOffset;
933 if (copy > size)
934 copy = size;
935 else
936 flush = true;
937
938 if (bytes)
939 {
940 bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
941 bytes += copy;
942 }
943 else
944 bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
945
946 size -= copy;
947 vars->bufferOffset += copy;
948 vars->position += copy;
949
950 if (flush && vars->bufferOffset)
951 {
952 uint64_t offset = (vars->position - vars->bufferOffset
953 - vars->extentPosition + vars->currentExtent->start);
954 uint32_t length = (vars->bufferOffset);
955
2d21ac55 956#if CRYPTO
0b4c1975
A		 957 if (cryptvars && vars->encryptStart
958 && (vars->position > vars->encryptStart)
959 && ((vars->position - length) < vars->encryptEnd))
3a60a9f5 960 {
6d2010ae
A		 961 AbsoluteTime startTime, endTime;
962
7ddcb079 963 uint64_t encryptLen, encryptStart;
3a60a9f5
A		 964 encryptLen = vars->position - vars->encryptStart;
965 if (encryptLen > length)
966 encryptLen = length;
967 encryptStart = length - encryptLen;
0b4c1975
A		 968 if (vars->position > vars->encryptEnd)
969 encryptLen -= (vars->position - vars->encryptEnd);
970
6d2010ae
A		 971 clock_get_uptime(&startTime);
972
3a60a9f5
A		 973 // encrypt the buffer
974 aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
975 &cryptvars->aes_iv[0],
976 encryptLen / AES_BLOCK_SIZE,
977 vars->buffer + vars->bufferHalf + encryptStart,
978 &cryptvars->ctx.encrypt);
6d2010ae
A		 979
980 clock_get_uptime(&endTime);
981 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
982 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
983 vars->cryptBytes += encryptLen;
984
3a60a9f5
A		 985 // save initial vector for following encrypts
986 bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
987 &cryptvars->aes_iv[0],
988 AES_BLOCK_SIZE);
989 }
2d21ac55 990#endif /* CRYPTO */
3a60a9f5
A		 991
992 if (vars->io)
993 {
2d21ac55 994 err = IOHibernatePollerIODone(vars, true);
3a60a9f5
A		 995 if (kIOReturnSuccess != err)
996 break;
997 }
998
999if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
1000//if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
1001
1002 err = IOHibernatePollerIO(vars, kIOPolledWrite, vars->bufferHalf, offset, length);
1003 if (kIOReturnSuccess != err)
1004 break;
1005 vars->io = true;
1006
1007 vars->extentRemaining -= vars->bufferOffset;
1008 if (!vars->extentRemaining)
1009 {
1010 vars->currentExtent++;
1011 vars->extentRemaining = vars->currentExtent->length;
1012 vars->extentPosition = vars->position;
1013 if (!vars->extentRemaining)
1014 {
1015 err = kIOReturnOverrun;
1016 break;
1017 }
1018 }
1019
1020 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
1021 vars->bufferOffset = 0;
1022 if (vars->bufferSize <= vars->extentRemaining)
1023 vars->bufferLimit = vars->bufferSize;
1024 else
1025 vars->bufferLimit = vars->extentRemaining;
1026
1027 flush = false;
1028 }
1029 }
1030 while (size);
1031
1032 return (err);
1033}
1034
1035static IOReturn
1036IOPolledFileRead(IOPolledFileIOVars * vars,
1037 uint8_t * bytes, IOByteCount size,
1038 hibernate_cryptvars_t * cryptvars)
1039{
1040 IOReturn err = kIOReturnSuccess;
1041 IOByteCount copy;
1042
1043// bytesWritten += size;
1044
1045 do
1046 {
1047 copy = vars->bufferLimit - vars->bufferOffset;
1048 if (copy > size)
1049 copy = size;
1050
1051 if (bytes)
1052 {
1053 bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
1054 bytes += copy;
1055 }
1056 size -= copy;
1057 vars->bufferOffset += copy;
1058// vars->position += copy;
1059
6d2010ae 1060 if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd))
3a60a9f5
A		 1061 {
1062 if (vars->io)
1063 {
2d21ac55 1064 err = IOHibernatePollerIODone(vars, false);
3a60a9f5
A		 1065 if (kIOReturnSuccess != err)
1066 break;
1067 }
1068 else
1069 cryptvars = 0;
1070
1071if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
1072
6d2010ae 1073 vars->position += vars->lastRead;
3a60a9f5 1074 vars->extentRemaining -= vars->lastRead;
6d2010ae 1075 vars->bufferLimit = vars->lastRead;
3a60a9f5
A		 1076
1077 if (!vars->extentRemaining)
1078 {
1079 vars->currentExtent++;
1080 vars->extentRemaining = vars->currentExtent->length;
1081 vars->extentPosition = vars->position;
1082 if (!vars->extentRemaining)
1083 {
1084 err = kIOReturnOverrun;
1085 break;
1086 }
1087 }
1088
36401178
A		 1089 uint64_t length;
1090 uint64_t lastReadLength = vars->lastRead;
3a60a9f5
A		 1091 uint64_t offset = (vars->position
1092 - vars->extentPosition + vars->currentExtent->start);
36401178
A		 1093 if (vars->extentRemaining <= vars->bufferSize)
1094 length = vars->extentRemaining;
1095 else
1096 length = vars->bufferSize;
6d2010ae
A		 1097 if ((length + vars->position) > vars->readEnd)
1098 length = vars->readEnd - vars->position;
3a60a9f5 1099
6d2010ae
A		 1100 vars->lastRead = length;
1101 if (length)
1102 {
3a60a9f5 1103//if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
6d2010ae
A		 1104 err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length);
1105 if (kIOReturnSuccess != err)
1106 break;
1107 vars->io = true;
1108 }
3a60a9f5
A		 1109
1110 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
1111 vars->bufferOffset = 0;
1112
2d21ac55 1113#if CRYPTO
3a60a9f5
A		 1114 if (cryptvars)
1115 {
1116 uint8_t thisVector[AES_BLOCK_SIZE];
6d2010ae
A		 1117 AbsoluteTime startTime, endTime;
1118
3a60a9f5
A		 1119 // save initial vector for following decrypts
1120 bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
36401178 1121 bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE,
3a60a9f5 1122 &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
6d2010ae 1123
3a60a9f5 1124 // decrypt the buffer
6d2010ae
A		 1125 clock_get_uptime(&startTime);
1126
3a60a9f5
A		 1127 aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
1128 &thisVector[0],
36401178 1129 lastReadLength / AES_BLOCK_SIZE,
3a60a9f5
A		 1130 vars->buffer + vars->bufferHalf,
1131 &cryptvars->ctx.decrypt);
6d2010ae
A		 1132
1133 clock_get_uptime(&endTime);
1134 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
1135 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
1136 vars->cryptBytes += lastReadLength;
3a60a9f5 1137 }
b0d623f7 1138#endif /* CRYPTO */
3a60a9f5
A		 1139 }
1140 }
1141 while (size);
1142
1143 return (err);
1144}
1145
1146/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
99c3a104 1147
3a60a9f5
A		 1148IOReturn
1149IOHibernateSystemSleep(void)
1150{
1151 IOReturn err;
1152 OSData * data;
1153 OSObject * obj;
1154 OSString * str;
db609669 1155 OSNumber * num;
39236c6e 1156 bool dsSSD, vmflush;
316670eb 1157 IOHibernateVars * vars;
3a60a9f5
A		 1158
1159 gIOHibernateState = kIOHibernateStateInactive;
1160
316670eb
A		 1161 if (!gIOChosenEntry)
1162 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1163
b0d623f7
A		 1164 gIOHibernateDebugFlags = 0;
1165 if (kIOLogHibernate & gIOKitDebug)
1166 gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
1167
0b4c1975
A		 1168 if (IOService::getPMRootDomain()->getHibernateSettings(
1169 &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime))
6d2010ae 1170 {
3a60a9f5
A		 1171 if (kIOHibernateModeSleep & gIOHibernateMode)
1172 // default to discard clean for safe sleep
1173 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
1174 | kIOHibernateModeDiscardCleanActive);
6d2010ae 1175 }
3a60a9f5 1176
3a60a9f5
A		 1177 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
1178 {
1179 if ((str = OSDynamicCast(OSString, obj)))
2d21ac55
A		 1180 strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
1181 sizeof(gIOHibernateFilename));
3a60a9f5
A		 1182 obj->release();
1183 }
1184
1185 if (!gIOHibernateMode || !gIOHibernateFilename[0])
1186 return (kIOReturnUnsupported);
1187
1188 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
1189
316670eb
A		 1190 vars = IONew(IOHibernateVars, 1);
1191 if (!vars) return (kIOReturnNoMemory);
1192 bzero(vars, sizeof(*vars));
1193
1194 IOLockLock(gFSLock);
1195 if (kFSIdle != gFSState)
1196 {
1197 HIBLOG("hibernate file busy\n");
1198 IOLockUnlock(gFSLock);
1199 IODelete(vars, IOHibernateVars, 1);
1200 return (kIOReturnBusy);
1201 }
1202 gFSState = kFSOpening;
1203 IOLockUnlock(gFSLock);
2d21ac55 1204
3a60a9f5
A		 1205 do
1206 {
0c530ab8 1207 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
39236c6e 1208 2 * page_size + WKdm_SCRATCH_BUF_SIZE, page_size);
0c530ab8
A		 1209 vars->ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1210 2 * kDefaultIOSize, page_size);
3a60a9f5 1211
6d2010ae
A		 1212 vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1213 ptoa_64(gIOHibernateHandoffPageCount), page_size);
1214
1215 if (!vars->srcBuffer || !vars->ioBuffer || !vars->handoffBuffer)
3a60a9f5
A		 1216 {
1217 err = kIOReturnNoMemory;
1218 break;
1219 }
1220
db609669
A		 1221 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey)))
1222 {
1223 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMinSize = num->unsigned64BitValue();
1224 obj->release();
1225 }
1226 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey)))
1227 {
1228 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMaxSize = num->unsigned64BitValue();
1229 obj->release();
1230 }
1231
1232 boolean_t encryptedswap = true;
1233 uint32_t pageCount;
1234 AbsoluteTime startTime, endTime;
1235 uint64_t nsec;
1236
1237 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
1238 gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
6d2010ae 1239 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
db609669 1240
39236c6e 1241 vmflush = (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey));
db609669 1242 uint64_t setFileSize = 0;
39236c6e
A		 1243 err = hibernate_alloc_page_lists(&vars->page_list,
1244 &vars->page_list_wired,
1245 &vars->page_list_pal);
1246 if (KERN_SUCCESS != err)
1247 break;
db609669
A		 1248
1249 if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode))
1250 {
1251 hibernate_page_list_setall(vars->page_list,
1252 vars->page_list_wired,
1253 vars->page_list_pal,
1254 true /* preflight */,
39236c6e 1255 vmflush /* discard */,
db609669
A		 1256 &pageCount);
1257 PE_Video consoleInfo;
1258 bzero(&consoleInfo, sizeof(consoleInfo));
1259 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1260
1261 // estimate: 5% increase in pages compressed
1262 // screen preview 2 images compressed 50%
1263 setFileSize = ((ptoa_64((105 * pageCount) / 100) * gIOHibernateCompression) >> 8)
1264 + vars->page_list->list_size
1265 + (consoleInfo.v_width * consoleInfo.v_height * 4);
1266
1267 HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n",
1268 pageCount, (100ULL * gIOHibernateCompression) >> 8,
1269 setFileSize, vars->fileMinSize);
1270
1271 if (!(kIOHibernateModeFileResize & gIOHibernateMode)
1272 && (setFileSize < vars->fileMinSize))
1273 {
1274 setFileSize = vars->fileMinSize;
1275 }
1276 }
1277
1278 // open & invalidate the image file
1279
1280 err = IOPolledFileOpen(gIOHibernateFilename, setFileSize, vars->ioBuffer,
6d2010ae
A		 1281 &vars->fileVars, &vars->fileExtents, &data,
1282 &vars->volumeCryptKey[0]);
3a60a9f5
A		 1283 if (KERN_SUCCESS != err)
1284 {
1285 HIBLOG("IOPolledFileOpen(%x)\n", err);
1286 break;
1287 }
3a60a9f5 1288
39236c6e
A		 1289 clock_get_uptime(&startTime);
1290 err = hibernate_setup(gIOHibernateCurrentHeader,
1291 gIOHibernateFreeRatio, gIOHibernateFreeTime,
1292 vmflush,
1293 vars->page_list, vars->page_list_wired, vars->page_list_pal);
1294 clock_get_uptime(&endTime);
1295 SUB_ABSOLUTETIME(&endTime, &startTime);
1296 absolutetime_to_nanoseconds(endTime, &nsec);
1297 HIBLOG("hibernate_setup(%d) took %qd ms\n", err, nsec / 1000000ULL);
1298
6d2010ae 1299 dsSSD = ((0 != (kIOHibernateOptionSSD & vars->fileVars->flags))
0b4c1975 1300 && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
0b4c1975
A		 1301 if (dsSSD)
1302 {
1303 gIOHibernateCurrentHeader->options |=
1304 kIOHibernateOptionSSD
1305 | kIOHibernateOptionColor;
6d2010ae
A		 1306
1307#if defined(__i386__) || defined(__x86_64__)
1308 if (!uuid_is_null(vars->volumeCryptKey) &&
1309 (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey)))
1310 {
1311 uintptr_t smcVars[2];
1312 smcVars[0] = sizeof(vars->volumeCryptKey);
316670eb 1313 smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
6d2010ae
A		 1314
1315 IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
1316 bzero(smcVars, sizeof(smcVars));
1317 }
1318#endif
0b4c1975
A		 1319 }
1320 else
1321 {
1322 gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
1323 }
1324
0b4c1975 1325
3a60a9f5 1326 if (KERN_SUCCESS != err)
3a60a9f5 1327 break;
3a60a9f5 1328
6d2010ae 1329 if (encryptedswap || !uuid_is_null(vars->volumeCryptKey))
3a60a9f5
A		 1330 gIOHibernateMode ^= kIOHibernateModeEncrypt;
1331
0b4c1975
A		 1332 if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1333 {
1334 vars->videoAllocSize = kVideoMapSize;
1335 if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize))
1336 vars->videoMapping = 0;
1337 }
3a60a9f5
A		 1338
1339 // generate crypt keys
1340 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
1341 vars->wiredCryptKey[i] = random();
1342 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
1343 vars->cryptKey[i] = random();
1344
1345 // set nvram
1346
1347 IORegistryEntry * regEntry;
1348 if (!gIOOptionsEntry)
1349 {
1350 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
1351 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
1352 if (regEntry && !gIOOptionsEntry)
1353 regEntry->release();
1354 }
3a60a9f5
A		 1355
1356 if (gIOOptionsEntry)
1357 {
1358 const OSSymbol * sym;
3a60a9f5
A		 1359
1360 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1361 if (sym)
1362 {
1363 gIOOptionsEntry->setProperty(sym, data);
1364 sym->release();
1365 }
1366 data->release();
1367
b0d623f7 1368#if defined(__i386__) || defined(__x86_64__)
0c530ab8
A		 1369 struct AppleRTCHibernateVars
1370 {
1371 uint8_t signature[4];
1372 uint32_t revision;
1373 uint8_t booterSignature[20];
1374 uint8_t wiredCryptKey[16];
1375 };
1376 AppleRTCHibernateVars rtcVars;
1377
1378 rtcVars.signature[0] = 'A';
1379 rtcVars.signature[1] = 'A';
1380 rtcVars.signature[2] = 'P';
1381 rtcVars.signature[3] = 'L';
1382 rtcVars.revision = 1;
1383 bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
1384 if (gIOHibernateBootSignature[0])
1385 {
1386 char c;
1387 uint8_t value = 0;
1388 for (uint32_t i = 0;
1389 (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1));
1390 i++)
1391 {
1392 if (c >= 'a')
1393 c -= 'a' - 10;
1394 else if (c >= 'A')
1395 c -= 'A' - 10;
1396 else if (c >= '0')
1397 c -= '0';
1398 else
1399 continue;
1400 value = (value << 4) | c;
1401 if (i & 1)
1402 rtcVars.booterSignature[i >> 1] = value;
1403 }
1404 }
1405 data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
1406 if (data)
2d21ac55 1407 {
060df5ea
A		 1408 if (!gIOHibernateRTCVariablesKey)
1409 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1410 if (gIOHibernateRTCVariablesKey)
1411 IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
1412
1413 if( gIOOptionsEntry )
1414 {
1415 if( gIOHibernateMode & kIOHibernateModeSwitch )
1416 {
1417 const OSSymbol *sym;
1418 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSwitchVarsKey);
1419 if( sym )
2d21ac55 1420 {
060df5ea
A		 1421 gIOOptionsEntry->setProperty(sym, data); /* intentional insecure backup of rtc boot vars */
1422 sym->release();
2d21ac55 1423 }
060df5ea
A		 1424 }
1425 }
2d21ac55 1426
060df5ea 1427 data->release();
0c530ab8
A		 1428 }
1429 if (gIOChosenEntry)
1430 {
1431 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
1432 if (data)
1433 gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
060df5ea
A		 1434 {
1435 // set BootNext
1436
1437 if (!gIOHibernateBoot0082Data)
1438 {
1439 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
1440 if (data)
1441 {
1442 // AppleNVRAM_EFI_LOAD_OPTION
1443 struct {
1444 uint32_t Attributes;
1445 uint16_t FilePathLength;
1446 uint16_t Desc;
1447 } loadOptionHeader;
1448 loadOptionHeader.Attributes = 1;
1449 loadOptionHeader.FilePathLength = data->getLength();
1450 loadOptionHeader.Desc = 0;
1451 gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
1452 if (gIOHibernateBoot0082Data)
1453 {
1454 gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
1455 gIOHibernateBoot0082Data->appendBytes(data);
1456 }
1457 }
1458 }
1459 if (!gIOHibernateBoot0082Key)
1460 gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
1461 if (!gIOHibernateBootNextKey)
1462 gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
1463 if (!gIOHibernateBootNextData)
1464 {
1465 uint16_t bits = 0x0082;
1466 gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
1467 }
1468 if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData)
1469 {
1470 gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
1471 gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
1472 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
1473 }
1474 }
0c530ab8 1475 }
b0d623f7 1476#else /* !i386 && !x86_64 */
3a60a9f5
A		 1477 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1478 {
1479 data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1480 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
1481 if (sym && data)
1482 gIOOptionsEntry->setProperty(sym, data);
1483 if (sym)
1484 sym->release();
1485 if (data)
1486 data->release();
2d21ac55 1487 if (false && gIOHibernateBootSignature[0])
3a60a9f5
A		 1488 {
1489 data = OSData::withCapacity(16);
1490 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
1491 if (sym && data)
1492 {
1493 char c;
0c530ab8 1494 uint8_t value = 0;
3a60a9f5
A		 1495 for (uint32_t i = 0; (c = gIOHibernateBootSignature[i]); i++)
1496 {
1497 if (c >= 'a')
1498 c -= 'a' - 10;
1499 else if (c >= 'A')
1500 c -= 'A' - 10;
1501 else if (c >= '0')
1502 c -= '0';
1503 else
1504 continue;
1505 value = (value << 4) | c;
1506 if (i & 1)
1507 data->appendBytes(&value, sizeof(value));
1508 }
1509 gIOOptionsEntry->setProperty(sym, data);
1510 }
1511 if (sym)
1512 sym->release();
1513 if (data)
1514 data->release();
1515 }
1516 }
3a60a9f5
A		 1517 if (!vars->haveFastBoot)
1518 {
1519 // set boot volume to zero
1520 IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
1521 if (platform && (kIOReturnSuccess == platform->readXPRAM(kXPRamAudioVolume,
1522 &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume))))
1523 {
1524 uint8_t newVolume;
1525 newVolume = vars->saveBootAudioVolume & 0xf8;
1526 platform->writeXPRAM(kXPRamAudioVolume,
1527 &newVolume, sizeof(newVolume));
1528 }
1529 }
b0d623f7 1530#endif /* !i386 && !x86_64 */
3a60a9f5
A		 1531 }
1532 // --
1533
316670eb
A		 1534 }
1535 while (false);
1536
1537 IOLockLock(gFSLock);
1538 if ((kIOReturnSuccess == err) && (kFSOpening == gFSState))
1539 {
1540 gFSState = kFSOpened;
1541 gIOHibernateVars = *vars;
1542 gFileVars = *vars->fileVars;
1543 gIOHibernateVars.fileVars = &gFileVars;
1544 gIOHibernateFileRef = gFileVars.fileRef;
3a60a9f5
A		 1545 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
1546 gIOHibernateState = kIOHibernateStateHibernating;
1547 }
316670eb
A		 1548 else
1549 {
1550 HIBLOG("hibernate file close due timeout\n");
1551 if (vars->fileVars && vars->fileVars->fileRef) kern_close_file_for_direct_io(vars->fileVars->fileRef, 0, 0, 0, 0, 0);
1552 IOHibernateDone(vars);
1553 gFSState = kFSIdle;
1554 }
1555 IOLockUnlock(gFSLock);
1556
1557 if (vars->fileVars) IODelete(vars->fileVars, IOPolledFileIOVars, 1);
1558 IODelete(vars, IOHibernateVars, 1);
3a60a9f5
A		 1559
1560 return (err);
1561}
1562
1563/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1564
2d21ac55
A		 1565DECLARE_IOHIBERNATEPROGRESSALPHA
1566
1567static void
1568ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
1569{
1570 uint32_t rowBytes, pixelShift;
1571 uint32_t x, y;
1572 int32_t blob;
1573 uint32_t alpha, in, color, result;
1574 uint8_t * out;
1575 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1576
1577 rowBytes = display->rowBytes;
1578 pixelShift = display->depth >> 4;
1579 if (pixelShift < 1) return;
1580
1581 screen += ((display->width
1582 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1583 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1584
1585 for (y = 0; y < kIOHibernateProgressHeight; y++)
1586 {
1587 out = screen + y * rowBytes;
1588 for (blob = 0; blob < kIOHibernateProgressCount; blob++)
1589 {
1590 color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
1591 for (x = 0; x < kIOHibernateProgressWidth; x++)
1592 {
1593 alpha = gIOHibernateProgressAlpha[y][x];
1594 result = color;
1595 if (alpha)
1596 {
1597 if (0xff != alpha)
1598 {
1599 if (1 == pixelShift)
1600 {
1601 in = *((uint16_t *)out) & 0x1f; // 16
1602 in = (in << 3) | (in >> 2);
1603 }
1604 else
1605 in = *((uint32_t *)out) & 0xff; // 32
1606 saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
1607 result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
1608 }
1609 if (1 == pixelShift)
1610 {
1611 result >>= 3;
1612 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1613 }
1614 else
1615 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1616 }
1617 out += (1 << pixelShift);
1618 }
1619 out += (kIOHibernateProgressSpacing << pixelShift);
1620 }
1621 }
1622}
1623
1624
1625static void
1626ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
1627{
1628 uint32_t rowBytes, pixelShift;
1629 uint32_t x, y;
1630 int32_t blob, lastBlob;
1631 uint32_t alpha, in, color, result;
1632 uint8_t * out;
1633 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1634
1635 pixelShift = display->depth >> 4;
1636 if (pixelShift < 1)
1637 return;
1638
1639 rowBytes = display->rowBytes;
1640
1641 screen += ((display->width
1642 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1643 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1644
1645 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
1646
1647 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
1648
1649 for (y = 0; y < kIOHibernateProgressHeight; y++)
1650 {
1651 out = screen + y * rowBytes;
1652 for (blob = firstBlob; blob <= lastBlob; blob++)
1653 {
1654 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
1655 for (x = 0; x < kIOHibernateProgressWidth; x++)
1656 {
1657 alpha = gIOHibernateProgressAlpha[y][x];
1658 result = color;
1659 if (alpha)
1660 {
1661 if (0xff != alpha)
1662 {
1663 in = display->progressSaveUnder[blob][saveindex[blob]++];
1664 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
1665 }
1666 if (1 == pixelShift)
1667 {
1668 result >>= 3;
1669 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1670 }
1671 else
1672 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1673 }
1674 out += (1 << pixelShift);
1675 }
1676 out += (kIOHibernateProgressSpacing << pixelShift);
1677 }
1678 }
1679}
1680
1681/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1682
316670eb
A		 1683IOReturn
1684IOHibernateIOKitSleep(void)
1685{
1686 IOReturn ret = kIOReturnSuccess;
1687 IOLockLock(gFSLock);
1688 if (kFSOpening == gFSState)
1689 {
1690 gFSState = kFSTimedOut;
1691 HIBLOG("hibernate file open timed out\n");
1692 ret = kIOReturnTimeout;
1693 }
1694 IOLockUnlock(gFSLock);
1695 return (ret);
1696}
1697
1698/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1699
3a60a9f5
A		 1700IOReturn
1701IOHibernateSystemHasSlept(void)
1702{
316670eb 1703 IOReturn ret = kIOReturnSuccess;
3a60a9f5 1704 IOHibernateVars * vars = &gIOHibernateVars;
316670eb 1705 OSObject * obj = 0;
2d21ac55
A		 1706 OSData * data;
1707
316670eb
A		 1708 IOLockLock(gFSLock);
1709 if ((kFSOpened != gFSState) && gIOHibernateMode)
1710 {
1711 ret = kIOReturnTimeout;
1712 }
1713 IOLockUnlock(gFSLock);
1714 if (kIOReturnSuccess != ret) return (ret);
1715
1716 if (gIOHibernateMode) obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
2d21ac55
A		 1717 vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
1718 if (obj && !vars->previewBuffer)
1719 obj->release();
3a60a9f5 1720
2d21ac55
A		 1721 vars->consoleMapping = NULL;
1722 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
3a60a9f5 1723 {
2d21ac55
A		 1724 vars->previewBuffer->release();
1725 vars->previewBuffer = 0;
1726 }
3a60a9f5 1727
0b4c1975
A		 1728 if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1729 && vars->previewBuffer
1730 && (data = OSDynamicCast(OSData,
2d21ac55
A		 1731 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey))))
1732 {
1733 UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
b0d623f7 1734 HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
2d21ac55
A		 1735
1736 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
1737
1738 if (kIOHibernatePreviewUpdates & flags)
1739 {
1740 PE_Video consoleInfo;
1741 hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
1742
1743 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1744
1745 graphicsInfo->width = consoleInfo.v_width;
1746 graphicsInfo->height = consoleInfo.v_height;
1747 graphicsInfo->rowBytes = consoleInfo.v_rowBytes;
1748 graphicsInfo->depth = consoleInfo.v_depth;
1749 vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
1750
1751 HIBPRINT("video %p %d %d %d\n",
0b4c1975
A		 1752 vars->consoleMapping, graphicsInfo->depth,
1753 graphicsInfo->width, graphicsInfo->height);
2d21ac55
A		 1754 if (vars->consoleMapping)
1755 ProgressInit(graphicsInfo, vars->consoleMapping,
1756 &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
1757 }
3a60a9f5 1758 }
2d21ac55 1759
3a60a9f5
A		 1760 if (gIOOptionsEntry)
1761 gIOOptionsEntry->sync();
1762
316670eb 1763 return (ret);
3a60a9f5
A		 1764}
1765
1766/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1767
6d2010ae
A		 1768static DeviceTreeNode *
1769MergeDeviceTree(DeviceTreeNode * entry, IORegistryEntry * regEntry)
1770{
1771 DeviceTreeNodeProperty * prop;
1772 DeviceTreeNode * child;
1773 IORegistryEntry * childRegEntry;
1774 const char * nameProp;
1775 unsigned int propLen, idx;
1776
1777 prop = (DeviceTreeNodeProperty *) (entry + 1);
1778 for (idx = 0; idx < entry->nProperties; idx++)
1779 {
1780 if (regEntry && (0 != strcmp("name", prop->name)))
1781 {
1782 regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
1783// HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
1784 }
1785 prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
1786 }
1787
1788 child = (DeviceTreeNode *) prop;
1789 for (idx = 0; idx < entry->nChildren; idx++)
1790 {
1791 if (kSuccess != DTGetProperty(child, "name", (void **) &nameProp, &propLen))
1792 panic("no name");
1793 childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
1794// HIBPRINT("%s == %p\n", nameProp, childRegEntry);
1795 child = MergeDeviceTree(child, childRegEntry);
1796 }
1797 return (child);
1798}
1799
3a60a9f5
A		 1800IOReturn
1801IOHibernateSystemWake(void)
1802{
316670eb
A		 1803 if (kFSOpened == gFSState)
1804 {
1805 IOHibernateDone(&gIOHibernateVars);
1806 }
1807 else
1808 {
1809 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1810 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1811 }
1812 return (kIOReturnSuccess);
1813}
3a60a9f5 1814
316670eb
A		 1815static IOReturn
1816IOHibernateDone(IOHibernateVars * vars)
1817{
bd504ef0
A		 1818 IORegistryEntry * next;
1819
99c3a104 1820 hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);
3a60a9f5
A		 1821
1822 if (vars->videoMapping)
1823 {
1824 if (vars->videoMapSize)
1825 // remove mappings
1826 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1827 if (vars->videoAllocSize)
1828 // dealloc range
b0d623f7 1829 kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
3a60a9f5
A		 1830 }
1831
1832 if (vars->previewBuffer)
1833 {
1834 vars->previewBuffer->release();
1835 vars->previewBuffer = 0;
1836 }
1837
0b4c1975
A		 1838 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1839 {
1840 IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey,
1841 gIOHibernateCurrentHeader->options, 32);
1842 }
1843 else
1844 {
1845 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1846 }
1847
1848 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1849 && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus))
1850 {
1851 IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey,
1852 &gIOHibernateGraphicsInfo->gfxStatus,
1853 sizeof(gIOHibernateGraphicsInfo->gfxStatus));
1854 }
1855 else
1856 {
1857 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1858 }
1859
3a60a9f5
A		 1860 if (vars->fileVars)
1861 {
bd504ef0
A		 1862 if ((next = vars->fileVars->media)) do
1863 {
1864 next->removeProperty(kIOPolledInterfaceActiveKey);
1865 next = next->getParentEntry(gIOServicePlane);
1866 }
1867 while (next);
3a60a9f5
A		 1868 IOPolledFileClose(vars->fileVars);
1869 }
1870
1871 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
1872
b0d623f7 1873#if defined(__i386__) || defined(__x86_64__)
060df5ea 1874 IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
6d2010ae 1875 IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
2d21ac55
A		 1876
1877 /*
1878 * Hibernate variable is written to NVRAM on platforms in which RtcRam
1879 * is not backed by coin cell. Remove Hibernate data from NVRAM.
1880 */
1881 if (gIOOptionsEntry) {
2d21ac55 1882
060df5ea
A		 1883 if (gIOHibernateRTCVariablesKey) {
1884 if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
1885 gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
1886 }
1887 }
1888
1889 if (gIOHibernateBootNextKey)
1890 {
1891 if (gIOHibernateBootNextSave)
1892 {
1893 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
1894 gIOHibernateBootNextSave->release();
1895 gIOHibernateBootNextSave = NULL;
2d21ac55 1896 }
060df5ea
A		 1897 else
1898 gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
1899 }
39236c6e 1900 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) gIOOptionsEntry->sync();
2d21ac55 1901 }
0c530ab8 1902#endif
3a60a9f5
A		 1903
1904 if (vars->srcBuffer)
1905 vars->srcBuffer->release();
1906 if (vars->ioBuffer)
1907 vars->ioBuffer->release();
6d2010ae 1908 bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
99c3a104 1909 if (vars->handoffBuffer)
6d2010ae 1910 {
99c3a104 1911 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
6d2010ae 1912 {
99c3a104
A		 1913 IOHibernateHandoff * handoff;
1914 bool done = false;
1915 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
1916 !done;
1917 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
6d2010ae 1918 {
99c3a104
A		 1919 HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
1920 uint8_t * data = &handoff->data[0];
1921 switch (handoff->type)
1922 {
1923 case kIOHibernateHandoffTypeEnd:
1924 done = true;
1925 break;
6d2010ae 1926
99c3a104
A		 1927 case kIOHibernateHandoffTypeDeviceTree:
1928 MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot());
1929 break;
1930
1931 case kIOHibernateHandoffTypeKeyStore:
6d2010ae 1932#if defined(__i386__) || defined(__x86_64__)
6d2010ae 1933 {
99c3a104
A		 1934 IOBufferMemoryDescriptor *
1935 md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
1936 if (md)
1937 {
1938 IOSetKeyStoreData(md);
1939 }
6d2010ae 1940 }
6d2010ae 1941#endif
99c3a104
A		 1942 break;
1943
1944 default:
1945 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
1946 break;
1947 }
1948 }
6d2010ae
A		 1949 }
1950 vars->handoffBuffer->release();
1951 }
3a60a9f5
A		 1952 if (vars->fileExtents)
1953 vars->fileExtents->release();
1954
1955 bzero(vars, sizeof(*vars));
1956
1957// gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
1958
1959 return (kIOReturnSuccess);
1960}
1961
1962IOReturn
1963IOHibernateSystemPostWake(void)
1964{
316670eb
A		 1965 struct kern_direct_file_io_ref_t * fileRef;
1966
1967 if (kFSOpened == gFSState)
3a60a9f5 1968 {
6d2010ae 1969 // invalidate & close the image file
0c530ab8 1970 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
316670eb
A		 1971 if ((fileRef = gIOHibernateFileRef))
1972 {
1973 gIOHibernateFileRef = 0;
39236c6e 1974 IOSleep(TRIM_DELAY);
316670eb 1975 kern_close_file_for_direct_io(fileRef,
39236c6e
A		 1976#if DISABLE_TRIM
1977 0, 0, 0, 0, 0);
1978#else
6d2010ae 1979 0, (caddr_t) gIOHibernateCurrentHeader,
7ddcb079 1980 sizeof(IOHibernateImageHeader),
39236c6e 1981 0,
7ddcb079 1982 gIOHibernateCurrentHeader->imageSize);
39236c6e 1983#endif
316670eb
A		 1984 }
1985 gFSState = kFSIdle;
3a60a9f5
A		 1986 }
1987 return (kIOReturnSuccess);
1988}
1989
316670eb
A		 1990bool IOHibernateWasScreenLocked(void)
1991{
1992 bool ret = false;
1993 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry)
1994 {
1995 OSData *
1996 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
1997 if (data) switch (*((uint32_t *)data->getBytesNoCopy()))
1998 {
1999 case kIOScreenLockLocked:
2000 case kIOScreenLockFileVaultDialog:
2001 ret = true;
2002 break;
2003 case kIOScreenLockNoLock:
2004 case kIOScreenLockUnlocked:
2005 default:
2006 ret = false;
2007 break;
2008 }
2009 }
2010 return (ret);
2011}
2012
3a60a9f5
A		 2013/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2014
2d21ac55 2015SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
6d2010ae 2016 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2d21ac55
A		 2017 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
2018SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
6d2010ae 2019 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2d21ac55
A		 2020 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
2021SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
6d2010ae 2022 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2d21ac55 2023 &gIOHibernateMode, 0, "");
39236c6e
A		 2024SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
2025 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2026 gIOHibernateStats, hibernate_statistics_t, "");
2027
2028SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
2029 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
2030 &gIOHibernateStats->graphicsReadyTime, 0, "");
2031SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
2032 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
2033 &gIOHibernateStats->wakeNotificationTime, 0, "");
2034SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
2035 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
2036 &gIOHibernateStats->lockScreenReadyTime, 0, "");
2037SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
2038 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
2039 &gIOHibernateStats->hidReadyTime, 0, "");
2040
2d21ac55 2041
3a60a9f5
A		 2042void
2043IOHibernateSystemInit(IOPMrootDomain * rootDomain)
2044{
2045 OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
2046 if (data)
2047 {
2048 rootDomain->setProperty(kIOHibernateStateKey, data);
2049 data->release();
2050 }
2051
2d21ac55 2052 if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename)))
3a60a9f5
A		 2053 gIOHibernateMode = kIOHibernateModeOn;
2054 else
2055 gIOHibernateFilename[0] = 0;
2056
3a60a9f5 2057 sysctl_register_oid(&sysctl__kern_hibernatefile);
3a60a9f5 2058 sysctl_register_oid(&sysctl__kern_bootsignature);
3a60a9f5 2059 sysctl_register_oid(&sysctl__kern_hibernatemode);
39236c6e
A		 2060 sysctl_register_oid(&sysctl__kern_hibernatestatistics);
2061 sysctl_register_oid(&sysctl__kern_hibernategraphicsready);
2062 sysctl_register_oid(&sysctl__kern_hibernatewakenotification);
2063 sysctl_register_oid(&sysctl__kern_hibernatelockscreenready);
2064 sysctl_register_oid(&sysctl__kern_hibernatehidready);
316670eb
A		 2065
2066 gFSLock = IOLockAlloc();
3a60a9f5
A		 2067}
2068
2d21ac55 2069
3a60a9f5
A		 2070/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2071
2072static void
2073hibernate_setup_for_wake(void)
2074{
3a60a9f5
A		 2075}
2076
2077/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2078
0c530ab8
A		 2079#define C_ASSERT(e) typedef char __C_ASSERT__[(e) ? 1 : -1]
2080
0b4c1975
A		 2081static bool
2082no_encrypt_page(vm_offset_t ppnum)
2083{
2084 if (pmap_is_noencrypt((ppnum_t)ppnum) == TRUE)
2085 {
2086 return true;
2087 }
2088 return false;
2089}
2090
6d2010ae
A		 2091static void
2092hibernate_pal_callback(void *vars_arg, vm_offset_t addr)
2093{
2094 IOHibernateVars *vars = (IOHibernateVars *)vars_arg;
2095 /* Make sure it's not in either of the save lists */
2096 hibernate_set_page_state(vars->page_list, vars->page_list_wired, atop_64(addr), 1, kIOHibernatePageStateFree);
2097
2098 /* Set it in the bitmap of pages owned by the PAL */
2099 hibernate_page_bitset(vars->page_list_pal, TRUE, atop_64(addr));
2100}
2101
2102static struct hibernate_cryptvars_t *local_cryptvars;
2103
2104extern "C" int
2105hibernate_pal_write(void *buffer, size_t size)
2106{
2107 IOHibernateVars * vars = &gIOHibernateVars;
2108
2109 IOReturn err = IOPolledFileWrite(vars->fileVars, (const uint8_t *)buffer, size, local_cryptvars);
2110 if (kIOReturnSuccess != err) {
2111 kprintf("epic hibernate fail! %d\n", err);
2112 return err;
2113 }
2114
2115 return 0;
2116}
2117
2118
2d21ac55 2119extern "C" uint32_t
3a60a9f5
A		 2120hibernate_write_image(void)
2121{
2122 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
2123 IOHibernateVars * vars = &gIOHibernateVars;
2124 IOPolledFileExtent * fileExtents;
2125
0c530ab8
A		 2126 C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
2127
3a60a9f5
A		 2128 uint32_t pageCount, pagesDone;
2129 IOReturn err;
0b4c1975
A		 2130 vm_offset_t ppnum, page;
2131 IOItemCount count;
3a60a9f5
A		 2132 uint8_t * src;
2133 uint8_t * data;
39236c6e
A		 2134 uint8_t * compressed;
2135 uint8_t * scratch;
2136 void * zerosCompressed;
2137 IOByteCount pageCompressedSize, zerosCompressedLen;
3a60a9f5
A		 2138 uint64_t compressedSize, uncompressedSize;
2139 uint64_t image1Size = 0;
2140 uint32_t bitmap_size;
0b4c1975 2141 bool iterDone, pollerOpen, needEncrypt;
3a60a9f5 2142 uint32_t restore1Sum, sum, sum1, sum2;
39236c6e 2143 int wkresult;
3a60a9f5
A		 2144 uint32_t tag;
2145 uint32_t pageType;
2146 uint32_t pageAndCount[2];
6d2010ae
A		 2147 addr64_t phys64;
2148 IOByteCount segLen;
3a60a9f5
A		 2149
2150 AbsoluteTime startTime, endTime;
6d2010ae 2151 AbsoluteTime allTime, compTime;
0b4c1975 2152 uint64_t compBytes;
3a60a9f5
A		 2153 uint64_t nsec;
2154 uint32_t lastProgressStamp = 0;
2155 uint32_t progressStamp;
2d21ac55 2156 uint32_t blob, lastBlob = (uint32_t) -1L;
3a60a9f5 2157
39236c6e
A		 2158 uint32_t wiredPagesEncrypted;
2159 uint32_t dirtyPagesEncrypted;
2160 uint32_t wiredPagesClear;
2161 uint32_t zeroPageCount;
2162
3a60a9f5
A		 2163 hibernate_cryptvars_t _cryptvars;
2164 hibernate_cryptvars_t * cryptvars = 0;
2165
39236c6e
A		 2166 wiredPagesEncrypted = 0;
2167 dirtyPagesEncrypted = 0;
2168 wiredPagesClear = 0;
2169 zeroPageCount = 0;
0b4c1975 2170
3a60a9f5
A		 2171 if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
2172 return (false /* sleep */ );
2173
0b4c1975
A		 2174 if (kIOHibernateModeSleep & gIOHibernateMode)
2175 kdebug_enable = save_kdebug_enable;
2176
2177 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START, 0, 0, 0, 0, 0);
6d2010ae 2178 IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
0b4c1975 2179
3a60a9f5
A		 2180 restore1Sum = sum1 = sum2 = 0;
2181
6d2010ae
A		 2182 hibernate_pal_prepare();
2183
2d21ac55 2184#if CRYPTO
3a60a9f5
A		 2185 // encryption data. "iv" is the "initial vector".
2186 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2187 {
2188 static const unsigned char first_iv[AES_BLOCK_SIZE]
2189 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
2190 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
2191
2192 cryptvars = &gIOHibernateCryptWakeContext;
2193 bzero(cryptvars, sizeof(hibernate_cryptvars_t));
2194 aes_encrypt_key(vars->cryptKey,
2195 kIOHibernateAESKeySize,
2196 &cryptvars->ctx.encrypt);
2197 aes_decrypt_key(vars->cryptKey,
2198 kIOHibernateAESKeySize,
2199 &cryptvars->ctx.decrypt);
2200
2201 cryptvars = &_cryptvars;
2202 bzero(cryptvars, sizeof(hibernate_cryptvars_t));
6d2010ae
A		 2203 for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++)
2204 vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
2205 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
3a60a9f5
A		 2206 aes_encrypt_key(vars->wiredCryptKey,
2207 kIOHibernateAESKeySize,
2208 &cryptvars->ctx.encrypt);
2209
2210 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
2211 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
2212 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
6d2010ae
A		 2213
2214 local_cryptvars = cryptvars;
3a60a9f5 2215 }
2d21ac55 2216#endif /* CRYPTO */
3a60a9f5
A		 2217
2218 hibernate_setup_for_wake();
2219
2220 hibernate_page_list_setall(vars->page_list,
2221 vars->page_list_wired,
db609669
A		 2222 vars->page_list_pal,
2223 false /* !preflight */,
39236c6e
A		 2224 /* discard_all */
2225 ((0 == (kIOHibernateModeSleep & gIOHibernateMode))
2226 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))),
3a60a9f5
A		 2227 &pageCount);
2228
2229 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
2230
2231 fileExtents = (IOPolledFileExtent *) vars->fileExtents->getBytesNoCopy();
2232
2233#if 0
2234 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
2235 for (page = 0; page < count; page++)
2236 {
2237 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
2238 fileExtents[page].start, fileExtents[page].length,
2239 fileExtents[page].start + fileExtents[page].length);
2240 }
2241#endif
2242
0b4c1975 2243 needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
3a60a9f5 2244 AbsoluteTime_to_scalar(&compTime) = 0;
6d2010ae 2245 compBytes = 0;
3a60a9f5
A		 2246
2247 clock_get_uptime(&allTime);
b0d623f7
A		 2248 IOService::getPMRootDomain()->pmStatsRecordEvent(
2249 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
3a60a9f5
A		 2250 do
2251 {
2252 compressedSize = 0;
2253 uncompressedSize = 0;
39236c6e 2254 zeroPageCount = 0;
3a60a9f5 2255
39236c6e 2256 IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);
3a60a9f5
A		 2257
2258 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
2259 ml_get_interrupts_enabled());
2260 err = IOHibernatePollerOpen(vars->fileVars, kIOPolledBeforeSleepState, vars->ioBuffer);
2261 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
2262 pollerOpen = (kIOReturnSuccess == err);
2263 if (!pollerOpen)
2264 break;
2265
2266 // copy file block extent list if larger than header
2267
2268 count = vars->fileExtents->getLength();
2269 if (count > sizeof(header->fileExtentMap))
2270 {
2271 count -= sizeof(header->fileExtentMap);
2272 err = IOPolledFileWrite(vars->fileVars,
2273 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
2274 if (kIOReturnSuccess != err)
2275 break;
2276 }
2277
b0d623f7
A		 2278 uintptr_t hibernateBase;
2279 uintptr_t hibernateEnd;
2280
6d2010ae 2281 hibernateBase = HIB_BASE; /* Defined in PAL headers */
b0d623f7 2282
316670eb 2283 hibernateEnd = (segHIBB + segSizeHIB);
6d2010ae 2284
3a60a9f5 2285 // copy out restore1 code
6d2010ae
A		 2286
2287 for (count = 0;
2288 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2289 count += segLen)
2290 {
2291 for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++)
2292 {
2293 gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone;
2294 }
2295 }
2296
2297 page = atop_32(kvtophys(hibernateBase));
2298 count = atop_32(round_page(hibernateEnd) - hibernateBase);
2299 header->restore1CodePhysPage = page;
2300 header->restore1CodeVirt = hibernateBase;
3a60a9f5 2301 header->restore1PageCount = count;
b0d623f7
A		 2302 header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
2303 header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
3a60a9f5 2304
316670eb 2305 // sum __HIB seg, with zeros for the stack
b0d623f7 2306 src = (uint8_t *) trunc_page(hibernateBase);
3a60a9f5
A		 2307 for (page = 0; page < count; page++)
2308 {
2309 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
6d2010ae 2310 restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page);
3a60a9f5 2311 else
0b4c1975 2312 restore1Sum += 0x00000000;
3a60a9f5
A		 2313 src += page_size;
2314 }
2315 sum1 = restore1Sum;
2316
316670eb 2317 // write the __HIB seg, with zeros for the stack
3a60a9f5 2318
b0d623f7
A		 2319 src = (uint8_t *) trunc_page(hibernateBase);
2320 count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
3a60a9f5
A		 2321 if (count)
2322 {
2323 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
2324 if (kIOReturnSuccess != err)
2325 break;
2326 }
2327 err = IOPolledFileWrite(vars->fileVars,
2328 (uint8_t *) 0,
2329 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
2330 cryptvars);
2331 if (kIOReturnSuccess != err)
2332 break;
2333 src = &gIOHibernateRestoreStackEnd[0];
b0d623f7 2334 count = round_page(hibernateEnd) - ((uintptr_t) src);
3a60a9f5
A		 2335 if (count)
2336 {
2337 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
2338 if (kIOReturnSuccess != err)
2339 break;
2340 }
2341
39236c6e
A		 2342 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2343 {
2344 vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
2345 vars->fileVars->encryptEnd = UINT64_MAX;
2346 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
2347 }
316670eb 2348
3a60a9f5
A		 2349 // write the preview buffer
2350
2d21ac55 2351 if (vars->previewBuffer)
3a60a9f5
A		 2352 {
2353 ppnum = 0;
2354 count = 0;
2355 do
2356 {
b0d623f7 2357 phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
3a60a9f5
A		 2358 pageAndCount[0] = atop_64(phys64);
2359 pageAndCount[1] = atop_32(segLen);
2360 err = IOPolledFileWrite(vars->fileVars,
2361 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
2362 cryptvars);
2363 if (kIOReturnSuccess != err)
2364 break;
2365 count += segLen;
2366 ppnum += sizeof(pageAndCount);
2367 }
2368 while (phys64);
2369 if (kIOReturnSuccess != err)
2370 break;
2371
b0d623f7 2372 src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
316670eb
A		 2373
2374 ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
2375
2d21ac55 2376 count = vars->previewBuffer->getLength();
3a60a9f5
A		 2377
2378 header->previewPageListSize = ppnum;
2379 header->previewSize = count + ppnum;
2380
2381 for (page = 0; page < count; page += page_size)
0b4c1975
A		 2382 {
2383 phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
2384 sum1 += hibernate_sum_page(src + page, atop_64(phys64));
2385 }
3a60a9f5
A		 2386 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
2387 if (kIOReturnSuccess != err)
2388 break;
2389 }
2390
2391 // mark areas for no save
2392
2393 for (count = 0;
b0d623f7 2394 (phys64 = vars->ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
3a60a9f5
A		 2395 count += segLen)
2396 {
2397 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2398 atop_64(phys64), atop_32(segLen),
2399 kIOHibernatePageStateFree);
2400 pageCount -= atop_32(segLen);
2401 }
2402
2403 for (count = 0;
b0d623f7 2404 (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
3a60a9f5
A		 2405 count += segLen)
2406 {
2407 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2408 atop_64(phys64), atop_32(segLen),
2409 kIOHibernatePageStateFree);
2410 pageCount -= atop_32(segLen);
2411 }
2412
2413 // copy out bitmap of pages available for trashing during restore
2414
2415 bitmap_size = vars->page_list_wired->list_size;
2416 src = (uint8_t *) vars->page_list_wired;
2417 err = IOPolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
2418 if (kIOReturnSuccess != err)
2419 break;
2420
2421 // mark more areas for no save, but these are not available
2422 // for trashing during restore
0c530ab8
A		 2423
2424 hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
3a60a9f5 2425
6d2010ae
A		 2426
2427 page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
2428 count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
3a60a9f5
A		 2429 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2430 page, count,
2431 kIOHibernatePageStateFree);
2432 pageCount -= count;
3a60a9f5
A		 2433
2434 if (vars->previewBuffer) for (count = 0;
b0d623f7 2435 (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
3a60a9f5
A		 2436 count += segLen)
2437 {
2438 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2439 atop_64(phys64), atop_32(segLen),
2440 kIOHibernatePageStateFree);
2441 pageCount -= atop_32(segLen);
2442 }
2443
6d2010ae
A		 2444 for (count = 0;
2445 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2446 count += segLen)
2447 {
2448 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2449 atop_64(phys64), atop_32(segLen),
2450 kIOHibernatePageStateFree);
2451 pageCount -= atop_32(segLen);
2452 }
2453
2454 (void)hibernate_pal_callback;
2455
3a60a9f5 2456 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
39236c6e
A		 2457 compressed = src + page_size;
2458 scratch = compressed + page_size;
2459
2460 // compress a zero page
2461 bzero(src, page_size);
2462 zerosCompressed = vars->handoffBuffer->getBytesNoCopy();
2463 zerosCompressedLen = WKdm_compress_new((WK_word*) src,
2464 (WK_word*) zerosCompressed,
2465 (WK_word*) scratch,
2466 page_size - 4);
2467
6d2010ae
A		 2468 pagesDone = 0;
2469 lastBlob = 0;
3a60a9f5 2470
db609669
A		 2471 HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
2472 bitmap_size, header->previewSize,
2473 pageCount, vars->fileVars->position);
3a60a9f5 2474
0b4c1975
A		 2475 enum
2476 // pageType
2477 {
2478 kWired = 0x02,
2479 kEncrypt = 0x01,
2480 kWiredEncrypt = kWired | kEncrypt,
2481 kWiredClear = kWired,
2482 kUnwiredEncrypt = kEncrypt
2483 };
2484
2485 for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--)
3a60a9f5 2486 {
316670eb
A		 2487 if (kUnwiredEncrypt == pageType)
2488 {
2489 // start unwired image
39236c6e
A		 2490 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2491 {
2492 vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
2493 vars->fileVars->encryptEnd = UINT64_MAX;
2494 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
2495 }
316670eb
A		 2496 bcopy(&cryptvars->aes_iv[0],
2497 &gIOHibernateCryptWakeContext.aes_iv[0],
2498 sizeof(cryptvars->aes_iv));
2499 cryptvars = &gIOHibernateCryptWakeContext;
0b4c1975
A		 2500 }
2501 for (iterDone = false, ppnum = 0; !iterDone; )
2502 {
2503 count = hibernate_page_list_iterate((kWired & pageType)
2504 ? vars->page_list_wired : vars->page_list,
2505 &ppnum);
2506// kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
2507 iterDone = !count;
3a60a9f5 2508
0b4c1975 2509 if (count && (kWired & pageType) && needEncrypt)
3a60a9f5 2510 {
0b4c1975
A		 2511 uint32_t checkIndex;
2512 for (checkIndex = 0;
2513 (checkIndex < count)
2514 && (((kEncrypt & pageType) == 0) == no_encrypt_page(ppnum + checkIndex));
2515 checkIndex++)
2516 {}
2517 if (!checkIndex)
2518 {
2519 ppnum++;
2520 continue;
2521 }
2522 count = checkIndex;
3a60a9f5
A		 2523 }
2524
0b4c1975
A		 2525 switch (pageType)
2526 {
39236c6e
A		 2527 case kWiredEncrypt: wiredPagesEncrypted += count; break;
2528 case kWiredClear: wiredPagesClear += count; break;
2529 case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
0b4c1975
A		 2530 }
2531
2532 if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */}
3a60a9f5 2533 else
3a60a9f5 2534 {
0b4c1975
A		 2535 pageAndCount[0] = ppnum;
2536 pageAndCount[1] = count;
2537 err = IOPolledFileWrite(vars->fileVars,
2538 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
2539 cryptvars);
2540 if (kIOReturnSuccess != err)
2541 break;
3a60a9f5 2542 }
0b4c1975
A		 2543
2544 for (page = ppnum; page < (ppnum + count); page++)
3a60a9f5 2545 {
0b4c1975
A		 2546 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
2547 if (err)
2548 {
2549 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
2550 break;
2551 }
2552
2553 sum = hibernate_sum_page(src, page);
2554 if (kWired & pageType)
2555 sum1 += sum;
2556 else
2557 sum2 += sum;
2558
2559 clock_get_uptime(&startTime);
39236c6e
A		 2560 wkresult = WKdm_compress_new((WK_word*) src,
2561 (WK_word*) compressed,
2562 (WK_word*) scratch,
2563 page_size - 4);
0b4c1975 2564
3a60a9f5 2565 clock_get_uptime(&endTime);
0b4c1975
A		 2566 ADD_ABSOLUTETIME(&compTime, &endTime);
2567 SUB_ABSOLUTETIME(&compTime, &startTime);
39236c6e 2568
0b4c1975 2569 compBytes += page_size;
39236c6e
A		 2570 pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
2571
2572 if ((pageCompressedSize == zerosCompressedLen)
2573 && !bcmp(compressed, zerosCompressed, zerosCompressedLen))
2574 {
2575 pageCompressedSize = 0;
2576 zeroPageCount++;
2577 }
2578
0b4c1975
A		 2579 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2580 pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1);
db609669 2581
0b4c1975 2582 if (pageCompressedSize != page_size)
39236c6e 2583 data = compressed;
0b4c1975
A		 2584 else
2585 data = src;
2586
2587 tag = pageCompressedSize | kIOHibernateTagSignature;
2588 err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
2589 if (kIOReturnSuccess != err)
2590 break;
2591
2592 err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
2593 if (kIOReturnSuccess != err)
2594 break;
2595
2596 compressedSize += pageCompressedSize;
39236c6e 2597 uncompressedSize += page_size;
0b4c1975
A		 2598 pagesDone++;
2599
2600 if (vars->consoleMapping && (0 == (1023 & pagesDone)))
2601 {
2602 blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
2603 if (blob != lastBlob)
2604 {
2605 ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
2606 lastBlob = blob;
2607 }
2608 }
2609 if (0 == (8191 & pagesDone))
2610 {
2611 clock_get_uptime(&endTime);
2612 SUB_ABSOLUTETIME(&endTime, &allTime);
2613 absolutetime_to_nanoseconds(endTime, &nsec);
2614 progressStamp = nsec / 750000000ULL;
2615 if (progressStamp != lastProgressStamp)
2616 {
2617 lastProgressStamp = progressStamp;
2618 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
2619 }
3a60a9f5
A		 2620 }
2621 }
0b4c1975
A		 2622 if (kIOReturnSuccess != err)
2623 break;
2624 ppnum = page;
3a60a9f5 2625 }
0b4c1975 2626
3a60a9f5
A		 2627 if (kIOReturnSuccess != err)
2628 break;
0b4c1975 2629
39236c6e 2630 if ((kEncrypt & pageType) && vars->fileVars->encryptStart)
3a60a9f5 2631 {
7ddcb079 2632 vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
0b4c1975
A		 2633 HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
2634 }
2635
2636 if (kWiredEncrypt != pageType)
2637 {
2638 // end of image1/2 - fill to next block
3a60a9f5
A		 2639 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
2640 if (kIOReturnSuccess != err)
2641 break;
0b4c1975
A		 2642 }
2643 if (kWiredClear == pageType)
2644 {
7ddcb079
A		 2645 // enlarge wired image for test
2646// err = IOPolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);
2647
0b4c1975
A		 2648 // end wired image
2649 header->encryptStart = vars->fileVars->encryptStart;
2650 header->encryptEnd = vars->fileVars->encryptEnd;
3a60a9f5 2651 image1Size = vars->fileVars->position;
7ddcb079 2652 HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
0b4c1975 2653 image1Size, header->encryptStart, header->encryptEnd);
3a60a9f5
A		 2654 }
2655 }
3a60a9f5
A		 2656 if (kIOReturnSuccess != err)
2657 break;
2658
2659 // Header:
2660
2661 header->imageSize = vars->fileVars->position;
2662 header->image1Size = image1Size;
2663 header->bitmapSize = bitmap_size;
2664 header->pageCount = pageCount;
3a60a9f5
A		 2665
2666 header->restore1Sum = restore1Sum;
2667 header->image1Sum = sum1;
2668 header->image2Sum = sum2;
316670eb 2669 header->sleepTime = gIOLastSleepTime.tv_sec;
db609669
A		 2670
2671 header->compression = (compressedSize << 8) / uncompressedSize;
2672 gIOHibernateCompression = header->compression;
3a60a9f5
A		 2673
2674 count = vars->fileExtents->getLength();
2675 if (count > sizeof(header->fileExtentMap))
2676 {
2677 header->fileExtentMapSize = count;
2678 count = sizeof(header->fileExtentMap);
2679 }
2680 else
2681 header->fileExtentMapSize = sizeof(header->fileExtentMap);
2682 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
0b4c1975 2683
39236c6e
A		 2684 header->deviceBase = vars->fileVars->block0;
2685 header->deviceBlockSize = vars->fileVars->blockSize;
6d2010ae 2686
3a60a9f5
A		 2687 IOPolledFileSeek(vars->fileVars, 0);
2688 err = IOPolledFileWrite(vars->fileVars,
2689 (uint8_t *) header, sizeof(IOHibernateImageHeader),
2690 cryptvars);
2691 if (kIOReturnSuccess != err)
2692 break;
2693 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
2694 if (kIOReturnSuccess != err)
2695 break;
2d21ac55 2696 err = IOHibernatePollerIODone(vars->fileVars, true);
3a60a9f5
A		 2697 if (kIOReturnSuccess != err)
2698 break;
2699 }
2700 while (false);
2701
2702 clock_get_uptime(&endTime);
b0d623f7
A		 2703
2704 IOService::getPMRootDomain()->pmStatsRecordEvent(
2705 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
2706
3a60a9f5
A		 2707 SUB_ABSOLUTETIME(&endTime, &allTime);
2708 absolutetime_to_nanoseconds(endTime, &nsec);
39236c6e 2709 HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
3a60a9f5
A		 2710
2711 absolutetime_to_nanoseconds(compTime, &nsec);
6d2010ae
A		 2712 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2713 compBytes,
2714 nsec / 1000000ULL,
2715 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
3a60a9f5 2716
6d2010ae
A		 2717 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2718 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
2719 vars->fileVars->cryptBytes,
2720 nsec / 1000000ULL,
2721 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
3a60a9f5 2722
db609669
A		 2723 HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
2724 header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
3a60a9f5 2725 uncompressedSize, atop_32(uncompressedSize), compressedSize,
8f6c56a5 2726 uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
3a60a9f5
A		 2727 sum1, sum2);
2728
39236c6e
A		 2729 HIBLOG("zeroPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n",
2730 zeroPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
0b4c1975 2731
c910b4d9
A		 2732 if (vars->fileVars->io)
2733 (void) IOHibernatePollerIODone(vars->fileVars, false);
2734
3a60a9f5
A		 2735 if (pollerOpen)
2736 IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState);
2737
2d21ac55
A		 2738 if (vars->consoleMapping)
2739 ProgressUpdate(gIOHibernateGraphicsInfo,
2740 vars->consoleMapping, 0, kIOHibernateProgressCount);
2741
3a60a9f5
A		 2742 HIBLOG("hibernate_write_image done(%x)\n", err);
2743
2744 // should we come back via regular wake, set the state in memory.
2745 gIOHibernateState = kIOHibernateStateInactive;
2746
0b4c1975 2747 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END,
39236c6e 2748 wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted, 0, 0);
0b4c1975 2749
2d21ac55
A		 2750 if (kIOReturnSuccess == err)
2751 {
2752 if (kIOHibernateModeSleep & gIOHibernateMode)
2753 {
2754 return (kIOHibernatePostWriteSleep);
2755 }
2756 else if(kIOHibernateModeRestart & gIOHibernateMode)
2757 {
2758 return (kIOHibernatePostWriteRestart);
2759 }
2760 else
2761 {
2762 /* by default, power down */
2763 return (kIOHibernatePostWriteHalt);
2764 }
2765 }
2766 else if (kIOReturnAborted == err)
2767 {
2768 return (kIOHibernatePostWriteWake);
2769 }
3a60a9f5 2770 else
3a60a9f5 2771 {
2d21ac55
A		 2772 /* on error, sleep */
2773 return (kIOHibernatePostWriteSleep);
3a60a9f5
A		 2774 }
2775}
2776
2777/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2778
2779extern "C" void
2780hibernate_machine_init(void)
2781{
2782 IOReturn err;
2783 uint32_t sum;
2784 uint32_t pagesDone;
0b4c1975 2785 uint32_t pagesRead = 0;
6d2010ae 2786 AbsoluteTime startTime, compTime;
3a60a9f5 2787 AbsoluteTime allTime, endTime;
39236c6e
A		 2788 AbsoluteTime startIOTime, endIOTime;
2789 uint64_t nsec, nsecIO;
6d2010ae 2790 uint64_t compBytes;
3a60a9f5
A		 2791 uint32_t lastProgressStamp = 0;
2792 uint32_t progressStamp;
3a60a9f5
A		 2793 hibernate_cryptvars_t * cryptvars = 0;
2794
2795 IOHibernateVars * vars = &gIOHibernateVars;
39236c6e 2796 bzero(gIOHibernateStats, sizeof(hibernate_statistics_t));
3a60a9f5
A		 2797
2798 if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
2799 return;
2800
3a60a9f5
A		 2801 sum = gIOHibernateCurrentHeader->actualImage1Sum;
2802 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
2803
3a60a9f5
A		 2804 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
2805 {
2806 HIBLOG("regular wake\n");
2807 return;
2808 }
2809
2810 HIBPRINT("diag %x %x %x %x\n",
2811 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
6d2010ae 2812 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
3a60a9f5 2813
39236c6e
A		 2814#define t40ms(x) (tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000)
2815#define tStat(x, y) gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y);
2816 tStat(booterStart, booterStart);
2817 gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart,
2818 tStat(booterDuration0, booterTime0);
2819 tStat(booterDuration1, booterTime1);
2820 tStat(booterDuration2, booterTime2);
2821 tStat(booterDuration, booterTime);
2822 tStat(booterConnectDisplayDuration, connectDisplayTime);
2823 tStat(booterSplashDuration, splashTime);
2824 tStat(trampolineDuration, trampolineTime);
2825
2826 gIOHibernateStats->image1Size = gIOHibernateCurrentHeader->image1Size;
2827 gIOHibernateStats->imageSize = gIOHibernateCurrentHeader->imageSize;
2828 gIOHibernateStats->image1Pages = pagesDone;
2829
2830 HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n",
2831 gIOHibernateStats->booterStart,
2832 gIOHibernateStats->smcStart,
2833 gIOHibernateStats->booterDuration0,
2834 gIOHibernateStats->booterDuration1,
2835 gIOHibernateStats->booterDuration2,
2836 gIOHibernateStats->booterDuration,
2837 gIOHibernateStats->booterConnectDisplayDuration,
2838 gIOHibernateStats->booterSplashDuration,
2839 gIOHibernateStats->trampolineDuration);
2840
2841 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, imageSize 0x%qx, image1Size 0x%qx, conflictCount %d, nextFree %x\n",
2842 gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size,
2843 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
bd504ef0 2844
39236c6e
A		 2845 if ((0 != (kIOHibernateModeSleep & gIOHibernateMode))
2846 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)))
2847 {
0c530ab8 2848 hibernate_page_list_discard(vars->page_list);
39236c6e 2849 }
0c530ab8 2850
6d2010ae
A		 2851 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
2852
2853 if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount)
2854 panic("handoff overflow");
2855
2856 IOHibernateHandoff * handoff;
2857 bool done = false;
2858 bool foundCryptData = false;
2859
2860 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
2861 !done;
2862 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
2863 {
2864// HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
2865 uint8_t * data = &handoff->data[0];
2866 switch (handoff->type)
2867 {
2868 case kIOHibernateHandoffTypeEnd:
2869 done = true;
2870 break;
2871
2872 case kIOHibernateHandoffTypeGraphicsInfo:
2873 bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
2874 break;
2875
2876 case kIOHibernateHandoffTypeCryptVars:
2877 if (cryptvars)
2878 {
2879 hibernate_cryptwakevars_t *
2880 wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
2881 bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
2882 }
2883 foundCryptData = true;
2884 bzero(data, handoff->bytecount);
2885 break;
2886
2887 case kIOHibernateHandoffTypeMemoryMap:
bd504ef0
A		 2888
2889 clock_get_uptime(&allTime);
2890
6d2010ae
A		 2891 hibernate_newruntime_map(data, handoff->bytecount,
2892 gIOHibernateCurrentHeader->systemTableOffset);
bd504ef0
A		 2893
2894 clock_get_uptime(&endTime);
2895
2896 SUB_ABSOLUTETIME(&endTime, &allTime);
2897 absolutetime_to_nanoseconds(endTime, &nsec);
2898
2899 HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL);
2900
6d2010ae
A		 2901 break;
2902
2903 case kIOHibernateHandoffTypeDeviceTree:
2904 {
2905// DTEntry chosen = NULL;
2906// HIBPRINT("DTLookupEntry %d\n", DTLookupEntry((const DTEntry) data, "/chosen", &chosen));
2907 }
2908 break;
2909
2910 default:
2911 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
2912 break;
2913 }
2914 }
2915 if (cryptvars && !foundCryptData)
2916 panic("hibernate handoff");
2917
316670eb
A		 2918 HIBPRINT("video %x %d %d %d status %x\n",
2919 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
2920 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus);
2921
2922 if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress)
3a60a9f5
A		 2923 {
2924 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
2925 * gIOHibernateGraphicsInfo->rowBytes);
2926 IOMapPages(kernel_map,
2927 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
2928 vars->videoMapSize, kIOMapInhibitCache );
2929 }
2930
316670eb
A		 2931 if (vars->videoMapSize)
2932 ProgressUpdate(gIOHibernateGraphicsInfo,
2933 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2934
0c530ab8 2935 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
39236c6e
A		 2936 uint8_t * compressed = src + page_size;
2937 uint8_t * scratch = compressed + page_size;
2938 uint32_t decoOffset;
3a60a9f5
A		 2939
2940 clock_get_uptime(&allTime);
6d2010ae
A		 2941 AbsoluteTime_to_scalar(&compTime) = 0;
2942 compBytes = 0;
3a60a9f5
A		 2943
2944 HIBLOG("IOHibernatePollerOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
2945 err = IOHibernatePollerOpen(vars->fileVars, kIOPolledAfterSleepState, 0);
39236c6e
A		 2946 clock_get_uptime(&startIOTime);
2947 endTime = startIOTime;
2948 SUB_ABSOLUTETIME(&endTime, &allTime);
2949 absolutetime_to_nanoseconds(endTime, &nsec);
2950 HIBLOG("IOHibernatePollerOpen(%x) %qd ms\n", err, nsec / 1000000ULL);
3a60a9f5 2951
3a60a9f5
A		 2952 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
2953
3a60a9f5
A		 2954 // kick off the read ahead
2955 vars->fileVars->io = false;
2956 vars->fileVars->bufferHalf = 0;
2957 vars->fileVars->bufferLimit = 0;
2958 vars->fileVars->lastRead = 0;
6d2010ae 2959 vars->fileVars->readEnd = gIOHibernateCurrentHeader->imageSize;
3a60a9f5 2960 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
6d2010ae
A		 2961 vars->fileVars->cryptBytes = 0;
2962 AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;
3a60a9f5 2963
6d2010ae 2964 err = IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
3a60a9f5
A		 2965 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2966 // --
2967
2968 HIBLOG("hibernate_machine_init reading\n");
2969
2970 uint32_t * header = (uint32_t *) src;
2971 sum = 0;
2972
6d2010ae 2973 while (kIOReturnSuccess == err)
3a60a9f5
A		 2974 {
2975 unsigned int count;
2976 unsigned int page;
2977 uint32_t tag;
2978 vm_offset_t ppnum, compressedSize;
2979
36401178
A		 2980 err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
2981 if (kIOReturnSuccess != err)
2982 break;
3a60a9f5
A		 2983
2984 ppnum = header[0];
2985 count = header[1];
2986
2987// HIBPRINT("(%x, %x)\n", ppnum, count);
2988
2989 if (!count)
2990 break;
2991
2992 for (page = 0; page < count; page++)
2993 {
36401178
A		 2994 err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
2995 if (kIOReturnSuccess != err)
2996 break;
3a60a9f5
A		 2997
2998 compressedSize = kIOHibernateTagLength & tag;
36401178
A		 2999 if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength))
3000 {
3001 err = kIOReturnIPCError;
3002 break;
3003 }
3004
39236c6e 3005 if (!compressedSize) bzero_phys(ptoa_64(ppnum), page_size);
3a60a9f5 3006 else
36401178 3007 {
39236c6e
A		 3008 err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
3009 if (kIOReturnSuccess != err) break;
3010 if (compressedSize < page_size)
3011 {
3012 decoOffset = page_size;
3013 clock_get_uptime(&startTime);
3014 WKdm_decompress_new((WK_word*) src, (WK_word*) compressed, (WK_word*) scratch, page_size);
3015 clock_get_uptime(&endTime);
3016 ADD_ABSOLUTETIME(&compTime, &endTime);
3017 SUB_ABSOLUTETIME(&compTime, &startTime);
3018 compBytes += page_size;
3019 }
3020 else decoOffset = 0;
3021
3022 sum += hibernate_sum_page((src + decoOffset), ppnum);
3023 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
3024 if (err)
3025 {
3026 HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
3027 break;
3028 }
36401178 3029 }
3a60a9f5
A		 3030
3031 ppnum++;
3032 pagesDone++;
0b4c1975 3033 pagesRead++;
3a60a9f5 3034
3a60a9f5
A		 3035 if (0 == (8191 & pagesDone))
3036 {
3037 clock_get_uptime(&endTime);
3038 SUB_ABSOLUTETIME(&endTime, &allTime);
3039 absolutetime_to_nanoseconds(endTime, &nsec);
3040 progressStamp = nsec / 750000000ULL;
3041 if (progressStamp != lastProgressStamp)
3042 {
3043 lastProgressStamp = progressStamp;
3044 HIBPRINT("pages %d (%d%%)\n", pagesDone,
3045 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
3046 }
3047 }
3048 }
3049 }
7ddcb079
A		 3050 if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages))
3051 err = kIOReturnLockedRead;
3a60a9f5 3052
36401178
A		 3053 if (kIOReturnSuccess != err)
3054 panic("Hibernate restore error %x", err);
3055
3a60a9f5 3056 gIOHibernateCurrentHeader->actualImage2Sum = sum;
db609669 3057 gIOHibernateCompression = gIOHibernateCurrentHeader->compression;
3a60a9f5
A		 3058
3059 if (vars->fileVars->io)
2d21ac55 3060 (void) IOHibernatePollerIODone(vars->fileVars, false);
3a60a9f5 3061
39236c6e
A		 3062 clock_get_uptime(&endIOTime);
3063
3a60a9f5
A		 3064 err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState);
3065
3a60a9f5 3066 clock_get_uptime(&endTime);
b0d623f7
A		 3067
3068 IOService::getPMRootDomain()->pmStatsRecordEvent(
3069 kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
3070 IOService::getPMRootDomain()->pmStatsRecordEvent(
3071 kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
3072
3a60a9f5
A		 3073 SUB_ABSOLUTETIME(&endTime, &allTime);
3074 absolutetime_to_nanoseconds(endTime, &nsec);
3075
39236c6e
A		 3076 SUB_ABSOLUTETIME(&endIOTime, &startIOTime);
3077 absolutetime_to_nanoseconds(endIOTime, &nsecIO);
3078
3079 gIOHibernateStats->kernelImageReadDuration = nsec / 1000000ULL;
3080 gIOHibernateStats->imagePages = pagesDone;
3081
3082 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ",
3083 pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize,
3084 nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0);
6d2010ae
A		 3085
3086 absolutetime_to_nanoseconds(compTime, &nsec);
3087 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
3088 compBytes,
3089 nsec / 1000000ULL,
3090 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
3091
3092 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
3093 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n",
3094 vars->fileVars->cryptBytes,
3095 nsec / 1000000ULL,
3096 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
3097
3098 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_NONE, pagesRead, pagesDone, 0, 0, 0);
3a60a9f5
A		 3099}
3100
3101/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
99c3a104 3102
39236c6e
A		 3103void IOHibernateSetWakeCapabilities(uint32_t capability)
3104{
3105 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
3106 {
3107 gIOHibernateStats->wakeCapability = capability;
3108 }
3109}
3110
3111/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
3112
99c3a104
A		 3113void IOHibernateSystemRestart(void)
3114{
3115 static uint8_t noteStore[32] __attribute__((aligned(32)));
3116 IORegistryEntry * regEntry;
3117 const OSSymbol * sym;
3118 OSData * noteProp;
3119 OSData * data;
3120 uintptr_t * smcVars;
3121 uint8_t * smcBytes;
3122 size_t len;
3123 addr64_t element;
3124
3125 data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
3126 if (!data) return;
3127
3128 smcVars = (typeof(smcVars)) data->getBytesNoCopy();
3129 smcBytes = (typeof(smcBytes)) smcVars[1];
3130 len = smcVars[0];
3131 if (len > sizeof(noteStore)) len = sizeof(noteStore);
3132 noteProp = OSData::withCapacity(3 * sizeof(element));
3133 if (!noteProp) return;
3134 element = len;
3135 noteProp->appendBytes(&element, sizeof(element));
3136 element = crc32(0, smcBytes, len);
3137 noteProp->appendBytes(&element, sizeof(element));
3138
3139 bcopy(smcBytes, noteStore, len);
3140 element = (addr64_t) &noteStore[0];
3141 element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
3142 noteProp->appendBytes(&element, sizeof(element));
3143
3144 if (!gIOOptionsEntry)
3145 {
3146 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
3147 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
3148 if (regEntry && !gIOOptionsEntry)
3149 regEntry->release();
3150 }
3151
3152 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
3153 if (gIOOptionsEntry && sym) gIOOptionsEntry->setProperty(sym, noteProp);
3154 if (noteProp) noteProp->release();
3155 if (sym) sym->release();
3156}
3157
3158
3159
mirror of XNU