]> git.saurik.com Git - apple/xnu.git/blame - iokit/Kernel/IOHibernateIO.cpp
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-792.13.8.tar.gz
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
CommitLineData
3a60a9f5
A		 1/*
2 * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
3 *
8ad349bb 4 * @APPLE_LICENSE_OSREFERENCE_HEADER_START@
3a60a9f5 5 *
8ad349bb
A		 6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the
10 * License may not be used to create, or enable the creation or
11 * redistribution of, unlawful or unlicensed copies of an Apple operating
12 * system, or to circumvent, violate, or enable the circumvention or
13 * violation of, any terms of an Apple operating system software license
14 * agreement.
15 *
16 * Please obtain a copy of the License at
17 * http://www.opensource.apple.com/apsl/ and read it before using this
18 * file.
19 *
20 * The Original Code and all software distributed under the License are
21 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
22 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
23 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
24 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
25 * Please see the License for the specific language governing rights and
26 * limitations under the License.
27 *
28 * @APPLE_LICENSE_OSREFERENCE_HEADER_END@
3a60a9f5
A		 29 */
30
31
32/*
33
34Sleep:
35
36- PMRootDomain calls IOHibernateSystemSleep() before system sleep
37(devices awake, normal execution context)
38- IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
39 grabs its extents and searches for a polling driver willing to work with that IOMedia.
40 The BSD code makes an ioctl to the storage driver to get the partition base offset to
41 the disk, and other ioctls to get the transfer constraints
42 If successful, the file is written to make sure its initially not bootable (in case of
43 later failure) and nvram set to point to the first block of the file. (Has to be done
44 here so blocking is possible in nvram support).
45 hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
46 page out any pages it wants to (currently zero, but probably some percentage of memory).
47 Its assumed just allocating pages will cause the VM system to naturally select the best
48 pages for eviction. It also copies processor flags needed for the restore path and sets
49 a flag in the boot processor proc info.
50 gIOHibernateState = kIOHibernateStateHibernating.
51- Regular sleep progresses - some drivers may inspect the root domain property
52 kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
53 as usual but leaves motherboard I/O on.
54- Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
55 at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
56 all ppc RC bits out of the hash table and caches into the mapping structures.
57- hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
58 hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
59 All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
60 by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
61 The image header and block list are written. The header includes the second file extent so
62 only the header block is needed to read the file, regardless of filesystem.
63 The kernel section "__HIB" is written uncompressed to the image. This section of code and data
64 (only) is used to decompress the image during wake/boot.
65 Some additional pages are removed from the bitmaps - the buffers used for hibernation.
66 The bitmaps are written to the image.
67 More areas are removed from the bitmaps (after they have been written to the image) - the
68 section "__HIB" pages and interrupt stack.
69 Each wired page is compressed and written and then each non-wired page. Compression and
70 disk writes are in parallel.
71 The image header is written to the start of the file and the polling driver closed.
72 The machine powers down (or sleeps).
73
74Boot/Wake:
75
76- BootX sees the boot-image nvram variable containing the device and block number of the image,
77 reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
78- BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
79 in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
80 that is in the kernel's __HIB section.
81- BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
82 hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
83 only code & data in that section is safe to call since all the other wired pages are still
84 compressed in the image.
85- hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
86 It uses the bitmaps to work out which pages can be uncompressed from the image to their final
87 location directly, and copies those that can't to interim free pages. When the image has been
88 completed, the copies are uncompressed, overwriting the wired image pages.
89 hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
90 is used to get pages into place for 64bit.
91- the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
92 changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
93 are removed from the software strutures, and the hash table is reinitialized.
94- After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
95 of memory, using the polled mode driver, before other threads can run or any devices are turned on.
96 This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
97 for the remaining non wired pages.
98- The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
99 (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
100 is closed via bsd.
101
102Polled Mode I/O:
103
104IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
105registry, specifying an object of calls IOPolledInterface.
106
107Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
108partition) that the image is going to live, looking for polled interface properties. If it finds
109one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
110interfaces found are kept in an ordered list.
111
112There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
113few different contexts things happen in:
114
115- there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
116up and running) and after wake - this is safe to allocate memory and do anything. The device
117ignores sleep requests from that point since its a waste of time if it goes to sleep and
118immediately wakes back up for the image write.
119
120- there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
121immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
122from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
123used to flush and set the disk to sleep.
124
125- there is an Open/Close (AfterSleep) pair made around the image read operations that happen
126immediately after sleep. These can't block or allocate memory. This is happening after the platform
127expert has woken the low level bits of the system, but most of the I/O system has not. There is only
128one thread running.
129
130For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
131(prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
132the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
133Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
134that is called for the hardware to check for events, and complete the I/O via the callback.
135The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
136to restrict I/O ops.
137*/
138
139#include <sys/systm.h>
140
141#include <IOKit/IOWorkLoop.h>
142#include <IOKit/IOCommandGate.h>
143#include <IOKit/IOTimerEventSource.h>
144#include <IOKit/IOPlatformExpert.h>
145#include <IOKit/IOKitDebug.h>
146#include <IOKit/IOTimeStamp.h>
147#include <IOKit/pwr_mgt/RootDomain.h>
148#include <IOKit/pwr_mgt/IOPMPrivate.h>
149#include <IOKit/IOMessage.h>
150#include <IOKit/IODeviceTreeSupport.h>
151#include <IOKit/IOBSD.h>
152#include "RootDomainUserClient.h"
153#include <IOKit/pwr_mgt/IOPowerConnection.h>
154#include "IOPMPowerStateQueue.h"
155#include <IOKit/IOBufferMemoryDescriptor.h>
156#include <crypto/aes.h>
157
158#include <sys/uio.h>
159#include <sys/conf.h>
160#include <sys/stat.h>
161#include <sys/fcntl.h> // (FWRITE, ...)
3a60a9f5 162#include <sys/sysctl.h>
3a60a9f5
A		 163
164#include <IOKit/IOHibernatePrivate.h>
165#include <IOKit/IOPolledInterface.h>
166#include <IOKit/IONVRAM.h>
167#include "IOHibernateInternal.h"
168#include "WKdm.h"
169#include "IOKitKernelInternal.h"
170
171/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
172
173OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject);
174
175OSMetaClassDefineReservedUnused(IOPolledInterface, 0);
176OSMetaClassDefineReservedUnused(IOPolledInterface, 1);
177OSMetaClassDefineReservedUnused(IOPolledInterface, 2);
178OSMetaClassDefineReservedUnused(IOPolledInterface, 3);
179OSMetaClassDefineReservedUnused(IOPolledInterface, 4);
180OSMetaClassDefineReservedUnused(IOPolledInterface, 5);
181OSMetaClassDefineReservedUnused(IOPolledInterface, 6);
182OSMetaClassDefineReservedUnused(IOPolledInterface, 7);
183OSMetaClassDefineReservedUnused(IOPolledInterface, 8);
184OSMetaClassDefineReservedUnused(IOPolledInterface, 9);
185OSMetaClassDefineReservedUnused(IOPolledInterface, 10);
186OSMetaClassDefineReservedUnused(IOPolledInterface, 11);
187OSMetaClassDefineReservedUnused(IOPolledInterface, 12);
188OSMetaClassDefineReservedUnused(IOPolledInterface, 13);
189OSMetaClassDefineReservedUnused(IOPolledInterface, 14);
190OSMetaClassDefineReservedUnused(IOPolledInterface, 15);
191
192/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
193
194extern uint32_t gIOHibernateState;
195uint32_t gIOHibernateMode;
196static char gIOHibernateBootSignature[256+1];
197static char gIOHibernateFilename[MAXPATHLEN+1];
198static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
199uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms)
200
201static IODTNVRAM * gIOOptionsEntry;
202static IORegistryEntry * gIOChosenEntry;
5d5c5d0d
A		 203#ifdef __i386__
204static const OSSymbol * gIOCreateEFIDevicePathSymbol;
205#endif
3a60a9f5
A		 206
207static IOPolledFileIOVars gFileVars;
208static IOHibernateVars gIOHibernateVars;
209static struct kern_direct_file_io_ref_t * gIOHibernateFileRef;
210static hibernate_cryptvars_t gIOHibernateCryptWakeContext;
211
212/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
213
214enum { kXPRamAudioVolume = 8 };
215enum { kDefaultIOSize = 128 * 1024 };
216enum { kVideoMapSize = 32 * 1024 * 1024 };
217
218#ifndef kIOMediaPreferredBlockSizeKey
219#define kIOMediaPreferredBlockSizeKey "Preferred Block Size"
220#endif
221
222#ifndef kIOBootPathKey
223#define kIOBootPathKey "bootpath"
224#endif
225#ifndef kIOSelectedBootDeviceKey
226#define kIOSelectedBootDeviceKey "boot-device"
227#endif
228
229
230enum { kIOHibernateMinPollersNeeded = 2 };
231
232/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
233
234// copy from phys addr to MD
235
236static IOReturn
237IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
238 IOByteCount offset, addr64_t bytes, IOByteCount length)
239{
240 addr64_t srcAddr = bytes;
241 IOByteCount remaining;
242
243 remaining = length = min(length, md->getLength() - offset);
244 while (remaining) { // (process another target segment?)
245 addr64_t dstAddr64;
246 IOByteCount dstLen;
247
248 dstAddr64 = md->getPhysicalSegment64(offset, &dstLen);
249 if (!dstAddr64)
250 break;
251
252 // Clip segment length to remaining
253 if (dstLen > remaining)
254 dstLen = remaining;
255
256#if 1
257 bcopy_phys(srcAddr, dstAddr64, dstLen);
258#else
259 copypv(srcAddr, dstAddr64, dstLen,
260 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
261#endif
262 srcAddr += dstLen;
263 offset += dstLen;
264 remaining -= dstLen;
265 }
266
267 assert(!remaining);
268
269 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
270}
271
272// copy from MD to phys addr
273
274static IOReturn
275IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
276 IOByteCount offset, addr64_t bytes, IOByteCount length)
277{
278 addr64_t dstAddr = bytes;
279 IOByteCount remaining;
280
281 remaining = length = min(length, md->getLength() - offset);
282 while (remaining) { // (process another target segment?)
283 addr64_t srcAddr64;
284 IOByteCount dstLen;
285
286 srcAddr64 = md->getPhysicalSegment64(offset, &dstLen);
287 if (!srcAddr64)
288 break;
289
290 // Clip segment length to remaining
291 if (dstLen > remaining)
292 dstLen = remaining;
293
294#if 1
295 bcopy_phys(srcAddr64, dstAddr, dstLen);
296#else
297 copypv(srcAddr, dstAddr64, dstLen,
298 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
299#endif
300 dstAddr += dstLen;
301 offset += dstLen;
302 remaining -= dstLen;
303 }
304
305 assert(!remaining);
306
307 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
308}
309
310/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
311
312void
313hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
314 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
315{
316 count += ppnum;
317 switch (kind)
318 {
319 case kIOHibernatePageStateUnwiredSave:
320 // unwired save
321 for (; ppnum < count; ppnum++)
322 {
323 hibernate_page_bitset(page_list, FALSE, ppnum);
324 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
325 }
326 break;
327 case kIOHibernatePageStateWiredSave:
328 // wired save
329 for (; ppnum < count; ppnum++)
330 {
331 hibernate_page_bitset(page_list, FALSE, ppnum);
332 hibernate_page_bitset(page_list_wired, FALSE, ppnum);
333 }
334 break;
335 case kIOHibernatePageStateFree:
336 // free page
337 for (; ppnum < count; ppnum++)
338 {
339 hibernate_page_bitset(page_list, TRUE, ppnum);
340 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
341 }
342 break;
343 default:
344 panic("hibernate_set_page_state");
345 }
346}
347
348static vm_offset_t
5d5c5d0d 349hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
3a60a9f5 350{
5d5c5d0d
A		 351 uint32_t page = *pPage;
352 uint32_t count;
353 hibernate_bitmap_t * bitmap;
3a60a9f5 354
5d5c5d0d
A		 355 while ((bitmap = hibernate_page_bitmap_pin(list, &page)))
356 {
357 count = hibernate_page_bitmap_count(bitmap, TRUE, page);
358 if (!count)
359 break;
360 page += count;
361 if (page <= bitmap->last_page)
362 break;
363 }
8ad349bb 364
5d5c5d0d
A		 365 *pPage = page;
366 if (bitmap)
367 count = hibernate_page_bitmap_count(bitmap, FALSE, page);
368 else
369 count = 0;
3a60a9f5
A		 370
371 return (count);
372}
373
374/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
375
376static IOReturn
377IOHibernatePollerProbe(IOPolledFileIOVars * vars, IOService * target)
378{
379 IOReturn err = kIOReturnError;
380 int32_t idx;
381 IOPolledInterface * poller;
382
383 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
384 {
385 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
386 err = poller->probe(target);
387 if (err)
388 {
389 HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
390 break;
391 }
392 }
393
394 return (err);
395}
396
397static IOReturn
398IOHibernatePollerOpen(IOPolledFileIOVars * vars, uint32_t state, IOMemoryDescriptor * md)
399{
400 IOReturn err = kIOReturnError;
401 int32_t idx;
402 IOPolledInterface * poller;
403
404 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
405 {
406 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
407 err = poller->open(state, md);
408 if (err)
409 {
410 HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
411 break;
412 }
413 }
414
415 return (err);
416}
417
418static IOReturn
419IOHibernatePollerClose(IOPolledFileIOVars * vars, uint32_t state)
420{
421 IOReturn err = kIOReturnError;
422 int32_t idx;
423 IOPolledInterface * poller;
424
425 for (idx = 0;
426 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
427 idx++)
428 {
429 err = poller->close(state);
430 if (err)
431 HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
432 }
433
434 return (err);
435}
436
437static void
438IOHibernatePollerIOComplete(void * target,
439 void * parameter,
440 IOReturn status,
441 UInt64 actualByteCount)
442{
443 IOPolledFileIOVars * vars = (IOPolledFileIOVars *) parameter;
444
445 vars->ioStatus = status;
446}
447
448static IOReturn
449IOHibernatePollerIO(IOPolledFileIOVars * vars,
450 uint32_t operation, uint32_t bufferOffset,
451 uint64_t deviceOffset, uint64_t length)
452{
453
454 IOReturn err = kIOReturnError;
455 IOPolledInterface * poller;
456 IOPolledCompletion completion;
457
458 completion.target = 0;
459 completion.action = &IOHibernatePollerIOComplete;
460 completion.parameter = vars;
461
462 vars->ioStatus = -1;
463
464 poller = (IOPolledInterface *) vars->pollers->getObject(0);
465 err = poller->startIO(operation, bufferOffset, deviceOffset + vars->block0, length, completion);
466 if (err)
467 HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
468
469 return (err);
470}
471
472static IOReturn
473IOHibernatePollerIODone(IOPolledFileIOVars * vars)
474{
475 IOReturn err = kIOReturnError;
476 int32_t idx;
477 IOPolledInterface * poller;
478
479 while (-1 == vars->ioStatus)
480 {
481 for (idx = 0;
482 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
483 idx++)
484 {
485 err = poller->checkForWork();
486 if (err)
487 HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
488 }
489 }
490
491 if (kIOReturnSuccess != vars->ioStatus)
492 HIBLOG("IOPolledInterface::ioStatus 0x%x\n", vars->ioStatus);
493
494 return (vars->ioStatus);
495}
496
497IOReturn
498IOPolledInterface::checkAllForWork(void)
499{
500 IOReturn err = kIOReturnNotReady;
501 int32_t idx;
502 IOPolledInterface * poller;
503
504 IOHibernateVars * vars = &gIOHibernateVars;
505
506 if (!vars->fileVars || !vars->fileVars->pollers)
507 return (err);
508
509 for (idx = 0;
510 (poller = (IOPolledInterface *) vars->fileVars->pollers->getObject(idx));
511 idx++)
512 {
513 err = poller->checkForWork();
514 if (err)
515 HIBLOG("IOPolledInterface::checkAllForWork[%d] 0x%x\n", idx, err);
516 }
517
518 return (err);
519}
520
521/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
522
523struct _OpenFileContext
524{
525 OSData * extents;
526 uint64_t size;
527};
528
529static void
530file_extent_callback(void * ref, uint64_t start, uint64_t length)
531{
532 _OpenFileContext * ctx = (_OpenFileContext *) ref;
533 IOPolledFileExtent extent;
534
535 extent.start = start;
536 extent.length = length;
537
538 ctx->extents->appendBytes(&extent, sizeof(extent));
539 ctx->size += length;
540}
541
542IOReturn
543IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer,
544 IOPolledFileIOVars ** fileVars, OSData ** fileExtents,
545 OSData ** imagePath)
546{
547 IOReturn err = kIOReturnError;
548 IOPolledFileIOVars * vars;
549 _OpenFileContext ctx;
550 OSData * extentsData;
551 OSNumber * num;
552 IORegistryEntry * part = 0;
553 OSDictionary * matching;
554 OSIterator * iter;
555 dev_t hibernate_image_dev;
556 uint64_t maxiobytes;
557
558 vars = &gFileVars;
559 do
560 {
561 HIBLOG("sizeof(IOHibernateImageHeader) == %ld\n", sizeof(IOHibernateImageHeader));
562 if (sizeof(IOHibernateImageHeader) != 512)
563 continue;
564
565 vars->io = false;
566 vars->buffer = (uint8_t *) ioBuffer->getBytesNoCopy();
567 vars->bufferHalf = 0;
568 vars->bufferOffset = 0;
569 vars->bufferSize = ioBuffer->getLength() >> 1;
570
571 extentsData = OSData::withCapacity(32);
572
573 ctx.extents = extentsData;
574 ctx.size = 0;
575 vars->fileRef = kern_open_file_for_direct_io(filename,
576 &file_extent_callback, &ctx,
577 &hibernate_image_dev,
578 &vars->block0,
579 &maxiobytes);
580 if (!vars->fileRef)
581 {
582 err = kIOReturnNoSpace;
583 break;
584 }
585 HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx\n", filename, ctx.size,
586 vars->block0, maxiobytes);
587 if (ctx.size < 1*1024*1024) // check against image size estimate!
588 {
589 err = kIOReturnNoSpace;
590 break;
591 }
592
593 if (maxiobytes < vars->bufferSize)
594 vars->bufferSize = maxiobytes;
595
596 vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
597
598 matching = IOService::serviceMatching("IOMedia");
599 num = OSNumber::withNumber(major(hibernate_image_dev), 32);
600 matching->setObject(kIOBSDMajorKey, num);
601 num->release();
602 num = OSNumber::withNumber(minor(hibernate_image_dev), 32);
603 matching->setObject(kIOBSDMinorKey, num);
604 num->release();
605 iter = IOService::getMatchingServices(matching);
606 matching->release();
607 if (iter)
608 {
609 part = (IORegistryEntry *) iter->getNextObject();
610 part->retain();
611 iter->release();
612 }
613
614 int minor, major;
615 IORegistryEntry * next;
616 IORegistryEntry * child;
617 OSData * data;
618
619 num = (OSNumber *) part->getProperty(kIOBSDMajorKey);
620 if (!num)
621 break;
622 major = num->unsigned32BitValue();
623 num = (OSNumber *) part->getProperty(kIOBSDMinorKey);
624 if (!num)
625 break;
626 minor = num->unsigned32BitValue();
627
628 hibernate_image_dev = makedev(major, minor);
629
630 vars->pollers = OSArray::withCapacity(4);
631 if (!vars->pollers)
632 break;
633
634 vars->blockSize = 512;
635 next = part;
636 do
637 {
638 IOPolledInterface * poller;
c0fea474
A		 639 OSObject * obj;
640
641 obj = next->getProperty(kIOPolledInterfaceSupportKey);
642 if (kOSBooleanFalse == obj)
643 {
644 vars->pollers->flushCollection();
645 break;
646 }
647 else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
3a60a9f5
A		 648 vars->pollers->setObject(poller);
649 if ((num = OSDynamicCast(OSNumber, next->getProperty(kIOMediaPreferredBlockSizeKey))))
650 vars->blockSize = num->unsigned32BitValue();
651 child = next;
652 }
653 while ((next = child->getParentEntry(gIOServicePlane))
654 && child->isParent(next, gIOServicePlane, true));
655
656 HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n",
657 major, minor, vars->blockSize, vars->pollers->getCount());
658 if (vars->pollers->getCount() < kIOHibernateMinPollersNeeded)
659 continue;
660
661 err = IOHibernatePollerProbe(vars, (IOService *) part);
662 if (kIOReturnSuccess != err)
663 break;
664
665 err = IOHibernatePollerOpen(vars, kIOPolledPreflightState, ioBuffer);
666 if (kIOReturnSuccess != err)
667 break;
668
669 *fileVars = vars;
670 *fileExtents = extentsData;
671
672 // make imagePath
3a60a9f5 673
5d5c5d0d 674 if ((extentsData->getLength() >= sizeof(IOPolledFileExtent)))
3a60a9f5 675 {
5d5c5d0d
A		 676 char str2[24];
677
678#if __i386__
679 if (!gIOCreateEFIDevicePathSymbol)
680 gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
681
682 sprintf(str2, "%qx", vars->extentMap[0]);
683
684 err = IOService::getPlatform()->callPlatformFunction(
685 gIOCreateEFIDevicePathSymbol, false,
686 (void *) part, (void *) str2, (void *) true,
687 (void *) &data);
688#else
689 char str1[256];
690 int len = sizeof(str1);
691
692 if (!part->getPath(str1, &len, gIODTPlane))
693 err = kIOReturnNotFound;
694 else
695 {
696 sprintf(str2, ",%qx", vars->extentMap[0]);
697 // (strip the plane name)
698 char * tail = strchr(str1, ':');
699 if (!tail)
700 tail = str1 - 1;
701 data = OSData::withBytes(tail + 1, strlen(tail + 1));
702 data->appendBytes(str2, strlen(str2));
703 }
704#endif
705 if (kIOReturnSuccess == err)
3a60a9f5 706 *imagePath = data;
5d5c5d0d
A		 707 else
708 HIBLOG("error 0x%x getting path\n", err);
3a60a9f5
A		 709 }
710 }
711 while (false);
712
713 if (kIOReturnSuccess != err)
714 {
715 HIBLOG("error 0x%x opening hibernation file\n", err);
716 if (vars->fileRef)
717 kern_close_file_for_direct_io(vars->fileRef);
718 }
719
720 if (part)
721 part->release();
722
723 return (err);
724}
725
726IOReturn
727IOPolledFileClose( IOPolledFileIOVars * vars )
728{
729 if (vars->pollers)
730 {
731 IOHibernatePollerClose(vars, kIOPolledPostflightState);
732 vars->pollers->release();
733 }
734
735 gIOHibernateFileRef = vars->fileRef;
736
737 bzero(vars, sizeof(IOPolledFileIOVars));
738
739 return (kIOReturnSuccess);
740}
741
742static IOReturn
743IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
744{
745 IOPolledFileExtent * extentMap;
746
747 extentMap = vars->extentMap;
748
749 vars->position = position;
750
751 while (position >= extentMap->length)
752 {
753 position -= extentMap->length;
754 extentMap++;
755 }
756
757 vars->currentExtent = extentMap;
758 vars->extentRemaining = extentMap->length - position;
759 vars->extentPosition = vars->position - position;
760
761 if (vars->bufferSize <= vars->extentRemaining)
762 vars->bufferLimit = vars->bufferSize;
763 else
764 vars->bufferLimit = vars->extentRemaining;
765
766 return (kIOReturnSuccess);
767}
768
769static IOReturn
770IOPolledFileWrite(IOPolledFileIOVars * vars,
771 const uint8_t * bytes, IOByteCount size,
772 hibernate_cryptvars_t * cryptvars)
773{
774 IOReturn err = kIOReturnSuccess;
775 IOByteCount copy;
776 bool flush = false;
777
778 do
779 {
780 if (!bytes && !size)
781 {
782 // seek to end of block & flush
783 size = vars->position & (vars->blockSize - 1);
784 if (size)
785 size = vars->blockSize - size;
786 flush = true;
787 // use some garbage for the fill
788 bytes = vars->buffer + vars->bufferOffset;
789 }
790
791 copy = vars->bufferLimit - vars->bufferOffset;
792 if (copy > size)
793 copy = size;
794 else
795 flush = true;
796
797 if (bytes)
798 {
799 bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
800 bytes += copy;
801 }
802 else
803 bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
804
805 size -= copy;
806 vars->bufferOffset += copy;
807 vars->position += copy;
808
809 if (flush && vars->bufferOffset)
810 {
811 uint64_t offset = (vars->position - vars->bufferOffset
812 - vars->extentPosition + vars->currentExtent->start);
813 uint32_t length = (vars->bufferOffset);
814
815 if (cryptvars && vars->encryptStart && (vars->position > vars->encryptStart))
816 {
817 uint32_t encryptLen, encryptStart;
818 encryptLen = vars->position - vars->encryptStart;
819 if (encryptLen > length)
820 encryptLen = length;
821 encryptStart = length - encryptLen;
822
823 // encrypt the buffer
824 aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
825 &cryptvars->aes_iv[0],
826 encryptLen / AES_BLOCK_SIZE,
827 vars->buffer + vars->bufferHalf + encryptStart,
828 &cryptvars->ctx.encrypt);
829 // save initial vector for following encrypts
830 bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
831 &cryptvars->aes_iv[0],
832 AES_BLOCK_SIZE);
833 }
834
835 if (vars->io)
836 {
837 err = IOHibernatePollerIODone(vars);
838 if (kIOReturnSuccess != err)
839 break;
840 }
841
842if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
843//if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
844
845 err = IOHibernatePollerIO(vars, kIOPolledWrite, vars->bufferHalf, offset, length);
846 if (kIOReturnSuccess != err)
847 break;
848 vars->io = true;
849
850 vars->extentRemaining -= vars->bufferOffset;
851 if (!vars->extentRemaining)
852 {
853 vars->currentExtent++;
854 vars->extentRemaining = vars->currentExtent->length;
855 vars->extentPosition = vars->position;
856 if (!vars->extentRemaining)
857 {
858 err = kIOReturnOverrun;
859 break;
860 }
861 }
862
863 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
864 vars->bufferOffset = 0;
865 if (vars->bufferSize <= vars->extentRemaining)
866 vars->bufferLimit = vars->bufferSize;
867 else
868 vars->bufferLimit = vars->extentRemaining;
869
870 flush = false;
871 }
872 }
873 while (size);
874
875 return (err);
876}
877
878static IOReturn
879IOPolledFileRead(IOPolledFileIOVars * vars,
880 uint8_t * bytes, IOByteCount size,
881 hibernate_cryptvars_t * cryptvars)
882{
883 IOReturn err = kIOReturnSuccess;
884 IOByteCount copy;
885
886// bytesWritten += size;
887
888 do
889 {
890 copy = vars->bufferLimit - vars->bufferOffset;
891 if (copy > size)
892 copy = size;
893
894 if (bytes)
895 {
896 bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
897 bytes += copy;
898 }
899 size -= copy;
900 vars->bufferOffset += copy;
901// vars->position += copy;
902
903 if (vars->bufferOffset == vars->bufferLimit)
904 {
905 if (vars->io)
906 {
907 err = IOHibernatePollerIODone(vars);
908 if (kIOReturnSuccess != err)
909 break;
910 }
911 else
912 cryptvars = 0;
913
914if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
915
916 vars->position += vars->lastRead;
917 vars->extentRemaining -= vars->lastRead;
918 vars->bufferLimit = vars->lastRead;
919
920 if (!vars->extentRemaining)
921 {
922 vars->currentExtent++;
923 vars->extentRemaining = vars->currentExtent->length;
924 vars->extentPosition = vars->position;
925 if (!vars->extentRemaining)
926 {
927 err = kIOReturnOverrun;
928 break;
929 }
930 }
931
932 if (vars->extentRemaining <= vars->bufferSize)
933 vars->lastRead = vars->extentRemaining;
934 else
935 vars->lastRead = vars->bufferSize;
936
937 uint64_t offset = (vars->position
938 - vars->extentPosition + vars->currentExtent->start);
939 uint64_t length = (vars->lastRead);
940
941//if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
942
943 err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length);
944 if (kIOReturnSuccess != err)
945 break;
946 vars->io = true;
947
948 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
949 vars->bufferOffset = 0;
950
951 if (cryptvars)
952 {
953 uint8_t thisVector[AES_BLOCK_SIZE];
954 // save initial vector for following decrypts
955 bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
956 bcopy(vars->buffer + vars->bufferHalf + vars->lastRead - AES_BLOCK_SIZE,
957 &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
958 // decrypt the buffer
959 aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
960 &thisVector[0],
961 vars->lastRead / AES_BLOCK_SIZE,
962 vars->buffer + vars->bufferHalf,
963 &cryptvars->ctx.decrypt);
964 }
965 }
966 }
967 while (size);
968
969 return (err);
970}
971
972/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
973
974IOReturn
975IOHibernateSystemSleep(void)
976{
977 IOReturn err;
978 OSData * data;
979 OSObject * obj;
980 OSString * str;
981 OSNumber * num;
982
983 IOHibernateVars * vars = &gIOHibernateVars;
984
985 if (vars->fileVars && vars->fileVars->fileRef)
986 // already on the way down
987 return (kIOReturnSuccess);
988
989 gIOHibernateState = kIOHibernateStateInactive;
990
991 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateModeKey)))
992 {
993 if ((num = OSDynamicCast(OSNumber, obj)))
994 gIOHibernateMode = num->unsigned32BitValue();
995 if (kIOHibernateModeSleep & gIOHibernateMode)
996 // default to discard clean for safe sleep
997 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
998 | kIOHibernateModeDiscardCleanActive);
999
1000 obj->release();
1001 }
1002 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeRatioKey)))
1003 {
1004 if ((num = OSDynamicCast(OSNumber, obj)))
1005 gIOHibernateFreeRatio = num->unsigned32BitValue();
1006 obj->release();
1007 }
1008 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeTimeKey)))
1009 {
1010 if ((num = OSDynamicCast(OSNumber, obj)))
1011 gIOHibernateFreeTime = num->unsigned32BitValue();
1012 obj->release();
1013 }
1014 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
1015 {
1016 if ((str = OSDynamicCast(OSString, obj)))
1017 strcpy(gIOHibernateFilename, str->getCStringNoCopy());
1018 obj->release();
1019 }
1020
1021 if (!gIOHibernateMode || !gIOHibernateFilename[0])
1022 return (kIOReturnUnsupported);
1023
1024 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
1025
1026 do
1027 {
5d5c5d0d
A		 1028 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1029 4 * page_size, page_size);
1030 vars->ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1031 2 * kDefaultIOSize, page_size);
3a60a9f5
A		 1032
1033 if (!vars->srcBuffer || !vars->ioBuffer)
1034 {
1035 err = kIOReturnNoMemory;
1036 break;
1037 }
1038
1039 err = IOPolledFileOpen(gIOHibernateFilename, vars->ioBuffer,
1040 &vars->fileVars, &vars->fileExtents, &data);
1041 if (KERN_SUCCESS != err)
1042 {
1043 HIBLOG("IOPolledFileOpen(%x)\n", err);
1044 break;
1045 }
1046 if (vars->fileVars->fileRef)
1047 {
1048 // invalidate the image file
1049 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1050 int err = kern_write_file(vars->fileVars->fileRef, 0,
1051 (caddr_t) gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
1052 if (KERN_SUCCESS != err)
1053 HIBLOG("kern_write_file(%d)\n", err);
1054 }
1055
1056 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
1057
1058 boolean_t encryptedswap;
1059 err = hibernate_setup(gIOHibernateCurrentHeader,
1060 gIOHibernateFreeRatio, gIOHibernateFreeTime,
1061 &vars->page_list, &vars->page_list_wired, &encryptedswap);
1062 if (KERN_SUCCESS != err)
1063 {
1064 HIBLOG("hibernate_setup(%d)\n", err);
1065 break;
1066 }
1067
1068 if (encryptedswap)
1069 gIOHibernateMode ^= kIOHibernateModeEncrypt;
1070
1071 vars->videoAllocSize = kVideoMapSize;
1072 if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize))
1073 vars->videoMapping = 0;
1074
1075 // generate crypt keys
1076 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
1077 vars->wiredCryptKey[i] = random();
1078 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
1079 vars->cryptKey[i] = random();
1080
1081 // set nvram
1082
1083 IORegistryEntry * regEntry;
1084 if (!gIOOptionsEntry)
1085 {
1086 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
1087 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
1088 if (regEntry && !gIOOptionsEntry)
1089 regEntry->release();
1090 }
1091 if (!gIOChosenEntry)
1092 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1093
1094 if (gIOOptionsEntry)
1095 {
1096 const OSSymbol * sym;
3a60a9f5
A		 1097
1098 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1099 if (sym)
1100 {
1101 gIOOptionsEntry->setProperty(sym, data);
1102 sym->release();
1103 }
1104 data->release();
1105
5d5c5d0d
A		 1106#ifdef __ppc__
1107 size_t len;
1108 char valueString[16];
1109
3a60a9f5
A		 1110 vars->saveBootDevice = gIOOptionsEntry->copyProperty(kIOSelectedBootDeviceKey);
1111 if (gIOChosenEntry)
1112 {
1113 OSData * bootDevice = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOBootPathKey));
1114 if (bootDevice)
1115 {
1116 sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey);
1117 OSString * str2 = OSString::withCStringNoCopy((const char *) bootDevice->getBytesNoCopy());
1118 if (sym && str2)
1119 gIOOptionsEntry->setProperty(sym, str2);
1120 if (sym)
1121 sym->release();
1122 if (str2)
1123 str2->release();
1124 }
3a60a9f5
A		 1125 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMemorySignatureKey));
1126 if (data)
1127 {
1128 vars->haveFastBoot = true;
1129
1130 len = sprintf(valueString, "0x%lx", *((UInt32 *)data->getBytesNoCopy()));
1131 data = OSData::withBytes(valueString, len + 1);
1132 sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey);
1133 if (sym && data)
1134 gIOOptionsEntry->setProperty(sym, data);
1135 if (sym)
1136 sym->release();
1137 if (data)
1138 data->release();
1139 }
1140 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
1141 if (data)
1142 gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
1143 }
5d5c5d0d
A		 1144#endif /* __ppc__ */
1145#ifdef __i386__
1146 struct AppleRTCHibernateVars
1147 {
1148 uint8_t signature[4];
1149 uint32_t revision;
1150 uint8_t booterSignature[20];
1151 uint8_t wiredCryptKey[16];
1152 };
1153 AppleRTCHibernateVars rtcVars;
1154
1155 rtcVars.signature[0] = 'A';
1156 rtcVars.signature[1] = 'A';
1157 rtcVars.signature[2] = 'P';
1158 rtcVars.signature[3] = 'L';
1159 rtcVars.revision = 1;
1160 bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
1161 if (gIOHibernateBootSignature[0])
1162 {
1163 char c;
1164 uint8_t value = 0;
1165 for (uint32_t i = 0;
1166 (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1));
1167 i++)
1168 {
1169 if (c >= 'a')
1170 c -= 'a' - 10;
1171 else if (c >= 'A')
1172 c -= 'A' - 10;
1173 else if (c >= '0')
1174 c -= '0';
1175 else
1176 continue;
1177 value = (value << 4) | c;
1178 if (i & 1)
1179 rtcVars.booterSignature[i >> 1] = value;
1180 }
1181 }
1182 data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
1183 if (data)
1184 {
1185 IOService::getPMRootDomain()->setProperty(kIOHibernateRTCVariablesKey, data);
1186 data->release();
1187 }
1188 if (gIOChosenEntry)
1189 {
1190 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
1191 if (data)
1192 gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
1193 }
1194#else /* !__i386__ */
3a60a9f5
A		 1195 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1196 {
1197 data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1198 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
1199 if (sym && data)
1200 gIOOptionsEntry->setProperty(sym, data);
1201 if (sym)
1202 sym->release();
1203 if (data)
1204 data->release();
1205 if (gIOHibernateBootSignature[0])
1206 {
1207 data = OSData::withCapacity(16);
1208 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
1209 if (sym && data)
1210 {
1211 char c;
5d5c5d0d 1212 uint8_t value = 0;
3a60a9f5
A		 1213 for (uint32_t i = 0; (c = gIOHibernateBootSignature[i]); i++)
1214 {
1215 if (c >= 'a')
1216 c -= 'a' - 10;
1217 else if (c >= 'A')
1218 c -= 'A' - 10;
1219 else if (c >= '0')
1220 c -= '0';
1221 else
1222 continue;
1223 value = (value << 4) | c;
1224 if (i & 1)
1225 data->appendBytes(&value, sizeof(value));
1226 }
1227 gIOOptionsEntry->setProperty(sym, data);
1228 }
1229 if (sym)
1230 sym->release();
1231 if (data)
1232 data->release();
1233 }
1234 }
3a60a9f5
A		 1235 if (!vars->haveFastBoot)
1236 {
1237 // set boot volume to zero
1238 IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
1239 if (platform && (kIOReturnSuccess == platform->readXPRAM(kXPRamAudioVolume,
1240 &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume))))
1241 {
1242 uint8_t newVolume;
1243 newVolume = vars->saveBootAudioVolume & 0xf8;
1244 platform->writeXPRAM(kXPRamAudioVolume,
1245 &newVolume, sizeof(newVolume));
1246 }
1247 }
5d5c5d0d 1248#endif /* !__i386__ */
3a60a9f5
A		 1249 }
1250 // --
1251
1252 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
1253 gIOHibernateState = kIOHibernateStateHibernating;
1254 }
1255 while (false);
1256
1257 return (err);
1258}
1259
1260/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1261
1262IOReturn
1263IOHibernateSystemHasSlept(void)
1264{
1265 IOHibernateVars * vars = &gIOHibernateVars;
1266
1267 if ((vars->previewData = OSDynamicCast(OSData,
1268 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewBufferKey))))
1269 {
1270 vars->previewBuffer = IOMemoryDescriptor::withAddress(
1271 (void *) vars->previewData->getBytesNoCopy(),
1272 vars->previewData->getLength(),
1273 kIODirectionInOut);
1274
1275 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
1276 {
1277 vars->previewBuffer->release();
1278 vars->previewBuffer = 0;
1279 }
1280 if (!vars->previewBuffer)
1281 vars->previewData = 0;
1282 }
1283 if (gIOOptionsEntry)
1284 gIOOptionsEntry->sync();
1285
1286 return (kIOReturnSuccess);
1287}
1288
1289/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1290
1291IOReturn
1292IOHibernateSystemWake(void)
1293{
1294 IOHibernateVars * vars = &gIOHibernateVars;
1295
1296 hibernate_teardown(vars->page_list, vars->page_list_wired);
1297
1298 if (vars->videoMapping)
1299 {
1300 if (vars->videoMapSize)
1301 // remove mappings
1302 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1303 if (vars->videoAllocSize)
1304 // dealloc range
1305 kmem_free(kernel_map, trunc_page_32(vars->videoMapping), vars->videoAllocSize);
1306 }
1307
1308 if (vars->previewBuffer)
1309 {
1310 vars->previewBuffer->release();
1311 vars->previewBuffer = 0;
1312 }
1313
1314 if (vars->fileVars)
1315 {
1316 IOPolledFileClose(vars->fileVars);
1317 }
1318
1319 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
1320
5d5c5d0d 1321#ifdef __ppc__
3a60a9f5
A		 1322 OSData * data = OSData::withCapacity(4);
1323 if (gIOOptionsEntry && data)
1324 {
1325 const OSSymbol * sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1326 if (sym)
1327 {
1328 gIOOptionsEntry->setProperty(sym, data);
1329 sym->release();
1330 }
1331 sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey);
1332 if (sym)
1333 {
1334 if (vars->saveBootDevice)
1335 {
1336 gIOOptionsEntry->setProperty(sym, vars->saveBootDevice);
1337 vars->saveBootDevice->release();
1338 }
1339 sym->release();
1340 }
1341 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
1342 if (sym)
1343 {
1344 gIOOptionsEntry->setProperty(sym, data);
1345 sym->release();
1346 }
1347 sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey);
1348 if (sym)
1349 {
1350 gIOOptionsEntry->removeProperty(sym);
1351 sym->release();
1352 }
1353 }
1354 if (data)
1355 data->release();
1356
1357 if (gIOOptionsEntry)
1358 {
1359 if (!vars->haveFastBoot)
1360 {
1361 // reset boot audio volume
1362 IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
1363 if (platform)
1364 platform->writeXPRAM(kXPRamAudioVolume,
1365 &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume));
1366 }
1367
1368 // sync now to hardware if the booter has not
1369 if (kIOHibernateStateInactive == gIOHibernateState)
1370 gIOOptionsEntry->sync();
1371 else
1372 // just sync the variables in case a later panic syncs nvram (it won't sync variables)
1373 gIOOptionsEntry->syncOFVariables();
1374 }
5d5c5d0d
A		 1375#endif
1376
1377#ifdef __i386__
1378 IOService::getPMRootDomain()->removeProperty(kIOHibernateRTCVariablesKey);
1379#endif
3a60a9f5
A		 1380
1381 if (vars->srcBuffer)
1382 vars->srcBuffer->release();
1383 if (vars->ioBuffer)
1384 vars->ioBuffer->release();
1385 if (vars->fileExtents)
1386 vars->fileExtents->release();
1387
1388 bzero(vars, sizeof(*vars));
1389
1390// gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
1391
1392 return (kIOReturnSuccess);
1393}
1394
1395IOReturn
1396IOHibernateSystemPostWake(void)
1397{
1398 if (gIOHibernateFileRef)
1399 {
5d5c5d0d
A		 1400 // invalidate the image file
1401 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1402 int err = kern_write_file(gIOHibernateFileRef, 0,
1403 (caddr_t) gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
1404 if (KERN_SUCCESS != err)
1405 HIBLOG("kern_write_file(%d)\n", err);
1406
3a60a9f5
A		 1407 kern_close_file_for_direct_io(gIOHibernateFileRef);
1408 gIOHibernateFileRef = 0;
1409 }
1410 return (kIOReturnSuccess);
1411}
1412
1413/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1414
1415void
1416IOHibernateSystemInit(IOPMrootDomain * rootDomain)
1417{
1418 OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
1419 if (data)
1420 {
1421 rootDomain->setProperty(kIOHibernateStateKey, data);
1422 data->release();
1423 }
1424
1425 if (PE_parse_boot_arg("hfile", gIOHibernateFilename))
1426 gIOHibernateMode = kIOHibernateModeOn;
1427 else
1428 gIOHibernateFilename[0] = 0;
1429
1430 static SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
1431 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
1432 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
1433 sysctl_register_oid(&sysctl__kern_hibernatefile);
1434
1435 static SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
1436 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
1437 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
1438 sysctl_register_oid(&sysctl__kern_bootsignature);
1439
1440 static SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
1441 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
1442 &gIOHibernateMode, 0, "");
1443 sysctl_register_oid(&sysctl__kern_hibernatemode);
1444}
1445
1446/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1447
1448static void
1449hibernate_setup_for_wake(void)
1450{
1451#if __ppc__
1452 // go slow (state needed for wake)
1453 ml_set_processor_speed(1);
1454#endif
1455}
1456
1457/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1458
5d5c5d0d
A		 1459#define C_ASSERT(e) typedef char __C_ASSERT__[(e) ? 1 : -1]
1460
3a60a9f5
A		 1461extern "C" boolean_t
1462hibernate_write_image(void)
1463{
1464 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
1465 IOHibernateVars * vars = &gIOHibernateVars;
1466 IOPolledFileExtent * fileExtents;
1467
5d5c5d0d
A		 1468 C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
1469
3a60a9f5
A		 1470 uint32_t pageCount, pagesDone;
1471 IOReturn err;
1472 vm_offset_t ppnum;
1473 IOItemCount page, count;
1474 uint8_t * src;
1475 uint8_t * data;
1476 IOByteCount pageCompressedSize;
1477 uint64_t compressedSize, uncompressedSize;
1478 uint64_t image1Size = 0;
1479 uint32_t bitmap_size;
1480 bool iterDone, pollerOpen, needEncryptStart;
1481 uint32_t restore1Sum, sum, sum1, sum2;
1482 uint32_t tag;
1483 uint32_t pageType;
1484 uint32_t pageAndCount[2];
1485
1486 AbsoluteTime startTime, endTime;
1487 AbsoluteTime allTime, compTime, decoTime;
1488 uint64_t nsec;
1489 uint32_t lastProgressStamp = 0;
1490 uint32_t progressStamp;
1491
1492 hibernate_cryptvars_t _cryptvars;
1493 hibernate_cryptvars_t * cryptvars = 0;
1494
1495 if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
1496 return (false /* sleep */ );
1497
1498 restore1Sum = sum1 = sum2 = 0;
1499
1500 // encryption data. "iv" is the "initial vector".
1501 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1502 {
1503 static const unsigned char first_iv[AES_BLOCK_SIZE]
1504 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
1505 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
1506
1507 cryptvars = &gIOHibernateCryptWakeContext;
1508 bzero(cryptvars, sizeof(hibernate_cryptvars_t));
1509 aes_encrypt_key(vars->cryptKey,
1510 kIOHibernateAESKeySize,
1511 &cryptvars->ctx.encrypt);
1512 aes_decrypt_key(vars->cryptKey,
1513 kIOHibernateAESKeySize,
1514 &cryptvars->ctx.decrypt);
1515
1516 cryptvars = &_cryptvars;
1517 bzero(cryptvars, sizeof(hibernate_cryptvars_t));
1518 aes_encrypt_key(vars->wiredCryptKey,
1519 kIOHibernateAESKeySize,
1520 &cryptvars->ctx.encrypt);
1521
1522 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1523 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1524 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
1525 bzero(gIOHibernateCryptWakeVars, sizeof(hibernate_cryptwakevars_t));
1526 }
1527
1528 hibernate_setup_for_wake();
1529
1530 hibernate_page_list_setall(vars->page_list,
1531 vars->page_list_wired,
1532 &pageCount);
1533
1534 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
1535
1536 fileExtents = (IOPolledFileExtent *) vars->fileExtents->getBytesNoCopy();
1537
1538#if 0
1539 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
1540 for (page = 0; page < count; page++)
1541 {
1542 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
1543 fileExtents[page].start, fileExtents[page].length,
1544 fileExtents[page].start + fileExtents[page].length);
1545 }
1546#endif
1547
1548 needEncryptStart = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
1549
1550 AbsoluteTime_to_scalar(&compTime) = 0;
1551 AbsoluteTime_to_scalar(&decoTime) = 0;
1552
1553 clock_get_uptime(&allTime);
1554
1555 do
1556 {
1557 compressedSize = 0;
1558 uncompressedSize = 0;
1559 iterDone = false;
1560 pageType = 0; // wired pages first
1561
1562 IOPolledFileSeek(vars->fileVars, sizeof(IOHibernateImageHeader));
1563
1564 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
1565 ml_get_interrupts_enabled());
1566 err = IOHibernatePollerOpen(vars->fileVars, kIOPolledBeforeSleepState, vars->ioBuffer);
1567 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
1568 pollerOpen = (kIOReturnSuccess == err);
1569 if (!pollerOpen)
1570 break;
1571
1572 // copy file block extent list if larger than header
1573
1574 count = vars->fileExtents->getLength();
1575 if (count > sizeof(header->fileExtentMap))
1576 {
1577 count -= sizeof(header->fileExtentMap);
1578 err = IOPolledFileWrite(vars->fileVars,
1579 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
1580 if (kIOReturnSuccess != err)
1581 break;
1582 }
1583
1584 // copy out restore1 code
1585
1586 page = atop_32(sectHIBB);
1587 count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page;
1588 header->restore1CodePage = page;
1589 header->restore1PageCount = count;
1590 header->restore1CodeOffset = ((uint32_t) &hibernate_machine_entrypoint) - sectHIBB;
1591 header->restore1StackOffset = ((uint32_t) &gIOHibernateRestoreStackEnd[0]) - 64 - sectHIBB;
1592
1593 // sum __HIB sect, with zeros for the stack
1594 src = (uint8_t *) trunc_page(sectHIBB);
1595 for (page = 0; page < count; page++)
1596 {
1597 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
1598 restore1Sum += hibernate_sum(src, page_size);
1599 else
1600 restore1Sum += 0x10000001;
1601 src += page_size;
1602 }
1603 sum1 = restore1Sum;
1604
1605 // write the __HIB sect, with zeros for the stack
1606
1607 src = (uint8_t *) trunc_page(sectHIBB);
1608 count = ((uint32_t) &gIOHibernateRestoreStack[0]) - trunc_page(sectHIBB);
1609 if (count)
1610 {
1611 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
1612 if (kIOReturnSuccess != err)
1613 break;
1614 }
1615 err = IOPolledFileWrite(vars->fileVars,
1616 (uint8_t *) 0,
1617 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
1618 cryptvars);
1619 if (kIOReturnSuccess != err)
1620 break;
1621 src = &gIOHibernateRestoreStackEnd[0];
1622 count = round_page(sectHIBB + sectSizeHIB) - ((uint32_t) src);
1623 if (count)
1624 {
1625 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
1626 if (kIOReturnSuccess != err)
1627 break;
1628 }
1629
1630 // write the preview buffer
1631
1632 addr64_t phys64;
1633 IOByteCount segLen;
1634
1635 if (vars->previewData)
1636 {
1637 ppnum = 0;
1638 count = 0;
1639 do
1640 {
1641 phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen);
1642 pageAndCount[0] = atop_64(phys64);
1643 pageAndCount[1] = atop_32(segLen);
1644 err = IOPolledFileWrite(vars->fileVars,
1645 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1646 cryptvars);
1647 if (kIOReturnSuccess != err)
1648 break;
1649 count += segLen;
1650 ppnum += sizeof(pageAndCount);
1651 }
1652 while (phys64);
1653 if (kIOReturnSuccess != err)
1654 break;
1655
1656 src = (uint8_t *) vars->previewData->getBytesNoCopy();
1657 count = vars->previewData->getLength();
1658
1659 header->previewPageListSize = ppnum;
1660 header->previewSize = count + ppnum;
1661
1662 for (page = 0; page < count; page += page_size)
1663 sum1 += hibernate_sum(src + page, page_size);
1664
1665 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
1666 if (kIOReturnSuccess != err)
1667 break;
1668 }
1669
1670 // mark areas for no save
1671
1672 for (count = 0;
1673 (phys64 = vars->ioBuffer->getPhysicalSegment64(count, &segLen));
1674 count += segLen)
1675 {
1676 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1677 atop_64(phys64), atop_32(segLen),
1678 kIOHibernatePageStateFree);
1679 pageCount -= atop_32(segLen);
1680 }
1681
1682 for (count = 0;
1683 (phys64 = vars->srcBuffer->getPhysicalSegment64(count, &segLen));
1684 count += segLen)
1685 {
1686 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1687 atop_64(phys64), atop_32(segLen),
1688 kIOHibernatePageStateFree);
1689 pageCount -= atop_32(segLen);
1690 }
1691
1692 // copy out bitmap of pages available for trashing during restore
1693
1694 bitmap_size = vars->page_list_wired->list_size;
1695 src = (uint8_t *) vars->page_list_wired;
1696 err = IOPolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
1697 if (kIOReturnSuccess != err)
1698 break;
1699
1700 // mark more areas for no save, but these are not available
1701 // for trashing during restore
5d5c5d0d
A		 1702
1703 hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
3a60a9f5 1704
3a60a9f5
A		 1705 page = atop_32(sectHIBB);
1706 count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page;
3a60a9f5
A		 1707 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1708 page, count,
1709 kIOHibernatePageStateFree);
1710 pageCount -= count;
3a60a9f5
A		 1711
1712 if (vars->previewBuffer) for (count = 0;
1713 (phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen));
1714 count += segLen)
1715 {
1716 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1717 atop_64(phys64), atop_32(segLen),
1718 kIOHibernatePageStateFree);
1719 pageCount -= atop_32(segLen);
1720 }
1721
1722 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
1723
5d5c5d0d
A		 1724 ppnum = 0;
1725 pagesDone = 0;
3a60a9f5
A		 1726
1727 HIBLOG("writing %d pages\n", pageCount);
1728
1729 do
1730 {
1731 count = hibernate_page_list_iterate(pageType ? vars->page_list : vars->page_list_wired,
5d5c5d0d 1732 &ppnum);
3a60a9f5
A		 1733// kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
1734
1735 iterDone = !count;
1736
1737 pageAndCount[0] = ppnum;
1738 pageAndCount[1] = count;
1739 err = IOPolledFileWrite(vars->fileVars,
1740 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1741 cryptvars);
1742 if (kIOReturnSuccess != err)
1743 break;
1744
1745 for (page = 0; page < count; page++)
1746 {
1747 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(ppnum), page_size);
1748 if (err)
1749 {
1750 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%d] %x\n", __LINE__, ppnum, err);
1751 break;
1752 }
1753
1754 sum = hibernate_sum(src, page_size);
1755
1756 clock_get_uptime(&startTime);
1757
1758 pageCompressedSize = WKdm_compress ((WK_word*) src, (WK_word*) (src + page_size), PAGE_SIZE_IN_WORDS);
1759
1760 clock_get_uptime(&endTime);
1761 ADD_ABSOLUTETIME(&compTime, &endTime);
1762 SUB_ABSOLUTETIME(&compTime, &startTime);
1763
1764 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1765 pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1);
1766
1767 if (pageCompressedSize > page_size)
1768 {
1769// HIBLOG("------------lose: %d\n", pageCompressedSize);
1770 pageCompressedSize = page_size;
1771 }
1772
1773 if (pageCompressedSize != page_size)
1774 data = (src + page_size);
1775 else
1776 data = src;
1777
1778 tag = pageCompressedSize | kIOHibernateTagSignature;
1779
1780 if (pageType)
1781 sum2 += sum;
1782 else
1783 sum1 += sum;
1784
1785 if (needEncryptStart && (ppnum >= atop_32(sectDATAB)))
1786 {
1787 // start encrypting partway into the data about to be written
1788 vars->fileVars->encryptStart = (vars->fileVars->position + AES_BLOCK_SIZE - 1)
1789 & ~(AES_BLOCK_SIZE - 1);
1790 needEncryptStart = false;
1791 }
1792
1793 err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
1794 if (kIOReturnSuccess != err)
1795 break;
1796
1797 err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
1798 if (kIOReturnSuccess != err)
1799 break;
1800
1801 compressedSize += pageCompressedSize;
1802 if (pageCompressedSize)
1803 uncompressedSize += page_size;
1804 ppnum++;
1805 pagesDone++;
1806
1807 if (0 == (8191 & pagesDone))
1808 {
1809 clock_get_uptime(&endTime);
1810 SUB_ABSOLUTETIME(&endTime, &allTime);
1811 absolutetime_to_nanoseconds(endTime, &nsec);
1812 progressStamp = nsec / 750000000ULL;
1813 if (progressStamp != lastProgressStamp)
1814 {
1815 lastProgressStamp = progressStamp;
1816 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
1817 }
1818 }
1819 }
1820 if (kIOReturnSuccess != err)
1821 break;
1822 if (iterDone && !pageType)
1823 {
1824 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
1825 if (kIOReturnSuccess != err)
1826 break;
1827
1828 iterDone = false;
1829 pageType = 1;
5d5c5d0d 1830 ppnum = 0;
3a60a9f5
A		 1831 image1Size = vars->fileVars->position;
1832 if (cryptvars)
1833 {
1834 bcopy(&cryptvars->aes_iv[0],
1835 &gIOHibernateCryptWakeContext.aes_iv[0],
1836 sizeof(cryptvars->aes_iv));
1837 cryptvars = &gIOHibernateCryptWakeContext;
1838 }
1839 HIBLOG("image1Size %qd\n", image1Size);
1840 }
1841 }
1842 while (!iterDone);
1843 if (kIOReturnSuccess != err)
1844 break;
1845 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
1846 if (kIOReturnSuccess != err)
1847 break;
1848
1849 // Header:
1850
1851 header->imageSize = vars->fileVars->position;
1852 header->image1Size = image1Size;
1853 header->bitmapSize = bitmap_size;
1854 header->pageCount = pageCount;
1855 header->encryptStart = vars->fileVars->encryptStart;
1856
1857 header->restore1Sum = restore1Sum;
1858 header->image1Sum = sum1;
1859 header->image2Sum = sum2;
1860
1861 count = vars->fileExtents->getLength();
1862 if (count > sizeof(header->fileExtentMap))
1863 {
1864 header->fileExtentMapSize = count;
1865 count = sizeof(header->fileExtentMap);
1866 }
1867 else
1868 header->fileExtentMapSize = sizeof(header->fileExtentMap);
1869 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
1870
1871 IOPolledFileSeek(vars->fileVars, 0);
1872 err = IOPolledFileWrite(vars->fileVars,
1873 (uint8_t *) header, sizeof(IOHibernateImageHeader),
1874 cryptvars);
1875 if (kIOReturnSuccess != err)
1876 break;
1877 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
1878 if (kIOReturnSuccess != err)
1879 break;
1880 err = IOHibernatePollerIODone(vars->fileVars);
1881 if (kIOReturnSuccess != err)
1882 break;
1883 }
1884 while (false);
1885
1886 clock_get_uptime(&endTime);
1887 SUB_ABSOLUTETIME(&endTime, &allTime);
1888 absolutetime_to_nanoseconds(endTime, &nsec);
1889 HIBLOG("all time: %qd ms, ",
1890 nsec / 1000000ULL);
1891
1892 absolutetime_to_nanoseconds(compTime, &nsec);
1893 HIBLOG("comp time: %qd ms, ",
1894 nsec / 1000000ULL);
1895
1896 absolutetime_to_nanoseconds(decoTime, &nsec);
1897 HIBLOG("deco time: %qd ms, ",
1898 nsec / 1000000ULL);
1899
1900 HIBLOG("\nimage %qd, uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
1901 header->imageSize,
1902 uncompressedSize, atop_32(uncompressedSize), compressedSize,
1903 (int) ((compressedSize * 100ULL) / uncompressedSize),
1904 sum1, sum2);
1905
1906 if (pollerOpen)
1907 IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState);
1908
1909 HIBLOG("hibernate_write_image done(%x)\n", err);
1910
1911 // should we come back via regular wake, set the state in memory.
1912 gIOHibernateState = kIOHibernateStateInactive;
1913
1914 if ((kIOReturnSuccess == err) && !(kIOHibernateModeSleep & gIOHibernateMode))
1915 return (true /* power down */ );
1916 else
1917 return (false /* sleep */ );
1918}
1919
1920/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1921
1922DECLARE_IOHIBERNATEPROGRESSALPHA
1923
1924static void
1925ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
1926{
1927 uint32_t rowBytes, pixelShift;
1928 uint32_t x, y;
1929 int32_t blob, lastBlob;
1930 uint32_t alpha, in, color, result;
1931 uint8_t * out;
1932 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1933
1934 pixelShift = display->depth >> 4;
1935 if (pixelShift < 1)
1936 return;
1937
1938 rowBytes = display->rowBytes;
1939
1940 screen += ((display->width
1941 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1942 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1943
1944 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
1945
1946 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
1947
1948 for (y = 0; y < kIOHibernateProgressHeight; y++)
1949 {
1950 out = screen + y * rowBytes;
1951 for (blob = firstBlob; blob <= lastBlob; blob++)
1952 {
1953 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
1954 for (x = 0; x < kIOHibernateProgressWidth; x++)
1955 {
1956 alpha = gIOHibernateProgressAlpha[y][x];
1957 result = color;
1958 if (alpha)
1959 {
1960 if (0xff != alpha)
1961 {
1962 in = display->progressSaveUnder[blob][saveindex[blob]++];
1963 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
1964 }
1965 if (1 == pixelShift)
1966 {
1967 result >>= 3;
1968 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1969 }
1970 else
1971 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1972 }
1973 out += (1 << pixelShift);
1974 }
1975 out += (kIOHibernateProgressSpacing << pixelShift);
1976 }
1977 }
1978}
1979
1980/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1981
1982extern "C" void
1983hibernate_machine_init(void)
1984{
1985 IOReturn err;
1986 uint32_t sum;
1987 uint32_t pagesDone;
1988 AbsoluteTime allTime, endTime;
1989 uint64_t nsec;
1990 uint32_t lastProgressStamp = 0;
1991 uint32_t progressStamp;
1992 uint64_t progressZeroPosition = 0;
1993 uint32_t blob, lastBlob = (uint32_t) -1L;
1994 hibernate_cryptvars_t * cryptvars = 0;
1995
1996 IOHibernateVars * vars = &gIOHibernateVars;
1997
1998 if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
1999 return;
2000
3a60a9f5
A		 2001 sum = gIOHibernateCurrentHeader->actualImage1Sum;
2002 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
2003
2004 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, image1Size %qx, conflictCount %d, nextFree %x\n",
2005 gIOHibernateState, pagesDone, sum, gIOHibernateCurrentHeader->image1Size,
2006 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
2007
2008 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
2009 {
2010 HIBLOG("regular wake\n");
2011 return;
2012 }
2013
2014 HIBPRINT("diag %x %x %x %x\n",
2015 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
2016 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
2017
2018 HIBPRINT("video %x %d %d %d\n",
2019 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
2020 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height);
2021
5d5c5d0d
A		 2022 if ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)
2023 hibernate_page_list_discard(vars->page_list);
2024
2025 boot_args *args = (boot_args *) PE_state.bootArgs;
2026
2027 if (vars->videoMapping
2028 && gIOHibernateGraphicsInfo->physicalAddress
2029 && (args->Video.v_baseAddr == gIOHibernateGraphicsInfo->physicalAddress))
3a60a9f5
A		 2030 {
2031 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
2032 * gIOHibernateGraphicsInfo->rowBytes);
2033 IOMapPages(kernel_map,
2034 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
2035 vars->videoMapSize, kIOMapInhibitCache );
2036 }
2037
5d5c5d0d
A		 2038 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
2039
2040 if (gIOHibernateWakeMapSize)
2041 {
2042 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(gIOHibernateWakeMap),
2043 gIOHibernateWakeMapSize);
2044 if (kIOReturnSuccess == err)
2045 hibernate_newruntime_map(src, gIOHibernateWakeMapSize,
2046 gIOHibernateCurrentHeader->systemTableOffset);
2047 gIOHibernateWakeMap = 0;
2048 gIOHibernateWakeMapSize = 0;
2049 }
2050
3a60a9f5
A		 2051 uint32_t decoOffset;
2052
2053 clock_get_uptime(&allTime);
2054
2055 HIBLOG("IOHibernatePollerOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
2056 err = IOHibernatePollerOpen(vars->fileVars, kIOPolledAfterSleepState, 0);
2057 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
2058
2059 if (gIOHibernateCurrentHeader->previewSize)
2060 progressZeroPosition = gIOHibernateCurrentHeader->previewSize
2061 + gIOHibernateCurrentHeader->fileExtentMapSize
2062 - sizeof(gIOHibernateCurrentHeader->fileExtentMap)
2063 + ptoa_64(gIOHibernateCurrentHeader->restore1PageCount);
2064
2065 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
2066
2067 if (vars->videoMapping)
2068 {
2069 lastBlob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
2070 / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
2071 ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, 0, lastBlob);
2072 }
2073
2074 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
2075 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2076 {
2077 cryptvars = &gIOHibernateCryptWakeContext;
2078 bcopy(&gIOHibernateCryptWakeVars->aes_iv[0],
2079 &cryptvars->aes_iv[0],
2080 sizeof(cryptvars->aes_iv));
2081 }
2082
2083 // kick off the read ahead
2084 vars->fileVars->io = false;
2085 vars->fileVars->bufferHalf = 0;
2086 vars->fileVars->bufferLimit = 0;
2087 vars->fileVars->lastRead = 0;
2088 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2089
2090 IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
2091 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2092 // --
2093
2094 HIBLOG("hibernate_machine_init reading\n");
2095
2096 uint32_t * header = (uint32_t *) src;
2097 sum = 0;
2098
2099 do
2100 {
2101 unsigned int count;
2102 unsigned int page;
2103 uint32_t tag;
2104 vm_offset_t ppnum, compressedSize;
2105
2106 IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
2107
2108 ppnum = header[0];
2109 count = header[1];
2110
2111// HIBPRINT("(%x, %x)\n", ppnum, count);
2112
2113 if (!count)
2114 break;
2115
2116 for (page = 0; page < count; page++)
2117 {
2118 IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
2119
2120 compressedSize = kIOHibernateTagLength & tag;
2121 if (!compressedSize)
2122 {
2123 ppnum++;
2124 pagesDone++;
2125 continue;
2126 }
2127
2128 IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
2129
2130 if (compressedSize != page_size)
2131 {
2132 decoOffset = page_size;
2133 WKdm_decompress((WK_word*) src, (WK_word*) (src + decoOffset), PAGE_SIZE_IN_WORDS);
2134 }
2135 else
2136 decoOffset = 0;
2137
2138 sum += hibernate_sum((src + decoOffset), page_size);
2139
2140 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
2141 if (err)
2142 HIBLOG("IOMemoryDescriptorReadToPhysical [%d] %x\n", ppnum, err);
2143
2144 ppnum++;
2145 pagesDone++;
2146
2147 if (vars->videoMapping && (0 == (255 & pagesDone)))
2148 {
2149 blob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
2150 / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
2151 if (blob != lastBlob)
2152 {
2153 ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, lastBlob, blob);
2154 lastBlob = blob;
2155 }
2156 }
2157
2158 if (0 == (8191 & pagesDone))
2159 {
2160 clock_get_uptime(&endTime);
2161 SUB_ABSOLUTETIME(&endTime, &allTime);
2162 absolutetime_to_nanoseconds(endTime, &nsec);
2163 progressStamp = nsec / 750000000ULL;
2164 if (progressStamp != lastProgressStamp)
2165 {
2166 lastProgressStamp = progressStamp;
2167 HIBPRINT("pages %d (%d%%)\n", pagesDone,
2168 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
2169 }
2170 }
2171 }
2172 }
2173 while (true);
2174
2175 gIOHibernateCurrentHeader->actualImage2Sum = sum;
2176
2177 if (vars->fileVars->io)
2178 (void) IOHibernatePollerIODone(vars->fileVars);
2179
2180 err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState);
2181
2182 if (vars->videoMapping)
2183 ProgressUpdate(gIOHibernateGraphicsInfo,
2184 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2185
2186 clock_get_uptime(&endTime);
2187 SUB_ABSOLUTETIME(&endTime, &allTime);
2188 absolutetime_to_nanoseconds(endTime, &nsec);
2189
2190 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %qd ms\n",
2191 pagesDone, sum, nsec / 1000000ULL);
2192}
2193
2194/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
mirror of XNU