]> git.saurik.com Git - apple/xnu.git/blame - iokit/Kernel/IOHibernateIO.cpp
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-792.22.5.tar.gz
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
CommitLineData
3a60a9f5
A		 1/*
2 * Copyright (c) 2004 Apple Computer, Inc. All rights reserved.
3 *
8f6c56a5 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
3a60a9f5 5 *
8f6c56a5
A		 6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
8ad349bb 24 * limitations under the License.
8f6c56a5
A		 25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
3a60a9f5
A		 27 */
28
29
30/*
31
32Sleep:
33
34- PMRootDomain calls IOHibernateSystemSleep() before system sleep
35(devices awake, normal execution context)
36- IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
37 grabs its extents and searches for a polling driver willing to work with that IOMedia.
38 The BSD code makes an ioctl to the storage driver to get the partition base offset to
39 the disk, and other ioctls to get the transfer constraints
40 If successful, the file is written to make sure its initially not bootable (in case of
41 later failure) and nvram set to point to the first block of the file. (Has to be done
42 here so blocking is possible in nvram support).
43 hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
44 page out any pages it wants to (currently zero, but probably some percentage of memory).
45 Its assumed just allocating pages will cause the VM system to naturally select the best
46 pages for eviction. It also copies processor flags needed for the restore path and sets
47 a flag in the boot processor proc info.
48 gIOHibernateState = kIOHibernateStateHibernating.
49- Regular sleep progresses - some drivers may inspect the root domain property
50 kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
51 as usual but leaves motherboard I/O on.
52- Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
53 at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
54 all ppc RC bits out of the hash table and caches into the mapping structures.
55- hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
56 hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
57 All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
58 by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
59 The image header and block list are written. The header includes the second file extent so
60 only the header block is needed to read the file, regardless of filesystem.
61 The kernel section "__HIB" is written uncompressed to the image. This section of code and data
62 (only) is used to decompress the image during wake/boot.
63 Some additional pages are removed from the bitmaps - the buffers used for hibernation.
64 The bitmaps are written to the image.
65 More areas are removed from the bitmaps (after they have been written to the image) - the
66 section "__HIB" pages and interrupt stack.
67 Each wired page is compressed and written and then each non-wired page. Compression and
68 disk writes are in parallel.
69 The image header is written to the start of the file and the polling driver closed.
70 The machine powers down (or sleeps).
71
72Boot/Wake:
73
74- BootX sees the boot-image nvram variable containing the device and block number of the image,
75 reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
76- BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
77 in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
78 that is in the kernel's __HIB section.
79- BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
80 hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
81 only code & data in that section is safe to call since all the other wired pages are still
82 compressed in the image.
83- hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
84 It uses the bitmaps to work out which pages can be uncompressed from the image to their final
85 location directly, and copies those that can't to interim free pages. When the image has been
86 completed, the copies are uncompressed, overwriting the wired image pages.
87 hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
88 is used to get pages into place for 64bit.
89- the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
90 changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
91 are removed from the software strutures, and the hash table is reinitialized.
92- After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
93 of memory, using the polled mode driver, before other threads can run or any devices are turned on.
94 This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
95 for the remaining non wired pages.
96- The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
97 (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
98 is closed via bsd.
99
100Polled Mode I/O:
101
102IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
103registry, specifying an object of calls IOPolledInterface.
104
105Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
106partition) that the image is going to live, looking for polled interface properties. If it finds
107one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
108interfaces found are kept in an ordered list.
109
110There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
111few different contexts things happen in:
112
113- there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
114up and running) and after wake - this is safe to allocate memory and do anything. The device
115ignores sleep requests from that point since its a waste of time if it goes to sleep and
116immediately wakes back up for the image write.
117
118- there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
119immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
120from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
121used to flush and set the disk to sleep.
122
123- there is an Open/Close (AfterSleep) pair made around the image read operations that happen
124immediately after sleep. These can't block or allocate memory. This is happening after the platform
125expert has woken the low level bits of the system, but most of the I/O system has not. There is only
126one thread running.
127
128For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
129(prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
130the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
131Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
132that is called for the hardware to check for events, and complete the I/O via the callback.
133The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
134to restrict I/O ops.
135*/
136
137#include <sys/systm.h>
138
139#include <IOKit/IOWorkLoop.h>
140#include <IOKit/IOCommandGate.h>
141#include <IOKit/IOTimerEventSource.h>
142#include <IOKit/IOPlatformExpert.h>
143#include <IOKit/IOKitDebug.h>
144#include <IOKit/IOTimeStamp.h>
145#include <IOKit/pwr_mgt/RootDomain.h>
146#include <IOKit/pwr_mgt/IOPMPrivate.h>
147#include <IOKit/IOMessage.h>
148#include <IOKit/IODeviceTreeSupport.h>
149#include <IOKit/IOBSD.h>
150#include "RootDomainUserClient.h"
151#include <IOKit/pwr_mgt/IOPowerConnection.h>
152#include "IOPMPowerStateQueue.h"
153#include <IOKit/IOBufferMemoryDescriptor.h>
154#include <crypto/aes.h>
155
156#include <sys/uio.h>
157#include <sys/conf.h>
158#include <sys/stat.h>
159#include <sys/fcntl.h> // (FWRITE, ...)
3a60a9f5 160#include <sys/sysctl.h>
3a60a9f5
A		 161
162#include <IOKit/IOHibernatePrivate.h>
163#include <IOKit/IOPolledInterface.h>
164#include <IOKit/IONVRAM.h>
165#include "IOHibernateInternal.h"
166#include "WKdm.h"
167#include "IOKitKernelInternal.h"
168
169/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
170
171OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject);
172
173OSMetaClassDefineReservedUnused(IOPolledInterface, 0);
174OSMetaClassDefineReservedUnused(IOPolledInterface, 1);
175OSMetaClassDefineReservedUnused(IOPolledInterface, 2);
176OSMetaClassDefineReservedUnused(IOPolledInterface, 3);
177OSMetaClassDefineReservedUnused(IOPolledInterface, 4);
178OSMetaClassDefineReservedUnused(IOPolledInterface, 5);
179OSMetaClassDefineReservedUnused(IOPolledInterface, 6);
180OSMetaClassDefineReservedUnused(IOPolledInterface, 7);
181OSMetaClassDefineReservedUnused(IOPolledInterface, 8);
182OSMetaClassDefineReservedUnused(IOPolledInterface, 9);
183OSMetaClassDefineReservedUnused(IOPolledInterface, 10);
184OSMetaClassDefineReservedUnused(IOPolledInterface, 11);
185OSMetaClassDefineReservedUnused(IOPolledInterface, 12);
186OSMetaClassDefineReservedUnused(IOPolledInterface, 13);
187OSMetaClassDefineReservedUnused(IOPolledInterface, 14);
188OSMetaClassDefineReservedUnused(IOPolledInterface, 15);
189
190/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
191
192extern uint32_t gIOHibernateState;
193uint32_t gIOHibernateMode;
194static char gIOHibernateBootSignature[256+1];
195static char gIOHibernateFilename[MAXPATHLEN+1];
196static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
197uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms)
198
199static IODTNVRAM * gIOOptionsEntry;
200static IORegistryEntry * gIOChosenEntry;
4452a7af
A		 201#ifdef __i386__
202static const OSSymbol * gIOCreateEFIDevicePathSymbol;
203#endif
3a60a9f5
A		 204
205static IOPolledFileIOVars gFileVars;
206static IOHibernateVars gIOHibernateVars;
207static struct kern_direct_file_io_ref_t * gIOHibernateFileRef;
208static hibernate_cryptvars_t gIOHibernateCryptWakeContext;
209
210/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
211
212enum { kXPRamAudioVolume = 8 };
213enum { kDefaultIOSize = 128 * 1024 };
214enum { kVideoMapSize = 32 * 1024 * 1024 };
215
216#ifndef kIOMediaPreferredBlockSizeKey
217#define kIOMediaPreferredBlockSizeKey "Preferred Block Size"
218#endif
219
220#ifndef kIOBootPathKey
221#define kIOBootPathKey "bootpath"
222#endif
223#ifndef kIOSelectedBootDeviceKey
224#define kIOSelectedBootDeviceKey "boot-device"
225#endif
226
227
228enum { kIOHibernateMinPollersNeeded = 2 };
229
230/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
231
232// copy from phys addr to MD
233
234static IOReturn
235IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
236 IOByteCount offset, addr64_t bytes, IOByteCount length)
237{
238 addr64_t srcAddr = bytes;
239 IOByteCount remaining;
240
241 remaining = length = min(length, md->getLength() - offset);
242 while (remaining) { // (process another target segment?)
243 addr64_t dstAddr64;
244 IOByteCount dstLen;
245
246 dstAddr64 = md->getPhysicalSegment64(offset, &dstLen);
247 if (!dstAddr64)
248 break;
249
250 // Clip segment length to remaining
251 if (dstLen > remaining)
252 dstLen = remaining;
253
254#if 1
255 bcopy_phys(srcAddr, dstAddr64, dstLen);
256#else
257 copypv(srcAddr, dstAddr64, dstLen,
258 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
259#endif
260 srcAddr += dstLen;
261 offset += dstLen;
262 remaining -= dstLen;
263 }
264
265 assert(!remaining);
266
267 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
268}
269
270// copy from MD to phys addr
271
272static IOReturn
273IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
274 IOByteCount offset, addr64_t bytes, IOByteCount length)
275{
276 addr64_t dstAddr = bytes;
277 IOByteCount remaining;
278
279 remaining = length = min(length, md->getLength() - offset);
280 while (remaining) { // (process another target segment?)
281 addr64_t srcAddr64;
282 IOByteCount dstLen;
283
284 srcAddr64 = md->getPhysicalSegment64(offset, &dstLen);
285 if (!srcAddr64)
286 break;
287
288 // Clip segment length to remaining
289 if (dstLen > remaining)
290 dstLen = remaining;
291
292#if 1
293 bcopy_phys(srcAddr64, dstAddr, dstLen);
294#else
295 copypv(srcAddr, dstAddr64, dstLen,
296 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
297#endif
298 dstAddr += dstLen;
299 offset += dstLen;
300 remaining -= dstLen;
301 }
302
303 assert(!remaining);
304
305 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
306}
307
308/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
309
310void
311hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
312 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
313{
314 count += ppnum;
315 switch (kind)
316 {
317 case kIOHibernatePageStateUnwiredSave:
318 // unwired save
319 for (; ppnum < count; ppnum++)
320 {
321 hibernate_page_bitset(page_list, FALSE, ppnum);
322 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
323 }
324 break;
325 case kIOHibernatePageStateWiredSave:
326 // wired save
327 for (; ppnum < count; ppnum++)
328 {
329 hibernate_page_bitset(page_list, FALSE, ppnum);
330 hibernate_page_bitset(page_list_wired, FALSE, ppnum);
331 }
332 break;
333 case kIOHibernatePageStateFree:
334 // free page
335 for (; ppnum < count; ppnum++)
336 {
337 hibernate_page_bitset(page_list, TRUE, ppnum);
338 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
339 }
340 break;
341 default:
342 panic("hibernate_set_page_state");
343 }
344}
345
346static vm_offset_t
4452a7af 347hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
3a60a9f5 348{
4452a7af
A		 349 uint32_t page = *pPage;
350 uint32_t count;
351 hibernate_bitmap_t * bitmap;
8f6c56a5 352
4452a7af
A		 353 while ((bitmap = hibernate_page_bitmap_pin(list, &page)))
354 {
355 count = hibernate_page_bitmap_count(bitmap, TRUE, page);
356 if (!count)
357 break;
358 page += count;
359 if (page <= bitmap->last_page)
360 break;
361 }
21362eb3 362
4452a7af
A		 363 *pPage = page;
364 if (bitmap)
365 count = hibernate_page_bitmap_count(bitmap, FALSE, page);
366 else
367 count = 0;
3a60a9f5
A		 368
369 return (count);
370}
371
372/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
373
374static IOReturn
375IOHibernatePollerProbe(IOPolledFileIOVars * vars, IOService * target)
376{
377 IOReturn err = kIOReturnError;
378 int32_t idx;
379 IOPolledInterface * poller;
380
381 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
382 {
383 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
384 err = poller->probe(target);
385 if (err)
386 {
387 HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
388 break;
389 }
390 }
391
392 return (err);
393}
394
395static IOReturn
396IOHibernatePollerOpen(IOPolledFileIOVars * vars, uint32_t state, IOMemoryDescriptor * md)
397{
398 IOReturn err = kIOReturnError;
399 int32_t idx;
400 IOPolledInterface * poller;
401
402 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
403 {
404 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
405 err = poller->open(state, md);
406 if (err)
407 {
408 HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
409 break;
410 }
411 }
412
413 return (err);
414}
415
416static IOReturn
417IOHibernatePollerClose(IOPolledFileIOVars * vars, uint32_t state)
418{
419 IOReturn err = kIOReturnError;
420 int32_t idx;
421 IOPolledInterface * poller;
422
423 for (idx = 0;
424 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
425 idx++)
426 {
427 err = poller->close(state);
428 if (err)
429 HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
430 }
431
432 return (err);
433}
434
435static void
436IOHibernatePollerIOComplete(void * target,
437 void * parameter,
438 IOReturn status,
439 UInt64 actualByteCount)
440{
441 IOPolledFileIOVars * vars = (IOPolledFileIOVars *) parameter;
442
443 vars->ioStatus = status;
444}
445
446static IOReturn
447IOHibernatePollerIO(IOPolledFileIOVars * vars,
448 uint32_t operation, uint32_t bufferOffset,
449 uint64_t deviceOffset, uint64_t length)
450{
451
452 IOReturn err = kIOReturnError;
453 IOPolledInterface * poller;
454 IOPolledCompletion completion;
455
456 completion.target = 0;
457 completion.action = &IOHibernatePollerIOComplete;
458 completion.parameter = vars;
459
460 vars->ioStatus = -1;
461
462 poller = (IOPolledInterface *) vars->pollers->getObject(0);
463 err = poller->startIO(operation, bufferOffset, deviceOffset + vars->block0, length, completion);
464 if (err)
465 HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
466
467 return (err);
468}
469
470static IOReturn
471IOHibernatePollerIODone(IOPolledFileIOVars * vars)
472{
473 IOReturn err = kIOReturnError;
474 int32_t idx;
475 IOPolledInterface * poller;
476
477 while (-1 == vars->ioStatus)
478 {
479 for (idx = 0;
480 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
481 idx++)
482 {
483 err = poller->checkForWork();
484 if (err)
485 HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
486 }
487 }
488
489 if (kIOReturnSuccess != vars->ioStatus)
490 HIBLOG("IOPolledInterface::ioStatus 0x%x\n", vars->ioStatus);
491
492 return (vars->ioStatus);
493}
494
495IOReturn
496IOPolledInterface::checkAllForWork(void)
497{
498 IOReturn err = kIOReturnNotReady;
499 int32_t idx;
500 IOPolledInterface * poller;
501
502 IOHibernateVars * vars = &gIOHibernateVars;
503
504 if (!vars->fileVars || !vars->fileVars->pollers)
505 return (err);
506
507 for (idx = 0;
508 (poller = (IOPolledInterface *) vars->fileVars->pollers->getObject(idx));
509 idx++)
510 {
511 err = poller->checkForWork();
512 if (err)
513 HIBLOG("IOPolledInterface::checkAllForWork[%d] 0x%x\n", idx, err);
514 }
515
516 return (err);
517}
518
519/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
520
521struct _OpenFileContext
522{
523 OSData * extents;
524 uint64_t size;
525};
526
527static void
528file_extent_callback(void * ref, uint64_t start, uint64_t length)
529{
530 _OpenFileContext * ctx = (_OpenFileContext *) ref;
531 IOPolledFileExtent extent;
532
533 extent.start = start;
534 extent.length = length;
535
536 ctx->extents->appendBytes(&extent, sizeof(extent));
537 ctx->size += length;
538}
539
540IOReturn
541IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer,
542 IOPolledFileIOVars ** fileVars, OSData ** fileExtents,
543 OSData ** imagePath)
544{
545 IOReturn err = kIOReturnError;
546 IOPolledFileIOVars * vars;
547 _OpenFileContext ctx;
548 OSData * extentsData;
549 OSNumber * num;
550 IORegistryEntry * part = 0;
551 OSDictionary * matching;
552 OSIterator * iter;
553 dev_t hibernate_image_dev;
554 uint64_t maxiobytes;
555
556 vars = &gFileVars;
557 do
558 {
559 HIBLOG("sizeof(IOHibernateImageHeader) == %ld\n", sizeof(IOHibernateImageHeader));
560 if (sizeof(IOHibernateImageHeader) != 512)
561 continue;
562
563 vars->io = false;
564 vars->buffer = (uint8_t *) ioBuffer->getBytesNoCopy();
565 vars->bufferHalf = 0;
566 vars->bufferOffset = 0;
567 vars->bufferSize = ioBuffer->getLength() >> 1;
568
569 extentsData = OSData::withCapacity(32);
570
571 ctx.extents = extentsData;
572 ctx.size = 0;
573 vars->fileRef = kern_open_file_for_direct_io(filename,
574 &file_extent_callback, &ctx,
575 &hibernate_image_dev,
576 &vars->block0,
577 &maxiobytes);
578 if (!vars->fileRef)
579 {
580 err = kIOReturnNoSpace;
581 break;
582 }
583 HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx\n", filename, ctx.size,
584 vars->block0, maxiobytes);
585 if (ctx.size < 1*1024*1024) // check against image size estimate!
586 {
587 err = kIOReturnNoSpace;
588 break;
589 }
590
591 if (maxiobytes < vars->bufferSize)
592 vars->bufferSize = maxiobytes;
593
594 vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
595
596 matching = IOService::serviceMatching("IOMedia");
597 num = OSNumber::withNumber(major(hibernate_image_dev), 32);
598 matching->setObject(kIOBSDMajorKey, num);
599 num->release();
600 num = OSNumber::withNumber(minor(hibernate_image_dev), 32);
601 matching->setObject(kIOBSDMinorKey, num);
602 num->release();
603 iter = IOService::getMatchingServices(matching);
604 matching->release();
605 if (iter)
606 {
607 part = (IORegistryEntry *) iter->getNextObject();
608 part->retain();
609 iter->release();
610 }
611
612 int minor, major;
613 IORegistryEntry * next;
614 IORegistryEntry * child;
615 OSData * data;
616
617 num = (OSNumber *) part->getProperty(kIOBSDMajorKey);
618 if (!num)
619 break;
620 major = num->unsigned32BitValue();
621 num = (OSNumber *) part->getProperty(kIOBSDMinorKey);
622 if (!num)
623 break;
624 minor = num->unsigned32BitValue();
625
626 hibernate_image_dev = makedev(major, minor);
627
628 vars->pollers = OSArray::withCapacity(4);
629 if (!vars->pollers)
630 break;
631
632 vars->blockSize = 512;
633 next = part;
634 do
635 {
636 IOPolledInterface * poller;
c0fea474
A		 637 OSObject * obj;
638
639 obj = next->getProperty(kIOPolledInterfaceSupportKey);
640 if (kOSBooleanFalse == obj)
641 {
642 vars->pollers->flushCollection();
643 break;
644 }
645 else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
3a60a9f5
A		 646 vars->pollers->setObject(poller);
647 if ((num = OSDynamicCast(OSNumber, next->getProperty(kIOMediaPreferredBlockSizeKey))))
648 vars->blockSize = num->unsigned32BitValue();
649 child = next;
650 }
651 while ((next = child->getParentEntry(gIOServicePlane))
652 && child->isParent(next, gIOServicePlane, true));
653
654 HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n",
655 major, minor, vars->blockSize, vars->pollers->getCount());
656 if (vars->pollers->getCount() < kIOHibernateMinPollersNeeded)
657 continue;
658
659 err = IOHibernatePollerProbe(vars, (IOService *) part);
660 if (kIOReturnSuccess != err)
661 break;
662
663 err = IOHibernatePollerOpen(vars, kIOPolledPreflightState, ioBuffer);
664 if (kIOReturnSuccess != err)
665 break;
666
667 *fileVars = vars;
668 *fileExtents = extentsData;
669
670 // make imagePath
3a60a9f5 671
4452a7af 672 if ((extentsData->getLength() >= sizeof(IOPolledFileExtent)))
3a60a9f5 673 {
4452a7af
A		 674 char str2[24];
675
676#if __i386__
677 if (!gIOCreateEFIDevicePathSymbol)
678 gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
679
680 sprintf(str2, "%qx", vars->extentMap[0]);
681
682 err = IOService::getPlatform()->callPlatformFunction(
683 gIOCreateEFIDevicePathSymbol, false,
684 (void *) part, (void *) str2, (void *) true,
685 (void *) &data);
686#else
687 char str1[256];
688 int len = sizeof(str1);
689
690 if (!part->getPath(str1, &len, gIODTPlane))
691 err = kIOReturnNotFound;
692 else
693 {
694 sprintf(str2, ",%qx", vars->extentMap[0]);
695 // (strip the plane name)
696 char * tail = strchr(str1, ':');
697 if (!tail)
698 tail = str1 - 1;
699 data = OSData::withBytes(tail + 1, strlen(tail + 1));
700 data->appendBytes(str2, strlen(str2));
701 }
702#endif
703 if (kIOReturnSuccess == err)
3a60a9f5 704 *imagePath = data;
4452a7af
A		 705 else
706 HIBLOG("error 0x%x getting path\n", err);
3a60a9f5
A		 707 }
708 }
709 while (false);
710
711 if (kIOReturnSuccess != err)
712 {
713 HIBLOG("error 0x%x opening hibernation file\n", err);
714 if (vars->fileRef)
715 kern_close_file_for_direct_io(vars->fileRef);
716 }
717
718 if (part)
719 part->release();
720
721 return (err);
722}
723
724IOReturn
725IOPolledFileClose( IOPolledFileIOVars * vars )
726{
727 if (vars->pollers)
728 {
729 IOHibernatePollerClose(vars, kIOPolledPostflightState);
730 vars->pollers->release();
731 }
732
733 gIOHibernateFileRef = vars->fileRef;
734
735 bzero(vars, sizeof(IOPolledFileIOVars));
736
737 return (kIOReturnSuccess);
738}
739
740static IOReturn
741IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
742{
743 IOPolledFileExtent * extentMap;
744
745 extentMap = vars->extentMap;
746
747 vars->position = position;
748
749 while (position >= extentMap->length)
750 {
751 position -= extentMap->length;
752 extentMap++;
753 }
754
755 vars->currentExtent = extentMap;
756 vars->extentRemaining = extentMap->length - position;
757 vars->extentPosition = vars->position - position;
758
759 if (vars->bufferSize <= vars->extentRemaining)
760 vars->bufferLimit = vars->bufferSize;
761 else
762 vars->bufferLimit = vars->extentRemaining;
763
764 return (kIOReturnSuccess);
765}
766
767static IOReturn
768IOPolledFileWrite(IOPolledFileIOVars * vars,
769 const uint8_t * bytes, IOByteCount size,
770 hibernate_cryptvars_t * cryptvars)
771{
772 IOReturn err = kIOReturnSuccess;
773 IOByteCount copy;
774 bool flush = false;
775
776 do
777 {
778 if (!bytes && !size)
779 {
780 // seek to end of block & flush
781 size = vars->position & (vars->blockSize - 1);
782 if (size)
783 size = vars->blockSize - size;
784 flush = true;
785 // use some garbage for the fill
786 bytes = vars->buffer + vars->bufferOffset;
787 }
788
789 copy = vars->bufferLimit - vars->bufferOffset;
790 if (copy > size)
791 copy = size;
792 else
793 flush = true;
794
795 if (bytes)
796 {
797 bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
798 bytes += copy;
799 }
800 else
801 bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
802
803 size -= copy;
804 vars->bufferOffset += copy;
805 vars->position += copy;
806
807 if (flush && vars->bufferOffset)
808 {
809 uint64_t offset = (vars->position - vars->bufferOffset
810 - vars->extentPosition + vars->currentExtent->start);
811 uint32_t length = (vars->bufferOffset);
812
813 if (cryptvars && vars->encryptStart && (vars->position > vars->encryptStart))
814 {
815 uint32_t encryptLen, encryptStart;
816 encryptLen = vars->position - vars->encryptStart;
817 if (encryptLen > length)
818 encryptLen = length;
819 encryptStart = length - encryptLen;
820
821 // encrypt the buffer
822 aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
823 &cryptvars->aes_iv[0],
824 encryptLen / AES_BLOCK_SIZE,
825 vars->buffer + vars->bufferHalf + encryptStart,
826 &cryptvars->ctx.encrypt);
827 // save initial vector for following encrypts
828 bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
829 &cryptvars->aes_iv[0],
830 AES_BLOCK_SIZE);
831 }
832
833 if (vars->io)
834 {
835 err = IOHibernatePollerIODone(vars);
836 if (kIOReturnSuccess != err)
837 break;
838 }
839
840if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
841//if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
842
843 err = IOHibernatePollerIO(vars, kIOPolledWrite, vars->bufferHalf, offset, length);
844 if (kIOReturnSuccess != err)
845 break;
846 vars->io = true;
847
848 vars->extentRemaining -= vars->bufferOffset;
849 if (!vars->extentRemaining)
850 {
851 vars->currentExtent++;
852 vars->extentRemaining = vars->currentExtent->length;
853 vars->extentPosition = vars->position;
854 if (!vars->extentRemaining)
855 {
856 err = kIOReturnOverrun;
857 break;
858 }
859 }
860
861 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
862 vars->bufferOffset = 0;
863 if (vars->bufferSize <= vars->extentRemaining)
864 vars->bufferLimit = vars->bufferSize;
865 else
866 vars->bufferLimit = vars->extentRemaining;
867
868 flush = false;
869 }
870 }
871 while (size);
872
873 return (err);
874}
875
876static IOReturn
877IOPolledFileRead(IOPolledFileIOVars * vars,
878 uint8_t * bytes, IOByteCount size,
879 hibernate_cryptvars_t * cryptvars)
880{
881 IOReturn err = kIOReturnSuccess;
882 IOByteCount copy;
883
884// bytesWritten += size;
885
886 do
887 {
888 copy = vars->bufferLimit - vars->bufferOffset;
889 if (copy > size)
890 copy = size;
891
892 if (bytes)
893 {
894 bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
895 bytes += copy;
896 }
897 size -= copy;
898 vars->bufferOffset += copy;
899// vars->position += copy;
900
901 if (vars->bufferOffset == vars->bufferLimit)
902 {
903 if (vars->io)
904 {
905 err = IOHibernatePollerIODone(vars);
906 if (kIOReturnSuccess != err)
907 break;
908 }
909 else
910 cryptvars = 0;
911
912if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
913
914 vars->position += vars->lastRead;
915 vars->extentRemaining -= vars->lastRead;
916 vars->bufferLimit = vars->lastRead;
917
918 if (!vars->extentRemaining)
919 {
920 vars->currentExtent++;
921 vars->extentRemaining = vars->currentExtent->length;
922 vars->extentPosition = vars->position;
923 if (!vars->extentRemaining)
924 {
925 err = kIOReturnOverrun;
926 break;
927 }
928 }
929
930 if (vars->extentRemaining <= vars->bufferSize)
931 vars->lastRead = vars->extentRemaining;
932 else
933 vars->lastRead = vars->bufferSize;
934
935 uint64_t offset = (vars->position
936 - vars->extentPosition + vars->currentExtent->start);
937 uint64_t length = (vars->lastRead);
938
939//if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
940
941 err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length);
942 if (kIOReturnSuccess != err)
943 break;
944 vars->io = true;
945
946 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
947 vars->bufferOffset = 0;
948
949 if (cryptvars)
950 {
951 uint8_t thisVector[AES_BLOCK_SIZE];
952 // save initial vector for following decrypts
953 bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
954 bcopy(vars->buffer + vars->bufferHalf + vars->lastRead - AES_BLOCK_SIZE,
955 &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
956 // decrypt the buffer
957 aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
958 &thisVector[0],
959 vars->lastRead / AES_BLOCK_SIZE,
960 vars->buffer + vars->bufferHalf,
961 &cryptvars->ctx.decrypt);
962 }
963 }
964 }
965 while (size);
966
967 return (err);
968}
969
970/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
971
972IOReturn
973IOHibernateSystemSleep(void)
974{
975 IOReturn err;
976 OSData * data;
977 OSObject * obj;
978 OSString * str;
979 OSNumber * num;
980
981 IOHibernateVars * vars = &gIOHibernateVars;
982
983 if (vars->fileVars && vars->fileVars->fileRef)
984 // already on the way down
985 return (kIOReturnSuccess);
986
987 gIOHibernateState = kIOHibernateStateInactive;
988
989 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateModeKey)))
990 {
991 if ((num = OSDynamicCast(OSNumber, obj)))
992 gIOHibernateMode = num->unsigned32BitValue();
993 if (kIOHibernateModeSleep & gIOHibernateMode)
994 // default to discard clean for safe sleep
995 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
996 | kIOHibernateModeDiscardCleanActive);
997
998 obj->release();
999 }
1000 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeRatioKey)))
1001 {
1002 if ((num = OSDynamicCast(OSNumber, obj)))
1003 gIOHibernateFreeRatio = num->unsigned32BitValue();
1004 obj->release();
1005 }
1006 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeTimeKey)))
1007 {
1008 if ((num = OSDynamicCast(OSNumber, obj)))
1009 gIOHibernateFreeTime = num->unsigned32BitValue();
1010 obj->release();
1011 }
1012 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
1013 {
1014 if ((str = OSDynamicCast(OSString, obj)))
1015 strcpy(gIOHibernateFilename, str->getCStringNoCopy());
1016 obj->release();
1017 }
1018
1019 if (!gIOHibernateMode || !gIOHibernateFilename[0])
1020 return (kIOReturnUnsupported);
1021
1022 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
1023
1024 do
1025 {
4452a7af
A		 1026 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1027 4 * page_size, page_size);
1028 vars->ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1029 2 * kDefaultIOSize, page_size);
3a60a9f5
A		 1030
1031 if (!vars->srcBuffer || !vars->ioBuffer)
1032 {
1033 err = kIOReturnNoMemory;
1034 break;
1035 }
1036
1037 err = IOPolledFileOpen(gIOHibernateFilename, vars->ioBuffer,
1038 &vars->fileVars, &vars->fileExtents, &data);
1039 if (KERN_SUCCESS != err)
1040 {
1041 HIBLOG("IOPolledFileOpen(%x)\n", err);
1042 break;
1043 }
1044 if (vars->fileVars->fileRef)
1045 {
1046 // invalidate the image file
1047 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1048 int err = kern_write_file(vars->fileVars->fileRef, 0,
1049 (caddr_t) gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
1050 if (KERN_SUCCESS != err)
1051 HIBLOG("kern_write_file(%d)\n", err);
1052 }
1053
1054 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
1055
1056 boolean_t encryptedswap;
1057 err = hibernate_setup(gIOHibernateCurrentHeader,
1058 gIOHibernateFreeRatio, gIOHibernateFreeTime,
1059 &vars->page_list, &vars->page_list_wired, &encryptedswap);
1060 if (KERN_SUCCESS != err)
1061 {
1062 HIBLOG("hibernate_setup(%d)\n", err);
1063 break;
1064 }
1065
1066 if (encryptedswap)
1067 gIOHibernateMode ^= kIOHibernateModeEncrypt;
1068
1069 vars->videoAllocSize = kVideoMapSize;
1070 if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize))
1071 vars->videoMapping = 0;
1072
1073 // generate crypt keys
1074 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
1075 vars->wiredCryptKey[i] = random();
1076 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
1077 vars->cryptKey[i] = random();
1078
1079 // set nvram
1080
1081 IORegistryEntry * regEntry;
1082 if (!gIOOptionsEntry)
1083 {
1084 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
1085 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
1086 if (regEntry && !gIOOptionsEntry)
1087 regEntry->release();
1088 }
1089 if (!gIOChosenEntry)
1090 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1091
1092 if (gIOOptionsEntry)
1093 {
1094 const OSSymbol * sym;
3a60a9f5
A		 1095
1096 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1097 if (sym)
1098 {
1099 gIOOptionsEntry->setProperty(sym, data);
1100 sym->release();
1101 }
1102 data->release();
1103
4452a7af
A		 1104#ifdef __ppc__
1105 size_t len;
1106 char valueString[16];
1107
3a60a9f5
A		 1108 vars->saveBootDevice = gIOOptionsEntry->copyProperty(kIOSelectedBootDeviceKey);
1109 if (gIOChosenEntry)
1110 {
1111 OSData * bootDevice = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOBootPathKey));
1112 if (bootDevice)
1113 {
1114 sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey);
1115 OSString * str2 = OSString::withCStringNoCopy((const char *) bootDevice->getBytesNoCopy());
1116 if (sym && str2)
1117 gIOOptionsEntry->setProperty(sym, str2);
1118 if (sym)
1119 sym->release();
1120 if (str2)
1121 str2->release();
1122 }
3a60a9f5
A		 1123 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMemorySignatureKey));
1124 if (data)
1125 {
1126 vars->haveFastBoot = true;
1127
1128 len = sprintf(valueString, "0x%lx", *((UInt32 *)data->getBytesNoCopy()));
1129 data = OSData::withBytes(valueString, len + 1);
1130 sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey);
1131 if (sym && data)
1132 gIOOptionsEntry->setProperty(sym, data);
1133 if (sym)
1134 sym->release();
1135 if (data)
1136 data->release();
1137 }
1138 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
1139 if (data)
1140 gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
1141 }
4452a7af
A		 1142#endif /* __ppc__ */
1143#ifdef __i386__
1144 struct AppleRTCHibernateVars
1145 {
1146 uint8_t signature[4];
1147 uint32_t revision;
1148 uint8_t booterSignature[20];
1149 uint8_t wiredCryptKey[16];
1150 };
1151 AppleRTCHibernateVars rtcVars;
1152
1153 rtcVars.signature[0] = 'A';
1154 rtcVars.signature[1] = 'A';
1155 rtcVars.signature[2] = 'P';
1156 rtcVars.signature[3] = 'L';
1157 rtcVars.revision = 1;
1158 bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
1159 if (gIOHibernateBootSignature[0])
1160 {
1161 char c;
1162 uint8_t value = 0;
1163 for (uint32_t i = 0;
1164 (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1));
1165 i++)
1166 {
1167 if (c >= 'a')
1168 c -= 'a' - 10;
1169 else if (c >= 'A')
1170 c -= 'A' - 10;
1171 else if (c >= '0')
1172 c -= '0';
1173 else
1174 continue;
1175 value = (value << 4) | c;
1176 if (i & 1)
1177 rtcVars.booterSignature[i >> 1] = value;
1178 }
1179 }
1180 data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
1181 if (data)
1182 {
1183 IOService::getPMRootDomain()->setProperty(kIOHibernateRTCVariablesKey, data);
1184 data->release();
1185 }
1186 if (gIOChosenEntry)
1187 {
1188 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
1189 if (data)
1190 gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
1191 }
1192#else /* !__i386__ */
3a60a9f5
A		 1193 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1194 {
1195 data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1196 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
1197 if (sym && data)
1198 gIOOptionsEntry->setProperty(sym, data);
1199 if (sym)
1200 sym->release();
1201 if (data)
1202 data->release();
1203 if (gIOHibernateBootSignature[0])
1204 {
1205 data = OSData::withCapacity(16);
1206 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
1207 if (sym && data)
1208 {
1209 char c;
4452a7af 1210 uint8_t value = 0;
3a60a9f5
A		 1211 for (uint32_t i = 0; (c = gIOHibernateBootSignature[i]); i++)
1212 {
1213 if (c >= 'a')
1214 c -= 'a' - 10;
1215 else if (c >= 'A')
1216 c -= 'A' - 10;
1217 else if (c >= '0')
1218 c -= '0';
1219 else
1220 continue;
1221 value = (value << 4) | c;
1222 if (i & 1)
1223 data->appendBytes(&value, sizeof(value));
1224 }
1225 gIOOptionsEntry->setProperty(sym, data);
1226 }
1227 if (sym)
1228 sym->release();
1229 if (data)
1230 data->release();
1231 }
1232 }
3a60a9f5
A		 1233 if (!vars->haveFastBoot)
1234 {
1235 // set boot volume to zero
1236 IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
1237 if (platform && (kIOReturnSuccess == platform->readXPRAM(kXPRamAudioVolume,
1238 &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume))))
1239 {
1240 uint8_t newVolume;
1241 newVolume = vars->saveBootAudioVolume & 0xf8;
1242 platform->writeXPRAM(kXPRamAudioVolume,
1243 &newVolume, sizeof(newVolume));
1244 }
1245 }
4452a7af 1246#endif /* !__i386__ */
3a60a9f5
A		 1247 }
1248 // --
1249
1250 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
1251 gIOHibernateState = kIOHibernateStateHibernating;
1252 }
1253 while (false);
1254
1255 return (err);
1256}
1257
1258/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1259
1260IOReturn
1261IOHibernateSystemHasSlept(void)
1262{
1263 IOHibernateVars * vars = &gIOHibernateVars;
1264
1265 if ((vars->previewData = OSDynamicCast(OSData,
1266 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewBufferKey))))
1267 {
1268 vars->previewBuffer = IOMemoryDescriptor::withAddress(
1269 (void *) vars->previewData->getBytesNoCopy(),
1270 vars->previewData->getLength(),
1271 kIODirectionInOut);
1272
1273 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
1274 {
1275 vars->previewBuffer->release();
1276 vars->previewBuffer = 0;
1277 }
1278 if (!vars->previewBuffer)
1279 vars->previewData = 0;
1280 }
1281 if (gIOOptionsEntry)
1282 gIOOptionsEntry->sync();
1283
1284 return (kIOReturnSuccess);
1285}
1286
1287/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1288
1289IOReturn
1290IOHibernateSystemWake(void)
1291{
1292 IOHibernateVars * vars = &gIOHibernateVars;
1293
1294 hibernate_teardown(vars->page_list, vars->page_list_wired);
1295
1296 if (vars->videoMapping)
1297 {
1298 if (vars->videoMapSize)
1299 // remove mappings
1300 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1301 if (vars->videoAllocSize)
1302 // dealloc range
1303 kmem_free(kernel_map, trunc_page_32(vars->videoMapping), vars->videoAllocSize);
1304 }
1305
1306 if (vars->previewBuffer)
1307 {
1308 vars->previewBuffer->release();
1309 vars->previewBuffer = 0;
1310 }
1311
1312 if (vars->fileVars)
1313 {
1314 IOPolledFileClose(vars->fileVars);
1315 }
1316
1317 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
1318
4452a7af 1319#ifdef __ppc__
3a60a9f5
A		 1320 OSData * data = OSData::withCapacity(4);
1321 if (gIOOptionsEntry && data)
1322 {
1323 const OSSymbol * sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1324 if (sym)
1325 {
1326 gIOOptionsEntry->setProperty(sym, data);
1327 sym->release();
1328 }
1329 sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey);
1330 if (sym)
1331 {
1332 if (vars->saveBootDevice)
1333 {
1334 gIOOptionsEntry->setProperty(sym, vars->saveBootDevice);
1335 vars->saveBootDevice->release();
1336 }
1337 sym->release();
1338 }
1339 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
1340 if (sym)
1341 {
1342 gIOOptionsEntry->setProperty(sym, data);
1343 sym->release();
1344 }
1345 sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey);
1346 if (sym)
1347 {
1348 gIOOptionsEntry->removeProperty(sym);
1349 sym->release();
1350 }
1351 }
1352 if (data)
1353 data->release();
1354
1355 if (gIOOptionsEntry)
1356 {
1357 if (!vars->haveFastBoot)
1358 {
1359 // reset boot audio volume
1360 IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
1361 if (platform)
1362 platform->writeXPRAM(kXPRamAudioVolume,
1363 &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume));
1364 }
1365
1366 // sync now to hardware if the booter has not
1367 if (kIOHibernateStateInactive == gIOHibernateState)
1368 gIOOptionsEntry->sync();
1369 else
1370 // just sync the variables in case a later panic syncs nvram (it won't sync variables)
1371 gIOOptionsEntry->syncOFVariables();
1372 }
4452a7af
A		 1373#endif
1374
1375#ifdef __i386__
1376 IOService::getPMRootDomain()->removeProperty(kIOHibernateRTCVariablesKey);
1377#endif
3a60a9f5
A		 1378
1379 if (vars->srcBuffer)
1380 vars->srcBuffer->release();
1381 if (vars->ioBuffer)
1382 vars->ioBuffer->release();
1383 if (vars->fileExtents)
1384 vars->fileExtents->release();
1385
1386 bzero(vars, sizeof(*vars));
1387
1388// gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
1389
1390 return (kIOReturnSuccess);
1391}
1392
1393IOReturn
1394IOHibernateSystemPostWake(void)
1395{
1396 if (gIOHibernateFileRef)
1397 {
4452a7af
A		 1398 // invalidate the image file
1399 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
1400 int err = kern_write_file(gIOHibernateFileRef, 0,
1401 (caddr_t) gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
1402 if (KERN_SUCCESS != err)
1403 HIBLOG("kern_write_file(%d)\n", err);
1404
3a60a9f5
A		 1405 kern_close_file_for_direct_io(gIOHibernateFileRef);
1406 gIOHibernateFileRef = 0;
1407 }
1408 return (kIOReturnSuccess);
1409}
1410
1411/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1412
1413void
1414IOHibernateSystemInit(IOPMrootDomain * rootDomain)
1415{
1416 OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
1417 if (data)
1418 {
1419 rootDomain->setProperty(kIOHibernateStateKey, data);
1420 data->release();
1421 }
1422
1423 if (PE_parse_boot_arg("hfile", gIOHibernateFilename))
1424 gIOHibernateMode = kIOHibernateModeOn;
1425 else
1426 gIOHibernateFilename[0] = 0;
1427
1428 static SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
1429 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
1430 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
1431 sysctl_register_oid(&sysctl__kern_hibernatefile);
1432
1433 static SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
1434 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
1435 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
1436 sysctl_register_oid(&sysctl__kern_bootsignature);
1437
1438 static SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
1439 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN,
1440 &gIOHibernateMode, 0, "");
1441 sysctl_register_oid(&sysctl__kern_hibernatemode);
1442}
1443
1444/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1445
1446static void
1447hibernate_setup_for_wake(void)
1448{
1449#if __ppc__
1450 // go slow (state needed for wake)
1451 ml_set_processor_speed(1);
1452#endif
1453}
1454
1455/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1456
4452a7af
A		 1457#define C_ASSERT(e) typedef char __C_ASSERT__[(e) ? 1 : -1]
1458
3a60a9f5
A		 1459extern "C" boolean_t
1460hibernate_write_image(void)
1461{
1462 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
1463 IOHibernateVars * vars = &gIOHibernateVars;
1464 IOPolledFileExtent * fileExtents;
1465
4452a7af
A		 1466 C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
1467
3a60a9f5
A		 1468 uint32_t pageCount, pagesDone;
1469 IOReturn err;
1470 vm_offset_t ppnum;
1471 IOItemCount page, count;
1472 uint8_t * src;
1473 uint8_t * data;
1474 IOByteCount pageCompressedSize;
1475 uint64_t compressedSize, uncompressedSize;
1476 uint64_t image1Size = 0;
1477 uint32_t bitmap_size;
1478 bool iterDone, pollerOpen, needEncryptStart;
1479 uint32_t restore1Sum, sum, sum1, sum2;
1480 uint32_t tag;
1481 uint32_t pageType;
1482 uint32_t pageAndCount[2];
1483
1484 AbsoluteTime startTime, endTime;
1485 AbsoluteTime allTime, compTime, decoTime;
1486 uint64_t nsec;
1487 uint32_t lastProgressStamp = 0;
1488 uint32_t progressStamp;
1489
1490 hibernate_cryptvars_t _cryptvars;
1491 hibernate_cryptvars_t * cryptvars = 0;
1492
1493 if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
1494 return (false /* sleep */ );
1495
1496 restore1Sum = sum1 = sum2 = 0;
1497
1498 // encryption data. "iv" is the "initial vector".
1499 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1500 {
1501 static const unsigned char first_iv[AES_BLOCK_SIZE]
1502 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
1503 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
1504
1505 cryptvars = &gIOHibernateCryptWakeContext;
1506 bzero(cryptvars, sizeof(hibernate_cryptvars_t));
1507 aes_encrypt_key(vars->cryptKey,
1508 kIOHibernateAESKeySize,
1509 &cryptvars->ctx.encrypt);
1510 aes_decrypt_key(vars->cryptKey,
1511 kIOHibernateAESKeySize,
1512 &cryptvars->ctx.decrypt);
1513
1514 cryptvars = &_cryptvars;
1515 bzero(cryptvars, sizeof(hibernate_cryptvars_t));
1516 aes_encrypt_key(vars->wiredCryptKey,
1517 kIOHibernateAESKeySize,
1518 &cryptvars->ctx.encrypt);
1519
1520 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1521 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1522 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
1523 bzero(gIOHibernateCryptWakeVars, sizeof(hibernate_cryptwakevars_t));
1524 }
1525
1526 hibernate_setup_for_wake();
1527
1528 hibernate_page_list_setall(vars->page_list,
1529 vars->page_list_wired,
1530 &pageCount);
1531
1532 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
1533
1534 fileExtents = (IOPolledFileExtent *) vars->fileExtents->getBytesNoCopy();
1535
1536#if 0
1537 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
1538 for (page = 0; page < count; page++)
1539 {
1540 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
1541 fileExtents[page].start, fileExtents[page].length,
1542 fileExtents[page].start + fileExtents[page].length);
1543 }
1544#endif
1545
1546 needEncryptStart = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
1547
1548 AbsoluteTime_to_scalar(&compTime) = 0;
1549 AbsoluteTime_to_scalar(&decoTime) = 0;
1550
1551 clock_get_uptime(&allTime);
1552
1553 do
1554 {
1555 compressedSize = 0;
1556 uncompressedSize = 0;
1557 iterDone = false;
1558 pageType = 0; // wired pages first
1559
1560 IOPolledFileSeek(vars->fileVars, sizeof(IOHibernateImageHeader));
1561
1562 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
1563 ml_get_interrupts_enabled());
1564 err = IOHibernatePollerOpen(vars->fileVars, kIOPolledBeforeSleepState, vars->ioBuffer);
1565 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
1566 pollerOpen = (kIOReturnSuccess == err);
1567 if (!pollerOpen)
1568 break;
1569
1570 // copy file block extent list if larger than header
1571
1572 count = vars->fileExtents->getLength();
1573 if (count > sizeof(header->fileExtentMap))
1574 {
1575 count -= sizeof(header->fileExtentMap);
1576 err = IOPolledFileWrite(vars->fileVars,
1577 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
1578 if (kIOReturnSuccess != err)
1579 break;
1580 }
1581
1582 // copy out restore1 code
1583
1584 page = atop_32(sectHIBB);
1585 count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page;
1586 header->restore1CodePage = page;
1587 header->restore1PageCount = count;
1588 header->restore1CodeOffset = ((uint32_t) &hibernate_machine_entrypoint) - sectHIBB;
1589 header->restore1StackOffset = ((uint32_t) &gIOHibernateRestoreStackEnd[0]) - 64 - sectHIBB;
1590
1591 // sum __HIB sect, with zeros for the stack
1592 src = (uint8_t *) trunc_page(sectHIBB);
1593 for (page = 0; page < count; page++)
1594 {
1595 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
1596 restore1Sum += hibernate_sum(src, page_size);
1597 else
1598 restore1Sum += 0x10000001;
1599 src += page_size;
1600 }
1601 sum1 = restore1Sum;
1602
1603 // write the __HIB sect, with zeros for the stack
1604
1605 src = (uint8_t *) trunc_page(sectHIBB);
1606 count = ((uint32_t) &gIOHibernateRestoreStack[0]) - trunc_page(sectHIBB);
1607 if (count)
1608 {
1609 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
1610 if (kIOReturnSuccess != err)
1611 break;
1612 }
1613 err = IOPolledFileWrite(vars->fileVars,
1614 (uint8_t *) 0,
1615 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
1616 cryptvars);
1617 if (kIOReturnSuccess != err)
1618 break;
1619 src = &gIOHibernateRestoreStackEnd[0];
1620 count = round_page(sectHIBB + sectSizeHIB) - ((uint32_t) src);
1621 if (count)
1622 {
1623 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
1624 if (kIOReturnSuccess != err)
1625 break;
1626 }
1627
1628 // write the preview buffer
1629
1630 addr64_t phys64;
1631 IOByteCount segLen;
1632
1633 if (vars->previewData)
1634 {
1635 ppnum = 0;
1636 count = 0;
1637 do
1638 {
1639 phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen);
1640 pageAndCount[0] = atop_64(phys64);
1641 pageAndCount[1] = atop_32(segLen);
1642 err = IOPolledFileWrite(vars->fileVars,
1643 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1644 cryptvars);
1645 if (kIOReturnSuccess != err)
1646 break;
1647 count += segLen;
1648 ppnum += sizeof(pageAndCount);
1649 }
1650 while (phys64);
1651 if (kIOReturnSuccess != err)
1652 break;
1653
1654 src = (uint8_t *) vars->previewData->getBytesNoCopy();
1655 count = vars->previewData->getLength();
1656
1657 header->previewPageListSize = ppnum;
1658 header->previewSize = count + ppnum;
1659
1660 for (page = 0; page < count; page += page_size)
1661 sum1 += hibernate_sum(src + page, page_size);
1662
1663 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
1664 if (kIOReturnSuccess != err)
1665 break;
1666 }
1667
1668 // mark areas for no save
1669
1670 for (count = 0;
1671 (phys64 = vars->ioBuffer->getPhysicalSegment64(count, &segLen));
1672 count += segLen)
1673 {
1674 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1675 atop_64(phys64), atop_32(segLen),
1676 kIOHibernatePageStateFree);
1677 pageCount -= atop_32(segLen);
1678 }
1679
1680 for (count = 0;
1681 (phys64 = vars->srcBuffer->getPhysicalSegment64(count, &segLen));
1682 count += segLen)
1683 {
1684 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1685 atop_64(phys64), atop_32(segLen),
1686 kIOHibernatePageStateFree);
1687 pageCount -= atop_32(segLen);
1688 }
1689
1690 // copy out bitmap of pages available for trashing during restore
1691
1692 bitmap_size = vars->page_list_wired->list_size;
1693 src = (uint8_t *) vars->page_list_wired;
1694 err = IOPolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
1695 if (kIOReturnSuccess != err)
1696 break;
1697
1698 // mark more areas for no save, but these are not available
1699 // for trashing during restore
4452a7af
A		 1700
1701 hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
3a60a9f5 1702
3a60a9f5
A		 1703 page = atop_32(sectHIBB);
1704 count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page;
3a60a9f5
A		 1705 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1706 page, count,
1707 kIOHibernatePageStateFree);
1708 pageCount -= count;
3a60a9f5
A		 1709
1710 if (vars->previewBuffer) for (count = 0;
1711 (phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen));
1712 count += segLen)
1713 {
1714 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
1715 atop_64(phys64), atop_32(segLen),
1716 kIOHibernatePageStateFree);
1717 pageCount -= atop_32(segLen);
1718 }
1719
1720 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
1721
4452a7af
A		 1722 ppnum = 0;
1723 pagesDone = 0;
3a60a9f5
A		 1724
1725 HIBLOG("writing %d pages\n", pageCount);
1726
1727 do
1728 {
1729 count = hibernate_page_list_iterate(pageType ? vars->page_list : vars->page_list_wired,
4452a7af 1730 &ppnum);
3a60a9f5
A		 1731// kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
1732
1733 iterDone = !count;
1734
1735 pageAndCount[0] = ppnum;
1736 pageAndCount[1] = count;
1737 err = IOPolledFileWrite(vars->fileVars,
1738 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
1739 cryptvars);
1740 if (kIOReturnSuccess != err)
1741 break;
1742
1743 for (page = 0; page < count; page++)
1744 {
1745 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(ppnum), page_size);
1746 if (err)
1747 {
1748 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%d] %x\n", __LINE__, ppnum, err);
1749 break;
1750 }
1751
1752 sum = hibernate_sum(src, page_size);
1753
1754 clock_get_uptime(&startTime);
1755
1756 pageCompressedSize = WKdm_compress ((WK_word*) src, (WK_word*) (src + page_size), PAGE_SIZE_IN_WORDS);
1757
1758 clock_get_uptime(&endTime);
1759 ADD_ABSOLUTETIME(&compTime, &endTime);
1760 SUB_ABSOLUTETIME(&compTime, &startTime);
1761
1762 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1763 pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1);
1764
1765 if (pageCompressedSize > page_size)
1766 {
1767// HIBLOG("------------lose: %d\n", pageCompressedSize);
1768 pageCompressedSize = page_size;
1769 }
1770
1771 if (pageCompressedSize != page_size)
1772 data = (src + page_size);
1773 else
1774 data = src;
1775
1776 tag = pageCompressedSize | kIOHibernateTagSignature;
1777
1778 if (pageType)
1779 sum2 += sum;
1780 else
1781 sum1 += sum;
1782
1783 if (needEncryptStart && (ppnum >= atop_32(sectDATAB)))
1784 {
1785 // start encrypting partway into the data about to be written
1786 vars->fileVars->encryptStart = (vars->fileVars->position + AES_BLOCK_SIZE - 1)
1787 & ~(AES_BLOCK_SIZE - 1);
1788 needEncryptStart = false;
1789 }
1790
1791 err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
1792 if (kIOReturnSuccess != err)
1793 break;
1794
1795 err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
1796 if (kIOReturnSuccess != err)
1797 break;
1798
1799 compressedSize += pageCompressedSize;
1800 if (pageCompressedSize)
1801 uncompressedSize += page_size;
1802 ppnum++;
1803 pagesDone++;
1804
1805 if (0 == (8191 & pagesDone))
1806 {
1807 clock_get_uptime(&endTime);
1808 SUB_ABSOLUTETIME(&endTime, &allTime);
1809 absolutetime_to_nanoseconds(endTime, &nsec);
1810 progressStamp = nsec / 750000000ULL;
1811 if (progressStamp != lastProgressStamp)
1812 {
1813 lastProgressStamp = progressStamp;
1814 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
1815 }
1816 }
1817 }
1818 if (kIOReturnSuccess != err)
1819 break;
1820 if (iterDone && !pageType)
1821 {
1822 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
1823 if (kIOReturnSuccess != err)
1824 break;
1825
1826 iterDone = false;
1827 pageType = 1;
4452a7af 1828 ppnum = 0;
3a60a9f5
A		 1829 image1Size = vars->fileVars->position;
1830 if (cryptvars)
1831 {
1832 bcopy(&cryptvars->aes_iv[0],
1833 &gIOHibernateCryptWakeContext.aes_iv[0],
1834 sizeof(cryptvars->aes_iv));
1835 cryptvars = &gIOHibernateCryptWakeContext;
1836 }
1837 HIBLOG("image1Size %qd\n", image1Size);
1838 }
1839 }
1840 while (!iterDone);
1841 if (kIOReturnSuccess != err)
1842 break;
1843 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
1844 if (kIOReturnSuccess != err)
1845 break;
1846
1847 // Header:
1848
1849 header->imageSize = vars->fileVars->position;
1850 header->image1Size = image1Size;
1851 header->bitmapSize = bitmap_size;
1852 header->pageCount = pageCount;
1853 header->encryptStart = vars->fileVars->encryptStart;
1854
1855 header->restore1Sum = restore1Sum;
1856 header->image1Sum = sum1;
1857 header->image2Sum = sum2;
1858
1859 count = vars->fileExtents->getLength();
1860 if (count > sizeof(header->fileExtentMap))
1861 {
1862 header->fileExtentMapSize = count;
1863 count = sizeof(header->fileExtentMap);
1864 }
1865 else
1866 header->fileExtentMapSize = sizeof(header->fileExtentMap);
1867 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
1868
1869 IOPolledFileSeek(vars->fileVars, 0);
1870 err = IOPolledFileWrite(vars->fileVars,
1871 (uint8_t *) header, sizeof(IOHibernateImageHeader),
1872 cryptvars);
1873 if (kIOReturnSuccess != err)
1874 break;
1875 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
1876 if (kIOReturnSuccess != err)
1877 break;
1878 err = IOHibernatePollerIODone(vars->fileVars);
1879 if (kIOReturnSuccess != err)
1880 break;
1881 }
1882 while (false);
1883
1884 clock_get_uptime(&endTime);
1885 SUB_ABSOLUTETIME(&endTime, &allTime);
1886 absolutetime_to_nanoseconds(endTime, &nsec);
1887 HIBLOG("all time: %qd ms, ",
1888 nsec / 1000000ULL);
1889
1890 absolutetime_to_nanoseconds(compTime, &nsec);
1891 HIBLOG("comp time: %qd ms, ",
1892 nsec / 1000000ULL);
1893
1894 absolutetime_to_nanoseconds(decoTime, &nsec);
1895 HIBLOG("deco time: %qd ms, ",
1896 nsec / 1000000ULL);
1897
1898 HIBLOG("\nimage %qd, uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
1899 header->imageSize,
1900 uncompressedSize, atop_32(uncompressedSize), compressedSize,
8f6c56a5 1901 uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
3a60a9f5
A		 1902 sum1, sum2);
1903
1904 if (pollerOpen)
1905 IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState);
1906
1907 HIBLOG("hibernate_write_image done(%x)\n", err);
1908
1909 // should we come back via regular wake, set the state in memory.
1910 gIOHibernateState = kIOHibernateStateInactive;
1911
1912 if ((kIOReturnSuccess == err) && !(kIOHibernateModeSleep & gIOHibernateMode))
1913 return (true /* power down */ );
1914 else
1915 return (false /* sleep */ );
1916}
1917
1918/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1919
1920DECLARE_IOHIBERNATEPROGRESSALPHA
1921
1922static void
1923ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
1924{
1925 uint32_t rowBytes, pixelShift;
1926 uint32_t x, y;
1927 int32_t blob, lastBlob;
1928 uint32_t alpha, in, color, result;
1929 uint8_t * out;
1930 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1931
1932 pixelShift = display->depth >> 4;
1933 if (pixelShift < 1)
1934 return;
1935
1936 rowBytes = display->rowBytes;
1937
1938 screen += ((display->width
1939 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1940 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1941
1942 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
1943
1944 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
1945
1946 for (y = 0; y < kIOHibernateProgressHeight; y++)
1947 {
1948 out = screen + y * rowBytes;
1949 for (blob = firstBlob; blob <= lastBlob; blob++)
1950 {
1951 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
1952 for (x = 0; x < kIOHibernateProgressWidth; x++)
1953 {
1954 alpha = gIOHibernateProgressAlpha[y][x];
1955 result = color;
1956 if (alpha)
1957 {
1958 if (0xff != alpha)
1959 {
1960 in = display->progressSaveUnder[blob][saveindex[blob]++];
1961 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
1962 }
1963 if (1 == pixelShift)
1964 {
1965 result >>= 3;
1966 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1967 }
1968 else
1969 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1970 }
1971 out += (1 << pixelShift);
1972 }
1973 out += (kIOHibernateProgressSpacing << pixelShift);
1974 }
1975 }
1976}
1977
1978/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1979
1980extern "C" void
1981hibernate_machine_init(void)
1982{
1983 IOReturn err;
1984 uint32_t sum;
1985 uint32_t pagesDone;
1986 AbsoluteTime allTime, endTime;
1987 uint64_t nsec;
1988 uint32_t lastProgressStamp = 0;
1989 uint32_t progressStamp;
1990 uint64_t progressZeroPosition = 0;
1991 uint32_t blob, lastBlob = (uint32_t) -1L;
1992 hibernate_cryptvars_t * cryptvars = 0;
1993
1994 IOHibernateVars * vars = &gIOHibernateVars;
1995
1996 if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
1997 return;
1998
3a60a9f5
A		 1999 sum = gIOHibernateCurrentHeader->actualImage1Sum;
2000 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
2001
2002 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, image1Size %qx, conflictCount %d, nextFree %x\n",
2003 gIOHibernateState, pagesDone, sum, gIOHibernateCurrentHeader->image1Size,
2004 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
2005
2006 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
2007 {
2008 HIBLOG("regular wake\n");
2009 return;
2010 }
2011
2012 HIBPRINT("diag %x %x %x %x\n",
2013 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
2014 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
2015
4452a7af 2016 HIBPRINT("video %x %d %d %d\n",
3a60a9f5
A		 2017 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
2018 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height);
2019
4452a7af
A		 2020 if ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)
2021 hibernate_page_list_discard(vars->page_list);
2022
2023 boot_args *args = (boot_args *) PE_state.bootArgs;
2024
2025 if (vars->videoMapping
2026 && gIOHibernateGraphicsInfo->physicalAddress
2027 && (args->Video.v_baseAddr == gIOHibernateGraphicsInfo->physicalAddress))
3a60a9f5
A		 2028 {
2029 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
2030 * gIOHibernateGraphicsInfo->rowBytes);
2031 IOMapPages(kernel_map,
2032 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
2033 vars->videoMapSize, kIOMapInhibitCache );
2034 }
2035
4452a7af
A		 2036 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
2037
2038 if (gIOHibernateWakeMapSize)
2039 {
2040 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(gIOHibernateWakeMap),
2041 gIOHibernateWakeMapSize);
2042 if (kIOReturnSuccess == err)
2043 hibernate_newruntime_map(src, gIOHibernateWakeMapSize,
2044 gIOHibernateCurrentHeader->systemTableOffset);
2045 gIOHibernateWakeMap = 0;
2046 gIOHibernateWakeMapSize = 0;
2047 }
2048
3a60a9f5
A		 2049 uint32_t decoOffset;
2050
2051 clock_get_uptime(&allTime);
2052
2053 HIBLOG("IOHibernatePollerOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
2054 err = IOHibernatePollerOpen(vars->fileVars, kIOPolledAfterSleepState, 0);
2055 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
2056
2057 if (gIOHibernateCurrentHeader->previewSize)
2058 progressZeroPosition = gIOHibernateCurrentHeader->previewSize
2059 + gIOHibernateCurrentHeader->fileExtentMapSize
2060 - sizeof(gIOHibernateCurrentHeader->fileExtentMap)
2061 + ptoa_64(gIOHibernateCurrentHeader->restore1PageCount);
2062
2063 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
2064
2065 if (vars->videoMapping)
2066 {
2067 lastBlob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
2068 / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
2069 ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, 0, lastBlob);
2070 }
2071
2072 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
2073 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2074 {
2075 cryptvars = &gIOHibernateCryptWakeContext;
2076 bcopy(&gIOHibernateCryptWakeVars->aes_iv[0],
2077 &cryptvars->aes_iv[0],
2078 sizeof(cryptvars->aes_iv));
2079 }
2080
2081 // kick off the read ahead
2082 vars->fileVars->io = false;
2083 vars->fileVars->bufferHalf = 0;
2084 vars->fileVars->bufferLimit = 0;
2085 vars->fileVars->lastRead = 0;
2086 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2087
2088 IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
2089 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2090 // --
2091
2092 HIBLOG("hibernate_machine_init reading\n");
2093
2094 uint32_t * header = (uint32_t *) src;
2095 sum = 0;
2096
2097 do
2098 {
2099 unsigned int count;
2100 unsigned int page;
2101 uint32_t tag;
2102 vm_offset_t ppnum, compressedSize;
2103
2104 IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
2105
2106 ppnum = header[0];
2107 count = header[1];
2108
2109// HIBPRINT("(%x, %x)\n", ppnum, count);
2110
2111 if (!count)
2112 break;
2113
2114 for (page = 0; page < count; page++)
2115 {
2116 IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
2117
2118 compressedSize = kIOHibernateTagLength & tag;
2119 if (!compressedSize)
2120 {
2121 ppnum++;
2122 pagesDone++;
2123 continue;
2124 }
2125
2126 IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
2127
2128 if (compressedSize != page_size)
2129 {
2130 decoOffset = page_size;
2131 WKdm_decompress((WK_word*) src, (WK_word*) (src + decoOffset), PAGE_SIZE_IN_WORDS);
2132 }
2133 else
2134 decoOffset = 0;
2135
2136 sum += hibernate_sum((src + decoOffset), page_size);
2137
2138 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
2139 if (err)
2140 HIBLOG("IOMemoryDescriptorReadToPhysical [%d] %x\n", ppnum, err);
2141
2142 ppnum++;
2143 pagesDone++;
2144
2145 if (vars->videoMapping && (0 == (255 & pagesDone)))
2146 {
2147 blob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
2148 / (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
2149 if (blob != lastBlob)
2150 {
2151 ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, lastBlob, blob);
2152 lastBlob = blob;
2153 }
2154 }
2155
2156 if (0 == (8191 & pagesDone))
2157 {
2158 clock_get_uptime(&endTime);
2159 SUB_ABSOLUTETIME(&endTime, &allTime);
2160 absolutetime_to_nanoseconds(endTime, &nsec);
2161 progressStamp = nsec / 750000000ULL;
2162 if (progressStamp != lastProgressStamp)
2163 {
2164 lastProgressStamp = progressStamp;
2165 HIBPRINT("pages %d (%d%%)\n", pagesDone,
2166 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
2167 }
2168 }
2169 }
2170 }
2171 while (true);
2172
2173 gIOHibernateCurrentHeader->actualImage2Sum = sum;
2174
2175 if (vars->fileVars->io)
2176 (void) IOHibernatePollerIODone(vars->fileVars);
2177
2178 err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState);
2179
2180 if (vars->videoMapping)
2181 ProgressUpdate(gIOHibernateGraphicsInfo,
2182 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2183
2184 clock_get_uptime(&endTime);
2185 SUB_ABSOLUTETIME(&endTime, &allTime);
2186 absolutetime_to_nanoseconds(endTime, &nsec);
2187
2188 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %qd ms\n",
2189 pagesDone, sum, nsec / 1000000ULL);
2190}
2191
2192/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
mirror of XNU