]> git.saurik.com Git - apple/xnu.git/blame - iokit/Kernel/IOHibernateIO.cpp
projects / apple / xnu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
xnu-2422.100.13.tar.gz
[apple/xnu.git] / iokit / Kernel / IOHibernateIO.cpp
CommitLineData
3a60a9f5 1/*
b0d623f7 2 * Copyright (c) 2004-2008 Apple Computer, Inc. All rights reserved.
3a60a9f5 3 *
2d21ac55 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
3a60a9f5 5 *
2d21ac55
A		 6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
8f6c56a5 14 *
2d21ac55
A		 15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
8f6c56a5
A		 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
2d21ac55
A		 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
8f6c56a5 25 *
2d21ac55 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
3a60a9f5
A		 27 */
28
29
30/*
31
32Sleep:
33
34- PMRootDomain calls IOHibernateSystemSleep() before system sleep
35(devices awake, normal execution context)
36- IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
37 grabs its extents and searches for a polling driver willing to work with that IOMedia.
38 The BSD code makes an ioctl to the storage driver to get the partition base offset to
39 the disk, and other ioctls to get the transfer constraints
40 If successful, the file is written to make sure its initially not bootable (in case of
41 later failure) and nvram set to point to the first block of the file. (Has to be done
42 here so blocking is possible in nvram support).
43 hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
44 page out any pages it wants to (currently zero, but probably some percentage of memory).
45 Its assumed just allocating pages will cause the VM system to naturally select the best
46 pages for eviction. It also copies processor flags needed for the restore path and sets
47 a flag in the boot processor proc info.
48 gIOHibernateState = kIOHibernateStateHibernating.
49- Regular sleep progresses - some drivers may inspect the root domain property
50 kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
51 as usual but leaves motherboard I/O on.
52- Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
53 at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
54 all ppc RC bits out of the hash table and caches into the mapping structures.
55- hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
56 hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
57 All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
58 by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
59 The image header and block list are written. The header includes the second file extent so
60 only the header block is needed to read the file, regardless of filesystem.
316670eb 61 The kernel segment "__HIB" is written uncompressed to the image. This segment of code and data
3a60a9f5
A		 62 (only) is used to decompress the image during wake/boot.
63 Some additional pages are removed from the bitmaps - the buffers used for hibernation.
64 The bitmaps are written to the image.
65 More areas are removed from the bitmaps (after they have been written to the image) - the
316670eb 66 segment "__HIB" pages and interrupt stack.
3a60a9f5
A		 67 Each wired page is compressed and written and then each non-wired page. Compression and
68 disk writes are in parallel.
69 The image header is written to the start of the file and the polling driver closed.
70 The machine powers down (or sleeps).
71
72Boot/Wake:
73
74- BootX sees the boot-image nvram variable containing the device and block number of the image,
75 reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
76- BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
77 in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
78 that is in the kernel's __HIB section.
79- BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
80 hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
81 only code & data in that section is safe to call since all the other wired pages are still
82 compressed in the image.
83- hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
84 It uses the bitmaps to work out which pages can be uncompressed from the image to their final
85 location directly, and copies those that can't to interim free pages. When the image has been
86 completed, the copies are uncompressed, overwriting the wired image pages.
87 hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
88 is used to get pages into place for 64bit.
89- the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
90 changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
91 are removed from the software strutures, and the hash table is reinitialized.
92- After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
93 of memory, using the polled mode driver, before other threads can run or any devices are turned on.
94 This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
95 for the remaining non wired pages.
96- The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
97 (normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
98 is closed via bsd.
99
100Polled Mode I/O:
101
102IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
103registry, specifying an object of calls IOPolledInterface.
104
105Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
106partition) that the image is going to live, looking for polled interface properties. If it finds
107one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
108interfaces found are kept in an ordered list.
109
110There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
111few different contexts things happen in:
112
113- there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
114up and running) and after wake - this is safe to allocate memory and do anything. The device
115ignores sleep requests from that point since its a waste of time if it goes to sleep and
116immediately wakes back up for the image write.
117
118- there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
119immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
120from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
121used to flush and set the disk to sleep.
122
123- there is an Open/Close (AfterSleep) pair made around the image read operations that happen
124immediately after sleep. These can't block or allocate memory. This is happening after the platform
125expert has woken the low level bits of the system, but most of the I/O system has not. There is only
126one thread running.
127
128For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
129(prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
130the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
131Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
132that is called for the hardware to check for events, and complete the I/O via the callback.
133The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
134to restrict I/O ops.
135*/
136
137#include <sys/systm.h>
138
139#include <IOKit/IOWorkLoop.h>
140#include <IOKit/IOCommandGate.h>
141#include <IOKit/IOTimerEventSource.h>
142#include <IOKit/IOPlatformExpert.h>
143#include <IOKit/IOKitDebug.h>
144#include <IOKit/IOTimeStamp.h>
145#include <IOKit/pwr_mgt/RootDomain.h>
146#include <IOKit/pwr_mgt/IOPMPrivate.h>
147#include <IOKit/IOMessage.h>
148#include <IOKit/IODeviceTreeSupport.h>
149#include <IOKit/IOBSD.h>
150#include "RootDomainUserClient.h"
151#include <IOKit/pwr_mgt/IOPowerConnection.h>
152#include "IOPMPowerStateQueue.h"
153#include <IOKit/IOBufferMemoryDescriptor.h>
6d2010ae 154#include <IOKit/AppleKeyStoreInterface.h>
316670eb 155#include <libkern/crypto/aes.h>
3a60a9f5
A		 156
157#include <sys/uio.h>
158#include <sys/conf.h>
159#include <sys/stat.h>
160#include <sys/fcntl.h> // (FWRITE, ...)
3a60a9f5 161#include <sys/sysctl.h>
6d2010ae 162#include <sys/kdebug.h>
3a60a9f5
A		 163
164#include <IOKit/IOHibernatePrivate.h>
165#include <IOKit/IOPolledInterface.h>
166#include <IOKit/IONVRAM.h>
167#include "IOHibernateInternal.h"
39236c6e 168#include <vm/WKdm_new.h>
3a60a9f5 169#include "IOKitKernelInternal.h"
6d2010ae
A		 170#include <pexpert/device_tree.h>
171
172#include <machine/pal_routines.h>
173#include <machine/pal_hibernate.h>
39236c6e 174#include <i386/tsc.h>
6d2010ae
A		 175
176extern "C" addr64_t kvtophys(vm_offset_t va);
99c3a104 177extern "C" ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
3a60a9f5
A		 178
179/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
180
39236c6e
A		 181#define DISABLE_TRIM 0
182#define TRIM_DELAY 5000
183
0b4c1975 184extern unsigned int save_kdebug_enable;
3a60a9f5
A		 185extern uint32_t gIOHibernateState;
186uint32_t gIOHibernateMode;
187static char gIOHibernateBootSignature[256+1];
188static char gIOHibernateFilename[MAXPATHLEN+1];
db609669
A		 189static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
190uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms)
191static uint64_t gIOHibernateCompression = 0x80; // default compression 50%
3a60a9f5
A		 192
193static IODTNVRAM * gIOOptionsEntry;
194static IORegistryEntry * gIOChosenEntry;
b0d623f7 195#if defined(__i386__) || defined(__x86_64__)
0c530ab8 196static const OSSymbol * gIOCreateEFIDevicePathSymbol;
060df5ea
A		 197static const OSSymbol * gIOHibernateRTCVariablesKey;
198static const OSSymbol * gIOHibernateBoot0082Key;
199static const OSSymbol * gIOHibernateBootNextKey;
200static OSData * gIOHibernateBoot0082Data;
201static OSData * gIOHibernateBootNextData;
202static OSObject * gIOHibernateBootNextSave;
0c530ab8 203#endif
3a60a9f5 204
316670eb
A		 205static IOLock * gFSLock;
206static uint32_t gFSState;
3a60a9f5
A		 207static IOPolledFileIOVars gFileVars;
208static IOHibernateVars gIOHibernateVars;
209static struct kern_direct_file_io_ref_t * gIOHibernateFileRef;
210static hibernate_cryptvars_t gIOHibernateCryptWakeContext;
6d2010ae
A		 211static hibernate_graphics_t _hibernateGraphics;
212static hibernate_graphics_t * gIOHibernateGraphicsInfo = &_hibernateGraphics;
39236c6e
A		 213static hibernate_statistics_t _hibernateStats;
214static hibernate_statistics_t * gIOHibernateStats = &_hibernateStats;
3a60a9f5 215
316670eb
A		 216enum
217{
218 kFSIdle = 0,
219 kFSOpening = 2,
220 kFSOpened = 3,
221 kFSTimedOut = 4,
222};
223
224static IOReturn IOHibernateDone(IOHibernateVars * vars);
225
3a60a9f5
A		 226/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
227
228enum { kXPRamAudioVolume = 8 };
229enum { kDefaultIOSize = 128 * 1024 };
22ba694c 230enum { kVideoMapSize = 80 * 1024 * 1024 };
3a60a9f5
A		 231
232#ifndef kIOMediaPreferredBlockSizeKey
233#define kIOMediaPreferredBlockSizeKey "Preferred Block Size"
234#endif
235
236#ifndef kIOBootPathKey
237#define kIOBootPathKey "bootpath"
238#endif
239#ifndef kIOSelectedBootDeviceKey
240#define kIOSelectedBootDeviceKey "boot-device"
241#endif
242
3a60a9f5
A		 243/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
244
245// copy from phys addr to MD
246
247static IOReturn
248IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
249 IOByteCount offset, addr64_t bytes, IOByteCount length)
250{
251 addr64_t srcAddr = bytes;
252 IOByteCount remaining;
253
254 remaining = length = min(length, md->getLength() - offset);
255 while (remaining) { // (process another target segment?)
256 addr64_t dstAddr64;
257 IOByteCount dstLen;
258
b0d623f7 259 dstAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
3a60a9f5
A		 260 if (!dstAddr64)
261 break;
262
263 // Clip segment length to remaining
264 if (dstLen > remaining)
265 dstLen = remaining;
266
267#if 1
268 bcopy_phys(srcAddr, dstAddr64, dstLen);
269#else
270 copypv(srcAddr, dstAddr64, dstLen,
271 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
272#endif
273 srcAddr += dstLen;
274 offset += dstLen;
275 remaining -= dstLen;
276 }
277
278 assert(!remaining);
279
280 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
281}
282
283// copy from MD to phys addr
284
285static IOReturn
286IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
287 IOByteCount offset, addr64_t bytes, IOByteCount length)
288{
289 addr64_t dstAddr = bytes;
290 IOByteCount remaining;
291
292 remaining = length = min(length, md->getLength() - offset);
293 while (remaining) { // (process another target segment?)
294 addr64_t srcAddr64;
295 IOByteCount dstLen;
296
b0d623f7 297 srcAddr64 = md->getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
3a60a9f5
A		 298 if (!srcAddr64)
299 break;
300
301 // Clip segment length to remaining
302 if (dstLen > remaining)
303 dstLen = remaining;
304
305#if 1
306 bcopy_phys(srcAddr64, dstAddr, dstLen);
307#else
308 copypv(srcAddr, dstAddr64, dstLen,
309 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
310#endif
311 dstAddr += dstLen;
312 offset += dstLen;
313 remaining -= dstLen;
314 }
315
316 assert(!remaining);
317
318 return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
319}
320
321/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
322
323void
324hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
325 vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
326{
327 count += ppnum;
328 switch (kind)
329 {
330 case kIOHibernatePageStateUnwiredSave:
331 // unwired save
332 for (; ppnum < count; ppnum++)
333 {
334 hibernate_page_bitset(page_list, FALSE, ppnum);
335 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
336 }
337 break;
338 case kIOHibernatePageStateWiredSave:
339 // wired save
340 for (; ppnum < count; ppnum++)
341 {
342 hibernate_page_bitset(page_list, FALSE, ppnum);
343 hibernate_page_bitset(page_list_wired, FALSE, ppnum);
344 }
345 break;
346 case kIOHibernatePageStateFree:
347 // free page
348 for (; ppnum < count; ppnum++)
349 {
350 hibernate_page_bitset(page_list, TRUE, ppnum);
351 hibernate_page_bitset(page_list_wired, TRUE, ppnum);
352 }
353 break;
354 default:
355 panic("hibernate_set_page_state");
356 }
357}
358
359static vm_offset_t
0c530ab8 360hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
3a60a9f5 361{
0c530ab8
A		 362 uint32_t page = *pPage;
363 uint32_t count;
364 hibernate_bitmap_t * bitmap;
21362eb3 365
0c530ab8
A		 366 while ((bitmap = hibernate_page_bitmap_pin(list, &page)))
367 {
368 count = hibernate_page_bitmap_count(bitmap, TRUE, page);
369 if (!count)
370 break;
371 page += count;
372 if (page <= bitmap->last_page)
373 break;
374 }
6601e61a 375
0c530ab8
A		 376 *pPage = page;
377 if (bitmap)
378 count = hibernate_page_bitmap_count(bitmap, FALSE, page);
379 else
380 count = 0;
3a60a9f5
A		 381
382 return (count);
383}
384
385/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
386
387static IOReturn
388IOHibernatePollerProbe(IOPolledFileIOVars * vars, IOService * target)
389{
390 IOReturn err = kIOReturnError;
391 int32_t idx;
392 IOPolledInterface * poller;
393
394 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
395 {
396 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
397 err = poller->probe(target);
398 if (err)
399 {
400 HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
401 break;
402 }
403 }
404
405 return (err);
406}
407
408static IOReturn
409IOHibernatePollerOpen(IOPolledFileIOVars * vars, uint32_t state, IOMemoryDescriptor * md)
410{
411 IOReturn err = kIOReturnError;
412 int32_t idx;
413 IOPolledInterface * poller;
414
415 for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
416 {
417 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
418 err = poller->open(state, md);
419 if (err)
420 {
421 HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
422 break;
423 }
424 }
425
426 return (err);
427}
428
429static IOReturn
430IOHibernatePollerClose(IOPolledFileIOVars * vars, uint32_t state)
431{
432 IOReturn err = kIOReturnError;
433 int32_t idx;
434 IOPolledInterface * poller;
435
436 for (idx = 0;
437 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
438 idx++)
439 {
440 err = poller->close(state);
441 if (err)
442 HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
443 }
444
445 return (err);
446}
447
448static void
449IOHibernatePollerIOComplete(void * target,
450 void * parameter,
451 IOReturn status,
452 UInt64 actualByteCount)
453{
454 IOPolledFileIOVars * vars = (IOPolledFileIOVars *) parameter;
455
456 vars->ioStatus = status;
457}
458
459static IOReturn
460IOHibernatePollerIO(IOPolledFileIOVars * vars,
461 uint32_t operation, uint32_t bufferOffset,
462 uint64_t deviceOffset, uint64_t length)
463{
464
465 IOReturn err = kIOReturnError;
466 IOPolledInterface * poller;
467 IOPolledCompletion completion;
468
469 completion.target = 0;
470 completion.action = &IOHibernatePollerIOComplete;
471 completion.parameter = vars;
472
473 vars->ioStatus = -1;
474
475 poller = (IOPolledInterface *) vars->pollers->getObject(0);
476 err = poller->startIO(operation, bufferOffset, deviceOffset + vars->block0, length, completion);
477 if (err)
478 HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
479
480 return (err);
481}
482
483static IOReturn
2d21ac55 484IOHibernatePollerIODone(IOPolledFileIOVars * vars, bool abortable)
3a60a9f5 485{
2d21ac55
A		 486 IOReturn err = kIOReturnSuccess;
487 int32_t idx = 0;
3a60a9f5
A		 488 IOPolledInterface * poller;
489
490 while (-1 == vars->ioStatus)
491 {
2d21ac55
A		 492 for (idx = 0;
493 (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
3a60a9f5
A		 494 idx++)
495 {
2d21ac55
A		 496 IOReturn newErr;
497 newErr = poller->checkForWork();
498 if ((newErr == kIOReturnAborted) && !abortable)
499 newErr = kIOReturnSuccess;
500 if (kIOReturnSuccess == err)
501 err = newErr;
3a60a9f5
A		 502 }
503 }
504
0b4c1975
A		 505 if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort())
506 {
507 err = kIOReturnAborted;
508 HIBLOG("IOPolledInterface::checkForWork sw abort\n");
509 }
510
2d21ac55
A		 511 if (err)
512 {
513 HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
514 }
515 else
516 {
517 err = vars->ioStatus;
518 if (kIOReturnSuccess != err)
519 HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
520 }
3a60a9f5 521
2d21ac55 522 return (err);
3a60a9f5
A		 523}
524
525IOReturn
526IOPolledInterface::checkAllForWork(void)
527{
528 IOReturn err = kIOReturnNotReady;
529 int32_t idx;
530 IOPolledInterface * poller;
531
532 IOHibernateVars * vars = &gIOHibernateVars;
533
534 if (!vars->fileVars || !vars->fileVars->pollers)
535 return (err);
536
537 for (idx = 0;
538 (poller = (IOPolledInterface *) vars->fileVars->pollers->getObject(idx));
539 idx++)
540 {
541 err = poller->checkForWork();
542 if (err)
543 HIBLOG("IOPolledInterface::checkAllForWork[%d] 0x%x\n", idx, err);
544 }
545
546 return (err);
547}
548
549/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
550
551struct _OpenFileContext
552{
553 OSData * extents;
554 uint64_t size;
555};
556
557static void
558file_extent_callback(void * ref, uint64_t start, uint64_t length)
559{
560 _OpenFileContext * ctx = (_OpenFileContext *) ref;
561 IOPolledFileExtent extent;
562
563 extent.start = start;
564 extent.length = length;
565
39236c6e
A		 566 HIBLOG("[0x%qx, 0x%qx]\n", start, length);
567
3a60a9f5
A		 568 ctx->extents->appendBytes(&extent, sizeof(extent));
569 ctx->size += length;
570}
571
6d2010ae
A		 572static IOService *
573IOCopyMediaForDev(dev_t device)
574{
575 OSDictionary * matching;
576 OSNumber * num;
577 OSIterator * iter;
578 IOService * result = 0;
579
580 matching = IOService::serviceMatching("IOMedia");
581 if (!matching)
582 return (0);
583 do
584 {
585 num = OSNumber::withNumber(major(device), 32);
586 if (!num)
587 break;
588 matching->setObject(kIOBSDMajorKey, num);
589 num->release();
590 num = OSNumber::withNumber(minor(device), 32);
591 if (!num)
592 break;
593 matching->setObject(kIOBSDMinorKey, num);
594 num->release();
595 if (!num)
596 break;
597 iter = IOService::getMatchingServices(matching);
598 if (iter)
599 {
600 result = (IOService *) iter->getNextObject();
601 result->retain();
602 iter->release();
603 }
604 }
605 while (false);
606 matching->release();
607
608 return (result);
609}
610
3a60a9f5 611IOReturn
db609669
A		 612IOPolledFileOpen( const char * filename, uint64_t setFileSize,
613 IOBufferMemoryDescriptor * ioBuffer,
614 IOPolledFileIOVars ** fileVars, OSData ** fileExtents,
615 OSData ** imagePath, uint8_t * volumeCryptKey)
3a60a9f5 616{
316670eb 617 IOReturn err = kIOReturnSuccess;
3a60a9f5
A		 618 IOPolledFileIOVars * vars;
619 _OpenFileContext ctx;
620 OSData * extentsData;
621 OSNumber * num;
6d2010ae
A		 622 IOService * part = 0;
623 OSString * keyUUID = 0;
624 OSString * keyStoreUUID = 0;
625 dev_t block_dev;
3a60a9f5
A		 626 dev_t hibernate_image_dev;
627 uint64_t maxiobytes;
316670eb
A		 628 AbsoluteTime startTime, endTime;
629 uint64_t nsec;
630
631 vars = IONew(IOPolledFileIOVars, 1);
632 if (!vars) return (kIOReturnNoMemory);
633 bzero(vars, sizeof(*vars));
3a60a9f5 634
3a60a9f5
A		 635 do
636 {
3a60a9f5
A		 637 vars->io = false;
638 vars->buffer = (uint8_t *) ioBuffer->getBytesNoCopy();
639 vars->bufferHalf = 0;
640 vars->bufferOffset = 0;
641 vars->bufferSize = ioBuffer->getLength() >> 1;
642
643 extentsData = OSData::withCapacity(32);
316670eb 644 ctx.extents = extentsData;
3a60a9f5 645 ctx.size = 0;
316670eb 646 clock_get_uptime(&startTime);
3a60a9f5
A		 647 vars->fileRef = kern_open_file_for_direct_io(filename,
648 &file_extent_callback, &ctx,
db609669
A		 649 setFileSize,
650 // write file:
651 0, (caddr_t) gIOHibernateCurrentHeader,
652 sizeof(IOHibernateImageHeader),
653 // results
6d2010ae 654 &block_dev,
3a60a9f5
A		 655 &hibernate_image_dev,
656 &vars->block0,
0b4c1975 657 &maxiobytes,
db609669 658 &vars->flags);
316670eb
A		 659#if 0
660 uint32_t msDelay = (131071 & random());
661 HIBLOG("sleep %d\n", msDelay);
662 IOSleep(msDelay);
663#endif
664 clock_get_uptime(&endTime);
665 SUB_ABSOLUTETIME(&endTime, &startTime);
666 absolutetime_to_nanoseconds(endTime, &nsec);
667
668 if (!vars->fileRef) err = kIOReturnNoSpace;
669
670 IOLockLock(gFSLock);
671 if (kFSOpening != gFSState) err = kIOReturnTimeout;
672 IOLockUnlock(gFSLock);
673
674 HIBLOG("kern_open_file_for_direct_io(%d) took %qd ms\n", err, nsec / 1000000ULL);
675 if (kIOReturnSuccess != err) break;
0b4c1975
A		 676
677 if (kIOHibernateModeSSDInvert & gIOHibernateMode)
6d2010ae 678 vars->flags ^= kIOHibernateOptionSSD;
0b4c1975
A		 679
680 HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx ssd %d\n", filename, ctx.size,
6d2010ae 681 vars->block0, maxiobytes, kIOHibernateOptionSSD & vars->flags);
3a60a9f5
A		 682 if (ctx.size < 1*1024*1024) // check against image size estimate!
683 {
684 err = kIOReturnNoSpace;
685 break;
686 }
687
db609669 688 vars->fileSize = ctx.size;
39236c6e 689 if (maxiobytes < vars->bufferSize) vars->bufferSize = maxiobytes;
3a60a9f5
A		 690
691 vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
2d21ac55 692
6d2010ae
A		 693 part = IOCopyMediaForDev(block_dev);
694 if (!part)
39236c6e
A		 695 {
696 err = kIOReturnNotFound;
6d2010ae 697 break;
39236c6e 698 }
6d2010ae
A		 699 err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false,
700 (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL);
701 if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID)
702 {
703// IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy());
704 uuid_t volumeKeyUUID;
705 aks_volume_key_t vek;
706 static IOService * sKeyStore;
707 static const OSSymbol * sAKSGetKey;
708
709 if (!sAKSGetKey)
710 sAKSGetKey = OSSymbol::withCStringNoCopy(AKS_PLATFORM_FUNCTION_GETKEY);
711 if (!sKeyStore)
712 sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane);
713 if (sKeyStore)
714 err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID);
715 else
716 err = kIOReturnNoResources;
717 if (kIOReturnSuccess == err)
718 err = sKeyStore->callPlatformFunction(sAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL);
719 if (kIOReturnSuccess != err)
720 IOLog("volume key err 0x%x\n", err);
721 else
722 {
723 size_t bytes = (kIOHibernateAESKeySize / 8);
724 if (vek.key.keybytecount < bytes)
725 bytes = vek.key.keybytecount;
726 bcopy(&vek.key.keybytes[0], volumeCryptKey, bytes);
727 }
728 bzero(&vek, sizeof(vek));
729 }
730 part->release();
731
732 part = IOCopyMediaForDev(hibernate_image_dev);
733 if (!part)
39236c6e
A		 734 {
735 err = kIOReturnNotFound;
6d2010ae 736 break;
39236c6e 737 }
6d2010ae 738
3a60a9f5
A		 739 IORegistryEntry * next;
740 IORegistryEntry * child;
15129b1c
A		 741 IOService * service;
742 OSData * data;
3a60a9f5 743
3a60a9f5
A		 744 vars->pollers = OSArray::withCapacity(4);
745 if (!vars->pollers)
39236c6e
A		 746 {
747 err = kIOReturnNoMemory;
748 break;
749 }
3a60a9f5
A		 750
751 vars->blockSize = 512;
752 next = part;
753 do
754 {
755 IOPolledInterface * poller;
c0fea474
A		 756 OSObject * obj;
757
758 obj = next->getProperty(kIOPolledInterfaceSupportKey);
759 if (kOSBooleanFalse == obj)
760 {
761 vars->pollers->flushCollection();
762 break;
763 }
764 else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
3a60a9f5 765 vars->pollers->setObject(poller);
15129b1c
A		 766
767 if ((service = OSDynamicCast(IOService, next))
768 && service->getDeviceMemory()
769 && !vars->pollers->getCount()) break;
770
3a60a9f5
A		 771 if ((num = OSDynamicCast(OSNumber, next->getProperty(kIOMediaPreferredBlockSizeKey))))
772 vars->blockSize = num->unsigned32BitValue();
773 child = next;
774 }
775 while ((next = child->getParentEntry(gIOServicePlane))
776 && child->isParent(next, gIOServicePlane, true));
777
39236c6e
A		 778 if (vars->blockSize < 4096) vars->blockSize = 4096;
779
3a60a9f5 780 HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n",
15129b1c
A		 781 major(hibernate_image_dev), minor(hibernate_image_dev), (long)vars->blockSize,
782 vars->pollers->getCount());
39236c6e 783
15129b1c 784 if (!vars->pollers->getCount())
39236c6e
A		 785 {
786 err = kIOReturnUnsupported;
3a60a9f5 787 continue;
39236c6e
A		 788 }
789 if (vars->blockSize < sizeof(IOHibernateImageHeader))
790 {
791 err = kIOReturnError;
792 continue;
793 }
3a60a9f5
A		 794
795 err = IOHibernatePollerProbe(vars, (IOService *) part);
39236c6e 796 if (kIOReturnSuccess != err) break;
3a60a9f5
A		 797
798 err = IOHibernatePollerOpen(vars, kIOPolledPreflightState, ioBuffer);
39236c6e 799 if (kIOReturnSuccess != err) break;
3a60a9f5 800
bd504ef0
A		 801 vars->media = part;
802 next = part;
803 while (next)
804 {
805 next->setProperty(kIOPolledInterfaceActiveKey, kOSBooleanTrue);
806 next = next->getParentEntry(gIOServicePlane);
807 }
808
3a60a9f5
A		 809 *fileVars = vars;
810 *fileExtents = extentsData;
811
812 // make imagePath
3a60a9f5 813
0c530ab8 814 if ((extentsData->getLength() >= sizeof(IOPolledFileExtent)))
3a60a9f5 815 {
6d2010ae 816 char str2[24 + sizeof(uuid_string_t) + 2];
0c530ab8 817
b0d623f7 818#if defined(__i386__) || defined(__x86_64__)
0c530ab8
A		 819 if (!gIOCreateEFIDevicePathSymbol)
820 gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
821
6d2010ae
A		 822 if (keyUUID)
823 snprintf(str2, sizeof(str2), "%qx:%s",
824 vars->extentMap[0].start, keyUUID->getCStringNoCopy());
825 else
826 snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
0c530ab8
A		 827
828 err = IOService::getPlatform()->callPlatformFunction(
829 gIOCreateEFIDevicePathSymbol, false,
6d2010ae
A		 830 (void *) part, (void *) str2,
831 (void *) (uintptr_t) true, (void *) &data);
0c530ab8
A		 832#else
833 char str1[256];
834 int len = sizeof(str1);
835
836 if (!part->getPath(str1, &len, gIODTPlane))
837 err = kIOReturnNotFound;
838 else
839 {
2d21ac55 840 snprintf(str2, sizeof(str2), ",%qx", vars->extentMap[0].start);
0c530ab8
A		 841 // (strip the plane name)
842 char * tail = strchr(str1, ':');
843 if (!tail)
844 tail = str1 - 1;
845 data = OSData::withBytes(tail + 1, strlen(tail + 1));
846 data->appendBytes(str2, strlen(str2));
847 }
848#endif
849 if (kIOReturnSuccess == err)
3a60a9f5 850 *imagePath = data;
0c530ab8
A		 851 else
852 HIBLOG("error 0x%x getting path\n", err);
3a60a9f5
A		 853 }
854 }
855 while (false);
856
857 if (kIOReturnSuccess != err)
858 {
859 HIBLOG("error 0x%x opening hibernation file\n", err);
860 if (vars->fileRef)
b0d623f7 861 {
7ddcb079 862 kern_close_file_for_direct_io(vars->fileRef, 0, 0, 0, 0, 0);
316670eb 863 vars->fileRef = NULL;
b0d623f7 864 }
3a60a9f5
A		 865 }
866
867 if (part)
868 part->release();
869
870 return (err);
871}
872
873IOReturn
874IOPolledFileClose( IOPolledFileIOVars * vars )
875{
876 if (vars->pollers)
877 {
878 IOHibernatePollerClose(vars, kIOPolledPostflightState);
879 vars->pollers->release();
880 }
881
3a60a9f5
A		 882 bzero(vars, sizeof(IOPolledFileIOVars));
883
884 return (kIOReturnSuccess);
885}
886
887static IOReturn
888IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
889{
890 IOPolledFileExtent * extentMap;
891
892 extentMap = vars->extentMap;
893
894 vars->position = position;
895
896 while (position >= extentMap->length)
897 {
898 position -= extentMap->length;
899 extentMap++;
900 }
901
902 vars->currentExtent = extentMap;
903 vars->extentRemaining = extentMap->length - position;
904 vars->extentPosition = vars->position - position;
905
906 if (vars->bufferSize <= vars->extentRemaining)
907 vars->bufferLimit = vars->bufferSize;
908 else
909 vars->bufferLimit = vars->extentRemaining;
910
911 return (kIOReturnSuccess);
912}
913
914static IOReturn
915IOPolledFileWrite(IOPolledFileIOVars * vars,
916 const uint8_t * bytes, IOByteCount size,
917 hibernate_cryptvars_t * cryptvars)
918{
919 IOReturn err = kIOReturnSuccess;
920 IOByteCount copy;
921 bool flush = false;
922
923 do
924 {
925 if (!bytes && !size)
926 {
927 // seek to end of block & flush
928 size = vars->position & (vars->blockSize - 1);
929 if (size)
930 size = vars->blockSize - size;
931 flush = true;
932 // use some garbage for the fill
933 bytes = vars->buffer + vars->bufferOffset;
934 }
935
936 copy = vars->bufferLimit - vars->bufferOffset;
937 if (copy > size)
938 copy = size;
939 else
940 flush = true;
941
942 if (bytes)
943 {
944 bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
945 bytes += copy;
946 }
947 else
948 bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
949
950 size -= copy;
951 vars->bufferOffset += copy;
952 vars->position += copy;
953
954 if (flush && vars->bufferOffset)
955 {
956 uint64_t offset = (vars->position - vars->bufferOffset
957 - vars->extentPosition + vars->currentExtent->start);
958 uint32_t length = (vars->bufferOffset);
959
2d21ac55 960#if CRYPTO
0b4c1975
A		 961 if (cryptvars && vars->encryptStart
962 && (vars->position > vars->encryptStart)
963 && ((vars->position - length) < vars->encryptEnd))
3a60a9f5 964 {
6d2010ae
A		 965 AbsoluteTime startTime, endTime;
966
7ddcb079 967 uint64_t encryptLen, encryptStart;
3a60a9f5
A		 968 encryptLen = vars->position - vars->encryptStart;
969 if (encryptLen > length)
970 encryptLen = length;
971 encryptStart = length - encryptLen;
0b4c1975
A		 972 if (vars->position > vars->encryptEnd)
973 encryptLen -= (vars->position - vars->encryptEnd);
974
6d2010ae
A		 975 clock_get_uptime(&startTime);
976
3a60a9f5
A		 977 // encrypt the buffer
978 aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
979 &cryptvars->aes_iv[0],
980 encryptLen / AES_BLOCK_SIZE,
981 vars->buffer + vars->bufferHalf + encryptStart,
982 &cryptvars->ctx.encrypt);
6d2010ae
A		 983
984 clock_get_uptime(&endTime);
985 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
986 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
987 vars->cryptBytes += encryptLen;
988
3a60a9f5
A		 989 // save initial vector for following encrypts
990 bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
991 &cryptvars->aes_iv[0],
992 AES_BLOCK_SIZE);
993 }
2d21ac55 994#endif /* CRYPTO */
3a60a9f5
A		 995
996 if (vars->io)
997 {
2d21ac55 998 err = IOHibernatePollerIODone(vars, true);
3a60a9f5
A		 999 if (kIOReturnSuccess != err)
1000 break;
1001 }
1002
1003if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
1004//if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
1005
1006 err = IOHibernatePollerIO(vars, kIOPolledWrite, vars->bufferHalf, offset, length);
1007 if (kIOReturnSuccess != err)
1008 break;
1009 vars->io = true;
1010
1011 vars->extentRemaining -= vars->bufferOffset;
1012 if (!vars->extentRemaining)
1013 {
1014 vars->currentExtent++;
1015 vars->extentRemaining = vars->currentExtent->length;
1016 vars->extentPosition = vars->position;
1017 if (!vars->extentRemaining)
1018 {
1019 err = kIOReturnOverrun;
1020 break;
1021 }
1022 }
1023
1024 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
1025 vars->bufferOffset = 0;
1026 if (vars->bufferSize <= vars->extentRemaining)
1027 vars->bufferLimit = vars->bufferSize;
1028 else
1029 vars->bufferLimit = vars->extentRemaining;
1030
1031 flush = false;
1032 }
1033 }
1034 while (size);
1035
1036 return (err);
1037}
1038
1039static IOReturn
1040IOPolledFileRead(IOPolledFileIOVars * vars,
1041 uint8_t * bytes, IOByteCount size,
1042 hibernate_cryptvars_t * cryptvars)
1043{
1044 IOReturn err = kIOReturnSuccess;
1045 IOByteCount copy;
1046
1047// bytesWritten += size;
1048
1049 do
1050 {
1051 copy = vars->bufferLimit - vars->bufferOffset;
1052 if (copy > size)
1053 copy = size;
1054
1055 if (bytes)
1056 {
1057 bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
1058 bytes += copy;
1059 }
1060 size -= copy;
1061 vars->bufferOffset += copy;
1062// vars->position += copy;
1063
6d2010ae 1064 if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd))
3a60a9f5
A		 1065 {
1066 if (vars->io)
1067 {
2d21ac55 1068 err = IOHibernatePollerIODone(vars, false);
3a60a9f5
A		 1069 if (kIOReturnSuccess != err)
1070 break;
1071 }
1072 else
1073 cryptvars = 0;
1074
1075if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
1076
6d2010ae 1077 vars->position += vars->lastRead;
3a60a9f5 1078 vars->extentRemaining -= vars->lastRead;
6d2010ae 1079 vars->bufferLimit = vars->lastRead;
3a60a9f5
A		 1080
1081 if (!vars->extentRemaining)
1082 {
1083 vars->currentExtent++;
1084 vars->extentRemaining = vars->currentExtent->length;
1085 vars->extentPosition = vars->position;
1086 if (!vars->extentRemaining)
1087 {
1088 err = kIOReturnOverrun;
1089 break;
1090 }
1091 }
1092
36401178
A		 1093 uint64_t length;
1094 uint64_t lastReadLength = vars->lastRead;
3a60a9f5
A		 1095 uint64_t offset = (vars->position
1096 - vars->extentPosition + vars->currentExtent->start);
36401178
A		 1097 if (vars->extentRemaining <= vars->bufferSize)
1098 length = vars->extentRemaining;
1099 else
1100 length = vars->bufferSize;
6d2010ae
A		 1101 if ((length + vars->position) > vars->readEnd)
1102 length = vars->readEnd - vars->position;
3a60a9f5 1103
6d2010ae
A		 1104 vars->lastRead = length;
1105 if (length)
1106 {
3a60a9f5 1107//if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
6d2010ae
A		 1108 err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length);
1109 if (kIOReturnSuccess != err)
1110 break;
1111 vars->io = true;
1112 }
3a60a9f5
A		 1113
1114 vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
1115 vars->bufferOffset = 0;
1116
2d21ac55 1117#if CRYPTO
3a60a9f5
A		 1118 if (cryptvars)
1119 {
1120 uint8_t thisVector[AES_BLOCK_SIZE];
6d2010ae
A		 1121 AbsoluteTime startTime, endTime;
1122
3a60a9f5
A		 1123 // save initial vector for following decrypts
1124 bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
36401178 1125 bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE,
3a60a9f5 1126 &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
6d2010ae 1127
3a60a9f5 1128 // decrypt the buffer
6d2010ae
A		 1129 clock_get_uptime(&startTime);
1130
3a60a9f5
A		 1131 aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
1132 &thisVector[0],
36401178 1133 lastReadLength / AES_BLOCK_SIZE,
3a60a9f5
A		 1134 vars->buffer + vars->bufferHalf,
1135 &cryptvars->ctx.decrypt);
6d2010ae
A		 1136
1137 clock_get_uptime(&endTime);
1138 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
1139 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
1140 vars->cryptBytes += lastReadLength;
3a60a9f5 1141 }
b0d623f7 1142#endif /* CRYPTO */
3a60a9f5
A		 1143 }
1144 }
1145 while (size);
1146
1147 return (err);
1148}
1149
1150/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
99c3a104 1151
3a60a9f5
A		 1152IOReturn
1153IOHibernateSystemSleep(void)
1154{
1155 IOReturn err;
1156 OSData * data;
1157 OSObject * obj;
1158 OSString * str;
db609669 1159 OSNumber * num;
39236c6e 1160 bool dsSSD, vmflush;
316670eb 1161 IOHibernateVars * vars;
3a60a9f5
A		 1162
1163 gIOHibernateState = kIOHibernateStateInactive;
1164
316670eb
A		 1165 if (!gIOChosenEntry)
1166 gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
1167
b0d623f7
A		 1168 gIOHibernateDebugFlags = 0;
1169 if (kIOLogHibernate & gIOKitDebug)
1170 gIOHibernateDebugFlags |= kIOHibernateDebugRestoreLogs;
1171
0b4c1975
A		 1172 if (IOService::getPMRootDomain()->getHibernateSettings(
1173 &gIOHibernateMode, &gIOHibernateFreeRatio, &gIOHibernateFreeTime))
6d2010ae 1174 {
3a60a9f5
A		 1175 if (kIOHibernateModeSleep & gIOHibernateMode)
1176 // default to discard clean for safe sleep
1177 gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
1178 | kIOHibernateModeDiscardCleanActive);
6d2010ae 1179 }
3a60a9f5 1180
3a60a9f5
A		 1181 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
1182 {
1183 if ((str = OSDynamicCast(OSString, obj)))
2d21ac55
A		 1184 strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
1185 sizeof(gIOHibernateFilename));
3a60a9f5
A		 1186 obj->release();
1187 }
1188
1189 if (!gIOHibernateMode || !gIOHibernateFilename[0])
1190 return (kIOReturnUnsupported);
1191
1192 HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
1193
316670eb
A		 1194 vars = IONew(IOHibernateVars, 1);
1195 if (!vars) return (kIOReturnNoMemory);
1196 bzero(vars, sizeof(*vars));
1197
1198 IOLockLock(gFSLock);
1199 if (kFSIdle != gFSState)
1200 {
1201 HIBLOG("hibernate file busy\n");
1202 IOLockUnlock(gFSLock);
1203 IODelete(vars, IOHibernateVars, 1);
1204 return (kIOReturnBusy);
1205 }
1206 gFSState = kFSOpening;
1207 IOLockUnlock(gFSLock);
2d21ac55 1208
3a60a9f5
A		 1209 do
1210 {
0c530ab8 1211 vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
39236c6e 1212 2 * page_size + WKdm_SCRATCH_BUF_SIZE, page_size);
0c530ab8
A		 1213 vars->ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1214 2 * kDefaultIOSize, page_size);
3a60a9f5 1215
6d2010ae
A		 1216 vars->handoffBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
1217 ptoa_64(gIOHibernateHandoffPageCount), page_size);
1218
1219 if (!vars->srcBuffer || !vars->ioBuffer || !vars->handoffBuffer)
3a60a9f5
A		 1220 {
1221 err = kIOReturnNoMemory;
1222 break;
1223 }
1224
db609669
A		 1225 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMinSizeKey)))
1226 {
1227 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMinSize = num->unsigned64BitValue();
1228 obj->release();
1229 }
1230 if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileMaxSizeKey)))
1231 {
1232 if ((num = OSDynamicCast(OSNumber, obj))) vars->fileMaxSize = num->unsigned64BitValue();
1233 obj->release();
1234 }
1235
1236 boolean_t encryptedswap = true;
1237 uint32_t pageCount;
1238 AbsoluteTime startTime, endTime;
1239 uint64_t nsec;
1240
1241 bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
1242 gIOHibernateCurrentHeader->debugFlags = gIOHibernateDebugFlags;
6d2010ae 1243 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
db609669 1244
39236c6e 1245 vmflush = (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey));
db609669 1246 uint64_t setFileSize = 0;
39236c6e
A		 1247 err = hibernate_alloc_page_lists(&vars->page_list,
1248 &vars->page_list_wired,
1249 &vars->page_list_pal);
1250 if (KERN_SUCCESS != err)
1251 break;
db609669
A		 1252
1253 if (vars->fileMinSize || (kIOHibernateModeFileResize & gIOHibernateMode))
1254 {
1255 hibernate_page_list_setall(vars->page_list,
1256 vars->page_list_wired,
1257 vars->page_list_pal,
1258 true /* preflight */,
39236c6e 1259 vmflush /* discard */,
db609669
A		 1260 &pageCount);
1261 PE_Video consoleInfo;
1262 bzero(&consoleInfo, sizeof(consoleInfo));
1263 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1264
1265 // estimate: 5% increase in pages compressed
1266 // screen preview 2 images compressed 50%
1267 setFileSize = ((ptoa_64((105 * pageCount) / 100) * gIOHibernateCompression) >> 8)
1268 + vars->page_list->list_size
1269 + (consoleInfo.v_width * consoleInfo.v_height * 4);
1270
1271 HIBLOG("hibernate_page_list_setall preflight pageCount %d est comp %qd setfile %qd min %qd\n",
1272 pageCount, (100ULL * gIOHibernateCompression) >> 8,
1273 setFileSize, vars->fileMinSize);
1274
1275 if (!(kIOHibernateModeFileResize & gIOHibernateMode)
1276 && (setFileSize < vars->fileMinSize))
1277 {
1278 setFileSize = vars->fileMinSize;
1279 }
1280 }
1281
1282 // open & invalidate the image file
1283
1284 err = IOPolledFileOpen(gIOHibernateFilename, setFileSize, vars->ioBuffer,
6d2010ae
A		 1285 &vars->fileVars, &vars->fileExtents, &data,
1286 &vars->volumeCryptKey[0]);
3a60a9f5
A		 1287 if (KERN_SUCCESS != err)
1288 {
1289 HIBLOG("IOPolledFileOpen(%x)\n", err);
1290 break;
1291 }
3a60a9f5 1292
39236c6e
A		 1293 clock_get_uptime(&startTime);
1294 err = hibernate_setup(gIOHibernateCurrentHeader,
1295 gIOHibernateFreeRatio, gIOHibernateFreeTime,
1296 vmflush,
1297 vars->page_list, vars->page_list_wired, vars->page_list_pal);
1298 clock_get_uptime(&endTime);
1299 SUB_ABSOLUTETIME(&endTime, &startTime);
1300 absolutetime_to_nanoseconds(endTime, &nsec);
1301 HIBLOG("hibernate_setup(%d) took %qd ms\n", err, nsec / 1000000ULL);
1302
6d2010ae 1303 dsSSD = ((0 != (kIOHibernateOptionSSD & vars->fileVars->flags))
0b4c1975 1304 && (kOSBooleanTrue == IOService::getPMRootDomain()->getProperty(kIOPMDeepSleepEnabledKey)));
0b4c1975
A		 1305 if (dsSSD)
1306 {
1307 gIOHibernateCurrentHeader->options |=
1308 kIOHibernateOptionSSD
1309 | kIOHibernateOptionColor;
6d2010ae
A		 1310
1311#if defined(__i386__) || defined(__x86_64__)
1312 if (!uuid_is_null(vars->volumeCryptKey) &&
1313 (kOSBooleanTrue != IOService::getPMRootDomain()->getProperty(kIOPMDestroyFVKeyOnStandbyKey)))
1314 {
1315 uintptr_t smcVars[2];
1316 smcVars[0] = sizeof(vars->volumeCryptKey);
316670eb 1317 smcVars[1] = (uintptr_t)(void *) &gIOHibernateVars.volumeCryptKey[0];
6d2010ae
A		 1318
1319 IOService::getPMRootDomain()->setProperty(kIOHibernateSMCVariablesKey, smcVars, sizeof(smcVars));
1320 bzero(smcVars, sizeof(smcVars));
1321 }
1322#endif
0b4c1975
A		 1323 }
1324 else
1325 {
1326 gIOHibernateCurrentHeader->options |= kIOHibernateOptionProgress;
1327 }
1328
0b4c1975 1329
3a60a9f5 1330 if (KERN_SUCCESS != err)
3a60a9f5 1331 break;
3a60a9f5 1332
6d2010ae 1333 if (encryptedswap || !uuid_is_null(vars->volumeCryptKey))
3a60a9f5
A		 1334 gIOHibernateMode ^= kIOHibernateModeEncrypt;
1335
0b4c1975
A		 1336 if (kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1337 {
1338 vars->videoAllocSize = kVideoMapSize;
1339 if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize))
1340 vars->videoMapping = 0;
1341 }
3a60a9f5
A		 1342
1343 // generate crypt keys
1344 for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
1345 vars->wiredCryptKey[i] = random();
1346 for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
1347 vars->cryptKey[i] = random();
1348
1349 // set nvram
1350
1351 IORegistryEntry * regEntry;
1352 if (!gIOOptionsEntry)
1353 {
1354 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
1355 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
1356 if (regEntry && !gIOOptionsEntry)
1357 regEntry->release();
1358 }
3a60a9f5
A		 1359
1360 if (gIOOptionsEntry)
1361 {
1362 const OSSymbol * sym;
3a60a9f5
A		 1363
1364 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
1365 if (sym)
1366 {
1367 gIOOptionsEntry->setProperty(sym, data);
1368 sym->release();
1369 }
1370 data->release();
1371
b0d623f7 1372#if defined(__i386__) || defined(__x86_64__)
0c530ab8
A		 1373 struct AppleRTCHibernateVars
1374 {
1375 uint8_t signature[4];
1376 uint32_t revision;
1377 uint8_t booterSignature[20];
1378 uint8_t wiredCryptKey[16];
1379 };
1380 AppleRTCHibernateVars rtcVars;
1381
1382 rtcVars.signature[0] = 'A';
1383 rtcVars.signature[1] = 'A';
1384 rtcVars.signature[2] = 'P';
1385 rtcVars.signature[3] = 'L';
1386 rtcVars.revision = 1;
1387 bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
1388 if (gIOHibernateBootSignature[0])
1389 {
1390 char c;
1391 uint8_t value = 0;
1392 for (uint32_t i = 0;
1393 (c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1));
1394 i++)
1395 {
1396 if (c >= 'a')
1397 c -= 'a' - 10;
1398 else if (c >= 'A')
1399 c -= 'A' - 10;
1400 else if (c >= '0')
1401 c -= '0';
1402 else
1403 continue;
1404 value = (value << 4) | c;
1405 if (i & 1)
1406 rtcVars.booterSignature[i >> 1] = value;
1407 }
1408 }
1409 data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
1410 if (data)
2d21ac55 1411 {
060df5ea
A		 1412 if (!gIOHibernateRTCVariablesKey)
1413 gIOHibernateRTCVariablesKey = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
1414 if (gIOHibernateRTCVariablesKey)
1415 IOService::getPMRootDomain()->setProperty(gIOHibernateRTCVariablesKey, data);
1416
1417 if( gIOOptionsEntry )
1418 {
1419 if( gIOHibernateMode & kIOHibernateModeSwitch )
1420 {
1421 const OSSymbol *sym;
1422 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSwitchVarsKey);
1423 if( sym )
2d21ac55 1424 {
060df5ea
A		 1425 gIOOptionsEntry->setProperty(sym, data); /* intentional insecure backup of rtc boot vars */
1426 sym->release();
2d21ac55 1427 }
060df5ea
A		 1428 }
1429 }
2d21ac55 1430
060df5ea 1431 data->release();
0c530ab8
A		 1432 }
1433 if (gIOChosenEntry)
1434 {
1435 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
1436 if (data)
1437 gIOHibernateCurrentHeader->machineSignature = *((UInt32 *)data->getBytesNoCopy());
060df5ea
A		 1438 {
1439 // set BootNext
1440
1441 if (!gIOHibernateBoot0082Data)
1442 {
1443 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty("boot-device-path"));
1444 if (data)
1445 {
1446 // AppleNVRAM_EFI_LOAD_OPTION
1447 struct {
1448 uint32_t Attributes;
1449 uint16_t FilePathLength;
1450 uint16_t Desc;
1451 } loadOptionHeader;
1452 loadOptionHeader.Attributes = 1;
1453 loadOptionHeader.FilePathLength = data->getLength();
1454 loadOptionHeader.Desc = 0;
1455 gIOHibernateBoot0082Data = OSData::withCapacity(sizeof(loadOptionHeader) + loadOptionHeader.FilePathLength);
1456 if (gIOHibernateBoot0082Data)
1457 {
1458 gIOHibernateBoot0082Data->appendBytes(&loadOptionHeader, sizeof(loadOptionHeader));
1459 gIOHibernateBoot0082Data->appendBytes(data);
1460 }
1461 }
1462 }
1463 if (!gIOHibernateBoot0082Key)
1464 gIOHibernateBoot0082Key = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:Boot0082");
1465 if (!gIOHibernateBootNextKey)
1466 gIOHibernateBootNextKey = OSSymbol::withCString("8BE4DF61-93CA-11D2-AA0D-00E098032B8C:BootNext");
1467 if (!gIOHibernateBootNextData)
1468 {
1469 uint16_t bits = 0x0082;
1470 gIOHibernateBootNextData = OSData::withBytes(&bits, sizeof(bits));
1471 }
1472 if (gIOHibernateBoot0082Key && gIOHibernateBoot0082Data && gIOHibernateBootNextKey && gIOHibernateBootNextData)
1473 {
1474 gIOHibernateBootNextSave = gIOOptionsEntry->copyProperty(gIOHibernateBootNextKey);
1475 gIOOptionsEntry->setProperty(gIOHibernateBoot0082Key, gIOHibernateBoot0082Data);
1476 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextData);
1477 }
1478 }
0c530ab8 1479 }
b0d623f7 1480#else /* !i386 && !x86_64 */
3a60a9f5
A		 1481 if (kIOHibernateModeEncrypt & gIOHibernateMode)
1482 {
1483 data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
1484 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
1485 if (sym && data)
1486 gIOOptionsEntry->setProperty(sym, data);
1487 if (sym)
1488 sym->release();
1489 if (data)
1490 data->release();
2d21ac55 1491 if (false && gIOHibernateBootSignature[0])
3a60a9f5
A		 1492 {
1493 data = OSData::withCapacity(16);
1494 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
1495 if (sym && data)
1496 {
1497 char c;
0c530ab8 1498 uint8_t value = 0;
3a60a9f5
A		 1499 for (uint32_t i = 0; (c = gIOHibernateBootSignature[i]); i++)
1500 {
1501 if (c >= 'a')
1502 c -= 'a' - 10;
1503 else if (c >= 'A')
1504 c -= 'A' - 10;
1505 else if (c >= '0')
1506 c -= '0';
1507 else
1508 continue;
1509 value = (value << 4) | c;
1510 if (i & 1)
1511 data->appendBytes(&value, sizeof(value));
1512 }
1513 gIOOptionsEntry->setProperty(sym, data);
1514 }
1515 if (sym)
1516 sym->release();
1517 if (data)
1518 data->release();
1519 }
1520 }
3a60a9f5
A		 1521 if (!vars->haveFastBoot)
1522 {
1523 // set boot volume to zero
1524 IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
1525 if (platform && (kIOReturnSuccess == platform->readXPRAM(kXPRamAudioVolume,
1526 &vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume))))
1527 {
1528 uint8_t newVolume;
1529 newVolume = vars->saveBootAudioVolume & 0xf8;
1530 platform->writeXPRAM(kXPRamAudioVolume,
1531 &newVolume, sizeof(newVolume));
1532 }
1533 }
b0d623f7 1534#endif /* !i386 && !x86_64 */
3a60a9f5
A		 1535 }
1536 // --
1537
316670eb
A		 1538 }
1539 while (false);
1540
1541 IOLockLock(gFSLock);
1542 if ((kIOReturnSuccess == err) && (kFSOpening == gFSState))
1543 {
1544 gFSState = kFSOpened;
1545 gIOHibernateVars = *vars;
1546 gFileVars = *vars->fileVars;
1547 gIOHibernateVars.fileVars = &gFileVars;
1548 gIOHibernateFileRef = gFileVars.fileRef;
3a60a9f5
A		 1549 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
1550 gIOHibernateState = kIOHibernateStateHibernating;
1551 }
316670eb
A		 1552 else
1553 {
1554 HIBLOG("hibernate file close due timeout\n");
1555 if (vars->fileVars && vars->fileVars->fileRef) kern_close_file_for_direct_io(vars->fileVars->fileRef, 0, 0, 0, 0, 0);
1556 IOHibernateDone(vars);
1557 gFSState = kFSIdle;
1558 }
1559 IOLockUnlock(gFSLock);
1560
1561 if (vars->fileVars) IODelete(vars->fileVars, IOPolledFileIOVars, 1);
1562 IODelete(vars, IOHibernateVars, 1);
3a60a9f5
A		 1563
1564 return (err);
1565}
1566
1567/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1568
2d21ac55
A		 1569DECLARE_IOHIBERNATEPROGRESSALPHA
1570
1571static void
1572ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
1573{
1574 uint32_t rowBytes, pixelShift;
1575 uint32_t x, y;
1576 int32_t blob;
1577 uint32_t alpha, in, color, result;
1578 uint8_t * out;
1579 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1580
1581 rowBytes = display->rowBytes;
1582 pixelShift = display->depth >> 4;
1583 if (pixelShift < 1) return;
1584
1585 screen += ((display->width
1586 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1587 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1588
1589 for (y = 0; y < kIOHibernateProgressHeight; y++)
1590 {
1591 out = screen + y * rowBytes;
1592 for (blob = 0; blob < kIOHibernateProgressCount; blob++)
1593 {
1594 color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
1595 for (x = 0; x < kIOHibernateProgressWidth; x++)
1596 {
1597 alpha = gIOHibernateProgressAlpha[y][x];
1598 result = color;
1599 if (alpha)
1600 {
1601 if (0xff != alpha)
1602 {
1603 if (1 == pixelShift)
1604 {
1605 in = *((uint16_t *)out) & 0x1f; // 16
1606 in = (in << 3) | (in >> 2);
1607 }
1608 else
1609 in = *((uint32_t *)out) & 0xff; // 32
1610 saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
1611 result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
1612 }
1613 if (1 == pixelShift)
1614 {
1615 result >>= 3;
1616 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1617 }
1618 else
1619 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1620 }
1621 out += (1 << pixelShift);
1622 }
1623 out += (kIOHibernateProgressSpacing << pixelShift);
1624 }
1625 }
1626}
1627
1628
1629static void
1630ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
1631{
1632 uint32_t rowBytes, pixelShift;
1633 uint32_t x, y;
1634 int32_t blob, lastBlob;
1635 uint32_t alpha, in, color, result;
1636 uint8_t * out;
1637 uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
1638
1639 pixelShift = display->depth >> 4;
1640 if (pixelShift < 1)
1641 return;
1642
1643 rowBytes = display->rowBytes;
1644
1645 screen += ((display->width
1646 - kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
1647 + (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
1648
1649 lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
1650
1651 screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
1652
1653 for (y = 0; y < kIOHibernateProgressHeight; y++)
1654 {
1655 out = screen + y * rowBytes;
1656 for (blob = firstBlob; blob <= lastBlob; blob++)
1657 {
1658 color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
1659 for (x = 0; x < kIOHibernateProgressWidth; x++)
1660 {
1661 alpha = gIOHibernateProgressAlpha[y][x];
1662 result = color;
1663 if (alpha)
1664 {
1665 if (0xff != alpha)
1666 {
1667 in = display->progressSaveUnder[blob][saveindex[blob]++];
1668 result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
1669 }
1670 if (1 == pixelShift)
1671 {
1672 result >>= 3;
1673 *((uint16_t *)out) = (result << 10) | (result << 5) | result; // 16
1674 }
1675 else
1676 *((uint32_t *)out) = (result << 16) | (result << 8) | result; // 32
1677 }
1678 out += (1 << pixelShift);
1679 }
1680 out += (kIOHibernateProgressSpacing << pixelShift);
1681 }
1682 }
1683}
1684
1685/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1686
316670eb
A		 1687IOReturn
1688IOHibernateIOKitSleep(void)
1689{
1690 IOReturn ret = kIOReturnSuccess;
1691 IOLockLock(gFSLock);
1692 if (kFSOpening == gFSState)
1693 {
1694 gFSState = kFSTimedOut;
1695 HIBLOG("hibernate file open timed out\n");
1696 ret = kIOReturnTimeout;
1697 }
1698 IOLockUnlock(gFSLock);
1699 return (ret);
1700}
1701
1702/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1703
3a60a9f5
A		 1704IOReturn
1705IOHibernateSystemHasSlept(void)
1706{
316670eb 1707 IOReturn ret = kIOReturnSuccess;
3a60a9f5 1708 IOHibernateVars * vars = &gIOHibernateVars;
316670eb 1709 OSObject * obj = 0;
2d21ac55
A		 1710 OSData * data;
1711
316670eb
A		 1712 IOLockLock(gFSLock);
1713 if ((kFSOpened != gFSState) && gIOHibernateMode)
1714 {
1715 ret = kIOReturnTimeout;
1716 }
1717 IOLockUnlock(gFSLock);
1718 if (kIOReturnSuccess != ret) return (ret);
1719
1720 if (gIOHibernateMode) obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
2d21ac55
A		 1721 vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
1722 if (obj && !vars->previewBuffer)
1723 obj->release();
3a60a9f5 1724
2d21ac55
A		 1725 vars->consoleMapping = NULL;
1726 if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
3a60a9f5 1727 {
2d21ac55
A		 1728 vars->previewBuffer->release();
1729 vars->previewBuffer = 0;
1730 }
3a60a9f5 1731
0b4c1975
A		 1732 if ((kIOHibernateOptionProgress & gIOHibernateCurrentHeader->options)
1733 && vars->previewBuffer
1734 && (data = OSDynamicCast(OSData,
2d21ac55
A		 1735 IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey))))
1736 {
1737 UInt32 flags = *((UInt32 *)data->getBytesNoCopy());
b0d623f7 1738 HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", (long)flags);
2d21ac55
A		 1739
1740 IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
1741
1742 if (kIOHibernatePreviewUpdates & flags)
1743 {
1744 PE_Video consoleInfo;
1745 hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
1746
1747 IOService::getPlatform()->getConsoleInfo(&consoleInfo);
1748
1749 graphicsInfo->width = consoleInfo.v_width;
1750 graphicsInfo->height = consoleInfo.v_height;
1751 graphicsInfo->rowBytes = consoleInfo.v_rowBytes;
1752 graphicsInfo->depth = consoleInfo.v_depth;
1753 vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
1754
1755 HIBPRINT("video %p %d %d %d\n",
0b4c1975
A		 1756 vars->consoleMapping, graphicsInfo->depth,
1757 graphicsInfo->width, graphicsInfo->height);
2d21ac55
A		 1758 if (vars->consoleMapping)
1759 ProgressInit(graphicsInfo, vars->consoleMapping,
1760 &graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
1761 }
3a60a9f5 1762 }
2d21ac55 1763
3a60a9f5
A		 1764 if (gIOOptionsEntry)
1765 gIOOptionsEntry->sync();
1766
316670eb 1767 return (ret);
3a60a9f5
A		 1768}
1769
1770/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1771
6d2010ae
A		 1772static DeviceTreeNode *
1773MergeDeviceTree(DeviceTreeNode * entry, IORegistryEntry * regEntry)
1774{
1775 DeviceTreeNodeProperty * prop;
1776 DeviceTreeNode * child;
1777 IORegistryEntry * childRegEntry;
1778 const char * nameProp;
1779 unsigned int propLen, idx;
1780
1781 prop = (DeviceTreeNodeProperty *) (entry + 1);
1782 for (idx = 0; idx < entry->nProperties; idx++)
1783 {
1784 if (regEntry && (0 != strcmp("name", prop->name)))
1785 {
1786 regEntry->setProperty((const char *) prop->name, (void *) (prop + 1), prop->length);
1787// HIBPRINT("%s: %s, %d\n", regEntry->getName(), prop->name, prop->length);
1788 }
1789 prop = (DeviceTreeNodeProperty *) (((uintptr_t)(prop + 1)) + ((prop->length + 3) & ~3));
1790 }
1791
1792 child = (DeviceTreeNode *) prop;
1793 for (idx = 0; idx < entry->nChildren; idx++)
1794 {
1795 if (kSuccess != DTGetProperty(child, "name", (void **) &nameProp, &propLen))
1796 panic("no name");
1797 childRegEntry = regEntry ? regEntry->childFromPath(nameProp, gIODTPlane) : NULL;
1798// HIBPRINT("%s == %p\n", nameProp, childRegEntry);
1799 child = MergeDeviceTree(child, childRegEntry);
1800 }
1801 return (child);
1802}
1803
3a60a9f5
A		 1804IOReturn
1805IOHibernateSystemWake(void)
1806{
316670eb
A		 1807 if (kFSOpened == gFSState)
1808 {
1809 IOHibernateDone(&gIOHibernateVars);
1810 }
1811 else
1812 {
1813 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1814 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1815 }
1816 return (kIOReturnSuccess);
1817}
3a60a9f5 1818
316670eb
A		 1819static IOReturn
1820IOHibernateDone(IOHibernateVars * vars)
1821{
bd504ef0
A		 1822 IORegistryEntry * next;
1823
99c3a104 1824 hibernate_teardown(vars->page_list, vars->page_list_wired, vars->page_list_pal);
3a60a9f5
A		 1825
1826 if (vars->videoMapping)
1827 {
1828 if (vars->videoMapSize)
1829 // remove mappings
1830 IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
1831 if (vars->videoAllocSize)
1832 // dealloc range
b0d623f7 1833 kmem_free(kernel_map, trunc_page(vars->videoMapping), vars->videoAllocSize);
3a60a9f5
A		 1834 }
1835
1836 if (vars->previewBuffer)
1837 {
1838 vars->previewBuffer->release();
1839 vars->previewBuffer = 0;
1840 }
1841
0b4c1975
A		 1842 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1843 {
1844 IOService::getPMRootDomain()->setProperty(kIOHibernateOptionsKey,
1845 gIOHibernateCurrentHeader->options, 32);
1846 }
1847 else
1848 {
1849 IOService::getPMRootDomain()->removeProperty(kIOHibernateOptionsKey);
1850 }
1851
1852 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState)
1853 && (kIOHibernateGfxStatusUnknown != gIOHibernateGraphicsInfo->gfxStatus))
1854 {
1855 IOService::getPMRootDomain()->setProperty(kIOHibernateGfxStatusKey,
1856 &gIOHibernateGraphicsInfo->gfxStatus,
1857 sizeof(gIOHibernateGraphicsInfo->gfxStatus));
1858 }
1859 else
1860 {
1861 IOService::getPMRootDomain()->removeProperty(kIOHibernateGfxStatusKey);
1862 }
1863
3a60a9f5
A		 1864 if (vars->fileVars)
1865 {
bd504ef0
A		 1866 if ((next = vars->fileVars->media)) do
1867 {
1868 next->removeProperty(kIOPolledInterfaceActiveKey);
1869 next = next->getParentEntry(gIOServicePlane);
1870 }
1871 while (next);
3a60a9f5
A		 1872 IOPolledFileClose(vars->fileVars);
1873 }
1874
1875 // invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
1876
b0d623f7 1877#if defined(__i386__) || defined(__x86_64__)
060df5ea 1878 IOService::getPMRootDomain()->removeProperty(gIOHibernateRTCVariablesKey);
6d2010ae 1879 IOService::getPMRootDomain()->removeProperty(kIOHibernateSMCVariablesKey);
2d21ac55
A		 1880
1881 /*
1882 * Hibernate variable is written to NVRAM on platforms in which RtcRam
1883 * is not backed by coin cell. Remove Hibernate data from NVRAM.
1884 */
1885 if (gIOOptionsEntry) {
2d21ac55 1886
060df5ea
A		 1887 if (gIOHibernateRTCVariablesKey) {
1888 if (gIOOptionsEntry->getProperty(gIOHibernateRTCVariablesKey)) {
1889 gIOOptionsEntry->removeProperty(gIOHibernateRTCVariablesKey);
1890 }
1891 }
1892
1893 if (gIOHibernateBootNextKey)
1894 {
1895 if (gIOHibernateBootNextSave)
1896 {
1897 gIOOptionsEntry->setProperty(gIOHibernateBootNextKey, gIOHibernateBootNextSave);
1898 gIOHibernateBootNextSave->release();
1899 gIOHibernateBootNextSave = NULL;
2d21ac55 1900 }
060df5ea
A		 1901 else
1902 gIOOptionsEntry->removeProperty(gIOHibernateBootNextKey);
1903 }
39236c6e 1904 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState) gIOOptionsEntry->sync();
2d21ac55 1905 }
0c530ab8 1906#endif
3a60a9f5
A		 1907
1908 if (vars->srcBuffer)
1909 vars->srcBuffer->release();
1910 if (vars->ioBuffer)
1911 vars->ioBuffer->release();
6d2010ae 1912 bzero(&gIOHibernateHandoffPages[0], gIOHibernateHandoffPageCount * sizeof(gIOHibernateHandoffPages[0]));
99c3a104 1913 if (vars->handoffBuffer)
6d2010ae 1914 {
99c3a104 1915 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
6d2010ae 1916 {
99c3a104
A		 1917 IOHibernateHandoff * handoff;
1918 bool done = false;
1919 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
1920 !done;
1921 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
6d2010ae 1922 {
99c3a104
A		 1923 HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
1924 uint8_t * data = &handoff->data[0];
1925 switch (handoff->type)
1926 {
1927 case kIOHibernateHandoffTypeEnd:
1928 done = true;
1929 break;
6d2010ae 1930
99c3a104
A		 1931 case kIOHibernateHandoffTypeDeviceTree:
1932 MergeDeviceTree((DeviceTreeNode *) data, IOService::getServiceRoot());
1933 break;
1934
1935 case kIOHibernateHandoffTypeKeyStore:
6d2010ae 1936#if defined(__i386__) || defined(__x86_64__)
6d2010ae 1937 {
99c3a104
A		 1938 IOBufferMemoryDescriptor *
1939 md = IOBufferMemoryDescriptor::withBytes(data, handoff->bytecount, kIODirectionOutIn);
1940 if (md)
1941 {
1942 IOSetKeyStoreData(md);
1943 }
6d2010ae 1944 }
6d2010ae 1945#endif
99c3a104
A		 1946 break;
1947
1948 default:
1949 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
1950 break;
1951 }
1952 }
6d2010ae
A		 1953 }
1954 vars->handoffBuffer->release();
1955 }
3a60a9f5
A		 1956 if (vars->fileExtents)
1957 vars->fileExtents->release();
1958
1959 bzero(vars, sizeof(*vars));
1960
1961// gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
1962
1963 return (kIOReturnSuccess);
1964}
1965
1966IOReturn
1967IOHibernateSystemPostWake(void)
1968{
316670eb
A		 1969 struct kern_direct_file_io_ref_t * fileRef;
1970
1971 if (kFSOpened == gFSState)
3a60a9f5 1972 {
6d2010ae 1973 // invalidate & close the image file
0c530ab8 1974 gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
316670eb
A		 1975 if ((fileRef = gIOHibernateFileRef))
1976 {
1977 gIOHibernateFileRef = 0;
39236c6e 1978 IOSleep(TRIM_DELAY);
316670eb 1979 kern_close_file_for_direct_io(fileRef,
39236c6e
A		 1980#if DISABLE_TRIM
1981 0, 0, 0, 0, 0);
1982#else
6d2010ae 1983 0, (caddr_t) gIOHibernateCurrentHeader,
7ddcb079 1984 sizeof(IOHibernateImageHeader),
39236c6e 1985 0,
7ddcb079 1986 gIOHibernateCurrentHeader->imageSize);
39236c6e 1987#endif
316670eb
A		 1988 }
1989 gFSState = kFSIdle;
3a60a9f5
A		 1990 }
1991 return (kIOReturnSuccess);
1992}
1993
316670eb
A		 1994bool IOHibernateWasScreenLocked(void)
1995{
1996 bool ret = false;
1997 if ((kIOHibernateStateWakingFromHibernate == gIOHibernateState) && gIOChosenEntry)
1998 {
1999 OSData *
2000 data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOScreenLockStateKey));
2001 if (data) switch (*((uint32_t *)data->getBytesNoCopy()))
2002 {
2003 case kIOScreenLockLocked:
2004 case kIOScreenLockFileVaultDialog:
2005 ret = true;
2006 break;
2007 case kIOScreenLockNoLock:
2008 case kIOScreenLockUnlocked:
2009 default:
2010 ret = false;
2011 break;
2012 }
2013 }
2014 return (ret);
2015}
2016
3a60a9f5
A		 2017/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2018
2d21ac55 2019SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
6d2010ae 2020 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2d21ac55
A		 2021 gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
2022SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
6d2010ae 2023 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2d21ac55
A		 2024 gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
2025SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
6d2010ae 2026 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2d21ac55 2027 &gIOHibernateMode, 0, "");
39236c6e
A		 2028SYSCTL_STRUCT(_kern, OID_AUTO, hibernatestatistics,
2029 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_LOCKED,
2030 gIOHibernateStats, hibernate_statistics_t, "");
2031
2032SYSCTL_UINT(_kern, OID_AUTO, hibernategraphicsready,
2033 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
2034 &gIOHibernateStats->graphicsReadyTime, 0, "");
2035SYSCTL_UINT(_kern, OID_AUTO, hibernatewakenotification,
2036 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
2037 &gIOHibernateStats->wakeNotificationTime, 0, "");
2038SYSCTL_UINT(_kern, OID_AUTO, hibernatelockscreenready,
2039 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
2040 &gIOHibernateStats->lockScreenReadyTime, 0, "");
2041SYSCTL_UINT(_kern, OID_AUTO, hibernatehidready,
2042 CTLFLAG_RW | CTLFLAG_NOAUTO | CTLFLAG_KERN | CTLFLAG_ANYBODY,
2043 &gIOHibernateStats->hidReadyTime, 0, "");
2044
2d21ac55 2045
3a60a9f5
A		 2046void
2047IOHibernateSystemInit(IOPMrootDomain * rootDomain)
2048{
2049 OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
2050 if (data)
2051 {
2052 rootDomain->setProperty(kIOHibernateStateKey, data);
2053 data->release();
2054 }
2055
2d21ac55 2056 if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename)))
3a60a9f5
A		 2057 gIOHibernateMode = kIOHibernateModeOn;
2058 else
2059 gIOHibernateFilename[0] = 0;
2060
3a60a9f5 2061 sysctl_register_oid(&sysctl__kern_hibernatefile);
3a60a9f5 2062 sysctl_register_oid(&sysctl__kern_bootsignature);
3a60a9f5 2063 sysctl_register_oid(&sysctl__kern_hibernatemode);
39236c6e
A		 2064 sysctl_register_oid(&sysctl__kern_hibernatestatistics);
2065 sysctl_register_oid(&sysctl__kern_hibernategraphicsready);
2066 sysctl_register_oid(&sysctl__kern_hibernatewakenotification);
2067 sysctl_register_oid(&sysctl__kern_hibernatelockscreenready);
2068 sysctl_register_oid(&sysctl__kern_hibernatehidready);
316670eb
A		 2069
2070 gFSLock = IOLockAlloc();
3a60a9f5
A		 2071}
2072
2d21ac55 2073
3a60a9f5
A		 2074/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2075
2076static void
2077hibernate_setup_for_wake(void)
2078{
3a60a9f5
A		 2079}
2080
2081/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2082
0c530ab8
A		 2083#define C_ASSERT(e) typedef char __C_ASSERT__[(e) ? 1 : -1]
2084
0b4c1975
A		 2085static bool
2086no_encrypt_page(vm_offset_t ppnum)
2087{
2088 if (pmap_is_noencrypt((ppnum_t)ppnum) == TRUE)
2089 {
2090 return true;
2091 }
2092 return false;
2093}
2094
6d2010ae
A		 2095static void
2096hibernate_pal_callback(void *vars_arg, vm_offset_t addr)
2097{
2098 IOHibernateVars *vars = (IOHibernateVars *)vars_arg;
2099 /* Make sure it's not in either of the save lists */
2100 hibernate_set_page_state(vars->page_list, vars->page_list_wired, atop_64(addr), 1, kIOHibernatePageStateFree);
2101
2102 /* Set it in the bitmap of pages owned by the PAL */
2103 hibernate_page_bitset(vars->page_list_pal, TRUE, atop_64(addr));
2104}
2105
2106static struct hibernate_cryptvars_t *local_cryptvars;
2107
2108extern "C" int
2109hibernate_pal_write(void *buffer, size_t size)
2110{
2111 IOHibernateVars * vars = &gIOHibernateVars;
2112
2113 IOReturn err = IOPolledFileWrite(vars->fileVars, (const uint8_t *)buffer, size, local_cryptvars);
2114 if (kIOReturnSuccess != err) {
2115 kprintf("epic hibernate fail! %d\n", err);
2116 return err;
2117 }
2118
2119 return 0;
2120}
2121
2122
2d21ac55 2123extern "C" uint32_t
3a60a9f5
A		 2124hibernate_write_image(void)
2125{
2126 IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
2127 IOHibernateVars * vars = &gIOHibernateVars;
2128 IOPolledFileExtent * fileExtents;
2129
0c530ab8
A		 2130 C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
2131
3a60a9f5
A		 2132 uint32_t pageCount, pagesDone;
2133 IOReturn err;
0b4c1975
A		 2134 vm_offset_t ppnum, page;
2135 IOItemCount count;
3a60a9f5
A		 2136 uint8_t * src;
2137 uint8_t * data;
39236c6e
A		 2138 uint8_t * compressed;
2139 uint8_t * scratch;
2140 void * zerosCompressed;
2141 IOByteCount pageCompressedSize, zerosCompressedLen;
3a60a9f5
A		 2142 uint64_t compressedSize, uncompressedSize;
2143 uint64_t image1Size = 0;
2144 uint32_t bitmap_size;
0b4c1975 2145 bool iterDone, pollerOpen, needEncrypt;
3a60a9f5 2146 uint32_t restore1Sum, sum, sum1, sum2;
39236c6e 2147 int wkresult;
3a60a9f5
A		 2148 uint32_t tag;
2149 uint32_t pageType;
2150 uint32_t pageAndCount[2];
6d2010ae
A		 2151 addr64_t phys64;
2152 IOByteCount segLen;
3a60a9f5
A		 2153
2154 AbsoluteTime startTime, endTime;
6d2010ae 2155 AbsoluteTime allTime, compTime;
0b4c1975 2156 uint64_t compBytes;
3a60a9f5
A		 2157 uint64_t nsec;
2158 uint32_t lastProgressStamp = 0;
2159 uint32_t progressStamp;
2d21ac55 2160 uint32_t blob, lastBlob = (uint32_t) -1L;
3a60a9f5 2161
39236c6e
A		 2162 uint32_t wiredPagesEncrypted;
2163 uint32_t dirtyPagesEncrypted;
2164 uint32_t wiredPagesClear;
2165 uint32_t zeroPageCount;
2166
3a60a9f5
A		 2167 hibernate_cryptvars_t _cryptvars;
2168 hibernate_cryptvars_t * cryptvars = 0;
2169
39236c6e
A		 2170 wiredPagesEncrypted = 0;
2171 dirtyPagesEncrypted = 0;
2172 wiredPagesClear = 0;
2173 zeroPageCount = 0;
0b4c1975 2174
3a60a9f5
A		 2175 if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
2176 return (false /* sleep */ );
2177
0b4c1975
A		 2178 if (kIOHibernateModeSleep & gIOHibernateMode)
2179 kdebug_enable = save_kdebug_enable;
2180
2181 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_START, 0, 0, 0, 0, 0);
6d2010ae 2182 IOService::getPMRootDomain()->tracePoint(kIOPMTracePointHibernate);
0b4c1975 2183
3a60a9f5
A		 2184 restore1Sum = sum1 = sum2 = 0;
2185
6d2010ae
A		 2186 hibernate_pal_prepare();
2187
2d21ac55 2188#if CRYPTO
3a60a9f5
A		 2189 // encryption data. "iv" is the "initial vector".
2190 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2191 {
2192 static const unsigned char first_iv[AES_BLOCK_SIZE]
2193 = { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
2194 0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
2195
2196 cryptvars = &gIOHibernateCryptWakeContext;
2197 bzero(cryptvars, sizeof(hibernate_cryptvars_t));
2198 aes_encrypt_key(vars->cryptKey,
2199 kIOHibernateAESKeySize,
2200 &cryptvars->ctx.encrypt);
2201 aes_decrypt_key(vars->cryptKey,
2202 kIOHibernateAESKeySize,
2203 &cryptvars->ctx.decrypt);
2204
2205 cryptvars = &_cryptvars;
2206 bzero(cryptvars, sizeof(hibernate_cryptvars_t));
6d2010ae
A		 2207 for (pageCount = 0; pageCount < sizeof(vars->wiredCryptKey); pageCount++)
2208 vars->wiredCryptKey[pageCount] ^= vars->volumeCryptKey[pageCount];
2209 bzero(&vars->volumeCryptKey[0], sizeof(vars->volumeCryptKey));
3a60a9f5
A		 2210 aes_encrypt_key(vars->wiredCryptKey,
2211 kIOHibernateAESKeySize,
2212 &cryptvars->ctx.encrypt);
2213
2214 bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
2215 bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
2216 bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
6d2010ae
A		 2217
2218 local_cryptvars = cryptvars;
3a60a9f5 2219 }
2d21ac55 2220#endif /* CRYPTO */
3a60a9f5
A		 2221
2222 hibernate_setup_for_wake();
2223
2224 hibernate_page_list_setall(vars->page_list,
2225 vars->page_list_wired,
db609669
A		 2226 vars->page_list_pal,
2227 false /* !preflight */,
39236c6e
A		 2228 /* discard_all */
2229 ((0 == (kIOHibernateModeSleep & gIOHibernateMode))
2230 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode))),
3a60a9f5
A		 2231 &pageCount);
2232
2233 HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
2234
2235 fileExtents = (IOPolledFileExtent *) vars->fileExtents->getBytesNoCopy();
2236
2237#if 0
2238 count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
2239 for (page = 0; page < count; page++)
2240 {
2241 HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
2242 fileExtents[page].start, fileExtents[page].length,
2243 fileExtents[page].start + fileExtents[page].length);
2244 }
2245#endif
2246
0b4c1975 2247 needEncrypt = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
3a60a9f5 2248 AbsoluteTime_to_scalar(&compTime) = 0;
6d2010ae 2249 compBytes = 0;
3a60a9f5
A		 2250
2251 clock_get_uptime(&allTime);
b0d623f7
A		 2252 IOService::getPMRootDomain()->pmStatsRecordEvent(
2253 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStartFlag, allTime);
3a60a9f5
A		 2254 do
2255 {
2256 compressedSize = 0;
2257 uncompressedSize = 0;
39236c6e 2258 zeroPageCount = 0;
3a60a9f5 2259
39236c6e 2260 IOPolledFileSeek(vars->fileVars, vars->fileVars->blockSize);
3a60a9f5
A		 2261
2262 HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
2263 ml_get_interrupts_enabled());
2264 err = IOHibernatePollerOpen(vars->fileVars, kIOPolledBeforeSleepState, vars->ioBuffer);
2265 HIBLOG("IOHibernatePollerOpen(%x)\n", err);
2266 pollerOpen = (kIOReturnSuccess == err);
2267 if (!pollerOpen)
2268 break;
2269
2270 // copy file block extent list if larger than header
2271
2272 count = vars->fileExtents->getLength();
2273 if (count > sizeof(header->fileExtentMap))
2274 {
2275 count -= sizeof(header->fileExtentMap);
2276 err = IOPolledFileWrite(vars->fileVars,
2277 ((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
2278 if (kIOReturnSuccess != err)
2279 break;
2280 }
2281
b0d623f7
A		 2282 uintptr_t hibernateBase;
2283 uintptr_t hibernateEnd;
2284
6d2010ae 2285 hibernateBase = HIB_BASE; /* Defined in PAL headers */
b0d623f7 2286
316670eb 2287 hibernateEnd = (segHIBB + segSizeHIB);
6d2010ae 2288
3a60a9f5 2289 // copy out restore1 code
6d2010ae
A		 2290
2291 for (count = 0;
2292 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2293 count += segLen)
2294 {
2295 for (pagesDone = 0; pagesDone < atop_32(segLen); pagesDone++)
2296 {
2297 gIOHibernateHandoffPages[atop_32(count) + pagesDone] = atop_64(phys64) + pagesDone;
2298 }
2299 }
2300
2301 page = atop_32(kvtophys(hibernateBase));
2302 count = atop_32(round_page(hibernateEnd) - hibernateBase);
2303 header->restore1CodePhysPage = page;
2304 header->restore1CodeVirt = hibernateBase;
3a60a9f5 2305 header->restore1PageCount = count;
b0d623f7
A		 2306 header->restore1CodeOffset = ((uintptr_t) &hibernate_machine_entrypoint) - hibernateBase;
2307 header->restore1StackOffset = ((uintptr_t) &gIOHibernateRestoreStackEnd[0]) - 64 - hibernateBase;
3a60a9f5 2308
316670eb 2309 // sum __HIB seg, with zeros for the stack
b0d623f7 2310 src = (uint8_t *) trunc_page(hibernateBase);
3a60a9f5
A		 2311 for (page = 0; page < count; page++)
2312 {
2313 if ((src < &gIOHibernateRestoreStack[0]) || (src >= &gIOHibernateRestoreStackEnd[0]))
6d2010ae 2314 restore1Sum += hibernate_sum_page(src, header->restore1CodeVirt + page);
3a60a9f5 2315 else
0b4c1975 2316 restore1Sum += 0x00000000;
3a60a9f5
A		 2317 src += page_size;
2318 }
2319 sum1 = restore1Sum;
2320
316670eb 2321 // write the __HIB seg, with zeros for the stack
3a60a9f5 2322
b0d623f7
A		 2323 src = (uint8_t *) trunc_page(hibernateBase);
2324 count = ((uintptr_t) &gIOHibernateRestoreStack[0]) - trunc_page(hibernateBase);
3a60a9f5
A		 2325 if (count)
2326 {
2327 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
2328 if (kIOReturnSuccess != err)
2329 break;
2330 }
2331 err = IOPolledFileWrite(vars->fileVars,
2332 (uint8_t *) 0,
2333 &gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
2334 cryptvars);
2335 if (kIOReturnSuccess != err)
2336 break;
2337 src = &gIOHibernateRestoreStackEnd[0];
b0d623f7 2338 count = round_page(hibernateEnd) - ((uintptr_t) src);
3a60a9f5
A		 2339 if (count)
2340 {
2341 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
2342 if (kIOReturnSuccess != err)
2343 break;
2344 }
2345
39236c6e
A		 2346 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2347 {
2348 vars->fileVars->encryptStart = (vars->fileVars->position & ~(AES_BLOCK_SIZE - 1));
2349 vars->fileVars->encryptEnd = UINT64_MAX;
2350 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
2351 }
316670eb 2352
3a60a9f5
A		 2353 // write the preview buffer
2354
2d21ac55 2355 if (vars->previewBuffer)
3a60a9f5
A		 2356 {
2357 ppnum = 0;
2358 count = 0;
2359 do
2360 {
b0d623f7 2361 phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone);
3a60a9f5
A		 2362 pageAndCount[0] = atop_64(phys64);
2363 pageAndCount[1] = atop_32(segLen);
2364 err = IOPolledFileWrite(vars->fileVars,
2365 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
2366 cryptvars);
2367 if (kIOReturnSuccess != err)
2368 break;
2369 count += segLen;
2370 ppnum += sizeof(pageAndCount);
2371 }
2372 while (phys64);
2373 if (kIOReturnSuccess != err)
2374 break;
2375
b0d623f7 2376 src = (uint8_t *) vars->previewBuffer->getPhysicalSegment(0, NULL, _kIOMemorySourceSegment);
316670eb
A		 2377
2378 ((hibernate_preview_t *)src)->lockTime = gIOConsoleLockTime;
2379
2d21ac55 2380 count = vars->previewBuffer->getLength();
3a60a9f5
A		 2381
2382 header->previewPageListSize = ppnum;
2383 header->previewSize = count + ppnum;
2384
2385 for (page = 0; page < count; page += page_size)
0b4c1975
A		 2386 {
2387 phys64 = vars->previewBuffer->getPhysicalSegment(page, NULL, kIOMemoryMapperNone);
2388 sum1 += hibernate_sum_page(src + page, atop_64(phys64));
2389 }
3a60a9f5
A		 2390 err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
2391 if (kIOReturnSuccess != err)
2392 break;
2393 }
2394
2395 // mark areas for no save
2396
2397 for (count = 0;
b0d623f7 2398 (phys64 = vars->ioBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
3a60a9f5
A		 2399 count += segLen)
2400 {
2401 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2402 atop_64(phys64), atop_32(segLen),
2403 kIOHibernatePageStateFree);
2404 pageCount -= atop_32(segLen);
2405 }
2406
2407 for (count = 0;
b0d623f7 2408 (phys64 = vars->srcBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
3a60a9f5
A		 2409 count += segLen)
2410 {
2411 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2412 atop_64(phys64), atop_32(segLen),
2413 kIOHibernatePageStateFree);
2414 pageCount -= atop_32(segLen);
2415 }
2416
2417 // copy out bitmap of pages available for trashing during restore
2418
2419 bitmap_size = vars->page_list_wired->list_size;
2420 src = (uint8_t *) vars->page_list_wired;
2421 err = IOPolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
2422 if (kIOReturnSuccess != err)
2423 break;
2424
2425 // mark more areas for no save, but these are not available
2426 // for trashing during restore
0c530ab8
A		 2427
2428 hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
3a60a9f5 2429
6d2010ae
A		 2430
2431 page = atop_32(KERNEL_IMAGE_TO_PHYS(hibernateBase));
2432 count = atop_32(round_page(KERNEL_IMAGE_TO_PHYS(hibernateEnd))) - page;
3a60a9f5
A		 2433 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2434 page, count,
2435 kIOHibernatePageStateFree);
2436 pageCount -= count;
3a60a9f5
A		 2437
2438 if (vars->previewBuffer) for (count = 0;
b0d623f7 2439 (phys64 = vars->previewBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
3a60a9f5
A		 2440 count += segLen)
2441 {
2442 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2443 atop_64(phys64), atop_32(segLen),
2444 kIOHibernatePageStateFree);
2445 pageCount -= atop_32(segLen);
2446 }
2447
6d2010ae
A		 2448 for (count = 0;
2449 (phys64 = vars->handoffBuffer->getPhysicalSegment(count, &segLen, kIOMemoryMapperNone));
2450 count += segLen)
2451 {
2452 hibernate_set_page_state(vars->page_list, vars->page_list_wired,
2453 atop_64(phys64), atop_32(segLen),
2454 kIOHibernatePageStateFree);
2455 pageCount -= atop_32(segLen);
2456 }
2457
2458 (void)hibernate_pal_callback;
2459
3a60a9f5 2460 src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
39236c6e
A		 2461 compressed = src + page_size;
2462 scratch = compressed + page_size;
2463
2464 // compress a zero page
2465 bzero(src, page_size);
2466 zerosCompressed = vars->handoffBuffer->getBytesNoCopy();
2467 zerosCompressedLen = WKdm_compress_new((WK_word*) src,
2468 (WK_word*) zerosCompressed,
2469 (WK_word*) scratch,
2470 page_size - 4);
2471
6d2010ae
A		 2472 pagesDone = 0;
2473 lastBlob = 0;
3a60a9f5 2474
db609669
A		 2475 HIBLOG("bitmap_size 0x%x, previewSize 0x%x, writing %d pages @ 0x%llx\n",
2476 bitmap_size, header->previewSize,
2477 pageCount, vars->fileVars->position);
3a60a9f5 2478
0b4c1975
A		 2479 enum
2480 // pageType
2481 {
2482 kWired = 0x02,
2483 kEncrypt = 0x01,
2484 kWiredEncrypt = kWired | kEncrypt,
2485 kWiredClear = kWired,
2486 kUnwiredEncrypt = kEncrypt
2487 };
2488
2489 for (pageType = kWiredEncrypt; pageType >= kUnwiredEncrypt; pageType--)
3a60a9f5 2490 {
316670eb
A		 2491 if (kUnwiredEncrypt == pageType)
2492 {
2493 // start unwired image
39236c6e
A		 2494 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2495 {
2496 vars->fileVars->encryptStart = (vars->fileVars->position & ~(((uint64_t)AES_BLOCK_SIZE) - 1));
2497 vars->fileVars->encryptEnd = UINT64_MAX;
2498 HIBLOG("encryptStart %qx\n", vars->fileVars->encryptStart);
2499 }
316670eb
A		 2500 bcopy(&cryptvars->aes_iv[0],
2501 &gIOHibernateCryptWakeContext.aes_iv[0],
2502 sizeof(cryptvars->aes_iv));
2503 cryptvars = &gIOHibernateCryptWakeContext;
0b4c1975
A		 2504 }
2505 for (iterDone = false, ppnum = 0; !iterDone; )
2506 {
2507 count = hibernate_page_list_iterate((kWired & pageType)
2508 ? vars->page_list_wired : vars->page_list,
2509 &ppnum);
2510// kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
2511 iterDone = !count;
3a60a9f5 2512
0b4c1975 2513 if (count && (kWired & pageType) && needEncrypt)
3a60a9f5 2514 {
0b4c1975
A		 2515 uint32_t checkIndex;
2516 for (checkIndex = 0;
2517 (checkIndex < count)
2518 && (((kEncrypt & pageType) == 0) == no_encrypt_page(ppnum + checkIndex));
2519 checkIndex++)
2520 {}
2521 if (!checkIndex)
2522 {
2523 ppnum++;
2524 continue;
2525 }
2526 count = checkIndex;
3a60a9f5
A		 2527 }
2528
0b4c1975
A		 2529 switch (pageType)
2530 {
39236c6e
A		 2531 case kWiredEncrypt: wiredPagesEncrypted += count; break;
2532 case kWiredClear: wiredPagesClear += count; break;
2533 case kUnwiredEncrypt: dirtyPagesEncrypted += count; break;
0b4c1975
A		 2534 }
2535
2536 if (iterDone && (kWiredEncrypt == pageType)) {/* not yet end of wired list */}
3a60a9f5 2537 else
3a60a9f5 2538 {
0b4c1975
A		 2539 pageAndCount[0] = ppnum;
2540 pageAndCount[1] = count;
2541 err = IOPolledFileWrite(vars->fileVars,
2542 (const uint8_t *) &pageAndCount, sizeof(pageAndCount),
2543 cryptvars);
2544 if (kIOReturnSuccess != err)
2545 break;
3a60a9f5 2546 }
0b4c1975
A		 2547
2548 for (page = ppnum; page < (ppnum + count); page++)
3a60a9f5 2549 {
0b4c1975
A		 2550 err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(page), page_size);
2551 if (err)
2552 {
2553 HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%ld] %x\n", __LINE__, (long)page, err);
2554 break;
2555 }
2556
2557 sum = hibernate_sum_page(src, page);
2558 if (kWired & pageType)
2559 sum1 += sum;
2560 else
2561 sum2 += sum;
2562
2563 clock_get_uptime(&startTime);
39236c6e
A		 2564 wkresult = WKdm_compress_new((WK_word*) src,
2565 (WK_word*) compressed,
2566 (WK_word*) scratch,
2567 page_size - 4);
0b4c1975 2568
3a60a9f5 2569 clock_get_uptime(&endTime);
0b4c1975
A		 2570 ADD_ABSOLUTETIME(&compTime, &endTime);
2571 SUB_ABSOLUTETIME(&compTime, &startTime);
39236c6e 2572
0b4c1975 2573 compBytes += page_size;
39236c6e
A		 2574 pageCompressedSize = (-1 == wkresult) ? page_size : wkresult;
2575
2576 if ((pageCompressedSize == zerosCompressedLen)
2577 && !bcmp(compressed, zerosCompressed, zerosCompressedLen))
2578 {
2579 pageCompressedSize = 0;
2580 zeroPageCount++;
2581 }
2582
0b4c1975
A		 2583 if (kIOHibernateModeEncrypt & gIOHibernateMode)
2584 pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1);
db609669 2585
0b4c1975 2586 if (pageCompressedSize != page_size)
39236c6e 2587 data = compressed;
0b4c1975
A		 2588 else
2589 data = src;
2590
2591 tag = pageCompressedSize | kIOHibernateTagSignature;
2592 err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
2593 if (kIOReturnSuccess != err)
2594 break;
2595
2596 err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
2597 if (kIOReturnSuccess != err)
2598 break;
2599
2600 compressedSize += pageCompressedSize;
39236c6e 2601 uncompressedSize += page_size;
0b4c1975
A		 2602 pagesDone++;
2603
2604 if (vars->consoleMapping && (0 == (1023 & pagesDone)))
2605 {
2606 blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
2607 if (blob != lastBlob)
2608 {
2609 ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
2610 lastBlob = blob;
2611 }
2612 }
2613 if (0 == (8191 & pagesDone))
2614 {
2615 clock_get_uptime(&endTime);
2616 SUB_ABSOLUTETIME(&endTime, &allTime);
2617 absolutetime_to_nanoseconds(endTime, &nsec);
2618 progressStamp = nsec / 750000000ULL;
2619 if (progressStamp != lastProgressStamp)
2620 {
2621 lastProgressStamp = progressStamp;
2622 HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
2623 }
3a60a9f5
A		 2624 }
2625 }
0b4c1975
A		 2626 if (kIOReturnSuccess != err)
2627 break;
2628 ppnum = page;
3a60a9f5 2629 }
0b4c1975 2630
3a60a9f5
A		 2631 if (kIOReturnSuccess != err)
2632 break;
0b4c1975 2633
39236c6e 2634 if ((kEncrypt & pageType) && vars->fileVars->encryptStart)
3a60a9f5 2635 {
7ddcb079 2636 vars->fileVars->encryptEnd = ((vars->fileVars->position + 511) & ~511ULL);
0b4c1975
A		 2637 HIBLOG("encryptEnd %qx\n", vars->fileVars->encryptEnd);
2638 }
2639
2640 if (kWiredEncrypt != pageType)
2641 {
2642 // end of image1/2 - fill to next block
3a60a9f5
A		 2643 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
2644 if (kIOReturnSuccess != err)
2645 break;
0b4c1975
A		 2646 }
2647 if (kWiredClear == pageType)
2648 {
7ddcb079
A		 2649 // enlarge wired image for test
2650// err = IOPolledFileWrite(vars->fileVars, 0, 0x60000000, cryptvars);
2651
0b4c1975
A		 2652 // end wired image
2653 header->encryptStart = vars->fileVars->encryptStart;
2654 header->encryptEnd = vars->fileVars->encryptEnd;
3a60a9f5 2655 image1Size = vars->fileVars->position;
7ddcb079 2656 HIBLOG("image1Size 0x%qx, encryptStart1 0x%qx, End1 0x%qx\n",
0b4c1975 2657 image1Size, header->encryptStart, header->encryptEnd);
3a60a9f5
A		 2658 }
2659 }
3a60a9f5
A		 2660 if (kIOReturnSuccess != err)
2661 break;
2662
2663 // Header:
2664
2665 header->imageSize = vars->fileVars->position;
2666 header->image1Size = image1Size;
2667 header->bitmapSize = bitmap_size;
2668 header->pageCount = pageCount;
3a60a9f5
A		 2669
2670 header->restore1Sum = restore1Sum;
2671 header->image1Sum = sum1;
2672 header->image2Sum = sum2;
316670eb 2673 header->sleepTime = gIOLastSleepTime.tv_sec;
db609669
A		 2674
2675 header->compression = (compressedSize << 8) / uncompressedSize;
2676 gIOHibernateCompression = header->compression;
3a60a9f5
A		 2677
2678 count = vars->fileExtents->getLength();
2679 if (count > sizeof(header->fileExtentMap))
2680 {
2681 header->fileExtentMapSize = count;
2682 count = sizeof(header->fileExtentMap);
2683 }
2684 else
2685 header->fileExtentMapSize = sizeof(header->fileExtentMap);
2686 bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
0b4c1975 2687
39236c6e
A		 2688 header->deviceBase = vars->fileVars->block0;
2689 header->deviceBlockSize = vars->fileVars->blockSize;
6d2010ae 2690
3a60a9f5
A		 2691 IOPolledFileSeek(vars->fileVars, 0);
2692 err = IOPolledFileWrite(vars->fileVars,
2693 (uint8_t *) header, sizeof(IOHibernateImageHeader),
2694 cryptvars);
2695 if (kIOReturnSuccess != err)
2696 break;
2697 err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
2698 if (kIOReturnSuccess != err)
2699 break;
2d21ac55 2700 err = IOHibernatePollerIODone(vars->fileVars, true);
3a60a9f5
A		 2701 if (kIOReturnSuccess != err)
2702 break;
2703 }
2704 while (false);
2705
2706 clock_get_uptime(&endTime);
b0d623f7
A		 2707
2708 IOService::getPMRootDomain()->pmStatsRecordEvent(
2709 kIOPMStatsHibernateImageWrite | kIOPMStatsEventStopFlag, endTime);
2710
3a60a9f5
A		 2711 SUB_ABSOLUTETIME(&endTime, &allTime);
2712 absolutetime_to_nanoseconds(endTime, &nsec);
39236c6e 2713 HIBLOG("all time: %qd ms, ", nsec / 1000000ULL);
3a60a9f5
A		 2714
2715 absolutetime_to_nanoseconds(compTime, &nsec);
6d2010ae
A		 2716 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
2717 compBytes,
2718 nsec / 1000000ULL,
2719 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
3a60a9f5 2720
6d2010ae
A		 2721 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
2722 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s, ",
2723 vars->fileVars->cryptBytes,
2724 nsec / 1000000ULL,
2725 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
3a60a9f5 2726
db609669
A		 2727 HIBLOG("\nimage %qd (%lld%%), uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
2728 header->imageSize, (header->imageSize * 100) / vars->fileVars->fileSize,
3a60a9f5 2729 uncompressedSize, atop_32(uncompressedSize), compressedSize,
8f6c56a5 2730 uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
3a60a9f5
A		 2731 sum1, sum2);
2732
39236c6e
A		 2733 HIBLOG("zeroPageCount %d, wiredPagesEncrypted %d, wiredPagesClear %d, dirtyPagesEncrypted %d\n",
2734 zeroPageCount, wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted);
0b4c1975 2735
c910b4d9
A		 2736 if (vars->fileVars->io)
2737 (void) IOHibernatePollerIODone(vars->fileVars, false);
2738
3a60a9f5
A		 2739 if (pollerOpen)
2740 IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState);
2741
2d21ac55
A		 2742 if (vars->consoleMapping)
2743 ProgressUpdate(gIOHibernateGraphicsInfo,
2744 vars->consoleMapping, 0, kIOHibernateProgressCount);
2745
3a60a9f5
A		 2746 HIBLOG("hibernate_write_image done(%x)\n", err);
2747
2748 // should we come back via regular wake, set the state in memory.
2749 gIOHibernateState = kIOHibernateStateInactive;
2750
0b4c1975 2751 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 1) | DBG_FUNC_END,
39236c6e 2752 wiredPagesEncrypted, wiredPagesClear, dirtyPagesEncrypted, 0, 0);
0b4c1975 2753
2d21ac55
A		 2754 if (kIOReturnSuccess == err)
2755 {
2756 if (kIOHibernateModeSleep & gIOHibernateMode)
2757 {
2758 return (kIOHibernatePostWriteSleep);
2759 }
2760 else if(kIOHibernateModeRestart & gIOHibernateMode)
2761 {
2762 return (kIOHibernatePostWriteRestart);
2763 }
2764 else
2765 {
2766 /* by default, power down */
2767 return (kIOHibernatePostWriteHalt);
2768 }
2769 }
2770 else if (kIOReturnAborted == err)
2771 {
2772 return (kIOHibernatePostWriteWake);
2773 }
3a60a9f5 2774 else
3a60a9f5 2775 {
2d21ac55
A		 2776 /* on error, sleep */
2777 return (kIOHibernatePostWriteSleep);
3a60a9f5
A		 2778 }
2779}
2780
2781/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2782
2783extern "C" void
2784hibernate_machine_init(void)
2785{
2786 IOReturn err;
2787 uint32_t sum;
2788 uint32_t pagesDone;
0b4c1975 2789 uint32_t pagesRead = 0;
6d2010ae 2790 AbsoluteTime startTime, compTime;
3a60a9f5 2791 AbsoluteTime allTime, endTime;
39236c6e
A		 2792 AbsoluteTime startIOTime, endIOTime;
2793 uint64_t nsec, nsecIO;
6d2010ae 2794 uint64_t compBytes;
3a60a9f5
A		 2795 uint32_t lastProgressStamp = 0;
2796 uint32_t progressStamp;
3a60a9f5
A		 2797 hibernate_cryptvars_t * cryptvars = 0;
2798
2799 IOHibernateVars * vars = &gIOHibernateVars;
39236c6e 2800 bzero(gIOHibernateStats, sizeof(hibernate_statistics_t));
3a60a9f5
A		 2801
2802 if (!vars->fileVars || !vars->fileVars->pollers || !vars->fileExtents)
2803 return;
2804
3a60a9f5
A		 2805 sum = gIOHibernateCurrentHeader->actualImage1Sum;
2806 pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
2807
3a60a9f5
A		 2808 if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
2809 {
2810 HIBLOG("regular wake\n");
2811 return;
2812 }
2813
2814 HIBPRINT("diag %x %x %x %x\n",
2815 gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
6d2010ae 2816 gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
3a60a9f5 2817
39236c6e
A		 2818#define t40ms(x) (tmrCvt((((uint64_t)(x)) << 8), tscFCvtt2n) / 1000000)
2819#define tStat(x, y) gIOHibernateStats->x = t40ms(gIOHibernateCurrentHeader->y);
2820 tStat(booterStart, booterStart);
2821 gIOHibernateStats->smcStart = gIOHibernateCurrentHeader->smcStart,
2822 tStat(booterDuration0, booterTime0);
2823 tStat(booterDuration1, booterTime1);
2824 tStat(booterDuration2, booterTime2);
2825 tStat(booterDuration, booterTime);
2826 tStat(booterConnectDisplayDuration, connectDisplayTime);
2827 tStat(booterSplashDuration, splashTime);
2828 tStat(trampolineDuration, trampolineTime);
2829
2830 gIOHibernateStats->image1Size = gIOHibernateCurrentHeader->image1Size;
2831 gIOHibernateStats->imageSize = gIOHibernateCurrentHeader->imageSize;
2832 gIOHibernateStats->image1Pages = pagesDone;
2833
2834 HIBLOG("booter start at %d ms smc %d ms, [%d, %d, %d] total %d ms, dsply %d, %d ms, tramp %d ms\n",
2835 gIOHibernateStats->booterStart,
2836 gIOHibernateStats->smcStart,
2837 gIOHibernateStats->booterDuration0,
2838 gIOHibernateStats->booterDuration1,
2839 gIOHibernateStats->booterDuration2,
2840 gIOHibernateStats->booterDuration,
2841 gIOHibernateStats->booterConnectDisplayDuration,
2842 gIOHibernateStats->booterSplashDuration,
2843 gIOHibernateStats->trampolineDuration);
2844
2845 HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, imageSize 0x%qx, image1Size 0x%qx, conflictCount %d, nextFree %x\n",
2846 gIOHibernateState, pagesDone, sum, gIOHibernateStats->imageSize, gIOHibernateStats->image1Size,
2847 gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
bd504ef0 2848
39236c6e
A		 2849 if ((0 != (kIOHibernateModeSleep & gIOHibernateMode))
2850 && (0 != ((kIOHibernateModeDiscardCleanActive | kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)))
2851 {
0c530ab8 2852 hibernate_page_list_discard(vars->page_list);
39236c6e 2853 }
0c530ab8 2854
6d2010ae
A		 2855 cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
2856
2857 if (gIOHibernateCurrentHeader->handoffPageCount > gIOHibernateHandoffPageCount)
2858 panic("handoff overflow");
2859
2860 IOHibernateHandoff * handoff;
2861 bool done = false;
2862 bool foundCryptData = false;
2863
2864 for (handoff = (IOHibernateHandoff *) vars->handoffBuffer->getBytesNoCopy();
2865 !done;
2866 handoff = (IOHibernateHandoff *) &handoff->data[handoff->bytecount])
2867 {
2868// HIBPRINT("handoff %p, %x, %x\n", handoff, handoff->type, handoff->bytecount);
2869 uint8_t * data = &handoff->data[0];
2870 switch (handoff->type)
2871 {
2872 case kIOHibernateHandoffTypeEnd:
2873 done = true;
2874 break;
2875
2876 case kIOHibernateHandoffTypeGraphicsInfo:
2877 bcopy(data, gIOHibernateGraphicsInfo, sizeof(*gIOHibernateGraphicsInfo));
2878 break;
2879
2880 case kIOHibernateHandoffTypeCryptVars:
2881 if (cryptvars)
2882 {
2883 hibernate_cryptwakevars_t *
2884 wakevars = (hibernate_cryptwakevars_t *) &handoff->data[0];
2885 bcopy(&wakevars->aes_iv[0], &cryptvars->aes_iv[0], sizeof(cryptvars->aes_iv));
2886 }
2887 foundCryptData = true;
2888 bzero(data, handoff->bytecount);
2889 break;
2890
2891 case kIOHibernateHandoffTypeMemoryMap:
bd504ef0
A		 2892
2893 clock_get_uptime(&allTime);
2894
6d2010ae
A		 2895 hibernate_newruntime_map(data, handoff->bytecount,
2896 gIOHibernateCurrentHeader->systemTableOffset);
bd504ef0
A		 2897
2898 clock_get_uptime(&endTime);
2899
2900 SUB_ABSOLUTETIME(&endTime, &allTime);
2901 absolutetime_to_nanoseconds(endTime, &nsec);
2902
2903 HIBLOG("hibernate_newruntime_map time: %qd ms, ", nsec / 1000000ULL);
2904
6d2010ae
A		 2905 break;
2906
2907 case kIOHibernateHandoffTypeDeviceTree:
2908 {
2909// DTEntry chosen = NULL;
2910// HIBPRINT("DTLookupEntry %d\n", DTLookupEntry((const DTEntry) data, "/chosen", &chosen));
2911 }
2912 break;
2913
2914 default:
2915 done = (kIOHibernateHandoffType != (handoff->type & 0xFFFF0000));
2916 break;
2917 }
2918 }
2919 if (cryptvars && !foundCryptData)
2920 panic("hibernate handoff");
2921
316670eb
A		 2922 HIBPRINT("video %x %d %d %d status %x\n",
2923 gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
2924 gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height, gIOHibernateGraphicsInfo->gfxStatus);
2925
2926 if (vars->videoMapping && gIOHibernateGraphicsInfo->physicalAddress)
3a60a9f5
A		 2927 {
2928 vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
2929 * gIOHibernateGraphicsInfo->rowBytes);
22ba694c
A		 2930 if (vars->videoMapSize > vars->videoAllocSize) vars->videoMapSize = 0;
2931 else
2932 {
2933 IOMapPages(kernel_map,
2934 vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
2935 vars->videoMapSize, kIOMapInhibitCache );
2936 }
3a60a9f5
A		 2937 }
2938
316670eb
A		 2939 if (vars->videoMapSize)
2940 ProgressUpdate(gIOHibernateGraphicsInfo,
2941 (uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
2942
0c530ab8 2943 uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
39236c6e
A		 2944 uint8_t * compressed = src + page_size;
2945 uint8_t * scratch = compressed + page_size;
2946 uint32_t decoOffset;
3a60a9f5
A		 2947
2948 clock_get_uptime(&allTime);
6d2010ae
A		 2949 AbsoluteTime_to_scalar(&compTime) = 0;
2950 compBytes = 0;
3a60a9f5
A		 2951
2952 HIBLOG("IOHibernatePollerOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
2953 err = IOHibernatePollerOpen(vars->fileVars, kIOPolledAfterSleepState, 0);
39236c6e
A		 2954 clock_get_uptime(&startIOTime);
2955 endTime = startIOTime;
2956 SUB_ABSOLUTETIME(&endTime, &allTime);
2957 absolutetime_to_nanoseconds(endTime, &nsec);
2958 HIBLOG("IOHibernatePollerOpen(%x) %qd ms\n", err, nsec / 1000000ULL);
3a60a9f5 2959
3a60a9f5
A		 2960 IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
2961
3a60a9f5
A		 2962 // kick off the read ahead
2963 vars->fileVars->io = false;
2964 vars->fileVars->bufferHalf = 0;
2965 vars->fileVars->bufferLimit = 0;
2966 vars->fileVars->lastRead = 0;
6d2010ae 2967 vars->fileVars->readEnd = gIOHibernateCurrentHeader->imageSize;
3a60a9f5 2968 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
6d2010ae
A		 2969 vars->fileVars->cryptBytes = 0;
2970 AbsoluteTime_to_scalar(&vars->fileVars->cryptTime) = 0;
3a60a9f5 2971
6d2010ae 2972 err = IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
3a60a9f5
A		 2973 vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
2974 // --
2975
2976 HIBLOG("hibernate_machine_init reading\n");
2977
2978 uint32_t * header = (uint32_t *) src;
2979 sum = 0;
2980
6d2010ae 2981 while (kIOReturnSuccess == err)
3a60a9f5
A		 2982 {
2983 unsigned int count;
2984 unsigned int page;
2985 uint32_t tag;
2986 vm_offset_t ppnum, compressedSize;
2987
36401178
A		 2988 err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
2989 if (kIOReturnSuccess != err)
2990 break;
3a60a9f5
A		 2991
2992 ppnum = header[0];
2993 count = header[1];
2994
2995// HIBPRINT("(%x, %x)\n", ppnum, count);
2996
2997 if (!count)
2998 break;
2999
3000 for (page = 0; page < count; page++)
3001 {
36401178
A		 3002 err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
3003 if (kIOReturnSuccess != err)
3004 break;
3a60a9f5
A		 3005
3006 compressedSize = kIOHibernateTagLength & tag;
36401178
A		 3007 if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength))
3008 {
3009 err = kIOReturnIPCError;
3010 break;
3011 }
3012
39236c6e 3013 if (!compressedSize) bzero_phys(ptoa_64(ppnum), page_size);
3a60a9f5 3014 else
36401178 3015 {
39236c6e
A		 3016 err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
3017 if (kIOReturnSuccess != err) break;
3018 if (compressedSize < page_size)
3019 {
3020 decoOffset = page_size;
3021 clock_get_uptime(&startTime);
3022 WKdm_decompress_new((WK_word*) src, (WK_word*) compressed, (WK_word*) scratch, page_size);
3023 clock_get_uptime(&endTime);
3024 ADD_ABSOLUTETIME(&compTime, &endTime);
3025 SUB_ABSOLUTETIME(&compTime, &startTime);
3026 compBytes += page_size;
3027 }
3028 else decoOffset = 0;
3029
3030 sum += hibernate_sum_page((src + decoOffset), ppnum);
3031 err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
3032 if (err)
3033 {
3034 HIBLOG("IOMemoryDescriptorReadToPhysical [%ld] %x\n", (long)ppnum, err);
3035 break;
3036 }
36401178 3037 }
3a60a9f5
A		 3038
3039 ppnum++;
3040 pagesDone++;
0b4c1975 3041 pagesRead++;
3a60a9f5 3042
3a60a9f5
A		 3043 if (0 == (8191 & pagesDone))
3044 {
3045 clock_get_uptime(&endTime);
3046 SUB_ABSOLUTETIME(&endTime, &allTime);
3047 absolutetime_to_nanoseconds(endTime, &nsec);
3048 progressStamp = nsec / 750000000ULL;
3049 if (progressStamp != lastProgressStamp)
3050 {
3051 lastProgressStamp = progressStamp;
3052 HIBPRINT("pages %d (%d%%)\n", pagesDone,
3053 (100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
3054 }
3055 }
3056 }
3057 }
7ddcb079
A		 3058 if ((kIOReturnSuccess == err) && (pagesDone == gIOHibernateCurrentHeader->actualUncompressedPages))
3059 err = kIOReturnLockedRead;
3a60a9f5 3060
36401178
A		 3061 if (kIOReturnSuccess != err)
3062 panic("Hibernate restore error %x", err);
3063
3a60a9f5 3064 gIOHibernateCurrentHeader->actualImage2Sum = sum;
db609669 3065 gIOHibernateCompression = gIOHibernateCurrentHeader->compression;
3a60a9f5
A		 3066
3067 if (vars->fileVars->io)
2d21ac55 3068 (void) IOHibernatePollerIODone(vars->fileVars, false);
3a60a9f5 3069
39236c6e
A		 3070 clock_get_uptime(&endIOTime);
3071
3a60a9f5
A		 3072 err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState);
3073
3a60a9f5 3074 clock_get_uptime(&endTime);
b0d623f7
A		 3075
3076 IOService::getPMRootDomain()->pmStatsRecordEvent(
3077 kIOPMStatsHibernateImageRead | kIOPMStatsEventStartFlag, allTime);
3078 IOService::getPMRootDomain()->pmStatsRecordEvent(
3079 kIOPMStatsHibernateImageRead | kIOPMStatsEventStopFlag, endTime);
3080
3a60a9f5
A		 3081 SUB_ABSOLUTETIME(&endTime, &allTime);
3082 absolutetime_to_nanoseconds(endTime, &nsec);
3083
39236c6e
A		 3084 SUB_ABSOLUTETIME(&endIOTime, &startIOTime);
3085 absolutetime_to_nanoseconds(endIOTime, &nsecIO);
3086
3087 gIOHibernateStats->kernelImageReadDuration = nsec / 1000000ULL;
3088 gIOHibernateStats->imagePages = pagesDone;
3089
3090 HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %d ms, disk(0x%x) %qd Mb/s, ",
3091 pagesDone, sum, gIOHibernateStats->kernelImageReadDuration, kDefaultIOSize,
3092 nsecIO ? ((((gIOHibernateCurrentHeader->imageSize - gIOHibernateCurrentHeader->image1Size) * 1000000000ULL) / 1024 / 1024) / nsecIO) : 0);
6d2010ae
A		 3093
3094 absolutetime_to_nanoseconds(compTime, &nsec);
3095 HIBLOG("comp bytes: %qd time: %qd ms %qd Mb/s, ",
3096 compBytes,
3097 nsec / 1000000ULL,
3098 nsec ? (((compBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
3099
3100 absolutetime_to_nanoseconds(vars->fileVars->cryptTime, &nsec);
3101 HIBLOG("crypt bytes: %qd time: %qd ms %qd Mb/s\n",
3102 vars->fileVars->cryptBytes,
3103 nsec / 1000000ULL,
3104 nsec ? (((vars->fileVars->cryptBytes * 1000000000ULL) / 1024 / 1024) / nsec) : 0);
3105
3106 KERNEL_DEBUG_CONSTANT(IOKDBG_CODE(DBG_HIBERNATE, 2) | DBG_FUNC_NONE, pagesRead, pagesDone, 0, 0, 0);
3a60a9f5
A		 3107}
3108
3109/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
99c3a104 3110
39236c6e
A		 3111void IOHibernateSetWakeCapabilities(uint32_t capability)
3112{
3113 if (kIOHibernateStateWakingFromHibernate == gIOHibernateState)
3114 {
3115 gIOHibernateStats->wakeCapability = capability;
3116 }
3117}
3118
3119/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
3120
99c3a104
A		 3121void IOHibernateSystemRestart(void)
3122{
3123 static uint8_t noteStore[32] __attribute__((aligned(32)));
3124 IORegistryEntry * regEntry;
3125 const OSSymbol * sym;
3126 OSData * noteProp;
3127 OSData * data;
3128 uintptr_t * smcVars;
3129 uint8_t * smcBytes;
3130 size_t len;
3131 addr64_t element;
3132
3133 data = OSDynamicCast(OSData, IOService::getPMRootDomain()->getProperty(kIOHibernateSMCVariablesKey));
3134 if (!data) return;
3135
3136 smcVars = (typeof(smcVars)) data->getBytesNoCopy();
3137 smcBytes = (typeof(smcBytes)) smcVars[1];
3138 len = smcVars[0];
3139 if (len > sizeof(noteStore)) len = sizeof(noteStore);
3140 noteProp = OSData::withCapacity(3 * sizeof(element));
3141 if (!noteProp) return;
3142 element = len;
3143 noteProp->appendBytes(&element, sizeof(element));
3144 element = crc32(0, smcBytes, len);
3145 noteProp->appendBytes(&element, sizeof(element));
3146
3147 bcopy(smcBytes, noteStore, len);
3148 element = (addr64_t) &noteStore[0];
3149 element = (element & page_mask) | ptoa_64(pmap_find_phys(kernel_pmap, element));
3150 noteProp->appendBytes(&element, sizeof(element));
3151
3152 if (!gIOOptionsEntry)
3153 {
3154 regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
3155 gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
3156 if (regEntry && !gIOOptionsEntry)
3157 regEntry->release();
3158 }
3159
3160 sym = OSSymbol::withCStringNoCopy(kIOHibernateBootNoteKey);
3161 if (gIOOptionsEntry && sym) gIOOptionsEntry->setProperty(sym, noteProp);
3162 if (noteProp) noteProp->release();
3163 if (sym) sym->release();
3164}
3165
3166
3167
mirror of XNU