git.saurik.com Git - apple/xnu.git/blame_incremental

... / ...

Commit	Line	Data
	1	/*
	2	* Copyright (c) 2004-2006 Apple Computer, Inc. All rights reserved.
	3	*
	4	* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
	5	*
	6	* This file contains Original Code and/or Modifications of Original Code
	7	* as defined in and that are subject to the Apple Public Source License
	8	* Version 2.0 (the 'License'). You may not use this file except in
	9	* compliance with the License. The rights granted to you under the License
	10	* may not be used to create, or enable the creation or redistribution of,
	11	* unlawful or unlicensed copies of an Apple operating system, or to
	12	* circumvent, violate, or enable the circumvention or violation of, any
	13	* terms of an Apple operating system software license agreement.
	14	*
	15	* Please obtain a copy of the License at
	16	* http://www.opensource.apple.com/apsl/ and read it before using this file.
	17	*
	18	* The Original Code and all software distributed under the License are
	19	* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
	20	* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
	21	* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
	22	* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
	23	* Please see the License for the specific language governing rights and
	24	* limitations under the License.
	25	*
	26	* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
	27	*/
	28
	29
	30	/*
	31
	32	Sleep:
	33
	34	- PMRootDomain calls IOHibernateSystemSleep() before system sleep
	35	(devices awake, normal execution context)
	36	- IOHibernateSystemSleep opens the hibernation file (or partition) at the bsd level,
	37	grabs its extents and searches for a polling driver willing to work with that IOMedia.
	38	The BSD code makes an ioctl to the storage driver to get the partition base offset to
	39	the disk, and other ioctls to get the transfer constraints
	40	If successful, the file is written to make sure its initially not bootable (in case of
	41	later failure) and nvram set to point to the first block of the file. (Has to be done
	42	here so blocking is possible in nvram support).
	43	hibernate_setup() in osfmk is called to allocate page bitmaps for all dram, and
	44	page out any pages it wants to (currently zero, but probably some percentage of memory).
	45	Its assumed just allocating pages will cause the VM system to naturally select the best
	46	pages for eviction. It also copies processor flags needed for the restore path and sets
	47	a flag in the boot processor proc info.
	48	gIOHibernateState = kIOHibernateStateHibernating.
	49	- Regular sleep progresses - some drivers may inspect the root domain property
	50	kIOHibernateStateKey to modify behavior. The platform driver saves state to memory
	51	as usual but leaves motherboard I/O on.
	52	- Eventually the platform calls ml_ppc_sleep() in the shutdown context on the last cpu,
	53	at which point memory is ready to be saved. mapping_hibernate_flush() is called to get
	54	all ppc RC bits out of the hash table and caches into the mapping structures.
	55	- hibernate_write_image() is called (still in shutdown context, no blocking or preemption).
	56	hibernate_page_list_setall() is called to get a bitmap of dram pages that need to be saved.
	57	All pages are assumed to be saved (as part of the wired image) unless explicitly subtracted
	58	by hibernate_page_list_setall(), avoiding having to find arch dependent low level bits.
	59	The image header and block list are written. The header includes the second file extent so
	60	only the header block is needed to read the file, regardless of filesystem.
	61	The kernel section "__HIB" is written uncompressed to the image. This section of code and data
	62	(only) is used to decompress the image during wake/boot.
	63	Some additional pages are removed from the bitmaps - the buffers used for hibernation.
	64	The bitmaps are written to the image.
	65	More areas are removed from the bitmaps (after they have been written to the image) - the
	66	section "__HIB" pages and interrupt stack.
	67	Each wired page is compressed and written and then each non-wired page. Compression and
	68	disk writes are in parallel.
	69	The image header is written to the start of the file and the polling driver closed.
	70	The machine powers down (or sleeps).
	71
	72	Boot/Wake:
	73
	74	- BootX sees the boot-image nvram variable containing the device and block number of the image,
	75	reads the header and if the signature is correct proceeds. The boot-image variable is cleared.
	76	- BootX reads the portion of the image used for wired pages, to memory. Its assumed this will fit
	77	in the OF memory environment, and the image is decrypted. There is no decompression in BootX,
	78	that is in the kernel's __HIB section.
	79	- BootX copies the "__HIB" section to its correct position in memory, quiesces and calls its entry
	80	hibernate_kernel_entrypoint(), passing the location of the image in memory. Translation is off,
	81	only code & data in that section is safe to call since all the other wired pages are still
	82	compressed in the image.
	83	- hibernate_kernel_entrypoint() removes pages occupied by the raw image from the page bitmaps.
	84	It uses the bitmaps to work out which pages can be uncompressed from the image to their final
	85	location directly, and copies those that can't to interim free pages. When the image has been
	86	completed, the copies are uncompressed, overwriting the wired image pages.
	87	hibernate_restore_phys_page() (in osfmk since its arch dependent, but part of the "__HIB" section)
	88	is used to get pages into place for 64bit.
	89	- the reset vector is called (at least on ppc), the kernel proceeds on a normal wake, with some
	90	changes conditional on the per proc flag - before VM is turned on the boot cpu, all mappings
	91	are removed from the software strutures, and the hash table is reinitialized.
	92	- After the platform CPU init code is called, hibernate_machine_init() is called to restore the rest
	93	of memory, using the polled mode driver, before other threads can run or any devices are turned on.
	94	This reduces the memory usage for BootX and allows decompression in parallel with disk reads,
	95	for the remaining non wired pages.
	96	- The polling driver is closed down and regular wake proceeds. When the kernel calls iokit to wake
	97	(normal execution context) hibernate_teardown() in osmfk is called to release any memory, the file
	98	is closed via bsd.
	99
	100	Polled Mode I/O:
	101
	102	IOHibernateSystemSleep() finds a polled mode interface to the ATA controller via a property in the
	103	registry, specifying an object of calls IOPolledInterface.
	104
	105	Before the system goes to sleep it searches from the IOMedia object (could be a filesystem or
	106	partition) that the image is going to live, looking for polled interface properties. If it finds
	107	one the IOMedia object is passed to a "probe" call for the interface to accept or reject. All the
	108	interfaces found are kept in an ordered list.
	109
	110	There is an Open/Close pair of calls made to each of the interfaces at various stages since there are
	111	few different contexts things happen in:
	112
	113	- there is an Open/Close (Preflight) made before any part of the system has slept (I/O is all
	114	up and running) and after wake - this is safe to allocate memory and do anything. The device
	115	ignores sleep requests from that point since its a waste of time if it goes to sleep and
	116	immediately wakes back up for the image write.
	117
	118	- there is an Open/Close (BeforeSleep) pair made around the image write operations that happen
	119	immediately before sleep. These can't block or allocate memory - the I/O system is asleep apart
	120	from the low level bits (motherboard I/O etc). There is only one thread running. The close can be
	121	used to flush and set the disk to sleep.
	122
	123	- there is an Open/Close (AfterSleep) pair made around the image read operations that happen
	124	immediately after sleep. These can't block or allocate memory. This is happening after the platform
	125	expert has woken the low level bits of the system, but most of the I/O system has not. There is only
	126	one thread running.
	127
	128	For the actual I/O, all the ops are with respect to a single IOMemoryDescriptor that was passed
	129	(prepared) to the Preflight Open() call. There is a read/write op, buffer offset to the IOMD for
	130	the data, an offset to the disk and length (block aligned 64 bit numbers), and completion callback.
	131	Each I/O is async but only one is ever outstanding. The polled interface has a checkForWork call
	132	that is called for the hardware to check for events, and complete the I/O via the callback.
	133	The hibernate path uses the same transfer constraints the regular cluster I/O path in BSD uses
	134	to restrict I/O ops.
	135	*/
	136
	137	#include <sys/systm.h>
	138
	139	#include <IOKit/IOWorkLoop.h>
	140	#include <IOKit/IOCommandGate.h>
	141	#include <IOKit/IOTimerEventSource.h>
	142	#include <IOKit/IOPlatformExpert.h>
	143	#include <IOKit/IOKitDebug.h>
	144	#include <IOKit/IOTimeStamp.h>
	145	#include <IOKit/pwr_mgt/RootDomain.h>
	146	#include <IOKit/pwr_mgt/IOPMPrivate.h>
	147	#include <IOKit/IOMessage.h>
	148	#include <IOKit/IODeviceTreeSupport.h>
	149	#include <IOKit/IOBSD.h>
	150	#include "RootDomainUserClient.h"
	151	#include <IOKit/pwr_mgt/IOPowerConnection.h>
	152	#include "IOPMPowerStateQueue.h"
	153	#include <IOKit/IOBufferMemoryDescriptor.h>
	154	#include <crypto/aes.h>
	155
	156	#include <sys/uio.h>
	157	#include <sys/conf.h>
	158	#include <sys/stat.h>
	159	#include <sys/fcntl.h> // (FWRITE, ...)
	160	#include <sys/sysctl.h>
	161
	162	#include <IOKit/IOHibernatePrivate.h>
	163	#include <IOKit/IOPolledInterface.h>
	164	#include <IOKit/IONVRAM.h>
	165	#include "IOHibernateInternal.h"
	166	#include "WKdm.h"
	167	#include "IOKitKernelInternal.h"
	168
	169	/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
	170
	171	OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject);
	172
	173	OSMetaClassDefineReservedUnused(IOPolledInterface, 0);
	174	OSMetaClassDefineReservedUnused(IOPolledInterface, 1);
	175	OSMetaClassDefineReservedUnused(IOPolledInterface, 2);
	176	OSMetaClassDefineReservedUnused(IOPolledInterface, 3);
	177	OSMetaClassDefineReservedUnused(IOPolledInterface, 4);
	178	OSMetaClassDefineReservedUnused(IOPolledInterface, 5);
	179	OSMetaClassDefineReservedUnused(IOPolledInterface, 6);
	180	OSMetaClassDefineReservedUnused(IOPolledInterface, 7);
	181	OSMetaClassDefineReservedUnused(IOPolledInterface, 8);
	182	OSMetaClassDefineReservedUnused(IOPolledInterface, 9);
	183	OSMetaClassDefineReservedUnused(IOPolledInterface, 10);
	184	OSMetaClassDefineReservedUnused(IOPolledInterface, 11);
	185	OSMetaClassDefineReservedUnused(IOPolledInterface, 12);
	186	OSMetaClassDefineReservedUnused(IOPolledInterface, 13);
	187	OSMetaClassDefineReservedUnused(IOPolledInterface, 14);
	188	OSMetaClassDefineReservedUnused(IOPolledInterface, 15);
	189
	190	/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
	191
	192	extern uint32_t gIOHibernateState;
	193	uint32_t gIOHibernateMode;
	194	static char gIOHibernateBootSignature[256+1];
	195	static char gIOHibernateFilename[MAXPATHLEN+1];
	196	static uint32_t gIOHibernateFreeRatio = 0; // free page target (percent)
	197	uint32_t gIOHibernateFreeTime = 0*1000; // max time to spend freeing pages (ms)
	198
	199	static IODTNVRAM * gIOOptionsEntry;
	200	static IORegistryEntry * gIOChosenEntry;
	201	#ifdef __i386__
	202	static const OSSymbol * gIOCreateEFIDevicePathSymbol;
	203	#endif
	204
	205	static IOPolledFileIOVars gFileVars;
	206	static IOHibernateVars gIOHibernateVars;
	207	static struct kern_direct_file_io_ref_t * gIOHibernateFileRef;
	208	static hibernate_cryptvars_t gIOHibernateCryptWakeContext;
	209
	210	/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
	211
	212	enum { kXPRamAudioVolume = 8 };
	213	enum { kDefaultIOSize = 128 * 1024 };
	214	enum { kVideoMapSize = 32 * 1024 * 1024 };
	215
	216	#ifndef kIOMediaPreferredBlockSizeKey
	217	#define kIOMediaPreferredBlockSizeKey "Preferred Block Size"
	218	#endif
	219
	220	#ifndef kIOBootPathKey
	221	#define kIOBootPathKey "bootpath"
	222	#endif
	223	#ifndef kIOSelectedBootDeviceKey
	224	#define kIOSelectedBootDeviceKey "boot-device"
	225	#endif
	226
	227
	228	enum { kIOHibernateMinPollersNeeded = 2 };
	229
	230	/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
	231
	232	// copy from phys addr to MD
	233
	234	static IOReturn
	235	IOMemoryDescriptorWriteFromPhysical(IOMemoryDescriptor * md,
	236	IOByteCount offset, addr64_t bytes, IOByteCount length)
	237	{
	238	addr64_t srcAddr = bytes;
	239	IOByteCount remaining;
	240
	241	remaining = length = min(length, md->getLength() - offset);
	242	while (remaining) { // (process another target segment?)
	243	addr64_t dstAddr64;
	244	IOByteCount dstLen;
	245
	246	dstAddr64 = md->getPhysicalSegment64(offset, &dstLen);
	247	if (!dstAddr64)
	248	break;
	249
	250	// Clip segment length to remaining
	251	if (dstLen > remaining)
	252	dstLen = remaining;
	253
	254	#if 1
	255	bcopy_phys(srcAddr, dstAddr64, dstLen);
	256	#else
	257	copypv(srcAddr, dstAddr64, dstLen,
	258	cppvPsnk \| cppvFsnk \| cppvNoRefSrc \| cppvNoModSnk \| cppvKmap);
	259	#endif
	260	srcAddr += dstLen;
	261	offset += dstLen;
	262	remaining -= dstLen;
	263	}
	264
	265	assert(!remaining);
	266
	267	return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
	268	}
	269
	270	// copy from MD to phys addr
	271
	272	static IOReturn
	273	IOMemoryDescriptorReadToPhysical(IOMemoryDescriptor * md,
	274	IOByteCount offset, addr64_t bytes, IOByteCount length)
	275	{
	276	addr64_t dstAddr = bytes;
	277	IOByteCount remaining;
	278
	279	remaining = length = min(length, md->getLength() - offset);
	280	while (remaining) { // (process another target segment?)
	281	addr64_t srcAddr64;
	282	IOByteCount dstLen;
	283
	284	srcAddr64 = md->getPhysicalSegment64(offset, &dstLen);
	285	if (!srcAddr64)
	286	break;
	287
	288	// Clip segment length to remaining
	289	if (dstLen > remaining)
	290	dstLen = remaining;
	291
	292	#if 1
	293	bcopy_phys(srcAddr64, dstAddr, dstLen);
	294	#else
	295	copypv(srcAddr, dstAddr64, dstLen,
	296	cppvPsnk \| cppvFsnk \| cppvNoRefSrc \| cppvNoModSnk \| cppvKmap);
	297	#endif
	298	dstAddr += dstLen;
	299	offset += dstLen;
	300	remaining -= dstLen;
	301	}
	302
	303	assert(!remaining);
	304
	305	return remaining ? kIOReturnUnderrun : kIOReturnSuccess;
	306	}
	307
	308	/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
	309
	310	void
	311	hibernate_set_page_state(hibernate_page_list_t * page_list, hibernate_page_list_t * page_list_wired,
	312	vm_offset_t ppnum, vm_offset_t count, uint32_t kind)
	313	{
	314	count += ppnum;
	315	switch (kind)
	316	{
	317	case kIOHibernatePageStateUnwiredSave:
	318	// unwired save
	319	for (; ppnum < count; ppnum++)
	320	{
	321	hibernate_page_bitset(page_list, FALSE, ppnum);
	322	hibernate_page_bitset(page_list_wired, TRUE, ppnum);
	323	}
	324	break;
	325	case kIOHibernatePageStateWiredSave:
	326	// wired save
	327	for (; ppnum < count; ppnum++)
	328	{
	329	hibernate_page_bitset(page_list, FALSE, ppnum);
	330	hibernate_page_bitset(page_list_wired, FALSE, ppnum);
	331	}
	332	break;
	333	case kIOHibernatePageStateFree:
	334	// free page
	335	for (; ppnum < count; ppnum++)
	336	{
	337	hibernate_page_bitset(page_list, TRUE, ppnum);
	338	hibernate_page_bitset(page_list_wired, TRUE, ppnum);
	339	}
	340	break;
	341	default:
	342	panic("hibernate_set_page_state");
	343	}
	344	}
	345
	346	static vm_offset_t
	347	hibernate_page_list_iterate(hibernate_page_list_t * list, vm_offset_t * pPage)
	348	{
	349	uint32_t page = *pPage;
	350	uint32_t count;
	351	hibernate_bitmap_t * bitmap;
	352
	353	while ((bitmap = hibernate_page_bitmap_pin(list, &page)))
	354	{
	355	count = hibernate_page_bitmap_count(bitmap, TRUE, page);
	356	if (!count)
	357	break;
	358	page += count;
	359	if (page <= bitmap->last_page)
	360	break;
	361	}
	362
	363	*pPage = page;
	364	if (bitmap)
	365	count = hibernate_page_bitmap_count(bitmap, FALSE, page);
	366	else
	367	count = 0;
	368
	369	return (count);
	370	}
	371
	372	/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
	373
	374	static IOReturn
	375	IOHibernatePollerProbe(IOPolledFileIOVars * vars, IOService * target)
	376	{
	377	IOReturn err = kIOReturnError;
	378	int32_t idx;
	379	IOPolledInterface * poller;
	380
	381	for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
	382	{
	383	poller = (IOPolledInterface *) vars->pollers->getObject(idx);
	384	err = poller->probe(target);
	385	if (err)
	386	{
	387	HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
	388	break;
	389	}
	390	}
	391
	392	return (err);
	393	}
	394
	395	static IOReturn
	396	IOHibernatePollerOpen(IOPolledFileIOVars * vars, uint32_t state, IOMemoryDescriptor * md)
	397	{
	398	IOReturn err = kIOReturnError;
	399	int32_t idx;
	400	IOPolledInterface * poller;
	401
	402	for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
	403	{
	404	poller = (IOPolledInterface *) vars->pollers->getObject(idx);
	405	err = poller->open(state, md);
	406	if (err)
	407	{
	408	HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
	409	break;
	410	}
	411	}
	412
	413	return (err);
	414	}
	415
	416	static IOReturn
	417	IOHibernatePollerClose(IOPolledFileIOVars * vars, uint32_t state)
	418	{
	419	IOReturn err = kIOReturnError;
	420	int32_t idx;
	421	IOPolledInterface * poller;
	422
	423	for (idx = 0;
	424	(poller = (IOPolledInterface *) vars->pollers->getObject(idx));
	425	idx++)
	426	{
	427	err = poller->close(state);
	428	if (err)
	429	HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
	430	}
	431
	432	return (err);
	433	}
	434
	435	static void
	436	IOHibernatePollerIOComplete(void * target,
	437	void * parameter,
	438	IOReturn status,
	439	UInt64 actualByteCount)
	440	{
	441	IOPolledFileIOVars * vars = (IOPolledFileIOVars *) parameter;
	442
	443	vars->ioStatus = status;
	444	}
	445
	446	static IOReturn
	447	IOHibernatePollerIO(IOPolledFileIOVars * vars,
	448	uint32_t operation, uint32_t bufferOffset,
	449	uint64_t deviceOffset, uint64_t length)
	450	{
	451
	452	IOReturn err = kIOReturnError;
	453	IOPolledInterface * poller;
	454	IOPolledCompletion completion;
	455
	456	completion.target = 0;
	457	completion.action = &IOHibernatePollerIOComplete;
	458	completion.parameter = vars;
	459
	460	vars->ioStatus = -1;
	461
	462	poller = (IOPolledInterface *) vars->pollers->getObject(0);
	463	err = poller->startIO(operation, bufferOffset, deviceOffset + vars->block0, length, completion);
	464	if (err)
	465	HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
	466
	467	return (err);
	468	}
	469
	470	static IOReturn
	471	IOHibernatePollerIODone(IOPolledFileIOVars * vars, bool abortable)
	472	{
	473	IOReturn err = kIOReturnSuccess;
	474	int32_t idx = 0;
	475	IOPolledInterface * poller;
	476
	477	while (-1 == vars->ioStatus)
	478	{
	479	for (idx = 0;
	480	(poller = (IOPolledInterface *) vars->pollers->getObject(idx));
	481	idx++)
	482	{
	483	IOReturn newErr;
	484	newErr = poller->checkForWork();
	485	if ((newErr == kIOReturnAborted) && !abortable)
	486	newErr = kIOReturnSuccess;
	487	if (kIOReturnSuccess == err)
	488	err = newErr;
	489	}
	490	}
	491
	492	if (err)
	493	{
	494	HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
	495	}
	496	else
	497	{
	498	err = vars->ioStatus;
	499	if (kIOReturnSuccess != err)
	500	HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
	501	}
	502
	503	return (err);
	504	}
	505
	506	IOReturn
	507	IOPolledInterface::checkAllForWork(void)
	508	{
	509	IOReturn err = kIOReturnNotReady;
	510	int32_t idx;
	511	IOPolledInterface * poller;
	512
	513	IOHibernateVars * vars = &gIOHibernateVars;
	514
	515	if (!vars->fileVars \|\| !vars->fileVars->pollers)
	516	return (err);
	517
	518	for (idx = 0;
	519	(poller = (IOPolledInterface *) vars->fileVars->pollers->getObject(idx));
	520	idx++)
	521	{
	522	err = poller->checkForWork();
	523	if (err)
	524	HIBLOG("IOPolledInterface::checkAllForWork[%d] 0x%x\n", idx, err);
	525	}
	526
	527	return (err);
	528	}
	529
	530	/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
	531
	532	struct _OpenFileContext
	533	{
	534	OSData * extents;
	535	uint64_t size;
	536	};
	537
	538	static void
	539	file_extent_callback(void * ref, uint64_t start, uint64_t length)
	540	{
	541	_OpenFileContext * ctx = (_OpenFileContext *) ref;
	542	IOPolledFileExtent extent;
	543
	544	extent.start = start;
	545	extent.length = length;
	546
	547	ctx->extents->appendBytes(&extent, sizeof(extent));
	548	ctx->size += length;
	549	}
	550
	551	IOReturn
	552	IOPolledFileOpen( const char * filename, IOBufferMemoryDescriptor * ioBuffer,
	553	IOPolledFileIOVars fileVars, OSData fileExtents,
	554	OSData ** imagePath)
	555	{
	556	IOReturn err = kIOReturnError;
	557	IOPolledFileIOVars * vars;
	558	_OpenFileContext ctx;
	559	OSData * extentsData;
	560	OSNumber * num;
	561	IORegistryEntry * part = 0;
	562	OSDictionary * matching;
	563	OSIterator * iter;
	564	dev_t hibernate_image_dev;
	565	uint64_t maxiobytes;
	566
	567	vars = &gFileVars;
	568	do
	569	{
	570	HIBLOG("sizeof(IOHibernateImageHeader) == %ld\n", sizeof(IOHibernateImageHeader));
	571	if (sizeof(IOHibernateImageHeader) != 512)
	572	continue;
	573
	574	vars->io = false;
	575	vars->buffer = (uint8_t *) ioBuffer->getBytesNoCopy();
	576	vars->bufferHalf = 0;
	577	vars->bufferOffset = 0;
	578	vars->bufferSize = ioBuffer->getLength() >> 1;
	579
	580	extentsData = OSData::withCapacity(32);
	581
	582	ctx.extents = extentsData;
	583	ctx.size = 0;
	584	vars->fileRef = kern_open_file_for_direct_io(filename,
	585	&file_extent_callback, &ctx,
	586	&hibernate_image_dev,
	587	&vars->block0,
	588	&maxiobytes);
	589	if (!vars->fileRef)
	590	{
	591	err = kIOReturnNoSpace;
	592	break;
	593	}
	594	HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx\n", filename, ctx.size,
	595	vars->block0, maxiobytes);
	596	if (ctx.size < 110241024) // check against image size estimate!
	597	{
	598	err = kIOReturnNoSpace;
	599	break;
	600	}
	601
	602	if (maxiobytes < vars->bufferSize)
	603	vars->bufferSize = maxiobytes;
	604
	605	vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
	606
	607	matching = IOService::serviceMatching("IOMedia");
	608	num = OSNumber::withNumber(major(hibernate_image_dev), 32);
	609	matching->setObject(kIOBSDMajorKey, num);
	610	num->release();
	611	num = OSNumber::withNumber(minor(hibernate_image_dev), 32);
	612	matching->setObject(kIOBSDMinorKey, num);
	613	num->release();
	614	iter = IOService::getMatchingServices(matching);
	615	matching->release();
	616	if (iter)
	617	{
	618	part = (IORegistryEntry *) iter->getNextObject();
	619	part->retain();
	620	iter->release();
	621	}
	622	if (!part)
	623	break;
	624
	625	int minor, major;
	626	IORegistryEntry * next;
	627	IORegistryEntry * child;
	628	OSData * data;
	629
	630	num = (OSNumber *) part->getProperty(kIOBSDMajorKey);
	631	if (!num)
	632	break;
	633	major = num->unsigned32BitValue();
	634	num = (OSNumber *) part->getProperty(kIOBSDMinorKey);
	635	if (!num)
	636	break;
	637	minor = num->unsigned32BitValue();
	638
	639	hibernate_image_dev = makedev(major, minor);
	640
	641	vars->pollers = OSArray::withCapacity(4);
	642	if (!vars->pollers)
	643	break;
	644
	645	vars->blockSize = 512;
	646	next = part;
	647	do
	648	{
	649	IOPolledInterface * poller;
	650	OSObject * obj;
	651
	652	obj = next->getProperty(kIOPolledInterfaceSupportKey);
	653	if (kOSBooleanFalse == obj)
	654	{
	655	vars->pollers->flushCollection();
	656	break;
	657	}
	658	else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
	659	vars->pollers->setObject(poller);
	660	if ((num = OSDynamicCast(OSNumber, next->getProperty(kIOMediaPreferredBlockSizeKey))))
	661	vars->blockSize = num->unsigned32BitValue();
	662	child = next;
	663	}
	664	while ((next = child->getParentEntry(gIOServicePlane))
	665	&& child->isParent(next, gIOServicePlane, true));
	666
	667	HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n",
	668	major, minor, vars->blockSize, vars->pollers->getCount());
	669	if (vars->pollers->getCount() < kIOHibernateMinPollersNeeded)
	670	continue;
	671
	672	err = IOHibernatePollerProbe(vars, (IOService *) part);
	673	if (kIOReturnSuccess != err)
	674	break;
	675
	676	err = IOHibernatePollerOpen(vars, kIOPolledPreflightState, ioBuffer);
	677	if (kIOReturnSuccess != err)
	678	break;
	679
	680	*fileVars = vars;
	681	*fileExtents = extentsData;
	682
	683	// make imagePath
	684
	685	if ((extentsData->getLength() >= sizeof(IOPolledFileExtent)))
	686	{
	687	char str2[24];
	688
	689	#if __i386__
	690	if (!gIOCreateEFIDevicePathSymbol)
	691	gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
	692
	693	snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
	694
	695	err = IOService::getPlatform()->callPlatformFunction(
	696	gIOCreateEFIDevicePathSymbol, false,
	697	(void ) part, (void ) str2, (void *) true,
	698	(void *) &data);
	699	#else
	700	char str1[256];
	701	int len = sizeof(str1);
	702
	703	if (!part->getPath(str1, &len, gIODTPlane))
	704	err = kIOReturnNotFound;
	705	else
	706	{
	707	snprintf(str2, sizeof(str2), ",%qx", vars->extentMap[0].start);
	708	// (strip the plane name)
	709	char * tail = strchr(str1, ':');
	710	if (!tail)
	711	tail = str1 - 1;
	712	data = OSData::withBytes(tail + 1, strlen(tail + 1));
	713	data->appendBytes(str2, strlen(str2));
	714	}
	715	#endif
	716	if (kIOReturnSuccess == err)
	717	*imagePath = data;
	718	else
	719	HIBLOG("error 0x%x getting path\n", err);
	720	}
	721	}
	722	while (false);
	723
	724	if (kIOReturnSuccess != err)
	725	{
	726	HIBLOG("error 0x%x opening hibernation file\n", err);
	727	if (vars->fileRef)
	728	kern_close_file_for_direct_io(vars->fileRef);
	729	}
	730
	731	if (part)
	732	part->release();
	733
	734	return (err);
	735	}
	736
	737	IOReturn
	738	IOPolledFileClose( IOPolledFileIOVars * vars )
	739	{
	740	if (vars->pollers)
	741	{
	742	IOHibernatePollerClose(vars, kIOPolledPostflightState);
	743	vars->pollers->release();
	744	}
	745
	746	gIOHibernateFileRef = vars->fileRef;
	747
	748	bzero(vars, sizeof(IOPolledFileIOVars));
	749
	750	return (kIOReturnSuccess);
	751	}
	752
	753	static IOReturn
	754	IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
	755	{
	756	IOPolledFileExtent * extentMap;
	757
	758	extentMap = vars->extentMap;
	759
	760	vars->position = position;
	761
	762	while (position >= extentMap->length)
	763	{
	764	position -= extentMap->length;
	765	extentMap++;
	766	}
	767
	768	vars->currentExtent = extentMap;
	769	vars->extentRemaining = extentMap->length - position;
	770	vars->extentPosition = vars->position - position;
	771
	772	if (vars->bufferSize <= vars->extentRemaining)
	773	vars->bufferLimit = vars->bufferSize;
	774	else
	775	vars->bufferLimit = vars->extentRemaining;
	776
	777	return (kIOReturnSuccess);
	778	}
	779
	780	static IOReturn
	781	IOPolledFileWrite(IOPolledFileIOVars * vars,
	782	const uint8_t * bytes, IOByteCount size,
	783	hibernate_cryptvars_t * cryptvars)
	784	{
	785	IOReturn err = kIOReturnSuccess;
	786	IOByteCount copy;
	787	bool flush = false;
	788
	789	do
	790	{
	791	if (!bytes && !size)
	792	{
	793	// seek to end of block & flush
	794	size = vars->position & (vars->blockSize - 1);
	795	if (size)
	796	size = vars->blockSize - size;
	797	flush = true;
	798	// use some garbage for the fill
	799	bytes = vars->buffer + vars->bufferOffset;
	800	}
	801
	802	copy = vars->bufferLimit - vars->bufferOffset;
	803	if (copy > size)
	804	copy = size;
	805	else
	806	flush = true;
	807
	808	if (bytes)
	809	{
	810	bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
	811	bytes += copy;
	812	}
	813	else
	814	bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
	815
	816	size -= copy;
	817	vars->bufferOffset += copy;
	818	vars->position += copy;
	819
	820	if (flush && vars->bufferOffset)
	821	{
	822	uint64_t offset = (vars->position - vars->bufferOffset
	823	- vars->extentPosition + vars->currentExtent->start);
	824	uint32_t length = (vars->bufferOffset);
	825
	826	#if CRYPTO
	827	if (cryptvars && vars->encryptStart && (vars->position > vars->encryptStart))
	828	{
	829	uint32_t encryptLen, encryptStart;
	830	encryptLen = vars->position - vars->encryptStart;
	831	if (encryptLen > length)
	832	encryptLen = length;
	833	encryptStart = length - encryptLen;
	834
	835	// encrypt the buffer
	836	aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
	837	&cryptvars->aes_iv[0],
	838	encryptLen / AES_BLOCK_SIZE,
	839	vars->buffer + vars->bufferHalf + encryptStart,
	840	&cryptvars->ctx.encrypt);
	841	// save initial vector for following encrypts
	842	bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
	843	&cryptvars->aes_iv[0],
	844	AES_BLOCK_SIZE);
	845	}
	846	#endif /* CRYPTO */
	847
	848	if (vars->io)
	849	{
	850	err = IOHibernatePollerIODone(vars, true);
	851	if (kIOReturnSuccess != err)
	852	break;
	853	}
	854
	855	if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
	856	//if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
	857
	858	err = IOHibernatePollerIO(vars, kIOPolledWrite, vars->bufferHalf, offset, length);
	859	if (kIOReturnSuccess != err)
	860	break;
	861	vars->io = true;
	862
	863	vars->extentRemaining -= vars->bufferOffset;
	864	if (!vars->extentRemaining)
	865	{
	866	vars->currentExtent++;
	867	vars->extentRemaining = vars->currentExtent->length;
	868	vars->extentPosition = vars->position;
	869	if (!vars->extentRemaining)
	870	{
	871	err = kIOReturnOverrun;
	872	break;
	873	}
	874	}
	875
	876	vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
	877	vars->bufferOffset = 0;
	878	if (vars->bufferSize <= vars->extentRemaining)
	879	vars->bufferLimit = vars->bufferSize;
	880	else
	881	vars->bufferLimit = vars->extentRemaining;
	882
	883	flush = false;
	884	}
	885	}
	886	while (size);
	887
	888	return (err);
	889	}
	890
	891	static IOReturn
	892	IOPolledFileRead(IOPolledFileIOVars * vars,
	893	uint8_t * bytes, IOByteCount size,
	894	hibernate_cryptvars_t * cryptvars)
	895	{
	896	IOReturn err = kIOReturnSuccess;
	897	IOByteCount copy;
	898
	899	// bytesWritten += size;
	900
	901	do
	902	{
	903	copy = vars->bufferLimit - vars->bufferOffset;
	904	if (copy > size)
	905	copy = size;
	906
	907	if (bytes)
	908	{
	909	bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
	910	bytes += copy;
	911	}
	912	size -= copy;
	913	vars->bufferOffset += copy;
	914	// vars->position += copy;
	915
	916	if (vars->bufferOffset == vars->bufferLimit)
	917	{
	918	if (vars->io)
	919	{
	920	err = IOHibernatePollerIODone(vars, false);
	921	if (kIOReturnSuccess != err)
	922	break;
	923	}
	924	else
	925	cryptvars = 0;
	926
	927	if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
	928
	929	vars->position += vars->lastRead;
	930	vars->extentRemaining -= vars->lastRead;
	931	vars->bufferLimit = vars->lastRead;
	932
	933	if (!vars->extentRemaining)
	934	{
	935	vars->currentExtent++;
	936	vars->extentRemaining = vars->currentExtent->length;
	937	vars->extentPosition = vars->position;
	938	if (!vars->extentRemaining)
	939	{
	940	err = kIOReturnOverrun;
	941	break;
	942	}
	943	}
	944
	945	uint64_t length;
	946	uint64_t lastReadLength = vars->lastRead;
	947	uint64_t offset = (vars->position
	948	- vars->extentPosition + vars->currentExtent->start);
	949	if (vars->extentRemaining <= vars->bufferSize)
	950	length = vars->extentRemaining;
	951	else
	952	length = vars->bufferSize;
	953	vars->lastRead = length;
	954
	955	//if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
	956
	957	err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length);
	958	if (kIOReturnSuccess != err)
	959	break;
	960	vars->io = true;
	961
	962	vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
	963	vars->bufferOffset = 0;
	964
	965	#if CRYPTO
	966	if (cryptvars)
	967	{
	968	uint8_t thisVector[AES_BLOCK_SIZE];
	969	// save initial vector for following decrypts
	970	bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
	971	bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE,
	972	&cryptvars->aes_iv[0], AES_BLOCK_SIZE);
	973	// decrypt the buffer
	974	aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
	975	&thisVector[0],
	976	lastReadLength / AES_BLOCK_SIZE,
	977	vars->buffer + vars->bufferHalf,
	978	&cryptvars->ctx.decrypt);
	979	}
	980	#endif CRYPTO
	981	}
	982	}
	983	while (size);
	984
	985	return (err);
	986	}
	987
	988	/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
	989
	990	IOReturn
	991	IOHibernateSystemSleep(void)
	992	{
	993	IOReturn err;
	994	OSData * data;
	995	OSObject * obj;
	996	OSString * str;
	997	OSNumber * num;
	998	OSDictionary *sleepOverrideOptions;
	999
	1000	IOHibernateVars * vars = &gIOHibernateVars;
	1001
	1002	if (vars->fileVars && vars->fileVars->fileRef)
	1003	// already on the way down
	1004	return (kIOReturnSuccess);
	1005
	1006	gIOHibernateState = kIOHibernateStateInactive;
	1007
	1008	/* The invocation of IOPMSleepSystemWithOptions() may override
	1009	* existing hibernation settings.
	1010	*/
	1011	sleepOverrideOptions = (OSDictionary *)OSDynamicCast( OSDictionary,
	1012	IOService::getPMRootDomain()->copyProperty(kRootDomainSleepOptionsKey));
	1013
	1014
	1015	/* Hibernate mode overriden by sleep otions ? */
	1016	obj = NULL;
	1017
	1018	if (sleepOverrideOptions) {
	1019	obj = sleepOverrideOptions->getObject(kIOHibernateModeKey);
	1020	if (obj) obj->retain();
	1021	}
	1022
	1023	if(!obj) {
	1024	obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateModeKey);
	1025	}
	1026
	1027	if (obj && (num = OSDynamicCast(OSNumber, obj)) )
	1028	{
	1029	gIOHibernateMode = num->unsigned32BitValue();
	1030	if (kIOHibernateModeSleep & gIOHibernateMode)
	1031	// default to discard clean for safe sleep
	1032	gIOHibernateMode ^= (kIOHibernateModeDiscardCleanInactive
	1033	\| kIOHibernateModeDiscardCleanActive);
	1034	}
	1035
	1036	if (obj) obj->release();
	1037
	1038	/* Hibernate free rotio overriden by sleep options ? */
	1039	obj = NULL;
	1040
	1041	if (sleepOverrideOptions) {
	1042	obj = sleepOverrideOptions->getObject(kIOHibernateFreeRatioKey);
	1043	if (obj) obj->retain();
	1044	}
	1045
	1046	if(!obj) {
	1047	obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeRatioKey);
	1048	}
	1049	if (obj && (num = OSDynamicCast(OSNumber, obj)))
	1050	{
	1051	gIOHibernateFreeRatio = num->unsigned32BitValue();
	1052	}
	1053	if (obj) obj->release();
	1054
	1055	if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFreeTimeKey)))
	1056	{
	1057	if ((num = OSDynamicCast(OSNumber, obj)))
	1058	gIOHibernateFreeTime = num->unsigned32BitValue();
	1059	obj->release();
	1060	}
	1061	if ((obj = IOService::getPMRootDomain()->copyProperty(kIOHibernateFileKey)))
	1062	{
	1063	if ((str = OSDynamicCast(OSString, obj)))
	1064	strlcpy(gIOHibernateFilename, str->getCStringNoCopy(),
	1065	sizeof(gIOHibernateFilename));
	1066	obj->release();
	1067	}
	1068
	1069	if (sleepOverrideOptions)
	1070	sleepOverrideOptions->release();
	1071
	1072	if (!gIOHibernateMode \|\| !gIOHibernateFilename[0])
	1073	return (kIOReturnUnsupported);
	1074
	1075	HIBLOG("hibernate image path: %s\n", gIOHibernateFilename);
	1076
	1077
	1078	do
	1079	{
	1080	vars->srcBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
	1081	4 * page_size, page_size);
	1082	vars->ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionOutIn,
	1083	2 * kDefaultIOSize, page_size);
	1084
	1085	if (!vars->srcBuffer \|\| !vars->ioBuffer)
	1086	{
	1087	err = kIOReturnNoMemory;
	1088	break;
	1089	}
	1090
	1091	err = IOPolledFileOpen(gIOHibernateFilename, vars->ioBuffer,
	1092	&vars->fileVars, &vars->fileExtents, &data);
	1093	if (KERN_SUCCESS != err)
	1094	{
	1095	HIBLOG("IOPolledFileOpen(%x)\n", err);
	1096	break;
	1097	}
	1098	if (vars->fileVars->fileRef)
	1099	{
	1100	// invalidate the image file
	1101	gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
	1102	int err = kern_write_file(vars->fileVars->fileRef, 0,
	1103	(caddr_t) gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
	1104	if (KERN_SUCCESS != err)
	1105	HIBLOG("kern_write_file(%d)\n", err);
	1106	}
	1107
	1108	bzero(gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
	1109
	1110	boolean_t encryptedswap;
	1111	err = hibernate_setup(gIOHibernateCurrentHeader,
	1112	gIOHibernateFreeRatio, gIOHibernateFreeTime,
	1113	&vars->page_list, &vars->page_list_wired, &encryptedswap);
	1114	if (KERN_SUCCESS != err)
	1115	{
	1116	HIBLOG("hibernate_setup(%d)\n", err);
	1117	break;
	1118	}
	1119
	1120	if (encryptedswap)
	1121	gIOHibernateMode ^= kIOHibernateModeEncrypt;
	1122
	1123	vars->videoAllocSize = kVideoMapSize;
	1124	if (KERN_SUCCESS != kmem_alloc_pageable(kernel_map, &vars->videoMapping, vars->videoAllocSize))
	1125	vars->videoMapping = 0;
	1126
	1127	// generate crypt keys
	1128	for (uint32_t i = 0; i < sizeof(vars->wiredCryptKey); i++)
	1129	vars->wiredCryptKey[i] = random();
	1130	for (uint32_t i = 0; i < sizeof(vars->cryptKey); i++)
	1131	vars->cryptKey[i] = random();
	1132
	1133	// set nvram
	1134
	1135	IORegistryEntry * regEntry;
	1136	if (!gIOOptionsEntry)
	1137	{
	1138	regEntry = IORegistryEntry::fromPath("/options", gIODTPlane);
	1139	gIOOptionsEntry = OSDynamicCast(IODTNVRAM, regEntry);
	1140	if (regEntry && !gIOOptionsEntry)
	1141	regEntry->release();
	1142	}
	1143	if (!gIOChosenEntry)
	1144	gIOChosenEntry = IORegistryEntry::fromPath("/chosen", gIODTPlane);
	1145
	1146	if (gIOOptionsEntry)
	1147	{
	1148	const OSSymbol * sym;
	1149
	1150	sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
	1151	if (sym)
	1152	{
	1153	gIOOptionsEntry->setProperty(sym, data);
	1154	sym->release();
	1155	}
	1156	data->release();
	1157
	1158	#ifdef __ppc__
	1159	size_t len;
	1160	char valueString[16];
	1161
	1162	vars->saveBootDevice = gIOOptionsEntry->copyProperty(kIOSelectedBootDeviceKey);
	1163	if (gIOChosenEntry)
	1164	{
	1165	OSData * bootDevice = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOBootPathKey));
	1166	if (bootDevice)
	1167	{
	1168	sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey);
	1169	OSString * str2 = OSString::withCStringNoCopy((const char *) bootDevice->getBytesNoCopy());
	1170	if (sym && str2)
	1171	gIOOptionsEntry->setProperty(sym, str2);
	1172	if (sym)
	1173	sym->release();
	1174	if (str2)
	1175	str2->release();
	1176	}
	1177	data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMemorySignatureKey));
	1178	if (data)
	1179	{
	1180	vars->haveFastBoot = true;
	1181
	1182	len = sprintf(valueString, "0x%lx", ((UInt32 )data->getBytesNoCopy()));
	1183	data = OSData::withBytes(valueString, len + 1);
	1184	sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey);
	1185	if (sym && data)
	1186	gIOOptionsEntry->setProperty(sym, data);
	1187	if (sym)
	1188	sym->release();
	1189	if (data)
	1190	data->release();
	1191	}
	1192	data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
	1193	if (data)
	1194	gIOHibernateCurrentHeader->machineSignature = ((UInt32 )data->getBytesNoCopy());
	1195	}
	1196	#endif /* __ppc__ */
	1197	#ifdef __i386__
	1198	struct AppleRTCHibernateVars
	1199	{
	1200	uint8_t signature[4];
	1201	uint32_t revision;
	1202	uint8_t booterSignature[20];
	1203	uint8_t wiredCryptKey[16];
	1204	};
	1205	AppleRTCHibernateVars rtcVars;
	1206
	1207	rtcVars.signature[0] = 'A';
	1208	rtcVars.signature[1] = 'A';
	1209	rtcVars.signature[2] = 'P';
	1210	rtcVars.signature[3] = 'L';
	1211	rtcVars.revision = 1;
	1212	bcopy(&vars->wiredCryptKey[0], &rtcVars.wiredCryptKey[0], sizeof(rtcVars.wiredCryptKey));
	1213	if (gIOHibernateBootSignature[0])
	1214	{
	1215	char c;
	1216	uint8_t value = 0;
	1217	for (uint32_t i = 0;
	1218	(c = gIOHibernateBootSignature[i]) && (i < (sizeof(rtcVars.booterSignature) << 1));
	1219	i++)
	1220	{
	1221	if (c >= 'a')
	1222	c -= 'a' - 10;
	1223	else if (c >= 'A')
	1224	c -= 'A' - 10;
	1225	else if (c >= '0')
	1226	c -= '0';
	1227	else
	1228	continue;
	1229	value = (value << 4) \| c;
	1230	if (i & 1)
	1231	rtcVars.booterSignature[i >> 1] = value;
	1232	}
	1233	}
	1234	data = OSData::withBytes(&rtcVars, sizeof(rtcVars));
	1235	if (data)
	1236	{
	1237	IOService::getPMRootDomain()->setProperty(kIOHibernateRTCVariablesKey, data);
	1238
	1239	if( gIOOptionsEntry )
	1240	{
	1241	if( gIOHibernateMode & kIOHibernateModeSwitch )
	1242	{
	1243	const OSSymbol *sym;
	1244	sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSwitchVarsKey);
	1245	if( sym )
	1246	{
	1247	gIOOptionsEntry->setProperty(sym, data); /* intentional insecure backup of rtc boot vars */
	1248	sym->release();
	1249	}
	1250	}
	1251	}
	1252
	1253	data->release();
	1254	}
	1255	if (gIOChosenEntry)
	1256	{
	1257	data = OSDynamicCast(OSData, gIOChosenEntry->getProperty(kIOHibernateMachineSignatureKey));
	1258	if (data)
	1259	gIOHibernateCurrentHeader->machineSignature = ((UInt32 )data->getBytesNoCopy());
	1260	}
	1261	#else /* !__i386__ */
	1262	if (kIOHibernateModeEncrypt & gIOHibernateMode)
	1263	{
	1264	data = OSData::withBytes(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
	1265	sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
	1266	if (sym && data)
	1267	gIOOptionsEntry->setProperty(sym, data);
	1268	if (sym)
	1269	sym->release();
	1270	if (data)
	1271	data->release();
	1272	if (false && gIOHibernateBootSignature[0])
	1273	{
	1274	data = OSData::withCapacity(16);
	1275	sym = OSSymbol::withCStringNoCopy(kIOHibernateBootSignatureKey);
	1276	if (sym && data)
	1277	{
	1278	char c;
	1279	uint8_t value = 0;
	1280	for (uint32_t i = 0; (c = gIOHibernateBootSignature[i]); i++)
	1281	{
	1282	if (c >= 'a')
	1283	c -= 'a' - 10;
	1284	else if (c >= 'A')
	1285	c -= 'A' - 10;
	1286	else if (c >= '0')
	1287	c -= '0';
	1288	else
	1289	continue;
	1290	value = (value << 4) \| c;
	1291	if (i & 1)
	1292	data->appendBytes(&value, sizeof(value));
	1293	}
	1294	gIOOptionsEntry->setProperty(sym, data);
	1295	}
	1296	if (sym)
	1297	sym->release();
	1298	if (data)
	1299	data->release();
	1300	}
	1301	}
	1302	if (!vars->haveFastBoot)
	1303	{
	1304	// set boot volume to zero
	1305	IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
	1306	if (platform && (kIOReturnSuccess == platform->readXPRAM(kXPRamAudioVolume,
	1307	&vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume))))
	1308	{
	1309	uint8_t newVolume;
	1310	newVolume = vars->saveBootAudioVolume & 0xf8;
	1311	platform->writeXPRAM(kXPRamAudioVolume,
	1312	&newVolume, sizeof(newVolume));
	1313	}
	1314	}
	1315	#endif /* !__i386__ */
	1316	}
	1317	// --
	1318
	1319	gIOHibernateCurrentHeader->signature = kIOHibernateHeaderSignature;
	1320	gIOHibernateState = kIOHibernateStateHibernating;
	1321	}
	1322	while (false);
	1323
	1324	return (err);
	1325	}
	1326
	1327	/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
	1328
	1329	DECLARE_IOHIBERNATEPROGRESSALPHA
	1330
	1331	static void
	1332	ProgressInit(hibernate_graphics_t * display, uint8_t * screen, uint8_t * saveunder, uint32_t savelen)
	1333	{
	1334	uint32_t rowBytes, pixelShift;
	1335	uint32_t x, y;
	1336	int32_t blob;
	1337	uint32_t alpha, in, color, result;
	1338	uint8_t * out;
	1339	uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
	1340
	1341	rowBytes = display->rowBytes;
	1342	pixelShift = display->depth >> 4;
	1343	if (pixelShift < 1) return;
	1344
	1345	screen += ((display->width
	1346	- kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
	1347	+ (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
	1348
	1349	for (y = 0; y < kIOHibernateProgressHeight; y++)
	1350	{
	1351	out = screen + y * rowBytes;
	1352	for (blob = 0; blob < kIOHibernateProgressCount; blob++)
	1353	{
	1354	color = blob ? kIOHibernateProgressDarkGray : kIOHibernateProgressMidGray;
	1355	for (x = 0; x < kIOHibernateProgressWidth; x++)
	1356	{
	1357	alpha = gIOHibernateProgressAlpha[y][x];
	1358	result = color;
	1359	if (alpha)
	1360	{
	1361	if (0xff != alpha)
	1362	{
	1363	if (1 == pixelShift)
	1364	{
	1365	in = ((uint16_t )out) & 0x1f; // 16
	1366	in = (in << 3) \| (in >> 2);
	1367	}
	1368	else
	1369	in = ((uint32_t )out) & 0xff; // 32
	1370	saveunder[blob * kIOHibernateProgressSaveUnderSize + saveindex[blob]++] = in;
	1371	result = ((255 - alpha) * in + alpha * result + 0xff) >> 8;
	1372	}
	1373	if (1 == pixelShift)
	1374	{
	1375	result >>= 3;
	1376	((uint16_t )out) = (result << 10) \| (result << 5) \| result; // 16
	1377	}
	1378	else
	1379	((uint32_t )out) = (result << 16) \| (result << 8) \| result; // 32
	1380	}
	1381	out += (1 << pixelShift);
	1382	}
	1383	out += (kIOHibernateProgressSpacing << pixelShift);
	1384	}
	1385	}
	1386	}
	1387
	1388
	1389	static void
	1390	ProgressUpdate(hibernate_graphics_t * display, uint8_t * screen, int32_t firstBlob, int32_t select)
	1391	{
	1392	uint32_t rowBytes, pixelShift;
	1393	uint32_t x, y;
	1394	int32_t blob, lastBlob;
	1395	uint32_t alpha, in, color, result;
	1396	uint8_t * out;
	1397	uint32_t saveindex[kIOHibernateProgressCount] = { 0 };
	1398
	1399	pixelShift = display->depth >> 4;
	1400	if (pixelShift < 1)
	1401	return;
	1402
	1403	rowBytes = display->rowBytes;
	1404
	1405	screen += ((display->width
	1406	- kIOHibernateProgressCount * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << (pixelShift - 1))
	1407	+ (display->height - kIOHibernateProgressOriginY - kIOHibernateProgressHeight) * rowBytes;
	1408
	1409	lastBlob = (select < kIOHibernateProgressCount) ? select : (kIOHibernateProgressCount - 1);
	1410
	1411	screen += (firstBlob * (kIOHibernateProgressWidth + kIOHibernateProgressSpacing)) << pixelShift;
	1412
	1413	for (y = 0; y < kIOHibernateProgressHeight; y++)
	1414	{
	1415	out = screen + y * rowBytes;
	1416	for (blob = firstBlob; blob <= lastBlob; blob++)
	1417	{
	1418	color = (blob < select) ? kIOHibernateProgressLightGray : kIOHibernateProgressMidGray;
	1419	for (x = 0; x < kIOHibernateProgressWidth; x++)
	1420	{
	1421	alpha = gIOHibernateProgressAlpha[y][x];
	1422	result = color;
	1423	if (alpha)
	1424	{
	1425	if (0xff != alpha)
	1426	{
	1427	in = display->progressSaveUnder[blob][saveindex[blob]++];
	1428	result = ((255 - alpha) * in + alpha * result + 0xff) / 255;
	1429	}
	1430	if (1 == pixelShift)
	1431	{
	1432	result >>= 3;
	1433	((uint16_t )out) = (result << 10) \| (result << 5) \| result; // 16
	1434	}
	1435	else
	1436	((uint32_t )out) = (result << 16) \| (result << 8) \| result; // 32
	1437	}
	1438	out += (1 << pixelShift);
	1439	}
	1440	out += (kIOHibernateProgressSpacing << pixelShift);
	1441	}
	1442	}
	1443	}
	1444
	1445	/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
	1446
	1447	IOReturn
	1448	IOHibernateSystemHasSlept(void)
	1449	{
	1450	IOHibernateVars * vars = &gIOHibernateVars;
	1451	OSObject * obj;
	1452	OSData * data;
	1453
	1454	obj = IOService::getPMRootDomain()->copyProperty(kIOHibernatePreviewBufferKey);
	1455	vars->previewBuffer = OSDynamicCast(IOMemoryDescriptor, obj);
	1456	if (obj && !vars->previewBuffer)
	1457	obj->release();
	1458
	1459	vars->consoleMapping = NULL;
	1460	if (vars->previewBuffer && (kIOReturnSuccess != vars->previewBuffer->prepare()))
	1461	{
	1462	vars->previewBuffer->release();
	1463	vars->previewBuffer = 0;
	1464	}
	1465
	1466	if (vars->previewBuffer && (data = OSDynamicCast(OSData,
	1467	IOService::getPMRootDomain()->getProperty(kIOHibernatePreviewActiveKey))))
	1468	{
	1469	UInt32 flags = ((UInt32 )data->getBytesNoCopy());
	1470	HIBPRINT("kIOHibernatePreviewActiveKey %08lx\n", flags);
	1471
	1472	IOService::getPMRootDomain()->removeProperty(kIOHibernatePreviewActiveKey);
	1473
	1474	if (kIOHibernatePreviewUpdates & flags)
	1475	{
	1476	PE_Video consoleInfo;
	1477	hibernate_graphics_t * graphicsInfo = gIOHibernateGraphicsInfo;
	1478
	1479	IOService::getPlatform()->getConsoleInfo(&consoleInfo);
	1480
	1481	graphicsInfo->width = consoleInfo.v_width;
	1482	graphicsInfo->height = consoleInfo.v_height;
	1483	graphicsInfo->rowBytes = consoleInfo.v_rowBytes;
	1484	graphicsInfo->depth = consoleInfo.v_depth;
	1485	vars->consoleMapping = (uint8_t *) consoleInfo.v_baseAddr;
	1486
	1487	HIBPRINT("video %p %d %d %d\n",
	1488	vars->consoleMapping, gIOHibernateGraphicsInfo->depth,
	1489	gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height);
	1490	if (vars->consoleMapping)
	1491	ProgressInit(graphicsInfo, vars->consoleMapping,
	1492	&graphicsInfo->progressSaveUnder[0][0], sizeof(graphicsInfo->progressSaveUnder));
	1493	}
	1494	}
	1495
	1496	if (gIOOptionsEntry)
	1497	gIOOptionsEntry->sync();
	1498
	1499	return (kIOReturnSuccess);
	1500	}
	1501
	1502	/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
	1503
	1504	IOReturn
	1505	IOHibernateSystemWake(void)
	1506	{
	1507	IOHibernateVars * vars = &gIOHibernateVars;
	1508
	1509	hibernate_teardown(vars->page_list, vars->page_list_wired);
	1510
	1511	if (vars->videoMapping)
	1512	{
	1513	if (vars->videoMapSize)
	1514	// remove mappings
	1515	IOUnmapPages(kernel_map, vars->videoMapping, vars->videoMapSize);
	1516	if (vars->videoAllocSize)
	1517	// dealloc range
	1518	kmem_free(kernel_map, trunc_page_32(vars->videoMapping), vars->videoAllocSize);
	1519	}
	1520
	1521	if (vars->previewBuffer)
	1522	{
	1523	vars->previewBuffer->release();
	1524	vars->previewBuffer = 0;
	1525	}
	1526
	1527	if (vars->fileVars)
	1528	{
	1529	IOPolledFileClose(vars->fileVars);
	1530	}
	1531
	1532	// invalidate nvram properties - (gIOOptionsEntry != 0) => nvram was touched
	1533
	1534	#ifdef __ppc__
	1535	OSData * data = OSData::withCapacity(4);
	1536	if (gIOOptionsEntry && data)
	1537	{
	1538	const OSSymbol * sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKey);
	1539	if (sym)
	1540	{
	1541	gIOOptionsEntry->setProperty(sym, data);
	1542	sym->release();
	1543	}
	1544	sym = OSSymbol::withCStringNoCopy(kIOSelectedBootDeviceKey);
	1545	if (sym)
	1546	{
	1547	if (vars->saveBootDevice)
	1548	{
	1549	gIOOptionsEntry->setProperty(sym, vars->saveBootDevice);
	1550	vars->saveBootDevice->release();
	1551	}
	1552	sym->release();
	1553	}
	1554	sym = OSSymbol::withCStringNoCopy(kIOHibernateBootImageKeyKey);
	1555	if (sym)
	1556	{
	1557	gIOOptionsEntry->setProperty(sym, data);
	1558	sym->release();
	1559	}
	1560	sym = OSSymbol::withCStringNoCopy(kIOHibernateMemorySignatureEnvKey);
	1561	if (sym)
	1562	{
	1563	gIOOptionsEntry->removeProperty(sym);
	1564	sym->release();
	1565	}
	1566	}
	1567	if (data)
	1568	data->release();
	1569
	1570	if (gIOOptionsEntry)
	1571	{
	1572	if (!vars->haveFastBoot)
	1573	{
	1574	// reset boot audio volume
	1575	IODTPlatformExpert * platform = OSDynamicCast(IODTPlatformExpert, IOService::getPlatform());
	1576	if (platform)
	1577	platform->writeXPRAM(kXPRamAudioVolume,
	1578	&vars->saveBootAudioVolume, sizeof(vars->saveBootAudioVolume));
	1579	}
	1580
	1581	// sync now to hardware if the booter has not
	1582	if (kIOHibernateStateInactive == gIOHibernateState)
	1583	gIOOptionsEntry->sync();
	1584	else
	1585	// just sync the variables in case a later panic syncs nvram (it won't sync variables)
	1586	gIOOptionsEntry->syncOFVariables();
	1587	}
	1588	#endif
	1589
	1590	#ifdef __i386__
	1591	IOService::getPMRootDomain()->removeProperty(kIOHibernateRTCVariablesKey);
	1592
	1593	/*
	1594	* Hibernate variable is written to NVRAM on platforms in which RtcRam
	1595	* is not backed by coin cell. Remove Hibernate data from NVRAM.
	1596	*/
	1597	if (gIOOptionsEntry) {
	1598	const OSSymbol * sym = OSSymbol::withCStringNoCopy(kIOHibernateRTCVariablesKey);
	1599
	1600	if (sym) {
	1601	if (gIOOptionsEntry->getProperty(sym)) {
	1602	gIOOptionsEntry->removeProperty(sym);
	1603	gIOOptionsEntry->sync();
	1604	}
	1605	sym->release();
	1606	}
	1607	}
	1608	#endif
	1609
	1610	if (vars->srcBuffer)
	1611	vars->srcBuffer->release();
	1612	if (vars->ioBuffer)
	1613	vars->ioBuffer->release();
	1614	if (vars->fileExtents)
	1615	vars->fileExtents->release();
	1616
	1617	bzero(vars, sizeof(*vars));
	1618
	1619	// gIOHibernateState = kIOHibernateStateInactive; // leave it for post wake code to see
	1620
	1621	return (kIOReturnSuccess);
	1622	}
	1623
	1624	IOReturn
	1625	IOHibernateSystemPostWake(void)
	1626	{
	1627	if (gIOHibernateFileRef)
	1628	{
	1629	// invalidate the image file
	1630	gIOHibernateCurrentHeader->signature = kIOHibernateHeaderInvalidSignature;
	1631	int err = kern_write_file(gIOHibernateFileRef, 0,
	1632	(caddr_t) gIOHibernateCurrentHeader, sizeof(IOHibernateImageHeader));
	1633	if (KERN_SUCCESS != err)
	1634	HIBLOG("kern_write_file(%d)\n", err);
	1635
	1636	kern_close_file_for_direct_io(gIOHibernateFileRef);
	1637	gIOHibernateFileRef = 0;
	1638	}
	1639	return (kIOReturnSuccess);
	1640	}
	1641
	1642	/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
	1643
	1644	SYSCTL_STRING(_kern, OID_AUTO, hibernatefile,
	1645	CTLFLAG_RW \| CTLFLAG_NOAUTO \| CTLFLAG_KERN,
	1646	gIOHibernateFilename, sizeof(gIOHibernateFilename), "");
	1647	SYSCTL_STRING(_kern, OID_AUTO, bootsignature,
	1648	CTLFLAG_RW \| CTLFLAG_NOAUTO \| CTLFLAG_KERN,
	1649	gIOHibernateBootSignature, sizeof(gIOHibernateBootSignature), "");
	1650	SYSCTL_UINT(_kern, OID_AUTO, hibernatemode,
	1651	CTLFLAG_RW \| CTLFLAG_NOAUTO \| CTLFLAG_KERN,
	1652	&gIOHibernateMode, 0, "");
	1653
	1654	void
	1655	IOHibernateSystemInit(IOPMrootDomain * rootDomain)
	1656	{
	1657	OSData * data = OSData::withBytesNoCopy(&gIOHibernateState, sizeof(gIOHibernateState));
	1658	if (data)
	1659	{
	1660	rootDomain->setProperty(kIOHibernateStateKey, data);
	1661	data->release();
	1662	}
	1663
	1664	if (PE_parse_boot_argn("hfile", gIOHibernateFilename, sizeof(gIOHibernateFilename)))
	1665	gIOHibernateMode = kIOHibernateModeOn;
	1666	else
	1667	gIOHibernateFilename[0] = 0;
	1668
	1669	sysctl_register_oid(&sysctl__kern_hibernatefile);
	1670	sysctl_register_oid(&sysctl__kern_bootsignature);
	1671	sysctl_register_oid(&sysctl__kern_hibernatemode);
	1672	}
	1673
	1674
	1675	/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
	1676
	1677	static void
	1678	hibernate_setup_for_wake(void)
	1679	{
	1680	#if __ppc__
	1681	// go slow (state needed for wake)
	1682	ml_set_processor_speed(1);
	1683	#endif
	1684	}
	1685
	1686	/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
	1687
	1688	#define C_ASSERT(e) typedef char __C_ASSERT__[(e) ? 1 : -1]
	1689
	1690	extern "C" uint32_t
	1691	hibernate_write_image(void)
	1692	{
	1693	IOHibernateImageHeader * header = gIOHibernateCurrentHeader;
	1694	IOHibernateVars * vars = &gIOHibernateVars;
	1695	IOPolledFileExtent * fileExtents;
	1696
	1697	C_ASSERT(sizeof(IOHibernateImageHeader) == 512);
	1698
	1699	uint32_t pageCount, pagesDone;
	1700	IOReturn err;
	1701	vm_offset_t ppnum;
	1702	IOItemCount page, count;
	1703	uint8_t * src;
	1704	uint8_t * data;
	1705	IOByteCount pageCompressedSize;
	1706	uint64_t compressedSize, uncompressedSize;
	1707	uint64_t image1Size = 0;
	1708	uint32_t bitmap_size;
	1709	bool iterDone, pollerOpen, needEncryptStart;
	1710	uint32_t restore1Sum, sum, sum1, sum2;
	1711	uint32_t tag;
	1712	uint32_t pageType;
	1713	uint32_t pageAndCount[2];
	1714
	1715	AbsoluteTime startTime, endTime;
	1716	AbsoluteTime allTime, compTime, decoTime;
	1717	uint64_t nsec;
	1718	uint32_t lastProgressStamp = 0;
	1719	uint32_t progressStamp;
	1720	uint32_t blob, lastBlob = (uint32_t) -1L;
	1721
	1722	hibernate_cryptvars_t _cryptvars;
	1723	hibernate_cryptvars_t * cryptvars = 0;
	1724
	1725	if (!vars->fileVars \|\| !vars->fileVars->pollers \|\| !vars->fileExtents)
	1726	return (false /* sleep */ );
	1727
	1728	restore1Sum = sum1 = sum2 = 0;
	1729
	1730	#if CRYPTO
	1731	// encryption data. "iv" is the "initial vector".
	1732	if (kIOHibernateModeEncrypt & gIOHibernateMode)
	1733	{
	1734	static const unsigned char first_iv[AES_BLOCK_SIZE]
	1735	= { 0xa3, 0x63, 0x65, 0xa9, 0x0b, 0x71, 0x7b, 0x1c,
	1736	0xdf, 0x9e, 0x5f, 0x32, 0xd7, 0x61, 0x63, 0xda };
	1737
	1738	cryptvars = &gIOHibernateCryptWakeContext;
	1739	bzero(cryptvars, sizeof(hibernate_cryptvars_t));
	1740	aes_encrypt_key(vars->cryptKey,
	1741	kIOHibernateAESKeySize,
	1742	&cryptvars->ctx.encrypt);
	1743	aes_decrypt_key(vars->cryptKey,
	1744	kIOHibernateAESKeySize,
	1745	&cryptvars->ctx.decrypt);
	1746
	1747	cryptvars = &_cryptvars;
	1748	bzero(cryptvars, sizeof(hibernate_cryptvars_t));
	1749	aes_encrypt_key(vars->wiredCryptKey,
	1750	kIOHibernateAESKeySize,
	1751	&cryptvars->ctx.encrypt);
	1752
	1753	bcopy(&first_iv[0], &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
	1754	bzero(&vars->wiredCryptKey[0], sizeof(vars->wiredCryptKey));
	1755	bzero(&vars->cryptKey[0], sizeof(vars->cryptKey));
	1756	bzero(gIOHibernateCryptWakeVars, sizeof(hibernate_cryptwakevars_t));
	1757	}
	1758	#endif /* CRYPTO */
	1759
	1760	hibernate_setup_for_wake();
	1761
	1762	hibernate_page_list_setall(vars->page_list,
	1763	vars->page_list_wired,
	1764	&pageCount);
	1765
	1766	HIBLOG("hibernate_page_list_setall found pageCount %d\n", pageCount);
	1767
	1768	fileExtents = (IOPolledFileExtent *) vars->fileExtents->getBytesNoCopy();
	1769
	1770	#if 0
	1771	count = vars->fileExtents->getLength() / sizeof(IOPolledFileExtent);
	1772	for (page = 0; page < count; page++)
	1773	{
	1774	HIBLOG("fileExtents[%d] %qx, %qx (%qx)\n", page,
	1775	fileExtents[page].start, fileExtents[page].length,
	1776	fileExtents[page].start + fileExtents[page].length);
	1777	}
	1778	#endif
	1779
	1780	needEncryptStart = (0 != (kIOHibernateModeEncrypt & gIOHibernateMode));
	1781
	1782	AbsoluteTime_to_scalar(&compTime) = 0;
	1783	AbsoluteTime_to_scalar(&decoTime) = 0;
	1784
	1785	clock_get_uptime(&allTime);
	1786
	1787	do
	1788	{
	1789	compressedSize = 0;
	1790	uncompressedSize = 0;
	1791	iterDone = false;
	1792	pageType = 0; // wired pages first
	1793
	1794	IOPolledFileSeek(vars->fileVars, sizeof(IOHibernateImageHeader));
	1795
	1796	HIBLOG("IOHibernatePollerOpen, ml_get_interrupts_enabled %d\n",
	1797	ml_get_interrupts_enabled());
	1798	err = IOHibernatePollerOpen(vars->fileVars, kIOPolledBeforeSleepState, vars->ioBuffer);
	1799	HIBLOG("IOHibernatePollerOpen(%x)\n", err);
	1800	pollerOpen = (kIOReturnSuccess == err);
	1801	if (!pollerOpen)
	1802	break;
	1803
	1804	// copy file block extent list if larger than header
	1805
	1806	count = vars->fileExtents->getLength();
	1807	if (count > sizeof(header->fileExtentMap))
	1808	{
	1809	count -= sizeof(header->fileExtentMap);
	1810	err = IOPolledFileWrite(vars->fileVars,
	1811	((uint8_t *) &fileExtents[0]) + sizeof(header->fileExtentMap), count, cryptvars);
	1812	if (kIOReturnSuccess != err)
	1813	break;
	1814	}
	1815
	1816	// copy out restore1 code
	1817
	1818	page = atop_32(sectHIBB);
	1819	count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page;
	1820	header->restore1CodePage = page;
	1821	header->restore1PageCount = count;
	1822	header->restore1CodeOffset = ((uint32_t) &hibernate_machine_entrypoint) - sectHIBB;
	1823	header->restore1StackOffset = ((uint32_t) &gIOHibernateRestoreStackEnd[0]) - 64 - sectHIBB;
	1824
	1825	// sum __HIB sect, with zeros for the stack
	1826	src = (uint8_t *) trunc_page(sectHIBB);
	1827	for (page = 0; page < count; page++)
	1828	{
	1829	if ((src < &gIOHibernateRestoreStack[0]) \|\| (src >= &gIOHibernateRestoreStackEnd[0]))
	1830	restore1Sum += hibernate_sum(src, page_size);
	1831	else
	1832	restore1Sum += 0x10000001;
	1833	src += page_size;
	1834	}
	1835	sum1 = restore1Sum;
	1836
	1837	// write the __HIB sect, with zeros for the stack
	1838
	1839	src = (uint8_t *) trunc_page(sectHIBB);
	1840	count = ((uint32_t) &gIOHibernateRestoreStack[0]) - trunc_page(sectHIBB);
	1841	if (count)
	1842	{
	1843	err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
	1844	if (kIOReturnSuccess != err)
	1845	break;
	1846	}
	1847	err = IOPolledFileWrite(vars->fileVars,
	1848	(uint8_t *) 0,
	1849	&gIOHibernateRestoreStackEnd[0] - &gIOHibernateRestoreStack[0],
	1850	cryptvars);
	1851	if (kIOReturnSuccess != err)
	1852	break;
	1853	src = &gIOHibernateRestoreStackEnd[0];
	1854	count = round_page(sectHIBB + sectSizeHIB) - ((uint32_t) src);
	1855	if (count)
	1856	{
	1857	err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
	1858	if (kIOReturnSuccess != err)
	1859	break;
	1860	}
	1861
	1862	// write the preview buffer
	1863
	1864	addr64_t phys64;
	1865	IOByteCount segLen;
	1866
	1867	if (vars->previewBuffer)
	1868	{
	1869	ppnum = 0;
	1870	count = 0;
	1871	do
	1872	{
	1873	phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen);
	1874	pageAndCount[0] = atop_64(phys64);
	1875	pageAndCount[1] = atop_32(segLen);
	1876	err = IOPolledFileWrite(vars->fileVars,
	1877	(const uint8_t *) &pageAndCount, sizeof(pageAndCount),
	1878	cryptvars);
	1879	if (kIOReturnSuccess != err)
	1880	break;
	1881	count += segLen;
	1882	ppnum += sizeof(pageAndCount);
	1883	}
	1884	while (phys64);
	1885	if (kIOReturnSuccess != err)
	1886	break;
	1887
	1888	src = (uint8_t *) vars->previewBuffer->getSourceSegment(0, NULL);
	1889	count = vars->previewBuffer->getLength();
	1890
	1891	header->previewPageListSize = ppnum;
	1892	header->previewSize = count + ppnum;
	1893
	1894	for (page = 0; page < count; page += page_size)
	1895	sum1 += hibernate_sum(src + page, page_size);
	1896
	1897	err = IOPolledFileWrite(vars->fileVars, src, count, cryptvars);
	1898	if (kIOReturnSuccess != err)
	1899	break;
	1900	}
	1901
	1902	// mark areas for no save
	1903
	1904	for (count = 0;
	1905	(phys64 = vars->ioBuffer->getPhysicalSegment64(count, &segLen));
	1906	count += segLen)
	1907	{
	1908	hibernate_set_page_state(vars->page_list, vars->page_list_wired,
	1909	atop_64(phys64), atop_32(segLen),
	1910	kIOHibernatePageStateFree);
	1911	pageCount -= atop_32(segLen);
	1912	}
	1913
	1914	for (count = 0;
	1915	(phys64 = vars->srcBuffer->getPhysicalSegment64(count, &segLen));
	1916	count += segLen)
	1917	{
	1918	hibernate_set_page_state(vars->page_list, vars->page_list_wired,
	1919	atop_64(phys64), atop_32(segLen),
	1920	kIOHibernatePageStateFree);
	1921	pageCount -= atop_32(segLen);
	1922	}
	1923
	1924	// copy out bitmap of pages available for trashing during restore
	1925
	1926	bitmap_size = vars->page_list_wired->list_size;
	1927	src = (uint8_t *) vars->page_list_wired;
	1928	err = IOPolledFileWrite(vars->fileVars, src, bitmap_size, cryptvars);
	1929	if (kIOReturnSuccess != err)
	1930	break;
	1931
	1932	// mark more areas for no save, but these are not available
	1933	// for trashing during restore
	1934
	1935	hibernate_page_list_set_volatile(vars->page_list, vars->page_list_wired, &pageCount);
	1936
	1937	page = atop_32(sectHIBB);
	1938	count = atop_32(round_page(sectHIBB + sectSizeHIB)) - page;
	1939	hibernate_set_page_state(vars->page_list, vars->page_list_wired,
	1940	page, count,
	1941	kIOHibernatePageStateFree);
	1942	pageCount -= count;
	1943
	1944	if (vars->previewBuffer) for (count = 0;
	1945	(phys64 = vars->previewBuffer->getPhysicalSegment64(count, &segLen));
	1946	count += segLen)
	1947	{
	1948	hibernate_set_page_state(vars->page_list, vars->page_list_wired,
	1949	atop_64(phys64), atop_32(segLen),
	1950	kIOHibernatePageStateFree);
	1951	pageCount -= atop_32(segLen);
	1952	}
	1953
	1954	src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
	1955
	1956	ppnum = 0;
	1957	pagesDone = 0;
	1958	lastBlob = 0;
	1959
	1960	HIBLOG("writing %d pages\n", pageCount);
	1961
	1962	do
	1963	{
	1964	count = hibernate_page_list_iterate(pageType ? vars->page_list : vars->page_list_wired,
	1965	&ppnum);
	1966	// kprintf("[%d](%x : %x)\n", pageType, ppnum, count);
	1967
	1968	iterDone = !count;
	1969
	1970	pageAndCount[0] = ppnum;
	1971	pageAndCount[1] = count;
	1972	err = IOPolledFileWrite(vars->fileVars,
	1973	(const uint8_t *) &pageAndCount, sizeof(pageAndCount),
	1974	cryptvars);
	1975	if (kIOReturnSuccess != err)
	1976	break;
	1977
	1978	for (page = 0; page < count; page++)
	1979	{
	1980	err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(ppnum), page_size);
	1981	if (err)
	1982	{
	1983	HIBLOG("IOMemoryDescriptorWriteFromPhysical %d [%d] %x\n", __LINE__, ppnum, err);
	1984	break;
	1985	}
	1986
	1987	sum = hibernate_sum(src, page_size);
	1988
	1989	clock_get_uptime(&startTime);
	1990
	1991	pageCompressedSize = WKdm_compress ((WK_word) src, (WK_word) (src + page_size), PAGE_SIZE_IN_WORDS);
	1992
	1993	clock_get_uptime(&endTime);
	1994	ADD_ABSOLUTETIME(&compTime, &endTime);
	1995	SUB_ABSOLUTETIME(&compTime, &startTime);
	1996
	1997	if (kIOHibernateModeEncrypt & gIOHibernateMode)
	1998	pageCompressedSize = (pageCompressedSize + AES_BLOCK_SIZE - 1) & ~(AES_BLOCK_SIZE - 1);
	1999
	2000	if (pageCompressedSize > page_size)
	2001	{
	2002	// HIBLOG("------------lose: %d\n", pageCompressedSize);
	2003	pageCompressedSize = page_size;
	2004	}
	2005
	2006	if (pageCompressedSize != page_size)
	2007	data = (src + page_size);
	2008	else
	2009	data = src;
	2010
	2011	tag = pageCompressedSize \| kIOHibernateTagSignature;
	2012
	2013	if (pageType)
	2014	sum2 += sum;
	2015	else
	2016	sum1 += sum;
	2017
	2018	if (needEncryptStart && (ppnum >= atop_32(sectDATAB)))
	2019	{
	2020	// start encrypting partway into the data about to be written
	2021	vars->fileVars->encryptStart = (vars->fileVars->position + AES_BLOCK_SIZE - 1)
	2022	& ~(AES_BLOCK_SIZE - 1);
	2023	needEncryptStart = false;
	2024	}
	2025
	2026	err = IOPolledFileWrite(vars->fileVars, (const uint8_t *) &tag, sizeof(tag), cryptvars);
	2027	if (kIOReturnSuccess != err)
	2028	break;
	2029
	2030	err = IOPolledFileWrite(vars->fileVars, data, (pageCompressedSize + 3) & ~3, cryptvars);
	2031	if (kIOReturnSuccess != err)
	2032	break;
	2033
	2034	compressedSize += pageCompressedSize;
	2035	if (pageCompressedSize)
	2036	uncompressedSize += page_size;
	2037	ppnum++;
	2038	pagesDone++;
	2039
	2040	if (vars->consoleMapping && (0 == (1023 & pagesDone)))
	2041	{
	2042	blob = ((pagesDone * kIOHibernateProgressCount) / pageCount);
	2043	if (blob != lastBlob)
	2044	{
	2045	ProgressUpdate(gIOHibernateGraphicsInfo, vars->consoleMapping, lastBlob, blob);
	2046	lastBlob = blob;
	2047	}
	2048	}
	2049	if (0 == (8191 & pagesDone))
	2050	{
	2051	clock_get_uptime(&endTime);
	2052	SUB_ABSOLUTETIME(&endTime, &allTime);
	2053	absolutetime_to_nanoseconds(endTime, &nsec);
	2054	progressStamp = nsec / 750000000ULL;
	2055	if (progressStamp != lastProgressStamp)
	2056	{
	2057	lastProgressStamp = progressStamp;
	2058	HIBPRINT("pages %d (%d%%)\n", pagesDone, (100 * pagesDone) / pageCount);
	2059	}
	2060	}
	2061	}
	2062	if (kIOReturnSuccess != err)
	2063	break;
	2064	if (iterDone && !pageType)
	2065	{
	2066	err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
	2067	if (kIOReturnSuccess != err)
	2068	break;
	2069
	2070	iterDone = false;
	2071	pageType = 1;
	2072	ppnum = 0;
	2073	image1Size = vars->fileVars->position;
	2074	if (cryptvars)
	2075	{
	2076	bcopy(&cryptvars->aes_iv[0],
	2077	&gIOHibernateCryptWakeContext.aes_iv[0],
	2078	sizeof(cryptvars->aes_iv));
	2079	cryptvars = &gIOHibernateCryptWakeContext;
	2080	}
	2081	HIBLOG("image1Size %qd\n", image1Size);
	2082	}
	2083	}
	2084	while (!iterDone);
	2085	if (kIOReturnSuccess != err)
	2086	break;
	2087	err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
	2088	if (kIOReturnSuccess != err)
	2089	break;
	2090
	2091	// Header:
	2092
	2093	header->imageSize = vars->fileVars->position;
	2094	header->image1Size = image1Size;
	2095	header->bitmapSize = bitmap_size;
	2096	header->pageCount = pageCount;
	2097	header->encryptStart = vars->fileVars->encryptStart;
	2098
	2099	header->restore1Sum = restore1Sum;
	2100	header->image1Sum = sum1;
	2101	header->image2Sum = sum2;
	2102
	2103	count = vars->fileExtents->getLength();
	2104	if (count > sizeof(header->fileExtentMap))
	2105	{
	2106	header->fileExtentMapSize = count;
	2107	count = sizeof(header->fileExtentMap);
	2108	}
	2109	else
	2110	header->fileExtentMapSize = sizeof(header->fileExtentMap);
	2111	bcopy(&fileExtents[0], &header->fileExtentMap[0], count);
	2112
	2113	IOPolledFileSeek(vars->fileVars, 0);
	2114	err = IOPolledFileWrite(vars->fileVars,
	2115	(uint8_t *) header, sizeof(IOHibernateImageHeader),
	2116	cryptvars);
	2117	if (kIOReturnSuccess != err)
	2118	break;
	2119	err = IOPolledFileWrite(vars->fileVars, 0, 0, cryptvars);
	2120	if (kIOReturnSuccess != err)
	2121	break;
	2122	err = IOHibernatePollerIODone(vars->fileVars, true);
	2123	if (kIOReturnSuccess != err)
	2124	break;
	2125	}
	2126	while (false);
	2127
	2128	clock_get_uptime(&endTime);
	2129	SUB_ABSOLUTETIME(&endTime, &allTime);
	2130	absolutetime_to_nanoseconds(endTime, &nsec);
	2131	HIBLOG("all time: %qd ms, ",
	2132	nsec / 1000000ULL);
	2133
	2134	absolutetime_to_nanoseconds(compTime, &nsec);
	2135	HIBLOG("comp time: %qd ms, ",
	2136	nsec / 1000000ULL);
	2137
	2138	absolutetime_to_nanoseconds(decoTime, &nsec);
	2139	HIBLOG("deco time: %qd ms, ",
	2140	nsec / 1000000ULL);
	2141
	2142	HIBLOG("\nimage %qd, uncompressed %qd (%d), compressed %qd (%d%%), sum1 %x, sum2 %x\n",
	2143	header->imageSize,
	2144	uncompressedSize, atop_32(uncompressedSize), compressedSize,
	2145	uncompressedSize ? ((int) ((compressedSize * 100ULL) / uncompressedSize)) : 0,
	2146	sum1, sum2);
	2147
	2148	if (vars->fileVars->io)
	2149	(void) IOHibernatePollerIODone(vars->fileVars, false);
	2150
	2151	if (pollerOpen)
	2152	IOHibernatePollerClose(vars->fileVars, kIOPolledBeforeSleepState);
	2153
	2154	if (vars->consoleMapping)
	2155	ProgressUpdate(gIOHibernateGraphicsInfo,
	2156	vars->consoleMapping, 0, kIOHibernateProgressCount);
	2157
	2158	HIBLOG("hibernate_write_image done(%x)\n", err);
	2159
	2160	// should we come back via regular wake, set the state in memory.
	2161	gIOHibernateState = kIOHibernateStateInactive;
	2162
	2163	if (kIOReturnSuccess == err)
	2164	{
	2165	if (kIOHibernateModeSleep & gIOHibernateMode)
	2166	{
	2167	return (kIOHibernatePostWriteSleep);
	2168	}
	2169	else if(kIOHibernateModeRestart & gIOHibernateMode)
	2170	{
	2171	return (kIOHibernatePostWriteRestart);
	2172	}
	2173	else
	2174	{
	2175	/* by default, power down */
	2176	return (kIOHibernatePostWriteHalt);
	2177	}
	2178	}
	2179	else if (kIOReturnAborted == err)
	2180	{
	2181	return (kIOHibernatePostWriteWake);
	2182	}
	2183	else
	2184	{
	2185	/* on error, sleep */
	2186	return (kIOHibernatePostWriteSleep);
	2187	}
	2188	}
	2189
	2190	/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
	2191
	2192	extern "C" void
	2193	hibernate_machine_init(void)
	2194	{
	2195	IOReturn err;
	2196	uint32_t sum;
	2197	uint32_t pagesDone;
	2198	AbsoluteTime allTime, endTime;
	2199	uint64_t nsec;
	2200	uint32_t lastProgressStamp = 0;
	2201	uint32_t progressStamp;
	2202	uint64_t progressZeroPosition = 0;
	2203	uint32_t blob, lastBlob = (uint32_t) -1L;
	2204	hibernate_cryptvars_t * cryptvars = 0;
	2205
	2206	IOHibernateVars * vars = &gIOHibernateVars;
	2207
	2208	if (!vars->fileVars \|\| !vars->fileVars->pollers \|\| !vars->fileExtents)
	2209	return;
	2210
	2211	sum = gIOHibernateCurrentHeader->actualImage1Sum;
	2212	pagesDone = gIOHibernateCurrentHeader->actualUncompressedPages;
	2213
	2214	HIBLOG("hibernate_machine_init: state %d, image pages %d, sum was %x, image1Size %qx, conflictCount %d, nextFree %x\n",
	2215	gIOHibernateState, pagesDone, sum, gIOHibernateCurrentHeader->image1Size,
	2216	gIOHibernateCurrentHeader->conflictCount, gIOHibernateCurrentHeader->nextFree);
	2217
	2218	if (kIOHibernateStateWakingFromHibernate != gIOHibernateState)
	2219	{
	2220	HIBLOG("regular wake\n");
	2221	return;
	2222	}
	2223
	2224	HIBPRINT("diag %x %x %x %x\n",
	2225	gIOHibernateCurrentHeader->diag[0], gIOHibernateCurrentHeader->diag[1],
	2226	gIOHibernateCurrentHeader->diag[2], gIOHibernateCurrentHeader->diag[3]);
	2227
	2228	HIBPRINT("video %x %d %d %d\n",
	2229	gIOHibernateGraphicsInfo->physicalAddress, gIOHibernateGraphicsInfo->depth,
	2230	gIOHibernateGraphicsInfo->width, gIOHibernateGraphicsInfo->height);
	2231
	2232	if ((kIOHibernateModeDiscardCleanActive \| kIOHibernateModeDiscardCleanInactive) & gIOHibernateMode)
	2233	hibernate_page_list_discard(vars->page_list);
	2234
	2235	boot_args args = (boot_args ) PE_state.bootArgs;
	2236
	2237	if (vars->videoMapping
	2238	&& gIOHibernateGraphicsInfo->physicalAddress
	2239	&& (args->Video.v_baseAddr == gIOHibernateGraphicsInfo->physicalAddress))
	2240	{
	2241	vars->videoMapSize = round_page(gIOHibernateGraphicsInfo->height
	2242	* gIOHibernateGraphicsInfo->rowBytes);
	2243	IOMapPages(kernel_map,
	2244	vars->videoMapping, gIOHibernateGraphicsInfo->physicalAddress,
	2245	vars->videoMapSize, kIOMapInhibitCache );
	2246	}
	2247
	2248	uint8_t * src = (uint8_t *) vars->srcBuffer->getBytesNoCopy();
	2249
	2250	if (gIOHibernateWakeMapSize)
	2251	{
	2252	err = IOMemoryDescriptorWriteFromPhysical(vars->srcBuffer, 0, ptoa_64(gIOHibernateWakeMap),
	2253	gIOHibernateWakeMapSize);
	2254	if (kIOReturnSuccess == err)
	2255	hibernate_newruntime_map(src, gIOHibernateWakeMapSize,
	2256	gIOHibernateCurrentHeader->systemTableOffset);
	2257	gIOHibernateWakeMap = 0;
	2258	gIOHibernateWakeMapSize = 0;
	2259	}
	2260
	2261	uint32_t decoOffset;
	2262
	2263	clock_get_uptime(&allTime);
	2264
	2265	HIBLOG("IOHibernatePollerOpen(), ml_get_interrupts_enabled %d\n", ml_get_interrupts_enabled());
	2266	err = IOHibernatePollerOpen(vars->fileVars, kIOPolledAfterSleepState, 0);
	2267	HIBLOG("IOHibernatePollerOpen(%x)\n", err);
	2268
	2269	if (gIOHibernateCurrentHeader->previewSize)
	2270	progressZeroPosition = gIOHibernateCurrentHeader->previewSize
	2271	+ gIOHibernateCurrentHeader->fileExtentMapSize
	2272	- sizeof(gIOHibernateCurrentHeader->fileExtentMap)
	2273	+ ptoa_64(gIOHibernateCurrentHeader->restore1PageCount);
	2274
	2275	IOPolledFileSeek(vars->fileVars, gIOHibernateCurrentHeader->image1Size);
	2276
	2277	if (vars->videoMapSize)
	2278	{
	2279	lastBlob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
	2280	/ (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
	2281	ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, 0, lastBlob);
	2282	}
	2283
	2284	cryptvars = (kIOHibernateModeEncrypt & gIOHibernateMode) ? &gIOHibernateCryptWakeContext : 0;
	2285	if (kIOHibernateModeEncrypt & gIOHibernateMode)
	2286	{
	2287	cryptvars = &gIOHibernateCryptWakeContext;
	2288	bcopy(&gIOHibernateCryptWakeVars->aes_iv[0],
	2289	&cryptvars->aes_iv[0],
	2290	sizeof(cryptvars->aes_iv));
	2291	}
	2292
	2293	// kick off the read ahead
	2294	vars->fileVars->io = false;
	2295	vars->fileVars->bufferHalf = 0;
	2296	vars->fileVars->bufferLimit = 0;
	2297	vars->fileVars->lastRead = 0;
	2298	vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
	2299
	2300	IOPolledFileRead(vars->fileVars, 0, 0, cryptvars);
	2301	vars->fileVars->bufferOffset = vars->fileVars->bufferLimit;
	2302	// --
	2303
	2304	HIBLOG("hibernate_machine_init reading\n");
	2305
	2306	uint32_t * header = (uint32_t *) src;
	2307	sum = 0;
	2308
	2309	do
	2310	{
	2311	unsigned int count;
	2312	unsigned int page;
	2313	uint32_t tag;
	2314	vm_offset_t ppnum, compressedSize;
	2315
	2316	err = IOPolledFileRead(vars->fileVars, src, 8, cryptvars);
	2317	if (kIOReturnSuccess != err)
	2318	break;
	2319
	2320	ppnum = header[0];
	2321	count = header[1];
	2322
	2323	// HIBPRINT("(%x, %x)\n", ppnum, count);
	2324
	2325	if (!count)
	2326	break;
	2327
	2328	for (page = 0; page < count; page++)
	2329	{
	2330	err = IOPolledFileRead(vars->fileVars, (uint8_t *) &tag, 4, cryptvars);
	2331	if (kIOReturnSuccess != err)
	2332	break;
	2333
	2334	compressedSize = kIOHibernateTagLength & tag;
	2335	if (kIOHibernateTagSignature != (tag & ~kIOHibernateTagLength))
	2336	{
	2337	err = kIOReturnIPCError;
	2338	break;
	2339	}
	2340
	2341	if (!compressedSize)
	2342	{
	2343	ppnum++;
	2344	pagesDone++;
	2345	continue;
	2346	}
	2347
	2348	err = IOPolledFileRead(vars->fileVars, src, (compressedSize + 3) & ~3, cryptvars);
	2349	if (kIOReturnSuccess != err)
	2350	break;
	2351
	2352	if (compressedSize < page_size)
	2353	{
	2354	decoOffset = page_size;
	2355	WKdm_decompress((WK_word) src, (WK_word) (src + decoOffset), PAGE_SIZE_IN_WORDS);
	2356	}
	2357	else
	2358	decoOffset = 0;
	2359
	2360	sum += hibernate_sum((src + decoOffset), page_size);
	2361
	2362	err = IOMemoryDescriptorReadToPhysical(vars->srcBuffer, decoOffset, ptoa_64(ppnum), page_size);
	2363	if (err)
	2364	{
	2365	HIBLOG("IOMemoryDescriptorReadToPhysical [%d] %x\n", ppnum, err);
	2366	break;
	2367	}
	2368
	2369	ppnum++;
	2370	pagesDone++;
	2371
	2372	if (vars->videoMapSize && (0 == (1023 & pagesDone)))
	2373	{
	2374	blob = ((vars->fileVars->position - progressZeroPosition) * kIOHibernateProgressCount)
	2375	/ (gIOHibernateCurrentHeader->imageSize - progressZeroPosition);
	2376	if (blob != lastBlob)
	2377	{
	2378	ProgressUpdate(gIOHibernateGraphicsInfo, (uint8_t *) vars->videoMapping, lastBlob, blob);
	2379	lastBlob = blob;
	2380	}
	2381	}
	2382
	2383	if (0 == (8191 & pagesDone))
	2384	{
	2385	clock_get_uptime(&endTime);
	2386	SUB_ABSOLUTETIME(&endTime, &allTime);
	2387	absolutetime_to_nanoseconds(endTime, &nsec);
	2388	progressStamp = nsec / 750000000ULL;
	2389	if (progressStamp != lastProgressStamp)
	2390	{
	2391	lastProgressStamp = progressStamp;
	2392	HIBPRINT("pages %d (%d%%)\n", pagesDone,
	2393	(100 * pagesDone) / gIOHibernateCurrentHeader->pageCount);
	2394	}
	2395	}
	2396	}
	2397	}
	2398	while (true);
	2399
	2400	if (kIOReturnSuccess != err)
	2401	panic("Hibernate restore error %x", err);
	2402
	2403	gIOHibernateCurrentHeader->actualImage2Sum = sum;
	2404
	2405	if (vars->fileVars->io)
	2406	(void) IOHibernatePollerIODone(vars->fileVars, false);
	2407
	2408	err = IOHibernatePollerClose(vars->fileVars, kIOPolledAfterSleepState);
	2409
	2410	if (vars->videoMapSize)
	2411	ProgressUpdate(gIOHibernateGraphicsInfo,
	2412	(uint8_t *) vars->videoMapping, 0, kIOHibernateProgressCount);
	2413
	2414	clock_get_uptime(&endTime);
	2415	SUB_ABSOLUTETIME(&endTime, &allTime);
	2416	absolutetime_to_nanoseconds(endTime, &nsec);
	2417
	2418	HIBLOG("hibernate_machine_init pagesDone %d sum2 %x, time: %qd ms\n",
	2419	pagesDone, sum, nsec / 1000000ULL);
	2420	}
	2421
	2422	/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */