- page += count;
- if (page <= bitmap->last_page)
- break;
- }
-
- *pPage = page;
- if (bitmap)
- count = hibernate_page_bitmap_count(bitmap, FALSE, page);
- else
- count = 0;
-
- return (count);
-}
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-static IOReturn
-IOHibernatePollerProbe(IOPolledFileIOVars * vars, IOService * target)
-{
- IOReturn err = kIOReturnError;
- int32_t idx;
- IOPolledInterface * poller;
-
- for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
- {
- poller = (IOPolledInterface *) vars->pollers->getObject(idx);
- err = poller->probe(target);
- if (err)
- {
- HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
- break;
- }
- }
-
- return (err);
-}
-
-static IOReturn
-IOHibernatePollerOpen(IOPolledFileIOVars * vars, uint32_t state, IOMemoryDescriptor * md)
-{
- IOReturn err = kIOReturnError;
- int32_t idx;
- IOPolledInterface * poller;
-
- for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
- {
- poller = (IOPolledInterface *) vars->pollers->getObject(idx);
- err = poller->open(state, md);
- if (err)
- {
- HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
- break;
- }
- }
-
- return (err);
-}
-
-static IOReturn
-IOHibernatePollerClose(IOPolledFileIOVars * vars, uint32_t state)
-{
- IOReturn err = kIOReturnError;
- int32_t idx;
- IOPolledInterface * poller;
-
- for (idx = 0;
- (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
- idx++)
- {
- err = poller->close(state);
- if (err)
- HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
- }
-
- return (err);
-}
-
-static void
-IOHibernatePollerIOComplete(void * target,
- void * parameter,
- IOReturn status,
- UInt64 actualByteCount)
-{
- IOPolledFileIOVars * vars = (IOPolledFileIOVars *) parameter;
-
- vars->ioStatus = status;
-}
-
-static IOReturn
-IOHibernatePollerIO(IOPolledFileIOVars * vars,
- uint32_t operation, uint32_t bufferOffset,
- uint64_t deviceOffset, uint64_t length)
-{
-
- IOReturn err = kIOReturnError;
- IOPolledInterface * poller;
- IOPolledCompletion completion;
-
- completion.target = 0;
- completion.action = &IOHibernatePollerIOComplete;
- completion.parameter = vars;
-
- vars->ioStatus = -1;
-
- poller = (IOPolledInterface *) vars->pollers->getObject(0);
- err = poller->startIO(operation, bufferOffset, deviceOffset + vars->block0, length, completion);
- if (err)
- HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
-
- return (err);
-}
-
-static IOReturn
-IOHibernatePollerIODone(IOPolledFileIOVars * vars, bool abortable)
-{
- IOReturn err = kIOReturnSuccess;
- int32_t idx = 0;
- IOPolledInterface * poller;
-
- while (-1 == vars->ioStatus)
- {
- for (idx = 0;
- (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
- idx++)
- {
- IOReturn newErr;
- newErr = poller->checkForWork();
- if ((newErr == kIOReturnAborted) && !abortable)
- newErr = kIOReturnSuccess;
- if (kIOReturnSuccess == err)
- err = newErr;
- }
- }
-
- if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort())
- {
- err = kIOReturnAborted;
- HIBLOG("IOPolledInterface::checkForWork sw abort\n");
- }
-
- if (err)
- {
- HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
- }
- else
- {
- err = vars->ioStatus;
- if (kIOReturnSuccess != err)
- HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
- }
-
- return (err);
-}
-
-IOReturn
-IOPolledInterface::checkAllForWork(void)
-{
- IOReturn err = kIOReturnNotReady;
- int32_t idx;
- IOPolledInterface * poller;
-
- IOHibernateVars * vars = &gIOHibernateVars;
-
- if (!vars->fileVars || !vars->fileVars->pollers)
- return (err);
-
- for (idx = 0;
- (poller = (IOPolledInterface *) vars->fileVars->pollers->getObject(idx));
- idx++)
- {
- err = poller->checkForWork();
- if (err)
- HIBLOG("IOPolledInterface::checkAllForWork[%d] 0x%x\n", idx, err);
- }
-
- return (err);
-}
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-struct _OpenFileContext
-{
- OSData * extents;
- uint64_t size;
-};
-
-static void
-file_extent_callback(void * ref, uint64_t start, uint64_t length)
-{
- _OpenFileContext * ctx = (_OpenFileContext *) ref;
- IOPolledFileExtent extent;
-
- extent.start = start;
- extent.length = length;
-
- HIBLOG("[0x%qx, 0x%qx]\n", start, length);
-
- ctx->extents->appendBytes(&extent, sizeof(extent));
- ctx->size += length;
-}
-
-static IOService *
-IOCopyMediaForDev(dev_t device)
-{
- OSDictionary * matching;
- OSNumber * num;
- OSIterator * iter;
- IOService * result = 0;
-
- matching = IOService::serviceMatching("IOMedia");
- if (!matching)
- return (0);
- do
- {
- num = OSNumber::withNumber(major(device), 32);
- if (!num)
- break;
- matching->setObject(kIOBSDMajorKey, num);
- num->release();
- num = OSNumber::withNumber(minor(device), 32);
- if (!num)
- break;
- matching->setObject(kIOBSDMinorKey, num);
- num->release();
- if (!num)
- break;
- iter = IOService::getMatchingServices(matching);
- if (iter)
- {
- result = (IOService *) iter->getNextObject();
- result->retain();
- iter->release();
- }
- }
- while (false);
- matching->release();
-
- return (result);
-}
-
-IOReturn
-IOPolledFileOpen( const char * filename, uint64_t setFileSize,
- IOBufferMemoryDescriptor * ioBuffer,
- IOPolledFileIOVars ** fileVars, OSData ** fileExtents,
- OSData ** imagePath, uint8_t * volumeCryptKey)
-{
- IOReturn err = kIOReturnSuccess;
- IOPolledFileIOVars * vars;
- _OpenFileContext ctx;
- OSData * extentsData;
- OSNumber * num;
- IOService * part = 0;
- OSString * keyUUID = 0;
- OSString * keyStoreUUID = 0;
- dev_t block_dev;
- dev_t hibernate_image_dev;
- uint64_t maxiobytes;
- AbsoluteTime startTime, endTime;
- uint64_t nsec;
-
- vars = IONew(IOPolledFileIOVars, 1);
- if (!vars) return (kIOReturnNoMemory);
- bzero(vars, sizeof(*vars));
-
- do
- {
- vars->io = false;
- vars->buffer = (uint8_t *) ioBuffer->getBytesNoCopy();
- vars->bufferHalf = 0;
- vars->bufferOffset = 0;
- vars->bufferSize = ioBuffer->getLength() >> 1;
-
- extentsData = OSData::withCapacity(32);
- ctx.extents = extentsData;
- ctx.size = 0;
- clock_get_uptime(&startTime);
- vars->fileRef = kern_open_file_for_direct_io(filename,
- &file_extent_callback, &ctx,
- setFileSize,
- // write file:
- 0, (caddr_t) gIOHibernateCurrentHeader,
- sizeof(IOHibernateImageHeader),
- // results
- &block_dev,
- &hibernate_image_dev,
- &vars->block0,
- &maxiobytes,
- &vars->flags);
-#if 0
- uint32_t msDelay = (131071 & random());
- HIBLOG("sleep %d\n", msDelay);
- IOSleep(msDelay);
-#endif
- clock_get_uptime(&endTime);
- SUB_ABSOLUTETIME(&endTime, &startTime);
- absolutetime_to_nanoseconds(endTime, &nsec);
-
- if (!vars->fileRef) err = kIOReturnNoSpace;
-
- IOLockLock(gFSLock);
- if (kFSOpening != gFSState) err = kIOReturnTimeout;
- IOLockUnlock(gFSLock);
-
- HIBLOG("kern_open_file_for_direct_io(%d) took %qd ms\n", err, nsec / 1000000ULL);
- if (kIOReturnSuccess != err) break;
-
- if (kIOHibernateModeSSDInvert & gIOHibernateMode)
- vars->flags ^= kIOHibernateOptionSSD;
-
- HIBLOG("Opened file %s, size %qd, partition base 0x%qx, maxio %qx ssd %d\n", filename, ctx.size,
- vars->block0, maxiobytes, kIOHibernateOptionSSD & vars->flags);
- if (ctx.size < 1*1024*1024) // check against image size estimate!
- {
- err = kIOReturnNoSpace;
- break;
- }
-
- vars->fileSize = ctx.size;
- if (maxiobytes < vars->bufferSize) vars->bufferSize = maxiobytes;
-
- vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
-
- part = IOCopyMediaForDev(block_dev);
- if (!part)
- {
- err = kIOReturnNotFound;
- break;
- }
- err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false,
- (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL);
- if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID)
- {
-// IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy());
- uuid_t volumeKeyUUID;
- aks_volume_key_t vek;
- static IOService * sKeyStore;
- static const OSSymbol * sAKSGetKey;
-
- if (!sAKSGetKey)
- sAKSGetKey = OSSymbol::withCStringNoCopy(AKS_PLATFORM_FUNCTION_GETKEY);
- if (!sKeyStore)
- sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane);
- if (sKeyStore)
- err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID);
- else
- err = kIOReturnNoResources;
- if (kIOReturnSuccess == err)
- err = sKeyStore->callPlatformFunction(sAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL);
- if (kIOReturnSuccess != err)
- IOLog("volume key err 0x%x\n", err);
- else
- {
- size_t bytes = (kIOHibernateAESKeySize / 8);
- if (vek.key.keybytecount < bytes)
- bytes = vek.key.keybytecount;
- bcopy(&vek.key.keybytes[0], volumeCryptKey, bytes);
- }
- bzero(&vek, sizeof(vek));
- }
- part->release();
-
- part = IOCopyMediaForDev(hibernate_image_dev);
- if (!part)
- {
- err = kIOReturnNotFound;
- break;
- }
-
- IORegistryEntry * next;
- IORegistryEntry * child;
- IOService * service;
- OSData * data;
-
- vars->pollers = OSArray::withCapacity(4);
- if (!vars->pollers)
- {
- err = kIOReturnNoMemory;
- break;
- }
-
- vars->blockSize = 512;
- next = part;
- do
- {
- IOPolledInterface * poller;
- OSObject * obj;
-
- obj = next->getProperty(kIOPolledInterfaceSupportKey);
- if (kOSBooleanFalse == obj)
- {
- vars->pollers->flushCollection();
- break;
- }
- else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
- vars->pollers->setObject(poller);
-
- if ((service = OSDynamicCast(IOService, next))
- && service->getDeviceMemory()
- && !vars->pollers->getCount()) break;
-
- if ((num = OSDynamicCast(OSNumber, next->getProperty(kIOMediaPreferredBlockSizeKey))))
- vars->blockSize = num->unsigned32BitValue();
- child = next;
- }
- while ((next = child->getParentEntry(gIOServicePlane))
- && child->isParent(next, gIOServicePlane, true));
-
- if (vars->blockSize < 4096) vars->blockSize = 4096;
-
- HIBLOG("hibernate image major %d, minor %d, blocksize %ld, pollers %d\n",
- major(hibernate_image_dev), minor(hibernate_image_dev), (long)vars->blockSize,
- vars->pollers->getCount());
-
- if (!vars->pollers->getCount())
- {
- err = kIOReturnUnsupported;
- continue;
- }
- if (vars->blockSize < sizeof(IOHibernateImageHeader))
- {
- err = kIOReturnError;
- continue;
- }
-
- err = IOHibernatePollerProbe(vars, (IOService *) part);
- if (kIOReturnSuccess != err) break;
-
- err = IOHibernatePollerOpen(vars, kIOPolledPreflightState, ioBuffer);
- if (kIOReturnSuccess != err) break;
-
- vars->media = part;
- next = part;
- while (next)
- {
- next->setProperty(kIOPolledInterfaceActiveKey, kOSBooleanTrue);
- next = next->getParentEntry(gIOServicePlane);
- }
-
- *fileVars = vars;
- *fileExtents = extentsData;
-
- // make imagePath
-
- if ((extentsData->getLength() >= sizeof(IOPolledFileExtent)))
- {
- char str2[24 + sizeof(uuid_string_t) + 2];
-
-#if defined(__i386__) || defined(__x86_64__)
- if (!gIOCreateEFIDevicePathSymbol)
- gIOCreateEFIDevicePathSymbol = OSSymbol::withCString("CreateEFIDevicePath");
-
- if (keyUUID)
- snprintf(str2, sizeof(str2), "%qx:%s",
- vars->extentMap[0].start, keyUUID->getCStringNoCopy());
- else
- snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
-
- err = IOService::getPlatform()->callPlatformFunction(
- gIOCreateEFIDevicePathSymbol, false,
- (void *) part, (void *) str2,
- (void *) (uintptr_t) true, (void *) &data);
-#else
- char str1[256];
- int len = sizeof(str1);
-
- if (!part->getPath(str1, &len, gIODTPlane))
- err = kIOReturnNotFound;
- else
- {
- snprintf(str2, sizeof(str2), ",%qx", vars->extentMap[0].start);
- // (strip the plane name)
- char * tail = strchr(str1, ':');
- if (!tail)
- tail = str1 - 1;
- data = OSData::withBytes(tail + 1, strlen(tail + 1));
- data->appendBytes(str2, strlen(str2));
- }
-#endif
- if (kIOReturnSuccess == err)
- *imagePath = data;
- else
- HIBLOG("error 0x%x getting path\n", err);
- }
- }
- while (false);
-
- if (kIOReturnSuccess != err)
- {
- HIBLOG("error 0x%x opening hibernation file\n", err);
- if (vars->fileRef)
- {
- kern_close_file_for_direct_io(vars->fileRef, 0, 0, 0, 0, 0);
- vars->fileRef = NULL;
- }
- }
-
- if (part)
- part->release();
-
- return (err);
-}
-
-IOReturn
-IOPolledFileClose( IOPolledFileIOVars * vars )
-{
- if (vars->pollers)
- {
- IOHibernatePollerClose(vars, kIOPolledPostflightState);
- vars->pollers->release();
- }
-
- bzero(vars, sizeof(IOPolledFileIOVars));
-
- return (kIOReturnSuccess);
-}
-
-static IOReturn
-IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
-{
- IOPolledFileExtent * extentMap;
-
- extentMap = vars->extentMap;
-
- vars->position = position;
-
- while (position >= extentMap->length)
- {
- position -= extentMap->length;
- extentMap++;
- }
-
- vars->currentExtent = extentMap;
- vars->extentRemaining = extentMap->length - position;
- vars->extentPosition = vars->position - position;
-
- if (vars->bufferSize <= vars->extentRemaining)
- vars->bufferLimit = vars->bufferSize;
- else
- vars->bufferLimit = vars->extentRemaining;
-
- return (kIOReturnSuccess);
-}
-
-static IOReturn
-IOPolledFileWrite(IOPolledFileIOVars * vars,
- const uint8_t * bytes, IOByteCount size,
- hibernate_cryptvars_t * cryptvars)
-{
- IOReturn err = kIOReturnSuccess;
- IOByteCount copy;
- bool flush = false;
-
- do
- {
- if (!bytes && !size)
- {
- // seek to end of block & flush
- size = vars->position & (vars->blockSize - 1);
- if (size)
- size = vars->blockSize - size;
- flush = true;
- // use some garbage for the fill
- bytes = vars->buffer + vars->bufferOffset;
- }
-
- copy = vars->bufferLimit - vars->bufferOffset;
- if (copy > size)
- copy = size;
- else
- flush = true;
-
- if (bytes)
- {
- bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
- bytes += copy;
- }
- else
- bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
-
- size -= copy;
- vars->bufferOffset += copy;
- vars->position += copy;
-
- if (flush && vars->bufferOffset)
- {
- uint64_t offset = (vars->position - vars->bufferOffset
- - vars->extentPosition + vars->currentExtent->start);
- uint32_t length = (vars->bufferOffset);
-
-#if CRYPTO
- if (cryptvars && vars->encryptStart
- && (vars->position > vars->encryptStart)
- && ((vars->position - length) < vars->encryptEnd))
- {
- AbsoluteTime startTime, endTime;
-
- uint64_t encryptLen, encryptStart;
- encryptLen = vars->position - vars->encryptStart;
- if (encryptLen > length)
- encryptLen = length;
- encryptStart = length - encryptLen;
- if (vars->position > vars->encryptEnd)
- encryptLen -= (vars->position - vars->encryptEnd);
-
- clock_get_uptime(&startTime);
-
- // encrypt the buffer
- aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
- &cryptvars->aes_iv[0],
- encryptLen / AES_BLOCK_SIZE,
- vars->buffer + vars->bufferHalf + encryptStart,
- &cryptvars->ctx.encrypt);
-
- clock_get_uptime(&endTime);
- ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
- SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
- vars->cryptBytes += encryptLen;
-
- // save initial vector for following encrypts
- bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
- &cryptvars->aes_iv[0],
- AES_BLOCK_SIZE);
- }
-#endif /* CRYPTO */
-
- if (vars->io)
- {
- err = IOHibernatePollerIODone(vars, true);
- if (kIOReturnSuccess != err)
- break;
- }
-
-if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
-//if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
-
- err = IOHibernatePollerIO(vars, kIOPolledWrite, vars->bufferHalf, offset, length);
- if (kIOReturnSuccess != err)
- break;
- vars->io = true;
-
- vars->extentRemaining -= vars->bufferOffset;
- if (!vars->extentRemaining)
- {
- vars->currentExtent++;
- vars->extentRemaining = vars->currentExtent->length;
- vars->extentPosition = vars->position;
- if (!vars->extentRemaining)
- {
- err = kIOReturnOverrun;
- break;
- }
- }
-
- vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
- vars->bufferOffset = 0;
- if (vars->bufferSize <= vars->extentRemaining)
- vars->bufferLimit = vars->bufferSize;
- else
- vars->bufferLimit = vars->extentRemaining;
-
- flush = false;
- }
- }
- while (size);
-
- return (err);
-}
-
-static IOReturn
-IOPolledFileRead(IOPolledFileIOVars * vars,
- uint8_t * bytes, IOByteCount size,
- hibernate_cryptvars_t * cryptvars)
-{
- IOReturn err = kIOReturnSuccess;
- IOByteCount copy;
-
-// bytesWritten += size;
-
- do
- {
- copy = vars->bufferLimit - vars->bufferOffset;
- if (copy > size)
- copy = size;
-
- if (bytes)
- {
- bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
- bytes += copy;
- }
- size -= copy;
- vars->bufferOffset += copy;
-// vars->position += copy;
-
- if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd))
- {
- if (vars->io)
- {
- err = IOHibernatePollerIODone(vars, false);
- if (kIOReturnSuccess != err)
- break;
- }
- else
- cryptvars = 0;
-
-if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
-
- vars->position += vars->lastRead;
- vars->extentRemaining -= vars->lastRead;
- vars->bufferLimit = vars->lastRead;
-
- if (!vars->extentRemaining)
- {
- vars->currentExtent++;
- vars->extentRemaining = vars->currentExtent->length;
- vars->extentPosition = vars->position;
- if (!vars->extentRemaining)
- {
- err = kIOReturnOverrun;
- break;
- }
- }
-
- uint64_t length;
- uint64_t lastReadLength = vars->lastRead;
- uint64_t offset = (vars->position
- - vars->extentPosition + vars->currentExtent->start);
- if (vars->extentRemaining <= vars->bufferSize)
- length = vars->extentRemaining;
- else
- length = vars->bufferSize;
- if ((length + vars->position) > vars->readEnd)
- length = vars->readEnd - vars->position;
-
- vars->lastRead = length;
- if (length)
- {
-//if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
- err = IOHibernatePollerIO(vars, kIOPolledRead, vars->bufferHalf, offset, length);
- if (kIOReturnSuccess != err)
- break;
- vars->io = true;
- }
-
- vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
- vars->bufferOffset = 0;
-
-#if CRYPTO
- if (cryptvars)
- {
- uint8_t thisVector[AES_BLOCK_SIZE];
- AbsoluteTime startTime, endTime;
-
- // save initial vector for following decrypts
- bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
- bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE,
- &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
-
- // decrypt the buffer
- clock_get_uptime(&startTime);
-
- aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
- &thisVector[0],
- lastReadLength / AES_BLOCK_SIZE,
- vars->buffer + vars->bufferHalf,
- &cryptvars->ctx.decrypt);
-
- clock_get_uptime(&endTime);
- ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
- SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
- vars->cryptBytes += lastReadLength;
- }
-#endif /* CRYPTO */
- }
projects / apple / xnu.git / blobdiff