* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
+#include <sys/uio.h>
+#include <sys/conf.h>
+
+#include <IOKit/IOLib.h>
+#include <IOKit/IOBSD.h>
#include <IOKit/IOService.h>
+#include <IOKit/IOPlatformExpert.h>
#include <IOKit/IOPolledInterface.h>
+#include <IOKit/IOHibernatePrivate.h>
+#include <IOKit/IOBufferMemoryDescriptor.h>
+#include <IOKit/AppleKeyStoreInterface.h>
+#include "IOKitKernelInternal.h"
+
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
OSMetaClassDefineReservedUnused(IOPolledInterface, 14);
OSMetaClassDefineReservedUnused(IOPolledInterface, 15);
-#if !HIBERNATION
-/* KPI stub if hibernate is configured off */
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+#ifndef kIOMediaPreferredBlockSizeKey
+#define kIOMediaPreferredBlockSizeKey "Preferred Block Size"
+#endif
+
+enum { kDefaultIOSize = 128*1024 };
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+class IOPolledFilePollers : public OSObject
+{
+ OSDeclareDefaultStructors(IOPolledFilePollers)
+
+public:
+ IOService * media;
+ OSArray * pollers;
+ IOBufferMemoryDescriptor * ioBuffer;
+ bool abortable;
+ bool io;
+ IOReturn ioStatus;
+ uint32_t openCount;
+ uint32_t openState;
+
+ static IOPolledFilePollers * copyPollers(IOService * media);
+};
+
+OSDefineMetaClassAndStructors(IOPolledFilePollers, OSObject)
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+IOPolledFilePollers *
+IOPolledFilePollers::copyPollers(IOService * media)
+{
+ IOPolledFilePollers * vars;
+ IOReturn err;
+ IOService * service;
+ OSObject * obj;
+ IORegistryEntry * next;
+ IORegistryEntry * child;
+
+ if ((obj = media->copyProperty(kIOPolledInterfaceStackKey)))
+ {
+ return (OSDynamicCast(IOPolledFilePollers, obj));
+ }
+
+ do
+ {
+ vars = OSTypeAlloc(IOPolledFilePollers);
+ vars->init();
+
+ vars->pollers = OSArray::withCapacity(4);
+ if (!vars->pollers)
+ {
+ err = kIOReturnNoMemory;
+ break;
+ }
+
+ next = vars->media = media;
+ do
+ {
+ IOPolledInterface * poller;
+ OSObject * obj;
+
+ obj = next->getProperty(kIOPolledInterfaceSupportKey);
+ if (kOSBooleanFalse == obj)
+ {
+ vars->pollers->flushCollection();
+ break;
+ }
+ else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
+ vars->pollers->setObject(poller);
+
+ if ((service = OSDynamicCast(IOService, next))
+ && service->getDeviceMemory()
+ && !vars->pollers->getCount()) break;
+
+ child = next;
+ }
+ while ((next = child->getParentEntry(gIOServicePlane))
+ && child->isParent(next, gIOServicePlane, true));
+
+ if (!vars->pollers->getCount())
+ {
+ err = kIOReturnUnsupported;
+ break;
+ }
+ }
+ while (false);
+
+ media->setProperty(kIOPolledInterfaceStackKey, vars);
+
+ return (vars);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+static IOReturn
+IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable);
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+static IOReturn
+IOPolledFilePollersProbe(IOPolledFilePollers * vars)
+{
+ IOReturn err = kIOReturnError;
+ int32_t idx;
+ IOPolledInterface * poller;
+
+ for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
+ {
+ poller = (IOPolledInterface *) vars->pollers->getObject(idx);
+ err = poller->probe(vars->media);
+ if (err)
+ {
+ HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
+ break;
+ }
+ }
+
+ return (err);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+IOReturn
+IOPolledFilePollersOpen(IOPolledFileIOVars * filevars, uint32_t state, bool abortable)
+{
+
+ IOPolledFilePollers * vars = filevars->pollers;
+ IOBufferMemoryDescriptor * ioBuffer;
+ IOPolledInterface * poller;
+ IOService * next;
+ IOReturn err = kIOReturnError;
+ int32_t idx;
+
+ vars->abortable = abortable;
+ ioBuffer = 0;
+
+ if (kIOPolledAfterSleepState == state)
+ {
+ vars->ioStatus = 0;
+ vars->io = false;
+ }
+ (void) IOPolledFilePollersIODone(vars, false);
+
+ if ((kIOPolledPreflightState == state) || (kIOPolledPreflightCoreDumpState == state))
+ {
+ ioBuffer = vars->ioBuffer;
+ if (!ioBuffer)
+ {
+ vars->ioBuffer = ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionInOut,
+ 2 * kDefaultIOSize, page_size);
+ if (!ioBuffer) return (kIOReturnNoMemory);
+ }
+ }
+
+ for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
+ {
+ poller = (IOPolledInterface *) vars->pollers->getObject(idx);
+ err = poller->open(state, ioBuffer);
+ if ((kIOReturnSuccess != err) && (kIOPolledPreflightCoreDumpState == state))
+ {
+ err = poller->open(kIOPolledPreflightState, ioBuffer);
+ }
+ if (kIOReturnSuccess != err)
+ {
+ HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
+ break;
+ }
+ }
+ if (kIOReturnSuccess == err)
+ {
+ next = vars->media;
+ while (next)
+ {
+ next->setProperty(kIOPolledInterfaceActiveKey, kOSBooleanTrue);
+ next = next->getProvider();
+ }
+ }
+
+ return (err);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+IOReturn
+IOPolledFilePollersClose(IOPolledFileIOVars * filevars, uint32_t state)
+{
+ IOPolledFilePollers * vars = filevars->pollers;
+ IOPolledInterface * poller;
+ IORegistryEntry * next;
+ IOReturn err;
+ int32_t idx;
+
+ (void) IOPolledFilePollersIODone(vars, false);
+
+ if (kIOPolledPostflightState == state)
+ {
+ vars->openCount--;
+ if (vars->openCount)
+ {
+ // 21207427
+ IOPolledFilePollersOpen(filevars, vars->openState, vars->abortable);
+ return (kIOReturnSuccess);
+ }
+ }
+
+ for (idx = 0, err = kIOReturnSuccess;
+ (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
+ idx++)
+ {
+ err = poller->close(state);
+ if (err) HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
+ }
+
+ if (kIOPolledPostflightState == state)
+ {
+ next = vars->media;
+ while (next)
+ {
+ next->removeProperty(kIOPolledInterfaceActiveKey);
+ next = next->getParentEntry(gIOServicePlane);
+ }
+
+ if (vars->ioBuffer)
+ {
+ vars->ioBuffer->release();
+ vars->ioBuffer = 0;
+ }
+ }
+ return (err);
+}
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+IOMemoryDescriptor *
+IOPolledFileGetIOBuffer(IOPolledFileIOVars * vars)
+{
+ return (vars->pollers->ioBuffer);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+static void
+IOPolledIOComplete(void * target,
+ void * parameter,
+ IOReturn status,
+ UInt64 actualByteCount)
+{
+ IOPolledFilePollers * vars = (IOPolledFilePollers *) parameter;
+
+ vars->ioStatus = status;
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+static IOReturn
+IOStartPolledIO(IOPolledFilePollers * vars,
+ uint32_t operation, uint32_t bufferOffset,
+ uint64_t deviceOffset, uint64_t length)
+{
+ IOReturn err;
+ IOPolledInterface * poller;
+ IOPolledCompletion completion;
+
+ err = vars->ioStatus;
+ if (kIOReturnSuccess != err) return (err);
+
+ completion.target = 0;
+ completion.action = &IOPolledIOComplete;
+ completion.parameter = vars;
+
+ vars->ioStatus = -1;
+
+ poller = (IOPolledInterface *) vars->pollers->getObject(0);
+ err = poller->startIO(operation, bufferOffset, deviceOffset, length, completion);
+ if (err)
+ HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
+
+ return (err);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+static IOReturn
+IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable)
+{
+ IOReturn err = kIOReturnSuccess;
+ int32_t idx = 0;
+ IOPolledInterface * poller;
+ AbsoluteTime deadline;
+
+ if (!vars->io) return (kIOReturnSuccess);
+
+ abortable &= vars->abortable;
+
+ clock_interval_to_deadline(2000, kMillisecondScale, &deadline);
+
+ while (-1 == vars->ioStatus)
+ {
+ for (idx = 0;
+ (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
+ idx++)
+ {
+ IOReturn newErr;
+ newErr = poller->checkForWork();
+ if ((newErr == kIOReturnAborted) && !abortable)
+ newErr = kIOReturnSuccess;
+ if (kIOReturnSuccess == err)
+ err = newErr;
+ }
+ if ((false) && (kIOReturnSuccess == err) && (mach_absolute_time() > AbsoluteTime_to_scalar(&deadline)))
+ {
+ HIBLOG("IOPolledInterface::forced timeout\n");
+ vars->ioStatus = kIOReturnTimeout;
+ }
+ }
+ vars->io = false;
+
+#if HIBERNATION
+ if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort())
+ {
+ err = kIOReturnAborted;
+ HIBLOG("IOPolledInterface::checkForWork sw abort\n");
+ }
+#endif
+
+ if (err)
+ {
+ HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
+ }
+ else
+ {
+ err = vars->ioStatus;
+ if (kIOReturnSuccess != err) HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
+ }
+
+ return (err);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+struct _OpenFileContext
+{
+ OSData * extents;
+ uint64_t size;
+};
+
+static void
+file_extent_callback(void * ref, uint64_t start, uint64_t length)
+{
+ _OpenFileContext * ctx = (_OpenFileContext *) ref;
+ IOPolledFileExtent extent;
+
+ extent.start = start;
+ extent.length = length;
+ ctx->extents->appendBytes(&extent, sizeof(extent));
+ ctx->size += length;
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+static IOService *
+IOCopyMediaForDev(dev_t device)
+{
+ OSDictionary * matching;
+ OSNumber * num;
+ OSIterator * iter;
+ IOService * result = 0;
+
+ matching = IOService::serviceMatching("IOMedia");
+ if (!matching)
+ return (0);
+ do
+ {
+ num = OSNumber::withNumber(major(device), 32);
+ if (!num)
+ break;
+ matching->setObject(kIOBSDMajorKey, num);
+ num->release();
+ num = OSNumber::withNumber(minor(device), 32);
+ if (!num)
+ break;
+ matching->setObject(kIOBSDMinorKey, num);
+ num->release();
+ if (!num)
+ break;
+ iter = IOService::getMatchingServices(matching);
+ if (iter)
+ {
+ result = (IOService *) iter->getNextObject();
+ result->retain();
+ iter->release();
+ }
+ }
+ while (false);
+ matching->release();
+
+ return (result);
+}
+
+static IOReturn
+IOGetVolumeCryptKey(dev_t block_dev, OSString ** pKeyUUID,
+ uint8_t * volumeCryptKey, size_t keySize)
+{
+ IOReturn err;
+ IOService * part;
+ OSString * keyUUID = 0;
+ OSString * keyStoreUUID = 0;
+ uuid_t volumeKeyUUID;
+ aks_volume_key_t vek;
+
+ static IOService * sKeyStore;
+
+ part = IOCopyMediaForDev(block_dev);
+ if (!part) return (kIOReturnNotFound);
+
+ err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false,
+ (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL);
+ if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID)
+ {
+// IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy());
+
+ if (!sKeyStore)
+ sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane);
+ if (sKeyStore)
+ err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID);
+ else
+ err = kIOReturnNoResources;
+ if (kIOReturnSuccess == err)
+ err = sKeyStore->callPlatformFunction(gAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL);
+ if (kIOReturnSuccess != err)
+ IOLog("volume key err 0x%x\n", err);
+ else
+ {
+ if (vek.key.keybytecount < keySize) keySize = vek.key.keybytecount;
+ bcopy(&vek.key.keybytes[0], volumeCryptKey, keySize);
+ }
+ bzero(&vek, sizeof(vek));
+
+ }
+ part->release();
+ if (pKeyUUID) *pKeyUUID = keyUUID;
+
+ return (err);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
IOReturn
-IOPolledInterface::checkAllForWork(void)
+IOPolledFileOpen(const char * filename,
+ uint64_t setFileSize, uint64_t fsFreeSize,
+ void * write_file_addr, size_t write_file_len,
+ IOPolledFileIOVars ** fileVars,
+ OSData ** imagePath,
+ uint8_t * volumeCryptKey, size_t keySize)
{
- IOReturn err = kIOReturnNotReady;
+ IOReturn err = kIOReturnSuccess;
+ IOPolledFileIOVars * vars;
+ _OpenFileContext ctx;
+ OSData * extentsData;
+ OSNumber * num;
+ IOService * part = 0;
+ dev_t block_dev;
+ dev_t image_dev;
+ AbsoluteTime startTime, endTime;
+ uint64_t nsec;
+
+ vars = IONew(IOPolledFileIOVars, 1);
+ if (!vars) return (kIOReturnNoMemory);
+ bzero(vars, sizeof(*vars));
+ vars->allocated = true;
+
+ do
+ {
+ extentsData = OSData::withCapacity(32);
+ ctx.extents = extentsData;
+ ctx.size = 0;
+ clock_get_uptime(&startTime);
+
+ vars->fileRef = kern_open_file_for_direct_io(filename,
+ (write_file_addr != NULL) || (0 != setFileSize),
+ &file_extent_callback, &ctx,
+ setFileSize,
+ fsFreeSize,
+ // write file:
+ 0, write_file_addr, write_file_len,
+ // results
+ &block_dev,
+ &image_dev,
+ &vars->block0,
+ &vars->maxiobytes,
+ &vars->flags);
+#if 0
+ uint32_t msDelay = (131071 & random());
+ HIBLOG("sleep %d\n", msDelay);
+ IOSleep(msDelay);
+#endif
+ clock_get_uptime(&endTime);
+ SUB_ABSOLUTETIME(&endTime, &startTime);
+ absolutetime_to_nanoseconds(endTime, &nsec);
+
+ if (!vars->fileRef) err = kIOReturnNoSpace;
+
+ HIBLOG("kern_open_file_for_direct_io took %qd ms\n", nsec / 1000000ULL);
+ if (kIOReturnSuccess != err) break;
+
+ HIBLOG("Opened file %s, size %qd, extents %ld, maxio %qx ssd %d\n", filename, ctx.size,
+ (extentsData->getLength() / sizeof(IOPolledFileExtent)) - 1,
+ vars->maxiobytes, kIOPolledFileSSD & vars->flags);
+ assert(!vars->block0);
+ if (extentsData->getLength() < sizeof(IOPolledFileExtent))
+ {
+ err = kIOReturnNoSpace;
+ break;
+ }
+
+ vars->fileSize = ctx.size;
+ vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
+
+ part = IOCopyMediaForDev(image_dev);
+ if (!part)
+ {
+ err = kIOReturnNotFound;
+ break;
+ }
+
+ if (!(vars->pollers = IOPolledFilePollers::copyPollers(part))) break;
+
+ if ((num = OSDynamicCast(OSNumber, part->getProperty(kIOMediaPreferredBlockSizeKey))))
+ vars->blockSize = num->unsigned32BitValue();
+ if (vars->blockSize < 4096) vars->blockSize = 4096;
+
+ HIBLOG("polled file major %d, minor %d, blocksize %ld, pollers %d\n",
+ major(image_dev), minor(image_dev), (long)vars->blockSize,
+ vars->pollers->pollers->getCount());
+
+ OSString * keyUUID = NULL;
+ if (volumeCryptKey)
+ {
+ err = IOGetVolumeCryptKey(block_dev, &keyUUID, volumeCryptKey, keySize);
+ }
+
+ *fileVars = vars;
+ vars->fileExtents = extentsData;
+
+ // make imagePath
+ OSData * data;
+ if (imagePath)
+ {
+#if defined(__i386__) || defined(__x86_64__)
+ char str2[24 + sizeof(uuid_string_t) + 2];
+
+ if (keyUUID)
+ snprintf(str2, sizeof(str2), "%qx:%s",
+ vars->extentMap[0].start, keyUUID->getCStringNoCopy());
+ else
+ snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
+
+ err = IOService::getPlatform()->callPlatformFunction(
+ gIOCreateEFIDevicePathSymbol, false,
+ (void *) part, (void *) str2,
+ (void *) (uintptr_t) true, (void *) &data);
+#else
+ data = 0;
+ err = kIOReturnSuccess;
+#endif
+ if (kIOReturnSuccess != err)
+ {
+ HIBLOG("error 0x%x getting path\n", err);
+ break;
+ }
+ *imagePath = data;
+ }
+ }
+ while (false);
+
+ if (kIOReturnSuccess != err)
+ {
+ HIBLOG("error 0x%x opening polled file\n", err);
+ IOPolledFileClose(&vars, 0, 0, 0, 0, 0);
+ }
+
+ if (part) part->release();
- return err;
+ return (err);
}
-#endif /* !HIBERNATION */
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+IOReturn
+IOPolledFileClose(IOPolledFileIOVars ** pVars,
+ off_t write_offset, void * addr, size_t write_length,
+ off_t discard_offset, off_t discard_end)
+{
+ IOPolledFileIOVars * vars;
+
+ vars = *pVars;
+ if (!vars) return(kIOReturnSuccess);
+
+ if (vars->fileRef)
+ {
+ kern_close_file_for_direct_io(vars->fileRef, write_offset, addr, write_length,
+ discard_offset, discard_end);
+ vars->fileRef = NULL;
+ }
+ if (vars->fileExtents)
+ {
+ vars->fileExtents->release();
+ vars->fileExtents = 0;
+ }
+ if (vars->pollers)
+ {
+ vars->pollers->release();
+ vars->pollers = 0;
+ }
+
+ if (vars->allocated) IODelete(vars, IOPolledFileIOVars, 1);
+ else bzero(vars, sizeof(IOPolledFileIOVars));
+ *pVars = NULL;
+
+ return (kIOReturnSuccess);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+IOReturn
+IOPolledFilePollersSetup(IOPolledFileIOVars * vars,
+ uint32_t openState)
+{
+ IOReturn err;
+
+ err = kIOReturnSuccess;
+ do
+ {
+ if (!vars->pollers->openCount)
+ {
+ err = IOPolledFilePollersProbe(vars->pollers);
+ if (kIOReturnSuccess != err) break;
+ err = IOPolledFilePollersOpen(vars, openState, false);
+ if (kIOReturnSuccess != err) break;
+ vars->pollers->openState = openState;
+ }
+ vars->pollers->openCount++;
+ vars->pollers->io = false;
+ vars->buffer = (uint8_t *) vars->pollers->ioBuffer->getBytesNoCopy();
+ vars->bufferHalf = 0;
+ vars->bufferOffset = 0;
+ vars->bufferSize = (vars->pollers->ioBuffer->getLength() >> 1);
+
+ if (vars->maxiobytes < vars->bufferSize) vars->bufferSize = vars->maxiobytes;
+ }
+ while (false);
+
+ if (kIOReturnSuccess != err) HIBLOG("IOPolledFilePollersSetup(%d) error 0x%x\n", openState, err);
+
+ return (err);
+}
+
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+IOReturn
+IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
+{
+ IOPolledFileExtent * extentMap;
+
+ extentMap = vars->extentMap;
+
+ vars->position = position;
+
+ while (position >= extentMap->length)
+ {
+ position -= extentMap->length;
+ extentMap++;
+ }
+
+ vars->currentExtent = extentMap;
+ vars->extentRemaining = extentMap->length - position;
+ vars->extentPosition = vars->position - position;
+
+ if (vars->bufferSize <= vars->extentRemaining)
+ vars->bufferLimit = vars->bufferSize;
+ else
+ vars->bufferLimit = vars->extentRemaining;
+
+ return (kIOReturnSuccess);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+IOReturn
+IOPolledFileWrite(IOPolledFileIOVars * vars,
+ const uint8_t * bytes, IOByteCount size,
+ IOPolledFileCryptVars * cryptvars)
+{
+ IOReturn err = kIOReturnSuccess;
+ IOByteCount copy;
+ bool flush = false;
+
+ do
+ {
+ if (!bytes && !size)
+ {
+ // seek to end of block & flush
+ size = vars->position & (vars->blockSize - 1);
+ if (size)
+ size = vars->blockSize - size;
+ flush = true;
+ // use some garbage for the fill
+ bytes = vars->buffer + vars->bufferOffset;
+ }
+
+ copy = vars->bufferLimit - vars->bufferOffset;
+ if (copy > size)
+ copy = size;
+ else
+ flush = true;
+
+ if (bytes)
+ {
+ bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
+ bytes += copy;
+ }
+ else
+ bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
+
+ size -= copy;
+ vars->bufferOffset += copy;
+ vars->position += copy;
+
+ if (flush && vars->bufferOffset)
+ {
+ uint64_t offset = (vars->position - vars->bufferOffset
+ - vars->extentPosition + vars->currentExtent->start);
+ uint32_t length = (vars->bufferOffset);
+
+#if CRYPTO
+ if (cryptvars && vars->encryptStart
+ && (vars->position > vars->encryptStart)
+ && ((vars->position - length) < vars->encryptEnd))
+ {
+ AbsoluteTime startTime, endTime;
+
+ uint64_t encryptLen, encryptStart;
+ encryptLen = vars->position - vars->encryptStart;
+ if (encryptLen > length)
+ encryptLen = length;
+ encryptStart = length - encryptLen;
+ if (vars->position > vars->encryptEnd)
+ encryptLen -= (vars->position - vars->encryptEnd);
+
+ clock_get_uptime(&startTime);
+
+ // encrypt the buffer
+ aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
+ &cryptvars->aes_iv[0],
+ encryptLen / AES_BLOCK_SIZE,
+ vars->buffer + vars->bufferHalf + encryptStart,
+ &cryptvars->ctx.encrypt);
+
+ clock_get_uptime(&endTime);
+ ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
+ SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
+ vars->cryptBytes += encryptLen;
+
+ // save initial vector for following encrypts
+ bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
+ &cryptvars->aes_iv[0],
+ AES_BLOCK_SIZE);
+ }
+#endif /* CRYPTO */
+
+ err = IOPolledFilePollersIODone(vars->pollers, true);
+ if (kIOReturnSuccess != err)
+ break;
+
+if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
+//if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
+
+ err = IOStartPolledIO(vars->pollers, kIOPolledWrite, vars->bufferHalf, offset, length);
+ if (kIOReturnSuccess != err)
+ break;
+ vars->pollers->io = true;
+
+ vars->extentRemaining -= vars->bufferOffset;
+ if (!vars->extentRemaining)
+ {
+ vars->currentExtent++;
+ vars->extentRemaining = vars->currentExtent->length;
+ vars->extentPosition = vars->position;
+ }
+
+ vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
+ vars->bufferOffset = 0;
+ if (vars->bufferSize <= vars->extentRemaining)
+ vars->bufferLimit = vars->bufferSize;
+ else
+ vars->bufferLimit = vars->extentRemaining;
+
+ if (!vars->extentRemaining)
+ {
+ err = kIOReturnOverrun;
+ break;
+ }
+
+ flush = false;
+ }
+ }
+ while (size);
+
+ return (err);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+IOReturn
+IOPolledFileRead(IOPolledFileIOVars * vars,
+ uint8_t * bytes, IOByteCount size,
+ IOPolledFileCryptVars * cryptvars)
+{
+ IOReturn err = kIOReturnSuccess;
+ IOByteCount copy;
+
+// bytesWritten += size;
+
+ do
+ {
+ copy = vars->bufferLimit - vars->bufferOffset;
+ if (copy > size)
+ copy = size;
+
+ if (bytes)
+ {
+ bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
+ bytes += copy;
+ }
+ size -= copy;
+ vars->bufferOffset += copy;
+// vars->position += copy;
+
+ if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd))
+ {
+ if (!vars->pollers->io) cryptvars = 0;
+ err = IOPolledFilePollersIODone(vars->pollers, true);
+ if (kIOReturnSuccess != err)
+ break;
+
+if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
+
+ vars->position += vars->lastRead;
+ vars->extentRemaining -= vars->lastRead;
+ vars->bufferLimit = vars->lastRead;
+
+ if (!vars->extentRemaining)
+ {
+ vars->currentExtent++;
+ vars->extentRemaining = vars->currentExtent->length;
+ vars->extentPosition = vars->position;
+ if (!vars->extentRemaining)
+ {
+ err = kIOReturnOverrun;
+ break;
+ }
+ }
+
+ uint64_t length;
+ uint64_t lastReadLength = vars->lastRead;
+ uint64_t offset = (vars->position
+ - vars->extentPosition + vars->currentExtent->start);
+ if (vars->extentRemaining <= vars->bufferSize)
+ length = vars->extentRemaining;
+ else
+ length = vars->bufferSize;
+ if ((length + vars->position) > vars->readEnd)
+ length = vars->readEnd - vars->position;
+
+ vars->lastRead = length;
+ if (length)
+ {
+//if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
+ err = IOStartPolledIO(vars->pollers, kIOPolledRead, vars->bufferHalf, offset, length);
+ if (kIOReturnSuccess != err)
+ break;
+ vars->pollers->io = true;
+ }
+
+ vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
+ vars->bufferOffset = 0;
+
+#if CRYPTO
+ if (cryptvars)
+ {
+ uint8_t thisVector[AES_BLOCK_SIZE];
+ AbsoluteTime startTime, endTime;
+
+ // save initial vector for following decrypts
+ bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
+ bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE,
+ &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
+
+ // decrypt the buffer
+ clock_get_uptime(&startTime);
+
+ aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
+ &thisVector[0],
+ lastReadLength / AES_BLOCK_SIZE,
+ vars->buffer + vars->bufferHalf,
+ &cryptvars->ctx.decrypt);
+
+ clock_get_uptime(&endTime);
+ ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
+ SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
+ vars->cryptBytes += lastReadLength;
+ }
+#endif /* CRYPTO */
+ }
+ }
+ while (size);
+
+ return (err);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
projects / apple / xnu.git / blobdiff