X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/d1ecb069dfe24481e4a83f44cb5217a2b06746d7..3e170ce000f1506b7b5d2c5c7faec85ceabb573d:/iokit/Kernel/IOPolledInterface.cpp

diff --git a/iokit/Kernel/IOPolledInterface.cpp b/iokit/Kernel/IOPolledInterface.cpp
index 440ac3235..1917714fc 100644
--- a/iokit/Kernel/IOPolledInterface.cpp
+++ b/iokit/Kernel/IOPolledInterface.cpp
@@ -26,8 +26,19 @@
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
 
+#include <sys/uio.h>
+#include <sys/conf.h>
+
+#include <IOKit/IOLib.h>
+#include <IOKit/IOBSD.h>
 #include <IOKit/IOService.h>
+#include <IOKit/IOPlatformExpert.h>
 #include <IOKit/IOPolledInterface.h>
+#include <IOKit/IOHibernatePrivate.h>
+#include <IOKit/IOBufferMemoryDescriptor.h>
+#include <IOKit/AppleKeyStoreInterface.h>
+#include "IOKitKernelInternal.h"
+
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
@@ -50,13 +61,929 @@ OSMetaClassDefineReservedUnused(IOPolledInterface, 13);
 OSMetaClassDefineReservedUnused(IOPolledInterface, 14);
 OSMetaClassDefineReservedUnused(IOPolledInterface, 15);
 
-#if !HIBERNATION
-/* KPI stub if hibernate is configured off */
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+#ifndef kIOMediaPreferredBlockSizeKey
+#define kIOMediaPreferredBlockSizeKey	"Preferred Block Size"
+#endif
+
+enum { kDefaultIOSize = 128*1024 };
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+class IOPolledFilePollers : public OSObject
+{
+    OSDeclareDefaultStructors(IOPolledFilePollers)
+
+public:
+    IOService                * media;
+    OSArray                  * pollers;
+    IOBufferMemoryDescriptor * ioBuffer;
+    bool                 abortable;
+    bool                 io;
+    IOReturn		 ioStatus;
+    uint32_t             openCount;
+    uint32_t             openState;
+
+    static IOPolledFilePollers * copyPollers(IOService * media);
+};
+
+OSDefineMetaClassAndStructors(IOPolledFilePollers, OSObject)
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+IOPolledFilePollers *
+IOPolledFilePollers::copyPollers(IOService * media)
+{
+    IOPolledFilePollers * vars;
+    IOReturn              err;
+    IOService       * service;
+    OSObject        * obj;
+    IORegistryEntry * next;
+    IORegistryEntry * child;
+
+    if ((obj = media->copyProperty(kIOPolledInterfaceStackKey)))
+    {
+	return (OSDynamicCast(IOPolledFilePollers, obj));
+    }
+
+    do
+    {
+	vars = OSTypeAlloc(IOPolledFilePollers);
+	vars->init();
+
+	vars->pollers = OSArray::withCapacity(4);
+	if (!vars->pollers)
+	{
+	    err = kIOReturnNoMemory;
+	    break;
+	}
+
+	next = vars->media = media;
+	do
+	{
+	    IOPolledInterface * poller;
+	    OSObject *          obj;
+
+	    obj = next->getProperty(kIOPolledInterfaceSupportKey);
+	    if (kOSBooleanFalse == obj)
+	    {
+		vars->pollers->flushCollection();
+		break;
+	    }
+	    else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
+		vars->pollers->setObject(poller);
+
+	    if ((service = OSDynamicCast(IOService, next)) 
+		&& service->getDeviceMemory()
+		&& !vars->pollers->getCount())	break;
+
+	    child = next;
+	}
+	while ((next = child->getParentEntry(gIOServicePlane)) 
+	       && child->isParent(next, gIOServicePlane, true));
+
+	if (!vars->pollers->getCount())
+	{
+	    err = kIOReturnUnsupported;
+	    break;
+	}
+    }
+    while (false);
+
+    media->setProperty(kIOPolledInterfaceStackKey, vars);
+
+    return (vars);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+static IOReturn 
+IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable);
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+static IOReturn
+IOPolledFilePollersProbe(IOPolledFilePollers * vars)
+{
+    IOReturn            err = kIOReturnError;
+    int32_t		idx;
+    IOPolledInterface * poller;
+
+    for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
+    {
+        poller = (IOPolledInterface *) vars->pollers->getObject(idx);
+        err = poller->probe(vars->media);
+        if (err)
+        {
+            HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
+            break;
+        }
+    }
+
+    return (err);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+IOReturn
+IOPolledFilePollersOpen(IOPolledFileIOVars * filevars, uint32_t state, bool abortable)
+{
+
+    IOPolledFilePollers      * vars = filevars->pollers;
+    IOBufferMemoryDescriptor * ioBuffer;
+    IOPolledInterface        * poller;
+    IOService                * next;
+    IOReturn                   err = kIOReturnError;
+    int32_t		       idx;
+
+    vars->abortable = abortable;
+    ioBuffer = 0;
+
+    if (kIOPolledAfterSleepState == state)
+    {
+        vars->ioStatus = 0;
+	vars->io = false;
+    }
+    (void) IOPolledFilePollersIODone(vars, false);
+
+    if ((kIOPolledPreflightState == state) || (kIOPolledPreflightCoreDumpState == state))
+    {
+        ioBuffer = vars->ioBuffer;
+        if (!ioBuffer)
+        {
+	    vars->ioBuffer = ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionInOut, 
+							    2 * kDefaultIOSize, page_size);
+	    if (!ioBuffer) return (kIOReturnNoMemory);
+        }
+    }
+
+    for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
+    {
+        poller = (IOPolledInterface *) vars->pollers->getObject(idx);
+        err = poller->open(state, ioBuffer);
+        if ((kIOReturnSuccess != err) && (kIOPolledPreflightCoreDumpState == state))
+        {
+	    err = poller->open(kIOPolledPreflightState, ioBuffer);
+        }
+        if (kIOReturnSuccess != err)
+        {
+            HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
+            break;
+        }
+    }
+    if (kIOReturnSuccess == err)
+    {
+        next = vars->media;
+	while (next)
+	{
+	    next->setProperty(kIOPolledInterfaceActiveKey, kOSBooleanTrue);
+	    next = next->getProvider();
+	}
+    }
+
+    return (err);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+IOReturn
+IOPolledFilePollersClose(IOPolledFileIOVars * filevars, uint32_t state)
+{
+    IOPolledFilePollers * vars = filevars->pollers;
+    IOPolledInterface * poller;
+    IORegistryEntry *   next;
+    IOReturn            err;
+    int32_t		idx;
+
+    (void) IOPolledFilePollersIODone(vars, false);
+
+    if (kIOPolledPostflightState == state)
+    {
+	vars->openCount--;
+	if (vars->openCount) 
+	{
+	    // 21207427
+            IOPolledFilePollersOpen(filevars, vars->openState, vars->abortable);
+	    return (kIOReturnSuccess);
+	}
+    }
+
+    for (idx = 0, err = kIOReturnSuccess;
+         (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
+         idx++)
+    {
+        err = poller->close(state);
+        if (err) HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
+    }
+
+    if (kIOPolledPostflightState == state)
+    {   
+	next = vars->media;
+	while (next)
+	{
+	    next->removeProperty(kIOPolledInterfaceActiveKey);
+	    next = next->getParentEntry(gIOServicePlane);
+	}
+
+	if (vars->ioBuffer)
+	{
+	    vars->ioBuffer->release();
+	    vars->ioBuffer = 0;
+	}
+    }
+    return (err);
+}
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+IOMemoryDescriptor *
+IOPolledFileGetIOBuffer(IOPolledFileIOVars * vars)
+{
+    return (vars->pollers->ioBuffer);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+static void
+IOPolledIOComplete(void *   target,
+		   void *   parameter,
+		   IOReturn status,
+		   UInt64   actualByteCount)
+{
+    IOPolledFilePollers * vars = (IOPolledFilePollers *) parameter;
+
+    vars->ioStatus = status;
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+static IOReturn
+IOStartPolledIO(IOPolledFilePollers * vars, 
+                    uint32_t operation, uint32_t bufferOffset, 
+		    uint64_t deviceOffset, uint64_t length)
+{
+    IOReturn            err;
+    IOPolledInterface * poller;
+    IOPolledCompletion  completion;
+
+    err = vars->ioStatus;
+    if (kIOReturnSuccess != err) return (err);
+
+    completion.target    = 0;
+    completion.action    = &IOPolledIOComplete;
+    completion.parameter = vars;
+
+    vars->ioStatus = -1;
+
+    poller = (IOPolledInterface *) vars->pollers->getObject(0);
+    err = poller->startIO(operation, bufferOffset, deviceOffset, length, completion);
+    if (err)
+        HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
+
+    return (err);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+static IOReturn
+IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable)
+{
+    IOReturn            err = kIOReturnSuccess;
+    int32_t		idx = 0;
+    IOPolledInterface * poller;
+    AbsoluteTime        deadline;
+
+    if (!vars->io) return (kIOReturnSuccess);
+
+    abortable &= vars->abortable;
+
+    clock_interval_to_deadline(2000, kMillisecondScale, &deadline);
+
+    while (-1 == vars->ioStatus)
+    {
+        for (idx = 0; 
+	    (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
+             idx++)
+        {
+	    IOReturn newErr;
+            newErr = poller->checkForWork();
+	    if ((newErr == kIOReturnAborted) && !abortable)
+		newErr = kIOReturnSuccess;
+	    if (kIOReturnSuccess == err)
+		err = newErr;
+        }
+        if ((false) && (kIOReturnSuccess == err) && (mach_absolute_time() > AbsoluteTime_to_scalar(&deadline)))
+    	{
+	    HIBLOG("IOPolledInterface::forced timeout\n");
+	    vars->ioStatus = kIOReturnTimeout;
+    	}
+    }
+    vars->io = false;
+
+#if HIBERNATION
+    if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort())
+    {
+        err = kIOReturnAborted;
+	HIBLOG("IOPolledInterface::checkForWork sw abort\n");
+    }
+#endif
+
+    if (err)
+    {
+	HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
+    }
+    else 
+    {
+	err = vars->ioStatus;
+	if (kIOReturnSuccess != err) HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
+    }
+
+    return (err);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+struct _OpenFileContext
+{
+    OSData * extents;
+    uint64_t size;
+};
+
+static void
+file_extent_callback(void * ref, uint64_t start, uint64_t length)
+{
+    _OpenFileContext * ctx = (_OpenFileContext *) ref;
+    IOPolledFileExtent extent;
+
+    extent.start  = start;
+    extent.length = length;
+    ctx->extents->appendBytes(&extent, sizeof(extent));
+    ctx->size += length;
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+static IOService * 
+IOCopyMediaForDev(dev_t device)
+{
+    OSDictionary * matching;
+    OSNumber *     num;
+    OSIterator *   iter;
+    IOService *    result = 0;
+
+    matching = IOService::serviceMatching("IOMedia");
+    if (!matching)
+        return (0);
+    do
+    {
+        num = OSNumber::withNumber(major(device), 32);
+        if (!num)
+            break;
+        matching->setObject(kIOBSDMajorKey, num);
+        num->release();
+        num = OSNumber::withNumber(minor(device), 32);
+        if (!num)
+            break;
+        matching->setObject(kIOBSDMinorKey, num);
+        num->release();
+        if (!num)
+            break;
+        iter = IOService::getMatchingServices(matching);
+        if (iter)
+        {
+            result = (IOService *) iter->getNextObject();
+            result->retain();
+            iter->release();
+        }
+    }
+    while (false);
+    matching->release();
+
+    return (result);
+}
+
+static IOReturn 
+IOGetVolumeCryptKey(dev_t block_dev,  OSString ** pKeyUUID, 
+		    uint8_t * volumeCryptKey, size_t keySize)
+{
+    IOReturn         err;
+    IOService *      part;
+    OSString *       keyUUID = 0;
+    OSString *       keyStoreUUID = 0;
+    uuid_t           volumeKeyUUID;
+    aks_volume_key_t vek;
+
+    static IOService * sKeyStore;
+
+    part = IOCopyMediaForDev(block_dev);
+    if (!part) return (kIOReturnNotFound);
+
+    err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false, 
+				      (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL);
+    if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID)
+    {
+//            IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy());
+
+	if (!sKeyStore)
+	    sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane);
+	if (sKeyStore)
+	    err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID);
+	else
+	    err = kIOReturnNoResources;
+	if (kIOReturnSuccess == err)    
+	    err = sKeyStore->callPlatformFunction(gAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL);
+	if (kIOReturnSuccess != err)    
+	    IOLog("volume key err 0x%x\n", err);
+	else
+	{
+	    if (vek.key.keybytecount < keySize) keySize = vek.key.keybytecount;
+	    bcopy(&vek.key.keybytes[0], volumeCryptKey, keySize);
+	}
+	bzero(&vek, sizeof(vek));
+
+    }
+    part->release();
+    if (pKeyUUID) *pKeyUUID = keyUUID;
+
+    return (err);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
 IOReturn
-IOPolledInterface::checkAllForWork(void)
+IOPolledFileOpen(const char * filename,
+		 uint64_t setFileSize, uint64_t fsFreeSize,
+		 void * write_file_addr, size_t write_file_len,
+		 IOPolledFileIOVars ** fileVars,
+		 OSData ** imagePath,
+		 uint8_t * volumeCryptKey, size_t keySize)
 {
-  IOReturn	err = kIOReturnNotReady;
+    IOReturn             err = kIOReturnSuccess;
+    IOPolledFileIOVars * vars;
+    _OpenFileContext     ctx;
+    OSData *             extentsData;
+    OSNumber *           num;
+    IOService *          part = 0;
+    dev_t                block_dev;
+    dev_t                image_dev;
+    AbsoluteTime         startTime, endTime;
+    uint64_t             nsec;
+
+    vars = IONew(IOPolledFileIOVars, 1);
+    if (!vars) return (kIOReturnNoMemory);
+    bzero(vars, sizeof(*vars));
+    vars->allocated = true;
+
+    do
+    {
+    	extentsData = OSData::withCapacity(32);
+   	ctx.extents = extentsData;
+	ctx.size    = 0;
+	clock_get_uptime(&startTime);
+
+	vars->fileRef = kern_open_file_for_direct_io(filename, 
+                                                     (write_file_addr != NULL) || (0 != setFileSize),
+						     &file_extent_callback, &ctx, 
+						     setFileSize,
+						     fsFreeSize,
+						     // write file:
+                                                     0, write_file_addr, write_file_len,
+                                                     // results
+						     &block_dev,
+						     &image_dev,
+                                                     &vars->block0,
+                                                     &vars->maxiobytes,
+                                                     &vars->flags);
+#if 0
+	uint32_t msDelay = (131071 & random());
+	HIBLOG("sleep %d\n", msDelay);
+	IOSleep(msDelay);
+#endif
+        clock_get_uptime(&endTime);
+        SUB_ABSOLUTETIME(&endTime, &startTime);
+        absolutetime_to_nanoseconds(endTime, &nsec);
+
+	if (!vars->fileRef) err = kIOReturnNoSpace;
+
+        HIBLOG("kern_open_file_for_direct_io took %qd ms\n", nsec / 1000000ULL);
+	if (kIOReturnSuccess != err) break;
+
+	HIBLOG("Opened file %s, size %qd, extents %ld, maxio %qx ssd %d\n", filename, ctx.size, 
+                    (extentsData->getLength() / sizeof(IOPolledFileExtent)) - 1,
+                    vars->maxiobytes, kIOPolledFileSSD & vars->flags);
+	assert(!vars->block0);
+	if (extentsData->getLength() < sizeof(IOPolledFileExtent))
+	{
+	    err = kIOReturnNoSpace;
+	    break;
+	}
+
+	vars->fileSize = ctx.size;
+	vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
+
+        part = IOCopyMediaForDev(image_dev);
+        if (!part)
+        {
+            err = kIOReturnNotFound;
+            break;
+	}
+
+	if (!(vars->pollers = IOPolledFilePollers::copyPollers(part))) break;
+
+	if ((num = OSDynamicCast(OSNumber, part->getProperty(kIOMediaPreferredBlockSizeKey))))
+               vars->blockSize = num->unsigned32BitValue();
+	if (vars->blockSize < 4096) vars->blockSize = 4096;
+
+        HIBLOG("polled file major %d, minor %d, blocksize %ld, pollers %d\n",
+               major(image_dev), minor(image_dev), (long)vars->blockSize, 
+               vars->pollers->pollers->getCount());
+
+        OSString * keyUUID = NULL;
+        if (volumeCryptKey)
+        {
+            err = IOGetVolumeCryptKey(block_dev, &keyUUID, volumeCryptKey, keySize);
+        }
+
+	*fileVars    = vars;
+	vars->fileExtents = extentsData;
+    
+	// make imagePath
+	OSData * data;
+	if (imagePath)
+	{
+#if defined(__i386__) || defined(__x86_64__)
+	    char str2[24 + sizeof(uuid_string_t) + 2];
+
+            if (keyUUID)
+                snprintf(str2, sizeof(str2), "%qx:%s", 
+                                vars->extentMap[0].start, keyUUID->getCStringNoCopy());
+            else
+                snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
+
+	    err = IOService::getPlatform()->callPlatformFunction(
+						gIOCreateEFIDevicePathSymbol, false,
+						(void *) part, (void *) str2,
+						(void *) (uintptr_t) true, (void *) &data);
+#else
+	    data = 0;
+	    err = kIOReturnSuccess;
+#endif
+	    if (kIOReturnSuccess != err)
+	    {
+		HIBLOG("error 0x%x getting path\n", err);
+		break;
+	    }
+	    *imagePath = data;
+	}
+    }
+    while (false);
+
+    if (kIOReturnSuccess != err)
+    {
+        HIBLOG("error 0x%x opening polled file\n", err);
+    	IOPolledFileClose(&vars, 0, 0, 0, 0, 0);
+    }
+
+    if (part) part->release();
 
-  return err;
+    return (err);
 }
-#endif /* !HIBERNATION */
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+IOReturn
+IOPolledFileClose(IOPolledFileIOVars ** pVars,
+		  off_t write_offset, void * addr, size_t write_length,
+		  off_t discard_offset, off_t discard_end)
+{
+    IOPolledFileIOVars * vars;
+
+    vars = *pVars;
+    if (!vars) return(kIOReturnSuccess);
+
+    if (vars->fileRef)
+    {
+	kern_close_file_for_direct_io(vars->fileRef, write_offset, addr, write_length, 
+				      discard_offset, discard_end);
+	vars->fileRef = NULL;
+    }
+    if (vars->fileExtents) 
+    {
+    	vars->fileExtents->release();
+    	vars->fileExtents = 0;
+    }
+    if (vars->pollers) 
+    {
+    	vars->pollers->release();
+    	vars->pollers = 0;
+    }
+
+    if (vars->allocated) IODelete(vars, IOPolledFileIOVars, 1);
+    else                 bzero(vars, sizeof(IOPolledFileIOVars));
+    *pVars = NULL;
+
+    return (kIOReturnSuccess);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+IOReturn
+IOPolledFilePollersSetup(IOPolledFileIOVars * vars,
+		         uint32_t openState)
+{
+    IOReturn err;
+
+    err = kIOReturnSuccess;
+    do
+    {
+        if (!vars->pollers->openCount)
+        {
+	    err = IOPolledFilePollersProbe(vars->pollers);
+	    if (kIOReturnSuccess != err) break;
+	    err = IOPolledFilePollersOpen(vars, openState, false);
+	    if (kIOReturnSuccess != err) break;
+	    vars->pollers->openState = openState;
+	}
+	vars->pollers->openCount++;
+	vars->pollers->io  = false;
+	vars->buffer       = (uint8_t *) vars->pollers->ioBuffer->getBytesNoCopy();
+	vars->bufferHalf   = 0;
+	vars->bufferOffset = 0;
+	vars->bufferSize   = (vars->pollers->ioBuffer->getLength() >> 1);
+
+        if (vars->maxiobytes < vars->bufferSize) vars->bufferSize = vars->maxiobytes;
+    }
+    while (false);
+
+    if (kIOReturnSuccess != err) HIBLOG("IOPolledFilePollersSetup(%d) error 0x%x\n", openState, err);
+
+    return (err);
+}
+
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+IOReturn
+IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
+{
+    IOPolledFileExtent * extentMap;
+
+    extentMap = vars->extentMap;
+
+    vars->position = position;
+
+    while (position >= extentMap->length)
+    {
+	position -= extentMap->length;
+	extentMap++;
+    }
+
+    vars->currentExtent   = extentMap;
+    vars->extentRemaining = extentMap->length - position;
+    vars->extentPosition  = vars->position - position;
+
+    if (vars->bufferSize <= vars->extentRemaining)
+	vars->bufferLimit = vars->bufferSize;
+    else
+	vars->bufferLimit = vars->extentRemaining;
+
+    return (kIOReturnSuccess);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+IOReturn
+IOPolledFileWrite(IOPolledFileIOVars * vars,
+                    const uint8_t * bytes, IOByteCount size,
+                    IOPolledFileCryptVars * cryptvars)
+{
+    IOReturn    err = kIOReturnSuccess;
+    IOByteCount copy;
+    bool	flush = false;
+
+    do
+    {
+	if (!bytes && !size)
+	{
+	    // seek to end of block & flush
+	    size = vars->position & (vars->blockSize - 1);
+	    if (size)
+		size = vars->blockSize - size;
+	    flush = true;
+            // use some garbage for the fill
+            bytes = vars->buffer + vars->bufferOffset;
+	}
+
+	copy = vars->bufferLimit - vars->bufferOffset;
+	if (copy > size)
+	    copy = size;
+	else
+	    flush = true;
+
+	if (bytes)
+	{
+	    bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
+	    bytes += copy;
+	}
+        else
+	    bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
+        
+	size -= copy;
+	vars->bufferOffset += copy;
+	vars->position += copy;
+
+	if (flush && vars->bufferOffset)
+	{
+	    uint64_t offset = (vars->position - vars->bufferOffset 
+				- vars->extentPosition + vars->currentExtent->start);
+	    uint32_t length = (vars->bufferOffset);
+
+#if CRYPTO
+            if (cryptvars && vars->encryptStart
+                && (vars->position > vars->encryptStart)
+                && ((vars->position - length) < vars->encryptEnd))
+            {
+                AbsoluteTime startTime, endTime;
+
+                uint64_t encryptLen, encryptStart;
+                encryptLen = vars->position - vars->encryptStart;
+                if (encryptLen > length)
+                    encryptLen = length;
+                encryptStart = length - encryptLen;
+                if (vars->position > vars->encryptEnd)
+                    encryptLen -= (vars->position - vars->encryptEnd);
+
+                clock_get_uptime(&startTime);
+
+                // encrypt the buffer
+                aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
+                                &cryptvars->aes_iv[0],
+                                encryptLen / AES_BLOCK_SIZE,
+                                vars->buffer + vars->bufferHalf + encryptStart,
+                                &cryptvars->ctx.encrypt);
+    
+                clock_get_uptime(&endTime);
+                ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
+                SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
+                vars->cryptBytes += encryptLen;
+
+                // save initial vector for following encrypts
+                bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
+                        &cryptvars->aes_iv[0],
+                        AES_BLOCK_SIZE);
+            }
+#endif /* CRYPTO */
+
+	    err = IOPolledFilePollersIODone(vars->pollers, true);
+	    if (kIOReturnSuccess != err)
+		break;
+
+if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
+//if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
+
+	    err = IOStartPolledIO(vars->pollers, kIOPolledWrite, vars->bufferHalf, offset, length);
+            if (kIOReturnSuccess != err)
+                break;
+	    vars->pollers->io = true;
+
+	    vars->extentRemaining -= vars->bufferOffset;
+	    if (!vars->extentRemaining)
+	    {
+		vars->currentExtent++;
+		vars->extentRemaining = vars->currentExtent->length;
+		vars->extentPosition  = vars->position;
+	    }
+
+	    vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
+	    vars->bufferOffset = 0;
+	    if (vars->bufferSize <= vars->extentRemaining)
+		vars->bufferLimit = vars->bufferSize;
+	    else
+		vars->bufferLimit = vars->extentRemaining;
+
+	    if (!vars->extentRemaining)
+	    {
+		err = kIOReturnOverrun;
+		break;
+	    }
+
+	    flush = false;
+	}
+    }
+    while (size);
+
+    return (err);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+IOReturn
+IOPolledFileRead(IOPolledFileIOVars * vars,
+                    uint8_t * bytes, IOByteCount size,
+                    IOPolledFileCryptVars * cryptvars)
+{
+    IOReturn    err = kIOReturnSuccess;
+    IOByteCount copy;
+
+//    bytesWritten += size;
+
+    do
+    {
+	copy = vars->bufferLimit - vars->bufferOffset;
+	if (copy > size)
+	    copy = size;
+
+	if (bytes)
+	{
+	    bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
+	    bytes += copy;
+	}
+	size -= copy;
+	vars->bufferOffset += copy;
+//	vars->position += copy;
+
+	if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd))
+	{
+	    if (!vars->pollers->io) cryptvars = 0;
+	    err = IOPolledFilePollersIODone(vars->pollers, true);
+	    if (kIOReturnSuccess != err)
+		break;
+
+if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
+
+	    vars->position        += vars->lastRead;
+	    vars->extentRemaining -= vars->lastRead;
+	    vars->bufferLimit      = vars->lastRead;
+
+	    if (!vars->extentRemaining)
+	    {
+		vars->currentExtent++;
+		vars->extentRemaining = vars->currentExtent->length;
+		vars->extentPosition  = vars->position;
+                if (!vars->extentRemaining)
+                {
+                    err = kIOReturnOverrun;
+                    break;
+                }
+	    }
+
+	    uint64_t length;
+	    uint64_t lastReadLength = vars->lastRead;
+	    uint64_t offset = (vars->position 
+				- vars->extentPosition + vars->currentExtent->start);
+	    if (vars->extentRemaining <= vars->bufferSize)
+		length = vars->extentRemaining;
+	    else
+		length = vars->bufferSize;
+	    if ((length + vars->position) > vars->readEnd)
+	    	length = vars->readEnd - vars->position;
+
+	    vars->lastRead = length;
+	    if (length)
+	    {
+//if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
+		err = IOStartPolledIO(vars->pollers, kIOPolledRead, vars->bufferHalf, offset, length);
+		if (kIOReturnSuccess != err)
+		    break;
+		vars->pollers->io = true;
+	    }
+
+	    vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
+	    vars->bufferOffset = 0;
+
+#if CRYPTO
+            if (cryptvars)
+            {
+                uint8_t thisVector[AES_BLOCK_SIZE];
+                AbsoluteTime startTime, endTime;
+
+                // save initial vector for following decrypts
+                bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
+                bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE, 
+                        &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
+
+                // decrypt the buffer
+                clock_get_uptime(&startTime);
+
+                aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
+                                &thisVector[0],
+                                lastReadLength / AES_BLOCK_SIZE,
+                                vars->buffer + vars->bufferHalf,
+                                &cryptvars->ctx.decrypt);
+
+                clock_get_uptime(&endTime);
+                ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
+                SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
+                vars->cryptBytes += lastReadLength;
+            }
+#endif /* CRYPTO */
+	}
+    }
+    while (size);
+
+    return (err);
+}
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+