iokit/Kernel/IOPolledInterface.cpp

   1 /*
   2  * Copyright (c) 2006-2009 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <sys/uio.h>
  30 #include <sys/conf.h>
  31
  32 #include <IOKit/IOLib.h>
  33 #include <IOKit/IOBSD.h>
  34 #include <IOKit/IOService.h>
  35 #include <IOKit/IOPlatformExpert.h>
  36 #include <IOKit/IOPolledInterface.h>
  37 #include <IOKit/IOHibernatePrivate.h>
  38 #include <IOKit/IOBufferMemoryDescriptor.h>
  39 #include <IOKit/AppleKeyStoreInterface.h>
  40 #include "IOKitKernelInternal.h"
  41
  42
  43 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  44
  45 OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject);
  46
  47 OSMetaClassDefineReservedUnused(IOPolledInterface, 0);
  48 OSMetaClassDefineReservedUnused(IOPolledInterface, 1);
  49 OSMetaClassDefineReservedUnused(IOPolledInterface, 2);
  50 OSMetaClassDefineReservedUnused(IOPolledInterface, 3);
  51 OSMetaClassDefineReservedUnused(IOPolledInterface, 4);
  52 OSMetaClassDefineReservedUnused(IOPolledInterface, 5);
  53 OSMetaClassDefineReservedUnused(IOPolledInterface, 6);
  54 OSMetaClassDefineReservedUnused(IOPolledInterface, 7);
  55 OSMetaClassDefineReservedUnused(IOPolledInterface, 8);
  56 OSMetaClassDefineReservedUnused(IOPolledInterface, 9);
  57 OSMetaClassDefineReservedUnused(IOPolledInterface, 10);
  58 OSMetaClassDefineReservedUnused(IOPolledInterface, 11);
  59 OSMetaClassDefineReservedUnused(IOPolledInterface, 12);
  60 OSMetaClassDefineReservedUnused(IOPolledInterface, 13);
  61 OSMetaClassDefineReservedUnused(IOPolledInterface, 14);
  62 OSMetaClassDefineReservedUnused(IOPolledInterface, 15);
  63
  64 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  65
  66 #ifndef kIOMediaPreferredBlockSizeKey
  67 #define kIOMediaPreferredBlockSizeKey   "Preferred Block Size"
  68 #endif
  69
  70 enum { kDefaultIOSize = 128*1024 };
  71
  72 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  73
  74 class IOPolledFilePollers : public OSObject
  75 {
  76     OSDeclareDefaultStructors(IOPolledFilePollers)
  77
  78 public:
  79     IOService                * media;
  80     OSArray                  * pollers;
  81     IOBufferMemoryDescriptor * ioBuffer;
  82     bool                 abortable;
  83     bool                 io;
  84     IOReturn             ioStatus;
  85     uint32_t             openCount;
  86
  87     static IOPolledFilePollers * copyPollers(IOService * media);
  88 };
  89
  90 OSDefineMetaClassAndStructors(IOPolledFilePollers, OSObject)
  91
  92 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  93
  94 IOPolledFilePollers *
  95 IOPolledFilePollers::copyPollers(IOService * media)
  96 {
  97     IOPolledFilePollers * vars;
  98     IOReturn              err;
  99     IOService       * service;
 100     OSObject        * obj;
 101     IORegistryEntry * next;
 102     IORegistryEntry * child;
 103
 104     if ((obj = media->copyProperty(kIOPolledInterfaceStackKey)))
 105     {
 106         return (OSDynamicCast(IOPolledFilePollers, obj));
 107     }
 108
 109     do
 110     {
 111         vars = OSTypeAlloc(IOPolledFilePollers);
 112         vars->init();
 113
 114         vars->pollers = OSArray::withCapacity(4);
 115         if (!vars->pollers)
 116         {
 117             err = kIOReturnNoMemory;
 118             break;
 119         }
 120
 121         next = vars->media = media;
 122         do
 123         {
 124             IOPolledInterface * poller;
 125             OSObject *          obj;
 126
 127             obj = next->getProperty(kIOPolledInterfaceSupportKey);
 128             if (kOSBooleanFalse == obj)
 129             {
 130                 vars->pollers->flushCollection();
 131                 break;
 132             }
 133             else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
 134                 vars->pollers->setObject(poller);
 135
 136             if ((service = OSDynamicCast(IOService, next))
 137                 && service->getDeviceMemory()
 138                 && !vars->pollers->getCount())  break;
 139
 140             child = next;
 141         }
 142         while ((next = child->getParentEntry(gIOServicePlane))
 143                && child->isParent(next, gIOServicePlane, true));
 144
 145         if (!vars->pollers->getCount())
 146         {
 147             err = kIOReturnUnsupported;
 148             break;
 149         }
 150     }
 151     while (false);
 152
 153     media->setProperty(kIOPolledInterfaceStackKey, vars);
 154
 155     return (vars);
 156 }
 157
 158 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 159
 160 static IOReturn
 161 IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable);
 162
 163 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 164
 165 static IOReturn
 166 IOPolledFilePollersProbe(IOPolledFilePollers * vars)
 167 {
 168     IOReturn            err = kIOReturnError;
 169     int32_t             idx;
 170     IOPolledInterface * poller;
 171
 172     for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
 173     {
 174         poller = (IOPolledInterface *) vars->pollers->getObject(idx);
 175         err = poller->probe(vars->media);
 176         if (err)
 177         {
 178             HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
 179             break;
 180         }
 181     }
 182
 183     return (err);
 184 }
 185
 186 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 187
 188 IOReturn
 189 IOPolledFilePollersOpen(IOPolledFileIOVars * filevars, uint32_t state, bool abortable)
 190 {
 191
 192     IOPolledFilePollers      * vars = filevars->pollers;
 193     IOBufferMemoryDescriptor * ioBuffer;
 194     IOPolledInterface        * poller;
 195     IOService                * next;
 196     IOReturn                   err = kIOReturnError;
 197     int32_t                    idx;
 198
 199     vars->abortable = abortable;
 200     ioBuffer = 0;
 201
 202     if (kIOPolledAfterSleepState == state)
 203     {
 204         vars->ioStatus = 0;
 205         vars->io = false;
 206     }
 207     (void) IOPolledFilePollersIODone(vars, false);
 208
 209     if ((kIOPolledPreflightState == state) || (kIOPolledPreflightCoreDumpState == state))
 210     {
 211         ioBuffer = vars->ioBuffer;
 212         if (!ioBuffer)
 213         {
 214             vars->ioBuffer = ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionInOut,
 215                                                             2 * kDefaultIOSize, page_size);
 216             if (!ioBuffer) return (kIOReturnNoMemory);
 217         }
 218     }
 219
 220     for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
 221     {
 222         poller = (IOPolledInterface *) vars->pollers->getObject(idx);
 223         err = poller->open(state, ioBuffer);
 224         if (kIOReturnSuccess != err)
 225         {
 226             HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
 227             break;
 228         }
 229     }
 230     if ((kIOReturnSuccess == err) && (kIOPolledPreflightState == state))
 231     {
 232         next = vars->media;
 233         while (next)
 234         {
 235             next->setProperty(kIOPolledInterfaceActiveKey, kOSBooleanTrue);
 236             next = next->getProvider();
 237         }
 238     }
 239
 240     return (err);
 241 }
 242
 243 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 244
 245 IOReturn
 246 IOPolledFilePollersClose(IOPolledFileIOVars * filevars, uint32_t state)
 247 {
 248     IOPolledFilePollers * vars = filevars->pollers;
 249     IOPolledInterface * poller;
 250     IORegistryEntry *   next;
 251     IOReturn            err;
 252     int32_t             idx;
 253
 254     (void) IOPolledFilePollersIODone(vars, false);
 255
 256     if ((kIOPolledPostflightState == state) || (kIOPolledPostflightCoreDumpState == state))
 257     {
 258         vars->openCount--;
 259     }
 260
 261     for (idx = 0, err = kIOReturnSuccess;
 262          (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
 263          idx++)
 264     {
 265         err = poller->close(state);
 266         if (err) HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
 267     }
 268
 269     if (kIOPolledPostflightState == state)
 270     {
 271         next = vars->media;
 272         while (next)
 273         {
 274             next->removeProperty(kIOPolledInterfaceActiveKey);
 275             next = next->getParentEntry(gIOServicePlane);
 276         }
 277     }
 278
 279     if ((kIOPolledPostflightState == state) || (kIOPolledPostflightCoreDumpState == state)) do
 280     {
 281         if (vars->openCount) break;
 282         if (vars->ioBuffer)
 283         {
 284             vars->ioBuffer->release();
 285             vars->ioBuffer = 0;
 286         }
 287     }
 288     while (false);
 289
 290     return (err);
 291 }
 292 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 293
 294 IOMemoryDescriptor *
 295 IOPolledFileGetIOBuffer(IOPolledFileIOVars * vars)
 296 {
 297     return (vars->pollers->ioBuffer);
 298 }
 299
 300 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 301
 302 static void
 303 IOPolledIOComplete(void *   target,
 304                    void *   parameter,
 305                    IOReturn status,
 306                    UInt64   actualByteCount)
 307 {
 308     IOPolledFilePollers * vars = (IOPolledFilePollers *) parameter;
 309
 310     vars->ioStatus = status;
 311 }
 312
 313 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 314
 315 static IOReturn
 316 IOStartPolledIO(IOPolledFilePollers * vars,
 317                     uint32_t operation, uint32_t bufferOffset,
 318                     uint64_t deviceOffset, uint64_t length)
 319 {
 320     IOReturn            err;
 321     IOPolledInterface * poller;
 322     IOPolledCompletion  completion;
 323
 324     err = vars->ioStatus;
 325     if (kIOReturnSuccess != err) return (err);
 326
 327     completion.target    = 0;
 328     completion.action    = &IOPolledIOComplete;
 329     completion.parameter = vars;
 330
 331     vars->ioStatus = -1;
 332
 333     poller = (IOPolledInterface *) vars->pollers->getObject(0);
 334     err = poller->startIO(operation, bufferOffset, deviceOffset, length, completion);
 335     if (err)
 336         HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
 337
 338     return (err);
 339 }
 340
 341 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 342
 343 static IOReturn
 344 IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable)
 345 {
 346     IOReturn            err = kIOReturnSuccess;
 347     int32_t             idx = 0;
 348     IOPolledInterface * poller;
 349     AbsoluteTime        deadline;
 350
 351     if (!vars->io) return (kIOReturnSuccess);
 352
 353     abortable &= vars->abortable;
 354
 355     clock_interval_to_deadline(2000, kMillisecondScale, &deadline);
 356
 357     while (-1 == vars->ioStatus)
 358     {
 359         for (idx = 0;
 360             (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
 361              idx++)
 362         {
 363             IOReturn newErr;
 364             newErr = poller->checkForWork();
 365             if ((newErr == kIOReturnAborted) && !abortable)
 366                 newErr = kIOReturnSuccess;
 367             if (kIOReturnSuccess == err)
 368                 err = newErr;
 369         }
 370         if ((false) && (kIOReturnSuccess == err) && (mach_absolute_time() > AbsoluteTime_to_scalar(&deadline)))
 371         {
 372             HIBLOG("IOPolledInterface::forced timeout\n");
 373             vars->ioStatus = kIOReturnTimeout;
 374         }
 375     }
 376     vars->io = false;
 377
 378 #if HIBERNATION
 379     if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort())
 380     {
 381         err = kIOReturnAborted;
 382         HIBLOG("IOPolledInterface::checkForWork sw abort\n");
 383     }
 384 #endif
 385
 386     if (err)
 387     {
 388         HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
 389     }
 390     else
 391     {
 392         err = vars->ioStatus;
 393         if (kIOReturnSuccess != err) HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
 394     }
 395
 396     return (err);
 397 }
 398
 399 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 400 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 401
 402 struct _OpenFileContext
 403 {
 404     OSData * extents;
 405     uint64_t size;
 406 };
 407
 408 static void
 409 file_extent_callback(void * ref, uint64_t start, uint64_t length)
 410 {
 411     _OpenFileContext * ctx = (_OpenFileContext *) ref;
 412     IOPolledFileExtent extent;
 413
 414     extent.start  = start;
 415     extent.length = length;
 416     ctx->extents->appendBytes(&extent, sizeof(extent));
 417     ctx->size += length;
 418 }
 419
 420 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 421
 422 static IOService *
 423 IOCopyMediaForDev(dev_t device)
 424 {
 425     OSDictionary * matching;
 426     OSNumber *     num;
 427     OSIterator *   iter;
 428     IOService *    result = 0;
 429
 430     matching = IOService::serviceMatching("IOMedia");
 431     if (!matching)
 432         return (0);
 433     do
 434     {
 435         num = OSNumber::withNumber(major(device), 32);
 436         if (!num)
 437             break;
 438         matching->setObject(kIOBSDMajorKey, num);
 439         num->release();
 440         num = OSNumber::withNumber(minor(device), 32);
 441         if (!num)
 442             break;
 443         matching->setObject(kIOBSDMinorKey, num);
 444         num->release();
 445         if (!num)
 446             break;
 447         iter = IOService::getMatchingServices(matching);
 448         if (iter)
 449         {
 450             result = (IOService *) iter->getNextObject();
 451             result->retain();
 452             iter->release();
 453         }
 454     }
 455     while (false);
 456     matching->release();
 457
 458     return (result);
 459 }
 460
 461 static IOReturn
 462 IOGetVolumeCryptKey(dev_t block_dev,  OSString ** pKeyUUID,
 463                     uint8_t * volumeCryptKey, size_t keySize)
 464 {
 465     IOReturn         err;
 466     IOService *      part;
 467     OSString *       keyUUID = 0;
 468     OSString *       keyStoreUUID = 0;
 469     uuid_t           volumeKeyUUID;
 470     aks_volume_key_t vek;
 471
 472     static IOService * sKeyStore;
 473
 474     part = IOCopyMediaForDev(block_dev);
 475     if (!part) return (kIOReturnNotFound);
 476
 477     err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false,
 478                                       (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL);
 479     if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID)
 480     {
 481 //            IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy());
 482
 483         if (!sKeyStore)
 484             sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane);
 485         if (sKeyStore)
 486             err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID);
 487         else
 488             err = kIOReturnNoResources;
 489         if (kIOReturnSuccess == err)
 490             err = sKeyStore->callPlatformFunction(gAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL);
 491         if (kIOReturnSuccess != err)
 492             IOLog("volume key err 0x%x\n", err);
 493         else
 494         {
 495             if (vek.key.keybytecount < keySize) keySize = vek.key.keybytecount;
 496             bcopy(&vek.key.keybytes[0], volumeCryptKey, keySize);
 497         }
 498         bzero(&vek, sizeof(vek));
 499
 500     }
 501     part->release();
 502     if (pKeyUUID) *pKeyUUID = keyUUID;
 503
 504     return (err);
 505 }
 506
 507 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 508
 509 IOReturn
 510 IOPolledFileOpen(const char * filename,
 511                  uint64_t setFileSize, uint64_t fsFreeSize,
 512                  void * write_file_addr, size_t write_file_len,
 513                  IOPolledFileIOVars ** fileVars,
 514                  OSData ** imagePath,
 515                  uint8_t * volumeCryptKey, size_t keySize)
 516 {
 517     IOReturn             err = kIOReturnSuccess;
 518     IOPolledFileIOVars * vars;
 519     _OpenFileContext     ctx;
 520     OSData *             extentsData;
 521     OSNumber *           num;
 522     IOService *          part = 0;
 523     dev_t                block_dev;
 524     dev_t                image_dev;
 525     AbsoluteTime         startTime, endTime;
 526     uint64_t             nsec;
 527
 528     vars = IONew(IOPolledFileIOVars, 1);
 529     if (!vars) return (kIOReturnNoMemory);
 530     bzero(vars, sizeof(*vars));
 531     vars->allocated = true;
 532
 533     do
 534     {
 535         extentsData = OSData::withCapacity(32);
 536         ctx.extents = extentsData;
 537         ctx.size    = 0;
 538         clock_get_uptime(&startTime);
 539
 540         vars->fileRef = kern_open_file_for_direct_io(filename,
 541                                                      (write_file_addr != NULL) || (0 != setFileSize),
 542                                                      &file_extent_callback, &ctx,
 543                                                      setFileSize,
 544                                                      fsFreeSize,
 545                                                      // write file:
 546                                                      0, write_file_addr, write_file_len,
 547                                                      // results
 548                                                      &block_dev,
 549                                                      &image_dev,
 550                                                      &vars->block0,
 551                                                      &vars->maxiobytes,
 552                                                      &vars->flags);
 553 #if 0
 554         uint32_t msDelay = (131071 & random());
 555         HIBLOG("sleep %d\n", msDelay);
 556         IOSleep(msDelay);
 557 #endif
 558         clock_get_uptime(&endTime);
 559         SUB_ABSOLUTETIME(&endTime, &startTime);
 560         absolutetime_to_nanoseconds(endTime, &nsec);
 561
 562         if (!vars->fileRef) err = kIOReturnNoSpace;
 563
 564         HIBLOG("kern_open_file_for_direct_io took %qd ms\n", nsec / 1000000ULL);
 565         if (kIOReturnSuccess != err) break;
 566
 567         HIBLOG("Opened file %s, size %qd, extents %ld, maxio %qx ssd %d\n", filename, ctx.size,
 568                     (extentsData->getLength() / sizeof(IOPolledFileExtent)) - 1,
 569                     vars->maxiobytes, kIOPolledFileSSD & vars->flags);
 570         assert(!vars->block0);
 571         if (extentsData->getLength() < sizeof(IOPolledFileExtent))
 572         {
 573             err = kIOReturnNoSpace;
 574             break;
 575         }
 576
 577         vars->fileSize = ctx.size;
 578         vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
 579
 580         part = IOCopyMediaForDev(image_dev);
 581         if (!part)
 582         {
 583             err = kIOReturnNotFound;
 584             break;
 585         }
 586
 587         if (!(vars->pollers = IOPolledFilePollers::copyPollers(part))) break;
 588
 589         if ((num = OSDynamicCast(OSNumber, part->getProperty(kIOMediaPreferredBlockSizeKey))))
 590                vars->blockSize = num->unsigned32BitValue();
 591         if (vars->blockSize < 4096) vars->blockSize = 4096;
 592
 593         HIBLOG("polled file major %d, minor %d, blocksize %ld, pollers %d\n",
 594                major(image_dev), minor(image_dev), (long)vars->blockSize,
 595                vars->pollers->pollers->getCount());
 596
 597         OSString * keyUUID = NULL;
 598         if (volumeCryptKey)
 599         {
 600             err = IOGetVolumeCryptKey(block_dev, &keyUUID, volumeCryptKey, keySize);
 601         }
 602
 603         *fileVars    = vars;
 604         vars->fileExtents = extentsData;
 605
 606         // make imagePath
 607         OSData * data;
 608         if (imagePath)
 609         {
 610 #if defined(__i386__) || defined(__x86_64__)
 611             char str2[24 + sizeof(uuid_string_t) + 2];
 612
 613             if (keyUUID)
 614                 snprintf(str2, sizeof(str2), "%qx:%s",
 615                                 vars->extentMap[0].start, keyUUID->getCStringNoCopy());
 616             else
 617                 snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
 618
 619             err = IOService::getPlatform()->callPlatformFunction(
 620                                                 gIOCreateEFIDevicePathSymbol, false,
 621                                                 (void *) part, (void *) str2,
 622                                                 (void *) (uintptr_t) true, (void *) &data);
 623 #else
 624             data = 0;
 625             err = kIOReturnSuccess;
 626 #endif
 627             if (kIOReturnSuccess != err)
 628             {
 629                 HIBLOG("error 0x%x getting path\n", err);
 630                 break;
 631             }
 632             *imagePath = data;
 633         }
 634     }
 635     while (false);
 636
 637     if (kIOReturnSuccess != err)
 638     {
 639         HIBLOG("error 0x%x opening polled file\n", err);
 640         IOPolledFileClose(&vars, 0, 0, 0, 0, 0);
 641     }
 642
 643     if (part) part->release();
 644
 645     return (err);
 646 }
 647
 648 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 649
 650 IOReturn
 651 IOPolledFileClose(IOPolledFileIOVars ** pVars,
 652                   off_t write_offset, void * addr, size_t write_length,
 653                   off_t discard_offset, off_t discard_end)
 654 {
 655     IOPolledFileIOVars * vars;
 656
 657     vars = *pVars;
 658     if (!vars) return(kIOReturnSuccess);
 659
 660     if (vars->fileRef)
 661     {
 662         kern_close_file_for_direct_io(vars->fileRef, write_offset, addr, write_length,
 663                                       discard_offset, discard_end);
 664         vars->fileRef = NULL;
 665     }
 666     if (vars->fileExtents)
 667     {
 668         vars->fileExtents->release();
 669         vars->fileExtents = 0;
 670     }
 671     if (vars->pollers)
 672     {
 673         vars->pollers->release();
 674         vars->pollers = 0;
 675     }
 676
 677     if (vars->allocated) IODelete(vars, IOPolledFileIOVars, 1);
 678     else                 bzero(vars, sizeof(IOPolledFileIOVars));
 679     *pVars = NULL;
 680
 681     return (kIOReturnSuccess);
 682 }
 683
 684 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 685
 686 IOReturn
 687 IOPolledFilePollersSetup(IOPolledFileIOVars * vars,
 688                          uint32_t openState)
 689 {
 690     IOReturn err;
 691
 692     err = kIOReturnSuccess;
 693     do
 694     {
 695         if (!vars->pollers->openCount)
 696         {
 697             err = IOPolledFilePollersProbe(vars->pollers);
 698             if (kIOReturnSuccess != err) break;
 699         }
 700         err = IOPolledFilePollersOpen(vars, openState, false);
 701         if (kIOReturnSuccess != err) break;
 702         if ((kIOPolledPreflightState == openState) || (kIOPolledPreflightCoreDumpState == openState))
 703         {
 704             vars->pollers->openCount++;
 705         }
 706         vars->pollers->io  = false;
 707         vars->buffer       = (uint8_t *) vars->pollers->ioBuffer->getBytesNoCopy();
 708         vars->bufferHalf   = 0;
 709         vars->bufferOffset = 0;
 710         vars->bufferSize   = (vars->pollers->ioBuffer->getLength() >> 1);
 711
 712         if (vars->maxiobytes < vars->bufferSize) vars->bufferSize = vars->maxiobytes;
 713     }
 714     while (false);
 715
 716     if (kIOReturnSuccess != err) HIBLOG("IOPolledFilePollersSetup(%d) error 0x%x\n", openState, err);
 717
 718     return (err);
 719 }
 720
 721
 722 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 723
 724 IOReturn
 725 IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
 726 {
 727     IOPolledFileExtent * extentMap;
 728
 729     extentMap = vars->extentMap;
 730
 731     vars->position = position;
 732
 733     while (position >= extentMap->length)
 734     {
 735         position -= extentMap->length;
 736         extentMap++;
 737     }
 738
 739     vars->currentExtent   = extentMap;
 740     vars->extentRemaining = extentMap->length - position;
 741     vars->extentPosition  = vars->position - position;
 742
 743     if (vars->bufferSize <= vars->extentRemaining)
 744         vars->bufferLimit = vars->bufferSize;
 745     else
 746         vars->bufferLimit = vars->extentRemaining;
 747
 748     return (kIOReturnSuccess);
 749 }
 750
 751 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 752
 753 IOReturn
 754 IOPolledFileWrite(IOPolledFileIOVars * vars,
 755                     const uint8_t * bytes, IOByteCount size,
 756                     IOPolledFileCryptVars * cryptvars)
 757 {
 758     IOReturn    err = kIOReturnSuccess;
 759     IOByteCount copy;
 760     bool        flush = false;
 761
 762     do
 763     {
 764         if (!bytes && !size)
 765         {
 766             // seek to end of block & flush
 767             size = vars->position & (vars->blockSize - 1);
 768             if (size)
 769                 size = vars->blockSize - size;
 770             flush = true;
 771             // use some garbage for the fill
 772             bytes = vars->buffer + vars->bufferOffset;
 773         }
 774
 775         copy = vars->bufferLimit - vars->bufferOffset;
 776         if (copy > size)
 777             copy = size;
 778         else
 779             flush = true;
 780
 781         if (bytes)
 782         {
 783             bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
 784             bytes += copy;
 785         }
 786         else
 787             bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
 788
 789         size -= copy;
 790         vars->bufferOffset += copy;
 791         vars->position += copy;
 792
 793         if (flush && vars->bufferOffset)
 794         {
 795             uint64_t offset = (vars->position - vars->bufferOffset
 796                                 - vars->extentPosition + vars->currentExtent->start);
 797             uint32_t length = (vars->bufferOffset);
 798
 799 #if CRYPTO
 800             if (cryptvars && vars->encryptStart
 801                 && (vars->position > vars->encryptStart)
 802                 && ((vars->position - length) < vars->encryptEnd))
 803             {
 804                 AbsoluteTime startTime, endTime;
 805
 806                 uint64_t encryptLen, encryptStart;
 807                 encryptLen = vars->position - vars->encryptStart;
 808                 if (encryptLen > length)
 809                     encryptLen = length;
 810                 encryptStart = length - encryptLen;
 811                 if (vars->position > vars->encryptEnd)
 812                     encryptLen -= (vars->position - vars->encryptEnd);
 813
 814                 clock_get_uptime(&startTime);
 815
 816                 // encrypt the buffer
 817                 aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
 818                                 &cryptvars->aes_iv[0],
 819                                 encryptLen / AES_BLOCK_SIZE,
 820                                 vars->buffer + vars->bufferHalf + encryptStart,
 821                                 &cryptvars->ctx.encrypt);
 822
 823                 clock_get_uptime(&endTime);
 824                 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
 825                 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
 826                 vars->cryptBytes += encryptLen;
 827
 828                 // save initial vector for following encrypts
 829                 bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
 830                         &cryptvars->aes_iv[0],
 831                         AES_BLOCK_SIZE);
 832             }
 833 #endif /* CRYPTO */
 834
 835             err = IOPolledFilePollersIODone(vars->pollers, true);
 836             if (kIOReturnSuccess != err)
 837                 break;
 838
 839 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
 840 //if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
 841
 842             err = IOStartPolledIO(vars->pollers, kIOPolledWrite, vars->bufferHalf, offset, length);
 843             if (kIOReturnSuccess != err)
 844                 break;
 845             vars->pollers->io = true;
 846
 847             vars->extentRemaining -= vars->bufferOffset;
 848             if (!vars->extentRemaining)
 849             {
 850                 vars->currentExtent++;
 851                 vars->extentRemaining = vars->currentExtent->length;
 852                 vars->extentPosition  = vars->position;
 853             }
 854
 855             vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
 856             vars->bufferOffset = 0;
 857             if (vars->bufferSize <= vars->extentRemaining)
 858                 vars->bufferLimit = vars->bufferSize;
 859             else
 860                 vars->bufferLimit = vars->extentRemaining;
 861
 862             if (!vars->extentRemaining)
 863             {
 864                 err = kIOReturnOverrun;
 865                 break;
 866             }
 867
 868             flush = false;
 869         }
 870     }
 871     while (size);
 872
 873     return (err);
 874 }
 875
 876 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 877
 878 IOReturn
 879 IOPolledFileRead(IOPolledFileIOVars * vars,
 880                     uint8_t * bytes, IOByteCount size,
 881                     IOPolledFileCryptVars * cryptvars)
 882 {
 883     IOReturn    err = kIOReturnSuccess;
 884     IOByteCount copy;
 885
 886 //    bytesWritten += size;
 887
 888     do
 889     {
 890         copy = vars->bufferLimit - vars->bufferOffset;
 891         if (copy > size)
 892             copy = size;
 893
 894         if (bytes)
 895         {
 896             bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
 897             bytes += copy;
 898         }
 899         size -= copy;
 900         vars->bufferOffset += copy;
 901 //      vars->position += copy;
 902
 903         if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd))
 904         {
 905             if (!vars->pollers->io) cryptvars = 0;
 906             err = IOPolledFilePollersIODone(vars->pollers, true);
 907             if (kIOReturnSuccess != err)
 908                 break;
 909
 910 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
 911
 912             vars->position        += vars->lastRead;
 913             vars->extentRemaining -= vars->lastRead;
 914             vars->bufferLimit      = vars->lastRead;
 915
 916             if (!vars->extentRemaining)
 917             {
 918                 vars->currentExtent++;
 919                 vars->extentRemaining = vars->currentExtent->length;
 920                 vars->extentPosition  = vars->position;
 921                 if (!vars->extentRemaining)
 922                 {
 923                     err = kIOReturnOverrun;
 924                     break;
 925                 }
 926             }
 927
 928             uint64_t length;
 929             uint64_t lastReadLength = vars->lastRead;
 930             uint64_t offset = (vars->position
 931                                 - vars->extentPosition + vars->currentExtent->start);
 932             if (vars->extentRemaining <= vars->bufferSize)
 933                 length = vars->extentRemaining;
 934             else
 935                 length = vars->bufferSize;
 936             if ((length + vars->position) > vars->readEnd)
 937                 length = vars->readEnd - vars->position;
 938
 939             vars->lastRead = length;
 940             if (length)
 941             {
 942 //if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
 943                 err = IOStartPolledIO(vars->pollers, kIOPolledRead, vars->bufferHalf, offset, length);
 944                 if (kIOReturnSuccess != err)
 945                     break;
 946                 vars->pollers->io = true;
 947             }
 948
 949             vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
 950             vars->bufferOffset = 0;
 951
 952 #if CRYPTO
 953             if (cryptvars)
 954             {
 955                 uint8_t thisVector[AES_BLOCK_SIZE];
 956                 AbsoluteTime startTime, endTime;
 957
 958                 // save initial vector for following decrypts
 959                 bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
 960                 bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE,
 961                         &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
 962
 963                 // decrypt the buffer
 964                 clock_get_uptime(&startTime);
 965
 966                 aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
 967                                 &thisVector[0],
 968                                 lastReadLength / AES_BLOCK_SIZE,
 969                                 vars->buffer + vars->bufferHalf,
 970                                 &cryptvars->ctx.decrypt);
 971
 972                 clock_get_uptime(&endTime);
 973                 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
 974                 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
 975                 vars->cryptBytes += lastReadLength;
 976             }
 977 #endif /* CRYPTO */
 978         }
 979     }
 980     while (size);
 981
 982     return (err);
 983 }
 984
 985 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 986