iokit/Kernel/IOPolledInterface.cpp

   1 /*
   2  * Copyright (c) 2006-2009 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <sys/uio.h>
  30 #include <sys/conf.h>
  31
  32 #include <IOKit/IOLib.h>
  33 #include <IOKit/IOBSD.h>
  34 #include <IOKit/IOService.h>
  35 #include <IOKit/IOPlatformExpert.h>
  36 #include <IOKit/IOPolledInterface.h>
  37 #include <IOKit/IOHibernatePrivate.h>
  38 #include <IOKit/IOBufferMemoryDescriptor.h>
  39 #include <IOKit/AppleKeyStoreInterface.h>
  40 #include "IOKitKernelInternal.h"
  41
  42
  43 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  44
  45 OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject);
  46
  47 OSMetaClassDefineReservedUnused(IOPolledInterface, 0);
  48 OSMetaClassDefineReservedUnused(IOPolledInterface, 1);
  49 OSMetaClassDefineReservedUnused(IOPolledInterface, 2);
  50 OSMetaClassDefineReservedUnused(IOPolledInterface, 3);
  51 OSMetaClassDefineReservedUnused(IOPolledInterface, 4);
  52 OSMetaClassDefineReservedUnused(IOPolledInterface, 5);
  53 OSMetaClassDefineReservedUnused(IOPolledInterface, 6);
  54 OSMetaClassDefineReservedUnused(IOPolledInterface, 7);
  55 OSMetaClassDefineReservedUnused(IOPolledInterface, 8);
  56 OSMetaClassDefineReservedUnused(IOPolledInterface, 9);
  57 OSMetaClassDefineReservedUnused(IOPolledInterface, 10);
  58 OSMetaClassDefineReservedUnused(IOPolledInterface, 11);
  59 OSMetaClassDefineReservedUnused(IOPolledInterface, 12);
  60 OSMetaClassDefineReservedUnused(IOPolledInterface, 13);
  61 OSMetaClassDefineReservedUnused(IOPolledInterface, 14);
  62 OSMetaClassDefineReservedUnused(IOPolledInterface, 15);
  63
  64 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  65
  66 #ifndef kIOMediaPreferredBlockSizeKey
  67 #define kIOMediaPreferredBlockSizeKey   "Preferred Block Size"
  68 #endif
  69
  70 enum { kDefaultIOSize = 128*1024 };
  71
  72 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  73
  74 class IOPolledFilePollers : public OSObject
  75 {
  76     OSDeclareDefaultStructors(IOPolledFilePollers)
  77
  78 public:
  79     IOService                * media;
  80     OSArray                  * pollers;
  81     IOBufferMemoryDescriptor * ioBuffer;
  82     bool                 abortable;
  83     bool                 io;
  84     IOReturn             ioStatus;
  85     uint32_t             openCount;
  86     uint32_t             openState;
  87
  88     static IOPolledFilePollers * copyPollers(IOService * media);
  89 };
  90
  91 OSDefineMetaClassAndStructors(IOPolledFilePollers, OSObject)
  92
  93 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  94
  95 IOPolledFilePollers *
  96 IOPolledFilePollers::copyPollers(IOService * media)
  97 {
  98     IOPolledFilePollers * vars;
  99     IOReturn              err;
 100     IOService       * service;
 101     OSObject        * obj;
 102     IORegistryEntry * next;
 103     IORegistryEntry * child;
 104
 105     if ((obj = media->copyProperty(kIOPolledInterfaceStackKey)))
 106     {
 107         return (OSDynamicCast(IOPolledFilePollers, obj));
 108     }
 109
 110     do
 111     {
 112         vars = OSTypeAlloc(IOPolledFilePollers);
 113         vars->init();
 114
 115         vars->pollers = OSArray::withCapacity(4);
 116         if (!vars->pollers)
 117         {
 118             err = kIOReturnNoMemory;
 119             break;
 120         }
 121
 122         next = vars->media = media;
 123         do
 124         {
 125             IOPolledInterface * poller;
 126             OSObject *          obj;
 127
 128             obj = next->getProperty(kIOPolledInterfaceSupportKey);
 129             if (kOSBooleanFalse == obj)
 130             {
 131                 vars->pollers->flushCollection();
 132                 break;
 133             }
 134             else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
 135                 vars->pollers->setObject(poller);
 136
 137             if ((service = OSDynamicCast(IOService, next))
 138                 && service->getDeviceMemory()
 139                 && !vars->pollers->getCount())  break;
 140
 141             child = next;
 142         }
 143         while ((next = child->getParentEntry(gIOServicePlane))
 144                && child->isParent(next, gIOServicePlane, true));
 145
 146         if (!vars->pollers->getCount())
 147         {
 148             err = kIOReturnUnsupported;
 149             break;
 150         }
 151     }
 152     while (false);
 153
 154     media->setProperty(kIOPolledInterfaceStackKey, vars);
 155
 156     return (vars);
 157 }
 158
 159 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 160
 161 static IOReturn
 162 IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable);
 163
 164 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 165
 166 static IOReturn
 167 IOPolledFilePollersProbe(IOPolledFilePollers * vars)
 168 {
 169     IOReturn            err = kIOReturnError;
 170     int32_t             idx;
 171     IOPolledInterface * poller;
 172
 173     for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
 174     {
 175         poller = (IOPolledInterface *) vars->pollers->getObject(idx);
 176         err = poller->probe(vars->media);
 177         if (err)
 178         {
 179             HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
 180             break;
 181         }
 182     }
 183
 184     return (err);
 185 }
 186
 187 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 188
 189 IOReturn
 190 IOPolledFilePollersOpen(IOPolledFileIOVars * filevars, uint32_t state, bool abortable)
 191 {
 192
 193     IOPolledFilePollers      * vars = filevars->pollers;
 194     IOBufferMemoryDescriptor * ioBuffer;
 195     IOPolledInterface        * poller;
 196     IOService                * next;
 197     IOReturn                   err = kIOReturnError;
 198     int32_t                    idx;
 199
 200     vars->abortable = abortable;
 201     ioBuffer = 0;
 202
 203     if (kIOPolledAfterSleepState == state)
 204     {
 205         vars->ioStatus = 0;
 206         vars->io = false;
 207     }
 208     (void) IOPolledFilePollersIODone(vars, false);
 209
 210     if ((kIOPolledPreflightState == state) || (kIOPolledPreflightCoreDumpState == state))
 211     {
 212         ioBuffer = vars->ioBuffer;
 213         if (!ioBuffer)
 214         {
 215             vars->ioBuffer = ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionInOut,
 216                                                             2 * kDefaultIOSize, page_size);
 217             if (!ioBuffer) return (kIOReturnNoMemory);
 218         }
 219     }
 220
 221     for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
 222     {
 223         poller = (IOPolledInterface *) vars->pollers->getObject(idx);
 224         err = poller->open(state, ioBuffer);
 225         if ((kIOReturnSuccess != err) && (kIOPolledPreflightCoreDumpState == state))
 226         {
 227             err = poller->open(kIOPolledPreflightState, ioBuffer);
 228         }
 229         if (kIOReturnSuccess != err)
 230         {
 231             HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
 232             break;
 233         }
 234     }
 235     if (kIOReturnSuccess == err)
 236     {
 237         next = vars->media;
 238         while (next)
 239         {
 240             next->setProperty(kIOPolledInterfaceActiveKey, kOSBooleanTrue);
 241             next = next->getProvider();
 242         }
 243     }
 244
 245     return (err);
 246 }
 247
 248 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 249
 250 IOReturn
 251 IOPolledFilePollersClose(IOPolledFileIOVars * filevars, uint32_t state)
 252 {
 253     IOPolledFilePollers * vars = filevars->pollers;
 254     IOPolledInterface * poller;
 255     IORegistryEntry *   next;
 256     IOReturn            err;
 257     int32_t             idx;
 258
 259     (void) IOPolledFilePollersIODone(vars, false);
 260
 261     if (kIOPolledPostflightState == state)
 262     {
 263         vars->openCount--;
 264         if (vars->openCount)
 265         {
 266             // 21207427
 267             IOPolledFilePollersOpen(filevars, vars->openState, vars->abortable);
 268             return (kIOReturnSuccess);
 269         }
 270     }
 271
 272     for (idx = 0, err = kIOReturnSuccess;
 273          (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
 274          idx++)
 275     {
 276         err = poller->close(state);
 277         if (err) HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
 278     }
 279
 280     if (kIOPolledPostflightState == state)
 281     {
 282         next = vars->media;
 283         while (next)
 284         {
 285             next->removeProperty(kIOPolledInterfaceActiveKey);
 286             next = next->getParentEntry(gIOServicePlane);
 287         }
 288
 289         if (vars->ioBuffer)
 290         {
 291             vars->ioBuffer->release();
 292             vars->ioBuffer = 0;
 293         }
 294     }
 295     return (err);
 296 }
 297 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 298
 299 IOMemoryDescriptor *
 300 IOPolledFileGetIOBuffer(IOPolledFileIOVars * vars)
 301 {
 302     return (vars->pollers->ioBuffer);
 303 }
 304
 305 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 306
 307 static void
 308 IOPolledIOComplete(void *   target,
 309                    void *   parameter,
 310                    IOReturn status,
 311                    UInt64   actualByteCount)
 312 {
 313     IOPolledFilePollers * vars = (IOPolledFilePollers *) parameter;
 314
 315     vars->ioStatus = status;
 316 }
 317
 318 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 319
 320 static IOReturn
 321 IOStartPolledIO(IOPolledFilePollers * vars,
 322                     uint32_t operation, uint32_t bufferOffset,
 323                     uint64_t deviceOffset, uint64_t length)
 324 {
 325     IOReturn            err;
 326     IOPolledInterface * poller;
 327     IOPolledCompletion  completion;
 328
 329     err = vars->ioStatus;
 330     if (kIOReturnSuccess != err) return (err);
 331
 332     completion.target    = 0;
 333     completion.action    = &IOPolledIOComplete;
 334     completion.parameter = vars;
 335
 336     vars->ioStatus = -1;
 337
 338     poller = (IOPolledInterface *) vars->pollers->getObject(0);
 339     err = poller->startIO(operation, bufferOffset, deviceOffset, length, completion);
 340     if (err)
 341         HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
 342
 343     return (err);
 344 }
 345
 346 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 347
 348 static IOReturn
 349 IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable)
 350 {
 351     IOReturn            err = kIOReturnSuccess;
 352     int32_t             idx = 0;
 353     IOPolledInterface * poller;
 354     AbsoluteTime        deadline;
 355
 356     if (!vars->io) return (kIOReturnSuccess);
 357
 358     abortable &= vars->abortable;
 359
 360     clock_interval_to_deadline(2000, kMillisecondScale, &deadline);
 361
 362     while (-1 == vars->ioStatus)
 363     {
 364         for (idx = 0;
 365             (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
 366              idx++)
 367         {
 368             IOReturn newErr;
 369             newErr = poller->checkForWork();
 370             if ((newErr == kIOReturnAborted) && !abortable)
 371                 newErr = kIOReturnSuccess;
 372             if (kIOReturnSuccess == err)
 373                 err = newErr;
 374         }
 375         if ((false) && (kIOReturnSuccess == err) && (mach_absolute_time() > AbsoluteTime_to_scalar(&deadline)))
 376         {
 377             HIBLOG("IOPolledInterface::forced timeout\n");
 378             vars->ioStatus = kIOReturnTimeout;
 379         }
 380     }
 381     vars->io = false;
 382
 383 #if HIBERNATION
 384     if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort())
 385     {
 386         err = kIOReturnAborted;
 387         HIBLOG("IOPolledInterface::checkForWork sw abort\n");
 388     }
 389 #endif
 390
 391     if (err)
 392     {
 393         HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
 394     }
 395     else
 396     {
 397         err = vars->ioStatus;
 398         if (kIOReturnSuccess != err) HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
 399     }
 400
 401     return (err);
 402 }
 403
 404 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 405 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 406
 407 struct _OpenFileContext
 408 {
 409     OSData * extents;
 410     uint64_t size;
 411 };
 412
 413 static void
 414 file_extent_callback(void * ref, uint64_t start, uint64_t length)
 415 {
 416     _OpenFileContext * ctx = (_OpenFileContext *) ref;
 417     IOPolledFileExtent extent;
 418
 419     extent.start  = start;
 420     extent.length = length;
 421     ctx->extents->appendBytes(&extent, sizeof(extent));
 422     ctx->size += length;
 423 }
 424
 425 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 426
 427 static IOService *
 428 IOCopyMediaForDev(dev_t device)
 429 {
 430     OSDictionary * matching;
 431     OSNumber *     num;
 432     OSIterator *   iter;
 433     IOService *    result = 0;
 434
 435     matching = IOService::serviceMatching("IOMedia");
 436     if (!matching)
 437         return (0);
 438     do
 439     {
 440         num = OSNumber::withNumber(major(device), 32);
 441         if (!num)
 442             break;
 443         matching->setObject(kIOBSDMajorKey, num);
 444         num->release();
 445         num = OSNumber::withNumber(minor(device), 32);
 446         if (!num)
 447             break;
 448         matching->setObject(kIOBSDMinorKey, num);
 449         num->release();
 450         if (!num)
 451             break;
 452         iter = IOService::getMatchingServices(matching);
 453         if (iter)
 454         {
 455             result = (IOService *) iter->getNextObject();
 456             result->retain();
 457             iter->release();
 458         }
 459     }
 460     while (false);
 461     matching->release();
 462
 463     return (result);
 464 }
 465
 466 static IOReturn
 467 IOGetVolumeCryptKey(dev_t block_dev,  OSString ** pKeyUUID,
 468                     uint8_t * volumeCryptKey, size_t keySize)
 469 {
 470     IOReturn         err;
 471     IOService *      part;
 472     OSString *       keyUUID = 0;
 473     OSString *       keyStoreUUID = 0;
 474     uuid_t           volumeKeyUUID;
 475     aks_volume_key_t vek;
 476
 477     static IOService * sKeyStore;
 478
 479     part = IOCopyMediaForDev(block_dev);
 480     if (!part) return (kIOReturnNotFound);
 481
 482     err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false,
 483                                       (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL);
 484     if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID)
 485     {
 486 //            IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy());
 487
 488         if (!sKeyStore)
 489             sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane);
 490         if (sKeyStore)
 491             err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID);
 492         else
 493             err = kIOReturnNoResources;
 494         if (kIOReturnSuccess == err)
 495             err = sKeyStore->callPlatformFunction(gAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL);
 496         if (kIOReturnSuccess != err)
 497             IOLog("volume key err 0x%x\n", err);
 498         else
 499         {
 500             if (vek.key.keybytecount < keySize) keySize = vek.key.keybytecount;
 501             bcopy(&vek.key.keybytes[0], volumeCryptKey, keySize);
 502         }
 503         bzero(&vek, sizeof(vek));
 504
 505     }
 506     part->release();
 507     if (pKeyUUID) *pKeyUUID = keyUUID;
 508
 509     return (err);
 510 }
 511
 512 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 513
 514 IOReturn
 515 IOPolledFileOpen(const char * filename,
 516                  uint64_t setFileSize, uint64_t fsFreeSize,
 517                  void * write_file_addr, size_t write_file_len,
 518                  IOPolledFileIOVars ** fileVars,
 519                  OSData ** imagePath,
 520                  uint8_t * volumeCryptKey, size_t keySize)
 521 {
 522     IOReturn             err = kIOReturnSuccess;
 523     IOPolledFileIOVars * vars;
 524     _OpenFileContext     ctx;
 525     OSData *             extentsData;
 526     OSNumber *           num;
 527     IOService *          part = 0;
 528     dev_t                block_dev;
 529     dev_t                image_dev;
 530     AbsoluteTime         startTime, endTime;
 531     uint64_t             nsec;
 532
 533     vars = IONew(IOPolledFileIOVars, 1);
 534     if (!vars) return (kIOReturnNoMemory);
 535     bzero(vars, sizeof(*vars));
 536     vars->allocated = true;
 537
 538     do
 539     {
 540         extentsData = OSData::withCapacity(32);
 541         ctx.extents = extentsData;
 542         ctx.size    = 0;
 543         clock_get_uptime(&startTime);
 544
 545         vars->fileRef = kern_open_file_for_direct_io(filename,
 546                                                      (write_file_addr != NULL) || (0 != setFileSize),
 547                                                      &file_extent_callback, &ctx,
 548                                                      setFileSize,
 549                                                      fsFreeSize,
 550                                                      // write file:
 551                                                      0, write_file_addr, write_file_len,
 552                                                      // results
 553                                                      &block_dev,
 554                                                      &image_dev,
 555                                                      &vars->block0,
 556                                                      &vars->maxiobytes,
 557                                                      &vars->flags);
 558 #if 0
 559         uint32_t msDelay = (131071 & random());
 560         HIBLOG("sleep %d\n", msDelay);
 561         IOSleep(msDelay);
 562 #endif
 563         clock_get_uptime(&endTime);
 564         SUB_ABSOLUTETIME(&endTime, &startTime);
 565         absolutetime_to_nanoseconds(endTime, &nsec);
 566
 567         if (!vars->fileRef) err = kIOReturnNoSpace;
 568
 569         HIBLOG("kern_open_file_for_direct_io took %qd ms\n", nsec / 1000000ULL);
 570         if (kIOReturnSuccess != err) break;
 571
 572         HIBLOG("Opened file %s, size %qd, extents %ld, maxio %qx ssd %d\n", filename, ctx.size,
 573                     (extentsData->getLength() / sizeof(IOPolledFileExtent)) - 1,
 574                     vars->maxiobytes, kIOPolledFileSSD & vars->flags);
 575         assert(!vars->block0);
 576         if (extentsData->getLength() < sizeof(IOPolledFileExtent))
 577         {
 578             err = kIOReturnNoSpace;
 579             break;
 580         }
 581
 582         vars->fileSize = ctx.size;
 583         vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
 584
 585         part = IOCopyMediaForDev(image_dev);
 586         if (!part)
 587         {
 588             err = kIOReturnNotFound;
 589             break;
 590         }
 591
 592         if (!(vars->pollers = IOPolledFilePollers::copyPollers(part))) break;
 593
 594         if ((num = OSDynamicCast(OSNumber, part->getProperty(kIOMediaPreferredBlockSizeKey))))
 595                vars->blockSize = num->unsigned32BitValue();
 596         if (vars->blockSize < 4096) vars->blockSize = 4096;
 597
 598         HIBLOG("polled file major %d, minor %d, blocksize %ld, pollers %d\n",
 599                major(image_dev), minor(image_dev), (long)vars->blockSize,
 600                vars->pollers->pollers->getCount());
 601
 602         OSString * keyUUID = NULL;
 603         if (volumeCryptKey)
 604         {
 605             err = IOGetVolumeCryptKey(block_dev, &keyUUID, volumeCryptKey, keySize);
 606         }
 607
 608         *fileVars    = vars;
 609         vars->fileExtents = extentsData;
 610
 611         // make imagePath
 612         OSData * data;
 613         if (imagePath)
 614         {
 615 #if defined(__i386__) || defined(__x86_64__)
 616             char str2[24 + sizeof(uuid_string_t) + 2];
 617
 618             if (keyUUID)
 619                 snprintf(str2, sizeof(str2), "%qx:%s",
 620                                 vars->extentMap[0].start, keyUUID->getCStringNoCopy());
 621             else
 622                 snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
 623
 624             err = IOService::getPlatform()->callPlatformFunction(
 625                                                 gIOCreateEFIDevicePathSymbol, false,
 626                                                 (void *) part, (void *) str2,
 627                                                 (void *) (uintptr_t) true, (void *) &data);
 628 #else
 629             data = 0;
 630             err = kIOReturnSuccess;
 631 #endif
 632             if (kIOReturnSuccess != err)
 633             {
 634                 HIBLOG("error 0x%x getting path\n", err);
 635                 break;
 636             }
 637             *imagePath = data;
 638         }
 639     }
 640     while (false);
 641
 642     if (kIOReturnSuccess != err)
 643     {
 644         HIBLOG("error 0x%x opening polled file\n", err);
 645         IOPolledFileClose(&vars, 0, 0, 0, 0, 0);
 646     }
 647
 648     if (part) part->release();
 649
 650     return (err);
 651 }
 652
 653 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 654
 655 IOReturn
 656 IOPolledFileClose(IOPolledFileIOVars ** pVars,
 657                   off_t write_offset, void * addr, size_t write_length,
 658                   off_t discard_offset, off_t discard_end)
 659 {
 660     IOPolledFileIOVars * vars;
 661
 662     vars = *pVars;
 663     if (!vars) return(kIOReturnSuccess);
 664
 665     if (vars->fileRef)
 666     {
 667         kern_close_file_for_direct_io(vars->fileRef, write_offset, addr, write_length,
 668                                       discard_offset, discard_end);
 669         vars->fileRef = NULL;
 670     }
 671     if (vars->fileExtents)
 672     {
 673         vars->fileExtents->release();
 674         vars->fileExtents = 0;
 675     }
 676     if (vars->pollers)
 677     {
 678         vars->pollers->release();
 679         vars->pollers = 0;
 680     }
 681
 682     if (vars->allocated) IODelete(vars, IOPolledFileIOVars, 1);
 683     else                 bzero(vars, sizeof(IOPolledFileIOVars));
 684     *pVars = NULL;
 685
 686     return (kIOReturnSuccess);
 687 }
 688
 689 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 690
 691 IOReturn
 692 IOPolledFilePollersSetup(IOPolledFileIOVars * vars,
 693                          uint32_t openState)
 694 {
 695     IOReturn err;
 696
 697     err = kIOReturnSuccess;
 698     do
 699     {
 700         if (!vars->pollers->openCount)
 701         {
 702             err = IOPolledFilePollersProbe(vars->pollers);
 703             if (kIOReturnSuccess != err) break;
 704             err = IOPolledFilePollersOpen(vars, openState, false);
 705             if (kIOReturnSuccess != err) break;
 706             vars->pollers->openState = openState;
 707         }
 708         vars->pollers->openCount++;
 709         vars->pollers->io  = false;
 710         vars->buffer       = (uint8_t *) vars->pollers->ioBuffer->getBytesNoCopy();
 711         vars->bufferHalf   = 0;
 712         vars->bufferOffset = 0;
 713         vars->bufferSize   = (vars->pollers->ioBuffer->getLength() >> 1);
 714
 715         if (vars->maxiobytes < vars->bufferSize) vars->bufferSize = vars->maxiobytes;
 716     }
 717     while (false);
 718
 719     if (kIOReturnSuccess != err) HIBLOG("IOPolledFilePollersSetup(%d) error 0x%x\n", openState, err);
 720
 721     return (err);
 722 }
 723
 724
 725 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 726
 727 IOReturn
 728 IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
 729 {
 730     IOPolledFileExtent * extentMap;
 731
 732     extentMap = vars->extentMap;
 733
 734     vars->position = position;
 735
 736     while (position >= extentMap->length)
 737     {
 738         position -= extentMap->length;
 739         extentMap++;
 740     }
 741
 742     vars->currentExtent   = extentMap;
 743     vars->extentRemaining = extentMap->length - position;
 744     vars->extentPosition  = vars->position - position;
 745
 746     if (vars->bufferSize <= vars->extentRemaining)
 747         vars->bufferLimit = vars->bufferSize;
 748     else
 749         vars->bufferLimit = vars->extentRemaining;
 750
 751     return (kIOReturnSuccess);
 752 }
 753
 754 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 755
 756 IOReturn
 757 IOPolledFileWrite(IOPolledFileIOVars * vars,
 758                     const uint8_t * bytes, IOByteCount size,
 759                     IOPolledFileCryptVars * cryptvars)
 760 {
 761     IOReturn    err = kIOReturnSuccess;
 762     IOByteCount copy;
 763     bool        flush = false;
 764
 765     do
 766     {
 767         if (!bytes && !size)
 768         {
 769             // seek to end of block & flush
 770             size = vars->position & (vars->blockSize - 1);
 771             if (size)
 772                 size = vars->blockSize - size;
 773             flush = true;
 774             // use some garbage for the fill
 775             bytes = vars->buffer + vars->bufferOffset;
 776         }
 777
 778         copy = vars->bufferLimit - vars->bufferOffset;
 779         if (copy > size)
 780             copy = size;
 781         else
 782             flush = true;
 783
 784         if (bytes)
 785         {
 786             bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
 787             bytes += copy;
 788         }
 789         else
 790             bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
 791
 792         size -= copy;
 793         vars->bufferOffset += copy;
 794         vars->position += copy;
 795
 796         if (flush && vars->bufferOffset)
 797         {
 798             uint64_t offset = (vars->position - vars->bufferOffset
 799                                 - vars->extentPosition + vars->currentExtent->start);
 800             uint32_t length = (vars->bufferOffset);
 801
 802 #if CRYPTO
 803             if (cryptvars && vars->encryptStart
 804                 && (vars->position > vars->encryptStart)
 805                 && ((vars->position - length) < vars->encryptEnd))
 806             {
 807                 AbsoluteTime startTime, endTime;
 808
 809                 uint64_t encryptLen, encryptStart;
 810                 encryptLen = vars->position - vars->encryptStart;
 811                 if (encryptLen > length)
 812                     encryptLen = length;
 813                 encryptStart = length - encryptLen;
 814                 if (vars->position > vars->encryptEnd)
 815                     encryptLen -= (vars->position - vars->encryptEnd);
 816
 817                 clock_get_uptime(&startTime);
 818
 819                 // encrypt the buffer
 820                 aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
 821                                 &cryptvars->aes_iv[0],
 822                                 encryptLen / AES_BLOCK_SIZE,
 823                                 vars->buffer + vars->bufferHalf + encryptStart,
 824                                 &cryptvars->ctx.encrypt);
 825
 826                 clock_get_uptime(&endTime);
 827                 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
 828                 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
 829                 vars->cryptBytes += encryptLen;
 830
 831                 // save initial vector for following encrypts
 832                 bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
 833                         &cryptvars->aes_iv[0],
 834                         AES_BLOCK_SIZE);
 835             }
 836 #endif /* CRYPTO */
 837
 838             err = IOPolledFilePollersIODone(vars->pollers, true);
 839             if (kIOReturnSuccess != err)
 840                 break;
 841
 842 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
 843 //if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
 844
 845             err = IOStartPolledIO(vars->pollers, kIOPolledWrite, vars->bufferHalf, offset, length);
 846             if (kIOReturnSuccess != err)
 847                 break;
 848             vars->pollers->io = true;
 849
 850             vars->extentRemaining -= vars->bufferOffset;
 851             if (!vars->extentRemaining)
 852             {
 853                 vars->currentExtent++;
 854                 vars->extentRemaining = vars->currentExtent->length;
 855                 vars->extentPosition  = vars->position;
 856             }
 857
 858             vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
 859             vars->bufferOffset = 0;
 860             if (vars->bufferSize <= vars->extentRemaining)
 861                 vars->bufferLimit = vars->bufferSize;
 862             else
 863                 vars->bufferLimit = vars->extentRemaining;
 864
 865             if (!vars->extentRemaining)
 866             {
 867                 err = kIOReturnOverrun;
 868                 break;
 869             }
 870
 871             flush = false;
 872         }
 873     }
 874     while (size);
 875
 876     return (err);
 877 }
 878
 879 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 880
 881 IOReturn
 882 IOPolledFileRead(IOPolledFileIOVars * vars,
 883                     uint8_t * bytes, IOByteCount size,
 884                     IOPolledFileCryptVars * cryptvars)
 885 {
 886     IOReturn    err = kIOReturnSuccess;
 887     IOByteCount copy;
 888
 889 //    bytesWritten += size;
 890
 891     do
 892     {
 893         copy = vars->bufferLimit - vars->bufferOffset;
 894         if (copy > size)
 895             copy = size;
 896
 897         if (bytes)
 898         {
 899             bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
 900             bytes += copy;
 901         }
 902         size -= copy;
 903         vars->bufferOffset += copy;
 904 //      vars->position += copy;
 905
 906         if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd))
 907         {
 908             if (!vars->pollers->io) cryptvars = 0;
 909             err = IOPolledFilePollersIODone(vars->pollers, true);
 910             if (kIOReturnSuccess != err)
 911                 break;
 912
 913 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
 914
 915             vars->position        += vars->lastRead;
 916             vars->extentRemaining -= vars->lastRead;
 917             vars->bufferLimit      = vars->lastRead;
 918
 919             if (!vars->extentRemaining)
 920             {
 921                 vars->currentExtent++;
 922                 vars->extentRemaining = vars->currentExtent->length;
 923                 vars->extentPosition  = vars->position;
 924                 if (!vars->extentRemaining)
 925                 {
 926                     err = kIOReturnOverrun;
 927                     break;
 928                 }
 929             }
 930
 931             uint64_t length;
 932             uint64_t lastReadLength = vars->lastRead;
 933             uint64_t offset = (vars->position
 934                                 - vars->extentPosition + vars->currentExtent->start);
 935             if (vars->extentRemaining <= vars->bufferSize)
 936                 length = vars->extentRemaining;
 937             else
 938                 length = vars->bufferSize;
 939             if ((length + vars->position) > vars->readEnd)
 940                 length = vars->readEnd - vars->position;
 941
 942             vars->lastRead = length;
 943             if (length)
 944             {
 945 //if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
 946                 err = IOStartPolledIO(vars->pollers, kIOPolledRead, vars->bufferHalf, offset, length);
 947                 if (kIOReturnSuccess != err)
 948                     break;
 949                 vars->pollers->io = true;
 950             }
 951
 952             vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
 953             vars->bufferOffset = 0;
 954
 955 #if CRYPTO
 956             if (cryptvars)
 957             {
 958                 uint8_t thisVector[AES_BLOCK_SIZE];
 959                 AbsoluteTime startTime, endTime;
 960
 961                 // save initial vector for following decrypts
 962                 bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
 963                 bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE,
 964                         &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
 965
 966                 // decrypt the buffer
 967                 clock_get_uptime(&startTime);
 968
 969                 aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
 970                                 &thisVector[0],
 971                                 lastReadLength / AES_BLOCK_SIZE,
 972                                 vars->buffer + vars->bufferHalf,
 973                                 &cryptvars->ctx.decrypt);
 974
 975                 clock_get_uptime(&endTime);
 976                 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
 977                 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
 978                 vars->cryptBytes += lastReadLength;
 979             }
 980 #endif /* CRYPTO */
 981         }
 982     }
 983     while (size);
 984
 985     return (err);
 986 }
 987
 988 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 989