iokit/Kernel/IOPolledInterface.cpp

   1 /*
   2  * Copyright (c) 2006-2009 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <sys/uio.h>
  30 #include <sys/conf.h>
  31
  32 #include <IOKit/IOLib.h>
  33 #include <IOKit/IOBSD.h>
  34 #include <IOKit/IOService.h>
  35 #include <IOKit/IOPlatformExpert.h>
  36 #include <IOKit/IOPolledInterface.h>
  37 #include <IOKit/IOHibernatePrivate.h>
  38 #include <IOKit/IOBufferMemoryDescriptor.h>
  39 #include <IOKit/AppleKeyStoreInterface.h>
  40 #include "IOKitKernelInternal.h"
  41
  42
  43 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  44
  45 OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject);
  46
  47 OSMetaClassDefineReservedUnused(IOPolledInterface, 0);
  48 OSMetaClassDefineReservedUnused(IOPolledInterface, 1);
  49 OSMetaClassDefineReservedUnused(IOPolledInterface, 2);
  50 OSMetaClassDefineReservedUnused(IOPolledInterface, 3);
  51 OSMetaClassDefineReservedUnused(IOPolledInterface, 4);
  52 OSMetaClassDefineReservedUnused(IOPolledInterface, 5);
  53 OSMetaClassDefineReservedUnused(IOPolledInterface, 6);
  54 OSMetaClassDefineReservedUnused(IOPolledInterface, 7);
  55 OSMetaClassDefineReservedUnused(IOPolledInterface, 8);
  56 OSMetaClassDefineReservedUnused(IOPolledInterface, 9);
  57 OSMetaClassDefineReservedUnused(IOPolledInterface, 10);
  58 OSMetaClassDefineReservedUnused(IOPolledInterface, 11);
  59 OSMetaClassDefineReservedUnused(IOPolledInterface, 12);
  60 OSMetaClassDefineReservedUnused(IOPolledInterface, 13);
  61 OSMetaClassDefineReservedUnused(IOPolledInterface, 14);
  62 OSMetaClassDefineReservedUnused(IOPolledInterface, 15);
  63
  64 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  65
  66 #ifndef kIOMediaPreferredBlockSizeKey
  67 #define kIOMediaPreferredBlockSizeKey   "Preferred Block Size"
  68 #endif
  69
  70 enum { kDefaultIOSize = 128*1024 };
  71
  72 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  73
  74 class IOPolledFilePollers : public OSObject
  75 {
  76     OSDeclareDefaultStructors(IOPolledFilePollers)
  77
  78 public:
  79     IOService                * media;
  80     OSArray                  * pollers;
  81     IOBufferMemoryDescriptor * ioBuffer;
  82     bool                 abortable;
  83     bool                 io;
  84     IOReturn             ioStatus;
  85     uint32_t             openCount;
  86
  87     static IOPolledFilePollers * copyPollers(IOService * media);
  88 };
  89
  90 OSDefineMetaClassAndStructors(IOPolledFilePollers, OSObject)
  91
  92 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  93
  94 IOPolledFilePollers *
  95 IOPolledFilePollers::copyPollers(IOService * media)
  96 {
  97     IOPolledFilePollers * vars;
  98     IOReturn              err;
  99     IOService       * service;
 100     OSObject        * obj;
 101     IORegistryEntry * next;
 102     IORegistryEntry * child;
 103
 104     if ((obj = media->copyProperty(kIOPolledInterfaceStackKey)))
 105     {
 106         return (OSDynamicCast(IOPolledFilePollers, obj));
 107     }
 108
 109     do
 110     {
 111         vars = OSTypeAlloc(IOPolledFilePollers);
 112         vars->init();
 113
 114         vars->pollers = OSArray::withCapacity(4);
 115         if (!vars->pollers)
 116         {
 117             err = kIOReturnNoMemory;
 118             break;
 119         }
 120
 121         next = vars->media = media;
 122         do
 123         {
 124             IOPolledInterface * poller;
 125             OSObject *          obj;
 126
 127             obj = next->getProperty(kIOPolledInterfaceSupportKey);
 128             if (kOSBooleanFalse == obj)
 129             {
 130                 vars->pollers->flushCollection();
 131                 break;
 132             }
 133             else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
 134                 vars->pollers->setObject(poller);
 135
 136             if ((service = OSDynamicCast(IOService, next))
 137                 && service->getDeviceMemory()
 138                 && !vars->pollers->getCount())  break;
 139
 140             child = next;
 141         }
 142         while ((next = child->getParentEntry(gIOServicePlane))
 143                && child->isParent(next, gIOServicePlane, true));
 144
 145         if (!vars->pollers->getCount())
 146         {
 147             err = kIOReturnUnsupported;
 148             break;
 149         }
 150     }
 151     while (false);
 152
 153     media->setProperty(kIOPolledInterfaceStackKey, vars);
 154
 155     return (vars);
 156 }
 157
 158 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 159
 160 static IOReturn
 161 IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable);
 162
 163 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 164
 165 static IOReturn
 166 IOPolledFilePollersProbe(IOPolledFilePollers * vars)
 167 {
 168     IOReturn            err = kIOReturnError;
 169     int32_t             idx;
 170     IOPolledInterface * poller;
 171
 172     for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
 173     {
 174         poller = (IOPolledInterface *) vars->pollers->getObject(idx);
 175         err = poller->probe(vars->media);
 176         if (err)
 177         {
 178             HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
 179             break;
 180         }
 181     }
 182
 183     return (err);
 184 }
 185
 186 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 187
 188 IOReturn
 189 IOPolledFilePollersOpen(IOPolledFileIOVars * filevars, uint32_t state, bool abortable)
 190 {
 191
 192     IOPolledFilePollers      * vars = filevars->pollers;
 193     IOBufferMemoryDescriptor * ioBuffer;
 194     IOPolledInterface        * poller;
 195     IOService                * next;
 196     IOReturn                   err = kIOReturnError;
 197     int32_t                    idx;
 198
 199     vars->abortable = abortable;
 200     ioBuffer = 0;
 201
 202     if (kIOPolledAfterSleepState == state)
 203     {
 204         vars->ioStatus = 0;
 205         vars->io = false;
 206     }
 207     (void) IOPolledFilePollersIODone(vars, false);
 208
 209     if ((kIOPolledPreflightState == state) || (kIOPolledPreflightCoreDumpState == state))
 210     {
 211         ioBuffer = vars->ioBuffer;
 212         if (!ioBuffer)
 213         {
 214             vars->ioBuffer = ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionInOut,
 215                                                             2 * kDefaultIOSize, page_size);
 216             if (!ioBuffer) return (kIOReturnNoMemory);
 217         }
 218     }
 219
 220     for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
 221     {
 222         poller = (IOPolledInterface *) vars->pollers->getObject(idx);
 223         err = poller->open(state, ioBuffer);
 224         if (kIOReturnSuccess != err)
 225         {
 226             HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
 227             break;
 228         }
 229     }
 230     if ((kIOReturnSuccess == err) && (kIOPolledPreflightState == state))
 231     {
 232         next = vars->media;
 233         while (next)
 234         {
 235             next->setProperty(kIOPolledInterfaceActiveKey, kOSBooleanTrue);
 236             next = next->getProvider();
 237         }
 238     }
 239
 240     return (err);
 241 }
 242
 243 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 244
 245 IOReturn
 246 IOPolledFilePollersClose(IOPolledFileIOVars * filevars, uint32_t state)
 247 {
 248     IOPolledFilePollers * vars = filevars->pollers;
 249     IOPolledInterface * poller;
 250     IORegistryEntry *   next;
 251     IOReturn            err;
 252     int32_t             idx;
 253
 254     (void) IOPolledFilePollersIODone(vars, false);
 255
 256     if ((kIOPolledPostflightState == state) || (kIOPolledPostflightCoreDumpState == state))
 257     {
 258         vars->openCount--;
 259     }
 260
 261     for (idx = 0, err = kIOReturnSuccess;
 262          (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
 263          idx++)
 264     {
 265         err = poller->close(state);
 266         if ((kIOReturnSuccess != err) && (kIOPolledBeforeSleepStateAborted == state))
 267         {
 268             err = poller->close(kIOPolledBeforeSleepState);
 269         }
 270         if (err) HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
 271     }
 272
 273     if (kIOPolledPostflightState == state)
 274     {
 275         next = vars->media;
 276         while (next)
 277         {
 278             next->removeProperty(kIOPolledInterfaceActiveKey);
 279             next = next->getParentEntry(gIOServicePlane);
 280         }
 281     }
 282
 283     if ((kIOPolledPostflightState == state) || (kIOPolledPostflightCoreDumpState == state)) do
 284     {
 285         if (vars->openCount) break;
 286         if (vars->ioBuffer)
 287         {
 288             vars->ioBuffer->release();
 289             vars->ioBuffer = 0;
 290         }
 291     }
 292     while (false);
 293
 294     return (err);
 295 }
 296 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 297
 298 IOMemoryDescriptor *
 299 IOPolledFileGetIOBuffer(IOPolledFileIOVars * vars)
 300 {
 301     return (vars->pollers->ioBuffer);
 302 }
 303
 304 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 305
 306 static void
 307 IOPolledIOComplete(void *   target,
 308                    void *   parameter,
 309                    IOReturn status,
 310                    UInt64   actualByteCount)
 311 {
 312     IOPolledFilePollers * vars = (IOPolledFilePollers *) parameter;
 313
 314     vars->ioStatus = status;
 315 }
 316
 317 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 318
 319 static IOReturn
 320 IOStartPolledIO(IOPolledFilePollers * vars,
 321                     uint32_t operation, uint32_t bufferOffset,
 322                     uint64_t deviceOffset, uint64_t length)
 323 {
 324     IOReturn            err;
 325     IOPolledInterface * poller;
 326     IOPolledCompletion  completion;
 327
 328     err = vars->ioStatus;
 329     if (kIOReturnSuccess != err) return (err);
 330
 331     completion.target    = 0;
 332     completion.action    = &IOPolledIOComplete;
 333     completion.parameter = vars;
 334
 335     vars->ioStatus = -1;
 336
 337     poller = (IOPolledInterface *) vars->pollers->getObject(0);
 338     err = poller->startIO(operation, bufferOffset, deviceOffset, length, completion);
 339     if (err) {
 340         if (kernel_debugger_entry_count) {
 341             HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
 342         } else {
 343             HIBLOGFROMPANIC("IOPolledInterface::IOStartPolledIO(0x%p, %d, 0x%x, 0x%llx, %llu) : poller->startIO(%d, 0x%x, 0x%llx, %llu, completion) returned 0x%x",
 344                         vars, operation, bufferOffset, deviceOffset, length, operation, bufferOffset, deviceOffset, length, err);
 345         }
 346     }
 347     return (err);
 348 }
 349
 350 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 351
 352 static IOReturn
 353 IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable)
 354 {
 355     IOReturn            err = kIOReturnSuccess;
 356     int32_t             idx = 0;
 357     IOPolledInterface * poller;
 358     AbsoluteTime        deadline;
 359
 360     if (!vars->io) return (kIOReturnSuccess);
 361
 362     abortable &= vars->abortable;
 363
 364     clock_interval_to_deadline(2000, kMillisecondScale, &deadline);
 365
 366     while (-1 == vars->ioStatus)
 367     {
 368         for (idx = 0;
 369             (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
 370              idx++)
 371         {
 372             IOReturn newErr;
 373             newErr = poller->checkForWork();
 374             if ((newErr == kIOReturnAborted) && !abortable)
 375                 newErr = kIOReturnSuccess;
 376             if (kIOReturnSuccess == err)
 377                 err = newErr;
 378         }
 379         if ((false) && (kIOReturnSuccess == err) && (mach_absolute_time() > AbsoluteTime_to_scalar(&deadline)))
 380         {
 381             HIBLOG("IOPolledInterface::forced timeout\n");
 382             vars->ioStatus = kIOReturnTimeout;
 383         }
 384     }
 385     vars->io = false;
 386
 387 #if HIBERNATION
 388     if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort())
 389     {
 390         err = kIOReturnAborted;
 391         HIBLOG("IOPolledInterface::checkForWork sw abort\n");
 392     }
 393 #endif
 394
 395     if (err)
 396     {
 397         HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
 398     }
 399     else
 400     {
 401         err = vars->ioStatus;
 402         if (kIOReturnSuccess != err) HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
 403     }
 404
 405     return (err);
 406 }
 407
 408 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 409 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 410
 411 struct _OpenFileContext
 412 {
 413     OSData * extents;
 414     uint64_t size;
 415 };
 416
 417 static void
 418 file_extent_callback(void * ref, uint64_t start, uint64_t length)
 419 {
 420     _OpenFileContext * ctx = (_OpenFileContext *) ref;
 421     IOPolledFileExtent extent;
 422
 423     extent.start  = start;
 424     extent.length = length;
 425     ctx->extents->appendBytes(&extent, sizeof(extent));
 426     ctx->size += length;
 427 }
 428
 429 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 430
 431 static IOService *
 432 IOCopyMediaForDev(dev_t device)
 433 {
 434     OSDictionary * matching;
 435     OSNumber *     num;
 436     OSIterator *   iter;
 437     IOService *    result = 0;
 438
 439     matching = IOService::serviceMatching("IOMedia");
 440     if (!matching)
 441         return (0);
 442     do
 443     {
 444         num = OSNumber::withNumber(major(device), 32);
 445         if (!num)
 446             break;
 447         matching->setObject(kIOBSDMajorKey, num);
 448         num->release();
 449         num = OSNumber::withNumber(minor(device), 32);
 450         if (!num)
 451             break;
 452         matching->setObject(kIOBSDMinorKey, num);
 453         num->release();
 454         if (!num)
 455             break;
 456         iter = IOService::getMatchingServices(matching);
 457         if (iter)
 458         {
 459             result = (IOService *) iter->getNextObject();
 460             result->retain();
 461             iter->release();
 462         }
 463     }
 464     while (false);
 465     matching->release();
 466
 467     return (result);
 468 }
 469
 470 #define APFSMEDIA_GETHIBERKEY         "getHiberKey"
 471
 472 static IOReturn
 473 IOGetVolumeCryptKey(dev_t block_dev,  OSString ** pKeyUUID,
 474                     uint8_t * volumeCryptKey, size_t keySize)
 475 {
 476     IOReturn         err;
 477     IOService *      part;
 478     OSString *       keyUUID = 0;
 479     OSString *       keyStoreUUID = 0;
 480     uuid_t           volumeKeyUUID;
 481     aks_volume_key_t vek;
 482
 483     static IOService * sKeyStore;
 484
 485     part = IOCopyMediaForDev(block_dev);
 486     if (!part) return (kIOReturnNotFound);
 487
 488     // Try APFS first
 489     {
 490         uuid_t volUuid = {0};
 491         size_t sizeOut = 0;
 492         err = part->callPlatformFunction(APFSMEDIA_GETHIBERKEY, false, &volUuid, volumeCryptKey, &keySize, &sizeOut);
 493         if (err == kIOReturnSuccess) {
 494             // No need to create uuid string if it's not requested
 495             if (pKeyUUID) {
 496                 uuid_string_t volUuidStr;
 497                 uuid_unparse(volUuid, volUuidStr);
 498                 *pKeyUUID = OSString::withCString(volUuidStr);
 499             }
 500
 501             part->release();
 502             return kIOReturnSuccess;
 503         }
 504     }
 505
 506     // Then old CS path
 507     err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false,
 508                   (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL);
 509     if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID)
 510     {
 511 //        IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy());
 512
 513         if (!sKeyStore)
 514             sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane);
 515         if (sKeyStore)
 516             err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID);
 517         else
 518             err = kIOReturnNoResources;
 519         if (kIOReturnSuccess == err)
 520             err = sKeyStore->callPlatformFunction(gAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL);
 521         if (kIOReturnSuccess != err)
 522             IOLog("volume key err 0x%x\n", err);
 523         else
 524         {
 525             if (vek.key.keybytecount < keySize) keySize = vek.key.keybytecount;
 526             bcopy(&vek.key.keybytes[0], volumeCryptKey, keySize);
 527         }
 528         bzero(&vek, sizeof(vek));
 529     }
 530
 531     part->release();
 532     if (pKeyUUID) *pKeyUUID = keyUUID;
 533
 534     return (err);
 535 }
 536
 537 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 538
 539 IOReturn
 540 IOPolledFileOpen(const char * filename,
 541                  uint64_t setFileSize, uint64_t fsFreeSize,
 542                  void * write_file_addr, size_t write_file_len,
 543                  IOPolledFileIOVars ** fileVars,
 544                  OSData ** imagePath,
 545                  uint8_t * volumeCryptKey, size_t keySize)
 546 {
 547     IOReturn             err = kIOReturnSuccess;
 548     IOPolledFileIOVars * vars;
 549     _OpenFileContext     ctx;
 550     OSData *             extentsData = NULL;
 551     OSNumber *           num;
 552     IOService *          part = 0;
 553     dev_t                block_dev;
 554     dev_t                image_dev;
 555     AbsoluteTime         startTime, endTime;
 556     uint64_t             nsec;
 557
 558     vars = IONew(IOPolledFileIOVars, 1);
 559     if (!vars) return (kIOReturnNoMemory);
 560     bzero(vars, sizeof(*vars));
 561     vars->allocated = true;
 562
 563     do
 564     {
 565         extentsData = OSData::withCapacity(32);
 566         ctx.extents = extentsData;
 567         ctx.size    = 0;
 568         clock_get_uptime(&startTime);
 569
 570         vars->fileRef = kern_open_file_for_direct_io(filename,
 571                                                      (write_file_addr != NULL) || (0 != setFileSize),
 572                                                      &file_extent_callback, &ctx,
 573                                                      setFileSize,
 574                                                      fsFreeSize,
 575                                                      // write file:
 576                                                      0, write_file_addr, write_file_len,
 577                                                      // results
 578                                                      &block_dev,
 579                                                      &image_dev,
 580                                                      &vars->block0,
 581                                                      &vars->maxiobytes,
 582                                                      &vars->flags);
 583 #if 0
 584         uint32_t msDelay = (131071 & random());
 585         HIBLOG("sleep %d\n", msDelay);
 586         IOSleep(msDelay);
 587 #endif
 588         clock_get_uptime(&endTime);
 589         SUB_ABSOLUTETIME(&endTime, &startTime);
 590         absolutetime_to_nanoseconds(endTime, &nsec);
 591
 592         if (!vars->fileRef) err = kIOReturnNoSpace;
 593
 594         HIBLOG("kern_open_file_for_direct_io took %qd ms\n", nsec / 1000000ULL);
 595         if (kIOReturnSuccess != err) break;
 596
 597         HIBLOG("Opened file %s, size %qd, extents %ld, maxio %qx ssd %d\n", filename, ctx.size,
 598                     (extentsData->getLength() / sizeof(IOPolledFileExtent)) - 1,
 599                     vars->maxiobytes, kIOPolledFileSSD & vars->flags);
 600         assert(!vars->block0);
 601         if (extentsData->getLength() < sizeof(IOPolledFileExtent))
 602         {
 603             err = kIOReturnNoSpace;
 604             break;
 605         }
 606
 607         vars->fileSize = ctx.size;
 608         vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
 609
 610         part = IOCopyMediaForDev(image_dev);
 611         if (!part)
 612         {
 613             err = kIOReturnNotFound;
 614             break;
 615         }
 616
 617         if (!(vars->pollers = IOPolledFilePollers::copyPollers(part))) break;
 618
 619         if ((num = OSDynamicCast(OSNumber, part->getProperty(kIOMediaPreferredBlockSizeKey))))
 620                vars->blockSize = num->unsigned32BitValue();
 621         if (vars->blockSize < 4096) vars->blockSize = 4096;
 622
 623         HIBLOG("polled file major %d, minor %d, blocksize %ld, pollers %d\n",
 624                major(image_dev), minor(image_dev), (long)vars->blockSize,
 625                vars->pollers->pollers->getCount());
 626
 627         OSString * keyUUID = NULL;
 628         if (volumeCryptKey)
 629         {
 630             err = IOGetVolumeCryptKey(block_dev, &keyUUID, volumeCryptKey, keySize);
 631         }
 632
 633         *fileVars    = vars;
 634         vars->fileExtents = extentsData;
 635
 636         // make imagePath
 637         OSData * data;
 638         if (imagePath)
 639         {
 640 #if defined(__i386__) || defined(__x86_64__)
 641             char str2[24 + sizeof(uuid_string_t) + 2];
 642
 643             if (keyUUID)
 644                 snprintf(str2, sizeof(str2), "%qx:%s",
 645                                 vars->extentMap[0].start, keyUUID->getCStringNoCopy());
 646             else
 647                 snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
 648
 649             err = IOService::getPlatform()->callPlatformFunction(
 650                                                 gIOCreateEFIDevicePathSymbol, false,
 651                                                 (void *) part, (void *) str2,
 652                                                 (void *) (uintptr_t) true, (void *) &data);
 653 #else
 654             data = 0;
 655             err = kIOReturnSuccess;
 656 #endif
 657             if (kIOReturnSuccess != err)
 658             {
 659                 HIBLOG("error 0x%x getting path\n", err);
 660                 break;
 661             }
 662             *imagePath = data;
 663         }
 664     }
 665     while (false);
 666
 667     if (kIOReturnSuccess != err)
 668     {
 669         HIBLOG("error 0x%x opening polled file\n", err);
 670         IOPolledFileClose(&vars, 0, 0, 0, 0, 0);
 671         if (extentsData) extentsData->release();
 672     }
 673
 674     if (part) part->release();
 675
 676     return (err);
 677 }
 678
 679 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 680
 681 IOReturn
 682 IOPolledFileClose(IOPolledFileIOVars ** pVars,
 683                   off_t write_offset, void * addr, size_t write_length,
 684                   off_t discard_offset, off_t discard_end)
 685 {
 686     IOPolledFileIOVars * vars;
 687
 688     vars = *pVars;
 689     if (!vars) return(kIOReturnSuccess);
 690
 691     if (vars->fileRef)
 692     {
 693         kern_close_file_for_direct_io(vars->fileRef, write_offset, addr, write_length,
 694                                       discard_offset, discard_end);
 695         vars->fileRef = NULL;
 696     }
 697     if (vars->fileExtents)
 698     {
 699         vars->fileExtents->release();
 700         vars->fileExtents = 0;
 701     }
 702     if (vars->pollers)
 703     {
 704         vars->pollers->release();
 705         vars->pollers = 0;
 706     }
 707
 708     if (vars->allocated) IODelete(vars, IOPolledFileIOVars, 1);
 709     else                 bzero(vars, sizeof(IOPolledFileIOVars));
 710     *pVars = NULL;
 711
 712     return (kIOReturnSuccess);
 713 }
 714
 715 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 716
 717 IOReturn
 718 IOPolledFilePollersSetup(IOPolledFileIOVars * vars,
 719                          uint32_t openState)
 720 {
 721     IOReturn err;
 722
 723     err = kIOReturnSuccess;
 724     do
 725     {
 726         if (!vars->pollers->openCount)
 727         {
 728             err = IOPolledFilePollersProbe(vars->pollers);
 729             if (kIOReturnSuccess != err) break;
 730         }
 731         err = IOPolledFilePollersOpen(vars, openState, false);
 732         if (kIOReturnSuccess != err) break;
 733         if ((kIOPolledPreflightState == openState) || (kIOPolledPreflightCoreDumpState == openState))
 734         {
 735             vars->pollers->openCount++;
 736         }
 737         vars->pollers->io  = false;
 738         vars->buffer       = (uint8_t *) vars->pollers->ioBuffer->getBytesNoCopy();
 739         vars->bufferHalf   = 0;
 740         vars->bufferOffset = 0;
 741         vars->bufferSize   = (vars->pollers->ioBuffer->getLength() >> 1);
 742
 743         if (vars->maxiobytes < vars->bufferSize) vars->bufferSize = vars->maxiobytes;
 744     }
 745     while (false);
 746
 747     if (kIOReturnSuccess != err) HIBLOG("IOPolledFilePollersSetup(%d) error 0x%x\n", openState, err);
 748
 749     return (err);
 750 }
 751
 752
 753 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 754
 755 IOReturn
 756 IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
 757 {
 758     IOPolledFileExtent * extentMap;
 759
 760     extentMap = vars->extentMap;
 761
 762     vars->position = position;
 763
 764     if (position > vars->fileSize) {
 765         HIBLOG("IOPolledFileSeek: called to seek to 0x%llx greater than file size of 0x%llx\n", vars->position,  vars->fileSize);
 766         return kIOReturnNoSpace;
 767     }
 768
 769     while (position >= extentMap->length)
 770     {
 771         position -= extentMap->length;
 772         extentMap++;
 773     }
 774
 775     vars->currentExtent   = extentMap;
 776     vars->extentRemaining = extentMap->length - position;
 777     vars->extentPosition  = vars->position - position;
 778
 779     if (vars->bufferSize <= vars->extentRemaining)
 780         vars->bufferLimit = vars->bufferSize;
 781     else
 782         vars->bufferLimit = vars->extentRemaining;
 783
 784     return (kIOReturnSuccess);
 785 }
 786
 787 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 788
 789 IOReturn
 790 IOPolledFileWrite(IOPolledFileIOVars * vars,
 791                     const uint8_t * bytes, IOByteCount size,
 792                     IOPolledFileCryptVars * cryptvars)
 793 {
 794     IOReturn    err = kIOReturnSuccess;
 795     IOByteCount copy, original_size = size;
 796     bool        flush = false;
 797
 798     do
 799     {
 800         if (!bytes && !size)
 801         {
 802             // seek to end of block & flush
 803             size = vars->position & (vars->blockSize - 1);
 804             if (size)
 805                 size = vars->blockSize - size;
 806             flush = true;
 807             // use some garbage for the fill
 808             bytes = vars->buffer + vars->bufferOffset;
 809         }
 810
 811         copy = vars->bufferLimit - vars->bufferOffset;
 812         if (copy > size)
 813             copy = size;
 814         else
 815             flush = true;
 816
 817         if (bytes)
 818         {
 819             bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
 820             bytes += copy;
 821         }
 822         else
 823             bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
 824
 825         size -= copy;
 826         vars->bufferOffset += copy;
 827         vars->position += copy;
 828
 829         if (flush && vars->bufferOffset)
 830         {
 831             uint64_t offset = (vars->position - vars->bufferOffset
 832                                 - vars->extentPosition + vars->currentExtent->start);
 833             uint32_t length = (vars->bufferOffset);
 834
 835 #if CRYPTO
 836             if (cryptvars && vars->encryptStart
 837                 && (vars->position > vars->encryptStart)
 838                 && ((vars->position - length) < vars->encryptEnd))
 839             {
 840                 AbsoluteTime startTime, endTime;
 841
 842                 uint64_t encryptLen, encryptStart;
 843                 encryptLen = vars->position - vars->encryptStart;
 844                 if (encryptLen > length)
 845                     encryptLen = length;
 846                 encryptStart = length - encryptLen;
 847                 if (vars->position > vars->encryptEnd)
 848                     encryptLen -= (vars->position - vars->encryptEnd);
 849
 850                 clock_get_uptime(&startTime);
 851
 852                 // encrypt the buffer
 853                 aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
 854                                 &cryptvars->aes_iv[0],
 855                                 encryptLen / AES_BLOCK_SIZE,
 856                                 vars->buffer + vars->bufferHalf + encryptStart,
 857                                 &cryptvars->ctx.encrypt);
 858
 859                 clock_get_uptime(&endTime);
 860                 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
 861                 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
 862                 vars->cryptBytes += encryptLen;
 863
 864                 // save initial vector for following encrypts
 865                 bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
 866                         &cryptvars->aes_iv[0],
 867                         AES_BLOCK_SIZE);
 868             }
 869 #endif /* CRYPTO */
 870
 871             err = IOPolledFilePollersIODone(vars->pollers, true);
 872             if (kIOReturnSuccess != err)
 873                 break;
 874
 875 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
 876 //if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
 877
 878             err = IOStartPolledIO(vars->pollers, kIOPolledWrite, vars->bufferHalf, offset, length);
 879             if (kIOReturnSuccess != err) {
 880                 HIBLOGFROMPANIC("IOPolledFileWrite(0x%p, 0x%p, %llu, 0x%p) : IOStartPolledIO(0x%p, kIOPolledWrite, %llu, 0x%llx, %d) returned 0x%x\n",
 881                     vars, bytes, (uint64_t) original_size, cryptvars, vars->pollers, (uint64_t) vars->bufferHalf, offset, length, err);
 882                 break;
 883             }
 884             vars->pollers->io = true;
 885
 886             vars->extentRemaining -= vars->bufferOffset;
 887             if (!vars->extentRemaining)
 888             {
 889                 vars->currentExtent++;
 890                 vars->extentRemaining = vars->currentExtent->length;
 891                 vars->extentPosition  = vars->position;
 892             }
 893
 894             vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
 895             vars->bufferOffset = 0;
 896             if (vars->bufferSize <= vars->extentRemaining)
 897                 vars->bufferLimit = vars->bufferSize;
 898             else
 899                 vars->bufferLimit = vars->extentRemaining;
 900
 901             if (!vars->extentRemaining)
 902             {
 903                 err = kIOReturnOverrun;
 904                 break;
 905             }
 906
 907             flush = false;
 908         }
 909     }
 910     while (size);
 911
 912     return (err);
 913 }
 914
 915 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 916
 917 IOReturn
 918 IOPolledFileFlush(IOPolledFileIOVars * vars)
 919 {
 920     // Only supported by the underlying polled mode driver on embedded currently (expect kIOReturnUnsupported on other platforms)
 921     IOReturn err = kIOReturnSuccess;
 922
 923     err = IOPolledFilePollersIODone(vars->pollers, true);
 924     if (kIOReturnSuccess != err)
 925             return err;
 926
 927     err = IOStartPolledIO(vars->pollers, kIOPolledFlush, 0, 0, 0);
 928     if (kIOReturnSuccess != err) {
 929             HIBLOGFROMPANIC("IOPolledFileFlush(0x%p) : IOStartPolledIO(0x%p, kIOPolledFlush, 0, 0, 0) returned 0x%x\n",
 930                     vars, vars->pollers, err);
 931             return err;
 932     }
 933     vars->pollers->io = true;
 934
 935     return err;
 936 }
 937
 938 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 939
 940 IOReturn
 941 IOPolledFileRead(IOPolledFileIOVars * vars,
 942                     uint8_t * bytes, IOByteCount size,
 943                     IOPolledFileCryptVars * cryptvars)
 944 {
 945     IOReturn    err = kIOReturnSuccess;
 946     IOByteCount copy;
 947
 948 //    bytesWritten += size;
 949
 950     do
 951     {
 952         copy = vars->bufferLimit - vars->bufferOffset;
 953         if (copy > size)
 954             copy = size;
 955
 956         if (bytes)
 957         {
 958             bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
 959             bytes += copy;
 960         }
 961         size -= copy;
 962         vars->bufferOffset += copy;
 963 //      vars->position += copy;
 964
 965         if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd))
 966         {
 967             if (!vars->pollers->io) cryptvars = 0;
 968             err = IOPolledFilePollersIODone(vars->pollers, true);
 969             if (kIOReturnSuccess != err)
 970                 break;
 971
 972 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
 973
 974             vars->position        += vars->lastRead;
 975             vars->extentRemaining -= vars->lastRead;
 976             vars->bufferLimit      = vars->lastRead;
 977
 978             if (!vars->extentRemaining)
 979             {
 980                 vars->currentExtent++;
 981                 vars->extentRemaining = vars->currentExtent->length;
 982                 vars->extentPosition  = vars->position;
 983                 if (!vars->extentRemaining)
 984                 {
 985                     err = kIOReturnOverrun;
 986                     break;
 987                 }
 988             }
 989
 990             uint64_t length;
 991             uint64_t lastReadLength = vars->lastRead;
 992             uint64_t offset = (vars->position
 993                                 - vars->extentPosition + vars->currentExtent->start);
 994             if (vars->extentRemaining <= vars->bufferSize)
 995                 length = vars->extentRemaining;
 996             else
 997                 length = vars->bufferSize;
 998             if ((length + vars->position) > vars->readEnd)
 999                 length = vars->readEnd - vars->position;
1000
1001             vars->lastRead = length;
1002             if (length)
1003             {
1004 //if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
1005                 err = IOStartPolledIO(vars->pollers, kIOPolledRead, vars->bufferHalf, offset, length);
1006                 if (kIOReturnSuccess != err)
1007                     break;
1008                 vars->pollers->io = true;
1009             }
1010
1011             vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
1012             vars->bufferOffset = 0;
1013
1014 #if CRYPTO
1015             if (cryptvars)
1016             {
1017                 uint8_t thisVector[AES_BLOCK_SIZE];
1018                 AbsoluteTime startTime, endTime;
1019
1020                 // save initial vector for following decrypts
1021                 bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
1022                 bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE,
1023                         &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1024
1025                 // decrypt the buffer
1026                 clock_get_uptime(&startTime);
1027
1028                 aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
1029                                 &thisVector[0],
1030                                 lastReadLength / AES_BLOCK_SIZE,
1031                                 vars->buffer + vars->bufferHalf,
1032                                 &cryptvars->ctx.decrypt);
1033
1034                 clock_get_uptime(&endTime);
1035                 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
1036                 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
1037                 vars->cryptBytes += lastReadLength;
1038             }
1039 #endif /* CRYPTO */
1040         }
1041     }
1042     while (size);
1043
1044     return (err);
1045 }
1046
1047 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1048