iokit/Kernel/IOPolledInterface.cpp

   1 /*
   2  * Copyright (c) 2006-2009 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <sys/uio.h>
  30 #include <sys/conf.h>
  31
  32 #include <IOKit/IOLib.h>
  33 #include <IOKit/IOBSD.h>
  34 #include <IOKit/IOService.h>
  35 #include <IOKit/IOPlatformExpert.h>
  36 #include <IOKit/IOPolledInterface.h>
  37 #include <IOKit/IOHibernatePrivate.h>
  38 #include <IOKit/IOBufferMemoryDescriptor.h>
  39 #include <IOKit/AppleKeyStoreInterface.h>
  40 #include "IOKitKernelInternal.h"
  41
  42
  43 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  44
  45 OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject);
  46
  47 OSMetaClassDefineReservedUnused(IOPolledInterface, 0);
  48 OSMetaClassDefineReservedUnused(IOPolledInterface, 1);
  49 OSMetaClassDefineReservedUnused(IOPolledInterface, 2);
  50 OSMetaClassDefineReservedUnused(IOPolledInterface, 3);
  51 OSMetaClassDefineReservedUnused(IOPolledInterface, 4);
  52 OSMetaClassDefineReservedUnused(IOPolledInterface, 5);
  53 OSMetaClassDefineReservedUnused(IOPolledInterface, 6);
  54 OSMetaClassDefineReservedUnused(IOPolledInterface, 7);
  55 OSMetaClassDefineReservedUnused(IOPolledInterface, 8);
  56 OSMetaClassDefineReservedUnused(IOPolledInterface, 9);
  57 OSMetaClassDefineReservedUnused(IOPolledInterface, 10);
  58 OSMetaClassDefineReservedUnused(IOPolledInterface, 11);
  59 OSMetaClassDefineReservedUnused(IOPolledInterface, 12);
  60 OSMetaClassDefineReservedUnused(IOPolledInterface, 13);
  61 OSMetaClassDefineReservedUnused(IOPolledInterface, 14);
  62 OSMetaClassDefineReservedUnused(IOPolledInterface, 15);
  63
  64 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  65
  66 #ifndef kIOMediaPreferredBlockSizeKey
  67 #define kIOMediaPreferredBlockSizeKey   "Preferred Block Size"
  68 #endif
  69
  70 enum { kDefaultIOSize = 128*1024 };
  71
  72 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  73
  74 class IOPolledFilePollers : public OSObject
  75 {
  76     OSDeclareDefaultStructors(IOPolledFilePollers)
  77
  78 public:
  79     IOService                * media;
  80     OSArray                  * pollers;
  81     IOBufferMemoryDescriptor * ioBuffer;
  82     bool                 abortable;
  83     bool                 io;
  84     IOReturn             ioStatus;
  85     uint32_t             openCount;
  86
  87     static IOPolledFilePollers * copyPollers(IOService * media);
  88 };
  89
  90 OSDefineMetaClassAndStructors(IOPolledFilePollers, OSObject)
  91
  92 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  93
  94 IOPolledFilePollers *
  95 IOPolledFilePollers::copyPollers(IOService * media)
  96 {
  97     IOPolledFilePollers * vars;
  98     IOReturn              err;
  99     IOService       * service;
 100     OSObject        * obj;
 101     IORegistryEntry * next;
 102     IORegistryEntry * child;
 103
 104     if ((obj = media->copyProperty(kIOPolledInterfaceStackKey)))
 105     {
 106         return (OSDynamicCast(IOPolledFilePollers, obj));
 107     }
 108
 109     do
 110     {
 111         vars = OSTypeAlloc(IOPolledFilePollers);
 112         vars->init();
 113
 114         vars->pollers = OSArray::withCapacity(4);
 115         if (!vars->pollers)
 116         {
 117             err = kIOReturnNoMemory;
 118             break;
 119         }
 120
 121         next = vars->media = media;
 122         do
 123         {
 124             IOPolledInterface * poller;
 125             OSObject *          obj;
 126
 127             obj = next->getProperty(kIOPolledInterfaceSupportKey);
 128             if (kOSBooleanFalse == obj)
 129             {
 130                 vars->pollers->flushCollection();
 131                 break;
 132             }
 133             else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
 134                 vars->pollers->setObject(poller);
 135
 136             if ((service = OSDynamicCast(IOService, next))
 137                 && service->getDeviceMemory()
 138                 && !vars->pollers->getCount())  break;
 139
 140             child = next;
 141         }
 142         while ((next = child->getParentEntry(gIOServicePlane))
 143                && child->isParent(next, gIOServicePlane, true));
 144
 145         if (!vars->pollers->getCount())
 146         {
 147             err = kIOReturnUnsupported;
 148             break;
 149         }
 150     }
 151     while (false);
 152
 153     media->setProperty(kIOPolledInterfaceStackKey, vars);
 154
 155     return (vars);
 156 }
 157
 158 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 159
 160 static IOReturn
 161 IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable);
 162
 163 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 164
 165 static IOReturn
 166 IOPolledFilePollersProbe(IOPolledFilePollers * vars)
 167 {
 168     IOReturn            err = kIOReturnError;
 169     int32_t             idx;
 170     IOPolledInterface * poller;
 171
 172     for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
 173     {
 174         poller = (IOPolledInterface *) vars->pollers->getObject(idx);
 175         err = poller->probe(vars->media);
 176         if (err)
 177         {
 178             HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
 179             break;
 180         }
 181     }
 182
 183     return (err);
 184 }
 185
 186 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 187
 188 IOReturn
 189 IOPolledFilePollersOpen(IOPolledFileIOVars * filevars, uint32_t state, bool abortable)
 190 {
 191
 192     IOPolledFilePollers      * vars = filevars->pollers;
 193     IOBufferMemoryDescriptor * ioBuffer;
 194     IOPolledInterface        * poller;
 195     IOService                * next;
 196     IOReturn                   err = kIOReturnError;
 197     int32_t                    idx;
 198
 199     vars->abortable = abortable;
 200     ioBuffer = 0;
 201
 202     if (kIOPolledAfterSleepState == state)
 203     {
 204         vars->ioStatus = 0;
 205         vars->io = false;
 206     }
 207     (void) IOPolledFilePollersIODone(vars, false);
 208
 209     if ((kIOPolledPreflightState == state) || (kIOPolledPreflightCoreDumpState == state))
 210     {
 211         ioBuffer = vars->ioBuffer;
 212         if (!ioBuffer)
 213         {
 214             vars->ioBuffer = ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionInOut,
 215                                                             2 * kDefaultIOSize, page_size);
 216             if (!ioBuffer) return (kIOReturnNoMemory);
 217         }
 218     }
 219
 220     for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
 221     {
 222         poller = (IOPolledInterface *) vars->pollers->getObject(idx);
 223         err = poller->open(state, ioBuffer);
 224         if (kIOReturnSuccess != err)
 225         {
 226             HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
 227             break;
 228         }
 229     }
 230     if ((kIOReturnSuccess == err) && (kIOPolledPreflightState == state))
 231     {
 232         next = vars->media;
 233         while (next)
 234         {
 235             next->setProperty(kIOPolledInterfaceActiveKey, kOSBooleanTrue);
 236             next = next->getProvider();
 237         }
 238     }
 239
 240     return (err);
 241 }
 242
 243 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 244
 245 IOReturn
 246 IOPolledFilePollersClose(IOPolledFileIOVars * filevars, uint32_t state)
 247 {
 248     IOPolledFilePollers * vars = filevars->pollers;
 249     IOPolledInterface * poller;
 250     IORegistryEntry *   next;
 251     IOReturn            err;
 252     int32_t             idx;
 253
 254     (void) IOPolledFilePollersIODone(vars, false);
 255
 256     if ((kIOPolledPostflightState == state) || (kIOPolledPostflightCoreDumpState == state))
 257     {
 258         vars->openCount--;
 259     }
 260
 261     for (idx = 0, err = kIOReturnSuccess;
 262          (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
 263          idx++)
 264     {
 265         err = poller->close(state);
 266         if ((kIOReturnSuccess != err) && (kIOPolledBeforeSleepStateAborted == state))
 267         {
 268             err = poller->close(kIOPolledBeforeSleepState);
 269         }
 270         if (err) HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
 271     }
 272
 273     if (kIOPolledPostflightState == state)
 274     {
 275         next = vars->media;
 276         while (next)
 277         {
 278             next->removeProperty(kIOPolledInterfaceActiveKey);
 279             next = next->getParentEntry(gIOServicePlane);
 280         }
 281     }
 282
 283     if ((kIOPolledPostflightState == state) || (kIOPolledPostflightCoreDumpState == state)) do
 284     {
 285         if (vars->openCount) break;
 286         if (vars->ioBuffer)
 287         {
 288             vars->ioBuffer->release();
 289             vars->ioBuffer = 0;
 290         }
 291     }
 292     while (false);
 293
 294     return (err);
 295 }
 296 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 297
 298 IOMemoryDescriptor *
 299 IOPolledFileGetIOBuffer(IOPolledFileIOVars * vars)
 300 {
 301     return (vars->pollers->ioBuffer);
 302 }
 303
 304 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 305
 306 static void
 307 IOPolledIOComplete(void *   target,
 308                    void *   parameter,
 309                    IOReturn status,
 310                    UInt64   actualByteCount)
 311 {
 312     IOPolledFilePollers * vars = (IOPolledFilePollers *) parameter;
 313
 314     vars->ioStatus = status;
 315 }
 316
 317 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 318
 319 static IOReturn
 320 IOStartPolledIO(IOPolledFilePollers * vars,
 321                     uint32_t operation, uint32_t bufferOffset,
 322                     uint64_t deviceOffset, uint64_t length)
 323 {
 324     IOReturn            err;
 325     IOPolledInterface * poller;
 326     IOPolledCompletion  completion;
 327
 328     err = vars->ioStatus;
 329     if (kIOReturnSuccess != err) return (err);
 330
 331     completion.target    = 0;
 332     completion.action    = &IOPolledIOComplete;
 333     completion.parameter = vars;
 334
 335     vars->ioStatus = -1;
 336
 337     poller = (IOPolledInterface *) vars->pollers->getObject(0);
 338     err = poller->startIO(operation, bufferOffset, deviceOffset, length, completion);
 339     if (err)
 340         HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
 341
 342     return (err);
 343 }
 344
 345 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 346
 347 static IOReturn
 348 IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable)
 349 {
 350     IOReturn            err = kIOReturnSuccess;
 351     int32_t             idx = 0;
 352     IOPolledInterface * poller;
 353     AbsoluteTime        deadline;
 354
 355     if (!vars->io) return (kIOReturnSuccess);
 356
 357     abortable &= vars->abortable;
 358
 359     clock_interval_to_deadline(2000, kMillisecondScale, &deadline);
 360
 361     while (-1 == vars->ioStatus)
 362     {
 363         for (idx = 0;
 364             (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
 365              idx++)
 366         {
 367             IOReturn newErr;
 368             newErr = poller->checkForWork();
 369             if ((newErr == kIOReturnAborted) && !abortable)
 370                 newErr = kIOReturnSuccess;
 371             if (kIOReturnSuccess == err)
 372                 err = newErr;
 373         }
 374         if ((false) && (kIOReturnSuccess == err) && (mach_absolute_time() > AbsoluteTime_to_scalar(&deadline)))
 375         {
 376             HIBLOG("IOPolledInterface::forced timeout\n");
 377             vars->ioStatus = kIOReturnTimeout;
 378         }
 379     }
 380     vars->io = false;
 381
 382 #if HIBERNATION
 383     if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort())
 384     {
 385         err = kIOReturnAborted;
 386         HIBLOG("IOPolledInterface::checkForWork sw abort\n");
 387     }
 388 #endif
 389
 390     if (err)
 391     {
 392         HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
 393     }
 394     else
 395     {
 396         err = vars->ioStatus;
 397         if (kIOReturnSuccess != err) HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
 398     }
 399
 400     return (err);
 401 }
 402
 403 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 404 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 405
 406 struct _OpenFileContext
 407 {
 408     OSData * extents;
 409     uint64_t size;
 410 };
 411
 412 static void
 413 file_extent_callback(void * ref, uint64_t start, uint64_t length)
 414 {
 415     _OpenFileContext * ctx = (_OpenFileContext *) ref;
 416     IOPolledFileExtent extent;
 417
 418     extent.start  = start;
 419     extent.length = length;
 420     ctx->extents->appendBytes(&extent, sizeof(extent));
 421     ctx->size += length;
 422 }
 423
 424 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 425
 426 static IOService *
 427 IOCopyMediaForDev(dev_t device)
 428 {
 429     OSDictionary * matching;
 430     OSNumber *     num;
 431     OSIterator *   iter;
 432     IOService *    result = 0;
 433
 434     matching = IOService::serviceMatching("IOMedia");
 435     if (!matching)
 436         return (0);
 437     do
 438     {
 439         num = OSNumber::withNumber(major(device), 32);
 440         if (!num)
 441             break;
 442         matching->setObject(kIOBSDMajorKey, num);
 443         num->release();
 444         num = OSNumber::withNumber(minor(device), 32);
 445         if (!num)
 446             break;
 447         matching->setObject(kIOBSDMinorKey, num);
 448         num->release();
 449         if (!num)
 450             break;
 451         iter = IOService::getMatchingServices(matching);
 452         if (iter)
 453         {
 454             result = (IOService *) iter->getNextObject();
 455             result->retain();
 456             iter->release();
 457         }
 458     }
 459     while (false);
 460     matching->release();
 461
 462     return (result);
 463 }
 464
 465 static IOReturn
 466 IOGetVolumeCryptKey(dev_t block_dev,  OSString ** pKeyUUID,
 467                     uint8_t * volumeCryptKey, size_t keySize)
 468 {
 469     IOReturn         err;
 470     IOService *      part;
 471     OSString *       keyUUID = 0;
 472     OSString *       keyStoreUUID = 0;
 473     uuid_t           volumeKeyUUID;
 474     aks_volume_key_t vek;
 475
 476     static IOService * sKeyStore;
 477
 478     part = IOCopyMediaForDev(block_dev);
 479     if (!part) return (kIOReturnNotFound);
 480
 481     err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false,
 482                                       (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL);
 483     if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID)
 484     {
 485 //            IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy());
 486
 487         if (!sKeyStore)
 488             sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane);
 489         if (sKeyStore)
 490             err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID);
 491         else
 492             err = kIOReturnNoResources;
 493         if (kIOReturnSuccess == err)
 494             err = sKeyStore->callPlatformFunction(gAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL);
 495         if (kIOReturnSuccess != err)
 496             IOLog("volume key err 0x%x\n", err);
 497         else
 498         {
 499             if (vek.key.keybytecount < keySize) keySize = vek.key.keybytecount;
 500             bcopy(&vek.key.keybytes[0], volumeCryptKey, keySize);
 501         }
 502         bzero(&vek, sizeof(vek));
 503
 504     }
 505     part->release();
 506     if (pKeyUUID) *pKeyUUID = keyUUID;
 507
 508     return (err);
 509 }
 510
 511 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 512
 513 IOReturn
 514 IOPolledFileOpen(const char * filename,
 515                  uint64_t setFileSize, uint64_t fsFreeSize,
 516                  void * write_file_addr, size_t write_file_len,
 517                  IOPolledFileIOVars ** fileVars,
 518                  OSData ** imagePath,
 519                  uint8_t * volumeCryptKey, size_t keySize)
 520 {
 521     IOReturn             err = kIOReturnSuccess;
 522     IOPolledFileIOVars * vars;
 523     _OpenFileContext     ctx;
 524     OSData *             extentsData;
 525     OSNumber *           num;
 526     IOService *          part = 0;
 527     dev_t                block_dev;
 528     dev_t                image_dev;
 529     AbsoluteTime         startTime, endTime;
 530     uint64_t             nsec;
 531
 532     vars = IONew(IOPolledFileIOVars, 1);
 533     if (!vars) return (kIOReturnNoMemory);
 534     bzero(vars, sizeof(*vars));
 535     vars->allocated = true;
 536
 537     do
 538     {
 539         extentsData = OSData::withCapacity(32);
 540         ctx.extents = extentsData;
 541         ctx.size    = 0;
 542         clock_get_uptime(&startTime);
 543
 544         vars->fileRef = kern_open_file_for_direct_io(filename,
 545                                                      (write_file_addr != NULL) || (0 != setFileSize),
 546                                                      &file_extent_callback, &ctx,
 547                                                      setFileSize,
 548                                                      fsFreeSize,
 549                                                      // write file:
 550                                                      0, write_file_addr, write_file_len,
 551                                                      // results
 552                                                      &block_dev,
 553                                                      &image_dev,
 554                                                      &vars->block0,
 555                                                      &vars->maxiobytes,
 556                                                      &vars->flags);
 557 #if 0
 558         uint32_t msDelay = (131071 & random());
 559         HIBLOG("sleep %d\n", msDelay);
 560         IOSleep(msDelay);
 561 #endif
 562         clock_get_uptime(&endTime);
 563         SUB_ABSOLUTETIME(&endTime, &startTime);
 564         absolutetime_to_nanoseconds(endTime, &nsec);
 565
 566         if (!vars->fileRef) err = kIOReturnNoSpace;
 567
 568         HIBLOG("kern_open_file_for_direct_io took %qd ms\n", nsec / 1000000ULL);
 569         if (kIOReturnSuccess != err) break;
 570
 571         HIBLOG("Opened file %s, size %qd, extents %ld, maxio %qx ssd %d\n", filename, ctx.size,
 572                     (extentsData->getLength() / sizeof(IOPolledFileExtent)) - 1,
 573                     vars->maxiobytes, kIOPolledFileSSD & vars->flags);
 574         assert(!vars->block0);
 575         if (extentsData->getLength() < sizeof(IOPolledFileExtent))
 576         {
 577             err = kIOReturnNoSpace;
 578             break;
 579         }
 580
 581         vars->fileSize = ctx.size;
 582         vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
 583
 584         part = IOCopyMediaForDev(image_dev);
 585         if (!part)
 586         {
 587             err = kIOReturnNotFound;
 588             break;
 589         }
 590
 591         if (!(vars->pollers = IOPolledFilePollers::copyPollers(part))) break;
 592
 593         if ((num = OSDynamicCast(OSNumber, part->getProperty(kIOMediaPreferredBlockSizeKey))))
 594                vars->blockSize = num->unsigned32BitValue();
 595         if (vars->blockSize < 4096) vars->blockSize = 4096;
 596
 597         HIBLOG("polled file major %d, minor %d, blocksize %ld, pollers %d\n",
 598                major(image_dev), minor(image_dev), (long)vars->blockSize,
 599                vars->pollers->pollers->getCount());
 600
 601         OSString * keyUUID = NULL;
 602         if (volumeCryptKey)
 603         {
 604             err = IOGetVolumeCryptKey(block_dev, &keyUUID, volumeCryptKey, keySize);
 605         }
 606
 607         *fileVars    = vars;
 608         vars->fileExtents = extentsData;
 609
 610         // make imagePath
 611         OSData * data;
 612         if (imagePath)
 613         {
 614 #if defined(__i386__) || defined(__x86_64__)
 615             char str2[24 + sizeof(uuid_string_t) + 2];
 616
 617             if (keyUUID)
 618                 snprintf(str2, sizeof(str2), "%qx:%s",
 619                                 vars->extentMap[0].start, keyUUID->getCStringNoCopy());
 620             else
 621                 snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
 622
 623             err = IOService::getPlatform()->callPlatformFunction(
 624                                                 gIOCreateEFIDevicePathSymbol, false,
 625                                                 (void *) part, (void *) str2,
 626                                                 (void *) (uintptr_t) true, (void *) &data);
 627 #else
 628             data = 0;
 629             err = kIOReturnSuccess;
 630 #endif
 631             if (kIOReturnSuccess != err)
 632             {
 633                 HIBLOG("error 0x%x getting path\n", err);
 634                 break;
 635             }
 636             *imagePath = data;
 637         }
 638     }
 639     while (false);
 640
 641     if (kIOReturnSuccess != err)
 642     {
 643         HIBLOG("error 0x%x opening polled file\n", err);
 644         IOPolledFileClose(&vars, 0, 0, 0, 0, 0);
 645     }
 646
 647     if (part) part->release();
 648
 649     return (err);
 650 }
 651
 652 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 653
 654 IOReturn
 655 IOPolledFileClose(IOPolledFileIOVars ** pVars,
 656                   off_t write_offset, void * addr, size_t write_length,
 657                   off_t discard_offset, off_t discard_end)
 658 {
 659     IOPolledFileIOVars * vars;
 660
 661     vars = *pVars;
 662     if (!vars) return(kIOReturnSuccess);
 663
 664     if (vars->fileRef)
 665     {
 666         kern_close_file_for_direct_io(vars->fileRef, write_offset, addr, write_length,
 667                                       discard_offset, discard_end);
 668         vars->fileRef = NULL;
 669     }
 670     if (vars->fileExtents)
 671     {
 672         vars->fileExtents->release();
 673         vars->fileExtents = 0;
 674     }
 675     if (vars->pollers)
 676     {
 677         vars->pollers->release();
 678         vars->pollers = 0;
 679     }
 680
 681     if (vars->allocated) IODelete(vars, IOPolledFileIOVars, 1);
 682     else                 bzero(vars, sizeof(IOPolledFileIOVars));
 683     *pVars = NULL;
 684
 685     return (kIOReturnSuccess);
 686 }
 687
 688 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 689
 690 IOReturn
 691 IOPolledFilePollersSetup(IOPolledFileIOVars * vars,
 692                          uint32_t openState)
 693 {
 694     IOReturn err;
 695
 696     err = kIOReturnSuccess;
 697     do
 698     {
 699         if (!vars->pollers->openCount)
 700         {
 701             err = IOPolledFilePollersProbe(vars->pollers);
 702             if (kIOReturnSuccess != err) break;
 703         }
 704         err = IOPolledFilePollersOpen(vars, openState, false);
 705         if (kIOReturnSuccess != err) break;
 706         if ((kIOPolledPreflightState == openState) || (kIOPolledPreflightCoreDumpState == openState))
 707         {
 708             vars->pollers->openCount++;
 709         }
 710         vars->pollers->io  = false;
 711         vars->buffer       = (uint8_t *) vars->pollers->ioBuffer->getBytesNoCopy();
 712         vars->bufferHalf   = 0;
 713         vars->bufferOffset = 0;
 714         vars->bufferSize   = (vars->pollers->ioBuffer->getLength() >> 1);
 715
 716         if (vars->maxiobytes < vars->bufferSize) vars->bufferSize = vars->maxiobytes;
 717     }
 718     while (false);
 719
 720     if (kIOReturnSuccess != err) HIBLOG("IOPolledFilePollersSetup(%d) error 0x%x\n", openState, err);
 721
 722     return (err);
 723 }
 724
 725
 726 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 727
 728 IOReturn
 729 IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
 730 {
 731     IOPolledFileExtent * extentMap;
 732
 733     extentMap = vars->extentMap;
 734
 735     vars->position = position;
 736
 737     while (position >= extentMap->length)
 738     {
 739         position -= extentMap->length;
 740         extentMap++;
 741     }
 742
 743     vars->currentExtent   = extentMap;
 744     vars->extentRemaining = extentMap->length - position;
 745     vars->extentPosition  = vars->position - position;
 746
 747     if (vars->bufferSize <= vars->extentRemaining)
 748         vars->bufferLimit = vars->bufferSize;
 749     else
 750         vars->bufferLimit = vars->extentRemaining;
 751
 752     return (kIOReturnSuccess);
 753 }
 754
 755 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 756
 757 IOReturn
 758 IOPolledFileWrite(IOPolledFileIOVars * vars,
 759                     const uint8_t * bytes, IOByteCount size,
 760                     IOPolledFileCryptVars * cryptvars)
 761 {
 762     IOReturn    err = kIOReturnSuccess;
 763     IOByteCount copy;
 764     bool        flush = false;
 765
 766     do
 767     {
 768         if (!bytes && !size)
 769         {
 770             // seek to end of block & flush
 771             size = vars->position & (vars->blockSize - 1);
 772             if (size)
 773                 size = vars->blockSize - size;
 774             flush = true;
 775             // use some garbage for the fill
 776             bytes = vars->buffer + vars->bufferOffset;
 777         }
 778
 779         copy = vars->bufferLimit - vars->bufferOffset;
 780         if (copy > size)
 781             copy = size;
 782         else
 783             flush = true;
 784
 785         if (bytes)
 786         {
 787             bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
 788             bytes += copy;
 789         }
 790         else
 791             bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
 792
 793         size -= copy;
 794         vars->bufferOffset += copy;
 795         vars->position += copy;
 796
 797         if (flush && vars->bufferOffset)
 798         {
 799             uint64_t offset = (vars->position - vars->bufferOffset
 800                                 - vars->extentPosition + vars->currentExtent->start);
 801             uint32_t length = (vars->bufferOffset);
 802
 803 #if CRYPTO
 804             if (cryptvars && vars->encryptStart
 805                 && (vars->position > vars->encryptStart)
 806                 && ((vars->position - length) < vars->encryptEnd))
 807             {
 808                 AbsoluteTime startTime, endTime;
 809
 810                 uint64_t encryptLen, encryptStart;
 811                 encryptLen = vars->position - vars->encryptStart;
 812                 if (encryptLen > length)
 813                     encryptLen = length;
 814                 encryptStart = length - encryptLen;
 815                 if (vars->position > vars->encryptEnd)
 816                     encryptLen -= (vars->position - vars->encryptEnd);
 817
 818                 clock_get_uptime(&startTime);
 819
 820                 // encrypt the buffer
 821                 aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
 822                                 &cryptvars->aes_iv[0],
 823                                 encryptLen / AES_BLOCK_SIZE,
 824                                 vars->buffer + vars->bufferHalf + encryptStart,
 825                                 &cryptvars->ctx.encrypt);
 826
 827                 clock_get_uptime(&endTime);
 828                 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
 829                 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
 830                 vars->cryptBytes += encryptLen;
 831
 832                 // save initial vector for following encrypts
 833                 bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
 834                         &cryptvars->aes_iv[0],
 835                         AES_BLOCK_SIZE);
 836             }
 837 #endif /* CRYPTO */
 838
 839             err = IOPolledFilePollersIODone(vars->pollers, true);
 840             if (kIOReturnSuccess != err)
 841                 break;
 842
 843 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
 844 //if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
 845
 846             err = IOStartPolledIO(vars->pollers, kIOPolledWrite, vars->bufferHalf, offset, length);
 847             if (kIOReturnSuccess != err)
 848                 break;
 849             vars->pollers->io = true;
 850
 851             vars->extentRemaining -= vars->bufferOffset;
 852             if (!vars->extentRemaining)
 853             {
 854                 vars->currentExtent++;
 855                 vars->extentRemaining = vars->currentExtent->length;
 856                 vars->extentPosition  = vars->position;
 857             }
 858
 859             vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
 860             vars->bufferOffset = 0;
 861             if (vars->bufferSize <= vars->extentRemaining)
 862                 vars->bufferLimit = vars->bufferSize;
 863             else
 864                 vars->bufferLimit = vars->extentRemaining;
 865
 866             if (!vars->extentRemaining)
 867             {
 868                 err = kIOReturnOverrun;
 869                 break;
 870             }
 871
 872             flush = false;
 873         }
 874     }
 875     while (size);
 876
 877     return (err);
 878 }
 879
 880 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 881
 882 IOReturn
 883 IOPolledFileRead(IOPolledFileIOVars * vars,
 884                     uint8_t * bytes, IOByteCount size,
 885                     IOPolledFileCryptVars * cryptvars)
 886 {
 887     IOReturn    err = kIOReturnSuccess;
 888     IOByteCount copy;
 889
 890 //    bytesWritten += size;
 891
 892     do
 893     {
 894         copy = vars->bufferLimit - vars->bufferOffset;
 895         if (copy > size)
 896             copy = size;
 897
 898         if (bytes)
 899         {
 900             bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
 901             bytes += copy;
 902         }
 903         size -= copy;
 904         vars->bufferOffset += copy;
 905 //      vars->position += copy;
 906
 907         if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd))
 908         {
 909             if (!vars->pollers->io) cryptvars = 0;
 910             err = IOPolledFilePollersIODone(vars->pollers, true);
 911             if (kIOReturnSuccess != err)
 912                 break;
 913
 914 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
 915
 916             vars->position        += vars->lastRead;
 917             vars->extentRemaining -= vars->lastRead;
 918             vars->bufferLimit      = vars->lastRead;
 919
 920             if (!vars->extentRemaining)
 921             {
 922                 vars->currentExtent++;
 923                 vars->extentRemaining = vars->currentExtent->length;
 924                 vars->extentPosition  = vars->position;
 925                 if (!vars->extentRemaining)
 926                 {
 927                     err = kIOReturnOverrun;
 928                     break;
 929                 }
 930             }
 931
 932             uint64_t length;
 933             uint64_t lastReadLength = vars->lastRead;
 934             uint64_t offset = (vars->position
 935                                 - vars->extentPosition + vars->currentExtent->start);
 936             if (vars->extentRemaining <= vars->bufferSize)
 937                 length = vars->extentRemaining;
 938             else
 939                 length = vars->bufferSize;
 940             if ((length + vars->position) > vars->readEnd)
 941                 length = vars->readEnd - vars->position;
 942
 943             vars->lastRead = length;
 944             if (length)
 945             {
 946 //if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
 947                 err = IOStartPolledIO(vars->pollers, kIOPolledRead, vars->bufferHalf, offset, length);
 948                 if (kIOReturnSuccess != err)
 949                     break;
 950                 vars->pollers->io = true;
 951             }
 952
 953             vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
 954             vars->bufferOffset = 0;
 955
 956 #if CRYPTO
 957             if (cryptvars)
 958             {
 959                 uint8_t thisVector[AES_BLOCK_SIZE];
 960                 AbsoluteTime startTime, endTime;
 961
 962                 // save initial vector for following decrypts
 963                 bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
 964                 bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE,
 965                         &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
 966
 967                 // decrypt the buffer
 968                 clock_get_uptime(&startTime);
 969
 970                 aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
 971                                 &thisVector[0],
 972                                 lastReadLength / AES_BLOCK_SIZE,
 973                                 vars->buffer + vars->bufferHalf,
 974                                 &cryptvars->ctx.decrypt);
 975
 976                 clock_get_uptime(&endTime);
 977                 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
 978                 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
 979                 vars->cryptBytes += lastReadLength;
 980             }
 981 #endif /* CRYPTO */
 982         }
 983     }
 984     while (size);
 985
 986     return (err);
 987 }
 988
 989 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 990