iokit/Kernel/IOPolledInterface.cpp

   1 /*
   2  * Copyright (c) 2006-2009 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <sys/uio.h>
  30 #include <sys/conf.h>
  31
  32 #include <IOKit/IOLib.h>
  33 #include <IOKit/IOBSD.h>
  34 #include <IOKit/IOService.h>
  35 #include <IOKit/IOPlatformExpert.h>
  36 #include <IOKit/IOPolledInterface.h>
  37 #include <IOKit/IOHibernatePrivate.h>
  38 #include <IOKit/IOBufferMemoryDescriptor.h>
  39 #include <IOKit/AppleKeyStoreInterface.h>
  40 #include "IOKitKernelInternal.h"
  41
  42
  43 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  44
  45 OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject);
  46
  47 OSMetaClassDefineReservedUnused(IOPolledInterface, 0);
  48 OSMetaClassDefineReservedUnused(IOPolledInterface, 1);
  49 OSMetaClassDefineReservedUnused(IOPolledInterface, 2);
  50 OSMetaClassDefineReservedUnused(IOPolledInterface, 3);
  51 OSMetaClassDefineReservedUnused(IOPolledInterface, 4);
  52 OSMetaClassDefineReservedUnused(IOPolledInterface, 5);
  53 OSMetaClassDefineReservedUnused(IOPolledInterface, 6);
  54 OSMetaClassDefineReservedUnused(IOPolledInterface, 7);
  55 OSMetaClassDefineReservedUnused(IOPolledInterface, 8);
  56 OSMetaClassDefineReservedUnused(IOPolledInterface, 9);
  57 OSMetaClassDefineReservedUnused(IOPolledInterface, 10);
  58 OSMetaClassDefineReservedUnused(IOPolledInterface, 11);
  59 OSMetaClassDefineReservedUnused(IOPolledInterface, 12);
  60 OSMetaClassDefineReservedUnused(IOPolledInterface, 13);
  61 OSMetaClassDefineReservedUnused(IOPolledInterface, 14);
  62 OSMetaClassDefineReservedUnused(IOPolledInterface, 15);
  63
  64 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  65
  66 #ifndef kIOMediaPreferredBlockSizeKey
  67 #define kIOMediaPreferredBlockSizeKey   "Preferred Block Size"
  68 #endif
  69
  70 enum { kDefaultIOSize = 128*1024 };
  71
  72 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  73
  74 class IOPolledFilePollers : public OSObject
  75 {
  76     OSDeclareDefaultStructors(IOPolledFilePollers)
  77
  78 public:
  79     IOService                * media;
  80     OSArray                  * pollers;
  81     IOBufferMemoryDescriptor * ioBuffer;
  82     bool                 abortable;
  83     bool                 io;
  84     IOReturn             ioStatus;
  85     uint32_t             openCount;
  86
  87     static IOPolledFilePollers * copyPollers(IOService * media);
  88 };
  89
  90 OSDefineMetaClassAndStructors(IOPolledFilePollers, OSObject)
  91
  92 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  93
  94 IOPolledFilePollers *
  95 IOPolledFilePollers::copyPollers(IOService * media)
  96 {
  97     IOPolledFilePollers * vars;
  98     IOReturn              err;
  99     IOService       * service;
 100     OSObject        * obj;
 101     IORegistryEntry * next;
 102     IORegistryEntry * child;
 103
 104     if ((obj = media->copyProperty(kIOPolledInterfaceStackKey)))
 105     {
 106         return (OSDynamicCast(IOPolledFilePollers, obj));
 107     }
 108
 109     do
 110     {
 111         vars = OSTypeAlloc(IOPolledFilePollers);
 112         vars->init();
 113
 114         vars->pollers = OSArray::withCapacity(4);
 115         if (!vars->pollers)
 116         {
 117             err = kIOReturnNoMemory;
 118             break;
 119         }
 120
 121         next = vars->media = media;
 122         do
 123         {
 124             IOPolledInterface * poller;
 125             OSObject *          obj;
 126
 127             obj = next->getProperty(kIOPolledInterfaceSupportKey);
 128             if (kOSBooleanFalse == obj)
 129             {
 130                 vars->pollers->flushCollection();
 131                 break;
 132             }
 133             else if ((poller = OSDynamicCast(IOPolledInterface, obj)))
 134                 vars->pollers->setObject(poller);
 135
 136             if ((service = OSDynamicCast(IOService, next))
 137                 && service->getDeviceMemory()
 138                 && !vars->pollers->getCount())  break;
 139
 140             child = next;
 141         }
 142         while ((next = child->getParentEntry(gIOServicePlane))
 143                && child->isParent(next, gIOServicePlane, true));
 144
 145         if (!vars->pollers->getCount())
 146         {
 147             err = kIOReturnUnsupported;
 148             break;
 149         }
 150     }
 151     while (false);
 152
 153     media->setProperty(kIOPolledInterfaceStackKey, vars);
 154
 155     return (vars);
 156 }
 157
 158 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 159
 160 static IOReturn
 161 IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable);
 162
 163 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 164
 165 static IOReturn
 166 IOPolledFilePollersProbe(IOPolledFilePollers * vars)
 167 {
 168     IOReturn            err = kIOReturnError;
 169     int32_t             idx;
 170     IOPolledInterface * poller;
 171
 172     for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
 173     {
 174         poller = (IOPolledInterface *) vars->pollers->getObject(idx);
 175         err = poller->probe(vars->media);
 176         if (err)
 177         {
 178             HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
 179             break;
 180         }
 181     }
 182
 183     return (err);
 184 }
 185
 186 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 187
 188 IOReturn
 189 IOPolledFilePollersOpen(IOPolledFileIOVars * filevars, uint32_t state, bool abortable)
 190 {
 191
 192     IOPolledFilePollers      * vars = filevars->pollers;
 193     IOBufferMemoryDescriptor * ioBuffer;
 194     IOPolledInterface        * poller;
 195     IOService                * next;
 196     IOReturn                   err = kIOReturnError;
 197     int32_t                    idx;
 198
 199     vars->abortable = abortable;
 200     ioBuffer = 0;
 201
 202     if (kIOPolledAfterSleepState == state)
 203     {
 204         vars->ioStatus = 0;
 205         vars->io = false;
 206     }
 207     (void) IOPolledFilePollersIODone(vars, false);
 208
 209     if ((kIOPolledPreflightState == state) || (kIOPolledPreflightCoreDumpState == state))
 210     {
 211         ioBuffer = vars->ioBuffer;
 212         if (!ioBuffer)
 213         {
 214             vars->ioBuffer = ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionInOut,
 215                                                             2 * kDefaultIOSize, page_size);
 216             if (!ioBuffer) return (kIOReturnNoMemory);
 217         }
 218     }
 219
 220     for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--)
 221     {
 222         poller = (IOPolledInterface *) vars->pollers->getObject(idx);
 223         err = poller->open(state, ioBuffer);
 224         if (kIOReturnSuccess != err)
 225         {
 226             HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
 227             break;
 228         }
 229     }
 230     if ((kIOReturnSuccess == err) && (kIOPolledPreflightState == state))
 231     {
 232         next = vars->media;
 233         while (next)
 234         {
 235             next->setProperty(kIOPolledInterfaceActiveKey, kOSBooleanTrue);
 236             next = next->getProvider();
 237         }
 238     }
 239
 240     return (err);
 241 }
 242
 243 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 244
 245 IOReturn
 246 IOPolledFilePollersClose(IOPolledFileIOVars * filevars, uint32_t state)
 247 {
 248     IOPolledFilePollers * vars = filevars->pollers;
 249     IOPolledInterface * poller;
 250     IORegistryEntry *   next;
 251     IOReturn            err;
 252     int32_t             idx;
 253
 254     (void) IOPolledFilePollersIODone(vars, false);
 255
 256     if ((kIOPolledPostflightState == state) || (kIOPolledPostflightCoreDumpState == state))
 257     {
 258         vars->openCount--;
 259     }
 260
 261     for (idx = 0, err = kIOReturnSuccess;
 262          (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
 263          idx++)
 264     {
 265         err = poller->close(state);
 266         if ((kIOReturnSuccess != err) && (kIOPolledBeforeSleepStateAborted == state))
 267         {
 268             err = poller->close(kIOPolledBeforeSleepState);
 269         }
 270         if (err) HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
 271     }
 272
 273     if (kIOPolledPostflightState == state)
 274     {
 275         next = vars->media;
 276         while (next)
 277         {
 278             next->removeProperty(kIOPolledInterfaceActiveKey);
 279             next = next->getParentEntry(gIOServicePlane);
 280         }
 281     }
 282
 283     if ((kIOPolledPostflightState == state) || (kIOPolledPostflightCoreDumpState == state)) do
 284     {
 285         if (vars->openCount) break;
 286         if (vars->ioBuffer)
 287         {
 288             vars->ioBuffer->release();
 289             vars->ioBuffer = 0;
 290         }
 291     }
 292     while (false);
 293
 294     return (err);
 295 }
 296 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 297
 298 IOMemoryDescriptor *
 299 IOPolledFileGetIOBuffer(IOPolledFileIOVars * vars)
 300 {
 301     return (vars->pollers->ioBuffer);
 302 }
 303
 304 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 305
 306 static void
 307 IOPolledIOComplete(void *   target,
 308                    void *   parameter,
 309                    IOReturn status,
 310                    UInt64   actualByteCount)
 311 {
 312     IOPolledFilePollers * vars = (IOPolledFilePollers *) parameter;
 313
 314     vars->ioStatus = status;
 315 }
 316
 317 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 318
 319 static IOReturn
 320 IOStartPolledIO(IOPolledFilePollers * vars,
 321                     uint32_t operation, uint32_t bufferOffset,
 322                     uint64_t deviceOffset, uint64_t length)
 323 {
 324     IOReturn            err;
 325     IOPolledInterface * poller;
 326     IOPolledCompletion  completion;
 327
 328     err = vars->ioStatus;
 329     if (kIOReturnSuccess != err) return (err);
 330
 331     completion.target    = 0;
 332     completion.action    = &IOPolledIOComplete;
 333     completion.parameter = vars;
 334
 335     vars->ioStatus = -1;
 336
 337     poller = (IOPolledInterface *) vars->pollers->getObject(0);
 338     err = poller->startIO(operation, bufferOffset, deviceOffset, length, completion);
 339     if (err) {
 340         if (kernel_debugger_entry_count) {
 341             HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
 342         } else {
 343             HIBLOGFROMPANIC("IOPolledInterface::IOStartPolledIO(0x%p, %d, 0x%x, 0x%llx, %llu) : poller->startIO(%d, 0x%x, 0x%llx, %llu, completion) returned 0x%x",
 344                         vars, operation, bufferOffset, deviceOffset, length, operation, bufferOffset, deviceOffset, length, err);
 345         }
 346     }
 347     return (err);
 348 }
 349
 350 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 351
 352 static IOReturn
 353 IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable)
 354 {
 355     IOReturn            err = kIOReturnSuccess;
 356     int32_t             idx = 0;
 357     IOPolledInterface * poller;
 358     AbsoluteTime        deadline;
 359
 360     if (!vars->io) return (kIOReturnSuccess);
 361
 362     abortable &= vars->abortable;
 363
 364     clock_interval_to_deadline(2000, kMillisecondScale, &deadline);
 365
 366     while (-1 == vars->ioStatus)
 367     {
 368         for (idx = 0;
 369             (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
 370              idx++)
 371         {
 372             IOReturn newErr;
 373             newErr = poller->checkForWork();
 374             if ((newErr == kIOReturnAborted) && !abortable)
 375                 newErr = kIOReturnSuccess;
 376             if (kIOReturnSuccess == err)
 377                 err = newErr;
 378         }
 379         if ((false) && (kIOReturnSuccess == err) && (mach_absolute_time() > AbsoluteTime_to_scalar(&deadline)))
 380         {
 381             HIBLOG("IOPolledInterface::forced timeout\n");
 382             vars->ioStatus = kIOReturnTimeout;
 383         }
 384     }
 385     vars->io = false;
 386
 387 #if HIBERNATION
 388     if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort())
 389     {
 390         err = kIOReturnAborted;
 391         HIBLOG("IOPolledInterface::checkForWork sw abort\n");
 392     }
 393 #endif
 394
 395     if (err)
 396     {
 397         HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
 398     }
 399     else
 400     {
 401         err = vars->ioStatus;
 402         if (kIOReturnSuccess != err) HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
 403     }
 404
 405     return (err);
 406 }
 407
 408 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 409 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 410
 411 struct _OpenFileContext
 412 {
 413     OSData * extents;
 414     uint64_t size;
 415 };
 416
 417 static void
 418 file_extent_callback(void * ref, uint64_t start, uint64_t length)
 419 {
 420     _OpenFileContext * ctx = (_OpenFileContext *) ref;
 421     IOPolledFileExtent extent;
 422
 423     extent.start  = start;
 424     extent.length = length;
 425     ctx->extents->appendBytes(&extent, sizeof(extent));
 426     ctx->size += length;
 427 }
 428
 429 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 430
 431 static IOService *
 432 IOCopyMediaForDev(dev_t device)
 433 {
 434     OSDictionary * matching;
 435     OSNumber *     num;
 436     OSIterator *   iter;
 437     IOService *    result = 0;
 438
 439     matching = IOService::serviceMatching("IOMedia");
 440     if (!matching)
 441         return (0);
 442     do
 443     {
 444         num = OSNumber::withNumber(major(device), 32);
 445         if (!num)
 446             break;
 447         matching->setObject(kIOBSDMajorKey, num);
 448         num->release();
 449         num = OSNumber::withNumber(minor(device), 32);
 450         if (!num)
 451             break;
 452         matching->setObject(kIOBSDMinorKey, num);
 453         num->release();
 454         if (!num)
 455             break;
 456         iter = IOService::getMatchingServices(matching);
 457         if (iter)
 458         {
 459             result = (IOService *) iter->getNextObject();
 460             result->retain();
 461             iter->release();
 462         }
 463     }
 464     while (false);
 465     matching->release();
 466
 467     return (result);
 468 }
 469
 470 #define APFSMEDIA_GETHIBERKEY         "getHiberKey"
 471
 472 static IOReturn
 473 IOGetVolumeCryptKey(dev_t block_dev,  OSString ** pKeyUUID,
 474                     uint8_t * volumeCryptKey, size_t keySize)
 475 {
 476     IOReturn         err;
 477     IOService *      part;
 478     OSString *       keyUUID = 0;
 479     OSString *       keyStoreUUID = 0;
 480     uuid_t           volumeKeyUUID;
 481     aks_volume_key_t vek;
 482
 483     static IOService * sKeyStore;
 484
 485     part = IOCopyMediaForDev(block_dev);
 486     if (!part) return (kIOReturnNotFound);
 487
 488     // Try APFS first
 489     {
 490         uuid_t volUuid = {0};
 491         size_t sizeOut = 0;
 492         err = part->callPlatformFunction(APFSMEDIA_GETHIBERKEY, false, &volUuid, volumeCryptKey, &keySize, &sizeOut);
 493         if (err == kIOReturnSuccess)
 494         {
 495             // No need to create uuid string if it's not requested
 496             if (pKeyUUID)
 497             {
 498                 uuid_string_t volUuidStr;
 499                 uuid_unparse(volUuid, volUuidStr);
 500                 *pKeyUUID = OSString::withCString(volUuidStr);
 501             }
 502
 503             part->release();
 504             return kIOReturnSuccess;
 505         }
 506     }
 507
 508     // Then old CS path
 509     err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false,
 510                   (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL);
 511     if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID)
 512     {
 513 //        IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy());
 514
 515         if (!sKeyStore)
 516             sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane);
 517         if (sKeyStore)
 518             err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID);
 519         else
 520             err = kIOReturnNoResources;
 521         if (kIOReturnSuccess == err)
 522             err = sKeyStore->callPlatformFunction(gAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL);
 523         if (kIOReturnSuccess != err)
 524             IOLog("volume key err 0x%x\n", err);
 525         else
 526         {
 527             if (vek.key.keybytecount < keySize) keySize = vek.key.keybytecount;
 528             bcopy(&vek.key.keybytes[0], volumeCryptKey, keySize);
 529         }
 530         bzero(&vek, sizeof(vek));
 531
 532         if (pKeyUUID)
 533         {
 534             // Create a copy because the caller would release it
 535             *pKeyUUID = OSString::withString(keyUUID);
 536         }
 537     }
 538
 539     part->release();
 540     return (err);
 541 }
 542
 543 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 544
 545 IOReturn
 546 IOPolledFileOpen(const char * filename,
 547                  uint64_t setFileSize, uint64_t fsFreeSize,
 548                  void * write_file_addr, size_t write_file_len,
 549                  IOPolledFileIOVars ** fileVars,
 550                  OSData ** imagePath,
 551                  uint8_t * volumeCryptKey, size_t keySize)
 552 {
 553     IOReturn             err = kIOReturnSuccess;
 554     IOPolledFileIOVars * vars;
 555     _OpenFileContext     ctx;
 556     OSData *             extentsData = NULL;
 557     OSNumber *           num;
 558     IOService *          part = 0;
 559     dev_t                block_dev;
 560     dev_t                image_dev;
 561     AbsoluteTime         startTime, endTime;
 562     uint64_t             nsec;
 563
 564     vars = IONew(IOPolledFileIOVars, 1);
 565     if (!vars) return (kIOReturnNoMemory);
 566     bzero(vars, sizeof(*vars));
 567     vars->allocated = true;
 568
 569     do
 570     {
 571         extentsData = OSData::withCapacity(32);
 572         ctx.extents = extentsData;
 573         ctx.size    = 0;
 574         clock_get_uptime(&startTime);
 575
 576         vars->fileRef = kern_open_file_for_direct_io(filename,
 577                                                      (write_file_addr != NULL) || (0 != setFileSize),
 578                                                      &file_extent_callback, &ctx,
 579                                                      setFileSize,
 580                                                      fsFreeSize,
 581                                                      // write file:
 582                                                      0, write_file_addr, write_file_len,
 583                                                      // results
 584                                                      &block_dev,
 585                                                      &image_dev,
 586                                                      &vars->block0,
 587                                                      &vars->maxiobytes,
 588                                                      &vars->flags);
 589 #if 0
 590         uint32_t msDelay = (131071 & random());
 591         HIBLOG("sleep %d\n", msDelay);
 592         IOSleep(msDelay);
 593 #endif
 594         clock_get_uptime(&endTime);
 595         SUB_ABSOLUTETIME(&endTime, &startTime);
 596         absolutetime_to_nanoseconds(endTime, &nsec);
 597
 598         if (!vars->fileRef) err = kIOReturnNoSpace;
 599
 600         HIBLOG("kern_open_file_for_direct_io took %qd ms\n", nsec / 1000000ULL);
 601         if (kIOReturnSuccess != err) break;
 602
 603         HIBLOG("Opened file %s, size %qd, extents %ld, maxio %qx ssd %d\n", filename, ctx.size,
 604                (extentsData->getLength() / sizeof(IOPolledFileExtent)) - 1,
 605                vars->maxiobytes, kIOPolledFileSSD & vars->flags);
 606         assert(!vars->block0);
 607         if (extentsData->getLength() < sizeof(IOPolledFileExtent))
 608         {
 609             err = kIOReturnNoSpace;
 610             break;
 611         }
 612
 613         vars->fileSize = ctx.size;
 614         vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
 615
 616         part = IOCopyMediaForDev(image_dev);
 617         if (!part)
 618         {
 619             err = kIOReturnNotFound;
 620             break;
 621         }
 622
 623         if (!(vars->pollers = IOPolledFilePollers::copyPollers(part))) break;
 624
 625         if ((num = OSDynamicCast(OSNumber, part->getProperty(kIOMediaPreferredBlockSizeKey))))
 626             vars->blockSize = num->unsigned32BitValue();
 627         if (vars->blockSize < 4096) vars->blockSize = 4096;
 628
 629         HIBLOG("polled file major %d, minor %d, blocksize %ld, pollers %d\n",
 630                major(image_dev), minor(image_dev), (long)vars->blockSize,
 631                vars->pollers->pollers->getCount());
 632
 633         OSString * keyUUID = NULL;
 634         if (volumeCryptKey)
 635         {
 636             err = IOGetVolumeCryptKey(block_dev, &keyUUID, volumeCryptKey, keySize);
 637         }
 638
 639         *fileVars    = vars;
 640         vars->fileExtents = extentsData;
 641
 642         // make imagePath
 643         OSData * data;
 644         if (imagePath)
 645         {
 646 #if defined(__i386__) || defined(__x86_64__)
 647             char str2[24 + sizeof(uuid_string_t) + 2];
 648
 649             if (keyUUID)
 650                 snprintf(str2, sizeof(str2), "%qx:%s",
 651                          vars->extentMap[0].start, keyUUID->getCStringNoCopy());
 652             else
 653                 snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
 654
 655             err = IOService::getPlatform()->callPlatformFunction(
 656                                                                  gIOCreateEFIDevicePathSymbol, false,
 657                                                                  (void *) part, (void *) str2,
 658                                                                  (void *) (uintptr_t) true, (void *) &data);
 659 #else
 660             data = 0;
 661             err = kIOReturnSuccess;
 662 #endif
 663             if (kIOReturnSuccess != err)
 664             {
 665                 HIBLOG("error 0x%x getting path\n", err);
 666                 break;
 667             }
 668             *imagePath = data;
 669         }
 670
 671         // Release key UUID if we have one
 672         if (keyUUID)
 673         {
 674             keyUUID->release();
 675             keyUUID = NULL; // Just in case
 676         }
 677     }
 678     while (false);
 679
 680     if (kIOReturnSuccess != err)
 681     {
 682         HIBLOG("error 0x%x opening polled file\n", err);
 683         IOPolledFileClose(&vars, 0, 0, 0, 0, 0);
 684         if (extentsData) extentsData->release();
 685     }
 686
 687     if (part) part->release();
 688
 689     return (err);
 690 }
 691
 692 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 693
 694 IOReturn
 695 IOPolledFileClose(IOPolledFileIOVars ** pVars,
 696                   off_t write_offset, void * addr, size_t write_length,
 697                   off_t discard_offset, off_t discard_end)
 698 {
 699     IOPolledFileIOVars * vars;
 700
 701     vars = *pVars;
 702     if (!vars) return(kIOReturnSuccess);
 703
 704     if (vars->fileRef)
 705     {
 706         kern_close_file_for_direct_io(vars->fileRef, write_offset, addr, write_length,
 707                                       discard_offset, discard_end);
 708         vars->fileRef = NULL;
 709     }
 710     if (vars->fileExtents)
 711     {
 712         vars->fileExtents->release();
 713         vars->fileExtents = 0;
 714     }
 715     if (vars->pollers)
 716     {
 717         vars->pollers->release();
 718         vars->pollers = 0;
 719     }
 720
 721     if (vars->allocated) IODelete(vars, IOPolledFileIOVars, 1);
 722     else                 bzero(vars, sizeof(IOPolledFileIOVars));
 723     *pVars = NULL;
 724
 725     return (kIOReturnSuccess);
 726 }
 727
 728 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 729
 730 IOReturn
 731 IOPolledFilePollersSetup(IOPolledFileIOVars * vars,
 732                          uint32_t openState)
 733 {
 734     IOReturn err;
 735
 736     err = kIOReturnSuccess;
 737     do
 738     {
 739         if (!vars->pollers->openCount)
 740         {
 741             err = IOPolledFilePollersProbe(vars->pollers);
 742             if (kIOReturnSuccess != err) break;
 743         }
 744         err = IOPolledFilePollersOpen(vars, openState, false);
 745         if (kIOReturnSuccess != err) break;
 746         if ((kIOPolledPreflightState == openState) || (kIOPolledPreflightCoreDumpState == openState))
 747         {
 748             vars->pollers->openCount++;
 749         }
 750         vars->pollers->io  = false;
 751         vars->buffer       = (uint8_t *) vars->pollers->ioBuffer->getBytesNoCopy();
 752         vars->bufferHalf   = 0;
 753         vars->bufferOffset = 0;
 754         vars->bufferSize   = (vars->pollers->ioBuffer->getLength() >> 1);
 755
 756         if (vars->maxiobytes < vars->bufferSize) vars->bufferSize = vars->maxiobytes;
 757     }
 758     while (false);
 759
 760     if (kIOReturnSuccess != err) HIBLOG("IOPolledFilePollersSetup(%d) error 0x%x\n", openState, err);
 761
 762     return (err);
 763 }
 764
 765
 766 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 767
 768 IOReturn
 769 IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
 770 {
 771     IOPolledFileExtent * extentMap;
 772
 773     extentMap = vars->extentMap;
 774
 775     vars->position = position;
 776
 777     if (position > vars->fileSize) {
 778         HIBLOG("IOPolledFileSeek: called to seek to 0x%llx greater than file size of 0x%llx\n", vars->position,  vars->fileSize);
 779         return kIOReturnNoSpace;
 780     }
 781
 782     while (position >= extentMap->length)
 783     {
 784         position -= extentMap->length;
 785         extentMap++;
 786     }
 787
 788     vars->currentExtent   = extentMap;
 789     vars->extentRemaining = extentMap->length - position;
 790     vars->extentPosition  = vars->position - position;
 791
 792     if (vars->bufferSize <= vars->extentRemaining)
 793         vars->bufferLimit = vars->bufferSize;
 794     else
 795         vars->bufferLimit = vars->extentRemaining;
 796
 797     return (kIOReturnSuccess);
 798 }
 799
 800 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 801
 802 IOReturn
 803 IOPolledFileWrite(IOPolledFileIOVars * vars,
 804                     const uint8_t * bytes, IOByteCount size,
 805                     IOPolledFileCryptVars * cryptvars)
 806 {
 807     IOReturn    err = kIOReturnSuccess;
 808     IOByteCount copy, original_size = size;
 809     bool        flush = false;
 810
 811     do
 812     {
 813         if (!bytes && !size)
 814         {
 815             // seek to end of block & flush
 816             size = vars->position & (vars->blockSize - 1);
 817             if (size)
 818                 size = vars->blockSize - size;
 819             flush = true;
 820             // use some garbage for the fill
 821             bytes = vars->buffer + vars->bufferOffset;
 822         }
 823
 824         copy = vars->bufferLimit - vars->bufferOffset;
 825         if (copy > size)
 826             copy = size;
 827         else
 828             flush = true;
 829
 830         if (bytes)
 831         {
 832 #if KASAN
 833           /* Since this may copy mach-o segments in bulk, use the nosan variants of bcopy to
 834            * avoid triggering global redzone sanitizer violations when accessing
 835            * interstices between 'C' structures
 836            */
 837             __nosan_bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
 838 #else
 839             bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
 840 #endif
 841             bytes += copy;
 842         }
 843         else
 844             bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
 845
 846         size -= copy;
 847         vars->bufferOffset += copy;
 848         vars->position += copy;
 849
 850         if (flush && vars->bufferOffset)
 851         {
 852             uint64_t offset = (vars->position - vars->bufferOffset
 853                                 - vars->extentPosition + vars->currentExtent->start);
 854             uint32_t length = (vars->bufferOffset);
 855
 856 #if CRYPTO
 857             if (cryptvars && vars->encryptStart
 858                 && (vars->position > vars->encryptStart)
 859                 && ((vars->position - length) < vars->encryptEnd))
 860             {
 861                 AbsoluteTime startTime, endTime;
 862
 863                 uint64_t encryptLen, encryptStart;
 864                 encryptLen = vars->position - vars->encryptStart;
 865                 if (encryptLen > length)
 866                     encryptLen = length;
 867                 encryptStart = length - encryptLen;
 868                 if (vars->position > vars->encryptEnd)
 869                     encryptLen -= (vars->position - vars->encryptEnd);
 870
 871                 clock_get_uptime(&startTime);
 872
 873                 // encrypt the buffer
 874                 aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
 875                                 &cryptvars->aes_iv[0],
 876                                 encryptLen / AES_BLOCK_SIZE,
 877                                 vars->buffer + vars->bufferHalf + encryptStart,
 878                                 &cryptvars->ctx.encrypt);
 879
 880                 clock_get_uptime(&endTime);
 881                 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
 882                 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
 883                 vars->cryptBytes += encryptLen;
 884
 885                 // save initial vector for following encrypts
 886                 bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
 887                         &cryptvars->aes_iv[0],
 888                         AES_BLOCK_SIZE);
 889             }
 890 #endif /* CRYPTO */
 891
 892             err = IOPolledFilePollersIODone(vars->pollers, true);
 893             if (kIOReturnSuccess != err)
 894                 break;
 895
 896 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
 897 //if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
 898
 899             err = IOStartPolledIO(vars->pollers, kIOPolledWrite, vars->bufferHalf, offset, length);
 900             if (kIOReturnSuccess != err) {
 901                 HIBLOGFROMPANIC("IOPolledFileWrite(0x%p, 0x%p, %llu, 0x%p) : IOStartPolledIO(0x%p, kIOPolledWrite, %llu, 0x%llx, %d) returned 0x%x\n",
 902                     vars, bytes, (uint64_t) original_size, cryptvars, vars->pollers, (uint64_t) vars->bufferHalf, offset, length, err);
 903                 break;
 904             }
 905             vars->pollers->io = true;
 906
 907             vars->extentRemaining -= vars->bufferOffset;
 908             if (!vars->extentRemaining)
 909             {
 910                 vars->currentExtent++;
 911                 vars->extentRemaining = vars->currentExtent->length;
 912                 vars->extentPosition  = vars->position;
 913             }
 914
 915             vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
 916             vars->bufferOffset = 0;
 917             if (vars->bufferSize <= vars->extentRemaining)
 918                 vars->bufferLimit = vars->bufferSize;
 919             else
 920                 vars->bufferLimit = vars->extentRemaining;
 921
 922             if (!vars->extentRemaining)
 923             {
 924                 err = kIOReturnOverrun;
 925                 break;
 926             }
 927
 928             flush = false;
 929         }
 930     }
 931     while (size);
 932
 933     return (err);
 934 }
 935
 936 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 937
 938 IOReturn
 939 IOPolledFileFlush(IOPolledFileIOVars * vars)
 940 {
 941     // Only supported by the underlying polled mode driver on embedded currently (expect kIOReturnUnsupported on other platforms)
 942     IOReturn err = kIOReturnSuccess;
 943
 944     err = IOPolledFilePollersIODone(vars->pollers, true);
 945     if (kIOReturnSuccess != err)
 946             return err;
 947
 948     err = IOStartPolledIO(vars->pollers, kIOPolledFlush, 0, 0, 0);
 949     if (kIOReturnSuccess != err) {
 950             HIBLOGFROMPANIC("IOPolledFileFlush(0x%p) : IOStartPolledIO(0x%p, kIOPolledFlush, 0, 0, 0) returned 0x%x\n",
 951                     vars, vars->pollers, err);
 952             return err;
 953     }
 954     vars->pollers->io = true;
 955
 956     return err;
 957 }
 958
 959 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 960
 961 IOReturn
 962 IOPolledFileRead(IOPolledFileIOVars * vars,
 963                     uint8_t * bytes, IOByteCount size,
 964                     IOPolledFileCryptVars * cryptvars)
 965 {
 966     IOReturn    err = kIOReturnSuccess;
 967     IOByteCount copy;
 968
 969 //    bytesWritten += size;
 970
 971     do
 972     {
 973         copy = vars->bufferLimit - vars->bufferOffset;
 974         if (copy > size)
 975             copy = size;
 976
 977         if (bytes)
 978         {
 979 #if KASAN
 980           __nosan_bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
 981 #else
 982           bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
 983 #endif
 984             bytes += copy;
 985         }
 986         size -= copy;
 987         vars->bufferOffset += copy;
 988 //      vars->position += copy;
 989
 990         if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd))
 991         {
 992             if (!vars->pollers->io) cryptvars = 0;
 993             err = IOPolledFilePollersIODone(vars->pollers, true);
 994             if (kIOReturnSuccess != err)
 995                 break;
 996
 997 if (vars->position & (vars->blockSize - 1)) HIBLOG("misaligned file pos %qx\n", vars->position);
 998
 999             vars->position        += vars->lastRead;
1000             vars->extentRemaining -= vars->lastRead;
1001             vars->bufferLimit      = vars->lastRead;
1002
1003             if (!vars->extentRemaining)
1004             {
1005                 vars->currentExtent++;
1006                 vars->extentRemaining = vars->currentExtent->length;
1007                 vars->extentPosition  = vars->position;
1008                 if (!vars->extentRemaining)
1009                 {
1010                     err = kIOReturnOverrun;
1011                     break;
1012                 }
1013             }
1014
1015             uint64_t length;
1016             uint64_t lastReadLength = vars->lastRead;
1017             uint64_t offset = (vars->position
1018                                 - vars->extentPosition + vars->currentExtent->start);
1019             if (vars->extentRemaining <= vars->bufferSize)
1020                 length = vars->extentRemaining;
1021             else
1022                 length = vars->bufferSize;
1023             if ((length + vars->position) > vars->readEnd)
1024                 length = vars->readEnd - vars->position;
1025
1026             vars->lastRead = length;
1027             if (length)
1028             {
1029 //if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
1030                 err = IOStartPolledIO(vars->pollers, kIOPolledRead, vars->bufferHalf, offset, length);
1031                 if (kIOReturnSuccess != err)
1032                     break;
1033                 vars->pollers->io = true;
1034             }
1035
1036             vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
1037             vars->bufferOffset = 0;
1038
1039 #if CRYPTO
1040             if (cryptvars)
1041             {
1042                 uint8_t thisVector[AES_BLOCK_SIZE];
1043                 AbsoluteTime startTime, endTime;
1044
1045                 // save initial vector for following decrypts
1046                 bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
1047                 bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE,
1048                         &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1049
1050                 // decrypt the buffer
1051                 clock_get_uptime(&startTime);
1052
1053                 aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
1054                                 &thisVector[0],
1055                                 lastReadLength / AES_BLOCK_SIZE,
1056                                 vars->buffer + vars->bufferHalf,
1057                                 &cryptvars->ctx.decrypt);
1058
1059                 clock_get_uptime(&endTime);
1060                 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
1061                 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
1062                 vars->cryptBytes += lastReadLength;
1063             }
1064 #endif /* CRYPTO */
1065         }
1066     }
1067     while (size);
1068
1069     return (err);
1070 }
1071
1072 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1073