iokit/Kernel/IOPolledInterface.cpp

   1 /*
   2  * Copyright (c) 2006-2009 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <sys/uio.h>
  30 #include <sys/conf.h>
  31
  32 #include <IOKit/IOLib.h>
  33 #include <IOKit/IOBSD.h>
  34 #include <IOKit/IOService.h>
  35 #include <IOKit/IOPlatformExpert.h>
  36 #include <IOKit/IOPolledInterface.h>
  37 #include <IOKit/IOHibernatePrivate.h>
  38 #include <IOKit/IOBufferMemoryDescriptor.h>
  39 #include <IOKit/AppleKeyStoreInterface.h>
  40 #include "IOKitKernelInternal.h"
  41
  42
  43 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  44
  45 OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject);
  46
  47 OSMetaClassDefineReservedUsed(IOPolledInterface, 0);
  48 OSMetaClassDefineReservedUnused(IOPolledInterface, 1);
  49 OSMetaClassDefineReservedUnused(IOPolledInterface, 2);
  50 OSMetaClassDefineReservedUnused(IOPolledInterface, 3);
  51 OSMetaClassDefineReservedUnused(IOPolledInterface, 4);
  52 OSMetaClassDefineReservedUnused(IOPolledInterface, 5);
  53 OSMetaClassDefineReservedUnused(IOPolledInterface, 6);
  54 OSMetaClassDefineReservedUnused(IOPolledInterface, 7);
  55 OSMetaClassDefineReservedUnused(IOPolledInterface, 8);
  56 OSMetaClassDefineReservedUnused(IOPolledInterface, 9);
  57 OSMetaClassDefineReservedUnused(IOPolledInterface, 10);
  58 OSMetaClassDefineReservedUnused(IOPolledInterface, 11);
  59 OSMetaClassDefineReservedUnused(IOPolledInterface, 12);
  60 OSMetaClassDefineReservedUnused(IOPolledInterface, 13);
  61 OSMetaClassDefineReservedUnused(IOPolledInterface, 14);
  62 OSMetaClassDefineReservedUnused(IOPolledInterface, 15);
  63
  64 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  65
  66 #ifndef kIOMediaPreferredBlockSizeKey
  67 #define kIOMediaPreferredBlockSizeKey   "Preferred Block Size"
  68 #endif
  69
  70 enum { kDefaultIOSize = 128 * 1024 };
  71
  72 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  73
  74 class IOPolledFilePollers : public OSObject
  75 {
  76         OSDeclareDefaultStructors(IOPolledFilePollers)
  77
  78 public:
  79         IOService                * media;
  80         OSArray                  * pollers;
  81         IOBufferMemoryDescriptor * ioBuffer;
  82         bool                 abortable;
  83         bool                 io;
  84         IOReturn             ioStatus;
  85         uint32_t             openCount;
  86
  87         static IOPolledFilePollers * copyPollers(IOService * media);
  88 };
  89
  90 OSDefineMetaClassAndStructors(IOPolledFilePollers, OSObject)
  91
  92 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  93
  94 IOPolledFilePollers *
  95 IOPolledFilePollers::copyPollers(IOService * media)
  96 {
  97         IOPolledFilePollers * vars;
  98         IOReturn              err;
  99         IOService       * service;
 100         OSObject        * obj;
 101         IORegistryEntry * next;
 102         IORegistryEntry * child;
 103
 104         if ((obj = media->copyProperty(kIOPolledInterfaceStackKey))) {
 105                 return OSDynamicCast(IOPolledFilePollers, obj);
 106         }
 107
 108         do{
 109                 vars = OSTypeAlloc(IOPolledFilePollers);
 110                 vars->init();
 111
 112                 vars->pollers = OSArray::withCapacity(4);
 113                 if (!vars->pollers) {
 114                         err = kIOReturnNoMemory;
 115                         break;
 116                 }
 117
 118                 next = vars->media = media;
 119                 do{
 120                         IOPolledInterface * poller;
 121                         OSObject *          obj;
 122
 123                         obj = next->getProperty(kIOPolledInterfaceSupportKey);
 124                         if (kOSBooleanFalse == obj) {
 125                                 vars->pollers->flushCollection();
 126                                 break;
 127                         } else if ((poller = OSDynamicCast(IOPolledInterface, obj))) {
 128                                 vars->pollers->setObject(poller);
 129                         }
 130
 131                         if ((service = OSDynamicCast(IOService, next))
 132                             && service->getDeviceMemory()
 133                             && !vars->pollers->getCount()) {
 134                                 break;
 135                         }
 136
 137                         child = next;
 138                 }while ((next = child->getParentEntry(gIOServicePlane))
 139                     && child->isParent(next, gIOServicePlane, true));
 140
 141                 if (!vars->pollers->getCount()) {
 142                         err = kIOReturnUnsupported;
 143                         break;
 144                 }
 145         }while (false);
 146
 147         media->setProperty(kIOPolledInterfaceStackKey, vars);
 148
 149         return vars;
 150 }
 151
 152 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 153
 154 static IOReturn
 155 IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable);
 156
 157 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 158
 159 static IOReturn
 160 IOPolledFilePollersProbe(IOPolledFilePollers * vars)
 161 {
 162         IOReturn            err = kIOReturnError;
 163         int32_t             idx;
 164         IOPolledInterface * poller;
 165
 166         for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--) {
 167                 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
 168                 err = poller->probe(vars->media);
 169                 if (err) {
 170                         HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
 171                         break;
 172                 }
 173         }
 174
 175         return err;
 176 }
 177
 178 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 179
 180 IOReturn
 181 IOPolledFilePollersOpen(IOPolledFileIOVars * filevars, uint32_t state, bool abortable)
 182 {
 183         IOPolledFilePollers      * vars = filevars->pollers;
 184         IOBufferMemoryDescriptor * ioBuffer;
 185         IOPolledInterface        * poller;
 186         IOService                * next;
 187         IOReturn                   err = kIOReturnError;
 188         int32_t                    idx;
 189
 190         vars->abortable = abortable;
 191         ioBuffer = 0;
 192
 193         if (kIOPolledAfterSleepState == state) {
 194                 vars->ioStatus = 0;
 195                 vars->io = false;
 196         }
 197         (void) IOPolledFilePollersIODone(vars, false);
 198
 199         if ((kIOPolledPreflightState == state) || (kIOPolledPreflightCoreDumpState == state)) {
 200                 ioBuffer = vars->ioBuffer;
 201                 if (!ioBuffer) {
 202                         vars->ioBuffer = ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionInOut,
 203                             2 * kDefaultIOSize, page_size);
 204                         if (!ioBuffer) {
 205                                 return kIOReturnNoMemory;
 206                         }
 207                 }
 208         }
 209
 210         for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--) {
 211                 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
 212                 err = poller->open(state, ioBuffer);
 213                 if (kIOReturnSuccess != err) {
 214                         HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
 215                         break;
 216                 }
 217         }
 218         if ((kIOReturnSuccess == err) && (kIOPolledPreflightState == state)) {
 219                 next = vars->media;
 220                 while (next) {
 221                         next->setProperty(kIOPolledInterfaceActiveKey, kOSBooleanTrue);
 222                         next = next->getProvider();
 223                 }
 224         }
 225
 226         return err;
 227 }
 228
 229 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 230
 231 IOReturn
 232 IOPolledFilePollersClose(IOPolledFileIOVars * filevars, uint32_t state)
 233 {
 234         IOPolledFilePollers * vars = filevars->pollers;
 235         IOPolledInterface * poller;
 236         IORegistryEntry *   next;
 237         IOReturn            err;
 238         int32_t             idx;
 239
 240         (void) IOPolledFilePollersIODone(vars, false);
 241
 242         if ((kIOPolledPostflightState == state) || (kIOPolledPostflightCoreDumpState == state)) {
 243                 vars->openCount--;
 244         }
 245
 246         for (idx = 0, err = kIOReturnSuccess;
 247             (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
 248             idx++) {
 249                 err = poller->close(state);
 250                 if ((kIOReturnSuccess != err) && (kIOPolledBeforeSleepStateAborted == state)) {
 251                         err = poller->close(kIOPolledBeforeSleepState);
 252                 }
 253                 if (err) {
 254                         HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
 255                 }
 256         }
 257
 258         if (kIOPolledPostflightState == state) {
 259                 next = vars->media;
 260                 while (next) {
 261                         next->removeProperty(kIOPolledInterfaceActiveKey);
 262                         next = next->getParentEntry(gIOServicePlane);
 263                 }
 264         }
 265
 266         if ((kIOPolledPostflightState == state) || (kIOPolledPostflightCoreDumpState == state)) {
 267                 do{
 268                         if (vars->openCount) {
 269                                 break;
 270                         }
 271                         if (vars->ioBuffer) {
 272                                 vars->ioBuffer->release();
 273                                 vars->ioBuffer = 0;
 274                         }
 275                 }while (false);
 276         }
 277
 278         return err;
 279 }
 280
 281 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 282
 283 IOReturn
 284 IOPolledInterface::setEncryptionKey(const uint8_t * key, size_t keySize)
 285 {
 286         return kIOReturnUnsupported;
 287 }
 288
 289 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 290
 291 IOReturn
 292 IOPolledFilePollersSetEncryptionKey(IOPolledFileIOVars * filevars,
 293     const uint8_t * key, size_t keySize)
 294 {
 295         IOReturn              ret = kIOReturnUnsupported;
 296         IOReturn              err;
 297         int32_t               idx;
 298         IOPolledFilePollers * vars = filevars->pollers;
 299         IOPolledInterface   * poller;
 300
 301         for (idx = 0;
 302             (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
 303             idx++) {
 304                 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
 305                 err = poller->setEncryptionKey(key, keySize);
 306                 if (kIOReturnSuccess == err) {
 307                         ret = err;
 308                 }
 309         }
 310
 311         return ret;
 312 }
 313
 314 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 315
 316 IOMemoryDescriptor *
 317 IOPolledFileGetIOBuffer(IOPolledFileIOVars * vars)
 318 {
 319         return vars->pollers->ioBuffer;
 320 }
 321
 322 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 323
 324 static void
 325 IOPolledIOComplete(void *   target,
 326     void *   parameter,
 327     IOReturn status,
 328     UInt64   actualByteCount)
 329 {
 330         IOPolledFilePollers * vars = (IOPolledFilePollers *) parameter;
 331
 332         vars->ioStatus = status;
 333 }
 334
 335 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 336
 337 static IOReturn
 338 IOStartPolledIO(IOPolledFilePollers * vars,
 339     uint32_t operation, uint32_t bufferOffset,
 340     uint64_t deviceOffset, uint64_t length)
 341 {
 342         IOReturn            err;
 343         IOPolledInterface * poller;
 344         IOPolledCompletion  completion;
 345
 346         err = vars->ioStatus;
 347         if (kIOReturnSuccess != err) {
 348                 return err;
 349         }
 350
 351         completion.target    = 0;
 352         completion.action    = &IOPolledIOComplete;
 353         completion.parameter = vars;
 354
 355         vars->ioStatus = -1;
 356
 357         poller = (IOPolledInterface *) vars->pollers->getObject(0);
 358         err = poller->startIO(operation, bufferOffset, deviceOffset, length, completion);
 359         if (err) {
 360                 if (kernel_debugger_entry_count) {
 361                         HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
 362                 } else {
 363                         HIBLOGFROMPANIC("IOPolledInterface::IOStartPolledIO(0x%p, %d, 0x%x, 0x%llx, %llu) : poller->startIO(%d, 0x%x, 0x%llx, %llu, completion) returned 0x%x",
 364                             vars, operation, bufferOffset, deviceOffset, length, operation, bufferOffset, deviceOffset, length, err);
 365                 }
 366         }
 367         return err;
 368 }
 369
 370 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 371
 372 static IOReturn
 373 IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable)
 374 {
 375         IOReturn            err = kIOReturnSuccess;
 376         int32_t             idx = 0;
 377         IOPolledInterface * poller;
 378         AbsoluteTime        deadline;
 379
 380         if (!vars->io) {
 381                 return kIOReturnSuccess;
 382         }
 383
 384         abortable &= vars->abortable;
 385
 386         clock_interval_to_deadline(2000, kMillisecondScale, &deadline);
 387
 388         while (-1 == vars->ioStatus) {
 389                 for (idx = 0;
 390                     (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
 391                     idx++) {
 392                         IOReturn newErr;
 393                         newErr = poller->checkForWork();
 394                         if ((newErr == kIOReturnAborted) && !abortable) {
 395                                 newErr = kIOReturnSuccess;
 396                         }
 397                         if (kIOReturnSuccess == err) {
 398                                 err = newErr;
 399                         }
 400                 }
 401                 if ((false) && (kIOReturnSuccess == err) && (mach_absolute_time() > AbsoluteTime_to_scalar(&deadline))) {
 402                         HIBLOG("IOPolledInterface::forced timeout\n");
 403                         vars->ioStatus = kIOReturnTimeout;
 404                 }
 405         }
 406         vars->io = false;
 407
 408 #if HIBERNATION
 409         if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort()) {
 410                 err = kIOReturnAborted;
 411                 HIBLOG("IOPolledInterface::checkForWork sw abort\n");
 412         }
 413 #endif
 414
 415         if (err) {
 416                 HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
 417         } else {
 418                 err = vars->ioStatus;
 419                 if (kIOReturnSuccess != err) {
 420                         HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
 421                 }
 422         }
 423
 424         return err;
 425 }
 426
 427 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 428 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 429
 430 struct _OpenFileContext {
 431         OSData * extents;
 432         uint64_t size;
 433 };
 434
 435 static void
 436 file_extent_callback(void * ref, uint64_t start, uint64_t length)
 437 {
 438         _OpenFileContext * ctx = (_OpenFileContext *) ref;
 439         IOPolledFileExtent extent;
 440
 441         extent.start  = start;
 442         extent.length = length;
 443         ctx->extents->appendBytes(&extent, sizeof(extent));
 444         ctx->size += length;
 445 }
 446
 447 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 448
 449 static IOService *
 450 IOCopyMediaForDev(dev_t device)
 451 {
 452         OSDictionary * matching;
 453         OSNumber *     num;
 454         OSIterator *   iter;
 455         IOService *    result = 0;
 456
 457         matching = IOService::serviceMatching("IOMedia");
 458         if (!matching) {
 459                 return 0;
 460         }
 461         do{
 462                 num = OSNumber::withNumber(major(device), 32);
 463                 if (!num) {
 464                         break;
 465                 }
 466                 matching->setObject(kIOBSDMajorKey, num);
 467                 num->release();
 468                 num = OSNumber::withNumber(minor(device), 32);
 469                 if (!num) {
 470                         break;
 471                 }
 472                 matching->setObject(kIOBSDMinorKey, num);
 473                 num->release();
 474                 if (!num) {
 475                         break;
 476                 }
 477                 iter = IOService::getMatchingServices(matching);
 478                 if (iter) {
 479                         result = (IOService *) iter->getNextObject();
 480                         result->retain();
 481                         iter->release();
 482                 }
 483         }while (false);
 484         matching->release();
 485
 486         return result;
 487 }
 488
 489 #define APFSMEDIA_GETHIBERKEY         "getHiberKey"
 490
 491 static IOReturn
 492 IOGetVolumeCryptKey(dev_t block_dev, OSString ** pKeyUUID,
 493     uint8_t * volumeCryptKey, size_t * keySize)
 494 {
 495         IOReturn         err;
 496         IOService *      part;
 497         OSString *       keyUUID = 0;
 498         OSString *       keyStoreUUID = 0;
 499         uuid_t           volumeKeyUUID;
 500         aks_volume_key_t vek;
 501         size_t           callerKeySize;
 502
 503         static IOService * sKeyStore;
 504
 505         part = IOCopyMediaForDev(block_dev);
 506         if (!part) {
 507                 return kIOReturnNotFound;
 508         }
 509
 510         callerKeySize = *keySize;
 511         // Try APFS first
 512         {
 513                 uuid_t volUuid = {0};
 514                 err = part->callPlatformFunction(APFSMEDIA_GETHIBERKEY, false, &volUuid, volumeCryptKey, keySize, keySize);
 515                 if (kIOReturnBadArgument == err) {
 516                         // apfs fails on buffer size >32
 517                         *keySize = 32;
 518                         err = part->callPlatformFunction(APFSMEDIA_GETHIBERKEY, false, &volUuid, volumeCryptKey, keySize, keySize);
 519                 }
 520                 if (err != kIOReturnSuccess) {
 521                         *keySize = 0;
 522                 } else {
 523                         // No need to create uuid string if it's not requested
 524                         if (pKeyUUID) {
 525                                 uuid_string_t volUuidStr;
 526                                 uuid_unparse(volUuid, volUuidStr);
 527                                 *pKeyUUID = OSString::withCString(volUuidStr);
 528                         }
 529
 530                         part->release();
 531                         return kIOReturnSuccess;
 532                 }
 533         }
 534
 535         // Then old CS path
 536         err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false,
 537             (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL);
 538         if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID) {
 539 //        IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy());
 540
 541                 if (!sKeyStore) {
 542                         sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane);
 543                 }
 544                 if (sKeyStore) {
 545                         err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID);
 546                 } else {
 547                         err = kIOReturnNoResources;
 548                 }
 549                 if (kIOReturnSuccess == err) {
 550                         err = sKeyStore->callPlatformFunction(gAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL);
 551                 }
 552                 if (kIOReturnSuccess != err) {
 553                         IOLog("volume key err 0x%x\n", err);
 554                 } else {
 555                         if (vek.key.keybytecount <= callerKeySize) {
 556                                 *keySize = vek.key.keybytecount;
 557                         }
 558                         bcopy(&vek.key.keybytes[0], volumeCryptKey, *keySize);
 559                 }
 560                 bzero(&vek, sizeof(vek));
 561
 562                 if (pKeyUUID) {
 563                         // Create a copy because the caller would release it
 564                         *pKeyUUID = OSString::withString(keyUUID);
 565                 }
 566         }
 567
 568         part->release();
 569         return err;
 570 }
 571
 572 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 573
 574 IOReturn
 575 IOPolledFileOpen(const char * filename,
 576     uint32_t flags,
 577     uint64_t setFileSize, uint64_t fsFreeSize,
 578     void * write_file_addr, size_t write_file_len,
 579     IOPolledFileIOVars ** fileVars,
 580     OSData ** imagePath,
 581     uint8_t * volumeCryptKey, size_t * keySize)
 582 {
 583         IOReturn             err = kIOReturnSuccess;
 584         IOPolledFileIOVars * vars;
 585         _OpenFileContext     ctx;
 586         OSData *             extentsData = NULL;
 587         OSNumber *           num;
 588         IOService *          part = 0;
 589         dev_t                block_dev;
 590         dev_t                image_dev;
 591         AbsoluteTime         startTime, endTime;
 592         uint64_t             nsec;
 593
 594         vars = IONew(IOPolledFileIOVars, 1);
 595         if (!vars) {
 596                 return kIOReturnNoMemory;
 597         }
 598         bzero(vars, sizeof(*vars));
 599         vars->allocated = true;
 600
 601         do{
 602                 extentsData = OSData::withCapacity(32);
 603                 ctx.extents = extentsData;
 604                 ctx.size    = 0;
 605                 clock_get_uptime(&startTime);
 606
 607                 vars->fileRef = kern_open_file_for_direct_io(filename,
 608                     flags,
 609                     &file_extent_callback, &ctx,
 610                     setFileSize,
 611                     fsFreeSize,
 612                     // write file:
 613                     0, write_file_addr, write_file_len,
 614                     // results
 615                     &block_dev,
 616                     &image_dev,
 617                     &vars->block0,
 618                     &vars->maxiobytes,
 619                     &vars->flags);
 620 #if 0
 621                 uint32_t msDelay = (131071 & random());
 622                 HIBLOG("sleep %d\n", msDelay);
 623                 IOSleep(msDelay);
 624 #endif
 625                 clock_get_uptime(&endTime);
 626                 SUB_ABSOLUTETIME(&endTime, &startTime);
 627                 absolutetime_to_nanoseconds(endTime, &nsec);
 628
 629                 if (!vars->fileRef) {
 630                         err = kIOReturnNoSpace;
 631                 }
 632
 633                 HIBLOG("kern_open_file_for_direct_io took %qd ms\n", nsec / 1000000ULL);
 634                 if (kIOReturnSuccess != err) {
 635                         break;
 636                 }
 637
 638                 HIBLOG("Opened file %s, size %qd, extents %ld, maxio %qx ssd %d\n", filename, ctx.size,
 639                     (extentsData->getLength() / sizeof(IOPolledFileExtent)) - 1,
 640                     vars->maxiobytes, kIOPolledFileSSD & vars->flags);
 641                 assert(!vars->block0);
 642                 if (extentsData->getLength() < sizeof(IOPolledFileExtent)) {
 643                         err = kIOReturnNoSpace;
 644                         break;
 645                 }
 646
 647                 vars->fileSize = ctx.size;
 648                 vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
 649
 650                 part = IOCopyMediaForDev(image_dev);
 651                 if (!part) {
 652                         err = kIOReturnNotFound;
 653                         break;
 654                 }
 655
 656                 if (!(vars->pollers = IOPolledFilePollers::copyPollers(part))) {
 657                         break;
 658                 }
 659
 660                 if ((num = OSDynamicCast(OSNumber, part->getProperty(kIOMediaPreferredBlockSizeKey)))) {
 661                         vars->blockSize = num->unsigned32BitValue();
 662                 }
 663                 if (vars->blockSize < 4096) {
 664                         vars->blockSize = 4096;
 665                 }
 666
 667                 HIBLOG("polled file major %d, minor %d, blocksize %ld, pollers %d\n",
 668                     major(image_dev), minor(image_dev), (long)vars->blockSize,
 669                     vars->pollers->pollers->getCount());
 670
 671                 OSString * keyUUID = NULL;
 672                 if (volumeCryptKey) {
 673                         err = IOGetVolumeCryptKey(block_dev, &keyUUID, volumeCryptKey, keySize);
 674                 }
 675
 676                 *fileVars    = vars;
 677                 vars->fileExtents = extentsData;
 678
 679                 // make imagePath
 680                 OSData * data;
 681                 if (imagePath) {
 682 #if defined(__i386__) || defined(__x86_64__)
 683                         char str2[24 + sizeof(uuid_string_t) + 2];
 684
 685                         if (keyUUID) {
 686                                 snprintf(str2, sizeof(str2), "%qx:%s",
 687                                     vars->extentMap[0].start, keyUUID->getCStringNoCopy());
 688                         } else {
 689                                 snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
 690                         }
 691
 692                         err = IOService::getPlatform()->callPlatformFunction(
 693                                 gIOCreateEFIDevicePathSymbol, false,
 694                                 (void *) part, (void *) str2,
 695                                 (void *) (uintptr_t) true, (void *) &data);
 696 #else
 697                         data = 0;
 698                         err = kIOReturnSuccess;
 699 #endif
 700                         if (kIOReturnSuccess != err) {
 701                                 HIBLOG("error 0x%x getting path\n", err);
 702                                 break;
 703                         }
 704                         *imagePath = data;
 705                 }
 706
 707                 // Release key UUID if we have one
 708                 if (keyUUID) {
 709                         keyUUID->release();
 710                         keyUUID = NULL; // Just in case
 711                 }
 712         }while (false);
 713
 714         if (kIOReturnSuccess != err) {
 715                 HIBLOG("error 0x%x opening polled file\n", err);
 716                 IOPolledFileClose(&vars, 0, 0, 0, 0, 0);
 717                 if (extentsData) {
 718                         extentsData->release();
 719                 }
 720         }
 721
 722         if (part) {
 723                 part->release();
 724         }
 725
 726         return err;
 727 }
 728
 729 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 730
 731 IOReturn
 732 IOPolledFileClose(IOPolledFileIOVars ** pVars,
 733     off_t write_offset, void * addr, size_t write_length,
 734     off_t discard_offset, off_t discard_end)
 735 {
 736         IOPolledFileIOVars * vars;
 737
 738         vars = *pVars;
 739         if (!vars) {
 740                 return kIOReturnSuccess;
 741         }
 742
 743         if (vars->fileRef) {
 744                 kern_close_file_for_direct_io(vars->fileRef, write_offset, addr, write_length,
 745                     discard_offset, discard_end);
 746                 vars->fileRef = NULL;
 747         }
 748         if (vars->fileExtents) {
 749                 vars->fileExtents->release();
 750                 vars->fileExtents = 0;
 751         }
 752         if (vars->pollers) {
 753                 vars->pollers->release();
 754                 vars->pollers = 0;
 755         }
 756
 757         if (vars->allocated) {
 758                 IODelete(vars, IOPolledFileIOVars, 1);
 759         } else {
 760                 bzero(vars, sizeof(IOPolledFileIOVars));
 761         }
 762         *pVars = NULL;
 763
 764         return kIOReturnSuccess;
 765 }
 766
 767 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 768
 769 IOReturn
 770 IOPolledFilePollersSetup(IOPolledFileIOVars * vars,
 771     uint32_t openState)
 772 {
 773         IOReturn err;
 774
 775         err = kIOReturnSuccess;
 776         do{
 777                 if (!vars->pollers->openCount) {
 778                         err = IOPolledFilePollersProbe(vars->pollers);
 779                         if (kIOReturnSuccess != err) {
 780                                 break;
 781                         }
 782                 }
 783                 err = IOPolledFilePollersOpen(vars, openState, false);
 784                 if (kIOReturnSuccess != err) {
 785                         break;
 786                 }
 787                 if ((kIOPolledPreflightState == openState) || (kIOPolledPreflightCoreDumpState == openState)) {
 788                         vars->pollers->openCount++;
 789                 }
 790                 vars->pollers->io  = false;
 791                 vars->buffer       = (uint8_t *) vars->pollers->ioBuffer->getBytesNoCopy();
 792                 vars->bufferHalf   = 0;
 793                 vars->bufferOffset = 0;
 794                 vars->bufferSize   = (vars->pollers->ioBuffer->getLength() >> 1);
 795
 796                 if (vars->maxiobytes < vars->bufferSize) {
 797                         vars->bufferSize = vars->maxiobytes;
 798                 }
 799         }while (false);
 800
 801         if (kIOReturnSuccess != err) {
 802                 HIBLOG("IOPolledFilePollersSetup(%d) error 0x%x\n", openState, err);
 803         }
 804
 805         return err;
 806 }
 807
 808
 809 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 810
 811 IOReturn
 812 IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
 813 {
 814         IOPolledFileExtent * extentMap;
 815
 816         extentMap = vars->extentMap;
 817
 818         vars->position = position;
 819
 820         if (position > vars->fileSize) {
 821                 HIBLOG("IOPolledFileSeek: called to seek to 0x%llx greater than file size of 0x%llx\n", vars->position, vars->fileSize);
 822                 return kIOReturnNoSpace;
 823         }
 824
 825         while (position >= extentMap->length) {
 826                 position -= extentMap->length;
 827                 extentMap++;
 828         }
 829
 830         vars->currentExtent   = extentMap;
 831         vars->extentRemaining = extentMap->length - position;
 832         vars->extentPosition  = vars->position - position;
 833
 834         if (vars->bufferSize <= vars->extentRemaining) {
 835                 vars->bufferLimit = vars->bufferSize;
 836         } else {
 837                 vars->bufferLimit = vars->extentRemaining;
 838         }
 839
 840         return kIOReturnSuccess;
 841 }
 842
 843 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 844
 845 IOReturn
 846 IOPolledFileWrite(IOPolledFileIOVars * vars,
 847     const uint8_t * bytes, IOByteCount size,
 848     IOPolledFileCryptVars * cryptvars)
 849 {
 850         IOReturn    err = kIOReturnSuccess;
 851         IOByteCount copy, original_size = size;
 852         bool        flush = false;
 853
 854         do{
 855                 if (!bytes && !size) {
 856                         // seek to end of block & flush
 857                         size = vars->position & (vars->blockSize - 1);
 858                         if (size) {
 859                                 size = vars->blockSize - size;
 860                         }
 861                         flush = true;
 862                         // use some garbage for the fill
 863                         bytes = vars->buffer + vars->bufferOffset;
 864                 }
 865
 866                 copy = vars->bufferLimit - vars->bufferOffset;
 867                 if (copy > size) {
 868                         copy = size;
 869                 } else {
 870                         flush = true;
 871                 }
 872
 873                 if (bytes) {
 874 #if KASAN
 875                         /* Since this may copy mach-o segments in bulk, use the nosan variants of bcopy to
 876                          * avoid triggering global redzone sanitizer violations when accessing
 877                          * interstices between 'C' structures
 878                          */
 879                         __nosan_bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
 880 #else
 881                         bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
 882 #endif
 883                         bytes += copy;
 884                 } else {
 885                         bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
 886                 }
 887
 888                 size -= copy;
 889                 vars->bufferOffset += copy;
 890                 vars->position += copy;
 891
 892                 if (flush && vars->bufferOffset) {
 893                         uint64_t offset = (vars->position - vars->bufferOffset
 894                             - vars->extentPosition + vars->currentExtent->start);
 895                         uint32_t length = (vars->bufferOffset);
 896
 897 #if CRYPTO
 898                         if (cryptvars && vars->encryptStart
 899                             && (vars->position > vars->encryptStart)
 900                             && ((vars->position - length) < vars->encryptEnd)) {
 901                                 AbsoluteTime startTime, endTime;
 902
 903                                 uint64_t encryptLen, encryptStart;
 904                                 encryptLen = vars->position - vars->encryptStart;
 905                                 if (encryptLen > length) {
 906                                         encryptLen = length;
 907                                 }
 908                                 encryptStart = length - encryptLen;
 909                                 if (vars->position > vars->encryptEnd) {
 910                                         encryptLen -= (vars->position - vars->encryptEnd);
 911                                 }
 912
 913                                 clock_get_uptime(&startTime);
 914
 915                                 // encrypt the buffer
 916                                 aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
 917                                     &cryptvars->aes_iv[0],
 918                                     encryptLen / AES_BLOCK_SIZE,
 919                                     vars->buffer + vars->bufferHalf + encryptStart,
 920                                     &cryptvars->ctx.encrypt);
 921
 922                                 clock_get_uptime(&endTime);
 923                                 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
 924                                 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
 925                                 vars->cryptBytes += encryptLen;
 926
 927                                 // save initial vector for following encrypts
 928                                 bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
 929                                     &cryptvars->aes_iv[0],
 930                                     AES_BLOCK_SIZE);
 931                         }
 932 #endif /* CRYPTO */
 933
 934                         err = IOPolledFilePollersIODone(vars->pollers, true);
 935                         if (kIOReturnSuccess != err) {
 936                                 break;
 937                         }
 938
 939                         if (vars->position & (vars->blockSize - 1)) {
 940                                 HIBLOG("misaligned file pos %qx\n", vars->position);
 941                         }
 942 //if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
 943
 944                         err = IOStartPolledIO(vars->pollers, kIOPolledWrite, vars->bufferHalf, offset, length);
 945                         if (kIOReturnSuccess != err) {
 946                                 HIBLOGFROMPANIC("IOPolledFileWrite(0x%p, 0x%p, %llu, 0x%p) : IOStartPolledIO(0x%p, kIOPolledWrite, %llu, 0x%llx, %d) returned 0x%x\n",
 947                                     vars, bytes, (uint64_t) original_size, cryptvars, vars->pollers, (uint64_t) vars->bufferHalf, offset, length, err);
 948                                 break;
 949                         }
 950                         vars->pollers->io = true;
 951
 952                         vars->extentRemaining -= vars->bufferOffset;
 953                         if (!vars->extentRemaining) {
 954                                 vars->currentExtent++;
 955                                 vars->extentRemaining = vars->currentExtent->length;
 956                                 vars->extentPosition  = vars->position;
 957                         }
 958
 959                         vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
 960                         vars->bufferOffset = 0;
 961                         if (vars->bufferSize <= vars->extentRemaining) {
 962                                 vars->bufferLimit = vars->bufferSize;
 963                         } else {
 964                                 vars->bufferLimit = vars->extentRemaining;
 965                         }
 966
 967                         if (!vars->extentRemaining) {
 968                                 err = kIOReturnOverrun;
 969                                 break;
 970                         }
 971
 972                         flush = false;
 973                 }
 974         }while (size);
 975
 976         return err;
 977 }
 978
 979 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 980
 981 IOReturn
 982 IOPolledFileFlush(IOPolledFileIOVars * vars)
 983 {
 984         // Only supported by the underlying polled mode driver on embedded currently (expect kIOReturnUnsupported on other platforms)
 985         IOReturn err = kIOReturnSuccess;
 986
 987         err = IOPolledFilePollersIODone(vars->pollers, true);
 988         if (kIOReturnSuccess != err) {
 989                 return err;
 990         }
 991
 992         err = IOStartPolledIO(vars->pollers, kIOPolledFlush, 0, 0, 0);
 993         if (kIOReturnSuccess != err) {
 994                 HIBLOGFROMPANIC("IOPolledFileFlush(0x%p) : IOStartPolledIO(0x%p, kIOPolledFlush, 0, 0, 0) returned 0x%x\n",
 995                     vars, vars->pollers, err);
 996                 return err;
 997         }
 998         vars->pollers->io = true;
 999
1000         return err;
1001 }
1002
1003 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1004
1005 IOReturn
1006 IOPolledFileRead(IOPolledFileIOVars * vars,
1007     uint8_t * bytes, IOByteCount size,
1008     IOPolledFileCryptVars * cryptvars)
1009 {
1010         IOReturn    err = kIOReturnSuccess;
1011         IOByteCount copy;
1012
1013 //    bytesWritten += size;
1014
1015         do{
1016                 copy = vars->bufferLimit - vars->bufferOffset;
1017                 if (copy > size) {
1018                         copy = size;
1019                 }
1020
1021                 if (bytes) {
1022 #if KASAN
1023                         __nosan_bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
1024 #else
1025                         bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
1026 #endif
1027                         bytes += copy;
1028                 }
1029                 size -= copy;
1030                 vars->bufferOffset += copy;
1031 //      vars->position += copy;
1032
1033                 if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd)) {
1034                         if (!vars->pollers->io) {
1035                                 cryptvars = 0;
1036                         }
1037                         err = IOPolledFilePollersIODone(vars->pollers, true);
1038                         if (kIOReturnSuccess != err) {
1039                                 break;
1040                         }
1041
1042                         if (vars->position & (vars->blockSize - 1)) {
1043                                 HIBLOG("misaligned file pos %qx\n", vars->position);
1044                         }
1045
1046                         vars->position        += vars->lastRead;
1047                         vars->extentRemaining -= vars->lastRead;
1048                         vars->bufferLimit      = vars->lastRead;
1049
1050                         if (!vars->extentRemaining) {
1051                                 vars->currentExtent++;
1052                                 vars->extentRemaining = vars->currentExtent->length;
1053                                 vars->extentPosition  = vars->position;
1054                                 if (!vars->extentRemaining) {
1055                                         err = kIOReturnOverrun;
1056                                         break;
1057                                 }
1058                         }
1059
1060                         uint64_t length;
1061                         uint64_t lastReadLength = vars->lastRead;
1062                         uint64_t offset = (vars->position
1063                             - vars->extentPosition + vars->currentExtent->start);
1064                         if (vars->extentRemaining <= vars->bufferSize) {
1065                                 length = vars->extentRemaining;
1066                         } else {
1067                                 length = vars->bufferSize;
1068                         }
1069                         if ((length + vars->position) > vars->readEnd) {
1070                                 length = vars->readEnd - vars->position;
1071                         }
1072
1073                         vars->lastRead = length;
1074                         if (length) {
1075 //if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
1076                                 err = IOStartPolledIO(vars->pollers, kIOPolledRead, vars->bufferHalf, offset, length);
1077                                 if (kIOReturnSuccess != err) {
1078                                         break;
1079                                 }
1080                                 vars->pollers->io = true;
1081                         }
1082
1083                         vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
1084                         vars->bufferOffset = 0;
1085
1086 #if CRYPTO
1087                         if (cryptvars) {
1088                                 uint8_t thisVector[AES_BLOCK_SIZE];
1089                                 AbsoluteTime startTime, endTime;
1090
1091                                 // save initial vector for following decrypts
1092                                 bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
1093                                 bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE,
1094                                     &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1095
1096                                 // decrypt the buffer
1097                                 clock_get_uptime(&startTime);
1098
1099                                 aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
1100                                     &thisVector[0],
1101                                     lastReadLength / AES_BLOCK_SIZE,
1102                                     vars->buffer + vars->bufferHalf,
1103                                     &cryptvars->ctx.decrypt);
1104
1105                                 clock_get_uptime(&endTime);
1106                                 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
1107                                 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
1108                                 vars->cryptBytes += lastReadLength;
1109                         }
1110 #endif /* CRYPTO */
1111                 }
1112         }while (size);
1113
1114         return err;
1115 }
1116
1117 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */