iokit/Kernel/IOPolledInterface.cpp

   1 /*
   2  * Copyright (c) 2006-2019 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <sys/uio.h>
  30 #include <sys/conf.h>
  31
  32 #include <IOKit/IOLib.h>
  33 #include <IOKit/IOBSD.h>
  34 #include <IOKit/IOService.h>
  35 #include <IOKit/IOPlatformExpert.h>
  36 #include <IOKit/IOPolledInterface.h>
  37 #include <IOKit/IOHibernatePrivate.h>
  38 #include <IOKit/IOBufferMemoryDescriptor.h>
  39 #include <IOKit/AppleKeyStoreInterface.h>
  40 #include "IOKitKernelInternal.h"
  41
  42
  43 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  44
  45 OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject);
  46
  47 OSMetaClassDefineReservedUsed(IOPolledInterface, 0);
  48 OSMetaClassDefineReservedUnused(IOPolledInterface, 1);
  49 OSMetaClassDefineReservedUnused(IOPolledInterface, 2);
  50 OSMetaClassDefineReservedUnused(IOPolledInterface, 3);
  51 OSMetaClassDefineReservedUnused(IOPolledInterface, 4);
  52 OSMetaClassDefineReservedUnused(IOPolledInterface, 5);
  53 OSMetaClassDefineReservedUnused(IOPolledInterface, 6);
  54 OSMetaClassDefineReservedUnused(IOPolledInterface, 7);
  55 OSMetaClassDefineReservedUnused(IOPolledInterface, 8);
  56 OSMetaClassDefineReservedUnused(IOPolledInterface, 9);
  57 OSMetaClassDefineReservedUnused(IOPolledInterface, 10);
  58 OSMetaClassDefineReservedUnused(IOPolledInterface, 11);
  59 OSMetaClassDefineReservedUnused(IOPolledInterface, 12);
  60 OSMetaClassDefineReservedUnused(IOPolledInterface, 13);
  61 OSMetaClassDefineReservedUnused(IOPolledInterface, 14);
  62 OSMetaClassDefineReservedUnused(IOPolledInterface, 15);
  63
  64 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  65
  66 #ifndef kIOMediaPreferredBlockSizeKey
  67 #define kIOMediaPreferredBlockSizeKey   "Preferred Block Size"
  68 #endif
  69
  70 enum { kDefaultIOSize = 128 * 1024 };
  71
  72 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  73
  74 class IOPolledFilePollers : public OSObject
  75 {
  76         OSDeclareDefaultStructors(IOPolledFilePollers);
  77
  78 public:
  79         IOService                * media;
  80         OSArray                  * pollers;
  81         IOBufferMemoryDescriptor * ioBuffer;
  82         bool                 abortable;
  83         bool                 io;
  84         IOReturn             ioStatus;
  85         uint32_t             openCount;
  86
  87         static IOPolledFilePollers * copyPollers(IOService * media);
  88 };
  89
  90 OSDefineMetaClassAndStructors(IOPolledFilePollers, OSObject)
  91
  92 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  93
  94 IOPolledFilePollers *
  95 IOPolledFilePollers::copyPollers(IOService * media)
  96 {
  97         IOPolledFilePollers * vars;
  98         IOReturn              err;
  99         IOService       * service;
 100         OSObject        * obj;
 101         IORegistryEntry * next;
 102         IORegistryEntry * child;
 103
 104         if ((obj = media->copyProperty(kIOPolledInterfaceStackKey))) {
 105                 return OSDynamicCast(IOPolledFilePollers, obj);
 106         }
 107
 108         do{
 109                 vars = OSTypeAlloc(IOPolledFilePollers);
 110                 vars->init();
 111
 112                 vars->pollers = OSArray::withCapacity(4);
 113                 if (!vars->pollers) {
 114                         err = kIOReturnNoMemory;
 115                         break;
 116                 }
 117
 118                 next = vars->media = media;
 119                 do{
 120                         IOPolledInterface * poller;
 121                         OSObject *          obj;
 122
 123                         obj = next->getProperty(kIOPolledInterfaceSupportKey);
 124                         if (kOSBooleanFalse == obj) {
 125                                 vars->pollers->flushCollection();
 126                                 break;
 127                         } else if ((poller = OSDynamicCast(IOPolledInterface, obj))) {
 128                                 vars->pollers->setObject(poller);
 129                         }
 130
 131                         if ((service = OSDynamicCast(IOService, next))
 132                             && service->getDeviceMemory()
 133                             && !vars->pollers->getCount()) {
 134                                 break;
 135                         }
 136
 137                         child = next;
 138                 }while ((next = child->getParentEntry(gIOServicePlane))
 139                     && child->isParent(next, gIOServicePlane, true));
 140
 141                 if (!vars->pollers->getCount()) {
 142                         err = kIOReturnUnsupported;
 143                         break;
 144                 }
 145         }while (false);
 146
 147         media->setProperty(kIOPolledInterfaceStackKey, vars);
 148
 149         return vars;
 150 }
 151
 152 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 153
 154 static IOReturn
 155 IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable);
 156
 157 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 158
 159 static IOReturn
 160 IOPolledFilePollersProbe(IOPolledFilePollers * vars)
 161 {
 162         IOReturn            err = kIOReturnError;
 163         int32_t             idx;
 164         IOPolledInterface * poller;
 165
 166         for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--) {
 167                 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
 168                 err = poller->probe(vars->media);
 169                 if (err) {
 170                         HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
 171                         break;
 172                 }
 173         }
 174
 175         return err;
 176 }
 177
 178 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 179
 180 IOReturn
 181 IOPolledFilePollersOpen(IOPolledFileIOVars * filevars, uint32_t state, bool abortable)
 182 {
 183         IOPolledFilePollers      * vars = filevars->pollers;
 184         IOBufferMemoryDescriptor * ioBuffer;
 185         IOPolledInterface        * poller;
 186         IOService                * next;
 187         IOReturn                   err = kIOReturnError;
 188         int32_t                    idx;
 189
 190         vars->abortable = abortable;
 191         ioBuffer = NULL;
 192
 193         if (kIOPolledAfterSleepState == state) {
 194                 vars->ioStatus = 0;
 195                 vars->io = false;
 196         }
 197         (void) IOPolledFilePollersIODone(vars, false);
 198
 199         if ((kIOPolledPreflightState == state) || (kIOPolledPreflightCoreDumpState == state)) {
 200                 ioBuffer = vars->ioBuffer;
 201                 if (!ioBuffer) {
 202                         vars->ioBuffer = ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionInOut,
 203                             2 * kDefaultIOSize, page_size);
 204                         if (!ioBuffer) {
 205                                 return kIOReturnNoMemory;
 206                         }
 207                 }
 208         }
 209
 210         for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--) {
 211                 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
 212                 err = poller->open(state, ioBuffer);
 213                 if (kIOReturnSuccess != err) {
 214                         HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
 215                         break;
 216                 }
 217         }
 218         if ((kIOReturnSuccess == err) && (kIOPolledPreflightState == state)) {
 219                 next = vars->media;
 220                 while (next) {
 221                         next->setProperty(kIOPolledInterfaceActiveKey, kOSBooleanTrue);
 222                         next = next->getProvider();
 223                 }
 224         }
 225
 226         return err;
 227 }
 228
 229 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 230
 231 IOReturn
 232 IOPolledFilePollersClose(IOPolledFileIOVars * filevars, uint32_t state)
 233 {
 234         IOPolledFilePollers * vars = filevars->pollers;
 235         IOPolledInterface * poller;
 236         IORegistryEntry *   next;
 237         IOReturn            err;
 238         int32_t             idx;
 239
 240         (void) IOPolledFilePollersIODone(vars, false);
 241
 242         if ((kIOPolledPostflightState == state) || (kIOPolledPostflightCoreDumpState == state)) {
 243                 vars->openCount--;
 244         }
 245
 246         for (idx = 0, err = kIOReturnSuccess;
 247             (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
 248             idx++) {
 249                 err = poller->close(state);
 250                 if ((kIOReturnSuccess != err) && (kIOPolledBeforeSleepStateAborted == state)) {
 251                         err = poller->close(kIOPolledBeforeSleepState);
 252                 }
 253                 if (err) {
 254                         HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
 255                 }
 256         }
 257
 258         if (kIOPolledPostflightState == state) {
 259                 next = vars->media;
 260                 while (next) {
 261                         next->removeProperty(kIOPolledInterfaceActiveKey);
 262                         next = next->getParentEntry(gIOServicePlane);
 263                 }
 264         }
 265
 266         if ((kIOPolledPostflightState == state) || (kIOPolledPostflightCoreDumpState == state)) {
 267                 do{
 268                         if (vars->openCount) {
 269                                 break;
 270                         }
 271                         if (vars->ioBuffer) {
 272                                 vars->ioBuffer->release();
 273                                 vars->ioBuffer = NULL;
 274                         }
 275                 }while (false);
 276         }
 277
 278         return err;
 279 }
 280
 281 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 282
 283 IOReturn
 284 IOPolledInterface::setEncryptionKey(const uint8_t * key, size_t keySize)
 285 {
 286         return kIOReturnUnsupported;
 287 }
 288
 289 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 290
 291 IOReturn
 292 IOPolledFilePollersSetEncryptionKey(IOPolledFileIOVars * filevars,
 293     const uint8_t * key, size_t keySize)
 294 {
 295         IOReturn              ret = kIOReturnUnsupported;
 296         IOReturn              err;
 297         int32_t               idx;
 298         IOPolledFilePollers * vars = filevars->pollers;
 299         IOPolledInterface   * poller;
 300
 301         for (idx = 0;
 302             (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
 303             idx++) {
 304                 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
 305                 err = poller->setEncryptionKey(key, keySize);
 306                 if (kIOReturnSuccess == err) {
 307                         ret = err;
 308                 }
 309         }
 310
 311         return ret;
 312 }
 313
 314 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 315
 316 IOMemoryDescriptor *
 317 IOPolledFileGetIOBuffer(IOPolledFileIOVars * vars)
 318 {
 319         return vars->pollers->ioBuffer;
 320 }
 321
 322 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 323
 324 static void
 325 IOPolledIOComplete(void *   target,
 326     void *   parameter,
 327     IOReturn status,
 328     UInt64   actualByteCount)
 329 {
 330         IOPolledFilePollers * vars = (IOPolledFilePollers *) parameter;
 331
 332         vars->ioStatus = status;
 333 }
 334
 335 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 336
 337 static IOReturn
 338 IOStartPolledIO(IOPolledFilePollers * vars,
 339     uint32_t operation, uint32_t bufferOffset,
 340     uint64_t deviceOffset, uint64_t length)
 341 {
 342         IOReturn            err;
 343         IOPolledInterface * poller;
 344         IOPolledCompletion  completion;
 345
 346         err = vars->ioStatus;
 347         if (kIOReturnSuccess != err) {
 348                 return err;
 349         }
 350
 351         completion.target    = NULL;
 352         completion.action    = &IOPolledIOComplete;
 353         completion.parameter = vars;
 354
 355         vars->ioStatus = -1;
 356
 357         poller = (IOPolledInterface *) vars->pollers->getObject(0);
 358         err = poller->startIO(operation, bufferOffset, deviceOffset, length, completion);
 359         if (err) {
 360                 if (kernel_debugger_entry_count) {
 361                         HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
 362                 } else {
 363                         HIBLOGFROMPANIC("IOPolledInterface::IOStartPolledIO(0x%p, %d, 0x%x, 0x%llx, %llu) : poller->startIO(%d, 0x%x, 0x%llx, %llu, completion) returned 0x%x",
 364                             vars, operation, bufferOffset, deviceOffset, length, operation, bufferOffset, deviceOffset, length, err);
 365                 }
 366         }
 367         return err;
 368 }
 369
 370 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 371
 372 static IOReturn
 373 IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable)
 374 {
 375         IOReturn            err = kIOReturnSuccess;
 376         int32_t             idx = 0;
 377         IOPolledInterface * poller;
 378         AbsoluteTime        deadline;
 379
 380         if (!vars->io) {
 381                 return kIOReturnSuccess;
 382         }
 383
 384         abortable &= vars->abortable;
 385
 386         clock_interval_to_deadline(2000, kMillisecondScale, &deadline);
 387
 388         while (-1 == vars->ioStatus) {
 389                 for (idx = 0;
 390                     (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
 391                     idx++) {
 392                         IOReturn newErr;
 393                         newErr = poller->checkForWork();
 394                         if ((newErr == kIOReturnAborted) && !abortable) {
 395                                 newErr = kIOReturnSuccess;
 396                         }
 397                         if (kIOReturnSuccess == err) {
 398                                 err = newErr;
 399                         }
 400                 }
 401                 if ((false) && (kIOReturnSuccess == err) && (mach_absolute_time() > AbsoluteTime_to_scalar(&deadline))) {
 402                         HIBLOG("IOPolledInterface::forced timeout\n");
 403                         vars->ioStatus = kIOReturnTimeout;
 404                 }
 405         }
 406         vars->io = false;
 407
 408 #if HIBERNATION
 409         if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort()) {
 410                 err = kIOReturnAborted;
 411                 HIBLOG("IOPolledInterface::checkForWork sw abort\n");
 412         }
 413 #endif
 414
 415         if (err) {
 416                 HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
 417         } else {
 418                 err = vars->ioStatus;
 419                 if (kIOReturnSuccess != err) {
 420                         HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
 421                 }
 422         }
 423
 424         return err;
 425 }
 426
 427 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 428 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 429
 430 struct _OpenFileContext {
 431         OSData * extents;
 432         uint64_t size;
 433 };
 434
 435 static void
 436 file_extent_callback(void * ref, uint64_t start, uint64_t length)
 437 {
 438         _OpenFileContext * ctx = (_OpenFileContext *) ref;
 439         IOPolledFileExtent extent;
 440
 441         extent.start  = start;
 442         extent.length = length;
 443         ctx->extents->appendBytes(&extent, sizeof(extent));
 444         ctx->size += length;
 445 }
 446
 447 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 448
 449 static IOService *
 450 IOCopyMediaForDev(dev_t device)
 451 {
 452         OSDictionary * matching;
 453         OSNumber *     num;
 454         OSIterator *   iter;
 455         IOService *    result = NULL;
 456
 457         matching = IOService::serviceMatching("IOMedia");
 458         if (!matching) {
 459                 return NULL;
 460         }
 461         do{
 462                 num = OSNumber::withNumber(major(device), 32);
 463                 if (!num) {
 464                         break;
 465                 }
 466                 matching->setObject(kIOBSDMajorKey, num);
 467                 num->release();
 468                 num = OSNumber::withNumber(minor(device), 32);
 469                 if (!num) {
 470                         break;
 471                 }
 472                 matching->setObject(kIOBSDMinorKey, num);
 473                 num->release();
 474                 if (!num) {
 475                         break;
 476                 }
 477                 iter = IOService::getMatchingServices(matching);
 478                 if (iter) {
 479                         result = (IOService *) iter->getNextObject();
 480                         result->retain();
 481                         iter->release();
 482                 }
 483         }while (false);
 484         matching->release();
 485
 486         return result;
 487 }
 488
 489 #define APFSMEDIA_GETHIBERKEY         "getHiberKey"
 490
 491 static IOReturn
 492 IOGetVolumeCryptKey(dev_t block_dev,
 493     LIBKERN_RETURNS_RETAINED OSString ** pKeyUUID,
 494     uint8_t * volumeCryptKey,
 495     size_t * keySize)
 496 {
 497         IOReturn         err;
 498         IOService *      part;
 499         OSString *       keyUUID = NULL;
 500         OSString *       keyStoreUUID = NULL;
 501         uuid_t           volumeKeyUUID;
 502         aks_volume_key_t vek;
 503         size_t           callerKeySize;
 504
 505         static IOService * sKeyStore;
 506
 507         part = IOCopyMediaForDev(block_dev);
 508         if (!part) {
 509                 return kIOReturnNotFound;
 510         }
 511
 512         callerKeySize = *keySize;
 513         // Try APFS first
 514         {
 515                 uuid_t volUuid = {0};
 516                 err = part->callPlatformFunction(APFSMEDIA_GETHIBERKEY, false, &volUuid, volumeCryptKey, keySize, keySize);
 517                 if (kIOReturnBadArgument == err) {
 518                         // apfs fails on buffer size >32
 519                         *keySize = 32;
 520                         err = part->callPlatformFunction(APFSMEDIA_GETHIBERKEY, false, &volUuid, volumeCryptKey, keySize, keySize);
 521                 }
 522                 if (err != kIOReturnSuccess) {
 523                         *keySize = 0;
 524                 } else {
 525                         // No need to create uuid string if it's not requested
 526                         if (pKeyUUID) {
 527                                 uuid_string_t volUuidStr;
 528                                 uuid_unparse(volUuid, volUuidStr);
 529                                 *pKeyUUID = OSString::withCString(volUuidStr);
 530                         }
 531
 532                         part->release();
 533                         return kIOReturnSuccess;
 534                 }
 535         }
 536
 537         // Then old CS path
 538         err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false,
 539             (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL);
 540         if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID) {
 541 //        IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy());
 542
 543                 if (!sKeyStore) {
 544                         sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane);
 545                 }
 546                 if (sKeyStore) {
 547                         err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID);
 548                 } else {
 549                         err = kIOReturnNoResources;
 550                 }
 551                 if (kIOReturnSuccess == err) {
 552                         err = sKeyStore->callPlatformFunction(gAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL);
 553                 }
 554                 if (kIOReturnSuccess != err) {
 555                         IOLog("volume key err 0x%x\n", err);
 556                 } else {
 557                         if (vek.key.keybytecount <= callerKeySize) {
 558                                 *keySize = vek.key.keybytecount;
 559                         }
 560                         bcopy(&vek.key.keybytes[0], volumeCryptKey, *keySize);
 561                 }
 562                 bzero(&vek, sizeof(vek));
 563
 564                 if (pKeyUUID) {
 565                         // Create a copy because the caller would release it
 566                         *pKeyUUID = OSString::withString(keyUUID);
 567                 }
 568         }
 569
 570         part->release();
 571         return err;
 572 }
 573
 574 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 575
 576 IOReturn
 577 IOPolledFileOpen(const char * filename,
 578     uint32_t flags,
 579     uint64_t setFileSize, uint64_t fsFreeSize,
 580     void * write_file_addr, size_t write_file_len,
 581     IOPolledFileIOVars ** fileVars,
 582     OSData ** imagePath,
 583     uint8_t * volumeCryptKey, size_t * keySize)
 584 {
 585         IOReturn             err = kIOReturnSuccess;
 586         IOPolledFileIOVars * vars;
 587         _OpenFileContext     ctx;
 588         OSData *             extentsData = NULL;
 589         OSNumber *           num;
 590         IOService *          part = NULL;
 591         dev_t                block_dev;
 592         dev_t                image_dev;
 593         AbsoluteTime         startTime, endTime;
 594         uint64_t             nsec;
 595
 596         vars = IONew(IOPolledFileIOVars, 1);
 597         if (!vars) {
 598                 return kIOReturnNoMemory;
 599         }
 600         bzero(vars, sizeof(*vars));
 601         vars->allocated = true;
 602
 603         do{
 604                 extentsData = OSData::withCapacity(32);
 605                 ctx.extents = extentsData;
 606                 ctx.size    = 0;
 607                 clock_get_uptime(&startTime);
 608
 609                 vars->fileRef = kern_open_file_for_direct_io(filename,
 610                     flags,
 611                     &file_extent_callback, &ctx,
 612                     setFileSize,
 613                     fsFreeSize,
 614                     // write file:
 615                     0, write_file_addr, write_file_len,
 616                     // results
 617                     &block_dev,
 618                     &image_dev,
 619                     &vars->block0,
 620                     &vars->maxiobytes,
 621                     &vars->flags);
 622 #if 0
 623                 uint32_t msDelay = (131071 & random());
 624                 HIBLOG("sleep %d\n", msDelay);
 625                 IOSleep(msDelay);
 626 #endif
 627                 clock_get_uptime(&endTime);
 628                 SUB_ABSOLUTETIME(&endTime, &startTime);
 629                 absolutetime_to_nanoseconds(endTime, &nsec);
 630
 631                 if (!vars->fileRef) {
 632                         err = kIOReturnNoSpace;
 633                 }
 634
 635                 HIBLOG("kern_open_file_for_direct_io took %qd ms\n", nsec / 1000000ULL);
 636                 if (kIOReturnSuccess != err) {
 637                         break;
 638                 }
 639
 640                 HIBLOG("Opened file %s, size %qd, extents %ld, maxio %qx ssd %d\n", filename, ctx.size,
 641                     (extentsData->getLength() / sizeof(IOPolledFileExtent)) - 1,
 642                     vars->maxiobytes, kIOPolledFileSSD & vars->flags);
 643                 assert(!vars->block0);
 644                 if (extentsData->getLength() < sizeof(IOPolledFileExtent)) {
 645                         err = kIOReturnNoSpace;
 646                         break;
 647                 }
 648
 649                 vars->fileSize = ctx.size;
 650                 vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
 651
 652                 part = IOCopyMediaForDev(image_dev);
 653                 if (!part) {
 654                         err = kIOReturnNotFound;
 655                         break;
 656                 }
 657
 658                 if (!(vars->pollers = IOPolledFilePollers::copyPollers(part))) {
 659                         break;
 660                 }
 661
 662                 if ((num = OSDynamicCast(OSNumber, part->getProperty(kIOMediaPreferredBlockSizeKey)))) {
 663                         vars->blockSize = num->unsigned32BitValue();
 664                 }
 665                 if (vars->blockSize < 4096) {
 666                         vars->blockSize = 4096;
 667                 }
 668
 669                 HIBLOG("polled file major %d, minor %d, blocksize %ld, pollers %d\n",
 670                     major(image_dev), minor(image_dev), (long)vars->blockSize,
 671                     vars->pollers->pollers->getCount());
 672
 673                 OSString * keyUUID = NULL;
 674                 if (volumeCryptKey) {
 675                         err = IOGetVolumeCryptKey(block_dev, &keyUUID, volumeCryptKey, keySize);
 676                 }
 677
 678                 *fileVars    = vars;
 679                 vars->fileExtents = extentsData;
 680
 681                 // make imagePath
 682                 OSData * data;
 683                 if (imagePath) {
 684 #if defined(__i386__) || defined(__x86_64__)
 685                         char str2[24 + sizeof(uuid_string_t) + 2];
 686
 687                         if (keyUUID) {
 688                                 snprintf(str2, sizeof(str2), "%qx:%s",
 689                                     vars->extentMap[0].start, keyUUID->getCStringNoCopy());
 690                         } else {
 691                                 snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
 692                         }
 693
 694                         err = IOService::getPlatform()->callPlatformFunction(
 695                                 gIOCreateEFIDevicePathSymbol, false,
 696                                 (void *) part, (void *) str2,
 697                                 (void *) (uintptr_t) true, (void *) &data);
 698 #else
 699                         data = NULL;
 700                         err = kIOReturnSuccess;
 701 #endif
 702                         if (kIOReturnSuccess != err) {
 703                                 HIBLOG("error 0x%x getting path\n", err);
 704                                 break;
 705                         }
 706                         *imagePath = data;
 707                 }
 708
 709                 // Release key UUID if we have one
 710                 if (keyUUID) {
 711                         keyUUID->release();
 712                         keyUUID = NULL; // Just in case
 713                 }
 714         }while (false);
 715
 716         if (kIOReturnSuccess != err) {
 717                 HIBLOG("error 0x%x opening polled file\n", err);
 718                 IOPolledFileClose(&vars, 0, NULL, 0, 0, 0);
 719                 if (extentsData) {
 720                         extentsData->release();
 721                 }
 722         }
 723
 724         if (part) {
 725                 part->release();
 726         }
 727
 728         return err;
 729 }
 730
 731 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 732
 733 IOReturn
 734 IOPolledFileClose(IOPolledFileIOVars ** pVars,
 735     off_t write_offset, void * addr, size_t write_length,
 736     off_t discard_offset, off_t discard_end)
 737 {
 738         IOPolledFileIOVars * vars;
 739
 740         vars = *pVars;
 741         if (!vars) {
 742                 return kIOReturnSuccess;
 743         }
 744
 745         if (vars->fileRef) {
 746                 kern_close_file_for_direct_io(vars->fileRef, write_offset, addr, write_length,
 747                     discard_offset, discard_end);
 748                 vars->fileRef = NULL;
 749         }
 750         if (vars->fileExtents) {
 751                 vars->fileExtents->release();
 752                 vars->fileExtents = NULL;
 753         }
 754         if (vars->pollers) {
 755                 vars->pollers->release();
 756                 vars->pollers = NULL;
 757         }
 758
 759         if (vars->allocated) {
 760                 IODelete(vars, IOPolledFileIOVars, 1);
 761         } else {
 762                 bzero(vars, sizeof(IOPolledFileIOVars));
 763         }
 764         *pVars = NULL;
 765
 766         return kIOReturnSuccess;
 767 }
 768
 769 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 770
 771 IOReturn
 772 IOPolledFilePollersSetup(IOPolledFileIOVars * vars,
 773     uint32_t openState)
 774 {
 775         IOReturn err;
 776
 777         err = kIOReturnSuccess;
 778         do{
 779                 if (!vars->pollers->openCount) {
 780                         err = IOPolledFilePollersProbe(vars->pollers);
 781                         if (kIOReturnSuccess != err) {
 782                                 break;
 783                         }
 784                 }
 785                 err = IOPolledFilePollersOpen(vars, openState, false);
 786                 if (kIOReturnSuccess != err) {
 787                         break;
 788                 }
 789                 if ((kIOPolledPreflightState == openState) || (kIOPolledPreflightCoreDumpState == openState)) {
 790                         vars->pollers->openCount++;
 791                 }
 792                 vars->pollers->io  = false;
 793                 vars->buffer       = (uint8_t *) vars->pollers->ioBuffer->getBytesNoCopy();
 794                 vars->bufferHalf   = 0;
 795                 vars->bufferOffset = 0;
 796                 vars->bufferSize   = (vars->pollers->ioBuffer->getLength() >> 1);
 797
 798                 if (vars->maxiobytes < vars->bufferSize) {
 799                         vars->bufferSize = vars->maxiobytes;
 800                 }
 801         }while (false);
 802
 803         if (kIOReturnSuccess != err) {
 804                 HIBLOG("IOPolledFilePollersSetup(%d) error 0x%x\n", openState, err);
 805         }
 806
 807         return err;
 808 }
 809
 810
 811 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 812
 813 IOReturn
 814 IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
 815 {
 816         IOPolledFileExtent * extentMap;
 817
 818         extentMap = vars->extentMap;
 819
 820         vars->position = position;
 821
 822         if (position > vars->fileSize) {
 823                 HIBLOG("IOPolledFileSeek: called to seek to 0x%llx greater than file size of 0x%llx\n", vars->position, vars->fileSize);
 824                 return kIOReturnNoSpace;
 825         }
 826
 827         while (position >= extentMap->length) {
 828                 position -= extentMap->length;
 829                 extentMap++;
 830         }
 831
 832         vars->currentExtent   = extentMap;
 833         vars->extentRemaining = extentMap->length - position;
 834         vars->extentPosition  = vars->position - position;
 835
 836         if (vars->bufferSize <= vars->extentRemaining) {
 837                 vars->bufferLimit = vars->bufferSize;
 838         } else {
 839                 vars->bufferLimit = vars->extentRemaining;
 840         }
 841
 842         return kIOReturnSuccess;
 843 }
 844
 845 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 846
 847 IOReturn
 848 IOPolledFileWrite(IOPolledFileIOVars * vars,
 849     const uint8_t * bytes, IOByteCount size,
 850     IOPolledFileCryptVars * cryptvars)
 851 {
 852         IOReturn    err = kIOReturnSuccess;
 853         IOByteCount copy, original_size = size;
 854         bool        flush = false;
 855
 856         do{
 857                 if (!bytes && !size) {
 858                         // seek to end of block & flush
 859                         size = vars->position & (vars->blockSize - 1);
 860                         if (size) {
 861                                 size = vars->blockSize - size;
 862                         }
 863                         flush = true;
 864                         // use some garbage for the fill
 865                         bytes = vars->buffer + vars->bufferOffset;
 866                 }
 867
 868                 copy = vars->bufferLimit - vars->bufferOffset;
 869                 if (copy > size) {
 870                         copy = size;
 871                 } else {
 872                         flush = true;
 873                 }
 874
 875                 if (bytes) {
 876 #if KASAN
 877                         /* Since this may copy mach-o segments in bulk, use the nosan variants of bcopy to
 878                          * avoid triggering global redzone sanitizer violations when accessing
 879                          * interstices between 'C' structures
 880                          */
 881                         __nosan_bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
 882 #else
 883                         bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
 884 #endif
 885                         bytes += copy;
 886                 } else {
 887                         bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
 888                 }
 889
 890                 size -= copy;
 891                 vars->bufferOffset += copy;
 892                 vars->position += copy;
 893
 894                 if (flush && vars->bufferOffset) {
 895                         uint64_t offset = (vars->position - vars->bufferOffset
 896                             - vars->extentPosition + vars->currentExtent->start);
 897                         uint32_t length = (vars->bufferOffset);
 898
 899 #if CRYPTO
 900                         if (cryptvars && vars->encryptStart
 901                             && (vars->position > vars->encryptStart)
 902                             && ((vars->position - length) < vars->encryptEnd)) {
 903                                 AbsoluteTime startTime, endTime;
 904
 905                                 uint64_t encryptLen, encryptStart;
 906                                 encryptLen = vars->position - vars->encryptStart;
 907                                 if (encryptLen > length) {
 908                                         encryptLen = length;
 909                                 }
 910                                 encryptStart = length - encryptLen;
 911                                 if (vars->position > vars->encryptEnd) {
 912                                         encryptLen -= (vars->position - vars->encryptEnd);
 913                                 }
 914
 915                                 clock_get_uptime(&startTime);
 916
 917                                 // encrypt the buffer
 918                                 aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
 919                                     &cryptvars->aes_iv[0],
 920                                     encryptLen / AES_BLOCK_SIZE,
 921                                     vars->buffer + vars->bufferHalf + encryptStart,
 922                                     &cryptvars->ctx.encrypt);
 923
 924                                 clock_get_uptime(&endTime);
 925                                 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
 926                                 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
 927                                 vars->cryptBytes += encryptLen;
 928
 929                                 // save initial vector for following encrypts
 930                                 bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
 931                                     &cryptvars->aes_iv[0],
 932                                     AES_BLOCK_SIZE);
 933                         }
 934 #endif /* CRYPTO */
 935
 936                         err = IOPolledFilePollersIODone(vars->pollers, true);
 937                         if (kIOReturnSuccess != err) {
 938                                 break;
 939                         }
 940
 941                         if (vars->position & (vars->blockSize - 1)) {
 942                                 HIBLOG("misaligned file pos %qx\n", vars->position);
 943                         }
 944 //if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
 945
 946                         err = IOStartPolledIO(vars->pollers, kIOPolledWrite, vars->bufferHalf, offset, length);
 947                         if (kIOReturnSuccess != err) {
 948                                 HIBLOGFROMPANIC("IOPolledFileWrite(0x%p, 0x%p, %llu, 0x%p) : IOStartPolledIO(0x%p, kIOPolledWrite, %llu, 0x%llx, %d) returned 0x%x\n",
 949                                     vars, bytes, (uint64_t) original_size, cryptvars, vars->pollers, (uint64_t) vars->bufferHalf, offset, length, err);
 950                                 break;
 951                         }
 952                         vars->pollers->io = true;
 953
 954                         vars->extentRemaining -= vars->bufferOffset;
 955                         if (!vars->extentRemaining) {
 956                                 vars->currentExtent++;
 957                                 vars->extentRemaining = vars->currentExtent->length;
 958                                 vars->extentPosition  = vars->position;
 959                         }
 960
 961                         vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
 962                         vars->bufferOffset = 0;
 963                         if (vars->bufferSize <= vars->extentRemaining) {
 964                                 vars->bufferLimit = vars->bufferSize;
 965                         } else {
 966                                 vars->bufferLimit = vars->extentRemaining;
 967                         }
 968
 969                         if (!vars->extentRemaining) {
 970                                 err = kIOReturnOverrun;
 971                                 break;
 972                         }
 973
 974                         flush = false;
 975                 }
 976         }while (size);
 977
 978         return err;
 979 }
 980
 981 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 982
 983 IOReturn
 984 IOPolledFileFlush(IOPolledFileIOVars * vars)
 985 {
 986         // Only supported by the underlying polled mode driver on embedded currently (expect kIOReturnUnsupported on other platforms)
 987         IOReturn err = kIOReturnSuccess;
 988
 989         err = IOPolledFilePollersIODone(vars->pollers, true);
 990         if (kIOReturnSuccess != err) {
 991                 return err;
 992         }
 993
 994         err = IOStartPolledIO(vars->pollers, kIOPolledFlush, 0, 0, 0);
 995         if (kIOReturnSuccess != err) {
 996                 HIBLOGFROMPANIC("IOPolledFileFlush(0x%p) : IOStartPolledIO(0x%p, kIOPolledFlush, 0, 0, 0) returned 0x%x\n",
 997                     vars, vars->pollers, err);
 998                 return err;
 999         }
1000         vars->pollers->io = true;
1001
1002         return err;
1003 }
1004
1005 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1006
1007 IOReturn
1008 IOPolledFileRead(IOPolledFileIOVars * vars,
1009     uint8_t * bytes, IOByteCount size,
1010     IOPolledFileCryptVars * cryptvars)
1011 {
1012         IOReturn    err = kIOReturnSuccess;
1013         IOByteCount copy;
1014
1015 //    bytesWritten += size;
1016
1017         do{
1018                 copy = vars->bufferLimit - vars->bufferOffset;
1019                 if (copy > size) {
1020                         copy = size;
1021                 }
1022
1023                 if (bytes) {
1024 #if KASAN
1025                         __nosan_bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
1026 #else
1027                         bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
1028 #endif
1029                         bytes += copy;
1030                 }
1031                 size -= copy;
1032                 vars->bufferOffset += copy;
1033 //      vars->position += copy;
1034
1035                 if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd)) {
1036                         if (!vars->pollers->io) {
1037                                 cryptvars = NULL;
1038                         }
1039                         err = IOPolledFilePollersIODone(vars->pollers, true);
1040                         if (kIOReturnSuccess != err) {
1041                                 break;
1042                         }
1043
1044                         if (vars->position & (vars->blockSize - 1)) {
1045                                 HIBLOG("misaligned file pos %qx\n", vars->position);
1046                         }
1047
1048                         vars->position        += vars->lastRead;
1049                         vars->extentRemaining -= vars->lastRead;
1050                         vars->bufferLimit      = vars->lastRead;
1051
1052                         if (!vars->extentRemaining) {
1053                                 vars->currentExtent++;
1054                                 vars->extentRemaining = vars->currentExtent->length;
1055                                 vars->extentPosition  = vars->position;
1056                                 if (!vars->extentRemaining) {
1057                                         err = kIOReturnOverrun;
1058                                         break;
1059                                 }
1060                         }
1061
1062                         uint64_t length;
1063                         uint64_t lastReadLength = vars->lastRead;
1064                         uint64_t offset = (vars->position
1065                             - vars->extentPosition + vars->currentExtent->start);
1066                         if (vars->extentRemaining <= vars->bufferSize) {
1067                                 length = vars->extentRemaining;
1068                         } else {
1069                                 length = vars->bufferSize;
1070                         }
1071                         if ((length + vars->position) > vars->readEnd) {
1072                                 length = vars->readEnd - vars->position;
1073                         }
1074
1075                         vars->lastRead = length;
1076                         if (length) {
1077 //if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
1078                                 err = IOStartPolledIO(vars->pollers, kIOPolledRead, vars->bufferHalf, offset, length);
1079                                 if (kIOReturnSuccess != err) {
1080                                         break;
1081                                 }
1082                                 vars->pollers->io = true;
1083                         }
1084
1085                         vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
1086                         vars->bufferOffset = 0;
1087
1088 #if CRYPTO
1089                         if (cryptvars) {
1090                                 uint8_t thisVector[AES_BLOCK_SIZE];
1091                                 AbsoluteTime startTime, endTime;
1092
1093                                 // save initial vector for following decrypts
1094                                 bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
1095                                 bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE,
1096                                     &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1097
1098                                 // decrypt the buffer
1099                                 clock_get_uptime(&startTime);
1100
1101                                 aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
1102                                     &thisVector[0],
1103                                     lastReadLength / AES_BLOCK_SIZE,
1104                                     vars->buffer + vars->bufferHalf,
1105                                     &cryptvars->ctx.decrypt);
1106
1107                                 clock_get_uptime(&endTime);
1108                                 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
1109                                 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
1110                                 vars->cryptBytes += lastReadLength;
1111                         }
1112 #endif /* CRYPTO */
1113                 }
1114         }while (size);
1115
1116         return err;
1117 }
1118
1119 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */