iokit/Kernel/IOPolledInterface.cpp

   1 /*
   2  * Copyright (c) 2006-2019 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <sys/uio.h>
  30 #include <sys/conf.h>
  31
  32 #include <IOKit/IOLib.h>
  33 #include <IOKit/IOBSD.h>
  34 #include <IOKit/IOService.h>
  35 #include <IOKit/IOPlatformExpert.h>
  36 #include <IOKit/IOPolledInterface.h>
  37 #include <IOKit/IOHibernatePrivate.h>
  38 #include <IOKit/IOBufferMemoryDescriptor.h>
  39 #include <IOKit/AppleKeyStoreInterface.h>
  40 #include <libkern/c++/OSSharedPtr.h>
  41 #include "IOKitKernelInternal.h"
  42
  43 #if defined(__arm64__)
  44 #include <pexpert/arm64/board_config.h>
  45 #if XNU_MONITOR_PPL_HIB
  46 #include <IOKit/SEPHibernator.h>
  47 #endif /* XNU_MONITOR_PPL_HIB */
  48 #endif /* defined(__arm64__) */
  49
  50 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  51
  52 OSDefineMetaClassAndAbstractStructors(IOPolledInterface, OSObject);
  53
  54 OSMetaClassDefineReservedUsedX86(IOPolledInterface, 0);
  55 OSMetaClassDefineReservedUnused(IOPolledInterface, 1);
  56 OSMetaClassDefineReservedUnused(IOPolledInterface, 2);
  57 OSMetaClassDefineReservedUnused(IOPolledInterface, 3);
  58 OSMetaClassDefineReservedUnused(IOPolledInterface, 4);
  59 OSMetaClassDefineReservedUnused(IOPolledInterface, 5);
  60 OSMetaClassDefineReservedUnused(IOPolledInterface, 6);
  61 OSMetaClassDefineReservedUnused(IOPolledInterface, 7);
  62 OSMetaClassDefineReservedUnused(IOPolledInterface, 8);
  63 OSMetaClassDefineReservedUnused(IOPolledInterface, 9);
  64 OSMetaClassDefineReservedUnused(IOPolledInterface, 10);
  65 OSMetaClassDefineReservedUnused(IOPolledInterface, 11);
  66 OSMetaClassDefineReservedUnused(IOPolledInterface, 12);
  67 OSMetaClassDefineReservedUnused(IOPolledInterface, 13);
  68 OSMetaClassDefineReservedUnused(IOPolledInterface, 14);
  69 OSMetaClassDefineReservedUnused(IOPolledInterface, 15);
  70
  71 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  72
  73 #ifndef kIOMediaPreferredBlockSizeKey
  74 #define kIOMediaPreferredBlockSizeKey   "Preferred Block Size"
  75 #endif
  76
  77 enum { kDefaultIOSize = 128 * 1024 };
  78
  79 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  80
  81 class IOPolledFilePollers : public OSObject
  82 {
  83         OSDeclareDefaultStructors(IOPolledFilePollers);
  84
  85 public:
  86         IOService                * media;
  87         OSArray                  * pollers;
  88         IOBufferMemoryDescriptor * ioBuffer;
  89         bool                 abortable;
  90         bool                 io;
  91         IOReturn             ioStatus;
  92         uint32_t             openCount;
  93
  94         static IOPolledFilePollers * copyPollers(IOService * media);
  95 };
  96
  97 OSDefineMetaClassAndStructors(IOPolledFilePollers, OSObject)
  98
  99 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 100
 101 IOPolledFilePollers *
 102 IOPolledFilePollers::copyPollers(IOService * media)
 103 {
 104         IOPolledFilePollers * vars;
 105         IOReturn              err;
 106         IOService       * service;
 107         OSObject        * obj;
 108         IORegistryEntry * next;
 109         IORegistryEntry * child;
 110
 111         if ((obj = media->copyProperty(kIOPolledInterfaceStackKey))) {
 112                 return OSDynamicCast(IOPolledFilePollers, obj);
 113         }
 114
 115         do{
 116                 vars = OSTypeAlloc(IOPolledFilePollers);
 117                 vars->init();
 118
 119                 vars->pollers = OSArray::withCapacity(4);
 120                 if (!vars->pollers) {
 121                         err = kIOReturnNoMemory;
 122                         break;
 123                 }
 124
 125                 next = vars->media = media;
 126                 do{
 127                         IOPolledInterface * poller;
 128                         OSObject *          obj;
 129
 130                         obj = next->getProperty(kIOPolledInterfaceSupportKey);
 131                         if (kOSBooleanFalse == obj) {
 132                                 vars->pollers->flushCollection();
 133                                 break;
 134                         } else if ((poller = OSDynamicCast(IOPolledInterface, obj))) {
 135                                 vars->pollers->setObject(poller);
 136                         }
 137
 138                         if ((service = OSDynamicCast(IOService, next))
 139                             && service->getDeviceMemory()
 140                             && !vars->pollers->getCount()) {
 141                                 break;
 142                         }
 143
 144                         child = next;
 145                 }while ((next = child->getParentEntry(gIOServicePlane))
 146                     && child->isParent(next, gIOServicePlane, true));
 147
 148                 if (!vars->pollers->getCount()) {
 149                         err = kIOReturnUnsupported;
 150                         break;
 151                 }
 152         }while (false);
 153
 154         media->setProperty(kIOPolledInterfaceStackKey, vars);
 155
 156         return vars;
 157 }
 158
 159 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 160
 161 static IOReturn
 162 IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable);
 163
 164 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 165
 166 static IOReturn
 167 IOPolledFilePollersProbe(IOPolledFilePollers * vars)
 168 {
 169         IOReturn            err = kIOReturnError;
 170         int32_t             idx;
 171         IOPolledInterface * poller;
 172
 173         for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--) {
 174                 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
 175                 err = poller->probe(vars->media);
 176                 if (err) {
 177                         HIBLOG("IOPolledInterface::probe[%d] 0x%x\n", idx, err);
 178                         break;
 179                 }
 180         }
 181
 182         return err;
 183 }
 184
 185 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 186
 187 IOReturn
 188 IOPolledFilePollersOpen(IOPolledFileIOVars * filevars, uint32_t state, bool abortable)
 189 {
 190         IOPolledFilePollers      * vars = filevars->pollers;
 191         IOBufferMemoryDescriptor * ioBuffer;
 192         IOPolledInterface        * poller;
 193         IOService                * next;
 194         IOReturn                   err = kIOReturnError;
 195         int32_t                    idx;
 196
 197         vars->abortable = abortable;
 198         ioBuffer = NULL;
 199
 200         if (kIOPolledAfterSleepState == state) {
 201                 vars->ioStatus = 0;
 202                 vars->io = false;
 203         }
 204         (void) IOPolledFilePollersIODone(vars, false);
 205
 206         if ((kIOPolledPreflightState == state) || (kIOPolledPreflightCoreDumpState == state)) {
 207                 ioBuffer = vars->ioBuffer;
 208                 if (!ioBuffer) {
 209                         vars->ioBuffer = ioBuffer = IOBufferMemoryDescriptor::withOptions(kIODirectionInOut,
 210                             2 * kDefaultIOSize, page_size);
 211                         if (!ioBuffer) {
 212                                 return kIOReturnNoMemory;
 213                         }
 214                 }
 215         }
 216
 217         for (idx = vars->pollers->getCount() - 1; idx >= 0; idx--) {
 218                 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
 219                 err = poller->open(state, ioBuffer);
 220                 if (kIOReturnSuccess != err) {
 221                         HIBLOG("IOPolledInterface::open[%d] 0x%x\n", idx, err);
 222                         break;
 223                 }
 224         }
 225         if ((kIOReturnSuccess == err) && (kIOPolledPreflightState == state)) {
 226                 next = vars->media;
 227                 while (next) {
 228                         next->setProperty(kIOPolledInterfaceActiveKey, kOSBooleanTrue);
 229                         next = next->getProvider();
 230                 }
 231         }
 232
 233         return err;
 234 }
 235
 236 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 237
 238 IOReturn
 239 IOPolledFilePollersClose(IOPolledFileIOVars * filevars, uint32_t state)
 240 {
 241         IOPolledFilePollers * vars = filevars->pollers;
 242         IOPolledInterface * poller;
 243         IORegistryEntry *   next;
 244         IOReturn            err;
 245         int32_t             idx;
 246
 247         (void) IOPolledFilePollersIODone(vars, false);
 248
 249         if ((kIOPolledPostflightState == state) || (kIOPolledPostflightCoreDumpState == state)) {
 250                 vars->openCount--;
 251         }
 252
 253         for (idx = 0, err = kIOReturnSuccess;
 254             (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
 255             idx++) {
 256                 err = poller->close(state);
 257                 if ((kIOReturnSuccess != err) && (kIOPolledBeforeSleepStateAborted == state)) {
 258                         err = poller->close(kIOPolledBeforeSleepState);
 259                 }
 260                 if (err) {
 261                         HIBLOG("IOPolledInterface::close[%d] 0x%x\n", idx, err);
 262                 }
 263         }
 264
 265         if (kIOPolledPostflightState == state) {
 266                 next = vars->media;
 267                 while (next) {
 268                         next->removeProperty(kIOPolledInterfaceActiveKey);
 269                         next = next->getParentEntry(gIOServicePlane);
 270                 }
 271         }
 272
 273         if ((kIOPolledPostflightState == state) || (kIOPolledPostflightCoreDumpState == state)) {
 274                 do{
 275                         if (vars->openCount) {
 276                                 break;
 277                         }
 278                         if (vars->ioBuffer) {
 279                                 vars->ioBuffer->release();
 280                                 vars->ioBuffer = NULL;
 281                         }
 282                 }while (false);
 283         }
 284
 285         return err;
 286 }
 287
 288 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 289
 290 IOReturn
 291 IOPolledInterface::setEncryptionKey(const uint8_t * key, size_t keySize)
 292 {
 293         return kIOReturnUnsupported;
 294 }
 295
 296 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 297
 298 IOReturn
 299 IOPolledFilePollersSetEncryptionKey(IOPolledFileIOVars * filevars,
 300     const uint8_t * key, size_t keySize)
 301 {
 302         IOReturn              ret = kIOReturnUnsupported;
 303         IOReturn              err;
 304         int32_t               idx;
 305         IOPolledFilePollers * vars = filevars->pollers;
 306         IOPolledInterface   * poller;
 307
 308         for (idx = 0;
 309             (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
 310             idx++) {
 311                 poller = (IOPolledInterface *) vars->pollers->getObject(idx);
 312                 err = poller->setEncryptionKey(key, keySize);
 313                 if (kIOReturnSuccess == err) {
 314                         ret = err;
 315                 }
 316         }
 317
 318         return ret;
 319 }
 320
 321 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 322
 323 IOMemoryDescriptor *
 324 IOPolledFileGetIOBuffer(IOPolledFileIOVars * vars)
 325 {
 326         return vars->pollers->ioBuffer;
 327 }
 328
 329 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 330
 331 static void
 332 IOPolledIOComplete(void *   target,
 333     void *   parameter,
 334     IOReturn status,
 335     UInt64   actualByteCount)
 336 {
 337         IOPolledFilePollers * vars = (IOPolledFilePollers *) parameter;
 338
 339         vars->ioStatus = status;
 340 }
 341
 342 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 343
 344 static IOReturn
 345 IOStartPolledIO(IOPolledFilePollers * vars,
 346     uint32_t operation, uint32_t bufferOffset,
 347     uint64_t deviceOffset, uint64_t length)
 348 {
 349         IOReturn            err;
 350         IOPolledInterface * poller;
 351         IOPolledCompletion  completion;
 352
 353         err = vars->ioStatus;
 354         if (kIOReturnSuccess != err) {
 355                 return err;
 356         }
 357
 358         completion.target    = NULL;
 359         completion.action    = &IOPolledIOComplete;
 360         completion.parameter = vars;
 361
 362         vars->ioStatus = -1;
 363
 364         poller = (IOPolledInterface *) vars->pollers->getObject(0);
 365         err = poller->startIO(operation, bufferOffset, deviceOffset, length, completion);
 366         if (err) {
 367                 if (kernel_debugger_entry_count) {
 368                         HIBLOG("IOPolledInterface::startIO[%d] 0x%x\n", 0, err);
 369                 } else {
 370                         HIBLOGFROMPANIC("IOPolledInterface::IOStartPolledIO(0x%p, %d, 0x%x, 0x%llx, %llu) : poller->startIO(%d, 0x%x, 0x%llx, %llu, completion) returned 0x%x",
 371                             vars, operation, bufferOffset, deviceOffset, length, operation, bufferOffset, deviceOffset, length, err);
 372                 }
 373         }
 374         return err;
 375 }
 376
 377 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 378
 379 static IOReturn
 380 IOPolledFilePollersIODone(IOPolledFilePollers * vars, bool abortable)
 381 {
 382         IOReturn            err = kIOReturnSuccess;
 383         int32_t             idx = 0;
 384         IOPolledInterface * poller;
 385         AbsoluteTime        deadline;
 386
 387         if (!vars->io) {
 388                 return kIOReturnSuccess;
 389         }
 390
 391         abortable &= vars->abortable;
 392
 393         clock_interval_to_deadline(2000, kMillisecondScale, &deadline);
 394
 395         while (-1 == vars->ioStatus) {
 396                 for (idx = 0;
 397                     (poller = (IOPolledInterface *) vars->pollers->getObject(idx));
 398                     idx++) {
 399                         IOReturn newErr;
 400                         newErr = poller->checkForWork();
 401                         if ((newErr == kIOReturnAborted) && !abortable) {
 402                                 newErr = kIOReturnSuccess;
 403                         }
 404                         if (kIOReturnSuccess == err) {
 405                                 err = newErr;
 406                         }
 407                 }
 408                 if ((false) && (kIOReturnSuccess == err) && (mach_absolute_time() > AbsoluteTime_to_scalar(&deadline))) {
 409                         HIBLOG("IOPolledInterface::forced timeout\n");
 410                         vars->ioStatus = kIOReturnTimeout;
 411                 }
 412         }
 413         vars->io = false;
 414
 415 #if HIBERNATION
 416         if ((kIOReturnSuccess == err) && abortable && hibernate_should_abort()) {
 417                 err = kIOReturnAborted;
 418                 HIBLOG("IOPolledInterface::checkForWork sw abort\n");
 419         }
 420 #endif
 421
 422         if (err) {
 423                 HIBLOG("IOPolledInterface::checkForWork[%d] 0x%x\n", idx, err);
 424         } else {
 425                 err = vars->ioStatus;
 426                 if (kIOReturnSuccess != err) {
 427                         HIBLOG("IOPolledInterface::ioStatus 0x%x\n", err);
 428                 }
 429         }
 430
 431         return err;
 432 }
 433
 434 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 435 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 436
 437 struct _OpenFileContext {
 438         OSData * extents;
 439         uint64_t size;
 440 };
 441
 442 static void
 443 file_extent_callback(void * ref, uint64_t start, uint64_t length)
 444 {
 445         _OpenFileContext * ctx = (_OpenFileContext *) ref;
 446         IOPolledFileExtent extent;
 447
 448         extent.start  = start;
 449         extent.length = length;
 450         ctx->extents->appendBytes(&extent, sizeof(extent));
 451         ctx->size += length;
 452 }
 453
 454 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 455
 456 static IOService *
 457 IOCopyMediaForDev(dev_t device)
 458 {
 459         OSDictionary * matching;
 460         OSNumber *     num;
 461         OSIterator *   iter;
 462         IOService *    result = NULL;
 463
 464         matching = IOService::serviceMatching("IOMedia");
 465         if (!matching) {
 466                 return NULL;
 467         }
 468         do{
 469                 num = OSNumber::withNumber(major(device), 32);
 470                 if (!num) {
 471                         break;
 472                 }
 473                 matching->setObject(kIOBSDMajorKey, num);
 474                 num->release();
 475                 num = OSNumber::withNumber(minor(device), 32);
 476                 if (!num) {
 477                         break;
 478                 }
 479                 matching->setObject(kIOBSDMinorKey, num);
 480                 num->release();
 481                 if (!num) {
 482                         break;
 483                 }
 484                 iter = IOService::getMatchingServices(matching);
 485                 if (iter) {
 486                         result = (IOService *) iter->getNextObject();
 487                         result->retain();
 488                         iter->release();
 489                 }
 490         }while (false);
 491         matching->release();
 492
 493         return result;
 494 }
 495
 496 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 497
 498 #if defined(__i386__) || defined(__x86_64__)
 499 #define APFSMEDIA_GETHIBERKEY         "getHiberKey"
 500
 501 static IOReturn
 502 IOGetVolumeCryptKey(dev_t block_dev,
 503     LIBKERN_RETURNS_RETAINED OSString ** pKeyUUID,
 504     uint8_t * volumeCryptKey,
 505     size_t * keySize)
 506 {
 507         IOReturn         err;
 508         IOService *      part;
 509         OSString *       keyUUID = NULL;
 510         OSString *       keyStoreUUID = NULL;
 511         uuid_t           volumeKeyUUID;
 512         aks_volume_key_t vek;
 513         size_t           callerKeySize;
 514
 515         static IOService * sKeyStore;
 516
 517         part = IOCopyMediaForDev(block_dev);
 518         if (!part) {
 519                 return kIOReturnNotFound;
 520         }
 521
 522         callerKeySize = *keySize;
 523         // Try APFS first
 524         {
 525                 uuid_t volUuid = {0};
 526                 err = part->callPlatformFunction(APFSMEDIA_GETHIBERKEY, false, &volUuid, volumeCryptKey, keySize, keySize);
 527                 if (kIOReturnBadArgument == err) {
 528                         // apfs fails on buffer size >32
 529                         *keySize = 32;
 530                         err = part->callPlatformFunction(APFSMEDIA_GETHIBERKEY, false, &volUuid, volumeCryptKey, keySize, keySize);
 531                 }
 532                 if (err != kIOReturnSuccess) {
 533                         *keySize = 0;
 534                 } else {
 535                         // No need to create uuid string if it's not requested
 536                         if (pKeyUUID) {
 537                                 uuid_string_t volUuidStr;
 538                                 uuid_unparse(volUuid, volUuidStr);
 539                                 *pKeyUUID = OSString::withCString(volUuidStr);
 540                         }
 541
 542                         part->release();
 543                         return kIOReturnSuccess;
 544                 }
 545         }
 546
 547         // Then old CS path
 548         err = part->callPlatformFunction(PLATFORM_FUNCTION_GET_MEDIA_ENCRYPTION_KEY_UUID, false,
 549             (void *) &keyUUID, (void *) &keyStoreUUID, NULL, NULL);
 550         if ((kIOReturnSuccess == err) && keyUUID && keyStoreUUID) {
 551 //        IOLog("got volume key %s\n", keyStoreUUID->getCStringNoCopy());
 552
 553                 if (!sKeyStore) {
 554                         sKeyStore = (IOService *) IORegistryEntry::fromPath(AKS_SERVICE_PATH, gIOServicePlane);
 555                 }
 556                 if (sKeyStore) {
 557                         err = uuid_parse(keyStoreUUID->getCStringNoCopy(), volumeKeyUUID);
 558                 } else {
 559                         err = kIOReturnNoResources;
 560                 }
 561                 if (kIOReturnSuccess == err) {
 562                         err = sKeyStore->callPlatformFunction(gAKSGetKey, true, volumeKeyUUID, &vek, NULL, NULL);
 563                 }
 564                 if (kIOReturnSuccess != err) {
 565                         IOLog("volume key err 0x%x\n", err);
 566                 } else {
 567                         if (vek.key.keybytecount <= callerKeySize) {
 568                                 *keySize = vek.key.keybytecount;
 569                         }
 570                         bcopy(&vek.key.keybytes[0], volumeCryptKey, *keySize);
 571                 }
 572                 bzero(&vek, sizeof(vek));
 573
 574                 if (pKeyUUID) {
 575                         // Create a copy because the caller would release it
 576                         *pKeyUUID = OSString::withString(keyUUID);
 577                 }
 578         }
 579
 580         part->release();
 581         return err;
 582 }
 583 #endif /* defined(__i386__) || defined(__x86_64__) */
 584
 585 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 586
 587 #if defined(__arm64__)
 588 static IOReturn
 589 IOGetHibernationCryptKey(uint8_t * hibernationKey,
 590     size_t * keySize,
 591     uint32_t *swSeed
 592     )
 593 {
 594 #if XNU_MONITOR_PPL_HIB
 595         SEPHibernator *hibernator = SEPHibernator::sepHibernator();
 596         sephib_wrapped_key_t wrappedKey = {};
 597         sephib_seprom_hib_payload_t sepromPayload = {};
 598         hibernator->prepareToHibernate(&wrappedKey, &sepromPayload);
 599         *swSeed = sepromPayload.sw_seed;
 600         assert(*keySize >= sizeof(wrappedKey.data));
 601         *keySize = sizeof(wrappedKey.data);
 602         memcpy(hibernationKey, wrappedKey.data, *keySize);
 603         return kIOReturnSuccess;
 604 #else
 605         return kIOReturnNotFound;
 606 #endif
 607 }
 608 #endif /* defined(__arm64__) */
 609
 610 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 611
 612 IOReturn
 613 IOPolledFileOpen(const char * filename,
 614     uint32_t flags,
 615     uint64_t setFileSize, uint64_t fsFreeSize,
 616     void * write_file_addr, size_t write_file_len,
 617     IOPolledFileIOVars ** fileVars,
 618     OSData ** imagePath,
 619     uint8_t * volumeCryptKey, size_t * keySize)
 620 {
 621         IOReturn             err = kIOReturnSuccess;
 622         IOPolledFileIOVars * vars;
 623         _OpenFileContext     ctx;
 624         OSData *             extentsData = NULL;
 625         OSNumber *           num;
 626         IOService *          part = NULL;
 627         dev_t                block_dev;
 628         dev_t                image_dev;
 629         AbsoluteTime         startTime, endTime;
 630         uint64_t             nsec;
 631
 632         vars = IONew(IOPolledFileIOVars, 1);
 633         if (!vars) {
 634                 return kIOReturnNoMemory;
 635         }
 636         bzero(vars, sizeof(*vars));
 637         vars->allocated = true;
 638
 639         do{
 640                 extentsData = OSData::withCapacity(32);
 641                 ctx.extents = extentsData;
 642                 ctx.size    = 0;
 643                 clock_get_uptime(&startTime);
 644
 645                 vars->fileRef = kern_open_file_for_direct_io(filename,
 646                     flags,
 647                     &file_extent_callback, &ctx,
 648                     setFileSize,
 649                     fsFreeSize,
 650                     // write file:
 651                     0, write_file_addr, write_file_len,
 652                     // results
 653                     &block_dev,
 654                     &image_dev,
 655                     &vars->block0,
 656                     &vars->maxiobytes,
 657                     &vars->flags);
 658 #if 0
 659                 uint32_t msDelay = (131071 & random());
 660                 HIBLOG("sleep %d\n", msDelay);
 661                 IOSleep(msDelay);
 662 #endif
 663                 clock_get_uptime(&endTime);
 664                 SUB_ABSOLUTETIME(&endTime, &startTime);
 665                 absolutetime_to_nanoseconds(endTime, &nsec);
 666
 667                 if (!vars->fileRef) {
 668                         err = kIOReturnNoSpace;
 669                 }
 670
 671                 HIBLOG("kern_open_file_for_direct_io took %qd ms\n", nsec / 1000000ULL);
 672                 if (kIOReturnSuccess != err) {
 673                         break;
 674                 }
 675
 676                 HIBLOG("Opened file %s, size %qd, extents %ld, maxio %qx ssd %d\n", filename, ctx.size,
 677                     (extentsData->getLength() / sizeof(IOPolledFileExtent)) - 1,
 678                     vars->maxiobytes, kIOPolledFileSSD & vars->flags);
 679                 assert(!vars->block0);
 680                 if (extentsData->getLength() < sizeof(IOPolledFileExtent)) {
 681                         err = kIOReturnNoSpace;
 682                         break;
 683                 }
 684
 685                 vars->fileSize = ctx.size;
 686                 vars->extentMap = (IOPolledFileExtent *) extentsData->getBytesNoCopy();
 687
 688                 part = IOCopyMediaForDev(image_dev);
 689                 if (!part) {
 690                         err = kIOReturnNotFound;
 691                         break;
 692                 }
 693
 694                 if (!(vars->pollers = IOPolledFilePollers::copyPollers(part))) {
 695                         break;
 696                 }
 697
 698                 if ((num = OSDynamicCast(OSNumber, part->getProperty(kIOMediaPreferredBlockSizeKey)))) {
 699                         vars->blockSize = num->unsigned32BitValue();
 700                 }
 701                 if (vars->blockSize < 4096) {
 702                         vars->blockSize = 4096;
 703                 }
 704
 705                 HIBLOG("polled file major %d, minor %d, blocksize %ld, pollers %d\n",
 706                     major(image_dev), minor(image_dev), (long)vars->blockSize,
 707                     vars->pollers->pollers->getCount());
 708
 709                 OSString * keyUUID = NULL;
 710 #if defined(__i386__) || defined(__x86_64__)
 711                 if (volumeCryptKey) {
 712                         err = IOGetVolumeCryptKey(block_dev, &keyUUID, volumeCryptKey, keySize);
 713                 }
 714 #elif defined(__arm64__)
 715                 uint32_t swSeed = 0;
 716                 if (volumeCryptKey) {
 717                         if (flags & kIOPolledFileHibernate) {
 718                                 err = IOGetHibernationCryptKey(volumeCryptKey, keySize, &swSeed);
 719                                 if (kIOReturnSuccess != err) {
 720                                         HIBLOG("error 0x%x from IOGetHibernationCryptKey\n", err);
 721                                         break;
 722                                 }
 723                         } else {
 724                                 *keySize = 0;
 725                         }
 726                 }
 727 #else
 728                 if (volumeCryptKey) {
 729                         HIBLOG("IOPolledFileOpen: unable to get volumeCryptKey\n");
 730                         err = kIOReturnNotFound;
 731                         break;
 732                 }
 733 #endif
 734
 735                 *fileVars    = vars;
 736                 vars->fileExtents = extentsData;
 737
 738                 // make imagePath
 739                 OSData * data = NULL;
 740                 if (imagePath) {
 741 #if defined(__i386__) || defined(__x86_64__)
 742                         char str2[24 + sizeof(uuid_string_t) + 2];
 743
 744                         if (keyUUID) {
 745                                 snprintf(str2, sizeof(str2), "%qx:%s",
 746                                     vars->extentMap[0].start, keyUUID->getCStringNoCopy());
 747                         } else {
 748                                 snprintf(str2, sizeof(str2), "%qx", vars->extentMap[0].start);
 749                         }
 750
 751                         err = IOService::getPlatform()->callPlatformFunction(
 752                                 gIOCreateEFIDevicePathSymbol, false,
 753                                 (void *) part, (void *) str2,
 754                                 (void *) (uintptr_t) true, (void *) &data);
 755 #elif defined(__arm64__)
 756                         char str2[26];
 757                         snprintf(str2, sizeof(str2), "%qx:%x", vars->extentMap[0].start, swSeed);
 758                         data = OSData::withBytes(str2, (unsigned int) strlen(str2));
 759                         err = kIOReturnSuccess;
 760 #else
 761                         err = kIOReturnNotFound;
 762 #endif
 763                         if (kIOReturnSuccess != err) {
 764                                 HIBLOG("error 0x%x getting path\n", err);
 765                                 break;
 766                         }
 767                         *imagePath = data;
 768                 }
 769
 770                 // Release key UUID if we have one
 771                 if (keyUUID) {
 772                         keyUUID->release();
 773                         keyUUID = NULL; // Just in case
 774                 }
 775         }while (false);
 776
 777         if (kIOReturnSuccess != err) {
 778                 HIBLOG("error 0x%x opening polled file\n", err);
 779                 IOPolledFileClose(&vars, 0, NULL, 0, 0, 0);
 780                 if (extentsData) {
 781                         extentsData->release();
 782                 }
 783         }
 784
 785         if (part) {
 786                 part->release();
 787         }
 788
 789         return err;
 790 }
 791
 792 IOReturn
 793 IOPolledFileOpen(const char * filename,
 794     uint32_t flags,
 795     uint64_t setFileSize, uint64_t fsFreeSize,
 796     void * write_file_addr, size_t write_file_len,
 797     IOPolledFileIOVars ** fileVars,
 798     OSSharedPtr<OSData>& imagePath,
 799     uint8_t * volumeCryptKey, size_t * keySize)
 800 {
 801         OSData* imagePathRaw = NULL;
 802         IOReturn result = IOPolledFileOpen(filename, flags, setFileSize, fsFreeSize, write_file_addr, write_file_len,
 803             fileVars, &imagePathRaw, volumeCryptKey, keySize);
 804         imagePath.reset(imagePathRaw, OSNoRetain);
 805         return result;
 806 }
 807
 808 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 809
 810 IOReturn
 811 IOPolledFileClose(IOPolledFileIOVars ** pVars,
 812     off_t write_offset, void * addr, size_t write_length,
 813     off_t discard_offset, off_t discard_end)
 814 {
 815         IOPolledFileIOVars * vars;
 816
 817         vars = *pVars;
 818         if (!vars) {
 819                 return kIOReturnSuccess;
 820         }
 821
 822         if (vars->fileRef) {
 823                 kern_close_file_for_direct_io(vars->fileRef, write_offset, addr, write_length,
 824                     discard_offset, discard_end);
 825                 vars->fileRef = NULL;
 826         }
 827         if (vars->fileExtents) {
 828                 vars->fileExtents->release();
 829                 vars->fileExtents = NULL;
 830         }
 831         if (vars->pollers) {
 832                 vars->pollers->release();
 833                 vars->pollers = NULL;
 834         }
 835
 836         if (vars->allocated) {
 837                 IODelete(vars, IOPolledFileIOVars, 1);
 838         } else {
 839                 bzero(vars, sizeof(IOPolledFileIOVars));
 840         }
 841         *pVars = NULL;
 842
 843         return kIOReturnSuccess;
 844 }
 845
 846 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 847
 848 IOReturn
 849 IOPolledFilePollersSetup(IOPolledFileIOVars * vars,
 850     uint32_t openState)
 851 {
 852         IOReturn err;
 853
 854         err = kIOReturnSuccess;
 855         do{
 856                 if (!vars->pollers->openCount) {
 857                         err = IOPolledFilePollersProbe(vars->pollers);
 858                         if (kIOReturnSuccess != err) {
 859                                 break;
 860                         }
 861                 }
 862                 err = IOPolledFilePollersOpen(vars, openState, false);
 863                 if (kIOReturnSuccess != err) {
 864                         break;
 865                 }
 866                 if ((kIOPolledPreflightState == openState) || (kIOPolledPreflightCoreDumpState == openState)) {
 867                         vars->pollers->openCount++;
 868                 }
 869                 vars->pollers->io  = false;
 870                 vars->buffer       = (uint8_t *) vars->pollers->ioBuffer->getBytesNoCopy();
 871                 vars->bufferHalf   = 0;
 872                 vars->bufferOffset = 0;
 873                 assert(vars->pollers->ioBuffer->getLength() <= UINT_MAX);
 874                 vars->bufferSize   = (typeof(vars->bufferSize))(vars->pollers->ioBuffer->getLength() >> 1);
 875
 876                 if (vars->maxiobytes < vars->bufferSize) {
 877                         vars->bufferSize = (typeof(vars->bufferSize))vars->maxiobytes;
 878                 }
 879         }while (false);
 880
 881         if (kIOReturnSuccess != err) {
 882                 HIBLOG("IOPolledFilePollersSetup(%d) error 0x%x\n", openState, err);
 883         }
 884
 885         return err;
 886 }
 887
 888
 889 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 890
 891 IOReturn
 892 IOPolledFileSeek(IOPolledFileIOVars * vars, uint64_t position)
 893 {
 894         IOPolledFileExtent * extentMap;
 895
 896         extentMap = vars->extentMap;
 897
 898         vars->position = position;
 899
 900         if (position > vars->fileSize) {
 901                 HIBLOG("IOPolledFileSeek: called to seek to 0x%llx greater than file size of 0x%llx\n", vars->position, vars->fileSize);
 902                 return kIOReturnNoSpace;
 903         }
 904
 905         while (position >= extentMap->length) {
 906                 position -= extentMap->length;
 907                 extentMap++;
 908         }
 909
 910         vars->currentExtent   = extentMap;
 911         vars->extentRemaining = extentMap->length - position;
 912         vars->extentPosition  = vars->position - position;
 913
 914         if (vars->bufferSize <= vars->extentRemaining) {
 915                 vars->bufferLimit = vars->bufferSize;
 916         } else {
 917                 vars->bufferLimit = ((uint32_t) vars->extentRemaining);
 918         }
 919
 920         return kIOReturnSuccess;
 921 }
 922
 923 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 924
 925 IOReturn
 926 IOPolledFileWrite(IOPolledFileIOVars * vars,
 927     const uint8_t * bytes, IOByteCount size,
 928     IOPolledFileCryptVars * cryptvars)
 929 {
 930         IOReturn    err = kIOReturnSuccess;
 931         IOByteCount copy, original_size = size;
 932         bool        flush = false;
 933
 934         do{
 935                 if (!bytes && !size) {
 936                         // seek to end of block & flush
 937                         size = vars->position & (vars->blockSize - 1);
 938                         if (size) {
 939                                 size = vars->blockSize - size;
 940                         }
 941                         flush = true;
 942                 }
 943
 944                 copy = vars->bufferLimit - vars->bufferOffset;
 945                 if (copy > size) {
 946                         copy = size;
 947                 } else {
 948                         flush = true;
 949                 }
 950
 951                 if (bytes) {
 952 #if KASAN
 953                         /* Since this may copy mach-o segments in bulk, use the nosan variants of bcopy to
 954                          * avoid triggering global redzone sanitizer violations when accessing
 955                          * interstices between 'C' structures
 956                          */
 957                         __nosan_bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
 958 #else
 959                         bcopy(bytes, vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
 960 #endif
 961                         bytes += copy;
 962                 } else {
 963                         bzero(vars->buffer + vars->bufferHalf + vars->bufferOffset, copy);
 964                 }
 965
 966                 size -= copy;
 967                 vars->bufferOffset += copy;
 968                 vars->position += copy;
 969
 970                 if (flush && vars->bufferOffset) {
 971                         uint64_t offset = (vars->position - vars->bufferOffset
 972                             - vars->extentPosition + vars->currentExtent->start);
 973                         uint32_t length = (vars->bufferOffset);
 974
 975 #if CRYPTO
 976                         if (cryptvars && vars->encryptStart
 977                             && (vars->position > vars->encryptStart)
 978                             && ((vars->position - length) < vars->encryptEnd)) {
 979                                 AbsoluteTime startTime, endTime;
 980
 981                                 uint64_t encryptLen, encryptStart;
 982                                 encryptLen = vars->position - vars->encryptStart;
 983                                 if (encryptLen > length) {
 984                                         encryptLen = length;
 985                                 }
 986                                 encryptStart = length - encryptLen;
 987                                 if (vars->position > vars->encryptEnd) {
 988                                         encryptLen -= (vars->position - vars->encryptEnd);
 989                                 }
 990
 991                                 clock_get_uptime(&startTime);
 992
 993                                 assert(encryptLen <= UINT_MAX);
 994                                 // encrypt the buffer
 995                                 aes_encrypt_cbc(vars->buffer + vars->bufferHalf + encryptStart,
 996                                     &cryptvars->aes_iv[0],
 997                                     (unsigned int) (encryptLen / AES_BLOCK_SIZE),
 998                                     vars->buffer + vars->bufferHalf + encryptStart,
 999                                     &cryptvars->ctx.encrypt);
1000
1001                                 clock_get_uptime(&endTime);
1002                                 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
1003                                 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
1004                                 vars->cryptBytes += encryptLen;
1005
1006                                 // save initial vector for following encrypts
1007                                 bcopy(vars->buffer + vars->bufferHalf + encryptStart + encryptLen - AES_BLOCK_SIZE,
1008                                     &cryptvars->aes_iv[0],
1009                                     AES_BLOCK_SIZE);
1010                         }
1011 #endif /* CRYPTO */
1012
1013                         err = IOPolledFilePollersIODone(vars->pollers, true);
1014                         if (kIOReturnSuccess != err) {
1015                                 break;
1016                         }
1017
1018                         if (vars->position & (vars->blockSize - 1)) {
1019                                 HIBLOG("misaligned file pos %qx\n", vars->position);
1020                         }
1021 //if (length != vars->bufferSize) HIBLOG("short write of %qx ends@ %qx\n", length, offset + length);
1022
1023                         err = IOStartPolledIO(vars->pollers, kIOPolledWrite, vars->bufferHalf, offset, length);
1024                         if (kIOReturnSuccess != err) {
1025                                 HIBLOGFROMPANIC("IOPolledFileWrite(0x%p, 0x%p, %llu, 0x%p) : IOStartPolledIO(0x%p, kIOPolledWrite, %llu, 0x%llx, %d) returned 0x%x\n",
1026                                     vars, bytes, (uint64_t) original_size, cryptvars, vars->pollers, (uint64_t) vars->bufferHalf, offset, length, err);
1027                                 break;
1028                         }
1029                         vars->pollers->io = true;
1030
1031                         vars->extentRemaining -= vars->bufferOffset;
1032                         if (!vars->extentRemaining) {
1033                                 vars->currentExtent++;
1034                                 vars->extentRemaining = vars->currentExtent->length;
1035                                 vars->extentPosition  = vars->position;
1036                         }
1037
1038                         vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
1039                         vars->bufferOffset = 0;
1040                         if (vars->bufferSize <= vars->extentRemaining) {
1041                                 vars->bufferLimit = vars->bufferSize;
1042                         } else {
1043                                 vars->bufferLimit =  ((uint32_t) vars->extentRemaining);
1044                         }
1045
1046                         if (!vars->extentRemaining) {
1047                                 err = kIOReturnOverrun;
1048                                 break;
1049                         }
1050
1051                         flush = false;
1052                 }
1053         }while (size);
1054
1055         return err;
1056 }
1057
1058 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1059
1060 IOReturn
1061 IOPolledFileFlush(IOPolledFileIOVars * vars)
1062 {
1063         // Only supported by the underlying polled mode driver on embedded currently (expect kIOReturnUnsupported on other platforms)
1064         IOReturn err = kIOReturnSuccess;
1065
1066         err = IOPolledFilePollersIODone(vars->pollers, true);
1067         if (kIOReturnSuccess != err) {
1068                 return err;
1069         }
1070
1071         err = IOStartPolledIO(vars->pollers, kIOPolledFlush, 0, 0, 0);
1072         if (kIOReturnSuccess != err) {
1073                 HIBLOGFROMPANIC("IOPolledFileFlush(0x%p) : IOStartPolledIO(0x%p, kIOPolledFlush, 0, 0, 0) returned 0x%x\n",
1074                     vars, vars->pollers, err);
1075                 return err;
1076         }
1077         vars->pollers->io = true;
1078
1079         return err;
1080 }
1081
1082 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1083
1084 IOReturn
1085 IOPolledFileRead(IOPolledFileIOVars * vars,
1086     uint8_t * bytes, IOByteCount size,
1087     IOPolledFileCryptVars * cryptvars)
1088 {
1089         IOReturn    err = kIOReturnSuccess;
1090         IOByteCount copy;
1091
1092 //    bytesWritten += size;
1093
1094         do{
1095                 copy = vars->bufferLimit - vars->bufferOffset;
1096                 if (copy > size) {
1097                         copy = size;
1098                 }
1099
1100                 if (bytes) {
1101 #if KASAN
1102                         __nosan_bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
1103 #else
1104                         bcopy(vars->buffer + vars->bufferHalf + vars->bufferOffset, bytes, copy);
1105 #endif
1106                         bytes += copy;
1107                 }
1108                 size -= copy;
1109                 vars->bufferOffset += copy;
1110 //      vars->position += copy;
1111
1112                 if ((vars->bufferOffset == vars->bufferLimit) && (vars->position < vars->readEnd)) {
1113                         if (!vars->pollers->io) {
1114                                 cryptvars = NULL;
1115                         }
1116                         err = IOPolledFilePollersIODone(vars->pollers, true);
1117                         if (kIOReturnSuccess != err) {
1118                                 break;
1119                         }
1120
1121                         if (vars->position & (vars->blockSize - 1)) {
1122                                 HIBLOG("misaligned file pos %qx\n", vars->position);
1123                         }
1124
1125                         vars->position        += vars->lastRead;
1126                         vars->extentRemaining -= vars->lastRead;
1127                         vars->bufferLimit      = vars->lastRead;
1128
1129                         if (!vars->extentRemaining) {
1130                                 vars->currentExtent++;
1131                                 vars->extentRemaining = vars->currentExtent->length;
1132                                 vars->extentPosition  = vars->position;
1133                                 if (!vars->extentRemaining) {
1134                                         err = kIOReturnOverrun;
1135                                         break;
1136                                 }
1137                         }
1138
1139                         uint32_t length;
1140                         uint32_t lastReadLength = vars->lastRead;
1141                         uint64_t offset = (vars->position
1142                             - vars->extentPosition + vars->currentExtent->start);
1143                         if (vars->extentRemaining <= vars->bufferSize) {
1144                                 length =  ((uint32_t) vars->extentRemaining);
1145                         } else {
1146                                 length = vars->bufferSize;
1147                         }
1148                         if ((length + vars->position) > vars->readEnd) {
1149                                 length = ((uint32_t) (vars->readEnd - vars->position));
1150                         }
1151
1152                         vars->lastRead = length;
1153                         if (length) {
1154 //if (length != vars->bufferSize) HIBLOG("short read of %qx ends@ %qx\n", length, offset + length);
1155                                 err = IOStartPolledIO(vars->pollers, kIOPolledRead, vars->bufferHalf, offset, length);
1156                                 if (kIOReturnSuccess != err) {
1157                                         break;
1158                                 }
1159                                 vars->pollers->io = true;
1160                         }
1161
1162                         vars->bufferHalf = vars->bufferHalf ? 0 : vars->bufferSize;
1163                         vars->bufferOffset = 0;
1164
1165 #if CRYPTO
1166                         if (cryptvars) {
1167                                 uint8_t thisVector[AES_BLOCK_SIZE];
1168                                 AbsoluteTime startTime, endTime;
1169
1170                                 // save initial vector for following decrypts
1171                                 bcopy(&cryptvars->aes_iv[0], &thisVector[0], AES_BLOCK_SIZE);
1172                                 bcopy(vars->buffer + vars->bufferHalf + lastReadLength - AES_BLOCK_SIZE,
1173                                     &cryptvars->aes_iv[0], AES_BLOCK_SIZE);
1174
1175                                 // decrypt the buffer
1176                                 clock_get_uptime(&startTime);
1177
1178                                 assert(lastReadLength <= UINT_MAX);
1179                                 aes_decrypt_cbc(vars->buffer + vars->bufferHalf,
1180                                     &thisVector[0],
1181                                     (unsigned int) (lastReadLength / AES_BLOCK_SIZE),
1182                                     vars->buffer + vars->bufferHalf,
1183                                     &cryptvars->ctx.decrypt);
1184
1185                                 clock_get_uptime(&endTime);
1186                                 ADD_ABSOLUTETIME(&vars->cryptTime, &endTime);
1187                                 SUB_ABSOLUTETIME(&vars->cryptTime, &startTime);
1188                                 vars->cryptBytes += lastReadLength;
1189                         }
1190 #endif /* CRYPTO */
1191                 }
1192         }while (size);
1193
1194         return err;
1195 }
1196
1197 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */