iokit/Kernel/IODMACommand.cpp

   1 /*
   2  * Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <IOKit/assert.h>
  30
  31 #include <libkern/OSTypes.h>
  32 #include <libkern/OSByteOrder.h>
  33
  34 #include <IOKit/IOReturn.h>
  35 #include <IOKit/IOLib.h>
  36 #include <IOKit/IODMACommand.h>
  37 #include <IOKit/IOMapper.h>
  38 #include <IOKit/IOMemoryDescriptor.h>
  39 #include <IOKit/IOBufferMemoryDescriptor.h>
  40
  41 #include "IOKitKernelInternal.h"
  42
  43 #define MAPTYPE(type)           ((UInt) (type) & kTypeMask)
  44 #define IS_MAPPED(type)         (MAPTYPE(type) == kMapped)
  45 #define IS_BYPASSED(type)       (MAPTYPE(type) == kBypassed)
  46 #define IS_NONCOHERENT(type)    (MAPTYPE(type) == kNonCoherent)
  47
  48 enum
  49 {
  50     kWalkSyncIn       = 0x01,   // bounce -> md
  51     kWalkSyncOut      = 0x02,   // bounce <- md
  52     kWalkSyncAlways   = 0x04,
  53     kWalkPreflight    = 0x08,
  54     kWalkDoubleBuffer = 0x10,
  55     kWalkPrepare      = 0x20,
  56     kWalkComplete     = 0x40,
  57     kWalkClient       = 0x80
  58 };
  59
  60
  61 #define fInternalState reserved
  62 #define fState         reserved->fState
  63 #define fMDSummary     reserved->fMDSummary
  64
  65
  66 #if 1
  67 // no direction => OutIn
  68 #define SHOULD_COPY_DIR(op, direction)                                      \
  69         ((kIODirectionNone == (direction))                                  \
  70             || (kWalkSyncAlways & (op))                                     \
  71             || (((kWalkSyncIn & (op)) ? kIODirectionIn : kIODirectionOut)   \
  72                                                     & (direction)))
  73
  74 #else
  75 #define SHOULD_COPY_DIR(state, direction) (true)
  76 #endif
  77
  78 #if 0
  79 #define DEBG(fmt, args...)      { IOLog(fmt, ## args); kprintf(fmt, ## args); }
  80 #else
  81 #define DEBG(fmt, args...)      {}
  82 #endif
  83
  84 /**************************** class IODMACommand ***************************/
  85
  86 #undef super
  87 #define super IOCommand
  88 OSDefineMetaClassAndStructors(IODMACommand, IOCommand);
  89
  90 OSMetaClassDefineReservedUsed(IODMACommand,  0);
  91 OSMetaClassDefineReservedUsed(IODMACommand,  1);
  92 OSMetaClassDefineReservedUsed(IODMACommand,  2);
  93 OSMetaClassDefineReservedUnused(IODMACommand,  3);
  94 OSMetaClassDefineReservedUnused(IODMACommand,  4);
  95 OSMetaClassDefineReservedUnused(IODMACommand,  5);
  96 OSMetaClassDefineReservedUnused(IODMACommand,  6);
  97 OSMetaClassDefineReservedUnused(IODMACommand,  7);
  98 OSMetaClassDefineReservedUnused(IODMACommand,  8);
  99 OSMetaClassDefineReservedUnused(IODMACommand,  9);
 100 OSMetaClassDefineReservedUnused(IODMACommand, 10);
 101 OSMetaClassDefineReservedUnused(IODMACommand, 11);
 102 OSMetaClassDefineReservedUnused(IODMACommand, 12);
 103 OSMetaClassDefineReservedUnused(IODMACommand, 13);
 104 OSMetaClassDefineReservedUnused(IODMACommand, 14);
 105 OSMetaClassDefineReservedUnused(IODMACommand, 15);
 106
 107 IODMACommand *
 108 IODMACommand::withSpecification(SegmentFunction outSegFunc,
 109                                 UInt8           numAddressBits,
 110                                 UInt64          maxSegmentSize,
 111                                 MappingOptions  mappingOptions,
 112                                 UInt64          maxTransferSize,
 113                                 UInt32          alignment,
 114                                 IOMapper       *mapper,
 115                                 void           *refCon)
 116 {
 117     IODMACommand * me = new IODMACommand;
 118
 119     if (me && !me->initWithSpecification(outSegFunc,
 120                                          numAddressBits, maxSegmentSize,
 121                                          mappingOptions, maxTransferSize,
 122                                          alignment,      mapper, refCon))
 123     {
 124         me->release();
 125         return 0;
 126     };
 127
 128     return me;
 129 }
 130
 131 IODMACommand *
 132 IODMACommand::cloneCommand(void *refCon)
 133 {
 134     return withSpecification(fOutSeg, fNumAddressBits, fMaxSegmentSize,
 135             fMappingOptions, fMaxTransferSize, fAlignMask + 1, fMapper, refCon);
 136 }
 137
 138 #define kLastOutputFunction ((SegmentFunction) kLastOutputFunction)
 139
 140 bool
 141 IODMACommand::initWithSpecification(SegmentFunction outSegFunc,
 142                                     UInt8           numAddressBits,
 143                                     UInt64          maxSegmentSize,
 144                                     MappingOptions  mappingOptions,
 145                                     UInt64          maxTransferSize,
 146                                     UInt32          alignment,
 147                                     IOMapper       *mapper,
 148                                     void           *refCon)
 149 {
 150     if (!super::init() || !outSegFunc)
 151         return false;
 152
 153     bool is32Bit = (OutputHost32   == outSegFunc || OutputBig32 == outSegFunc
 154                  || OutputLittle32 == outSegFunc);
 155     if (is32Bit)
 156     {
 157         if (!numAddressBits)
 158             numAddressBits = 32;
 159         else if (numAddressBits > 32)
 160             return false;               // Wrong output function for bits
 161     }
 162
 163     if (numAddressBits && (numAddressBits < PAGE_SHIFT))
 164         return false;
 165
 166     if (!maxSegmentSize)
 167         maxSegmentSize--;       // Set Max segment to -1
 168     if (!maxTransferSize)
 169         maxTransferSize--;      // Set Max transfer to -1
 170
 171     if (!mapper)
 172     {
 173         IOMapper::checkForSystemMapper();
 174         mapper = IOMapper::gSystem;
 175     }
 176
 177     fNumSegments     = 0;
 178     fBypassMask      = 0;
 179     fOutSeg          = outSegFunc;
 180     fNumAddressBits  = numAddressBits;
 181     fMaxSegmentSize  = maxSegmentSize;
 182     fMappingOptions  = mappingOptions;
 183     fMaxTransferSize = maxTransferSize;
 184     if (!alignment)
 185         alignment = 1;
 186     fAlignMask       = alignment - 1;
 187     fMapper          = mapper;
 188     fRefCon          = refCon;
 189
 190     switch (MAPTYPE(mappingOptions))
 191     {
 192     case kMapped:                   break;
 193     case kNonCoherent: fMapper = 0; break;
 194     case kBypassed:
 195         if (mapper && !mapper->getBypassMask(&fBypassMask))
 196             return false;
 197         break;
 198     default:
 199         return false;
 200     };
 201
 202     if (fMapper)
 203         fMapper->retain();
 204
 205     reserved = IONew(IODMACommandInternal, 1);
 206     if (!reserved)
 207         return false;
 208     bzero(reserved, sizeof(IODMACommandInternal));
 209
 210     fInternalState->fIterateOnly = (0 != (kIterateOnly & mappingOptions));
 211
 212     return true;
 213 }
 214
 215 void
 216 IODMACommand::free()
 217 {
 218     if (reserved)
 219         IODelete(reserved, IODMACommandInternal, 1);
 220
 221     if (fMapper)
 222         fMapper->release();
 223
 224     super::free();
 225 }
 226
 227 IOReturn
 228 IODMACommand::setMemoryDescriptor(const IOMemoryDescriptor *mem, bool autoPrepare)
 229 {
 230         IOReturn err = kIOReturnSuccess;
 231
 232     if (mem == fMemory)
 233     {
 234         if (!autoPrepare)
 235         {
 236             while (fActive)
 237                 complete();
 238         }
 239         return kIOReturnSuccess;
 240     }
 241
 242     if (fMemory) {
 243         // As we are almost certainly being called from a work loop thread
 244         // if fActive is true it is probably not a good time to potentially
 245         // block.  Just test for it and return an error
 246         if (fActive)
 247             return kIOReturnBusy;
 248         clearMemoryDescriptor();
 249     }
 250
 251     if (mem) {
 252         bzero(&fMDSummary, sizeof(fMDSummary));
 253         err = mem->dmaCommandOperation(
 254                 kIOMDGetCharacteristics,
 255                 &fMDSummary, sizeof(fMDSummary));
 256         if (err)
 257             return err;
 258
 259         ppnum_t highPage = fMDSummary.fHighestPage ? fMDSummary.fHighestPage : gIOLastPage;
 260
 261         if ((kMapped == MAPTYPE(fMappingOptions))
 262             && fMapper
 263             && (!fNumAddressBits || (fNumAddressBits >= 31)))
 264             // assuming mapped space is 2G
 265             fInternalState->fCheckAddressing = false;
 266         else
 267             fInternalState->fCheckAddressing = (fNumAddressBits && (highPage >= (1UL << (fNumAddressBits - PAGE_SHIFT))));
 268
 269         fInternalState->fNewMD = true;
 270         mem->retain();
 271         fMemory = mem;
 272
 273         mem->dmaCommandOperation(kIOMDSetDMAActive, this, 0);
 274         if (autoPrepare) {
 275                 err = prepare();
 276                 if (err) {
 277                         clearMemoryDescriptor();
 278                 }
 279         }
 280     }
 281
 282     return err;
 283 }
 284
 285 IOReturn
 286 IODMACommand::clearMemoryDescriptor(bool autoComplete)
 287 {
 288     if (fActive && !autoComplete)
 289         return (kIOReturnNotReady);
 290
 291     if (fMemory) {
 292         while (fActive)
 293             complete();
 294         fMemory->dmaCommandOperation(kIOMDSetDMAInactive, this, 0);
 295         fMemory->release();
 296         fMemory = 0;
 297     }
 298
 299     return (kIOReturnSuccess);
 300 }
 301
 302 const IOMemoryDescriptor *
 303 IODMACommand::getMemoryDescriptor() const
 304 {
 305     return fMemory;
 306 }
 307
 308
 309 IOReturn
 310 IODMACommand::segmentOp(
 311                         void         *reference,
 312                         IODMACommand *target,
 313                         Segment64     segment,
 314                         void         *segments,
 315                         UInt32        segmentIndex)
 316 {
 317     IOOptionBits op = (uintptr_t) reference;
 318     addr64_t     maxPhys, address;
 319     uint64_t     length;
 320     uint32_t     numPages;
 321
 322     IODMACommandInternal * state = target->reserved;
 323
 324     if (target->fNumAddressBits && (target->fNumAddressBits < 64) && !state->fLocalMapper)
 325         maxPhys = (1ULL << target->fNumAddressBits);
 326     else
 327         maxPhys = 0;
 328     maxPhys--;
 329
 330     address = segment.fIOVMAddr;
 331     length = segment.fLength;
 332
 333     assert(address);
 334     assert(length);
 335
 336     if (!state->fMisaligned)
 337     {
 338         state->fMisaligned |= (0 != (state->fSourceAlignMask & address));
 339         if (state->fMisaligned) DEBG("misaligned %qx:%qx, %lx\n", address, length, state->fSourceAlignMask);
 340     }
 341
 342     if (state->fMisaligned && (kWalkPreflight & op))
 343         return (kIOReturnNotAligned);
 344
 345     if (!state->fDoubleBuffer)
 346     {
 347         if ((address + length - 1) <= maxPhys)
 348         {
 349             length = 0;
 350         }
 351         else if (address <= maxPhys)
 352         {
 353             DEBG("tail %qx, %qx", address, length);
 354             length = (address + length - maxPhys - 1);
 355             address = maxPhys + 1;
 356             DEBG("-> %qx, %qx\n", address, length);
 357         }
 358     }
 359
 360     if (!length)
 361         return (kIOReturnSuccess);
 362
 363     numPages = atop_64(round_page_64((address & PAGE_MASK) + length));
 364
 365     if (kWalkPreflight & op)
 366     {
 367         state->fCopyPageCount += numPages;
 368     }
 369     else
 370     {
 371         vm_page_t lastPage;
 372         lastPage = NULL;
 373         if (kWalkPrepare & op)
 374         {
 375             lastPage = state->fCopyNext;
 376             for (IOItemCount idx = 0; idx < numPages; idx++)
 377             {
 378                 vm_page_set_offset(lastPage, atop_64(address) + idx);
 379                 lastPage = vm_page_get_next(lastPage);
 380             }
 381         }
 382
 383         if (!lastPage || SHOULD_COPY_DIR(op, target->fMDSummary.fDirection))
 384         {
 385             lastPage = state->fCopyNext;
 386             for (IOItemCount idx = 0; idx < numPages; idx++)
 387             {
 388                 if (SHOULD_COPY_DIR(op, target->fMDSummary.fDirection))
 389                 {
 390                     addr64_t remapAddr;
 391                     uint64_t chunk;
 392
 393                     remapAddr = ptoa_64(vm_page_get_phys_page(lastPage));
 394                     if (!state->fDoubleBuffer)
 395                     {
 396                         remapAddr += (address & PAGE_MASK);
 397                     }
 398                     chunk = PAGE_SIZE - (address & PAGE_MASK);
 399                     if (chunk > length)
 400                         chunk = length;
 401
 402                     DEBG("cpv: 0x%qx %s 0x%qx, 0x%qx, 0x%02lx\n", remapAddr,
 403                                 (kWalkSyncIn & op) ? "->" : "<-",
 404                                 address, chunk, op);
 405
 406                     if (kWalkSyncIn & op)
 407                     { // cppvNoModSnk
 408                         copypv(remapAddr, address, chunk,
 409                                         cppvPsnk | cppvFsnk | cppvPsrc | cppvNoRefSrc );
 410                     }
 411                     else
 412                     {
 413                         copypv(address, remapAddr, chunk,
 414                                         cppvPsnk | cppvFsnk | cppvPsrc | cppvNoRefSrc );
 415                     }
 416                     address += chunk;
 417                     length -= chunk;
 418                 }
 419                 lastPage = vm_page_get_next(lastPage);
 420             }
 421         }
 422         state->fCopyNext = lastPage;
 423     }
 424
 425     return kIOReturnSuccess;
 426 }
 427
 428 IOReturn
 429 IODMACommand::walkAll(UInt8 op)
 430 {
 431     IODMACommandInternal * state = fInternalState;
 432
 433     IOReturn     ret = kIOReturnSuccess;
 434     UInt32       numSegments;
 435     UInt64       offset;
 436
 437     if (kWalkPreflight & op)
 438     {
 439         state->fMapContig      = false;
 440         state->fMisaligned     = false;
 441         state->fDoubleBuffer   = false;
 442         state->fPrepared       = false;
 443         state->fCopyNext       = NULL;
 444         state->fCopyPageAlloc  = 0;
 445         state->fLocalMapperPageAlloc = 0;
 446         state->fCopyPageCount  = 0;
 447         state->fNextRemapPage  = NULL;
 448         state->fCopyMD         = 0;
 449
 450         if (!(kWalkDoubleBuffer & op))
 451         {
 452             offset = 0;
 453             numSegments = 0-1;
 454             ret = genIOVMSegments(op, segmentOp, (void *) op, &offset, state, &numSegments);
 455         }
 456
 457         op &= ~kWalkPreflight;
 458
 459         state->fDoubleBuffer = (state->fMisaligned || (kWalkDoubleBuffer & op));
 460         if (state->fDoubleBuffer)
 461             state->fCopyPageCount = atop_64(round_page(state->fPreparedLength));
 462
 463         if (state->fCopyPageCount)
 464         {
 465             vm_page_t mapBase = NULL;
 466
 467             DEBG("preflight fCopyPageCount %d\n", state->fCopyPageCount);
 468
 469             if (!state->fDoubleBuffer)
 470             {
 471                 kern_return_t kr;
 472                 kr = vm_page_alloc_list(state->fCopyPageCount,
 473                                         KMA_LOMEM | KMA_NOPAGEWAIT, &mapBase);
 474                 if (KERN_SUCCESS != kr)
 475                 {
 476                     DEBG("vm_page_alloc_list(%d) failed (%d)\n", state->fCopyPageCount, kr);
 477                     mapBase = NULL;
 478                 }
 479             }
 480
 481             if (mapBase)
 482             {
 483                 state->fCopyPageAlloc = mapBase;
 484                 state->fCopyNext = state->fCopyPageAlloc;
 485                 offset = 0;
 486                 numSegments = 0-1;
 487                 ret = genIOVMSegments(op, segmentOp, (void *) op, &offset, state, &numSegments);
 488                 state->fPrepared = true;
 489                 op &= ~(kWalkSyncIn | kWalkSyncOut);
 490             }
 491             else
 492             {
 493                 DEBG("alloc IOBMD\n");
 494                 mach_vm_address_t mask = 0xFFFFF000; //state->fSourceAlignMask
 495                 state->fCopyMD = IOBufferMemoryDescriptor::inTaskWithPhysicalMask(kernel_task,
 496                                     fMDSummary.fDirection, state->fPreparedLength, mask);
 497
 498                 if (state->fCopyMD)
 499                 {
 500                     ret = kIOReturnSuccess;
 501                     state->fPrepared = true;
 502                 }
 503                 else
 504                 {
 505                     DEBG("IODMACommand !alloc IOBMD");
 506                     return (kIOReturnNoResources);
 507                 }
 508             }
 509         }
 510
 511         if (state->fLocalMapper)
 512         {
 513             state->fLocalMapperPageCount = atop_64(round_page(
 514                     state->fPreparedLength + ((state->fPreparedOffset + fMDSummary.fPageAlign) & page_mask)));
 515             state->fLocalMapperPageAlloc = fMapper->iovmAllocDMACommand(this, state->fLocalMapperPageCount);
 516             if (!state->fLocalMapperPageAlloc)
 517             {
 518                 DEBG("IODMACommand !iovmAlloc");
 519                 return (kIOReturnNoResources);
 520             }
 521             state->fMapContig = true;
 522         }
 523     }
 524
 525     if (state->fPrepared && ((kWalkSyncIn | kWalkSyncOut) & op))
 526     {
 527         if (state->fCopyPageCount)
 528         {
 529             DEBG("sync fCopyPageCount %d\n", state->fCopyPageCount);
 530
 531             if (state->fCopyPageAlloc)
 532             {
 533                 state->fCopyNext = state->fCopyPageAlloc;
 534                 offset = 0;
 535                 numSegments = 0-1;
 536                 ret = genIOVMSegments(op, segmentOp, (void *) op, &offset, state, &numSegments);
 537             }
 538             else if (state->fCopyMD)
 539             {
 540                 DEBG("sync IOBMD\n");
 541
 542                 if (SHOULD_COPY_DIR(op, fMDSummary.fDirection))
 543                 {
 544                     IOMemoryDescriptor *poMD = const_cast<IOMemoryDescriptor *>(fMemory);
 545
 546                     IOByteCount bytes;
 547
 548                     if (kWalkSyncIn & op)
 549                         bytes = poMD->writeBytes(state->fPreparedOffset,
 550                                                     state->fCopyMD->getBytesNoCopy(),
 551                                                     state->fPreparedLength);
 552                     else
 553                         bytes = poMD->readBytes(state->fPreparedOffset,
 554                                                     state->fCopyMD->getBytesNoCopy(),
 555                                                     state->fPreparedLength);
 556                     DEBG("fCopyMD %s %lx bytes\n", (kWalkSyncIn & op) ? "wrote" : "read", bytes);
 557                     ret = (bytes == state->fPreparedLength) ? kIOReturnSuccess : kIOReturnUnderrun;
 558                 }
 559                 else
 560                     ret = kIOReturnSuccess;
 561             }
 562         }
 563     }
 564
 565     if (kWalkComplete & op)
 566     {
 567         if (state->fLocalMapperPageAlloc)
 568         {
 569             fMapper->iovmFreeDMACommand(this, state->fLocalMapperPageAlloc, state->fLocalMapperPageCount);
 570             state->fLocalMapperPageAlloc = 0;
 571             state->fLocalMapperPageCount = 0;
 572         }
 573         if (state->fCopyPageAlloc)
 574         {
 575             vm_page_free_list(state->fCopyPageAlloc, FALSE);
 576             state->fCopyPageAlloc = 0;
 577             state->fCopyPageCount = 0;
 578         }
 579         if (state->fCopyMD)
 580         {
 581             state->fCopyMD->release();
 582             state->fCopyMD = 0;
 583         }
 584
 585         state->fPrepared = false;
 586     }
 587     return (ret);
 588 }
 589
 590 UInt8
 591 IODMACommand::getNumAddressBits(void)
 592 {
 593     return (fNumAddressBits);
 594 }
 595
 596 UInt32
 597 IODMACommand::getAlignment(void)
 598 {
 599     return (fAlignMask + 1);
 600 }
 601
 602 IOReturn
 603 IODMACommand::prepareWithSpecification(SegmentFunction  outSegFunc,
 604                                        UInt8            numAddressBits,
 605                                        UInt64           maxSegmentSize,
 606                                        MappingOptions   mappingOptions,
 607                                        UInt64           maxTransferSize,
 608                                        UInt32           alignment,
 609                                        IOMapper         *mapper,
 610                                        UInt64           offset,
 611                                        UInt64           length,
 612                                        bool             flushCache,
 613                                        bool             synchronize)
 614 {
 615     if (fActive)
 616         return kIOReturnNotPermitted;
 617
 618     if (!outSegFunc)
 619         return kIOReturnBadArgument;
 620
 621     bool is32Bit = (OutputHost32   == outSegFunc || OutputBig32 == outSegFunc
 622                  || OutputLittle32 == outSegFunc);
 623     if (is32Bit)
 624     {
 625         if (!numAddressBits)
 626             numAddressBits = 32;
 627         else if (numAddressBits > 32)
 628             return kIOReturnBadArgument;                // Wrong output function for bits
 629     }
 630
 631     if (numAddressBits && (numAddressBits < PAGE_SHIFT))
 632         return kIOReturnBadArgument;
 633
 634     if (!maxSegmentSize)
 635         maxSegmentSize--;       // Set Max segment to -1
 636     if (!maxTransferSize)
 637         maxTransferSize--;      // Set Max transfer to -1
 638
 639     if (!mapper)
 640     {
 641         IOMapper::checkForSystemMapper();
 642         mapper = IOMapper::gSystem;
 643     }
 644
 645     switch (MAPTYPE(mappingOptions))
 646     {
 647     case kMapped:                   break;
 648     case kNonCoherent: fMapper = 0; break;
 649     case kBypassed:
 650         if (mapper && !mapper->getBypassMask(&fBypassMask))
 651             return kIOReturnBadArgument;
 652         break;
 653     default:
 654         return kIOReturnBadArgument;
 655     };
 656
 657     fNumSegments     = 0;
 658     fBypassMask      = 0;
 659     fOutSeg          = outSegFunc;
 660     fNumAddressBits  = numAddressBits;
 661     fMaxSegmentSize  = maxSegmentSize;
 662     fMappingOptions  = mappingOptions;
 663     fMaxTransferSize = maxTransferSize;
 664     if (!alignment)
 665         alignment = 1;
 666     fAlignMask       = alignment - 1;
 667     if (mapper != fMapper)
 668     {
 669         mapper->retain();
 670         fMapper->release();
 671         fMapper = mapper;
 672     }
 673
 674     fInternalState->fIterateOnly = (0 != (kIterateOnly & mappingOptions));
 675
 676     return prepare(offset, length, flushCache, synchronize);
 677 }
 678
 679
 680 IOReturn
 681 IODMACommand::prepare(UInt64 offset, UInt64 length, bool flushCache, bool synchronize)
 682 {
 683     IODMACommandInternal * state = fInternalState;
 684     IOReturn               ret   = kIOReturnSuccess;
 685     MappingOptions mappingOptions    = fMappingOptions;
 686
 687     if (!length)
 688         length = fMDSummary.fLength;
 689
 690     if (length > fMaxTransferSize)
 691         return kIOReturnNoSpace;
 692
 693     if (IS_NONCOHERENT(mappingOptions) && flushCache) {
 694         IOMemoryDescriptor *poMD = const_cast<IOMemoryDescriptor *>(fMemory);
 695
 696         poMD->performOperation(kIOMemoryIncoherentIOStore, offset, length);
 697     }
 698     if (fActive++)
 699     {
 700         if ((state->fPreparedOffset != offset)
 701           || (state->fPreparedLength != length))
 702         ret = kIOReturnNotReady;
 703     }
 704     else
 705     {
 706         state->fPreparedOffset = offset;
 707         state->fPreparedLength = length;
 708
 709         state->fMapContig      = false;
 710         state->fMisaligned     = false;
 711         state->fDoubleBuffer   = false;
 712         state->fPrepared       = false;
 713         state->fCopyNext       = NULL;
 714         state->fCopyPageAlloc  = 0;
 715         state->fCopyPageCount  = 0;
 716         state->fNextRemapPage  = NULL;
 717         state->fCopyMD         = 0;
 718         state->fLocalMapperPageAlloc = 0;
 719         state->fLocalMapperPageCount = 0;
 720
 721         state->fLocalMapper    = (fMapper && (fMapper != IOMapper::gSystem));
 722
 723         state->fSourceAlignMask = fAlignMask;
 724         if (state->fLocalMapper)
 725             state->fSourceAlignMask &= page_mask;
 726
 727         state->fCursor = state->fIterateOnly
 728                         || (!state->fCheckAddressing
 729                             && !state->fLocalMapper
 730                             && (!state->fSourceAlignMask
 731                                 || ((fMDSummary.fPageAlign & (1 << 31)) && (0 == (fMDSummary.fPageAlign & state->fSourceAlignMask)))));
 732
 733         if (!state->fCursor)
 734         {
 735             IOOptionBits op = kWalkPrepare | kWalkPreflight;
 736             if (synchronize)
 737                 op |= kWalkSyncOut;
 738             ret = walkAll(op);
 739         }
 740         if (kIOReturnSuccess == ret)
 741             state->fPrepared = true;
 742     }
 743     return ret;
 744 }
 745
 746 IOReturn
 747 IODMACommand::complete(bool invalidateCache, bool synchronize)
 748 {
 749     IODMACommandInternal * state = fInternalState;
 750     IOReturn               ret   = kIOReturnSuccess;
 751
 752     if (fActive < 1)
 753         return kIOReturnNotReady;
 754
 755     if (!--fActive)
 756     {
 757         if (!state->fCursor)
 758         {
 759                 IOOptionBits op = kWalkComplete;
 760                 if (synchronize)
 761                         op |= kWalkSyncIn;
 762                 ret = walkAll(op);
 763         }
 764         state->fPrepared = false;
 765
 766         if (IS_NONCOHERENT(fMappingOptions) && invalidateCache)
 767         {
 768             IOMemoryDescriptor *poMD = const_cast<IOMemoryDescriptor *>(fMemory);
 769
 770             poMD->performOperation(kIOMemoryIncoherentIOFlush, state->fPreparedOffset, state->fPreparedLength);
 771         }
 772     }
 773
 774     return ret;
 775 }
 776
 777 IOReturn
 778 IODMACommand::getPreparedOffsetAndLength(UInt64 * offset, UInt64 * length)
 779 {
 780     IODMACommandInternal * state = fInternalState;
 781     if (fActive < 1)
 782         return (kIOReturnNotReady);
 783
 784     if (offset)
 785         *offset = state->fPreparedOffset;
 786     if (length)
 787         *length = state->fPreparedLength;
 788
 789     return (kIOReturnSuccess);
 790 }
 791
 792 IOReturn
 793 IODMACommand::synchronize(IOOptionBits options)
 794 {
 795     IODMACommandInternal * state = fInternalState;
 796     IOReturn               ret   = kIOReturnSuccess;
 797     IOOptionBits           op;
 798
 799     if (kIODirectionOutIn == (kIODirectionOutIn & options))
 800         return kIOReturnBadArgument;
 801
 802     if (fActive < 1)
 803         return kIOReturnNotReady;
 804
 805     op = 0;
 806     if (kForceDoubleBuffer & options)
 807     {
 808         if (state->fDoubleBuffer)
 809             return kIOReturnSuccess;
 810         if (state->fCursor)
 811             state->fCursor = false;
 812         else
 813             ret = walkAll(kWalkComplete);
 814
 815         op |= kWalkPrepare | kWalkPreflight | kWalkDoubleBuffer;
 816     }
 817     else if (state->fCursor)
 818         return kIOReturnSuccess;
 819
 820     if (kIODirectionIn & options)
 821         op |= kWalkSyncIn | kWalkSyncAlways;
 822     else if (kIODirectionOut & options)
 823         op |= kWalkSyncOut | kWalkSyncAlways;
 824
 825     ret = walkAll(op);
 826
 827     return ret;
 828 }
 829
 830 struct IODMACommandTransferContext
 831 {
 832     void *   buffer;
 833     UInt64   bufferOffset;
 834     UInt64   remaining;
 835     UInt32   op;
 836 };
 837 enum
 838 {
 839     kIODMACommandTransferOpReadBytes  = 1,
 840     kIODMACommandTransferOpWriteBytes = 2
 841 };
 842
 843 IOReturn
 844 IODMACommand::transferSegment(void   *reference,
 845                         IODMACommand *target,
 846                         Segment64     segment,
 847                         void         *segments,
 848                         UInt32        segmentIndex)
 849 {
 850     IODMACommandTransferContext * context = (IODMACommandTransferContext *) reference;
 851     UInt64   length  = min(segment.fLength, context->remaining);
 852     addr64_t ioAddr  = segment.fIOVMAddr;
 853     addr64_t cpuAddr = ioAddr;
 854
 855     context->remaining -= length;
 856
 857     while (length)
 858     {
 859         UInt64 copyLen = length;
 860         if ((kMapped == MAPTYPE(target->fMappingOptions))
 861             && target->fMapper)
 862         {
 863             cpuAddr = target->fMapper->mapAddr(ioAddr);
 864             copyLen = min(copyLen, page_size - (ioAddr & (page_size - 1)));
 865             ioAddr += copyLen;
 866         }
 867
 868         switch (context->op)
 869         {
 870             case kIODMACommandTransferOpReadBytes:
 871                 copypv(cpuAddr, context->bufferOffset + (addr64_t) context->buffer, copyLen,
 872                                     cppvPsrc | cppvNoRefSrc | cppvFsnk | cppvKmap);
 873                 break;
 874             case kIODMACommandTransferOpWriteBytes:
 875                 copypv(context->bufferOffset + (addr64_t) context->buffer, cpuAddr, copyLen,
 876                                 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
 877                 break;
 878         }
 879         length                -= copyLen;
 880         context->bufferOffset += copyLen;
 881     }
 882
 883     return (context->remaining ? kIOReturnSuccess : kIOReturnOverrun);
 884 }
 885
 886 UInt64
 887 IODMACommand::transfer(IOOptionBits transferOp, UInt64 offset, void * buffer, UInt64 length)
 888 {
 889     IODMACommandInternal *      state = fInternalState;
 890     IODMACommandTransferContext context;
 891     Segment64                   segments[1];
 892     UInt32                      numSegments = 0-1;
 893
 894     if (fActive < 1)
 895         return (0);
 896
 897     if (offset >= state->fPreparedLength)
 898         return (0);
 899     length = min(length, state->fPreparedLength - offset);
 900
 901     context.buffer       = buffer;
 902     context.bufferOffset = 0;
 903     context.remaining    = length;
 904     context.op           = transferOp;
 905     (void) genIOVMSegments(kWalkClient, transferSegment, &context, &offset, &segments[0], &numSegments);
 906
 907     return (length - context.remaining);
 908 }
 909
 910 UInt64
 911 IODMACommand::readBytes(UInt64 offset, void *bytes, UInt64 length)
 912 {
 913     return (transfer(kIODMACommandTransferOpReadBytes, offset, bytes, length));
 914 }
 915
 916 UInt64
 917 IODMACommand::writeBytes(UInt64 offset, const void *bytes, UInt64 length)
 918 {
 919     return (transfer(kIODMACommandTransferOpWriteBytes, offset, const_cast<void *>(bytes), length));
 920 }
 921
 922 IOReturn
 923 IODMACommand::genIOVMSegments(UInt64 *offsetP,
 924                               void   *segmentsP,
 925                               UInt32 *numSegmentsP)
 926 {
 927     return (genIOVMSegments(kWalkClient, clientOutputSegment, (void *) fOutSeg,
 928                             offsetP, segmentsP, numSegmentsP));
 929 }
 930
 931 IOReturn
 932 IODMACommand::genIOVMSegments(uint32_t op,
 933                               InternalSegmentFunction outSegFunc,
 934                               void   *reference,
 935                               UInt64 *offsetP,
 936                               void   *segmentsP,
 937                               UInt32 *numSegmentsP)
 938 {
 939     IODMACommandInternal * internalState = fInternalState;
 940     IOOptionBits           mdOp = kIOMDWalkSegments;
 941     IOReturn               ret  = kIOReturnSuccess;
 942
 943     if (!(kWalkComplete & op) && !fActive)
 944         return kIOReturnNotReady;
 945
 946     if (!offsetP || !segmentsP || !numSegmentsP || !*numSegmentsP)
 947         return kIOReturnBadArgument;
 948
 949     IOMDDMAWalkSegmentArgs *state =
 950         (IOMDDMAWalkSegmentArgs *) fState;
 951
 952     UInt64 offset    = *offsetP + internalState->fPreparedOffset;
 953     UInt64 memLength = internalState->fPreparedOffset + internalState->fPreparedLength;
 954
 955     if (offset >= memLength)
 956         return kIOReturnOverrun;
 957
 958     if ((offset == internalState->fPreparedOffset) || (offset != state->fOffset) || internalState->fNewMD) {
 959         state->fOffset                 = 0;
 960         state->fIOVMAddr               = 0;
 961         internalState->fNextRemapPage  = NULL;
 962         internalState->fNewMD          = false;
 963         state->fMapped                 = (IS_MAPPED(fMappingOptions) && fMapper);
 964         mdOp                           = kIOMDFirstSegment;
 965     };
 966
 967     UInt64    bypassMask = fBypassMask;
 968     UInt32    segIndex = 0;
 969     UInt32    numSegments = *numSegmentsP;
 970     Segment64 curSeg = { 0, 0 };
 971     addr64_t  maxPhys;
 972
 973     if (fNumAddressBits && (fNumAddressBits < 64))
 974         maxPhys = (1ULL << fNumAddressBits);
 975     else
 976         maxPhys = 0;
 977     maxPhys--;
 978
 979     while (state->fIOVMAddr || (state->fOffset < memLength))
 980     {
 981         // state = next seg
 982         if (!state->fIOVMAddr) {
 983
 984             IOReturn rtn;
 985
 986             state->fOffset = offset;
 987             state->fLength = memLength - offset;
 988
 989             if (internalState->fMapContig && (kWalkClient & op))
 990             {
 991                 ppnum_t pageNum = internalState->fLocalMapperPageAlloc;
 992                 state->fIOVMAddr = ptoa_64(pageNum)
 993                                             + offset - internalState->fPreparedOffset;
 994                 rtn = kIOReturnSuccess;
 995             }
 996             else
 997             {
 998                 const IOMemoryDescriptor * memory =
 999                     internalState->fCopyMD ? internalState->fCopyMD : fMemory;
1000                 rtn = memory->dmaCommandOperation(mdOp, fState, sizeof(fState));
1001                 mdOp = kIOMDWalkSegments;
1002             }
1003
1004             if (rtn == kIOReturnSuccess)
1005             {
1006                 assert(state->fIOVMAddr);
1007                 assert(state->fLength);
1008                 if ((curSeg.fIOVMAddr + curSeg.fLength) == state->fIOVMAddr) {
1009                     UInt64 length = state->fLength;
1010                     offset          += length;
1011                     curSeg.fLength  += length;
1012                     state->fIOVMAddr = 0;
1013                 }
1014             }
1015             else if (rtn == kIOReturnOverrun)
1016                 state->fIOVMAddr = state->fLength = 0;  // At end
1017             else
1018                 return rtn;
1019         }
1020
1021         // seg = state, offset = end of seg
1022         if (!curSeg.fIOVMAddr)
1023         {
1024             UInt64 length = state->fLength;
1025             offset          += length;
1026             curSeg.fIOVMAddr = state->fIOVMAddr | bypassMask;
1027             curSeg.fLength   = length;
1028             state->fIOVMAddr = 0;
1029         }
1030
1031         if (!state->fIOVMAddr)
1032         {
1033             if ((kWalkClient & op) && (curSeg.fIOVMAddr + curSeg.fLength - 1) > maxPhys)
1034             {
1035                 if (internalState->fCursor)
1036                 {
1037                     curSeg.fIOVMAddr = 0;
1038                     ret = kIOReturnMessageTooLarge;
1039                     break;
1040                 }
1041                 else if (curSeg.fIOVMAddr <= maxPhys)
1042                 {
1043                     UInt64 remain, newLength;
1044
1045                     newLength        = (maxPhys + 1 - curSeg.fIOVMAddr);
1046                     DEBG("trunc %qx, %qx-> %qx\n", curSeg.fIOVMAddr, curSeg.fLength, newLength);
1047                     remain           = curSeg.fLength - newLength;
1048                     state->fIOVMAddr = newLength + curSeg.fIOVMAddr;
1049                     curSeg.fLength   = newLength;
1050                     state->fLength   = remain;
1051                     offset          -= remain;
1052                 }
1053                 else
1054                 {
1055                     UInt64    addr = curSeg.fIOVMAddr;
1056                     ppnum_t   addrPage = atop_64(addr);
1057                     vm_page_t remap = NULL;
1058                     UInt64    remain, newLength;
1059
1060                     DEBG("sparse switch %qx, %qx ", addr, curSeg.fLength);
1061
1062                     remap = internalState->fNextRemapPage;
1063                     if (remap && (addrPage == vm_page_get_offset(remap)))
1064                     {
1065                     }
1066                     else for (remap = internalState->fCopyPageAlloc;
1067                                 remap && (addrPage != vm_page_get_offset(remap));
1068                                 remap = vm_page_get_next(remap))
1069                     {
1070                     }
1071
1072                     if (!remap) panic("no remap page found");
1073
1074                     curSeg.fIOVMAddr = ptoa_64(vm_page_get_phys_page(remap))
1075                                         + (addr & PAGE_MASK);
1076                     internalState->fNextRemapPage = vm_page_get_next(remap);
1077
1078                     newLength            = PAGE_SIZE - (addr & PAGE_MASK);
1079                     if (newLength < curSeg.fLength)
1080                     {
1081                         remain           = curSeg.fLength - newLength;
1082                         state->fIOVMAddr = addr + newLength;
1083                         curSeg.fLength   = newLength;
1084                         state->fLength   = remain;
1085                         offset          -= remain;
1086                     }
1087                     DEBG("-> %qx, %qx offset %qx\n", curSeg.fIOVMAddr, curSeg.fLength, offset);
1088                 }
1089             }
1090
1091             if (curSeg.fLength > fMaxSegmentSize)
1092             {
1093                 UInt64 remain = curSeg.fLength - fMaxSegmentSize;
1094
1095                 state->fIOVMAddr = fMaxSegmentSize + curSeg.fIOVMAddr;
1096                 curSeg.fLength   = fMaxSegmentSize;
1097
1098                 state->fLength   = remain;
1099                 offset          -= remain;
1100             }
1101
1102             if (internalState->fCursor
1103                 && (0 != (internalState->fSourceAlignMask & curSeg.fIOVMAddr)))
1104             {
1105                 curSeg.fIOVMAddr = 0;
1106                 ret = kIOReturnNotAligned;
1107                 break;
1108             }
1109
1110             if (offset >= memLength)
1111             {
1112                 curSeg.fLength   -= (offset - memLength);
1113                 offset = memLength;
1114                 state->fIOVMAddr = state->fLength = 0;  // At end
1115                 break;
1116             }
1117         }
1118
1119         if (state->fIOVMAddr) {
1120             if ((segIndex + 1 == numSegments))
1121                 break;
1122
1123             ret = (*outSegFunc)(reference, this, curSeg, segmentsP, segIndex++);
1124             curSeg.fIOVMAddr = 0;
1125             if (kIOReturnSuccess != ret)
1126                 break;
1127         }
1128     }
1129
1130     if (curSeg.fIOVMAddr) {
1131         ret = (*outSegFunc)(reference, this, curSeg, segmentsP, segIndex++);
1132     }
1133
1134     if (kIOReturnSuccess == ret)
1135     {
1136         state->fOffset = offset;
1137         *offsetP       = offset - internalState->fPreparedOffset;
1138         *numSegmentsP  = segIndex;
1139     }
1140     return ret;
1141 }
1142
1143 IOReturn
1144 IODMACommand::clientOutputSegment(
1145         void *reference, IODMACommand *target,
1146         Segment64 segment, void *vSegList, UInt32 outSegIndex)
1147 {
1148     SegmentFunction segmentFunction = (SegmentFunction) reference;
1149     IOReturn ret = kIOReturnSuccess;
1150
1151     if (target->fNumAddressBits && (target->fNumAddressBits < 64)
1152         && ((segment.fIOVMAddr + segment.fLength - 1) >> target->fNumAddressBits)
1153         && (target->reserved->fLocalMapperPageAlloc || !target->reserved->fLocalMapper))
1154     {
1155         DEBG("kIOReturnMessageTooLarge(fNumAddressBits) %qx, %qx\n", segment.fIOVMAddr, segment.fLength);
1156         ret = kIOReturnMessageTooLarge;
1157     }
1158
1159     if (!(*segmentFunction)(target, segment, vSegList, outSegIndex))
1160     {
1161         DEBG("kIOReturnMessageTooLarge(fOutSeg) %qx, %qx\n", segment.fIOVMAddr, segment.fLength);
1162         ret = kIOReturnMessageTooLarge;
1163     }
1164
1165     return (ret);
1166 }
1167
1168 IOReturn
1169 IODMACommand::genIOVMSegments(SegmentFunction segmentFunction,
1170                                     UInt64   *offsetP,
1171                                     void     *segmentsP,
1172                                     UInt32   *numSegmentsP)
1173 {
1174     return (genIOVMSegments(kWalkClient, clientOutputSegment, (void *) segmentFunction,
1175                             offsetP, segmentsP, numSegmentsP));
1176 }
1177
1178 bool
1179 IODMACommand::OutputHost32(IODMACommand *,
1180         Segment64 segment, void *vSegList, UInt32 outSegIndex)
1181 {
1182     Segment32 *base = (Segment32 *) vSegList;
1183     base[outSegIndex].fIOVMAddr = (UInt32) segment.fIOVMAddr;
1184     base[outSegIndex].fLength   = (UInt32) segment.fLength;
1185     return true;
1186 }
1187
1188 bool
1189 IODMACommand::OutputBig32(IODMACommand *,
1190         Segment64 segment, void *vSegList, UInt32 outSegIndex)
1191 {
1192     const UInt offAddr = outSegIndex * sizeof(Segment32);
1193     const UInt offLen  = offAddr + sizeof(UInt32);
1194     OSWriteBigInt32(vSegList, offAddr, (UInt32) segment.fIOVMAddr);
1195     OSWriteBigInt32(vSegList, offLen,  (UInt32) segment.fLength);
1196     return true;
1197 }
1198
1199 bool
1200 IODMACommand::OutputLittle32(IODMACommand *,
1201         Segment64 segment, void *vSegList, UInt32 outSegIndex)
1202 {
1203     const UInt offAddr = outSegIndex * sizeof(Segment32);
1204     const UInt offLen  = offAddr + sizeof(UInt32);
1205     OSWriteLittleInt32(vSegList, offAddr, (UInt32) segment.fIOVMAddr);
1206     OSWriteLittleInt32(vSegList, offLen,  (UInt32) segment.fLength);
1207     return true;
1208 }
1209
1210 bool
1211 IODMACommand::OutputHost64(IODMACommand *,
1212         Segment64 segment, void *vSegList, UInt32 outSegIndex)
1213 {
1214     Segment64 *base = (Segment64 *) vSegList;
1215     base[outSegIndex] = segment;
1216     return true;
1217 }
1218
1219 bool
1220 IODMACommand::OutputBig64(IODMACommand *,
1221         Segment64 segment, void *vSegList, UInt32 outSegIndex)
1222 {
1223     const UInt offAddr = outSegIndex * sizeof(Segment64);
1224     const UInt offLen  = offAddr + sizeof(UInt64);
1225     OSWriteBigInt64(vSegList, offAddr, (UInt64) segment.fIOVMAddr);
1226     OSWriteBigInt64(vSegList, offLen,  (UInt64) segment.fLength);
1227     return true;
1228 }
1229
1230 bool
1231 IODMACommand::OutputLittle64(IODMACommand *,
1232         Segment64 segment, void *vSegList, UInt32 outSegIndex)
1233 {
1234     const UInt offAddr = outSegIndex * sizeof(Segment64);
1235     const UInt offLen  = offAddr + sizeof(UInt64);
1236     OSWriteLittleInt64(vSegList, offAddr, (UInt64) segment.fIOVMAddr);
1237     OSWriteLittleInt64(vSegList, offLen,  (UInt64) segment.fLength);
1238     return true;
1239 }
1240
1241