iokit/Kernel/IODMACommand.cpp

   1 /*
   2  * Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <IOKit/assert.h>
  30
  31 #include <libkern/OSTypes.h>
  32 #include <libkern/OSByteOrder.h>
  33
  34 #include <IOKit/IOReturn.h>
  35 #include <IOKit/IOLib.h>
  36 #include <IOKit/IODMACommand.h>
  37 #include <IOKit/IOMapper.h>
  38 #include <IOKit/IOMemoryDescriptor.h>
  39 #include <IOKit/IOBufferMemoryDescriptor.h>
  40
  41 #include "IOKitKernelInternal.h"
  42
  43 #define MAPTYPE(type)           ((UInt) (type) & kTypeMask)
  44 #define IS_MAPPED(type)         (MAPTYPE(type) == kMapped)
  45 #define IS_BYPASSED(type)       (MAPTYPE(type) == kBypassed)
  46 #define IS_NONCOHERENT(type)    (MAPTYPE(type) == kNonCoherent)
  47
  48 enum
  49 {
  50     kWalkSyncIn       = 0x01,   // bounce -> md
  51     kWalkSyncOut      = 0x02,   // bounce <- md
  52     kWalkSyncAlways   = 0x04,
  53     kWalkPreflight    = 0x08,
  54     kWalkDoubleBuffer = 0x10,
  55     kWalkPrepare      = 0x20,
  56     kWalkComplete     = 0x40,
  57     kWalkClient       = 0x80
  58 };
  59
  60
  61 #define fInternalState reserved
  62 #define fState         reserved->fState
  63 #define fMDSummary     reserved->fMDSummary
  64
  65
  66 #if 1
  67 // no direction => OutIn
  68 #define SHOULD_COPY_DIR(op, direction)                                      \
  69         ((kIODirectionNone == (direction))                                  \
  70             || (kWalkSyncAlways & (op))                                     \
  71             || (((kWalkSyncIn & (op)) ? kIODirectionIn : kIODirectionOut)   \
  72                                                     & (direction)))
  73
  74 #else
  75 #define SHOULD_COPY_DIR(state, direction) (true)
  76 #endif
  77
  78 #if 0
  79 #define DEBG(fmt, args...)      { IOLog(fmt, ## args); kprintf(fmt, ## args); }
  80 #else
  81 #define DEBG(fmt, args...)      {}
  82 #endif
  83
  84 /**************************** class IODMACommand ***************************/
  85
  86 #undef super
  87 #define super OSObject
  88 OSDefineMetaClassAndStructors(IODMACommand, IOCommand);
  89
  90 OSMetaClassDefineReservedUsed(IODMACommand,  0);
  91 OSMetaClassDefineReservedUsed(IODMACommand,  1);
  92 OSMetaClassDefineReservedUsed(IODMACommand,  2);
  93 OSMetaClassDefineReservedUnused(IODMACommand,  3);
  94 OSMetaClassDefineReservedUnused(IODMACommand,  4);
  95 OSMetaClassDefineReservedUnused(IODMACommand,  5);
  96 OSMetaClassDefineReservedUnused(IODMACommand,  6);
  97 OSMetaClassDefineReservedUnused(IODMACommand,  7);
  98 OSMetaClassDefineReservedUnused(IODMACommand,  8);
  99 OSMetaClassDefineReservedUnused(IODMACommand,  9);
 100 OSMetaClassDefineReservedUnused(IODMACommand, 10);
 101 OSMetaClassDefineReservedUnused(IODMACommand, 11);
 102 OSMetaClassDefineReservedUnused(IODMACommand, 12);
 103 OSMetaClassDefineReservedUnused(IODMACommand, 13);
 104 OSMetaClassDefineReservedUnused(IODMACommand, 14);
 105 OSMetaClassDefineReservedUnused(IODMACommand, 15);
 106
 107 IODMACommand *
 108 IODMACommand::withSpecification(SegmentFunction outSegFunc,
 109                                 UInt8           numAddressBits,
 110                                 UInt64          maxSegmentSize,
 111                                 MappingOptions  mappingOptions,
 112                                 UInt64          maxTransferSize,
 113                                 UInt32          alignment,
 114                                 IOMapper       *mapper,
 115                                 void           *refCon)
 116 {
 117     IODMACommand * me = new IODMACommand;
 118
 119     if (me && !me->initWithSpecification(outSegFunc,
 120                                          numAddressBits, maxSegmentSize,
 121                                          mappingOptions, maxTransferSize,
 122                                          alignment,      mapper, refCon))
 123     {
 124         me->release();
 125         return 0;
 126     };
 127
 128     return me;
 129 }
 130
 131 IODMACommand *
 132 IODMACommand::cloneCommand(void *refCon)
 133 {
 134     return withSpecification(fOutSeg, fNumAddressBits, fMaxSegmentSize,
 135             fMappingOptions, fMaxTransferSize, fAlignMask + 1, fMapper, refCon);
 136 }
 137
 138 #define kLastOutputFunction ((SegmentFunction) kLastOutputFunction)
 139
 140 bool
 141 IODMACommand::initWithSpecification(SegmentFunction outSegFunc,
 142                                     UInt8           numAddressBits,
 143                                     UInt64          maxSegmentSize,
 144                                     MappingOptions  mappingOptions,
 145                                     UInt64          maxTransferSize,
 146                                     UInt32          alignment,
 147                                     IOMapper       *mapper,
 148                                     void           *refCon)
 149 {
 150     if (!super::init() || !outSegFunc || !numAddressBits)
 151         return false;
 152
 153     bool is32Bit = (OutputHost32   == outSegFunc || OutputBig32 == outSegFunc
 154                  || OutputLittle32 == outSegFunc);
 155     if (is32Bit)
 156     {
 157         if (!numAddressBits)
 158             numAddressBits = 32;
 159         else if (numAddressBits > 32)
 160             return false;               // Wrong output function for bits
 161     }
 162
 163     if (numAddressBits && (numAddressBits < PAGE_SHIFT))
 164         return false;
 165
 166     if (!maxSegmentSize)
 167         maxSegmentSize--;       // Set Max segment to -1
 168     if (!maxTransferSize)
 169         maxTransferSize--;      // Set Max transfer to -1
 170
 171     if (!mapper)
 172     {
 173         IOMapper::checkForSystemMapper();
 174         mapper = IOMapper::gSystem;
 175     }
 176
 177     fNumSegments     = 0;
 178     fBypassMask      = 0;
 179     fOutSeg          = outSegFunc;
 180     fNumAddressBits  = numAddressBits;
 181     fMaxSegmentSize  = maxSegmentSize;
 182     fMappingOptions  = mappingOptions;
 183     fMaxTransferSize = maxTransferSize;
 184     if (!alignment)
 185         alignment = 1;
 186     fAlignMask       = alignment - 1;
 187     fMapper          = mapper;
 188     fRefCon          = refCon;
 189
 190     switch (MAPTYPE(mappingOptions))
 191     {
 192     case kMapped:                   break;
 193     case kNonCoherent: fMapper = 0; break;
 194     case kBypassed:
 195         if (mapper && !mapper->getBypassMask(&fBypassMask))
 196             return false;
 197         break;
 198     default:
 199         return false;
 200     };
 201
 202     if (fMapper)
 203         fMapper->retain();
 204
 205     reserved = IONew(IODMACommandInternal, 1);
 206     if (!reserved)
 207         return false;
 208     bzero(reserved, sizeof(IODMACommandInternal));
 209
 210     fInternalState->fIterateOnly = (0 != (kIterateOnly & mappingOptions));
 211
 212     return true;
 213 }
 214
 215 void
 216 IODMACommand::free()
 217 {
 218     if (reserved)
 219         IODelete(reserved, IODMACommandInternal, 1);
 220
 221     if (fMapper)
 222         fMapper->release();
 223
 224     super::free();
 225 }
 226
 227 IOReturn
 228 IODMACommand::setMemoryDescriptor(const IOMemoryDescriptor *mem, bool autoPrepare)
 229 {
 230     if (mem == fMemory)
 231     {
 232         if (!autoPrepare)
 233         {
 234             while (fActive)
 235                 complete();
 236         }
 237         return kIOReturnSuccess;
 238     }
 239
 240     if (fMemory) {
 241         // As we are almost certainly being called from a work loop thread
 242         // if fActive is true it is probably not a good time to potentially
 243         // block.  Just test for it and return an error
 244         if (fActive)
 245             return kIOReturnBusy;
 246         clearMemoryDescriptor();
 247     };
 248
 249     if (mem) {
 250         bzero(&fMDSummary, sizeof(fMDSummary));
 251         IOReturn rtn = mem->dmaCommandOperation(
 252                 kIOMDGetCharacteristics,
 253                 &fMDSummary, sizeof(fMDSummary));
 254         if (rtn)
 255             return rtn;
 256
 257         ppnum_t highPage = fMDSummary.fHighestPage ? fMDSummary.fHighestPage : gIOLastPage;
 258
 259         if ((kMapped == MAPTYPE(fMappingOptions))
 260             && fMapper
 261             && (!fNumAddressBits || (fNumAddressBits >= 31)))
 262             // assuming mapped space is 2G
 263             fInternalState->fCheckAddressing = false;
 264         else
 265             fInternalState->fCheckAddressing = (fNumAddressBits && (highPage >= (1UL << (fNumAddressBits - PAGE_SHIFT))));
 266
 267         fInternalState->fNewMD = true;
 268         mem->retain();
 269         fMemory = mem;
 270
 271         mem->dmaCommandOperation(kIOMDSetDMAActive, this, 0);
 272         if (autoPrepare)
 273             return prepare();
 274     };
 275
 276     return kIOReturnSuccess;
 277 }
 278
 279 IOReturn
 280 IODMACommand::clearMemoryDescriptor(bool autoComplete)
 281 {
 282     if (fActive && !autoComplete)
 283         return (kIOReturnNotReady);
 284
 285     if (fMemory) {
 286         while (fActive)
 287             complete();
 288         fMemory->dmaCommandOperation(kIOMDSetDMAInactive, this, 0);
 289         fMemory->release();
 290         fMemory = 0;
 291     }
 292
 293     return (kIOReturnSuccess);
 294 }
 295
 296 const IOMemoryDescriptor *
 297 IODMACommand::getMemoryDescriptor() const
 298 {
 299     return fMemory;
 300 }
 301
 302
 303 IOReturn
 304 IODMACommand::segmentOp(
 305                         void         *reference,
 306                         IODMACommand *target,
 307                         Segment64     segment,
 308                         void         *segments,
 309                         UInt32        segmentIndex)
 310 {
 311     IOOptionBits op = (uintptr_t) reference;
 312     addr64_t     maxPhys, address;
 313     uint64_t     length;
 314     uint32_t     numPages;
 315
 316     IODMACommandInternal * state = target->reserved;
 317
 318     if (target->fNumAddressBits && (target->fNumAddressBits < 64) && !state->fLocalMapper)
 319         maxPhys = (1ULL << target->fNumAddressBits);
 320     else
 321         maxPhys = 0;
 322     maxPhys--;
 323
 324     address = segment.fIOVMAddr;
 325     length = segment.fLength;
 326
 327     assert(address);
 328     assert(length);
 329
 330     if (!state->fMisaligned)
 331     {
 332         state->fMisaligned |= (0 != (state->fSourceAlignMask & address));
 333         if (state->fMisaligned) DEBG("misaligned %qx:%qx, %lx\n", address, length, state->fSourceAlignMask);
 334     }
 335
 336     if (state->fMisaligned && (kWalkPreflight & op))
 337         return (kIOReturnNotAligned);
 338
 339     if (!state->fDoubleBuffer)
 340     {
 341         if ((address + length - 1) <= maxPhys)
 342         {
 343             length = 0;
 344         }
 345         else if (address <= maxPhys)
 346         {
 347             DEBG("tail %qx, %qx", address, length);
 348             length = (address + length - maxPhys - 1);
 349             address = maxPhys + 1;
 350             DEBG("-> %qx, %qx\n", address, length);
 351         }
 352     }
 353
 354     if (!length)
 355         return (kIOReturnSuccess);
 356
 357     numPages = atop_64(round_page_64((address & PAGE_MASK) + length));
 358
 359     if (kWalkPreflight & op)
 360     {
 361         state->fCopyPageCount += numPages;
 362     }
 363     else
 364     {
 365         vm_page_t lastPage;
 366         lastPage = NULL;
 367         if (kWalkPrepare & op)
 368         {
 369             lastPage = state->fCopyNext;
 370             for (IOItemCount idx = 0; idx < numPages; idx++)
 371             {
 372                 vm_page_set_offset(lastPage, atop_64(address) + idx);
 373                 lastPage = vm_page_get_next(lastPage);
 374             }
 375         }
 376
 377         if (!lastPage || SHOULD_COPY_DIR(op, target->fMDSummary.fDirection))
 378         {
 379             lastPage = state->fCopyNext;
 380             for (IOItemCount idx = 0; idx < numPages; idx++)
 381             {
 382                 if (SHOULD_COPY_DIR(op, target->fMDSummary.fDirection))
 383                 {
 384                     addr64_t remapAddr;
 385                     uint64_t chunk;
 386
 387                     remapAddr = ptoa_64(vm_page_get_phys_page(lastPage));
 388                     if (!state->fDoubleBuffer)
 389                     {
 390                         remapAddr += (address & PAGE_MASK);
 391                     }
 392                     chunk = PAGE_SIZE - (address & PAGE_MASK);
 393                     if (chunk > length)
 394                         chunk = length;
 395
 396                     DEBG("cpv: 0x%qx %s 0x%qx, 0x%qx, 0x%02lx\n", remapAddr,
 397                                 (kWalkSyncIn & op) ? "->" : "<-",
 398                                 address, chunk, op);
 399
 400                     if (kWalkSyncIn & op)
 401                     { // cppvNoModSnk
 402                         copypv(remapAddr, address, chunk,
 403                                         cppvPsnk | cppvFsnk | cppvPsrc | cppvNoRefSrc );
 404                     }
 405                     else
 406                     {
 407                         copypv(address, remapAddr, chunk,
 408                                         cppvPsnk | cppvFsnk | cppvPsrc | cppvNoRefSrc );
 409                     }
 410                     address += chunk;
 411                     length -= chunk;
 412                 }
 413                 lastPage = vm_page_get_next(lastPage);
 414             }
 415         }
 416         state->fCopyNext = lastPage;
 417     }
 418
 419     return kIOReturnSuccess;
 420 }
 421
 422 IOReturn
 423 IODMACommand::walkAll(UInt8 op)
 424 {
 425     IODMACommandInternal * state = fInternalState;
 426
 427     IOReturn     ret = kIOReturnSuccess;
 428     UInt32       numSegments;
 429     UInt64       offset;
 430
 431     if (kWalkPreflight & op)
 432     {
 433         state->fMapContig      = false;
 434         state->fMisaligned     = false;
 435         state->fDoubleBuffer   = false;
 436         state->fPrepared       = false;
 437         state->fCopyNext       = NULL;
 438         state->fCopyPageAlloc  = 0;
 439         state->fLocalMapperPageAlloc = 0;
 440         state->fCopyPageCount  = 0;
 441         state->fNextRemapPage  = NULL;
 442         state->fCopyMD         = 0;
 443
 444         if (!(kWalkDoubleBuffer & op))
 445         {
 446             offset = 0;
 447             numSegments = 0-1;
 448             ret = genIOVMSegments(op, segmentOp, (void *) op, &offset, state, &numSegments);
 449         }
 450
 451         op &= ~kWalkPreflight;
 452
 453         state->fDoubleBuffer = (state->fMisaligned || (kWalkDoubleBuffer & op));
 454         if (state->fDoubleBuffer)
 455             state->fCopyPageCount = atop_64(round_page(state->fPreparedLength));
 456
 457         if (state->fCopyPageCount)
 458         {
 459             vm_page_t mapBase = NULL;
 460
 461             DEBG("preflight fCopyPageCount %d\n", state->fCopyPageCount);
 462
 463             if (!state->fDoubleBuffer)
 464             {
 465                 kern_return_t kr;
 466                 kr = vm_page_alloc_list(state->fCopyPageCount,
 467                                         KMA_LOMEM | KMA_NOPAGEWAIT, &mapBase);
 468                 if (KERN_SUCCESS != kr)
 469                 {
 470                     DEBG("vm_page_alloc_list(%d) failed (%d)\n", state->fCopyPageCount, kr);
 471                     mapBase = NULL;
 472                 }
 473             }
 474
 475             if (mapBase)
 476             {
 477                 state->fCopyPageAlloc = mapBase;
 478                 state->fCopyNext = state->fCopyPageAlloc;
 479                 offset = 0;
 480                 numSegments = 0-1;
 481                 ret = genIOVMSegments(op, segmentOp, (void *) op, &offset, state, &numSegments);
 482                 state->fPrepared = true;
 483                 op &= ~(kWalkSyncIn | kWalkSyncOut);
 484             }
 485             else
 486             {
 487                 DEBG("alloc IOBMD\n");
 488                 mach_vm_address_t mask = 0xFFFFF000; //state->fSourceAlignMask
 489                 state->fCopyMD = IOBufferMemoryDescriptor::inTaskWithPhysicalMask(kernel_task,
 490                                     fMDSummary.fDirection, state->fPreparedLength, mask);
 491
 492                 if (state->fCopyMD)
 493                 {
 494                     ret = kIOReturnSuccess;
 495                     state->fPrepared = true;
 496                 }
 497                 else
 498                 {
 499                     DEBG("IODMACommand !iovmAlloc");
 500                     return (kIOReturnNoResources);
 501                 }
 502             }
 503         }
 504
 505         if (state->fLocalMapper)
 506         {
 507             state->fLocalMapperPageCount = atop_64(round_page(
 508                     state->fPreparedLength + ((state->fPreparedOffset + fMDSummary.fPageAlign) & page_mask)));
 509             state->fLocalMapperPageAlloc = fMapper->iovmAllocDMACommand(this, state->fLocalMapperPageCount);
 510             state->fMapContig = true;
 511         }
 512     }
 513
 514     if (state->fPrepared && ((kWalkSyncIn | kWalkSyncOut) & op))
 515     {
 516         if (state->fCopyPageCount)
 517         {
 518             DEBG("sync fCopyPageCount %d\n", state->fCopyPageCount);
 519
 520             if (state->fCopyPageAlloc)
 521             {
 522                 state->fCopyNext = state->fCopyPageAlloc;
 523                 offset = 0;
 524                 numSegments = 0-1;
 525                 ret = genIOVMSegments(op, segmentOp, (void *) op, &offset, state, &numSegments);
 526             }
 527             else if (state->fCopyMD)
 528             {
 529                 DEBG("sync IOBMD\n");
 530
 531                 if (SHOULD_COPY_DIR(op, fMDSummary.fDirection))
 532                 {
 533                     IOMemoryDescriptor *poMD = const_cast<IOMemoryDescriptor *>(fMemory);
 534
 535                     IOByteCount bytes;
 536
 537                     if (kWalkSyncIn & op)
 538                         bytes = poMD->writeBytes(state->fPreparedOffset,
 539                                                     state->fCopyMD->getBytesNoCopy(),
 540                                                     state->fPreparedLength);
 541                     else
 542                         bytes = poMD->readBytes(state->fPreparedOffset,
 543                                                     state->fCopyMD->getBytesNoCopy(),
 544                                                     state->fPreparedLength);
 545                     DEBG("fCopyMD %s %lx bytes\n", (kWalkSyncIn & op) ? "wrote" : "read", bytes);
 546                     ret = (bytes == state->fPreparedLength) ? kIOReturnSuccess : kIOReturnUnderrun;
 547                 }
 548                 else
 549                     ret = kIOReturnSuccess;
 550             }
 551         }
 552     }
 553
 554     if (kWalkComplete & op)
 555     {
 556         if (state->fLocalMapperPageAlloc)
 557         {
 558             fMapper->iovmFreeDMACommand(this, state->fLocalMapperPageAlloc, state->fLocalMapperPageCount);
 559             state->fLocalMapperPageAlloc = 0;
 560             state->fLocalMapperPageCount = 0;
 561         }
 562         if (state->fCopyPageAlloc)
 563         {
 564             vm_page_free_list(state->fCopyPageAlloc, FALSE);
 565             state->fCopyPageAlloc = 0;
 566             state->fCopyPageCount = 0;
 567         }
 568         if (state->fCopyMD)
 569         {
 570             state->fCopyMD->release();
 571             state->fCopyMD = 0;
 572         }
 573
 574         state->fPrepared = false;
 575     }
 576     return (ret);
 577 }
 578
 579 UInt8
 580 IODMACommand::getNumAddressBits(void)
 581 {
 582     return (fNumAddressBits);
 583 }
 584
 585 UInt32
 586 IODMACommand::getAlignment(void)
 587 {
 588     return (fAlignMask + 1);
 589 }
 590
 591 IOReturn
 592 IODMACommand::prepareWithSpecification(SegmentFunction  outSegFunc,
 593                                        UInt8            numAddressBits,
 594                                        UInt64           maxSegmentSize,
 595                                        MappingOptions   mappingOptions,
 596                                        UInt64           maxTransferSize,
 597                                        UInt32           alignment,
 598                                        IOMapper         *mapper,
 599                                        UInt64           offset,
 600                                        UInt64           length,
 601                                        bool             flushCache,
 602                                        bool             synchronize)
 603 {
 604     if (fActive)
 605         return kIOReturnNotPermitted;
 606
 607     if (!outSegFunc || !numAddressBits)
 608         return kIOReturnBadArgument;
 609
 610     bool is32Bit = (OutputHost32   == outSegFunc || OutputBig32 == outSegFunc
 611                  || OutputLittle32 == outSegFunc);
 612     if (is32Bit)
 613     {
 614         if (!numAddressBits)
 615             numAddressBits = 32;
 616         else if (numAddressBits > 32)
 617             return kIOReturnBadArgument;                // Wrong output function for bits
 618     }
 619
 620     if (numAddressBits && (numAddressBits < PAGE_SHIFT))
 621         return kIOReturnBadArgument;
 622
 623     if (!maxSegmentSize)
 624         maxSegmentSize--;       // Set Max segment to -1
 625     if (!maxTransferSize)
 626         maxTransferSize--;      // Set Max transfer to -1
 627
 628     if (!mapper)
 629     {
 630         IOMapper::checkForSystemMapper();
 631         mapper = IOMapper::gSystem;
 632     }
 633
 634     switch (MAPTYPE(mappingOptions))
 635     {
 636     case kMapped:                   break;
 637     case kNonCoherent: fMapper = 0; break;
 638     case kBypassed:
 639         if (mapper && !mapper->getBypassMask(&fBypassMask))
 640             return kIOReturnBadArgument;
 641         break;
 642     default:
 643         return kIOReturnBadArgument;
 644     };
 645
 646     fNumSegments     = 0;
 647     fBypassMask      = 0;
 648     fOutSeg          = outSegFunc;
 649     fNumAddressBits  = numAddressBits;
 650     fMaxSegmentSize  = maxSegmentSize;
 651     fMappingOptions  = mappingOptions;
 652     fMaxTransferSize = maxTransferSize;
 653     if (!alignment)
 654         alignment = 1;
 655     fAlignMask       = alignment - 1;
 656     if (mapper != fMapper)
 657     {
 658         mapper->retain();
 659         fMapper->release();
 660         fMapper = mapper;
 661     }
 662
 663     fInternalState->fIterateOnly = (0 != (kIterateOnly & mappingOptions));
 664
 665     return prepare(offset, length, flushCache, synchronize);
 666 }
 667
 668
 669 IOReturn
 670 IODMACommand::prepare(UInt64 offset, UInt64 length, bool flushCache, bool synchronize)
 671 {
 672     IODMACommandInternal * state = fInternalState;
 673     IOReturn               ret   = kIOReturnSuccess;
 674     MappingOptions mappingOptions    = fMappingOptions;
 675
 676     if (!length)
 677         length = fMDSummary.fLength;
 678
 679     if (length > fMaxTransferSize)
 680         return kIOReturnNoSpace;
 681
 682     if (IS_NONCOHERENT(mappingOptions) && flushCache) {
 683         IOMemoryDescriptor *poMD = const_cast<IOMemoryDescriptor *>(fMemory);
 684
 685         poMD->performOperation(kIOMemoryIncoherentIOStore, offset, length);
 686     }
 687     if (fActive++)
 688     {
 689         if ((state->fPreparedOffset != offset)
 690           || (state->fPreparedLength != length))
 691         ret = kIOReturnNotReady;
 692     }
 693     else
 694     {
 695         state->fPreparedOffset = offset;
 696         state->fPreparedLength = length;
 697
 698         state->fMapContig      = false;
 699         state->fMisaligned     = false;
 700         state->fDoubleBuffer   = false;
 701         state->fPrepared       = false;
 702         state->fCopyNext       = NULL;
 703         state->fCopyPageAlloc  = 0;
 704         state->fCopyPageCount  = 0;
 705         state->fNextRemapPage  = NULL;
 706         state->fCopyMD         = 0;
 707         state->fLocalMapperPageAlloc = 0;
 708         state->fLocalMapperPageCount = 0;
 709
 710         state->fLocalMapper    = (fMapper && (fMapper != IOMapper::gSystem));
 711
 712         state->fSourceAlignMask = fAlignMask;
 713         if (state->fLocalMapper)
 714             state->fSourceAlignMask &= page_mask;
 715
 716         state->fCursor = state->fIterateOnly
 717                         || (!state->fCheckAddressing
 718                             && !state->fLocalMapper
 719                             && (!state->fSourceAlignMask
 720                                 || ((fMDSummary.fPageAlign & (1 << 31)) && (0 == (fMDSummary.fPageAlign & state->fSourceAlignMask)))));
 721
 722         if (!state->fCursor)
 723         {
 724             IOOptionBits op = kWalkPrepare | kWalkPreflight;
 725             if (synchronize)
 726                 op |= kWalkSyncOut;
 727             ret = walkAll(op);
 728         }
 729         if (kIOReturnSuccess == ret)
 730             state->fPrepared = true;
 731     }
 732     return ret;
 733 }
 734
 735 IOReturn
 736 IODMACommand::complete(bool invalidateCache, bool synchronize)
 737 {
 738     IODMACommandInternal * state = fInternalState;
 739     IOReturn               ret   = kIOReturnSuccess;
 740
 741     if (fActive < 1)
 742         return kIOReturnNotReady;
 743
 744     if (!--fActive)
 745     {
 746         if (!state->fCursor)
 747         {
 748                 IOOptionBits op = kWalkComplete;
 749                 if (synchronize)
 750                         op |= kWalkSyncIn;
 751                 ret = walkAll(op);
 752         }
 753         state->fPrepared = false;
 754
 755         if (IS_NONCOHERENT(fMappingOptions) && invalidateCache)
 756         {
 757             IOMemoryDescriptor *poMD = const_cast<IOMemoryDescriptor *>(fMemory);
 758
 759             poMD->performOperation(kIOMemoryIncoherentIOFlush, state->fPreparedOffset, state->fPreparedLength);
 760         }
 761     }
 762
 763     return ret;
 764 }
 765
 766 IOReturn
 767 IODMACommand::getPreparedOffsetAndLength(UInt64 * offset, UInt64 * length)
 768 {
 769     IODMACommandInternal * state = fInternalState;
 770     if (fActive < 1)
 771         return (kIOReturnNotReady);
 772
 773     if (offset)
 774         *offset = state->fPreparedOffset;
 775     if (length)
 776         *length = state->fPreparedLength;
 777
 778     return (kIOReturnSuccess);
 779 }
 780
 781 IOReturn
 782 IODMACommand::synchronize(IOOptionBits options)
 783 {
 784     IODMACommandInternal * state = fInternalState;
 785     IOReturn               ret   = kIOReturnSuccess;
 786     IOOptionBits           op;
 787
 788     if (kIODirectionOutIn == (kIODirectionOutIn & options))
 789         return kIOReturnBadArgument;
 790
 791     if (fActive < 1)
 792         return kIOReturnNotReady;
 793
 794     op = 0;
 795     if (kForceDoubleBuffer & options)
 796     {
 797         if (state->fDoubleBuffer)
 798             return kIOReturnSuccess;
 799         if (state->fCursor)
 800             state->fCursor = false;
 801         else
 802             ret = walkAll(kWalkComplete);
 803
 804         op |= kWalkPrepare | kWalkPreflight | kWalkDoubleBuffer;
 805     }
 806     else if (state->fCursor)
 807         return kIOReturnSuccess;
 808
 809     if (kIODirectionIn & options)
 810         op |= kWalkSyncIn | kWalkSyncAlways;
 811     else if (kIODirectionOut & options)
 812         op |= kWalkSyncOut | kWalkSyncAlways;
 813
 814     ret = walkAll(op);
 815
 816     return ret;
 817 }
 818
 819 struct IODMACommandTransferContext
 820 {
 821     void *   buffer;
 822     UInt64   bufferOffset;
 823     UInt64   remaining;
 824     UInt32   op;
 825 };
 826 enum
 827 {
 828     kIODMACommandTransferOpReadBytes  = 1,
 829     kIODMACommandTransferOpWriteBytes = 2
 830 };
 831
 832 IOReturn
 833 IODMACommand::transferSegment(void   *reference,
 834                         IODMACommand *target,
 835                         Segment64     segment,
 836                         void         *segments,
 837                         UInt32        segmentIndex)
 838 {
 839     IODMACommandTransferContext * context = (IODMACommandTransferContext *) reference;
 840     UInt64   length  = min(segment.fLength, context->remaining);
 841     addr64_t ioAddr  = segment.fIOVMAddr;
 842     addr64_t cpuAddr = ioAddr;
 843
 844     context->remaining -= length;
 845
 846     while (length)
 847     {
 848         UInt64 copyLen = length;
 849         if ((kMapped == MAPTYPE(target->fMappingOptions))
 850             && target->fMapper)
 851         {
 852             cpuAddr = target->fMapper->mapAddr(ioAddr);
 853             copyLen = min(copyLen, page_size - (ioAddr & (page_size - 1)));
 854             ioAddr += copyLen;
 855         }
 856
 857         switch (context->op)
 858         {
 859             case kIODMACommandTransferOpReadBytes:
 860                 copypv(cpuAddr, context->bufferOffset + (addr64_t) context->buffer, copyLen,
 861                                     cppvPsrc | cppvNoRefSrc | cppvFsnk | cppvKmap);
 862                 break;
 863             case kIODMACommandTransferOpWriteBytes:
 864                 copypv(context->bufferOffset + (addr64_t) context->buffer, cpuAddr, copyLen,
 865                                 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
 866                 break;
 867         }
 868         length                -= copyLen;
 869         context->bufferOffset += copyLen;
 870     }
 871
 872     return (context->remaining ? kIOReturnSuccess : kIOReturnOverrun);
 873 }
 874
 875 UInt64
 876 IODMACommand::transfer(IOOptionBits transferOp, UInt64 offset, void * buffer, UInt64 length)
 877 {
 878     IODMACommandInternal *      state = fInternalState;
 879     IODMACommandTransferContext context;
 880     Segment64                   segments[1];
 881     UInt32                      numSegments = 0-1;
 882
 883     if (fActive < 1)
 884         return (0);
 885
 886     if (offset >= state->fPreparedLength)
 887         return (0);
 888     length = min(length, state->fPreparedLength - offset);
 889
 890     context.buffer       = buffer;
 891     context.bufferOffset = 0;
 892     context.remaining    = length;
 893     context.op           = transferOp;
 894     (void) genIOVMSegments(kWalkClient, transferSegment, &context, &offset, &segments[0], &numSegments);
 895
 896     return (length - context.remaining);
 897 }
 898
 899 UInt64
 900 IODMACommand::readBytes(UInt64 offset, void *bytes, UInt64 length)
 901 {
 902     return (transfer(kIODMACommandTransferOpReadBytes, offset, bytes, length));
 903 }
 904
 905 UInt64
 906 IODMACommand::writeBytes(UInt64 offset, const void *bytes, UInt64 length)
 907 {
 908     return (transfer(kIODMACommandTransferOpWriteBytes, offset, const_cast<void *>(bytes), length));
 909 }
 910
 911 IOReturn
 912 IODMACommand::genIOVMSegments(UInt64 *offsetP,
 913                               void   *segmentsP,
 914                               UInt32 *numSegmentsP)
 915 {
 916     return (genIOVMSegments(kWalkClient, clientOutputSegment, (void *) fOutSeg,
 917                             offsetP, segmentsP, numSegmentsP));
 918 }
 919
 920 IOReturn
 921 IODMACommand::genIOVMSegments(uint32_t op,
 922                               InternalSegmentFunction outSegFunc,
 923                               void   *reference,
 924                               UInt64 *offsetP,
 925                               void   *segmentsP,
 926                               UInt32 *numSegmentsP)
 927 {
 928     IODMACommandInternal * internalState = fInternalState;
 929     IOOptionBits           mdOp = kIOMDWalkSegments;
 930     IOReturn               ret  = kIOReturnSuccess;
 931
 932     if (!(kWalkComplete & op) && !fActive)
 933         return kIOReturnNotReady;
 934
 935     if (!offsetP || !segmentsP || !numSegmentsP || !*numSegmentsP)
 936         return kIOReturnBadArgument;
 937
 938     IOMDDMAWalkSegmentArgs *state =
 939         (IOMDDMAWalkSegmentArgs *) fState;
 940
 941     UInt64 offset    = *offsetP + internalState->fPreparedOffset;
 942     UInt64 memLength = internalState->fPreparedOffset + internalState->fPreparedLength;
 943
 944     if (offset >= memLength)
 945         return kIOReturnOverrun;
 946
 947     if ((offset == internalState->fPreparedOffset) || (offset != state->fOffset) || internalState->fNewMD) {
 948         state->fOffset                 = 0;
 949         state->fIOVMAddr               = 0;
 950         internalState->fNextRemapPage  = NULL;
 951         internalState->fNewMD          = false;
 952         state->fMapped                 = (IS_MAPPED(fMappingOptions) && fMapper);
 953         mdOp                           = kIOMDFirstSegment;
 954     };
 955
 956     UInt64    bypassMask = fBypassMask;
 957     UInt32    segIndex = 0;
 958     UInt32    numSegments = *numSegmentsP;
 959     Segment64 curSeg = { 0, 0 };
 960     addr64_t  maxPhys;
 961
 962     if (fNumAddressBits && (fNumAddressBits < 64))
 963         maxPhys = (1ULL << fNumAddressBits);
 964     else
 965         maxPhys = 0;
 966     maxPhys--;
 967
 968     while (state->fIOVMAddr || (state->fOffset < memLength))
 969     {
 970         // state = next seg
 971         if (!state->fIOVMAddr) {
 972
 973             IOReturn rtn;
 974
 975             state->fOffset = offset;
 976             state->fLength = memLength - offset;
 977
 978             if (internalState->fMapContig && (kWalkClient & op))
 979             {
 980                 ppnum_t pageNum = internalState->fLocalMapperPageAlloc;
 981                 state->fIOVMAddr = ptoa_64(pageNum)
 982                                             + offset - internalState->fPreparedOffset;
 983                 rtn = kIOReturnSuccess;
 984             }
 985             else
 986             {
 987                 const IOMemoryDescriptor * memory =
 988                     internalState->fCopyMD ? internalState->fCopyMD : fMemory;
 989                 rtn = memory->dmaCommandOperation(mdOp, fState, sizeof(fState));
 990                 mdOp = kIOMDWalkSegments;
 991             }
 992
 993             if (rtn == kIOReturnSuccess)
 994             {
 995                 assert(state->fIOVMAddr);
 996                 assert(state->fLength);
 997                 if ((curSeg.fIOVMAddr + curSeg.fLength) == state->fIOVMAddr) {
 998                     UInt64 length = state->fLength;
 999                     offset          += length;
1000                     curSeg.fLength  += length;
1001                     state->fIOVMAddr = 0;
1002                 }
1003             }
1004             else if (rtn == kIOReturnOverrun)
1005                 state->fIOVMAddr = state->fLength = 0;  // At end
1006             else
1007                 return rtn;
1008         }
1009
1010         // seg = state, offset = end of seg
1011         if (!curSeg.fIOVMAddr)
1012         {
1013             UInt64 length = state->fLength;
1014             offset          += length;
1015             curSeg.fIOVMAddr = state->fIOVMAddr | bypassMask;
1016             curSeg.fLength   = length;
1017             state->fIOVMAddr = 0;
1018         }
1019
1020         if (!state->fIOVMAddr)
1021         {
1022             if ((kWalkClient & op) && (curSeg.fIOVMAddr + curSeg.fLength - 1) > maxPhys)
1023             {
1024                 if (internalState->fCursor)
1025                 {
1026                     curSeg.fIOVMAddr = 0;
1027                     ret = kIOReturnMessageTooLarge;
1028                     break;
1029                 }
1030                 else if (curSeg.fIOVMAddr <= maxPhys)
1031                 {
1032                     UInt64 remain, newLength;
1033
1034                     newLength        = (maxPhys + 1 - curSeg.fIOVMAddr);
1035                     DEBG("trunc %qx, %qx-> %qx\n", curSeg.fIOVMAddr, curSeg.fLength, newLength);
1036                     remain           = curSeg.fLength - newLength;
1037                     state->fIOVMAddr = newLength + curSeg.fIOVMAddr;
1038                     curSeg.fLength   = newLength;
1039                     state->fLength   = remain;
1040                     offset          -= remain;
1041                 }
1042                 else
1043                 {
1044                     UInt64    addr = curSeg.fIOVMAddr;
1045                     ppnum_t   addrPage = atop_64(addr);
1046                     vm_page_t remap = NULL;
1047                     UInt64    remain, newLength;
1048
1049                     DEBG("sparse switch %qx, %qx ", addr, curSeg.fLength);
1050
1051                     remap = internalState->fNextRemapPage;
1052                     if (remap && (addrPage == vm_page_get_offset(remap)))
1053                     {
1054                     }
1055                     else for (remap = internalState->fCopyPageAlloc;
1056                                 remap && (addrPage != vm_page_get_offset(remap));
1057                                 remap = vm_page_get_next(remap))
1058                     {
1059                     }
1060
1061                     if (!remap) panic("no remap page found");
1062
1063                     curSeg.fIOVMAddr = ptoa_64(vm_page_get_phys_page(remap))
1064                                         + (addr & PAGE_MASK);
1065                     internalState->fNextRemapPage = vm_page_get_next(remap);
1066
1067                     newLength            = PAGE_SIZE - (addr & PAGE_MASK);
1068                     if (newLength < curSeg.fLength)
1069                     {
1070                         remain           = curSeg.fLength - newLength;
1071                         state->fIOVMAddr = addr + newLength;
1072                         curSeg.fLength   = newLength;
1073                         state->fLength   = remain;
1074                         offset          -= remain;
1075                     }
1076                     DEBG("-> %qx, %qx offset %qx\n", curSeg.fIOVMAddr, curSeg.fLength, offset);
1077                 }
1078             }
1079
1080             if (curSeg.fLength > fMaxSegmentSize)
1081             {
1082                 UInt64 remain = curSeg.fLength - fMaxSegmentSize;
1083
1084                 state->fIOVMAddr = fMaxSegmentSize + curSeg.fIOVMAddr;
1085                 curSeg.fLength   = fMaxSegmentSize;
1086
1087                 state->fLength   = remain;
1088                 offset          -= remain;
1089             }
1090
1091             if (internalState->fCursor
1092                 && (0 != (internalState->fSourceAlignMask & curSeg.fIOVMAddr)))
1093             {
1094                 curSeg.fIOVMAddr = 0;
1095                 ret = kIOReturnNotAligned;
1096                 break;
1097             }
1098
1099             if (offset >= memLength)
1100             {
1101                 curSeg.fLength   -= (offset - memLength);
1102                 offset = memLength;
1103                 state->fIOVMAddr = state->fLength = 0;  // At end
1104                 break;
1105             }
1106         }
1107
1108         if (state->fIOVMAddr) {
1109             if ((segIndex + 1 == numSegments))
1110                 break;
1111
1112             ret = (*outSegFunc)(reference, this, curSeg, segmentsP, segIndex++);
1113             curSeg.fIOVMAddr = 0;
1114             if (kIOReturnSuccess != ret)
1115                 break;
1116         }
1117     }
1118
1119     if (curSeg.fIOVMAddr) {
1120         ret = (*outSegFunc)(reference, this, curSeg, segmentsP, segIndex++);
1121     }
1122
1123     if (kIOReturnSuccess == ret)
1124     {
1125         state->fOffset = offset;
1126         *offsetP       = offset - internalState->fPreparedOffset;
1127         *numSegmentsP  = segIndex;
1128     }
1129     return ret;
1130 }
1131
1132 IOReturn
1133 IODMACommand::clientOutputSegment(
1134         void *reference, IODMACommand *target,
1135         Segment64 segment, void *vSegList, UInt32 outSegIndex)
1136 {
1137     SegmentFunction segmentFunction = (SegmentFunction) reference;
1138     IOReturn ret = kIOReturnSuccess;
1139
1140     if ((target->fNumAddressBits < 64)
1141         && ((segment.fIOVMAddr + segment.fLength - 1) >> target->fNumAddressBits)
1142         && (target->reserved->fLocalMapperPageAlloc || !target->reserved->fLocalMapper))
1143     {
1144         DEBG("kIOReturnMessageTooLarge(fNumAddressBits) %qx, %qx\n", segment.fIOVMAddr, segment.fLength);
1145         ret = kIOReturnMessageTooLarge;
1146     }
1147
1148     if (!(*segmentFunction)(target, segment, vSegList, outSegIndex))
1149     {
1150         DEBG("kIOReturnMessageTooLarge(fOutSeg) %qx, %qx\n", segment.fIOVMAddr, segment.fLength);
1151         ret = kIOReturnMessageTooLarge;
1152     }
1153
1154     return (ret);
1155 }
1156
1157 IOReturn
1158 IODMACommand::genIOVMSegments(SegmentFunction segmentFunction,
1159                                     UInt64   *offsetP,
1160                                     void     *segmentsP,
1161                                     UInt32   *numSegmentsP)
1162 {
1163     return (genIOVMSegments(kWalkClient, clientOutputSegment, (void *) segmentFunction,
1164                             offsetP, segmentsP, numSegmentsP));
1165 }
1166
1167 bool
1168 IODMACommand::OutputHost32(IODMACommand *,
1169         Segment64 segment, void *vSegList, UInt32 outSegIndex)
1170 {
1171     Segment32 *base = (Segment32 *) vSegList;
1172     base[outSegIndex].fIOVMAddr = (UInt32) segment.fIOVMAddr;
1173     base[outSegIndex].fLength   = (UInt32) segment.fLength;
1174     return true;
1175 }
1176
1177 bool
1178 IODMACommand::OutputBig32(IODMACommand *,
1179         Segment64 segment, void *vSegList, UInt32 outSegIndex)
1180 {
1181     const UInt offAddr = outSegIndex * sizeof(Segment32);
1182     const UInt offLen  = offAddr + sizeof(UInt32);
1183     OSWriteBigInt32(vSegList, offAddr, (UInt32) segment.fIOVMAddr);
1184     OSWriteBigInt32(vSegList, offLen,  (UInt32) segment.fLength);
1185     return true;
1186 }
1187
1188 bool
1189 IODMACommand::OutputLittle32(IODMACommand *,
1190         Segment64 segment, void *vSegList, UInt32 outSegIndex)
1191 {
1192     const UInt offAddr = outSegIndex * sizeof(Segment32);
1193     const UInt offLen  = offAddr + sizeof(UInt32);
1194     OSWriteLittleInt32(vSegList, offAddr, (UInt32) segment.fIOVMAddr);
1195     OSWriteLittleInt32(vSegList, offLen,  (UInt32) segment.fLength);
1196     return true;
1197 }
1198
1199 bool
1200 IODMACommand::OutputHost64(IODMACommand *,
1201         Segment64 segment, void *vSegList, UInt32 outSegIndex)
1202 {
1203     Segment64 *base = (Segment64 *) vSegList;
1204     base[outSegIndex] = segment;
1205     return true;
1206 }
1207
1208 bool
1209 IODMACommand::OutputBig64(IODMACommand *,
1210         Segment64 segment, void *vSegList, UInt32 outSegIndex)
1211 {
1212     const UInt offAddr = outSegIndex * sizeof(Segment64);
1213     const UInt offLen  = offAddr + sizeof(UInt64);
1214     OSWriteBigInt64(vSegList, offAddr, (UInt64) segment.fIOVMAddr);
1215     OSWriteBigInt64(vSegList, offLen,  (UInt64) segment.fLength);
1216     return true;
1217 }
1218
1219 bool
1220 IODMACommand::OutputLittle64(IODMACommand *,
1221         Segment64 segment, void *vSegList, UInt32 outSegIndex)
1222 {
1223     const UInt offAddr = outSegIndex * sizeof(Segment64);
1224     const UInt offLen  = offAddr + sizeof(UInt64);
1225     OSWriteLittleInt64(vSegList, offAddr, (UInt64) segment.fIOVMAddr);
1226     OSWriteLittleInt64(vSegList, offLen,  (UInt64) segment.fLength);
1227     return true;
1228 }
1229
1230