iokit/Kernel/IODMACommand.cpp

   1 /*
   2  * Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <IOKit/assert.h>
  30
  31 #include <libkern/OSTypes.h>
  32 #include <libkern/OSByteOrder.h>
  33 #include <libkern/OSDebug.h>
  34
  35 #include <IOKit/IOReturn.h>
  36 #include <IOKit/IOLib.h>
  37 #include <IOKit/IODMACommand.h>
  38 #include <IOKit/IOMapper.h>
  39 #include <IOKit/IOMemoryDescriptor.h>
  40 #include <IOKit/IOBufferMemoryDescriptor.h>
  41
  42 #include "IOKitKernelInternal.h"
  43
  44 #define MAPTYPE(type)           ((UInt) (type) & kTypeMask)
  45 #define IS_NONCOHERENT(type)    (MAPTYPE(type) == kNonCoherent)
  46
  47 enum
  48 {
  49     kWalkSyncIn       = 0x01,   // bounce -> md
  50     kWalkSyncOut      = 0x02,   // bounce <- md
  51     kWalkSyncAlways   = 0x04,
  52     kWalkPreflight    = 0x08,
  53     kWalkDoubleBuffer = 0x10,
  54     kWalkPrepare      = 0x20,
  55     kWalkComplete     = 0x40,
  56     kWalkClient       = 0x80
  57 };
  58
  59
  60 #define fInternalState reserved
  61 #define fState         reserved->fState
  62 #define fMDSummary     reserved->fMDSummary
  63
  64
  65 #if 1
  66 // no direction => OutIn
  67 #define SHOULD_COPY_DIR(op, direction)                                      \
  68         ((kIODirectionNone == (direction))                                  \
  69             || (kWalkSyncAlways & (op))                                     \
  70             || (((kWalkSyncIn & (op)) ? kIODirectionIn : kIODirectionOut)   \
  71                                                     & (direction)))
  72
  73 #else
  74 #define SHOULD_COPY_DIR(state, direction) (true)
  75 #endif
  76
  77 #if 0
  78 #define DEBG(fmt, args...)      { IOLog(fmt, ## args); kprintf(fmt, ## args); }
  79 #else
  80 #define DEBG(fmt, args...)      {}
  81 #endif
  82
  83 /**************************** class IODMACommand ***************************/
  84
  85 #undef super
  86 #define super IOCommand
  87 OSDefineMetaClassAndStructors(IODMACommand, IOCommand);
  88
  89 OSMetaClassDefineReservedUsed(IODMACommand,  0);
  90 OSMetaClassDefineReservedUsed(IODMACommand,  1);
  91 OSMetaClassDefineReservedUsed(IODMACommand,  2);
  92 OSMetaClassDefineReservedUsed(IODMACommand,  3);
  93 OSMetaClassDefineReservedUsed(IODMACommand,  4);
  94 OSMetaClassDefineReservedUsed(IODMACommand,  5);
  95 OSMetaClassDefineReservedUsed(IODMACommand,  6);
  96 OSMetaClassDefineReservedUnused(IODMACommand,  7);
  97 OSMetaClassDefineReservedUnused(IODMACommand,  8);
  98 OSMetaClassDefineReservedUnused(IODMACommand,  9);
  99 OSMetaClassDefineReservedUnused(IODMACommand, 10);
 100 OSMetaClassDefineReservedUnused(IODMACommand, 11);
 101 OSMetaClassDefineReservedUnused(IODMACommand, 12);
 102 OSMetaClassDefineReservedUnused(IODMACommand, 13);
 103 OSMetaClassDefineReservedUnused(IODMACommand, 14);
 104 OSMetaClassDefineReservedUnused(IODMACommand, 15);
 105
 106 IODMACommand *
 107 IODMACommand::withRefCon(void * refCon)
 108 {
 109     IODMACommand * me = new IODMACommand;
 110
 111     if (me && !me->initWithRefCon(refCon))
 112     {
 113         me->release();
 114         return 0;
 115     }
 116
 117     return me;
 118 }
 119
 120 IODMACommand *
 121 IODMACommand::withSpecification(SegmentFunction  outSegFunc,
 122                           const SegmentOptions * segmentOptions,
 123                           uint32_t               mappingOptions,
 124                           IOMapper             * mapper,
 125                           void                 * refCon)
 126 {
 127     IODMACommand * me = new IODMACommand;
 128
 129     if (me && !me->initWithSpecification(outSegFunc, segmentOptions, mappingOptions,
 130                                          mapper, refCon))
 131     {
 132         me->release();
 133         return 0;
 134     }
 135
 136     return me;
 137 }
 138
 139 IODMACommand *
 140 IODMACommand::withSpecification(SegmentFunction outSegFunc,
 141                                 UInt8           numAddressBits,
 142                                 UInt64          maxSegmentSize,
 143                                 MappingOptions  mappingOptions,
 144                                 UInt64          maxTransferSize,
 145                                 UInt32          alignment,
 146                                 IOMapper       *mapper,
 147                                 void           *refCon)
 148 {
 149     IODMACommand * me = new IODMACommand;
 150
 151     if (me && !me->initWithSpecification(outSegFunc,
 152                                          numAddressBits, maxSegmentSize,
 153                                          mappingOptions, maxTransferSize,
 154                                          alignment,      mapper, refCon))
 155     {
 156         me->release();
 157         return 0;
 158     }
 159
 160     return me;
 161 }
 162
 163 IODMACommand *
 164 IODMACommand::cloneCommand(void *refCon)
 165 {
 166     SegmentOptions segmentOptions =
 167     {
 168         .fStructSize                = sizeof(segmentOptions),
 169         .fNumAddressBits            = (uint8_t)fNumAddressBits,
 170         .fMaxSegmentSize            = fMaxSegmentSize,
 171         .fMaxTransferSize           = fMaxTransferSize,
 172         .fAlignment                 = fAlignMask + 1,
 173         .fAlignmentLength           = fAlignMaskInternalSegments + 1,
 174         .fAlignmentInternalSegments = fAlignMaskLength + 1
 175     };
 176
 177     return (IODMACommand::withSpecification(fOutSeg, &segmentOptions,
 178                                             fMappingOptions, fMapper, refCon));
 179 }
 180
 181 #define kLastOutputFunction ((SegmentFunction) kLastOutputFunction)
 182
 183 bool
 184 IODMACommand::initWithRefCon(void * refCon)
 185 {
 186     if (!super::init()) return (false);
 187
 188     if (!reserved)
 189     {
 190         reserved = IONew(IODMACommandInternal, 1);
 191         if (!reserved) return false;
 192     }
 193     bzero(reserved, sizeof(IODMACommandInternal));
 194     fRefCon = refCon;
 195
 196     return (true);
 197 }
 198
 199 bool
 200 IODMACommand::initWithSpecification(SegmentFunction        outSegFunc,
 201                                     const SegmentOptions * segmentOptions,
 202                                     uint32_t               mappingOptions,
 203                                     IOMapper             * mapper,
 204                                     void                 * refCon)
 205 {
 206     if (!initWithRefCon(refCon)) return false;
 207
 208     if (kIOReturnSuccess != setSpecification(outSegFunc, segmentOptions,
 209                                              mappingOptions, mapper))      return false;
 210
 211     return (true);
 212 }
 213
 214 bool
 215 IODMACommand::initWithSpecification(SegmentFunction outSegFunc,
 216                                     UInt8           numAddressBits,
 217                                     UInt64          maxSegmentSize,
 218                                     MappingOptions  mappingOptions,
 219                                     UInt64          maxTransferSize,
 220                                     UInt32          alignment,
 221                                     IOMapper       *mapper,
 222                                     void           *refCon)
 223 {
 224     SegmentOptions segmentOptions =
 225     {
 226         .fStructSize                = sizeof(segmentOptions),
 227         .fNumAddressBits            = numAddressBits,
 228         .fMaxSegmentSize            = maxSegmentSize,
 229         .fMaxTransferSize           = maxTransferSize,
 230         .fAlignment                 = alignment,
 231         .fAlignmentLength           = 1,
 232         .fAlignmentInternalSegments = alignment
 233     };
 234
 235     return (initWithSpecification(outSegFunc, &segmentOptions, mappingOptions, mapper, refCon));
 236 }
 237
 238 IOReturn
 239 IODMACommand::setSpecification(SegmentFunction        outSegFunc,
 240                                const SegmentOptions * segmentOptions,
 241                                uint32_t               mappingOptions,
 242                                IOMapper             * mapper)
 243 {
 244     IOService * device = 0;
 245     UInt8       numAddressBits;
 246     UInt64      maxSegmentSize;
 247     UInt64      maxTransferSize;
 248     UInt32      alignment;
 249
 250     bool        is32Bit;
 251
 252     if (!outSegFunc || !segmentOptions) return (kIOReturnBadArgument);
 253
 254     is32Bit = ((OutputHost32 == outSegFunc)
 255                 || (OutputBig32 == outSegFunc)
 256                 || (OutputLittle32 == outSegFunc));
 257
 258     numAddressBits = segmentOptions->fNumAddressBits;
 259     maxSegmentSize = segmentOptions->fMaxSegmentSize;
 260     maxTransferSize = segmentOptions->fMaxTransferSize;
 261     alignment = segmentOptions->fAlignment;
 262     if (is32Bit)
 263     {
 264         if (!numAddressBits)
 265             numAddressBits = 32;
 266         else if (numAddressBits > 32)
 267             return (kIOReturnBadArgument);              // Wrong output function for bits
 268     }
 269
 270     if (numAddressBits && (numAddressBits < PAGE_SHIFT)) return (kIOReturnBadArgument);
 271
 272     if (!maxSegmentSize)  maxSegmentSize--;     // Set Max segment to -1
 273     if (!maxTransferSize) maxTransferSize--;    // Set Max transfer to -1
 274
 275     if (mapper && !OSDynamicCast(IOMapper, mapper))
 276     {
 277         device = mapper;
 278         mapper = 0;
 279     }
 280     if (!mapper && (kUnmapped != MAPTYPE(mappingOptions)))
 281     {
 282         IOMapper::checkForSystemMapper();
 283         mapper = IOMapper::gSystem;
 284     }
 285
 286     fNumSegments     = 0;
 287     fOutSeg          = outSegFunc;
 288     fNumAddressBits  = numAddressBits;
 289     fMaxSegmentSize  = maxSegmentSize;
 290     fMappingOptions  = mappingOptions;
 291     fMaxTransferSize = maxTransferSize;
 292     if (!alignment)    alignment = 1;
 293     fAlignMask       = alignment - 1;
 294
 295     alignment = segmentOptions->fAlignmentLength;
 296     if (!alignment) alignment = 1;
 297     fAlignMaskLength = alignment - 1;
 298
 299     alignment = segmentOptions->fAlignmentInternalSegments;
 300     if (!alignment) alignment = (fAlignMask + 1);
 301     fAlignMaskInternalSegments = alignment - 1;
 302
 303     switch (MAPTYPE(mappingOptions))
 304     {
 305     case kMapped:       break;
 306     case kUnmapped:     break;
 307     case kNonCoherent:  break;
 308
 309     case kBypassed:
 310         if (!mapper)    break;
 311         return (kIOReturnBadArgument);
 312
 313     default:
 314         return (kIOReturnBadArgument);
 315     };
 316
 317     if (mapper != fMapper)
 318     {
 319         if (mapper)  mapper->retain();
 320         if (fMapper) fMapper->release();
 321         fMapper = mapper;
 322     }
 323
 324     fInternalState->fIterateOnly = (0 != (kIterateOnly & mappingOptions));
 325     fInternalState->fDevice = device;
 326
 327     return (kIOReturnSuccess);
 328 }
 329
 330 void
 331 IODMACommand::free()
 332 {
 333     if (reserved) IODelete(reserved, IODMACommandInternal, 1);
 334
 335     if (fMapper) fMapper->release();
 336
 337     super::free();
 338 }
 339
 340 IOReturn
 341 IODMACommand::setMemoryDescriptor(const IOMemoryDescriptor *mem, bool autoPrepare)
 342 {
 343     IOReturn err = kIOReturnSuccess;
 344
 345     if (mem == fMemory)
 346     {
 347         if (!autoPrepare)
 348         {
 349             while (fActive)
 350                 complete();
 351         }
 352         return kIOReturnSuccess;
 353     }
 354
 355     if (fMemory) {
 356         // As we are almost certainly being called from a work loop thread
 357         // if fActive is true it is probably not a good time to potentially
 358         // block.  Just test for it and return an error
 359         if (fActive)
 360             return kIOReturnBusy;
 361         clearMemoryDescriptor();
 362     }
 363
 364     if (mem) {
 365         bzero(&fMDSummary, sizeof(fMDSummary));
 366         err = mem->dmaCommandOperation(kIOMDGetCharacteristics | (kMapped == MAPTYPE(fMappingOptions)),
 367                                        &fMDSummary, sizeof(fMDSummary));
 368         if (err)
 369             return err;
 370
 371         ppnum_t highPage = fMDSummary.fHighestPage ? fMDSummary.fHighestPage : gIOLastPage;
 372
 373         if ((kMapped == MAPTYPE(fMappingOptions))
 374             && fMapper)
 375             fInternalState->fCheckAddressing = false;
 376         else
 377             fInternalState->fCheckAddressing = (fNumAddressBits && (highPage >= (1UL << (fNumAddressBits - PAGE_SHIFT))));
 378
 379         fInternalState->fNewMD = true;
 380         mem->retain();
 381         fMemory = mem;
 382         if (!fMapper) mem->dmaCommandOperation(kIOMDSetDMAActive, this, 0);
 383         if (autoPrepare) {
 384             err = prepare();
 385             if (err) {
 386                 clearMemoryDescriptor();
 387             }
 388         }
 389     }
 390
 391     return err;
 392 }
 393
 394 IOReturn
 395 IODMACommand::clearMemoryDescriptor(bool autoComplete)
 396 {
 397     if (fActive && !autoComplete) return (kIOReturnNotReady);
 398
 399     if (fMemory)
 400     {
 401         while (fActive) complete();
 402         if (!fMapper) fMemory->dmaCommandOperation(kIOMDSetDMAInactive, this, 0);
 403         fMemory->release();
 404         fMemory = 0;
 405     }
 406
 407     return (kIOReturnSuccess);
 408 }
 409
 410 const IOMemoryDescriptor *
 411 IODMACommand::getMemoryDescriptor() const
 412 {
 413     return fMemory;
 414 }
 415
 416 IOMemoryDescriptor *
 417 IODMACommand::getIOMemoryDescriptor() const
 418 {
 419     IOMemoryDescriptor * mem;
 420
 421     mem = reserved->fCopyMD;
 422     if (!mem) mem = __IODEQUALIFY(IOMemoryDescriptor *, fMemory);
 423
 424     return (mem);
 425 }
 426
 427 IOReturn
 428 IODMACommand::segmentOp(
 429                         void         *reference,
 430                         IODMACommand *target,
 431                         Segment64     segment,
 432                         void         *segments,
 433                         UInt32        segmentIndex)
 434 {
 435     IOOptionBits op = (uintptr_t) reference;
 436     addr64_t     maxPhys, address;
 437     uint64_t     length;
 438     uint32_t     numPages;
 439     uint32_t     mask;
 440
 441     IODMACommandInternal * state = target->reserved;
 442
 443     if (target->fNumAddressBits && (target->fNumAddressBits < 64) && (state->fLocalMapperAllocValid || !target->fMapper))
 444         maxPhys = (1ULL << target->fNumAddressBits);
 445     else
 446         maxPhys = 0;
 447     maxPhys--;
 448
 449     address = segment.fIOVMAddr;
 450     length = segment.fLength;
 451
 452     assert(length);
 453
 454     if (!state->fMisaligned)
 455     {
 456         mask = (segmentIndex ? target->fAlignMaskInternalSegments : state->fSourceAlignMask);
 457         state->fMisaligned |= (0 != (mask & address));
 458         if (state->fMisaligned) DEBG("misaligned address %qx:%qx, %x\n", address, length, mask);
 459     }
 460     if (!state->fMisaligned)
 461     {
 462         mask = target->fAlignMaskLength;
 463         state->fMisaligned |= (0 != (mask & length));
 464         if (state->fMisaligned) DEBG("misaligned length %qx:%qx, %x\n", address, length, mask);
 465     }
 466
 467     if (state->fMisaligned && (kWalkPreflight & op))
 468         return (kIOReturnNotAligned);
 469
 470     if (!state->fDoubleBuffer)
 471     {
 472         if ((address + length - 1) <= maxPhys)
 473         {
 474             length = 0;
 475         }
 476         else if (address <= maxPhys)
 477         {
 478             DEBG("tail %qx, %qx", address, length);
 479             length = (address + length - maxPhys - 1);
 480             address = maxPhys + 1;
 481             DEBG("-> %qx, %qx\n", address, length);
 482         }
 483     }
 484
 485     if (!length)
 486         return (kIOReturnSuccess);
 487
 488     numPages = atop_64(round_page_64((address & PAGE_MASK) + length));
 489
 490     if (kWalkPreflight & op)
 491     {
 492         state->fCopyPageCount += numPages;
 493     }
 494     else
 495     {
 496         vm_page_t lastPage;
 497         lastPage = NULL;
 498         if (kWalkPrepare & op)
 499         {
 500             lastPage = state->fCopyNext;
 501             for (IOItemCount idx = 0; idx < numPages; idx++)
 502             {
 503                 vm_page_set_offset(lastPage, atop_64(address) + idx);
 504                 lastPage = vm_page_get_next(lastPage);
 505             }
 506         }
 507
 508         if (!lastPage || SHOULD_COPY_DIR(op, target->fMDSummary.fDirection))
 509         {
 510             lastPage = state->fCopyNext;
 511             for (IOItemCount idx = 0; idx < numPages; idx++)
 512             {
 513                 if (SHOULD_COPY_DIR(op, target->fMDSummary.fDirection))
 514                 {
 515                     addr64_t cpuAddr = address;
 516                     addr64_t remapAddr;
 517                     uint64_t chunk;
 518
 519                     if ((kMapped == MAPTYPE(target->fMappingOptions))
 520                         && target->fMapper)
 521                     {
 522                         cpuAddr = target->fMapper->mapToPhysicalAddress(address);
 523                     }
 524
 525                     remapAddr = ptoa_64(vm_page_get_phys_page(lastPage));
 526                     if (!state->fDoubleBuffer)
 527                     {
 528                         remapAddr += (address & PAGE_MASK);
 529                     }
 530                     chunk = PAGE_SIZE - (address & PAGE_MASK);
 531                     if (chunk > length)
 532                         chunk = length;
 533
 534                     DEBG("cpv: 0x%qx %s 0x%qx, 0x%qx, 0x%02lx\n", remapAddr,
 535                                 (kWalkSyncIn & op) ? "->" : "<-",
 536                                 address, chunk, op);
 537
 538                     if (kWalkSyncIn & op)
 539                     { // cppvNoModSnk
 540                         copypv(remapAddr, cpuAddr, chunk,
 541                                         cppvPsnk | cppvFsnk | cppvPsrc | cppvNoRefSrc );
 542                     }
 543                     else
 544                     {
 545                         copypv(cpuAddr, remapAddr, chunk,
 546                                         cppvPsnk | cppvFsnk | cppvPsrc | cppvNoRefSrc );
 547                     }
 548                     address += chunk;
 549                     length -= chunk;
 550                 }
 551                 lastPage = vm_page_get_next(lastPage);
 552             }
 553         }
 554         state->fCopyNext = lastPage;
 555     }
 556
 557     return kIOReturnSuccess;
 558 }
 559
 560 IOBufferMemoryDescriptor *
 561 IODMACommand::createCopyBuffer(IODirection direction, UInt64 length)
 562 {
 563     mach_vm_address_t mask = 0xFFFFF000;        //state->fSourceAlignMask
 564     return (IOBufferMemoryDescriptor::inTaskWithPhysicalMask(kernel_task,
 565                                                         direction, length, mask));
 566 }
 567
 568 IOReturn
 569 IODMACommand::walkAll(UInt8 op)
 570 {
 571     IODMACommandInternal * state = fInternalState;
 572
 573     IOReturn     ret = kIOReturnSuccess;
 574     UInt32       numSegments;
 575     UInt64       offset;
 576
 577     if (kWalkPreflight & op)
 578     {
 579         state->fMisaligned     = false;
 580         state->fDoubleBuffer   = false;
 581         state->fPrepared       = false;
 582         state->fCopyNext       = NULL;
 583         state->fCopyPageAlloc  = 0;
 584         state->fCopyPageCount  = 0;
 585         state->fNextRemapPage  = NULL;
 586         state->fCopyMD         = 0;
 587
 588         if (!(kWalkDoubleBuffer & op))
 589         {
 590             offset = 0;
 591             numSegments = 0-1;
 592             ret = genIOVMSegments(op, segmentOp, (void *)(uintptr_t) op, &offset, state, &numSegments);
 593         }
 594
 595         op &= ~kWalkPreflight;
 596
 597         state->fDoubleBuffer = (state->fMisaligned || state->fForceDoubleBuffer);
 598         state->fForceDoubleBuffer = false;
 599         if (state->fDoubleBuffer)
 600             state->fCopyPageCount = atop_64(round_page(state->fPreparedLength));
 601
 602         if (state->fCopyPageCount)
 603         {
 604             vm_page_t mapBase = NULL;
 605
 606             DEBG("preflight fCopyPageCount %d\n", state->fCopyPageCount);
 607
 608             if (!fMapper && !state->fDoubleBuffer)
 609             {
 610                 kern_return_t kr;
 611
 612                 if (fMapper) panic("fMapper copying");
 613
 614                 kr = vm_page_alloc_list(state->fCopyPageCount,
 615                                         KMA_LOMEM | KMA_NOPAGEWAIT, &mapBase);
 616                 if (KERN_SUCCESS != kr)
 617                 {
 618                     DEBG("vm_page_alloc_list(%d) failed (%d)\n", state->fCopyPageCount, kr);
 619                     mapBase = NULL;
 620                 }
 621             }
 622
 623             if (mapBase)
 624             {
 625                 state->fCopyPageAlloc = mapBase;
 626                 state->fCopyNext = state->fCopyPageAlloc;
 627                 offset = 0;
 628                 numSegments = 0-1;
 629                 ret = genIOVMSegments(op, segmentOp, (void *)(uintptr_t) op, &offset, state, &numSegments);
 630                 state->fPrepared = true;
 631                 op &= ~(kWalkSyncIn | kWalkSyncOut);
 632             }
 633             else
 634             {
 635                 DEBG("alloc IOBMD\n");
 636                 state->fCopyMD = createCopyBuffer(fMDSummary.fDirection, state->fPreparedLength);
 637
 638                 if (state->fCopyMD)
 639                 {
 640                     ret = kIOReturnSuccess;
 641                     state->fPrepared = true;
 642                 }
 643                 else
 644                 {
 645                     DEBG("IODMACommand !alloc IOBMD");
 646                     return (kIOReturnNoResources);
 647                 }
 648             }
 649         }
 650     }
 651
 652     if (state->fPrepared && ((kWalkSyncIn | kWalkSyncOut) & op))
 653     {
 654         if (state->fCopyPageCount)
 655         {
 656             DEBG("sync fCopyPageCount %d\n", state->fCopyPageCount);
 657
 658             if (state->fCopyPageAlloc)
 659             {
 660                 state->fCopyNext = state->fCopyPageAlloc;
 661                 offset = 0;
 662                 numSegments = 0-1;
 663                 ret = genIOVMSegments(op, segmentOp, (void *)(uintptr_t) op, &offset, state, &numSegments);
 664             }
 665             else if (state->fCopyMD)
 666             {
 667                 DEBG("sync IOBMD\n");
 668
 669                 if (SHOULD_COPY_DIR(op, fMDSummary.fDirection))
 670                 {
 671                     IOMemoryDescriptor *poMD = const_cast<IOMemoryDescriptor *>(fMemory);
 672
 673                     IOByteCount bytes;
 674
 675                     if (kWalkSyncIn & op)
 676                         bytes = poMD->writeBytes(state->fPreparedOffset,
 677                                                     state->fCopyMD->getBytesNoCopy(),
 678                                                     state->fPreparedLength);
 679                     else
 680                         bytes = poMD->readBytes(state->fPreparedOffset,
 681                                                     state->fCopyMD->getBytesNoCopy(),
 682                                                     state->fPreparedLength);
 683                     DEBG("fCopyMD %s %lx bytes\n", (kWalkSyncIn & op) ? "wrote" : "read", bytes);
 684                     ret = (bytes == state->fPreparedLength) ? kIOReturnSuccess : kIOReturnUnderrun;
 685                 }
 686                 else
 687                     ret = kIOReturnSuccess;
 688             }
 689         }
 690     }
 691
 692     if (kWalkComplete & op)
 693     {
 694         if (state->fCopyPageAlloc)
 695         {
 696             vm_page_free_list(state->fCopyPageAlloc, FALSE);
 697             state->fCopyPageAlloc = 0;
 698             state->fCopyPageCount = 0;
 699         }
 700         if (state->fCopyMD)
 701         {
 702             state->fCopyMD->release();
 703             state->fCopyMD = 0;
 704         }
 705
 706         state->fPrepared = false;
 707     }
 708     return (ret);
 709 }
 710
 711 UInt8
 712 IODMACommand::getNumAddressBits(void)
 713 {
 714     return (fNumAddressBits);
 715 }
 716
 717 UInt32
 718 IODMACommand::getAlignment(void)
 719 {
 720     return (fAlignMask + 1);
 721 }
 722
 723 uint32_t
 724 IODMACommand::getAlignmentLength(void)
 725 {
 726     return (fAlignMaskLength + 1);
 727 }
 728
 729 uint32_t
 730 IODMACommand::getAlignmentInternalSegments(void)
 731 {
 732     return (fAlignMaskInternalSegments + 1);
 733 }
 734
 735 IOReturn
 736 IODMACommand::prepareWithSpecification(SegmentFunction        outSegFunc,
 737                                        const SegmentOptions * segmentOptions,
 738                                        uint32_t               mappingOptions,
 739                                        IOMapper             * mapper,
 740                                        UInt64                 offset,
 741                                        UInt64                 length,
 742                                        bool                   flushCache,
 743                                        bool                   synchronize)
 744 {
 745     IOReturn ret;
 746
 747     if (fActive) return kIOReturnNotPermitted;
 748
 749     ret = setSpecification(outSegFunc, segmentOptions, mappingOptions, mapper);
 750     if (kIOReturnSuccess != ret) return (ret);
 751
 752     ret = prepare(offset, length, flushCache, synchronize);
 753
 754     return (ret);
 755 }
 756
 757 IOReturn
 758 IODMACommand::prepareWithSpecification(SegmentFunction  outSegFunc,
 759                                        UInt8            numAddressBits,
 760                                        UInt64           maxSegmentSize,
 761                                        MappingOptions   mappingOptions,
 762                                        UInt64           maxTransferSize,
 763                                        UInt32           alignment,
 764                                        IOMapper         *mapper,
 765                                        UInt64           offset,
 766                                        UInt64           length,
 767                                        bool             flushCache,
 768                                        bool             synchronize)
 769 {
 770     SegmentOptions segmentOptions =
 771     {
 772         .fStructSize                = sizeof(segmentOptions),
 773         .fNumAddressBits            = numAddressBits,
 774         .fMaxSegmentSize            = maxSegmentSize,
 775         .fMaxTransferSize           = maxTransferSize,
 776         .fAlignment                 = alignment,
 777         .fAlignmentLength           = 1,
 778         .fAlignmentInternalSegments = alignment
 779     };
 780
 781     return (prepareWithSpecification(outSegFunc, &segmentOptions, mappingOptions, mapper,
 782                                         offset, length, flushCache, synchronize));
 783 }
 784
 785
 786 IOReturn
 787 IODMACommand::prepare(UInt64 offset, UInt64 length, bool flushCache, bool synchronize)
 788 {
 789     IODMACommandInternal *  state = fInternalState;
 790     IOReturn                  ret = kIOReturnSuccess;
 791     uint32_t       mappingOptions = fMappingOptions;
 792
 793     // check specification has been set
 794     if (!fOutSeg) return (kIOReturnNotReady);
 795
 796     if (!length) length = fMDSummary.fLength;
 797
 798     if (length > fMaxTransferSize) return kIOReturnNoSpace;
 799
 800     if (fActive++)
 801     {
 802         if ((state->fPreparedOffset != offset)
 803           || (state->fPreparedLength != length))
 804         ret = kIOReturnNotReady;
 805     }
 806     else
 807     {
 808         if (fAlignMaskLength & length) return (kIOReturnNotAligned);
 809
 810         state->fPreparedOffset = offset;
 811         state->fPreparedLength = length;
 812
 813         state->fMapContig      = false;
 814         state->fMisaligned     = false;
 815         state->fDoubleBuffer   = false;
 816         state->fPrepared       = false;
 817         state->fCopyNext       = NULL;
 818         state->fCopyPageAlloc  = 0;
 819         state->fCopyPageCount  = 0;
 820         state->fNextRemapPage  = NULL;
 821         state->fCopyMD         = 0;
 822         state->fLocalMapperAlloc       = 0;
 823         state->fLocalMapperAllocValid  = false;
 824         state->fLocalMapperAllocLength = 0;
 825
 826         state->fLocalMapper    = (fMapper && (fMapper != IOMapper::gSystem));
 827
 828         state->fSourceAlignMask = fAlignMask;
 829         if (fMapper)
 830             state->fSourceAlignMask &= page_mask;
 831
 832         state->fCursor = state->fIterateOnly
 833                         || (!state->fCheckAddressing
 834                             && (!state->fSourceAlignMask
 835                                 || ((fMDSummary.fPageAlign & (1 << 31)) && (0 == (fMDSummary.fPageAlign & state->fSourceAlignMask)))));
 836
 837         if (!state->fCursor)
 838         {
 839             IOOptionBits op = kWalkPrepare | kWalkPreflight;
 840             if (synchronize)
 841                 op |= kWalkSyncOut;
 842             ret = walkAll(op);
 843         }
 844
 845         if (IS_NONCOHERENT(mappingOptions) && flushCache)
 846         {
 847             if (state->fCopyMD)
 848             {
 849                 state->fCopyMD->performOperation(kIOMemoryIncoherentIOStore, 0, length);
 850             }
 851             else
 852             {
 853                 IOMemoryDescriptor * md = const_cast<IOMemoryDescriptor *>(fMemory);
 854                 md->performOperation(kIOMemoryIncoherentIOStore, offset, length);
 855             }
 856         }
 857
 858         if (fMapper)
 859         {
 860             IOMDDMAMapArgs mapArgs;
 861             bzero(&mapArgs, sizeof(mapArgs));
 862             mapArgs.fMapper = fMapper;
 863             mapArgs.fCommand = this;
 864             mapArgs.fMapSpec.device         = state->fDevice;
 865             mapArgs.fMapSpec.alignment      = fAlignMask + 1;
 866             mapArgs.fMapSpec.numAddressBits = fNumAddressBits ? fNumAddressBits : 64;
 867             mapArgs.fLength = state->fPreparedLength;
 868             const IOMemoryDescriptor * md = state->fCopyMD;
 869             if (md) { mapArgs.fOffset = 0; } else
 870             {
 871                 md = fMemory;
 872                 mapArgs.fOffset = state->fPreparedOffset;
 873             }
 874             ret = md->dmaCommandOperation(kIOMDDMAMap | state->fIterateOnly, &mapArgs, sizeof(mapArgs));
 875 //IOLog("dma %p 0x%x 0x%qx-0x%qx 0x%qx-0x%qx\n", this, ret, state->fPreparedOffset, state->fPreparedLength, mapArgs.fAlloc, mapArgs.fAllocLength);
 876
 877             if (kIOReturnSuccess == ret)
 878             {
 879                 state->fLocalMapperAlloc       = mapArgs.fAlloc;
 880                 state->fLocalMapperAllocValid  = true;
 881                 state->fLocalMapperAllocLength = mapArgs.fAllocLength;
 882                 state->fMapContig = mapArgs.fMapContig;
 883             }
 884             if (NULL != IOMapper::gSystem) ret = kIOReturnSuccess;
 885         }
 886         if (kIOReturnSuccess == ret) state->fPrepared = true;
 887     }
 888     return ret;
 889 }
 890
 891 IOReturn
 892 IODMACommand::complete(bool invalidateCache, bool synchronize)
 893 {
 894     IODMACommandInternal * state = fInternalState;
 895     IOReturn               ret   = kIOReturnSuccess;
 896     IOMemoryDescriptor   * copyMD;
 897
 898     if (fActive < 1)
 899         return kIOReturnNotReady;
 900
 901     if (!--fActive)
 902     {
 903         copyMD = state->fCopyMD;
 904         if (copyMD) copyMD->retain();
 905
 906         if (IS_NONCOHERENT(fMappingOptions) && invalidateCache)
 907         {
 908             if (copyMD)
 909             {
 910                 copyMD->performOperation(kIOMemoryIncoherentIOFlush, 0, state->fPreparedLength);
 911             }
 912             else
 913             {
 914                 IOMemoryDescriptor * md = const_cast<IOMemoryDescriptor *>(fMemory);
 915                 md->performOperation(kIOMemoryIncoherentIOFlush, state->fPreparedOffset, state->fPreparedLength);
 916             }
 917         }
 918
 919         if (!state->fCursor)
 920         {
 921                 IOOptionBits op = kWalkComplete;
 922                 if (synchronize)
 923                         op |= kWalkSyncIn;
 924                 ret = walkAll(op);
 925         }
 926
 927         if (state->fLocalMapperAllocValid)
 928         {
 929             IOMDDMAMapArgs mapArgs;
 930             bzero(&mapArgs, sizeof(mapArgs));
 931             mapArgs.fMapper = fMapper;
 932             mapArgs.fCommand = this;
 933             mapArgs.fAlloc = state->fLocalMapperAlloc;
 934             mapArgs.fAllocLength = state->fLocalMapperAllocLength;
 935             const IOMemoryDescriptor * md = copyMD;
 936             if (md) { mapArgs.fOffset = 0; }
 937             else
 938             {
 939                 md = fMemory;
 940                 mapArgs.fOffset = state->fPreparedOffset;
 941             }
 942
 943             ret = md->dmaCommandOperation(kIOMDDMAUnmap, &mapArgs, sizeof(mapArgs));
 944
 945             state->fLocalMapperAlloc       = 0;
 946             state->fLocalMapperAllocValid  = false;
 947             state->fLocalMapperAllocLength = 0;
 948         }
 949         if (copyMD) copyMD->release();
 950         state->fPrepared = false;
 951     }
 952
 953     return ret;
 954 }
 955
 956 IOReturn
 957 IODMACommand::getPreparedOffsetAndLength(UInt64 * offset, UInt64 * length)
 958 {
 959     IODMACommandInternal * state = fInternalState;
 960     if (fActive < 1)
 961         return (kIOReturnNotReady);
 962
 963     if (offset)
 964         *offset = state->fPreparedOffset;
 965     if (length)
 966         *length = state->fPreparedLength;
 967
 968     return (kIOReturnSuccess);
 969 }
 970
 971 IOReturn
 972 IODMACommand::synchronize(IOOptionBits options)
 973 {
 974     IODMACommandInternal * state = fInternalState;
 975     IOReturn               ret   = kIOReturnSuccess;
 976     IOOptionBits           op;
 977
 978     if (kIODirectionOutIn == (kIODirectionOutIn & options))
 979         return kIOReturnBadArgument;
 980
 981     if (fActive < 1)
 982         return kIOReturnNotReady;
 983
 984     op = 0;
 985     if (kForceDoubleBuffer & options)
 986     {
 987         if (state->fDoubleBuffer) return kIOReturnSuccess;
 988         ret = complete(false /* invalidateCache */, true /* synchronize */);
 989         state->fCursor = false;
 990         state->fForceDoubleBuffer = true;
 991         ret = prepare(state->fPreparedOffset, state->fPreparedLength, false /* flushCache */, true /* synchronize */);
 992
 993         return (ret);
 994     }
 995     else if (state->fCursor)
 996         return kIOReturnSuccess;
 997
 998     if (kIODirectionIn & options)
 999         op |= kWalkSyncIn | kWalkSyncAlways;
1000     else if (kIODirectionOut & options)
1001         op |= kWalkSyncOut | kWalkSyncAlways;
1002
1003     ret = walkAll(op);
1004
1005     return ret;
1006 }
1007
1008 struct IODMACommandTransferContext
1009 {
1010     void *   buffer;
1011     UInt64   bufferOffset;
1012     UInt64   remaining;
1013     UInt32   op;
1014 };
1015 enum
1016 {
1017     kIODMACommandTransferOpReadBytes  = 1,
1018     kIODMACommandTransferOpWriteBytes = 2
1019 };
1020
1021 IOReturn
1022 IODMACommand::transferSegment(void   *reference,
1023                         IODMACommand *target,
1024                         Segment64     segment,
1025                         void         *segments,
1026                         UInt32        segmentIndex)
1027 {
1028     IODMACommandTransferContext * context = (IODMACommandTransferContext *) reference;
1029     UInt64   length  = min(segment.fLength, context->remaining);
1030     addr64_t ioAddr  = segment.fIOVMAddr;
1031     addr64_t cpuAddr = ioAddr;
1032
1033     context->remaining -= length;
1034
1035     while (length)
1036     {
1037         UInt64 copyLen = length;
1038         if ((kMapped == MAPTYPE(target->fMappingOptions))
1039             && target->fMapper)
1040         {
1041             cpuAddr = target->fMapper->mapToPhysicalAddress(ioAddr);
1042             copyLen = min(copyLen, page_size - (ioAddr & (page_size - 1)));
1043             ioAddr += copyLen;
1044         }
1045
1046         switch (context->op)
1047         {
1048             case kIODMACommandTransferOpReadBytes:
1049                 copypv(cpuAddr, context->bufferOffset + (addr64_t) context->buffer, copyLen,
1050                                     cppvPsrc | cppvNoRefSrc | cppvFsnk | cppvKmap);
1051                 break;
1052             case kIODMACommandTransferOpWriteBytes:
1053                 copypv(context->bufferOffset + (addr64_t) context->buffer, cpuAddr, copyLen,
1054                                 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
1055                 break;
1056         }
1057         length                -= copyLen;
1058         context->bufferOffset += copyLen;
1059     }
1060
1061     return (context->remaining ? kIOReturnSuccess : kIOReturnOverrun);
1062 }
1063
1064 UInt64
1065 IODMACommand::transfer(IOOptionBits transferOp, UInt64 offset, void * buffer, UInt64 length)
1066 {
1067     IODMACommandInternal *      state = fInternalState;
1068     IODMACommandTransferContext context;
1069     Segment64                   segments[1];
1070     UInt32                      numSegments = 0-1;
1071
1072     if (fActive < 1)
1073         return (0);
1074
1075     if (offset >= state->fPreparedLength)
1076         return (0);
1077     length = min(length, state->fPreparedLength - offset);
1078
1079     context.buffer       = buffer;
1080     context.bufferOffset = 0;
1081     context.remaining    = length;
1082     context.op           = transferOp;
1083     (void) genIOVMSegments(kWalkClient, transferSegment, &context, &offset, &segments[0], &numSegments);
1084
1085     return (length - context.remaining);
1086 }
1087
1088 UInt64
1089 IODMACommand::readBytes(UInt64 offset, void *bytes, UInt64 length)
1090 {
1091     return (transfer(kIODMACommandTransferOpReadBytes, offset, bytes, length));
1092 }
1093
1094 UInt64
1095 IODMACommand::writeBytes(UInt64 offset, const void *bytes, UInt64 length)
1096 {
1097     return (transfer(kIODMACommandTransferOpWriteBytes, offset, const_cast<void *>(bytes), length));
1098 }
1099
1100 IOReturn
1101 IODMACommand::genIOVMSegments(UInt64 *offsetP,
1102                               void   *segmentsP,
1103                               UInt32 *numSegmentsP)
1104 {
1105     return (genIOVMSegments(kWalkClient, clientOutputSegment, (void *) fOutSeg,
1106                             offsetP, segmentsP, numSegmentsP));
1107 }
1108
1109 IOReturn
1110 IODMACommand::genIOVMSegments(uint32_t op,
1111                               InternalSegmentFunction outSegFunc,
1112                               void   *reference,
1113                               UInt64 *offsetP,
1114                               void   *segmentsP,
1115                               UInt32 *numSegmentsP)
1116 {
1117     IODMACommandInternal * internalState = fInternalState;
1118     IOOptionBits           mdOp = kIOMDWalkSegments;
1119     IOReturn               ret  = kIOReturnSuccess;
1120
1121     if (!(kWalkComplete & op) && !fActive)
1122         return kIOReturnNotReady;
1123
1124     if (!offsetP || !segmentsP || !numSegmentsP || !*numSegmentsP)
1125         return kIOReturnBadArgument;
1126
1127     IOMDDMAWalkSegmentArgs *state =
1128         (IOMDDMAWalkSegmentArgs *)(void *) fState;
1129
1130     UInt64 offset    = *offsetP + internalState->fPreparedOffset;
1131     UInt64 memLength = internalState->fPreparedOffset + internalState->fPreparedLength;
1132
1133     if (offset >= memLength)
1134         return kIOReturnOverrun;
1135
1136     if ((offset == internalState->fPreparedOffset) || (offset != state->fOffset) || internalState->fNewMD) {
1137         state->fOffset                                   = 0;
1138         internalState->fIOVMAddrValid = state->fIOVMAddr = 0;
1139         internalState->fNextRemapPage                    = NULL;
1140         internalState->fNewMD                            = false;
1141         mdOp                                             = kIOMDFirstSegment;
1142         if (fMapper)
1143         {
1144             if (internalState->fLocalMapperAllocValid)
1145             {
1146                 state->fMapped = kIOMDDMAWalkMappedLocal;
1147                 state->fMappedBase = internalState->fLocalMapperAlloc;
1148             }
1149             else state->fMapped = true;
1150         }
1151     };
1152
1153     UInt32    segIndex = 0;
1154     UInt32    numSegments = *numSegmentsP;
1155     Segment64 curSeg = { 0, 0 };
1156     bool      curSegValid = false;
1157     addr64_t  maxPhys;
1158
1159     if (fNumAddressBits && (fNumAddressBits < 64))
1160         maxPhys = (1ULL << fNumAddressBits);
1161     else
1162         maxPhys = 0;
1163     maxPhys--;
1164
1165     while (internalState->fIOVMAddrValid || (state->fOffset < memLength))
1166     {
1167         // state = next seg
1168         if (!internalState->fIOVMAddrValid) {
1169
1170             IOReturn rtn;
1171
1172             state->fOffset = offset;
1173             state->fLength = memLength - offset;
1174
1175             if (internalState->fMapContig && internalState->fLocalMapperAllocValid)
1176             {
1177                 state->fIOVMAddr = internalState->fLocalMapperAlloc + offset - internalState->fPreparedOffset;
1178                 rtn = kIOReturnSuccess;
1179 #if 0
1180                 {
1181                     uint64_t checkOffset;
1182                     IOPhysicalLength segLen;
1183                     for (checkOffset = 0; checkOffset < state->fLength; )
1184                     {
1185                         addr64_t phys = const_cast<IOMemoryDescriptor *>(fMemory)->getPhysicalSegment(checkOffset + offset, &segLen, kIOMemoryMapperNone);
1186                         if (fMapper->mapAddr(state->fIOVMAddr + checkOffset) != phys)
1187                         {
1188                             panic("%llx != %llx:%llx, %llx phys: %llx %llx\n", offset,
1189                                     state->fIOVMAddr + checkOffset, fMapper->mapAddr(state->fIOVMAddr + checkOffset), state->fLength,
1190                                     phys, checkOffset);
1191                         }
1192                         checkOffset += page_size - (phys & page_mask);
1193                     }
1194                 }
1195 #endif
1196             }
1197             else
1198             {
1199                 const IOMemoryDescriptor * memory =
1200                     internalState->fCopyMD ? internalState->fCopyMD : fMemory;
1201                 rtn = memory->dmaCommandOperation(mdOp, fState, sizeof(fState));
1202                 mdOp = kIOMDWalkSegments;
1203             }
1204
1205             if (rtn == kIOReturnSuccess)
1206             {
1207                 internalState->fIOVMAddrValid = true;
1208                 assert(state->fLength);
1209                 if (curSegValid && ((curSeg.fIOVMAddr + curSeg.fLength) == state->fIOVMAddr)) {
1210                     UInt64 length = state->fLength;
1211                     offset          += length;
1212                     curSeg.fLength  += length;
1213                     internalState->fIOVMAddrValid = state->fIOVMAddr = 0;
1214                 }
1215             }
1216             else if (rtn == kIOReturnOverrun)
1217                 internalState->fIOVMAddrValid = state->fIOVMAddr = state->fLength = 0;  // At end
1218             else
1219                 return rtn;
1220         }
1221
1222         // seg = state, offset = end of seg
1223         if (!curSegValid)
1224         {
1225             UInt64 length                 = state->fLength;
1226             offset                       += length;
1227             curSeg.fIOVMAddr              = state->fIOVMAddr;
1228             curSeg.fLength                = length;
1229             curSegValid                   = true;
1230             internalState->fIOVMAddrValid = state->fIOVMAddr = 0;
1231         }
1232
1233         if (!internalState->fIOVMAddrValid)
1234         {
1235             // maxPhys
1236             if ((kWalkClient & op) && (curSeg.fIOVMAddr + curSeg.fLength - 1) > maxPhys)
1237             {
1238                 if (internalState->fCursor)
1239                 {
1240                     curSegValid = curSeg.fIOVMAddr = 0;
1241                     ret = kIOReturnMessageTooLarge;
1242                     break;
1243                 }
1244                 else if (curSeg.fIOVMAddr <= maxPhys)
1245                 {
1246                     UInt64 remain, newLength;
1247
1248                     newLength        = (maxPhys + 1 - curSeg.fIOVMAddr);
1249                     DEBG("trunc %qx, %qx-> %qx\n", curSeg.fIOVMAddr, curSeg.fLength, newLength);
1250                     remain           = curSeg.fLength - newLength;
1251                     state->fIOVMAddr = newLength + curSeg.fIOVMAddr;
1252                     internalState->fIOVMAddrValid = true;
1253                     curSeg.fLength   = newLength;
1254                     state->fLength   = remain;
1255                     offset          -= remain;
1256                 }
1257                 else
1258                 {
1259                     UInt64    addr = curSeg.fIOVMAddr;
1260                     ppnum_t   addrPage = atop_64(addr);
1261                     vm_page_t remap = NULL;
1262                     UInt64    remain, newLength;
1263
1264                     DEBG("sparse switch %qx, %qx ", addr, curSeg.fLength);
1265
1266                     remap = internalState->fNextRemapPage;
1267                     if (remap && (addrPage == vm_page_get_offset(remap)))
1268                     {
1269                     }
1270                     else for (remap = internalState->fCopyPageAlloc;
1271                                 remap && (addrPage != vm_page_get_offset(remap));
1272                                 remap = vm_page_get_next(remap))
1273                     {
1274                     }
1275
1276                     if (!remap) panic("no remap page found");
1277
1278                     curSeg.fIOVMAddr = ptoa_64(vm_page_get_phys_page(remap))
1279                                         + (addr & PAGE_MASK);
1280                     curSegValid = true;
1281                     internalState->fNextRemapPage = vm_page_get_next(remap);
1282
1283                     newLength            = PAGE_SIZE - (addr & PAGE_MASK);
1284                     if (newLength < curSeg.fLength)
1285                     {
1286                         remain           = curSeg.fLength - newLength;
1287                         state->fIOVMAddr = addr + newLength;
1288                         internalState->fIOVMAddrValid = true;
1289                         curSeg.fLength   = newLength;
1290                         state->fLength   = remain;
1291                         offset          -= remain;
1292                     }
1293                     DEBG("-> %qx, %qx offset %qx\n", curSeg.fIOVMAddr, curSeg.fLength, offset);
1294                 }
1295             }
1296
1297             // reduce size of output segment
1298             uint64_t reduce, leftover = 0;
1299
1300             // fMaxSegmentSize
1301             if (curSeg.fLength > fMaxSegmentSize)
1302             {
1303                 leftover      += curSeg.fLength - fMaxSegmentSize;
1304                 curSeg.fLength = fMaxSegmentSize;
1305                 state->fIOVMAddr = curSeg.fLength + curSeg.fIOVMAddr;
1306                 internalState->fIOVMAddrValid = true;
1307             }
1308
1309             // alignment current length
1310
1311             reduce = (curSeg.fLength & fAlignMaskLength);
1312             if (reduce && (curSeg.fLength > reduce))
1313             {
1314                 leftover       += reduce;
1315                 curSeg.fLength -= reduce;
1316                 state->fIOVMAddr = curSeg.fLength + curSeg.fIOVMAddr;
1317                 internalState->fIOVMAddrValid = true;
1318             }
1319
1320             // alignment next address
1321
1322             reduce = (state->fIOVMAddr & fAlignMaskInternalSegments);
1323             if (reduce && (curSeg.fLength > reduce))
1324             {
1325                 leftover       += reduce;
1326                 curSeg.fLength -= reduce;
1327                 state->fIOVMAddr = curSeg.fLength + curSeg.fIOVMAddr;
1328                 internalState->fIOVMAddrValid = true;
1329             }
1330
1331             if (leftover)
1332             {
1333                 DEBG("reduce seg by 0x%llx @ 0x%llx [0x%llx, 0x%llx]\n",
1334                       leftover, offset,
1335                       curSeg.fIOVMAddr, curSeg.fLength);
1336                 state->fLength   = leftover;
1337                 offset          -= leftover;
1338             }
1339
1340             //
1341
1342             if (internalState->fCursor)
1343             {
1344                 bool misaligned;
1345                 uint32_t mask;
1346
1347                 mask = (segIndex ? fAlignMaskInternalSegments : internalState->fSourceAlignMask);
1348                 misaligned = (0 != (mask & curSeg.fIOVMAddr));
1349                 if (!misaligned)
1350                 {
1351                     mask = fAlignMaskLength;
1352                     misaligned |= (0 != (mask &  curSeg.fLength));
1353                 }
1354                 if (misaligned)
1355                 {
1356                     if (misaligned) DEBG("cursor misaligned %qx:%qx\n", curSeg.fIOVMAddr, curSeg.fLength);
1357                     curSegValid = curSeg.fIOVMAddr = 0;
1358                     ret = kIOReturnNotAligned;
1359                     break;
1360                 }
1361             }
1362
1363             if (offset >= memLength)
1364             {
1365                 curSeg.fLength   -= (offset - memLength);
1366                 offset = memLength;
1367                 internalState->fIOVMAddrValid = state->fIOVMAddr = state->fLength = 0;  // At end
1368                 break;
1369             }
1370         }
1371
1372         if (internalState->fIOVMAddrValid) {
1373             if ((segIndex + 1 == numSegments))
1374                 break;
1375
1376             ret = (*outSegFunc)(reference, this, curSeg, segmentsP, segIndex++);
1377             curSegValid = curSeg.fIOVMAddr = 0;
1378             if (kIOReturnSuccess != ret)
1379                 break;
1380         }
1381     }
1382
1383     if (curSegValid) {
1384         ret = (*outSegFunc)(reference, this, curSeg, segmentsP, segIndex++);
1385     }
1386
1387     if (kIOReturnSuccess == ret)
1388     {
1389         state->fOffset = offset;
1390         *offsetP       = offset - internalState->fPreparedOffset;
1391         *numSegmentsP  = segIndex;
1392     }
1393     return ret;
1394 }
1395
1396 IOReturn
1397 IODMACommand::clientOutputSegment(
1398         void *reference, IODMACommand *target,
1399         Segment64 segment, void *vSegList, UInt32 outSegIndex)
1400 {
1401     SegmentFunction segmentFunction = (SegmentFunction) reference;
1402     IOReturn ret = kIOReturnSuccess;
1403
1404     if (target->fNumAddressBits && (target->fNumAddressBits < 64)
1405         && ((segment.fIOVMAddr + segment.fLength - 1) >> target->fNumAddressBits)
1406         && (target->reserved->fLocalMapperAllocValid || !target->fMapper))
1407     {
1408         DEBG("kIOReturnMessageTooLarge(fNumAddressBits) %qx, %qx\n", segment.fIOVMAddr, segment.fLength);
1409         ret = kIOReturnMessageTooLarge;
1410     }
1411
1412     if (!(*segmentFunction)(target, segment, vSegList, outSegIndex))
1413     {
1414         DEBG("kIOReturnMessageTooLarge(fOutSeg) %qx, %qx\n", segment.fIOVMAddr, segment.fLength);
1415         ret = kIOReturnMessageTooLarge;
1416     }
1417
1418     return (ret);
1419 }
1420
1421 IOReturn
1422 IODMACommand::genIOVMSegments(SegmentFunction segmentFunction,
1423                                     UInt64   *offsetP,
1424                                     void     *segmentsP,
1425                                     UInt32   *numSegmentsP)
1426 {
1427     return (genIOVMSegments(kWalkClient, clientOutputSegment, (void *) segmentFunction,
1428                             offsetP, segmentsP, numSegmentsP));
1429 }
1430
1431 bool
1432 IODMACommand::OutputHost32(IODMACommand *,
1433         Segment64 segment, void *vSegList, UInt32 outSegIndex)
1434 {
1435     Segment32 *base = (Segment32 *) vSegList;
1436     base[outSegIndex].fIOVMAddr = (UInt32) segment.fIOVMAddr;
1437     base[outSegIndex].fLength   = (UInt32) segment.fLength;
1438     return true;
1439 }
1440
1441 bool
1442 IODMACommand::OutputBig32(IODMACommand *,
1443         Segment64 segment, void *vSegList, UInt32 outSegIndex)
1444 {
1445     const UInt offAddr = outSegIndex * sizeof(Segment32);
1446     const UInt offLen  = offAddr + sizeof(UInt32);
1447     OSWriteBigInt32(vSegList, offAddr, (UInt32) segment.fIOVMAddr);
1448     OSWriteBigInt32(vSegList, offLen,  (UInt32) segment.fLength);
1449     return true;
1450 }
1451
1452 bool
1453 IODMACommand::OutputLittle32(IODMACommand *,
1454         Segment64 segment, void *vSegList, UInt32 outSegIndex)
1455 {
1456     const UInt offAddr = outSegIndex * sizeof(Segment32);
1457     const UInt offLen  = offAddr + sizeof(UInt32);
1458     OSWriteLittleInt32(vSegList, offAddr, (UInt32) segment.fIOVMAddr);
1459     OSWriteLittleInt32(vSegList, offLen,  (UInt32) segment.fLength);
1460     return true;
1461 }
1462
1463 bool
1464 IODMACommand::OutputHost64(IODMACommand *,
1465         Segment64 segment, void *vSegList, UInt32 outSegIndex)
1466 {
1467     Segment64 *base = (Segment64 *) vSegList;
1468     base[outSegIndex] = segment;
1469     return true;
1470 }
1471
1472 bool
1473 IODMACommand::OutputBig64(IODMACommand *,
1474         Segment64 segment, void *vSegList, UInt32 outSegIndex)
1475 {
1476     const UInt offAddr = outSegIndex * sizeof(Segment64);
1477     const UInt offLen  = offAddr + sizeof(UInt64);
1478     OSWriteBigInt64(vSegList, offAddr, (UInt64) segment.fIOVMAddr);
1479     OSWriteBigInt64(vSegList, offLen,  (UInt64) segment.fLength);
1480     return true;
1481 }
1482
1483 bool
1484 IODMACommand::OutputLittle64(IODMACommand *,
1485         Segment64 segment, void *vSegList, UInt32 outSegIndex)
1486 {
1487     const UInt offAddr = outSegIndex * sizeof(Segment64);
1488     const UInt offLen  = offAddr + sizeof(UInt64);
1489     OSWriteLittleInt64(vSegList, offAddr, (UInt64) segment.fIOVMAddr);
1490     OSWriteLittleInt64(vSegList, offLen,  (UInt64) segment.fLength);
1491     return true;
1492 }
1493
1494