X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/4452a7af2eac33dbad800bcc91f2399d62c18f53..04b8595b18b1b41ac7a206e4b3d51a635f8413d7:/iokit/Kernel/IODMACommand.cpp?ds=inline diff --git a/iokit/Kernel/IODMACommand.cpp b/iokit/Kernel/IODMACommand.cpp index 0c2956ce7..3b3c0ee3a 100644 --- a/iokit/Kernel/IODMACommand.cpp +++ b/iokit/Kernel/IODMACommand.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2005 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -30,6 +30,7 @@ #include #include +#include #include #include @@ -39,16 +40,12 @@ #include #include "IOKitKernelInternal.h" -#include "IOCopyMapper.h" #define MAPTYPE(type) ((UInt) (type) & kTypeMask) -#define IS_MAPPED(type) (MAPTYPE(type) == kMapped) +#define IS_MAPPED(type) (MAPTYPE(type) != kBypassed) #define IS_BYPASSED(type) (MAPTYPE(type) == kBypassed) #define IS_NONCOHERENT(type) (MAPTYPE(type) == kNonCoherent) - -static bool gIOEnableCopyMapper = true; - enum { kWalkSyncIn = 0x01, // bounce -> md @@ -61,30 +58,6 @@ enum kWalkClient = 0x80 }; -struct ExpansionData -{ - IOMDDMAWalkSegmentState fState; - IOMDDMACharacteristics fMDSummary; - - UInt64 fPreparedOffset; - UInt64 fPreparedLength; - - UInt8 fCursor; - UInt8 fCheckAddressing; - UInt8 fIterateOnly; - UInt8 fMisaligned; - UInt8 fCopyContig; - UInt8 fPrepared; - UInt8 fDoubleBuffer; - UInt8 __pad[1]; - - ppnum_t fCopyPageAlloc; - ppnum_t fCopyPageCount; - addr64_t fCopyNext; - - class IOBufferMemoryDescriptor * fCopyMD; -}; -typedef ExpansionData IODMACommandInternal; #define fInternalState reserved #define fState reserved->fState @@ -104,21 +77,20 @@ typedef ExpansionData IODMACommandInternal; #endif #if 0 -#define DEBG(fmt, args...) { kprintf(fmt, ## args); } +#define DEBG(fmt, args...) { IOLog(fmt, ## args); kprintf(fmt, ## args); } #else #define DEBG(fmt, args...) {} #endif - /**************************** class IODMACommand ***************************/ #undef super -#define super OSObject +#define super IOCommand OSDefineMetaClassAndStructors(IODMACommand, IOCommand); -OSMetaClassDefineReservedUnused(IODMACommand, 0); -OSMetaClassDefineReservedUnused(IODMACommand, 1); -OSMetaClassDefineReservedUnused(IODMACommand, 2); +OSMetaClassDefineReservedUsed(IODMACommand, 0); +OSMetaClassDefineReservedUsed(IODMACommand, 1); +OSMetaClassDefineReservedUsed(IODMACommand, 2); OSMetaClassDefineReservedUnused(IODMACommand, 3); OSMetaClassDefineReservedUnused(IODMACommand, 4); OSMetaClassDefineReservedUnused(IODMACommand, 5); @@ -176,7 +148,9 @@ IODMACommand::initWithSpecification(SegmentFunction outSegFunc, IOMapper *mapper, void *refCon) { - if (!super::init() || !outSegFunc || !numAddressBits) + IOService * device = 0; + + if (!super::init() || !outSegFunc) return false; bool is32Bit = (OutputHost32 == outSegFunc || OutputBig32 == outSegFunc @@ -197,6 +171,12 @@ IODMACommand::initWithSpecification(SegmentFunction outSegFunc, if (!maxTransferSize) maxTransferSize--; // Set Max transfer to -1 + + if (mapper && !OSDynamicCast(IOMapper, mapper)) + { + device = mapper; + mapper = 0; + } if (!mapper) { IOMapper::checkForSystemMapper(); @@ -219,7 +199,7 @@ IODMACommand::initWithSpecification(SegmentFunction outSegFunc, switch (MAPTYPE(mappingOptions)) { case kMapped: break; - case kNonCoherent: fMapper = 0; break; + case kNonCoherent: /*fMapper = 0;*/ break; case kBypassed: if (mapper && !mapper->getBypassMask(&fBypassMask)) return false; @@ -228,13 +208,17 @@ IODMACommand::initWithSpecification(SegmentFunction outSegFunc, return false; }; - reserved = IONew(ExpansionData, 1); + if (fMapper) + fMapper->retain(); + + reserved = IONew(IODMACommandInternal, 1); if (!reserved) return false; - bzero(reserved, sizeof(ExpansionData)); + bzero(reserved, sizeof(IODMACommandInternal)); fInternalState->fIterateOnly = (0 != (kIterateOnly & mappingOptions)); - + fInternalState->fDevice = device; + return true; } @@ -242,7 +226,10 @@ void IODMACommand::free() { if (reserved) - IODelete(reserved, ExpansionData, 1); + IODelete(reserved, IODMACommandInternal, 1); + + if (fMapper) + fMapper->release(); super::free(); } @@ -250,6 +237,8 @@ IODMACommand::free() IOReturn IODMACommand::setMemoryDescriptor(const IOMemoryDescriptor *mem, bool autoPrepare) { + IOReturn err = kIOReturnSuccess; + if (mem == fMemory) { if (!autoPrepare) @@ -267,34 +256,37 @@ IODMACommand::setMemoryDescriptor(const IOMemoryDescriptor *mem, bool autoPrepar if (fActive) return kIOReturnBusy; clearMemoryDescriptor(); - }; + } if (mem) { bzero(&fMDSummary, sizeof(fMDSummary)); - IOReturn rtn = mem->dmaCommandOperation( - kIOMDGetCharacteristics, - &fMDSummary, sizeof(fMDSummary)); - if (rtn) - return rtn; + err = mem->dmaCommandOperation(kIOMDGetCharacteristics | (kMapped == MAPTYPE(fMappingOptions)), + &fMDSummary, sizeof(fMDSummary)); + if (err) + return err; ppnum_t highPage = fMDSummary.fHighestPage ? fMDSummary.fHighestPage : gIOLastPage; if ((kMapped == MAPTYPE(fMappingOptions)) - && fMapper - && (!fNumAddressBits || (fNumAddressBits >= 31))) - // assuming mapped space is 2G + && fMapper) fInternalState->fCheckAddressing = false; else fInternalState->fCheckAddressing = (fNumAddressBits && (highPage >= (1UL << (fNumAddressBits - PAGE_SHIFT)))); + fInternalState->fNewMD = true; mem->retain(); fMemory = mem; - if (autoPrepare) - return prepare(); - }; - - return kIOReturnSuccess; + mem->dmaCommandOperation(kIOMDSetDMAActive, this, 0); + if (autoPrepare) { + err = prepare(); + if (err) { + clearMemoryDescriptor(); + } + } + } + + return err; } IOReturn @@ -306,6 +298,7 @@ IODMACommand::clearMemoryDescriptor(bool autoComplete) if (fMemory) { while (fActive) complete(); + fMemory->dmaCommandOperation(kIOMDSetDMAInactive, this, 0); fMemory->release(); fMemory = 0; } @@ -328,15 +321,14 @@ IODMACommand::segmentOp( void *segments, UInt32 segmentIndex) { - IOOptionBits op = (IOOptionBits) reference; + IOOptionBits op = (uintptr_t) reference; addr64_t maxPhys, address; - addr64_t remapAddr = 0; uint64_t length; uint32_t numPages; IODMACommandInternal * state = target->reserved; - if (target->fNumAddressBits && (target->fNumAddressBits < 64)) + if (target->fNumAddressBits && (target->fNumAddressBits < 64) && (state->fLocalMapperPageAlloc || !target->fMapper)) maxPhys = (1ULL << target->fNumAddressBits); else maxPhys = 0; @@ -350,8 +342,8 @@ IODMACommand::segmentOp( if (!state->fMisaligned) { - state->fMisaligned |= (0 != (target->fAlignMask & address)); - if (state->fMisaligned) DEBG("misaligned %qx:%qx, %lx\n", address, length, target->fAlignMask); + state->fMisaligned |= (0 != (state->fSourceAlignMask & address)); + if (state->fMisaligned) DEBG("misaligned %qx:%qx, %lx\n", address, length, state->fSourceAlignMask); } if (state->fMisaligned && (kWalkPreflight & op)) @@ -375,8 +367,7 @@ IODMACommand::segmentOp( if (!length) return (kIOReturnSuccess); - numPages = atop_64(round_page_64(length)); - remapAddr = state->fCopyNext; + numPages = atop_64(round_page_64((address & PAGE_MASK) + length)); if (kWalkPreflight & op) { @@ -384,35 +375,65 @@ IODMACommand::segmentOp( } else { + vm_page_t lastPage; + lastPage = NULL; if (kWalkPrepare & op) { + lastPage = state->fCopyNext; for (IOItemCount idx = 0; idx < numPages; idx++) - gIOCopyMapper->iovmInsert(atop_64(remapAddr), idx, atop_64(address) + idx); - } - if (state->fDoubleBuffer) - state->fCopyNext += length; - else - { - state->fCopyNext += round_page(length); - remapAddr += (address & PAGE_MASK); + { + vm_page_set_offset(lastPage, atop_64(address) + idx); + lastPage = vm_page_get_next(lastPage); + } } - if (SHOULD_COPY_DIR(op, target->fMDSummary.fDirection)) + if (!lastPage || SHOULD_COPY_DIR(op, target->fMDSummary.fDirection)) { - DEBG("cpv: 0x%qx %s 0x%qx, 0x%qx, 0x%02lx\n", remapAddr, - (kWalkSyncIn & op) ? "->" : "<-", - address, length, op); - if (kWalkSyncIn & op) - { // cppvNoModSnk - copypv(remapAddr, address, length, - cppvPsnk | cppvFsnk | cppvPsrc | cppvNoRefSrc ); - } - else + lastPage = state->fCopyNext; + for (IOItemCount idx = 0; idx < numPages; idx++) { - copypv(address, remapAddr, length, - cppvPsnk | cppvFsnk | cppvPsrc | cppvNoRefSrc ); + if (SHOULD_COPY_DIR(op, target->fMDSummary.fDirection)) + { + addr64_t cpuAddr = address; + addr64_t remapAddr; + uint64_t chunk; + + if ((kMapped == MAPTYPE(target->fMappingOptions)) + && target->fMapper) + { + cpuAddr = target->fMapper->mapAddr(address); + } + + remapAddr = ptoa_64(vm_page_get_phys_page(lastPage)); + if (!state->fDoubleBuffer) + { + remapAddr += (address & PAGE_MASK); + } + chunk = PAGE_SIZE - (address & PAGE_MASK); + if (chunk > length) + chunk = length; + + DEBG("cpv: 0x%qx %s 0x%qx, 0x%qx, 0x%02lx\n", remapAddr, + (kWalkSyncIn & op) ? "->" : "<-", + address, chunk, op); + + if (kWalkSyncIn & op) + { // cppvNoModSnk + copypv(remapAddr, cpuAddr, chunk, + cppvPsnk | cppvFsnk | cppvPsrc | cppvNoRefSrc ); + } + else + { + copypv(cpuAddr, remapAddr, chunk, + cppvPsnk | cppvFsnk | cppvPsrc | cppvNoRefSrc ); + } + address += chunk; + length -= chunk; + } + lastPage = vm_page_get_next(lastPage); } } + state->fCopyNext = lastPage; } return kIOReturnSuccess; @@ -427,22 +448,22 @@ IODMACommand::walkAll(UInt8 op) UInt32 numSegments; UInt64 offset; - if (gIOEnableCopyMapper && (kWalkPreflight & op)) + if (kWalkPreflight & op) { - state->fCopyContig = false; state->fMisaligned = false; state->fDoubleBuffer = false; state->fPrepared = false; - state->fCopyNext = 0; + state->fCopyNext = NULL; state->fCopyPageAlloc = 0; state->fCopyPageCount = 0; - state->fCopyMD = 0; + state->fNextRemapPage = NULL; + state->fCopyMD = 0; if (!(kWalkDoubleBuffer & op)) { offset = 0; numSegments = 0-1; - ret = genIOVMSegments(segmentOp, (void *) op, &offset, state, &numSegments); + ret = genIOVMSegments(op, segmentOp, (void *)(uintptr_t) op, &offset, state, &numSegments); } op &= ~kWalkPreflight; @@ -453,35 +474,41 @@ IODMACommand::walkAll(UInt8 op) if (state->fCopyPageCount) { - IOMapper * mapper; - ppnum_t mapBase = 0; + vm_page_t mapBase = NULL; DEBG("preflight fCopyPageCount %d\n", state->fCopyPageCount); - mapper = gIOCopyMapper; - if (mapper) - mapBase = mapper->iovmAlloc(state->fCopyPageCount); - if (mapBase) + if (!state->fDoubleBuffer) { - state->fCopyPageAlloc = mapBase; - if (state->fCopyPageAlloc && state->fDoubleBuffer) + kern_return_t kr; + + if (fMapper) panic("fMapper copying"); + + kr = vm_page_alloc_list(state->fCopyPageCount, + KMA_LOMEM | KMA_NOPAGEWAIT, &mapBase); + if (KERN_SUCCESS != kr) { - DEBG("contig copy map\n"); - state->fCopyContig = true; + DEBG("vm_page_alloc_list(%d) failed (%d)\n", state->fCopyPageCount, kr); + mapBase = NULL; } + } - state->fCopyNext = ptoa_64(state->fCopyPageAlloc); + if (mapBase) + { + state->fCopyPageAlloc = mapBase; + state->fCopyNext = state->fCopyPageAlloc; offset = 0; numSegments = 0-1; - ret = genIOVMSegments(segmentOp, (void *) op, &offset, state, &numSegments); + ret = genIOVMSegments(op, segmentOp, (void *)(uintptr_t) op, &offset, state, &numSegments); state->fPrepared = true; op &= ~(kWalkSyncIn | kWalkSyncOut); } else { DEBG("alloc IOBMD\n"); - state->fCopyMD = IOBufferMemoryDescriptor::withOptions( - fMDSummary.fDirection, state->fPreparedLength, page_size); + mach_vm_address_t mask = 0xFFFFF000; //state->fSourceAlignMask + state->fCopyMD = IOBufferMemoryDescriptor::inTaskWithPhysicalMask(kernel_task, + fMDSummary.fDirection, state->fPreparedLength, mask); if (state->fCopyMD) { @@ -490,14 +517,14 @@ IODMACommand::walkAll(UInt8 op) } else { - DEBG("IODMACommand !iovmAlloc"); + DEBG("IODMACommand !alloc IOBMD"); return (kIOReturnNoResources); } } } } - if (gIOEnableCopyMapper && state->fPrepared && ((kWalkSyncIn | kWalkSyncOut) & op)) + if (state->fPrepared && ((kWalkSyncIn | kWalkSyncOut) & op)) { if (state->fCopyPageCount) { @@ -505,10 +532,10 @@ IODMACommand::walkAll(UInt8 op) if (state->fCopyPageAlloc) { - state->fCopyNext = ptoa_64(state->fCopyPageAlloc); + state->fCopyNext = state->fCopyPageAlloc; offset = 0; numSegments = 0-1; - ret = genIOVMSegments(segmentOp, (void *) op, &offset, state, &numSegments); + ret = genIOVMSegments(op, segmentOp, (void *)(uintptr_t) op, &offset, state, &numSegments); } else if (state->fCopyMD) { @@ -541,7 +568,7 @@ IODMACommand::walkAll(UInt8 op) { if (state->fCopyPageAlloc) { - gIOCopyMapper->iovmFree(state->fCopyPageAlloc, state->fCopyPageCount); + vm_page_free_list(state->fCopyPageAlloc, FALSE); state->fCopyPageAlloc = 0; state->fCopyPageCount = 0; } @@ -556,11 +583,107 @@ IODMACommand::walkAll(UInt8 op) return (ret); } +UInt8 +IODMACommand::getNumAddressBits(void) +{ + return (fNumAddressBits); +} + +UInt32 +IODMACommand::getAlignment(void) +{ + return (fAlignMask + 1); +} + +IOReturn +IODMACommand::prepareWithSpecification(SegmentFunction outSegFunc, + UInt8 numAddressBits, + UInt64 maxSegmentSize, + MappingOptions mappingOptions, + UInt64 maxTransferSize, + UInt32 alignment, + IOMapper *mapper, + UInt64 offset, + UInt64 length, + bool flushCache, + bool synchronize) +{ + if (fActive) + return kIOReturnNotPermitted; + + if (!outSegFunc) + return kIOReturnBadArgument; + + bool is32Bit = (OutputHost32 == outSegFunc || OutputBig32 == outSegFunc + || OutputLittle32 == outSegFunc); + if (is32Bit) + { + if (!numAddressBits) + numAddressBits = 32; + else if (numAddressBits > 32) + return kIOReturnBadArgument; // Wrong output function for bits + } + + if (numAddressBits && (numAddressBits < PAGE_SHIFT)) + return kIOReturnBadArgument; + + if (!maxSegmentSize) + maxSegmentSize--; // Set Max segment to -1 + if (!maxTransferSize) + maxTransferSize--; // Set Max transfer to -1 + + if (mapper && !OSDynamicCast(IOMapper, mapper)) + { + fInternalState->fDevice = mapper; + mapper = 0; + } + if (!mapper) + { + IOMapper::checkForSystemMapper(); + mapper = IOMapper::gSystem; + } + + switch (MAPTYPE(mappingOptions)) + { + case kMapped: break; + case kNonCoherent: break; + case kBypassed: + if (mapper && !mapper->getBypassMask(&fBypassMask)) + return kIOReturnBadArgument; + break; + default: + return kIOReturnBadArgument; + }; + + fNumSegments = 0; + fBypassMask = 0; + fOutSeg = outSegFunc; + fNumAddressBits = numAddressBits; + fMaxSegmentSize = maxSegmentSize; + fMappingOptions = mappingOptions; + fMaxTransferSize = maxTransferSize; + if (!alignment) + alignment = 1; + fAlignMask = alignment - 1; + if (mapper != fMapper) + { + mapper->retain(); + fMapper->release(); + fMapper = mapper; + } + + fInternalState->fIterateOnly = (0 != (kIterateOnly & mappingOptions)); + + return prepare(offset, length, flushCache, synchronize); +} + + IOReturn IODMACommand::prepare(UInt64 offset, UInt64 length, bool flushCache, bool synchronize) { IODMACommandInternal * state = fInternalState; IOReturn ret = kIOReturnSuccess; + MappingOptions mappingOptions = fMappingOptions; if (!length) length = fMDSummary.fLength; @@ -568,13 +691,11 @@ IODMACommand::prepare(UInt64 offset, UInt64 length, bool flushCache, bool synchr if (length > fMaxTransferSize) return kIOReturnNoSpace; -#if 0 if (IS_NONCOHERENT(mappingOptions) && flushCache) { IOMemoryDescriptor *poMD = const_cast(fMemory); - poMD->performOperation(kIOMemoryIncoherentIOStore, 0, fMDSummary.fLength); + poMD->performOperation(kIOMemoryIncoherentIOStore, offset, length); } -#endif if (fActive++) { if ((state->fPreparedOffset != offset) @@ -586,19 +707,29 @@ IODMACommand::prepare(UInt64 offset, UInt64 length, bool flushCache, bool synchr state->fPreparedOffset = offset; state->fPreparedLength = length; - state->fCopyContig = false; + state->fMapContig = false; state->fMisaligned = false; state->fDoubleBuffer = false; state->fPrepared = false; - state->fCopyNext = 0; + state->fCopyNext = NULL; state->fCopyPageAlloc = 0; state->fCopyPageCount = 0; + state->fNextRemapPage = NULL; state->fCopyMD = 0; + state->fLocalMapperPageAlloc = 0; + state->fLocalMapperPageCount = 0; + state->fLocalMapper = (fMapper && (fMapper != IOMapper::gSystem)); + + state->fSourceAlignMask = fAlignMask; + if (fMapper) + state->fSourceAlignMask &= page_mask; + state->fCursor = state->fIterateOnly || (!state->fCheckAddressing - && (!fAlignMask - || ((fMDSummary.fPageAlign & (1 << 31)) && (0 == (fMDSummary.fPageAlign & fAlignMask))))); + && (!state->fSourceAlignMask + || ((fMDSummary.fPageAlign & (1 << 31)) && (0 == (fMDSummary.fPageAlign & state->fSourceAlignMask))))); + if (!state->fCursor) { IOOptionBits op = kWalkPrepare | kWalkPreflight; @@ -606,6 +737,45 @@ IODMACommand::prepare(UInt64 offset, UInt64 length, bool flushCache, bool synchr op |= kWalkSyncOut; ret = walkAll(op); } + + if (fMapper) + { + if (state->fLocalMapper) + { + state->fLocalMapperPageCount = atop_64(round_page( + state->fPreparedLength + ((state->fPreparedOffset + fMDSummary.fPageAlign) & page_mask))); + state->fLocalMapperPageAlloc = ptoa_64(fMapper->iovmAllocDMACommand(this, state->fLocalMapperPageCount)); + if (!state->fLocalMapperPageAlloc) + { + DEBG("IODMACommand !iovmAlloc"); + return (kIOReturnNoResources); + } + state->fMapContig = true; + } + else + { + IOMDDMAMapArgs mapArgs; + bzero(&mapArgs, sizeof(mapArgs)); + mapArgs.fMapper = fMapper; + mapArgs.fMapSpec.device = state->fDevice; + mapArgs.fMapSpec.alignment = fAlignMask + 1; + mapArgs.fMapSpec.numAddressBits = fNumAddressBits ? fNumAddressBits : 64; + mapArgs.fOffset = state->fPreparedOffset; + mapArgs.fLength = state->fPreparedLength; + const IOMemoryDescriptor * md = state->fCopyMD; + if (!md) md = fMemory; + ret = md->dmaCommandOperation(kIOMDDMAMap | state->fIterateOnly, &mapArgs, sizeof(mapArgs)); + if (kIOReturnSuccess == ret) + { + state->fLocalMapperPageAlloc = mapArgs.fAlloc; + state->fLocalMapperPageCount = mapArgs.fAllocCount; + state->fMapContig = mapArgs.fMapContig; + } + ret = kIOReturnSuccess; + } + } + + if (kIOReturnSuccess == ret) state->fPrepared = true; } @@ -625,27 +795,53 @@ IODMACommand::complete(bool invalidateCache, bool synchronize) { if (!state->fCursor) { - IOOptionBits op = kWalkComplete; - if (synchronize) - op |= kWalkSyncIn; - ret = walkAll(op); + IOOptionBits op = kWalkComplete; + if (synchronize) + op |= kWalkSyncIn; + ret = walkAll(op); } + if (state->fLocalMapperPageAlloc) + { + if (state->fLocalMapper) + { + fMapper->iovmFreeDMACommand(this, atop_64(state->fLocalMapperPageAlloc), state->fLocalMapperPageCount); + } + else if (state->fLocalMapperPageCount) + { + fMapper->iovmFree(atop_64(state->fLocalMapperPageAlloc), state->fLocalMapperPageCount); + } + state->fLocalMapperPageAlloc = 0; + state->fLocalMapperPageCount = 0; + } + state->fPrepared = false; -#if 0 if (IS_NONCOHERENT(fMappingOptions) && invalidateCache) { - // XXX gvdl: need invalidate before Chardonnay ships IOMemoryDescriptor *poMD = const_cast(fMemory); - poMD->performOperation(kIOMemoryIncoherentIOInvalidate, 0, fMDSummary.fLength); + poMD->performOperation(kIOMemoryIncoherentIOFlush, state->fPreparedOffset, state->fPreparedLength); } -#endif } return ret; } +IOReturn +IODMACommand::getPreparedOffsetAndLength(UInt64 * offset, UInt64 * length) +{ + IODMACommandInternal * state = fInternalState; + if (fActive < 1) + return (kIOReturnNotReady); + + if (offset) + *offset = state->fPreparedOffset; + if (length) + *length = state->fPreparedLength; + + return (kIOReturnSuccess); +} + IOReturn IODMACommand::synchronize(IOOptionBits options) { @@ -684,22 +880,115 @@ IODMACommand::synchronize(IOOptionBits options) return ret; } +struct IODMACommandTransferContext +{ + void * buffer; + UInt64 bufferOffset; + UInt64 remaining; + UInt32 op; +}; +enum +{ + kIODMACommandTransferOpReadBytes = 1, + kIODMACommandTransferOpWriteBytes = 2 +}; + +IOReturn +IODMACommand::transferSegment(void *reference, + IODMACommand *target, + Segment64 segment, + void *segments, + UInt32 segmentIndex) +{ + IODMACommandTransferContext * context = (IODMACommandTransferContext *) reference; + UInt64 length = min(segment.fLength, context->remaining); + addr64_t ioAddr = segment.fIOVMAddr; + addr64_t cpuAddr = ioAddr; + + context->remaining -= length; + + while (length) + { + UInt64 copyLen = length; + if ((kMapped == MAPTYPE(target->fMappingOptions)) + && target->fMapper) + { + cpuAddr = target->fMapper->mapAddr(ioAddr); + copyLen = min(copyLen, page_size - (ioAddr & (page_size - 1))); + ioAddr += copyLen; + } + + switch (context->op) + { + case kIODMACommandTransferOpReadBytes: + copypv(cpuAddr, context->bufferOffset + (addr64_t) context->buffer, copyLen, + cppvPsrc | cppvNoRefSrc | cppvFsnk | cppvKmap); + break; + case kIODMACommandTransferOpWriteBytes: + copypv(context->bufferOffset + (addr64_t) context->buffer, cpuAddr, copyLen, + cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap); + break; + } + length -= copyLen; + context->bufferOffset += copyLen; + } + + return (context->remaining ? kIOReturnSuccess : kIOReturnOverrun); +} + +UInt64 +IODMACommand::transfer(IOOptionBits transferOp, UInt64 offset, void * buffer, UInt64 length) +{ + IODMACommandInternal * state = fInternalState; + IODMACommandTransferContext context; + Segment64 segments[1]; + UInt32 numSegments = 0-1; + + if (fActive < 1) + return (0); + + if (offset >= state->fPreparedLength) + return (0); + length = min(length, state->fPreparedLength - offset); + + context.buffer = buffer; + context.bufferOffset = 0; + context.remaining = length; + context.op = transferOp; + (void) genIOVMSegments(kWalkClient, transferSegment, &context, &offset, &segments[0], &numSegments); + + return (length - context.remaining); +} + +UInt64 +IODMACommand::readBytes(UInt64 offset, void *bytes, UInt64 length) +{ + return (transfer(kIODMACommandTransferOpReadBytes, offset, bytes, length)); +} + +UInt64 +IODMACommand::writeBytes(UInt64 offset, const void *bytes, UInt64 length) +{ + return (transfer(kIODMACommandTransferOpWriteBytes, offset, const_cast(bytes), length)); +} + IOReturn IODMACommand::genIOVMSegments(UInt64 *offsetP, void *segmentsP, UInt32 *numSegmentsP) { - return (genIOVMSegments(clientOutputSegment, (void *) kWalkClient, offsetP, segmentsP, numSegmentsP)); + return (genIOVMSegments(kWalkClient, clientOutputSegment, (void *) fOutSeg, + offsetP, segmentsP, numSegmentsP)); } IOReturn -IODMACommand::genIOVMSegments(InternalSegmentFunction outSegFunc, +IODMACommand::genIOVMSegments(uint32_t op, + InternalSegmentFunction outSegFunc, void *reference, UInt64 *offsetP, void *segmentsP, UInt32 *numSegmentsP) { - IOOptionBits op = (IOOptionBits) reference; IODMACommandInternal * internalState = fInternalState; IOOptionBits mdOp = kIOMDWalkSegments; IOReturn ret = kIOReturnSuccess; @@ -711,19 +1000,21 @@ IODMACommand::genIOVMSegments(InternalSegmentFunction outSegFunc, return kIOReturnBadArgument; IOMDDMAWalkSegmentArgs *state = - (IOMDDMAWalkSegmentArgs *) fState; + (IOMDDMAWalkSegmentArgs *)(void *) fState; - UInt64 offset = *offsetP + internalState->fPreparedOffset; + UInt64 offset = *offsetP + internalState->fPreparedOffset; UInt64 memLength = internalState->fPreparedOffset + internalState->fPreparedLength; if (offset >= memLength) return kIOReturnOverrun; - if (!offset || offset != state->fOffset) { - state->fOffset = 0; - state->fIOVMAddr = 0; - state->fMapped = (IS_MAPPED(fMappingOptions) && fMapper); - mdOp = kIOMDFirstSegment; + if ((offset == internalState->fPreparedOffset) || (offset != state->fOffset) || internalState->fNewMD) { + state->fOffset = 0; + state->fIOVMAddr = 0; + internalState->fNextRemapPage = NULL; + internalState->fNewMD = false; + state->fMapped = (IS_MAPPED(fMappingOptions) && fMapper); + mdOp = kIOMDFirstSegment; }; UInt64 bypassMask = fBypassMask; @@ -738,20 +1029,37 @@ IODMACommand::genIOVMSegments(InternalSegmentFunction outSegFunc, maxPhys = 0; maxPhys--; - while ((state->fIOVMAddr) || state->fOffset < memLength) + while (state->fIOVMAddr || (state->fOffset < memLength)) { - if (!state->fIOVMAddr) { + // state = next seg + if (!state->fIOVMAddr) { IOReturn rtn; state->fOffset = offset; state->fLength = memLength - offset; - if (internalState->fCopyContig && (kWalkClient & op)) + if (internalState->fMapContig && internalState->fLocalMapperPageAlloc) { - state->fIOVMAddr = ptoa_64(internalState->fCopyPageAlloc) - + offset - internalState->fPreparedOffset; + state->fIOVMAddr = internalState->fLocalMapperPageAlloc + offset; rtn = kIOReturnSuccess; +#if 0 + { + uint64_t checkOffset; + IOPhysicalLength segLen; + for (checkOffset = 0; checkOffset < state->fLength; ) + { + addr64_t phys = const_cast(fMemory)->getPhysicalSegment(checkOffset + offset, &segLen, kIOMemoryMapperNone); + if (fMapper->mapAddr(state->fIOVMAddr + checkOffset) != phys) + { + panic("%llx != %llx:%llx, %llx phys: %llx %llx\n", offset, + state->fIOVMAddr + checkOffset, fMapper->mapAddr(state->fIOVMAddr + checkOffset), state->fLength, + phys, checkOffset); + } + checkOffset += page_size - (phys & page_mask); + } + } +#endif } else { @@ -761,71 +1069,90 @@ IODMACommand::genIOVMSegments(InternalSegmentFunction outSegFunc, mdOp = kIOMDWalkSegments; } - if (rtn == kIOReturnSuccess) { + if (rtn == kIOReturnSuccess) + { assert(state->fIOVMAddr); assert(state->fLength); + if ((curSeg.fIOVMAddr + curSeg.fLength) == state->fIOVMAddr) { + UInt64 length = state->fLength; + offset += length; + curSeg.fLength += length; + state->fIOVMAddr = 0; + } } else if (rtn == kIOReturnOverrun) state->fIOVMAddr = state->fLength = 0; // At end else return rtn; - }; - - if (!curSeg.fIOVMAddr) { - UInt64 length = state->fLength; + } - offset += length; - curSeg.fIOVMAddr = state->fIOVMAddr | bypassMask; - curSeg.fLength = length; - state->fIOVMAddr = 0; - } - else if ((curSeg.fIOVMAddr + curSeg.fLength == state->fIOVMAddr)) { + // seg = state, offset = end of seg + if (!curSeg.fIOVMAddr) + { UInt64 length = state->fLength; - offset += length; - curSeg.fLength += length; - state->fIOVMAddr = 0; - }; - + offset += length; + curSeg.fIOVMAddr = state->fIOVMAddr | bypassMask; + curSeg.fLength = length; + state->fIOVMAddr = 0; + } if (!state->fIOVMAddr) { - if (kWalkClient & op) + if ((kWalkClient & op) && (curSeg.fIOVMAddr + curSeg.fLength - 1) > maxPhys) { - if ((curSeg.fIOVMAddr + curSeg.fLength - 1) > maxPhys) + if (internalState->fCursor) { - if (internalState->fCursor) + curSeg.fIOVMAddr = 0; + ret = kIOReturnMessageTooLarge; + break; + } + else if (curSeg.fIOVMAddr <= maxPhys) + { + UInt64 remain, newLength; + + newLength = (maxPhys + 1 - curSeg.fIOVMAddr); + DEBG("trunc %qx, %qx-> %qx\n", curSeg.fIOVMAddr, curSeg.fLength, newLength); + remain = curSeg.fLength - newLength; + state->fIOVMAddr = newLength + curSeg.fIOVMAddr; + curSeg.fLength = newLength; + state->fLength = remain; + offset -= remain; + } + else + { + UInt64 addr = curSeg.fIOVMAddr; + ppnum_t addrPage = atop_64(addr); + vm_page_t remap = NULL; + UInt64 remain, newLength; + + DEBG("sparse switch %qx, %qx ", addr, curSeg.fLength); + + remap = internalState->fNextRemapPage; + if (remap && (addrPage == vm_page_get_offset(remap))) { - curSeg.fIOVMAddr = 0; - ret = kIOReturnMessageTooLarge; - break; } - else if (curSeg.fIOVMAddr <= maxPhys) + else for (remap = internalState->fCopyPageAlloc; + remap && (addrPage != vm_page_get_offset(remap)); + remap = vm_page_get_next(remap)) { - UInt64 remain, newLength; - - newLength = (maxPhys + 1 - curSeg.fIOVMAddr); - DEBG("trunc %qx, %qx-> %qx\n", curSeg.fIOVMAddr, curSeg.fLength, newLength); - remain = curSeg.fLength - newLength; - state->fIOVMAddr = newLength + curSeg.fIOVMAddr; - curSeg.fLength = newLength; - state->fLength = remain; - offset -= remain; } - else if (gIOCopyMapper) + + if (!remap) panic("no remap page found"); + + curSeg.fIOVMAddr = ptoa_64(vm_page_get_phys_page(remap)) + + (addr & PAGE_MASK); + internalState->fNextRemapPage = vm_page_get_next(remap); + + newLength = PAGE_SIZE - (addr & PAGE_MASK); + if (newLength < curSeg.fLength) { - DEBG("sparse switch %qx, %qx ", curSeg.fIOVMAddr, curSeg.fLength); - // Cache this! - for (UInt checkRemapIndex = 0; checkRemapIndex < internalState->fCopyPageCount; checkRemapIndex++) - { - if (trunc_page_64(curSeg.fIOVMAddr) == gIOCopyMapper->mapAddr( - ptoa_64(internalState->fCopyPageAlloc + checkRemapIndex))) - { - curSeg.fIOVMAddr = ptoa_64(internalState->fCopyPageAlloc + checkRemapIndex) + (curSeg.fIOVMAddr & PAGE_MASK); - break; - } - } - DEBG("-> %qx, %qx\n", curSeg.fIOVMAddr, curSeg.fLength); + remain = curSeg.fLength - newLength; + state->fIOVMAddr = addr + newLength; + curSeg.fLength = newLength; + state->fLength = remain; + offset -= remain; } + DEBG("-> %qx, %qx offset %qx\n", curSeg.fIOVMAddr, curSeg.fLength, offset); } } @@ -841,7 +1168,7 @@ IODMACommand::genIOVMSegments(InternalSegmentFunction outSegFunc, } if (internalState->fCursor - && (0 != (fAlignMask & curSeg.fIOVMAddr))) + && (0 != (internalState->fSourceAlignMask & curSeg.fIOVMAddr))) { curSeg.fIOVMAddr = 0; ret = kIOReturnNotAligned; @@ -886,16 +1213,18 @@ IODMACommand::clientOutputSegment( void *reference, IODMACommand *target, Segment64 segment, void *vSegList, UInt32 outSegIndex) { + SegmentFunction segmentFunction = (SegmentFunction) reference; IOReturn ret = kIOReturnSuccess; - if ((target->fNumAddressBits < 64) - && ((segment.fIOVMAddr + segment.fLength - 1) >> target->fNumAddressBits)) + if (target->fNumAddressBits && (target->fNumAddressBits < 64) + && ((segment.fIOVMAddr + segment.fLength - 1) >> target->fNumAddressBits) + && (target->reserved->fLocalMapperPageAlloc || !target->fMapper)) { DEBG("kIOReturnMessageTooLarge(fNumAddressBits) %qx, %qx\n", segment.fIOVMAddr, segment.fLength); ret = kIOReturnMessageTooLarge; } - if (!(*target->fOutSeg)(target, segment, vSegList, outSegIndex)) + if (!(*segmentFunction)(target, segment, vSegList, outSegIndex)) { DEBG("kIOReturnMessageTooLarge(fOutSeg) %qx, %qx\n", segment.fIOVMAddr, segment.fLength); ret = kIOReturnMessageTooLarge; @@ -904,6 +1233,16 @@ IODMACommand::clientOutputSegment( return (ret); } +IOReturn +IODMACommand::genIOVMSegments(SegmentFunction segmentFunction, + UInt64 *offsetP, + void *segmentsP, + UInt32 *numSegmentsP) +{ + return (genIOVMSegments(kWalkClient, clientOutputSegment, (void *) segmentFunction, + offsetP, segmentsP, numSegmentsP)); +} + bool IODMACommand::OutputHost32(IODMACommand *, Segment64 segment, void *vSegList, UInt32 outSegIndex)