-IODMACommand::genIOVMSegments(InternalSegmentFunction outSegFunc,
- void *reference,
- UInt64 *offsetP,
- void *segmentsP,
- UInt32 *numSegmentsP)
-{
- IOOptionBits op = (IOOptionBits) reference;
- IODMACommandInternal * internalState = fInternalState;
- IOOptionBits mdOp = kIOMDWalkSegments;
- IOReturn ret = kIOReturnSuccess;
-
- if (!(kWalkComplete & op) && !fActive)
- return kIOReturnNotReady;
-
- if (!offsetP || !segmentsP || !numSegmentsP || !*numSegmentsP)
- return kIOReturnBadArgument;
-
- IOMDDMAWalkSegmentArgs *state =
- (IOMDDMAWalkSegmentArgs *) fState;
-
- UInt64 offset = *offsetP + internalState->fPreparedOffset;
- UInt64 memLength = internalState->fPreparedOffset + internalState->fPreparedLength;
-
- if (offset >= memLength)
- return kIOReturnOverrun;
-
- if (!offset || offset != state->fOffset) {
- state->fOffset = 0;
- state->fIOVMAddr = 0;
- state->fMapped = (IS_MAPPED(fMappingOptions) && fMapper);
- mdOp = kIOMDFirstSegment;
- };
-
- UInt64 bypassMask = fBypassMask;
- UInt32 segIndex = 0;
- UInt32 numSegments = *numSegmentsP;
- Segment64 curSeg = { 0, 0 };
- addr64_t maxPhys;
-
- if (fNumAddressBits && (fNumAddressBits < 64))
- maxPhys = (1ULL << fNumAddressBits);
- else
- maxPhys = 0;
- maxPhys--;
-
- while ((state->fIOVMAddr) || state->fOffset < memLength)
- {
- if (!state->fIOVMAddr) {
-
- IOReturn rtn;
-
- state->fOffset = offset;
- state->fLength = memLength - offset;
-
- if (internalState->fCopyContig && (kWalkClient & op))
- {
- state->fIOVMAddr = ptoa_64(internalState->fCopyPageAlloc)
- + offset - internalState->fPreparedOffset;
- rtn = kIOReturnSuccess;
- }
- else
- {
- const IOMemoryDescriptor * memory =
- internalState->fCopyMD ? internalState->fCopyMD : fMemory;
- rtn = memory->dmaCommandOperation(mdOp, fState, sizeof(fState));
- mdOp = kIOMDWalkSegments;
- }
-
- if (rtn == kIOReturnSuccess) {
- assert(state->fIOVMAddr);
- assert(state->fLength);
- }
- else if (rtn == kIOReturnOverrun)
- state->fIOVMAddr = state->fLength = 0; // At end
- else
- return rtn;
- };
-
- if (!curSeg.fIOVMAddr) {
- UInt64 length = state->fLength;
-
- offset += length;
- curSeg.fIOVMAddr = state->fIOVMAddr | bypassMask;
- curSeg.fLength = length;
- state->fIOVMAddr = 0;
- }
- else if ((curSeg.fIOVMAddr + curSeg.fLength == state->fIOVMAddr)) {
- UInt64 length = state->fLength;
- offset += length;
- curSeg.fLength += length;
- state->fIOVMAddr = 0;
- };
-
-
- if (!state->fIOVMAddr)
- {
- if (kWalkClient & op)
- {
- if ((curSeg.fIOVMAddr + curSeg.fLength - 1) > maxPhys)
- {
- if (internalState->fCursor)
- {
- curSeg.fIOVMAddr = 0;
- ret = kIOReturnMessageTooLarge;
- break;
- }
- else if (curSeg.fIOVMAddr <= maxPhys)
- {
- UInt64 remain, newLength;
-
- newLength = (maxPhys + 1 - curSeg.fIOVMAddr);
- DEBG("trunc %qx, %qx-> %qx\n", curSeg.fIOVMAddr, curSeg.fLength, newLength);
- remain = curSeg.fLength - newLength;
- state->fIOVMAddr = newLength + curSeg.fIOVMAddr;
- curSeg.fLength = newLength;
- state->fLength = remain;
- offset -= remain;
- }
- else if (gIOCopyMapper)
- {
- DEBG("sparse switch %qx, %qx ", curSeg.fIOVMAddr, curSeg.fLength);
- // Cache this!
- for (UInt checkRemapIndex = 0; checkRemapIndex < internalState->fCopyPageCount; checkRemapIndex++)
- {
- if (trunc_page_64(curSeg.fIOVMAddr) == gIOCopyMapper->mapAddr(
- ptoa_64(internalState->fCopyPageAlloc + checkRemapIndex)))
- {
- curSeg.fIOVMAddr = ptoa_64(internalState->fCopyPageAlloc + checkRemapIndex) + (curSeg.fIOVMAddr & PAGE_MASK);
+IODMACommand::genIOVMSegments(uint32_t op,
+ InternalSegmentFunction outSegFunc,
+ void *reference,
+ UInt64 *offsetP,
+ void *segmentsP,
+ UInt32 *numSegmentsP)
+{
+ IODMACommandInternal * internalState = fInternalState;
+ IOOptionBits mdOp = kIOMDWalkSegments;
+ IOReturn ret = kIOReturnSuccess;
+
+ if (!(kWalkComplete & op) && !fActive) {
+ return kIOReturnNotReady;
+ }
+
+ if (!offsetP || !segmentsP || !numSegmentsP || !*numSegmentsP) {
+ return kIOReturnBadArgument;
+ }
+
+ IOMDDMAWalkSegmentArgs *state =
+ (IOMDDMAWalkSegmentArgs *)(void *) fState;
+
+ UInt64 offset = *offsetP + internalState->fPreparedOffset;
+ UInt64 memLength = internalState->fPreparedOffset + internalState->fPreparedLength;
+
+ if (offset >= memLength) {
+ return kIOReturnOverrun;
+ }
+
+ if ((offset == internalState->fPreparedOffset) || (offset != state->fOffset) || internalState->fNewMD) {
+ state->fOffset = 0;
+ internalState->fIOVMAddrValid = state->fIOVMAddr = 0;
+ internalState->fNextRemapPage = NULL;
+ internalState->fNewMD = false;
+ mdOp = kIOMDFirstSegment;
+ if (fMapper) {
+ if (internalState->fLocalMapperAllocValid) {
+ state->fMapped = true;
+ state->fMappedBase = internalState->fLocalMapperAlloc;
+ } else {
+ state->fMapped = false;
+ }
+ }
+ }
+
+ UInt32 segIndex = 0;
+ UInt32 numSegments = *numSegmentsP;
+ Segment64 curSeg = { 0, 0 };
+ bool curSegValid = false;
+ addr64_t maxPhys;
+
+ if (fNumAddressBits && (fNumAddressBits < 64)) {
+ maxPhys = (1ULL << fNumAddressBits);
+ } else {
+ maxPhys = 0;
+ }
+ maxPhys--;
+
+ while (internalState->fIOVMAddrValid || (state->fOffset < memLength)) {
+ // state = next seg
+ if (!internalState->fIOVMAddrValid) {
+ IOReturn rtn;
+
+ state->fOffset = offset;
+ state->fLength = memLength - offset;
+
+ bool done = false;
+ bool check = false;
+
+ if (internalState->fLocalMapperAllocValid) {
+ if (!internalState->fMapSegmentsCount) {
+ state->fIOVMAddr = internalState->fLocalMapperAlloc + offset - internalState->fPreparedOffset;
+ rtn = kIOReturnSuccess;
+ done = true;
+ check = true;
+ } else {
+ uint64_t address;
+ uint64_t length;
+ uint64_t runOffset;
+ uint64_t ind;
+ uint64_t off2Ind = internalState->fOffset2Index;
+
+ // Validate the previous offset
+ if (offset
+ && (offset == internalState->fNextOffset || off2Ind <= offset)) {
+ ind = internalState->fIndex;
+ } else {
+ ind = off2Ind = 0; // Start from beginning
+ }
+#if defined(LOGTAG)
+ if (LOGTAG == fMemory->getTag()) {
+ IOLog("DMA[%p] offsets 0x%qx, 0x%qx, 0x%qx ind %qd\n", this, offset, internalState->fPreparedOffset, internalState->fNextOffset, ind);
+ }
+#endif /* defined(LOGTAG) */
+
+ // Scan through iopl info blocks looking for block containing offset
+ while (ind < internalState->fMapSegmentsCount && offset >= internalState->fMapSegments[ind].fDMAOffset) {
+ ind++;
+ }
+ if (ind < internalState->fMapSegmentsCount) {
+ length = internalState->fMapSegments[ind].fDMAOffset;
+ } else {
+ length = memLength;
+ }
+ length -= offset; // Remainder within iopl
+
+ // Go back to actual range as search goes past it
+ ind--;
+ off2Ind = internalState->fMapSegments[ind].fDMAOffset;
+
+ // Subtract offset till this iopl in total list
+ runOffset = offset - off2Ind;
+
+ // Compute an offset relative to the mapped base
+
+ runOffset += internalState->fMapSegments[ind].fPageOffset;
+ address = internalState->fLocalMapperAllocBase + internalState->fMapSegments[ind].fMapOffset + runOffset;
+#if defined(LOGTAG)
+ if (LOGTAG == fMemory->getTag()) {
+ IOLog("DMA[%p] addrlen 0x%qx, 0x%qx\n", this, address, length);
+ }
+#endif /* defined(LOGTAG) */
+
+ state->fIOVMAddr = address;
+ state->fLength = length;
+
+ internalState->fIndex = ind;
+ internalState->fOffset2Index = off2Ind;
+ internalState->fNextOffset = state->fOffset + length;
+
+ rtn = kIOReturnSuccess;
+ done = true;
+ check = true;
+ }
+ }
+
+ if (!done) {
+ IOMemoryDescriptor * memory =
+ internalState->fCopyMD ? internalState->fCopyMD.get() : fMemory.get();
+ rtn = memory->dmaCommandOperation(mdOp, fState, sizeof(fState));
+ mdOp = kIOMDWalkSegments;
+ }
+#if 0
+ if (check
+ && !ml_at_interrupt_context()
+ && (rtn == kIOReturnSuccess)
+ && fMapper
+ && strcmp("AppleNVMeMMU", fMapper->getName())) {
+ uint64_t checkOffset;
+ IOPhysicalLength segLen;
+ IOMemoryDescriptor * memory =
+ internalState->fCopyMD ? internalState->fCopyMD.get() : fMemory.get();
+ for (checkOffset = 0; checkOffset < state->fLength;) {
+ addr64_t phys = memory->getPhysicalSegment(offset + checkOffset, &segLen, kIOMemoryMapperNone);
+ addr64_t mapperPhys;
+
+ mapperPhys = fMapper->mapToPhysicalAddress(state->fIOVMAddr + checkOffset);
+ mapperPhys |= (phys & (fMapper->getPageSize() - 1));
+ if (mapperPhys != phys) {
+ panic("DMA[%p] mismatch at offset %llx + %llx, dma %llx mapperPhys %llx != %llx, len %llx\n",
+ this, offset, checkOffset,
+ state->fIOVMAddr + checkOffset, mapperPhys, phys, state->fLength);
+ }
+ checkOffset += page_size - (phys & page_mask);
+ }
+ }
+#endif
+ if (rtn == kIOReturnSuccess) {
+ internalState->fIOVMAddrValid = true;
+ assert(state->fLength);
+ if (curSegValid && ((curSeg.fIOVMAddr + curSeg.fLength) == state->fIOVMAddr)) {
+ UInt64 length = state->fLength;
+ offset += length;
+ curSeg.fLength += length;
+ internalState->fIOVMAddrValid = state->fIOVMAddr = 0;
+ }
+ } else if (rtn == kIOReturnOverrun) {
+ internalState->fIOVMAddrValid = state->fIOVMAddr = state->fLength = 0; // At end
+ } else {
+ return rtn;
+ }
+ }
+
+ // seg = state, offset = end of seg
+ if (!curSegValid) {
+ UInt64 length = state->fLength;
+ offset += length;
+ curSeg.fIOVMAddr = state->fIOVMAddr;
+ curSeg.fLength = length;
+ curSegValid = true;
+ internalState->fIOVMAddrValid = state->fIOVMAddr = 0;
+ }
+
+ if (!internalState->fIOVMAddrValid) {
+ // maxPhys
+ if ((kWalkClient & op) && (curSeg.fIOVMAddr + curSeg.fLength - 1) > maxPhys) {
+ if (internalState->fCursor) {
+ curSegValid = curSeg.fIOVMAddr = 0;
+ ret = kIOReturnMessageTooLarge;
+ break;
+ } else if (curSeg.fIOVMAddr <= maxPhys) {
+ UInt64 remain, newLength;
+
+ newLength = (maxPhys + 1 - curSeg.fIOVMAddr);
+ DEBG("trunc %qx, %qx-> %qx\n", curSeg.fIOVMAddr, curSeg.fLength, newLength);
+ remain = curSeg.fLength - newLength;
+ state->fIOVMAddr = newLength + curSeg.fIOVMAddr;
+ internalState->fIOVMAddrValid = true;
+ curSeg.fLength = newLength;
+ state->fLength = remain;
+ offset -= remain;
+ } else {
+ UInt64 addr = curSeg.fIOVMAddr;
+ ppnum_t addrPage = (ppnum_t) atop_64(addr);
+ vm_page_t remap = NULL;
+ UInt64 remain, newLength;
+
+ DEBG("sparse switch %qx, %qx ", addr, curSeg.fLength);
+
+ remap = internalState->fNextRemapPage;
+ if (remap && (addrPage == vm_page_get_offset(remap))) {
+ } else {
+ for (remap = internalState->fCopyPageAlloc;
+ remap && (addrPage != vm_page_get_offset(remap));
+ remap = vm_page_get_next(remap)) {
+ }
+ }
+
+ if (!remap) {
+ panic("no remap page found");
+ }
+
+ curSeg.fIOVMAddr = ptoa_64(vm_page_get_phys_page(remap))
+ + (addr & PAGE_MASK);
+ curSegValid = true;
+ internalState->fNextRemapPage = vm_page_get_next(remap);
+
+ newLength = PAGE_SIZE - (addr & PAGE_MASK);
+ if (newLength < curSeg.fLength) {
+ remain = curSeg.fLength - newLength;
+ state->fIOVMAddr = addr + newLength;
+ internalState->fIOVMAddrValid = true;
+ curSeg.fLength = newLength;
+ state->fLength = remain;
+ offset -= remain;
+ }
+ DEBG("-> %qx, %qx offset %qx\n", curSeg.fIOVMAddr, curSeg.fLength, offset);
+ }
+ }
+
+ // reduce size of output segment
+ uint64_t reduce, leftover = 0;
+
+ // fMaxSegmentSize
+ if (curSeg.fLength > fMaxSegmentSize) {
+ leftover += curSeg.fLength - fMaxSegmentSize;
+ curSeg.fLength = fMaxSegmentSize;
+ state->fIOVMAddr = curSeg.fLength + curSeg.fIOVMAddr;
+ internalState->fIOVMAddrValid = true;
+ }
+
+ // alignment current length
+
+ reduce = (curSeg.fLength & fAlignMaskLength);
+ if (reduce && (curSeg.fLength > reduce)) {
+ leftover += reduce;
+ curSeg.fLength -= reduce;
+ state->fIOVMAddr = curSeg.fLength + curSeg.fIOVMAddr;
+ internalState->fIOVMAddrValid = true;
+ }
+
+ // alignment next address
+
+ reduce = (state->fIOVMAddr & fAlignMaskInternalSegments);
+ if (reduce && (curSeg.fLength > reduce)) {
+ leftover += reduce;
+ curSeg.fLength -= reduce;
+ state->fIOVMAddr = curSeg.fLength + curSeg.fIOVMAddr;
+ internalState->fIOVMAddrValid = true;
+ }
+
+ if (leftover) {
+ DEBG("reduce seg by 0x%llx @ 0x%llx [0x%llx, 0x%llx]\n",
+ leftover, offset,
+ curSeg.fIOVMAddr, curSeg.fLength);
+ state->fLength = leftover;
+ offset -= leftover;
+ }
+
+ //
+
+ if (internalState->fCursor) {
+ bool misaligned;
+ uint32_t mask;
+
+ mask = (segIndex ? fAlignMaskInternalSegments : internalState->fSourceAlignMask);
+ misaligned = (0 != (mask & curSeg.fIOVMAddr));
+ if (!misaligned) {
+ mask = fAlignMaskLength;
+ misaligned |= (0 != (mask & curSeg.fLength));
+ }
+ if (misaligned) {
+ if (misaligned) {
+ DEBG("cursor misaligned %qx:%qx\n", curSeg.fIOVMAddr, curSeg.fLength);
+ }
+ curSegValid = curSeg.fIOVMAddr = 0;
+ ret = kIOReturnNotAligned;
+ break;
+ }
+ }
+
+ if (offset >= memLength) {
+ curSeg.fLength -= (offset - memLength);
+ offset = memLength;
+ internalState->fIOVMAddrValid = state->fIOVMAddr = state->fLength = 0; // At end
+ break;
+ }
+ }
+
+ if (internalState->fIOVMAddrValid) {
+ if ((segIndex + 1 == numSegments)) {
projects / apple / xnu.git / blobdiff