#include <libkern/OSTypes.h>
#include <libkern/OSByteOrder.h>
+#include <libkern/OSDebug.h>
#include <IOKit/IOReturn.h>
#include <IOKit/IOLib.h>
#include "IOKitKernelInternal.h"
#define MAPTYPE(type) ((UInt) (type) & kTypeMask)
-#define IS_MAPPED(type) (MAPTYPE(type) == kMapped)
+#define IS_MAPPED(type) (MAPTYPE(type) != kBypassed)
#define IS_BYPASSED(type) (MAPTYPE(type) == kBypassed)
#define IS_NONCOHERENT(type) (MAPTYPE(type) == kNonCoherent)
/**************************** class IODMACommand ***************************/
#undef super
-#define super OSObject
+#define super IOCommand
OSDefineMetaClassAndStructors(IODMACommand, IOCommand);
OSMetaClassDefineReservedUsed(IODMACommand, 0);
IOMapper *mapper,
void *refCon)
{
- if (!super::init() || !outSegFunc || !numAddressBits)
+ IOService * device = 0;
+
+ if (!super::init() || !outSegFunc)
return false;
bool is32Bit = (OutputHost32 == outSegFunc || OutputBig32 == outSegFunc
if (!maxTransferSize)
maxTransferSize--; // Set Max transfer to -1
+
+ if (mapper && !OSDynamicCast(IOMapper, mapper))
+ {
+ device = mapper;
+ mapper = 0;
+ }
if (!mapper)
{
IOMapper::checkForSystemMapper();
switch (MAPTYPE(mappingOptions))
{
case kMapped: break;
- case kNonCoherent: fMapper = 0; break;
+ case kNonCoherent: /*fMapper = 0;*/ break;
case kBypassed:
if (mapper && !mapper->getBypassMask(&fBypassMask))
return false;
bzero(reserved, sizeof(IODMACommandInternal));
fInternalState->fIterateOnly = (0 != (kIterateOnly & mappingOptions));
-
+ fInternalState->fDevice = device;
+
return true;
}
IOReturn
IODMACommand::setMemoryDescriptor(const IOMemoryDescriptor *mem, bool autoPrepare)
{
+ IOReturn err = kIOReturnSuccess;
+
if (mem == fMemory)
{
if (!autoPrepare)
if (fActive)
return kIOReturnBusy;
clearMemoryDescriptor();
- };
+ }
if (mem) {
bzero(&fMDSummary, sizeof(fMDSummary));
- IOReturn rtn = mem->dmaCommandOperation(
- kIOMDGetCharacteristics,
- &fMDSummary, sizeof(fMDSummary));
- if (rtn)
- return rtn;
+ err = mem->dmaCommandOperation(kIOMDGetCharacteristics | (kMapped == MAPTYPE(fMappingOptions)),
+ &fMDSummary, sizeof(fMDSummary));
+ if (err)
+ return err;
ppnum_t highPage = fMDSummary.fHighestPage ? fMDSummary.fHighestPage : gIOLastPage;
if ((kMapped == MAPTYPE(fMappingOptions))
- && fMapper
- && (!fNumAddressBits || (fNumAddressBits >= 31)))
- // assuming mapped space is 2G
+ && fMapper)
fInternalState->fCheckAddressing = false;
else
fInternalState->fCheckAddressing = (fNumAddressBits && (highPage >= (1UL << (fNumAddressBits - PAGE_SHIFT))));
fMemory = mem;
mem->dmaCommandOperation(kIOMDSetDMAActive, this, 0);
- if (autoPrepare)
- return prepare();
- };
-
- return kIOReturnSuccess;
+ if (autoPrepare) {
+ err = prepare();
+ if (err) {
+ clearMemoryDescriptor();
+ }
+ }
+ }
+
+ return err;
}
IOReturn
IODMACommandInternal * state = target->reserved;
- if (target->fNumAddressBits && (target->fNumAddressBits < 64) && !state->fLocalMapper)
+ if (target->fNumAddressBits && (target->fNumAddressBits < 64) && (state->fLocalMapperPageAlloc || !target->fMapper))
maxPhys = (1ULL << target->fNumAddressBits);
else
maxPhys = 0;
{
if (SHOULD_COPY_DIR(op, target->fMDSummary.fDirection))
{
+ addr64_t cpuAddr = address;
addr64_t remapAddr;
uint64_t chunk;
+ if ((kMapped == MAPTYPE(target->fMappingOptions))
+ && target->fMapper)
+ {
+ cpuAddr = target->fMapper->mapAddr(address);
+ }
+
remapAddr = ptoa_64(vm_page_get_phys_page(lastPage));
if (!state->fDoubleBuffer)
{
if (kWalkSyncIn & op)
{ // cppvNoModSnk
- copypv(remapAddr, address, chunk,
+ copypv(remapAddr, cpuAddr, chunk,
cppvPsnk | cppvFsnk | cppvPsrc | cppvNoRefSrc );
}
else
{
- copypv(address, remapAddr, chunk,
+ copypv(cpuAddr, remapAddr, chunk,
cppvPsnk | cppvFsnk | cppvPsrc | cppvNoRefSrc );
}
address += chunk;
if (kWalkPreflight & op)
{
- state->fMapContig = false;
state->fMisaligned = false;
state->fDoubleBuffer = false;
state->fPrepared = false;
state->fCopyNext = NULL;
state->fCopyPageAlloc = 0;
- state->fLocalMapperPageAlloc = 0;
state->fCopyPageCount = 0;
state->fNextRemapPage = NULL;
state->fCopyMD = 0;
{
offset = 0;
numSegments = 0-1;
- ret = genIOVMSegments(op, segmentOp, (void *) op, &offset, state, &numSegments);
+ ret = genIOVMSegments(op, segmentOp, (void *)(uintptr_t) op, &offset, state, &numSegments);
}
op &= ~kWalkPreflight;
if (!state->fDoubleBuffer)
{
kern_return_t kr;
+
+ if (fMapper) panic("fMapper copying");
+
kr = vm_page_alloc_list(state->fCopyPageCount,
KMA_LOMEM | KMA_NOPAGEWAIT, &mapBase);
if (KERN_SUCCESS != kr)
state->fCopyNext = state->fCopyPageAlloc;
offset = 0;
numSegments = 0-1;
- ret = genIOVMSegments(op, segmentOp, (void *) op, &offset, state, &numSegments);
+ ret = genIOVMSegments(op, segmentOp, (void *)(uintptr_t) op, &offset, state, &numSegments);
state->fPrepared = true;
op &= ~(kWalkSyncIn | kWalkSyncOut);
}
}
else
{
- DEBG("IODMACommand !iovmAlloc");
+ DEBG("IODMACommand !alloc IOBMD");
return (kIOReturnNoResources);
}
}
}
-
- if (state->fLocalMapper)
- {
- state->fLocalMapperPageCount = atop_64(round_page(
- state->fPreparedLength + ((state->fPreparedOffset + fMDSummary.fPageAlign) & page_mask)));
- state->fLocalMapperPageAlloc = fMapper->iovmAllocDMACommand(this, state->fLocalMapperPageCount);
- state->fMapContig = true;
- }
}
if (state->fPrepared && ((kWalkSyncIn | kWalkSyncOut) & op))
state->fCopyNext = state->fCopyPageAlloc;
offset = 0;
numSegments = 0-1;
- ret = genIOVMSegments(op, segmentOp, (void *) op, &offset, state, &numSegments);
+ ret = genIOVMSegments(op, segmentOp, (void *)(uintptr_t) op, &offset, state, &numSegments);
}
else if (state->fCopyMD)
{
if (kWalkComplete & op)
{
- if (state->fLocalMapperPageAlloc)
- {
- fMapper->iovmFreeDMACommand(this, state->fLocalMapperPageAlloc, state->fLocalMapperPageCount);
- state->fLocalMapperPageAlloc = 0;
- state->fLocalMapperPageCount = 0;
- }
if (state->fCopyPageAlloc)
{
vm_page_free_list(state->fCopyPageAlloc, FALSE);
if (fActive)
return kIOReturnNotPermitted;
- if (!outSegFunc || !numAddressBits)
+ if (!outSegFunc)
return kIOReturnBadArgument;
bool is32Bit = (OutputHost32 == outSegFunc || OutputBig32 == outSegFunc
if (!maxTransferSize)
maxTransferSize--; // Set Max transfer to -1
+ if (mapper && !OSDynamicCast(IOMapper, mapper))
+ {
+ fInternalState->fDevice = mapper;
+ mapper = 0;
+ }
if (!mapper)
{
IOMapper::checkForSystemMapper();
switch (MAPTYPE(mappingOptions))
{
case kMapped: break;
- case kNonCoherent: fMapper = 0; break;
+ case kNonCoherent: break;
case kBypassed:
if (mapper && !mapper->getBypassMask(&fBypassMask))
return kIOReturnBadArgument;
state->fLocalMapper = (fMapper && (fMapper != IOMapper::gSystem));
state->fSourceAlignMask = fAlignMask;
- if (state->fLocalMapper)
+ if (fMapper)
state->fSourceAlignMask &= page_mask;
state->fCursor = state->fIterateOnly
|| (!state->fCheckAddressing
- && !state->fLocalMapper
&& (!state->fSourceAlignMask
|| ((fMDSummary.fPageAlign & (1 << 31)) && (0 == (fMDSummary.fPageAlign & state->fSourceAlignMask)))));
-
+
if (!state->fCursor)
{
IOOptionBits op = kWalkPrepare | kWalkPreflight;
op |= kWalkSyncOut;
ret = walkAll(op);
}
+
+ if (fMapper)
+ {
+ if (state->fLocalMapper)
+ {
+ state->fLocalMapperPageCount = atop_64(round_page(
+ state->fPreparedLength + ((state->fPreparedOffset + fMDSummary.fPageAlign) & page_mask)));
+ state->fLocalMapperPageAlloc = ptoa_64(fMapper->iovmAllocDMACommand(this, state->fLocalMapperPageCount));
+ if (!state->fLocalMapperPageAlloc)
+ {
+ DEBG("IODMACommand !iovmAlloc");
+ return (kIOReturnNoResources);
+ }
+ state->fMapContig = true;
+ }
+ else
+ {
+ IOMDDMAMapArgs mapArgs;
+ bzero(&mapArgs, sizeof(mapArgs));
+ mapArgs.fMapper = fMapper;
+ mapArgs.fMapSpec.device = state->fDevice;
+ mapArgs.fMapSpec.alignment = fAlignMask + 1;
+ mapArgs.fMapSpec.numAddressBits = fNumAddressBits ? fNumAddressBits : 64;
+ mapArgs.fOffset = state->fPreparedOffset;
+ mapArgs.fLength = state->fPreparedLength;
+ const IOMemoryDescriptor * md = state->fCopyMD;
+ if (!md) md = fMemory;
+ ret = md->dmaCommandOperation(kIOMDDMAMap | state->fIterateOnly, &mapArgs, sizeof(mapArgs));
+ if (kIOReturnSuccess == ret)
+ {
+ state->fLocalMapperPageAlloc = mapArgs.fAlloc;
+ state->fLocalMapperPageCount = mapArgs.fAllocCount;
+ state->fMapContig = mapArgs.fMapContig;
+ }
+ ret = kIOReturnSuccess;
+ }
+ }
+
+
if (kIOReturnSuccess == ret)
state->fPrepared = true;
}
op |= kWalkSyncIn;
ret = walkAll(op);
}
+ if (state->fLocalMapperPageAlloc)
+ {
+ if (state->fLocalMapper)
+ {
+ fMapper->iovmFreeDMACommand(this, atop_64(state->fLocalMapperPageAlloc), state->fLocalMapperPageCount);
+ }
+ else if (state->fLocalMapperPageCount)
+ {
+ fMapper->iovmFree(atop_64(state->fLocalMapperPageAlloc), state->fLocalMapperPageCount);
+ }
+ state->fLocalMapperPageAlloc = 0;
+ state->fLocalMapperPageCount = 0;
+ }
+
state->fPrepared = false;
if (IS_NONCOHERENT(fMappingOptions) && invalidateCache)
return kIOReturnBadArgument;
IOMDDMAWalkSegmentArgs *state =
- (IOMDDMAWalkSegmentArgs *) fState;
+ (IOMDDMAWalkSegmentArgs *)(void *) fState;
UInt64 offset = *offsetP + internalState->fPreparedOffset;
UInt64 memLength = internalState->fPreparedOffset + internalState->fPreparedLength;
state->fOffset = offset;
state->fLength = memLength - offset;
- if (internalState->fMapContig && (kWalkClient & op))
+ if (internalState->fMapContig && internalState->fLocalMapperPageAlloc)
{
- ppnum_t pageNum = internalState->fLocalMapperPageAlloc;
- state->fIOVMAddr = ptoa_64(pageNum)
- + offset - internalState->fPreparedOffset;
+ state->fIOVMAddr = internalState->fLocalMapperPageAlloc + offset;
rtn = kIOReturnSuccess;
+#if 0
+ {
+ uint64_t checkOffset;
+ IOPhysicalLength segLen;
+ for (checkOffset = 0; checkOffset < state->fLength; )
+ {
+ addr64_t phys = const_cast<IOMemoryDescriptor *>(fMemory)->getPhysicalSegment(checkOffset + offset, &segLen, kIOMemoryMapperNone);
+ if (fMapper->mapAddr(state->fIOVMAddr + checkOffset) != phys)
+ {
+ panic("%llx != %llx:%llx, %llx phys: %llx %llx\n", offset,
+ state->fIOVMAddr + checkOffset, fMapper->mapAddr(state->fIOVMAddr + checkOffset), state->fLength,
+ phys, checkOffset);
+ }
+ checkOffset += page_size - (phys & page_mask);
+ }
+ }
+#endif
}
else
{
SegmentFunction segmentFunction = (SegmentFunction) reference;
IOReturn ret = kIOReturnSuccess;
- if ((target->fNumAddressBits < 64)
+ if (target->fNumAddressBits && (target->fNumAddressBits < 64)
&& ((segment.fIOVMAddr + segment.fLength - 1) >> target->fNumAddressBits)
- && (target->reserved->fLocalMapperPageAlloc || !target->reserved->fLocalMapper))
+ && (target->reserved->fLocalMapperPageAlloc || !target->fMapper))
{
DEBG("kIOReturnMessageTooLarge(fNumAddressBits) %qx, %qx\n", segment.fIOVMAddr, segment.fLength);
ret = kIOReturnMessageTooLarge;
projects / apple / xnu.git / blobdiff