#include <IOKit/IOBufferMemoryDescriptor.h>
#include "IOKitKernelInternal.h"
-#include "IOCopyMapper.h"
#define MAPTYPE(type) ((UInt) (type) & kTypeMask)
#define IS_MAPPED(type) (MAPTYPE(type) == kMapped)
#endif
#if 0
-#define DEBG(fmt, args...) { kprintf(fmt, ## args); }
+#define DEBG(fmt, args...) { IOLog(fmt, ## args); kprintf(fmt, ## args); }
#else
#define DEBG(fmt, args...) {}
#endif
-
/**************************** class IODMACommand ***************************/
#undef super
-#define super OSObject
+#define super IOCommand
OSDefineMetaClassAndStructors(IODMACommand, IOCommand);
OSMetaClassDefineReservedUsed(IODMACommand, 0);
IOMapper *mapper,
void *refCon)
{
- if (!super::init() || !outSegFunc || !numAddressBits)
+ if (!super::init() || !outSegFunc)
return false;
bool is32Bit = (OutputHost32 == outSegFunc || OutputBig32 == outSegFunc
IOReturn
IODMACommand::setMemoryDescriptor(const IOMemoryDescriptor *mem, bool autoPrepare)
{
+ IOReturn err = kIOReturnSuccess;
+
if (mem == fMemory)
{
if (!autoPrepare)
if (fActive)
return kIOReturnBusy;
clearMemoryDescriptor();
- };
+ }
if (mem) {
bzero(&fMDSummary, sizeof(fMDSummary));
- IOReturn rtn = mem->dmaCommandOperation(
+ err = mem->dmaCommandOperation(
kIOMDGetCharacteristics,
&fMDSummary, sizeof(fMDSummary));
- if (rtn)
- return rtn;
+ if (err)
+ return err;
ppnum_t highPage = fMDSummary.fHighestPage ? fMDSummary.fHighestPage : gIOLastPage;
fMemory = mem;
mem->dmaCommandOperation(kIOMDSetDMAActive, this, 0);
- if (autoPrepare)
- return prepare();
- };
-
- return kIOReturnSuccess;
+ if (autoPrepare) {
+ err = prepare();
+ if (err) {
+ clearMemoryDescriptor();
+ }
+ }
+ }
+
+ return err;
}
IOReturn
{
IOOptionBits op = (uintptr_t) reference;
addr64_t maxPhys, address;
- addr64_t remapAddr = 0;
uint64_t length;
uint32_t numPages;
if (!length)
return (kIOReturnSuccess);
- numPages = atop_64(round_page_64(length));
- remapAddr = state->fCopyNext;
+ numPages = atop_64(round_page_64((address & PAGE_MASK) + length));
if (kWalkPreflight & op)
{
}
else
{
+ vm_page_t lastPage;
+ lastPage = NULL;
if (kWalkPrepare & op)
{
+ lastPage = state->fCopyNext;
for (IOItemCount idx = 0; idx < numPages; idx++)
- gIOCopyMapper->iovmInsert(atop_64(remapAddr), idx, atop_64(address) + idx);
- }
- if (state->fDoubleBuffer)
- state->fCopyNext += length;
- else
- {
- state->fCopyNext += round_page(length);
- remapAddr += (address & PAGE_MASK);
+ {
+ vm_page_set_offset(lastPage, atop_64(address) + idx);
+ lastPage = vm_page_get_next(lastPage);
+ }
}
- if (SHOULD_COPY_DIR(op, target->fMDSummary.fDirection))
+ if (!lastPage || SHOULD_COPY_DIR(op, target->fMDSummary.fDirection))
{
- DEBG("cpv: 0x%qx %s 0x%qx, 0x%qx, 0x%02lx\n", remapAddr,
- (kWalkSyncIn & op) ? "->" : "<-",
- address, length, op);
- if (kWalkSyncIn & op)
- { // cppvNoModSnk
- copypv(remapAddr, address, length,
- cppvPsnk | cppvFsnk | cppvPsrc | cppvNoRefSrc );
- }
- else
+ lastPage = state->fCopyNext;
+ for (IOItemCount idx = 0; idx < numPages; idx++)
{
- copypv(address, remapAddr, length,
- cppvPsnk | cppvFsnk | cppvPsrc | cppvNoRefSrc );
+ if (SHOULD_COPY_DIR(op, target->fMDSummary.fDirection))
+ {
+ addr64_t remapAddr;
+ uint64_t chunk;
+
+ remapAddr = ptoa_64(vm_page_get_phys_page(lastPage));
+ if (!state->fDoubleBuffer)
+ {
+ remapAddr += (address & PAGE_MASK);
+ }
+ chunk = PAGE_SIZE - (address & PAGE_MASK);
+ if (chunk > length)
+ chunk = length;
+
+ DEBG("cpv: 0x%qx %s 0x%qx, 0x%qx, 0x%02lx\n", remapAddr,
+ (kWalkSyncIn & op) ? "->" : "<-",
+ address, chunk, op);
+
+ if (kWalkSyncIn & op)
+ { // cppvNoModSnk
+ copypv(remapAddr, address, chunk,
+ cppvPsnk | cppvFsnk | cppvPsrc | cppvNoRefSrc );
+ }
+ else
+ {
+ copypv(address, remapAddr, chunk,
+ cppvPsnk | cppvFsnk | cppvPsrc | cppvNoRefSrc );
+ }
+ address += chunk;
+ length -= chunk;
+ }
+ lastPage = vm_page_get_next(lastPage);
}
}
+ state->fCopyNext = lastPage;
}
return kIOReturnSuccess;
state->fMisaligned = false;
state->fDoubleBuffer = false;
state->fPrepared = false;
- state->fCopyNext = 0;
- state->fCopyMapperPageAlloc = 0;
+ state->fCopyNext = NULL;
+ state->fCopyPageAlloc = 0;
state->fLocalMapperPageAlloc = 0;
state->fCopyPageCount = 0;
- state->fNextRemapIndex = 0;
- state->fCopyMD = 0;
+ state->fNextRemapPage = NULL;
+ state->fCopyMD = 0;
if (!(kWalkDoubleBuffer & op))
{
if (state->fCopyPageCount)
{
- IOMapper * mapper;
- ppnum_t mapBase = 0;
+ vm_page_t mapBase = NULL;
DEBG("preflight fCopyPageCount %d\n", state->fCopyPageCount);
- mapper = gIOCopyMapper;
- if (mapper)
- mapBase = mapper->iovmAlloc(state->fCopyPageCount);
- if (mapBase)
+ if (!state->fDoubleBuffer)
{
- state->fCopyMapperPageAlloc = mapBase;
- if (state->fCopyMapperPageAlloc && state->fDoubleBuffer)
+ kern_return_t kr;
+ kr = vm_page_alloc_list(state->fCopyPageCount,
+ KMA_LOMEM | KMA_NOPAGEWAIT, &mapBase);
+ if (KERN_SUCCESS != kr)
{
- DEBG("contig copy map\n");
- state->fMapContig = true;
+ DEBG("vm_page_alloc_list(%d) failed (%d)\n", state->fCopyPageCount, kr);
+ mapBase = NULL;
}
+ }
- state->fCopyNext = ptoa_64(state->fCopyMapperPageAlloc);
+ if (mapBase)
+ {
+ state->fCopyPageAlloc = mapBase;
+ state->fCopyNext = state->fCopyPageAlloc;
offset = 0;
numSegments = 0-1;
ret = genIOVMSegments(op, segmentOp, (void *) op, &offset, state, &numSegments);
else
{
DEBG("alloc IOBMD\n");
- state->fCopyMD = IOBufferMemoryDescriptor::withOptions(
- fMDSummary.fDirection, state->fPreparedLength, state->fSourceAlignMask);
+ mach_vm_address_t mask = 0xFFFFF000; //state->fSourceAlignMask
+ state->fCopyMD = IOBufferMemoryDescriptor::inTaskWithPhysicalMask(kernel_task,
+ fMDSummary.fDirection, state->fPreparedLength, mask);
if (state->fCopyMD)
{
}
else
{
- DEBG("IODMACommand !iovmAlloc");
+ DEBG("IODMACommand !alloc IOBMD");
return (kIOReturnNoResources);
}
}
if (state->fLocalMapper)
{
- state->fLocalMapperPageCount = atop_64(round_page(state->fPreparedLength));
+ state->fLocalMapperPageCount = atop_64(round_page(
+ state->fPreparedLength + ((state->fPreparedOffset + fMDSummary.fPageAlign) & page_mask)));
state->fLocalMapperPageAlloc = fMapper->iovmAllocDMACommand(this, state->fLocalMapperPageCount);
+ if (!state->fLocalMapperPageAlloc)
+ {
+ DEBG("IODMACommand !iovmAlloc");
+ return (kIOReturnNoResources);
+ }
state->fMapContig = true;
}
}
{
DEBG("sync fCopyPageCount %d\n", state->fCopyPageCount);
- if (state->fCopyMapperPageAlloc)
+ if (state->fCopyPageAlloc)
{
- state->fCopyNext = ptoa_64(state->fCopyMapperPageAlloc);
+ state->fCopyNext = state->fCopyPageAlloc;
offset = 0;
numSegments = 0-1;
ret = genIOVMSegments(op, segmentOp, (void *) op, &offset, state, &numSegments);
fMapper->iovmFreeDMACommand(this, state->fLocalMapperPageAlloc, state->fLocalMapperPageCount);
state->fLocalMapperPageAlloc = 0;
state->fLocalMapperPageCount = 0;
- }
- if (state->fCopyMapperPageAlloc)
+ }
+ if (state->fCopyPageAlloc)
{
- gIOCopyMapper->iovmFree(state->fCopyMapperPageAlloc, state->fCopyPageCount);
- state->fCopyMapperPageAlloc = 0;
+ vm_page_free_list(state->fCopyPageAlloc, FALSE);
+ state->fCopyPageAlloc = 0;
state->fCopyPageCount = 0;
}
if (state->fCopyMD)
if (fActive)
return kIOReturnNotPermitted;
- if (!outSegFunc || !numAddressBits)
+ if (!outSegFunc)
return kIOReturnBadArgument;
bool is32Bit = (OutputHost32 == outSegFunc || OutputBig32 == outSegFunc
state->fMisaligned = false;
state->fDoubleBuffer = false;
state->fPrepared = false;
- state->fCopyNext = 0;
- state->fCopyMapperPageAlloc = 0;
+ state->fCopyNext = NULL;
+ state->fCopyPageAlloc = 0;
state->fCopyPageCount = 0;
- state->fNextRemapIndex = 0;
+ state->fNextRemapPage = NULL;
state->fCopyMD = 0;
state->fLocalMapperPageAlloc = 0;
state->fLocalMapperPageCount = 0;
if ((offset == internalState->fPreparedOffset) || (offset != state->fOffset) || internalState->fNewMD) {
state->fOffset = 0;
state->fIOVMAddr = 0;
- internalState->fNextRemapIndex = 0;
+ internalState->fNextRemapPage = NULL;
internalState->fNewMD = false;
state->fMapped = (IS_MAPPED(fMappingOptions) && fMapper);
mdOp = kIOMDFirstSegment;
maxPhys = 0;
maxPhys--;
- while ((state->fIOVMAddr) || state->fOffset < memLength)
+ while (state->fIOVMAddr || (state->fOffset < memLength))
{
- if (!state->fIOVMAddr) {
+ // state = next seg
+ if (!state->fIOVMAddr) {
IOReturn rtn;
if (internalState->fMapContig && (kWalkClient & op))
{
ppnum_t pageNum = internalState->fLocalMapperPageAlloc;
- if (!pageNum)
- pageNum = internalState->fCopyMapperPageAlloc;
state->fIOVMAddr = ptoa_64(pageNum)
+ offset - internalState->fPreparedOffset;
rtn = kIOReturnSuccess;
mdOp = kIOMDWalkSegments;
}
- if (rtn == kIOReturnSuccess) {
+ if (rtn == kIOReturnSuccess)
+ {
assert(state->fIOVMAddr);
assert(state->fLength);
+ if ((curSeg.fIOVMAddr + curSeg.fLength) == state->fIOVMAddr) {
+ UInt64 length = state->fLength;
+ offset += length;
+ curSeg.fLength += length;
+ state->fIOVMAddr = 0;
+ }
}
else if (rtn == kIOReturnOverrun)
state->fIOVMAddr = state->fLength = 0; // At end
else
return rtn;
- };
-
- if (!curSeg.fIOVMAddr) {
- UInt64 length = state->fLength;
+ }
- offset += length;
- curSeg.fIOVMAddr = state->fIOVMAddr | bypassMask;
- curSeg.fLength = length;
- state->fIOVMAddr = 0;
- }
- else if ((curSeg.fIOVMAddr + curSeg.fLength == state->fIOVMAddr)) {
+ // seg = state, offset = end of seg
+ if (!curSeg.fIOVMAddr)
+ {
UInt64 length = state->fLength;
- offset += length;
- curSeg.fLength += length;
- state->fIOVMAddr = 0;
- };
-
+ offset += length;
+ curSeg.fIOVMAddr = state->fIOVMAddr | bypassMask;
+ curSeg.fLength = length;
+ state->fIOVMAddr = 0;
+ }
if (!state->fIOVMAddr)
{
- if (kWalkClient & op)
+ if ((kWalkClient & op) && (curSeg.fIOVMAddr + curSeg.fLength - 1) > maxPhys)
{
- if ((curSeg.fIOVMAddr + curSeg.fLength - 1) > maxPhys)
+ if (internalState->fCursor)
+ {
+ curSeg.fIOVMAddr = 0;
+ ret = kIOReturnMessageTooLarge;
+ break;
+ }
+ else if (curSeg.fIOVMAddr <= maxPhys)
{
- if (internalState->fCursor)
+ UInt64 remain, newLength;
+
+ newLength = (maxPhys + 1 - curSeg.fIOVMAddr);
+ DEBG("trunc %qx, %qx-> %qx\n", curSeg.fIOVMAddr, curSeg.fLength, newLength);
+ remain = curSeg.fLength - newLength;
+ state->fIOVMAddr = newLength + curSeg.fIOVMAddr;
+ curSeg.fLength = newLength;
+ state->fLength = remain;
+ offset -= remain;
+ }
+ else
+ {
+ UInt64 addr = curSeg.fIOVMAddr;
+ ppnum_t addrPage = atop_64(addr);
+ vm_page_t remap = NULL;
+ UInt64 remain, newLength;
+
+ DEBG("sparse switch %qx, %qx ", addr, curSeg.fLength);
+
+ remap = internalState->fNextRemapPage;
+ if (remap && (addrPage == vm_page_get_offset(remap)))
{
- curSeg.fIOVMAddr = 0;
- ret = kIOReturnMessageTooLarge;
- break;
}
- else if (curSeg.fIOVMAddr <= maxPhys)
+ else for (remap = internalState->fCopyPageAlloc;
+ remap && (addrPage != vm_page_get_offset(remap));
+ remap = vm_page_get_next(remap))
{
- UInt64 remain, newLength;
-
- newLength = (maxPhys + 1 - curSeg.fIOVMAddr);
- DEBG("trunc %qx, %qx-> %qx\n", curSeg.fIOVMAddr, curSeg.fLength, newLength);
- remain = curSeg.fLength - newLength;
- state->fIOVMAddr = newLength + curSeg.fIOVMAddr;
- curSeg.fLength = newLength;
- state->fLength = remain;
- offset -= remain;
}
- else if (gIOCopyMapper)
+
+ if (!remap) panic("no remap page found");
+
+ curSeg.fIOVMAddr = ptoa_64(vm_page_get_phys_page(remap))
+ + (addr & PAGE_MASK);
+ internalState->fNextRemapPage = vm_page_get_next(remap);
+
+ newLength = PAGE_SIZE - (addr & PAGE_MASK);
+ if (newLength < curSeg.fLength)
{
- DEBG("sparse switch %qx, %qx ", curSeg.fIOVMAddr, curSeg.fLength);
- if (trunc_page_64(curSeg.fIOVMAddr) == gIOCopyMapper->mapAddr(
- ptoa_64(internalState->fCopyMapperPageAlloc + internalState->fNextRemapIndex)))
- {
-
- curSeg.fIOVMAddr = ptoa_64(internalState->fCopyMapperPageAlloc + internalState->fNextRemapIndex)
- + (curSeg.fIOVMAddr & PAGE_MASK);
- internalState->fNextRemapIndex += atop_64(round_page(curSeg.fLength));
- }
- else for (UInt checkRemapIndex = 0; checkRemapIndex < internalState->fCopyPageCount; checkRemapIndex++)
- {
- if (trunc_page_64(curSeg.fIOVMAddr) == gIOCopyMapper->mapAddr(
- ptoa_64(internalState->fCopyMapperPageAlloc + checkRemapIndex)))
- {
- curSeg.fIOVMAddr = ptoa_64(internalState->fCopyMapperPageAlloc + checkRemapIndex)
- + (curSeg.fIOVMAddr & PAGE_MASK);
- internalState->fNextRemapIndex = checkRemapIndex + atop_64(round_page(curSeg.fLength));
- break;
- }
- }
- DEBG("-> %qx, %qx\n", curSeg.fIOVMAddr, curSeg.fLength);
+ remain = curSeg.fLength - newLength;
+ state->fIOVMAddr = addr + newLength;
+ curSeg.fLength = newLength;
+ state->fLength = remain;
+ offset -= remain;
}
+ DEBG("-> %qx, %qx offset %qx\n", curSeg.fIOVMAddr, curSeg.fLength, offset);
}
}
SegmentFunction segmentFunction = (SegmentFunction) reference;
IOReturn ret = kIOReturnSuccess;
- if ((target->fNumAddressBits < 64)
+ if (target->fNumAddressBits && (target->fNumAddressBits < 64)
&& ((segment.fIOVMAddr + segment.fLength - 1) >> target->fNumAddressBits)
&& (target->reserved->fLocalMapperPageAlloc || !target->reserved->fLocalMapper))
{
projects / apple / xnu.git / blobdiff