xnu-7195.101.1.tar.gz

[apple/xnu.git] / iokit / Kernel / IODMACommand.cpp

/*
 * Copyright (c) 2005-2006 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#define IOKIT_ENABLE_SHARED_PTR

#include <IOKit/assert.h>

#include <libkern/OSTypes.h>
#include <libkern/OSByteOrder.h>
#include <libkern/OSDebug.h>

#include <IOKit/IOReturn.h>
#include <IOKit/IOLib.h>
#include <IOKit/IODMACommand.h>
#include <IOKit/IOMapper.h>
#include <IOKit/IOMemoryDescriptor.h>
#include <IOKit/IOBufferMemoryDescriptor.h>

#include "IOKitKernelInternal.h"

#define MAPTYPE(type)           ((UInt) (type) & kTypeMask)
#define IS_NONCOHERENT(type)    (MAPTYPE(type) == kNonCoherent)

enum{
	kWalkSyncIn       = 0x01,// bounce -> md
	kWalkSyncOut      = 0x02,// bounce <- md
	kWalkSyncAlways   = 0x04,
	kWalkPreflight    = 0x08,
	kWalkDoubleBuffer = 0x10,
	kWalkPrepare      = 0x20,
	kWalkComplete     = 0x40,
	kWalkClient       = 0x80
};


#define fInternalState reserved
#define fState         reserved->fState
#define fMDSummary     reserved->fMDSummary


#if 1
// no direction => OutIn
#define SHOULD_COPY_DIR(op, direction)                                      \
	((kIODirectionNone == (direction))                                  \
	    || (kWalkSyncAlways & (op))                                     \
	    || (((kWalkSyncIn & (op)) ? kIODirectionIn : kIODirectionOut)   \
	                                            & (direction)))

#else
#define SHOULD_COPY_DIR(state, direction) (true)
#endif

#if 0
#define DEBG(fmt, args...)      { IOLog(fmt, ## args); kprintf(fmt, ## args); }
#else
#define DEBG(fmt, args...)      {}
#endif

#if 0
#define LOGTAG          0x87654321
#endif

/**************************** class IODMACommand ***************************/

#undef super
#define super IOCommand
OSDefineMetaClassAndStructorsWithZone(IODMACommand, IOCommand, ZC_NONE);

OSMetaClassDefineReservedUsedX86(IODMACommand, 0);
OSMetaClassDefineReservedUsedX86(IODMACommand, 1);
OSMetaClassDefineReservedUsedX86(IODMACommand, 2);
OSMetaClassDefineReservedUsedX86(IODMACommand, 3);
OSMetaClassDefineReservedUsedX86(IODMACommand, 4);
OSMetaClassDefineReservedUsedX86(IODMACommand, 5);
OSMetaClassDefineReservedUsedX86(IODMACommand, 6);
OSMetaClassDefineReservedUnused(IODMACommand, 7);
OSMetaClassDefineReservedUnused(IODMACommand, 8);
OSMetaClassDefineReservedUnused(IODMACommand, 9);
OSMetaClassDefineReservedUnused(IODMACommand, 10);
OSMetaClassDefineReservedUnused(IODMACommand, 11);
OSMetaClassDefineReservedUnused(IODMACommand, 12);
OSMetaClassDefineReservedUnused(IODMACommand, 13);
OSMetaClassDefineReservedUnused(IODMACommand, 14);
OSMetaClassDefineReservedUnused(IODMACommand, 15);


OSSharedPtr<IODMACommand>
IODMACommand::withRefCon(void * refCon)
{
	OSSharedPtr<IODMACommand> me = OSMakeShared<IODMACommand>();

	if (me && !me->initWithRefCon(refCon)) {
		return nullptr;
	}

	return me;
}

OSSharedPtr<IODMACommand>
IODMACommand::withSpecification(SegmentFunction  outSegFunc,
    const SegmentOptions * segmentOptions,
    uint32_t               mappingOptions,
    IOMapper             * mapper,
    void                 * refCon)
{
	OSSharedPtr<IODMACommand> me = OSMakeShared<IODMACommand>();

	if (me && !me->initWithSpecification(outSegFunc, segmentOptions, mappingOptions,
	    mapper, refCon)) {
		return nullptr;
	}

	return me;
}

OSSharedPtr<IODMACommand>
IODMACommand::withSpecification(SegmentFunction outSegFunc,
    UInt8           numAddressBits,
    UInt64          maxSegmentSize,
    MappingOptions  mappingOptions,
    UInt64          maxTransferSize,
    UInt32          alignment,
    IOMapper       *mapper,
    void           *refCon)
{
	OSSharedPtr<IODMACommand> me = OSMakeShared<IODMACommand>();

	if (me && !me->initWithSpecification(outSegFunc,
	    numAddressBits, maxSegmentSize,
	    mappingOptions, maxTransferSize,
	    alignment, mapper, refCon)) {
		return nullptr;
	}

	return me;
}

OSSharedPtr<IODMACommand>
IODMACommand::cloneCommand(void *refCon)
{
	SegmentOptions segmentOptions =
	{
		.fStructSize                = sizeof(segmentOptions),
		.fNumAddressBits            = (uint8_t)fNumAddressBits,
		.fMaxSegmentSize            = fMaxSegmentSize,
		.fMaxTransferSize           = fMaxTransferSize,
		.fAlignment                 = fAlignMask + 1,
		.fAlignmentLength           = fAlignMaskInternalSegments + 1,
		.fAlignmentInternalSegments = fAlignMaskLength + 1
	};

	return IODMACommand::withSpecification(fOutSeg, &segmentOptions,
	           fMappingOptions, fMapper.get(), refCon);
}

#define kLastOutputFunction ((SegmentFunction) kLastOutputFunction)

bool
IODMACommand::initWithRefCon(void * refCon)
{
	if (!super::init()) {
		return false;
	}

	if (!reserved) {
		reserved = IONew(IODMACommandInternal, 1);
		if (!reserved) {
			return false;
		}
	}
	bzero(reserved, sizeof(IODMACommandInternal));
	fRefCon = refCon;

	return true;
}

bool
IODMACommand::initWithSpecification(SegmentFunction        outSegFunc,
    const SegmentOptions * segmentOptions,
    uint32_t               mappingOptions,
    IOMapper             * mapper,
    void                 * refCon)
{
	if (!initWithRefCon(refCon)) {
		return false;
	}

	if (kIOReturnSuccess != setSpecification(outSegFunc, segmentOptions,
	    mappingOptions, mapper)) {
		return false;
	}

	return true;
}

bool
IODMACommand::initWithSpecification(SegmentFunction outSegFunc,
    UInt8           numAddressBits,
    UInt64          maxSegmentSize,
    MappingOptions  mappingOptions,
    UInt64          maxTransferSize,
    UInt32          alignment,
    IOMapper       *mapper,
    void           *refCon)
{
	SegmentOptions segmentOptions =
	{
		.fStructSize                = sizeof(segmentOptions),
		.fNumAddressBits            = numAddressBits,
		.fMaxSegmentSize            = maxSegmentSize,
		.fMaxTransferSize           = maxTransferSize,
		.fAlignment                 = alignment,
		.fAlignmentLength           = 1,
		.fAlignmentInternalSegments = alignment
	};

	return initWithSpecification(outSegFunc, &segmentOptions, mappingOptions, mapper, refCon);
}

IOReturn
IODMACommand::setSpecification(SegmentFunction        outSegFunc,
    const SegmentOptions * segmentOptions,
    uint32_t               mappingOptions,
    IOMapper             * mapper)
{
	IOService * device = NULL;
	UInt8       numAddressBits;
	UInt64      maxSegmentSize;
	UInt64      maxTransferSize;
	UInt32      alignment;

	bool        is32Bit;

	if (!outSegFunc || !segmentOptions) {
		return kIOReturnBadArgument;
	}

	is32Bit = ((OutputHost32 == outSegFunc)
	    || (OutputBig32 == outSegFunc)
	    || (OutputLittle32 == outSegFunc));

	numAddressBits = segmentOptions->fNumAddressBits;
	maxSegmentSize = segmentOptions->fMaxSegmentSize;
	maxTransferSize = segmentOptions->fMaxTransferSize;
	alignment = segmentOptions->fAlignment;
	if (is32Bit) {
		if (!numAddressBits) {
			numAddressBits = 32;
		} else if (numAddressBits > 32) {
			return kIOReturnBadArgument;  // Wrong output function for bits
		}
	}

	if (numAddressBits && (numAddressBits < PAGE_SHIFT)) {
		return kIOReturnBadArgument;
	}

	if (!maxSegmentSize) {
		maxSegmentSize--;               // Set Max segment to -1
	}
	if (!maxTransferSize) {
		maxTransferSize--;              // Set Max transfer to -1
	}
	if (mapper && !OSDynamicCast(IOMapper, mapper)) {
		device = mapper;
		mapper = NULL;
	}
	if (!mapper && (kUnmapped != MAPTYPE(mappingOptions))) {
		IOMapper::checkForSystemMapper();
		mapper = IOMapper::gSystem;
	}

	fNumSegments     = 0;
	fOutSeg          = outSegFunc;
	fNumAddressBits  = numAddressBits;
	fMaxSegmentSize  = maxSegmentSize;
	fMappingOptions  = mappingOptions;
	fMaxTransferSize = maxTransferSize;
	if (!alignment) {
		alignment = 1;
	}
	fAlignMask       = alignment - 1;

	alignment = segmentOptions->fAlignmentLength;
	if (!alignment) {
		alignment = 1;
	}
	fAlignMaskLength = alignment - 1;

	alignment = segmentOptions->fAlignmentInternalSegments;
	if (!alignment) {
		alignment = (fAlignMask + 1);
	}
	fAlignMaskInternalSegments = alignment - 1;

	switch (MAPTYPE(mappingOptions)) {
	case kMapped:       break;
	case kUnmapped:     break;
	case kNonCoherent:  break;

	case kBypassed:
		if (!mapper) {
			break;
		}
		return kIOReturnBadArgument;

	default:
		return kIOReturnBadArgument;
	}
	;

	if (mapper != fMapper) {
		fMapper.reset(mapper, OSRetain);
	}

	fInternalState->fIterateOnly = (0 != (kIterateOnly & mappingOptions));
	fInternalState->fDevice = device;

	return kIOReturnSuccess;
}

void
IODMACommand::free()
{
	if (reserved) {
		IODelete(reserved, IODMACommandInternal, 1);
	}

	fMapper.reset();

	// Correct use of this class when setting an IOMemoryDescriptor
	// in fMemory via setMemoryDescriptor(desc) is, for the caller, to
	// have a matching call to clearMemoryDescriptor() before releasing
	// the object. The matching call has also the effect of releasing
	// the ref taken on the IOMemoryDescriptor in setMemoryDescriptor().
	//
	// A number of "misbehaving" drivers has been found during testing,
	// whereby a matching call to clearMemoryDescriptor() is missing:
	//
	// rdar://59947343
	// rdar://59946968
	//
	// Both the approaches taken in said drivers are wrong, but have gone
	// basically silent with fMemory being a regular pointer. With fMemory
	// becoming a OSSharedPtr, the IODMACommand destructor expects to find
	// either fMemory reset (through the call to clearMemoryDescriptor()) or
	// a reference hold for the release.
	//
	// For this reason, this workaround of detaching fMemory is put in
	// place here, choosing the leak over the panic for misbehaving
	// drivers. Once all instances are fixed, this workaround will be
	// removed.
	//
	// Note: all well behaving drivers that have matching calls for
	// setMemoryDescriptor() and clearMemoryDescriptor() are unaffected
	// since fMemory will be null at this point.
	fMemory.detach();

	super::free();
}

IOReturn
IODMACommand::setMemoryDescriptor(const IOMemoryDescriptor *mem, bool autoPrepare)
{
	IOReturn err = kIOReturnSuccess;

	if (mem == fMemory) {
		if (!autoPrepare) {
			while (fActive) {
				complete();
			}
		}
		return kIOReturnSuccess;
	}

	if (fMemory) {
		// As we are almost certainly being called from a work loop thread
		// if fActive is true it is probably not a good time to potentially
		// block.  Just test for it and return an error
		if (fActive) {
			return kIOReturnBusy;
		}
		clearMemoryDescriptor();
	}

	if (mem) {
		bzero(&fMDSummary, sizeof(fMDSummary));
		err = mem->dmaCommandOperation(kIOMDGetCharacteristics | (kMapped == MAPTYPE(fMappingOptions)),
		    &fMDSummary, sizeof(fMDSummary));
		if (err) {
			return err;
		}

		ppnum_t highPage = fMDSummary.fHighestPage ? fMDSummary.fHighestPage : gIOLastPage;

		if ((kMapped == MAPTYPE(fMappingOptions))
		    && fMapper) {
			fInternalState->fCheckAddressing = false;
		} else {
			fInternalState->fCheckAddressing = (fNumAddressBits && (highPage >= (1UL << (fNumAddressBits - PAGE_SHIFT))));
		}

		fInternalState->fNewMD = true;
		fMemory.reset(const_cast<IOMemoryDescriptor *>(mem), OSRetain);
		fInternalState->fSetActiveNoMapper = (!fMapper);
		if (fInternalState->fSetActiveNoMapper) {
			mem->dmaCommandOperation(kIOMDSetDMAActive, this, 0);
		}
		if (autoPrepare) {
			err = prepare();
			if (err) {
				clearMemoryDescriptor();
			}
		}
	}

	return err;
}

IOReturn
IODMACommand::clearMemoryDescriptor(bool autoComplete)
{
	if (fActive && !autoComplete) {
		return kIOReturnNotReady;
	}

	if (fMemory) {
		while (fActive) {
			complete();
		}
		if (fInternalState->fSetActiveNoMapper) {
			fMemory->dmaCommandOperation(kIOMDSetDMAInactive, this, 0);
		}
		fMemory.reset();
	}

	return kIOReturnSuccess;
}

const IOMemoryDescriptor *
IODMACommand::getMemoryDescriptor() const
{
	return fMemory.get();
}

IOMemoryDescriptor *
IODMACommand::getIOMemoryDescriptor() const
{
	OSSharedPtr<IOMemoryDescriptor> mem;

	mem = reserved->fCopyMD;
	if (!mem) {
		mem = fMemory;
	}

	return mem.get();
}

IOReturn
IODMACommand::segmentOp(
	void         *reference,
	IODMACommand *target,
	Segment64     segment,
	void         *segments,
	UInt32        segmentIndex)
{
	IOOptionBits op = (IOOptionBits)(uintptr_t) reference;
	addr64_t     maxPhys, address;
	uint64_t     length;
	uint32_t     numPages;
	uint32_t     mask;

	IODMACommandInternal * state = target->reserved;

	if (target->fNumAddressBits && (target->fNumAddressBits < 64) && (state->fLocalMapperAllocValid || !target->fMapper)) {
		maxPhys = (1ULL << target->fNumAddressBits);
	} else {
		maxPhys = 0;
	}
	maxPhys--;

	address = segment.fIOVMAddr;
	length = segment.fLength;

	assert(length);

	if (!state->fMisaligned) {
		mask = (segmentIndex ? target->fAlignMaskInternalSegments : state->fSourceAlignMask);
		state->fMisaligned |= (0 != (mask & address));
		if (state->fMisaligned) {
			DEBG("misaligned address %qx:%qx, %x\n", address, length, mask);
		}
	}
	if (!state->fMisaligned) {
		mask = target->fAlignMaskLength;
		state->fMisaligned |= (0 != (mask & length));
		if (state->fMisaligned) {
			DEBG("misaligned length %qx:%qx, %x\n", address, length, mask);
		}
	}

	if (state->fMisaligned && (kWalkPreflight & op)) {
		return kIOReturnNotAligned;
	}

	if (!state->fDoubleBuffer) {
		if ((address + length - 1) <= maxPhys) {
			length = 0;
		} else if (address <= maxPhys) {
			DEBG("tail %qx, %qx", address, length);
			length = (address + length - maxPhys - 1);
			address = maxPhys + 1;
			DEBG("-> %qx, %qx\n", address, length);
		}
	}

	if (!length) {
		return kIOReturnSuccess;
	}

	uint64_t numPages64 = atop_64(round_page_64((address & PAGE_MASK) + length));
	if (numPages64 > UINT_MAX) {
		return kIOReturnVMError;
	}
	numPages = (typeof(numPages))numPages64;

	if (kWalkPreflight & op) {
		state->fCopyPageCount += numPages;
	} else {
		vm_page_t lastPage;
		lastPage = NULL;
		if (kWalkPrepare & op) {
			lastPage = state->fCopyNext;
			for (IOItemCount idx = 0; idx < numPages; idx++) {
				vm_page_set_offset(lastPage, atop_64(address) + idx);
				lastPage = vm_page_get_next(lastPage);
			}
		}

		if (!lastPage || SHOULD_COPY_DIR(op, target->fMDSummary.fDirection)) {
			lastPage = state->fCopyNext;
			for (IOItemCount idx = 0; idx < numPages; idx++) {
				if (SHOULD_COPY_DIR(op, target->fMDSummary.fDirection)) {
					addr64_t cpuAddr = address;
					addr64_t remapAddr;
					uint64_t chunk;

					if ((kMapped == MAPTYPE(target->fMappingOptions))
					    && target->fMapper) {
						cpuAddr = target->fMapper->mapToPhysicalAddress(address);
					}

					remapAddr = ptoa_64(vm_page_get_phys_page(lastPage));
					if (!state->fDoubleBuffer) {
						remapAddr += (address & PAGE_MASK);
					}
					chunk = PAGE_SIZE - (address & PAGE_MASK);
					if (chunk > length) {
						chunk = length;
					}
					if (chunk > (UINT_MAX - PAGE_SIZE + 1)) {
						chunk = (UINT_MAX - PAGE_SIZE + 1);
					}

					DEBG("cpv: 0x%qx %s 0x%qx, 0x%qx, 0x%02lx\n", remapAddr,
					    (kWalkSyncIn & op) ? "->" : "<-",
					    address, chunk, op);

					if (kWalkSyncIn & op) { // cppvNoModSnk
						copypv(remapAddr, cpuAddr, (unsigned int) chunk,
						    cppvPsnk | cppvFsnk | cppvPsrc | cppvNoRefSrc );
					} else {
						copypv(cpuAddr, remapAddr, (unsigned int) chunk,
						    cppvPsnk | cppvFsnk | cppvPsrc | cppvNoRefSrc );
					}
					address += chunk;
					length -= chunk;
				}
				lastPage = vm_page_get_next(lastPage);
			}
		}
		state->fCopyNext = lastPage;
	}

	return kIOReturnSuccess;
}

OSSharedPtr<IOBufferMemoryDescriptor>
IODMACommand::createCopyBuffer(IODirection direction, UInt64 length)
{
	mach_vm_address_t mask = 0xFFFFF000;    //state->fSourceAlignMask
	return IOBufferMemoryDescriptor::inTaskWithPhysicalMask(kernel_task,
	           direction, length, mask);
}

IOReturn
IODMACommand::walkAll(uint32_t op)
{
	IODMACommandInternal * state = fInternalState;

	IOReturn     ret = kIOReturnSuccess;
	UInt32       numSegments;
	UInt64       offset;

	if (kWalkPreflight & op) {
		state->fMisaligned     = false;
		state->fDoubleBuffer   = false;
		state->fPrepared       = false;
		state->fCopyNext       = NULL;
		state->fCopyPageAlloc  = NULL;
		state->fCopyPageCount  = 0;
		state->fNextRemapPage  = NULL;
		state->fCopyMD         = NULL;

		if (!(kWalkDoubleBuffer & op)) {
			offset = 0;
			numSegments = 0 - 1;
			ret = genIOVMSegments(op, segmentOp, (void *)(uintptr_t) op, &offset, state, &numSegments);
		}

		op &= ~kWalkPreflight;

		state->fDoubleBuffer = (state->fMisaligned || state->fForceDoubleBuffer);
		state->fForceDoubleBuffer = false;
		if (state->fDoubleBuffer) {
			state->fCopyPageCount = (typeof(state->fCopyPageCount))(atop_64(round_page(state->fPreparedLength)));
		}

		if (state->fCopyPageCount) {
			vm_page_t mapBase = NULL;

			DEBG("preflight fCopyPageCount %d\n", state->fCopyPageCount);

			if (!fMapper && !state->fDoubleBuffer) {
				kern_return_t kr;

				if (fMapper) {
					panic("fMapper copying");
				}

				kr = vm_page_alloc_list(state->fCopyPageCount,
				    (kma_flags_t)(KMA_LOMEM | KMA_NOPAGEWAIT), &mapBase);
				if (KERN_SUCCESS != kr) {
					DEBG("vm_page_alloc_list(%d) failed (%d)\n", state->fCopyPageCount, kr);
					mapBase = NULL;
				}
			}

			if (mapBase) {
				state->fCopyPageAlloc = mapBase;
				state->fCopyNext = state->fCopyPageAlloc;
				offset = 0;
				numSegments = 0 - 1;
				ret = genIOVMSegments(op, segmentOp, (void *)(uintptr_t) op, &offset, state, &numSegments);
				state->fPrepared = true;
				op &= ~(kWalkSyncIn | kWalkSyncOut);
			} else {
				DEBG("alloc IOBMD\n");
				state->fCopyMD = createCopyBuffer(fMDSummary.fDirection, state->fPreparedLength);

				if (state->fCopyMD) {
					ret = kIOReturnSuccess;
					state->fPrepared = true;
				} else {
					DEBG("IODMACommand !alloc IOBMD");
					return kIOReturnNoResources;
				}
			}
		}
	}

	if (state->fPrepared && ((kWalkSyncIn | kWalkSyncOut) & op)) {
		if (state->fCopyPageCount) {
			DEBG("sync fCopyPageCount %d\n", state->fCopyPageCount);

			if (state->fCopyPageAlloc) {
				state->fCopyNext = state->fCopyPageAlloc;
				offset = 0;
				numSegments = 0 - 1;
				ret = genIOVMSegments(op, segmentOp, (void *)(uintptr_t) op, &offset, state, &numSegments);
			} else if (state->fCopyMD) {
				DEBG("sync IOBMD\n");

				if (SHOULD_COPY_DIR(op, fMDSummary.fDirection)) {
					OSSharedPtr<IOMemoryDescriptor> poMD = fMemory;

					IOByteCount bytes;

					if (kWalkSyncIn & op) {
						bytes = poMD->writeBytes(state->fPreparedOffset,
						    state->fCopyMD->getBytesNoCopy(),
						    state->fPreparedLength);
					} else {
						bytes = poMD->readBytes(state->fPreparedOffset,
						    state->fCopyMD->getBytesNoCopy(),
						    state->fPreparedLength);
					}
					DEBG("fCopyMD %s %lx bytes\n", (kWalkSyncIn & op) ? "wrote" : "read", bytes);
					ret = (bytes == state->fPreparedLength) ? kIOReturnSuccess : kIOReturnUnderrun;
				} else {
					ret = kIOReturnSuccess;
				}
			}
		}
	}

	if (kWalkComplete & op) {
		if (state->fCopyPageAlloc) {
			vm_page_free_list(state->fCopyPageAlloc, FALSE);
			state->fCopyPageAlloc = NULL;
			state->fCopyPageCount = 0;
		}
		if (state->fCopyMD) {
			state->fCopyMD.reset();
		}

		state->fPrepared = false;
	}
	return ret;
}

UInt8
IODMACommand::getNumAddressBits(void)
{
	return (UInt8) fNumAddressBits;
}

UInt32
IODMACommand::getAlignment(void)
{
	return fAlignMask + 1;
}

uint32_t
IODMACommand::getAlignmentLength(void)
{
	return fAlignMaskLength + 1;
}

uint32_t
IODMACommand::getAlignmentInternalSegments(void)
{
	return fAlignMaskInternalSegments + 1;
}

IOReturn
IODMACommand::prepareWithSpecification(SegmentFunction        outSegFunc,
    const SegmentOptions * segmentOptions,
    uint32_t               mappingOptions,
    IOMapper             * mapper,
    UInt64                 offset,
    UInt64                 length,
    bool                   flushCache,
    bool                   synchronize)
{
	IOReturn ret;

	if (fActive) {
		return kIOReturnNotPermitted;
	}

	ret = setSpecification(outSegFunc, segmentOptions, mappingOptions, mapper);
	if (kIOReturnSuccess != ret) {
		return ret;
	}

	ret = prepare(offset, length, flushCache, synchronize);

	return ret;
}

IOReturn
IODMACommand::prepareWithSpecification(SegmentFunction  outSegFunc,
    UInt8            numAddressBits,
    UInt64           maxSegmentSize,
    MappingOptions   mappingOptions,
    UInt64           maxTransferSize,
    UInt32           alignment,
    IOMapper         *mapper,
    UInt64           offset,
    UInt64           length,
    bool             flushCache,
    bool             synchronize)
{
	SegmentOptions segmentOptions =
	{
		.fStructSize                = sizeof(segmentOptions),
		.fNumAddressBits            = numAddressBits,
		.fMaxSegmentSize            = maxSegmentSize,
		.fMaxTransferSize           = maxTransferSize,
		.fAlignment                 = alignment,
		.fAlignmentLength           = 1,
		.fAlignmentInternalSegments = alignment
	};

	return prepareWithSpecification(outSegFunc, &segmentOptions, mappingOptions, mapper,
	           offset, length, flushCache, synchronize);
}


IOReturn
IODMACommand::prepare(UInt64 offset, UInt64 length, bool flushCache, bool synchronize)
{
	IODMACommandInternal *  state = fInternalState;
	IOReturn                  ret = kIOReturnSuccess;
	uint32_t       mappingOptions = fMappingOptions;

	// check specification has been set
	if (!fOutSeg) {
		return kIOReturnNotReady;
	}

	if (!length) {
		length = fMDSummary.fLength;
	}

	if (length > fMaxTransferSize) {
		return kIOReturnNoSpace;
	}

	if (fActive++) {
		if ((state->fPreparedOffset != offset)
		    || (state->fPreparedLength != length)) {
			ret = kIOReturnNotReady;
		}
	} else {
		if (fAlignMaskLength & length) {
			return kIOReturnNotAligned;
		}

		if (atop_64(state->fPreparedLength) > UINT_MAX) {
			return kIOReturnVMError;
		}
		state->fPreparedOffset = offset;
		state->fPreparedLength = length;

		state->fMisaligned     = false;
		state->fDoubleBuffer   = false;
		state->fPrepared       = false;
		state->fCopyNext       = NULL;
		state->fCopyPageAlloc  = NULL;
		state->fCopyPageCount  = 0;
		state->fNextRemapPage  = NULL;
		state->fCopyMD         = NULL;
		state->fLocalMapperAlloc       = 0;
		state->fLocalMapperAllocValid  = false;
		state->fLocalMapperAllocLength = 0;

		state->fSourceAlignMask = fAlignMask;
		if (fMapper) {
			state->fSourceAlignMask &= page_mask;
		}

		state->fCursor = state->fIterateOnly
		    || (!state->fCheckAddressing
		    && (!state->fSourceAlignMask
		    || ((fMDSummary.fPageAlign & (1 << 31)) && (0 == (fMDSummary.fPageAlign & state->fSourceAlignMask)))));

		if (!state->fCursor) {
			IOOptionBits op = kWalkPrepare | kWalkPreflight;
			if (synchronize) {
				op |= kWalkSyncOut;
			}
			ret = walkAll(op);
		}

		if (IS_NONCOHERENT(mappingOptions) && flushCache) {
			if (state->fCopyMD) {
				state->fCopyMD->performOperation(kIOMemoryIncoherentIOStore, 0, length);
			} else {
				fMemory->performOperation(kIOMemoryIncoherentIOStore, offset, length);
			}
		}

		if (fMapper) {
			IOMDDMAMapArgs mapArgs;
			bzero(&mapArgs, sizeof(mapArgs));
			mapArgs.fMapper = fMapper.get();
			mapArgs.fCommand = this;
			mapArgs.fMapSpec.device         = state->fDevice;
			mapArgs.fMapSpec.alignment      = fAlignMask + 1;
			mapArgs.fMapSpec.numAddressBits = fNumAddressBits ? ((UInt8) fNumAddressBits) : 64;
			mapArgs.fLength = state->fPreparedLength;
			OSSharedPtr<IOMemoryDescriptor> md = state->fCopyMD;
			if (md) {
				mapArgs.fOffset = 0;
			} else {
				md = fMemory;
				mapArgs.fOffset = state->fPreparedOffset;
			}

			ret = md->dmaCommandOperation(kIOMDDMAMap, &mapArgs, sizeof(mapArgs));

			if ((kIOReturnSuccess == ret)
			    && mapArgs.fAllocLength
			    && (mapArgs.fAllocLength != mapArgs.fLength)) {
				do {
					// multisegment case
					IOMDDMAWalkSegmentState  walkState;
					IOMDDMAWalkSegmentArgs * walkArgs = (IOMDDMAWalkSegmentArgs *) (void *)&walkState;
					IOOptionBits             mdOp;
					uint64_t                 index;
					IOPhysicalLength         segLen;
					uint32_t                         segCount;
					uint64_t                         phys, align;
					uint64_t                         mapperPageMask;
					uint64_t                         mapperPageShift;
					uint64_t                         insertOffset;
					uint32_t                         mapOptions;
					uint64_t                         length;

					assert(mapArgs.fAllocLength > mapArgs.fLength);

					mapperPageMask    = fMapper->getPageSize();
					assert(mapperPageMask);
					mapperPageMask   -= 1;
					mapperPageShift   = (64 - __builtin_clzll(mapperPageMask));
					walkArgs->fMapped = false;
					length            = state->fPreparedLength;
					mdOp              = kIOMDFirstSegment;
					segCount          = 0;
					for (index = 0; index < length; segCount++) {
						walkArgs->fOffset = state->fPreparedOffset + index;

						ret    = md->dmaCommandOperation(mdOp, &walkState, sizeof(walkState));
						mdOp   = kIOMDWalkSegments;
						assert(kIOReturnSuccess == ret);
						if (ret != kIOReturnSuccess) {
							panic("dmaCommandOperation");
						}
						segLen = walkArgs->fLength;
						index += segLen;
					}
					if (ret != kIOReturnSuccess) {
						break;
					}

#if defined(LOGTAG)
					if (LOGTAG == fMemory->getTag()) {
						IOLog("DMA[%p] alloc 0x%qx, 0x%qx\n", this, mapArgs.fAlloc, mapArgs.fAllocLength);
					}
#endif /* defined(LOGTAG) */

					state->fMapSegments = IONewZero(IODMACommandMapSegment, segCount);
					if (!state->fMapSegments) {
						ret = kIOReturnNoMemory;
						break;
					}
					state->fMapSegmentsCount = segCount;

					switch (kIODirectionOutIn & fMDSummary.fDirection) {
					case kIODirectionOut:
						mapOptions = kIODMAMapReadAccess;
						break;
					case kIODirectionIn:
						mapOptions = kIODMAMapWriteAccess;
						break;
					default:
						mapOptions = kIODMAMapReadAccess | kIODMAMapWriteAccess;
						break;
					}

					mdOp = kIOMDFirstSegment;
					segCount = 0;
					for (insertOffset = 0, index = 0; index < length; segCount++) {
						walkArgs->fOffset = state->fPreparedOffset + index;
						ret = md->dmaCommandOperation(mdOp, &walkState, sizeof(walkState));
						mdOp = kIOMDWalkSegments;
						if (ret != kIOReturnSuccess) {
							panic("dmaCommandOperation 0x%x", ret);
						}
						phys = walkArgs->fIOVMAddr;
						segLen = walkArgs->fLength;

#if defined(LOGTAG)
						if (LOGTAG == fMemory->getTag()) {
							IOLog("DMA[%p] phys[%d] 0x%qx, 0x%qx\n", this, segCount, (uint64_t) phys, (uint64_t) segLen);
						}
#endif /* defined(LOGTAG) */

						align = (phys & mapperPageMask);

#if defined(LOGTAG)
						if (LOGTAG == fMemory->getTag()) {
							IOLog("DMA[%p] runs[%d] dmaoff 0x%qx, mapoff 0x%qx, align 0x%qx\n", this, segCount, index, insertOffset, align);
						}
#endif /* defined(LOGTAG) */

						assert(segCount < state->fMapSegmentsCount);
						state->fMapSegments[segCount].fDMAOffset = state->fPreparedOffset + index;
						state->fMapSegments[segCount].fMapOffset = insertOffset;
						state->fMapSegments[segCount].fPageOffset = align;
						index  += segLen;

						// segment page align
						segLen  = ((phys + segLen + mapperPageMask) & ~mapperPageMask);
						phys   -= align;
						segLen -= phys;
						insertOffset += segLen;
					}
					state->fLocalMapperAllocBase = (mapArgs.fAlloc & ~mapperPageMask);
#if defined(LOGTAG)
					if (LOGTAG == fMemory->getTag()) {
						IOLog("IODMACommand fMapSegmentsCount %d\n", state->fMapSegmentsCount);
					}
#endif /* defined(LOGTAG) */
				} while (false);
			}
			if (kIOReturnSuccess == ret) {
				state->fLocalMapperAlloc       = mapArgs.fAlloc;
				state->fLocalMapperAllocValid  = true;
				state->fLocalMapperAllocLength = mapArgs.fAllocLength;
			}
		}
		if (kIOReturnSuccess == ret) {
			state->fPrepared = true;
		}
	}
	return ret;
}

IOReturn
IODMACommand::complete(bool invalidateCache, bool synchronize)
{
	IODMACommandInternal * state = fInternalState;
	IOReturn               ret   = kIOReturnSuccess;
	OSSharedPtr<IOMemoryDescriptor> copyMD;

	if (fActive < 1) {
		return kIOReturnNotReady;
	}

	if (!--fActive) {
		copyMD = state->fCopyMD;

		if (IS_NONCOHERENT(fMappingOptions) && invalidateCache) {
			if (copyMD) {
				copyMD->performOperation(kIOMemoryIncoherentIOFlush, 0, state->fPreparedLength);
			} else {
				OSSharedPtr<IOMemoryDescriptor> md = fMemory;
				md->performOperation(kIOMemoryIncoherentIOFlush, state->fPreparedOffset, state->fPreparedLength);
			}
		}

		if (!state->fCursor) {
			IOOptionBits op = kWalkComplete;
			if (synchronize) {
				op |= kWalkSyncIn;
			}
			ret = walkAll(op);
		}

		if (state->fLocalMapperAllocValid) {
			IOMDDMAMapArgs mapArgs;
			bzero(&mapArgs, sizeof(mapArgs));
			mapArgs.fMapper = fMapper.get();
			mapArgs.fCommand = this;
			mapArgs.fAlloc = state->fLocalMapperAlloc;
			mapArgs.fAllocLength = state->fLocalMapperAllocLength;
			OSSharedPtr<IOMemoryDescriptor> md = copyMD;
			if (md) {
				mapArgs.fOffset = 0;
			} else {
				md = fMemory;
				mapArgs.fOffset = state->fPreparedOffset;
			}

			ret = md->dmaCommandOperation(kIOMDDMAUnmap, &mapArgs, sizeof(mapArgs));

			state->fLocalMapperAlloc       = 0;
			state->fLocalMapperAllocValid  = false;
			state->fLocalMapperAllocLength = 0;
			if (state->fMapSegments) {
				IODelete(state->fMapSegments, IODMACommandMapSegment, state->fMapSegmentsCount);
				state->fMapSegments      = NULL;
				state->fMapSegmentsCount = 0;
			}
		}

		state->fPrepared = false;
	}

	return ret;
}

IOReturn
IODMACommand::getPreparedOffsetAndLength(UInt64 * offset, UInt64 * length)
{
	IODMACommandInternal * state = fInternalState;
	if (fActive < 1) {
		return kIOReturnNotReady;
	}

	if (offset) {
		*offset = state->fPreparedOffset;
	}
	if (length) {
		*length = state->fPreparedLength;
	}

	return kIOReturnSuccess;
}

IOReturn
IODMACommand::synchronize(IOOptionBits options)
{
	IODMACommandInternal * state = fInternalState;
	IOReturn               ret   = kIOReturnSuccess;
	IOOptionBits           op;

	if (kIODirectionOutIn == (kIODirectionOutIn & options)) {
		return kIOReturnBadArgument;
	}

	if (fActive < 1) {
		return kIOReturnNotReady;
	}

	op = 0;
	if (kForceDoubleBuffer & options) {
		if (state->fDoubleBuffer) {
			return kIOReturnSuccess;
		}
		ret = complete(false /* invalidateCache */, true /* synchronize */);
		state->fCursor = false;
		state->fForceDoubleBuffer = true;
		ret = prepare(state->fPreparedOffset, state->fPreparedLength, false /* flushCache */, true /* synchronize */);

		return ret;
	} else if (state->fCursor) {
		return kIOReturnSuccess;
	}

	if (kIODirectionIn & options) {
		op |= kWalkSyncIn | kWalkSyncAlways;
	} else if (kIODirectionOut & options) {
		op |= kWalkSyncOut | kWalkSyncAlways;
	}

	ret = walkAll(op);

	return ret;
}

struct IODMACommandTransferContext {
	void *   buffer;
	UInt64   bufferOffset;
	UInt64   remaining;
	UInt32   op;
};
enum{
	kIODMACommandTransferOpReadBytes  = 1,
	kIODMACommandTransferOpWriteBytes = 2
};

IOReturn
IODMACommand::transferSegment(void   *reference,
    IODMACommand *target,
    Segment64     segment,
    void         *segments,
    UInt32        segmentIndex)
{
	IODMACommandTransferContext * context = (IODMACommandTransferContext *) reference;
	UInt64   length  = min(segment.fLength, context->remaining);
	addr64_t ioAddr  = segment.fIOVMAddr;
	addr64_t cpuAddr = ioAddr;

	context->remaining -= length;

	while (length) {
		UInt64 copyLen = length;
		if ((kMapped == MAPTYPE(target->fMappingOptions))
		    && target->fMapper) {
			cpuAddr = target->fMapper->mapToPhysicalAddress(ioAddr);
			copyLen = min(copyLen, page_size - (ioAddr & (page_size - 1)));
			ioAddr += copyLen;
		}
		if (copyLen > (UINT_MAX - PAGE_SIZE + 1)) {
			copyLen = (UINT_MAX - PAGE_SIZE + 1);
		}

		switch (context->op) {
		case kIODMACommandTransferOpReadBytes:
			copypv(cpuAddr, context->bufferOffset + (addr64_t) context->buffer, (unsigned int) copyLen,
			    cppvPsrc | cppvNoRefSrc | cppvFsnk | cppvKmap);
			break;
		case kIODMACommandTransferOpWriteBytes:
			copypv(context->bufferOffset + (addr64_t) context->buffer, cpuAddr, (unsigned int) copyLen,
			    cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
			break;
		}
		length                -= copyLen;
		context->bufferOffset += copyLen;
	}

	return context->remaining ? kIOReturnSuccess : kIOReturnOverrun;
}

UInt64
IODMACommand::transfer(IOOptionBits transferOp, UInt64 offset, void * buffer, UInt64 length)
{
	IODMACommandInternal *      state = fInternalState;
	IODMACommandTransferContext context;
	Segment64                   segments[1];
	UInt32                      numSegments = 0 - 1;

	if (fActive < 1) {
		return 0;
	}

	if (offset >= state->fPreparedLength) {
		return 0;
	}
	length = min(length, state->fPreparedLength - offset);

	context.buffer       = buffer;
	context.bufferOffset = 0;
	context.remaining    = length;
	context.op           = transferOp;
	(void) genIOVMSegments(kWalkClient, transferSegment, &context, &offset, &segments[0], &numSegments);

	return length - context.remaining;
}

UInt64
IODMACommand::readBytes(UInt64 offset, void *bytes, UInt64 length)
{
	return transfer(kIODMACommandTransferOpReadBytes, offset, bytes, length);
}

UInt64
IODMACommand::writeBytes(UInt64 offset, const void *bytes, UInt64 length)
{
	return transfer(kIODMACommandTransferOpWriteBytes, offset, const_cast<void *>(bytes), length);
}

IOReturn
IODMACommand::genIOVMSegments(UInt64 *offsetP,
    void   *segmentsP,
    UInt32 *numSegmentsP)
{
	return genIOVMSegments(kWalkClient, clientOutputSegment, (void *) fOutSeg,
	           offsetP, segmentsP, numSegmentsP);
}

IOReturn
IODMACommand::genIOVMSegments(uint32_t op,
    InternalSegmentFunction outSegFunc,
    void   *reference,
    UInt64 *offsetP,
    void   *segmentsP,
    UInt32 *numSegmentsP)
{
	IODMACommandInternal * internalState = fInternalState;
	IOOptionBits           mdOp = kIOMDWalkSegments;
	IOReturn               ret  = kIOReturnSuccess;

	if (!(kWalkComplete & op) && !fActive) {
		return kIOReturnNotReady;
	}

	if (!offsetP || !segmentsP || !numSegmentsP || !*numSegmentsP) {
		return kIOReturnBadArgument;
	}

	IOMDDMAWalkSegmentArgs *state =
	    (IOMDDMAWalkSegmentArgs *)(void *) fState;

	UInt64 offset    = *offsetP + internalState->fPreparedOffset;
	UInt64 memLength = internalState->fPreparedOffset + internalState->fPreparedLength;

	if (offset >= memLength) {
		return kIOReturnOverrun;
	}

	if ((offset == internalState->fPreparedOffset) || (offset != state->fOffset) || internalState->fNewMD) {
		state->fOffset                                   = 0;
		internalState->fIOVMAddrValid = state->fIOVMAddr = 0;
		internalState->fNextRemapPage                    = NULL;
		internalState->fNewMD                            = false;
		mdOp                                             = kIOMDFirstSegment;
		if (fMapper) {
			if (internalState->fLocalMapperAllocValid) {
				state->fMapped = true;
				state->fMappedBase = internalState->fLocalMapperAlloc;
			} else {
				state->fMapped = false;
			}
		}
	}

	UInt32    segIndex = 0;
	UInt32    numSegments = *numSegmentsP;
	Segment64 curSeg = { 0, 0 };
	bool      curSegValid = false;
	addr64_t  maxPhys;

	if (fNumAddressBits && (fNumAddressBits < 64)) {
		maxPhys = (1ULL << fNumAddressBits);
	} else {
		maxPhys = 0;
	}
	maxPhys--;

	while (internalState->fIOVMAddrValid || (state->fOffset < memLength)) {
		// state = next seg
		if (!internalState->fIOVMAddrValid) {
			IOReturn rtn;

			state->fOffset = offset;
			state->fLength = memLength - offset;

			bool done = false;
			bool check = false;

			if (internalState->fLocalMapperAllocValid) {
				if (!internalState->fMapSegmentsCount) {
					state->fIOVMAddr = internalState->fLocalMapperAlloc + offset - internalState->fPreparedOffset;
					rtn = kIOReturnSuccess;
					done = true;
					check = true;
				} else {
					uint64_t address;
					uint64_t length;
					uint64_t runOffset;
					uint64_t ind;
					uint64_t off2Ind = internalState->fOffset2Index;

					// Validate the previous offset
					if (offset
					    && (offset == internalState->fNextOffset || off2Ind <= offset)) {
						ind = internalState->fIndex;
					} else {
						ind = off2Ind = 0; // Start from beginning
					}
#if defined(LOGTAG)
					if (LOGTAG == fMemory->getTag()) {
						IOLog("DMA[%p] offsets 0x%qx, 0x%qx, 0x%qx ind %qd\n", this, offset, internalState->fPreparedOffset, internalState->fNextOffset, ind);
					}
#endif /* defined(LOGTAG) */

					// Scan through iopl info blocks looking for block containing offset
					while (ind < internalState->fMapSegmentsCount && offset >= internalState->fMapSegments[ind].fDMAOffset) {
						ind++;
					}
					if (ind < internalState->fMapSegmentsCount) {
						length = internalState->fMapSegments[ind].fDMAOffset;
					} else {
						length = memLength;
					}
					length -= offset;       // Remainder within iopl

					// Go back to actual range as search goes past it
					ind--;
					off2Ind = internalState->fMapSegments[ind].fDMAOffset;

					// Subtract offset till this iopl in total list
					runOffset = offset - off2Ind;

					// Compute an offset relative to the mapped base

					runOffset += internalState->fMapSegments[ind].fPageOffset;
					address = internalState->fLocalMapperAllocBase + internalState->fMapSegments[ind].fMapOffset + runOffset;
#if defined(LOGTAG)
					if (LOGTAG == fMemory->getTag()) {
						IOLog("DMA[%p] addrlen 0x%qx, 0x%qx\n", this, address, length);
					}
#endif /* defined(LOGTAG) */

					state->fIOVMAddr = address;
					state->fLength   = length;

					internalState->fIndex        = ind;
					internalState->fOffset2Index = off2Ind;
					internalState->fNextOffset   = state->fOffset + length;

					rtn = kIOReturnSuccess;
					done = true;
					check = true;
				}
			}

			if (!done) {
				IOMemoryDescriptor * memory =
				    internalState->fCopyMD ? internalState->fCopyMD.get() : fMemory.get();
				rtn = memory->dmaCommandOperation(mdOp, fState, sizeof(fState));
				mdOp = kIOMDWalkSegments;
			}
#if 0
			if (check
			    && !ml_at_interrupt_context()
			    && (rtn == kIOReturnSuccess)
			    && fMapper
			    && strcmp("AppleNVMeMMU", fMapper->getName())) {
				uint64_t checkOffset;
				IOPhysicalLength segLen;
				IOMemoryDescriptor * memory =
				    internalState->fCopyMD ? internalState->fCopyMD.get() : fMemory.get();
				for (checkOffset = 0; checkOffset < state->fLength;) {
					addr64_t phys = memory->getPhysicalSegment(offset + checkOffset, &segLen, kIOMemoryMapperNone);
					addr64_t mapperPhys;

					mapperPhys = fMapper->mapToPhysicalAddress(state->fIOVMAddr + checkOffset);
					mapperPhys |= (phys & (fMapper->getPageSize() - 1));
					if (mapperPhys != phys) {
						panic("DMA[%p] mismatch at offset %llx + %llx, dma %llx mapperPhys %llx != %llx, len %llx\n",
						    this, offset, checkOffset,
						    state->fIOVMAddr + checkOffset, mapperPhys, phys, state->fLength);
					}
					checkOffset += page_size - (phys & page_mask);
				}
			}
#endif
			if (rtn == kIOReturnSuccess) {
				internalState->fIOVMAddrValid = true;
				assert(state->fLength);
				if (curSegValid && ((curSeg.fIOVMAddr + curSeg.fLength) == state->fIOVMAddr)) {
					UInt64 length = state->fLength;
					offset          += length;
					curSeg.fLength  += length;
					internalState->fIOVMAddrValid = state->fIOVMAddr = 0;
				}
			} else if (rtn == kIOReturnOverrun) {
				internalState->fIOVMAddrValid = state->fIOVMAddr = state->fLength = 0; // At end
			} else {
				return rtn;
			}
		}

		// seg = state, offset = end of seg
		if (!curSegValid) {
			UInt64 length                 = state->fLength;
			offset                       += length;
			curSeg.fIOVMAddr              = state->fIOVMAddr;
			curSeg.fLength                = length;
			curSegValid                   = true;
			internalState->fIOVMAddrValid = state->fIOVMAddr = 0;
		}

		if (!internalState->fIOVMAddrValid) {
			// maxPhys
			if ((kWalkClient & op) && (curSeg.fIOVMAddr + curSeg.fLength - 1) > maxPhys) {
				if (internalState->fCursor) {
					curSegValid = curSeg.fIOVMAddr = 0;
					ret = kIOReturnMessageTooLarge;
					break;
				} else if (curSeg.fIOVMAddr <= maxPhys) {
					UInt64 remain, newLength;

					newLength        = (maxPhys + 1 - curSeg.fIOVMAddr);
					DEBG("trunc %qx, %qx-> %qx\n", curSeg.fIOVMAddr, curSeg.fLength, newLength);
					remain           = curSeg.fLength - newLength;
					state->fIOVMAddr = newLength + curSeg.fIOVMAddr;
					internalState->fIOVMAddrValid = true;
					curSeg.fLength   = newLength;
					state->fLength   = remain;
					offset          -= remain;
				} else {
					UInt64    addr = curSeg.fIOVMAddr;
					ppnum_t   addrPage = (ppnum_t) atop_64(addr);
					vm_page_t remap = NULL;
					UInt64    remain, newLength;

					DEBG("sparse switch %qx, %qx ", addr, curSeg.fLength);

					remap = internalState->fNextRemapPage;
					if (remap && (addrPage == vm_page_get_offset(remap))) {
					} else {
						for (remap = internalState->fCopyPageAlloc;
						    remap && (addrPage != vm_page_get_offset(remap));
						    remap = vm_page_get_next(remap)) {
						}
					}

					if (!remap) {
						panic("no remap page found");
					}

					curSeg.fIOVMAddr = ptoa_64(vm_page_get_phys_page(remap))
					    + (addr & PAGE_MASK);
					curSegValid = true;
					internalState->fNextRemapPage = vm_page_get_next(remap);

					newLength            = PAGE_SIZE - (addr & PAGE_MASK);
					if (newLength < curSeg.fLength) {
						remain           = curSeg.fLength - newLength;
						state->fIOVMAddr = addr + newLength;
						internalState->fIOVMAddrValid = true;
						curSeg.fLength   = newLength;
						state->fLength   = remain;
						offset          -= remain;
					}
					DEBG("-> %qx, %qx offset %qx\n", curSeg.fIOVMAddr, curSeg.fLength, offset);
				}
			}

			// reduce size of output segment
			uint64_t reduce, leftover = 0;

			// fMaxSegmentSize
			if (curSeg.fLength > fMaxSegmentSize) {
				leftover      += curSeg.fLength - fMaxSegmentSize;
				curSeg.fLength = fMaxSegmentSize;
				state->fIOVMAddr = curSeg.fLength + curSeg.fIOVMAddr;
				internalState->fIOVMAddrValid = true;
			}

			// alignment current length

			reduce = (curSeg.fLength & fAlignMaskLength);
			if (reduce && (curSeg.fLength > reduce)) {
				leftover       += reduce;
				curSeg.fLength -= reduce;
				state->fIOVMAddr = curSeg.fLength + curSeg.fIOVMAddr;
				internalState->fIOVMAddrValid = true;
			}

			// alignment next address

			reduce = (state->fIOVMAddr & fAlignMaskInternalSegments);
			if (reduce && (curSeg.fLength > reduce)) {
				leftover       += reduce;
				curSeg.fLength -= reduce;
				state->fIOVMAddr = curSeg.fLength + curSeg.fIOVMAddr;
				internalState->fIOVMAddrValid = true;
			}

			if (leftover) {
				DEBG("reduce seg by 0x%llx @ 0x%llx [0x%llx, 0x%llx]\n",
				    leftover, offset,
				    curSeg.fIOVMAddr, curSeg.fLength);
				state->fLength   = leftover;
				offset          -= leftover;
			}

			//

			if (internalState->fCursor) {
				bool misaligned;
				uint32_t mask;

				mask = (segIndex ? fAlignMaskInternalSegments : internalState->fSourceAlignMask);
				misaligned = (0 != (mask & curSeg.fIOVMAddr));
				if (!misaligned) {
					mask = fAlignMaskLength;
					misaligned |= (0 != (mask &  curSeg.fLength));
				}
				if (misaligned) {
					if (misaligned) {
						DEBG("cursor misaligned %qx:%qx\n", curSeg.fIOVMAddr, curSeg.fLength);
					}
					curSegValid = curSeg.fIOVMAddr = 0;
					ret = kIOReturnNotAligned;
					break;
				}
			}

			if (offset >= memLength) {
				curSeg.fLength   -= (offset - memLength);
				offset = memLength;
				internalState->fIOVMAddrValid = state->fIOVMAddr = state->fLength = 0; // At end
				break;
			}
		}

		if (internalState->fIOVMAddrValid) {
			if ((segIndex + 1 == numSegments)) {
				break;
			}
#if defined(LOGTAG)
			if ((LOGTAG == fMemory->getTag()) && (kWalkClient == op)) {
				IOLog("DMA[%p] outseg 0x%qx, 0x%qx\n", this, curSeg.fIOVMAddr, curSeg.fLength);
			}
#endif /* defined(LOGTAG) */
			ret = (*outSegFunc)(reference, this, curSeg, segmentsP, segIndex++);
			curSegValid = curSeg.fIOVMAddr = 0;
			if (kIOReturnSuccess != ret) {
				break;
			}
		}
	}

	if (curSegValid) {
#if defined(LOGTAG)
		if ((LOGTAG == fMemory->getTag()) && (kWalkClient == op)) {
			IOLog("DMA[%p] outseg 0x%qx, 0x%qx\n", this, curSeg.fIOVMAddr, curSeg.fLength);
		}
#endif /* defined(LOGTAG) */
		ret = (*outSegFunc)(reference, this, curSeg, segmentsP, segIndex++);
	}

	if (kIOReturnSuccess == ret) {
		state->fOffset = offset;
		*offsetP       = offset - internalState->fPreparedOffset;
		*numSegmentsP  = segIndex;
	}
	return ret;
}

IOReturn
IODMACommand::clientOutputSegment(
	void *reference, IODMACommand *target,
	Segment64 segment, void *vSegList, UInt32 outSegIndex)
{
	SegmentFunction segmentFunction = (SegmentFunction) reference;
	IOReturn ret = kIOReturnSuccess;

	if (target->fNumAddressBits && (target->fNumAddressBits < 64)
	    && ((segment.fIOVMAddr + segment.fLength - 1) >> target->fNumAddressBits)
	    && (target->reserved->fLocalMapperAllocValid || !target->fMapper)) {
		DEBG("kIOReturnMessageTooLarge(fNumAddressBits) %qx, %qx\n", segment.fIOVMAddr, segment.fLength);
		ret = kIOReturnMessageTooLarge;
	}

	if (!(*segmentFunction)(target, segment, vSegList, outSegIndex)) {
		DEBG("kIOReturnMessageTooLarge(fOutSeg) %qx, %qx\n", segment.fIOVMAddr, segment.fLength);
		ret = kIOReturnMessageTooLarge;
	}

	return ret;
}

IOReturn
IODMACommand::genIOVMSegments(SegmentFunction segmentFunction,
    UInt64   *offsetP,
    void     *segmentsP,
    UInt32   *numSegmentsP)
{
	return genIOVMSegments(kWalkClient, clientOutputSegment, (void *) segmentFunction,
	           offsetP, segmentsP, numSegmentsP);
}

bool
IODMACommand::OutputHost32(IODMACommand *,
    Segment64 segment, void *vSegList, UInt32 outSegIndex)
{
	Segment32 *base = (Segment32 *) vSegList;
	base[outSegIndex].fIOVMAddr = (UInt32) segment.fIOVMAddr;
	base[outSegIndex].fLength   = (UInt32) segment.fLength;
	return true;
}

bool
IODMACommand::OutputBig32(IODMACommand *,
    Segment64 segment, void *vSegList, UInt32 outSegIndex)
{
	const UInt offAddr = outSegIndex * sizeof(Segment32);
	const UInt offLen  = offAddr + sizeof(UInt32);
	OSWriteBigInt32(vSegList, offAddr, (UInt32) segment.fIOVMAddr);
	OSWriteBigInt32(vSegList, offLen, (UInt32) segment.fLength);
	return true;
}

bool
IODMACommand::OutputLittle32(IODMACommand *,
    Segment64 segment, void *vSegList, UInt32 outSegIndex)
{
	const UInt offAddr = outSegIndex * sizeof(Segment32);
	const UInt offLen  = offAddr + sizeof(UInt32);
	OSWriteLittleInt32(vSegList, offAddr, (UInt32) segment.fIOVMAddr);
	OSWriteLittleInt32(vSegList, offLen, (UInt32) segment.fLength);
	return true;
}

bool
IODMACommand::OutputHost64(IODMACommand *,
    Segment64 segment, void *vSegList, UInt32 outSegIndex)
{
	Segment64 *base = (Segment64 *) vSegList;
	base[outSegIndex] = segment;
	return true;
}

bool
IODMACommand::OutputBig64(IODMACommand *,
    Segment64 segment, void *vSegList, UInt32 outSegIndex)
{
	const UInt offAddr = outSegIndex * sizeof(Segment64);
	const UInt offLen  = offAddr + sizeof(UInt64);
	OSWriteBigInt64(vSegList, offAddr, (UInt64) segment.fIOVMAddr);
	OSWriteBigInt64(vSegList, offLen, (UInt64) segment.fLength);
	return true;
}

bool
IODMACommand::OutputLittle64(IODMACommand *,
    Segment64 segment, void *vSegList, UInt32 outSegIndex)
{
	const UInt offAddr = outSegIndex * sizeof(Segment64);
	const UInt offLen  = offAddr + sizeof(UInt64);
	OSWriteLittleInt64(vSegList, offAddr, (UInt64) segment.fIOVMAddr);
	OSWriteLittleInt64(vSegList, offLen, (UInt64) segment.fLength);
	return true;
}