X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/b0d623f7f2ae71ed96e60569f61f9a9a27016e80..440d4c6cfad24426bfddca7518f16c17f4e443f7:/iokit/Kernel/IOBufferMemoryDescriptor.cpp diff --git a/iokit/Kernel/IOBufferMemoryDescriptor.cpp b/iokit/Kernel/IOBufferMemoryDescriptor.cpp index bca55b535..a56b469ee 100644 --- a/iokit/Kernel/IOBufferMemoryDescriptor.cpp +++ b/iokit/Kernel/IOBufferMemoryDescriptor.cpp @@ -37,7 +37,6 @@ #include #include "IOKitKernelInternal.h" -#include "IOCopyMapper.h" __BEGIN_DECLS void ipc_port_release_send(ipc_port_t port); @@ -49,11 +48,10 @@ __END_DECLS enum { - kInternalFlagRealloc = 0x00000001, + kInternalFlagPhysical = 0x00000001, + kInternalFlagPageSized = 0x00000002 }; -volatile ppnum_t gIOHighestAllocatedPage; - /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #define super IOGeneralMemoryDescriptor @@ -84,9 +82,7 @@ bool IOBufferMemoryDescriptor::initWithPhysicalMask( kern_return_t kr; task_t mapTask = NULL; vm_map_t vmmap = NULL; - addr64_t lastIOAddr; mach_vm_address_t highestMask = 0; - bool usePhys; IOOptionBits iomdOptions = kIOMemoryTypeVirtual64 | kIOMemoryAsReference; if (!capacity) @@ -103,6 +99,8 @@ bool IOBufferMemoryDescriptor::initWithPhysicalMask( return (false); _ranges.v64->address = 0; _ranges.v64->length = 0; + // make sure super::free doesn't dealloc _ranges before super::init + _flags = kIOMemoryAsReference; // Grab IOMD bits from the Buffer MD options iomdOptions |= (options & kIOBufferDescriptorMemoryFlags); @@ -112,9 +110,11 @@ bool IOBufferMemoryDescriptor::initWithPhysicalMask( alignment = ((physicalMask ^ (-1ULL)) & (physicalMask - 1)); highestMask = (physicalMask | alignment); alignment++; + if (alignment < page_size) + alignment = page_size; } - if ((options & (kIOMemorySharingTypeMask | kIOMapCacheMask)) && (alignment < page_size)) + if ((options & (kIOMemorySharingTypeMask | kIOMapCacheMask | kIOMemoryClearEncrypt)) && (alignment < page_size)) alignment = page_size; if (alignment >= page_size) @@ -128,9 +128,6 @@ bool IOBufferMemoryDescriptor::initWithPhysicalMask( if ((inTask != kernel_task) && !(options & kIOMemoryPageable)) return false; - if ((options & kIOMemoryPhysicallyContiguous) && !physicalMask) - physicalMask = 0xFFFFFFFF; - // set flags for entry + object create vm_prot_t memEntryCacheMode = VM_PROT_READ | VM_PROT_WRITE; @@ -153,6 +150,10 @@ bool IOBufferMemoryDescriptor::initWithPhysicalMask( SET_MAP_MEM(MAP_MEM_COPYBACK, memEntryCacheMode); break; + case kIOMapCopybackInnerCache: + SET_MAP_MEM(MAP_MEM_INNERWBACK, memEntryCacheMode); + break; + case kIOMapDefaultCache: default: SET_MAP_MEM(MAP_MEM_NOOP, memEntryCacheMode); @@ -174,63 +175,41 @@ bool IOBufferMemoryDescriptor::initWithPhysicalMask( else { memEntryCacheMode |= MAP_MEM_NAMED_REUSE; + vmmap = kernel_map; - if (IOMapper::gSystem) - // assuming mapped space is 2G - lastIOAddr = (1UL << 31) - PAGE_SIZE; - else - lastIOAddr = ptoa_64(gIOHighestAllocatedPage); + // Buffer shouldn't auto prepare they should be prepared explicitly + // But it never was enforced so what are you going to do? + iomdOptions |= kIOMemoryAutoPrepare; - usePhys = (highestMask && (lastIOAddr != (lastIOAddr & highestMask)) - && (alignment <= page_size)); + /* Allocate a wired-down buffer inside kernel space. */ - if (!usePhys && (options & kIOMemoryPhysicallyContiguous)) + if ((options & kIOMemoryPhysicallyContiguous) || highestMask || (alignment > page_size)) { - _buffer = (void *) IOKernelAllocateContiguous(capacity, highestMask, alignment); - usePhys = (NULL == _buffer); + _internalFlags |= kInternalFlagPhysical; + if (highestMask) + { + _internalFlags |= kInternalFlagPageSized; + capacity = round_page(capacity); + } + _buffer = (void *) IOKernelAllocateWithPhysicalRestrict(capacity, highestMask, alignment, + (0 != (options & kIOMemoryPhysicallyContiguous))); } - if (usePhys) + else if (alignment > 1) { - mach_vm_address_t address; - iomdOptions &= ~kIOMemoryTypeVirtual64; - iomdOptions |= kIOMemoryTypePhysical64; - - address = IOMallocPhysical(capacity, highestMask); - _buffer = (void *) address; - if (!_buffer) - return false; - - mapTask = inTask; - inTask = 0; + _buffer = IOMallocAligned(capacity, alignment); } else { - vmmap = kernel_map; - - // Buffer shouldn't auto prepare they should be prepared explicitly - // But it never was enforced so what are you going to do? - iomdOptions |= kIOMemoryAutoPrepare; + _buffer = IOMalloc(capacity); + } - /* Allocate a wired-down buffer inside kernel space. */ - if (options & kIOMemoryPhysicallyContiguous) - { - // attempted allocate already - } - else if (alignment > 1) - { - _buffer = IOMallocAligned(capacity, alignment); - } - else - { - _buffer = IOMalloc(capacity); - } - if (!_buffer) - return false; + if (!_buffer) + { + return false; } } - if( (kIOMemoryTypePhysical64 != (kIOMemoryTypeMask & iomdOptions)) - && (options & (kIOMemoryPageable | kIOMapCacheMask))) { + if( (options & (kIOMemoryPageable | kIOMapCacheMask))) { ipc_port_t sharedMem; vm_size_t size = round_page(capacity); @@ -277,42 +256,6 @@ bool IOBufferMemoryDescriptor::initWithPhysicalMask( inTask, iomdOptions, /* System mapper */ 0)) return false; - if (highestMask && !IOMapper::gSystem) - { - IOMDDMACharacteristics mdSummary; - - bzero(&mdSummary, sizeof(mdSummary)); - IOReturn rtn = dmaCommandOperation( - kIOMDGetCharacteristics, - &mdSummary, sizeof(mdSummary)); - if (rtn) - return false; - - if (mdSummary.fHighestPage) - { - ppnum_t highest; - while (mdSummary.fHighestPage > (highest = gIOHighestAllocatedPage)) - { - if (OSCompareAndSwap(highest, mdSummary.fHighestPage, - (UInt32 *) &gIOHighestAllocatedPage)) - break; - } - lastIOAddr = ptoa_64(mdSummary.fHighestPage); - } - else - lastIOAddr = ptoa_64(gIOLastPage); - - if (lastIOAddr != (lastIOAddr & highestMask)) - { - if (kIOMemoryTypePhysical64 != (_flags & kIOMemoryTypeMask)) - { - // flag a retry - _internalFlags |= kInternalFlagRealloc; - } - return false; - } - } - if (mapTask) { if (!reserved) { @@ -350,18 +293,8 @@ IOBufferMemoryDescriptor * IOBufferMemoryDescriptor::inTaskWithOptions( IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor; if (me && !me->initWithPhysicalMask(inTask, options, capacity, alignment, 0)) { - bool retry = (0 != (kInternalFlagRealloc & me->_internalFlags)); me->release(); me = 0; - if (retry) - { - me = new IOBufferMemoryDescriptor; - if (me && !me->initWithPhysicalMask(inTask, options, capacity, alignment, 0)) - { - me->release(); - me = 0; - } - } } return me; } @@ -376,18 +309,8 @@ IOBufferMemoryDescriptor * IOBufferMemoryDescriptor::inTaskWithPhysicalMask( if (me && !me->initWithPhysicalMask(inTask, options, capacity, 1, physicalMask)) { - bool retry = (0 != (kInternalFlagRealloc & me->_internalFlags)); me->release(); me = 0; - if (retry) - { - me = new IOBufferMemoryDescriptor; - if (me && !me->initWithPhysicalMask(inTask, options, capacity, 1, physicalMask)) - { - me->release(); - me = 0; - } - } } return me; } @@ -410,18 +333,8 @@ IOBufferMemoryDescriptor * IOBufferMemoryDescriptor::withOptions( IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor; if (me && !me->initWithPhysicalMask(kernel_task, options, capacity, alignment, 0)) { - bool retry = (0 != (kInternalFlagRealloc & me->_internalFlags)); me->release(); me = 0; - if (retry) - { - me = new IOBufferMemoryDescriptor; - if (me && !me->initWithPhysicalMask(kernel_task, options, capacity, alignment, 0)) - { - me->release(); - me = 0; - } - } } return me; } @@ -490,22 +403,8 @@ IOBufferMemoryDescriptor::withBytes(const void * inBytes, | (inContiguous ? kIOMemoryPhysicallyContiguous : 0), inLength, inLength, 0 )) { - bool retry = (0 != (kInternalFlagRealloc & me->_internalFlags)); me->release(); me = 0; - if (retry) - { - me = new IOBufferMemoryDescriptor; - if (me && !me->initWithPhysicalMask( - kernel_task, inDirection | kIOMemoryUnshared - | (inContiguous ? kIOMemoryPhysicallyContiguous : 0), - inLength, inLength, 0 )) - { - me->release(); - me = 0; - } - } - } if (me) @@ -531,13 +430,13 @@ void IOBufferMemoryDescriptor::free() { // Cache all of the relevant information on the stack for use // after we call super::free()! - IOOptionBits flags = _flags; + IOOptionBits flags = _flags; + IOOptionBits internalFlags = _internalFlags; IOOptionBits options = _options; vm_size_t size = _capacity; void * buffer = _buffer; IOMemoryMap * map = 0; IOAddressRange * range = _ranges.v64; - mach_vm_address_t source = range ? range->address : 0; vm_offset_t alignment = _alignment; if (alignment >= page_size) @@ -562,10 +461,12 @@ void IOBufferMemoryDescriptor::free() } else if (buffer) { - if (kIOMemoryTypePhysical64 == (flags & kIOMemoryTypeMask)) - IOFreePhysical(source, size); - else if (options & kIOMemoryPhysicallyContiguous) - IOKernelFreeContiguous((mach_vm_address_t) buffer, size); + if (internalFlags & kInternalFlagPhysical) + { + if (kInternalFlagPageSized & internalFlags) + size = round_page(size); + IOKernelFreePhysical((mach_vm_address_t) buffer, size); + } else if (alignment > 1) IOFreeAligned(buffer, size); else