X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/8ad349bb6ed4a0be06e34c92be0d98b92e078db4..89b3af67bb32e691275bf6fa803d1834b2284115:/iokit/Kernel/IOBufferMemoryDescriptor.cpp diff --git a/iokit/Kernel/IOBufferMemoryDescriptor.cpp b/iokit/Kernel/IOBufferMemoryDescriptor.cpp index 69c787546..4821b81dc 100644 --- a/iokit/Kernel/IOBufferMemoryDescriptor.cpp +++ b/iokit/Kernel/IOBufferMemoryDescriptor.cpp @@ -1,39 +1,39 @@ /* * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved. * - * @APPLE_LICENSE_OSREFERENCE_HEADER_START@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * - * This file contains Original Code and/or Modifications of Original Code - * as defined in and that are subject to the Apple Public Source License - * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. The rights granted to you under the - * License may not be used to create, or enable the creation or - * redistribution of, unlawful or unlicensed copies of an Apple operating - * system, or to circumvent, violate, or enable the circumvention or - * violation of, any terms of an Apple operating system software license - * agreement. - * - * Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this - * file. - * - * The Original Code and all software distributed under the License are - * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER - * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, - * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. - * Please see the License for the specific language governing rights and + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and * limitations under the License. - * - * @APPLE_LICENSE_OSREFERENCE_HEADER_END@ + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ #include #include #include +#include #include #include "IOKitKernelInternal.h" +#include "IOCopyMapper.h" __BEGIN_DECLS void ipc_port_release_send(ipc_port_t port); @@ -42,6 +42,12 @@ void ipc_port_release_send(ipc_port_t port); vm_map_t IOPageableMapForAddress( vm_address_t address ); __END_DECLS +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +volatile ppnum_t gIOHighestAllocatedPage; + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + #define super IOGeneralMemoryDescriptor OSDefineMetaClassAndStructors(IOBufferMemoryDescriptor, IOGeneralMemoryDescriptor); @@ -95,8 +101,20 @@ bool IOBufferMemoryDescriptor::initWithOptions( vm_size_t capacity, vm_offset_t alignment, task_t inTask) +{ + mach_vm_address_t physicalMask = 0; + return (initWithPhysicalMask(inTask, options, capacity, alignment, physicalMask)); +} + +bool IOBufferMemoryDescriptor::initWithPhysicalMask( + task_t inTask, + IOOptionBits options, + mach_vm_size_t capacity, + mach_vm_address_t alignment, + mach_vm_address_t physicalMask) { kern_return_t kr; + addr64_t lastIOAddr; vm_map_t vmmap = 0; IOOptionBits iomdOptions = kIOMemoryAsReference | kIOMemoryTypeVirtual; @@ -118,6 +136,12 @@ bool IOBufferMemoryDescriptor::initWithOptions( if ((inTask != kernel_task) && !(options & kIOMemoryPageable)) return false; + if (physicalMask && (alignment <= 1)) + alignment = ((physicalMask ^ PAGE_MASK) & PAGE_MASK) + 1; + + if ((options & kIOMemoryPhysicallyContiguous) && !physicalMask) + physicalMask = 0xFFFFFFFF; + _alignment = alignment; if (options & kIOMemoryPageable) { @@ -176,7 +200,7 @@ bool IOBufferMemoryDescriptor::initWithOptions( #if IOALLOCDEBUG debug_iomallocpageable_size += size; #endif - if ((NULL == inTask) && (options & kIOMemoryPageable)) + if (NULL == inTask) inTask = kernel_task; else if (inTask == kernel_task) { @@ -184,7 +208,6 @@ bool IOBufferMemoryDescriptor::initWithOptions( } else { - if( !reserved) { reserved = IONew( ExpansionData, 1 ); if( !reserved) @@ -195,23 +218,58 @@ bool IOBufferMemoryDescriptor::initWithOptions( reserved->map = vmmap; } } - else + else { - // @@@ gvdl: Need to remove this - // Buffer should never auto prepare they should be prepared explicitly - // But it never was enforced so what are you going to do? - iomdOptions |= kIOMemoryAutoPrepare; - - /* Allocate a wired-down buffer inside kernel space. */ - if (options & kIOMemoryPhysicallyContiguous) - _buffer = IOMallocContiguous(capacity, alignment, 0); - else if (alignment > 1) - _buffer = IOMallocAligned(capacity, alignment); + if (IOMapper::gSystem) + // assuming mapped space is 2G + lastIOAddr = (1UL << 31) - PAGE_SIZE; else - _buffer = IOMalloc(capacity); + lastIOAddr = ptoa_64(gIOHighestAllocatedPage); - if (!_buffer) - return false; + if (physicalMask && (lastIOAddr != (lastIOAddr & physicalMask))) + { + mach_vm_address_t address; + iomdOptions &= ~kIOMemoryTypeVirtual; + iomdOptions |= kIOMemoryTypePhysical; + + address = IOMallocPhysical(capacity, physicalMask); + _buffer = (void *) address; + if (!_buffer) + return false; + + if (inTask == kernel_task) + { + vmmap = kernel_map; + } + else if (NULL != inTask) + { + if( !reserved) { + reserved = IONew( ExpansionData, 1 ); + if( !reserved) + return( false ); + } + vmmap = get_task_map(inTask); + vm_map_reference(vmmap); + reserved->map = vmmap; + } + inTask = 0; + } + else + { + // Buffer shouldn't auto prepare they should be prepared explicitly + // But it never was enforced so what are you going to do? + iomdOptions |= kIOMemoryAutoPrepare; + + /* Allocate a wired-down buffer inside kernel space. */ + if (options & kIOMemoryPhysicallyContiguous) + _buffer = (void *) IOKernelAllocateContiguous(capacity, alignment); + else if (alignment > 1) + _buffer = IOMallocAligned(capacity, alignment); + else + _buffer = IOMalloc(capacity); + if (!_buffer) + return false; + } } _singleRange.v.address = (vm_address_t) _buffer; @@ -221,22 +279,55 @@ bool IOBufferMemoryDescriptor::initWithOptions( inTask, iomdOptions, /* System mapper */ 0)) return false; - if (options & kIOMemoryPageable) + if (physicalMask && !IOMapper::gSystem) { - kern_return_t kr; + IOMDDMACharacteristics mdSummary; + + bzero(&mdSummary, sizeof(mdSummary)); + IOReturn rtn = dmaCommandOperation( + kIOMDGetCharacteristics, + &mdSummary, sizeof(mdSummary)); + if (rtn) + return false; - if (vmmap) + if (mdSummary.fHighestPage) { - kr = doMap(vmmap, (IOVirtualAddress *) &_buffer, kIOMapAnywhere, 0, round_page_32(capacity)); - if (KERN_SUCCESS != kr) + ppnum_t highest; + while (mdSummary.fHighestPage > (highest = gIOHighestAllocatedPage)) { - _buffer = 0; - return( false ); + if (OSCompareAndSwap(highest, mdSummary.fHighestPage, + (UInt32 *) &gIOHighestAllocatedPage)) + break; } - _singleRange.v.address = (vm_address_t) _buffer; + lastIOAddr = ptoa_64(mdSummary.fHighestPage); + } + else + lastIOAddr = ptoa_64(gIOLastPage); + + if (lastIOAddr != (lastIOAddr & physicalMask)) + { + if (kIOMemoryTypePhysical != (_flags & kIOMemoryTypeMask)) + { + // flag a retry + _physSegCount = 1; + } + return false; } } + if (vmmap) + { + kr = doMap(vmmap, (IOVirtualAddress *) &_buffer, kIOMapAnywhere, 0, capacity); + if (KERN_SUCCESS != kr) + { + _buffer = 0; + return( false ); + } + + if (kIOMemoryTypeVirtual & iomdOptions) + _singleRange.v.address = (vm_address_t) _buffer; + } + setLength(capacity); return true; @@ -251,8 +342,44 @@ IOBufferMemoryDescriptor * IOBufferMemoryDescriptor::inTaskWithOptions( IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor; if (me && !me->initWithOptions(options, capacity, alignment, inTask)) { + bool retry = me->_physSegCount; me->release(); me = 0; + if (retry) + { + me = new IOBufferMemoryDescriptor; + if (me && !me->initWithOptions(options, capacity, alignment, inTask)) + { + me->release(); + me = 0; + } + } + } + return me; +} + +IOBufferMemoryDescriptor * IOBufferMemoryDescriptor::inTaskWithPhysicalMask( + task_t inTask, + IOOptionBits options, + mach_vm_size_t capacity, + mach_vm_address_t physicalMask) +{ + IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor; + + if (me && !me->initWithPhysicalMask(inTask, options, capacity, 1, physicalMask)) + { + bool retry = me->_physSegCount; + me->release(); + me = 0; + if (retry) + { + me = new IOBufferMemoryDescriptor; + if (me && !me->initWithPhysicalMask(inTask, options, capacity, 1, physicalMask)) + { + me->release(); + me = 0; + } + } } return me; } @@ -270,13 +397,7 @@ IOBufferMemoryDescriptor * IOBufferMemoryDescriptor::withOptions( vm_size_t capacity, vm_offset_t alignment) { - IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor; - - if (me && !me->initWithOptions(options, capacity, alignment, kernel_task)) { - me->release(); - me = 0; - } - return me; + return(IOBufferMemoryDescriptor::inTaskWithOptions(kernel_task, options, capacity, alignment)); } @@ -337,9 +458,21 @@ IOBufferMemoryDescriptor::withBytes(const void * inBytes, { IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor; - if (me && !me->initWithBytes(inBytes, inLength, inDirection, inContiguous)){ - me->release(); - me = 0; + if (me && !me->initWithBytes(inBytes, inLength, inDirection, inContiguous)) + { + bool retry = me->_physSegCount; + me->release(); + me = 0; + if (retry) + { + me = new IOBufferMemoryDescriptor; + if (me && !me->initWithBytes(inBytes, inLength, inDirection, inContiguous)) + { + me->release(); + me = 0; + } + } + } return me; } @@ -353,11 +486,13 @@ void IOBufferMemoryDescriptor::free() { // Cache all of the relevant information on the stack for use // after we call super::free()! - IOOptionBits options = _options; - vm_size_t size = _capacity; - void * buffer = _buffer; - vm_map_t vmmap = 0; - vm_offset_t alignment = _alignment; + IOOptionBits flags = _flags; + IOOptionBits options = _options; + vm_size_t size = _capacity; + void * buffer = _buffer; + IOVirtualAddress source = _singleRange.v.address; + vm_map_t vmmap = 0; + vm_offset_t alignment = _alignment; if (reserved) { @@ -384,8 +519,14 @@ void IOBufferMemoryDescriptor::free() } else if (buffer) { - if (options & kIOMemoryPhysicallyContiguous) - IOFreeContiguous(buffer, size); + if (kIOMemoryTypePhysical == (flags & kIOMemoryTypeMask)) + { + if (vmmap) + vm_deallocate(vmmap, (vm_address_t) buffer, round_page_32(size)); + IOFreePhysical((mach_vm_address_t) source, size); + } + else if (options & kIOMemoryPhysicallyContiguous) + IOKernelFreeContiguous((mach_vm_address_t) buffer, size); else if (alignment > 1) IOFreeAligned(buffer, size); else @@ -445,14 +586,21 @@ void IOBufferMemoryDescriptor::setDirection(IODirection direction) bool IOBufferMemoryDescriptor::appendBytes(const void * bytes, vm_size_t withLength) { - vm_size_t actualBytesToCopy = min(withLength, _capacity - _length); + vm_size_t actualBytesToCopy = min(withLength, _capacity - _length); + IOByteCount offset; assert(_length <= _capacity); - bcopy(/* from */ bytes, (void *)(_singleRange.v.address + _length), - actualBytesToCopy); + + offset = _length; _length += actualBytesToCopy; _singleRange.v.length += actualBytesToCopy; + if (_task == kernel_task) + bcopy(/* from */ bytes, (void *)(_singleRange.v.address + offset), + actualBytesToCopy); + else + writeBytes(offset, bytes, actualBytesToCopy); + return true; } @@ -463,9 +611,13 @@ IOBufferMemoryDescriptor::appendBytes(const void * bytes, vm_size_t withLength) */ void * IOBufferMemoryDescriptor::getBytesNoCopy() { - return (void *)_singleRange.v.address; + if (kIOMemoryTypePhysical == (_flags & kIOMemoryTypeMask)) + return _buffer; + else + return (void *)_singleRange.v.address; } + /* * getBytesNoCopy: * @@ -474,13 +626,30 @@ void * IOBufferMemoryDescriptor::getBytesNoCopy() void * IOBufferMemoryDescriptor::getBytesNoCopy(vm_size_t start, vm_size_t withLength) { - if (start < _length && (start + withLength) <= _length) - return (void *)(_singleRange.v.address + start); + IOVirtualAddress address; + if (kIOMemoryTypePhysical == (_flags & kIOMemoryTypeMask)) + address = (IOVirtualAddress) _buffer; + else + address = _singleRange.v.address; + + if (start < _length && (start + withLength) <= _length) + return (void *)(address + start); return 0; } +/* DEPRECATED */ void * IOBufferMemoryDescriptor::getVirtualSegment(IOByteCount offset, +/* DEPRECATED */ IOByteCount * lengthOfSegment) +{ + void * bytes = getBytesNoCopy(offset, 0); + + if (bytes && lengthOfSegment) + *lengthOfSegment = _length - offset; + + return bytes; +} + OSMetaClassDefineReservedUsed(IOBufferMemoryDescriptor, 0); -OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 1); +OSMetaClassDefineReservedUsed(IOBufferMemoryDescriptor, 1); OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 2); OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 3); OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 4);