xnu-4570.1.46.tar.gz

[apple/xnu.git] / iokit / Kernel / IOBufferMemoryDescriptor.cpp

diff --git a/iokit/Kernel/IOBufferMemoryDescriptor.cpp b/iokit/Kernel/IOBufferMemoryDescriptor.cpp
index 4821b81dc39980046bbf1d7c1b5d564ebad7eb3f..b3ff13ddaed3dfc816b0447ab573c5ade716ce0f 100644 (file)
--- a/iokit/Kernel/IOBufferMemoryDescriptor.cpp
+++ b/iokit/Kernel/IOBufferMemoryDescriptor.cpp
@@ -25,26 +25,50 @@
  * 
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */
  * 
  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  */

+
+#define _IOMEMORYDESCRIPTOR_INTERNAL_
+

 #include <IOKit/assert.h>
 #include <IOKit/system.h>
 
 #include <IOKit/IOLib.h>
 #include <IOKit/IOMapper.h>
 #include <IOKit/IOBufferMemoryDescriptor.h>
 #include <IOKit/assert.h>
 #include <IOKit/system.h>
 
 #include <IOKit/IOLib.h>
 #include <IOKit/IOMapper.h>
 #include <IOKit/IOBufferMemoryDescriptor.h>

+#include <libkern/OSDebug.h>
+#include <mach/mach_vm.h>

 
 #include "IOKitKernelInternal.h"
 
 #include "IOKitKernelInternal.h"

-#include "IOCopyMapper.h"
+
+#ifdef IOALLOCDEBUG
+#include <libkern/c++/OSCPPDebug.h>
+#endif
+#include <IOKit/IOStatisticsPrivate.h>
+
+#if IOKITSTATS
+#define IOStatisticsAlloc(type, size) \
+do { \
+       IOStatistics::countAlloc(type, size); \
+} while (0)
+#else
+#define IOStatisticsAlloc(type, size)
+#endif /* IOKITSTATS */
+

 
 __BEGIN_DECLS
 void ipc_port_release_send(ipc_port_t port);
 #include <vm/pmap.h>
 
 
 __BEGIN_DECLS
 void ipc_port_release_send(ipc_port_t port);
 #include <vm/pmap.h>
 

-vm_map_t IOPageableMapForAddress( vm_address_t address );

 __END_DECLS
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 __END_DECLS
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 

-volatile ppnum_t gIOHighestAllocatedPage;
+enum
+{
+    kInternalFlagPhysical      = 0x00000001,
+    kInternalFlagPageSized     = 0x00000002,
+    kInternalFlagPageAllocated = 0x00000004,
+    kInternalFlagInit          = 0x00000008
+};

 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 


@@ -52,50 +76,26 @@ volatile ppnum_t gIOHighestAllocatedPage;
 OSDefineMetaClassAndStructors(IOBufferMemoryDescriptor,
                                IOGeneralMemoryDescriptor);
 
 OSDefineMetaClassAndStructors(IOBufferMemoryDescriptor,
                                IOGeneralMemoryDescriptor);
 

-bool IOBufferMemoryDescriptor::initWithAddress(
-                                  void *      /* address       */ ,
-                                  IOByteCount /* withLength    */ ,
-                                  IODirection /* withDirection */ )
-{
-    return false;
-}
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

 
 

-bool IOBufferMemoryDescriptor::initWithAddress(
-                                  vm_address_t /* address       */ ,
-                                  IOByteCount  /* withLength    */ ,
-                                  IODirection  /* withDirection */ ,
-                                  task_t       /* withTask      */ )
+static uintptr_t IOBMDPageProc(iopa_t * a)

 {
 {

-    return false;
-}
+    kern_return_t kr;
+    vm_address_t  vmaddr  = 0;
+    int           options = 0; // KMA_LOMEM;

 
 

-bool IOBufferMemoryDescriptor::initWithPhysicalAddress(
-                                  IOPhysicalAddress /* address       */ ,
-                                  IOByteCount       /* withLength    */ ,
-                                  IODirection       /* withDirection */ )
-{
-    return false;
-}
+    kr = kernel_memory_allocate(kernel_map, &vmaddr,
+                               page_size, 0, options, VM_KERN_MEMORY_IOKIT);

 
 

-bool IOBufferMemoryDescriptor::initWithPhysicalRanges(
-                                  IOPhysicalRange * /* ranges        */ ,
-                                  UInt32            /* withCount     */ ,
-                                  IODirection       /* withDirection */ ,
-                                  bool              /* asReference   */ )
-{
-    return false;
-}
+    if (KERN_SUCCESS != kr) vmaddr = 0;
+    else                   bzero((void *) vmaddr, page_size);

 
 

-bool IOBufferMemoryDescriptor::initWithRanges(
-                                  IOVirtualRange * /* ranges        */ ,
-                                  UInt32           /* withCount     */ ,
-                                  IODirection      /* withDirection */ ,
-                                  task_t           /* withTask      */ ,
-                                  bool             /* asReference   */ )
-{
-    return false;
+    return ((uintptr_t) vmaddr);

 }
 
 }
 

+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+#ifndef __LP64__

 bool IOBufferMemoryDescriptor::initWithOptions(
                                IOOptionBits options,
                                vm_size_t    capacity,
 bool IOBufferMemoryDescriptor::initWithOptions(
                                IOOptionBits options,
                                vm_size_t    capacity,


@@ -105,6 +105,7 @@ bool IOBufferMemoryDescriptor::initWithOptions(
     mach_vm_address_t physicalMask = 0;
     return (initWithPhysicalMask(inTask, options, capacity, alignment, physicalMask));
 }
     mach_vm_address_t physicalMask = 0;
     return (initWithPhysicalMask(inTask, options, capacity, alignment, physicalMask));
 }

+#endif /* !__LP64__ */

 
 bool IOBufferMemoryDescriptor::initWithPhysicalMask(
                                task_t            inTask,
 
 bool IOBufferMemoryDescriptor::initWithPhysicalMask(
                                task_t            inTask,


@@ -113,223 +114,217 @@ bool IOBufferMemoryDescriptor::initWithPhysicalMask(
                                mach_vm_address_t alignment,
                                mach_vm_address_t physicalMask)
 {
                                mach_vm_address_t alignment,
                                mach_vm_address_t physicalMask)
 {

-    kern_return_t      kr;
-    addr64_t            lastIOAddr;
-    vm_map_t           vmmap = 0;
-    IOOptionBits       iomdOptions = kIOMemoryAsReference | kIOMemoryTypeVirtual;
+    task_t               mapTask = NULL;
+    vm_map_t             vmmap = NULL;
+    mach_vm_address_t     highestMask = 0;
+    IOOptionBits         iomdOptions = kIOMemoryTypeVirtual64 | kIOMemoryAsReference;
+    IODMAMapSpecification mapSpec;
+    bool                  mapped = false;
+    bool                  needZero;
+
+    if (!capacity) return false;
+
+    _options                 = options;
+    _capacity         = capacity;
+    _internalFlags    = 0;
+    _internalReserved = 0;
+    _buffer          = 0;
+
+    _ranges.v64 = IONew(IOAddressRange, 1);
+    if (!_ranges.v64)
+       return (false);
+    _ranges.v64->address = 0;
+    _ranges.v64->length  = 0;
+    //  make sure super::free doesn't dealloc _ranges before super::init
+    _flags = kIOMemoryAsReference;
+
+    // Grab IOMD bits from the Buffer MD options
+    iomdOptions  |= (options & kIOBufferDescriptorMemoryFlags);
+
+    if (!(kIOMemoryMapperNone & options))
+    {
+       IOMapper::checkForSystemMapper();
+       mapped = (0 != IOMapper::gSystem);
+    }
+    needZero = (mapped || (0 != (kIOMemorySharingTypeMask & options)));

 
 

-    if (!capacity)
-        return false;
+    if (physicalMask && (alignment <= 1))
+    {
+       alignment   = ((physicalMask ^ (-1ULL)) & (physicalMask - 1));
+       highestMask = (physicalMask | alignment);
+       alignment++;
+       if (alignment < page_size)
+            alignment = page_size;
+    }

 
 

-    _options             = options;
-    _capacity     = capacity;
-    _physAddrs    = 0;
-    _physSegCount = 0;
-    _buffer      = 0;
+    if ((options & (kIOMemorySharingTypeMask | kIOMapCacheMask | kIOMemoryClearEncrypt)) && (alignment < page_size))
+       alignment = page_size;

 
 

-    // Grab the direction and the Auto Prepare bits from the Buffer MD options
-    iomdOptions  |= options & (kIOMemoryDirectionMask | kIOMemoryAutoPrepare);
+    if (alignment >= page_size)
+       capacity = round_page(capacity);

 
 

-    if ((options & kIOMemorySharingTypeMask) && (alignment < page_size))
-        alignment = page_size;
+    if (alignment > page_size)
+       options |= kIOMemoryPhysicallyContiguous;

 
 

-    if ((inTask != kernel_task) && !(options & kIOMemoryPageable))
-        return false;
+    _alignment = alignment;

 
 

-    if (physicalMask && (alignment <= 1))
-       alignment = ((physicalMask ^ PAGE_MASK) & PAGE_MASK) + 1;
+    if ((capacity + alignment) < _capacity) return (false);

 
 

-    if ((options & kIOMemoryPhysicallyContiguous) && !physicalMask)
-       physicalMask = 0xFFFFFFFF;
+    if ((inTask != kernel_task) && !(options & kIOMemoryPageable))
+       return false;

 
 

-    _alignment = alignment;
-    if (options & kIOMemoryPageable)
+    bzero(&mapSpec, sizeof(mapSpec));
+    mapSpec.alignment      = _alignment;
+    mapSpec.numAddressBits = 64;
+    if (highestMask && mapped)

     {
     {

-        iomdOptions |= kIOMemoryBufferPageable;
+       if (highestMask <= 0xFFFFFFFF)
+           mapSpec.numAddressBits = (32 - __builtin_clz((unsigned int) highestMask));
+       else
+           mapSpec.numAddressBits = (64 - __builtin_clz((unsigned int) (highestMask >> 32)));
+       highestMask = 0;
+    }

 
 

-        ipc_port_t sharedMem;
-        vm_size_t size = round_page_32(capacity);
+    // set memory entry cache mode, pageable, purgeable
+    iomdOptions |= ((options & kIOMapCacheMask) >> kIOMapCacheShift) << kIOMemoryBufferCacheShift;
+    if (options & kIOMemoryPageable)
+    {
+       iomdOptions |= kIOMemoryBufferPageable;
+       if (options & kIOMemoryPurgeable) iomdOptions |= kIOMemoryBufferPurgeable;
+    }
+    else
+    {
+       vmmap = kernel_map;

 
 

-        // must create the entry before any pages are allocated
+       // Buffer shouldn't auto prepare they should be prepared explicitly
+       // But it never was enforced so what are you going to do?
+       iomdOptions |= kIOMemoryAutoPrepare;

 
 

-       // set flags for entry + object create
-       vm_prot_t memEntryCacheMode = VM_PROT_READ | VM_PROT_WRITE
-                                   | MAP_MEM_NAMED_CREATE;
+       /* Allocate a wired-down buffer inside kernel space. */

 
 

-        if (options & kIOMemoryPurgeable)
-            memEntryCacheMode |= MAP_MEM_PURGABLE;
+       bool contig = (0 != (options & kIOMemoryHostPhysicallyContiguous));

 
 

-       // set memory entry cache mode
-       switch (options & kIOMapCacheMask)
+       if (!contig && (0 != (options & kIOMemoryPhysicallyContiguous)))

        {
        {

-           case kIOMapInhibitCache:
-               SET_MAP_MEM(MAP_MEM_IO, memEntryCacheMode);
-               break;
-    
-           case kIOMapWriteThruCache:
-               SET_MAP_MEM(MAP_MEM_WTHRU, memEntryCacheMode);
-               break;
-
-           case kIOMapWriteCombineCache:
-               SET_MAP_MEM(MAP_MEM_WCOMB, memEntryCacheMode);
-               break;
-
-           case kIOMapCopybackCache:
-               SET_MAP_MEM(MAP_MEM_COPYBACK, memEntryCacheMode);
-               break;
-
-           case kIOMapDefaultCache:
-           default:
-               SET_MAP_MEM(MAP_MEM_NOOP, memEntryCacheMode);
-               break;
+           contig |= (!mapped);
+           contig |= (0 != (kIOMemoryMapperNone & options));
+#if 0
+           // treat kIOMemoryPhysicallyContiguous as kIOMemoryHostPhysicallyContiguous for now
+           contig |= true;
+#endif

        }
 
        }
 

-       kr = mach_make_memory_entry( vmmap,
-                   &size, 0,
-                   memEntryCacheMode, &sharedMem,
-                   NULL );
-
-       if( (KERN_SUCCESS == kr) && (size != round_page_32(capacity))) {
-           ipc_port_release_send( sharedMem );
-           kr = kIOReturnVMError;
+       if (contig || highestMask || (alignment > page_size))
+       {
+            _internalFlags |= kInternalFlagPhysical;
+            if (highestMask)
+            {
+                _internalFlags |= kInternalFlagPageSized;
+                capacity = round_page(capacity);
+            }
+            _buffer = (void *) IOKernelAllocateWithPhysicalRestrict(
+                                       capacity, highestMask, alignment, contig);

        }
        }

-       if( KERN_SUCCESS != kr)
-           return( false );
-
-       _memEntry = (void *) sharedMem;
+       else if (needZero
+                 && ((capacity + alignment) <= (page_size - gIOPageAllocChunkBytes)))
+       {
+            _internalFlags |= kInternalFlagPageAllocated;
+            needZero        = false;
+            _buffer         = (void *) iopa_alloc(&gIOBMDPageAllocator, &IOBMDPageProc, capacity, alignment);
+           if (_buffer)
+           {
+               IOStatisticsAlloc(kIOStatisticsMallocAligned, capacity);

 #if IOALLOCDEBUG
 #if IOALLOCDEBUG

-       debug_iomallocpageable_size += size;
+               OSAddAtomic(capacity, &debug_iomalloc_size);

 #endif
 #endif

-       if (NULL == inTask)
-           inTask = kernel_task;
-       else if (inTask == kernel_task)
+           }
+       }
+       else if (alignment > 1)

        {
        {

-           vmmap = kernel_map;
+            _buffer = IOMallocAligned(capacity, alignment);

        }
        else
        {
        }
        else
        {

-           if( !reserved) {
-               reserved = IONew( ExpansionData, 1 );
-               if( !reserved)
-                   return( false );
-           }
-           vmmap = get_task_map(inTask);
-           vm_map_reference(vmmap);
-           reserved->map = vmmap;
+            _buffer = IOMalloc(capacity);
+       }
+       if (!_buffer)
+       {
+            return false;

        }
        }

+       if (needZero) bzero(_buffer, capacity);

     }
     }

-    else
-    {
-       if (IOMapper::gSystem)
-           // assuming mapped space is 2G
-           lastIOAddr = (1UL << 31) - PAGE_SIZE;
-       else
-           lastIOAddr = ptoa_64(gIOHighestAllocatedPage);

 
 

-       if (physicalMask && (lastIOAddr != (lastIOAddr & physicalMask)))
-       {
-           mach_vm_address_t address;
-           iomdOptions &= ~kIOMemoryTypeVirtual;
-           iomdOptions |= kIOMemoryTypePhysical;
+    if( (options & (kIOMemoryPageable | kIOMapCacheMask))) {
+       vm_size_t       size = round_page(capacity);

 
 

-           address = IOMallocPhysical(capacity, physicalMask);
-           _buffer = (void *) address;
-           if (!_buffer)
-               return false;
+       // initWithOptions will create memory entry
+       iomdOptions |= kIOMemoryPersistent;

 
 

-           if (inTask == kernel_task)
-           {
-               vmmap = kernel_map;
-           }
-           else if (NULL != inTask)
-           {
-               if( !reserved) {
-                   reserved = IONew( ExpansionData, 1 );
-                   if( !reserved)
-                       return( false );
-               }
-               vmmap = get_task_map(inTask);
-               vm_map_reference(vmmap);
-               reserved->map = vmmap;
-           }
-           inTask = 0;
+       if( options & kIOMemoryPageable) {
+#if IOALLOCDEBUG
+           OSAddAtomicLong(size, &debug_iomallocpageable_size);
+#endif
+           mapTask = inTask;
+           if (NULL == inTask)
+               inTask = kernel_task;

        }
        }

-       else
+       else if (options & kIOMapCacheMask)

        {
        {

-           // Buffer shouldn't auto prepare they should be prepared explicitly
-           // But it never was enforced so what are you going to do?
-           iomdOptions |= kIOMemoryAutoPrepare;
-
-           /* Allocate a wired-down buffer inside kernel space. */
-           if (options & kIOMemoryPhysicallyContiguous)
-               _buffer = (void *) IOKernelAllocateContiguous(capacity, alignment);
-           else if (alignment > 1)
-               _buffer = IOMallocAligned(capacity, alignment);
-           else
-               _buffer = IOMalloc(capacity);
-           if (!_buffer)
-               return false;
+           // Prefetch each page to put entries into the pmap
+           volatile UInt8 *    startAddr = (UInt8 *)_buffer;
+           volatile UInt8 *    endAddr   = (UInt8 *)_buffer + capacity;
+
+           while (startAddr < endAddr)
+           {
+               UInt8 dummyVar = *startAddr;
+               (void) dummyVar;
+               startAddr += page_size;
+           }

        }
     }
 
        }
     }
 

-    _singleRange.v.address = (vm_address_t) _buffer;
-    _singleRange.v.length  = capacity;
+    _ranges.v64->address = (mach_vm_address_t) _buffer;;
+    _ranges.v64->length  = _capacity;

 
 

-    if (!super::initWithOptions(&_singleRange.v, 1, 0,
-                               inTask, iomdOptions, /* System mapper */ 0))
+    if (!super::initWithOptions(_ranges.v64, 1, 0,
+                               inTask, iomdOptions, /* System mapper */ 0))

        return false;
 
        return false;
 

-    if (physicalMask && !IOMapper::gSystem)
-    {
-       IOMDDMACharacteristics mdSummary;
-
-       bzero(&mdSummary, sizeof(mdSummary));
-       IOReturn rtn = dmaCommandOperation(
-               kIOMDGetCharacteristics,
-               &mdSummary, sizeof(mdSummary));
-       if (rtn)
-           return false;
-
-       if (mdSummary.fHighestPage)
-       {
-           ppnum_t highest;
-           while (mdSummary.fHighestPage > (highest = gIOHighestAllocatedPage))
-           {
-               if (OSCompareAndSwap(highest, mdSummary.fHighestPage, 
-                                       (UInt32 *) &gIOHighestAllocatedPage))
-                   break;
-           }
-           lastIOAddr = ptoa_64(mdSummary.fHighestPage);
-       }
-       else
-           lastIOAddr = ptoa_64(gIOLastPage);
+    _internalFlags |= kInternalFlagInit;
+#if IOTRACKING
+    if (!(options & kIOMemoryPageable))        trackingAccumSize(capacity);
+#endif /* IOTRACKING */

 
 

-       if (lastIOAddr != (lastIOAddr & physicalMask))
-       {
-           if (kIOMemoryTypePhysical != (_flags & kIOMemoryTypeMask))
-           {
-               // flag a retry
-               _physSegCount = 1;
-           }
-           return false;
-       }
-    }
+    // give any system mapper the allocation params
+    if (kIOReturnSuccess != dmaCommandOperation(kIOMDAddDMAMapSpec, 
+                                               &mapSpec, sizeof(mapSpec)))
+       return false;

 
 

-    if (vmmap)
+    if (mapTask)

     {
     {

-       kr = doMap(vmmap, (IOVirtualAddress *) &_buffer, kIOMapAnywhere, 0, capacity);
-       if (KERN_SUCCESS != kr)
+       if (!reserved) {
+           reserved = IONew( ExpansionData, 1 );
+           if( !reserved)
+               return( false );
+       }
+       reserved->map = createMappingInTask(mapTask, 0, 
+                           kIOMapAnywhere | (options & kIOMapPrefault) | (options & kIOMapCacheMask), 0, 0);
+       if (!reserved->map)

        {
            _buffer = 0;
            return( false );
        }
        {
            _buffer = 0;
            return( false );
        }

-
-       if (kIOMemoryTypeVirtual & iomdOptions)
-           _singleRange.v.address = (vm_address_t) _buffer;
+       release();          // map took a retain on this
+       reserved->map->retain();
+       removeMapping(reserved->map);
+       mach_vm_address_t buffer = reserved->map->getAddress();
+       _buffer = (void *) buffer;
+       if (kIOMemoryTypeVirtual64 == (kIOMemoryTypeMask & iomdOptions))
+           _ranges.v64->address = buffer;

     }
 
     }
 

-    setLength(capacity);
-    
+    setLength(_capacity);
+

     return true;
 }
 
     return true;
 }
 


@@ -341,19 +336,9 @@ IOBufferMemoryDescriptor * IOBufferMemoryDescriptor::inTaskWithOptions(
 {
     IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor;
     
 {
     IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor;
     

-    if (me && !me->initWithOptions(options, capacity, alignment, inTask)) {
-       bool retry = me->_physSegCount;
+    if (me && !me->initWithPhysicalMask(inTask, options, capacity, alignment, 0)) {

        me->release();
        me = 0;
        me->release();
        me = 0;

-       if (retry)
-       {
-           me = new IOBufferMemoryDescriptor;
-           if (me && !me->initWithOptions(options, capacity, alignment, inTask))
-           {
-               me->release();
-               me = 0;
-           }
-       }

     }
     return me;
 }
     }
     return me;
 }


@@ -368,36 +353,34 @@ IOBufferMemoryDescriptor * IOBufferMemoryDescriptor::inTaskWithPhysicalMask(
     
     if (me && !me->initWithPhysicalMask(inTask, options, capacity, 1, physicalMask))
     {
     
     if (me && !me->initWithPhysicalMask(inTask, options, capacity, 1, physicalMask))
     {

-       bool retry = me->_physSegCount;

        me->release();
        me = 0;
        me->release();
        me = 0;

-       if (retry)
-       {
-           me = new IOBufferMemoryDescriptor;
-           if (me && !me->initWithPhysicalMask(inTask, options, capacity, 1, physicalMask))
-           {
-               me->release();
-               me = 0;
-           }
-       }

     }
     return me;
 }
 
     }
     return me;
 }
 

+#ifndef __LP64__

 bool IOBufferMemoryDescriptor::initWithOptions(
                                IOOptionBits options,
                                vm_size_t    capacity,
                                vm_offset_t  alignment)
 {
 bool IOBufferMemoryDescriptor::initWithOptions(
                                IOOptionBits options,
                                vm_size_t    capacity,
                                vm_offset_t  alignment)
 {

-    return( initWithOptions(options, capacity, alignment, kernel_task) );
+    return (initWithPhysicalMask(kernel_task, options, capacity, alignment, (mach_vm_address_t)0));

 }
 }

+#endif /* !__LP64__ */

 
 IOBufferMemoryDescriptor * IOBufferMemoryDescriptor::withOptions(
                                             IOOptionBits options,
                                             vm_size_t    capacity,
                                             vm_offset_t  alignment)
 {
 
 IOBufferMemoryDescriptor * IOBufferMemoryDescriptor::withOptions(
                                             IOOptionBits options,
                                             vm_size_t    capacity,
                                             vm_offset_t  alignment)
 {

-    return(IOBufferMemoryDescriptor::inTaskWithOptions(kernel_task, options, capacity, alignment));
+    IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor;
+    
+    if (me && !me->initWithPhysicalMask(kernel_task, options, capacity, alignment, 0)) {
+       me->release();
+       me = 0;
+    }
+    return me;

 }
 
 
 }
 
 


@@ -418,6 +401,7 @@ IOBufferMemoryDescriptor::withCapacity(vm_size_t   inCapacity,
                inCapacity, inContiguous ? inCapacity : 1 ));
 }
 
                inCapacity, inContiguous ? inCapacity : 1 ));
 }
 

+#ifndef __LP64__

 /*
  * initWithBytes:
  *
 /*
  * initWithBytes:
  *


@@ -429,10 +413,9 @@ bool IOBufferMemoryDescriptor::initWithBytes(const void * inBytes,
                                              IODirection  inDirection,
                                              bool         inContiguous)
 {
                                              IODirection  inDirection,
                                              bool         inContiguous)
 {

-    if (!initWithOptions(
-               inDirection | kIOMemoryUnshared
-                | (inContiguous ? kIOMemoryPhysicallyContiguous : 0),
-               inLength, inLength ))
+    if (!initWithPhysicalMask(kernel_task, inDirection | kIOMemoryUnshared
+                             | (inContiguous ? kIOMemoryPhysicallyContiguous : 0),
+                             inLength, inLength, (mach_vm_address_t)0))

         return false;
 
     // start out with no data
         return false;
 
     // start out with no data


@@ -443,6 +426,7 @@ bool IOBufferMemoryDescriptor::initWithBytes(const void * inBytes,
 
     return true;
 }
 
     return true;
 }

+#endif /* !__LP64__ */

 
 /*
  * withBytes:
 
 /*
  * withBytes:


@@ -458,21 +442,25 @@ IOBufferMemoryDescriptor::withBytes(const void * inBytes,
 {
     IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor;
 
 {
     IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor;
 

-    if (me && !me->initWithBytes(inBytes, inLength, inDirection, inContiguous))
+    if (me && !me->initWithPhysicalMask(
+               kernel_task, inDirection | kIOMemoryUnshared
+                | (inContiguous ? kIOMemoryPhysicallyContiguous : 0),
+               inLength, inLength, 0 ))

     {
     {

-       bool retry = me->_physSegCount;

        me->release();
        me = 0;
        me->release();
        me = 0;

-       if (retry)
+    }
+
+    if (me)
+    {
+       // start out with no data
+       me->setLength(0);
+
+       if (!me->appendBytes(inBytes, inLength))

        {
        {

-           me = new IOBufferMemoryDescriptor;
-           if (me && !me->initWithBytes(inBytes, inLength, inDirection, inContiguous))
-           {
-               me->release();
-               me = 0;
-           }
+           me->release();
+           me = 0;

        }
        }

-

     }
     return me;
 }
     }
     return me;
 }


@@ -486,54 +474,74 @@ void IOBufferMemoryDescriptor::free()
 {
     // Cache all of the relevant information on the stack for use
     // after we call super::free()!
 {
     // Cache all of the relevant information on the stack for use
     // after we call super::free()!

-    IOOptionBits     flags     = _flags;
+    IOOptionBits     flags         = _flags;
+    IOOptionBits     internalFlags = _internalFlags;

     IOOptionBits     options   = _options;
     vm_size_t        size      = _capacity;
     void *           buffer    = _buffer;
     IOOptionBits     options   = _options;
     vm_size_t        size      = _capacity;
     void *           buffer    = _buffer;

-    IOVirtualAddress source    = _singleRange.v.address;
-    vm_map_t        vmmap     = 0;
+    IOMemoryMap *    map       = 0;
+    IOAddressRange * range     = _ranges.v64;

     vm_offset_t      alignment = _alignment;
 
     vm_offset_t      alignment = _alignment;
 

+    if (alignment >= page_size)
+       size = round_page(size);
+

     if (reserved)
     {
     if (reserved)
     {

-       vmmap = reserved->map;
+       map = reserved->map;

         IODelete( reserved, ExpansionData, 1 );
         IODelete( reserved, ExpansionData, 1 );

+       if (map)
+           map->release();

     }
 
     }
 

+    if ((options & kIOMemoryPageable)
+        || (kInternalFlagPageSized & internalFlags)) size = round_page(size);
+
+#if IOTRACKING
+    if (!(options & kIOMemoryPageable)
+       && buffer
+       && (kInternalFlagInit & _internalFlags)) trackingAccumSize(-size);
+#endif /* IOTRACKING */
+

     /* super::free may unwire - deallocate buffer afterwards */
     super::free();
 
     if (options & kIOMemoryPageable)
     {
 #if IOALLOCDEBUG
     /* super::free may unwire - deallocate buffer afterwards */
     super::free();
 
     if (options & kIOMemoryPageable)
     {
 #if IOALLOCDEBUG

-        if (!buffer || vmmap)
-            debug_iomallocpageable_size -= round_page_32(size);
+       OSAddAtomicLong(-size, &debug_iomallocpageable_size);

 #endif
 #endif

-        if (buffer)
-        {
-            if (vmmap)
-                vm_deallocate(vmmap, (vm_address_t) buffer, round_page_32(size));
-            else
-                IOFreePageable(buffer, size);
-        }

     }
     else if (buffer)
     {
     }
     else if (buffer)
     {

-       if (kIOMemoryTypePhysical == (flags & kIOMemoryTypeMask))
+        if (kInternalFlagPhysical & internalFlags)
+        {
+            IOKernelFreePhysical((mach_vm_address_t) buffer, size);
+       }
+       else if (kInternalFlagPageAllocated & internalFlags)

        {
        {

-            if (vmmap)
-                vm_deallocate(vmmap, (vm_address_t) buffer, round_page_32(size));
-           IOFreePhysical((mach_vm_address_t) source, size);
+           uintptr_t page;
+            page = iopa_free(&gIOBMDPageAllocator, (uintptr_t) buffer, size);
+           if (page)
+           {
+               kmem_free(kernel_map, page, page_size);
+           }
+#if IOALLOCDEBUG
+               OSAddAtomic(-size, &debug_iomalloc_size);
+#endif
+           IOStatisticsAlloc(kIOStatisticsFreeAligned, size);

        }
        }

-        else if (options & kIOMemoryPhysicallyContiguous)
-            IOKernelFreeContiguous((mach_vm_address_t) buffer, size);

         else if (alignment > 1)
         else if (alignment > 1)

+       {

             IOFreeAligned(buffer, size);
             IOFreeAligned(buffer, size);

+       }

         else
         else

+       {

             IOFree(buffer, size);
             IOFree(buffer, size);

+       }

     }
     }

-    if (vmmap)
-       vm_map_deallocate(vmmap);
+    if (range && (kIOMemoryAsReference & flags))
+       IODelete(range, IOAddressRange, 1);

 }
 
 /*
 }
 
 /*


@@ -559,9 +567,10 @@ vm_size_t IOBufferMemoryDescriptor::getCapacity() const
 void IOBufferMemoryDescriptor::setLength(vm_size_t length)
 {
     assert(length <= _capacity);
 void IOBufferMemoryDescriptor::setLength(vm_size_t length)
 {
     assert(length <= _capacity);

+    if (length > _capacity) return;

 
     _length = length;
 
     _length = length;

-    _singleRange.v.length = length;
+    _ranges.v64->length = length;

 }
 
 /*
 }
 
 /*


@@ -573,7 +582,10 @@ void IOBufferMemoryDescriptor::setLength(vm_size_t length)
  */
 void IOBufferMemoryDescriptor::setDirection(IODirection direction)
 {
  */
 void IOBufferMemoryDescriptor::setDirection(IODirection direction)
 {

-    _direction = direction;
+    _flags = (_flags & ~kIOMemoryDirectionMask) | direction;
+#ifndef __LP64__
+    _direction = (IODirection) (_flags & kIOMemoryDirectionMask);
+#endif /* !__LP64__ */

 }
 
 /*
 }
 
 /*


@@ -593,10 +605,10 @@ IOBufferMemoryDescriptor::appendBytes(const void * bytes, vm_size_t withLength)
 
     offset = _length;
     _length += actualBytesToCopy;
 
     offset = _length;
     _length += actualBytesToCopy;

-    _singleRange.v.length += actualBytesToCopy;
+    _ranges.v64->length += actualBytesToCopy;

 
     if (_task == kernel_task)
 
     if (_task == kernel_task)

-       bcopy(/* from */ bytes, (void *)(_singleRange.v.address + offset),
+       bcopy(/* from */ bytes, (void *)(_ranges.v64->address + offset),

              actualBytesToCopy);
     else
        writeBytes(offset, bytes, actualBytesToCopy);
              actualBytesToCopy);
     else
        writeBytes(offset, bytes, actualBytesToCopy);


@@ -611,10 +623,10 @@ IOBufferMemoryDescriptor::appendBytes(const void * bytes, vm_size_t withLength)
  */
 void * IOBufferMemoryDescriptor::getBytesNoCopy()
 {
  */
 void * IOBufferMemoryDescriptor::getBytesNoCopy()
 {

-    if (kIOMemoryTypePhysical == (_flags & kIOMemoryTypeMask))
+    if (kIOMemoryTypePhysical64 == (_flags & kIOMemoryTypeMask))

        return _buffer;
     else
        return _buffer;
     else

-       return (void *)_singleRange.v.address;
+       return (void *)_ranges.v64->address;

 }
 
 
 }
 
 


@@ -627,18 +639,22 @@ void *
 IOBufferMemoryDescriptor::getBytesNoCopy(vm_size_t start, vm_size_t withLength)
 {
     IOVirtualAddress address;
 IOBufferMemoryDescriptor::getBytesNoCopy(vm_size_t start, vm_size_t withLength)
 {
     IOVirtualAddress address;

-    if (kIOMemoryTypePhysical == (_flags & kIOMemoryTypeMask))
+
+    if ((start + withLength) < start) return 0;
+
+    if (kIOMemoryTypePhysical64 == (_flags & kIOMemoryTypeMask))

        address = (IOVirtualAddress) _buffer;
     else
        address = (IOVirtualAddress) _buffer;
     else

-       address = _singleRange.v.address;
+       address = _ranges.v64->address;

 
    if (start < _length && (start + withLength) <= _length)
         return (void *)(address + start);
     return 0;
 }
 
 
    if (start < _length && (start + withLength) <= _length)
         return (void *)(address + start);
     return 0;
 }
 

-/* DEPRECATED */ void * IOBufferMemoryDescriptor::getVirtualSegment(IOByteCount offset,
-/* DEPRECATED */                                                       IOByteCount * lengthOfSegment)
+#ifndef __LP64__
+void * IOBufferMemoryDescriptor::getVirtualSegment(IOByteCount offset,
+                                                       IOByteCount * lengthOfSegment)

 {
     void * bytes = getBytesNoCopy(offset, 0);
     
 {
     void * bytes = getBytesNoCopy(offset, 0);
     


@@ -647,9 +663,15 @@ IOBufferMemoryDescriptor::getBytesNoCopy(vm_size_t start, vm_size_t withLength)
 
     return bytes;
 }
 
     return bytes;
 }

+#endif /* !__LP64__ */

 
 

+#ifdef __LP64__
+OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 0);
+OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 1);
+#else /* !__LP64__ */

 OSMetaClassDefineReservedUsed(IOBufferMemoryDescriptor, 0);
 OSMetaClassDefineReservedUsed(IOBufferMemoryDescriptor, 1);
 OSMetaClassDefineReservedUsed(IOBufferMemoryDescriptor, 0);
 OSMetaClassDefineReservedUsed(IOBufferMemoryDescriptor, 1);

+#endif /* !__LP64__ */

 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 2);
 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 3);
 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 4);
 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 2);
 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 3);
 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 4);