xnu-2050.9.2.tar.gz

[apple/xnu.git] / iokit / Kernel / IOBufferMemoryDescriptor.cpp

diff --git a/iokit/Kernel/IOBufferMemoryDescriptor.cpp b/iokit/Kernel/IOBufferMemoryDescriptor.cpp
index bca55b5354cd0ad928748386efb2d7956fa62e67..a56b469ee999d7373044d28e131688e0fad6272b 100644 (file)
--- a/iokit/Kernel/IOBufferMemoryDescriptor.cpp
+++ b/iokit/Kernel/IOBufferMemoryDescriptor.cpp
@@ -37,7 +37,6 @@
 #include <libkern/OSDebug.h>
 
 #include "IOKitKernelInternal.h"
 #include <libkern/OSDebug.h>
 
 #include "IOKitKernelInternal.h"

-#include "IOCopyMapper.h"

 
 __BEGIN_DECLS
 void ipc_port_release_send(ipc_port_t port);
 
 __BEGIN_DECLS
 void ipc_port_release_send(ipc_port_t port);


@@ -49,11 +48,10 @@ __END_DECLS
 
 enum
 {
 
 enum
 {

-    kInternalFlagRealloc = 0x00000001,
+    kInternalFlagPhysical  = 0x00000001,
+    kInternalFlagPageSized = 0x00000002

 };
 
 };
 

-volatile ppnum_t gIOHighestAllocatedPage;
-

 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 #define super IOGeneralMemoryDescriptor
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 #define super IOGeneralMemoryDescriptor


@@ -84,9 +82,7 @@ bool IOBufferMemoryDescriptor::initWithPhysicalMask(
     kern_return_t      kr;
     task_t             mapTask = NULL;
     vm_map_t           vmmap = NULL;
     kern_return_t      kr;
     task_t             mapTask = NULL;
     vm_map_t           vmmap = NULL;

-    addr64_t            lastIOAddr;

     mach_vm_address_t   highestMask = 0;
     mach_vm_address_t   highestMask = 0;

-    bool               usePhys;

     IOOptionBits       iomdOptions = kIOMemoryTypeVirtual64 | kIOMemoryAsReference;
 
     if (!capacity)
     IOOptionBits       iomdOptions = kIOMemoryTypeVirtual64 | kIOMemoryAsReference;
 
     if (!capacity)


@@ -103,6 +99,8 @@ bool IOBufferMemoryDescriptor::initWithPhysicalMask(
        return (false);
     _ranges.v64->address = 0;
     _ranges.v64->length  = 0;
        return (false);
     _ranges.v64->address = 0;
     _ranges.v64->length  = 0;

+       //  make sure super::free doesn't dealloc _ranges before super::init
+       _flags = kIOMemoryAsReference;

 
     // Grab IOMD bits from the Buffer MD options
     iomdOptions  |= (options & kIOBufferDescriptorMemoryFlags);
 
     // Grab IOMD bits from the Buffer MD options
     iomdOptions  |= (options & kIOBufferDescriptorMemoryFlags);


@@ -112,9 +110,11 @@ bool IOBufferMemoryDescriptor::initWithPhysicalMask(
        alignment   = ((physicalMask ^ (-1ULL)) & (physicalMask - 1));
        highestMask = (physicalMask | alignment);
        alignment++;
        alignment   = ((physicalMask ^ (-1ULL)) & (physicalMask - 1));
        highestMask = (physicalMask | alignment);
        alignment++;

+       if (alignment < page_size)
+            alignment = page_size;

     }
 
     }
 

-    if ((options & (kIOMemorySharingTypeMask | kIOMapCacheMask)) && (alignment < page_size))
+    if ((options & (kIOMemorySharingTypeMask | kIOMapCacheMask | kIOMemoryClearEncrypt)) && (alignment < page_size))

        alignment = page_size;
 
     if (alignment >= page_size)
        alignment = page_size;
 
     if (alignment >= page_size)


@@ -128,9 +128,6 @@ bool IOBufferMemoryDescriptor::initWithPhysicalMask(
     if ((inTask != kernel_task) && !(options & kIOMemoryPageable))
        return false;
 
     if ((inTask != kernel_task) && !(options & kIOMemoryPageable))
        return false;
 

-    if ((options & kIOMemoryPhysicallyContiguous) && !physicalMask)
-       physicalMask = 0xFFFFFFFF;
-

     // set flags for entry + object create
     vm_prot_t memEntryCacheMode = VM_PROT_READ | VM_PROT_WRITE;
 
     // set flags for entry + object create
     vm_prot_t memEntryCacheMode = VM_PROT_READ | VM_PROT_WRITE;
 


@@ -153,6 +150,10 @@ bool IOBufferMemoryDescriptor::initWithPhysicalMask(
            SET_MAP_MEM(MAP_MEM_COPYBACK, memEntryCacheMode);
            break;
 
            SET_MAP_MEM(MAP_MEM_COPYBACK, memEntryCacheMode);
            break;
 

+       case kIOMapCopybackInnerCache:
+           SET_MAP_MEM(MAP_MEM_INNERWBACK, memEntryCacheMode);
+           break;
+

        case kIOMapDefaultCache:
        default:
            SET_MAP_MEM(MAP_MEM_NOOP, memEntryCacheMode);
        case kIOMapDefaultCache:
        default:
            SET_MAP_MEM(MAP_MEM_NOOP, memEntryCacheMode);


@@ -174,63 +175,41 @@ bool IOBufferMemoryDescriptor::initWithPhysicalMask(
     else
     {
        memEntryCacheMode |= MAP_MEM_NAMED_REUSE;
     else
     {
        memEntryCacheMode |= MAP_MEM_NAMED_REUSE;

+       vmmap = kernel_map;

 
 

-       if (IOMapper::gSystem)
-           // assuming mapped space is 2G
-           lastIOAddr = (1UL << 31) - PAGE_SIZE;
-       else
-           lastIOAddr = ptoa_64(gIOHighestAllocatedPage);
+       // Buffer shouldn't auto prepare they should be prepared explicitly
+       // But it never was enforced so what are you going to do?
+       iomdOptions |= kIOMemoryAutoPrepare;

 
 

-       usePhys = (highestMask && (lastIOAddr != (lastIOAddr & highestMask))
-                   && (alignment <= page_size));
+       /* Allocate a wired-down buffer inside kernel space. */

 
 

-       if (!usePhys && (options & kIOMemoryPhysicallyContiguous))
+       if ((options & kIOMemoryPhysicallyContiguous) || highestMask || (alignment > page_size))

        {
        {

-           _buffer = (void *) IOKernelAllocateContiguous(capacity, highestMask, alignment);
-           usePhys = (NULL == _buffer);
+            _internalFlags |= kInternalFlagPhysical;
+            if (highestMask)
+            {
+                _internalFlags |= kInternalFlagPageSized;
+                capacity = round_page(capacity);
+            }
+            _buffer = (void *) IOKernelAllocateWithPhysicalRestrict(capacity, highestMask, alignment, 
+                                        (0 != (options & kIOMemoryPhysicallyContiguous)));

        }
        }

-       if (usePhys)
+       else if (alignment > 1)

        {
        {

-           mach_vm_address_t address;
-           iomdOptions &= ~kIOMemoryTypeVirtual64;
-           iomdOptions |= kIOMemoryTypePhysical64;
-
-           address = IOMallocPhysical(capacity, highestMask);
-           _buffer = (void *) address;
-           if (!_buffer)
-               return false;
-
-           mapTask = inTask;
-           inTask = 0;
+            _buffer = IOMallocAligned(capacity, alignment);

        }
        else
        {
        }
        else
        {

-           vmmap = kernel_map;
-
-           // Buffer shouldn't auto prepare they should be prepared explicitly
-           // But it never was enforced so what are you going to do?
-           iomdOptions |= kIOMemoryAutoPrepare;
+            _buffer = IOMalloc(capacity);
+       }

 
 

-           /* Allocate a wired-down buffer inside kernel space. */
-           if (options & kIOMemoryPhysicallyContiguous)
-           {
-               // attempted allocate already
-           }
-           else if (alignment > 1)
-           {
-               _buffer = IOMallocAligned(capacity, alignment);
-           }
-           else
-           {
-               _buffer = IOMalloc(capacity);
-           }
-           if (!_buffer)
-               return false;
+       if (!_buffer)
+       {
+            return false;

        }
     }
 
        }
     }
 

-    if( (kIOMemoryTypePhysical64 != (kIOMemoryTypeMask & iomdOptions)) 
-       && (options & (kIOMemoryPageable | kIOMapCacheMask))) {
+    if( (options & (kIOMemoryPageable | kIOMapCacheMask))) {

        ipc_port_t      sharedMem;
        vm_size_t       size = round_page(capacity);
 
        ipc_port_t      sharedMem;
        vm_size_t       size = round_page(capacity);
 


@@ -277,42 +256,6 @@ bool IOBufferMemoryDescriptor::initWithPhysicalMask(
                                inTask, iomdOptions, /* System mapper */ 0))
        return false;
 
                                inTask, iomdOptions, /* System mapper */ 0))
        return false;
 

-    if (highestMask && !IOMapper::gSystem)
-    {
-       IOMDDMACharacteristics mdSummary;
-
-       bzero(&mdSummary, sizeof(mdSummary));
-       IOReturn rtn = dmaCommandOperation(
-               kIOMDGetCharacteristics,
-               &mdSummary, sizeof(mdSummary));
-       if (rtn)
-           return false;
-
-       if (mdSummary.fHighestPage)
-       {
-           ppnum_t highest;
-           while (mdSummary.fHighestPage > (highest = gIOHighestAllocatedPage))
-           {
-               if (OSCompareAndSwap(highest, mdSummary.fHighestPage, 
-                                       (UInt32 *) &gIOHighestAllocatedPage))
-                   break;
-           }
-           lastIOAddr = ptoa_64(mdSummary.fHighestPage);
-       }
-       else
-           lastIOAddr = ptoa_64(gIOLastPage);
-
-       if (lastIOAddr != (lastIOAddr & highestMask))
-       {
-           if (kIOMemoryTypePhysical64 != (_flags & kIOMemoryTypeMask))
-           {
-               // flag a retry
-               _internalFlags |= kInternalFlagRealloc;
-           }
-           return false;
-       }
-    }
-

     if (mapTask)
     {
        if (!reserved) {
     if (mapTask)
     {
        if (!reserved) {


@@ -350,18 +293,8 @@ IOBufferMemoryDescriptor * IOBufferMemoryDescriptor::inTaskWithOptions(
     IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor;
     
     if (me && !me->initWithPhysicalMask(inTask, options, capacity, alignment, 0)) {
     IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor;
     
     if (me && !me->initWithPhysicalMask(inTask, options, capacity, alignment, 0)) {

-       bool retry = (0 != (kInternalFlagRealloc & me->_internalFlags));

        me->release();
        me = 0;
        me->release();
        me = 0;

-       if (retry)
-       {
-           me = new IOBufferMemoryDescriptor;
-           if (me && !me->initWithPhysicalMask(inTask, options, capacity, alignment, 0))
-           {
-               me->release();
-               me = 0;
-           }
-       }

     }
     return me;
 }
     }
     return me;
 }


@@ -376,18 +309,8 @@ IOBufferMemoryDescriptor * IOBufferMemoryDescriptor::inTaskWithPhysicalMask(
     
     if (me && !me->initWithPhysicalMask(inTask, options, capacity, 1, physicalMask))
     {
     
     if (me && !me->initWithPhysicalMask(inTask, options, capacity, 1, physicalMask))
     {

-       bool retry = (0 != (kInternalFlagRealloc & me->_internalFlags));

        me->release();
        me = 0;
        me->release();
        me = 0;

-       if (retry)
-       {
-           me = new IOBufferMemoryDescriptor;
-           if (me && !me->initWithPhysicalMask(inTask, options, capacity, 1, physicalMask))
-           {
-               me->release();
-               me = 0;
-           }
-       }

     }
     return me;
 }
     }
     return me;
 }


@@ -410,18 +333,8 @@ IOBufferMemoryDescriptor * IOBufferMemoryDescriptor::withOptions(
     IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor;
     
     if (me && !me->initWithPhysicalMask(kernel_task, options, capacity, alignment, 0)) {
     IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor;
     
     if (me && !me->initWithPhysicalMask(kernel_task, options, capacity, alignment, 0)) {

-       bool retry = (0 != (kInternalFlagRealloc & me->_internalFlags));

        me->release();
        me = 0;
        me->release();
        me = 0;

-       if (retry)
-       {
-           me = new IOBufferMemoryDescriptor;
-           if (me && !me->initWithPhysicalMask(kernel_task, options, capacity, alignment, 0))
-           {
-               me->release();
-               me = 0;
-           }
-       }

     }
     return me;
 }
     }
     return me;
 }


@@ -490,22 +403,8 @@ IOBufferMemoryDescriptor::withBytes(const void * inBytes,
                 | (inContiguous ? kIOMemoryPhysicallyContiguous : 0),
                inLength, inLength, 0 ))
     {
                 | (inContiguous ? kIOMemoryPhysicallyContiguous : 0),
                inLength, inLength, 0 ))
     {

-       bool retry = (0 != (kInternalFlagRealloc & me->_internalFlags));

        me->release();
        me = 0;
        me->release();
        me = 0;

-       if (retry)
-       {
-           me = new IOBufferMemoryDescriptor;
-           if (me && !me->initWithPhysicalMask(
-                  kernel_task, inDirection | kIOMemoryUnshared
-                   | (inContiguous ? kIOMemoryPhysicallyContiguous : 0),
-                  inLength, inLength, 0 ))
-           {
-               me->release();
-               me = 0;
-           }
-       }
-

     }
 
     if (me)
     }
 
     if (me)


@@ -531,13 +430,13 @@ void IOBufferMemoryDescriptor::free()
 {
     // Cache all of the relevant information on the stack for use
     // after we call super::free()!
 {
     // Cache all of the relevant information on the stack for use
     // after we call super::free()!

-    IOOptionBits     flags     = _flags;
+    IOOptionBits     flags         = _flags;
+    IOOptionBits     internalFlags = _internalFlags;

     IOOptionBits     options   = _options;
     vm_size_t        size      = _capacity;
     void *           buffer    = _buffer;
     IOMemoryMap *    map       = 0;
     IOAddressRange * range     = _ranges.v64;
     IOOptionBits     options   = _options;
     vm_size_t        size      = _capacity;
     void *           buffer    = _buffer;
     IOMemoryMap *    map       = 0;
     IOAddressRange * range     = _ranges.v64;

-    mach_vm_address_t source   = range ? range->address : 0;

     vm_offset_t      alignment = _alignment;
 
     if (alignment >= page_size)
     vm_offset_t      alignment = _alignment;
 
     if (alignment >= page_size)


@@ -562,10 +461,12 @@ void IOBufferMemoryDescriptor::free()
     }
     else if (buffer)
     {
     }
     else if (buffer)
     {

-       if (kIOMemoryTypePhysical64 == (flags & kIOMemoryTypeMask))
-           IOFreePhysical(source, size);
-        else if (options & kIOMemoryPhysicallyContiguous)
-            IOKernelFreeContiguous((mach_vm_address_t) buffer, size);
+        if (internalFlags & kInternalFlagPhysical)
+        {
+            if (kInternalFlagPageSized & internalFlags)
+                size = round_page(size);
+            IOKernelFreePhysical((mach_vm_address_t) buffer, size);
+        }

         else if (alignment > 1)
             IOFreeAligned(buffer, size);
         else
         else if (alignment > 1)
             IOFreeAligned(buffer, size);
         else