xnu-1699.26.8.tar.gz

[apple/xnu.git] / iokit / Kernel / IOMemoryDescriptor.cpp

diff --git a/iokit/Kernel/IOMemoryDescriptor.cpp b/iokit/Kernel/IOMemoryDescriptor.cpp
index 12ace713b3c89a412ccffac8e0b7bd6c8e80ba71..9b459094542f5dc4046584676621eae5dd633939 100644 (file)
--- a/iokit/Kernel/IOMemoryDescriptor.cpp
+++ b/iokit/Kernel/IOMemoryDescriptor.cpp
@@ -1,16 +1,19 @@
 /*
 /*

- * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2007 Apple Inc. All rights reserved.

  *
  *

- * @APPLE_LICENSE_HEADER_START@
- * 
- * Copyright (c) 1999-2003 Apple Computer, Inc.  All Rights Reserved.
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

  * 
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in
  * 
  * This file contains Original Code and/or Modifications of Original Code
  * as defined in and that are subject to the Apple Public Source License
  * Version 2.0 (the 'License'). You may not use this file except in

- * compliance with the License. Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this
- * file.
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.

  * 
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  * 
  * The Original Code and all software distributed under the License are
  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER


@@ -20,7 +23,7 @@
  * Please see the License for the specific language governing rights and
  * limitations under the License.
  * 
  * Please see the License for the specific language governing rights and
  * limitations under the License.
  * 

- * @APPLE_LICENSE_HEADER_END@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@

  */
 /*
  * Copyright (c) 1998 Apple Computer, Inc.  All rights reserved. 
  */
 /*
  * Copyright (c) 1998 Apple Computer, Inc.  All rights reserved. 


@@ -28,7 +31,8 @@
  * HISTORY
  *
  */
  * HISTORY
  *
  */

-// 45678901234567890123456789012345678901234567890123456789012345678901234567890
+
+

 #include <sys/cdefs.h>
 
 #include <IOKit/assert.h>
 #include <sys/cdefs.h>
 
 #include <IOKit/assert.h>


@@ -36,68 +40,60 @@
 #include <IOKit/IOLib.h>
 #include <IOKit/IOMemoryDescriptor.h>
 #include <IOKit/IOMapper.h>
 #include <IOKit/IOLib.h>
 #include <IOKit/IOMemoryDescriptor.h>
 #include <IOKit/IOMapper.h>

+#include <IOKit/IOKitKeysPrivate.h>
+
+#ifndef __LP64__
+#include <IOKit/IOSubMemoryDescriptor.h>
+#endif /* !__LP64__ */

 
 #include <IOKit/IOKitDebug.h>
 
 #include <IOKit/IOKitDebug.h>

+#include <libkern/OSDebug.h>
+
+#include "IOKitKernelInternal.h"

 
 #include <libkern/c++/OSContainers.h>
 #include <libkern/c++/OSDictionary.h>
 #include <libkern/c++/OSArray.h>
 #include <libkern/c++/OSSymbol.h>
 #include <libkern/c++/OSNumber.h>
 
 #include <libkern/c++/OSContainers.h>
 #include <libkern/c++/OSDictionary.h>
 #include <libkern/c++/OSArray.h>
 #include <libkern/c++/OSSymbol.h>
 #include <libkern/c++/OSNumber.h>

-#include <sys/cdefs.h>
+
+#include <sys/uio.h>

 
 __BEGIN_DECLS
 #include <vm/pmap.h>
 
 __BEGIN_DECLS
 #include <vm/pmap.h>

+#include <vm/vm_pageout.h>

 #include <mach/memory_object_types.h>
 #include <device/device_port.h>
 
 #include <mach/memory_object_types.h>
 #include <device/device_port.h>
 

-#ifndef i386
-struct phys_entry      *pmap_find_physentry(ppnum_t pa);
-#endif
-void ipc_port_release_send(ipc_port_t port);
-
-/* Copy between a physical page and a virtual address in the given vm_map */
-kern_return_t copypv(addr64_t source, addr64_t sink, unsigned int size, int which);
-
-memory_object_t
-device_pager_setup(
-       memory_object_t pager,
-       int             device_handle,
-       vm_size_t       size,
-       int             flags);
-void
-device_pager_deallocate(
-        memory_object_t);
-kern_return_t
-device_pager_populate_object(
-       memory_object_t         pager,
-       vm_object_offset_t      offset,
-       ppnum_t                 phys_addr,
-       vm_size_t               size);
+#include <mach/vm_prot.h>
+#include <mach/mach_vm.h>
+#include <vm/vm_fault.h>
+#include <vm/vm_protos.h>

 
 

-/*
- *     Page fault handling based on vm_map (or entries therein)
- */
-extern kern_return_t vm_fault(
-               vm_map_t        map,
-               vm_offset_t     vaddr,
-               vm_prot_t       fault_type,
-               boolean_t       change_wiring,
-               int             interruptible,
-               pmap_t          caller_pmap,
-               vm_offset_t     caller_pmap_addr);
+extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
+extern void ipc_port_release_send(ipc_port_t port);
+
+kern_return_t
+memory_object_iopl_request(
+       ipc_port_t              port,
+       memory_object_offset_t  offset,
+       vm_size_t               *upl_size,
+       upl_t                   *upl_ptr,
+       upl_page_info_array_t   user_page_list,
+       unsigned int            *page_list_count,
+       int                     *flags);

 
 unsigned int  IOTranslateCacheBits(struct phys_entry *pp);
 
 
 unsigned int  IOTranslateCacheBits(struct phys_entry *pp);
 

-vm_map_t IOPageableMapForAddress( vm_address_t address );
+__END_DECLS

 
 

-typedef kern_return_t (*IOIteratePageableMapsCallback)(vm_map_t map, void * ref);
+#define kIOMaximumMappedIOByteCount    (512*1024*1024)

 
 

-kern_return_t IOIteratePageableMaps(vm_size_t size,
-                    IOIteratePageableMapsCallback callback, void * ref);
-__END_DECLS
+static IOMapper * gIOSystemMapper = NULL;

 
 

-static IOMapper * gIOSystemMapper;
+static ppnum_t   gIOMaximumMappedIOPageCount = atop_32(kIOMaximumMappedIOByteCount);
+
+ppnum_t                  gIOLastPage;

 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 


@@ -117,15 +113,69 @@ static IORecursiveLock * gIOMemoryLock;
 #define WAKEUP \
     IORecursiveLockWakeup( gIOMemoryLock, (void *)this, /* one-thread */ false)
 
 #define WAKEUP \
     IORecursiveLockWakeup( gIOMemoryLock, (void *)this, /* one-thread */ false)
 

+#if 0
+#define DEBG(fmt, args...)     { kprintf(fmt, ## args); }
+#else
+#define DEBG(fmt, args...)     {}
+#endif
+
+#define IOMD_DEBUG_DMAACTIVE   1
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+// Some data structures and accessor macros used by the initWithOptions
+// Function
+
+enum ioPLBlockFlags {
+    kIOPLOnDevice  = 0x00000001,
+    kIOPLExternUPL = 0x00000002,
+};
+
+struct typePersMDData
+{
+    const IOGeneralMemoryDescriptor *fMD;
+    ipc_port_t fMemEntry;
+};
+
+struct ioPLBlock {
+    upl_t fIOPL;
+    vm_address_t fPageInfo;   // Pointer to page list or index into it
+    uint32_t fIOMDOffset;          // The offset of this iopl in descriptor
+    ppnum_t fMappedBase;           // Page number of first page in this iopl
+    unsigned int fPageOffset;      // Offset within first page of iopl
+    unsigned int fFlags;           // Flags
+};
+
+struct ioGMDData {
+    IOMapper *fMapper;
+    uint64_t fPreparationID;
+    unsigned int fPageCnt;
+#if __LP64__
+    // align arrays to 8 bytes so following macros work
+    unsigned int fPad;
+#endif
+    upl_page_info_t fPageList[1]; /* variable length */
+    ioPLBlock fBlocks[1]; /* variable length */
+};
+
+#define getDataP(osd)  ((ioGMDData *) (osd)->getBytesNoCopy())
+#define getIOPLList(d) ((ioPLBlock *) &(d->fPageList[d->fPageCnt]))
+#define getNumIOPL(osd, d)     \
+    (((osd)->getLength() - ((char *) getIOPLList(d) - (char *) d)) / sizeof(ioPLBlock))
+#define getPageList(d) (&(d->fPageList[0]))
+#define computeDataSize(p, u) \
+    (offsetof(ioGMDData, fPageList) + p * sizeof(upl_page_info_t) + u * sizeof(ioPLBlock))
+
+

 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 

-#define next_page(a) ( trunc_page_32(a) + PAGE_SIZE )
+#define next_page(a) ( trunc_page(a) + PAGE_SIZE )

 
 
 extern "C" {
 
 kern_return_t device_data_action(
 
 
 extern "C" {
 
 kern_return_t device_data_action(

-               int                     device_handle, 
+               uintptr_t               device_handle, 

                ipc_port_t              device_pager,
                vm_prot_t               protection, 
                vm_object_offset_t      offset, 
                ipc_port_t              device_pager,
                vm_prot_t               protection, 
                vm_object_offset_t      offset, 


@@ -144,8 +194,12 @@ kern_return_t device_data_action(
     LOCK;
     memDesc = ref->memory;
     if( memDesc)
     LOCK;
     memDesc = ref->memory;
     if( memDesc)

+    {
+       memDesc->retain();

        kr = memDesc->handleFault( device_pager, 0, 0,
                 offset, size, kIOMapDefaultCache /*?*/);
        kr = memDesc->handleFault( device_pager, 0, 0,
                 offset, size, kIOMapDefaultCache /*?*/);

+       memDesc->release();
+    }

     else
        kr = KERN_ABORTED;
     UNLOCK;
     else
        kr = KERN_ABORTED;
     UNLOCK;


@@ -154,7 +208,7 @@ kern_return_t device_data_action(
 }
 
 kern_return_t device_close(
 }
 
 kern_return_t device_close(

-               int     device_handle)
+               uintptr_t     device_handle)

 {
     struct ExpansionData {
         void *                         devicePager;
 {
     struct ExpansionData {
         void *                         devicePager;


@@ -168,29 +222,50 @@ kern_return_t device_close(
 
     return( kIOReturnSuccess );
 }
 
     return( kIOReturnSuccess );
 }

-
+};     // end extern "C"
+
+// Note this inline function uses C++ reference arguments to return values
+// This means that pointers are not passed and NULLs don't have to be
+// checked for as a NULL reference is illegal.
+static inline void
+getAddrLenForInd(user_addr_t &addr, IOPhysicalLength &len, // Output variables
+     UInt32 type, IOGeneralMemoryDescriptor::Ranges r, UInt32 ind)
+{
+    assert(kIOMemoryTypeUIO       == type
+       || kIOMemoryTypeVirtual   == type || kIOMemoryTypeVirtual64 == type
+       || kIOMemoryTypePhysical  == type || kIOMemoryTypePhysical64 == type);
+    if (kIOMemoryTypeUIO == type) {
+       user_size_t us;
+       uio_getiov((uio_t) r.uio, ind, &addr, &us); len = us;
+    }
+#ifndef __LP64__
+    else if ((kIOMemoryTypeVirtual64 == type) || (kIOMemoryTypePhysical64 == type)) {
+       IOAddressRange cur = r.v64[ind];
+       addr = cur.address;
+       len  = cur.length;
+    }
+#endif /* !__LP64__ */
+    else {
+       IOVirtualRange cur = r.v[ind];
+       addr = cur.address;
+       len  = cur.length;
+    }

 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 

-/*
- * withAddress:
- *
- * Create a new IOMemoryDescriptor.  The buffer is a virtual address
- * relative to the specified task.  If no task is supplied, the kernel
- * task is implied.
- */

 IOMemoryDescriptor *
 IOMemoryDescriptor::withAddress(void *      address,
                                 IOByteCount   length,
                                 IODirection direction)
 {
     return IOMemoryDescriptor::
 IOMemoryDescriptor *
 IOMemoryDescriptor::withAddress(void *      address,
                                 IOByteCount   length,
                                 IODirection direction)
 {
     return IOMemoryDescriptor::

-        withAddress((vm_address_t) address, length, direction, kernel_task);
+        withAddressRange((IOVirtualAddress) address, length, direction | kIOMemoryAutoPrepare, kernel_task);

 }
 
 }
 

+#ifndef __LP64__

 IOMemoryDescriptor *
 IOMemoryDescriptor *

-IOMemoryDescriptor::withAddress(vm_address_t address,
+IOMemoryDescriptor::withAddress(IOVirtualAddress address,

                                 IOByteCount  length,
                                 IODirection  direction,
                                 task_t       task)
                                 IOByteCount  length,
                                 IODirection  direction,
                                 task_t       task)


@@ -205,6 +280,7 @@ IOMemoryDescriptor::withAddress(vm_address_t address,
     }
     return 0;
 }
     }
     return 0;
 }

+#endif /* !__LP64__ */

 
 IOMemoryDescriptor *
 IOMemoryDescriptor::withPhysicalAddress(
 
 IOMemoryDescriptor *
 IOMemoryDescriptor::withPhysicalAddress(


@@ -212,16 +288,10 @@ IOMemoryDescriptor::withPhysicalAddress(
                                IOByteCount             length,
                                IODirection             direction )
 {
                                IOByteCount             length,
                                IODirection             direction )
 {

-    IOGeneralMemoryDescriptor *self = new IOGeneralMemoryDescriptor;
-    if (self
-    && !self->initWithPhysicalAddress(address, length, direction)) {
-        self->release();
-        return 0;
-    }
-
-    return self;
+    return (IOMemoryDescriptor::withAddressRange(address, length, direction, TASK_NULL));

 }
 
 }
 

+#ifndef __LP64__

 IOMemoryDescriptor *
 IOMemoryDescriptor::withRanges(        IOVirtualRange * ranges,
                                UInt32           withCount,
 IOMemoryDescriptor *
 IOMemoryDescriptor::withRanges(        IOVirtualRange * ranges,
                                UInt32           withCount,


@@ -239,10 +309,44 @@ IOMemoryDescriptor::withRanges(   IOVirtualRange * ranges,
     }
     return 0;
 }
     }
     return 0;
 }

+#endif /* !__LP64__ */
+
+IOMemoryDescriptor *
+IOMemoryDescriptor::withAddressRange(mach_vm_address_t address,
+                                       mach_vm_size_t length,
+                                       IOOptionBits   options,
+                                       task_t         task)
+{
+    IOAddressRange range = { address, length };
+    return (IOMemoryDescriptor::withAddressRanges(&range, 1, options, task));
+}
+
+IOMemoryDescriptor *
+IOMemoryDescriptor::withAddressRanges(IOAddressRange *   ranges,
+                                       UInt32           rangeCount,
+                                       IOOptionBits     options,
+                                       task_t           task)
+{
+    IOGeneralMemoryDescriptor * that = new IOGeneralMemoryDescriptor;
+    if (that)
+    {
+       if (task)
+           options |= kIOMemoryTypeVirtual64;
+       else
+           options |= kIOMemoryTypePhysical64;
+
+       if (that->initWithOptions(ranges, rangeCount, 0, task, options, /* mapper */ 0))
+           return that;
+
+       that->release();
+    }
+
+    return 0;
+}

 
 
 /*
 
 
 /*

- * withRanges:
+ * withOptions:

  *
  * Create a new IOMemoryDescriptor. The buffer is made up of several
  * virtual address ranges, from a given task.
  *
  * Create a new IOMemoryDescriptor. The buffer is made up of several
  * virtual address ranges, from a given task.


@@ -269,7 +373,6 @@ IOMemoryDescriptor::withOptions(void *              buffers,
     return self;
 }
 
     return self;
 }
 

-// Can't leave abstract but this should never be used directly,

 bool IOMemoryDescriptor::initWithOptions(void *                buffers,
                                          UInt32                count,
                                          UInt32                offset,
 bool IOMemoryDescriptor::initWithOptions(void *                buffers,
                                          UInt32                count,
                                          UInt32                offset,


@@ -277,11 +380,10 @@ bool IOMemoryDescriptor::initWithOptions(void *           buffers,
                                          IOOptionBits  options,
                                          IOMapper *    mapper)
 {
                                          IOOptionBits  options,
                                          IOMapper *    mapper)
 {

-    // @@@ gvdl: Should I panic?
-    panic("IOMD::initWithOptions called\n");
-    return 0;
+    return( false );

 }
 
 }
 

+#ifndef __LP64__

 IOMemoryDescriptor *
 IOMemoryDescriptor::withPhysicalRanges(        IOPhysicalRange * ranges,
                                         UInt32          withCount,
 IOMemoryDescriptor *
 IOMemoryDescriptor::withPhysicalRanges(        IOPhysicalRange * ranges,
                                         UInt32          withCount,


@@ -305,40 +407,108 @@ IOMemoryDescriptor::withSubRange(IOMemoryDescriptor *    of,
                                IOByteCount             length,
                                IODirection             direction)
 {
                                IOByteCount             length,
                                IODirection             direction)
 {

-    IOSubMemoryDescriptor *self = new IOSubMemoryDescriptor;
+    return (IOSubMemoryDescriptor::withSubRange(of, offset, length, direction | kIOMemoryThreadSafe));
+}
+#endif /* !__LP64__ */
+
+IOMemoryDescriptor *
+IOMemoryDescriptor::withPersistentMemoryDescriptor(IOMemoryDescriptor *originalMD)
+{
+    IOGeneralMemoryDescriptor *origGenMD = 
+       OSDynamicCast(IOGeneralMemoryDescriptor, originalMD);
+
+    if (origGenMD)
+       return IOGeneralMemoryDescriptor::
+           withPersistentMemoryDescriptor(origGenMD);
+    else
+       return 0;
+}
+
+IOMemoryDescriptor *
+IOGeneralMemoryDescriptor::withPersistentMemoryDescriptor(IOGeneralMemoryDescriptor *originalMD)
+{
+    ipc_port_t sharedMem = (ipc_port_t) originalMD->createNamedEntry();
+
+    if (!sharedMem)
+       return 0;
+   
+    if (sharedMem == originalMD->_memEntry) {
+       originalMD->retain();               // Add a new reference to ourselves
+       ipc_port_release_send(sharedMem);   // Remove extra send right
+       return originalMD;
+    }

 
 

-    if (self && !self->initSubRange(of, offset, length, direction)) {
+    IOGeneralMemoryDescriptor * self = new IOGeneralMemoryDescriptor;
+    typePersMDData initData = { originalMD, sharedMem };
+
+    if (self
+    && !self->initWithOptions(&initData, 1, 0, 0, kIOMemoryTypePersistentMD, 0)) {

         self->release();
        self = 0;
     }
     return self;
 }
 
         self->release();
        self = 0;
     }
     return self;
 }
 

-/*
- * initWithAddress:
- *
- * Initialize an IOMemoryDescriptor. The buffer is a virtual address
- * relative to the specified task.  If no task is supplied, the kernel
- * task is implied.
- *
- * An IOMemoryDescriptor can be re-used by calling initWithAddress or
- * initWithRanges again on an existing instance -- note this behavior
- * is not commonly supported in other I/O Kit classes, although it is
- * supported here.
- */
+void *IOGeneralMemoryDescriptor::createNamedEntry()
+{
+    kern_return_t error;
+    ipc_port_t sharedMem;
+
+    IOOptionBits type = _flags & kIOMemoryTypeMask;
+
+    user_addr_t range0Addr;
+    IOByteCount range0Len;
+    getAddrLenForInd(range0Addr, range0Len, type, _ranges, 0);
+    range0Addr = trunc_page_64(range0Addr);
+
+    vm_size_t size = ptoa_32(_pages);
+    vm_address_t kernelPage = (vm_address_t) range0Addr;
+
+    vm_map_t theMap = ((_task == kernel_task)
+                       && (kIOMemoryBufferPageable & _flags)) 
+                   ? IOPageableMapForAddress(kernelPage)
+                   : get_task_map(_task);
+
+    memory_object_size_t  actualSize = size;
+    vm_prot_t             prot       = VM_PROT_READ;
+    if (kIODirectionOut != (kIODirectionOutIn & _flags))
+       prot |= VM_PROT_WRITE;
+
+    if (_memEntry)
+       prot |= MAP_MEM_NAMED_REUSE;
+
+    error = mach_make_memory_entry_64(theMap,
+           &actualSize, range0Addr, prot, &sharedMem, (ipc_port_t) _memEntry);
+
+    if (KERN_SUCCESS == error) {
+       if (actualSize == size) {
+           return sharedMem;
+       } else {
+#if IOASSERT
+           IOLog("IOGMD::mach_make_memory_entry_64 (%08llx) size (%08llx:%08llx)\n",
+                 (UInt64)range0Addr, (UInt64)actualSize, (UInt64)size);
+#endif    
+           ipc_port_release_send( sharedMem );
+       }
+    }
+
+    return MACH_PORT_NULL;
+}
+
+#ifndef __LP64__

 bool
 IOGeneralMemoryDescriptor::initWithAddress(void *      address,
                                     IOByteCount   withLength,
                                     IODirection withDirection)
 {
 bool
 IOGeneralMemoryDescriptor::initWithAddress(void *      address,
                                     IOByteCount   withLength,
                                     IODirection withDirection)
 {

-    _singleRange.v.address = (vm_address_t) address;
+    _singleRange.v.address = (vm_offset_t) address;

     _singleRange.v.length  = withLength;
 
     return initWithRanges(&_singleRange.v, 1, withDirection, kernel_task, true);
 }
 
 bool
     _singleRange.v.length  = withLength;
 
     return initWithRanges(&_singleRange.v, 1, withDirection, kernel_task, true);
 }
 
 bool

-IOGeneralMemoryDescriptor::initWithAddress(vm_address_t address,
+IOGeneralMemoryDescriptor::initWithAddress(IOVirtualAddress address,

                                     IOByteCount    withLength,
                                     IODirection  withDirection,
                                     task_t       withTask)
                                     IOByteCount    withLength,
                                     IODirection  withDirection,
                                     task_t       withTask)


@@ -391,26 +561,26 @@ IOGeneralMemoryDescriptor::initWithRanges(
 
     if (task) {
         mdOpts |= kIOMemoryTypeVirtual;
 
     if (task) {
         mdOpts |= kIOMemoryTypeVirtual;

+
+       // Auto-prepare if this is a kernel memory descriptor as very few
+       // clients bother to prepare() kernel memory.
+       // But it was not enforced so what are you going to do?

         if (task == kernel_task)
             mdOpts |= kIOMemoryAutoPrepare;
     }
     else
         mdOpts |= kIOMemoryTypePhysical;
         if (task == kernel_task)
             mdOpts |= kIOMemoryAutoPrepare;
     }
     else
         mdOpts |= kIOMemoryTypePhysical;

-
-    // @@@ gvdl: Need to remove this
-    // Auto-prepare if this is a kernel memory descriptor as very few
-    // clients bother to prepare() kernel memory.
-    // But it has been enforced so what are you going to do?

     
     return initWithOptions(ranges, count, 0, task, mdOpts, /* mapper */ 0);
 }
     
     return initWithOptions(ranges, count, 0, task, mdOpts, /* mapper */ 0);
 }

+#endif /* !__LP64__ */

 
 /*
  * initWithOptions:
  *
  *  IOMemoryDescriptor. The buffer is made up of several virtual address ranges,
 
 /*
  * initWithOptions:
  *
  *  IOMemoryDescriptor. The buffer is made up of several virtual address ranges,

- * from a given task or several physical ranges or finally an UPL from the ubc
- * system.
+ * from a given task, several physical ranges, an UPL from the ubc
+ * system or a uio (may be 64bit) from the BSD subsystem.

  *
  * Passing the ranges as a reference will avoid an extra allocation.
  *
  *
  * Passing the ranges as a reference will avoid an extra allocation.
  *


@@ -419,36 +589,6 @@ IOGeneralMemoryDescriptor::initWithRanges(
  * I/O Kit classes, although it is supported here.
  */
 
  * I/O Kit classes, although it is supported here.
  */
 

-enum ioPLBlockFlags {
-    kIOPLOnDevice  = 0x00000001,
-    kIOPLExternUPL = 0x00000002,
-};
-
-struct ioPLBlock {
-    upl_t fIOPL;
-    vm_address_t fIOMDOffset;  // The offset of this iopl in descriptor
-    vm_offset_t fPageInfo;     // Pointer to page list or index into it
-    ppnum_t fMappedBase;       // Page number of first page in this iopl
-    unsigned int fPageOffset;  // Offset within first page of iopl
-    unsigned int fFlags;       // Flags
-};
-
-struct ioGMDData {
-    IOMapper *fMapper;
-    unsigned int fPageCnt;
-    upl_page_info_t fPageList[0];      // @@@ gvdl need to get rid of this
-                                        //  should be able to use upl directly
-    ioPLBlock fBlocks[0];
-};
-
-#define getDataP(osd)  ((ioGMDData *) (osd)->getBytesNoCopy())
-#define getIOPLList(d) ((ioPLBlock *) &(d->fPageList[d->fPageCnt]))
-#define getNumIOPL(d,len)      \
-    ((len - ((char *) getIOPLList(d) - (char *) d)) / sizeof(ioPLBlock))
-#define getPageList(d) (&(d->fPageList[0]))
-#define computeDataSize(p, u) \
-    (sizeof(ioGMDData) + p * sizeof(upl_page_info_t) + u * sizeof(ioPLBlock))
-

 bool
 IOGeneralMemoryDescriptor::initWithOptions(void *      buffers,
                                            UInt32      count,
 bool
 IOGeneralMemoryDescriptor::initWithOptions(void *      buffers,
                                            UInt32      count,


@@ -457,22 +597,64 @@ IOGeneralMemoryDescriptor::initWithOptions(void * buffers,
                                            IOOptionBits        options,
                                            IOMapper *  mapper)
 {
                                            IOOptionBits        options,
                                            IOMapper *  mapper)
 {

+    IOOptionBits type = options & kIOMemoryTypeMask;
+
+#ifndef __LP64__
+    if (task
+        && (kIOMemoryTypeVirtual == type)
+        && vm_map_is_64bit(get_task_map(task)) 
+        && ((IOVirtualRange *) buffers)->address)
+    {
+        OSReportWithBacktrace("IOMemoryDescriptor: attempt to create 32b virtual in 64b task, use ::withAddressRange()");
+        return false;
+    }
+#endif /* !__LP64__ */

 
 

-    switch (options & kIOMemoryTypeMask) {
+    // Grab the original MD's configuation data to initialse the
+    // arguments to this function.
+    if (kIOMemoryTypePersistentMD == type) {
+
+       typePersMDData *initData = (typePersMDData *) buffers;
+       const IOGeneralMemoryDescriptor *orig = initData->fMD;
+       ioGMDData *dataP = getDataP(orig->_memoryEntries);
+
+       // Only accept persistent memory descriptors with valid dataP data.
+       assert(orig->_rangesCount == 1);
+       if ( !(orig->_flags & kIOMemoryPersistent) || !dataP)
+           return false;
+
+       _memEntry = initData->fMemEntry;        // Grab the new named entry
+       options = orig->_flags & ~kIOMemoryAsReference; 
+        type = options & kIOMemoryTypeMask;
+       buffers = orig->_ranges.v;
+       count = orig->_rangesCount;
+
+       // Now grab the original task and whatever mapper was previously used
+       task = orig->_task;
+       mapper = dataP->fMapper;
+
+       // We are ready to go through the original initialisation now
+    }
+
+    switch (type) {
+    case kIOMemoryTypeUIO:

     case kIOMemoryTypeVirtual:
     case kIOMemoryTypeVirtual:

+#ifndef __LP64__
+    case kIOMemoryTypeVirtual64:
+#endif /* !__LP64__ */

         assert(task);
         if (!task)
             return false;
         assert(task);
         if (!task)
             return false;

-        else
-            break;
+       break;

 
     case kIOMemoryTypePhysical:                // Neither Physical nor UPL should have a task
 
     case kIOMemoryTypePhysical:                // Neither Physical nor UPL should have a task

-       mapper = kIOMapperNone;
+#ifndef __LP64__
+    case kIOMemoryTypePhysical64:
+#endif /* !__LP64__ */

     case kIOMemoryTypeUPL:
         assert(!task);
         break;
     default:
     case kIOMemoryTypeUPL:
         assert(!task);
         break;
     default:

-panic("IOGMD::iWO(): bad type");       // @@@ gvdl: for testing

         return false;  /* bad argument */
     }
 
         return false;  /* bad argument */
     }
 


@@ -490,13 +672,31 @@ panic("IOGMD::iWO(): bad type");  // @@@ gvdl: for testing
          * An existing memory descriptor is being retargeted to point to
          * somewhere else.  Clean up our present state.
          */
          * An existing memory descriptor is being retargeted to point to
          * somewhere else.  Clean up our present state.
          */

+       IOOptionBits type = _flags & kIOMemoryTypeMask;
+       if ((kIOMemoryTypePhysical != type) && (kIOMemoryTypePhysical64 != type))
+       {
+           while (_wireCount)
+               complete();
+       }
+        if (_ranges.v && !(kIOMemoryAsReference & _flags))
+       {
+           if (kIOMemoryTypeUIO == type)
+               uio_free((uio_t) _ranges.v);
+#ifndef __LP64__
+           else if ((kIOMemoryTypeVirtual64 == type) || (kIOMemoryTypePhysical64 == type))
+               IODelete(_ranges.v64, IOAddressRange, _rangesCount);
+#endif /* !__LP64__ */
+           else
+               IODelete(_ranges.v, IOVirtualRange, _rangesCount);
+       }

 
 

-        while (_wireCount)
-            complete();
-        if (_kernPtrAligned)
-            unmapFromKernel();
-        if (_ranges.v && _rangesIsAllocated)
-            IODelete(_ranges.v, IOVirtualRange, _rangesCount);
+       if (_memEntry)
+       {
+           ipc_port_release_send((ipc_port_t) _memEntry);
+           _memEntry = 0;
+       }
+       if (_mappings)
+           _mappings->flushCollection();

     }
     else {
         if (!super::init())
     }
     else {
         if (!super::init())


@@ -505,24 +705,44 @@ panic("IOGMD::iWO(): bad type");  // @@@ gvdl: for testing
     }
 
     // Grab the appropriate mapper
     }
 
     // Grab the appropriate mapper

-    if (mapper == kIOMapperNone)
+    if (kIOMemoryMapperNone & options)

         mapper = 0;    // No Mapper
         mapper = 0;    // No Mapper

-    else if (!mapper) {
+    else if (mapper == kIOMapperSystem) {

         IOMapper::checkForSystemMapper();
         gIOSystemMapper = mapper = IOMapper::gSystem;
     }
 
         IOMapper::checkForSystemMapper();
         gIOSystemMapper = mapper = IOMapper::gSystem;
     }
 

+    // Temp binary compatibility for kIOMemoryThreadSafe
+    if (kIOMemoryReserved6156215 & options)
+    {
+       options &= ~kIOMemoryReserved6156215;
+       options |= kIOMemoryThreadSafe;
+    }
+    // Remove the dynamic internal use flags from the initial setting
+    options              &= ~(kIOMemoryPreparedReadOnly);

     _flags                = options;
     _task                  = task;
 
     _flags                = options;
     _task                  = task;
 

-    // DEPRECATED variable initialisation
+#ifndef __LP64__

     _direction             = (IODirection) (_flags & kIOMemoryDirectionMask);
     _direction             = (IODirection) (_flags & kIOMemoryDirectionMask);

-    _position              = 0;
-    _kernPtrAligned        = 0;
-    _cachedPhysicalAddress = 0;
-    _cachedVirtualAddress  = 0;
+#endif /* !__LP64__ */
+
+    __iomd_reservedA = 0;
+    __iomd_reservedB = 0;
+    _highestPage = 0;

 
 

-    if ( (options & kIOMemoryTypeMask) == kIOMemoryTypeUPL) {
+    if (kIOMemoryThreadSafe & options)
+    {
+       if (!_prepareLock)
+           _prepareLock = IOLockAlloc();
+    }
+    else if (_prepareLock)
+    {
+       IOLockFree(_prepareLock);
+       _prepareLock = NULL;
+    }
+       
+    if (kIOMemoryTypeUPL == type) {

 
         ioGMDData *dataP;
         unsigned int dataSize = computeDataSize(/* pages */ 0, /* upls */ 1);
 
         ioGMDData *dataP;
         unsigned int dataSize = computeDataSize(/* pages */ 0, /* upls */ 1);


@@ -535,26 +755,31 @@ panic("IOGMD::iWO(): bad type");  // @@@ gvdl: for testing
         else if (!_memoryEntries->initWithCapacity(dataSize))
             return false;
 
         else if (!_memoryEntries->initWithCapacity(dataSize))
             return false;
 

-        _memoryEntries->appendBytes(0, sizeof(ioGMDData));
+        _memoryEntries->appendBytes(0, computeDataSize(0, 0));

         dataP = getDataP(_memoryEntries);
         dataP->fMapper = mapper;
         dataP->fPageCnt = 0;
 
         dataP = getDataP(_memoryEntries);
         dataP->fMapper = mapper;
         dataP->fPageCnt = 0;
 

-        _wireCount++;  // UPLs start out life wired
+ //       _wireCount++;        // UPLs start out life wired

 
         _length    = count;
         _pages    += atop_32(offset + count + PAGE_MASK) - atop_32(offset);
 
         ioPLBlock iopl;
 
         _length    = count;
         _pages    += atop_32(offset + count + PAGE_MASK) - atop_32(offset);
 
         ioPLBlock iopl;

-        upl_page_info_t *pageList = UPL_GET_INTERNAL_PAGE_LIST((upl_t) buffers);
-

         iopl.fIOPL = (upl_t) buffers;
         iopl.fIOPL = (upl_t) buffers;

+        upl_set_referenced(iopl.fIOPL, true);
+        upl_page_info_t *pageList = UPL_GET_INTERNAL_PAGE_LIST(iopl.fIOPL);
+
+       if (upl_get_size(iopl.fIOPL) < (count + offset))
+           panic("short external upl");
+

         // Set the flag kIOPLOnDevice convieniently equal to 1
         iopl.fFlags  = pageList->device | kIOPLExternUPL;
         iopl.fIOMDOffset = 0;
         // Set the flag kIOPLOnDevice convieniently equal to 1
         iopl.fFlags  = pageList->device | kIOPLExternUPL;
         iopl.fIOMDOffset = 0;

+
+        _highestPage = upl_get_highest_page(iopl.fIOPL);
+

         if (!pageList->device) {
         if (!pageList->device) {

-            // @@@ gvdl: Ask JoeS are the pages contiguious with the list?
-            // or there a chance that we may be inserting 0 phys_addrs?

             // Pre-compute the offset into the UPL's page list
             pageList = &pageList[atop_32(offset)];
             offset &= PAGE_MASK;
             // Pre-compute the offset into the UPL's page list
             pageList = &pageList[atop_32(offset)];
             offset &= PAGE_MASK;


@@ -572,59 +797,123 @@ panic("IOGMD::iWO(): bad type"); // @@@ gvdl: for testing
 
         _memoryEntries->appendBytes(&iopl, sizeof(iopl));
     }
 
         _memoryEntries->appendBytes(&iopl, sizeof(iopl));
     }

-    else {     /* kIOMemoryTypeVirtual | kIOMemoryTypePhysical */
-        IOVirtualRange *ranges = (IOVirtualRange *) buffers;
-
-        /*
-         * Initialize the memory descriptor.
-         */
-
-        _length                        = 0;
-        _pages                 = 0;
-        for (unsigned ind = 0; ind < count; ind++) {
-            IOVirtualRange cur = ranges[ind];
-    
-            _length += cur.length;
-            _pages += atop_32(cur.address + cur.length + PAGE_MASK)
-                   -  atop_32(cur.address);
-        }
-
-        _ranges.v              = 0;
-        _rangesIsAllocated     = !(options & kIOMemoryAsReference);
-        _rangesCount           = count;
-
-        if (options & kIOMemoryAsReference)
-            _ranges.v = ranges;
-        else {
-            _ranges.v = IONew(IOVirtualRange, count);
-            if (!_ranges.v)
-                return false;
-            bcopy(/* from */ ranges, _ranges.v,
-                  count * sizeof(IOVirtualRange));
-        } 
+    else {
+       // kIOMemoryTypeVirtual  | kIOMemoryTypeVirtual64 | kIOMemoryTypeUIO 
+       // kIOMemoryTypePhysical | kIOMemoryTypePhysical64
+       
+       // Initialize the memory descriptor
+       if (options & kIOMemoryAsReference) {
+#ifndef __LP64__
+           _rangesIsAllocated = false;
+#endif /* !__LP64__ */
+
+           // Hack assignment to get the buffer arg into _ranges.
+           // I'd prefer to do _ranges = (Ranges) buffers, but that doesn't
+           // work, C++ sigh.
+           // This also initialises the uio & physical ranges.
+           _ranges.v = (IOVirtualRange *) buffers;
+       }
+       else {
+#ifndef __LP64__
+           _rangesIsAllocated = true;
+#endif /* !__LP64__ */
+           switch (type)
+           {
+             case kIOMemoryTypeUIO:
+               _ranges.v = (IOVirtualRange *) uio_duplicate((uio_t) buffers);
+               break;
+
+#ifndef __LP64__
+             case kIOMemoryTypeVirtual64:
+             case kIOMemoryTypePhysical64:
+               if (count == 1
+                   && (((IOAddressRange *) buffers)->address + ((IOAddressRange *) buffers)->length) <= 0x100000000ULL
+                   ) {
+                   if (kIOMemoryTypeVirtual64 == type)
+                       type = kIOMemoryTypeVirtual;
+                   else
+                       type = kIOMemoryTypePhysical;
+                   _flags = (_flags & ~kIOMemoryTypeMask) | type | kIOMemoryAsReference;
+                   _rangesIsAllocated = false;
+                   _ranges.v = &_singleRange.v;
+                   _singleRange.v.address = ((IOAddressRange *) buffers)->address;
+                   _singleRange.v.length  = ((IOAddressRange *) buffers)->length;
+                   break;
+               }
+               _ranges.v64 = IONew(IOAddressRange, count);
+               if (!_ranges.v64)
+                   return false;
+               bcopy(buffers, _ranges.v, count * sizeof(IOAddressRange));
+               break;
+#endif /* !__LP64__ */
+             case kIOMemoryTypeVirtual:
+             case kIOMemoryTypePhysical:
+               if (count == 1) {
+                   _flags |= kIOMemoryAsReference;
+#ifndef __LP64__
+                   _rangesIsAllocated = false;
+#endif /* !__LP64__ */
+                   _ranges.v = &_singleRange.v;
+               } else {
+                   _ranges.v = IONew(IOVirtualRange, count);
+                   if (!_ranges.v)
+                       return false;
+               }
+               bcopy(buffers, _ranges.v, count * sizeof(IOVirtualRange));
+               break;
+           }
+       } 
+
+       // Find starting address within the vector of ranges
+       Ranges vec = _ranges;
+       UInt32 length = 0;
+       UInt32 pages = 0;
+       for (unsigned ind = 0; ind < count;  ind++) {
+           user_addr_t addr;
+           IOPhysicalLength len;
+
+           // addr & len are returned by this function
+           getAddrLenForInd(addr, len, type, vec, ind);
+           pages += (atop_64(addr + len + PAGE_MASK) - atop_64(addr));
+           len += length;
+           assert(len >= length);      // Check for 32 bit wrap around
+           length = len;
+
+           if ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type))
+           {
+               ppnum_t highPage = atop_64(addr + len - 1);
+               if (highPage > _highestPage)
+                   _highestPage = highPage;
+           }
+       } 
+       _length      = length;
+       _pages       = pages;
+       _rangesCount = count;

 
         // Auto-prepare memory at creation time.
         // Implied completion when descriptor is free-ed
 
         // Auto-prepare memory at creation time.
         // Implied completion when descriptor is free-ed

-        if ( (options & kIOMemoryTypeMask) == kIOMemoryTypePhysical)
-            _wireCount++;      // Physical MDs are start out wired
-        else { /* kIOMemoryTypeVirtual */
+        if ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type))
+            _wireCount++;      // Physical MDs are, by definition, wired
+        else { /* kIOMemoryTypeVirtual | kIOMemoryTypeVirtual64 | kIOMemoryTypeUIO */

             ioGMDData *dataP;
             ioGMDData *dataP;

-            unsigned int dataSize =
-                computeDataSize(_pages, /* upls */ _rangesCount * 2);
+            unsigned dataSize = computeDataSize(_pages, /* upls */ count * 2);

 
             if (!_memoryEntries) {
                 _memoryEntries = OSData::withCapacity(dataSize);
                 if (!_memoryEntries)
 
             if (!_memoryEntries) {
                 _memoryEntries = OSData::withCapacity(dataSize);
                 if (!_memoryEntries)

-                    return false;
+                   return false;

             }
             else if (!_memoryEntries->initWithCapacity(dataSize))
                 return false;
     
             }
             else if (!_memoryEntries->initWithCapacity(dataSize))
                 return false;
     

-            _memoryEntries->appendBytes(0, sizeof(ioGMDData));
+            _memoryEntries->appendBytes(0, computeDataSize(0, 0));

             dataP = getDataP(_memoryEntries);
             dataP->fMapper = mapper;
             dataP->fPageCnt = _pages;
 
             dataP = getDataP(_memoryEntries);
             dataP->fMapper = mapper;
             dataP->fPageCnt = _pages;
 

+           if ( (kIOMemoryPersistent & _flags) && !_memEntry)
+               _memEntry = createNamedEntry();
+

             if ((_flags & kIOMemoryAutoPrepare)
              && prepare() != kIOReturnSuccess)
                 return false;
             if ((_flags & kIOMemoryAutoPrepare)
              && prepare() != kIOReturnSuccess)
                 return false;


@@ -641,20 +930,36 @@ panic("IOGMD::iWO(): bad type");  // @@@ gvdl: for testing
  */
 void IOGeneralMemoryDescriptor::free()
 {
  */
 void IOGeneralMemoryDescriptor::free()
 {

-    LOCK;
+    IOOptionBits type = _flags & kIOMemoryTypeMask;
+

     if( reserved)
     if( reserved)

+    {
+       LOCK;

        reserved->memory = 0;
        reserved->memory = 0;

-    UNLOCK;
+       UNLOCK;
+    }

 
 

-    while (_wireCount)
-        complete();
+    if ((kIOMemoryTypePhysical != type) && (kIOMemoryTypePhysical64 != type))
+    {
+       while (_wireCount)
+           complete();
+    }

     if (_memoryEntries)
         _memoryEntries->release();
 
     if (_memoryEntries)
         _memoryEntries->release();
 

-    if (_kernPtrAligned)
-        unmapFromKernel();
-    if (_ranges.v && _rangesIsAllocated)
-        IODelete(_ranges.v, IOVirtualRange, _rangesCount);
+    if (_ranges.v && !(kIOMemoryAsReference & _flags))
+    {
+       if (kIOMemoryTypeUIO == type)
+           uio_free((uio_t) _ranges.v);
+#ifndef __LP64__
+       else if ((kIOMemoryTypeVirtual64 == type) || (kIOMemoryTypePhysical64 == type))
+           IODelete(_ranges.v64, IOAddressRange, _rangesCount);
+#endif /* !__LP64__ */
+       else
+           IODelete(_ranges.v, IOVirtualRange, _rangesCount);
+
+       _ranges.v = NULL;
+    }

 
     if (reserved && reserved->devicePager)
        device_pager_deallocate( (memory_object_t) reserved->devicePager );
 
     if (reserved && reserved->devicePager)
        device_pager_deallocate( (memory_object_t) reserved->devicePager );


@@ -664,18 +969,23 @@ void IOGeneralMemoryDescriptor::free()
     if (_memEntry)
         ipc_port_release_send( (ipc_port_t) _memEntry );
 
     if (_memEntry)
         ipc_port_release_send( (ipc_port_t) _memEntry );
 

+    if (_prepareLock)
+       IOLockFree(_prepareLock);
+

     super::free();
 }
 
     super::free();
 }
 

-/* DEPRECATED */ void IOGeneralMemoryDescriptor::unmapFromKernel()
-/* DEPRECATED */ {
-                    panic("IOGMD::unmapFromKernel deprecated");
-/* DEPRECATED */ }
-/* DEPRECATED */ 
-/* DEPRECATED */ void IOGeneralMemoryDescriptor::mapIntoKernel(unsigned rangeIndex)
-/* DEPRECATED */ {
-                    panic("IOGMD::mapIntoKernel deprecated");
-/* DEPRECATED */ }
+#ifndef __LP64__
+void IOGeneralMemoryDescriptor::unmapFromKernel()
+{
+    panic("IOGMD::unmapFromKernel deprecated");
+}
+
+void IOGeneralMemoryDescriptor::mapIntoKernel(unsigned rangeIndex)
+{
+    panic("IOGMD::mapIntoKernel deprecated");
+}
+#endif /* !__LP64__ */

 
 /*
  * getDirection:
 
 /*
  * getDirection:


@@ -684,7 +994,11 @@ void IOGeneralMemoryDescriptor::free()
  */
 IODirection IOMemoryDescriptor::getDirection() const
 {
  */
 IODirection IOMemoryDescriptor::getDirection() const
 {

-    return _direction;
+#ifndef __LP64__
+    if (_direction)
+       return _direction;
+#endif /* !__LP64__ */
+    return (IODirection) (_flags & kIOMemoryDirectionMask);

 }
 
 /*
 }
 
 /*


@@ -707,40 +1021,44 @@ IOOptionBits IOMemoryDescriptor::getTag( void )
     return( _tag);
 }
 
     return( _tag);
 }
 

+#ifndef __LP64__

 // @@@ gvdl: who is using this API?  Seems like a wierd thing to implement.
 // @@@ gvdl: who is using this API?  Seems like a wierd thing to implement.

-IOPhysicalAddress IOMemoryDescriptor::getSourceSegment( IOByteCount   offset,
-                                                        IOByteCount * length )
+IOPhysicalAddress
+IOMemoryDescriptor::getSourceSegment( IOByteCount   offset, IOByteCount * length )

 {
 {

-    IOPhysicalAddress physAddr = 0;
+    addr64_t physAddr = 0;

 
     if( prepare() == kIOReturnSuccess) {
 
     if( prepare() == kIOReturnSuccess) {

-        physAddr = getPhysicalSegment( offset, length );
+        physAddr = getPhysicalSegment64( offset, length );

         complete();
     }
 
         complete();
     }
 

-    return( physAddr );
+    return( (IOPhysicalAddress) physAddr ); // truncated but only page offset is used

 }
 }

+#endif /* !__LP64__ */

 
 IOByteCount IOMemoryDescriptor::readBytes
                 (IOByteCount offset, void *bytes, IOByteCount length)
 {
 
 IOByteCount IOMemoryDescriptor::readBytes
                 (IOByteCount offset, void *bytes, IOByteCount length)
 {

-    addr64_t dstAddr = (addr64_t) (UInt32) bytes;
+    addr64_t dstAddr = CAST_DOWN(addr64_t, bytes);

     IOByteCount remaining;
 
     // Assert that this entire I/O is withing the available range
     assert(offset < _length);
     assert(offset + length <= _length);
     if (offset >= _length) {
     IOByteCount remaining;
 
     // Assert that this entire I/O is withing the available range
     assert(offset < _length);
     assert(offset + length <= _length);
     if (offset >= _length) {

-IOLog("IOGMD(%p): rB = o%lx, l%lx\n", this, offset, length);   // @@@ gvdl

         return 0;
     }
 
         return 0;
     }
 

+    if (kIOMemoryThreadSafe & _flags)
+       LOCK;
+

     remaining = length = min(length, _length - offset);
     while (remaining) {        // (process another target segment?)
         addr64_t       srcAddr64;
         IOByteCount    srcLen;
 
     remaining = length = min(length, _length - offset);
     while (remaining) {        // (process another target segment?)
         addr64_t       srcAddr64;
         IOByteCount    srcLen;
 

-        srcAddr64 = getPhysicalSegment64(offset, &srcLen);
+        srcAddr64 = getPhysicalSegment(offset, &srcLen, kIOMemoryMapperNone);

         if (!srcAddr64)
             break;
 
         if (!srcAddr64)
             break;
 


@@ -749,13 +1067,16 @@ IOLog("IOGMD(%p): rB = o%lx, l%lx\n", this, offset, length);     // @@@ gvdl
             srcLen = remaining;
 
         copypv(srcAddr64, dstAddr, srcLen,
             srcLen = remaining;
 
         copypv(srcAddr64, dstAddr, srcLen,

-                            cppvPsrc | cppvFsnk | cppvKmap);
+                            cppvPsrc | cppvNoRefSrc | cppvFsnk | cppvKmap);

 
         dstAddr   += srcLen;
         offset    += srcLen;
         remaining -= srcLen;
     }
 
 
         dstAddr   += srcLen;
         offset    += srcLen;
         remaining -= srcLen;
     }
 

+    if (kIOMemoryThreadSafe & _flags)
+       UNLOCK;
+

     assert(!remaining);
 
     return length - remaining;
     assert(!remaining);
 
     return length - remaining;


@@ -764,7 +1085,7 @@ IOLog("IOGMD(%p): rB = o%lx, l%lx\n", this, offset, length);       // @@@ gvdl
 IOByteCount IOMemoryDescriptor::writeBytes
                 (IOByteCount offset, const void *bytes, IOByteCount length)
 {
 IOByteCount IOMemoryDescriptor::writeBytes
                 (IOByteCount offset, const void *bytes, IOByteCount length)
 {

-    addr64_t srcAddr = (addr64_t) (UInt32) bytes;
+    addr64_t srcAddr = CAST_DOWN(addr64_t, bytes);

     IOByteCount remaining;
 
     // Assert that this entire I/O is withing the available range
     IOByteCount remaining;
 
     // Assert that this entire I/O is withing the available range


@@ -774,16 +1095,18 @@ IOByteCount IOMemoryDescriptor::writeBytes
     assert( !(kIOMemoryPreparedReadOnly & _flags) );
 
     if ( (kIOMemoryPreparedReadOnly & _flags) || offset >= _length) {
     assert( !(kIOMemoryPreparedReadOnly & _flags) );
 
     if ( (kIOMemoryPreparedReadOnly & _flags) || offset >= _length) {

-IOLog("IOGMD(%p): wB = o%lx, l%lx\n", this, offset, length);   // @@@ gvdl

         return 0;
     }
 
         return 0;
     }
 

+    if (kIOMemoryThreadSafe & _flags)
+       LOCK;
+

     remaining = length = min(length, _length - offset);
     while (remaining) {        // (process another target segment?)
         addr64_t    dstAddr64;
         IOByteCount dstLen;
 
     remaining = length = min(length, _length - offset);
     while (remaining) {        // (process another target segment?)
         addr64_t    dstAddr64;
         IOByteCount dstLen;
 

-        dstAddr64 = getPhysicalSegment64(offset, &dstLen);
+        dstAddr64 = getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);

         if (!dstAddr64)
             break;
 
         if (!dstAddr64)
             break;
 


@@ -792,13 +1115,16 @@ IOLog("IOGMD(%p): wB = o%lx, l%lx\n", this, offset, length);     // @@@ gvdl
             dstLen = remaining;
 
         copypv(srcAddr, (addr64_t) dstAddr64, dstLen,
             dstLen = remaining;
 
         copypv(srcAddr, (addr64_t) dstAddr64, dstLen,

-                            cppvPsnk | cppvFsnk | cppvNoModSnk | cppvKmap);
+                            cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);

 
         srcAddr   += dstLen;
         offset    += dstLen;
         remaining -= dstLen;
     }
 
 
         srcAddr   += dstLen;
         offset    += dstLen;
         remaining -= dstLen;
     }
 

+    if (kIOMemoryThreadSafe & _flags)
+       UNLOCK;
+

     assert(!remaining);
 
     return length - remaining;
     assert(!remaining);
 
     return length - remaining;


@@ -807,173 +1133,455 @@ IOLog("IOGMD(%p): wB = o%lx, l%lx\n", this, offset, length);   // @@@ gvdl
 // osfmk/device/iokit_rpc.c
 extern "C" unsigned int IODefaultCacheBits(addr64_t pa);
 
 // osfmk/device/iokit_rpc.c
 extern "C" unsigned int IODefaultCacheBits(addr64_t pa);
 

-/* DEPRECATED */ void IOGeneralMemoryDescriptor::setPosition(IOByteCount position)
-/* DEPRECATED */ {
-                    panic("IOGMD::setPosition deprecated");
-/* DEPRECATED */ }
+#ifndef __LP64__
+void IOGeneralMemoryDescriptor::setPosition(IOByteCount position)
+{
+    panic("IOGMD::setPosition deprecated");
+}
+#endif /* !__LP64__ */
+
+static volatile SInt64 gIOMDPreparationID __attribute__((aligned(8))) = (1ULL << 32);

 
 

-IOPhysicalAddress IOGeneralMemoryDescriptor::getPhysicalSegment
-                        (IOByteCount offset, IOByteCount *lengthOfSegment)
+uint64_t
+IOGeneralMemoryDescriptor::getPreparationID( void )

 {
 {

-    IOPhysicalAddress address = 0;
-    IOPhysicalLength  length  = 0;
+    ioGMDData *dataP;
+
+    if (!_wireCount)
+       return (kIOPreparationIDUnprepared);

 
 

-//  assert(offset <= _length);
-    if (offset < _length) // (within bounds?)
+    if (_flags & (kIOMemoryTypePhysical | kIOMemoryTypePhysical64))
+       return (kIOPreparationIDAlwaysPrepared);
+
+    if (!_memoryEntries || !(dataP = getDataP(_memoryEntries)))
+       return (kIOPreparationIDUnprepared);
+
+    if (kIOPreparationIDUnprepared == dataP->fPreparationID)

     {
     {

-        if ( (_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical) {
-            unsigned int ind;
+       dataP->fPreparationID = OSIncrementAtomic64(&gIOMDPreparationID);
+    }
+    return (dataP->fPreparationID);
+}

 
 

-            // Physical address based memory descriptor
+uint64_t
+IOMemoryDescriptor::getPreparationID( void )
+{
+    return (kIOPreparationIDUnsupported);    
+}

 
 

-            // Find offset within descriptor and make it relative
-            // to the current _range.
-            for (ind = 0 ; offset >= _ranges.p[ind].length; ind++ )
-                offset -= _ranges.p[ind].length;
-    
-            IOPhysicalRange cur = _ranges.p[ind];
-            address = cur.address + offset;
-            length  = cur.length  - offset;
-
-            // see how far we can coalesce ranges
-            for (++ind; ind < _rangesCount; ind++) {
-                cur =  _ranges.p[ind];
-        
-                if (address + length != cur.address)
-                    break;
-    
-                length += cur.length;
-            }
+IOReturn IOGeneralMemoryDescriptor::dmaCommandOperation(DMACommandOps op, void *vData, UInt dataSize) const
+{
+    if (kIOMDGetCharacteristics == op) {

 
 

-            // @@@ gvdl: should assert(address);
-            // but can't as NVidia GeForce creates a bogus physical mem
-            {
-                assert(address || /*nvidia*/(!_ranges.p[0].address && 1 == _rangesCount));
-            }
-            assert(length);
-        }
-        else do {
-            // We need wiring & we are wired.
-            assert(_wireCount);
+       if (dataSize < sizeof(IOMDDMACharacteristics))
+           return kIOReturnUnderrun;

 
 

-            if (!_wireCount)
-           {
-               panic("IOGMD: not wired for getPhysicalSegment()");
-                continue;
+       IOMDDMACharacteristics *data = (IOMDDMACharacteristics *) vData;
+       data->fLength = _length;
+       data->fSGCount = _rangesCount;
+       data->fPages = _pages;
+       data->fDirection = getDirection();
+       if (!_wireCount)
+           data->fIsPrepared = false;
+       else {
+           data->fIsPrepared = true;
+           data->fHighestPage = _highestPage;
+           if (_memoryEntries) {
+               ioGMDData *gmdData = getDataP(_memoryEntries);
+               ioPLBlock *ioplList = getIOPLList(gmdData);
+               UInt count = getNumIOPL(_memoryEntries, gmdData);
+
+               data->fIsMapped = (gmdData->fMapper && _pages && (count > 0)
+                              && ioplList[0].fMappedBase);
+               if (count == 1)
+                   data->fPageAlign = (ioplList[0].fPageOffset & PAGE_MASK) | ~PAGE_MASK;

            }
            }

+           else
+               data->fIsMapped = false;
+       }

 
 

-            assert(_memoryEntries);
+       return kIOReturnSuccess;

 
 

-            ioGMDData * dataP = getDataP(_memoryEntries);
-            const ioPLBlock *ioplList = getIOPLList(dataP);
-            UInt ind, numIOPLs = getNumIOPL(dataP, _memoryEntries->getLength());
-            upl_page_info_t *pageList = getPageList(dataP);
+#if IOMD_DEBUG_DMAACTIVE
+    } else if (kIOMDSetDMAActive == op) {
+       IOGeneralMemoryDescriptor * md = const_cast<IOGeneralMemoryDescriptor *>(this);
+       OSIncrementAtomic(&md->__iomd_reservedA);
+    } else if (kIOMDSetDMAInactive == op) {
+       IOGeneralMemoryDescriptor * md = const_cast<IOGeneralMemoryDescriptor *>(this);
+       if (md->__iomd_reservedA)
+           OSDecrementAtomic(&md->__iomd_reservedA);
+       else
+           panic("kIOMDSetDMAInactive");
+#endif /* IOMD_DEBUG_DMAACTIVE */
+
+    } else if (!(kIOMDWalkSegments & op))
+       return kIOReturnBadArgument;
+
+    // Get the next segment
+    struct InternalState {
+       IOMDDMAWalkSegmentArgs fIO;
+       UInt fOffset2Index;
+       UInt fIndex;
+       UInt fNextOffset;
+    } *isP;
+
+    // Find the next segment
+    if (dataSize < sizeof(*isP))
+       return kIOReturnUnderrun;
+
+    isP = (InternalState *) vData;
+    UInt offset = isP->fIO.fOffset;
+    bool mapped = isP->fIO.fMapped;
+
+    if (offset >= _length)
+       return (offset == _length)? kIOReturnOverrun : kIOReturnInternalError;
+
+    // Validate the previous offset
+    UInt ind, off2Ind = isP->fOffset2Index;
+    if ((kIOMDFirstSegment != op) 
+       && offset 
+       && (offset == isP->fNextOffset || off2Ind <= offset))
+       ind = isP->fIndex;
+    else
+       ind = off2Ind = 0;      // Start from beginning

 
 

-            assert(numIOPLs > 0);
+    UInt length;
+    UInt64 address;
+    if ( (_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical) {

 
 

-            // Scan through iopl info blocks looking for block containing offset
-            for (ind = 1; ind < numIOPLs; ind++) {
-                if (offset < ioplList[ind].fIOMDOffset)
-                    break;
-            }
+       // Physical address based memory descriptor
+       const IOPhysicalRange *physP = (IOPhysicalRange *) &_ranges.p[0];

 
 

-            // Go back to actual range as search goes past it
-            ioPLBlock ioplInfo = ioplList[ind - 1];
+       // Find the range after the one that contains the offset
+       mach_vm_size_t len;
+       for (len = 0; off2Ind <= offset; ind++) {
+           len = physP[ind].length;
+           off2Ind += len;
+       }

 
 

-            if (ind < numIOPLs)
-                length = ioplList[ind].fIOMDOffset;
-            else
-                length = _length;
-            length -= offset;                  // Remainder within iopl
+       // Calculate length within range and starting address
+       length   = off2Ind - offset;
+       address  = physP[ind - 1].address + len - length;

 
 

-            // Subtract offset till this iopl in total list
-            offset -= ioplInfo.fIOMDOffset;
+       // see how far we can coalesce ranges
+       while (ind < _rangesCount && address + length == physP[ind].address) {
+           len = physP[ind].length;
+           length += len;
+           off2Ind += len;
+           ind++;
+       }

 
 

-            // This is a mapped IOPL so we just need to compute an offset
-            // relative to the mapped base.
-            if (ioplInfo.fMappedBase) {
-                offset += (ioplInfo.fPageOffset & PAGE_MASK);
-                address = ptoa_32(ioplInfo.fMappedBase) + offset;
-                continue;
-            }
+       // correct contiguous check overshoot
+       ind--;
+       off2Ind -= len;
+    }
+#ifndef __LP64__
+    else if ( (_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical64) {

 
 

-            // Currently the offset is rebased into the current iopl.
-            // Now add the iopl 1st page offset.
-            offset += ioplInfo.fPageOffset;
+       // Physical address based memory descriptor
+       const IOAddressRange *physP = (IOAddressRange *) &_ranges.v64[0];

 
 

-            // For external UPLs the fPageInfo field points directly to
-            // the upl's upl_page_info_t array.
-            if (ioplInfo.fFlags & kIOPLExternUPL)
-                pageList = (upl_page_info_t *) ioplInfo.fPageInfo;
-            else
-                pageList = &pageList[ioplInfo.fPageInfo];
+       // Find the range after the one that contains the offset
+       mach_vm_size_t len;
+       for (len = 0; off2Ind <= offset; ind++) {
+           len = physP[ind].length;
+           off2Ind += len;
+       }

 
 

-            // Check for direct device non-paged memory
-            if ( ioplInfo.fFlags & kIOPLOnDevice ) {
-                address = ptoa_32(pageList->phys_addr) + offset;
-                continue;
-            }
+       // Calculate length within range and starting address
+       length   = off2Ind - offset;
+       address  = physP[ind - 1].address + len - length;

 
 

-            // Now we need compute the index into the pageList
-            ind = atop_32(offset);
-            offset &= PAGE_MASK;
+       // see how far we can coalesce ranges
+       while (ind < _rangesCount && address + length == physP[ind].address) {
+           len = physP[ind].length;
+           length += len;
+           off2Ind += len;
+           ind++;
+       }

 
 

-            IOPhysicalAddress pageAddr = pageList[ind].phys_addr;
-            address = ptoa_32(pageAddr) + offset;
-
-            // Check for the remaining data in this upl being longer than the
-            // remainder on the current page.  This should be checked for
-            // contiguous pages
-            if (length > PAGE_SIZE - offset) {
-                // See if the next page is contiguous.  Stop looking when we hit
-                // the end of this upl, which is indicated by the
-                // contigLength >= length.
-                IOByteCount contigLength = PAGE_SIZE - offset;
-
-                // Look for contiguous segment
-                while (contigLength < length
-                &&     ++pageAddr == pageList[++ind].phys_addr) {
-                    contigLength += PAGE_SIZE;
-                }
-                if (length > contigLength)
-                    length = contigLength;
-            }
-    
-            assert(address);
-            assert(length);
+       // correct contiguous check overshoot
+       ind--;
+       off2Ind -= len;
+    }
+#endif /* !__LP64__ */
+    else do {
+       if (!_wireCount)
+           panic("IOGMD: not wired for the IODMACommand");
+
+       assert(_memoryEntries);
+
+       ioGMDData * dataP = getDataP(_memoryEntries);
+       const ioPLBlock *ioplList = getIOPLList(dataP);
+       UInt numIOPLs = getNumIOPL(_memoryEntries, dataP);
+       upl_page_info_t *pageList = getPageList(dataP);
+
+       assert(numIOPLs > 0);
+
+       // Scan through iopl info blocks looking for block containing offset
+       while (ind < numIOPLs && offset >= ioplList[ind].fIOMDOffset)
+           ind++;
+
+       // Go back to actual range as search goes past it
+       ioPLBlock ioplInfo = ioplList[ind - 1];
+       off2Ind = ioplInfo.fIOMDOffset;
+
+       if (ind < numIOPLs)
+           length = ioplList[ind].fIOMDOffset;
+       else
+           length = _length;
+       length -= offset;                       // Remainder within iopl
+
+       // Subtract offset till this iopl in total list
+       offset -= off2Ind;
+
+       // If a mapped address is requested and this is a pre-mapped IOPL
+       // then just need to compute an offset relative to the mapped base.
+       if (mapped && ioplInfo.fMappedBase) {
+           offset += (ioplInfo.fPageOffset & PAGE_MASK);
+           address = ptoa_64(ioplInfo.fMappedBase) + offset;
+           continue;   // Done leave do/while(false) now
+       }
+
+       // The offset is rebased into the current iopl.
+       // Now add the iopl 1st page offset.
+       offset += ioplInfo.fPageOffset;
+
+       // For external UPLs the fPageInfo field points directly to
+       // the upl's upl_page_info_t array.
+       if (ioplInfo.fFlags & kIOPLExternUPL)
+           pageList = (upl_page_info_t *) ioplInfo.fPageInfo;
+       else
+           pageList = &pageList[ioplInfo.fPageInfo];
+
+       // Check for direct device non-paged memory
+       if ( ioplInfo.fFlags & kIOPLOnDevice ) {
+           address = ptoa_64(pageList->phys_addr) + offset;
+           continue;   // Done leave do/while(false) now
+       }
+
+       // Now we need compute the index into the pageList
+       UInt pageInd = atop_32(offset);
+       offset &= PAGE_MASK;

 
 

-        } while (0);
+       // Compute the starting address of this segment
+       IOPhysicalAddress pageAddr = pageList[pageInd].phys_addr;
+       if (!pageAddr) {
+           panic("!pageList phys_addr");
+       }
+
+       address = ptoa_64(pageAddr) + offset;

 
 

-        if (!address)
-            length = 0;
+       // length is currently set to the length of the remainider of the iopl.
+       // We need to check that the remainder of the iopl is contiguous.
+       // This is indicated by pageList[ind].phys_addr being sequential.
+       IOByteCount contigLength = PAGE_SIZE - offset;
+       while (contigLength < length
+               && ++pageAddr == pageList[++pageInd].phys_addr)
+       {
+           contigLength += PAGE_SIZE;
+       }
+
+       if (contigLength < length)
+           length = contigLength;
+       
+
+       assert(address);
+       assert(length);
+
+    } while (false);
+
+    // Update return values and state
+    isP->fIO.fIOVMAddr = address;
+    isP->fIO.fLength   = length;
+    isP->fIndex        = ind;
+    isP->fOffset2Index = off2Ind;
+    isP->fNextOffset   = isP->fIO.fOffset + length;
+
+    return kIOReturnSuccess;
+}
+
+addr64_t
+IOGeneralMemoryDescriptor::getPhysicalSegment(IOByteCount offset, IOByteCount *lengthOfSegment, IOOptionBits options)
+{
+    IOReturn     ret;
+    addr64_t     address = 0;
+    IOByteCount  length  = 0;
+    IOMapper *   mapper  = gIOSystemMapper;
+    IOOptionBits type    = _flags & kIOMemoryTypeMask;
+
+    if (lengthOfSegment)
+        *lengthOfSegment = 0;
+
+    if (offset >= _length)
+        return 0;
+
+    // IOMemoryDescriptor::doMap() cannot use getPhysicalSegment() to obtain the page offset, since it must
+    // support the unwired memory case in IOGeneralMemoryDescriptor, and hibernate_write_image() cannot use
+    // map()->getVirtualAddress() to obtain the kernel pointer, since it must prevent the memory allocation
+    // due to IOMemoryMap, so _kIOMemorySourceSegment is a necessary evil until all of this gets cleaned up
+
+    if ((options & _kIOMemorySourceSegment) && (kIOMemoryTypeUPL != type))
+    {
+        unsigned rangesIndex = 0;
+       Ranges vec = _ranges;
+       user_addr_t addr;
+
+       // Find starting address within the vector of ranges
+       for (;;) {
+           getAddrLenForInd(addr, length, type, vec, rangesIndex);
+           if (offset < length)
+               break;
+           offset -= length; // (make offset relative)
+           rangesIndex++;
+       } 
+
+       // Now that we have the starting range,
+       // lets find the last contiguous range
+        addr   += offset;
+        length -= offset;
+
+        for ( ++rangesIndex; rangesIndex < _rangesCount; rangesIndex++ ) {
+           user_addr_t      newAddr;
+           IOPhysicalLength newLen;
+
+           getAddrLenForInd(newAddr, newLen, type, vec, rangesIndex);
+           if (addr + length != newAddr)
+               break;
+           length += newLen;
+       } 
+        if (addr)
+           address = (IOPhysicalAddress) addr; // Truncate address to 32bit

     }
     }

+    else
+    {
+       IOMDDMAWalkSegmentState _state;
+       IOMDDMAWalkSegmentArgs * state = (IOMDDMAWalkSegmentArgs *) &_state;
+
+       state->fOffset = offset;
+       state->fLength = _length - offset;
+       state->fMapped = (0 == (options & kIOMemoryMapperNone));
+
+       ret = dmaCommandOperation(kIOMDFirstSegment, _state, sizeof(_state));
+
+       if ((kIOReturnSuccess != ret) && (kIOReturnOverrun != ret))
+               DEBG("getPhysicalSegment dmaCommandOperation(%lx), %p, offset %qx, addr %qx, len %qx\n", 
+                                       ret, this, state->fOffset,
+                                       state->fIOVMAddr, state->fLength);
+       if (kIOReturnSuccess == ret)
+       {
+           address = state->fIOVMAddr;
+           length  = state->fLength;
+       }
+
+       // dmaCommandOperation() does not distinguish between "mapped" and "unmapped" physical memory, even
+       // with fMapped set correctly, so we must handle the transformation here until this gets cleaned up
+
+       if (mapper && ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type)))
+       {
+           if ((options & kIOMemoryMapperNone) && !(_flags & kIOMemoryMapperNone))
+           {
+               addr64_t    origAddr = address;
+               IOByteCount origLen  = length;
+
+               address = mapper->mapAddr(origAddr);
+               length = page_size - (address & (page_size - 1));
+               while ((length < origLen)
+                   && ((address + length) == mapper->mapAddr(origAddr + length)))
+                   length += page_size;
+               if (length > origLen)
+                   length = origLen;
+           }
+#ifdef __LP64__
+           else if (!(options & kIOMemoryMapperNone) && (_flags & kIOMemoryMapperNone))
+           {
+               panic("getPhysicalSegment not mapped for I/O");
+           }
+#endif /* __LP64__ */
+       }
+    }
+
+    if (!address)
+        length = 0;

 
     if (lengthOfSegment)
         *lengthOfSegment = length;
 
 
     if (lengthOfSegment)
         *lengthOfSegment = length;
 

-    return address;
+    return (address);
+}
+
+#ifndef __LP64__
+addr64_t
+IOMemoryDescriptor::getPhysicalSegment(IOByteCount offset, IOByteCount *lengthOfSegment, IOOptionBits options)
+{
+    addr64_t address = 0;
+
+    if (options & _kIOMemorySourceSegment)
+    {
+        address = getSourceSegment(offset, lengthOfSegment);
+    }
+    else if (options & kIOMemoryMapperNone)
+    {
+        address = getPhysicalSegment64(offset, lengthOfSegment);
+    }
+    else
+    {
+        address = getPhysicalSegment(offset, lengthOfSegment);
+    }
+
+    return (address);
+}
+
+addr64_t
+IOGeneralMemoryDescriptor::getPhysicalSegment64(IOByteCount offset, IOByteCount *lengthOfSegment)
+{
+    return (getPhysicalSegment(offset, lengthOfSegment, kIOMemoryMapperNone));
+}
+
+IOPhysicalAddress
+IOGeneralMemoryDescriptor::getPhysicalSegment(IOByteCount offset, IOByteCount *lengthOfSegment)
+{
+    addr64_t    address = 0;
+    IOByteCount length  = 0;
+
+    address = getPhysicalSegment(offset, lengthOfSegment, 0);
+
+    if (lengthOfSegment)
+       length = *lengthOfSegment;
+
+    if ((address + length) > 0x100000000ULL)
+    {
+       panic("getPhysicalSegment() out of 32b range 0x%qx, len 0x%lx, class %s",
+                   address, (long) length, (getMetaClass())->getClassName());
+    }
+
+    return ((IOPhysicalAddress) address);

 }
 
 }
 

-addr64_t IOMemoryDescriptor::getPhysicalSegment64
-                        (IOByteCount offset, IOByteCount *lengthOfSegment)
+addr64_t
+IOMemoryDescriptor::getPhysicalSegment64(IOByteCount offset, IOByteCount *lengthOfSegment)

 {
     IOPhysicalAddress phys32;
     IOByteCount              length;
     addr64_t         phys64;
 {
     IOPhysicalAddress phys32;
     IOByteCount              length;
     addr64_t         phys64;

+    IOMapper *        mapper = 0;

 
     phys32 = getPhysicalSegment(offset, lengthOfSegment);
     if (!phys32)
        return 0;
 
     if (gIOSystemMapper)
 
     phys32 = getPhysicalSegment(offset, lengthOfSegment);
     if (!phys32)
        return 0;
 
     if (gIOSystemMapper)

+       mapper = gIOSystemMapper;
+
+    if (mapper)

     {
        IOByteCount origLen;
 
     {
        IOByteCount origLen;
 

-       phys64 = gIOSystemMapper->mapAddr(phys32);
+       phys64 = mapper->mapAddr(phys32);

        origLen = *lengthOfSegment;
        length = page_size - (phys64 & (page_size - 1));
        while ((length < origLen)
        origLen = *lengthOfSegment;
        length = page_size - (phys64 & (page_size - 1));
        while ((length < origLen)

-           && ((phys64 + length) == gIOSystemMapper->mapAddr(phys32 + length)))
+           && ((phys64 + length) == mapper->mapAddr(phys32 + length)))

            length += page_size;
        if (length > origLen)
            length = origLen;
            length += page_size;
        if (length > origLen)
            length = origLen;


@@ -986,66 +1594,351 @@ addr64_t IOMemoryDescriptor::getPhysicalSegment64
     return phys64;
 }
 
     return phys64;
 }
 

-IOPhysicalAddress IOGeneralMemoryDescriptor::getSourceSegment
-                            (IOByteCount offset, IOByteCount *lengthOfSegment)
+IOPhysicalAddress
+IOMemoryDescriptor::getPhysicalSegment(IOByteCount offset, IOByteCount *lengthOfSegment)

 {
 {

-    IOPhysicalAddress address = 0;
-    IOPhysicalLength  length  = 0;
+    return ((IOPhysicalAddress) getPhysicalSegment(offset, lengthOfSegment, 0));
+}

 
 

-    assert(offset <= _length);
+IOPhysicalAddress
+IOGeneralMemoryDescriptor::getSourceSegment(IOByteCount offset, IOByteCount *lengthOfSegment)
+{
+    return ((IOPhysicalAddress) getPhysicalSegment(offset, lengthOfSegment, _kIOMemorySourceSegment));
+}

 
 

-    if ( (_flags & kIOMemoryTypeMask) == kIOMemoryTypeUPL)
-       return super::getSourceSegment( offset, lengthOfSegment );
+void * IOGeneralMemoryDescriptor::getVirtualSegment(IOByteCount offset,
+                                                       IOByteCount * lengthOfSegment)
+{
+    if (_task == kernel_task)
+        return (void *) getSourceSegment(offset, lengthOfSegment);
+    else
+        panic("IOGMD::getVirtualSegment deprecated");

 
 

-    if ( offset < _length ) // (within bounds?)
+    return 0;
+}
+#endif /* !__LP64__ */
+
+IOReturn 
+IOMemoryDescriptor::dmaCommandOperation(DMACommandOps op, void *vData, UInt dataSize) const
+{
+    if (kIOMDGetCharacteristics == op) {
+       if (dataSize < sizeof(IOMDDMACharacteristics))
+           return kIOReturnUnderrun;
+
+       IOMDDMACharacteristics *data = (IOMDDMACharacteristics *) vData;
+       data->fLength = getLength();
+       data->fSGCount = 0;
+       data->fDirection = getDirection();
+       if (IOMapper::gSystem)
+           data->fIsMapped = true;
+       data->fIsPrepared = true;       // Assume prepared - fails safe
+    }
+    else if (kIOMDWalkSegments & op) {
+       if (dataSize < sizeof(IOMDDMAWalkSegmentArgs))
+           return kIOReturnUnderrun;
+
+       IOMDDMAWalkSegmentArgs *data = (IOMDDMAWalkSegmentArgs *) vData;
+       IOByteCount offset  = (IOByteCount) data->fOffset;
+
+       IOPhysicalLength length;
+       IOMemoryDescriptor *ncmd = const_cast<IOMemoryDescriptor *>(this);
+       if (data->fMapped && IOMapper::gSystem)
+           data->fIOVMAddr = ncmd->getPhysicalSegment(offset, &length);
+       else
+           data->fIOVMAddr = ncmd->getPhysicalSegment(offset, &length, kIOMemoryMapperNone);
+       data->fLength = length;
+    }
+    else
+       return kIOReturnBadArgument;
+
+    return kIOReturnSuccess;
+}
+
+static IOReturn 
+purgeableControlBits(IOOptionBits newState, vm_purgable_t * control, int * state)
+{
+    IOReturn err = kIOReturnSuccess;
+
+    *control = VM_PURGABLE_SET_STATE;
+    switch (newState)

     {
     {

-        unsigned rangesIndex = 0;
+       case kIOMemoryPurgeableKeepCurrent:
+           *control = VM_PURGABLE_GET_STATE;
+           break;
+
+       case kIOMemoryPurgeableNonVolatile:
+           *state = VM_PURGABLE_NONVOLATILE;
+           break;
+       case kIOMemoryPurgeableVolatile:
+           *state = VM_PURGABLE_VOLATILE;
+           break;
+       case kIOMemoryPurgeableEmpty:
+           *state = VM_PURGABLE_EMPTY;
+           break;
+       default:
+           err = kIOReturnBadArgument;
+           break;
+    }
+    return (err);
+}

 
 

-        for ( ; offset >= _ranges.v[rangesIndex].length; rangesIndex++ )
+static IOReturn 
+purgeableStateBits(int * state)
+{
+    IOReturn err = kIOReturnSuccess;
+
+    switch (*state)
+    {
+       case VM_PURGABLE_NONVOLATILE:
+           *state = kIOMemoryPurgeableNonVolatile;
+           break;
+       case VM_PURGABLE_VOLATILE:
+           *state = kIOMemoryPurgeableVolatile;
+           break;
+       case VM_PURGABLE_EMPTY:
+           *state = kIOMemoryPurgeableEmpty;
+           break;
+       default:
+           *state = kIOMemoryPurgeableNonVolatile;
+           err = kIOReturnNotReady;
+           break;
+    }
+    return (err);
+}
+
+IOReturn 
+IOGeneralMemoryDescriptor::setPurgeable( IOOptionBits newState,
+                                                  IOOptionBits * oldState )
+{
+    IOReturn     err = kIOReturnSuccess;
+    vm_purgable_t control;
+    int           state;
+
+    if (_memEntry)
+    {
+       err = super::setPurgeable(newState, oldState);
+    }
+    else
+    {
+       if (kIOMemoryThreadSafe & _flags)
+           LOCK;
+       do
+       {
+           // Find the appropriate vm_map for the given task
+           vm_map_t curMap;
+           if (_task == kernel_task && (kIOMemoryBufferPageable & _flags))
+           {
+               err = kIOReturnNotReady;
+               break;
+           }
+           else
+               curMap = get_task_map(_task);
+
+           // can only do one range
+           Ranges vec = _ranges;
+           IOOptionBits type = _flags & kIOMemoryTypeMask;
+           user_addr_t addr; 
+           IOByteCount len;
+           getAddrLenForInd(addr, len, type, vec, 0);
+
+           err = purgeableControlBits(newState, &control, &state);
+           if (kIOReturnSuccess != err)
+               break;
+           err = mach_vm_purgable_control(curMap, addr, control, &state);
+           if (oldState)
+           {
+               if (kIOReturnSuccess == err)
+               {
+                   err = purgeableStateBits(&state);
+                   *oldState = state;
+               }
+           }
+       }
+       while (false);
+       if (kIOMemoryThreadSafe & _flags)
+           UNLOCK;
+    }
+    return (err);
+}
+
+IOReturn IOMemoryDescriptor::setPurgeable( IOOptionBits newState,
+                                           IOOptionBits * oldState )
+{
+    IOReturn     err = kIOReturnSuccess;
+    vm_purgable_t control;
+    int           state;
+
+    if (kIOMemoryThreadSafe & _flags)
+       LOCK;
+
+    do 
+    {
+        if (!_memEntry)

         {
         {

-            offset -= _ranges.v[rangesIndex].length; // (make offset relative)
+            err = kIOReturnNotReady;
+            break;

         }
         }

+       err = purgeableControlBits(newState, &control, &state);
+       if (kIOReturnSuccess != err)
+           break;
+        err = mach_memory_entry_purgable_control((ipc_port_t) _memEntry, control, &state);
+       if (oldState)
+       {
+           if (kIOReturnSuccess == err)
+           {
+               err = purgeableStateBits(&state);
+               *oldState = state;
+           }
+       }
+    }
+    while (false);

 
 

-        address = _ranges.v[rangesIndex].address + offset;
-        length  = _ranges.v[rangesIndex].length  - offset;
+    if (kIOMemoryThreadSafe & _flags)
+       UNLOCK;

 
 

-        for ( ++rangesIndex; rangesIndex < _rangesCount; rangesIndex++ )
-        {
-            if ( address + length != _ranges.v[rangesIndex].address )  break;
+    return (err);
+}

 
 

-            length += _ranges.v[rangesIndex].length; // (coalesce ranges)
-        }
+extern "C" void dcache_incoherent_io_flush64(addr64_t pa, unsigned int count);
+extern "C" void dcache_incoherent_io_store64(addr64_t pa, unsigned int count);
+
+static void SetEncryptOp(addr64_t pa, unsigned int count)
+{
+    ppnum_t page, end;

 
 

-        assert(address);
-        if ( address == 0 )  length = 0;
+    page = atop_64(round_page_64(pa));
+    end  = atop_64(trunc_page_64(pa + count));
+    for (; page < end; page++)
+    {
+        pmap_clear_noencrypt(page);    

     }
     }

+}

 
 

-    if ( lengthOfSegment )  *lengthOfSegment = length;
+static void ClearEncryptOp(addr64_t pa, unsigned int count)
+{
+    ppnum_t page, end;

 
 

-    return address;
+    page = atop_64(round_page_64(pa));
+    end  = atop_64(trunc_page_64(pa + count));
+    for (; page < end; page++)
+    {
+        pmap_set_noencrypt(page);    
+    }

 }
 
 }
 

-/* DEPRECATED */ /* USE INSTEAD: map(), readBytes(), writeBytes() */
-/* DEPRECATED */ void * IOGeneralMemoryDescriptor::getVirtualSegment(IOByteCount offset,
-/* DEPRECATED */                                                       IOByteCount * lengthOfSegment)
-/* DEPRECATED */ {
-                    if (_task == kernel_task)
-                        return (void *) getSourceSegment(offset, lengthOfSegment);
-                    else
-                        panic("IOGMD::getVirtualSegment deprecated");
+IOReturn IOMemoryDescriptor::performOperation( IOOptionBits options,
+                                                IOByteCount offset, IOByteCount length )
+{
+    IOByteCount remaining;
+    void (*func)(addr64_t pa, unsigned int count) = 0;
+
+    switch (options)
+    {
+        case kIOMemoryIncoherentIOFlush:
+            func = &dcache_incoherent_io_flush64;
+            break;
+        case kIOMemoryIncoherentIOStore:
+            func = &dcache_incoherent_io_store64;
+            break;

 
 

-                    return 0;
-/* DEPRECATED */ }
-/* DEPRECATED */ /* USE INSTEAD: map(), readBytes(), writeBytes() */
+        case kIOMemorySetEncrypted:
+            func = &SetEncryptOp;
+            break;
+        case kIOMemoryClearEncrypted:
+            func = &ClearEncryptOp;
+            break;
+    }
+
+    if (!func)
+        return (kIOReturnUnsupported);
+
+    if (kIOMemoryThreadSafe & _flags)
+       LOCK;
+
+    remaining = length = min(length, getLength() - offset);
+    while (remaining)
+    // (process another target segment?)
+    {
+        addr64_t    dstAddr64;
+        IOByteCount dstLen;
+
+        dstAddr64 = getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
+        if (!dstAddr64)
+            break;
+
+        // Clip segment length to remaining
+        if (dstLen > remaining)
+            dstLen = remaining;
+
+       (*func)(dstAddr64, dstLen);
+
+        offset    += dstLen;
+        remaining -= dstLen;
+    }
+
+    if (kIOMemoryThreadSafe & _flags)
+       UNLOCK;
+
+    return (remaining ? kIOReturnUnderrun : kIOReturnSuccess);
+}
+
+extern vm_offset_t             first_avail;
+#define io_kernel_static_end   first_avail
+
+static kern_return_t
+io_get_kernel_static_upl(
+       vm_map_t                /* map */,
+       uintptr_t               offset,
+       vm_size_t               *upl_size,
+       upl_t                   *upl,
+       upl_page_info_array_t   page_list,
+       unsigned int            *count,
+       ppnum_t                 *highest_page)
+{
+    unsigned int pageCount, page;
+    ppnum_t phys;
+    ppnum_t highestPage = 0;
+
+    pageCount = atop_32(*upl_size);
+    if (pageCount > *count)
+       pageCount = *count;
+
+    *upl = NULL;
+
+    for (page = 0; page < pageCount; page++)
+    {
+       phys = pmap_find_phys(kernel_pmap, ((addr64_t)offset) + ptoa_64(page));
+       if (!phys)
+           break;
+       page_list[page].phys_addr = phys;
+       page_list[page].pageout   = 0;
+       page_list[page].absent    = 0;
+       page_list[page].dirty     = 0;
+       page_list[page].precious  = 0;
+       page_list[page].device    = 0;
+       if (phys > highestPage)
+           highestPage = phys;
+    }
+
+    *highest_page = highestPage;
+
+    return ((page >= pageCount) ? kIOReturnSuccess : kIOReturnVMError);
+}

 
 IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection)
 {
 
 IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection)
 {

-    IOReturn error = kIOReturnNoMemory;
+    IOOptionBits type = _flags & kIOMemoryTypeMask;
+    IOReturn error = kIOReturnCannotWire;

     ioGMDData *dataP;
     ppnum_t mapBase = 0;
     IOMapper *mapper;
     ioGMDData *dataP;
     ppnum_t mapBase = 0;
     IOMapper *mapper;

+    ipc_port_t sharedMem = (ipc_port_t) _memEntry;

 
     assert(!_wireCount);
 
     assert(!_wireCount);

+    assert(kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type || kIOMemoryTypeUIO == type);
+
+    if (_pages > gIOMaximumMappedIOPageCount)
+       return kIOReturnNoResources;

 
     dataP = getDataP(_memoryEntries);
     mapper = dataP->fMapper;
 
     dataP = getDataP(_memoryEntries);
     mapper = dataP->fMapper;


@@ -1058,10 +1951,10 @@ IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection)
     dataP = 0; // May no longer be valid so lets not get tempted.
 
     if (forDirection == kIODirectionNone)
     dataP = 0; // May no longer be valid so lets not get tempted.
 
     if (forDirection == kIODirectionNone)

-        forDirection = _direction;
+        forDirection = getDirection();

 
     int uplFlags;    // This Mem Desc's default flags for upl creation
 
     int uplFlags;    // This Mem Desc's default flags for upl creation

-    switch (forDirection)
+    switch (kIODirectionOutIn & forDirection)

     {
     case kIODirectionOut:
         // Pages do not need to be marked as dirty on commit
     {
     case kIODirectionOut:
         // Pages do not need to be marked as dirty on commit


@@ -1076,55 +1969,100 @@ IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection)
     }
     uplFlags |= UPL_SET_IO_WIRE | UPL_SET_LITE;
 
     }
     uplFlags |= UPL_SET_IO_WIRE | UPL_SET_LITE;
 

-    //
-    // Check user read/write access to the data buffer.
-    //
-    unsigned int pageIndex = 0;
-    IOByteCount mdOffset = 0;
+#ifdef UPL_NEED_32BIT_ADDR
+    if (kIODirectionPrepareToPhys32 & forDirection) 
+       uplFlags |= UPL_NEED_32BIT_ADDR;
+#endif
+
+    // Find the appropriate vm_map for the given task

     vm_map_t curMap;
     if (_task == kernel_task && (kIOMemoryBufferPageable & _flags))
         curMap = 0;
     else
         { curMap = get_task_map(_task); }
 
     vm_map_t curMap;
     if (_task == kernel_task && (kIOMemoryBufferPageable & _flags))
         curMap = 0;
     else
         { curMap = get_task_map(_task); }
 

+    // Iterate over the vector of virtual ranges
+    Ranges vec = _ranges;
+    unsigned int pageIndex = 0;
+    IOByteCount mdOffset = 0;
+    ppnum_t highestPage = 0;

     for (UInt range = 0; range < _rangesCount; range++) {
         ioPLBlock iopl;
     for (UInt range = 0; range < _rangesCount; range++) {
         ioPLBlock iopl;

-        IOVirtualRange curRange = _ranges.v[range];
-        vm_address_t startPage;
+       user_addr_t startPage;

         IOByteCount numBytes;
         IOByteCount numBytes;

+       ppnum_t highPage = 0;
+
+       // Get the startPage address and length of vec[range]
+       getAddrLenForInd(startPage, numBytes, type, vec, range);
+       iopl.fPageOffset = startPage & PAGE_MASK;
+       numBytes += iopl.fPageOffset;
+       startPage = trunc_page_64(startPage);

 
 

-        startPage = trunc_page_32(curRange.address);
-        iopl.fPageOffset = (short) curRange.address & PAGE_MASK;

        if (mapper)
            iopl.fMappedBase = mapBase + pageIndex;
        else
            iopl.fMappedBase = 0;
        if (mapper)
            iopl.fMappedBase = mapBase + pageIndex;
        else
            iopl.fMappedBase = 0;

-        numBytes = iopl.fPageOffset + curRange.length;

 
 

+       // Iterate over the current range, creating UPLs

         while (numBytes) {
             dataP = getDataP(_memoryEntries);
         while (numBytes) {
             dataP = getDataP(_memoryEntries);

-            vm_map_t theMap =
-                (curMap)? curMap 
-                        : IOPageableMapForAddress(startPage);
+           vm_address_t kernelStart = (vm_address_t) startPage;
+            vm_map_t theMap;
+           if (curMap)
+               theMap = curMap;
+           else if (!sharedMem) {
+               assert(_task == kernel_task);
+               theMap = IOPageableMapForAddress(kernelStart);
+           }
+           else
+               theMap = NULL;
+

             upl_page_info_array_t pageInfo = getPageList(dataP);
             int ioplFlags = uplFlags;
             upl_page_list_ptr_t baseInfo = &pageInfo[pageIndex];
 
             upl_page_info_array_t pageInfo = getPageList(dataP);
             int ioplFlags = uplFlags;
             upl_page_list_ptr_t baseInfo = &pageInfo[pageIndex];
 

-            vm_size_t ioplSize = round_page_32(numBytes);
+            vm_size_t ioplSize = round_page(numBytes);

             unsigned int numPageInfo = atop_32(ioplSize);
             unsigned int numPageInfo = atop_32(ioplSize);

-            error = vm_map_get_upl(theMap, 
-                                   startPage,
-                                   &ioplSize,
-                                   &iopl.fIOPL,
-                                   baseInfo,
-                                   &numPageInfo,
-                                   &ioplFlags,
-                                   false);
+
+           if (theMap == kernel_map && kernelStart < io_kernel_static_end) {
+               error = io_get_kernel_static_upl(theMap, 
+                                               kernelStart,
+                                               &ioplSize,
+                                               &iopl.fIOPL,
+                                               baseInfo,
+                                               &numPageInfo,
+                                               &highPage);
+           }
+           else if (sharedMem) {
+               error = memory_object_iopl_request(sharedMem, 
+                                               ptoa_32(pageIndex),
+                                               &ioplSize,
+                                               &iopl.fIOPL,
+                                               baseInfo,
+                                               &numPageInfo,
+                                               &ioplFlags);
+           }
+           else {
+               assert(theMap);
+               error = vm_map_create_upl(theMap,
+                                               startPage,
+                                               (upl_size_t*)&ioplSize,
+                                               &iopl.fIOPL,
+                                               baseInfo,
+                                               &numPageInfo,
+                                               &ioplFlags);
+           }
+

             assert(ioplSize);
             if (error != KERN_SUCCESS)
                 goto abortExit;
 
             assert(ioplSize);
             if (error != KERN_SUCCESS)
                 goto abortExit;
 

-            error = kIOReturnNoMemory;
+           if (iopl.fIOPL)
+               highPage = upl_get_highest_page(iopl.fIOPL);
+           if (highPage > highestPage)
+               highestPage = highPage;
+
+            error = kIOReturnCannotWire;

 
             if (baseInfo->device) {
                 numPageInfo = 1;
 
             if (baseInfo->device) {
                 numPageInfo = 1;


@@ -1138,7 +2076,7 @@ IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection)
             }
             else {
                 iopl.fFlags = 0;
             }
             else {
                 iopl.fFlags = 0;

-                if (mapper)
+               if (mapper)

                     mapper->iovmInsert(mapBase, pageIndex,
                                        baseInfo, numPageInfo);
             }
                     mapper->iovmInsert(mapBase, pageIndex,
                                        baseInfo, numPageInfo);
             }


@@ -1146,16 +2084,23 @@ IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection)
             iopl.fIOMDOffset = mdOffset;
             iopl.fPageInfo = pageIndex;
 
             iopl.fIOMDOffset = mdOffset;
             iopl.fPageInfo = pageIndex;
 

-           if (_flags & kIOMemoryAutoPrepare)
+#if 0
+           // used to remove the upl for auto prepares here, for some errant code
+           // that freed memory before the descriptor pointing at it
+           if ((_flags & kIOMemoryAutoPrepare) && iopl.fIOPL)

            {
            {

-               kernel_upl_commit(iopl.fIOPL, 0, 0);
+               upl_commit(iopl.fIOPL, 0, 0);
+               upl_deallocate(iopl.fIOPL);

                iopl.fIOPL = 0;
            }
                iopl.fIOPL = 0;
            }

+#endif

 
             if (!_memoryEntries->appendBytes(&iopl, sizeof(iopl))) {
                 // Clean up partial created and unsaved iopl
 
             if (!_memoryEntries->appendBytes(&iopl, sizeof(iopl))) {
                 // Clean up partial created and unsaved iopl

-               if (iopl.fIOPL)
-                   kernel_upl_abort(iopl.fIOPL, 0);
+                if (iopl.fIOPL) {
+                    upl_abort(iopl.fIOPL, 0);
+                    upl_deallocate(iopl.fIOPL);
+                }

                 goto abortExit;
             }
 
                 goto abortExit;
             }
 


@@ -1177,24 +2122,32 @@ IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection)
         }
     }
 
         }
     }
 

+    _highestPage = highestPage;
+

     return kIOReturnSuccess;
 
 abortExit:
     {
         dataP = getDataP(_memoryEntries);
     return kIOReturnSuccess;
 
 abortExit:
     {
         dataP = getDataP(_memoryEntries);

-        UInt done = getNumIOPL(dataP, _memoryEntries->getLength());
+        UInt done = getNumIOPL(_memoryEntries, dataP);

         ioPLBlock *ioplList = getIOPLList(dataP);
     
         for (UInt range = 0; range < done; range++)
        {
         ioPLBlock *ioplList = getIOPLList(dataP);
     
         for (UInt range = 0; range < done; range++)
        {

-           if (ioplList[range].fIOPL)
-               kernel_upl_abort(ioplList[range].fIOPL, 0);
+           if (ioplList[range].fIOPL) {
+             upl_abort(ioplList[range].fIOPL, 0);
+             upl_deallocate(ioplList[range].fIOPL);
+           }

        }
        }

+       (void) _memoryEntries->initWithBytes(dataP, computeDataSize(0, 0)); // == setLength()

 
         if (mapper && mapBase)
             mapper->iovmFree(mapBase, _pages);
     }
 
 
         if (mapper && mapBase)
             mapper->iovmFree(mapBase, _pages);
     }
 

+    if (error == KERN_FAILURE)
+        error = kIOReturnCannotWire;
+

     return error;
 }
 
     return error;
 }
 


@@ -1209,17 +2162,35 @@ abortExit:
  */
 IOReturn IOGeneralMemoryDescriptor::prepare(IODirection forDirection)
 {
  */
 IOReturn IOGeneralMemoryDescriptor::prepare(IODirection forDirection)
 {

-    IOReturn error = kIOReturnSuccess;
+    IOReturn error    = kIOReturnSuccess;
+    IOOptionBits type = _flags & kIOMemoryTypeMask;
+
+    if ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type))
+       return kIOReturnSuccess;

 
 

-    if (!_wireCount && (_flags & kIOMemoryTypeMask) == kIOMemoryTypeVirtual) {
+    if (_prepareLock)
+       IOLockLock(_prepareLock);
+
+    if (!_wireCount
+    && (kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type || kIOMemoryTypeUIO == type) ) {

         error = wireVirtual(forDirection);
         error = wireVirtual(forDirection);

-        if (error)
-            return error;

     }
 
     }
 

-    _wireCount++;
+    if (kIOReturnSuccess == error)
+       _wireCount++;

 
 

-    return kIOReturnSuccess;
+    if (1 == _wireCount)
+    {
+        if (kIOMemoryClearEncrypt & _flags)
+        {
+            performOperation(kIOMemoryClearEncrypted, 0, _length);
+        }
+    }
+
+    if (_prepareLock)
+       IOLockUnlock(_prepareLock);
+
+    return error;

 }
 
 /*
 }
 
 /*


@@ -1230,92 +2201,234 @@ IOReturn IOGeneralMemoryDescriptor::prepare(IODirection forDirection)
  * issued; the prepare() and complete() must occur in pairs, before
  * before and after an I/O transfer involving pageable memory.
  */
  * issued; the prepare() and complete() must occur in pairs, before
  * before and after an I/O transfer involving pageable memory.
  */

- 
+

 IOReturn IOGeneralMemoryDescriptor::complete(IODirection /* forDirection */)
 {
 IOReturn IOGeneralMemoryDescriptor::complete(IODirection /* forDirection */)
 {

-    assert(_wireCount);
+    IOOptionBits type = _flags & kIOMemoryTypeMask;

 
 

-    if (!_wireCount)
-        return kIOReturnSuccess;
+    if ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type))
+       return kIOReturnSuccess;

 
 

-    _wireCount--;
-    if (!_wireCount) {
-        if ((_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical) {
-            /* kIOMemoryTypePhysical */
-            // DO NOTHING
-        }
-        else {
-            ioGMDData * dataP = getDataP(_memoryEntries);
-            ioPLBlock *ioplList = getIOPLList(dataP);
-           UInt count = getNumIOPL(dataP, _memoryEntries->getLength());
-
-            if (dataP->fMapper && _pages && ioplList[0].fMappedBase)
-                dataP->fMapper->iovmFree(ioplList[0].fMappedBase, _pages);
-
-            // Only complete iopls that we created which are for TypeVirtual
-            if ( (_flags & kIOMemoryTypeMask) == kIOMemoryTypeVirtual) {
-                for (UInt ind = 0; ind < count; ind++)
-                   if (ioplList[ind].fIOPL)
-                       kernel_upl_commit(ioplList[ind].fIOPL, 0, 0);
-            }
+    if (_prepareLock)
+       IOLockLock(_prepareLock);
+
+    assert(_wireCount);

 
 

-            (void) _memoryEntries->initWithBytes(dataP, sizeof(ioGMDData)); // == setLength()
+    if (_wireCount)
+    {
+        if ((kIOMemoryClearEncrypt & _flags) && (1 == _wireCount))
+        {
+            performOperation(kIOMemorySetEncrypted, 0, _length);

         }
         }

+
+       _wireCount--;
+       if (!_wireCount)
+       {
+           IOOptionBits type = _flags & kIOMemoryTypeMask;
+           ioGMDData * dataP = getDataP(_memoryEntries);
+           ioPLBlock *ioplList = getIOPLList(dataP);
+           UInt count = getNumIOPL(_memoryEntries, dataP);
+
+#if IOMD_DEBUG_DMAACTIVE
+           if (__iomd_reservedA) panic("complete() while dma active");
+#endif /* IOMD_DEBUG_DMAACTIVE */
+
+           if (dataP->fMapper && _pages && ioplList[0].fMappedBase)
+               dataP->fMapper->iovmFree(ioplList[0].fMappedBase, _pages);
+
+           // Only complete iopls that we created which are for TypeVirtual
+           if (kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type || kIOMemoryTypeUIO == type) {
+               for (UInt ind = 0; ind < count; ind++)
+                   if (ioplList[ind].fIOPL) {
+                        upl_commit(ioplList[ind].fIOPL, 0, 0);
+                        upl_deallocate(ioplList[ind].fIOPL);
+                   }
+           } else if (kIOMemoryTypeUPL == type) {
+               upl_set_referenced(ioplList[0].fIOPL, false);
+           }
+
+           (void) _memoryEntries->initWithBytes(dataP, computeDataSize(0, 0)); // == setLength()
+
+           dataP->fPreparationID = kIOPreparationIDUnprepared;
+       }

     }
     }

+
+    if (_prepareLock)
+       IOLockUnlock(_prepareLock);
+

     return kIOReturnSuccess;
 }
 
 IOReturn IOGeneralMemoryDescriptor::doMap(
     return kIOReturnSuccess;
 }
 
 IOReturn IOGeneralMemoryDescriptor::doMap(

-       vm_map_t                addressMap,
-       IOVirtualAddress *      atAddress,
+       vm_map_t                __addressMap,
+       IOVirtualAddress *      __address,

        IOOptionBits            options,
        IOOptionBits            options,

-       IOByteCount             sourceOffset,
-       IOByteCount             length )
+       IOByteCount             __offset,
+       IOByteCount             __length )
+

 {
 {

-    kern_return_t kr;
+#ifndef __LP64__
+    if (!(kIOMap64Bit & options)) panic("IOGeneralMemoryDescriptor::doMap !64bit");
+#endif /* !__LP64__ */
+
+    IOMemoryMap *  mapping = (IOMemoryMap *) *__address;
+    mach_vm_size_t offset  = mapping->fOffset + __offset;
+    mach_vm_size_t length  = mapping->fLength;
+
+    kern_return_t kr = kIOReturnVMError;

     ipc_port_t sharedMem = (ipc_port_t) _memEntry;
 
     ipc_port_t sharedMem = (ipc_port_t) _memEntry;
 

+    IOOptionBits type = _flags & kIOMemoryTypeMask;
+    Ranges vec = _ranges;
+
+    user_addr_t range0Addr = 0;
+    IOByteCount range0Len = 0;
+
+    if ((offset >= _length) || ((offset + length) > _length))
+       return( kIOReturnBadArgument );
+
+    if (vec.v)
+       getAddrLenForInd(range0Addr, range0Len, type, vec, 0);
+

     // mapping source == dest? (could be much better)
     // mapping source == dest? (could be much better)

-    if( _task && (addressMap == get_task_map(_task)) && (options & kIOMapAnywhere)
-       && (1 == _rangesCount) && (0 == sourceOffset)
-       && (length <= _ranges.v[0].length) ) {
-           *atAddress = _ranges.v[0].address;
-           return( kIOReturnSuccess );
+    if( _task
+     && (mapping->fAddressMap == get_task_map(_task)) && (options & kIOMapAnywhere)
+     && (1 == _rangesCount) && (0 == offset)
+     && range0Addr && (length <= range0Len) )
+    {
+       mapping->fAddress = range0Addr;
+       mapping->fOptions |= kIOMapStatic;
+
+       return( kIOReturnSuccess );

     }
 
     if( 0 == sharedMem) {
 
     }
 
     if( 0 == sharedMem) {
 

-        vm_size_t size = _pages << PAGE_SHIFT;
+        vm_size_t size = ptoa_32(_pages);

 
         if( _task) {
 
         if( _task) {

-#ifndef i386
-            vm_size_t actualSize = size;
-            kr = mach_make_memory_entry( get_task_map(_task),
-                        &actualSize, _ranges.v[0].address,
-                        VM_PROT_READ | VM_PROT_WRITE, &sharedMem,
-                        NULL );
-
-            if( (KERN_SUCCESS == kr) && (actualSize != round_page_32(size))) {
+
+            memory_object_size_t actualSize = size;
+           vm_prot_t            prot       = VM_PROT_READ;
+           if (!(kIOMapReadOnly & options))
+               prot |= VM_PROT_WRITE;
+           else if (kIOMapDefaultCache != (options & kIOMapCacheMask))
+               prot |= VM_PROT_WRITE;
+
+            if (_rangesCount == 1)
+            {
+                kr = mach_make_memory_entry_64(get_task_map(_task),
+                                                &actualSize, range0Addr,
+                                                prot, &sharedMem,
+                                                NULL);
+            }
+            if( (_rangesCount != 1) 
+                || ((KERN_SUCCESS == kr) && (actualSize != round_page(size))))
+            do
+           {

 #if IOASSERT
 #if IOASSERT

-                IOLog("mach_make_memory_entry_64 (%08x) size (%08lx:%08x)\n",
-                            _ranges.v[0].address, (UInt32)actualSize, size);
+                IOLog("mach_vm_remap path for ranges %d size (%08llx:%08llx)\n",
+                     _rangesCount, (UInt64)actualSize, (UInt64)size);

 #endif
                 kr = kIOReturnVMError;
 #endif
                 kr = kIOReturnVMError;

-                ipc_port_release_send( sharedMem );
-            }
-
-            if( KERN_SUCCESS != kr)
-#endif /* i386 */
-                sharedMem = MACH_PORT_NULL;
+                if (sharedMem)
+                {
+                    ipc_port_release_send(sharedMem);
+                    sharedMem = MACH_PORT_NULL;
+                }

 
 

-        } else do {
+               mach_vm_address_t address, segDestAddr;
+                mach_vm_size_t    mapLength;
+                unsigned          rangesIndex;
+                IOOptionBits      type = _flags & kIOMemoryTypeMask;
+                user_addr_t       srcAddr;
+                IOPhysicalLength  segLen = 0;
+
+                // Find starting address within the vector of ranges
+                for (rangesIndex = 0; rangesIndex < _rangesCount; rangesIndex++) {
+                    getAddrLenForInd(srcAddr, segLen, type, _ranges, rangesIndex);
+                    if (offset < segLen)
+                        break;
+                    offset -= segLen; // (make offset relative)
+                } 
+
+               mach_vm_size_t    pageOffset = (srcAddr & PAGE_MASK);
+               address = trunc_page_64(mapping->fAddress);
+
+               if ((options & kIOMapAnywhere) || ((mapping->fAddress - address) == pageOffset))
+               {
+                   vm_map_t map = mapping->fAddressMap;
+                   kr = IOMemoryDescriptorMapCopy(&map, 
+                                                   options,
+                                                   offset, &address, round_page_64(length + pageOffset));
+                    if (kr == KERN_SUCCESS)
+                    {
+                        segDestAddr  = address;
+                        segLen      -= offset;
+                        mapLength    = length;
+
+                        while (true)
+                        {
+                            vm_prot_t cur_prot, max_prot;
+                            kr = mach_vm_remap(map, &segDestAddr, round_page_64(segLen), PAGE_MASK, 
+                                                    VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE,
+                                                    get_task_map(_task), trunc_page_64(srcAddr),
+                                                    FALSE /* copy */,
+                                                    &cur_prot,
+                                                    &max_prot,
+                                                    VM_INHERIT_NONE);
+                            if (KERN_SUCCESS == kr)
+                            {
+                                if ((!(VM_PROT_READ & cur_prot))
+                                    || (!(kIOMapReadOnly & options) && !(VM_PROT_WRITE & cur_prot)))
+                                {
+                                    kr = KERN_PROTECTION_FAILURE;
+                                }
+                            }
+                            if (KERN_SUCCESS != kr)
+                                break;
+                            segDestAddr += segLen;
+                            mapLength   -= segLen;
+                            if (!mapLength)
+                                break;
+                            rangesIndex++;
+                            if (rangesIndex >= _rangesCount)
+                            {
+                                kr = kIOReturnBadArgument;
+                                break;
+                            }
+                            getAddrLenForInd(srcAddr, segLen, type, vec, rangesIndex);
+                            if (srcAddr & PAGE_MASK)
+                            {
+                                kr = kIOReturnBadArgument;
+                                break;
+                            }
+                            if (segLen > mapLength)
+                                segLen = mapLength;
+                        } 
+                        if (KERN_SUCCESS != kr)
+                        {
+                            mach_vm_deallocate(mapping->fAddressMap, address, round_page_64(length + pageOffset));
+                        }
+                    }
+
+                   if (KERN_SUCCESS == kr)
+                       mapping->fAddress = address + pageOffset;
+                   else
+                       mapping->fAddress = NULL;
+               }
+            }
+            while (false);
+        } 
+       else do
+       {       // _task == 0, must be physical

 
             memory_object_t    pager;
            unsigned int        flags = 0;
            addr64_t            pa;
            IOPhysicalLength    segLen;
 
 
             memory_object_t    pager;
            unsigned int        flags = 0;
            addr64_t            pa;
            IOPhysicalLength    segLen;
 

-           pa = getPhysicalSegment64( sourceOffset, &segLen );
+           pa = getPhysicalSegment( offset, &segLen, kIOMemoryMapperNone );

 
             if( !reserved) {
                 reserved = IONew( ExpansionData, 1 );
 
             if( !reserved) {
                 reserved = IONew( ExpansionData, 1 );


@@ -1331,6 +2444,17 @@ IOReturn IOGeneralMemoryDescriptor::doMap(
                case kIOMapDefaultCache:
                default:
                    flags = IODefaultCacheBits(pa);
                case kIOMapDefaultCache:
                default:
                    flags = IODefaultCacheBits(pa);

+                   if (DEVICE_PAGER_CACHE_INHIB & flags)
+                   {
+                       if (DEVICE_PAGER_GUARDED & flags)
+                           mapping->fOptions |= kIOMapInhibitCache;
+                       else
+                           mapping->fOptions |= kIOMapWriteCombineCache;
+                   }
+                   else if (DEVICE_PAGER_WRITE_THROUGH & flags)
+                       mapping->fOptions |= kIOMapWriteThruCache;
+                   else
+                       mapping->fOptions |= kIOMapCopybackCache;

                    break;
        
                case kIOMapInhibitCache:
                    break;
        
                case kIOMapInhibitCache:


@@ -1355,7 +2479,7 @@ IOReturn IOGeneralMemoryDescriptor::doMap(
 
            flags |= reserved->pagerContig ? DEVICE_PAGER_CONTIGUOUS : 0;
 
 
            flags |= reserved->pagerContig ? DEVICE_PAGER_CONTIGUOUS : 0;
 

-            pager = device_pager_setup( (memory_object_t) 0, (int) reserved, 
+            pager = device_pager_setup( (memory_object_t) 0, (uintptr_t) reserved, 

                                                                size, flags);
             assert( pager );
 
                                                                size, flags);
             assert( pager );
 


@@ -1364,7 +2488,8 @@ IOReturn IOGeneralMemoryDescriptor::doMap(
                             size, VM_PROT_READ | VM_PROT_WRITE, pager, &sharedMem );
 
                 assert( KERN_SUCCESS == kr );
                             size, VM_PROT_READ | VM_PROT_WRITE, pager, &sharedMem );
 
                 assert( KERN_SUCCESS == kr );

-                if( KERN_SUCCESS != kr) {
+                if( KERN_SUCCESS != kr)
+               {

                    device_pager_deallocate( pager );
                     pager = MACH_PORT_NULL;
                     sharedMem = MACH_PORT_NULL;
                    device_pager_deallocate( pager );
                     pager = MACH_PORT_NULL;
                     sharedMem = MACH_PORT_NULL;


@@ -1382,203 +2507,104 @@ IOReturn IOGeneralMemoryDescriptor::doMap(
         _memEntry = (void *) sharedMem;
     }
 
         _memEntry = (void *) sharedMem;
     }
 

-#ifndef i386
-    if( 0 == sharedMem)
-      kr = kIOReturnVMError;
+    IOReturn result;
+    if (0 == sharedMem)
+      result = kr;

     else
     else

-#endif
-      kr = super::doMap( addressMap, atAddress,
-                           options, sourceOffset, length );
+      result = super::doMap( __addressMap, __address,
+                                       options, __offset, __length );

 
 

-    return( kr );
+    return( result );

 }
 
 IOReturn IOGeneralMemoryDescriptor::doUnmap(
        vm_map_t                addressMap,
 }
 
 IOReturn IOGeneralMemoryDescriptor::doUnmap(
        vm_map_t                addressMap,

-       IOVirtualAddress        logical,
-       IOByteCount             length )
+       IOVirtualAddress        __address,
+       IOByteCount             __length )

 {
 {

-    // could be much better
-    if( _task && (addressMap == get_task_map(_task)) && (1 == _rangesCount)
-        && (logical == _ranges.v[0].address)
-        && (length <= _ranges.v[0].length) )
-           return( kIOReturnSuccess );
-
-    return( super::doUnmap( addressMap, logical, length ));
+    return (super::doUnmap(addressMap, __address, __length));

 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 

-extern "C" {
-// osfmk/device/iokit_rpc.c
-extern kern_return_t IOMapPages( vm_map_t map, vm_offset_t va, vm_offset_t pa,
-                                 vm_size_t length, unsigned int mapFlags);
-extern kern_return_t IOUnmapPages(vm_map_t map, vm_offset_t va, vm_size_t length);
-};
+#undef super
+#define super OSObject

 
 

-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+OSDefineMetaClassAndStructors( IOMemoryMap, OSObject )

 
 

-OSDefineMetaClassAndAbstractStructors( IOMemoryMap, OSObject )
+OSMetaClassDefineReservedUnused(IOMemoryMap, 0);
+OSMetaClassDefineReservedUnused(IOMemoryMap, 1);
+OSMetaClassDefineReservedUnused(IOMemoryMap, 2);
+OSMetaClassDefineReservedUnused(IOMemoryMap, 3);
+OSMetaClassDefineReservedUnused(IOMemoryMap, 4);
+OSMetaClassDefineReservedUnused(IOMemoryMap, 5);
+OSMetaClassDefineReservedUnused(IOMemoryMap, 6);
+OSMetaClassDefineReservedUnused(IOMemoryMap, 7);

 
 

-/* inline function implementation */
+/* ex-inline function implementation */

 IOPhysicalAddress IOMemoryMap::getPhysicalAddress()
     { return( getPhysicalSegment( 0, 0 )); }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 IOPhysicalAddress IOMemoryMap::getPhysicalAddress()
     { return( getPhysicalSegment( 0, 0 )); }
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 

-class _IOMemoryMap : public IOMemoryMap
-{
-    OSDeclareDefaultStructors(_IOMemoryMap)
-
-    IOMemoryDescriptor * memory;
-    IOMemoryMap *      superMap;
-    IOByteCount                offset;
-    IOByteCount                length;
-    IOVirtualAddress   logical;
-    task_t             addressTask;
-    vm_map_t           addressMap;
-    IOOptionBits       options;
-
-protected:
-    virtual void taggedRelease(const void *tag = 0) const;
-    virtual void free();
-
-public:
-
-    // IOMemoryMap methods
-    virtual IOVirtualAddress   getVirtualAddress();
-    virtual IOByteCount        getLength();
-    virtual task_t             getAddressTask();
-    virtual IOMemoryDescriptor * getMemoryDescriptor();
-    virtual IOOptionBits       getMapOptions();
-
-    virtual IOReturn           unmap();
-    virtual void               taskDied();
-
-    virtual IOPhysicalAddress  getPhysicalSegment(IOByteCount offset,
-                                                  IOByteCount * length);
-
-    // for IOMemoryDescriptor use
-    _IOMemoryMap * copyCompatible(
-               IOMemoryDescriptor *    owner,
-                task_t                 intoTask,
-                IOVirtualAddress       toAddress,
-                IOOptionBits           options,
-                IOByteCount            offset,
-                IOByteCount            length );
-
-    bool initCompatible(
-       IOMemoryDescriptor *    memory,
-       IOMemoryMap *           superMap,
-        IOByteCount            offset,
-        IOByteCount            length );
-
-    bool initWithDescriptor(
-       IOMemoryDescriptor *    memory,
-       task_t                  intoTask,
-       IOVirtualAddress        toAddress,
-       IOOptionBits            options,
-        IOByteCount            offset,
-        IOByteCount            length );
-
-    IOReturn redirect(
-       task_t                  intoTask, bool redirect );
-};
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-#undef super
-#define super IOMemoryMap
-
-OSDefineMetaClassAndStructors(_IOMemoryMap, IOMemoryMap)
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-bool _IOMemoryMap::initCompatible(
-       IOMemoryDescriptor *    _memory,
-       IOMemoryMap *           _superMap,
-        IOByteCount            _offset,
-        IOByteCount            _length )
+bool IOMemoryMap::init(
+        task_t                 intoTask,
+        mach_vm_address_t      toAddress,
+        IOOptionBits           _options,
+        mach_vm_size_t         _offset,
+        mach_vm_size_t         _length )

 {
 {

-
-    if( !super::init())
+    if (!intoTask)

        return( false);
 
        return( false);
 

-    if( (_offset + _length) > _superMap->getLength())
-       return( false);
+    if (!super::init())
+       return(false);

 
 

-    _memory->retain();
-    memory     = _memory;
-    _superMap->retain();
-    superMap   = _superMap;
-
-    offset     = _offset;
-    if( _length)
-        length = _length;
-    else
-        length = _memory->getLength();
+    fAddressMap  = get_task_map(intoTask);
+    if (!fAddressMap)
+       return(false);
+    vm_map_reference(fAddressMap);

 
 

-    options    = superMap->getMapOptions();
-    logical    = superMap->getVirtualAddress() + offset;
+    fAddressTask = intoTask;
+    fOptions     = _options;
+    fLength      = _length;
+    fOffset     = _offset;
+    fAddress     = toAddress;

 
 

-    return( true );
+    return (true);

 }
 
 }
 

-bool _IOMemoryMap::initWithDescriptor(
-        IOMemoryDescriptor *   _memory,
-        task_t                 intoTask,
-        IOVirtualAddress       toAddress,
-        IOOptionBits           _options,
-        IOByteCount            _offset,
-        IOByteCount            _length )
+bool IOMemoryMap::setMemoryDescriptor(IOMemoryDescriptor * _memory, mach_vm_size_t _offset)

 {
 {

-    bool       ok;
+    if (!_memory)
+       return(false);

 
 

-    if( (!_memory) || (!intoTask) || !super::init())
-       return( false);
-
-    if( (_offset + _length) > _memory->getLength())
-       return( false);
-
-    addressMap  = get_task_map(intoTask);
-    if( !addressMap)
-       return( false);
-    vm_map_reference(addressMap);
+    if (!fSuperMap)
+    {
+       if( (_offset + fLength) > _memory->getLength())
+           return( false);
+       fOffset = _offset;
+    }

 
     _memory->retain();
 
     _memory->retain();

-    memory     = _memory;
-
-    offset     = _offset;
-    if( _length)
-        length = _length;
-    else
-        length = _memory->getLength();
-
-    addressTask        = intoTask;
-    logical    = toAddress;
-    options    = _options;
-
-    if( options & kIOMapStatic)
-       ok = true;
-    else
-       ok = (kIOReturnSuccess == memory->doMap( addressMap, &logical,
-                                                options, offset, length ));
-    if( !ok) {
-       logical = 0;
-        memory->release();
-        memory = 0;
-        vm_map_deallocate(addressMap);
-        addressMap = 0;
+    if (fMemory)
+    {
+       if (fMemory != _memory)
+           fMemory->removeMapping(this);
+       fMemory->release();

     }
     }

-    return( ok );
+    fMemory = _memory;
+
+    return( true );

 }
 
 struct IOMemoryDescriptorMapAllocRef
 {
     ipc_port_t         sharedMem;
 }
 
 struct IOMemoryDescriptorMapAllocRef
 {
     ipc_port_t         sharedMem;

-    vm_size_t          size;
-    vm_offset_t                mapped;
-    IOByteCount                sourceOffset;
+    vm_map_t            map;
+    mach_vm_address_t  mapped;
+    mach_vm_size_t     size;
+    mach_vm_size_t     sourceOffset;

     IOOptionBits       options;
 };
 
     IOOptionBits       options;
 };
 


@@ -1588,10 +2614,15 @@ static kern_return_t IOMemoryDescriptorMapAlloc(vm_map_t map, void * _ref)
     IOReturn                       err;
 
     do {
     IOReturn                       err;
 
     do {

-        if( ref->sharedMem) {
+        if( ref->sharedMem)
+       {

             vm_prot_t prot = VM_PROT_READ
                             | ((ref->options & kIOMapReadOnly) ? 0 : VM_PROT_WRITE);
 
             vm_prot_t prot = VM_PROT_READ
                             | ((ref->options & kIOMapReadOnly) ? 0 : VM_PROT_WRITE);
 

+           // VM system requires write access to change cache mode
+           if (kIOMapDefaultCache != (ref->options & kIOMapCacheMask))
+               prot |= VM_PROT_WRITE;
+

             // set memory entry cache
             vm_prot_t memEntryCacheMode = prot | MAP_MEM_ONLY;
             switch (ref->options & kIOMapCacheMask)
             // set memory entry cache
             vm_prot_t memEntryCacheMode = prot | MAP_MEM_ONLY;
             switch (ref->options & kIOMapCacheMask)


@@ -1625,7 +2656,7 @@ static kern_return_t IOMemoryDescriptorMapAlloc(vm_map_t map, void * _ref)
             if (KERN_SUCCESS != err)
                 IOLog("MAP_MEM_ONLY failed %d\n", err);
 
             if (KERN_SUCCESS != err)
                 IOLog("MAP_MEM_ONLY failed %d\n", err);
 

-            err = vm_map( map,
+            err = mach_vm_map( map,

                             &ref->mapped,
                             ref->size, 0 /* mask */, 
                             (( ref->options & kIOMapAnywhere ) ? VM_FLAGS_ANYWHERE : VM_FLAGS_FIXED)
                             &ref->mapped,
                             ref->size, 0 /* mask */, 
                             (( ref->options & kIOMapAnywhere ) ? VM_FLAGS_ANYWHERE : VM_FLAGS_FIXED)


@@ -1635,129 +2666,250 @@ static kern_return_t IOMemoryDescriptorMapAlloc(vm_map_t map, void * _ref)
                             prot, // cur
                             prot, // max
                             VM_INHERIT_NONE);
                             prot, // cur
                             prot, // max
                             VM_INHERIT_NONE);

-    
+

             if( KERN_SUCCESS != err) {
                 ref->mapped = 0;
                 continue;
             }
             if( KERN_SUCCESS != err) {
                 ref->mapped = 0;
                 continue;
             }

-    
-        } else {
-    
-            err = vm_allocate( map, &ref->mapped, ref->size,
+            ref->map = map;
+        }
+       else
+       {
+            err = mach_vm_allocate(map, &ref->mapped, ref->size,

                             ((ref->options & kIOMapAnywhere) ? VM_FLAGS_ANYWHERE : VM_FLAGS_FIXED)
                             | VM_MAKE_TAG(VM_MEMORY_IOKIT) );
                             ((ref->options & kIOMapAnywhere) ? VM_FLAGS_ANYWHERE : VM_FLAGS_FIXED)
                             | VM_MAKE_TAG(VM_MEMORY_IOKIT) );

-    

             if( KERN_SUCCESS != err) {
                 ref->mapped = 0;
                 continue;
             }
             if( KERN_SUCCESS != err) {
                 ref->mapped = 0;
                 continue;
             }

-    
+            ref->map = map;

             // we have to make sure that these guys don't get copied if we fork.
             // we have to make sure that these guys don't get copied if we fork.

-            err = vm_inherit( map, ref->mapped, ref->size, VM_INHERIT_NONE);
+            err = vm_inherit(map, ref->mapped, ref->size, VM_INHERIT_NONE);

             assert( KERN_SUCCESS == err );
         }
             assert( KERN_SUCCESS == err );
         }

-
-    } while( false );
+    }
+    while( false );

 
     return( err );
 }
 
 
     return( err );
 }
 

+kern_return_t 
+IOMemoryDescriptorMapMemEntry(vm_map_t * map, ipc_port_t entry, IOOptionBits options, bool pageable,
+                               mach_vm_size_t offset, 
+                               mach_vm_address_t * address, mach_vm_size_t length)
+{
+    IOReturn err;
+    IOMemoryDescriptorMapAllocRef ref;
+
+    ref.map          = *map;
+    ref.sharedMem    = entry;
+    ref.sourceOffset = trunc_page_64(offset);
+    ref.options             = options;
+    ref.size         = length;
+
+    if (options & kIOMapAnywhere)
+       // vm_map looks for addresses above here, even when VM_FLAGS_ANYWHERE
+       ref.mapped = 0;
+    else
+       ref.mapped = *address;
+
+    if( ref.sharedMem && (ref.map == kernel_map) && pageable)
+       err = IOIteratePageableMaps( ref.size, &IOMemoryDescriptorMapAlloc, &ref );
+    else
+       err = IOMemoryDescriptorMapAlloc( ref.map, &ref );
+
+    *address = ref.mapped;
+    *map     = ref.map;
+
+    return (err);
+}
+
+kern_return_t 
+IOMemoryDescriptorMapCopy(vm_map_t * map, 
+                               IOOptionBits options,
+                               mach_vm_size_t offset, 
+                               mach_vm_address_t * address, mach_vm_size_t length)
+{
+    IOReturn err;
+    IOMemoryDescriptorMapAllocRef ref;
+
+    ref.map          = *map;
+    ref.sharedMem    = NULL;
+    ref.sourceOffset = trunc_page_64(offset);
+    ref.options             = options;
+    ref.size         = length;
+
+    if (options & kIOMapAnywhere)
+       // vm_map looks for addresses above here, even when VM_FLAGS_ANYWHERE
+       ref.mapped = 0;
+    else
+       ref.mapped = *address;
+
+    if (ref.map == kernel_map)
+       err = IOIteratePageableMaps(ref.size, &IOMemoryDescriptorMapAlloc, &ref);
+    else
+       err = IOMemoryDescriptorMapAlloc(ref.map, &ref);
+
+    *address = ref.mapped;
+    *map     = ref.map;
+
+    return (err);
+}

 
 IOReturn IOMemoryDescriptor::doMap(
 
 IOReturn IOMemoryDescriptor::doMap(

-       vm_map_t                addressMap,
-       IOVirtualAddress *      atAddress,
+       vm_map_t                __addressMap,
+       IOVirtualAddress *      __address,

        IOOptionBits            options,
        IOOptionBits            options,

-       IOByteCount             sourceOffset,
-       IOByteCount             length )
+       IOByteCount             __offset,
+       IOByteCount             __length )

 {
 {

-    IOReturn           err = kIOReturnSuccess;
-    memory_object_t    pager;
-    vm_address_t       logical;
-    IOByteCount                pageOffset;
-    IOPhysicalAddress  sourceAddr;
-    IOMemoryDescriptorMapAllocRef      ref;
+#ifndef __LP64__
+    if (!(kIOMap64Bit & options)) panic("IOMemoryDescriptor::doMap !64bit");
+#endif /* !__LP64__ */

 
 

-    ref.sharedMem      = (ipc_port_t) _memEntry;
-    ref.sourceOffset   = sourceOffset;
-    ref.options                = options;
+    IOMemoryMap *  mapping = (IOMemoryMap *) *__address;
+    mach_vm_size_t offset  = mapping->fOffset + __offset;
+    mach_vm_size_t length  = mapping->fLength;

 
 

-    do {
+    IOReturn         err = kIOReturnSuccess;
+    memory_object_t   pager;
+    mach_vm_size_t    pageOffset;
+    IOPhysicalAddress sourceAddr;
+    unsigned int lock_count;

 
 

-        if( 0 == length)
-            length = getLength();
+    do
+    {
+       sourceAddr = getPhysicalSegment( offset, NULL, _kIOMemorySourceSegment );
+       pageOffset = sourceAddr - trunc_page( sourceAddr );

 
 

-        sourceAddr = getSourceSegment( sourceOffset, NULL );
-        assert( sourceAddr );
-        pageOffset = sourceAddr - trunc_page_32( sourceAddr );
+       if( reserved)
+           pager = (memory_object_t) reserved->devicePager;
+       else
+           pager = MACH_PORT_NULL;

 
 

-        ref.size = round_page_32( length + pageOffset );
+       if ((kIOMapReference|kIOMapUnique) == ((kIOMapReference|kIOMapUnique) & options))
+       {
+           upl_t          redirUPL2;
+           vm_size_t      size;
+           int            flags;

 
 

-        logical = *atAddress;
-        if( options & kIOMapAnywhere) 
-            // vm_map looks for addresses above here, even when VM_FLAGS_ANYWHERE
-            ref.mapped = 0;
-        else {
-            ref.mapped = trunc_page_32( logical );
-            if( (logical - ref.mapped) != pageOffset) {
-                err = kIOReturnVMError;
-                continue;
-            }
-        }
+           if (!_memEntry)
+           {
+               err = kIOReturnNotReadable;
+               continue;
+           }

 
 

-        if( ref.sharedMem && (addressMap == kernel_map) && (kIOMemoryBufferPageable & _flags))
-            err = IOIteratePageableMaps( ref.size, &IOMemoryDescriptorMapAlloc, &ref );
-        else
-            err = IOMemoryDescriptorMapAlloc( addressMap, &ref );
+           size = round_page(mapping->fLength + pageOffset);
+           flags = UPL_COPYOUT_FROM | UPL_SET_INTERNAL 
+                       | UPL_SET_LITE | UPL_SET_IO_WIRE | UPL_BLOCK_ACCESS;

 
 

-        if( err != KERN_SUCCESS)
-            continue;
+           if (KERN_SUCCESS != memory_object_iopl_request((ipc_port_t) _memEntry, 0, &size, &redirUPL2,
+                                           NULL, NULL,
+                                           &flags))
+               redirUPL2 = NULL;

 
 

-        if( reserved)
-            pager = (memory_object_t) reserved->devicePager;
-        else
-            pager = MACH_PORT_NULL;
+           for (lock_count = 0;
+                IORecursiveLockHaveLock(gIOMemoryLock);
+                lock_count++) {
+             UNLOCK;
+           }
+           err = upl_transpose(redirUPL2, mapping->fRedirUPL);
+           for (;
+                lock_count;
+                lock_count--) {
+             LOCK;
+           }

 
 

-        if( !ref.sharedMem || pager )
-            err = handleFault( pager, addressMap, ref.mapped, sourceOffset, length, options );
+           if (kIOReturnSuccess != err)
+           {
+               IOLog("upl_transpose(%x)\n", err);
+               err = kIOReturnSuccess;
+           }

 
 

-    } while( false );
+           if (redirUPL2)
+           {
+               upl_commit(redirUPL2, NULL, 0);
+               upl_deallocate(redirUPL2);
+               redirUPL2 = 0;
+           }
+           {
+               // swap the memEntries since they now refer to different vm_objects
+               void * me = _memEntry;
+               _memEntry = mapping->fMemory->_memEntry;
+               mapping->fMemory->_memEntry = me;
+           }
+           if (pager)
+               err = handleFault( reserved->devicePager, mapping->fAddressMap, mapping->fAddress, offset, length, options );
+       }
+       else
+       {
+           mach_vm_address_t address;

 
 

-    if( err != KERN_SUCCESS) {
-        if( ref.mapped)
-            doUnmap( addressMap, ref.mapped, ref.size );
-        *atAddress = NULL;
-    } else
-        *atAddress = ref.mapped + pageOffset;
+           if (!(options & kIOMapAnywhere))
+           {
+               address = trunc_page_64(mapping->fAddress);
+               if( (mapping->fAddress - address) != pageOffset)
+               {
+                   err = kIOReturnVMError;
+                   continue;
+               }
+           }

 
 

-    return( err );
-}
+            vm_map_t map = mapping->fAddressMap;
+           err = IOMemoryDescriptorMapMemEntry(&map, (ipc_port_t) _memEntry,
+                                                   options, (kIOMemoryBufferPageable & _flags),
+                                                   offset, &address, round_page_64(length + pageOffset));
+           if( err != KERN_SUCCESS)
+               continue;

 
 

-enum {
-    kIOMemoryRedirected        = 0x00010000
-};
+           if (!_memEntry || pager)
+           {
+               err = handleFault( pager, mapping->fAddressMap, address, offset, length, options );
+               if (err != KERN_SUCCESS)
+                   doUnmap( mapping->fAddressMap, (IOVirtualAddress) mapping, 0 );
+           }
+
+#if DEBUG
+       if (kIOLogMapping & gIOKitDebug)
+           IOLog("mapping(%x) desc %p @ %lx, map %p, address %qx, offset %qx, length %qx\n", 
+                   err, this, sourceAddr, mapping, address, offset, length);
+#endif
+
+           if (err == KERN_SUCCESS)
+               mapping->fAddress = address + pageOffset;
+           else
+               mapping->fAddress = NULL;
+       }
+    }
+    while( false );
+
+    return (err);
+}

 
 IOReturn IOMemoryDescriptor::handleFault(
         void *                 _pager,
        vm_map_t                addressMap,
 
 IOReturn IOMemoryDescriptor::handleFault(
         void *                 _pager,
        vm_map_t                addressMap,

-       IOVirtualAddress        address,
-       IOByteCount             sourceOffset,
-       IOByteCount             length,
+       mach_vm_address_t       address,
+       mach_vm_size_t          sourceOffset,
+       mach_vm_size_t          length,

         IOOptionBits           options )
 {
     IOReturn           err = kIOReturnSuccess;
     memory_object_t    pager = (memory_object_t) _pager;
         IOOptionBits           options )
 {
     IOReturn           err = kIOReturnSuccess;
     memory_object_t    pager = (memory_object_t) _pager;

-    vm_size_t          size;
-    vm_size_t          bytes;
-    vm_size_t          page;
-    IOByteCount                pageOffset;
+    mach_vm_size_t     size;
+    mach_vm_size_t     bytes;
+    mach_vm_size_t     page;
+    mach_vm_size_t     pageOffset;
+    mach_vm_size_t     pagerOffset;

     IOPhysicalLength   segLen;
     addr64_t           physAddr;
 
     IOPhysicalLength   segLen;
     addr64_t           physAddr;
 

-    if( !addressMap) {
-
-        if( kIOMemoryRedirected & _flags) {
-#ifdef DEBUG
-            IOLog("sleep mem redirect %p, %lx\n", this, sourceOffset);
+    if( !addressMap)
+    {
+        if( kIOMemoryRedirected & _flags)
+       {
+#if DEBUG
+            IOLog("sleep mem redirect %p, %qx\n", this, sourceOffset);

 #endif
             do {
                SLEEP;
 #endif
             do {
                SLEEP;


@@ -1767,346 +2919,415 @@ IOReturn IOMemoryDescriptor::handleFault(
         return( kIOReturnSuccess );
     }
 
         return( kIOReturnSuccess );
     }
 

-    physAddr = getPhysicalSegment64( sourceOffset, &segLen );
+    physAddr = getPhysicalSegment( sourceOffset, &segLen, kIOMemoryMapperNone );

     assert( physAddr );
     pageOffset = physAddr - trunc_page_64( physAddr );
     assert( physAddr );
     pageOffset = physAddr - trunc_page_64( physAddr );

+    pagerOffset = sourceOffset;

 
     size = length + pageOffset;
     physAddr -= pageOffset;
 
     segLen += pageOffset;
     bytes = size;
 
     size = length + pageOffset;
     physAddr -= pageOffset;
 
     segLen += pageOffset;
     bytes = size;

-    do {
+    do
+    {

        // in the middle of the loop only map whole pages
        if( segLen >= bytes)
            segLen = bytes;
        // in the middle of the loop only map whole pages
        if( segLen >= bytes)
            segLen = bytes;

-       else if( segLen != trunc_page_32( segLen))
+       else if( segLen != trunc_page( segLen))

            err = kIOReturnVMError;
         if( physAddr != trunc_page_64( physAddr))
            err = kIOReturnBadArgument;
            err = kIOReturnVMError;
         if( physAddr != trunc_page_64( physAddr))
            err = kIOReturnBadArgument;

+       if (kIOReturnSuccess != err)
+           break;

 
 

-#ifdef DEBUG
+#if DEBUG

        if( kIOLogMapping & gIOKitDebug)
        if( kIOLogMapping & gIOKitDebug)

-           IOLog("_IOMemoryMap::map(%p) %08lx->%08qx:%08lx\n",
+           IOLog("IOMemoryMap::map(%p) 0x%qx->0x%qx:0x%qx\n",

                 addressMap, address + pageOffset, physAddr + pageOffset,
                segLen - pageOffset);
 #endif
 
 
                 addressMap, address + pageOffset, physAddr + pageOffset,
                segLen - pageOffset);
 #endif
 
 

-
-
-
-#ifdef i386  
-       /* i386 doesn't support faulting on device memory yet */
-       if( addressMap && (kIOReturnSuccess == err))
-            err = IOMapPages( addressMap, address, (IOPhysicalAddress) physAddr, segLen, options );
-        assert( KERN_SUCCESS == err );
-       if( err)
-           break;
-#endif
-

         if( pager) {
             if( reserved && reserved->pagerContig) {
                 IOPhysicalLength       allLen;
                 addr64_t               allPhys;
 
         if( pager) {
             if( reserved && reserved->pagerContig) {
                 IOPhysicalLength       allLen;
                 addr64_t               allPhys;
 

-                allPhys = getPhysicalSegment64( 0, &allLen );
+                allPhys = getPhysicalSegment( 0, &allLen, kIOMemoryMapperNone );

                 assert( allPhys );
                 assert( allPhys );

-                err = device_pager_populate_object( pager, 0, allPhys >> PAGE_SHIFT, round_page_32(allLen) );
-
-            } else {
+               err = device_pager_populate_object( pager, 0, atop_64(allPhys), round_page(allLen) );
+            }
+           else
+           {

 
 

-                for( page = 0;
+               for( page = 0;

                      (page < segLen) && (KERN_SUCCESS == err);
                      (page < segLen) && (KERN_SUCCESS == err);

-                     page += page_size) {
-                        err = device_pager_populate_object(pager, sourceOffset + page,
-                               (ppnum_t)((physAddr + page) >> PAGE_SHIFT), page_size);
+                     page += page_size)
+               {
+                   err = device_pager_populate_object(pager, pagerOffset,
+                           (ppnum_t)(atop_64(physAddr + page)), page_size);
+                   pagerOffset += page_size;

                 }
             }
             assert( KERN_SUCCESS == err );
             if( err)
                 break;
         }
                 }
             }
             assert( KERN_SUCCESS == err );
             if( err)
                 break;
         }

-#ifndef i386
+
+       // This call to vm_fault causes an early pmap level resolution
+       // of the mappings created above for kernel mappings, since
+       // faulting in later can't take place from interrupt level.

        /*  *** ALERT *** */
        /*  *** Temporary Workaround *** */
 
        /*  *** ALERT *** */
        /*  *** Temporary Workaround *** */
 

-       /* This call to vm_fault causes an early pmap level resolution  */
-       /* of the mappings created above.  Need for this is in absolute */
-       /* violation of the basic tenet that the pmap layer is a cache. */
-       /* Further, it implies a serious I/O architectural violation on */
-       /* the part of some user of the mapping.  As of this writing,   */
-       /* the call to vm_fault is needed because the NVIDIA driver     */
-       /* makes a call to pmap_extract.  The NVIDIA driver needs to be */
-       /* fixed as soon as possible.  The NVIDIA driver should not     */
-       /* need to query for this info as it should know from the doMap */
-       /* call where the physical memory is mapped.  When a query is   */
-       /* necessary to find a physical mapping, it should be done      */
-       /* through an iokit call which includes the mapped memory       */
-       /* handle.  This is required for machine architecture independence.*/
-
-       if(!(kIOMemoryRedirected & _flags)) {
-               vm_fault(addressMap, address, 3, FALSE, FALSE, NULL, 0);
+       if ((addressMap == kernel_map) && !(kIOMemoryRedirected & _flags))
+       {
+               vm_fault(addressMap, 
+                        (vm_map_offset_t)address, 
+                        VM_PROT_READ|VM_PROT_WRITE, 
+                        FALSE, THREAD_UNINT, NULL, 
+                        (vm_map_offset_t)0);

        }
 
        /*  *** Temporary Workaround *** */
        /*  *** ALERT *** */
        }
 
        /*  *** Temporary Workaround *** */
        /*  *** ALERT *** */

-#endif
+

        sourceOffset += segLen - pageOffset;
        address += segLen;
        bytes -= segLen;
        pageOffset = 0;
 
        sourceOffset += segLen - pageOffset;
        address += segLen;
        bytes -= segLen;
        pageOffset = 0;
 

-    } while( bytes
-       && (physAddr = getPhysicalSegment64( sourceOffset, &segLen )));
+    } 
+    while (bytes && (physAddr = getPhysicalSegment( sourceOffset, &segLen, kIOMemoryMapperNone )));

 
 

-    if( bytes)
+    if (bytes)

         err = kIOReturnBadArgument;
 
         err = kIOReturnBadArgument;
 

-    return( err );
+    return (err);

 }
 
 IOReturn IOMemoryDescriptor::doUnmap(
        vm_map_t                addressMap,
 }
 
 IOReturn IOMemoryDescriptor::doUnmap(
        vm_map_t                addressMap,

-       IOVirtualAddress        logical,
-       IOByteCount             length )
+       IOVirtualAddress        __address,
+       IOByteCount             __length )

 {
 {

-    IOReturn   err;
+    IOReturn         err;
+    mach_vm_address_t address;
+    mach_vm_size_t    length;

 
 

-#ifdef DEBUG
-    if( kIOLogMapping & gIOKitDebug)
-       kprintf("IOMemoryDescriptor::doUnmap(%x) %08x:%08x\n",
-                addressMap, logical, length );
-#endif
-
-    if( true /* && (addressMap == kernel_map) || (addressMap == get_task_map(current_task()))*/) {
+    if (__length)
+    {
+       address = __address;
+       length  = __length;
+    }
+    else
+    {
+       addressMap = ((IOMemoryMap *) __address)->fAddressMap;
+       address    = ((IOMemoryMap *) __address)->fAddress;
+       length     = ((IOMemoryMap *) __address)->fLength;
+    }

 
 

-        if( _memEntry && (addressMap == kernel_map) && (kIOMemoryBufferPageable & _flags))
-            addressMap = IOPageableMapForAddress( logical );
+    if ((addressMap == kernel_map) 
+        && ((kIOMemoryBufferPageable & _flags) || !_memEntry))
+       addressMap = IOPageableMapForAddress( address );

 
 

-        err = vm_deallocate( addressMap, logical, length );
+#if DEBUG
+    if( kIOLogMapping & gIOKitDebug)
+       IOLog("IOMemoryDescriptor::doUnmap map %p, 0x%qx:0x%qx\n",
+               addressMap, address, length );
+#endif

 
 

-    } else
-        err = kIOReturnSuccess;
+    err = mach_vm_deallocate( addressMap, address, length );

 
 

-    return( err );
+    return (err);

 }
 
 }
 

-IOReturn IOMemoryDescriptor::redirect( task_t safeTask, bool redirect )
+IOReturn IOMemoryDescriptor::redirect( task_t safeTask, bool doRedirect )

 {
 {

-    IOReturn           err;
-    _IOMemoryMap *     mapping = 0;
+    IOReturn           err = kIOReturnSuccess;
+    IOMemoryMap *      mapping = 0;

     OSIterator *       iter;
 
     LOCK;
 
     OSIterator *       iter;
 
     LOCK;
 

+    if( doRedirect)
+        _flags |= kIOMemoryRedirected;
+    else
+        _flags &= ~kIOMemoryRedirected;
+

     do {
        if( (iter = OSCollectionIterator::withCollection( _mappings))) {
     do {
        if( (iter = OSCollectionIterator::withCollection( _mappings))) {

-            while( (mapping = (_IOMemoryMap *) iter->getNextObject()))
-                mapping->redirect( safeTask, redirect );
+           while( (mapping = (IOMemoryMap *) iter->getNextObject()))
+               mapping->redirect( safeTask, doRedirect );

 
 

-            iter->release();
-        }
+           iter->release();
+       }

     } while( false );
 
     } while( false );
 

-    if( redirect)
-        _flags |= kIOMemoryRedirected;
-    else {
-        _flags &= ~kIOMemoryRedirected;
+    if (!doRedirect)
+    {

         WAKEUP;
     }
 
     UNLOCK;
 
         WAKEUP;
     }
 
     UNLOCK;
 

+#ifndef __LP64__

     // temporary binary compatibility
     IOSubMemoryDescriptor * subMem;
     if( (subMem = OSDynamicCast( IOSubMemoryDescriptor, this)))
     // temporary binary compatibility
     IOSubMemoryDescriptor * subMem;
     if( (subMem = OSDynamicCast( IOSubMemoryDescriptor, this)))

-        err = subMem->redirect( safeTask, redirect );
+       err = subMem->redirect( safeTask, doRedirect );

     else
     else

-        err = kIOReturnSuccess;
+       err = kIOReturnSuccess;
+#endif /* !__LP64__ */

 
     return( err );
 }
 
 
     return( err );
 }
 

-IOReturn IOSubMemoryDescriptor::redirect( task_t safeTask, bool redirect )
-{
-    return( _parent->redirect( safeTask, redirect ));
-}
-
-IOReturn _IOMemoryMap::redirect( task_t safeTask, bool redirect )
+IOReturn IOMemoryMap::redirect( task_t safeTask, bool doRedirect )

 {
     IOReturn err = kIOReturnSuccess;
 
 {
     IOReturn err = kIOReturnSuccess;
 

-    if( superMap) {
-//        err = ((_IOMemoryMap *)superMap)->redirect( safeTask, redirect );
+    if( fSuperMap) {
+//        err = ((IOMemoryMap *)superMap)->redirect( safeTask, doRedirect );

     } else {
 
         LOCK;
     } else {
 
         LOCK;

-        if( logical && addressMap
-        && (get_task_map( safeTask) != addressMap)
-        && (0 == (options & kIOMapStatic))) {
-
-            IOUnmapPages( addressMap, logical, length );
-            if( !redirect) {
-                err = vm_deallocate( addressMap, logical, length );
-                err = memory->doMap( addressMap, &logical,
-                                     (options & ~kIOMapAnywhere) /*| kIOMapReserve*/,
-                                     offset, length );
-            } else
-                err = kIOReturnSuccess;
-#ifdef DEBUG
-            IOLog("IOMemoryMap::redirect(%d, %p) %x:%lx from %p\n", redirect, this, logical, length, addressMap);
+
+       do
+       {
+           if (!fAddress)
+               break;
+           if (!fAddressMap)
+               break;
+
+           if ((!safeTask || (get_task_map(safeTask) != fAddressMap))
+             && (0 == (fOptions & kIOMapStatic)))
+           {
+               IOUnmapPages( fAddressMap, fAddress, fLength );
+               err = kIOReturnSuccess;
+#if DEBUG
+               IOLog("IOMemoryMap::redirect(%d, %p) 0x%qx:0x%qx from %p\n", doRedirect, this, fAddress, fLength, fAddressMap);

 #endif
 #endif

-        }
-        UNLOCK;
+           }
+           else if (kIOMapWriteCombineCache == (fOptions & kIOMapCacheMask))
+           {
+               IOOptionBits newMode;
+               newMode = (fOptions & ~kIOMapCacheMask) | (doRedirect ? kIOMapInhibitCache : kIOMapWriteCombineCache);
+               IOProtectCacheMode(fAddressMap, fAddress, fLength, newMode);
+           }
+       }
+       while (false);
+       UNLOCK;

     }
 
     }
 

+    if ((((fMemory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical)
+        || ((fMemory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical64))
+     && safeTask
+     && (doRedirect != (0 != (fMemory->_flags & kIOMemoryRedirected))))
+       fMemory->redirect(safeTask, doRedirect);
+

     return( err );
 }
 
     return( err );
 }
 

-IOReturn _IOMemoryMap::unmap( void )
+IOReturn IOMemoryMap::unmap( void )

 {
     IOReturn   err;
 
     LOCK;
 
 {
     IOReturn   err;
 
     LOCK;
 

-    if( logical && addressMap && (0 == superMap)
-       && (0 == (options & kIOMapStatic))) {
+    if( fAddress && fAddressMap && (0 == fSuperMap) && fMemory
+       && (0 == (fOptions & kIOMapStatic))) {

 
 

-        err = memory->doUnmap( addressMap, logical, length );
-        vm_map_deallocate(addressMap);
-        addressMap = 0;
+        err = fMemory->doUnmap(fAddressMap, (IOVirtualAddress) this, 0);

 
     } else
        err = kIOReturnSuccess;
 
 
     } else
        err = kIOReturnSuccess;
 

-    logical = 0;
+    if (fAddressMap)
+    {
+        vm_map_deallocate(fAddressMap);
+        fAddressMap = 0;
+    }
+
+    fAddress = 0;

 
     UNLOCK;
 
     return( err );
 }
 
 
     UNLOCK;
 
     return( err );
 }
 

-void _IOMemoryMap::taskDied( void )
+void IOMemoryMap::taskDied( void )

 {
     LOCK;
 {
     LOCK;

-    if( addressMap) {
-        vm_map_deallocate(addressMap);
-        addressMap = 0;
+    if (fUserClientUnmap)
+       unmap();
+    if( fAddressMap) {
+        vm_map_deallocate(fAddressMap);
+        fAddressMap = 0;

     }
     }

-    addressTask        = 0;
-    logical    = 0;
+    fAddressTask = 0;
+    fAddress    = 0;

     UNLOCK;
 }
 
     UNLOCK;
 }
 

+IOReturn IOMemoryMap::userClientUnmap( void )
+{
+    fUserClientUnmap = true;
+    return (kIOReturnSuccess);
+}
+

 // Overload the release mechanism.  All mappings must be a member
 // of a memory descriptors _mappings set.  This means that we
 // always have 2 references on a mapping.  When either of these mappings
 // are released we need to free ourselves.
 // Overload the release mechanism.  All mappings must be a member
 // of a memory descriptors _mappings set.  This means that we
 // always have 2 references on a mapping.  When either of these mappings
 // are released we need to free ourselves.

-void _IOMemoryMap::taggedRelease(const void *tag) const
+void IOMemoryMap::taggedRelease(const void *tag) const

 {
     LOCK;
     super::taggedRelease(tag, 2);
     UNLOCK;
 }
 
 {
     LOCK;
     super::taggedRelease(tag, 2);
     UNLOCK;
 }
 

-void _IOMemoryMap::free()
+void IOMemoryMap::free()

 {
     unmap();
 
 {
     unmap();
 

-    if( memory) {
+    if (fMemory)
+    {
+        LOCK;
+       fMemory->removeMapping(this);
+       UNLOCK;
+       fMemory->release();
+    }
+
+    if (fOwner && (fOwner != fMemory))
+    {

         LOCK;
         LOCK;

-       memory->removeMapping( this);
+       fOwner->removeMapping(this);

        UNLOCK;
        UNLOCK;

-       memory->release();

     }
 
     }
 

-    if( superMap)
-       superMap->release();
+    if (fSuperMap)
+       fSuperMap->release();
+
+    if (fRedirUPL) {
+       upl_commit(fRedirUPL, NULL, 0);
+       upl_deallocate(fRedirUPL);
+    }

 
     super::free();
 }
 
 
     super::free();
 }
 

-IOByteCount _IOMemoryMap::getLength()
+IOByteCount IOMemoryMap::getLength()
+{
+    return( fLength );
+}
+
+IOVirtualAddress IOMemoryMap::getVirtualAddress()
+{
+#ifndef __LP64__
+    if (fSuperMap)
+       fSuperMap->getVirtualAddress();
+    else if (fAddressMap 
+               && vm_map_is_64bit(fAddressMap)
+               && (sizeof(IOVirtualAddress) < 8))
+    {
+       OSReportWithBacktrace("IOMemoryMap::getVirtualAddress(0x%qx) called on 64b map; use ::getAddress()", fAddress);
+    }
+#endif /* !__LP64__ */
+
+    return (fAddress);
+}
+
+#ifndef __LP64__
+mach_vm_address_t      IOMemoryMap::getAddress()

 {
 {

-    return( length );
+    return( fAddress);

 }
 
 }
 

-IOVirtualAddress _IOMemoryMap::getVirtualAddress()
+mach_vm_size_t         IOMemoryMap::getSize()

 {
 {

-    return( logical);
+    return( fLength );

 }
 }

+#endif /* !__LP64__ */
+

 
 

-task_t _IOMemoryMap::getAddressTask()
+task_t IOMemoryMap::getAddressTask()

 {
 {

-    if( superMap)
-       return( superMap->getAddressTask());
+    if( fSuperMap)
+       return( fSuperMap->getAddressTask());

     else
     else

-        return( addressTask);
+        return( fAddressTask);

 }
 
 }
 

-IOOptionBits _IOMemoryMap::getMapOptions()
+IOOptionBits IOMemoryMap::getMapOptions()

 {
 {

-    return( options);
+    return( fOptions);

 }
 
 }
 

-IOMemoryDescriptor * _IOMemoryMap::getMemoryDescriptor()
+IOMemoryDescriptor * IOMemoryMap::getMemoryDescriptor()

 {
 {

-    return( memory );
+    return( fMemory );

 }
 
 }
 

-_IOMemoryMap * _IOMemoryMap::copyCompatible(
-               IOMemoryDescriptor *    owner,
-                task_t                 task,
-                IOVirtualAddress       toAddress,
-                IOOptionBits           _options,
-                IOByteCount            _offset,
-                IOByteCount            _length )
+IOMemoryMap * IOMemoryMap::copyCompatible(
+               IOMemoryMap * newMapping )

 {
 {

-    _IOMemoryMap * mapping;
+    task_t             task      = newMapping->getAddressTask();
+    mach_vm_address_t  toAddress = newMapping->fAddress;
+    IOOptionBits       _options  = newMapping->fOptions;
+    mach_vm_size_t     _offset   = newMapping->fOffset;
+    mach_vm_size_t     _length   = newMapping->fLength;

 
 

-    if( (!task) || (task != getAddressTask()))
+    if( (!task) || (!fAddressMap) || (fAddressMap != get_task_map(task)))

        return( 0 );
        return( 0 );

-    if( (options ^ _options) & kIOMapReadOnly)
+    if( (fOptions ^ _options) & kIOMapReadOnly)

        return( 0 );
     if( (kIOMapDefaultCache != (_options & kIOMapCacheMask)) 
        return( 0 );
     if( (kIOMapDefaultCache != (_options & kIOMapCacheMask)) 

-     && ((options ^ _options) & kIOMapCacheMask))
+     && ((fOptions ^ _options) & kIOMapCacheMask))

        return( 0 );
 
        return( 0 );
 

-    if( (0 == (_options & kIOMapAnywhere)) && (logical != toAddress))
+    if( (0 == (_options & kIOMapAnywhere)) && (fAddress != toAddress))

        return( 0 );
 
        return( 0 );
 

-    if( _offset < offset)
+    if( _offset < fOffset)

        return( 0 );
 
        return( 0 );
 

-    _offset -= offset;
+    _offset -= fOffset;

 
 

-    if( (_offset + _length) > length)
+    if( (_offset + _length) > fLength)

        return( 0 );
 
        return( 0 );
 

-    if( (length == _length) && (!_offset)) {
-        retain();
-       mapping = this;
-
-    } else {
-        mapping = new _IOMemoryMap;
-        if( mapping
-        && !mapping->initCompatible( owner, this, _offset, _length )) {
-            mapping->release();
-            mapping = 0;
-        }
+    retain();
+    if( (fLength == _length) && (!_offset))
+    {
+       newMapping = this;
+    }
+    else
+    {
+       newMapping->fSuperMap = this;
+       newMapping->fOffset   = fOffset + _offset;
+       newMapping->fAddress  = fAddress + _offset;

     }
 
     }
 

-    return( mapping );
+    return( newMapping );

 }
 
 }
 

-IOPhysicalAddress _IOMemoryMap::getPhysicalSegment( IOByteCount _offset,
-                                                   IOPhysicalLength * length)
+IOPhysicalAddress 
+#ifdef __LP64__
+IOMemoryMap::getPhysicalSegment( IOByteCount _offset, IOPhysicalLength * _length, IOOptionBits _options)
+#else /* !__LP64__ */
+IOMemoryMap::getPhysicalSegment( IOByteCount _offset, IOPhysicalLength * _length)
+#endif /* !__LP64__ */

 {
     IOPhysicalAddress  address;
 
     LOCK;
 {
     IOPhysicalAddress  address;
 
     LOCK;

-    address = memory->getPhysicalSegment( offset + _offset, length );
+#ifdef __LP64__
+    address = fMemory->getPhysicalSegment( fOffset + _offset, _length, _options );
+#else /* !__LP64__ */
+    address = fMemory->getPhysicalSegment( fOffset + _offset, _length );
+#endif /* !__LP64__ */

     UNLOCK;
 
     return( address );
     UNLOCK;
 
     return( address );


@@ -2123,6 +3344,10 @@ void IOMemoryDescriptor::initialize( void )
 {
     if( 0 == gIOMemoryLock)
        gIOMemoryLock = IORecursiveLockAlloc();
 {
     if( 0 == gIOMemoryLock)
        gIOMemoryLock = IORecursiveLockAlloc();

+
+    IORegistryEntry::getRegistryRoot()->setProperty(kIOMaximumMappedIOByteCountKey,
+                                                   ptoa_64(gIOMaximumMappedIOPageCount), 64);
+    gIOLastPage = IOGetLastPageNumber();

 }
 
 void IOMemoryDescriptor::free( void )
 }
 
 void IOMemoryDescriptor::free( void )


@@ -2138,322 +3363,286 @@ IOMemoryMap * IOMemoryDescriptor::setMapping(
        IOVirtualAddress        mapAddress,
        IOOptionBits            options )
 {
        IOVirtualAddress        mapAddress,
        IOOptionBits            options )
 {

-    _IOMemoryMap *             map;
-
-    map = new _IOMemoryMap;
-
-    LOCK;
-
-    if( map
-     && !map->initWithDescriptor( this, intoTask, mapAddress,
-                    options | kIOMapStatic, 0, getLength() )) {
-       map->release();
-       map = 0;
-    }
-
-    addMapping( map);
-
-    UNLOCK;
-
-    return( map);
+    return (createMappingInTask( intoTask, mapAddress,
+                                   options | kIOMapStatic,
+                                   0, getLength() ));

 }
 
 IOMemoryMap * IOMemoryDescriptor::map( 
        IOOptionBits            options )
 {
 }
 
 IOMemoryMap * IOMemoryDescriptor::map( 
        IOOptionBits            options )
 {

-
-    return( makeMapping( this, kernel_task, 0,
-                       options | kIOMapAnywhere,
-                       0, getLength() ));
+    return (createMappingInTask( kernel_task, 0,
+                               options | kIOMapAnywhere,
+                               0, getLength() ));

 }
 
 }
 

-IOMemoryMap * IOMemoryDescriptor::map(
-       task_t                  intoTask,
-       IOVirtualAddress        toAddress,
+#ifndef __LP64__
+IOMemoryMap * IOMemoryDescriptor::map( 
+       task_t                  intoTask,
+       IOVirtualAddress        atAddress,

        IOOptionBits            options,
        IOByteCount             offset,
        IOByteCount             length )
 {
        IOOptionBits            options,
        IOByteCount             offset,
        IOByteCount             length )
 {

-    if( 0 == length)
-       length = getLength();
+    if ((!(kIOMapAnywhere & options)) && vm_map_is_64bit(get_task_map(intoTask)))
+    {
+       OSReportWithBacktrace("IOMemoryDescriptor::map() in 64b task, use ::createMappingInTask()");
+       return (0);
+    }

 
 

-    return( makeMapping( this, intoTask, toAddress, options, offset, length ));
+    return (createMappingInTask(intoTask, atAddress,
+                               options, offset, length));

 }
 }

+#endif /* !__LP64__ */

 
 

-IOMemoryMap * IOMemoryDescriptor::makeMapping(
-       IOMemoryDescriptor *    owner,
+IOMemoryMap * IOMemoryDescriptor::createMappingInTask(

        task_t                  intoTask,
        task_t                  intoTask,

-       IOVirtualAddress        toAddress,
+       mach_vm_address_t       atAddress,

        IOOptionBits            options,
        IOOptionBits            options,

-       IOByteCount             offset,
-       IOByteCount             length )
+       mach_vm_size_t          offset,
+       mach_vm_size_t          length)

 {
 {

-    _IOMemoryMap *     mapping = 0;
-    OSIterator *       iter;
-
-    LOCK;
-
-    do {
-       // look for an existing mapping
-       if( (iter = OSCollectionIterator::withCollection( _mappings))) {
-
-            while( (mapping = (_IOMemoryMap *) iter->getNextObject())) {
+    IOMemoryMap * result;
+    IOMemoryMap * mapping;

 
 

-               if( (mapping = mapping->copyCompatible( 
-                                       owner, intoTask, toAddress,
-                                       options | kIOMapReference,
-                                       offset, length )))
-                   break;
-            }
-            iter->release();
-            if( mapping)
-                continue;
-        }
+    if (0 == length)
+       length = getLength();

 
 

+    mapping = new IOMemoryMap;

 
 

-       if( mapping || (options & kIOMapReference))
-           continue;
+    if( mapping
+     && !mapping->init( intoTask, atAddress,
+                       options, offset, length )) {
+       mapping->release();
+       mapping = 0;
+    }

 
 

-       owner = this;
+    if (mapping)
+       result = makeMapping(this, intoTask, (IOVirtualAddress) mapping, options | kIOMap64Bit, 0, 0);
+    else
+       result = 0;

 
 

-        mapping = new _IOMemoryMap;
-       if( mapping
-       && !mapping->initWithDescriptor( owner, intoTask, toAddress, options,
-                          offset, length )) {
-#ifdef DEBUG
-           IOLog("Didn't make map %08lx : %08lx\n", offset, length );
+#if DEBUG
+    if (!result)
+       IOLog("createMappingInTask failed desc %p, addr %qx, options %lx, offset %qx, length %qx\n",
+                   this, atAddress, options, offset, length);

 #endif
 #endif

-           mapping->release();
-            mapping = 0;
-       }
-
-    } while( false );
-
-    owner->addMapping( mapping);
-
-    UNLOCK;

 
 

-    return( mapping);
+    return (result);

 }
 
 }
 

-void IOMemoryDescriptor::addMapping(
-       IOMemoryMap * mapping )
-{
-    if( mapping) {
-        if( 0 == _mappings)
-            _mappings = OSSet::withCapacity(1);
-       if( _mappings )
-           _mappings->setObject( mapping );
-    }
-}
-
-void IOMemoryDescriptor::removeMapping(
-       IOMemoryMap * mapping )
+#ifndef __LP64__ // there is only a 64 bit version for LP64
+IOReturn IOMemoryMap::redirect(IOMemoryDescriptor * newBackingMemory,
+                               IOOptionBits         options,
+                               IOByteCount          offset)

 {
 {

-    if( _mappings)
-        _mappings->removeObject( mapping);
+    return (redirect(newBackingMemory, options, (mach_vm_size_t)offset));

 }
 }

+#endif

 
 

-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-#undef super
-#define super IOMemoryDescriptor
-
-OSDefineMetaClassAndStructors(IOSubMemoryDescriptor, IOMemoryDescriptor)
-
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-bool IOSubMemoryDescriptor::initSubRange( IOMemoryDescriptor * parent,
-                                       IOByteCount offset, IOByteCount length,
-                                       IODirection direction )
+IOReturn IOMemoryMap::redirect(IOMemoryDescriptor * newBackingMemory,
+                               IOOptionBits         options,
+                               mach_vm_size_t       offset)

 {
 {

-    if( !parent)
-       return( false);
+    IOReturn err = kIOReturnSuccess;
+    IOMemoryDescriptor * physMem = 0;

 
 

-    if( (offset + length) > parent->getLength())
-       return( false);
+    LOCK;

 
 

-    /*
-     * We can check the _parent instance variable before having ever set it
-     * to an initial value because I/O Kit guarantees that all our instance
-     * variables are zeroed on an object's allocation.
-     */
+    if (fAddress && fAddressMap) do 
+    {
+       if (((fMemory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical)
+           || ((fMemory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical64))
+       {
+           physMem = fMemory;
+           physMem->retain();
+       }

 
 

-    if( !_parent) {
-       if( !super::init())
-           return( false );
-    } else {
-       /*
-        * An existing memory descriptor is being retargeted to
-        * point to somewhere else.  Clean up our present state.
-        */
+       if (!fRedirUPL)
+       {
+           vm_size_t size = round_page(fLength);
+           int flags = UPL_COPYOUT_FROM | UPL_SET_INTERNAL 
+                       | UPL_SET_LITE | UPL_SET_IO_WIRE | UPL_BLOCK_ACCESS;
+           if (KERN_SUCCESS != memory_object_iopl_request((ipc_port_t) fMemory->_memEntry, 0, &size, &fRedirUPL,
+                                           NULL, NULL,
+                                           &flags))
+               fRedirUPL = 0;
+
+           if (physMem)
+           {
+               IOUnmapPages( fAddressMap, fAddress, fLength );
+               if (false)
+                   physMem->redirect(0, true);
+           }
+       }

 
 

-       _parent->release();
-       _parent = 0;
+       if (newBackingMemory)
+       {
+           if (newBackingMemory != fMemory)
+           {
+               fOffset = 0;
+               if (this != newBackingMemory->makeMapping(newBackingMemory, fAddressTask, (IOVirtualAddress) this, 
+                                                           options | kIOMapUnique | kIOMapReference | kIOMap64Bit,
+                                                           offset, fLength))
+                   err = kIOReturnError;
+           }
+           if (fRedirUPL)
+           {
+               upl_commit(fRedirUPL, NULL, 0);
+               upl_deallocate(fRedirUPL);
+               fRedirUPL = 0;
+           }
+           if (false && physMem)
+               physMem->redirect(0, false);
+       }

     }
     }

+    while (false);

 
 

-    parent->retain();
-    _parent    = parent;
-    _start     = offset;
-    _length    = length;
-    _direction  = direction;
-    _tag       = parent->getTag();
-
-    return( true );
-}
-
-void IOSubMemoryDescriptor::free( void )
-{
-    if( _parent)
-       _parent->release();
-
-    super::free();
-}
-
-
-IOPhysicalAddress IOSubMemoryDescriptor::getPhysicalSegment( IOByteCount offset,
-                                                       IOByteCount * length )
-{
-    IOPhysicalAddress  address;
-    IOByteCount                actualLength;
-
-    assert(offset <= _length);
-
-    if( length)
-        *length = 0;
-
-    if( offset >= _length)
-        return( 0 );
-
-    address = _parent->getPhysicalSegment( offset + _start, &actualLength );
-
-    if( address && length)
-       *length = min( _length - offset, actualLength );
-
-    return( address );
-}
-
-IOPhysicalAddress IOSubMemoryDescriptor::getSourceSegment( IOByteCount offset,
-                                                          IOByteCount * length )
-{
-    IOPhysicalAddress  address;
-    IOByteCount                actualLength;
-
-    assert(offset <= _length);
-
-    if( length)
-        *length = 0;
-
-    if( offset >= _length)
-        return( 0 );
-
-    address = _parent->getSourceSegment( offset + _start, &actualLength );
+    UNLOCK;

 
 

-    if( address && length)
-       *length = min( _length - offset, actualLength );
+    if (physMem)
+       physMem->release();

 
 

-    return( address );
+    return (err);

 }
 
 }
 

-void * IOSubMemoryDescriptor::getVirtualSegment(IOByteCount offset,
-                                       IOByteCount * lengthOfSegment)
+IOMemoryMap * IOMemoryDescriptor::makeMapping(
+       IOMemoryDescriptor *    owner,
+       task_t                  __intoTask,
+       IOVirtualAddress        __address,
+       IOOptionBits            options,
+       IOByteCount             __offset,
+       IOByteCount             __length )

 {
 {

-    return( 0 );
-}
+#ifndef __LP64__
+    if (!(kIOMap64Bit & options)) panic("IOMemoryDescriptor::makeMapping !64bit");
+#endif /* !__LP64__ */

 
 

-IOByteCount IOSubMemoryDescriptor::readBytes(IOByteCount offset,
-                                       void * bytes, IOByteCount length)
-{
-    IOByteCount        byteCount;
+    IOMemoryDescriptor * mapDesc = 0;
+    IOMemoryMap *       result = 0;
+    OSIterator *        iter;

 
 

-    assert(offset <= _length);
+    IOMemoryMap *  mapping = (IOMemoryMap *) __address;
+    mach_vm_size_t offset  = mapping->fOffset + __offset;
+    mach_vm_size_t length  = mapping->fLength;

 
 

-    if( offset >= _length)
-        return( 0 );
+    mapping->fOffset = offset;

 
     LOCK;
 
     LOCK;

-    byteCount = _parent->readBytes( _start + offset, bytes,
-                               min(length, _length - offset) );
-    UNLOCK;
-
-    return( byteCount );
-}
-
-IOByteCount IOSubMemoryDescriptor::writeBytes(IOByteCount offset,
-                               const void* bytes, IOByteCount length)
-{
-    IOByteCount        byteCount;

 
 

-    assert(offset <= _length);
+    do
+    {
+       if (kIOMapStatic & options)
+       {
+           result = mapping;
+           addMapping(mapping);
+           mapping->setMemoryDescriptor(this, 0);
+           continue;
+       }

 
 

-    if( offset >= _length)
-        return( 0 );
+       if (kIOMapUnique & options)
+       {
+           addr64_t phys;
+           IOByteCount       physLen;

 
 

-    LOCK;
-    byteCount = _parent->writeBytes( _start + offset, bytes,
-                               min(length, _length - offset) );
-    UNLOCK;
+//         if (owner != this)          continue;

 
 

-    return( byteCount );
-}
+           if (((_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical)
+               || ((_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical64))
+           {
+               phys = getPhysicalSegment(offset, &physLen, kIOMemoryMapperNone);
+               if (!phys || (physLen < length))
+                   continue;
+    
+               mapDesc = IOMemoryDescriptor::withAddressRange(
+                               phys, length, getDirection() | kIOMemoryMapperNone, NULL);
+               if (!mapDesc)
+                   continue;
+               offset = 0;
+               mapping->fOffset = offset;
+           }
+       }
+       else
+       {
+           // look for a compatible existing mapping
+           if( (iter = OSCollectionIterator::withCollection(_mappings)))
+           {
+               IOMemoryMap * lookMapping;
+               while ((lookMapping = (IOMemoryMap *) iter->getNextObject()))
+               {
+                   if ((result = lookMapping->copyCompatible(mapping)))
+                   {
+                       addMapping(result);
+                       result->setMemoryDescriptor(this, offset);
+                       break;
+                   }
+               }
+               iter->release();
+           }
+           if (result || (options & kIOMapReference))
+           {
+               if (result != mapping)
+               {
+                    mapping->release();
+                    mapping = NULL;
+                }
+               continue;
+           }
+       }

 
 

-IOReturn IOSubMemoryDescriptor::prepare(
-               IODirection forDirection)
-{
-    IOReturn   err;
+       if (!mapDesc)
+       {
+           mapDesc = this;
+           mapDesc->retain();
+       }
+       IOReturn
+       kr = mapDesc->doMap( 0, (IOVirtualAddress *) &mapping, options, 0, 0 );
+       if (kIOReturnSuccess == kr)
+       {
+           result = mapping;
+           mapDesc->addMapping(result);
+           result->setMemoryDescriptor(mapDesc, offset);
+       }
+       else
+       {
+           mapping->release();
+           mapping = NULL;
+       }
+    }
+    while( false );

 
 

-    LOCK;
-    err = _parent->prepare( forDirection);

     UNLOCK;
 
     UNLOCK;
 

-    return( err );
+    if (mapDesc)
+       mapDesc->release();
+
+    return (result);

 }
 
 }
 

-IOReturn IOSubMemoryDescriptor::complete(
-               IODirection forDirection)
+void IOMemoryDescriptor::addMapping(
+       IOMemoryMap * mapping )

 {
 {

-    IOReturn   err;
-
-    LOCK;
-    err = _parent->complete( forDirection);
-    UNLOCK;
-
-    return( err );
+    if( mapping)
+    {
+        if( 0 == _mappings)
+            _mappings = OSSet::withCapacity(1);
+       if( _mappings )
+           _mappings->setObject( mapping );
+    }

 }
 
 }
 

-IOMemoryMap * IOSubMemoryDescriptor::makeMapping(
-       IOMemoryDescriptor *    owner,
-       task_t                  intoTask,
-       IOVirtualAddress        toAddress,
-       IOOptionBits            options,
-       IOByteCount             offset,
-       IOByteCount             length )
+void IOMemoryDescriptor::removeMapping(
+       IOMemoryMap * mapping )

 {
 {

-    IOMemoryMap * mapping;
-
-     mapping = (IOMemoryMap *) _parent->makeMapping(
-                                       _parent, intoTask,
-                                       toAddress - (_start + offset),
-                                       options | kIOMapReference,
-                                       _start + offset, length );
-
-    if( !mapping)
-        mapping = (IOMemoryMap *) _parent->makeMapping(
-                                       _parent, intoTask,
-                                       toAddress,
-                                       options, _start + offset, length );
-
-    if( !mapping)
-       mapping = super::makeMapping( owner, intoTask, toAddress, options,
-                                       offset, length );
-
-    return( mapping );
+    if( _mappings)
+        _mappings->removeObject( mapping);

 }
 
 }
 

-/* ick */
-
+#ifndef __LP64__
+// obsolete initializers
+// - initWithOptions is the designated initializer 

 bool
 bool

-IOSubMemoryDescriptor::initWithAddress(void *      address,
+IOMemoryDescriptor::initWithAddress(void *      address,

                                     IOByteCount   length,
                                     IODirection direction)
 {
                                     IOByteCount   length,
                                     IODirection direction)
 {


@@ -2461,7 +3650,7 @@ IOSubMemoryDescriptor::initWithAddress(void *      address,
 }
 
 bool
 }
 
 bool

-IOSubMemoryDescriptor::initWithAddress(vm_address_t address,
+IOMemoryDescriptor::initWithAddress(IOVirtualAddress address,

                                     IOByteCount    length,
                                     IODirection  direction,
                                     task_t       task)
                                     IOByteCount    length,
                                     IODirection  direction,
                                     task_t       task)


@@ -2470,7 +3659,7 @@ IOSubMemoryDescriptor::initWithAddress(vm_address_t address,
 }
 
 bool
 }
 
 bool

-IOSubMemoryDescriptor::initWithPhysicalAddress(
+IOMemoryDescriptor::initWithPhysicalAddress(

                                 IOPhysicalAddress      address,
                                 IOByteCount            length,
                                 IODirection            direction )
                                 IOPhysicalAddress      address,
                                 IOByteCount            length,
                                 IODirection            direction )


@@ -2479,7 +3668,7 @@ IOSubMemoryDescriptor::initWithPhysicalAddress(
 }
 
 bool
 }
 
 bool

-IOSubMemoryDescriptor::initWithRanges(
+IOMemoryDescriptor::initWithRanges(

                                        IOVirtualRange * ranges,
                                        UInt32           withCount,
                                        IODirection      direction,
                                        IOVirtualRange * ranges,
                                        UInt32           withCount,
                                        IODirection      direction,


@@ -2490,7 +3679,7 @@ IOSubMemoryDescriptor::initWithRanges(
 }
 
 bool
 }
 
 bool

-IOSubMemoryDescriptor::initWithPhysicalRanges( IOPhysicalRange * ranges,
+IOMemoryDescriptor::initWithPhysicalRanges(    IOPhysicalRange * ranges,

                                                UInt32           withCount,
                                                IODirection      direction,
                                                bool             asReference)
                                                UInt32           withCount,
                                                IODirection      direction,
                                                bool             asReference)


@@ -2498,16 +3687,28 @@ IOSubMemoryDescriptor::initWithPhysicalRanges(  IOPhysicalRange * ranges,
     return( false );
 }
 
     return( false );
 }
 

+void * IOMemoryDescriptor::getVirtualSegment(IOByteCount offset,
+                                       IOByteCount * lengthOfSegment)
+{
+    return( 0 );
+}
+#endif /* !__LP64__ */
+

 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 bool IOGeneralMemoryDescriptor::serialize(OSSerialize * s) const
 {
     OSSymbol const *keys[2];
     OSObject *values[2];
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 bool IOGeneralMemoryDescriptor::serialize(OSSerialize * s) const
 {
     OSSymbol const *keys[2];
     OSObject *values[2];

-    IOVirtualRange *vcopy;
+    struct SerData {
+       user_addr_t address;
+       user_size_t length;
+    } *vcopy;

     unsigned int index, nRanges;
     bool result;
 
     unsigned int index, nRanges;
     bool result;
 

+    IOOptionBits type = _flags & kIOMemoryTypeMask;
+

     if (s == NULL) return false;
     if (s->previouslySerialized(this)) return true;
 
     if (s == NULL) return false;
     if (s->previouslySerialized(this)) return true;
 


@@ -2515,7 +3716,7 @@ bool IOGeneralMemoryDescriptor::serialize(OSSerialize * s) const
     if (!s->addXMLStartTag(this, "array")) return false;
 
     nRanges = _rangesCount;
     if (!s->addXMLStartTag(this, "array")) return false;
 
     nRanges = _rangesCount;

-    vcopy = (IOVirtualRange *) IOMalloc(sizeof(IOVirtualRange) * nRanges);
+    vcopy = (SerData *) IOMalloc(sizeof(SerData) * nRanges);

     if (vcopy == 0) return false;
 
     keys[0] = OSSymbol::withCString("address");
     if (vcopy == 0) return false;
 
     keys[0] = OSSymbol::withCString("address");


@@ -2530,8 +3731,12 @@ bool IOGeneralMemoryDescriptor::serialize(OSSerialize * s) const
     // while the lock is held.
     LOCK;
     if (nRanges == _rangesCount) {
     // while the lock is held.
     LOCK;
     if (nRanges == _rangesCount) {

+       Ranges vec = _ranges;

         for (index = 0; index < nRanges; index++) {
         for (index = 0; index < nRanges; index++) {

-            vcopy[index] = _ranges.v[index];
+           user_addr_t addr; IOByteCount len;
+           getAddrLenForInd(addr, len, type, vec, index);
+            vcopy[index].address = addr;
+            vcopy[index].length  = len;

         }
     } else {
        // The descriptor changed out from under us.  Give up.
         }
     } else {
        // The descriptor changed out from under us.  Give up.


@@ -2543,12 +3748,15 @@ bool IOGeneralMemoryDescriptor::serialize(OSSerialize * s) const
 
     for (index = 0; index < nRanges; index++)
     {
 
     for (index = 0; index < nRanges; index++)
     {

-       values[0] = OSNumber::withNumber(_ranges.v[index].address, sizeof(_ranges.v[index].address) * 8);
+       user_addr_t addr = vcopy[index].address;
+       IOByteCount len = (IOByteCount) vcopy[index].length;
+       values[0] =
+           OSNumber::withNumber(addr, sizeof(addr) * 8);

        if (values[0] == 0) {
          result = false;
          goto bail;
        }
        if (values[0] == 0) {
          result = false;
          goto bail;
        }

-       values[1] = OSNumber::withNumber(_ranges.v[index].length, sizeof(_ranges.v[index].length) * 8);
+       values[1] = OSNumber::withNumber(len, sizeof(len) * 8);

        if (values[1] == 0) {
          result = false;
          goto bail;
        if (values[1] == 0) {
          result = false;
          goto bail;


@@ -2580,67 +3788,30 @@ bool IOGeneralMemoryDescriptor::serialize(OSSerialize * s) const
     if (keys[1])
       keys[1]->release();
     if (vcopy)
     if (keys[1])
       keys[1]->release();
     if (vcopy)

-        IOFree(vcopy, sizeof(IOVirtualRange) * nRanges);
+        IOFree(vcopy, sizeof(SerData) * nRanges);

     return result;
 }
 
     return result;
 }
 

-bool IOSubMemoryDescriptor::serialize(OSSerialize * s) const
-{
-    if (!s) {
-       return (false);
-    }
-    if (s->previouslySerialized(this)) return true;
-
-    // Pretend we are a dictionary.
-    // We must duplicate the functionality of OSDictionary here
-    // because otherwise object references will not work;
-    // they are based on the value of the object passed to
-    // previouslySerialized and addXMLStartTag.
-
-    if (!s->addXMLStartTag(this, "dict")) return false;
-
-    char const *keys[3] = {"offset", "length", "parent"};
-
-    OSObject *values[3];
-    values[0] = OSNumber::withNumber(_start, sizeof(_start) * 8);
-    if (values[0] == 0)
-       return false;
-    values[1] = OSNumber::withNumber(_length, sizeof(_length) * 8);
-    if (values[1] == 0) {
-       values[0]->release();
-       return false;
-    }
-    values[2] = _parent;
-
-    bool result = true;
-    for (int i=0; i<3; i++) {
-        if (!s->addString("<key>") ||
-           !s->addString(keys[i]) ||
-           !s->addXMLEndTag("key") ||
-           !values[i]->serialize(s)) {
-         result = false;
-         break;
-        }
-    }
-    values[0]->release();
-    values[1]->release();
-    if (!result) {
-      return false;
-    }
-
-    return s->addXMLEndTag("dict");
-}
-

 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 0);
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 0);

-OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 1);
-OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 2);
+#ifdef __LP64__
+OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 1);
+OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 2);

 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 3);
 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 4);
 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 5);
 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 6);
 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 7);
 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 3);
 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 4);
 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 5);
 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 6);
 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 7);

+#else /* !__LP64__ */
+OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 1);
+OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 2);
+OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 3);
+OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 4);
+OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 5);
+OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 6);
+OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 7);
+#endif /* !__LP64__ */

 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 8);
 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 9);
 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 10);
 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 8);
 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 9);
 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 10);


@@ -2651,5 +3822,6 @@ OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 14);
 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 15);
 
 /* ex-inline function implementation */
 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 15);
 
 /* ex-inline function implementation */

-IOPhysicalAddress IOMemoryDescriptor::getPhysicalAddress()
+IOPhysicalAddress 
+IOMemoryDescriptor::getPhysicalAddress()

         { return( getPhysicalSegment( 0, 0 )); }
         { return( getPhysicalSegment( 0, 0 )); }