xnu-2782.20.48.tar.gz

[apple/xnu.git] / iokit / Kernel / IOMemoryDescriptor.cpp

diff --git a/iokit/Kernel/IOMemoryDescriptor.cpp b/iokit/Kernel/IOMemoryDescriptor.cpp
index 0e1873b30aceb7ea335b17c3c88241aa8a8de857..0c77443861c05e2b02c8ac46dd5ffafc0181b486 100644 (file)
--- a/iokit/Kernel/IOMemoryDescriptor.cpp
+++ b/iokit/Kernel/IOMemoryDescriptor.cpp
@@ -1,31 +1,29 @@
 /*
 /*

- * Copyright (c) 1998-2004 Apple Computer, Inc. All rights reserved.
+ * Copyright (c) 1998-2007 Apple Inc. All rights reserved.

  *
  *

- * @APPLE_LICENSE_OSREFERENCE_HEADER_START@
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_START@

  * 
  * 

- * This file contains Original Code and/or Modifications of Original Code 
- * as defined in and that are subject to the Apple Public Source License 
- * Version 2.0 (the 'License'). You may not use this file except in 
- * compliance with the License.  The rights granted to you under the 
- * License may not be used to create, or enable the creation or 
- * redistribution of, unlawful or unlicensed copies of an Apple operating 
- * system, or to circumvent, violate, or enable the circumvention or 
- * violation of, any terms of an Apple operating system software license 
- * agreement.
- *
- * Please obtain a copy of the License at 
- * http://www.opensource.apple.com/apsl/ and read it before using this 
- * file.
- *
- * The Original Code and all software distributed under the License are 
- * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 
- * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 
- * Please see the License for the specific language governing rights and 
+ * This file contains Original Code and/or Modifications of Original Code
+ * as defined in and that are subject to the Apple Public Source License
+ * Version 2.0 (the 'License'). You may not use this file except in
+ * compliance with the License. The rights granted to you under the License
+ * may not be used to create, or enable the creation or redistribution of,
+ * unlawful or unlicensed copies of an Apple operating system, or to
+ * circumvent, violate, or enable the circumvention or violation of, any
+ * terms of an Apple operating system software license agreement.
+ * 
+ * Please obtain a copy of the License at
+ * http://www.opensource.apple.com/apsl/ and read it before using this file.
+ * 
+ * The Original Code and all software distributed under the License are
+ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
+ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
+ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
+ * Please see the License for the specific language governing rights and

  * limitations under the License.
  * limitations under the License.

- *
- * @APPLE_LICENSE_OSREFERENCE_HEADER_END@
+ * 
+ * @APPLE_OSREFERENCE_LICENSE_HEADER_END@

  */
 /*
  * Copyright (c) 1998 Apple Computer, Inc.  All rights reserved. 
  */
 /*
  * Copyright (c) 1998 Apple Computer, Inc.  All rights reserved. 


@@ -33,7 +31,8 @@
  * HISTORY
  *
  */
  * HISTORY
  *
  */

-// 45678901234567890123456789012345678901234567890123456789012345678901234567890
+
+

 #include <sys/cdefs.h>
 
 #include <IOKit/assert.h>
 #include <sys/cdefs.h>
 
 #include <IOKit/assert.h>


@@ -41,12 +40,17 @@
 #include <IOKit/IOLib.h>
 #include <IOKit/IOMemoryDescriptor.h>
 #include <IOKit/IOMapper.h>
 #include <IOKit/IOLib.h>
 #include <IOKit/IOMemoryDescriptor.h>
 #include <IOKit/IOMapper.h>

+#include <IOKit/IODMACommand.h>

 #include <IOKit/IOKitKeysPrivate.h>
 
 #include <IOKit/IOKitKeysPrivate.h>
 

+#ifndef __LP64__
+#include <IOKit/IOSubMemoryDescriptor.h>
+#endif /* !__LP64__ */
+

 #include <IOKit/IOKitDebug.h>
 #include <IOKit/IOKitDebug.h>

+#include <libkern/OSDebug.h>

 
 #include "IOKitKernelInternal.h"
 
 #include "IOKitKernelInternal.h"

-#include "IOCopyMapper.h"

 
 #include <libkern/c++/OSContainers.h>
 #include <libkern/c++/OSDictionary.h>
 
 #include <libkern/c++/OSContainers.h>
 #include <libkern/c++/OSDictionary.h>


@@ -59,34 +63,17 @@
 __BEGIN_DECLS
 #include <vm/pmap.h>
 #include <vm/vm_pageout.h>
 __BEGIN_DECLS
 #include <vm/pmap.h>
 #include <vm/vm_pageout.h>

-#include <vm/vm_shared_memory_server.h>

 #include <mach/memory_object_types.h>
 #include <device/device_port.h>
 
 #include <mach/vm_prot.h>
 #include <mach/memory_object_types.h>
 #include <device/device_port.h>
 
 #include <mach/vm_prot.h>

+#include <mach/mach_vm.h>

 #include <vm/vm_fault.h>
 #include <vm/vm_fault.h>

+#include <vm/vm_protos.h>

 
 extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
 
 extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);

-void ipc_port_release_send(ipc_port_t port);
-
-/* Copy between a physical page and a virtual address in the given vm_map */
-kern_return_t copypv(addr64_t source, addr64_t sink, unsigned int size, int which);
-
-memory_object_t
-device_pager_setup(
-       memory_object_t pager,
-       int             device_handle,
-       vm_size_t       size,
-       int             flags);
-void
-device_pager_deallocate(
-        memory_object_t);
-kern_return_t
-device_pager_populate_object(
-       memory_object_t         pager,
-       vm_object_offset_t      offset,
-       ppnum_t                 phys_addr,
-       vm_size_t               size);
+extern void ipc_port_release_send(ipc_port_t port);
+

 kern_return_t
 memory_object_iopl_request(
        ipc_port_t              port,
 kern_return_t
 memory_object_iopl_request(
        ipc_port_t              port,


@@ -97,18 +84,16 @@ memory_object_iopl_request(
        unsigned int            *page_list_count,
        int                     *flags);
 
        unsigned int            *page_list_count,
        int                     *flags);
 

+// osfmk/device/iokit_rpc.c
+unsigned int IODefaultCacheBits(addr64_t pa);

 unsigned int  IOTranslateCacheBits(struct phys_entry *pp);
 
 __END_DECLS
 
 unsigned int  IOTranslateCacheBits(struct phys_entry *pp);
 
 __END_DECLS
 

-#define kIOMaximumMappedIOByteCount    (512*1024*1024)
+#define kIOMapperWaitSystem    ((IOMapper *) 1)

 
 static IOMapper * gIOSystemMapper = NULL;
 
 
 static IOMapper * gIOSystemMapper = NULL;
 

-IOCopyMapper *   gIOCopyMapper = NULL;
-
-static ppnum_t   gIOMaximumMappedIOPageCount = atop_32(kIOMaximumMappedIOByteCount);
-

 ppnum_t                  gIOLastPage;
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 ppnum_t                  gIOLastPage;
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */


@@ -135,73 +120,7 @@ static IORecursiveLock * gIOMemoryLock;
 #define DEBG(fmt, args...)     {}
 #endif
 
 #define DEBG(fmt, args...)     {}
 #endif
 

-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-class _IOMemoryMap : public IOMemoryMap
-{
-    OSDeclareDefaultStructors(_IOMemoryMap)
-public:
-    IOMemoryDescriptor * memory;
-    IOMemoryMap *      superMap;
-    IOByteCount                offset;
-    IOByteCount                length;
-    IOVirtualAddress   logical;
-    task_t             addressTask;
-    vm_map_t           addressMap;
-    IOOptionBits       options;
-    upl_t              redirUPL;
-    ipc_port_t         redirEntry;
-    IOMemoryDescriptor * owner;
-
-protected:
-    virtual void taggedRelease(const void *tag = 0) const;
-    virtual void free();
-
-public:
-
-    // IOMemoryMap methods
-    virtual IOVirtualAddress   getVirtualAddress();
-    virtual IOByteCount        getLength();
-    virtual task_t             getAddressTask();
-    virtual IOMemoryDescriptor * getMemoryDescriptor();
-    virtual IOOptionBits       getMapOptions();
-
-    virtual IOReturn           unmap();
-    virtual void               taskDied();
-
-    virtual IOReturn           redirect(IOMemoryDescriptor * newBackingMemory,
-                                        IOOptionBits         options,
-                                        IOByteCount          offset = 0);
-
-    virtual IOPhysicalAddress  getPhysicalSegment(IOByteCount offset,
-                                                  IOByteCount * length);
-
-    // for IOMemoryDescriptor use
-    _IOMemoryMap * copyCompatible(
-               IOMemoryDescriptor *    owner,
-                task_t                 intoTask,
-                IOVirtualAddress       toAddress,
-                IOOptionBits           options,
-                IOByteCount            offset,
-                IOByteCount            length );
-
-    bool initCompatible(
-       IOMemoryDescriptor *    memory,
-       IOMemoryMap *           superMap,
-        IOByteCount            offset,
-        IOByteCount            length );
-
-    bool initWithDescriptor(
-       IOMemoryDescriptor *    memory,
-       task_t                  intoTask,
-       IOVirtualAddress        toAddress,
-       IOOptionBits            options,
-        IOByteCount            offset,
-        IOByteCount            length );
-
-    IOReturn redirect(
-       task_t                  intoTask, bool redirect );
-};
+#define IOMD_DEBUG_DMAACTIVE   1

 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 


@@ -213,68 +132,70 @@ enum ioPLBlockFlags {
     kIOPLExternUPL = 0x00000002,
 };
 
     kIOPLExternUPL = 0x00000002,
 };
 

-struct typePersMDData
+struct IOMDPersistentInitData

 {
 {

-    const IOGeneralMemoryDescriptor *fMD;
-    ipc_port_t fMemEntry;
+    const IOGeneralMemoryDescriptor * fMD;
+    IOMemoryReference               * fMemRef;

 };
 
 struct ioPLBlock {
     upl_t fIOPL;
 };
 
 struct ioPLBlock {
     upl_t fIOPL;

-    vm_address_t fIOMDOffset;  // The offset of this iopl in descriptor
-    vm_offset_t fPageInfo;     // Pointer to page list or index into it
-    ppnum_t fMappedBase;       // Page number of first page in this iopl
-    unsigned int fPageOffset;  // Offset within first page of iopl
-    unsigned int fFlags;       // Flags
+    vm_address_t fPageInfo;   // Pointer to page list or index into it
+    uint32_t fIOMDOffset;          // The offset of this iopl in descriptor
+    ppnum_t fMappedPage;           // Page number of first page in this iopl
+    unsigned int fPageOffset;      // Offset within first page of iopl
+    unsigned int fFlags;           // Flags

 };
 
 struct ioGMDData {
 };
 
 struct ioGMDData {

-    IOMapper *fMapper;
+    IOMapper *  fMapper;
+    uint8_t    fDMAMapNumAddressBits;
+    uint64_t    fDMAMapAlignment;
+    addr64_t    fMappedBase;
+    uint64_t fPreparationID;

     unsigned int fPageCnt;
     unsigned int fPageCnt;

-    upl_page_info_t fPageList[];
-    ioPLBlock fBlocks[];
+    unsigned char fDiscontig:1;
+    unsigned char fCompletionError:1;
+    unsigned char _resv:6;
+#if __LP64__
+    // align arrays to 8 bytes so following macros work
+    unsigned char fPad[3];
+#endif
+    upl_page_info_t fPageList[1]; /* variable length */
+    ioPLBlock fBlocks[1]; /* variable length */

 };
 
 #define getDataP(osd)  ((ioGMDData *) (osd)->getBytesNoCopy())
 };
 
 #define getDataP(osd)  ((ioGMDData *) (osd)->getBytesNoCopy())

-#define getIOPLList(d) ((ioPLBlock *) &(d->fPageList[d->fPageCnt]))
+#define getIOPLList(d) ((ioPLBlock *) (void *)&(d->fPageList[d->fPageCnt]))

 #define getNumIOPL(osd, d)     \
     (((osd)->getLength() - ((char *) getIOPLList(d) - (char *) d)) / sizeof(ioPLBlock))
 #define getPageList(d) (&(d->fPageList[0]))
 #define computeDataSize(p, u) \
 #define getNumIOPL(osd, d)     \
     (((osd)->getLength() - ((char *) getIOPLList(d) - (char *) d)) / sizeof(ioPLBlock))
 #define getPageList(d) (&(d->fPageList[0]))
 #define computeDataSize(p, u) \

-    (sizeof(ioGMDData) + p * sizeof(upl_page_info_t) + u * sizeof(ioPLBlock))
-
+    (offsetof(ioGMDData, fPageList) + p * sizeof(upl_page_info_t) + u * sizeof(ioPLBlock))

 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 

-#define next_page(a) ( trunc_page_32(a) + PAGE_SIZE )
-
+#define next_page(a) ( trunc_page(a) + PAGE_SIZE )

 
 extern "C" {
 
 kern_return_t device_data_action(
 
 extern "C" {
 
 kern_return_t device_data_action(

-               int                     device_handle, 
+               uintptr_t               device_handle, 

                ipc_port_t              device_pager,
                vm_prot_t               protection, 
                vm_object_offset_t      offset, 
                vm_size_t               size)
 {
                ipc_port_t              device_pager,
                vm_prot_t               protection, 
                vm_object_offset_t      offset, 
                vm_size_t               size)
 {

-    struct ExpansionData {
-        void *                         devicePager;
-        unsigned int                   pagerContig:1;
-        unsigned int                   unused:31;
-       IOMemoryDescriptor *            memory;
-    };

     kern_return_t       kr;
     kern_return_t       kr;

-    ExpansionData *      ref = (ExpansionData *) device_handle;
+    IOMemoryDescriptorReserved * ref = (IOMemoryDescriptorReserved *) device_handle;

     IOMemoryDescriptor * memDesc;
 
     LOCK;
     IOMemoryDescriptor * memDesc;
 
     LOCK;

-    memDesc = ref->memory;
+    memDesc = ref->dp.memory;

     if( memDesc)
     {
        memDesc->retain();
     if( memDesc)
     {
        memDesc->retain();

-       kr = memDesc->handleFault( device_pager, 0, 0,
-                offset, size, kIOMapDefaultCache /*?*/);
+       kr = memDesc->handleFault(device_pager, offset, size);

        memDesc->release();
     }
     else
        memDesc->release();
     }
     else


@@ -285,27 +206,23 @@ kern_return_t device_data_action(
 }
 
 kern_return_t device_close(
 }
 
 kern_return_t device_close(

-               int     device_handle)
+               uintptr_t     device_handle)

 {
 {

-    struct ExpansionData {
-        void *                         devicePager;
-        unsigned int                   pagerContig:1;
-        unsigned int                   unused:31;
-       IOMemoryDescriptor *            memory;
-    };
-    ExpansionData *   ref = (ExpansionData *) device_handle;
+    IOMemoryDescriptorReserved * ref = (IOMemoryDescriptorReserved *) device_handle;

 
 

-    IODelete( ref, ExpansionData, 1 );
+    IODelete( ref, IOMemoryDescriptorReserved, 1 );

 
     return( kIOReturnSuccess );
 }
 };     // end extern "C"
 
 
     return( kIOReturnSuccess );
 }
 };     // end extern "C"
 

+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+

 // Note this inline function uses C++ reference arguments to return values
 // This means that pointers are not passed and NULLs don't have to be
 // checked for as a NULL reference is illegal.
 static inline void
 // Note this inline function uses C++ reference arguments to return values
 // This means that pointers are not passed and NULLs don't have to be
 // checked for as a NULL reference is illegal.
 static inline void

-getAddrLenForInd(addr64_t &addr, IOPhysicalLength &len, // Output variables
+getAddrLenForInd(mach_vm_address_t &addr, mach_vm_size_t &len, // Output variables

      UInt32 type, IOGeneralMemoryDescriptor::Ranges r, UInt32 ind)
 {
     assert(kIOMemoryTypeUIO       == type
      UInt32 type, IOGeneralMemoryDescriptor::Ranges r, UInt32 ind)
 {
     assert(kIOMemoryTypeUIO       == type


@@ -313,13 +230,16 @@ getAddrLenForInd(addr64_t &addr, IOPhysicalLength &len, // Output variables
        || kIOMemoryTypePhysical  == type || kIOMemoryTypePhysical64 == type);
     if (kIOMemoryTypeUIO == type) {
        user_size_t us;
        || kIOMemoryTypePhysical  == type || kIOMemoryTypePhysical64 == type);
     if (kIOMemoryTypeUIO == type) {
        user_size_t us;

-       uio_getiov((uio_t) r.uio, ind, &addr, &us); len = us;
+       user_addr_t ad;
+       uio_getiov((uio_t) r.uio, ind, &ad, &us); addr = ad; len = us;

     }
     }

+#ifndef __LP64__

     else if ((kIOMemoryTypeVirtual64 == type) || (kIOMemoryTypePhysical64 == type)) {
        IOAddressRange cur = r.v64[ind];
        addr = cur.address;
        len  = cur.length;
     }
     else if ((kIOMemoryTypeVirtual64 == type) || (kIOMemoryTypePhysical64 == type)) {
        IOAddressRange cur = r.v64[ind];
        addr = cur.address;
        len  = cur.length;
     }

+#endif /* !__LP64__ */

     else {
        IOVirtualRange cur = r.v[ind];
        addr = cur.address;
     else {
        IOVirtualRange cur = r.v[ind];
        addr = cur.address;


@@ -329,3216 +249,4057 @@ getAddrLenForInd(addr64_t &addr, IOPhysicalLength &len, // Output variables
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 

-/*
- * withAddress:
- *
- * Create a new IOMemoryDescriptor.  The buffer is a virtual address
- * relative to the specified task.  If no task is supplied, the kernel
- * task is implied.
- */
-IOMemoryDescriptor *
-IOMemoryDescriptor::withAddress(void *      address,
-                                IOByteCount   length,
-                                IODirection direction)
+static IOReturn 
+purgeableControlBits(IOOptionBits newState, vm_purgable_t * control, int * state)

 {
 {

-    return IOMemoryDescriptor::
-        withAddress((vm_address_t) address, length, direction, kernel_task);
-}
+    IOReturn err = kIOReturnSuccess;

 
 

-IOMemoryDescriptor *
-IOMemoryDescriptor::withAddress(vm_address_t address,
-                                IOByteCount  length,
-                                IODirection  direction,
-                                task_t       task)
-{
-#if TEST_V64
-    if (task)
-    {
-       IOOptionBits options = (IOOptionBits) direction;
-       if (task == kernel_task)
-           options |= kIOMemoryAutoPrepare;
-       return (IOMemoryDescriptor::withAddressRange(address, length, options, task));
-    }
-#endif
-    IOGeneralMemoryDescriptor * that = new IOGeneralMemoryDescriptor;
-    if (that)
+    *control = VM_PURGABLE_SET_STATE;
+
+    enum { kIOMemoryPurgeableControlMask = 15 };
+
+    switch (kIOMemoryPurgeableControlMask & newState)

     {
     {

-       if (that->initWithAddress(address, length, direction, task))
-           return that;
+       case kIOMemoryPurgeableKeepCurrent:
+           *control = VM_PURGABLE_GET_STATE;
+           break;

 
 

-        that->release();
+       case kIOMemoryPurgeableNonVolatile:
+           *state = VM_PURGABLE_NONVOLATILE;
+           break;
+       case kIOMemoryPurgeableVolatile:
+           *state = VM_PURGABLE_VOLATILE | (newState & ~kIOMemoryPurgeableControlMask);
+           break;
+       case kIOMemoryPurgeableEmpty:
+           *state = VM_PURGABLE_EMPTY;
+           break;
+       default:
+           err = kIOReturnBadArgument;
+           break;

     }
     }

-    return 0;
+    return (err);

 }
 
 }
 

-IOMemoryDescriptor *
-IOMemoryDescriptor::withPhysicalAddress(
-                               IOPhysicalAddress       address,
-                               IOByteCount             length,
-                               IODirection             direction )
+static IOReturn 
+purgeableStateBits(int * state)

 {
 {

-#if TEST_P64
-    return (IOMemoryDescriptor::withAddressRange(address, length, (IOOptionBits) direction, NULL));
-#endif
-    IOGeneralMemoryDescriptor *self = new IOGeneralMemoryDescriptor;
-    if (self
-    && !self->initWithPhysicalAddress(address, length, direction)) {
-        self->release();
-        return 0;
-    }
+    IOReturn err = kIOReturnSuccess;

 
 

-    return self;
+    switch (VM_PURGABLE_STATE_MASK & *state)
+    {
+       case VM_PURGABLE_NONVOLATILE:
+           *state = kIOMemoryPurgeableNonVolatile;
+           break;
+       case VM_PURGABLE_VOLATILE:
+           *state = kIOMemoryPurgeableVolatile;
+           break;
+       case VM_PURGABLE_EMPTY:
+           *state = kIOMemoryPurgeableEmpty;
+           break;
+       default:
+           *state = kIOMemoryPurgeableNonVolatile;
+           err = kIOReturnNotReady;
+           break;
+    }
+    return (err);

 }
 
 }
 

-IOMemoryDescriptor *
-IOMemoryDescriptor::withRanges(        IOVirtualRange * ranges,
-                               UInt32           withCount,
-                               IODirection      direction,
-                               task_t           task,
-                               bool             asReference)
+
+static vm_prot_t 
+vmProtForCacheMode(IOOptionBits cacheMode)

 {
 {

-    IOGeneralMemoryDescriptor * that = new IOGeneralMemoryDescriptor;
-    if (that)
+    vm_prot_t prot = 0;
+    switch (cacheMode)

     {
     {

-       if (that->initWithRanges(ranges, withCount, direction, task, asReference))
-           return that;
+       case kIOInhibitCache:
+           SET_MAP_MEM(MAP_MEM_IO, prot);
+           break;

 
 

-        that->release();
+       case kIOWriteThruCache:
+           SET_MAP_MEM(MAP_MEM_WTHRU, prot);
+           break;
+
+       case kIOWriteCombineCache:
+           SET_MAP_MEM(MAP_MEM_WCOMB, prot);
+           break;
+
+       case kIOCopybackCache:
+           SET_MAP_MEM(MAP_MEM_COPYBACK, prot);
+           break;
+
+       case kIOCopybackInnerCache:
+           SET_MAP_MEM(MAP_MEM_INNERWBACK, prot);
+           break;
+
+       case kIODefaultCache:
+       default:
+           SET_MAP_MEM(MAP_MEM_NOOP, prot);
+           break;

     }
     }

-    return 0;
-}

 
 

-IOMemoryDescriptor *
-IOMemoryDescriptor::withAddressRange(mach_vm_address_t address,
-                                       mach_vm_size_t length,
-                                       IOOptionBits   options,
-                                       task_t         task)
-{
-    IOAddressRange range = { address, length };
-    return (IOMemoryDescriptor::withAddressRanges(&range, 1, options, task));
+    return (prot);

 }
 
 }
 

-IOMemoryDescriptor *
-IOMemoryDescriptor::withAddressRanges(IOAddressRange *   ranges,
-                                       UInt32           rangeCount,
-                                       IOOptionBits     options,
-                                       task_t           task)
+static unsigned int
+pagerFlagsForCacheMode(IOOptionBits cacheMode)

 {
 {

-    IOGeneralMemoryDescriptor * that = new IOGeneralMemoryDescriptor;
-    if (that)
+    unsigned int pagerFlags = 0;
+    switch (cacheMode)

     {
     {

-       if (task)
-           options |= kIOMemoryTypeVirtual64;
-       else
-           options |= kIOMemoryTypePhysical64;
+       case kIOInhibitCache:
+           pagerFlags = DEVICE_PAGER_CACHE_INHIB |  DEVICE_PAGER_COHERENT | DEVICE_PAGER_GUARDED;
+           break;

 
 

-       if (that->initWithOptions(ranges, rangeCount, 0, task, options, /* mapper */ 0))
-           return that;
+       case kIOWriteThruCache:
+           pagerFlags = DEVICE_PAGER_WRITE_THROUGH | DEVICE_PAGER_COHERENT | DEVICE_PAGER_GUARDED;
+           break;

 
 

-       that->release();
-    }
+       case kIOWriteCombineCache:
+           pagerFlags = DEVICE_PAGER_CACHE_INHIB | DEVICE_PAGER_COHERENT;
+           break;

 
 

-    return 0;
+       case kIOCopybackCache:
+           pagerFlags = DEVICE_PAGER_COHERENT;
+           break;
+
+       case kIOCopybackInnerCache:
+           pagerFlags = DEVICE_PAGER_COHERENT;
+           break;
+
+       case kIODefaultCache:
+       default:
+           pagerFlags = -1U;
+           break;
+    }
+    return (pagerFlags);

 }
 
 }
 

+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

 
 

-/*
- * withRanges:
- *
- * Create a new IOMemoryDescriptor. The buffer is made up of several
- * virtual address ranges, from a given task.
- *
- * Passing the ranges as a reference will avoid an extra allocation.
- */
-IOMemoryDescriptor *
-IOMemoryDescriptor::withOptions(void *         buffers,
-                                UInt32         count,
-                                UInt32         offset,
-                                task_t         task,
-                                IOOptionBits   opts,
-                                IOMapper *     mapper)
+struct IOMemoryEntry

 {
 {

-    IOGeneralMemoryDescriptor *self = new IOGeneralMemoryDescriptor;
-
-    if (self
-    && !self->initWithOptions(buffers, count, offset, task, opts, mapper))
-    {
-        self->release();
-        return 0;
-    }
+    ipc_port_t entry;
+    int64_t    offset;
+    uint64_t   size;
+};

 
 

-    return self;
-}
+struct IOMemoryReference
+{
+    volatile SInt32 refCount;
+    vm_prot_t       prot;
+    uint32_t        capacity;
+    uint32_t        count;
+    IOMemoryEntry   entries[0];
+};

 
 

-// Can't leave abstract but this should never be used directly,
-bool IOMemoryDescriptor::initWithOptions(void *                buffers,
-                                         UInt32                count,
-                                         UInt32                offset,
-                                         task_t                task,
-                                         IOOptionBits  options,
-                                         IOMapper *    mapper)
+enum

 {
 {

-    // @@@ gvdl: Should I panic?
-    panic("IOMD::initWithOptions called\n");
-    return 0;
-}
+    kIOMemoryReferenceReuse = 0x00000001,
+    kIOMemoryReferenceWrite = 0x00000002,
+};

 
 

-IOMemoryDescriptor *
-IOMemoryDescriptor::withPhysicalRanges(        IOPhysicalRange * ranges,
-                                        UInt32          withCount,
-                                        IODirection     direction,
-                                        bool            asReference)
+SInt32 gIOMemoryReferenceCount;
+
+IOMemoryReference *
+IOGeneralMemoryDescriptor::memoryReferenceAlloc(uint32_t capacity, IOMemoryReference * realloc)

 {
 {

-    IOGeneralMemoryDescriptor * that = new IOGeneralMemoryDescriptor;
-    if (that)
+    IOMemoryReference * ref;
+    size_t              newSize, oldSize, copySize;
+
+    newSize = (sizeof(IOMemoryReference) 
+                 - sizeof(ref->entries) 
+                 + capacity * sizeof(ref->entries[0]));
+    ref = (typeof(ref)) IOMalloc(newSize);
+    if (realloc)

     {
     {

-       if (that->initWithPhysicalRanges(ranges, withCount, direction, asReference))
-           return that;
-
-        that->release();
+       oldSize = (sizeof(IOMemoryReference) 
+                       - sizeof(realloc->entries) 
+                       + realloc->capacity * sizeof(realloc->entries[0]));
+       copySize = oldSize;
+        if (copySize > newSize) copySize = newSize;
+       if (ref) bcopy(realloc, ref, copySize);
+       IOFree(realloc, oldSize);

     }
     }

-    return 0;
+    else if (ref)
+    {
+       bzero(ref, sizeof(*ref));
+       ref->refCount = 1;
+       OSIncrementAtomic(&gIOMemoryReferenceCount);
+    }
+    if (!ref) return (0);
+    ref->capacity = capacity;
+    return (ref);

 }
 
 }
 

-IOMemoryDescriptor *
-IOMemoryDescriptor::withSubRange(IOMemoryDescriptor *  of,
-                               IOByteCount             offset,
-                               IOByteCount             length,
-                               IODirection             direction)
+void 
+IOGeneralMemoryDescriptor::memoryReferenceFree(IOMemoryReference * ref)

 {
 {

-    IOSubMemoryDescriptor *self = new IOSubMemoryDescriptor;
+    IOMemoryEntry * entries;
+    size_t          size;

 
 

-    if (self && !self->initSubRange(of, offset, length, direction)) {
-        self->release();
-       self = 0;
+    entries = ref->entries + ref->count;
+    while (entries > &ref->entries[0])
+    {
+        entries--;
+        ipc_port_release_send(entries->entry);

     }
     }

-    return self;
+    size = (sizeof(IOMemoryReference) 
+                 - sizeof(ref->entries) 
+                 + ref->capacity * sizeof(ref->entries[0]));
+    IOFree(ref, size);
+
+    OSDecrementAtomic(&gIOMemoryReferenceCount);

 }
 
 }
 

-IOMemoryDescriptor *
-IOMemoryDescriptor::withPersistentMemoryDescriptor(IOMemoryDescriptor *originalMD)
+void 
+IOGeneralMemoryDescriptor::memoryReferenceRelease(IOMemoryReference * ref)

 {
 {

-    IOGeneralMemoryDescriptor *origGenMD = 
-       OSDynamicCast(IOGeneralMemoryDescriptor, originalMD);
-
-    if (origGenMD)
-       return IOGeneralMemoryDescriptor::
-           withPersistentMemoryDescriptor(origGenMD);
-    else
-       return 0;
+    if (1 == OSDecrementAtomic(&ref->refCount)) memoryReferenceFree(ref);

 }
 
 }
 

-IOMemoryDescriptor *
-IOGeneralMemoryDescriptor::withPersistentMemoryDescriptor(IOGeneralMemoryDescriptor *originalMD)
-{
-    ipc_port_t sharedMem = (ipc_port_t) originalMD->createNamedEntry();

 
 

-    if (!sharedMem)
-       return 0;
-   
-    if (sharedMem == originalMD->_memEntry) {
-       originalMD->retain();               // Add a new reference to ourselves
-       ipc_port_release_send(sharedMem);   // Remove extra send right
-       return originalMD;
+IOReturn
+IOGeneralMemoryDescriptor::memoryReferenceCreate(
+                        IOOptionBits         options,
+                        IOMemoryReference ** reference)
+{
+    enum { kCapacity = 4, kCapacityInc = 4 };
+
+    kern_return_t        err;
+    IOMemoryReference *  ref;
+    IOMemoryEntry *      entries;
+    IOMemoryEntry *      cloneEntries;
+    vm_map_t             map;
+    ipc_port_t           entry, cloneEntry;
+    vm_prot_t            prot;
+    memory_object_size_t actualSize;
+    uint32_t             rangeIdx;
+    uint32_t             count;
+    mach_vm_address_t    entryAddr, endAddr, entrySize;
+    mach_vm_size_t       srcAddr, srcLen;
+    mach_vm_size_t       nextAddr, nextLen;
+    mach_vm_size_t       offset, remain;
+    IOByteCount          physLen;
+    IOOptionBits         type = (_flags & kIOMemoryTypeMask);
+    IOOptionBits         cacheMode;
+    unsigned int        pagerFlags;
+
+    ref = memoryReferenceAlloc(kCapacity, NULL);
+    if (!ref) return (kIOReturnNoMemory);
+    entries = &ref->entries[0];
+    count = 0;
+
+    offset = 0;
+    rangeIdx = 0;
+    if (_task) getAddrLenForInd(nextAddr, nextLen, type, _ranges, rangeIdx);
+    else
+    {
+        nextAddr = getPhysicalSegment(offset, &physLen, kIOMemoryMapperNone);
+        nextLen = physLen;
+       // default cache mode for physical
+       if (kIODefaultCache == ((_flags & kIOMemoryBufferCacheMask) >> kIOMemoryBufferCacheShift))
+       {
+           IOOptionBits mode;
+           pagerFlags = IODefaultCacheBits(nextAddr);
+           if (DEVICE_PAGER_CACHE_INHIB & pagerFlags)
+           {
+               if (DEVICE_PAGER_GUARDED & pagerFlags)
+                   mode = kIOInhibitCache;
+               else
+                   mode = kIOWriteCombineCache;
+           }
+           else if (DEVICE_PAGER_WRITE_THROUGH & pagerFlags)
+               mode = kIOWriteThruCache;
+           else
+               mode = kIOCopybackCache;
+           _flags |= (mode << kIOMemoryBufferCacheShift);
+       }

     }
 
     }
 

-    IOGeneralMemoryDescriptor * self = new IOGeneralMemoryDescriptor;
-    typePersMDData initData = { originalMD, sharedMem };
+    // cache mode & vm_prot
+    prot = VM_PROT_READ;
+    cacheMode = ((_flags & kIOMemoryBufferCacheMask) >> kIOMemoryBufferCacheShift);
+    prot |= vmProtForCacheMode(cacheMode);
+    // VM system requires write access to change cache mode
+    if (kIODefaultCache != cacheMode)                    prot |= VM_PROT_WRITE;
+    if (kIODirectionOut != (kIODirectionOutIn & _flags)) prot |= VM_PROT_WRITE;
+    if (kIOMemoryReferenceWrite & options)               prot |= VM_PROT_WRITE;

 
 

-    if (self
-    && !self->initWithOptions(&initData, 1, 0, 0, kIOMemoryTypePersistentMD, 0)) {
-        self->release();
-       self = 0;
+    if ((kIOMemoryReferenceReuse & options) && _memRef)
+    {
+        cloneEntries = &_memRef->entries[0];
+       prot |= MAP_MEM_NAMED_REUSE;

     }
     }

-    return self;
-}

 
 

-void *IOGeneralMemoryDescriptor::createNamedEntry()
-{
-    kern_return_t error;
-    ipc_port_t sharedMem;
+    if (_task)
+    {
+       // virtual ranges

 
 

-    IOOptionBits type = _flags & kIOMemoryTypeMask;
+       if (kIOMemoryBufferPageable & _flags)
+       {
+           // IOBufferMemoryDescriptor alloc - set flags for entry + object create
+           prot |= MAP_MEM_NAMED_CREATE;
+           if (kIOMemoryBufferPurgeable & _flags) prot |= MAP_MEM_PURGABLE;
+           prot |= VM_PROT_WRITE;
+           map = NULL;
+       }
+       else map = get_task_map(_task);

 
 

-    user_addr_t range0Addr;
-    IOByteCount range0Len;
-    getAddrLenForInd(range0Addr, range0Len, type, _ranges, 0);
-    range0Addr = trunc_page_64(range0Addr);
+       remain = _length;
+       while (remain)
+       {
+           srcAddr  = nextAddr;
+           srcLen   = nextLen;
+           nextAddr = 0;
+           nextLen  = 0;
+           // coalesce addr range
+           for (++rangeIdx; rangeIdx < _rangesCount; rangeIdx++)
+           {
+               getAddrLenForInd(nextAddr, nextLen, type, _ranges, rangeIdx);
+               if ((srcAddr + srcLen) != nextAddr) break;
+               srcLen += nextLen;
+           }
+           entryAddr = trunc_page_64(srcAddr);
+           endAddr   = round_page_64(srcAddr + srcLen);
+           do
+           {
+               entrySize = (endAddr - entryAddr);
+               if (!entrySize) break;
+               actualSize = entrySize;

 
 

-    vm_size_t size = ptoa_32(_pages);
-    vm_address_t kernelPage = (vm_address_t) range0Addr;
+               cloneEntry = MACH_PORT_NULL;
+               if (MAP_MEM_NAMED_REUSE & prot)
+               {
+                   if (cloneEntries < &_memRef->entries[_memRef->count]) cloneEntry = cloneEntries->entry;
+                   else                                                  prot &= ~MAP_MEM_NAMED_REUSE;
+               }

 
 

-    vm_map_t theMap = ((_task == kernel_task)
-                       && (kIOMemoryBufferPageable & _flags)) 
-                   ? IOPageableMapForAddress(kernelPage)
-                   : get_task_map(_task);
+               err = mach_make_memory_entry_64(map,
+                       &actualSize, entryAddr, prot, &entry, cloneEntry);

 
 

-    memory_object_size_t  actualSize = size;
-    vm_prot_t prot = VM_PROT_READ | VM_PROT_WRITE;
-    if (_memEntry)
-       prot |= MAP_MEM_NAMED_REUSE;
+               if (KERN_SUCCESS != err) break;
+               if (actualSize > entrySize) panic("mach_make_memory_entry_64 actualSize");

 
 

-    error = mach_make_memory_entry_64(theMap,
-           &actualSize, range0Addr, prot, &sharedMem, (ipc_port_t) _memEntry);
-
-    if (KERN_SUCCESS == error) {
-       if (actualSize == size) {
-           return sharedMem;
-       } else {
-#if IOASSERT
-           IOLog("IOGMD::mach_make_memory_entry_64 (%08llx) size (%08lx:%08x)\n",
-                       (UInt64)range0Addr, (UInt32)actualSize, size);
-#endif    
-           ipc_port_release_send( sharedMem );
+               if (count >= ref->capacity)
+               {
+                   ref = memoryReferenceAlloc(ref->capacity + kCapacityInc, ref);
+                   entries = &ref->entries[count];
+               }
+               entries->entry  = entry;
+               entries->size   = actualSize;
+               entries->offset = offset + (entryAddr - srcAddr);
+               entryAddr += actualSize;
+               if (MAP_MEM_NAMED_REUSE & prot)
+               {
+                   if ((cloneEntries->entry  == entries->entry)
+                    && (cloneEntries->size   == entries->size)
+                    && (cloneEntries->offset == entries->offset))         cloneEntries++;
+                    else                                    prot &= ~MAP_MEM_NAMED_REUSE;
+               }
+               entries++;
+               count++;
+           }
+           while (true);
+           offset += srcLen;
+           remain -= srcLen;

        }
     }
        }
     }

+    else
+    {
+       // _task == 0, physical
+       memory_object_t pager;
+        vm_size_t       size = ptoa_32(_pages);

 
 

-    return MACH_PORT_NULL;
-}
+       if (!getKernelReserved()) panic("getKernelReserved");

 
 

-/*
- * initWithAddress:
- *
- * Initialize an IOMemoryDescriptor. The buffer is a virtual address
- * relative to the specified task.  If no task is supplied, the kernel
- * task is implied.
- *
- * An IOMemoryDescriptor can be re-used by calling initWithAddress or
- * initWithRanges again on an existing instance -- note this behavior
- * is not commonly supported in other I/O Kit classes, although it is
- * supported here.
- */
-bool
-IOGeneralMemoryDescriptor::initWithAddress(void *      address,
-                                    IOByteCount   withLength,
-                                    IODirection withDirection)
-{
-    _singleRange.v.address = (vm_address_t) address;
-    _singleRange.v.length  = withLength;
+       reserved->dp.pagerContig = (1 == _rangesCount);
+       reserved->dp.memory      = this;

 
 

-    return initWithRanges(&_singleRange.v, 1, withDirection, kernel_task, true);
-}
+       pagerFlags = pagerFlagsForCacheMode(cacheMode);
+       if (-1U == pagerFlags) panic("phys is kIODefaultCache");
+       if (reserved->dp.pagerContig) pagerFlags |= DEVICE_PAGER_CONTIGUOUS;

 
 

-bool
-IOGeneralMemoryDescriptor::initWithAddress(vm_address_t address,
-                                    IOByteCount    withLength,
-                                    IODirection  withDirection,
-                                    task_t       withTask)
-{
-    _singleRange.v.address = address;
-    _singleRange.v.length  = withLength;
+       pager = device_pager_setup((memory_object_t) 0, (uintptr_t) reserved, 
+                                                           size, pagerFlags);
+       assert (pager);
+       if (!pager) err = kIOReturnVMError;
+       else
+       {
+           srcAddr  = nextAddr;
+           entryAddr = trunc_page_64(srcAddr);
+           err = mach_memory_object_memory_entry_64((host_t) 1, false /*internal*/, 
+                       size, VM_PROT_READ | VM_PROT_WRITE, pager, &entry);
+           assert (KERN_SUCCESS == err);
+           if (KERN_SUCCESS != err) device_pager_deallocate(pager);
+           else
+           {
+               reserved->dp.devicePager = pager;
+               entries->entry  = entry;
+               entries->size   = size;
+               entries->offset = offset + (entryAddr - srcAddr);
+               entries++;
+               count++;
+           }
+       }
+    }
+    
+    ref->count = count;
+    ref->prot  = prot;

 
 

-    return initWithRanges(&_singleRange.v, 1, withDirection, withTask, true);
-}
+    if (KERN_SUCCESS == err)
+    {
+       if (MAP_MEM_NAMED_REUSE & prot)
+       {
+           memoryReferenceFree(ref);
+           OSIncrementAtomic(&_memRef->refCount);
+           ref = _memRef;
+       }
+    }
+    else
+    {
+        memoryReferenceFree(ref);
+        ref = NULL;    
+    }

 
 

-bool
-IOGeneralMemoryDescriptor::initWithPhysicalAddress(
-                                IOPhysicalAddress      address,
-                                IOByteCount            withLength,
-                                IODirection            withDirection )
-{
-    _singleRange.p.address = address;
-    _singleRange.p.length  = withLength;
+    *reference = ref;

 
 

-    return initWithPhysicalRanges( &_singleRange.p, 1, withDirection, true);
+    return (err);

 }
 
 }
 

-bool
-IOGeneralMemoryDescriptor::initWithPhysicalRanges(
-                                IOPhysicalRange * ranges,
-                                UInt32            count,
-                                IODirection       direction,
-                                bool              reference)
+struct IOMemoryDescriptorMapAllocRef

 {
 {

-    IOOptionBits mdOpts = direction | kIOMemoryTypePhysical;
-
-    if (reference)
-        mdOpts |= kIOMemoryAsReference;
-
-    return initWithOptions(ranges, count, 0, 0, mdOpts, /* mapper */ 0);
-}
+    vm_map_t          map;
+    mach_vm_address_t mapped;
+    mach_vm_size_t    size;
+    vm_prot_t         prot;
+    IOOptionBits      options;
+};

 
 

-bool
-IOGeneralMemoryDescriptor::initWithRanges(
-                                   IOVirtualRange * ranges,
-                                   UInt32           count,
-                                   IODirection      direction,
-                                   task_t           task,
-                                   bool             reference)
+static kern_return_t 
+IOMemoryDescriptorMapAlloc(vm_map_t map, void * _ref)

 {
 {

-    IOOptionBits mdOpts = direction;
-
-    if (reference)
-        mdOpts |= kIOMemoryAsReference;
-
-    if (task) {
-        mdOpts |= kIOMemoryTypeVirtual;
-
-       // Auto-prepare if this is a kernel memory descriptor as very few
-       // clients bother to prepare() kernel memory.
-       // But it was not  enforced so what are you going to do?
-        if (task == kernel_task)
-            mdOpts |= kIOMemoryAutoPrepare;
+    IOMemoryDescriptorMapAllocRef * ref = (typeof(ref))_ref;
+    IOReturn                       err;
+    vm_map_offset_t                addr;
+
+    addr = ref->mapped;
+    err = vm_map_enter_mem_object(map, &addr, ref->size,
+                                 (vm_map_offset_t) 0,
+                                 (((ref->options & kIOMapAnywhere)
+                                   ? VM_FLAGS_ANYWHERE
+                                   : VM_FLAGS_FIXED)
+                                  | VM_MAKE_TAG(VM_MEMORY_IOKIT)
+                                  | VM_FLAGS_IOKIT_ACCT), /* iokit accounting */
+                                 IPC_PORT_NULL,
+                                 (memory_object_offset_t) 0,
+                                 false, /* copy */
+                                 ref->prot,
+                                 ref->prot,
+                                 VM_INHERIT_NONE);
+    if (KERN_SUCCESS == err)
+    {
+       ref->mapped = (mach_vm_address_t) addr;
+       ref->map = map;

     }
     }

-    else
-        mdOpts |= kIOMemoryTypePhysical;
-    
-    return initWithOptions(ranges, count, 0, task, mdOpts, /* mapper */ 0);
-}

 
 

-/*
- * initWithOptions:
- *
- *  IOMemoryDescriptor. The buffer is made up of several virtual address ranges,
- * from a given task, several physical ranges, an UPL from the ubc
- * system or a uio (may be 64bit) from the BSD subsystem.
- *
- * Passing the ranges as a reference will avoid an extra allocation.
- *
- * An IOMemoryDescriptor can be re-used by calling initWithOptions again on an
- * existing instance -- note this behavior is not commonly supported in other
- * I/O Kit classes, although it is supported here.
- */
+    return( err );
+}

 
 

-bool
-IOGeneralMemoryDescriptor::initWithOptions(void *      buffers,
-                                           UInt32      count,
-                                           UInt32      offset,
-                                           task_t      task,
-                                           IOOptionBits        options,
-                                           IOMapper *  mapper)
+IOReturn 
+IOGeneralMemoryDescriptor::memoryReferenceMap(
+                    IOMemoryReference * ref,
+                     vm_map_t            map,
+                     mach_vm_size_t      inoffset,
+                     mach_vm_size_t      size,
+                     IOOptionBits        options,
+                     mach_vm_address_t * inaddr)

 {
 {

-    IOOptionBits type = options & kIOMemoryTypeMask;
+    IOReturn        err;
+    int64_t         offset = inoffset;
+    uint32_t        rangeIdx, entryIdx;
+    vm_map_offset_t addr, mapAddr;
+    vm_map_offset_t pageOffset, entryOffset, remain, chunk;
+
+    mach_vm_address_t srcAddr, nextAddr;
+    mach_vm_size_t    srcLen, nextLen;
+    IOByteCount       physLen;
+    IOMemoryEntry   * entry;
+    vm_prot_t         prot, memEntryCacheMode;
+    IOOptionBits      type;
+    IOOptionBits      cacheMode;

 
 

-    // Grab the original MD's configuation data to initialse the
-    // arguments to this function.
-    if (kIOMemoryTypePersistentMD == type) {
+    /*
+     * For the kIOMapPrefault option.
+     */
+    upl_page_info_t *pageList = NULL;
+    UInt currentPageIndex = 0;

 
 

-       typePersMDData *initData = (typePersMDData *) buffers;
-       const IOGeneralMemoryDescriptor *orig = initData->fMD;
-       ioGMDData *dataP = getDataP(orig->_memoryEntries);
+    type = _flags & kIOMemoryTypeMask;
+    prot = VM_PROT_READ;
+    if (!(kIOMapReadOnly & options)) prot |= VM_PROT_WRITE;
+    prot &= ref->prot;

 
 

-       // Only accept persistent memory descriptors with valid dataP data.
-       assert(orig->_rangesCount == 1);
-       if ( !(orig->_flags & kIOMemoryPersistent) || !dataP)
-           return false;
+    cacheMode = ((options & kIOMapCacheMask) >> kIOMapCacheShift);
+    if (kIODefaultCache != cacheMode)
+    {
+       // VM system requires write access to update named entry cache mode
+       memEntryCacheMode = (MAP_MEM_ONLY | VM_PROT_WRITE | prot | vmProtForCacheMode(cacheMode));
+    }

 
 

-       _memEntry = initData->fMemEntry;        // Grab the new named entry
-       options = orig->_flags | kIOMemoryAsReference; 
-       _singleRange = orig->_singleRange;      // Initialise our range
-       buffers = &_singleRange;
-       count = 1;
+    if (_task)
+    {
+       // Find first range for offset
+       for (remain = offset, rangeIdx = 0; rangeIdx < _rangesCount; rangeIdx++)
+       {
+           getAddrLenForInd(nextAddr, nextLen, type, _ranges, rangeIdx);
+           if (remain < nextLen) break;
+           remain -= nextLen;
+       } 
+    }
+    else
+    {
+        rangeIdx = 0;
+        remain   = 0;
+        nextAddr = getPhysicalSegment(offset, &physLen, kIOMemoryMapperNone);
+        nextLen  = size;
+    }

 
 

-       // Now grab the original task and whatever mapper was previously used
-       task = orig->_task;
-       mapper = dataP->fMapper;
+    assert(remain < nextLen);
+    if (remain >= nextLen) return (kIOReturnBadArgument);

 
 

-       // We are ready to go through the original initialisation now
+    nextAddr  += remain;
+    nextLen   -= remain;
+    pageOffset = (page_mask & nextAddr);
+    addr = 0;
+    if (!(options & kIOMapAnywhere))
+    {
+        addr = *inaddr;
+        if (pageOffset != (page_mask & addr)) return (kIOReturnNotAligned);
+        addr -= pageOffset;

     }
 
     }
 

-    switch (type) {
-    case kIOMemoryTypeUIO:
-    case kIOMemoryTypeVirtual:
-    case kIOMemoryTypeVirtual64:
-        assert(task);
-        if (!task)
-            return false;
-        else
-            break;
+    // find first entry for offset
+    for (entryIdx = 0; 
+       (entryIdx < ref->count) && (offset >= ref->entries[entryIdx].offset);
+       entryIdx++) {}
+    entryIdx--;
+    entry = &ref->entries[entryIdx];

 
 

-    case kIOMemoryTypePhysical:                // Neither Physical nor UPL should have a task
-    case kIOMemoryTypePhysical64:
-       mapper = kIOMapperNone;
+    // allocate VM
+    size = round_page_64(size + pageOffset);
+    {
+       IOMemoryDescriptorMapAllocRef ref;
+       ref.map     = map;
+       ref.options = options;
+       ref.size    = size;
+       ref.prot    = prot;
+       if (options & kIOMapAnywhere)
+           // vm_map looks for addresses above here, even when VM_FLAGS_ANYWHERE
+           ref.mapped = 0;
+       else
+           ref.mapped = addr;

 
 

-    case kIOMemoryTypeUPL:
-        assert(!task);
-        break;
-    default:
-        return false;  /* bad argument */
+       if ((ref.map == kernel_map) && (kIOMemoryBufferPageable & _flags))
+           err = IOIteratePageableMaps( ref.size, &IOMemoryDescriptorMapAlloc, &ref );
+       else
+           err = IOMemoryDescriptorMapAlloc(ref.map, &ref);
+       if (KERN_SUCCESS == err)
+       {
+           addr = ref.mapped;
+           map  = ref.map;
+       }

     }
 
     }
 

-    assert(buffers);
-    assert(count);
-

     /*
     /*

-     * We can check the _initialized  instance variable before having ever set
-     * it to an initial value because I/O Kit guarantees that all our instance
-     * variables are zeroed on an object's allocation.
+     * Prefaulting is only possible if we wired the memory earlier. Check the
+     * memory type, and the underlying data.

      */
      */

-
-    if (_initialized) {
+    if (options & kIOMapPrefault) {

         /*
         /*

-         * An existing memory descriptor is being retargeted to point to
-         * somewhere else.  Clean up our present state.
+         * The memory must have been wired by calling ::prepare(), otherwise
+         * we don't have the UPL. Without UPLs, pages cannot be pre-faulted
+         */
+        assert(map != kernel_map);
+        assert(_wireCount != 0);
+        assert(_memoryEntries != NULL);
+        if ((map == kernel_map) ||
+            (_wireCount == 0) ||
+            (_memoryEntries == NULL))
+        {
+            return kIOReturnBadArgument;
+        }
+        
+        // Get the page list.
+        ioGMDData* dataP = getDataP(_memoryEntries);
+        ioPLBlock const* ioplList = getIOPLList(dataP);
+        pageList = getPageList(dataP);
+        
+        // Get the number of IOPLs.
+        UInt numIOPLs = getNumIOPL(_memoryEntries, dataP);
+        
+        /*
+         * Scan through the IOPL Info Blocks, looking for the first block containing
+         * the offset. The research will go past it, so we'll need to go back to the
+         * right range at the end.
+         */
+        UInt ioplIndex = 0;
+        while (ioplIndex < numIOPLs && offset >= ioplList[ioplIndex].fIOMDOffset)
+            ioplIndex++;
+        ioplIndex--;
+        
+        // Retrieve the IOPL info block.
+        ioPLBlock ioplInfo = ioplList[ioplIndex];
+            
+        /*
+         * For external UPLs, the fPageInfo points directly to the UPL's page_info_t
+         * array.

          */
          */

+        if (ioplInfo.fFlags & kIOPLExternUPL)
+            pageList = (upl_page_info_t*) ioplInfo.fPageInfo;
+        else
+            pageList = &pageList[ioplInfo.fPageInfo];
+        
+        // Rebase [offset] into the IOPL in order to looks for the first page index.
+        mach_vm_size_t offsetInIOPL = offset - ioplInfo.fIOMDOffset + ioplInfo.fPageOffset;
+        
+        // Retrieve the index of the first page corresponding to the offset.
+        currentPageIndex = atop_32(offsetInIOPL);
+    }

 
 

-        while (_wireCount)
-            complete();
-        if (_ranges.v && _rangesIsAllocated)
+    // enter mappings
+    remain  = size;
+    mapAddr = addr;
+    addr    += pageOffset;
+    while (remain && nextLen && (KERN_SUCCESS == err))
+    {
+       srcAddr  = nextAddr;
+       srcLen   = nextLen;
+       nextAddr = 0;
+       nextLen  = 0;
+       // coalesce addr range
+       for (++rangeIdx; rangeIdx < _rangesCount; rangeIdx++)

        {
        {

-           if (kIOMemoryTypeUIO == type)
-               uio_free((uio_t) _ranges.v);
-           else if ((kIOMemoryTypeVirtual64 == type) || (kIOMemoryTypePhysical64 == type))
-               IODelete(_ranges.v64, IOAddressRange, _rangesCount);
-           else
-               IODelete(_ranges.v, IOVirtualRange, _rangesCount);
+           getAddrLenForInd(nextAddr, nextLen, type, _ranges, rangeIdx);
+           if ((srcAddr + srcLen) != nextAddr) break;
+           srcLen += nextLen;

        }
        }

-       if (_memEntry)
-           { ipc_port_release_send((ipc_port_t) _memEntry); _memEntry = 0; }
-    }
-    else {
-        if (!super::init())
-            return false;
-        _initialized = true;
-    }
-
-    // Grab the appropriate mapper
-    if (mapper == kIOMapperNone)
-        mapper = 0;    // No Mapper
-    else if (mapper == kIOMapperSystem) {
-        IOMapper::checkForSystemMapper();
-        gIOSystemMapper = mapper = IOMapper::gSystem;
-    }

 
 

-    // Remove the dynamic internal use flags from the initial setting
-    options              &= ~(kIOMemoryPreparedReadOnly);
-    _flags                = options;
-    _task                  = task;
+        while (srcLen && (KERN_SUCCESS == err))
+        {
+            entryOffset = offset - entry->offset;
+            if ((page_mask & entryOffset) != pageOffset) 
+            {
+                err = kIOReturnNotAligned;
+                break;
+            }

 
 

-    // DEPRECATED variable initialisation
-    _direction             = (IODirection) (_flags & kIOMemoryDirectionMask);
+           if (kIODefaultCache != cacheMode)
+           {
+               vm_size_t unused = 0;
+               err = mach_make_memory_entry(NULL /*unused*/, &unused, 0 /*unused*/, 
+                                            memEntryCacheMode, NULL, entry->entry);
+               assert (KERN_SUCCESS == err);
+           }

 
 

-    __iomd_reservedA = 0;
-    __iomd_reservedB = 0;
-    __iomd_reservedC = 0;
+            entryOffset -= pageOffset;
+            if (entryOffset >= entry->size) panic("entryOffset");
+            chunk = entry->size - entryOffset;
+            if (chunk)
+            {
+                if (chunk > remain) chunk = remain;
+
+                if (options & kIOMapPrefault) {
+                    UInt nb_pages = round_page(chunk) / PAGE_SIZE;
+                    err = vm_map_enter_mem_object_prefault(map,
+                                                           &mapAddr,
+                                                           chunk, 0 /* mask */, 
+                                                            (VM_FLAGS_FIXED
+                                                           | VM_FLAGS_OVERWRITE
+                                                           | VM_MAKE_TAG(VM_MEMORY_IOKIT)
+                                                           | VM_FLAGS_IOKIT_ACCT), /* iokit accounting */
+                                                           entry->entry,
+                                                           entryOffset,
+                                                           prot, // cur
+                                                           prot, // max
+                                                           &pageList[currentPageIndex],
+                                                          nb_pages);
+
+                    // Compute the next index in the page list.
+                    currentPageIndex += nb_pages;
+                    assert(currentPageIndex <= _pages);
+                } else {
+                    err = vm_map_enter_mem_object(map,
+                                                  &mapAddr,
+                                                  chunk, 0 /* mask */, 
+                                                   (VM_FLAGS_FIXED
+                                                  | VM_FLAGS_OVERWRITE
+                                                  | VM_MAKE_TAG(VM_MEMORY_IOKIT)
+                                                  | VM_FLAGS_IOKIT_ACCT), /* iokit accounting */
+                                                  entry->entry,
+                                                  entryOffset,
+                                                  false, // copy
+                                                  prot, // cur
+                                                  prot, // max
+                                                  VM_INHERIT_NONE);
+                }

 
 

-    _highestPage = 0;
+                if (KERN_SUCCESS != err) break;
+                remain -= chunk;
+                if (!remain) break;
+                mapAddr  += chunk;
+                offset   += chunk - pageOffset;
+            }
+            pageOffset = 0;
+            entry++;
+            entryIdx++;
+            if (entryIdx >= ref->count) 
+            {
+                err = kIOReturnOverrun;
+                break;
+            }
+        }
+    }

 
 

-    if (kIOMemoryTypeUPL == type) {
+    if ((KERN_SUCCESS != err) && addr)
+    {
+        (void) mach_vm_deallocate(map, trunc_page_64(addr), size);
+        addr = 0;
+    }
+    *inaddr = addr;

 
 

-        ioGMDData *dataP;
-        unsigned int dataSize = computeDataSize(/* pages */ 0, /* upls */ 1);
+    return (err);
+}

 
 

-        if (!_memoryEntries) {
-            _memoryEntries = OSData::withCapacity(dataSize);
-            if (!_memoryEntries)
-                return false;
-        }
-        else if (!_memoryEntries->initWithCapacity(dataSize))
-            return false;
+IOReturn 
+IOGeneralMemoryDescriptor::memoryReferenceGetPageCounts(
+                              IOMemoryReference * ref,
+                               IOByteCount       * residentPageCount,
+                               IOByteCount       * dirtyPageCount)
+{
+    IOReturn        err;
+    IOMemoryEntry * entries;
+    unsigned int resident, dirty;
+    unsigned int totalResident, totalDirty;
+
+    totalResident = totalDirty = 0;
+    entries = ref->entries + ref->count;
+    while (entries > &ref->entries[0])
+    {
+        entries--;
+       err = mach_memory_entry_get_page_counts(entries->entry, &resident, &dirty);
+       if (KERN_SUCCESS != err) break;
+       totalResident += resident;
+       totalDirty    += dirty;
+    }

 
 

-        _memoryEntries->appendBytes(0, sizeof(ioGMDData));
-        dataP = getDataP(_memoryEntries);
-        dataP->fMapper = mapper;
-        dataP->fPageCnt = 0;
+    if (residentPageCount) *residentPageCount = totalResident;
+    if (dirtyPageCount)    *dirtyPageCount    = totalDirty;
+    return (err);
+}

 
 

- //       _wireCount++;        // UPLs start out life wired
+IOReturn
+IOGeneralMemoryDescriptor::memoryReferenceSetPurgeable(
+                               IOMemoryReference * ref,
+                               IOOptionBits        newState,
+                               IOOptionBits      * oldState)
+{
+    IOReturn        err;
+    IOMemoryEntry * entries;
+    vm_purgable_t   control;
+    int             totalState, state;
+
+    entries = ref->entries + ref->count;
+    totalState = kIOMemoryPurgeableNonVolatile;
+    while (entries > &ref->entries[0])
+    {
+        entries--;
+
+       err = purgeableControlBits(newState, &control, &state);
+       if (KERN_SUCCESS != err) break;
+       err = mach_memory_entry_purgable_control(entries->entry, control, &state);
+       if (KERN_SUCCESS != err) break;
+       err = purgeableStateBits(&state);
+       if (KERN_SUCCESS != err) break;
+
+       if (kIOMemoryPurgeableEmpty == state)              totalState = kIOMemoryPurgeableEmpty;
+       else if (kIOMemoryPurgeableEmpty == totalState)    continue;
+       else if (kIOMemoryPurgeableVolatile == totalState) continue;
+       else if (kIOMemoryPurgeableVolatile == state)      totalState = kIOMemoryPurgeableVolatile;
+       else totalState = kIOMemoryPurgeableNonVolatile;
+    }

 
 

-        _length    = count;
-        _pages    += atop_32(offset + count + PAGE_MASK) - atop_32(offset);
+    if (oldState) *oldState = totalState;
+    return (err);
+}

 
 

-        ioPLBlock iopl;
-        upl_page_info_t *pageList = UPL_GET_INTERNAL_PAGE_LIST((upl_t) buffers);
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

 
 

-        iopl.fIOPL = (upl_t) buffers;
-        // Set the flag kIOPLOnDevice convieniently equal to 1
-        iopl.fFlags  = pageList->device | kIOPLExternUPL;
-        iopl.fIOMDOffset = 0;
+IOMemoryDescriptor *
+IOMemoryDescriptor::withAddress(void *      address,
+                                IOByteCount   length,
+                                IODirection direction)
+{
+    return IOMemoryDescriptor::
+        withAddressRange((IOVirtualAddress) address, length, direction | kIOMemoryAutoPrepare, kernel_task);
+}

 
 

-        _highestPage = upl_get_highest_page(iopl.fIOPL);
+#ifndef __LP64__
+IOMemoryDescriptor *
+IOMemoryDescriptor::withAddress(IOVirtualAddress address,
+                                IOByteCount  length,
+                                IODirection  direction,
+                                task_t       task)
+{
+    IOGeneralMemoryDescriptor * that = new IOGeneralMemoryDescriptor;
+    if (that)
+    {
+       if (that->initWithAddress(address, length, direction, task))
+           return that;

 
 

-        if (!pageList->device) {
-            // Pre-compute the offset into the UPL's page list
-            pageList = &pageList[atop_32(offset)];
-            offset &= PAGE_MASK;
-            if (mapper) {
-                iopl.fMappedBase = mapper->iovmAlloc(_pages);
-                mapper->iovmInsert(iopl.fMappedBase, 0, pageList, _pages);
-            }
-           else
-               iopl.fMappedBase = 0;
-        }
-       else
-           iopl.fMappedBase = 0;
-        iopl.fPageInfo = (vm_address_t) pageList;
-        iopl.fPageOffset = offset;
-
-        _memoryEntries->appendBytes(&iopl, sizeof(iopl));
+        that->release();

     }
     }

-    else {
-       // kIOMemoryTypeVirtual  | kIOMemoryTypeVirtual64 | kIOMemoryTypeUIO 
-       // kIOMemoryTypePhysical | kIOMemoryTypePhysical64
-       
-       // Initialize the memory descriptor
-       if (options & kIOMemoryAsReference) {
-           _rangesIsAllocated = false;
-
-           // Hack assignment to get the buffer arg into _ranges.
-           // I'd prefer to do _ranges = (Ranges) buffers, but that doesn't
-           // work, C++ sigh.
-           // This also initialises the uio & physical ranges.
-           _ranges.v = (IOVirtualRange *) buffers;
-       }
-       else {
-           _rangesIsAllocated = true;
-           switch (_flags & kIOMemoryTypeMask)
-           {
-             case kIOMemoryTypeUIO:
-               _ranges.v = (IOVirtualRange *) uio_duplicate((uio_t) buffers);
-               break;
-
-             case kIOMemoryTypeVirtual64:
-             case kIOMemoryTypePhysical64:
-               _ranges.v64 = IONew(IOAddressRange, count);
-               if (!_ranges.v64)
-                   return false;
-               bcopy(buffers, _ranges.v, count * sizeof(IOAddressRange));
-               break;
-             case kIOMemoryTypeVirtual:
-               _ranges.v = IONew(IOVirtualRange, count);
-               if (!_ranges.v)
-                   return false;
-               bcopy(buffers, _ranges.v, count * sizeof(IOVirtualRange));
-               break;
-           }
-       } 
+    return 0;
+}
+#endif /* !__LP64__ */

 
 

-       // Find starting address within the vector of ranges
-       Ranges vec = _ranges;
-       UInt32 length = 0;
-       UInt32 pages = 0;
-       for (unsigned ind = 0; ind < count;  ind++) {
-           user_addr_t addr;
-           UInt32 len;
+IOMemoryDescriptor *
+IOMemoryDescriptor::withPhysicalAddress(
+                               IOPhysicalAddress       address,
+                               IOByteCount             length,
+                               IODirection             direction )
+{
+    return (IOMemoryDescriptor::withAddressRange(address, length, direction, TASK_NULL));
+}

 
 

-           // addr & len are returned by this function
-           getAddrLenForInd(addr, len, type, vec, ind);
-           pages += (atop_64(addr + len + PAGE_MASK) - atop_64(addr));
-           len += length;
-           assert(len >= length);      // Check for 32 bit wrap around
-           length = len;
+#ifndef __LP64__
+IOMemoryDescriptor *
+IOMemoryDescriptor::withRanges(        IOVirtualRange * ranges,
+                               UInt32           withCount,
+                               IODirection      direction,
+                               task_t           task,
+                               bool             asReference)
+{
+    IOGeneralMemoryDescriptor * that = new IOGeneralMemoryDescriptor;
+    if (that)
+    {
+       if (that->initWithRanges(ranges, withCount, direction, task, asReference))
+           return that;

 
 

-           if ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type))
-           {
-               ppnum_t highPage = atop_64(addr + len - 1);
-               if (highPage > _highestPage)
-                   _highestPage = highPage;
-           }
-       } 
-       _length      = length;
-       _pages       = pages;
-       _rangesCount = count;
+        that->release();
+    }
+    return 0;
+}
+#endif /* !__LP64__ */

 
 

-        // Auto-prepare memory at creation time.
-        // Implied completion when descriptor is free-ed
-        if ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type))
-            _wireCount++;      // Physical MDs are, by definition, wired
-        else { /* kIOMemoryTypeVirtual | kIOMemoryTypeVirtual64 | kIOMemoryTypeUIO */
-            ioGMDData *dataP;
-            unsigned dataSize = computeDataSize(_pages, /* upls */ count * 2);
+IOMemoryDescriptor *
+IOMemoryDescriptor::withAddressRange(mach_vm_address_t address,
+                                       mach_vm_size_t length,
+                                       IOOptionBits   options,
+                                       task_t         task)
+{
+    IOAddressRange range = { address, length };
+    return (IOMemoryDescriptor::withAddressRanges(&range, 1, options, task));
+}

 
 

-            if (!_memoryEntries) {
-                _memoryEntries = OSData::withCapacity(dataSize);
-                if (!_memoryEntries)
-                   return false;
-            }
-            else if (!_memoryEntries->initWithCapacity(dataSize))
-                return false;
-    
-            _memoryEntries->appendBytes(0, sizeof(ioGMDData));
-            dataP = getDataP(_memoryEntries);
-            dataP->fMapper = mapper;
-            dataP->fPageCnt = _pages;
+IOMemoryDescriptor *
+IOMemoryDescriptor::withAddressRanges(IOAddressRange *   ranges,
+                                       UInt32           rangeCount,
+                                       IOOptionBits     options,
+                                       task_t           task)
+{
+    IOGeneralMemoryDescriptor * that = new IOGeneralMemoryDescriptor;
+    if (that)
+    {
+       if (task)
+           options |= kIOMemoryTypeVirtual64;
+       else
+           options |= kIOMemoryTypePhysical64;

 
 

-           if ( (kIOMemoryPersistent & _flags) && !_memEntry)
-               _memEntry = createNamedEntry();
+       if (that->initWithOptions(ranges, rangeCount, 0, task, options, /* mapper */ 0))
+           return that;

 
 

-            if ((_flags & kIOMemoryAutoPrepare)
-             && prepare() != kIOReturnSuccess)
-                return false;
-        }
+       that->release();

     }
 
     }
 

-    return true;
+    return 0;

 }
 
 }
 

+

 /*
 /*

- * free
+ * withOptions:

  *
  *

- * Free resources.
+ * Create a new IOMemoryDescriptor. The buffer is made up of several
+ * virtual address ranges, from a given task.
+ *
+ * Passing the ranges as a reference will avoid an extra allocation.

  */
  */

-void IOGeneralMemoryDescriptor::free()
+IOMemoryDescriptor *
+IOMemoryDescriptor::withOptions(void *         buffers,
+                                UInt32         count,
+                                UInt32         offset,
+                                task_t         task,
+                                IOOptionBits   opts,
+                                IOMapper *     mapper)

 {
 {

-    LOCK;
-    if( reserved)
-       reserved->memory = 0;
-    UNLOCK;
-
-    while (_wireCount)
-        complete();
-    if (_memoryEntries)
-        _memoryEntries->release();
+    IOGeneralMemoryDescriptor *self = new IOGeneralMemoryDescriptor;

 
 

-    if (_ranges.v && _rangesIsAllocated)
+    if (self
+    && !self->initWithOptions(buffers, count, offset, task, opts, mapper))

     {
     {

-       IOOptionBits type = _flags & kIOMemoryTypeMask;
-       if (kIOMemoryTypeUIO == type)
-           uio_free((uio_t) _ranges.v);
-       else if ((kIOMemoryTypeVirtual64 == type) || (kIOMemoryTypePhysical64 == type))
-           IODelete(_ranges.v64, IOAddressRange, _rangesCount);
-       else
-           IODelete(_ranges.v, IOVirtualRange, _rangesCount);
+        self->release();
+        return 0;

     }
 
     }
 

-    if (reserved && reserved->devicePager)
-       device_pager_deallocate( (memory_object_t) reserved->devicePager );
-
-    // memEntry holds a ref on the device pager which owns reserved
-    // (ExpansionData) so no reserved access after this point
-    if (_memEntry)
-        ipc_port_release_send( (ipc_port_t) _memEntry );
-
-    super::free();
+    return self;

 }
 
 }
 

-/* DEPRECATED */ void IOGeneralMemoryDescriptor::unmapFromKernel()
-/* DEPRECATED */ {
-                    panic("IOGMD::unmapFromKernel deprecated");
-/* DEPRECATED */ }
-/* DEPRECATED */ 
-/* DEPRECATED */ void IOGeneralMemoryDescriptor::mapIntoKernel(unsigned rangeIndex)
-/* DEPRECATED */ {
-                    panic("IOGMD::mapIntoKernel deprecated");
-/* DEPRECATED */ }
-
-/*
- * getDirection:
- *
- * Get the direction of the transfer.
- */
-IODirection IOMemoryDescriptor::getDirection() const
+bool IOMemoryDescriptor::initWithOptions(void *                buffers,
+                                         UInt32                count,
+                                         UInt32                offset,
+                                         task_t                task,
+                                         IOOptionBits  options,
+                                         IOMapper *    mapper)

 {
 {

-    return _direction;
+    return( false );

 }
 
 }
 

-/*
- * getLength:
- *
- * Get the length of the transfer (over all ranges).
- */
-IOByteCount IOMemoryDescriptor::getLength() const
+#ifndef __LP64__
+IOMemoryDescriptor *
+IOMemoryDescriptor::withPhysicalRanges(        IOPhysicalRange * ranges,
+                                        UInt32          withCount,
+                                        IODirection     direction,
+                                        bool            asReference)

 {
 {

-    return _length;
+    IOGeneralMemoryDescriptor * that = new IOGeneralMemoryDescriptor;
+    if (that)
+    {
+       if (that->initWithPhysicalRanges(ranges, withCount, direction, asReference))
+           return that;
+
+        that->release();
+    }
+    return 0;

 }
 
 }
 

-void IOMemoryDescriptor::setTag( IOOptionBits tag )
+IOMemoryDescriptor *
+IOMemoryDescriptor::withSubRange(IOMemoryDescriptor *  of,
+                               IOByteCount             offset,
+                               IOByteCount             length,
+                               IODirection             direction)

 {
 {

-    _tag = tag;    
+    return (IOSubMemoryDescriptor::withSubRange(of, offset, length, direction | kIOMemoryThreadSafe));

 }
 }

+#endif /* !__LP64__ */

 
 

-IOOptionBits IOMemoryDescriptor::getTag( void )
+IOMemoryDescriptor *
+IOMemoryDescriptor::withPersistentMemoryDescriptor(IOMemoryDescriptor *originalMD)

 {
 {

-    return( _tag);
+    IOGeneralMemoryDescriptor *origGenMD = 
+       OSDynamicCast(IOGeneralMemoryDescriptor, originalMD);
+
+    if (origGenMD)
+       return IOGeneralMemoryDescriptor::
+           withPersistentMemoryDescriptor(origGenMD);
+    else
+       return 0;

 }
 
 }
 

-// @@@ gvdl: who is using this API?  Seems like a wierd thing to implement.
-IOPhysicalAddress
-IOMemoryDescriptor::getSourceSegment( IOByteCount   offset, IOByteCount * length )
+IOMemoryDescriptor *
+IOGeneralMemoryDescriptor::withPersistentMemoryDescriptor(IOGeneralMemoryDescriptor *originalMD)

 {
 {

-    addr64_t physAddr = 0;
+    IOMemoryReference * memRef;
+    
+    if (kIOReturnSuccess != originalMD->memoryReferenceCreate(kIOMemoryReferenceReuse, &memRef)) return (0);

 
 

-    if( prepare() == kIOReturnSuccess) {
-        physAddr = getPhysicalSegment64( offset, length );
-        complete();
+    if (memRef == originalMD->_memRef)
+    {
+       originalMD->retain();               // Add a new reference to ourselves
+        originalMD->memoryReferenceRelease(memRef);
+       return originalMD;

     }
 
     }
 

-    return( (IOPhysicalAddress) physAddr ); // truncated but only page offset is used
+    IOGeneralMemoryDescriptor * self = new IOGeneralMemoryDescriptor;
+    IOMDPersistentInitData initData = { originalMD, memRef };
+
+    if (self
+    && !self->initWithOptions(&initData, 1, 0, 0, kIOMemoryTypePersistentMD, 0)) {
+        self->release();
+       self = 0;
+    }
+    return self;

 }
 
 }
 

-IOByteCount IOMemoryDescriptor::readBytes
-                (IOByteCount offset, void *bytes, IOByteCount length)
+#ifndef __LP64__
+bool
+IOGeneralMemoryDescriptor::initWithAddress(void *      address,
+                                    IOByteCount   withLength,
+                                    IODirection withDirection)

 {
 {

-    addr64_t dstAddr = (addr64_t) (UInt32) bytes;
-    IOByteCount remaining;
+    _singleRange.v.address = (vm_offset_t) address;
+    _singleRange.v.length  = withLength;

 
 

-    // Assert that this entire I/O is withing the available range
-    assert(offset < _length);
-    assert(offset + length <= _length);
-    if (offset >= _length) {
-IOLog("IOGMD(%p): rB = o%lx, l%lx\n", this, offset, length);   // @@@ gvdl
-        return 0;
-    }
+    return initWithRanges(&_singleRange.v, 1, withDirection, kernel_task, true);
+}

 
 

-    remaining = length = min(length, _length - offset);
-    while (remaining) {        // (process another target segment?)
-        addr64_t       srcAddr64;
-        IOByteCount    srcLen;
+bool
+IOGeneralMemoryDescriptor::initWithAddress(IOVirtualAddress address,
+                                    IOByteCount    withLength,
+                                    IODirection  withDirection,
+                                    task_t       withTask)
+{
+    _singleRange.v.address = address;
+    _singleRange.v.length  = withLength;

 
 

-        srcAddr64 = getPhysicalSegment64(offset, &srcLen);
-        if (!srcAddr64)
-            break;
+    return initWithRanges(&_singleRange.v, 1, withDirection, withTask, true);
+}

 
 

-        // Clip segment length to remaining
-        if (srcLen > remaining)
-            srcLen = remaining;
+bool
+IOGeneralMemoryDescriptor::initWithPhysicalAddress(
+                                IOPhysicalAddress      address,
+                                IOByteCount            withLength,
+                                IODirection            withDirection )
+{
+    _singleRange.p.address = address;
+    _singleRange.p.length  = withLength;

 
 

-        copypv(srcAddr64, dstAddr, srcLen,
-                            cppvPsrc | cppvNoRefSrc | cppvFsnk | cppvKmap);
+    return initWithPhysicalRanges( &_singleRange.p, 1, withDirection, true);
+}

 
 

-        dstAddr   += srcLen;
-        offset    += srcLen;
-        remaining -= srcLen;
-    }
+bool
+IOGeneralMemoryDescriptor::initWithPhysicalRanges(
+                                IOPhysicalRange * ranges,
+                                UInt32            count,
+                                IODirection       direction,
+                                bool              reference)
+{
+    IOOptionBits mdOpts = direction | kIOMemoryTypePhysical;

 
 

-    assert(!remaining);
+    if (reference)
+        mdOpts |= kIOMemoryAsReference;

 
 

-    return length - remaining;
+    return initWithOptions(ranges, count, 0, 0, mdOpts, /* mapper */ 0);

 }
 
 }
 

-IOByteCount IOMemoryDescriptor::writeBytes
-                (IOByteCount offset, const void *bytes, IOByteCount length)
+bool
+IOGeneralMemoryDescriptor::initWithRanges(
+                                   IOVirtualRange * ranges,
+                                   UInt32           count,
+                                   IODirection      direction,
+                                   task_t           task,
+                                   bool             reference)

 {
 {

-    addr64_t srcAddr = (addr64_t) (UInt32) bytes;
-    IOByteCount remaining;
+    IOOptionBits mdOpts = direction;

 
 

-    // Assert that this entire I/O is withing the available range
-    assert(offset < _length);
-    assert(offset + length <= _length);
+    if (reference)
+        mdOpts |= kIOMemoryAsReference;

 
 

-    assert( !(kIOMemoryPreparedReadOnly & _flags) );
+    if (task) {
+        mdOpts |= kIOMemoryTypeVirtual;

 
 

-    if ( (kIOMemoryPreparedReadOnly & _flags) || offset >= _length) {
-IOLog("IOGMD(%p): wB = o%lx, l%lx\n", this, offset, length);   // @@@ gvdl
-        return 0;
+       // Auto-prepare if this is a kernel memory descriptor as very few
+       // clients bother to prepare() kernel memory.
+       // But it was not enforced so what are you going to do?
+        if (task == kernel_task)
+            mdOpts |= kIOMemoryAutoPrepare;

     }
     }

+    else
+        mdOpts |= kIOMemoryTypePhysical;
+    
+    return initWithOptions(ranges, count, 0, task, mdOpts, /* mapper */ 0);
+}
+#endif /* !__LP64__ */

 
 

-    remaining = length = min(length, _length - offset);
-    while (remaining) {        // (process another target segment?)
-        addr64_t    dstAddr64;
-        IOByteCount dstLen;
+/*
+ * initWithOptions:
+ *
+ *  IOMemoryDescriptor. The buffer is made up of several virtual address ranges,
+ * from a given task, several physical ranges, an UPL from the ubc
+ * system or a uio (may be 64bit) from the BSD subsystem.
+ *
+ * Passing the ranges as a reference will avoid an extra allocation.
+ *
+ * An IOMemoryDescriptor can be re-used by calling initWithOptions again on an
+ * existing instance -- note this behavior is not commonly supported in other
+ * I/O Kit classes, although it is supported here.
+ */

 
 

-        dstAddr64 = getPhysicalSegment64(offset, &dstLen);
-        if (!dstAddr64)
-            break;
+bool
+IOGeneralMemoryDescriptor::initWithOptions(void *      buffers,
+                                           UInt32      count,
+                                           UInt32      offset,
+                                           task_t      task,
+                                           IOOptionBits        options,
+                                           IOMapper *  mapper)
+{
+    IOOptionBits type = options & kIOMemoryTypeMask;

 
 

-        // Clip segment length to remaining
-        if (dstLen > remaining)
-            dstLen = remaining;
+#ifndef __LP64__
+    if (task
+        && (kIOMemoryTypeVirtual == type)
+        && vm_map_is_64bit(get_task_map(task)) 
+        && ((IOVirtualRange *) buffers)->address)
+    {
+        OSReportWithBacktrace("IOMemoryDescriptor: attempt to create 32b virtual in 64b task, use ::withAddressRange()");
+        return false;
+    }
+#endif /* !__LP64__ */

 
 

-        copypv(srcAddr, (addr64_t) dstAddr64, dstLen,
-                            cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
+    // Grab the original MD's configuation data to initialse the
+    // arguments to this function.
+    if (kIOMemoryTypePersistentMD == type) {

 
 

-        srcAddr   += dstLen;
-        offset    += dstLen;
-        remaining -= dstLen;
-    }
+       IOMDPersistentInitData *initData = (typeof(initData)) buffers;
+       const IOGeneralMemoryDescriptor *orig = initData->fMD;
+       ioGMDData *dataP = getDataP(orig->_memoryEntries);

 
 

-    assert(!remaining);
+       // Only accept persistent memory descriptors with valid dataP data.
+       assert(orig->_rangesCount == 1);
+       if ( !(orig->_flags & kIOMemoryPersistent) || !dataP)
+           return false;

 
 

-    return length - remaining;
-}
+       _memRef = initData->fMemRef;    // Grab the new named entry
+       options = orig->_flags & ~kIOMemoryAsReference; 
+        type = options & kIOMemoryTypeMask;
+       buffers = orig->_ranges.v;
+       count = orig->_rangesCount;

 
 

-// osfmk/device/iokit_rpc.c
-extern "C" unsigned int IODefaultCacheBits(addr64_t pa);
+       // Now grab the original task and whatever mapper was previously used
+       task = orig->_task;
+       mapper = dataP->fMapper;
+
+       // We are ready to go through the original initialisation now
+    }

 
 

-/* DEPRECATED */ void IOGeneralMemoryDescriptor::setPosition(IOByteCount position)
-/* DEPRECATED */ {
-                    panic("IOGMD::setPosition deprecated");
-/* DEPRECATED */ }
+    switch (type) {
+    case kIOMemoryTypeUIO:
+    case kIOMemoryTypeVirtual:
+#ifndef __LP64__
+    case kIOMemoryTypeVirtual64:
+#endif /* !__LP64__ */
+        assert(task);
+        if (!task)
+            return false;
+       break;

 
 

-IOReturn IOGeneralMemoryDescriptor::dmaCommandOperation(DMACommandOps op, void *vData, UInt dataSize) const
-{
-    if (kIOMDGetCharacteristics == op) {
+    case kIOMemoryTypePhysical:                // Neither Physical nor UPL should have a task
+#ifndef __LP64__
+    case kIOMemoryTypePhysical64:
+#endif /* !__LP64__ */
+    case kIOMemoryTypeUPL:
+        assert(!task);
+        break;
+    default:
+        return false;  /* bad argument */
+    }

 
 

-       if (dataSize < sizeof(IOMDDMACharacteristics))
-           return kIOReturnUnderrun;
+    assert(buffers);
+    assert(count);

 
 

-       IOMDDMACharacteristics *data = (IOMDDMACharacteristics *) vData;
-       data->fLength = _length;
-       data->fSGCount = _rangesCount;
-       data->fPages = _pages;
-       data->fDirection = _direction;
-       if (!_wireCount)
-           data->fIsPrepared = false;
-       else {
-           data->fIsPrepared = true;
-           data->fHighestPage = _highestPage;
-           if (_memoryEntries) {
-               ioGMDData *gmdData = getDataP(_memoryEntries);
-               ioPLBlock *ioplList = getIOPLList(gmdData);
-               UInt count = getNumIOPL(_memoryEntries, gmdData);
+    /*
+     * We can check the _initialized  instance variable before having ever set
+     * it to an initial value because I/O Kit guarantees that all our instance
+     * variables are zeroed on an object's allocation.
+     */

 
 

-               data->fIsMapped = (gmdData->fMapper && _pages && (count > 0)
-                              && ioplList[0].fMappedBase);
-               if (count == 1)
-                   data->fPageAlign = (ioplList[0].fPageOffset & PAGE_MASK) | ~PAGE_MASK;
-           }
+    if (_initialized) {
+        /*
+         * An existing memory descriptor is being retargeted to point to
+         * somewhere else.  Clean up our present state.
+         */
+       IOOptionBits type = _flags & kIOMemoryTypeMask;
+       if ((kIOMemoryTypePhysical != type) && (kIOMemoryTypePhysical64 != type))
+       {
+           while (_wireCount)
+               complete();
+       }
+        if (_ranges.v && !(kIOMemoryAsReference & _flags))
+       {
+           if (kIOMemoryTypeUIO == type)
+               uio_free((uio_t) _ranges.v);
+#ifndef __LP64__
+           else if ((kIOMemoryTypeVirtual64 == type) || (kIOMemoryTypePhysical64 == type))
+               IODelete(_ranges.v64, IOAddressRange, _rangesCount);
+#endif /* !__LP64__ */

            else
            else

-               data->fIsMapped = false;
+               IODelete(_ranges.v, IOVirtualRange, _rangesCount);

        }
 
        }
 

-       return kIOReturnSuccess;
+       options |= (kIOMemoryRedirected & _flags);
+       if (!(kIOMemoryRedirected & options))
+       {
+           if (_memRef)
+           {
+               memoryReferenceRelease(_memRef);
+               _memRef = 0;
+           }
+           if (_mappings)
+               _mappings->flushCollection();
+       }
+    }
+    else {
+        if (!super::init())
+            return false;
+        _initialized = true;

     }
     }

-    else if (!(kIOMDWalkSegments & op))
-       return kIOReturnBadArgument;
-
-    // Get the next segment
-    struct InternalState {
-       IOMDDMAWalkSegmentArgs fIO;
-       UInt fOffset2Index;
-       UInt fIndex;
-       UInt fNextOffset;
-    } *isP;

 
 

-    // Find the next segment
-    if (dataSize < sizeof(*isP))
-       return kIOReturnUnderrun;
+    // Grab the appropriate mapper
+    if (kIOMemoryHostOnly & options) options |= kIOMemoryMapperNone;
+    if (kIOMemoryMapperNone & options)
+        mapper = 0;    // No Mapper
+    else if (mapper == kIOMapperSystem) {
+        IOMapper::checkForSystemMapper();
+        gIOSystemMapper = mapper = IOMapper::gSystem;
+    }

 
 

-    isP = (InternalState *) vData;
-    UInt offset = isP->fIO.fOffset;
-    bool mapped = isP->fIO.fMapped;
+    // Temp binary compatibility for kIOMemoryThreadSafe
+    if (kIOMemoryReserved6156215 & options)
+    {
+       options &= ~kIOMemoryReserved6156215;
+       options |= kIOMemoryThreadSafe;
+    }
+    // Remove the dynamic internal use flags from the initial setting
+    options              &= ~(kIOMemoryPreparedReadOnly);
+    _flags                = options;
+    _task                  = task;

 
 

-    if (offset >= _length)
-       return (offset == _length)? kIOReturnOverrun : kIOReturnInternalError;
+#ifndef __LP64__
+    _direction             = (IODirection) (_flags & kIOMemoryDirectionMask);
+#endif /* !__LP64__ */

 
 

-    // Validate the previous offset
-    UInt ind, off2Ind = isP->fOffset2Index;
-    if ((kIOMDFirstSegment != op) 
-       && offset 
-       && (offset == isP->fNextOffset || off2Ind <= offset))
-       ind = isP->fIndex;
-    else
-       ind = off2Ind = 0;      // Start from beginning
+    __iomd_reservedA = 0;
+    __iomd_reservedB = 0;
+    _highestPage = 0;

 
 

-    UInt length;
-    UInt64 address;
-    if ( (_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical) {
+    if (kIOMemoryThreadSafe & options)
+    {
+       if (!_prepareLock)
+           _prepareLock = IOLockAlloc();
+    }
+    else if (_prepareLock)
+    {
+       IOLockFree(_prepareLock);
+       _prepareLock = NULL;
+    }
+       
+    if (kIOMemoryTypeUPL == type) {

 
 

-       // Physical address based memory descriptor
-       const IOPhysicalRange *physP = (IOPhysicalRange *) &_ranges.p[0];
+        ioGMDData *dataP;
+        unsigned int dataSize = computeDataSize(/* pages */ 0, /* upls */ 1);

 
 

-       // Find the range after the one that contains the offset
-       UInt len;
-       for (len = 0; off2Ind <= offset; ind++) {
-           len = physP[ind].length;
-           off2Ind += len;
-       }
+        if (!initMemoryEntries(dataSize, mapper)) return (false);
+        dataP = getDataP(_memoryEntries);
+        dataP->fPageCnt = 0;

 
 

-       // Calculate length within range and starting address
-       length   = off2Ind - offset;
-       address  = physP[ind - 1].address + len - length;
+ //       _wireCount++;        // UPLs start out life wired

 
 

-       // see how far we can coalesce ranges
-       while (ind < _rangesCount && address + length == physP[ind].address) {
-           len = physP[ind].length;
-           length += len;
-           off2Ind += len;
-           ind++;
+        _length    = count;
+        _pages    += atop_32(offset + count + PAGE_MASK) - atop_32(offset);
+
+        ioPLBlock iopl;
+        iopl.fIOPL = (upl_t) buffers;
+        upl_set_referenced(iopl.fIOPL, true);
+        upl_page_info_t *pageList = UPL_GET_INTERNAL_PAGE_LIST(iopl.fIOPL);
+
+       if (upl_get_size(iopl.fIOPL) < (count + offset))
+           panic("short external upl");
+
+        _highestPage = upl_get_highest_page(iopl.fIOPL);
+
+        // Set the flag kIOPLOnDevice convieniently equal to 1
+        iopl.fFlags  = pageList->device | kIOPLExternUPL;
+        if (!pageList->device) {
+            // Pre-compute the offset into the UPL's page list
+            pageList = &pageList[atop_32(offset)];
+            offset &= PAGE_MASK;
+        }
+        iopl.fIOMDOffset = 0;
+        iopl.fMappedPage = 0;
+        iopl.fPageInfo = (vm_address_t) pageList;
+        iopl.fPageOffset = offset;
+        _memoryEntries->appendBytes(&iopl, sizeof(iopl));
+    }
+    else {
+       // kIOMemoryTypeVirtual  | kIOMemoryTypeVirtual64 | kIOMemoryTypeUIO 
+       // kIOMemoryTypePhysical | kIOMemoryTypePhysical64
+       
+       // Initialize the memory descriptor
+       if (options & kIOMemoryAsReference) {
+#ifndef __LP64__
+           _rangesIsAllocated = false;
+#endif /* !__LP64__ */
+
+           // Hack assignment to get the buffer arg into _ranges.
+           // I'd prefer to do _ranges = (Ranges) buffers, but that doesn't
+           // work, C++ sigh.
+           // This also initialises the uio & physical ranges.
+           _ranges.v = (IOVirtualRange *) buffers;

        }
        }

+       else {
+#ifndef __LP64__
+           _rangesIsAllocated = true;
+#endif /* !__LP64__ */
+           switch (type)
+           {
+             case kIOMemoryTypeUIO:
+               _ranges.v = (IOVirtualRange *) uio_duplicate((uio_t) buffers);
+               break;

 
 

-       // correct contiguous check overshoot
-       ind--;
-       off2Ind -= len;
+#ifndef __LP64__
+             case kIOMemoryTypeVirtual64:
+             case kIOMemoryTypePhysical64:
+               if (count == 1
+                   && (((IOAddressRange *) buffers)->address + ((IOAddressRange *) buffers)->length) <= 0x100000000ULL
+                   ) {
+                   if (kIOMemoryTypeVirtual64 == type)
+                       type = kIOMemoryTypeVirtual;
+                   else
+                       type = kIOMemoryTypePhysical;
+                   _flags = (_flags & ~kIOMemoryTypeMask) | type | kIOMemoryAsReference;
+                   _rangesIsAllocated = false;
+                   _ranges.v = &_singleRange.v;
+                   _singleRange.v.address = ((IOAddressRange *) buffers)->address;
+                   _singleRange.v.length  = ((IOAddressRange *) buffers)->length;
+                   break;
+               }
+               _ranges.v64 = IONew(IOAddressRange, count);
+               if (!_ranges.v64)
+                   return false;
+               bcopy(buffers, _ranges.v, count * sizeof(IOAddressRange));
+               break;
+#endif /* !__LP64__ */
+             case kIOMemoryTypeVirtual:
+             case kIOMemoryTypePhysical:
+               if (count == 1) {
+                   _flags |= kIOMemoryAsReference;
+#ifndef __LP64__
+                   _rangesIsAllocated = false;
+#endif /* !__LP64__ */
+                   _ranges.v = &_singleRange.v;
+               } else {
+                   _ranges.v = IONew(IOVirtualRange, count);
+                   if (!_ranges.v)
+                       return false;
+               }
+               bcopy(buffers, _ranges.v, count * sizeof(IOVirtualRange));
+               break;
+           }
+       } 
+
+       // Find starting address within the vector of ranges
+       Ranges vec = _ranges;
+       UInt32 length = 0;
+       UInt32 pages = 0;
+       for (unsigned ind = 0; ind < count;  ind++) {
+           mach_vm_address_t addr;
+           mach_vm_size_t len;
+
+           // addr & len are returned by this function
+           getAddrLenForInd(addr, len, type, vec, ind);
+           pages += (atop_64(addr + len + PAGE_MASK) - atop_64(addr));
+           len += length;
+           assert(len >= length);      // Check for 32 bit wrap around
+           length = len;
+
+           if ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type))
+           {
+               ppnum_t highPage = atop_64(addr + len - 1);
+               if (highPage > _highestPage)
+                   _highestPage = highPage;
+           }
+       } 
+       _length      = length;
+       _pages       = pages;
+       _rangesCount = count;
+
+        // Auto-prepare memory at creation time.
+        // Implied completion when descriptor is free-ed
+        if ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type))
+            _wireCount++;      // Physical MDs are, by definition, wired
+        else { /* kIOMemoryTypeVirtual | kIOMemoryTypeVirtual64 | kIOMemoryTypeUIO */
+            ioGMDData *dataP;
+            unsigned dataSize = computeDataSize(_pages, /* upls */ count * 2);
+
+            if (!initMemoryEntries(dataSize, mapper)) return false;
+            dataP = getDataP(_memoryEntries);
+            dataP->fPageCnt = _pages;
+
+           if ( (kIOMemoryPersistent & _flags) && !_memRef)
+           {
+               IOReturn 
+               err = memoryReferenceCreate(0, &_memRef);
+               if (kIOReturnSuccess != err) return false;
+           }
+
+            if ((_flags & kIOMemoryAutoPrepare)
+             && prepare() != kIOReturnSuccess)
+                return false;
+        }

     }
     }

-    else if ( (_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical64) {

 
 

-       // Physical address based memory descriptor
-       const IOAddressRange *physP = (IOAddressRange *) &_ranges.v64[0];
+    return true;
+}

 
 

-       // Find the range after the one that contains the offset
-       mach_vm_size_t len;
-       for (len = 0; off2Ind <= offset; ind++) {
-           len = physP[ind].length;
-           off2Ind += len;
+/*
+ * free
+ *
+ * Free resources.
+ */
+void IOGeneralMemoryDescriptor::free()
+{
+    IOOptionBits type = _flags & kIOMemoryTypeMask;
+
+    if( reserved)
+    {
+       LOCK;
+       reserved->dp.memory = 0;
+       UNLOCK;
+    }
+    if ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type))
+    {
+       ioGMDData * dataP;
+       if (_memoryEntries && (dataP = getDataP(_memoryEntries)) && dataP->fMappedBase)
+       {
+           dataP->fMapper->iovmFree(atop_64(dataP->fMappedBase), _pages);
+           dataP->fMappedBase = 0;

        }
        }

+    }
+    else
+    {
+       while (_wireCount) complete();
+    }

 
 

-       // Calculate length within range and starting address
-       length   = off2Ind - offset;
-       address  = physP[ind - 1].address + len - length;
+    if (_memoryEntries) _memoryEntries->release();

 
 

-       // see how far we can coalesce ranges
-       while (ind < _rangesCount && address + length == physP[ind].address) {
-           len = physP[ind].length;
-           length += len;
-           off2Ind += len;
-           ind++;
-       }
+    if (_ranges.v && !(kIOMemoryAsReference & _flags))
+    {
+       if (kIOMemoryTypeUIO == type)
+           uio_free((uio_t) _ranges.v);
+#ifndef __LP64__
+       else if ((kIOMemoryTypeVirtual64 == type) || (kIOMemoryTypePhysical64 == type))
+           IODelete(_ranges.v64, IOAddressRange, _rangesCount);
+#endif /* !__LP64__ */
+       else
+           IODelete(_ranges.v, IOVirtualRange, _rangesCount);

 
 

-       // correct contiguous check overshoot
-       ind--;
-       off2Ind -= len;
+       _ranges.v = NULL;

     }
     }

-    else do {
-       if (!_wireCount)
-           panic("IOGMD: not wired for the IODMACommand");

 
 

-       assert(_memoryEntries);
+    if (reserved)
+    {
+        if (reserved->dp.devicePager)
+        {
+            // memEntry holds a ref on the device pager which owns reserved
+            // (IOMemoryDescriptorReserved) so no reserved access after this point
+            device_pager_deallocate( (memory_object_t) reserved->dp.devicePager );
+        }
+        else
+            IODelete(reserved, IOMemoryDescriptorReserved, 1);
+        reserved = NULL;
+    }

 
 

-       ioGMDData * dataP = getDataP(_memoryEntries);
-       const ioPLBlock *ioplList = getIOPLList(dataP);
-       UInt numIOPLs = getNumIOPL(_memoryEntries, dataP);
-       upl_page_info_t *pageList = getPageList(dataP);
+    if (_memRef)      memoryReferenceRelease(_memRef);
+    if (_prepareLock) IOLockFree(_prepareLock);

 
 

-       assert(numIOPLs > 0);
+    super::free();
+}

 
 

-       // Scan through iopl info blocks looking for block containing offset
-       while (ind < numIOPLs && offset >= ioplList[ind].fIOMDOffset)
-           ind++;
+#ifndef __LP64__
+void IOGeneralMemoryDescriptor::unmapFromKernel()
+{
+    panic("IOGMD::unmapFromKernel deprecated");
+}

 
 

-       // Go back to actual range as search goes past it
-       ioPLBlock ioplInfo = ioplList[ind - 1];
-       off2Ind = ioplInfo.fIOMDOffset;
+void IOGeneralMemoryDescriptor::mapIntoKernel(unsigned rangeIndex)
+{
+    panic("IOGMD::mapIntoKernel deprecated");
+}
+#endif /* !__LP64__ */

 
 

-       if (ind < numIOPLs)
-           length = ioplList[ind].fIOMDOffset;
-       else
-           length = _length;
-       length -= offset;                       // Remainder within iopl
+/*
+ * getDirection:
+ *
+ * Get the direction of the transfer.
+ */
+IODirection IOMemoryDescriptor::getDirection() const
+{
+#ifndef __LP64__
+    if (_direction)
+       return _direction;
+#endif /* !__LP64__ */
+    return (IODirection) (_flags & kIOMemoryDirectionMask);
+}

 
 

-       // Subtract offset till this iopl in total list
-       offset -= off2Ind;
+/*
+ * getLength:
+ *
+ * Get the length of the transfer (over all ranges).
+ */
+IOByteCount IOMemoryDescriptor::getLength() const
+{
+    return _length;
+}

 
 

-       // If a mapped address is requested and this is a pre-mapped IOPL
-       // then just need to compute an offset relative to the mapped base.
-       if (mapped && ioplInfo.fMappedBase) {
-           offset += (ioplInfo.fPageOffset & PAGE_MASK);
-           address = ptoa_64(ioplInfo.fMappedBase) + offset;
-           continue;   // Done leave do/while(false) now
-       }
+void IOMemoryDescriptor::setTag( IOOptionBits tag )
+{
+    _tag = tag;    
+}

 
 

-       // The offset is rebased into the current iopl.
-       // Now add the iopl 1st page offset.
-       offset += ioplInfo.fPageOffset;
+IOOptionBits IOMemoryDescriptor::getTag( void )
+{
+    return( _tag);
+}

 
 

-       // For external UPLs the fPageInfo field points directly to
-       // the upl's upl_page_info_t array.
-       if (ioplInfo.fFlags & kIOPLExternUPL)
-           pageList = (upl_page_info_t *) ioplInfo.fPageInfo;
-       else
-           pageList = &pageList[ioplInfo.fPageInfo];
+#ifndef __LP64__
+// @@@ gvdl: who is using this API?  Seems like a wierd thing to implement.
+IOPhysicalAddress
+IOMemoryDescriptor::getSourceSegment( IOByteCount   offset, IOByteCount * length )
+{
+    addr64_t physAddr = 0;

 
 

-       // Check for direct device non-paged memory
-       if ( ioplInfo.fFlags & kIOPLOnDevice ) {
-           address = ptoa_64(pageList->phys_addr) + offset;
-           continue;   // Done leave do/while(false) now
-       }
+    if( prepare() == kIOReturnSuccess) {
+        physAddr = getPhysicalSegment64( offset, length );
+        complete();
+    }

 
 

-       // Now we need compute the index into the pageList
-       UInt pageInd = atop_32(offset);
-       offset &= PAGE_MASK;
+    return( (IOPhysicalAddress) physAddr ); // truncated but only page offset is used
+}
+#endif /* !__LP64__ */

 
 

-       // Compute the starting address of this segment
-       IOPhysicalAddress pageAddr = pageList[pageInd].phys_addr;
-       address = ptoa_64(pageAddr) + offset;
+IOByteCount IOMemoryDescriptor::readBytes
+                (IOByteCount offset, void *bytes, IOByteCount length)
+{
+    addr64_t dstAddr = CAST_DOWN(addr64_t, bytes);
+    IOByteCount remaining;

 
 

-       // length is currently set to the length of the remainider of the iopl.
-       // We need to check that the remainder of the iopl is contiguous.
-       // This is indicated by pageList[ind].phys_addr being sequential.
-       IOByteCount contigLength = PAGE_SIZE - offset;
-       while (contigLength < length
-               && ++pageAddr == pageList[++pageInd].phys_addr)
-       {
-           contigLength += PAGE_SIZE;
-       }
+    // Assert that this entire I/O is withing the available range
+    assert(offset <= _length);
+    assert(offset + length <= _length);
+    if (offset >= _length) {
+        return 0;
+    }

 
 

-       if (contigLength < length)
-           length = contigLength;
-       
+    if (kIOMemoryThreadSafe & _flags)
+       LOCK;

 
 

-       assert(address);
-       assert(length);
+    remaining = length = min(length, _length - offset);
+    while (remaining) {        // (process another target segment?)
+        addr64_t       srcAddr64;
+        IOByteCount    srcLen;

 
 

-    } while (false);
+        srcAddr64 = getPhysicalSegment(offset, &srcLen, kIOMemoryMapperNone);
+        if (!srcAddr64)
+            break;

 
 

-    // Update return values and state
-    isP->fIO.fIOVMAddr = address;
-    isP->fIO.fLength   = length;
-    isP->fIndex        = ind;
-    isP->fOffset2Index = off2Ind;
-    isP->fNextOffset   = isP->fIO.fOffset + length;
+        // Clip segment length to remaining
+        if (srcLen > remaining)
+            srcLen = remaining;

 
 

-    return kIOReturnSuccess;
+        copypv(srcAddr64, dstAddr, srcLen,
+                            cppvPsrc | cppvNoRefSrc | cppvFsnk | cppvKmap);
+
+        dstAddr   += srcLen;
+        offset    += srcLen;
+        remaining -= srcLen;
+    }
+
+    if (kIOMemoryThreadSafe & _flags)
+       UNLOCK;
+
+    assert(!remaining);
+
+    return length - remaining;

 }
 
 }
 

-addr64_t
-IOGeneralMemoryDescriptor::getPhysicalSegment64(IOByteCount offset, IOByteCount *lengthOfSegment)
+IOByteCount IOMemoryDescriptor::writeBytes
+                (IOByteCount inoffset, const void *bytes, IOByteCount length)

 {
 {

-    IOReturn    ret;
-    IOByteCount length  = 0;
-    addr64_t    address = 0;
+    addr64_t srcAddr = CAST_DOWN(addr64_t, bytes);
+    IOByteCount remaining;
+    IOByteCount offset = inoffset;

 
 

-    if (offset < _length) // (within bounds?)
-    {
-       IOMDDMAWalkSegmentState _state;
-       IOMDDMAWalkSegmentArgs * state = (IOMDDMAWalkSegmentArgs *) &_state;
+    // Assert that this entire I/O is withing the available range
+    assert(offset <= _length);
+    assert(offset + length <= _length);

 
 

-       state->fOffset = offset;
-       state->fLength = _length - offset;
-       state->fMapped = false;
+    assert( !(kIOMemoryPreparedReadOnly & _flags) );

 
 

-       ret = dmaCommandOperation(kIOMDFirstSegment, _state, sizeof(_state));
+    if ( (kIOMemoryPreparedReadOnly & _flags) || offset >= _length) {
+        return 0;
+    }

 
 

-       if ((kIOReturnSuccess != ret) && (kIOReturnOverrun != ret))
-               DEBG("getPhysicalSegment64 dmaCommandOperation(%lx), %p, offset %qx, addr %qx, len %qx\n", 
-                                       ret, this, state->fOffset,
-                                       state->fIOVMAddr, state->fLength);
-       if (kIOReturnSuccess == ret)
+    if (kIOMemoryThreadSafe & _flags)
+       LOCK;
+
+    remaining = length = min(length, _length - offset);
+    while (remaining) {        // (process another target segment?)
+        addr64_t    dstAddr64;
+        IOByteCount dstLen;
+
+        dstAddr64 = getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
+        if (!dstAddr64)
+            break;
+
+        // Clip segment length to remaining
+        if (dstLen > remaining)
+            dstLen = remaining;
+
+       if (!srcAddr) bzero_phys(dstAddr64, dstLen);
+       else

        {
        {

-           address = state->fIOVMAddr;
-           length  = state->fLength;
+           copypv(srcAddr, (addr64_t) dstAddr64, dstLen,
+                   cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
+           srcAddr   += dstLen;

        }
        }

-        if (!address)
-            length = 0;
+        offset    += dstLen;
+        remaining -= dstLen;

     }
 
     }
 

-    if (lengthOfSegment)
-        *lengthOfSegment = length;
+    if (kIOMemoryThreadSafe & _flags)
+       UNLOCK;

 
 

-    return (address);
+    assert(!remaining);
+
+    if (!srcAddr) performOperation(kIOMemoryIncoherentIOFlush, inoffset, length);
+
+    return length - remaining;

 }
 
 }
 

-IOPhysicalAddress
-IOGeneralMemoryDescriptor::getPhysicalSegment(IOByteCount offset, IOByteCount *lengthOfSegment)
+#ifndef __LP64__
+void IOGeneralMemoryDescriptor::setPosition(IOByteCount position)

 {
 {

-    IOReturn          ret;
-    IOByteCount       length  = 0;
-    addr64_t         address = 0;
+    panic("IOGMD::setPosition deprecated");
+}
+#endif /* !__LP64__ */

 
 

-//  assert(offset <= _length);
+static volatile SInt64 gIOMDPreparationID __attribute__((aligned(8))) = (1ULL << 32);

 
 

-    if (offset < _length) // (within bounds?)
-    {
-       IOMDDMAWalkSegmentState _state;
-       IOMDDMAWalkSegmentArgs * state = (IOMDDMAWalkSegmentArgs *) &_state;
+uint64_t
+IOGeneralMemoryDescriptor::getPreparationID( void )
+{
+    ioGMDData *dataP;

 
 

-       state->fOffset = offset;
-       state->fLength = _length - offset;
-       state->fMapped = true;
+    if (!_wireCount)
+       return (kIOPreparationIDUnprepared);

 
 

-       ret = dmaCommandOperation(
-               kIOMDFirstSegment, _state, sizeof(_state));
+    if (((kIOMemoryTypeMask & _flags) == kIOMemoryTypePhysical)
+      || ((kIOMemoryTypeMask & _flags) == kIOMemoryTypePhysical64))
+    {
+        IOMemoryDescriptor::setPreparationID();
+        return (IOMemoryDescriptor::getPreparationID());
+    }

 
 

-       if ((kIOReturnSuccess != ret) && (kIOReturnOverrun != ret))
-           DEBG("getPhysicalSegment dmaCommandOperation(%lx), %p, offset %qx, addr %qx, len %qx\n", 
-                                   ret, this, state->fOffset,
-                                   state->fIOVMAddr, state->fLength);
-       if (kIOReturnSuccess == ret)
-       {
-           address = state->fIOVMAddr;
-           length  = state->fLength;
-       }
+    if (!_memoryEntries || !(dataP = getDataP(_memoryEntries)))
+       return (kIOPreparationIDUnprepared);

 
 

-        if (!address)
-            length = 0;
+    if (kIOPreparationIDUnprepared == dataP->fPreparationID)
+    {
+       dataP->fPreparationID = OSIncrementAtomic64(&gIOMDPreparationID);

     }
     }

+    return (dataP->fPreparationID);
+}

 
 

-    if ((address + length) > 0x100000000ULL)
+IOMemoryDescriptorReserved * IOMemoryDescriptor::getKernelReserved( void )
+{
+    if (!reserved)

     {
     {

-       panic("getPhysicalSegment() out of 32b range 0x%qx, len 0x%x, class %s",
-                   address, length, (getMetaClass())->getClassName());
+        reserved = IONew(IOMemoryDescriptorReserved, 1);
+        if (reserved)
+            bzero(reserved, sizeof(IOMemoryDescriptorReserved));

     }
     }

+    return (reserved);
+}

 
 

-    if (lengthOfSegment)
-        *lengthOfSegment = length;
+void IOMemoryDescriptor::setPreparationID( void )
+{
+    if (getKernelReserved() && (kIOPreparationIDUnprepared == reserved->preparationID))
+    {
+#if defined(__ppc__ )
+        reserved->preparationID = gIOMDPreparationID++;
+#else
+        reserved->preparationID = OSIncrementAtomic64(&gIOMDPreparationID);
+#endif
+    }
+}

 
 

-    return ((IOPhysicalAddress) address);
+uint64_t IOMemoryDescriptor::getPreparationID( void )
+{
+    if (reserved)
+        return (reserved->preparationID);    
+    else
+        return (kIOPreparationIDUnsupported);    

 }
 
 }
 

-addr64_t
-IOMemoryDescriptor::getPhysicalSegment64(IOByteCount offset, IOByteCount *lengthOfSegment)
+IOReturn IOGeneralMemoryDescriptor::dmaCommandOperation(DMACommandOps op, void *vData, UInt dataSize) const

 {
 {

-    IOPhysicalAddress phys32;
-    IOByteCount              length;
-    addr64_t         phys64;
-    IOMapper *        mapper = 0;
+    IOReturn err = kIOReturnSuccess;
+    DMACommandOps params;
+    IOGeneralMemoryDescriptor * md = const_cast<IOGeneralMemoryDescriptor *>(this);
+    ioGMDData *dataP;

 
 

-    phys32 = getPhysicalSegment(offset, lengthOfSegment);
-    if (!phys32)
-       return 0;
+    params = (op & ~kIOMDDMACommandOperationMask & op);
+    op &= kIOMDDMACommandOperationMask;

 
 

-    if (gIOSystemMapper)
-       mapper = gIOSystemMapper;
+    if (kIOMDDMAMap == op)
+    {
+       if (dataSize < sizeof(IOMDDMAMapArgs))
+           return kIOReturnUnderrun;

 
 

-    if (mapper)
+       IOMDDMAMapArgs * data = (IOMDDMAMapArgs *) vData;
+
+       if (!_memoryEntries 
+           && !md->initMemoryEntries(computeDataSize(0, 0), kIOMapperWaitSystem)) return (kIOReturnNoMemory);
+
+       if (_memoryEntries && data->fMapper)
+       {
+           bool remap;
+           bool whole = ((data->fOffset == 0) && (data->fLength == _length));
+           dataP = getDataP(_memoryEntries);
+
+           if (data->fMapSpec.numAddressBits < dataP->fDMAMapNumAddressBits) dataP->fDMAMapNumAddressBits = data->fMapSpec.numAddressBits;
+           if (data->fMapSpec.alignment      > dataP->fDMAMapAlignment)      dataP->fDMAMapAlignment      = data->fMapSpec.alignment;
+
+           remap = (dataP->fDMAMapNumAddressBits < 64)
+                && ((dataP->fMappedBase + _length) > (1ULL << dataP->fDMAMapNumAddressBits));
+           remap |= (dataP->fDMAMapAlignment > page_size);
+           remap |= (!whole);
+           if (remap || !dataP->fMappedBase)
+           {
+//             if (dataP->fMappedBase) OSReportWithBacktrace("kIOMDDMAMap whole %d remap %d params %d\n", whole, remap, params);
+               err = md->dmaMap(data->fMapper, &data->fMapSpec, data->fOffset, data->fLength, &data->fAlloc, &data->fAllocCount);
+               if ((kIOReturnSuccess == err) && whole && !dataP->fMappedBase)
+               {
+                   dataP->fMappedBase = data->fAlloc;
+                   data->fAllocCount = 0;                      // IOMD owns the alloc now
+               }
+           }
+           else
+           {
+               data->fAlloc = dataP->fMappedBase;
+               data->fAllocCount = 0;                          // IOMD owns the alloc
+           }
+           data->fMapContig = !dataP->fDiscontig;
+       }
+
+       return (err);                           
+    }
+
+    if (kIOMDAddDMAMapSpec == op)

     {
     {

-       IOByteCount origLen;
+       if (dataSize < sizeof(IODMAMapSpecification))
+           return kIOReturnUnderrun;

 
 

-       phys64 = mapper->mapAddr(phys32);
-       origLen = *lengthOfSegment;
-       length = page_size - (phys64 & (page_size - 1));
-       while ((length < origLen)
-           && ((phys64 + length) == mapper->mapAddr(phys32 + length)))
-           length += page_size;
-       if (length > origLen)
-           length = origLen;
+       IODMAMapSpecification * data = (IODMAMapSpecification *) vData;

 
 

-       *lengthOfSegment = length;
+       if (!_memoryEntries 
+           && !md->initMemoryEntries(computeDataSize(0, 0), kIOMapperWaitSystem)) return (kIOReturnNoMemory);
+
+       if (_memoryEntries)
+       {
+           dataP = getDataP(_memoryEntries);
+           if (data->numAddressBits < dataP->fDMAMapNumAddressBits)
+               dataP->fDMAMapNumAddressBits = data->numAddressBits;
+           if (data->alignment > dataP->fDMAMapAlignment)
+               dataP->fDMAMapAlignment = data->alignment;
+       }
+       return kIOReturnSuccess;

     }
     }

-    else
-       phys64 = (addr64_t) phys32;

 
 

-    return phys64;
-}
+    if (kIOMDGetCharacteristics == op) {

 
 

-IOPhysicalAddress
-IOGeneralMemoryDescriptor::getSourceSegment(IOByteCount offset, IOByteCount *lengthOfSegment)
-{
-    IOPhysicalAddress address = 0;
-    IOPhysicalLength  length  = 0;
-    IOOptionBits      type    = _flags & kIOMemoryTypeMask;
+       if (dataSize < sizeof(IOMDDMACharacteristics))
+           return kIOReturnUnderrun;

 
 

-    assert(offset <= _length);
+       IOMDDMACharacteristics *data = (IOMDDMACharacteristics *) vData;
+       data->fLength = _length;
+       data->fSGCount = _rangesCount;
+       data->fPages = _pages;
+       data->fDirection = getDirection();
+       if (!_wireCount)
+           data->fIsPrepared = false;
+       else {
+           data->fIsPrepared = true;
+           data->fHighestPage = _highestPage;
+           if (_memoryEntries)
+           {
+               dataP = getDataP(_memoryEntries);
+               ioPLBlock *ioplList = getIOPLList(dataP);
+               UInt count = getNumIOPL(_memoryEntries, dataP);
+               if (count == 1)
+                   data->fPageAlign = (ioplList[0].fPageOffset & PAGE_MASK) | ~PAGE_MASK;
+           }
+       }

 
 

-    if ( type == kIOMemoryTypeUPL)
-       return super::getSourceSegment( offset, lengthOfSegment );
-    else if ( offset < _length ) // (within bounds?)
-    {
-        unsigned rangesIndex = 0;
-       Ranges vec = _ranges;
-       user_addr_t addr;
+       return kIOReturnSuccess;

 
 

-       // Find starting address within the vector of ranges
-       for (;;) {
-           getAddrLenForInd(addr, length, type, vec, rangesIndex);
-           if (offset < length)
-               break;
-           offset -= length; // (make offset relative)
-           rangesIndex++;
-       } 
+#if IOMD_DEBUG_DMAACTIVE
+    } else if (kIOMDDMAActive == op) {
+       if (params) OSIncrementAtomic(&md->__iomd_reservedA);
+       else {
+           if (md->__iomd_reservedA)
+               OSDecrementAtomic(&md->__iomd_reservedA);
+           else
+               panic("kIOMDSetDMAInactive");
+       }
+#endif /* IOMD_DEBUG_DMAACTIVE */

 
 

-       // Now that we have the starting range,
-       // lets find the last contiguous range
-        addr   += offset;
-        length -= offset;
+    } else if (kIOMDWalkSegments != op)
+       return kIOReturnBadArgument;

 
 

-        for ( ++rangesIndex; rangesIndex < _rangesCount; rangesIndex++ ) {
-           user_addr_t      newAddr;
-           IOPhysicalLength newLen;
+    // Get the next segment
+    struct InternalState {
+       IOMDDMAWalkSegmentArgs fIO;
+       UInt fOffset2Index;
+       UInt fIndex;
+       UInt fNextOffset;
+    } *isP;
+
+    // Find the next segment
+    if (dataSize < sizeof(*isP))
+       return kIOReturnUnderrun;
+
+    isP = (InternalState *) vData;
+    UInt offset = isP->fIO.fOffset;
+    bool mapped = isP->fIO.fMapped;
+
+    if (IOMapper::gSystem && mapped
+        && (!(kIOMemoryHostOnly & _flags))
+       && (!_memoryEntries || !getDataP(_memoryEntries)->fMappedBase))
+//     && (_memoryEntries && !getDataP(_memoryEntries)->fMappedBase))
+    {
+       if (!_memoryEntries 
+           && !md->initMemoryEntries(computeDataSize(0, 0), kIOMapperWaitSystem)) return (kIOReturnNoMemory);

 
 

-           getAddrLenForInd(newAddr, newLen, type, vec, rangesIndex);
-           if (addr + length != newAddr)
-               break;
-           length += newLen;
-       } 
-        if (addr)
-           address = (IOPhysicalAddress) addr; // Truncate address to 32bit
-       else
-           length = 0;
+       dataP = getDataP(_memoryEntries);
+       if (dataP->fMapper)
+       {
+           IODMAMapSpecification mapSpec;
+           bzero(&mapSpec, sizeof(mapSpec));
+           mapSpec.numAddressBits = dataP->fDMAMapNumAddressBits;
+           mapSpec.alignment = dataP->fDMAMapAlignment;
+           err = md->dmaMap(dataP->fMapper, &mapSpec, 0, _length, &dataP->fMappedBase, NULL);
+           if (kIOReturnSuccess != err) return (err);
+       }

     }
 
     }
 

-    if ( lengthOfSegment )  *lengthOfSegment = length;
-
-    return address;
-}
+    if (offset >= _length)
+       return (offset == _length)? kIOReturnOverrun : kIOReturnInternalError;

 
 

-/* DEPRECATED */ /* USE INSTEAD: map(), readBytes(), writeBytes() */
-/* DEPRECATED */ void * IOGeneralMemoryDescriptor::getVirtualSegment(IOByteCount offset,
-/* DEPRECATED */                                                       IOByteCount * lengthOfSegment)
-/* DEPRECATED */ {
-                    if (_task == kernel_task)
-                        return (void *) getSourceSegment(offset, lengthOfSegment);
-                    else
-                        panic("IOGMD::getVirtualSegment deprecated");
+    // Validate the previous offset
+    UInt ind, off2Ind = isP->fOffset2Index;
+    if (!params
+       && offset 
+       && (offset == isP->fNextOffset || off2Ind <= offset))
+       ind = isP->fIndex;
+    else
+       ind = off2Ind = 0;      // Start from beginning

 
 

-                    return 0;
-/* DEPRECATED */ }
-/* DEPRECATED */ /* USE INSTEAD: map(), readBytes(), writeBytes() */
+    UInt length;
+    UInt64 address;

 
 
 
 

+    if ( (_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical) {

 
 

-IOReturn 
-IOMemoryDescriptor::dmaCommandOperation(DMACommandOps op, void *vData, UInt dataSize) const
-{
-    if (kIOMDGetCharacteristics == op) {
-       if (dataSize < sizeof(IOMDDMACharacteristics))
-           return kIOReturnUnderrun;
+       // Physical address based memory descriptor
+       const IOPhysicalRange *physP = (IOPhysicalRange *) &_ranges.p[0];

 
 

-       IOMDDMACharacteristics *data = (IOMDDMACharacteristics *) vData;
-       data->fLength = getLength();
-       data->fSGCount = 0;
-       data->fDirection = _direction;
-       if (IOMapper::gSystem)
-           data->fIsMapped = true;
-       data->fIsPrepared = true;       // Assume prepared - fails safe
-    }
-    else if (kIOMDWalkSegments & op) {
-       if (dataSize < sizeof(IOMDDMAWalkSegmentArgs))
-           return kIOReturnUnderrun;
+       // Find the range after the one that contains the offset
+       mach_vm_size_t len;
+       for (len = 0; off2Ind <= offset; ind++) {
+           len = physP[ind].length;
+           off2Ind += len;
+       }

 
 

-       IOMDDMAWalkSegmentArgs *data = (IOMDDMAWalkSegmentArgs *) vData;
-       IOByteCount offset  = (IOByteCount) data->fOffset;
+       // Calculate length within range and starting address
+       length   = off2Ind - offset;
+       address  = physP[ind - 1].address + len - length;

 
 

-       IOPhysicalLength length;
-       IOMemoryDescriptor *ncmd = const_cast<IOMemoryDescriptor *>(this);
-       if (data->fMapped && IOMapper::gSystem)
-           data->fIOVMAddr = ncmd->getPhysicalSegment(offset, &length);
+       if (true && mapped && _memoryEntries 
+               && (dataP = getDataP(_memoryEntries)) && dataP->fMappedBase)
+       {
+           address = dataP->fMappedBase + offset;
+       }

        else
        else

-           data->fIOVMAddr = ncmd->getPhysicalSegment64(offset, &length);
-       data->fLength = length;
-    }
-    else
-       return kIOReturnBadArgument;
-
-    return kIOReturnSuccess;
-}
+       {
+           // see how far we can coalesce ranges
+           while (ind < _rangesCount && address + length == physP[ind].address) {
+               len = physP[ind].length;
+               length += len;
+               off2Ind += len;
+               ind++;
+           }
+       }

 
 

-IOReturn IOMemoryDescriptor::setPurgeable( IOOptionBits newState,
-                                           IOOptionBits * oldState )
-{
-    IOReturn     err = kIOReturnSuccess;
-    vm_purgable_t control;
-    int           state;
+       // correct contiguous check overshoot
+       ind--;
+       off2Ind -= len;
+    }
+#ifndef __LP64__
+    else if ( (_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical64) {

 
 

-    do 
-    {
-        if (!_memEntry)
-        {
-            err = kIOReturnNotReady;
-            break;
-        }
+       // Physical address based memory descriptor
+       const IOAddressRange *physP = (IOAddressRange *) &_ranges.v64[0];

 
 

-        control = VM_PURGABLE_SET_STATE;
-        switch (newState)
-        {
-            case kIOMemoryPurgeableKeepCurrent:
-                control = VM_PURGABLE_GET_STATE;
-                break;
+       // Find the range after the one that contains the offset
+       mach_vm_size_t len;
+       for (len = 0; off2Ind <= offset; ind++) {
+           len = physP[ind].length;
+           off2Ind += len;
+       }

 
 

-            case kIOMemoryPurgeableNonVolatile:
-                state = VM_PURGABLE_NONVOLATILE;
-                break;
-            case kIOMemoryPurgeableVolatile:
-                state = VM_PURGABLE_VOLATILE;
-                break;
-            case kIOMemoryPurgeableEmpty:
-                state = VM_PURGABLE_EMPTY;
-                break;
-            default:
-                err = kIOReturnBadArgument;
-                break;
-        }
+       // Calculate length within range and starting address
+       length   = off2Ind - offset;
+       address  = physP[ind - 1].address + len - length;

 
 

-        if (kIOReturnSuccess != err)
-            break;
+       if (true && mapped && _memoryEntries 
+               && (dataP = getDataP(_memoryEntries)) && dataP->fMappedBase)
+       {
+           address = dataP->fMappedBase + offset;
+       }
+       else
+       {
+           // see how far we can coalesce ranges
+           while (ind < _rangesCount && address + length == physP[ind].address) {
+               len = physP[ind].length;
+               length += len;
+               off2Ind += len;
+               ind++;
+           }
+       }
+       // correct contiguous check overshoot
+       ind--;
+       off2Ind -= len;
+    } 
+#endif /* !__LP64__ */
+    else do {
+       if (!_wireCount)
+           panic("IOGMD: not wired for the IODMACommand");

 
 

-        err = mach_memory_entry_purgable_control((ipc_port_t) _memEntry, control, &state);
+       assert(_memoryEntries);

 
 

-        if (oldState)
-        {
-            if (kIOReturnSuccess == err)
-            {
-                switch (state)
-                {
-                    case VM_PURGABLE_NONVOLATILE:
-                        state = kIOMemoryPurgeableNonVolatile;
-                        break;
-                    case VM_PURGABLE_VOLATILE:
-                        state = kIOMemoryPurgeableVolatile;
-                        break;
-                    case VM_PURGABLE_EMPTY:
-                        state = kIOMemoryPurgeableEmpty;
-                        break;
-                    default:
-                        state = kIOMemoryPurgeableNonVolatile;
-                        err = kIOReturnNotReady;
-                        break;
-                }
-                *oldState = state;
-            }
-        }
-    }
-    while (false);
+       dataP = getDataP(_memoryEntries);
+       const ioPLBlock *ioplList = getIOPLList(dataP);
+       UInt numIOPLs = getNumIOPL(_memoryEntries, dataP);
+       upl_page_info_t *pageList = getPageList(dataP);

 
 

-    return (err);
-}
+       assert(numIOPLs > 0);

 
 

-extern "C" void dcache_incoherent_io_flush64(addr64_t pa, unsigned int count);
-extern "C" void dcache_incoherent_io_store64(addr64_t pa, unsigned int count);
+       // Scan through iopl info blocks looking for block containing offset
+       while (ind < numIOPLs && offset >= ioplList[ind].fIOMDOffset)
+           ind++;

 
 

-IOReturn IOMemoryDescriptor::performOperation( IOOptionBits options,
-                                                IOByteCount offset, IOByteCount length )
-{
-    IOByteCount remaining;
-    void (*func)(addr64_t pa, unsigned int count) = 0;
+       // Go back to actual range as search goes past it
+       ioPLBlock ioplInfo = ioplList[ind - 1];
+       off2Ind = ioplInfo.fIOMDOffset;

 
 

-    switch (options)
-    {
-        case kIOMemoryIncoherentIOFlush:
-            func = &dcache_incoherent_io_flush64;
-            break;
-        case kIOMemoryIncoherentIOStore:
-            func = &dcache_incoherent_io_store64;
-            break;
-    }
+       if (ind < numIOPLs)
+           length = ioplList[ind].fIOMDOffset;
+       else
+           length = _length;
+       length -= offset;                       // Remainder within iopl

 
 

-    if (!func)
-        return (kIOReturnUnsupported);
+       // Subtract offset till this iopl in total list
+       offset -= off2Ind;

 
 

-    remaining = length = min(length, getLength() - offset);
-    while (remaining)
-    // (process another target segment?)
-    {
-        addr64_t    dstAddr64;
-        IOByteCount dstLen;
+       // If a mapped address is requested and this is a pre-mapped IOPL
+       // then just need to compute an offset relative to the mapped base.
+       if (mapped && dataP->fMappedBase) {
+           offset += (ioplInfo.fPageOffset & PAGE_MASK);
+           address = trunc_page_64(dataP->fMappedBase) + ptoa_64(ioplInfo.fMappedPage) + offset;
+           continue;   // Done leave do/while(false) now
+       }

 
 

-        dstAddr64 = getPhysicalSegment64(offset, &dstLen);
-        if (!dstAddr64)
-            break;
+       // The offset is rebased into the current iopl.
+       // Now add the iopl 1st page offset.
+       offset += ioplInfo.fPageOffset;

 
 

-        // Clip segment length to remaining
-        if (dstLen > remaining)
-            dstLen = remaining;
+       // For external UPLs the fPageInfo field points directly to
+       // the upl's upl_page_info_t array.
+       if (ioplInfo.fFlags & kIOPLExternUPL)
+           pageList = (upl_page_info_t *) ioplInfo.fPageInfo;
+       else
+           pageList = &pageList[ioplInfo.fPageInfo];

 
 

-       (*func)(dstAddr64, dstLen);
+       // Check for direct device non-paged memory
+       if ( ioplInfo.fFlags & kIOPLOnDevice ) {
+           address = ptoa_64(pageList->phys_addr) + offset;
+           continue;   // Done leave do/while(false) now
+       }

 
 

-        offset    += dstLen;
-        remaining -= dstLen;
-    }
+       // Now we need compute the index into the pageList
+       UInt pageInd = atop_32(offset);
+       offset &= PAGE_MASK;

 
 

-    return (remaining ? kIOReturnUnderrun : kIOReturnSuccess);
-}
+       // Compute the starting address of this segment
+       IOPhysicalAddress pageAddr = pageList[pageInd].phys_addr;
+       if (!pageAddr) {
+           panic("!pageList phys_addr");
+       }

 
 

-#ifdef __ppc__
-extern vm_offset_t             static_memory_end;
-#define io_kernel_static_end   static_memory_end
-#else
-extern vm_offset_t             first_avail;
-#define io_kernel_static_end   first_avail
-#endif
+       address = ptoa_64(pageAddr) + offset;

 
 

-static kern_return_t
-io_get_kernel_static_upl(
-       vm_map_t                /* map */,
-       vm_address_t            offset,
-       vm_size_t               *upl_size,
-       upl_t                   *upl,
-       upl_page_info_array_t   page_list,
-       unsigned int            *count,
-       ppnum_t                 *highest_page)
-{
-    unsigned int pageCount, page;
-    ppnum_t phys;
-    ppnum_t highestPage = 0;
+       // length is currently set to the length of the remainider of the iopl.
+       // We need to check that the remainder of the iopl is contiguous.
+       // This is indicated by pageList[ind].phys_addr being sequential.
+       IOByteCount contigLength = PAGE_SIZE - offset;
+       while (contigLength < length
+               && ++pageAddr == pageList[++pageInd].phys_addr)
+       {
+           contigLength += PAGE_SIZE;
+       }

 
 

-    pageCount = atop_32(*upl_size);
-    if (pageCount > *count)
-       pageCount = *count;
+       if (contigLength < length)
+           length = contigLength;
+       

 
 

-    *upl = NULL;
+       assert(address);
+       assert(length);

 
 

-    for (page = 0; page < pageCount; page++)
-    {
-       phys = pmap_find_phys(kernel_pmap, ((addr64_t)offset) + ptoa_64(page));
-       if (!phys)
-           break;
-       page_list[page].phys_addr = phys;
-       page_list[page].pageout   = 0;
-       page_list[page].absent    = 0;
-       page_list[page].dirty     = 0;
-       page_list[page].precious  = 0;
-       page_list[page].device    = 0;
-       if (phys > highestPage)
-           highestPage = page;
-    }
+    } while (false);

 
 

-    *highest_page = highestPage;
+    // Update return values and state
+    isP->fIO.fIOVMAddr = address;
+    isP->fIO.fLength   = length;
+    isP->fIndex        = ind;
+    isP->fOffset2Index = off2Ind;
+    isP->fNextOffset   = isP->fIO.fOffset + length;

 
 

-    return ((page >= pageCount) ? kIOReturnSuccess : kIOReturnVMError);
+    return kIOReturnSuccess;

 }
 
 }
 

-IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection)
+addr64_t
+IOGeneralMemoryDescriptor::getPhysicalSegment(IOByteCount offset, IOByteCount *lengthOfSegment, IOOptionBits options)

 {
 {

-    IOOptionBits type = _flags & kIOMemoryTypeMask;
-    IOReturn error = kIOReturnNoMemory;
-    ioGMDData *dataP;
-    ppnum_t mapBase = 0;
-    IOMapper *mapper;
-    ipc_port_t sharedMem = (ipc_port_t) _memEntry;
-
-    assert(!_wireCount);
-    assert(kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type || kIOMemoryTypeUIO == type);
-
-    if (_pages >= gIOMaximumMappedIOPageCount)
-       return kIOReturnNoResources;
+    IOReturn          ret;
+    mach_vm_address_t address = 0;
+    mach_vm_size_t    length  = 0;
+    IOMapper *        mapper  = gIOSystemMapper;
+    IOOptionBits      type    = _flags & kIOMemoryTypeMask;

 
 

-    dataP = getDataP(_memoryEntries);
-    mapper = dataP->fMapper;
-    if (mapper && _pages)
-        mapBase = mapper->iovmAlloc(_pages);
+    if (lengthOfSegment)
+        *lengthOfSegment = 0;

 
 

-    // Note that appendBytes(NULL) zeros the data up to the
-    // desired length.
-    _memoryEntries->appendBytes(0, dataP->fPageCnt * sizeof(upl_page_info_t));
-    dataP = 0; // May no longer be valid so lets not get tempted.
+    if (offset >= _length)
+        return 0;

 
 

-    if (forDirection == kIODirectionNone)
-        forDirection = _direction;
+    // IOMemoryDescriptor::doMap() cannot use getPhysicalSegment() to obtain the page offset, since it must
+    // support the unwired memory case in IOGeneralMemoryDescriptor, and hibernate_write_image() cannot use
+    // map()->getVirtualAddress() to obtain the kernel pointer, since it must prevent the memory allocation
+    // due to IOMemoryMap, so _kIOMemorySourceSegment is a necessary evil until all of this gets cleaned up

 
 

-    int uplFlags;    // This Mem Desc's default flags for upl creation
-    switch (kIODirectionOutIn & forDirection)
+    if ((options & _kIOMemorySourceSegment) && (kIOMemoryTypeUPL != type))

     {
     {

-    case kIODirectionOut:
-        // Pages do not need to be marked as dirty on commit
-        uplFlags = UPL_COPYOUT_FROM;
-        _flags |= kIOMemoryPreparedReadOnly;
-        break;
-
-    case kIODirectionIn:
-    default:
-        uplFlags = 0;  // i.e. ~UPL_COPYOUT_FROM
-        break;
-    }
-    uplFlags |= UPL_SET_IO_WIRE | UPL_SET_LITE;
+        unsigned rangesIndex = 0;
+       Ranges vec = _ranges;
+       mach_vm_address_t addr;

 
 

-#ifdef UPL_NEED_32BIT_ADDR
-    if (kIODirectionPrepareToPhys32 & forDirection) 
-       uplFlags |= UPL_NEED_32BIT_ADDR;
-#endif
+       // Find starting address within the vector of ranges
+       for (;;) {
+           getAddrLenForInd(addr, length, type, vec, rangesIndex);
+           if (offset < length)
+               break;
+           offset -= length; // (make offset relative)
+           rangesIndex++;
+       } 

 
 

-    // Find the appropriate vm_map for the given task
-    vm_map_t curMap;
-    if (_task == kernel_task && (kIOMemoryBufferPageable & _flags))
-        curMap = 0;
-    else
-        { curMap = get_task_map(_task); }
+       // Now that we have the starting range,
+       // lets find the last contiguous range
+        addr   += offset;
+        length -= offset;

 
 

-    // Iterate over the vector of virtual ranges
-    Ranges vec = _ranges;
-    unsigned int pageIndex = 0;
-    IOByteCount mdOffset = 0;
-    ppnum_t highestPage = 0;
-    for (UInt range = 0; range < _rangesCount; range++) {
-        ioPLBlock iopl;
-       user_addr_t startPage;
-        IOByteCount numBytes;
-       ppnum_t highPage = 0;
+        for ( ++rangesIndex; rangesIndex < _rangesCount; rangesIndex++ ) {
+           mach_vm_address_t newAddr;
+           mach_vm_size_t    newLen;

 
 

-       // Get the startPage address and length of vec[range]
-       getAddrLenForInd(startPage, numBytes, type, vec, range);
-       iopl.fPageOffset = (short) startPage & PAGE_MASK;
-       numBytes += iopl.fPageOffset;
-       startPage = trunc_page_64(startPage);
+           getAddrLenForInd(newAddr, newLen, type, vec, rangesIndex);
+           if (addr + length != newAddr)
+               break;
+           length += newLen;
+       } 
+        if (addr)
+           address = (IOPhysicalAddress) addr; // Truncate address to 32bit
+    }
+    else
+    {
+       IOMDDMAWalkSegmentState _state;
+       IOMDDMAWalkSegmentArgs * state = (IOMDDMAWalkSegmentArgs *) (void *)&_state;

 
 

-       if (mapper)
-           iopl.fMappedBase = mapBase + pageIndex;
-       else
-           iopl.fMappedBase = 0;
+       state->fOffset = offset;
+       state->fLength = _length - offset;
+       state->fMapped = (0 == (options & kIOMemoryMapperNone)) && !(_flags & kIOMemoryHostOnly);

 
 

-       // Iterate over the current range, creating UPLs
-        while (numBytes) {
-            dataP = getDataP(_memoryEntries);
-           vm_address_t kernelStart = (vm_address_t) startPage;
-            vm_map_t theMap;
-           if (curMap)
-               theMap = curMap;
-           else if (!sharedMem) {
-               assert(_task == kernel_task);
-               theMap = IOPageableMapForAddress(kernelStart);
-           }
-           else
-               theMap = NULL;
+       ret = dmaCommandOperation(kIOMDFirstSegment, _state, sizeof(_state));

 
 

-            upl_page_info_array_t pageInfo = getPageList(dataP);
-            int ioplFlags = uplFlags;
-            upl_page_list_ptr_t baseInfo = &pageInfo[pageIndex];
+       if ((kIOReturnSuccess != ret) && (kIOReturnOverrun != ret))
+               DEBG("getPhysicalSegment dmaCommandOperation(%lx), %p, offset %qx, addr %qx, len %qx\n", 
+                                       ret, this, state->fOffset,
+                                       state->fIOVMAddr, state->fLength);
+       if (kIOReturnSuccess == ret)
+       {
+           address = state->fIOVMAddr;
+           length  = state->fLength;
+       }

 
 

-            vm_size_t ioplSize = round_page_32(numBytes);
-            unsigned int numPageInfo = atop_32(ioplSize);
+       // dmaCommandOperation() does not distinguish between "mapped" and "unmapped" physical memory, even
+       // with fMapped set correctly, so we must handle the transformation here until this gets cleaned up

 
 

-           if (theMap == kernel_map && kernelStart < io_kernel_static_end) {
-               error = io_get_kernel_static_upl(theMap, 
-                                               kernelStart,
-                                               &ioplSize,
-                                               &iopl.fIOPL,
-                                               baseInfo,
-                                               &numPageInfo,
-                                               &highPage);
-           }
-           else if (sharedMem) {
-               error = memory_object_iopl_request(sharedMem, 
-                                               ptoa_32(pageIndex),
-                                               &ioplSize,
-                                               &iopl.fIOPL,
-                                               baseInfo,
-                                               &numPageInfo,
-                                               &ioplFlags);
-           }
-           else {
-               assert(theMap);
-               error = vm_map_create_upl(theMap,
-                                               startPage,
-                                               &ioplSize,
-                                               &iopl.fIOPL,
-                                               baseInfo,
-                                               &numPageInfo,
-                                               &ioplFlags);
+       if (mapper && ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type)))
+       {
+           if ((options & kIOMemoryMapperNone) && !(_flags & kIOMemoryMapperNone))
+           {
+               addr64_t    origAddr = address;
+               IOByteCount origLen  = length;
+
+               address = mapper->mapAddr(origAddr);
+               length = page_size - (address & (page_size - 1));
+               while ((length < origLen)
+                   && ((address + length) == mapper->mapAddr(origAddr + length)))
+                   length += page_size;
+               if (length > origLen)
+                   length = origLen;

            }
            }

+       }
+    }

 
 

-            assert(ioplSize);
-            if (error != KERN_SUCCESS)
-                goto abortExit;
+    if (!address)
+        length = 0;

 
 

-           if (iopl.fIOPL)
-               highPage = upl_get_highest_page(iopl.fIOPL);
-           if (highPage > highestPage)
-               highestPage = highPage;
+    if (lengthOfSegment)
+        *lengthOfSegment = length;

 
 

-            error = kIOReturnNoMemory;
+    return (address);
+}

 
 

-            if (baseInfo->device) {
-                numPageInfo = 1;
-                iopl.fFlags  = kIOPLOnDevice;
-                // Don't translate device memory at all 
-               if (mapper && mapBase) {
-                   mapper->iovmFree(mapBase, _pages);
-                   mapBase = 0;
-                   iopl.fMappedBase = 0;
-               }
-            }
-            else {
-                iopl.fFlags = 0;
-               if (mapper)
-                    mapper->iovmInsert(mapBase, pageIndex,
-                                       baseInfo, numPageInfo);
-            }
+#ifndef __LP64__
+addr64_t
+IOMemoryDescriptor::getPhysicalSegment(IOByteCount offset, IOByteCount *lengthOfSegment, IOOptionBits options)
+{
+    addr64_t address = 0;

 
 

-            iopl.fIOMDOffset = mdOffset;
-            iopl.fPageInfo = pageIndex;
+    if (options & _kIOMemorySourceSegment)
+    {
+        address = getSourceSegment(offset, lengthOfSegment);
+    }
+    else if (options & kIOMemoryMapperNone)
+    {
+        address = getPhysicalSegment64(offset, lengthOfSegment);
+    }
+    else
+    {
+        address = getPhysicalSegment(offset, lengthOfSegment);
+    }

 
 

-           if ((_flags & kIOMemoryAutoPrepare) && iopl.fIOPL)
-           {
-               upl_commit(iopl.fIOPL, 0, 0);
-               upl_deallocate(iopl.fIOPL);
-               iopl.fIOPL = 0;
-           }
+    return (address);
+}

 
 

-            if (!_memoryEntries->appendBytes(&iopl, sizeof(iopl))) {
-                // Clean up partial created and unsaved iopl
-                if (iopl.fIOPL) {
-                    upl_abort(iopl.fIOPL, 0);
-                    upl_deallocate(iopl.fIOPL);
-                }
-                goto abortExit;
-            }
+addr64_t
+IOGeneralMemoryDescriptor::getPhysicalSegment64(IOByteCount offset, IOByteCount *lengthOfSegment)
+{
+    return (getPhysicalSegment(offset, lengthOfSegment, kIOMemoryMapperNone));
+}

 
 

-            // Check for a multiple iopl's in one virtual range
-            pageIndex += numPageInfo;
-            mdOffset -= iopl.fPageOffset;
-            if (ioplSize < numBytes) {
-                numBytes -= ioplSize;
-                startPage += ioplSize;
-                mdOffset += ioplSize;
-                iopl.fPageOffset = 0;
-               if (mapper)
-                   iopl.fMappedBase = mapBase + pageIndex;
-            }
-            else {
-                mdOffset += numBytes;
-                break;
-            }
-        }
-    }
+IOPhysicalAddress
+IOGeneralMemoryDescriptor::getPhysicalSegment(IOByteCount offset, IOByteCount *lengthOfSegment)
+{
+    addr64_t    address = 0;
+    IOByteCount length  = 0;

 
 

-    _highestPage = highestPage;
+    address = getPhysicalSegment(offset, lengthOfSegment, 0);

 
 

-    return kIOReturnSuccess;
+    if (lengthOfSegment)
+       length = *lengthOfSegment;

 
 

-abortExit:
+    if ((address + length) > 0x100000000ULL)

     {
     {

-        dataP = getDataP(_memoryEntries);
-        UInt done = getNumIOPL(_memoryEntries, dataP);
-        ioPLBlock *ioplList = getIOPLList(dataP);
-    
-        for (UInt range = 0; range < done; range++)
-       {
-           if (ioplList[range].fIOPL) {
-             upl_abort(ioplList[range].fIOPL, 0);
-             upl_deallocate(ioplList[range].fIOPL);
-           }
-       }
-       (void) _memoryEntries->initWithBytes(dataP, sizeof(ioGMDData)); // == setLength()
-
-        if (mapper && mapBase)
-            mapper->iovmFree(mapBase, _pages);
+       panic("getPhysicalSegment() out of 32b range 0x%qx, len 0x%lx, class %s",
+                   address, (long) length, (getMetaClass())->getClassName());

     }
 
     }
 

-    return error;
+    return ((IOPhysicalAddress) address);

 }
 
 }
 

-/*
- * prepare
- *
- * Prepare the memory for an I/O transfer.  This involves paging in
- * the memory, if necessary, and wiring it down for the duration of
- * the transfer.  The complete() method completes the processing of
- * the memory after the I/O transfer finishes.  This method needn't
- * called for non-pageable memory.
- */
-IOReturn IOGeneralMemoryDescriptor::prepare(IODirection forDirection)
+addr64_t
+IOMemoryDescriptor::getPhysicalSegment64(IOByteCount offset, IOByteCount *lengthOfSegment)

 {
 {

-    IOReturn error    = kIOReturnSuccess;
-    IOOptionBits type = _flags & kIOMemoryTypeMask;
+    IOPhysicalAddress phys32;
+    IOByteCount              length;
+    addr64_t         phys64;
+    IOMapper *        mapper = 0;

 
 

-    if (!_wireCount
-    && (kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type || kIOMemoryTypeUIO == type) ) {
-        error = wireVirtual(forDirection);
-        if (error)
-            return error;
-    }
+    phys32 = getPhysicalSegment(offset, lengthOfSegment);
+    if (!phys32)
+       return 0;

 
 

-    _wireCount++;
+    if (gIOSystemMapper)
+       mapper = gIOSystemMapper;

 
 

-    return kIOReturnSuccess;
-}
+    if (mapper)
+    {
+       IOByteCount origLen;

 
 

-/*
- * complete
- *
- * Complete processing of the memory after an I/O transfer finishes.
- * This method should not be called unless a prepare was previously
- * issued; the prepare() and complete() must occur in pairs, before
- * before and after an I/O transfer involving pageable memory.
- */
- 
-IOReturn IOGeneralMemoryDescriptor::complete(IODirection /* forDirection */)
-{
-    assert(_wireCount);
+       phys64 = mapper->mapAddr(phys32);
+       origLen = *lengthOfSegment;
+       length = page_size - (phys64 & (page_size - 1));
+       while ((length < origLen)
+           && ((phys64 + length) == mapper->mapAddr(phys32 + length)))
+           length += page_size;
+       if (length > origLen)
+           length = origLen;

 
 

-    if (!_wireCount)
-        return kIOReturnSuccess;
+       *lengthOfSegment = length;
+    }
+    else
+       phys64 = (addr64_t) phys32;

 
 

-    _wireCount--;
-    if (!_wireCount) {
-       IOOptionBits type = _flags & kIOMemoryTypeMask;
+    return phys64;
+}

 
 

-        if ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type)) {
-            /* kIOMemoryTypePhysical */
-            // DO NOTHING
-        }
-        else {
-            ioGMDData * dataP = getDataP(_memoryEntries);
-            ioPLBlock *ioplList = getIOPLList(dataP);
-           UInt count = getNumIOPL(_memoryEntries, dataP);
-
-            if (dataP->fMapper && _pages && ioplList[0].fMappedBase)
-                dataP->fMapper->iovmFree(ioplList[0].fMappedBase, _pages);
-
-            // Only complete iopls that we created which are for TypeVirtual
-            if (kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type || kIOMemoryTypeUIO == type) {
-                for (UInt ind = 0; ind < count; ind++)
-                   if (ioplList[ind].fIOPL) {
-                        upl_commit(ioplList[ind].fIOPL, 0, 0);
-                        upl_deallocate(ioplList[ind].fIOPL);
-                   }
-            }
+IOPhysicalAddress
+IOMemoryDescriptor::getPhysicalSegment(IOByteCount offset, IOByteCount *lengthOfSegment)
+{
+    return ((IOPhysicalAddress) getPhysicalSegment(offset, lengthOfSegment, 0));
+}

 
 

-            (void) _memoryEntries->initWithBytes(dataP, sizeof(ioGMDData)); // == setLength()
-        }
-    }
-    return kIOReturnSuccess;
+IOPhysicalAddress
+IOGeneralMemoryDescriptor::getSourceSegment(IOByteCount offset, IOByteCount *lengthOfSegment)
+{
+    return ((IOPhysicalAddress) getPhysicalSegment(offset, lengthOfSegment, _kIOMemorySourceSegment));

 }
 
 }
 

-IOReturn IOGeneralMemoryDescriptor::doMap(
-       vm_map_t                addressMap,
-       IOVirtualAddress *      atAddress,
-       IOOptionBits            options,
-       IOByteCount             sourceOffset,
-       IOByteCount             length )
+void * IOGeneralMemoryDescriptor::getVirtualSegment(IOByteCount offset,
+                                                       IOByteCount * lengthOfSegment)

 {
 {

-    kern_return_t kr;
-    ipc_port_t sharedMem = (ipc_port_t) _memEntry;
+    if (_task == kernel_task)
+        return (void *) getSourceSegment(offset, lengthOfSegment);
+    else
+        panic("IOGMD::getVirtualSegment deprecated");

 
 

-    IOOptionBits type = _flags & kIOMemoryTypeMask;
-    Ranges vec = _ranges;
+    return 0;
+}
+#endif /* !__LP64__ */
+
+IOReturn 
+IOMemoryDescriptor::dmaCommandOperation(DMACommandOps op, void *vData, UInt dataSize) const
+{
+    IOMemoryDescriptor *md = const_cast<IOMemoryDescriptor *>(this);
+    DMACommandOps params;
+    IOReturn err;

 
 

-    user_addr_t range0Addr = 0;
-    IOByteCount range0Len = 0;
+    params = (op & ~kIOMDDMACommandOperationMask & op);
+    op &= kIOMDDMACommandOperationMask;

 
 

-    if (vec.v)
-       getAddrLenForInd(range0Addr, range0Len, type, vec, 0);
+    if (kIOMDGetCharacteristics == op) {
+       if (dataSize < sizeof(IOMDDMACharacteristics))
+           return kIOReturnUnderrun;

 
 

-    // mapping source == dest? (could be much better)
-    if( _task
-    && (addressMap == get_task_map(_task)) && (options & kIOMapAnywhere)
-    && (1 == _rangesCount) && (0 == sourceOffset)
-    && range0Addr && (length <= range0Len) ) {
-       if (sizeof(user_addr_t) > 4 && ((UInt64) range0Addr) >> 32)
-           return kIOReturnOverrun;    // Doesn't fit in 32bit return field
-       else {
-           *atAddress = range0Addr;
-           return( kIOReturnSuccess );
-       }
+       IOMDDMACharacteristics *data = (IOMDDMACharacteristics *) vData;
+       data->fLength = getLength();
+       data->fSGCount = 0;
+       data->fDirection = getDirection();
+       data->fIsPrepared = true;       // Assume prepared - fails safe

     }
     }

+    else if (kIOMDWalkSegments == op) {
+       if (dataSize < sizeof(IOMDDMAWalkSegmentArgs))
+           return kIOReturnUnderrun;

 
 

-    if( 0 == sharedMem) {
-
-        vm_size_t size = ptoa_32(_pages);
+       IOMDDMAWalkSegmentArgs *data = (IOMDDMAWalkSegmentArgs *) vData;
+       IOByteCount offset  = (IOByteCount) data->fOffset;

 
 

-        if( _task) {
+       IOPhysicalLength length;
+       if (data->fMapped && IOMapper::gSystem)
+           data->fIOVMAddr = md->getPhysicalSegment(offset, &length);
+       else
+           data->fIOVMAddr = md->getPhysicalSegment(offset, &length, kIOMemoryMapperNone);
+       data->fLength = length;
+    }
+    else if (kIOMDAddDMAMapSpec == op) return kIOReturnUnsupported;
+    else if (kIOMDDMAMap == op)
+    {
+       if (dataSize < sizeof(IOMDDMAMapArgs))
+           return kIOReturnUnderrun;
+       IOMDDMAMapArgs * data = (IOMDDMAMapArgs *) vData;

 
 

-            memory_object_size_t actualSize = size;
-            kr = mach_make_memory_entry_64(get_task_map(_task),
-                        &actualSize, range0Addr,
-                        VM_PROT_READ | VM_PROT_WRITE, &sharedMem,
-                        NULL );
+       if (params) panic("class %s does not support IODMACommand::kIterateOnly", getMetaClass()->getClassName());

 
 

-            if( (KERN_SUCCESS == kr) && (actualSize != round_page_32(size))) {
-#if IOASSERT
-                IOLog("mach_make_memory_entry_64 (%08llx) size (%08lx:%08x)\n",
-                            range0Addr, (UInt32) actualSize, size);
-#endif
-                kr = kIOReturnVMError;
-                ipc_port_release_send( sharedMem );
-            }
+       data->fMapContig = true;
+       err = md->dmaMap(data->fMapper, &data->fMapSpec, data->fOffset, data->fLength, &data->fAlloc, &data->fAllocCount);
+       return (err);                           
+    }
+    else return kIOReturnBadArgument;

 
 

-            if( KERN_SUCCESS != kr)
-                sharedMem = MACH_PORT_NULL;
+    return kIOReturnSuccess;
+}

 
 

-        } else do {    // _task == 0, must be physical
+IOReturn 
+IOGeneralMemoryDescriptor::setPurgeable( IOOptionBits newState,
+                                                  IOOptionBits * oldState )
+{
+    IOReturn     err = kIOReturnSuccess;

 
 

-            memory_object_t    pager;
-           unsigned int        flags = 0;
-           addr64_t            pa;
-           IOPhysicalLength    segLen;
+    vm_purgable_t control;
+    int           state;

 
 

-           pa = getPhysicalSegment64( sourceOffset, &segLen );
+    if (_memRef)
+    {
+       err = super::setPurgeable(newState, oldState);
+    }
+    else
+    {
+       if (kIOMemoryThreadSafe & _flags)
+           LOCK;
+       do
+       {
+           // Find the appropriate vm_map for the given task
+           vm_map_t curMap;
+           if (_task == kernel_task && (kIOMemoryBufferPageable & _flags))
+           {
+               err = kIOReturnNotReady;
+               break;
+           }
+           else if (!_task)
+           {
+               err = kIOReturnUnsupported;
+               break;
+           }
+           else
+               curMap = get_task_map(_task);

 
 

-            if( !reserved) {
-                reserved = IONew( ExpansionData, 1 );
-                if( !reserved)
-                    continue;
-            }
-            reserved->pagerContig = (1 == _rangesCount);
-           reserved->memory = this;
+           // can only do one range
+           Ranges vec = _ranges;
+           IOOptionBits type = _flags & kIOMemoryTypeMask;
+           mach_vm_address_t addr; 
+           mach_vm_size_t    len;
+           getAddrLenForInd(addr, len, type, vec, 0);

 
 

-           /*What cache mode do we need*/
-            switch(options & kIOMapCacheMask ) {
+           err = purgeableControlBits(newState, &control, &state);
+           if (kIOReturnSuccess != err)
+               break;
+           err = mach_vm_purgable_control(curMap, addr, control, &state);
+           if (oldState)
+           {
+               if (kIOReturnSuccess == err)
+               {
+                   err = purgeableStateBits(&state);
+                   *oldState = state;
+               }
+           }
+       }
+       while (false);
+       if (kIOMemoryThreadSafe & _flags)
+           UNLOCK;
+    }

 
 

-               case kIOMapDefaultCache:
-               default:
-                   flags = IODefaultCacheBits(pa);
-                   break;
-       
-               case kIOMapInhibitCache:
-                   flags = DEVICE_PAGER_CACHE_INHIB | 
-                                   DEVICE_PAGER_COHERENT | DEVICE_PAGER_GUARDED;
-                   break;
-       
-               case kIOMapWriteThruCache:
-                   flags = DEVICE_PAGER_WRITE_THROUGH |
-                                   DEVICE_PAGER_COHERENT | DEVICE_PAGER_GUARDED;
-                   break;
+    return (err);
+}

 
 

-               case kIOMapCopybackCache:
-                   flags = DEVICE_PAGER_COHERENT;
-                   break;
+IOReturn IOMemoryDescriptor::setPurgeable( IOOptionBits newState,
+                                           IOOptionBits * oldState )
+{
+    IOReturn err = kIOReturnNotReady;

 
 

-               case kIOMapWriteCombineCache:
-                   flags = DEVICE_PAGER_CACHE_INHIB |
-                                   DEVICE_PAGER_COHERENT;
-                   break;
-            }
+    if (kIOMemoryThreadSafe & _flags) LOCK;
+    if (_memRef) err = IOGeneralMemoryDescriptor::memoryReferenceSetPurgeable(_memRef, newState, oldState);
+    if (kIOMemoryThreadSafe & _flags) UNLOCK;

 
 

-           flags |= reserved->pagerContig ? DEVICE_PAGER_CONTIGUOUS : 0;
+    return (err);
+}
+ 
+IOReturn IOMemoryDescriptor::getPageCounts( IOByteCount * residentPageCount,
+                                           IOByteCount * dirtyPageCount )
+{
+   IOReturn err = kIOReturnNotReady;

 
 

-            pager = device_pager_setup( (memory_object_t) 0, (int) reserved, 
-                                                               size, flags);
-            assert( pager );
+    if (kIOMemoryThreadSafe & _flags) LOCK;
+    if (_memRef) err = IOGeneralMemoryDescriptor::memoryReferenceGetPageCounts(_memRef, residentPageCount, dirtyPageCount);
+    if (kIOMemoryThreadSafe & _flags) UNLOCK;

 
 

-            if( pager) {
-                kr = mach_memory_object_memory_entry_64( (host_t) 1, false /*internal*/, 
-                            size, VM_PROT_READ | VM_PROT_WRITE, pager, &sharedMem );
+    return (err);
+}
+ 

 
 

-                assert( KERN_SUCCESS == kr );
-                if( KERN_SUCCESS != kr) {
-                   device_pager_deallocate( pager );
-                    pager = MACH_PORT_NULL;
-                    sharedMem = MACH_PORT_NULL;
-                }
-            }
-           if( pager && sharedMem)
-               reserved->devicePager    = pager;
-           else {
-               IODelete( reserved, ExpansionData, 1 );
-               reserved = 0;
-           }
+extern "C" void dcache_incoherent_io_flush64(addr64_t pa, unsigned int count);
+extern "C" void dcache_incoherent_io_store64(addr64_t pa, unsigned int count);

 
 

-        } while( false );
+static void SetEncryptOp(addr64_t pa, unsigned int count)
+{
+    ppnum_t page, end;

 
 

-        _memEntry = (void *) sharedMem;
+    page = atop_64(round_page_64(pa));
+    end  = atop_64(trunc_page_64(pa + count));
+    for (; page < end; page++)
+    {
+        pmap_clear_noencrypt(page);    

     }
     }

-
-
-    if( 0 == sharedMem)
-      kr = kIOReturnVMError;
-    else
-      kr = super::doMap( addressMap, atAddress,
-                           options, sourceOffset, length );
-
-    return( kr );

 }
 
 }
 

-IOReturn IOGeneralMemoryDescriptor::doUnmap(
-       vm_map_t                addressMap,
-       IOVirtualAddress        logical,
-       IOByteCount             length )
+static void ClearEncryptOp(addr64_t pa, unsigned int count)

 {
 {

-    // could be much better
-    if( _task && (addressMap == get_task_map(_task)) && (1 == _rangesCount)) {
-
-       IOOptionBits type = _flags & kIOMemoryTypeMask;
-       user_addr_t range0Addr;
-       IOByteCount range0Len;
+    ppnum_t page, end;

 
 

-       getAddrLenForInd(range0Addr, range0Len, type, _ranges, 0);
-       if (logical == range0Addr && length <= range0Len)
-           return( kIOReturnSuccess );
+    page = atop_64(round_page_64(pa));
+    end  = atop_64(trunc_page_64(pa + count));
+    for (; page < end; page++)
+    {
+        pmap_set_noencrypt(page);    

     }
     }

-
-    return( super::doUnmap( addressMap, logical, length ));

 }
 
 }
 

-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-OSDefineMetaClassAndAbstractStructors( IOMemoryMap, OSObject )
-
-/* inline function implementation */
-IOPhysicalAddress IOMemoryMap::getPhysicalAddress()
-    { return( getPhysicalSegment( 0, 0 )); }
+IOReturn IOMemoryDescriptor::performOperation( IOOptionBits options,
+                                                IOByteCount offset, IOByteCount length )
+{
+    IOByteCount remaining;
+    unsigned int res;
+    void (*func)(addr64_t pa, unsigned int count) = 0;

 
 

+    switch (options)
+    {
+        case kIOMemoryIncoherentIOFlush:
+            func = &dcache_incoherent_io_flush64;
+            break;
+        case kIOMemoryIncoherentIOStore:
+            func = &dcache_incoherent_io_store64;
+            break;

 
 

-#undef super
-#define super IOMemoryMap
+        case kIOMemorySetEncrypted:
+            func = &SetEncryptOp;
+            break;
+        case kIOMemoryClearEncrypted:
+            func = &ClearEncryptOp;
+            break;
+    }

 
 

-OSDefineMetaClassAndStructors(_IOMemoryMap, IOMemoryMap)
+    if (!func)
+        return (kIOReturnUnsupported);

 
 

-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+    if (kIOMemoryThreadSafe & _flags)
+       LOCK;

 
 

-bool _IOMemoryMap::initCompatible(
-       IOMemoryDescriptor *    _memory,
-       IOMemoryMap *           _superMap,
-        IOByteCount            _offset,
-        IOByteCount            _length )
-{
+    res = 0x0UL;
+    remaining = length = min(length, getLength() - offset);
+    while (remaining)
+    // (process another target segment?)
+    {
+        addr64_t    dstAddr64;
+        IOByteCount dstLen;

 
 

-    if( !super::init())
-       return( false);
+        dstAddr64 = getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
+        if (!dstAddr64)
+            break;

 
 

-    if( (_offset + _length) > _superMap->getLength())
-       return( false);
+        // Clip segment length to remaining
+        if (dstLen > remaining)
+            dstLen = remaining;

 
 

-    _memory->retain();
-    memory     = _memory;
-    _superMap->retain();
-    superMap   = _superMap;
+       (*func)(dstAddr64, dstLen);

 
 

-    offset     = _offset;
-    if( _length)
-        length = _length;
-    else
-        length = _memory->getLength();
+        offset    += dstLen;
+        remaining -= dstLen;
+    }

 
 

-    options    = superMap->getMapOptions();
-    logical    = superMap->getVirtualAddress() + offset;
+    if (kIOMemoryThreadSafe & _flags)
+       UNLOCK;

 
 

-    return( true );
+    return (remaining ? kIOReturnUnderrun : kIOReturnSuccess);

 }
 
 }
 

-bool _IOMemoryMap::initWithDescriptor(
-        IOMemoryDescriptor *   _memory,
-        task_t                 intoTask,
-        IOVirtualAddress       toAddress,
-        IOOptionBits           _options,
-        IOByteCount            _offset,
-        IOByteCount            _length )
+#if defined(__i386__) || defined(__x86_64__)
+extern vm_offset_t             first_avail;
+#define io_kernel_static_end   first_avail
+#else
+#error io_kernel_static_end is undefined for this architecture
+#endif
+
+static kern_return_t
+io_get_kernel_static_upl(
+       vm_map_t                /* map */,
+       uintptr_t               offset,
+       vm_size_t               *upl_size,
+       upl_t                   *upl,
+       upl_page_info_array_t   page_list,
+       unsigned int            *count,
+       ppnum_t                 *highest_page)

 {
 {

-    bool ok;
-    bool redir = ((kIOMapUnique|kIOMapReference) == ((kIOMapUnique|kIOMapReference) & _options));
+    unsigned int pageCount, page;
+    ppnum_t phys;
+    ppnum_t highestPage = 0;

 
 

-    if ((!_memory) || (!intoTask))
-       return( false);
+    pageCount = atop_32(*upl_size);
+    if (pageCount > *count)
+       pageCount = *count;

 
 

-    if( (_offset + _length) > _memory->getLength())
-       return( false);
+    *upl = NULL;

 
 

-    if (!redir)
+    for (page = 0; page < pageCount; page++)

     {
     {

-       if (!super::init())
-           return(false);
-       addressMap  = get_task_map(intoTask);
-       if( !addressMap)
-           return( false);
-       vm_map_reference(addressMap);
-       addressTask = intoTask;
-       logical     = toAddress;
-       options     = _options;
+       phys = pmap_find_phys(kernel_pmap, ((addr64_t)offset) + ptoa_64(page));
+       if (!phys)
+           break;
+       page_list[page].phys_addr = phys;
+       page_list[page].pageout   = 0;
+       page_list[page].absent    = 0;
+       page_list[page].dirty     = 0;
+       page_list[page].precious  = 0;
+       page_list[page].device    = 0;
+       if (phys > highestPage)
+           highestPage = phys;

     }
 
     }
 

-    _memory->retain();
-
-    offset     = _offset;
-    if( _length)
-        length = _length;
-    else
-        length = _memory->getLength();
-
-    if( options & kIOMapStatic)
-       ok = true;
-    else
-       ok = (kIOReturnSuccess == _memory->doMap( addressMap, &toAddress,
-                                                 _options, offset, length ));
-    if (ok || redir)
-    {
-       if (memory)
-           memory->release();
-       memory  = _memory;
-       logical = toAddress;
-    }
-    else
-    {
-        _memory->release();
-       if (!redir)
-       {
-           logical = 0;
-           memory = 0;
-           vm_map_deallocate(addressMap);
-           addressMap = 0;
-       }
-    }
+    *highest_page = highestPage;

 
 

-    return( ok );
+    return ((page >= pageCount) ? kIOReturnSuccess : kIOReturnVMError);

 }
 
 }
 

-/* LP64todo - these need to expand */
-struct IOMemoryDescriptorMapAllocRef
+IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection)

 {
 {

-    ipc_port_t         sharedMem;
-    vm_size_t          size;
-    vm_offset_t                mapped;
-    IOByteCount                sourceOffset;
-    IOOptionBits       options;
-};
+    IOOptionBits type = _flags & kIOMemoryTypeMask;
+    IOReturn error = kIOReturnCannotWire;
+    ioGMDData *dataP;
+    upl_page_info_array_t pageInfo;
+    ppnum_t mapBase;

 
 

-static kern_return_t IOMemoryDescriptorMapAlloc(vm_map_t map, void * _ref)
-{
-    IOMemoryDescriptorMapAllocRef * ref = (IOMemoryDescriptorMapAllocRef *)_ref;
-    IOReturn                       err;
+    assert(kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type || kIOMemoryTypeUIO == type);

 
 

-    do {
-        if( ref->sharedMem) {
-            vm_prot_t prot = VM_PROT_READ
-                            | ((ref->options & kIOMapReadOnly) ? 0 : VM_PROT_WRITE);
+    if ((kIODirectionOutIn & forDirection) == kIODirectionNone)
+        forDirection = (IODirection) (forDirection | getDirection());

 
 

-            // set memory entry cache
-            vm_prot_t memEntryCacheMode = prot | MAP_MEM_ONLY;
-            switch (ref->options & kIOMapCacheMask)
-            {
-               case kIOMapInhibitCache:
-                    SET_MAP_MEM(MAP_MEM_IO, memEntryCacheMode);
-                    break;
-       
-               case kIOMapWriteThruCache:
-                    SET_MAP_MEM(MAP_MEM_WTHRU, memEntryCacheMode);
-                    break;
-
-               case kIOMapWriteCombineCache:
-                    SET_MAP_MEM(MAP_MEM_WCOMB, memEntryCacheMode);
-                    break;
-
-               case kIOMapCopybackCache:
-                    SET_MAP_MEM(MAP_MEM_COPYBACK, memEntryCacheMode);
-                    break;
-
-               case kIOMapDefaultCache:
-               default:
-                    SET_MAP_MEM(MAP_MEM_NOOP, memEntryCacheMode);
-                    break;
-            }
+    int uplFlags;    // This Mem Desc's default flags for upl creation
+    switch (kIODirectionOutIn & forDirection)
+    {
+    case kIODirectionOut:
+        // Pages do not need to be marked as dirty on commit
+        uplFlags = UPL_COPYOUT_FROM;
+        break;
+
+    case kIODirectionIn:
+    default:
+        uplFlags = 0;  // i.e. ~UPL_COPYOUT_FROM
+        break;
+    }

 
 

-            vm_size_t unused = 0;
-
-            err = mach_make_memory_entry( NULL /*unused*/, &unused, 0 /*unused*/, 
-                                            memEntryCacheMode, NULL, ref->sharedMem );
-            if (KERN_SUCCESS != err)
-                IOLog("MAP_MEM_ONLY failed %d\n", err);
-
-            err = vm_map( map,
-                            &ref->mapped,
-                            ref->size, 0 /* mask */, 
-                            (( ref->options & kIOMapAnywhere ) ? VM_FLAGS_ANYWHERE : VM_FLAGS_FIXED)
-                            | VM_MAKE_TAG(VM_MEMORY_IOKIT), 
-                            ref->sharedMem, ref->sourceOffset,
-                            false, // copy
-                            prot, // cur
-                            prot, // max
-                            VM_INHERIT_NONE);
-
-            if( KERN_SUCCESS != err) {
-                ref->mapped = 0;
-                continue;
-            }
-    
-        } else {
-    
-            err = vm_allocate( map, &ref->mapped, ref->size,
-                            ((ref->options & kIOMapAnywhere) ? VM_FLAGS_ANYWHERE : VM_FLAGS_FIXED)
-                            | VM_MAKE_TAG(VM_MEMORY_IOKIT) );
-    
-            if( KERN_SUCCESS != err) {
-                ref->mapped = 0;
-                continue;
-            }
-    
-            // we have to make sure that these guys don't get copied if we fork.
-            err = vm_inherit( map, ref->mapped, ref->size, VM_INHERIT_NONE);
-            assert( KERN_SUCCESS == err );
+    if (_wireCount)
+    {
+        if ((kIOMemoryPreparedReadOnly & _flags) && !(UPL_COPYOUT_FROM & uplFlags))
+        {
+           OSReportWithBacktrace("IOMemoryDescriptor 0x%lx prepared read only", VM_KERNEL_ADDRPERM(this));
+           error = kIOReturnNotWritable;

         }
         }

+        else error = kIOReturnSuccess;
+       return (error);
+    }

 
 

-    } while( false );
+    dataP = getDataP(_memoryEntries);
+    IOMapper *mapper;
+    mapper = dataP->fMapper;
+    dataP->fMappedBase = 0;

 
 

-    return( err );
-}
+    uplFlags |= UPL_SET_IO_WIRE | UPL_SET_LITE;
+    if (kIODirectionPrepareToPhys32 & forDirection) 
+    {
+       if (!mapper) uplFlags |= UPL_NEED_32BIT_ADDR;
+       if (dataP->fDMAMapNumAddressBits > 32) dataP->fDMAMapNumAddressBits = 32;
+    }
+    if (kIODirectionPrepareNoFault    & forDirection) uplFlags |= UPL_REQUEST_NO_FAULT;
+    if (kIODirectionPrepareNoZeroFill & forDirection) uplFlags |= UPL_NOZEROFILLIO;
+    if (kIODirectionPrepareNonCoherent & forDirection) uplFlags |= UPL_REQUEST_FORCE_COHERENCY;

 
 

+    mapBase = 0;

 
 

-IOReturn IOMemoryDescriptor::doMap(
-       vm_map_t                addressMap,
-       IOVirtualAddress *      atAddress,
-       IOOptionBits            options,
-       IOByteCount             sourceOffset,
-       IOByteCount             length )
-{
-    IOReturn           err = kIOReturnSuccess;
-    memory_object_t    pager;
-    vm_address_t       logical;
-    IOByteCount                pageOffset;
-    IOPhysicalAddress  sourceAddr;
-    IOMemoryDescriptorMapAllocRef      ref;
+    // Note that appendBytes(NULL) zeros the data up to the desired length
+    //           and the length parameter is an unsigned int
+    size_t uplPageSize = dataP->fPageCnt * sizeof(upl_page_info_t);
+    if (uplPageSize > ((unsigned int)uplPageSize))    return (kIOReturnNoMemory);
+    if (!_memoryEntries->appendBytes(0, uplPageSize)) return (kIOReturnNoMemory);
+    dataP = 0;

 
 

-    ref.sharedMem      = (ipc_port_t) _memEntry;
-    ref.sourceOffset   = sourceOffset;
-    ref.options                = options;
+    // Find the appropriate vm_map for the given task
+    vm_map_t curMap;
+    if (_task == kernel_task && (kIOMemoryBufferPageable & _flags))            curMap = 0;
+    else                                                     curMap = get_task_map(_task);

 
 

-    do {
+    // Iterate over the vector of virtual ranges
+    Ranges vec = _ranges;
+    unsigned int pageIndex  = 0;
+    IOByteCount mdOffset    = 0;
+    ppnum_t highestPage     = 0;

 
 

-        if( 0 == length)
-            length = getLength();
+    IOMemoryEntry * memRefEntry = 0;
+    if (_memRef) memRefEntry = &_memRef->entries[0];

 
 

-       sourceAddr = getSourceSegment( sourceOffset, NULL );
-       pageOffset = sourceAddr - trunc_page_32( sourceAddr );
+    for (UInt range = 0; range < _rangesCount; range++) {
+        ioPLBlock iopl;
+       mach_vm_address_t startPage;
+        mach_vm_size_t    numBytes;
+       ppnum_t highPage = 0;

 
 

-       ref.size = round_page_32( length + pageOffset );
+       // Get the startPage address and length of vec[range]
+       getAddrLenForInd(startPage, numBytes, type, vec, range);
+       iopl.fPageOffset = startPage & PAGE_MASK;
+       numBytes += iopl.fPageOffset;
+       startPage = trunc_page_64(startPage);

 
 

-       if ((kIOMapReference|kIOMapUnique) == ((kIOMapReference|kIOMapUnique) & options))
-       {
-           upl_t         redirUPL2;
-           vm_size_t     size;
-           int           flags;
+       if (mapper)
+           iopl.fMappedPage = mapBase + pageIndex;
+       else
+           iopl.fMappedPage = 0;

 
 

-           _IOMemoryMap * mapping = (_IOMemoryMap *) *atAddress;
-           ref.mapped = mapping->getVirtualAddress();
-    
-           if (!_memEntry)
+       // Iterate over the current range, creating UPLs
+        while (numBytes) {
+           vm_address_t kernelStart = (vm_address_t) startPage;
+            vm_map_t theMap;
+           if (curMap) theMap = curMap;
+           else if (_memRef)

            {
            {

-               err = kIOReturnNotReadable;
-               continue;
+               theMap = NULL;

            }
            }

-
-           size = length;
-           flags = UPL_COPYOUT_FROM | UPL_SET_INTERNAL 
-                       | UPL_SET_LITE | UPL_SET_IO_WIRE | UPL_BLOCK_ACCESS;
-
-           if (KERN_SUCCESS != memory_object_iopl_request((ipc_port_t) _memEntry, 0, &size, &redirUPL2,
-                                           NULL, NULL,
-                                           &flags))
-               redirUPL2 = NULL;
-
-           err = upl_transpose(redirUPL2, mapping->redirUPL);
-           if (kIOReturnSuccess != err)
+           else

            {
            {

-               IOLog("upl_transpose(%x)\n", err);
-               err = kIOReturnSuccess;
+               assert(_task == kernel_task);
+               theMap = IOPageableMapForAddress(kernelStart);

            }
 
            }
 

-           if (redirUPL2)
-           {
-               upl_commit(redirUPL2, NULL, 0);
-               upl_deallocate(redirUPL2);
-               redirUPL2 = 0;
+            int ioplFlags = uplFlags;
+           dataP = getDataP(_memoryEntries);
+           pageInfo = getPageList(dataP);
+            upl_page_list_ptr_t baseInfo = &pageInfo[pageIndex];
+
+            vm_size_t ioplSize = round_page(numBytes);
+            unsigned int numPageInfo = atop_32(ioplSize);
+
+           if ((theMap == kernel_map) && (kernelStart < io_kernel_static_end)) {
+               error = io_get_kernel_static_upl(theMap, 
+                                               kernelStart,
+                                               &ioplSize,
+                                               &iopl.fIOPL,
+                                               baseInfo,
+                                               &numPageInfo,
+                                               &highPage);

            }
            }

-           {
-               // swap the memEntries since they now refer to different vm_objects
-               void * me = _memEntry;
-               _memEntry = mapping->memory->_memEntry;
-               mapping->memory->_memEntry = me;
+           else if (_memRef) {
+               memory_object_offset_t entryOffset;
+
+               entryOffset = (mdOffset - iopl.fPageOffset - memRefEntry->offset);
+               if (entryOffset >= memRefEntry->size) {
+                   memRefEntry++;
+                   if (memRefEntry >= &_memRef->entries[_memRef->count]) panic("memRefEntry");
+                   entryOffset = 0;
+               }
+               if (ioplSize > (memRefEntry->size - entryOffset)) ioplSize = (memRefEntry->size - entryOffset);
+               error = memory_object_iopl_request(memRefEntry->entry, 
+                                                  entryOffset,
+                                                  &ioplSize,
+                                                  &iopl.fIOPL,
+                                                  baseInfo,
+                                                  &numPageInfo,
+                                                  &ioplFlags);

            }
            }

-       }
-       else
-       {
-    
-           logical = *atAddress;
-           if( options & kIOMapAnywhere) 
-               // vm_map looks for addresses above here, even when VM_FLAGS_ANYWHERE
-               ref.mapped = 0;

            else {
            else {

-               ref.mapped = trunc_page_32( logical );
-               if( (logical - ref.mapped) != pageOffset) {
-                   err = kIOReturnVMError;
-                   continue;
-               }
+               assert(theMap);
+               error = vm_map_create_upl(theMap,
+                                               startPage,
+                                               (upl_size_t*)&ioplSize,
+                                               &iopl.fIOPL,
+                                               baseInfo,
+                                               &numPageInfo,
+                                               &ioplFlags);

            }
            }

-    
-           if( ref.sharedMem && (addressMap == kernel_map) && (kIOMemoryBufferPageable & _flags))
-               err = IOIteratePageableMaps( ref.size, &IOMemoryDescriptorMapAlloc, &ref );
-           else
-               err = IOMemoryDescriptorMapAlloc( addressMap, &ref );
-       }

 
 

-        if( err != KERN_SUCCESS)
-            continue;
+            assert(ioplSize);
+            if (error != KERN_SUCCESS)
+                goto abortExit;

 
 

-        if( reserved)
-            pager = (memory_object_t) reserved->devicePager;
-        else
-            pager = MACH_PORT_NULL;
+           if (iopl.fIOPL)
+               highPage = upl_get_highest_page(iopl.fIOPL);
+           if (highPage > highestPage)
+               highestPage = highPage;

 
 

-        if( !ref.sharedMem || pager )
-            err = handleFault( pager, addressMap, ref.mapped, sourceOffset, length, options );
+            error = kIOReturnCannotWire;

 
 

-    } while( false );
+            if (baseInfo->device) {
+                numPageInfo = 1;
+                iopl.fFlags = kIOPLOnDevice;
+            }
+            else {
+                iopl.fFlags = 0;
+            }

 
 

-    if( err != KERN_SUCCESS) {
-        if( ref.mapped)
-            doUnmap( addressMap, ref.mapped, ref.size );
-        *atAddress = NULL;
-    } else
-        *atAddress = ref.mapped + pageOffset;
+            iopl.fIOMDOffset = mdOffset;
+            iopl.fPageInfo = pageIndex;
+            if (mapper && pageIndex && (page_mask & (mdOffset + iopl.fPageOffset))) dataP->fDiscontig = true;

 
 

-    return( err );
-}
+#if 0
+           // used to remove the upl for auto prepares here, for some errant code
+           // that freed memory before the descriptor pointing at it
+           if ((_flags & kIOMemoryAutoPrepare) && iopl.fIOPL)
+           {
+               upl_commit(iopl.fIOPL, 0, 0);
+               upl_deallocate(iopl.fIOPL);
+               iopl.fIOPL = 0;
+           }
+#endif

 
 

-enum {
-    kIOMemoryRedirected        = 0x00010000
-};
+            if (!_memoryEntries->appendBytes(&iopl, sizeof(iopl))) {
+                // Clean up partial created and unsaved iopl
+                if (iopl.fIOPL) {
+                    upl_abort(iopl.fIOPL, 0);
+                    upl_deallocate(iopl.fIOPL);
+                }
+                goto abortExit;
+            }
+           dataP = 0;

 
 

-IOReturn IOMemoryDescriptor::handleFault(
-        void *                 _pager,
-       vm_map_t                addressMap,
-       IOVirtualAddress        address,
-       IOByteCount             sourceOffset,
-       IOByteCount             length,
-        IOOptionBits           options )
-{
-    IOReturn           err = kIOReturnSuccess;
-    memory_object_t    pager = (memory_object_t) _pager;
-    vm_size_t          size;
-    vm_size_t          bytes;
-    vm_size_t          page;
-    IOByteCount                pageOffset;
-    IOByteCount                pagerOffset;
-    IOPhysicalLength   segLen;
-    addr64_t           physAddr;
+            // Check for a multiple iopl's in one virtual range
+            pageIndex += numPageInfo;
+            mdOffset -= iopl.fPageOffset;
+            if (ioplSize < numBytes) {
+                numBytes -= ioplSize;
+                startPage += ioplSize;
+                mdOffset += ioplSize;
+                iopl.fPageOffset = 0;
+               if (mapper) iopl.fMappedPage = mapBase + pageIndex;
+            }
+            else {
+                mdOffset += numBytes;
+                break;
+            }
+        }
+    }

 
 

-    if( !addressMap) {
+    _highestPage = highestPage;

 
 

-        if( kIOMemoryRedirected & _flags) {
-#ifdef DEBUG
-            IOLog("sleep mem redirect %p, %lx\n", this, sourceOffset);
-#endif
-            do {
-               SLEEP;
-            } while( kIOMemoryRedirected & _flags );
-        }
+    if (UPL_COPYOUT_FROM & uplFlags) _flags |= kIOMemoryPreparedReadOnly;

 
 

-        return( kIOReturnSuccess );
-    }
+    return kIOReturnSuccess;

 
 

-    physAddr = getPhysicalSegment64( sourceOffset, &segLen );
-    assert( physAddr );
-    pageOffset = physAddr - trunc_page_64( physAddr );
-    pagerOffset = sourceOffset;
+abortExit:
+    {
+        dataP = getDataP(_memoryEntries);
+        UInt done = getNumIOPL(_memoryEntries, dataP);
+        ioPLBlock *ioplList = getIOPLList(dataP);
+    
+        for (UInt range = 0; range < done; range++)
+       {
+           if (ioplList[range].fIOPL) {
+             upl_abort(ioplList[range].fIOPL, 0);
+             upl_deallocate(ioplList[range].fIOPL);
+           }
+       }
+       (void) _memoryEntries->initWithBytes(dataP, computeDataSize(0, 0)); // == setLength()
+    }

 
 

-    size = length + pageOffset;
-    physAddr -= pageOffset;
+    if (error == KERN_FAILURE)
+        error = kIOReturnCannotWire;
+    else if (error == KERN_MEMORY_ERROR)
+        error = kIOReturnNoResources;

 
 

-    segLen += pageOffset;
-    bytes = size;
-    do {
-       // in the middle of the loop only map whole pages
-       if( segLen >= bytes)
-           segLen = bytes;
-       else if( segLen != trunc_page_32( segLen))
-           err = kIOReturnVMError;
-        if( physAddr != trunc_page_64( physAddr))
-           err = kIOReturnBadArgument;
+    return error;
+}

 
 

-#ifdef DEBUG
-       if( kIOLogMapping & gIOKitDebug)
-           IOLog("_IOMemoryMap::map(%p) %08lx->%08qx:%08lx\n",
-                addressMap, address + pageOffset, physAddr + pageOffset,
-               segLen - pageOffset);
-#endif
+bool IOGeneralMemoryDescriptor::initMemoryEntries(size_t size, IOMapper * mapper)
+{
+    ioGMDData * dataP;
+    unsigned    dataSize = size;

 
 

-        if( pager) {
-            if( reserved && reserved->pagerContig) {
-                IOPhysicalLength       allLen;
-                addr64_t               allPhys;
+    if (!_memoryEntries) {
+       _memoryEntries = OSData::withCapacity(dataSize);
+       if (!_memoryEntries)
+           return false;
+    }
+    else if (!_memoryEntries->initWithCapacity(dataSize))
+       return false;

 
 

-                allPhys = getPhysicalSegment64( 0, &allLen );
-                assert( allPhys );
-                err = device_pager_populate_object( pager, 0, allPhys >> PAGE_SHIFT, round_page_32(allLen) );
+    _memoryEntries->appendBytes(0, computeDataSize(0, 0));
+    dataP = getDataP(_memoryEntries);

 
 

-            } else {
+    if (mapper == kIOMapperWaitSystem) {
+        IOMapper::checkForSystemMapper();
+        mapper = IOMapper::gSystem;
+    }
+    dataP->fMapper               = mapper;
+    dataP->fPageCnt              = 0;
+    dataP->fMappedBase           = 0;
+    dataP->fDMAMapNumAddressBits = 64;
+    dataP->fDMAMapAlignment      = 0;
+    dataP->fPreparationID        = kIOPreparationIDUnprepared;
+    dataP->fDiscontig            = false;
+    dataP->fCompletionError      = false;
+
+    return (true);
+}

 
 

-           for( page = 0;
-                     (page < segLen) && (KERN_SUCCESS == err);
-                     page += page_size) {
-                        err = device_pager_populate_object(pager, pagerOffset,
-                               (ppnum_t)((physAddr + page) >> PAGE_SHIFT), page_size);
-                       pagerOffset += page_size;
-                }
-            }
-            assert( KERN_SUCCESS == err );
-            if( err)
-                break;
-        }
+IOReturn IOMemoryDescriptor::dmaMap(
+    IOMapper                    * mapper,
+    const IODMAMapSpecification * mapSpec,
+    uint64_t                      offset,
+    uint64_t                      length,
+    uint64_t                    * address,
+    ppnum_t                     * mapPages)
+{
+    IOMDDMAWalkSegmentState  walkState;
+    IOMDDMAWalkSegmentArgs * walkArgs = (IOMDDMAWalkSegmentArgs *) (void *)&walkState;
+    IOOptionBits             mdOp;
+    IOReturn                 ret;
+    IOPhysicalLength         segLen;
+    addr64_t                 phys, align, pageOffset;
+    ppnum_t                  base, pageIndex, pageCount;
+    uint64_t                 index;
+    uint32_t                 mapOptions = 0;
+
+    if (!(kIOMemoryPreparedReadOnly & _flags)) mapOptions |= kIODMAMapWriteAccess;
+
+    walkArgs->fMapped = false;
+    mdOp = kIOMDFirstSegment;
+    pageCount = 0;
+    for (index = 0; index < length; )
+    {
+       if (index && (page_mask & (index + pageOffset))) break;
+
+       walkArgs->fOffset = offset + index;
+       ret = dmaCommandOperation(mdOp, &walkState, sizeof(walkState));
+       mdOp = kIOMDWalkSegments;
+       if (ret != kIOReturnSuccess) break;
+       phys = walkArgs->fIOVMAddr;
+       segLen = walkArgs->fLength;
+
+       align = (phys & page_mask);
+       if (!index) pageOffset = align;
+       else if (align) break;
+       pageCount += atop_64(round_page_64(align + segLen));
+       index += segLen;
+    }

 
 

-       /*  *** ALERT *** */
-       /*  *** Temporary Workaround *** */
-
-       /* This call to vm_fault causes an early pmap level resolution  */
-       /* of the mappings created above.  Need for this is in absolute */
-       /* violation of the basic tenet that the pmap layer is a cache. */
-       /* Further, it implies a serious I/O architectural violation on */
-       /* the part of some user of the mapping.  As of this writing,   */
-       /* the call to vm_fault is needed because the NVIDIA driver     */
-       /* makes a call to pmap_extract.  The NVIDIA driver needs to be */
-       /* fixed as soon as possible.  The NVIDIA driver should not     */
-       /* need to query for this info as it should know from the doMap */
-       /* call where the physical memory is mapped.  When a query is   */
-       /* necessary to find a physical mapping, it should be done      */
-       /* through an iokit call which includes the mapped memory       */
-       /* handle.  This is required for machine architecture independence.*/
-
-       if(!(kIOMemoryRedirected & _flags)) {
-               vm_fault(addressMap, 
-                        (vm_map_offset_t)address, 
-                        VM_PROT_READ|VM_PROT_WRITE, 
-                        FALSE, THREAD_UNINT, NULL, 
-                        (vm_map_offset_t)0);
-       }
+    if (index < length) return (kIOReturnVMError);

 
 

-       /*  *** Temporary Workaround *** */
-       /*  *** ALERT *** */
+    base = mapper->iovmMapMemory(this, offset, pageCount, 
+                                mapOptions, NULL, mapSpec);

 
 

-       sourceOffset += segLen - pageOffset;
-       address += segLen;
-       bytes -= segLen;
-       pageOffset = 0;
+    if (!base) return (kIOReturnNoResources);

 
 

-    } while( bytes
-       && (physAddr = getPhysicalSegment64( sourceOffset, &segLen )));
+    mdOp = kIOMDFirstSegment;
+    for (pageIndex = 0, index = 0; index < length; )
+    {
+       walkArgs->fOffset = offset + index;
+       ret = dmaCommandOperation(mdOp, &walkState, sizeof(walkState));
+       mdOp = kIOMDWalkSegments;
+       if (ret != kIOReturnSuccess) break;
+       phys = walkArgs->fIOVMAddr;
+       segLen = walkArgs->fLength;
+
+       ppnum_t page = atop_64(phys);
+       ppnum_t count = atop_64(round_page_64(phys + segLen)) - page;
+       while (count--)
+       {
+           mapper->iovmInsert(base, pageIndex, page);
+           page++;
+           pageIndex++;
+       }
+       index += segLen;
+    }
+    if (pageIndex != pageCount) panic("pageIndex");

 
 

-    if( bytes)
-        err = kIOReturnBadArgument;
+    *address = ptoa_64(base) + pageOffset;
+    if (mapPages) *mapPages = pageCount;

 
 

-    return( err );
+    return (kIOReturnSuccess);

 }
 
 }
 

-IOReturn IOMemoryDescriptor::doUnmap(
-       vm_map_t                addressMap,
-       IOVirtualAddress        logical,
-       IOByteCount             length )
+IOReturn IOGeneralMemoryDescriptor::dmaMap(
+    IOMapper                    * mapper,
+    const IODMAMapSpecification * mapSpec,
+    uint64_t                      offset,
+    uint64_t                      length,
+    uint64_t                    * address,
+    ppnum_t                     * mapPages)

 {
 {

-    IOReturn   err;
+    IOReturn          err = kIOReturnSuccess;
+    ioGMDData *       dataP;
+    IOOptionBits      type = _flags & kIOMemoryTypeMask;

 
 

-#ifdef DEBUG
-    if( kIOLogMapping & gIOKitDebug)
-       kprintf("IOMemoryDescriptor::doUnmap(%x) %08x:%08x\n",
-                addressMap, logical, length );
-#endif
+    *address = 0;
+    if (kIOMemoryHostOnly & _flags) return (kIOReturnSuccess);

 
 

-    if( true /* && (addressMap == kernel_map) || (addressMap == get_task_map(current_task()))*/) {
+    if ((type == kIOMemoryTypePhysical) || (type == kIOMemoryTypePhysical64)
+     || offset || (length != _length))
+    {
+       err = super::dmaMap(mapper, mapSpec, offset, length, address, mapPages);
+    }
+    else if (_memoryEntries && _pages && (dataP = getDataP(_memoryEntries)))
+    {
+       const ioPLBlock * ioplList = getIOPLList(dataP);
+       upl_page_info_t * pageList;
+       uint32_t          mapOptions = 0;
+       ppnum_t           base;

 
 

-        if( _memEntry && (addressMap == kernel_map) && (kIOMemoryBufferPageable & _flags))
-            addressMap = IOPageableMapForAddress( logical );
+       IODMAMapSpecification mapSpec;
+       bzero(&mapSpec, sizeof(mapSpec));
+       mapSpec.numAddressBits = dataP->fDMAMapNumAddressBits;
+       mapSpec.alignment = dataP->fDMAMapAlignment;

 
 

-        err = vm_deallocate( addressMap, logical, length );
+       // For external UPLs the fPageInfo field points directly to
+       // the upl's upl_page_info_t array.
+       if (ioplList->fFlags & kIOPLExternUPL)
+       {
+           pageList = (upl_page_info_t *) ioplList->fPageInfo;
+           mapOptions |= kIODMAMapPagingPath;
+       }
+       else
+           pageList = getPageList(dataP);

 
 

-    } else
-        err = kIOReturnSuccess;
+    if (!(kIOMemoryPreparedReadOnly & _flags)) mapOptions |= kIODMAMapWriteAccess;

 
 

-    return( err );
-}
+       // Check for direct device non-paged memory
+       if (ioplList->fFlags & kIOPLOnDevice) mapOptions |= kIODMAMapPhysicallyContiguous;

 
 

-IOReturn IOMemoryDescriptor::redirect( task_t safeTask, bool doRedirect )
-{
-    IOReturn           err = kIOReturnSuccess;
-    _IOMemoryMap *     mapping = 0;
-    OSIterator *       iter;
+       base = mapper->iovmMapMemory(
+                       this, offset, _pages, mapOptions, &pageList[0], &mapSpec);
+       *address = ptoa_64(base) + (ioplList->fPageOffset & PAGE_MASK);
+       if (mapPages) *mapPages = _pages;
+    }

 
 

-    LOCK;
+    return (err);
+}

 
 

-    if( doRedirect)
-        _flags |= kIOMemoryRedirected;
-    else
-        _flags &= ~kIOMemoryRedirected;
+/*
+ * prepare
+ *
+ * Prepare the memory for an I/O transfer.  This involves paging in
+ * the memory, if necessary, and wiring it down for the duration of
+ * the transfer.  The complete() method completes the processing of
+ * the memory after the I/O transfer finishes.  This method needn't
+ * called for non-pageable memory.
+ */

 
 

-    do {
-       if( (iter = OSCollectionIterator::withCollection( _mappings))) {
-           while( (mapping = (_IOMemoryMap *) iter->getNextObject()))
-               mapping->redirect( safeTask, doRedirect );
+IOReturn IOGeneralMemoryDescriptor::prepare(IODirection forDirection)
+{
+    IOReturn error    = kIOReturnSuccess;
+    IOOptionBits type = _flags & kIOMemoryTypeMask;

 
 

-           iter->release();
-       }
-    } while( false );
+    if ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type))
+       return kIOReturnSuccess;

 
 

-    if (!doRedirect)
+    if (_prepareLock)
+       IOLockLock(_prepareLock);
+
+    if (kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type || kIOMemoryTypeUIO == type)

     {
     {

-        WAKEUP;
+       error = wireVirtual(forDirection);

     }
 
     }
 

-    UNLOCK;
+    if (kIOReturnSuccess == error)
+    {
+       if (1 == ++_wireCount)
+       {
+           if (kIOMemoryClearEncrypt & _flags)
+           {
+               performOperation(kIOMemoryClearEncrypted, 0, _length);
+           }
+       }
+    }

 
 

-    // temporary binary compatibility
-    IOSubMemoryDescriptor * subMem;
-    if( (subMem = OSDynamicCast( IOSubMemoryDescriptor, this)))
-       err = subMem->redirect( safeTask, doRedirect );
-    else
-       err = kIOReturnSuccess;
+    if (_prepareLock)
+       IOLockUnlock(_prepareLock);

 
 

-    return( err );
+    return error;

 }
 
 }
 

-IOReturn IOSubMemoryDescriptor::redirect( task_t safeTask, bool doRedirect )
-{
-    return( _parent->redirect( safeTask, doRedirect ));
-}
+/*
+ * complete
+ *
+ * Complete processing of the memory after an I/O transfer finishes.
+ * This method should not be called unless a prepare was previously
+ * issued; the prepare() and complete() must occur in pairs, before
+ * before and after an I/O transfer involving pageable memory.
+ */

 
 

-IOReturn _IOMemoryMap::redirect( task_t safeTask, bool doRedirect )
+IOReturn IOGeneralMemoryDescriptor::complete(IODirection forDirection)

 {
 {

-    IOReturn err = kIOReturnSuccess;
+    IOOptionBits type = _flags & kIOMemoryTypeMask;
+    ioGMDData * dataP;

 
 

-    if( superMap) {
-//        err = ((_IOMemoryMap *)superMap)->redirect( safeTask, doRedirect );
-    } else {
+    if ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type))
+       return kIOReturnSuccess;

 
 

-        LOCK;
+    if (_prepareLock)
+       IOLockLock(_prepareLock);

 
 

-       do
+    assert(_wireCount);
+
+    if ((kIODirectionCompleteWithError & forDirection) 
+     && (dataP = getDataP(_memoryEntries)))
+        dataP->fCompletionError = true;
+
+    if (_wireCount)
+    {
+        if ((kIOMemoryClearEncrypt & _flags) && (1 == _wireCount))
+        {
+            performOperation(kIOMemorySetEncrypted, 0, _length);
+        }
+
+       _wireCount--;
+       if (!_wireCount || (kIODirectionCompleteWithDataValid & forDirection))

        {
        {

-           if (!logical)
-               break;
-           if (!addressMap)
-               break;
+           IOOptionBits type = _flags & kIOMemoryTypeMask;
+           dataP = getDataP(_memoryEntries);
+           ioPLBlock *ioplList = getIOPLList(dataP);
+           UInt ind, count = getNumIOPL(_memoryEntries, dataP);

 
 

-           if ((!safeTask || (get_task_map(safeTask) != addressMap))
-             && (0 == (options & kIOMapStatic)))
+           if (_wireCount)

            {
            {

-               IOUnmapPages( addressMap, logical, length );
-               if(!doRedirect && safeTask
-                && (((memory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical) 
-                   || ((memory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical64)))
-                {
-                   err = vm_deallocate( addressMap, logical, length );
-                   err = memory->doMap( addressMap, &logical,
-                                        (options & ~kIOMapAnywhere) /*| kIOMapReserve*/,
-                                        offset, length );
-               } else
-                   err = kIOReturnSuccess;
-#ifdef DEBUG
-               IOLog("IOMemoryMap::redirect(%d, %p) %x:%lx from %p\n", doRedirect, this, logical, length, addressMap);
-#endif
+               // kIODirectionCompleteWithDataValid & forDirection
+               if (kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type || kIOMemoryTypeUIO == type)
+               {
+                   for (ind = 0; ind < count; ind++)
+                   {
+                       if (ioplList[ind].fIOPL) iopl_valid_data(ioplList[ind].fIOPL);
+                   }
+               }

            }
            }

-           else if (kIOMapWriteCombineCache == (options & kIOMapCacheMask))
+           else

            {
            {

-               IOOptionBits newMode;
-               newMode = (options & ~kIOMapCacheMask) | (doRedirect ? kIOMapInhibitCache : kIOMapWriteCombineCache);
-               IOProtectCacheMode(addressMap, logical, length, newMode);
+#if IOMD_DEBUG_DMAACTIVE
+               if (__iomd_reservedA) panic("complete() while dma active");
+#endif /* IOMD_DEBUG_DMAACTIVE */
+
+               if (dataP->fMappedBase) {
+                   dataP->fMapper->iovmFree(atop_64(dataP->fMappedBase), _pages);
+                   dataP->fMappedBase = 0;
+               }
+               // Only complete iopls that we created which are for TypeVirtual
+               if (kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type || kIOMemoryTypeUIO == type) {
+                   for (ind = 0; ind < count; ind++)
+                       if (ioplList[ind].fIOPL) {
+                           if (dataP->fCompletionError) 
+                               upl_abort(ioplList[ind].fIOPL, 0 /*!UPL_ABORT_DUMP_PAGES*/);
+                           else
+                               upl_commit(ioplList[ind].fIOPL, 0, 0);
+                           upl_deallocate(ioplList[ind].fIOPL);
+                       }
+               } else if (kIOMemoryTypeUPL == type) {
+                   upl_set_referenced(ioplList[0].fIOPL, false);
+               }
+
+               (void) _memoryEntries->initWithBytes(dataP, computeDataSize(0, 0)); // == setLength()
+
+               dataP->fPreparationID = kIOPreparationIDUnprepared;

            }
        }
            }
        }

-       while (false);
-
-       UNLOCK;

     }
 
     }
 

-    if ((((memory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical)
-        || ((memory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical64))
-     && safeTask
-     && (doRedirect != (0 != (memory->_flags & kIOMemoryRedirected))))
-       memory->redirect(safeTask, doRedirect);
+    if (_prepareLock)
+       IOLockUnlock(_prepareLock);

 
 

-    return( err );
+    return kIOReturnSuccess;

 }
 
 }
 

-IOReturn _IOMemoryMap::unmap( void )
+IOReturn IOGeneralMemoryDescriptor::doMap(
+       vm_map_t                __addressMap,
+       IOVirtualAddress *      __address,
+       IOOptionBits            options,
+       IOByteCount             __offset,
+       IOByteCount             __length )
+

 {
 {

-    IOReturn   err;
+#ifndef __LP64__
+    if (!(kIOMap64Bit & options)) panic("IOGeneralMemoryDescriptor::doMap !64bit");
+#endif /* !__LP64__ */

 
 

-    LOCK;
+    kern_return_t  err;

 
 

-    if( logical && addressMap && (0 == superMap)
-       && (0 == (options & kIOMapStatic))) {
+    IOMemoryMap *  mapping = (IOMemoryMap *) *__address;
+    mach_vm_size_t offset  = mapping->fOffset + __offset;
+    mach_vm_size_t length  = mapping->fLength;

 
 

-        err = memory->doUnmap( addressMap, logical, length );
-        vm_map_deallocate(addressMap);
-        addressMap = 0;
+    IOOptionBits type = _flags & kIOMemoryTypeMask;
+    Ranges vec = _ranges;

 
 

-    } else
-       err = kIOReturnSuccess;
+    mach_vm_address_t range0Addr = 0;
+    mach_vm_size_t    range0Len = 0;

 
 

-    logical = 0;
+    if ((offset >= _length) || ((offset + length) > _length))
+       return( kIOReturnBadArgument );

 
 

-    UNLOCK;
+    if (vec.v)
+       getAddrLenForInd(range0Addr, range0Len, type, vec, 0);

 
 

-    return( err );
-}
+    // mapping source == dest? (could be much better)
+    if (_task
+     && (mapping->fAddressTask == _task)
+     && (mapping->fAddressMap == get_task_map(_task)) 
+     && (options & kIOMapAnywhere)
+     && (1 == _rangesCount) 
+     && (0 == offset)
+     && range0Addr 
+     && (length <= range0Len))
+    {
+       mapping->fAddress = range0Addr;
+       mapping->fOptions |= kIOMapStatic;

 
 

-void _IOMemoryMap::taskDied( void )
-{
-    LOCK;
-    if( addressMap) {
-        vm_map_deallocate(addressMap);
-        addressMap = 0;
+       return( kIOReturnSuccess );

     }
     }

-    addressTask        = 0;
-    logical    = 0;
-    UNLOCK;
-}
-
-// Overload the release mechanism.  All mappings must be a member
-// of a memory descriptors _mappings set.  This means that we
-// always have 2 references on a mapping.  When either of these mappings
-// are released we need to free ourselves.
-void _IOMemoryMap::taggedRelease(const void *tag) const
-{
-    LOCK;
-    super::taggedRelease(tag, 2);
-    UNLOCK;
-}
-
-void _IOMemoryMap::free()
-{
-    unmap();

 
 

-    if( memory) {
-        LOCK;
-       memory->removeMapping( this);
-       UNLOCK;
-       memory->release();
+    if (!_memRef)
+    {
+        IOOptionBits createOptions = 0;
+       if (!(kIOMapReadOnly & options)) 
+       {
+           createOptions |= kIOMemoryReferenceWrite;
+#if DEVELOPMENT || DEBUG
+            if (kIODirectionOut == (kIODirectionOutIn & _flags))
+            {
+                OSReportWithBacktrace("warning: creating writable mapping from IOMemoryDescriptor(kIODirectionOut) - use kIOMapReadOnly or change direction");
+           }
+#endif
+       }
+       err = memoryReferenceCreate(createOptions, &_memRef);
+       if (kIOReturnSuccess != err) return (err);

     }
 
     }
 

-    if (owner && (owner != memory))
+    memory_object_t pager;
+    pager = (memory_object_t) (reserved ? reserved->dp.devicePager : 0);
+
+    // <upl_transpose //
+    if ((kIOMapReference|kIOMapUnique) == ((kIOMapReference|kIOMapUnique) & options))

     {
     {

-        LOCK;
-       owner->removeMapping(this);
-       UNLOCK;
-    }
+        do
+       {
+           upl_t          redirUPL2;
+           vm_size_t      size;
+           int            flags;
+           unsigned int   lock_count;

 
 

-    if( superMap)
-       superMap->release();
+           if (!_memRef || (1 != _memRef->count))
+           {
+               err = kIOReturnNotReadable;
+               break;
+           }

 
 

-    if (redirUPL) {
-       upl_commit(redirUPL, NULL, 0);
-       upl_deallocate(redirUPL);
-    }
+           size = round_page(mapping->fLength);
+           flags = UPL_COPYOUT_FROM | UPL_SET_INTERNAL 
+                       | UPL_SET_LITE | UPL_SET_IO_WIRE | UPL_BLOCK_ACCESS;

 
 

-    super::free();
-}
+           if (KERN_SUCCESS != memory_object_iopl_request(_memRef->entries[0].entry, 0, &size, &redirUPL2,
+                                           NULL, NULL,
+                                           &flags))
+               redirUPL2 = NULL;

 
 

-IOByteCount _IOMemoryMap::getLength()
-{
-    return( length );
-}
+           for (lock_count = 0;
+                IORecursiveLockHaveLock(gIOMemoryLock);
+                lock_count++) {
+             UNLOCK;
+           }
+           err = upl_transpose(redirUPL2, mapping->fRedirUPL);
+           for (;
+                lock_count;
+                lock_count--) {
+             LOCK;
+           }

 
 

-IOVirtualAddress _IOMemoryMap::getVirtualAddress()
-{
-    return( logical);
-}
+           if (kIOReturnSuccess != err)
+           {
+               IOLog("upl_transpose(%x)\n", err);
+               err = kIOReturnSuccess;
+           }

 
 

-task_t _IOMemoryMap::getAddressTask()
-{
-    if( superMap)
-       return( superMap->getAddressTask());
+           if (redirUPL2)
+           {
+               upl_commit(redirUPL2, NULL, 0);
+               upl_deallocate(redirUPL2);
+               redirUPL2 = 0;
+           }
+           {
+               // swap the memEntries since they now refer to different vm_objects
+               IOMemoryReference * me = _memRef;
+               _memRef = mapping->fMemory->_memRef;
+               mapping->fMemory->_memRef = me;
+           }
+           if (pager)
+               err = populateDevicePager( pager, mapping->fAddressMap, mapping->fAddress, offset, length, options );
+       }
+       while (false);
+    }
+    // upl_transpose> //

     else
     else

-        return( addressTask);
-}
+    {
+       err = memoryReferenceMap(_memRef, mapping->fAddressMap, offset, length, options, &mapping->fAddress);

 
 

-IOOptionBits _IOMemoryMap::getMapOptions()
-{
-    return( options);
-}
+       if ((err == KERN_SUCCESS) && pager)
+       {
+           err = populateDevicePager(pager, mapping->fAddressMap, mapping->fAddress, offset, length, options);
+           if (err != KERN_SUCCESS)
+           {
+               doUnmap(mapping->fAddressMap, (IOVirtualAddress) mapping, 0);
+           }
+           else if (kIOMapDefaultCache == (options & kIOMapCacheMask))
+           {
+               mapping->fOptions |= ((_flags & kIOMemoryBufferCacheMask) >> kIOMemoryBufferCacheShift);
+           }
+       }
+    }

 
 

-IOMemoryDescriptor * _IOMemoryMap::getMemoryDescriptor()
-{
-    return( memory );
+    return (err);

 }
 
 }
 

-_IOMemoryMap * _IOMemoryMap::copyCompatible(
-               IOMemoryDescriptor *    owner,
-                task_t                 task,
-                IOVirtualAddress       toAddress,
-                IOOptionBits           _options,
-                IOByteCount            _offset,
-                IOByteCount            _length )
+IOReturn IOGeneralMemoryDescriptor::doUnmap(
+       vm_map_t                addressMap,
+       IOVirtualAddress        __address,
+       IOByteCount             __length )

 {
 {

-    _IOMemoryMap * mapping;
-
-    if( (!task) || (!addressMap) || (addressMap != get_task_map(task)))
-       return( 0 );
-    if( options & kIOMapUnique)
-       return( 0 );
-    if( (options ^ _options) & kIOMapReadOnly)
-       return( 0 );
-    if( (kIOMapDefaultCache != (_options & kIOMapCacheMask)) 
-     && ((options ^ _options) & kIOMapCacheMask))
-       return( 0 );
-
-    if( (0 == (_options & kIOMapAnywhere)) && (logical != toAddress))
-       return( 0 );
+    return (super::doUnmap(addressMap, __address, __length));
+}

 
 

-    if( _offset < offset)
-       return( 0 );
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

 
 

-    _offset -= offset;
+#undef super
+#define super OSObject

 
 

-    if( (_offset + _length) > length)
-       return( 0 );
+OSDefineMetaClassAndStructors( IOMemoryMap, OSObject )

 
 

-    if( (length == _length) && (!_offset)) {
-        retain();
-       mapping = this;
+OSMetaClassDefineReservedUnused(IOMemoryMap, 0);
+OSMetaClassDefineReservedUnused(IOMemoryMap, 1);
+OSMetaClassDefineReservedUnused(IOMemoryMap, 2);
+OSMetaClassDefineReservedUnused(IOMemoryMap, 3);
+OSMetaClassDefineReservedUnused(IOMemoryMap, 4);
+OSMetaClassDefineReservedUnused(IOMemoryMap, 5);
+OSMetaClassDefineReservedUnused(IOMemoryMap, 6);
+OSMetaClassDefineReservedUnused(IOMemoryMap, 7);

 
 

-    } else {
-        mapping = new _IOMemoryMap;
-        if( mapping
-        && !mapping->initCompatible( owner, this, _offset, _length )) {
-            mapping->release();
-            mapping = 0;
-        }
-    }
+/* ex-inline function implementation */
+IOPhysicalAddress IOMemoryMap::getPhysicalAddress()
+    { return( getPhysicalSegment( 0, 0 )); }

 
 

-    return( mapping );
-}
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

 
 

-IOPhysicalAddress 
-_IOMemoryMap::getPhysicalSegment( IOByteCount _offset, IOPhysicalLength * _length)
+bool IOMemoryMap::init(
+        task_t                 intoTask,
+        mach_vm_address_t      toAddress,
+        IOOptionBits           _options,
+        mach_vm_size_t         _offset,
+        mach_vm_size_t         _length )

 {
 {

-    IOPhysicalAddress  address;
-
-    LOCK;
-    address = memory->getPhysicalSegment( offset + _offset, _length );
-    UNLOCK;
+    if (!intoTask)
+       return( false);

 
 

-    return( address );
-}
+    if (!super::init())
+       return(false);

 
 

-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+    fAddressMap  = get_task_map(intoTask);
+    if (!fAddressMap)
+       return(false);
+    vm_map_reference(fAddressMap);

 
 

-#undef super
-#define super OSObject
+    fAddressTask = intoTask;
+    fOptions     = _options;
+    fLength      = _length;
+    fOffset     = _offset;
+    fAddress     = toAddress;

 
 

-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+    return (true);
+}

 
 

-void IOMemoryDescriptor::initialize( void )
+bool IOMemoryMap::setMemoryDescriptor(IOMemoryDescriptor * _memory, mach_vm_size_t _offset)

 {
 {

-    if( 0 == gIOMemoryLock)
-       gIOMemoryLock = IORecursiveLockAlloc();
+    if (!_memory)
+       return(false);

 
 

-    IORegistryEntry::getRegistryRoot()->setProperty(kIOMaximumMappedIOByteCountKey,
-                                                   ptoa_64(gIOMaximumMappedIOPageCount), 64);
-    if (!gIOCopyMapper)
+    if (!fSuperMap)

     {
     {

-       IOMapper *
-       mapper = new IOCopyMapper;
-       if (mapper)
-       {
-           if (mapper->init() && mapper->start(NULL))
-               gIOCopyMapper = (IOCopyMapper *) mapper;
-           else
-               mapper->release();
-       }
+       if( (_offset + fLength) > _memory->getLength())
+           return( false);
+       fOffset = _offset;

     }
 
     }
 

-    gIOLastPage = IOGetLastPageNumber();
+    _memory->retain();
+    if (fMemory)
+    {
+       if (fMemory != _memory)
+           fMemory->removeMapping(this);
+       fMemory->release();
+    }
+    fMemory = _memory;
+
+    return( true );

 }
 
 }
 

-void IOMemoryDescriptor::free( void )
+IOReturn IOMemoryDescriptor::doMap(
+       vm_map_t                __addressMap,
+       IOVirtualAddress *      __address,
+       IOOptionBits            options,
+       IOByteCount             __offset,
+       IOByteCount             __length )

 {
 {

-    if( _mappings)
-       _mappings->release();
+    return (kIOReturnUnsupported);
+}

 
 

-    super::free();
+IOReturn IOMemoryDescriptor::handleFault(
+        void *                 _pager,
+       mach_vm_size_t          sourceOffset,
+       mach_vm_size_t          length)
+{
+    if( kIOMemoryRedirected & _flags)
+    {
+#if DEBUG
+       IOLog("sleep mem redirect %p, %qx\n", this, sourceOffset);
+#endif
+       do {
+           SLEEP;
+       } while( kIOMemoryRedirected & _flags );
+    }
+    return (kIOReturnSuccess);

 }
 
 }
 

-IOMemoryMap * IOMemoryDescriptor::setMapping(
-       task_t                  intoTask,
-       IOVirtualAddress        mapAddress,
-       IOOptionBits            options )
+IOReturn IOMemoryDescriptor::populateDevicePager(
+        void *                 _pager,
+       vm_map_t                addressMap,
+       mach_vm_address_t       address,
+       mach_vm_size_t          sourceOffset,
+       mach_vm_size_t          length,
+        IOOptionBits           options )

 {
 {

-    _IOMemoryMap *             newMap;
+    IOReturn           err = kIOReturnSuccess;
+    memory_object_t    pager = (memory_object_t) _pager;
+    mach_vm_size_t     size;
+    mach_vm_size_t     bytes;
+    mach_vm_size_t     page;
+    mach_vm_size_t     pageOffset;
+    mach_vm_size_t     pagerOffset;
+    IOPhysicalLength   segLen;
+    addr64_t           physAddr;
+
+    physAddr = getPhysicalSegment( sourceOffset, &segLen, kIOMemoryMapperNone );
+    assert( physAddr );
+    pageOffset = physAddr - trunc_page_64( physAddr );
+    pagerOffset = sourceOffset;
+
+    size = length + pageOffset;
+    physAddr -= pageOffset;
+
+    segLen += pageOffset;
+    bytes = size;
+    do
+    {
+       // in the middle of the loop only map whole pages
+       if( segLen >= bytes) segLen = bytes;
+       else if (segLen != trunc_page(segLen))    err = kIOReturnVMError;
+        if (physAddr != trunc_page_64(physAddr))  err = kIOReturnBadArgument;

 
 

-    newMap = new _IOMemoryMap;
+       if (kIOReturnSuccess != err) break;

 
 

-    LOCK;
+       if (reserved && reserved->dp.pagerContig)
+       {
+           IOPhysicalLength    allLen;
+           addr64_t            allPhys;

 
 

-    if( newMap
-     && !newMap->initWithDescriptor( this, intoTask, mapAddress,
-                    options | kIOMapStatic, 0, getLength() )) {
-       newMap->release();
-       newMap = 0;
-    }
+           allPhys = getPhysicalSegment( 0, &allLen, kIOMemoryMapperNone );
+           assert( allPhys );
+           err = device_pager_populate_object( pager, 0, atop_64(allPhys), round_page(allLen) );
+       }
+       else
+       {
+           for( page = 0;
+                (page < segLen) && (KERN_SUCCESS == err);
+                page += page_size)
+           {
+               err = device_pager_populate_object(pager, pagerOffset,
+                       (ppnum_t)(atop_64(physAddr + page)), page_size);
+               pagerOffset += page_size;
+           }
+       }
+       assert (KERN_SUCCESS == err);
+       if (err) break;

 
 

-    addMapping( newMap);
+       // This call to vm_fault causes an early pmap level resolution
+       // of the mappings created above for kernel mappings, since
+       // faulting in later can't take place from interrupt level.
+       if ((addressMap == kernel_map) && !(kIOMemoryRedirected & _flags))
+       {
+           vm_fault(addressMap, 
+                    (vm_map_offset_t)trunc_page_64(address),
+                    VM_PROT_READ|VM_PROT_WRITE, 
+                    FALSE, THREAD_UNINT, NULL, 
+                    (vm_map_offset_t)0);
+       }

 
 

-    UNLOCK;
+       sourceOffset += segLen - pageOffset;
+       address += segLen;
+       bytes -= segLen;
+       pageOffset = 0;
+    } 
+    while (bytes && (physAddr = getPhysicalSegment( sourceOffset, &segLen, kIOMemoryMapperNone )));
+
+    if (bytes)
+        err = kIOReturnBadArgument;

 
 

-    return( newMap);
+    return (err);

 }
 
 }
 

-IOMemoryMap * IOMemoryDescriptor::map( 
-       IOOptionBits            options )
+IOReturn IOMemoryDescriptor::doUnmap(
+       vm_map_t                addressMap,
+       IOVirtualAddress        __address,
+       IOByteCount             __length )

 {
 {

+    IOReturn         err;
+    mach_vm_address_t address;
+    mach_vm_size_t    length;

 
 

-    return( makeMapping( this, kernel_task, 0,
-                       options | kIOMapAnywhere,
-                       0, getLength() ));
-}
+    if (__length)
+    {
+       address = __address;
+       length  = __length;
+    }
+    else
+    {
+       addressMap = ((IOMemoryMap *) __address)->fAddressMap;
+       address    = ((IOMemoryMap *) __address)->fAddress;
+       length     = ((IOMemoryMap *) __address)->fLength;
+    }

 
 

-IOMemoryMap * IOMemoryDescriptor::map(
-       task_t                  intoTask,
-       IOVirtualAddress        toAddress,
-       IOOptionBits            options,
-       IOByteCount             offset,
-       IOByteCount             length )
-{
-    if( 0 == length)
-       length = getLength();
+    if ((addressMap == kernel_map) && (kIOMemoryBufferPageable & _flags))
+       addressMap = IOPageableMapForAddress( address );
+
+#if DEBUG
+    if( kIOLogMapping & gIOKitDebug)
+       IOLog("IOMemoryDescriptor::doUnmap map %p, 0x%qx:0x%qx\n",
+               addressMap, address, length );
+#endif
+
+    err = mach_vm_deallocate( addressMap, address, length );

 
 

-    return( makeMapping( this, intoTask, toAddress, options, offset, length ));
+    return (err);

 }
 
 }
 

-IOReturn _IOMemoryMap::redirect(IOMemoryDescriptor * newBackingMemory,
-                               IOOptionBits         options,
-                               IOByteCount          offset)
+IOReturn IOMemoryDescriptor::redirect( task_t safeTask, bool doRedirect )

 {
 {

-    IOReturn err = kIOReturnSuccess;
-    IOMemoryDescriptor * physMem = 0;
+    IOReturn           err = kIOReturnSuccess;
+    IOMemoryMap *      mapping = 0;
+    OSIterator *       iter;

 
     LOCK;
 
 
     LOCK;
 

-    if (logical && addressMap) do 
-    {
-       if (((memory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical)
-           || ((memory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical64))
-       {
-           physMem = memory;
-           physMem->retain();
-       }
+    if( doRedirect)
+        _flags |= kIOMemoryRedirected;
+    else
+        _flags &= ~kIOMemoryRedirected;

 
 

-       if (!redirUPL)
-       {
-           vm_size_t size = length;
-           int flags = UPL_COPYOUT_FROM | UPL_SET_INTERNAL 
-                       | UPL_SET_LITE | UPL_SET_IO_WIRE | UPL_BLOCK_ACCESS;
-           if (KERN_SUCCESS != memory_object_iopl_request((ipc_port_t) memory->_memEntry, 0, &size, &redirUPL,
-                                           NULL, NULL,
-                                           &flags))
-               redirUPL = 0;
+    do {
+       if( (iter = OSCollectionIterator::withCollection( _mappings))) {

 
 

-           if (physMem)
-           {
-               IOUnmapPages( addressMap, logical, length );
-               physMem->redirect(0, true);
-           }
-       }
+           memory_object_t   pager;

 
 

-       if (newBackingMemory)
-       {
-           if (newBackingMemory != memory)
-           {
-               if (this != newBackingMemory->makeMapping(newBackingMemory, addressTask, (IOVirtualAddress) this, 
-                                                           options | kIOMapUnique | kIOMapReference,
-                                                           offset, length))
-                   err = kIOReturnError;
-           }
-           if (redirUPL)
+           if( reserved)
+               pager = (memory_object_t) reserved->dp.devicePager;
+           else
+               pager = MACH_PORT_NULL;
+
+           while( (mapping = (IOMemoryMap *) iter->getNextObject()))

            {
            {

-               upl_commit(redirUPL, NULL, 0);
-               upl_deallocate(redirUPL);
-               redirUPL = 0;
+               mapping->redirect( safeTask, doRedirect );
+               if (!doRedirect && !safeTask && pager && (kernel_map == mapping->fAddressMap))
+               {
+                   err = populateDevicePager(pager, mapping->fAddressMap, mapping->fAddress, mapping->fOffset, mapping->fLength, kIOMapDefaultCache );
+               }

            }
            }

-           if (physMem)
-               physMem->redirect(0, false);
+
+           iter->release();

        }
        }

+    } while( false );
+
+    if (!doRedirect)
+    {
+        WAKEUP;

     }
     }

-    while (false);

 
     UNLOCK;
 
 
     UNLOCK;
 

-    if (physMem)
-       physMem->release();
+#ifndef __LP64__
+    // temporary binary compatibility
+    IOSubMemoryDescriptor * subMem;
+    if( (subMem = OSDynamicCast( IOSubMemoryDescriptor, this)))
+       err = subMem->redirect( safeTask, doRedirect );
+    else
+       err = kIOReturnSuccess;
+#endif /* !__LP64__ */

 
 

-    return (err);
+    return( err );

 }
 
 }
 

-IOMemoryMap * IOMemoryDescriptor::makeMapping(
-       IOMemoryDescriptor *    owner,
-       task_t                  intoTask,
-       IOVirtualAddress        toAddress,
-       IOOptionBits            options,
-       IOByteCount             offset,
-       IOByteCount             length )
+IOReturn IOMemoryMap::redirect( task_t safeTask, bool doRedirect )

 {
 {

-    IOMemoryDescriptor * mapDesc = 0;
-    _IOMemoryMap *     mapping = 0;
-    OSIterator *       iter;
+    IOReturn err = kIOReturnSuccess;

 
 

-    LOCK;
+    if( fSuperMap) {
+//        err = ((IOMemoryMap *)superMap)->redirect( safeTask, doRedirect );
+    } else {

 
 

-    do
-    {
-       if (kIOMapUnique & options)
-       {
-           IOPhysicalAddress phys;
-           IOByteCount       physLen;
+        LOCK;

 
 

-           if (owner != this)
-               continue;
+       do
+       {
+           if (!fAddress)
+               break;
+           if (!fAddressMap)
+               break;

 
 

-           if (((_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical)
-               || ((_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical64))
-           {
-               phys = getPhysicalSegment(offset, &physLen);
-               if (!phys || (physLen < length))
-                   continue;
-    
-               mapDesc = IOMemoryDescriptor::withPhysicalAddress(
-                               phys, length, _direction);
-               if (!mapDesc)
-                   continue;
-               offset = 0;
-           }
-           else
+           if ((!safeTask || (get_task_map(safeTask) != fAddressMap))
+             && (0 == (fOptions & kIOMapStatic)))

            {
            {

-               mapDesc = this;
-               mapDesc->retain();
+               IOUnmapPages( fAddressMap, fAddress, fLength );
+               err = kIOReturnSuccess;
+#if DEBUG
+               IOLog("IOMemoryMap::redirect(%d, %p) 0x%qx:0x%qx from %p\n", doRedirect, this, fAddress, fLength, fAddressMap);
+#endif

            }
            }

-
-           if (kIOMapReference & options)
+           else if (kIOMapWriteCombineCache == (fOptions & kIOMapCacheMask))

            {
            {

-               mapping = (_IOMemoryMap *) toAddress;
-               mapping->retain();
-
-#if 1
-               uint32_t pageOffset1 = mapDesc->getSourceSegment( offset, NULL );
-               pageOffset1 -= trunc_page_32( pageOffset1 );
-
-               uint32_t pageOffset2 = mapping->getVirtualAddress();
-               pageOffset2 -= trunc_page_32( pageOffset2 );
-               
-               if (pageOffset1 != pageOffset2)
-                   IOLog("::redirect can't map offset %x to addr %x\n", 
-                           pageOffset1, mapping->getVirtualAddress());
-#endif
-
-
-               if (!mapping->initWithDescriptor( mapDesc, intoTask, toAddress, options,
-                                                   offset, length ))
-               {
-#ifdef DEBUG
-                   IOLog("Didn't redirect map %08lx : %08lx\n", offset, length );
-#endif
-               }
-
-               if (mapping->owner)
-                   mapping->owner->removeMapping(mapping);
-               continue;
+               IOOptionBits newMode;
+               newMode = (fOptions & ~kIOMapCacheMask) | (doRedirect ? kIOMapInhibitCache : kIOMapWriteCombineCache);
+               IOProtectCacheMode(fAddressMap, fAddress, fLength, newMode);

            }
        }
            }
        }

-       else
-       {
-           // look for an existing mapping
-           if( (iter = OSCollectionIterator::withCollection( _mappings))) {
-    
-               while( (mapping = (_IOMemoryMap *) iter->getNextObject())) {
-    
-                   if( (mapping = mapping->copyCompatible( 
-                                           owner, intoTask, toAddress,
-                                           options | kIOMapReference,
-                                           offset, length )))
-                       break;
-               }
-               iter->release();
-           }
+       while (false);
+       UNLOCK;
+    }

 
 

+    if ((((fMemory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical)
+        || ((fMemory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical64))
+     && safeTask
+     && (doRedirect != (0 != (fMemory->_flags & kIOMemoryRedirected))))
+       fMemory->redirect(safeTask, doRedirect);

 
 

-       if (mapping)
-           mapping->retain();
+    return( err );
+}

 
 

-           if( mapping || (options & kIOMapReference))
-               continue;
+IOReturn IOMemoryMap::unmap( void )
+{
+    IOReturn   err;

 
 

-           mapDesc = owner;
-           mapDesc->retain();
-       }
-       owner = this;
-
-        mapping = new _IOMemoryMap;
-       if( mapping
-       && !mapping->initWithDescriptor( mapDesc, intoTask, toAddress, options,
-                          offset, length )) {
-#ifdef DEBUG
-           IOLog("Didn't make map %08lx : %08lx\n", offset, length );
-#endif
-           mapping->release();
-            mapping = 0;
-       }
+    LOCK;

 
 

-       if (mapping)
-           mapping->retain();
+    if( fAddress && fAddressMap && (0 == fSuperMap) && fMemory
+       && (0 == (fOptions & kIOMapStatic))) {

 
 

-    } while( false );
+        err = fMemory->doUnmap(fAddressMap, (IOVirtualAddress) this, 0);

 
 

-    if (mapping)
+    } else
+       err = kIOReturnSuccess;
+
+    if (fAddressMap)

     {
     {

-       mapping->owner = owner;
-       owner->addMapping( mapping);
-       mapping->release();
+        vm_map_deallocate(fAddressMap);
+        fAddressMap = 0;

     }
 
     }
 

-    UNLOCK;
+    fAddress = 0;

 
 

-    if (mapDesc)
-       mapDesc->release();
+    UNLOCK;

 
 

-    return( mapping);
+    return( err );

 }
 
 }
 

-void IOMemoryDescriptor::addMapping(
-       IOMemoryMap * mapping )
+void IOMemoryMap::taskDied( void )

 {
 {

-    if( mapping) {
-        if( 0 == _mappings)
-            _mappings = OSSet::withCapacity(1);
-       if( _mappings )
-           _mappings->setObject( mapping );
+    LOCK;
+    if (fUserClientUnmap)
+       unmap();
+    if( fAddressMap) {
+        vm_map_deallocate(fAddressMap);
+        fAddressMap = 0;

     }
     }

+    fAddressTask = 0;
+    fAddress    = 0;
+    UNLOCK;

 }
 
 }
 

-void IOMemoryDescriptor::removeMapping(
-       IOMemoryMap * mapping )
+IOReturn IOMemoryMap::userClientUnmap( void )

 {
 {

-    if( _mappings)
-        _mappings->removeObject( mapping);
+    fUserClientUnmap = true;
+    return (kIOReturnSuccess);

 }
 
 }
 

-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+// Overload the release mechanism.  All mappings must be a member
+// of a memory descriptors _mappings set.  This means that we
+// always have 2 references on a mapping.  When either of these mappings
+// are released we need to free ourselves.
+void IOMemoryMap::taggedRelease(const void *tag) const
+{
+    LOCK;
+    super::taggedRelease(tag, 2);
+    UNLOCK;
+}

 
 

-#undef super
-#define super IOMemoryDescriptor
+void IOMemoryMap::free()
+{
+    unmap();

 
 

-OSDefineMetaClassAndStructors(IOSubMemoryDescriptor, IOMemoryDescriptor)
+    if (fMemory)
+    {
+        LOCK;
+       fMemory->removeMapping(this);
+       UNLOCK;
+       fMemory->release();
+    }

 
 

-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+    if (fOwner && (fOwner != fMemory))
+    {
+        LOCK;
+       fOwner->removeMapping(this);
+       UNLOCK;
+    }

 
 

-bool IOSubMemoryDescriptor::initSubRange( IOMemoryDescriptor * parent,
-                                       IOByteCount offset, IOByteCount length,
-                                       IODirection direction )
-{
-    if( !parent)
-       return( false);
+    if (fSuperMap)
+       fSuperMap->release();

 
 

-    if( (offset + length) > parent->getLength())
-       return( false);
+    if (fRedirUPL) {
+       upl_commit(fRedirUPL, NULL, 0);
+       upl_deallocate(fRedirUPL);
+    }

 
 

-    /*
-     * We can check the _parent instance variable before having ever set it
-     * to an initial value because I/O Kit guarantees that all our instance
-     * variables are zeroed on an object's allocation.
-     */
+    super::free();
+}

 
 

-    if( !_parent) {
-       if( !super::init())
-           return( false );
-    } else {
-       /*
-        * An existing memory descriptor is being retargeted to
-        * point to somewhere else.  Clean up our present state.
-        */
+IOByteCount IOMemoryMap::getLength()
+{
+    return( fLength );
+}

 
 

-       _parent->release();
-       _parent = 0;
+IOVirtualAddress IOMemoryMap::getVirtualAddress()
+{
+#ifndef __LP64__
+    if (fSuperMap)
+       fSuperMap->getVirtualAddress();
+    else if (fAddressMap 
+               && vm_map_is_64bit(fAddressMap)
+               && (sizeof(IOVirtualAddress) < 8))
+    {
+       OSReportWithBacktrace("IOMemoryMap::getVirtualAddress(0x%qx) called on 64b map; use ::getAddress()", fAddress);

     }
     }

+#endif /* !__LP64__ */

 
 

-    parent->retain();
-    _parent    = parent;
-    _start     = offset;
-    _length    = length;
-    _direction  = direction;
-    _tag       = parent->getTag();
+    return (fAddress);
+}

 
 

-    return( true );
+#ifndef __LP64__
+mach_vm_address_t      IOMemoryMap::getAddress()
+{
+    return( fAddress);

 }
 
 }
 

-void IOSubMemoryDescriptor::free( void )
+mach_vm_size_t         IOMemoryMap::getSize()

 {
 {

-    if( _parent)
-       _parent->release();
+    return( fLength );
+}
+#endif /* !__LP64__ */

 
 

-    super::free();
+
+task_t IOMemoryMap::getAddressTask()
+{
+    if( fSuperMap)
+       return( fSuperMap->getAddressTask());
+    else
+        return( fAddressTask);

 }
 
 }
 

+IOOptionBits IOMemoryMap::getMapOptions()
+{
+    return( fOptions);
+}

 
 

-IOReturn
-IOSubMemoryDescriptor::dmaCommandOperation(DMACommandOps op, void *vData, UInt dataSize) const
+IOMemoryDescriptor * IOMemoryMap::getMemoryDescriptor()

 {
 {

-    IOReturn rtn;
+    return( fMemory );
+}

 
 

-    if (kIOMDGetCharacteristics == op) {
+IOMemoryMap * IOMemoryMap::copyCompatible(
+               IOMemoryMap * newMapping )
+{
+    task_t             task      = newMapping->getAddressTask();
+    mach_vm_address_t  toAddress = newMapping->fAddress;
+    IOOptionBits       _options  = newMapping->fOptions;
+    mach_vm_size_t     _offset   = newMapping->fOffset;
+    mach_vm_size_t     _length   = newMapping->fLength;

 
 

-       rtn = _parent->dmaCommandOperation(op, vData, dataSize);
-       if (kIOReturnSuccess == rtn) {
-           IOMDDMACharacteristics *data = (IOMDDMACharacteristics *) vData;
-           data->fLength = _length;
-           data->fSGCount = 0; // XXX gvdl: need to compute and pages
-           data->fPages = 0;
-           data->fPageAlign = 0;
-       }
+    if( (!task) || (!fAddressMap) || (fAddressMap != get_task_map(task)))
+       return( 0 );
+    if( (fOptions ^ _options) & kIOMapReadOnly)
+       return( 0 );
+    if( (kIOMapDefaultCache != (_options & kIOMapCacheMask)) 
+     && ((fOptions ^ _options) & kIOMapCacheMask))
+       return( 0 );

 
 

-       return rtn;
-    }
-    else if (kIOMDWalkSegments & op) {
-       if (dataSize < sizeof(IOMDDMAWalkSegmentArgs))
-           return kIOReturnUnderrun;
+    if( (0 == (_options & kIOMapAnywhere)) && (fAddress != toAddress))
+       return( 0 );

 
 

-       IOMDDMAWalkSegmentArgs *data =
-           reinterpret_cast<IOMDDMAWalkSegmentArgs *>(vData);
-       UInt offset = data->fOffset;
-       UInt remain = _length - offset;
-       if ((int) remain <= 0)
-           return (!remain)? kIOReturnOverrun : kIOReturnInternalError;
+    if( _offset < fOffset)
+       return( 0 );
+
+    _offset -= fOffset;

 
 

-       data->fOffset = offset + _start;
-       rtn = _parent->dmaCommandOperation(op, vData, dataSize);
-       if (data->fLength > remain)
-           data->fLength = remain;
-       data->fOffset  = offset;
+    if( (_offset + _length) > fLength)
+       return( 0 );

 
 

-       return rtn;
+    retain();
+    if( (fLength == _length) && (!_offset))
+    {
+       newMapping = this;

     }
     else
     }
     else

-       return kIOReturnBadArgument;
+    {
+       newMapping->fSuperMap = this;
+       newMapping->fOffset   = fOffset + _offset;
+       newMapping->fAddress  = fAddress + _offset;
+    }
+
+    return( newMapping );

 }
 
 }
 

-addr64_t
-IOSubMemoryDescriptor::getPhysicalSegment64(IOByteCount offset, IOByteCount * length)
+IOReturn IOMemoryMap::wireRange(
+       uint32_t                options,
+        mach_vm_size_t         offset,
+        mach_vm_size_t         length)

 {
 {

-    addr64_t   address;
-    IOByteCount        actualLength;
-
-    assert(offset <= _length);
+    IOReturn kr;
+    mach_vm_address_t start = trunc_page_64(fAddress + offset);
+    mach_vm_address_t end   = round_page_64(fAddress + offset + length);
+    
+    if (kIODirectionOutIn & options)
+    {
+       kr = vm_map_wire(fAddressMap, start, end, (kIODirectionOutIn & options), FALSE);
+    }
+    else
+    {
+       kr = vm_map_unwire(fAddressMap, start, end, FALSE);
+    }

 
 

-    if( length)
-        *length = 0;
+    return (kr);
+}

 
 

-    if( offset >= _length)
-        return( 0 );

 
 

-    address = _parent->getPhysicalSegment64( offset + _start, &actualLength );
+IOPhysicalAddress 
+#ifdef __LP64__
+IOMemoryMap::getPhysicalSegment( IOByteCount _offset, IOPhysicalLength * _length, IOOptionBits _options)
+#else /* !__LP64__ */
+IOMemoryMap::getPhysicalSegment( IOByteCount _offset, IOPhysicalLength * _length)
+#endif /* !__LP64__ */
+{
+    IOPhysicalAddress  address;

 
 

-    if( address && length)
-       *length = min( _length - offset, actualLength );
+    LOCK;
+#ifdef __LP64__
+    address = fMemory->getPhysicalSegment( fOffset + _offset, _length, _options );
+#else /* !__LP64__ */
+    address = fMemory->getPhysicalSegment( fOffset + _offset, _length );
+#endif /* !__LP64__ */
+    UNLOCK;

 
     return( address );
 }
 
 
     return( address );
 }
 

-IOPhysicalAddress
-IOSubMemoryDescriptor::getPhysicalSegment( IOByteCount offset, IOByteCount * length )
-{
-    IOPhysicalAddress  address;
-    IOByteCount                actualLength;
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

 
 

-    assert(offset <= _length);
+#undef super
+#define super OSObject
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */

 
 

-    if( length)
-        *length = 0;
+void IOMemoryDescriptor::initialize( void )
+{
+    if( 0 == gIOMemoryLock)
+       gIOMemoryLock = IORecursiveLockAlloc();

 
 

-    if( offset >= _length)
-        return( 0 );
+    gIOLastPage = IOGetLastPageNumber();
+}

 
 

-    address = _parent->getPhysicalSegment( offset + _start, &actualLength );
+void IOMemoryDescriptor::free( void )
+{
+    if( _mappings)
+       _mappings->release();

 
 

-    if( address && length)
-       *length = min( _length - offset, actualLength );
+    super::free();
+}

 
 

-    return( address );
+IOMemoryMap * IOMemoryDescriptor::setMapping(
+       task_t                  intoTask,
+       IOVirtualAddress        mapAddress,
+       IOOptionBits            options )
+{
+    return (createMappingInTask( intoTask, mapAddress,
+                                   options | kIOMapStatic,
+                                   0, getLength() ));

 }
 
 }
 

+IOMemoryMap * IOMemoryDescriptor::map( 
+       IOOptionBits            options )
+{
+    return (createMappingInTask( kernel_task, 0,
+                               options | kIOMapAnywhere,
+                               0, getLength() ));
+}

 
 

-IOReturn IOSubMemoryDescriptor::doMap(
-       vm_map_t                addressMap,
-       IOVirtualAddress *      atAddress,
+#ifndef __LP64__
+IOMemoryMap * IOMemoryDescriptor::map( 
+       task_t                  intoTask,
+       IOVirtualAddress        atAddress,

        IOOptionBits            options,
        IOOptionBits            options,

-       IOByteCount             sourceOffset,
+       IOByteCount             offset,

        IOByteCount             length )
 {
        IOByteCount             length )
 {

-    if( sourceOffset >= _length)
-        return( kIOReturnOverrun );
-    return (_parent->doMap(addressMap, atAddress, options, sourceOffset + _start, length));
+    if ((!(kIOMapAnywhere & options)) && vm_map_is_64bit(get_task_map(intoTask)))
+    {
+       OSReportWithBacktrace("IOMemoryDescriptor::map() in 64b task, use ::createMappingInTask()");
+       return (0);
+    }
+
+    return (createMappingInTask(intoTask, atAddress,
+                               options, offset, length));

 }
 }

+#endif /* !__LP64__ */

 
 

-IOPhysicalAddress 
-IOSubMemoryDescriptor::getSourceSegment( IOByteCount offset, IOByteCount * length )
+IOMemoryMap * IOMemoryDescriptor::createMappingInTask(
+       task_t                  intoTask,
+       mach_vm_address_t       atAddress,
+       IOOptionBits            options,
+       mach_vm_size_t          offset,
+       mach_vm_size_t          length)

 {
 {

-    IOPhysicalAddress  address;
-    IOByteCount                actualLength;
+    IOMemoryMap * result;
+    IOMemoryMap * mapping;

 
 

-    assert(offset <= _length);
+    if (0 == length)
+       length = getLength();

 
 

-    if( length)
-        *length = 0;
+    mapping = new IOMemoryMap;

 
 

-    if( offset >= _length)
-        return( 0 );
+    if( mapping
+     && !mapping->init( intoTask, atAddress,
+                       options, offset, length )) {
+       mapping->release();
+       mapping = 0;
+    }

 
 

-    address = _parent->getSourceSegment( offset + _start, &actualLength );
+    if (mapping)
+       result = makeMapping(this, intoTask, (IOVirtualAddress) mapping, options | kIOMap64Bit, 0, 0);
+    else
+       result = 0;

 
 

-    if( address && length)
-       *length = min( _length - offset, actualLength );
+#if DEBUG
+    if (!result)
+       IOLog("createMappingInTask failed desc %p, addr %qx, options %x, offset %qx, length %llx\n",
+               this, atAddress, (uint32_t) options, offset, length);
+#endif

 
 

-    return( address );
+    return (result);

 }
 
 }
 

-void * IOSubMemoryDescriptor::getVirtualSegment(IOByteCount offset,
-                                       IOByteCount * lengthOfSegment)
+#ifndef __LP64__ // there is only a 64 bit version for LP64
+IOReturn IOMemoryMap::redirect(IOMemoryDescriptor * newBackingMemory,
+                               IOOptionBits         options,
+                               IOByteCount          offset)

 {
 {

-    return( 0 );
+    return (redirect(newBackingMemory, options, (mach_vm_size_t)offset));

 }
 }

+#endif

 
 

-IOByteCount IOSubMemoryDescriptor::readBytes(IOByteCount offset,
-                                       void * bytes, IOByteCount length)
+IOReturn IOMemoryMap::redirect(IOMemoryDescriptor * newBackingMemory,
+                               IOOptionBits         options,
+                               mach_vm_size_t       offset)

 {
 {

-    IOByteCount        byteCount;
-
-    assert(offset <= _length);
-
-    if( offset >= _length)
-        return( 0 );
+    IOReturn err = kIOReturnSuccess;
+    IOMemoryDescriptor * physMem = 0;

 
     LOCK;
 
     LOCK;

-    byteCount = _parent->readBytes( _start + offset, bytes,
-                               min(length, _length - offset) );
-    UNLOCK;

 
 

-    return( byteCount );
-}
+    if (fAddress && fAddressMap) do 
+    {
+       if (((fMemory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical)
+           || ((fMemory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical64))
+       {
+           physMem = fMemory;
+           physMem->retain();
+       }

 
 

-IOByteCount IOSubMemoryDescriptor::writeBytes(IOByteCount offset,
-                               const void* bytes, IOByteCount length)
-{
-    IOByteCount        byteCount;
+       if (!fRedirUPL && fMemory->_memRef && (1 == fMemory->_memRef->count))
+       {
+           vm_size_t size = round_page(fLength);
+           int flags = UPL_COPYOUT_FROM | UPL_SET_INTERNAL 
+                       | UPL_SET_LITE | UPL_SET_IO_WIRE | UPL_BLOCK_ACCESS;
+           if (KERN_SUCCESS != memory_object_iopl_request(fMemory->_memRef->entries[0].entry, 0, &size, &fRedirUPL,
+                                           NULL, NULL,
+                                           &flags))
+               fRedirUPL = 0;

 
 

-    assert(offset <= _length);
+           if (physMem)
+           {
+               IOUnmapPages( fAddressMap, fAddress, fLength );
+               if ((false))
+                   physMem->redirect(0, true);
+           }
+       }

 
 

-    if( offset >= _length)
-        return( 0 );
+       if (newBackingMemory)
+       {
+           if (newBackingMemory != fMemory)
+           {
+               fOffset = 0;
+               if (this != newBackingMemory->makeMapping(newBackingMemory, fAddressTask, (IOVirtualAddress) this, 
+                                                           options | kIOMapUnique | kIOMapReference | kIOMap64Bit,
+                                                           offset, fLength))
+                   err = kIOReturnError;
+           }
+           if (fRedirUPL)
+           {
+               upl_commit(fRedirUPL, NULL, 0);
+               upl_deallocate(fRedirUPL);
+               fRedirUPL = 0;
+           }
+           if ((false) && physMem)
+               physMem->redirect(0, false);
+       }
+    }
+    while (false);

 
 

-    LOCK;
-    byteCount = _parent->writeBytes( _start + offset, bytes,
-                               min(length, _length - offset) );

     UNLOCK;
 
     UNLOCK;
 

-    return( byteCount );
-}
-
-IOReturn IOSubMemoryDescriptor::setPurgeable( IOOptionBits newState,
-                                    IOOptionBits * oldState )
-{
-    IOReturn err;
-
-    LOCK;
-    err = _parent->setPurgeable( newState, oldState );
-    UNLOCK;
+    if (physMem)
+       physMem->release();

 
 

-    return( err );
+    return (err);

 }
 
 }
 

-IOReturn IOSubMemoryDescriptor::performOperation( IOOptionBits options,
-                                        IOByteCount offset, IOByteCount length )
+IOMemoryMap * IOMemoryDescriptor::makeMapping(
+       IOMemoryDescriptor *    owner,
+       task_t                  __intoTask,
+       IOVirtualAddress        __address,
+       IOOptionBits            options,
+       IOByteCount             __offset,
+       IOByteCount             __length )

 {
 {

-    IOReturn err;
+#ifndef __LP64__
+    if (!(kIOMap64Bit & options)) panic("IOMemoryDescriptor::makeMapping !64bit");
+#endif /* !__LP64__ */

 
 

-    assert(offset <= _length);
+    IOMemoryDescriptor * mapDesc = 0;
+    IOMemoryMap *       result = 0;
+    OSIterator *        iter;
+
+    IOMemoryMap *  mapping = (IOMemoryMap *) __address;
+    mach_vm_size_t offset  = mapping->fOffset + __offset;
+    mach_vm_size_t length  = mapping->fLength;

 
 

-    if( offset >= _length)
-        return( kIOReturnOverrun );
+    mapping->fOffset = offset;

 
     LOCK;
 
     LOCK;

-    err = _parent->performOperation( options, _start + offset,
-                                      min(length, _length - offset) );
-    UNLOCK;

 
 

-    return( err );
-}
+    do
+    {
+       if (kIOMapStatic & options)
+       {
+           result = mapping;
+           addMapping(mapping);
+           mapping->setMemoryDescriptor(this, 0);
+           continue;
+       }

 
 

-IOReturn IOSubMemoryDescriptor::prepare(
-               IODirection forDirection)
-{
-    IOReturn   err;
+       if (kIOMapUnique & options)
+       {
+           addr64_t phys;
+           IOByteCount       physLen;

 
 

-    LOCK;
-    err = _parent->prepare( forDirection);
-    UNLOCK;
+//         if (owner != this)          continue;

 
 

-    return( err );
-}
+           if (((_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical)
+               || ((_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical64))
+           {
+               phys = getPhysicalSegment(offset, &physLen, kIOMemoryMapperNone);
+               if (!phys || (physLen < length))
+                   continue;
+    
+               mapDesc = IOMemoryDescriptor::withAddressRange(
+                               phys, length, getDirection() | kIOMemoryMapperNone, NULL);
+               if (!mapDesc)
+                   continue;
+               offset = 0;
+               mapping->fOffset = offset;
+           }
+       }
+       else
+       {
+           // look for a compatible existing mapping
+           if( (iter = OSCollectionIterator::withCollection(_mappings)))
+           {
+               IOMemoryMap * lookMapping;
+               while ((lookMapping = (IOMemoryMap *) iter->getNextObject()))
+               {
+                   if ((result = lookMapping->copyCompatible(mapping)))
+                   {
+                       addMapping(result);
+                       result->setMemoryDescriptor(this, offset);
+                       break;
+                   }
+               }
+               iter->release();
+           }
+           if (result || (options & kIOMapReference))
+           {
+               if (result != mapping)
+               {
+                    mapping->release();
+                    mapping = NULL;
+                }
+               continue;
+           }
+       }

 
 

-IOReturn IOSubMemoryDescriptor::complete(
-               IODirection forDirection)
-{
-    IOReturn   err;
+       if (!mapDesc)
+       {
+           mapDesc = this;
+           mapDesc->retain();
+       }
+       IOReturn
+       kr = mapDesc->doMap( 0, (IOVirtualAddress *) &mapping, options, 0, 0 );
+       if (kIOReturnSuccess == kr)
+       {
+           result = mapping;
+           mapDesc->addMapping(result);
+           result->setMemoryDescriptor(mapDesc, offset);
+       }
+       else
+       {
+           mapping->release();
+           mapping = NULL;
+       }
+    }
+    while( false );

 
 

-    LOCK;
-    err = _parent->complete( forDirection);

     UNLOCK;
 
     UNLOCK;
 

-    return( err );
+    if (mapDesc)
+       mapDesc->release();
+
+    return (result);

 }
 
 }
 

-IOMemoryMap * IOSubMemoryDescriptor::makeMapping(
-       IOMemoryDescriptor *    owner,
-       task_t                  intoTask,
-       IOVirtualAddress        toAddress,
-       IOOptionBits            options,
-       IOByteCount             offset,
-       IOByteCount             length )
+void IOMemoryDescriptor::addMapping(
+       IOMemoryMap * mapping )

 {
 {

-    IOMemoryMap * mapping = 0;
-
-    if (!(kIOMapUnique & options))
-       mapping = (IOMemoryMap *) _parent->makeMapping(
-                                       _parent, intoTask,
-                                       toAddress - (_start + offset),
-                                       options | kIOMapReference,
-                                       _start + offset, length );
-
-    if( !mapping)
-        mapping = (IOMemoryMap *) _parent->makeMapping(
-                                       _parent, intoTask,
-                                       toAddress,
-                                       options, _start + offset, length );
-
-    if( !mapping)
-       mapping = super::makeMapping( owner, intoTask, toAddress, options,
-                                       offset, length );
-
-    return( mapping );
+    if( mapping)
+    {
+        if( 0 == _mappings)
+            _mappings = OSSet::withCapacity(1);
+       if( _mappings )
+           _mappings->setObject( mapping );
+    }

 }
 
 }
 

-/* ick */
+void IOMemoryDescriptor::removeMapping(
+       IOMemoryMap * mapping )
+{
+    if( _mappings)
+        _mappings->removeObject( mapping);
+}

 
 

+#ifndef __LP64__
+// obsolete initializers
+// - initWithOptions is the designated initializer 

 bool
 bool

-IOSubMemoryDescriptor::initWithAddress(void *      address,
+IOMemoryDescriptor::initWithAddress(void *      address,

                                     IOByteCount   length,
                                     IODirection direction)
 {
                                     IOByteCount   length,
                                     IODirection direction)
 {


@@ -3546,7 +4307,7 @@ IOSubMemoryDescriptor::initWithAddress(void *      address,
 }
 
 bool
 }
 
 bool

-IOSubMemoryDescriptor::initWithAddress(vm_address_t address,
+IOMemoryDescriptor::initWithAddress(IOVirtualAddress address,

                                     IOByteCount    length,
                                     IODirection  direction,
                                     task_t       task)
                                     IOByteCount    length,
                                     IODirection  direction,
                                     task_t       task)


@@ -3555,7 +4316,7 @@ IOSubMemoryDescriptor::initWithAddress(vm_address_t address,
 }
 
 bool
 }
 
 bool

-IOSubMemoryDescriptor::initWithPhysicalAddress(
+IOMemoryDescriptor::initWithPhysicalAddress(

                                 IOPhysicalAddress      address,
                                 IOByteCount            length,
                                 IODirection            direction )
                                 IOPhysicalAddress      address,
                                 IOByteCount            length,
                                 IODirection            direction )


@@ -3564,7 +4325,7 @@ IOSubMemoryDescriptor::initWithPhysicalAddress(
 }
 
 bool
 }
 
 bool

-IOSubMemoryDescriptor::initWithRanges(
+IOMemoryDescriptor::initWithRanges(

                                        IOVirtualRange * ranges,
                                        UInt32           withCount,
                                        IODirection      direction,
                                        IOVirtualRange * ranges,
                                        UInt32           withCount,
                                        IODirection      direction,


@@ -3575,7 +4336,7 @@ IOSubMemoryDescriptor::initWithRanges(
 }
 
 bool
 }
 
 bool

-IOSubMemoryDescriptor::initWithPhysicalRanges( IOPhysicalRange * ranges,
+IOMemoryDescriptor::initWithPhysicalRanges(    IOPhysicalRange * ranges,

                                                UInt32           withCount,
                                                IODirection      direction,
                                                bool             asReference)
                                                UInt32           withCount,
                                                IODirection      direction,
                                                bool             asReference)


@@ -3583,12 +4344,21 @@ IOSubMemoryDescriptor::initWithPhysicalRanges(  IOPhysicalRange * ranges,
     return( false );
 }
 
     return( false );
 }
 

+void * IOMemoryDescriptor::getVirtualSegment(IOByteCount offset,
+                                       IOByteCount * lengthOfSegment)
+{
+    return( 0 );
+}
+#endif /* !__LP64__ */
+

 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 bool IOGeneralMemoryDescriptor::serialize(OSSerialize * s) const
 {
     OSSymbol const *keys[2];
     OSObject *values[2];
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 bool IOGeneralMemoryDescriptor::serialize(OSSerialize * s) const
 {
     OSSymbol const *keys[2];
     OSObject *values[2];

+    OSArray * array;
+

     struct SerData {
        user_addr_t address;
        user_size_t length;
     struct SerData {
        user_addr_t address;
        user_size_t length;


@@ -3599,10 +4369,9 @@ bool IOGeneralMemoryDescriptor::serialize(OSSerialize * s) const
     IOOptionBits type = _flags & kIOMemoryTypeMask;
 
     if (s == NULL) return false;
     IOOptionBits type = _flags & kIOMemoryTypeMask;
 
     if (s == NULL) return false;

-    if (s->previouslySerialized(this)) return true;

 
 

-    // Pretend we are an array.
-    if (!s->addXMLStartTag(this, "array")) return false;
+    array = OSArray::withCapacity(4);
+    if (!array)  return (false);

 
     nRanges = _rangesCount;
     vcopy = (SerData *) IOMalloc(sizeof(SerData) * nRanges);
 
     nRanges = _rangesCount;
     vcopy = (SerData *) IOMalloc(sizeof(SerData) * nRanges);


@@ -3622,7 +4391,7 @@ bool IOGeneralMemoryDescriptor::serialize(OSSerialize * s) const
     if (nRanges == _rangesCount) {
        Ranges vec = _ranges;
         for (index = 0; index < nRanges; index++) {
     if (nRanges == _rangesCount) {
        Ranges vec = _ranges;
         for (index = 0; index < nRanges; index++) {

-           user_addr_t addr; IOByteCount len;
+           mach_vm_address_t addr; mach_vm_size_t len;

            getAddrLenForInd(addr, len, type, vec, index);
             vcopy[index].address = addr;
             vcopy[index].length  = len;
            getAddrLenForInd(addr, len, type, vec, index);
             vcopy[index].address = addr;
             vcopy[index].length  = len;


@@ -3639,8 +4408,7 @@ bool IOGeneralMemoryDescriptor::serialize(OSSerialize * s) const
     {
        user_addr_t addr = vcopy[index].address;
        IOByteCount len = (IOByteCount) vcopy[index].length;
     {
        user_addr_t addr = vcopy[index].address;
        IOByteCount len = (IOByteCount) vcopy[index].length;

-       values[0] =
-           OSNumber::withNumber(addr, (((UInt64) addr) >> 32)? 64 : 32);
+       values[0] = OSNumber::withNumber(addr, sizeof(addr) * 8);

        if (values[0] == 0) {
          result = false;
          goto bail;
        if (values[0] == 0) {
          result = false;
          goto bail;


@@ -3655,19 +4423,18 @@ bool IOGeneralMemoryDescriptor::serialize(OSSerialize * s) const
          result = false;
          goto bail;
        }
          result = false;
          goto bail;
        }

+       array->setObject(dict);
+       dict->release();

        values[0]->release();
        values[1]->release();
        values[0] = values[1] = 0;
        values[0]->release();
        values[1]->release();
        values[0] = values[1] = 0;

-
-       result = dict->serialize(s);
-       dict->release();
-       if (!result) {
-         goto bail;
-       }

     }
     }

-    result = s->addXMLEndTag("array");
+
+    result = array->serialize(s);

 
  bail:
 
  bail:

+    if (array)
+      array->release();

     if (values[0])
       values[0]->release();
     if (values[1])
     if (values[0])
       values[0]->release();
     if (values[1])


@@ -3677,67 +4444,169 @@ bool IOGeneralMemoryDescriptor::serialize(OSSerialize * s) const
     if (keys[1])
       keys[1]->release();
     if (vcopy)
     if (keys[1])
       keys[1]->release();
     if (vcopy)

-        IOFree(vcopy, sizeof(IOVirtualRange) * nRanges);
+        IOFree(vcopy, sizeof(SerData) * nRanges);
+

     return result;
 }
 
     return result;
 }
 

-bool IOSubMemoryDescriptor::serialize(OSSerialize * s) const
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+#if DEVELOPMENT || DEBUG
+
+extern "C" void IOMemoryDescriptorTest(int x)

 {
 {

-    if (!s) {
-       return (false);
-    }
-    if (s->previouslySerialized(this)) return true;
+    IOGeneralMemoryDescriptor * md;

 
 

-    // Pretend we are a dictionary.
-    // We must duplicate the functionality of OSDictionary here
-    // because otherwise object references will not work;
-    // they are based on the value of the object passed to
-    // previouslySerialized and addXMLStartTag.
+    vm_offset_t data[2];
+    vm_size_t  bsize = 16*1024*1024;

 
 

-    if (!s->addXMLStartTag(this, "dict")) return false;
+    vm_size_t  srcsize, srcoffset, mapoffset, size;
+    
+    kern_return_t kr;

 
 

-    char const *keys[3] = {"offset", "length", "parent"};
+    kr = vm_allocate(kernel_map, &data[0], bsize, VM_FLAGS_ANYWHERE);
+    vm_inherit(kernel_map, data[0] + 1*4096, 4096, VM_INHERIT_NONE);
+    vm_inherit(kernel_map, data[0] + 16*4096, 4096, VM_INHERIT_NONE);

 
 

-    OSObject *values[3];
-    values[0] = OSNumber::withNumber(_start, sizeof(_start) * 8);
-    if (values[0] == 0)
-       return false;
-    values[1] = OSNumber::withNumber(_length, sizeof(_length) * 8);
-    if (values[1] == 0) {
-       values[0]->release();
-       return false;
-    }
-    values[2] = _parent;
+    kprintf("data 0x%lx, 0x%lx\n", (long)data[0], (long)data[1]);

 
 

-    bool result = true;
-    for (int i=0; i<3; i++) {
-        if (!s->addString("<key>") ||
-           !s->addString(keys[i]) ||
-           !s->addXMLEndTag("key") ||
-           !values[i]->serialize(s)) {
-         result = false;
-         break;
-        }
+    uint32_t idx, offidx;
+    for (idx = 0; idx < (bsize / sizeof(uint32_t)); idx++)
+    {
+       ((uint32_t*)data[0])[idx] = idx;    

     }
     }

-    values[0]->release();
-    values[1]->release();
-    if (!result) {
-      return false;
+
+    for (srcoffset = 0; srcoffset < bsize; srcoffset = ((srcoffset << 1) + 0x40c))
+    {
+       for (srcsize = 4; srcsize < (bsize - srcoffset - 1); srcsize = ((srcsize << 1) + 0x3fc))
+       {
+           IOAddressRange ranges[3];
+           uint32_t rangeCount = 1;
+
+           bzero(&ranges[0], sizeof(ranges));
+           ranges[0].address = data[0] + srcoffset;
+           ranges[0].length  = srcsize;
+
+           if (srcsize > 5*page_size)
+           {
+               ranges[0].length  = 7634;
+               ranges[1].length  = 9870;
+               ranges[2].length  = srcsize - ranges[0].length - ranges[1].length;
+               ranges[1].address = ranges[0].address + ranges[0].length;
+               ranges[2].address = ranges[1].address + ranges[1].length;
+               rangeCount = 3;     
+           }
+           else if ((srcsize > 2*page_size) && !(page_mask & srcoffset))
+           {
+               ranges[0].length  = 4096;
+               ranges[1].length  = 4096;
+               ranges[2].length  = srcsize - ranges[0].length - ranges[1].length;
+               ranges[0].address = data[0] + srcoffset + 4096;
+               ranges[1].address = data[0] + srcoffset;
+               ranges[2].address = ranges[0].address + ranges[0].length;
+               rangeCount = 3;     
+           }
+
+           md = OSDynamicCast(IOGeneralMemoryDescriptor, 
+               IOMemoryDescriptor::withAddressRanges(&ranges[0], rangeCount, kIODirectionInOut, kernel_task));
+           assert(md);
+
+           kprintf("IOMemoryReferenceCreate [0x%lx @ 0x%lx]\n[0x%llx, 0x%llx],\n[0x%llx, 0x%llx],\n[0x%llx, 0x%llx]\n", 
+                   (long) srcsize, (long) srcoffset,
+                   (long long) ranges[0].address - data[0], (long long) ranges[0].length,
+                   (long long) ranges[1].address - data[0], (long long) ranges[1].length,
+                   (long long) ranges[2].address - data[0], (long long) ranges[2].length);
+
+           if (kIOReturnSuccess == kr)
+           {
+               for (mapoffset = 0; mapoffset < srcsize; mapoffset = ((mapoffset << 1) + 0xf00))
+               {
+                   for (size = 4; size < (srcsize - mapoffset - 1); size = ((size << 1) + 0x20))
+                   {
+                       IOMemoryMap     * map;
+                       mach_vm_address_t addr = 0;
+                       uint32_t          data;
+
+                       kprintf("<mapRef [0x%lx @ 0x%lx]\n", (long) size, (long) mapoffset);
+
+                       map = md->createMappingInTask(kernel_task, 0, kIOMapAnywhere, mapoffset, size);
+                       if (map) addr = map->getAddress();
+                       else kr = kIOReturnError;
+
+                       kprintf(">mapRef 0x%x %llx\n", kr, addr);
+
+                       if (kIOReturnSuccess != kr) break;
+                       kr = md->prepare();
+                       if (kIOReturnSuccess != kr)
+                       {
+                           kprintf("prepare() fail 0x%x\n", kr);
+                           break;
+                       }
+                       for (idx = 0; idx < size; idx += sizeof(uint32_t))
+                       {
+                           offidx = (idx + mapoffset + srcoffset);
+                           if ((srcsize <= 5*page_size) && (srcsize > 2*page_size) && !(page_mask & srcoffset))
+                           {
+                               if (offidx < 8192) offidx ^= 0x1000;
+                           }
+                           offidx /= sizeof(uint32_t);
+
+                           if (offidx != ((uint32_t*)addr)[idx/sizeof(uint32_t)]) 
+                           {
+                               kprintf("vm mismatch @ 0x%x, 0x%lx, 0x%lx, \n", idx, (long) srcoffset, (long) mapoffset);
+                               kr = kIOReturnBadMedia;
+                           }
+                           else
+                           {
+                               if (sizeof(data) != md->readBytes(mapoffset + idx, &data, sizeof(data))) data = 0;
+                               if (offidx != data) 
+                               {
+                                   kprintf("phys mismatch @ 0x%x, 0x%lx, 0x%lx, \n", idx, (long) srcoffset, (long) mapoffset);
+                                   kr = kIOReturnBadMedia;
+                               }
+                           }
+                       }
+                       md->complete();
+                       map->release();
+                       kprintf("unmapRef %llx\n", addr);
+                   }
+                   if (kIOReturnSuccess != kr) break;
+               }
+           }
+            if (kIOReturnSuccess != kr) break;
+       }
+       if (kIOReturnSuccess != kr) break;

     }
 
     }
 

-    return s->addXMLEndTag("dict");
+    if (kIOReturnSuccess != kr) kprintf("FAIL: src 0x%lx @ 0x%lx, map 0x%lx @ 0x%lx\n", 
+                                       (long) srcsize, (long) srcoffset, (long) size, (long) mapoffset);
+
+    vm_deallocate(kernel_map, data[0], bsize);
+//    vm_deallocate(kernel_map, data[1], size);

 }
 
 }
 

+#endif  /* DEVELOPMENT || DEBUG */
+

 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 0);
 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 0);

+#ifdef __LP64__
+OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 1);
+OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 2);
+OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 3);
+OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 4);
+OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 5);
+OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 6);
+OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 7);
+#else /* !__LP64__ */

 OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 1);
 OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 2);
 OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 3);
 OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 4);
 OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 5);
 OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 1);
 OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 2);
 OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 3);
 OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 4);
 OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 5);

-OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 6);
-OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 7);
+OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 6);
+OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 7);
+#endif /* !__LP64__ */

 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 8);
 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 9);
 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 10);
 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 8);
 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 9);
 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 10);


@@ -3751,6 +4620,3 @@ OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 15);
 IOPhysicalAddress 
 IOMemoryDescriptor::getPhysicalAddress()
         { return( getPhysicalSegment( 0, 0 )); }
 IOPhysicalAddress 
 IOMemoryDescriptor::getPhysicalAddress()
         { return( getPhysicalSegment( 0, 0 )); }

-
-
-