iokit/Kernel/IOMemoryDescriptor.cpp

   1 /*
   2  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * The contents of this file constitute Original Code as defined in and
   7  * are subject to the Apple Public Source License Version 1.1 (the
   8  * "License").  You may not use this file except in compliance with the
   9  * License.  Please obtain a copy of the License at
  10  * http://www.apple.com/publicsource and read it before using this file.
  11  *
  12  * This Original Code and all software distributed under the License are
  13  * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  14  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  15  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
  17  * License for the specific language governing rights and limitations
  18  * under the License.
  19  *
  20  * @APPLE_LICENSE_HEADER_END@
  21  */
  22 /*
  23  * Copyright (c) 1998 Apple Computer, Inc.  All rights reserved.
  24  *
  25  * HISTORY
  26  *
  27  */
  28 // 45678901234567890123456789012345678901234567890123456789012345678901234567890
  29 #include <sys/cdefs.h>
  30
  31 #include <IOKit/assert.h>
  32 #include <IOKit/system.h>
  33 #include <IOKit/IOLib.h>
  34 #include <IOKit/IOMemoryDescriptor.h>
  35 #include <IOKit/IOMapper.h>
  36 #include <IOKit/IOKitKeysPrivate.h>
  37
  38 #include <IOKit/IOKitDebug.h>
  39
  40 #include <libkern/c++/OSContainers.h>
  41 #include <libkern/c++/OSDictionary.h>
  42 #include <libkern/c++/OSArray.h>
  43 #include <libkern/c++/OSSymbol.h>
  44 #include <libkern/c++/OSNumber.h>
  45 #include <sys/cdefs.h>
  46
  47 __BEGIN_DECLS
  48 #include <vm/pmap.h>
  49 #include <mach/memory_object_types.h>
  50 #include <device/device_port.h>
  51
  52 #ifndef i386
  53 struct phys_entry      *pmap_find_physentry(ppnum_t pa);
  54 #endif
  55 extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
  56 void ipc_port_release_send(ipc_port_t port);
  57
  58 /* Copy between a physical page and a virtual address in the given vm_map */
  59 kern_return_t copypv(addr64_t source, addr64_t sink, unsigned int size, int which);
  60
  61 memory_object_t
  62 device_pager_setup(
  63         memory_object_t pager,
  64         int             device_handle,
  65         vm_size_t       size,
  66         int             flags);
  67 void
  68 device_pager_deallocate(
  69         memory_object_t);
  70 kern_return_t
  71 device_pager_populate_object(
  72         memory_object_t         pager,
  73         vm_object_offset_t      offset,
  74         ppnum_t                 phys_addr,
  75         vm_size_t               size);
  76 kern_return_t
  77 memory_object_iopl_request(
  78         ipc_port_t              port,
  79         memory_object_offset_t  offset,
  80         vm_size_t               *upl_size,
  81         upl_t                   *upl_ptr,
  82         upl_page_info_array_t   user_page_list,
  83         unsigned int            *page_list_count,
  84         int                     *flags);
  85
  86 /*
  87  *      Page fault handling based on vm_map (or entries therein)
  88  */
  89 extern kern_return_t vm_fault(
  90                 vm_map_t        map,
  91                 vm_offset_t     vaddr,
  92                 vm_prot_t       fault_type,
  93                 boolean_t       change_wiring,
  94                 int             interruptible,
  95                 pmap_t          caller_pmap,
  96                 vm_offset_t     caller_pmap_addr);
  97
  98 unsigned int  IOTranslateCacheBits(struct phys_entry *pp);
  99
 100 vm_map_t IOPageableMapForAddress( vm_address_t address );
 101
 102 typedef kern_return_t (*IOIteratePageableMapsCallback)(vm_map_t map, void * ref);
 103
 104 kern_return_t IOIteratePageableMaps(vm_size_t size,
 105                     IOIteratePageableMapsCallback callback, void * ref);
 106 __END_DECLS
 107
 108 #define kIOMaximumMappedIOByteCount     (512*1024*1024)
 109
 110 static IOMapper * gIOSystemMapper;
 111 static ppnum_t    gIOMaximumMappedIOPageCount = atop_32(kIOMaximumMappedIOByteCount);
 112
 113 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 114
 115 OSDefineMetaClassAndAbstractStructors( IOMemoryDescriptor, OSObject )
 116
 117 #define super IOMemoryDescriptor
 118
 119 OSDefineMetaClassAndStructors(IOGeneralMemoryDescriptor, IOMemoryDescriptor)
 120
 121 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 122
 123 static IORecursiveLock * gIOMemoryLock;
 124
 125 #define LOCK    IORecursiveLockLock( gIOMemoryLock)
 126 #define UNLOCK  IORecursiveLockUnlock( gIOMemoryLock)
 127 #define SLEEP   IORecursiveLockSleep( gIOMemoryLock, (void *)this, THREAD_UNINT)
 128 #define WAKEUP  \
 129     IORecursiveLockWakeup( gIOMemoryLock, (void *)this, /* one-thread */ false)
 130
 131 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 132
 133 #define next_page(a) ( trunc_page_32(a) + PAGE_SIZE )
 134
 135
 136 extern "C" {
 137
 138 kern_return_t device_data_action(
 139                int                     device_handle,
 140                ipc_port_t              device_pager,
 141                vm_prot_t               protection,
 142                vm_object_offset_t      offset,
 143                vm_size_t               size)
 144 {
 145     struct ExpansionData {
 146         void *                          devicePager;
 147         unsigned int                    pagerContig:1;
 148         unsigned int                    unused:31;
 149         IOMemoryDescriptor *            memory;
 150     };
 151     kern_return_t        kr;
 152     ExpansionData *      ref = (ExpansionData *) device_handle;
 153     IOMemoryDescriptor * memDesc;
 154
 155     LOCK;
 156     memDesc = ref->memory;
 157     if( memDesc)
 158         kr = memDesc->handleFault( device_pager, 0, 0,
 159                 offset, size, kIOMapDefaultCache /*?*/);
 160     else
 161         kr = KERN_ABORTED;
 162     UNLOCK;
 163
 164     return( kr );
 165 }
 166
 167 kern_return_t device_close(
 168                int     device_handle)
 169 {
 170     struct ExpansionData {
 171         void *                          devicePager;
 172         unsigned int                    pagerContig:1;
 173         unsigned int                    unused:31;
 174         IOMemoryDescriptor *            memory;
 175     };
 176     ExpansionData *   ref = (ExpansionData *) device_handle;
 177
 178     IODelete( ref, ExpansionData, 1 );
 179
 180     return( kIOReturnSuccess );
 181 }
 182
 183 }
 184
 185 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 186
 187 /*
 188  * withAddress:
 189  *
 190  * Create a new IOMemoryDescriptor.  The buffer is a virtual address
 191  * relative to the specified task.  If no task is supplied, the kernel
 192  * task is implied.
 193  */
 194 IOMemoryDescriptor *
 195 IOMemoryDescriptor::withAddress(void *      address,
 196                                 IOByteCount   length,
 197                                 IODirection direction)
 198 {
 199     return IOMemoryDescriptor::
 200         withAddress((vm_address_t) address, length, direction, kernel_task);
 201 }
 202
 203 IOMemoryDescriptor *
 204 IOMemoryDescriptor::withAddress(vm_address_t address,
 205                                 IOByteCount  length,
 206                                 IODirection  direction,
 207                                 task_t       task)
 208 {
 209     IOGeneralMemoryDescriptor * that = new IOGeneralMemoryDescriptor;
 210     if (that)
 211     {
 212         if (that->initWithAddress(address, length, direction, task))
 213             return that;
 214
 215         that->release();
 216     }
 217     return 0;
 218 }
 219
 220 IOMemoryDescriptor *
 221 IOMemoryDescriptor::withPhysicalAddress(
 222                                 IOPhysicalAddress       address,
 223                                 IOByteCount             length,
 224                                 IODirection             direction )
 225 {
 226     IOGeneralMemoryDescriptor *self = new IOGeneralMemoryDescriptor;
 227     if (self
 228     && !self->initWithPhysicalAddress(address, length, direction)) {
 229         self->release();
 230         return 0;
 231     }
 232
 233     return self;
 234 }
 235
 236 IOMemoryDescriptor *
 237 IOMemoryDescriptor::withRanges( IOVirtualRange * ranges,
 238                                 UInt32           withCount,
 239                                 IODirection      direction,
 240                                 task_t           task,
 241                                 bool             asReference)
 242 {
 243     IOGeneralMemoryDescriptor * that = new IOGeneralMemoryDescriptor;
 244     if (that)
 245     {
 246         if (that->initWithRanges(ranges, withCount, direction, task, asReference))
 247             return that;
 248
 249         that->release();
 250     }
 251     return 0;
 252 }
 253
 254
 255 /*
 256  * withRanges:
 257  *
 258  * Create a new IOMemoryDescriptor. The buffer is made up of several
 259  * virtual address ranges, from a given task.
 260  *
 261  * Passing the ranges as a reference will avoid an extra allocation.
 262  */
 263 IOMemoryDescriptor *
 264 IOMemoryDescriptor::withOptions(void *          buffers,
 265                                 UInt32          count,
 266                                 UInt32          offset,
 267                                 task_t          task,
 268                                 IOOptionBits    opts,
 269                                 IOMapper *      mapper)
 270 {
 271     IOGeneralMemoryDescriptor *self = new IOGeneralMemoryDescriptor;
 272
 273     if (self
 274     && !self->initWithOptions(buffers, count, offset, task, opts, mapper))
 275     {
 276         self->release();
 277         return 0;
 278     }
 279
 280     return self;
 281 }
 282
 283 // Can't leave abstract but this should never be used directly,
 284 bool IOMemoryDescriptor::initWithOptions(void *         buffers,
 285                                          UInt32         count,
 286                                          UInt32         offset,
 287                                          task_t         task,
 288                                          IOOptionBits   options,
 289                                          IOMapper *     mapper)
 290 {
 291     // @@@ gvdl: Should I panic?
 292     panic("IOMD::initWithOptions called\n");
 293     return 0;
 294 }
 295
 296 IOMemoryDescriptor *
 297 IOMemoryDescriptor::withPhysicalRanges( IOPhysicalRange * ranges,
 298                                         UInt32          withCount,
 299                                         IODirection     direction,
 300                                         bool            asReference)
 301 {
 302     IOGeneralMemoryDescriptor * that = new IOGeneralMemoryDescriptor;
 303     if (that)
 304     {
 305         if (that->initWithPhysicalRanges(ranges, withCount, direction, asReference))
 306             return that;
 307
 308         that->release();
 309     }
 310     return 0;
 311 }
 312
 313 IOMemoryDescriptor *
 314 IOMemoryDescriptor::withSubRange(IOMemoryDescriptor *   of,
 315                                 IOByteCount             offset,
 316                                 IOByteCount             length,
 317                                 IODirection             direction)
 318 {
 319     IOSubMemoryDescriptor *self = new IOSubMemoryDescriptor;
 320
 321     if (self && !self->initSubRange(of, offset, length, direction)) {
 322         self->release();
 323         self = 0;
 324     }
 325     return self;
 326 }
 327
 328 /*
 329  * initWithAddress:
 330  *
 331  * Initialize an IOMemoryDescriptor. The buffer is a virtual address
 332  * relative to the specified task.  If no task is supplied, the kernel
 333  * task is implied.
 334  *
 335  * An IOMemoryDescriptor can be re-used by calling initWithAddress or
 336  * initWithRanges again on an existing instance -- note this behavior
 337  * is not commonly supported in other I/O Kit classes, although it is
 338  * supported here.
 339  */
 340 bool
 341 IOGeneralMemoryDescriptor::initWithAddress(void *      address,
 342                                     IOByteCount   withLength,
 343                                     IODirection withDirection)
 344 {
 345     _singleRange.v.address = (vm_address_t) address;
 346     _singleRange.v.length  = withLength;
 347
 348     return initWithRanges(&_singleRange.v, 1, withDirection, kernel_task, true);
 349 }
 350
 351 bool
 352 IOGeneralMemoryDescriptor::initWithAddress(vm_address_t address,
 353                                     IOByteCount    withLength,
 354                                     IODirection  withDirection,
 355                                     task_t       withTask)
 356 {
 357     _singleRange.v.address = address;
 358     _singleRange.v.length  = withLength;
 359
 360     return initWithRanges(&_singleRange.v, 1, withDirection, withTask, true);
 361 }
 362
 363 bool
 364 IOGeneralMemoryDescriptor::initWithPhysicalAddress(
 365                                  IOPhysicalAddress      address,
 366                                  IOByteCount            withLength,
 367                                  IODirection            withDirection )
 368 {
 369     _singleRange.p.address = address;
 370     _singleRange.p.length  = withLength;
 371
 372     return initWithPhysicalRanges( &_singleRange.p, 1, withDirection, true);
 373 }
 374
 375 bool
 376 IOGeneralMemoryDescriptor::initWithPhysicalRanges(
 377                                 IOPhysicalRange * ranges,
 378                                 UInt32            count,
 379                                 IODirection       direction,
 380                                 bool              reference)
 381 {
 382     IOOptionBits mdOpts = direction | kIOMemoryTypePhysical;
 383
 384     if (reference)
 385         mdOpts |= kIOMemoryAsReference;
 386
 387     return initWithOptions(ranges, count, 0, 0, mdOpts, /* mapper */ 0);
 388 }
 389
 390 bool
 391 IOGeneralMemoryDescriptor::initWithRanges(
 392                                    IOVirtualRange * ranges,
 393                                    UInt32           count,
 394                                    IODirection      direction,
 395                                    task_t           task,
 396                                    bool             reference)
 397 {
 398     IOOptionBits mdOpts = direction;
 399
 400     if (reference)
 401         mdOpts |= kIOMemoryAsReference;
 402
 403     if (task) {
 404         mdOpts |= kIOMemoryTypeVirtual;
 405         if (task == kernel_task)
 406             mdOpts |= kIOMemoryAutoPrepare;
 407     }
 408     else
 409         mdOpts |= kIOMemoryTypePhysical;
 410
 411     // @@@ gvdl: Need to remove this
 412     // Auto-prepare if this is a kernel memory descriptor as very few
 413     // clients bother to prepare() kernel memory.
 414     // But it has been enforced so what are you going to do?
 415
 416     return initWithOptions(ranges, count, 0, task, mdOpts, /* mapper */ 0);
 417 }
 418
 419 /*
 420  * initWithOptions:
 421  *
 422  *  IOMemoryDescriptor. The buffer is made up of several virtual address ranges,
 423  * from a given task or several physical ranges or finally an UPL from the ubc
 424  * system.
 425  *
 426  * Passing the ranges as a reference will avoid an extra allocation.
 427  *
 428  * An IOMemoryDescriptor can be re-used by calling initWithOptions again on an
 429  * existing instance -- note this behavior is not commonly supported in other
 430  * I/O Kit classes, although it is supported here.
 431  */
 432
 433 enum ioPLBlockFlags {
 434     kIOPLOnDevice  = 0x00000001,
 435     kIOPLExternUPL = 0x00000002,
 436 };
 437
 438 struct ioPLBlock {
 439     upl_t fIOPL;
 440     vm_address_t fIOMDOffset;   // The offset of this iopl in descriptor
 441     vm_offset_t fPageInfo;      // Pointer to page list or index into it
 442     ppnum_t fMappedBase;        // Page number of first page in this iopl
 443     unsigned int fPageOffset;   // Offset within first page of iopl
 444     unsigned int fFlags;        // Flags
 445 };
 446
 447 struct ioGMDData {
 448     IOMapper *fMapper;
 449     unsigned int fPageCnt;
 450     upl_page_info_t fPageList[0];       // @@@ gvdl need to get rid of this
 451                                         //  should be able to use upl directly
 452     ioPLBlock fBlocks[0];
 453 };
 454
 455 #define getDataP(osd)   ((ioGMDData *) (osd)->getBytesNoCopy())
 456 #define getIOPLList(d)  ((ioPLBlock *) &(d->fPageList[d->fPageCnt]))
 457 #define getNumIOPL(d,len)       \
 458     ((len - ((char *) getIOPLList(d) - (char *) d)) / sizeof(ioPLBlock))
 459 #define getPageList(d)  (&(d->fPageList[0]))
 460 #define computeDataSize(p, u) \
 461     (sizeof(ioGMDData) + p * sizeof(upl_page_info_t) + u * sizeof(ioPLBlock))
 462
 463 bool
 464 IOGeneralMemoryDescriptor::initWithOptions(void *       buffers,
 465                                            UInt32       count,
 466                                            UInt32       offset,
 467                                            task_t       task,
 468                                            IOOptionBits options,
 469                                            IOMapper *   mapper)
 470 {
 471
 472     switch (options & kIOMemoryTypeMask) {
 473     case kIOMemoryTypeVirtual:
 474         assert(task);
 475         if (!task)
 476             return false;
 477         else
 478             break;
 479
 480     case kIOMemoryTypePhysical:         // Neither Physical nor UPL should have a task
 481         mapper = kIOMapperNone;
 482     case kIOMemoryTypeUPL:
 483         assert(!task);
 484         break;
 485     default:
 486 panic("IOGMD::iWO(): bad type");        // @@@ gvdl: for testing
 487         return false;   /* bad argument */
 488     }
 489
 490     assert(buffers);
 491     assert(count);
 492
 493     /*
 494      * We can check the _initialized  instance variable before having ever set
 495      * it to an initial value because I/O Kit guarantees that all our instance
 496      * variables are zeroed on an object's allocation.
 497      */
 498
 499     if (_initialized) {
 500         /*
 501          * An existing memory descriptor is being retargeted to point to
 502          * somewhere else.  Clean up our present state.
 503          */
 504
 505         while (_wireCount)
 506             complete();
 507         if (_kernPtrAligned)
 508             unmapFromKernel();
 509         if (_ranges.v && _rangesIsAllocated)
 510             IODelete(_ranges.v, IOVirtualRange, _rangesCount);
 511     }
 512     else {
 513         if (!super::init())
 514             return false;
 515         _initialized = true;
 516     }
 517
 518     // Grab the appropriate mapper
 519     if (mapper == kIOMapperNone)
 520         mapper = 0;     // No Mapper
 521     else if (!mapper) {
 522         IOMapper::checkForSystemMapper();
 523         gIOSystemMapper = mapper = IOMapper::gSystem;
 524     }
 525
 526     _flags                 = options;
 527     _task                  = task;
 528
 529     // DEPRECATED variable initialisation
 530     _direction             = (IODirection) (_flags & kIOMemoryDirectionMask);
 531     _position              = 0;
 532     _kernPtrAligned        = 0;
 533     _cachedPhysicalAddress = 0;
 534     _cachedVirtualAddress  = 0;
 535
 536     if ( (options & kIOMemoryTypeMask) == kIOMemoryTypeUPL) {
 537
 538         ioGMDData *dataP;
 539         unsigned int dataSize = computeDataSize(/* pages */ 0, /* upls */ 1);
 540
 541         if (!_memoryEntries) {
 542             _memoryEntries = OSData::withCapacity(dataSize);
 543             if (!_memoryEntries)
 544                 return false;
 545         }
 546         else if (!_memoryEntries->initWithCapacity(dataSize))
 547             return false;
 548
 549         _memoryEntries->appendBytes(0, sizeof(ioGMDData));
 550         dataP = getDataP(_memoryEntries);
 551         dataP->fMapper = mapper;
 552         dataP->fPageCnt = 0;
 553
 554         _wireCount++;   // UPLs start out life wired
 555
 556         _length    = count;
 557         _pages    += atop_32(offset + count + PAGE_MASK) - atop_32(offset);
 558
 559         ioPLBlock iopl;
 560         upl_page_info_t *pageList = UPL_GET_INTERNAL_PAGE_LIST((upl_t) buffers);
 561
 562         iopl.fIOPL = (upl_t) buffers;
 563         // Set the flag kIOPLOnDevice convieniently equal to 1
 564         iopl.fFlags  = pageList->device | kIOPLExternUPL;
 565         iopl.fIOMDOffset = 0;
 566         if (!pageList->device) {
 567             // @@@ gvdl: Ask JoeS are the pages contiguious with the list?
 568             // or there a chance that we may be inserting 0 phys_addrs?
 569             // Pre-compute the offset into the UPL's page list
 570             pageList = &pageList[atop_32(offset)];
 571             offset &= PAGE_MASK;
 572             if (mapper) {
 573                 iopl.fMappedBase = mapper->iovmAlloc(_pages);
 574                 mapper->iovmInsert(iopl.fMappedBase, 0, pageList, _pages);
 575             }
 576             else
 577                 iopl.fMappedBase = 0;
 578         }
 579         else
 580             iopl.fMappedBase = 0;
 581         iopl.fPageInfo = (vm_address_t) pageList;
 582         iopl.fPageOffset = offset;
 583
 584         _memoryEntries->appendBytes(&iopl, sizeof(iopl));
 585     }
 586     else {      /* kIOMemoryTypeVirtual | kIOMemoryTypePhysical */
 587         IOVirtualRange *ranges = (IOVirtualRange *) buffers;
 588
 589         /*
 590          * Initialize the memory descriptor.
 591          */
 592
 593         _length                 = 0;
 594         _pages                  = 0;
 595         for (unsigned ind = 0; ind < count; ind++) {
 596             IOVirtualRange cur = ranges[ind];
 597
 598             _length += cur.length;
 599             _pages += atop_32(cur.address + cur.length + PAGE_MASK)
 600                    -  atop_32(cur.address);
 601         }
 602
 603         _ranges.v               = 0;
 604         _rangesIsAllocated      = !(options & kIOMemoryAsReference);
 605         _rangesCount            = count;
 606
 607         if (options & kIOMemoryAsReference)
 608             _ranges.v = ranges;
 609         else {
 610             _ranges.v = IONew(IOVirtualRange, count);
 611             if (!_ranges.v)
 612                 return false;
 613             bcopy(/* from */ ranges, _ranges.v,
 614                   count * sizeof(IOVirtualRange));
 615         }
 616
 617         // Auto-prepare memory at creation time.
 618         // Implied completion when descriptor is free-ed
 619         if ( (options & kIOMemoryTypeMask) == kIOMemoryTypePhysical)
 620             _wireCount++;       // Physical MDs are start out wired
 621         else { /* kIOMemoryTypeVirtual */
 622             ioGMDData *dataP;
 623             unsigned int dataSize =
 624                 computeDataSize(_pages, /* upls */ _rangesCount * 2);
 625
 626             if (!_memoryEntries) {
 627                 _memoryEntries = OSData::withCapacity(dataSize);
 628                 if (!_memoryEntries)
 629                     return false;
 630             }
 631             else if (!_memoryEntries->initWithCapacity(dataSize))
 632                 return false;
 633
 634             _memoryEntries->appendBytes(0, sizeof(ioGMDData));
 635             dataP = getDataP(_memoryEntries);
 636             dataP->fMapper = mapper;
 637             dataP->fPageCnt = _pages;
 638
 639             if (kIOMemoryPersistent & _flags)
 640             {
 641                 kern_return_t error;
 642                 ipc_port_t sharedMem;
 643
 644                 vm_size_t size = _pages << PAGE_SHIFT;
 645                 vm_address_t startPage;
 646
 647                 startPage = trunc_page_32(_ranges.v[0].address);
 648
 649                 vm_map_t theMap = ((_task == kernel_task) && (kIOMemoryBufferPageable & _flags))
 650                                 ? IOPageableMapForAddress(startPage)
 651                                 : get_task_map(_task);
 652
 653                 vm_size_t actualSize = size;
 654                 error = mach_make_memory_entry( theMap,
 655                             &actualSize, startPage,
 656                             VM_PROT_READ | VM_PROT_WRITE, &sharedMem,
 657                             NULL );
 658
 659                 if (KERN_SUCCESS == error) {
 660                     if (actualSize == round_page_32(size)) {
 661                         _memEntry = (void *) sharedMem;
 662                     } else {
 663 #if IOASSERT
 664                         IOLog("mach_make_memory_entry_64 (%08x) size (%08lx:%08x)\n",
 665                                     startPage, (UInt32)actualSize, size);
 666 #endif
 667                         ipc_port_release_send( sharedMem );
 668                     }
 669                 }
 670             }
 671
 672             if ((_flags & kIOMemoryAutoPrepare)
 673              && prepare() != kIOReturnSuccess)
 674                 return false;
 675         }
 676     }
 677
 678     return true;
 679 }
 680
 681 /*
 682  * free
 683  *
 684  * Free resources.
 685  */
 686 void IOGeneralMemoryDescriptor::free()
 687 {
 688     LOCK;
 689     if( reserved)
 690         reserved->memory = 0;
 691     UNLOCK;
 692
 693     while (_wireCount)
 694         complete();
 695     if (_memoryEntries)
 696         _memoryEntries->release();
 697
 698     if (_kernPtrAligned)
 699         unmapFromKernel();
 700     if (_ranges.v && _rangesIsAllocated)
 701         IODelete(_ranges.v, IOVirtualRange, _rangesCount);
 702
 703     if (reserved && reserved->devicePager)
 704         device_pager_deallocate( (memory_object_t) reserved->devicePager );
 705
 706     // memEntry holds a ref on the device pager which owns reserved
 707     // (ExpansionData) so no reserved access after this point
 708     if (_memEntry)
 709         ipc_port_release_send( (ipc_port_t) _memEntry );
 710
 711     super::free();
 712 }
 713
 714 /* DEPRECATED */ void IOGeneralMemoryDescriptor::unmapFromKernel()
 715 /* DEPRECATED */ {
 716                     panic("IOGMD::unmapFromKernel deprecated");
 717 /* DEPRECATED */ }
 718 /* DEPRECATED */
 719 /* DEPRECATED */ void IOGeneralMemoryDescriptor::mapIntoKernel(unsigned rangeIndex)
 720 /* DEPRECATED */ {
 721                     panic("IOGMD::mapIntoKernel deprecated");
 722 /* DEPRECATED */ }
 723
 724 /*
 725  * getDirection:
 726  *
 727  * Get the direction of the transfer.
 728  */
 729 IODirection IOMemoryDescriptor::getDirection() const
 730 {
 731     return _direction;
 732 }
 733
 734 /*
 735  * getLength:
 736  *
 737  * Get the length of the transfer (over all ranges).
 738  */
 739 IOByteCount IOMemoryDescriptor::getLength() const
 740 {
 741     return _length;
 742 }
 743
 744 void IOMemoryDescriptor::setTag( IOOptionBits tag )
 745 {
 746     _tag = tag;
 747 }
 748
 749 IOOptionBits IOMemoryDescriptor::getTag( void )
 750 {
 751     return( _tag);
 752 }
 753
 754 // @@@ gvdl: who is using this API?  Seems like a wierd thing to implement.
 755 IOPhysicalAddress IOMemoryDescriptor::getSourceSegment( IOByteCount   offset,
 756                                                         IOByteCount * length )
 757 {
 758     IOPhysicalAddress physAddr = 0;
 759
 760     if( prepare() == kIOReturnSuccess) {
 761         physAddr = getPhysicalSegment( offset, length );
 762         complete();
 763     }
 764
 765     return( physAddr );
 766 }
 767
 768 IOByteCount IOMemoryDescriptor::readBytes
 769                 (IOByteCount offset, void *bytes, IOByteCount length)
 770 {
 771     addr64_t dstAddr = (addr64_t) (UInt32) bytes;
 772     IOByteCount remaining;
 773
 774     // Assert that this entire I/O is withing the available range
 775     assert(offset < _length);
 776     assert(offset + length <= _length);
 777     if (offset >= _length) {
 778 IOLog("IOGMD(%p): rB = o%lx, l%lx\n", this, offset, length);    // @@@ gvdl
 779         return 0;
 780     }
 781
 782     remaining = length = min(length, _length - offset);
 783     while (remaining) { // (process another target segment?)
 784         addr64_t        srcAddr64;
 785         IOByteCount     srcLen;
 786
 787         srcAddr64 = getPhysicalSegment64(offset, &srcLen);
 788         if (!srcAddr64)
 789             break;
 790
 791         // Clip segment length to remaining
 792         if (srcLen > remaining)
 793             srcLen = remaining;
 794
 795         copypv(srcAddr64, dstAddr, srcLen,
 796                             cppvPsrc | cppvNoRefSrc | cppvFsnk | cppvKmap);
 797
 798         dstAddr   += srcLen;
 799         offset    += srcLen;
 800         remaining -= srcLen;
 801     }
 802
 803     assert(!remaining);
 804
 805     return length - remaining;
 806 }
 807
 808 IOByteCount IOMemoryDescriptor::writeBytes
 809                 (IOByteCount offset, const void *bytes, IOByteCount length)
 810 {
 811     addr64_t srcAddr = (addr64_t) (UInt32) bytes;
 812     IOByteCount remaining;
 813
 814     // Assert that this entire I/O is withing the available range
 815     assert(offset < _length);
 816     assert(offset + length <= _length);
 817
 818     assert( !(kIOMemoryPreparedReadOnly & _flags) );
 819
 820     if ( (kIOMemoryPreparedReadOnly & _flags) || offset >= _length) {
 821 IOLog("IOGMD(%p): wB = o%lx, l%lx\n", this, offset, length);    // @@@ gvdl
 822         return 0;
 823     }
 824
 825     remaining = length = min(length, _length - offset);
 826     while (remaining) { // (process another target segment?)
 827         addr64_t    dstAddr64;
 828         IOByteCount dstLen;
 829
 830         dstAddr64 = getPhysicalSegment64(offset, &dstLen);
 831         if (!dstAddr64)
 832             break;
 833
 834         // Clip segment length to remaining
 835         if (dstLen > remaining)
 836             dstLen = remaining;
 837
 838         copypv(srcAddr, (addr64_t) dstAddr64, dstLen,
 839                             cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
 840
 841         srcAddr   += dstLen;
 842         offset    += dstLen;
 843         remaining -= dstLen;
 844     }
 845
 846     assert(!remaining);
 847
 848     return length - remaining;
 849 }
 850
 851 // osfmk/device/iokit_rpc.c
 852 extern "C" unsigned int IODefaultCacheBits(addr64_t pa);
 853
 854 /* DEPRECATED */ void IOGeneralMemoryDescriptor::setPosition(IOByteCount position)
 855 /* DEPRECATED */ {
 856                     panic("IOGMD::setPosition deprecated");
 857 /* DEPRECATED */ }
 858
 859 IOPhysicalAddress IOGeneralMemoryDescriptor::getPhysicalSegment
 860                         (IOByteCount offset, IOByteCount *lengthOfSegment)
 861 {
 862     IOPhysicalAddress address = 0;
 863     IOPhysicalLength  length  = 0;
 864
 865 //  assert(offset <= _length);
 866     if (offset < _length) // (within bounds?)
 867     {
 868         if ( (_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical) {
 869             unsigned int ind;
 870
 871             // Physical address based memory descriptor
 872
 873             // Find offset within descriptor and make it relative
 874             // to the current _range.
 875             for (ind = 0 ; offset >= _ranges.p[ind].length; ind++ )
 876                 offset -= _ranges.p[ind].length;
 877
 878             IOPhysicalRange cur = _ranges.p[ind];
 879             address = cur.address + offset;
 880             length  = cur.length  - offset;
 881
 882             // see how far we can coalesce ranges
 883             for (++ind; ind < _rangesCount; ind++) {
 884                 cur =  _ranges.p[ind];
 885
 886                 if (address + length != cur.address)
 887                     break;
 888
 889                 length += cur.length;
 890             }
 891
 892             // @@@ gvdl: should assert(address);
 893             // but can't as NVidia GeForce creates a bogus physical mem
 894             {
 895                 assert(address || /*nvidia*/(!_ranges.p[0].address && 1 == _rangesCount));
 896             }
 897             assert(length);
 898         }
 899         else do {
 900             // We need wiring & we are wired.
 901             assert(_wireCount);
 902
 903             if (!_wireCount)
 904             {
 905                 panic("IOGMD: not wired for getPhysicalSegment()");
 906                 continue;
 907             }
 908
 909             assert(_memoryEntries);
 910
 911             ioGMDData * dataP = getDataP(_memoryEntries);
 912             const ioPLBlock *ioplList = getIOPLList(dataP);
 913             UInt ind, numIOPLs = getNumIOPL(dataP, _memoryEntries->getLength());
 914             upl_page_info_t *pageList = getPageList(dataP);
 915
 916             assert(numIOPLs > 0);
 917
 918             // Scan through iopl info blocks looking for block containing offset
 919             for (ind = 1; ind < numIOPLs; ind++) {
 920                 if (offset < ioplList[ind].fIOMDOffset)
 921                     break;
 922             }
 923
 924             // Go back to actual range as search goes past it
 925             ioPLBlock ioplInfo = ioplList[ind - 1];
 926
 927             if (ind < numIOPLs)
 928                 length = ioplList[ind].fIOMDOffset;
 929             else
 930                 length = _length;
 931             length -= offset;                   // Remainder within iopl
 932
 933             // Subtract offset till this iopl in total list
 934             offset -= ioplInfo.fIOMDOffset;
 935
 936             // This is a mapped IOPL so we just need to compute an offset
 937             // relative to the mapped base.
 938             if (ioplInfo.fMappedBase) {
 939                 offset += (ioplInfo.fPageOffset & PAGE_MASK);
 940                 address = ptoa_32(ioplInfo.fMappedBase) + offset;
 941                 continue;
 942             }
 943
 944             // Currently the offset is rebased into the current iopl.
 945             // Now add the iopl 1st page offset.
 946             offset += ioplInfo.fPageOffset;
 947
 948             // For external UPLs the fPageInfo field points directly to
 949             // the upl's upl_page_info_t array.
 950             if (ioplInfo.fFlags & kIOPLExternUPL)
 951                 pageList = (upl_page_info_t *) ioplInfo.fPageInfo;
 952             else
 953                 pageList = &pageList[ioplInfo.fPageInfo];
 954
 955             // Check for direct device non-paged memory
 956             if ( ioplInfo.fFlags & kIOPLOnDevice ) {
 957                 address = ptoa_32(pageList->phys_addr) + offset;
 958                 continue;
 959             }
 960
 961             // Now we need compute the index into the pageList
 962             ind = atop_32(offset);
 963             offset &= PAGE_MASK;
 964
 965             IOPhysicalAddress pageAddr = pageList[ind].phys_addr;
 966             address = ptoa_32(pageAddr) + offset;
 967
 968             // Check for the remaining data in this upl being longer than the
 969             // remainder on the current page.  This should be checked for
 970             // contiguous pages
 971             if (length > PAGE_SIZE - offset) {
 972                 // See if the next page is contiguous.  Stop looking when we hit
 973                 // the end of this upl, which is indicated by the
 974                 // contigLength >= length.
 975                 IOByteCount contigLength = PAGE_SIZE - offset;
 976
 977                 // Look for contiguous segment
 978                 while (contigLength < length
 979                 &&     ++pageAddr == pageList[++ind].phys_addr) {
 980                     contigLength += PAGE_SIZE;
 981                 }
 982                 if (length > contigLength)
 983                     length = contigLength;
 984             }
 985
 986             assert(address);
 987             assert(length);
 988
 989         } while (0);
 990
 991         if (!address)
 992             length = 0;
 993     }
 994
 995     if (lengthOfSegment)
 996         *lengthOfSegment = length;
 997
 998     return address;
 999 }
1000
1001 addr64_t IOMemoryDescriptor::getPhysicalSegment64
1002                         (IOByteCount offset, IOByteCount *lengthOfSegment)
1003 {
1004     IOPhysicalAddress phys32;
1005     IOByteCount       length;
1006     addr64_t          phys64;
1007
1008     phys32 = getPhysicalSegment(offset, lengthOfSegment);
1009     if (!phys32)
1010         return 0;
1011
1012     if (gIOSystemMapper)
1013     {
1014         IOByteCount origLen;
1015
1016         phys64 = gIOSystemMapper->mapAddr(phys32);
1017         origLen = *lengthOfSegment;
1018         length = page_size - (phys64 & (page_size - 1));
1019         while ((length < origLen)
1020             && ((phys64 + length) == gIOSystemMapper->mapAddr(phys32 + length)))
1021             length += page_size;
1022         if (length > origLen)
1023             length = origLen;
1024
1025         *lengthOfSegment = length;
1026     }
1027     else
1028         phys64 = (addr64_t) phys32;
1029
1030     return phys64;
1031 }
1032
1033 // Note this function is NOT a virtual function
1034 void * IOGeneralMemoryDescriptor::getBackingID() const
1035 {
1036     if (!_memEntry) // Not created as a persistent memory descriptor
1037         return 0;
1038
1039     vm_size_t size = _pages << PAGE_SHIFT;
1040     vm_size_t seenSize = 0;
1041     vm_address_t basePage = trunc_page_32(_ranges.v[0].address);
1042     void *retObjID = 0;
1043
1044     vm_map_t theMap =
1045         ((_task == kernel_task) && (kIOMemoryBufferPageable & _flags))
1046             ? IOPageableMapForAddress(basePage)
1047             : get_task_map(_task);
1048
1049
1050     for (;;) {
1051         vm_region_object_info_data_64_t objInfo;
1052         vm_address_t actualPage = basePage;
1053         vm_size_t actualSize;
1054         mach_msg_type_number_t objInfoSize;
1055         kern_return_t error;
1056
1057         objInfoSize = VM_REGION_OBJECT_INFO_COUNT_64;
1058         error = vm_region_64(theMap,
1059                              &actualPage,
1060                              &actualSize,
1061                              VM_REGION_OBJECT_INFO_64,
1062                              (vm_region_info_t) &objInfo,
1063                              &objInfoSize,
1064                              0);
1065
1066         if (KERN_SUCCESS != error || actualSize == 0 || actualPage > basePage
1067         || (retObjID && retObjID != (void *) objInfo.object_id))
1068             return 0;
1069
1070         actualPage += actualSize;               // Calculate the end address
1071         seenSize += actualPage - basePage;      // Size of overlap
1072         basePage = actualPage;                  // Start here for next loop
1073         if (seenSize >= size)
1074             return (void *) objInfo.object_id;
1075
1076         if (!retObjID)
1077             retObjID = (void *) objInfo.object_id;
1078     }
1079 }
1080
1081
1082 IOPhysicalAddress IOGeneralMemoryDescriptor::getSourceSegment
1083                             (IOByteCount offset, IOByteCount *lengthOfSegment)
1084 {
1085     IOPhysicalAddress address = 0;
1086     IOPhysicalLength  length  = 0;
1087
1088     assert(offset <= _length);
1089
1090     if ( (_flags & kIOMemoryTypeMask) == kIOMemoryTypeUPL)
1091         return super::getSourceSegment( offset, lengthOfSegment );
1092
1093     if ( offset < _length ) // (within bounds?)
1094     {
1095         unsigned rangesIndex = 0;
1096
1097         for ( ; offset >= _ranges.v[rangesIndex].length; rangesIndex++ )
1098         {
1099             offset -= _ranges.v[rangesIndex].length; // (make offset relative)
1100         }
1101
1102         address = _ranges.v[rangesIndex].address + offset;
1103         length  = _ranges.v[rangesIndex].length  - offset;
1104
1105         for ( ++rangesIndex; rangesIndex < _rangesCount; rangesIndex++ )
1106         {
1107             if ( address + length != _ranges.v[rangesIndex].address )  break;
1108
1109             length += _ranges.v[rangesIndex].length; // (coalesce ranges)
1110         }
1111
1112         assert(address);
1113         if ( address == 0 )  length = 0;
1114     }
1115
1116     if ( lengthOfSegment )  *lengthOfSegment = length;
1117
1118     return address;
1119 }
1120
1121 /* DEPRECATED */ /* USE INSTEAD: map(), readBytes(), writeBytes() */
1122 /* DEPRECATED */ void * IOGeneralMemoryDescriptor::getVirtualSegment(IOByteCount offset,
1123 /* DEPRECATED */                                                        IOByteCount * lengthOfSegment)
1124 /* DEPRECATED */ {
1125                     if (_task == kernel_task)
1126                         return (void *) getSourceSegment(offset, lengthOfSegment);
1127                     else
1128                         panic("IOGMD::getVirtualSegment deprecated");
1129
1130                     return 0;
1131 /* DEPRECATED */ }
1132 /* DEPRECATED */ /* USE INSTEAD: map(), readBytes(), writeBytes() */
1133
1134 #ifdef __ppc__
1135 extern vm_offset_t              static_memory_end;
1136 #define io_kernel_static_end    static_memory_end
1137 #else
1138 extern vm_offset_t              first_avail;
1139 #define io_kernel_static_end    first_avail
1140 #endif
1141
1142 static kern_return_t
1143 io_get_kernel_static_upl(
1144         vm_map_t                map,
1145         vm_address_t            offset,
1146         vm_size_t               *upl_size,
1147         upl_t                   *upl,
1148         upl_page_info_array_t   page_list,
1149         unsigned int            *count,
1150         int                     *flags,
1151         int                     force_data_sync)
1152 {
1153     unsigned int pageCount, page;
1154     ppnum_t phys;
1155
1156     pageCount = atop_32(*upl_size);
1157     if (pageCount > *count)
1158         pageCount = *count;
1159
1160     *upl = NULL;
1161
1162     for (page = 0; page < pageCount; page++)
1163     {
1164         phys = pmap_find_phys(kernel_pmap, ((addr64_t)offset) + ptoa_64(page));
1165         if (!phys)
1166             break;
1167         page_list[page].phys_addr = phys;
1168         page_list[page].pageout   = 0;
1169         page_list[page].absent    = 0;
1170         page_list[page].dirty     = 0;
1171         page_list[page].precious  = 0;
1172         page_list[page].device    = 0;
1173     }
1174
1175     return ((page >= pageCount) ? kIOReturnSuccess : kIOReturnVMError);
1176 }
1177
1178 IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection)
1179 {
1180     IOReturn error = kIOReturnNoMemory;
1181     ioGMDData *dataP;
1182     ppnum_t mapBase = 0;
1183     IOMapper *mapper;
1184     ipc_port_t sharedMem = (ipc_port_t) _memEntry;
1185
1186     assert(!_wireCount);
1187
1188     if (_pages >= gIOMaximumMappedIOPageCount)
1189         return kIOReturnNoResources;
1190
1191     dataP = getDataP(_memoryEntries);
1192     mapper = dataP->fMapper;
1193     if (mapper && _pages)
1194         mapBase = mapper->iovmAlloc(_pages);
1195
1196     // Note that appendBytes(NULL) zeros the data up to the
1197     // desired length.
1198     _memoryEntries->appendBytes(0, dataP->fPageCnt * sizeof(upl_page_info_t));
1199     dataP = 0;  // May no longer be valid so lets not get tempted.
1200
1201     if (forDirection == kIODirectionNone)
1202         forDirection = _direction;
1203
1204     int uplFlags;    // This Mem Desc's default flags for upl creation
1205     switch (forDirection)
1206     {
1207     case kIODirectionOut:
1208         // Pages do not need to be marked as dirty on commit
1209         uplFlags = UPL_COPYOUT_FROM;
1210         _flags |= kIOMemoryPreparedReadOnly;
1211         break;
1212
1213     case kIODirectionIn:
1214     default:
1215         uplFlags = 0;   // i.e. ~UPL_COPYOUT_FROM
1216         break;
1217     }
1218     uplFlags |= UPL_SET_IO_WIRE | UPL_SET_LITE;
1219
1220     //
1221     // Check user read/write access to the data buffer.
1222     //
1223     unsigned int pageIndex = 0;
1224     IOByteCount mdOffset = 0;
1225     vm_map_t curMap;
1226     if (_task == kernel_task && (kIOMemoryBufferPageable & _flags))
1227         curMap = 0;
1228     else
1229         { curMap = get_task_map(_task); }
1230
1231     for (UInt range = 0; range < _rangesCount; range++) {
1232         ioPLBlock iopl;
1233         IOVirtualRange curRange = _ranges.v[range];
1234         vm_address_t startPage;
1235         IOByteCount numBytes;
1236
1237         startPage = trunc_page_32(curRange.address);
1238         iopl.fPageOffset = (short) curRange.address & PAGE_MASK;
1239         if (mapper)
1240             iopl.fMappedBase = mapBase + pageIndex;
1241         else
1242             iopl.fMappedBase = 0;
1243         numBytes = iopl.fPageOffset + curRange.length;
1244
1245         while (numBytes) {
1246             dataP = getDataP(_memoryEntries);
1247             vm_map_t theMap =
1248                 (curMap)? curMap
1249                         : IOPageableMapForAddress(startPage);
1250             upl_page_info_array_t pageInfo = getPageList(dataP);
1251             int ioplFlags = uplFlags;
1252             upl_page_list_ptr_t baseInfo = &pageInfo[pageIndex];
1253
1254             vm_size_t ioplSize = round_page_32(numBytes);
1255             unsigned int numPageInfo = atop_32(ioplSize);
1256
1257             if ((theMap == kernel_map) && (startPage < io_kernel_static_end))
1258             {
1259                 error = io_get_kernel_static_upl(theMap,
1260                                     startPage,
1261                                     &ioplSize,
1262                                     &iopl.fIOPL,
1263                                     baseInfo,
1264                                     &numPageInfo,
1265                                     &ioplFlags,
1266                                     false);
1267
1268             } else if (sharedMem && (kIOMemoryPersistent & _flags)) {
1269
1270                 error = memory_object_iopl_request(sharedMem,
1271                                     ptoa_32(pageIndex),
1272                                     &ioplSize,
1273                                     &iopl.fIOPL,
1274                                     baseInfo,
1275                                     &numPageInfo,
1276                                     &ioplFlags);
1277
1278             } else {
1279                 error = vm_map_get_upl(theMap,
1280                                     startPage,
1281                                     &ioplSize,
1282                                     &iopl.fIOPL,
1283                                     baseInfo,
1284                                     &numPageInfo,
1285                                     &ioplFlags,
1286                                     false);
1287             }
1288
1289             assert(ioplSize);
1290             if (error != KERN_SUCCESS)
1291                 goto abortExit;
1292
1293             error = kIOReturnNoMemory;
1294
1295             if (baseInfo->device) {
1296                 numPageInfo = 1;
1297                 iopl.fFlags  = kIOPLOnDevice;
1298                 // Don't translate device memory at all
1299                 if (mapper && mapBase) {
1300                     mapper->iovmFree(mapBase, _pages);
1301                     mapBase = 0;
1302                     iopl.fMappedBase = 0;
1303                 }
1304             }
1305             else {
1306                 iopl.fFlags = 0;
1307                 if (mapper)
1308                     mapper->iovmInsert(mapBase, pageIndex,
1309                                        baseInfo, numPageInfo);
1310             }
1311
1312             iopl.fIOMDOffset = mdOffset;
1313             iopl.fPageInfo = pageIndex;
1314
1315             if ((_flags & kIOMemoryAutoPrepare) && iopl.fIOPL)
1316             {
1317                 kernel_upl_commit(iopl.fIOPL, 0, 0);
1318                 iopl.fIOPL = 0;
1319             }
1320
1321             if (!_memoryEntries->appendBytes(&iopl, sizeof(iopl))) {
1322                 // Clean up partial created and unsaved iopl
1323                 if (iopl.fIOPL)
1324                     kernel_upl_abort(iopl.fIOPL, 0);
1325                 goto abortExit;
1326             }
1327
1328             // Check for a multiple iopl's in one virtual range
1329             pageIndex += numPageInfo;
1330             mdOffset -= iopl.fPageOffset;
1331             if (ioplSize < numBytes) {
1332                 numBytes -= ioplSize;
1333                 startPage += ioplSize;
1334                 mdOffset += ioplSize;
1335                 iopl.fPageOffset = 0;
1336                 if (mapper)
1337                     iopl.fMappedBase = mapBase + pageIndex;
1338             }
1339             else {
1340                 mdOffset += numBytes;
1341                 break;
1342             }
1343         }
1344     }
1345
1346     return kIOReturnSuccess;
1347
1348 abortExit:
1349     {
1350         dataP = getDataP(_memoryEntries);
1351         UInt done = getNumIOPL(dataP, _memoryEntries->getLength());
1352         ioPLBlock *ioplList = getIOPLList(dataP);
1353
1354         for (UInt range = 0; range < done; range++)
1355         {
1356             if (ioplList[range].fIOPL)
1357                 kernel_upl_abort(ioplList[range].fIOPL, 0);
1358         }
1359
1360         if (mapper && mapBase)
1361             mapper->iovmFree(mapBase, _pages);
1362     }
1363
1364     return error;
1365 }
1366
1367 /*
1368  * prepare
1369  *
1370  * Prepare the memory for an I/O transfer.  This involves paging in
1371  * the memory, if necessary, and wiring it down for the duration of
1372  * the transfer.  The complete() method completes the processing of
1373  * the memory after the I/O transfer finishes.  This method needn't
1374  * called for non-pageable memory.
1375  */
1376 IOReturn IOGeneralMemoryDescriptor::prepare(IODirection forDirection)
1377 {
1378     IOReturn error = kIOReturnSuccess;
1379
1380     if (!_wireCount && (_flags & kIOMemoryTypeMask) == kIOMemoryTypeVirtual) {
1381         error = wireVirtual(forDirection);
1382         if (error)
1383             return error;
1384     }
1385
1386     _wireCount++;
1387
1388     return kIOReturnSuccess;
1389 }
1390
1391 /*
1392  * complete
1393  *
1394  * Complete processing of the memory after an I/O transfer finishes.
1395  * This method should not be called unless a prepare was previously
1396  * issued; the prepare() and complete() must occur in pairs, before
1397  * before and after an I/O transfer involving pageable memory.
1398  */
1399
1400 IOReturn IOGeneralMemoryDescriptor::complete(IODirection /* forDirection */)
1401 {
1402     assert(_wireCount);
1403
1404     if (!_wireCount)
1405         return kIOReturnSuccess;
1406
1407     _wireCount--;
1408     if (!_wireCount) {
1409         if ((_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical) {
1410             /* kIOMemoryTypePhysical */
1411             // DO NOTHING
1412         }
1413         else {
1414             ioGMDData * dataP = getDataP(_memoryEntries);
1415             ioPLBlock *ioplList = getIOPLList(dataP);
1416             UInt count = getNumIOPL(dataP, _memoryEntries->getLength());
1417
1418             if (dataP->fMapper && _pages && ioplList[0].fMappedBase)
1419                 dataP->fMapper->iovmFree(ioplList[0].fMappedBase, _pages);
1420
1421             // Only complete iopls that we created which are for TypeVirtual
1422             if ( (_flags & kIOMemoryTypeMask) == kIOMemoryTypeVirtual) {
1423                 for (UInt ind = 0; ind < count; ind++)
1424                     if (ioplList[ind].fIOPL)
1425                         kernel_upl_commit(ioplList[ind].fIOPL, 0, 0);
1426             }
1427
1428             (void) _memoryEntries->initWithBytes(dataP, sizeof(ioGMDData)); // == setLength()
1429         }
1430     }
1431     return kIOReturnSuccess;
1432 }
1433
1434 IOReturn IOGeneralMemoryDescriptor::doMap(
1435         vm_map_t                addressMap,
1436         IOVirtualAddress *      atAddress,
1437         IOOptionBits            options,
1438         IOByteCount             sourceOffset,
1439         IOByteCount             length )
1440 {
1441     kern_return_t kr;
1442     ipc_port_t sharedMem = (ipc_port_t) _memEntry;
1443
1444     // mapping source == dest? (could be much better)
1445     if( _task && (addressMap == get_task_map(_task)) && (options & kIOMapAnywhere)
1446         && (1 == _rangesCount) && (0 == sourceOffset)
1447         && (length <= _ranges.v[0].length) ) {
1448             *atAddress = _ranges.v[0].address;
1449             return( kIOReturnSuccess );
1450     }
1451
1452     if( 0 == sharedMem) {
1453
1454         vm_size_t size = _pages << PAGE_SHIFT;
1455
1456         if( _task) {
1457 #ifndef i386
1458             vm_size_t actualSize = size;
1459             kr = mach_make_memory_entry( get_task_map(_task),
1460                         &actualSize, _ranges.v[0].address,
1461                         VM_PROT_READ | VM_PROT_WRITE, &sharedMem,
1462                         NULL );
1463
1464             if( (KERN_SUCCESS == kr) && (actualSize != round_page_32(size))) {
1465 #if IOASSERT
1466                 IOLog("mach_make_memory_entry_64 (%08x) size (%08lx:%08x)\n",
1467                             _ranges.v[0].address, (UInt32)actualSize, size);
1468 #endif
1469                 kr = kIOReturnVMError;
1470                 ipc_port_release_send( sharedMem );
1471             }
1472
1473             if( KERN_SUCCESS != kr)
1474 #endif /* i386 */
1475                 sharedMem = MACH_PORT_NULL;
1476
1477         } else do {
1478
1479             memory_object_t     pager;
1480             unsigned int        flags = 0;
1481             addr64_t            pa;
1482             IOPhysicalLength    segLen;
1483
1484             pa = getPhysicalSegment64( sourceOffset, &segLen );
1485
1486             if( !reserved) {
1487                 reserved = IONew( ExpansionData, 1 );
1488                 if( !reserved)
1489                     continue;
1490             }
1491             reserved->pagerContig = (1 == _rangesCount);
1492             reserved->memory = this;
1493
1494             /*What cache mode do we need*/
1495             switch(options & kIOMapCacheMask ) {
1496
1497                 case kIOMapDefaultCache:
1498                 default:
1499                     flags = IODefaultCacheBits(pa);
1500                     break;
1501
1502                 case kIOMapInhibitCache:
1503                     flags = DEVICE_PAGER_CACHE_INHIB |
1504                                     DEVICE_PAGER_COHERENT | DEVICE_PAGER_GUARDED;
1505                     break;
1506
1507                 case kIOMapWriteThruCache:
1508                     flags = DEVICE_PAGER_WRITE_THROUGH |
1509                                     DEVICE_PAGER_COHERENT | DEVICE_PAGER_GUARDED;
1510                     break;
1511
1512                 case kIOMapCopybackCache:
1513                     flags = DEVICE_PAGER_COHERENT;
1514                     break;
1515
1516                 case kIOMapWriteCombineCache:
1517                     flags = DEVICE_PAGER_CACHE_INHIB |
1518                                     DEVICE_PAGER_COHERENT;
1519                     break;
1520             }
1521
1522             flags |= reserved->pagerContig ? DEVICE_PAGER_CONTIGUOUS : 0;
1523
1524             pager = device_pager_setup( (memory_object_t) 0, (int) reserved,
1525                                                                 size, flags);
1526             assert( pager );
1527
1528             if( pager) {
1529                 kr = mach_memory_object_memory_entry_64( (host_t) 1, false /*internal*/,
1530                             size, VM_PROT_READ | VM_PROT_WRITE, pager, &sharedMem );
1531
1532                 assert( KERN_SUCCESS == kr );
1533                 if( KERN_SUCCESS != kr) {
1534                     device_pager_deallocate( pager );
1535                     pager = MACH_PORT_NULL;
1536                     sharedMem = MACH_PORT_NULL;
1537                 }
1538             }
1539             if( pager && sharedMem)
1540                 reserved->devicePager    = pager;
1541             else {
1542                 IODelete( reserved, ExpansionData, 1 );
1543                 reserved = 0;
1544             }
1545
1546         } while( false );
1547
1548         _memEntry = (void *) sharedMem;
1549     }
1550
1551 #ifndef i386
1552     if( 0 == sharedMem)
1553       kr = kIOReturnVMError;
1554     else
1555 #endif
1556       kr = super::doMap( addressMap, atAddress,
1557                            options, sourceOffset, length );
1558
1559     return( kr );
1560 }
1561
1562 IOReturn IOGeneralMemoryDescriptor::doUnmap(
1563         vm_map_t                addressMap,
1564         IOVirtualAddress        logical,
1565         IOByteCount             length )
1566 {
1567     // could be much better
1568     if( _task && (addressMap == get_task_map(_task)) && (1 == _rangesCount)
1569          && (logical == _ranges.v[0].address)
1570          && (length <= _ranges.v[0].length) )
1571             return( kIOReturnSuccess );
1572
1573     return( super::doUnmap( addressMap, logical, length ));
1574 }
1575
1576 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1577
1578 extern "C" {
1579 // osfmk/device/iokit_rpc.c
1580 extern kern_return_t IOMapPages( vm_map_t map, vm_offset_t va, vm_offset_t pa,
1581                                  vm_size_t length, unsigned int mapFlags);
1582 extern kern_return_t IOUnmapPages(vm_map_t map, vm_offset_t va, vm_size_t length);
1583 };
1584
1585 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1586
1587 OSDefineMetaClassAndAbstractStructors( IOMemoryMap, OSObject )
1588
1589 /* inline function implementation */
1590 IOPhysicalAddress IOMemoryMap::getPhysicalAddress()
1591     { return( getPhysicalSegment( 0, 0 )); }
1592
1593 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1594
1595 class _IOMemoryMap : public IOMemoryMap
1596 {
1597     OSDeclareDefaultStructors(_IOMemoryMap)
1598
1599     IOMemoryDescriptor * memory;
1600     IOMemoryMap *       superMap;
1601     IOByteCount         offset;
1602     IOByteCount         length;
1603     IOVirtualAddress    logical;
1604     task_t              addressTask;
1605     vm_map_t            addressMap;
1606     IOOptionBits        options;
1607
1608 protected:
1609     virtual void taggedRelease(const void *tag = 0) const;
1610     virtual void free();
1611
1612 public:
1613
1614     // IOMemoryMap methods
1615     virtual IOVirtualAddress    getVirtualAddress();
1616     virtual IOByteCount         getLength();
1617     virtual task_t              getAddressTask();
1618     virtual IOMemoryDescriptor * getMemoryDescriptor();
1619     virtual IOOptionBits        getMapOptions();
1620
1621     virtual IOReturn            unmap();
1622     virtual void                taskDied();
1623
1624     virtual IOPhysicalAddress   getPhysicalSegment(IOByteCount offset,
1625                                                    IOByteCount * length);
1626
1627     // for IOMemoryDescriptor use
1628     _IOMemoryMap * copyCompatible(
1629                 IOMemoryDescriptor *    owner,
1630                 task_t                  intoTask,
1631                 IOVirtualAddress        toAddress,
1632                 IOOptionBits            options,
1633                 IOByteCount             offset,
1634                 IOByteCount             length );
1635
1636     bool initCompatible(
1637         IOMemoryDescriptor *    memory,
1638         IOMemoryMap *           superMap,
1639         IOByteCount             offset,
1640         IOByteCount             length );
1641
1642     bool initWithDescriptor(
1643         IOMemoryDescriptor *    memory,
1644         task_t                  intoTask,
1645         IOVirtualAddress        toAddress,
1646         IOOptionBits            options,
1647         IOByteCount             offset,
1648         IOByteCount             length );
1649
1650     IOReturn redirect(
1651         task_t                  intoTask, bool redirect );
1652 };
1653
1654 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1655
1656 #undef super
1657 #define super IOMemoryMap
1658
1659 OSDefineMetaClassAndStructors(_IOMemoryMap, IOMemoryMap)
1660
1661 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1662
1663 bool _IOMemoryMap::initCompatible(
1664         IOMemoryDescriptor *    _memory,
1665         IOMemoryMap *           _superMap,
1666         IOByteCount             _offset,
1667         IOByteCount             _length )
1668 {
1669
1670     if( !super::init())
1671         return( false);
1672
1673     if( (_offset + _length) > _superMap->getLength())
1674         return( false);
1675
1676     _memory->retain();
1677     memory      = _memory;
1678     _superMap->retain();
1679     superMap    = _superMap;
1680
1681     offset      = _offset;
1682     if( _length)
1683         length  = _length;
1684     else
1685         length  = _memory->getLength();
1686
1687     options     = superMap->getMapOptions();
1688     logical     = superMap->getVirtualAddress() + offset;
1689
1690     return( true );
1691 }
1692
1693 bool _IOMemoryMap::initWithDescriptor(
1694         IOMemoryDescriptor *    _memory,
1695         task_t                  intoTask,
1696         IOVirtualAddress        toAddress,
1697         IOOptionBits            _options,
1698         IOByteCount             _offset,
1699         IOByteCount             _length )
1700 {
1701     bool        ok;
1702
1703     if( (!_memory) || (!intoTask) || !super::init())
1704         return( false);
1705
1706     if( (_offset + _length) > _memory->getLength())
1707         return( false);
1708
1709     addressMap  = get_task_map(intoTask);
1710     if( !addressMap)
1711         return( false);
1712     vm_map_reference(addressMap);
1713
1714     _memory->retain();
1715     memory      = _memory;
1716
1717     offset      = _offset;
1718     if( _length)
1719         length  = _length;
1720     else
1721         length  = _memory->getLength();
1722
1723     addressTask = intoTask;
1724     logical     = toAddress;
1725     options     = _options;
1726
1727     if( options & kIOMapStatic)
1728         ok = true;
1729     else
1730         ok = (kIOReturnSuccess == memory->doMap( addressMap, &logical,
1731                                                  options, offset, length ));
1732     if( !ok) {
1733         logical = 0;
1734         memory->release();
1735         memory = 0;
1736         vm_map_deallocate(addressMap);
1737         addressMap = 0;
1738     }
1739     return( ok );
1740 }
1741
1742 struct IOMemoryDescriptorMapAllocRef
1743 {
1744     ipc_port_t          sharedMem;
1745     vm_size_t           size;
1746     vm_offset_t         mapped;
1747     IOByteCount         sourceOffset;
1748     IOOptionBits        options;
1749 };
1750
1751 static kern_return_t IOMemoryDescriptorMapAlloc(vm_map_t map, void * _ref)
1752 {
1753     IOMemoryDescriptorMapAllocRef * ref = (IOMemoryDescriptorMapAllocRef *)_ref;
1754     IOReturn                        err;
1755
1756     do {
1757         if( ref->sharedMem) {
1758             vm_prot_t prot = VM_PROT_READ
1759                             | ((ref->options & kIOMapReadOnly) ? 0 : VM_PROT_WRITE);
1760
1761             // set memory entry cache
1762             vm_prot_t memEntryCacheMode = prot | MAP_MEM_ONLY;
1763             switch (ref->options & kIOMapCacheMask)
1764             {
1765                 case kIOMapInhibitCache:
1766                     SET_MAP_MEM(MAP_MEM_IO, memEntryCacheMode);
1767                     break;
1768
1769                 case kIOMapWriteThruCache:
1770                     SET_MAP_MEM(MAP_MEM_WTHRU, memEntryCacheMode);
1771                     break;
1772
1773                 case kIOMapWriteCombineCache:
1774                     SET_MAP_MEM(MAP_MEM_WCOMB, memEntryCacheMode);
1775                     break;
1776
1777                 case kIOMapCopybackCache:
1778                     SET_MAP_MEM(MAP_MEM_COPYBACK, memEntryCacheMode);
1779                     break;
1780
1781                 case kIOMapDefaultCache:
1782                 default:
1783                     SET_MAP_MEM(MAP_MEM_NOOP, memEntryCacheMode);
1784                     break;
1785             }
1786
1787             vm_size_t unused = 0;
1788
1789             err = mach_make_memory_entry( NULL /*unused*/, &unused, 0 /*unused*/,
1790                                             memEntryCacheMode, NULL, ref->sharedMem );
1791             if (KERN_SUCCESS != err)
1792                 IOLog("MAP_MEM_ONLY failed %d\n", err);
1793
1794             err = vm_map( map,
1795                             &ref->mapped,
1796                             ref->size, 0 /* mask */,
1797                             (( ref->options & kIOMapAnywhere ) ? VM_FLAGS_ANYWHERE : VM_FLAGS_FIXED)
1798                             | VM_MAKE_TAG(VM_MEMORY_IOKIT),
1799                             ref->sharedMem, ref->sourceOffset,
1800                             false, // copy
1801                             prot, // cur
1802                             prot, // max
1803                             VM_INHERIT_NONE);
1804
1805             if( KERN_SUCCESS != err) {
1806                 ref->mapped = 0;
1807                 continue;
1808             }
1809
1810         } else {
1811
1812             err = vm_allocate( map, &ref->mapped, ref->size,
1813                             ((ref->options & kIOMapAnywhere) ? VM_FLAGS_ANYWHERE : VM_FLAGS_FIXED)
1814                             | VM_MAKE_TAG(VM_MEMORY_IOKIT) );
1815
1816             if( KERN_SUCCESS != err) {
1817                 ref->mapped = 0;
1818                 continue;
1819             }
1820
1821             // we have to make sure that these guys don't get copied if we fork.
1822             err = vm_inherit( map, ref->mapped, ref->size, VM_INHERIT_NONE);
1823             assert( KERN_SUCCESS == err );
1824         }
1825
1826     } while( false );
1827
1828     return( err );
1829 }
1830
1831
1832 IOReturn IOMemoryDescriptor::doMap(
1833         vm_map_t                addressMap,
1834         IOVirtualAddress *      atAddress,
1835         IOOptionBits            options,
1836         IOByteCount             sourceOffset,
1837         IOByteCount             length )
1838 {
1839     IOReturn            err = kIOReturnSuccess;
1840     memory_object_t     pager;
1841     vm_address_t        logical;
1842     IOByteCount         pageOffset;
1843     IOPhysicalAddress   sourceAddr;
1844     IOMemoryDescriptorMapAllocRef       ref;
1845
1846     ref.sharedMem       = (ipc_port_t) _memEntry;
1847     ref.sourceOffset    = sourceOffset;
1848     ref.options         = options;
1849
1850     do {
1851
1852         if( 0 == length)
1853             length = getLength();
1854
1855         sourceAddr = getSourceSegment( sourceOffset, NULL );
1856         assert( sourceAddr );
1857         pageOffset = sourceAddr - trunc_page_32( sourceAddr );
1858
1859         ref.size = round_page_32( length + pageOffset );
1860
1861         logical = *atAddress;
1862         if( options & kIOMapAnywhere)
1863             // vm_map looks for addresses above here, even when VM_FLAGS_ANYWHERE
1864             ref.mapped = 0;
1865         else {
1866             ref.mapped = trunc_page_32( logical );
1867             if( (logical - ref.mapped) != pageOffset) {
1868                 err = kIOReturnVMError;
1869                 continue;
1870             }
1871         }
1872
1873         if( ref.sharedMem && (addressMap == kernel_map) && (kIOMemoryBufferPageable & _flags))
1874             err = IOIteratePageableMaps( ref.size, &IOMemoryDescriptorMapAlloc, &ref );
1875         else
1876             err = IOMemoryDescriptorMapAlloc( addressMap, &ref );
1877
1878         if( err != KERN_SUCCESS)
1879             continue;
1880
1881         if( reserved)
1882             pager = (memory_object_t) reserved->devicePager;
1883         else
1884             pager = MACH_PORT_NULL;
1885
1886         if( !ref.sharedMem || pager )
1887             err = handleFault( pager, addressMap, ref.mapped, sourceOffset, length, options );
1888
1889     } while( false );
1890
1891     if( err != KERN_SUCCESS) {
1892         if( ref.mapped)
1893             doUnmap( addressMap, ref.mapped, ref.size );
1894         *atAddress = NULL;
1895     } else
1896         *atAddress = ref.mapped + pageOffset;
1897
1898     return( err );
1899 }
1900
1901 enum {
1902     kIOMemoryRedirected = 0x00010000
1903 };
1904
1905 IOReturn IOMemoryDescriptor::handleFault(
1906         void *                  _pager,
1907         vm_map_t                addressMap,
1908         IOVirtualAddress        address,
1909         IOByteCount             sourceOffset,
1910         IOByteCount             length,
1911         IOOptionBits            options )
1912 {
1913     IOReturn            err = kIOReturnSuccess;
1914     memory_object_t     pager = (memory_object_t) _pager;
1915     vm_size_t           size;
1916     vm_size_t           bytes;
1917     vm_size_t           page;
1918     IOByteCount         pageOffset;
1919     IOByteCount         pagerOffset;
1920     IOPhysicalLength    segLen;
1921     addr64_t            physAddr;
1922
1923     if( !addressMap) {
1924
1925         if( kIOMemoryRedirected & _flags) {
1926 #ifdef DEBUG
1927             IOLog("sleep mem redirect %p, %lx\n", this, sourceOffset);
1928 #endif
1929             do {
1930                 SLEEP;
1931             } while( kIOMemoryRedirected & _flags );
1932         }
1933
1934         return( kIOReturnSuccess );
1935     }
1936
1937     physAddr = getPhysicalSegment64( sourceOffset, &segLen );
1938     assert( physAddr );
1939     pageOffset = physAddr - trunc_page_64( physAddr );
1940     pagerOffset = sourceOffset;
1941
1942     size = length + pageOffset;
1943     physAddr -= pageOffset;
1944
1945     segLen += pageOffset;
1946     bytes = size;
1947     do {
1948         // in the middle of the loop only map whole pages
1949         if( segLen >= bytes)
1950             segLen = bytes;
1951         else if( segLen != trunc_page_32( segLen))
1952             err = kIOReturnVMError;
1953         if( physAddr != trunc_page_64( physAddr))
1954             err = kIOReturnBadArgument;
1955
1956 #ifdef DEBUG
1957         if( kIOLogMapping & gIOKitDebug)
1958             IOLog("_IOMemoryMap::map(%p) %08lx->%08qx:%08lx\n",
1959                 addressMap, address + pageOffset, physAddr + pageOffset,
1960                 segLen - pageOffset);
1961 #endif
1962
1963
1964
1965
1966
1967 #ifdef i386
1968         /* i386 doesn't support faulting on device memory yet */
1969         if( addressMap && (kIOReturnSuccess == err))
1970             err = IOMapPages( addressMap, address, (IOPhysicalAddress) physAddr, segLen, options );
1971         assert( KERN_SUCCESS == err );
1972         if( err)
1973             break;
1974 #endif
1975
1976         if( pager) {
1977             if( reserved && reserved->pagerContig) {
1978                 IOPhysicalLength        allLen;
1979                 addr64_t                allPhys;
1980
1981                 allPhys = getPhysicalSegment64( 0, &allLen );
1982                 assert( allPhys );
1983                 err = device_pager_populate_object( pager, 0, allPhys >> PAGE_SHIFT, round_page_32(allLen) );
1984
1985             } else {
1986
1987             for( page = 0;
1988                      (page < segLen) && (KERN_SUCCESS == err);
1989                      page += page_size) {
1990                         err = device_pager_populate_object(pager, pagerOffset,
1991                                 (ppnum_t)((physAddr + page) >> PAGE_SHIFT), page_size);
1992                         pagerOffset += page_size;
1993                 }
1994             }
1995             assert( KERN_SUCCESS == err );
1996             if( err)
1997                 break;
1998         }
1999 #ifndef i386
2000         /*  *** ALERT *** */
2001         /*  *** Temporary Workaround *** */
2002
2003         /* This call to vm_fault causes an early pmap level resolution  */
2004         /* of the mappings created above.  Need for this is in absolute */
2005         /* violation of the basic tenet that the pmap layer is a cache. */
2006         /* Further, it implies a serious I/O architectural violation on */
2007         /* the part of some user of the mapping.  As of this writing,   */
2008         /* the call to vm_fault is needed because the NVIDIA driver     */
2009         /* makes a call to pmap_extract.  The NVIDIA driver needs to be */
2010         /* fixed as soon as possible.  The NVIDIA driver should not     */
2011         /* need to query for this info as it should know from the doMap */
2012         /* call where the physical memory is mapped.  When a query is   */
2013         /* necessary to find a physical mapping, it should be done      */
2014         /* through an iokit call which includes the mapped memory       */
2015         /* handle.  This is required for machine architecture independence.*/
2016
2017         if(!(kIOMemoryRedirected & _flags)) {
2018                 vm_fault(addressMap, address, 3, FALSE, FALSE, NULL, 0);
2019         }
2020
2021         /*  *** Temporary Workaround *** */
2022         /*  *** ALERT *** */
2023 #endif
2024         sourceOffset += segLen - pageOffset;
2025         address += segLen;
2026         bytes -= segLen;
2027         pageOffset = 0;
2028
2029     } while( bytes
2030         && (physAddr = getPhysicalSegment64( sourceOffset, &segLen )));
2031
2032     if( bytes)
2033         err = kIOReturnBadArgument;
2034
2035     return( err );
2036 }
2037
2038 IOReturn IOMemoryDescriptor::doUnmap(
2039         vm_map_t                addressMap,
2040         IOVirtualAddress        logical,
2041         IOByteCount             length )
2042 {
2043     IOReturn    err;
2044
2045 #ifdef DEBUG
2046     if( kIOLogMapping & gIOKitDebug)
2047         kprintf("IOMemoryDescriptor::doUnmap(%x) %08x:%08x\n",
2048                 addressMap, logical, length );
2049 #endif
2050
2051     if( true /* && (addressMap == kernel_map) || (addressMap == get_task_map(current_task()))*/) {
2052
2053         if( _memEntry && (addressMap == kernel_map) && (kIOMemoryBufferPageable & _flags))
2054             addressMap = IOPageableMapForAddress( logical );
2055
2056         err = vm_deallocate( addressMap, logical, length );
2057
2058     } else
2059         err = kIOReturnSuccess;
2060
2061     return( err );
2062 }
2063
2064 IOReturn IOMemoryDescriptor::redirect( task_t safeTask, bool redirect )
2065 {
2066     IOReturn            err;
2067     _IOMemoryMap *      mapping = 0;
2068     OSIterator *        iter;
2069
2070     LOCK;
2071
2072     do {
2073         if( (iter = OSCollectionIterator::withCollection( _mappings))) {
2074             while( (mapping = (_IOMemoryMap *) iter->getNextObject()))
2075                 mapping->redirect( safeTask, redirect );
2076
2077             iter->release();
2078         }
2079     } while( false );
2080
2081     if( redirect)
2082         _flags |= kIOMemoryRedirected;
2083     else {
2084         _flags &= ~kIOMemoryRedirected;
2085         WAKEUP;
2086     }
2087
2088     UNLOCK;
2089
2090     // temporary binary compatibility
2091     IOSubMemoryDescriptor * subMem;
2092     if( (subMem = OSDynamicCast( IOSubMemoryDescriptor, this)))
2093         err = subMem->redirect( safeTask, redirect );
2094     else
2095         err = kIOReturnSuccess;
2096
2097     return( err );
2098 }
2099
2100 IOReturn IOSubMemoryDescriptor::redirect( task_t safeTask, bool redirect )
2101 {
2102     return( _parent->redirect( safeTask, redirect ));
2103 }
2104
2105 IOReturn _IOMemoryMap::redirect( task_t safeTask, bool redirect )
2106 {
2107     IOReturn err = kIOReturnSuccess;
2108
2109     if( superMap) {
2110 //        err = ((_IOMemoryMap *)superMap)->redirect( safeTask, redirect );
2111     } else {
2112
2113         LOCK;
2114         if( logical && addressMap
2115         && (get_task_map( safeTask) != addressMap)
2116         && (0 == (options & kIOMapStatic))) {
2117
2118             IOUnmapPages( addressMap, logical, length );
2119             if( !redirect) {
2120                 err = vm_deallocate( addressMap, logical, length );
2121                 err = memory->doMap( addressMap, &logical,
2122                                      (options & ~kIOMapAnywhere) /*| kIOMapReserve*/,
2123                                      offset, length );
2124             } else
2125                 err = kIOReturnSuccess;
2126 #ifdef DEBUG
2127             IOLog("IOMemoryMap::redirect(%d, %p) %x:%lx from %p\n", redirect, this, logical, length, addressMap);
2128 #endif
2129         }
2130         UNLOCK;
2131     }
2132
2133     return( err );
2134 }
2135
2136 IOReturn _IOMemoryMap::unmap( void )
2137 {
2138     IOReturn    err;
2139
2140     LOCK;
2141
2142     if( logical && addressMap && (0 == superMap)
2143         && (0 == (options & kIOMapStatic))) {
2144
2145         err = memory->doUnmap( addressMap, logical, length );
2146         vm_map_deallocate(addressMap);
2147         addressMap = 0;
2148
2149     } else
2150         err = kIOReturnSuccess;
2151
2152     logical = 0;
2153
2154     UNLOCK;
2155
2156     return( err );
2157 }
2158
2159 void _IOMemoryMap::taskDied( void )
2160 {
2161     LOCK;
2162     if( addressMap) {
2163         vm_map_deallocate(addressMap);
2164         addressMap = 0;
2165     }
2166     addressTask = 0;
2167     logical     = 0;
2168     UNLOCK;
2169 }
2170
2171 // Overload the release mechanism.  All mappings must be a member
2172 // of a memory descriptors _mappings set.  This means that we
2173 // always have 2 references on a mapping.  When either of these mappings
2174 // are released we need to free ourselves.
2175 void _IOMemoryMap::taggedRelease(const void *tag) const
2176 {
2177     LOCK;
2178     super::taggedRelease(tag, 2);
2179     UNLOCK;
2180 }
2181
2182 void _IOMemoryMap::free()
2183 {
2184     unmap();
2185
2186     if( memory) {
2187         LOCK;
2188         memory->removeMapping( this);
2189         UNLOCK;
2190         memory->release();
2191     }
2192
2193     if( superMap)
2194         superMap->release();
2195
2196     super::free();
2197 }
2198
2199 IOByteCount _IOMemoryMap::getLength()
2200 {
2201     return( length );
2202 }
2203
2204 IOVirtualAddress _IOMemoryMap::getVirtualAddress()
2205 {
2206     return( logical);
2207 }
2208
2209 task_t _IOMemoryMap::getAddressTask()
2210 {
2211     if( superMap)
2212         return( superMap->getAddressTask());
2213     else
2214         return( addressTask);
2215 }
2216
2217 IOOptionBits _IOMemoryMap::getMapOptions()
2218 {
2219     return( options);
2220 }
2221
2222 IOMemoryDescriptor * _IOMemoryMap::getMemoryDescriptor()
2223 {
2224     return( memory );
2225 }
2226
2227 _IOMemoryMap * _IOMemoryMap::copyCompatible(
2228                 IOMemoryDescriptor *    owner,
2229                 task_t                  task,
2230                 IOVirtualAddress        toAddress,
2231                 IOOptionBits            _options,
2232                 IOByteCount             _offset,
2233                 IOByteCount             _length )
2234 {
2235     _IOMemoryMap * mapping;
2236
2237     if( (!task) || (!addressMap) || (addressMap != get_task_map(task)))
2238         return( 0 );
2239     if( (options ^ _options) & kIOMapReadOnly)
2240         return( 0 );
2241     if( (kIOMapDefaultCache != (_options & kIOMapCacheMask))
2242      && ((options ^ _options) & kIOMapCacheMask))
2243         return( 0 );
2244
2245     if( (0 == (_options & kIOMapAnywhere)) && (logical != toAddress))
2246         return( 0 );
2247
2248     if( _offset < offset)
2249         return( 0 );
2250
2251     _offset -= offset;
2252
2253     if( (_offset + _length) > length)
2254         return( 0 );
2255
2256     if( (length == _length) && (!_offset)) {
2257         retain();
2258         mapping = this;
2259
2260     } else {
2261         mapping = new _IOMemoryMap;
2262         if( mapping
2263         && !mapping->initCompatible( owner, this, _offset, _length )) {
2264             mapping->release();
2265             mapping = 0;
2266         }
2267     }
2268
2269     return( mapping );
2270 }
2271
2272 IOPhysicalAddress _IOMemoryMap::getPhysicalSegment( IOByteCount _offset,
2273                                                     IOPhysicalLength * length)
2274 {
2275     IOPhysicalAddress   address;
2276
2277     LOCK;
2278     address = memory->getPhysicalSegment( offset + _offset, length );
2279     UNLOCK;
2280
2281     return( address );
2282 }
2283
2284 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2285
2286 #undef super
2287 #define super OSObject
2288
2289 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2290
2291 void IOMemoryDescriptor::initialize( void )
2292 {
2293     if( 0 == gIOMemoryLock)
2294         gIOMemoryLock = IORecursiveLockAlloc();
2295
2296     IORegistryEntry::getRegistryRoot()->setProperty(kIOMaximumMappedIOByteCountKey,
2297                                                     ptoa_64(gIOMaximumMappedIOPageCount), 64);
2298 }
2299
2300 void IOMemoryDescriptor::free( void )
2301 {
2302     if( _mappings)
2303         _mappings->release();
2304
2305     super::free();
2306 }
2307
2308 IOMemoryMap * IOMemoryDescriptor::setMapping(
2309         task_t                  intoTask,
2310         IOVirtualAddress        mapAddress,
2311         IOOptionBits            options )
2312 {
2313     _IOMemoryMap *              map;
2314
2315     map = new _IOMemoryMap;
2316
2317     LOCK;
2318
2319     if( map
2320      && !map->initWithDescriptor( this, intoTask, mapAddress,
2321                     options | kIOMapStatic, 0, getLength() )) {
2322         map->release();
2323         map = 0;
2324     }
2325
2326     addMapping( map);
2327
2328     UNLOCK;
2329
2330     return( map);
2331 }
2332
2333 IOMemoryMap * IOMemoryDescriptor::map(
2334         IOOptionBits            options )
2335 {
2336
2337     return( makeMapping( this, kernel_task, 0,
2338                         options | kIOMapAnywhere,
2339                         0, getLength() ));
2340 }
2341
2342 IOMemoryMap * IOMemoryDescriptor::map(
2343         task_t                  intoTask,
2344         IOVirtualAddress        toAddress,
2345         IOOptionBits            options,
2346         IOByteCount             offset,
2347         IOByteCount             length )
2348 {
2349     if( 0 == length)
2350         length = getLength();
2351
2352     return( makeMapping( this, intoTask, toAddress, options, offset, length ));
2353 }
2354
2355 IOMemoryMap * IOMemoryDescriptor::makeMapping(
2356         IOMemoryDescriptor *    owner,
2357         task_t                  intoTask,
2358         IOVirtualAddress        toAddress,
2359         IOOptionBits            options,
2360         IOByteCount             offset,
2361         IOByteCount             length )
2362 {
2363     _IOMemoryMap *      mapping = 0;
2364     OSIterator *        iter;
2365
2366     LOCK;
2367
2368     do {
2369         // look for an existing mapping
2370         if( (iter = OSCollectionIterator::withCollection( _mappings))) {
2371
2372             while( (mapping = (_IOMemoryMap *) iter->getNextObject())) {
2373
2374                 if( (mapping = mapping->copyCompatible(
2375                                         owner, intoTask, toAddress,
2376                                         options | kIOMapReference,
2377                                         offset, length )))
2378                     break;
2379             }
2380             iter->release();
2381             if( mapping)
2382                 continue;
2383         }
2384
2385
2386         if( mapping || (options & kIOMapReference))
2387             continue;
2388
2389         owner = this;
2390
2391         mapping = new _IOMemoryMap;
2392         if( mapping
2393         && !mapping->initWithDescriptor( owner, intoTask, toAddress, options,
2394                            offset, length )) {
2395 #ifdef DEBUG
2396             IOLog("Didn't make map %08lx : %08lx\n", offset, length );
2397 #endif
2398             mapping->release();
2399             mapping = 0;
2400         }
2401
2402     } while( false );
2403
2404     owner->addMapping( mapping);
2405
2406     UNLOCK;
2407
2408     return( mapping);
2409 }
2410
2411 void IOMemoryDescriptor::addMapping(
2412         IOMemoryMap * mapping )
2413 {
2414     if( mapping) {
2415         if( 0 == _mappings)
2416             _mappings = OSSet::withCapacity(1);
2417         if( _mappings )
2418             _mappings->setObject( mapping );
2419     }
2420 }
2421
2422 void IOMemoryDescriptor::removeMapping(
2423         IOMemoryMap * mapping )
2424 {
2425     if( _mappings)
2426         _mappings->removeObject( mapping);
2427 }
2428
2429 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2430
2431 #undef super
2432 #define super IOMemoryDescriptor
2433
2434 OSDefineMetaClassAndStructors(IOSubMemoryDescriptor, IOMemoryDescriptor)
2435
2436 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2437
2438 bool IOSubMemoryDescriptor::initSubRange( IOMemoryDescriptor * parent,
2439                                         IOByteCount offset, IOByteCount length,
2440                                         IODirection direction )
2441 {
2442     if( !parent)
2443         return( false);
2444
2445     if( (offset + length) > parent->getLength())
2446         return( false);
2447
2448     /*
2449      * We can check the _parent instance variable before having ever set it
2450      * to an initial value because I/O Kit guarantees that all our instance
2451      * variables are zeroed on an object's allocation.
2452      */
2453
2454     if( !_parent) {
2455         if( !super::init())
2456             return( false );
2457     } else {
2458         /*
2459          * An existing memory descriptor is being retargeted to
2460          * point to somewhere else.  Clean up our present state.
2461          */
2462
2463         _parent->release();
2464         _parent = 0;
2465     }
2466
2467     parent->retain();
2468     _parent     = parent;
2469     _start      = offset;
2470     _length     = length;
2471     _direction  = direction;
2472     _tag        = parent->getTag();
2473
2474     return( true );
2475 }
2476
2477 void IOSubMemoryDescriptor::free( void )
2478 {
2479     if( _parent)
2480         _parent->release();
2481
2482     super::free();
2483 }
2484
2485
2486 IOPhysicalAddress IOSubMemoryDescriptor::getPhysicalSegment( IOByteCount offset,
2487                                                         IOByteCount * length )
2488 {
2489     IOPhysicalAddress   address;
2490     IOByteCount         actualLength;
2491
2492     assert(offset <= _length);
2493
2494     if( length)
2495         *length = 0;
2496
2497     if( offset >= _length)
2498         return( 0 );
2499
2500     address = _parent->getPhysicalSegment( offset + _start, &actualLength );
2501
2502     if( address && length)
2503         *length = min( _length - offset, actualLength );
2504
2505     return( address );
2506 }
2507
2508 IOPhysicalAddress IOSubMemoryDescriptor::getSourceSegment( IOByteCount offset,
2509                                                            IOByteCount * length )
2510 {
2511     IOPhysicalAddress   address;
2512     IOByteCount         actualLength;
2513
2514     assert(offset <= _length);
2515
2516     if( length)
2517         *length = 0;
2518
2519     if( offset >= _length)
2520         return( 0 );
2521
2522     address = _parent->getSourceSegment( offset + _start, &actualLength );
2523
2524     if( address && length)
2525         *length = min( _length - offset, actualLength );
2526
2527     return( address );
2528 }
2529
2530 void * IOSubMemoryDescriptor::getVirtualSegment(IOByteCount offset,
2531                                         IOByteCount * lengthOfSegment)
2532 {
2533     return( 0 );
2534 }
2535
2536 IOByteCount IOSubMemoryDescriptor::readBytes(IOByteCount offset,
2537                                         void * bytes, IOByteCount length)
2538 {
2539     IOByteCount byteCount;
2540
2541     assert(offset <= _length);
2542
2543     if( offset >= _length)
2544         return( 0 );
2545
2546     LOCK;
2547     byteCount = _parent->readBytes( _start + offset, bytes,
2548                                 min(length, _length - offset) );
2549     UNLOCK;
2550
2551     return( byteCount );
2552 }
2553
2554 IOByteCount IOSubMemoryDescriptor::writeBytes(IOByteCount offset,
2555                                 const void* bytes, IOByteCount length)
2556 {
2557     IOByteCount byteCount;
2558
2559     assert(offset <= _length);
2560
2561     if( offset >= _length)
2562         return( 0 );
2563
2564     LOCK;
2565     byteCount = _parent->writeBytes( _start + offset, bytes,
2566                                 min(length, _length - offset) );
2567     UNLOCK;
2568
2569     return( byteCount );
2570 }
2571
2572 IOReturn IOSubMemoryDescriptor::prepare(
2573                 IODirection forDirection)
2574 {
2575     IOReturn    err;
2576
2577     LOCK;
2578     err = _parent->prepare( forDirection);
2579     UNLOCK;
2580
2581     return( err );
2582 }
2583
2584 IOReturn IOSubMemoryDescriptor::complete(
2585                 IODirection forDirection)
2586 {
2587     IOReturn    err;
2588
2589     LOCK;
2590     err = _parent->complete( forDirection);
2591     UNLOCK;
2592
2593     return( err );
2594 }
2595
2596 IOMemoryMap * IOSubMemoryDescriptor::makeMapping(
2597         IOMemoryDescriptor *    owner,
2598         task_t                  intoTask,
2599         IOVirtualAddress        toAddress,
2600         IOOptionBits            options,
2601         IOByteCount             offset,
2602         IOByteCount             length )
2603 {
2604     IOMemoryMap * mapping;
2605
2606      mapping = (IOMemoryMap *) _parent->makeMapping(
2607                                         _parent, intoTask,
2608                                         toAddress - (_start + offset),
2609                                         options | kIOMapReference,
2610                                         _start + offset, length );
2611
2612     if( !mapping)
2613         mapping = (IOMemoryMap *) _parent->makeMapping(
2614                                         _parent, intoTask,
2615                                         toAddress,
2616                                         options, _start + offset, length );
2617
2618     if( !mapping)
2619         mapping = super::makeMapping( owner, intoTask, toAddress, options,
2620                                         offset, length );
2621
2622     return( mapping );
2623 }
2624
2625 /* ick */
2626
2627 bool
2628 IOSubMemoryDescriptor::initWithAddress(void *      address,
2629                                     IOByteCount   length,
2630                                     IODirection direction)
2631 {
2632     return( false );
2633 }
2634
2635 bool
2636 IOSubMemoryDescriptor::initWithAddress(vm_address_t address,
2637                                     IOByteCount    length,
2638                                     IODirection  direction,
2639                                     task_t       task)
2640 {
2641     return( false );
2642 }
2643
2644 bool
2645 IOSubMemoryDescriptor::initWithPhysicalAddress(
2646                                  IOPhysicalAddress      address,
2647                                  IOByteCount            length,
2648                                  IODirection            direction )
2649 {
2650     return( false );
2651 }
2652
2653 bool
2654 IOSubMemoryDescriptor::initWithRanges(
2655                                         IOVirtualRange * ranges,
2656                                         UInt32           withCount,
2657                                         IODirection      direction,
2658                                         task_t           task,
2659                                         bool             asReference)
2660 {
2661     return( false );
2662 }
2663
2664 bool
2665 IOSubMemoryDescriptor::initWithPhysicalRanges(  IOPhysicalRange * ranges,
2666                                                 UInt32           withCount,
2667                                                 IODirection      direction,
2668                                                 bool             asReference)
2669 {
2670     return( false );
2671 }
2672
2673 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2674
2675 bool IOGeneralMemoryDescriptor::serialize(OSSerialize * s) const
2676 {
2677     OSSymbol const *keys[2];
2678     OSObject *values[2];
2679     IOVirtualRange *vcopy;
2680     unsigned int index, nRanges;
2681     bool result;
2682
2683     if (s == NULL) return false;
2684     if (s->previouslySerialized(this)) return true;
2685
2686     // Pretend we are an array.
2687     if (!s->addXMLStartTag(this, "array")) return false;
2688
2689     nRanges = _rangesCount;
2690     vcopy = (IOVirtualRange *) IOMalloc(sizeof(IOVirtualRange) * nRanges);
2691     if (vcopy == 0) return false;
2692
2693     keys[0] = OSSymbol::withCString("address");
2694     keys[1] = OSSymbol::withCString("length");
2695
2696     result = false;
2697     values[0] = values[1] = 0;
2698
2699     // From this point on we can go to bail.
2700
2701     // Copy the volatile data so we don't have to allocate memory
2702     // while the lock is held.
2703     LOCK;
2704     if (nRanges == _rangesCount) {
2705         for (index = 0; index < nRanges; index++) {
2706             vcopy[index] = _ranges.v[index];
2707         }
2708     } else {
2709         // The descriptor changed out from under us.  Give up.
2710         UNLOCK;
2711         result = false;
2712         goto bail;
2713     }
2714     UNLOCK;
2715
2716     for (index = 0; index < nRanges; index++)
2717     {
2718         values[0] = OSNumber::withNumber(_ranges.v[index].address, sizeof(_ranges.v[index].address) * 8);
2719         if (values[0] == 0) {
2720           result = false;
2721           goto bail;
2722         }
2723         values[1] = OSNumber::withNumber(_ranges.v[index].length, sizeof(_ranges.v[index].length) * 8);
2724         if (values[1] == 0) {
2725           result = false;
2726           goto bail;
2727         }
2728         OSDictionary *dict = OSDictionary::withObjects((const OSObject **)values, (const OSSymbol **)keys, 2);
2729         if (dict == 0) {
2730           result = false;
2731           goto bail;
2732         }
2733         values[0]->release();
2734         values[1]->release();
2735         values[0] = values[1] = 0;
2736
2737         result = dict->serialize(s);
2738         dict->release();
2739         if (!result) {
2740           goto bail;
2741         }
2742     }
2743     result = s->addXMLEndTag("array");
2744
2745  bail:
2746     if (values[0])
2747       values[0]->release();
2748     if (values[1])
2749       values[1]->release();
2750     if (keys[0])
2751       keys[0]->release();
2752     if (keys[1])
2753       keys[1]->release();
2754     if (vcopy)
2755         IOFree(vcopy, sizeof(IOVirtualRange) * nRanges);
2756     return result;
2757 }
2758
2759 bool IOSubMemoryDescriptor::serialize(OSSerialize * s) const
2760 {
2761     if (!s) {
2762         return (false);
2763     }
2764     if (s->previouslySerialized(this)) return true;
2765
2766     // Pretend we are a dictionary.
2767     // We must duplicate the functionality of OSDictionary here
2768     // because otherwise object references will not work;
2769     // they are based on the value of the object passed to
2770     // previouslySerialized and addXMLStartTag.
2771
2772     if (!s->addXMLStartTag(this, "dict")) return false;
2773
2774     char const *keys[3] = {"offset", "length", "parent"};
2775
2776     OSObject *values[3];
2777     values[0] = OSNumber::withNumber(_start, sizeof(_start) * 8);
2778     if (values[0] == 0)
2779         return false;
2780     values[1] = OSNumber::withNumber(_length, sizeof(_length) * 8);
2781     if (values[1] == 0) {
2782         values[0]->release();
2783         return false;
2784     }
2785     values[2] = _parent;
2786
2787     bool result = true;
2788     for (int i=0; i<3; i++) {
2789         if (!s->addString("<key>") ||
2790             !s->addString(keys[i]) ||
2791             !s->addXMLEndTag("key") ||
2792             !values[i]->serialize(s)) {
2793           result = false;
2794           break;
2795         }
2796     }
2797     values[0]->release();
2798     values[1]->release();
2799     if (!result) {
2800       return false;
2801     }
2802
2803     return s->addXMLEndTag("dict");
2804 }
2805
2806 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
2807
2808 OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 0);
2809 OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 1);
2810 OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 2);
2811 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 3);
2812 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 4);
2813 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 5);
2814 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 6);
2815 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 7);
2816 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 8);
2817 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 9);
2818 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 10);
2819 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 11);
2820 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 12);
2821 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 13);
2822 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 14);
2823 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 15);
2824
2825 /* ex-inline function implementation */
2826 IOPhysicalAddress IOMemoryDescriptor::getPhysicalAddress()
2827         { return( getPhysicalSegment( 0, 0 )); }