iokit/Kernel/IOBufferMemoryDescriptor.cpp

   1 /*
   2  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #define _IOMEMORYDESCRIPTOR_INTERNAL_
  30
  31 #include <IOKit/assert.h>
  32 #include <IOKit/system.h>
  33
  34 #include <IOKit/IOLib.h>
  35 #include <IOKit/IOMapper.h>
  36 #include <IOKit/IOBufferMemoryDescriptor.h>
  37 #include <libkern/OSDebug.h>
  38 #include <mach/mach_vm.h>
  39
  40 #include "IOKitKernelInternal.h"
  41
  42 #ifdef IOALLOCDEBUG
  43 #include <libkern/c++/OSCPPDebug.h>
  44 #endif
  45 #include <IOKit/IOStatisticsPrivate.h>
  46
  47 #if IOKITSTATS
  48 #define IOStatisticsAlloc(type, size) \
  49 do { \
  50         IOStatistics::countAlloc(type, size); \
  51 } while (0)
  52 #else
  53 #define IOStatisticsAlloc(type, size)
  54 #endif /* IOKITSTATS */
  55
  56
  57 __BEGIN_DECLS
  58 void ipc_port_release_send(ipc_port_t port);
  59 #include <vm/pmap.h>
  60
  61 __END_DECLS
  62
  63 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  64
  65 enum
  66 {
  67     kInternalFlagPhysical      = 0x00000001,
  68     kInternalFlagPageSized     = 0x00000002,
  69     kInternalFlagPageAllocated = 0x00000004
  70 };
  71
  72 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  73
  74 #if 0
  75 #undef assert
  76 #define assert(ex)  \
  77         ((ex) ? (void)0 : Assert(__FILE__, __LINE__, # ex))
  78 #endif
  79
  80 enum
  81 {
  82     kIOPageAllocChunkBytes = (PAGE_SIZE / 64),
  83     kIOPageAllocSignature  = 'iopa'
  84 };
  85
  86 struct io_pagealloc_t
  87 {
  88     queue_chain_t link;
  89     uint64_t      avail;
  90     uint32_t      signature;
  91 };
  92 typedef struct io_pagealloc_t io_pagealloc_t;
  93
  94 typedef char io_pagealloc_t_assert[(sizeof(io_pagealloc_t) <= kIOPageAllocChunkBytes) ? 1 : -1];
  95
  96 IOSimpleLock * gIOPageAllocLock;
  97 queue_head_t   gIOPageAllocList;
  98 vm_size_t      gIOPageAllocCount;
  99 vm_size_t      gIOPageAllocBytes;
 100
 101 static io_pagealloc_t *
 102 iopa_allocpage(void)
 103 {
 104     kern_return_t    kr;
 105     io_pagealloc_t * pa;
 106     vm_address_t     vmaddr = 0;
 107
 108     int options = 0; // KMA_LOMEM;
 109     kr = kernel_memory_allocate(kernel_map, &vmaddr,
 110                                 page_size, 0, options);
 111     if (KERN_SUCCESS != kr) return (0);
 112
 113     bzero((void *) vmaddr, page_size);
 114     pa = (typeof(pa)) (vmaddr + page_size - kIOPageAllocChunkBytes);
 115
 116     pa->signature = kIOPageAllocSignature;
 117     pa->avail     = -2ULL;
 118
 119     return (pa);
 120 }
 121
 122 static void
 123 iopa_freepage(io_pagealloc_t * pa)
 124 {
 125     kmem_free( kernel_map, trunc_page((uintptr_t) pa), page_size);
 126 }
 127
 128 static uintptr_t
 129 iopa_allocinpage(io_pagealloc_t * pa, uint32_t count, uint64_t align)
 130 {
 131     uint32_t n, s;
 132     uint64_t avail = pa->avail;
 133
 134     assert(avail);
 135
 136     // find strings of count 1 bits in avail
 137     for (n = count; n > 1; n -= s)
 138     {
 139         s = n >> 1;
 140         avail = avail & (avail << s);
 141     }
 142     // and aligned
 143     avail &= align;
 144
 145     if (avail)
 146     {
 147         n = __builtin_clzll(avail);
 148         pa->avail &= ~((-1ULL << (64 - count)) >> n);
 149         if (!pa->avail && pa->link.next)
 150         {
 151             remque(&pa->link);
 152             pa->link.next = 0;
 153         }
 154         return (n * kIOPageAllocChunkBytes + trunc_page((uintptr_t) pa));
 155     }
 156
 157     return (0);
 158 }
 159
 160 static uint32_t
 161 log2up(uint32_t size)
 162 {
 163     if (size <= 1) size = 0;
 164     else size = 32 - __builtin_clz(size - 1);
 165     return (size);
 166 }
 167
 168 static uintptr_t
 169 iopa_alloc(vm_size_t bytes, uint32_t balign)
 170 {
 171     static const uint64_t align_masks[] = {
 172         0xFFFFFFFFFFFFFFFF,
 173         0xAAAAAAAAAAAAAAAA,
 174         0x8888888888888888,
 175         0x8080808080808080,
 176         0x8000800080008000,
 177         0x8000000080000000,
 178         0x8000000000000000,
 179     };
 180     io_pagealloc_t * pa;
 181     uintptr_t        addr = 0;
 182     uint32_t         count;
 183     uint64_t         align;
 184
 185     if (!bytes) bytes = 1;
 186     count = (bytes + kIOPageAllocChunkBytes - 1) / kIOPageAllocChunkBytes;
 187     align = align_masks[log2up((balign + kIOPageAllocChunkBytes - 1) / kIOPageAllocChunkBytes)];
 188
 189     IOSimpleLockLock(gIOPageAllocLock);
 190     pa = (typeof(pa)) queue_first(&gIOPageAllocList);
 191     while (!queue_end(&gIOPageAllocList, &pa->link))
 192     {
 193         addr = iopa_allocinpage(pa, count, align);
 194         if (addr)
 195         {
 196             gIOPageAllocBytes += bytes;
 197             break;
 198         }
 199         pa = (typeof(pa)) queue_next(&pa->link);
 200     }
 201     IOSimpleLockUnlock(gIOPageAllocLock);
 202     if (!addr)
 203     {
 204         pa = iopa_allocpage();
 205         if (pa)
 206         {
 207             addr = iopa_allocinpage(pa, count, align);
 208             IOSimpleLockLock(gIOPageAllocLock);
 209             if (pa->avail) enqueue_head(&gIOPageAllocList, &pa->link);
 210             gIOPageAllocCount++;
 211             if (addr) gIOPageAllocBytes += bytes;
 212             IOSimpleLockUnlock(gIOPageAllocLock);
 213         }
 214     }
 215
 216     if (addr)
 217     {
 218         assert((addr & ((1 << log2up(balign)) - 1)) == 0);
 219         IOStatisticsAlloc(kIOStatisticsMallocAligned, bytes);
 220 #if IOALLOCDEBUG
 221         debug_iomalloc_size += bytes;
 222 #endif
 223     }
 224
 225     return (addr);
 226 }
 227
 228 static void
 229 iopa_free(uintptr_t addr, vm_size_t bytes)
 230 {
 231     io_pagealloc_t * pa;
 232     uint32_t         count;
 233     uintptr_t        chunk;
 234
 235     if (!bytes) bytes = 1;
 236
 237     chunk = (addr & page_mask);
 238     assert(0 == (chunk & (kIOPageAllocChunkBytes - 1)));
 239
 240     pa = (typeof(pa)) (addr | (page_size - kIOPageAllocChunkBytes));
 241     assert(kIOPageAllocSignature == pa->signature);
 242
 243     count = (bytes + kIOPageAllocChunkBytes - 1) / kIOPageAllocChunkBytes;
 244     chunk /= kIOPageAllocChunkBytes;
 245
 246     IOSimpleLockLock(gIOPageAllocLock);
 247     if (!pa->avail)
 248     {
 249         assert(!pa->link.next);
 250         enqueue_tail(&gIOPageAllocList, &pa->link);
 251     }
 252     pa->avail |= ((-1ULL << (64 - count)) >> chunk);
 253     if (pa->avail != -2ULL) pa = 0;
 254     else
 255     {
 256         remque(&pa->link);
 257         pa->link.next = 0;
 258         pa->signature = 0;
 259         gIOPageAllocCount--;
 260     }
 261     gIOPageAllocBytes -= bytes;
 262     IOSimpleLockUnlock(gIOPageAllocLock);
 263     if (pa) iopa_freepage(pa);
 264
 265 #if IOALLOCDEBUG
 266     debug_iomalloc_size -= bytes;
 267 #endif
 268     IOStatisticsAlloc(kIOStatisticsFreeAligned, bytes);
 269 }
 270
 271 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 272
 273 #define super IOGeneralMemoryDescriptor
 274 OSDefineMetaClassAndStructors(IOBufferMemoryDescriptor,
 275                                 IOGeneralMemoryDescriptor);
 276
 277 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 278
 279 #ifndef __LP64__
 280 bool IOBufferMemoryDescriptor::initWithOptions(
 281                                IOOptionBits options,
 282                                vm_size_t    capacity,
 283                                vm_offset_t  alignment,
 284                                task_t       inTask)
 285 {
 286     mach_vm_address_t physicalMask = 0;
 287     return (initWithPhysicalMask(inTask, options, capacity, alignment, physicalMask));
 288 }
 289 #endif /* !__LP64__ */
 290
 291 bool IOBufferMemoryDescriptor::initWithPhysicalMask(
 292                                 task_t            inTask,
 293                                 IOOptionBits      options,
 294                                 mach_vm_size_t    capacity,
 295                                 mach_vm_address_t alignment,
 296                                 mach_vm_address_t physicalMask)
 297 {
 298     kern_return_t         kr;
 299     task_t                mapTask = NULL;
 300     vm_map_t              vmmap = NULL;
 301     mach_vm_address_t     highestMask = 0;
 302     IOOptionBits          iomdOptions = kIOMemoryTypeVirtual64 | kIOMemoryAsReference;
 303     IODMAMapSpecification mapSpec;
 304     bool                  mapped = false;
 305     bool                  needZero;
 306
 307     if (!capacity)
 308         return false;
 309
 310     _options          = options;
 311     _capacity         = capacity;
 312     _internalFlags    = 0;
 313     _internalReserved = 0;
 314     _buffer           = 0;
 315
 316     _ranges.v64 = IONew(IOAddressRange, 1);
 317     if (!_ranges.v64)
 318         return (false);
 319     _ranges.v64->address = 0;
 320     _ranges.v64->length  = 0;
 321     //  make sure super::free doesn't dealloc _ranges before super::init
 322     _flags = kIOMemoryAsReference;
 323
 324     // Grab IOMD bits from the Buffer MD options
 325     iomdOptions  |= (options & kIOBufferDescriptorMemoryFlags);
 326
 327     if (!(kIOMemoryMapperNone & options))
 328     {
 329         IOMapper::checkForSystemMapper();
 330         mapped = (0 != IOMapper::gSystem);
 331     }
 332     needZero = mapped;
 333
 334     if (physicalMask && (alignment <= 1))
 335     {
 336         alignment   = ((physicalMask ^ (-1ULL)) & (physicalMask - 1));
 337         highestMask = (physicalMask | alignment);
 338         alignment++;
 339         if (alignment < page_size)
 340             alignment = page_size;
 341     }
 342
 343     if ((options & (kIOMemorySharingTypeMask | kIOMapCacheMask | kIOMemoryClearEncrypt)) && (alignment < page_size))
 344         alignment = page_size;
 345
 346     if (alignment >= page_size)
 347         capacity = round_page(capacity);
 348
 349     if (alignment > page_size)
 350         options |= kIOMemoryPhysicallyContiguous;
 351
 352     _alignment = alignment;
 353
 354     if ((inTask != kernel_task) && !(options & kIOMemoryPageable))
 355         return false;
 356
 357     bzero(&mapSpec, sizeof(mapSpec));
 358     mapSpec.alignment      = _alignment;
 359     mapSpec.numAddressBits = 64;
 360     if (highestMask && mapped)
 361     {
 362         if (highestMask <= 0xFFFFFFFF)
 363             mapSpec.numAddressBits = (32 - __builtin_clz((unsigned int) highestMask));
 364         else
 365             mapSpec.numAddressBits = (64 - __builtin_clz((unsigned int) (highestMask >> 32)));
 366         highestMask = 0;
 367     }
 368
 369     // set flags for entry + object create
 370     vm_prot_t memEntryCacheMode = VM_PROT_READ | VM_PROT_WRITE;
 371
 372     // set memory entry cache mode
 373     switch (options & kIOMapCacheMask)
 374     {
 375         case kIOMapInhibitCache:
 376             SET_MAP_MEM(MAP_MEM_IO, memEntryCacheMode);
 377             break;
 378
 379         case kIOMapWriteThruCache:
 380             SET_MAP_MEM(MAP_MEM_WTHRU, memEntryCacheMode);
 381             break;
 382
 383         case kIOMapWriteCombineCache:
 384             SET_MAP_MEM(MAP_MEM_WCOMB, memEntryCacheMode);
 385             break;
 386
 387         case kIOMapCopybackCache:
 388             SET_MAP_MEM(MAP_MEM_COPYBACK, memEntryCacheMode);
 389             break;
 390
 391         case kIOMapCopybackInnerCache:
 392             SET_MAP_MEM(MAP_MEM_INNERWBACK, memEntryCacheMode);
 393             break;
 394
 395         case kIOMapDefaultCache:
 396         default:
 397             SET_MAP_MEM(MAP_MEM_NOOP, memEntryCacheMode);
 398             break;
 399     }
 400
 401     if (options & kIOMemoryPageable)
 402     {
 403         iomdOptions |= kIOMemoryBufferPageable;
 404
 405         // must create the entry before any pages are allocated
 406
 407         // set flags for entry + object create
 408         memEntryCacheMode |= MAP_MEM_NAMED_CREATE;
 409
 410         if (options & kIOMemoryPurgeable)
 411             memEntryCacheMode |= MAP_MEM_PURGABLE;
 412     }
 413     else
 414     {
 415         memEntryCacheMode |= MAP_MEM_NAMED_REUSE;
 416         vmmap = kernel_map;
 417
 418         // Buffer shouldn't auto prepare they should be prepared explicitly
 419         // But it never was enforced so what are you going to do?
 420         iomdOptions |= kIOMemoryAutoPrepare;
 421
 422         /* Allocate a wired-down buffer inside kernel space. */
 423
 424         bool contig = (0 != (options & kIOMemoryHostPhysicallyContiguous));
 425
 426         if (!contig && (0 != (options & kIOMemoryPhysicallyContiguous)))
 427         {
 428             contig |= (!mapped);
 429             contig |= (0 != (kIOMemoryMapperNone & options));
 430 #if 0
 431             // treat kIOMemoryPhysicallyContiguous as kIOMemoryHostPhysicallyContiguous for now
 432             contig |= true;
 433 #endif
 434         }
 435
 436         if (contig || highestMask || (alignment > page_size))
 437         {
 438             _internalFlags |= kInternalFlagPhysical;
 439             if (highestMask)
 440             {
 441                 _internalFlags |= kInternalFlagPageSized;
 442                 capacity = round_page(capacity);
 443             }
 444             _buffer = (void *) IOKernelAllocateWithPhysicalRestrict(
 445                                         capacity, highestMask, alignment, contig);
 446         }
 447         else if (needZero
 448                   && ((capacity + alignment) <= (page_size - kIOPageAllocChunkBytes)))
 449         {
 450             _internalFlags |= kInternalFlagPageAllocated;
 451             needZero        = false;
 452             _buffer         = (void *) iopa_alloc(capacity, alignment);
 453         }
 454         else if (alignment > 1)
 455         {
 456             _buffer = IOMallocAligned(capacity, alignment);
 457         }
 458         else
 459         {
 460             _buffer = IOMalloc(capacity);
 461         }
 462         if (!_buffer)
 463         {
 464             return false;
 465         }
 466         if (needZero) bzero(_buffer, capacity);
 467     }
 468
 469     if( (options & (kIOMemoryPageable | kIOMapCacheMask))) {
 470         ipc_port_t      sharedMem;
 471         vm_size_t       size = round_page(capacity);
 472
 473         kr = mach_make_memory_entry(vmmap,
 474                                     &size, (vm_offset_t)_buffer,
 475                                     memEntryCacheMode, &sharedMem,
 476                                     NULL );
 477
 478         if( (KERN_SUCCESS == kr) && (size != round_page(capacity))) {
 479             ipc_port_release_send( sharedMem );
 480             kr = kIOReturnVMError;
 481         }
 482         if( KERN_SUCCESS != kr)
 483             return( false );
 484
 485         _memEntry = (void *) sharedMem;
 486
 487         if( options & kIOMemoryPageable) {
 488 #if IOALLOCDEBUG
 489             debug_iomallocpageable_size += size;
 490 #endif
 491             mapTask = inTask;
 492             if (NULL == inTask)
 493                 inTask = kernel_task;
 494         }
 495         else if (options & kIOMapCacheMask)
 496         {
 497             // Prefetch each page to put entries into the pmap
 498             volatile UInt8 *    startAddr = (UInt8 *)_buffer;
 499             volatile UInt8 *    endAddr   = (UInt8 *)_buffer + capacity;
 500
 501             while (startAddr < endAddr)
 502             {
 503                 *startAddr;
 504                 startAddr += page_size;
 505             }
 506         }
 507     }
 508
 509     _ranges.v64->address = (mach_vm_address_t) _buffer;;
 510     _ranges.v64->length  = _capacity;
 511
 512     if (!super::initWithOptions(_ranges.v64, 1, 0,
 513                                 inTask, iomdOptions, /* System mapper */ 0))
 514         return false;
 515
 516     // give any system mapper the allocation params
 517     if (kIOReturnSuccess != dmaCommandOperation(kIOMDAddDMAMapSpec,
 518                                                 &mapSpec, sizeof(mapSpec)))
 519         return false;
 520
 521     if (mapTask)
 522     {
 523         if (!reserved) {
 524             reserved = IONew( ExpansionData, 1 );
 525             if( !reserved)
 526                 return( false );
 527         }
 528         reserved->map = createMappingInTask(mapTask, 0,
 529                             kIOMapAnywhere | (options & kIOMapCacheMask), 0, 0);
 530         if (!reserved->map)
 531         {
 532             _buffer = 0;
 533             return( false );
 534         }
 535         release();          // map took a retain on this
 536         reserved->map->retain();
 537         removeMapping(reserved->map);
 538         mach_vm_address_t buffer = reserved->map->getAddress();
 539         _buffer = (void *) buffer;
 540         if (kIOMemoryTypeVirtual64 == (kIOMemoryTypeMask & iomdOptions))
 541             _ranges.v64->address = buffer;
 542     }
 543
 544     setLength(_capacity);
 545
 546     return true;
 547 }
 548
 549 IOBufferMemoryDescriptor * IOBufferMemoryDescriptor::inTaskWithOptions(
 550                                             task_t       inTask,
 551                                             IOOptionBits options,
 552                                             vm_size_t    capacity,
 553                                             vm_offset_t  alignment)
 554 {
 555     IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor;
 556
 557     if (me && !me->initWithPhysicalMask(inTask, options, capacity, alignment, 0)) {
 558         me->release();
 559         me = 0;
 560     }
 561     return me;
 562 }
 563
 564 IOBufferMemoryDescriptor * IOBufferMemoryDescriptor::inTaskWithPhysicalMask(
 565                                             task_t            inTask,
 566                                             IOOptionBits      options,
 567                                             mach_vm_size_t    capacity,
 568                                             mach_vm_address_t physicalMask)
 569 {
 570     IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor;
 571
 572     if (me && !me->initWithPhysicalMask(inTask, options, capacity, 1, physicalMask))
 573     {
 574         me->release();
 575         me = 0;
 576     }
 577     return me;
 578 }
 579
 580 #ifndef __LP64__
 581 bool IOBufferMemoryDescriptor::initWithOptions(
 582                                IOOptionBits options,
 583                                vm_size_t    capacity,
 584                                vm_offset_t  alignment)
 585 {
 586     return (initWithPhysicalMask(kernel_task, options, capacity, alignment, (mach_vm_address_t)0));
 587 }
 588 #endif /* !__LP64__ */
 589
 590 IOBufferMemoryDescriptor * IOBufferMemoryDescriptor::withOptions(
 591                                             IOOptionBits options,
 592                                             vm_size_t    capacity,
 593                                             vm_offset_t  alignment)
 594 {
 595     IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor;
 596
 597     if (me && !me->initWithPhysicalMask(kernel_task, options, capacity, alignment, 0)) {
 598         me->release();
 599         me = 0;
 600     }
 601     return me;
 602 }
 603
 604
 605 /*
 606  * withCapacity:
 607  *
 608  * Returns a new IOBufferMemoryDescriptor with a buffer large enough to
 609  * hold capacity bytes.  The descriptor's length is initially set to the capacity.
 610  */
 611 IOBufferMemoryDescriptor *
 612 IOBufferMemoryDescriptor::withCapacity(vm_size_t   inCapacity,
 613                                        IODirection inDirection,
 614                                        bool        inContiguous)
 615 {
 616     return( IOBufferMemoryDescriptor::withOptions(
 617                inDirection | kIOMemoryUnshared
 618                 | (inContiguous ? kIOMemoryPhysicallyContiguous : 0),
 619                inCapacity, inContiguous ? inCapacity : 1 ));
 620 }
 621
 622 #ifndef __LP64__
 623 /*
 624  * initWithBytes:
 625  *
 626  * Initialize a new IOBufferMemoryDescriptor preloaded with bytes (copied).
 627  * The descriptor's length and capacity are set to the input buffer's size.
 628  */
 629 bool IOBufferMemoryDescriptor::initWithBytes(const void * inBytes,
 630                                              vm_size_t    inLength,
 631                                              IODirection  inDirection,
 632                                              bool         inContiguous)
 633 {
 634     if (!initWithPhysicalMask(kernel_task, inDirection | kIOMemoryUnshared
 635                               | (inContiguous ? kIOMemoryPhysicallyContiguous : 0),
 636                               inLength, inLength, (mach_vm_address_t)0))
 637         return false;
 638
 639     // start out with no data
 640     setLength(0);
 641
 642     if (!appendBytes(inBytes, inLength))
 643         return false;
 644
 645     return true;
 646 }
 647 #endif /* !__LP64__ */
 648
 649 /*
 650  * withBytes:
 651  *
 652  * Returns a new IOBufferMemoryDescriptor preloaded with bytes (copied).
 653  * The descriptor's length and capacity are set to the input buffer's size.
 654  */
 655 IOBufferMemoryDescriptor *
 656 IOBufferMemoryDescriptor::withBytes(const void * inBytes,
 657                                     vm_size_t    inLength,
 658                                     IODirection  inDirection,
 659                                     bool         inContiguous)
 660 {
 661     IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor;
 662
 663     if (me && !me->initWithPhysicalMask(
 664                kernel_task, inDirection | kIOMemoryUnshared
 665                 | (inContiguous ? kIOMemoryPhysicallyContiguous : 0),
 666                inLength, inLength, 0 ))
 667     {
 668         me->release();
 669         me = 0;
 670     }
 671
 672     if (me)
 673     {
 674         // start out with no data
 675         me->setLength(0);
 676
 677         if (!me->appendBytes(inBytes, inLength))
 678         {
 679             me->release();
 680             me = 0;
 681         }
 682     }
 683     return me;
 684 }
 685
 686 /*
 687  * free:
 688  *
 689  * Free resources
 690  */
 691 void IOBufferMemoryDescriptor::free()
 692 {
 693     // Cache all of the relevant information on the stack for use
 694     // after we call super::free()!
 695     IOOptionBits     flags         = _flags;
 696     IOOptionBits     internalFlags = _internalFlags;
 697     IOOptionBits     options   = _options;
 698     vm_size_t        size      = _capacity;
 699     void *           buffer    = _buffer;
 700     IOMemoryMap *    map       = 0;
 701     IOAddressRange * range     = _ranges.v64;
 702     vm_offset_t      alignment = _alignment;
 703
 704     if (alignment >= page_size)
 705         size = round_page(size);
 706
 707     if (reserved)
 708     {
 709         map = reserved->map;
 710         IODelete( reserved, ExpansionData, 1 );
 711         if (map)
 712             map->release();
 713     }
 714
 715     /* super::free may unwire - deallocate buffer afterwards */
 716     super::free();
 717
 718     if (options & kIOMemoryPageable)
 719     {
 720 #if IOALLOCDEBUG
 721         debug_iomallocpageable_size -= round_page(size);
 722 #endif
 723     }
 724     else if (buffer)
 725     {
 726         if (kInternalFlagPageSized & internalFlags) size = round_page(size);
 727
 728         if (kInternalFlagPhysical & internalFlags)
 729         {
 730             IOKernelFreePhysical((mach_vm_address_t) buffer, size);
 731         }
 732         else if (kInternalFlagPageAllocated & internalFlags)
 733         {
 734             iopa_free((uintptr_t) buffer, size);
 735         }
 736         else if (alignment > 1)
 737         {
 738             IOFreeAligned(buffer, size);
 739         }
 740         else
 741         {
 742             IOFree(buffer, size);
 743         }
 744     }
 745     if (range && (kIOMemoryAsReference & flags))
 746         IODelete(range, IOAddressRange, 1);
 747 }
 748
 749 /*
 750  * getCapacity:
 751  *
 752  * Get the buffer capacity
 753  */
 754 vm_size_t IOBufferMemoryDescriptor::getCapacity() const
 755 {
 756     return _capacity;
 757 }
 758
 759 /*
 760  * setLength:
 761  *
 762  * Change the buffer length of the memory descriptor.  When a new buffer
 763  * is created, the initial length of the buffer is set to be the same as
 764  * the capacity.  The length can be adjusted via setLength for a shorter
 765  * transfer (there is no need to create more buffer descriptors when you
 766  * can reuse an existing one, even for different transfer sizes).   Note
 767  * that the specified length must not exceed the capacity of the buffer.
 768  */
 769 void IOBufferMemoryDescriptor::setLength(vm_size_t length)
 770 {
 771     assert(length <= _capacity);
 772
 773     _length = length;
 774     _ranges.v64->length = length;
 775 }
 776
 777 /*
 778  * setDirection:
 779  *
 780  * Change the direction of the transfer.  This method allows one to redirect
 781  * the descriptor's transfer direction.  This eliminates the need to destroy
 782  * and create new buffers when different transfer directions are needed.
 783  */
 784 void IOBufferMemoryDescriptor::setDirection(IODirection direction)
 785 {
 786     _flags = (_flags & ~kIOMemoryDirectionMask) | direction;
 787 #ifndef __LP64__
 788     _direction = (IODirection) (_flags & kIOMemoryDirectionMask);
 789 #endif /* !__LP64__ */
 790 }
 791
 792 /*
 793  * appendBytes:
 794  *
 795  * Add some data to the end of the buffer.  This method automatically
 796  * maintains the memory descriptor buffer length.  Note that appendBytes
 797  * will not copy past the end of the memory descriptor's current capacity.
 798  */
 799 bool
 800 IOBufferMemoryDescriptor::appendBytes(const void * bytes, vm_size_t withLength)
 801 {
 802     vm_size_t   actualBytesToCopy = min(withLength, _capacity - _length);
 803     IOByteCount offset;
 804
 805     assert(_length <= _capacity);
 806
 807     offset = _length;
 808     _length += actualBytesToCopy;
 809     _ranges.v64->length += actualBytesToCopy;
 810
 811     if (_task == kernel_task)
 812         bcopy(/* from */ bytes, (void *)(_ranges.v64->address + offset),
 813               actualBytesToCopy);
 814     else
 815         writeBytes(offset, bytes, actualBytesToCopy);
 816
 817     return true;
 818 }
 819
 820 /*
 821  * getBytesNoCopy:
 822  *
 823  * Return the virtual address of the beginning of the buffer
 824  */
 825 void * IOBufferMemoryDescriptor::getBytesNoCopy()
 826 {
 827     if (kIOMemoryTypePhysical64 == (_flags & kIOMemoryTypeMask))
 828         return _buffer;
 829     else
 830         return (void *)_ranges.v64->address;
 831 }
 832
 833
 834 /*
 835  * getBytesNoCopy:
 836  *
 837  * Return the virtual address of an offset from the beginning of the buffer
 838  */
 839 void *
 840 IOBufferMemoryDescriptor::getBytesNoCopy(vm_size_t start, vm_size_t withLength)
 841 {
 842     IOVirtualAddress address;
 843     if (kIOMemoryTypePhysical64 == (_flags & kIOMemoryTypeMask))
 844         address = (IOVirtualAddress) _buffer;
 845     else
 846         address = _ranges.v64->address;
 847
 848    if (start < _length && (start + withLength) <= _length)
 849         return (void *)(address + start);
 850     return 0;
 851 }
 852
 853 #ifndef __LP64__
 854 void * IOBufferMemoryDescriptor::getVirtualSegment(IOByteCount offset,
 855                                                         IOByteCount * lengthOfSegment)
 856 {
 857     void * bytes = getBytesNoCopy(offset, 0);
 858
 859     if (bytes && lengthOfSegment)
 860         *lengthOfSegment = _length - offset;
 861
 862     return bytes;
 863 }
 864 #endif /* !__LP64__ */
 865
 866 #ifdef __LP64__
 867 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 0);
 868 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 1);
 869 #else /* !__LP64__ */
 870 OSMetaClassDefineReservedUsed(IOBufferMemoryDescriptor, 0);
 871 OSMetaClassDefineReservedUsed(IOBufferMemoryDescriptor, 1);
 872 #endif /* !__LP64__ */
 873 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 2);
 874 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 3);
 875 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 4);
 876 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 5);
 877 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 6);
 878 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 7);
 879 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 8);
 880 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 9);
 881 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 10);
 882 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 11);
 883 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 12);
 884 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 13);
 885 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 14);
 886 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 15);