iokit/Kernel/IOBufferMemoryDescriptor.cpp

   1 /*
   2  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #define _IOMEMORYDESCRIPTOR_INTERNAL_
  30
  31 #include <IOKit/assert.h>
  32 #include <IOKit/system.h>
  33
  34 #include <IOKit/IOLib.h>
  35 #include <IOKit/IOMapper.h>
  36 #include <IOKit/IOBufferMemoryDescriptor.h>
  37 #include <libkern/OSDebug.h>
  38 #include <mach/mach_vm.h>
  39
  40 #include "IOKitKernelInternal.h"
  41
  42 #ifdef IOALLOCDEBUG
  43 #include <libkern/c++/OSCPPDebug.h>
  44 #endif
  45 #include <IOKit/IOStatisticsPrivate.h>
  46
  47 #if IOKITSTATS
  48 #define IOStatisticsAlloc(type, size) \
  49 do { \
  50         IOStatistics::countAlloc(type, size); \
  51 } while (0)
  52 #else
  53 #define IOStatisticsAlloc(type, size)
  54 #endif /* IOKITSTATS */
  55
  56
  57 __BEGIN_DECLS
  58 void ipc_port_release_send(ipc_port_t port);
  59 #include <vm/pmap.h>
  60
  61 __END_DECLS
  62
  63 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  64
  65 enum{
  66         kInternalFlagPhysical      = 0x00000001,
  67         kInternalFlagPageSized     = 0x00000002,
  68         kInternalFlagPageAllocated = 0x00000004,
  69         kInternalFlagInit          = 0x00000008
  70 };
  71
  72 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  73
  74 #define super IOGeneralMemoryDescriptor
  75 OSDefineMetaClassAndStructors(IOBufferMemoryDescriptor,
  76     IOGeneralMemoryDescriptor);
  77
  78 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  79
  80 static uintptr_t
  81 IOBMDPageProc(iopa_t * a)
  82 {
  83         kern_return_t kr;
  84         vm_address_t  vmaddr  = 0;
  85         int           options = 0;// KMA_LOMEM;
  86
  87         kr = kernel_memory_allocate(kernel_map, &vmaddr,
  88             page_size, 0, options, VM_KERN_MEMORY_IOKIT);
  89
  90         if (KERN_SUCCESS != kr) {
  91                 vmaddr = 0;
  92         } else {
  93                 bzero((void *) vmaddr, page_size);
  94         }
  95
  96         return (uintptr_t) vmaddr;
  97 }
  98
  99 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 100
 101 #ifndef __LP64__
 102 bool
 103 IOBufferMemoryDescriptor::initWithOptions(
 104         IOOptionBits options,
 105         vm_size_t    capacity,
 106         vm_offset_t  alignment,
 107         task_t       inTask)
 108 {
 109         mach_vm_address_t physicalMask = 0;
 110         return initWithPhysicalMask(inTask, options, capacity, alignment, physicalMask);
 111 }
 112 #endif /* !__LP64__ */
 113
 114 IOBufferMemoryDescriptor *
 115 IOBufferMemoryDescriptor::withCopy(
 116         task_t                inTask,
 117         IOOptionBits      options,
 118         vm_map_t              sourceMap,
 119         mach_vm_address_t source,
 120         mach_vm_size_t    size)
 121 {
 122         IOBufferMemoryDescriptor * inst;
 123         kern_return_t              err;
 124         vm_map_copy_t              copy;
 125         vm_map_address_t           address;
 126
 127         copy = NULL;
 128         do {
 129                 err = kIOReturnNoMemory;
 130                 inst = new IOBufferMemoryDescriptor;
 131                 if (!inst) {
 132                         break;
 133                 }
 134                 inst->_ranges.v64 = IONew(IOAddressRange, 1);
 135                 if (!inst->_ranges.v64) {
 136                         break;
 137                 }
 138
 139                 err = vm_map_copyin(sourceMap, source, size,
 140                     false /* src_destroy */, &copy);
 141                 if (KERN_SUCCESS != err) {
 142                         break;
 143                 }
 144
 145                 err = vm_map_copyout(get_task_map(inTask), &address, copy);
 146                 if (KERN_SUCCESS != err) {
 147                         break;
 148                 }
 149                 copy = NULL;
 150
 151                 inst->_ranges.v64->address = address;
 152                 inst->_ranges.v64->length  = size;
 153
 154                 if (!inst->initWithPhysicalMask(inTask, options, size, page_size, 0)) {
 155                         err = kIOReturnError;
 156                 }
 157         } while (false);
 158
 159         if (KERN_SUCCESS == err) {
 160                 return inst;
 161         }
 162
 163         if (copy) {
 164                 vm_map_copy_discard(copy);
 165         }
 166         OSSafeReleaseNULL(inst);
 167         return NULL;
 168 }
 169
 170
 171 bool
 172 IOBufferMemoryDescriptor::initWithPhysicalMask(
 173         task_t            inTask,
 174         IOOptionBits      options,
 175         mach_vm_size_t    capacity,
 176         mach_vm_address_t alignment,
 177         mach_vm_address_t physicalMask)
 178 {
 179         task_t                mapTask = NULL;
 180         vm_map_t              vmmap = NULL;
 181         mach_vm_address_t     highestMask = 0;
 182         IOOptionBits          iomdOptions = kIOMemoryTypeVirtual64 | kIOMemoryAsReference;
 183         IODMAMapSpecification mapSpec;
 184         bool                  mapped = false;
 185         bool                  withCopy = false;
 186         bool                  needZero;
 187
 188         if (!capacity) {
 189                 return false;
 190         }
 191
 192         _options          = options;
 193         _capacity         = capacity;
 194         _internalFlags    = 0;
 195         _internalReserved = 0;
 196         _buffer           = NULL;
 197
 198         if (!_ranges.v64) {
 199                 _ranges.v64 = IONew(IOAddressRange, 1);
 200                 if (!_ranges.v64) {
 201                         return false;
 202                 }
 203                 _ranges.v64->address = 0;
 204                 _ranges.v64->length  = 0;
 205         } else {
 206                 if (!_ranges.v64->address) {
 207                         return false;
 208                 }
 209                 if (!(kIOMemoryPageable & options)) {
 210                         return false;
 211                 }
 212                 if (!inTask) {
 213                         return false;
 214                 }
 215                 _buffer = (void *) _ranges.v64->address;
 216                 withCopy = true;
 217         }
 218         //  make sure super::free doesn't dealloc _ranges before super::init
 219         _flags = kIOMemoryAsReference;
 220
 221         // Grab IOMD bits from the Buffer MD options
 222         iomdOptions  |= (options & kIOBufferDescriptorMemoryFlags);
 223
 224         if (!(kIOMemoryMapperNone & options)) {
 225                 IOMapper::checkForSystemMapper();
 226                 mapped = (NULL != IOMapper::gSystem);
 227         }
 228         needZero = (mapped || (0 != (kIOMemorySharingTypeMask & options)));
 229
 230         if (physicalMask && (alignment <= 1)) {
 231                 alignment   = ((physicalMask ^ (-1ULL)) & (physicalMask - 1));
 232                 highestMask = (physicalMask | alignment);
 233                 alignment++;
 234                 if (alignment < page_size) {
 235                         alignment = page_size;
 236                 }
 237         }
 238
 239         if ((options & (kIOMemorySharingTypeMask | kIOMapCacheMask | kIOMemoryClearEncrypt)) && (alignment < page_size)) {
 240                 alignment = page_size;
 241         }
 242
 243         if (alignment >= page_size) {
 244                 capacity = round_page(capacity);
 245         }
 246
 247         if (alignment > page_size) {
 248                 options |= kIOMemoryPhysicallyContiguous;
 249         }
 250
 251         _alignment = alignment;
 252
 253         if ((capacity + alignment) < _capacity) {
 254                 return false;
 255         }
 256
 257         if ((inTask != kernel_task) && !(options & kIOMemoryPageable)) {
 258                 return false;
 259         }
 260
 261         bzero(&mapSpec, sizeof(mapSpec));
 262         mapSpec.alignment      = _alignment;
 263         mapSpec.numAddressBits = 64;
 264         if (highestMask && mapped) {
 265                 if (highestMask <= 0xFFFFFFFF) {
 266                         mapSpec.numAddressBits = (32 - __builtin_clz((unsigned int) highestMask));
 267                 } else {
 268                         mapSpec.numAddressBits = (64 - __builtin_clz((unsigned int) (highestMask >> 32)));
 269                 }
 270                 highestMask = 0;
 271         }
 272
 273         // set memory entry cache mode, pageable, purgeable
 274         iomdOptions |= ((options & kIOMapCacheMask) >> kIOMapCacheShift) << kIOMemoryBufferCacheShift;
 275         if (options & kIOMemoryPageable) {
 276                 iomdOptions |= kIOMemoryBufferPageable;
 277                 if (options & kIOMemoryPurgeable) {
 278                         iomdOptions |= kIOMemoryBufferPurgeable;
 279                 }
 280         } else {
 281                 vmmap = kernel_map;
 282
 283                 // Buffer shouldn't auto prepare they should be prepared explicitly
 284                 // But it never was enforced so what are you going to do?
 285                 iomdOptions |= kIOMemoryAutoPrepare;
 286
 287                 /* Allocate a wired-down buffer inside kernel space. */
 288
 289                 bool contig = (0 != (options & kIOMemoryHostPhysicallyContiguous));
 290
 291                 if (!contig && (0 != (options & kIOMemoryPhysicallyContiguous))) {
 292                         contig |= (!mapped);
 293                         contig |= (0 != (kIOMemoryMapperNone & options));
 294 #if 0
 295                         // treat kIOMemoryPhysicallyContiguous as kIOMemoryHostPhysicallyContiguous for now
 296                         contig |= true;
 297 #endif
 298                 }
 299
 300                 if (contig || highestMask || (alignment > page_size)) {
 301                         _internalFlags |= kInternalFlagPhysical;
 302                         if (highestMask) {
 303                                 _internalFlags |= kInternalFlagPageSized;
 304                                 capacity = round_page(capacity);
 305                         }
 306                         _buffer = (void *) IOKernelAllocateWithPhysicalRestrict(
 307                                 capacity, highestMask, alignment, contig);
 308                 } else if (needZero
 309                     && ((capacity + alignment) <= (page_size - gIOPageAllocChunkBytes))) {
 310                         _internalFlags |= kInternalFlagPageAllocated;
 311                         needZero        = false;
 312                         _buffer         = (void *) iopa_alloc(&gIOBMDPageAllocator, &IOBMDPageProc, capacity, alignment);
 313                         if (_buffer) {
 314                                 IOStatisticsAlloc(kIOStatisticsMallocAligned, capacity);
 315 #if IOALLOCDEBUG
 316                                 OSAddAtomic(capacity, &debug_iomalloc_size);
 317 #endif
 318                         }
 319                 } else if (alignment > 1) {
 320                         _buffer = IOMallocAligned(capacity, alignment);
 321                 } else {
 322                         _buffer = IOMalloc(capacity);
 323                 }
 324                 if (!_buffer) {
 325                         return false;
 326                 }
 327                 if (needZero) {
 328                         bzero(_buffer, capacity);
 329                 }
 330         }
 331
 332         if ((options & (kIOMemoryPageable | kIOMapCacheMask))) {
 333                 vm_size_t       size = round_page(capacity);
 334
 335                 // initWithOptions will create memory entry
 336                 if (!withCopy) {
 337                         iomdOptions |= kIOMemoryPersistent;
 338                 }
 339
 340                 if (options & kIOMemoryPageable) {
 341 #if IOALLOCDEBUG
 342                         OSAddAtomicLong(size, &debug_iomallocpageable_size);
 343 #endif
 344                         if (!withCopy) {
 345                                 mapTask = inTask;
 346                         }
 347                         if (NULL == inTask) {
 348                                 inTask = kernel_task;
 349                         }
 350                 } else if (options & kIOMapCacheMask) {
 351                         // Prefetch each page to put entries into the pmap
 352                         volatile UInt8 *    startAddr = (UInt8 *)_buffer;
 353                         volatile UInt8 *    endAddr   = (UInt8 *)_buffer + capacity;
 354
 355                         while (startAddr < endAddr) {
 356                                 UInt8 dummyVar = *startAddr;
 357                                 (void) dummyVar;
 358                                 startAddr += page_size;
 359                         }
 360                 }
 361         }
 362
 363         _ranges.v64->address = (mach_vm_address_t) _buffer;
 364         _ranges.v64->length  = _capacity;
 365
 366         if (!super::initWithOptions(_ranges.v64, 1, 0,
 367             inTask, iomdOptions, /* System mapper */ NULL)) {
 368                 return false;
 369         }
 370
 371         _internalFlags |= kInternalFlagInit;
 372 #if IOTRACKING
 373         if (!(options & kIOMemoryPageable)) {
 374                 trackingAccumSize(capacity);
 375         }
 376 #endif /* IOTRACKING */
 377
 378         // give any system mapper the allocation params
 379         if (kIOReturnSuccess != dmaCommandOperation(kIOMDAddDMAMapSpec,
 380             &mapSpec, sizeof(mapSpec))) {
 381                 return false;
 382         }
 383
 384         if (mapTask) {
 385                 if (!reserved) {
 386                         reserved = IONew( ExpansionData, 1 );
 387                         if (!reserved) {
 388                                 return false;
 389                         }
 390                 }
 391                 reserved->map = createMappingInTask(mapTask, 0,
 392                     kIOMapAnywhere | (options & kIOMapPrefault) | (options & kIOMapCacheMask), 0, 0);
 393                 if (!reserved->map) {
 394                         _buffer = NULL;
 395                         return false;
 396                 }
 397                 release();  // map took a retain on this
 398                 reserved->map->retain();
 399                 removeMapping(reserved->map);
 400                 mach_vm_address_t buffer = reserved->map->getAddress();
 401                 _buffer = (void *) buffer;
 402                 if (kIOMemoryTypeVirtual64 == (kIOMemoryTypeMask & iomdOptions)) {
 403                         _ranges.v64->address = buffer;
 404                 }
 405         }
 406
 407         setLength(_capacity);
 408
 409         return true;
 410 }
 411
 412 IOBufferMemoryDescriptor *
 413 IOBufferMemoryDescriptor::inTaskWithOptions(
 414         task_t       inTask,
 415         IOOptionBits options,
 416         vm_size_t    capacity,
 417         vm_offset_t  alignment)
 418 {
 419         IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor;
 420
 421         if (me && !me->initWithPhysicalMask(inTask, options, capacity, alignment, 0)) {
 422                 me->release();
 423                 me = NULL;
 424         }
 425         return me;
 426 }
 427
 428 IOBufferMemoryDescriptor *
 429 IOBufferMemoryDescriptor::inTaskWithOptions(
 430         task_t       inTask,
 431         IOOptionBits options,
 432         vm_size_t    capacity,
 433         vm_offset_t  alignment,
 434         uint32_t     kernTag,
 435         uint32_t     userTag)
 436 {
 437         IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor;
 438
 439         if (me) {
 440                 me->setVMTags(kernTag, userTag);
 441
 442                 if (!me->initWithPhysicalMask(inTask, options, capacity, alignment, 0)) {
 443                         me->release();
 444                         me = NULL;
 445                 }
 446         }
 447         return me;
 448 }
 449
 450 IOBufferMemoryDescriptor *
 451 IOBufferMemoryDescriptor::inTaskWithPhysicalMask(
 452         task_t            inTask,
 453         IOOptionBits      options,
 454         mach_vm_size_t    capacity,
 455         mach_vm_address_t physicalMask)
 456 {
 457         IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor;
 458
 459         if (me && !me->initWithPhysicalMask(inTask, options, capacity, 1, physicalMask)) {
 460                 me->release();
 461                 me = NULL;
 462         }
 463         return me;
 464 }
 465
 466 #ifndef __LP64__
 467 bool
 468 IOBufferMemoryDescriptor::initWithOptions(
 469         IOOptionBits options,
 470         vm_size_t    capacity,
 471         vm_offset_t  alignment)
 472 {
 473         return initWithPhysicalMask(kernel_task, options, capacity, alignment, (mach_vm_address_t)0);
 474 }
 475 #endif /* !__LP64__ */
 476
 477 IOBufferMemoryDescriptor *
 478 IOBufferMemoryDescriptor::withOptions(
 479         IOOptionBits options,
 480         vm_size_t    capacity,
 481         vm_offset_t  alignment)
 482 {
 483         IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor;
 484
 485         if (me && !me->initWithPhysicalMask(kernel_task, options, capacity, alignment, 0)) {
 486                 me->release();
 487                 me = NULL;
 488         }
 489         return me;
 490 }
 491
 492
 493 /*
 494  * withCapacity:
 495  *
 496  * Returns a new IOBufferMemoryDescriptor with a buffer large enough to
 497  * hold capacity bytes.  The descriptor's length is initially set to the capacity.
 498  */
 499 IOBufferMemoryDescriptor *
 500 IOBufferMemoryDescriptor::withCapacity(vm_size_t   inCapacity,
 501     IODirection inDirection,
 502     bool        inContiguous)
 503 {
 504         return IOBufferMemoryDescriptor::withOptions(
 505                 inDirection | kIOMemoryUnshared
 506                 | (inContiguous ? kIOMemoryPhysicallyContiguous : 0),
 507                 inCapacity, inContiguous ? inCapacity : 1 );
 508 }
 509
 510 #ifndef __LP64__
 511 /*
 512  * initWithBytes:
 513  *
 514  * Initialize a new IOBufferMemoryDescriptor preloaded with bytes (copied).
 515  * The descriptor's length and capacity are set to the input buffer's size.
 516  */
 517 bool
 518 IOBufferMemoryDescriptor::initWithBytes(const void * inBytes,
 519     vm_size_t    inLength,
 520     IODirection  inDirection,
 521     bool         inContiguous)
 522 {
 523         if (!initWithPhysicalMask(kernel_task, inDirection | kIOMemoryUnshared
 524             | (inContiguous ? kIOMemoryPhysicallyContiguous : 0),
 525             inLength, inLength, (mach_vm_address_t)0)) {
 526                 return false;
 527         }
 528
 529         // start out with no data
 530         setLength(0);
 531
 532         if (!appendBytes(inBytes, inLength)) {
 533                 return false;
 534         }
 535
 536         return true;
 537 }
 538 #endif /* !__LP64__ */
 539
 540 /*
 541  * withBytes:
 542  *
 543  * Returns a new IOBufferMemoryDescriptor preloaded with bytes (copied).
 544  * The descriptor's length and capacity are set to the input buffer's size.
 545  */
 546 IOBufferMemoryDescriptor *
 547 IOBufferMemoryDescriptor::withBytes(const void * inBytes,
 548     vm_size_t    inLength,
 549     IODirection  inDirection,
 550     bool         inContiguous)
 551 {
 552         IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor;
 553
 554         if (me && !me->initWithPhysicalMask(
 555                     kernel_task, inDirection | kIOMemoryUnshared
 556                     | (inContiguous ? kIOMemoryPhysicallyContiguous : 0),
 557                     inLength, inLength, 0 )) {
 558                 me->release();
 559                 me = NULL;
 560         }
 561
 562         if (me) {
 563                 // start out with no data
 564                 me->setLength(0);
 565
 566                 if (!me->appendBytes(inBytes, inLength)) {
 567                         me->release();
 568                         me = NULL;
 569                 }
 570         }
 571         return me;
 572 }
 573
 574 /*
 575  * free:
 576  *
 577  * Free resources
 578  */
 579 void
 580 IOBufferMemoryDescriptor::free()
 581 {
 582         // Cache all of the relevant information on the stack for use
 583         // after we call super::free()!
 584         IOOptionBits     flags         = _flags;
 585         IOOptionBits     internalFlags = _internalFlags;
 586         IOOptionBits     options   = _options;
 587         vm_size_t        size      = _capacity;
 588         void *           buffer    = _buffer;
 589         IOMemoryMap *    map       = NULL;
 590         IOAddressRange * range     = _ranges.v64;
 591         vm_offset_t      alignment = _alignment;
 592
 593         if (alignment >= page_size) {
 594                 size = round_page(size);
 595         }
 596
 597         if (reserved) {
 598                 map = reserved->map;
 599                 IODelete( reserved, ExpansionData, 1 );
 600                 if (map) {
 601                         map->release();
 602                 }
 603         }
 604
 605         if ((options & kIOMemoryPageable)
 606             || (kInternalFlagPageSized & internalFlags)) {
 607                 size = round_page(size);
 608         }
 609
 610 #if IOTRACKING
 611         if (!(options & kIOMemoryPageable)
 612             && buffer
 613             && (kInternalFlagInit & _internalFlags)) {
 614                 trackingAccumSize(-size);
 615         }
 616 #endif /* IOTRACKING */
 617
 618         /* super::free may unwire - deallocate buffer afterwards */
 619         super::free();
 620
 621         if (options & kIOMemoryPageable) {
 622 #if IOALLOCDEBUG
 623                 OSAddAtomicLong(-size, &debug_iomallocpageable_size);
 624 #endif
 625         } else if (buffer) {
 626                 if (kInternalFlagPhysical & internalFlags) {
 627                         IOKernelFreePhysical((mach_vm_address_t) buffer, size);
 628                 } else if (kInternalFlagPageAllocated & internalFlags) {
 629                         uintptr_t page;
 630                         page = iopa_free(&gIOBMDPageAllocator, (uintptr_t) buffer, size);
 631                         if (page) {
 632                                 kmem_free(kernel_map, page, page_size);
 633                         }
 634 #if IOALLOCDEBUG
 635                         OSAddAtomic(-size, &debug_iomalloc_size);
 636 #endif
 637                         IOStatisticsAlloc(kIOStatisticsFreeAligned, size);
 638                 } else if (alignment > 1) {
 639                         IOFreeAligned(buffer, size);
 640                 } else {
 641                         IOFree(buffer, size);
 642                 }
 643         }
 644         if (range && (kIOMemoryAsReference & flags)) {
 645                 IODelete(range, IOAddressRange, 1);
 646         }
 647 }
 648
 649 /*
 650  * getCapacity:
 651  *
 652  * Get the buffer capacity
 653  */
 654 vm_size_t
 655 IOBufferMemoryDescriptor::getCapacity() const
 656 {
 657         return _capacity;
 658 }
 659
 660 /*
 661  * setLength:
 662  *
 663  * Change the buffer length of the memory descriptor.  When a new buffer
 664  * is created, the initial length of the buffer is set to be the same as
 665  * the capacity.  The length can be adjusted via setLength for a shorter
 666  * transfer (there is no need to create more buffer descriptors when you
 667  * can reuse an existing one, even for different transfer sizes).   Note
 668  * that the specified length must not exceed the capacity of the buffer.
 669  */
 670 void
 671 IOBufferMemoryDescriptor::setLength(vm_size_t length)
 672 {
 673         assert(length <= _capacity);
 674         if (length > _capacity) {
 675                 return;
 676         }
 677
 678         _length = length;
 679         _ranges.v64->length = length;
 680 }
 681
 682 /*
 683  * setDirection:
 684  *
 685  * Change the direction of the transfer.  This method allows one to redirect
 686  * the descriptor's transfer direction.  This eliminates the need to destroy
 687  * and create new buffers when different transfer directions are needed.
 688  */
 689 void
 690 IOBufferMemoryDescriptor::setDirection(IODirection direction)
 691 {
 692         _flags = (_flags & ~kIOMemoryDirectionMask) | direction;
 693 #ifndef __LP64__
 694         _direction = (IODirection) (_flags & kIOMemoryDirectionMask);
 695 #endif /* !__LP64__ */
 696 }
 697
 698 /*
 699  * appendBytes:
 700  *
 701  * Add some data to the end of the buffer.  This method automatically
 702  * maintains the memory descriptor buffer length.  Note that appendBytes
 703  * will not copy past the end of the memory descriptor's current capacity.
 704  */
 705 bool
 706 IOBufferMemoryDescriptor::appendBytes(const void * bytes, vm_size_t withLength)
 707 {
 708         vm_size_t   actualBytesToCopy = min(withLength, _capacity - _length);
 709         IOByteCount offset;
 710
 711         assert(_length <= _capacity);
 712
 713         offset = _length;
 714         _length += actualBytesToCopy;
 715         _ranges.v64->length += actualBytesToCopy;
 716
 717         if (_task == kernel_task) {
 718                 bcopy(/* from */ bytes, (void *)(_ranges.v64->address + offset),
 719                     actualBytesToCopy);
 720         } else {
 721                 writeBytes(offset, bytes, actualBytesToCopy);
 722         }
 723
 724         return true;
 725 }
 726
 727 /*
 728  * getBytesNoCopy:
 729  *
 730  * Return the virtual address of the beginning of the buffer
 731  */
 732 void *
 733 IOBufferMemoryDescriptor::getBytesNoCopy()
 734 {
 735         if (kIOMemoryTypePhysical64 == (_flags & kIOMemoryTypeMask)) {
 736                 return _buffer;
 737         } else {
 738                 return (void *)_ranges.v64->address;
 739         }
 740 }
 741
 742
 743 /*
 744  * getBytesNoCopy:
 745  *
 746  * Return the virtual address of an offset from the beginning of the buffer
 747  */
 748 void *
 749 IOBufferMemoryDescriptor::getBytesNoCopy(vm_size_t start, vm_size_t withLength)
 750 {
 751         IOVirtualAddress address;
 752
 753         if ((start + withLength) < start) {
 754                 return NULL;
 755         }
 756
 757         if (kIOMemoryTypePhysical64 == (_flags & kIOMemoryTypeMask)) {
 758                 address = (IOVirtualAddress) _buffer;
 759         } else {
 760                 address = _ranges.v64->address;
 761         }
 762
 763         if (start < _length && (start + withLength) <= _length) {
 764                 return (void *)(address + start);
 765         }
 766         return NULL;
 767 }
 768
 769 #ifndef __LP64__
 770 void *
 771 IOBufferMemoryDescriptor::getVirtualSegment(IOByteCount offset,
 772     IOByteCount * lengthOfSegment)
 773 {
 774         void * bytes = getBytesNoCopy(offset, 0);
 775
 776         if (bytes && lengthOfSegment) {
 777                 *lengthOfSegment = _length - offset;
 778         }
 779
 780         return bytes;
 781 }
 782 #endif /* !__LP64__ */
 783
 784 #ifdef __LP64__
 785 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 0);
 786 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 1);
 787 #else /* !__LP64__ */
 788 OSMetaClassDefineReservedUsed(IOBufferMemoryDescriptor, 0);
 789 OSMetaClassDefineReservedUsed(IOBufferMemoryDescriptor, 1);
 790 #endif /* !__LP64__ */
 791 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 2);
 792 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 3);
 793 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 4);
 794 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 5);
 795 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 6);
 796 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 7);
 797 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 8);
 798 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 9);
 799 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 10);
 800 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 11);
 801 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 12);
 802 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 13);
 803 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 14);
 804 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 15);