iokit/Kernel/IOBufferMemoryDescriptor.cpp

   1 /*
   2  * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #define _IOMEMORYDESCRIPTOR_INTERNAL_
  30
  31 #include <IOKit/assert.h>
  32 #include <IOKit/system.h>
  33
  34 #include <IOKit/IOLib.h>
  35 #include <IOKit/IOMapper.h>
  36 #include <IOKit/IOBufferMemoryDescriptor.h>
  37 #include <libkern/OSDebug.h>
  38
  39 #include "IOKitKernelInternal.h"
  40
  41 __BEGIN_DECLS
  42 void ipc_port_release_send(ipc_port_t port);
  43 #include <vm/pmap.h>
  44
  45 __END_DECLS
  46
  47 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  48
  49 enum
  50 {
  51     kInternalFlagPhysical  = 0x00000001,
  52     kInternalFlagPageSized = 0x00000002
  53 };
  54
  55 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  56
  57 #define super IOGeneralMemoryDescriptor
  58 OSDefineMetaClassAndStructors(IOBufferMemoryDescriptor,
  59                                 IOGeneralMemoryDescriptor);
  60
  61 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
  62
  63 #ifndef __LP64__
  64 bool IOBufferMemoryDescriptor::initWithOptions(
  65                                IOOptionBits options,
  66                                vm_size_t    capacity,
  67                                vm_offset_t  alignment,
  68                                task_t       inTask)
  69 {
  70     mach_vm_address_t physicalMask = 0;
  71     return (initWithPhysicalMask(inTask, options, capacity, alignment, physicalMask));
  72 }
  73 #endif /* !__LP64__ */
  74
  75 bool IOBufferMemoryDescriptor::initWithPhysicalMask(
  76                                 task_t            inTask,
  77                                 IOOptionBits      options,
  78                                 mach_vm_size_t    capacity,
  79                                 mach_vm_address_t alignment,
  80                                 mach_vm_address_t physicalMask)
  81 {
  82     kern_return_t       kr;
  83     task_t              mapTask = NULL;
  84     vm_map_t            vmmap = NULL;
  85     mach_vm_address_t   highestMask = 0;
  86     IOOptionBits        iomdOptions = kIOMemoryTypeVirtual64 | kIOMemoryAsReference;
  87
  88     if (!capacity)
  89         return false;
  90
  91     _options          = options;
  92     _capacity         = capacity;
  93     _internalFlags    = 0;
  94     _internalReserved = 0;
  95     _buffer           = 0;
  96
  97     _ranges.v64 = IONew(IOAddressRange, 1);
  98     if (!_ranges.v64)
  99         return (false);
 100     _ranges.v64->address = 0;
 101     _ranges.v64->length  = 0;
 102         //  make sure super::free doesn't dealloc _ranges before super::init
 103         _flags = kIOMemoryAsReference;
 104
 105     // Grab IOMD bits from the Buffer MD options
 106     iomdOptions  |= (options & kIOBufferDescriptorMemoryFlags);
 107
 108     if (physicalMask && (alignment <= 1))
 109     {
 110         alignment   = ((physicalMask ^ (-1ULL)) & (physicalMask - 1));
 111         highestMask = (physicalMask | alignment);
 112         alignment++;
 113         if (alignment < page_size)
 114             alignment = page_size;
 115     }
 116
 117     if ((options & (kIOMemorySharingTypeMask | kIOMapCacheMask | kIOMemoryClearEncrypt)) && (alignment < page_size))
 118         alignment = page_size;
 119
 120     if (alignment >= page_size)
 121         capacity = round_page(capacity);
 122
 123     if (alignment > page_size)
 124         options |= kIOMemoryPhysicallyContiguous;
 125
 126     _alignment = alignment;
 127
 128     if ((inTask != kernel_task) && !(options & kIOMemoryPageable))
 129         return false;
 130
 131     // set flags for entry + object create
 132     vm_prot_t memEntryCacheMode = VM_PROT_READ | VM_PROT_WRITE;
 133
 134     // set memory entry cache mode
 135     switch (options & kIOMapCacheMask)
 136     {
 137         case kIOMapInhibitCache:
 138             SET_MAP_MEM(MAP_MEM_IO, memEntryCacheMode);
 139             break;
 140
 141         case kIOMapWriteThruCache:
 142             SET_MAP_MEM(MAP_MEM_WTHRU, memEntryCacheMode);
 143             break;
 144
 145         case kIOMapWriteCombineCache:
 146             SET_MAP_MEM(MAP_MEM_WCOMB, memEntryCacheMode);
 147             break;
 148
 149         case kIOMapCopybackCache:
 150             SET_MAP_MEM(MAP_MEM_COPYBACK, memEntryCacheMode);
 151             break;
 152
 153         case kIOMapCopybackInnerCache:
 154             SET_MAP_MEM(MAP_MEM_INNERWBACK, memEntryCacheMode);
 155             break;
 156
 157         case kIOMapDefaultCache:
 158         default:
 159             SET_MAP_MEM(MAP_MEM_NOOP, memEntryCacheMode);
 160             break;
 161     }
 162
 163     if (options & kIOMemoryPageable)
 164     {
 165         iomdOptions |= kIOMemoryBufferPageable;
 166
 167         // must create the entry before any pages are allocated
 168
 169         // set flags for entry + object create
 170         memEntryCacheMode |= MAP_MEM_NAMED_CREATE;
 171
 172         if (options & kIOMemoryPurgeable)
 173             memEntryCacheMode |= MAP_MEM_PURGABLE;
 174     }
 175     else
 176     {
 177         memEntryCacheMode |= MAP_MEM_NAMED_REUSE;
 178         vmmap = kernel_map;
 179
 180         // Buffer shouldn't auto prepare they should be prepared explicitly
 181         // But it never was enforced so what are you going to do?
 182         iomdOptions |= kIOMemoryAutoPrepare;
 183
 184         /* Allocate a wired-down buffer inside kernel space. */
 185
 186         if ((options & kIOMemoryPhysicallyContiguous) || highestMask || (alignment > page_size))
 187         {
 188             _internalFlags |= kInternalFlagPhysical;
 189             if (highestMask)
 190             {
 191                 _internalFlags |= kInternalFlagPageSized;
 192                 capacity = round_page(capacity);
 193             }
 194             _buffer = (void *) IOKernelAllocateWithPhysicalRestrict(capacity, highestMask, alignment,
 195                                         (0 != (options & kIOMemoryPhysicallyContiguous)));
 196         }
 197         else if (alignment > 1)
 198         {
 199             _buffer = IOMallocAligned(capacity, alignment);
 200         }
 201         else
 202         {
 203             _buffer = IOMalloc(capacity);
 204         }
 205
 206         if (!_buffer)
 207         {
 208             return false;
 209         }
 210     }
 211
 212     if( (options & (kIOMemoryPageable | kIOMapCacheMask))) {
 213         ipc_port_t      sharedMem;
 214         vm_size_t       size = round_page(capacity);
 215
 216         kr = mach_make_memory_entry(vmmap,
 217                                     &size, (vm_offset_t)_buffer,
 218                                     memEntryCacheMode, &sharedMem,
 219                                     NULL );
 220
 221         if( (KERN_SUCCESS == kr) && (size != round_page(capacity))) {
 222             ipc_port_release_send( sharedMem );
 223             kr = kIOReturnVMError;
 224         }
 225         if( KERN_SUCCESS != kr)
 226             return( false );
 227
 228         _memEntry = (void *) sharedMem;
 229
 230         if( options & kIOMemoryPageable) {
 231 #if IOALLOCDEBUG
 232             debug_iomallocpageable_size += size;
 233 #endif
 234             mapTask = inTask;
 235             if (NULL == inTask)
 236                 inTask = kernel_task;
 237         }
 238         else if (options & kIOMapCacheMask)
 239         {
 240             // Prefetch each page to put entries into the pmap
 241             volatile UInt8 *    startAddr = (UInt8 *)_buffer;
 242             volatile UInt8 *    endAddr   = (UInt8 *)_buffer + capacity;
 243
 244             while (startAddr < endAddr)
 245             {
 246                 *startAddr;
 247                 startAddr += page_size;
 248             }
 249         }
 250     }
 251
 252     _ranges.v64->address = (mach_vm_address_t) _buffer;;
 253     _ranges.v64->length  = _capacity;
 254
 255     if (!super::initWithOptions(_ranges.v64, 1, 0,
 256                                 inTask, iomdOptions, /* System mapper */ 0))
 257         return false;
 258
 259     if (mapTask)
 260     {
 261         if (!reserved) {
 262             reserved = IONew( ExpansionData, 1 );
 263             if( !reserved)
 264                 return( false );
 265         }
 266         reserved->map = createMappingInTask(mapTask, 0,
 267                             kIOMapAnywhere | (options & kIOMapCacheMask), 0, 0);
 268         if (!reserved->map)
 269         {
 270             _buffer = 0;
 271             return( false );
 272         }
 273         release();          // map took a retain on this
 274         reserved->map->retain();
 275         removeMapping(reserved->map);
 276         mach_vm_address_t buffer = reserved->map->getAddress();
 277         _buffer = (void *) buffer;
 278         if (kIOMemoryTypeVirtual64 == (kIOMemoryTypeMask & iomdOptions))
 279             _ranges.v64->address = buffer;
 280     }
 281
 282     setLength(_capacity);
 283
 284     return true;
 285 }
 286
 287 IOBufferMemoryDescriptor * IOBufferMemoryDescriptor::inTaskWithOptions(
 288                                             task_t       inTask,
 289                                             IOOptionBits options,
 290                                             vm_size_t    capacity,
 291                                             vm_offset_t  alignment)
 292 {
 293     IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor;
 294
 295     if (me && !me->initWithPhysicalMask(inTask, options, capacity, alignment, 0)) {
 296         me->release();
 297         me = 0;
 298     }
 299     return me;
 300 }
 301
 302 IOBufferMemoryDescriptor * IOBufferMemoryDescriptor::inTaskWithPhysicalMask(
 303                                             task_t            inTask,
 304                                             IOOptionBits      options,
 305                                             mach_vm_size_t    capacity,
 306                                             mach_vm_address_t physicalMask)
 307 {
 308     IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor;
 309
 310     if (me && !me->initWithPhysicalMask(inTask, options, capacity, 1, physicalMask))
 311     {
 312         me->release();
 313         me = 0;
 314     }
 315     return me;
 316 }
 317
 318 #ifndef __LP64__
 319 bool IOBufferMemoryDescriptor::initWithOptions(
 320                                IOOptionBits options,
 321                                vm_size_t    capacity,
 322                                vm_offset_t  alignment)
 323 {
 324     return (initWithPhysicalMask(kernel_task, options, capacity, alignment, (mach_vm_address_t)0));
 325 }
 326 #endif /* !__LP64__ */
 327
 328 IOBufferMemoryDescriptor * IOBufferMemoryDescriptor::withOptions(
 329                                             IOOptionBits options,
 330                                             vm_size_t    capacity,
 331                                             vm_offset_t  alignment)
 332 {
 333     IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor;
 334
 335     if (me && !me->initWithPhysicalMask(kernel_task, options, capacity, alignment, 0)) {
 336         me->release();
 337         me = 0;
 338     }
 339     return me;
 340 }
 341
 342
 343 /*
 344  * withCapacity:
 345  *
 346  * Returns a new IOBufferMemoryDescriptor with a buffer large enough to
 347  * hold capacity bytes.  The descriptor's length is initially set to the capacity.
 348  */
 349 IOBufferMemoryDescriptor *
 350 IOBufferMemoryDescriptor::withCapacity(vm_size_t   inCapacity,
 351                                        IODirection inDirection,
 352                                        bool        inContiguous)
 353 {
 354     return( IOBufferMemoryDescriptor::withOptions(
 355                inDirection | kIOMemoryUnshared
 356                 | (inContiguous ? kIOMemoryPhysicallyContiguous : 0),
 357                inCapacity, inContiguous ? inCapacity : 1 ));
 358 }
 359
 360 #ifndef __LP64__
 361 /*
 362  * initWithBytes:
 363  *
 364  * Initialize a new IOBufferMemoryDescriptor preloaded with bytes (copied).
 365  * The descriptor's length and capacity are set to the input buffer's size.
 366  */
 367 bool IOBufferMemoryDescriptor::initWithBytes(const void * inBytes,
 368                                              vm_size_t    inLength,
 369                                              IODirection  inDirection,
 370                                              bool         inContiguous)
 371 {
 372     if (!initWithPhysicalMask(kernel_task, inDirection | kIOMemoryUnshared
 373                               | (inContiguous ? kIOMemoryPhysicallyContiguous : 0),
 374                               inLength, inLength, (mach_vm_address_t)0))
 375         return false;
 376
 377     // start out with no data
 378     setLength(0);
 379
 380     if (!appendBytes(inBytes, inLength))
 381         return false;
 382
 383     return true;
 384 }
 385 #endif /* !__LP64__ */
 386
 387 /*
 388  * withBytes:
 389  *
 390  * Returns a new IOBufferMemoryDescriptor preloaded with bytes (copied).
 391  * The descriptor's length and capacity are set to the input buffer's size.
 392  */
 393 IOBufferMemoryDescriptor *
 394 IOBufferMemoryDescriptor::withBytes(const void * inBytes,
 395                                     vm_size_t    inLength,
 396                                     IODirection  inDirection,
 397                                     bool         inContiguous)
 398 {
 399     IOBufferMemoryDescriptor *me = new IOBufferMemoryDescriptor;
 400
 401     if (me && !me->initWithPhysicalMask(
 402                kernel_task, inDirection | kIOMemoryUnshared
 403                 | (inContiguous ? kIOMemoryPhysicallyContiguous : 0),
 404                inLength, inLength, 0 ))
 405     {
 406         me->release();
 407         me = 0;
 408     }
 409
 410     if (me)
 411     {
 412         // start out with no data
 413         me->setLength(0);
 414
 415         if (!me->appendBytes(inBytes, inLength))
 416         {
 417             me->release();
 418             me = 0;
 419         }
 420     }
 421     return me;
 422 }
 423
 424 /*
 425  * free:
 426  *
 427  * Free resources
 428  */
 429 void IOBufferMemoryDescriptor::free()
 430 {
 431     // Cache all of the relevant information on the stack for use
 432     // after we call super::free()!
 433     IOOptionBits     flags         = _flags;
 434     IOOptionBits     internalFlags = _internalFlags;
 435     IOOptionBits     options   = _options;
 436     vm_size_t        size      = _capacity;
 437     void *           buffer    = _buffer;
 438     IOMemoryMap *    map       = 0;
 439     IOAddressRange * range     = _ranges.v64;
 440     vm_offset_t      alignment = _alignment;
 441
 442     if (alignment >= page_size)
 443         size = round_page(size);
 444
 445     if (reserved)
 446     {
 447         map = reserved->map;
 448         IODelete( reserved, ExpansionData, 1 );
 449         if (map)
 450             map->release();
 451     }
 452
 453     /* super::free may unwire - deallocate buffer afterwards */
 454     super::free();
 455
 456     if (options & kIOMemoryPageable)
 457     {
 458 #if IOALLOCDEBUG
 459         debug_iomallocpageable_size -= round_page(size);
 460 #endif
 461     }
 462     else if (buffer)
 463     {
 464         if (internalFlags & kInternalFlagPhysical)
 465         {
 466             if (kInternalFlagPageSized & internalFlags)
 467                 size = round_page(size);
 468             IOKernelFreePhysical((mach_vm_address_t) buffer, size);
 469         }
 470         else if (alignment > 1)
 471             IOFreeAligned(buffer, size);
 472         else
 473             IOFree(buffer, size);
 474     }
 475     if (range && (kIOMemoryAsReference & flags))
 476         IODelete(range, IOAddressRange, 1);
 477 }
 478
 479 /*
 480  * getCapacity:
 481  *
 482  * Get the buffer capacity
 483  */
 484 vm_size_t IOBufferMemoryDescriptor::getCapacity() const
 485 {
 486     return _capacity;
 487 }
 488
 489 /*
 490  * setLength:
 491  *
 492  * Change the buffer length of the memory descriptor.  When a new buffer
 493  * is created, the initial length of the buffer is set to be the same as
 494  * the capacity.  The length can be adjusted via setLength for a shorter
 495  * transfer (there is no need to create more buffer descriptors when you
 496  * can reuse an existing one, even for different transfer sizes).   Note
 497  * that the specified length must not exceed the capacity of the buffer.
 498  */
 499 void IOBufferMemoryDescriptor::setLength(vm_size_t length)
 500 {
 501     assert(length <= _capacity);
 502
 503     _length = length;
 504     _ranges.v64->length = length;
 505 }
 506
 507 /*
 508  * setDirection:
 509  *
 510  * Change the direction of the transfer.  This method allows one to redirect
 511  * the descriptor's transfer direction.  This eliminates the need to destroy
 512  * and create new buffers when different transfer directions are needed.
 513  */
 514 void IOBufferMemoryDescriptor::setDirection(IODirection direction)
 515 {
 516     _flags = (_flags & ~kIOMemoryDirectionMask) | direction;
 517 #ifndef __LP64__
 518     _direction = (IODirection) (_flags & kIOMemoryDirectionMask);
 519 #endif /* !__LP64__ */
 520 }
 521
 522 /*
 523  * appendBytes:
 524  *
 525  * Add some data to the end of the buffer.  This method automatically
 526  * maintains the memory descriptor buffer length.  Note that appendBytes
 527  * will not copy past the end of the memory descriptor's current capacity.
 528  */
 529 bool
 530 IOBufferMemoryDescriptor::appendBytes(const void * bytes, vm_size_t withLength)
 531 {
 532     vm_size_t   actualBytesToCopy = min(withLength, _capacity - _length);
 533     IOByteCount offset;
 534
 535     assert(_length <= _capacity);
 536
 537     offset = _length;
 538     _length += actualBytesToCopy;
 539     _ranges.v64->length += actualBytesToCopy;
 540
 541     if (_task == kernel_task)
 542         bcopy(/* from */ bytes, (void *)(_ranges.v64->address + offset),
 543               actualBytesToCopy);
 544     else
 545         writeBytes(offset, bytes, actualBytesToCopy);
 546
 547     return true;
 548 }
 549
 550 /*
 551  * getBytesNoCopy:
 552  *
 553  * Return the virtual address of the beginning of the buffer
 554  */
 555 void * IOBufferMemoryDescriptor::getBytesNoCopy()
 556 {
 557     if (kIOMemoryTypePhysical64 == (_flags & kIOMemoryTypeMask))
 558         return _buffer;
 559     else
 560         return (void *)_ranges.v64->address;
 561 }
 562
 563
 564 /*
 565  * getBytesNoCopy:
 566  *
 567  * Return the virtual address of an offset from the beginning of the buffer
 568  */
 569 void *
 570 IOBufferMemoryDescriptor::getBytesNoCopy(vm_size_t start, vm_size_t withLength)
 571 {
 572     IOVirtualAddress address;
 573     if (kIOMemoryTypePhysical64 == (_flags & kIOMemoryTypeMask))
 574         address = (IOVirtualAddress) _buffer;
 575     else
 576         address = _ranges.v64->address;
 577
 578    if (start < _length && (start + withLength) <= _length)
 579         return (void *)(address + start);
 580     return 0;
 581 }
 582
 583 #ifndef __LP64__
 584 void * IOBufferMemoryDescriptor::getVirtualSegment(IOByteCount offset,
 585                                                         IOByteCount * lengthOfSegment)
 586 {
 587     void * bytes = getBytesNoCopy(offset, 0);
 588
 589     if (bytes && lengthOfSegment)
 590         *lengthOfSegment = _length - offset;
 591
 592     return bytes;
 593 }
 594 #endif /* !__LP64__ */
 595
 596 #ifdef __LP64__
 597 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 0);
 598 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 1);
 599 #else /* !__LP64__ */
 600 OSMetaClassDefineReservedUsed(IOBufferMemoryDescriptor, 0);
 601 OSMetaClassDefineReservedUsed(IOBufferMemoryDescriptor, 1);
 602 #endif /* !__LP64__ */
 603 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 2);
 604 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 3);
 605 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 4);
 606 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 5);
 607 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 6);
 608 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 7);
 609 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 8);
 610 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 9);
 611 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 10);
 612 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 11);
 613 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 12);
 614 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 13);
 615 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 14);
 616 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor, 15);