2 * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
28 #include <IOKit/assert.h>
29 #include <IOKit/system.h>
31 #include <IOKit/IOLib.h>
32 #include <IOKit/IOMapper.h>
33 #include <IOKit/IOBufferMemoryDescriptor.h>
34 #include <libkern/OSDebug.h>
36 #include "IOKitKernelInternal.h"
37 #include "IOCopyMapper.h"
40 void ipc_port_release_send(ipc_port_t port
);
43 vm_map_t
IOPageableMapForAddress( vm_address_t address
);
46 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
48 volatile ppnum_t gIOHighestAllocatedPage
;
50 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
52 #define super IOGeneralMemoryDescriptor
53 OSDefineMetaClassAndStructors(IOBufferMemoryDescriptor
,
54 IOGeneralMemoryDescriptor
);
56 bool IOBufferMemoryDescriptor::initWithAddress(
57 void * /* address */ ,
58 IOByteCount
/* withLength */ ,
59 IODirection
/* withDirection */ )
64 bool IOBufferMemoryDescriptor::initWithAddress(
65 vm_address_t
/* address */ ,
66 IOByteCount
/* withLength */ ,
67 IODirection
/* withDirection */ ,
68 task_t
/* withTask */ )
73 bool IOBufferMemoryDescriptor::initWithPhysicalAddress(
74 IOPhysicalAddress
/* address */ ,
75 IOByteCount
/* withLength */ ,
76 IODirection
/* withDirection */ )
81 bool IOBufferMemoryDescriptor::initWithPhysicalRanges(
82 IOPhysicalRange
* /* ranges */ ,
83 UInt32
/* withCount */ ,
84 IODirection
/* withDirection */ ,
85 bool /* asReference */ )
90 bool IOBufferMemoryDescriptor::initWithRanges(
91 IOVirtualRange
* /* ranges */ ,
92 UInt32
/* withCount */ ,
93 IODirection
/* withDirection */ ,
94 task_t
/* withTask */ ,
95 bool /* asReference */ )
100 bool IOBufferMemoryDescriptor::initWithOptions(
101 IOOptionBits options
,
103 vm_offset_t alignment
,
106 mach_vm_address_t physicalMask
= 0;
107 return (initWithPhysicalMask(inTask
, options
, capacity
, alignment
, physicalMask
));
110 bool IOBufferMemoryDescriptor::initWithPhysicalMask(
112 IOOptionBits options
,
113 mach_vm_size_t capacity
,
114 mach_vm_address_t alignment
,
115 mach_vm_address_t physicalMask
)
118 task_t mapTask
= NULL
;
119 vm_map_t vmmap
= NULL
;
121 IOAddressRange range
;
122 IOOptionBits iomdOptions
= kIOMemoryTypeVirtual64
;
128 _capacity
= capacity
;
134 _ranges
.v64
= &range
;
136 // Grab IOMD bits from the Buffer MD options
137 iomdOptions
|= (options
& kIOBufferDescriptorMemoryFlags
);
139 if ((options
& (kIOMemorySharingTypeMask
| kIOMapCacheMask
)) && (alignment
< page_size
))
140 alignment
= page_size
;
142 if (physicalMask
&& (alignment
<= 1))
143 alignment
= ((physicalMask
^ PAGE_MASK
) & PAGE_MASK
) + 1;
145 _alignment
= alignment
;
147 if (((inTask
!= kernel_task
) && !(options
& kIOMemoryPageable
)) ||
148 (physicalMask
&& (options
& kIOMapCacheMask
)))
151 if ((options
& kIOMemoryPhysicallyContiguous
) && !physicalMask
)
152 physicalMask
= 0xFFFFFFFF;
154 // set flags for entry + object create
155 vm_prot_t memEntryCacheMode
= VM_PROT_READ
| VM_PROT_WRITE
;
157 // set memory entry cache mode
158 switch (options
& kIOMapCacheMask
)
160 case kIOMapInhibitCache
:
161 SET_MAP_MEM(MAP_MEM_IO
, memEntryCacheMode
);
164 case kIOMapWriteThruCache
:
165 SET_MAP_MEM(MAP_MEM_WTHRU
, memEntryCacheMode
);
168 case kIOMapWriteCombineCache
:
169 SET_MAP_MEM(MAP_MEM_WCOMB
, memEntryCacheMode
);
172 case kIOMapCopybackCache
:
173 SET_MAP_MEM(MAP_MEM_COPYBACK
, memEntryCacheMode
);
176 case kIOMapDefaultCache
:
178 SET_MAP_MEM(MAP_MEM_NOOP
, memEntryCacheMode
);
182 if (options
& kIOMemoryPageable
)
184 iomdOptions
|= kIOMemoryBufferPageable
;
186 // must create the entry before any pages are allocated
188 // set flags for entry + object create
189 memEntryCacheMode
|= MAP_MEM_NAMED_CREATE
;
191 if (options
& kIOMemoryPurgeable
)
192 memEntryCacheMode
|= MAP_MEM_PURGABLE
;
196 memEntryCacheMode
|= MAP_MEM_NAMED_REUSE
;
198 if (IOMapper::gSystem
)
199 // assuming mapped space is 2G
200 lastIOAddr
= (1UL << 31) - PAGE_SIZE
;
202 lastIOAddr
= ptoa_64(gIOHighestAllocatedPage
);
204 if (physicalMask
&& (lastIOAddr
!= (lastIOAddr
& physicalMask
)))
206 mach_vm_address_t address
;
207 iomdOptions
&= ~kIOMemoryTypeVirtual64
;
208 iomdOptions
|= kIOMemoryTypePhysical64
;
210 address
= IOMallocPhysical(capacity
, physicalMask
);
211 _buffer
= (void *) address
;
222 // Buffer shouldn't auto prepare they should be prepared explicitly
223 // But it never was enforced so what are you going to do?
224 iomdOptions
|= kIOMemoryAutoPrepare
;
226 /* Allocate a wired-down buffer inside kernel space. */
227 if (options
& kIOMemoryPhysicallyContiguous
)
228 _buffer
= (void *) IOKernelAllocateContiguous(capacity
, alignment
);
229 else if (alignment
> 1)
230 _buffer
= IOMallocAligned(capacity
, alignment
);
232 _buffer
= IOMalloc(capacity
);
238 if( (kIOMemoryTypePhysical64
!= (kIOMemoryTypeMask
& iomdOptions
))
239 && (options
& (kIOMemoryPageable
| kIOMapCacheMask
))) {
240 ipc_port_t sharedMem
;
241 vm_size_t size
= round_page_32(capacity
);
243 kr
= mach_make_memory_entry(vmmap
,
244 &size
, (vm_offset_t
)_buffer
,
245 memEntryCacheMode
, &sharedMem
,
248 if( (KERN_SUCCESS
== kr
) && (size
!= round_page_32(capacity
))) {
249 ipc_port_release_send( sharedMem
);
250 kr
= kIOReturnVMError
;
252 if( KERN_SUCCESS
!= kr
)
255 _memEntry
= (void *) sharedMem
;
257 if( options
& kIOMemoryPageable
) {
259 debug_iomallocpageable_size
+= size
;
263 inTask
= kernel_task
;
265 else if (options
& kIOMapCacheMask
)
267 // Prefetch each page to put entries into the pmap
268 volatile UInt8
* startAddr
= (UInt8
*)_buffer
;
269 volatile UInt8
* endAddr
= (UInt8
*)_buffer
+ capacity
;
271 while (startAddr
< endAddr
)
274 startAddr
+= page_size
;
279 range
.address
= (mach_vm_address_t
) _buffer
;
280 range
.length
= capacity
;
282 if (!super::initWithOptions(&range
, 1, 0,
283 inTask
, iomdOptions
, /* System mapper */ 0))
286 if (physicalMask
&& !IOMapper::gSystem
)
288 IOMDDMACharacteristics mdSummary
;
290 bzero(&mdSummary
, sizeof(mdSummary
));
291 IOReturn rtn
= dmaCommandOperation(
292 kIOMDGetCharacteristics
,
293 &mdSummary
, sizeof(mdSummary
));
297 if (mdSummary
.fHighestPage
)
300 while (mdSummary
.fHighestPage
> (highest
= gIOHighestAllocatedPage
))
302 if (OSCompareAndSwap(highest
, mdSummary
.fHighestPage
,
303 (UInt32
*) &gIOHighestAllocatedPage
))
306 lastIOAddr
= ptoa_64(mdSummary
.fHighestPage
);
309 lastIOAddr
= ptoa_64(gIOLastPage
);
311 if (lastIOAddr
!= (lastIOAddr
& physicalMask
))
313 if (kIOMemoryTypePhysical64
!= (_flags
& kIOMemoryTypeMask
))
325 reserved
= IONew( ExpansionData
, 1 );
329 reserved
->map
= map(mapTask
, 0, kIOMapAnywhere
, 0, 0);
335 release(); // map took a retain on this
336 mach_vm_address_t buffer
= reserved
->map
->getAddress();
337 _buffer
= (void *) buffer
;
338 if (kIOMemoryTypeVirtual64
== (kIOMemoryTypeMask
& iomdOptions
))
339 _ranges
.v64
->address
= buffer
;
347 IOBufferMemoryDescriptor
* IOBufferMemoryDescriptor::inTaskWithOptions(
349 IOOptionBits options
,
351 vm_offset_t alignment
)
353 IOBufferMemoryDescriptor
*me
= new IOBufferMemoryDescriptor
;
355 if (me
&& !me
->initWithOptions(options
, capacity
, alignment
, inTask
)) {
356 bool retry
= me
->_physSegCount
;
361 me
= new IOBufferMemoryDescriptor
;
362 if (me
&& !me
->initWithOptions(options
, capacity
, alignment
, inTask
))
372 IOBufferMemoryDescriptor
* IOBufferMemoryDescriptor::inTaskWithPhysicalMask(
374 IOOptionBits options
,
375 mach_vm_size_t capacity
,
376 mach_vm_address_t physicalMask
)
378 IOBufferMemoryDescriptor
*me
= new IOBufferMemoryDescriptor
;
380 if (me
&& !me
->initWithPhysicalMask(inTask
, options
, capacity
, 1, physicalMask
))
382 bool retry
= me
->_physSegCount
;
387 me
= new IOBufferMemoryDescriptor
;
388 if (me
&& !me
->initWithPhysicalMask(inTask
, options
, capacity
, 1, physicalMask
))
398 bool IOBufferMemoryDescriptor::initWithOptions(
399 IOOptionBits options
,
401 vm_offset_t alignment
)
403 return( initWithOptions(options
, capacity
, alignment
, kernel_task
) );
406 IOBufferMemoryDescriptor
* IOBufferMemoryDescriptor::withOptions(
407 IOOptionBits options
,
409 vm_offset_t alignment
)
411 return(IOBufferMemoryDescriptor::inTaskWithOptions(kernel_task
, options
, capacity
, alignment
));
418 * Returns a new IOBufferMemoryDescriptor with a buffer large enough to
419 * hold capacity bytes. The descriptor's length is initially set to the capacity.
421 IOBufferMemoryDescriptor
*
422 IOBufferMemoryDescriptor::withCapacity(vm_size_t inCapacity
,
423 IODirection inDirection
,
426 return( IOBufferMemoryDescriptor::withOptions(
427 inDirection
| kIOMemoryUnshared
428 | (inContiguous
? kIOMemoryPhysicallyContiguous
: 0),
429 inCapacity
, inContiguous
? inCapacity
: 1 ));
435 * Initialize a new IOBufferMemoryDescriptor preloaded with bytes (copied).
436 * The descriptor's length and capacity are set to the input buffer's size.
438 bool IOBufferMemoryDescriptor::initWithBytes(const void * inBytes
,
440 IODirection inDirection
,
443 if (!initWithOptions(
444 inDirection
| kIOMemoryUnshared
445 | (inContiguous
? kIOMemoryPhysicallyContiguous
: 0),
446 inLength
, inLength
))
449 // start out with no data
452 if (!appendBytes(inBytes
, inLength
))
461 * Returns a new IOBufferMemoryDescriptor preloaded with bytes (copied).
462 * The descriptor's length and capacity are set to the input buffer's size.
464 IOBufferMemoryDescriptor
*
465 IOBufferMemoryDescriptor::withBytes(const void * inBytes
,
467 IODirection inDirection
,
470 IOBufferMemoryDescriptor
*me
= new IOBufferMemoryDescriptor
;
472 if (me
&& !me
->initWithBytes(inBytes
, inLength
, inDirection
, inContiguous
))
474 bool retry
= me
->_physSegCount
;
479 me
= new IOBufferMemoryDescriptor
;
480 if (me
&& !me
->initWithBytes(inBytes
, inLength
, inDirection
, inContiguous
))
496 void IOBufferMemoryDescriptor::free()
498 // Cache all of the relevant information on the stack for use
499 // after we call super::free()!
500 IOOptionBits flags
= _flags
;
501 IOOptionBits options
= _options
;
502 vm_size_t size
= _capacity
;
503 void * buffer
= _buffer
;
504 mach_vm_address_t source
= (_ranges
.v
) ? _ranges
.v64
->address
: 0;
505 IOMemoryMap
* map
= 0;
506 vm_offset_t alignment
= _alignment
;
511 IODelete( reserved
, ExpansionData
, 1 );
516 /* super::free may unwire - deallocate buffer afterwards */
519 if (options
& kIOMemoryPageable
)
522 debug_iomallocpageable_size
-= round_page_32(size
);
527 if (kIOMemoryTypePhysical64
== (flags
& kIOMemoryTypeMask
))
528 IOFreePhysical(source
, size
);
529 else if (options
& kIOMemoryPhysicallyContiguous
)
530 IOKernelFreeContiguous((mach_vm_address_t
) buffer
, size
);
531 else if (alignment
> 1)
532 IOFreeAligned(buffer
, size
);
534 IOFree(buffer
, size
);
541 * Get the buffer capacity
543 vm_size_t
IOBufferMemoryDescriptor::getCapacity() const
551 * Change the buffer length of the memory descriptor. When a new buffer
552 * is created, the initial length of the buffer is set to be the same as
553 * the capacity. The length can be adjusted via setLength for a shorter
554 * transfer (there is no need to create more buffer descriptors when you
555 * can reuse an existing one, even for different transfer sizes). Note
556 * that the specified length must not exceed the capacity of the buffer.
558 void IOBufferMemoryDescriptor::setLength(vm_size_t length
)
560 assert(length
<= _capacity
);
563 _ranges
.v64
->length
= length
;
569 * Change the direction of the transfer. This method allows one to redirect
570 * the descriptor's transfer direction. This eliminates the need to destroy
571 * and create new buffers when different transfer directions are needed.
573 void IOBufferMemoryDescriptor::setDirection(IODirection direction
)
575 _direction
= direction
;
581 * Add some data to the end of the buffer. This method automatically
582 * maintains the memory descriptor buffer length. Note that appendBytes
583 * will not copy past the end of the memory descriptor's current capacity.
586 IOBufferMemoryDescriptor::appendBytes(const void * bytes
, vm_size_t withLength
)
588 vm_size_t actualBytesToCopy
= min(withLength
, _capacity
- _length
);
591 assert(_length
<= _capacity
);
594 _length
+= actualBytesToCopy
;
595 _ranges
.v64
->length
+= actualBytesToCopy
;
597 if (_task
== kernel_task
)
598 bcopy(/* from */ bytes
, (void *)(_ranges
.v64
->address
+ offset
),
601 writeBytes(offset
, bytes
, actualBytesToCopy
);
609 * Return the virtual address of the beginning of the buffer
611 void * IOBufferMemoryDescriptor::getBytesNoCopy()
613 if (kIOMemoryTypePhysical64
== (_flags
& kIOMemoryTypeMask
))
616 return (void *)_ranges
.v64
->address
;
623 * Return the virtual address of an offset from the beginning of the buffer
626 IOBufferMemoryDescriptor::getBytesNoCopy(vm_size_t start
, vm_size_t withLength
)
628 IOVirtualAddress address
;
629 if (kIOMemoryTypePhysical64
== (_flags
& kIOMemoryTypeMask
))
630 address
= (IOVirtualAddress
) _buffer
;
632 address
= _ranges
.v64
->address
;
634 if (start
< _length
&& (start
+ withLength
) <= _length
)
635 return (void *)(address
+ start
);
639 /* DEPRECATED */ void * IOBufferMemoryDescriptor::getVirtualSegment(IOByteCount offset
,
640 /* DEPRECATED */ IOByteCount
* lengthOfSegment
)
642 void * bytes
= getBytesNoCopy(offset
, 0);
644 if (bytes
&& lengthOfSegment
)
645 *lengthOfSegment
= _length
- offset
;
650 OSMetaClassDefineReservedUsed(IOBufferMemoryDescriptor
, 0);
651 OSMetaClassDefineReservedUsed(IOBufferMemoryDescriptor
, 1);
652 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 2);
653 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 3);
654 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 4);
655 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 5);
656 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 6);
657 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 7);
658 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 8);
659 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 9);
660 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 10);
661 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 11);
662 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 12);
663 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 13);
664 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 14);
665 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 15);