2 * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
4 * @APPLE_LICENSE_OSREFERENCE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the
10 * License may not be used to create, or enable the creation or
11 * redistribution of, unlawful or unlicensed copies of an Apple operating
12 * system, or to circumvent, violate, or enable the circumvention or
13 * violation of, any terms of an Apple operating system software license
16 * Please obtain a copy of the License at
17 * http://www.opensource.apple.com/apsl/ and read it before using this
20 * The Original Code and all software distributed under the License are
21 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
22 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
23 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
24 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
25 * Please see the License for the specific language governing rights and
26 * limitations under the License.
28 * @APPLE_LICENSE_OSREFERENCE_HEADER_END@
30 #include <IOKit/assert.h>
31 #include <IOKit/system.h>
33 #include <IOKit/IOLib.h>
34 #include <IOKit/IOMapper.h>
35 #include <IOKit/IOBufferMemoryDescriptor.h>
37 #include "IOKitKernelInternal.h"
38 #include "IOCopyMapper.h"
41 void ipc_port_release_send(ipc_port_t port
);
44 vm_map_t
IOPageableMapForAddress( vm_address_t address
);
47 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
49 volatile ppnum_t gIOHighestAllocatedPage
;
51 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
53 #define super IOGeneralMemoryDescriptor
54 OSDefineMetaClassAndStructors(IOBufferMemoryDescriptor
,
55 IOGeneralMemoryDescriptor
);
57 bool IOBufferMemoryDescriptor::initWithAddress(
58 void * /* address */ ,
59 IOByteCount
/* withLength */ ,
60 IODirection
/* withDirection */ )
65 bool IOBufferMemoryDescriptor::initWithAddress(
66 vm_address_t
/* address */ ,
67 IOByteCount
/* withLength */ ,
68 IODirection
/* withDirection */ ,
69 task_t
/* withTask */ )
74 bool IOBufferMemoryDescriptor::initWithPhysicalAddress(
75 IOPhysicalAddress
/* address */ ,
76 IOByteCount
/* withLength */ ,
77 IODirection
/* withDirection */ )
82 bool IOBufferMemoryDescriptor::initWithPhysicalRanges(
83 IOPhysicalRange
* /* ranges */ ,
84 UInt32
/* withCount */ ,
85 IODirection
/* withDirection */ ,
86 bool /* asReference */ )
91 bool IOBufferMemoryDescriptor::initWithRanges(
92 IOVirtualRange
* /* ranges */ ,
93 UInt32
/* withCount */ ,
94 IODirection
/* withDirection */ ,
95 task_t
/* withTask */ ,
96 bool /* asReference */ )
101 bool IOBufferMemoryDescriptor::initWithOptions(
102 IOOptionBits options
,
104 vm_offset_t alignment
,
107 mach_vm_address_t physicalMask
= 0;
108 return (initWithPhysicalMask(inTask
, options
, capacity
, alignment
, physicalMask
));
111 bool IOBufferMemoryDescriptor::initWithPhysicalMask(
113 IOOptionBits options
,
114 mach_vm_size_t capacity
,
115 mach_vm_address_t alignment
,
116 mach_vm_address_t physicalMask
)
121 IOOptionBits iomdOptions
= kIOMemoryAsReference
| kIOMemoryTypeVirtual
;
127 _capacity
= capacity
;
132 // Grab the direction and the Auto Prepare bits from the Buffer MD options
133 iomdOptions
|= options
& (kIOMemoryDirectionMask
| kIOMemoryAutoPrepare
);
135 if ((options
& kIOMemorySharingTypeMask
) && (alignment
< page_size
))
136 alignment
= page_size
;
138 if ((inTask
!= kernel_task
) && !(options
& kIOMemoryPageable
))
141 if (physicalMask
&& (alignment
<= 1))
142 alignment
= ((physicalMask
^ PAGE_MASK
) & PAGE_MASK
) + 1;
144 if ((options
& kIOMemoryPhysicallyContiguous
) && !physicalMask
)
145 physicalMask
= 0xFFFFFFFF;
147 _alignment
= alignment
;
148 if (options
& kIOMemoryPageable
)
150 iomdOptions
|= kIOMemoryBufferPageable
;
152 ipc_port_t sharedMem
;
153 vm_size_t size
= round_page_32(capacity
);
155 // must create the entry before any pages are allocated
157 // set flags for entry + object create
158 vm_prot_t memEntryCacheMode
= VM_PROT_READ
| VM_PROT_WRITE
159 | MAP_MEM_NAMED_CREATE
;
161 if (options
& kIOMemoryPurgeable
)
162 memEntryCacheMode
|= MAP_MEM_PURGABLE
;
164 // set memory entry cache mode
165 switch (options
& kIOMapCacheMask
)
167 case kIOMapInhibitCache
:
168 SET_MAP_MEM(MAP_MEM_IO
, memEntryCacheMode
);
171 case kIOMapWriteThruCache
:
172 SET_MAP_MEM(MAP_MEM_WTHRU
, memEntryCacheMode
);
175 case kIOMapWriteCombineCache
:
176 SET_MAP_MEM(MAP_MEM_WCOMB
, memEntryCacheMode
);
179 case kIOMapCopybackCache
:
180 SET_MAP_MEM(MAP_MEM_COPYBACK
, memEntryCacheMode
);
183 case kIOMapDefaultCache
:
185 SET_MAP_MEM(MAP_MEM_NOOP
, memEntryCacheMode
);
189 kr
= mach_make_memory_entry( vmmap
,
191 memEntryCacheMode
, &sharedMem
,
194 if( (KERN_SUCCESS
== kr
) && (size
!= round_page_32(capacity
))) {
195 ipc_port_release_send( sharedMem
);
196 kr
= kIOReturnVMError
;
198 if( KERN_SUCCESS
!= kr
)
201 _memEntry
= (void *) sharedMem
;
203 debug_iomallocpageable_size
+= size
;
206 inTask
= kernel_task
;
207 else if (inTask
== kernel_task
)
214 reserved
= IONew( ExpansionData
, 1 );
218 vmmap
= get_task_map(inTask
);
219 vm_map_reference(vmmap
);
220 reserved
->map
= vmmap
;
225 if (IOMapper::gSystem
)
226 // assuming mapped space is 2G
227 lastIOAddr
= (1UL << 31) - PAGE_SIZE
;
229 lastIOAddr
= ptoa_64(gIOHighestAllocatedPage
);
231 if (physicalMask
&& (lastIOAddr
!= (lastIOAddr
& physicalMask
)))
233 mach_vm_address_t address
;
234 iomdOptions
&= ~kIOMemoryTypeVirtual
;
235 iomdOptions
|= kIOMemoryTypePhysical
;
237 address
= IOMallocPhysical(capacity
, physicalMask
);
238 _buffer
= (void *) address
;
242 if (inTask
== kernel_task
)
246 else if (NULL
!= inTask
)
249 reserved
= IONew( ExpansionData
, 1 );
253 vmmap
= get_task_map(inTask
);
254 vm_map_reference(vmmap
);
255 reserved
->map
= vmmap
;
261 // Buffer shouldn't auto prepare they should be prepared explicitly
262 // But it never was enforced so what are you going to do?
263 iomdOptions
|= kIOMemoryAutoPrepare
;
265 /* Allocate a wired-down buffer inside kernel space. */
266 if (options
& kIOMemoryPhysicallyContiguous
)
267 _buffer
= (void *) IOKernelAllocateContiguous(capacity
, alignment
);
268 else if (alignment
> 1)
269 _buffer
= IOMallocAligned(capacity
, alignment
);
271 _buffer
= IOMalloc(capacity
);
277 _singleRange
.v
.address
= (vm_address_t
) _buffer
;
278 _singleRange
.v
.length
= capacity
;
280 if (!super::initWithOptions(&_singleRange
.v
, 1, 0,
281 inTask
, iomdOptions
, /* System mapper */ 0))
284 if (physicalMask
&& !IOMapper::gSystem
)
286 IOMDDMACharacteristics mdSummary
;
288 bzero(&mdSummary
, sizeof(mdSummary
));
289 IOReturn rtn
= dmaCommandOperation(
290 kIOMDGetCharacteristics
,
291 &mdSummary
, sizeof(mdSummary
));
295 if (mdSummary
.fHighestPage
)
298 while (mdSummary
.fHighestPage
> (highest
= gIOHighestAllocatedPage
))
300 if (OSCompareAndSwap(highest
, mdSummary
.fHighestPage
,
301 (UInt32
*) &gIOHighestAllocatedPage
))
304 lastIOAddr
= ptoa_64(mdSummary
.fHighestPage
);
307 lastIOAddr
= ptoa_64(gIOLastPage
);
309 if (lastIOAddr
!= (lastIOAddr
& physicalMask
))
311 if (kIOMemoryTypePhysical
!= (_flags
& kIOMemoryTypeMask
))
322 kr
= doMap(vmmap
, (IOVirtualAddress
*) &_buffer
, kIOMapAnywhere
, 0, capacity
);
323 if (KERN_SUCCESS
!= kr
)
329 if (kIOMemoryTypeVirtual
& iomdOptions
)
330 _singleRange
.v
.address
= (vm_address_t
) _buffer
;
338 IOBufferMemoryDescriptor
* IOBufferMemoryDescriptor::inTaskWithOptions(
340 IOOptionBits options
,
342 vm_offset_t alignment
)
344 IOBufferMemoryDescriptor
*me
= new IOBufferMemoryDescriptor
;
346 if (me
&& !me
->initWithOptions(options
, capacity
, alignment
, inTask
)) {
347 bool retry
= me
->_physSegCount
;
352 me
= new IOBufferMemoryDescriptor
;
353 if (me
&& !me
->initWithOptions(options
, capacity
, alignment
, inTask
))
363 IOBufferMemoryDescriptor
* IOBufferMemoryDescriptor::inTaskWithPhysicalMask(
365 IOOptionBits options
,
366 mach_vm_size_t capacity
,
367 mach_vm_address_t physicalMask
)
369 IOBufferMemoryDescriptor
*me
= new IOBufferMemoryDescriptor
;
371 if (me
&& !me
->initWithPhysicalMask(inTask
, options
, capacity
, 1, physicalMask
))
373 bool retry
= me
->_physSegCount
;
378 me
= new IOBufferMemoryDescriptor
;
379 if (me
&& !me
->initWithPhysicalMask(inTask
, options
, capacity
, 1, physicalMask
))
389 bool IOBufferMemoryDescriptor::initWithOptions(
390 IOOptionBits options
,
392 vm_offset_t alignment
)
394 return( initWithOptions(options
, capacity
, alignment
, kernel_task
) );
397 IOBufferMemoryDescriptor
* IOBufferMemoryDescriptor::withOptions(
398 IOOptionBits options
,
400 vm_offset_t alignment
)
402 return(IOBufferMemoryDescriptor::inTaskWithOptions(kernel_task
, options
, capacity
, alignment
));
409 * Returns a new IOBufferMemoryDescriptor with a buffer large enough to
410 * hold capacity bytes. The descriptor's length is initially set to the capacity.
412 IOBufferMemoryDescriptor
*
413 IOBufferMemoryDescriptor::withCapacity(vm_size_t inCapacity
,
414 IODirection inDirection
,
417 return( IOBufferMemoryDescriptor::withOptions(
418 inDirection
| kIOMemoryUnshared
419 | (inContiguous
? kIOMemoryPhysicallyContiguous
: 0),
420 inCapacity
, inContiguous
? inCapacity
: 1 ));
426 * Initialize a new IOBufferMemoryDescriptor preloaded with bytes (copied).
427 * The descriptor's length and capacity are set to the input buffer's size.
429 bool IOBufferMemoryDescriptor::initWithBytes(const void * inBytes
,
431 IODirection inDirection
,
434 if (!initWithOptions(
435 inDirection
| kIOMemoryUnshared
436 | (inContiguous
? kIOMemoryPhysicallyContiguous
: 0),
437 inLength
, inLength
))
440 // start out with no data
443 if (!appendBytes(inBytes
, inLength
))
452 * Returns a new IOBufferMemoryDescriptor preloaded with bytes (copied).
453 * The descriptor's length and capacity are set to the input buffer's size.
455 IOBufferMemoryDescriptor
*
456 IOBufferMemoryDescriptor::withBytes(const void * inBytes
,
458 IODirection inDirection
,
461 IOBufferMemoryDescriptor
*me
= new IOBufferMemoryDescriptor
;
463 if (me
&& !me
->initWithBytes(inBytes
, inLength
, inDirection
, inContiguous
))
465 bool retry
= me
->_physSegCount
;
470 me
= new IOBufferMemoryDescriptor
;
471 if (me
&& !me
->initWithBytes(inBytes
, inLength
, inDirection
, inContiguous
))
487 void IOBufferMemoryDescriptor::free()
489 // Cache all of the relevant information on the stack for use
490 // after we call super::free()!
491 IOOptionBits flags
= _flags
;
492 IOOptionBits options
= _options
;
493 vm_size_t size
= _capacity
;
494 void * buffer
= _buffer
;
495 IOVirtualAddress source
= _singleRange
.v
.address
;
497 vm_offset_t alignment
= _alignment
;
501 vmmap
= reserved
->map
;
502 IODelete( reserved
, ExpansionData
, 1 );
505 /* super::free may unwire - deallocate buffer afterwards */
508 if (options
& kIOMemoryPageable
)
511 if (!buffer
|| vmmap
)
512 debug_iomallocpageable_size
-= round_page_32(size
);
517 vm_deallocate(vmmap
, (vm_address_t
) buffer
, round_page_32(size
));
519 IOFreePageable(buffer
, size
);
524 if (kIOMemoryTypePhysical
== (flags
& kIOMemoryTypeMask
))
527 vm_deallocate(vmmap
, (vm_address_t
) buffer
, round_page_32(size
));
528 IOFreePhysical((mach_vm_address_t
) source
, size
);
530 else if (options
& kIOMemoryPhysicallyContiguous
)
531 IOKernelFreeContiguous((mach_vm_address_t
) buffer
, size
);
532 else if (alignment
> 1)
533 IOFreeAligned(buffer
, size
);
535 IOFree(buffer
, size
);
538 vm_map_deallocate(vmmap
);
544 * Get the buffer capacity
546 vm_size_t
IOBufferMemoryDescriptor::getCapacity() const
554 * Change the buffer length of the memory descriptor. When a new buffer
555 * is created, the initial length of the buffer is set to be the same as
556 * the capacity. The length can be adjusted via setLength for a shorter
557 * transfer (there is no need to create more buffer descriptors when you
558 * can reuse an existing one, even for different transfer sizes). Note
559 * that the specified length must not exceed the capacity of the buffer.
561 void IOBufferMemoryDescriptor::setLength(vm_size_t length
)
563 assert(length
<= _capacity
);
566 _singleRange
.v
.length
= length
;
572 * Change the direction of the transfer. This method allows one to redirect
573 * the descriptor's transfer direction. This eliminates the need to destroy
574 * and create new buffers when different transfer directions are needed.
576 void IOBufferMemoryDescriptor::setDirection(IODirection direction
)
578 _direction
= direction
;
584 * Add some data to the end of the buffer. This method automatically
585 * maintains the memory descriptor buffer length. Note that appendBytes
586 * will not copy past the end of the memory descriptor's current capacity.
589 IOBufferMemoryDescriptor::appendBytes(const void * bytes
, vm_size_t withLength
)
591 vm_size_t actualBytesToCopy
= min(withLength
, _capacity
- _length
);
594 assert(_length
<= _capacity
);
597 _length
+= actualBytesToCopy
;
598 _singleRange
.v
.length
+= actualBytesToCopy
;
600 if (_task
== kernel_task
)
601 bcopy(/* from */ bytes
, (void *)(_singleRange
.v
.address
+ offset
),
604 writeBytes(offset
, bytes
, actualBytesToCopy
);
612 * Return the virtual address of the beginning of the buffer
614 void * IOBufferMemoryDescriptor::getBytesNoCopy()
616 if (kIOMemoryTypePhysical
== (_flags
& kIOMemoryTypeMask
))
619 return (void *)_singleRange
.v
.address
;
626 * Return the virtual address of an offset from the beginning of the buffer
629 IOBufferMemoryDescriptor::getBytesNoCopy(vm_size_t start
, vm_size_t withLength
)
631 IOVirtualAddress address
;
632 if (kIOMemoryTypePhysical
== (_flags
& kIOMemoryTypeMask
))
633 address
= (IOVirtualAddress
) _buffer
;
635 address
= _singleRange
.v
.address
;
637 if (start
< _length
&& (start
+ withLength
) <= _length
)
638 return (void *)(address
+ start
);
642 /* DEPRECATED */ void * IOBufferMemoryDescriptor::getVirtualSegment(IOByteCount offset
,
643 /* DEPRECATED */ IOByteCount
* lengthOfSegment
)
645 void * bytes
= getBytesNoCopy(offset
, 0);
647 if (bytes
&& lengthOfSegment
)
648 *lengthOfSegment
= _length
- offset
;
653 OSMetaClassDefineReservedUsed(IOBufferMemoryDescriptor
, 0);
654 OSMetaClassDefineReservedUsed(IOBufferMemoryDescriptor
, 1);
655 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 2);
656 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 3);
657 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 4);
658 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 5);
659 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 6);
660 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 7);
661 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 8);
662 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 9);
663 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 10);
664 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 11);
665 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 12);
666 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 13);
667 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 14);
668 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 15);