2 * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
29 #define _IOMEMORYDESCRIPTOR_INTERNAL_
31 #include <IOKit/assert.h>
32 #include <IOKit/system.h>
34 #include <IOKit/IOLib.h>
35 #include <IOKit/IOMapper.h>
36 #include <IOKit/IOBufferMemoryDescriptor.h>
37 #include <libkern/OSDebug.h>
38 #include <mach/mach_vm.h>
40 #include "IOKitKernelInternal.h"
43 #include <libkern/c++/OSCPPDebug.h>
45 #include <IOKit/IOStatisticsPrivate.h>
48 #define IOStatisticsAlloc(type, size) \
50 IOStatistics::countAlloc(type, size); \
53 #define IOStatisticsAlloc(type, size)
54 #endif /* IOKITSTATS */
58 void ipc_port_release_send(ipc_port_t port
);
63 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
67 kInternalFlagPhysical
= 0x00000001,
68 kInternalFlagPageSized
= 0x00000002,
69 kInternalFlagPageAllocated
= 0x00000004
72 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
74 #define super IOGeneralMemoryDescriptor
75 OSDefineMetaClassAndStructors(IOBufferMemoryDescriptor
,
76 IOGeneralMemoryDescriptor
);
78 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
80 static uintptr_t IOBMDPageProc(iopa_t
* a
)
83 vm_address_t vmaddr
= 0;
84 int options
= 0; // KMA_LOMEM;
86 kr
= kernel_memory_allocate(kernel_map
, &vmaddr
,
87 page_size
, 0, options
, VM_KERN_MEMORY_IOKIT
);
89 if (KERN_SUCCESS
!= kr
) vmaddr
= 0;
90 else bzero((void *) vmaddr
, page_size
);
92 return ((uintptr_t) vmaddr
);
95 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
98 bool IOBufferMemoryDescriptor::initWithOptions(
101 vm_offset_t alignment
,
104 mach_vm_address_t physicalMask
= 0;
105 return (initWithPhysicalMask(inTask
, options
, capacity
, alignment
, physicalMask
));
107 #endif /* !__LP64__ */
109 bool IOBufferMemoryDescriptor::initWithPhysicalMask(
111 IOOptionBits options
,
112 mach_vm_size_t capacity
,
113 mach_vm_address_t alignment
,
114 mach_vm_address_t physicalMask
)
116 task_t mapTask
= NULL
;
117 vm_map_t vmmap
= NULL
;
118 mach_vm_address_t highestMask
= 0;
119 IOOptionBits iomdOptions
= kIOMemoryTypeVirtual64
| kIOMemoryAsReference
;
120 IODMAMapSpecification mapSpec
;
124 if (!capacity
) return false;
127 _capacity
= capacity
;
129 _internalReserved
= 0;
132 _ranges
.v64
= IONew(IOAddressRange
, 1);
135 _ranges
.v64
->address
= 0;
136 _ranges
.v64
->length
= 0;
137 // make sure super::free doesn't dealloc _ranges before super::init
138 _flags
= kIOMemoryAsReference
;
140 // Grab IOMD bits from the Buffer MD options
141 iomdOptions
|= (options
& kIOBufferDescriptorMemoryFlags
);
143 if (!(kIOMemoryMapperNone
& options
))
145 IOMapper::checkForSystemMapper();
146 mapped
= (0 != IOMapper::gSystem
);
148 needZero
= (mapped
|| (0 != (kIOMemorySharingTypeMask
& options
)));
150 if (physicalMask
&& (alignment
<= 1))
152 alignment
= ((physicalMask
^ (-1ULL)) & (physicalMask
- 1));
153 highestMask
= (physicalMask
| alignment
);
155 if (alignment
< page_size
)
156 alignment
= page_size
;
159 if ((options
& (kIOMemorySharingTypeMask
| kIOMapCacheMask
| kIOMemoryClearEncrypt
)) && (alignment
< page_size
))
160 alignment
= page_size
;
162 if (alignment
>= page_size
)
163 capacity
= round_page(capacity
);
165 if (alignment
> page_size
)
166 options
|= kIOMemoryPhysicallyContiguous
;
168 _alignment
= alignment
;
170 if ((capacity
+ alignment
) < _capacity
) return (false);
172 if ((inTask
!= kernel_task
) && !(options
& kIOMemoryPageable
))
175 bzero(&mapSpec
, sizeof(mapSpec
));
176 mapSpec
.alignment
= _alignment
;
177 mapSpec
.numAddressBits
= 64;
178 if (highestMask
&& mapped
)
180 if (highestMask
<= 0xFFFFFFFF)
181 mapSpec
.numAddressBits
= (32 - __builtin_clz((unsigned int) highestMask
));
183 mapSpec
.numAddressBits
= (64 - __builtin_clz((unsigned int) (highestMask
>> 32)));
187 // set memory entry cache mode, pageable, purgeable
188 iomdOptions
|= ((options
& kIOMapCacheMask
) >> kIOMapCacheShift
) << kIOMemoryBufferCacheShift
;
189 if (options
& kIOMemoryPageable
)
191 iomdOptions
|= kIOMemoryBufferPageable
;
192 if (options
& kIOMemoryPurgeable
) iomdOptions
|= kIOMemoryBufferPurgeable
;
198 // Buffer shouldn't auto prepare they should be prepared explicitly
199 // But it never was enforced so what are you going to do?
200 iomdOptions
|= kIOMemoryAutoPrepare
;
202 /* Allocate a wired-down buffer inside kernel space. */
204 bool contig
= (0 != (options
& kIOMemoryHostPhysicallyContiguous
));
206 if (!contig
&& (0 != (options
& kIOMemoryPhysicallyContiguous
)))
209 contig
|= (0 != (kIOMemoryMapperNone
& options
));
211 // treat kIOMemoryPhysicallyContiguous as kIOMemoryHostPhysicallyContiguous for now
216 if (contig
|| highestMask
|| (alignment
> page_size
))
218 _internalFlags
|= kInternalFlagPhysical
;
221 _internalFlags
|= kInternalFlagPageSized
;
222 capacity
= round_page(capacity
);
224 _buffer
= (void *) IOKernelAllocateWithPhysicalRestrict(
225 capacity
, highestMask
, alignment
, contig
);
228 && ((capacity
+ alignment
) <= (page_size
- gIOPageAllocChunkBytes
)))
230 _internalFlags
|= kInternalFlagPageAllocated
;
232 _buffer
= (void *) iopa_alloc(&gIOBMDPageAllocator
, &IOBMDPageProc
, capacity
, alignment
);
235 IOStatisticsAlloc(kIOStatisticsMallocAligned
, capacity
);
237 OSAddAtomic(capacity
, &debug_iomalloc_size
);
241 else if (alignment
> 1)
243 _buffer
= IOMallocAligned(capacity
, alignment
);
247 _buffer
= IOMalloc(capacity
);
253 if (needZero
) bzero(_buffer
, capacity
);
256 if( (options
& (kIOMemoryPageable
| kIOMapCacheMask
))) {
257 vm_size_t size
= round_page(capacity
);
259 // initWithOptions will create memory entry
260 iomdOptions
|= kIOMemoryPersistent
;
262 if( options
& kIOMemoryPageable
) {
264 OSAddAtomicLong(size
, &debug_iomallocpageable_size
);
268 inTask
= kernel_task
;
270 else if (options
& kIOMapCacheMask
)
272 // Prefetch each page to put entries into the pmap
273 volatile UInt8
* startAddr
= (UInt8
*)_buffer
;
274 volatile UInt8
* endAddr
= (UInt8
*)_buffer
+ capacity
;
276 while (startAddr
< endAddr
)
278 UInt8 dummyVar
= *startAddr
;
280 startAddr
+= page_size
;
285 _ranges
.v64
->address
= (mach_vm_address_t
) _buffer
;;
286 _ranges
.v64
->length
= _capacity
;
288 if (!super::initWithOptions(_ranges
.v64
, 1, 0,
289 inTask
, iomdOptions
, /* System mapper */ 0))
292 // give any system mapper the allocation params
293 if (kIOReturnSuccess
!= dmaCommandOperation(kIOMDAddDMAMapSpec
,
294 &mapSpec
, sizeof(mapSpec
)))
300 reserved
= IONew( ExpansionData
, 1 );
304 reserved
->map
= createMappingInTask(mapTask
, 0,
305 kIOMapAnywhere
| (options
& kIOMapPrefault
) | (options
& kIOMapCacheMask
), 0, 0);
311 release(); // map took a retain on this
312 reserved
->map
->retain();
313 removeMapping(reserved
->map
);
314 mach_vm_address_t buffer
= reserved
->map
->getAddress();
315 _buffer
= (void *) buffer
;
316 if (kIOMemoryTypeVirtual64
== (kIOMemoryTypeMask
& iomdOptions
))
317 _ranges
.v64
->address
= buffer
;
320 setLength(_capacity
);
325 IOBufferMemoryDescriptor
* IOBufferMemoryDescriptor::inTaskWithOptions(
327 IOOptionBits options
,
329 vm_offset_t alignment
)
331 IOBufferMemoryDescriptor
*me
= new IOBufferMemoryDescriptor
;
333 if (me
&& !me
->initWithPhysicalMask(inTask
, options
, capacity
, alignment
, 0)) {
340 IOBufferMemoryDescriptor
* IOBufferMemoryDescriptor::inTaskWithPhysicalMask(
342 IOOptionBits options
,
343 mach_vm_size_t capacity
,
344 mach_vm_address_t physicalMask
)
346 IOBufferMemoryDescriptor
*me
= new IOBufferMemoryDescriptor
;
348 if (me
&& !me
->initWithPhysicalMask(inTask
, options
, capacity
, 1, physicalMask
))
357 bool IOBufferMemoryDescriptor::initWithOptions(
358 IOOptionBits options
,
360 vm_offset_t alignment
)
362 return (initWithPhysicalMask(kernel_task
, options
, capacity
, alignment
, (mach_vm_address_t
)0));
364 #endif /* !__LP64__ */
366 IOBufferMemoryDescriptor
* IOBufferMemoryDescriptor::withOptions(
367 IOOptionBits options
,
369 vm_offset_t alignment
)
371 IOBufferMemoryDescriptor
*me
= new IOBufferMemoryDescriptor
;
373 if (me
&& !me
->initWithPhysicalMask(kernel_task
, options
, capacity
, alignment
, 0)) {
384 * Returns a new IOBufferMemoryDescriptor with a buffer large enough to
385 * hold capacity bytes. The descriptor's length is initially set to the capacity.
387 IOBufferMemoryDescriptor
*
388 IOBufferMemoryDescriptor::withCapacity(vm_size_t inCapacity
,
389 IODirection inDirection
,
392 return( IOBufferMemoryDescriptor::withOptions(
393 inDirection
| kIOMemoryUnshared
394 | (inContiguous
? kIOMemoryPhysicallyContiguous
: 0),
395 inCapacity
, inContiguous
? inCapacity
: 1 ));
402 * Initialize a new IOBufferMemoryDescriptor preloaded with bytes (copied).
403 * The descriptor's length and capacity are set to the input buffer's size.
405 bool IOBufferMemoryDescriptor::initWithBytes(const void * inBytes
,
407 IODirection inDirection
,
410 if (!initWithPhysicalMask(kernel_task
, inDirection
| kIOMemoryUnshared
411 | (inContiguous
? kIOMemoryPhysicallyContiguous
: 0),
412 inLength
, inLength
, (mach_vm_address_t
)0))
415 // start out with no data
418 if (!appendBytes(inBytes
, inLength
))
423 #endif /* !__LP64__ */
428 * Returns a new IOBufferMemoryDescriptor preloaded with bytes (copied).
429 * The descriptor's length and capacity are set to the input buffer's size.
431 IOBufferMemoryDescriptor
*
432 IOBufferMemoryDescriptor::withBytes(const void * inBytes
,
434 IODirection inDirection
,
437 IOBufferMemoryDescriptor
*me
= new IOBufferMemoryDescriptor
;
439 if (me
&& !me
->initWithPhysicalMask(
440 kernel_task
, inDirection
| kIOMemoryUnshared
441 | (inContiguous
? kIOMemoryPhysicallyContiguous
: 0),
442 inLength
, inLength
, 0 ))
450 // start out with no data
453 if (!me
->appendBytes(inBytes
, inLength
))
467 void IOBufferMemoryDescriptor::free()
469 // Cache all of the relevant information on the stack for use
470 // after we call super::free()!
471 IOOptionBits flags
= _flags
;
472 IOOptionBits internalFlags
= _internalFlags
;
473 IOOptionBits options
= _options
;
474 vm_size_t size
= _capacity
;
475 void * buffer
= _buffer
;
476 IOMemoryMap
* map
= 0;
477 IOAddressRange
* range
= _ranges
.v64
;
478 vm_offset_t alignment
= _alignment
;
480 if (alignment
>= page_size
)
481 size
= round_page(size
);
486 IODelete( reserved
, ExpansionData
, 1 );
491 /* super::free may unwire - deallocate buffer afterwards */
494 if (options
& kIOMemoryPageable
)
497 OSAddAtomicLong(-(round_page(size
)), &debug_iomallocpageable_size
);
502 if (kInternalFlagPageSized
& internalFlags
) size
= round_page(size
);
504 if (kInternalFlagPhysical
& internalFlags
)
506 IOKernelFreePhysical((mach_vm_address_t
) buffer
, size
);
508 else if (kInternalFlagPageAllocated
& internalFlags
)
511 page
= iopa_free(&gIOBMDPageAllocator
, (uintptr_t) buffer
, size
);
514 kmem_free(kernel_map
, page
, page_size
);
517 OSAddAtomic(-size
, &debug_iomalloc_size
);
519 IOStatisticsAlloc(kIOStatisticsFreeAligned
, size
);
521 else if (alignment
> 1)
523 IOFreeAligned(buffer
, size
);
527 IOFree(buffer
, size
);
530 if (range
&& (kIOMemoryAsReference
& flags
))
531 IODelete(range
, IOAddressRange
, 1);
537 * Get the buffer capacity
539 vm_size_t
IOBufferMemoryDescriptor::getCapacity() const
547 * Change the buffer length of the memory descriptor. When a new buffer
548 * is created, the initial length of the buffer is set to be the same as
549 * the capacity. The length can be adjusted via setLength for a shorter
550 * transfer (there is no need to create more buffer descriptors when you
551 * can reuse an existing one, even for different transfer sizes). Note
552 * that the specified length must not exceed the capacity of the buffer.
554 void IOBufferMemoryDescriptor::setLength(vm_size_t length
)
556 assert(length
<= _capacity
);
557 if (length
> _capacity
) return;
560 _ranges
.v64
->length
= length
;
566 * Change the direction of the transfer. This method allows one to redirect
567 * the descriptor's transfer direction. This eliminates the need to destroy
568 * and create new buffers when different transfer directions are needed.
570 void IOBufferMemoryDescriptor::setDirection(IODirection direction
)
572 _flags
= (_flags
& ~kIOMemoryDirectionMask
) | direction
;
574 _direction
= (IODirection
) (_flags
& kIOMemoryDirectionMask
);
575 #endif /* !__LP64__ */
581 * Add some data to the end of the buffer. This method automatically
582 * maintains the memory descriptor buffer length. Note that appendBytes
583 * will not copy past the end of the memory descriptor's current capacity.
586 IOBufferMemoryDescriptor::appendBytes(const void * bytes
, vm_size_t withLength
)
588 vm_size_t actualBytesToCopy
= min(withLength
, _capacity
- _length
);
591 assert(_length
<= _capacity
);
594 _length
+= actualBytesToCopy
;
595 _ranges
.v64
->length
+= actualBytesToCopy
;
597 if (_task
== kernel_task
)
598 bcopy(/* from */ bytes
, (void *)(_ranges
.v64
->address
+ offset
),
601 writeBytes(offset
, bytes
, actualBytesToCopy
);
609 * Return the virtual address of the beginning of the buffer
611 void * IOBufferMemoryDescriptor::getBytesNoCopy()
613 if (kIOMemoryTypePhysical64
== (_flags
& kIOMemoryTypeMask
))
616 return (void *)_ranges
.v64
->address
;
623 * Return the virtual address of an offset from the beginning of the buffer
626 IOBufferMemoryDescriptor::getBytesNoCopy(vm_size_t start
, vm_size_t withLength
)
628 IOVirtualAddress address
;
629 if (kIOMemoryTypePhysical64
== (_flags
& kIOMemoryTypeMask
))
630 address
= (IOVirtualAddress
) _buffer
;
632 address
= _ranges
.v64
->address
;
634 if (start
< _length
&& (start
+ withLength
) <= _length
)
635 return (void *)(address
+ start
);
640 void * IOBufferMemoryDescriptor::getVirtualSegment(IOByteCount offset
,
641 IOByteCount
* lengthOfSegment
)
643 void * bytes
= getBytesNoCopy(offset
, 0);
645 if (bytes
&& lengthOfSegment
)
646 *lengthOfSegment
= _length
- offset
;
650 #endif /* !__LP64__ */
653 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 0);
654 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 1);
655 #else /* !__LP64__ */
656 OSMetaClassDefineReservedUsed(IOBufferMemoryDescriptor
, 0);
657 OSMetaClassDefineReservedUsed(IOBufferMemoryDescriptor
, 1);
658 #endif /* !__LP64__ */
659 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 2);
660 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 3);
661 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 4);
662 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 5);
663 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 6);
664 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 7);
665 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 8);
666 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 9);
667 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 10);
668 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 11);
669 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 12);
670 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 13);
671 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 14);
672 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 15);