2 * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
29 #define _IOMEMORYDESCRIPTOR_INTERNAL_
31 #include <IOKit/assert.h>
32 #include <IOKit/system.h>
34 #include <IOKit/IOLib.h>
35 #include <IOKit/IOMapper.h>
36 #include <IOKit/IOBufferMemoryDescriptor.h>
37 #include <libkern/OSDebug.h>
38 #include <mach/mach_vm.h>
40 #include "IOKitKernelInternal.h"
43 #include <libkern/c++/OSCPPDebug.h>
45 #include <IOKit/IOStatisticsPrivate.h>
48 #define IOStatisticsAlloc(type, size) \
50 IOStatistics::countAlloc(type, size); \
53 #define IOStatisticsAlloc(type, size)
54 #endif /* IOKITSTATS */
58 void ipc_port_release_send(ipc_port_t port
);
63 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
67 kInternalFlagPhysical
= 0x00000001,
68 kInternalFlagPageSized
= 0x00000002,
69 kInternalFlagPageAllocated
= 0x00000004
72 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
74 #define super IOGeneralMemoryDescriptor
75 OSDefineMetaClassAndStructors(IOBufferMemoryDescriptor
,
76 IOGeneralMemoryDescriptor
);
78 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
80 static uintptr_t IOBMDPageProc(iopa_t
* a
)
83 vm_address_t vmaddr
= 0;
84 int options
= 0; // KMA_LOMEM;
86 kr
= kernel_memory_allocate(kernel_map
, &vmaddr
,
87 page_size
, 0, options
);
89 if (KERN_SUCCESS
!= kr
) vmaddr
= 0;
90 else bzero((void *) vmaddr
, page_size
);
92 return ((uintptr_t) vmaddr
);
95 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
98 bool IOBufferMemoryDescriptor::initWithOptions(
101 vm_offset_t alignment
,
104 mach_vm_address_t physicalMask
= 0;
105 return (initWithPhysicalMask(inTask
, options
, capacity
, alignment
, physicalMask
));
107 #endif /* !__LP64__ */
109 bool IOBufferMemoryDescriptor::initWithPhysicalMask(
111 IOOptionBits options
,
112 mach_vm_size_t capacity
,
113 mach_vm_address_t alignment
,
114 mach_vm_address_t physicalMask
)
116 task_t mapTask
= NULL
;
117 vm_map_t vmmap
= NULL
;
118 mach_vm_address_t highestMask
= 0;
119 IOOptionBits iomdOptions
= kIOMemoryTypeVirtual64
| kIOMemoryAsReference
;
120 IODMAMapSpecification mapSpec
;
124 if (!capacity
) return false;
127 _capacity
= capacity
;
129 _internalReserved
= 0;
132 _ranges
.v64
= IONew(IOAddressRange
, 1);
135 _ranges
.v64
->address
= 0;
136 _ranges
.v64
->length
= 0;
137 // make sure super::free doesn't dealloc _ranges before super::init
138 _flags
= kIOMemoryAsReference
;
140 // Grab IOMD bits from the Buffer MD options
141 iomdOptions
|= (options
& kIOBufferDescriptorMemoryFlags
);
143 if (!(kIOMemoryMapperNone
& options
))
145 IOMapper::checkForSystemMapper();
146 mapped
= (0 != IOMapper::gSystem
);
148 needZero
= (mapped
|| (0 != (kIOMemorySharingTypeMask
& options
)));
150 if (physicalMask
&& (alignment
<= 1))
152 alignment
= ((physicalMask
^ (-1ULL)) & (physicalMask
- 1));
153 highestMask
= (physicalMask
| alignment
);
155 if (alignment
< page_size
)
156 alignment
= page_size
;
159 if ((options
& (kIOMemorySharingTypeMask
| kIOMapCacheMask
| kIOMemoryClearEncrypt
)) && (alignment
< page_size
))
160 alignment
= page_size
;
162 if (alignment
>= page_size
)
163 capacity
= round_page(capacity
);
165 if (alignment
> page_size
)
166 options
|= kIOMemoryPhysicallyContiguous
;
168 _alignment
= alignment
;
170 if ((inTask
!= kernel_task
) && !(options
& kIOMemoryPageable
))
173 bzero(&mapSpec
, sizeof(mapSpec
));
174 mapSpec
.alignment
= _alignment
;
175 mapSpec
.numAddressBits
= 64;
176 if (highestMask
&& mapped
)
178 if (highestMask
<= 0xFFFFFFFF)
179 mapSpec
.numAddressBits
= (32 - __builtin_clz((unsigned int) highestMask
));
181 mapSpec
.numAddressBits
= (64 - __builtin_clz((unsigned int) (highestMask
>> 32)));
185 // set memory entry cache mode, pageable, purgeable
186 iomdOptions
|= ((options
& kIOMapCacheMask
) >> kIOMapCacheShift
) << kIOMemoryBufferCacheShift
;
187 if (options
& kIOMemoryPageable
)
189 iomdOptions
|= kIOMemoryBufferPageable
;
190 if (options
& kIOMemoryPurgeable
) iomdOptions
|= kIOMemoryBufferPurgeable
;
196 // Buffer shouldn't auto prepare they should be prepared explicitly
197 // But it never was enforced so what are you going to do?
198 iomdOptions
|= kIOMemoryAutoPrepare
;
200 /* Allocate a wired-down buffer inside kernel space. */
202 bool contig
= (0 != (options
& kIOMemoryHostPhysicallyContiguous
));
204 if (!contig
&& (0 != (options
& kIOMemoryPhysicallyContiguous
)))
207 contig
|= (0 != (kIOMemoryMapperNone
& options
));
209 // treat kIOMemoryPhysicallyContiguous as kIOMemoryHostPhysicallyContiguous for now
214 if (contig
|| highestMask
|| (alignment
> page_size
))
216 _internalFlags
|= kInternalFlagPhysical
;
219 _internalFlags
|= kInternalFlagPageSized
;
220 capacity
= round_page(capacity
);
222 _buffer
= (void *) IOKernelAllocateWithPhysicalRestrict(
223 capacity
, highestMask
, alignment
, contig
);
226 && ((capacity
+ alignment
) <= (page_size
- gIOPageAllocChunkBytes
)))
228 _internalFlags
|= kInternalFlagPageAllocated
;
230 _buffer
= (void *) iopa_alloc(&gIOBMDPageAllocator
, &IOBMDPageProc
, capacity
, alignment
);
233 IOStatisticsAlloc(kIOStatisticsMallocAligned
, capacity
);
235 debug_iomalloc_size
+= capacity
;
239 else if (alignment
> 1)
241 _buffer
= IOMallocAligned(capacity
, alignment
);
245 _buffer
= IOMalloc(capacity
);
251 if (needZero
) bzero(_buffer
, capacity
);
254 if( (options
& (kIOMemoryPageable
| kIOMapCacheMask
))) {
255 vm_size_t size
= round_page(capacity
);
257 // initWithOptions will create memory entry
258 iomdOptions
|= kIOMemoryPersistent
;
260 if( options
& kIOMemoryPageable
) {
262 debug_iomallocpageable_size
+= size
;
266 inTask
= kernel_task
;
268 else if (options
& kIOMapCacheMask
)
270 // Prefetch each page to put entries into the pmap
271 volatile UInt8
* startAddr
= (UInt8
*)_buffer
;
272 volatile UInt8
* endAddr
= (UInt8
*)_buffer
+ capacity
;
274 while (startAddr
< endAddr
)
276 UInt8 dummyVar
= *startAddr
;
278 startAddr
+= page_size
;
283 _ranges
.v64
->address
= (mach_vm_address_t
) _buffer
;;
284 _ranges
.v64
->length
= _capacity
;
286 if (!super::initWithOptions(_ranges
.v64
, 1, 0,
287 inTask
, iomdOptions
, /* System mapper */ 0))
290 // give any system mapper the allocation params
291 if (kIOReturnSuccess
!= dmaCommandOperation(kIOMDAddDMAMapSpec
,
292 &mapSpec
, sizeof(mapSpec
)))
298 reserved
= IONew( ExpansionData
, 1 );
302 reserved
->map
= createMappingInTask(mapTask
, 0,
303 kIOMapAnywhere
| (options
& kIOMapPrefault
) | (options
& kIOMapCacheMask
), 0, 0);
309 release(); // map took a retain on this
310 reserved
->map
->retain();
311 removeMapping(reserved
->map
);
312 mach_vm_address_t buffer
= reserved
->map
->getAddress();
313 _buffer
= (void *) buffer
;
314 if (kIOMemoryTypeVirtual64
== (kIOMemoryTypeMask
& iomdOptions
))
315 _ranges
.v64
->address
= buffer
;
318 setLength(_capacity
);
323 IOBufferMemoryDescriptor
* IOBufferMemoryDescriptor::inTaskWithOptions(
325 IOOptionBits options
,
327 vm_offset_t alignment
)
329 IOBufferMemoryDescriptor
*me
= new IOBufferMemoryDescriptor
;
331 if (me
&& !me
->initWithPhysicalMask(inTask
, options
, capacity
, alignment
, 0)) {
338 IOBufferMemoryDescriptor
* IOBufferMemoryDescriptor::inTaskWithPhysicalMask(
340 IOOptionBits options
,
341 mach_vm_size_t capacity
,
342 mach_vm_address_t physicalMask
)
344 IOBufferMemoryDescriptor
*me
= new IOBufferMemoryDescriptor
;
346 if (me
&& !me
->initWithPhysicalMask(inTask
, options
, capacity
, 1, physicalMask
))
355 bool IOBufferMemoryDescriptor::initWithOptions(
356 IOOptionBits options
,
358 vm_offset_t alignment
)
360 return (initWithPhysicalMask(kernel_task
, options
, capacity
, alignment
, (mach_vm_address_t
)0));
362 #endif /* !__LP64__ */
364 IOBufferMemoryDescriptor
* IOBufferMemoryDescriptor::withOptions(
365 IOOptionBits options
,
367 vm_offset_t alignment
)
369 IOBufferMemoryDescriptor
*me
= new IOBufferMemoryDescriptor
;
371 if (me
&& !me
->initWithPhysicalMask(kernel_task
, options
, capacity
, alignment
, 0)) {
382 * Returns a new IOBufferMemoryDescriptor with a buffer large enough to
383 * hold capacity bytes. The descriptor's length is initially set to the capacity.
385 IOBufferMemoryDescriptor
*
386 IOBufferMemoryDescriptor::withCapacity(vm_size_t inCapacity
,
387 IODirection inDirection
,
390 return( IOBufferMemoryDescriptor::withOptions(
391 inDirection
| kIOMemoryUnshared
392 | (inContiguous
? kIOMemoryPhysicallyContiguous
: 0),
393 inCapacity
, inContiguous
? inCapacity
: 1 ));
400 * Initialize a new IOBufferMemoryDescriptor preloaded with bytes (copied).
401 * The descriptor's length and capacity are set to the input buffer's size.
403 bool IOBufferMemoryDescriptor::initWithBytes(const void * inBytes
,
405 IODirection inDirection
,
408 if (!initWithPhysicalMask(kernel_task
, inDirection
| kIOMemoryUnshared
409 | (inContiguous
? kIOMemoryPhysicallyContiguous
: 0),
410 inLength
, inLength
, (mach_vm_address_t
)0))
413 // start out with no data
416 if (!appendBytes(inBytes
, inLength
))
421 #endif /* !__LP64__ */
426 * Returns a new IOBufferMemoryDescriptor preloaded with bytes (copied).
427 * The descriptor's length and capacity are set to the input buffer's size.
429 IOBufferMemoryDescriptor
*
430 IOBufferMemoryDescriptor::withBytes(const void * inBytes
,
432 IODirection inDirection
,
435 IOBufferMemoryDescriptor
*me
= new IOBufferMemoryDescriptor
;
437 if (me
&& !me
->initWithPhysicalMask(
438 kernel_task
, inDirection
| kIOMemoryUnshared
439 | (inContiguous
? kIOMemoryPhysicallyContiguous
: 0),
440 inLength
, inLength
, 0 ))
448 // start out with no data
451 if (!me
->appendBytes(inBytes
, inLength
))
465 void IOBufferMemoryDescriptor::free()
467 // Cache all of the relevant information on the stack for use
468 // after we call super::free()!
469 IOOptionBits flags
= _flags
;
470 IOOptionBits internalFlags
= _internalFlags
;
471 IOOptionBits options
= _options
;
472 vm_size_t size
= _capacity
;
473 void * buffer
= _buffer
;
474 IOMemoryMap
* map
= 0;
475 IOAddressRange
* range
= _ranges
.v64
;
476 vm_offset_t alignment
= _alignment
;
478 if (alignment
>= page_size
)
479 size
= round_page(size
);
484 IODelete( reserved
, ExpansionData
, 1 );
489 /* super::free may unwire - deallocate buffer afterwards */
492 if (options
& kIOMemoryPageable
)
495 debug_iomallocpageable_size
-= round_page(size
);
500 if (kInternalFlagPageSized
& internalFlags
) size
= round_page(size
);
502 if (kInternalFlagPhysical
& internalFlags
)
504 IOKernelFreePhysical((mach_vm_address_t
) buffer
, size
);
506 else if (kInternalFlagPageAllocated
& internalFlags
)
509 page
= iopa_free(&gIOBMDPageAllocator
, (uintptr_t) buffer
, size
);
512 kmem_free(kernel_map
, page
, page_size
);
515 debug_iomalloc_size
-= size
;
517 IOStatisticsAlloc(kIOStatisticsFreeAligned
, size
);
519 else if (alignment
> 1)
521 IOFreeAligned(buffer
, size
);
525 IOFree(buffer
, size
);
528 if (range
&& (kIOMemoryAsReference
& flags
))
529 IODelete(range
, IOAddressRange
, 1);
535 * Get the buffer capacity
537 vm_size_t
IOBufferMemoryDescriptor::getCapacity() const
545 * Change the buffer length of the memory descriptor. When a new buffer
546 * is created, the initial length of the buffer is set to be the same as
547 * the capacity. The length can be adjusted via setLength for a shorter
548 * transfer (there is no need to create more buffer descriptors when you
549 * can reuse an existing one, even for different transfer sizes). Note
550 * that the specified length must not exceed the capacity of the buffer.
552 void IOBufferMemoryDescriptor::setLength(vm_size_t length
)
554 assert(length
<= _capacity
);
557 _ranges
.v64
->length
= length
;
563 * Change the direction of the transfer. This method allows one to redirect
564 * the descriptor's transfer direction. This eliminates the need to destroy
565 * and create new buffers when different transfer directions are needed.
567 void IOBufferMemoryDescriptor::setDirection(IODirection direction
)
569 _flags
= (_flags
& ~kIOMemoryDirectionMask
) | direction
;
571 _direction
= (IODirection
) (_flags
& kIOMemoryDirectionMask
);
572 #endif /* !__LP64__ */
578 * Add some data to the end of the buffer. This method automatically
579 * maintains the memory descriptor buffer length. Note that appendBytes
580 * will not copy past the end of the memory descriptor's current capacity.
583 IOBufferMemoryDescriptor::appendBytes(const void * bytes
, vm_size_t withLength
)
585 vm_size_t actualBytesToCopy
= min(withLength
, _capacity
- _length
);
588 assert(_length
<= _capacity
);
591 _length
+= actualBytesToCopy
;
592 _ranges
.v64
->length
+= actualBytesToCopy
;
594 if (_task
== kernel_task
)
595 bcopy(/* from */ bytes
, (void *)(_ranges
.v64
->address
+ offset
),
598 writeBytes(offset
, bytes
, actualBytesToCopy
);
606 * Return the virtual address of the beginning of the buffer
608 void * IOBufferMemoryDescriptor::getBytesNoCopy()
610 if (kIOMemoryTypePhysical64
== (_flags
& kIOMemoryTypeMask
))
613 return (void *)_ranges
.v64
->address
;
620 * Return the virtual address of an offset from the beginning of the buffer
623 IOBufferMemoryDescriptor::getBytesNoCopy(vm_size_t start
, vm_size_t withLength
)
625 IOVirtualAddress address
;
626 if (kIOMemoryTypePhysical64
== (_flags
& kIOMemoryTypeMask
))
627 address
= (IOVirtualAddress
) _buffer
;
629 address
= _ranges
.v64
->address
;
631 if (start
< _length
&& (start
+ withLength
) <= _length
)
632 return (void *)(address
+ start
);
637 void * IOBufferMemoryDescriptor::getVirtualSegment(IOByteCount offset
,
638 IOByteCount
* lengthOfSegment
)
640 void * bytes
= getBytesNoCopy(offset
, 0);
642 if (bytes
&& lengthOfSegment
)
643 *lengthOfSegment
= _length
- offset
;
647 #endif /* !__LP64__ */
650 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 0);
651 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 1);
652 #else /* !__LP64__ */
653 OSMetaClassDefineReservedUsed(IOBufferMemoryDescriptor
, 0);
654 OSMetaClassDefineReservedUsed(IOBufferMemoryDescriptor
, 1);
655 #endif /* !__LP64__ */
656 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 2);
657 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 3);
658 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 4);
659 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 5);
660 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 6);
661 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 7);
662 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 8);
663 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 9);
664 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 10);
665 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 11);
666 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 12);
667 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 13);
668 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 14);
669 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 15);