2 * Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
29 #define _IOMEMORYDESCRIPTOR_INTERNAL_
31 #include <IOKit/assert.h>
32 #include <IOKit/system.h>
34 #include <IOKit/IOLib.h>
35 #include <IOKit/IOMapper.h>
36 #include <IOKit/IOBufferMemoryDescriptor.h>
37 #include <libkern/OSDebug.h>
38 #include <mach/mach_vm.h>
40 #include "IOKitKernelInternal.h"
43 #include <libkern/c++/OSCPPDebug.h>
45 #include <IOKit/IOStatisticsPrivate.h>
48 #define IOStatisticsAlloc(type, size) \
50 IOStatistics::countAlloc(type, size); \
53 #define IOStatisticsAlloc(type, size)
54 #endif /* IOKITSTATS */
58 void ipc_port_release_send(ipc_port_t port
);
63 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
67 kInternalFlagPhysical
= 0x00000001,
68 kInternalFlagPageSized
= 0x00000002,
69 kInternalFlagPageAllocated
= 0x00000004,
70 kInternalFlagInit
= 0x00000008
73 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
75 #define super IOGeneralMemoryDescriptor
76 OSDefineMetaClassAndStructors(IOBufferMemoryDescriptor
,
77 IOGeneralMemoryDescriptor
);
79 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
81 static uintptr_t IOBMDPageProc(iopa_t
* a
)
84 vm_address_t vmaddr
= 0;
85 int options
= 0; // KMA_LOMEM;
87 kr
= kernel_memory_allocate(kernel_map
, &vmaddr
,
88 page_size
, 0, options
, VM_KERN_MEMORY_IOKIT
);
90 if (KERN_SUCCESS
!= kr
) vmaddr
= 0;
91 else bzero((void *) vmaddr
, page_size
);
93 return ((uintptr_t) vmaddr
);
96 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
99 bool IOBufferMemoryDescriptor::initWithOptions(
100 IOOptionBits options
,
102 vm_offset_t alignment
,
105 mach_vm_address_t physicalMask
= 0;
106 return (initWithPhysicalMask(inTask
, options
, capacity
, alignment
, physicalMask
));
108 #endif /* !__LP64__ */
110 bool IOBufferMemoryDescriptor::initWithPhysicalMask(
112 IOOptionBits options
,
113 mach_vm_size_t capacity
,
114 mach_vm_address_t alignment
,
115 mach_vm_address_t physicalMask
)
117 task_t mapTask
= NULL
;
118 vm_map_t vmmap
= NULL
;
119 mach_vm_address_t highestMask
= 0;
120 IOOptionBits iomdOptions
= kIOMemoryTypeVirtual64
| kIOMemoryAsReference
;
121 IODMAMapSpecification mapSpec
;
125 if (!capacity
) return false;
128 _capacity
= capacity
;
130 _internalReserved
= 0;
133 _ranges
.v64
= IONew(IOAddressRange
, 1);
136 _ranges
.v64
->address
= 0;
137 _ranges
.v64
->length
= 0;
138 // make sure super::free doesn't dealloc _ranges before super::init
139 _flags
= kIOMemoryAsReference
;
141 // Grab IOMD bits from the Buffer MD options
142 iomdOptions
|= (options
& kIOBufferDescriptorMemoryFlags
);
144 if (!(kIOMemoryMapperNone
& options
))
146 IOMapper::checkForSystemMapper();
147 mapped
= (0 != IOMapper::gSystem
);
149 needZero
= (mapped
|| (0 != (kIOMemorySharingTypeMask
& options
)));
151 if (physicalMask
&& (alignment
<= 1))
153 alignment
= ((physicalMask
^ (-1ULL)) & (physicalMask
- 1));
154 highestMask
= (physicalMask
| alignment
);
156 if (alignment
< page_size
)
157 alignment
= page_size
;
160 if ((options
& (kIOMemorySharingTypeMask
| kIOMapCacheMask
| kIOMemoryClearEncrypt
)) && (alignment
< page_size
))
161 alignment
= page_size
;
163 if (alignment
>= page_size
)
164 capacity
= round_page(capacity
);
166 if (alignment
> page_size
)
167 options
|= kIOMemoryPhysicallyContiguous
;
169 _alignment
= alignment
;
171 if ((capacity
+ alignment
) < _capacity
) return (false);
173 if ((inTask
!= kernel_task
) && !(options
& kIOMemoryPageable
))
176 bzero(&mapSpec
, sizeof(mapSpec
));
177 mapSpec
.alignment
= _alignment
;
178 mapSpec
.numAddressBits
= 64;
179 if (highestMask
&& mapped
)
181 if (highestMask
<= 0xFFFFFFFF)
182 mapSpec
.numAddressBits
= (32 - __builtin_clz((unsigned int) highestMask
));
184 mapSpec
.numAddressBits
= (64 - __builtin_clz((unsigned int) (highestMask
>> 32)));
188 // set memory entry cache mode, pageable, purgeable
189 iomdOptions
|= ((options
& kIOMapCacheMask
) >> kIOMapCacheShift
) << kIOMemoryBufferCacheShift
;
190 if (options
& kIOMemoryPageable
)
192 iomdOptions
|= kIOMemoryBufferPageable
;
193 if (options
& kIOMemoryPurgeable
) iomdOptions
|= kIOMemoryBufferPurgeable
;
199 // Buffer shouldn't auto prepare they should be prepared explicitly
200 // But it never was enforced so what are you going to do?
201 iomdOptions
|= kIOMemoryAutoPrepare
;
203 /* Allocate a wired-down buffer inside kernel space. */
205 bool contig
= (0 != (options
& kIOMemoryHostPhysicallyContiguous
));
207 if (!contig
&& (0 != (options
& kIOMemoryPhysicallyContiguous
)))
210 contig
|= (0 != (kIOMemoryMapperNone
& options
));
212 // treat kIOMemoryPhysicallyContiguous as kIOMemoryHostPhysicallyContiguous for now
217 if (contig
|| highestMask
|| (alignment
> page_size
))
219 _internalFlags
|= kInternalFlagPhysical
;
222 _internalFlags
|= kInternalFlagPageSized
;
223 capacity
= round_page(capacity
);
225 _buffer
= (void *) IOKernelAllocateWithPhysicalRestrict(
226 capacity
, highestMask
, alignment
, contig
);
229 && ((capacity
+ alignment
) <= (page_size
- gIOPageAllocChunkBytes
)))
231 _internalFlags
|= kInternalFlagPageAllocated
;
233 _buffer
= (void *) iopa_alloc(&gIOBMDPageAllocator
, &IOBMDPageProc
, capacity
, alignment
);
236 IOStatisticsAlloc(kIOStatisticsMallocAligned
, capacity
);
238 OSAddAtomic(capacity
, &debug_iomalloc_size
);
242 else if (alignment
> 1)
244 _buffer
= IOMallocAligned(capacity
, alignment
);
248 _buffer
= IOMalloc(capacity
);
254 if (needZero
) bzero(_buffer
, capacity
);
257 if( (options
& (kIOMemoryPageable
| kIOMapCacheMask
))) {
258 vm_size_t size
= round_page(capacity
);
260 // initWithOptions will create memory entry
261 iomdOptions
|= kIOMemoryPersistent
;
263 if( options
& kIOMemoryPageable
) {
265 OSAddAtomicLong(size
, &debug_iomallocpageable_size
);
269 inTask
= kernel_task
;
271 else if (options
& kIOMapCacheMask
)
273 // Prefetch each page to put entries into the pmap
274 volatile UInt8
* startAddr
= (UInt8
*)_buffer
;
275 volatile UInt8
* endAddr
= (UInt8
*)_buffer
+ capacity
;
277 while (startAddr
< endAddr
)
279 UInt8 dummyVar
= *startAddr
;
281 startAddr
+= page_size
;
286 _ranges
.v64
->address
= (mach_vm_address_t
) _buffer
;;
287 _ranges
.v64
->length
= _capacity
;
289 if (!super::initWithOptions(_ranges
.v64
, 1, 0,
290 inTask
, iomdOptions
, /* System mapper */ 0))
293 _internalFlags
|= kInternalFlagInit
;
295 if (!(options
& kIOMemoryPageable
)) trackingAccumSize(capacity
);
296 #endif /* IOTRACKING */
298 // give any system mapper the allocation params
299 if (kIOReturnSuccess
!= dmaCommandOperation(kIOMDAddDMAMapSpec
,
300 &mapSpec
, sizeof(mapSpec
)))
306 reserved
= IONew( ExpansionData
, 1 );
310 reserved
->map
= createMappingInTask(mapTask
, 0,
311 kIOMapAnywhere
| (options
& kIOMapPrefault
) | (options
& kIOMapCacheMask
), 0, 0);
317 release(); // map took a retain on this
318 reserved
->map
->retain();
319 removeMapping(reserved
->map
);
320 mach_vm_address_t buffer
= reserved
->map
->getAddress();
321 _buffer
= (void *) buffer
;
322 if (kIOMemoryTypeVirtual64
== (kIOMemoryTypeMask
& iomdOptions
))
323 _ranges
.v64
->address
= buffer
;
326 setLength(_capacity
);
331 IOBufferMemoryDescriptor
* IOBufferMemoryDescriptor::inTaskWithOptions(
333 IOOptionBits options
,
335 vm_offset_t alignment
)
337 IOBufferMemoryDescriptor
*me
= new IOBufferMemoryDescriptor
;
339 if (me
&& !me
->initWithPhysicalMask(inTask
, options
, capacity
, alignment
, 0)) {
346 IOBufferMemoryDescriptor
* IOBufferMemoryDescriptor::inTaskWithPhysicalMask(
348 IOOptionBits options
,
349 mach_vm_size_t capacity
,
350 mach_vm_address_t physicalMask
)
352 IOBufferMemoryDescriptor
*me
= new IOBufferMemoryDescriptor
;
354 if (me
&& !me
->initWithPhysicalMask(inTask
, options
, capacity
, 1, physicalMask
))
363 bool IOBufferMemoryDescriptor::initWithOptions(
364 IOOptionBits options
,
366 vm_offset_t alignment
)
368 return (initWithPhysicalMask(kernel_task
, options
, capacity
, alignment
, (mach_vm_address_t
)0));
370 #endif /* !__LP64__ */
372 IOBufferMemoryDescriptor
* IOBufferMemoryDescriptor::withOptions(
373 IOOptionBits options
,
375 vm_offset_t alignment
)
377 IOBufferMemoryDescriptor
*me
= new IOBufferMemoryDescriptor
;
379 if (me
&& !me
->initWithPhysicalMask(kernel_task
, options
, capacity
, alignment
, 0)) {
390 * Returns a new IOBufferMemoryDescriptor with a buffer large enough to
391 * hold capacity bytes. The descriptor's length is initially set to the capacity.
393 IOBufferMemoryDescriptor
*
394 IOBufferMemoryDescriptor::withCapacity(vm_size_t inCapacity
,
395 IODirection inDirection
,
398 return( IOBufferMemoryDescriptor::withOptions(
399 inDirection
| kIOMemoryUnshared
400 | (inContiguous
? kIOMemoryPhysicallyContiguous
: 0),
401 inCapacity
, inContiguous
? inCapacity
: 1 ));
408 * Initialize a new IOBufferMemoryDescriptor preloaded with bytes (copied).
409 * The descriptor's length and capacity are set to the input buffer's size.
411 bool IOBufferMemoryDescriptor::initWithBytes(const void * inBytes
,
413 IODirection inDirection
,
416 if (!initWithPhysicalMask(kernel_task
, inDirection
| kIOMemoryUnshared
417 | (inContiguous
? kIOMemoryPhysicallyContiguous
: 0),
418 inLength
, inLength
, (mach_vm_address_t
)0))
421 // start out with no data
424 if (!appendBytes(inBytes
, inLength
))
429 #endif /* !__LP64__ */
434 * Returns a new IOBufferMemoryDescriptor preloaded with bytes (copied).
435 * The descriptor's length and capacity are set to the input buffer's size.
437 IOBufferMemoryDescriptor
*
438 IOBufferMemoryDescriptor::withBytes(const void * inBytes
,
440 IODirection inDirection
,
443 IOBufferMemoryDescriptor
*me
= new IOBufferMemoryDescriptor
;
445 if (me
&& !me
->initWithPhysicalMask(
446 kernel_task
, inDirection
| kIOMemoryUnshared
447 | (inContiguous
? kIOMemoryPhysicallyContiguous
: 0),
448 inLength
, inLength
, 0 ))
456 // start out with no data
459 if (!me
->appendBytes(inBytes
, inLength
))
473 void IOBufferMemoryDescriptor::free()
475 // Cache all of the relevant information on the stack for use
476 // after we call super::free()!
477 IOOptionBits flags
= _flags
;
478 IOOptionBits internalFlags
= _internalFlags
;
479 IOOptionBits options
= _options
;
480 vm_size_t size
= _capacity
;
481 void * buffer
= _buffer
;
482 IOMemoryMap
* map
= 0;
483 IOAddressRange
* range
= _ranges
.v64
;
484 vm_offset_t alignment
= _alignment
;
486 if (alignment
>= page_size
)
487 size
= round_page(size
);
492 IODelete( reserved
, ExpansionData
, 1 );
497 if ((options
& kIOMemoryPageable
)
498 || (kInternalFlagPageSized
& internalFlags
)) size
= round_page(size
);
501 if (!(options
& kIOMemoryPageable
)
503 && (kInternalFlagInit
& _internalFlags
)) trackingAccumSize(-size
);
504 #endif /* IOTRACKING */
506 /* super::free may unwire - deallocate buffer afterwards */
509 if (options
& kIOMemoryPageable
)
512 OSAddAtomicLong(-size
, &debug_iomallocpageable_size
);
517 if (kInternalFlagPhysical
& internalFlags
)
519 IOKernelFreePhysical((mach_vm_address_t
) buffer
, size
);
521 else if (kInternalFlagPageAllocated
& internalFlags
)
524 page
= iopa_free(&gIOBMDPageAllocator
, (uintptr_t) buffer
, size
);
527 kmem_free(kernel_map
, page
, page_size
);
530 OSAddAtomic(-size
, &debug_iomalloc_size
);
532 IOStatisticsAlloc(kIOStatisticsFreeAligned
, size
);
534 else if (alignment
> 1)
536 IOFreeAligned(buffer
, size
);
540 IOFree(buffer
, size
);
543 if (range
&& (kIOMemoryAsReference
& flags
))
544 IODelete(range
, IOAddressRange
, 1);
550 * Get the buffer capacity
552 vm_size_t
IOBufferMemoryDescriptor::getCapacity() const
560 * Change the buffer length of the memory descriptor. When a new buffer
561 * is created, the initial length of the buffer is set to be the same as
562 * the capacity. The length can be adjusted via setLength for a shorter
563 * transfer (there is no need to create more buffer descriptors when you
564 * can reuse an existing one, even for different transfer sizes). Note
565 * that the specified length must not exceed the capacity of the buffer.
567 void IOBufferMemoryDescriptor::setLength(vm_size_t length
)
569 assert(length
<= _capacity
);
570 if (length
> _capacity
) return;
573 _ranges
.v64
->length
= length
;
579 * Change the direction of the transfer. This method allows one to redirect
580 * the descriptor's transfer direction. This eliminates the need to destroy
581 * and create new buffers when different transfer directions are needed.
583 void IOBufferMemoryDescriptor::setDirection(IODirection direction
)
585 _flags
= (_flags
& ~kIOMemoryDirectionMask
) | direction
;
587 _direction
= (IODirection
) (_flags
& kIOMemoryDirectionMask
);
588 #endif /* !__LP64__ */
594 * Add some data to the end of the buffer. This method automatically
595 * maintains the memory descriptor buffer length. Note that appendBytes
596 * will not copy past the end of the memory descriptor's current capacity.
599 IOBufferMemoryDescriptor::appendBytes(const void * bytes
, vm_size_t withLength
)
601 vm_size_t actualBytesToCopy
= min(withLength
, _capacity
- _length
);
604 assert(_length
<= _capacity
);
607 _length
+= actualBytesToCopy
;
608 _ranges
.v64
->length
+= actualBytesToCopy
;
610 if (_task
== kernel_task
)
611 bcopy(/* from */ bytes
, (void *)(_ranges
.v64
->address
+ offset
),
614 writeBytes(offset
, bytes
, actualBytesToCopy
);
622 * Return the virtual address of the beginning of the buffer
624 void * IOBufferMemoryDescriptor::getBytesNoCopy()
626 if (kIOMemoryTypePhysical64
== (_flags
& kIOMemoryTypeMask
))
629 return (void *)_ranges
.v64
->address
;
636 * Return the virtual address of an offset from the beginning of the buffer
639 IOBufferMemoryDescriptor::getBytesNoCopy(vm_size_t start
, vm_size_t withLength
)
641 IOVirtualAddress address
;
643 if ((start
+ withLength
) < start
) return 0;
645 if (kIOMemoryTypePhysical64
== (_flags
& kIOMemoryTypeMask
))
646 address
= (IOVirtualAddress
) _buffer
;
648 address
= _ranges
.v64
->address
;
650 if (start
< _length
&& (start
+ withLength
) <= _length
)
651 return (void *)(address
+ start
);
656 void * IOBufferMemoryDescriptor::getVirtualSegment(IOByteCount offset
,
657 IOByteCount
* lengthOfSegment
)
659 void * bytes
= getBytesNoCopy(offset
, 0);
661 if (bytes
&& lengthOfSegment
)
662 *lengthOfSegment
= _length
- offset
;
666 #endif /* !__LP64__ */
669 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 0);
670 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 1);
671 #else /* !__LP64__ */
672 OSMetaClassDefineReservedUsed(IOBufferMemoryDescriptor
, 0);
673 OSMetaClassDefineReservedUsed(IOBufferMemoryDescriptor
, 1);
674 #endif /* !__LP64__ */
675 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 2);
676 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 3);
677 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 4);
678 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 5);
679 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 6);
680 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 7);
681 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 8);
682 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 9);
683 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 10);
684 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 11);
685 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 12);
686 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 13);
687 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 14);
688 OSMetaClassDefineReservedUnused(IOBufferMemoryDescriptor
, 15);