]> git.saurik.com Git - apple/xnu.git/blob - iokit/Kernel/IOMemoryDescriptor.cpp
projects / apple / xnu.git / blob
? search:


f6ab8e93b8f599b6612ee3ab53c85c4a69970f96
[apple/xnu.git] / iokit / Kernel / IOMemoryDescriptor.cpp
1 /*
2 * Copyright (c) 1998-2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * Copyright (c) 1998 Apple Computer, Inc. All rights reserved.
30 *
31 * HISTORY
32 *
33 */
34
35
36 #include <sys/cdefs.h>
37
38 #include <IOKit/assert.h>
39 #include <IOKit/system.h>
40 #include <IOKit/IOLib.h>
41 #include <IOKit/IOMemoryDescriptor.h>
42 #include <IOKit/IOMapper.h>
43 #include <IOKit/IODMACommand.h>
44 #include <IOKit/IOKitKeysPrivate.h>
45
46 #ifndef __LP64__
47 #include <IOKit/IOSubMemoryDescriptor.h>
48 #endif /* !__LP64__ */
49
50 #include <IOKit/IOKitDebug.h>
51 #include <libkern/OSDebug.h>
52
53 #include "IOKitKernelInternal.h"
54
55 #include <libkern/c++/OSContainers.h>
56 #include <libkern/c++/OSDictionary.h>
57 #include <libkern/c++/OSArray.h>
58 #include <libkern/c++/OSSymbol.h>
59 #include <libkern/c++/OSNumber.h>
60
61 #include <sys/uio.h>
62
63 __BEGIN_DECLS
64 #include <vm/pmap.h>
65 #include <vm/vm_pageout.h>
66 #include <mach/memory_object_types.h>
67 #include <device/device_port.h>
68
69 #include <mach/vm_prot.h>
70 #include <mach/mach_vm.h>
71 #include <vm/vm_fault.h>
72 #include <vm/vm_protos.h>
73
74 extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
75 extern void ipc_port_release_send(ipc_port_t port);
76
77 kern_return_t
78 memory_object_iopl_request(
79 ipc_port_t port,
80 memory_object_offset_t offset,
81 vm_size_t *upl_size,
82 upl_t *upl_ptr,
83 upl_page_info_array_t user_page_list,
84 unsigned int *page_list_count,
85 int *flags);
86
87 // osfmk/device/iokit_rpc.c
88 unsigned int IODefaultCacheBits(addr64_t pa);
89 unsigned int IOTranslateCacheBits(struct phys_entry *pp);
90
91 __END_DECLS
92
93 #define kIOMapperWaitSystem ((IOMapper *) 1)
94
95 static IOMapper * gIOSystemMapper = NULL;
96
97 ppnum_t gIOLastPage;
98
99 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
100
101 OSDefineMetaClassAndAbstractStructors( IOMemoryDescriptor, OSObject )
102
103 #define super IOMemoryDescriptor
104
105 OSDefineMetaClassAndStructors(IOGeneralMemoryDescriptor, IOMemoryDescriptor)
106
107 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
108
109 static IORecursiveLock * gIOMemoryLock;
110
111 #define LOCK IORecursiveLockLock( gIOMemoryLock)
112 #define UNLOCK IORecursiveLockUnlock( gIOMemoryLock)
113 #define SLEEP IORecursiveLockSleep( gIOMemoryLock, (void *)this, THREAD_UNINT)
114 #define WAKEUP \
115 IORecursiveLockWakeup( gIOMemoryLock, (void *)this, /* one-thread */ false)
116
117 #if 0
118 #define DEBG(fmt, args...) { kprintf(fmt, ## args); }
119 #else
120 #define DEBG(fmt, args...) {}
121 #endif
122
123 #define IOMD_DEBUG_DMAACTIVE 1
124
125 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
126
127 // Some data structures and accessor macros used by the initWithOptions
128 // Function
129
130 enum ioPLBlockFlags {
131 kIOPLOnDevice = 0x00000001,
132 kIOPLExternUPL = 0x00000002,
133 };
134
135 struct IOMDPersistentInitData
136 {
137 const IOGeneralMemoryDescriptor * fMD;
138 IOMemoryReference * fMemRef;
139 };
140
141 struct ioPLBlock {
142 upl_t fIOPL;
143 vm_address_t fPageInfo; // Pointer to page list or index into it
144 uint32_t fIOMDOffset; // The offset of this iopl in descriptor
145 ppnum_t fMappedPage; // Page number of first page in this iopl
146 unsigned int fPageOffset; // Offset within first page of iopl
147 unsigned int fFlags; // Flags
148 };
149
150 struct ioGMDData {
151 IOMapper * fMapper;
152 uint8_t fDMAMapNumAddressBits;
153 uint64_t fDMAMapAlignment;
154 addr64_t fMappedBase;
155 uint64_t fPreparationID;
156 unsigned int fPageCnt;
157 unsigned char fDiscontig:1;
158 unsigned char fCompletionError:1;
159 unsigned char _resv:6;
160 #if __LP64__
161 // align arrays to 8 bytes so following macros work
162 unsigned char fPad[3];
163 #endif
164 upl_page_info_t fPageList[1]; /* variable length */
165 ioPLBlock fBlocks[1]; /* variable length */
166 };
167
168 #define getDataP(osd) ((ioGMDData *) (osd)->getBytesNoCopy())
169 #define getIOPLList(d) ((ioPLBlock *) (void *)&(d->fPageList[d->fPageCnt]))
170 #define getNumIOPL(osd, d) \
171 (((osd)->getLength() - ((char *) getIOPLList(d) - (char *) d)) / sizeof(ioPLBlock))
172 #define getPageList(d) (&(d->fPageList[0]))
173 #define computeDataSize(p, u) \
174 (offsetof(ioGMDData, fPageList) + p * sizeof(upl_page_info_t) + u * sizeof(ioPLBlock))
175
176 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
177
178 #define next_page(a) ( trunc_page(a) + PAGE_SIZE )
179
180 extern "C" {
181
182 kern_return_t device_data_action(
183 uintptr_t device_handle,
184 ipc_port_t device_pager,
185 vm_prot_t protection,
186 vm_object_offset_t offset,
187 vm_size_t size)
188 {
189 kern_return_t kr;
190 IOMemoryDescriptorReserved * ref = (IOMemoryDescriptorReserved *) device_handle;
191 IOMemoryDescriptor * memDesc;
192
193 LOCK;
194 memDesc = ref->dp.memory;
195 if( memDesc)
196 {
197 memDesc->retain();
198 kr = memDesc->handleFault(device_pager, offset, size);
199 memDesc->release();
200 }
201 else
202 kr = KERN_ABORTED;
203 UNLOCK;
204
205 return( kr );
206 }
207
208 kern_return_t device_close(
209 uintptr_t device_handle)
210 {
211 IOMemoryDescriptorReserved * ref = (IOMemoryDescriptorReserved *) device_handle;
212
213 IODelete( ref, IOMemoryDescriptorReserved, 1 );
214
215 return( kIOReturnSuccess );
216 }
217 }; // end extern "C"
218
219 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
220
221 // Note this inline function uses C++ reference arguments to return values
222 // This means that pointers are not passed and NULLs don't have to be
223 // checked for as a NULL reference is illegal.
224 static inline void
225 getAddrLenForInd(mach_vm_address_t &addr, mach_vm_size_t &len, // Output variables
226 UInt32 type, IOGeneralMemoryDescriptor::Ranges r, UInt32 ind)
227 {
228 assert(kIOMemoryTypeUIO == type
229 || kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type
230 || kIOMemoryTypePhysical == type || kIOMemoryTypePhysical64 == type);
231 if (kIOMemoryTypeUIO == type) {
232 user_size_t us;
233 user_addr_t ad;
234 uio_getiov((uio_t) r.uio, ind, &ad, &us); addr = ad; len = us;
235 }
236 #ifndef __LP64__
237 else if ((kIOMemoryTypeVirtual64 == type) || (kIOMemoryTypePhysical64 == type)) {
238 IOAddressRange cur = r.v64[ind];
239 addr = cur.address;
240 len = cur.length;
241 }
242 #endif /* !__LP64__ */
243 else {
244 IOVirtualRange cur = r.v[ind];
245 addr = cur.address;
246 len = cur.length;
247 }
248 }
249
250 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
251
252 static IOReturn
253 purgeableControlBits(IOOptionBits newState, vm_purgable_t * control, int * state)
254 {
255 IOReturn err = kIOReturnSuccess;
256
257 *control = VM_PURGABLE_SET_STATE;
258
259 enum { kIOMemoryPurgeableControlMask = 15 };
260
261 switch (kIOMemoryPurgeableControlMask & newState)
262 {
263 case kIOMemoryPurgeableKeepCurrent:
264 *control = VM_PURGABLE_GET_STATE;
265 break;
266
267 case kIOMemoryPurgeableNonVolatile:
268 *state = VM_PURGABLE_NONVOLATILE;
269 break;
270 case kIOMemoryPurgeableVolatile:
271 *state = VM_PURGABLE_VOLATILE | (newState & ~kIOMemoryPurgeableControlMask);
272 break;
273 case kIOMemoryPurgeableEmpty:
274 *state = VM_PURGABLE_EMPTY;
275 break;
276 default:
277 err = kIOReturnBadArgument;
278 break;
279 }
280 return (err);
281 }
282
283 static IOReturn
284 purgeableStateBits(int * state)
285 {
286 IOReturn err = kIOReturnSuccess;
287
288 switch (VM_PURGABLE_STATE_MASK & *state)
289 {
290 case VM_PURGABLE_NONVOLATILE:
291 *state = kIOMemoryPurgeableNonVolatile;
292 break;
293 case VM_PURGABLE_VOLATILE:
294 *state = kIOMemoryPurgeableVolatile;
295 break;
296 case VM_PURGABLE_EMPTY:
297 *state = kIOMemoryPurgeableEmpty;
298 break;
299 default:
300 *state = kIOMemoryPurgeableNonVolatile;
301 err = kIOReturnNotReady;
302 break;
303 }
304 return (err);
305 }
306
307
308 static vm_prot_t
309 vmProtForCacheMode(IOOptionBits cacheMode)
310 {
311 vm_prot_t prot = 0;
312 switch (cacheMode)
313 {
314 case kIOInhibitCache:
315 SET_MAP_MEM(MAP_MEM_IO, prot);
316 break;
317
318 case kIOWriteThruCache:
319 SET_MAP_MEM(MAP_MEM_WTHRU, prot);
320 break;
321
322 case kIOWriteCombineCache:
323 SET_MAP_MEM(MAP_MEM_WCOMB, prot);
324 break;
325
326 case kIOCopybackCache:
327 SET_MAP_MEM(MAP_MEM_COPYBACK, prot);
328 break;
329
330 case kIOCopybackInnerCache:
331 SET_MAP_MEM(MAP_MEM_INNERWBACK, prot);
332 break;
333
334 case kIODefaultCache:
335 default:
336 SET_MAP_MEM(MAP_MEM_NOOP, prot);
337 break;
338 }
339
340 return (prot);
341 }
342
343 static unsigned int
344 pagerFlagsForCacheMode(IOOptionBits cacheMode)
345 {
346 unsigned int pagerFlags = 0;
347 switch (cacheMode)
348 {
349 case kIOInhibitCache:
350 pagerFlags = DEVICE_PAGER_CACHE_INHIB | DEVICE_PAGER_COHERENT | DEVICE_PAGER_GUARDED;
351 break;
352
353 case kIOWriteThruCache:
354 pagerFlags = DEVICE_PAGER_WRITE_THROUGH | DEVICE_PAGER_COHERENT | DEVICE_PAGER_GUARDED;
355 break;
356
357 case kIOWriteCombineCache:
358 pagerFlags = DEVICE_PAGER_CACHE_INHIB | DEVICE_PAGER_COHERENT;
359 break;
360
361 case kIOCopybackCache:
362 pagerFlags = DEVICE_PAGER_COHERENT;
363 break;
364
365 case kIOCopybackInnerCache:
366 pagerFlags = DEVICE_PAGER_COHERENT;
367 break;
368
369 case kIODefaultCache:
370 default:
371 pagerFlags = -1U;
372 break;
373 }
374 return (pagerFlags);
375 }
376
377 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
378 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
379
380 struct IOMemoryEntry
381 {
382 ipc_port_t entry;
383 int64_t offset;
384 uint64_t size;
385 };
386
387 struct IOMemoryReference
388 {
389 volatile SInt32 refCount;
390 vm_prot_t prot;
391 uint32_t capacity;
392 uint32_t count;
393 IOMemoryEntry entries[0];
394 };
395
396 enum
397 {
398 kIOMemoryReferenceReuse = 0x00000001,
399 kIOMemoryReferenceWrite = 0x00000002,
400 };
401
402 SInt32 gIOMemoryReferenceCount;
403
404 IOMemoryReference *
405 IOGeneralMemoryDescriptor::memoryReferenceAlloc(uint32_t capacity, IOMemoryReference * realloc)
406 {
407 IOMemoryReference * ref;
408 size_t newSize, oldSize, copySize;
409
410 newSize = (sizeof(IOMemoryReference)
411 - sizeof(ref->entries)
412 + capacity * sizeof(ref->entries[0]));
413 ref = (typeof(ref)) IOMalloc(newSize);
414 if (realloc)
415 {
416 oldSize = (sizeof(IOMemoryReference)
417 - sizeof(realloc->entries)
418 + realloc->capacity * sizeof(realloc->entries[0]));
419 copySize = oldSize;
420 if (copySize > newSize) copySize = newSize;
421 if (ref) bcopy(realloc, ref, copySize);
422 IOFree(realloc, oldSize);
423 }
424 else if (ref)
425 {
426 bzero(ref, sizeof(*ref));
427 ref->refCount = 1;
428 OSIncrementAtomic(&gIOMemoryReferenceCount);
429 }
430 if (!ref) return (0);
431 ref->capacity = capacity;
432 return (ref);
433 }
434
435 void
436 IOGeneralMemoryDescriptor::memoryReferenceFree(IOMemoryReference * ref)
437 {
438 IOMemoryEntry * entries;
439 size_t size;
440
441 entries = ref->entries + ref->count;
442 while (entries > &ref->entries[0])
443 {
444 entries--;
445 ipc_port_release_send(entries->entry);
446 }
447 size = (sizeof(IOMemoryReference)
448 - sizeof(ref->entries)
449 + ref->capacity * sizeof(ref->entries[0]));
450 IOFree(ref, size);
451
452 OSDecrementAtomic(&gIOMemoryReferenceCount);
453 }
454
455 void
456 IOGeneralMemoryDescriptor::memoryReferenceRelease(IOMemoryReference * ref)
457 {
458 if (1 == OSDecrementAtomic(&ref->refCount)) memoryReferenceFree(ref);
459 }
460
461
462 IOReturn
463 IOGeneralMemoryDescriptor::memoryReferenceCreate(
464 IOOptionBits options,
465 IOMemoryReference ** reference)
466 {
467 enum { kCapacity = 4, kCapacityInc = 4 };
468
469 kern_return_t err;
470 IOMemoryReference * ref;
471 IOMemoryEntry * entries;
472 IOMemoryEntry * cloneEntries;
473 vm_map_t map;
474 ipc_port_t entry, cloneEntry;
475 vm_prot_t prot;
476 memory_object_size_t actualSize;
477 uint32_t rangeIdx;
478 uint32_t count;
479 mach_vm_address_t entryAddr, endAddr, entrySize;
480 mach_vm_size_t srcAddr, srcLen;
481 mach_vm_size_t nextAddr, nextLen;
482 mach_vm_size_t offset, remain;
483 IOByteCount physLen;
484 IOOptionBits type = (_flags & kIOMemoryTypeMask);
485 IOOptionBits cacheMode;
486 unsigned int pagerFlags;
487
488 ref = memoryReferenceAlloc(kCapacity, NULL);
489 if (!ref) return (kIOReturnNoMemory);
490 entries = &ref->entries[0];
491 count = 0;
492
493 offset = 0;
494 rangeIdx = 0;
495 if (_task) getAddrLenForInd(nextAddr, nextLen, type, _ranges, rangeIdx);
496 else
497 {
498 nextAddr = getPhysicalSegment(offset, &physLen, kIOMemoryMapperNone);
499 nextLen = physLen;
500 // default cache mode for physical
501 if (kIODefaultCache == ((_flags & kIOMemoryBufferCacheMask) >> kIOMemoryBufferCacheShift))
502 {
503 IOOptionBits mode;
504 pagerFlags = IODefaultCacheBits(nextAddr);
505 if (DEVICE_PAGER_CACHE_INHIB & pagerFlags)
506 {
507 if (DEVICE_PAGER_GUARDED & pagerFlags)
508 mode = kIOInhibitCache;
509 else
510 mode = kIOWriteCombineCache;
511 }
512 else if (DEVICE_PAGER_WRITE_THROUGH & pagerFlags)
513 mode = kIOWriteThruCache;
514 else
515 mode = kIOCopybackCache;
516 _flags |= (mode << kIOMemoryBufferCacheShift);
517 }
518 }
519
520 // cache mode & vm_prot
521 prot = VM_PROT_READ;
522 cacheMode = ((_flags & kIOMemoryBufferCacheMask) >> kIOMemoryBufferCacheShift);
523 prot |= vmProtForCacheMode(cacheMode);
524 // VM system requires write access to change cache mode
525 if (kIODefaultCache != cacheMode) prot |= VM_PROT_WRITE;
526 if (kIODirectionOut != (kIODirectionOutIn & _flags)) prot |= VM_PROT_WRITE;
527 if (kIOMemoryReferenceWrite & options) prot |= VM_PROT_WRITE;
528
529 if ((kIOMemoryReferenceReuse & options) && _memRef)
530 {
531 cloneEntries = &_memRef->entries[0];
532 prot |= MAP_MEM_NAMED_REUSE;
533 }
534
535 if (_task)
536 {
537 // virtual ranges
538
539 if (kIOMemoryBufferPageable & _flags)
540 {
541 // IOBufferMemoryDescriptor alloc - set flags for entry + object create
542 prot |= MAP_MEM_NAMED_CREATE;
543 if (kIOMemoryBufferPurgeable & _flags) prot |= MAP_MEM_PURGABLE;
544 prot |= VM_PROT_WRITE;
545 map = NULL;
546 }
547 else map = get_task_map(_task);
548
549 remain = _length;
550 while (remain)
551 {
552 srcAddr = nextAddr;
553 srcLen = nextLen;
554 nextAddr = 0;
555 nextLen = 0;
556 // coalesce addr range
557 for (++rangeIdx; rangeIdx < _rangesCount; rangeIdx++)
558 {
559 getAddrLenForInd(nextAddr, nextLen, type, _ranges, rangeIdx);
560 if ((srcAddr + srcLen) != nextAddr) break;
561 srcLen += nextLen;
562 }
563 entryAddr = trunc_page_64(srcAddr);
564 endAddr = round_page_64(srcAddr + srcLen);
565 do
566 {
567 entrySize = (endAddr - entryAddr);
568 if (!entrySize) break;
569 actualSize = entrySize;
570
571 cloneEntry = MACH_PORT_NULL;
572 if (MAP_MEM_NAMED_REUSE & prot)
573 {
574 if (cloneEntries < &_memRef->entries[_memRef->count]) cloneEntry = cloneEntries->entry;
575 else prot &= ~MAP_MEM_NAMED_REUSE;
576 }
577
578 err = mach_make_memory_entry_64(map,
579 &actualSize, entryAddr, prot, &entry, cloneEntry);
580
581 if (KERN_SUCCESS != err) break;
582 if (actualSize > entrySize) panic("mach_make_memory_entry_64 actualSize");
583
584 if (count >= ref->capacity)
585 {
586 ref = memoryReferenceAlloc(ref->capacity + kCapacityInc, ref);
587 entries = &ref->entries[count];
588 }
589 entries->entry = entry;
590 entries->size = actualSize;
591 entries->offset = offset + (entryAddr - srcAddr);
592 entryAddr += actualSize;
593 if (MAP_MEM_NAMED_REUSE & prot)
594 {
595 if ((cloneEntries->entry == entries->entry)
596 && (cloneEntries->size == entries->size)
597 && (cloneEntries->offset == entries->offset)) cloneEntries++;
598 else prot &= ~MAP_MEM_NAMED_REUSE;
599 }
600 entries++;
601 count++;
602 }
603 while (true);
604 offset += srcLen;
605 remain -= srcLen;
606 }
607 }
608 else
609 {
610 // _task == 0, physical
611 memory_object_t pager;
612 vm_size_t size = ptoa_32(_pages);
613
614 if (!getKernelReserved()) panic("getKernelReserved");
615
616 reserved->dp.pagerContig = (1 == _rangesCount);
617 reserved->dp.memory = this;
618
619 pagerFlags = pagerFlagsForCacheMode(cacheMode);
620 if (-1U == pagerFlags) panic("phys is kIODefaultCache");
621 if (reserved->dp.pagerContig) pagerFlags |= DEVICE_PAGER_CONTIGUOUS;
622
623 pager = device_pager_setup((memory_object_t) 0, (uintptr_t) reserved,
624 size, pagerFlags);
625 assert (pager);
626 if (!pager) err = kIOReturnVMError;
627 else
628 {
629 srcAddr = nextAddr;
630 entryAddr = trunc_page_64(srcAddr);
631 err = mach_memory_object_memory_entry_64((host_t) 1, false /*internal*/,
632 size, VM_PROT_READ | VM_PROT_WRITE, pager, &entry);
633 assert (KERN_SUCCESS == err);
634 if (KERN_SUCCESS != err) device_pager_deallocate(pager);
635 else
636 {
637 reserved->dp.devicePager = pager;
638 entries->entry = entry;
639 entries->size = size;
640 entries->offset = offset + (entryAddr - srcAddr);
641 entries++;
642 count++;
643 }
644 }
645 }
646
647 ref->count = count;
648 ref->prot = prot;
649
650 if (KERN_SUCCESS == err)
651 {
652 if (MAP_MEM_NAMED_REUSE & prot)
653 {
654 memoryReferenceFree(ref);
655 OSIncrementAtomic(&_memRef->refCount);
656 ref = _memRef;
657 }
658 }
659 else
660 {
661 memoryReferenceFree(ref);
662 ref = NULL;
663 }
664
665 *reference = ref;
666
667 return (err);
668 }
669
670 struct IOMemoryDescriptorMapAllocRef
671 {
672 vm_map_t map;
673 mach_vm_address_t mapped;
674 mach_vm_size_t size;
675 vm_prot_t prot;
676 IOOptionBits options;
677 };
678
679 static kern_return_t
680 IOMemoryDescriptorMapAlloc(vm_map_t map, void * _ref)
681 {
682 IOMemoryDescriptorMapAllocRef * ref = (typeof(ref))_ref;
683 IOReturn err;
684 vm_map_offset_t addr;
685
686 addr = ref->mapped;
687 err = vm_map_enter_mem_object(map, &addr, ref->size,
688 (vm_map_offset_t) 0,
689 (((ref->options & kIOMapAnywhere)
690 ? VM_FLAGS_ANYWHERE
691 : VM_FLAGS_FIXED)
692 | VM_MAKE_TAG(VM_MEMORY_IOKIT)
693 | VM_FLAGS_IOKIT_ACCT), /* iokit accounting */
694 IPC_PORT_NULL,
695 (memory_object_offset_t) 0,
696 false, /* copy */
697 ref->prot,
698 ref->prot,
699 VM_INHERIT_NONE);
700 if (KERN_SUCCESS == err)
701 {
702 ref->mapped = (mach_vm_address_t) addr;
703 ref->map = map;
704 }
705
706 return( err );
707 }
708
709 IOReturn
710 IOGeneralMemoryDescriptor::memoryReferenceMap(
711 IOMemoryReference * ref,
712 vm_map_t map,
713 mach_vm_size_t inoffset,
714 mach_vm_size_t size,
715 IOOptionBits options,
716 mach_vm_address_t * inaddr)
717 {
718 IOReturn err;
719 int64_t offset = inoffset;
720 uint32_t rangeIdx, entryIdx;
721 vm_map_offset_t addr, mapAddr;
722 vm_map_offset_t pageOffset, entryOffset, remain, chunk;
723
724 mach_vm_address_t srcAddr, nextAddr;
725 mach_vm_size_t srcLen, nextLen;
726 IOByteCount physLen;
727 IOMemoryEntry * entry;
728 vm_prot_t prot, memEntryCacheMode;
729 IOOptionBits type;
730 IOOptionBits cacheMode;
731
732 /*
733 * For the kIOMapPrefault option.
734 */
735 upl_page_info_t *pageList = NULL;
736 UInt currentPageIndex = 0;
737
738 type = _flags & kIOMemoryTypeMask;
739 prot = VM_PROT_READ;
740 if (!(kIOMapReadOnly & options)) prot |= VM_PROT_WRITE;
741 prot &= ref->prot;
742
743 cacheMode = ((options & kIOMapCacheMask) >> kIOMapCacheShift);
744 if (kIODefaultCache != cacheMode)
745 {
746 // VM system requires write access to change cache mode
747 prot |= VM_PROT_WRITE;
748 // update named entries cache mode
749 memEntryCacheMode = (MAP_MEM_ONLY | prot | vmProtForCacheMode(cacheMode));
750 }
751
752 if (_task)
753 {
754 // Find first range for offset
755 for (remain = offset, rangeIdx = 0; rangeIdx < _rangesCount; rangeIdx++)
756 {
757 getAddrLenForInd(nextAddr, nextLen, type, _ranges, rangeIdx);
758 if (remain < nextLen) break;
759 remain -= nextLen;
760 }
761 }
762 else
763 {
764 rangeIdx = 0;
765 remain = 0;
766 nextAddr = getPhysicalSegment(offset, &physLen, kIOMemoryMapperNone);
767 nextLen = size;
768 }
769
770 assert(remain < nextLen);
771 if (remain >= nextLen) return (kIOReturnBadArgument);
772
773 nextAddr += remain;
774 nextLen -= remain;
775 pageOffset = (page_mask & nextAddr);
776 addr = 0;
777 if (!(options & kIOMapAnywhere))
778 {
779 addr = *inaddr;
780 if (pageOffset != (page_mask & addr)) return (kIOReturnNotAligned);
781 addr -= pageOffset;
782 }
783
784 // find first entry for offset
785 for (entryIdx = 0;
786 (entryIdx < ref->count) && (offset >= ref->entries[entryIdx].offset);
787 entryIdx++) {}
788 entryIdx--;
789 entry = &ref->entries[entryIdx];
790
791 // allocate VM
792 size = round_page_64(size + pageOffset);
793 {
794 IOMemoryDescriptorMapAllocRef ref;
795 ref.map = map;
796 ref.options = options;
797 ref.size = size;
798 ref.prot = prot;
799 if (options & kIOMapAnywhere)
800 // vm_map looks for addresses above here, even when VM_FLAGS_ANYWHERE
801 ref.mapped = 0;
802 else
803 ref.mapped = addr;
804
805 if ((ref.map == kernel_map) && (kIOMemoryBufferPageable & _flags))
806 err = IOIteratePageableMaps( ref.size, &IOMemoryDescriptorMapAlloc, &ref );
807 else
808 err = IOMemoryDescriptorMapAlloc(ref.map, &ref);
809 if (KERN_SUCCESS == err)
810 {
811 addr = ref.mapped;
812 map = ref.map;
813 }
814 }
815
816 /*
817 * Prefaulting is only possible if we wired the memory earlier. Check the
818 * memory type, and the underlying data.
819 */
820 if (options & kIOMapPrefault) {
821 /*
822 * The memory must have been wired by calling ::prepare(), otherwise
823 * we don't have the UPL. Without UPLs, pages cannot be pre-faulted
824 */
825 assert(map != kernel_map);
826 assert(_wireCount != 0);
827 assert(_memoryEntries != NULL);
828 if ((map == kernel_map) ||
829 (_wireCount == 0) ||
830 (_memoryEntries == NULL))
831 {
832 return kIOReturnBadArgument;
833 }
834
835 // Get the page list.
836 ioGMDData* dataP = getDataP(_memoryEntries);
837 ioPLBlock const* ioplList = getIOPLList(dataP);
838 pageList = getPageList(dataP);
839
840 // Get the number of IOPLs.
841 UInt numIOPLs = getNumIOPL(_memoryEntries, dataP);
842
843 /*
844 * Scan through the IOPL Info Blocks, looking for the first block containing
845 * the offset. The research will go past it, so we'll need to go back to the
846 * right range at the end.
847 */
848 UInt ioplIndex = 0;
849 while (ioplIndex < numIOPLs && offset >= ioplList[ioplIndex].fIOMDOffset)
850 ioplIndex++;
851 ioplIndex--;
852
853 // Retrieve the IOPL info block.
854 ioPLBlock ioplInfo = ioplList[ioplIndex];
855
856 /*
857 * For external UPLs, the fPageInfo points directly to the UPL's page_info_t
858 * array.
859 */
860 if (ioplInfo.fFlags & kIOPLExternUPL)
861 pageList = (upl_page_info_t*) ioplInfo.fPageInfo;
862 else
863 pageList = &pageList[ioplInfo.fPageInfo];
864
865 // Rebase [offset] into the IOPL in order to looks for the first page index.
866 mach_vm_size_t offsetInIOPL = offset - ioplInfo.fIOMDOffset + ioplInfo.fPageOffset;
867
868 // Retrieve the index of the first page corresponding to the offset.
869 currentPageIndex = atop_32(offsetInIOPL);
870 }
871
872 // enter mappings
873 remain = size;
874 mapAddr = addr;
875 addr += pageOffset;
876 while (remain && nextLen && (KERN_SUCCESS == err))
877 {
878 srcAddr = nextAddr;
879 srcLen = nextLen;
880 nextAddr = 0;
881 nextLen = 0;
882 // coalesce addr range
883 for (++rangeIdx; rangeIdx < _rangesCount; rangeIdx++)
884 {
885 getAddrLenForInd(nextAddr, nextLen, type, _ranges, rangeIdx);
886 if ((srcAddr + srcLen) != nextAddr) break;
887 srcLen += nextLen;
888 }
889
890 while (srcLen && (KERN_SUCCESS == err))
891 {
892 entryOffset = offset - entry->offset;
893 if ((page_mask & entryOffset) != pageOffset)
894 {
895 err = kIOReturnNotAligned;
896 break;
897 }
898
899 if (kIODefaultCache != cacheMode)
900 {
901 vm_size_t unused = 0;
902 err = mach_make_memory_entry(NULL /*unused*/, &unused, 0 /*unused*/,
903 memEntryCacheMode, NULL, entry->entry);
904 assert (KERN_SUCCESS == err);
905 }
906
907 entryOffset -= pageOffset;
908 if (entryOffset >= entry->size) panic("entryOffset");
909 chunk = entry->size - entryOffset;
910 if (chunk)
911 {
912 if (chunk > remain) chunk = remain;
913
914 if (options & kIOMapPrefault) {
915 UInt nb_pages = round_page(chunk) / PAGE_SIZE;
916 err = vm_map_enter_mem_object_prefault(map,
917 &mapAddr,
918 chunk, 0 /* mask */,
919 (VM_FLAGS_FIXED
920 | VM_FLAGS_OVERWRITE
921 | VM_MAKE_TAG(VM_MEMORY_IOKIT)
922 | VM_FLAGS_IOKIT_ACCT), /* iokit accounting */
923 entry->entry,
924 entryOffset,
925 prot, // cur
926 prot, // max
927 &pageList[currentPageIndex],
928 nb_pages);
929
930 // Compute the next index in the page list.
931 currentPageIndex += nb_pages;
932 assert(currentPageIndex <= _pages);
933 } else {
934 err = vm_map_enter_mem_object(map,
935 &mapAddr,
936 chunk, 0 /* mask */,
937 (VM_FLAGS_FIXED
938 | VM_FLAGS_OVERWRITE
939 | VM_MAKE_TAG(VM_MEMORY_IOKIT)
940 | VM_FLAGS_IOKIT_ACCT), /* iokit accounting */
941 entry->entry,
942 entryOffset,
943 false, // copy
944 prot, // cur
945 prot, // max
946 VM_INHERIT_NONE);
947 }
948
949 if (KERN_SUCCESS != err) break;
950 remain -= chunk;
951 if (!remain) break;
952 mapAddr += chunk;
953 offset += chunk - pageOffset;
954 }
955 pageOffset = 0;
956 entry++;
957 entryIdx++;
958 if (entryIdx >= ref->count)
959 {
960 err = kIOReturnOverrun;
961 break;
962 }
963 }
964 }
965
966 if ((KERN_SUCCESS != err) && addr)
967 {
968 (void) mach_vm_deallocate(map, trunc_page_64(addr), size);
969 addr = 0;
970 }
971 *inaddr = addr;
972
973 return (err);
974 }
975
976 IOReturn
977 IOGeneralMemoryDescriptor::memoryReferenceGetPageCounts(
978 IOMemoryReference * ref,
979 IOByteCount * residentPageCount,
980 IOByteCount * dirtyPageCount)
981 {
982 IOReturn err;
983 IOMemoryEntry * entries;
984 unsigned int resident, dirty;
985 unsigned int totalResident, totalDirty;
986
987 totalResident = totalDirty = 0;
988 entries = ref->entries + ref->count;
989 while (entries > &ref->entries[0])
990 {
991 entries--;
992 err = mach_memory_entry_get_page_counts(entries->entry, &resident, &dirty);
993 if (KERN_SUCCESS != err) break;
994 totalResident += resident;
995 totalDirty += dirty;
996 }
997
998 if (residentPageCount) *residentPageCount = totalResident;
999 if (dirtyPageCount) *dirtyPageCount = totalDirty;
1000 return (err);
1001 }
1002
1003 IOReturn
1004 IOGeneralMemoryDescriptor::memoryReferenceSetPurgeable(
1005 IOMemoryReference * ref,
1006 IOOptionBits newState,
1007 IOOptionBits * oldState)
1008 {
1009 IOReturn err;
1010 IOMemoryEntry * entries;
1011 vm_purgable_t control;
1012 int totalState, state;
1013
1014 entries = ref->entries + ref->count;
1015 totalState = kIOMemoryPurgeableNonVolatile;
1016 while (entries > &ref->entries[0])
1017 {
1018 entries--;
1019
1020 err = purgeableControlBits(newState, &control, &state);
1021 if (KERN_SUCCESS != err) break;
1022 err = mach_memory_entry_purgable_control(entries->entry, control, &state);
1023 if (KERN_SUCCESS != err) break;
1024 err = purgeableStateBits(&state);
1025 if (KERN_SUCCESS != err) break;
1026
1027 if (kIOMemoryPurgeableEmpty == state) totalState = kIOMemoryPurgeableEmpty;
1028 else if (kIOMemoryPurgeableEmpty == totalState) continue;
1029 else if (kIOMemoryPurgeableVolatile == totalState) continue;
1030 else if (kIOMemoryPurgeableVolatile == state) totalState = kIOMemoryPurgeableVolatile;
1031 else totalState = kIOMemoryPurgeableNonVolatile;
1032 }
1033
1034 if (oldState) *oldState = totalState;
1035 return (err);
1036 }
1037
1038 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
1039
1040 IOMemoryDescriptor *
1041 IOMemoryDescriptor::withAddress(void * address,
1042 IOByteCount length,
1043 IODirection direction)
1044 {
1045 return IOMemoryDescriptor::
1046 withAddressRange((IOVirtualAddress) address, length, direction | kIOMemoryAutoPrepare, kernel_task);
1047 }
1048
1049 #ifndef __LP64__
1050 IOMemoryDescriptor *
1051 IOMemoryDescriptor::withAddress(IOVirtualAddress address,
1052 IOByteCount length,
1053 IODirection direction,
1054 task_t task)
1055 {
1056 IOGeneralMemoryDescriptor * that = new IOGeneralMemoryDescriptor;
1057 if (that)
1058 {
1059 if (that->initWithAddress(address, length, direction, task))
1060 return that;
1061
1062 that->release();
1063 }
1064 return 0;
1065 }
1066 #endif /* !__LP64__ */
1067
1068 IOMemoryDescriptor *
1069 IOMemoryDescriptor::withPhysicalAddress(
1070 IOPhysicalAddress address,
1071 IOByteCount length,
1072 IODirection direction )
1073 {
1074 return (IOMemoryDescriptor::withAddressRange(address, length, direction, TASK_NULL));
1075 }
1076
1077 #ifndef __LP64__
1078 IOMemoryDescriptor *
1079 IOMemoryDescriptor::withRanges( IOVirtualRange * ranges,
1080 UInt32 withCount,
1081 IODirection direction,
1082 task_t task,
1083 bool asReference)
1084 {
1085 IOGeneralMemoryDescriptor * that = new IOGeneralMemoryDescriptor;
1086 if (that)
1087 {
1088 if (that->initWithRanges(ranges, withCount, direction, task, asReference))
1089 return that;
1090
1091 that->release();
1092 }
1093 return 0;
1094 }
1095 #endif /* !__LP64__ */
1096
1097 IOMemoryDescriptor *
1098 IOMemoryDescriptor::withAddressRange(mach_vm_address_t address,
1099 mach_vm_size_t length,
1100 IOOptionBits options,
1101 task_t task)
1102 {
1103 IOAddressRange range = { address, length };
1104 return (IOMemoryDescriptor::withAddressRanges(&range, 1, options, task));
1105 }
1106
1107 IOMemoryDescriptor *
1108 IOMemoryDescriptor::withAddressRanges(IOAddressRange * ranges,
1109 UInt32 rangeCount,
1110 IOOptionBits options,
1111 task_t task)
1112 {
1113 IOGeneralMemoryDescriptor * that = new IOGeneralMemoryDescriptor;
1114 if (that)
1115 {
1116 if (task)
1117 options |= kIOMemoryTypeVirtual64;
1118 else
1119 options |= kIOMemoryTypePhysical64;
1120
1121 if (that->initWithOptions(ranges, rangeCount, 0, task, options, /* mapper */ 0))
1122 return that;
1123
1124 that->release();
1125 }
1126
1127 return 0;
1128 }
1129
1130
1131 /*
1132 * withOptions:
1133 *
1134 * Create a new IOMemoryDescriptor. The buffer is made up of several
1135 * virtual address ranges, from a given task.
1136 *
1137 * Passing the ranges as a reference will avoid an extra allocation.
1138 */
1139 IOMemoryDescriptor *
1140 IOMemoryDescriptor::withOptions(void * buffers,
1141 UInt32 count,
1142 UInt32 offset,
1143 task_t task,
1144 IOOptionBits opts,
1145 IOMapper * mapper)
1146 {
1147 IOGeneralMemoryDescriptor *self = new IOGeneralMemoryDescriptor;
1148
1149 if (self
1150 && !self->initWithOptions(buffers, count, offset, task, opts, mapper))
1151 {
1152 self->release();
1153 return 0;
1154 }
1155
1156 return self;
1157 }
1158
1159 bool IOMemoryDescriptor::initWithOptions(void * buffers,
1160 UInt32 count,
1161 UInt32 offset,
1162 task_t task,
1163 IOOptionBits options,
1164 IOMapper * mapper)
1165 {
1166 return( false );
1167 }
1168
1169 #ifndef __LP64__
1170 IOMemoryDescriptor *
1171 IOMemoryDescriptor::withPhysicalRanges( IOPhysicalRange * ranges,
1172 UInt32 withCount,
1173 IODirection direction,
1174 bool asReference)
1175 {
1176 IOGeneralMemoryDescriptor * that = new IOGeneralMemoryDescriptor;
1177 if (that)
1178 {
1179 if (that->initWithPhysicalRanges(ranges, withCount, direction, asReference))
1180 return that;
1181
1182 that->release();
1183 }
1184 return 0;
1185 }
1186
1187 IOMemoryDescriptor *
1188 IOMemoryDescriptor::withSubRange(IOMemoryDescriptor * of,
1189 IOByteCount offset,
1190 IOByteCount length,
1191 IODirection direction)
1192 {
1193 return (IOSubMemoryDescriptor::withSubRange(of, offset, length, direction | kIOMemoryThreadSafe));
1194 }
1195 #endif /* !__LP64__ */
1196
1197 IOMemoryDescriptor *
1198 IOMemoryDescriptor::withPersistentMemoryDescriptor(IOMemoryDescriptor *originalMD)
1199 {
1200 IOGeneralMemoryDescriptor *origGenMD =
1201 OSDynamicCast(IOGeneralMemoryDescriptor, originalMD);
1202
1203 if (origGenMD)
1204 return IOGeneralMemoryDescriptor::
1205 withPersistentMemoryDescriptor(origGenMD);
1206 else
1207 return 0;
1208 }
1209
1210 IOMemoryDescriptor *
1211 IOGeneralMemoryDescriptor::withPersistentMemoryDescriptor(IOGeneralMemoryDescriptor *originalMD)
1212 {
1213 IOMemoryReference * memRef;
1214
1215 if (kIOReturnSuccess != originalMD->memoryReferenceCreate(kIOMemoryReferenceReuse, &memRef)) return (0);
1216
1217 if (memRef == originalMD->_memRef)
1218 {
1219 originalMD->retain(); // Add a new reference to ourselves
1220 originalMD->memoryReferenceRelease(memRef);
1221 return originalMD;
1222 }
1223
1224 IOGeneralMemoryDescriptor * self = new IOGeneralMemoryDescriptor;
1225 IOMDPersistentInitData initData = { originalMD, memRef };
1226
1227 if (self
1228 && !self->initWithOptions(&initData, 1, 0, 0, kIOMemoryTypePersistentMD, 0)) {
1229 self->release();
1230 self = 0;
1231 }
1232 return self;
1233 }
1234
1235 #ifndef __LP64__
1236 bool
1237 IOGeneralMemoryDescriptor::initWithAddress(void * address,
1238 IOByteCount withLength,
1239 IODirection withDirection)
1240 {
1241 _singleRange.v.address = (vm_offset_t) address;
1242 _singleRange.v.length = withLength;
1243
1244 return initWithRanges(&_singleRange.v, 1, withDirection, kernel_task, true);
1245 }
1246
1247 bool
1248 IOGeneralMemoryDescriptor::initWithAddress(IOVirtualAddress address,
1249 IOByteCount withLength,
1250 IODirection withDirection,
1251 task_t withTask)
1252 {
1253 _singleRange.v.address = address;
1254 _singleRange.v.length = withLength;
1255
1256 return initWithRanges(&_singleRange.v, 1, withDirection, withTask, true);
1257 }
1258
1259 bool
1260 IOGeneralMemoryDescriptor::initWithPhysicalAddress(
1261 IOPhysicalAddress address,
1262 IOByteCount withLength,
1263 IODirection withDirection )
1264 {
1265 _singleRange.p.address = address;
1266 _singleRange.p.length = withLength;
1267
1268 return initWithPhysicalRanges( &_singleRange.p, 1, withDirection, true);
1269 }
1270
1271 bool
1272 IOGeneralMemoryDescriptor::initWithPhysicalRanges(
1273 IOPhysicalRange * ranges,
1274 UInt32 count,
1275 IODirection direction,
1276 bool reference)
1277 {
1278 IOOptionBits mdOpts = direction | kIOMemoryTypePhysical;
1279
1280 if (reference)
1281 mdOpts |= kIOMemoryAsReference;
1282
1283 return initWithOptions(ranges, count, 0, 0, mdOpts, /* mapper */ 0);
1284 }
1285
1286 bool
1287 IOGeneralMemoryDescriptor::initWithRanges(
1288 IOVirtualRange * ranges,
1289 UInt32 count,
1290 IODirection direction,
1291 task_t task,
1292 bool reference)
1293 {
1294 IOOptionBits mdOpts = direction;
1295
1296 if (reference)
1297 mdOpts |= kIOMemoryAsReference;
1298
1299 if (task) {
1300 mdOpts |= kIOMemoryTypeVirtual;
1301
1302 // Auto-prepare if this is a kernel memory descriptor as very few
1303 // clients bother to prepare() kernel memory.
1304 // But it was not enforced so what are you going to do?
1305 if (task == kernel_task)
1306 mdOpts |= kIOMemoryAutoPrepare;
1307 }
1308 else
1309 mdOpts |= kIOMemoryTypePhysical;
1310
1311 return initWithOptions(ranges, count, 0, task, mdOpts, /* mapper */ 0);
1312 }
1313 #endif /* !__LP64__ */
1314
1315 /*
1316 * initWithOptions:
1317 *
1318 * IOMemoryDescriptor. The buffer is made up of several virtual address ranges,
1319 * from a given task, several physical ranges, an UPL from the ubc
1320 * system or a uio (may be 64bit) from the BSD subsystem.
1321 *
1322 * Passing the ranges as a reference will avoid an extra allocation.
1323 *
1324 * An IOMemoryDescriptor can be re-used by calling initWithOptions again on an
1325 * existing instance -- note this behavior is not commonly supported in other
1326 * I/O Kit classes, although it is supported here.
1327 */
1328
1329 bool
1330 IOGeneralMemoryDescriptor::initWithOptions(void * buffers,
1331 UInt32 count,
1332 UInt32 offset,
1333 task_t task,
1334 IOOptionBits options,
1335 IOMapper * mapper)
1336 {
1337 IOOptionBits type = options & kIOMemoryTypeMask;
1338
1339 #ifndef __LP64__
1340 if (task
1341 && (kIOMemoryTypeVirtual == type)
1342 && vm_map_is_64bit(get_task_map(task))
1343 && ((IOVirtualRange *) buffers)->address)
1344 {
1345 OSReportWithBacktrace("IOMemoryDescriptor: attempt to create 32b virtual in 64b task, use ::withAddressRange()");
1346 return false;
1347 }
1348 #endif /* !__LP64__ */
1349
1350 // Grab the original MD's configuation data to initialse the
1351 // arguments to this function.
1352 if (kIOMemoryTypePersistentMD == type) {
1353
1354 IOMDPersistentInitData *initData = (typeof(initData)) buffers;
1355 const IOGeneralMemoryDescriptor *orig = initData->fMD;
1356 ioGMDData *dataP = getDataP(orig->_memoryEntries);
1357
1358 // Only accept persistent memory descriptors with valid dataP data.
1359 assert(orig->_rangesCount == 1);
1360 if ( !(orig->_flags & kIOMemoryPersistent) || !dataP)
1361 return false;
1362
1363 _memRef = initData->fMemRef; // Grab the new named entry
1364 options = orig->_flags & ~kIOMemoryAsReference;
1365 type = options & kIOMemoryTypeMask;
1366 buffers = orig->_ranges.v;
1367 count = orig->_rangesCount;
1368
1369 // Now grab the original task and whatever mapper was previously used
1370 task = orig->_task;
1371 mapper = dataP->fMapper;
1372
1373 // We are ready to go through the original initialisation now
1374 }
1375
1376 switch (type) {
1377 case kIOMemoryTypeUIO:
1378 case kIOMemoryTypeVirtual:
1379 #ifndef __LP64__
1380 case kIOMemoryTypeVirtual64:
1381 #endif /* !__LP64__ */
1382 assert(task);
1383 if (!task)
1384 return false;
1385 break;
1386
1387 case kIOMemoryTypePhysical: // Neither Physical nor UPL should have a task
1388 #ifndef __LP64__
1389 case kIOMemoryTypePhysical64:
1390 #endif /* !__LP64__ */
1391 case kIOMemoryTypeUPL:
1392 assert(!task);
1393 break;
1394 default:
1395 return false; /* bad argument */
1396 }
1397
1398 assert(buffers);
1399 assert(count);
1400
1401 /*
1402 * We can check the _initialized instance variable before having ever set
1403 * it to an initial value because I/O Kit guarantees that all our instance
1404 * variables are zeroed on an object's allocation.
1405 */
1406
1407 if (_initialized) {
1408 /*
1409 * An existing memory descriptor is being retargeted to point to
1410 * somewhere else. Clean up our present state.
1411 */
1412 IOOptionBits type = _flags & kIOMemoryTypeMask;
1413 if ((kIOMemoryTypePhysical != type) && (kIOMemoryTypePhysical64 != type))
1414 {
1415 while (_wireCount)
1416 complete();
1417 }
1418 if (_ranges.v && !(kIOMemoryAsReference & _flags))
1419 {
1420 if (kIOMemoryTypeUIO == type)
1421 uio_free((uio_t) _ranges.v);
1422 #ifndef __LP64__
1423 else if ((kIOMemoryTypeVirtual64 == type) || (kIOMemoryTypePhysical64 == type))
1424 IODelete(_ranges.v64, IOAddressRange, _rangesCount);
1425 #endif /* !__LP64__ */
1426 else
1427 IODelete(_ranges.v, IOVirtualRange, _rangesCount);
1428 }
1429
1430 options |= (kIOMemoryRedirected & _flags);
1431 if (!(kIOMemoryRedirected & options))
1432 {
1433 if (_memRef)
1434 {
1435 memoryReferenceRelease(_memRef);
1436 _memRef = 0;
1437 }
1438 if (_mappings)
1439 _mappings->flushCollection();
1440 }
1441 }
1442 else {
1443 if (!super::init())
1444 return false;
1445 _initialized = true;
1446 }
1447
1448 // Grab the appropriate mapper
1449 if (kIOMemoryHostOnly & options) options |= kIOMemoryMapperNone;
1450 if (kIOMemoryMapperNone & options)
1451 mapper = 0; // No Mapper
1452 else if (mapper == kIOMapperSystem) {
1453 IOMapper::checkForSystemMapper();
1454 gIOSystemMapper = mapper = IOMapper::gSystem;
1455 }
1456
1457 // Temp binary compatibility for kIOMemoryThreadSafe
1458 if (kIOMemoryReserved6156215 & options)
1459 {
1460 options &= ~kIOMemoryReserved6156215;
1461 options |= kIOMemoryThreadSafe;
1462 }
1463 // Remove the dynamic internal use flags from the initial setting
1464 options &= ~(kIOMemoryPreparedReadOnly);
1465 _flags = options;
1466 _task = task;
1467
1468 #ifndef __LP64__
1469 _direction = (IODirection) (_flags & kIOMemoryDirectionMask);
1470 #endif /* !__LP64__ */
1471
1472 __iomd_reservedA = 0;
1473 __iomd_reservedB = 0;
1474 _highestPage = 0;
1475
1476 if (kIOMemoryThreadSafe & options)
1477 {
1478 if (!_prepareLock)
1479 _prepareLock = IOLockAlloc();
1480 }
1481 else if (_prepareLock)
1482 {
1483 IOLockFree(_prepareLock);
1484 _prepareLock = NULL;
1485 }
1486
1487 if (kIOMemoryTypeUPL == type) {
1488
1489 ioGMDData *dataP;
1490 unsigned int dataSize = computeDataSize(/* pages */ 0, /* upls */ 1);
1491
1492 if (!initMemoryEntries(dataSize, mapper)) return (false);
1493 dataP = getDataP(_memoryEntries);
1494 dataP->fPageCnt = 0;
1495
1496 // _wireCount++; // UPLs start out life wired
1497
1498 _length = count;
1499 _pages += atop_32(offset + count + PAGE_MASK) - atop_32(offset);
1500
1501 ioPLBlock iopl;
1502 iopl.fIOPL = (upl_t) buffers;
1503 upl_set_referenced(iopl.fIOPL, true);
1504 upl_page_info_t *pageList = UPL_GET_INTERNAL_PAGE_LIST(iopl.fIOPL);
1505
1506 if (upl_get_size(iopl.fIOPL) < (count + offset))
1507 panic("short external upl");
1508
1509 _highestPage = upl_get_highest_page(iopl.fIOPL);
1510
1511 // Set the flag kIOPLOnDevice convieniently equal to 1
1512 iopl.fFlags = pageList->device | kIOPLExternUPL;
1513 if (!pageList->device) {
1514 // Pre-compute the offset into the UPL's page list
1515 pageList = &pageList[atop_32(offset)];
1516 offset &= PAGE_MASK;
1517 }
1518 iopl.fIOMDOffset = 0;
1519 iopl.fMappedPage = 0;
1520 iopl.fPageInfo = (vm_address_t) pageList;
1521 iopl.fPageOffset = offset;
1522 _memoryEntries->appendBytes(&iopl, sizeof(iopl));
1523 }
1524 else {
1525 // kIOMemoryTypeVirtual | kIOMemoryTypeVirtual64 | kIOMemoryTypeUIO
1526 // kIOMemoryTypePhysical | kIOMemoryTypePhysical64
1527
1528 // Initialize the memory descriptor
1529 if (options & kIOMemoryAsReference) {
1530 #ifndef __LP64__
1531 _rangesIsAllocated = false;
1532 #endif /* !__LP64__ */
1533
1534 // Hack assignment to get the buffer arg into _ranges.
1535 // I'd prefer to do _ranges = (Ranges) buffers, but that doesn't
1536 // work, C++ sigh.
1537 // This also initialises the uio & physical ranges.
1538 _ranges.v = (IOVirtualRange *) buffers;
1539 }
1540 else {
1541 #ifndef __LP64__
1542 _rangesIsAllocated = true;
1543 #endif /* !__LP64__ */
1544 switch (type)
1545 {
1546 case kIOMemoryTypeUIO:
1547 _ranges.v = (IOVirtualRange *) uio_duplicate((uio_t) buffers);
1548 break;
1549
1550 #ifndef __LP64__
1551 case kIOMemoryTypeVirtual64:
1552 case kIOMemoryTypePhysical64:
1553 if (count == 1
1554 && (((IOAddressRange *) buffers)->address + ((IOAddressRange *) buffers)->length) <= 0x100000000ULL
1555 ) {
1556 if (kIOMemoryTypeVirtual64 == type)
1557 type = kIOMemoryTypeVirtual;
1558 else
1559 type = kIOMemoryTypePhysical;
1560 _flags = (_flags & ~kIOMemoryTypeMask) | type | kIOMemoryAsReference;
1561 _rangesIsAllocated = false;
1562 _ranges.v = &_singleRange.v;
1563 _singleRange.v.address = ((IOAddressRange *) buffers)->address;
1564 _singleRange.v.length = ((IOAddressRange *) buffers)->length;
1565 break;
1566 }
1567 _ranges.v64 = IONew(IOAddressRange, count);
1568 if (!_ranges.v64)
1569 return false;
1570 bcopy(buffers, _ranges.v, count * sizeof(IOAddressRange));
1571 break;
1572 #endif /* !__LP64__ */
1573 case kIOMemoryTypeVirtual:
1574 case kIOMemoryTypePhysical:
1575 if (count == 1) {
1576 _flags |= kIOMemoryAsReference;
1577 #ifndef __LP64__
1578 _rangesIsAllocated = false;
1579 #endif /* !__LP64__ */
1580 _ranges.v = &_singleRange.v;
1581 } else {
1582 _ranges.v = IONew(IOVirtualRange, count);
1583 if (!_ranges.v)
1584 return false;
1585 }
1586 bcopy(buffers, _ranges.v, count * sizeof(IOVirtualRange));
1587 break;
1588 }
1589 }
1590
1591 // Find starting address within the vector of ranges
1592 Ranges vec = _ranges;
1593 UInt32 length = 0;
1594 UInt32 pages = 0;
1595 for (unsigned ind = 0; ind < count; ind++) {
1596 mach_vm_address_t addr;
1597 mach_vm_size_t len;
1598
1599 // addr & len are returned by this function
1600 getAddrLenForInd(addr, len, type, vec, ind);
1601 pages += (atop_64(addr + len + PAGE_MASK) - atop_64(addr));
1602 len += length;
1603 assert(len >= length); // Check for 32 bit wrap around
1604 length = len;
1605
1606 if ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type))
1607 {
1608 ppnum_t highPage = atop_64(addr + len - 1);
1609 if (highPage > _highestPage)
1610 _highestPage = highPage;
1611 }
1612 }
1613 _length = length;
1614 _pages = pages;
1615 _rangesCount = count;
1616
1617 // Auto-prepare memory at creation time.
1618 // Implied completion when descriptor is free-ed
1619 if ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type))
1620 _wireCount++; // Physical MDs are, by definition, wired
1621 else { /* kIOMemoryTypeVirtual | kIOMemoryTypeVirtual64 | kIOMemoryTypeUIO */
1622 ioGMDData *dataP;
1623 unsigned dataSize = computeDataSize(_pages, /* upls */ count * 2);
1624
1625 if (!initMemoryEntries(dataSize, mapper)) return false;
1626 dataP = getDataP(_memoryEntries);
1627 dataP->fPageCnt = _pages;
1628
1629 if ( (kIOMemoryPersistent & _flags) && !_memRef)
1630 {
1631 IOReturn
1632 err = memoryReferenceCreate(0, &_memRef);
1633 if (kIOReturnSuccess != err) return false;
1634 }
1635
1636 if ((_flags & kIOMemoryAutoPrepare)
1637 && prepare() != kIOReturnSuccess)
1638 return false;
1639 }
1640 }
1641
1642 return true;
1643 }
1644
1645 /*
1646 * free
1647 *
1648 * Free resources.
1649 */
1650 void IOGeneralMemoryDescriptor::free()
1651 {
1652 IOOptionBits type = _flags & kIOMemoryTypeMask;
1653
1654 if( reserved)
1655 {
1656 LOCK;
1657 reserved->dp.memory = 0;
1658 UNLOCK;
1659 }
1660 if ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type))
1661 {
1662 ioGMDData * dataP;
1663 if (_memoryEntries && (dataP = getDataP(_memoryEntries)) && dataP->fMappedBase)
1664 {
1665 dataP->fMapper->iovmFree(atop_64(dataP->fMappedBase), _pages);
1666 dataP->fMappedBase = 0;
1667 }
1668 }
1669 else
1670 {
1671 while (_wireCount) complete();
1672 }
1673
1674 if (_memoryEntries) _memoryEntries->release();
1675
1676 if (_ranges.v && !(kIOMemoryAsReference & _flags))
1677 {
1678 if (kIOMemoryTypeUIO == type)
1679 uio_free((uio_t) _ranges.v);
1680 #ifndef __LP64__
1681 else if ((kIOMemoryTypeVirtual64 == type) || (kIOMemoryTypePhysical64 == type))
1682 IODelete(_ranges.v64, IOAddressRange, _rangesCount);
1683 #endif /* !__LP64__ */
1684 else
1685 IODelete(_ranges.v, IOVirtualRange, _rangesCount);
1686
1687 _ranges.v = NULL;
1688 }
1689
1690 if (reserved)
1691 {
1692 if (reserved->dp.devicePager)
1693 {
1694 // memEntry holds a ref on the device pager which owns reserved
1695 // (IOMemoryDescriptorReserved) so no reserved access after this point
1696 device_pager_deallocate( (memory_object_t) reserved->dp.devicePager );
1697 }
1698 else
1699 IODelete(reserved, IOMemoryDescriptorReserved, 1);
1700 reserved = NULL;
1701 }
1702
1703 if (_memRef) memoryReferenceRelease(_memRef);
1704 if (_prepareLock) IOLockFree(_prepareLock);
1705
1706 super::free();
1707 }
1708
1709 #ifndef __LP64__
1710 void IOGeneralMemoryDescriptor::unmapFromKernel()
1711 {
1712 panic("IOGMD::unmapFromKernel deprecated");
1713 }
1714
1715 void IOGeneralMemoryDescriptor::mapIntoKernel(unsigned rangeIndex)
1716 {
1717 panic("IOGMD::mapIntoKernel deprecated");
1718 }
1719 #endif /* !__LP64__ */
1720
1721 /*
1722 * getDirection:
1723 *
1724 * Get the direction of the transfer.
1725 */
1726 IODirection IOMemoryDescriptor::getDirection() const
1727 {
1728 #ifndef __LP64__
1729 if (_direction)
1730 return _direction;
1731 #endif /* !__LP64__ */
1732 return (IODirection) (_flags & kIOMemoryDirectionMask);
1733 }
1734
1735 /*
1736 * getLength:
1737 *
1738 * Get the length of the transfer (over all ranges).
1739 */
1740 IOByteCount IOMemoryDescriptor::getLength() const
1741 {
1742 return _length;
1743 }
1744
1745 void IOMemoryDescriptor::setTag( IOOptionBits tag )
1746 {
1747 _tag = tag;
1748 }
1749
1750 IOOptionBits IOMemoryDescriptor::getTag( void )
1751 {
1752 return( _tag);
1753 }
1754
1755 #ifndef __LP64__
1756 // @@@ gvdl: who is using this API? Seems like a wierd thing to implement.
1757 IOPhysicalAddress
1758 IOMemoryDescriptor::getSourceSegment( IOByteCount offset, IOByteCount * length )
1759 {
1760 addr64_t physAddr = 0;
1761
1762 if( prepare() == kIOReturnSuccess) {
1763 physAddr = getPhysicalSegment64( offset, length );
1764 complete();
1765 }
1766
1767 return( (IOPhysicalAddress) physAddr ); // truncated but only page offset is used
1768 }
1769 #endif /* !__LP64__ */
1770
1771 IOByteCount IOMemoryDescriptor::readBytes
1772 (IOByteCount offset, void *bytes, IOByteCount length)
1773 {
1774 addr64_t dstAddr = CAST_DOWN(addr64_t, bytes);
1775 IOByteCount remaining;
1776
1777 // Assert that this entire I/O is withing the available range
1778 assert(offset <= _length);
1779 assert(offset + length <= _length);
1780 if (offset >= _length) {
1781 return 0;
1782 }
1783
1784 if (kIOMemoryThreadSafe & _flags)
1785 LOCK;
1786
1787 remaining = length = min(length, _length - offset);
1788 while (remaining) { // (process another target segment?)
1789 addr64_t srcAddr64;
1790 IOByteCount srcLen;
1791
1792 srcAddr64 = getPhysicalSegment(offset, &srcLen, kIOMemoryMapperNone);
1793 if (!srcAddr64)
1794 break;
1795
1796 // Clip segment length to remaining
1797 if (srcLen > remaining)
1798 srcLen = remaining;
1799
1800 copypv(srcAddr64, dstAddr, srcLen,
1801 cppvPsrc | cppvNoRefSrc | cppvFsnk | cppvKmap);
1802
1803 dstAddr += srcLen;
1804 offset += srcLen;
1805 remaining -= srcLen;
1806 }
1807
1808 if (kIOMemoryThreadSafe & _flags)
1809 UNLOCK;
1810
1811 assert(!remaining);
1812
1813 return length - remaining;
1814 }
1815
1816 IOByteCount IOMemoryDescriptor::writeBytes
1817 (IOByteCount inoffset, const void *bytes, IOByteCount length)
1818 {
1819 addr64_t srcAddr = CAST_DOWN(addr64_t, bytes);
1820 IOByteCount remaining;
1821 IOByteCount offset = inoffset;
1822
1823 // Assert that this entire I/O is withing the available range
1824 assert(offset <= _length);
1825 assert(offset + length <= _length);
1826
1827 assert( !(kIOMemoryPreparedReadOnly & _flags) );
1828
1829 if ( (kIOMemoryPreparedReadOnly & _flags) || offset >= _length) {
1830 return 0;
1831 }
1832
1833 if (kIOMemoryThreadSafe & _flags)
1834 LOCK;
1835
1836 remaining = length = min(length, _length - offset);
1837 while (remaining) { // (process another target segment?)
1838 addr64_t dstAddr64;
1839 IOByteCount dstLen;
1840
1841 dstAddr64 = getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
1842 if (!dstAddr64)
1843 break;
1844
1845 // Clip segment length to remaining
1846 if (dstLen > remaining)
1847 dstLen = remaining;
1848
1849 if (!srcAddr) bzero_phys(dstAddr64, dstLen);
1850 else
1851 {
1852 copypv(srcAddr, (addr64_t) dstAddr64, dstLen,
1853 cppvPsnk | cppvFsnk | cppvNoRefSrc | cppvNoModSnk | cppvKmap);
1854 srcAddr += dstLen;
1855 }
1856 offset += dstLen;
1857 remaining -= dstLen;
1858 }
1859
1860 if (kIOMemoryThreadSafe & _flags)
1861 UNLOCK;
1862
1863 assert(!remaining);
1864
1865 if (!srcAddr) performOperation(kIOMemoryIncoherentIOFlush, inoffset, length);
1866
1867 return length - remaining;
1868 }
1869
1870 #ifndef __LP64__
1871 void IOGeneralMemoryDescriptor::setPosition(IOByteCount position)
1872 {
1873 panic("IOGMD::setPosition deprecated");
1874 }
1875 #endif /* !__LP64__ */
1876
1877 static volatile SInt64 gIOMDPreparationID __attribute__((aligned(8))) = (1ULL << 32);
1878
1879 uint64_t
1880 IOGeneralMemoryDescriptor::getPreparationID( void )
1881 {
1882 ioGMDData *dataP;
1883
1884 if (!_wireCount)
1885 return (kIOPreparationIDUnprepared);
1886
1887 if (((kIOMemoryTypeMask & _flags) == kIOMemoryTypePhysical)
1888 || ((kIOMemoryTypeMask & _flags) == kIOMemoryTypePhysical64))
1889 {
1890 IOMemoryDescriptor::setPreparationID();
1891 return (IOMemoryDescriptor::getPreparationID());
1892 }
1893
1894 if (!_memoryEntries || !(dataP = getDataP(_memoryEntries)))
1895 return (kIOPreparationIDUnprepared);
1896
1897 if (kIOPreparationIDUnprepared == dataP->fPreparationID)
1898 {
1899 dataP->fPreparationID = OSIncrementAtomic64(&gIOMDPreparationID);
1900 }
1901 return (dataP->fPreparationID);
1902 }
1903
1904 IOMemoryDescriptorReserved * IOMemoryDescriptor::getKernelReserved( void )
1905 {
1906 if (!reserved)
1907 {
1908 reserved = IONew(IOMemoryDescriptorReserved, 1);
1909 if (reserved)
1910 bzero(reserved, sizeof(IOMemoryDescriptorReserved));
1911 }
1912 return (reserved);
1913 }
1914
1915 void IOMemoryDescriptor::setPreparationID( void )
1916 {
1917 if (getKernelReserved() && (kIOPreparationIDUnprepared == reserved->preparationID))
1918 {
1919 #if defined(__ppc__ )
1920 reserved->preparationID = gIOMDPreparationID++;
1921 #else
1922 reserved->preparationID = OSIncrementAtomic64(&gIOMDPreparationID);
1923 #endif
1924 }
1925 }
1926
1927 uint64_t IOMemoryDescriptor::getPreparationID( void )
1928 {
1929 if (reserved)
1930 return (reserved->preparationID);
1931 else
1932 return (kIOPreparationIDUnsupported);
1933 }
1934
1935 IOReturn IOGeneralMemoryDescriptor::dmaCommandOperation(DMACommandOps op, void *vData, UInt dataSize) const
1936 {
1937 IOReturn err = kIOReturnSuccess;
1938 DMACommandOps params;
1939 IOGeneralMemoryDescriptor * md = const_cast<IOGeneralMemoryDescriptor *>(this);
1940 ioGMDData *dataP;
1941
1942 params = (op & ~kIOMDDMACommandOperationMask & op);
1943 op &= kIOMDDMACommandOperationMask;
1944
1945 if (kIOMDDMAMap == op)
1946 {
1947 if (dataSize < sizeof(IOMDDMAMapArgs))
1948 return kIOReturnUnderrun;
1949
1950 IOMDDMAMapArgs * data = (IOMDDMAMapArgs *) vData;
1951
1952 if (!_memoryEntries
1953 && !md->initMemoryEntries(computeDataSize(0, 0), kIOMapperWaitSystem)) return (kIOReturnNoMemory);
1954
1955 if (_memoryEntries && data->fMapper)
1956 {
1957 bool remap;
1958 bool whole = ((data->fOffset == 0) && (data->fLength == _length));
1959 dataP = getDataP(_memoryEntries);
1960
1961 if (data->fMapSpec.numAddressBits < dataP->fDMAMapNumAddressBits) dataP->fDMAMapNumAddressBits = data->fMapSpec.numAddressBits;
1962 if (data->fMapSpec.alignment > dataP->fDMAMapAlignment) dataP->fDMAMapAlignment = data->fMapSpec.alignment;
1963
1964 remap = (dataP->fDMAMapNumAddressBits < 64)
1965 && ((dataP->fMappedBase + _length) > (1ULL << dataP->fDMAMapNumAddressBits));
1966 remap |= (dataP->fDMAMapAlignment > page_size);
1967 remap |= (!whole);
1968 if (remap || !dataP->fMappedBase)
1969 {
1970 // if (dataP->fMappedBase) OSReportWithBacktrace("kIOMDDMAMap whole %d remap %d params %d\n", whole, remap, params);
1971 err = md->dmaMap(data->fMapper, &data->fMapSpec, data->fOffset, data->fLength, &data->fAlloc, &data->fAllocCount);
1972 if ((kIOReturnSuccess == err) && whole && !dataP->fMappedBase)
1973 {
1974 dataP->fMappedBase = data->fAlloc;
1975 data->fAllocCount = 0; // IOMD owns the alloc now
1976 }
1977 }
1978 else
1979 {
1980 data->fAlloc = dataP->fMappedBase;
1981 data->fAllocCount = 0; // IOMD owns the alloc
1982 }
1983 data->fMapContig = !dataP->fDiscontig;
1984 }
1985
1986 return (err);
1987 }
1988
1989 if (kIOMDAddDMAMapSpec == op)
1990 {
1991 if (dataSize < sizeof(IODMAMapSpecification))
1992 return kIOReturnUnderrun;
1993
1994 IODMAMapSpecification * data = (IODMAMapSpecification *) vData;
1995
1996 if (!_memoryEntries
1997 && !md->initMemoryEntries(computeDataSize(0, 0), kIOMapperWaitSystem)) return (kIOReturnNoMemory);
1998
1999 if (_memoryEntries)
2000 {
2001 dataP = getDataP(_memoryEntries);
2002 if (data->numAddressBits < dataP->fDMAMapNumAddressBits)
2003 dataP->fDMAMapNumAddressBits = data->numAddressBits;
2004 if (data->alignment > dataP->fDMAMapAlignment)
2005 dataP->fDMAMapAlignment = data->alignment;
2006 }
2007 return kIOReturnSuccess;
2008 }
2009
2010 if (kIOMDGetCharacteristics == op) {
2011
2012 if (dataSize < sizeof(IOMDDMACharacteristics))
2013 return kIOReturnUnderrun;
2014
2015 IOMDDMACharacteristics *data = (IOMDDMACharacteristics *) vData;
2016 data->fLength = _length;
2017 data->fSGCount = _rangesCount;
2018 data->fPages = _pages;
2019 data->fDirection = getDirection();
2020 if (!_wireCount)
2021 data->fIsPrepared = false;
2022 else {
2023 data->fIsPrepared = true;
2024 data->fHighestPage = _highestPage;
2025 if (_memoryEntries)
2026 {
2027 dataP = getDataP(_memoryEntries);
2028 ioPLBlock *ioplList = getIOPLList(dataP);
2029 UInt count = getNumIOPL(_memoryEntries, dataP);
2030 if (count == 1)
2031 data->fPageAlign = (ioplList[0].fPageOffset & PAGE_MASK) | ~PAGE_MASK;
2032 }
2033 }
2034
2035 return kIOReturnSuccess;
2036
2037 #if IOMD_DEBUG_DMAACTIVE
2038 } else if (kIOMDDMAActive == op) {
2039 if (params) OSIncrementAtomic(&md->__iomd_reservedA);
2040 else {
2041 if (md->__iomd_reservedA)
2042 OSDecrementAtomic(&md->__iomd_reservedA);
2043 else
2044 panic("kIOMDSetDMAInactive");
2045 }
2046 #endif /* IOMD_DEBUG_DMAACTIVE */
2047
2048 } else if (kIOMDWalkSegments != op)
2049 return kIOReturnBadArgument;
2050
2051 // Get the next segment
2052 struct InternalState {
2053 IOMDDMAWalkSegmentArgs fIO;
2054 UInt fOffset2Index;
2055 UInt fIndex;
2056 UInt fNextOffset;
2057 } *isP;
2058
2059 // Find the next segment
2060 if (dataSize < sizeof(*isP))
2061 return kIOReturnUnderrun;
2062
2063 isP = (InternalState *) vData;
2064 UInt offset = isP->fIO.fOffset;
2065 bool mapped = isP->fIO.fMapped;
2066
2067 if (IOMapper::gSystem && mapped
2068 && (!(kIOMemoryHostOnly & _flags))
2069 && (!_memoryEntries || !getDataP(_memoryEntries)->fMappedBase))
2070 // && (_memoryEntries && !getDataP(_memoryEntries)->fMappedBase))
2071 {
2072 if (!_memoryEntries
2073 && !md->initMemoryEntries(computeDataSize(0, 0), kIOMapperWaitSystem)) return (kIOReturnNoMemory);
2074
2075 dataP = getDataP(_memoryEntries);
2076 if (dataP->fMapper)
2077 {
2078 IODMAMapSpecification mapSpec;
2079 bzero(&mapSpec, sizeof(mapSpec));
2080 mapSpec.numAddressBits = dataP->fDMAMapNumAddressBits;
2081 mapSpec.alignment = dataP->fDMAMapAlignment;
2082 err = md->dmaMap(dataP->fMapper, &mapSpec, 0, _length, &dataP->fMappedBase, NULL);
2083 if (kIOReturnSuccess != err) return (err);
2084 }
2085 }
2086
2087 if (offset >= _length)
2088 return (offset == _length)? kIOReturnOverrun : kIOReturnInternalError;
2089
2090 // Validate the previous offset
2091 UInt ind, off2Ind = isP->fOffset2Index;
2092 if (!params
2093 && offset
2094 && (offset == isP->fNextOffset || off2Ind <= offset))
2095 ind = isP->fIndex;
2096 else
2097 ind = off2Ind = 0; // Start from beginning
2098
2099 UInt length;
2100 UInt64 address;
2101
2102
2103 if ( (_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical) {
2104
2105 // Physical address based memory descriptor
2106 const IOPhysicalRange *physP = (IOPhysicalRange *) &_ranges.p[0];
2107
2108 // Find the range after the one that contains the offset
2109 mach_vm_size_t len;
2110 for (len = 0; off2Ind <= offset; ind++) {
2111 len = physP[ind].length;
2112 off2Ind += len;
2113 }
2114
2115 // Calculate length within range and starting address
2116 length = off2Ind - offset;
2117 address = physP[ind - 1].address + len - length;
2118
2119 if (true && mapped && _memoryEntries
2120 && (dataP = getDataP(_memoryEntries)) && dataP->fMappedBase)
2121 {
2122 address = dataP->fMappedBase + offset;
2123 }
2124 else
2125 {
2126 // see how far we can coalesce ranges
2127 while (ind < _rangesCount && address + length == physP[ind].address) {
2128 len = physP[ind].length;
2129 length += len;
2130 off2Ind += len;
2131 ind++;
2132 }
2133 }
2134
2135 // correct contiguous check overshoot
2136 ind--;
2137 off2Ind -= len;
2138 }
2139 #ifndef __LP64__
2140 else if ( (_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical64) {
2141
2142 // Physical address based memory descriptor
2143 const IOAddressRange *physP = (IOAddressRange *) &_ranges.v64[0];
2144
2145 // Find the range after the one that contains the offset
2146 mach_vm_size_t len;
2147 for (len = 0; off2Ind <= offset; ind++) {
2148 len = physP[ind].length;
2149 off2Ind += len;
2150 }
2151
2152 // Calculate length within range and starting address
2153 length = off2Ind - offset;
2154 address = physP[ind - 1].address + len - length;
2155
2156 if (true && mapped && _memoryEntries
2157 && (dataP = getDataP(_memoryEntries)) && dataP->fMappedBase)
2158 {
2159 address = dataP->fMappedBase + offset;
2160 }
2161 else
2162 {
2163 // see how far we can coalesce ranges
2164 while (ind < _rangesCount && address + length == physP[ind].address) {
2165 len = physP[ind].length;
2166 length += len;
2167 off2Ind += len;
2168 ind++;
2169 }
2170 }
2171 // correct contiguous check overshoot
2172 ind--;
2173 off2Ind -= len;
2174 }
2175 #endif /* !__LP64__ */
2176 else do {
2177 if (!_wireCount)
2178 panic("IOGMD: not wired for the IODMACommand");
2179
2180 assert(_memoryEntries);
2181
2182 dataP = getDataP(_memoryEntries);
2183 const ioPLBlock *ioplList = getIOPLList(dataP);
2184 UInt numIOPLs = getNumIOPL(_memoryEntries, dataP);
2185 upl_page_info_t *pageList = getPageList(dataP);
2186
2187 assert(numIOPLs > 0);
2188
2189 // Scan through iopl info blocks looking for block containing offset
2190 while (ind < numIOPLs && offset >= ioplList[ind].fIOMDOffset)
2191 ind++;
2192
2193 // Go back to actual range as search goes past it
2194 ioPLBlock ioplInfo = ioplList[ind - 1];
2195 off2Ind = ioplInfo.fIOMDOffset;
2196
2197 if (ind < numIOPLs)
2198 length = ioplList[ind].fIOMDOffset;
2199 else
2200 length = _length;
2201 length -= offset; // Remainder within iopl
2202
2203 // Subtract offset till this iopl in total list
2204 offset -= off2Ind;
2205
2206 // If a mapped address is requested and this is a pre-mapped IOPL
2207 // then just need to compute an offset relative to the mapped base.
2208 if (mapped && dataP->fMappedBase) {
2209 offset += (ioplInfo.fPageOffset & PAGE_MASK);
2210 address = trunc_page_64(dataP->fMappedBase) + ptoa_64(ioplInfo.fMappedPage) + offset;
2211 continue; // Done leave do/while(false) now
2212 }
2213
2214 // The offset is rebased into the current iopl.
2215 // Now add the iopl 1st page offset.
2216 offset += ioplInfo.fPageOffset;
2217
2218 // For external UPLs the fPageInfo field points directly to
2219 // the upl's upl_page_info_t array.
2220 if (ioplInfo.fFlags & kIOPLExternUPL)
2221 pageList = (upl_page_info_t *) ioplInfo.fPageInfo;
2222 else
2223 pageList = &pageList[ioplInfo.fPageInfo];
2224
2225 // Check for direct device non-paged memory
2226 if ( ioplInfo.fFlags & kIOPLOnDevice ) {
2227 address = ptoa_64(pageList->phys_addr) + offset;
2228 continue; // Done leave do/while(false) now
2229 }
2230
2231 // Now we need compute the index into the pageList
2232 UInt pageInd = atop_32(offset);
2233 offset &= PAGE_MASK;
2234
2235 // Compute the starting address of this segment
2236 IOPhysicalAddress pageAddr = pageList[pageInd].phys_addr;
2237 if (!pageAddr) {
2238 panic("!pageList phys_addr");
2239 }
2240
2241 address = ptoa_64(pageAddr) + offset;
2242
2243 // length is currently set to the length of the remainider of the iopl.
2244 // We need to check that the remainder of the iopl is contiguous.
2245 // This is indicated by pageList[ind].phys_addr being sequential.
2246 IOByteCount contigLength = PAGE_SIZE - offset;
2247 while (contigLength < length
2248 && ++pageAddr == pageList[++pageInd].phys_addr)
2249 {
2250 contigLength += PAGE_SIZE;
2251 }
2252
2253 if (contigLength < length)
2254 length = contigLength;
2255
2256
2257 assert(address);
2258 assert(length);
2259
2260 } while (false);
2261
2262 // Update return values and state
2263 isP->fIO.fIOVMAddr = address;
2264 isP->fIO.fLength = length;
2265 isP->fIndex = ind;
2266 isP->fOffset2Index = off2Ind;
2267 isP->fNextOffset = isP->fIO.fOffset + length;
2268
2269 return kIOReturnSuccess;
2270 }
2271
2272 addr64_t
2273 IOGeneralMemoryDescriptor::getPhysicalSegment(IOByteCount offset, IOByteCount *lengthOfSegment, IOOptionBits options)
2274 {
2275 IOReturn ret;
2276 mach_vm_address_t address = 0;
2277 mach_vm_size_t length = 0;
2278 IOMapper * mapper = gIOSystemMapper;
2279 IOOptionBits type = _flags & kIOMemoryTypeMask;
2280
2281 if (lengthOfSegment)
2282 *lengthOfSegment = 0;
2283
2284 if (offset >= _length)
2285 return 0;
2286
2287 // IOMemoryDescriptor::doMap() cannot use getPhysicalSegment() to obtain the page offset, since it must
2288 // support the unwired memory case in IOGeneralMemoryDescriptor, and hibernate_write_image() cannot use
2289 // map()->getVirtualAddress() to obtain the kernel pointer, since it must prevent the memory allocation
2290 // due to IOMemoryMap, so _kIOMemorySourceSegment is a necessary evil until all of this gets cleaned up
2291
2292 if ((options & _kIOMemorySourceSegment) && (kIOMemoryTypeUPL != type))
2293 {
2294 unsigned rangesIndex = 0;
2295 Ranges vec = _ranges;
2296 mach_vm_address_t addr;
2297
2298 // Find starting address within the vector of ranges
2299 for (;;) {
2300 getAddrLenForInd(addr, length, type, vec, rangesIndex);
2301 if (offset < length)
2302 break;
2303 offset -= length; // (make offset relative)
2304 rangesIndex++;
2305 }
2306
2307 // Now that we have the starting range,
2308 // lets find the last contiguous range
2309 addr += offset;
2310 length -= offset;
2311
2312 for ( ++rangesIndex; rangesIndex < _rangesCount; rangesIndex++ ) {
2313 mach_vm_address_t newAddr;
2314 mach_vm_size_t newLen;
2315
2316 getAddrLenForInd(newAddr, newLen, type, vec, rangesIndex);
2317 if (addr + length != newAddr)
2318 break;
2319 length += newLen;
2320 }
2321 if (addr)
2322 address = (IOPhysicalAddress) addr; // Truncate address to 32bit
2323 }
2324 else
2325 {
2326 IOMDDMAWalkSegmentState _state;
2327 IOMDDMAWalkSegmentArgs * state = (IOMDDMAWalkSegmentArgs *) (void *)&_state;
2328
2329 state->fOffset = offset;
2330 state->fLength = _length - offset;
2331 state->fMapped = (0 == (options & kIOMemoryMapperNone)) && !(_flags & kIOMemoryHostOnly);
2332
2333 ret = dmaCommandOperation(kIOMDFirstSegment, _state, sizeof(_state));
2334
2335 if ((kIOReturnSuccess != ret) && (kIOReturnOverrun != ret))
2336 DEBG("getPhysicalSegment dmaCommandOperation(%lx), %p, offset %qx, addr %qx, len %qx\n",
2337 ret, this, state->fOffset,
2338 state->fIOVMAddr, state->fLength);
2339 if (kIOReturnSuccess == ret)
2340 {
2341 address = state->fIOVMAddr;
2342 length = state->fLength;
2343 }
2344
2345 // dmaCommandOperation() does not distinguish between "mapped" and "unmapped" physical memory, even
2346 // with fMapped set correctly, so we must handle the transformation here until this gets cleaned up
2347
2348 if (mapper && ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type)))
2349 {
2350 if ((options & kIOMemoryMapperNone) && !(_flags & kIOMemoryMapperNone))
2351 {
2352 addr64_t origAddr = address;
2353 IOByteCount origLen = length;
2354
2355 address = mapper->mapAddr(origAddr);
2356 length = page_size - (address & (page_size - 1));
2357 while ((length < origLen)
2358 && ((address + length) == mapper->mapAddr(origAddr + length)))
2359 length += page_size;
2360 if (length > origLen)
2361 length = origLen;
2362 }
2363 }
2364 }
2365
2366 if (!address)
2367 length = 0;
2368
2369 if (lengthOfSegment)
2370 *lengthOfSegment = length;
2371
2372 return (address);
2373 }
2374
2375 #ifndef __LP64__
2376 addr64_t
2377 IOMemoryDescriptor::getPhysicalSegment(IOByteCount offset, IOByteCount *lengthOfSegment, IOOptionBits options)
2378 {
2379 addr64_t address = 0;
2380
2381 if (options & _kIOMemorySourceSegment)
2382 {
2383 address = getSourceSegment(offset, lengthOfSegment);
2384 }
2385 else if (options & kIOMemoryMapperNone)
2386 {
2387 address = getPhysicalSegment64(offset, lengthOfSegment);
2388 }
2389 else
2390 {
2391 address = getPhysicalSegment(offset, lengthOfSegment);
2392 }
2393
2394 return (address);
2395 }
2396
2397 addr64_t
2398 IOGeneralMemoryDescriptor::getPhysicalSegment64(IOByteCount offset, IOByteCount *lengthOfSegment)
2399 {
2400 return (getPhysicalSegment(offset, lengthOfSegment, kIOMemoryMapperNone));
2401 }
2402
2403 IOPhysicalAddress
2404 IOGeneralMemoryDescriptor::getPhysicalSegment(IOByteCount offset, IOByteCount *lengthOfSegment)
2405 {
2406 addr64_t address = 0;
2407 IOByteCount length = 0;
2408
2409 address = getPhysicalSegment(offset, lengthOfSegment, 0);
2410
2411 if (lengthOfSegment)
2412 length = *lengthOfSegment;
2413
2414 if ((address + length) > 0x100000000ULL)
2415 {
2416 panic("getPhysicalSegment() out of 32b range 0x%qx, len 0x%lx, class %s",
2417 address, (long) length, (getMetaClass())->getClassName());
2418 }
2419
2420 return ((IOPhysicalAddress) address);
2421 }
2422
2423 addr64_t
2424 IOMemoryDescriptor::getPhysicalSegment64(IOByteCount offset, IOByteCount *lengthOfSegment)
2425 {
2426 IOPhysicalAddress phys32;
2427 IOByteCount length;
2428 addr64_t phys64;
2429 IOMapper * mapper = 0;
2430
2431 phys32 = getPhysicalSegment(offset, lengthOfSegment);
2432 if (!phys32)
2433 return 0;
2434
2435 if (gIOSystemMapper)
2436 mapper = gIOSystemMapper;
2437
2438 if (mapper)
2439 {
2440 IOByteCount origLen;
2441
2442 phys64 = mapper->mapAddr(phys32);
2443 origLen = *lengthOfSegment;
2444 length = page_size - (phys64 & (page_size - 1));
2445 while ((length < origLen)
2446 && ((phys64 + length) == mapper->mapAddr(phys32 + length)))
2447 length += page_size;
2448 if (length > origLen)
2449 length = origLen;
2450
2451 *lengthOfSegment = length;
2452 }
2453 else
2454 phys64 = (addr64_t) phys32;
2455
2456 return phys64;
2457 }
2458
2459 IOPhysicalAddress
2460 IOMemoryDescriptor::getPhysicalSegment(IOByteCount offset, IOByteCount *lengthOfSegment)
2461 {
2462 return ((IOPhysicalAddress) getPhysicalSegment(offset, lengthOfSegment, 0));
2463 }
2464
2465 IOPhysicalAddress
2466 IOGeneralMemoryDescriptor::getSourceSegment(IOByteCount offset, IOByteCount *lengthOfSegment)
2467 {
2468 return ((IOPhysicalAddress) getPhysicalSegment(offset, lengthOfSegment, _kIOMemorySourceSegment));
2469 }
2470
2471 void * IOGeneralMemoryDescriptor::getVirtualSegment(IOByteCount offset,
2472 IOByteCount * lengthOfSegment)
2473 {
2474 if (_task == kernel_task)
2475 return (void *) getSourceSegment(offset, lengthOfSegment);
2476 else
2477 panic("IOGMD::getVirtualSegment deprecated");
2478
2479 return 0;
2480 }
2481 #endif /* !__LP64__ */
2482
2483 IOReturn
2484 IOMemoryDescriptor::dmaCommandOperation(DMACommandOps op, void *vData, UInt dataSize) const
2485 {
2486 IOMemoryDescriptor *md = const_cast<IOMemoryDescriptor *>(this);
2487 DMACommandOps params;
2488 IOReturn err;
2489
2490 params = (op & ~kIOMDDMACommandOperationMask & op);
2491 op &= kIOMDDMACommandOperationMask;
2492
2493 if (kIOMDGetCharacteristics == op) {
2494 if (dataSize < sizeof(IOMDDMACharacteristics))
2495 return kIOReturnUnderrun;
2496
2497 IOMDDMACharacteristics *data = (IOMDDMACharacteristics *) vData;
2498 data->fLength = getLength();
2499 data->fSGCount = 0;
2500 data->fDirection = getDirection();
2501 data->fIsPrepared = true; // Assume prepared - fails safe
2502 }
2503 else if (kIOMDWalkSegments == op) {
2504 if (dataSize < sizeof(IOMDDMAWalkSegmentArgs))
2505 return kIOReturnUnderrun;
2506
2507 IOMDDMAWalkSegmentArgs *data = (IOMDDMAWalkSegmentArgs *) vData;
2508 IOByteCount offset = (IOByteCount) data->fOffset;
2509
2510 IOPhysicalLength length;
2511 if (data->fMapped && IOMapper::gSystem)
2512 data->fIOVMAddr = md->getPhysicalSegment(offset, &length);
2513 else
2514 data->fIOVMAddr = md->getPhysicalSegment(offset, &length, kIOMemoryMapperNone);
2515 data->fLength = length;
2516 }
2517 else if (kIOMDAddDMAMapSpec == op) return kIOReturnUnsupported;
2518 else if (kIOMDDMAMap == op)
2519 {
2520 if (dataSize < sizeof(IOMDDMAMapArgs))
2521 return kIOReturnUnderrun;
2522 IOMDDMAMapArgs * data = (IOMDDMAMapArgs *) vData;
2523
2524 if (params) panic("class %s does not support IODMACommand::kIterateOnly", getMetaClass()->getClassName());
2525
2526 data->fMapContig = true;
2527 err = md->dmaMap(data->fMapper, &data->fMapSpec, data->fOffset, data->fLength, &data->fAlloc, &data->fAllocCount);
2528 return (err);
2529 }
2530 else return kIOReturnBadArgument;
2531
2532 return kIOReturnSuccess;
2533 }
2534
2535 IOReturn
2536 IOGeneralMemoryDescriptor::setPurgeable( IOOptionBits newState,
2537 IOOptionBits * oldState )
2538 {
2539 IOReturn err = kIOReturnSuccess;
2540
2541 vm_purgable_t control;
2542 int state;
2543
2544 if (_memRef)
2545 {
2546 err = super::setPurgeable(newState, oldState);
2547 }
2548 else
2549 {
2550 if (kIOMemoryThreadSafe & _flags)
2551 LOCK;
2552 do
2553 {
2554 // Find the appropriate vm_map for the given task
2555 vm_map_t curMap;
2556 if (_task == kernel_task && (kIOMemoryBufferPageable & _flags))
2557 {
2558 err = kIOReturnNotReady;
2559 break;
2560 }
2561 else if (!_task)
2562 {
2563 err = kIOReturnUnsupported;
2564 break;
2565 }
2566 else
2567 curMap = get_task_map(_task);
2568
2569 // can only do one range
2570 Ranges vec = _ranges;
2571 IOOptionBits type = _flags & kIOMemoryTypeMask;
2572 mach_vm_address_t addr;
2573 mach_vm_size_t len;
2574 getAddrLenForInd(addr, len, type, vec, 0);
2575
2576 err = purgeableControlBits(newState, &control, &state);
2577 if (kIOReturnSuccess != err)
2578 break;
2579 err = mach_vm_purgable_control(curMap, addr, control, &state);
2580 if (oldState)
2581 {
2582 if (kIOReturnSuccess == err)
2583 {
2584 err = purgeableStateBits(&state);
2585 *oldState = state;
2586 }
2587 }
2588 }
2589 while (false);
2590 if (kIOMemoryThreadSafe & _flags)
2591 UNLOCK;
2592 }
2593
2594 return (err);
2595 }
2596
2597 IOReturn IOMemoryDescriptor::setPurgeable( IOOptionBits newState,
2598 IOOptionBits * oldState )
2599 {
2600 IOReturn err = kIOReturnNotReady;
2601
2602 if (kIOMemoryThreadSafe & _flags) LOCK;
2603 if (_memRef) err = IOGeneralMemoryDescriptor::memoryReferenceSetPurgeable(_memRef, newState, oldState);
2604 if (kIOMemoryThreadSafe & _flags) UNLOCK;
2605
2606 return (err);
2607 }
2608
2609 IOReturn IOMemoryDescriptor::getPageCounts( IOByteCount * residentPageCount,
2610 IOByteCount * dirtyPageCount )
2611 {
2612 IOReturn err = kIOReturnNotReady;
2613
2614 if (kIOMemoryThreadSafe & _flags) LOCK;
2615 if (_memRef) err = IOGeneralMemoryDescriptor::memoryReferenceGetPageCounts(_memRef, residentPageCount, dirtyPageCount);
2616 if (kIOMemoryThreadSafe & _flags) UNLOCK;
2617
2618 return (err);
2619 }
2620
2621
2622 extern "C" void dcache_incoherent_io_flush64(addr64_t pa, unsigned int count);
2623 extern "C" void dcache_incoherent_io_store64(addr64_t pa, unsigned int count);
2624
2625 static void SetEncryptOp(addr64_t pa, unsigned int count)
2626 {
2627 ppnum_t page, end;
2628
2629 page = atop_64(round_page_64(pa));
2630 end = atop_64(trunc_page_64(pa + count));
2631 for (; page < end; page++)
2632 {
2633 pmap_clear_noencrypt(page);
2634 }
2635 }
2636
2637 static void ClearEncryptOp(addr64_t pa, unsigned int count)
2638 {
2639 ppnum_t page, end;
2640
2641 page = atop_64(round_page_64(pa));
2642 end = atop_64(trunc_page_64(pa + count));
2643 for (; page < end; page++)
2644 {
2645 pmap_set_noencrypt(page);
2646 }
2647 }
2648
2649 IOReturn IOMemoryDescriptor::performOperation( IOOptionBits options,
2650 IOByteCount offset, IOByteCount length )
2651 {
2652 IOByteCount remaining;
2653 unsigned int res;
2654 void (*func)(addr64_t pa, unsigned int count) = 0;
2655
2656 switch (options)
2657 {
2658 case kIOMemoryIncoherentIOFlush:
2659 func = &dcache_incoherent_io_flush64;
2660 break;
2661 case kIOMemoryIncoherentIOStore:
2662 func = &dcache_incoherent_io_store64;
2663 break;
2664
2665 case kIOMemorySetEncrypted:
2666 func = &SetEncryptOp;
2667 break;
2668 case kIOMemoryClearEncrypted:
2669 func = &ClearEncryptOp;
2670 break;
2671 }
2672
2673 if (!func)
2674 return (kIOReturnUnsupported);
2675
2676 if (kIOMemoryThreadSafe & _flags)
2677 LOCK;
2678
2679 res = 0x0UL;
2680 remaining = length = min(length, getLength() - offset);
2681 while (remaining)
2682 // (process another target segment?)
2683 {
2684 addr64_t dstAddr64;
2685 IOByteCount dstLen;
2686
2687 dstAddr64 = getPhysicalSegment(offset, &dstLen, kIOMemoryMapperNone);
2688 if (!dstAddr64)
2689 break;
2690
2691 // Clip segment length to remaining
2692 if (dstLen > remaining)
2693 dstLen = remaining;
2694
2695 (*func)(dstAddr64, dstLen);
2696
2697 offset += dstLen;
2698 remaining -= dstLen;
2699 }
2700
2701 if (kIOMemoryThreadSafe & _flags)
2702 UNLOCK;
2703
2704 return (remaining ? kIOReturnUnderrun : kIOReturnSuccess);
2705 }
2706
2707 #if defined(__i386__) || defined(__x86_64__)
2708 extern vm_offset_t first_avail;
2709 #define io_kernel_static_end first_avail
2710 #else
2711 #error io_kernel_static_end is undefined for this architecture
2712 #endif
2713
2714 static kern_return_t
2715 io_get_kernel_static_upl(
2716 vm_map_t /* map */,
2717 uintptr_t offset,
2718 vm_size_t *upl_size,
2719 upl_t *upl,
2720 upl_page_info_array_t page_list,
2721 unsigned int *count,
2722 ppnum_t *highest_page)
2723 {
2724 unsigned int pageCount, page;
2725 ppnum_t phys;
2726 ppnum_t highestPage = 0;
2727
2728 pageCount = atop_32(*upl_size);
2729 if (pageCount > *count)
2730 pageCount = *count;
2731
2732 *upl = NULL;
2733
2734 for (page = 0; page < pageCount; page++)
2735 {
2736 phys = pmap_find_phys(kernel_pmap, ((addr64_t)offset) + ptoa_64(page));
2737 if (!phys)
2738 break;
2739 page_list[page].phys_addr = phys;
2740 page_list[page].pageout = 0;
2741 page_list[page].absent = 0;
2742 page_list[page].dirty = 0;
2743 page_list[page].precious = 0;
2744 page_list[page].device = 0;
2745 if (phys > highestPage)
2746 highestPage = phys;
2747 }
2748
2749 *highest_page = highestPage;
2750
2751 return ((page >= pageCount) ? kIOReturnSuccess : kIOReturnVMError);
2752 }
2753
2754 IOReturn IOGeneralMemoryDescriptor::wireVirtual(IODirection forDirection)
2755 {
2756 IOOptionBits type = _flags & kIOMemoryTypeMask;
2757 IOReturn error = kIOReturnCannotWire;
2758 ioGMDData *dataP;
2759 upl_page_info_array_t pageInfo;
2760 ppnum_t mapBase;
2761
2762 assert(kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type || kIOMemoryTypeUIO == type);
2763
2764 if ((kIODirectionOutIn & forDirection) == kIODirectionNone)
2765 forDirection = (IODirection) (forDirection | getDirection());
2766
2767 int uplFlags; // This Mem Desc's default flags for upl creation
2768 switch (kIODirectionOutIn & forDirection)
2769 {
2770 case kIODirectionOut:
2771 // Pages do not need to be marked as dirty on commit
2772 uplFlags = UPL_COPYOUT_FROM;
2773 break;
2774
2775 case kIODirectionIn:
2776 default:
2777 uplFlags = 0; // i.e. ~UPL_COPYOUT_FROM
2778 break;
2779 }
2780
2781 if (_wireCount)
2782 {
2783 if ((kIOMemoryPreparedReadOnly & _flags) && !(UPL_COPYOUT_FROM & uplFlags))
2784 {
2785 OSReportWithBacktrace("IOMemoryDescriptor 0x%lx prepared read only", VM_KERNEL_ADDRPERM(this));
2786 error = kIOReturnNotWritable;
2787 }
2788 else error = kIOReturnSuccess;
2789 return (error);
2790 }
2791
2792 dataP = getDataP(_memoryEntries);
2793 IOMapper *mapper;
2794 mapper = dataP->fMapper;
2795 dataP->fMappedBase = 0;
2796
2797 uplFlags |= UPL_SET_IO_WIRE | UPL_SET_LITE;
2798 if (kIODirectionPrepareToPhys32 & forDirection)
2799 {
2800 if (!mapper) uplFlags |= UPL_NEED_32BIT_ADDR;
2801 if (dataP->fDMAMapNumAddressBits > 32) dataP->fDMAMapNumAddressBits = 32;
2802 }
2803 if (kIODirectionPrepareNoFault & forDirection) uplFlags |= UPL_REQUEST_NO_FAULT;
2804 if (kIODirectionPrepareNoZeroFill & forDirection) uplFlags |= UPL_NOZEROFILLIO;
2805 if (kIODirectionPrepareNonCoherent & forDirection) uplFlags |= UPL_REQUEST_FORCE_COHERENCY;
2806
2807 mapBase = 0;
2808
2809 // Note that appendBytes(NULL) zeros the data up to the desired length
2810 // and the length parameter is an unsigned int
2811 size_t uplPageSize = dataP->fPageCnt * sizeof(upl_page_info_t);
2812 if (uplPageSize > ((unsigned int)uplPageSize)) return (kIOReturnNoMemory);
2813 if (!_memoryEntries->appendBytes(0, uplPageSize)) return (kIOReturnNoMemory);
2814 dataP = 0;
2815
2816 // Find the appropriate vm_map for the given task
2817 vm_map_t curMap;
2818 if (_task == kernel_task && (kIOMemoryBufferPageable & _flags)) curMap = 0;
2819 else curMap = get_task_map(_task);
2820
2821 // Iterate over the vector of virtual ranges
2822 Ranges vec = _ranges;
2823 unsigned int pageIndex = 0;
2824 IOByteCount mdOffset = 0;
2825 ppnum_t highestPage = 0;
2826
2827 IOMemoryEntry * memRefEntry = 0;
2828 if (_memRef) memRefEntry = &_memRef->entries[0];
2829
2830 for (UInt range = 0; range < _rangesCount; range++) {
2831 ioPLBlock iopl;
2832 mach_vm_address_t startPage;
2833 mach_vm_size_t numBytes;
2834 ppnum_t highPage = 0;
2835
2836 // Get the startPage address and length of vec[range]
2837 getAddrLenForInd(startPage, numBytes, type, vec, range);
2838 iopl.fPageOffset = startPage & PAGE_MASK;
2839 numBytes += iopl.fPageOffset;
2840 startPage = trunc_page_64(startPage);
2841
2842 if (mapper)
2843 iopl.fMappedPage = mapBase + pageIndex;
2844 else
2845 iopl.fMappedPage = 0;
2846
2847 // Iterate over the current range, creating UPLs
2848 while (numBytes) {
2849 vm_address_t kernelStart = (vm_address_t) startPage;
2850 vm_map_t theMap;
2851 if (curMap) theMap = curMap;
2852 else if (_memRef)
2853 {
2854 theMap = NULL;
2855 }
2856 else
2857 {
2858 assert(_task == kernel_task);
2859 theMap = IOPageableMapForAddress(kernelStart);
2860 }
2861
2862 int ioplFlags = uplFlags;
2863 dataP = getDataP(_memoryEntries);
2864 pageInfo = getPageList(dataP);
2865 upl_page_list_ptr_t baseInfo = &pageInfo[pageIndex];
2866
2867 vm_size_t ioplSize = round_page(numBytes);
2868 unsigned int numPageInfo = atop_32(ioplSize);
2869
2870 if ((theMap == kernel_map) && (kernelStart < io_kernel_static_end)) {
2871 error = io_get_kernel_static_upl(theMap,
2872 kernelStart,
2873 &ioplSize,
2874 &iopl.fIOPL,
2875 baseInfo,
2876 &numPageInfo,
2877 &highPage);
2878 }
2879 else if (_memRef) {
2880 memory_object_offset_t entryOffset;
2881
2882 entryOffset = (mdOffset - iopl.fPageOffset - memRefEntry->offset);
2883 if (entryOffset >= memRefEntry->size) {
2884 memRefEntry++;
2885 if (memRefEntry >= &_memRef->entries[_memRef->count]) panic("memRefEntry");
2886 entryOffset = 0;
2887 }
2888 if (ioplSize > (memRefEntry->size - entryOffset)) ioplSize = (memRefEntry->size - entryOffset);
2889 error = memory_object_iopl_request(memRefEntry->entry,
2890 entryOffset,
2891 &ioplSize,
2892 &iopl.fIOPL,
2893 baseInfo,
2894 &numPageInfo,
2895 &ioplFlags);
2896 }
2897 else {
2898 assert(theMap);
2899 error = vm_map_create_upl(theMap,
2900 startPage,
2901 (upl_size_t*)&ioplSize,
2902 &iopl.fIOPL,
2903 baseInfo,
2904 &numPageInfo,
2905 &ioplFlags);
2906 }
2907
2908 assert(ioplSize);
2909 if (error != KERN_SUCCESS)
2910 goto abortExit;
2911
2912 if (iopl.fIOPL)
2913 highPage = upl_get_highest_page(iopl.fIOPL);
2914 if (highPage > highestPage)
2915 highestPage = highPage;
2916
2917 error = kIOReturnCannotWire;
2918
2919 if (baseInfo->device) {
2920 numPageInfo = 1;
2921 iopl.fFlags = kIOPLOnDevice;
2922 }
2923 else {
2924 iopl.fFlags = 0;
2925 }
2926
2927 iopl.fIOMDOffset = mdOffset;
2928 iopl.fPageInfo = pageIndex;
2929 if (mapper && pageIndex && (page_mask & (mdOffset + iopl.fPageOffset))) dataP->fDiscontig = true;
2930
2931 #if 0
2932 // used to remove the upl for auto prepares here, for some errant code
2933 // that freed memory before the descriptor pointing at it
2934 if ((_flags & kIOMemoryAutoPrepare) && iopl.fIOPL)
2935 {
2936 upl_commit(iopl.fIOPL, 0, 0);
2937 upl_deallocate(iopl.fIOPL);
2938 iopl.fIOPL = 0;
2939 }
2940 #endif
2941
2942 if (!_memoryEntries->appendBytes(&iopl, sizeof(iopl))) {
2943 // Clean up partial created and unsaved iopl
2944 if (iopl.fIOPL) {
2945 upl_abort(iopl.fIOPL, 0);
2946 upl_deallocate(iopl.fIOPL);
2947 }
2948 goto abortExit;
2949 }
2950 dataP = 0;
2951
2952 // Check for a multiple iopl's in one virtual range
2953 pageIndex += numPageInfo;
2954 mdOffset -= iopl.fPageOffset;
2955 if (ioplSize < numBytes) {
2956 numBytes -= ioplSize;
2957 startPage += ioplSize;
2958 mdOffset += ioplSize;
2959 iopl.fPageOffset = 0;
2960 if (mapper) iopl.fMappedPage = mapBase + pageIndex;
2961 }
2962 else {
2963 mdOffset += numBytes;
2964 break;
2965 }
2966 }
2967 }
2968
2969 _highestPage = highestPage;
2970
2971 if (UPL_COPYOUT_FROM & uplFlags) _flags |= kIOMemoryPreparedReadOnly;
2972
2973 return kIOReturnSuccess;
2974
2975 abortExit:
2976 {
2977 dataP = getDataP(_memoryEntries);
2978 UInt done = getNumIOPL(_memoryEntries, dataP);
2979 ioPLBlock *ioplList = getIOPLList(dataP);
2980
2981 for (UInt range = 0; range < done; range++)
2982 {
2983 if (ioplList[range].fIOPL) {
2984 upl_abort(ioplList[range].fIOPL, 0);
2985 upl_deallocate(ioplList[range].fIOPL);
2986 }
2987 }
2988 (void) _memoryEntries->initWithBytes(dataP, computeDataSize(0, 0)); // == setLength()
2989 }
2990
2991 if (error == KERN_FAILURE)
2992 error = kIOReturnCannotWire;
2993 else if (error == KERN_MEMORY_ERROR)
2994 error = kIOReturnNoResources;
2995
2996 return error;
2997 }
2998
2999 bool IOGeneralMemoryDescriptor::initMemoryEntries(size_t size, IOMapper * mapper)
3000 {
3001 ioGMDData * dataP;
3002 unsigned dataSize = size;
3003
3004 if (!_memoryEntries) {
3005 _memoryEntries = OSData::withCapacity(dataSize);
3006 if (!_memoryEntries)
3007 return false;
3008 }
3009 else if (!_memoryEntries->initWithCapacity(dataSize))
3010 return false;
3011
3012 _memoryEntries->appendBytes(0, computeDataSize(0, 0));
3013 dataP = getDataP(_memoryEntries);
3014
3015 if (mapper == kIOMapperWaitSystem) {
3016 IOMapper::checkForSystemMapper();
3017 mapper = IOMapper::gSystem;
3018 }
3019 dataP->fMapper = mapper;
3020 dataP->fPageCnt = 0;
3021 dataP->fMappedBase = 0;
3022 dataP->fDMAMapNumAddressBits = 64;
3023 dataP->fDMAMapAlignment = 0;
3024 dataP->fPreparationID = kIOPreparationIDUnprepared;
3025 dataP->fDiscontig = false;
3026 dataP->fCompletionError = false;
3027
3028 return (true);
3029 }
3030
3031 IOReturn IOMemoryDescriptor::dmaMap(
3032 IOMapper * mapper,
3033 const IODMAMapSpecification * mapSpec,
3034 uint64_t offset,
3035 uint64_t length,
3036 uint64_t * address,
3037 ppnum_t * mapPages)
3038 {
3039 IOMDDMAWalkSegmentState walkState;
3040 IOMDDMAWalkSegmentArgs * walkArgs = (IOMDDMAWalkSegmentArgs *) (void *)&walkState;
3041 IOOptionBits mdOp;
3042 IOReturn ret;
3043 IOPhysicalLength segLen;
3044 addr64_t phys, align, pageOffset;
3045 ppnum_t base, pageIndex, pageCount;
3046 uint64_t index;
3047 uint32_t mapOptions = 0;
3048
3049 if (!(kIOMemoryPreparedReadOnly & _flags)) mapOptions |= kIODMAMapWriteAccess;
3050
3051 walkArgs->fMapped = false;
3052 mdOp = kIOMDFirstSegment;
3053 pageCount = 0;
3054 for (index = 0; index < length; )
3055 {
3056 if (index && (page_mask & (index + pageOffset))) break;
3057
3058 walkArgs->fOffset = offset + index;
3059 ret = dmaCommandOperation(mdOp, &walkState, sizeof(walkState));
3060 mdOp = kIOMDWalkSegments;
3061 if (ret != kIOReturnSuccess) break;
3062 phys = walkArgs->fIOVMAddr;
3063 segLen = walkArgs->fLength;
3064
3065 align = (phys & page_mask);
3066 if (!index) pageOffset = align;
3067 else if (align) break;
3068 pageCount += atop_64(round_page_64(align + segLen));
3069 index += segLen;
3070 }
3071
3072 if (index < length) return (kIOReturnVMError);
3073
3074 base = mapper->iovmMapMemory(this, offset, pageCount,
3075 mapOptions, NULL, mapSpec);
3076
3077 if (!base) return (kIOReturnNoResources);
3078
3079 mdOp = kIOMDFirstSegment;
3080 for (pageIndex = 0, index = 0; index < length; )
3081 {
3082 walkArgs->fOffset = offset + index;
3083 ret = dmaCommandOperation(mdOp, &walkState, sizeof(walkState));
3084 mdOp = kIOMDWalkSegments;
3085 if (ret != kIOReturnSuccess) break;
3086 phys = walkArgs->fIOVMAddr;
3087 segLen = walkArgs->fLength;
3088
3089 ppnum_t page = atop_64(phys);
3090 ppnum_t count = atop_64(round_page_64(phys + segLen)) - page;
3091 while (count--)
3092 {
3093 mapper->iovmInsert(base, pageIndex, page);
3094 page++;
3095 pageIndex++;
3096 }
3097 index += segLen;
3098 }
3099 if (pageIndex != pageCount) panic("pageIndex");
3100
3101 *address = ptoa_64(base) + pageOffset;
3102 if (mapPages) *mapPages = pageCount;
3103
3104 return (kIOReturnSuccess);
3105 }
3106
3107 IOReturn IOGeneralMemoryDescriptor::dmaMap(
3108 IOMapper * mapper,
3109 const IODMAMapSpecification * mapSpec,
3110 uint64_t offset,
3111 uint64_t length,
3112 uint64_t * address,
3113 ppnum_t * mapPages)
3114 {
3115 IOReturn err = kIOReturnSuccess;
3116 ioGMDData * dataP;
3117 IOOptionBits type = _flags & kIOMemoryTypeMask;
3118
3119 *address = 0;
3120 if (kIOMemoryHostOnly & _flags) return (kIOReturnSuccess);
3121
3122 if ((type == kIOMemoryTypePhysical) || (type == kIOMemoryTypePhysical64)
3123 || offset || (length != _length))
3124 {
3125 err = super::dmaMap(mapper, mapSpec, offset, length, address, mapPages);
3126 }
3127 else if (_memoryEntries && _pages && (dataP = getDataP(_memoryEntries)))
3128 {
3129 const ioPLBlock * ioplList = getIOPLList(dataP);
3130 upl_page_info_t * pageList;
3131 uint32_t mapOptions = 0;
3132 ppnum_t base;
3133
3134 IODMAMapSpecification mapSpec;
3135 bzero(&mapSpec, sizeof(mapSpec));
3136 mapSpec.numAddressBits = dataP->fDMAMapNumAddressBits;
3137 mapSpec.alignment = dataP->fDMAMapAlignment;
3138
3139 // For external UPLs the fPageInfo field points directly to
3140 // the upl's upl_page_info_t array.
3141 if (ioplList->fFlags & kIOPLExternUPL)
3142 {
3143 pageList = (upl_page_info_t *) ioplList->fPageInfo;
3144 mapOptions |= kIODMAMapPagingPath;
3145 }
3146 else
3147 pageList = getPageList(dataP);
3148
3149 if (!(kIOMemoryPreparedReadOnly & _flags)) mapOptions |= kIODMAMapWriteAccess;
3150
3151 // Check for direct device non-paged memory
3152 if (ioplList->fFlags & kIOPLOnDevice) mapOptions |= kIODMAMapPhysicallyContiguous;
3153
3154 base = mapper->iovmMapMemory(
3155 this, offset, _pages, mapOptions, &pageList[0], &mapSpec);
3156 *address = ptoa_64(base) + (ioplList->fPageOffset & PAGE_MASK);
3157 if (mapPages) *mapPages = _pages;
3158 }
3159
3160 return (err);
3161 }
3162
3163 /*
3164 * prepare
3165 *
3166 * Prepare the memory for an I/O transfer. This involves paging in
3167 * the memory, if necessary, and wiring it down for the duration of
3168 * the transfer. The complete() method completes the processing of
3169 * the memory after the I/O transfer finishes. This method needn't
3170 * called for non-pageable memory.
3171 */
3172
3173 IOReturn IOGeneralMemoryDescriptor::prepare(IODirection forDirection)
3174 {
3175 IOReturn error = kIOReturnSuccess;
3176 IOOptionBits type = _flags & kIOMemoryTypeMask;
3177
3178 if ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type))
3179 return kIOReturnSuccess;
3180
3181 if (_prepareLock)
3182 IOLockLock(_prepareLock);
3183
3184 if (kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type || kIOMemoryTypeUIO == type)
3185 {
3186 error = wireVirtual(forDirection);
3187 }
3188
3189 if (kIOReturnSuccess == error)
3190 {
3191 if (1 == ++_wireCount)
3192 {
3193 if (kIOMemoryClearEncrypt & _flags)
3194 {
3195 performOperation(kIOMemoryClearEncrypted, 0, _length);
3196 }
3197 }
3198 }
3199
3200 if (_prepareLock)
3201 IOLockUnlock(_prepareLock);
3202
3203 return error;
3204 }
3205
3206 /*
3207 * complete
3208 *
3209 * Complete processing of the memory after an I/O transfer finishes.
3210 * This method should not be called unless a prepare was previously
3211 * issued; the prepare() and complete() must occur in pairs, before
3212 * before and after an I/O transfer involving pageable memory.
3213 */
3214
3215 IOReturn IOGeneralMemoryDescriptor::complete(IODirection forDirection)
3216 {
3217 IOOptionBits type = _flags & kIOMemoryTypeMask;
3218 ioGMDData * dataP;
3219
3220 if ((kIOMemoryTypePhysical == type) || (kIOMemoryTypePhysical64 == type))
3221 return kIOReturnSuccess;
3222
3223 if (_prepareLock)
3224 IOLockLock(_prepareLock);
3225
3226 assert(_wireCount);
3227
3228 if ((kIODirectionCompleteWithError & forDirection)
3229 && (dataP = getDataP(_memoryEntries)))
3230 dataP->fCompletionError = true;
3231
3232 if (_wireCount)
3233 {
3234 if ((kIOMemoryClearEncrypt & _flags) && (1 == _wireCount))
3235 {
3236 performOperation(kIOMemorySetEncrypted, 0, _length);
3237 }
3238
3239 _wireCount--;
3240 if (!_wireCount || (kIODirectionCompleteWithDataValid & forDirection))
3241 {
3242 IOOptionBits type = _flags & kIOMemoryTypeMask;
3243 dataP = getDataP(_memoryEntries);
3244 ioPLBlock *ioplList = getIOPLList(dataP);
3245 UInt ind, count = getNumIOPL(_memoryEntries, dataP);
3246
3247 if (_wireCount)
3248 {
3249 // kIODirectionCompleteWithDataValid & forDirection
3250 if (kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type || kIOMemoryTypeUIO == type)
3251 {
3252 for (ind = 0; ind < count; ind++)
3253 {
3254 if (ioplList[ind].fIOPL) iopl_valid_data(ioplList[ind].fIOPL);
3255 }
3256 }
3257 }
3258 else
3259 {
3260 #if IOMD_DEBUG_DMAACTIVE
3261 if (__iomd_reservedA) panic("complete() while dma active");
3262 #endif /* IOMD_DEBUG_DMAACTIVE */
3263
3264 if (dataP->fMappedBase) {
3265 dataP->fMapper->iovmFree(atop_64(dataP->fMappedBase), _pages);
3266 dataP->fMappedBase = 0;
3267 }
3268 // Only complete iopls that we created which are for TypeVirtual
3269 if (kIOMemoryTypeVirtual == type || kIOMemoryTypeVirtual64 == type || kIOMemoryTypeUIO == type) {
3270 for (ind = 0; ind < count; ind++)
3271 if (ioplList[ind].fIOPL) {
3272 if (dataP->fCompletionError)
3273 upl_abort(ioplList[ind].fIOPL, 0 /*!UPL_ABORT_DUMP_PAGES*/);
3274 else
3275 upl_commit(ioplList[ind].fIOPL, 0, 0);
3276 upl_deallocate(ioplList[ind].fIOPL);
3277 }
3278 } else if (kIOMemoryTypeUPL == type) {
3279 upl_set_referenced(ioplList[0].fIOPL, false);
3280 }
3281
3282 (void) _memoryEntries->initWithBytes(dataP, computeDataSize(0, 0)); // == setLength()
3283
3284 dataP->fPreparationID = kIOPreparationIDUnprepared;
3285 }
3286 }
3287 }
3288
3289 if (_prepareLock)
3290 IOLockUnlock(_prepareLock);
3291
3292 return kIOReturnSuccess;
3293 }
3294
3295 IOReturn IOGeneralMemoryDescriptor::doMap(
3296 vm_map_t __addressMap,
3297 IOVirtualAddress * __address,
3298 IOOptionBits options,
3299 IOByteCount __offset,
3300 IOByteCount __length )
3301
3302 {
3303 #ifndef __LP64__
3304 if (!(kIOMap64Bit & options)) panic("IOGeneralMemoryDescriptor::doMap !64bit");
3305 #endif /* !__LP64__ */
3306
3307 kern_return_t err;
3308
3309 IOMemoryMap * mapping = (IOMemoryMap *) *__address;
3310 mach_vm_size_t offset = mapping->fOffset + __offset;
3311 mach_vm_size_t length = mapping->fLength;
3312
3313 IOOptionBits type = _flags & kIOMemoryTypeMask;
3314 Ranges vec = _ranges;
3315
3316 mach_vm_address_t range0Addr = 0;
3317 mach_vm_size_t range0Len = 0;
3318
3319 if ((offset >= _length) || ((offset + length) > _length))
3320 return( kIOReturnBadArgument );
3321
3322 if (vec.v)
3323 getAddrLenForInd(range0Addr, range0Len, type, vec, 0);
3324
3325 // mapping source == dest? (could be much better)
3326 if (_task
3327 && (mapping->fAddressTask == _task)
3328 && (mapping->fAddressMap == get_task_map(_task))
3329 && (options & kIOMapAnywhere)
3330 && (1 == _rangesCount)
3331 && (0 == offset)
3332 && range0Addr
3333 && (length <= range0Len))
3334 {
3335 mapping->fAddress = range0Addr;
3336 mapping->fOptions |= kIOMapStatic;
3337
3338 return( kIOReturnSuccess );
3339 }
3340
3341 if (!_memRef)
3342 {
3343 IOOptionBits createOptions = 0;
3344 if (!(kIOMapReadOnly & options))
3345 {
3346 createOptions |= kIOMemoryReferenceWrite;
3347 #if DEVELOPMENT || DEBUG
3348 if (kIODirectionOut == (kIODirectionOutIn & _flags))
3349 {
3350 OSReportWithBacktrace("warning: creating writable mapping from IOMemoryDescriptor(kIODirectionOut) - use kIOMapReadOnly or change direction");
3351 }
3352 #endif
3353 }
3354 err = memoryReferenceCreate(createOptions, &_memRef);
3355 if (kIOReturnSuccess != err) return (err);
3356 }
3357
3358 memory_object_t pager;
3359 pager = (memory_object_t) (reserved ? reserved->dp.devicePager : 0);
3360
3361 // <upl_transpose //
3362 if ((kIOMapReference|kIOMapUnique) == ((kIOMapReference|kIOMapUnique) & options))
3363 {
3364 do
3365 {
3366 upl_t redirUPL2;
3367 vm_size_t size;
3368 int flags;
3369 unsigned int lock_count;
3370
3371 if (!_memRef || (1 != _memRef->count))
3372 {
3373 err = kIOReturnNotReadable;
3374 break;
3375 }
3376
3377 size = round_page(mapping->fLength);
3378 flags = UPL_COPYOUT_FROM | UPL_SET_INTERNAL
3379 | UPL_SET_LITE | UPL_SET_IO_WIRE | UPL_BLOCK_ACCESS;
3380
3381 if (KERN_SUCCESS != memory_object_iopl_request(_memRef->entries[0].entry, 0, &size, &redirUPL2,
3382 NULL, NULL,
3383 &flags))
3384 redirUPL2 = NULL;
3385
3386 for (lock_count = 0;
3387 IORecursiveLockHaveLock(gIOMemoryLock);
3388 lock_count++) {
3389 UNLOCK;
3390 }
3391 err = upl_transpose(redirUPL2, mapping->fRedirUPL);
3392 for (;
3393 lock_count;
3394 lock_count--) {
3395 LOCK;
3396 }
3397
3398 if (kIOReturnSuccess != err)
3399 {
3400 IOLog("upl_transpose(%x)\n", err);
3401 err = kIOReturnSuccess;
3402 }
3403
3404 if (redirUPL2)
3405 {
3406 upl_commit(redirUPL2, NULL, 0);
3407 upl_deallocate(redirUPL2);
3408 redirUPL2 = 0;
3409 }
3410 {
3411 // swap the memEntries since they now refer to different vm_objects
3412 IOMemoryReference * me = _memRef;
3413 _memRef = mapping->fMemory->_memRef;
3414 mapping->fMemory->_memRef = me;
3415 }
3416 if (pager)
3417 err = populateDevicePager( pager, mapping->fAddressMap, mapping->fAddress, offset, length, options );
3418 }
3419 while (false);
3420 }
3421 // upl_transpose> //
3422 else
3423 {
3424 err = memoryReferenceMap(_memRef, mapping->fAddressMap, offset, length, options, &mapping->fAddress);
3425
3426 if ((err == KERN_SUCCESS) && pager)
3427 {
3428 err = populateDevicePager(pager, mapping->fAddressMap, mapping->fAddress, offset, length, options);
3429 if (err != KERN_SUCCESS)
3430 {
3431 doUnmap(mapping->fAddressMap, (IOVirtualAddress) mapping, 0);
3432 }
3433 else if (kIOMapDefaultCache == (options & kIOMapCacheMask))
3434 {
3435 mapping->fOptions |= ((_flags & kIOMemoryBufferCacheMask) >> kIOMemoryBufferCacheShift);
3436 }
3437 }
3438 }
3439
3440 return (err);
3441 }
3442
3443 IOReturn IOGeneralMemoryDescriptor::doUnmap(
3444 vm_map_t addressMap,
3445 IOVirtualAddress __address,
3446 IOByteCount __length )
3447 {
3448 return (super::doUnmap(addressMap, __address, __length));
3449 }
3450
3451 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
3452
3453 #undef super
3454 #define super OSObject
3455
3456 OSDefineMetaClassAndStructors( IOMemoryMap, OSObject )
3457
3458 OSMetaClassDefineReservedUnused(IOMemoryMap, 0);
3459 OSMetaClassDefineReservedUnused(IOMemoryMap, 1);
3460 OSMetaClassDefineReservedUnused(IOMemoryMap, 2);
3461 OSMetaClassDefineReservedUnused(IOMemoryMap, 3);
3462 OSMetaClassDefineReservedUnused(IOMemoryMap, 4);
3463 OSMetaClassDefineReservedUnused(IOMemoryMap, 5);
3464 OSMetaClassDefineReservedUnused(IOMemoryMap, 6);
3465 OSMetaClassDefineReservedUnused(IOMemoryMap, 7);
3466
3467 /* ex-inline function implementation */
3468 IOPhysicalAddress IOMemoryMap::getPhysicalAddress()
3469 { return( getPhysicalSegment( 0, 0 )); }
3470
3471 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
3472
3473 bool IOMemoryMap::init(
3474 task_t intoTask,
3475 mach_vm_address_t toAddress,
3476 IOOptionBits _options,
3477 mach_vm_size_t _offset,
3478 mach_vm_size_t _length )
3479 {
3480 if (!intoTask)
3481 return( false);
3482
3483 if (!super::init())
3484 return(false);
3485
3486 fAddressMap = get_task_map(intoTask);
3487 if (!fAddressMap)
3488 return(false);
3489 vm_map_reference(fAddressMap);
3490
3491 fAddressTask = intoTask;
3492 fOptions = _options;
3493 fLength = _length;
3494 fOffset = _offset;
3495 fAddress = toAddress;
3496
3497 return (true);
3498 }
3499
3500 bool IOMemoryMap::setMemoryDescriptor(IOMemoryDescriptor * _memory, mach_vm_size_t _offset)
3501 {
3502 if (!_memory)
3503 return(false);
3504
3505 if (!fSuperMap)
3506 {
3507 if( (_offset + fLength) > _memory->getLength())
3508 return( false);
3509 fOffset = _offset;
3510 }
3511
3512 _memory->retain();
3513 if (fMemory)
3514 {
3515 if (fMemory != _memory)
3516 fMemory->removeMapping(this);
3517 fMemory->release();
3518 }
3519 fMemory = _memory;
3520
3521 return( true );
3522 }
3523
3524 IOReturn IOMemoryDescriptor::doMap(
3525 vm_map_t __addressMap,
3526 IOVirtualAddress * __address,
3527 IOOptionBits options,
3528 IOByteCount __offset,
3529 IOByteCount __length )
3530 {
3531 return (kIOReturnUnsupported);
3532 }
3533
3534 IOReturn IOMemoryDescriptor::handleFault(
3535 void * _pager,
3536 mach_vm_size_t sourceOffset,
3537 mach_vm_size_t length)
3538 {
3539 if( kIOMemoryRedirected & _flags)
3540 {
3541 #if DEBUG
3542 IOLog("sleep mem redirect %p, %qx\n", this, sourceOffset);
3543 #endif
3544 do {
3545 SLEEP;
3546 } while( kIOMemoryRedirected & _flags );
3547 }
3548 return (kIOReturnSuccess);
3549 }
3550
3551 IOReturn IOMemoryDescriptor::populateDevicePager(
3552 void * _pager,
3553 vm_map_t addressMap,
3554 mach_vm_address_t address,
3555 mach_vm_size_t sourceOffset,
3556 mach_vm_size_t length,
3557 IOOptionBits options )
3558 {
3559 IOReturn err = kIOReturnSuccess;
3560 memory_object_t pager = (memory_object_t) _pager;
3561 mach_vm_size_t size;
3562 mach_vm_size_t bytes;
3563 mach_vm_size_t page;
3564 mach_vm_size_t pageOffset;
3565 mach_vm_size_t pagerOffset;
3566 IOPhysicalLength segLen;
3567 addr64_t physAddr;
3568
3569 physAddr = getPhysicalSegment( sourceOffset, &segLen, kIOMemoryMapperNone );
3570 assert( physAddr );
3571 pageOffset = physAddr - trunc_page_64( physAddr );
3572 pagerOffset = sourceOffset;
3573
3574 size = length + pageOffset;
3575 physAddr -= pageOffset;
3576
3577 segLen += pageOffset;
3578 bytes = size;
3579 do
3580 {
3581 // in the middle of the loop only map whole pages
3582 if( segLen >= bytes) segLen = bytes;
3583 else if (segLen != trunc_page(segLen)) err = kIOReturnVMError;
3584 if (physAddr != trunc_page_64(physAddr)) err = kIOReturnBadArgument;
3585
3586 if (kIOReturnSuccess != err) break;
3587
3588 if (reserved && reserved->dp.pagerContig)
3589 {
3590 IOPhysicalLength allLen;
3591 addr64_t allPhys;
3592
3593 allPhys = getPhysicalSegment( 0, &allLen, kIOMemoryMapperNone );
3594 assert( allPhys );
3595 err = device_pager_populate_object( pager, 0, atop_64(allPhys), round_page(allLen) );
3596 }
3597 else
3598 {
3599 for( page = 0;
3600 (page < segLen) && (KERN_SUCCESS == err);
3601 page += page_size)
3602 {
3603 err = device_pager_populate_object(pager, pagerOffset,
3604 (ppnum_t)(atop_64(physAddr + page)), page_size);
3605 pagerOffset += page_size;
3606 }
3607 }
3608 assert (KERN_SUCCESS == err);
3609 if (err) break;
3610
3611 // This call to vm_fault causes an early pmap level resolution
3612 // of the mappings created above for kernel mappings, since
3613 // faulting in later can't take place from interrupt level.
3614 if ((addressMap == kernel_map) && !(kIOMemoryRedirected & _flags))
3615 {
3616 vm_fault(addressMap,
3617 (vm_map_offset_t)trunc_page_64(address),
3618 VM_PROT_READ|VM_PROT_WRITE,
3619 FALSE, THREAD_UNINT, NULL,
3620 (vm_map_offset_t)0);
3621 }
3622
3623 sourceOffset += segLen - pageOffset;
3624 address += segLen;
3625 bytes -= segLen;
3626 pageOffset = 0;
3627 }
3628 while (bytes && (physAddr = getPhysicalSegment( sourceOffset, &segLen, kIOMemoryMapperNone )));
3629
3630 if (bytes)
3631 err = kIOReturnBadArgument;
3632
3633 return (err);
3634 }
3635
3636 IOReturn IOMemoryDescriptor::doUnmap(
3637 vm_map_t addressMap,
3638 IOVirtualAddress __address,
3639 IOByteCount __length )
3640 {
3641 IOReturn err;
3642 mach_vm_address_t address;
3643 mach_vm_size_t length;
3644
3645 if (__length)
3646 {
3647 address = __address;
3648 length = __length;
3649 }
3650 else
3651 {
3652 addressMap = ((IOMemoryMap *) __address)->fAddressMap;
3653 address = ((IOMemoryMap *) __address)->fAddress;
3654 length = ((IOMemoryMap *) __address)->fLength;
3655 }
3656
3657 if ((addressMap == kernel_map) && (kIOMemoryBufferPageable & _flags))
3658 addressMap = IOPageableMapForAddress( address );
3659
3660 #if DEBUG
3661 if( kIOLogMapping & gIOKitDebug)
3662 IOLog("IOMemoryDescriptor::doUnmap map %p, 0x%qx:0x%qx\n",
3663 addressMap, address, length );
3664 #endif
3665
3666 err = mach_vm_deallocate( addressMap, address, length );
3667
3668 return (err);
3669 }
3670
3671 IOReturn IOMemoryDescriptor::redirect( task_t safeTask, bool doRedirect )
3672 {
3673 IOReturn err = kIOReturnSuccess;
3674 IOMemoryMap * mapping = 0;
3675 OSIterator * iter;
3676
3677 LOCK;
3678
3679 if( doRedirect)
3680 _flags |= kIOMemoryRedirected;
3681 else
3682 _flags &= ~kIOMemoryRedirected;
3683
3684 do {
3685 if( (iter = OSCollectionIterator::withCollection( _mappings))) {
3686
3687 memory_object_t pager;
3688
3689 if( reserved)
3690 pager = (memory_object_t) reserved->dp.devicePager;
3691 else
3692 pager = MACH_PORT_NULL;
3693
3694 while( (mapping = (IOMemoryMap *) iter->getNextObject()))
3695 {
3696 mapping->redirect( safeTask, doRedirect );
3697 if (!doRedirect && !safeTask && pager && (kernel_map == mapping->fAddressMap))
3698 {
3699 err = populateDevicePager(pager, mapping->fAddressMap, mapping->fAddress, mapping->fOffset, mapping->fLength, kIOMapDefaultCache );
3700 }
3701 }
3702
3703 iter->release();
3704 }
3705 } while( false );
3706
3707 if (!doRedirect)
3708 {
3709 WAKEUP;
3710 }
3711
3712 UNLOCK;
3713
3714 #ifndef __LP64__
3715 // temporary binary compatibility
3716 IOSubMemoryDescriptor * subMem;
3717 if( (subMem = OSDynamicCast( IOSubMemoryDescriptor, this)))
3718 err = subMem->redirect( safeTask, doRedirect );
3719 else
3720 err = kIOReturnSuccess;
3721 #endif /* !__LP64__ */
3722
3723 return( err );
3724 }
3725
3726 IOReturn IOMemoryMap::redirect( task_t safeTask, bool doRedirect )
3727 {
3728 IOReturn err = kIOReturnSuccess;
3729
3730 if( fSuperMap) {
3731 // err = ((IOMemoryMap *)superMap)->redirect( safeTask, doRedirect );
3732 } else {
3733
3734 LOCK;
3735
3736 do
3737 {
3738 if (!fAddress)
3739 break;
3740 if (!fAddressMap)
3741 break;
3742
3743 if ((!safeTask || (get_task_map(safeTask) != fAddressMap))
3744 && (0 == (fOptions & kIOMapStatic)))
3745 {
3746 IOUnmapPages( fAddressMap, fAddress, fLength );
3747 err = kIOReturnSuccess;
3748 #if DEBUG
3749 IOLog("IOMemoryMap::redirect(%d, %p) 0x%qx:0x%qx from %p\n", doRedirect, this, fAddress, fLength, fAddressMap);
3750 #endif
3751 }
3752 else if (kIOMapWriteCombineCache == (fOptions & kIOMapCacheMask))
3753 {
3754 IOOptionBits newMode;
3755 newMode = (fOptions & ~kIOMapCacheMask) | (doRedirect ? kIOMapInhibitCache : kIOMapWriteCombineCache);
3756 IOProtectCacheMode(fAddressMap, fAddress, fLength, newMode);
3757 }
3758 }
3759 while (false);
3760 UNLOCK;
3761 }
3762
3763 if ((((fMemory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical)
3764 || ((fMemory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical64))
3765 && safeTask
3766 && (doRedirect != (0 != (fMemory->_flags & kIOMemoryRedirected))))
3767 fMemory->redirect(safeTask, doRedirect);
3768
3769 return( err );
3770 }
3771
3772 IOReturn IOMemoryMap::unmap( void )
3773 {
3774 IOReturn err;
3775
3776 LOCK;
3777
3778 if( fAddress && fAddressMap && (0 == fSuperMap) && fMemory
3779 && (0 == (fOptions & kIOMapStatic))) {
3780
3781 err = fMemory->doUnmap(fAddressMap, (IOVirtualAddress) this, 0);
3782
3783 } else
3784 err = kIOReturnSuccess;
3785
3786 if (fAddressMap)
3787 {
3788 vm_map_deallocate(fAddressMap);
3789 fAddressMap = 0;
3790 }
3791
3792 fAddress = 0;
3793
3794 UNLOCK;
3795
3796 return( err );
3797 }
3798
3799 void IOMemoryMap::taskDied( void )
3800 {
3801 LOCK;
3802 if (fUserClientUnmap)
3803 unmap();
3804 if( fAddressMap) {
3805 vm_map_deallocate(fAddressMap);
3806 fAddressMap = 0;
3807 }
3808 fAddressTask = 0;
3809 fAddress = 0;
3810 UNLOCK;
3811 }
3812
3813 IOReturn IOMemoryMap::userClientUnmap( void )
3814 {
3815 fUserClientUnmap = true;
3816 return (kIOReturnSuccess);
3817 }
3818
3819 // Overload the release mechanism. All mappings must be a member
3820 // of a memory descriptors _mappings set. This means that we
3821 // always have 2 references on a mapping. When either of these mappings
3822 // are released we need to free ourselves.
3823 void IOMemoryMap::taggedRelease(const void *tag) const
3824 {
3825 LOCK;
3826 super::taggedRelease(tag, 2);
3827 UNLOCK;
3828 }
3829
3830 void IOMemoryMap::free()
3831 {
3832 unmap();
3833
3834 if (fMemory)
3835 {
3836 LOCK;
3837 fMemory->removeMapping(this);
3838 UNLOCK;
3839 fMemory->release();
3840 }
3841
3842 if (fOwner && (fOwner != fMemory))
3843 {
3844 LOCK;
3845 fOwner->removeMapping(this);
3846 UNLOCK;
3847 }
3848
3849 if (fSuperMap)
3850 fSuperMap->release();
3851
3852 if (fRedirUPL) {
3853 upl_commit(fRedirUPL, NULL, 0);
3854 upl_deallocate(fRedirUPL);
3855 }
3856
3857 super::free();
3858 }
3859
3860 IOByteCount IOMemoryMap::getLength()
3861 {
3862 return( fLength );
3863 }
3864
3865 IOVirtualAddress IOMemoryMap::getVirtualAddress()
3866 {
3867 #ifndef __LP64__
3868 if (fSuperMap)
3869 fSuperMap->getVirtualAddress();
3870 else if (fAddressMap
3871 && vm_map_is_64bit(fAddressMap)
3872 && (sizeof(IOVirtualAddress) < 8))
3873 {
3874 OSReportWithBacktrace("IOMemoryMap::getVirtualAddress(0x%qx) called on 64b map; use ::getAddress()", fAddress);
3875 }
3876 #endif /* !__LP64__ */
3877
3878 return (fAddress);
3879 }
3880
3881 #ifndef __LP64__
3882 mach_vm_address_t IOMemoryMap::getAddress()
3883 {
3884 return( fAddress);
3885 }
3886
3887 mach_vm_size_t IOMemoryMap::getSize()
3888 {
3889 return( fLength );
3890 }
3891 #endif /* !__LP64__ */
3892
3893
3894 task_t IOMemoryMap::getAddressTask()
3895 {
3896 if( fSuperMap)
3897 return( fSuperMap->getAddressTask());
3898 else
3899 return( fAddressTask);
3900 }
3901
3902 IOOptionBits IOMemoryMap::getMapOptions()
3903 {
3904 return( fOptions);
3905 }
3906
3907 IOMemoryDescriptor * IOMemoryMap::getMemoryDescriptor()
3908 {
3909 return( fMemory );
3910 }
3911
3912 IOMemoryMap * IOMemoryMap::copyCompatible(
3913 IOMemoryMap * newMapping )
3914 {
3915 task_t task = newMapping->getAddressTask();
3916 mach_vm_address_t toAddress = newMapping->fAddress;
3917 IOOptionBits _options = newMapping->fOptions;
3918 mach_vm_size_t _offset = newMapping->fOffset;
3919 mach_vm_size_t _length = newMapping->fLength;
3920
3921 if( (!task) || (!fAddressMap) || (fAddressMap != get_task_map(task)))
3922 return( 0 );
3923 if( (fOptions ^ _options) & kIOMapReadOnly)
3924 return( 0 );
3925 if( (kIOMapDefaultCache != (_options & kIOMapCacheMask))
3926 && ((fOptions ^ _options) & kIOMapCacheMask))
3927 return( 0 );
3928
3929 if( (0 == (_options & kIOMapAnywhere)) && (fAddress != toAddress))
3930 return( 0 );
3931
3932 if( _offset < fOffset)
3933 return( 0 );
3934
3935 _offset -= fOffset;
3936
3937 if( (_offset + _length) > fLength)
3938 return( 0 );
3939
3940 retain();
3941 if( (fLength == _length) && (!_offset))
3942 {
3943 newMapping = this;
3944 }
3945 else
3946 {
3947 newMapping->fSuperMap = this;
3948 newMapping->fOffset = fOffset + _offset;
3949 newMapping->fAddress = fAddress + _offset;
3950 }
3951
3952 return( newMapping );
3953 }
3954
3955 IOReturn IOMemoryMap::wireRange(
3956 uint32_t options,
3957 mach_vm_size_t offset,
3958 mach_vm_size_t length)
3959 {
3960 IOReturn kr;
3961 mach_vm_address_t start = trunc_page_64(fAddress + offset);
3962 mach_vm_address_t end = round_page_64(fAddress + offset + length);
3963
3964 if (kIODirectionOutIn & options)
3965 {
3966 kr = vm_map_wire(fAddressMap, start, end, (kIODirectionOutIn & options), FALSE);
3967 }
3968 else
3969 {
3970 kr = vm_map_unwire(fAddressMap, start, end, FALSE);
3971 }
3972
3973 return (kr);
3974 }
3975
3976
3977 IOPhysicalAddress
3978 #ifdef __LP64__
3979 IOMemoryMap::getPhysicalSegment( IOByteCount _offset, IOPhysicalLength * _length, IOOptionBits _options)
3980 #else /* !__LP64__ */
3981 IOMemoryMap::getPhysicalSegment( IOByteCount _offset, IOPhysicalLength * _length)
3982 #endif /* !__LP64__ */
3983 {
3984 IOPhysicalAddress address;
3985
3986 LOCK;
3987 #ifdef __LP64__
3988 address = fMemory->getPhysicalSegment( fOffset + _offset, _length, _options );
3989 #else /* !__LP64__ */
3990 address = fMemory->getPhysicalSegment( fOffset + _offset, _length );
3991 #endif /* !__LP64__ */
3992 UNLOCK;
3993
3994 return( address );
3995 }
3996
3997 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
3998
3999 #undef super
4000 #define super OSObject
4001
4002 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
4003
4004 void IOMemoryDescriptor::initialize( void )
4005 {
4006 if( 0 == gIOMemoryLock)
4007 gIOMemoryLock = IORecursiveLockAlloc();
4008
4009 gIOLastPage = IOGetLastPageNumber();
4010 }
4011
4012 void IOMemoryDescriptor::free( void )
4013 {
4014 if( _mappings)
4015 _mappings->release();
4016
4017 super::free();
4018 }
4019
4020 IOMemoryMap * IOMemoryDescriptor::setMapping(
4021 task_t intoTask,
4022 IOVirtualAddress mapAddress,
4023 IOOptionBits options )
4024 {
4025 return (createMappingInTask( intoTask, mapAddress,
4026 options | kIOMapStatic,
4027 0, getLength() ));
4028 }
4029
4030 IOMemoryMap * IOMemoryDescriptor::map(
4031 IOOptionBits options )
4032 {
4033 return (createMappingInTask( kernel_task, 0,
4034 options | kIOMapAnywhere,
4035 0, getLength() ));
4036 }
4037
4038 #ifndef __LP64__
4039 IOMemoryMap * IOMemoryDescriptor::map(
4040 task_t intoTask,
4041 IOVirtualAddress atAddress,
4042 IOOptionBits options,
4043 IOByteCount offset,
4044 IOByteCount length )
4045 {
4046 if ((!(kIOMapAnywhere & options)) && vm_map_is_64bit(get_task_map(intoTask)))
4047 {
4048 OSReportWithBacktrace("IOMemoryDescriptor::map() in 64b task, use ::createMappingInTask()");
4049 return (0);
4050 }
4051
4052 return (createMappingInTask(intoTask, atAddress,
4053 options, offset, length));
4054 }
4055 #endif /* !__LP64__ */
4056
4057 IOMemoryMap * IOMemoryDescriptor::createMappingInTask(
4058 task_t intoTask,
4059 mach_vm_address_t atAddress,
4060 IOOptionBits options,
4061 mach_vm_size_t offset,
4062 mach_vm_size_t length)
4063 {
4064 IOMemoryMap * result;
4065 IOMemoryMap * mapping;
4066
4067 if (0 == length)
4068 length = getLength();
4069
4070 mapping = new IOMemoryMap;
4071
4072 if( mapping
4073 && !mapping->init( intoTask, atAddress,
4074 options, offset, length )) {
4075 mapping->release();
4076 mapping = 0;
4077 }
4078
4079 if (mapping)
4080 result = makeMapping(this, intoTask, (IOVirtualAddress) mapping, options | kIOMap64Bit, 0, 0);
4081 else
4082 result = 0;
4083
4084 #if DEBUG
4085 if (!result)
4086 IOLog("createMappingInTask failed desc %p, addr %qx, options %x, offset %qx, length %llx\n",
4087 this, atAddress, (uint32_t) options, offset, length);
4088 #endif
4089
4090 return (result);
4091 }
4092
4093 #ifndef __LP64__ // there is only a 64 bit version for LP64
4094 IOReturn IOMemoryMap::redirect(IOMemoryDescriptor * newBackingMemory,
4095 IOOptionBits options,
4096 IOByteCount offset)
4097 {
4098 return (redirect(newBackingMemory, options, (mach_vm_size_t)offset));
4099 }
4100 #endif
4101
4102 IOReturn IOMemoryMap::redirect(IOMemoryDescriptor * newBackingMemory,
4103 IOOptionBits options,
4104 mach_vm_size_t offset)
4105 {
4106 IOReturn err = kIOReturnSuccess;
4107 IOMemoryDescriptor * physMem = 0;
4108
4109 LOCK;
4110
4111 if (fAddress && fAddressMap) do
4112 {
4113 if (((fMemory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical)
4114 || ((fMemory->_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical64))
4115 {
4116 physMem = fMemory;
4117 physMem->retain();
4118 }
4119
4120 if (!fRedirUPL && fMemory->_memRef && (1 == fMemory->_memRef->count))
4121 {
4122 vm_size_t size = round_page(fLength);
4123 int flags = UPL_COPYOUT_FROM | UPL_SET_INTERNAL
4124 | UPL_SET_LITE | UPL_SET_IO_WIRE | UPL_BLOCK_ACCESS;
4125 if (KERN_SUCCESS != memory_object_iopl_request(fMemory->_memRef->entries[0].entry, 0, &size, &fRedirUPL,
4126 NULL, NULL,
4127 &flags))
4128 fRedirUPL = 0;
4129
4130 if (physMem)
4131 {
4132 IOUnmapPages( fAddressMap, fAddress, fLength );
4133 if ((false))
4134 physMem->redirect(0, true);
4135 }
4136 }
4137
4138 if (newBackingMemory)
4139 {
4140 if (newBackingMemory != fMemory)
4141 {
4142 fOffset = 0;
4143 if (this != newBackingMemory->makeMapping(newBackingMemory, fAddressTask, (IOVirtualAddress) this,
4144 options | kIOMapUnique | kIOMapReference | kIOMap64Bit,
4145 offset, fLength))
4146 err = kIOReturnError;
4147 }
4148 if (fRedirUPL)
4149 {
4150 upl_commit(fRedirUPL, NULL, 0);
4151 upl_deallocate(fRedirUPL);
4152 fRedirUPL = 0;
4153 }
4154 if ((false) && physMem)
4155 physMem->redirect(0, false);
4156 }
4157 }
4158 while (false);
4159
4160 UNLOCK;
4161
4162 if (physMem)
4163 physMem->release();
4164
4165 return (err);
4166 }
4167
4168 IOMemoryMap * IOMemoryDescriptor::makeMapping(
4169 IOMemoryDescriptor * owner,
4170 task_t __intoTask,
4171 IOVirtualAddress __address,
4172 IOOptionBits options,
4173 IOByteCount __offset,
4174 IOByteCount __length )
4175 {
4176 #ifndef __LP64__
4177 if (!(kIOMap64Bit & options)) panic("IOMemoryDescriptor::makeMapping !64bit");
4178 #endif /* !__LP64__ */
4179
4180 IOMemoryDescriptor * mapDesc = 0;
4181 IOMemoryMap * result = 0;
4182 OSIterator * iter;
4183
4184 IOMemoryMap * mapping = (IOMemoryMap *) __address;
4185 mach_vm_size_t offset = mapping->fOffset + __offset;
4186 mach_vm_size_t length = mapping->fLength;
4187
4188 mapping->fOffset = offset;
4189
4190 LOCK;
4191
4192 do
4193 {
4194 if (kIOMapStatic & options)
4195 {
4196 result = mapping;
4197 addMapping(mapping);
4198 mapping->setMemoryDescriptor(this, 0);
4199 continue;
4200 }
4201
4202 if (kIOMapUnique & options)
4203 {
4204 addr64_t phys;
4205 IOByteCount physLen;
4206
4207 // if (owner != this) continue;
4208
4209 if (((_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical)
4210 || ((_flags & kIOMemoryTypeMask) == kIOMemoryTypePhysical64))
4211 {
4212 phys = getPhysicalSegment(offset, &physLen, kIOMemoryMapperNone);
4213 if (!phys || (physLen < length))
4214 continue;
4215
4216 mapDesc = IOMemoryDescriptor::withAddressRange(
4217 phys, length, getDirection() | kIOMemoryMapperNone, NULL);
4218 if (!mapDesc)
4219 continue;
4220 offset = 0;
4221 mapping->fOffset = offset;
4222 }
4223 }
4224 else
4225 {
4226 // look for a compatible existing mapping
4227 if( (iter = OSCollectionIterator::withCollection(_mappings)))
4228 {
4229 IOMemoryMap * lookMapping;
4230 while ((lookMapping = (IOMemoryMap *) iter->getNextObject()))
4231 {
4232 if ((result = lookMapping->copyCompatible(mapping)))
4233 {
4234 addMapping(result);
4235 result->setMemoryDescriptor(this, offset);
4236 break;
4237 }
4238 }
4239 iter->release();
4240 }
4241 if (result || (options & kIOMapReference))
4242 {
4243 if (result != mapping)
4244 {
4245 mapping->release();
4246 mapping = NULL;
4247 }
4248 continue;
4249 }
4250 }
4251
4252 if (!mapDesc)
4253 {
4254 mapDesc = this;
4255 mapDesc->retain();
4256 }
4257 IOReturn
4258 kr = mapDesc->doMap( 0, (IOVirtualAddress *) &mapping, options, 0, 0 );
4259 if (kIOReturnSuccess == kr)
4260 {
4261 result = mapping;
4262 mapDesc->addMapping(result);
4263 result->setMemoryDescriptor(mapDesc, offset);
4264 }
4265 else
4266 {
4267 mapping->release();
4268 mapping = NULL;
4269 }
4270 }
4271 while( false );
4272
4273 UNLOCK;
4274
4275 if (mapDesc)
4276 mapDesc->release();
4277
4278 return (result);
4279 }
4280
4281 void IOMemoryDescriptor::addMapping(
4282 IOMemoryMap * mapping )
4283 {
4284 if( mapping)
4285 {
4286 if( 0 == _mappings)
4287 _mappings = OSSet::withCapacity(1);
4288 if( _mappings )
4289 _mappings->setObject( mapping );
4290 }
4291 }
4292
4293 void IOMemoryDescriptor::removeMapping(
4294 IOMemoryMap * mapping )
4295 {
4296 if( _mappings)
4297 _mappings->removeObject( mapping);
4298 }
4299
4300 #ifndef __LP64__
4301 // obsolete initializers
4302 // - initWithOptions is the designated initializer
4303 bool
4304 IOMemoryDescriptor::initWithAddress(void * address,
4305 IOByteCount length,
4306 IODirection direction)
4307 {
4308 return( false );
4309 }
4310
4311 bool
4312 IOMemoryDescriptor::initWithAddress(IOVirtualAddress address,
4313 IOByteCount length,
4314 IODirection direction,
4315 task_t task)
4316 {
4317 return( false );
4318 }
4319
4320 bool
4321 IOMemoryDescriptor::initWithPhysicalAddress(
4322 IOPhysicalAddress address,
4323 IOByteCount length,
4324 IODirection direction )
4325 {
4326 return( false );
4327 }
4328
4329 bool
4330 IOMemoryDescriptor::initWithRanges(
4331 IOVirtualRange * ranges,
4332 UInt32 withCount,
4333 IODirection direction,
4334 task_t task,
4335 bool asReference)
4336 {
4337 return( false );
4338 }
4339
4340 bool
4341 IOMemoryDescriptor::initWithPhysicalRanges( IOPhysicalRange * ranges,
4342 UInt32 withCount,
4343 IODirection direction,
4344 bool asReference)
4345 {
4346 return( false );
4347 }
4348
4349 void * IOMemoryDescriptor::getVirtualSegment(IOByteCount offset,
4350 IOByteCount * lengthOfSegment)
4351 {
4352 return( 0 );
4353 }
4354 #endif /* !__LP64__ */
4355
4356 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
4357
4358 bool IOGeneralMemoryDescriptor::serialize(OSSerialize * s) const
4359 {
4360 OSSymbol const *keys[2];
4361 OSObject *values[2];
4362 OSArray * array;
4363
4364 struct SerData {
4365 user_addr_t address;
4366 user_size_t length;
4367 } *vcopy;
4368 unsigned int index, nRanges;
4369 bool result;
4370
4371 IOOptionBits type = _flags & kIOMemoryTypeMask;
4372
4373 if (s == NULL) return false;
4374
4375 array = OSArray::withCapacity(4);
4376 if (!array) return (false);
4377
4378 nRanges = _rangesCount;
4379 vcopy = (SerData *) IOMalloc(sizeof(SerData) * nRanges);
4380 if (vcopy == 0) return false;
4381
4382 keys[0] = OSSymbol::withCString("address");
4383 keys[1] = OSSymbol::withCString("length");
4384
4385 result = false;
4386 values[0] = values[1] = 0;
4387
4388 // From this point on we can go to bail.
4389
4390 // Copy the volatile data so we don't have to allocate memory
4391 // while the lock is held.
4392 LOCK;
4393 if (nRanges == _rangesCount) {
4394 Ranges vec = _ranges;
4395 for (index = 0; index < nRanges; index++) {
4396 mach_vm_address_t addr; mach_vm_size_t len;
4397 getAddrLenForInd(addr, len, type, vec, index);
4398 vcopy[index].address = addr;
4399 vcopy[index].length = len;
4400 }
4401 } else {
4402 // The descriptor changed out from under us. Give up.
4403 UNLOCK;
4404 result = false;
4405 goto bail;
4406 }
4407 UNLOCK;
4408
4409 for (index = 0; index < nRanges; index++)
4410 {
4411 user_addr_t addr = vcopy[index].address;
4412 IOByteCount len = (IOByteCount) vcopy[index].length;
4413 values[0] = OSNumber::withNumber(addr, sizeof(addr) * 8);
4414 if (values[0] == 0) {
4415 result = false;
4416 goto bail;
4417 }
4418 values[1] = OSNumber::withNumber(len, sizeof(len) * 8);
4419 if (values[1] == 0) {
4420 result = false;
4421 goto bail;
4422 }
4423 OSDictionary *dict = OSDictionary::withObjects((const OSObject **)values, (const OSSymbol **)keys, 2);
4424 if (dict == 0) {
4425 result = false;
4426 goto bail;
4427 }
4428 array->setObject(dict);
4429 dict->release();
4430 values[0]->release();
4431 values[1]->release();
4432 values[0] = values[1] = 0;
4433 }
4434
4435 result = array->serialize(s);
4436
4437 bail:
4438 if (array)
4439 array->release();
4440 if (values[0])
4441 values[0]->release();
4442 if (values[1])
4443 values[1]->release();
4444 if (keys[0])
4445 keys[0]->release();
4446 if (keys[1])
4447 keys[1]->release();
4448 if (vcopy)
4449 IOFree(vcopy, sizeof(SerData) * nRanges);
4450
4451 return result;
4452 }
4453
4454 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
4455
4456 #if DEVELOPMENT || DEBUG
4457
4458 extern "C" void IOMemoryDescriptorTest(int x)
4459 {
4460 IOGeneralMemoryDescriptor * md;
4461
4462 vm_offset_t data[2];
4463 vm_size_t bsize = 16*1024*1024;
4464
4465 vm_size_t srcsize, srcoffset, mapoffset, size;
4466
4467 kern_return_t kr;
4468
4469 kr = vm_allocate(kernel_map, &data[0], bsize, VM_FLAGS_ANYWHERE);
4470 vm_inherit(kernel_map, data[0] + 1*4096, 4096, VM_INHERIT_NONE);
4471 vm_inherit(kernel_map, data[0] + 16*4096, 4096, VM_INHERIT_NONE);
4472
4473 kprintf("data 0x%lx, 0x%lx\n", (long)data[0], (long)data[1]);
4474
4475 uint32_t idx, offidx;
4476 for (idx = 0; idx < (bsize / sizeof(uint32_t)); idx++)
4477 {
4478 ((uint32_t*)data[0])[idx] = idx;
4479 }
4480
4481 for (srcoffset = 0; srcoffset < bsize; srcoffset = ((srcoffset << 1) + 0x40c))
4482 {
4483 for (srcsize = 4; srcsize < (bsize - srcoffset - 1); srcsize = ((srcsize << 1) + 0x3fc))
4484 {
4485 IOAddressRange ranges[3];
4486 uint32_t rangeCount = 1;
4487
4488 bzero(&ranges[0], sizeof(ranges));
4489 ranges[0].address = data[0] + srcoffset;
4490 ranges[0].length = srcsize;
4491
4492 if (srcsize > 5*page_size)
4493 {
4494 ranges[0].length = 7634;
4495 ranges[1].length = 9870;
4496 ranges[2].length = srcsize - ranges[0].length - ranges[1].length;
4497 ranges[1].address = ranges[0].address + ranges[0].length;
4498 ranges[2].address = ranges[1].address + ranges[1].length;
4499 rangeCount = 3;
4500 }
4501 else if ((srcsize > 2*page_size) && !(page_mask & srcoffset))
4502 {
4503 ranges[0].length = 4096;
4504 ranges[1].length = 4096;
4505 ranges[2].length = srcsize - ranges[0].length - ranges[1].length;
4506 ranges[0].address = data[0] + srcoffset + 4096;
4507 ranges[1].address = data[0] + srcoffset;
4508 ranges[2].address = ranges[0].address + ranges[0].length;
4509 rangeCount = 3;
4510 }
4511
4512 md = OSDynamicCast(IOGeneralMemoryDescriptor,
4513 IOMemoryDescriptor::withAddressRanges(&ranges[0], rangeCount, kIODirectionInOut, kernel_task));
4514 assert(md);
4515
4516 kprintf("IOMemoryReferenceCreate [0x%lx @ 0x%lx]\n[0x%llx, 0x%llx],\n[0x%llx, 0x%llx],\n[0x%llx, 0x%llx]\n",
4517 (long) srcsize, (long) srcoffset,
4518 (long long) ranges[0].address - data[0], (long long) ranges[0].length,
4519 (long long) ranges[1].address - data[0], (long long) ranges[1].length,
4520 (long long) ranges[2].address - data[0], (long long) ranges[2].length);
4521
4522 if (kIOReturnSuccess == kr)
4523 {
4524 for (mapoffset = 0; mapoffset < srcsize; mapoffset = ((mapoffset << 1) + 0xf00))
4525 {
4526 for (size = 4; size < (srcsize - mapoffset - 1); size = ((size << 1) + 0x20))
4527 {
4528 IOMemoryMap * map;
4529 mach_vm_address_t addr = 0;
4530 uint32_t data;
4531
4532 kprintf("<mapRef [0x%lx @ 0x%lx]\n", (long) size, (long) mapoffset);
4533
4534 map = md->createMappingInTask(kernel_task, 0, kIOMapAnywhere, mapoffset, size);
4535 if (map) addr = map->getAddress();
4536 else kr = kIOReturnError;
4537
4538 kprintf(">mapRef 0x%x %llx\n", kr, addr);
4539
4540 if (kIOReturnSuccess != kr) break;
4541 kr = md->prepare();
4542 if (kIOReturnSuccess != kr)
4543 {
4544 kprintf("prepare() fail 0x%x\n", kr);
4545 break;
4546 }
4547 for (idx = 0; idx < size; idx += sizeof(uint32_t))
4548 {
4549 offidx = (idx + mapoffset + srcoffset);
4550 if ((srcsize <= 5*page_size) && (srcsize > 2*page_size) && !(page_mask & srcoffset))
4551 {
4552 if (offidx < 8192) offidx ^= 0x1000;
4553 }
4554 offidx /= sizeof(uint32_t);
4555
4556 if (offidx != ((uint32_t*)addr)[idx/sizeof(uint32_t)])
4557 {
4558 kprintf("vm mismatch @ 0x%x, 0x%lx, 0x%lx, \n", idx, (long) srcoffset, (long) mapoffset);
4559 kr = kIOReturnBadMedia;
4560 }
4561 else
4562 {
4563 if (sizeof(data) != md->readBytes(mapoffset + idx, &data, sizeof(data))) data = 0;
4564 if (offidx != data)
4565 {
4566 kprintf("phys mismatch @ 0x%x, 0x%lx, 0x%lx, \n", idx, (long) srcoffset, (long) mapoffset);
4567 kr = kIOReturnBadMedia;
4568 }
4569 }
4570 }
4571 md->complete();
4572 map->release();
4573 kprintf("unmapRef %llx\n", addr);
4574 }
4575 if (kIOReturnSuccess != kr) break;
4576 }
4577 }
4578 if (kIOReturnSuccess != kr) break;
4579 }
4580 if (kIOReturnSuccess != kr) break;
4581 }
4582
4583 if (kIOReturnSuccess != kr) kprintf("FAIL: src 0x%lx @ 0x%lx, map 0x%lx @ 0x%lx\n",
4584 (long) srcsize, (long) srcoffset, (long) size, (long) mapoffset);
4585
4586 vm_deallocate(kernel_map, data[0], bsize);
4587 // vm_deallocate(kernel_map, data[1], size);
4588 }
4589
4590 #endif /* DEVELOPMENT || DEBUG */
4591
4592 /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
4593
4594 OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 0);
4595 #ifdef __LP64__
4596 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 1);
4597 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 2);
4598 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 3);
4599 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 4);
4600 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 5);
4601 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 6);
4602 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 7);
4603 #else /* !__LP64__ */
4604 OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 1);
4605 OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 2);
4606 OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 3);
4607 OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 4);
4608 OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 5);
4609 OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 6);
4610 OSMetaClassDefineReservedUsed(IOMemoryDescriptor, 7);
4611 #endif /* !__LP64__ */
4612 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 8);
4613 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 9);
4614 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 10);
4615 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 11);
4616 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 12);
4617 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 13);
4618 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 14);
4619 OSMetaClassDefineReservedUnused(IOMemoryDescriptor, 15);
4620
4621 /* ex-inline function implementation */
4622 IOPhysicalAddress
4623 IOMemoryDescriptor::getPhysicalAddress()
4624 { return( getPhysicalSegment( 0, 0 )); }
mirror of XNU