X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/21362eb3e66fd2c787aee132bce100a44d71a99c..4452a7af2eac33dbad800bcc91f2399d62c18f53:/iokit/Kernel/IOCopyMapper.cpp diff --git a/iokit/Kernel/IOCopyMapper.cpp b/iokit/Kernel/IOCopyMapper.cpp new file mode 100644 index 000000000..eef092d65 --- /dev/null +++ b/iokit/Kernel/IOCopyMapper.cpp @@ -0,0 +1,426 @@ +/* + * Copyright (c) 2006 Apple Computer, Inc. All rights reserved. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ + */ +// 45678901234567890123456789012345678901234567890123456789012345678901234567890 + +#include "IOCopyMapper.h" +#include + +#if 0 +#define DEBG(fmt, args...) { kprintf(fmt, ## args); } +#else +#define DEBG(fmt, args...) {} +#endif + +extern "C" { +extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va); +extern void ml_get_bouncepool_info( + vm_offset_t *phys_addr, + vm_size_t *size); +extern unsigned int vm_lopage_max_count; +extern unsigned int vm_himemory_mode; +} + +#define super IOMapper + +OSDefineMetaClassAndStructors(IOCopyMapper, IOMapper); + +// Remember no value can be bigger than 31 bits as the sign bit indicates +// that this entry is valid to the hardware and that would be bad if it wasn't +typedef struct FreeDARTEntry { +#if __BIG_ENDIAN__ + unsigned int + /* bool */ fValid : 1, + /* bool */ fInUse : 1, // Allocated but not inserted yet + /* bool */ : 5, // Align size on nibble boundary for debugging + /* uint */ fSize : 5, + /* uint */ : 2, + /* uint */ fNext :18; // offset of FreeDARTEntry's + +#elif __LITTLE_ENDIAN__ + unsigned int + /* uint */ fNext :18, // offset of FreeDARTEntry's + /* uint */ : 2, + /* uint */ fSize : 5, + /* bool */ : 5, // Align size on nibble boundary for debugging + /* bool */ fInUse : 1, // Allocated but not inserted yet + /* bool */ fValid : 1; +#endif +#if __BIG_ENDIAN__ + unsigned int + /* uint */ :14, + /* uint */ fPrev :18; // offset of FreeDARTEntry's + +#elif __LITTLE_ENDIAN__ + unsigned int + /* uint */ fPrev :18, // offset of FreeDARTEntry's + /* uint */ :14; +#endif +} FreeDARTEntry; + +typedef struct ActiveDARTEntry { +#if __BIG_ENDIAN__ + unsigned int + /* bool */ fValid : 1, // Must be set to one if valid + /* uint */ fPPNum :31; // ppnum_t page of translation +#define ACTIVEDARTENTRY(page) { true, page } + +#elif __LITTLE_ENDIAN__ + unsigned int + /* uint */ fPPNum :31, // ppnum_t page of translation + /* bool */ fValid : 1; // Must be set to one if valid +#define ACTIVEDARTENTRY(page) { page, true } + +#endif +}; + +#define kActivePerFree (sizeof(freeDART[0]) / sizeof(ActiveDARTEntry)) + +static SYSCTL_UINT(_kern, OID_AUTO, copyregionmax, + CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN, + NULL, 0, ""); + +static SYSCTL_UINT(_kern, OID_AUTO, lowpagemax, + CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN, + &vm_lopage_max_count, 0, ""); + +static SYSCTL_UINT(_kern, OID_AUTO, himemorymode, + CTLFLAG_RD | CTLFLAG_NOAUTO | CTLFLAG_KERN, + &vm_himemory_mode, 0, ""); + +bool IOCopyMapper::initHardware(IOService * provider) +{ + UInt32 dartSizePages = 0; + + vm_offset_t phys_addr; + vm_size_t size; + ml_get_bouncepool_info(&phys_addr, &size); + + if (!size) + return (false); + + fBufferPage = atop_32(phys_addr); + dartSizePages = (atop_32(size) + kTransPerPage - 1) / kTransPerPage; + + fTableLock = IOLockAlloc(); + + if (!fTableLock) + return false; + + if (!allocTable(dartSizePages * kMapperPage)) + return false; + + UInt32 canMapPages = dartSizePages * kTransPerPage; + fMapperRegionSize = canMapPages; + for (fNumZones = 0; canMapPages; fNumZones++) + canMapPages >>= 1; + fNumZones -= 3; // correct for overshoot and minumum 16K pages allocation + + invalidateDART(0, fMapperRegionSize); + + breakUp(0, fNumZones, 0); + ((FreeDARTEntry *) fTable)->fInUse = true; + + fMapperRegionUsed = kMinZoneSize; + fMapperRegionMaxUsed = fMapperRegionUsed; + + sysctl__kern_copyregionmax.oid_arg1 = &fMapperRegionMaxUsed; + + sysctl_register_oid(&sysctl__kern_copyregionmax); + sysctl_register_oid(&sysctl__kern_lowpagemax); + sysctl_register_oid(&sysctl__kern_himemorymode); + + fDummyPage = IOMallocAligned(0x1000, 0x1000); + fDummyPageNumber = + pmap_find_phys(kernel_pmap, (addr64_t) (uintptr_t) fDummyPage); + + return true; +} + +void IOCopyMapper::free() +{ + if (fDummyPage) { + IOFreeAligned(fDummyPage, 0x1000); + fDummyPage = 0; + fDummyPageNumber = 0; + } + + if (fTableLock) { + IOLockFree(fTableLock); + fTableLock = 0; + } + + super::free(); +} + +// Must be called while locked +void IOCopyMapper::breakUp(unsigned startIndex, unsigned endIndex, unsigned freeInd) +{ + unsigned int zoneSize; + FreeDARTEntry *freeDART = (FreeDARTEntry *) fTable; + + do { + // Need to break up bigger blocks of memory till we get one in our + // desired zone. + endIndex--; + zoneSize = (kMinZoneSize/2 << endIndex); + ppnum_t tail = freeInd + zoneSize; + + DEBG("breakup z %d start %x tail %x\n", endIndex, freeInd, tail); + + // By definition free lists must be empty + fFreeLists[endIndex] = tail; + freeDART[tail].fSize = endIndex; + freeDART[tail].fNext = freeDART[tail].fPrev = 0; + } while (endIndex != startIndex); + freeDART[freeInd].fSize = endIndex; +} + +// Zero is never a valid page to return +ppnum_t IOCopyMapper::iovmAlloc(IOItemCount pages) +{ + unsigned int zone, zoneSize, z, cnt; + ppnum_t next, ret = 0; + FreeDARTEntry *freeDART = (FreeDARTEntry *) fTable; + + // Can't alloc anything of less than minumum + if (pages < kMinZoneSize) + pages = kMinZoneSize; + + // Can't alloc anything bigger than 1/2 table + if (pages >= fMapperRegionSize/2) + { + panic("iovmAlloc 0x%x", pages); + return 0; + } + + // Find the appropriate zone for this allocation + for (zone = 0, zoneSize = kMinZoneSize; pages > zoneSize; zone++) + zoneSize <<= 1; + + { + IOLockLock(fTableLock); + + for (;;) { + for (z = zone; z < fNumZones; z++) { + if ( (ret = fFreeLists[z]) ) + break; + } + if (ret) + break; + + fFreeSleepers++; + IOLockSleep(fTableLock, fFreeLists, THREAD_UNINT); + fFreeSleepers--; + } + + // If we didn't find a entry in our size then break up the free block + // that we did find. + if (zone != z) + { + DEBG("breakup %d, %d, 0x%x\n", zone, z, ret); + breakUp(zone, z, ret); + } + + freeDART[ret].fInUse = true; // Mark entry as In Use + next = freeDART[ret].fNext; + DEBG("va: 0x%x, %d, ret %x next %x\n", (ret * kActivePerFree) + fBufferPage, pages, ret, next); + + fFreeLists[z] = next; + if (next) + freeDART[next].fPrev = 0; + + // ret is free list offset not page offset; + ret *= kActivePerFree; + + ActiveDARTEntry pageEntry = ACTIVEDARTENTRY(fDummyPageNumber); + for (cnt = 0; cnt < pages; cnt++) { + ActiveDARTEntry *activeDART = &fMappings[ret + cnt]; + *activeDART = pageEntry; + } + + fMapperRegionUsed += pages; + if (fMapperRegionUsed > fMapperRegionMaxUsed) + fMapperRegionMaxUsed = fMapperRegionUsed; + + IOLockUnlock(fTableLock); + } + + if (ret) + ret += fBufferPage; + + return ret; +} + + +void IOCopyMapper::invalidateDART(ppnum_t pnum, IOItemCount size) +{ + bzero((void *) &fMappings[pnum], size * sizeof(fMappings[0])); +} + +void IOCopyMapper::iovmFree(ppnum_t addr, IOItemCount pages) +{ + unsigned int zone, zoneSize, z; + FreeDARTEntry *freeDART = (FreeDARTEntry *) fTable; + + if (addr < fBufferPage) + IOPanic("addr < fBufferPage"); + addr -= fBufferPage; + + // Can't free anything of less than minumum + if (pages < kMinZoneSize) + pages = kMinZoneSize; + + // Can't free anything bigger than 1/2 table + if (pages >= fMapperRegionSize/2) + return; + + // Find the appropriate zone for this allocation + for (zone = 0, zoneSize = kMinZoneSize; pages > zoneSize; zone++) + zoneSize <<= 1; + + // Grab lock that protects the dart + IOLockLock(fTableLock); + + invalidateDART(addr, pages); + + addr /= kActivePerFree; + + // We are freeing a block, check to see if pairs are available for + // coalescing. We will walk up the entire chain if we can. + for (z = zone; z < fNumZones; z++) { + ppnum_t pair = addr ^ (kMinZoneSize/2 << z); // Find pair address + if (freeDART[pair].fValid || freeDART[pair].fInUse || (freeDART[pair].fSize != z)) + break; + + // The paired alloc entry is free if we are here + ppnum_t next = freeDART[pair].fNext; + ppnum_t prev = freeDART[pair].fPrev; + + // Remove the pair from its freeList + if (prev) + freeDART[prev].fNext = next; + else + fFreeLists[z] = next; + + if (next) + freeDART[next].fPrev = prev; + + // Sort the addr and the pair + if (addr > pair) + addr = pair; + } + + DEBG("vf: 0x%x, %d, z %d, head %x, new %x\n", addr * kActivePerFree + fBufferPage, pages, z, fFreeLists[z], addr); + + // Add the allocation entry into it's free list and re-init it + freeDART[addr].fSize = z; + freeDART[addr].fNext = fFreeLists[z]; + if (fFreeLists[z]) + freeDART[fFreeLists[z]].fPrev = addr; + freeDART[addr].fPrev = 0; + fFreeLists[z] = addr; + + fMapperRegionUsed -= pages; + + if (fFreeSleepers) + IOLockWakeup(fTableLock, fFreeLists, /* oneThread */ false); + + IOLockUnlock(fTableLock); +} + +addr64_t IOCopyMapper::mapAddr(IOPhysicalAddress addr) +{ + if (addr < ptoa_32(fBufferPage)) + { + return (addr64_t) addr; // Not mapped by us anyway + } + + addr -= ptoa_32(fBufferPage); + if (addr >= ptoa_32(fMapperRegionSize)) + { + return (addr64_t) addr; // Not mapped by us anyway + } + else + { + ActiveDARTEntry *activeDART = (ActiveDARTEntry *) fTable; + UInt offset = addr & PAGE_MASK; + + ActiveDARTEntry mappedPage = activeDART[atop_32(addr)]; + if (mappedPage.fValid) + { + return (ptoa_64(mappedPage.fPPNum) | offset); + } + + panic("%s::mapAddr(0x%08lx) not mapped for I/O\n", getName(), addr); + return 0; + } +} + +void IOCopyMapper::iovmInsert(ppnum_t addr, IOItemCount offset, ppnum_t page) +{ + addr -= fBufferPage; + addr += offset; // Add the offset page to the base address + + ActiveDARTEntry *activeDART = &fMappings[addr]; + ActiveDARTEntry entry = ACTIVEDARTENTRY(page); + *activeDART = entry; +} + +void IOCopyMapper::iovmInsert(ppnum_t addr, IOItemCount offset, + ppnum_t *pageList, IOItemCount pageCount) +{ + addr -= fBufferPage; + addr += offset; // Add the offset page to the base address + + IOItemCount i; + ActiveDARTEntry *activeDART = &fMappings[addr]; + + for (i = 0; i < pageCount; i++) + { + ActiveDARTEntry entry = ACTIVEDARTENTRY(pageList[i]); + activeDART[i] = entry; + } +} + +void IOCopyMapper::iovmInsert(ppnum_t addr, IOItemCount offset, + upl_page_info_t *pageList, IOItemCount pageCount) +{ + addr -= fBufferPage; + addr += offset; // Add the offset page to the base address + + IOItemCount i; + ActiveDARTEntry *activeDART = &fMappings[addr]; + + for (i = 0; i < pageCount; i++) + { + ActiveDARTEntry entry = ACTIVEDARTENTRY(pageList[i].phys_addr); + activeDART[i] = entry; + } +} + +