X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/0b4c1975fb5e4eccf1012a35081f7e7799b81046..04b8595b18b1b41ac7a206e4b3d51a635f8413d7:/iokit/Kernel/IOLib.cpp diff --git a/iokit/Kernel/IOLib.cpp b/iokit/Kernel/IOLib.cpp index a5415e71c..3714d1d4f 100644 --- a/iokit/Kernel/IOLib.cpp +++ b/iokit/Kernel/IOLib.cpp @@ -55,6 +55,24 @@ #include #endif +#include "libkern/OSAtomic.h" +#include +#include +#include + +#if IOKITSTATS + +#define IOStatisticsAlloc(type, size) \ +do { \ + IOStatistics::countAlloc(type, size); \ +} while (0) + +#else + +#define IOStatisticsAlloc(type, size) + +#endif /* IOKITSTATS */ + extern "C" { @@ -63,7 +81,7 @@ mach_timespec_t IOZeroTvalspec = { 0, 0 }; extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va); -int +extern int __doprnt( const char *fmt, va_list argp, @@ -71,7 +89,10 @@ __doprnt( void *arg, int radix); -extern void conslog_putc(char); +extern void cons_putc_locked(char); +extern void bsd_log_lock(void); +extern void bsd_log_unlock(void); +extern void logwakeup(); /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ @@ -89,6 +110,7 @@ void *_giDebugLogDataInternal = NULL; void *_giDebugReserved1 = NULL; void *_giDebugReserved2 = NULL; +iopa_t gIOBMDPageAllocator; /* * Static variables for this module. @@ -97,9 +119,15 @@ void *_giDebugReserved2 = NULL; static queue_head_t gIOMallocContiguousEntries; static lck_mtx_t * gIOMallocContiguousEntriesLock; -enum { kIOMaxPageableMaps = 16 }; -enum { kIOPageableMapSize = 96 * 1024 * 1024 }; +#if __x86_64__ +enum { kIOMaxPageableMaps = 8 }; +enum { kIOPageableMapSize = 512 * 1024 * 1024 }; +enum { kIOPageableMaxMapSize = 512 * 1024 * 1024 }; +#else +enum { kIOMaxPageableMaps = 16 }; +enum { kIOPageableMapSize = 96 * 1024 * 1024 }; enum { kIOPageableMaxMapSize = 96 * 1024 * 1024 }; +#endif typedef struct { vm_map_t map; @@ -114,6 +142,10 @@ static struct { lck_mtx_t * lock; } gIOKitPageableSpace; +static iopa_t gIOPageablePageAllocator; + +uint32_t gIOPageAllocChunkBytes; + /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ void IOLibInit(void) @@ -145,6 +177,11 @@ void IOLibInit(void) gIOMallocContiguousEntriesLock = lck_mtx_alloc_init(IOLockGroup, LCK_ATTR_NULL); queue_init( &gIOMallocContiguousEntries ); + gIOPageAllocChunkBytes = PAGE_SIZE/64; + assert(sizeof(iopa_page_t) <= gIOPageAllocChunkBytes); + iopa_init(&gIOBMDPageAllocator); + iopa_init(&gIOPageablePageAllocator); + libInitialized = true; } @@ -178,11 +215,13 @@ void * IOMalloc(vm_size_t size) void * address; address = (void *)kalloc(size); + if ( address ) { #if IOALLOCDEBUG - if (address) { debug_iomalloc_size += size; - } #endif + IOStatisticsAlloc(kIOStatisticsMalloc, size); + } + return address; } @@ -193,6 +232,7 @@ void IOFree(void * address, vm_size_t size) #if IOALLOCDEBUG debug_iomalloc_size -= size; #endif + IOStatisticsAlloc(kIOStatisticsFree, size); } } @@ -214,7 +254,10 @@ void * IOMallocAligned(vm_size_t size, vm_size_t alignment) alignMask = alignment - 1; adjustedSize = size + sizeof(vm_size_t) + sizeof(vm_address_t); - if (adjustedSize >= page_size) { + if (size > adjustedSize) { + address = 0; /* overflow detected */ + } + else if (adjustedSize >= page_size) { kr = kernel_memory_allocate(kernel_map, &address, size, alignMask, 0); @@ -250,11 +293,12 @@ void * IOMallocAligned(vm_size_t size, vm_size_t alignment) assert(0 == (address & alignMask)); -#if IOALLOCDEBUG if( address) { +#if IOALLOCDEBUG debug_iomalloc_size += size; - } #endif + IOStatisticsAlloc(kIOStatisticsMallocAligned, size); + } return (void *) address; } @@ -289,6 +333,8 @@ void IOFreeAligned(void * address, vm_size_t size) #if IOALLOCDEBUG debug_iomalloc_size -= size; #endif + + IOStatisticsAlloc(kIOStatisticsFreeAligned, size); } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ @@ -318,14 +364,16 @@ IOKernelFreePhysical(mach_vm_address_t address, mach_vm_size_t size) kfree((void *)allocationAddress, adjustedSize); } + IOStatisticsAlloc(kIOStatisticsFreeContiguous, size); #if IOALLOCDEBUG debug_iomalloc_size -= size; #endif } + mach_vm_address_t IOKernelAllocateWithPhysicalRestrict(mach_vm_size_t size, mach_vm_address_t maxPhys, - mach_vm_size_t alignment, bool contiguous) + mach_vm_size_t alignment, bool contiguous) { kern_return_t kr; mach_vm_address_t address; @@ -353,12 +401,18 @@ IOKernelAllocateWithPhysicalRestrict(mach_vm_size_t size, mach_vm_address_t maxP contiguous = (contiguous && (adjustedSize > page_size)) || (alignment > page_size); - if ((!contiguous) && (maxPhys <= 0xFFFFFFFF)) - { - maxPhys = 0; - options |= KMA_LOMEM; - } - + if (!contiguous) + { + if (maxPhys <= 0xFFFFFFFF) + { + maxPhys = 0; + options |= KMA_LOMEM; + } + else if (gIOLastPage && (atop_64(maxPhys) > gIOLastPage)) + { + maxPhys = 0; + } + } if (contiguous || maxPhys) { kr = kmem_alloc_contig(kernel_map, &virt, size, @@ -396,15 +450,17 @@ IOKernelAllocateWithPhysicalRestrict(mach_vm_size_t size, mach_vm_address_t maxP address = 0; } -#if IOALLOCDEBUG if (address) { + IOStatisticsAlloc(kIOStatisticsMallocContiguous, size); +#if IOALLOCDEBUG debug_iomalloc_size += size; - } #endif + } return (address); } + /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ struct _IOMallocContiguousEntry @@ -588,7 +644,7 @@ static kern_return_t IOMallocPageableCallback(vm_map_t map, void * _ref) return( kr ); } -void * IOMallocPageable(vm_size_t size, vm_size_t alignment) +static void * IOMallocPageablePages(vm_size_t size, vm_size_t alignment) { kern_return_t kr = kIOReturnNotReady; struct IOMallocPageableRef ref; @@ -603,11 +659,6 @@ void * IOMallocPageable(vm_size_t size, vm_size_t alignment) if( kIOReturnSuccess != kr) ref.address = 0; -#if IOALLOCDEBUG - if( ref.address) - debug_iomallocpageable_size += round_page(size); -#endif - return( (void *) ref.address ); } @@ -629,17 +680,198 @@ vm_map_t IOPageableMapForAddress( uintptr_t address ) return( map ); } -void IOFreePageable(void * address, vm_size_t size) +static void IOFreePageablePages(void * address, vm_size_t size) { vm_map_t map; map = IOPageableMapForAddress( (vm_address_t) address); if( map) kmem_free( map, (vm_offset_t) address, size); +} +static uintptr_t IOMallocOnePageablePage(iopa_t * a) +{ + return ((uintptr_t) IOMallocPageablePages(page_size, page_size)); +} + +void * IOMallocPageable(vm_size_t size, vm_size_t alignment) +{ + void * addr; + + if (size >= (page_size - 4*gIOPageAllocChunkBytes)) addr = IOMallocPageablePages(size, alignment); + else addr = ((void * ) iopa_alloc(&gIOPageablePageAllocator, &IOMallocOnePageablePage, size, alignment)); + + if (addr) { #if IOALLOCDEBUG - debug_iomallocpageable_size -= round_page(size); + debug_iomallocpageable_size += size; #endif + IOStatisticsAlloc(kIOStatisticsMallocPageable, size); + } + + return (addr); +} + +void IOFreePageable(void * address, vm_size_t size) +{ +#if IOALLOCDEBUG + debug_iomallocpageable_size -= size; +#endif + IOStatisticsAlloc(kIOStatisticsFreePageable, size); + + if (size < (page_size - 4*gIOPageAllocChunkBytes)) + { + address = (void *) iopa_free(&gIOPageablePageAllocator, (uintptr_t) address, size); + size = page_size; + } + if (address) IOFreePageablePages(address, size); +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +extern "C" void +iopa_init(iopa_t * a) +{ + bzero(a, sizeof(*a)); + a->lock = IOLockAlloc(); + queue_init(&a->list); +} + +static uintptr_t +iopa_allocinpage(iopa_page_t * pa, uint32_t count, uint64_t align) +{ + uint32_t n, s; + uint64_t avail = pa->avail; + + assert(avail); + + // find strings of count 1 bits in avail + for (n = count; n > 1; n -= s) + { + s = n >> 1; + avail = avail & (avail << s); + } + // and aligned + avail &= align; + + if (avail) + { + n = __builtin_clzll(avail); + pa->avail &= ~((-1ULL << (64 - count)) >> n); + if (!pa->avail && pa->link.next) + { + remque(&pa->link); + pa->link.next = 0; + } + return (n * gIOPageAllocChunkBytes + trunc_page((uintptr_t) pa)); + } + + return (0); +} + +static uint32_t +log2up(uint32_t size) +{ + if (size <= 1) size = 0; + else size = 32 - __builtin_clz(size - 1); + return (size); +} + +uintptr_t +iopa_alloc(iopa_t * a, iopa_proc_t alloc, vm_size_t bytes, uint32_t balign) +{ + static const uint64_t align_masks[] = { + 0xFFFFFFFFFFFFFFFF, + 0xAAAAAAAAAAAAAAAA, + 0x8888888888888888, + 0x8080808080808080, + 0x8000800080008000, + 0x8000000080000000, + 0x8000000000000000, + }; + iopa_page_t * pa; + uintptr_t addr = 0; + uint32_t count; + uint64_t align; + + if (!bytes) bytes = 1; + count = (bytes + gIOPageAllocChunkBytes - 1) / gIOPageAllocChunkBytes; + align = align_masks[log2up((balign + gIOPageAllocChunkBytes - 1) / gIOPageAllocChunkBytes)]; + + IOLockLock(a->lock); + pa = (typeof(pa)) queue_first(&a->list); + while (!queue_end(&a->list, &pa->link)) + { + addr = iopa_allocinpage(pa, count, align); + if (addr) + { + a->bytecount += bytes; + break; + } + pa = (typeof(pa)) queue_next(&pa->link); + } + IOLockUnlock(a->lock); + + if (!addr) + { + addr = alloc(a); + if (addr) + { + pa = (typeof(pa)) (addr + page_size - gIOPageAllocChunkBytes); + pa->signature = kIOPageAllocSignature; + pa->avail = -2ULL; + + addr = iopa_allocinpage(pa, count, align); + IOLockLock(a->lock); + if (pa->avail) enqueue_head(&a->list, &pa->link); + a->pagecount++; + if (addr) a->bytecount += bytes; + IOLockUnlock(a->lock); + } + } + + assert((addr & ((1 << log2up(balign)) - 1)) == 0); + return (addr); +} + +uintptr_t +iopa_free(iopa_t * a, uintptr_t addr, vm_size_t bytes) +{ + iopa_page_t * pa; + uint32_t count; + uintptr_t chunk; + + if (!bytes) bytes = 1; + + chunk = (addr & page_mask); + assert(0 == (chunk & (gIOPageAllocChunkBytes - 1))); + + pa = (typeof(pa)) (addr | (page_size - gIOPageAllocChunkBytes)); + assert(kIOPageAllocSignature == pa->signature); + + count = (bytes + gIOPageAllocChunkBytes - 1) / gIOPageAllocChunkBytes; + chunk /= gIOPageAllocChunkBytes; + + IOLockLock(a->lock); + if (!pa->avail) + { + assert(!pa->link.next); + enqueue_tail(&a->list, &pa->link); + } + pa->avail |= ((-1ULL << (64 - count)) >> chunk); + if (pa->avail != -2ULL) pa = 0; + else + { + remque(&pa->link); + pa->link.next = 0; + pa->signature = 0; + a->pagecount--; + // page to free + pa = (typeof(pa)) trunc_page(pa); + } + a->bytecount -= bytes; + IOLockUnlock(a->lock); + + return ((uintptr_t) pa); } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ @@ -727,23 +959,37 @@ void IOPause(unsigned nanoseconds) /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -static void _iolog_putc(int ch, void *arg __unused) +static void _iolog_consputc(int ch, void *arg __unused) { - conslog_putc(ch); + cons_putc_locked(ch); +} + +static void _iolog_logputc(int ch, void *arg __unused) +{ + log_putc_locked(ch); } void IOLog(const char *format, ...) { - va_list ap; + va_list ap; - va_start(ap, format); - __doprnt(format, ap, _iolog_putc, NULL, 16); - va_end(ap); + va_start(ap, format); + IOLogv(format, ap); + va_end(ap); } void IOLogv(const char *format, va_list ap) { - __doprnt(format, ap, _iolog_putc, NULL, 16); + va_list ap2; + + va_copy(ap2, ap); + + bsd_log_lock(); + __doprnt(format, ap, _iolog_logputc, NULL, 16); + bsd_log_unlock(); + logwakeup(); + + __doprnt(format, ap2, _iolog_consputc, NULL, 16); } #if !__LP64__