X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/39236c6e673c41db228275375ab7fdb0f837b292..0a7de7458d150b5d4dffc935ba399be265ef0a1a:/iokit/Kernel/IOLib.cpp diff --git a/iokit/Kernel/IOLib.cpp b/iokit/Kernel/IOLib.cpp index 45504157a..0dedff70f 100644 --- a/iokit/Kernel/IOLib.cpp +++ b/iokit/Kernel/IOLib.cpp @@ -2,7 +2,7 @@ * Copyright (c) 1998-2006 Apple Computer, Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ - * + * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -11,10 +11,10 @@ * unlawful or unlicensed copies of an Apple operating system, or to * circumvent, violate, or enable the circumvention or violation of, any * terms of an Apple operating system software license agreement. - * + * * Please obtain a copy of the License at * http://www.opensource.apple.com/apsl/ and read it before using this file. - * + * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, @@ -22,7 +22,7 @@ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. * Please see the License for the specific language governing rights and * limitations under the License. - * + * * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* @@ -42,11 +42,11 @@ #include #include -#include -#include +#include +#include #include #include -#include +#include #include "IOKitKernelInternal.h" @@ -58,7 +58,9 @@ #include "libkern/OSAtomic.h" #include #include +#include #include +#include #if IOKITSTATS @@ -73,31 +75,33 @@ do { \ #endif /* IOKITSTATS */ -extern "C" -{ +#define TRACK_ALLOC (IOTRACKING && (kIOTracking & gIOKitDebug)) + +extern "C" +{ mach_timespec_t IOZeroTvalspec = { 0, 0 }; extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va); extern int __doprnt( - const char *fmt, - va_list argp, - void (*putc)(int, void *), + const char *fmt, + va_list argp, + void (*putc)(int, void *), void *arg, - int radix); + int radix, + int is_log); extern void cons_putc_locked(char); extern void bsd_log_lock(void); extern void bsd_log_unlock(void); -extern void logwakeup(); /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -lck_grp_t *IOLockGroup; +lck_grp_t *IOLockGroup; /* * Global variables for use by iLogger @@ -105,10 +109,10 @@ lck_grp_t *IOLockGroup; * Binary compatibility is not guaranteed for kexts that reference these symbols. */ -void *_giDebugLogInternal = NULL; -void *_giDebugLogDataInternal = NULL; -void *_giDebugReserved1 = NULL; -void *_giDebugReserved2 = NULL; +void *_giDebugLogInternal = NULL; +void *_giDebugLogDataInternal = NULL; +void *_giDebugReserved1 = NULL; +void *_giDebugReserved2 = NULL; iopa_t gIOBMDPageAllocator; @@ -119,212 +123,385 @@ iopa_t gIOBMDPageAllocator; static queue_head_t gIOMallocContiguousEntries; static lck_mtx_t * gIOMallocContiguousEntriesLock; -enum { kIOMaxPageableMaps = 16 }; -enum { kIOPageableMapSize = 96 * 1024 * 1024 }; +#if __x86_64__ +enum { kIOMaxPageableMaps = 8 }; +enum { kIOPageableMapSize = 512 * 1024 * 1024 }; +enum { kIOPageableMaxMapSize = 512 * 1024 * 1024 }; +#else +enum { kIOMaxPageableMaps = 16 }; +enum { kIOPageableMapSize = 96 * 1024 * 1024 }; enum { kIOPageableMaxMapSize = 96 * 1024 * 1024 }; +#endif typedef struct { - vm_map_t map; - vm_offset_t address; - vm_offset_t end; + vm_map_t map; + vm_offset_t address; + vm_offset_t end; } IOMapData; static struct { - UInt32 count; - UInt32 hint; - IOMapData maps[ kIOMaxPageableMaps ]; - lck_mtx_t * lock; + UInt32 count; + UInt32 hint; + IOMapData maps[kIOMaxPageableMaps]; + lck_mtx_t * lock; } gIOKitPageableSpace; static iopa_t gIOPageablePageAllocator; +uint32_t gIOPageAllocChunkBytes; + +#if IOTRACKING +IOTrackingQueue * gIOMallocTracking; +IOTrackingQueue * gIOWireTracking; +IOTrackingQueue * gIOMapTracking; +#endif /* IOTRACKING */ + /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -void IOLibInit(void) +void +IOLibInit(void) { - kern_return_t ret; + kern_return_t ret; + + static bool libInitialized; - static bool libInitialized; + if (libInitialized) { + return; + } - if(libInitialized) - return; + IOLockGroup = lck_grp_alloc_init("IOKit", LCK_GRP_ATTR_NULL); + +#if IOTRACKING + IOTrackingInit(); + gIOMallocTracking = IOTrackingQueueAlloc(kIOMallocTrackingName, 0, 0, 0, + kIOTrackingQueueTypeAlloc, + 37); + gIOWireTracking = IOTrackingQueueAlloc(kIOWireTrackingName, 0, 0, page_size, 0, 0); + + size_t mapCaptureSize = (kIOTracking & gIOKitDebug) ? page_size : (1024 * 1024); + gIOMapTracking = IOTrackingQueueAlloc(kIOMapTrackingName, 0, 0, mapCaptureSize, + kIOTrackingQueueTypeDefaultOn + | kIOTrackingQueueTypeMap + | kIOTrackingQueueTypeUser, + 0); +#endif - gIOKitPageableSpace.maps[0].address = 0; - ret = kmem_suballoc(kernel_map, - &gIOKitPageableSpace.maps[0].address, - kIOPageableMapSize, - TRUE, - VM_FLAGS_ANYWHERE, - &gIOKitPageableSpace.maps[0].map); - if (ret != KERN_SUCCESS) - panic("failed to allocate iokit pageable map\n"); + gIOKitPageableSpace.maps[0].address = 0; + ret = kmem_suballoc(kernel_map, + &gIOKitPageableSpace.maps[0].address, + kIOPageableMapSize, + TRUE, + VM_FLAGS_ANYWHERE, + VM_MAP_KERNEL_FLAGS_NONE, + VM_KERN_MEMORY_IOKIT, + &gIOKitPageableSpace.maps[0].map); + if (ret != KERN_SUCCESS) { + panic("failed to allocate iokit pageable map\n"); + } - IOLockGroup = lck_grp_alloc_init("IOKit", LCK_GRP_ATTR_NULL); + gIOKitPageableSpace.lock = lck_mtx_alloc_init(IOLockGroup, LCK_ATTR_NULL); + gIOKitPageableSpace.maps[0].end = gIOKitPageableSpace.maps[0].address + kIOPageableMapSize; + gIOKitPageableSpace.hint = 0; + gIOKitPageableSpace.count = 1; - gIOKitPageableSpace.lock = lck_mtx_alloc_init(IOLockGroup, LCK_ATTR_NULL); - gIOKitPageableSpace.maps[0].end = gIOKitPageableSpace.maps[0].address + kIOPageableMapSize; - gIOKitPageableSpace.hint = 0; - gIOKitPageableSpace.count = 1; + gIOMallocContiguousEntriesLock = lck_mtx_alloc_init(IOLockGroup, LCK_ATTR_NULL); + queue_init( &gIOMallocContiguousEntries ); - gIOMallocContiguousEntriesLock = lck_mtx_alloc_init(IOLockGroup, LCK_ATTR_NULL); - queue_init( &gIOMallocContiguousEntries ); + gIOPageAllocChunkBytes = PAGE_SIZE / 64; + assert(sizeof(iopa_page_t) <= gIOPageAllocChunkBytes); + iopa_init(&gIOBMDPageAllocator); + iopa_init(&gIOPageablePageAllocator); - iopa_init(&gIOBMDPageAllocator); - iopa_init(&gIOPageablePageAllocator); - libInitialized = true; + libInitialized = true; } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -IOThread IOCreateThread(IOThreadFunc fcn, void *arg) +static uint32_t +log2up(uint32_t size) { - kern_return_t result; - thread_t thread; + if (size <= 1) { + size = 0; + } else { + size = 32 - __builtin_clz(size - 1); + } + return size; +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +IOThread +IOCreateThread(IOThreadFunc fcn, void *arg) +{ + kern_return_t result; + thread_t thread; result = kernel_thread_start((thread_continue_t)fcn, arg, &thread); - if (result != KERN_SUCCESS) - return (NULL); + if (result != KERN_SUCCESS) { + return NULL; + } thread_deallocate(thread); - return (thread); + return thread; } -void IOExitThread(void) +void +IOExitThread(void) { - (void) thread_terminate(current_thread()); + (void) thread_terminate(current_thread()); } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +#if IOTRACKING +struct IOLibMallocHeader { + IOTrackingAddress tracking; +}; +#endif -void * IOMalloc(vm_size_t size) +#if IOTRACKING +#define sizeofIOLibMallocHeader (sizeof(IOLibMallocHeader) - (TRACK_ALLOC ? 0 : sizeof(IOTrackingAddress))) +#else +#define sizeofIOLibMallocHeader (0) +#endif + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +void * +IOMalloc(vm_size_t size) { - void * address; + void * address; + vm_size_t allocSize; + + allocSize = size + sizeofIOLibMallocHeader; +#if IOTRACKING + if (sizeofIOLibMallocHeader && (allocSize <= size)) { + return NULL; // overflow + } +#endif + address = kalloc_tag_bt(allocSize, VM_KERN_MEMORY_IOKIT); + + if (address) { +#if IOTRACKING + if (TRACK_ALLOC) { + IOLibMallocHeader * hdr; + hdr = (typeof(hdr))address; + bzero(&hdr->tracking, sizeof(hdr->tracking)); + hdr->tracking.address = ~(((uintptr_t) address) + sizeofIOLibMallocHeader); + hdr->tracking.size = size; + IOTrackingAdd(gIOMallocTracking, &hdr->tracking.tracking, size, true, VM_KERN_MEMORY_NONE); + } +#endif + address = (typeof(address))(((uintptr_t) address) + sizeofIOLibMallocHeader); - address = (void *)kalloc(size); - if ( address ) { #if IOALLOCDEBUG - debug_iomalloc_size += size; + OSAddAtomic(size, &debug_iomalloc_size); #endif IOStatisticsAlloc(kIOStatisticsMalloc, size); - } + } - return address; + return address; } -void IOFree(void * address, vm_size_t size) +void +IOFree(void * inAddress, vm_size_t size) { - if (address) { - kfree(address, size); + void * address; + + if ((address = inAddress)) { + address = (typeof(address))(((uintptr_t) address) - sizeofIOLibMallocHeader); + +#if IOTRACKING + if (TRACK_ALLOC) { + IOLibMallocHeader * hdr; + struct ptr_reference { void * ptr; }; + volatile struct ptr_reference ptr; + + // we're about to block in IOTrackingRemove(), make sure the original pointer + // exists in memory or a register for leak scanning to find + ptr.ptr = inAddress; + + hdr = (typeof(hdr))address; + if (size != hdr->tracking.size) { + OSReportWithBacktrace("bad IOFree size 0x%lx should be 0x%lx", size, hdr->tracking.size); + size = hdr->tracking.size; + } + IOTrackingRemove(gIOMallocTracking, &hdr->tracking.tracking, size); + ptr.ptr = NULL; + } +#endif + + kfree(address, size + sizeofIOLibMallocHeader); #if IOALLOCDEBUG - debug_iomalloc_size -= size; + OSAddAtomic(-size, &debug_iomalloc_size); #endif IOStatisticsAlloc(kIOStatisticsFree, size); - } + } } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -void * IOMallocAligned(vm_size_t size, vm_size_t alignment) +vm_tag_t +IOMemoryTag(vm_map_t map) { - kern_return_t kr; - vm_offset_t address; - vm_offset_t allocationAddress; - vm_size_t adjustedSize; - uintptr_t alignMask; + vm_tag_t tag; - if (size == 0) - return 0; - if (alignment == 0) - alignment = 1; - - alignMask = alignment - 1; - adjustedSize = size + sizeof(vm_size_t) + sizeof(vm_address_t); + if (!vm_kernel_map_is_kernel(map)) { + return VM_MEMORY_IOKIT; + } - if (size > adjustedSize) { - address = 0; /* overflow detected */ - } - else if (adjustedSize >= page_size) { + tag = vm_tag_bt(); + if (tag == VM_KERN_MEMORY_NONE) { + tag = VM_KERN_MEMORY_IOKIT; + } - kr = kernel_memory_allocate(kernel_map, &address, - size, alignMask, 0); - if (KERN_SUCCESS != kr) - address = 0; + return tag; +} - } else { +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - adjustedSize += alignMask; +struct IOLibPageMallocHeader { + mach_vm_size_t allocationSize; + mach_vm_address_t allocationAddress; +#if IOTRACKING + IOTrackingAddress tracking; +#endif +}; - if (adjustedSize >= page_size) { +#if IOTRACKING +#define sizeofIOLibPageMallocHeader (sizeof(IOLibPageMallocHeader) - (TRACK_ALLOC ? 0 : sizeof(IOTrackingAddress))) +#else +#define sizeofIOLibPageMallocHeader (sizeof(IOLibPageMallocHeader)) +#endif - kr = kernel_memory_allocate(kernel_map, &allocationAddress, - adjustedSize, 0, 0); - if (KERN_SUCCESS != kr) - allocationAddress = 0; +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - } else - allocationAddress = (vm_address_t) kalloc(adjustedSize); +void * +IOMallocAligned(vm_size_t size, vm_size_t alignment) +{ + kern_return_t kr; + vm_offset_t address; + vm_offset_t allocationAddress; + vm_size_t adjustedSize; + uintptr_t alignMask; + IOLibPageMallocHeader * hdr; + + if (size == 0) { + return 0; + } - if (allocationAddress) { - address = (allocationAddress + alignMask - + (sizeof(vm_size_t) + sizeof(vm_address_t))) - & (~alignMask); + alignment = (1UL << log2up(alignment)); + alignMask = alignment - 1; + adjustedSize = size + sizeofIOLibPageMallocHeader; + + if (size > adjustedSize) { + address = 0; /* overflow detected */ + } else if (adjustedSize >= page_size) { + kr = kernel_memory_allocate(kernel_map, &address, + size, alignMask, 0, IOMemoryTag(kernel_map)); + if (KERN_SUCCESS != kr) { + address = 0; + } +#if IOTRACKING + else if (TRACK_ALLOC) { + IOTrackingAlloc(gIOMallocTracking, address, size); + } +#endif + } else { + adjustedSize += alignMask; + + if (adjustedSize >= page_size) { + kr = kernel_memory_allocate(kernel_map, &allocationAddress, + adjustedSize, 0, 0, IOMemoryTag(kernel_map)); + if (KERN_SUCCESS != kr) { + allocationAddress = 0; + } + } else { + allocationAddress = (vm_address_t) kalloc_tag_bt(adjustedSize, VM_KERN_MEMORY_IOKIT); + } - *((vm_size_t *)(address - sizeof(vm_size_t) - sizeof(vm_address_t))) - = adjustedSize; - *((vm_address_t *)(address - sizeof(vm_address_t))) - = allocationAddress; - } else - address = 0; - } + if (allocationAddress) { + address = (allocationAddress + alignMask + sizeofIOLibPageMallocHeader) + & (~alignMask); + + hdr = (typeof(hdr))(address - sizeofIOLibPageMallocHeader); + hdr->allocationSize = adjustedSize; + hdr->allocationAddress = allocationAddress; +#if IOTRACKING + if (TRACK_ALLOC) { + bzero(&hdr->tracking, sizeof(hdr->tracking)); + hdr->tracking.address = ~address; + hdr->tracking.size = size; + IOTrackingAdd(gIOMallocTracking, &hdr->tracking.tracking, size, true, VM_KERN_MEMORY_NONE); + } +#endif + } else { + address = 0; + } + } - assert(0 == (address & alignMask)); + assert(0 == (address & alignMask)); - if( address) { + if (address) { #if IOALLOCDEBUG - debug_iomalloc_size += size; + OSAddAtomic(size, &debug_iomalloc_size); #endif - IOStatisticsAlloc(kIOStatisticsMallocAligned, size); + IOStatisticsAlloc(kIOStatisticsMallocAligned, size); } - return (void *) address; + return (void *) address; } -void IOFreeAligned(void * address, vm_size_t size) +void +IOFreeAligned(void * address, vm_size_t size) { - vm_address_t allocationAddress; - vm_size_t adjustedSize; + vm_address_t allocationAddress; + vm_size_t adjustedSize; + IOLibPageMallocHeader * hdr; - if( !address) - return; - - assert(size); - - adjustedSize = size + sizeof(vm_size_t) + sizeof(vm_address_t); - if (adjustedSize >= page_size) { - - kmem_free( kernel_map, (vm_offset_t) address, size); + if (!address) { + return; + } - } else { - adjustedSize = *((vm_size_t *)( (vm_address_t) address - - sizeof(vm_address_t) - sizeof(vm_size_t))); - allocationAddress = *((vm_address_t *)( (vm_address_t) address - - sizeof(vm_address_t) )); + assert(size); - if (adjustedSize >= page_size) - kmem_free( kernel_map, allocationAddress, adjustedSize); - else - kfree((void *)allocationAddress, adjustedSize); - } + adjustedSize = size + sizeofIOLibPageMallocHeader; + if (adjustedSize >= page_size) { +#if IOTRACKING + if (TRACK_ALLOC) { + IOTrackingFree(gIOMallocTracking, (uintptr_t) address, size); + } +#endif + kmem_free( kernel_map, (vm_offset_t) address, size); + } else { + hdr = (typeof(hdr))(((uintptr_t)address) - sizeofIOLibPageMallocHeader); + adjustedSize = hdr->allocationSize; + allocationAddress = hdr->allocationAddress; + +#if IOTRACKING + if (TRACK_ALLOC) { + if (size != hdr->tracking.size) { + OSReportWithBacktrace("bad IOFreeAligned size 0x%lx should be 0x%lx", size, hdr->tracking.size); + size = hdr->tracking.size; + } + IOTrackingRemove(gIOMallocTracking, &hdr->tracking.tracking, size); + } +#endif + if (adjustedSize >= page_size) { + kmem_free( kernel_map, allocationAddress, adjustedSize); + } else { + kfree(allocationAddress, adjustedSize); + } + } #if IOALLOCDEBUG - debug_iomalloc_size -= size; + OSAddAtomic(-size, &debug_iomalloc_size); #endif - IOStatisticsAlloc(kIOStatisticsFreeAligned, size); + IOStatisticsAlloc(kIOStatisticsFreeAligned, size); } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ @@ -332,600 +509,639 @@ void IOFreeAligned(void * address, vm_size_t size) void IOKernelFreePhysical(mach_vm_address_t address, mach_vm_size_t size) { - mach_vm_address_t allocationAddress; - mach_vm_size_t adjustedSize; + mach_vm_address_t allocationAddress; + mach_vm_size_t adjustedSize; + IOLibPageMallocHeader * hdr; - if (!address) - return; - - assert(size); - - adjustedSize = (2 * size) + sizeof(mach_vm_size_t) + sizeof(mach_vm_address_t); - if (adjustedSize >= page_size) { - - kmem_free( kernel_map, (vm_offset_t) address, size); + if (!address) { + return; + } - } else { + assert(size); - adjustedSize = *((mach_vm_size_t *) - (address - sizeof(mach_vm_address_t) - sizeof(mach_vm_size_t))); - allocationAddress = *((mach_vm_address_t *) - (address - sizeof(mach_vm_address_t) )); - kfree((void *)allocationAddress, adjustedSize); - } + adjustedSize = (2 * size) + sizeofIOLibPageMallocHeader; + if (adjustedSize >= page_size) { +#if IOTRACKING + if (TRACK_ALLOC) { + IOTrackingFree(gIOMallocTracking, address, size); + } +#endif + kmem_free( kernel_map, (vm_offset_t) address, size); + } else { + hdr = (typeof(hdr))(((uintptr_t)address) - sizeofIOLibPageMallocHeader); + adjustedSize = hdr->allocationSize; + allocationAddress = hdr->allocationAddress; +#if IOTRACKING + if (TRACK_ALLOC) { + IOTrackingRemove(gIOMallocTracking, &hdr->tracking.tracking, size); + } +#endif + kfree(allocationAddress, adjustedSize); + } - IOStatisticsAlloc(kIOStatisticsFreeContiguous, size); + IOStatisticsAlloc(kIOStatisticsFreeContiguous, size); #if IOALLOCDEBUG - debug_iomalloc_size -= size; + OSAddAtomic(-size, &debug_iomalloc_size); #endif } +#if __arm__ || __arm64__ +extern unsigned long gPhysBase, gPhysSize; +#endif + mach_vm_address_t -IOKernelAllocateWithPhysicalRestrict(mach_vm_size_t size, mach_vm_address_t maxPhys, - mach_vm_size_t alignment, bool contiguous) +IOKernelAllocateWithPhysicalRestrict(mach_vm_size_t size, mach_vm_address_t maxPhys, + mach_vm_size_t alignment, bool contiguous) { - kern_return_t kr; - mach_vm_address_t address; - mach_vm_address_t allocationAddress; - mach_vm_size_t adjustedSize; - mach_vm_address_t alignMask; + kern_return_t kr; + mach_vm_address_t address; + mach_vm_address_t allocationAddress; + mach_vm_size_t adjustedSize; + mach_vm_address_t alignMask; + IOLibPageMallocHeader * hdr; + + if (size == 0) { + return 0; + } + if (alignment == 0) { + alignment = 1; + } - if (size == 0) - return (0); - if (alignment == 0) - alignment = 1; + alignMask = alignment - 1; - alignMask = alignment - 1; - adjustedSize = (2 * size) + sizeof(mach_vm_size_t) + sizeof(mach_vm_address_t); + if (os_mul_and_add_overflow(2, size, sizeofIOLibPageMallocHeader, &adjustedSize)) { + return 0; + } - contiguous = (contiguous && (adjustedSize > page_size)) - || (alignment > page_size); + contiguous = (contiguous && (adjustedSize > page_size)) + || (alignment > page_size); - if (contiguous || maxPhys) - { - int options = 0; - vm_offset_t virt; + if (contiguous || maxPhys) { + int options = 0; + vm_offset_t virt; - adjustedSize = size; - contiguous = (contiguous && (adjustedSize > page_size)) - || (alignment > page_size); + adjustedSize = size; + contiguous = (contiguous && (adjustedSize > page_size)) + || (alignment > page_size); - if (!contiguous) - { - if (maxPhys <= 0xFFFFFFFF) - { - maxPhys = 0; - options |= KMA_LOMEM; - } - else if (gIOLastPage && (atop_64(maxPhys) > gIOLastPage)) - { - maxPhys = 0; - } - } - if (contiguous || maxPhys) - { - kr = kmem_alloc_contig(kernel_map, &virt, size, - alignMask, atop(maxPhys), atop(alignMask), 0); - } - else - { - kr = kernel_memory_allocate(kernel_map, &virt, - size, alignMask, options); + if (!contiguous) { +#if __arm__ || __arm64__ + if (maxPhys >= (mach_vm_address_t)(gPhysBase + gPhysSize)) { + maxPhys = 0; + } else +#endif + if (maxPhys <= 0xFFFFFFFF) { + maxPhys = 0; + options |= KMA_LOMEM; + } else if (gIOLastPage && (atop_64(maxPhys) > gIOLastPage)) { + maxPhys = 0; + } + } + if (contiguous || maxPhys) { + kr = kmem_alloc_contig(kernel_map, &virt, size, + alignMask, atop(maxPhys), atop(alignMask), 0, IOMemoryTag(kernel_map)); + } else { + kr = kernel_memory_allocate(kernel_map, &virt, + size, alignMask, options, IOMemoryTag(kernel_map)); + } + if (KERN_SUCCESS == kr) { + address = virt; +#if IOTRACKING + if (TRACK_ALLOC) { + IOTrackingAlloc(gIOMallocTracking, address, size); + } +#endif + } else { + address = 0; + } + } else { + adjustedSize += alignMask; + if (adjustedSize < size) { + return 0; + } + allocationAddress = (mach_vm_address_t) kalloc_tag_bt(adjustedSize, VM_KERN_MEMORY_IOKIT); + + if (allocationAddress) { + address = (allocationAddress + alignMask + sizeofIOLibPageMallocHeader) + & (~alignMask); + + if (atop_32(address) != atop_32(address + size - 1)) { + address = round_page(address); + } + + hdr = (typeof(hdr))(address - sizeofIOLibPageMallocHeader); + hdr->allocationSize = adjustedSize; + hdr->allocationAddress = allocationAddress; +#if IOTRACKING + if (TRACK_ALLOC) { + bzero(&hdr->tracking, sizeof(hdr->tracking)); + hdr->tracking.address = ~address; + hdr->tracking.size = size; + IOTrackingAdd(gIOMallocTracking, &hdr->tracking.tracking, size, true, VM_KERN_MEMORY_NONE); + } +#endif + } else { + address = 0; + } } - if (KERN_SUCCESS == kr) - address = virt; - else - address = 0; - } - else - { - adjustedSize += alignMask; - allocationAddress = (mach_vm_address_t) kalloc(adjustedSize); - - if (allocationAddress) { - - address = (allocationAddress + alignMask - + (sizeof(mach_vm_size_t) + sizeof(mach_vm_address_t))) - & (~alignMask); - - if (atop_32(address) != atop_32(address + size - 1)) - address = round_page(address); - - *((mach_vm_size_t *)(address - sizeof(mach_vm_size_t) - - sizeof(mach_vm_address_t))) = adjustedSize; - *((mach_vm_address_t *)(address - sizeof(mach_vm_address_t))) - = allocationAddress; - } else - address = 0; - } - - if (address) { - IOStatisticsAlloc(kIOStatisticsMallocContiguous, size); + + if (address) { + IOStatisticsAlloc(kIOStatisticsMallocContiguous, size); #if IOALLOCDEBUG - debug_iomalloc_size += size; + OSAddAtomic(size, &debug_iomalloc_size); #endif - } + } - return (address); + return address; } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -struct _IOMallocContiguousEntry -{ - mach_vm_address_t virtualAddr; - IOBufferMemoryDescriptor * md; - queue_chain_t link; +struct _IOMallocContiguousEntry { + mach_vm_address_t virtualAddr; + IOBufferMemoryDescriptor * md; + queue_chain_t link; }; typedef struct _IOMallocContiguousEntry _IOMallocContiguousEntry; -void * IOMallocContiguous(vm_size_t size, vm_size_t alignment, - IOPhysicalAddress * physicalAddress) +void * +IOMallocContiguous(vm_size_t size, vm_size_t alignment, + IOPhysicalAddress * physicalAddress) { - mach_vm_address_t address = 0; - - if (size == 0) - return 0; - if (alignment == 0) - alignment = 1; - - /* Do we want a physical address? */ - if (!physicalAddress) - { - address = IOKernelAllocateWithPhysicalRestrict(size, 0 /*maxPhys*/, alignment, true); - } - else do - { - IOBufferMemoryDescriptor * bmd; - mach_vm_address_t physicalMask; - vm_offset_t alignMask; + mach_vm_address_t address = 0; - alignMask = alignment - 1; - physicalMask = (0xFFFFFFFF ^ alignMask); - - bmd = IOBufferMemoryDescriptor::inTaskWithPhysicalMask( - kernel_task, kIOMemoryPhysicallyContiguous, size, physicalMask); - if (!bmd) - break; - - _IOMallocContiguousEntry * - entry = IONew(_IOMallocContiguousEntry, 1); - if (!entry) - { - bmd->release(); - break; + if (size == 0) { + return 0; } - entry->virtualAddr = (mach_vm_address_t) bmd->getBytesNoCopy(); - entry->md = bmd; - lck_mtx_lock(gIOMallocContiguousEntriesLock); - queue_enter( &gIOMallocContiguousEntries, entry, - _IOMallocContiguousEntry *, link ); - lck_mtx_unlock(gIOMallocContiguousEntriesLock); + if (alignment == 0) { + alignment = 1; + } + + /* Do we want a physical address? */ + if (!physicalAddress) { + address = IOKernelAllocateWithPhysicalRestrict(size, 0 /*maxPhys*/, alignment, true); + } else { + do { + IOBufferMemoryDescriptor * bmd; + mach_vm_address_t physicalMask; + vm_offset_t alignMask; + + alignMask = alignment - 1; + physicalMask = (0xFFFFFFFF ^ alignMask); + + bmd = IOBufferMemoryDescriptor::inTaskWithPhysicalMask( + kernel_task, kIOMemoryPhysicallyContiguous, size, physicalMask); + if (!bmd) { + break; + } + + _IOMallocContiguousEntry * + entry = IONew(_IOMallocContiguousEntry, 1); + if (!entry) { + bmd->release(); + break; + } + entry->virtualAddr = (mach_vm_address_t) bmd->getBytesNoCopy(); + entry->md = bmd; + lck_mtx_lock(gIOMallocContiguousEntriesLock); + queue_enter( &gIOMallocContiguousEntries, entry, + _IOMallocContiguousEntry *, link ); + lck_mtx_unlock(gIOMallocContiguousEntriesLock); - address = (mach_vm_address_t) entry->virtualAddr; - *physicalAddress = bmd->getPhysicalAddress(); - } - while (false); + address = (mach_vm_address_t) entry->virtualAddr; + *physicalAddress = bmd->getPhysicalAddress(); + }while (false); + } - return (void *) address; + return (void *) address; } -void IOFreeContiguous(void * _address, vm_size_t size) +void +IOFreeContiguous(void * _address, vm_size_t size) { - _IOMallocContiguousEntry * entry; - IOMemoryDescriptor * md = NULL; + _IOMallocContiguousEntry * entry; + IOMemoryDescriptor * md = NULL; - mach_vm_address_t address = (mach_vm_address_t) _address; + mach_vm_address_t address = (mach_vm_address_t) _address; - if( !address) - return; + if (!address) { + return; + } - assert(size); + assert(size); - lck_mtx_lock(gIOMallocContiguousEntriesLock); - queue_iterate( &gIOMallocContiguousEntries, entry, - _IOMallocContiguousEntry *, link ) - { - if( entry->virtualAddr == address ) { - md = entry->md; - queue_remove( &gIOMallocContiguousEntries, entry, + lck_mtx_lock(gIOMallocContiguousEntriesLock); + queue_iterate( &gIOMallocContiguousEntries, entry, + _IOMallocContiguousEntry *, link ) + { + if (entry->virtualAddr == address) { + md = entry->md; + queue_remove( &gIOMallocContiguousEntries, entry, _IOMallocContiguousEntry *, link ); - break; + break; + } + } + lck_mtx_unlock(gIOMallocContiguousEntriesLock); + + if (md) { + md->release(); + IODelete(entry, _IOMallocContiguousEntry, 1); + } else { + IOKernelFreePhysical((mach_vm_address_t) address, size); } - } - lck_mtx_unlock(gIOMallocContiguousEntriesLock); - - if (md) - { - md->release(); - IODelete(entry, _IOMallocContiguousEntry, 1); - } - else - { - IOKernelFreePhysical((mach_vm_address_t) address, size); - } } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -kern_return_t IOIteratePageableMaps(vm_size_t size, - IOIteratePageableMapsCallback callback, void * ref) +kern_return_t +IOIteratePageableMaps(vm_size_t size, + IOIteratePageableMapsCallback callback, void * ref) { - kern_return_t kr = kIOReturnNotReady; - vm_size_t segSize; - UInt32 attempts; - UInt32 index; - vm_offset_t min; - vm_map_t map; - - if (size > kIOPageableMaxMapSize) - return( kIOReturnBadArgument ); - - do { - index = gIOKitPageableSpace.hint; - attempts = gIOKitPageableSpace.count; - while( attempts--) { - kr = (*callback)(gIOKitPageableSpace.maps[index].map, ref); - if( KERN_SUCCESS == kr) { - gIOKitPageableSpace.hint = index; - break; - } - if( index) - index--; - else - index = gIOKitPageableSpace.count - 1; - } - if( KERN_SUCCESS == kr) - break; - - lck_mtx_lock( gIOKitPageableSpace.lock ); - - index = gIOKitPageableSpace.count; - if( index >= (kIOMaxPageableMaps - 1)) { - lck_mtx_unlock( gIOKitPageableSpace.lock ); - break; - } - - if( size < kIOPageableMapSize) - segSize = kIOPageableMapSize; - else - segSize = size; - - min = 0; - kr = kmem_suballoc(kernel_map, - &min, - segSize, - TRUE, - VM_FLAGS_ANYWHERE, - &map); - if( KERN_SUCCESS != kr) { - lck_mtx_unlock( gIOKitPageableSpace.lock ); - break; - } - - gIOKitPageableSpace.maps[index].map = map; - gIOKitPageableSpace.maps[index].address = min; - gIOKitPageableSpace.maps[index].end = min + segSize; - gIOKitPageableSpace.hint = index; - gIOKitPageableSpace.count = index + 1; - - lck_mtx_unlock( gIOKitPageableSpace.lock ); - - } while( true ); - - return kr; + kern_return_t kr = kIOReturnNotReady; + vm_size_t segSize; + UInt32 attempts; + UInt32 index; + vm_offset_t min; + vm_map_t map; + + if (size > kIOPageableMaxMapSize) { + return kIOReturnBadArgument; + } + + do { + index = gIOKitPageableSpace.hint; + attempts = gIOKitPageableSpace.count; + while (attempts--) { + kr = (*callback)(gIOKitPageableSpace.maps[index].map, ref); + if (KERN_SUCCESS == kr) { + gIOKitPageableSpace.hint = index; + break; + } + if (index) { + index--; + } else { + index = gIOKitPageableSpace.count - 1; + } + } + if (KERN_NO_SPACE != kr) { + break; + } + + lck_mtx_lock( gIOKitPageableSpace.lock ); + + index = gIOKitPageableSpace.count; + if (index >= (kIOMaxPageableMaps - 1)) { + lck_mtx_unlock( gIOKitPageableSpace.lock ); + break; + } + + if (size < kIOPageableMapSize) { + segSize = kIOPageableMapSize; + } else { + segSize = size; + } + + min = 0; + kr = kmem_suballoc(kernel_map, + &min, + segSize, + TRUE, + VM_FLAGS_ANYWHERE, + VM_MAP_KERNEL_FLAGS_NONE, + VM_KERN_MEMORY_IOKIT, + &map); + if (KERN_SUCCESS != kr) { + lck_mtx_unlock( gIOKitPageableSpace.lock ); + break; + } + + gIOKitPageableSpace.maps[index].map = map; + gIOKitPageableSpace.maps[index].address = min; + gIOKitPageableSpace.maps[index].end = min + segSize; + gIOKitPageableSpace.hint = index; + gIOKitPageableSpace.count = index + 1; + + lck_mtx_unlock( gIOKitPageableSpace.lock ); + } while (true); + + return kr; } -struct IOMallocPageableRef -{ - vm_offset_t address; - vm_size_t size; +struct IOMallocPageableRef { + vm_offset_t address; + vm_size_t size; + vm_tag_t tag; }; -static kern_return_t IOMallocPageableCallback(vm_map_t map, void * _ref) +static kern_return_t +IOMallocPageableCallback(vm_map_t map, void * _ref) { - struct IOMallocPageableRef * ref = (struct IOMallocPageableRef *) _ref; - kern_return_t kr; + struct IOMallocPageableRef * ref = (struct IOMallocPageableRef *) _ref; + kern_return_t kr; - kr = kmem_alloc_pageable( map, &ref->address, ref->size ); + kr = kmem_alloc_pageable( map, &ref->address, ref->size, ref->tag ); - return( kr ); + return kr; } -static void * IOMallocPageablePages(vm_size_t size, vm_size_t alignment) +static void * +IOMallocPageablePages(vm_size_t size, vm_size_t alignment, vm_tag_t tag) { - kern_return_t kr = kIOReturnNotReady; - struct IOMallocPageableRef ref; + kern_return_t kr = kIOReturnNotReady; + struct IOMallocPageableRef ref; - if (alignment > page_size) - return( 0 ); - if (size > kIOPageableMaxMapSize) - return( 0 ); + if (alignment > page_size) { + return 0; + } + if (size > kIOPageableMaxMapSize) { + return 0; + } - ref.size = size; - kr = IOIteratePageableMaps( size, &IOMallocPageableCallback, &ref ); - if( kIOReturnSuccess != kr) - ref.address = 0; + ref.size = size; + ref.tag = tag; + kr = IOIteratePageableMaps( size, &IOMallocPageableCallback, &ref ); + if (kIOReturnSuccess != kr) { + ref.address = 0; + } - return( (void *) ref.address ); + return (void *) ref.address; } -vm_map_t IOPageableMapForAddress( uintptr_t address ) +vm_map_t +IOPageableMapForAddress( uintptr_t address ) { - vm_map_t map = 0; - UInt32 index; - - for( index = 0; index < gIOKitPageableSpace.count; index++) { - if( (address >= gIOKitPageableSpace.maps[index].address) - && (address < gIOKitPageableSpace.maps[index].end) ) { - map = gIOKitPageableSpace.maps[index].map; - break; - } - } - if( !map) - panic("IOPageableMapForAddress: null"); - - return( map ); + vm_map_t map = 0; + UInt32 index; + + for (index = 0; index < gIOKitPageableSpace.count; index++) { + if ((address >= gIOKitPageableSpace.maps[index].address) + && (address < gIOKitPageableSpace.maps[index].end)) { + map = gIOKitPageableSpace.maps[index].map; + break; + } + } + if (!map) { + panic("IOPageableMapForAddress: null"); + } + + return map; } -static void IOFreePageablePages(void * address, vm_size_t size) +static void +IOFreePageablePages(void * address, vm_size_t size) { - vm_map_t map; - - map = IOPageableMapForAddress( (vm_address_t) address); - if( map) - kmem_free( map, (vm_offset_t) address, size); + vm_map_t map; + + map = IOPageableMapForAddress((vm_address_t) address); + if (map) { + kmem_free( map, (vm_offset_t) address, size); + } } -static uintptr_t IOMallocOnePageablePage(iopa_t * a) +static uintptr_t +IOMallocOnePageablePage(iopa_t * a) { - return ((uintptr_t) IOMallocPageablePages(page_size, page_size)); + return (uintptr_t) IOMallocPageablePages(page_size, page_size, VM_KERN_MEMORY_IOKIT); } -void * IOMallocPageable(vm_size_t size, vm_size_t alignment) +void * +IOMallocPageable(vm_size_t size, vm_size_t alignment) { - void * addr; + void * addr; - if (size >= (page_size - 4*kIOPageAllocChunkBytes)) addr = IOMallocPageablePages(size, alignment); - else addr = ((void * ) iopa_alloc(&gIOPageablePageAllocator, &IOMallocOnePageablePage, size, alignment)); + if (size >= (page_size - 4 * gIOPageAllocChunkBytes)) { + addr = IOMallocPageablePages(size, alignment, IOMemoryTag(kernel_map)); + } else { + addr = ((void *) iopa_alloc(&gIOPageablePageAllocator, &IOMallocOnePageablePage, size, alignment)); + } - if (addr) { + if (addr) { #if IOALLOCDEBUG - debug_iomallocpageable_size += size; + OSAddAtomicLong(size, &debug_iomallocpageable_size); #endif - IOStatisticsAlloc(kIOStatisticsMallocPageable, size); - } + IOStatisticsAlloc(kIOStatisticsMallocPageable, size); + } - return (addr); + return addr; } -void IOFreePageable(void * address, vm_size_t size) +void +IOFreePageable(void * address, vm_size_t size) { #if IOALLOCDEBUG - debug_iomallocpageable_size -= size; + OSAddAtomicLong(-size, &debug_iomallocpageable_size); #endif - IOStatisticsAlloc(kIOStatisticsFreePageable, size); - - if (size < (page_size - 4*kIOPageAllocChunkBytes)) - { - address = (void *) iopa_free(&gIOPageablePageAllocator, (uintptr_t) address, size); - size = page_size; - } - if (address) IOFreePageablePages(address, size); + IOStatisticsAlloc(kIOStatisticsFreePageable, size); + + if (size < (page_size - 4 * gIOPageAllocChunkBytes)) { + address = (void *) iopa_free(&gIOPageablePageAllocator, (uintptr_t) address, size); + size = page_size; + } + if (address) { + IOFreePageablePages(address, size); + } } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#if 0 -#undef assert -#define assert(ex) \ - ((ex) ? (void)0 : Assert(__FILE__, __LINE__, # ex)) -#endif - -typedef char iopa_page_t_assert[(sizeof(iopa_page_t) <= kIOPageAllocChunkBytes) ? 1 : -1]; - -extern "C" void +extern "C" void iopa_init(iopa_t * a) { - bzero(a, sizeof(*a)); - a->lock = IOLockAlloc(); - queue_init(&a->list); + bzero(a, sizeof(*a)); + a->lock = IOLockAlloc(); + queue_init(&a->list); } static uintptr_t iopa_allocinpage(iopa_page_t * pa, uint32_t count, uint64_t align) { - uint32_t n, s; - uint64_t avail = pa->avail; - - assert(avail); - - // find strings of count 1 bits in avail - for (n = count; n > 1; n -= s) - { - s = n >> 1; - avail = avail & (avail << s); - } - // and aligned - avail &= align; - - if (avail) - { - n = __builtin_clzll(avail); - pa->avail &= ~((-1ULL << (64 - count)) >> n); - if (!pa->avail && pa->link.next) - { - remque(&pa->link); - pa->link.next = 0; - } - return (n * kIOPageAllocChunkBytes + trunc_page((uintptr_t) pa)); - } + uint32_t n, s; + uint64_t avail = pa->avail; - return (0); -} + assert(avail); -static uint32_t -log2up(uint32_t size) -{ - if (size <= 1) size = 0; - else size = 32 - __builtin_clz(size - 1); - return (size); + // find strings of count 1 bits in avail + for (n = count; n > 1; n -= s) { + s = n >> 1; + avail = avail & (avail << s); + } + // and aligned + avail &= align; + + if (avail) { + n = __builtin_clzll(avail); + pa->avail &= ~((-1ULL << (64 - count)) >> n); + if (!pa->avail && pa->link.next) { + remque(&pa->link); + pa->link.next = 0; + } + return n * gIOPageAllocChunkBytes + trunc_page((uintptr_t) pa); + } + + return 0; } -uintptr_t +uintptr_t iopa_alloc(iopa_t * a, iopa_proc_t alloc, vm_size_t bytes, uint32_t balign) { - static const uint64_t align_masks[] = { - 0xFFFFFFFFFFFFFFFF, - 0xAAAAAAAAAAAAAAAA, - 0x8888888888888888, - 0x8080808080808080, - 0x8000800080008000, - 0x8000000080000000, - 0x8000000000000000, - }; - iopa_page_t * pa; - uintptr_t addr = 0; - uint32_t count; - uint64_t align; - - if (!bytes) bytes = 1; - count = (bytes + kIOPageAllocChunkBytes - 1) / kIOPageAllocChunkBytes; - align = align_masks[log2up((balign + kIOPageAllocChunkBytes - 1) / kIOPageAllocChunkBytes)]; - - IOLockLock(a->lock); - pa = (typeof(pa)) queue_first(&a->list); - while (!queue_end(&a->list, &pa->link)) - { - addr = iopa_allocinpage(pa, count, align); - if (addr) - { - a->bytecount += bytes; - break; + static const uint64_t align_masks[] = { + 0xFFFFFFFFFFFFFFFF, + 0xAAAAAAAAAAAAAAAA, + 0x8888888888888888, + 0x8080808080808080, + 0x8000800080008000, + 0x8000000080000000, + 0x8000000000000000, + }; + iopa_page_t * pa; + uintptr_t addr = 0; + uint32_t count; + uint64_t align; + + if (!bytes) { + bytes = 1; } - pa = (typeof(pa)) queue_next(&pa->link); - } - IOLockUnlock(a->lock); - - if (!addr) - { - addr = alloc(a); - if (addr) - { - pa = (typeof(pa)) (addr + page_size - kIOPageAllocChunkBytes); - pa->signature = kIOPageAllocSignature; - pa->avail = -2ULL; - - addr = iopa_allocinpage(pa, count, align); - IOLockLock(a->lock); - if (pa->avail) enqueue_head(&a->list, &pa->link); - a->pagecount++; - if (addr) a->bytecount += bytes; - IOLockUnlock(a->lock); + count = (bytes + gIOPageAllocChunkBytes - 1) / gIOPageAllocChunkBytes; + align = align_masks[log2up((balign + gIOPageAllocChunkBytes - 1) / gIOPageAllocChunkBytes)]; + + IOLockLock(a->lock); + __IGNORE_WCASTALIGN(pa = (typeof(pa))queue_first(&a->list)); + while (!queue_end(&a->list, &pa->link)) { + addr = iopa_allocinpage(pa, count, align); + if (addr) { + a->bytecount += bytes; + break; + } + __IGNORE_WCASTALIGN(pa = (typeof(pa))queue_next(&pa->link)); + } + IOLockUnlock(a->lock); + + if (!addr) { + addr = alloc(a); + if (addr) { + pa = (typeof(pa))(addr + page_size - gIOPageAllocChunkBytes); + pa->signature = kIOPageAllocSignature; + pa->avail = -2ULL; + + addr = iopa_allocinpage(pa, count, align); + IOLockLock(a->lock); + if (pa->avail) { + enqueue_head(&a->list, &pa->link); + } + a->pagecount++; + if (addr) { + a->bytecount += bytes; + } + IOLockUnlock(a->lock); + } } - } - assert((addr & ((1 << log2up(balign)) - 1)) == 0); - return (addr); + assert((addr & ((1 << log2up(balign)) - 1)) == 0); + return addr; } -uintptr_t +uintptr_t iopa_free(iopa_t * a, uintptr_t addr, vm_size_t bytes) { - iopa_page_t * pa; - uint32_t count; - uintptr_t chunk; - - if (!bytes) bytes = 1; - - chunk = (addr & page_mask); - assert(0 == (chunk & (kIOPageAllocChunkBytes - 1))); - - pa = (typeof(pa)) (addr | (page_size - kIOPageAllocChunkBytes)); - assert(kIOPageAllocSignature == pa->signature); - - count = (bytes + kIOPageAllocChunkBytes - 1) / kIOPageAllocChunkBytes; - chunk /= kIOPageAllocChunkBytes; - - IOLockLock(a->lock); - if (!pa->avail) - { - assert(!pa->link.next); - enqueue_tail(&a->list, &pa->link); - } - pa->avail |= ((-1ULL << (64 - count)) >> chunk); - if (pa->avail != -2ULL) pa = 0; - else - { - remque(&pa->link); - pa->link.next = 0; - pa->signature = 0; - a->pagecount--; - // page to free - pa = (typeof(pa)) trunc_page(pa); - } - a->bytecount -= bytes; - IOLockUnlock(a->lock); - - return ((uintptr_t) pa); + iopa_page_t * pa; + uint32_t count; + uintptr_t chunk; + + if (!bytes) { + bytes = 1; + } + + chunk = (addr & page_mask); + assert(0 == (chunk & (gIOPageAllocChunkBytes - 1))); + + pa = (typeof(pa))(addr | (page_size - gIOPageAllocChunkBytes)); + assert(kIOPageAllocSignature == pa->signature); + + count = (bytes + gIOPageAllocChunkBytes - 1) / gIOPageAllocChunkBytes; + chunk /= gIOPageAllocChunkBytes; + + IOLockLock(a->lock); + if (!pa->avail) { + assert(!pa->link.next); + enqueue_tail(&a->list, &pa->link); + } + pa->avail |= ((-1ULL << (64 - count)) >> chunk); + if (pa->avail != -2ULL) { + pa = 0; + } else { + remque(&pa->link); + pa->link.next = 0; + pa->signature = 0; + a->pagecount--; + // page to free + pa = (typeof(pa))trunc_page(pa); + } + a->bytecount -= bytes; + IOLockUnlock(a->lock); + + return (uintptr_t) pa; } - + /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -IOReturn IOSetProcessorCacheMode( task_t task, IOVirtualAddress address, - IOByteCount length, IOOptionBits cacheMode ) +IOReturn +IOSetProcessorCacheMode( task_t task, IOVirtualAddress address, + IOByteCount length, IOOptionBits cacheMode ) { - IOReturn ret = kIOReturnSuccess; - ppnum_t pagenum; + IOReturn ret = kIOReturnSuccess; + ppnum_t pagenum; - if( task != kernel_task) - return( kIOReturnUnsupported ); - if ((address | length) & PAGE_MASK) - { + if (task != kernel_task) { + return kIOReturnUnsupported; + } + if ((address | length) & PAGE_MASK) { // OSReportWithBacktrace("IOSetProcessorCacheMode(0x%x, 0x%x, 0x%x) fails\n", address, length, cacheMode); - return( kIOReturnUnsupported ); - } - length = round_page(address + length) - trunc_page( address ); - address = trunc_page( address ); - - // make map mode - cacheMode = (cacheMode << kIOMapCacheShift) & kIOMapCacheMask; - - while( (kIOReturnSuccess == ret) && (length > 0) ) { - - // Get the physical page number - pagenum = pmap_find_phys(kernel_pmap, (addr64_t)address); - if( pagenum) { - ret = IOUnmapPages( get_task_map(task), address, page_size ); - ret = IOMapPages( get_task_map(task), address, ptoa_64(pagenum), page_size, cacheMode ); - } else - ret = kIOReturnVMError; + return kIOReturnUnsupported; + } + length = round_page(address + length) - trunc_page( address ); + address = trunc_page( address ); + + // make map mode + cacheMode = (cacheMode << kIOMapCacheShift) & kIOMapCacheMask; + + while ((kIOReturnSuccess == ret) && (length > 0)) { + // Get the physical page number + pagenum = pmap_find_phys(kernel_pmap, (addr64_t)address); + if (pagenum) { + ret = IOUnmapPages( get_task_map(task), address, page_size ); + ret = IOMapPages( get_task_map(task), address, ptoa_64(pagenum), page_size, cacheMode ); + } else { + ret = kIOReturnVMError; + } - address += page_size; - length -= page_size; - } + address += page_size; + length -= page_size; + } - return( ret ); + return ret; } -IOReturn IOFlushProcessorCache( task_t task, IOVirtualAddress address, - IOByteCount length ) +IOReturn +IOFlushProcessorCache( task_t task, IOVirtualAddress address, + IOByteCount length ) { - if( task != kernel_task) - return( kIOReturnUnsupported ); + if (task != kernel_task) { + return kIOReturnUnsupported; + } - flush_dcache64( (addr64_t) address, (unsigned) length, false ); + flush_dcache64((addr64_t) address, (unsigned) length, false ); - return( kIOReturnSuccess ); + return kIOReturnSuccess; } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -vm_offset_t OSKernelStackRemaining( void ) +vm_offset_t +OSKernelStackRemaining( void ) { - return (ml_stack_remaining()); + return ml_stack_remaining(); } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ @@ -933,64 +1149,85 @@ vm_offset_t OSKernelStackRemaining( void ) /* * Spin for indicated number of milliseconds. */ -void IOSleep(unsigned milliseconds) +void +IOSleep(unsigned milliseconds) { - delay_for_interval(milliseconds, kMillisecondScale); + delay_for_interval(milliseconds, kMillisecondScale); +} + +/* + * Spin for indicated number of milliseconds, and potentially an + * additional number of milliseconds up to the leeway values. + */ +void +IOSleepWithLeeway(unsigned intervalMilliseconds, unsigned leewayMilliseconds) +{ + delay_for_interval_with_leeway(intervalMilliseconds, leewayMilliseconds, kMillisecondScale); } /* * Spin for indicated number of microseconds. */ -void IODelay(unsigned microseconds) +void +IODelay(unsigned microseconds) { - delay_for_interval(microseconds, kMicrosecondScale); + delay_for_interval(microseconds, kMicrosecondScale); } /* * Spin for indicated number of nanoseconds. */ -void IOPause(unsigned nanoseconds) +void +IOPause(unsigned nanoseconds) { - delay_for_interval(nanoseconds, kNanosecondScale); + delay_for_interval(nanoseconds, kNanosecondScale); } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -static void _iolog_consputc(int ch, void *arg __unused) -{ - cons_putc_locked(ch); -} +static void _IOLogv(const char *format, va_list ap, void *caller) __printflike(1, 0); -static void _iolog_logputc(int ch, void *arg __unused) +__attribute__((noinline, not_tail_called)) +void +IOLog(const char *format, ...) { - log_putc_locked(ch); + void *caller = __builtin_return_address(0); + va_list ap; + + va_start(ap, format); + _IOLogv(format, ap, caller); + va_end(ap); } -void IOLog(const char *format, ...) +__attribute__((noinline, not_tail_called)) +void +IOLogv(const char *format, va_list ap) { - va_list ap; - - va_start(ap, format); - IOLogv(format, ap); - va_end(ap); + void *caller = __builtin_return_address(0); + _IOLogv(format, ap, caller); } -void IOLogv(const char *format, va_list ap) +void +_IOLogv(const char *format, va_list ap, void *caller) { - va_list ap2; + va_list ap2; + struct console_printbuf_state info_data; + console_printbuf_state_init(&info_data, TRUE, TRUE); - va_copy(ap2, ap); + va_copy(ap2, ap); - bsd_log_lock(); - __doprnt(format, ap, _iolog_logputc, NULL, 16); - bsd_log_unlock(); - logwakeup(); + os_log_with_args(OS_LOG_DEFAULT, OS_LOG_TYPE_DEFAULT, format, ap, caller); - __doprnt(format, ap2, _iolog_consputc, NULL, 16); + __doprnt(format, ap2, console_printbuf_putc, &info_data, 16, TRUE); + console_printbuf_clear(&info_data); + va_end(ap2); + + assertf(ml_get_interrupts_enabled() || ml_is_quiescing() || debug_mode_active() || !gCPUsRunning, "IOLog called with interrupts disabled"); } #if !__LP64__ -void IOPanic(const char *reason) +void +IOPanic(const char *reason) { panic("%s", reason); } @@ -998,27 +1235,72 @@ void IOPanic(const char *reason) /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +void +IOKitKernelLogBuffer(const char * title, const void * buffer, size_t size, + void (*output)(const char *format, ...)) +{ + uint8_t c, chars[17]; + size_t idx; + + output("%s(0x%x):\n", title, size); + if (size > 4096) { + size = 4096; + } + chars[16] = idx = 0; + while (true) { + if (!(idx & 15)) { + if (idx) { + output(" |%s|\n", chars); + } + if (idx >= size) { + break; + } + output("%04x: ", idx); + } else if (!(idx & 7)) { + output(" "); + } + + c = ((char *)buffer)[idx]; + output("%02x ", c); + chars[idx & 15] = ((c >= 0x20) && (c <= 0x7f)) ? c : ' '; + + idx++; + if ((idx == size) && (idx & 15)) { + chars[idx & 15] = 0; + while (idx & 15) { + idx++; + output(" "); + } + } + } +} + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + /* * Convert a integer constant (typically a #define or enum) to a string. */ -static char noValue[80]; // that's pretty +static char noValue[80]; // that's pretty -const char *IOFindNameForValue(int value, const IONamedValue *regValueArray) +const char * +IOFindNameForValue(int value, const IONamedValue *regValueArray) { - for( ; regValueArray->name; regValueArray++) { - if(regValueArray->value == value) - return(regValueArray->name); + for (; regValueArray->name; regValueArray++) { + if (regValueArray->value == value) { + return regValueArray->name; + } } snprintf(noValue, sizeof(noValue), "0x%x (UNDEFINED)", value); - return((const char *)noValue); + return (const char *)noValue; } -IOReturn IOFindValueForName(const char *string, - const IONamedValue *regValueArray, - int *value) +IOReturn +IOFindValueForName(const char *string, + const IONamedValue *regValueArray, + int *value) { - for( ; regValueArray->name; regValueArray++) { - if(!strcmp(regValueArray->name, string)) { + for (; regValueArray->name; regValueArray++) { + if (!strcmp(regValueArray->name, string)) { *value = regValueArray->value; return kIOReturnSuccess; } @@ -1026,42 +1308,42 @@ IOReturn IOFindValueForName(const char *string, return kIOReturnBadArgument; } -OSString * IOCopyLogNameForPID(int pid) +OSString * +IOCopyLogNameForPID(int pid) { - char buf[128]; - size_t len; - snprintf(buf, sizeof(buf), "pid %d, ", pid); - len = strlen(buf); - proc_name(pid, buf + len, sizeof(buf) - len); - return (OSString::withCString(buf)); + char buf[128]; + size_t len; + snprintf(buf, sizeof(buf), "pid %d, ", pid); + len = strlen(buf); + proc_name(pid, buf + len, sizeof(buf) - len); + return OSString::withCString(buf); } /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -IOAlignment IOSizeToAlignment(unsigned int size) +IOAlignment +IOSizeToAlignment(unsigned int size) { - register int shift; - const int intsize = sizeof(unsigned int) * 8; - - for (shift = 1; shift < intsize; shift++) { - if (size & 0x80000000) - return (IOAlignment)(intsize - shift); - size <<= 1; - } - return 0; + int shift; + const int intsize = sizeof(unsigned int) * 8; + + for (shift = 1; shift < intsize; shift++) { + if (size & 0x80000000) { + return (IOAlignment)(intsize - shift); + } + size <<= 1; + } + return 0; } -unsigned int IOAlignmentToSize(IOAlignment align) +unsigned int +IOAlignmentToSize(IOAlignment align) { - unsigned int size; - - for (size = 1; align; align--) { - size <<= 1; - } - return size; -} + unsigned int size; + for (size = 1; align; align--) { + size <<= 1; + } + return size; +} } /* extern "C" */ - - -