X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/ff6e181ae92fc6f1e89841290f461d1f2f9badd9..e2d2fc5c71f7d145cba7267989251af45e3bb5ba:/osfmk/kern/kalloc.c?ds=sidebyside diff --git a/osfmk/kern/kalloc.c b/osfmk/kern/kalloc.c index 36a4c90e1..f84a19956 100644 --- a/osfmk/kern/kalloc.c +++ b/osfmk/kern/kalloc.c @@ -1,14 +1,19 @@ /* - * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2000-2006 Apple Computer, Inc. All rights reserved. * - * @APPLE_LICENSE_HEADER_START@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in - * compliance with the License. Please obtain a copy of the License at - * http://www.opensource.apple.com/apsl/ and read it before using this - * file. + * compliance with the License. The rights granted to you under the License + * may not be used to create, or enable the creation or redistribution of, + * unlawful or unlicensed copies of an Apple operating system, or to + * circumvent, violate, or enable the circumvention or violation of, any + * terms of an Apple operating system software license agreement. + * + * Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this file. * * The Original Code and all software distributed under the License are * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER @@ -18,7 +23,7 @@ * Please see the License for the specific language governing rights and * limitations under the License. * - * @APPLE_LICENSE_HEADER_END@ + * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ */ /* * @OSF_COPYRIGHT@ @@ -77,14 +82,56 @@ zone_t kalloc_zone(vm_size_t); #endif +#define KALLOC_MAP_SIZE_MIN (16 * 1024 * 1024) +#define KALLOC_MAP_SIZE_MAX (128 * 1024 * 1024) vm_map_t kalloc_map; -vm_size_t kalloc_map_size = 16 * 1024 * 1024; vm_size_t kalloc_max; vm_size_t kalloc_max_prerounded; +vm_size_t kalloc_kernmap_size; /* size of kallocs that can come from kernel map */ unsigned int kalloc_large_inuse; vm_size_t kalloc_large_total; vm_size_t kalloc_large_max; +vm_size_t kalloc_largest_allocated = 0; +uint64_t kalloc_large_sum; + +int kalloc_fake_zone_index = -1; /* index of our fake zone in statistics arrays */ + +vm_offset_t kalloc_map_min; +vm_offset_t kalloc_map_max; + +#ifdef MUTEX_ZONE +/* + * Diagnostic code to track mutexes separately rather than via the 2^ zones + */ + zone_t lck_mtx_zone; +#endif + +static void +KALLOC_ZINFO_SALLOC(vm_size_t bytes) +{ + thread_t thr = current_thread(); + task_t task; + zinfo_usage_t zinfo; + + thr->tkm_shared.alloc += bytes; + if (kalloc_fake_zone_index != -1 && + (task = thr->task) != NULL && (zinfo = task->tkm_zinfo) != NULL) + zinfo[kalloc_fake_zone_index].alloc += bytes; +} + +static void +KALLOC_ZINFO_SFREE(vm_size_t bytes) +{ + thread_t thr = current_thread(); + task_t task; + zinfo_usage_t zinfo; + + thr->tkm_shared.free += bytes; + if (kalloc_fake_zone_index != -1 && + (task = thr->task) != NULL && (zinfo = task->tkm_zinfo) != NULL) + zinfo[kalloc_fake_zone_index].free += bytes; +} /* * All allocations of size less than kalloc_max are rounded to the @@ -99,6 +146,8 @@ vm_size_t kalloc_large_max; * It represents the first power of two for which no zone exists. * kalloc_max_prerounded is the smallest allocation size, before * rounding, for which no zone exists. + * Also if the allocation size is more than kalloc_kernmap_size + * then allocate from kernel map rather than kalloc_map. */ int first_k_zone = -1; @@ -145,11 +194,23 @@ void * kalloc_canblock( boolean_t canblock); +lck_grp_t *kalloc_lck_grp; +lck_mtx_t kalloc_lock; + +#define kalloc_spin_lock() lck_mtx_lock_spin(&kalloc_lock) +#define kalloc_unlock() lck_mtx_unlock(&kalloc_lock) + + /* OSMalloc local data declarations */ static queue_head_t OSMalloc_tag_list; -decl_simple_lock_data(static,OSMalloc_tag_lock) +lck_grp_t *OSMalloc_tag_lck_grp; +lck_mtx_t OSMalloc_tag_lock; + +#define OSMalloc_tag_spin_lock() lck_mtx_lock_spin(&OSMalloc_tag_lock) +#define OSMalloc_tag_unlock() lck_mtx_unlock(&OSMalloc_tag_lock) + /* OSMalloc forward declarations */ void OSMalloc_init(void); @@ -170,15 +231,31 @@ kalloc_init( { kern_return_t retval; vm_offset_t min; - vm_size_t size; + vm_size_t size, kalloc_map_size; register int i; + /* + * Scale the kalloc_map_size to physical memory size: stay below + * 1/8th the total zone map size, or 128 MB (for a 32-bit kernel). + */ + kalloc_map_size = (vm_size_t)(sane_size >> 5); +#if !__LP64__ + if (kalloc_map_size > KALLOC_MAP_SIZE_MAX) + kalloc_map_size = KALLOC_MAP_SIZE_MAX; +#endif /* !__LP64__ */ + if (kalloc_map_size < KALLOC_MAP_SIZE_MIN) + kalloc_map_size = KALLOC_MAP_SIZE_MIN; + retval = kmem_suballoc(kernel_map, &min, kalloc_map_size, - FALSE, VM_FLAGS_ANYWHERE, &kalloc_map); + FALSE, VM_FLAGS_ANYWHERE | VM_FLAGS_PERMANENT, + &kalloc_map); if (retval != KERN_SUCCESS) panic("kalloc_init: kmem_suballoc failed"); + kalloc_map_min = min; + kalloc_map_max = min + kalloc_map_size - 1; + /* * Ensure that zones up to size 8192 bytes exist. * This is desirable because messages are allocated @@ -190,14 +267,19 @@ kalloc_init( else kalloc_max = PAGE_SIZE; kalloc_max_prerounded = kalloc_max / 2 + 1; + /* size it to be more than 16 times kalloc_max (256k) for allocations from kernel map */ + kalloc_kernmap_size = (kalloc_max * 16) + 1; + kalloc_largest_allocated = kalloc_kernmap_size; /* * Allocate a zone for each size we are going to handle. - * We specify non-paged memory. + * We specify non-paged memory. Don't charge the caller + * for the allocation, as we aren't sure how the memory + * will be handled. */ for (i = 0, size = 1; size < kalloc_max; i++, size <<= 1) { if (size < KALLOC_MINSIZE) { - k_zone[i] = 0; + k_zone[i] = NULL; continue; } if (size == KALLOC_MINSIZE) { @@ -205,8 +287,15 @@ kalloc_init( } k_zone[i] = zinit(size, k_zone_max[i] * size, size, k_zone_name[i]); + zone_change(k_zone[i], Z_CALLERACCT, FALSE); } + kalloc_lck_grp = lck_grp_alloc_init("kalloc.large", LCK_GRP_ATTR_NULL); + lck_mtx_init(&kalloc_lock, kalloc_lck_grp, LCK_ATTR_NULL); OSMalloc_init(); +#ifdef MUTEX_ZONE + lck_mtx_zone = zinit(sizeof(struct _lck_mtx_), 1024*256, 4096, "lck_mtx"); +#endif + } void * @@ -216,10 +305,11 @@ kalloc_canblock( { register int zindex; register vm_size_t allocsize; + vm_map_t alloc_map = VM_MAP_NULL; /* * If size is too large for a zone, then use kmem_alloc. - * (We use kmem_alloc instead of kmem_alloc_wired so that + * (We use kmem_alloc instead of kmem_alloc_kobject so that * krealloc can use kmem_realloc.) */ @@ -228,17 +318,42 @@ kalloc_canblock( /* kmem_alloc could block so we return if noblock */ if (!canblock) { - return(0); + return(NULL); } - if (kmem_alloc(kalloc_map, (vm_offset_t *)&addr, size) != KERN_SUCCESS) - addr = 0; - if (addr) { + if (size >= kalloc_kernmap_size) + alloc_map = kernel_map; + else + alloc_map = kalloc_map; + + if (kmem_alloc(alloc_map, (vm_offset_t *)&addr, size) != KERN_SUCCESS) { + if (alloc_map != kernel_map) { + if (kmem_alloc(kernel_map, (vm_offset_t *)&addr, size) != KERN_SUCCESS) + addr = NULL; + } + else + addr = NULL; + } + + if (addr != NULL) { + kalloc_spin_lock(); + /* + * Thread-safe version of the workaround for 4740071 + * (a double FREE()) + */ + if (size > kalloc_largest_allocated) + kalloc_largest_allocated = size; + kalloc_large_inuse++; kalloc_large_total += size; + kalloc_large_sum += size; if (kalloc_large_total > kalloc_large_max) kalloc_large_max = kalloc_large_total; + + kalloc_unlock(); + + KALLOC_ZINFO_SALLOC(size); } return(addr); } @@ -282,6 +397,7 @@ krealloc( register int zindex; register vm_size_t allocsize; void *naddr; + vm_map_t alloc_map = VM_MAP_NULL; /* can only be used for increasing allocation size */ @@ -300,24 +416,28 @@ krealloc( /* if old block was kmem_alloc'd, then use kmem_realloc if necessary */ if (old_size >= kalloc_max_prerounded) { + if (old_size >= kalloc_kernmap_size) + alloc_map = kernel_map; + else + alloc_map = kalloc_map; + old_size = round_page(old_size); new_size = round_page(new_size); if (new_size > old_size) { - if (KERN_SUCCESS != kmem_realloc(kalloc_map, + if (KERN_SUCCESS != kmem_realloc(alloc_map, (vm_offset_t)*addrp, old_size, - (vm_offset_t *)&naddr, new_size)) { + (vm_offset_t *)&naddr, new_size)) panic("krealloc: kmem_realloc"); - naddr = 0; - } simple_lock(lock); *addrp = (void *) naddr; /* kmem_realloc() doesn't free old page range. */ - kmem_free(kalloc_map, (vm_offset_t)*addrp, old_size); + kmem_free(alloc_map, (vm_offset_t)*addrp, old_size); kalloc_large_total += (new_size - old_size); + kalloc_large_sum += (new_size - old_size); if (kalloc_large_total > kalloc_large_max) kalloc_large_max = kalloc_large_total; @@ -345,18 +465,29 @@ krealloc( simple_unlock(lock); if (new_size >= kalloc_max_prerounded) { - if (KERN_SUCCESS != kmem_alloc(kalloc_map, + if (new_size >= kalloc_kernmap_size) + alloc_map = kernel_map; + else + alloc_map = kalloc_map; + if (KERN_SUCCESS != kmem_alloc(alloc_map, (vm_offset_t *)&naddr, new_size)) { panic("krealloc: kmem_alloc"); simple_lock(lock); *addrp = NULL; return; } + kalloc_spin_lock(); + kalloc_large_inuse++; + kalloc_large_sum += new_size; kalloc_large_total += new_size; if (kalloc_large_total > kalloc_large_max) kalloc_large_max = kalloc_large_total; + + kalloc_unlock(); + + KALLOC_ZINFO_SALLOC(new_size); } else { register int new_zindex; @@ -413,6 +544,8 @@ kget( return(zget(k_zone[zindex])); } +volatile SInt32 kfree_nop_count = 0; + void kfree( void *data, @@ -420,15 +553,47 @@ kfree( { register int zindex; register vm_size_t freesize; + vm_map_t alloc_map = kernel_map; /* if size was too large for a zone, then use kmem_free */ if (size >= kalloc_max_prerounded) { - kmem_free(kalloc_map, (vm_offset_t)data, size); + if ((((vm_offset_t) data) >= kalloc_map_min) && (((vm_offset_t) data) <= kalloc_map_max)) + alloc_map = kalloc_map; + if (size > kalloc_largest_allocated) { + /* + * work around double FREEs of small MALLOCs + * this use to end up being a nop + * since the pointer being freed from an + * alloc backed by the zalloc world could + * never show up in the kalloc_map... however, + * the kernel_map is a different issue... since it + * was released back into the zalloc pool, a pointer + * would have gotten written over the 'size' that + * the MALLOC was retaining in the first 4 bytes of + * the underlying allocation... that pointer ends up + * looking like a really big size on the 2nd FREE and + * pushes the kfree into the kernel_map... we + * end up removing a ton of virutal space before we panic + * this check causes us to ignore the kfree for a size + * that must be 'bogus'... note that it might not be due + * to the above scenario, but it would still be wrong and + * cause serious damage. + */ + + OSAddAtomic(1, &kfree_nop_count); + return; + } + kmem_free(alloc_map, (vm_offset_t)data, size); + + kalloc_spin_lock(); kalloc_large_total -= size; kalloc_large_inuse--; + kalloc_unlock(); + + KALLOC_ZINFO_SFREE(size); return; } @@ -471,18 +636,32 @@ kalloc_zone( } #endif +void +kalloc_fake_zone_init(int zone_index) +{ + kalloc_fake_zone_index = zone_index; +} void -kalloc_fake_zone_info(int *count, vm_size_t *cur_size, vm_size_t *max_size, vm_size_t *elem_size, - vm_size_t *alloc_size, int *collectable, int *exhaustable) +kalloc_fake_zone_info(int *count, + vm_size_t *cur_size, vm_size_t *max_size, vm_size_t *elem_size, vm_size_t *alloc_size, + uint64_t *sum_size, int *collectable, int *exhaustable, int *caller_acct) { *count = kalloc_large_inuse; *cur_size = kalloc_large_total; *max_size = kalloc_large_max; - *elem_size = kalloc_large_total / kalloc_large_inuse; - *alloc_size = kalloc_large_total / kalloc_large_inuse; + + if (kalloc_large_inuse) { + *elem_size = kalloc_large_total / kalloc_large_inuse; + *alloc_size = kalloc_large_total / kalloc_large_inuse; + } else { + *elem_size = 0; + *alloc_size = 0; + } + *sum_size = kalloc_large_sum; *collectable = 0; *exhaustable = 0; + *caller_acct = 0; } @@ -491,7 +670,9 @@ OSMalloc_init( void) { queue_init(&OSMalloc_tag_list); - simple_lock_init(&OSMalloc_tag_lock, 0); + + OSMalloc_tag_lck_grp = lck_grp_alloc_init("OSMalloc_tag", LCK_GRP_ATTR_NULL); + lck_mtx_init(&OSMalloc_tag_lock, OSMalloc_tag_lck_grp, LCK_ATTR_NULL); } OSMallocTag @@ -512,9 +693,9 @@ OSMalloc_Tagalloc( strncpy(OSMTag->OSMT_name, str, OSMT_MAX_NAME); - simple_lock(&OSMalloc_tag_lock); + OSMalloc_tag_spin_lock(); enqueue_tail(&OSMalloc_tag_list, (queue_entry_t)OSMTag); - simple_unlock(&OSMalloc_tag_lock); + OSMalloc_tag_unlock(); OSMTag->OSMT_state = OSMT_VALID; return(OSMTag); } @@ -526,7 +707,7 @@ OSMalloc_Tagref( if (!((tag->OSMT_state & OSMT_VALID_MASK) == OSMT_VALID)) panic("OSMalloc_Tagref(): bad state 0x%08X\n",tag->OSMT_state); - (void)hw_atomic_add((uint32_t *)(&tag->OSMT_refcnt), 1); + (void)hw_atomic_add(&tag->OSMT_refcnt, 1); } void @@ -536,11 +717,11 @@ OSMalloc_Tagrele( if (!((tag->OSMT_state & OSMT_VALID_MASK) == OSMT_VALID)) panic("OSMalloc_Tagref(): bad state 0x%08X\n",tag->OSMT_state); - if (hw_atomic_sub((uint32_t *)(&tag->OSMT_refcnt), 1) == 0) { + if (hw_atomic_sub(&tag->OSMT_refcnt, 1) == 0) { if (hw_compare_and_store(OSMT_VALID|OSMT_RELEASED, OSMT_VALID|OSMT_RELEASED, &tag->OSMT_state)) { - simple_lock(&OSMalloc_tag_lock); + OSMalloc_tag_spin_lock(); (void)remque((queue_entry_t)tag); - simple_unlock(&OSMalloc_tag_lock); + OSMalloc_tag_unlock(); kfree((void*)tag, sizeof(*tag)); } else panic("OSMalloc_Tagrele(): refcnt 0\n"); @@ -554,10 +735,10 @@ OSMalloc_Tagfree( if (!hw_compare_and_store(OSMT_VALID, OSMT_VALID|OSMT_RELEASED, &tag->OSMT_state)) panic("OSMalloc_Tagfree(): bad state 0x%08X\n", tag->OSMT_state); - if (hw_atomic_sub((uint32_t *)(&tag->OSMT_refcnt), 1) == 0) { - simple_lock(&OSMalloc_tag_lock); + if (hw_atomic_sub(&tag->OSMT_refcnt, 1) == 0) { + OSMalloc_tag_spin_lock(); (void)remque((queue_entry_t)tag); - simple_unlock(&OSMalloc_tag_lock); + OSMalloc_tag_unlock(); kfree((void*)tag, sizeof(*tag)); } } @@ -575,10 +756,13 @@ OSMalloc( && (size & ~PAGE_MASK)) { if ((kr = kmem_alloc_pageable(kernel_map, (vm_offset_t *)&addr, size)) != KERN_SUCCESS) - panic("OSMalloc(): kmem_alloc_pageable() failed 0x%08X\n", kr); + addr = NULL; } else addr = kalloc((vm_size_t)size); + if (!addr) + OSMalloc_Tagrele(tag); + return(addr); }