vm_size_t kalloc_large_total;
vm_size_t kalloc_large_max;
vm_size_t kalloc_largest_allocated = 0;
+uint64_t kalloc_large_sum;
+
+int kalloc_fake_zone_index = -1; /* index of our fake zone in statistics arrays */
+
+vm_offset_t kalloc_map_min;
+vm_offset_t kalloc_map_max;
+
+#ifdef MUTEX_ZONE
+/*
+ * Diagnostic code to track mutexes separately rather than via the 2^ zones
+ */
+ zone_t lck_mtx_zone;
+#endif
+
+static void
+KALLOC_ZINFO_SALLOC(vm_size_t bytes)
+{
+ thread_t thr = current_thread();
+ task_t task;
+ zinfo_usage_t zinfo;
+
+ thr->tkm_shared.alloc += bytes;
+ if (kalloc_fake_zone_index != -1 &&
+ (task = thr->task) != NULL && (zinfo = task->tkm_zinfo) != NULL)
+ zinfo[kalloc_fake_zone_index].alloc += bytes;
+}
+
+static void
+KALLOC_ZINFO_SFREE(vm_size_t bytes)
+{
+ thread_t thr = current_thread();
+ task_t task;
+ zinfo_usage_t zinfo;
+
+ thr->tkm_shared.free += bytes;
+ if (kalloc_fake_zone_index != -1 &&
+ (task = thr->task) != NULL && (zinfo = task->tkm_zinfo) != NULL)
+ zinfo[kalloc_fake_zone_index].free += bytes;
+}
/*
* All allocations of size less than kalloc_max are rounded to the
boolean_t canblock);
+lck_grp_t *kalloc_lck_grp;
+lck_mtx_t kalloc_lock;
+
+#define kalloc_spin_lock() lck_mtx_lock_spin(&kalloc_lock)
+#define kalloc_unlock() lck_mtx_unlock(&kalloc_lock)
+
+
/* OSMalloc local data declarations */
static
queue_head_t OSMalloc_tag_list;
-decl_simple_lock_data(static,OSMalloc_tag_lock)
+lck_grp_t *OSMalloc_tag_lck_grp;
+lck_mtx_t OSMalloc_tag_lock;
+
+#define OSMalloc_tag_spin_lock() lck_mtx_lock_spin(&OSMalloc_tag_lock)
+#define OSMalloc_tag_unlock() lck_mtx_unlock(&OSMalloc_tag_lock)
+
/* OSMalloc forward declarations */
void OSMalloc_init(void);
/*
* Scale the kalloc_map_size to physical memory size: stay below
- * 1/8th the total zone map size, or 128 MB.
+ * 1/8th the total zone map size, or 128 MB (for a 32-bit kernel).
*/
- kalloc_map_size = sane_size >> 5;
+ kalloc_map_size = (vm_size_t)(sane_size >> 5);
+#if !__LP64__
if (kalloc_map_size > KALLOC_MAP_SIZE_MAX)
kalloc_map_size = KALLOC_MAP_SIZE_MAX;
+#endif /* !__LP64__ */
if (kalloc_map_size < KALLOC_MAP_SIZE_MIN)
kalloc_map_size = KALLOC_MAP_SIZE_MIN;
retval = kmem_suballoc(kernel_map, &min, kalloc_map_size,
- FALSE, VM_FLAGS_ANYWHERE, &kalloc_map);
+ FALSE, VM_FLAGS_ANYWHERE | VM_FLAGS_PERMANENT,
+ &kalloc_map);
if (retval != KERN_SUCCESS)
panic("kalloc_init: kmem_suballoc failed");
+ kalloc_map_min = min;
+ kalloc_map_max = min + kalloc_map_size - 1;
+
/*
* Ensure that zones up to size 8192 bytes exist.
* This is desirable because messages are allocated
kalloc_max_prerounded = kalloc_max / 2 + 1;
/* size it to be more than 16 times kalloc_max (256k) for allocations from kernel map */
kalloc_kernmap_size = (kalloc_max * 16) + 1;
+ kalloc_largest_allocated = kalloc_kernmap_size;
/*
* Allocate a zone for each size we are going to handle.
- * We specify non-paged memory.
+ * We specify non-paged memory. Don't charge the caller
+ * for the allocation, as we aren't sure how the memory
+ * will be handled.
*/
for (i = 0, size = 1; size < kalloc_max; i++, size <<= 1) {
if (size < KALLOC_MINSIZE) {
}
k_zone[i] = zinit(size, k_zone_max[i] * size, size,
k_zone_name[i]);
+ zone_change(k_zone[i], Z_CALLERACCT, FALSE);
}
+ kalloc_lck_grp = lck_grp_alloc_init("kalloc.large", LCK_GRP_ATTR_NULL);
+ lck_mtx_init(&kalloc_lock, kalloc_lck_grp, LCK_ATTR_NULL);
OSMalloc_init();
+#ifdef MUTEX_ZONE
+ lck_mtx_zone = zinit(sizeof(struct _lck_mtx_), 1024*256, 4096, "lck_mtx");
+#endif
+
}
void *
/*
* If size is too large for a zone, then use kmem_alloc.
- * (We use kmem_alloc instead of kmem_alloc_wired so that
+ * (We use kmem_alloc instead of kmem_alloc_kobject so that
* krealloc can use kmem_realloc.)
*/
/* kmem_alloc could block so we return if noblock */
if (!canblock) {
- return(NULL);
+ return(NULL);
}
- if (size >= kalloc_kernmap_size) {
+ if (size >= kalloc_kernmap_size)
alloc_map = kernel_map;
-
- if (size > kalloc_largest_allocated)
- kalloc_largest_allocated = size;
- } else
+ else
alloc_map = kalloc_map;
- if (kmem_alloc(alloc_map, (vm_offset_t *)&addr, size) != KERN_SUCCESS)
- addr = NULL;
+ if (kmem_alloc(alloc_map, (vm_offset_t *)&addr, size) != KERN_SUCCESS) {
+ if (alloc_map != kernel_map) {
+ if (kmem_alloc(kernel_map, (vm_offset_t *)&addr, size) != KERN_SUCCESS)
+ addr = NULL;
+ }
+ else
+ addr = NULL;
+ }
+
+ if (addr != NULL) {
+ kalloc_spin_lock();
+ /*
+ * Thread-safe version of the workaround for 4740071
+ * (a double FREE())
+ */
+ if (size > kalloc_largest_allocated)
+ kalloc_largest_allocated = size;
- if (addr) {
kalloc_large_inuse++;
kalloc_large_total += size;
+ kalloc_large_sum += size;
if (kalloc_large_total > kalloc_large_max)
kalloc_large_max = kalloc_large_total;
+
+ kalloc_unlock();
+
+ KALLOC_ZINFO_SALLOC(size);
}
return(addr);
}
kmem_free(alloc_map, (vm_offset_t)*addrp, old_size);
kalloc_large_total += (new_size - old_size);
+ kalloc_large_sum += (new_size - old_size);
if (kalloc_large_total > kalloc_large_max)
kalloc_large_max = kalloc_large_total;
*addrp = NULL;
return;
}
+ kalloc_spin_lock();
+
kalloc_large_inuse++;
+ kalloc_large_sum += new_size;
kalloc_large_total += new_size;
if (kalloc_large_total > kalloc_large_max)
kalloc_large_max = kalloc_large_total;
+
+ kalloc_unlock();
+
+ KALLOC_ZINFO_SALLOC(new_size);
} else {
register int new_zindex;
return(zget(k_zone[zindex]));
}
+volatile SInt32 kfree_nop_count = 0;
+
void
kfree(
void *data,
{
register int zindex;
register vm_size_t freesize;
- vm_map_t alloc_map = VM_MAP_NULL;
+ vm_map_t alloc_map = kernel_map;
/* if size was too large for a zone, then use kmem_free */
if (size >= kalloc_max_prerounded) {
- if (size >= kalloc_kernmap_size) {
- alloc_map = kernel_map;
-
- if (size > kalloc_largest_allocated)
+ if ((((vm_offset_t) data) >= kalloc_map_min) && (((vm_offset_t) data) <= kalloc_map_max))
+ alloc_map = kalloc_map;
+ if (size > kalloc_largest_allocated) {
/*
* work around double FREEs of small MALLOCs
* this use to end up being a nop
* to the above scenario, but it would still be wrong and
* cause serious damage.
*/
+
+ OSAddAtomic(1, &kfree_nop_count);
return;
- } else
- alloc_map = kalloc_map;
+ }
kmem_free(alloc_map, (vm_offset_t)data, size);
+ kalloc_spin_lock();
+
kalloc_large_total -= size;
kalloc_large_inuse--;
+ kalloc_unlock();
+
+ KALLOC_ZINFO_SFREE(size);
return;
}
}
#endif
+void
+kalloc_fake_zone_init(int zone_index)
+{
+ kalloc_fake_zone_index = zone_index;
+}
void
-kalloc_fake_zone_info(int *count, vm_size_t *cur_size, vm_size_t *max_size, vm_size_t *elem_size,
- vm_size_t *alloc_size, int *collectable, int *exhaustable)
+kalloc_fake_zone_info(int *count,
+ vm_size_t *cur_size, vm_size_t *max_size, vm_size_t *elem_size, vm_size_t *alloc_size,
+ uint64_t *sum_size, int *collectable, int *exhaustable, int *caller_acct)
{
*count = kalloc_large_inuse;
*cur_size = kalloc_large_total;
*max_size = kalloc_large_max;
- *elem_size = kalloc_large_total / kalloc_large_inuse;
- *alloc_size = kalloc_large_total / kalloc_large_inuse;
+
+ if (kalloc_large_inuse) {
+ *elem_size = kalloc_large_total / kalloc_large_inuse;
+ *alloc_size = kalloc_large_total / kalloc_large_inuse;
+ } else {
+ *elem_size = 0;
+ *alloc_size = 0;
+ }
+ *sum_size = kalloc_large_sum;
*collectable = 0;
*exhaustable = 0;
+ *caller_acct = 0;
}
void)
{
queue_init(&OSMalloc_tag_list);
- simple_lock_init(&OSMalloc_tag_lock, 0);
+
+ OSMalloc_tag_lck_grp = lck_grp_alloc_init("OSMalloc_tag", LCK_GRP_ATTR_NULL);
+ lck_mtx_init(&OSMalloc_tag_lock, OSMalloc_tag_lck_grp, LCK_ATTR_NULL);
}
OSMallocTag
strncpy(OSMTag->OSMT_name, str, OSMT_MAX_NAME);
- simple_lock(&OSMalloc_tag_lock);
+ OSMalloc_tag_spin_lock();
enqueue_tail(&OSMalloc_tag_list, (queue_entry_t)OSMTag);
- simple_unlock(&OSMalloc_tag_lock);
+ OSMalloc_tag_unlock();
OSMTag->OSMT_state = OSMT_VALID;
return(OSMTag);
}
if (hw_atomic_sub(&tag->OSMT_refcnt, 1) == 0) {
if (hw_compare_and_store(OSMT_VALID|OSMT_RELEASED, OSMT_VALID|OSMT_RELEASED, &tag->OSMT_state)) {
- simple_lock(&OSMalloc_tag_lock);
+ OSMalloc_tag_spin_lock();
(void)remque((queue_entry_t)tag);
- simple_unlock(&OSMalloc_tag_lock);
+ OSMalloc_tag_unlock();
kfree((void*)tag, sizeof(*tag));
} else
panic("OSMalloc_Tagrele(): refcnt 0\n");
panic("OSMalloc_Tagfree(): bad state 0x%08X\n", tag->OSMT_state);
if (hw_atomic_sub(&tag->OSMT_refcnt, 1) == 0) {
- simple_lock(&OSMalloc_tag_lock);
+ OSMalloc_tag_spin_lock();
(void)remque((queue_entry_t)tag);
- simple_unlock(&OSMalloc_tag_lock);
+ OSMalloc_tag_unlock();
kfree((void*)tag, sizeof(*tag));
}
}
projects / apple / xnu.git / blobdiff