X-Git-Url: https://git.saurik.com/apple/libc.git/blobdiff_plain/734aad71947a79037af64f74c683f5eb36fe6065..51282358e8fdbfc483c0c34e7eae9b89b51f2570:/gen/malloc.c diff --git a/gen/malloc.c b/gen/malloc.c index 536aa8a..e89051b 100644 --- a/gen/malloc.c +++ b/gen/malloc.c @@ -1,10 +1,8 @@ /* - * Copyright (c) 1999 Apple Computer, Inc. All rights reserved. + * Copyright (c) 1999, 2006-2008 Apple Inc. All rights reserved. * * @APPLE_LICENSE_HEADER_START@ * - * Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved. - * * This file contains Original Code and/or Modifications of Original Code * as defined in and that are subject to the Apple Public Source License * Version 2.0 (the 'License'). You may not use this file except in @@ -23,32 +21,56 @@ * @APPLE_LICENSE_HEADER_END@ */ -#define __POSIX_LIB__ +#include +#include "magmallocProvider.h" + #import #import #import #import -#import -#import // for spin lock -#import -#include +#import +#import +#import +#import +#import +#import +#import +#import +#import +#import #import "scalable_malloc.h" #import "stack_logging.h" +#import "malloc_printf.h" +#import "_simple.h" -#define USE_SLEEP_RATHER_THAN_ABORT 0 +/* + * MALLOC_ABSOLUTE_MAX_SIZE - There are many instances of addition to a + * user-specified size_t, which can cause overflow (and subsequent crashes) + * for values near SIZE_T_MAX. Rather than add extra "if" checks everywhere + * this occurs, it is easier to just set an absolute maximum request size, + * and immediately return an error if the requested size exceeds this maximum. + * Of course, values less than this absolute max can fail later if the value + * is still too large for the available memory. The largest value added + * seems to be PAGE_SIZE (in the macro round_page()), so to be safe, we set + * the maximum to be 2 * PAGE_SIZE less than SIZE_T_MAX. + */ +#define MALLOC_ABSOLUTE_MAX_SIZE (SIZE_T_MAX - (2 * PAGE_SIZE)) -#define MAX_ALLOCATION 0xc0000000 // beyond this, assume a programming error -#define INITIAL_ZONES 8 // After this number, we reallocate for new zones +#define USE_SLEEP_RATHER_THAN_ABORT 0 -typedef void (malloc_logger_t)(unsigned type, unsigned arg1, unsigned arg2, unsigned arg3, unsigned result, unsigned num_hot_frames_to_skip); +typedef void (malloc_logger_t)(uint32_t type, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3, uintptr_t result, uint32_t num_hot_frames_to_skip); -static pthread_lock_t _malloc_lock; -static malloc_zone_t *initial_malloc_zones[INITIAL_ZONES] = {0}; +__private_extern__ pthread_lock_t _malloc_lock = 0; // initialized in __libc_init -/* The following variables are exported for the benefit of performance tools */ +/* The following variables are exported for the benefit of performance tools + * + * It should always be safe to first read malloc_num_zones, then read + * malloc_zones without taking the lock, if only iteration is required + */ unsigned malloc_num_zones = 0; -malloc_zone_t **malloc_zones = initial_malloc_zones; +unsigned malloc_num_zones_allocated = 0; +malloc_zone_t **malloc_zones = 0; malloc_logger_t *malloc_logger = NULL; unsigned malloc_debug_flags = 0; @@ -57,6 +79,25 @@ unsigned malloc_check_start = 0; // 0 means don't check unsigned malloc_check_counter = 0; unsigned malloc_check_each = 1000; +/* global flag to suppress ASL logging e.g. for syslogd */ +int _malloc_no_asl_log = 0; + +static int malloc_check_sleep = 100; // default 100 second sleep +static int malloc_check_abort = 0; // default is to sleep, not abort + +static int malloc_debug_file = STDERR_FILENO; +/* + * State indicated by malloc_def_zone_state + * 0 - the default zone has not yet been created + * 1 - a Malloc* environment variable has been set + * 2 - the default zone has been created and an environment variable scan done + * 3 - a new default zone has been created and another environment variable scan + */ +__private_extern__ int malloc_def_zone_state = 0; +__private_extern__ malloc_zone_t *__zone0 = NULL; + +static const char Malloc_Facility[] = "com.apple.Libsystem.malloc"; + #define MALLOC_LOCK() LOCK(_malloc_lock) #define MALLOC_UNLOCK() UNLOCK(_malloc_lock) @@ -67,82 +108,291 @@ unsigned malloc_check_each = 1000; /********* Utilities ************/ -static inline malloc_zone_t *find_registered_zone(const void *ptr, size_t *returned_size) { - // locates the proper zone - // if zone found fills returnedSize; else returns NULL - // See comment in malloc_zone_register() about clients non locking to call this function - // Speed is critical for this function - unsigned index = malloc_num_zones; +static inline malloc_zone_t * find_registered_zone(const void *, size_t *) __attribute__((always_inline)); +static inline malloc_zone_t * +find_registered_zone(const void *ptr, size_t *returned_size) { + // Returns a zone which contains ptr, else NULL + unsigned index; malloc_zone_t **zones = malloc_zones; - while (index--) { - malloc_zone_t *zone = *zones++; - size_t size; - size = zone->size(zone, ptr); - if (size) { + + for (index = 0; index < malloc_num_zones; ++index, ++zones) { + malloc_zone_t *zone = *zones; + size_t size = zone->size(zone, ptr); + if (size) { // Claimed by this zone? if (returned_size) *returned_size = size; return zone; } } + // Unclaimed by any zone. + if (returned_size) *returned_size = 0; return NULL; } +__private_extern__ __attribute__((noinline)) void +malloc_error_break(void) { + // Provides a non-inlined place for various malloc error procedures to call + // that will be called after an error message appears. It does not make + // sense for developers to call this function, so it is marked + // __private_extern__ to prevent it from becoming API. + MAGMALLOC_MALLOCERRORBREAK(); // DTrace USDT probe +} + +__private_extern__ boolean_t __stack_logging_locked(); + +__private_extern__ __attribute__((noinline)) int +malloc_gdb_po_unsafe(void) { + // In order to implement "po" other data formatters in gdb, the debugger + // calls functions that call malloc. The debugger will only run one thread + // of the program in this case, so if another thread is holding a zone lock, + // gdb may deadlock in this case. + // + // Iterate over the zones in malloc_zones, and call "trylock" on the zone + // lock. If trylock succeeds, unlock it, otherwise return "locked". Returns + // 0 == safe, 1 == locked/unsafe. + + if (__stack_logging_locked()) + return 1; + + malloc_zone_t **zones = malloc_zones; + unsigned i, e = malloc_num_zones; + + for (i = 0; i != e; ++i) { + malloc_zone_t *zone = zones[i]; + + // Version must be >= 5 to look at the new introspection field. + if (zone->version < 5) + continue; + + if (zone->introspect->zone_locked && zone->introspect->zone_locked(zone)) + return 1; + } + return 0; +} + /********* Creation and destruction ************/ -static void _malloc_initialize(void) { - // guaranteed to be called only once - (void)malloc_create_zone(0, 0); - malloc_set_zone_name(malloc_zones[0], "DefaultMallocZone"); - LOCK_INIT(_malloc_lock); - // malloc_printf("Malloc: %d registered zones\n", malloc_num_zones); - // malloc_printf("malloc: malloc_zones is at 0x%x; malloc_num_zones is at 0x%x\n", (unsigned)&malloc_zones, (unsigned)&malloc_num_zones); +static void set_flags_from_environment(void); + +static void +malloc_zone_register_while_locked(malloc_zone_t *zone) { + size_t protect_size; + unsigned i; + + /* scan the list of zones, to see if this zone is already registered. If + * so, print an error message and return. */ + for (i = 0; i != malloc_num_zones; ++i) + if (zone == malloc_zones[i]) { + _malloc_printf(ASL_LEVEL_ERR, "Attempted to register zone more than once: %p\n", zone); + return; + } + + if (malloc_num_zones == malloc_num_zones_allocated) { + size_t malloc_zones_size = malloc_num_zones * sizeof(malloc_zone_t *); + size_t alloc_size = malloc_zones_size + vm_page_size; + + malloc_zone_t **new_zones = mmap(0, alloc_size, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, VM_MAKE_TAG(VM_MEMORY_MALLOC), 0); + + /* If there were previously allocated malloc zones, we need to copy them + * out of the previous array and into the new zones array */ + if (malloc_zones) + memcpy(new_zones, malloc_zones, malloc_zones_size); + + /* Update the malloc_zones pointer, which we leak if it was previously + * allocated, and the number of zones allocated */ + protect_size = alloc_size; + malloc_zones = new_zones; + malloc_num_zones_allocated = alloc_size / sizeof(malloc_zone_t *); + } else { + /* If we don't need to reallocate zones, we need to briefly change the + * page protection the malloc zones to allow writes */ + protect_size = malloc_num_zones_allocated * sizeof(malloc_zone_t *); + vm_protect(mach_task_self(), (uintptr_t)malloc_zones, protect_size, 0, VM_PROT_READ | VM_PROT_WRITE); + } + malloc_zones[malloc_num_zones++] = zone; + + /* Finally, now that the zone is registered, disallow write access to the + * malloc_zones array */ + vm_protect(mach_task_self(), (uintptr_t)malloc_zones, protect_size, 0, VM_PROT_READ); + //_malloc_printf(ASL_LEVEL_INFO, "Registered malloc_zone %p in malloc_zones %p [%u zones, %u bytes]\n", zone, malloc_zones, malloc_num_zones, protect_size); +} + +static void +_malloc_initialize(void) { + MALLOC_LOCK(); + if (malloc_def_zone_state < 2) { + unsigned n; + malloc_zone_t *zone; + + malloc_def_zone_state += 2; + set_flags_from_environment(); // will only set flags up to two times + n = malloc_num_zones; + zone = create_scalable_zone(0, malloc_debug_flags); + malloc_zone_register_while_locked(zone); + malloc_set_zone_name(zone, "DefaultMallocZone"); + if (n != 0) { // make the default first, for efficiency + unsigned protect_size = malloc_num_zones_allocated * sizeof(malloc_zone_t *); + malloc_zone_t *hold = malloc_zones[0]; + if(hold->zone_name && strcmp(hold->zone_name, "DefaultMallocZone") == 0) { + free((void *)hold->zone_name); + hold->zone_name = NULL; + } + vm_protect(mach_task_self(), (uintptr_t)malloc_zones, protect_size, 0, VM_PROT_READ | VM_PROT_WRITE); + malloc_zones[0] = malloc_zones[n]; + malloc_zones[n] = hold; + vm_protect(mach_task_self(), (uintptr_t)malloc_zones, protect_size, 0, VM_PROT_READ); + } + // _malloc_printf(ASL_LEVEL_INFO, "%d registered zones\n", malloc_num_zones); + // _malloc_printf(ASL_LEVEL_INFO, "malloc_zones is at %p; malloc_num_zones is at %p\n", (unsigned)&malloc_zones, (unsigned)&malloc_num_zones); + } + MALLOC_UNLOCK(); } -static inline malloc_zone_t *inline_malloc_default_zone(void) { - if (!malloc_num_zones) _malloc_initialize(); - // malloc_printf("In inline_malloc_default_zone with %d %d\n", malloc_num_zones, malloc_has_debug_zone); +static inline malloc_zone_t *inline_malloc_default_zone(void) __attribute__((always_inline)); +static inline malloc_zone_t * +inline_malloc_default_zone(void) { + if (malloc_def_zone_state < 2) _malloc_initialize(); + // _malloc_printf(ASL_LEVEL_INFO, "In inline_malloc_default_zone with %d %d\n", malloc_num_zones, malloc_has_debug_zone); return malloc_zones[0]; } -malloc_zone_t *malloc_default_zone(void) { +malloc_zone_t * +malloc_default_zone(void) { return inline_malloc_default_zone(); } -static void set_flags_from_environment(void) { - const char *flag; +malloc_zone_t * +malloc_default_purgeable_zone(void) { + static malloc_zone_t *dpz; + + if (!dpz) { + malloc_zone_t *tmp = create_purgeable_zone(0, malloc_default_zone(), malloc_debug_flags); + malloc_zone_register(tmp); + malloc_set_zone_name(tmp, "DefaultPurgeableMallocZone"); + if (!__sync_bool_compare_and_swap(&dpz, NULL, tmp)) + malloc_destroy_zone(tmp); + } + return dpz; +} + +// For debugging, allow stack logging to both memory and disk to compare their results. +static void +stack_logging_log_stack_debug(uint32_t type_flags, uintptr_t zone_ptr, uintptr_t size, uintptr_t ptr_arg, uintptr_t return_val, uint32_t num_hot_to_skip) +{ + __disk_stack_logging_log_stack(type_flags, zone_ptr, size, ptr_arg, return_val, num_hot_to_skip); + stack_logging_log_stack(type_flags, zone_ptr, size, ptr_arg, return_val, num_hot_to_skip); +} + +static void +set_flags_from_environment(void) { + const char *flag; + int fd; + char **env = * _NSGetEnviron(); + char **p; + char *c; + + if (malloc_debug_file != STDERR_FILENO) { + close(malloc_debug_file); + malloc_debug_file = STDERR_FILENO; + } +#if __LP64__ + malloc_debug_flags = SCALABLE_MALLOC_ABORT_ON_CORRUPTION; // Set always on 64-bit processes +#else + malloc_debug_flags = 0; +#endif + stack_logging_enable_logging = 0; + stack_logging_dontcompact = 0; + malloc_logger = NULL; + malloc_check_start = 0; + malloc_check_each = 1000; + malloc_check_abort = 0; + malloc_check_sleep = 100; + /* + * Given that all environment variables start with "Malloc" we optimize by scanning quickly + * first the environment, therefore avoiding repeated calls to getenv(). + * If we are setu/gid these flags are ignored to prevent a malicious invoker from changing + * our behaviour. + */ + for (p = env; (c = *p) != NULL; ++p) { + if (!strncmp(c, "Malloc", 6)) { + if (issetugid()) + return; + break; + } + } + if (c == NULL) + return; + flag = getenv("MallocLogFile"); + if (flag) { + fd = open(flag, O_WRONLY|O_APPEND|O_CREAT, 0644); + if (fd >= 0) { + malloc_debug_file = fd; + fcntl(fd, F_SETFD, 0); // clear close-on-exec flag XXX why? + } else { + malloc_printf("Could not open %s, using stderr\n", flag); + } + } if (getenv("MallocGuardEdges")) { malloc_debug_flags = SCALABLE_MALLOC_ADD_GUARD_PAGES; - malloc_printf("malloc[%d]: protecting edges\n", getpid()); + _malloc_printf(ASL_LEVEL_INFO, "protecting edges\n"); if (getenv("MallocDoNotProtectPrelude")) { malloc_debug_flags |= SCALABLE_MALLOC_DONT_PROTECT_PRELUDE; - malloc_printf("malloc[%d]: ... but not protecting prelude guard page\n", getpid()); + _malloc_printf(ASL_LEVEL_INFO, "... but not protecting prelude guard page\n"); } if (getenv("MallocDoNotProtectPostlude")) { malloc_debug_flags |= SCALABLE_MALLOC_DONT_PROTECT_POSTLUDE; - malloc_printf("malloc[%d]: ... but not protecting postlude guard page\n", getpid()); + _malloc_printf(ASL_LEVEL_INFO, "... but not protecting postlude guard page\n"); } } flag = getenv("MallocStackLogging"); if (!flag) { flag = getenv("MallocStackLoggingNoCompact"); stack_logging_dontcompact = 1; - } + } + // For debugging, the MallocStackLogging or MallocStackLoggingNoCompact environment variables can be set to + // values of "memory", "disk", or "both" to control which stack logging mechanism to use. Those strings appear + // in the flag variable, and the strtoul() call below will return 0, so then we can do string comparison on the + // value of flag. The default stack logging now is disk stack logging, since memory stack logging is not 64-bit-aware. if (flag) { - unsigned val = strtoul(flag, NULL, 0); + unsigned long val = strtoul(flag, NULL, 0); if (val == 1) val = 0; if (val == -1) val = 0; - malloc_logger = (val) ? (void *)val : stack_logging_log_stack; + if (val) { + malloc_logger = (void *)val; + _malloc_printf(ASL_LEVEL_INFO, "recording stacks using recorder %p\n", malloc_logger); + } else if (strcmp(flag,"memory") == 0) { + malloc_logger = (malloc_logger_t *)stack_logging_log_stack; + _malloc_printf(ASL_LEVEL_INFO, "recording malloc stacks in memory using standard recorder\n"); + } else if (strcmp(flag,"both") == 0) { + malloc_logger = stack_logging_log_stack_debug; + _malloc_printf(ASL_LEVEL_INFO, "recording malloc stacks to both memory and disk for comparison debugging\n"); + } else { // the default is to log to disk + malloc_logger = __disk_stack_logging_log_stack; + _malloc_printf(ASL_LEVEL_INFO, "recording malloc stacks to disk using standard recorder\n"); + } stack_logging_enable_logging = 1; - if (malloc_logger == stack_logging_log_stack) { - malloc_printf("malloc[%d]: recording stacks using standard recorder\n", getpid()); - } else { - malloc_printf("malloc[%d]: recording stacks using recorder %p\n", getpid(), malloc_logger); + if (stack_logging_dontcompact) { + if (malloc_logger == __disk_stack_logging_log_stack) { + _malloc_printf(ASL_LEVEL_INFO, "stack logging compaction turned off; size of log files on disk can increase rapidly\n"); + } else { + _malloc_printf(ASL_LEVEL_INFO, "stack logging compaction turned off; VM can increase rapidly\n"); + } } - if (stack_logging_dontcompact) malloc_printf("malloc[%d]: stack logging compaction turned off; VM can increase rapidly\n", getpid()); } if (getenv("MallocScribble")) { - malloc_debug_flags |= SCALABLE_MALLOC_DO_SCRIBBLE; - malloc_printf("malloc[%d]: enabling scribbling to detect mods to free blocks\n", getpid()); + malloc_debug_flags |= SCALABLE_MALLOC_DO_SCRIBBLE; + _malloc_printf(ASL_LEVEL_INFO, "enabling scribbling to detect mods to free blocks\n"); + } + if (getenv("MallocErrorAbort")) { + malloc_debug_flags |= SCALABLE_MALLOC_ABORT_ON_ERROR; + _malloc_printf(ASL_LEVEL_INFO, "enabling abort() on bad malloc or free\n"); + } +#if __LP64__ + /* initialization above forces SCALABLE_MALLOC_ABORT_ON_CORRUPTION of 64-bit processes */ +#else + if (getenv("MallocCorruptionAbort")) { // Set from an environment variable in 32-bit processes + malloc_debug_flags |= SCALABLE_MALLOC_ABORT_ON_CORRUPTION; } +#endif flag = getenv("MallocCheckHeapStart"); if (flag) { malloc_check_start = strtoul(flag, NULL, 0); @@ -154,184 +404,322 @@ static void set_flags_from_environment(void) { if (malloc_check_each == 0) malloc_check_each = 1; if (malloc_check_each == -1) malloc_check_each = 1; } - malloc_printf("malloc[%d]: checks heap after %dth operation and each %d operations\n", getpid(), malloc_check_start, malloc_check_each); + _malloc_printf(ASL_LEVEL_INFO, "checks heap after %dth operation and each %d operations\n", malloc_check_start, malloc_check_each); + flag = getenv("MallocCheckHeapAbort"); + if (flag) + malloc_check_abort = strtol(flag, NULL, 0); + if (malloc_check_abort) + _malloc_printf(ASL_LEVEL_INFO, "will abort on heap corruption\n"); + else { + flag = getenv("MallocCheckHeapSleep"); + if (flag) + malloc_check_sleep = strtol(flag, NULL, 0); + if (malloc_check_sleep > 0) + _malloc_printf(ASL_LEVEL_INFO, "will sleep for %d seconds on heap corruption\n", malloc_check_sleep); + else if (malloc_check_sleep < 0) + _malloc_printf(ASL_LEVEL_INFO, "will sleep once for %d seconds on heap corruption\n", -malloc_check_sleep); + else + _malloc_printf(ASL_LEVEL_INFO, "no sleep on heap corruption\n"); + } } if (getenv("MallocHelp")) { - malloc_printf( - "malloc[%d]: environment variables that can be set for debug:\n" + _malloc_printf(ASL_LEVEL_INFO, + "environment variables that can be set for debug:\n" + "- MallocLogFile to create/append messages to file instead of stderr\n" "- MallocGuardEdges to add 2 guard pages for each large block\n" "- MallocDoNotProtectPrelude to disable protection (when previous flag set)\n" "- MallocDoNotProtectPostlude to disable protection (when previous flag set)\n" "- MallocStackLogging to record all stacks. Tools like leaks can then be applied\n" "- MallocStackLoggingNoCompact to record all stacks. Needed for malloc_history\n" - "- MallocScribble to detect writing on free blocks: 0x55 is written upon free\n" - "- MallocCheckHeapStart to check the heap from time to time after operations \n" - "- MallocHelp - this help!\n", getpid()); + "- MallocStackLoggingDirectory to set location of stack logs, which can grow large; default is /tmp\n" + "- MallocScribble to detect writing on free blocks and missing initializers:\n" + " 0x55 is written upon free and 0xaa is written on allocation\n" + "- MallocCheckHeapStart to start checking the heap after operations\n" + "- MallocCheckHeapEach to repeat the checking of the heap after operations\n" + "- MallocCheckHeapSleep to sleep seconds on heap corruption\n" + "- MallocCheckHeapAbort to abort on heap corruption if is non-zero\n" + "- MallocCorruptionAbort to abort on malloc errors, but not on out of memory for 32-bit processes\n" + " MallocCorruptionAbort is always set on 64-bit processes\n" + "- MallocErrorAbort to abort on any malloc error, including out of memory\n" + "- MallocHelp - this help!\n"); } } -malloc_zone_t *malloc_create_zone(vm_size_t start_size, unsigned flags) { +malloc_zone_t * +malloc_create_zone(vm_size_t start_size, unsigned flags) +{ malloc_zone_t *zone; - if (!malloc_num_zones) { - char **env = * _NSGetEnviron(); - char **p; - char *c; - /* Given that all environment variables start with "Malloc" we optimize by scanning quickly first the environment, therefore avoiding repeated calls to getenv() */ - for (p = env; (c = *p) != NULL; ++p) { - if (!strncmp(c, "Malloc", 6)) { - set_flags_from_environment(); - break; - } - } + /* start_size doesn't seemed to actually be used, but we test anyways */ + if (start_size > MALLOC_ABSOLUTE_MAX_SIZE) { + return NULL; } - zone = create_scalable_zone(start_size, malloc_debug_flags); + if (malloc_def_zone_state < 2) _malloc_initialize(); + zone = create_scalable_zone(start_size, flags | malloc_debug_flags); malloc_zone_register(zone); return zone; } -void malloc_destroy_zone(malloc_zone_t *zone) { +/* + * For use by CheckFix: establish a new default zone whose behavior is, apart from + * the use of death-row and per-CPU magazines, that of Leopard. + */ +void +malloc_create_legacy_default_zone(void) +{ + malloc_zone_t *zone; + int i; + + if (malloc_def_zone_state < 2) _malloc_initialize(); + zone = create_legacy_scalable_zone(0, malloc_debug_flags); + + MALLOC_LOCK(); + malloc_zone_register_while_locked(zone); + + // + // Establish the legacy scalable zone just created as the default zone. + // + malloc_zone_t *hold = malloc_zones[0]; + if(hold->zone_name && strcmp(hold->zone_name, "DefaultMallocZone") == 0) { + free((void *)hold->zone_name); + hold->zone_name = NULL; + } + malloc_set_zone_name(zone, "DefaultMallocZone"); + + unsigned protect_size = malloc_num_zones_allocated * sizeof(malloc_zone_t *); + vm_protect(mach_task_self(), (uintptr_t)malloc_zones, protect_size, 0, VM_PROT_READ | VM_PROT_WRITE); + + // assert(zone == malloc_zones[malloc_num_zones - 1]; + for (i = malloc_num_zones - 1; i > 0; --i) { + malloc_zones[i] = malloc_zones[i - 1]; + } + malloc_zones[0] = zone; + + vm_protect(mach_task_self(), (uintptr_t)malloc_zones, protect_size, 0, VM_PROT_READ); + MALLOC_UNLOCK(); +} + +void +malloc_destroy_zone(malloc_zone_t *zone) { malloc_zone_unregister(zone); zone->destroy(zone); } +/* called from the {put,set,unset}env routine */ +__private_extern__ void +__malloc_check_env_name(const char *name) +{ + MALLOC_LOCK(); + if(malloc_def_zone_state == 2 && strncmp(name, "Malloc", 6) == 0) + malloc_def_zone_state = 1; + MALLOC_UNLOCK(); +} + /********* Block creation and manipulation ************/ -static void internal_check(void) { +extern const char *__crashreporter_info__; + +static void +internal_check(void) { static vm_address_t *frames = NULL; static unsigned num_frames; if (malloc_zone_check(NULL)) { - malloc_printf("MallocCheckHeap: PASSED check at %dth operation\n", malloc_check_counter-1); - if (!frames) vm_allocate(mach_task_self(), (void *)&frames, vm_page_size, 1); - thread_stack_pcs(frames, vm_page_size/sizeof(vm_address_t) - 1, &num_frames); + if (!frames) vm_allocate(mach_task_self(), (void *)&frames, vm_page_size, 1); + thread_stack_pcs(frames, vm_page_size/sizeof(vm_address_t) - 1, &num_frames); } else { + _SIMPLE_STRING b = _simple_salloc(); + if (b) + _simple_sprintf(b, "*** MallocCheckHeap: FAILED check at %dth operation\n", malloc_check_counter-1); + else + _malloc_printf(MALLOC_PRINTF_NOLOG, "*** MallocCheckHeap: FAILED check at %dth operation\n", malloc_check_counter-1); malloc_printf("*** MallocCheckHeap: FAILED check at %dth operation\n", malloc_check_counter-1); if (frames) { unsigned index = 1; - malloc_printf("Stack for last operation where the malloc check succeeded: "); - while (index < num_frames) malloc_printf("0x%x ", frames[index++]); - malloc_printf("\n(Use 'atos' for a symbolic stack)\n"); + if (b) { + _simple_sappend(b, "Stack for last operation where the malloc check succeeded: "); + while (index < num_frames) _simple_sprintf(b, "%p ", frames[index++]); + malloc_printf("%s\n(Use 'atos' for a symbolic stack)\n", _simple_string(b)); + } else { + /* + * Should only get here if vm_allocate() can't get a single page of + * memory, implying _simple_asl_log() would also fail. So we just + * print to the file descriptor. + */ + _malloc_printf(MALLOC_PRINTF_NOLOG, "Stack for last operation where the malloc check succeeded: "); + while (index < num_frames) _malloc_printf(MALLOC_PRINTF_NOLOG, "%p ", frames[index++]); + _malloc_printf(MALLOC_PRINTF_NOLOG, "\n(Use 'atos' for a symbolic stack)\n"); + } } if (malloc_check_each > 1) { unsigned recomm_each = (malloc_check_each > 10) ? malloc_check_each/10 : 1; unsigned recomm_start = (malloc_check_counter > malloc_check_each+1) ? malloc_check_counter-1-malloc_check_each : 1; malloc_printf("*** Recommend using 'setenv MallocCheckHeapStart %d; setenv MallocCheckHeapEach %d' to narrow down failure\n", recomm_start, recomm_each); } - malloc_printf("*** Sleeping for 100 seconds to leave time to attach\n"); - sleep(100); + if (malloc_check_abort) { + __crashreporter_info__ = b ? _simple_string(b) : "*** MallocCheckHeap: FAILED check"; + abort(); + } else if (b) + _simple_sfree(b); + if (malloc_check_sleep > 0) { + _malloc_printf(ASL_LEVEL_NOTICE, "*** Sleeping for %d seconds to leave time to attach\n", + malloc_check_sleep); + sleep(malloc_check_sleep); + } else if (malloc_check_sleep < 0) { + _malloc_printf(ASL_LEVEL_NOTICE, "*** Sleeping once for %d seconds to leave time to attach\n", + -malloc_check_sleep); + sleep(-malloc_check_sleep); + malloc_check_sleep = 0; + } } malloc_check_start += malloc_check_each; } -void *malloc_zone_malloc(malloc_zone_t *zone, size_t size) { +void * +malloc_zone_malloc(malloc_zone_t *zone, size_t size) { void *ptr; - if ((unsigned)size >= MAX_ALLOCATION) { - /* Probably a programming error */ - fprintf(stderr, "*** malloc_zone_malloc[%d]: argument too large: %d\n", getpid(), (unsigned)size); - return NULL; - } if (malloc_check_start && (malloc_check_counter++ >= malloc_check_start)) { internal_check(); } + if (size > MALLOC_ABSOLUTE_MAX_SIZE) { + return NULL; + } ptr = zone->malloc(zone, size); - if (malloc_logger) malloc_logger(MALLOC_LOG_TYPE_ALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE, (unsigned)zone, size, 0, (unsigned)ptr, 0); + if (malloc_logger) malloc_logger(MALLOC_LOG_TYPE_ALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE, (uintptr_t)zone, (uintptr_t)size, 0, (uintptr_t)ptr, 0); return ptr; } -void *malloc_zone_calloc(malloc_zone_t *zone, size_t num_items, size_t size) { +void * +malloc_zone_calloc(malloc_zone_t *zone, size_t num_items, size_t size) { void *ptr; if (malloc_check_start && (malloc_check_counter++ >= malloc_check_start)) { internal_check(); } - if (((unsigned)num_items >= MAX_ALLOCATION) || ((unsigned)size >= MAX_ALLOCATION) || ((long long)size * num_items >= (long long) MAX_ALLOCATION)) { - /* Probably a programming error */ - fprintf(stderr, "*** malloc_zone_calloc[%d]: arguments too large: %d,%d\n", getpid(), (unsigned)num_items, (unsigned)size); - return NULL; + if (size > MALLOC_ABSOLUTE_MAX_SIZE) { + return NULL; } ptr = zone->calloc(zone, num_items, size); - if (malloc_logger) malloc_logger(MALLOC_LOG_TYPE_ALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE | MALLOC_LOG_TYPE_CLEARED, (unsigned)zone, num_items * size, 0, (unsigned)ptr, 0); + if (malloc_logger) malloc_logger(MALLOC_LOG_TYPE_ALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE | MALLOC_LOG_TYPE_CLEARED, (uintptr_t)zone, (uintptr_t)(num_items * size), 0, (uintptr_t)ptr, 0); return ptr; } -void *malloc_zone_valloc(malloc_zone_t *zone, size_t size) { +void * +malloc_zone_valloc(malloc_zone_t *zone, size_t size) { void *ptr; - if ((unsigned)size >= MAX_ALLOCATION) { - /* Probably a programming error */ - fprintf(stderr, "*** malloc_zone_valloc[%d]: argument too large: %d\n", getpid(), (unsigned)size); - return NULL; - } if (malloc_check_start && (malloc_check_counter++ >= malloc_check_start)) { internal_check(); } + if (size > MALLOC_ABSOLUTE_MAX_SIZE) { + return NULL; + } ptr = zone->valloc(zone, size); - if (malloc_logger) malloc_logger(MALLOC_LOG_TYPE_ALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE, (unsigned)zone, size, 0, (unsigned)ptr, 0); + if (malloc_logger) malloc_logger(MALLOC_LOG_TYPE_ALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE, (uintptr_t)zone, (uintptr_t)size, 0, (uintptr_t)ptr, 0); return ptr; } -void *malloc_zone_realloc(malloc_zone_t *zone, void *ptr, size_t size) { +void * +malloc_zone_realloc(malloc_zone_t *zone, void *ptr, size_t size) { void *new_ptr; if (malloc_check_start && (malloc_check_counter++ >= malloc_check_start)) { internal_check(); } + if (size > MALLOC_ABSOLUTE_MAX_SIZE) { + return NULL; + } new_ptr = zone->realloc(zone, ptr, size); - if (malloc_logger) malloc_logger(MALLOC_LOG_TYPE_ALLOCATE | MALLOC_LOG_TYPE_DEALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE, (unsigned)zone, (unsigned)ptr, size, (unsigned)new_ptr, 0); + if (malloc_logger) malloc_logger(MALLOC_LOG_TYPE_ALLOCATE | MALLOC_LOG_TYPE_DEALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE, (uintptr_t)zone, (uintptr_t)ptr, (uintptr_t)size, (uintptr_t)new_ptr, 0); return new_ptr; } -void malloc_zone_free(malloc_zone_t *zone, void *ptr) { - if (malloc_logger) malloc_logger(MALLOC_LOG_TYPE_DEALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE, (unsigned)zone, (unsigned)ptr, 0, 0, 0); +void +malloc_zone_free(malloc_zone_t *zone, void *ptr) { + if (malloc_logger) malloc_logger(MALLOC_LOG_TYPE_DEALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE, (uintptr_t)zone, (uintptr_t)ptr, 0, 0, 0); if (malloc_check_start && (malloc_check_counter++ >= malloc_check_start)) { internal_check(); } zone->free(zone, ptr); } -malloc_zone_t *malloc_zone_from_ptr(const void *ptr) { - malloc_zone_t *zone; - if (!ptr) return NULL; - zone = find_registered_zone(ptr, NULL); - return zone; +static void +malloc_zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size) { + if (malloc_logger) malloc_logger(MALLOC_LOG_TYPE_DEALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE, (uintptr_t)zone, (uintptr_t)ptr, 0, 0, 0); + if (malloc_check_start && (malloc_check_counter++ >= malloc_check_start)) { + internal_check(); + } + zone->free_definite_size(zone, ptr, size); +} + +malloc_zone_t * +malloc_zone_from_ptr(const void *ptr) { + if (!ptr) + return NULL; + else + return find_registered_zone(ptr, NULL); +} + +void * +malloc_zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size) { + void *ptr; + if (zone->version < 5) // Version must be >= 5 to look at the new memalign field. + return NULL; + if (!(zone->memalign)) + return NULL; + if (malloc_check_start && (malloc_check_counter++ >= malloc_check_start)) { + internal_check(); + } + if (size > MALLOC_ABSOLUTE_MAX_SIZE) { + return NULL; + } + if (alignment < sizeof( void *) || // excludes 0 == alignment + 0 != (alignment & (alignment - 1))) { // relies on sizeof(void *) being a power of two. + return NULL; + } + ptr = zone->memalign(zone, alignment, size); + if (malloc_logger) malloc_logger(MALLOC_LOG_TYPE_ALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE, (uintptr_t)zone, (uintptr_t)size, 0, (uintptr_t)ptr, 0); + return ptr; } /********* Functions for zone implementors ************/ -void malloc_zone_register(malloc_zone_t *zone) { - /* Note that given the sequencing it is always safe to first get the number of zones, then get malloc_zones without taking the lock, if all you need is to iterate through the list */ +void +malloc_zone_register(malloc_zone_t *zone) { MALLOC_LOCK(); - if (malloc_num_zones >= INITIAL_ZONES) { - malloc_zone_t **zones = malloc_zones; - malloc_zone_t *pzone = malloc_zones[0]; - boolean_t copy = malloc_num_zones == INITIAL_ZONES; - if (copy) zones = NULL; // to avoid realloc on something not allocated - MALLOC_UNLOCK(); - zones = pzone->realloc(pzone, zones, (malloc_num_zones + 1) * sizeof(malloc_zone_t *)); // we leak initial_malloc_zones, not worth tracking it - MALLOC_LOCK(); - if (copy) memcpy(zones, malloc_zones, malloc_num_zones * sizeof(malloc_zone_t *)); - malloc_zones = zones; - } - malloc_zones[malloc_num_zones] = zone; - malloc_num_zones++; // note that we do this after setting malloc_num_zones, so enumerations without taking the lock are safe + malloc_zone_register_while_locked(zone); MALLOC_UNLOCK(); - // malloc_printf("Registered %p malloc_zones at address %p is %p [%d zones]\n", zone, &malloc_zones, malloc_zones, malloc_num_zones); } -void malloc_zone_unregister(malloc_zone_t *z) { +void +malloc_zone_unregister(malloc_zone_t *z) { unsigned index; + + if (malloc_num_zones == 0) + return; + MALLOC_LOCK(); - index = malloc_num_zones; - while (index--) { - malloc_zone_t *zone = malloc_zones[index]; - if (zone == z) { - malloc_zones[index] = malloc_zones[--malloc_num_zones]; - MALLOC_UNLOCK(); - return; - } + for (index = 0; index < malloc_num_zones; ++index) { + if (z != malloc_zones[index]) + continue; + + // Modify the page to be allow write access, so that we can update the + // malloc_zones array. + size_t protect_size = malloc_num_zones_allocated * sizeof(malloc_zone_t *); + vm_protect(mach_task_self(), (uintptr_t)malloc_zones, protect_size, 0, VM_PROT_READ | VM_PROT_WRITE); + + // If we found a match, swap it with the entry on the back of the list + // and null out the back of the list. + malloc_zones[index] = malloc_zones[malloc_num_zones - 1]; + malloc_zones[malloc_num_zones - 1] = NULL; + --malloc_num_zones; + + vm_protect(mach_task_self(), (uintptr_t)malloc_zones, protect_size, 0, VM_PROT_READ); + MALLOC_UNLOCK(); + return; } MALLOC_UNLOCK(); - fprintf(stderr, "*** malloc[%d]: malloc_zone_unregister() failed for %p\n", getpid(), z); + malloc_printf("*** malloc_zone_unregister() failed for %p\n", z); } -void malloc_set_zone_name(malloc_zone_t *z, const char *name) { +void +malloc_set_zone_name(malloc_zone_t *z, const char *name) { char *newName; if (z->zone_name) { free((char *)z->zone_name); @@ -342,115 +730,325 @@ void malloc_set_zone_name(malloc_zone_t *z, const char *name) { z->zone_name = (const char *)newName; } -const char *malloc_get_zone_name(malloc_zone_t *zone) { +const char * +malloc_get_zone_name(malloc_zone_t *zone) { return zone->zone_name; } -static char *_malloc_append_unsigned(unsigned value, unsigned base, char *head) { - if (!value) { - head[0] = '0'; - } else { - if (value >= base) head = _malloc_append_unsigned(value / base, base, head); - value = value % base; - head[0] = (value < 10) ? '0' + value : 'a' + value - 10; - } - return head+1; -} - -void malloc_printf(const char *format, ...) { - va_list args; - char buf[1024]; - char *head = buf; - char ch; - unsigned *nums; - va_start(args, format); -#if LOG_THREAD - head = _malloc_append_unsigned(((unsigned)&args) >> 12, 16, head); - *head++ = ' '; -#endif - nums = args; - while (ch = *format++) { - if (ch == '%') { - ch = *format++; - if (ch == 's') { - char *str = (char *)(*nums++); - write(2, buf, head - buf); - head = buf; - write(2, str, strlen(str)); - } else { - if (ch == 'p') { - *head++ = '0'; *head++ = 'x'; - } - head = _malloc_append_unsigned(*nums++, (ch == 'd') ? 10 : 16, head); - } - } else { - *head++ = ch; - } +/* + * XXX malloc_printf now uses _simple_*printf. It only deals with a + * subset of printf format specifiers, but it doesn't call malloc. + */ + +__private_extern__ void +_malloc_vprintf(int flags, const char *format, va_list ap) +{ + _SIMPLE_STRING b; + + if (_malloc_no_asl_log || (flags & MALLOC_PRINTF_NOLOG) || (b = _simple_salloc()) == NULL) { + if (!(flags & MALLOC_PRINTF_NOPREFIX)) { + if (__is_threaded) { + /* XXX somewhat rude 'knowing' that pthread_t is a pointer */ + _simple_dprintf(malloc_debug_file, "%s(%d,%p) malloc: ", getprogname(), getpid(), (void *)pthread_self()); + } else { + _simple_dprintf(malloc_debug_file, "%s(%d) malloc: ", getprogname(), getpid()); + } + } + _simple_vdprintf(malloc_debug_file, format, ap); + return; } - write(2, buf, head - buf); fflush(stderr); - va_end(args); + if (!(flags & MALLOC_PRINTF_NOPREFIX)) { + if (__is_threaded) { + /* XXX somewhat rude 'knowing' that pthread_t is a pointer */ + _simple_sprintf(b, "%s(%d,%p) malloc: ", getprogname(), getpid(), (void *)pthread_self()); + } else { + _simple_sprintf(b, "%s(%d) malloc: ", getprogname(), getpid()); + } + } + _simple_vsprintf(b, format, ap); + _simple_put(b, malloc_debug_file); + _simple_asl_log(flags & MALLOC_PRINTF_LEVEL_MASK, Malloc_Facility, _simple_string(b)); + _simple_sfree(b); +} + +__private_extern__ void +_malloc_printf(int flags, const char *format, ...) +{ + va_list ap; + + va_start(ap, format); + _malloc_vprintf(flags, format, ap); + va_end(ap); +} + +void +malloc_printf(const char *format, ...) +{ + va_list ap; + + va_start(ap, format); + _malloc_vprintf(ASL_LEVEL_ERR, format, ap); + va_end(ap); } /********* Generic ANSI callouts ************/ -void *malloc(size_t size) { - return malloc_zone_malloc(inline_malloc_default_zone(), size); +void * +malloc(size_t size) { + void *retval; + retval = malloc_zone_malloc(inline_malloc_default_zone(), size); + if (retval == NULL) { + errno = ENOMEM; + } + return retval; } -void *calloc(size_t num_items, size_t size) { - return malloc_zone_calloc(inline_malloc_default_zone(), num_items, size); +void * +calloc(size_t num_items, size_t size) { + void *retval; + retval = malloc_zone_calloc(inline_malloc_default_zone(), num_items, size); + if (retval == NULL) { + errno = ENOMEM; + } + return retval; } -void free(void *ptr) { +void +free(void *ptr) { malloc_zone_t *zone; - if (!ptr) return; - zone = find_registered_zone(ptr, NULL); - if (zone) { + size_t size; + if (!ptr) + return; + zone = find_registered_zone(ptr, &size); + if (!zone) { + malloc_printf("*** error for object %p: pointer being freed was not allocated\n" + "*** set a breakpoint in malloc_error_break to debug\n", ptr); + malloc_error_break(); + if ((malloc_debug_flags & (SCALABLE_MALLOC_ABORT_ON_CORRUPTION|SCALABLE_MALLOC_ABORT_ON_ERROR))) + abort(); + } else if (zone->version >= 6 && zone->free_definite_size) + malloc_zone_free_definite_size(zone, ptr, size); + else malloc_zone_free(zone, ptr); - } else { - fprintf(stderr, "*** malloc[%d]: Deallocation of a pointer not malloced: %p; This could be a double free(), or free() called with the middle of an allocated block; Try setting environment variable MallocHelp to see tools to help debug\n", getpid(), ptr); - } } -void *realloc(void *old_ptr, size_t new_size) { +void * +realloc(void *in_ptr, size_t new_size) { + void *retval; + void *old_ptr; malloc_zone_t *zone; - size_t old_size = 0; - if (!old_ptr) return malloc_zone_malloc(inline_malloc_default_zone(), new_size); - zone = find_registered_zone(old_ptr, &old_size); - if (zone && (old_size >= new_size)) return old_ptr; - if (!zone) zone = inline_malloc_default_zone(); - return malloc_zone_realloc(zone, old_ptr, new_size); + size_t old_size = 0; + + // SUSv3: "If size is 0 and ptr is not a null pointer, the object + // pointed to is freed. If the space cannot be allocated, the object + // shall remain unchanged." Also "If size is 0, either a null pointer + // or a unique pointer that can be successfully passed to free() shall + // be returned." We choose to allocate a minimum size object by calling + // malloc_zone_malloc with zero size, which matches "If ptr is a null + // pointer, realloc() shall be equivalent to malloc() for the specified + // size." So we only free the original memory if the allocation succeeds. + old_ptr = (new_size == 0) ? NULL : in_ptr; + if (!old_ptr) { + retval = malloc_zone_malloc(inline_malloc_default_zone(), new_size); + } else { + zone = find_registered_zone(old_ptr, &old_size); + if (zone && old_size >= new_size) + return old_ptr; + + if (!zone) + zone = inline_malloc_default_zone(); + + retval = malloc_zone_realloc(zone, old_ptr, new_size); + } + if (retval == NULL) { + errno = ENOMEM; + } else if (new_size == 0) { + free(in_ptr); + } + return retval; } -void *valloc(size_t size) { +void * +valloc(size_t size) { + void *retval; malloc_zone_t *zone = inline_malloc_default_zone(); - return malloc_zone_valloc(zone, size); + retval = malloc_zone_valloc(zone, size); + if (retval == NULL) { + errno = ENOMEM; + } + return retval; } -extern void vfree(void *ptr) { +extern void +vfree(void *ptr) { free(ptr); } -size_t malloc_size(const void *ptr) { +size_t +malloc_size(const void *ptr) { size_t size = 0; - if (!ptr) return size; + + if (!ptr) + return size; + (void)find_registered_zone(ptr, &size); return size; } -size_t malloc_good_size (size_t size) { +size_t +malloc_good_size (size_t size) { malloc_zone_t *zone = inline_malloc_default_zone(); return zone->introspect->good_size(zone, size); } +/* + * The posix_memalign() function shall allocate size bytes aligned on a boundary specified by alignment, + * and shall return a pointer to the allocated memory in memptr. + * The value of alignment shall be a multiple of sizeof( void *), that is also a power of two. + * Upon successful completion, the value pointed to by memptr shall be a multiple of alignment. + * + * Upon successful completion, posix_memalign() shall return zero; otherwise, + * an error number shall be returned to indicate the error. + * + * The posix_memalign() function shall fail if: + * EINVAL + * The value of the alignment parameter is not a power of two multiple of sizeof( void *). + * ENOMEM + * There is insufficient memory available with the requested alignment. + */ + +int +posix_memalign(void **memptr, size_t alignment, size_t size) +{ + void *retval; + + /* POSIX is silent on NULL == memptr !?! */ + + retval = malloc_zone_memalign(inline_malloc_default_zone(), alignment, size); + if (retval == NULL) { + // To avoid testing the alignment constraints redundantly, we'll rely on the + // test made in malloc_zone_memalign to vet each request. Only if that test fails + // and returns NULL, do we arrive here to detect the bogus alignment and give the + // required EINVAL return. + if (alignment < sizeof( void *) || // excludes 0 == alignment + 0 != (alignment & (alignment - 1))) { // relies on sizeof(void *) being a power of two. + return EINVAL; + } + return ENOMEM; + } else { + *memptr = retval; // Set iff allocation succeeded + return 0; + } +} + +static malloc_zone_t * +find_registered_purgeable_zone(void *ptr) { + if (!ptr) + return NULL; + + /* + * Look for a zone which contains ptr. If that zone does not have the purgeable malloc flag + * set, or the allocation is too small, do nothing. Otherwise, set the allocation volatile. + * FIXME: for performance reasons, we should probably keep a separate list of purgeable zones + * and only search those. + */ + size_t size = 0; + malloc_zone_t *zone = find_registered_zone(ptr, &size); + + /* FIXME: would really like a zone->introspect->flags->purgeable check, but haven't determined + * binary compatibility impact of changing the introspect struct yet. */ + if (!zone) + return NULL; + + /* Check to make sure pointer is page aligned and size is multiple of page size */ + if ((size < vm_page_size) || ((size % vm_page_size) != 0)) + return NULL; + + return zone; +} + +void +malloc_make_purgeable(void *ptr) { + malloc_zone_t *zone = find_registered_purgeable_zone(ptr); + if (!zone) + return; + + int state = VM_PURGABLE_VOLATILE; + vm_purgable_control(mach_task_self(), (vm_address_t)ptr, VM_PURGABLE_SET_STATE, &state); + return; +} + +/* Returns true if ptr is valid. Ignore the return value from vm_purgeable_control and only report + * state. */ +int +malloc_make_nonpurgeable(void *ptr) { + malloc_zone_t *zone = find_registered_purgeable_zone(ptr); + if (!zone) + return 0; + + int state = VM_PURGABLE_NONVOLATILE; + vm_purgable_control(mach_task_self(), (vm_address_t)ptr, VM_PURGABLE_SET_STATE, &state); + + if (state == VM_PURGABLE_EMPTY) + return EFAULT; + + return 0; +} + +/********* Batch methods ************/ + +unsigned +malloc_zone_batch_malloc(malloc_zone_t *zone, size_t size, void **results, unsigned num_requested) { + unsigned (*batch_malloc)(malloc_zone_t *, size_t, void **, unsigned) = zone-> batch_malloc; + if (! batch_malloc) return 0; + if (malloc_check_start && (malloc_check_counter++ >= malloc_check_start)) { + internal_check(); + } + unsigned batched = batch_malloc(zone, size, results, num_requested); + if (malloc_logger) { + unsigned index = 0; + while (index < batched) { + malloc_logger(MALLOC_LOG_TYPE_ALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE, (uintptr_t)zone, (uintptr_t)size, 0, (uintptr_t)results[index], 0); + index++; + } + } + return batched; +} + +void +malloc_zone_batch_free(malloc_zone_t *zone, void **to_be_freed, unsigned num) { + if (malloc_check_start && (malloc_check_counter++ >= malloc_check_start)) { + internal_check(); + } + if (malloc_logger) { + unsigned index = 0; + while (index < num) { + malloc_logger(MALLOC_LOG_TYPE_DEALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE, (uintptr_t)zone, (uintptr_t)to_be_freed[index], 0, 0, 0); + index++; + } + } + void (*batch_free)(malloc_zone_t *, void **, unsigned) = zone-> batch_free; + if (batch_free) { + batch_free(zone, to_be_freed, num); + } else { + void (*free_fun)(malloc_zone_t *, void *) = zone->free; + while (num--) { + void *ptr = *to_be_freed++; + free_fun(zone, ptr); + } + } +} + /********* Functions for performance tools ************/ -static kern_return_t _malloc_default_reader(task_t task, vm_address_t address, vm_size_t size, void **ptr) { +static kern_return_t +_malloc_default_reader(task_t task, vm_address_t address, vm_size_t size, void **ptr) { *ptr = (void *)address; return 0; } -kern_return_t malloc_get_all_zones(task_t task, memory_reader_t reader, vm_address_t **addresses, unsigned *count) { +kern_return_t +malloc_get_all_zones(task_t task, memory_reader_t reader, vm_address_t **addresses, unsigned *count) { // Note that the 2 following addresses are not correct if the address of the target is different from your own. This notably occurs if the address of System.framework is slid (e.g. different than at B & I ) vm_address_t remote_malloc_zones = (vm_address_t)&malloc_zones; vm_address_t remote_malloc_num_zones = (vm_address_t)&malloc_num_zones; @@ -462,25 +1060,25 @@ kern_return_t malloc_get_all_zones(task_t task, memory_reader_t reader, vm_addre if (!reader) reader = _malloc_default_reader; // printf("Read malloc_zones at address %p should be %p\n", &malloc_zones, malloc_zones); err = reader(task, remote_malloc_zones, sizeof(void *), (void **)&zones_address_ref); - // printf("Read malloc_zones[0x%x]=%p\n", remote_malloc_zones, *zones_address_ref); + // printf("Read malloc_zones[%p]=%p\n", remote_malloc_zones, *zones_address_ref); if (err) { - fprintf(stderr, "*** malloc[%d]: malloc_get_all_zones: error reading zones_address at 0x%x\n", getpid(), (unsigned)remote_malloc_zones); + malloc_printf("*** malloc_get_all_zones: error reading zones_address at %p\n", (unsigned)remote_malloc_zones); return err; } zones_address = *zones_address_ref; // printf("Reading num_zones at address %p\n", remote_malloc_num_zones); err = reader(task, remote_malloc_num_zones, sizeof(unsigned), (void **)&num_zones_ref); if (err) { - fprintf(stderr, "*** malloc[%d]: malloc_get_all_zones: error reading num_zones at 0x%x\n", getpid(), (unsigned)remote_malloc_num_zones); + malloc_printf("*** malloc_get_all_zones: error reading num_zones at %p\n", (unsigned)remote_malloc_num_zones); return err; } num_zones = *num_zones_ref; - // printf("Read malloc_num_zones[0x%x]=%d\n", remote_malloc_num_zones, num_zones); + // printf("Read malloc_num_zones[%p]=%d\n", remote_malloc_num_zones, num_zones); *count = num_zones; // printf("malloc_get_all_zones succesfully found %d zones\n", num_zones); err = reader(task, zones_address, sizeof(malloc_zone_t *) * num_zones, (void **)addresses); if (err) { - fprintf(stderr, "*** malloc[%d]: malloc_get_all_zones: error reading zones at 0x%x\n", getpid(), (unsigned)&zones_address); + malloc_printf("*** malloc_get_all_zones: error reading zones at %p\n", &zones_address); return err; } // printf("malloc_get_all_zones succesfully read %d zones\n", num_zones); @@ -489,10 +1087,11 @@ kern_return_t malloc_get_all_zones(task_t task, memory_reader_t reader, vm_addre /********* Debug helpers ************/ -void malloc_zone_print_ptr_info(void *ptr) { +void +malloc_zone_print_ptr_info(void *ptr) { malloc_zone_t *zone; if (!ptr) return; - zone = find_registered_zone(ptr, NULL); + zone = malloc_zone_from_ptr(ptr); if (zone) { printf("ptr %p in registered zone %p\n", ptr, zone); } else { @@ -500,7 +1099,8 @@ void malloc_zone_print_ptr_info(void *ptr) { } } -boolean_t malloc_zone_check(malloc_zone_t *zone) { +boolean_t +malloc_zone_check(malloc_zone_t *zone) { boolean_t ok = 1; if (!zone) { unsigned index = 0; @@ -514,7 +1114,8 @@ boolean_t malloc_zone_check(malloc_zone_t *zone) { return ok; } -void malloc_zone_print(malloc_zone_t *zone, boolean_t verbose) { +void +malloc_zone_print(malloc_zone_t *zone, boolean_t verbose) { if (!zone) { unsigned index = 0; while (index < malloc_num_zones) { @@ -526,7 +1127,27 @@ void malloc_zone_print(malloc_zone_t *zone, boolean_t verbose) { } } -void malloc_zone_log(malloc_zone_t *zone, void *address) { +void +malloc_zone_statistics(malloc_zone_t *zone, malloc_statistics_t *stats) { + if (!zone) { + memset(stats, 0, sizeof(*stats)); + unsigned index = 0; + while (index < malloc_num_zones) { + zone = malloc_zones[index++]; + malloc_statistics_t this_stats; + zone->introspect->statistics(zone, &this_stats); + stats->blocks_in_use += this_stats.blocks_in_use; + stats->size_in_use += this_stats.size_in_use; + stats->max_size_in_use += this_stats.max_size_in_use; + stats->size_allocated += this_stats.size_allocated; + } + } else { + zone->introspect->statistics(zone, stats); + } +} + +void +malloc_zone_log(malloc_zone_t *zone, void *address) { if (!zone) { unsigned index = 0; while (index < malloc_num_zones) { @@ -540,20 +1161,37 @@ void malloc_zone_log(malloc_zone_t *zone, void *address) { /********* Misc other entry points ************/ -static void DefaultMallocError(int x) { - fprintf(stderr, "*** malloc[%d]: error %d\n", getpid(), x); +static void +DefaultMallocError(int x) { #if USE_SLEEP_RATHER_THAN_ABORT + malloc_printf("*** error %d\n", x); sleep(3600); #else + _SIMPLE_STRING b = _simple_salloc(); + if (b) { + _simple_sprintf(b, "*** error %d", x); + malloc_printf("%s\n", _simple_string(b)); + __crashreporter_info__ = _simple_string(b); + } else { + _malloc_printf(MALLOC_PRINTF_NOLOG, "*** error %d", x); + __crashreporter_info__ = "*** DefaultMallocError called"; + } abort(); #endif } -void (*malloc_error(void (*func)(int)))(int) { +void (* +malloc_error(void (*func)(int)))(int) { return DefaultMallocError; } -void _malloc_fork_prepare() { +/* Stack logging fork-handling prototypes */ +extern void __stack_logging_fork_prepare(); +extern void __stack_logging_fork_parent(); +extern void __stack_logging_fork_child(); + +void +_malloc_fork_prepare() { /* Prepare the malloc module for a fork by insuring that no thread is in a malloc critical section */ unsigned index = 0; MALLOC_LOCK(); @@ -561,11 +1199,14 @@ void _malloc_fork_prepare() { malloc_zone_t *zone = malloc_zones[index++]; zone->introspect->force_lock(zone); } + __stack_logging_fork_prepare(); } -void _malloc_fork_parent() { +void +_malloc_fork_parent() { /* Called in the parent process after a fork() to resume normal operation. */ unsigned index = 0; + __stack_logging_fork_parent(); MALLOC_UNLOCK(); while (index < malloc_num_zones) { malloc_zone_t *zone = malloc_zones[index++]; @@ -573,9 +1214,11 @@ void _malloc_fork_parent() { } } -void _malloc_fork_child() { +void +_malloc_fork_child() { /* Called in the child process after a fork() to resume normal operation. In the MTASK case we also have to change memory inheritance so that the child does not share memory with the parent. */ unsigned index = 0; + __stack_logging_fork_child(); MALLOC_UNLOCK(); while (index < malloc_num_zones) { malloc_zone_t *zone = malloc_zones[index++]; @@ -583,9 +1226,27 @@ void _malloc_fork_child() { } } -size_t mstats(void) { - malloc_zone_print(NULL, 0); - return 1; +/* + * A Glibc-like mstats() interface. + * + * Note that this interface really isn't very good, as it doesn't understand + * that we may have multiple allocators running at once. We just massage + * the result from malloc_zone_statistics in any case. + */ +struct mstats +mstats(void) +{ + malloc_statistics_t s; + struct mstats m; + + malloc_zone_statistics(NULL, &s); + m.bytes_total = s.size_allocated; + m.chunks_used = s.blocks_in_use; + m.bytes_used = s.size_in_use; + m.chunks_free = 0; + m.bytes_free = m.bytes_total - m.bytes_used; /* isn't this somewhat obvious? */ + + return(m); } /***************** OBSOLETE ENTRY POINTS ********************/ @@ -596,25 +1257,28 @@ size_t mstats(void) { #warning PHASE OUT THE FOLLOWING FUNCTIONS #endif -void set_malloc_singlethreaded(boolean_t single) { +void +set_malloc_singlethreaded(boolean_t single) { static boolean_t warned = 0; if (!warned) { #if PHASE_OUT_OLD_MALLOC - fprintf(stderr, "*** malloc[%d]: OBSOLETE: set_malloc_singlethreaded(%d)\n", getpid(), single); + malloc_printf("*** OBSOLETE: set_malloc_singlethreaded(%d)\n", single); #endif warned = 1; } } -void malloc_singlethreaded() { +void +malloc_singlethreaded() { static boolean_t warned = 0; if (!warned) { - fprintf(stderr, "*** malloc[%d]: OBSOLETE: malloc_singlethreaded()\n", getpid()); + malloc_printf("*** OBSOLETE: malloc_singlethreaded()\n"); warned = 1; } } -int malloc_debug(int level) { - fprintf(stderr, "*** malloc[%d]: OBSOLETE: malloc_debug()\n", getpid()); +int +malloc_debug(int level) { + malloc_printf("*** OBSOLETE: malloc_debug()\n"); return 0; }