gen/malloc.c

   1 /*
   2  * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. Please obtain a copy of the License at
  10  * http://www.opensource.apple.com/apsl/ and read it before using this
  11  * file.
  12  *
  13  * The Original Code and all software distributed under the License are
  14  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  15  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  16  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  18  * Please see the License for the specific language governing rights and
  19  * limitations under the License.
  20  *
  21  * @APPLE_LICENSE_HEADER_END@
  22  */
  23
  24 #include <pthread_internals.h>
  25
  26 #import <stdlib.h>
  27 #import <stdio.h>
  28 #import <string.h>
  29 #import <unistd.h>
  30 #import <objc/zone.h>
  31 #import <malloc/malloc.h>
  32 #import <fcntl.h>
  33 #include <crt_externs.h>
  34
  35 #import "scalable_malloc.h"
  36 #import "stack_logging.h"
  37
  38 #define USE_SLEEP_RATHER_THAN_ABORT     0
  39
  40 #define MAX_ALLOCATION 0xc0000000 // beyond this, assume a programming error
  41 #define INITIAL_ZONES   8  // After this number, we reallocate for new zones
  42
  43 typedef void (malloc_logger_t)(unsigned type, unsigned arg1, unsigned arg2, unsigned arg3, unsigned result, unsigned num_hot_frames_to_skip);
  44
  45 static pthread_lock_t _malloc_lock;
  46 static malloc_zone_t *initial_malloc_zones[INITIAL_ZONES] = {0};
  47
  48 /* The following variables are exported for the benefit of performance tools */
  49 unsigned malloc_num_zones = 0;
  50 malloc_zone_t **malloc_zones = initial_malloc_zones;
  51 malloc_logger_t *malloc_logger = NULL;
  52
  53 unsigned malloc_debug_flags = 0;
  54
  55 unsigned malloc_check_start = 0; // 0 means don't check
  56 unsigned malloc_check_counter = 0;
  57 unsigned malloc_check_each = 1000;
  58
  59 static int malloc_check_sleep = 100; // default 100 second sleep
  60 static int malloc_check_abort = 0; // default is to sleep, not abort
  61
  62 static int malloc_free_abort = 0; // default is not to abort
  63
  64 static int logfd = 2;   // malloc_printf file descriptor
  65
  66 #define MALLOC_LOCK()           LOCK(_malloc_lock)
  67 #define MALLOC_UNLOCK()         UNLOCK(_malloc_lock)
  68
  69 #define MALLOC_LOG_TYPE_ALLOCATE        stack_logging_type_alloc
  70 #define MALLOC_LOG_TYPE_DEALLOCATE      stack_logging_type_dealloc
  71 #define MALLOC_LOG_TYPE_HAS_ZONE        stack_logging_flag_zone
  72 #define MALLOC_LOG_TYPE_CLEARED         stack_logging_flag_cleared
  73
  74 /*********      Utilities       ************/
  75
  76 static inline malloc_zone_t *
  77 find_registered_zone(const void *ptr, size_t *returned_size) {
  78     // locates the proper zone
  79     // if zone found fills returnedSize; else returns NULL
  80     // See comment in malloc_zone_register() about clients non locking to call this function
  81     // Speed is critical for this function
  82     unsigned    index = malloc_num_zones;
  83     malloc_zone_t       **zones = malloc_zones;
  84     while (index--) {
  85         malloc_zone_t   *zone = *zones++;
  86         size_t  size;
  87         size = zone->size(zone, ptr);
  88         if (size) {
  89             if (returned_size) *returned_size = size;
  90             return zone;
  91         }
  92     }
  93     return NULL;
  94 }
  95
  96 /*********      Creation and destruction        ************/
  97
  98 static void
  99 _malloc_initialize(void) {
 100     // guaranteed to be called only once
 101     (void)malloc_create_zone(0, 0);
 102     malloc_set_zone_name(malloc_zones[0], "DefaultMallocZone");
 103     LOCK_INIT(_malloc_lock);
 104     // malloc_printf("Malloc: %d registered zones\n", malloc_num_zones);
 105     // malloc_printf("malloc: malloc_zones is at %p; malloc_num_zones is at %p\n", (unsigned)&malloc_zones, (unsigned)&malloc_num_zones);
 106 }
 107
 108 static inline malloc_zone_t *
 109 inline_malloc_default_zone(void) {
 110     if (!malloc_num_zones) _malloc_initialize();
 111     // malloc_printf("In inline_malloc_default_zone with %d %d\n", malloc_num_zones, malloc_has_debug_zone);
 112     return malloc_zones[0];
 113 }
 114
 115 malloc_zone_t *
 116 malloc_default_zone(void) {
 117     return inline_malloc_default_zone();
 118 }
 119
 120 static void
 121 set_flags_from_environment(void) {
 122     const char *flag;
 123     flag = getenv("MallocLogFile");
 124     if (flag) {
 125         int fd = open(flag, O_WRONLY|O_APPEND|O_CREAT, 0644);
 126         if (fd >= 0) {
 127             logfd = fd;
 128             fcntl(fd, F_SETFD, 0); // clear close-on-exec flag
 129         } else
 130             malloc_printf("malloc[%d]: Could not open %s, using stderr\n",
 131                 getpid(), flag);
 132     }
 133     if (getenv("MallocGuardEdges")) {
 134         malloc_debug_flags = SCALABLE_MALLOC_ADD_GUARD_PAGES;
 135         malloc_printf("malloc[%d]: protecting edges\n", getpid());
 136         if (getenv("MallocDoNotProtectPrelude")) {
 137             malloc_debug_flags |= SCALABLE_MALLOC_DONT_PROTECT_PRELUDE;
 138             malloc_printf("malloc[%d]: ... but not protecting prelude guard page\n", getpid());
 139         }
 140         if (getenv("MallocDoNotProtectPostlude")) {
 141             malloc_debug_flags |= SCALABLE_MALLOC_DONT_PROTECT_POSTLUDE;
 142             malloc_printf("malloc[%d]: ... but not protecting postlude guard page\n", getpid());
 143         }
 144     }
 145     flag = getenv("MallocStackLogging");
 146     if (!flag) {
 147         flag = getenv("MallocStackLoggingNoCompact");
 148         stack_logging_dontcompact = 1;
 149     }
 150     if (flag) {
 151         unsigned        val = strtoul(flag, NULL, 0);
 152         if (val == 1) val = 0;
 153         if (val == -1) val = 0;
 154         malloc_logger = (val) ? (void *)val : stack_logging_log_stack;
 155         stack_logging_enable_logging = 1;
 156         if (malloc_logger == stack_logging_log_stack) {
 157             malloc_printf("malloc[%d]: recording stacks using standard recorder\n", getpid());
 158         } else {
 159             malloc_printf("malloc[%d]: recording stacks using recorder %p\n", getpid(), malloc_logger);
 160         }
 161         if (stack_logging_dontcompact) malloc_printf("malloc[%d]: stack logging compaction turned off; VM can increase rapidly\n", getpid());
 162     }
 163     if (getenv("MallocScribble")) {
 164         malloc_debug_flags |= SCALABLE_MALLOC_DO_SCRIBBLE;
 165         malloc_printf("malloc[%d]: enabling scribbling to detect mods to free blocks\n", getpid());
 166     }
 167     flag = getenv("MallocCheckHeapStart");
 168     if (flag) {
 169         malloc_check_start = strtoul(flag, NULL, 0);
 170         if (malloc_check_start == 0) malloc_check_start = 1;
 171         if (malloc_check_start == -1) malloc_check_start = 1;
 172         flag = getenv("MallocCheckHeapEach");
 173         if (flag) {
 174             malloc_check_each = strtoul(flag, NULL, 0);
 175             if (malloc_check_each == 0) malloc_check_each = 1;
 176             if (malloc_check_each == -1) malloc_check_each = 1;
 177         }
 178         malloc_printf("malloc[%d]: checks heap after %dth operation and each %d operations\n", getpid(), malloc_check_start, malloc_check_each);
 179         flag = getenv("MallocCheckHeapAbort");
 180         if (flag)
 181             malloc_check_abort = strtol(flag, NULL, 0);
 182         if (malloc_check_abort)
 183             malloc_printf("malloc[%d]: will abort on heap corruption\n",
 184                 getpid());
 185         else {
 186             flag = getenv("MallocCheckHeapSleep");
 187             if (flag)
 188                 malloc_check_sleep = strtol(flag, NULL, 0);
 189             if (malloc_check_sleep > 0)
 190                 malloc_printf("malloc[%d]: will sleep for %d seconds on heap corruption\n",
 191                     getpid(), malloc_check_sleep);
 192             else if (malloc_check_sleep < 0)
 193                 malloc_printf("malloc[%d]: will sleep once for %d seconds on heap corruption\n",
 194                     getpid(), -malloc_check_sleep);
 195             else
 196                 malloc_printf("malloc[%d]: no sleep on heap corruption\n",
 197                     getpid());
 198         }
 199     }
 200     flag = getenv("MallocBadFreeAbort");
 201     if (flag)
 202         malloc_free_abort = strtol(flag, NULL, 0);
 203     if (getenv("MallocHelp")) {
 204         malloc_printf(
 205             "malloc[%d]: environment variables that can be set for debug:\n"
 206             "- MallocLogFile <f> to create/append messages to file <f> instead of stderr\n"
 207             "- MallocGuardEdges to add 2 guard pages for each large block\n"
 208             "- MallocDoNotProtectPrelude to disable protection (when previous flag set)\n"
 209             "- MallocDoNotProtectPostlude to disable protection (when previous flag set)\n"
 210             "- MallocStackLogging to record all stacks.  Tools like leaks can then be applied\n"
 211             "- MallocStackLoggingNoCompact to record all stacks.  Needed for malloc_history\n"
 212             "- MallocScribble to detect writing on free blocks: 0x55 is written upon free\n"
 213             "- MallocCheckHeapStart <n> to start checking the heap after <n> operations\n"
 214             "- MallocCheckHeapEach <s> to repeat the checking of the heap after <s> operations\n"
 215             "- MallocCheckHeapSleep <t> to sleep <t> seconds on heap corruption\n"
 216             "- MallocCheckHeapAbort <b> to abort on heap corruption if <b> is non-zero\n"
 217             "- MallocBadFreeAbort <b> to abort on a bad free if <b> is non-zero\n"
 218             "- MallocHelp - this help!\n", getpid());
 219     }
 220 }
 221
 222 malloc_zone_t *
 223 malloc_create_zone(vm_size_t start_size, unsigned flags) {
 224     malloc_zone_t       *zone;
 225     if (!malloc_num_zones) {
 226         char    **env = * _NSGetEnviron();
 227         char    **p;
 228         char    *c;
 229         /* Given that all environment variables start with "Malloc" we optimize by scanning quickly first the environment, therefore avoiding repeated calls to getenv() */
 230         for (p = env; (c = *p) != NULL; ++p) {
 231             if (!strncmp(c, "Malloc", 6)) {
 232                 set_flags_from_environment();
 233                 break;
 234             }
 235         }
 236
 237     }
 238     zone = create_scalable_zone(start_size, malloc_debug_flags);
 239     malloc_zone_register(zone);
 240     return zone;
 241 }
 242
 243 void
 244 malloc_destroy_zone(malloc_zone_t *zone) {
 245     malloc_zone_unregister(zone);
 246     zone->destroy(zone);
 247 }
 248
 249 /*********      Block creation and manipulation ************/
 250
 251 static void
 252 internal_check(void) {
 253     static vm_address_t *frames = NULL;
 254     static unsigned     num_frames;
 255     if (malloc_zone_check(NULL)) {
 256         malloc_printf("MallocCheckHeap: PASSED check at %dth operation\n", malloc_check_counter-1);
 257         if (!frames) vm_allocate(mach_task_self(), (void *)&frames, vm_page_size, 1);
 258         thread_stack_pcs(frames, vm_page_size/sizeof(vm_address_t) - 1, &num_frames);
 259     } else {
 260         malloc_printf("*** MallocCheckHeap: FAILED check at %dth operation\n", malloc_check_counter-1);
 261         if (frames) {
 262             unsigned    index = 1;
 263             malloc_printf("Stack for last operation where the malloc check succeeded: ");
 264             while (index < num_frames) malloc_printf("%p ", frames[index++]);
 265             malloc_printf("\n(Use 'atos' for a symbolic stack)\n");
 266         }
 267         if (malloc_check_each > 1) {
 268             unsigned    recomm_each = (malloc_check_each > 10) ? malloc_check_each/10 : 1;
 269             unsigned    recomm_start = (malloc_check_counter > malloc_check_each+1) ? malloc_check_counter-1-malloc_check_each : 1;
 270             malloc_printf("*** Recommend using 'setenv MallocCheckHeapStart %d; setenv MallocCheckHeapEach %d' to narrow down failure\n", recomm_start, recomm_each);
 271         }
 272         if (malloc_check_abort)
 273             abort();
 274         if (malloc_check_sleep > 0) {
 275             malloc_printf("*** Sleeping for %d seconds to leave time to attach\n",
 276                 malloc_check_sleep);
 277             sleep(malloc_check_sleep);
 278         } else if (malloc_check_sleep < 0) {
 279             malloc_printf("*** Sleeping once for %d seconds to leave time to attach\n",
 280                 -malloc_check_sleep);
 281             sleep(-malloc_check_sleep);
 282             malloc_check_sleep = 0;
 283         }
 284     }
 285     malloc_check_start += malloc_check_each;
 286 }
 287
 288 void *
 289 malloc_zone_malloc(malloc_zone_t *zone, size_t size) {
 290     void        *ptr;
 291     if ((unsigned)size >= MAX_ALLOCATION) {
 292         /* Probably a programming error */
 293         malloc_printf("*** malloc_zone_malloc[%d]: argument too large: %d\n", getpid(), size);
 294         return NULL;
 295     }
 296     if (malloc_check_start && (malloc_check_counter++ >= malloc_check_start)) {
 297         internal_check();
 298     }
 299     ptr = zone->malloc(zone, size);
 300     if (malloc_logger) malloc_logger(MALLOC_LOG_TYPE_ALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE, (unsigned)zone, size, 0, (unsigned)ptr, 0);
 301     return ptr;
 302 }
 303
 304 void *
 305 malloc_zone_calloc(malloc_zone_t *zone, size_t num_items, size_t size) {
 306     void        *ptr;
 307     if (malloc_check_start && (malloc_check_counter++ >= malloc_check_start)) {
 308         internal_check();
 309     }
 310     if (((unsigned)num_items >= MAX_ALLOCATION) || ((unsigned)size >= MAX_ALLOCATION) || ((long long)size * num_items >= (long long) MAX_ALLOCATION)) {
 311         /* Probably a programming error */
 312         malloc_printf("*** malloc_zone_calloc[%d]: arguments too large: %d,%d\n", getpid(), num_items, size);
 313         return NULL;
 314     }
 315     ptr = zone->calloc(zone, num_items, size);
 316     if (malloc_logger) malloc_logger(MALLOC_LOG_TYPE_ALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE | MALLOC_LOG_TYPE_CLEARED, (unsigned)zone, num_items * size, 0, (unsigned)ptr, 0);
 317     return ptr;
 318 }
 319
 320 void *
 321 malloc_zone_valloc(malloc_zone_t *zone, size_t size) {
 322     void        *ptr;
 323     if ((unsigned)size >= MAX_ALLOCATION) {
 324         /* Probably a programming error */
 325         malloc_printf("*** malloc_zone_valloc[%d]: argument too large: %d\n", getpid(), size);
 326         return NULL;
 327     }
 328     if (malloc_check_start && (malloc_check_counter++ >= malloc_check_start)) {
 329         internal_check();
 330     }
 331     ptr = zone->valloc(zone, size);
 332     if (malloc_logger) malloc_logger(MALLOC_LOG_TYPE_ALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE, (unsigned)zone, size, 0, (unsigned)ptr, 0);
 333     return ptr;
 334 }
 335
 336 void *
 337 malloc_zone_realloc(malloc_zone_t *zone, void *ptr, size_t size) {
 338     void        *new_ptr;
 339     if (malloc_check_start && (malloc_check_counter++ >= malloc_check_start)) {
 340         internal_check();
 341     }
 342     new_ptr = zone->realloc(zone, ptr, size);
 343     if (malloc_logger) malloc_logger(MALLOC_LOG_TYPE_ALLOCATE | MALLOC_LOG_TYPE_DEALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE, (unsigned)zone, (unsigned)ptr, size, (unsigned)new_ptr, 0);
 344     return new_ptr;
 345 }
 346
 347 void
 348 malloc_zone_free(malloc_zone_t *zone, void *ptr) {
 349     if (malloc_logger) malloc_logger(MALLOC_LOG_TYPE_DEALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE, (unsigned)zone, (unsigned)ptr, 0, 0, 0);
 350     if (malloc_check_start && (malloc_check_counter++ >= malloc_check_start)) {
 351         internal_check();
 352     }
 353     zone->free(zone, ptr);
 354 }
 355
 356 malloc_zone_t *
 357 malloc_zone_from_ptr(const void *ptr) {
 358     malloc_zone_t       *zone;
 359     if (!ptr) return NULL;
 360     zone = find_registered_zone(ptr, NULL);
 361     return zone;
 362 }
 363
 364 /*********      Functions for zone implementors ************/
 365
 366 void
 367 malloc_zone_register(malloc_zone_t *zone) {
 368     /* Note that given the sequencing it is always safe to first get the number of zones, then get malloc_zones without taking the lock, if all you need is to iterate through the list */
 369     MALLOC_LOCK();
 370     if (malloc_num_zones >= INITIAL_ZONES) {
 371         malloc_zone_t           **zones = malloc_zones;
 372         malloc_zone_t           *pzone = malloc_zones[0];
 373         boolean_t               copy = malloc_num_zones == INITIAL_ZONES;
 374         if (copy) zones = NULL; // to avoid realloc on something not allocated
 375         MALLOC_UNLOCK();
 376         zones = pzone->realloc(pzone, zones, (malloc_num_zones + 1) * sizeof(malloc_zone_t *)); // we leak initial_malloc_zones, not worth tracking it
 377         MALLOC_LOCK();
 378         if (copy) memcpy(zones, malloc_zones, malloc_num_zones * sizeof(malloc_zone_t *));
 379         malloc_zones = zones;
 380     }
 381     malloc_zones[malloc_num_zones] = zone;
 382     malloc_num_zones++; // note that we do this after setting malloc_num_zones, so enumerations without taking the lock are safe
 383     MALLOC_UNLOCK();
 384     // malloc_printf("Registered %p malloc_zones at address %p is %p [%d zones]\n", zone, &malloc_zones, malloc_zones, malloc_num_zones);
 385 }
 386
 387 void
 388 malloc_zone_unregister(malloc_zone_t *z) {
 389     unsigned    index;
 390     MALLOC_LOCK();
 391     index = malloc_num_zones;
 392     while (index--) {
 393         malloc_zone_t   *zone = malloc_zones[index];
 394         if (zone == z) {
 395             malloc_zones[index] = malloc_zones[--malloc_num_zones];
 396             MALLOC_UNLOCK();
 397             return;
 398         }
 399     }
 400     MALLOC_UNLOCK();
 401     malloc_printf("*** malloc[%d]: malloc_zone_unregister() failed for %p\n", getpid(), z);
 402 }
 403
 404 void
 405 malloc_set_zone_name(malloc_zone_t *z, const char *name) {
 406     char        *newName;
 407     if (z->zone_name) {
 408         free((char *)z->zone_name);
 409         z->zone_name = NULL;
 410     }
 411     newName = malloc_zone_malloc(z, strlen(name) + 1);
 412     strcpy(newName, name);
 413     z->zone_name = (const char *)newName;
 414 }
 415
 416 const char *
 417 malloc_get_zone_name(malloc_zone_t *zone) {
 418     return zone->zone_name;
 419 }
 420
 421 static char *
 422 _malloc_append_unsigned(unsigned value, unsigned base, char *head) {
 423     if (!value) {
 424         head[0] = '0';
 425     } else {
 426         if (value >= base) head = _malloc_append_unsigned(value / base, base, head);
 427         value = value % base;
 428         head[0] = (value < 10) ? '0' + value : 'a' + value - 10;
 429     }
 430     return head+1;
 431 }
 432
 433 void
 434 malloc_printf(const char *format, ...) {
 435     va_list     args;
 436     char        buf[1024];
 437     char        *head = buf;
 438     char        ch;
 439     unsigned    *nums;
 440     va_start(args, format);
 441 #if LOG_THREAD
 442     head = _malloc_append_unsigned(((unsigned)&args) >> 12, 16, head);
 443     *head++ = ' ';
 444 #endif
 445     nums = (void *)args;
 446     while (ch = *format++) {
 447         if (ch == '%') {
 448             ch = *format++;
 449             if (ch == 's') {
 450                 char    *str = (char *)(*nums++);
 451                 write(logfd, buf, head - buf);
 452                 head = buf;
 453                 write(logfd, str, strlen(str));
 454             } else if (ch == 'y') {
 455                 unsigned        num = *nums++;
 456                 if (num == 0) {
 457                     *head++ = '0';
 458                 } else if (num >= 10 * 1024 *1024) {
 459                     // use a round number of MB
 460                     head = _malloc_append_unsigned(num >> 20, 10, head);
 461                     *head++ = 'M'; *head++ = 'B';
 462                 } else if (num >= 10 * 1024) {
 463                     // use a round amount of KB
 464                     head = _malloc_append_unsigned(num >> 10, 10, head);
 465                     *head++ = 'K'; *head++ = 'B';
 466                 } else {
 467                     head = _malloc_append_unsigned(num, 10, head);
 468                     *head++ = 'b';
 469                 }
 470             } else {
 471                 if (ch == 'p') {
 472                     *head++ = '0'; *head++ = 'x';
 473                 }
 474                 head = _malloc_append_unsigned(*nums++, (ch == 'd') ? 10 : 16, head);
 475             }
 476         } else {
 477             *head++ = ch;
 478         }
 479     }
 480     write(logfd, buf, head - buf); fflush(stderr);
 481     va_end(args);
 482 }
 483
 484 /*********      Generic ANSI callouts   ************/
 485
 486 void *
 487 malloc(size_t size) {
 488     void        *retval;
 489     retval = malloc_zone_malloc(inline_malloc_default_zone(), size);
 490     if (retval == NULL) {
 491         errno = ENOMEM;
 492     }
 493     return retval;
 494 }
 495
 496 void *
 497 calloc(size_t num_items, size_t size) {
 498     void        *retval;
 499     retval = malloc_zone_calloc(inline_malloc_default_zone(), num_items, size);
 500     if (retval == NULL) {
 501         errno = ENOMEM;
 502     }
 503     return retval;
 504 }
 505
 506 void
 507 free(void *ptr) {
 508     malloc_zone_t       *zone;
 509     if (!ptr) return;
 510     zone = find_registered_zone(ptr, NULL);
 511     if (zone) {
 512         malloc_zone_free(zone, ptr);
 513     } else {
 514         malloc_printf("*** malloc[%d]: Deallocation of a pointer not malloced: %p; This could be a double free(), or free() called with the middle of an allocated block; Try setting environment variable MallocHelp to see tools to help debug\n", getpid(), ptr);
 515         if (malloc_free_abort)
 516             abort();
 517     }
 518 }
 519
 520 void *
 521 realloc(void *old_ptr, size_t new_size) {
 522     void        *retval;
 523     malloc_zone_t       *zone;
 524     size_t      old_size = 0;
 525     if (!old_ptr) {
 526         retval = malloc_zone_malloc(inline_malloc_default_zone(), new_size);
 527     } else {
 528         zone = find_registered_zone(old_ptr, &old_size);
 529         if (zone && (old_size >= new_size)) return old_ptr;
 530         if (!zone) zone = inline_malloc_default_zone();
 531         retval = malloc_zone_realloc(zone, old_ptr, new_size);
 532     }
 533     if (retval == NULL) {
 534         errno = ENOMEM;
 535     }
 536     return retval;
 537 }
 538
 539 void *
 540 valloc(size_t size) {
 541     void        *retval;
 542     malloc_zone_t       *zone = inline_malloc_default_zone();
 543     retval = malloc_zone_valloc(zone, size);
 544     if (retval == NULL) {
 545         errno = ENOMEM;
 546     }
 547     return retval;
 548 }
 549
 550 extern void
 551 vfree(void *ptr) {
 552     free(ptr);
 553 }
 554
 555 size_t
 556 malloc_size(const void *ptr) {
 557     size_t      size = 0;
 558     if (!ptr) return size;
 559     (void)find_registered_zone(ptr, &size);
 560     return size;
 561 }
 562
 563 size_t
 564 malloc_good_size (size_t size) {
 565     malloc_zone_t       *zone = inline_malloc_default_zone();
 566     return zone->introspect->good_size(zone, size);
 567 }
 568
 569 /*********      Batch methods   ************/
 570
 571 unsigned
 572 malloc_zone_batch_malloc(malloc_zone_t *zone, size_t size, void **results, unsigned num_requested) {
 573     unsigned    (*batch_malloc)(malloc_zone_t *, size_t, void **, unsigned) = zone-> batch_malloc;
 574     if (! batch_malloc) return 0;
 575     if (malloc_check_start && (malloc_check_counter++ >= malloc_check_start)) {
 576         internal_check();
 577     }
 578     unsigned    batched = batch_malloc(zone, size, results, num_requested);
 579     if (malloc_logger) {
 580         unsigned        index = 0;
 581         while (index < batched) {
 582             malloc_logger(MALLOC_LOG_TYPE_ALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE, (unsigned)zone, size, 0, (unsigned)results[index], 0);
 583             index++;
 584         }
 585     }
 586     return batched;
 587 }
 588
 589 void
 590 malloc_zone_batch_free(malloc_zone_t *zone, void **to_be_freed, unsigned num) {
 591     if (malloc_check_start && (malloc_check_counter++ >= malloc_check_start)) {
 592         internal_check();
 593     }
 594     if (malloc_logger) {
 595         unsigned        index = 0;
 596         while (index < num) {
 597             malloc_logger(MALLOC_LOG_TYPE_DEALLOCATE | MALLOC_LOG_TYPE_HAS_ZONE, (unsigned)zone, (unsigned)to_be_freed[index], 0, 0, 0);
 598             index++;
 599         }
 600     }
 601     void        (*batch_free)(malloc_zone_t *, void **, unsigned) = zone-> batch_free;
 602     if (batch_free) {
 603         batch_free(zone, to_be_freed, num);
 604     } else {
 605         void    (*free_fun)(malloc_zone_t *, void *) = zone->free;
 606         while (num--) {
 607             void        *ptr = *to_be_freed++;
 608             free_fun(zone, ptr);
 609         }
 610     }
 611 }
 612
 613 /*********      Functions for performance tools ************/
 614
 615 static kern_return_t
 616 _malloc_default_reader(task_t task, vm_address_t address, vm_size_t size, void **ptr) {
 617     *ptr = (void *)address;
 618     return 0;
 619 }
 620
 621 kern_return_t
 622 malloc_get_all_zones(task_t task, memory_reader_t reader, vm_address_t **addresses, unsigned *count) {
 623     // Note that the 2 following addresses are not correct if the address of the target is different from your own.  This notably occurs if the address of System.framework is slid (e.g. different than at B & I )
 624     vm_address_t        remote_malloc_zones = (vm_address_t)&malloc_zones;
 625     vm_address_t        remote_malloc_num_zones = (vm_address_t)&malloc_num_zones;
 626     kern_return_t       err;
 627     vm_address_t        zones_address;
 628     vm_address_t        *zones_address_ref;
 629     unsigned            num_zones;
 630     unsigned            *num_zones_ref;
 631     if (!reader) reader = _malloc_default_reader;
 632     // printf("Read malloc_zones at address %p should be %p\n", &malloc_zones, malloc_zones);
 633     err = reader(task, remote_malloc_zones, sizeof(void *), (void **)&zones_address_ref);
 634     // printf("Read malloc_zones[%p]=%p\n", remote_malloc_zones, *zones_address_ref);
 635     if (err) {
 636         malloc_printf("*** malloc[%d]: malloc_get_all_zones: error reading zones_address at %p\n", getpid(), (unsigned)remote_malloc_zones);
 637         return err;
 638     }
 639     zones_address = *zones_address_ref;
 640     // printf("Reading num_zones at address %p\n", remote_malloc_num_zones);
 641     err = reader(task, remote_malloc_num_zones, sizeof(unsigned), (void **)&num_zones_ref);
 642     if (err) {
 643         malloc_printf("*** malloc[%d]: malloc_get_all_zones: error reading num_zones at %p\n",  getpid(), (unsigned)remote_malloc_num_zones);
 644         return err;
 645     }
 646     num_zones = *num_zones_ref;
 647     // printf("Read malloc_num_zones[%p]=%d\n", remote_malloc_num_zones, num_zones);
 648     *count = num_zones;
 649     // printf("malloc_get_all_zones succesfully found %d zones\n", num_zones);
 650     err = reader(task, zones_address, sizeof(malloc_zone_t *) * num_zones, (void **)addresses);
 651     if (err) {
 652         malloc_printf("*** malloc[%d]: malloc_get_all_zones: error reading zones at %p\n", getpid(), (unsigned)&zones_address);
 653         return err;
 654     }
 655     // printf("malloc_get_all_zones succesfully read %d zones\n", num_zones);
 656     return err;
 657 }
 658
 659 /*********      Debug helpers   ************/
 660
 661 void
 662 malloc_zone_print_ptr_info(void *ptr) {
 663     malloc_zone_t       *zone;
 664     if (!ptr) return;
 665     zone = find_registered_zone(ptr, NULL);
 666     if (zone) {
 667         printf("ptr %p in registered zone %p\n", ptr, zone);
 668     } else {
 669         printf("ptr %p not in heap\n", ptr);
 670     }
 671 }
 672
 673 boolean_t
 674 malloc_zone_check(malloc_zone_t *zone) {
 675     boolean_t   ok = 1;
 676     if (!zone) {
 677         unsigned        index = 0;
 678         while (index < malloc_num_zones) {
 679             zone = malloc_zones[index++];
 680             if (!zone->introspect->check(zone)) ok = 0;
 681         }
 682     } else {
 683         ok = zone->introspect->check(zone);
 684     }
 685     return ok;
 686 }
 687
 688 void
 689 malloc_zone_print(malloc_zone_t *zone, boolean_t verbose) {
 690     if (!zone) {
 691         unsigned        index = 0;
 692         while (index < malloc_num_zones) {
 693             zone = malloc_zones[index++];
 694             zone->introspect->print(zone, verbose);
 695         }
 696     } else {
 697         zone->introspect->print(zone, verbose);
 698     }
 699 }
 700
 701 void
 702 malloc_zone_statistics(malloc_zone_t *zone, malloc_statistics_t *stats) {
 703     if (!zone) {
 704         memset(stats, 0, sizeof(stats));
 705         unsigned        index = 0;
 706         while (index < malloc_num_zones) {
 707             zone = malloc_zones[index++];
 708             malloc_statistics_t this_stats;
 709             zone->introspect->statistics(zone, &this_stats);
 710             stats->blocks_in_use += this_stats.blocks_in_use;
 711             stats->size_in_use += this_stats.size_in_use;
 712             stats->max_size_in_use += this_stats.max_size_in_use;
 713             stats->size_allocated += this_stats.size_allocated;
 714         }
 715     } else {
 716         zone->introspect->statistics(zone, stats);
 717     }
 718 }
 719
 720 void
 721 malloc_zone_log(malloc_zone_t *zone, void *address) {
 722     if (!zone) {
 723         unsigned        index = 0;
 724         while (index < malloc_num_zones) {
 725             zone = malloc_zones[index++];
 726             zone->introspect->log(zone, address);
 727         }
 728     } else {
 729         zone->introspect->log(zone, address);
 730     }
 731 }
 732
 733 /*********      Misc other entry points ************/
 734
 735 static void
 736 DefaultMallocError(int x) {
 737     malloc_printf("*** malloc[%d]: error %d\n", getpid(), x);
 738 #if USE_SLEEP_RATHER_THAN_ABORT
 739     sleep(3600);
 740 #else
 741     abort();
 742 #endif
 743 }
 744
 745 void (*
 746 malloc_error(void (*func)(int)))(int) {
 747     return DefaultMallocError;
 748 }
 749
 750 void
 751 _malloc_fork_prepare() {
 752     /* Prepare the malloc module for a fork by insuring that no thread is in a malloc critical section */
 753     unsigned    index = 0;
 754     MALLOC_LOCK();
 755     while (index < malloc_num_zones) {
 756         malloc_zone_t   *zone = malloc_zones[index++];
 757         zone->introspect->force_lock(zone);
 758     }
 759 }
 760
 761 void
 762 _malloc_fork_parent() {
 763     /* Called in the parent process after a fork() to resume normal operation. */
 764     unsigned    index = 0;
 765     MALLOC_UNLOCK();
 766     while (index < malloc_num_zones) {
 767         malloc_zone_t   *zone = malloc_zones[index++];
 768         zone->introspect->force_unlock(zone);
 769     }
 770 }
 771
 772 void
 773 _malloc_fork_child() {
 774     /* Called in the child process after a fork() to resume normal operation.  In the MTASK case we also have to change memory inheritance so that the child does not share memory with the parent. */
 775     unsigned    index = 0;
 776     MALLOC_UNLOCK();
 777     while (index < malloc_num_zones) {
 778         malloc_zone_t   *zone = malloc_zones[index++];
 779         zone->introspect->force_unlock(zone);
 780     }
 781 }
 782
 783 size_t
 784 mstats(void) {
 785     malloc_zone_print(NULL, 0);
 786     return 1;
 787 }
 788
 789 /*****************      OBSOLETE ENTRY POINTS   ********************/
 790
 791 #if PHASE_OUT_OLD_MALLOC
 792 #error PHASE OUT THE FOLLOWING FUNCTIONS
 793 #else
 794 #warning PHASE OUT THE FOLLOWING FUNCTIONS
 795 #endif
 796
 797 void
 798 set_malloc_singlethreaded(boolean_t single) {
 799     static boolean_t warned = 0;
 800     if (!warned) {
 801 #if PHASE_OUT_OLD_MALLOC
 802         malloc_printf("*** malloc[%d]: OBSOLETE: set_malloc_singlethreaded(%d)\n", getpid(), single);
 803 #endif
 804         warned = 1;
 805     }
 806 }
 807
 808 void
 809 malloc_singlethreaded() {
 810     static boolean_t warned = 0;
 811     if (!warned) {
 812         malloc_printf("*** malloc[%d]: OBSOLETE: malloc_singlethreaded()\n", getpid());
 813         warned = 1;
 814     }
 815 }
 816
 817 int
 818 malloc_debug(int level) {
 819     malloc_printf("*** malloc[%d]: OBSOLETE: malloc_debug()\n", getpid());
 820     return 0;
 821 }