2 * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
4 * @APPLE_LICENSE_HEADER_START@
6 * The contents of this file constitute Original Code as defined in and
7 * are subject to the Apple Public Source License Version 1.1 (the
8 * "License"). You may not use this file except in compliance with the
9 * License. Please obtain a copy of the License at
10 * http://www.apple.com/publicsource and read it before using this file.
12 * This Original Code and all software distributed under the License are
13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17 * License for the specific language governing rights and limitations
20 * @APPLE_LICENSE_HEADER_END@
29 #import <pthread_internals.h> // for spin lock
30 #import <objc/malloc.h>
31 #include <crt_externs.h>
33 #import "scalable_malloc.h"
34 #import "stack_logging.h"
36 #define USE_SLEEP_RATHER_THAN_ABORT 0
38 #define MAX_ALLOCATION 0xc0000000 // beyond this, assume a programming error
39 #define INITIAL_ZONES 8 // After this number, we reallocate for new zones
41 typedef void (malloc_logger_t
)(unsigned type
, unsigned arg1
, unsigned arg2
, unsigned arg3
, unsigned result
, unsigned num_hot_frames_to_skip
);
43 static pthread_lock_t _malloc_lock
;
44 static malloc_zone_t
*initial_malloc_zones
[INITIAL_ZONES
] = {0};
46 /* The following variables are exported for the benefit of performance tools */
47 unsigned malloc_num_zones
= 0;
48 malloc_zone_t
**malloc_zones
= initial_malloc_zones
;
49 malloc_logger_t
*malloc_logger
= NULL
;
51 unsigned malloc_debug_flags
= 0;
53 unsigned malloc_check_start
= 0; // 0 means don't check
54 unsigned malloc_check_counter
= 0;
55 unsigned malloc_check_each
= 1000;
57 #define MALLOC_LOCK() LOCK(_malloc_lock)
58 #define MALLOC_UNLOCK() UNLOCK(_malloc_lock)
60 #define MALLOC_LOG_TYPE_ALLOCATE stack_logging_type_alloc
61 #define MALLOC_LOG_TYPE_DEALLOCATE stack_logging_type_dealloc
62 #define MALLOC_LOG_TYPE_HAS_ZONE stack_logging_flag_zone
63 #define MALLOC_LOG_TYPE_CLEARED stack_logging_flag_cleared
65 /********* Utilities ************/
67 static inline malloc_zone_t
*find_registered_zone(const void *ptr
, size_t *returned_size
) {
68 // locates the proper zone
69 // if zone found fills returnedSize; else returns NULL
70 // See comment in malloc_zone_register() about clients non locking to call this function
71 // Speed is critical for this function
72 unsigned index
= malloc_num_zones
;
73 malloc_zone_t
**zones
= malloc_zones
;
75 malloc_zone_t
*zone
= *zones
++;
77 size
= zone
->size(zone
, ptr
);
79 if (returned_size
) *returned_size
= size
;
86 /********* Creation and destruction ************/
88 static void _malloc_initialize(void) {
89 // guaranteed to be called only once
90 (void)malloc_create_zone(0, 0);
91 malloc_set_zone_name(malloc_zones
[0], "DefaultMallocZone");
92 LOCK_INIT(_malloc_lock
);
93 // malloc_printf("Malloc: %d registered zones\n", malloc_num_zones);
94 // malloc_printf("malloc: malloc_zones is at 0x%x; malloc_num_zones is at 0x%x\n", (unsigned)&malloc_zones, (unsigned)&malloc_num_zones);
97 static inline malloc_zone_t
*inline_malloc_default_zone(void) {
98 if (!malloc_num_zones
) _malloc_initialize();
99 // malloc_printf("In inline_malloc_default_zone with %d %d\n", malloc_num_zones, malloc_has_debug_zone);
100 return malloc_zones
[0];
103 malloc_zone_t
*malloc_default_zone(void) {
104 return inline_malloc_default_zone();
107 static void set_flags_from_environment(void) {
109 if (getenv("MallocGuardEdges")) {
110 malloc_debug_flags
= SCALABLE_MALLOC_ADD_GUARD_PAGES
;
111 malloc_printf("malloc[%d]: protecting edges\n", getpid());
112 if (getenv("MallocDoNotProtectPrelude")) {
113 malloc_debug_flags
|= SCALABLE_MALLOC_DONT_PROTECT_PRELUDE
;
114 malloc_printf("malloc[%d]: ... but not protecting prelude guard page\n", getpid());
116 if (getenv("MallocDoNotProtectPostlude")) {
117 malloc_debug_flags
|= SCALABLE_MALLOC_DONT_PROTECT_POSTLUDE
;
118 malloc_printf("malloc[%d]: ... but not protecting postlude guard page\n", getpid());
121 flag
= getenv("MallocStackLogging");
123 flag
= getenv("MallocStackLoggingNoCompact");
124 stack_logging_dontcompact
= 1;
127 unsigned val
= strtoul(flag
, NULL
, 0);
128 if (val
== 1) val
= 0;
129 if (val
== -1) val
= 0;
130 malloc_logger
= (val
) ? (void *)val
: stack_logging_log_stack
;
131 stack_logging_enable_logging
= 1;
132 if (malloc_logger
== stack_logging_log_stack
) {
133 malloc_printf("malloc[%d]: recording stacks using standard recorder\n", getpid());
135 malloc_printf("malloc[%d]: recording stacks using recorder %p\n", getpid(), malloc_logger
);
137 if (stack_logging_dontcompact
) malloc_printf("malloc[%d]: stack logging compaction turned off; VM can increase rapidly\n", getpid());
139 if (getenv("MallocScribble")) {
140 malloc_debug_flags
|= SCALABLE_MALLOC_DO_SCRIBBLE
;
141 malloc_printf("malloc[%d]: enabling scribbling to detect mods to free blocks\n", getpid());
143 flag
= getenv("MallocCheckHeapStart");
145 malloc_check_start
= strtoul(flag
, NULL
, 0);
146 if (malloc_check_start
== 0) malloc_check_start
= 1;
147 if (malloc_check_start
== -1) malloc_check_start
= 1;
148 flag
= getenv("MallocCheckHeapEach");
150 malloc_check_each
= strtoul(flag
, NULL
, 0);
151 if (malloc_check_each
== 0) malloc_check_each
= 1;
152 if (malloc_check_each
== -1) malloc_check_each
= 1;
154 malloc_printf("malloc[%d]: checks heap after %dth operation and each %d operations\n", getpid(), malloc_check_start
, malloc_check_each
);
156 if (getenv("MallocHelp")) {
158 "malloc[%d]: environment variables that can be set for debug:\n"
159 "- MallocGuardEdges to add 2 guard pages for each large block\n"
160 "- MallocDoNotProtectPrelude to disable protection (when previous flag set)\n"
161 "- MallocDoNotProtectPostlude to disable protection (when previous flag set)\n"
162 "- MallocStackLogging to record all stacks. Tools like leaks can then be applied\n"
163 "- MallocStackLoggingNoCompact to record all stacks. Needed for malloc_history\n"
164 "- MallocScribble to detect writing on free blocks: 0x55 is written upon free\n"
165 "- MallocCheckHeapStart <n> to check the heap from time to time after <n> operations \n"
166 "- MallocHelp - this help!\n", getpid());
170 malloc_zone_t
*malloc_create_zone(vm_size_t start_size
, unsigned flags
) {
172 if (!malloc_num_zones
) {
173 char **env
= * _NSGetEnviron();
176 /* Given that all environment variables start with "Malloc" we optimize by scanning quickly first the environment, therefore avoiding repeated calls to getenv() */
177 for (p
= env
; (c
= *p
) != NULL
; ++p
) {
178 if (!strncmp(c
, "Malloc", 6)) {
179 set_flags_from_environment();
185 zone
= create_scalable_zone(start_size
, malloc_debug_flags
);
186 malloc_zone_register(zone
);
190 void malloc_destroy_zone(malloc_zone_t
*zone
) {
191 malloc_zone_unregister(zone
);
195 /********* Block creation and manipulation ************/
197 static void internal_check(void) {
198 static vm_address_t
*frames
= NULL
;
199 static unsigned num_frames
;
200 if (malloc_zone_check(NULL
)) {
201 malloc_printf("MallocCheckHeap: PASSED check at %dth operation\n", malloc_check_counter
-1);
202 if (!frames
) vm_allocate(mach_task_self(), (void *)&frames
, vm_page_size
, 1);
203 thread_stack_pcs(frames
, vm_page_size
/sizeof(vm_address_t
) - 1, &num_frames
);
205 malloc_printf("*** MallocCheckHeap: FAILED check at %dth operation\n", malloc_check_counter
-1);
208 malloc_printf("Stack for last operation where the malloc check succeeded: ");
209 while (index
< num_frames
) malloc_printf("0x%x ", frames
[index
++]);
210 malloc_printf("\n(Use 'atos' for a symbolic stack)\n");
212 if (malloc_check_each
> 1) {
213 unsigned recomm_each
= (malloc_check_each
> 10) ? malloc_check_each
/10 : 1;
214 unsigned recomm_start
= (malloc_check_counter
> malloc_check_each
+1) ? malloc_check_counter
-1-malloc_check_each
: 1;
215 malloc_printf("*** Recommend using 'setenv MallocCheckHeapStart %d; setenv MallocCheckHeapEach %d' to narrow down failure\n", recomm_start
, recomm_each
);
217 malloc_printf("*** Sleeping for 100 seconds to leave time to attach\n");
220 malloc_check_start
+= malloc_check_each
;
223 void *malloc_zone_malloc(malloc_zone_t
*zone
, size_t size
) {
225 if ((unsigned)size
>= MAX_ALLOCATION
) {
226 /* Probably a programming error */
227 fprintf(stderr
, "*** malloc_zone_malloc[%d]: argument too large: %d\n", getpid(), (unsigned)size
);
230 if (malloc_check_start
&& (malloc_check_counter
++ >= malloc_check_start
)) {
233 ptr
= zone
->malloc(zone
, size
);
234 if (malloc_logger
) malloc_logger(MALLOC_LOG_TYPE_ALLOCATE
| MALLOC_LOG_TYPE_HAS_ZONE
, (unsigned)zone
, size
, 0, (unsigned)ptr
, 0);
238 void *malloc_zone_calloc(malloc_zone_t
*zone
, size_t num_items
, size_t size
) {
240 if (malloc_check_start
&& (malloc_check_counter
++ >= malloc_check_start
)) {
243 if (((unsigned)num_items
>= MAX_ALLOCATION
) || ((unsigned)size
>= MAX_ALLOCATION
) || ((long long)size
* num_items
>= (long long) MAX_ALLOCATION
)) {
244 /* Probably a programming error */
245 fprintf(stderr
, "*** malloc_zone_calloc[%d]: arguments too large: %d,%d\n", getpid(), (unsigned)num_items
, (unsigned)size
);
248 ptr
= zone
->calloc(zone
, num_items
, size
);
249 if (malloc_logger
) malloc_logger(MALLOC_LOG_TYPE_ALLOCATE
| MALLOC_LOG_TYPE_HAS_ZONE
| MALLOC_LOG_TYPE_CLEARED
, (unsigned)zone
, num_items
* size
, 0, (unsigned)ptr
, 0);
253 void *malloc_zone_valloc(malloc_zone_t
*zone
, size_t size
) {
255 if ((unsigned)size
>= MAX_ALLOCATION
) {
256 /* Probably a programming error */
257 fprintf(stderr
, "*** malloc_zone_valloc[%d]: argument too large: %d\n", getpid(), (unsigned)size
);
260 if (malloc_check_start
&& (malloc_check_counter
++ >= malloc_check_start
)) {
263 ptr
= zone
->valloc(zone
, size
);
264 if (malloc_logger
) malloc_logger(MALLOC_LOG_TYPE_ALLOCATE
| MALLOC_LOG_TYPE_HAS_ZONE
, (unsigned)zone
, size
, 0, (unsigned)ptr
, 0);
268 void *malloc_zone_realloc(malloc_zone_t
*zone
, void *ptr
, size_t size
) {
270 if (malloc_check_start
&& (malloc_check_counter
++ >= malloc_check_start
)) {
273 new_ptr
= zone
->realloc(zone
, ptr
, size
);
274 if (malloc_logger
) malloc_logger(MALLOC_LOG_TYPE_ALLOCATE
| MALLOC_LOG_TYPE_DEALLOCATE
| MALLOC_LOG_TYPE_HAS_ZONE
, (unsigned)zone
, (unsigned)ptr
, size
, (unsigned)new_ptr
, 0);
278 void malloc_zone_free(malloc_zone_t
*zone
, void *ptr
) {
279 if (malloc_logger
) malloc_logger(MALLOC_LOG_TYPE_DEALLOCATE
| MALLOC_LOG_TYPE_HAS_ZONE
, (unsigned)zone
, (unsigned)ptr
, 0, 0, 0);
280 if (malloc_check_start
&& (malloc_check_counter
++ >= malloc_check_start
)) {
283 zone
->free(zone
, ptr
);
286 malloc_zone_t
*malloc_zone_from_ptr(const void *ptr
) {
288 if (!ptr
) return NULL
;
289 zone
= find_registered_zone(ptr
, NULL
);
293 /********* Functions for zone implementors ************/
295 void malloc_zone_register(malloc_zone_t
*zone
) {
296 /* Note that given the sequencing it is always safe to first get the number of zones, then get malloc_zones without taking the lock, if all you need is to iterate through the list */
298 if (malloc_num_zones
>= INITIAL_ZONES
) {
299 malloc_zone_t
**zones
= malloc_zones
;
300 malloc_zone_t
*pzone
= malloc_zones
[0];
301 boolean_t copy
= malloc_num_zones
== INITIAL_ZONES
;
302 if (copy
) zones
= NULL
; // to avoid realloc on something not allocated
304 zones
= pzone
->realloc(pzone
, zones
, (malloc_num_zones
+ 1) * sizeof(malloc_zone_t
*)); // we leak initial_malloc_zones, not worth tracking it
306 if (copy
) memcpy(zones
, malloc_zones
, malloc_num_zones
* sizeof(malloc_zone_t
*));
307 malloc_zones
= zones
;
309 malloc_zones
[malloc_num_zones
] = zone
;
310 malloc_num_zones
++; // note that we do this after setting malloc_num_zones, so enumerations without taking the lock are safe
312 // malloc_printf("Registered %p malloc_zones at address %p is %p [%d zones]\n", zone, &malloc_zones, malloc_zones, malloc_num_zones);
315 void malloc_zone_unregister(malloc_zone_t
*z
) {
318 index
= malloc_num_zones
;
320 malloc_zone_t
*zone
= malloc_zones
[index
];
322 malloc_zones
[index
] = malloc_zones
[--malloc_num_zones
];
328 fprintf(stderr
, "*** malloc[%d]: malloc_zone_unregister() failed for %p\n", getpid(), z
);
331 void malloc_set_zone_name(malloc_zone_t
*z
, const char *name
) {
334 free((char *)z
->zone_name
);
337 newName
= malloc_zone_malloc(z
, strlen(name
) + 1);
338 strcpy(newName
, name
);
339 z
->zone_name
= (const char *)newName
;
342 const char *malloc_get_zone_name(malloc_zone_t
*zone
) {
343 return zone
->zone_name
;
346 static char *_malloc_append_unsigned(unsigned value
, unsigned base
, char *head
) {
350 if (value
>= base
) head
= _malloc_append_unsigned(value
/ base
, base
, head
);
351 value
= value
% base
;
352 head
[0] = (value
< 10) ? '0' + value
: 'a' + value
- 10;
357 void malloc_printf(const char *format
, ...) {
363 va_start(args
, format
);
365 head
= _malloc_append_unsigned(((unsigned)&args
) >> 12, 16, head
);
369 while (ch
= *format
++) {
373 char *str
= (char *)(*nums
++);
374 write(2, buf
, head
- buf
);
376 write(2, str
, strlen(str
));
379 *head
++ = '0'; *head
++ = 'x';
381 head
= _malloc_append_unsigned(*nums
++, (ch
== 'd') ? 10 : 16, head
);
387 write(2, buf
, head
- buf
); fflush(stderr
);
391 /********* Generic ANSI callouts ************/
393 void *malloc(size_t size
) {
394 return malloc_zone_malloc(inline_malloc_default_zone(), size
);
397 void *calloc(size_t num_items
, size_t size
) {
398 return malloc_zone_calloc(inline_malloc_default_zone(), num_items
, size
);
401 void free(void *ptr
) {
404 zone
= find_registered_zone(ptr
, NULL
);
406 malloc_zone_free(zone
, ptr
);
408 fprintf(stderr
, "*** malloc[%d]: Deallocation of a pointer not malloced: %p; This could be a double free(), or free() called with the middle of an allocated block; Try setting environment variable MallocHelp to see tools to help debug\n", getpid(), ptr
);
412 void *realloc(void *old_ptr
, size_t new_size
) {
415 if (!old_ptr
) return malloc_zone_malloc(inline_malloc_default_zone(), new_size
);
416 zone
= find_registered_zone(old_ptr
, &old_size
);
417 if (zone
&& (old_size
>= new_size
)) return old_ptr
;
418 if (!zone
) zone
= inline_malloc_default_zone();
419 return malloc_zone_realloc(zone
, old_ptr
, new_size
);
422 void *valloc(size_t size
) {
423 malloc_zone_t
*zone
= inline_malloc_default_zone();
424 return malloc_zone_valloc(zone
, size
);
427 extern void vfree(void *ptr
) {
431 size_t malloc_size(const void *ptr
) {
433 if (!ptr
) return size
;
434 (void)find_registered_zone(ptr
, &size
);
438 size_t malloc_good_size (size_t size
) {
439 malloc_zone_t
*zone
= inline_malloc_default_zone();
440 return zone
->introspect
->good_size(zone
, size
);
443 /********* Functions for performance tools ************/
445 static kern_return_t
_malloc_default_reader(task_t task
, vm_address_t address
, vm_size_t size
, void **ptr
) {
446 *ptr
= (void *)address
;
450 kern_return_t
malloc_get_all_zones(task_t task
, memory_reader_t reader
, vm_address_t
**addresses
, unsigned *count
) {
451 // Note that the 2 following addresses are not correct if the address of the target is different from your own. This notably occurs if the address of System.framework is slid (e.g. different than at B & I )
452 vm_address_t remote_malloc_zones
= (vm_address_t
)&malloc_zones
;
453 vm_address_t remote_malloc_num_zones
= (vm_address_t
)&malloc_num_zones
;
455 vm_address_t zones_address
;
456 vm_address_t
*zones_address_ref
;
458 unsigned *num_zones_ref
;
459 if (!reader
) reader
= _malloc_default_reader
;
460 // printf("Read malloc_zones at address %p should be %p\n", &malloc_zones, malloc_zones);
461 err
= reader(task
, remote_malloc_zones
, sizeof(void *), (void **)&zones_address_ref
);
462 // printf("Read malloc_zones[0x%x]=%p\n", remote_malloc_zones, *zones_address_ref);
464 fprintf(stderr
, "*** malloc[%d]: malloc_get_all_zones: error reading zones_address at 0x%x\n", getpid(), (unsigned)remote_malloc_zones
);
467 zones_address
= *zones_address_ref
;
468 // printf("Reading num_zones at address %p\n", remote_malloc_num_zones);
469 err
= reader(task
, remote_malloc_num_zones
, sizeof(unsigned), (void **)&num_zones_ref
);
471 fprintf(stderr
, "*** malloc[%d]: malloc_get_all_zones: error reading num_zones at 0x%x\n", getpid(), (unsigned)remote_malloc_num_zones
);
474 num_zones
= *num_zones_ref
;
475 // printf("Read malloc_num_zones[0x%x]=%d\n", remote_malloc_num_zones, num_zones);
477 // printf("malloc_get_all_zones succesfully found %d zones\n", num_zones);
478 err
= reader(task
, zones_address
, sizeof(malloc_zone_t
*) * num_zones
, (void **)addresses
);
480 fprintf(stderr
, "*** malloc[%d]: malloc_get_all_zones: error reading zones at 0x%x\n", getpid(), (unsigned)&zones_address
);
483 // printf("malloc_get_all_zones succesfully read %d zones\n", num_zones);
487 /********* Debug helpers ************/
489 void malloc_zone_print_ptr_info(void *ptr
) {
492 zone
= find_registered_zone(ptr
, NULL
);
494 printf("ptr %p in registered zone %p\n", ptr
, zone
);
496 printf("ptr %p not in heap\n", ptr
);
500 boolean_t
malloc_zone_check(malloc_zone_t
*zone
) {
504 while (index
< malloc_num_zones
) {
505 zone
= malloc_zones
[index
++];
506 if (!zone
->introspect
->check(zone
)) ok
= 0;
509 ok
= zone
->introspect
->check(zone
);
514 void malloc_zone_print(malloc_zone_t
*zone
, boolean_t verbose
) {
517 while (index
< malloc_num_zones
) {
518 zone
= malloc_zones
[index
++];
519 zone
->introspect
->print(zone
, verbose
);
522 zone
->introspect
->print(zone
, verbose
);
526 void malloc_zone_log(malloc_zone_t
*zone
, void *address
) {
529 while (index
< malloc_num_zones
) {
530 zone
= malloc_zones
[index
++];
531 zone
->introspect
->log(zone
, address
);
534 zone
->introspect
->log(zone
, address
);
538 /********* Misc other entry points ************/
540 static void DefaultMallocError(int x
) {
541 fprintf(stderr
, "*** malloc[%d]: error %d\n", getpid(), x
);
542 #if USE_SLEEP_RATHER_THAN_ABORT
549 void (*malloc_error(void (*func
)(int)))(int) {
550 return DefaultMallocError
;
553 void _malloc_fork_prepare() {
554 /* Prepare the malloc module for a fork by insuring that no thread is in a malloc critical section */
557 while (index
< malloc_num_zones
) {
558 malloc_zone_t
*zone
= malloc_zones
[index
++];
559 zone
->introspect
->force_lock(zone
);
563 void _malloc_fork_parent() {
564 /* Called in the parent process after a fork() to resume normal operation. */
567 while (index
< malloc_num_zones
) {
568 malloc_zone_t
*zone
= malloc_zones
[index
++];
569 zone
->introspect
->force_unlock(zone
);
573 void _malloc_fork_child() {
574 /* Called in the child process after a fork() to resume normal operation. In the MTASK case we also have to change memory inheritance so that the child does not share memory with the parent. */
577 while (index
< malloc_num_zones
) {
578 malloc_zone_t
*zone
= malloc_zones
[index
++];
579 zone
->introspect
->force_unlock(zone
);
583 size_t mstats(void) {
584 malloc_zone_print(NULL
, 0);
588 /***************** OBSOLETE ENTRY POINTS ********************/
590 #if PHASE_OUT_OLD_MALLOC
591 #error PHASE OUT THE FOLLOWING FUNCTIONS
593 #warning PHASE OUT THE FOLLOWING FUNCTIONS
596 void set_malloc_singlethreaded(boolean_t single
) {
597 static boolean_t warned
= 0;
599 #if PHASE_OUT_OLD_MALLOC
600 fprintf(stderr
, "*** malloc[%d]: OBSOLETE: set_malloc_singlethreaded(%d)\n", getpid(), single
);
606 void malloc_singlethreaded() {
607 static boolean_t warned
= 0;
609 fprintf(stderr
, "*** malloc[%d]: OBSOLETE: malloc_singlethreaded()\n", getpid());
614 int malloc_debug(int level
) {
615 fprintf(stderr
, "*** malloc[%d]: OBSOLETE: malloc_debug()\n", getpid());