2 * Copyright (c) 1999 Apple Computer, Inc. All rights reserved.
4 * @APPLE_LICENSE_HEADER_START@
6 * Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved.
8 * This file contains Original Code and/or Modifications of Original Code
9 * as defined in and that are subject to the Apple Public Source License
10 * Version 2.0 (the 'License'). You may not use this file except in
11 * compliance with the License. Please obtain a copy of the License at
12 * http://www.opensource.apple.com/apsl/ and read it before using this
15 * The Original Code and all software distributed under the License are
16 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
17 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
18 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
20 * Please see the License for the specific language governing rights and
21 * limitations under the License.
23 * @APPLE_LICENSE_HEADER_END@
32 #import <pthread_internals.h> // for spin lock
33 #import <objc/malloc.h>
34 #include <crt_externs.h>
36 #import "scalable_malloc.h"
37 #import "stack_logging.h"
39 #define USE_SLEEP_RATHER_THAN_ABORT 0
41 #define MAX_ALLOCATION 0xc0000000 // beyond this, assume a programming error
42 #define INITIAL_ZONES 8 // After this number, we reallocate for new zones
44 typedef void (malloc_logger_t
)(unsigned type
, unsigned arg1
, unsigned arg2
, unsigned arg3
, unsigned result
, unsigned num_hot_frames_to_skip
);
46 static pthread_lock_t _malloc_lock
;
47 static malloc_zone_t
*initial_malloc_zones
[INITIAL_ZONES
] = {0};
49 /* The following variables are exported for the benefit of performance tools */
50 unsigned malloc_num_zones
= 0;
51 malloc_zone_t
**malloc_zones
= initial_malloc_zones
;
52 malloc_logger_t
*malloc_logger
= NULL
;
54 unsigned malloc_debug_flags
= 0;
56 unsigned malloc_check_start
= 0; // 0 means don't check
57 unsigned malloc_check_counter
= 0;
58 unsigned malloc_check_each
= 1000;
60 #define MALLOC_LOCK() LOCK(_malloc_lock)
61 #define MALLOC_UNLOCK() UNLOCK(_malloc_lock)
63 #define MALLOC_LOG_TYPE_ALLOCATE stack_logging_type_alloc
64 #define MALLOC_LOG_TYPE_DEALLOCATE stack_logging_type_dealloc
65 #define MALLOC_LOG_TYPE_HAS_ZONE stack_logging_flag_zone
66 #define MALLOC_LOG_TYPE_CLEARED stack_logging_flag_cleared
68 /********* Utilities ************/
70 static inline malloc_zone_t
*find_registered_zone(const void *ptr
, size_t *returned_size
) {
71 // locates the proper zone
72 // if zone found fills returnedSize; else returns NULL
73 // See comment in malloc_zone_register() about clients non locking to call this function
74 // Speed is critical for this function
75 unsigned index
= malloc_num_zones
;
76 malloc_zone_t
**zones
= malloc_zones
;
78 malloc_zone_t
*zone
= *zones
++;
80 size
= zone
->size(zone
, ptr
);
82 if (returned_size
) *returned_size
= size
;
89 /********* Creation and destruction ************/
91 static void _malloc_initialize(void) {
92 // guaranteed to be called only once
93 (void)malloc_create_zone(0, 0);
94 malloc_set_zone_name(malloc_zones
[0], "DefaultMallocZone");
95 LOCK_INIT(_malloc_lock
);
96 // malloc_printf("Malloc: %d registered zones\n", malloc_num_zones);
97 // malloc_printf("malloc: malloc_zones is at 0x%x; malloc_num_zones is at 0x%x\n", (unsigned)&malloc_zones, (unsigned)&malloc_num_zones);
100 static inline malloc_zone_t
*inline_malloc_default_zone(void) {
101 if (!malloc_num_zones
) _malloc_initialize();
102 // malloc_printf("In inline_malloc_default_zone with %d %d\n", malloc_num_zones, malloc_has_debug_zone);
103 return malloc_zones
[0];
106 malloc_zone_t
*malloc_default_zone(void) {
107 return inline_malloc_default_zone();
110 static void set_flags_from_environment(void) {
112 if (getenv("MallocGuardEdges")) {
113 malloc_debug_flags
= SCALABLE_MALLOC_ADD_GUARD_PAGES
;
114 malloc_printf("malloc[%d]: protecting edges\n", getpid());
115 if (getenv("MallocDoNotProtectPrelude")) {
116 malloc_debug_flags
|= SCALABLE_MALLOC_DONT_PROTECT_PRELUDE
;
117 malloc_printf("malloc[%d]: ... but not protecting prelude guard page\n", getpid());
119 if (getenv("MallocDoNotProtectPostlude")) {
120 malloc_debug_flags
|= SCALABLE_MALLOC_DONT_PROTECT_POSTLUDE
;
121 malloc_printf("malloc[%d]: ... but not protecting postlude guard page\n", getpid());
124 flag
= getenv("MallocStackLogging");
126 flag
= getenv("MallocStackLoggingNoCompact");
127 stack_logging_dontcompact
= 1;
130 unsigned val
= strtoul(flag
, NULL
, 0);
131 if (val
== 1) val
= 0;
132 if (val
== -1) val
= 0;
133 malloc_logger
= (val
) ? (void *)val
: stack_logging_log_stack
;
134 stack_logging_enable_logging
= 1;
135 if (malloc_logger
== stack_logging_log_stack
) {
136 malloc_printf("malloc[%d]: recording stacks using standard recorder\n", getpid());
138 malloc_printf("malloc[%d]: recording stacks using recorder %p\n", getpid(), malloc_logger
);
140 if (stack_logging_dontcompact
) malloc_printf("malloc[%d]: stack logging compaction turned off; VM can increase rapidly\n", getpid());
142 if (getenv("MallocScribble")) {
143 malloc_debug_flags
|= SCALABLE_MALLOC_DO_SCRIBBLE
;
144 malloc_printf("malloc[%d]: enabling scribbling to detect mods to free blocks\n", getpid());
146 flag
= getenv("MallocCheckHeapStart");
148 malloc_check_start
= strtoul(flag
, NULL
, 0);
149 if (malloc_check_start
== 0) malloc_check_start
= 1;
150 if (malloc_check_start
== -1) malloc_check_start
= 1;
151 flag
= getenv("MallocCheckHeapEach");
153 malloc_check_each
= strtoul(flag
, NULL
, 0);
154 if (malloc_check_each
== 0) malloc_check_each
= 1;
155 if (malloc_check_each
== -1) malloc_check_each
= 1;
157 malloc_printf("malloc[%d]: checks heap after %dth operation and each %d operations\n", getpid(), malloc_check_start
, malloc_check_each
);
159 if (getenv("MallocHelp")) {
161 "malloc[%d]: environment variables that can be set for debug:\n"
162 "- MallocGuardEdges to add 2 guard pages for each large block\n"
163 "- MallocDoNotProtectPrelude to disable protection (when previous flag set)\n"
164 "- MallocDoNotProtectPostlude to disable protection (when previous flag set)\n"
165 "- MallocStackLogging to record all stacks. Tools like leaks can then be applied\n"
166 "- MallocStackLoggingNoCompact to record all stacks. Needed for malloc_history\n"
167 "- MallocScribble to detect writing on free blocks: 0x55 is written upon free\n"
168 "- MallocCheckHeapStart <n> to check the heap from time to time after <n> operations \n"
169 "- MallocHelp - this help!\n", getpid());
173 malloc_zone_t
*malloc_create_zone(vm_size_t start_size
, unsigned flags
) {
175 if (!malloc_num_zones
) {
176 char **env
= * _NSGetEnviron();
179 /* Given that all environment variables start with "Malloc" we optimize by scanning quickly first the environment, therefore avoiding repeated calls to getenv() */
180 for (p
= env
; (c
= *p
) != NULL
; ++p
) {
181 if (!strncmp(c
, "Malloc", 6)) {
182 set_flags_from_environment();
188 zone
= create_scalable_zone(start_size
, malloc_debug_flags
);
189 malloc_zone_register(zone
);
193 void malloc_destroy_zone(malloc_zone_t
*zone
) {
194 malloc_zone_unregister(zone
);
198 /********* Block creation and manipulation ************/
200 static void internal_check(void) {
201 static vm_address_t
*frames
= NULL
;
202 static unsigned num_frames
;
203 if (malloc_zone_check(NULL
)) {
204 malloc_printf("MallocCheckHeap: PASSED check at %dth operation\n", malloc_check_counter
-1);
205 if (!frames
) vm_allocate(mach_task_self(), (void *)&frames
, vm_page_size
, 1);
206 thread_stack_pcs(frames
, vm_page_size
/sizeof(vm_address_t
) - 1, &num_frames
);
208 malloc_printf("*** MallocCheckHeap: FAILED check at %dth operation\n", malloc_check_counter
-1);
211 malloc_printf("Stack for last operation where the malloc check succeeded: ");
212 while (index
< num_frames
) malloc_printf("0x%x ", frames
[index
++]);
213 malloc_printf("\n(Use 'atos' for a symbolic stack)\n");
215 if (malloc_check_each
> 1) {
216 unsigned recomm_each
= (malloc_check_each
> 10) ? malloc_check_each
/10 : 1;
217 unsigned recomm_start
= (malloc_check_counter
> malloc_check_each
+1) ? malloc_check_counter
-1-malloc_check_each
: 1;
218 malloc_printf("*** Recommend using 'setenv MallocCheckHeapStart %d; setenv MallocCheckHeapEach %d' to narrow down failure\n", recomm_start
, recomm_each
);
220 malloc_printf("*** Sleeping for 100 seconds to leave time to attach\n");
223 malloc_check_start
+= malloc_check_each
;
226 void *malloc_zone_malloc(malloc_zone_t
*zone
, size_t size
) {
228 if ((unsigned)size
>= MAX_ALLOCATION
) {
229 /* Probably a programming error */
230 fprintf(stderr
, "*** malloc_zone_malloc[%d]: argument too large: %d\n", getpid(), (unsigned)size
);
233 if (malloc_check_start
&& (malloc_check_counter
++ >= malloc_check_start
)) {
236 ptr
= zone
->malloc(zone
, size
);
237 if (malloc_logger
) malloc_logger(MALLOC_LOG_TYPE_ALLOCATE
| MALLOC_LOG_TYPE_HAS_ZONE
, (unsigned)zone
, size
, 0, (unsigned)ptr
, 0);
241 void *malloc_zone_calloc(malloc_zone_t
*zone
, size_t num_items
, size_t size
) {
243 if (malloc_check_start
&& (malloc_check_counter
++ >= malloc_check_start
)) {
246 if (((unsigned)num_items
>= MAX_ALLOCATION
) || ((unsigned)size
>= MAX_ALLOCATION
) || ((long long)size
* num_items
>= (long long) MAX_ALLOCATION
)) {
247 /* Probably a programming error */
248 fprintf(stderr
, "*** malloc_zone_calloc[%d]: arguments too large: %d,%d\n", getpid(), (unsigned)num_items
, (unsigned)size
);
251 ptr
= zone
->calloc(zone
, num_items
, size
);
252 if (malloc_logger
) malloc_logger(MALLOC_LOG_TYPE_ALLOCATE
| MALLOC_LOG_TYPE_HAS_ZONE
| MALLOC_LOG_TYPE_CLEARED
, (unsigned)zone
, num_items
* size
, 0, (unsigned)ptr
, 0);
256 void *malloc_zone_valloc(malloc_zone_t
*zone
, size_t size
) {
258 if ((unsigned)size
>= MAX_ALLOCATION
) {
259 /* Probably a programming error */
260 fprintf(stderr
, "*** malloc_zone_valloc[%d]: argument too large: %d\n", getpid(), (unsigned)size
);
263 if (malloc_check_start
&& (malloc_check_counter
++ >= malloc_check_start
)) {
266 ptr
= zone
->valloc(zone
, size
);
267 if (malloc_logger
) malloc_logger(MALLOC_LOG_TYPE_ALLOCATE
| MALLOC_LOG_TYPE_HAS_ZONE
, (unsigned)zone
, size
, 0, (unsigned)ptr
, 0);
271 void *malloc_zone_realloc(malloc_zone_t
*zone
, void *ptr
, size_t size
) {
273 if (malloc_check_start
&& (malloc_check_counter
++ >= malloc_check_start
)) {
276 new_ptr
= zone
->realloc(zone
, ptr
, size
);
277 if (malloc_logger
) malloc_logger(MALLOC_LOG_TYPE_ALLOCATE
| MALLOC_LOG_TYPE_DEALLOCATE
| MALLOC_LOG_TYPE_HAS_ZONE
, (unsigned)zone
, (unsigned)ptr
, size
, (unsigned)new_ptr
, 0);
281 void malloc_zone_free(malloc_zone_t
*zone
, void *ptr
) {
282 if (malloc_logger
) malloc_logger(MALLOC_LOG_TYPE_DEALLOCATE
| MALLOC_LOG_TYPE_HAS_ZONE
, (unsigned)zone
, (unsigned)ptr
, 0, 0, 0);
283 if (malloc_check_start
&& (malloc_check_counter
++ >= malloc_check_start
)) {
286 zone
->free(zone
, ptr
);
289 malloc_zone_t
*malloc_zone_from_ptr(const void *ptr
) {
291 if (!ptr
) return NULL
;
292 zone
= find_registered_zone(ptr
, NULL
);
296 /********* Functions for zone implementors ************/
298 void malloc_zone_register(malloc_zone_t
*zone
) {
299 /* Note that given the sequencing it is always safe to first get the number of zones, then get malloc_zones without taking the lock, if all you need is to iterate through the list */
301 if (malloc_num_zones
>= INITIAL_ZONES
) {
302 malloc_zone_t
**zones
= malloc_zones
;
303 malloc_zone_t
*pzone
= malloc_zones
[0];
304 boolean_t copy
= malloc_num_zones
== INITIAL_ZONES
;
305 if (copy
) zones
= NULL
; // to avoid realloc on something not allocated
307 zones
= pzone
->realloc(pzone
, zones
, (malloc_num_zones
+ 1) * sizeof(malloc_zone_t
*)); // we leak initial_malloc_zones, not worth tracking it
309 if (copy
) memcpy(zones
, malloc_zones
, malloc_num_zones
* sizeof(malloc_zone_t
*));
310 malloc_zones
= zones
;
312 malloc_zones
[malloc_num_zones
] = zone
;
313 malloc_num_zones
++; // note that we do this after setting malloc_num_zones, so enumerations without taking the lock are safe
315 // malloc_printf("Registered %p malloc_zones at address %p is %p [%d zones]\n", zone, &malloc_zones, malloc_zones, malloc_num_zones);
318 void malloc_zone_unregister(malloc_zone_t
*z
) {
321 index
= malloc_num_zones
;
323 malloc_zone_t
*zone
= malloc_zones
[index
];
325 malloc_zones
[index
] = malloc_zones
[--malloc_num_zones
];
331 fprintf(stderr
, "*** malloc[%d]: malloc_zone_unregister() failed for %p\n", getpid(), z
);
334 void malloc_set_zone_name(malloc_zone_t
*z
, const char *name
) {
337 free((char *)z
->zone_name
);
340 newName
= malloc_zone_malloc(z
, strlen(name
) + 1);
341 strcpy(newName
, name
);
342 z
->zone_name
= (const char *)newName
;
345 const char *malloc_get_zone_name(malloc_zone_t
*zone
) {
346 return zone
->zone_name
;
349 static char *_malloc_append_unsigned(unsigned value
, unsigned base
, char *head
) {
353 if (value
>= base
) head
= _malloc_append_unsigned(value
/ base
, base
, head
);
354 value
= value
% base
;
355 head
[0] = (value
< 10) ? '0' + value
: 'a' + value
- 10;
360 void malloc_printf(const char *format
, ...) {
366 va_start(args
, format
);
368 head
= _malloc_append_unsigned(((unsigned)&args
) >> 12, 16, head
);
372 while (ch
= *format
++) {
376 char *str
= (char *)(*nums
++);
377 write(2, buf
, head
- buf
);
379 write(2, str
, strlen(str
));
382 *head
++ = '0'; *head
++ = 'x';
384 head
= _malloc_append_unsigned(*nums
++, (ch
== 'd') ? 10 : 16, head
);
390 write(2, buf
, head
- buf
); fflush(stderr
);
394 /********* Generic ANSI callouts ************/
396 void *malloc(size_t size
) {
397 return malloc_zone_malloc(inline_malloc_default_zone(), size
);
400 void *calloc(size_t num_items
, size_t size
) {
401 return malloc_zone_calloc(inline_malloc_default_zone(), num_items
, size
);
404 void free(void *ptr
) {
407 zone
= find_registered_zone(ptr
, NULL
);
409 malloc_zone_free(zone
, ptr
);
411 fprintf(stderr
, "*** malloc[%d]: Deallocation of a pointer not malloced: %p; This could be a double free(), or free() called with the middle of an allocated block; Try setting environment variable MallocHelp to see tools to help debug\n", getpid(), ptr
);
415 void *realloc(void *old_ptr
, size_t new_size
) {
418 if (!old_ptr
) return malloc_zone_malloc(inline_malloc_default_zone(), new_size
);
419 zone
= find_registered_zone(old_ptr
, &old_size
);
420 if (zone
&& (old_size
>= new_size
)) return old_ptr
;
421 if (!zone
) zone
= inline_malloc_default_zone();
422 return malloc_zone_realloc(zone
, old_ptr
, new_size
);
425 void *valloc(size_t size
) {
426 malloc_zone_t
*zone
= inline_malloc_default_zone();
427 return malloc_zone_valloc(zone
, size
);
430 extern void vfree(void *ptr
) {
434 size_t malloc_size(const void *ptr
) {
436 if (!ptr
) return size
;
437 (void)find_registered_zone(ptr
, &size
);
441 size_t malloc_good_size (size_t size
) {
442 malloc_zone_t
*zone
= inline_malloc_default_zone();
443 return zone
->introspect
->good_size(zone
, size
);
446 /********* Functions for performance tools ************/
448 static kern_return_t
_malloc_default_reader(task_t task
, vm_address_t address
, vm_size_t size
, void **ptr
) {
449 *ptr
= (void *)address
;
453 kern_return_t
malloc_get_all_zones(task_t task
, memory_reader_t reader
, vm_address_t
**addresses
, unsigned *count
) {
454 // Note that the 2 following addresses are not correct if the address of the target is different from your own. This notably occurs if the address of System.framework is slid (e.g. different than at B & I )
455 vm_address_t remote_malloc_zones
= (vm_address_t
)&malloc_zones
;
456 vm_address_t remote_malloc_num_zones
= (vm_address_t
)&malloc_num_zones
;
458 vm_address_t zones_address
;
459 vm_address_t
*zones_address_ref
;
461 unsigned *num_zones_ref
;
462 if (!reader
) reader
= _malloc_default_reader
;
463 // printf("Read malloc_zones at address %p should be %p\n", &malloc_zones, malloc_zones);
464 err
= reader(task
, remote_malloc_zones
, sizeof(void *), (void **)&zones_address_ref
);
465 // printf("Read malloc_zones[0x%x]=%p\n", remote_malloc_zones, *zones_address_ref);
467 fprintf(stderr
, "*** malloc[%d]: malloc_get_all_zones: error reading zones_address at 0x%x\n", getpid(), (unsigned)remote_malloc_zones
);
470 zones_address
= *zones_address_ref
;
471 // printf("Reading num_zones at address %p\n", remote_malloc_num_zones);
472 err
= reader(task
, remote_malloc_num_zones
, sizeof(unsigned), (void **)&num_zones_ref
);
474 fprintf(stderr
, "*** malloc[%d]: malloc_get_all_zones: error reading num_zones at 0x%x\n", getpid(), (unsigned)remote_malloc_num_zones
);
477 num_zones
= *num_zones_ref
;
478 // printf("Read malloc_num_zones[0x%x]=%d\n", remote_malloc_num_zones, num_zones);
480 // printf("malloc_get_all_zones succesfully found %d zones\n", num_zones);
481 err
= reader(task
, zones_address
, sizeof(malloc_zone_t
*) * num_zones
, (void **)addresses
);
483 fprintf(stderr
, "*** malloc[%d]: malloc_get_all_zones: error reading zones at 0x%x\n", getpid(), (unsigned)&zones_address
);
486 // printf("malloc_get_all_zones succesfully read %d zones\n", num_zones);
490 /********* Debug helpers ************/
492 void malloc_zone_print_ptr_info(void *ptr
) {
495 zone
= find_registered_zone(ptr
, NULL
);
497 printf("ptr %p in registered zone %p\n", ptr
, zone
);
499 printf("ptr %p not in heap\n", ptr
);
503 boolean_t
malloc_zone_check(malloc_zone_t
*zone
) {
507 while (index
< malloc_num_zones
) {
508 zone
= malloc_zones
[index
++];
509 if (!zone
->introspect
->check(zone
)) ok
= 0;
512 ok
= zone
->introspect
->check(zone
);
517 void malloc_zone_print(malloc_zone_t
*zone
, boolean_t verbose
) {
520 while (index
< malloc_num_zones
) {
521 zone
= malloc_zones
[index
++];
522 zone
->introspect
->print(zone
, verbose
);
525 zone
->introspect
->print(zone
, verbose
);
529 void malloc_zone_log(malloc_zone_t
*zone
, void *address
) {
532 while (index
< malloc_num_zones
) {
533 zone
= malloc_zones
[index
++];
534 zone
->introspect
->log(zone
, address
);
537 zone
->introspect
->log(zone
, address
);
541 /********* Misc other entry points ************/
543 static void DefaultMallocError(int x
) {
544 fprintf(stderr
, "*** malloc[%d]: error %d\n", getpid(), x
);
545 #if USE_SLEEP_RATHER_THAN_ABORT
552 void (*malloc_error(void (*func
)(int)))(int) {
553 return DefaultMallocError
;
556 void _malloc_fork_prepare() {
557 /* Prepare the malloc module for a fork by insuring that no thread is in a malloc critical section */
560 while (index
< malloc_num_zones
) {
561 malloc_zone_t
*zone
= malloc_zones
[index
++];
562 zone
->introspect
->force_lock(zone
);
566 void _malloc_fork_parent() {
567 /* Called in the parent process after a fork() to resume normal operation. */
570 while (index
< malloc_num_zones
) {
571 malloc_zone_t
*zone
= malloc_zones
[index
++];
572 zone
->introspect
->force_unlock(zone
);
576 void _malloc_fork_child() {
577 /* Called in the child process after a fork() to resume normal operation. In the MTASK case we also have to change memory inheritance so that the child does not share memory with the parent. */
580 while (index
< malloc_num_zones
) {
581 malloc_zone_t
*zone
= malloc_zones
[index
++];
582 zone
->introspect
->force_unlock(zone
);
586 size_t mstats(void) {
587 malloc_zone_print(NULL
, 0);
591 /***************** OBSOLETE ENTRY POINTS ********************/
593 #if PHASE_OUT_OLD_MALLOC
594 #error PHASE OUT THE FOLLOWING FUNCTIONS
596 #warning PHASE OUT THE FOLLOWING FUNCTIONS
599 void set_malloc_singlethreaded(boolean_t single
) {
600 static boolean_t warned
= 0;
602 #if PHASE_OUT_OLD_MALLOC
603 fprintf(stderr
, "*** malloc[%d]: OBSOLETE: set_malloc_singlethreaded(%d)\n", getpid(), single
);
609 void malloc_singlethreaded() {
610 static boolean_t warned
= 0;
612 fprintf(stderr
, "*** malloc[%d]: OBSOLETE: malloc_singlethreaded()\n", getpid());
617 int malloc_debug(int level
) {
618 fprintf(stderr
, "*** malloc[%d]: OBSOLETE: malloc_debug()\n", getpid());