[redis.git] / deps / jemalloc / src / zone.c

#include "jemalloc/internal/jemalloc_internal.h"
#ifndef JEMALLOC_ZONE
#  error "This source file is for zones on Darwin (OS X)."
#endif

/*
 * The malloc_default_purgeable_zone function is only available on >= 10.6.
 * We need to check whether it is present at runtime, thus the weak_import.
 */
extern malloc_zone_t *malloc_default_purgeable_zone(void)
JEMALLOC_ATTR(weak_import);

/******************************************************************************/
/* Data. */

static malloc_zone_t zone;
static struct malloc_introspection_t zone_introspect;

/******************************************************************************/
/* Function prototypes for non-inline static functions. */

static size_t	zone_size(malloc_zone_t *zone, void *ptr);
static void	*zone_malloc(malloc_zone_t *zone, size_t size);
static void	*zone_calloc(malloc_zone_t *zone, size_t num, size_t size);
static void	*zone_valloc(malloc_zone_t *zone, size_t size);
static void	zone_free(malloc_zone_t *zone, void *ptr);
static void	*zone_realloc(malloc_zone_t *zone, void *ptr, size_t size);
#if (JEMALLOC_ZONE_VERSION >= 5)
static void	*zone_memalign(malloc_zone_t *zone, size_t alignment,
#endif
#if (JEMALLOC_ZONE_VERSION >= 6)
    size_t size);
static void	zone_free_definite_size(malloc_zone_t *zone, void *ptr,
    size_t size);
#endif
static void	*zone_destroy(malloc_zone_t *zone);
static size_t	zone_good_size(malloc_zone_t *zone, size_t size);
static void	zone_force_lock(malloc_zone_t *zone);
static void	zone_force_unlock(malloc_zone_t *zone);

/******************************************************************************/
/*
 * Functions.
 */

static size_t
zone_size(malloc_zone_t *zone, void *ptr)
{

	/*
	 * There appear to be places within Darwin (such as setenv(3)) that
	 * cause calls to this function with pointers that *no* zone owns.  If
	 * we knew that all pointers were owned by *some* zone, we could split
	 * our zone into two parts, and use one as the default allocator and
	 * the other as the default deallocator/reallocator.  Since that will
	 * not work in practice, we must check all pointers to assure that they
	 * reside within a mapped chunk before determining size.
	 */
	return (ivsalloc(ptr, config_prof));
}

static void *
zone_malloc(malloc_zone_t *zone, size_t size)
{

	return (je_malloc(size));
}

static void *
zone_calloc(malloc_zone_t *zone, size_t num, size_t size)
{

	return (je_calloc(num, size));
}

static void *
zone_valloc(malloc_zone_t *zone, size_t size)
{
	void *ret = NULL; /* Assignment avoids useless compiler warning. */

	je_posix_memalign(&ret, PAGE, size);

	return (ret);
}

static void
zone_free(malloc_zone_t *zone, void *ptr)
{

	if (ivsalloc(ptr, config_prof) != 0) {
		je_free(ptr);
		return;
	}

	free(ptr);
}

static void *
zone_realloc(malloc_zone_t *zone, void *ptr, size_t size)
{

	if (ivsalloc(ptr, config_prof) != 0)
		return (je_realloc(ptr, size));

	return (realloc(ptr, size));
}

#if (JEMALLOC_ZONE_VERSION >= 5)
static void *
zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size)
{
	void *ret = NULL; /* Assignment avoids useless compiler warning. */

	je_posix_memalign(&ret, alignment, size);

	return (ret);
}
#endif

#if (JEMALLOC_ZONE_VERSION >= 6)
static void
zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size)
{

	if (ivsalloc(ptr, config_prof) != 0) {
		assert(ivsalloc(ptr, config_prof) == size);
		je_free(ptr);
		return;
	}

	free(ptr);
}
#endif

static void *
zone_destroy(malloc_zone_t *zone)
{

	/* This function should never be called. */
	assert(false);
	return (NULL);
}

static size_t
zone_good_size(malloc_zone_t *zone, size_t size)
{

	if (size == 0)
		size = 1;
	return (s2u(size));
}

static void
zone_force_lock(malloc_zone_t *zone)
{

	if (isthreaded)
		jemalloc_prefork();
}

static void
zone_force_unlock(malloc_zone_t *zone)
{

	if (isthreaded)
		jemalloc_postfork_parent();
}

JEMALLOC_ATTR(constructor)
void
register_zone(void)
{

	zone.size = (void *)zone_size;
	zone.malloc = (void *)zone_malloc;
	zone.calloc = (void *)zone_calloc;
	zone.valloc = (void *)zone_valloc;
	zone.free = (void *)zone_free;
	zone.realloc = (void *)zone_realloc;
	zone.destroy = (void *)zone_destroy;
	zone.zone_name = "jemalloc_zone";
	zone.batch_malloc = NULL;
	zone.batch_free = NULL;
	zone.introspect = &zone_introspect;
	zone.version = JEMALLOC_ZONE_VERSION;
#if (JEMALLOC_ZONE_VERSION >= 5)
	zone.memalign = zone_memalign;
#endif
#if (JEMALLOC_ZONE_VERSION >= 6)
	zone.free_definite_size = zone_free_definite_size;
#endif
#if (JEMALLOC_ZONE_VERSION >= 8)
	zone.pressure_relief = NULL;
#endif

	zone_introspect.enumerator = NULL;
	zone_introspect.good_size = (void *)zone_good_size;
	zone_introspect.check = NULL;
	zone_introspect.print = NULL;
	zone_introspect.log = NULL;
	zone_introspect.force_lock = (void *)zone_force_lock;
	zone_introspect.force_unlock = (void *)zone_force_unlock;
	zone_introspect.statistics = NULL;
#if (JEMALLOC_ZONE_VERSION >= 6)
	zone_introspect.zone_locked = NULL;
#endif
#if (JEMALLOC_ZONE_VERSION >= 7)
	zone_introspect.enable_discharge_checking = NULL;
	zone_introspect.disable_discharge_checking = NULL;
	zone_introspect.discharge = NULL;
#ifdef __BLOCKS__
	zone_introspect.enumerate_discharged_pointers = NULL;
#else
	zone_introspect.enumerate_unavailable_without_blocks = NULL;
#endif
#endif

	/*
	 * The default purgeable zone is created lazily by OSX's libc.  It uses
	 * the default zone when it is created for "small" allocations
	 * (< 15 KiB), but assumes the default zone is a scalable_zone.  This
	 * obviously fails when the default zone is the jemalloc zone, so
	 * malloc_default_purgeable_zone is called beforehand so that the
	 * default purgeable zone is created when the default zone is still
	 * a scalable_zone.  As purgeable zones only exist on >= 10.6, we need
	 * to check for the existence of malloc_default_purgeable_zone() at
	 * run time.
	 */
	if (malloc_default_purgeable_zone != NULL)
		malloc_default_purgeable_zone();

	/* Register the custom zone.  At this point it won't be the default. */
	malloc_zone_register(&zone);

	/*
	 * Unregister and reregister the default zone.  On OSX >= 10.6,
	 * unregistering takes the last registered zone and places it at the
	 * location of the specified zone.  Unregistering the default zone thus
	 * makes the last registered one the default.  On OSX < 10.6,
	 * unregistering shifts all registered zones.  The first registered zone
	 * then becomes the default.
	 */
	do {
		malloc_zone_t *default_zone = malloc_default_zone();
		malloc_zone_unregister(default_zone);
		malloc_zone_register(default_zone);
	} while (malloc_default_zone() != &zone);
}
Commit	Line	Data
a78e148b	1	#include "jemalloc/internal/jemalloc_internal.h"
	2	#ifndef JEMALLOC_ZONE
	3	# error "This source file is for zones on Darwin (OS X)."
	4	#endif
	5
ad4c0b41	6	/*
	7	* The malloc_default_purgeable_zone function is only available on >= 10.6.
	8	* We need to check whether it is present at runtime, thus the weak_import.
	9	*/
	10	extern malloc_zone_t *malloc_default_purgeable_zone(void)
	11	JEMALLOC_ATTR(weak_import);
	12
a78e148b	13	/******************************************************************************/
	14	/* Data. */
	15
ad4c0b41	16	static malloc_zone_t zone;
ad4c0b41	17	static struct malloc_introspection_t zone_introspect;
a78e148b	18
	19	/******************************************************************************/
	20	/* Function prototypes for non-inline static functions. */
	21
	22	static size_t zone_size(malloc_zone_t zone, void ptr);
	23	static void zone_malloc(malloc_zone_t zone, size_t size);
	24	static void zone_calloc(malloc_zone_t zone, size_t num, size_t size);
	25	static void zone_valloc(malloc_zone_t zone, size_t size);
	26	static void zone_free(malloc_zone_t zone, void ptr);
	27	static void zone_realloc(malloc_zone_t zone, void *ptr, size_t size);
ad4c0b41	28	#if (JEMALLOC_ZONE_VERSION >= 5)
a78e148b	29	static void zone_memalign(malloc_zone_t zone, size_t alignment,
ad4c0b41	30	#endif
ad4c0b41	31	#if (JEMALLOC_ZONE_VERSION >= 6)
a78e148b	32	size_t size);
	33	static void zone_free_definite_size(malloc_zone_t zone, void ptr,
	34	size_t size);
	35	#endif
	36	static void zone_destroy(malloc_zone_t zone);
	37	static size_t zone_good_size(malloc_zone_t *zone, size_t size);
	38	static void zone_force_lock(malloc_zone_t *zone);
	39	static void zone_force_unlock(malloc_zone_t *zone);
a78e148b	40
	41	/******************************************************************************/
	42	/*
	43	* Functions.
	44	*/
	45
	46	static size_t
	47	zone_size(malloc_zone_t zone, void ptr)
	48	{
	49
	50	/*
	51	* There appear to be places within Darwin (such as setenv(3)) that
	52	* cause calls to this function with pointers that no zone owns. If
	53	* we knew that all pointers were owned by some zone, we could split
	54	* our zone into two parts, and use one as the default allocator and
	55	* the other as the default deallocator/reallocator. Since that will
	56	* not work in practice, we must check all pointers to assure that they
	57	* reside within a mapped chunk before determining size.
	58	*/
ad4c0b41	59	return (ivsalloc(ptr, config_prof));
a78e148b	60	}
	61
	62	static void *
	63	zone_malloc(malloc_zone_t *zone, size_t size)
	64	{
	65
ad4c0b41	66	return (je_malloc(size));
a78e148b	67	}
	68
	69	static void *
	70	zone_calloc(malloc_zone_t *zone, size_t num, size_t size)
	71	{
	72
ad4c0b41	73	return (je_calloc(num, size));
a78e148b	74	}
	75
	76	static void *
	77	zone_valloc(malloc_zone_t *zone, size_t size)
	78	{
	79	void ret = NULL; / Assignment avoids useless compiler warning. */
	80
ad4c0b41	81	je_posix_memalign(&ret, PAGE, size);
a78e148b	82
	83	return (ret);
	84	}
	85
	86	static void
	87	zone_free(malloc_zone_t zone, void ptr)
	88	{
	89
ad4c0b41	90	if (ivsalloc(ptr, config_prof) != 0) {
	91	je_free(ptr);
	92	return;
	93	}
	94
	95	free(ptr);
a78e148b	96	}
	97
	98	static void *
	99	zone_realloc(malloc_zone_t zone, void ptr, size_t size)
	100	{
	101
ad4c0b41	102	if (ivsalloc(ptr, config_prof) != 0)
	103	return (je_realloc(ptr, size));
	104
	105	return (realloc(ptr, size));
a78e148b	106	}
a78e148b	107
ad4c0b41	108	#if (JEMALLOC_ZONE_VERSION >= 5)
a78e148b	109	static void *
	110	zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size)
	111	{
	112	void ret = NULL; / Assignment avoids useless compiler warning. */
	113
ad4c0b41	114	je_posix_memalign(&ret, alignment, size);
a78e148b	115
	116	return (ret);
	117	}
ad4c0b41	118	#endif
a78e148b	119
ad4c0b41	120	#if (JEMALLOC_ZONE_VERSION >= 6)
a78e148b	121	static void
	122	zone_free_definite_size(malloc_zone_t zone, void ptr, size_t size)
	123	{
	124
ad4c0b41	125	if (ivsalloc(ptr, config_prof) != 0) {
	126	assert(ivsalloc(ptr, config_prof) == size);
	127	je_free(ptr);
	128	return;
	129	}
	130
	131	free(ptr);
a78e148b	132	}
	133	#endif
	134
	135	static void *
	136	zone_destroy(malloc_zone_t *zone)
	137	{
	138
	139	/* This function should never be called. */
	140	assert(false);
	141	return (NULL);
	142	}
	143
	144	static size_t
	145	zone_good_size(malloc_zone_t *zone, size_t size)
	146	{
a78e148b	147
ad4c0b41	148	if (size == 0)
	149	size = 1;
	150	return (s2u(size));
a78e148b	151	}
	152
	153	static void
	154	zone_force_lock(malloc_zone_t *zone)
	155	{
	156
	157	if (isthreaded)
	158	jemalloc_prefork();
	159	}
	160
	161	static void
	162	zone_force_unlock(malloc_zone_t *zone)
	163	{
	164
	165	if (isthreaded)
ad4c0b41	166	jemalloc_postfork_parent();
a78e148b	167	}
a78e148b	168
ad4c0b41	169	JEMALLOC_ATTR(constructor)
	170	void
	171	register_zone(void)
a78e148b	172	{
	173
	174	zone.size = (void *)zone_size;
	175	zone.malloc = (void *)zone_malloc;
	176	zone.calloc = (void *)zone_calloc;
	177	zone.valloc = (void *)zone_valloc;
	178	zone.free = (void *)zone_free;
	179	zone.realloc = (void *)zone_realloc;
	180	zone.destroy = (void *)zone_destroy;
	181	zone.zone_name = "jemalloc_zone";
	182	zone.batch_malloc = NULL;
	183	zone.batch_free = NULL;
	184	zone.introspect = &zone_introspect;
	185	zone.version = JEMALLOC_ZONE_VERSION;
ad4c0b41	186	#if (JEMALLOC_ZONE_VERSION >= 5)
a78e148b	187	zone.memalign = zone_memalign;
ad4c0b41	188	#endif
ad4c0b41	189	#if (JEMALLOC_ZONE_VERSION >= 6)
a78e148b	190	zone.free_definite_size = zone_free_definite_size;
a78e148b	191	#endif
ad4c0b41	192	#if (JEMALLOC_ZONE_VERSION >= 8)
	193	zone.pressure_relief = NULL;
	194	#endif
a78e148b	195
	196	zone_introspect.enumerator = NULL;
	197	zone_introspect.good_size = (void *)zone_good_size;
	198	zone_introspect.check = NULL;
	199	zone_introspect.print = NULL;
	200	zone_introspect.log = NULL;
	201	zone_introspect.force_lock = (void *)zone_force_lock;
	202	zone_introspect.force_unlock = (void *)zone_force_unlock;
	203	zone_introspect.statistics = NULL;
	204	#if (JEMALLOC_ZONE_VERSION >= 6)
	205	zone_introspect.zone_locked = NULL;
	206	#endif
ad4c0b41	207	#if (JEMALLOC_ZONE_VERSION >= 7)
	208	zone_introspect.enable_discharge_checking = NULL;
	209	zone_introspect.disable_discharge_checking = NULL;
	210	zone_introspect.discharge = NULL;
	211	#ifdef __BLOCKS__
	212	zone_introspect.enumerate_discharged_pointers = NULL;
	213	#else
	214	zone_introspect.enumerate_unavailable_without_blocks = NULL;
	215	#endif
a78e148b	216	#endif
a78e148b	217
ad4c0b41	218	/*
	219	* The default purgeable zone is created lazily by OSX's libc. It uses
	220	* the default zone when it is created for "small" allocations
	221	* (< 15 KiB), but assumes the default zone is a scalable_zone. This
	222	* obviously fails when the default zone is the jemalloc zone, so
	223	* malloc_default_purgeable_zone is called beforehand so that the
	224	* default purgeable zone is created when the default zone is still
	225	* a scalable_zone. As purgeable zones only exist on >= 10.6, we need
	226	* to check for the existence of malloc_default_purgeable_zone() at
	227	* run time.
	228	*/
	229	if (malloc_default_purgeable_zone != NULL)
	230	malloc_default_purgeable_zone();
a78e148b	231
ad4c0b41	232	/* Register the custom zone. At this point it won't be the default. */
ad4c0b41	233	malloc_zone_register(&zone);
a78e148b	234
a78e148b	235	/*
ad4c0b41	236	* Unregister and reregister the default zone. On OSX >= 10.6,
	237	* unregistering takes the last registered zone and places it at the
	238	* location of the specified zone. Unregistering the default zone thus
	239	* makes the last registered one the default. On OSX < 10.6,
	240	* unregistering shifts all registered zones. The first registered zone
	241	* then becomes the default.
a78e148b	242	*/
ad4c0b41	243	do {
	244	malloc_zone_t *default_zone = malloc_default_zone();
	245	malloc_zone_unregister(default_zone);
	246	malloc_zone_register(default_zone);
	247	} while (malloc_default_zone() != &zone);
a78e148b	248	}