Merge pull request #208 from jbergstroem/jemalloc-2.2.5

[redis.git] / deps / jemalloc / src / jemalloc.c

diff --git a/deps/jemalloc/src/jemalloc.c b/deps/jemalloc/src/jemalloc.c
index e28751692d8532fb9ad555b1abd358532417250e..a161c2e26e173484f438c165c290c6d01abad846 100644 (file)
--- a/deps/jemalloc/src/jemalloc.c
+++ b/deps/jemalloc/src/jemalloc.c
@@ -84,6 +84,7 @@ static void   malloc_conf_error(const char *msg, const char *k, size_t klen,
     const char *v, size_t vlen);
 static void    malloc_conf_init(void);
 static bool    malloc_init_hard(void);
+static int     imemalign(void **memptr, size_t alignment, size_t size);
 
 /******************************************************************************/
 /* malloc_message() setup. */
@@ -688,7 +689,7 @@ malloc_init_hard(void)
 
                result = sysconf(_SC_PAGESIZE);
                assert(result != -1);
-               pagesize = (unsigned)result;
+               pagesize = (size_t)result;
 
                /*
                 * We assume that pagesize is a power of 2 when calculating
@@ -768,6 +769,14 @@ malloc_init_hard(void)
        }
 #endif
 
+       if (malloc_mutex_init(&arenas_lock))
+               return (true);
+
+       if (pthread_key_create(&arenas_tsd, arenas_cleanup) != 0) {
+               malloc_mutex_unlock(&init_lock);
+               return (true);
+       }
+
        /*
         * Create enough scaffolding to allow recursive allocation in
         * malloc_ncpus().
@@ -794,14 +803,6 @@ malloc_init_hard(void)
        ARENA_SET(arenas[0]);
        arenas[0]->nthreads++;
 
-       if (malloc_mutex_init(&arenas_lock))
-               return (true);
-
-       if (pthread_key_create(&arenas_tsd, arenas_cleanup) != 0) {
-               malloc_mutex_unlock(&init_lock);
-               return (true);
-       }
-
 #ifdef JEMALLOC_PROF
        if (prof_boot2()) {
                malloc_mutex_unlock(&init_lock);
@@ -939,7 +940,8 @@ JEMALLOC_P(malloc)(size_t size)
 #ifdef JEMALLOC_PROF
        if (opt_prof) {
                usize = s2u(size);
-               if ((cnt = prof_alloc_prep(usize)) == NULL) {
+               PROF_ALLOC_PREP(1, usize, cnt);
+               if (cnt == NULL) {
                        ret = NULL;
                        goto OOM;
                }
@@ -988,9 +990,15 @@ RETURN:
 }
 
 JEMALLOC_ATTR(nonnull(1))
-JEMALLOC_ATTR(visibility("default"))
-int
-JEMALLOC_P(posix_memalign)(void **memptr, size_t alignment, size_t size)
+#ifdef JEMALLOC_PROF
+/*
+ * Avoid any uncertainty as to how many backtrace frames to ignore in 
+ * PROF_ALLOC_PREP().
+ */
+JEMALLOC_ATTR(noinline)
+#endif
+static int
+imemalign(void **memptr, size_t alignment, size_t size)
 {
        int ret;
        size_t usize
@@ -1057,7 +1065,8 @@ JEMALLOC_P(posix_memalign)(void **memptr, size_t alignment, size_t size)
 
 #ifdef JEMALLOC_PROF
                if (opt_prof) {
-                       if ((cnt = prof_alloc_prep(usize)) == NULL) {
+                       PROF_ALLOC_PREP(2, usize, cnt);
+                       if (cnt == NULL) {
                                result = NULL;
                                ret = EINVAL;
                        } else {
@@ -1110,6 +1119,15 @@ RETURN:
        return (ret);
 }
 
+JEMALLOC_ATTR(nonnull(1))
+JEMALLOC_ATTR(visibility("default"))
+int
+JEMALLOC_P(posix_memalign)(void **memptr, size_t alignment, size_t size)
+{
+
+       return imemalign(memptr, alignment, size);
+}
+
 JEMALLOC_ATTR(malloc)
 JEMALLOC_ATTR(visibility("default"))
 void *
@@ -1165,7 +1183,8 @@ JEMALLOC_P(calloc)(size_t num, size_t size)
 #ifdef JEMALLOC_PROF
        if (opt_prof) {
                usize = s2u(num_size);
-               if ((cnt = prof_alloc_prep(usize)) == NULL) {
+               PROF_ALLOC_PREP(1, usize, cnt);
+               if (cnt == NULL) {
                        ret = NULL;
                        goto RETURN;
                }
@@ -1278,7 +1297,9 @@ JEMALLOC_P(realloc)(void *ptr, size_t size)
                if (opt_prof) {
                        usize = s2u(size);
                        old_ctx = prof_ctx_get(ptr);
-                       if ((cnt = prof_alloc_prep(usize)) == NULL) {
+                       PROF_ALLOC_PREP(1, usize, cnt);
+                       if (cnt == NULL) {
+                               old_ctx = NULL;
                                ret = NULL;
                                goto OOM;
                        }
@@ -1288,8 +1309,13 @@ JEMALLOC_P(realloc)(void *ptr, size_t size)
                                    false, false);
                                if (ret != NULL)
                                        arena_prof_promoted(ret, usize);
-                       } else
+                               else
+                                       old_ctx = NULL;
+                       } else {
                                ret = iralloc(ptr, size, 0, 0, false, false);
+                               if (ret == NULL)
+                                       old_ctx = NULL;
+                       }
                } else
 #endif
                {
@@ -1327,7 +1353,8 @@ OOM:
 #ifdef JEMALLOC_PROF
                        if (opt_prof) {
                                usize = s2u(size);
-                               if ((cnt = prof_alloc_prep(usize)) == NULL)
+                               PROF_ALLOC_PREP(1, usize, cnt);
+                               if (cnt == NULL)
                                        ret = NULL;
                                else {
                                        if (prof_promote && (uintptr_t)cnt !=
@@ -1432,7 +1459,7 @@ JEMALLOC_P(memalign)(size_t alignment, size_t size)
 #ifdef JEMALLOC_CC_SILENCE
        int result =
 #endif
-           JEMALLOC_P(posix_memalign)(&ret, alignment, size);
+           imemalign(&ret, alignment, size);
 #ifdef JEMALLOC_CC_SILENCE
        if (result != 0)
                return (NULL);
@@ -1451,7 +1478,7 @@ JEMALLOC_P(valloc)(size_t size)
 #ifdef JEMALLOC_CC_SILENCE
        int result =
 #endif
-           JEMALLOC_P(posix_memalign)(&ret, PAGE_SIZE, size);
+           imemalign(&ret, PAGE_SIZE, size);
 #ifdef JEMALLOC_CC_SILENCE
        if (result != 0)
                return (NULL);
@@ -1566,14 +1593,14 @@ JEMALLOC_P(allocm)(void **ptr, size_t *rsize, size_t size, int flags)
        if (malloc_init())
                goto OOM;
 
-       usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment,
-           NULL);
+       usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment, NULL);
        if (usize == 0)
                goto OOM;
 
 #ifdef JEMALLOC_PROF
        if (opt_prof) {
-               if ((cnt = prof_alloc_prep(usize)) == NULL)
+               PROF_ALLOC_PREP(1, usize, cnt);
+               if (cnt == NULL)
                        goto OOM;
                if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize <=
                    small_maxclass) {
@@ -1590,7 +1617,7 @@ JEMALLOC_P(allocm)(void **ptr, size_t *rsize, size_t size, int flags)
                        if (p == NULL)
                                goto OOM;
                }
-
+               prof_malloc(p, usize, cnt);
                if (rsize != NULL)
                        *rsize = usize;
        } else
@@ -1645,7 +1672,6 @@ JEMALLOC_P(rallocm)(void **ptr, size_t *rsize, size_t size, size_t extra,
        bool no_move = flags & ALLOCM_NO_MOVE;
 #ifdef JEMALLOC_PROF
        prof_thr_cnt_t *cnt;
-       prof_ctx_t *old_ctx;
 #endif
 
        assert(ptr != NULL);
@@ -1660,25 +1686,33 @@ JEMALLOC_P(rallocm)(void **ptr, size_t *rsize, size_t size, size_t extra,
                /*
                 * usize isn't knowable before iralloc() returns when extra is
                 * non-zero.  Therefore, compute its maximum possible value and
-                * use that in prof_alloc_prep() to decide whether to capture a
+                * use that in PROF_ALLOC_PREP() to decide whether to capture a
                 * backtrace.  prof_realloc() will use the actual usize to
                 * decide whether to sample.
                 */
                size_t max_usize = (alignment == 0) ? s2u(size+extra) :
                    sa2u(size+extra, alignment, NULL);
+               prof_ctx_t *old_ctx = prof_ctx_get(p);
                old_size = isalloc(p);
-               old_ctx = prof_ctx_get(p);
-               if ((cnt = prof_alloc_prep(max_usize)) == NULL)
+               PROF_ALLOC_PREP(1, max_usize, cnt);
+               if (cnt == NULL)
                        goto OOM;
-               if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && max_usize
-                   <= small_maxclass) {
+               /*
+                * Use minimum usize to determine whether promotion may happen.
+                */
+               if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U
+                   && ((alignment == 0) ? s2u(size) : sa2u(size,
+                   alignment, NULL)) <= small_maxclass) {
                        q = iralloc(p, small_maxclass+1, (small_maxclass+1 >=
                            size+extra) ? 0 : size+extra - (small_maxclass+1),
                            alignment, zero, no_move);
                        if (q == NULL)
                                goto ERR;
-                       usize = isalloc(q);
-                       arena_prof_promoted(q, usize);
+                       if (max_usize < PAGE_SIZE) {
+                               usize = max_usize;
+                               arena_prof_promoted(q, usize);
+                       } else
+                               usize = isalloc(q);
                } else {
                        q = iralloc(p, size, extra, alignment, zero, no_move);
                        if (q == NULL)