+++ /dev/null
-#define JEMALLOC_TCACHE_C_
-#include "jemalloc/internal/jemalloc_internal.h"
-#ifdef JEMALLOC_TCACHE
-/******************************************************************************/
-/* Data. */
-
-bool opt_tcache = true;
-ssize_t opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT;
-ssize_t opt_lg_tcache_gc_sweep = LG_TCACHE_GC_SWEEP_DEFAULT;
-
-tcache_bin_info_t *tcache_bin_info;
-static unsigned stack_nelms; /* Total stack elms per tcache. */
-
-/* Map of thread-specific caches. */
-#ifndef NO_TLS
-__thread tcache_t *tcache_tls JEMALLOC_ATTR(tls_model("initial-exec"));
-#endif
-
-/*
- * Same contents as tcache, but initialized such that the TSD destructor is
- * called when a thread exits, so that the cache can be cleaned up.
- */
-pthread_key_t tcache_tsd;
-
-size_t nhbins;
-size_t tcache_maxclass;
-unsigned tcache_gc_incr;
-
-/******************************************************************************/
-/* Function prototypes for non-inline static functions. */
-
-static void tcache_thread_cleanup(void *arg);
-
-/******************************************************************************/
-
-void *
-tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin, size_t binind)
-{
- void *ret;
-
- arena_tcache_fill_small(tcache->arena, tbin, binind
-#ifdef JEMALLOC_PROF
- , tcache->prof_accumbytes
-#endif
- );
-#ifdef JEMALLOC_PROF
- tcache->prof_accumbytes = 0;
-#endif
- ret = tcache_alloc_easy(tbin);
-
- return (ret);
-}
-
-void
-tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem
-#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
- , tcache_t *tcache
-#endif
- )
-{
- void *ptr;
- unsigned i, nflush, ndeferred;
-#ifdef JEMALLOC_STATS
- bool merged_stats = false;
-#endif
-
- assert(binind < nbins);
- assert(rem <= tbin->ncached);
-
- for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) {
- /* Lock the arena bin associated with the first object. */
- arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(
- tbin->avail[0]);
- arena_t *arena = chunk->arena;
- arena_bin_t *bin = &arena->bins[binind];
-
-#ifdef JEMALLOC_PROF
- if (arena == tcache->arena) {
- malloc_mutex_lock(&arena->lock);
- arena_prof_accum(arena, tcache->prof_accumbytes);
- malloc_mutex_unlock(&arena->lock);
- tcache->prof_accumbytes = 0;
- }
-#endif
-
- malloc_mutex_lock(&bin->lock);
-#ifdef JEMALLOC_STATS
- if (arena == tcache->arena) {
- assert(merged_stats == false);
- merged_stats = true;
- bin->stats.nflushes++;
- bin->stats.nrequests += tbin->tstats.nrequests;
- tbin->tstats.nrequests = 0;
- }
-#endif
- ndeferred = 0;
- for (i = 0; i < nflush; i++) {
- ptr = tbin->avail[i];
- assert(ptr != NULL);
- chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
- if (chunk->arena == arena) {
- size_t pageind = ((uintptr_t)ptr -
- (uintptr_t)chunk) >> PAGE_SHIFT;
- arena_chunk_map_t *mapelm =
- &chunk->map[pageind-map_bias];
- arena_dalloc_bin(arena, chunk, ptr, mapelm);
- } else {
- /*
- * This object was allocated via a different
- * arena bin than the one that is currently
- * locked. Stash the object, so that it can be
- * handled in a future pass.
- */
- tbin->avail[ndeferred] = ptr;
- ndeferred++;
- }
- }
- malloc_mutex_unlock(&bin->lock);
- }
-#ifdef JEMALLOC_STATS
- if (merged_stats == false) {
- /*
- * The flush loop didn't happen to flush to this thread's
- * arena, so the stats didn't get merged. Manually do so now.
- */
- arena_bin_t *bin = &tcache->arena->bins[binind];
- malloc_mutex_lock(&bin->lock);
- bin->stats.nflushes++;
- bin->stats.nrequests += tbin->tstats.nrequests;
- tbin->tstats.nrequests = 0;
- malloc_mutex_unlock(&bin->lock);
- }
-#endif
-
- memmove(tbin->avail, &tbin->avail[tbin->ncached - rem],
- rem * sizeof(void *));
- tbin->ncached = rem;
- if ((int)tbin->ncached < tbin->low_water)
- tbin->low_water = tbin->ncached;
-}
-
-void
-tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem
-#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
- , tcache_t *tcache
-#endif
- )
-{
- void *ptr;
- unsigned i, nflush, ndeferred;
-#ifdef JEMALLOC_STATS
- bool merged_stats = false;
-#endif
-
- assert(binind < nhbins);
- assert(rem <= tbin->ncached);
-
- for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) {
- /* Lock the arena associated with the first object. */
- arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(
- tbin->avail[0]);
- arena_t *arena = chunk->arena;
-
- malloc_mutex_lock(&arena->lock);
-#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS))
- if (arena == tcache->arena) {
-#endif
-#ifdef JEMALLOC_PROF
- arena_prof_accum(arena, tcache->prof_accumbytes);
- tcache->prof_accumbytes = 0;
-#endif
-#ifdef JEMALLOC_STATS
- merged_stats = true;
- arena->stats.nrequests_large += tbin->tstats.nrequests;
- arena->stats.lstats[binind - nbins].nrequests +=
- tbin->tstats.nrequests;
- tbin->tstats.nrequests = 0;
-#endif
-#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS))
- }
-#endif
- ndeferred = 0;
- for (i = 0; i < nflush; i++) {
- ptr = tbin->avail[i];
- assert(ptr != NULL);
- chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
- if (chunk->arena == arena)
- arena_dalloc_large(arena, chunk, ptr);
- else {
- /*
- * This object was allocated via a different
- * arena than the one that is currently locked.
- * Stash the object, so that it can be handled
- * in a future pass.
- */
- tbin->avail[ndeferred] = ptr;
- ndeferred++;
- }
- }
- malloc_mutex_unlock(&arena->lock);
- }
-#ifdef JEMALLOC_STATS
- if (merged_stats == false) {
- /*
- * The flush loop didn't happen to flush to this thread's
- * arena, so the stats didn't get merged. Manually do so now.
- */
- arena_t *arena = tcache->arena;
- malloc_mutex_lock(&arena->lock);
- arena->stats.nrequests_large += tbin->tstats.nrequests;
- arena->stats.lstats[binind - nbins].nrequests +=
- tbin->tstats.nrequests;
- tbin->tstats.nrequests = 0;
- malloc_mutex_unlock(&arena->lock);
- }
-#endif
-
- memmove(tbin->avail, &tbin->avail[tbin->ncached - rem],
- rem * sizeof(void *));
- tbin->ncached = rem;
- if ((int)tbin->ncached < tbin->low_water)
- tbin->low_water = tbin->ncached;
-}
-
-tcache_t *
-tcache_create(arena_t *arena)
-{
- tcache_t *tcache;
- size_t size, stack_offset;
- unsigned i;
-
- size = offsetof(tcache_t, tbins) + (sizeof(tcache_bin_t) * nhbins);
- /* Naturally align the pointer stacks. */
- size = PTR_CEILING(size);
- stack_offset = size;
- size += stack_nelms * sizeof(void *);
- /*
- * Round up to the nearest multiple of the cacheline size, in order to
- * avoid the possibility of false cacheline sharing.
- *
- * That this works relies on the same logic as in ipalloc(), but we
- * cannot directly call ipalloc() here due to tcache bootstrapping
- * issues.
- */
- size = (size + CACHELINE_MASK) & (-CACHELINE);
-
- if (size <= small_maxclass)
- tcache = (tcache_t *)arena_malloc_small(arena, size, true);
- else if (size <= tcache_maxclass)
- tcache = (tcache_t *)arena_malloc_large(arena, size, true);
- else
- tcache = (tcache_t *)icalloc(size);
-
- if (tcache == NULL)
- return (NULL);
-
-#ifdef JEMALLOC_STATS
- /* Link into list of extant tcaches. */
- malloc_mutex_lock(&arena->lock);
- ql_elm_new(tcache, link);
- ql_tail_insert(&arena->tcache_ql, tcache, link);
- malloc_mutex_unlock(&arena->lock);
-#endif
-
- tcache->arena = arena;
- assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0);
- for (i = 0; i < nhbins; i++) {
- tcache->tbins[i].lg_fill_div = 1;
- tcache->tbins[i].avail = (void **)((uintptr_t)tcache +
- (uintptr_t)stack_offset);
- stack_offset += tcache_bin_info[i].ncached_max * sizeof(void *);
- }
-
- TCACHE_SET(tcache);
-
- return (tcache);
-}
-
-void
-tcache_destroy(tcache_t *tcache)
-{
- unsigned i;
- size_t tcache_size;
-
-#ifdef JEMALLOC_STATS
- /* Unlink from list of extant tcaches. */
- malloc_mutex_lock(&tcache->arena->lock);
- ql_remove(&tcache->arena->tcache_ql, tcache, link);
- malloc_mutex_unlock(&tcache->arena->lock);
- tcache_stats_merge(tcache, tcache->arena);
-#endif
-
- for (i = 0; i < nbins; i++) {
- tcache_bin_t *tbin = &tcache->tbins[i];
- tcache_bin_flush_small(tbin, i, 0
-#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
- , tcache
-#endif
- );
-
-#ifdef JEMALLOC_STATS
- if (tbin->tstats.nrequests != 0) {
- arena_t *arena = tcache->arena;
- arena_bin_t *bin = &arena->bins[i];
- malloc_mutex_lock(&bin->lock);
- bin->stats.nrequests += tbin->tstats.nrequests;
- malloc_mutex_unlock(&bin->lock);
- }
-#endif
- }
-
- for (; i < nhbins; i++) {
- tcache_bin_t *tbin = &tcache->tbins[i];
- tcache_bin_flush_large(tbin, i, 0
-#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF))
- , tcache
-#endif
- );
-
-#ifdef JEMALLOC_STATS
- if (tbin->tstats.nrequests != 0) {
- arena_t *arena = tcache->arena;
- malloc_mutex_lock(&arena->lock);
- arena->stats.nrequests_large += tbin->tstats.nrequests;
- arena->stats.lstats[i - nbins].nrequests +=
- tbin->tstats.nrequests;
- malloc_mutex_unlock(&arena->lock);
- }
-#endif
- }
-
-#ifdef JEMALLOC_PROF
- if (tcache->prof_accumbytes > 0) {
- malloc_mutex_lock(&tcache->arena->lock);
- arena_prof_accum(tcache->arena, tcache->prof_accumbytes);
- malloc_mutex_unlock(&tcache->arena->lock);
- }
-#endif
-
- tcache_size = arena_salloc(tcache);
- if (tcache_size <= small_maxclass) {
- arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache);
- arena_t *arena = chunk->arena;
- size_t pageind = ((uintptr_t)tcache - (uintptr_t)chunk) >>
- PAGE_SHIFT;
- arena_chunk_map_t *mapelm = &chunk->map[pageind-map_bias];
- arena_run_t *run = (arena_run_t *)((uintptr_t)chunk +
- (uintptr_t)((pageind - (mapelm->bits >> PAGE_SHIFT)) <<
- PAGE_SHIFT));
- arena_bin_t *bin = run->bin;
-
- malloc_mutex_lock(&bin->lock);
- arena_dalloc_bin(arena, chunk, tcache, mapelm);
- malloc_mutex_unlock(&bin->lock);
- } else if (tcache_size <= tcache_maxclass) {
- arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache);
- arena_t *arena = chunk->arena;
-
- malloc_mutex_lock(&arena->lock);
- arena_dalloc_large(arena, chunk, tcache);
- malloc_mutex_unlock(&arena->lock);
- } else
- idalloc(tcache);
-}
-
-static void
-tcache_thread_cleanup(void *arg)
-{
- tcache_t *tcache = (tcache_t *)arg;
-
- if (tcache == (void *)(uintptr_t)1) {
- /*
- * The previous time this destructor was called, we set the key
- * to 1 so that other destructors wouldn't cause re-creation of
- * the tcache. This time, do nothing, so that the destructor
- * will not be called again.
- */
- } else if (tcache == (void *)(uintptr_t)2) {
- /*
- * Another destructor called an allocator function after this
- * destructor was called. Reset tcache to 1 in order to
- * receive another callback.
- */
- TCACHE_SET((uintptr_t)1);
- } else if (tcache != NULL) {
- assert(tcache != (void *)(uintptr_t)1);
- tcache_destroy(tcache);
- TCACHE_SET((uintptr_t)1);
- }
-}
-
-#ifdef JEMALLOC_STATS
-void
-tcache_stats_merge(tcache_t *tcache, arena_t *arena)
-{
- unsigned i;
-
- /* Merge and reset tcache stats. */
- for (i = 0; i < nbins; i++) {
- arena_bin_t *bin = &arena->bins[i];
- tcache_bin_t *tbin = &tcache->tbins[i];
- malloc_mutex_lock(&bin->lock);
- bin->stats.nrequests += tbin->tstats.nrequests;
- malloc_mutex_unlock(&bin->lock);
- tbin->tstats.nrequests = 0;
- }
-
- for (; i < nhbins; i++) {
- malloc_large_stats_t *lstats = &arena->stats.lstats[i - nbins];
- tcache_bin_t *tbin = &tcache->tbins[i];
- arena->stats.nrequests_large += tbin->tstats.nrequests;
- lstats->nrequests += tbin->tstats.nrequests;
- tbin->tstats.nrequests = 0;
- }
-}
-#endif
-
-bool
-tcache_boot(void)
-{
-
- if (opt_tcache) {
- unsigned i;
-
- /*
- * If necessary, clamp opt_lg_tcache_max, now that
- * small_maxclass and arena_maxclass are known.
- */
- if (opt_lg_tcache_max < 0 || (1U <<
- opt_lg_tcache_max) < small_maxclass)
- tcache_maxclass = small_maxclass;
- else if ((1U << opt_lg_tcache_max) > arena_maxclass)
- tcache_maxclass = arena_maxclass;
- else
- tcache_maxclass = (1U << opt_lg_tcache_max);
-
- nhbins = nbins + (tcache_maxclass >> PAGE_SHIFT);
-
- /* Initialize tcache_bin_info. */
- tcache_bin_info = (tcache_bin_info_t *)base_alloc(nhbins *
- sizeof(tcache_bin_info_t));
- if (tcache_bin_info == NULL)
- return (true);
- stack_nelms = 0;
- for (i = 0; i < nbins; i++) {
- if ((arena_bin_info[i].nregs << 1) <=
- TCACHE_NSLOTS_SMALL_MAX) {
- tcache_bin_info[i].ncached_max =
- (arena_bin_info[i].nregs << 1);
- } else {
- tcache_bin_info[i].ncached_max =
- TCACHE_NSLOTS_SMALL_MAX;
- }
- stack_nelms += tcache_bin_info[i].ncached_max;
- }
- for (; i < nhbins; i++) {
- tcache_bin_info[i].ncached_max = TCACHE_NSLOTS_LARGE;
- stack_nelms += tcache_bin_info[i].ncached_max;
- }
-
- /* Compute incremental GC event threshold. */
- if (opt_lg_tcache_gc_sweep >= 0) {
- tcache_gc_incr = ((1U << opt_lg_tcache_gc_sweep) /
- nbins) + (((1U << opt_lg_tcache_gc_sweep) % nbins ==
- 0) ? 0 : 1);
- } else
- tcache_gc_incr = 0;
-
- if (pthread_key_create(&tcache_tsd, tcache_thread_cleanup) !=
- 0) {
- malloc_write(
- "<jemalloc>: Error in pthread_key_create()\n");
- abort();
- }
- }
-
- return (false);
-}
-/******************************************************************************/
-#endif /* JEMALLOC_TCACHE */
projects / redis.git / blobdiff