+++ /dev/null
-/******************************************************************************/
-#ifdef JEMALLOC_H_TYPES
-
-/*
- * Subpages are an artificially designated partitioning of pages. Their only
- * purpose is to support subpage-spaced size classes.
- *
- * There must be at least 4 subpages per page, due to the way size classes are
- * handled.
- */
-#define LG_SUBPAGE 8
-#define SUBPAGE ((size_t)(1U << LG_SUBPAGE))
-#define SUBPAGE_MASK (SUBPAGE - 1)
-
-/* Return the smallest subpage multiple that is >= s. */
-#define SUBPAGE_CEILING(s) \
- (((s) + SUBPAGE_MASK) & ~SUBPAGE_MASK)
-
-#ifdef JEMALLOC_TINY
- /* Smallest size class to support. */
-# define LG_TINY_MIN LG_SIZEOF_PTR
-# define TINY_MIN (1U << LG_TINY_MIN)
-#endif
-
-/*
- * Maximum size class that is a multiple of the quantum, but not (necessarily)
- * a power of 2. Above this size, allocations are rounded up to the nearest
- * power of 2.
- */
-#define LG_QSPACE_MAX_DEFAULT 7
-
-/*
- * Maximum size class that is a multiple of the cacheline, but not (necessarily)
- * a power of 2. Above this size, allocations are rounded up to the nearest
- * power of 2.
- */
-#define LG_CSPACE_MAX_DEFAULT 9
-
-/*
- * RUN_MAX_OVRHD indicates maximum desired run header overhead. Runs are sized
- * as small as possible such that this setting is still honored, without
- * violating other constraints. The goal is to make runs as small as possible
- * without exceeding a per run external fragmentation threshold.
- *
- * We use binary fixed point math for overhead computations, where the binary
- * point is implicitly RUN_BFP bits to the left.
- *
- * Note that it is possible to set RUN_MAX_OVRHD low enough that it cannot be
- * honored for some/all object sizes, since when heap profiling is enabled
- * there is one pointer of header overhead per object (plus a constant). This
- * constraint is relaxed (ignored) for runs that are so small that the
- * per-region overhead is greater than:
- *
- * (RUN_MAX_OVRHD / (reg_size << (3+RUN_BFP))
- */
-#define RUN_BFP 12
-/* \/ Implicit binary fixed point. */
-#define RUN_MAX_OVRHD 0x0000003dU
-#define RUN_MAX_OVRHD_RELAX 0x00001800U
-
-/* Maximum number of regions in one run. */
-#define LG_RUN_MAXREGS 11
-#define RUN_MAXREGS (1U << LG_RUN_MAXREGS)
-
-/*
- * The minimum ratio of active:dirty pages per arena is computed as:
- *
- * (nactive >> opt_lg_dirty_mult) >= ndirty
- *
- * So, supposing that opt_lg_dirty_mult is 5, there can be no less than 32
- * times as many active pages as dirty pages.
- */
-#define LG_DIRTY_MULT_DEFAULT 5
-
-typedef struct arena_chunk_map_s arena_chunk_map_t;
-typedef struct arena_chunk_s arena_chunk_t;
-typedef struct arena_run_s arena_run_t;
-typedef struct arena_bin_info_s arena_bin_info_t;
-typedef struct arena_bin_s arena_bin_t;
-typedef struct arena_s arena_t;
-
-#endif /* JEMALLOC_H_TYPES */
-/******************************************************************************/
-#ifdef JEMALLOC_H_STRUCTS
-
-/* Each element of the chunk map corresponds to one page within the chunk. */
-struct arena_chunk_map_s {
- union {
- /*
- * Linkage for run trees. There are two disjoint uses:
- *
- * 1) arena_t's runs_avail_{clean,dirty} trees.
- * 2) arena_run_t conceptually uses this linkage for in-use
- * non-full runs, rather than directly embedding linkage.
- */
- rb_node(arena_chunk_map_t) rb_link;
- /*
- * List of runs currently in purgatory. arena_chunk_purge()
- * temporarily allocates runs that contain dirty pages while
- * purging, so that other threads cannot use the runs while the
- * purging thread is operating without the arena lock held.
- */
- ql_elm(arena_chunk_map_t) ql_link;
- } u;
-
-#ifdef JEMALLOC_PROF
- /* Profile counters, used for large object runs. */
- prof_ctx_t *prof_ctx;
-#endif
-
- /*
- * Run address (or size) and various flags are stored together. The bit
- * layout looks like (assuming 32-bit system):
- *
- * ???????? ???????? ????---- ----dula
- *
- * ? : Unallocated: Run address for first/last pages, unset for internal
- * pages.
- * Small: Run page offset.
- * Large: Run size for first page, unset for trailing pages.
- * - : Unused.
- * d : dirty?
- * u : unzeroed?
- * l : large?
- * a : allocated?
- *
- * Following are example bit patterns for the three types of runs.
- *
- * p : run page offset
- * s : run size
- * c : (binind+1) for size class (used only if prof_promote is true)
- * x : don't care
- * - : 0
- * + : 1
- * [DULA] : bit set
- * [dula] : bit unset
- *
- * Unallocated (clean):
- * ssssssss ssssssss ssss---- ----du-a
- * xxxxxxxx xxxxxxxx xxxx---- -----Uxx
- * ssssssss ssssssss ssss---- ----dU-a
- *
- * Unallocated (dirty):
- * ssssssss ssssssss ssss---- ----D--a
- * xxxxxxxx xxxxxxxx xxxx---- ----xxxx
- * ssssssss ssssssss ssss---- ----D--a
- *
- * Small:
- * pppppppp pppppppp pppp---- ----d--A
- * pppppppp pppppppp pppp---- -------A
- * pppppppp pppppppp pppp---- ----d--A
- *
- * Large:
- * ssssssss ssssssss ssss---- ----D-LA
- * xxxxxxxx xxxxxxxx xxxx---- ----xxxx
- * -------- -------- -------- ----D-LA
- *
- * Large (sampled, size <= PAGE_SIZE):
- * ssssssss ssssssss sssscccc ccccD-LA
- *
- * Large (not sampled, size == PAGE_SIZE):
- * ssssssss ssssssss ssss---- ----D-LA
- */
- size_t bits;
-#ifdef JEMALLOC_PROF
-#define CHUNK_MAP_CLASS_SHIFT 4
-#define CHUNK_MAP_CLASS_MASK ((size_t)0xff0U)
-#endif
-#define CHUNK_MAP_FLAGS_MASK ((size_t)0xfU)
-#define CHUNK_MAP_DIRTY ((size_t)0x8U)
-#define CHUNK_MAP_UNZEROED ((size_t)0x4U)
-#define CHUNK_MAP_LARGE ((size_t)0x2U)
-#define CHUNK_MAP_ALLOCATED ((size_t)0x1U)
-#define CHUNK_MAP_KEY CHUNK_MAP_ALLOCATED
-};
-typedef rb_tree(arena_chunk_map_t) arena_avail_tree_t;
-typedef rb_tree(arena_chunk_map_t) arena_run_tree_t;
-
-/* Arena chunk header. */
-struct arena_chunk_s {
- /* Arena that owns the chunk. */
- arena_t *arena;
-
- /* Linkage for the arena's chunks_dirty list. */
- ql_elm(arena_chunk_t) link_dirty;
-
- /*
- * True if the chunk is currently in the chunks_dirty list, due to
- * having at some point contained one or more dirty pages. Removal
- * from chunks_dirty is lazy, so (dirtied && ndirty == 0) is possible.
- */
- bool dirtied;
-
- /* Number of dirty pages. */
- size_t ndirty;
-
- /*
- * Map of pages within chunk that keeps track of free/large/small. The
- * first map_bias entries are omitted, since the chunk header does not
- * need to be tracked in the map. This omission saves a header page
- * for common chunk sizes (e.g. 4 MiB).
- */
- arena_chunk_map_t map[1]; /* Dynamically sized. */
-};
-typedef rb_tree(arena_chunk_t) arena_chunk_tree_t;
-
-struct arena_run_s {
-#ifdef JEMALLOC_DEBUG
- uint32_t magic;
-# define ARENA_RUN_MAGIC 0x384adf93
-#endif
-
- /* Bin this run is associated with. */
- arena_bin_t *bin;
-
- /* Index of next region that has never been allocated, or nregs. */
- uint32_t nextind;
-
- /* Number of free regions in run. */
- unsigned nfree;
-};
-
-/*
- * Read-only information associated with each element of arena_t's bins array
- * is stored separately, partly to reduce memory usage (only one copy, rather
- * than one per arena), but mainly to avoid false cacheline sharing.
- */
-struct arena_bin_info_s {
- /* Size of regions in a run for this bin's size class. */
- size_t reg_size;
-
- /* Total size of a run for this bin's size class. */
- size_t run_size;
-
- /* Total number of regions in a run for this bin's size class. */
- uint32_t nregs;
-
- /*
- * Offset of first bitmap_t element in a run header for this bin's size
- * class.
- */
- uint32_t bitmap_offset;
-
- /*
- * Metadata used to manipulate bitmaps for runs associated with this
- * bin.
- */
- bitmap_info_t bitmap_info;
-
-#ifdef JEMALLOC_PROF
- /*
- * Offset of first (prof_ctx_t *) in a run header for this bin's size
- * class, or 0 if (opt_prof == false).
- */
- uint32_t ctx0_offset;
-#endif
-
- /* Offset of first region in a run for this bin's size class. */
- uint32_t reg0_offset;
-};
-
-struct arena_bin_s {
- /*
- * All operations on runcur, runs, and stats require that lock be
- * locked. Run allocation/deallocation are protected by the arena lock,
- * which may be acquired while holding one or more bin locks, but not
- * vise versa.
- */
- malloc_mutex_t lock;
-
- /*
- * Current run being used to service allocations of this bin's size
- * class.
- */
- arena_run_t *runcur;
-
- /*
- * Tree of non-full runs. This tree is used when looking for an
- * existing run when runcur is no longer usable. We choose the
- * non-full run that is lowest in memory; this policy tends to keep
- * objects packed well, and it can also help reduce the number of
- * almost-empty chunks.
- */
- arena_run_tree_t runs;
-
-#ifdef JEMALLOC_STATS
- /* Bin statistics. */
- malloc_bin_stats_t stats;
-#endif
-};
-
-struct arena_s {
-#ifdef JEMALLOC_DEBUG
- uint32_t magic;
-# define ARENA_MAGIC 0x947d3d24
-#endif
-
- /* This arena's index within the arenas array. */
- unsigned ind;
-
- /*
- * Number of threads currently assigned to this arena. This field is
- * protected by arenas_lock.
- */
- unsigned nthreads;
-
- /*
- * There are three classes of arena operations from a locking
- * perspective:
- * 1) Thread asssignment (modifies nthreads) is protected by
- * arenas_lock.
- * 2) Bin-related operations are protected by bin locks.
- * 3) Chunk- and run-related operations are protected by this mutex.
- */
- malloc_mutex_t lock;
-
-#ifdef JEMALLOC_STATS
- arena_stats_t stats;
-# ifdef JEMALLOC_TCACHE
- /*
- * List of tcaches for extant threads associated with this arena.
- * Stats from these are merged incrementally, and at exit.
- */
- ql_head(tcache_t) tcache_ql;
-# endif
-#endif
-
-#ifdef JEMALLOC_PROF
- uint64_t prof_accumbytes;
-#endif
-
- /* List of dirty-page-containing chunks this arena manages. */
- ql_head(arena_chunk_t) chunks_dirty;
-
- /*
- * In order to avoid rapid chunk allocation/deallocation when an arena
- * oscillates right on the cusp of needing a new chunk, cache the most
- * recently freed chunk. The spare is left in the arena's chunk trees
- * until it is deleted.
- *
- * There is one spare chunk per arena, rather than one spare total, in
- * order to avoid interactions between multiple threads that could make
- * a single spare inadequate.
- */
- arena_chunk_t *spare;
-
- /* Number of pages in active runs. */
- size_t nactive;
-
- /*
- * Current count of pages within unused runs that are potentially
- * dirty, and for which madvise(... MADV_DONTNEED) has not been called.
- * By tracking this, we can institute a limit on how much dirty unused
- * memory is mapped for each arena.
- */
- size_t ndirty;
-
- /*
- * Approximate number of pages being purged. It is possible for
- * multiple threads to purge dirty pages concurrently, and they use
- * npurgatory to indicate the total number of pages all threads are
- * attempting to purge.
- */
- size_t npurgatory;
-
- /*
- * Size/address-ordered trees of this arena's available runs. The trees
- * are used for first-best-fit run allocation. The dirty tree contains
- * runs with dirty pages (i.e. very likely to have been touched and
- * therefore have associated physical pages), whereas the clean tree
- * contains runs with pages that either have no associated physical
- * pages, or have pages that the kernel may recycle at any time due to
- * previous madvise(2) calls. The dirty tree is used in preference to
- * the clean tree for allocations, because using dirty pages reduces
- * the amount of dirty purging necessary to keep the active:dirty page
- * ratio below the purge threshold.
- */
- arena_avail_tree_t runs_avail_clean;
- arena_avail_tree_t runs_avail_dirty;
-
- /*
- * bins is used to store trees of free regions of the following sizes,
- * assuming a 64-bit system with 16-byte quantum, 4 KiB page size, and
- * default MALLOC_CONF.
- *
- * bins[i] | size |
- * --------+--------+
- * 0 | 8 |
- * --------+--------+
- * 1 | 16 |
- * 2 | 32 |
- * 3 | 48 |
- * : :
- * 6 | 96 |
- * 7 | 112 |
- * 8 | 128 |
- * --------+--------+
- * 9 | 192 |
- * 10 | 256 |
- * 11 | 320 |
- * 12 | 384 |
- * 13 | 448 |
- * 14 | 512 |
- * --------+--------+
- * 15 | 768 |
- * 16 | 1024 |
- * 17 | 1280 |
- * : :
- * 25 | 3328 |
- * 26 | 3584 |
- * 27 | 3840 |
- * --------+--------+
- */
- arena_bin_t bins[1]; /* Dynamically sized. */
-};
-
-#endif /* JEMALLOC_H_STRUCTS */
-/******************************************************************************/
-#ifdef JEMALLOC_H_EXTERNS
-
-extern size_t opt_lg_qspace_max;
-extern size_t opt_lg_cspace_max;
-extern ssize_t opt_lg_dirty_mult;
-/*
- * small_size2bin is a compact lookup table that rounds request sizes up to
- * size classes. In order to reduce cache footprint, the table is compressed,
- * and all accesses are via the SMALL_SIZE2BIN macro.
- */
-extern uint8_t const *small_size2bin;
-#define SMALL_SIZE2BIN(s) (small_size2bin[(s-1) >> LG_TINY_MIN])
-
-extern arena_bin_info_t *arena_bin_info;
-
-/* Various bin-related settings. */
-#ifdef JEMALLOC_TINY /* Number of (2^n)-spaced tiny bins. */
-# define ntbins ((unsigned)(LG_QUANTUM - LG_TINY_MIN))
-#else
-# define ntbins 0
-#endif
-extern unsigned nqbins; /* Number of quantum-spaced bins. */
-extern unsigned ncbins; /* Number of cacheline-spaced bins. */
-extern unsigned nsbins; /* Number of subpage-spaced bins. */
-extern unsigned nbins;
-#ifdef JEMALLOC_TINY
-# define tspace_max ((size_t)(QUANTUM >> 1))
-#endif
-#define qspace_min QUANTUM
-extern size_t qspace_max;
-extern size_t cspace_min;
-extern size_t cspace_max;
-extern size_t sspace_min;
-extern size_t sspace_max;
-#define small_maxclass sspace_max
-
-#define nlclasses (chunk_npages - map_bias)
-
-void arena_purge_all(arena_t *arena);
-#ifdef JEMALLOC_PROF
-void arena_prof_accum(arena_t *arena, uint64_t accumbytes);
-#endif
-#ifdef JEMALLOC_TCACHE
-void arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin,
- size_t binind
-# ifdef JEMALLOC_PROF
- , uint64_t prof_accumbytes
-# endif
- );
-#endif
-void *arena_malloc_small(arena_t *arena, size_t size, bool zero);
-void *arena_malloc_large(arena_t *arena, size_t size, bool zero);
-void *arena_malloc(size_t size, bool zero);
-void *arena_palloc(arena_t *arena, size_t size, size_t alloc_size,
- size_t alignment, bool zero);
-size_t arena_salloc(const void *ptr);
-#ifdef JEMALLOC_PROF
-void arena_prof_promoted(const void *ptr, size_t size);
-size_t arena_salloc_demote(const void *ptr);
-#endif
-void arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr,
- arena_chunk_map_t *mapelm);
-void arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr);
-#ifdef JEMALLOC_STATS
-void arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty,
- arena_stats_t *astats, malloc_bin_stats_t *bstats,
- malloc_large_stats_t *lstats);
-#endif
-void *arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size,
- size_t extra, bool zero);
-void *arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra,
- size_t alignment, bool zero);
-bool arena_new(arena_t *arena, unsigned ind);
-bool arena_boot(void);
-
-#endif /* JEMALLOC_H_EXTERNS */
-/******************************************************************************/
-#ifdef JEMALLOC_H_INLINES
-
-#ifndef JEMALLOC_ENABLE_INLINE
-size_t arena_bin_index(arena_t *arena, arena_bin_t *bin);
-unsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info,
- const void *ptr);
-# ifdef JEMALLOC_PROF
-prof_ctx_t *arena_prof_ctx_get(const void *ptr);
-void arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx);
-# endif
-void arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr);
-#endif
-
-#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_))
-JEMALLOC_INLINE size_t
-arena_bin_index(arena_t *arena, arena_bin_t *bin)
-{
- size_t binind = bin - arena->bins;
- assert(binind < nbins);
- return (binind);
-}
-
-JEMALLOC_INLINE unsigned
-arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr)
-{
- unsigned shift, diff, regind;
- size_t size;
-
- dassert(run->magic == ARENA_RUN_MAGIC);
- /*
- * Freeing a pointer lower than region zero can cause assertion
- * failure.
- */
- assert((uintptr_t)ptr >= (uintptr_t)run +
- (uintptr_t)bin_info->reg0_offset);
-
- /*
- * Avoid doing division with a variable divisor if possible. Using
- * actual division here can reduce allocator throughput by over 20%!
- */
- diff = (unsigned)((uintptr_t)ptr - (uintptr_t)run -
- bin_info->reg0_offset);
-
- /* Rescale (factor powers of 2 out of the numerator and denominator). */
- size = bin_info->reg_size;
- shift = ffs(size) - 1;
- diff >>= shift;
- size >>= shift;
-
- if (size == 1) {
- /* The divisor was a power of 2. */
- regind = diff;
- } else {
- /*
- * To divide by a number D that is not a power of two we
- * multiply by (2^21 / D) and then right shift by 21 positions.
- *
- * X / D
- *
- * becomes
- *
- * (X * size_invs[D - 3]) >> SIZE_INV_SHIFT
- *
- * We can omit the first three elements, because we never
- * divide by 0, and 1 and 2 are both powers of two, which are
- * handled above.
- */
-#define SIZE_INV_SHIFT ((sizeof(unsigned) << 3) - LG_RUN_MAXREGS)
-#define SIZE_INV(s) (((1U << SIZE_INV_SHIFT) / (s)) + 1)
- static const unsigned size_invs[] = {
- SIZE_INV(3),
- SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7),
- SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11),
- SIZE_INV(12), SIZE_INV(13), SIZE_INV(14), SIZE_INV(15),
- SIZE_INV(16), SIZE_INV(17), SIZE_INV(18), SIZE_INV(19),
- SIZE_INV(20), SIZE_INV(21), SIZE_INV(22), SIZE_INV(23),
- SIZE_INV(24), SIZE_INV(25), SIZE_INV(26), SIZE_INV(27),
- SIZE_INV(28), SIZE_INV(29), SIZE_INV(30), SIZE_INV(31)
- };
-
- if (size <= ((sizeof(size_invs) / sizeof(unsigned)) + 2))
- regind = (diff * size_invs[size - 3]) >> SIZE_INV_SHIFT;
- else
- regind = diff / size;
-#undef SIZE_INV
-#undef SIZE_INV_SHIFT
- }
- assert(diff == regind * size);
- assert(regind < bin_info->nregs);
-
- return (regind);
-}
-
-#ifdef JEMALLOC_PROF
-JEMALLOC_INLINE prof_ctx_t *
-arena_prof_ctx_get(const void *ptr)
-{
- prof_ctx_t *ret;
- arena_chunk_t *chunk;
- size_t pageind, mapbits;
-
- assert(ptr != NULL);
- assert(CHUNK_ADDR2BASE(ptr) != ptr);
-
- chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
- pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT;
- mapbits = chunk->map[pageind-map_bias].bits;
- assert((mapbits & CHUNK_MAP_ALLOCATED) != 0);
- if ((mapbits & CHUNK_MAP_LARGE) == 0) {
- if (prof_promote)
- ret = (prof_ctx_t *)(uintptr_t)1U;
- else {
- arena_run_t *run = (arena_run_t *)((uintptr_t)chunk +
- (uintptr_t)((pageind - (mapbits >> PAGE_SHIFT)) <<
- PAGE_SHIFT));
- size_t binind = arena_bin_index(chunk->arena, run->bin);
- arena_bin_info_t *bin_info = &arena_bin_info[binind];
- unsigned regind;
-
- dassert(run->magic == ARENA_RUN_MAGIC);
- regind = arena_run_regind(run, bin_info, ptr);
- ret = *(prof_ctx_t **)((uintptr_t)run +
- bin_info->ctx0_offset + (regind *
- sizeof(prof_ctx_t *)));
- }
- } else
- ret = chunk->map[pageind-map_bias].prof_ctx;
-
- return (ret);
-}
-
-JEMALLOC_INLINE void
-arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx)
-{
- arena_chunk_t *chunk;
- size_t pageind, mapbits;
-
- assert(ptr != NULL);
- assert(CHUNK_ADDR2BASE(ptr) != ptr);
-
- chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
- pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT;
- mapbits = chunk->map[pageind-map_bias].bits;
- assert((mapbits & CHUNK_MAP_ALLOCATED) != 0);
- if ((mapbits & CHUNK_MAP_LARGE) == 0) {
- if (prof_promote == false) {
- arena_run_t *run = (arena_run_t *)((uintptr_t)chunk +
- (uintptr_t)((pageind - (mapbits >> PAGE_SHIFT)) <<
- PAGE_SHIFT));
- arena_bin_t *bin = run->bin;
- size_t binind;
- arena_bin_info_t *bin_info;
- unsigned regind;
-
- dassert(run->magic == ARENA_RUN_MAGIC);
- binind = arena_bin_index(chunk->arena, bin);
- bin_info = &arena_bin_info[binind];
- regind = arena_run_regind(run, bin_info, ptr);
-
- *((prof_ctx_t **)((uintptr_t)run + bin_info->ctx0_offset
- + (regind * sizeof(prof_ctx_t *)))) = ctx;
- } else
- assert((uintptr_t)ctx == (uintptr_t)1U);
- } else
- chunk->map[pageind-map_bias].prof_ctx = ctx;
-}
-#endif
-
-JEMALLOC_INLINE void
-arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr)
-{
- size_t pageind;
- arena_chunk_map_t *mapelm;
-
- assert(arena != NULL);
- dassert(arena->magic == ARENA_MAGIC);
- assert(chunk->arena == arena);
- assert(ptr != NULL);
- assert(CHUNK_ADDR2BASE(ptr) != ptr);
-
- pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT;
- mapelm = &chunk->map[pageind-map_bias];
- assert((mapelm->bits & CHUNK_MAP_ALLOCATED) != 0);
- if ((mapelm->bits & CHUNK_MAP_LARGE) == 0) {
- /* Small allocation. */
-#ifdef JEMALLOC_TCACHE
- tcache_t *tcache;
-
- if ((tcache = tcache_get()) != NULL)
- tcache_dalloc_small(tcache, ptr);
- else {
-#endif
- arena_run_t *run;
- arena_bin_t *bin;
-
- run = (arena_run_t *)((uintptr_t)chunk +
- (uintptr_t)((pageind - (mapelm->bits >>
- PAGE_SHIFT)) << PAGE_SHIFT));
- dassert(run->magic == ARENA_RUN_MAGIC);
- bin = run->bin;
-#ifdef JEMALLOC_DEBUG
- {
- size_t binind = arena_bin_index(arena, bin);
- arena_bin_info_t *bin_info =
- &arena_bin_info[binind];
- assert(((uintptr_t)ptr - ((uintptr_t)run +
- (uintptr_t)bin_info->reg0_offset)) %
- bin_info->reg_size == 0);
- }
-#endif
- malloc_mutex_lock(&bin->lock);
- arena_dalloc_bin(arena, chunk, ptr, mapelm);
- malloc_mutex_unlock(&bin->lock);
-#ifdef JEMALLOC_TCACHE
- }
-#endif
- } else {
-#ifdef JEMALLOC_TCACHE
- size_t size = mapelm->bits & ~PAGE_MASK;
-
- assert(((uintptr_t)ptr & PAGE_MASK) == 0);
- if (size <= tcache_maxclass) {
- tcache_t *tcache;
-
- if ((tcache = tcache_get()) != NULL)
- tcache_dalloc_large(tcache, ptr, size);
- else {
- malloc_mutex_lock(&arena->lock);
- arena_dalloc_large(arena, chunk, ptr);
- malloc_mutex_unlock(&arena->lock);
- }
- } else {
- malloc_mutex_lock(&arena->lock);
- arena_dalloc_large(arena, chunk, ptr);
- malloc_mutex_unlock(&arena->lock);
- }
-#else
- assert(((uintptr_t)ptr & PAGE_MASK) == 0);
- malloc_mutex_lock(&arena->lock);
- arena_dalloc_large(arena, chunk, ptr);
- malloc_mutex_unlock(&arena->lock);
-#endif
- }
-}
-#endif
-
-#endif /* JEMALLOC_H_INLINES */
-/******************************************************************************/
projects / redis.git / blobdiff