+++ /dev/null
-#define JEMALLOC_CHUNK_SWAP_C_
-#include "jemalloc/internal/jemalloc_internal.h"
-#ifdef JEMALLOC_SWAP
-/******************************************************************************/
-/* Data. */
-
-malloc_mutex_t swap_mtx;
-bool swap_enabled;
-bool swap_prezeroed;
-size_t swap_nfds;
-int *swap_fds;
-#ifdef JEMALLOC_STATS
-size_t swap_avail;
-#endif
-
-/* Base address of the mmap()ed file(s). */
-static void *swap_base;
-/* Current end of the space in use (<= swap_max). */
-static void *swap_end;
-/* Absolute upper limit on file-backed addresses. */
-static void *swap_max;
-
-/*
- * Trees of chunks that were previously allocated (trees differ only in node
- * ordering). These are used when allocating chunks, in an attempt to re-use
- * address space. Depending on function, different tree orderings are needed,
- * which is why there are two trees with the same contents.
- */
-static extent_tree_t swap_chunks_szad;
-static extent_tree_t swap_chunks_ad;
-
-/******************************************************************************/
-/* Function prototypes for non-inline static functions. */
-
-static void *chunk_recycle_swap(size_t size, bool *zero);
-static extent_node_t *chunk_dealloc_swap_record(void *chunk, size_t size);
-
-/******************************************************************************/
-
-static void *
-chunk_recycle_swap(size_t size, bool *zero)
-{
- extent_node_t *node, key;
-
- key.addr = NULL;
- key.size = size;
- malloc_mutex_lock(&swap_mtx);
- node = extent_tree_szad_nsearch(&swap_chunks_szad, &key);
- if (node != NULL) {
- void *ret = node->addr;
-
- /* Remove node from the tree. */
- extent_tree_szad_remove(&swap_chunks_szad, node);
- if (node->size == size) {
- extent_tree_ad_remove(&swap_chunks_ad, node);
- base_node_dealloc(node);
- } else {
- /*
- * Insert the remainder of node's address range as a
- * smaller chunk. Its position within swap_chunks_ad
- * does not change.
- */
- assert(node->size > size);
- node->addr = (void *)((uintptr_t)node->addr + size);
- node->size -= size;
- extent_tree_szad_insert(&swap_chunks_szad, node);
- }
-#ifdef JEMALLOC_STATS
- swap_avail -= size;
-#endif
- malloc_mutex_unlock(&swap_mtx);
-
- if (*zero)
- memset(ret, 0, size);
- return (ret);
- }
- malloc_mutex_unlock(&swap_mtx);
-
- return (NULL);
-}
-
-void *
-chunk_alloc_swap(size_t size, bool *zero)
-{
- void *ret;
-
- assert(swap_enabled);
-
- ret = chunk_recycle_swap(size, zero);
- if (ret != NULL)
- return (ret);
-
- malloc_mutex_lock(&swap_mtx);
- if ((uintptr_t)swap_end + size <= (uintptr_t)swap_max) {
- ret = swap_end;
- swap_end = (void *)((uintptr_t)swap_end + size);
-#ifdef JEMALLOC_STATS
- swap_avail -= size;
-#endif
- malloc_mutex_unlock(&swap_mtx);
-
- if (swap_prezeroed)
- *zero = true;
- else if (*zero)
- memset(ret, 0, size);
- } else {
- malloc_mutex_unlock(&swap_mtx);
- return (NULL);
- }
-
- return (ret);
-}
-
-static extent_node_t *
-chunk_dealloc_swap_record(void *chunk, size_t size)
-{
- extent_node_t *xnode, *node, *prev, key;
-
- xnode = NULL;
- while (true) {
- key.addr = (void *)((uintptr_t)chunk + size);
- node = extent_tree_ad_nsearch(&swap_chunks_ad, &key);
- /* Try to coalesce forward. */
- if (node != NULL && node->addr == key.addr) {
- /*
- * Coalesce chunk with the following address range.
- * This does not change the position within
- * swap_chunks_ad, so only remove/insert from/into
- * swap_chunks_szad.
- */
- extent_tree_szad_remove(&swap_chunks_szad, node);
- node->addr = chunk;
- node->size += size;
- extent_tree_szad_insert(&swap_chunks_szad, node);
- break;
- } else if (xnode == NULL) {
- /*
- * It is possible that base_node_alloc() will cause a
- * new base chunk to be allocated, so take care not to
- * deadlock on swap_mtx, and recover if another thread
- * deallocates an adjacent chunk while this one is busy
- * allocating xnode.
- */
- malloc_mutex_unlock(&swap_mtx);
- xnode = base_node_alloc();
- malloc_mutex_lock(&swap_mtx);
- if (xnode == NULL)
- return (NULL);
- } else {
- /* Coalescing forward failed, so insert a new node. */
- node = xnode;
- xnode = NULL;
- node->addr = chunk;
- node->size = size;
- extent_tree_ad_insert(&swap_chunks_ad, node);
- extent_tree_szad_insert(&swap_chunks_szad, node);
- break;
- }
- }
- /* Discard xnode if it ended up unused do to a race. */
- if (xnode != NULL)
- base_node_dealloc(xnode);
-
- /* Try to coalesce backward. */
- prev = extent_tree_ad_prev(&swap_chunks_ad, node);
- if (prev != NULL && (void *)((uintptr_t)prev->addr + prev->size) ==
- chunk) {
- /*
- * Coalesce chunk with the previous address range. This does
- * not change the position within swap_chunks_ad, so only
- * remove/insert node from/into swap_chunks_szad.
- */
- extent_tree_szad_remove(&swap_chunks_szad, prev);
- extent_tree_ad_remove(&swap_chunks_ad, prev);
-
- extent_tree_szad_remove(&swap_chunks_szad, node);
- node->addr = prev->addr;
- node->size += prev->size;
- extent_tree_szad_insert(&swap_chunks_szad, node);
-
- base_node_dealloc(prev);
- }
-
- return (node);
-}
-
-bool
-chunk_in_swap(void *chunk)
-{
- bool ret;
-
- assert(swap_enabled);
-
- malloc_mutex_lock(&swap_mtx);
- if ((uintptr_t)chunk >= (uintptr_t)swap_base
- && (uintptr_t)chunk < (uintptr_t)swap_max)
- ret = true;
- else
- ret = false;
- malloc_mutex_unlock(&swap_mtx);
-
- return (ret);
-}
-
-bool
-chunk_dealloc_swap(void *chunk, size_t size)
-{
- bool ret;
-
- assert(swap_enabled);
-
- malloc_mutex_lock(&swap_mtx);
- if ((uintptr_t)chunk >= (uintptr_t)swap_base
- && (uintptr_t)chunk < (uintptr_t)swap_max) {
- extent_node_t *node;
-
- /* Try to coalesce with other unused chunks. */
- node = chunk_dealloc_swap_record(chunk, size);
- if (node != NULL) {
- chunk = node->addr;
- size = node->size;
- }
-
- /*
- * Try to shrink the in-use memory if this chunk is at the end
- * of the in-use memory.
- */
- if ((void *)((uintptr_t)chunk + size) == swap_end) {
- swap_end = (void *)((uintptr_t)swap_end - size);
-
- if (node != NULL) {
- extent_tree_szad_remove(&swap_chunks_szad,
- node);
- extent_tree_ad_remove(&swap_chunks_ad, node);
- base_node_dealloc(node);
- }
- } else
- madvise(chunk, size, MADV_DONTNEED);
-
-#ifdef JEMALLOC_STATS
- swap_avail += size;
-#endif
- ret = false;
- goto RETURN;
- }
-
- ret = true;
-RETURN:
- malloc_mutex_unlock(&swap_mtx);
- return (ret);
-}
-
-bool
-chunk_swap_enable(const int *fds, unsigned nfds, bool prezeroed)
-{
- bool ret;
- unsigned i;
- off_t off;
- void *vaddr;
- size_t cumsize, voff;
- size_t sizes[nfds];
-
- malloc_mutex_lock(&swap_mtx);
-
- /* Get file sizes. */
- for (i = 0, cumsize = 0; i < nfds; i++) {
- off = lseek(fds[i], 0, SEEK_END);
- if (off == ((off_t)-1)) {
- ret = true;
- goto RETURN;
- }
- if (PAGE_CEILING(off) != off) {
- /* Truncate to a multiple of the page size. */
- off &= ~PAGE_MASK;
- if (ftruncate(fds[i], off) != 0) {
- ret = true;
- goto RETURN;
- }
- }
- sizes[i] = off;
- if (cumsize + off < cumsize) {
- /*
- * Cumulative file size is greater than the total
- * address space. Bail out while it's still obvious
- * what the problem is.
- */
- ret = true;
- goto RETURN;
- }
- cumsize += off;
- }
-
- /* Round down to a multiple of the chunk size. */
- cumsize &= ~chunksize_mask;
- if (cumsize == 0) {
- ret = true;
- goto RETURN;
- }
-
- /*
- * Allocate a chunk-aligned region of anonymous memory, which will
- * be the final location for the memory-mapped files.
- */
- vaddr = chunk_alloc_mmap_noreserve(cumsize);
- if (vaddr == NULL) {
- ret = true;
- goto RETURN;
- }
-
- /* Overlay the files onto the anonymous mapping. */
- for (i = 0, voff = 0; i < nfds; i++) {
- void *addr = mmap((void *)((uintptr_t)vaddr + voff), sizes[i],
- PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, fds[i], 0);
- if (addr == MAP_FAILED) {
- char buf[BUFERROR_BUF];
-
-
- buferror(errno, buf, sizeof(buf));
- malloc_write(
- "<jemalloc>: Error in mmap(..., MAP_FIXED, ...): ");
- malloc_write(buf);
- malloc_write("\n");
- if (opt_abort)
- abort();
- if (munmap(vaddr, voff) == -1) {
- buferror(errno, buf, sizeof(buf));
- malloc_write("<jemalloc>: Error in munmap(): ");
- malloc_write(buf);
- malloc_write("\n");
- }
- ret = true;
- goto RETURN;
- }
- assert(addr == (void *)((uintptr_t)vaddr + voff));
-
- /*
- * Tell the kernel that the mapping will be accessed randomly,
- * and that it should not gratuitously sync pages to the
- * filesystem.
- */
-#ifdef MADV_RANDOM
- madvise(addr, sizes[i], MADV_RANDOM);
-#endif
-#ifdef MADV_NOSYNC
- madvise(addr, sizes[i], MADV_NOSYNC);
-#endif
-
- voff += sizes[i];
- }
-
- swap_prezeroed = prezeroed;
- swap_base = vaddr;
- swap_end = swap_base;
- swap_max = (void *)((uintptr_t)vaddr + cumsize);
-
- /* Copy the fds array for mallctl purposes. */
- swap_fds = (int *)base_alloc(nfds * sizeof(int));
- if (swap_fds == NULL) {
- ret = true;
- goto RETURN;
- }
- memcpy(swap_fds, fds, nfds * sizeof(int));
- swap_nfds = nfds;
-
-#ifdef JEMALLOC_STATS
- swap_avail = cumsize;
-#endif
-
- swap_enabled = true;
-
- ret = false;
-RETURN:
- malloc_mutex_unlock(&swap_mtx);
- return (ret);
-}
-
-bool
-chunk_swap_boot(void)
-{
-
- if (malloc_mutex_init(&swap_mtx))
- return (true);
-
- swap_enabled = false;
- swap_prezeroed = false; /* swap.* mallctl's depend on this. */
- swap_nfds = 0;
- swap_fds = NULL;
-#ifdef JEMALLOC_STATS
- swap_avail = 0;
-#endif
- swap_base = NULL;
- swap_end = NULL;
- swap_max = NULL;
-
- extent_tree_szad_new(&swap_chunks_szad);
- extent_tree_ad_new(&swap_chunks_ad);
-
- return (false);
-}
-
-/******************************************************************************/
-#endif /* JEMALLOC_SWAP */
projects / redis.git / blobdiff