-/*
- * Copyright (c) 1999, 2006, 2008 Apple Inc. All rights reserved.
- *
- * @APPLE_LICENSE_HEADER_START@
- *
- * This file contains Original Code and/or Modifications of Original Code
- * as defined in and that are subject to the Apple Public Source License
- * Version 2.0 (the 'License'). You may not use this file except in
- * compliance with the License. Please obtain a copy of the License at
- * http://www.opensource.apple.com/apsl/ and read it before using this
- * file.
- *
- * The Original Code and all software distributed under the License are
- * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
- * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
- * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
- * Please see the License for the specific language governing rights and
- * limitations under the License.
- *
- * @APPLE_LICENSE_HEADER_END@
- */
-
-/* Author: Bertrand Serlet, August 1999 */
-
-/*
- Multithread enhancements for "tiny" allocations introduced February 2008.
- These are in the spirit of "Hoard". See:
- Berger, E.D.; McKinley, K.S.; Blumofe, R.D.; Wilson, P.R. (2000).
- "Hoard: a scalable memory allocator for multithreaded applications".
- ACM SIGPLAN Notices 35 (11): 117-128. Berger2000.
- <http://portal.acm.org/citation.cfm?id=356989.357000>
- Retrieved on 2008-02-22.
-*/
-
-/* gcc -g -O3 magazine_malloc.c malloc.c -o libmagmalloc.dylib -I. \
- -I/System/Library/Frameworks/System.framework/PrivateHeaders/ -funit-at-a-time \
- -dynamiclib -Wall -arch x86_64 -arch i386 -arch ppc */
-
-#include <TargetConditionals.h>
-
-#include "scalable_malloc.h"
-#include "malloc_printf.h"
-#include "_simple.h"
-#include "magmallocProvider.h"
-
-#include <pthread_internals.h> /* for pthread_lock_t SPI */
-#include <pthread.h> /* for pthread API */
-
-#include <stdint.h>
-#include <unistd.h>
-#include <mach/vm_statistics.h>
-#include <mach/mach_init.h>
-#include <sys/types.h>
-#include <sys/mman.h>
-#include <sys/param.h>
-
-#if defined(__i386__) || defined(__x86_64__)
-#define __APPLE_API_PRIVATE
-#include <machine/cpu_capabilities.h>
-#define _COMM_PAGE_VERSION_REQD 9
-#undef __APPLE_API_PRIVATE
-#else
-#include <sys/sysctl.h>
-#endif
-
-#include <libkern/OSAtomic.h>
-#include <mach-o/dyld.h> /* for NSVersionOfLinkTimeLibrary() */
-#include <mach-o/dyld_priv.h> /* for _dyld_get_image_slide() */
-#include <crt_externs.h> /* for _NSGetMachExecuteHeader() */
-#include <mach/vm_param.h>
-#include <sys/vmparam.h>
-
-#include <CrashReporterClient.h>
-
-/********************* DEFINITIONS ************************/
-
-#define DEBUG_MALLOC 0 // set to one to debug malloc itself
-
-#define DEBUG_CLIENT 0 // set to one to debug malloc client
-
-#define DEBUG_MADVISE 0
-
-// <rdar://problem/10397726>
-#define RELAXED_INVARIANT_CHECKS 1
-
-#if DEBUG_MALLOC
-#warning DEBUG_MALLOC ENABLED
-# define INLINE
-# define ALWAYSINLINE
-# define CHECK_MAGAZINE_PTR_LOCKED(szone, mag_ptr, fun) \
-do { \
- if (__is_threaded && TRY_LOCK(mag_ptr->magazine_lock)) { \
- malloc_printf("*** magazine_lock was not set %p in %s\n", \
- mag_ptr->magazine_lock, fun); \
- } \
-} while (0)
-#else
-# define INLINE __inline__
-# define ALWAYSINLINE __attribute__((always_inline))
-# define CHECK_MAGAZINE_PTR_LOCKED(szone, mag_ptr, fun) {}
-#endif
-
-# define NOINLINE __attribute__((noinline))
-
-#if defined(__i386__) || defined(__x86_64__)
-#define CACHE_ALIGN __attribute__ ((aligned (128) )) /* Future-proofing at 128B */
-#elif defined(__ppc__) || defined(__ppc64__)
-#define CACHE_ALIGN __attribute__ ((aligned (128) ))
-#else
-#define CACHE_ALIGN
-#endif
-
-#if !__LP64__
-#define ASLR_INTERNAL 1
-#endif
-
-/*
- * Access to global variables is slow, so optimise our handling of vm_page_size
- * and vm_page_shift.
- */
-#define _vm_page_size vm_page_size /* to get to the originals */
-#define _vm_page_shift vm_page_shift
-#define vm_page_size 4096 /* our normal working sizes */
-#define vm_page_shift 12
-
-/*
- * msize - a type to refer to the number of quanta of a tiny or small
- * allocation. A tiny block with an msize of 3 would be 3 << SHIFT_TINY_QUANTUM
- * bytes in size.
- */
-typedef unsigned short msize_t;
-
-typedef union {
- void *p;
- uintptr_t u;
-} ptr_union;
-
-typedef struct {
- ptr_union previous;
- ptr_union next;
-} free_list_t;
-
-typedef unsigned int grain_t; // N.B. wide enough to index all free slots
-
-typedef int mag_index_t;
-
-#define CHECK_REGIONS (1 << 31)
-#define DISABLE_ASLR (1 << 30)
-
-#define MAX_RECORDER_BUFFER 256
-
-/********************* DEFINITIONS for tiny ************************/
-
-/*
- * Memory in the Tiny range is allocated from regions (heaps) pointed to by the
- * szone's hashed_regions pointer.
- *
- * Each region is laid out as a heap, followed by a header block, all within
- * a 1MB (2^20) block. This means there are 64520 16-byte blocks and the header
- * is 16138 bytes, making the total 1048458 bytes, leaving 118 bytes unused.
- *
- * The header block is arranged as in struct tiny_region defined just below, and
- * consists of two bitfields (or bit arrays) interleaved 32 bits by 32 bits.
- *
- * Each bitfield comprises NUM_TINY_BLOCKS bits, and refers to the corresponding
- * TINY_QUANTUM block within the heap.
- *
- * The bitfields are used to encode the state of memory within the heap. The header bit indicates
- * that the corresponding quantum is the first quantum in a block (either in use or free). The
- * in-use bit is set for the header if the block has been handed out (allocated). If the header
- * bit is not set, the in-use bit is invalid.
- *
- * The szone maintains an array of NUM_TINY_SLOTS freelists, each of which is used to hold
- * free objects of the corresponding quantum size.
- *
- * A free block is laid out depending on its size, in order to fit all free
- * blocks in 16 bytes, on both 32 and 64 bit platforms. One quantum blocks do
- * not store their size in the block, instead relying on the header information
- * to determine their size. Blocks of two or more quanta have room to store
- * their size in the block, and store it both after the 'next' pointer, and in
- * the last 2 bytes of the block.
- *
- * 1-quantum block
- * Offset (32-bit mode) (64-bit mode)
- * 0x0 0x0 : previous
- * 0x4 0x08 : next
- * end end
- *
- * >1-quantum block
- * Offset (32-bit mode) (64-bit mode)
- * 0x0 0x0 : previous
- * 0x4 0x08 : next
- * 0x8 0x10 : size (in quantum counts)
- * end - 2 end - 2 : size (in quantum counts)
- * end end
- *
- * All fields are pointer-sized, except for the size which is an unsigned short.
- *
- */
-
-#define SHIFT_TINY_QUANTUM 4 // Required for AltiVec
-#define TINY_QUANTUM (1 << SHIFT_TINY_QUANTUM)
-
-#define FOLLOWING_TINY_PTR(ptr,msize) (((unsigned char *)(ptr)) + ((msize) << SHIFT_TINY_QUANTUM))
-
-#ifdef __LP64__
-#define NUM_TINY_SLOTS 64 // number of slots for free-lists
-#else
-#define NUM_TINY_SLOTS 32 // number of slots for free-lists
-#endif
-
-#define NUM_TINY_BLOCKS 64520
-#define SHIFT_TINY_CEIL_BLOCKS 16 // ceil(log2(NUM_TINY_BLOCKS))
-#define NUM_TINY_CEIL_BLOCKS (1 << SHIFT_TINY_CEIL_BLOCKS)
-#define TINY_BLOCKS_ALIGN (SHIFT_TINY_CEIL_BLOCKS + SHIFT_TINY_QUANTUM) // 20
-
-#define TINY_ENTROPY_BITS 15
-#define TINY_ENTROPY_MASK ((1 << TINY_ENTROPY_BITS) - 1)
-
-/*
- * Avoid having so much entropy that the end of a valid tiny allocation
- * might overrun the end of the tiny region.
- */
-#if TINY_ENTROPY_MASK + NUM_TINY_SLOTS > NUM_TINY_BLOCKS
-#error Too many entropy bits for tiny region requested
-#endif
-
-/*
- * Enough room for the data, followed by the bit arrays (2-bits per block)
- * plus rounding to the nearest page.
- */
-#define CEIL_NUM_TINY_BLOCKS_WORDS (((NUM_TINY_BLOCKS + 31) & ~31) >> 5)
-#define TINY_METADATA_SIZE (sizeof(region_trailer_t) + sizeof(tiny_header_inuse_pair_t) * CEIL_NUM_TINY_BLOCKS_WORDS)
-#define TINY_REGION_SIZE \
- ((NUM_TINY_BLOCKS * TINY_QUANTUM + TINY_METADATA_SIZE + vm_page_size - 1) & ~ (vm_page_size - 1))
-
-#define TINY_METADATA_START (NUM_TINY_BLOCKS * TINY_QUANTUM)
-
-/*
- * Beginning and end pointers for a region's heap.
- */
-#define TINY_REGION_ADDRESS(region) ((void *)(region))
-#define TINY_REGION_END(region) ((void *)(((uintptr_t)(region)) + (NUM_TINY_BLOCKS * TINY_QUANTUM)))
-
-/*
- * Locate the heap base for a pointer known to be within a tiny region.
- */
-#define TINY_REGION_FOR_PTR(_p) ((void *)((uintptr_t)(_p) & ~((1 << TINY_BLOCKS_ALIGN) - 1)))
-
-/*
- * Convert between byte and msize units.
- */
-#define TINY_BYTES_FOR_MSIZE(_m) ((_m) << SHIFT_TINY_QUANTUM)
-#define TINY_MSIZE_FOR_BYTES(_b) ((_b) >> SHIFT_TINY_QUANTUM)
-
-#ifdef __LP64__
-# define TINY_FREE_SIZE(ptr) (((msize_t *)(ptr))[8])
-#else
-# define TINY_FREE_SIZE(ptr) (((msize_t *)(ptr))[4])
-#endif
-#define TINY_PREVIOUS_MSIZE(ptr) ((msize_t *)(ptr))[-1]
-
-/*
- * Layout of a tiny region
- */
-typedef uint32_t tiny_block_t[4]; // assert(TINY_QUANTUM == sizeof(tiny_block_t))
-
-typedef struct tiny_header_inuse_pair
-{
- uint32_t header;
- uint32_t inuse;
-} tiny_header_inuse_pair_t;
-
-typedef struct region_trailer
-{
- struct region_trailer *prev;
- struct region_trailer *next;
- boolean_t recirc_suitable;
- boolean_t failedREUSE;
- volatile int pinned_to_depot;
- unsigned bytes_used;
- mag_index_t mag_index;
-} region_trailer_t;
-
-typedef struct tiny_region
-{
- tiny_block_t blocks[NUM_TINY_BLOCKS];
-
- region_trailer_t trailer;
-
- // The interleaved bit arrays comprising the header and inuse bitfields.
- // The unused bits of each component in the last pair will be initialized to sentinel values.
- tiny_header_inuse_pair_t pairs[CEIL_NUM_TINY_BLOCKS_WORDS];
-
- uint8_t pad[TINY_REGION_SIZE - (NUM_TINY_BLOCKS * sizeof(tiny_block_t)) - TINY_METADATA_SIZE];
-} *tiny_region_t;
-
-/*
- * Per-region meta data for tiny allocator
- */
-#define REGION_TRAILER_FOR_TINY_REGION(r) (&(((tiny_region_t)(r))->trailer))
-#define MAGAZINE_INDEX_FOR_TINY_REGION(r) (REGION_TRAILER_FOR_TINY_REGION(r)->mag_index)
-#define BYTES_USED_FOR_TINY_REGION(r) (REGION_TRAILER_FOR_TINY_REGION(r)->bytes_used)
-
-/*
- * Locate the block header for a pointer known to be within a tiny region.
- */
-#define TINY_BLOCK_HEADER_FOR_PTR(_p) ((void *)&(((tiny_region_t)TINY_REGION_FOR_PTR(_p))->pairs))
-
-/*
- * Locate the inuse map for a given block header pointer.
- */
-#define TINY_INUSE_FOR_HEADER(_h) ((void *)&(((tiny_header_inuse_pair_t *)(_h))->inuse))
-
-/*
- * Compute the bitmap index for a pointer known to be within a tiny region.
- */
-#define TINY_INDEX_FOR_PTR(_p) (((uintptr_t)(_p) >> SHIFT_TINY_QUANTUM) & (NUM_TINY_CEIL_BLOCKS - 1))
-
-#define TINY_CACHE 1 // This governs a last-free cache of 1 that bypasses the free-list
-
-#if ! TINY_CACHE
-#warning TINY_CACHE turned off
-#endif
-
-#define TINY_REGION_PAYLOAD_BYTES (NUM_TINY_BLOCKS * TINY_QUANTUM)
-
-/********************* DEFINITIONS for small ************************/
-
-/*
- * Memory in the Small range is allocated from regions (heaps) pointed to by the szone's hashed_regions
- * pointer.
- *
- * Each region is laid out as a heap, followed by the metadata array, all within an 8MB (2^23) block.
- * The array is arranged as an array of shorts, one for each SMALL_QUANTUM in the heap.
- * This means there are 16320 512-blocks and the array is 16320*2 bytes, which totals 8388480, leaving
- * 128 bytes unused.
- *
- * The MSB of each short is set for the first quantum in a free block. The low 15 bits encode the
- * block size (in SMALL_QUANTUM units), or are zero if the quantum is not the first in a block.
- *
- * The szone maintains an array of 32 freelists, each of which is used to hold free objects
- * of the corresponding quantum size.
- *
- * A free block is laid out as:
- *
- * Offset (32-bit mode) (64-bit mode)
- * 0x0 0x0 : previous
- * 0x4 0x08 : next
- * 0x8 0x10 : size (in quantum counts)
- * end - 2 end - 2 : size (in quantum counts)
- * end end
- *
- * All fields are pointer-sized, except for the size which is an unsigned short.
- *
- */
-
-#define SMALL_IS_FREE (1 << 15)
-
-#define SHIFT_SMALL_QUANTUM (SHIFT_TINY_QUANTUM + 5) // 9
-#define SMALL_QUANTUM (1 << SHIFT_SMALL_QUANTUM) // 512 bytes
-
-#define FOLLOWING_SMALL_PTR(ptr,msize) (((unsigned char *)(ptr)) + ((msize) << SHIFT_SMALL_QUANTUM))
-
-/*
- * The number of slots in the free-list for small blocks. To avoid going to
- * vm system as often on large memory machines, increase the number of free list
- * spots above some amount of RAM installed in the system.
- */
-#define NUM_SMALL_SLOTS 32
-#define NUM_SMALL_SLOTS_LARGEMEM 256
-#define SMALL_BITMAP_WORDS 8
-
-/*
- * We can only represent up to 1<<15 for msize; but we choose to stay even below that to avoid the
- * convention msize=0 => msize = (1<<15)
- */
-#define NUM_SMALL_BLOCKS 16320
-#define SHIFT_SMALL_CEIL_BLOCKS 14 // ceil(log2(NUM_SMALL_BLOCKs))
-#define NUM_SMALL_CEIL_BLOCKS (1 << SHIFT_SMALL_CEIL_BLOCKS)
-#define SMALL_BLOCKS_ALIGN (SHIFT_SMALL_CEIL_BLOCKS + SHIFT_SMALL_QUANTUM) // 23
-
-#define SMALL_ENTROPY_BITS 13
-#define SMALL_ENTROPY_MASK ((1 << SMALL_ENTROPY_BITS) - 1)
-
-/*
- * Avoid having so much entropy that the end of a valid small allocation
- * might overrun the end of the small region.
- */
-#if SMALL_ENTROPY_MASK + NUM_SMALL_SLOTS > NUM_SMALL_BLOCKS
-#error Too many entropy bits for small region requested
-#endif
-
-#define SMALL_METADATA_SIZE (sizeof(region_trailer_t) + NUM_SMALL_BLOCKS * sizeof(msize_t))
-#define SMALL_REGION_SIZE \
- ((NUM_SMALL_BLOCKS * SMALL_QUANTUM + SMALL_METADATA_SIZE + vm_page_size - 1) & ~ (vm_page_size - 1))
-
-#define SMALL_METADATA_START (NUM_SMALL_BLOCKS * SMALL_QUANTUM)
-
-/*
- * Beginning and end pointers for a region's heap.
- */
-#define SMALL_REGION_ADDRESS(region) ((unsigned char *)region)
-#define SMALL_REGION_END(region) (SMALL_REGION_ADDRESS(region) + (NUM_SMALL_BLOCKS * SMALL_QUANTUM))
-
-/*
- * Locate the heap base for a pointer known to be within a small region.
- */
-#define SMALL_REGION_FOR_PTR(_p) ((void *)((uintptr_t)(_p) & ~((1 << SMALL_BLOCKS_ALIGN) - 1)))
-
-/*
- * Convert between byte and msize units.
- */
-#define SMALL_BYTES_FOR_MSIZE(_m) ((_m) << SHIFT_SMALL_QUANTUM)
-#define SMALL_MSIZE_FOR_BYTES(_b) ((_b) >> SHIFT_SMALL_QUANTUM)
-
-#define SMALL_PREVIOUS_MSIZE(ptr) ((msize_t *)(ptr))[-1]
-
-/*
- * Layout of a small region
- */
-typedef uint32_t small_block_t[SMALL_QUANTUM/sizeof(uint32_t)];
-
-typedef struct small_region
-{
- small_block_t blocks[NUM_SMALL_BLOCKS];
-
- region_trailer_t trailer;
-
- msize_t small_meta_words[NUM_SMALL_BLOCKS];
-
- uint8_t pad[SMALL_REGION_SIZE - (NUM_SMALL_BLOCKS * sizeof(small_block_t)) - SMALL_METADATA_SIZE];
-} *small_region_t;
-
-/*
- * Per-region meta data for small allocator
- */
-#define REGION_TRAILER_FOR_SMALL_REGION(r) (&(((small_region_t)(r))->trailer))
-#define MAGAZINE_INDEX_FOR_SMALL_REGION(r) (REGION_TRAILER_FOR_SMALL_REGION(r)->mag_index)
-#define BYTES_USED_FOR_SMALL_REGION(r) (REGION_TRAILER_FOR_SMALL_REGION(r)->bytes_used)
-
-/*
- * Locate the metadata base for a pointer known to be within a small region.
- */
-#define SMALL_META_HEADER_FOR_PTR(_p) (((small_region_t)SMALL_REGION_FOR_PTR(_p))->small_meta_words)
-
-/*
- * Compute the metadata index for a pointer known to be within a small region.
- */
-#define SMALL_META_INDEX_FOR_PTR(_p) (((uintptr_t)(_p) >> SHIFT_SMALL_QUANTUM) & (NUM_SMALL_CEIL_BLOCKS - 1))
-
-/*
- * Find the metadata word for a pointer known to be within a small region.
- */
-#define SMALL_METADATA_FOR_PTR(_p) (SMALL_META_HEADER_FOR_PTR(_p) + SMALL_META_INDEX_FOR_PTR(_p))
-
-/*
- * Determine whether a pointer known to be within a small region points to memory which is free.
- */
-#define SMALL_PTR_IS_FREE(_p) (*SMALL_METADATA_FOR_PTR(_p) & SMALL_IS_FREE)
-
-/*
- * Extract the msize value for a pointer known to be within a small region.
- */
-#define SMALL_PTR_SIZE(_p) (*SMALL_METADATA_FOR_PTR(_p) & ~SMALL_IS_FREE)
-
-#define SMALL_CACHE 1
-#if !SMALL_CACHE
-#warning SMALL_CACHE turned off
-#endif
-
-#define SMALL_REGION_PAYLOAD_BYTES (NUM_SMALL_BLOCKS * SMALL_QUANTUM)
-
-/************************* DEFINITIONS for large ****************************/
-
-#define LARGE_THRESHOLD (15 * 1024) // strictly above this use "large"
-#define LARGE_THRESHOLD_LARGEMEM (127 * 1024)
-
-#if (LARGE_THRESHOLD > NUM_SMALL_SLOTS * SMALL_QUANTUM)
-#error LARGE_THRESHOLD should always be less than NUM_SMALL_SLOTS * SMALL_QUANTUM
-#endif
-
-#if (LARGE_THRESHOLD_LARGEMEM > NUM_SMALL_SLOTS_LARGEMEM * SMALL_QUANTUM)
-#error LARGE_THRESHOLD_LARGEMEM should always be less than NUM_SMALL_SLOTS * SMALL_QUANTUM
-#endif
-
-/*
- * When all memory is touched after a copy, vm_copy() is always a lose
- * But if the memory is only read, vm_copy() wins over memmove() at 3 or 4 pages
- * (on a G3/300MHz)
- *
- * This must be larger than LARGE_THRESHOLD
- */
-#define VM_COPY_THRESHOLD (40 * 1024)
-#define VM_COPY_THRESHOLD_LARGEMEM (128 * 1024)
-
-typedef struct {
- vm_address_t address;
- vm_size_t size;
- boolean_t did_madvise_reusable;
-} large_entry_t;
-
-#if !TARGET_OS_EMBEDDED
-#define LARGE_CACHE 1
-#else
-#define LARGE_CACHE 0
-#endif
-#if !LARGE_CACHE
-#warning LARGE_CACHE turned off
-#endif
-#if defined(__LP64__)
-#define LARGE_ENTRY_CACHE_SIZE 16
-#define LARGE_CACHE_SIZE_LIMIT ((vm_size_t)0x80000000) /* 2Gb */
-#else
-#define LARGE_ENTRY_CACHE_SIZE 8
-#define LARGE_CACHE_SIZE_LIMIT ((vm_size_t)0x02000000) /* 32Mb */
-#endif
-#define LARGE_CACHE_SIZE_ENTRY_LIMIT (LARGE_CACHE_SIZE_LIMIT/LARGE_ENTRY_CACHE_SIZE)
-
-#define SZONE_FLOTSAM_THRESHOLD_LOW (1024 * 512)
-#define SZONE_FLOTSAM_THRESHOLD_HIGH (1024 * 1024)
-
-/*******************************************************************************
- * Definitions for region hash
- ******************************************************************************/
-
-typedef void * region_t;
-typedef region_t * rgnhdl_t; /* A pointer into hashed_regions array. */
-
-#define INITIAL_NUM_REGIONS_SHIFT 6 // log2(INITIAL_NUM_REGIONS)
-#define INITIAL_NUM_REGIONS (1 << INITIAL_NUM_REGIONS_SHIFT) // Must be a power of 2!
-#define HASHRING_OPEN_ENTRY ((region_t) 0) // Initial value and sentinel marking end of collision chain
-#define HASHRING_REGION_DEALLOCATED ((region_t)-1) // Region at this slot reclaimed by OS
-#define HASH_BLOCKS_ALIGN TINY_BLOCKS_ALIGN // MIN( TINY_BLOCKS_ALIGN, SMALL_BLOCKS_ALIGN, ... )
-
-typedef struct region_hash_generation {
- size_t num_regions_allocated;
- size_t num_regions_allocated_shift; // log2(num_regions_allocated)
- region_t *hashed_regions; // hashed by location
- struct region_hash_generation *nextgen;
-} region_hash_generation_t;
-
-/*******************************************************************************
- * Per-processor magazine for tiny and small allocators
- ******************************************************************************/
-
-typedef struct { // vm_allocate()'d, so the array of magazines is page-aligned to begin with.
- // Take magazine_lock first, Depot lock when needed for recirc, then szone->{tiny,small}_regions_lock when needed for alloc
- pthread_lock_t magazine_lock CACHE_ALIGN;
- // Protection for the crtical section that does allocate_pages outside the magazine_lock
- volatile boolean_t alloc_underway;
-
- // One element deep "death row", optimizes malloc/free/malloc for identical size.
- void *mag_last_free; // low SHIFT_{TINY,SMALL}_QUANTUM bits indicate the msize
- region_t mag_last_free_rgn; // holds the region for mag_last_free
-
- free_list_t *mag_free_list[256]; // assert( 256 >= MAX( NUM_TINY_SLOTS, NUM_SMALL_SLOTS_LARGEMEM ))
- unsigned mag_bitmap[8]; // assert( sizeof(mag_bitmap) << 3 >= sizeof(mag_free_list)/sizeof(free_list_t) )
-
- // the first and last free region in the last block are treated as big blocks in use that are not accounted for
- size_t mag_bytes_free_at_end;
- size_t mag_bytes_free_at_start;
- region_t mag_last_region; // Valid iff mag_bytes_free_at_end || mag_bytes_free_at_start > 0
-
- // bean counting ...
- unsigned mag_num_objects;
- size_t mag_num_bytes_in_objects;
- size_t num_bytes_in_magazine;
-
- // recirculation list -- invariant: all regions owned by this magazine that meet the emptiness criteria
- // are located nearer to the head of the list than any region that doesn't satisfy that criteria.
- // Doubly linked list for efficient extraction.
- unsigned recirculation_entries;
- region_trailer_t *firstNode;
- region_trailer_t *lastNode;
-
-#if __LP64__
- uint64_t pad[48]; // So sizeof(magazine_t) is 2560 bytes. FIXME: assert this at compile time
-#else
- uint32_t pad[12]; // So sizeof(magazine_t) is 1280 bytes. FIXME: assert this at compile time
-#endif
-} magazine_t;
-
-#define TINY_MAX_MAGAZINES 32 /* MUST BE A POWER OF 2! */
-#define TINY_MAGAZINE_PAGED_SIZE \
- (((sizeof(magazine_t) * (TINY_MAX_MAGAZINES + 1)) + vm_page_size - 1) &\
- ~ (vm_page_size - 1)) /* + 1 for the Depot */
-
-#define SMALL_MAX_MAGAZINES 32 /* MUST BE A POWER OF 2! */
-#define SMALL_MAGAZINE_PAGED_SIZE \
- (((sizeof(magazine_t) * (SMALL_MAX_MAGAZINES + 1)) + vm_page_size - 1) &\
- ~ (vm_page_size - 1)) /* + 1 for the Depot */
-
-#define DEPOT_MAGAZINE_INDEX -1
-
-/****************************** zone itself ***********************************/
-
-/*
- * Note that objects whose adddress are held in pointers here must be pursued
- * individually in the {tiny,small}_in_use_enumeration() routines. See for
- * example the treatment of region_hash_generation and tiny_magazines below.
- */
-
-typedef struct szone_s { // vm_allocate()'d, so page-aligned to begin with.
- malloc_zone_t basic_zone; // first page will be given read-only protection
- uint8_t pad[vm_page_size - sizeof(malloc_zone_t)];
-
- pthread_key_t cpu_id_key; // remainder of structure is R/W (contains no function pointers)
- unsigned debug_flags;
- void *log_address;
-
- /* Regions for tiny objects */
- pthread_lock_t tiny_regions_lock CACHE_ALIGN;
- size_t num_tiny_regions;
- size_t num_tiny_regions_dealloc;
- region_hash_generation_t *tiny_region_generation;
- region_hash_generation_t trg[2];
-
- int num_tiny_magazines;
- unsigned num_tiny_magazines_mask;
- int num_tiny_magazines_mask_shift;
- magazine_t *tiny_magazines; // array of per-processor magazines
-
-#if TARGET_OS_EMBEDDED
- uintptr_t last_tiny_advise;
-#endif
-
- /* Regions for small objects */
- pthread_lock_t small_regions_lock CACHE_ALIGN;
- size_t num_small_regions;
- size_t num_small_regions_dealloc;
- region_hash_generation_t *small_region_generation;
- region_hash_generation_t srg[2];
-
- unsigned num_small_slots; // determined by physmem size
-
- int num_small_magazines;
- unsigned num_small_magazines_mask;
- int num_small_magazines_mask_shift;
- magazine_t *small_magazines; // array of per-processor magazines
-
-#if TARGET_OS_EMBEDDED
- uintptr_t last_small_advise;
-#endif
-
- /* large objects: all the rest */
- pthread_lock_t large_szone_lock CACHE_ALIGN; // One customer at a time for large
- unsigned num_large_objects_in_use;
- unsigned num_large_entries;
- large_entry_t *large_entries; // hashed by location; null entries don't count
- size_t num_bytes_in_large_objects;
-
-#if LARGE_CACHE
- int large_entry_cache_oldest;
- int large_entry_cache_newest;
- large_entry_t large_entry_cache[LARGE_ENTRY_CACHE_SIZE]; // "death row" for large malloc/free
- boolean_t large_legacy_reset_mprotect;
- size_t large_entry_cache_reserve_bytes;
- size_t large_entry_cache_reserve_limit;
- size_t large_entry_cache_bytes; // total size of death row, bytes
-#endif
-
- /* flag and limits pertaining to altered malloc behavior for systems with
- large amounts of physical memory */
- unsigned is_largemem;
- unsigned large_threshold;
- unsigned vm_copy_threshold;
-
- /* security cookie */
- uintptr_t cookie;
-
- /* Initial region list */
- region_t initial_tiny_regions[INITIAL_NUM_REGIONS];
- region_t initial_small_regions[INITIAL_NUM_REGIONS];
-
- /* The purgeable zone constructed by create_purgeable_zone() would like to hand off tiny and small
- * allocations to the default scalable zone. Record the latter as the "helper" zone here. */
- struct szone_s *helper_zone;
-
- boolean_t flotsam_enabled;
-} szone_t;
-
-#define SZONE_PAGED_SIZE ((sizeof(szone_t) + vm_page_size - 1) & ~ (vm_page_size - 1))
-
-#if DEBUG_MALLOC || DEBUG_CLIENT
-static void szone_sleep(void);
-#endif
-__private_extern__ void malloc_error_break(void);
-
-// msg prints after fmt, ...
-static NOINLINE void szone_error(szone_t *szone, int is_corruption, const char *msg, const void *ptr, const char *fmt, ...)
- __printflike(5, 6);
-
-static void protect(void *address, size_t size, unsigned protection, unsigned debug_flags);
-static void *allocate_pages(szone_t *szone, size_t size, unsigned char align, unsigned debug_flags,
- int vm_page_label);
-static void *allocate_pages_securely(szone_t *szone, size_t size, unsigned char align,
- int vm_page_label);
-static void deallocate_pages(szone_t *szone, void *addr, size_t size, unsigned debug_flags);
-#if TARGET_OS_EMBEDDED
-static int madvise_free_range(szone_t *szone, region_t r, uintptr_t pgLo, uintptr_t pgHi, uintptr_t *last);
-#else
-static int madvise_free_range(szone_t *szone, region_t r, uintptr_t pgLo, uintptr_t pgHi);
-#endif
-static kern_return_t _szone_default_reader(task_t task, vm_address_t address, vm_size_t size, void **ptr);
-
-static INLINE mag_index_t mag_get_thread_index(szone_t *szone) ALWAYSINLINE;
-static magazine_t *mag_lock_zine_for_region_trailer(szone_t *szone, magazine_t *magazines, region_trailer_t *trailer,
- mag_index_t mag_index);
-
-static INLINE rgnhdl_t hash_lookup_region_no_lock(region_t *regions, size_t num_entries, size_t shift, region_t r)
- ALWAYSINLINE;
-static void hash_region_insert_no_lock(region_t *regions, size_t num_entries, size_t shift, region_t r);
-static region_t *hash_regions_alloc_no_lock(szone_t *szone, size_t num_entries);
-static region_t *hash_regions_grow_no_lock(szone_t *szone, region_t *regions, size_t old_size,
- size_t *mutable_shift, size_t *new_size);
-
-static INLINE uintptr_t free_list_gen_checksum(uintptr_t ptr) ALWAYSINLINE;
-static INLINE uintptr_t free_list_checksum_ptr(szone_t *szone, void *p) ALWAYSINLINE;
-static INLINE void *free_list_unchecksum_ptr(szone_t *szone, ptr_union *ptr) ALWAYSINLINE;
-static unsigned free_list_count(szone_t *szone, free_list_t *ptr);
-
-static INLINE void recirc_list_extract(szone_t *szone, magazine_t *mag_ptr, region_trailer_t *node) ALWAYSINLINE;
-static INLINE void recirc_list_splice_last(szone_t *szone, magazine_t *mag_ptr, region_trailer_t *node) ALWAYSINLINE;
-static INLINE void recirc_list_splice_first(szone_t *szone, magazine_t *mag_ptr, region_trailer_t *node) ALWAYSINLINE;
-
-static INLINE void BITARRAY_SET(uint32_t *bits, msize_t index) ALWAYSINLINE;
-static INLINE void BITARRAY_CLR(uint32_t *bits, msize_t index) ALWAYSINLINE;
-static INLINE boolean_t BITARRAY_BIT(uint32_t *bits, msize_t index) ALWAYSINLINE;
-
-static msize_t get_tiny_free_size(const void *ptr);
-static msize_t get_tiny_previous_free_msize(const void *ptr);
-static INLINE msize_t get_tiny_meta_header(const void *ptr, boolean_t *is_free) ALWAYSINLINE;
-static INLINE void set_tiny_meta_header_in_use(const void *ptr, msize_t msize) ALWAYSINLINE;
-static INLINE void set_tiny_meta_header_in_use_1(const void *ptr) ALWAYSINLINE;
-static INLINE void set_tiny_meta_header_middle(const void *ptr) ALWAYSINLINE;
-static INLINE void set_tiny_meta_header_free(const void *ptr, msize_t msize) ALWAYSINLINE;
-static INLINE boolean_t tiny_meta_header_is_free(const void *ptr) ALWAYSINLINE;
-static INLINE void *tiny_previous_preceding_free(void *ptr, msize_t *prev_msize) ALWAYSINLINE;
-
-static void tiny_free_list_add_ptr(szone_t *szone, magazine_t *tiny_mag_ptr, void *ptr, msize_t msize);
-static void tiny_free_list_remove_ptr(szone_t *szone, magazine_t *tiny_mag_ptr, void *ptr, msize_t msize);
-static INLINE region_t tiny_region_for_ptr_no_lock(szone_t *szone, const void *ptr) ALWAYSINLINE;
-
-static void tiny_finalize_region(szone_t *szone, magazine_t *tiny_mag_ptr);
-static int tiny_free_detach_region(szone_t *szone, magazine_t *tiny_mag_ptr, region_t r);
-static size_t tiny_free_reattach_region(szone_t *szone, magazine_t *tiny_mag_ptr, region_t r);
-static void tiny_free_scan_madvise_free(szone_t *szone, magazine_t *depot_ptr, region_t r);
-static region_t tiny_free_try_depot_unmap_no_lock(szone_t *szone, magazine_t *depot_ptr, region_trailer_t *node);
-static boolean_t tiny_free_do_recirc_to_depot(szone_t *szone, magazine_t *tiny_mag_ptr, mag_index_t mag_index);
-static region_t tiny_find_msize_region(szone_t *szone, magazine_t *tiny_mag_ptr, mag_index_t mag_index, msize_t msize);
-static boolean_t tiny_get_region_from_depot(szone_t *szone, magazine_t *tiny_mag_ptr, mag_index_t mag_index, msize_t msize);
-
-static INLINE boolean_t tiny_free_no_lock(szone_t *szone, magazine_t *tiny_mag_ptr, mag_index_t mag_index, region_t region,
- void *ptr, msize_t msize) ALWAYSINLINE;
-static void *tiny_malloc_from_region_no_lock(szone_t *szone, magazine_t *tiny_mag_ptr, mag_index_t mag_index,
- msize_t msize, void *fresh_region);
-static boolean_t tiny_try_realloc_in_place(szone_t *szone, void *ptr, size_t old_size, size_t new_size);
-static boolean_t tiny_check_region(szone_t *szone, region_t region);
-static kern_return_t tiny_in_use_enumerator(task_t task, void *context, unsigned type_mask, szone_t *szone,
- memory_reader_t reader, vm_range_recorder_t recorder);
-static void *tiny_malloc_from_free_list(szone_t *szone, magazine_t *tiny_mag_ptr, mag_index_t mag_index,
- msize_t msize);
-static INLINE void *tiny_malloc_should_clear(szone_t *szone, msize_t msize, boolean_t cleared_requested) ALWAYSINLINE;
-static INLINE void free_tiny(szone_t *szone, void *ptr, region_t tiny_region, size_t known_size) ALWAYSINLINE;
-static void print_tiny_free_list(szone_t *szone);
-static void print_tiny_region(boolean_t verbose, region_t region, size_t bytes_at_start, size_t bytes_at_end);
-static boolean_t tiny_free_list_check(szone_t *szone, grain_t slot);
-
-static INLINE void small_meta_header_set_is_free(msize_t *meta_headers, unsigned index, msize_t msize) ALWAYSINLINE;
-static INLINE void small_meta_header_set_in_use(msize_t *meta_headers, msize_t index, msize_t msize) ALWAYSINLINE;
-static INLINE void small_meta_header_set_middle(msize_t *meta_headers, msize_t index) ALWAYSINLINE;
-static void small_free_list_add_ptr(szone_t *szone, magazine_t *small_mag_ptr, void *ptr, msize_t msize);
-static void small_free_list_remove_ptr(szone_t *szone, magazine_t *small_mag_ptr, void *ptr, msize_t msize);
-static INLINE region_t small_region_for_ptr_no_lock(szone_t *szone, const void *ptr) ALWAYSINLINE;
-
-static void small_finalize_region(szone_t *szone, magazine_t *small_mag_ptr);
-static int small_free_detach_region(szone_t *szone, magazine_t *small_mag_ptr, region_t r);
-static size_t small_free_reattach_region(szone_t *szone, magazine_t *small_mag_ptr, region_t r);
-static void small_free_scan_madvise_free(szone_t *szone, magazine_t *depot_ptr, region_t r);
-static region_t small_free_try_depot_unmap_no_lock(szone_t *szone, magazine_t *depot_ptr, region_trailer_t *node);
-static boolean_t small_free_do_recirc_to_depot(szone_t *szone, magazine_t *small_mag_ptr, mag_index_t mag_index);
-static region_t small_find_msize_region(szone_t *szone, magazine_t *small_mag_ptr, mag_index_t mag_index, msize_t msize);
-static boolean_t small_get_region_from_depot(szone_t *szone, magazine_t *small_mag_ptr, mag_index_t mag_index, msize_t msize);
-static INLINE boolean_t small_free_no_lock(szone_t *szone, magazine_t *small_mag_ptr, mag_index_t mag_index, region_t region,
- void *ptr, msize_t msize) ALWAYSINLINE;
-static void *small_malloc_from_region_no_lock(szone_t *szone, magazine_t *small_mag_ptr, mag_index_t mag_index,
- msize_t msize, void *fresh_region);
-static boolean_t small_try_realloc_in_place(szone_t *szone, void *ptr, size_t old_size, size_t new_size);
-static boolean_t small_check_region(szone_t *szone, region_t region);
-static kern_return_t small_in_use_enumerator(task_t task, void *context, unsigned type_mask, szone_t *szone,
- memory_reader_t reader, vm_range_recorder_t recorder);
-static void *small_malloc_from_free_list(szone_t *szone, magazine_t *small_mag_ptr, mag_index_t mag_index,
- msize_t msize);
-static INLINE void *small_malloc_should_clear(szone_t *szone, msize_t msize, boolean_t cleared_requested) ALWAYSINLINE;
-static INLINE void free_small(szone_t *szone, void *ptr, region_t small_region, size_t known_size) ALWAYSINLINE;
-static void print_small_free_list(szone_t *szone);
-static void print_small_region(szone_t *szone, boolean_t verbose, region_t region, size_t bytes_at_start, size_t bytes_at_end);
-static boolean_t small_free_list_check(szone_t *szone, grain_t grain);
-
-#if DEBUG_MALLOC
-static void large_debug_print(szone_t *szone);
-#endif
-static large_entry_t *large_entry_for_pointer_no_lock(szone_t *szone, const void *ptr);
-static void large_entry_insert_no_lock(szone_t *szone, large_entry_t range);
-static INLINE void large_entries_rehash_after_entry_no_lock(szone_t *szone, large_entry_t *entry) ALWAYSINLINE;
-static INLINE large_entry_t *large_entries_alloc_no_lock(szone_t *szone, unsigned num) ALWAYSINLINE;
-static void large_entries_free_no_lock(szone_t *szone, large_entry_t *entries, unsigned num,
- vm_range_t *range_to_deallocate);
-static large_entry_t *large_entries_grow_no_lock(szone_t *szone, vm_range_t *range_to_deallocate);
-static vm_range_t large_entry_free_no_lock(szone_t *szone, large_entry_t *entry);
-static NOINLINE kern_return_t large_in_use_enumerator(task_t task, void *context,
- unsigned type_mask, vm_address_t large_entries_address,
- unsigned num_entries, memory_reader_t reader,
- vm_range_recorder_t recorder);
-static void *large_malloc(szone_t *szone, size_t num_pages, unsigned char alignment, boolean_t cleared_requested);
-static NOINLINE void free_large(szone_t *szone, void *ptr);
-static INLINE int large_try_realloc_in_place(szone_t *szone, void *ptr, size_t old_size, size_t new_size) ALWAYSINLINE;
-
-/*
- * Mark these NOINLINE to avoid bloating the purgeable zone call backs
- */
-static NOINLINE void szone_free(szone_t *szone, void *ptr);
-static NOINLINE void *szone_malloc_should_clear(szone_t *szone, size_t size, boolean_t cleared_requested);
-static NOINLINE void *szone_malloc(szone_t *szone, size_t size);
-static NOINLINE void *szone_calloc(szone_t *szone, size_t num_items, size_t size);
-static NOINLINE void *szone_valloc(szone_t *szone, size_t size);
-static NOINLINE size_t szone_size_try_large(szone_t *szone, const void *ptr);
-static NOINLINE size_t szone_size(szone_t *szone, const void *ptr);
-static NOINLINE void *szone_realloc(szone_t *szone, void *ptr, size_t new_size);
-static NOINLINE void *szone_memalign(szone_t *szone, size_t alignment, size_t size);
-static NOINLINE void szone_free_definite_size(szone_t *szone, void *ptr, size_t size);
-static NOINLINE unsigned szone_batch_malloc(szone_t *szone, size_t size, void **results, unsigned count);
-static NOINLINE void szone_batch_free(szone_t *szone, void **to_be_freed, unsigned count);
-static void szone_destroy(szone_t *szone);
-static NOINLINE size_t szone_good_size(szone_t *szone, size_t size);
-
-static NOINLINE boolean_t szone_check_all(szone_t *szone, const char *function);
-static boolean_t szone_check(szone_t *szone);
-static kern_return_t szone_ptr_in_use_enumerator(task_t task, void *context,
- unsigned type_mask, vm_address_t zone_address,
- memory_reader_t reader, vm_range_recorder_t recorder);
-static NOINLINE void szone_print(szone_t *szone, boolean_t verbose);
-static void szone_log(malloc_zone_t *zone, void *log_address);
-static void szone_force_lock(szone_t *szone);
-static void szone_force_unlock(szone_t *szone);
-static boolean_t szone_locked(szone_t *szone);
-
-static void szone_statistics(szone_t *szone, malloc_statistics_t *stats);
-
-static void purgeable_free(szone_t *szone, void *ptr);
-static void *purgeable_malloc(szone_t *szone, size_t size);
-static void *purgeable_calloc(szone_t *szone, size_t num_items, size_t size);
-static void *purgeable_valloc(szone_t *szone, size_t size);
-static size_t purgeable_size(szone_t *szone, const void *ptr);
-static void *purgeable_realloc(szone_t *szone, void *ptr, size_t new_size);
-static void *purgeable_memalign(szone_t *szone, size_t alignment, size_t size);
-static void purgeable_free_definite_size(szone_t *szone, void *ptr, size_t size);
-static unsigned purgeable_batch_malloc(szone_t *szone, size_t size, void **results, unsigned count);
-static void purgeable_batch_free(szone_t *szone, void **to_be_freed, unsigned count);
-static void purgeable_destroy(szone_t *szone);
-static size_t purgeable_good_size(szone_t *szone, size_t size);
-
-static boolean_t purgeable_check(szone_t *szone);
-static kern_return_t purgeable_ptr_in_use_enumerator(task_t task, void *context,
- unsigned type_mask, vm_address_t zone_address,
- memory_reader_t reader, vm_range_recorder_t recorder);
-static void purgeable_print(szone_t *szone, boolean_t verbose);
-static void purgeable_log(malloc_zone_t *zone, void *log_address);
-static void purgeable_force_lock(szone_t *szone);
-static void purgeable_force_unlock(szone_t *szone);
-static boolean_t purgeable_locked(szone_t *szone);
-
-static void purgeable_statistics(szone_t *szone, malloc_statistics_t *stats);
-
-static void *frozen_malloc(szone_t *zone, size_t new_size);
-static void *frozen_calloc(szone_t *zone, size_t num_items, size_t size);
-static void *frozen_valloc(szone_t *zone, size_t new_size);
-static void *frozen_realloc(szone_t *zone, void *ptr, size_t new_size);
-static void frozen_free(szone_t *zone, void *ptr);
-static void frozen_destroy(szone_t *zone);
-
-static volatile uintptr_t entropic_address = 0;
-static volatile uintptr_t entropic_limit = 0;
-#define ENTROPIC_KABILLION 0x10000000 /* 256Mb */
-
-__private_extern__ uint64_t malloc_entropy[2];
-
-#define SZONE_LOCK(szone) \
- do { \
- LOCK(szone->large_szone_lock); \
- } while (0)
-
-#define SZONE_UNLOCK(szone) \
- do { \
- UNLOCK(szone->large_szone_lock); \
- } while (0)
-
-#define SZONE_TRY_LOCK(szone) \
- TRY_LOCK(szone->large_szone_lock);
-
-#define SZONE_MAGAZINE_PTR_LOCK(szone, mag_ptr) \
- do { \
- LOCK(mag_ptr->magazine_lock); \
- } while(0)
-
-#define SZONE_MAGAZINE_PTR_UNLOCK(szone, mag_ptr) \
- do { \
- UNLOCK(mag_ptr->magazine_lock); \
- } while(0)
-
-#define SZONE_MAGAZINE_PTR_TRY_LOCK(szone, mag_ptr) \
- TRY_LOCK(mag_ptr->magazine_lock);
-
-#if DEBUG_MALLOC
-# define LOG(szone,ptr) \
- (szone->log_address && (((uintptr_t)szone->log_address == -1) || \
- (szone->log_address == (void *)(ptr))))
-#else
-# define LOG(szone,ptr) 0
-#endif
-
-#if DEBUG_MALLOC || DEBUG_CLIENT
-# define CHECK(szone,fun) \
- if ((szone)->debug_flags & CHECK_REGIONS) \
- szone_check_all(szone, fun)
-#else
-# define CHECK(szone,fun) \
- do {} while (0)
-#endif
-
-/********************* VERY LOW LEVEL UTILITIES ************************/
-
-static void
-szone_sleep(void)
-{
- if (getenv("MallocErrorStop")) {
- _malloc_printf(ASL_LEVEL_NOTICE, "*** sending SIGSTOP to help debug\n");
- kill(getpid(), SIGSTOP);
- } else if (getenv("MallocErrorSleep")) {
- _malloc_printf(ASL_LEVEL_NOTICE, "*** sleeping to help debug\n");
- sleep(3600); // to help debug
- }
-}
-
-// msg prints after fmt, ...
-static NOINLINE void
-szone_error(szone_t *szone, int is_corruption, const char *msg, const void *ptr, const char *fmt, ...)
-{
- va_list ap;
- _SIMPLE_STRING b = _simple_salloc();
-
- if (szone) SZONE_UNLOCK(szone); // FIXME: unlock magazine and region locks?
- if (b) {
- if (fmt) {
- va_start(ap, fmt);
- _simple_vsprintf(b, fmt, ap);
- va_end(ap);
- }
- if (ptr) {
- _simple_sprintf(b, "*** error for object %p: %s\n", ptr, msg);
- } else {
- _simple_sprintf(b, "*** error: %s\n", msg);
- }
- malloc_printf("%s*** set a breakpoint in malloc_error_break to debug\n", _simple_string(b));
- } else {
- /*
- * Should only get here if vm_allocate() can't get a single page of
- * memory, implying _simple_asl_log() would also fail. So we just
- * print to the file descriptor.
- */
- if (fmt) {
- va_start(ap, fmt);
- _malloc_vprintf(MALLOC_PRINTF_NOLOG, fmt, ap);
- va_end(ap);
- }
- if (ptr) {
- _malloc_printf(MALLOC_PRINTF_NOLOG, "*** error for object %p: %s\n", ptr, msg);
- } else {
- _malloc_printf(MALLOC_PRINTF_NOLOG, "*** error: %s\n", msg);
- }
- _malloc_printf(MALLOC_PRINTF_NOLOG, "*** set a breakpoint in malloc_error_break to debug\n");
- }
- malloc_error_break();
-#if DEBUG_MALLOC
- szone_print(szone, 1);
-#endif
- szone_sleep();
- // Call abort() if this is a memory corruption error and the abort on
- // corruption flag is set, or if any error should abort.
- if ((is_corruption && (szone->debug_flags & SCALABLE_MALLOC_ABORT_ON_CORRUPTION)) ||
- (szone->debug_flags & SCALABLE_MALLOC_ABORT_ON_ERROR)) {
- CRSetCrashLogMessage(b ? _simple_string(b) : msg);
- abort();
- } else if (b) {
- _simple_sfree(b);
- }
-}
-
-static void
-protect(void *address, size_t size, unsigned protection, unsigned debug_flags)
-{
- kern_return_t err;
-
- if (!(debug_flags & SCALABLE_MALLOC_DONT_PROTECT_PRELUDE)) {
- err = mprotect((void *)((uintptr_t)address - vm_page_size), vm_page_size, protection);
- if (err) {
- malloc_printf("*** can't protect(%p) region for prelude guard page at %p\n",
- protection,(uintptr_t)address - (1 << vm_page_shift));
- }
- }
- if (!(debug_flags & SCALABLE_MALLOC_DONT_PROTECT_POSTLUDE)) {
- err = mprotect((void *)((uintptr_t)address + size), vm_page_size, protection);
- if (err) {
- malloc_printf("*** can't protect(%p) region for postlude guard page at %p\n",
- protection, (uintptr_t)address + size);
- }
- }
-}
-
-static void *
-allocate_pages(szone_t *szone, size_t size, unsigned char align, unsigned debug_flags, int vm_page_label)
-{
- // align specifies a desired alignment (as a log) or 0 if no alignment requested
- void *vm_addr;
- uintptr_t addr = 0, aligned_address;
- boolean_t add_guard_pages = debug_flags & SCALABLE_MALLOC_ADD_GUARD_PAGES;
- boolean_t purgeable = debug_flags & SCALABLE_MALLOC_PURGEABLE;
- size_t allocation_size = round_page(size);
- size_t delta;
- int alloc_flags = VM_MAKE_TAG(vm_page_label);
-
- if (align) add_guard_pages = 0; // too cumbersome to deal with that
- if (!allocation_size) allocation_size = 1 << vm_page_shift;
- if (add_guard_pages) allocation_size += 2 * (1 << vm_page_shift);
- if (align) allocation_size += (size_t)1 << align;
- if (purgeable) alloc_flags |= VM_FLAGS_PURGABLE;
-
- if (allocation_size < size) // size_t arithmetic wrapped!
- return NULL;
-
- vm_addr = mmap(0 /* addr */,
- allocation_size /* size */,
- PROT_READ | PROT_WRITE /* prot */,
- MAP_ANON | MAP_PRIVATE /* flags */,
- alloc_flags /* fd being used to pass "purgeable" and "vm_page_label" */,
- 0 /* offset */);
- if ((uintptr_t)vm_addr == -1) {
- szone_error(szone, 0, "can't allocate region", NULL, "*** mmap(size=%lu) failed (error code=%d)\n",
- allocation_size, errno);
- return NULL;
- }
- addr = (uintptr_t)vm_addr;
-
- if (align) {
- aligned_address = (addr + ((uintptr_t)1 << align) - 1) & ~ (((uintptr_t)1 << align) - 1);
- if (aligned_address != addr) {
- delta = aligned_address - addr;
- if (munmap((void *)addr, delta) == -1)
- malloc_printf("*** munmap unaligned header failed with %d\n", errno);
- addr = aligned_address;
- allocation_size -= delta;
- }
- if (allocation_size > size) {
- if (munmap((void *)(addr + size), allocation_size - size) == -1)
- malloc_printf("*** munmap unaligned footer failed with %d\n", errno);
- }
- }
- if (add_guard_pages) {
- addr += (uintptr_t)1 << vm_page_shift;
- protect((void *)addr, size, PROT_NONE, debug_flags);
- }
- return (void *)addr;
-}
-
-static void *
-allocate_pages_securely(szone_t *szone, size_t size, unsigned char align, int vm_page_label)
-{
- // align specifies a desired alignment (as a log) or 0 if no alignment requested
- void *vm_addr;
- uintptr_t addr, aligned_address;
- size_t delta, allocation_size = MAX(round_page(size), vm_page_size);
- int alloc_flags = VM_MAKE_TAG(vm_page_label);
-
- if (szone->debug_flags & DISABLE_ASLR)
- return allocate_pages(szone, size, align, 0, vm_page_label);
-
- if (align)
- allocation_size += (size_t)1 << align;
-
- if (allocation_size < size) // size_t arithmetic wrapped!
- return NULL;
-
-retry:
- vm_addr = mmap((void *)entropic_address /* kernel finds next available range at or above this address */,
- allocation_size /* size */,
- PROT_READ | PROT_WRITE /* prot */,
- MAP_ANON | MAP_PRIVATE /* flags */,
- alloc_flags /* fd being used to pass "vm_page_label" */,
- 0 /* offset */);
- if (MAP_FAILED == vm_addr) {
- szone_error(szone, 0, "can't allocate region securely", NULL, "*** mmap(size=%lu) failed (error code=%d)\n",
- size, errno);
- return NULL;
- }
- addr = (uintptr_t)vm_addr;
-
- // Don't allow allocation to rise above entropic_limit (for tidiness).
- if (addr + allocation_size > entropic_limit) { // Exhausted current range?
- uintptr_t t = entropic_address;
- uintptr_t u = t - ENTROPIC_KABILLION;
-
- if (u < t) { // provided we don't wrap, unmap and retry, in the expanded entropic range
- munmap((void *)addr, allocation_size);
- (void)__sync_bool_compare_and_swap(&entropic_address, t, u); // Just one reduction please
- goto retry;
- }
- // fall through to use what we got
- }
-
- if (addr < entropic_address) { // mmap wrapped to find this allocation, expand the entropic range
- uintptr_t t = entropic_address;
- uintptr_t u = t - ENTROPIC_KABILLION;
- if (u < t)
- (void)__sync_bool_compare_and_swap(&entropic_address, t, u); // Just one reduction please
- // fall through to use what we got
- }
-
- // unmap any excess address range used for alignment padding
- if (align) {
- aligned_address = (addr + ((uintptr_t)1 << align) - 1) & ~ (((uintptr_t)1 << align) - 1);
- if (aligned_address != addr) {
- delta = aligned_address - addr;
- if (munmap((void *)addr, delta) == -1)
- malloc_printf("*** munmap unaligned header failed with %d\n", errno);
- addr = aligned_address;
- allocation_size -= delta;
- }
- if (allocation_size > size) {
- if (munmap((void *)(addr + size), allocation_size - size) == -1)
- malloc_printf("*** munmap unaligned footer failed with %d\n", errno);
- }
- }
- return (void *)addr;
-}
-
-static void
-deallocate_pages(szone_t *szone, void *addr, size_t size, unsigned debug_flags)
-{
- int err;
- boolean_t add_guard_pages = debug_flags & SCALABLE_MALLOC_ADD_GUARD_PAGES;
-
- if (add_guard_pages) {
- addr = (void *)((uintptr_t)addr - (1 << vm_page_shift));
- size += 2 * (1 << vm_page_shift);
- }
- err = munmap(addr, size);
- if ((err == -1) && szone)
- szone_error(szone, 0, "Can't deallocate_pages region", addr, NULL);
-}
-
-static int
-#if TARGET_OS_EMBEDDED
-madvise_free_range(szone_t *szone, region_t r, uintptr_t pgLo, uintptr_t pgHi, uintptr_t *last)
-#else
-madvise_free_range(szone_t *szone, region_t r, uintptr_t pgLo, uintptr_t pgHi)
-#endif
-{
- if (pgHi > pgLo) {
- size_t len = pgHi - pgLo;
-
-#if DEBUG_MALLOC
- if (szone->debug_flags & SCALABLE_MALLOC_DO_SCRIBBLE)
- memset((void *)pgLo, 0xed, len); // Scribble on MADV_FREEd memory
-#endif
-
-#if TARGET_OS_EMBEDDED
- if (last) {
- if (*last == pgLo)
- return 0;
-
- *last = pgLo;
- }
-#endif
-
- MAGMALLOC_MADVFREEREGION((void *)szone, (void *)r, (void *)pgLo, len); // DTrace USDT Probe
-#if TARGET_OS_EMBEDDED
- if (-1 == madvise((void *)pgLo, len, MADV_FREE)) {
-#else
- if (-1 == madvise((void *)pgLo, len, MADV_FREE_REUSABLE)) {
-#endif
- /* -1 return: VM map entry change makes this unfit for reuse. Something evil lurks. */
-#if DEBUG_MADVISE
- szone_error(szone, 0, "madvise_free_range madvise(..., MADV_FREE_REUSABLE) failed",
- (void *)pgLo, "length=%d\n", len);
-#endif
- }
- }
- return 0;
-}
-
-static kern_return_t
-_szone_default_reader(task_t task, vm_address_t address, vm_size_t size, void **ptr)
-{
- *ptr = (void *)address;
- return 0;
-}
-
-// Multiplicative hash where the multiplier is a prime near (ULONG_MAX / phi). [phi = 1.618033...]
-// pthread_t's are page aligned, (sometimes even in ascending sequence). These hash well.
-// See Knuth TAOCP, Vol. 3.
-#if __LP64__
-#define HASH_SELF() \
- ((((uintptr_t)pthread_self()) >> vm_page_shift) * 11400714819323198549ULL) >> (64 - szone->num_tiny_magazines_mask_shift)
-#else
-#define HASH_SELF() \
- ((((uintptr_t)pthread_self()) >> vm_page_shift) * 2654435761UL) >> (32 - szone->num_tiny_magazines_mask_shift)
-#endif
-
-#if defined(__i386__) || defined(__x86_64__) || defined(__arm__)
-/*
- * These commpage routines provide fast access to the logical cpu number
- * of the calling processor assuming no pre-emption occurs.
- */
-
-static INLINE mag_index_t
-mag_get_thread_index(szone_t *szone)
-{
- if (!__is_threaded)
- return 0;
- else
- return cpu_number() & (TINY_MAX_MAGAZINES - 1);
-}
-
-#else
-#warning deriving magazine index from pthread_self() [want processor number]
-
-static INLINE mag_index_t
-mag_get_thread_index(szone_t *szone)
-{
- if (!__is_threaded)
- return 0;
- else if ((pthread_key_t) -1 == szone->cpu_id_key) { // In case pthread_key_create() failed.
- return HASH_SELF();
- } else {
- mag_index_t idx = (mag_index_t)(intptr_t)pthread_getspecific(szone->cpu_id_key);
-
- // Has this thread been hinted with a non-zero value [i.e. 1 + cpuid()] ?
- // If so, bump down the hint to a zero-based magazine index and return it.
- if (idx) {
- return idx - 1;
- } else {
- // No hint available. Contruct a magazine index for this thread ...
- idx = HASH_SELF();
-
- // bump up the hint to exclude zero and try to memorize it ...
- pthread_setspecific(szone->cpu_id_key, (const void *)((uintptr_t)idx + 1));
-
- // and return the (zero-based) magazine index.
- return idx;
- }
- }
-}
-#endif
-
-static magazine_t *
-mag_lock_zine_for_region_trailer(szone_t *szone, magazine_t *magazines, region_trailer_t *trailer, mag_index_t mag_index)
-{
- mag_index_t refreshed_index;
- magazine_t *mag_ptr = &(magazines[mag_index]);
-
- // Take the lock on entry.
- SZONE_MAGAZINE_PTR_LOCK(szone, mag_ptr);
-
- // Now in the time it took to acquire the lock, the region may have migrated
- // from one magazine to another. In which case the magazine lock we obtained
- // (namely magazines[mag_index].mag_lock) is stale. If so, keep on tryin' ...
- while (mag_index != (refreshed_index = trailer->mag_index)) { // Note assignment
-
- SZONE_MAGAZINE_PTR_UNLOCK(szone, mag_ptr);
-
- mag_index = refreshed_index;
- mag_ptr = &(magazines[mag_index]);
- SZONE_MAGAZINE_PTR_LOCK(szone, mag_ptr);
- }
-
- return mag_ptr;
-}
-
-/*******************************************************************************
- * Region hash implementation
- *
- * This is essentially a duplicate of the existing Large allocator hash, minus
- * the ability to remove entries. The two should be combined eventually.
- ******************************************************************************/
-#pragma mark region hash
-
-/*
- * hash_lookup_region_no_lock - Scan a hash ring looking for an entry for a
- * given region.
- *
- * FIXME: If consecutive queries of the same region are likely, a one-entry
- * cache would likely be a significant performance win here.
- */
-static INLINE rgnhdl_t
-hash_lookup_region_no_lock(region_t *regions, size_t num_entries, size_t shift, region_t r) {
- size_t index, hash_index;
- rgnhdl_t entry;
-
- if (!num_entries)
- return 0;
-
- // Multiplicative hash where the multiplier is a prime near (ULONG_MAX / phi). [phi = 1.618033...]
- // Since the values of (((uintptr_t)r >> HASH_BLOCKS_ALIGN) are (roughly) an ascending sequence of integers,
- // this hash works really well. See Knuth TAOCP, Vol. 3.
-#if __LP64__
- index = hash_index = (((uintptr_t)r >> HASH_BLOCKS_ALIGN) * 11400714819323198549ULL) >> (64 - shift);
-#else
- index = hash_index = (((uintptr_t)r >> HASH_BLOCKS_ALIGN) * 2654435761UL) >> (32 - shift);
-#endif
- do {
- entry = regions + index;
- if (*entry == 0)
- return 0;
- if (*entry == r)
- return entry;
- if (++index == num_entries)
- index = 0;
- } while (index != hash_index);
- return 0;
-}
-
-/*
- * hash_region_insert_no_lock - Insert a region into the hash ring.
- */
-static void
-hash_region_insert_no_lock(region_t *regions, size_t num_entries, size_t shift, region_t r) {
- size_t index, hash_index;
- rgnhdl_t entry;
-
- // Multiplicative hash where the multiplier is a prime near (ULONG_MAX / phi). [phi = 1.618033...]
- // Since the values of (((uintptr_t)r >> HASH_BLOCKS_ALIGN) are (roughly) an ascending sequence of integers,
- // this hash works really well. See Knuth TAOCP, Vol. 3.
-#if __LP64__
- index = hash_index = (((uintptr_t)r >> HASH_BLOCKS_ALIGN) * 11400714819323198549ULL) >> (64 - shift);
-#else
- index = hash_index = (((uintptr_t)r >> HASH_BLOCKS_ALIGN) * 2654435761UL) >> (32 - shift);
-#endif
- do {
- entry = regions + index;
- if (*entry == HASHRING_OPEN_ENTRY || *entry == HASHRING_REGION_DEALLOCATED) {
- *entry = r;
- return;
- }
- if (++index == num_entries)
- index = 0;
- } while (index != hash_index);
-}
-
-/*
- * hash_regions_alloc_no_lock - Allocate space for a number of entries. This
- * must be a VM allocation as to avoid recursing between allocating a new small
- * region, and asking the small region to allocate space for the new list of
- * regions.
- */
-static region_t *
-hash_regions_alloc_no_lock(szone_t *szone, size_t num_entries)
-{
- size_t size = num_entries * sizeof(region_t);
-
- return allocate_pages(szone, round_page(size), 0, 0, VM_MEMORY_MALLOC);
-}
-
-/*
- * hash_regions_grow_no_lock - Grow the hash ring, and rehash the entries.
- * Return the new region and new size to update the szone. Do not deallocate
- * the old entries since someone may still be allocating them.
- */
-static region_t *
-hash_regions_grow_no_lock(szone_t *szone, region_t *regions, size_t old_size, size_t *mutable_shift,
- size_t *new_size)
-{
- // double in size and allocate memory for the regions
- *new_size = old_size + old_size;
- *mutable_shift = *mutable_shift + 1;
- region_t *new_regions = hash_regions_alloc_no_lock(szone, *new_size);
-
- // rehash the entries into the new list
- size_t index;
- for (index = 0; index < old_size; ++index) {
- region_t r = regions[index];
- if (r != HASHRING_OPEN_ENTRY && r != HASHRING_REGION_DEALLOCATED)
- hash_region_insert_no_lock(new_regions, *new_size, *mutable_shift, r);
- }
- return new_regions;
-}
-
-/********************* FREE LIST UTILITIES ************************/
-
-// A free list entry is comprised of a pair of pointers, previous and next.
-// These are used to implement a doubly-linked list, which permits efficient
-// extraction.
-//
-// Because the free list entries are previously freed objects, a misbehaved
-// program may write to a pointer after it has called free() on that pointer,
-// either by dereference or buffer overflow from an adjacent pointer. This write
-// would then corrupt the free list's previous and next pointers, leading to a
-// crash. In order to detect this case, we take advantage of the fact that
-// malloc'd pointers are known to be at least 16 byte aligned, and thus have
-// at least 4 trailing zero bits.
-//
-// When an entry is added to the free list, a checksum of the previous and next
-// pointers is calculated and written to the high four bits of the respective
-// pointers. Upon detection of an invalid checksum, an error is logged and NULL
-// is returned. Since all code which un-checksums pointers checks for a NULL
-// return, a potentially crashing or malicious dereference is avoided at the
-// cost of leaking the corrupted block, and any subsequent blocks on the free
-// list of that size.
-
-static NOINLINE void
-free_list_checksum_botch(szone_t *szone, free_list_t *ptr)
-{
- szone_error(szone, 1, "incorrect checksum for freed object "
- "- object was probably modified after being freed.", ptr, NULL);
-}
-
-static INLINE uintptr_t free_list_gen_checksum(uintptr_t ptr)
-{
- uint8_t chk;
-
- chk = (unsigned char)(ptr >> 0);
- chk += (unsigned char)(ptr >> 8);
- chk += (unsigned char)(ptr >> 16);
- chk += (unsigned char)(ptr >> 24);
-#if __LP64__
- chk += (unsigned char)(ptr >> 32);
- chk += (unsigned char)(ptr >> 40);
- chk += (unsigned char)(ptr >> 48);
- chk += (unsigned char)(ptr >> 56);
-#endif
-
- return chk & (uintptr_t)0xF;
-}
-
-#define NYBBLE 4
-#if __LP64__
-#define ANTI_NYBBLE (64 - NYBBLE)
-#else
-#define ANTI_NYBBLE (32 - NYBBLE)
-#endif
-
-static INLINE uintptr_t
-free_list_checksum_ptr(szone_t *szone, void *ptr)
-{
- uintptr_t p = (uintptr_t)ptr;
- return (p >> NYBBLE) | (free_list_gen_checksum(p ^ szone->cookie) << ANTI_NYBBLE); // compiles to rotate instruction
-}
-
-static INLINE void *
-free_list_unchecksum_ptr(szone_t *szone, ptr_union *ptr)
-{
- ptr_union p;
- uintptr_t t = ptr->u;
-
- t = (t << NYBBLE) | (t >> ANTI_NYBBLE); // compiles to rotate instruction
- p.u = t & ~(uintptr_t)0xF;
-
- if ((t & (uintptr_t)0xF) != free_list_gen_checksum(p.u ^ szone->cookie))
- {
- free_list_checksum_botch(szone, (free_list_t *)ptr);
- return NULL;
- }
- return p.p;
-}
-
-#undef ANTI_NYBBLE
-#undef NYBBLE
-
-static unsigned
-free_list_count(szone_t *szone, free_list_t *ptr)
-{
- unsigned count = 0;
-
- while (ptr) {
- count++;
- ptr = free_list_unchecksum_ptr(szone, &ptr->next);
- }
- return count;
-}
-
-static INLINE void
-recirc_list_extract(szone_t *szone, magazine_t *mag_ptr, region_trailer_t *node)
-{
- // excise node from list
- if (NULL == node->prev)
- mag_ptr->firstNode = node->next;
- else
- node->prev->next = node->next;
-
- if (NULL == node->next)
- mag_ptr->lastNode = node->prev;
- else
- node->next->prev = node->prev;
-
- mag_ptr->recirculation_entries--;
-}
-
-static INLINE void
-recirc_list_splice_last(szone_t *szone, magazine_t *mag_ptr, region_trailer_t *node)
-{
- if (NULL == mag_ptr->lastNode) {
- mag_ptr->firstNode = node;
- node->prev = NULL;
- } else {
- node->prev = mag_ptr->lastNode;
- mag_ptr->lastNode->next = node;
- }
- mag_ptr->lastNode = node;
- node->next = NULL;
- node->recirc_suitable = FALSE;
- mag_ptr->recirculation_entries++;
-}
-
-static INLINE void
-recirc_list_splice_first(szone_t *szone, magazine_t *mag_ptr, region_trailer_t *node)
-{
- if (NULL == mag_ptr->firstNode) {
- mag_ptr->lastNode = node;
- node->next = NULL;
- } else {
- node->next = mag_ptr->firstNode;
- mag_ptr->firstNode->prev = node;
- }
- mag_ptr->firstNode = node;
- node->prev = NULL;
- node->recirc_suitable = FALSE;
- mag_ptr->recirculation_entries++;
-}
-
-/* Macros used to manipulate the uint32_t quantity mag_bitmap. */
-
-/* BITMAPV variants are used by tiny. */
-#if defined(__LP64__)
-// assert(NUM_SLOTS == 64) in which case (slot >> 5) is either 0 or 1
-#define BITMAPV_SET(bitmap,slot) (bitmap[(slot) >> 5] |= 1 << ((slot) & 31))
-#define BITMAPV_CLR(bitmap,slot) (bitmap[(slot) >> 5] &= ~ (1 << ((slot) & 31)))
-#define BITMAPV_BIT(bitmap,slot) ((bitmap[(slot) >> 5] >> ((slot) & 31)) & 1)
-#define BITMAPV_CTZ(bitmap) (__builtin_ctzl(bitmap))
-#else
-// assert(NUM_SLOTS == 32) in which case (slot >> 5) is always 0, so code it that way
-#define BITMAPV_SET(bitmap,slot) (bitmap[0] |= 1 << (slot))
-#define BITMAPV_CLR(bitmap,slot) (bitmap[0] &= ~ (1 << (slot)))
-#define BITMAPV_BIT(bitmap,slot) ((bitmap[0] >> (slot)) & 1)
-#define BITMAPV_CTZ(bitmap) (__builtin_ctz(bitmap))
-#endif
-
-/* BITMAPN is used by small. (slot >> 5) takes on values from 0 to 7. */
-#define BITMAPN_SET(bitmap,slot) (bitmap[(slot) >> 5] |= 1 << ((slot) & 31))
-#define BITMAPN_CLR(bitmap,slot) (bitmap[(slot) >> 5] &= ~ (1 << ((slot) & 31)))
-#define BITMAPN_BIT(bitmap,slot) ((bitmap[(slot) >> 5] >> ((slot) & 31)) & 1)
-
-/* returns bit # of least-significant one bit, starting at 0 (undefined if !bitmap) */
-#define BITMAP32_CTZ(bitmap) (__builtin_ctz(bitmap[0]))
-
-/********************* TINY FREE LIST UTILITIES ************************/
-
-// We encode the meta-headers as follows:
-// Each quantum has an associated set of 2 bits:
-// block_header when 1 says this block is the beginning of a block
-// in_use when 1 says this block is in use
-// so a block in use of size 3 is 1-1 0-X 0-X
-// for a free block TINY_FREE_SIZE(ptr) carries the size and the bits are 1-0 X-X X-X
-// for a block middle the bits are 0-0
-
-// We store the meta-header bit arrays by interleaving them 32 bits at a time.
-// Initial 32 bits of block_header, followed by initial 32 bits of in_use, followed
-// by next 32 bits of block_header, followed by next 32 bits of in_use, etc.
-// This localizes memory references thereby reducing cache and TLB pressures.
-
-static INLINE void
-BITARRAY_SET(uint32_t *bits, msize_t index)
-{
- // index >> 5 identifies the uint32_t to manipulate in the conceptually contiguous bits array
- // (index >> 5) << 1 identifies the uint32_t allowing for the actual interleaving
- bits[(index >> 5) << 1] |= (1 << (index & 31));
-}
-
-static INLINE void
-BITARRAY_CLR(uint32_t *bits, msize_t index)
-{
- bits[(index >> 5) << 1] &= ~(1 << (index & 31));
-}
-
-static INLINE boolean_t
-BITARRAY_BIT(uint32_t *bits, msize_t index)
-{
- return ((bits[(index >> 5) << 1]) >> (index & 31)) & 1;
-}
-
-#if 0
-static INLINE void bitarray_mclr(uint32_t *bits, unsigned start, unsigned end) ALWAYSINLINE;
-
-static INLINE void
-bitarray_mclr(uint32_t *bits, unsigned start, unsigned end)
-{
- // start >> 5 identifies the uint32_t to manipulate in the conceptually contiguous bits array
- // (start >> 5) << 1 identifies the uint32_t allowing for the actual interleaving
- uint32_t *addr = bits + ((start >> 5) << 1);
-
- uint32_t span = end - start;
- start = start & 31;
- end = start + span;
-
- if (end > 31) {
- addr[0] &= (0xFFFFFFFFU >> (31 - start)) >> 1;
- addr[2] &= (0xFFFFFFFFU << (end - 32));
- } else {
- unsigned mask = (0xFFFFFFFFU >> (31 - start)) >> 1;
- mask |= (0xFFFFFFFFU << end);
- addr[0] &= mask;
- }
-}
-#endif
-
-/*
- * Obtain the size of a free tiny block (in msize_t units).
- */
-static msize_t
-get_tiny_free_size(const void *ptr)
-{
- void *next_block = (void *)((uintptr_t)ptr + TINY_QUANTUM);
- void *region_end = TINY_REGION_END(TINY_REGION_FOR_PTR(ptr));
-
- // check whether the next block is outside the tiny region or a block header
- // if so, then the size of this block is one, and there is no stored size.
- if (next_block < region_end)
- {
- uint32_t *next_header = TINY_BLOCK_HEADER_FOR_PTR(next_block);
- msize_t next_index = TINY_INDEX_FOR_PTR(next_block);
-
- if (!BITARRAY_BIT(next_header, next_index))
- return TINY_FREE_SIZE(ptr);
- }
- return 1;
-}
-
-/*
- * Get the size of the previous free block, which is stored in the last two
- * bytes of the block. If the previous block is not free, then the result is
- * undefined.
- */
-static msize_t
-get_tiny_previous_free_msize(const void *ptr)
-{
- // check whether the previous block is in the tiny region and a block header
- // if so, then the size of the previous block is one, and there is no stored
- // size.
- if (ptr != TINY_REGION_FOR_PTR(ptr))
- {
- void *prev_block = (void *)((uintptr_t)ptr - TINY_QUANTUM);
- uint32_t *prev_header = TINY_BLOCK_HEADER_FOR_PTR(prev_block);
- msize_t prev_index = TINY_INDEX_FOR_PTR(prev_block);
- if (BITARRAY_BIT(prev_header, prev_index))
- return 1;
- return TINY_PREVIOUS_MSIZE(ptr);
- }
- // don't read possibly unmapped memory before the beginning of the region
- return 0;
-}
-
-static INLINE msize_t
-get_tiny_meta_header(const void *ptr, boolean_t *is_free)
-{
- // returns msize and is_free
- // may return 0 for the msize component (meaning 65536)
- uint32_t *block_header;
- msize_t index;
-
- block_header = TINY_BLOCK_HEADER_FOR_PTR(ptr);
- index = TINY_INDEX_FOR_PTR(ptr);
-
- msize_t midx = (index >> 5) << 1;
- uint32_t mask = 1 << (index & 31);
- *is_free = 0;
- if (0 == (block_header[midx] & mask)) // if (!BITARRAY_BIT(block_header, index))
- return 0;
- if (0 == (block_header[midx + 1] & mask)) { // if (!BITARRAY_BIT(in_use, index))
- *is_free = 1;
- return get_tiny_free_size(ptr);
- }
-
- // index >> 5 identifies the uint32_t to manipulate in the conceptually contiguous bits array
- // (index >> 5) << 1 identifies the uint32_t allowing for the actual interleaving
-#if defined(__LP64__)
- // The return value, msize, is computed as the distance to the next 1 bit in block_header.
- // That's guaranteed to be somewhwere in the next 64 bits. And those bits could span three
- // uint32_t block_header elements. Collect the bits into a single uint64_t and measure up with ffsl.
- uint32_t *addr = ((uint32_t *)block_header) + ((index >> 5) << 1);
- uint32_t bitidx = index & 31;
- uint64_t word_lo = addr[0];
- uint64_t word_mid = addr[2];
- uint64_t word_hi = addr[4];
- uint64_t word_lomid = (word_lo >> bitidx) | (word_mid << (32 - bitidx));
- uint64_t word = bitidx ? word_lomid | (word_hi << (64 - bitidx)) : word_lomid;
- uint32_t result = __builtin_ffsl(word >> 1);
-#else
- // The return value, msize, is computed as the distance to the next 1 bit in block_header.
- // That's guaranteed to be somwhwere in the next 32 bits. And those bits could span two
- // uint32_t block_header elements. Collect the bits into a single uint32_t and measure up with ffs.
- uint32_t *addr = ((uint32_t *)block_header) + ((index >> 5) << 1);
- uint32_t bitidx = index & 31;
- uint32_t word = bitidx ? (addr[0] >> bitidx) | (addr[2] << (32 - bitidx)) : addr[0];
- uint32_t result = __builtin_ffs(word >> 1);
-#endif
- return result;
-}
-
-static INLINE void
-set_tiny_meta_header_in_use(const void *ptr, msize_t msize)
-{
- uint32_t *block_header = TINY_BLOCK_HEADER_FOR_PTR(ptr);
- msize_t index = TINY_INDEX_FOR_PTR(ptr);
- msize_t clr_msize = msize - 1;
- msize_t midx = (index >> 5) << 1;
- uint32_t val = (1 << (index & 31));
-
-#if DEBUG_MALLOC
- if (msize >= NUM_TINY_SLOTS)
- malloc_printf("set_tiny_meta_header_in_use() invariant broken %p %d\n", ptr, msize);
- if ((unsigned)index + (unsigned)msize > 0x10000)
- malloc_printf("set_tiny_meta_header_in_use() invariant broken (2) %p %d\n", ptr, msize);
-#endif
-
- block_header[midx] |= val; // BITARRAY_SET(block_header, index);
- block_header[midx + 1] |= val; // BITARRAY_SET(in_use, index);
-
- // bitarray_mclr(block_header, index, end_bit);
- // bitarray_mclr(in_use, index, end_bit);
-
- index++;
- midx = (index >> 5) << 1;
-
- unsigned start = index & 31;
- unsigned end = start + clr_msize;
-
-#if defined(__LP64__)
- if (end > 63) {
- unsigned mask0 = (0xFFFFFFFFU >> (31 - start)) >> 1;
- unsigned mask1 = (0xFFFFFFFFU << (end - 64));
- block_header[midx + 0] &= mask0; // clear header
- block_header[midx + 1] &= mask0; // clear in_use
- block_header[midx + 2] = 0; // clear header
- block_header[midx + 3] = 0; // clear in_use
- block_header[midx + 4] &= mask1; // clear header
- block_header[midx + 5] &= mask1; // clear in_use
- } else
-#endif
- if (end > 31) {
- unsigned mask0 = (0xFFFFFFFFU >> (31 - start)) >> 1;
- unsigned mask1 = (0xFFFFFFFFU << (end - 32));
- block_header[midx + 0] &= mask0;
- block_header[midx + 1] &= mask0;
- block_header[midx + 2] &= mask1;
- block_header[midx + 3] &= mask1;
- } else {
- unsigned mask = (0xFFFFFFFFU >> (31 - start)) >> 1;
- mask |= (0xFFFFFFFFU << end);
- block_header[midx + 0] &= mask;
- block_header[midx + 1] &= mask;
- }
-
- // we set the block_header bit for the following block to reaffirm next block is a block
- index += clr_msize;
- midx = (index >> 5) << 1;
- val = (1 << (index & 31));
- block_header[midx] |= val; // BITARRAY_SET(block_header, (index+clr_msize));
-#if DEBUG_MALLOC
- {
- boolean_t ff;
- msize_t mf;
-
- mf = get_tiny_meta_header(ptr, &ff);
- if (msize != mf) {
- malloc_printf("setting header for tiny in_use %p : %d\n", ptr, msize);
- malloc_printf("reading header for tiny %p : %d %d\n", ptr, mf, ff);
- }
- }
-#endif
-}
-
-static INLINE void
-set_tiny_meta_header_in_use_1(const void *ptr) // As above with msize == 1
-{
- uint32_t *block_header = TINY_BLOCK_HEADER_FOR_PTR(ptr);
- msize_t index = TINY_INDEX_FOR_PTR(ptr);
- msize_t midx = (index >> 5) << 1;
- uint32_t val = (1 << (index & 31));
-
- block_header[midx] |= val; // BITARRAY_SET(block_header, index);
- block_header[midx + 1] |= val; // BITARRAY_SET(in_use, index);
-
- index++;
- midx = (index >> 5) << 1;
- val = (1 << (index & 31));
-
- block_header[midx] |= val; // BITARRAY_SET(block_header, (index+clr_msize))
-}
-
-static INLINE void
-set_tiny_meta_header_middle(const void *ptr)
-{
- // indicates this block is in the middle of an in use block
- uint32_t *block_header;
- uint32_t *in_use;
- msize_t index;
-
- block_header = TINY_BLOCK_HEADER_FOR_PTR(ptr);
- in_use = TINY_INUSE_FOR_HEADER(block_header);
- index = TINY_INDEX_FOR_PTR(ptr);
-
- BITARRAY_CLR(block_header, index);
- BITARRAY_CLR(in_use, index);
-}
-
-static INLINE void
-set_tiny_meta_header_free(const void *ptr, msize_t msize)
-{
- // !msize is acceptable and means 65536
- uint32_t *block_header = TINY_BLOCK_HEADER_FOR_PTR(ptr);
- msize_t index = TINY_INDEX_FOR_PTR(ptr);
- msize_t midx = (index >> 5) << 1;
- uint32_t val = (1 << (index & 31));
-
-#if DEBUG_MALLOC
- if ((unsigned)index + (unsigned)msize > 0x10000) {
- malloc_printf("setting header for tiny free %p msize too large: %d\n", ptr, msize);
- }
-#endif
-
- block_header[midx] |= val; // BITARRAY_SET(block_header, index);
- block_header[midx + 1] &= ~val; // BITARRAY_CLR(in_use, index);
-
- // mark the end of this block if msize is > 1. For msize == 0, the whole
- // region is free, so there is no following block. For msize == 1, there is
- // no space to write the size on 64 bit systems. The size for 1 quantum
- // blocks is computed from the metadata bitmaps.
- if (msize > 1) {
- void *follower = FOLLOWING_TINY_PTR(ptr, msize);
- TINY_PREVIOUS_MSIZE(follower) = msize;
- TINY_FREE_SIZE(ptr) = msize;
- }
- if (msize == 0) {
- TINY_FREE_SIZE(ptr) = msize;
- }
-#if DEBUG_MALLOC
- boolean_t ff;
- msize_t mf = get_tiny_meta_header(ptr, &ff);
- if ((msize != mf) || !ff) {
- malloc_printf("setting header for tiny free %p : %u\n", ptr, msize);
- malloc_printf("reading header for tiny %p : %u %u\n", ptr, mf, ff);
- }
-#endif
-}
-
-static INLINE boolean_t
-tiny_meta_header_is_free(const void *ptr)
-{
- uint32_t *block_header;
- uint32_t *in_use;
- msize_t index;
-
- block_header = TINY_BLOCK_HEADER_FOR_PTR(ptr);
- in_use = TINY_INUSE_FOR_HEADER(block_header);
- index = TINY_INDEX_FOR_PTR(ptr);
- if (!BITARRAY_BIT(block_header, index))
- return 0;
- return !BITARRAY_BIT(in_use, index);
-}
-
-static INLINE void *
-tiny_previous_preceding_free(void *ptr, msize_t *prev_msize)
-{
- // returns the previous block, assuming and verifying it's free
- uint32_t *block_header;
- uint32_t *in_use;
- msize_t index;
- msize_t previous_msize;
- msize_t previous_index;
- void *previous_ptr;
-
- block_header = TINY_BLOCK_HEADER_FOR_PTR(ptr);
- in_use = TINY_INUSE_FOR_HEADER(block_header);
- index = TINY_INDEX_FOR_PTR(ptr);
-
- if (!index)
- return NULL;
- if ((previous_msize = get_tiny_previous_free_msize(ptr)) > index)
- return NULL;
-
- previous_index = index - previous_msize;
- previous_ptr = (void *)((uintptr_t)TINY_REGION_FOR_PTR(ptr) + TINY_BYTES_FOR_MSIZE(previous_index));
- if (!BITARRAY_BIT(block_header, previous_index))
- return NULL;
- if (BITARRAY_BIT(in_use, previous_index))
- return NULL;
- if (get_tiny_free_size(previous_ptr) != previous_msize)
- return NULL;
-
- // conservative check did match true check
- *prev_msize = previous_msize;
- return previous_ptr;
-}
-
-/*
- * Adds an item to the proper free list, and also marks the meta-header of the
- * block properly.
- * Assumes szone has been locked
- */
-static void
-tiny_free_list_add_ptr(szone_t *szone, magazine_t *tiny_mag_ptr, void *ptr, msize_t msize)
-{
- grain_t slot = (!msize || (msize >= NUM_TINY_SLOTS)) ? NUM_TINY_SLOTS - 1 : msize - 1;
- free_list_t *free_ptr = ptr;
- free_list_t *free_head = tiny_mag_ptr->mag_free_list[slot];
-
-#if DEBUG_MALLOC
- if (LOG(szone,ptr)) {
- malloc_printf("in %s, ptr=%p, msize=%d\n", __FUNCTION__, ptr, msize);
- }
- if (((uintptr_t)ptr) & (TINY_QUANTUM - 1)) {
- szone_error(szone, 1, "tiny_free_list_add_ptr: Unaligned ptr", ptr, NULL);
- }
-#endif
- set_tiny_meta_header_free(ptr, msize);
- if (free_head) {
-#if DEBUG_MALLOC
- if (free_list_unchecksum_ptr(szone, &free_head->previous)) {
- szone_error(szone, 1, "tiny_free_list_add_ptr: Internal invariant broken (free_head->previous)", ptr,
- "ptr=%p slot=%d free_head=%p previous=%p\n", ptr, slot, (void *)free_head, free_head->previous.p);
- }
- if (! tiny_meta_header_is_free(free_head)) {
- szone_error(szone, 1, "tiny_free_list_add_ptr: Internal invariant broken (free_head is not a free pointer)", ptr,
- "ptr=%p slot=%d free_head=%p\n", ptr, slot, (void *)free_head);
- }
-#endif
- free_head->previous.u = free_list_checksum_ptr(szone, free_ptr);
- } else {
- BITMAPV_SET(tiny_mag_ptr->mag_bitmap, slot);
- }
- free_ptr->previous.u = free_list_checksum_ptr(szone, NULL);
- free_ptr->next.u = free_list_checksum_ptr(szone, free_head);
-
- tiny_mag_ptr->mag_free_list[slot] = free_ptr;
-}
-
-/*
- * Removes the item pointed to by ptr in the proper free list.
- * Assumes szone has been locked
- */
-static void
-tiny_free_list_remove_ptr(szone_t *szone, magazine_t *tiny_mag_ptr, void *ptr, msize_t msize)
-{
- grain_t slot = (!msize || (msize >= NUM_TINY_SLOTS)) ? NUM_TINY_SLOTS - 1 : msize - 1;
- free_list_t *free_ptr = ptr, *next, *previous;
-
- next = free_list_unchecksum_ptr(szone, &free_ptr->next);
- previous = free_list_unchecksum_ptr(szone, &free_ptr->previous);
-
-#if DEBUG_MALLOC
- if (LOG(szone,ptr)) {
- malloc_printf("In %s, ptr=%p, msize=%d\n", __FUNCTION__, ptr, msize);
- }
-#endif
- if (!previous) {
- // The block to remove is the head of the free list
-#if DEBUG_MALLOC
- if (tiny_mag_ptr->mag_free_list[slot] != ptr) {
- szone_error(szone, 1, "tiny_free_list_remove_ptr: Internal invariant broken (tiny_mag_ptr->mag_free_list[slot])", ptr,
- "ptr=%p slot=%d msize=%d tiny_mag_ptr->mag_free_list[slot]=%p\n",
- ptr, slot, msize, (void *)tiny_mag_ptr->mag_free_list[slot]);
- return;
- }
-#endif
- tiny_mag_ptr->mag_free_list[slot] = next;
- if (!next) BITMAPV_CLR(tiny_mag_ptr->mag_bitmap, slot);
- } else {
- // We know free_ptr is already checksummed, so we don't need to do it
- // again.
- previous->next = free_ptr->next;
- }
- if (next) {
- // We know free_ptr is already checksummed, so we don't need to do it
- // again.
- next->previous = free_ptr->previous;
- }
-}
-
-/*
- * tiny_region_for_ptr_no_lock - Returns the tiny region containing the pointer,
- * or NULL if not found.
- */
-static INLINE region_t
-tiny_region_for_ptr_no_lock(szone_t *szone, const void *ptr)
-{
- rgnhdl_t r = hash_lookup_region_no_lock(szone->tiny_region_generation->hashed_regions,
- szone->tiny_region_generation->num_regions_allocated,
- szone->tiny_region_generation->num_regions_allocated_shift,
- TINY_REGION_FOR_PTR(ptr));
- return r ? *r : r;
-}
-
-static void
-tiny_finalize_region(szone_t *szone, magazine_t *tiny_mag_ptr) {
- void *last_block, *previous_block;
- uint32_t *last_header;
- msize_t last_msize, previous_msize, last_index;
-
- // It is possible that the block prior to the last block in the region has
- // been free'd, but was not coalesced with the free bytes at the end of the
- // block, since we treat the bytes at the end of the region as "in use" in
- // the meta headers. Attempt to coalesce the last block with the previous
- // block, so we don't violate the "no consecutive free blocks" invariant.
- //
- // FIXME: Need to investigate how much work would be required to increase
- // 'mag_bytes_free_at_end' when freeing the preceding block, rather
- // than performing this workaround.
- //
-
- if (tiny_mag_ptr->mag_bytes_free_at_end) {
- last_block = (void *)
- ((uintptr_t)TINY_REGION_END(tiny_mag_ptr->mag_last_region) - tiny_mag_ptr->mag_bytes_free_at_end);
- last_msize = TINY_MSIZE_FOR_BYTES(tiny_mag_ptr->mag_bytes_free_at_end);
- last_header = TINY_BLOCK_HEADER_FOR_PTR(last_block);
- last_index = TINY_INDEX_FOR_PTR(last_block);
-
- // Before anything we transform any remaining mag_bytes_free_at_end into a
- // regular free block. We take special care here to update the bitfield
- // information, since we are bypassing the normal free codepath. If there
- // is more than one quanta worth of memory in mag_bytes_free_at_end, then
- // there will be two block headers:
- // 1) header for the free space at end, msize = 1
- // 2) header inserted by set_tiny_meta_header_in_use after block
- // We must clear the second one so that when the free block's size is
- // queried, we do not think the block is only 1 quantum in size because
- // of the second set header bit.
- if (last_index != (NUM_TINY_BLOCKS - 1))
- BITARRAY_CLR(last_header, (last_index + 1));
-
- previous_block = tiny_previous_preceding_free(last_block, &previous_msize);
- if (previous_block) {
- set_tiny_meta_header_middle(last_block);
- tiny_free_list_remove_ptr(szone, tiny_mag_ptr, previous_block, previous_msize);
- last_block = previous_block;
- last_msize += previous_msize;
- }
-
- // splice last_block into the free list
- tiny_free_list_add_ptr(szone, tiny_mag_ptr, last_block, last_msize);
- tiny_mag_ptr->mag_bytes_free_at_end = 0;
- }
-
-#if ASLR_INTERNAL
- // Coalesce the big free block at start with any following free blocks
- if (tiny_mag_ptr->mag_bytes_free_at_start) {
- last_block = TINY_REGION_ADDRESS(tiny_mag_ptr->mag_last_region);
- last_msize = TINY_MSIZE_FOR_BYTES(tiny_mag_ptr->mag_bytes_free_at_start);
-
- void *next_block = (void *) ((uintptr_t)last_block + tiny_mag_ptr->mag_bytes_free_at_start);
-
- // clear the in use bit we were using to mark the end of the big start block
- set_tiny_meta_header_middle((uintptr_t)next_block - TINY_QUANTUM);
-
- // Coalesce the big start block with any following free blocks
- if (tiny_meta_header_is_free(next_block)) {
- msize_t next_msize = get_tiny_free_size(next_block);
- set_tiny_meta_header_middle(next_block);
- tiny_free_list_remove_ptr(szone, tiny_mag_ptr, next_block, next_msize);
- last_msize += next_msize;
- }
-
- // splice last_block into the free list
- tiny_free_list_add_ptr(szone, tiny_mag_ptr, last_block, last_msize);
- tiny_mag_ptr->mag_bytes_free_at_start = 0;
- }
-#endif
-
- tiny_mag_ptr->mag_last_region = NULL;
-}
-
-static int
-tiny_free_detach_region(szone_t *szone, magazine_t *tiny_mag_ptr, region_t r) {
- uintptr_t start = (uintptr_t)TINY_REGION_ADDRESS(r);
- uintptr_t current = start;
- uintptr_t limit = (uintptr_t)TINY_REGION_END(r);
- boolean_t is_free;
- msize_t msize;
- int total_alloc = 0;
-
- while (current < limit) {
- msize = get_tiny_meta_header((void *)current, &is_free);
- if (is_free && !msize && (current == start)) {
- // first block is all free
- break;
- }
- if (!msize) {
-#if DEBUG_MALLOC
- malloc_printf("*** tiny_free_detach_region error with %p: msize=%d is_free =%d\n",
- (void *)current, msize, is_free);
-#endif
- break;
- }
- if (is_free) {
- tiny_free_list_remove_ptr(szone, tiny_mag_ptr, (void *)current, msize);
- } else {
- total_alloc++;
- }
- current += TINY_BYTES_FOR_MSIZE(msize);
- }
- return total_alloc;
-}
-
-static size_t
-tiny_free_reattach_region(szone_t *szone, magazine_t *tiny_mag_ptr, region_t r) {
- uintptr_t start = (uintptr_t)TINY_REGION_ADDRESS(r);
- uintptr_t current = start;
- uintptr_t limit = (uintptr_t)TINY_REGION_END(r);
- boolean_t is_free;
- msize_t msize;
- size_t total_alloc = 0;
-
- while (current < limit) {
- msize = get_tiny_meta_header((void *)current, &is_free);
- if (is_free && !msize && (current == start)) {
- // first block is all free
- break;
- }
- if (!msize) {
-#if DEBUG_MALLOC
- malloc_printf("*** tiny_free_reattach_region error with %p: msize=%d is_free =%d\n",
- (void *)current, msize, is_free);
-#endif
- break;
- }
- if (is_free) {
- tiny_free_list_add_ptr(szone, tiny_mag_ptr, (void *)current, msize);
- } else {
- total_alloc += TINY_BYTES_FOR_MSIZE(msize);
- }
- current += TINY_BYTES_FOR_MSIZE(msize);
- }
- return total_alloc;
-}
-
-typedef struct {
- uint8_t pnum, size;
-} tiny_pg_pair_t;
-
-static void NOINLINE /* want private stack frame for automatic array */
-tiny_free_scan_madvise_free(szone_t *szone, magazine_t *depot_ptr, region_t r) {
- uintptr_t start = (uintptr_t)TINY_REGION_ADDRESS(r);
- uintptr_t current = start;
- uintptr_t limit = (uintptr_t)TINY_REGION_END(r);
- boolean_t is_free;
- msize_t msize;
- tiny_pg_pair_t advisory[((TINY_REGION_PAYLOAD_BYTES + vm_page_size - 1) >> vm_page_shift) >> 1]; // 256bytes stack allocated
- int advisories = 0;
-
- // Scan the metadata identifying blocks which span one or more pages. Mark the pages MADV_FREE taking care to preserve free list
- // management data.
- while (current < limit) {
- msize = get_tiny_meta_header((void *)current, &is_free);
- if (is_free && !msize && (current == start)) {
- // first block is all free
-#if DEBUG_MALLOC
- malloc_printf("*** tiny_free_scan_madvise_free first block is all free! %p: msize=%d is_free =%d\n",
- (void *)current, msize, is_free);
-#endif
- uintptr_t pgLo = round_page(start + sizeof(free_list_t) + sizeof(msize_t));
- uintptr_t pgHi = trunc_page(start + TINY_REGION_SIZE - sizeof(msize_t));
-
- if (pgLo < pgHi) {
- advisory[advisories].pnum = (pgLo - start) >> vm_page_shift;
- advisory[advisories].size = (pgHi - pgLo) >> vm_page_shift;
- advisories++;
- }
- break;
- }
- if (!msize) {
-#if DEBUG_MALLOC
- malloc_printf("*** tiny_free_scan_madvise_free error with %p: msize=%d is_free =%d\n",
- (void *)current, msize, is_free);
-#endif
- break;
- }
- if (is_free) {
- uintptr_t pgLo = round_page(current + sizeof(free_list_t) + sizeof(msize_t));
- uintptr_t pgHi = trunc_page(current + TINY_BYTES_FOR_MSIZE(msize) - sizeof(msize_t));
-
- if (pgLo < pgHi) {
- advisory[advisories].pnum = (pgLo - start) >> vm_page_shift;
- advisory[advisories].size = (pgHi - pgLo) >> vm_page_shift;
- advisories++;
- }
- }
- current += TINY_BYTES_FOR_MSIZE(msize);
- }
-
- if (advisories > 0) {
- int i;
-
- // So long as the following hold for this region:
- // (1) No malloc()'s are ever performed from the depot (hence free pages remain free,)
- // (2) The region is not handed over to a per-CPU magazine (where malloc()'s could be performed),
- // (3) The entire region is not mumap()'d (so the madvise's are applied to the intended addresses),
- // then the madvise opportunities collected just above can be applied outside all locks.
- // (1) is ensured by design, (2) and (3) are ensured by bumping the globally visible counter node->pinned_to_depot.
-
- OSAtomicIncrement32Barrier(&(REGION_TRAILER_FOR_TINY_REGION(r)->pinned_to_depot));
- SZONE_MAGAZINE_PTR_UNLOCK(szone, depot_ptr);
- for (i = 0; i < advisories; ++i) {
- uintptr_t addr = (advisory[i].pnum << vm_page_shift) + start;
- size_t size = advisory[i].size << vm_page_shift;
-
-#if TARGET_OS_EMBEDDED
- madvise_free_range(szone, r, addr, addr + size, NULL);
-#else
- madvise_free_range(szone, r, addr, addr + size);
-#endif
- }
- SZONE_MAGAZINE_PTR_LOCK(szone, depot_ptr);
- OSAtomicDecrement32Barrier(&(REGION_TRAILER_FOR_TINY_REGION(r)->pinned_to_depot));
- }
-}
-
-static region_t
-tiny_free_try_depot_unmap_no_lock(szone_t *szone, magazine_t *depot_ptr, region_trailer_t *node)
-{
- if (0 < node->bytes_used ||
- 0 < node->pinned_to_depot ||
- depot_ptr->recirculation_entries < (szone->num_tiny_magazines * 2)) {
- return NULL;
- }
-
- // disconnect node from Depot
- recirc_list_extract(szone, depot_ptr, node);
-
- // Iterate the region pulling its free entries off the (locked) Depot's free list
- region_t sparse_region = TINY_REGION_FOR_PTR(node);
- int objects_in_use = tiny_free_detach_region(szone, depot_ptr, sparse_region);
-
- if (0 == objects_in_use) {
- // Invalidate the hash table entry for this region with HASHRING_REGION_DEALLOCATED.
- // Using HASHRING_REGION_DEALLOCATED preserves the collision chain, using HASHRING_OPEN_ENTRY (0) would not.
- rgnhdl_t pSlot = hash_lookup_region_no_lock(szone->tiny_region_generation->hashed_regions,
- szone->tiny_region_generation->num_regions_allocated,
- szone->tiny_region_generation->num_regions_allocated_shift, sparse_region);
- if (NULL == pSlot) {
- szone_error(szone, 1, "tiny_free_try_depot_unmap_no_lock hash lookup failed:", NULL, "%p\n", sparse_region);
- return NULL;
- }
- *pSlot = HASHRING_REGION_DEALLOCATED;
- depot_ptr->num_bytes_in_magazine -= TINY_REGION_PAYLOAD_BYTES;
- __sync_fetch_and_add( &(szone->num_tiny_regions_dealloc), 1); // Atomically increment num_tiny_regions_dealloc
-
- // Caller will transfer ownership of the region back to the OS with no locks held
- MAGMALLOC_DEALLOCREGION((void *)szone, (void *)sparse_region, TINY_REGION_SIZE); // DTrace USDT Probe
- return sparse_region;
- } else {
- szone_error(szone, 1, "tiny_free_try_depot_unmap_no_lock objects_in_use not zero:", NULL, "%d\n", objects_in_use);
- return NULL;
- }
-}
-
-static boolean_t
-tiny_free_do_recirc_to_depot(szone_t *szone, magazine_t *tiny_mag_ptr, mag_index_t mag_index)
-{
- // The entire magazine crossed the "emptiness threshold". Transfer a region
- // from this magazine to the Depot. Choose a region that itself has crossed the emptiness threshold (i.e
- // is at least fraction "f" empty.) Such a region will be marked "suitable" on the recirculation list.
- region_trailer_t *node = tiny_mag_ptr->firstNode;
-
- while (node && !node->recirc_suitable) {
- node = node->next;
- }
-
- if (NULL == node) {
-#if DEBUG_MALLOC
- malloc_printf("*** tiny_free_do_recirc_to_depot end of list\n");
-#endif
- return TRUE; // Caller must SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr);
- }
-
- region_t sparse_region = TINY_REGION_FOR_PTR(node);
-
- // Deal with unclaimed memory -- mag_bytes_free_at_end or mag_bytes_free_at_start
- if (sparse_region == tiny_mag_ptr->mag_last_region && (tiny_mag_ptr->mag_bytes_free_at_end || tiny_mag_ptr->mag_bytes_free_at_start)) {
- tiny_finalize_region(szone, tiny_mag_ptr);
- }
-
- // disconnect "suitable" node from magazine
- recirc_list_extract(szone, tiny_mag_ptr, node);
-
- // Iterate the region pulling its free entries off its (locked) magazine's free list
- int objects_in_use = tiny_free_detach_region(szone, tiny_mag_ptr, sparse_region);
- magazine_t *depot_ptr = &(szone->tiny_magazines[DEPOT_MAGAZINE_INDEX]);
-
- // hand over the region to the (locked) Depot
- SZONE_MAGAZINE_PTR_LOCK(szone,depot_ptr);
- // this will cause tiny_free_list_add_ptr called by tiny_free_reattach_region to use
- // the depot as its target magazine, rather than magazine formerly associated with sparse_region
- MAGAZINE_INDEX_FOR_TINY_REGION(sparse_region) = DEPOT_MAGAZINE_INDEX;
- node->pinned_to_depot = 0;
-
- // Iterate the region putting its free entries on Depot's free list
- size_t bytes_inplay = tiny_free_reattach_region(szone, depot_ptr, sparse_region);
-
- tiny_mag_ptr->mag_num_bytes_in_objects -= bytes_inplay;
- tiny_mag_ptr->num_bytes_in_magazine -= TINY_REGION_PAYLOAD_BYTES;
- tiny_mag_ptr->mag_num_objects -= objects_in_use;
-
- SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr); // Unlock the originating magazine
-
- depot_ptr->mag_num_bytes_in_objects += bytes_inplay;
- depot_ptr->num_bytes_in_magazine += TINY_REGION_PAYLOAD_BYTES;
- depot_ptr->mag_num_objects += objects_in_use;
-
- // connect to Depot as last node
- recirc_list_splice_last(szone, depot_ptr, node);
-
- MAGMALLOC_RECIRCREGION((void *)szone, (int)mag_index, (void *)sparse_region, TINY_REGION_SIZE,
- (int)BYTES_USED_FOR_TINY_REGION(sparse_region)); // DTrace USDT Probe
-
- // Mark free'd dirty pages with MADV_FREE to reduce memory pressure
- tiny_free_scan_madvise_free(szone, depot_ptr, sparse_region);
-
- // If the region is entirely empty vm_deallocate() it outside the depot lock
- region_t r_dealloc = tiny_free_try_depot_unmap_no_lock(szone, depot_ptr, node);
- SZONE_MAGAZINE_PTR_UNLOCK(szone,depot_ptr);
- if (r_dealloc)
- deallocate_pages(szone, r_dealloc, TINY_REGION_SIZE, 0);
- return FALSE; // Caller need not unlock the originating magazine
-}
-
-static region_t
-tiny_find_msize_region(szone_t *szone, magazine_t *tiny_mag_ptr, mag_index_t mag_index, msize_t msize)
-{
- free_list_t *ptr;
- grain_t slot = msize - 1;
- free_list_t **free_list = tiny_mag_ptr->mag_free_list;
- free_list_t **the_slot = free_list + slot;
- free_list_t **limit;
-#if defined(__LP64__)
- uint64_t bitmap;
-#else
- uint32_t bitmap;
-#endif
- // Assumes we've locked the magazine
- CHECK_MAGAZINE_PTR_LOCKED(szone, tiny_mag_ptr, __PRETTY_FUNCTION__);
-
- // Look for an exact match by checking the freelist for this msize.
- ptr = *the_slot;
- if (ptr)
- return TINY_REGION_FOR_PTR(ptr);
-
- // Mask off the bits representing slots holding free blocks smaller than the
- // size we need. If there are no larger free blocks, try allocating from
- // the free space at the end of the tiny region.
-#if defined(__LP64__)
- bitmap = ((uint64_t *)(tiny_mag_ptr->mag_bitmap))[0] & ~ ((1ULL << slot) - 1);
-#else
- bitmap = tiny_mag_ptr->mag_bitmap[0] & ~ ((1 << slot) - 1);
-#endif
- if (!bitmap)
- return NULL;
-
- slot = BITMAPV_CTZ(bitmap);
- limit = free_list + NUM_TINY_SLOTS - 1;
- free_list += slot;
-
- if (free_list < limit) {
- ptr = *free_list;
- if (ptr)
- return TINY_REGION_FOR_PTR(ptr);
- else {
- /* Shouldn't happen. Fall through to look at last slot. */
-#if DEBUG_MALLOC
- malloc_printf("in tiny_find_msize_region(), mag_bitmap out of sync, slot=%d\n",slot);
-#endif
- }
- }
-
- // We are now looking at the last slot, which contains blocks equal to, or
- // due to coalescing of free blocks, larger than (NUM_TINY_SLOTS - 1) * tiny quantum size.
- ptr = *limit;
- if (ptr)
- return TINY_REGION_FOR_PTR(ptr);
-
- return NULL;
-}
-
-static boolean_t
-tiny_get_region_from_depot(szone_t *szone, magazine_t *tiny_mag_ptr, mag_index_t mag_index, msize_t msize)
-{
- magazine_t *depot_ptr = &(szone->tiny_magazines[DEPOT_MAGAZINE_INDEX]);
-
- /* FIXME: Would Uniprocessor benefit from recirc and MADV_FREE? */
- if (szone->num_tiny_magazines == 1) // Uniprocessor, single magazine, so no recirculation necessary
- return 0;
-
-#if DEBUG_MALLOC
- if (DEPOT_MAGAZINE_INDEX == mag_index) {
- szone_error(szone, 1, "tiny_get_region_from_depot called for magazine index -1", NULL, NULL);
- return 0;
- }
-#endif
-
- SZONE_MAGAZINE_PTR_LOCK(szone,depot_ptr);
-
- // Appropriate a Depot'd region that can satisfy requested msize.
- region_trailer_t *node;
- region_t sparse_region;
-
- while (1) {
- sparse_region = tiny_find_msize_region(szone, depot_ptr, DEPOT_MAGAZINE_INDEX, msize);
- if (NULL == sparse_region) { // Depot empty?
- SZONE_MAGAZINE_PTR_UNLOCK(szone,depot_ptr);
- return 0;
- }
-
- node = REGION_TRAILER_FOR_TINY_REGION(sparse_region);
- if (0 >= node->pinned_to_depot)
- break;
-
- SZONE_MAGAZINE_PTR_UNLOCK(szone,depot_ptr);
- pthread_yield_np();
- SZONE_MAGAZINE_PTR_LOCK(szone,depot_ptr);
- }
-
- // disconnect node from Depot
- recirc_list_extract(szone, depot_ptr, node);
-
- // Iterate the region pulling its free entries off the (locked) Depot's free list
- int objects_in_use = tiny_free_detach_region(szone, depot_ptr, sparse_region);
-
- // Transfer ownership of the region
- MAGAZINE_INDEX_FOR_TINY_REGION(sparse_region) = mag_index;
- node->pinned_to_depot = 0;
-
- // Iterate the region putting its free entries on its new (locked) magazine's free list
- size_t bytes_inplay = tiny_free_reattach_region(szone, tiny_mag_ptr, sparse_region);
-
- depot_ptr->mag_num_bytes_in_objects -= bytes_inplay;
- depot_ptr->num_bytes_in_magazine -= TINY_REGION_PAYLOAD_BYTES;
- depot_ptr->mag_num_objects -= objects_in_use;
-
- tiny_mag_ptr->mag_num_bytes_in_objects += bytes_inplay;
- tiny_mag_ptr->num_bytes_in_magazine += TINY_REGION_PAYLOAD_BYTES;
- tiny_mag_ptr->mag_num_objects += objects_in_use;
-
- // connect to magazine as first node
- recirc_list_splice_first(szone, tiny_mag_ptr, node);
-
- SZONE_MAGAZINE_PTR_UNLOCK(szone,depot_ptr);
-
- // madvise() outside the Depot lock
-#if TARGET_OS_EMBEDDED
- if (node->failedREUSE) {
-#else
- if (node->failedREUSE ||
- -1 == madvise((void *)sparse_region, TINY_REGION_PAYLOAD_BYTES, MADV_FREE_REUSE)) {
-#endif
- /* -1 return: VM map entry change makes this unfit for reuse. Something evil lurks. */
-#if DEBUG_MADVISE
- szone_error(szone, 0, "tiny_get_region_from_depot madvise(..., MADV_FREE_REUSE) failed",
- sparse_region, "length=%d\n", TINY_REGION_PAYLOAD_BYTES);
-#endif
- node->failedREUSE = TRUE;
- }
-
- MAGMALLOC_DEPOTREGION((void *)szone, (int)mag_index, (void *)sparse_region, TINY_REGION_SIZE,
- (int)BYTES_USED_FOR_TINY_REGION(sparse_region)); // DTrace USDT Probe
-
- return 1;
-}
-
-#define K 1.5 // headroom measured in number of 1Mb regions
-#define DENSITY_THRESHOLD(a) \
- ((a) - ((a) >> 2)) // "Emptiness" f = 0.25, so "Density" is (1 - f)*a. Generally: ((a) - ((a) >> -log2(f)))
-
-static INLINE boolean_t
-tiny_free_no_lock(szone_t *szone, magazine_t *tiny_mag_ptr, mag_index_t mag_index, region_t region, void *ptr,
- msize_t msize)
-{
- void *original_ptr = ptr;
- size_t original_size = TINY_BYTES_FOR_MSIZE(msize);
- void *next_block = ((unsigned char *)ptr + original_size);
- msize_t previous_msize, next_msize;
- void *previous;
- free_list_t *big_free_block;
- free_list_t *after_next_block;
- free_list_t *before_next_block;
-
-#if DEBUG_MALLOC
- if (LOG(szone,ptr)) {
- malloc_printf("in tiny_free_no_lock(), ptr=%p, msize=%d\n", ptr, msize);
- }
- if (!msize) {
- szone_error(szone, 1, "trying to free tiny block that is too small", ptr,
- "in tiny_free_no_lock(), ptr=%p, msize=%d\n", ptr, msize);
- }
-#endif
-
- // We try to coalesce this block with the preceeding one
- previous = tiny_previous_preceding_free(ptr, &previous_msize);
- if (previous) {
-#if DEBUG_MALLOC
- if (LOG(szone, ptr) || LOG(szone,previous)) {
- malloc_printf("in tiny_free_no_lock(), coalesced backwards for %p previous=%p\n", ptr, previous);
- }
-#endif
-
- // clear the meta_header since this is no longer the start of a block
- set_tiny_meta_header_middle(ptr);
- tiny_free_list_remove_ptr(szone, tiny_mag_ptr, previous, previous_msize);
- ptr = previous;
- msize += previous_msize;
- }
- // We try to coalesce with the next block
- if ((next_block < TINY_REGION_END(region)) && tiny_meta_header_is_free(next_block)) {
- next_msize = get_tiny_free_size(next_block);
-#if DEBUG_MALLOC
- if (LOG(szone, ptr) || LOG(szone, next_block)) {
- malloc_printf("in tiny_free_no_lock(), for ptr=%p, msize=%d coalesced forward=%p next_msize=%d\n",
- ptr, msize, next_block, next_msize);
- }
-#endif
- // If we are coalescing with the next block, and the next block is in
- // the last slot of the free list, then we optimize this case here to
- // avoid removing next_block from the slot (NUM_TINY_SLOTS - 1) and then adding ptr back
- // to slot (NUM_TINY_SLOTS - 1).
- if (next_msize >= NUM_TINY_SLOTS) {
- msize += next_msize;
-
- big_free_block = (free_list_t *)next_block;
- after_next_block = free_list_unchecksum_ptr(szone, &big_free_block->next);
- before_next_block = free_list_unchecksum_ptr(szone, &big_free_block->previous);
-
- if (!before_next_block) {
- tiny_mag_ptr->mag_free_list[NUM_TINY_SLOTS-1] = ptr;
- } else {
- before_next_block->next.u = free_list_checksum_ptr(szone, ptr);
- }
-
- if (after_next_block) {
- after_next_block->previous.u = free_list_checksum_ptr(szone, ptr);
- }
-
- // we don't need to checksum these since they are already checksummed
- ((free_list_t *)ptr)->previous = big_free_block->previous;
- ((free_list_t *)ptr)->next = big_free_block->next;
-
- // clear the meta_header to enable coalescing backwards
- set_tiny_meta_header_middle(big_free_block);
- set_tiny_meta_header_free(ptr, msize);
-
- goto tiny_free_ending;
- }
- tiny_free_list_remove_ptr(szone, tiny_mag_ptr, next_block, next_msize);
- set_tiny_meta_header_middle(next_block); // clear the meta_header to enable coalescing backwards
- msize += next_msize;
- }
-
- // The tiny cache already scribbles free blocks as they go through the
- // cache whenever msize < TINY_QUANTUM , so we do not need to do it here.
- if ((szone->debug_flags & SCALABLE_MALLOC_DO_SCRIBBLE) && msize && (msize >= TINY_QUANTUM))
- memset(ptr, 0x55, TINY_BYTES_FOR_MSIZE(msize));
-
- tiny_free_list_add_ptr(szone, tiny_mag_ptr, ptr, msize);
-
- tiny_free_ending:
-
- tiny_mag_ptr->mag_num_objects--;
- // we use original_size and not msize to avoid double counting the coalesced blocks
- tiny_mag_ptr->mag_num_bytes_in_objects -= original_size;
-
- // Update this region's bytes in use count
- region_trailer_t *node = REGION_TRAILER_FOR_TINY_REGION(region);
- size_t bytes_used = node->bytes_used - original_size;
- node->bytes_used = bytes_used;
-
-#if !TARGET_OS_EMBEDDED // Always madvise for embedded platforms
- /* FIXME: Would Uniprocessor benefit from recirc and MADV_FREE? */
- if (szone->num_tiny_magazines == 1) { // Uniprocessor, single magazine, so no recirculation necessary
- /* NOTHING */
- } else if (DEPOT_MAGAZINE_INDEX != mag_index) {
- // Emptiness discriminant
- if (bytes_used < DENSITY_THRESHOLD(TINY_REGION_PAYLOAD_BYTES)) {
- /* Region has crossed threshold from density to sparsity. Mark it "suitable" on the
- recirculation candidates list. */
- node->recirc_suitable = TRUE;
- } else {
- /* After this free, we've found the region is still dense, so it must have been even more so before
- the free. That implies the region is already correctly marked. Do nothing. */
- }
-
- // Has the entire magazine crossed the "emptiness threshold"? If so, transfer a region
- // from this magazine to the Depot. Choose a region that itself has crossed the emptiness threshold (i.e
- // is at least fraction "f" empty.) Such a region will be marked "suitable" on the recirculation list.
- size_t a = tiny_mag_ptr->num_bytes_in_magazine; // Total bytes allocated to this magazine
- size_t u = tiny_mag_ptr->mag_num_bytes_in_objects; // In use (malloc'd) from this magaqzine
-
- if (a - u > ((3 * TINY_REGION_PAYLOAD_BYTES) / 2) && u < DENSITY_THRESHOLD(a)) {
- return tiny_free_do_recirc_to_depot(szone, tiny_mag_ptr, mag_index);
- }
-
- } else {
-#endif
- // Freed to Depot. N.B. Lock on tiny_magazines[DEPOT_MAGAZINE_INDEX] is already held
- // Calcuate the first page in the coalesced block that would be safe to mark MADV_FREE
- uintptr_t safe_ptr = (uintptr_t)ptr + sizeof(free_list_t) + sizeof(msize_t);
- uintptr_t round_safe = round_page(safe_ptr);
-
- // Calcuate the last page in the coalesced block that would be safe to mark MADV_FREE
- uintptr_t safe_extent = (uintptr_t)ptr + TINY_BYTES_FOR_MSIZE(msize) - sizeof(msize_t);
- uintptr_t trunc_extent = trunc_page(safe_extent);
-
- // The newly freed block may complete a span of bytes that cover a page. Mark it with MADV_FREE.
- if (round_safe < trunc_extent) { // Safe area covers a page (perhaps many)
- uintptr_t lo = trunc_page((uintptr_t)original_ptr);
- uintptr_t hi = round_page((uintptr_t)original_ptr + original_size);
-
- tiny_free_list_remove_ptr(szone, tiny_mag_ptr, ptr, msize);
- set_tiny_meta_header_in_use(ptr, msize);
-
- OSAtomicIncrement32Barrier(&(node->pinned_to_depot));
- SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr);
-#if TARGET_OS_EMBEDDED
- madvise_free_range(szone, region, MAX(round_safe, lo), MIN(trunc_extent, hi), &szone->last_tiny_advise);
-#else
- madvise_free_range(szone, region, MAX(round_safe, lo), MIN(trunc_extent, hi));
-#endif
- SZONE_MAGAZINE_PTR_LOCK(szone, tiny_mag_ptr);
- OSAtomicDecrement32Barrier(&(node->pinned_to_depot));
-
- set_tiny_meta_header_free(ptr, msize);
- tiny_free_list_add_ptr(szone, tiny_mag_ptr, ptr, msize);
- }
-
-#if !TARGET_OS_EMBEDDED
- if (0 < bytes_used || 0 < node->pinned_to_depot) {
- /* Depot'd region is still live. Leave it in place on the Depot's recirculation list
- so as to avoid thrashing between the Depot's free list and a magazines's free list
- with detach_region/reattach_region */
- } else {
- /* Depot'd region is just now empty. Consider return to OS. */
- region_t r_dealloc = tiny_free_try_depot_unmap_no_lock(szone, tiny_mag_ptr, node);
- SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr);
- if (r_dealloc)
- deallocate_pages(szone, r_dealloc, TINY_REGION_SIZE, 0);
- return FALSE; // Caller need not unlock
- }
- }
-#endif
-
- return TRUE; // Caller must do SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr)
-}
-
-// Allocates from the last region or a freshly allocated region
-static void *
-tiny_malloc_from_region_no_lock(szone_t *szone, magazine_t *tiny_mag_ptr, mag_index_t mag_index,
- msize_t msize, void * aligned_address)
-{
- void *ptr;
-
- // Deal with unclaimed memory -- mag_bytes_free_at_end or mag_bytes_free_at_start
- if (tiny_mag_ptr->mag_bytes_free_at_end || tiny_mag_ptr->mag_bytes_free_at_start)
- tiny_finalize_region(szone, tiny_mag_ptr);
-
- // We set the unused bits of the header in the last pair to be all ones, and those of the inuse to zeroes.
- ((tiny_region_t)aligned_address)->pairs[CEIL_NUM_TINY_BLOCKS_WORDS-1].header =
- (NUM_TINY_BLOCKS & 31) ? (0xFFFFFFFFU << (NUM_TINY_BLOCKS & 31)) : 0;
- ((tiny_region_t)aligned_address)->pairs[CEIL_NUM_TINY_BLOCKS_WORDS-1].inuse = 0;
-
- // Here find the only place in tinyland that (infrequently) takes the tiny_regions_lock.
- // Only one thread at a time should be permitted to assess the density of the hash
- // ring and adjust if needed.
- // Only one thread at a time should be permitted to insert its new region on
- // the hash ring.
- // It is safe for all other threads to read the hash ring (hashed_regions) and
- // the associated sizes (num_regions_allocated and num_tiny_regions).
-
- LOCK(szone->tiny_regions_lock);
-
- // Check to see if the hash ring of tiny regions needs to grow. Try to
- // avoid the hash ring becoming too dense.
- if (szone->tiny_region_generation->num_regions_allocated < (2 * szone->num_tiny_regions)) {
- region_t *new_regions;
- size_t new_size;
- size_t new_shift = szone->tiny_region_generation->num_regions_allocated_shift; // In/Out parameter
- new_regions = hash_regions_grow_no_lock(szone, szone->tiny_region_generation->hashed_regions,
- szone->tiny_region_generation->num_regions_allocated,
- &new_shift,
- &new_size);
- // Do not deallocate the current hashed_regions allocation since someone may
- // be iterating it. Instead, just leak it.
-
- // Prepare to advance to the "next generation" of the hash ring.
- szone->tiny_region_generation->nextgen->hashed_regions = new_regions;
- szone->tiny_region_generation->nextgen->num_regions_allocated = new_size;
- szone->tiny_region_generation->nextgen->num_regions_allocated_shift = new_shift;
-
- // Throw the switch to atomically advance to the next generation.
- szone->tiny_region_generation = szone->tiny_region_generation->nextgen;
- // Ensure everyone sees the advance.
- OSMemoryBarrier();
- }
- // Tag the region at "aligned_address" as belonging to us,
- // and so put it under the protection of the magazine lock we are holding.
- // Do this before advertising "aligned_address" on the hash ring(!)
- MAGAZINE_INDEX_FOR_TINY_REGION(aligned_address) = mag_index;
-
- // Insert the new region into the hash ring, and update malloc statistics
- hash_region_insert_no_lock(szone->tiny_region_generation->hashed_regions,
- szone->tiny_region_generation->num_regions_allocated,
- szone->tiny_region_generation->num_regions_allocated_shift,
- aligned_address);
-
- szone->num_tiny_regions++;
- UNLOCK(szone->tiny_regions_lock);
-
- tiny_mag_ptr->mag_last_region = aligned_address;
- BYTES_USED_FOR_TINY_REGION(aligned_address) = TINY_BYTES_FOR_MSIZE(msize);
-#if ASLR_INTERNAL
- int offset_msize = malloc_entropy[0] & TINY_ENTROPY_MASK;
-#if DEBUG_MALLOC
- if (getenv("MallocASLRForce")) offset_msize = strtol(getenv("MallocASLRForce"), NULL, 0) & TINY_ENTROPY_MASK;
- if (getenv("MallocASLRPrint")) malloc_printf("Region: %p offset: %d\n", aligned_address, offset_msize);
-#endif
-#else
- int offset_msize = 0;
-#endif
- ptr = (void *)((uintptr_t) aligned_address + TINY_BYTES_FOR_MSIZE(offset_msize));
- set_tiny_meta_header_in_use(ptr, msize);
- tiny_mag_ptr->mag_num_objects++;
- tiny_mag_ptr->mag_num_bytes_in_objects += TINY_BYTES_FOR_MSIZE(msize);
- tiny_mag_ptr->num_bytes_in_magazine += TINY_REGION_PAYLOAD_BYTES;
-
- // We put a header on the last block so that it appears in use (for coalescing, etc...)
- set_tiny_meta_header_in_use_1((void *)((uintptr_t)ptr + TINY_BYTES_FOR_MSIZE(msize)));
- tiny_mag_ptr->mag_bytes_free_at_end = TINY_BYTES_FOR_MSIZE(NUM_TINY_BLOCKS - msize - offset_msize);
-
-#if ASLR_INTERNAL
- // Put a header on the previous block for same reason
- tiny_mag_ptr->mag_bytes_free_at_start = TINY_BYTES_FOR_MSIZE(offset_msize);
- if (offset_msize) {
- set_tiny_meta_header_in_use_1((void *)((uintptr_t)ptr - TINY_QUANTUM));
- }
-#else
- tiny_mag_ptr->mag_bytes_free_at_start = 0;
-#endif
-
- // connect to magazine as last node
- recirc_list_splice_last(szone, tiny_mag_ptr, REGION_TRAILER_FOR_TINY_REGION(aligned_address));
-
-#if DEBUG_MALLOC
- if (LOG(szone,ptr)) {
- malloc_printf("in tiny_malloc_from_region_no_lock(), ptr=%p, msize=%d\n", ptr, msize);
- }
-#endif
- return ptr;
-}
-
-static INLINE void *
-tiny_try_shrink_in_place(szone_t *szone, void *ptr, size_t old_size, size_t new_good_size)
-{
- msize_t new_msize = TINY_MSIZE_FOR_BYTES(new_good_size);
- msize_t mshrinkage = TINY_MSIZE_FOR_BYTES(old_size) - new_msize;
-
- if (mshrinkage) {
- void *q = (void *)((uintptr_t)ptr + TINY_BYTES_FOR_MSIZE(new_msize));
- magazine_t *tiny_mag_ptr = mag_lock_zine_for_region_trailer(szone, szone->tiny_magazines,
- REGION_TRAILER_FOR_TINY_REGION(TINY_REGION_FOR_PTR(ptr)),
- MAGAZINE_INDEX_FOR_TINY_REGION(TINY_REGION_FOR_PTR(ptr)));
-
- // Mark q as block header and in-use, thus creating two blocks.
- set_tiny_meta_header_in_use(q, mshrinkage);
- tiny_mag_ptr->mag_num_objects++;
-
- SZONE_MAGAZINE_PTR_UNLOCK(szone,tiny_mag_ptr);
- szone_free(szone, q); // avoid inlining free_tiny(szone, q, ...);
- }
- return ptr;
-}
-
-static INLINE boolean_t
-tiny_try_realloc_in_place(szone_t *szone, void *ptr, size_t old_size, size_t new_size)
-{
- // returns 1 on success
- msize_t index;
- msize_t old_msize;
- unsigned next_index;
- void *next_block;
- boolean_t is_free;
- msize_t next_msize, coalesced_msize, leftover_msize;
- void *leftover;
-
- index = TINY_INDEX_FOR_PTR(ptr);
- old_msize = TINY_MSIZE_FOR_BYTES(old_size);
- next_index = index + old_msize;
-
- if (next_index >= NUM_TINY_BLOCKS) {
- return 0;
- }
- next_block = (char *)ptr + old_size;
-
- magazine_t *tiny_mag_ptr = mag_lock_zine_for_region_trailer(szone, szone->tiny_magazines,
- REGION_TRAILER_FOR_TINY_REGION(TINY_REGION_FOR_PTR(ptr)),
- MAGAZINE_INDEX_FOR_TINY_REGION(TINY_REGION_FOR_PTR(ptr)));
-
- /*
- * Look for a free block immediately afterwards. If it's large enough, we can consume (part of)
- * it.
- */
- is_free = tiny_meta_header_is_free(next_block);
- if (!is_free) {
- SZONE_MAGAZINE_PTR_UNLOCK(szone,tiny_mag_ptr);
- return 0; // next_block is in use;
- }
- next_msize = get_tiny_free_size(next_block);
- if (old_size + TINY_BYTES_FOR_MSIZE(next_msize) < new_size) {
- SZONE_MAGAZINE_PTR_UNLOCK(szone,tiny_mag_ptr);
- return 0; // even with next block, not enough
- }
- /*
- * The following block is big enough; pull it from its freelist and chop off enough to satisfy
- * our needs.
- */
- tiny_free_list_remove_ptr(szone, tiny_mag_ptr, next_block, next_msize);
- set_tiny_meta_header_middle(next_block); // clear the meta_header to enable coalescing backwards
- coalesced_msize = TINY_MSIZE_FOR_BYTES(new_size - old_size + TINY_QUANTUM - 1);
- leftover_msize = next_msize - coalesced_msize;
- if (leftover_msize) {
- /* there's some left, so put the remainder back */
- leftover = (void *)((uintptr_t)next_block + TINY_BYTES_FOR_MSIZE(coalesced_msize));
-
- tiny_free_list_add_ptr(szone, tiny_mag_ptr, leftover, leftover_msize);
- }
- set_tiny_meta_header_in_use(ptr, old_msize + coalesced_msize);
-#if DEBUG_MALLOC
- if (LOG(szone,ptr)) {
- malloc_printf("in tiny_try_realloc_in_place(), ptr=%p, msize=%d\n", ptr, old_msize + coalesced_msize);
- }
-#endif
- tiny_mag_ptr->mag_num_bytes_in_objects += TINY_BYTES_FOR_MSIZE(coalesced_msize);
-
- // Update this region's bytes in use count
- region_trailer_t *node = REGION_TRAILER_FOR_TINY_REGION(TINY_REGION_FOR_PTR(ptr));
- size_t bytes_used = node->bytes_used + TINY_BYTES_FOR_MSIZE(coalesced_msize);
- node->bytes_used = bytes_used;
-
- // Emptiness discriminant
- if (bytes_used < DENSITY_THRESHOLD(TINY_REGION_PAYLOAD_BYTES)) {
- /* After this reallocation the region is still sparse, so it must have been even more so before
- the reallocation. That implies the region is already correctly marked. Do nothing. */
- } else {
- /* Region has crossed threshold from sparsity to density. Mark it not "suitable" on the
- recirculation candidates list. */
- node->recirc_suitable = FALSE;
- }
-
- SZONE_MAGAZINE_PTR_UNLOCK(szone,tiny_mag_ptr);
- CHECK(szone, __PRETTY_FUNCTION__);
- return 1;
-}
-
-static boolean_t
-tiny_check_region(szone_t *szone, region_t region)
-{
- uintptr_t start, ptr, region_end;
- boolean_t prev_free = 0;
- boolean_t is_free;
- msize_t msize;
- free_list_t *free_head;
- void *follower, *previous, *next;
- mag_index_t mag_index = MAGAZINE_INDEX_FOR_TINY_REGION(region);
- magazine_t *tiny_mag_ptr = &(szone->tiny_magazines[mag_index]);
-
- // Assumes locked
- CHECK_MAGAZINE_PTR_LOCKED(szone, tiny_mag_ptr, __PRETTY_FUNCTION__);
-
- /* establish region limits */
- start = (uintptr_t)TINY_REGION_ADDRESS(region);
- ptr = start;
- if (region == tiny_mag_ptr->mag_last_region) {
- ptr += tiny_mag_ptr->mag_bytes_free_at_start;
-
- /*
- * Check the leading block's integrity here also.
- */
- if (tiny_mag_ptr->mag_bytes_free_at_start) {
- msize = get_tiny_meta_header((void *)(ptr - TINY_QUANTUM), &is_free);
- if (is_free || (msize != 1)) {
- malloc_printf("*** invariant broken for leader block %p - %d %d\n", ptr - TINY_QUANTUM, msize, is_free);
- }
- }
- }
- region_end = (uintptr_t)TINY_REGION_END(region);
-
- /*
- * The last region may have a trailing chunk which has not been converted into inuse/freelist
- * blocks yet.
- */
- if (region == tiny_mag_ptr->mag_last_region)
- region_end -= tiny_mag_ptr->mag_bytes_free_at_end;
-
- /*
- * Scan blocks within the region.
- */
- while (ptr < region_end) {
- /*
- * If the first block is free, and its size is 65536 (msize = 0) then the entire region is
- * free.
- */
- msize = get_tiny_meta_header((void *)ptr, &is_free);
- if (is_free && !msize && (ptr == start)) {
- return 1;
- }
-
- /*
- * If the block's size is 65536 (msize = 0) then since we're not the first entry the size is
- * corrupt.
- */
- if (!msize) {
- malloc_printf("*** invariant broken for tiny block %p this msize=%d - size is too small\n",
- ptr, msize);
- return 0;
- }
-
- if (!is_free) {
- /*
- * In use blocks cannot be more than (NUM_TINY_SLOTS - 1) quanta large.
- */
- prev_free = 0;
- if (msize > (NUM_TINY_SLOTS - 1)) {
- malloc_printf("*** invariant broken for %p this tiny msize=%d - size is too large\n",
- ptr, msize);
- return 0;
- }
- /* move to next block */
- ptr += TINY_BYTES_FOR_MSIZE(msize);
- } else {
-#if !RELAXED_INVARIANT_CHECKS
- /*
- * Free blocks must have been coalesced, we cannot have a free block following another
- * free block.
- */
- if (prev_free) {
- malloc_printf("*** invariant broken for free block %p this tiny msize=%d: two free blocks in a row\n",
- ptr, msize);
- return 0;
- }
-#endif // RELAXED_INVARIANT_CHECKS
- prev_free = 1;
- /*
- * Check the integrity of this block's entry in its freelist.
- */
- free_head = (free_list_t *)ptr;
- previous = free_list_unchecksum_ptr(szone, &free_head->previous);
- next = free_list_unchecksum_ptr(szone, &free_head->next);
- if (previous && !tiny_meta_header_is_free(previous)) {
- malloc_printf("*** invariant broken for %p (previous %p is not a free pointer)\n",
- ptr, previous);
- return 0;
- }
- if (next && !tiny_meta_header_is_free(next)) {
- malloc_printf("*** invariant broken for %p (next in free list %p is not a free pointer)\n",
- ptr, next);
- return 0;
- }
- /*
- * Check the free block's trailing size value.
- */
- follower = FOLLOWING_TINY_PTR(ptr, msize);
- if (((uintptr_t)follower != region_end) && (get_tiny_previous_free_msize(follower) != msize)) {
- malloc_printf("*** invariant broken for tiny free %p followed by %p in region [%p-%p] "
- "(end marker incorrect) should be %d; in fact %d\n",
- ptr, follower, TINY_REGION_ADDRESS(region), region_end, msize, get_tiny_previous_free_msize(follower));
- return 0;
- }
- /* move to next block */
- ptr = (uintptr_t)follower;
- }
- }
- /*
- * Ensure that we scanned the entire region
- */
- if (ptr != region_end) {
- malloc_printf("*** invariant broken for region end %p - %p\n", ptr, region_end);
- return 0;
- }
- /*
- * Check the trailing block's integrity.
- */
- if (region == tiny_mag_ptr->mag_last_region) {
- if (tiny_mag_ptr->mag_bytes_free_at_end) {
- msize = get_tiny_meta_header((void *)ptr, &is_free);
- if (is_free || (msize != 1)) {
- malloc_printf("*** invariant broken for blocker block %p - %d %d\n", ptr, msize, is_free);
- }
- }
- }
- return 1;
-}
-
-static kern_return_t
-tiny_in_use_enumerator(task_t task, void *context, unsigned type_mask, szone_t *szone,
- memory_reader_t reader, vm_range_recorder_t recorder)
-{
- size_t num_regions;
- size_t index;
- region_t *regions;
- vm_range_t buffer[MAX_RECORDER_BUFFER];
- unsigned count = 0;
- kern_return_t err;
- region_t region;
- vm_range_t range;
- vm_range_t admin_range;
- vm_range_t ptr_range;
- unsigned char *mapped_region;
- uint32_t *block_header;
- uint32_t *in_use;
- unsigned block_index;
- unsigned block_limit;
- boolean_t is_free;
- msize_t msize;
- void *mapped_ptr;
- unsigned bit;
- magazine_t *tiny_mag_base = NULL;
-
- region_hash_generation_t *trg_ptr;
- err = reader(task, (vm_address_t)szone->tiny_region_generation, sizeof(region_hash_generation_t), (void **)&trg_ptr);
- if (err) return err;
-
- num_regions = trg_ptr->num_regions_allocated;
- err = reader(task, (vm_address_t)trg_ptr->hashed_regions, sizeof(region_t) * num_regions, (void **)®ions);
- if (err) return err;
-
- if (type_mask & MALLOC_PTR_IN_USE_RANGE_TYPE) {
- // Map in all active magazines. Do this outside the iteration over regions.
- err = reader(task, (vm_address_t)(szone->tiny_magazines),
- szone->num_tiny_magazines*sizeof(magazine_t),(void **)&tiny_mag_base);
- if (err) return err;
- }
-
- for (index = 0; index < num_regions; ++index) {
- region = regions[index];
- if (HASHRING_OPEN_ENTRY != region && HASHRING_REGION_DEALLOCATED != region) {
- range.address = (vm_address_t)TINY_REGION_ADDRESS(region);
- range.size = (vm_size_t)TINY_REGION_SIZE;
- if (type_mask & MALLOC_ADMIN_REGION_RANGE_TYPE) {
- admin_range.address = range.address + TINY_METADATA_START;
- admin_range.size = TINY_METADATA_SIZE;
- recorder(task, context, MALLOC_ADMIN_REGION_RANGE_TYPE, &admin_range, 1);
- }
- if (type_mask & (MALLOC_PTR_REGION_RANGE_TYPE | MALLOC_ADMIN_REGION_RANGE_TYPE)) {
- ptr_range.address = range.address;
- ptr_range.size = NUM_TINY_BLOCKS * TINY_QUANTUM;
- recorder(task, context, MALLOC_PTR_REGION_RANGE_TYPE, &ptr_range, 1);
- }
- if (type_mask & MALLOC_PTR_IN_USE_RANGE_TYPE) {
- void *mag_last_free;
- vm_address_t mag_last_free_ptr = 0;
- msize_t mag_last_free_msize = 0;
-
- err = reader(task, range.address, range.size, (void **)&mapped_region);
- if (err)
- return err;
-
- mag_index_t mag_index = MAGAZINE_INDEX_FOR_TINY_REGION(mapped_region);
- magazine_t *tiny_mag_ptr = tiny_mag_base + mag_index;
-
- if (DEPOT_MAGAZINE_INDEX != mag_index) {
- mag_last_free = tiny_mag_ptr->mag_last_free;
- if (mag_last_free) {
- mag_last_free_ptr = (uintptr_t) mag_last_free & ~(TINY_QUANTUM - 1);
- mag_last_free_msize = (uintptr_t) mag_last_free & (TINY_QUANTUM - 1);
- }
- } else {
- for (mag_index = 0; mag_index < szone->num_tiny_magazines; mag_index++) {
- if ((void *)range.address == (tiny_mag_base + mag_index)->mag_last_free_rgn) {
- mag_last_free = (tiny_mag_base + mag_index)->mag_last_free;
- if (mag_last_free) {
- mag_last_free_ptr = (uintptr_t) mag_last_free & ~(TINY_QUANTUM - 1);
- mag_last_free_msize = (uintptr_t) mag_last_free & (TINY_QUANTUM - 1);
- }
- }
- }
- }
-
- block_header = (uint32_t *)(mapped_region + TINY_METADATA_START + sizeof(region_trailer_t));
- in_use = TINY_INUSE_FOR_HEADER(block_header);
- block_index = 0;
- block_limit = NUM_TINY_BLOCKS;
- if (region == tiny_mag_ptr->mag_last_region) {
- block_index += TINY_MSIZE_FOR_BYTES(tiny_mag_ptr->mag_bytes_free_at_start);
- block_limit -= TINY_MSIZE_FOR_BYTES(tiny_mag_ptr->mag_bytes_free_at_end);
- }
-
- while (block_index < block_limit) {
- vm_size_t block_offset = TINY_BYTES_FOR_MSIZE(block_index);
- is_free = !BITARRAY_BIT(in_use, block_index);
- if (is_free) {
- mapped_ptr = mapped_region + block_offset;
-
- // mapped_region, the address at which 'range' in 'task' has been
- // mapped into our process, is not necessarily aligned to
- // TINY_BLOCKS_ALIGN.
- //
- // Since the code in get_tiny_free_size() assumes the pointer came
- // from a properly aligned tiny region, and mapped_region is not
- // necessarily aligned, then do the size calculation directly.
- // If the next bit is set in the header bitmap, then the size is one
- // quantum. Otherwise, read the size field.
- if (!BITARRAY_BIT(block_header, (block_index+1)))
- msize = TINY_FREE_SIZE(mapped_ptr);
- else
- msize = 1;
-
- } else if (range.address + block_offset != mag_last_free_ptr) {
- msize = 1;
- bit = block_index + 1;
- while (! BITARRAY_BIT(block_header, bit)) {
- bit++;
- msize ++;
- }
- buffer[count].address = range.address + block_offset;
- buffer[count].size = TINY_BYTES_FOR_MSIZE(msize);
- count++;
- if (count >= MAX_RECORDER_BUFFER) {
- recorder(task, context, MALLOC_PTR_IN_USE_RANGE_TYPE, buffer, count);
- count = 0;
- }
- } else {
- // Block is not free but it matches mag_last_free_ptr so even
- // though it is not marked free in the bitmap, we treat it as if
- // it is and move on
- msize = mag_last_free_msize;
- }
-
- if (!msize)
- return KERN_FAILURE; // Somethings amiss. Avoid looping at this block_index.
-
- block_index += msize;
- }
- if (count) {
- recorder(task, context, MALLOC_PTR_IN_USE_RANGE_TYPE, buffer, count);
- count = 0;
- }
- }
- }
- }
- return 0;
-}
-
-static void *
-tiny_malloc_from_free_list(szone_t *szone, magazine_t *tiny_mag_ptr, mag_index_t mag_index, msize_t msize)
-{
- free_list_t *ptr;
- msize_t this_msize;
- grain_t slot = msize - 1;
- free_list_t **free_list = tiny_mag_ptr->mag_free_list;
- free_list_t **the_slot = free_list + slot;
- free_list_t *next;
- free_list_t **limit;
-#if defined(__LP64__)
- uint64_t bitmap;
-#else
- uint32_t bitmap;
-#endif
- msize_t leftover_msize;
- free_list_t *leftover_ptr;
-
- // Assumes we've locked the region
- CHECK_MAGAZINE_PTR_LOCKED(szone, tiny_mag_ptr, __PRETTY_FUNCTION__);
-
- // Look for an exact match by checking the freelist for this msize.
- //
- ptr = *the_slot;
- if (ptr) {
- next = free_list_unchecksum_ptr(szone, &ptr->next);
- if (next) {
- next->previous = ptr->previous;
- } else {
- BITMAPV_CLR(tiny_mag_ptr->mag_bitmap, slot);
- }
- *the_slot = next;
- this_msize = msize;
-#if DEBUG_MALLOC
- if (LOG(szone, ptr)) {
- malloc_printf("in tiny_malloc_from_free_list(), exact match ptr=%p, this_msize=%d\n", ptr, this_msize);
- }
-#endif
- goto return_tiny_alloc;
- }
-
- // Mask off the bits representing slots holding free blocks smaller than the
- // size we need. If there are no larger free blocks, try allocating from
- // the free space at the end of the tiny region.
-#if defined(__LP64__)
- bitmap = ((uint64_t *)(tiny_mag_ptr->mag_bitmap))[0] & ~ ((1ULL << slot) - 1);
-#else
- bitmap = tiny_mag_ptr->mag_bitmap[0] & ~ ((1 << slot) - 1);
-#endif
- if (!bitmap)
- goto try_tiny_malloc_from_end;
-
- slot = BITMAPV_CTZ(bitmap);
- limit = free_list + NUM_TINY_SLOTS - 1;
- free_list += slot;
-
- if (free_list < limit) {
- ptr = *free_list;
- if (ptr) {
- next = free_list_unchecksum_ptr(szone, &ptr->next);
- *free_list = next;
- if (next) {
- next->previous = ptr->previous;
- } else {
- BITMAPV_CLR(tiny_mag_ptr->mag_bitmap, slot);
- }
- this_msize = get_tiny_free_size(ptr);
- goto add_leftover_and_proceed;
- }
-#if DEBUG_MALLOC
- malloc_printf("in tiny_malloc_from_free_list(), mag_bitmap out of sync, slot=%d\n",slot);
-#endif
- }
-
- // We are now looking at the last slot, which contains blocks equal to, or
- // due to coalescing of free blocks, larger than (NUM_TINY_SLOTS - 1) * tiny quantum size.
- // If the last freelist is not empty, and the head contains a block that is
- // larger than our request, then the remainder is put back on the free list.
- ptr = *limit;
- if (ptr) {
- this_msize = get_tiny_free_size(ptr);
- next = free_list_unchecksum_ptr(szone, &ptr->next);
- if (this_msize - msize >= NUM_TINY_SLOTS) {
- // the leftover will go back to the free list, so we optimize by
- // modifying the free list rather than a pop and push of the head
- leftover_msize = this_msize - msize;
- leftover_ptr = (free_list_t *)((unsigned char *)ptr + TINY_BYTES_FOR_MSIZE(msize));
- *limit = leftover_ptr;
- if (next) {
- next->previous.u = free_list_checksum_ptr(szone, leftover_ptr);
- }
- leftover_ptr->previous = ptr->previous;
- leftover_ptr->next = ptr->next;
- set_tiny_meta_header_free(leftover_ptr, leftover_msize);
-#if DEBUG_MALLOC
- if (LOG(szone,ptr)) {
- malloc_printf("in tiny_malloc_from_free_list(), last slot ptr=%p, msize=%d this_msize=%d\n",
- ptr, msize, this_msize);
- }
-#endif
- this_msize = msize;
- goto return_tiny_alloc;
- }
- if (next) {
- next->previous = ptr->previous;
- }
- *limit = next;
- goto add_leftover_and_proceed;
- /* NOTREACHED */
- }
-
-try_tiny_malloc_from_end:
- // Let's see if we can use tiny_mag_ptr->mag_bytes_free_at_end
- if (tiny_mag_ptr->mag_bytes_free_at_end >= TINY_BYTES_FOR_MSIZE(msize)) {
- ptr = (free_list_t *)((uintptr_t)TINY_REGION_END(tiny_mag_ptr->mag_last_region) -
- tiny_mag_ptr->mag_bytes_free_at_end);
- tiny_mag_ptr->mag_bytes_free_at_end -= TINY_BYTES_FOR_MSIZE(msize);
- if (tiny_mag_ptr->mag_bytes_free_at_end) {
- // let's add an in use block after ptr to serve as boundary
- set_tiny_meta_header_in_use_1((unsigned char *)ptr + TINY_BYTES_FOR_MSIZE(msize));
- }
- this_msize = msize;
-#if DEBUG_MALLOC
- if (LOG(szone, ptr)) {
- malloc_printf("in tiny_malloc_from_free_list(), from end ptr=%p, msize=%d\n", ptr, msize);
- }
-#endif
- goto return_tiny_alloc;
- }
-#if ASLR_INTERNAL
- // Try from start if nothing left at end
- if (tiny_mag_ptr->mag_bytes_free_at_start >= TINY_BYTES_FOR_MSIZE(msize)) {
- ptr = (free_list_t *)(TINY_REGION_ADDRESS(tiny_mag_ptr->mag_last_region) +
- tiny_mag_ptr->mag_bytes_free_at_start - TINY_BYTES_FOR_MSIZE(msize));
- tiny_mag_ptr->mag_bytes_free_at_start -= TINY_BYTES_FOR_MSIZE(msize);
- if (tiny_mag_ptr->mag_bytes_free_at_start) {
- // let's add an in use block before ptr to serve as boundary
- set_tiny_meta_header_in_use_1((unsigned char *)ptr - TINY_QUANTUM);
- }
- this_msize = msize;
-#if DEBUG_MALLOC
- if (LOG(szone, ptr)) {
- malloc_printf("in tiny_malloc_from_free_list(), from start ptr=%p, msize=%d\n", ptr, msize);
- }
-#endif
- goto return_tiny_alloc;
- }
-#endif
- return NULL;
-
-add_leftover_and_proceed:
- if (!this_msize || (this_msize > msize)) {
- leftover_msize = this_msize - msize;
- leftover_ptr = (free_list_t *)((unsigned char *)ptr + TINY_BYTES_FOR_MSIZE(msize));
-#if DEBUG_MALLOC
- if (LOG(szone,ptr)) {
- malloc_printf("in tiny_malloc_from_free_list(), adding leftover ptr=%p, this_msize=%d\n", ptr, this_msize);
- }
-#endif
- tiny_free_list_add_ptr(szone, tiny_mag_ptr, leftover_ptr, leftover_msize);
- this_msize = msize;
- }
-
-return_tiny_alloc:
- tiny_mag_ptr->mag_num_objects++;
- tiny_mag_ptr->mag_num_bytes_in_objects += TINY_BYTES_FOR_MSIZE(this_msize);
-
- // Update this region's bytes in use count
- region_trailer_t *node = REGION_TRAILER_FOR_TINY_REGION(TINY_REGION_FOR_PTR(ptr));
- size_t bytes_used = node->bytes_used + TINY_BYTES_FOR_MSIZE(this_msize);
- node->bytes_used = bytes_used;
-
- // Emptiness discriminant
- if (bytes_used < DENSITY_THRESHOLD(TINY_REGION_PAYLOAD_BYTES)) {
- /* After this allocation the region is still sparse, so it must have been even more so before
- the allocation. That implies the region is already correctly marked. Do nothing. */
- } else {
- /* Region has crossed threshold from sparsity to density. Mark it not "suitable" on the
- recirculation candidates list. */
- node->recirc_suitable = FALSE;
- }
-#if DEBUG_MALLOC
- if (LOG(szone,ptr)) {
- malloc_printf("in tiny_malloc_from_free_list(), ptr=%p, this_msize=%d, msize=%d\n", ptr, this_msize, msize);
- }
-#endif
- if (this_msize > 1)
- set_tiny_meta_header_in_use(ptr, this_msize);
- else
- set_tiny_meta_header_in_use_1(ptr);
- return ptr;
-}
-#undef DENSITY_THRESHOLD
-#undef K
-
-static INLINE void *
-tiny_malloc_should_clear(szone_t *szone, msize_t msize, boolean_t cleared_requested)
-{
- void *ptr;
- mag_index_t mag_index = mag_get_thread_index(szone);
- magazine_t *tiny_mag_ptr = &(szone->tiny_magazines[mag_index]);
-
-#if DEBUG_MALLOC
- if (DEPOT_MAGAZINE_INDEX == mag_index) {
- szone_error(szone, 1, "malloc called for magazine index -1", NULL, NULL);
- return(NULL);
- }
-
- if (!msize) {
- szone_error(szone, 1, "invariant broken (!msize) in allocation (region)", NULL, NULL);
- return(NULL);
- }
-#endif
-
- SZONE_MAGAZINE_PTR_LOCK(szone, tiny_mag_ptr);
-
-#if TINY_CACHE
- ptr = tiny_mag_ptr->mag_last_free;
-
- if ((((uintptr_t)ptr) & (TINY_QUANTUM - 1)) == msize) {
- // we have a winner
- tiny_mag_ptr->mag_last_free = NULL;
- tiny_mag_ptr->mag_last_free_rgn = NULL;
- SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr);
- CHECK(szone, __PRETTY_FUNCTION__);
- ptr = (void *)((uintptr_t)ptr & ~ (TINY_QUANTUM - 1));
- if (cleared_requested) {
- memset(ptr, 0, TINY_BYTES_FOR_MSIZE(msize));
- }
-#if DEBUG_MALLOC
- if (LOG(szone,ptr)) {
- malloc_printf("in tiny_malloc_should_clear(), tiny cache ptr=%p, msize=%d\n", ptr, msize);
- }
-#endif
- return ptr;
- }
-#endif /* TINY_CACHE */
-
- while (1) {
- ptr = tiny_malloc_from_free_list(szone, tiny_mag_ptr, mag_index, msize);
- if (ptr) {
- SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr);
- CHECK(szone, __PRETTY_FUNCTION__);
- if (cleared_requested) {
- memset(ptr, 0, TINY_BYTES_FOR_MSIZE(msize));
- }
- return ptr;
- }
-
- if (tiny_get_region_from_depot(szone, tiny_mag_ptr, mag_index, msize)) {
- ptr = tiny_malloc_from_free_list(szone, tiny_mag_ptr, mag_index, msize);
- if (ptr) {
- SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr);
- CHECK(szone, __PRETTY_FUNCTION__);
- if (cleared_requested) {
- memset(ptr, 0, TINY_BYTES_FOR_MSIZE(msize));
- }
- return ptr;
- }
- }
-
- // The magazine is exhausted. A new region (heap) must be allocated to satisfy this call to malloc().
- // The allocation, an mmap() system call, will be performed outside the magazine spin locks by the first
- // thread that suffers the exhaustion. That thread sets "alloc_underway" and enters a critical section.
- // Threads arriving here later are excluded from the critical section, yield the CPU, and then retry the
- // allocation. After some time the magazine is resupplied, the original thread leaves with its allocation,
- // and retry-ing threads succeed in the code just above.
- if (!tiny_mag_ptr->alloc_underway) {
- void *fresh_region;
-
- // time to create a new region (do this outside the magazine lock)
- tiny_mag_ptr->alloc_underway = TRUE;
- OSMemoryBarrier();
- SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr);
- fresh_region = allocate_pages_securely(szone, TINY_REGION_SIZE, TINY_BLOCKS_ALIGN, VM_MEMORY_MALLOC_TINY);
- SZONE_MAGAZINE_PTR_LOCK(szone, tiny_mag_ptr);
-
- MAGMALLOC_ALLOCREGION((void *)szone, (int)mag_index, fresh_region, TINY_REGION_SIZE); // DTrace USDT Probe
-
- if (!fresh_region) { // out of memory!
- tiny_mag_ptr->alloc_underway = FALSE;
- OSMemoryBarrier();
- SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr);
- return NULL;
- }
-
- ptr = tiny_malloc_from_region_no_lock(szone, tiny_mag_ptr, mag_index, msize, fresh_region);
-
- // we don't clear because this freshly allocated space is pristine
- tiny_mag_ptr->alloc_underway = FALSE;
- OSMemoryBarrier();
- SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr);
- CHECK(szone, __PRETTY_FUNCTION__);
- return ptr;
- } else {
- SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr);
- pthread_yield_np();
- SZONE_MAGAZINE_PTR_LOCK(szone, tiny_mag_ptr);
- }
- }
- /* NOTREACHED */
-}
-
-static NOINLINE void
-free_tiny_botch(szone_t *szone, free_list_t *ptr)
-{
- mag_index_t mag_index = MAGAZINE_INDEX_FOR_TINY_REGION(TINY_REGION_FOR_PTR(ptr));
- magazine_t *tiny_mag_ptr = &(szone->tiny_magazines[mag_index]);
- SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr);
- szone_error(szone, 1, "double free", ptr, NULL);
-}
-
-static INLINE void
-free_tiny(szone_t *szone, void *ptr, region_t tiny_region, size_t known_size)
-{
- msize_t msize;
- boolean_t is_free;
- mag_index_t mag_index = MAGAZINE_INDEX_FOR_TINY_REGION(tiny_region);
- magazine_t *tiny_mag_ptr = &(szone->tiny_magazines[mag_index]);
-
- // ptr is known to be in tiny_region
- if (known_size) {
- msize = TINY_MSIZE_FOR_BYTES(known_size + TINY_QUANTUM - 1);
- } else {
- msize = get_tiny_meta_header(ptr, &is_free);
- if (is_free) {
- free_tiny_botch(szone, ptr);
- return;
- }
- }
-#if DEBUG_MALLOC
- if (!msize) {
- malloc_printf("*** free_tiny() block in use is too large: %p\n", ptr);
- return;
- }
-#endif
-
- SZONE_MAGAZINE_PTR_LOCK(szone, tiny_mag_ptr);
-
-#if TINY_CACHE
- // Depot does not participate in TINY_CACHE since it can't be directly malloc()'d
- if (DEPOT_MAGAZINE_INDEX != mag_index) {
- if (msize < TINY_QUANTUM) { // to see if the bits fit in the last 4 bits
- void *ptr2 = tiny_mag_ptr->mag_last_free; // Might be NULL
- region_t rgn2 = tiny_mag_ptr->mag_last_free_rgn;
-
- /* check that we don't already have this pointer in the cache */
- if (ptr == (void *)((uintptr_t)ptr2 & ~ (TINY_QUANTUM - 1))) {
- free_tiny_botch(szone, ptr);
- return;
- }
-
- if ((szone->debug_flags & SCALABLE_MALLOC_DO_SCRIBBLE) && msize)
- memset(ptr, 0x55, TINY_BYTES_FOR_MSIZE(msize));
-
- tiny_mag_ptr->mag_last_free = (void *)(((uintptr_t)ptr) | msize);
- tiny_mag_ptr->mag_last_free_rgn = tiny_region;
-
- if (!ptr2) {
- SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr);
- CHECK(szone, __PRETTY_FUNCTION__);
- return;
- }
-
- msize = (uintptr_t)ptr2 & (TINY_QUANTUM - 1);
- ptr = (void *)(((uintptr_t)ptr2) & ~(TINY_QUANTUM - 1));
- tiny_region = rgn2;
- }
- }
-#endif /* TINY_CACHE */
-
- // Now in the time it took to acquire the lock, the region may have migrated
- // from one magazine to another. I.e. trailer->mag_index is volatile.
- // In which case the magazine lock we obtained (namely magazines[mag_index].mag_lock)
- // is stale. If so, keep on tryin' ...
- region_trailer_t *trailer = REGION_TRAILER_FOR_TINY_REGION(tiny_region);
- mag_index_t refreshed_index;
-
- while (mag_index != (refreshed_index = trailer->mag_index)) { // Note assignment
-
- SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr);
-
- mag_index = refreshed_index;
- tiny_mag_ptr = &(szone->tiny_magazines[mag_index]);
- SZONE_MAGAZINE_PTR_LOCK(szone, tiny_mag_ptr);
- }
-
- if (tiny_free_no_lock(szone, tiny_mag_ptr, mag_index, tiny_region, ptr, msize))
- SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr);
-
- CHECK(szone, __PRETTY_FUNCTION__);
-}
-
-static void
-print_tiny_free_list(szone_t *szone)
-{
- free_list_t *ptr;
- _SIMPLE_STRING b = _simple_salloc();
- mag_index_t mag_index;
-
- if (b) {
- _simple_sappend(b, "tiny free sizes:\n");
- for (mag_index = -1; mag_index < szone->num_tiny_magazines; mag_index++) {
- grain_t slot = 0;
- _simple_sprintf(b,"\tMagazine %d: ", mag_index);
- while (slot < NUM_TINY_SLOTS) {
- ptr = szone->tiny_magazines[mag_index].mag_free_list[slot];
- if (ptr) {
- _simple_sprintf(b, "%s%y[%d]; ", (slot == NUM_TINY_SLOTS-1) ? ">=" : "",
- (slot+1)*TINY_QUANTUM, free_list_count(szone, ptr));
- }
- slot++;
- }
- _simple_sappend(b,"\n");
- }
- _malloc_printf(MALLOC_PRINTF_NOLOG | MALLOC_PRINTF_NOPREFIX, "%s\n", _simple_string(b));
- _simple_sfree(b);
- }
-}
-
-static void
-print_tiny_region(boolean_t verbose, region_t region, size_t bytes_at_start, size_t bytes_at_end)
-{
- unsigned counts[1024];
- unsigned in_use = 0;
- uintptr_t start = (uintptr_t)TINY_REGION_ADDRESS(region);
- uintptr_t current = start + bytes_at_end;
- uintptr_t limit = (uintptr_t)TINY_REGION_END(region) - bytes_at_end;
- boolean_t is_free;
- msize_t msize;
- unsigned ci;
- _SIMPLE_STRING b;
- uintptr_t pgTot = 0;
-
- if (region == HASHRING_REGION_DEALLOCATED) {
- if ((b = _simple_salloc()) != NULL) {
- _simple_sprintf(b, "Tiny region [unknown address] was returned to the OS\n");
- _malloc_printf(MALLOC_PRINTF_NOLOG | MALLOC_PRINTF_NOPREFIX, "%s\n", _simple_string(b));
- _simple_sfree(b);
- }
- return;
- }
-
- memset(counts, 0, sizeof(counts));
- while (current < limit) {
- msize = get_tiny_meta_header((void *)current, &is_free);
- if (is_free & !msize && (current == start)) {
- // first block is all free
- uintptr_t pgLo = round_page(start + sizeof(free_list_t) + sizeof(msize_t));
- uintptr_t pgHi = trunc_page(start + TINY_REGION_SIZE - sizeof(msize_t));
-
- if (pgLo < pgHi) {
- pgTot += (pgHi - pgLo);
- }
- break;
- }
- if (!msize) {
- malloc_printf("*** error with %p: msize=%d\n", (void *)current, (unsigned)msize);
- break;
- }
- if (!is_free) {
- // block in use
- if (msize > NUM_TINY_SLOTS)
- malloc_printf("*** error at %p msize for in_use is %d\n", (void *)current, msize);
- if (msize < 1024)
- counts[msize]++;
- in_use++;
- } else {
- uintptr_t pgLo = round_page(current + sizeof(free_list_t) + sizeof(msize_t));
- uintptr_t pgHi = trunc_page(current + TINY_BYTES_FOR_MSIZE(msize) - sizeof(msize_t));
-
- if (pgLo < pgHi) {
- pgTot += (pgHi - pgLo);
- }
- }
- current += TINY_BYTES_FOR_MSIZE(msize);
- }
- if ((b = _simple_salloc()) != NULL) {
- _simple_sprintf(b, "Tiny region [%p-%p, %y] \t", (void *)start, TINY_REGION_END(region), (int)TINY_REGION_SIZE);
- _simple_sprintf(b, "Magazine=%d \t", MAGAZINE_INDEX_FOR_TINY_REGION(region));
- _simple_sprintf(b, "Allocations in use=%d \t Bytes in use=%ly \t", in_use, BYTES_USED_FOR_TINY_REGION(region));
- if (bytes_at_end || bytes_at_start)
- _simple_sprintf(b, "Untouched=%ly ", bytes_at_end + bytes_at_start);
- if (DEPOT_MAGAZINE_INDEX == MAGAZINE_INDEX_FOR_TINY_REGION(region)) {
- _simple_sprintf(b, "Advised MADV_FREE=%ly", pgTot);
- } else {
- _simple_sprintf(b, "Fragments subject to reclamation=%ly", pgTot);
- }
- if (verbose && in_use) {
- _simple_sappend(b, "\n\tSizes in use: ");
- for (ci = 0; ci < 1024; ci++)
- if (counts[ci])
- _simple_sprintf(b, "%d[%d] ", TINY_BYTES_FOR_MSIZE(ci), counts[ci]);
- }
- _malloc_printf(MALLOC_PRINTF_NOLOG | MALLOC_PRINTF_NOPREFIX, "%s\n", _simple_string(b));
- _simple_sfree(b);
- }
-}
-
-static boolean_t
-tiny_free_list_check(szone_t *szone, grain_t slot)
-{
- mag_index_t mag_index;
-
- for (mag_index = -1; mag_index < szone->num_tiny_magazines; mag_index++) {
- magazine_t *tiny_mag_ptr = &(szone->tiny_magazines[mag_index]);
- SZONE_MAGAZINE_PTR_LOCK(szone, tiny_mag_ptr);
-
- unsigned count = 0;
- free_list_t *ptr = szone->tiny_magazines[mag_index].mag_free_list[slot];
- boolean_t is_free;
- free_list_t *previous = NULL;
-
- while (ptr) {
- is_free = tiny_meta_header_is_free(ptr);
- if (! is_free) {
- malloc_printf("*** in-use ptr in free list slot=%d count=%d ptr=%p\n", slot, count, ptr);
- SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr);
- return 0;
- }
- if (((uintptr_t)ptr) & (TINY_QUANTUM - 1)) {
- malloc_printf("*** unaligned ptr in free list slot=%d count=%d ptr=%p\n", slot, count, ptr);
- SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr);
- return 0;
- }
- if (!tiny_region_for_ptr_no_lock(szone, ptr)) {
- malloc_printf("*** ptr not in szone slot=%d count=%d ptr=%p\n", slot, count, ptr);
- SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr);
- return 0;
- }
- if (free_list_unchecksum_ptr(szone, &ptr->previous) != previous) {
- malloc_printf("*** previous incorrectly set slot=%d count=%d ptr=%p\n", slot, count, ptr);
- SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr);
- return 0;
- }
- previous = ptr;
- ptr = free_list_unchecksum_ptr(szone, &ptr->next);
- count++;
- }
-
- SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr);
- }
- return 1;
-}
-
-/********************* SMALL FREE LIST UTILITIES ************************/
-
-/*
- * Mark a block as free. Only the first quantum of a block is marked thusly,
- * the remainder are marked "middle".
- */
-static INLINE void
-small_meta_header_set_is_free(msize_t *meta_headers, unsigned index, msize_t msize)
-{
- meta_headers[index] = msize | SMALL_IS_FREE;
-}
-
-/*
- * Mark a block as in use. Only the first quantum of a block is marked thusly,
- * the remainder are marked "middle".
- */
-static INLINE void
-small_meta_header_set_in_use(msize_t *meta_headers, msize_t index, msize_t msize)
-{
- meta_headers[index] = msize;
-}
-
-/*
- * Mark a quantum as being the second or later in a block.
- */
-static INLINE void
-small_meta_header_set_middle(msize_t *meta_headers, msize_t index)
-{
- meta_headers[index] = 0;
-}
-
-/*
- * Adds an item to the proper free list, and also marks the meta-header of the
- * block properly.
- * Assumes szone has been locked
- */
-static void
-small_free_list_add_ptr(szone_t *szone, magazine_t *small_mag_ptr, void *ptr, msize_t msize)
-{
- grain_t slot = (msize <= szone->num_small_slots) ? msize - 1 : szone->num_small_slots - 1;
- free_list_t *free_ptr = ptr;
- free_list_t *free_head = small_mag_ptr->mag_free_list[slot];
- void *follower;
-
-#if DEBUG_MALLOC
- if (LOG(szone,ptr)) {
- malloc_printf("in %s, ptr=%p, msize=%d\n", __FUNCTION__, ptr, msize);
- }
- if (((uintptr_t)ptr) & (SMALL_QUANTUM - 1)) {
- szone_error(szone, 1, "small_free_list_add_ptr: Unaligned ptr", ptr, NULL);
- }
-#endif
- small_meta_header_set_is_free(SMALL_META_HEADER_FOR_PTR(ptr), SMALL_META_INDEX_FOR_PTR(ptr), msize);
-
- if (free_head) {
-#if DEBUG_MALLOC
- if (free_list_unchecksum_ptr(szone, &free_head->previous)) {
- szone_error(szone, 1, "small_free_list_add_ptr: Internal invariant broken (free_head->previous)", ptr,
- "ptr=%p slot=%d free_head=%p previous=%p\n", ptr, slot, (void *)free_head, free_head->previous.p);
- }
- if (!SMALL_PTR_IS_FREE(free_head)) {
- szone_error(szone, 1, "small_free_list_add_ptr: Internal invariant broken (free_head is not a free pointer)", ptr,
- "ptr=%p slot=%d free_head=%p\n", ptr, slot, (void *)free_head);
- }
-#endif
- free_head->previous.u = free_list_checksum_ptr(szone, free_ptr);
- } else {
- BITMAPN_SET(small_mag_ptr->mag_bitmap, slot);
- }
- free_ptr->previous.u = free_list_checksum_ptr(szone, NULL);
- free_ptr->next.u = free_list_checksum_ptr(szone, free_head);
-
- small_mag_ptr->mag_free_list[slot] = free_ptr;
-
- // Store msize at the end of the block denoted by "ptr" (i.e. at a negative offset from "follower")
- follower = (void *)((uintptr_t)ptr + SMALL_BYTES_FOR_MSIZE(msize));
- SMALL_PREVIOUS_MSIZE(follower) = msize;
-}
-
-/*
- * Removes the item pointed to by ptr in the proper free list.
- * Assumes szone has been locked
- */
-static void
-small_free_list_remove_ptr(szone_t *szone, magazine_t *small_mag_ptr, void *ptr, msize_t msize)
-{
- grain_t slot = (msize <= szone->num_small_slots) ? msize - 1 : szone->num_small_slots - 1;
- free_list_t *free_ptr = ptr, *next, *previous;
-
- next = free_list_unchecksum_ptr(szone, &free_ptr->next);
- previous = free_list_unchecksum_ptr(szone, &free_ptr->previous);
-
-#if DEBUG_MALLOC
- if (LOG(szone,ptr)) {
- malloc_printf("In %s, ptr=%p, msize=%d\n", __FUNCTION__, ptr, msize);
- }
-#endif
-
- if (!previous) {
- // The block to remove is the head of the free list
-#if DEBUG_MALLOC
- if (small_mag_ptr->mag_free_list[slot] != ptr) {
- szone_error(szone, 1, "small_free_list_remove_ptr: Internal invariant broken (small_mag_ptr->mag_free_list[slot])", ptr,
- "ptr=%p slot=%d msize=%d small_mag_ptr->mag_free_list[slot]=%p\n",
- ptr, slot, msize, (void *)small_mag_ptr->mag_free_list[slot]);
- return;
- }
-#endif
- small_mag_ptr->mag_free_list[slot] = next;
- if (!next) BITMAPN_CLR(small_mag_ptr->mag_bitmap, slot);
- } else {
- // We know free_ptr is already checksummed, so we don't need to do it
- // again.
- previous->next = free_ptr->next;
- }
- if (next) {
- // We know free_ptr is already checksummed, so we don't need to do it
- // again.
- next->previous = free_ptr->previous;
- }
-}
-
-/*
- * small_region_for_ptr_no_lock - Returns the small region containing the pointer,
- * or NULL if not found.
- */
-static INLINE region_t
-small_region_for_ptr_no_lock(szone_t *szone, const void *ptr)
-{
- rgnhdl_t r = hash_lookup_region_no_lock(szone->small_region_generation->hashed_regions,
- szone->small_region_generation->num_regions_allocated,
- szone->small_region_generation->num_regions_allocated_shift,
- SMALL_REGION_FOR_PTR(ptr));
- return r ? *r : r;
-}
-
-static void
-small_finalize_region(szone_t *szone, magazine_t *small_mag_ptr) {
- void *last_block, *previous_block;
- msize_t last_msize, previous_msize, last_index;
-
- // It is possible that the block prior to the last block in the region has
- // been free'd, but was not coalesced with the free bytes at the end of the
- // block, since we treat the bytes at the end of the region as "in use" in
- // the meta headers. Attempt to coalesce the last block with the previous
- // block, so we don't violate the "no consecutive free blocks" invariant.
- //
- // FIXME: If we could calculate the previous small free size in the same
- // manner as tiny_previous_preceding_free, it would eliminate the
- // index & previous msize checks, which are a guard against reading
- // bogus data out of in-use or written-on-freed memory.
- //
- // FIXME: Need to investigate how much work would be required to increase
- // 'mag_bytes_free_at_end' when freeing the preceding block, rather
- // than performing this workaround.
- //
- if (small_mag_ptr->mag_bytes_free_at_end) {
- last_block = SMALL_REGION_END(small_mag_ptr->mag_last_region) - small_mag_ptr->mag_bytes_free_at_end;
- last_msize = SMALL_MSIZE_FOR_BYTES(small_mag_ptr->mag_bytes_free_at_end);
-
- last_index = SMALL_META_INDEX_FOR_PTR(last_block);
- previous_msize = SMALL_PREVIOUS_MSIZE(last_block);
-
- if (last_index && (previous_msize <= last_index)) {
- previous_block = (void *)((uintptr_t)last_block - SMALL_BYTES_FOR_MSIZE(previous_msize));
- if (*SMALL_METADATA_FOR_PTR(previous_block) == (previous_msize | SMALL_IS_FREE)) {
- msize_t *meta_headers = SMALL_META_HEADER_FOR_PTR(last_block);
-
- small_meta_header_set_middle(meta_headers, last_index);
- small_free_list_remove_ptr(szone, small_mag_ptr, previous_block, previous_msize);
- last_block = (void *)((uintptr_t)last_block - SMALL_BYTES_FOR_MSIZE(previous_msize));
- last_msize += previous_msize;
- }
- }
-
- // splice last_block into the free list
- small_free_list_add_ptr(szone, small_mag_ptr, last_block, last_msize);
- small_mag_ptr->mag_bytes_free_at_end = 0;
- }
-
-#if ASLR_INTERNAL
- if (small_mag_ptr->mag_bytes_free_at_start) {
- last_block = SMALL_REGION_ADDRESS(small_mag_ptr->mag_last_region);
- last_msize = SMALL_MSIZE_FOR_BYTES(small_mag_ptr->mag_bytes_free_at_start);
-
- void *next_block = (void *) ((uintptr_t)last_block + small_mag_ptr->mag_bytes_free_at_start);
- if (SMALL_PTR_IS_FREE(next_block)) {
- msize_t next_msize = SMALL_PTR_SIZE(next_block);
-
- small_meta_header_set_middle(SMALL_META_HEADER_FOR_PTR(next_block), SMALL_META_INDEX_FOR_PTR(next_block));
- small_free_list_remove_ptr(szone, small_mag_ptr, next_block, next_msize);
- last_msize += next_msize;
- }
-
- // splice last_block into the free list
- small_free_list_add_ptr(szone, small_mag_ptr, last_block, last_msize);
- small_mag_ptr->mag_bytes_free_at_start = 0;
- }
-#endif
-
- // TODO: Will we ever need to coalesce the blocks at the beginning and end when we finalize?
-
- small_mag_ptr->mag_last_region = NULL;
-}
-
-static int
-small_free_detach_region(szone_t *szone, magazine_t *small_mag_ptr, region_t r) {
- unsigned char *ptr = SMALL_REGION_ADDRESS(r);
- msize_t *meta_headers = SMALL_META_HEADER_FOR_PTR(ptr);
- uintptr_t start = (uintptr_t)SMALL_REGION_ADDRESS(r);
- uintptr_t current = start;
- uintptr_t limit = (uintptr_t)SMALL_REGION_END(r);
- int total_alloc = 0;
-
- while (current < limit) {
- unsigned index = SMALL_META_INDEX_FOR_PTR(current);
- msize_t msize_and_free = meta_headers[index];
- boolean_t is_free = msize_and_free & SMALL_IS_FREE;
- msize_t msize = msize_and_free & ~ SMALL_IS_FREE;
-
- if (!msize) {
-#if DEBUG_MALLOC
- malloc_printf("*** small_free_detach_region error with %p: msize=%d is_free =%d\n",
- (void *)current, msize, is_free);
-#endif
- break;
- }
- if (is_free) {
- small_free_list_remove_ptr(szone, small_mag_ptr, (void *)current, msize);
- } else {
- total_alloc++;
- }
- current += SMALL_BYTES_FOR_MSIZE(msize);
- }
- return total_alloc;
-}
-
-static size_t
-small_free_reattach_region(szone_t *szone, magazine_t *small_mag_ptr, region_t r) {
- unsigned char *ptr = SMALL_REGION_ADDRESS(r);
- msize_t *meta_headers = SMALL_META_HEADER_FOR_PTR(ptr);
- uintptr_t start = (uintptr_t)SMALL_REGION_ADDRESS(r);
- uintptr_t current = start;
- uintptr_t limit = (uintptr_t)SMALL_REGION_END(r);
- size_t total_alloc = 0;
-
- while (current < limit) {
- unsigned index = SMALL_META_INDEX_FOR_PTR(current);
- msize_t msize_and_free = meta_headers[index];
- boolean_t is_free = msize_and_free & SMALL_IS_FREE;
- msize_t msize = msize_and_free & ~ SMALL_IS_FREE;
-
- if (!msize) {
-#if DEBUG_MALLOC
- malloc_printf("*** small_free_reattach_region error with %p: msize=%d is_free =%d\n",
- (void *)current, msize, is_free);
-#endif
- break;
- }
- if (is_free) {
- small_free_list_add_ptr(szone, small_mag_ptr, (void *)current, msize);
- } else {
- total_alloc += SMALL_BYTES_FOR_MSIZE(msize);
- }
- current += SMALL_BYTES_FOR_MSIZE(msize);
- }
- return total_alloc;
-}
-
-typedef struct {
- uint16_t pnum, size;
-} small_pg_pair_t;
-
-static void NOINLINE /* want private stack frame for automatic array */
-small_free_scan_madvise_free(szone_t *szone, magazine_t *depot_ptr, region_t r) {
- uintptr_t start = (uintptr_t)SMALL_REGION_ADDRESS(r);
- uintptr_t current = start;
- uintptr_t limit = (uintptr_t)SMALL_REGION_END(r);
- msize_t *meta_headers = SMALL_META_HEADER_FOR_PTR(start);
- small_pg_pair_t advisory[((SMALL_REGION_PAYLOAD_BYTES + vm_page_size - 1) >> vm_page_shift) >> 1]; // 4096bytes stack allocated
- int advisories = 0;
-
- // Scan the metadata identifying blocks which span one or more pages. Mark the pages MADV_FREE taking care to preserve free list
- // management data.
- while (current < limit) {
- unsigned index = SMALL_META_INDEX_FOR_PTR(current);
- msize_t msize_and_free = meta_headers[index];
- boolean_t is_free = msize_and_free & SMALL_IS_FREE;
- msize_t msize = msize_and_free & ~ SMALL_IS_FREE;
-
- if (is_free && !msize && (current == start)) {
-#if DEBUG_MALLOC
- // first block is all free
- malloc_printf("*** small_free_scan_madvise_free first block is all free! %p: msize=%d is_free =%d\n",
- (void *)current, msize, is_free);
-#endif
- uintptr_t pgLo = round_page(start + sizeof(free_list_t) + sizeof(msize_t));
- uintptr_t pgHi = trunc_page(start + SMALL_REGION_SIZE - sizeof(msize_t));
-
- if (pgLo < pgHi) {
- advisory[advisories].pnum = (pgLo - start) >> vm_page_shift;
- advisory[advisories].size = (pgHi - pgLo) >> vm_page_shift;
- advisories++;
- }
- break;
- }
- if (!msize) {
-#if DEBUG_MALLOC
- malloc_printf("*** small_free_scan_madvise_free error with %p: msize=%d is_free =%d\n",
- (void *)current, msize, is_free);
-#endif
- break;
- }
- if (is_free) {
- uintptr_t pgLo = round_page(current + sizeof(free_list_t) + sizeof(msize_t));
- uintptr_t pgHi = trunc_page(current + SMALL_BYTES_FOR_MSIZE(msize) - sizeof(msize_t));
-
- if (pgLo < pgHi) {
- advisory[advisories].pnum = (pgLo - start) >> vm_page_shift;
- advisory[advisories].size = (pgHi - pgLo) >> vm_page_shift;
- advisories++;
- }
- }
- current += SMALL_BYTES_FOR_MSIZE(msize);
- }
-
- if (advisories > 0) {
- int i;
-
- OSAtomicIncrement32Barrier(&(REGION_TRAILER_FOR_SMALL_REGION(r)->pinned_to_depot));
- SZONE_MAGAZINE_PTR_UNLOCK(szone, depot_ptr);
- for (i = 0; i < advisories; ++i) {
- uintptr_t addr = (advisory[i].pnum << vm_page_shift) + start;
- size_t size = advisory[i].size << vm_page_shift;
-
-#if TARGET_OS_EMBEDDED
- madvise_free_range(szone, r, addr, addr + size, NULL);
-#else
- madvise_free_range(szone, r, addr, addr + size);
-#endif
- }
- SZONE_MAGAZINE_PTR_LOCK(szone, depot_ptr);
- OSAtomicDecrement32Barrier(&(REGION_TRAILER_FOR_SMALL_REGION(r)->pinned_to_depot));
- }
-}
-
-static region_t
-small_free_try_depot_unmap_no_lock(szone_t *szone, magazine_t *depot_ptr, region_trailer_t *node)
-{
- if (0 < node->bytes_used ||
- 0 < node->pinned_to_depot ||
- depot_ptr->recirculation_entries < (szone->num_small_magazines * 2)) {
- return NULL;
- }
-
- // disconnect first node from Depot
- recirc_list_extract(szone, depot_ptr, node);
-
- // Iterate the region pulling its free entries off the (locked) Depot's free list
- region_t sparse_region = SMALL_REGION_FOR_PTR(node);
- int objects_in_use = small_free_detach_region(szone, depot_ptr, sparse_region);
-
- if (0 == objects_in_use) {
- // Invalidate the hash table entry for this region with HASHRING_REGION_DEALLOCATED.
- // Using HASHRING_REGION_DEALLOCATED preserves the collision chain, using HASHRING_OPEN_ENTRY (0) would not.
- rgnhdl_t pSlot = hash_lookup_region_no_lock(szone->small_region_generation->hashed_regions,
- szone->small_region_generation->num_regions_allocated,
- szone->small_region_generation->num_regions_allocated_shift, sparse_region);
- if (NULL == pSlot) {
- szone_error(szone, 1, "small_free_try_depot_unmap_no_lock hash lookup failed:", NULL, "%p\n", sparse_region);
- return NULL;
- }
- *pSlot = HASHRING_REGION_DEALLOCATED;
- depot_ptr->num_bytes_in_magazine -= SMALL_REGION_PAYLOAD_BYTES;
- __sync_fetch_and_add( &(szone->num_small_regions_dealloc), 1); // Atomically increment num_small_regions_dealloc
-
- // Caller will transfer ownership of the region back to the OS with no locks held
- MAGMALLOC_DEALLOCREGION((void *)szone, (void *)sparse_region, SMALL_REGION_SIZE); // DTrace USDT Probe
- return sparse_region;
-
- } else {
- szone_error(szone, 1, "small_free_try_depot_unmap_no_lock objects_in_use not zero:", NULL, "%d\n", objects_in_use);
- return NULL;
- }
-}
-
-static boolean_t
-small_free_do_recirc_to_depot(szone_t *szone, magazine_t *small_mag_ptr, mag_index_t mag_index)
-{
- // The entire magazine crossed the "emptiness threshold". Transfer a region
- // from this magazine to the Depot. Choose a region that itself has crossed the emptiness threshold (i.e
- // is at least fraction "f" empty.) Such a region will be marked "suitable" on the recirculation list.
- region_trailer_t *node = small_mag_ptr->firstNode;
-
- while (node && !node->recirc_suitable) {
- node = node->next;
- }
-
- if (NULL == node) {
-#if DEBUG_MALLOC
- malloc_printf("*** small_free_do_recirc_to_depot end of list\n");
-#endif
- return TRUE; // Caller must SZONE_MAGAZINE_PTR_UNLOCK(szone, small_mag_ptr);
- }
-
- region_t sparse_region = SMALL_REGION_FOR_PTR(node);
-
- // Deal with unclaimed memory -- mag_bytes_free_at_end or mag_bytes_free_at start
- if (sparse_region == small_mag_ptr->mag_last_region && (small_mag_ptr->mag_bytes_free_at_end || small_mag_ptr->mag_bytes_free_at_start)) {
- small_finalize_region(szone, small_mag_ptr);
- }
-
- // disconnect "suitable" node from magazine
- recirc_list_extract(szone, small_mag_ptr, node);
-
- // Iterate the region pulling its free entries off its (locked) magazine's free list
- int objects_in_use = small_free_detach_region(szone, small_mag_ptr, sparse_region);
- magazine_t *depot_ptr = &(szone->small_magazines[DEPOT_MAGAZINE_INDEX]);
-
- // hand over the region to the (locked) Depot
- SZONE_MAGAZINE_PTR_LOCK(szone,depot_ptr);
- // this will cause small_free_list_add_ptr called by small_free_reattach_region to use
- // the depot as its target magazine, rather than magazine formerly associated with sparse_region
- MAGAZINE_INDEX_FOR_SMALL_REGION(sparse_region) = DEPOT_MAGAZINE_INDEX;
- node->pinned_to_depot = 0;
-
- // Iterate the region putting its free entries on Depot's free list
- size_t bytes_inplay = small_free_reattach_region(szone, depot_ptr, sparse_region);
-
- small_mag_ptr->mag_num_bytes_in_objects -= bytes_inplay;
- small_mag_ptr->num_bytes_in_magazine -= SMALL_REGION_PAYLOAD_BYTES;
- small_mag_ptr->mag_num_objects -= objects_in_use;
-
- SZONE_MAGAZINE_PTR_UNLOCK(szone, small_mag_ptr); // Unlock the originating magazine
-
- depot_ptr->mag_num_bytes_in_objects += bytes_inplay;
- depot_ptr->num_bytes_in_magazine += SMALL_REGION_PAYLOAD_BYTES;
- depot_ptr->mag_num_objects += objects_in_use;
-
- // connect to Depot as last node
- recirc_list_splice_last(szone, depot_ptr, node);
-
- MAGMALLOC_RECIRCREGION((void *)szone, (int)mag_index, (void *)sparse_region, SMALL_REGION_SIZE,
- (int)BYTES_USED_FOR_SMALL_REGION(sparse_region)); // DTrace USDT Probe
-
- // Mark free'd dirty pages with MADV_FREE to reduce memory pressure
- small_free_scan_madvise_free(szone, depot_ptr, sparse_region);
-
- // If the region is entirely empty vm_deallocate() it outside the depot lock
- region_t r_dealloc = small_free_try_depot_unmap_no_lock(szone, depot_ptr, node);
- SZONE_MAGAZINE_PTR_UNLOCK(szone,depot_ptr);
- if (r_dealloc)
- deallocate_pages(szone, r_dealloc, SMALL_REGION_SIZE, 0);
- return FALSE; // Caller need not unlock the originating magazine
-}
-
-static region_t
-small_find_msize_region(szone_t *szone, magazine_t *small_mag_ptr, mag_index_t mag_index, msize_t msize)
-{
- free_list_t *ptr;
- grain_t slot = (msize <= szone->num_small_slots) ? msize - 1 : szone->num_small_slots - 1;
- free_list_t **free_list = small_mag_ptr->mag_free_list;
- free_list_t **the_slot = free_list + slot;
- free_list_t **limit;
- unsigned bitmap;
-
- // Assumes we've locked the magazine
- CHECK_MAGAZINE_PTR_LOCKED(szone, small_mag_ptr, __PRETTY_FUNCTION__);
-
- // Look for an exact match by checking the freelist for this msize.
- ptr = *the_slot;
- if (ptr)
- return SMALL_REGION_FOR_PTR(ptr);
-
- // Mask off the bits representing slots holding free blocks smaller than
- // the size we need.
- if (szone->is_largemem) {
- // BITMAPN_CTZ implementation
- unsigned idx = slot >> 5;
- bitmap = 0;
- unsigned mask = ~ ((1 << (slot & 31)) - 1);
- for ( ; idx < SMALL_BITMAP_WORDS; ++idx ) {
- bitmap = small_mag_ptr->mag_bitmap[idx] & mask;
- if (bitmap != 0)
- break;
- mask = ~0U;
- }
- // Check for fallthrough: No bits set in bitmap
- if ((bitmap == 0) && (idx == SMALL_BITMAP_WORDS))
- return NULL;
-
- // Start looking at the first set bit, plus 32 bits for every word of
- // zeroes or entries that were too small.
- slot = BITMAP32_CTZ((&bitmap)) + (idx * 32);
- } else {
- bitmap = small_mag_ptr->mag_bitmap[0] & ~ ((1 << slot) - 1);
- if (!bitmap)
- return NULL;
-
- slot = BITMAP32_CTZ((&bitmap));
- }
- limit = free_list + szone->num_small_slots - 1;
- free_list += slot;
-
- if (free_list < limit) {
- ptr = *free_list;
- if (ptr)
- return SMALL_REGION_FOR_PTR(ptr);
- else {
- /* Shouldn't happen. Fall through to look at last slot. */
-#if DEBUG_MALLOC
- malloc_printf("in small_malloc_from_free_list(), mag_bitmap out of sync, slot=%d\n",slot);
-#endif
- }
- }
-
- // We are now looking at the last slot, which contains blocks equal to, or
- // due to coalescing of free blocks, larger than (num_small_slots - 1) * (small quantum size).
- ptr = *limit;
- if (ptr)
- return SMALL_REGION_FOR_PTR(ptr);
-
- return NULL;
-}
-
-static boolean_t
-small_get_region_from_depot(szone_t *szone, magazine_t *small_mag_ptr, mag_index_t mag_index, msize_t msize)
-{
- magazine_t *depot_ptr = &(szone->small_magazines[DEPOT_MAGAZINE_INDEX]);
-
- /* FIXME: Would Uniprocessor benefit from recirc and MADV_FREE? */
- if (szone->num_small_magazines == 1) // Uniprocessor, single magazine, so no recirculation necessary
- return 0;
-
-#if DEBUG_MALLOC
- if (DEPOT_MAGAZINE_INDEX == mag_index) {
- szone_error(szone, 1, "small_get_region_from_depot called for magazine index -1", NULL, NULL);
- return 0;
- }
-#endif
-
- SZONE_MAGAZINE_PTR_LOCK(szone,depot_ptr);
-
- // Appropriate a Depot'd region that can satisfy requested msize.
- region_trailer_t *node;
- region_t sparse_region;
-
- while (1) {
- sparse_region = small_find_msize_region(szone, depot_ptr, DEPOT_MAGAZINE_INDEX, msize);
- if (NULL == sparse_region) { // Depot empty?
- SZONE_MAGAZINE_PTR_UNLOCK(szone,depot_ptr);
- return 0;
- }
-
- node = REGION_TRAILER_FOR_SMALL_REGION(sparse_region);
- if (0 >= node->pinned_to_depot)
- break;
-
- SZONE_MAGAZINE_PTR_UNLOCK(szone,depot_ptr);
- pthread_yield_np();
- SZONE_MAGAZINE_PTR_LOCK(szone,depot_ptr);
- }
-
- // disconnect node from Depot
- recirc_list_extract(szone, depot_ptr, node);
-
- // Iterate the region pulling its free entries off the (locked) Depot's free list
- int objects_in_use = small_free_detach_region(szone, depot_ptr, sparse_region);
-
- // Transfer ownership of the region
- MAGAZINE_INDEX_FOR_SMALL_REGION(sparse_region) = mag_index;
- node->pinned_to_depot = 0;
-
- // Iterate the region putting its free entries on its new (locked) magazine's free list
- size_t bytes_inplay = small_free_reattach_region(szone, small_mag_ptr, sparse_region);
-
- depot_ptr->mag_num_bytes_in_objects -= bytes_inplay;
- depot_ptr->num_bytes_in_magazine -= SMALL_REGION_PAYLOAD_BYTES;
- depot_ptr->mag_num_objects -= objects_in_use;
-
- small_mag_ptr->mag_num_bytes_in_objects += bytes_inplay;
- small_mag_ptr->num_bytes_in_magazine += SMALL_REGION_PAYLOAD_BYTES;
- small_mag_ptr->mag_num_objects += objects_in_use;
-
- // connect to magazine as first node
- recirc_list_splice_first(szone, small_mag_ptr, node);
-
- SZONE_MAGAZINE_PTR_UNLOCK(szone,depot_ptr);
-
- // madvise() outside the Depot lock
-#if TARGET_OS_EMBEDDED
- if (node->failedREUSE) {
-#else
- if (node->failedREUSE ||
- -1 == madvise((void *)sparse_region, SMALL_REGION_PAYLOAD_BYTES, MADV_FREE_REUSE)) {
-#endif
- /* -1 return: VM map entry change makes this unfit for reuse. Something evil lurks. */
-#if DEBUG_MADVISE
- szone_error(szone, 0, "small_get_region_from_depot madvise(..., MADV_FREE_REUSE) failed",
- sparse_region, "length=%d\n", SMALL_REGION_PAYLOAD_BYTES);
-#endif
- node->failedREUSE = TRUE;
- }
-
- MAGMALLOC_DEPOTREGION((void *)szone, (int)mag_index, (void *)sparse_region, SMALL_REGION_SIZE,
- (int)BYTES_USED_FOR_SMALL_REGION(sparse_region)); // DTrace USDT Probe
-
- return 1;
-}
-
-#define K 1.5 // headroom measured in number of 8Mb regions
-#define DENSITY_THRESHOLD(a) \
- ((a) - ((a) >> 2)) // "Emptiness" f = 0.25, so "Density" is (1 - f)*a. Generally: ((a) - ((a) >> -log2(f)))
-
-static INLINE boolean_t
-small_free_no_lock(szone_t *szone, magazine_t *small_mag_ptr, mag_index_t mag_index, region_t region, void *ptr, msize_t msize)
-{
- msize_t *meta_headers = SMALL_META_HEADER_FOR_PTR(ptr);
- unsigned index = SMALL_META_INDEX_FOR_PTR(ptr);
- void *original_ptr = ptr;
- size_t original_size = SMALL_BYTES_FOR_MSIZE(msize);
- unsigned char *next_block = ((unsigned char *)ptr + original_size);
- msize_t next_index = index + msize;
- msize_t previous_msize, next_msize;
- void *previous;
-
-#if DEBUG_MALLOC
- if (LOG(szone,ptr)) {
- malloc_printf("in small_free_no_lock(), ptr=%p, msize=%d\n", ptr, msize);
- }
- if (!msize) {
- szone_error(szone, 1, "trying to free small block that is too small", ptr,
- "in small_free_no_lock(), ptr=%p, msize=%d\n", ptr, msize);
- }
-#endif
-
- // We try to coalesce this block with the preceeding one
- if (index && (SMALL_PREVIOUS_MSIZE(ptr) <= index)) {
- previous_msize = SMALL_PREVIOUS_MSIZE(ptr);
- if (meta_headers[index - previous_msize] == (previous_msize | SMALL_IS_FREE)) {
- previous = (void *)((uintptr_t)ptr - SMALL_BYTES_FOR_MSIZE(previous_msize));
- // previous is really to be coalesced
-#if DEBUG_MALLOC
- if (LOG(szone, ptr) || LOG(szone,previous)) {
- malloc_printf("in small_free_no_lock(), coalesced backwards for %p previous=%p\n", ptr, previous);
- }
-#endif
- small_free_list_remove_ptr(szone, small_mag_ptr, previous, previous_msize);
- small_meta_header_set_middle(meta_headers, index);
- ptr = previous;
- msize += previous_msize;
- index -= previous_msize;
- }
- }
- // We try to coalesce with the next block
- if ((next_block < SMALL_REGION_END(region)) && (meta_headers[next_index] & SMALL_IS_FREE)) {
- // next block is free, we coalesce
- next_msize = meta_headers[next_index] & ~ SMALL_IS_FREE;
-#if DEBUG_MALLOC
- if (LOG(szone,ptr))
- malloc_printf("In small_free_no_lock(), for ptr=%p, msize=%d coalesced next block=%p next_msize=%d\n",
- ptr, msize, next_block, next_msize);
-#endif
- small_free_list_remove_ptr(szone, small_mag_ptr, next_block, next_msize);
- small_meta_header_set_middle(meta_headers, next_index);
- msize += next_msize;
- }
- if (szone->debug_flags & SCALABLE_MALLOC_DO_SCRIBBLE) {
- if (!msize) {
- szone_error(szone, 1, "incorrect size information - block header was damaged", ptr, NULL);
- } else {
- memset(ptr, 0x55, SMALL_BYTES_FOR_MSIZE(msize));
- }
- }
- small_free_list_add_ptr(szone, small_mag_ptr, ptr, msize);
- small_mag_ptr->mag_num_objects--;
- // we use original_size and not msize to avoid double counting the coalesced blocks
- small_mag_ptr->mag_num_bytes_in_objects -= original_size;
-
- // Update this region's bytes in use count
- region_trailer_t *node = REGION_TRAILER_FOR_SMALL_REGION(region);
- size_t bytes_used = node->bytes_used - original_size;
- node->bytes_used = bytes_used;
-
-#if !TARGET_OS_EMBEDDED // Always madvise for embedded platforms
- /* FIXME: Would Uniprocessor benefit from recirc and MADV_FREE? */
- if (szone->num_small_magazines == 1) { // Uniprocessor, single magazine, so no recirculation necessary
- /* NOTHING */
- } else if (DEPOT_MAGAZINE_INDEX != mag_index) {
- // Emptiness discriminant
- if (bytes_used < DENSITY_THRESHOLD(SMALL_REGION_PAYLOAD_BYTES)) {
- /* Region has crossed threshold from density to sparsity. Mark it "suitable" on the
- recirculation candidates list. */
- node->recirc_suitable = TRUE;
- } else {
- /* After this free, we've found the region is still dense, so it must have been even more so before
- the free. That implies the region is already correctly marked. Do nothing. */
- }
-
- // Has the entire magazine crossed the "emptiness threshold"? If so, transfer a region
- // from this magazine to the Depot. Choose a region that itself has crossed the emptiness threshold (i.e
- // is at least fraction "f" empty.) Such a region will be marked "suitable" on the recirculation list.
-
- size_t a = small_mag_ptr->num_bytes_in_magazine; // Total bytes allocated to this magazine
- size_t u = small_mag_ptr->mag_num_bytes_in_objects; // In use (malloc'd) from this magaqzine
-
- if (a - u > ((3 * SMALL_REGION_PAYLOAD_BYTES) / 2) && u < DENSITY_THRESHOLD(a)) {
- return small_free_do_recirc_to_depot(szone, small_mag_ptr, mag_index);
- }
-
- } else {
-#endif
- // Freed to Depot. N.B. Lock on small_magazines[DEPOT_MAGAZINE_INDEX] is already held
- // Calcuate the first page in the coalesced block that would be safe to mark MADV_FREE
- uintptr_t safe_ptr = (uintptr_t)ptr + sizeof(free_list_t) + sizeof(msize_t);
- uintptr_t round_safe = round_page(safe_ptr);
-
- // Calcuate the last page in the coalesced block that would be safe to mark MADV_FREE
- uintptr_t safe_extent = (uintptr_t)ptr + SMALL_BYTES_FOR_MSIZE(msize) - sizeof(msize_t);
- uintptr_t trunc_extent = trunc_page(safe_extent);
-
- // The newly freed block may complete a span of bytes that cover one or more pages. Mark the span with MADV_FREE.
- if (round_safe < trunc_extent) { // Safe area covers a page (perhaps many)
- uintptr_t lo = trunc_page((uintptr_t)original_ptr);
- uintptr_t hi = round_page((uintptr_t)original_ptr + original_size);
-
- small_free_list_remove_ptr(szone, small_mag_ptr, ptr, msize);
- small_meta_header_set_in_use(meta_headers, index, msize);
-
- OSAtomicIncrement32Barrier(&(node->pinned_to_depot));
- SZONE_MAGAZINE_PTR_UNLOCK(szone, small_mag_ptr);
-#if TARGET_OS_EMBEDDED
- madvise_free_range(szone, region, MAX(round_safe, lo), MIN(trunc_extent, hi), &szone->last_small_advise);
-#else
- madvise_free_range(szone, region, MAX(round_safe, lo), MIN(trunc_extent, hi));
-#endif
- SZONE_MAGAZINE_PTR_LOCK(szone, small_mag_ptr);
- OSAtomicDecrement32Barrier(&(node->pinned_to_depot));
-
- small_meta_header_set_is_free(meta_headers, index, msize);
- small_free_list_add_ptr(szone, small_mag_ptr, ptr, msize);
- }
-
-#if !TARGET_OS_EMBEDDED
- if (0 < bytes_used || 0 < node->pinned_to_depot) {
- /* Depot'd region is still live. Leave it in place on the Depot's recirculation list
- so as to avoid thrashing between the Depot's free list and a magazines's free list
- with detach_region/reattach_region */
- } else {
- /* Depot'd region is just now empty. Consider return to OS. */
- region_t r_dealloc = small_free_try_depot_unmap_no_lock(szone, small_mag_ptr, node);
- SZONE_MAGAZINE_PTR_UNLOCK(szone, small_mag_ptr);
- if (r_dealloc)
- deallocate_pages(szone, r_dealloc, SMALL_REGION_SIZE, 0);
- return FALSE; // Caller need not unlock
- }
- }
-#endif
-
- return TRUE; // Caller must do SZONE_MAGAZINE_PTR_UNLOCK(szone, small_mag_ptr)
-}
-
-// Allocates from the last region or a freshly allocated region
-static void *
-small_malloc_from_region_no_lock(szone_t *szone, magazine_t *small_mag_ptr, mag_index_t mag_index,
- msize_t msize, void *aligned_address)
-{
- void *ptr;
-
- // Before anything we transform the mag_bytes_free_at_end or mag_bytes_free_at_start - if any - to a regular free block
- /* FIXME: last_block needs to be coalesced with previous entry if free, <rdar://5462322> */
- if (small_mag_ptr->mag_bytes_free_at_end || small_mag_ptr->mag_bytes_free_at_start)
- small_finalize_region(szone, small_mag_ptr);
-
- // Here find the only place in smallville that (infrequently) takes the small_regions_lock.
- // Only one thread at a time should be permitted to assess the density of the hash
- // ring and adjust if needed.
- // Only one thread at a time should be permitted to insert its new region on
- // the hash ring.
- // It is safe for all other threads to read the hash ring (hashed_regions) and
- // the associated sizes (num_regions_allocated and num_small_regions).
-
- LOCK(szone->small_regions_lock);
- // Check to see if the hash ring of small regions needs to grow. Try to
- // avoid the hash ring becoming too dense.
- if (szone->small_region_generation->num_regions_allocated < (2 * szone->num_small_regions)) {
- region_t *new_regions;
- size_t new_size;
- size_t new_shift = szone->small_region_generation->num_regions_allocated_shift; // In/Out parameter
- new_regions = hash_regions_grow_no_lock(szone, szone->small_region_generation->hashed_regions,
- szone->small_region_generation->num_regions_allocated,
- &new_shift,
- &new_size);
- // Do not deallocate the current hashed_regions allocation since someone
- // may be iterating it. Instead, just leak it.
-
- // Prepare to advance to the "next generation" of the hash ring.
- szone->small_region_generation->nextgen->hashed_regions = new_regions;
- szone->small_region_generation->nextgen->num_regions_allocated = new_size;
- szone->small_region_generation->nextgen->num_regions_allocated_shift = new_shift;
-
- // Throw the switch to atomically advance to the next generation.
- szone->small_region_generation = szone->small_region_generation->nextgen;
- // Ensure everyone sees the advance.
- OSMemoryBarrier();
- }
- // Tag the region at "aligned_address" as belonging to us,
- // and so put it under the protection of the magazine lock we are holding.
- // Do this before advertising "aligned_address" on the hash ring(!)
- MAGAZINE_INDEX_FOR_SMALL_REGION(aligned_address) = mag_index;
-
- // Insert the new region into the hash ring, and update malloc statistics
- hash_region_insert_no_lock(szone->small_region_generation->hashed_regions,
- szone->small_region_generation->num_regions_allocated,
- szone->small_region_generation->num_regions_allocated_shift,
- aligned_address);
-
- szone->num_small_regions++;
-
- UNLOCK(szone->small_regions_lock);
-
- small_mag_ptr->mag_last_region = aligned_address;
- BYTES_USED_FOR_SMALL_REGION(aligned_address) = SMALL_BYTES_FOR_MSIZE(msize);
-#if ASLR_INTERNAL
- int offset_msize = malloc_entropy[1] & SMALL_ENTROPY_MASK;
-#if DEBUG_MALLOC
- if (getenv("MallocASLRForce")) offset_msize = strtol(getenv("MallocASLRForce"), NULL, 0) & SMALL_ENTROPY_MASK;
- if (getenv("MallocASLRPrint")) malloc_printf("Region: %p offset: %d\n", aligned_address, offset_msize);
-#endif
-#else
- int offset_msize = 0;
-#endif
- ptr = (void *)((uintptr_t) aligned_address + SMALL_BYTES_FOR_MSIZE(offset_msize));
- small_meta_header_set_in_use(SMALL_META_HEADER_FOR_PTR(ptr), offset_msize, msize);
- small_mag_ptr->mag_num_objects++;
- small_mag_ptr->mag_num_bytes_in_objects += SMALL_BYTES_FOR_MSIZE(msize);
- small_mag_ptr->num_bytes_in_magazine += SMALL_REGION_PAYLOAD_BYTES;
-
- // add a big free block at the end
- small_meta_header_set_in_use(SMALL_META_HEADER_FOR_PTR(ptr), offset_msize + msize, NUM_SMALL_BLOCKS - msize - offset_msize);
- small_mag_ptr->mag_bytes_free_at_end = SMALL_BYTES_FOR_MSIZE(NUM_SMALL_BLOCKS - msize - offset_msize);
-
-#if ASLR_INTERNAL
- // add a big free block at the start
- small_mag_ptr->mag_bytes_free_at_start = SMALL_BYTES_FOR_MSIZE(offset_msize);
- if (offset_msize) {
- small_meta_header_set_in_use(SMALL_META_HEADER_FOR_PTR(ptr), 0, offset_msize);
- }
-#else
- small_mag_ptr->mag_bytes_free_at_start = 0;
-#endif
-
- // connect to magazine as last node
- recirc_list_splice_last(szone, small_mag_ptr, REGION_TRAILER_FOR_SMALL_REGION(aligned_address));
-
- return ptr;
-}
-
-static INLINE void *
-small_try_shrink_in_place(szone_t *szone, void *ptr, size_t old_size, size_t new_good_size)
-{
- msize_t new_msize = SMALL_MSIZE_FOR_BYTES(new_good_size);
- msize_t mshrinkage = SMALL_MSIZE_FOR_BYTES(old_size) - new_msize;
-
- if (mshrinkage) {
- void *q = (void *)((uintptr_t)ptr + SMALL_BYTES_FOR_MSIZE(new_msize));
- magazine_t *small_mag_ptr = mag_lock_zine_for_region_trailer(szone, szone->small_magazines,
- REGION_TRAILER_FOR_SMALL_REGION(SMALL_REGION_FOR_PTR(ptr)),
- MAGAZINE_INDEX_FOR_SMALL_REGION(SMALL_REGION_FOR_PTR(ptr)));
-
- // Mark q as block header and in-use, thus creating two blocks.
- small_meta_header_set_in_use(SMALL_META_HEADER_FOR_PTR(ptr), SMALL_META_INDEX_FOR_PTR(ptr), new_msize);
- small_meta_header_set_in_use(SMALL_META_HEADER_FOR_PTR(q), SMALL_META_INDEX_FOR_PTR(q), mshrinkage);
- small_mag_ptr->mag_num_objects++;
-
- SZONE_MAGAZINE_PTR_UNLOCK(szone,small_mag_ptr);
- szone_free(szone, q); // avoid inlining free_small(szone, q, ...);
- }
-
- return ptr;
-}
-
-static INLINE boolean_t
-small_try_realloc_in_place(szone_t *szone, void *ptr, size_t old_size, size_t new_size)
-{
- // returns 1 on success
- msize_t *meta_headers = SMALL_META_HEADER_FOR_PTR(ptr);
- unsigned index;
- msize_t old_msize, new_msize;
- unsigned next_index;
- void *next_block;
- msize_t next_msize_and_free;
- boolean_t is_free;
- msize_t next_msize, leftover_msize;
- void *leftover;
-
- index = SMALL_META_INDEX_FOR_PTR(ptr);
- old_msize = SMALL_MSIZE_FOR_BYTES(old_size);
- new_msize = SMALL_MSIZE_FOR_BYTES(new_size + SMALL_QUANTUM - 1);
- next_index = index + old_msize;
-
- if (next_index >= NUM_SMALL_BLOCKS) {
- return 0;
- }
- next_block = (char *)ptr + old_size;
-
-#if DEBUG_MALLOC
- if ((uintptr_t)next_block & (SMALL_QUANTUM - 1)) {
- szone_error(szone, 1, "internal invariant broken in realloc(next_block)", next_block, NULL);
- }
- if (meta_headers[index] != old_msize)
- malloc_printf("*** small_try_realloc_in_place incorrect old %d %d\n",
- meta_headers[index], old_msize);
-#endif
-
- magazine_t *small_mag_ptr = mag_lock_zine_for_region_trailer(szone, szone->small_magazines,
- REGION_TRAILER_FOR_SMALL_REGION(SMALL_REGION_FOR_PTR(ptr)),
- MAGAZINE_INDEX_FOR_SMALL_REGION(SMALL_REGION_FOR_PTR(ptr)));
-
- /*
- * Look for a free block immediately afterwards. If it's large enough, we can consume (part of)
- * it.
- */
- next_msize_and_free = meta_headers[next_index];
- is_free = next_msize_and_free & SMALL_IS_FREE;
- if (!is_free) {
- SZONE_MAGAZINE_PTR_UNLOCK(szone, small_mag_ptr);
- return 0; // next_block is in use;
- }
- next_msize = next_msize_and_free & ~ SMALL_IS_FREE;
- if (old_msize + next_msize < new_msize) {
- SZONE_MAGAZINE_PTR_UNLOCK(szone, small_mag_ptr);
- return 0; // even with next block, not enough
- }
- /*
- * The following block is big enough; pull it from its freelist and chop off enough to satisfy
- * our needs.
- */
- small_free_list_remove_ptr(szone, small_mag_ptr, next_block, next_msize);
- small_meta_header_set_middle(meta_headers, next_index);
- leftover_msize = old_msize + next_msize - new_msize;
- if (leftover_msize) {
- /* there's some left, so put the remainder back */
- leftover = (unsigned char *)ptr + SMALL_BYTES_FOR_MSIZE(new_msize);
-
- small_free_list_add_ptr(szone, small_mag_ptr, leftover, leftover_msize);
- }
-#if DEBUG_MALLOC
- if (SMALL_BYTES_FOR_MSIZE(new_msize) > szone->large_threshold) {
- malloc_printf("*** realloc in place for %p exceeded msize=%d\n", new_msize);
- }
-#endif
- small_meta_header_set_in_use(meta_headers, index, new_msize);
-#if DEBUG_MALLOC
- if (LOG(szone,ptr)) {
- malloc_printf("in small_try_realloc_in_place(), ptr=%p, msize=%d\n", ptr, *SMALL_METADATA_FOR_PTR(ptr));
- }
-#endif
- small_mag_ptr->mag_num_bytes_in_objects += SMALL_BYTES_FOR_MSIZE(new_msize - old_msize);
-
- // Update this region's bytes in use count
- region_trailer_t *node = REGION_TRAILER_FOR_SMALL_REGION(SMALL_REGION_FOR_PTR(ptr));
- size_t bytes_used = node->bytes_used + SMALL_BYTES_FOR_MSIZE(new_msize - old_msize);
- node->bytes_used = bytes_used;
-
- // Emptiness discriminant
- if (bytes_used < DENSITY_THRESHOLD(SMALL_REGION_PAYLOAD_BYTES)) {
- /* After this reallocation the region is still sparse, so it must have been even more so before
- the reallocation. That implies the region is already correctly marked. Do nothing. */
- } else {
- /* Region has crossed threshold from sparsity to density. Mark it not "suitable" on the
- recirculation candidates list. */
- node->recirc_suitable = FALSE;
- }
-
- SZONE_MAGAZINE_PTR_UNLOCK(szone, small_mag_ptr);
- CHECK(szone, __PRETTY_FUNCTION__);
- return 1;
-}
-
-static boolean_t
-small_check_region(szone_t *szone, region_t region)
-{
- unsigned char *ptr = SMALL_REGION_ADDRESS(region);
- msize_t *meta_headers = SMALL_META_HEADER_FOR_PTR(ptr);
- unsigned char *region_end = SMALL_REGION_END(region);
- msize_t prev_free = 0;
- unsigned index;
- msize_t msize_and_free;
- msize_t msize;
- free_list_t *free_head;
- void *previous, *next;
- msize_t *follower;
- mag_index_t mag_index = MAGAZINE_INDEX_FOR_SMALL_REGION(SMALL_REGION_FOR_PTR(ptr));
- magazine_t *small_mag_ptr = &(szone->small_magazines[mag_index]);
-
- // Assumes locked
- CHECK_MAGAZINE_PTR_LOCKED(szone, small_mag_ptr, __PRETTY_FUNCTION__);
-
- if (region == small_mag_ptr->mag_last_region) {
- ptr += small_mag_ptr->mag_bytes_free_at_start;
- region_end -= small_mag_ptr->mag_bytes_free_at_end;
- }
-
- while (ptr < region_end) {
- index = SMALL_META_INDEX_FOR_PTR(ptr);
- msize_and_free = meta_headers[index];
- if (!(msize_and_free & SMALL_IS_FREE)) {
- // block is in use
- msize = msize_and_free;
- if (!msize) {
- malloc_printf("*** invariant broken: null msize ptr=%p num_small_regions=%d end=%p\n",
- ptr, szone->num_small_regions, region_end);
- return 0;
- }
-#if !RELAXED_INVARIANT_CHECKS
- if (SMALL_BYTES_FOR_MSIZE(msize) > szone->large_threshold) {
- malloc_printf("*** invariant broken for %p this small msize=%d - size is too large\n",
- ptr, msize_and_free);
- return 0;
- }
-#endif // RELAXED_INVARIANT_CHECKS
- ptr += SMALL_BYTES_FOR_MSIZE(msize);
- prev_free = 0;
- } else {
- // free pointer
- msize = msize_and_free & ~ SMALL_IS_FREE;
- free_head = (free_list_t *)ptr;
- follower = (msize_t *)FOLLOWING_SMALL_PTR(ptr, msize);
- if (!msize) {
- malloc_printf("*** invariant broken for free block %p this msize=%d\n", ptr, msize);
- return 0;
- }
-#if !RELAXED_INVARIANT_CHECKS
- if (prev_free) {
- malloc_printf("*** invariant broken for %p (2 free in a row)\n", ptr);
- return 0;
- }
-#endif
- previous = free_list_unchecksum_ptr(szone, &free_head->previous);
- next = free_list_unchecksum_ptr(szone, &free_head->next);
- if (previous && !SMALL_PTR_IS_FREE(previous)) {
- malloc_printf("*** invariant broken for %p (previous %p is not a free pointer)\n",
- ptr, free_head->previous);
- return 0;
- }
- if (next && !SMALL_PTR_IS_FREE(next)) {
- malloc_printf("*** invariant broken for %p (next is not a free pointer)\n", ptr);
- return 0;
- }
- if (SMALL_PREVIOUS_MSIZE(follower) != msize) {
- malloc_printf("*** invariant broken for small free %p followed by %p in region [%p-%p] "
- "(end marker incorrect) should be %d; in fact %d\n",
- ptr, follower, SMALL_REGION_ADDRESS(region), region_end, msize, SMALL_PREVIOUS_MSIZE(follower));
- return 0;
- }
- ptr = (unsigned char *)follower;
- prev_free = SMALL_IS_FREE;
- }
- }
- return 1;
-}
-
-static kern_return_t
-small_in_use_enumerator(task_t task, void *context, unsigned type_mask, szone_t *szone,
- memory_reader_t reader, vm_range_recorder_t recorder)
-{
- size_t num_regions;
- size_t index;
- region_t *regions;
- vm_range_t buffer[MAX_RECORDER_BUFFER];
- unsigned count = 0;
- kern_return_t err;
- region_t region;
- vm_range_t range;
- vm_range_t admin_range;
- vm_range_t ptr_range;
- unsigned char *mapped_region;
- msize_t *block_header;
- unsigned block_index;
- unsigned block_limit;
- msize_t msize_and_free;
- msize_t msize;
- magazine_t *small_mag_base = NULL;
-
- region_hash_generation_t *srg_ptr;
- err = reader(task, (vm_address_t)szone->small_region_generation, sizeof(region_hash_generation_t), (void **)&srg_ptr);
- if (err) return err;
-
- num_regions = srg_ptr->num_regions_allocated;
- err = reader(task, (vm_address_t)srg_ptr->hashed_regions, sizeof(region_t) * num_regions, (void **)®ions);
- if (err) return err;
-
- if (type_mask & MALLOC_PTR_IN_USE_RANGE_TYPE) {
- // Map in all active magazines. Do this outside the iteration over regions.
- err = reader(task, (vm_address_t)(szone->small_magazines),
- szone->num_small_magazines*sizeof(magazine_t),(void **)&small_mag_base);
- if (err) return err;
- }
-
- for (index = 0; index < num_regions; ++index) {
- region = regions[index];
- if (HASHRING_OPEN_ENTRY != region && HASHRING_REGION_DEALLOCATED != region) {
- range.address = (vm_address_t)SMALL_REGION_ADDRESS(region);
- range.size = SMALL_REGION_SIZE;
- if (type_mask & MALLOC_ADMIN_REGION_RANGE_TYPE) {
- admin_range.address = range.address + SMALL_METADATA_START;
- admin_range.size = SMALL_METADATA_SIZE;
- recorder(task, context, MALLOC_ADMIN_REGION_RANGE_TYPE, &admin_range, 1);
- }
- if (type_mask & (MALLOC_PTR_REGION_RANGE_TYPE | MALLOC_ADMIN_REGION_RANGE_TYPE)) {
- ptr_range.address = range.address;
- ptr_range.size = NUM_SMALL_BLOCKS * SMALL_QUANTUM;
- recorder(task, context, MALLOC_PTR_REGION_RANGE_TYPE, &ptr_range, 1);
- }
- if (type_mask & MALLOC_PTR_IN_USE_RANGE_TYPE) {
- void *mag_last_free;
- vm_address_t mag_last_free_ptr = 0;
- msize_t mag_last_free_msize = 0;
-
- err = reader(task, range.address, range.size, (void **)&mapped_region);
- if (err)
- return err;
-
- mag_index_t mag_index = MAGAZINE_INDEX_FOR_SMALL_REGION(mapped_region);
- magazine_t *small_mag_ptr = small_mag_base + mag_index;
-
- if (DEPOT_MAGAZINE_INDEX != mag_index) {
- mag_last_free = small_mag_ptr->mag_last_free;
- if (mag_last_free) {
- mag_last_free_ptr = (uintptr_t) mag_last_free & ~(SMALL_QUANTUM - 1);
- mag_last_free_msize = (uintptr_t) mag_last_free & (SMALL_QUANTUM - 1);
- }
- } else {
- for (mag_index = 0; mag_index < szone->num_small_magazines; mag_index++) {
- if ((void *)range.address == (small_mag_base + mag_index)->mag_last_free_rgn) {
- mag_last_free = (small_mag_base + mag_index)->mag_last_free;
- if (mag_last_free) {
- mag_last_free_ptr = (uintptr_t) mag_last_free & ~(SMALL_QUANTUM - 1);
- mag_last_free_msize = (uintptr_t) mag_last_free & (SMALL_QUANTUM - 1);
- }
- }
- }
- }
-
- block_header = (msize_t *)(mapped_region + SMALL_METADATA_START + sizeof(region_trailer_t));
- block_index = 0;
- block_limit = NUM_SMALL_BLOCKS;
- if (region == small_mag_ptr->mag_last_region) {
- block_index += SMALL_MSIZE_FOR_BYTES(small_mag_ptr->mag_bytes_free_at_start);
- block_limit -= SMALL_MSIZE_FOR_BYTES(small_mag_ptr->mag_bytes_free_at_end);
- }
- while (block_index < block_limit) {
- msize_and_free = block_header[block_index];
- msize = msize_and_free & ~ SMALL_IS_FREE;
- if (! (msize_and_free & SMALL_IS_FREE) &&
- range.address + SMALL_BYTES_FOR_MSIZE(block_index) != mag_last_free_ptr) {
- // Block in use
- buffer[count].address = range.address + SMALL_BYTES_FOR_MSIZE(block_index);
- buffer[count].size = SMALL_BYTES_FOR_MSIZE(msize);
- count++;
- if (count >= MAX_RECORDER_BUFFER) {
- recorder(task, context, MALLOC_PTR_IN_USE_RANGE_TYPE, buffer, count);
- count = 0;
- }
- }
-
- if (!msize)
- return KERN_FAILURE; // Somethings amiss. Avoid looping at this block_index.
-
- block_index += msize;
- }
- if (count) {
- recorder(task, context, MALLOC_PTR_IN_USE_RANGE_TYPE, buffer, count);
- count = 0;
- }
- }
- }
- }
- return 0;
-}
-
-static void *
-small_malloc_from_free_list(szone_t *szone, magazine_t *small_mag_ptr, mag_index_t mag_index, msize_t msize)
-{
- free_list_t *ptr;
- msize_t this_msize;
- grain_t slot = (msize <= szone->num_small_slots) ? msize - 1 : szone->num_small_slots - 1;
- free_list_t **free_list = small_mag_ptr->mag_free_list;
- free_list_t **the_slot = free_list + slot;
- free_list_t *next;
- free_list_t **limit;
- unsigned bitmap;
- msize_t leftover_msize;
- free_list_t *leftover_ptr;
-
- // Assumes we've locked the region
- CHECK_MAGAZINE_PTR_LOCKED(szone, small_mag_ptr, __PRETTY_FUNCTION__);
-
- // Look for an exact match by checking the freelist for this msize.
- //
- ptr = *the_slot;
- if (ptr) {
- next = free_list_unchecksum_ptr(szone, &ptr->next);
- if (next) {
- next->previous = ptr->previous;
- } else {
- BITMAPN_CLR(small_mag_ptr->mag_bitmap, slot);
- }
- *the_slot = next;
- this_msize = msize;
- goto return_small_alloc;
- }
-
- // Mask off the bits representing slots holding free blocks smaller than
- // the size we need. If there are no larger free blocks, try allocating
- // from the free space at the end of the small region.
- if (szone->is_largemem) {
- // BITMAPN_CTZ implementation
- unsigned idx = slot >> 5;
- bitmap = 0;
- unsigned mask = ~ ((1 << (slot & 31)) - 1);
- for ( ; idx < SMALL_BITMAP_WORDS; ++idx ) {
- bitmap = small_mag_ptr->mag_bitmap[idx] & mask;
- if (bitmap != 0)
- break;
- mask = ~0U;
- }
- // Check for fallthrough: No bits set in bitmap
- if ((bitmap == 0) && (idx == SMALL_BITMAP_WORDS))
- goto try_small_from_end;
-
- // Start looking at the first set bit, plus 32 bits for every word of
- // zeroes or entries that were too small.
- slot = BITMAP32_CTZ((&bitmap)) + (idx * 32);
- } else {
- bitmap = small_mag_ptr->mag_bitmap[0] & ~ ((1 << slot) - 1);
- if (!bitmap)
- goto try_small_from_end;
-
- slot = BITMAP32_CTZ((&bitmap));
- }
- // FIXME: Explain use of - 1 here, last slot has special meaning
- limit = free_list + szone->num_small_slots - 1;
- free_list += slot;
-
- if (free_list < limit) {
- ptr = *free_list;
- if (ptr) {
-
- next = free_list_unchecksum_ptr(szone, &ptr->next);
- *free_list = next;
- if (next) {
- next->previous = ptr->previous;
- } else {
- BITMAPN_CLR(small_mag_ptr->mag_bitmap, slot);
- }
- this_msize = SMALL_PTR_SIZE(ptr);
- goto add_leftover_and_proceed;
- }
-#if DEBUG_MALLOC
- malloc_printf("in small_malloc_from_free_list(), mag_bitmap out of sync, slot=%d\n",slot);
-#endif
- }
-
- // We are now looking at the last slot, which contains blocks equal to, or
- // due to coalescing of free blocks, larger than (num_small_slots - 1) * (small quantum size).
- // If the last freelist is not empty, and the head contains a block that is
- // larger than our request, then the remainder is put back on the free list.
- //
- ptr = *limit;
- if (ptr) {
- this_msize = SMALL_PTR_SIZE(ptr);
- next = free_list_unchecksum_ptr(szone, &ptr->next);
- if (this_msize - msize >= szone->num_small_slots) {
- // the leftover will go back to the free list, so we optimize by
- // modifying the free list rather than a pop and push of the head
- leftover_msize = this_msize - msize;
- leftover_ptr = (free_list_t *)((unsigned char *)ptr + SMALL_BYTES_FOR_MSIZE(msize));
- *limit = leftover_ptr;
- if (next) {
- next->previous.u = free_list_checksum_ptr(szone, leftover_ptr);
- }
- leftover_ptr->previous = ptr->previous;
- leftover_ptr->next = ptr->next;
- small_meta_header_set_is_free(SMALL_META_HEADER_FOR_PTR(leftover_ptr),
- SMALL_META_INDEX_FOR_PTR(leftover_ptr), leftover_msize);
- // Store msize at the end of the block denoted by "leftover_ptr" (i.e. at a negative offset from follower)
- SMALL_PREVIOUS_MSIZE(FOLLOWING_SMALL_PTR(leftover_ptr, leftover_msize)) = leftover_msize; // Access is safe
-#if DEBUG_MALLOC
- if (LOG(szone,ptr)) {
- malloc_printf("in small_malloc_from_free_list(), last slot ptr=%p, msize=%d this_msize=%d\n", ptr, msize, this_msize);
- }
-#endif
- this_msize = msize;
- goto return_small_alloc;
- }
- if (next) {
- next->previous = ptr->previous;
- }
- *limit = next;
- goto add_leftover_and_proceed;
- }
-
-try_small_from_end:
- // Let's see if we can use small_mag_ptr->mag_bytes_free_at_end
- if (small_mag_ptr->mag_bytes_free_at_end >= SMALL_BYTES_FOR_MSIZE(msize)) {
- ptr = (free_list_t *)(SMALL_REGION_END(small_mag_ptr->mag_last_region) -
- small_mag_ptr->mag_bytes_free_at_end);
- small_mag_ptr->mag_bytes_free_at_end -= SMALL_BYTES_FOR_MSIZE(msize);
- if (small_mag_ptr->mag_bytes_free_at_end) {
- // let's mark this block as in use to serve as boundary
- small_meta_header_set_in_use(SMALL_META_HEADER_FOR_PTR(ptr),
- SMALL_META_INDEX_FOR_PTR((unsigned char *)ptr + SMALL_BYTES_FOR_MSIZE(msize)),
- SMALL_MSIZE_FOR_BYTES(small_mag_ptr->mag_bytes_free_at_end));
- }
- this_msize = msize;
- goto return_small_alloc;
- }
-#if ASLR_INTERNAL
- // Try from start if nothing left at end
- if (small_mag_ptr->mag_bytes_free_at_start >= SMALL_BYTES_FOR_MSIZE(msize)) {
- ptr = (free_list_t *)(SMALL_REGION_ADDRESS(small_mag_ptr->mag_last_region) +
- small_mag_ptr->mag_bytes_free_at_start - SMALL_BYTES_FOR_MSIZE(msize));
- small_mag_ptr->mag_bytes_free_at_start -= SMALL_BYTES_FOR_MSIZE(msize);
- if (small_mag_ptr->mag_bytes_free_at_start) {
- // let's mark this block as in use to serve as boundary
- small_meta_header_set_in_use(SMALL_META_HEADER_FOR_PTR(ptr), 0, SMALL_MSIZE_FOR_BYTES(small_mag_ptr->mag_bytes_free_at_start));
- }
- this_msize = msize;
- goto return_small_alloc;
- }
-#endif
- return NULL;
-
-add_leftover_and_proceed:
- if (this_msize > msize) {
- leftover_msize = this_msize - msize;
- leftover_ptr = (free_list_t *)((unsigned char *)ptr + SMALL_BYTES_FOR_MSIZE(msize));
-#if DEBUG_MALLOC
- if (LOG(szone,ptr)) {
- malloc_printf("in small_malloc_from_free_list(), adding leftover ptr=%p, this_msize=%d\n", ptr, this_msize);
- }
-#endif
- small_free_list_add_ptr(szone, small_mag_ptr, leftover_ptr, leftover_msize);
- this_msize = msize;
- }
-
-return_small_alloc:
- small_mag_ptr->mag_num_objects++;
- small_mag_ptr->mag_num_bytes_in_objects += SMALL_BYTES_FOR_MSIZE(this_msize);
-
- // Update this region's bytes in use count
- region_trailer_t *node = REGION_TRAILER_FOR_SMALL_REGION(SMALL_REGION_FOR_PTR(ptr));
- size_t bytes_used = node->bytes_used + SMALL_BYTES_FOR_MSIZE(this_msize);
- node->bytes_used = bytes_used;
-
- // Emptiness discriminant
- if (bytes_used < DENSITY_THRESHOLD(SMALL_REGION_PAYLOAD_BYTES)) {
- /* After this allocation the region is still sparse, so it must have been even more so before
- the allocation. That implies the region is already correctly marked. Do nothing. */
- } else {
- /* Region has crossed threshold from sparsity to density. Mark in not "suitable" on the
- recirculation candidates list. */
- node->recirc_suitable = FALSE;
- }
-#if DEBUG_MALLOC
- if (LOG(szone,ptr)) {
- malloc_printf("in small_malloc_from_free_list(), ptr=%p, this_msize=%d, msize=%d\n", ptr, this_msize, msize);
- }
-#endif
- small_meta_header_set_in_use(SMALL_META_HEADER_FOR_PTR(ptr), SMALL_META_INDEX_FOR_PTR(ptr), this_msize);
- return ptr;
-}
-#undef DENSITY_THRESHOLD
-#undef K
-
-static INLINE void *
-small_malloc_should_clear(szone_t *szone, msize_t msize, boolean_t cleared_requested)
-{
- void *ptr;
- mag_index_t mag_index = mag_get_thread_index(szone);
- magazine_t *small_mag_ptr = &(szone->small_magazines[mag_index]);
-
- SZONE_MAGAZINE_PTR_LOCK(szone, small_mag_ptr);
-
-#if SMALL_CACHE
- ptr = (void *)small_mag_ptr->mag_last_free;
-
- if ((((uintptr_t)ptr) & (SMALL_QUANTUM - 1)) == msize) {
- // we have a winner
- small_mag_ptr->mag_last_free = NULL;
- small_mag_ptr->mag_last_free_rgn = NULL;
- SZONE_MAGAZINE_PTR_UNLOCK(szone, small_mag_ptr);
- CHECK(szone, __PRETTY_FUNCTION__);
- ptr = (void *)((uintptr_t)ptr & ~ (SMALL_QUANTUM - 1));
- if (cleared_requested) {
- memset(ptr, 0, SMALL_BYTES_FOR_MSIZE(msize));
- }
- return ptr;
- }
-#endif /* SMALL_CACHE */
-
- while(1) {
- ptr = small_malloc_from_free_list(szone, small_mag_ptr, mag_index, msize);
- if (ptr) {
- SZONE_MAGAZINE_PTR_UNLOCK(szone, small_mag_ptr);
- CHECK(szone, __PRETTY_FUNCTION__);
- if (cleared_requested) {
- memset(ptr, 0, SMALL_BYTES_FOR_MSIZE(msize));
- }
- return ptr;
- }
-
- if (small_get_region_from_depot(szone, small_mag_ptr, mag_index, msize)) {
- ptr = small_malloc_from_free_list(szone, small_mag_ptr, mag_index, msize);
- if (ptr) {
- SZONE_MAGAZINE_PTR_UNLOCK(szone, small_mag_ptr);
- CHECK(szone, __PRETTY_FUNCTION__);
- if (cleared_requested) {
- memset(ptr, 0, SMALL_BYTES_FOR_MSIZE(msize));
- }
- return ptr;
- }
- }
-
- // The magazine is exhausted. A new region (heap) must be allocated to satisfy this call to malloc().
- // The allocation, an mmap() system call, will be performed outside the magazine spin locks by the first
- // thread that suffers the exhaustion. That thread sets "alloc_underway" and enters a critical section.
- // Threads arriving here later are excluded from the critical section, yield the CPU, and then retry the
- // allocation. After some time the magazine is resupplied, the original thread leaves with its allocation,
- // and retry-ing threads succeed in the code just above.
- if (!small_mag_ptr->alloc_underway) {
- void *fresh_region;
-
- // time to create a new region (do this outside the magazine lock)
- small_mag_ptr->alloc_underway = TRUE;
- OSMemoryBarrier();
- SZONE_MAGAZINE_PTR_UNLOCK(szone, small_mag_ptr);
- fresh_region = allocate_pages_securely(szone, SMALL_REGION_SIZE, SMALL_BLOCKS_ALIGN, VM_MEMORY_MALLOC_SMALL);
- SZONE_MAGAZINE_PTR_LOCK(szone, small_mag_ptr);
-
- MAGMALLOC_ALLOCREGION((void *)szone, (int)mag_index, fresh_region, SMALL_REGION_SIZE); // DTrace USDT Probe
-
- if (!fresh_region) { // out of memory!
- small_mag_ptr->alloc_underway = FALSE;
- OSMemoryBarrier();
- SZONE_MAGAZINE_PTR_UNLOCK(szone, small_mag_ptr);
- return NULL;
- }
-
- ptr = small_malloc_from_region_no_lock(szone, small_mag_ptr, mag_index, msize, fresh_region);
-
- // we don't clear because this freshly allocated space is pristine
- small_mag_ptr->alloc_underway = FALSE;
- OSMemoryBarrier();
- SZONE_MAGAZINE_PTR_UNLOCK(szone, small_mag_ptr);
- CHECK(szone, __PRETTY_FUNCTION__);
- return ptr;
- } else {
- SZONE_MAGAZINE_PTR_UNLOCK(szone, small_mag_ptr);
- pthread_yield_np();
- SZONE_MAGAZINE_PTR_LOCK(szone, small_mag_ptr);
- }
- }
- /* NOTREACHED */
-}
-
-static NOINLINE void
-free_small_botch(szone_t *szone, free_list_t *ptr)
-{
- mag_index_t mag_index = MAGAZINE_INDEX_FOR_SMALL_REGION(SMALL_REGION_FOR_PTR(ptr));
- magazine_t *small_mag_ptr = &(szone->small_magazines[mag_index]);
- SZONE_MAGAZINE_PTR_UNLOCK(szone, small_mag_ptr);
- szone_error(szone, 1, "double free", ptr, NULL);
-}
-
-static INLINE void
-free_small(szone_t *szone, void *ptr, region_t small_region, size_t known_size)
-{
- msize_t msize;
- mag_index_t mag_index = MAGAZINE_INDEX_FOR_SMALL_REGION(SMALL_REGION_FOR_PTR(ptr));
- magazine_t *small_mag_ptr = &(szone->small_magazines[mag_index]);
-
- // ptr is known to be in small_region
- if (known_size) {
- msize = SMALL_MSIZE_FOR_BYTES(known_size + SMALL_QUANTUM - 1);
- } else {
- msize = SMALL_PTR_SIZE(ptr);
- if (SMALL_PTR_IS_FREE(ptr)) {
- free_small_botch(szone, ptr);
- return;
- }
- }
-
- SZONE_MAGAZINE_PTR_LOCK(szone, small_mag_ptr);
-
-#if SMALL_CACHE
- // Depot does not participate in SMALL_CACHE since it can't be directly malloc()'d
- if (DEPOT_MAGAZINE_INDEX != mag_index) {
-
- void *ptr2 = small_mag_ptr->mag_last_free; // Might be NULL
- region_t rgn2 = small_mag_ptr->mag_last_free_rgn;
-
- /* check that we don't already have this pointer in the cache */
- if (ptr == (void *)((uintptr_t)ptr2 & ~ (SMALL_QUANTUM - 1))) {
- free_small_botch(szone, ptr);
- return;
- }
-
- if ((szone->debug_flags & SCALABLE_MALLOC_DO_SCRIBBLE) && msize)
- memset(ptr, 0x55, SMALL_BYTES_FOR_MSIZE(msize));
-
- small_mag_ptr->mag_last_free = (void *)(((uintptr_t)ptr) | msize);
- small_mag_ptr->mag_last_free_rgn = small_region;
-
- if (!ptr2) {
- SZONE_MAGAZINE_PTR_UNLOCK(szone, small_mag_ptr);
- CHECK(szone, __PRETTY_FUNCTION__);
- return;
- }
-
- msize = (uintptr_t)ptr2 & (SMALL_QUANTUM - 1);
- ptr = (void *)(((uintptr_t)ptr2) & ~(SMALL_QUANTUM - 1));
- small_region = rgn2;
- }
-#endif /* SMALL_CACHE */
-
- // Now in the time it took to acquire the lock, the region may have migrated
- // from one magazine to another. I.e. trailer->mag_index is volatile.
- // In which case the magazine lock we obtained (namely magazines[mag_index].mag_lock)
- // is stale. If so, keep on tryin' ...
- region_trailer_t *trailer = REGION_TRAILER_FOR_SMALL_REGION(small_region);
- mag_index_t refreshed_index;
-
- while (mag_index != (refreshed_index = trailer->mag_index)) { // Note assignment
-
- SZONE_MAGAZINE_PTR_UNLOCK(szone, small_mag_ptr);
-
- mag_index = refreshed_index;
- small_mag_ptr = &(szone->small_magazines[mag_index]);
- SZONE_MAGAZINE_PTR_LOCK(szone, small_mag_ptr);
- }
-
- if (small_free_no_lock(szone, small_mag_ptr, mag_index, small_region, ptr, msize))
- SZONE_MAGAZINE_PTR_UNLOCK(szone, small_mag_ptr);
-
- CHECK(szone, __PRETTY_FUNCTION__);
-}
-
-static void
-print_small_free_list(szone_t *szone)
-{
- free_list_t *ptr;
- _SIMPLE_STRING b = _simple_salloc();
- mag_index_t mag_index;
-
- if (b) {
- _simple_sappend(b, "small free sizes:\n");
- for (mag_index = -1; mag_index < szone->num_small_magazines; mag_index++) {
- grain_t slot = 0;
- _simple_sprintf(b,"\tMagazine %d: ", mag_index);
- while (slot < szone->num_small_slots) {
- ptr = szone->small_magazines[mag_index].mag_free_list[slot];
- if (ptr) {
- _simple_sprintf(b, "%s%y[%d]; ", (slot == szone->num_small_slots-1) ? ">=" : "",
- (slot + 1) * SMALL_QUANTUM, free_list_count(szone, ptr));
- }
- slot++;
- }
- _simple_sappend(b,"\n");
- }
- _malloc_printf(MALLOC_PRINTF_NOLOG | MALLOC_PRINTF_NOPREFIX, "%s\n", _simple_string(b));
- _simple_sfree(b);
- }
-}
-
-static void
-print_small_region(szone_t *szone, boolean_t verbose, region_t region, size_t bytes_at_start, size_t bytes_at_end)
-{
- unsigned counts[1024];
- unsigned in_use = 0;
- uintptr_t start = (uintptr_t)SMALL_REGION_ADDRESS(region);
- uintptr_t current = start + bytes_at_start;
- uintptr_t limit = (uintptr_t)SMALL_REGION_END(region) - bytes_at_end;
- msize_t msize_and_free;
- msize_t msize;
- unsigned ci;
- _SIMPLE_STRING b;
- uintptr_t pgTot = 0;
-
- if (region == HASHRING_REGION_DEALLOCATED) {
- if ((b = _simple_salloc()) != NULL) {
- _simple_sprintf(b, "Small region [unknown address] was returned to the OS\n");
- _malloc_printf(MALLOC_PRINTF_NOLOG | MALLOC_PRINTF_NOPREFIX, "%s\n", _simple_string(b));
- _simple_sfree(b);
- }
- return;
- }
-
- memset(counts, 0, sizeof(counts));
- while (current < limit) {
- msize_and_free = *SMALL_METADATA_FOR_PTR(current);
- msize = msize_and_free & ~ SMALL_IS_FREE;
- if (!msize) {
- malloc_printf("*** error with %p: msize=%d\n", (void *)current, (unsigned)msize);
- break;
- }
- if (!(msize_and_free & SMALL_IS_FREE)) {
- // block in use
- if (msize < 1024)
- counts[msize]++;
- in_use++;
- } else {
- uintptr_t pgLo = round_page(current + sizeof(free_list_t) + sizeof(msize_t));
- uintptr_t pgHi = trunc_page(current + SMALL_BYTES_FOR_MSIZE(msize) - sizeof(msize_t));
-
- if (pgLo < pgHi) {
- pgTot += (pgHi - pgLo);
- }
- }
- current += SMALL_BYTES_FOR_MSIZE(msize);
- }
- if ((b = _simple_salloc()) != NULL) {
- _simple_sprintf(b, "Small region [%p-%p, %y] \t", (void *)start, SMALL_REGION_END(region), (int)SMALL_REGION_SIZE);
- _simple_sprintf(b, "Magazine=%d \t", MAGAZINE_INDEX_FOR_SMALL_REGION(region));
- _simple_sprintf(b, "Allocations in use=%d \t Bytes in use=%ly \t", in_use, BYTES_USED_FOR_SMALL_REGION(region));
- if (bytes_at_end || bytes_at_start)
- _simple_sprintf(b, "Untouched=%ly ", bytes_at_end + bytes_at_start);
- if (DEPOT_MAGAZINE_INDEX == MAGAZINE_INDEX_FOR_SMALL_REGION(region)) {
- _simple_sprintf(b, "Advised MADV_FREE=%ly", pgTot);
- } else {
- _simple_sprintf(b, "Fragments subject to reclamation=%ly", pgTot);
- }
- if (verbose && in_use) {
- _simple_sappend(b, "\n\tSizes in use: ");
- for (ci = 0; ci < 1024; ci++)
- if (counts[ci])
- _simple_sprintf(b, "%d[%d] ", SMALL_BYTES_FOR_MSIZE(ci), counts[ci]);
- }
- _malloc_printf(MALLOC_PRINTF_NOLOG | MALLOC_PRINTF_NOPREFIX, "%s\n", _simple_string(b));
- _simple_sfree(b);
- }
-}
-
-static boolean_t
-small_free_list_check(szone_t *szone, grain_t slot)
-{
- mag_index_t mag_index;
-
- for (mag_index = -1; mag_index < szone->num_small_magazines; mag_index++) {
- magazine_t *small_mag_ptr = &(szone->small_magazines[mag_index]);
- SZONE_MAGAZINE_PTR_LOCK(szone, small_mag_ptr);
-
- unsigned count = 0;
- free_list_t *ptr = szone->small_magazines[mag_index].mag_free_list[slot];
- msize_t msize_and_free;
- free_list_t *previous = NULL;
-
- while (ptr) {
- msize_and_free = *SMALL_METADATA_FOR_PTR(ptr);
- if (!(msize_and_free & SMALL_IS_FREE)) {
- malloc_printf("*** in-use ptr in free list slot=%d count=%d ptr=%p\n", slot, count, ptr);
- SZONE_MAGAZINE_PTR_UNLOCK(szone, small_mag_ptr);
- return 0;
- }
- if (((uintptr_t)ptr) & (SMALL_QUANTUM - 1)) {
- malloc_printf("*** unaligned ptr in free list slot=%d count=%d ptr=%p\n", slot, count, ptr);
- SZONE_MAGAZINE_PTR_UNLOCK(szone, small_mag_ptr);
- return 0;
- }
- if (!small_region_for_ptr_no_lock(szone, ptr)) {
- malloc_printf("*** ptr not in szone slot=%d count=%d ptr=%p\n", slot, count, ptr);
- SZONE_MAGAZINE_PTR_UNLOCK(szone, small_mag_ptr);
- return 0;
- }
- if (free_list_unchecksum_ptr(szone, &ptr->previous) != previous) {
- malloc_printf("*** previous incorrectly set slot=%d count=%d ptr=%p\n", slot, count, ptr);
- SZONE_MAGAZINE_PTR_UNLOCK(szone, small_mag_ptr);
- return 0;
- }
- previous = ptr;
- ptr = free_list_unchecksum_ptr(szone, &ptr->next);
- count++;
- }
-
- SZONE_MAGAZINE_PTR_UNLOCK(szone, small_mag_ptr);
- }
- return 1;
-}
-
-/*******************************************************************************
- * Large allocator implementation
- ******************************************************************************/
-#pragma mark large allocator
-
-#if DEBUG_MALLOC
-
-static void
-large_debug_print(szone_t *szone)
-{
- unsigned index;
- large_entry_t *range;
- _SIMPLE_STRING b = _simple_salloc();
-
- if (b) {
- for (index = 0, range = szone->large_entries; index < szone->num_large_entries; index++, range++)
- if (range->address)
- _simple_sprintf(b, "%d: %p(%y); ", index, range->address, range->size);
-
- _malloc_printf(MALLOC_PRINTF_NOLOG | MALLOC_PRINTF_NOPREFIX, "%s\n", _simple_string(b));
- _simple_sfree(b);
- }
-}
-#endif
-
-/*
- * Scan the hash ring looking for an entry for the given pointer.
- */
-static large_entry_t *
-large_entry_for_pointer_no_lock(szone_t *szone, const void *ptr)
-{
- // result only valid with lock held
- unsigned num_large_entries = szone->num_large_entries;
- unsigned hash_index;
- unsigned index;
- large_entry_t *range;
-
- if (!num_large_entries)
- return NULL;
-
- hash_index = ((uintptr_t)ptr >> vm_page_shift) % num_large_entries;
- index = hash_index;
-
- do {
- range = szone->large_entries + index;
- if (range->address == (vm_address_t)ptr)
- return range;
- if (0 == range->address)
- return NULL; // end of chain
- index++;
- if (index == num_large_entries)
- index = 0;
- } while (index != hash_index);
-
- return NULL;
-}
-
-static void
-large_entry_insert_no_lock(szone_t *szone, large_entry_t range)
-{
- unsigned num_large_entries = szone->num_large_entries;
- unsigned hash_index = (((uintptr_t)(range.address)) >> vm_page_shift) % num_large_entries;
- unsigned index = hash_index;
- large_entry_t *entry;
-
- // assert(szone->num_large_objects_in_use < szone->num_large_entries); /* must be called with room to spare */
-
- do {
- entry = szone->large_entries + index;
- if (0 == entry->address) {
- *entry = range;
- return; // end of chain
- }
- index++;
- if (index == num_large_entries)
- index = 0;
- } while (index != hash_index);
-
- // assert(0); /* must not fallthrough! */
-}
-
-// FIXME: can't we simply swap the (now empty) entry with the last entry on the collision chain for this hash slot?
-static INLINE void
-large_entries_rehash_after_entry_no_lock(szone_t *szone, large_entry_t *entry)
-{
- unsigned num_large_entries = szone->num_large_entries;
- unsigned hash_index = entry - szone->large_entries;
- unsigned index = hash_index;
- large_entry_t range;
-
- // assert(entry->address == 0) /* caller must have cleared *entry */
-
- do {
- index++;
- if (index == num_large_entries)
- index = 0;
- range = szone->large_entries[index];
- if (0 == range.address)
- return;
- szone->large_entries[index].address = (vm_address_t)0;
- szone->large_entries[index].size = 0;
- szone->large_entries[index].did_madvise_reusable = FALSE;
- large_entry_insert_no_lock(szone, range); // this will reinsert in the
- // proper place
- } while (index != hash_index);
-
- // assert(0); /* since entry->address == 0, must not fallthrough! */
-}
-
-// FIXME: num should probably be a size_t, since you can theoretically allocate
-// more than 2^32-1 large_threshold objects in 64 bit.
-static INLINE large_entry_t *
-large_entries_alloc_no_lock(szone_t *szone, unsigned num)
-{
- size_t size = num * sizeof(large_entry_t);
-
- // Note that we allocate memory (via a system call) under a spin lock
- // That is certainly evil, however it's very rare in the lifetime of a process
- // The alternative would slow down the normal case
- return allocate_pages(szone, round_page(size), 0, 0, VM_MEMORY_MALLOC_LARGE);
-}
-
-static void
-large_entries_free_no_lock(szone_t *szone, large_entry_t *entries, unsigned num, vm_range_t *range_to_deallocate)
-{
- size_t size = num * sizeof(large_entry_t);
-
- range_to_deallocate->address = (vm_address_t)entries;
- range_to_deallocate->size = round_page(size);
-}
-
-static large_entry_t *
-large_entries_grow_no_lock(szone_t *szone, vm_range_t *range_to_deallocate)
-{
- // sets range_to_deallocate
- unsigned old_num_entries = szone->num_large_entries;
- large_entry_t *old_entries = szone->large_entries;
- // always an odd number for good hashing
- unsigned new_num_entries = (old_num_entries) ? old_num_entries * 2 + 1 :
- ((vm_page_size / sizeof(large_entry_t)) - 1);
- large_entry_t *new_entries = large_entries_alloc_no_lock(szone, new_num_entries);
- unsigned index = old_num_entries;
- large_entry_t oldRange;
-
- // if the allocation of new entries failed, bail
- if (new_entries == NULL)
- return NULL;
-
- szone->num_large_entries = new_num_entries;
- szone->large_entries = new_entries;
-
- /* rehash entries into the new list */
- while (index--) {
- oldRange = old_entries[index];
- if (oldRange.address) {
- large_entry_insert_no_lock(szone, oldRange);
- }
- }
-
- if (old_entries) {
- large_entries_free_no_lock(szone, old_entries, old_num_entries, range_to_deallocate);
- } else {
- range_to_deallocate->address = (vm_address_t)0;
- range_to_deallocate->size = 0;
- }
-
- return new_entries;
-}
-
-// frees the specific entry in the size table
-// returns a range to truly deallocate
-static vm_range_t
-large_entry_free_no_lock(szone_t *szone, large_entry_t *entry)
-{
- vm_range_t range;
-
- range.address = entry->address;
- range.size = entry->size;
-
- if (szone->debug_flags & SCALABLE_MALLOC_ADD_GUARD_PAGES) {
- protect((void *)range.address, range.size, PROT_READ | PROT_WRITE, szone->debug_flags);
- range.address -= vm_page_size;
- range.size += 2 * vm_page_size;
- }
-
- entry->address = 0;
- entry->size = 0;
- entry->did_madvise_reusable = FALSE;
- large_entries_rehash_after_entry_no_lock(szone, entry);
-
-#if DEBUG_MALLOC
- if (large_entry_for_pointer_no_lock(szone, (void *)range.address)) {
- malloc_printf("*** freed entry %p still in use; num_large_entries=%d\n",
- range.address, szone->num_large_entries);
- large_debug_print(szone);
- szone_sleep();
- }
-#endif
- return range;
-}
-
-static NOINLINE kern_return_t
-large_in_use_enumerator(task_t task, void *context, unsigned type_mask, vm_address_t large_entries_address,
- unsigned num_entries, memory_reader_t reader, vm_range_recorder_t recorder)
-{
- unsigned index = 0;
- vm_range_t buffer[MAX_RECORDER_BUFFER];
- unsigned count = 0;
- large_entry_t *entries;
- kern_return_t err;
- vm_range_t range;
- large_entry_t entry;
-
- err = reader(task, large_entries_address, sizeof(large_entry_t) * num_entries, (void **)&entries);
- if (err)
- return err;
-
- index = num_entries;
- if (type_mask & MALLOC_ADMIN_REGION_RANGE_TYPE) {
- range.address = large_entries_address;
- range.size = round_page(num_entries * sizeof(large_entry_t));
- recorder(task, context, MALLOC_ADMIN_REGION_RANGE_TYPE, &range, 1);
- }
- if (type_mask & (MALLOC_PTR_IN_USE_RANGE_TYPE | MALLOC_PTR_REGION_RANGE_TYPE)) {
- while (index--) {
- entry = entries[index];
- if (entry.address) {
- range.address = entry.address;
- range.size = entry.size;
- buffer[count++] = range;
- if (count >= MAX_RECORDER_BUFFER) {
- recorder(task, context, MALLOC_PTR_IN_USE_RANGE_TYPE | MALLOC_PTR_REGION_RANGE_TYPE,
- buffer, count);
- count = 0;
- }
- }
- }
- }
- if (count) {
- recorder(task, context, MALLOC_PTR_IN_USE_RANGE_TYPE | MALLOC_PTR_REGION_RANGE_TYPE,
- buffer, count);
- }
- return 0;
-}
-
-static void *
-large_malloc(szone_t *szone, size_t num_pages, unsigned char alignment,
- boolean_t cleared_requested)
-{
- void *addr;
- vm_range_t range_to_deallocate;
- size_t size;
- large_entry_t large_entry;
-
- if (!num_pages)
- num_pages = 1; // minimal allocation size for this szone
- size = (size_t)num_pages << vm_page_shift;
- range_to_deallocate.size = 0;
- range_to_deallocate.address = 0;
-
-#if LARGE_CACHE
- if (size < LARGE_CACHE_SIZE_ENTRY_LIMIT) { // Look for a large_entry_t on the death-row cache?
- SZONE_LOCK(szone);
-
- int i, best = -1, idx = szone->large_entry_cache_newest, stop_idx = szone->large_entry_cache_oldest;
- size_t best_size = SIZE_T_MAX;
-
- while (1) { // Scan large_entry_cache for best fit, starting with most recent entry
- size_t this_size = szone->large_entry_cache[idx].size;
- addr = (void *)szone->large_entry_cache[idx].address;
-
- if (0 == alignment || 0 == (((uintptr_t) addr) & (((uintptr_t) 1 << alignment) - 1))) {
- if (size == this_size) { // size match!
- best = idx;
- best_size = this_size;
- break;
- }
-
- if (size <= this_size && this_size < best_size) { // improved fit?
- best = idx;
- best_size = this_size;
- }
- }
-
- if (idx == stop_idx) // exhausted live ring?
- break;
-
- if (idx)
- idx--; // bump idx down
- else
- idx = LARGE_ENTRY_CACHE_SIZE - 1; // wrap idx
- }
-
- if (best > -1 && (best_size - size) < size) { //limit fragmentation to 50%
- addr = (void *)szone->large_entry_cache[best].address;
- boolean_t was_madvised_reusable = szone->large_entry_cache[best].did_madvise_reusable;
-
- // Compact live ring to fill entry now vacated at large_entry_cache[best]
- // while preserving time-order
- if (szone->large_entry_cache_oldest < szone->large_entry_cache_newest) {
-
- // Ring hasn't wrapped. Fill in from right.
- for (i = best; i < szone->large_entry_cache_newest; ++i)
- szone->large_entry_cache[i] = szone->large_entry_cache[i + 1];
-
- szone->large_entry_cache_newest--; // Pull in right endpoint.
-
- } else if (szone->large_entry_cache_newest < szone->large_entry_cache_oldest) {
-
- // Ring has wrapped. Arrange to fill in from the contiguous side.
- if (best <= szone->large_entry_cache_newest) {
- // Fill from right.
- for (i = best; i < szone->large_entry_cache_newest; ++i)
- szone->large_entry_cache[i] = szone->large_entry_cache[i + 1];
-
- if (0 < szone->large_entry_cache_newest)
- szone->large_entry_cache_newest--;
- else
- szone->large_entry_cache_newest = LARGE_ENTRY_CACHE_SIZE - 1;
- } else {
- // Fill from left.
- for ( i = best; i > szone->large_entry_cache_oldest; --i)
- szone->large_entry_cache[i] = szone->large_entry_cache[i - 1];
-
- if (szone->large_entry_cache_oldest < LARGE_ENTRY_CACHE_SIZE - 1)
- szone->large_entry_cache_oldest++;
- else
- szone->large_entry_cache_oldest = 0;
- }
-
- } else {
- // By trichotomy, large_entry_cache_newest == large_entry_cache_oldest.
- // That implies best == large_entry_cache_newest == large_entry_cache_oldest
- // and the ring is now empty.
- szone->large_entry_cache[best].address = 0;
- szone->large_entry_cache[best].size = 0;
- szone->large_entry_cache[best].did_madvise_reusable = FALSE;
- }
-
- if ((szone->num_large_objects_in_use + 1) * 4 > szone->num_large_entries) {
- // density of hash table too high; grow table
- // we do that under lock to avoid a race
- large_entry_t *entries = large_entries_grow_no_lock(szone, &range_to_deallocate);
- if (entries == NULL) {
- SZONE_UNLOCK(szone);
- return NULL;
- }
- }
-
- large_entry.address = (vm_address_t)addr;
- large_entry.size = best_size;
- large_entry.did_madvise_reusable = FALSE;
- large_entry_insert_no_lock(szone, large_entry);
-
- szone->num_large_objects_in_use ++;
- szone->num_bytes_in_large_objects += best_size;
- if (!was_madvised_reusable)
- szone->large_entry_cache_reserve_bytes -= best_size;
-
- szone->large_entry_cache_bytes -= best_size;
-
- if (szone->flotsam_enabled && szone->large_entry_cache_bytes < SZONE_FLOTSAM_THRESHOLD_LOW) {
- szone->flotsam_enabled = FALSE;
- }
-
- SZONE_UNLOCK(szone);
-
- if (range_to_deallocate.size) {
- // we deallocate outside the lock
- deallocate_pages(szone, (void *)range_to_deallocate.address, range_to_deallocate.size, 0);
- }
-
- // Perform the madvise() outside the lock.
- // Typically the madvise() is successful and we'll quickly return from this routine.
- // In the unusual case of failure, reacquire the lock to unwind.
-#if TARGET_OS_EMBEDDED
- // Ok to do this madvise on embedded because we won't call MADV_FREE_REUSABLE on a large
- // cache block twice without MADV_FREE_REUSE in between.
-#endif
- if (was_madvised_reusable && -1 == madvise(addr, size, MADV_FREE_REUSE)) {
- /* -1 return: VM map entry change makes this unfit for reuse. */
-#if DEBUG_MADVISE
- szone_error(szone, 0, "large_malloc madvise(..., MADV_FREE_REUSE) failed",
- addr, "length=%d\n", size);
-#endif
-
- SZONE_LOCK(szone);
- szone->num_large_objects_in_use--;
- szone->num_bytes_in_large_objects -= large_entry.size;
-
- // Re-acquire "entry" after interval just above where we let go the lock.
- large_entry_t *entry = large_entry_for_pointer_no_lock(szone, addr);
- if (NULL == entry) {
- szone_error(szone, 1, "entry for pointer being discarded from death-row vanished", addr, NULL);
- SZONE_UNLOCK(szone);
- } else {
-
- range_to_deallocate = large_entry_free_no_lock(szone, entry);
- SZONE_UNLOCK(szone);
-
- if (range_to_deallocate.size) {
- // we deallocate outside the lock
- deallocate_pages(szone, (void *)range_to_deallocate.address, range_to_deallocate.size, 0);
- }
- }
- /* Fall through to allocate_pages() afresh. */
- } else {
- if (cleared_requested) {
- memset(addr, 0, size);
- }
-
- return addr;
- }
- } else {
- SZONE_UNLOCK(szone);
- }
- }
-
- range_to_deallocate.size = 0;
- range_to_deallocate.address = 0;
-#endif /* LARGE_CACHE */
-
- addr = allocate_pages(szone, size, alignment, szone->debug_flags, VM_MEMORY_MALLOC_LARGE);
- if (addr == NULL) {
- return NULL;
- }
-
- SZONE_LOCK(szone);
- if ((szone->num_large_objects_in_use + 1) * 4 > szone->num_large_entries) {
- // density of hash table too high; grow table
- // we do that under lock to avoid a race
- large_entry_t *entries = large_entries_grow_no_lock(szone, &range_to_deallocate);
- if (entries == NULL) {
- SZONE_UNLOCK(szone);
- return NULL;
- }
- }
-
- large_entry.address = (vm_address_t)addr;
- large_entry.size = size;
- large_entry.did_madvise_reusable = FALSE;
- large_entry_insert_no_lock(szone, large_entry);
-
- szone->num_large_objects_in_use ++;
- szone->num_bytes_in_large_objects += size;
- SZONE_UNLOCK(szone);
-
- if (range_to_deallocate.size) {
- // we deallocate outside the lock
- deallocate_pages(szone, (void *)range_to_deallocate.address, range_to_deallocate.size, 0);
- }
- return addr;
-}
-
-static NOINLINE void
-free_large(szone_t *szone, void *ptr)
-{
- // We have established ptr is page-aligned and neither tiny nor small
- large_entry_t *entry;
- vm_range_t vm_range_to_deallocate;
-
- SZONE_LOCK(szone);
- entry = large_entry_for_pointer_no_lock(szone, ptr);
- if (entry) {
-#if LARGE_CACHE
-#ifndef MADV_CAN_REUSE
-#define MADV_CAN_REUSE 9 /* per Francois, for testing until xnu is resubmitted to B&I */
-#endif
- if (entry->size < LARGE_CACHE_SIZE_ENTRY_LIMIT &&
- -1 != madvise((void *)(entry->address), entry->size, MADV_CAN_REUSE)) { // Put the large_entry_t on the death-row cache?
- int idx = szone->large_entry_cache_newest, stop_idx = szone->large_entry_cache_oldest;
- large_entry_t this_entry = *entry; // Make a local copy, "entry" is volatile when lock is let go.
- boolean_t reusable = TRUE;
- boolean_t should_madvise = szone->large_entry_cache_reserve_bytes + this_entry.size > szone->large_entry_cache_reserve_limit;
-
- // Already freed?
- // [Note that repeated entries in death-row risk vending the same entry subsequently
- // to two different malloc() calls. By checking here the (illegal) double free
- // is accommodated, matching the behavior of the previous implementation.]
- while (1) { // Scan large_entry_cache starting with most recent entry
- if (szone->large_entry_cache[idx].address == entry->address) {
- szone_error(szone, 1, "pointer being freed already on death-row", ptr, NULL);
- SZONE_UNLOCK(szone);
- return;
- }
-
- if (idx == stop_idx) // exhausted live ring?
- break;
-
- if (idx)
- idx--; // bump idx down
- else
- idx = LARGE_ENTRY_CACHE_SIZE - 1; // wrap idx
- }
-
- SZONE_UNLOCK(szone);
-
- if (szone->debug_flags & SCALABLE_MALLOC_PURGEABLE) { // Are we a purgable zone?
- int state = VM_PURGABLE_NONVOLATILE; // restore to default condition
-
- if (KERN_SUCCESS != vm_purgable_control(mach_task_self(), this_entry.address, VM_PURGABLE_SET_STATE, &state)) {
- malloc_printf("*** can't vm_purgable_control(..., VM_PURGABLE_SET_STATE) for large freed block at %p\n",
- this_entry.address);
- reusable = FALSE;
- }
- }
-
- if (szone->large_legacy_reset_mprotect) { // Linked for Leopard?
- // Accomodate Leopard apps that (illegally) mprotect() their own guard pages on large malloc'd allocations
- int err = mprotect((void *)(this_entry.address), this_entry.size, PROT_READ | PROT_WRITE);
- if (err) {
- malloc_printf("*** can't reset protection for large freed block at %p\n", this_entry.address);
- reusable = FALSE;
- }
- }
-
- // madvise(..., MADV_REUSABLE) death-row arrivals if hoarding would exceed large_entry_cache_reserve_limit
- if (should_madvise) {
- // Issue madvise to avoid paging out the dirtied free()'d pages in "entry"
- MAGMALLOC_MADVFREEREGION((void *)szone, (void *)0, (void *)(this_entry.address), this_entry.size); // DTrace USDT Probe
-
-#if TARGET_OS_EMBEDDED
- // Ok to do this madvise on embedded because we won't call MADV_FREE_REUSABLE on a large
- // cache block twice without MADV_FREE_REUSE in between.
-#endif
- if (-1 == madvise((void *)(this_entry.address), this_entry.size, MADV_FREE_REUSABLE)) {
- /* -1 return: VM map entry change makes this unfit for reuse. */
-#if DEBUG_MADVISE
- szone_error(szone, 0, "free_large madvise(..., MADV_FREE_REUSABLE) failed",
- (void *)this_entry.address, "length=%d\n", this_entry.size);
-#endif
- reusable = FALSE;
- }
- }
-
- SZONE_LOCK(szone);
-
- // Re-acquire "entry" after interval just above where we let go the lock.
- entry = large_entry_for_pointer_no_lock(szone, ptr);
- if (NULL == entry) {
- szone_error(szone, 1, "entry for pointer being freed from death-row vanished", ptr, NULL);
- SZONE_UNLOCK(szone);
- return;
- }
-
- // Add "entry" to death-row ring
- if (reusable) {
- int idx = szone->large_entry_cache_newest; // Most recently occupied
- vm_address_t addr;
- size_t adjsize;
-
- if (szone->large_entry_cache_newest == szone->large_entry_cache_oldest &&
- 0 == szone->large_entry_cache[idx].address) {
- // Ring is empty, idx is good as it stands
- addr = 0;
- adjsize = 0;
- } else {
- // Extend the queue to the "right" by bumping up large_entry_cache_newest
- if (idx == LARGE_ENTRY_CACHE_SIZE - 1)
- idx = 0; // Wrap index
- else
- idx++; // Bump index
-
- if (idx == szone->large_entry_cache_oldest) { // Fully occupied
- // Drop this entry from the cache and deallocate the VM
- addr = szone->large_entry_cache[idx].address;
- adjsize = szone->large_entry_cache[idx].size;
- szone->large_entry_cache_bytes -= adjsize;
- if (!szone->large_entry_cache[idx].did_madvise_reusable)
- szone->large_entry_cache_reserve_bytes -= adjsize;
- } else {
- // Using an unoccupied cache slot
- addr = 0;
- adjsize = 0;
- }
- }
-
- if ((szone->debug_flags & SCALABLE_MALLOC_DO_SCRIBBLE))
- memset((void *)(entry->address), 0x55, entry->size);
-
- entry->did_madvise_reusable = should_madvise; // Was madvise()'d above?
- if (!should_madvise) // Entered on death-row without madvise() => up the hoard total
- szone->large_entry_cache_reserve_bytes += entry->size;
-
- szone->large_entry_cache_bytes += entry->size;
-
- if (!szone->flotsam_enabled && szone->large_entry_cache_bytes > SZONE_FLOTSAM_THRESHOLD_HIGH) {
- szone->flotsam_enabled = TRUE;
- }
-
- szone->large_entry_cache[idx] = *entry;
- szone->large_entry_cache_newest = idx;
-
- szone->num_large_objects_in_use--;
- szone->num_bytes_in_large_objects -= entry->size;
-
- (void)large_entry_free_no_lock(szone, entry);
-
- if (0 == addr) {
- SZONE_UNLOCK(szone);
- return;
- }
-
- // Fall through to drop large_entry_cache_oldest from the cache,
- // and then deallocate its pages.
-
- // Trim the queue on the "left" by bumping up large_entry_cache_oldest
- if (szone->large_entry_cache_oldest == LARGE_ENTRY_CACHE_SIZE - 1)
- szone->large_entry_cache_oldest = 0;
- else
- szone->large_entry_cache_oldest++;
-
- // we deallocate_pages, including guard pages, outside the lock
- SZONE_UNLOCK(szone);
- deallocate_pages(szone, (void *)addr, (size_t)adjsize, 0);
- return;
- } else {
- /* fall through to discard an allocation that is not reusable */
- }
- }
-#endif /* LARGE_CACHE */
-
- szone->num_large_objects_in_use--;
- szone->num_bytes_in_large_objects -= entry->size;
-
- vm_range_to_deallocate = large_entry_free_no_lock(szone, entry);
- } else {
-#if DEBUG_MALLOC
- large_debug_print(szone);
-#endif
- szone_error(szone, 1, "pointer being freed was not allocated", ptr, NULL);
- SZONE_UNLOCK(szone);
- return;
- }
- SZONE_UNLOCK(szone); // we release the lock asap
- CHECK(szone, __PRETTY_FUNCTION__);
-
- // we deallocate_pages, including guard pages, outside the lock
- if (vm_range_to_deallocate.address) {
-#if DEBUG_MALLOC
- // FIXME: large_entry_for_pointer_no_lock() needs the lock held ...
- if (large_entry_for_pointer_no_lock(szone, (void *)vm_range_to_deallocate.address)) {
- malloc_printf("*** invariant broken: %p still in use num_large_entries=%d\n",
- vm_range_to_deallocate.address, szone->num_large_entries);
- large_debug_print(szone);
- szone_sleep();
- }
-#endif
- deallocate_pages(szone, (void *)vm_range_to_deallocate.address, (size_t)vm_range_to_deallocate.size, 0);
- }
-}
-
-static INLINE void *
-large_try_shrink_in_place(szone_t *szone, void *ptr, size_t old_size, size_t new_good_size)
-{
- size_t shrinkage = old_size - new_good_size;
-
- if (shrinkage) {
- SZONE_LOCK(szone);
- /* contract existing large entry */
- large_entry_t *large_entry = large_entry_for_pointer_no_lock(szone, ptr);
- if (!large_entry) {
- szone_error(szone, 1, "large entry reallocated is not properly in table", ptr, NULL);
- SZONE_UNLOCK(szone);
- return ptr;
- }
-
- large_entry->address = (vm_address_t)ptr;
- large_entry->size = new_good_size;
- szone->num_bytes_in_large_objects -= shrinkage;
- SZONE_UNLOCK(szone); // we release the lock asap
-
- deallocate_pages(szone, (void *)((uintptr_t)ptr + new_good_size), shrinkage, 0);
- }
- return ptr;
-}
-
-static INLINE int
-large_try_realloc_in_place(szone_t *szone, void *ptr, size_t old_size, size_t new_size)
-{
- vm_address_t addr = (vm_address_t)ptr + old_size;
- large_entry_t *large_entry;
- kern_return_t err;
-
- SZONE_LOCK(szone);
- large_entry = large_entry_for_pointer_no_lock(szone, (void *)addr);
- SZONE_UNLOCK(szone);
-
- if (large_entry) { // check if "addr = ptr + old_size" is already spoken for
- return 0; // large pointer already exists in table - extension is not going to work
- }
-
- new_size = round_page(new_size);
- /*
- * Ask for allocation at a specific address, and mark as realloc
- * to request coalescing with previous realloc'ed extensions.
- */
- err = vm_allocate(mach_task_self(), &addr, new_size - old_size, VM_MAKE_TAG(VM_MEMORY_REALLOC));
- if (err != KERN_SUCCESS) {
- return 0;
- }
-
- SZONE_LOCK(szone);
- /* extend existing large entry */
- large_entry = large_entry_for_pointer_no_lock(szone, ptr);
- if (!large_entry) {
- szone_error(szone, 1, "large entry reallocated is not properly in table", ptr, NULL);
- SZONE_UNLOCK(szone);
- return 0; // Bail, leaking "addr"
- }
-
- large_entry->address = (vm_address_t)ptr;
- large_entry->size = new_size;
- szone->num_bytes_in_large_objects += new_size - old_size;
- SZONE_UNLOCK(szone); // we release the lock asap
-
- return 1;
-}
-
-/********************* Zone call backs ************************/
-/*
- * Mark these NOINLINE to avoid bloating the purgeable zone call backs
- */
-static NOINLINE void
-szone_free(szone_t *szone, void *ptr)
-{
- region_t tiny_region;
- region_t small_region;
-
-#if DEBUG_MALLOC
- if (LOG(szone, ptr))
- malloc_printf("in szone_free with %p\n", ptr);
-#endif
- if (!ptr)
- return;
- /*
- * Try to free to a tiny region.
- */
- if ((uintptr_t)ptr & (TINY_QUANTUM - 1)) {
- szone_error(szone, 1, "Non-aligned pointer being freed", ptr, NULL);
- return;
- }
- if ((tiny_region = tiny_region_for_ptr_no_lock(szone, ptr)) != NULL) {
- if (TINY_INDEX_FOR_PTR(ptr) >= NUM_TINY_BLOCKS) {
- szone_error(szone, 1, "Pointer to metadata being freed", ptr, NULL);
- return;
- }
- free_tiny(szone, ptr, tiny_region, 0);
- return;
- }
-
- /*
- * Try to free to a small region.
- */
- if ((uintptr_t)ptr & (SMALL_QUANTUM - 1)) {
- szone_error(szone, 1, "Non-aligned pointer being freed (2)", ptr, NULL);
- return;
- }
- if ((small_region = small_region_for_ptr_no_lock(szone, ptr)) != NULL) {
- if (SMALL_META_INDEX_FOR_PTR(ptr) >= NUM_SMALL_BLOCKS) {
- szone_error(szone, 1, "Pointer to metadata being freed (2)", ptr, NULL);
- return;
- }
- free_small(szone, ptr, small_region, 0);
- return;
- }
-
- /* check that it's a legal large allocation */
- if ((uintptr_t)ptr & (vm_page_size - 1)) {
- szone_error(szone, 1, "non-page-aligned, non-allocated pointer being freed", ptr, NULL);
- return;
- }
- free_large(szone, ptr);
-}
-
-static NOINLINE void
-szone_free_definite_size(szone_t *szone, void *ptr, size_t size)
-{
-#if DEBUG_MALLOC
- if (LOG(szone, ptr))
- malloc_printf("in szone_free_definite_size with %p\n", ptr);
-
- if (0 == size) {
- szone_error(szone, 1, "pointer of size zero being freed", ptr, NULL);
- return;
- }
-
-#endif
- if (!ptr)
- return;
-
- /*
- * Try to free to a tiny region.
- */
- if ((uintptr_t)ptr & (TINY_QUANTUM - 1)) {
- szone_error(szone, 1, "Non-aligned pointer being freed", ptr, NULL);
- return;
- }
- if (size <= (NUM_TINY_SLOTS - 1)*TINY_QUANTUM) {
- if (TINY_INDEX_FOR_PTR(ptr) >= NUM_TINY_BLOCKS) {
- szone_error(szone, 1, "Pointer to metadata being freed", ptr, NULL);
- return;
- }
- free_tiny(szone, ptr, TINY_REGION_FOR_PTR(ptr), size);
- return;
- }
-
- /*
- * Try to free to a small region.
- */
- if ((uintptr_t)ptr & (SMALL_QUANTUM - 1)) {
- szone_error(szone, 1, "Non-aligned pointer being freed (2)", ptr, NULL);
- return;
- }
- if (size <= szone->large_threshold) {
- if (SMALL_META_INDEX_FOR_PTR(ptr) >= NUM_SMALL_BLOCKS) {
- szone_error(szone, 1, "Pointer to metadata being freed (2)", ptr, NULL);
- return;
- }
- free_small(szone, ptr, SMALL_REGION_FOR_PTR(ptr), size);
- return;
- }
-
- /* check that it's a legal large allocation */
- if ((uintptr_t)ptr & (vm_page_size - 1)) {
- szone_error(szone, 1, "non-page-aligned, non-allocated pointer being freed", ptr, NULL);
- return;
- }
- free_large(szone, ptr);
-}
-
-static NOINLINE void *
-szone_malloc_should_clear(szone_t *szone, size_t size, boolean_t cleared_requested)
-{
- void *ptr;
- msize_t msize;
-
- if (size <= (NUM_TINY_SLOTS - 1)*TINY_QUANTUM) {
- // think tiny
- msize = TINY_MSIZE_FOR_BYTES(size + TINY_QUANTUM - 1);
- if (!msize)
- msize = 1;
- ptr = tiny_malloc_should_clear(szone, msize, cleared_requested);
- } else if (size <= szone->large_threshold) {
- // think small
- msize = SMALL_MSIZE_FOR_BYTES(size + SMALL_QUANTUM - 1);
- if (!msize)
- msize = 1;
- ptr = small_malloc_should_clear(szone, msize, cleared_requested);
- } else {
- // large
- size_t num_pages = round_page(size) >> vm_page_shift;
- if (num_pages == 0) /* Overflowed */
- ptr = 0;
- else
- ptr = large_malloc(szone, num_pages, 0, cleared_requested);
- }
-#if DEBUG_MALLOC
- if (LOG(szone, ptr))
- malloc_printf("szone_malloc returned %p\n", ptr);
-#endif
- /*
- * If requested, scribble on allocated memory.
- */
- if ((szone->debug_flags & SCALABLE_MALLOC_DO_SCRIBBLE) && ptr && !cleared_requested && size)
- memset(ptr, 0xaa, size);
-
- return ptr;
-}
-
-static NOINLINE void *
-szone_malloc(szone_t *szone, size_t size) {
- return szone_malloc_should_clear(szone, size, 0);
-}
-
-static NOINLINE void *
-szone_calloc(szone_t *szone, size_t num_items, size_t size)
-{
- size_t total_bytes = num_items * size;
-
- // Check for overflow of integer multiplication
- if (num_items > 1) {
-#if __LP64__ /* size_t is uint64_t */
- if ((num_items | size) & 0xffffffff00000000ul) {
- // num_items or size equals or exceeds sqrt(2^64) == 2^32, appeal to wider arithmetic
- __uint128_t product = ((__uint128_t)num_items) * ((__uint128_t)size);
- if ((uint64_t)(product >> 64)) // compiles to test on upper register of register pair
- return NULL;
- }
-#else /* size_t is uint32_t */
- if ((num_items | size) & 0xffff0000ul) {
- // num_items or size equals or exceeds sqrt(2^32) == 2^16, appeal to wider arithmetic
- uint64_t product = ((uint64_t)num_items) * ((uint64_t)size);
- if ((uint32_t)(product >> 32)) // compiles to test on upper register of register pair
- return NULL;
- }
-#endif
- }
-
- return szone_malloc_should_clear(szone, total_bytes, 1);
-}
-
-static NOINLINE void *
-szone_valloc(szone_t *szone, size_t size)
-{
- void *ptr;
-
- if (size <= szone->large_threshold) {
- ptr = szone_memalign(szone, vm_page_size, size);
- } else {
- size_t num_pages;
-
- num_pages = round_page(size) >> vm_page_shift;
- ptr = large_malloc(szone, num_pages, 0, 0);
- }
-
-#if DEBUG_MALLOC
- if (LOG(szone, ptr))
- malloc_printf("szone_valloc returned %p\n", ptr);
-#endif
- return ptr;
-}
-
-/* Isolate PIC-base load (for __is_threaded) here. */
-static NOINLINE size_t
-szone_size_try_large(szone_t *szone, const void *ptr)
-{
- size_t size = 0;
- large_entry_t *entry;
-
- SZONE_LOCK(szone);
- entry = large_entry_for_pointer_no_lock(szone, ptr);
- if (entry) {
- size = entry->size;
- }
- SZONE_UNLOCK(szone);
-#if DEBUG_MALLOC
- if (LOG(szone, ptr)) {
- malloc_printf("szone_size for %p returned %d\n", ptr, (unsigned)size);
- }
-#endif
- return size;
-}
-
-static NOINLINE size_t
-szone_size(szone_t *szone, const void *ptr)
-{
- boolean_t is_free;
- msize_t msize, msize_and_free;
-
- if (!ptr)
- return 0;
-#if DEBUG_MALLOC
- if (LOG(szone, ptr)) {
- malloc_printf("in szone_size for %p (szone=%p)\n", ptr, szone);
- }
-#endif
-
- /*
- * Look for it in a tiny region.
- */
- if ((uintptr_t)ptr & (TINY_QUANTUM - 1))
- return 0;
- if (tiny_region_for_ptr_no_lock(szone, ptr)) {
- if (TINY_INDEX_FOR_PTR(ptr) >= NUM_TINY_BLOCKS)
- return 0;
- msize = get_tiny_meta_header(ptr, &is_free);
- if (is_free)
- return 0;
-#if TINY_CACHE
- {
- mag_index_t mag_index = MAGAZINE_INDEX_FOR_TINY_REGION(TINY_REGION_FOR_PTR(ptr));
- if (DEPOT_MAGAZINE_INDEX != mag_index) {
- magazine_t *tiny_mag_ptr = &(szone->tiny_magazines[mag_index]);
-
- if (msize < TINY_QUANTUM && ptr == (void *)((uintptr_t)(tiny_mag_ptr->mag_last_free) & ~ (TINY_QUANTUM - 1)))
- return 0;
- } else {
- for (mag_index = 0; mag_index < szone->num_tiny_magazines; mag_index++) {
- magazine_t *tiny_mag_ptr = &(szone->tiny_magazines[mag_index]);
-
- if (msize < TINY_QUANTUM && ptr == (void *)((uintptr_t)(tiny_mag_ptr->mag_last_free) & ~ (TINY_QUANTUM - 1)))
- return 0;
- }
- }
- }
-#endif
- return TINY_BYTES_FOR_MSIZE(msize);
- }
-
- /*
- * Look for it in a small region.
- */
- if ((uintptr_t)ptr & (SMALL_QUANTUM - 1))
- return 0;
- if (small_region_for_ptr_no_lock(szone, ptr)) {
- if (SMALL_META_INDEX_FOR_PTR(ptr) >= NUM_SMALL_BLOCKS)
- return 0;
- msize_and_free = *SMALL_METADATA_FOR_PTR(ptr);
- if (msize_and_free & SMALL_IS_FREE)
- return 0;
-#if SMALL_CACHE
- {
- mag_index_t mag_index = MAGAZINE_INDEX_FOR_SMALL_REGION(SMALL_REGION_FOR_PTR(ptr));
- if (DEPOT_MAGAZINE_INDEX != mag_index) {
- magazine_t *small_mag_ptr = &(szone->small_magazines[mag_index]);
-
- if (ptr == (void *)((uintptr_t)(small_mag_ptr->mag_last_free) & ~ (SMALL_QUANTUM - 1)))
- return 0;
- } else {
- for (mag_index = 0; mag_index < szone->num_small_magazines; mag_index++) {
- magazine_t *small_mag_ptr = &(szone->small_magazines[mag_index]);
-
- if (ptr == (void *)((uintptr_t)(small_mag_ptr->mag_last_free) & ~ (SMALL_QUANTUM - 1)))
- return 0;
- }
- }
- }
-#endif
- return SMALL_BYTES_FOR_MSIZE(msize_and_free);
- }
-
- /*
- * If not page-aligned, it cannot have come from a large allocation.
- */
- if ((uintptr_t)ptr & (vm_page_size - 1))
- return 0;
-
- /*
- * Look for it in a large entry.
- */
- return szone_size_try_large(szone, ptr);
-}
-
-static NOINLINE void *
-szone_realloc(szone_t *szone, void *ptr, size_t new_size)
-{
- size_t old_size, new_good_size, valid_size;
- void *new_ptr;
-
-#if DEBUG_MALLOC
- if (LOG(szone, ptr)) {
- malloc_printf("in szone_realloc for %p, %d\n", ptr, (unsigned)new_size);
- }
-#endif
- if (NULL == ptr) {
- // If ptr is a null pointer, realloc() shall be equivalent to malloc() for the specified size.
- return szone_malloc(szone, new_size);
- } else if (0 == new_size) {
- // If size is 0 and ptr is not a null pointer, the object pointed to is freed.
- szone_free(szone, ptr);
- // If size is 0, either a null pointer or a unique pointer that can be successfully passed
- // to free() shall be returned.
- return szone_malloc(szone, 1);
- }
-
- old_size = szone_size(szone, ptr);
- if (!old_size) {
- szone_error(szone, 1, "pointer being reallocated was not allocated", ptr, NULL);
- return NULL;
- }
-
- new_good_size = szone_good_size(szone, new_size);
- if (new_good_size == old_size) { // Existing allocation is best fit evar?
- return ptr;
- }
-
- /*
- * If the new size suits the tiny allocator and the pointer being resized
- * belongs to a tiny region, try to reallocate in-place.
- */
- if (new_good_size <= (NUM_TINY_SLOTS - 1) * TINY_QUANTUM) {
- if (old_size <= (NUM_TINY_SLOTS - 1) * TINY_QUANTUM) {
- if (new_good_size <= (old_size >> 1)) {
- /*
- * Serious shrinkage (more than half). free() the excess.
- */
- return tiny_try_shrink_in_place(szone, ptr, old_size, new_good_size);
- } else if (new_good_size <= old_size) {
- /*
- * new_good_size smaller than old_size but not by much (less than half).
- * Avoid thrashing at the expense of some wasted storage.
- */
- return ptr;
- } else if (tiny_try_realloc_in_place(szone, ptr, old_size, new_good_size)) { // try to grow the allocation
- return ptr;
- }
- }
-
- /*
- * Else if the new size suits the small allocator and the pointer being resized
- * belongs to a small region, and we're not protecting the small allocations
- * try to reallocate in-place.
- */
- } else if (new_good_size <= szone->large_threshold) {
- if ((NUM_TINY_SLOTS - 1) * TINY_QUANTUM < old_size && old_size <= szone->large_threshold) {
- if (new_good_size <= (old_size >> 1)) {
- return small_try_shrink_in_place(szone, ptr, old_size, new_good_size);
- } else if (new_good_size <= old_size) {
- return ptr;
- } else if (small_try_realloc_in_place(szone, ptr, old_size, new_good_size)) {
- return ptr;
- }
- }
- /*
- * Else if the allocation's a large allocation, try to reallocate in-place there.
- */
- } else if (!(szone->debug_flags & SCALABLE_MALLOC_PURGEABLE) && // purgeable needs fresh allocation
- (old_size > szone->large_threshold) &&
- (new_good_size > szone->large_threshold)) {
- if (new_good_size <= (old_size >> 1)) {
- return large_try_shrink_in_place(szone, ptr, old_size, new_good_size);
- } else if (new_good_size <= old_size) {
- return ptr;
- } else if (large_try_realloc_in_place(szone, ptr, old_size, new_good_size)) {
- return ptr;
- }
- }
-
- /*
- * Can't reallocate in place for whatever reason; allocate a new buffer and copy.
- */
- if (new_good_size <= (old_size >> 1)) {
- /* Serious shrinkage (more than half). FALL THROUGH to alloc/copy/free. */
- } else if (new_good_size <= old_size) {
- return ptr;
- }
-
- new_ptr = szone_malloc(szone, new_size);
- if (new_ptr == NULL)
- return NULL;
-
- /*
- * If the allocation's large enough, try to copy using VM. If that fails, or
- * if it's too small, just copy by hand.
- */
- valid_size = MIN(old_size, new_size);
- if ((valid_size < szone->vm_copy_threshold) ||
- vm_copy(mach_task_self(), (vm_address_t)ptr, valid_size, (vm_address_t)new_ptr))
- memcpy(new_ptr, ptr, valid_size);
- szone_free(szone, ptr);
-
-#if DEBUG_MALLOC
- if (LOG(szone, ptr)) {
- malloc_printf("szone_realloc returned %p for %d\n", new_ptr, (unsigned)new_size);
- }
-#endif
- return new_ptr;
-}
-
-static NOINLINE void *
-szone_memalign(szone_t *szone, size_t alignment, size_t size)
-{
- if ((size + alignment) < size) // size_t arithmetic wrapped!
- return NULL;
-
- // alignment is gauranteed a power of 2 at least as large as sizeof(void *), hence non-zero.
- // Since size + alignment didn't wrap, 0 <= size + alignment - 1 < size + alignment
- size_t span = size + alignment - 1;
-
- if (alignment <= TINY_QUANTUM) {
- return szone_malloc(szone, size); // Trivially satisfied by tiny, small, or large
-
- } else if (span <= (NUM_TINY_SLOTS - 1)*TINY_QUANTUM) {
- msize_t mspan = TINY_MSIZE_FOR_BYTES(span + TINY_QUANTUM - 1);
- void *p = szone_malloc(szone, span); // avoids inlining tiny_malloc_should_clear(szone, mspan, 0);
-
- if (NULL == p)
- return NULL;
-
- size_t offset = ((uintptr_t) p) & (alignment - 1); // p % alignment
- size_t pad = (0 == offset) ? 0 : alignment - offset; // p + pad achieves desired alignment
-
- msize_t msize = TINY_MSIZE_FOR_BYTES(size + TINY_QUANTUM - 1);
- msize_t mpad = TINY_MSIZE_FOR_BYTES(pad + TINY_QUANTUM - 1);
- msize_t mwaste = mspan - msize - mpad; // excess blocks
-
- if (mpad > 0) {
- void *q = (void *)(((uintptr_t) p) + pad);
-
- // Mark q as a block header and in-use, thus creating two blocks.
- magazine_t *tiny_mag_ptr = mag_lock_zine_for_region_trailer(szone, szone->tiny_magazines,
- REGION_TRAILER_FOR_TINY_REGION(TINY_REGION_FOR_PTR(p)),
- MAGAZINE_INDEX_FOR_TINY_REGION(TINY_REGION_FOR_PTR(p)));
- set_tiny_meta_header_in_use(q, msize);
- tiny_mag_ptr->mag_num_objects++;
-
- // set_tiny_meta_header_in_use() "reaffirms" the block_header on the *following* block, so
- // now set its in_use bit as well. But only if its within the original allocation made above.
- if (mwaste > 0)
- BITARRAY_SET(TINY_INUSE_FOR_HEADER(TINY_BLOCK_HEADER_FOR_PTR(q)), TINY_INDEX_FOR_PTR(q) + msize);
- SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr);
-
- // Give up mpad blocks beginning at p to the tiny free list
- // region_t r = TINY_REGION_FOR_PTR(p);
- szone_free(szone, p); // avoids inlining free_tiny(szone, p, &r);
-
- p = q; // advance p to the desired alignment
- }
-
- if (mwaste > 0) {
- void *q = (void *)(((uintptr_t) p) + TINY_BYTES_FOR_MSIZE(msize));
- // Mark q as block header and in-use, thus creating two blocks.
- magazine_t *tiny_mag_ptr = mag_lock_zine_for_region_trailer(szone, szone->tiny_magazines,
- REGION_TRAILER_FOR_TINY_REGION(TINY_REGION_FOR_PTR(p)),
- MAGAZINE_INDEX_FOR_TINY_REGION(TINY_REGION_FOR_PTR(p)));
- set_tiny_meta_header_in_use(q, mwaste);
- tiny_mag_ptr->mag_num_objects++;
- SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr);
-
- // Give up mwaste blocks beginning at q to the tiny free list
- // region_t r = TINY_REGION_FOR_PTR(q);
- szone_free(szone, q); // avoids inlining free_tiny(szone, q, &r);
- }
-
- return p; // p has the desired size and alignment, and can later be free()'d
-
- } else if ((NUM_TINY_SLOTS - 1)*TINY_QUANTUM < size && alignment <= SMALL_QUANTUM) {
- return szone_malloc(szone, size); // Trivially satisfied by small or large
-
- } else if (span <= szone->large_threshold) {
-
- if (size <= (NUM_TINY_SLOTS - 1)*TINY_QUANTUM) {
- size = (NUM_TINY_SLOTS - 1)*TINY_QUANTUM + TINY_QUANTUM; // ensure block allocated by small does not have a tiny-possible size
- span = size + alignment - 1;
- }
-
- msize_t mspan = SMALL_MSIZE_FOR_BYTES(span + SMALL_QUANTUM - 1);
- void *p = szone_malloc(szone, span); // avoid inlining small_malloc_should_clear(szone, mspan, 0);
-
- if (NULL == p)
- return NULL;
-
- size_t offset = ((uintptr_t) p) & (alignment - 1); // p % alignment
- size_t pad = (0 == offset) ? 0 : alignment - offset; // p + pad achieves desired alignment
-
- msize_t msize = SMALL_MSIZE_FOR_BYTES(size + SMALL_QUANTUM - 1);
- msize_t mpad = SMALL_MSIZE_FOR_BYTES(pad + SMALL_QUANTUM - 1);
- msize_t mwaste = mspan - msize - mpad; // excess blocks
-
- if (mpad > 0) {
- void *q = (void *)(((uintptr_t) p) + pad);
-
- // Mark q as block header and in-use, thus creating two blocks.
- magazine_t *small_mag_ptr = mag_lock_zine_for_region_trailer(szone, szone->small_magazines,
- REGION_TRAILER_FOR_SMALL_REGION(SMALL_REGION_FOR_PTR(p)),
- MAGAZINE_INDEX_FOR_SMALL_REGION(SMALL_REGION_FOR_PTR(p)));
- small_meta_header_set_in_use(SMALL_META_HEADER_FOR_PTR(p), SMALL_META_INDEX_FOR_PTR(p), mpad);
- small_meta_header_set_in_use(SMALL_META_HEADER_FOR_PTR(q), SMALL_META_INDEX_FOR_PTR(q), msize + mwaste);
- small_mag_ptr->mag_num_objects++;
- SZONE_MAGAZINE_PTR_UNLOCK(szone, small_mag_ptr);
-
- // Give up mpad blocks beginning at p to the small free list
- // region_t r = SMALL_REGION_FOR_PTR(p);
- szone_free(szone, p); // avoid inlining free_small(szone, p, &r);
-
- p = q; // advance p to the desired alignment
- }
- if (mwaste > 0) {
- void *q = (void *)(((uintptr_t) p) + SMALL_BYTES_FOR_MSIZE(msize));
- // Mark q as block header and in-use, thus creating two blocks.
- magazine_t *small_mag_ptr = mag_lock_zine_for_region_trailer(szone, szone->small_magazines,
- REGION_TRAILER_FOR_SMALL_REGION(SMALL_REGION_FOR_PTR(p)),
- MAGAZINE_INDEX_FOR_SMALL_REGION(SMALL_REGION_FOR_PTR(p)));
- small_meta_header_set_in_use(SMALL_META_HEADER_FOR_PTR(p), SMALL_META_INDEX_FOR_PTR(p), msize);
- small_meta_header_set_in_use(SMALL_META_HEADER_FOR_PTR(q), SMALL_META_INDEX_FOR_PTR(q), mwaste);
- small_mag_ptr->mag_num_objects++;
- SZONE_MAGAZINE_PTR_UNLOCK(szone, small_mag_ptr);
-
- // Give up mwaste blocks beginning at q to the small free list
- // region_t r = SMALL_REGION_FOR_PTR(q);
- szone_free(szone, q); // avoid inlining free_small(szone, q, &r);
- }
-
- return p; // p has the desired size and alignment, and can later be free()'d
-
- } else if (szone->large_threshold < size && alignment <= vm_page_size) {
- return szone_malloc(szone, size); // Trivially satisfied by large
-
- } else {
- // ensure block allocated by large does not have a small-possible size
- size_t num_pages = round_page(MAX(szone->large_threshold + 1, size)) >> vm_page_shift;
- void *p;
-
- if (num_pages == 0) /* Overflowed */
- p = NULL;
- else
- p = large_malloc(szone, num_pages, MAX(vm_page_shift, __builtin_ctz(alignment)), 0);
-
- return p;
- }
- /* NOTREACHED */
-}
-
-// given a size, returns the number of pointers allocated capable of holding
-// that size, up to the limit specified by the 'count' argument. These pointers
-// are stored in the 'results' array, which must be allocated by the caller.
-// may return zero, since this function is only a best attempt at allocating
-// the pointers. clients should be prepared to call malloc for any additional
-// blocks they need.
-static NOINLINE unsigned
-szone_batch_malloc(szone_t *szone, size_t size, void **results, unsigned count)
-{
- msize_t msize = TINY_MSIZE_FOR_BYTES(size + TINY_QUANTUM - 1);
- unsigned found = 0;
- mag_index_t mag_index = mag_get_thread_index(szone);
- magazine_t *tiny_mag_ptr = &(szone->tiny_magazines[mag_index]);
-
- // only bother implementing this for tiny
- if (size > (NUM_TINY_SLOTS - 1)*TINY_QUANTUM)
- return 0;
- // make sure to return objects at least one quantum in size
- if (!msize)
- msize = 1;
-
- CHECK(szone, __PRETTY_FUNCTION__);
-
- // We must lock the zone now, since tiny_malloc_from_free_list assumes that
- // the caller has done so.
- SZONE_MAGAZINE_PTR_LOCK(szone, tiny_mag_ptr);
-
- // with the zone locked, allocate objects from the free list until all
- // sufficiently large objects have been exhausted, or we have met our quota
- // of objects to allocate.
- while (found < count) {
- void *ptr = tiny_malloc_from_free_list(szone, tiny_mag_ptr, mag_index, msize);
- if (!ptr)
- break;
-
- *results++ = ptr;
- found++;
- }
- SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr);
- return found;
-}
-
-/* Try caching the tiny_region and checking if the next ptr hits there. */
-static NOINLINE void
-szone_batch_free(szone_t *szone, void **to_be_freed, unsigned count)
-{
- unsigned cc = 0;
- void *ptr;
- region_t tiny_region = NULL;
- boolean_t is_free;
- msize_t msize;
- magazine_t *tiny_mag_ptr = NULL;
- mag_index_t mag_index = -1;
-
- // frees all the pointers in to_be_freed
- // note that to_be_freed may be overwritten during the process
- if (!count)
- return;
-
- CHECK(szone, __PRETTY_FUNCTION__);
- while (cc < count) {
- ptr = to_be_freed[cc];
- if (ptr) {
- if (NULL == tiny_region || tiny_region != TINY_REGION_FOR_PTR(ptr)) { // region same as last iteration?
- if (tiny_mag_ptr) { // non-NULL iff magazine lock taken
- SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr);
- tiny_mag_ptr = NULL;
- }
-
- tiny_region = tiny_region_for_ptr_no_lock(szone, ptr);
-
- if (tiny_region) {
- tiny_mag_ptr = mag_lock_zine_for_region_trailer(szone, szone->tiny_magazines,
- REGION_TRAILER_FOR_TINY_REGION(tiny_region),
- MAGAZINE_INDEX_FOR_TINY_REGION(tiny_region));
- mag_index = MAGAZINE_INDEX_FOR_TINY_REGION(tiny_region);
- }
- }
- if (tiny_region) {
- // this is a tiny pointer
- if (TINY_INDEX_FOR_PTR(ptr) >= NUM_TINY_BLOCKS)
- break; // pointer to metadata; let the standard free deal with it
- msize = get_tiny_meta_header(ptr, &is_free);
- if (is_free)
- break; // a double free; let the standard free deal with it
-
- if (!tiny_free_no_lock(szone, tiny_mag_ptr, mag_index, tiny_region, ptr, msize)) {
- // Arrange to re-acquire magazine lock
- tiny_mag_ptr = NULL;
- tiny_region = NULL;
- }
- to_be_freed[cc] = NULL;
- } else {
- // No region in this zone claims ptr; let the standard free deal with it
- break;
- }
- }
- cc++;
- }
-
- if (tiny_mag_ptr) {
- SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr);
- tiny_mag_ptr = NULL;
- }
-
- CHECK(szone, __PRETTY_FUNCTION__);
- while (count--) {
- ptr = to_be_freed[count];
- if (ptr)
- szone_free(szone, ptr);
- }
-}
-
-// FIXME: Suppose one of the locks is held?
-static void
-szone_destroy(szone_t *szone)
-{
- size_t index;
- large_entry_t *large;
- vm_range_t range_to_deallocate;
-
-#if LARGE_CACHE
- SZONE_LOCK(szone);
-
- /* disable any memory pressure responder */
- szone->flotsam_enabled = FALSE;
-
- // stack allocated copy of the death-row cache
- int idx = szone->large_entry_cache_oldest, idx_max = szone->large_entry_cache_newest;
- large_entry_t local_entry_cache[LARGE_ENTRY_CACHE_SIZE];
-
- memcpy((void *)local_entry_cache, (void *)szone->large_entry_cache, sizeof(local_entry_cache));
-
- szone->large_entry_cache_oldest = szone->large_entry_cache_newest = 0;
- szone->large_entry_cache[0].address = 0x0;
- szone->large_entry_cache[0].size = 0;
- szone->large_entry_cache_bytes = 0;
- szone->large_entry_cache_reserve_bytes = 0;
-
- SZONE_UNLOCK(szone);
-
- // deallocate the death-row cache outside the zone lock
- while (idx != idx_max) {
- deallocate_pages(szone, (void *) local_entry_cache[idx].address, local_entry_cache[idx].size, 0);
- if (++idx == LARGE_ENTRY_CACHE_SIZE) idx = 0;
- }
- if (0 != local_entry_cache[idx].address && 0 != local_entry_cache[idx].size) {
- deallocate_pages(szone, (void *) local_entry_cache[idx].address, local_entry_cache[idx].size, 0);
- }
-#endif
-
- /* destroy large entries */
- index = szone->num_large_entries;
- while (index--) {
- large = szone->large_entries + index;
- if (large->address) {
- // we deallocate_pages, including guard pages
- deallocate_pages(szone, (void *)(large->address), large->size, szone->debug_flags);
- }
- }
- large_entries_free_no_lock(szone, szone->large_entries, szone->num_large_entries, &range_to_deallocate);
- if (range_to_deallocate.size)
- deallocate_pages(szone, (void *)range_to_deallocate.address, (size_t)range_to_deallocate.size, 0);
-
- /* destroy tiny regions */
- for (index = 0; index < szone->tiny_region_generation->num_regions_allocated; ++index)
- if ((HASHRING_OPEN_ENTRY != szone->tiny_region_generation->hashed_regions[index]) &&
- (HASHRING_REGION_DEALLOCATED != szone->tiny_region_generation->hashed_regions[index]))
- deallocate_pages(szone, szone->tiny_region_generation->hashed_regions[index], TINY_REGION_SIZE, 0);
-
- /* destroy small regions */
- for (index = 0; index < szone->small_region_generation->num_regions_allocated; ++index)
- if ((HASHRING_OPEN_ENTRY != szone->small_region_generation->hashed_regions[index]) &&
- (HASHRING_REGION_DEALLOCATED != szone->small_region_generation->hashed_regions[index]))
- deallocate_pages(szone, szone->small_region_generation->hashed_regions[index], SMALL_REGION_SIZE, 0);
-
- /* destroy region hash rings, if any */
- if (szone->tiny_region_generation->hashed_regions != szone->initial_tiny_regions) {
- size_t size = round_page(szone->tiny_region_generation->num_regions_allocated * sizeof(region_t));
- deallocate_pages(szone, szone->tiny_region_generation->hashed_regions, size, 0);
- }
- if (szone->small_region_generation->hashed_regions != szone->initial_small_regions) {
- size_t size = round_page(szone->small_region_generation->num_regions_allocated * sizeof(region_t));
- deallocate_pages(szone, szone->small_region_generation->hashed_regions, size, 0);
- }
-
- /* Now destroy the separate szone region */
- if (szone->cpu_id_key != (pthread_key_t) -1)
- (void)pthread_key_delete(szone->cpu_id_key);
- deallocate_pages(szone, (void *)&(szone->tiny_magazines[-1]), TINY_MAGAZINE_PAGED_SIZE, SCALABLE_MALLOC_ADD_GUARD_PAGES);
- deallocate_pages(szone, (void *)&(szone->small_magazines[-1]), SMALL_MAGAZINE_PAGED_SIZE, SCALABLE_MALLOC_ADD_GUARD_PAGES);
- deallocate_pages(szone, (void *)szone, SZONE_PAGED_SIZE, 0);
-}
-
-static NOINLINE size_t
-szone_good_size(szone_t *szone, size_t size)
-{
- msize_t msize;
-
- // Find a good size for this tiny allocation.
- if (size <= (NUM_TINY_SLOTS - 1) * TINY_QUANTUM) {
- msize = TINY_MSIZE_FOR_BYTES(size + TINY_QUANTUM - 1);
- if (!msize)
- msize = 1;
- return TINY_BYTES_FOR_MSIZE(msize);
- }
-
- // Find a good size for this small allocation.
- if (size <= szone->large_threshold) {
- msize = SMALL_MSIZE_FOR_BYTES(size + SMALL_QUANTUM - 1);
- if (!msize)
- msize = 1;
- return SMALL_BYTES_FOR_MSIZE(msize);
- }
-
- // Check for integer overflow on the size, since unlike the two cases above,
- // there is no upper bound on allocation size at this point.
- if (size > round_page(size))
- return (size_t)(-1LL);
-
-#if DEBUG_MALLOC
- // It is not acceptable to see a size of zero here, since that means we
- // failed to catch a request for zero bytes in the tiny check, or the size
- // overflowed to zero during some arithmetic.
- if (size == 0)
- malloc_printf("szone_good_size() invariant broken %y\n", size);
-#endif
- return round_page(size);
-}
-
-unsigned szone_check_counter = 0;
-unsigned szone_check_start = 0;
-unsigned szone_check_modulo = 1;
-
-static NOINLINE boolean_t
-szone_check_all(szone_t *szone, const char *function)
-{
- size_t index;
-
- /* check tiny regions - chould check region count */
- for (index = 0; index < szone->tiny_region_generation->num_regions_allocated; ++index) {
- region_t tiny = szone->tiny_region_generation->hashed_regions[index];
-
- if (HASHRING_REGION_DEALLOCATED == tiny)
- continue;
-
- if (tiny) {
- magazine_t *tiny_mag_ptr = mag_lock_zine_for_region_trailer(szone, szone->tiny_magazines,
- REGION_TRAILER_FOR_TINY_REGION(tiny), MAGAZINE_INDEX_FOR_TINY_REGION(tiny));
-
- if (!tiny_check_region(szone, tiny)) {
- SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr);
- szone->debug_flags &= ~ CHECK_REGIONS;
- szone_error(szone, 1, "check: tiny region incorrect", NULL,
- "*** tiny region %ld incorrect szone_check_all(%s) counter=%d\n",
- index, function, szone_check_counter);
- return 0;
- }
- SZONE_MAGAZINE_PTR_UNLOCK(szone, tiny_mag_ptr);
- }
- }
- /* check tiny free lists */
- for (index = 0; index < NUM_TINY_SLOTS; ++index) {
- if (!tiny_free_list_check(szone, index)) {
- szone->debug_flags &= ~ CHECK_REGIONS;
- szone_error(szone, 1, "check: tiny free list incorrect", NULL,
- "*** tiny free list incorrect (slot=%ld) szone_check_all(%s) counter=%d\n",
- index, function, szone_check_counter);
- return 0;
- }
- }
-
- /* check small regions - could check region count */
- for (index = 0; index < szone->small_region_generation->num_regions_allocated; ++index) {
- region_t small = szone->small_region_generation->hashed_regions[index];
-
- if (HASHRING_REGION_DEALLOCATED == small)
- continue;
-
- if (small) {
- magazine_t *small_mag_ptr = mag_lock_zine_for_region_trailer(szone, szone->small_magazines,
- REGION_TRAILER_FOR_SMALL_REGION(small), MAGAZINE_INDEX_FOR_SMALL_REGION(small));
-
- if (!small_check_region(szone, small)) {
- SZONE_MAGAZINE_PTR_UNLOCK(szone, small_mag_ptr);
- szone->debug_flags &= ~ CHECK_REGIONS;
- szone_error(szone, 1, "check: small region incorrect", NULL,
- "*** small region %ld incorrect szone_check_all(%s) counter=%d\n",
- index, function, szone_check_counter);
- return 0;
- }
- SZONE_MAGAZINE_PTR_UNLOCK(szone, small_mag_ptr);
- }
- }
- /* check small free lists */
- for (index = 0; index < szone->num_small_slots; ++index) {
- if (!small_free_list_check(szone, index)) {
- szone->debug_flags &= ~ CHECK_REGIONS;
- szone_error(szone, 1, "check: small free list incorrect", NULL,
- "*** small free list incorrect (slot=%ld) szone_check_all(%s) counter=%d\n",
- index, function, szone_check_counter);
- return 0;
- }
- }
-
- return 1;
-}
-
-static boolean_t
-szone_check(szone_t *szone)
-{
- if ((++szone_check_counter % 10000) == 0)
- _malloc_printf(ASL_LEVEL_NOTICE, "at szone_check counter=%d\n", szone_check_counter);
-
- if (szone_check_counter < szone_check_start)
- return 1;
-
- if (szone_check_counter % szone_check_modulo)
- return 1;
-
- return szone_check_all(szone, "");
-}
-
-static kern_return_t
-szone_ptr_in_use_enumerator(task_t task, void *context, unsigned type_mask, vm_address_t zone_address,
- memory_reader_t reader, vm_range_recorder_t recorder)
-{
- szone_t *szone;
- kern_return_t err;
-
- if (!reader) reader = _szone_default_reader;
-
- err = reader(task, zone_address, sizeof(szone_t), (void **)&szone);
- if (err) return err;
-
- err = tiny_in_use_enumerator(task, context, type_mask, szone, reader, recorder);
- if (err) return err;
-
- err = small_in_use_enumerator(task, context, type_mask, szone, reader, recorder);
- if (err) return err;
-
- err = large_in_use_enumerator(task, context, type_mask,
- (vm_address_t)szone->large_entries, szone->num_large_entries, reader, recorder);
- return err;
-}
-
-// Following method is deprecated: use scalable_zone_statistics instead
-void
-scalable_zone_info(malloc_zone_t *zone, unsigned *info_to_fill, unsigned count)
-{
- szone_t *szone = (void *)zone;
- unsigned info[13];
-
- // We do not lock to facilitate debug
-
- size_t s = 0;
- unsigned t = 0;
- size_t u = 0;
- mag_index_t mag_index;
-
- for (mag_index = -1; mag_index < szone->num_tiny_magazines; mag_index++) {
- s += szone->tiny_magazines[mag_index].mag_bytes_free_at_start;
- s += szone->tiny_magazines[mag_index].mag_bytes_free_at_end;
- t += szone->tiny_magazines[mag_index].mag_num_objects;
- u += szone->tiny_magazines[mag_index].mag_num_bytes_in_objects;
- }
-
- info[4] = t;
- info[5] = u;
-
- for (t = 0, u = 0, mag_index = -1; mag_index < szone->num_small_magazines; mag_index++) {
- s += szone->small_magazines[mag_index].mag_bytes_free_at_start;
- s += szone->small_magazines[mag_index].mag_bytes_free_at_end;
- t += szone->small_magazines[mag_index].mag_num_objects;
- u += szone->small_magazines[mag_index].mag_num_bytes_in_objects;
- }
-
- info[6] = t;
- info[7] = u;
-
- info[8] = szone->num_large_objects_in_use;
- info[9] = szone->num_bytes_in_large_objects;
-
- info[10] = 0; // DEPRECATED szone->num_huge_entries;
- info[11] = 0; // DEPRECATED szone->num_bytes_in_huge_objects;
-
- info[12] = szone->debug_flags;
-
- info[0] = info[4] + info[6] + info[8] + info[10];
- info[1] = info[5] + info[7] + info[9] + info[11];
-
- info[3] = (szone->num_tiny_regions - szone->num_tiny_regions_dealloc) * TINY_REGION_SIZE +
- (szone->num_small_regions - szone->num_small_regions_dealloc) * SMALL_REGION_SIZE + info[9] + info[11];
-
- info[2] = info[3] - s;
- memcpy(info_to_fill, info, sizeof(unsigned)*count);
-}
-
-// FIXME: consistent picture requires locking!
-static NOINLINE void
-szone_print(szone_t *szone, boolean_t verbose)
-{
- unsigned info[13];
- size_t index;
- region_t region;
-
- scalable_zone_info((void *)szone, info, 13);
- _malloc_printf(MALLOC_PRINTF_NOLOG | MALLOC_PRINTF_NOPREFIX,
- "Scalable zone %p: inUse=%u(%y) touched=%y allocated=%y flags=%d\n",
- szone, info[0], info[1], info[2], info[3], info[12]);
- _malloc_printf(MALLOC_PRINTF_NOLOG | MALLOC_PRINTF_NOPREFIX,
- "\ttiny=%u(%y) small=%u(%y) large=%u(%y) huge=%u(%y)\n",
- info[4], info[5], info[6], info[7], info[8], info[9], info[10], info[11]);
- // tiny
- _malloc_printf(MALLOC_PRINTF_NOLOG | MALLOC_PRINTF_NOPREFIX,
- "%lu tiny regions:\n", szone->num_tiny_regions);
- if (szone->num_tiny_regions_dealloc)
- _malloc_printf(MALLOC_PRINTF_NOLOG | MALLOC_PRINTF_NOPREFIX,
- "[%lu tiny regions have been vm_deallocate'd]\n", szone->num_tiny_regions_dealloc);
- for (index = 0; index < szone->tiny_region_generation->num_regions_allocated; ++index) {
- region = szone->tiny_region_generation->hashed_regions[index];
- if (HASHRING_OPEN_ENTRY != region && HASHRING_REGION_DEALLOCATED != region) {
- mag_index_t mag_index = MAGAZINE_INDEX_FOR_TINY_REGION(region);
- print_tiny_region(verbose, region,
- (region == szone->tiny_magazines[mag_index].mag_last_region) ?
- szone->tiny_magazines[mag_index].mag_bytes_free_at_start : 0,
- (region == szone->tiny_magazines[mag_index].mag_last_region) ?
- szone->tiny_magazines[mag_index].mag_bytes_free_at_end : 0);
- }
- }
- if (verbose)
- print_tiny_free_list(szone);
- // small
- _malloc_printf(MALLOC_PRINTF_NOLOG | MALLOC_PRINTF_NOPREFIX,
- "%lu small regions:\n", szone->num_small_regions);
- if (szone->num_small_regions_dealloc)
- _malloc_printf(MALLOC_PRINTF_NOLOG | MALLOC_PRINTF_NOPREFIX,
- "[%lu small regions have been vm_deallocate'd]\n", szone->num_small_regions_dealloc);
- for (index = 0; index < szone->small_region_generation->num_regions_allocated; ++index) {
- region = szone->small_region_generation->hashed_regions[index];
- if (HASHRING_OPEN_ENTRY != region && HASHRING_REGION_DEALLOCATED != region) {
- mag_index_t mag_index = MAGAZINE_INDEX_FOR_SMALL_REGION(region);
- print_small_region(szone, verbose, region,
- (region == szone->small_magazines[mag_index].mag_last_region) ?
- szone->small_magazines[mag_index].mag_bytes_free_at_start : 0,
- (region == szone->small_magazines[mag_index].mag_last_region) ?
- szone->small_magazines[mag_index].mag_bytes_free_at_end : 0);
- }
- }
- if (verbose)
- print_small_free_list(szone);
-}
-
-static void
-szone_log(malloc_zone_t *zone, void *log_address)
-{
- szone_t *szone = (szone_t *)zone;
-
- szone->log_address = log_address;
-}
-
-static void
-szone_force_lock(szone_t *szone)
-{
- mag_index_t i;
-
- for (i = 0; i < szone->num_tiny_magazines; ++i) {
- SZONE_MAGAZINE_PTR_LOCK(szone, (&(szone->tiny_magazines[i])));
- }
- SZONE_MAGAZINE_PTR_LOCK(szone, (&(szone->tiny_magazines[DEPOT_MAGAZINE_INDEX])));
-
- for (i = 0; i < szone->num_small_magazines; ++i) {
- SZONE_MAGAZINE_PTR_LOCK(szone, (&(szone->small_magazines[i])));
- }
- SZONE_MAGAZINE_PTR_LOCK(szone, (&(szone->small_magazines[DEPOT_MAGAZINE_INDEX])));
-
- SZONE_LOCK(szone);
-}
-
-static void
-szone_force_unlock(szone_t *szone)
-{
- mag_index_t i;
-
- SZONE_UNLOCK(szone);
-
- for (i = -1; i < szone->num_small_magazines; ++i) {
- SZONE_MAGAZINE_PTR_UNLOCK(szone, (&(szone->small_magazines[i])));
- }
-
- for (i = -1; i < szone->num_tiny_magazines; ++i) {
- SZONE_MAGAZINE_PTR_UNLOCK(szone, (&(szone->tiny_magazines[i])));
- }
-}
-
-static boolean_t
-szone_locked(szone_t *szone)
-{
- mag_index_t i;
- int tookLock;
-
- tookLock = SZONE_TRY_LOCK(szone);
- if (tookLock == 0)
- return 1;
- SZONE_UNLOCK(szone);
-
- for (i = -1; i < szone->num_small_magazines; ++i) {
- tookLock = SZONE_MAGAZINE_PTR_TRY_LOCK(szone, (&(szone->small_magazines[i])));
- if (tookLock == 0)
- return 1;
- SZONE_MAGAZINE_PTR_UNLOCK(szone, (&(szone->small_magazines[i])));
- }
-
- for (i = -1; i < szone->num_tiny_magazines; ++i) {
- tookLock = SZONE_MAGAZINE_PTR_TRY_LOCK(szone, (&(szone->tiny_magazines[i])));
- if (tookLock == 0)
- return 1;
- SZONE_MAGAZINE_PTR_UNLOCK(szone, (&(szone->tiny_magazines[i])));
- }
- return 0;
-}
-
-static size_t
-szone_pressure_relief(szone_t *szone, size_t goal)
-{
-#if LARGE_CACHE
- if (!szone->flotsam_enabled)
- return 0;
-
- SZONE_LOCK(szone);
-
- // stack allocated copy of the death-row cache
- int idx = szone->large_entry_cache_oldest, idx_max = szone->large_entry_cache_newest;
- large_entry_t local_entry_cache[LARGE_ENTRY_CACHE_SIZE];
-
- memcpy((void *)local_entry_cache, (void *)szone->large_entry_cache, sizeof(local_entry_cache));
-
- szone->large_entry_cache_oldest = szone->large_entry_cache_newest = 0;
- szone->large_entry_cache[0].address = 0x0;
- szone->large_entry_cache[0].size = 0;
- szone->large_entry_cache_bytes = 0;
- szone->large_entry_cache_reserve_bytes = 0;
-
- szone->flotsam_enabled = FALSE;
-
- SZONE_UNLOCK(szone);
-
- // deallocate the death-row cache outside the zone lock
- size_t total = 0;
- while (idx != idx_max) {
- deallocate_pages(szone, (void *) local_entry_cache[idx].address, local_entry_cache[idx].size, 0);
- total += local_entry_cache[idx].size;
- if (++idx == LARGE_ENTRY_CACHE_SIZE) idx = 0;
- }
- if (0 != local_entry_cache[idx].address && 0 != local_entry_cache[idx].size) {
- deallocate_pages(szone, (void *) local_entry_cache[idx].address, local_entry_cache[idx].size, 0);
- total += local_entry_cache[idx].size;
- }
- MAGMALLOC_PRESSURERELIEF((void *)szone, goal, total); // DTrace USDT Probe
- return total;
-#else
- return 0;
-#endif
-}
-
-boolean_t
-scalable_zone_statistics(malloc_zone_t *zone, malloc_statistics_t *stats, unsigned subzone)
-{
- szone_t *szone = (szone_t *)zone;
-
- switch (subzone) {
- case 0:
- {
- size_t s = 0;
- unsigned t = 0;
- size_t u = 0;
- mag_index_t mag_index;
-
- for (mag_index = -1; mag_index < szone->num_tiny_magazines; mag_index++) {
- s += szone->tiny_magazines[mag_index].mag_bytes_free_at_start;
- s += szone->tiny_magazines[mag_index].mag_bytes_free_at_end;
- t += szone->tiny_magazines[mag_index].mag_num_objects;
- u += szone->tiny_magazines[mag_index].mag_num_bytes_in_objects;
- }
-
- stats->blocks_in_use = t;
- stats->size_in_use = u;
- stats->size_allocated = (szone->num_tiny_regions - szone->num_tiny_regions_dealloc) * TINY_REGION_SIZE;
- stats->max_size_in_use = stats->size_allocated - s;
- return 1;
- }
- case 1:
- {
- size_t s = 0;
- unsigned t = 0;
- size_t u = 0;
- mag_index_t mag_index;
-
- for (mag_index = -1; mag_index < szone->num_small_magazines; mag_index++) {
- s += szone->small_magazines[mag_index].mag_bytes_free_at_start;
- s += szone->small_magazines[mag_index].mag_bytes_free_at_end;
- t += szone->small_magazines[mag_index].mag_num_objects;
- u += szone->small_magazines[mag_index].mag_num_bytes_in_objects;
- }
-
- stats->blocks_in_use = t;
- stats->size_in_use = u;
- stats->size_allocated = (szone->num_small_regions - szone->num_small_regions_dealloc) * SMALL_REGION_SIZE;
- stats->max_size_in_use = stats->size_allocated - s;
- return 1;
- }
- case 2:
- stats->blocks_in_use = szone->num_large_objects_in_use;
- stats->size_in_use = szone->num_bytes_in_large_objects;
- stats->max_size_in_use = stats->size_allocated = stats->size_in_use;
- return 1;
- case 3:
- stats->blocks_in_use = 0; // DEPRECATED szone->num_huge_entries;
- stats->size_in_use = 0; // DEPRECATED szone->num_bytes_in_huge_objects;
- stats->max_size_in_use = stats->size_allocated = 0;
- return 1;
- }
- return 0;
-}
-
-static void
-szone_statistics(szone_t *szone, malloc_statistics_t *stats)
-{
- size_t large;
-
- size_t s = 0;
- unsigned t = 0;
- size_t u = 0;
- mag_index_t mag_index;
-
- for (mag_index = -1; mag_index < szone->num_tiny_magazines; mag_index++) {
- s += szone->tiny_magazines[mag_index].mag_bytes_free_at_start;
- s += szone->tiny_magazines[mag_index].mag_bytes_free_at_end;
- t += szone->tiny_magazines[mag_index].mag_num_objects;
- u += szone->tiny_magazines[mag_index].mag_num_bytes_in_objects;
- }
-
- for (mag_index = -1; mag_index < szone->num_small_magazines; mag_index++) {
- s += szone->small_magazines[mag_index].mag_bytes_free_at_start;
- s += szone->small_magazines[mag_index].mag_bytes_free_at_end;
- t += szone->small_magazines[mag_index].mag_num_objects;
- u += szone->small_magazines[mag_index].mag_num_bytes_in_objects;
- }
-
- large = szone->num_bytes_in_large_objects + 0; // DEPRECATED szone->num_bytes_in_huge_objects;
-
- stats->blocks_in_use = t + szone->num_large_objects_in_use + 0; // DEPRECATED szone->num_huge_entries;
- stats->size_in_use = u + large;
- stats->max_size_in_use = stats->size_allocated =
- (szone->num_tiny_regions - szone->num_tiny_regions_dealloc) * TINY_REGION_SIZE +
- (szone->num_small_regions - szone->num_small_regions_dealloc) * SMALL_REGION_SIZE + large;
- // Now we account for the untouched areas
- stats->max_size_in_use -= s;
-}
-
-static void *
-legacy_zeroing_large_malloc(szone_t *szone, size_t size) {
- if (size > LARGE_THRESHOLD) // Leopard and earlier returned a ZFOD range, so ...
- return szone_calloc(szone, 1, size); // Clear to zero always, ham-handedly touching in each page
- else
- return szone_malloc(szone, size);
-}
-
-static void *
-legacy_zeroing_large_valloc(szone_t *szone, size_t size) {
- void *p = szone_valloc(szone, size);
-
- // Leopard and earlier returned a ZFOD range, so ...
- memset(p, 0, size); // Clear to zero always, ham-handedly touching in each page
- return p;
-}
-
-void zeroify_scalable_zone(malloc_zone_t *zone)
-{
- szone_t *szone = (szone_t *)zone;
-
- if (szone) {
- mprotect(szone, sizeof(szone->basic_zone), PROT_READ | PROT_WRITE);
- szone->basic_zone.malloc = (void *)legacy_zeroing_large_malloc;
- szone->basic_zone.valloc = (void *)legacy_zeroing_large_valloc;
- mprotect(szone, sizeof(szone->basic_zone), PROT_READ);
- }
-}
-
-static const struct malloc_introspection_t szone_introspect = {
- (void *)szone_ptr_in_use_enumerator,
- (void *)szone_good_size,
- (void *)szone_check,
- (void *)szone_print,
- szone_log,
- (void *)szone_force_lock,
- (void *)szone_force_unlock,
- (void *)szone_statistics,
- (void *)szone_locked,
- NULL, NULL, NULL, NULL, /* Zone enumeration version 7 and forward. */
-}; // marked as const to spare the DATA section
-
-malloc_zone_t *
-create_scalable_zone(size_t initial_size, unsigned debug_flags)
-{
- szone_t *szone;
- uint64_t hw_memsize = 0;
-
- /*
- * Sanity-check our build-time assumptions about the size of a page.
- * Since we have sized various things assuming the default page size,
- * attempting to determine it dynamically is not useful.
- */
- if ((vm_page_size != _vm_page_size) || (vm_page_shift != _vm_page_shift)) {
- malloc_printf("*** FATAL ERROR - machine page size does not match our assumptions.\n");
- exit(-1);
- }
-
-#if defined(__i386__) || defined(__x86_64__)
- if (_COMM_PAGE_VERSION_REQD > (*((short *) _COMM_PAGE_VERSION))) { // _COMM_PAGE_CPU_NUMBER must be present at runtime
- malloc_printf("*** ERROR - comm page version mismatch.\n");
- exit(-1);
- }
-#endif
-
- /* get memory for the zone. */
- szone = allocate_pages(NULL, SZONE_PAGED_SIZE, 0, 0, VM_MEMORY_MALLOC);
- if (!szone)
- return NULL;
-
- /* set up the szone structure */
-#if 0
-#warning CHECK_REGIONS enabled
- debug_flags |= CHECK_REGIONS;
-#endif
-#if 0
-#warning LOG enabled
- szone->log_address = ~0;
-#endif
- szone->trg[0].nextgen = &(szone->trg[1]);
- szone->trg[1].nextgen = &(szone->trg[0]);
- szone->tiny_region_generation = &(szone->trg[0]);
-
- szone->tiny_region_generation->hashed_regions = szone->initial_tiny_regions;
- szone->tiny_region_generation->num_regions_allocated = INITIAL_NUM_REGIONS;
- szone->tiny_region_generation->num_regions_allocated_shift = INITIAL_NUM_REGIONS_SHIFT;
-
- szone->srg[0].nextgen = &(szone->srg[1]);
- szone->srg[1].nextgen = &(szone->srg[0]);
- szone->small_region_generation = &(szone->srg[0]);
-
- szone->small_region_generation->hashed_regions = szone->initial_small_regions;
- szone->small_region_generation->num_regions_allocated = INITIAL_NUM_REGIONS;
- szone->small_region_generation->num_regions_allocated_shift = INITIAL_NUM_REGIONS_SHIFT;
-
-
- /*
- * Initialize variables that size the free list for SMALL allocations based
- * upon the amount of memory in the system. Switch to a larger number of
- * free list entries at 1GB.
- */
-#if defined(__i386__) || defined(__x86_64__) || defined(__arm__)
- if ((hw_memsize = *(uint64_t *)(uintptr_t)_COMM_PAGE_MEMORY_SIZE) >= (1ULL << 30))
-#else
- size_t uint64_t_size = sizeof(hw_memsize);
-
- if (0 == sysctlbyname("hw.memsize", &hw_memsize, &uint64_t_size, 0, 0) &&
- hw_memsize >= (1ULL << 30))
-#endif
- {
- szone->is_largemem = 1;
- szone->num_small_slots = NUM_SMALL_SLOTS_LARGEMEM;
- szone->large_threshold = LARGE_THRESHOLD_LARGEMEM;
- szone->vm_copy_threshold = VM_COPY_THRESHOLD_LARGEMEM;
- } else {
- szone->is_largemem = 0;
- szone->num_small_slots = NUM_SMALL_SLOTS;
- szone->large_threshold = LARGE_THRESHOLD;
- szone->vm_copy_threshold = VM_COPY_THRESHOLD;
- }
-#if LARGE_CACHE
- szone->large_entry_cache_reserve_limit =
- hw_memsize >> 10; // madvise(..., MADV_REUSABLE) death-row arrivals above this threshold [~0.1%]
-
- /* <rdar://problem/6610904> Reset protection when returning a previous large allocation? */
- int32_t libSystemVersion = NSVersionOfLinkTimeLibrary("System");
- if ((-1 != libSystemVersion) && ((libSystemVersion >> 16) < 112) /* CFSystemVersionSnowLeopard */)
- szone->large_legacy_reset_mprotect = TRUE;
- else
- szone->large_legacy_reset_mprotect = FALSE;
-#endif
-
- // Initialize the security token.
- szone->cookie = (uintptr_t)malloc_entropy[0];
-
- // Prepare ASLR
-#if __i386__ || __LP64__ || TARGET_OS_EMBEDDED
-#if __i386__
- uintptr_t stackbase = 0x8fe00000;
- int entropic_bits = 3;
-#elif __LP64__
- uintptr_t stackbase = USRSTACK64;
- int entropic_bits = 16;
-#else
- uintptr_t stackbase = USRSTACK;
- int entropic_bits = 3;
-#endif
- if (0 != _dyld_get_image_slide((const struct mach_header*)_NSGetMachExecuteHeader())) {
- if (0 == entropic_address) {
- uintptr_t t = stackbase - MAXSSIZ - ((uintptr_t) (malloc_entropy[1] & ((1 << entropic_bits) - 1)) << SMALL_BLOCKS_ALIGN);
- (void)__sync_bool_compare_and_swap(&entropic_limit, 0, t); // Just one initialization please
- (void)__sync_bool_compare_and_swap(&entropic_address, 0, t - ENTROPIC_KABILLION); // Just one initialization please
- }
- debug_flags &= ~DISABLE_ASLR;
- } else {
- debug_flags |= DISABLE_ASLR;
- }
-
-#else
- debug_flags |= DISABLE_ASLR;
-#endif
-
- szone->basic_zone.version = 8;
- szone->basic_zone.size = (void *)szone_size;
- szone->basic_zone.malloc = (void *)szone_malloc;
- szone->basic_zone.calloc = (void *)szone_calloc;
- szone->basic_zone.valloc = (void *)szone_valloc;
- szone->basic_zone.free = (void *)szone_free;
- szone->basic_zone.realloc = (void *)szone_realloc;
- szone->basic_zone.destroy = (void *)szone_destroy;
- szone->basic_zone.batch_malloc = (void *)szone_batch_malloc;
- szone->basic_zone.batch_free = (void *)szone_batch_free;
- szone->basic_zone.introspect = (struct malloc_introspection_t *)&szone_introspect;
- szone->basic_zone.memalign = (void *)szone_memalign;
- szone->basic_zone.free_definite_size = (void *)szone_free_definite_size;
- szone->basic_zone.pressure_relief = (void *)szone_pressure_relief;
-
- szone->basic_zone.reserved1 = 0; /* Set to zero once and for all as required by CFAllocator. */
- szone->basic_zone.reserved2 = 0; /* Set to zero once and for all as required by CFAllocator. */
- mprotect(szone, sizeof(szone->basic_zone), PROT_READ); /* Prevent overwriting the function pointers in basic_zone. */
-
- szone->debug_flags = debug_flags;
- LOCK_INIT(szone->large_szone_lock);
-
-#if defined(__ppc__) || defined(__ppc64__)
- /*
- * In the interest of compatibility for PPC applications executing via Rosetta,
- * arrange to zero-fill allocations as occurred by side effect in Leopard and earlier.
- */
- zeroify_scalable_zone((malloc_zone_t *)szone);
-#endif
-
-#if defined(__i386__) || defined(__x86_64__)
- szone->cpu_id_key = (pthread_key_t) -1; // Unused. _COMM_PAGE_CPU_NUMBER preferred.
-#else
- int err;
- if ((err = pthread_key_create(&(szone->cpu_id_key), NULL))) {
- malloc_printf("*** ERROR -pthread_key_create failure err=%d.\n", err);
- szone->cpu_id_key = (pthread_key_t) -1;
- }
-#endif
-
- // Query the number of configured processors.
- // Uniprocessor case gets just one tiny and one small magazine (whose index is zero). This gives
- // the same behavior as the original scalable malloc. MP gets per-CPU magazines
- // that scale (way) better.
-#if defined(__i386__) || defined(__x86_64__) || defined(__arm__)
- int nproc = *(uint8_t *)(uintptr_t)_COMM_PAGE_NCPUS;
-#else
- int nproc = sysconf(_SC_NPROCESSORS_CONF);
-#endif
- szone->num_tiny_magazines = (nproc > 1) ? MIN(nproc, TINY_MAX_MAGAZINES) : 1;
-
- // FIXME vm_allocate() based on number of configured CPUs
- magazine_t *tiny_magazines = allocate_pages(NULL, TINY_MAGAZINE_PAGED_SIZE, 0,
- SCALABLE_MALLOC_ADD_GUARD_PAGES, VM_MEMORY_MALLOC);
- if (NULL == tiny_magazines)
- return NULL;
-
- szone->tiny_magazines = &(tiny_magazines[1]); // szone->tiny_magazines[-1] is the Depot
-
- // The magazines are indexed in [0 .. (num_tiny_magazines - 1)]
- // Find the smallest power of 2 that exceeds (num_tiny_magazines - 1)
- szone->num_tiny_magazines_mask_shift = 0;
- int i = 1;
- while( i <= (szone->num_tiny_magazines - 1) ) {
- szone->num_tiny_magazines_mask_shift++;
- i <<= 1;
- }
-
- // Now if i <= TINY_MAX_MAGAZINES we'll never access tiny_magazines[] out of bounds.
- if (i > TINY_MAX_MAGAZINES) {
- malloc_printf("*** FATAL ERROR - magazine mask exceeds allocated magazines.\n");
- exit(-1);
- }
-
- // Reduce i by 1 to obtain a mask covering [0 .. (num_tiny_magazines - 1)]
- szone->num_tiny_magazines_mask = i - 1; // A mask used for hashing to a magazine index (and a safety aid)
-#if TARGET_OS_EMBEDDED
- szone->last_tiny_advise = 0;
-#endif
-
- // Init the tiny_magazine locks
- LOCK_INIT(szone->tiny_regions_lock);
- LOCK_INIT(szone->tiny_magazines[DEPOT_MAGAZINE_INDEX].magazine_lock);
- for (i = 0; i < szone->num_tiny_magazines; ++i) {
- LOCK_INIT(szone->tiny_magazines[i].magazine_lock);
- }
-
- szone->num_small_magazines = (nproc > 1) ? MIN(nproc, SMALL_MAX_MAGAZINES) : 1;
-
- // FIXME vm_allocate() based on number of configured CPUs
- magazine_t *small_magazines = allocate_pages(NULL, SMALL_MAGAZINE_PAGED_SIZE, 0,
- SCALABLE_MALLOC_ADD_GUARD_PAGES, VM_MEMORY_MALLOC);
- if (NULL == small_magazines)
- return NULL;
-
- szone->small_magazines = &(small_magazines[1]); // szone->small_magazines[-1] is the Depot
-
- // The magazines are indexed in [0 .. (num_small_magazines - 1)]
- // Find the smallest power of 2 that exceeds (num_small_magazines - 1)
- szone->num_small_magazines_mask_shift = 0;
- while( i <= (szone->num_small_magazines - 1) ) {
- szone->num_small_magazines_mask_shift++;
- i <<= 1;
- }
-
- // Now if i <= SMALL_MAX_MAGAZINES we'll never access small_magazines[] out of bounds.
- if (i > SMALL_MAX_MAGAZINES) {
- malloc_printf("*** FATAL ERROR - magazine mask exceeds allocated magazines.\n");
- exit(-1);
- }
-
- // Reduce i by 1 to obtain a mask covering [0 .. (num_small_magazines - 1)]
- szone->num_small_magazines_mask = i - 1; // A mask used for hashing to a magazine index (and a safety aid)
-#if TARGET_OS_EMBEDDED
- szone->last_small_advise = 0;
-#endif
-
- // Init the small_magazine locks
- LOCK_INIT(szone->small_regions_lock);
- LOCK_INIT(szone->small_magazines[DEPOT_MAGAZINE_INDEX].magazine_lock);
- for (i = 0; i < szone->num_small_magazines; ++i) {
- LOCK_INIT(szone->small_magazines[i].magazine_lock);
- }
-
- CHECK(szone, __PRETTY_FUNCTION__);
- return (malloc_zone_t *)szone;
-}
-
-//
-// purgeable zones have their own "large" allocation pool, but share "tiny" and "large"
-// heaps with a helper_zone identified in the call to create_purgeable_zone()
-//
-static size_t
-purgeable_size(szone_t *szone, const void *ptr)
-{
- // Only claim our large allocations, leave the shared tiny/small for the helper zone to claim.
- return szone_size_try_large(szone, ptr);
-}
-
-static void *
-purgeable_malloc(szone_t *szone, size_t size) {
- if (size <= szone->large_threshold)
- return szone_malloc(szone->helper_zone, size);
- else
- return szone_malloc(szone, size);
-}
-
-static void *
-purgeable_calloc(szone_t *szone, size_t num_items, size_t size)
-{
- size_t total_bytes = num_items * size;
-
- // Check for overflow of integer multiplication
- if (num_items > 1) {
-#if __LP64__ /* size_t is uint64_t */
- if ((num_items | size) & 0xffffffff00000000ul) {
- // num_items or size equals or exceeds sqrt(2^64) == 2^32, appeal to wider arithmetic
- __uint128_t product = ((__uint128_t)num_items) * ((__uint128_t)size);
- if ((uint64_t)(product >> 64)) // compiles to test on upper register of register pair
- return NULL;
- }
-#else /* size_t is uint32_t */
- if ((num_items | size) & 0xffff0000ul) {
- // num_items or size equals or exceeds sqrt(2^32) == 2^16, appeal to wider arithmetic
- uint64_t product = ((uint64_t)num_items) * ((uint64_t)size);
- if ((uint32_t)(product >> 32)) // compiles to test on upper register of register pair
- return NULL;
- }
-#endif
- }
-
- if (total_bytes <= szone->large_threshold)
- return szone_calloc(szone->helper_zone, 1, total_bytes);
- else
- return szone_calloc(szone, 1, total_bytes);
-}
-
-static void *
-purgeable_valloc(szone_t *szone, size_t size)
-{
- if (size <= szone->large_threshold)
- return szone_valloc(szone->helper_zone, size);
- else
- return szone_valloc(szone, size);
-}
-
-static void
-purgeable_free(szone_t *szone, void *ptr)
-{
- large_entry_t *entry;
-
- SZONE_LOCK(szone);
- entry = large_entry_for_pointer_no_lock(szone, ptr);
- SZONE_UNLOCK(szone);
- if (entry) {
- return free_large(szone, ptr);
- } else {
- return szone_free(szone->helper_zone, ptr);
- }
-}
-
-static void
-purgeable_free_definite_size(szone_t *szone, void *ptr, size_t size)
-{
- if (size <= szone->large_threshold)
- return szone_free_definite_size(szone->helper_zone, ptr, size);
- else
- return szone_free_definite_size(szone, ptr, size);
-}
-
-static void *
-purgeable_realloc(szone_t *szone, void *ptr, size_t new_size)
-{
- size_t old_size;
-
- if (NULL == ptr) {
- // If ptr is a null pointer, realloc() shall be equivalent to malloc() for the specified size.
- return purgeable_malloc(szone, new_size);
- } else if (0 == new_size) {
- // If size is 0 and ptr is not a null pointer, the object pointed to is freed.
- purgeable_free(szone, ptr);
- // If size is 0, either a null pointer or a unique pointer that can be successfully passed
- // to free() shall be returned.
- return purgeable_malloc(szone, 1);
- }
-
- old_size = purgeable_size(szone, ptr); // Now ptr can be safely size()'d
- if (!old_size)
- old_size = szone_size(szone->helper_zone, ptr);
-
- if (!old_size) {
- szone_error(szone, 1, "pointer being reallocated was not allocated", ptr, NULL);
- return NULL;
- }
-
- // Distinguish 4 cases: {oldsize, newsize} x { <= , > large_threshold }
- // and deal with the allocation crossing from the purgeable zone to the helper zone and vice versa.
- if (old_size <= szone->large_threshold) {
- if (new_size <= szone->large_threshold)
- return szone_realloc(szone->helper_zone, ptr, new_size);
- else {
- // allocation crosses from helper to purgeable zone
- void * new_ptr = purgeable_malloc(szone, new_size);
- if (new_ptr) {
- memcpy(new_ptr, ptr, old_size);
- szone_free_definite_size(szone->helper_zone, ptr, old_size);
- }
- return new_ptr; // in state VM_PURGABLE_NONVOLATILE
- }
- } else {
- if (new_size <= szone->large_threshold) {
- // allocation crosses from purgeable to helper zone
- void * new_ptr = szone_malloc(szone->helper_zone, new_size);
- if (new_ptr) {
- memcpy(new_ptr, ptr, new_size);
- purgeable_free_definite_size(szone, ptr, old_size);
- }
- return new_ptr;
- } else {
- void * new_ptr = purgeable_malloc(szone, new_size);
- if (new_ptr) {
- memcpy(new_ptr, ptr, MIN(old_size, new_size));
- purgeable_free_definite_size(szone, ptr, old_size);
- }
- return new_ptr; // in state VM_PURGABLE_NONVOLATILE
- }
- }
- /* NOTREACHED */
-}
-
-static void
-purgeable_destroy(szone_t *szone)
-{
- /* destroy large entries */
- size_t index = szone->num_large_entries;
- large_entry_t *large;
- vm_range_t range_to_deallocate;
-
- while (index--) {
- large = szone->large_entries + index;
- if (large->address) {
- // we deallocate_pages, including guard pages
- deallocate_pages(szone, (void *)(large->address), large->size, szone->debug_flags);
- }
- }
- large_entries_free_no_lock(szone, szone->large_entries, szone->num_large_entries, &range_to_deallocate);
- if (range_to_deallocate.size)
- deallocate_pages(szone, (void *)range_to_deallocate.address, (size_t)range_to_deallocate.size, 0);
-
- /* Now destroy the separate szone region */
- deallocate_pages(szone, (void *)szone, SZONE_PAGED_SIZE, 0);
-}
-
-static unsigned
-purgeable_batch_malloc(szone_t *szone, size_t size, void **results, unsigned count)
-{
- return szone_batch_malloc(szone->helper_zone, size, results, count);
-}
-
-static void
-purgeable_batch_free(szone_t *szone, void **to_be_freed, unsigned count)
-{
- return szone_batch_free(szone->helper_zone, to_be_freed, count);
-}
-
-static void *
-purgeable_memalign(szone_t *szone, size_t alignment, size_t size)
-{
- if (size <= szone->large_threshold)
- return szone_memalign(szone->helper_zone, alignment, size);
- else
- return szone_memalign(szone, alignment, size);
-}
-
-static kern_return_t
-purgeable_ptr_in_use_enumerator(task_t task, void *context, unsigned type_mask, vm_address_t zone_address,
- memory_reader_t reader, vm_range_recorder_t recorder)
-{
- szone_t *szone;
- kern_return_t err;
-
- if (!reader) reader = _szone_default_reader;
-
- err = reader(task, zone_address, sizeof(szone_t), (void **)&szone);
- if (err) return err;
-
- err = large_in_use_enumerator(task, context, type_mask,
- (vm_address_t)szone->large_entries, szone->num_large_entries, reader, recorder);
- return err;
-}
-
-static size_t
-purgeable_good_size(szone_t *szone, size_t size)
-{
- if (size <= szone->large_threshold)
- return szone_good_size(szone->helper_zone, size);
- else
- return szone_good_size(szone, size);
-}
-
-static boolean_t
-purgeable_check(szone_t *szone)
-{
- return 1;
-}
-
-static void
-purgeable_print(szone_t *szone, boolean_t verbose)
-{
- _malloc_printf(MALLOC_PRINTF_NOLOG | MALLOC_PRINTF_NOPREFIX,
- "Scalable zone %p: inUse=%u(%y) flags=%d\n",
- szone, szone->num_large_objects_in_use, szone->num_bytes_in_large_objects, szone->debug_flags);
-}
-
-static void
-purgeable_log(malloc_zone_t *zone, void *log_address)
-{
- szone_t *szone = (szone_t *)zone;
-
- szone->log_address = log_address;
-}
-
-static void
-purgeable_force_lock(szone_t *szone)
-{
- SZONE_LOCK(szone);
-}
-
-static void
-purgeable_force_unlock(szone_t *szone)
-{
- SZONE_UNLOCK(szone);
-}
-
-static void
-purgeable_statistics(szone_t *szone, malloc_statistics_t *stats)
-{
- stats->blocks_in_use = szone->num_large_objects_in_use;
- stats->size_in_use = stats->max_size_in_use = stats->size_allocated = szone->num_bytes_in_large_objects;
-}
-
-static boolean_t
-purgeable_locked(szone_t *szone)
-{
- int tookLock;
-
- tookLock = SZONE_TRY_LOCK(szone);
- if (tookLock == 0)
- return 1;
- SZONE_UNLOCK(szone);
- return 0;
-}
-
-static size_t
-purgeable_pressure_relief(szone_t *szone, size_t goal)
-{
- return szone_pressure_relief(szone, goal) + szone_pressure_relief(szone->helper_zone, goal);
-}
-
-static const struct malloc_introspection_t purgeable_introspect = {
- (void *)purgeable_ptr_in_use_enumerator,
- (void *)purgeable_good_size,
- (void *)purgeable_check,
- (void *)purgeable_print,
- purgeable_log,
- (void *)purgeable_force_lock,
- (void *)purgeable_force_unlock,
- (void *)purgeable_statistics,
- (void *)purgeable_locked,
- NULL, NULL, NULL, NULL, /* Zone enumeration version 7 and forward. */
-}; // marked as const to spare the DATA section
-
-__private_extern__ malloc_zone_t *
-create_purgeable_zone(size_t initial_size, malloc_zone_t *malloc_default_zone, unsigned debug_flags)
-{
- szone_t *szone;
- uint64_t hw_memsize = 0;
-
- /* get memory for the zone. */
- szone = allocate_pages(NULL, SZONE_PAGED_SIZE, 0, 0, VM_MEMORY_MALLOC);
- if (!szone)
- return NULL;
-
- /* set up the szone structure */
-#if 0
-#warning LOG enabled
- szone->log_address = ~0;
-#endif
-
-#if defined(__i386__) || defined(__x86_64__) || defined(__arm__)
- hw_memsize = *(uint64_t *)(uintptr_t)_COMM_PAGE_MEMORY_SIZE;
-#else
- size_t uint64_t_size = sizeof(hw_memsize);
-
- sysctlbyname("hw.memsize", &hw_memsize, &uint64_t_size, 0, 0);
-#endif
- /* Purgeable zone does not participate in the adaptive "largemem" sizing. */
- szone->is_largemem = 0;
- szone->large_threshold = LARGE_THRESHOLD;
- szone->vm_copy_threshold = VM_COPY_THRESHOLD;
-
-#if LARGE_CACHE
- szone->large_entry_cache_reserve_limit =
- hw_memsize >> 10; // madvise(..., MADV_REUSABLE) death-row arrivals above this threshold [~0.1%]
-
- /* <rdar://problem/6610904> Reset protection when returning a previous large allocation? */
- int32_t libSystemVersion = NSVersionOfLinkTimeLibrary("System");
- if ((-1 != libSystemVersion) && ((libSystemVersion >> 16) < 112) /* CFSystemVersionSnowLeopard */)
- szone->large_legacy_reset_mprotect = TRUE;
- else
- szone->large_legacy_reset_mprotect = FALSE;
-#endif
-
- szone->basic_zone.version = 8;
- szone->basic_zone.size = (void *)purgeable_size;
- szone->basic_zone.malloc = (void *)purgeable_malloc;
- szone->basic_zone.calloc = (void *)purgeable_calloc;
- szone->basic_zone.valloc = (void *)purgeable_valloc;
- szone->basic_zone.free = (void *)purgeable_free;
- szone->basic_zone.realloc = (void *)purgeable_realloc;
- szone->basic_zone.destroy = (void *)purgeable_destroy;
- szone->basic_zone.batch_malloc = (void *)purgeable_batch_malloc;
- szone->basic_zone.batch_free = (void *)purgeable_batch_free;
- szone->basic_zone.introspect = (struct malloc_introspection_t *)&purgeable_introspect;
- szone->basic_zone.memalign = (void *)purgeable_memalign;
- szone->basic_zone.free_definite_size = (void *)purgeable_free_definite_size;
- szone->basic_zone.pressure_relief = (void *)purgeable_pressure_relief;
-
- szone->basic_zone.reserved1 = 0; /* Set to zero once and for all as required by CFAllocator. */
- szone->basic_zone.reserved2 = 0; /* Set to zero once and for all as required by CFAllocator. */
- mprotect(szone, sizeof(szone->basic_zone), PROT_READ); /* Prevent overwriting the function pointers in basic_zone. */
-
- szone->debug_flags = debug_flags | SCALABLE_MALLOC_PURGEABLE;
-
- /* Purgeable zone does not support SCALABLE_MALLOC_ADD_GUARD_PAGES. */
- if (szone->debug_flags & SCALABLE_MALLOC_ADD_GUARD_PAGES) {
- _malloc_printf(ASL_LEVEL_INFO, "purgeable zone does not support guard pages\n");
- szone->debug_flags &= ~SCALABLE_MALLOC_ADD_GUARD_PAGES;
- }
-
- LOCK_INIT(szone->large_szone_lock);
-
- szone->helper_zone = (struct szone_s *)malloc_default_zone;
-
- CHECK(szone, __PRETTY_FUNCTION__);
- return (malloc_zone_t *)szone;
-}
-
-/*
- * For use by CheckFix: create a new zone whose behavior is, apart from
- * the use of death-row and per-CPU magazines, that of Leopard.
- */
-static NOINLINE void *
-legacy_valloc(szone_t *szone, size_t size)
-{
- void *ptr;
- size_t num_pages;
-
- num_pages = round_page(size) >> vm_page_shift;
- ptr = large_malloc(szone, num_pages, 0, TRUE);
-#if DEBUG_MALLOC
- if (LOG(szone, ptr))
- malloc_printf("legacy_valloc returned %p\n", ptr);
-#endif
- return ptr;
-}
-
-__private_extern__ malloc_zone_t *
-create_legacy_scalable_zone(size_t initial_size, unsigned debug_flags)
-{
- malloc_zone_t *mzone = create_scalable_zone(initial_size, debug_flags);
- szone_t *szone = (szone_t *)mzone;
-
- if (!szone)
- return NULL;
-
- szone->is_largemem = 0;
- szone->num_small_slots = NUM_SMALL_SLOTS;
- szone->large_threshold = LARGE_THRESHOLD;
- szone->vm_copy_threshold = VM_COPY_THRESHOLD;
-
- mprotect(szone, sizeof(szone->basic_zone), PROT_READ | PROT_WRITE);
- szone->basic_zone.valloc = (void *)legacy_valloc;
- szone->basic_zone.free_definite_size = NULL;
- mprotect(szone, sizeof(szone->basic_zone), PROT_READ);
-
- return mzone;
-}
-
-/********* Support code for emacs unexec ************/
-
-/* History of freezedry version numbers:
- *
- * 1) Old malloc (before the scalable malloc implementation in this file
- * existed).
- * 2) Original freezedrying code for scalable malloc. This code was apparently
- * based on the old freezedrying code and was fundamentally flawed in its
- * assumption that tracking allocated memory regions was adequate to fake
- * operations on freezedried memory. This doesn't work, since scalable
- * malloc does not store flags in front of large page-aligned allocations.
- * 3) Original szone-based freezedrying code.
- * 4) Fresher malloc with tiny zone
- * 5) 32/64bit compatible malloc
- * 6) Metadata within 1MB and 8MB region for tiny and small
- *
- * No version backward compatibility is provided, but the version number does
- * make it possible for malloc_jumpstart() to return an error if the application
- * was freezedried with an older version of malloc.
- */
-#define MALLOC_FREEZEDRY_VERSION 6
-
-typedef struct {
- unsigned version;
- unsigned nszones;
- szone_t *szones;
-} malloc_frozen;
-
-static void *
-frozen_malloc(szone_t *zone, size_t new_size)
-{
- return malloc(new_size);
-}
-
-static void *
-frozen_calloc(szone_t *zone, size_t num_items, size_t size)
-{
- return calloc(num_items, size);
-}
-
-static void *
-frozen_valloc(szone_t *zone, size_t new_size)
-{
- return valloc(new_size);
-}
-
-static void *
-frozen_realloc(szone_t *zone, void *ptr, size_t new_size)
-{
- size_t old_size = szone_size(zone, ptr);
- void *new_ptr;
-
- if (new_size <= old_size) {
- return ptr;
- }
- new_ptr = malloc(new_size);
- if (old_size > 0) {
- memcpy(new_ptr, ptr, old_size);
- }
- return new_ptr;
-}
-
-static void
-frozen_free(szone_t *zone, void *ptr)
-{
-}
-
-static void
-frozen_destroy(szone_t *zone)
-{
-}
-
-/********* Pseudo-private API for emacs unexec ************/
-
-/*
- * malloc_freezedry() records all of the szones in use, so that they can be
- * partially reconstituted by malloc_jumpstart(). Due to the differences
- * between reconstituted memory regions and those created by the szone code,
- * care is taken not to reallocate from the freezedried memory, except in the
- * case of a non-growing realloc().
- *
- * Due to the flexibility provided by the zone registration mechanism, it is
- * impossible to implement generic freezedrying for any zone type. This code
- * only handles applications that use the szone allocator, so malloc_freezedry()
- * returns 0 (error) if any non-szone zones are encountered.
- */
-
-uintptr_t
-malloc_freezedry(void)
-{
- extern unsigned malloc_num_zones;
- extern malloc_zone_t **malloc_zones;
- malloc_frozen *data;
- unsigned i;
-
- /* Allocate space in which to store the freezedry state. */
- data = (malloc_frozen *) malloc(sizeof(malloc_frozen));
-
- /* Set freezedry version number so that malloc_jumpstart() can check for compatibility. */
- data->version = MALLOC_FREEZEDRY_VERSION;
-
- /* Allocate the array of szone pointers. */
- data->nszones = malloc_num_zones;
- data->szones = (szone_t *) calloc(malloc_num_zones, sizeof(szone_t));
-
- /*
- * Fill in the array of szone structures. They are copied rather than
- * referenced, since the originals are likely to be clobbered during malloc
- * initialization.
- */
- for (i = 0; i < malloc_num_zones; i++) {
- if (strcmp(malloc_zones[i]->zone_name, "DefaultMallocZone")) {
- /* Unknown zone type. */
- free(data->szones);
- free(data);
- return 0;
- }
- memcpy(&data->szones[i], malloc_zones[i], sizeof(szone_t));
- }
-
- return((uintptr_t)data);
-}
-
-int
-malloc_jumpstart(uintptr_t cookie)
-{
- malloc_frozen *data = (malloc_frozen *)cookie;
- unsigned i;
-
- if (data->version != MALLOC_FREEZEDRY_VERSION) {
- /* Unsupported freezedry version. */
- return 1;
- }
-
- for (i = 0; i < data->nszones; i++) {
- /* Set function pointers. Even the functions that stay the same must be
- * set, since there are no guarantees that they will be mapped to the
- * same addresses. */
- data->szones[i].basic_zone.size = (void *) szone_size;
- data->szones[i].basic_zone.malloc = (void *) frozen_malloc;
- data->szones[i].basic_zone.calloc = (void *) frozen_calloc;
- data->szones[i].basic_zone.valloc = (void *) frozen_valloc;
- data->szones[i].basic_zone.free = (void *) frozen_free;
- data->szones[i].basic_zone.realloc = (void *) frozen_realloc;
- data->szones[i].basic_zone.destroy = (void *) frozen_destroy;
- data->szones[i].basic_zone.introspect = (struct malloc_introspection_t *)&szone_introspect;
-
- /* Register the freezedried zone. */
- malloc_zone_register(&data->szones[i].basic_zone);
- }
-
- return 0;
-}
projects / apple / libc.git / commitdiff
