deps/jemalloc.orig/include/jemalloc/internal/jemalloc_internal.h.in

   1 #include <sys/mman.h>
   2 #include <sys/param.h>
   3 #include <sys/time.h>
   4 #include <sys/types.h>
   5 #include <sys/sysctl.h>
   6 #include <sys/uio.h>
   7
   8 #include <errno.h>
   9 #include <limits.h>
  10 #ifndef SIZE_T_MAX
  11 #  define SIZE_T_MAX    SIZE_MAX
  12 #endif
  13 #include <pthread.h>
  14 #include <sched.h>
  15 #include <stdarg.h>
  16 #include <stdbool.h>
  17 #include <stdio.h>
  18 #include <stdlib.h>
  19 #include <stdint.h>
  20 #include <stddef.h>
  21 #ifndef offsetof
  22 #  define offsetof(type, member)        ((size_t)&(((type *)NULL)->member))
  23 #endif
  24 #include <inttypes.h>
  25 #include <string.h>
  26 #include <strings.h>
  27 #include <ctype.h>
  28 #include <unistd.h>
  29 #include <fcntl.h>
  30 #include <pthread.h>
  31 #include <math.h>
  32
  33 #define JEMALLOC_MANGLE
  34 #include "../jemalloc@install_suffix@.h"
  35
  36 #include "jemalloc/internal/private_namespace.h"
  37
  38 #if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN))
  39 #include <libkern/OSAtomic.h>
  40 #endif
  41
  42 #ifdef JEMALLOC_ZONE
  43 #include <mach/mach_error.h>
  44 #include <mach/mach_init.h>
  45 #include <mach/vm_map.h>
  46 #include <malloc/malloc.h>
  47 #endif
  48
  49 #ifdef JEMALLOC_LAZY_LOCK
  50 #include <dlfcn.h>
  51 #endif
  52
  53 #define RB_COMPACT
  54 #include "jemalloc/internal/rb.h"
  55 #include "jemalloc/internal/qr.h"
  56 #include "jemalloc/internal/ql.h"
  57
  58 extern void     (*JEMALLOC_P(malloc_message))(void *wcbopaque, const char *s);
  59
  60 /*
  61  * Define a custom assert() in order to reduce the chances of deadlock during
  62  * assertion failure.
  63  */
  64 #ifndef assert
  65 #  ifdef JEMALLOC_DEBUG
  66 #    define assert(e) do {                                              \
  67         if (!(e)) {                                                     \
  68                 char line_buf[UMAX2S_BUFSIZE];                          \
  69                 malloc_write("<jemalloc>: ");                           \
  70                 malloc_write(__FILE__);                                 \
  71                 malloc_write(":");                                      \
  72                 malloc_write(u2s(__LINE__, 10, line_buf));              \
  73                 malloc_write(": Failed assertion: ");                   \
  74                 malloc_write("\"");                                     \
  75                 malloc_write(#e);                                       \
  76                 malloc_write("\"\n");                                   \
  77                 abort();                                                \
  78         }                                                               \
  79 } while (0)
  80 #  else
  81 #    define assert(e)
  82 #  endif
  83 #endif
  84
  85 #ifdef JEMALLOC_DEBUG
  86 #  define dassert(e) assert(e)
  87 #else
  88 #  define dassert(e)
  89 #endif
  90
  91 /*
  92  * jemalloc can conceptually be broken into components (arena, tcache, etc.),
  93  * but there are circular dependencies that cannot be broken without
  94  * substantial performance degradation.  In order to reduce the effect on
  95  * visual code flow, read the header files in multiple passes, with one of the
  96  * following cpp variables defined during each pass:
  97  *
  98  *   JEMALLOC_H_TYPES   : Preprocessor-defined constants and psuedo-opaque data
  99  *                        types.
 100  *   JEMALLOC_H_STRUCTS : Data structures.
 101  *   JEMALLOC_H_EXTERNS : Extern data declarations and function prototypes.
 102  *   JEMALLOC_H_INLINES : Inline functions.
 103  */
 104 /******************************************************************************/
 105 #define JEMALLOC_H_TYPES
 106
 107 #define ALLOCM_LG_ALIGN_MASK    ((int)0x3f)
 108
 109 #define ZU(z)   ((size_t)z)
 110
 111 #ifndef __DECONST
 112 #  define       __DECONST(type, var)    ((type)(uintptr_t)(const void *)(var))
 113 #endif
 114
 115 #ifdef JEMALLOC_DEBUG
 116    /* Disable inlining to make debugging easier. */
 117 #  define JEMALLOC_INLINE
 118 #  define inline
 119 #else
 120 #  define JEMALLOC_ENABLE_INLINE
 121 #  define JEMALLOC_INLINE static inline
 122 #endif
 123
 124 /* Size of stack-allocated buffer passed to buferror(). */
 125 #define BUFERROR_BUF            64
 126
 127 /* Minimum alignment of allocations is 2^LG_QUANTUM bytes. */
 128 #ifdef __i386__
 129 #  define LG_QUANTUM            4
 130 #endif
 131 #ifdef __ia64__
 132 #  define LG_QUANTUM            4
 133 #endif
 134 #ifdef __alpha__
 135 #  define LG_QUANTUM            4
 136 #endif
 137 #ifdef __sparc64__
 138 #  define LG_QUANTUM            4
 139 #endif
 140 #if (defined(__amd64__) || defined(__x86_64__))
 141 #  define LG_QUANTUM            4
 142 #endif
 143 #ifdef __arm__
 144 #  define LG_QUANTUM            3
 145 #endif
 146 #ifdef __mips__
 147 #  define LG_QUANTUM            3
 148 #endif
 149 #ifdef __powerpc__
 150 #  define LG_QUANTUM            4
 151 #endif
 152 #ifdef __s390x__
 153 #  define LG_QUANTUM            4
 154 #endif
 155
 156 #define QUANTUM                 ((size_t)(1U << LG_QUANTUM))
 157 #define QUANTUM_MASK            (QUANTUM - 1)
 158
 159 /* Return the smallest quantum multiple that is >= a. */
 160 #define QUANTUM_CEILING(a)                                              \
 161         (((a) + QUANTUM_MASK) & ~QUANTUM_MASK)
 162
 163 #define LONG                    ((size_t)(1U << LG_SIZEOF_LONG))
 164 #define LONG_MASK               (LONG - 1)
 165
 166 /* Return the smallest long multiple that is >= a. */
 167 #define LONG_CEILING(a)                                         \
 168         (((a) + LONG_MASK) & ~LONG_MASK)
 169
 170 #define SIZEOF_PTR              (1U << LG_SIZEOF_PTR)
 171 #define PTR_MASK                (SIZEOF_PTR - 1)
 172
 173 /* Return the smallest (void *) multiple that is >= a. */
 174 #define PTR_CEILING(a)                                          \
 175         (((a) + PTR_MASK) & ~PTR_MASK)
 176
 177 /*
 178  * Maximum size of L1 cache line.  This is used to avoid cache line aliasing.
 179  * In addition, this controls the spacing of cacheline-spaced size classes.
 180  */
 181 #define LG_CACHELINE            6
 182 #define CACHELINE               ((size_t)(1U << LG_CACHELINE))
 183 #define CACHELINE_MASK          (CACHELINE - 1)
 184
 185 /* Return the smallest cacheline multiple that is >= s. */
 186 #define CACHELINE_CEILING(s)                                            \
 187         (((s) + CACHELINE_MASK) & ~CACHELINE_MASK)
 188
 189 /*
 190  * Page size.  STATIC_PAGE_SHIFT is determined by the configure script.  If
 191  * DYNAMIC_PAGE_SHIFT is enabled, only use the STATIC_PAGE_* macros where
 192  * compile-time values are required for the purposes of defining data
 193  * structures.
 194  */
 195 #define STATIC_PAGE_SIZE ((size_t)(1U << STATIC_PAGE_SHIFT))
 196 #define STATIC_PAGE_MASK ((size_t)(STATIC_PAGE_SIZE - 1))
 197
 198 #ifdef PAGE_SHIFT
 199 #  undef PAGE_SHIFT
 200 #endif
 201 #ifdef PAGE_SIZE
 202 #  undef PAGE_SIZE
 203 #endif
 204 #ifdef PAGE_MASK
 205 #  undef PAGE_MASK
 206 #endif
 207
 208 #ifdef DYNAMIC_PAGE_SHIFT
 209 #  define PAGE_SHIFT    lg_pagesize
 210 #  define PAGE_SIZE     pagesize
 211 #  define PAGE_MASK     pagesize_mask
 212 #else
 213 #  define PAGE_SHIFT    STATIC_PAGE_SHIFT
 214 #  define PAGE_SIZE     STATIC_PAGE_SIZE
 215 #  define PAGE_MASK     STATIC_PAGE_MASK
 216 #endif
 217
 218 /* Return the smallest pagesize multiple that is >= s. */
 219 #define PAGE_CEILING(s)                                                 \
 220         (((s) + PAGE_MASK) & ~PAGE_MASK)
 221
 222 #include "jemalloc/internal/atomic.h"
 223 #include "jemalloc/internal/prn.h"
 224 #include "jemalloc/internal/ckh.h"
 225 #include "jemalloc/internal/stats.h"
 226 #include "jemalloc/internal/ctl.h"
 227 #include "jemalloc/internal/mutex.h"
 228 #include "jemalloc/internal/mb.h"
 229 #include "jemalloc/internal/extent.h"
 230 #include "jemalloc/internal/arena.h"
 231 #include "jemalloc/internal/bitmap.h"
 232 #include "jemalloc/internal/base.h"
 233 #include "jemalloc/internal/chunk.h"
 234 #include "jemalloc/internal/huge.h"
 235 #include "jemalloc/internal/rtree.h"
 236 #include "jemalloc/internal/tcache.h"
 237 #include "jemalloc/internal/hash.h"
 238 #ifdef JEMALLOC_ZONE
 239 #include "jemalloc/internal/zone.h"
 240 #endif
 241 #include "jemalloc/internal/prof.h"
 242
 243 #undef JEMALLOC_H_TYPES
 244 /******************************************************************************/
 245 #define JEMALLOC_H_STRUCTS
 246
 247 #include "jemalloc/internal/atomic.h"
 248 #include "jemalloc/internal/prn.h"
 249 #include "jemalloc/internal/ckh.h"
 250 #include "jemalloc/internal/stats.h"
 251 #include "jemalloc/internal/ctl.h"
 252 #include "jemalloc/internal/mutex.h"
 253 #include "jemalloc/internal/mb.h"
 254 #include "jemalloc/internal/bitmap.h"
 255 #include "jemalloc/internal/extent.h"
 256 #include "jemalloc/internal/arena.h"
 257 #include "jemalloc/internal/base.h"
 258 #include "jemalloc/internal/chunk.h"
 259 #include "jemalloc/internal/huge.h"
 260 #include "jemalloc/internal/rtree.h"
 261 #include "jemalloc/internal/tcache.h"
 262 #include "jemalloc/internal/hash.h"
 263 #ifdef JEMALLOC_ZONE
 264 #include "jemalloc/internal/zone.h"
 265 #endif
 266 #include "jemalloc/internal/prof.h"
 267
 268 #ifdef JEMALLOC_STATS
 269 typedef struct {
 270         uint64_t        allocated;
 271         uint64_t        deallocated;
 272 } thread_allocated_t;
 273 #endif
 274
 275 #undef JEMALLOC_H_STRUCTS
 276 /******************************************************************************/
 277 #define JEMALLOC_H_EXTERNS
 278
 279 extern bool     opt_abort;
 280 #ifdef JEMALLOC_FILL
 281 extern bool     opt_junk;
 282 #endif
 283 #ifdef JEMALLOC_SYSV
 284 extern bool     opt_sysv;
 285 #endif
 286 #ifdef JEMALLOC_XMALLOC
 287 extern bool     opt_xmalloc;
 288 #endif
 289 #ifdef JEMALLOC_FILL
 290 extern bool     opt_zero;
 291 #endif
 292 extern size_t   opt_narenas;
 293
 294 #ifdef DYNAMIC_PAGE_SHIFT
 295 extern size_t           pagesize;
 296 extern size_t           pagesize_mask;
 297 extern size_t           lg_pagesize;
 298 #endif
 299
 300 /* Number of CPUs. */
 301 extern unsigned         ncpus;
 302
 303 extern malloc_mutex_t   arenas_lock; /* Protects arenas initialization. */
 304 extern pthread_key_t    arenas_tsd;
 305 #ifndef NO_TLS
 306 /*
 307  * Map of pthread_self() --> arenas[???], used for selecting an arena to use
 308  * for allocations.
 309  */
 310 extern __thread arena_t *arenas_tls JEMALLOC_ATTR(tls_model("initial-exec"));
 311 #  define ARENA_GET()   arenas_tls
 312 #  define ARENA_SET(v)  do {                                            \
 313         arenas_tls = (v);                                               \
 314         pthread_setspecific(arenas_tsd, (void *)(v));                   \
 315 } while (0)
 316 #else
 317 #  define ARENA_GET()   ((arena_t *)pthread_getspecific(arenas_tsd))
 318 #  define ARENA_SET(v)  do {                                            \
 319         pthread_setspecific(arenas_tsd, (void *)(v));                   \
 320 } while (0)
 321 #endif
 322
 323 /*
 324  * Arenas that are used to service external requests.  Not all elements of the
 325  * arenas array are necessarily used; arenas are created lazily as needed.
 326  */
 327 extern arena_t          **arenas;
 328 extern unsigned         narenas;
 329
 330 #ifdef JEMALLOC_STATS
 331 #  ifndef NO_TLS
 332 extern __thread thread_allocated_t      thread_allocated_tls;
 333 #    define ALLOCATED_GET() (thread_allocated_tls.allocated)
 334 #    define ALLOCATEDP_GET() (&thread_allocated_tls.allocated)
 335 #    define DEALLOCATED_GET() (thread_allocated_tls.deallocated)
 336 #    define DEALLOCATEDP_GET() (&thread_allocated_tls.deallocated)
 337 #    define ALLOCATED_ADD(a, d) do {                                    \
 338         thread_allocated_tls.allocated += a;                            \
 339         thread_allocated_tls.deallocated += d;                          \
 340 } while (0)
 341 #  else
 342 extern pthread_key_t    thread_allocated_tsd;
 343 thread_allocated_t      *thread_allocated_get_hard(void);
 344
 345 #    define ALLOCATED_GET() (thread_allocated_get()->allocated)
 346 #    define ALLOCATEDP_GET() (&thread_allocated_get()->allocated)
 347 #    define DEALLOCATED_GET() (thread_allocated_get()->deallocated)
 348 #    define DEALLOCATEDP_GET() (&thread_allocated_get()->deallocated)
 349 #    define ALLOCATED_ADD(a, d) do {                                    \
 350         thread_allocated_t *thread_allocated = thread_allocated_get();  \
 351         thread_allocated->allocated += (a);                             \
 352         thread_allocated->deallocated += (d);                           \
 353 } while (0)
 354 #  endif
 355 #endif
 356
 357 arena_t *arenas_extend(unsigned ind);
 358 arena_t *choose_arena_hard(void);
 359 int     buferror(int errnum, char *buf, size_t buflen);
 360 void    jemalloc_prefork(void);
 361 void    jemalloc_postfork(void);
 362
 363 #include "jemalloc/internal/atomic.h"
 364 #include "jemalloc/internal/prn.h"
 365 #include "jemalloc/internal/ckh.h"
 366 #include "jemalloc/internal/stats.h"
 367 #include "jemalloc/internal/ctl.h"
 368 #include "jemalloc/internal/mutex.h"
 369 #include "jemalloc/internal/mb.h"
 370 #include "jemalloc/internal/bitmap.h"
 371 #include "jemalloc/internal/extent.h"
 372 #include "jemalloc/internal/arena.h"
 373 #include "jemalloc/internal/base.h"
 374 #include "jemalloc/internal/chunk.h"
 375 #include "jemalloc/internal/huge.h"
 376 #include "jemalloc/internal/rtree.h"
 377 #include "jemalloc/internal/tcache.h"
 378 #include "jemalloc/internal/hash.h"
 379 #ifdef JEMALLOC_ZONE
 380 #include "jemalloc/internal/zone.h"
 381 #endif
 382 #include "jemalloc/internal/prof.h"
 383
 384 #undef JEMALLOC_H_EXTERNS
 385 /******************************************************************************/
 386 #define JEMALLOC_H_INLINES
 387
 388 #include "jemalloc/internal/atomic.h"
 389 #include "jemalloc/internal/prn.h"
 390 #include "jemalloc/internal/ckh.h"
 391 #include "jemalloc/internal/stats.h"
 392 #include "jemalloc/internal/ctl.h"
 393 #include "jemalloc/internal/mutex.h"
 394 #include "jemalloc/internal/mb.h"
 395 #include "jemalloc/internal/extent.h"
 396 #include "jemalloc/internal/base.h"
 397 #include "jemalloc/internal/chunk.h"
 398 #include "jemalloc/internal/huge.h"
 399
 400 #ifndef JEMALLOC_ENABLE_INLINE
 401 size_t  pow2_ceil(size_t x);
 402 size_t  s2u(size_t size);
 403 size_t  sa2u(size_t size, size_t alignment, size_t *run_size_p);
 404 void    malloc_write(const char *s);
 405 arena_t *choose_arena(void);
 406 #  if (defined(JEMALLOC_STATS) && defined(NO_TLS))
 407 thread_allocated_t      *thread_allocated_get(void);
 408 #  endif
 409 #endif
 410
 411 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
 412 /* Compute the smallest power of 2 that is >= x. */
 413 JEMALLOC_INLINE size_t
 414 pow2_ceil(size_t x)
 415 {
 416
 417         x--;
 418         x |= x >> 1;
 419         x |= x >> 2;
 420         x |= x >> 4;
 421         x |= x >> 8;
 422         x |= x >> 16;
 423 #if (LG_SIZEOF_PTR == 3)
 424         x |= x >> 32;
 425 #endif
 426         x++;
 427         return (x);
 428 }
 429
 430 /*
 431  * Compute usable size that would result from allocating an object with the
 432  * specified size.
 433  */
 434 JEMALLOC_INLINE size_t
 435 s2u(size_t size)
 436 {
 437
 438         if (size <= small_maxclass)
 439                 return (arena_bin_info[SMALL_SIZE2BIN(size)].reg_size);
 440         if (size <= arena_maxclass)
 441                 return (PAGE_CEILING(size));
 442         return (CHUNK_CEILING(size));
 443 }
 444
 445 /*
 446  * Compute usable size that would result from allocating an object with the
 447  * specified size and alignment.
 448  */
 449 JEMALLOC_INLINE size_t
 450 sa2u(size_t size, size_t alignment, size_t *run_size_p)
 451 {
 452         size_t usize;
 453
 454         /*
 455          * Round size up to the nearest multiple of alignment.
 456          *
 457          * This done, we can take advantage of the fact that for each small
 458          * size class, every object is aligned at the smallest power of two
 459          * that is non-zero in the base two representation of the size.  For
 460          * example:
 461          *
 462          *   Size |   Base 2 | Minimum alignment
 463          *   -----+----------+------------------
 464          *     96 |  1100000 |  32
 465          *    144 | 10100000 |  32
 466          *    192 | 11000000 |  64
 467          *
 468          * Depending on runtime settings, it is possible that arena_malloc()
 469          * will further round up to a power of two, but that never causes
 470          * correctness issues.
 471          */
 472         usize = (size + (alignment - 1)) & (-alignment);
 473         /*
 474          * (usize < size) protects against the combination of maximal
 475          * alignment and size greater than maximal alignment.
 476          */
 477         if (usize < size) {
 478                 /* size_t overflow. */
 479                 return (0);
 480         }
 481
 482         if (usize <= arena_maxclass && alignment <= PAGE_SIZE) {
 483                 if (usize <= small_maxclass)
 484                         return (arena_bin_info[SMALL_SIZE2BIN(usize)].reg_size);
 485                 return (PAGE_CEILING(usize));
 486         } else {
 487                 size_t run_size;
 488
 489                 /*
 490                  * We can't achieve subpage alignment, so round up alignment
 491                  * permanently; it makes later calculations simpler.
 492                  */
 493                 alignment = PAGE_CEILING(alignment);
 494                 usize = PAGE_CEILING(size);
 495                 /*
 496                  * (usize < size) protects against very large sizes within
 497                  * PAGE_SIZE of SIZE_T_MAX.
 498                  *
 499                  * (usize + alignment < usize) protects against the
 500                  * combination of maximal alignment and usize large enough
 501                  * to cause overflow.  This is similar to the first overflow
 502                  * check above, but it needs to be repeated due to the new
 503                  * usize value, which may now be *equal* to maximal
 504                  * alignment, whereas before we only detected overflow if the
 505                  * original size was *greater* than maximal alignment.
 506                  */
 507                 if (usize < size || usize + alignment < usize) {
 508                         /* size_t overflow. */
 509                         return (0);
 510                 }
 511
 512                 /*
 513                  * Calculate the size of the over-size run that arena_palloc()
 514                  * would need to allocate in order to guarantee the alignment.
 515                  */
 516                 if (usize >= alignment)
 517                         run_size = usize + alignment - PAGE_SIZE;
 518                 else {
 519                         /*
 520                          * It is possible that (alignment << 1) will cause
 521                          * overflow, but it doesn't matter because we also
 522                          * subtract PAGE_SIZE, which in the case of overflow
 523                          * leaves us with a very large run_size.  That causes
 524                          * the first conditional below to fail, which means
 525                          * that the bogus run_size value never gets used for
 526                          * anything important.
 527                          */
 528                         run_size = (alignment << 1) - PAGE_SIZE;
 529                 }
 530                 if (run_size_p != NULL)
 531                         *run_size_p = run_size;
 532
 533                 if (run_size <= arena_maxclass)
 534                         return (PAGE_CEILING(usize));
 535                 return (CHUNK_CEILING(usize));
 536         }
 537 }
 538
 539 /*
 540  * Wrapper around malloc_message() that avoids the need for
 541  * JEMALLOC_P(malloc_message)(...) throughout the code.
 542  */
 543 JEMALLOC_INLINE void
 544 malloc_write(const char *s)
 545 {
 546
 547         JEMALLOC_P(malloc_message)(NULL, s);
 548 }
 549
 550 /*
 551  * Choose an arena based on a per-thread value (fast-path code, calls slow-path
 552  * code if necessary).
 553  */
 554 JEMALLOC_INLINE arena_t *
 555 choose_arena(void)
 556 {
 557         arena_t *ret;
 558
 559         ret = ARENA_GET();
 560         if (ret == NULL) {
 561                 ret = choose_arena_hard();
 562                 assert(ret != NULL);
 563         }
 564
 565         return (ret);
 566 }
 567
 568 #if (defined(JEMALLOC_STATS) && defined(NO_TLS))
 569 JEMALLOC_INLINE thread_allocated_t *
 570 thread_allocated_get(void)
 571 {
 572         thread_allocated_t *thread_allocated = (thread_allocated_t *)
 573             pthread_getspecific(thread_allocated_tsd);
 574
 575         if (thread_allocated == NULL)
 576                 return (thread_allocated_get_hard());
 577         return (thread_allocated);
 578 }
 579 #endif
 580 #endif
 581
 582 #include "jemalloc/internal/bitmap.h"
 583 #include "jemalloc/internal/rtree.h"
 584 #include "jemalloc/internal/tcache.h"
 585 #include "jemalloc/internal/arena.h"
 586 #include "jemalloc/internal/hash.h"
 587 #ifdef JEMALLOC_ZONE
 588 #include "jemalloc/internal/zone.h"
 589 #endif
 590
 591 #ifndef JEMALLOC_ENABLE_INLINE
 592 void    *imalloc(size_t size);
 593 void    *icalloc(size_t size);
 594 void    *ipalloc(size_t usize, size_t alignment, bool zero);
 595 size_t  isalloc(const void *ptr);
 596 #  ifdef JEMALLOC_IVSALLOC
 597 size_t  ivsalloc(const void *ptr);
 598 #  endif
 599 void    idalloc(void *ptr);
 600 void    *iralloc(void *ptr, size_t size, size_t extra, size_t alignment,
 601     bool zero, bool no_move);
 602 #endif
 603
 604 #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_))
 605 JEMALLOC_INLINE void *
 606 imalloc(size_t size)
 607 {
 608
 609         assert(size != 0);
 610
 611         if (size <= arena_maxclass)
 612                 return (arena_malloc(size, false));
 613         else
 614                 return (huge_malloc(size, false));
 615 }
 616
 617 JEMALLOC_INLINE void *
 618 icalloc(size_t size)
 619 {
 620
 621         if (size <= arena_maxclass)
 622                 return (arena_malloc(size, true));
 623         else
 624                 return (huge_malloc(size, true));
 625 }
 626
 627 JEMALLOC_INLINE void *
 628 ipalloc(size_t usize, size_t alignment, bool zero)
 629 {
 630         void *ret;
 631
 632         assert(usize != 0);
 633         assert(usize == sa2u(usize, alignment, NULL));
 634
 635         if (usize <= arena_maxclass && alignment <= PAGE_SIZE)
 636                 ret = arena_malloc(usize, zero);
 637         else {
 638                 size_t run_size
 639 #ifdef JEMALLOC_CC_SILENCE
 640                     = 0
 641 #endif
 642                     ;
 643
 644                 /*
 645                  * Ideally we would only ever call sa2u() once per aligned
 646                  * allocation request, and the caller of this function has
 647                  * already done so once.  However, it's rather burdensome to
 648                  * require every caller to pass in run_size, especially given
 649                  * that it's only relevant to large allocations.  Therefore,
 650                  * just call it again here in order to get run_size.
 651                  */
 652                 sa2u(usize, alignment, &run_size);
 653                 if (run_size <= arena_maxclass) {
 654                         ret = arena_palloc(choose_arena(), usize, run_size,
 655                             alignment, zero);
 656                 } else if (alignment <= chunksize)
 657                         ret = huge_malloc(usize, zero);
 658                 else
 659                         ret = huge_palloc(usize, alignment, zero);
 660         }
 661
 662         assert(((uintptr_t)ret & (alignment - 1)) == 0);
 663         return (ret);
 664 }
 665
 666 JEMALLOC_INLINE size_t
 667 isalloc(const void *ptr)
 668 {
 669         size_t ret;
 670         arena_chunk_t *chunk;
 671
 672         assert(ptr != NULL);
 673
 674         chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
 675         if (chunk != ptr) {
 676                 /* Region. */
 677                 dassert(chunk->arena->magic == ARENA_MAGIC);
 678
 679 #ifdef JEMALLOC_PROF
 680                 ret = arena_salloc_demote(ptr);
 681 #else
 682                 ret = arena_salloc(ptr);
 683 #endif
 684         } else
 685                 ret = huge_salloc(ptr);
 686
 687         return (ret);
 688 }
 689
 690 #ifdef JEMALLOC_IVSALLOC
 691 JEMALLOC_INLINE size_t
 692 ivsalloc(const void *ptr)
 693 {
 694
 695         /* Return 0 if ptr is not within a chunk managed by jemalloc. */
 696         if (rtree_get(chunks_rtree, (uintptr_t)CHUNK_ADDR2BASE(ptr)) == NULL)
 697                 return (0);
 698
 699         return (isalloc(ptr));
 700 }
 701 #endif
 702
 703 JEMALLOC_INLINE void
 704 idalloc(void *ptr)
 705 {
 706         arena_chunk_t *chunk;
 707
 708         assert(ptr != NULL);
 709
 710         chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr);
 711         if (chunk != ptr)
 712                 arena_dalloc(chunk->arena, chunk, ptr);
 713         else
 714                 huge_dalloc(ptr, true);
 715 }
 716
 717 JEMALLOC_INLINE void *
 718 iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero,
 719     bool no_move)
 720 {
 721         void *ret;
 722         size_t oldsize;
 723
 724         assert(ptr != NULL);
 725         assert(size != 0);
 726
 727         oldsize = isalloc(ptr);
 728
 729         if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1))
 730             != 0) {
 731                 size_t usize, copysize;
 732
 733                 /*
 734                  * Existing object alignment is inadquate; allocate new space
 735                  * and copy.
 736                  */
 737                 if (no_move)
 738                         return (NULL);
 739                 usize = sa2u(size + extra, alignment, NULL);
 740                 if (usize == 0)
 741                         return (NULL);
 742                 ret = ipalloc(usize, alignment, zero);
 743                 if (ret == NULL) {
 744                         if (extra == 0)
 745                                 return (NULL);
 746                         /* Try again, without extra this time. */
 747                         usize = sa2u(size, alignment, NULL);
 748                         if (usize == 0)
 749                                 return (NULL);
 750                         ret = ipalloc(usize, alignment, zero);
 751                         if (ret == NULL)
 752                                 return (NULL);
 753                 }
 754                 /*
 755                  * Copy at most size bytes (not size+extra), since the caller
 756                  * has no expectation that the extra bytes will be reliably
 757                  * preserved.
 758                  */
 759                 copysize = (size < oldsize) ? size : oldsize;
 760                 memcpy(ret, ptr, copysize);
 761                 idalloc(ptr);
 762                 return (ret);
 763         }
 764
 765         if (no_move) {
 766                 if (size <= arena_maxclass) {
 767                         return (arena_ralloc_no_move(ptr, oldsize, size,
 768                             extra, zero));
 769                 } else {
 770                         return (huge_ralloc_no_move(ptr, oldsize, size,
 771                             extra));
 772                 }
 773         } else {
 774                 if (size + extra <= arena_maxclass) {
 775                         return (arena_ralloc(ptr, oldsize, size, extra,
 776                             alignment, zero));
 777                 } else {
 778                         return (huge_ralloc(ptr, oldsize, size, extra,
 779                             alignment, zero));
 780                 }
 781         }
 782 }
 783 #endif
 784
 785 #include "jemalloc/internal/prof.h"
 786
 787 #undef JEMALLOC_H_INLINES
 788 /******************************************************************************/