deps/jemalloc/src/zone.c

   1 #include "jemalloc/internal/jemalloc_internal.h"
   2 #ifndef JEMALLOC_ZONE
   3 #  error "This source file is for zones on Darwin (OS X)."
   4 #endif
   5
   6 /******************************************************************************/
   7 /* Data. */
   8
   9 static malloc_zone_t zone, szone;
  10 static struct malloc_introspection_t zone_introspect, ozone_introspect;
  11
  12 /******************************************************************************/
  13 /* Function prototypes for non-inline static functions. */
  14
  15 static size_t   zone_size(malloc_zone_t *zone, void *ptr);
  16 static void     *zone_malloc(malloc_zone_t *zone, size_t size);
  17 static void     *zone_calloc(malloc_zone_t *zone, size_t num, size_t size);
  18 static void     *zone_valloc(malloc_zone_t *zone, size_t size);
  19 static void     zone_free(malloc_zone_t *zone, void *ptr);
  20 static void     *zone_realloc(malloc_zone_t *zone, void *ptr, size_t size);
  21 #if (JEMALLOC_ZONE_VERSION >= 6)
  22 static void     *zone_memalign(malloc_zone_t *zone, size_t alignment,
  23     size_t size);
  24 static void     zone_free_definite_size(malloc_zone_t *zone, void *ptr,
  25     size_t size);
  26 #endif
  27 static void     *zone_destroy(malloc_zone_t *zone);
  28 static size_t   zone_good_size(malloc_zone_t *zone, size_t size);
  29 static void     zone_force_lock(malloc_zone_t *zone);
  30 static void     zone_force_unlock(malloc_zone_t *zone);
  31 static size_t   ozone_size(malloc_zone_t *zone, void *ptr);
  32 static void     ozone_free(malloc_zone_t *zone, void *ptr);
  33 static void     *ozone_realloc(malloc_zone_t *zone, void *ptr, size_t size);
  34 static unsigned ozone_batch_malloc(malloc_zone_t *zone, size_t size,
  35     void **results, unsigned num_requested);
  36 static void     ozone_batch_free(malloc_zone_t *zone, void **to_be_freed,
  37     unsigned num);
  38 #if (JEMALLOC_ZONE_VERSION >= 6)
  39 static void     ozone_free_definite_size(malloc_zone_t *zone, void *ptr,
  40     size_t size);
  41 #endif
  42 static void     ozone_force_lock(malloc_zone_t *zone);
  43 static void     ozone_force_unlock(malloc_zone_t *zone);
  44
  45 /******************************************************************************/
  46 /*
  47  * Functions.
  48  */
  49
  50 static size_t
  51 zone_size(malloc_zone_t *zone, void *ptr)
  52 {
  53
  54         /*
  55          * There appear to be places within Darwin (such as setenv(3)) that
  56          * cause calls to this function with pointers that *no* zone owns.  If
  57          * we knew that all pointers were owned by *some* zone, we could split
  58          * our zone into two parts, and use one as the default allocator and
  59          * the other as the default deallocator/reallocator.  Since that will
  60          * not work in practice, we must check all pointers to assure that they
  61          * reside within a mapped chunk before determining size.
  62          */
  63         return (ivsalloc(ptr));
  64 }
  65
  66 static void *
  67 zone_malloc(malloc_zone_t *zone, size_t size)
  68 {
  69
  70         return (JEMALLOC_P(malloc)(size));
  71 }
  72
  73 static void *
  74 zone_calloc(malloc_zone_t *zone, size_t num, size_t size)
  75 {
  76
  77         return (JEMALLOC_P(calloc)(num, size));
  78 }
  79
  80 static void *
  81 zone_valloc(malloc_zone_t *zone, size_t size)
  82 {
  83         void *ret = NULL; /* Assignment avoids useless compiler warning. */
  84
  85         JEMALLOC_P(posix_memalign)(&ret, PAGE_SIZE, size);
  86
  87         return (ret);
  88 }
  89
  90 static void
  91 zone_free(malloc_zone_t *zone, void *ptr)
  92 {
  93
  94         JEMALLOC_P(free)(ptr);
  95 }
  96
  97 static void *
  98 zone_realloc(malloc_zone_t *zone, void *ptr, size_t size)
  99 {
 100
 101         return (JEMALLOC_P(realloc)(ptr, size));
 102 }
 103
 104 #if (JEMALLOC_ZONE_VERSION >= 6)
 105 static void *
 106 zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size)
 107 {
 108         void *ret = NULL; /* Assignment avoids useless compiler warning. */
 109
 110         JEMALLOC_P(posix_memalign)(&ret, alignment, size);
 111
 112         return (ret);
 113 }
 114
 115 static void
 116 zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size)
 117 {
 118
 119         assert(ivsalloc(ptr) == size);
 120         JEMALLOC_P(free)(ptr);
 121 }
 122 #endif
 123
 124 static void *
 125 zone_destroy(malloc_zone_t *zone)
 126 {
 127
 128         /* This function should never be called. */
 129         assert(false);
 130         return (NULL);
 131 }
 132
 133 static size_t
 134 zone_good_size(malloc_zone_t *zone, size_t size)
 135 {
 136         size_t ret;
 137         void *p;
 138
 139         /*
 140          * Actually create an object of the appropriate size, then find out
 141          * how large it could have been without moving up to the next size
 142          * class.
 143          */
 144         p = JEMALLOC_P(malloc)(size);
 145         if (p != NULL) {
 146                 ret = isalloc(p);
 147                 JEMALLOC_P(free)(p);
 148         } else
 149                 ret = size;
 150
 151         return (ret);
 152 }
 153
 154 static void
 155 zone_force_lock(malloc_zone_t *zone)
 156 {
 157
 158         if (isthreaded)
 159                 jemalloc_prefork();
 160 }
 161
 162 static void
 163 zone_force_unlock(malloc_zone_t *zone)
 164 {
 165
 166         if (isthreaded)
 167                 jemalloc_postfork();
 168 }
 169
 170 malloc_zone_t *
 171 create_zone(void)
 172 {
 173
 174         zone.size = (void *)zone_size;
 175         zone.malloc = (void *)zone_malloc;
 176         zone.calloc = (void *)zone_calloc;
 177         zone.valloc = (void *)zone_valloc;
 178         zone.free = (void *)zone_free;
 179         zone.realloc = (void *)zone_realloc;
 180         zone.destroy = (void *)zone_destroy;
 181         zone.zone_name = "jemalloc_zone";
 182         zone.batch_malloc = NULL;
 183         zone.batch_free = NULL;
 184         zone.introspect = &zone_introspect;
 185         zone.version = JEMALLOC_ZONE_VERSION;
 186 #if (JEMALLOC_ZONE_VERSION >= 6)
 187         zone.memalign = zone_memalign;
 188         zone.free_definite_size = zone_free_definite_size;
 189 #endif
 190
 191         zone_introspect.enumerator = NULL;
 192         zone_introspect.good_size = (void *)zone_good_size;
 193         zone_introspect.check = NULL;
 194         zone_introspect.print = NULL;
 195         zone_introspect.log = NULL;
 196         zone_introspect.force_lock = (void *)zone_force_lock;
 197         zone_introspect.force_unlock = (void *)zone_force_unlock;
 198         zone_introspect.statistics = NULL;
 199 #if (JEMALLOC_ZONE_VERSION >= 6)
 200         zone_introspect.zone_locked = NULL;
 201 #endif
 202
 203         return (&zone);
 204 }
 205
 206 static size_t
 207 ozone_size(malloc_zone_t *zone, void *ptr)
 208 {
 209         size_t ret;
 210
 211         ret = ivsalloc(ptr);
 212         if (ret == 0)
 213                 ret = szone.size(zone, ptr);
 214
 215         return (ret);
 216 }
 217
 218 static void
 219 ozone_free(malloc_zone_t *zone, void *ptr)
 220 {
 221
 222         if (ivsalloc(ptr) != 0)
 223                 JEMALLOC_P(free)(ptr);
 224         else {
 225                 size_t size = szone.size(zone, ptr);
 226                 if (size != 0)
 227                         (szone.free)(zone, ptr);
 228         }
 229 }
 230
 231 static void *
 232 ozone_realloc(malloc_zone_t *zone, void *ptr, size_t size)
 233 {
 234         size_t oldsize;
 235
 236         if (ptr == NULL)
 237                 return (JEMALLOC_P(malloc)(size));
 238
 239         oldsize = ivsalloc(ptr);
 240         if (oldsize != 0)
 241                 return (JEMALLOC_P(realloc)(ptr, size));
 242         else {
 243                 oldsize = szone.size(zone, ptr);
 244                 if (oldsize == 0)
 245                         return (JEMALLOC_P(malloc)(size));
 246                 else {
 247                         void *ret = JEMALLOC_P(malloc)(size);
 248                         if (ret != NULL) {
 249                                 memcpy(ret, ptr, (oldsize < size) ? oldsize :
 250                                     size);
 251                                 (szone.free)(zone, ptr);
 252                         }
 253                         return (ret);
 254                 }
 255         }
 256 }
 257
 258 static unsigned
 259 ozone_batch_malloc(malloc_zone_t *zone, size_t size, void **results,
 260     unsigned num_requested)
 261 {
 262
 263         /* Don't bother implementing this interface, since it isn't required. */
 264         return (0);
 265 }
 266
 267 static void
 268 ozone_batch_free(malloc_zone_t *zone, void **to_be_freed, unsigned num)
 269 {
 270         unsigned i;
 271
 272         for (i = 0; i < num; i++)
 273                 ozone_free(zone, to_be_freed[i]);
 274 }
 275
 276 #if (JEMALLOC_ZONE_VERSION >= 6)
 277 static void
 278 ozone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size)
 279 {
 280
 281         if (ivsalloc(ptr) != 0) {
 282                 assert(ivsalloc(ptr) == size);
 283                 JEMALLOC_P(free)(ptr);
 284         } else {
 285                 assert(size == szone.size(zone, ptr));
 286                 szone.free_definite_size(zone, ptr, size);
 287         }
 288 }
 289 #endif
 290
 291 static void
 292 ozone_force_lock(malloc_zone_t *zone)
 293 {
 294
 295         /* jemalloc locking is taken care of by the normal jemalloc zone. */
 296         szone.introspect->force_lock(zone);
 297 }
 298
 299 static void
 300 ozone_force_unlock(malloc_zone_t *zone)
 301 {
 302
 303         /* jemalloc locking is taken care of by the normal jemalloc zone. */
 304         szone.introspect->force_unlock(zone);
 305 }
 306
 307 /*
 308  * Overlay the default scalable zone (szone) such that existing allocations are
 309  * drained, and further allocations come from jemalloc.  This is necessary
 310  * because Core Foundation directly accesses and uses the szone before the
 311  * jemalloc library is even loaded.
 312  */
 313 void
 314 szone2ozone(malloc_zone_t *zone)
 315 {
 316
 317         /*
 318          * Stash a copy of the original szone so that we can call its
 319          * functions as needed.  Note that the internally, the szone stores its
 320          * bookkeeping data structures immediately following the malloc_zone_t
 321          * header, so when calling szone functions, we need to pass a pointer
 322          * to the original zone structure.
 323          */
 324         memcpy(&szone, zone, sizeof(malloc_zone_t));
 325
 326         zone->size = (void *)ozone_size;
 327         zone->malloc = (void *)zone_malloc;
 328         zone->calloc = (void *)zone_calloc;
 329         zone->valloc = (void *)zone_valloc;
 330         zone->free = (void *)ozone_free;
 331         zone->realloc = (void *)ozone_realloc;
 332         zone->destroy = (void *)zone_destroy;
 333         zone->zone_name = "jemalloc_ozone";
 334         zone->batch_malloc = ozone_batch_malloc;
 335         zone->batch_free = ozone_batch_free;
 336         zone->introspect = &ozone_introspect;
 337         zone->version = JEMALLOC_ZONE_VERSION;
 338 #if (JEMALLOC_ZONE_VERSION >= 6)
 339         zone->memalign = zone_memalign;
 340         zone->free_definite_size = ozone_free_definite_size;
 341 #endif
 342
 343         ozone_introspect.enumerator = NULL;
 344         ozone_introspect.good_size = (void *)zone_good_size;
 345         ozone_introspect.check = NULL;
 346         ozone_introspect.print = NULL;
 347         ozone_introspect.log = NULL;
 348         ozone_introspect.force_lock = (void *)ozone_force_lock;
 349         ozone_introspect.force_unlock = (void *)ozone_force_unlock;
 350         ozone_introspect.statistics = NULL;
 351 #if (JEMALLOC_ZONE_VERSION >= 6)
 352         ozone_introspect.zone_locked = NULL;
 353 #endif
 354 }