]> git.saurik.com Git - redis.git/blob - deps/jemalloc/src/prof.c
projects / redis.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge pull request #496 from pietern/2.6-makeinstall
[redis.git] / deps / jemalloc / src / prof.c
1 #define JEMALLOC_PROF_C_
2 #include "jemalloc/internal/jemalloc_internal.h"
3 #ifdef JEMALLOC_PROF
4 /******************************************************************************/
5
6 #ifdef JEMALLOC_PROF_LIBUNWIND
7 #define UNW_LOCAL_ONLY
8 #include <libunwind.h>
9 #endif
10
11 #ifdef JEMALLOC_PROF_LIBGCC
12 #include <unwind.h>
13 #endif
14
15 /******************************************************************************/
16 /* Data. */
17
18 bool opt_prof = false;
19 bool opt_prof_active = true;
20 size_t opt_lg_prof_bt_max = LG_PROF_BT_MAX_DEFAULT;
21 size_t opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT;
22 ssize_t opt_lg_prof_interval = LG_PROF_INTERVAL_DEFAULT;
23 bool opt_prof_gdump = false;
24 bool opt_prof_leak = false;
25 bool opt_prof_accum = true;
26 ssize_t opt_lg_prof_tcmax = LG_PROF_TCMAX_DEFAULT;
27 char opt_prof_prefix[PATH_MAX + 1];
28
29 uint64_t prof_interval;
30 bool prof_promote;
31
32 unsigned prof_bt_max;
33
34 #ifndef NO_TLS
35 __thread prof_tdata_t *prof_tdata_tls
36 JEMALLOC_ATTR(tls_model("initial-exec"));
37 #endif
38 pthread_key_t prof_tdata_tsd;
39
40 /*
41 * Global hash of (prof_bt_t *)-->(prof_ctx_t *). This is the master data
42 * structure that knows about all backtraces currently captured.
43 */
44 static ckh_t bt2ctx;
45 static malloc_mutex_t bt2ctx_mtx;
46
47 static malloc_mutex_t prof_dump_seq_mtx;
48 static uint64_t prof_dump_seq;
49 static uint64_t prof_dump_iseq;
50 static uint64_t prof_dump_mseq;
51 static uint64_t prof_dump_useq;
52
53 /*
54 * This buffer is rather large for stack allocation, so use a single buffer for
55 * all profile dumps. The buffer is implicitly protected by bt2ctx_mtx, since
56 * it must be locked anyway during dumping.
57 */
58 static char prof_dump_buf[PROF_DUMP_BUF_SIZE];
59 static unsigned prof_dump_buf_end;
60 static int prof_dump_fd;
61
62 /* Do not dump any profiles until bootstrapping is complete. */
63 static bool prof_booted = false;
64
65 static malloc_mutex_t enq_mtx;
66 static bool enq;
67 static bool enq_idump;
68 static bool enq_gdump;
69
70 /******************************************************************************/
71 /* Function prototypes for non-inline static functions. */
72
73 static prof_bt_t *bt_dup(prof_bt_t *bt);
74 static void bt_destroy(prof_bt_t *bt);
75 #ifdef JEMALLOC_PROF_LIBGCC
76 static _Unwind_Reason_Code prof_unwind_init_callback(
77 struct _Unwind_Context *context, void *arg);
78 static _Unwind_Reason_Code prof_unwind_callback(
79 struct _Unwind_Context *context, void *arg);
80 #endif
81 static bool prof_flush(bool propagate_err);
82 static bool prof_write(const char *s, bool propagate_err);
83 static void prof_ctx_sum(prof_ctx_t *ctx, prof_cnt_t *cnt_all,
84 size_t *leak_nctx);
85 static void prof_ctx_destroy(prof_ctx_t *ctx);
86 static void prof_ctx_merge(prof_ctx_t *ctx, prof_thr_cnt_t *cnt);
87 static bool prof_dump_ctx(prof_ctx_t *ctx, prof_bt_t *bt,
88 bool propagate_err);
89 static bool prof_dump_maps(bool propagate_err);
90 static bool prof_dump(const char *filename, bool leakcheck,
91 bool propagate_err);
92 static void prof_dump_filename(char *filename, char v, int64_t vseq);
93 static void prof_fdump(void);
94 static void prof_bt_hash(const void *key, unsigned minbits, size_t *hash1,
95 size_t *hash2);
96 static bool prof_bt_keycomp(const void *k1, const void *k2);
97 static void prof_tdata_cleanup(void *arg);
98
99 /******************************************************************************/
100
101 void
102 bt_init(prof_bt_t *bt, void **vec)
103 {
104
105 bt->vec = vec;
106 bt->len = 0;
107 }
108
109 static void
110 bt_destroy(prof_bt_t *bt)
111 {
112
113 idalloc(bt);
114 }
115
116 static prof_bt_t *
117 bt_dup(prof_bt_t *bt)
118 {
119 prof_bt_t *ret;
120
121 /*
122 * Create a single allocation that has space for vec immediately
123 * following the prof_bt_t structure. The backtraces that get
124 * stored in the backtrace caches are copied from stack-allocated
125 * temporary variables, so size is known at creation time. Making this
126 * a contiguous object improves cache locality.
127 */
128 ret = (prof_bt_t *)imalloc(QUANTUM_CEILING(sizeof(prof_bt_t)) +
129 (bt->len * sizeof(void *)));
130 if (ret == NULL)
131 return (NULL);
132 ret->vec = (void **)((uintptr_t)ret +
133 QUANTUM_CEILING(sizeof(prof_bt_t)));
134 memcpy(ret->vec, bt->vec, bt->len * sizeof(void *));
135 ret->len = bt->len;
136
137 return (ret);
138 }
139
140 static inline void
141 prof_enter(void)
142 {
143
144 malloc_mutex_lock(&enq_mtx);
145 enq = true;
146 malloc_mutex_unlock(&enq_mtx);
147
148 malloc_mutex_lock(&bt2ctx_mtx);
149 }
150
151 static inline void
152 prof_leave(void)
153 {
154 bool idump, gdump;
155
156 malloc_mutex_unlock(&bt2ctx_mtx);
157
158 malloc_mutex_lock(&enq_mtx);
159 enq = false;
160 idump = enq_idump;
161 enq_idump = false;
162 gdump = enq_gdump;
163 enq_gdump = false;
164 malloc_mutex_unlock(&enq_mtx);
165
166 if (idump)
167 prof_idump();
168 if (gdump)
169 prof_gdump();
170 }
171
172 #ifdef JEMALLOC_PROF_LIBUNWIND
173 void
174 prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max)
175 {
176 unw_context_t uc;
177 unw_cursor_t cursor;
178 unsigned i;
179 int err;
180
181 assert(bt->len == 0);
182 assert(bt->vec != NULL);
183 assert(max <= (1U << opt_lg_prof_bt_max));
184
185 unw_getcontext(&uc);
186 unw_init_local(&cursor, &uc);
187
188 /* Throw away (nignore+1) stack frames, if that many exist. */
189 for (i = 0; i < nignore + 1; i++) {
190 err = unw_step(&cursor);
191 if (err <= 0)
192 return;
193 }
194
195 /*
196 * Iterate over stack frames until there are no more, or until no space
197 * remains in bt.
198 */
199 for (i = 0; i < max; i++) {
200 unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *)&bt->vec[i]);
201 bt->len++;
202 err = unw_step(&cursor);
203 if (err <= 0)
204 break;
205 }
206 }
207 #endif
208 #ifdef JEMALLOC_PROF_LIBGCC
209 static _Unwind_Reason_Code
210 prof_unwind_init_callback(struct _Unwind_Context *context, void *arg)
211 {
212
213 return (_URC_NO_REASON);
214 }
215
216 static _Unwind_Reason_Code
217 prof_unwind_callback(struct _Unwind_Context *context, void *arg)
218 {
219 prof_unwind_data_t *data = (prof_unwind_data_t *)arg;
220
221 if (data->nignore > 0)
222 data->nignore--;
223 else {
224 data->bt->vec[data->bt->len] = (void *)_Unwind_GetIP(context);
225 data->bt->len++;
226 if (data->bt->len == data->max)
227 return (_URC_END_OF_STACK);
228 }
229
230 return (_URC_NO_REASON);
231 }
232
233 void
234 prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max)
235 {
236 prof_unwind_data_t data = {bt, nignore, max};
237
238 _Unwind_Backtrace(prof_unwind_callback, &data);
239 }
240 #endif
241 #ifdef JEMALLOC_PROF_GCC
242 void
243 prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max)
244 {
245 #define BT_FRAME(i) \
246 if ((i) < nignore + max) { \
247 void *p; \
248 if (__builtin_frame_address(i) == 0) \
249 return; \
250 p = __builtin_return_address(i); \
251 if (p == NULL) \
252 return; \
253 if (i >= nignore) { \
254 bt->vec[(i) - nignore] = p; \
255 bt->len = (i) - nignore + 1; \
256 } \
257 } else \
258 return;
259
260 assert(nignore <= 3);
261 assert(max <= (1U << opt_lg_prof_bt_max));
262
263 BT_FRAME(0)
264 BT_FRAME(1)
265 BT_FRAME(2)
266 BT_FRAME(3)
267 BT_FRAME(4)
268 BT_FRAME(5)
269 BT_FRAME(6)
270 BT_FRAME(7)
271 BT_FRAME(8)
272 BT_FRAME(9)
273
274 BT_FRAME(10)
275 BT_FRAME(11)
276 BT_FRAME(12)
277 BT_FRAME(13)
278 BT_FRAME(14)
279 BT_FRAME(15)
280 BT_FRAME(16)
281 BT_FRAME(17)
282 BT_FRAME(18)
283 BT_FRAME(19)
284
285 BT_FRAME(20)
286 BT_FRAME(21)
287 BT_FRAME(22)
288 BT_FRAME(23)
289 BT_FRAME(24)
290 BT_FRAME(25)
291 BT_FRAME(26)
292 BT_FRAME(27)
293 BT_FRAME(28)
294 BT_FRAME(29)
295
296 BT_FRAME(30)
297 BT_FRAME(31)
298 BT_FRAME(32)
299 BT_FRAME(33)
300 BT_FRAME(34)
301 BT_FRAME(35)
302 BT_FRAME(36)
303 BT_FRAME(37)
304 BT_FRAME(38)
305 BT_FRAME(39)
306
307 BT_FRAME(40)
308 BT_FRAME(41)
309 BT_FRAME(42)
310 BT_FRAME(43)
311 BT_FRAME(44)
312 BT_FRAME(45)
313 BT_FRAME(46)
314 BT_FRAME(47)
315 BT_FRAME(48)
316 BT_FRAME(49)
317
318 BT_FRAME(50)
319 BT_FRAME(51)
320 BT_FRAME(52)
321 BT_FRAME(53)
322 BT_FRAME(54)
323 BT_FRAME(55)
324 BT_FRAME(56)
325 BT_FRAME(57)
326 BT_FRAME(58)
327 BT_FRAME(59)
328
329 BT_FRAME(60)
330 BT_FRAME(61)
331 BT_FRAME(62)
332 BT_FRAME(63)
333 BT_FRAME(64)
334 BT_FRAME(65)
335 BT_FRAME(66)
336 BT_FRAME(67)
337 BT_FRAME(68)
338 BT_FRAME(69)
339
340 BT_FRAME(70)
341 BT_FRAME(71)
342 BT_FRAME(72)
343 BT_FRAME(73)
344 BT_FRAME(74)
345 BT_FRAME(75)
346 BT_FRAME(76)
347 BT_FRAME(77)
348 BT_FRAME(78)
349 BT_FRAME(79)
350
351 BT_FRAME(80)
352 BT_FRAME(81)
353 BT_FRAME(82)
354 BT_FRAME(83)
355 BT_FRAME(84)
356 BT_FRAME(85)
357 BT_FRAME(86)
358 BT_FRAME(87)
359 BT_FRAME(88)
360 BT_FRAME(89)
361
362 BT_FRAME(90)
363 BT_FRAME(91)
364 BT_FRAME(92)
365 BT_FRAME(93)
366 BT_FRAME(94)
367 BT_FRAME(95)
368 BT_FRAME(96)
369 BT_FRAME(97)
370 BT_FRAME(98)
371 BT_FRAME(99)
372
373 BT_FRAME(100)
374 BT_FRAME(101)
375 BT_FRAME(102)
376 BT_FRAME(103)
377 BT_FRAME(104)
378 BT_FRAME(105)
379 BT_FRAME(106)
380 BT_FRAME(107)
381 BT_FRAME(108)
382 BT_FRAME(109)
383
384 BT_FRAME(110)
385 BT_FRAME(111)
386 BT_FRAME(112)
387 BT_FRAME(113)
388 BT_FRAME(114)
389 BT_FRAME(115)
390 BT_FRAME(116)
391 BT_FRAME(117)
392 BT_FRAME(118)
393 BT_FRAME(119)
394
395 BT_FRAME(120)
396 BT_FRAME(121)
397 BT_FRAME(122)
398 BT_FRAME(123)
399 BT_FRAME(124)
400 BT_FRAME(125)
401 BT_FRAME(126)
402 BT_FRAME(127)
403
404 /* Extras to compensate for nignore. */
405 BT_FRAME(128)
406 BT_FRAME(129)
407 BT_FRAME(130)
408 #undef BT_FRAME
409 }
410 #endif
411
412 prof_thr_cnt_t *
413 prof_lookup(prof_bt_t *bt)
414 {
415 union {
416 prof_thr_cnt_t *p;
417 void *v;
418 } ret;
419 prof_tdata_t *prof_tdata;
420
421 prof_tdata = PROF_TCACHE_GET();
422 if (prof_tdata == NULL) {
423 prof_tdata = prof_tdata_init();
424 if (prof_tdata == NULL)
425 return (NULL);
426 }
427
428 if (ckh_search(&prof_tdata->bt2cnt, bt, NULL, &ret.v)) {
429 union {
430 prof_bt_t *p;
431 void *v;
432 } btkey;
433 union {
434 prof_ctx_t *p;
435 void *v;
436 } ctx;
437 bool new_ctx;
438
439 /*
440 * This thread's cache lacks bt. Look for it in the global
441 * cache.
442 */
443 prof_enter();
444 if (ckh_search(&bt2ctx, bt, &btkey.v, &ctx.v)) {
445 /* bt has never been seen before. Insert it. */
446 ctx.v = imalloc(sizeof(prof_ctx_t));
447 if (ctx.v == NULL) {
448 prof_leave();
449 return (NULL);
450 }
451 btkey.p = bt_dup(bt);
452 if (btkey.v == NULL) {
453 prof_leave();
454 idalloc(ctx.v);
455 return (NULL);
456 }
457 ctx.p->bt = btkey.p;
458 if (malloc_mutex_init(&ctx.p->lock)) {
459 prof_leave();
460 idalloc(btkey.v);
461 idalloc(ctx.v);
462 return (NULL);
463 }
464 memset(&ctx.p->cnt_merged, 0, sizeof(prof_cnt_t));
465 ql_new(&ctx.p->cnts_ql);
466 if (ckh_insert(&bt2ctx, btkey.v, ctx.v)) {
467 /* OOM. */
468 prof_leave();
469 malloc_mutex_destroy(&ctx.p->lock);
470 idalloc(btkey.v);
471 idalloc(ctx.v);
472 return (NULL);
473 }
474 /*
475 * Artificially raise curobjs, in order to avoid a race
476 * condition with prof_ctx_merge()/prof_ctx_destroy().
477 *
478 * No locking is necessary for ctx here because no other
479 * threads have had the opportunity to fetch it from
480 * bt2ctx yet.
481 */
482 ctx.p->cnt_merged.curobjs++;
483 new_ctx = true;
484 } else {
485 /*
486 * Artificially raise curobjs, in order to avoid a race
487 * condition with prof_ctx_merge()/prof_ctx_destroy().
488 */
489 malloc_mutex_lock(&ctx.p->lock);
490 ctx.p->cnt_merged.curobjs++;
491 malloc_mutex_unlock(&ctx.p->lock);
492 new_ctx = false;
493 }
494 prof_leave();
495
496 /* Link a prof_thd_cnt_t into ctx for this thread. */
497 if (opt_lg_prof_tcmax >= 0 && ckh_count(&prof_tdata->bt2cnt)
498 == (ZU(1) << opt_lg_prof_tcmax)) {
499 assert(ckh_count(&prof_tdata->bt2cnt) > 0);
500 /*
501 * Flush the least recently used cnt in order to keep
502 * bt2cnt from becoming too large.
503 */
504 ret.p = ql_last(&prof_tdata->lru_ql, lru_link);
505 assert(ret.v != NULL);
506 if (ckh_remove(&prof_tdata->bt2cnt, ret.p->ctx->bt,
507 NULL, NULL))
508 assert(false);
509 ql_remove(&prof_tdata->lru_ql, ret.p, lru_link);
510 prof_ctx_merge(ret.p->ctx, ret.p);
511 /* ret can now be re-used. */
512 } else {
513 assert(opt_lg_prof_tcmax < 0 ||
514 ckh_count(&prof_tdata->bt2cnt) < (ZU(1) <<
515 opt_lg_prof_tcmax));
516 /* Allocate and partially initialize a new cnt. */
517 ret.v = imalloc(sizeof(prof_thr_cnt_t));
518 if (ret.p == NULL) {
519 if (new_ctx)
520 prof_ctx_destroy(ctx.p);
521 return (NULL);
522 }
523 ql_elm_new(ret.p, cnts_link);
524 ql_elm_new(ret.p, lru_link);
525 }
526 /* Finish initializing ret. */
527 ret.p->ctx = ctx.p;
528 ret.p->epoch = 0;
529 memset(&ret.p->cnts, 0, sizeof(prof_cnt_t));
530 if (ckh_insert(&prof_tdata->bt2cnt, btkey.v, ret.v)) {
531 if (new_ctx)
532 prof_ctx_destroy(ctx.p);
533 idalloc(ret.v);
534 return (NULL);
535 }
536 ql_head_insert(&prof_tdata->lru_ql, ret.p, lru_link);
537 malloc_mutex_lock(&ctx.p->lock);
538 ql_tail_insert(&ctx.p->cnts_ql, ret.p, cnts_link);
539 ctx.p->cnt_merged.curobjs--;
540 malloc_mutex_unlock(&ctx.p->lock);
541 } else {
542 /* Move ret to the front of the LRU. */
543 ql_remove(&prof_tdata->lru_ql, ret.p, lru_link);
544 ql_head_insert(&prof_tdata->lru_ql, ret.p, lru_link);
545 }
546
547 return (ret.p);
548 }
549
550 static bool
551 prof_flush(bool propagate_err)
552 {
553 bool ret = false;
554 ssize_t err;
555
556 err = write(prof_dump_fd, prof_dump_buf, prof_dump_buf_end);
557 if (err == -1) {
558 if (propagate_err == false) {
559 malloc_write("<jemalloc>: write() failed during heap "
560 "profile flush\n");
561 if (opt_abort)
562 abort();
563 }
564 ret = true;
565 }
566 prof_dump_buf_end = 0;
567
568 return (ret);
569 }
570
571 static bool
572 prof_write(const char *s, bool propagate_err)
573 {
574 unsigned i, slen, n;
575
576 i = 0;
577 slen = strlen(s);
578 while (i < slen) {
579 /* Flush the buffer if it is full. */
580 if (prof_dump_buf_end == PROF_DUMP_BUF_SIZE)
581 if (prof_flush(propagate_err) && propagate_err)
582 return (true);
583
584 if (prof_dump_buf_end + slen <= PROF_DUMP_BUF_SIZE) {
585 /* Finish writing. */
586 n = slen - i;
587 } else {
588 /* Write as much of s as will fit. */
589 n = PROF_DUMP_BUF_SIZE - prof_dump_buf_end;
590 }
591 memcpy(&prof_dump_buf[prof_dump_buf_end], &s[i], n);
592 prof_dump_buf_end += n;
593 i += n;
594 }
595
596 return (false);
597 }
598
599 static void
600 prof_ctx_sum(prof_ctx_t *ctx, prof_cnt_t *cnt_all, size_t *leak_nctx)
601 {
602 prof_thr_cnt_t *thr_cnt;
603 prof_cnt_t tcnt;
604
605 malloc_mutex_lock(&ctx->lock);
606
607 memcpy(&ctx->cnt_summed, &ctx->cnt_merged, sizeof(prof_cnt_t));
608 ql_foreach(thr_cnt, &ctx->cnts_ql, cnts_link) {
609 volatile unsigned *epoch = &thr_cnt->epoch;
610
611 while (true) {
612 unsigned epoch0 = *epoch;
613
614 /* Make sure epoch is even. */
615 if (epoch0 & 1U)
616 continue;
617
618 memcpy(&tcnt, &thr_cnt->cnts, sizeof(prof_cnt_t));
619
620 /* Terminate if epoch didn't change while reading. */
621 if (*epoch == epoch0)
622 break;
623 }
624
625 ctx->cnt_summed.curobjs += tcnt.curobjs;
626 ctx->cnt_summed.curbytes += tcnt.curbytes;
627 if (opt_prof_accum) {
628 ctx->cnt_summed.accumobjs += tcnt.accumobjs;
629 ctx->cnt_summed.accumbytes += tcnt.accumbytes;
630 }
631 }
632
633 if (ctx->cnt_summed.curobjs != 0)
634 (*leak_nctx)++;
635
636 /* Add to cnt_all. */
637 cnt_all->curobjs += ctx->cnt_summed.curobjs;
638 cnt_all->curbytes += ctx->cnt_summed.curbytes;
639 if (opt_prof_accum) {
640 cnt_all->accumobjs += ctx->cnt_summed.accumobjs;
641 cnt_all->accumbytes += ctx->cnt_summed.accumbytes;
642 }
643
644 malloc_mutex_unlock(&ctx->lock);
645 }
646
647 static void
648 prof_ctx_destroy(prof_ctx_t *ctx)
649 {
650
651 /*
652 * Check that ctx is still unused by any thread cache before destroying
653 * it. prof_lookup() artificially raises ctx->cnt_merge.curobjs in
654 * order to avoid a race condition with this function, as does
655 * prof_ctx_merge() in order to avoid a race between the main body of
656 * prof_ctx_merge() and entry into this function.
657 */
658 prof_enter();
659 malloc_mutex_lock(&ctx->lock);
660 if (ql_first(&ctx->cnts_ql) == NULL && ctx->cnt_merged.curobjs == 1) {
661 assert(ctx->cnt_merged.curbytes == 0);
662 assert(ctx->cnt_merged.accumobjs == 0);
663 assert(ctx->cnt_merged.accumbytes == 0);
664 /* Remove ctx from bt2ctx. */
665 if (ckh_remove(&bt2ctx, ctx->bt, NULL, NULL))
666 assert(false);
667 prof_leave();
668 /* Destroy ctx. */
669 malloc_mutex_unlock(&ctx->lock);
670 bt_destroy(ctx->bt);
671 malloc_mutex_destroy(&ctx->lock);
672 idalloc(ctx);
673 } else {
674 /*
675 * Compensate for increment in prof_ctx_merge() or
676 * prof_lookup().
677 */
678 ctx->cnt_merged.curobjs--;
679 malloc_mutex_unlock(&ctx->lock);
680 prof_leave();
681 }
682 }
683
684 static void
685 prof_ctx_merge(prof_ctx_t *ctx, prof_thr_cnt_t *cnt)
686 {
687 bool destroy;
688
689 /* Merge cnt stats and detach from ctx. */
690 malloc_mutex_lock(&ctx->lock);
691 ctx->cnt_merged.curobjs += cnt->cnts.curobjs;
692 ctx->cnt_merged.curbytes += cnt->cnts.curbytes;
693 ctx->cnt_merged.accumobjs += cnt->cnts.accumobjs;
694 ctx->cnt_merged.accumbytes += cnt->cnts.accumbytes;
695 ql_remove(&ctx->cnts_ql, cnt, cnts_link);
696 if (opt_prof_accum == false && ql_first(&ctx->cnts_ql) == NULL &&
697 ctx->cnt_merged.curobjs == 0) {
698 /*
699 * Artificially raise ctx->cnt_merged.curobjs in order to keep
700 * another thread from winning the race to destroy ctx while
701 * this one has ctx->lock dropped. Without this, it would be
702 * possible for another thread to:
703 *
704 * 1) Sample an allocation associated with ctx.
705 * 2) Deallocate the sampled object.
706 * 3) Successfully prof_ctx_destroy(ctx).
707 *
708 * The result would be that ctx no longer exists by the time
709 * this thread accesses it in prof_ctx_destroy().
710 */
711 ctx->cnt_merged.curobjs++;
712 destroy = true;
713 } else
714 destroy = false;
715 malloc_mutex_unlock(&ctx->lock);
716 if (destroy)
717 prof_ctx_destroy(ctx);
718 }
719
720 static bool
721 prof_dump_ctx(prof_ctx_t *ctx, prof_bt_t *bt, bool propagate_err)
722 {
723 char buf[UMAX2S_BUFSIZE];
724 unsigned i;
725
726 if (opt_prof_accum == false && ctx->cnt_summed.curobjs == 0) {
727 assert(ctx->cnt_summed.curbytes == 0);
728 assert(ctx->cnt_summed.accumobjs == 0);
729 assert(ctx->cnt_summed.accumbytes == 0);
730 return (false);
731 }
732
733 if (prof_write(u2s(ctx->cnt_summed.curobjs, 10, buf), propagate_err)
734 || prof_write(": ", propagate_err)
735 || prof_write(u2s(ctx->cnt_summed.curbytes, 10, buf),
736 propagate_err)
737 || prof_write(" [", propagate_err)
738 || prof_write(u2s(ctx->cnt_summed.accumobjs, 10, buf),
739 propagate_err)
740 || prof_write(": ", propagate_err)
741 || prof_write(u2s(ctx->cnt_summed.accumbytes, 10, buf),
742 propagate_err)
743 || prof_write("] @", propagate_err))
744 return (true);
745
746 for (i = 0; i < bt->len; i++) {
747 if (prof_write(" 0x", propagate_err)
748 || prof_write(u2s((uintptr_t)bt->vec[i], 16, buf),
749 propagate_err))
750 return (true);
751 }
752
753 if (prof_write("\n", propagate_err))
754 return (true);
755
756 return (false);
757 }
758
759 static bool
760 prof_dump_maps(bool propagate_err)
761 {
762 int mfd;
763 char buf[UMAX2S_BUFSIZE];
764 char *s;
765 unsigned i, slen;
766 /* /proc/<pid>/maps\0 */
767 char mpath[6 + UMAX2S_BUFSIZE
768 + 5 + 1];
769
770 i = 0;
771
772 s = "/proc/";
773 slen = strlen(s);
774 memcpy(&mpath[i], s, slen);
775 i += slen;
776
777 s = u2s(getpid(), 10, buf);
778 slen = strlen(s);
779 memcpy(&mpath[i], s, slen);
780 i += slen;
781
782 s = "/maps";
783 slen = strlen(s);
784 memcpy(&mpath[i], s, slen);
785 i += slen;
786
787 mpath[i] = '\0';
788
789 mfd = open(mpath, O_RDONLY);
790 if (mfd != -1) {
791 ssize_t nread;
792
793 if (prof_write("\nMAPPED_LIBRARIES:\n", propagate_err) &&
794 propagate_err)
795 return (true);
796 nread = 0;
797 do {
798 prof_dump_buf_end += nread;
799 if (prof_dump_buf_end == PROF_DUMP_BUF_SIZE) {
800 /* Make space in prof_dump_buf before read(). */
801 if (prof_flush(propagate_err) && propagate_err)
802 return (true);
803 }
804 nread = read(mfd, &prof_dump_buf[prof_dump_buf_end],
805 PROF_DUMP_BUF_SIZE - prof_dump_buf_end);
806 } while (nread > 0);
807 close(mfd);
808 } else
809 return (true);
810
811 return (false);
812 }
813
814 static bool
815 prof_dump(const char *filename, bool leakcheck, bool propagate_err)
816 {
817 prof_cnt_t cnt_all;
818 size_t tabind;
819 union {
820 prof_bt_t *p;
821 void *v;
822 } bt;
823 union {
824 prof_ctx_t *p;
825 void *v;
826 } ctx;
827 char buf[UMAX2S_BUFSIZE];
828 size_t leak_nctx;
829
830 prof_enter();
831 prof_dump_fd = creat(filename, 0644);
832 if (prof_dump_fd == -1) {
833 if (propagate_err == false) {
834 malloc_write("<jemalloc>: creat(\"");
835 malloc_write(filename);
836 malloc_write("\", 0644) failed\n");
837 if (opt_abort)
838 abort();
839 }
840 goto ERROR;
841 }
842
843 /* Merge per thread profile stats, and sum them in cnt_all. */
844 memset(&cnt_all, 0, sizeof(prof_cnt_t));
845 leak_nctx = 0;
846 for (tabind = 0; ckh_iter(&bt2ctx, &tabind, NULL, &ctx.v) == false;)
847 prof_ctx_sum(ctx.p, &cnt_all, &leak_nctx);
848
849 /* Dump profile header. */
850 if (prof_write("heap profile: ", propagate_err)
851 || prof_write(u2s(cnt_all.curobjs, 10, buf), propagate_err)
852 || prof_write(": ", propagate_err)
853 || prof_write(u2s(cnt_all.curbytes, 10, buf), propagate_err)
854 || prof_write(" [", propagate_err)
855 || prof_write(u2s(cnt_all.accumobjs, 10, buf), propagate_err)
856 || prof_write(": ", propagate_err)
857 || prof_write(u2s(cnt_all.accumbytes, 10, buf), propagate_err))
858 goto ERROR;
859
860 if (opt_lg_prof_sample == 0) {
861 if (prof_write("] @ heapprofile\n", propagate_err))
862 goto ERROR;
863 } else {
864 if (prof_write("] @ heap_v2/", propagate_err)
865 || prof_write(u2s((uint64_t)1U << opt_lg_prof_sample, 10,
866 buf), propagate_err)
867 || prof_write("\n", propagate_err))
868 goto ERROR;
869 }
870
871 /* Dump per ctx profile stats. */
872 for (tabind = 0; ckh_iter(&bt2ctx, &tabind, &bt.v, &ctx.v)
873 == false;) {
874 if (prof_dump_ctx(ctx.p, bt.p, propagate_err))
875 goto ERROR;
876 }
877
878 /* Dump /proc/<pid>/maps if possible. */
879 if (prof_dump_maps(propagate_err))
880 goto ERROR;
881
882 if (prof_flush(propagate_err))
883 goto ERROR;
884 close(prof_dump_fd);
885 prof_leave();
886
887 if (leakcheck && cnt_all.curbytes != 0) {
888 malloc_write("<jemalloc>: Leak summary: ");
889 malloc_write(u2s(cnt_all.curbytes, 10, buf));
890 malloc_write((cnt_all.curbytes != 1) ? " bytes, " : " byte, ");
891 malloc_write(u2s(cnt_all.curobjs, 10, buf));
892 malloc_write((cnt_all.curobjs != 1) ? " objects, " :
893 " object, ");
894 malloc_write(u2s(leak_nctx, 10, buf));
895 malloc_write((leak_nctx != 1) ? " contexts\n" : " context\n");
896 malloc_write("<jemalloc>: Run pprof on \"");
897 malloc_write(filename);
898 malloc_write("\" for leak detail\n");
899 }
900
901 return (false);
902 ERROR:
903 prof_leave();
904 return (true);
905 }
906
907 #define DUMP_FILENAME_BUFSIZE (PATH_MAX+ UMAX2S_BUFSIZE \
908 + 1 \
909 + UMAX2S_BUFSIZE \
910 + 2 \
911 + UMAX2S_BUFSIZE \
912 + 5 + 1)
913 static void
914 prof_dump_filename(char *filename, char v, int64_t vseq)
915 {
916 char buf[UMAX2S_BUFSIZE];
917 char *s;
918 unsigned i, slen;
919
920 /*
921 * Construct a filename of the form:
922 *
923 * <prefix>.<pid>.<seq>.v<vseq>.heap\0
924 */
925
926 i = 0;
927
928 s = opt_prof_prefix;
929 slen = strlen(s);
930 memcpy(&filename[i], s, slen);
931 i += slen;
932
933 s = ".";
934 slen = strlen(s);
935 memcpy(&filename[i], s, slen);
936 i += slen;
937
938 s = u2s(getpid(), 10, buf);
939 slen = strlen(s);
940 memcpy(&filename[i], s, slen);
941 i += slen;
942
943 s = ".";
944 slen = strlen(s);
945 memcpy(&filename[i], s, slen);
946 i += slen;
947
948 s = u2s(prof_dump_seq, 10, buf);
949 prof_dump_seq++;
950 slen = strlen(s);
951 memcpy(&filename[i], s, slen);
952 i += slen;
953
954 s = ".";
955 slen = strlen(s);
956 memcpy(&filename[i], s, slen);
957 i += slen;
958
959 filename[i] = v;
960 i++;
961
962 if (vseq != 0xffffffffffffffffLLU) {
963 s = u2s(vseq, 10, buf);
964 slen = strlen(s);
965 memcpy(&filename[i], s, slen);
966 i += slen;
967 }
968
969 s = ".heap";
970 slen = strlen(s);
971 memcpy(&filename[i], s, slen);
972 i += slen;
973
974 filename[i] = '\0';
975 }
976
977 static void
978 prof_fdump(void)
979 {
980 char filename[DUMP_FILENAME_BUFSIZE];
981
982 if (prof_booted == false)
983 return;
984
985 if (opt_prof_prefix[0] != '\0') {
986 malloc_mutex_lock(&prof_dump_seq_mtx);
987 prof_dump_filename(filename, 'f', 0xffffffffffffffffLLU);
988 malloc_mutex_unlock(&prof_dump_seq_mtx);
989 prof_dump(filename, opt_prof_leak, false);
990 }
991 }
992
993 void
994 prof_idump(void)
995 {
996 char filename[DUMP_FILENAME_BUFSIZE];
997
998 if (prof_booted == false)
999 return;
1000 malloc_mutex_lock(&enq_mtx);
1001 if (enq) {
1002 enq_idump = true;
1003 malloc_mutex_unlock(&enq_mtx);
1004 return;
1005 }
1006 malloc_mutex_unlock(&enq_mtx);
1007
1008 if (opt_prof_prefix[0] != '\0') {
1009 malloc_mutex_lock(&prof_dump_seq_mtx);
1010 prof_dump_filename(filename, 'i', prof_dump_iseq);
1011 prof_dump_iseq++;
1012 malloc_mutex_unlock(&prof_dump_seq_mtx);
1013 prof_dump(filename, false, false);
1014 }
1015 }
1016
1017 bool
1018 prof_mdump(const char *filename)
1019 {
1020 char filename_buf[DUMP_FILENAME_BUFSIZE];
1021
1022 if (opt_prof == false || prof_booted == false)
1023 return (true);
1024
1025 if (filename == NULL) {
1026 /* No filename specified, so automatically generate one. */
1027 if (opt_prof_prefix[0] == '\0')
1028 return (true);
1029 malloc_mutex_lock(&prof_dump_seq_mtx);
1030 prof_dump_filename(filename_buf, 'm', prof_dump_mseq);
1031 prof_dump_mseq++;
1032 malloc_mutex_unlock(&prof_dump_seq_mtx);
1033 filename = filename_buf;
1034 }
1035 return (prof_dump(filename, false, true));
1036 }
1037
1038 void
1039 prof_gdump(void)
1040 {
1041 char filename[DUMP_FILENAME_BUFSIZE];
1042
1043 if (prof_booted == false)
1044 return;
1045 malloc_mutex_lock(&enq_mtx);
1046 if (enq) {
1047 enq_gdump = true;
1048 malloc_mutex_unlock(&enq_mtx);
1049 return;
1050 }
1051 malloc_mutex_unlock(&enq_mtx);
1052
1053 if (opt_prof_prefix[0] != '\0') {
1054 malloc_mutex_lock(&prof_dump_seq_mtx);
1055 prof_dump_filename(filename, 'u', prof_dump_useq);
1056 prof_dump_useq++;
1057 malloc_mutex_unlock(&prof_dump_seq_mtx);
1058 prof_dump(filename, false, false);
1059 }
1060 }
1061
1062 static void
1063 prof_bt_hash(const void *key, unsigned minbits, size_t *hash1, size_t *hash2)
1064 {
1065 size_t ret1, ret2;
1066 uint64_t h;
1067 prof_bt_t *bt = (prof_bt_t *)key;
1068
1069 assert(minbits <= 32 || (SIZEOF_PTR == 8 && minbits <= 64));
1070 assert(hash1 != NULL);
1071 assert(hash2 != NULL);
1072
1073 h = hash(bt->vec, bt->len * sizeof(void *), 0x94122f335b332aeaLLU);
1074 if (minbits <= 32) {
1075 /*
1076 * Avoid doing multiple hashes, since a single hash provides
1077 * enough bits.
1078 */
1079 ret1 = h & ZU(0xffffffffU);
1080 ret2 = h >> 32;
1081 } else {
1082 ret1 = h;
1083 ret2 = hash(bt->vec, bt->len * sizeof(void *),
1084 0x8432a476666bbc13LLU);
1085 }
1086
1087 *hash1 = ret1;
1088 *hash2 = ret2;
1089 }
1090
1091 static bool
1092 prof_bt_keycomp(const void *k1, const void *k2)
1093 {
1094 const prof_bt_t *bt1 = (prof_bt_t *)k1;
1095 const prof_bt_t *bt2 = (prof_bt_t *)k2;
1096
1097 if (bt1->len != bt2->len)
1098 return (false);
1099 return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0);
1100 }
1101
1102 prof_tdata_t *
1103 prof_tdata_init(void)
1104 {
1105 prof_tdata_t *prof_tdata;
1106
1107 /* Initialize an empty cache for this thread. */
1108 prof_tdata = (prof_tdata_t *)imalloc(sizeof(prof_tdata_t));
1109 if (prof_tdata == NULL)
1110 return (NULL);
1111
1112 if (ckh_new(&prof_tdata->bt2cnt, PROF_CKH_MINITEMS,
1113 prof_bt_hash, prof_bt_keycomp)) {
1114 idalloc(prof_tdata);
1115 return (NULL);
1116 }
1117 ql_new(&prof_tdata->lru_ql);
1118
1119 prof_tdata->vec = imalloc(sizeof(void *) * prof_bt_max);
1120 if (prof_tdata->vec == NULL) {
1121 ckh_delete(&prof_tdata->bt2cnt);
1122 idalloc(prof_tdata);
1123 return (NULL);
1124 }
1125
1126 prof_tdata->prn_state = 0;
1127 prof_tdata->threshold = 0;
1128 prof_tdata->accum = 0;
1129
1130 PROF_TCACHE_SET(prof_tdata);
1131
1132 return (prof_tdata);
1133 }
1134
1135 static void
1136 prof_tdata_cleanup(void *arg)
1137 {
1138 prof_thr_cnt_t *cnt;
1139 prof_tdata_t *prof_tdata = (prof_tdata_t *)arg;
1140
1141 /*
1142 * Delete the hash table. All of its contents can still be iterated
1143 * over via the LRU.
1144 */
1145 ckh_delete(&prof_tdata->bt2cnt);
1146
1147 /* Iteratively merge cnt's into the global stats and delete them. */
1148 while ((cnt = ql_last(&prof_tdata->lru_ql, lru_link)) != NULL) {
1149 ql_remove(&prof_tdata->lru_ql, cnt, lru_link);
1150 prof_ctx_merge(cnt->ctx, cnt);
1151 idalloc(cnt);
1152 }
1153
1154 idalloc(prof_tdata->vec);
1155
1156 idalloc(prof_tdata);
1157 PROF_TCACHE_SET(NULL);
1158 }
1159
1160 void
1161 prof_boot0(void)
1162 {
1163
1164 memcpy(opt_prof_prefix, PROF_PREFIX_DEFAULT,
1165 sizeof(PROF_PREFIX_DEFAULT));
1166 }
1167
1168 void
1169 prof_boot1(void)
1170 {
1171
1172 /*
1173 * opt_prof and prof_promote must be in their final state before any
1174 * arenas are initialized, so this function must be executed early.
1175 */
1176
1177 if (opt_prof_leak && opt_prof == false) {
1178 /*
1179 * Enable opt_prof, but in such a way that profiles are never
1180 * automatically dumped.
1181 */
1182 opt_prof = true;
1183 opt_prof_gdump = false;
1184 prof_interval = 0;
1185 } else if (opt_prof) {
1186 if (opt_lg_prof_interval >= 0) {
1187 prof_interval = (((uint64_t)1U) <<
1188 opt_lg_prof_interval);
1189 } else
1190 prof_interval = 0;
1191 }
1192
1193 prof_promote = (opt_prof && opt_lg_prof_sample > PAGE_SHIFT);
1194 }
1195
1196 bool
1197 prof_boot2(void)
1198 {
1199
1200 if (opt_prof) {
1201 if (ckh_new(&bt2ctx, PROF_CKH_MINITEMS, prof_bt_hash,
1202 prof_bt_keycomp))
1203 return (true);
1204 if (malloc_mutex_init(&bt2ctx_mtx))
1205 return (true);
1206 if (pthread_key_create(&prof_tdata_tsd, prof_tdata_cleanup)
1207 != 0) {
1208 malloc_write(
1209 "<jemalloc>: Error in pthread_key_create()\n");
1210 abort();
1211 }
1212
1213 prof_bt_max = (1U << opt_lg_prof_bt_max);
1214 if (malloc_mutex_init(&prof_dump_seq_mtx))
1215 return (true);
1216
1217 if (malloc_mutex_init(&enq_mtx))
1218 return (true);
1219 enq = false;
1220 enq_idump = false;
1221 enq_gdump = false;
1222
1223 if (atexit(prof_fdump) != 0) {
1224 malloc_write("<jemalloc>: Error in atexit()\n");
1225 if (opt_abort)
1226 abort();
1227 }
1228 }
1229
1230 #ifdef JEMALLOC_PROF_LIBGCC
1231 /*
1232 * Cause the backtracing machinery to allocate its internal state
1233 * before enabling profiling.
1234 */
1235 _Unwind_Backtrace(prof_unwind_init_callback, NULL);
1236 #endif
1237
1238 prof_booted = true;
1239
1240 return (false);
1241 }
1242
1243 /******************************************************************************/
1244 #endif /* JEMALLOC_PROF */
Redis += OpenLDAP MDB