+/* This function turns on mbuf leak detection */
+static void
+mleak_activate(void)
+{
+ mleak_table.mleak_sample_factor = MLEAK_SAMPLE_FACTOR;
+ PE_parse_boot_argn("mleak_sample_factor",
+ &mleak_table.mleak_sample_factor,
+ sizeof (mleak_table.mleak_sample_factor));
+
+ if (mleak_table.mleak_sample_factor == 0)
+ mclfindleak = 0;
+
+ if (mclfindleak == 0)
+ return;
+
+ vm_size_t alloc_size =
+ mleak_alloc_buckets * sizeof (struct mallocation);
+ vm_size_t trace_size = mleak_trace_buckets * sizeof (struct mtrace);
+
+ MALLOC(mleak_allocations, struct mallocation *, alloc_size,
+ M_TEMP, M_WAITOK | M_ZERO);
+ VERIFY(mleak_allocations != NULL);
+
+ MALLOC(mleak_traces, struct mtrace *, trace_size,
+ M_TEMP, M_WAITOK | M_ZERO);
+ VERIFY(mleak_traces != NULL);
+
+ MALLOC(mleak_stat, mleak_stat_t *, MLEAK_STAT_SIZE(MLEAK_NUM_TRACES),
+ M_TEMP, M_WAITOK | M_ZERO);
+ VERIFY(mleak_stat != NULL);
+ mleak_stat->ml_cnt = MLEAK_NUM_TRACES;
+#ifdef __LP64__
+ mleak_stat->ml_isaddr64 = 1;
+#endif /* __LP64__ */
+}
+
+static void
+mleak_logger(u_int32_t num, mcache_obj_t *addr, boolean_t alloc)
+{
+ int temp;
+
+ if (mclfindleak == 0)
+ return;
+
+ if (!alloc)
+ return (mleak_free(addr));
+
+ temp = atomic_add_32_ov(&mleak_table.mleak_capture, 1);
+
+ if ((temp % mleak_table.mleak_sample_factor) == 0 && addr != NULL) {
+ uintptr_t bt[MLEAK_STACK_DEPTH];
+ int logged = fastbacktrace(bt, MLEAK_STACK_DEPTH);
+ mleak_log(bt, addr, logged, num);
+ }
+}
+
+/*
+ * This function records the allocation in the mleak_allocations table
+ * and the backtrace in the mleak_traces table; if allocation slot is in use,
+ * replace old allocation with new one if the trace slot is in use, return
+ * (or increment refcount if same trace).
+ */
+static boolean_t
+mleak_log(uintptr_t *bt, mcache_obj_t *addr, uint32_t depth, int num)
+{
+ struct mallocation *allocation;
+ struct mtrace *trace;
+ uint32_t trace_index;
+
+ /* Quit if someone else modifying the tables */
+ if (!lck_mtx_try_lock_spin(mleak_lock)) {
+ mleak_table.total_conflicts++;
+ return (FALSE);
+ }
+
+ allocation = &mleak_allocations[hashaddr((uintptr_t)addr,
+ mleak_alloc_buckets)];
+ trace_index = hashbacktrace(bt, depth, mleak_trace_buckets);
+ trace = &mleak_traces[trace_index];
+
+ VERIFY(allocation <= &mleak_allocations[mleak_alloc_buckets - 1]);
+ VERIFY(trace <= &mleak_traces[mleak_trace_buckets - 1]);
+
+ allocation->hitcount++;
+ trace->hitcount++;
+
+ /*
+ * If the allocation bucket we want is occupied
+ * and the occupier has the same trace, just bail.
+ */
+ if (allocation->element != NULL &&
+ trace_index == allocation->trace_index) {
+ mleak_table.alloc_collisions++;
+ lck_mtx_unlock(mleak_lock);
+ return (TRUE);
+ }
+
+ /*
+ * Store the backtrace in the traces array;
+ * Size of zero = trace bucket is free.
+ */
+ if (trace->allocs > 0 &&
+ bcmp(trace->addr, bt, (depth * sizeof (uintptr_t))) != 0) {
+ /* Different, unique trace, but the same hash! Bail out. */
+ trace->collisions++;
+ mleak_table.trace_collisions++;
+ lck_mtx_unlock(mleak_lock);
+ return (TRUE);
+ } else if (trace->allocs > 0) {
+ /* Same trace, already added, so increment refcount */
+ trace->allocs++;
+ } else {
+ /* Found an unused trace bucket, so record the trace here */
+ if (trace->depth != 0) {
+ /* this slot previously used but not currently in use */
+ mleak_table.trace_overwrites++;
+ }
+ mleak_table.trace_recorded++;
+ trace->allocs = 1;
+ memcpy(trace->addr, bt, (depth * sizeof (uintptr_t)));
+ trace->depth = depth;
+ trace->collisions = 0;
+ }
+
+ /* Step 2: Store the allocation record in the allocations array */
+ if (allocation->element != NULL) {
+ /*
+ * Replace an existing allocation. No need to preserve
+ * because only a subset of the allocations are being
+ * recorded anyway.
+ */
+ mleak_table.alloc_collisions++;
+ } else if (allocation->trace_index != 0) {
+ mleak_table.alloc_overwrites++;
+ }
+ allocation->element = addr;
+ allocation->trace_index = trace_index;
+ allocation->count = num;
+ mleak_table.alloc_recorded++;
+ mleak_table.outstanding_allocs++;
+
+ lck_mtx_unlock(mleak_lock);
+ return (TRUE);
+}
+
+static void
+mleak_free(mcache_obj_t *addr)
+{
+ while (addr != NULL) {
+ struct mallocation *allocation = &mleak_allocations
+ [hashaddr((uintptr_t)addr, mleak_alloc_buckets)];
+
+ if (allocation->element == addr &&
+ allocation->trace_index < mleak_trace_buckets) {
+ lck_mtx_lock_spin(mleak_lock);
+ if (allocation->element == addr &&
+ allocation->trace_index < mleak_trace_buckets) {
+ struct mtrace *trace;
+ trace = &mleak_traces[allocation->trace_index];
+ /* allocs = 0 means trace bucket is unused */
+ if (trace->allocs > 0)
+ trace->allocs--;
+ if (trace->allocs == 0)
+ trace->depth = 0;
+ /* NULL element means alloc bucket is unused */
+ allocation->element = NULL;
+ mleak_table.outstanding_allocs--;
+ }
+ lck_mtx_unlock(mleak_lock);
+ }
+ addr = addr->obj_next;
+ }
+}
+
+static void
+mleak_sort_traces()
+{
+ int i, j, k;
+ struct mtrace *swap;
+
+ for(i = 0; i < MLEAK_NUM_TRACES; i++)
+ mleak_top_trace[i] = NULL;
+
+ for(i = 0, j = 0; j < MLEAK_NUM_TRACES && i < mleak_trace_buckets; i++)
+ {
+ if (mleak_traces[i].allocs <= 0)
+ continue;
+
+ mleak_top_trace[j] = &mleak_traces[i];
+ for (k = j; k > 0; k--) {
+ if (mleak_top_trace[k]->allocs <=
+ mleak_top_trace[k-1]->allocs)
+ break;
+
+ swap = mleak_top_trace[k-1];
+ mleak_top_trace[k-1] = mleak_top_trace[k];
+ mleak_top_trace[k] = swap;
+ }
+ j++;
+ }
+
+ j--;
+ for(; i < mleak_trace_buckets; i++) {
+ if (mleak_traces[i].allocs <= mleak_top_trace[j]->allocs)
+ continue;
+
+ mleak_top_trace[j] = &mleak_traces[i];
+
+ for (k = j; k > 0; k--) {
+ if (mleak_top_trace[k]->allocs <=
+ mleak_top_trace[k-1]->allocs)
+ break;
+
+ swap = mleak_top_trace[k-1];
+ mleak_top_trace[k-1] = mleak_top_trace[k];
+ mleak_top_trace[k] = swap;
+ }
+ }
+}
+
+static void
+mleak_update_stats()
+{
+ mleak_trace_stat_t *mltr;
+ int i;
+
+ VERIFY(mleak_stat != NULL);
+#ifdef __LP64__
+ VERIFY(mleak_stat->ml_isaddr64);
+#else
+ VERIFY(!mleak_stat->ml_isaddr64);
+#endif /* !__LP64__ */
+ VERIFY(mleak_stat->ml_cnt == MLEAK_NUM_TRACES);
+
+ mleak_sort_traces();
+
+ mltr = &mleak_stat->ml_trace[0];
+ bzero(mltr, sizeof (*mltr) * MLEAK_NUM_TRACES);
+ for (i = 0; i < MLEAK_NUM_TRACES; i++) {
+ int j;
+
+ if (mleak_top_trace[i] == NULL ||
+ mleak_top_trace[i]->allocs == 0)
+ continue;
+
+ mltr->mltr_collisions = mleak_top_trace[i]->collisions;
+ mltr->mltr_hitcount = mleak_top_trace[i]->hitcount;
+ mltr->mltr_allocs = mleak_top_trace[i]->allocs;
+ mltr->mltr_depth = mleak_top_trace[i]->depth;
+
+ VERIFY(mltr->mltr_depth <= MLEAK_STACK_DEPTH);
+ for (j = 0; j < mltr->mltr_depth; j++)
+ mltr->mltr_addr[j] = mleak_top_trace[i]->addr[j];
+
+ mltr++;
+ }
+}
+
+static struct mbtypes {
+ int mt_type;
+ const char *mt_name;
+} mbtypes[] = {
+ { MT_DATA, "data" },
+ { MT_OOBDATA, "oob data" },
+ { MT_CONTROL, "ancillary data" },
+ { MT_HEADER, "packet headers" },
+ { MT_SOCKET, "socket structures" },
+ { MT_PCB, "protocol control blocks" },
+ { MT_RTABLE, "routing table entries" },
+ { MT_HTABLE, "IMP host table entries" },
+ { MT_ATABLE, "address resolution tables" },
+ { MT_FTABLE, "fragment reassembly queue headers" },
+ { MT_SONAME, "socket names and addresses" },
+ { MT_SOOPTS, "socket options" },
+ { MT_RIGHTS, "access rights" },
+ { MT_IFADDR, "interface addresses" },
+ { MT_TAG, "packet tags" },
+ { 0, NULL }
+};
+
+#define MBUF_DUMP_BUF_CHK() { \
+ clen -= k; \
+ if (clen < 1) \
+ goto done; \
+ c += k; \
+}
+
+static char *
+mbuf_dump(void)
+{
+ unsigned long totmem = 0, totfree = 0, totmbufs, totused, totpct;
+ u_int32_t m_mbufs = 0, m_clfree = 0, m_bigclfree = 0;
+ u_int32_t m_mbufclfree = 0, m_mbufbigclfree = 0;
+ u_int32_t m_16kclusters = 0, m_16kclfree = 0, m_mbuf16kclfree = 0;
+ int nmbtypes = sizeof (mbstat.m_mtypes) / sizeof (short);
+ uint8_t seen[256];
+ struct mbtypes *mp;
+ mb_class_stat_t *sp;
+ mleak_trace_stat_t *mltr;
+ char *c = mbuf_dump_buf;
+ int i, k, clen = MBUF_DUMP_BUF_SIZE;
+
+ mbuf_dump_buf[0] = '\0';
+
+ /* synchronize all statistics in the mbuf table */
+ mbuf_stat_sync();
+ mbuf_mtypes_sync(TRUE);
+
+ sp = &mb_stat->mbs_class[0];
+ for (i = 0; i < mb_stat->mbs_cnt; i++, sp++) {
+ u_int32_t mem;
+
+ if (m_class(i) == MC_MBUF) {
+ m_mbufs = sp->mbcl_active;
+ } else if (m_class(i) == MC_CL) {
+ m_clfree = sp->mbcl_total - sp->mbcl_active;
+ } else if (m_class(i) == MC_BIGCL) {
+ m_bigclfree = sp->mbcl_total - sp->mbcl_active;
+ } else if (njcl > 0 && m_class(i) == MC_16KCL) {
+ m_16kclfree = sp->mbcl_total - sp->mbcl_active;
+ m_16kclusters = sp->mbcl_total;
+ } else if (m_class(i) == MC_MBUF_CL) {
+ m_mbufclfree = sp->mbcl_total - sp->mbcl_active;
+ } else if (m_class(i) == MC_MBUF_BIGCL) {
+ m_mbufbigclfree = sp->mbcl_total - sp->mbcl_active;
+ } else if (njcl > 0 && m_class(i) == MC_MBUF_16KCL) {
+ m_mbuf16kclfree = sp->mbcl_total - sp->mbcl_active;
+ }
+
+ mem = sp->mbcl_ctotal * sp->mbcl_size;
+ totmem += mem;
+ totfree += (sp->mbcl_mc_cached + sp->mbcl_infree) *
+ sp->mbcl_size;
+
+ }
+
+ /* adjust free counts to include composite caches */
+ m_clfree += m_mbufclfree;
+ m_bigclfree += m_mbufbigclfree;
+ m_16kclfree += m_mbuf16kclfree;
+
+ totmbufs = 0;
+ for (mp = mbtypes; mp->mt_name != NULL; mp++)
+ totmbufs += mbstat.m_mtypes[mp->mt_type];
+ if (totmbufs > m_mbufs)
+ totmbufs = m_mbufs;
+ k = snprintf(c, clen, "%lu/%u mbufs in use:\n", totmbufs, m_mbufs);
+ MBUF_DUMP_BUF_CHK();
+
+ bzero(&seen, sizeof (seen));
+ for (mp = mbtypes; mp->mt_name != NULL; mp++) {
+ if (mbstat.m_mtypes[mp->mt_type] != 0) {
+ seen[mp->mt_type] = 1;
+ k = snprintf(c, clen, "\t%u mbufs allocated to %s\n",
+ mbstat.m_mtypes[mp->mt_type], mp->mt_name);
+ MBUF_DUMP_BUF_CHK();
+ }
+ }
+ seen[MT_FREE] = 1;
+ for (i = 0; i < nmbtypes; i++)
+ if (!seen[i] && mbstat.m_mtypes[i] != 0) {
+ k = snprintf(c, clen, "\t%u mbufs allocated to "
+ "<mbuf type %d>\n", mbstat.m_mtypes[i], i);
+ MBUF_DUMP_BUF_CHK();
+ }
+ if ((m_mbufs - totmbufs) > 0) {
+ k = snprintf(c, clen, "\t%lu mbufs allocated to caches\n",
+ m_mbufs - totmbufs);
+ MBUF_DUMP_BUF_CHK();
+ }
+ k = snprintf(c, clen, "%u/%u mbuf 2KB clusters in use\n"
+ "%u/%u mbuf 4KB clusters in use\n",
+ (unsigned int)(mbstat.m_clusters - m_clfree),
+ (unsigned int)mbstat.m_clusters,
+ (unsigned int)(mbstat.m_bigclusters - m_bigclfree),
+ (unsigned int)mbstat.m_bigclusters);
+ MBUF_DUMP_BUF_CHK();
+
+ if (njcl > 0) {
+ k = snprintf(c, clen, "%u/%u mbuf %uKB clusters in use\n",
+ m_16kclusters - m_16kclfree, m_16kclusters,
+ njclbytes / 1024);
+ MBUF_DUMP_BUF_CHK();
+ }
+ totused = totmem - totfree;
+ if (totmem == 0) {
+ totpct = 0;
+ } else if (totused < (ULONG_MAX / 100)) {
+ totpct = (totused * 100) / totmem;
+ } else {
+ u_long totmem1 = totmem / 100;
+ u_long totused1 = totused / 100;
+ totpct = (totused1 * 100) / totmem1;
+ }
+ k = snprintf(c, clen, "%lu KB allocated to network (approx. %lu%% "
+ "in use)\n", totmem / 1024, totpct);
+ MBUF_DUMP_BUF_CHK();
+
+ /* mbuf leak detection statistics */
+ mleak_update_stats();
+
+ k = snprintf(c, clen, "\nmbuf leak detection table:\n");
+ MBUF_DUMP_BUF_CHK();
+ k = snprintf(c, clen, "\ttotal captured: %u (one per %u)\n",
+ mleak_table.mleak_capture / mleak_table.mleak_sample_factor,
+ mleak_table.mleak_sample_factor);
+ MBUF_DUMP_BUF_CHK();
+ k = snprintf(c, clen, "\ttotal allocs outstanding: %llu\n",
+ mleak_table.outstanding_allocs);
+ MBUF_DUMP_BUF_CHK();
+ k = snprintf(c, clen, "\tnew hash recorded: %llu allocs, %llu traces\n",
+ mleak_table.alloc_recorded, mleak_table.trace_recorded);
+ MBUF_DUMP_BUF_CHK();
+ k = snprintf(c, clen, "\thash collisions: %llu allocs, %llu traces\n",
+ mleak_table.alloc_collisions, mleak_table.trace_collisions);
+ MBUF_DUMP_BUF_CHK();
+ k = snprintf(c, clen, "\toverwrites: %llu allocs, %llu traces\n",
+ mleak_table.alloc_overwrites, mleak_table.trace_overwrites);
+ MBUF_DUMP_BUF_CHK();
+ k = snprintf(c, clen, "\tlock conflicts: %llu\n\n",
+ mleak_table.total_conflicts);
+ MBUF_DUMP_BUF_CHK();
+
+ k = snprintf(c, clen, "top %d outstanding traces:\n",
+ mleak_stat->ml_cnt);
+ MBUF_DUMP_BUF_CHK();
+ for (i = 0; i < mleak_stat->ml_cnt; i++) {
+ mltr = &mleak_stat->ml_trace[i];
+ k = snprintf(c, clen, "[%d] %llu outstanding alloc(s), "
+ "%llu hit(s), %llu collision(s)\n", (i + 1),
+ mltr->mltr_allocs, mltr->mltr_hitcount,
+ mltr->mltr_collisions);
+ MBUF_DUMP_BUF_CHK();
+ }
+
+ if (mleak_stat->ml_isaddr64)
+ k = snprintf(c, clen, MB_LEAK_HDR_64);
+ else
+ k = snprintf(c, clen, MB_LEAK_HDR_32);
+ MBUF_DUMP_BUF_CHK();
+
+ for (i = 0; i < MLEAK_STACK_DEPTH; i++) {
+ int j;
+ k = snprintf(c, clen, "%2d: ", (i + 1));
+ MBUF_DUMP_BUF_CHK();
+ for (j = 0; j < mleak_stat->ml_cnt; j++) {
+ mltr = &mleak_stat->ml_trace[j];
+ if (i < mltr->mltr_depth) {
+ if (mleak_stat->ml_isaddr64) {
+ k = snprintf(c, clen, "0x%0llx ",
+ mltr->mltr_addr[i]);
+ } else {
+ k = snprintf(c, clen,
+ "0x%08x ",
+ (u_int32_t)mltr->mltr_addr[i]);
+ }
+ } else {
+ if (mleak_stat->ml_isaddr64)
+ k = snprintf(c, clen,
+ MB_LEAK_SPACING_64);
+ else
+ k = snprintf(c, clen,
+ MB_LEAK_SPACING_32);
+ }
+ MBUF_DUMP_BUF_CHK();
+ }
+ k = snprintf(c, clen, "\n");
+ MBUF_DUMP_BUF_CHK();
+ }
+done:
+ return (mbuf_dump_buf);
+}
+
+#undef MBUF_DUMP_BUF_CHK
+
projects / apple / xnu.git / blobdiff