/*
- * Copyright (c) 1998-2011 Apple Inc. All rights reserved.
+ * Copyright (c) 1998-2013 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
#include <sys/queue.h>
#include <sys/proc.h>
+#include <dev/random/randomdev.h>
+
#include <kern/kern_types.h>
#include <kern/simple_lock.h>
#include <kern/queue.h>
#include <kern/zalloc.h>
#include <libkern/OSAtomic.h>
+#include <libkern/OSDebug.h>
#include <libkern/libkern.h>
#include <IOKit/IOMapper.h>
extern vm_map_t mb_map; /* special map */
/* Global lock */
-static lck_mtx_t *mbuf_mlock;
+decl_lck_mtx_data(static, mbuf_mlock_data);
+static lck_mtx_t *mbuf_mlock = &mbuf_mlock_data;
static lck_attr_t *mbuf_mlock_attr;
static lck_grp_t *mbuf_mlock_grp;
static lck_grp_attr_t *mbuf_mlock_grp_attr;
mcache_audit_t *cl_audit[NMBPBG]; /* array of audits */
} mcl_audit_t;
-/*
- * Size of data from the beginning of an mbuf that covers m_hdr, pkthdr
- * and m_ext structures. If auditing is enabled, we allocate a shadow
- * mbuf structure of this size inside each audit structure, and the
- * contents of the real mbuf gets copied into it when the mbuf is freed.
- * This allows us to pattern-fill the mbuf for integrity check, and to
- * preserve any constructed mbuf fields (e.g. mbuf + cluster cache case).
- * Note that we don't save the contents of clusters when they are freed;
- * we simply pattern-fill them.
- */
-#define AUDIT_CONTENTS_SIZE ((MSIZE - MHLEN) + sizeof (_m_ext_t))
+typedef struct {
+ struct thread *msa_thread; /* thread doing transaction */
+ struct thread *msa_pthread; /* previous transaction thread */
+ uint32_t msa_tstamp; /* transaction timestamp (ms) */
+ uint32_t msa_ptstamp; /* prev transaction timestamp (ms) */
+ uint16_t msa_depth; /* pc stack depth */
+ uint16_t msa_pdepth; /* previous transaction pc stack */
+ void *msa_stack[MCACHE_STACK_DEPTH];
+ void *msa_pstack[MCACHE_STACK_DEPTH];
+} mcl_scratch_audit_t;
+
+typedef struct {
+ /*
+ * Size of data from the beginning of an mbuf that covers m_hdr,
+ * pkthdr and m_ext structures. If auditing is enabled, we allocate
+ * a shadow mbuf structure of this size inside each audit structure,
+ * and the contents of the real mbuf gets copied into it when the mbuf
+ * is freed. This allows us to pattern-fill the mbuf for integrity
+ * check, and to preserve any constructed mbuf fields (e.g. mbuf +
+ * cluster cache case). Note that we don't save the contents of
+ * clusters when they are freed; we simply pattern-fill them.
+ */
+ u_int8_t sc_mbuf[(MSIZE - _MHLEN) + sizeof (_m_ext_t)];
+ mcl_scratch_audit_t sc_scratch __attribute__((aligned(8)));
+} mcl_saved_contents_t;
+
+#define AUDIT_CONTENTS_SIZE (sizeof (mcl_saved_contents_t))
+
+#define MCA_SAVED_MBUF_PTR(_mca) \
+ ((struct mbuf *)(void *)((mcl_saved_contents_t *) \
+ (_mca)->mca_contents)->sc_mbuf)
+#define MCA_SAVED_MBUF_SIZE \
+ (sizeof (((mcl_saved_contents_t *)0)->sc_mbuf))
+#define MCA_SAVED_SCRATCH_PTR(_mca) \
+ (&((mcl_saved_contents_t *)(_mca)->mca_contents)->sc_scratch)
/*
* mbuf specific mcache audit flags
int njclbytes; /* size of a jumbo cluster */
union mbigcluster *mbutl; /* first mapped cluster address */
union mbigcluster *embutl; /* ending virtual address of mclusters */
-int max_linkhdr; /* largest link-level header */
-int max_protohdr; /* largest protocol header */
+int _max_linkhdr; /* largest link-level header */
+int _max_protohdr; /* largest protocol header */
int max_hdr; /* largest link+protocol header */
int max_datalen; /* MHLEN - max_hdr */
static boolean_t mclverify; /* debug: pattern-checking */
static boolean_t mcltrace; /* debug: stack tracing */
static boolean_t mclfindleak; /* debug: leak detection */
+static boolean_t mclexpleak; /* debug: expose leak info to user space */
+
+static struct timeval mb_start; /* beginning of time */
/* mbuf leak detection variables */
static struct mleak_table mleak_table;
*/
#define MLEAK_NUM_TRACES 5
+#define MB_LEAK_SPACING_64 " "
+#define MB_LEAK_SPACING_32 " "
+
+
+#define MB_LEAK_HDR_32 "\n\
+ trace [1] trace [2] trace [3] trace [4] trace [5] \n\
+ ---------- ---------- ---------- ---------- ---------- \n\
+"
+
+#define MB_LEAK_HDR_64 "\n\
+ trace [1] trace [2] trace [3] \
+ trace [4] trace [5] \n\
+ ------------------ ------------------ ------------------ \
+ ------------------ ------------------ \n\
+"
+
static uint32_t mleak_alloc_buckets = MLEAK_ALLOCATION_MAP_NUM;
static uint32_t mleak_trace_buckets = MLEAK_TRACE_MAP_NUM;
static struct mtrace *mleak_top_trace[MLEAK_NUM_TRACES];
/* Lock to protect mleak tables from concurrent modification */
-static lck_mtx_t *mleak_lock;
+decl_lck_mtx_data(static, mleak_lock_data);
+static lck_mtx_t *mleak_lock = &mleak_lock_data;
static lck_attr_t *mleak_lock_attr;
static lck_grp_t *mleak_lock_grp;
static lck_grp_attr_t *mleak_lock_grp_attr;
extern u_int32_t high_sb_max;
-/* TODO: should be in header file */
-int do_reclaim = 0;
-
/* The minimum number of objects that are allocated, to start. */
#define MINCL 32
#define MINBIGCL (MINCL >> 1)
#define MB_WDT_MAXTIME 10 /* # of secs before watchdog panic */
static struct timeval mb_wdtstart; /* watchdog start timestamp */
-static char mbuf_dump_buf[256];
+static char *mbuf_dump_buf;
+
+#define MBUF_DUMP_BUF_SIZE 2048
/*
* mbuf watchdog is enabled by default on embedded platforms. It is
* also toggeable via the kern.ipc.mb_watchdog sysctl.
*/
-#if CONFIG_EMBEDDED
-static unsigned int mb_watchdog = 1;
-#else
static unsigned int mb_watchdog = 0;
-#endif /* CONFIG_EMBEDDED */
+
+/* Red zone */
+static u_int32_t mb_redzone_cookie;
+static void m_redzone_init(struct mbuf *);
+static void m_redzone_verify(struct mbuf *m);
/* The following are used to serialize m_clalloc() */
static boolean_t mb_clalloc_busy;
boolean_t);
static void mcl_audit_restore_mbuf(struct mbuf *, mcache_audit_t *, boolean_t);
static void mcl_audit_save_mbuf(struct mbuf *, mcache_audit_t *);
+static void mcl_audit_scratch(mcache_audit_t *);
static void mcl_audit_mcheck_panic(struct mbuf *);
static void mcl_audit_verify_nextptr(void *, mcache_audit_t *);
static void mleak_logger(u_int32_t, mcache_obj_t *, boolean_t);
static boolean_t mleak_log(uintptr_t *, mcache_obj_t *, uint32_t, int);
static void mleak_free(mcache_obj_t *);
+static void mleak_sort_traces(void);
+static void mleak_update_stats(void);
static mcl_slab_t *slab_get(void *);
static void slab_init(mcl_slab_t *, mbuf_class_t, u_int32_t,
/*
* Macros used during mbuf and cluster initialization.
*/
+#define MBUF_INIT_PKTHDR(m) { \
+ (m)->m_pkthdr.rcvif = NULL; \
+ (m)->m_pkthdr.pkt_hdr = NULL; \
+ (m)->m_pkthdr.len = 0; \
+ (m)->m_pkthdr.csum_flags = 0; \
+ (m)->m_pkthdr.csum_data = 0; \
+ (m)->m_pkthdr.vlan_tag = 0; \
+ m_classifier_init(m, 0); \
+ m_tag_init(m, 1); \
+ m_scratch_init(m); \
+ m_redzone_init(m); \
+}
+
#define MBUF_INIT(m, pkthdr, type) { \
_MCHECK(m); \
(m)->m_next = (m)->m_nextpkt = NULL; \
} else { \
(m)->m_data = (m)->m_pktdat; \
(m)->m_flags = M_PKTHDR; \
- (m)->m_pkthdr.rcvif = NULL; \
- (m)->m_pkthdr.len = 0; \
- (m)->m_pkthdr.header = NULL; \
- (m)->m_pkthdr.csum_flags = 0; \
- (m)->m_pkthdr.csum_data = 0; \
- (m)->m_pkthdr.tso_segsz = 0; \
- (m)->m_pkthdr.vlan_tag = 0; \
- (m)->m_pkthdr.socket_id = 0; \
- (m)->m_pkthdr.vt_nrecs = 0; \
- m_tag_init(m); \
- m_prio_init(m); \
+ MBUF_INIT_PKTHDR(m); \
} \
}
*/
typedef struct {
unsigned int cpu_mtypes[MT_MAX];
-} __attribute__((aligned(CPU_CACHE_SIZE), packed)) mtypes_cpu_t;
+} __attribute__((aligned(MAX_CPU_CACHE_LINE_SIZE), packed)) mtypes_cpu_t;
typedef struct {
mtypes_cpu_t mbs_cpu[1];
((size_t)(&((mbuf_mtypes_t *)0)->mbs_cpu[n]))
#define MTYPES_CPU(p) \
- ((mtypes_cpu_t *)((char *)(p) + MBUF_MTYPES_SIZE(cpu_number())))
+ ((mtypes_cpu_t *)(void *)((char *)(p) + MBUF_MTYPES_SIZE(cpu_number())))
#define mtype_stat_add(type, n) { \
if ((unsigned)(type) < MT_MAX) { \
mleak_top_trace_sysctl SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
- mleak_trace_stat_t *mltr;
int i;
/* Ensure leak tracing turned on */
- if (!mclfindleak)
+ if (!mclfindleak || !mclexpleak)
return (ENXIO);
- VERIFY(mleak_stat != NULL);
-#ifdef __LP64__
- VERIFY(mleak_stat->ml_isaddr64);
-#else
- VERIFY(!mleak_stat->ml_isaddr64);
-#endif /* !__LP64__ */
- VERIFY(mleak_stat->ml_cnt == MLEAK_NUM_TRACES);
-
lck_mtx_lock(mleak_lock);
- mltr = &mleak_stat->ml_trace[0];
- bzero(mltr, sizeof (*mltr) * MLEAK_NUM_TRACES);
- for (i = 0; i < MLEAK_NUM_TRACES; i++) {
- int j;
-
- if (mleak_top_trace[i] == NULL ||
- mleak_top_trace[i]->allocs == 0)
- continue;
-
- mltr->mltr_collisions = mleak_top_trace[i]->collisions;
- mltr->mltr_hitcount = mleak_top_trace[i]->hitcount;
- mltr->mltr_allocs = mleak_top_trace[i]->allocs;
- mltr->mltr_depth = mleak_top_trace[i]->depth;
-
- VERIFY(mltr->mltr_depth <= MLEAK_STACK_DEPTH);
- for (j = 0; j < mltr->mltr_depth; j++)
- mltr->mltr_addr[j] = mleak_top_trace[i]->addr[j];
-
- mltr++;
- }
+ mleak_update_stats();
i = SYSCTL_OUT(req, mleak_stat, MLEAK_STAT_SIZE(MLEAK_NUM_TRACES));
lck_mtx_unlock(mleak_lock);
int i = 0;
/* Ensure leak tracing turned on */
- if (!mclfindleak)
+ if (!mclfindleak || !mclexpleak)
return (ENXIO);
lck_mtx_lock(mleak_lock);
/* Non-server */
static ncl_tbl_t ncl_table[] = {
- { (1ULL << GBSHIFT) /* 1 GB */, (64 << MBSHIFT) /* 64 MB */ },
+ { (1ULL << GBSHIFT) /* 1 GB */, (64 << MBSHIFT) /* 64 MB */ },
{ (1ULL << (GBSHIFT + 3)) /* 8 GB */, (96 << MBSHIFT) /* 96 MB */ },
{ (1ULL << (GBSHIFT + 4)) /* 16 GB */, (128 << MBSHIFT) /* 128 MB */ },
{ 0, 0 }
/* Server */
static ncl_tbl_t ncl_table_srv[] = {
- { (1ULL << GBSHIFT) /* 1 GB */, (96 << MBSHIFT) /* 96 MB */ },
+ { (1ULL << GBSHIFT) /* 1 GB */, (96 << MBSHIFT) /* 96 MB */ },
{ (1ULL << (GBSHIFT + 2)) /* 4 GB */, (128 << MBSHIFT) /* 128 MB */ },
{ (1ULL << (GBSHIFT + 3)) /* 8 GB */, (160 << MBSHIFT) /* 160 MB */ },
{ (1ULL << (GBSHIFT + 4)) /* 16 GB */, (192 << MBSHIFT) /* 192 MB */ },
void *buf;
thread_t thread = THREAD_NULL;
+ microuptime(&mb_start);
+
+ /*
+ * These MBUF_ values must be equal to their private counterparts.
+ */
+ _CASSERT(MBUF_EXT == M_EXT);
+ _CASSERT(MBUF_PKTHDR == M_PKTHDR);
+ _CASSERT(MBUF_EOR == M_EOR);
+ _CASSERT(MBUF_LOOP == M_LOOP);
+ _CASSERT(MBUF_BCAST == M_BCAST);
+ _CASSERT(MBUF_MCAST == M_MCAST);
+ _CASSERT(MBUF_FRAG == M_FRAG);
+ _CASSERT(MBUF_FIRSTFRAG == M_FIRSTFRAG);
+ _CASSERT(MBUF_LASTFRAG == M_LASTFRAG);
+ _CASSERT(MBUF_PROMISC == M_PROMISC);
+ _CASSERT(MBUF_HASFCS == M_HASFCS);
+
+ _CASSERT(MBUF_TYPE_FREE == MT_FREE);
+ _CASSERT(MBUF_TYPE_DATA == MT_DATA);
+ _CASSERT(MBUF_TYPE_HEADER == MT_HEADER);
+ _CASSERT(MBUF_TYPE_SOCKET == MT_SOCKET);
+ _CASSERT(MBUF_TYPE_PCB == MT_PCB);
+ _CASSERT(MBUF_TYPE_RTABLE == MT_RTABLE);
+ _CASSERT(MBUF_TYPE_HTABLE == MT_HTABLE);
+ _CASSERT(MBUF_TYPE_ATABLE == MT_ATABLE);
+ _CASSERT(MBUF_TYPE_SONAME == MT_SONAME);
+ _CASSERT(MBUF_TYPE_SOOPTS == MT_SOOPTS);
+ _CASSERT(MBUF_TYPE_FTABLE == MT_FTABLE);
+ _CASSERT(MBUF_TYPE_RIGHTS == MT_RIGHTS);
+ _CASSERT(MBUF_TYPE_IFADDR == MT_IFADDR);
+ _CASSERT(MBUF_TYPE_CONTROL == MT_CONTROL);
+ _CASSERT(MBUF_TYPE_OOBDATA == MT_OOBDATA);
+
+ _CASSERT(MBUF_TSO_IPV4 == CSUM_TSO_IPV4);
+ _CASSERT(MBUF_TSO_IPV6 == CSUM_TSO_IPV6);
+ _CASSERT(MBUF_CSUM_REQ_SUM16 == CSUM_PARTIAL);
+ _CASSERT(MBUF_CSUM_TCP_SUM16 == MBUF_CSUM_REQ_SUM16);
+ _CASSERT(MBUF_CSUM_REQ_IP == CSUM_IP);
+ _CASSERT(MBUF_CSUM_REQ_TCP == CSUM_TCP);
+ _CASSERT(MBUF_CSUM_REQ_UDP == CSUM_UDP);
+ _CASSERT(MBUF_CSUM_REQ_TCPIPV6 == CSUM_TCPIPV6);
+ _CASSERT(MBUF_CSUM_REQ_UDPIPV6 == CSUM_UDPIPV6);
+ _CASSERT(MBUF_CSUM_DID_IP == CSUM_IP_CHECKED);
+ _CASSERT(MBUF_CSUM_IP_GOOD == CSUM_IP_VALID);
+ _CASSERT(MBUF_CSUM_DID_DATA == CSUM_DATA_VALID);
+ _CASSERT(MBUF_CSUM_PSEUDO_HDR == CSUM_PSEUDO_HDR);
+
+ _CASSERT(MBUF_WAITOK == M_WAIT);
+ _CASSERT(MBUF_DONTWAIT == M_DONTWAIT);
+ _CASSERT(MBUF_COPYALL == M_COPYALL);
+
+ _CASSERT(MBUF_SC2TC(MBUF_SC_BK_SYS) == MBUF_TC_BK);
+ _CASSERT(MBUF_SC2TC(MBUF_SC_BK) == MBUF_TC_BK);
+ _CASSERT(MBUF_SC2TC(MBUF_SC_BE) == MBUF_TC_BE);
+ _CASSERT(MBUF_SC2TC(MBUF_SC_RD) == MBUF_TC_BE);
+ _CASSERT(MBUF_SC2TC(MBUF_SC_OAM) == MBUF_TC_BE);
+ _CASSERT(MBUF_SC2TC(MBUF_SC_AV) == MBUF_TC_VI);
+ _CASSERT(MBUF_SC2TC(MBUF_SC_RV) == MBUF_TC_VI);
+ _CASSERT(MBUF_SC2TC(MBUF_SC_VI) == MBUF_TC_VI);
+ _CASSERT(MBUF_SC2TC(MBUF_SC_VO) == MBUF_TC_VO);
+ _CASSERT(MBUF_SC2TC(MBUF_SC_CTL) == MBUF_TC_VO);
+
+ _CASSERT(MBUF_TC2SCVAL(MBUF_TC_BK) == SCVAL_BK);
+ _CASSERT(MBUF_TC2SCVAL(MBUF_TC_BE) == SCVAL_BE);
+ _CASSERT(MBUF_TC2SCVAL(MBUF_TC_VI) == SCVAL_VI);
+ _CASSERT(MBUF_TC2SCVAL(MBUF_TC_VO) == SCVAL_VO);
+
+ /* Module specific scratch space (32-bit alignment requirement) */
+ _CASSERT(!(offsetof(struct mbuf, m_pkthdr.pkt_mpriv) %
+ sizeof (uint32_t)));
+
+ /* Initialize random red zone cookie value */
+ _CASSERT(sizeof (mb_redzone_cookie) ==
+ sizeof (((struct pkthdr *)0)->redzone));
+ read_random(&mb_redzone_cookie, sizeof (mb_redzone_cookie));
+
+ /* Make sure we don't save more than we should */
+ _CASSERT(MCA_SAVED_MBUF_SIZE <= sizeof (struct mbuf));
+
if (nmbclusters == 0)
nmbclusters = NMBCLUSTERS;
mbuf_mlock_grp_attr = lck_grp_attr_alloc_init();
mbuf_mlock_grp = lck_grp_alloc_init("mbuf", mbuf_mlock_grp_attr);
mbuf_mlock_attr = lck_attr_alloc_init();
- mbuf_mlock = lck_mtx_alloc_init(mbuf_mlock_grp, mbuf_mlock_attr);
+ lck_mtx_init(mbuf_mlock, mbuf_mlock_grp, mbuf_mlock_attr);
/*
* Allocate cluster slabs table:
VERIFY(mclaudit != NULL);
mcl_audit_con_cache = mcache_create("mcl_audit_contents",
- AUDIT_CONTENTS_SIZE, 0, 0, MCR_SLEEP);
+ AUDIT_CONTENTS_SIZE, sizeof (u_int64_t), 0, MCR_SLEEP);
VERIFY(mcl_audit_con_cache != NULL);
}
mclverify = (mbuf_debug & MCF_VERIFY);
mcltrace = (mbuf_debug & MCF_TRACE);
mclfindleak = !(mbuf_debug & MCF_NOLEAKLOG);
+ mclexpleak = mclfindleak && (mbuf_debug & MCF_EXPLEAKLOG);
/* Enable mbuf leak logging, with a lock to protect the tables */
mleak_lock_grp_attr = lck_grp_attr_alloc_init();
mleak_lock_grp = lck_grp_alloc_init("mleak_lock", mleak_lock_grp_attr);
mleak_lock_attr = lck_attr_alloc_init();
- mleak_lock = lck_mtx_alloc_init(mleak_lock_grp, mleak_lock_attr);
+ lck_mtx_init(mleak_lock, mleak_lock_grp, mleak_lock_attr);
mleak_activate();
bzero((char *)mcl_paddr, mcl_pages * sizeof (ppnum_t));
embutl = (union mbigcluster *)
- ((unsigned char *)mbutl + (nmbclusters * MCLBYTES));
+ ((void *)((unsigned char *)mbutl + (nmbclusters * MCLBYTES)));
VERIFY((((char *)embutl - (char *)mbutl) % MBIGCLBYTES) == 0);
/* Prime up the freelist */
* before alignment is not saved.
*/
ncpu = ml_get_max_cpus();
- MALLOC(buf, void *, MBUF_MTYPES_SIZE(ncpu) + CPU_CACHE_SIZE,
+ MALLOC(buf, void *, MBUF_MTYPES_SIZE(ncpu) + CPU_CACHE_LINE_SIZE,
M_TEMP, M_WAITOK);
VERIFY(buf != NULL);
- mbuf_mtypes = (mbuf_mtypes_t *)P2ROUNDUP((intptr_t)buf, CPU_CACHE_SIZE);
+ mbuf_mtypes = (mbuf_mtypes_t *)P2ROUNDUP((intptr_t)buf,
+ CPU_CACHE_LINE_SIZE);
bzero(mbuf_mtypes, MBUF_MTYPES_SIZE(ncpu));
/*
}
}
- printf("mbinit: done [%d MB total pool size, (%d/%d) split]\n",
+ /* allocate space for mbuf_dump_buf */
+ MALLOC(mbuf_dump_buf, char *, MBUF_DUMP_BUF_SIZE, M_TEMP, M_WAITOK);
+ VERIFY(mbuf_dump_buf != NULL);
+
+ if (mbuf_debug & MCF_DEBUG) {
+ printf("%s: MLEN %d, MHLEN %d\n", __func__,
+ (int)_MLEN, (int)_MHLEN);
+ }
+
+ printf("%s: done [%d MB total pool size, (%d/%d) split]\n", __func__,
(nmbclusters << MCLSHIFT) >> MBSHIFT,
(nclusters << MCLSHIFT) >> MBSHIFT,
(njcl << MCLSHIFT) >> MBSHIFT);
}
/* Record this transaction */
if (mcltrace)
- mcache_buffer_log(mca, list, m_cache(class));
+ mcache_buffer_log(mca, list, m_cache(class), &mb_start);
if (alloc)
mca->mca_uflags |= MB_INUSE;
mcache_audit_free_verify(mca, m, 0,
m_maxsize(MC_MBUF));
}
- ms = (struct mbuf *)mca->mca_contents;
+ ms = MCA_SAVED_MBUF_PTR(mca);
}
/* Do the cluster sanity checks */
MEXT_REF(m) = 0;
MEXT_FLAGS(m) = 0;
- rfa = (mcache_obj_t *)MEXT_RFA(m);
+ rfa = (mcache_obj_t *)(void *)MEXT_RFA(m);
rfa->obj_next = ref_list;
ref_list = rfa;
MEXT_RFA(m) = NULL;
rfa = (struct ext_ref *)ref_list;
ref_list = ref_list->obj_next;
- ((mcache_obj_t *)rfa)->obj_next = NULL;
+ ((mcache_obj_t *)(void *)rfa)->obj_next = NULL;
/*
* If auditing is enabled, construct the shadow mbuf
lck_mtx_lock(mbuf_mlock);
mca = mcl_audit_buf2mca(MC_MBUF, (mcache_obj_t *)m);
- ms = ((struct mbuf *)mca->mca_contents);
+ ms = MCA_SAVED_MBUF_PTR(mca);
cl_mca = mcl_audit_buf2mca(MC_CL, (mcache_obj_t *)cl);
/*
mca = mcl_audit_buf2mca(MC_MBUF, (mcache_obj_t *)m);
mcl_audit_mbuf(mca, m, TRUE, alloc);
if (mcltrace)
- mcache_buffer_log(mca, m, m_cache(class));
+ mcache_buffer_log(mca, m, m_cache(class), &mb_start);
if (alloc)
mca->mca_uflags |= MB_COMP_INUSE;
* pattern-filled by the above call to mcl_audit_mbuf().
*/
if (!alloc && mclverify)
- ms = (struct mbuf *)mca->mca_contents;
+ ms = MCA_SAVED_MBUF_PTR(mca);
/* Do the cluster sanity checks and record its transaction */
cl = ms->m_ext.ext_buf;
size = m_maxsize(MC_16KCL);
mcl_audit_cluster(mca, cl, size, alloc, FALSE);
if (mcltrace)
- mcache_buffer_log(mca, cl, m_cache(class));
+ mcache_buffer_log(mca, cl, m_cache(class), &mb_start);
if (alloc)
mca->mca_uflags |= MB_COMP_INUSE;
for (i = 0; i < numpages; i++, page += NBPG) {
ppnum_t offset = ((char *)page - (char *)mbutl) / NBPG;
- ppnum_t new_page = pmap_find_phys(kernel_pmap,
- (vm_offset_t)page);
+ ppnum_t new_page = pmap_find_phys(kernel_pmap, page);
/*
- * In the case of no mapper being available the following
- * code noops and returns the input page; if there is a
- * mapper the appropriate I/O page is returned.
+ * If there is a mapper the appropriate I/O page is returned;
+ * zero out the page to discard its past contents to prevent
+ * exposing leftover kernel memory.
*/
VERIFY(offset < mcl_pages);
- new_page = IOMapperInsertPage(mcl_paddr_base, offset, new_page);
- mcl_paddr[offset] = new_page << PGSHIFT;
+ if (mcl_paddr_base != 0) {
+ bzero((void *)(uintptr_t) page, page_size);
+ new_page = IOMapperInsertPage(mcl_paddr_base,
+ offset, new_page);
+ }
+ mcl_paddr[offset] = new_page;
/* Pattern-fill this fresh page */
if (mclverify) {
struct mbuf *ms;
mca = mcl_audit_buf2mca(MC_MBUF,
(mcache_obj_t *)m);
- ms = ((struct mbuf *)
- mca->mca_contents);
+ ms = MCA_SAVED_MBUF_PTR(mca);
ms->m_type = MT_FREE;
} else {
m->m_type = MT_FREE;
lck_mtx_unlock(mbuf_mlock);
if (bmap != 0) {
- /* drain is performed in pfslowtimo(), to avoid deadlocks */
- do_reclaim = 1;
+ /* signal the domains to drain */
+ net_drain_domains();
/* Sigh; we have no other choices but to ask mcache to purge */
for (m = 0; m < NELEM(mbuf_table); m++) {
if (m->m_type == MT_FREE)
panic("m_free: freeing an already freed mbuf");
- /* Free the aux data and tags if there is any */
if (m->m_flags & M_PKTHDR) {
+ /* Check for scratch area overflow */
+ m_redzone_verify(m);
+ /* Free the aux data and tags if there is any */
m_tag_delete_chain(m, NULL);
}
void
m_copy_pkthdr(struct mbuf *to, struct mbuf *from)
{
- /* We will be taking over the tags of 'to' */
- if (to->m_flags & M_PKTHDR)
+ VERIFY(from->m_flags & M_PKTHDR);
+
+ /* Check for scratch area overflow */
+ m_redzone_verify(from);
+
+ if (to->m_flags & M_PKTHDR) {
+ /* Check for scratch area overflow */
+ m_redzone_verify(to);
+ /* We will be taking over the tags of 'to' */
m_tag_delete_chain(to, NULL);
+ }
to->m_pkthdr = from->m_pkthdr; /* especially tags */
- m_tag_init(from); /* purge tags from src */
- m_prio_init(from); /* reset priority from src */
+ m_classifier_init(from, 0); /* purge classifier info */
+ m_tag_init(from, 1); /* purge all tags from src */
+ m_scratch_init(from); /* clear src scratch area */
to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
if ((to->m_flags & M_EXT) == 0)
to->m_data = to->m_pktdat;
+ m_redzone_init(to); /* setup red zone on dst */
}
/*
static int
m_dup_pkthdr(struct mbuf *to, struct mbuf *from, int how)
{
- if (to->m_flags & M_PKTHDR)
+ VERIFY(from->m_flags & M_PKTHDR);
+
+ /* Check for scratch area overflow */
+ m_redzone_verify(from);
+
+ if (to->m_flags & M_PKTHDR) {
+ /* Check for scratch area overflow */
+ m_redzone_verify(to);
+ /* We will be taking over the tags of 'to' */
m_tag_delete_chain(to, NULL);
+ }
to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
if ((to->m_flags & M_EXT) == 0)
to->m_data = to->m_pktdat;
to->m_pkthdr = from->m_pkthdr;
- m_tag_init(to);
+ m_redzone_init(to); /* setup red zone on dst */
+ m_tag_init(to, 0); /* preserve dst static tags */
return (m_tag_copy_chain(to, from, how));
}
+void
+m_copy_pftag(struct mbuf *to, struct mbuf *from)
+{
+ to->m_pkthdr.pf_mtag = from->m_pkthdr.pf_mtag;
+#if PF_ECN
+ to->m_pkthdr.pf_mtag.pftag_hdr = NULL;
+ to->m_pkthdr.pf_mtag.pftag_flags &= ~(PF_TAG_HDR_INET|PF_TAG_HDR_INET6);
+#endif /* PF_ECN */
+}
+
+void
+m_classifier_init(struct mbuf *m, uint32_t pktf_mask)
+{
+ VERIFY(m->m_flags & M_PKTHDR);
+
+ m->m_pkthdr.pkt_proto = 0;
+ m->m_pkthdr.pkt_flowsrc = 0;
+ m->m_pkthdr.pkt_flowid = 0;
+ m->m_pkthdr.pkt_flags &= pktf_mask; /* caller-defined mask */
+ /* preserve service class and interface info for loopback packets */
+ if (!(m->m_pkthdr.pkt_flags & PKTF_LOOP))
+ (void) m_set_service_class(m, MBUF_SC_BE);
+ if (!(m->m_pkthdr.pkt_flags & PKTF_IFAINFO))
+ m->m_pkthdr.pkt_ifainfo = 0;
+#if MEASURE_BW
+ m->m_pkthdr.pkt_bwseq = 0;
+#endif /* MEASURE_BW */
+}
+
+void
+m_copy_classifier(struct mbuf *to, struct mbuf *from)
+{
+ VERIFY(to->m_flags & M_PKTHDR);
+ VERIFY(from->m_flags & M_PKTHDR);
+
+ to->m_pkthdr.pkt_proto = from->m_pkthdr.pkt_proto;
+ to->m_pkthdr.pkt_flowsrc = from->m_pkthdr.pkt_flowsrc;
+ to->m_pkthdr.pkt_flowid = from->m_pkthdr.pkt_flowid;
+ to->m_pkthdr.pkt_flags = from->m_pkthdr.pkt_flags;
+ (void) m_set_service_class(to, from->m_pkthdr.pkt_svc);
+ to->m_pkthdr.pkt_ifainfo = from->m_pkthdr.pkt_ifainfo;
+ to->m_pkthdr.ipsec_policy = from->m_pkthdr.ipsec_policy;
+#if MEASURE_BW
+ to->m_pkthdr.pkt_bwseq = from->m_pkthdr.pkt_bwseq;
+#endif /* MEASURE_BW */
+}
+
/*
* Return a list of mbuf hdrs that point to clusters. Try for num_needed;
* if wantall is not set, return whatever number were available. Set up the
return (NULL);
}
+ if (pnum > *num_needed) {
+ printf("%s: File a radar related to <rdar://10146739>. \
+ needed = %u, pnum = %u, num_needed = %u \n",
+ __func__, needed, pnum, *num_needed);
+ }
+
*num_needed = pnum;
return (top);
}
mt_free++;
if (m->m_flags & M_PKTHDR) {
+ /* Check for scratch area overflow */
+ m_redzone_verify(m);
+ /* Free the aux data and tags if there is any */
m_tag_delete_chain(m, NULL);
}
if (!(m->m_flags & M_EXT))
goto simple_free;
- o = (mcache_obj_t *)m->m_ext.ext_buf;
+ o = (mcache_obj_t *)(void *)m->m_ext.ext_buf;
refcnt = m_decref(m);
composite = (MEXT_FLAGS(m) & EXTF_COMPOSITE);
if (refcnt == 0 && !composite) {
m->m_ext.ext_size,
m->m_ext.ext_arg);
}
- rfa = (mcache_obj_t *)MEXT_RFA(m);
+ rfa = (mcache_obj_t *)(void *)MEXT_RFA(m);
rfa->obj_next = ref_list;
ref_list = rfa;
MEXT_RFA(m) = NULL;
int MCFail;
struct mbuf *
-m_copym(struct mbuf *m, int off0, int len, int wait)
+m_copym_mode(struct mbuf *m, int off0, int len, int wait, uint32_t mode)
{
struct mbuf *n, *mhdr = NULL, **np;
int off = off0;
goto nospace;
if (copyhdr != 0) {
- M_COPY_PKTHDR(n, mhdr);
+ if (mode == M_COPYM_MOVE_HDR) {
+ M_COPY_PKTHDR(n, mhdr);
+ } else if (mode == M_COPYM_COPY_HDR) {
+ if (m_dup_pkthdr(n, mhdr, wait) == 0)
+ goto nospace;
+ }
if (len == M_COPYALL)
n->m_pkthdr.len -= off0;
else
return (NULL);
}
+
+struct mbuf *
+m_copym(struct mbuf *m, int off0, int len, int wait)
+{
+ return (m_copym_mode(m, off0, len, wait, M_COPYM_MOVE_HDR));
+}
+
/*
* Equivalent to m_copym except that all necessary mbuf hdrs are allocated
* within this routine also, the last mbuf and offset accessed are passed
*/
struct mbuf *
m_copym_with_hdrs(struct mbuf *m, int off0, int len0, int wait,
- struct mbuf **m_lastm, int *m_off)
+ struct mbuf **m_lastm, int *m_off, uint32_t mode)
{
struct mbuf *n, **np = NULL;
int off = off0, len = len0;
}
if (copyhdr) {
- M_COPY_PKTHDR(n, m);
+ if (mode == M_COPYM_MOVE_HDR) {
+ M_COPY_PKTHDR(n, m);
+ } else if (mode == M_COPYM_COPY_HDR) {
+ if (m_dup_pkthdr(n, m, wait) == 0)
+ goto nospace;
+ }
n->m_pkthdr.len = len;
copyhdr = 0;
}
* allocate a new writable mbuf and try again.
*/
-#if defined(DIAGNOSTIC)
+#if DIAGNOSTIC
if (!(flags & M_COPYBACK0_COW))
panic("m_copyback0: read-only");
-#endif /* defined(DIAGNOSTIC) */
+#endif /* DIAGNOSTIC */
/*
* if we're going to write into the middle of
return (ENOBUFS);
}
-char *
+uint64_t
mcl_to_paddr(char *addr)
{
vm_offset_t base_phys;
if (!MBUF_IN_MAP(addr))
- return (NULL);
- base_phys = mcl_paddr[(addr - (char *)mbutl) >> PGSHIFT];
+ return (0);
+ base_phys = mcl_paddr[atop_64(addr - (char *)mbutl)];
if (base_phys == 0)
- return (NULL);
- return ((char *)((uintptr_t)base_phys | ((uintptr_t)addr & PGOFSET)));
+ return (0);
+ return ((uint64_t)(ptoa_64(base_phys) | ((uint64_t)addr & PAGE_MASK)));
}
/*
return (m);
}
+unsigned int
+m_fixhdr(struct mbuf *m0)
+{
+ u_int len;
+
+ VERIFY(m0->m_flags & M_PKTHDR);
+
+ len = m_length2(m0, NULL);
+ m0->m_pkthdr.len = len;
+ return (len);
+}
+
+unsigned int
+m_length2(struct mbuf *m0, struct mbuf **last)
+{
+ struct mbuf *m;
+ u_int len;
+
+ len = 0;
+ for (m = m0; m != NULL; m = m->m_next) {
+ len += m->m_len;
+ if (m->m_next == NULL)
+ break;
+ }
+ if (last != NULL)
+ *last = m;
+ return (len);
+}
+
+/*
+ * Defragment a mbuf chain, returning the shortest possible chain of mbufs
+ * and clusters. If allocation fails and this cannot be completed, NULL will
+ * be returned, but the passed in chain will be unchanged. Upon success,
+ * the original chain will be freed, and the new chain will be returned.
+ *
+ * If a non-packet header is passed in, the original mbuf (chain?) will
+ * be returned unharmed.
+ *
+ * If offset is specfied, the first mbuf in the chain will have a leading
+ * space of the amount stated by the "off" parameter.
+ *
+ * This routine requires that the m_pkthdr.header field of the original
+ * mbuf chain is cleared by the caller.
+ */
+struct mbuf *
+m_defrag_offset(struct mbuf *m0, u_int32_t off, int how)
+{
+ struct mbuf *m_new = NULL, *m_final = NULL;
+ int progress = 0, length, pktlen;
+
+ if (!(m0->m_flags & M_PKTHDR))
+ return (m0);
+
+ VERIFY(off < MHLEN);
+ m_fixhdr(m0); /* Needed sanity check */
+
+ pktlen = m0->m_pkthdr.len + off;
+ if (pktlen > MHLEN)
+ m_final = m_getcl(how, MT_DATA, M_PKTHDR);
+ else
+ m_final = m_gethdr(how, MT_DATA);
+
+ if (m_final == NULL)
+ goto nospace;
+
+ if (off > 0) {
+ pktlen -= off;
+ m_final->m_data += off;
+ }
+
+ /*
+ * Caller must have handled the contents pointed to by this
+ * pointer before coming here, as otherwise it will point to
+ * the original mbuf which will get freed upon success.
+ */
+ VERIFY(m0->m_pkthdr.pkt_hdr == NULL);
+
+ if (m_dup_pkthdr(m_final, m0, how) == 0)
+ goto nospace;
+
+ m_new = m_final;
+
+ while (progress < pktlen) {
+ length = pktlen - progress;
+ if (length > MCLBYTES)
+ length = MCLBYTES;
+ length -= ((m_new == m_final) ? off : 0);
+
+ if (m_new == NULL) {
+ if (length > MLEN)
+ m_new = m_getcl(how, MT_DATA, 0);
+ else
+ m_new = m_get(how, MT_DATA);
+ if (m_new == NULL)
+ goto nospace;
+ }
+
+ m_copydata(m0, progress, length, mtod(m_new, caddr_t));
+ progress += length;
+ m_new->m_len = length;
+ if (m_new != m_final)
+ m_cat(m_final, m_new);
+ m_new = NULL;
+ }
+ m_freem(m0);
+ m0 = m_final;
+ return (m0);
+nospace:
+ if (m_final)
+ m_freem(m_final);
+ return (NULL);
+}
+
+struct mbuf *
+m_defrag(struct mbuf *m0, int how)
+{
+ return (m_defrag_offset(m0, 0, how));
+}
+
void
m_mchtype(struct mbuf *m, int t)
{
/* Attach the contents buffer if requested */
if (save_contents) {
- VERIFY(con != NULL);
+ mcl_saved_contents_t *msc =
+ (mcl_saved_contents_t *)(void *)con;
+
+ VERIFY(msc != NULL);
+ VERIFY(IS_P2ALIGNED(msc, sizeof (u_int64_t)));
+ VERIFY(con_size == sizeof (*msc));
mca->mca_contents_size = con_size;
- mca->mca_contents = con;
+ mca->mca_contents = msc;
con = con->obj_next;
bzero(mca->mca_contents, mca->mca_contents_size);
}
static void
mcl_audit_restore_mbuf(struct mbuf *m, mcache_audit_t *mca, boolean_t composite)
{
- struct mbuf *ms = (struct mbuf *)mca->mca_contents;
+ struct mbuf *ms = MCA_SAVED_MBUF_PTR(mca);
if (composite) {
struct mbuf *next = m->m_next;
VERIFY(ms->m_flags == M_EXT && MEXT_RFA(ms) != NULL &&
MBUF_IS_COMPOSITE(ms));
+ VERIFY(mca->mca_contents_size == AUDIT_CONTENTS_SIZE);
/*
* We could have hand-picked the mbuf fields and restore
* them individually, but that will be a maintenance
* headache. Instead, restore everything that was saved;
* the mbuf layer will recheck and reinitialize anyway.
*/
- bcopy(ms, m, mca->mca_contents_size);
+ bcopy(ms, m, MCA_SAVED_MBUF_SIZE);
m->m_next = next;
} else {
/*
static void
mcl_audit_save_mbuf(struct mbuf *m, mcache_audit_t *mca)
{
+ VERIFY(mca->mca_contents_size == AUDIT_CONTENTS_SIZE);
_MCHECK(m);
- bcopy(m, mca->mca_contents, mca->mca_contents_size);
+ bcopy(m, MCA_SAVED_MBUF_PTR(mca), MCA_SAVED_MBUF_SIZE);
}
static void
}
}
+static void
+mcl_audit_scratch(mcache_audit_t *mca)
+{
+ void *stack[MCACHE_STACK_DEPTH + 1];
+ mcl_scratch_audit_t *msa;
+ struct timeval now;
+
+ VERIFY(mca->mca_contents != NULL);
+ msa = MCA_SAVED_SCRATCH_PTR(mca);
+
+ msa->msa_pthread = msa->msa_thread;
+ msa->msa_thread = current_thread();
+ bcopy(msa->msa_stack, msa->msa_pstack, sizeof (msa->msa_pstack));
+ msa->msa_pdepth = msa->msa_depth;
+ bzero(stack, sizeof (stack));
+ msa->msa_depth = OSBacktrace(stack, MCACHE_STACK_DEPTH + 1) - 1;
+ bcopy(&stack[1], msa->msa_stack, sizeof (mca->mca_pstack));
+
+ msa->msa_ptstamp = msa->msa_tstamp;
+ microuptime(&now);
+ /* tstamp is in ms relative to base_ts */
+ msa->msa_tstamp = ((now.tv_usec - mb_start.tv_usec) / 1000);
+ if ((now.tv_sec - mb_start.tv_sec) > 0)
+ msa->msa_tstamp += ((now.tv_sec - mb_start.tv_sec) * 1000);
+}
+
static void
mcl_audit_mcheck_panic(struct mbuf *m)
{
struct mallocation *allocation;
struct mtrace *trace;
uint32_t trace_index;
- int i;
/* Quit if someone else modifying the tables */
if (!lck_mtx_try_lock_spin(mleak_lock)) {
mleak_table.alloc_recorded++;
mleak_table.outstanding_allocs++;
- /* keep a log of the last 5 traces to be top trace, in order */
- for (i = 0; i < MLEAK_NUM_TRACES; i++) {
- if (mleak_top_trace[i] == NULL ||
- mleak_top_trace[i]->allocs <= trace->allocs) {
- if (mleak_top_trace[i] != trace) {
- int j = MLEAK_NUM_TRACES;
- while (--j > i) {
- mleak_top_trace[j] =
- mleak_top_trace[j - 1];
- }
- mleak_top_trace[i] = trace;
- }
- break;
- }
- }
-
lck_mtx_unlock(mleak_lock);
return (TRUE);
}
}
}
+static void
+mleak_sort_traces()
+{
+ int i, j, k;
+ struct mtrace *swap;
+
+ for(i = 0; i < MLEAK_NUM_TRACES; i++)
+ mleak_top_trace[i] = NULL;
+
+ for(i = 0, j = 0; j < MLEAK_NUM_TRACES && i < mleak_trace_buckets; i++)
+ {
+ if (mleak_traces[i].allocs <= 0)
+ continue;
+
+ mleak_top_trace[j] = &mleak_traces[i];
+ for (k = j; k > 0; k--) {
+ if (mleak_top_trace[k]->allocs <=
+ mleak_top_trace[k-1]->allocs)
+ break;
+
+ swap = mleak_top_trace[k-1];
+ mleak_top_trace[k-1] = mleak_top_trace[k];
+ mleak_top_trace[k] = swap;
+ }
+ j++;
+ }
+
+ j--;
+ for(; i < mleak_trace_buckets; i++) {
+ if (mleak_traces[i].allocs <= mleak_top_trace[j]->allocs)
+ continue;
+
+ mleak_top_trace[j] = &mleak_traces[i];
+
+ for (k = j; k > 0; k--) {
+ if (mleak_top_trace[k]->allocs <=
+ mleak_top_trace[k-1]->allocs)
+ break;
+
+ swap = mleak_top_trace[k-1];
+ mleak_top_trace[k-1] = mleak_top_trace[k];
+ mleak_top_trace[k] = swap;
+ }
+ }
+}
+
+static void
+mleak_update_stats()
+{
+ mleak_trace_stat_t *mltr;
+ int i;
+
+ VERIFY(mleak_stat != NULL);
+#ifdef __LP64__
+ VERIFY(mleak_stat->ml_isaddr64);
+#else
+ VERIFY(!mleak_stat->ml_isaddr64);
+#endif /* !__LP64__ */
+ VERIFY(mleak_stat->ml_cnt == MLEAK_NUM_TRACES);
+
+ mleak_sort_traces();
+
+ mltr = &mleak_stat->ml_trace[0];
+ bzero(mltr, sizeof (*mltr) * MLEAK_NUM_TRACES);
+ for (i = 0; i < MLEAK_NUM_TRACES; i++) {
+ int j;
+
+ if (mleak_top_trace[i] == NULL ||
+ mleak_top_trace[i]->allocs == 0)
+ continue;
+
+ mltr->mltr_collisions = mleak_top_trace[i]->collisions;
+ mltr->mltr_hitcount = mleak_top_trace[i]->hitcount;
+ mltr->mltr_allocs = mleak_top_trace[i]->allocs;
+ mltr->mltr_depth = mleak_top_trace[i]->depth;
+
+ VERIFY(mltr->mltr_depth <= MLEAK_STACK_DEPTH);
+ for (j = 0; j < mltr->mltr_depth; j++)
+ mltr->mltr_addr[j] = mleak_top_trace[i]->addr[j];
+
+ mltr++;
+ }
+}
+
static struct mbtypes {
int mt_type;
const char *mt_name;
uint8_t seen[256];
struct mbtypes *mp;
mb_class_stat_t *sp;
+ mleak_trace_stat_t *mltr;
char *c = mbuf_dump_buf;
- int i, k, clen = sizeof (mbuf_dump_buf);
+ int i, k, clen = MBUF_DUMP_BUF_SIZE;
mbuf_dump_buf[0] = '\0';
"in use)\n", totmem / 1024, totpct);
MBUF_DUMP_BUF_CHK();
+ /* mbuf leak detection statistics */
+ mleak_update_stats();
+
+ k = snprintf(c, clen, "\nmbuf leak detection table:\n");
+ MBUF_DUMP_BUF_CHK();
+ k = snprintf(c, clen, "\ttotal captured: %u (one per %u)\n",
+ mleak_table.mleak_capture / mleak_table.mleak_sample_factor,
+ mleak_table.mleak_sample_factor);
+ MBUF_DUMP_BUF_CHK();
+ k = snprintf(c, clen, "\ttotal allocs outstanding: %llu\n",
+ mleak_table.outstanding_allocs);
+ MBUF_DUMP_BUF_CHK();
+ k = snprintf(c, clen, "\tnew hash recorded: %llu allocs, %llu traces\n",
+ mleak_table.alloc_recorded, mleak_table.trace_recorded);
+ MBUF_DUMP_BUF_CHK();
+ k = snprintf(c, clen, "\thash collisions: %llu allocs, %llu traces\n",
+ mleak_table.alloc_collisions, mleak_table.trace_collisions);
+ MBUF_DUMP_BUF_CHK();
+ k = snprintf(c, clen, "\toverwrites: %llu allocs, %llu traces\n",
+ mleak_table.alloc_overwrites, mleak_table.trace_overwrites);
+ MBUF_DUMP_BUF_CHK();
+ k = snprintf(c, clen, "\tlock conflicts: %llu\n\n",
+ mleak_table.total_conflicts);
+ MBUF_DUMP_BUF_CHK();
+
+ k = snprintf(c, clen, "top %d outstanding traces:\n",
+ mleak_stat->ml_cnt);
+ MBUF_DUMP_BUF_CHK();
+ for (i = 0; i < mleak_stat->ml_cnt; i++) {
+ mltr = &mleak_stat->ml_trace[i];
+ k = snprintf(c, clen, "[%d] %llu outstanding alloc(s), "
+ "%llu hit(s), %llu collision(s)\n", (i + 1),
+ mltr->mltr_allocs, mltr->mltr_hitcount,
+ mltr->mltr_collisions);
+ MBUF_DUMP_BUF_CHK();
+ }
+
+ if (mleak_stat->ml_isaddr64)
+ k = snprintf(c, clen, MB_LEAK_HDR_64);
+ else
+ k = snprintf(c, clen, MB_LEAK_HDR_32);
+ MBUF_DUMP_BUF_CHK();
+
+ for (i = 0; i < MLEAK_STACK_DEPTH; i++) {
+ int j;
+ k = snprintf(c, clen, "%2d: ", (i + 1));
+ MBUF_DUMP_BUF_CHK();
+ for (j = 0; j < mleak_stat->ml_cnt; j++) {
+ mltr = &mleak_stat->ml_trace[j];
+ if (i < mltr->mltr_depth) {
+ if (mleak_stat->ml_isaddr64) {
+ k = snprintf(c, clen, "0x%0llx ",
+ mltr->mltr_addr[i]);
+ } else {
+ k = snprintf(c, clen,
+ "0x%08x ",
+ (u_int32_t)mltr->mltr_addr[i]);
+ }
+ } else {
+ if (mleak_stat->ml_isaddr64)
+ k = snprintf(c, clen,
+ MB_LEAK_SPACING_64);
+ else
+ k = snprintf(c, clen,
+ MB_LEAK_SPACING_32);
+ }
+ MBUF_DUMP_BUF_CHK();
+ }
+ k = snprintf(c, clen, "\n");
+ MBUF_DUMP_BUF_CHK();
+ }
done:
return (mbuf_dump_buf);
}
#undef MBUF_DUMP_BUF_CHK
+/*
+ * Convert between a regular and a packet header mbuf. Caller is responsible
+ * for setting or clearing M_PKTHDR; this routine does the rest of the work.
+ */
+int
+m_reinit(struct mbuf *m, int hdr)
+{
+ int ret = 0;
+
+ if (hdr) {
+ VERIFY(!(m->m_flags & M_PKTHDR));
+ if (!(m->m_flags & M_EXT) &&
+ (m->m_data != m->m_dat || m->m_len > 0)) {
+ /*
+ * If there's no external cluster attached and the
+ * mbuf appears to contain user data, we cannot
+ * safely convert this to a packet header mbuf,
+ * as the packet header structure might overlap
+ * with the data.
+ */
+ printf("%s: cannot set M_PKTHDR on altered mbuf %p, "
+ "m_data %p (expected %p), m_len %d (expected 0)\n",
+ __func__, m, m->m_data, m->m_dat, m->m_len);
+ ret = EBUSY;
+ } else {
+ VERIFY((m->m_flags & M_EXT) || m->m_data == m->m_dat);
+ m->m_flags |= M_PKTHDR;
+ MBUF_INIT_PKTHDR(m);
+ }
+ } else {
+ /* Check for scratch area overflow */
+ m_redzone_verify(m);
+ /* Free the aux data and tags if there is any */
+ m_tag_delete_chain(m, NULL);
+ m->m_flags &= ~M_PKTHDR;
+ }
+
+ return (ret);
+}
+
+void
+m_scratch_init(struct mbuf *m)
+{
+ VERIFY(m->m_flags & M_PKTHDR);
+
+ bzero(&m->m_pkthdr.pkt_mpriv, sizeof (m->m_pkthdr.pkt_mpriv));
+}
+
+u_int32_t
+m_scratch_get(struct mbuf *m, u_int8_t **p)
+{
+ VERIFY(m->m_flags & M_PKTHDR);
+
+ if (mcltrace) {
+ mcache_audit_t *mca;
+
+ lck_mtx_lock(mbuf_mlock);
+ mca = mcl_audit_buf2mca(MC_MBUF, (mcache_obj_t *)m);
+ if (mca->mca_uflags & MB_SCVALID)
+ mcl_audit_scratch(mca);
+ lck_mtx_unlock(mbuf_mlock);
+ }
+
+ *p = (u_int8_t *)&m->m_pkthdr.pkt_mpriv;
+ return (sizeof (m->m_pkthdr.pkt_mpriv));
+}
+
+static void
+m_redzone_init(struct mbuf *m)
+{
+ VERIFY(m->m_flags & M_PKTHDR);
+ /*
+ * Each mbuf has a unique red zone pattern, which is a XOR
+ * of the red zone cookie and the address of the mbuf.
+ */
+ m->m_pkthdr.redzone = ((u_int32_t)(uintptr_t)m) ^ mb_redzone_cookie;
+}
+
+static void
+m_redzone_verify(struct mbuf *m)
+{
+ u_int32_t mb_redzone;
+
+ VERIFY(m->m_flags & M_PKTHDR);
+
+ mb_redzone = ((u_int32_t)(uintptr_t)m) ^ mb_redzone_cookie;
+ if (m->m_pkthdr.redzone != mb_redzone) {
+ panic("mbuf %p redzone violation with value 0x%x "
+ "(instead of 0x%x, using cookie 0x%x)\n",
+ m, m->m_pkthdr.redzone, mb_redzone, mb_redzone_cookie);
+ /* NOTREACHED */
+ }
+}
+
SYSCTL_DECL(_kern_ipc);
SYSCTL_PROC(_kern_ipc, KIPC_MBSTAT, mbstat,
CTLFLAG_RD | CTLFLAG_LOCKED,
projects / apple / xnu.git / blobdiff