X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/2d21ac55c334faf3a56e5634905ed6987fc787d4..bd504ef0e0b883cdd7917b73b3574eb9ce669905:/bsd/kern/uipc_mbuf.c?ds=sidebyside diff --git a/bsd/kern/uipc_mbuf.c b/bsd/kern/uipc_mbuf.c index 38fe262e6..3edca9510 100644 --- a/bsd/kern/uipc_mbuf.c +++ b/bsd/kern/uipc_mbuf.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000-2007 Apple Inc. All rights reserved. + * Copyright (c) 1998-2012 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -77,12 +77,14 @@ #include #include #include +#include #include #include #include #include #include +#include #include #include @@ -114,7 +116,7 @@ * preserve the contents of the objects during its transactions. * * MC_BIGCL: - * This is a cache of rudimentary objects of NBPG in size; each + * This is a cache of rudimentary objects of MBIGCLBYTES in size; each * object represents a mbigcluster structure. This cache does not * preserve the contents of the objects during its transaction. * @@ -263,8 +265,9 @@ * Debugging can be enabled by adding "mbuf_debug=0x3" to boot-args; this * translates to the mcache flags (MCF_VERIFY | MCF_AUDIT). Additionally, * the CPU layer cache can be disabled by setting the MCF_NOCPUCACHE flag, - * i.e. modify the boot argument parameter to "mbuf_debug=0x13". Note - * that debugging consumes more CPU and memory. + * i.e. modify the boot argument parameter to "mbuf_debug=0x13". Leak + * detection may also be disabled by setting the MCF_NOLEAKLOG flag, e.g. + * "mbuf_debug=0x113". Note that debugging consumes more CPU and memory. * * Each object is associated with exactly one mcache_audit_t structure that * contains the information related to its last buffer transaction. Given @@ -275,9 +278,9 @@ * | mbuf addr | | mclaudit[i] | * +------------+ +=============+ * | | cl_audit[0] | - * i = MTOCL(addr) +-------------+ + * i = MTOBG(addr) +-------------+ * | +-----> | cl_audit[1] | -----> mcache_audit_t - * b = CLTOM(i) | +-------------+ + * b = BGTOM(i) | +-------------+ * | | | ... | * x = MCLIDX(b, addr) | +-------------+ * | | | cl_audit[7] | @@ -285,22 +288,23 @@ * (e.g. x == 1) * * The mclaudit[] array is allocated at initialization time, but its contents - * get populated when the corresponding cluster is created. Because a cluster - * can be turned into NMBPCL number of mbufs, we preserve enough space for the - * mbufs so that there is a 1-to-1 mapping between them. A cluster that never + * get populated when the corresponding cluster is created. Because a page + * can be turned into NMBPBG number of mbufs, we preserve enough space for the + * mbufs so that there is a 1-to-1 mapping between them. A page that never * gets (or has not yet) turned into mbufs will use only cl_audit[0] with the - * remaining entries unused. For big clusters, only one entry is allocated - * and used for the entire cluster pair. + * remaining entries unused. For 16KB cluster, only one entry from the first + * page is allocated and used for the entire object. */ /* TODO: should be in header file */ /* kernel translater */ -extern vm_offset_t kmem_mb_alloc(vm_map_t, int); +extern vm_offset_t kmem_mb_alloc(vm_map_t, int, int); extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va); extern vm_map_t mb_map; /* special map */ /* Global lock */ -static lck_mtx_t *mbuf_mlock; +decl_lck_mtx_data(static, mbuf_mlock_data); +static lck_mtx_t *mbuf_mlock = &mbuf_mlock_data; static lck_attr_t *mbuf_mlock_attr; static lck_grp_t *mbuf_mlock_grp; static lck_grp_attr_t *mbuf_mlock_grp_attr; @@ -310,23 +314,27 @@ static void *mbuf_worker_run; /* wait channel for worker thread */ static int mbuf_worker_ready; /* worker thread is runnable */ static int mbuf_expand_mcl; /* number of cluster creation requets */ static int mbuf_expand_big; /* number of big cluster creation requests */ -static int mbuf_expand_16k; /* number of 16K cluster creation requests */ +static int mbuf_expand_16k; /* number of 16KB cluster creation requests */ static int ncpu; /* number of CPUs */ -static int *mcl_paddr; /* Array of cluster physical addresses */ +static ppnum_t *mcl_paddr; /* Array of cluster physical addresses */ +static ppnum_t mcl_pages; /* Size of array (# physical pages) */ static ppnum_t mcl_paddr_base; /* Handle returned by IOMapper::iovmAlloc() */ static mcache_t *ref_cache; /* Cache of cluster reference & flags */ static mcache_t *mcl_audit_con_cache; /* Audit contents cache */ static unsigned int mbuf_debug; /* patchable mbuf mcache flags */ static unsigned int mb_normalized; /* number of packets "normalized" */ +#define MB_GROWTH_AGGRESSIVE 1 /* Threshold: 1/2 of total */ +#define MB_GROWTH_NORMAL 2 /* Threshold: 3/4 of total */ + typedef enum { MC_MBUF = 0, /* Regular mbuf */ MC_CL, /* Cluster */ - MC_BIGCL, /* Large (4K) cluster */ - MC_16KCL, /* Jumbo (16K) cluster */ + MC_BIGCL, /* Large (4KB) cluster */ + MC_16KCL, /* Jumbo (16KB) cluster */ MC_MBUF_CL, /* mbuf + cluster */ - MC_MBUF_BIGCL, /* mbuf + large (4K) cluster */ - MC_MBUF_16KCL /* mbuf + jumbo (16K) cluster */ + MC_MBUF_BIGCL, /* mbuf + large (4KB) cluster */ + MC_MBUF_16KCL /* mbuf + jumbo (16KB) cluster */ } mbuf_class_t; #define MBUF_CLASS_MIN MC_MBUF @@ -365,6 +373,8 @@ typedef enum { * a cluster's size. In this case, only the slab of the first cluster is * used. The rest of the slabs are marked with SLF_PARTIAL to indicate * that they are part of the larger slab. + * + * Each slab controls a page of memory. */ typedef struct mcl_slab { struct mcl_slab *sl_next; /* neighboring slab */ @@ -388,24 +398,24 @@ typedef struct mcl_slab { * whenever a new piece of memory mapped in from the VM crosses the 1MB * boundary. */ -#define MBSHIFT 20 /* 1MB */ -#define NSLABSPMB ((1 << MBSHIFT) >> MCLSHIFT) /* 512 slabs/grp */ +#define NSLABSPMB ((1 << MBSHIFT) >> PGSHIFT) /* 256 slabs/grp */ typedef struct mcl_slabg { mcl_slab_t slg_slab[NSLABSPMB]; /* group of slabs */ } mcl_slabg_t; +/* + * Number of slabs needed to control a 16KB cluster object. + */ +#define NSLABSP16KB (M16KCLBYTES >> PGSHIFT) + /* * Per-cluster audit structure. */ typedef struct { - mcache_audit_t *cl_audit[NMBPCL]; /* array of audits */ + mcache_audit_t *cl_audit[NMBPBG]; /* array of audits */ } mcl_audit_t; -#if CONFIG_MBUF_NOEXPAND -static unsigned int maxmbufcl; -#endif /* CONFIG_MBUF_NOEXPAND */ - /* * Size of data from the beginning of an mbuf that covers m_hdr, pkthdr * and m_ext structures. If auditing is enabled, we allocate a shadow @@ -416,11 +426,7 @@ static unsigned int maxmbufcl; * Note that we don't save the contents of clusters when they are freed; * we simply pattern-fill them. */ -#if defined(__LP64__) -#define AUDIT_CONTENTS_SIZE 160 -#else -#define AUDIT_CONTENTS_SIZE 80 -#endif /* __LP64__ */ +#define AUDIT_CONTENTS_SIZE ((MSIZE - MHLEN) + sizeof (_m_ext_t)) /* * mbuf specific mcache audit flags @@ -433,6 +439,7 @@ static unsigned int maxmbufcl; * Each of the following two arrays hold up to nmbclusters elements. */ static mcl_audit_t *mclaudit; /* array of cluster audit information */ +static unsigned int maxclaudit; /* max # of entries in audit table */ static mcl_slabg_t **slabstbl; /* cluster slabs table */ static unsigned int maxslabgrp; /* max # of entries in slabs table */ static unsigned int slabgrp; /* # of entries in slabs table */ @@ -441,13 +448,88 @@ static unsigned int slabgrp; /* # of entries in slabs table */ int nclusters; /* # of clusters for non-jumbo (legacy) sizes */ int njcl; /* # of clusters for jumbo sizes */ int njclbytes; /* size of a jumbo cluster */ -union mcluster *mbutl; /* first mapped cluster address */ -union mcluster *embutl; /* ending virtual address of mclusters */ -int max_linkhdr; /* largest link-level header */ -int max_protohdr; /* largest protocol header */ +union mbigcluster *mbutl; /* first mapped cluster address */ +union mbigcluster *embutl; /* ending virtual address of mclusters */ +int _max_linkhdr; /* largest link-level header */ +int _max_protohdr; /* largest protocol header */ int max_hdr; /* largest link+protocol header */ int max_datalen; /* MHLEN - max_hdr */ +static boolean_t mclverify; /* debug: pattern-checking */ +static boolean_t mcltrace; /* debug: stack tracing */ +static boolean_t mclfindleak; /* debug: leak detection */ +static boolean_t mclexpleak; /* debug: expose leak info to user space */ + +/* mbuf leak detection variables */ +static struct mleak_table mleak_table; +static mleak_stat_t *mleak_stat; + +#define MLEAK_STAT_SIZE(n) \ + ((size_t)(&((mleak_stat_t *)0)->ml_trace[n])) + +struct mallocation { + mcache_obj_t *element; /* the alloc'ed element, NULL if unused */ + u_int32_t trace_index; /* mtrace index for corresponding backtrace */ + u_int32_t count; /* How many objects were requested */ + u_int64_t hitcount; /* for determining hash effectiveness */ +}; + +struct mtrace { + u_int64_t collisions; + u_int64_t hitcount; + u_int64_t allocs; + u_int64_t depth; + uintptr_t addr[MLEAK_STACK_DEPTH]; +}; + +/* Size must be a power of two for the zhash to be able to just mask off bits */ +#define MLEAK_ALLOCATION_MAP_NUM 512 +#define MLEAK_TRACE_MAP_NUM 256 + +/* + * Sample factor for how often to record a trace. This is overwritable + * by the boot-arg mleak_sample_factor. + */ +#define MLEAK_SAMPLE_FACTOR 500 + +/* + * Number of top leakers recorded. + */ +#define MLEAK_NUM_TRACES 5 + +#define MB_LEAK_SPACING_64 " " +#define MB_LEAK_SPACING_32 " " + + +#define MB_LEAK_HDR_32 "\n\ + trace [1] trace [2] trace [3] trace [4] trace [5] \n\ + ---------- ---------- ---------- ---------- ---------- \n\ +" + +#define MB_LEAK_HDR_64 "\n\ + trace [1] trace [2] trace [3] \ + trace [4] trace [5] \n\ + ------------------ ------------------ ------------------ \ + ------------------ ------------------ \n\ +" + +static uint32_t mleak_alloc_buckets = MLEAK_ALLOCATION_MAP_NUM; +static uint32_t mleak_trace_buckets = MLEAK_TRACE_MAP_NUM; + +/* Hashmaps of allocations and their corresponding traces */ +static struct mallocation *mleak_allocations; +static struct mtrace *mleak_traces; +static struct mtrace *mleak_top_trace[MLEAK_NUM_TRACES]; + +/* Lock to protect mleak tables from concurrent modification */ +decl_lck_mtx_data(static, mleak_lock_data); +static lck_mtx_t *mleak_lock = &mleak_lock_data; +static lck_attr_t *mleak_lock_attr; +static lck_grp_t *mleak_lock_grp; +static lck_grp_attr_t *mleak_lock_grp_attr; + +extern u_int32_t high_sb_max; + /* TODO: should be in header file */ int do_reclaim = 0; @@ -457,7 +539,6 @@ int do_reclaim = 0; #define MIN16KCL (MINCL >> 2) /* Low watermarks (only map in pages once free counts go below) */ -#define MCL_LOWAT MINCL #define MBIGCL_LOWAT MINBIGCL #define M16KCL_LOWAT MIN16KCL @@ -522,15 +603,36 @@ static mbuf_table_t mbuf_table[] = { #define NELEM(a) (sizeof (a) / sizeof ((a)[0])) static void *mb_waitchan = &mbuf_table; /* wait channel for all caches */ -static int mb_waiters; /* number of sleepers */ +static int mb_waiters; /* number of waiters */ + +#define MB_WDT_MAXTIME 10 /* # of secs before watchdog panic */ +static struct timeval mb_wdtstart; /* watchdog start timestamp */ +static char *mbuf_dump_buf; + +#define MBUF_DUMP_BUF_SIZE 2048 + +/* + * mbuf watchdog is enabled by default on embedded platforms. It is + * also toggeable via the kern.ipc.mb_watchdog sysctl. + */ +#if CONFIG_EMBEDDED +static unsigned int mb_watchdog = 1; +#else +static unsigned int mb_watchdog = 0; +#endif /* CONFIG_EMBEDDED */ /* The following are used to serialize m_clalloc() */ static boolean_t mb_clalloc_busy; static void *mb_clalloc_waitchan = &mb_clalloc_busy; static int mb_clalloc_waiters; +static void mbuf_mtypes_sync(boolean_t); static int mbstat_sysctl SYSCTL_HANDLER_ARGS; +static void mbuf_stat_sync(void); static int mb_stat_sysctl SYSCTL_HANDLER_ARGS; +static int mleak_top_trace_sysctl SYSCTL_HANDLER_ARGS; +static int mleak_table_sysctl SYSCTL_HANDLER_ARGS; +static char *mbuf_dump(void); static void mbuf_table_init(void); static inline void m_incref(struct mbuf *); static inline u_int32_t m_decref(struct mbuf *); @@ -551,11 +653,13 @@ static unsigned int mbuf_cslab_alloc(void *, mcache_obj_t ***, static void mbuf_cslab_free(void *, mcache_obj_t *, int); static void mbuf_cslab_audit(void *, mcache_obj_t *, boolean_t); static int freelist_populate(mbuf_class_t, unsigned int, int); +static void freelist_init(mbuf_class_t); static boolean_t mbuf_cached_above(mbuf_class_t, int); static boolean_t mbuf_steal(mbuf_class_t, unsigned int); static void m_reclaim(mbuf_class_t, unsigned int, boolean_t); static int m_howmany(int, size_t); static void mbuf_worker_thread(void); +static void mbuf_watchdog(void); static boolean_t mbuf_sleep(mbuf_class_t, unsigned int, int); static void mcl_audit_init(void *, mcache_audit_t **, mcache_obj_t **, @@ -569,6 +673,13 @@ static void mcl_audit_save_mbuf(struct mbuf *, mcache_audit_t *); static void mcl_audit_mcheck_panic(struct mbuf *); static void mcl_audit_verify_nextptr(void *, mcache_audit_t *); +static void mleak_activate(void); +static void mleak_logger(u_int32_t, mcache_obj_t *, boolean_t); +static boolean_t mleak_log(uintptr_t *, mcache_obj_t *, uint32_t, int); +static void mleak_free(mcache_obj_t *); +static void mleak_sort_traces(void); +static void mleak_update_stats(void); + static mcl_slab_t *slab_get(void *); static void slab_init(mcl_slab_t *, mbuf_class_t, u_int32_t, void *, void *, unsigned int, int, int); @@ -579,6 +690,15 @@ static void slab_nextptr_panic(mcl_slab_t *, void *); static void slab_detach(mcl_slab_t *); static boolean_t slab_is_detached(mcl_slab_t *); +static int m_copyback0(struct mbuf **, int, int, const void *, int, int); +static struct mbuf *m_split0(struct mbuf *, int, int, int); + +/* flags for m_copyback0 */ +#define M_COPYBACK0_COPYBACK 0x0001 /* copyback from cp */ +#define M_COPYBACK0_PRESERVE 0x0002 /* preserve original data */ +#define M_COPYBACK0_COW 0x0004 /* do copy-on-write */ +#define M_COPYBACK0_EXTEND 0x0008 /* extend chain */ + /* * This flag is set for all mbufs that come out of and into the composite * mbuf + cluster caches, i.e. MC_MBUF_CL and MC_MBUF_BIGCL. mbufs that @@ -592,11 +712,19 @@ static boolean_t slab_is_detached(mcl_slab_t *); */ #define EXTF_COMPOSITE 0x1 +/* + * This flag indicates that the external cluster is read-only, i.e. it is + * or was referred to by more than one mbufs. Once set, this flag is never + * cleared. + */ +#define EXTF_READONLY 0x2 +#define EXTF_MASK (EXTF_COMPOSITE | EXTF_READONLY) + #define MEXT_RFA(m) ((m)->m_ext.ext_refflags) #define MEXT_REF(m) (MEXT_RFA(m)->refcnt) #define MEXT_FLAGS(m) (MEXT_RFA(m)->flags) #define MBUF_IS_COMPOSITE(m) \ - (MEXT_REF(m) == 0 && (MEXT_FLAGS(m) & EXTF_COMPOSITE)) + (MEXT_REF(m) == 0 && (MEXT_FLAGS(m) & EXTF_MASK) == EXTF_COMPOSITE) /* * Macros used to verify the integrity of the mbuf. @@ -625,15 +753,21 @@ static boolean_t slab_is_detached(mcl_slab_t *); #define MTOD(m, t) ((t)((m)->m_data)) /* - * Macros to obtain cluster index and base cluster address. + * Macros to obtain (4KB) cluster index and base cluster address. + */ + +#define MTOBG(x) (((char *)(x) - (char *)mbutl) >> MBIGCLSHIFT) +#define BGTOM(x) ((union mbigcluster *)(mbutl + (x))) + +/* + * Macro to find the mbuf index relative to a base. */ -#define MTOCL(x) (((char *)(x) - (char *)mbutl) >> MCLSHIFT) -#define CLTOM(x) ((union mcluster *)(mbutl + (x))) +#define MCLIDX(c, m) (((char *)(m) - (char *)(c)) >> MSIZESHIFT) /* - * Macro to find the mbuf index relative to the cluster base. + * Same thing for 2KB cluster index. */ -#define MCLIDX(c, m) (((char *)(m) - (char *)(c)) >> 8) +#define CLBGIDX(c, m) (((char *)(m) - (char *)(c)) >> MCLSHIFT) /* * Macros used during mbuf and cluster initialization. @@ -654,10 +788,13 @@ static boolean_t slab_is_detached(mcl_slab_t *); (m)->m_pkthdr.header = NULL; \ (m)->m_pkthdr.csum_flags = 0; \ (m)->m_pkthdr.csum_data = 0; \ - (m)->m_pkthdr.reserved0 = NULL; \ + (m)->m_pkthdr.tso_segsz = 0; \ (m)->m_pkthdr.vlan_tag = 0; \ (m)->m_pkthdr.socket_id = 0; \ + (m)->m_pkthdr.vt_nrecs = 0; \ + (m)->m_pkthdr.aux_flags = 0; \ m_tag_init(m); \ + m_service_class_init(m); \ } \ } @@ -695,9 +832,12 @@ static boolean_t slab_is_detached(mcl_slab_t *); * that allows for a more accurate view of the state of the allocator. */ struct mb_stat *mb_stat; +struct omb_stat *omb_stat; /* For backwards compatibility */ #define MB_STAT_SIZE(n) \ ((size_t)(&((mb_stat_t *)0)->mbs_class[n])) +#define OMB_STAT_SIZE(n) \ + ((size_t)(&((struct omb_stat *)0)->mbs_class[n])) /* * The legacy structure holding all of the mbuf allocation statistics. @@ -740,17 +880,14 @@ static mbuf_mtypes_t *mbuf_mtypes; /* per-CPU statistics */ ((size_t)(&((mbuf_mtypes_t *)0)->mbs_cpu[n])) #define MTYPES_CPU(p) \ - ((mtypes_cpu_t *)((char *)(p) + MBUF_MTYPES_SIZE(cpu_number()))) - -/* This should be in a header file */ -#define atomic_add_32(a, n) ((void) OSAddAtomic(n, (volatile SInt32 *)a)) + ((mtypes_cpu_t *)(void *)((char *)(p) + MBUF_MTYPES_SIZE(cpu_number()))) #define mtype_stat_add(type, n) { \ if ((unsigned)(type) < MT_MAX) { \ mtypes_cpu_t *mbs = MTYPES_CPU(mbuf_mtypes); \ atomic_add_32(&mbs->cpu_mtypes[type], n); \ - } else if ((unsigned)(type) < MBSTAT_MTYPES_MAX) { \ - atomic_add_32(&mbstat.m_mtypes[type], n); \ + } else if ((unsigned)(type) < (unsigned)MBSTAT_MTYPES_MAX) { \ + atomic_add_16((int16_t *)&mbstat.m_mtypes[type], n); \ } \ } @@ -758,13 +895,15 @@ static mbuf_mtypes_t *mbuf_mtypes; /* per-CPU statistics */ #define mtype_stat_inc(t) mtype_stat_add(t, 1) #define mtype_stat_dec(t) mtype_stat_sub(t, 1) -static int -mbstat_sysctl SYSCTL_HANDLER_ARGS +static void +mbuf_mtypes_sync(boolean_t locked) { -#pragma unused(oidp, arg1, arg2) int m, n; mtypes_cpu_t mtc; + if (locked) + lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + bzero(&mtc, sizeof (mtc)); for (m = 0; m < ncpu; m++) { mtypes_cpu_t *scp = &mbuf_mtypes->mbs_cpu[m]; @@ -776,24 +915,33 @@ mbstat_sysctl SYSCTL_HANDLER_ARGS for (n = 0; n < MT_MAX; n++) mtc.cpu_mtypes[n] += temp.cpu_mtypes[n]; } - lck_mtx_lock(mbuf_mlock); + if (!locked) + lck_mtx_lock(mbuf_mlock); for (n = 0; n < MT_MAX; n++) mbstat.m_mtypes[n] = mtc.cpu_mtypes[n]; - lck_mtx_unlock(mbuf_mlock); - - return (SYSCTL_OUT(req, &mbstat, sizeof (mbstat))); + if (!locked) + lck_mtx_unlock(mbuf_mlock); } static int -mb_stat_sysctl SYSCTL_HANDLER_ARGS +mbstat_sysctl SYSCTL_HANDLER_ARGS { #pragma unused(oidp, arg1, arg2) - mcache_t *cp; - mcache_cpu_t *ccp; + mbuf_mtypes_sync(FALSE); + + return (SYSCTL_OUT(req, &mbstat, sizeof (mbstat))); +} + +static void +mbuf_stat_sync(void) +{ mb_class_stat_t *sp; + mcache_cpu_t *ccp; + mcache_t *cp; int k, m, bktsize; - lck_mtx_lock(mbuf_mlock); + lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + for (k = 0; k < NELEM(mbuf_table); k++) { cp = m_cache(k); ccp = &cp->mc_cpu[0]; @@ -835,9 +983,8 @@ mb_stat_sysctl SYSCTL_HANDLER_ARGS break; case MC_CL: - /* Deduct clusters used in composite cache and mbufs */ - sp->mbcl_ctotal -= (m_total(MC_MBUF_CL) + - (P2ROUNDUP(m_total(MC_MBUF), NMBPCL)/NMBPCL)); + /* Deduct clusters used in composite cache */ + sp->mbcl_ctotal -= m_total(MC_MBUF_CL); break; case MC_BIGCL: @@ -854,9 +1001,90 @@ mb_stat_sysctl SYSCTL_HANDLER_ARGS break; } } +} + +static int +mb_stat_sysctl SYSCTL_HANDLER_ARGS +{ +#pragma unused(oidp, arg1, arg2) + void *statp; + int k, statsz, proc64 = proc_is64bit(req->p); + + lck_mtx_lock(mbuf_mlock); + mbuf_stat_sync(); + + if (!proc64) { + struct omb_class_stat *oc; + struct mb_class_stat *c; + + omb_stat->mbs_cnt = mb_stat->mbs_cnt; + oc = &omb_stat->mbs_class[0]; + c = &mb_stat->mbs_class[0]; + for (k = 0; k < omb_stat->mbs_cnt; k++, oc++, c++) { + (void) snprintf(oc->mbcl_cname, sizeof (oc->mbcl_cname), + "%s", c->mbcl_cname); + oc->mbcl_size = c->mbcl_size; + oc->mbcl_total = c->mbcl_total; + oc->mbcl_active = c->mbcl_active; + oc->mbcl_infree = c->mbcl_infree; + oc->mbcl_slab_cnt = c->mbcl_slab_cnt; + oc->mbcl_alloc_cnt = c->mbcl_alloc_cnt; + oc->mbcl_free_cnt = c->mbcl_free_cnt; + oc->mbcl_notified = c->mbcl_notified; + oc->mbcl_purge_cnt = c->mbcl_purge_cnt; + oc->mbcl_fail_cnt = c->mbcl_fail_cnt; + oc->mbcl_ctotal = c->mbcl_ctotal; + oc->mbcl_mc_state = c->mbcl_mc_state; + oc->mbcl_mc_cached = c->mbcl_mc_cached; + oc->mbcl_mc_waiter_cnt = c->mbcl_mc_waiter_cnt; + oc->mbcl_mc_wretry_cnt = c->mbcl_mc_wretry_cnt; + oc->mbcl_mc_nwretry_cnt = c->mbcl_mc_nwretry_cnt; + } + statp = omb_stat; + statsz = OMB_STAT_SIZE(NELEM(mbuf_table)); + } else { + statp = mb_stat; + statsz = MB_STAT_SIZE(NELEM(mbuf_table)); + } + lck_mtx_unlock(mbuf_mlock); - return (SYSCTL_OUT(req, mb_stat, MB_STAT_SIZE(NELEM(mbuf_table)))); + return (SYSCTL_OUT(req, statp, statsz)); +} + +static int +mleak_top_trace_sysctl SYSCTL_HANDLER_ARGS +{ +#pragma unused(oidp, arg1, arg2) + int i; + + /* Ensure leak tracing turned on */ + if (!mclfindleak || !mclexpleak) + return (ENXIO); + + lck_mtx_lock(mleak_lock); + mleak_update_stats(); + i = SYSCTL_OUT(req, mleak_stat, MLEAK_STAT_SIZE(MLEAK_NUM_TRACES)); + lck_mtx_unlock(mleak_lock); + + return (i); +} + +static int +mleak_table_sysctl SYSCTL_HANDLER_ARGS +{ +#pragma unused(oidp, arg1, arg2) + int i = 0; + + /* Ensure leak tracing turned on */ + if (!mclfindleak || !mclexpleak) + return (ENXIO); + + lck_mtx_lock(mleak_lock); + i = SYSCTL_OUT(req, &mleak_table, sizeof (mleak_table)); + lck_mtx_unlock(mleak_lock); + + return (i); } static inline void @@ -870,6 +1098,14 @@ m_incref(struct mbuf *m) new = old + 1; ASSERT(new != 0); } while (!OSCompareAndSwap(old, new, addr)); + + /* + * If cluster is shared, mark it with (sticky) EXTF_READONLY; + * we don't clear the flag when the refcount goes back to 1 + * to simplify code calling m_mclhasreference(). + */ + if (new > 1 && !(MEXT_FLAGS(m) & EXTF_READONLY)) + (void) OSBitOrAtomic(EXTF_READONLY, &MEXT_FLAGS(m)); } static inline u_int32_t @@ -890,8 +1126,13 @@ m_decref(struct mbuf *m) static void mbuf_table_init(void) { + unsigned int b, c, s; int m; + MALLOC(omb_stat, struct omb_stat *, OMB_STAT_SIZE(NELEM(mbuf_table)), + M_TEMP, M_WAITOK | M_ZERO); + VERIFY(omb_stat != NULL); + MALLOC(mb_stat, mb_stat_t *, MB_STAT_SIZE(NELEM(mbuf_table)), M_TEMP, M_WAITOK | M_ZERO); VERIFY(mb_stat != NULL); @@ -910,66 +1151,78 @@ mbuf_table_init(void) #endif /* CONFIG_MBUF_JUMBO */ /* - * nclusters is going to be split in 2 to hold both the 2K - * and the 4K pools, so make sure each half is even. + * nclusters holds both the 2KB and 4KB pools, so ensure it's + * a multiple of 4KB clusters. */ - nclusters = P2ROUNDDOWN(nmbclusters - njcl, 4); + nclusters = P2ROUNDDOWN(nmbclusters - njcl, NCLPBG); if (njcl > 0) { /* - * Each jumbo cluster takes 8 2K clusters, so make - * sure that the pool size is evenly divisible by 8. + * Each jumbo cluster takes 8 2KB clusters, so make + * sure that the pool size is evenly divisible by 8; + * njcl is in 2KB unit, hence treated as such. */ njcl = P2ROUNDDOWN(nmbclusters - nclusters, 8); - } -#if CONFIG_MBUF_NOEXPAND - /* Only use 4k clusters if we're setting aside more than 256k */ - if (nmbclusters <= 128) { - maxmbufcl = nmbclusters / 4; - } else { - /* Half to big clusters, half to small */ - maxmbufcl = (nmbclusters / 4) * 3; + /* Update nclusters with rounded down value of njcl */ + nclusters = P2ROUNDDOWN(nmbclusters - njcl, NCLPBG); } -#endif /* CONFIG_MBUF_NOEXPAND */ /* - * 1/2 of the map is reserved for 2K clusters. Out of this, 1/16th - * of the total number of 2K clusters allocated is reserved and cannot - * be turned into mbufs. It can only be used for pure cluster objects. + * njcl is valid only on platforms with 16KB jumbo clusters, where + * it is configured to 1/3 of the pool size. On these platforms, + * the remaining is used for 2KB and 4KB clusters. On platforms + * without 16KB jumbo clusters, the entire pool is used for both + * 2KB and 4KB clusters. A 4KB cluster can either be splitted into + * 16 mbufs, or into 2 2KB clusters. + * + * +---+---+------------ ... -----------+------- ... -------+ + * | c | b | s | njcl | + * +---+---+------------ ... -----------+------- ... -------+ + * + * 1/32th of the shared region is reserved for pure 2KB and 4KB + * clusters (1/64th each.) + */ + c = P2ROUNDDOWN((nclusters >> 6), 2); /* in 2KB unit */ + b = P2ROUNDDOWN((nclusters >> (6 + NCLPBGSHIFT)), 2); /* in 4KB unit */ + s = nclusters - (c + (b << NCLPBGSHIFT)); /* in 2KB unit */ + + /* + * 1/64th (c) is reserved for 2KB clusters. */ - m_minlimit(MC_CL) = (nclusters >> 5); - m_maxlimit(MC_CL) = (nclusters >> 1); + m_minlimit(MC_CL) = c; + m_maxlimit(MC_CL) = s + c; /* in 2KB unit */ m_maxsize(MC_CL) = m_size(MC_CL) = MCLBYTES; (void) snprintf(m_cname(MC_CL), MAX_MBUF_CNAME, "cl"); /* - * The remaining (15/16th) can be turned into mbufs. + * Another 1/64th (b) of the map is reserved for 4KB clusters. + * It cannot be turned into 2KB clusters or mbufs. */ - m_minlimit(MC_MBUF) = 0; - m_maxlimit(MC_MBUF) = (m_maxlimit(MC_CL) - m_minlimit(MC_CL)) * NMBPCL; - m_maxsize(MC_MBUF) = m_size(MC_MBUF) = MSIZE; - (void) snprintf(m_cname(MC_MBUF), MAX_MBUF_CNAME, "mbuf"); + m_minlimit(MC_BIGCL) = b; + m_maxlimit(MC_BIGCL) = (s >> NCLPBGSHIFT) + b; /* in 4KB unit */ + m_maxsize(MC_BIGCL) = m_size(MC_BIGCL) = MBIGCLBYTES; + (void) snprintf(m_cname(MC_BIGCL), MAX_MBUF_CNAME, "bigcl"); /* - * The other 1/2 of the map is reserved for 4K clusters. + * The remaining 31/32ths (s) are all-purpose (mbufs, 2KB, or 4KB) */ - m_minlimit(MC_BIGCL) = 0; - m_maxlimit(MC_BIGCL) = m_maxlimit(MC_CL) >> 1; - m_maxsize(MC_BIGCL) = m_size(MC_BIGCL) = NBPG; - (void) snprintf(m_cname(MC_BIGCL), MAX_MBUF_CNAME, "bigcl"); + m_minlimit(MC_MBUF) = 0; + m_maxlimit(MC_MBUF) = (s << NMBPCLSHIFT); /* in mbuf unit */ + m_maxsize(MC_MBUF) = m_size(MC_MBUF) = MSIZE; + (void) snprintf(m_cname(MC_MBUF), MAX_MBUF_CNAME, "mbuf"); /* * Set limits for the composite classes. */ m_minlimit(MC_MBUF_CL) = 0; - m_maxlimit(MC_MBUF_CL) = m_maxlimit(MC_CL) - m_minlimit(MC_CL); + m_maxlimit(MC_MBUF_CL) = m_maxlimit(MC_CL); m_maxsize(MC_MBUF_CL) = MCLBYTES; m_size(MC_MBUF_CL) = m_size(MC_MBUF) + m_size(MC_CL); (void) snprintf(m_cname(MC_MBUF_CL), MAX_MBUF_CNAME, "mbuf_cl"); m_minlimit(MC_MBUF_BIGCL) = 0; m_maxlimit(MC_MBUF_BIGCL) = m_maxlimit(MC_BIGCL); - m_maxsize(MC_MBUF_BIGCL) = NBPG; + m_maxsize(MC_MBUF_BIGCL) = MBIGCLBYTES; m_size(MC_MBUF_BIGCL) = m_size(MC_MBUF) + m_size(MC_BIGCL); (void) snprintf(m_cname(MC_MBUF_BIGCL), MAX_MBUF_CNAME, "mbuf_bigcl"); @@ -977,7 +1230,7 @@ mbuf_table_init(void) * And for jumbo classes. */ m_minlimit(MC_16KCL) = 0; - m_maxlimit(MC_16KCL) = (njcl >> 3); + m_maxlimit(MC_16KCL) = (njcl >> NCLPJCLSHIFT); /* in 16KB unit */ m_maxsize(MC_16KCL) = m_size(MC_16KCL) = M16KCLBYTES; (void) snprintf(m_cname(MC_16KCL), MAX_MBUF_CNAME, "16kcl"); @@ -999,17 +1252,142 @@ mbuf_table_init(void) mbstat.m_bigmclbytes = m_maxsize(MC_BIGCL); } +#if defined(__LP64__) +typedef struct ncl_tbl { + uint64_t nt_maxmem; /* memory (sane) size */ + uint32_t nt_mbpool; /* mbuf pool size */ +} ncl_tbl_t; + +/* Non-server */ +static ncl_tbl_t ncl_table[] = { + { (1ULL << GBSHIFT) /* 1 GB */, (64 << MBSHIFT) /* 64 MB */ }, + { (1ULL << (GBSHIFT + 3)) /* 8 GB */, (96 << MBSHIFT) /* 96 MB */ }, + { (1ULL << (GBSHIFT + 4)) /* 16 GB */, (128 << MBSHIFT) /* 128 MB */ }, + { 0, 0 } +}; + +/* Server */ +static ncl_tbl_t ncl_table_srv[] = { + { (1ULL << GBSHIFT) /* 1 GB */, (96 << MBSHIFT) /* 96 MB */ }, + { (1ULL << (GBSHIFT + 2)) /* 4 GB */, (128 << MBSHIFT) /* 128 MB */ }, + { (1ULL << (GBSHIFT + 3)) /* 8 GB */, (160 << MBSHIFT) /* 160 MB */ }, + { (1ULL << (GBSHIFT + 4)) /* 16 GB */, (192 << MBSHIFT) /* 192 MB */ }, + { (1ULL << (GBSHIFT + 5)) /* 32 GB */, (256 << MBSHIFT) /* 256 MB */ }, + { (1ULL << (GBSHIFT + 6)) /* 64 GB */, (384 << MBSHIFT) /* 384 MB */ }, + { 0, 0 } +}; +#endif /* __LP64__ */ + +__private_extern__ unsigned int +mbuf_default_ncl(int server, uint64_t mem) +{ +#if !defined(__LP64__) +#pragma unused(server) + unsigned int n; + /* + * 32-bit kernel (default to 64MB of mbuf pool for >= 1GB RAM). + */ + if ((n = ((mem / 16) / MCLBYTES)) > 32768) + n = 32768; +#else + unsigned int n, i; + ncl_tbl_t *tbl = (server ? ncl_table_srv : ncl_table); + /* + * 64-bit kernel (mbuf pool size based on table). + */ + n = tbl[0].nt_mbpool; + for (i = 0; tbl[i].nt_mbpool != 0; i++) { + if (mem < tbl[i].nt_maxmem) + break; + n = tbl[i].nt_mbpool; + } + n >>= MCLSHIFT; +#endif /* !__LP64__ */ + return (n); +} + __private_extern__ void mbinit(void) { unsigned int m; - int initmcl = MINCL; - int mcl_pages; + unsigned int initmcl = 0; void *buf; + thread_t thread = THREAD_NULL; + + /* + * These MBUF_ values must be equal to their private counterparts. + */ + _CASSERT(MBUF_EXT == M_EXT); + _CASSERT(MBUF_PKTHDR == M_PKTHDR); + _CASSERT(MBUF_EOR == M_EOR); + _CASSERT(MBUF_LOOP == M_LOOP); + _CASSERT(MBUF_BCAST == M_BCAST); + _CASSERT(MBUF_MCAST == M_MCAST); + _CASSERT(MBUF_FRAG == M_FRAG); + _CASSERT(MBUF_FIRSTFRAG == M_FIRSTFRAG); + _CASSERT(MBUF_LASTFRAG == M_LASTFRAG); + _CASSERT(MBUF_PROMISC == M_PROMISC); + _CASSERT(MBUF_HASFCS == M_HASFCS); + + _CASSERT(MBUF_TYPE_FREE == MT_FREE); + _CASSERT(MBUF_TYPE_DATA == MT_DATA); + _CASSERT(MBUF_TYPE_HEADER == MT_HEADER); + _CASSERT(MBUF_TYPE_SOCKET == MT_SOCKET); + _CASSERT(MBUF_TYPE_PCB == MT_PCB); + _CASSERT(MBUF_TYPE_RTABLE == MT_RTABLE); + _CASSERT(MBUF_TYPE_HTABLE == MT_HTABLE); + _CASSERT(MBUF_TYPE_ATABLE == MT_ATABLE); + _CASSERT(MBUF_TYPE_SONAME == MT_SONAME); + _CASSERT(MBUF_TYPE_SOOPTS == MT_SOOPTS); + _CASSERT(MBUF_TYPE_FTABLE == MT_FTABLE); + _CASSERT(MBUF_TYPE_RIGHTS == MT_RIGHTS); + _CASSERT(MBUF_TYPE_IFADDR == MT_IFADDR); + _CASSERT(MBUF_TYPE_CONTROL == MT_CONTROL); + _CASSERT(MBUF_TYPE_OOBDATA == MT_OOBDATA); + + _CASSERT(MBUF_TSO_IPV4 == CSUM_TSO_IPV4); + _CASSERT(MBUF_TSO_IPV6 == CSUM_TSO_IPV6); + _CASSERT(MBUF_CSUM_REQ_SUM16 == CSUM_TCP_SUM16); + _CASSERT(MBUF_CSUM_TCP_SUM16 == MBUF_CSUM_REQ_SUM16); + _CASSERT(MBUF_CSUM_REQ_IP == CSUM_IP); + _CASSERT(MBUF_CSUM_REQ_TCP == CSUM_TCP); + _CASSERT(MBUF_CSUM_REQ_UDP == CSUM_UDP); + _CASSERT(MBUF_CSUM_REQ_TCPIPV6 == CSUM_TCPIPV6); + _CASSERT(MBUF_CSUM_REQ_UDPIPV6 == CSUM_UDPIPV6); + _CASSERT(MBUF_CSUM_DID_IP == CSUM_IP_CHECKED); + _CASSERT(MBUF_CSUM_IP_GOOD == CSUM_IP_VALID); + _CASSERT(MBUF_CSUM_DID_DATA == CSUM_DATA_VALID); + _CASSERT(MBUF_CSUM_PSEUDO_HDR == CSUM_PSEUDO_HDR); + + _CASSERT(MBUF_WAITOK == M_WAIT); + _CASSERT(MBUF_DONTWAIT == M_DONTWAIT); + _CASSERT(MBUF_COPYALL == M_COPYALL); + + _CASSERT(MBUF_PKTAUXF_INET_RESOLVE_RTR == MAUXF_INET_RESOLVE_RTR); + _CASSERT(MBUF_PKTAUXF_INET6_RESOLVE_RTR == MAUXF_INET6_RESOLVE_RTR); + + _CASSERT(MBUF_SC2TC(MBUF_SC_BK_SYS) == MBUF_TC_BK); + _CASSERT(MBUF_SC2TC(MBUF_SC_BK) == MBUF_TC_BK); + _CASSERT(MBUF_SC2TC(MBUF_SC_BE) == MBUF_TC_BE); + _CASSERT(MBUF_SC2TC(MBUF_SC_RD) == MBUF_TC_BE); + _CASSERT(MBUF_SC2TC(MBUF_SC_OAM) == MBUF_TC_BE); + _CASSERT(MBUF_SC2TC(MBUF_SC_AV) == MBUF_TC_VI); + _CASSERT(MBUF_SC2TC(MBUF_SC_RV) == MBUF_TC_VI); + _CASSERT(MBUF_SC2TC(MBUF_SC_VI) == MBUF_TC_VI); + _CASSERT(MBUF_SC2TC(MBUF_SC_VO) == MBUF_TC_VO); + _CASSERT(MBUF_SC2TC(MBUF_SC_CTL) == MBUF_TC_VO); + + _CASSERT(MBUF_TC2SCVAL(MBUF_TC_BK) == SCVAL_BK); + _CASSERT(MBUF_TC2SCVAL(MBUF_TC_BE) == SCVAL_BE); + _CASSERT(MBUF_TC2SCVAL(MBUF_TC_VI) == SCVAL_VI); + _CASSERT(MBUF_TC2SCVAL(MBUF_TC_VO) == SCVAL_VO); if (nmbclusters == 0) nmbclusters = NMBCLUSTERS; + /* This should be a sane (at least even) value by now */ + VERIFY(nmbclusters != 0 && !(nmbclusters & 0x1)); + /* Setup the mbuf table */ mbuf_table_init(); @@ -1017,57 +1395,108 @@ mbinit(void) mbuf_mlock_grp_attr = lck_grp_attr_alloc_init(); mbuf_mlock_grp = lck_grp_alloc_init("mbuf", mbuf_mlock_grp_attr); mbuf_mlock_attr = lck_attr_alloc_init(); - mbuf_mlock = lck_mtx_alloc_init(mbuf_mlock_grp, mbuf_mlock_attr); + lck_mtx_init(mbuf_mlock, mbuf_mlock_grp, mbuf_mlock_attr); - /* Allocate cluster slabs table */ - maxslabgrp = P2ROUNDUP(nmbclusters, NSLABSPMB) / NSLABSPMB; + /* + * Allocate cluster slabs table: + * + * maxslabgrp = (N * 2048) / (1024 * 1024) + * + * Where N is nmbclusters rounded up to the nearest 512. This yields + * mcl_slab_g_t units, each one representing a MB of memory. + */ + maxslabgrp = + (P2ROUNDUP(nmbclusters, (MBSIZE >> 11)) << MCLSHIFT) >> MBSHIFT; MALLOC(slabstbl, mcl_slabg_t **, maxslabgrp * sizeof (mcl_slabg_t *), M_TEMP, M_WAITOK | M_ZERO); VERIFY(slabstbl != NULL); - /* Allocate audit structures if needed */ - PE_parse_boot_arg("mbuf_debug", &mbuf_debug); + /* + * Allocate audit structures, if needed: + * + * maxclaudit = (maxslabgrp * 1024 * 1024) / 4096 + * + * This yields mcl_audit_t units, each one representing a page. + */ + PE_parse_boot_argn("mbuf_debug", &mbuf_debug, sizeof (mbuf_debug)); mbuf_debug |= mcache_getflags(); - if (mbuf_debug & MCF_AUDIT) { - MALLOC(mclaudit, mcl_audit_t *, - nmbclusters * sizeof (*mclaudit), M_TEMP, - M_WAITOK | M_ZERO); + if (mbuf_debug & MCF_DEBUG) { + maxclaudit = ((maxslabgrp << MBSHIFT) >> PGSHIFT); + MALLOC(mclaudit, mcl_audit_t *, maxclaudit * sizeof (*mclaudit), + M_TEMP, M_WAITOK | M_ZERO); VERIFY(mclaudit != NULL); mcl_audit_con_cache = mcache_create("mcl_audit_contents", AUDIT_CONTENTS_SIZE, 0, 0, MCR_SLEEP); VERIFY(mcl_audit_con_cache != NULL); } + mclverify = (mbuf_debug & MCF_VERIFY); + mcltrace = (mbuf_debug & MCF_TRACE); + mclfindleak = !(mbuf_debug & MCF_NOLEAKLOG); + mclexpleak = mclfindleak && (mbuf_debug & MCF_EXPLEAKLOG); + + /* Enable mbuf leak logging, with a lock to protect the tables */ + + mleak_lock_grp_attr = lck_grp_attr_alloc_init(); + mleak_lock_grp = lck_grp_alloc_init("mleak_lock", mleak_lock_grp_attr); + mleak_lock_attr = lck_attr_alloc_init(); + lck_mtx_init(mleak_lock, mleak_lock_grp, mleak_lock_attr); + + mleak_activate(); /* Calculate the number of pages assigned to the cluster pool */ - mcl_pages = nmbclusters/(NBPG/CLBYTES); - MALLOC(mcl_paddr, int *, mcl_pages * sizeof (int), M_TEMP, M_WAITOK); + mcl_pages = (nmbclusters * MCLBYTES) / CLBYTES; + MALLOC(mcl_paddr, ppnum_t *, mcl_pages * sizeof (ppnum_t), + M_TEMP, M_WAITOK); VERIFY(mcl_paddr != NULL); /* Register with the I/O Bus mapper */ mcl_paddr_base = IOMapperIOVMAlloc(mcl_pages); - bzero((char *)mcl_paddr, mcl_pages * sizeof (int)); - - embutl = (union mcluster *) - ((unsigned char *)mbutl + (nmbclusters * MCLBYTES)); - - PE_parse_boot_arg("initmcl", &initmcl); + bzero((char *)mcl_paddr, mcl_pages * sizeof (ppnum_t)); + + embutl = (union mbigcluster *) + ((void *)((unsigned char *)mbutl + (nmbclusters * MCLBYTES))); + VERIFY((((char *)embutl - (char *)mbutl) % MBIGCLBYTES) == 0); + + /* Prime up the freelist */ + PE_parse_boot_argn("initmcl", &initmcl, sizeof (initmcl)); + if (initmcl != 0) { + initmcl >>= NCLPBGSHIFT; /* become a 4K unit */ + if (initmcl > m_maxlimit(MC_BIGCL)) + initmcl = m_maxlimit(MC_BIGCL); + } + if (initmcl < m_minlimit(MC_BIGCL)) + initmcl = m_minlimit(MC_BIGCL); lck_mtx_lock(mbuf_mlock); - if (m_clalloc(MAX(NBPG/CLBYTES, 1) * initmcl, M_WAIT, MCLBYTES) == 0) - panic("mbinit: m_clalloc failed\n"); + /* + * For classes with non-zero minimum limits, populate their freelists + * so that m_total(class) is at least m_minlimit(class). + */ + VERIFY(m_total(MC_BIGCL) == 0 && m_minlimit(MC_BIGCL) != 0); + freelist_populate(m_class(MC_BIGCL), initmcl, M_WAIT); + VERIFY(m_total(MC_BIGCL) >= m_minlimit(MC_BIGCL)); + freelist_init(m_class(MC_CL)); + + for (m = 0; m < NELEM(mbuf_table); m++) { + /* Make sure we didn't miss any */ + VERIFY(m_minlimit(m_class(m)) == 0 || + m_total(m_class(m)) >= m_minlimit(m_class(m))); + } lck_mtx_unlock(mbuf_mlock); - (void) kernel_thread(kernel_task, mbuf_worker_thread_init); + (void) kernel_thread_start((thread_continue_t)mbuf_worker_thread_init, + NULL, &thread); + thread_deallocate(thread); ref_cache = mcache_create("mext_ref", sizeof (struct ext_ref), 0, 0, MCR_SLEEP); /* Create the cache for each class */ for (m = 0; m < NELEM(mbuf_table); m++) { - void *allocfunc, *freefunc, *auditfunc; + void *allocfunc, *freefunc, *auditfunc, *logfunc; u_int32_t flags; flags = mbuf_debug; @@ -1076,10 +1505,12 @@ mbinit(void) allocfunc = mbuf_cslab_alloc; freefunc = mbuf_cslab_free; auditfunc = mbuf_cslab_audit; + logfunc = mleak_logger; } else { allocfunc = mbuf_slab_alloc; freefunc = mbuf_slab_free; auditfunc = mbuf_slab_audit; + logfunc = mleak_logger; } /* @@ -1092,9 +1523,12 @@ mbinit(void) njcl == 0) flags |= MCF_NOCPUCACHE; + if (!mclfindleak) + flags |= MCF_NOLEAKLOG; + m_cache(m) = mcache_create_ext(m_cname(m), m_maxsize(m), - allocfunc, freefunc, auditfunc, mbuf_slab_notify, - (void *)m, flags, MCR_SLEEP); + allocfunc, freefunc, auditfunc, logfunc, mbuf_slab_notify, + (void *)(uintptr_t)m, flags, MCR_SLEEP); } /* @@ -1111,7 +1545,35 @@ mbinit(void) mbuf_mtypes = (mbuf_mtypes_t *)P2ROUNDUP((intptr_t)buf, CPU_CACHE_SIZE); bzero(mbuf_mtypes, MBUF_MTYPES_SIZE(ncpu)); - printf("mbinit: done\n"); + /* + * Set the max limit on sb_max to be 1/16 th of the size of + * memory allocated for mbuf clusters. + */ + high_sb_max = (nmbclusters << (MCLSHIFT - 4)); + if (high_sb_max < sb_max) { + /* sb_max is too large for this configuration, scale it down */ + if (high_sb_max > (1 << MBSHIFT)) { + /* We have atleast 16 M of mbuf pool */ + sb_max = high_sb_max; + } else if ((nmbclusters << MCLSHIFT) > (1 << MBSHIFT)) { + /* + * If we have more than 1M of mbufpool, cap the size of + * max sock buf at 1M + */ + sb_max = high_sb_max = (1 << MBSHIFT); + } else { + sb_max = high_sb_max; + } + } + + /* allocate space for mbuf_dump_buf */ + MALLOC(mbuf_dump_buf, char *, MBUF_DUMP_BUF_SIZE, M_TEMP, M_WAITOK); + VERIFY(mbuf_dump_buf != NULL); + + printf("mbinit: done [%d MB total pool size, (%d/%d) split]\n", + (nmbclusters << MCLSHIFT) >> MBSHIFT, + (nclusters << MCLSHIFT) >> MBSHIFT, + (njcl << MCLSHIFT) >> MBSHIFT); } /* @@ -1137,7 +1599,7 @@ slab_alloc(mbuf_class_t class, int wait) * more than one buffer chunks (e.g. mbuf slabs). For other * slabs, this probably doesn't make much of a difference. */ - if (class == MC_MBUF && (wait & MCR_COMP)) + if ((class == MC_MBUF || class == MC_CL) && (wait & MCR_COMP)) sp = (mcl_slab_t *)TAILQ_LAST(&m_slablist(class), mcl_slhead); else sp = (mcl_slab_t *)TAILQ_FIRST(&m_slablist(class)); @@ -1157,7 +1619,10 @@ slab_alloc(mbuf_class_t class, int wait) if (class == MC_MBUF) { sp->sl_head = buf->obj_next; - VERIFY(sp->sl_head != NULL || sp->sl_refcnt == (NMBPCL - 1)); + VERIFY(sp->sl_head != NULL || sp->sl_refcnt == (NMBPBG - 1)); + } else if (class == MC_CL) { + sp->sl_head = buf->obj_next; + VERIFY(sp->sl_head != NULL || sp->sl_refcnt == (NCLPBG - 1)); } else { sp->sl_head = NULL; } @@ -1182,41 +1647,33 @@ slab_alloc(mbuf_class_t class, int wait) if (class == MC_CL) { mbstat.m_clfree = (--m_infree(MC_CL)) + m_infree(MC_MBUF_CL); /* - * A 2K cluster slab can have at most 1 reference. + * A 2K cluster slab can have at most NCLPBG references. */ - VERIFY(sp->sl_refcnt == 1 && sp->sl_chunks == 1 && - sp->sl_len == m_maxsize(MC_CL) && sp->sl_head == NULL); + VERIFY(sp->sl_refcnt >= 1 && sp->sl_refcnt <= NCLPBG && + sp->sl_chunks == NCLPBG && + sp->sl_len == m_maxsize(MC_BIGCL)); + VERIFY(sp->sl_refcnt < NCLPBG || sp->sl_head == NULL); } else if (class == MC_BIGCL) { - mcl_slab_t *nsp = sp->sl_next; mbstat.m_bigclfree = (--m_infree(MC_BIGCL)) + m_infree(MC_MBUF_BIGCL); /* - * Increment 2nd slab. A 4K big cluster takes - * 2 slabs, each having at most 1 reference. + * A 4K cluster slab can have at most 1 reference. */ VERIFY(sp->sl_refcnt == 1 && sp->sl_chunks == 1 && - sp->sl_len == m_maxsize(MC_BIGCL) && sp->sl_head == NULL); - /* Next slab must already be present */ - VERIFY(nsp != NULL); - nsp->sl_refcnt++; - VERIFY(!slab_is_detached(nsp)); - VERIFY(nsp->sl_class == MC_BIGCL && - nsp->sl_flags == (SLF_MAPPED | SLF_PARTIAL) && - nsp->sl_refcnt == 1 && nsp->sl_chunks == 0 && - nsp->sl_len == 0 && nsp->sl_base == sp->sl_base && - nsp->sl_head == NULL); + sp->sl_len == m_maxsize(class) && sp->sl_head == NULL); } else if (class == MC_16KCL) { mcl_slab_t *nsp; int k; --m_infree(MC_16KCL); VERIFY(sp->sl_refcnt == 1 && sp->sl_chunks == 1 && - sp->sl_len == m_maxsize(MC_16KCL) && sp->sl_head == NULL); + sp->sl_len == m_maxsize(class) && sp->sl_head == NULL); /* - * Increment 2nd-8th slab. A 16K big cluster takes - * 8 cluster slabs, each having at most 1 reference. + * Increment 2nd-Nth slab reference, where N is NSLABSP16KB. + * A 16KB big cluster takes NSLABSP16KB slabs, each having at + * most 1 reference. */ - for (nsp = sp, k = 1; k < (M16KCLBYTES / MCLBYTES); k++) { + for (nsp = sp, k = 1; k < NSLABSP16KB; k++) { nsp = nsp->sl_next; /* Next slab must already be present */ VERIFY(nsp != NULL); @@ -1229,7 +1686,7 @@ slab_alloc(mbuf_class_t class, int wait) nsp->sl_head == NULL); } } else { - ASSERT(class == MC_MBUF); + VERIFY(class == MC_MBUF); --m_infree(MC_MBUF); /* * If auditing is turned on, this check is @@ -1239,20 +1696,20 @@ slab_alloc(mbuf_class_t class, int wait) _MCHECK((struct mbuf *)buf); /* * Since we have incremented the reference count above, - * an mbuf slab (formerly a 2K cluster slab that was cut + * an mbuf slab (formerly a 4KB cluster slab that was cut * up into mbufs) must have a reference count between 1 - * and NMBPCL at this point. + * and NMBPBG at this point. */ - VERIFY(sp->sl_refcnt >= 1 && - (unsigned short)sp->sl_refcnt <= NMBPCL && - sp->sl_chunks == NMBPCL && sp->sl_len == m_maxsize(MC_CL)); - VERIFY((unsigned short)sp->sl_refcnt < NMBPCL || - sp->sl_head == NULL); + VERIFY(sp->sl_refcnt >= 1 && sp->sl_refcnt <= NMBPBG && + sp->sl_chunks == NMBPBG && + sp->sl_len == m_maxsize(MC_BIGCL)); + VERIFY(sp->sl_refcnt < NMBPBG || sp->sl_head == NULL); } /* If empty, remove this slab from the class's freelist */ if (sp->sl_head == NULL) { - VERIFY(class != MC_MBUF || sp->sl_refcnt == NMBPCL); + VERIFY(class != MC_MBUF || sp->sl_refcnt == NMBPBG); + VERIFY(class != MC_CL || sp->sl_refcnt == NCLPBG); slab_remove(sp, class); } @@ -1278,45 +1735,38 @@ slab_free(mbuf_class_t class, mcache_obj_t *buf) /* Decrement slab reference */ sp->sl_refcnt--; - if (class == MC_CL || class == MC_BIGCL) { + if (class == MC_CL) { + VERIFY(IS_P2ALIGNED(buf, MCLBYTES)); + /* + * A slab that has been splitted for 2KB clusters can have + * at most 1 outstanding reference at this point. + */ + VERIFY(sp->sl_refcnt >= 0 && sp->sl_refcnt <= (NCLPBG - 1) && + sp->sl_chunks == NCLPBG && + sp->sl_len == m_maxsize(MC_BIGCL)); + VERIFY(sp->sl_refcnt < (NCLPBG - 1) || + (slab_is_detached(sp) && sp->sl_head == NULL)); + } else if (class == MC_BIGCL) { VERIFY(IS_P2ALIGNED(buf, MCLBYTES)); /* - * A 2K cluster slab can have at most 1 reference + * A 4KB cluster slab can have at most 1 reference * which must be 0 at this point. */ VERIFY(sp->sl_refcnt == 0 && sp->sl_chunks == 1 && sp->sl_len == m_maxsize(class) && sp->sl_head == NULL); VERIFY(slab_is_detached(sp)); - if (class == MC_BIGCL) { - mcl_slab_t *nsp = sp->sl_next; - VERIFY(IS_P2ALIGNED(buf, NBPG)); - /* Next slab must already be present */ - VERIFY(nsp != NULL); - /* Decrement 2nd slab reference */ - nsp->sl_refcnt--; - /* - * A 4K big cluster takes 2 slabs, both - * must now have 0 reference. - */ - VERIFY(slab_is_detached(nsp)); - VERIFY(nsp->sl_class == MC_BIGCL && - (nsp->sl_flags & (SLF_MAPPED | SLF_PARTIAL)) && - nsp->sl_refcnt == 0 && nsp->sl_chunks == 0 && - nsp->sl_len == 0 && nsp->sl_base == sp->sl_base && - nsp->sl_head == NULL); - } } else if (class == MC_16KCL) { mcl_slab_t *nsp; int k; /* - * A 16K cluster takes 8 cluster slabs, all must + * A 16KB cluster takes NSLABSP16KB slabs, all must * now have 0 reference. */ - VERIFY(IS_P2ALIGNED(buf, NBPG)); + VERIFY(IS_P2ALIGNED(buf, MBIGCLBYTES)); VERIFY(sp->sl_refcnt == 0 && sp->sl_chunks == 1 && - sp->sl_len == m_maxsize(MC_16KCL) && sp->sl_head == NULL); + sp->sl_len == m_maxsize(class) && sp->sl_head == NULL); VERIFY(slab_is_detached(sp)); - for (nsp = sp, k = 1; k < (M16KCLBYTES / MCLBYTES); k++) { + for (nsp = sp, k = 1; k < NSLABSP16KB; k++) { nsp = nsp->sl_next; /* Next slab must already be present */ VERIFY(nsp != NULL); @@ -1330,14 +1780,15 @@ slab_free(mbuf_class_t class, mcache_obj_t *buf) } } else { /* - * An mbuf slab has a total of NMBPL reference counts. - * Since we have decremented the reference above, it - * must now be between 0 and NMBPCL-1. + * A slab that has been splitted for mbufs has at most NMBPBG + * reference counts. Since we have decremented one reference + * above, it must now be between 0 and NMBPBG-1. */ - VERIFY(sp->sl_refcnt >= 0 && - (unsigned short)sp->sl_refcnt <= (NMBPCL - 1) && - sp->sl_chunks == NMBPCL && sp->sl_len == m_maxsize(MC_CL)); - VERIFY(sp->sl_refcnt < (NMBPCL - 1) || + VERIFY(class == MC_MBUF); + VERIFY(sp->sl_refcnt >= 0 && sp->sl_refcnt <= (NMBPBG - 1) && + sp->sl_chunks == NMBPBG && + sp->sl_len == m_maxsize(MC_BIGCL)); + VERIFY(sp->sl_refcnt < (NMBPBG - 1) || (slab_is_detached(sp) && sp->sl_head == NULL)); } @@ -1348,12 +1799,15 @@ slab_free(mbuf_class_t class, mcache_obj_t *buf) */ if (mclaudit != NULL) { mcache_audit_t *mca = mcl_audit_buf2mca(class, buf); - mcache_audit_free_verify(mca, buf, 0, m_maxsize(class)); + if (mclverify) { + mcache_audit_free_verify(mca, buf, 0, m_maxsize(class)); + } mca->mca_uflags &= ~MB_SCVALID; } if (class == MC_CL) { mbstat.m_clfree = (++m_infree(MC_CL)) + m_infree(MC_MBUF_CL); + buf->obj_next = sp->sl_head; } else if (class == MC_BIGCL) { mbstat.m_bigclfree = (++m_infree(MC_BIGCL)) + m_infree(MC_MBUF_BIGCL); @@ -1365,14 +1819,25 @@ slab_free(mbuf_class_t class, mcache_obj_t *buf) } sp->sl_head = buf; - /* All mbufs are freed; return the cluster that we stole earlier */ - if (sp->sl_refcnt == 0 && class == MC_MBUF) { - int i = NMBPCL; - - m_total(MC_MBUF) -= NMBPCL; + /* + * If a slab has been splitted to either one which holds 2KB clusters, + * or one which holds mbufs, turn it back to one which holds a 4KB + * cluster. + */ + if (class == MC_MBUF && sp->sl_refcnt == 0 && + m_total(class) > m_minlimit(class) && + m_total(MC_BIGCL) < m_maxlimit(MC_BIGCL)) { + int i = NMBPBG; + + m_total(MC_BIGCL)++; + mbstat.m_bigclusters = m_total(MC_BIGCL); + m_total(MC_MBUF) -= NMBPBG; mbstat.m_mbufs = m_total(MC_MBUF); - m_infree(MC_MBUF) -= NMBPCL; - mtype_stat_add(MT_FREE, -NMBPCL); + m_infree(MC_MBUF) -= NMBPBG; + mtype_stat_add(MT_FREE, -((unsigned)NMBPBG)); + + VERIFY(m_total(MC_BIGCL) <= m_maxlimit(MC_BIGCL)); + VERIFY(m_total(MC_MBUF) >= m_minlimit(MC_MBUF)); while (i--) { struct mbuf *m = sp->sl_head; @@ -1385,19 +1850,58 @@ slab_free(mbuf_class_t class, mcache_obj_t *buf) /* Remove the slab from the mbuf class's slab list */ slab_remove(sp, class); - /* Reinitialize it as a 2K cluster slab */ - slab_init(sp, MC_CL, sp->sl_flags, sp->sl_base, sp->sl_base, + /* Reinitialize it as a 4KB cluster slab */ + slab_init(sp, MC_BIGCL, sp->sl_flags, sp->sl_base, sp->sl_base, sp->sl_len, 0, 1); - if (mclaudit != NULL) + if (mclverify) { mcache_set_pattern(MCACHE_FREE_PATTERN, - (caddr_t)sp->sl_head, m_maxsize(MC_CL)); + (caddr_t)sp->sl_head, m_maxsize(MC_BIGCL)); + } + mbstat.m_bigclfree = (++m_infree(MC_BIGCL)) + + m_infree(MC_MBUF_BIGCL); - mbstat.m_clfree = (++m_infree(MC_CL)) + m_infree(MC_MBUF_CL); + VERIFY(slab_is_detached(sp)); + /* And finally switch class */ + class = MC_BIGCL; + } else if (class == MC_CL && sp->sl_refcnt == 0 && + m_total(class) > m_minlimit(class) && + m_total(MC_BIGCL) < m_maxlimit(MC_BIGCL)) { + int i = NCLPBG; + + m_total(MC_BIGCL)++; + mbstat.m_bigclusters = m_total(MC_BIGCL); + m_total(MC_CL) -= NCLPBG; + mbstat.m_clusters = m_total(MC_CL); + m_infree(MC_CL) -= NCLPBG; + VERIFY(m_total(MC_BIGCL) <= m_maxlimit(MC_BIGCL)); + VERIFY(m_total(MC_CL) >= m_minlimit(MC_CL)); + + while (i--) { + union mcluster *c = sp->sl_head; + VERIFY(c != NULL); + sp->sl_head = c->mcl_next; + c->mcl_next = NULL; + } + VERIFY(sp->sl_head == NULL); + + /* Remove the slab from the 2KB cluster class's slab list */ + slab_remove(sp, class); + + /* Reinitialize it as a 4KB cluster slab */ + slab_init(sp, MC_BIGCL, sp->sl_flags, sp->sl_base, sp->sl_base, + sp->sl_len, 0, 1); + + if (mclverify) { + mcache_set_pattern(MCACHE_FREE_PATTERN, + (caddr_t)sp->sl_head, m_maxsize(MC_BIGCL)); + } + mbstat.m_bigclfree = (++m_infree(MC_BIGCL)) + + m_infree(MC_MBUF_BIGCL); VERIFY(slab_is_detached(sp)); /* And finally switch class */ - class = MC_CL; + class = MC_BIGCL; } /* Reinsert the slab to the class's slab list */ @@ -1456,6 +1960,9 @@ mbuf_slab_alloc(void *arg, mcache_obj_t ***plist, unsigned int num, int wait) if (mbuf_cached_above(class, wait)) break; + /* watchdog checkpoint */ + mbuf_watchdog(); + /* We have nothing and cannot block; give up */ if (wait & MCR_NOSLEEP) { if (!(wait & MCR_TRYHARD)) { @@ -1552,7 +2059,9 @@ mbuf_slab_audit(void *arg, mcache_obj_t *list, boolean_t alloc) ASSERT(!(mca->mca_uflags & MB_SCVALID)); } /* Record this transaction */ - mcache_buffer_log(mca, list, m_cache(class)); + if (mcltrace) + mcache_buffer_log(mca, list, m_cache(class)); + if (alloc) mca->mca_uflags |= MB_INUSE; else @@ -1619,16 +2128,17 @@ cslab_alloc(mbuf_class_t class, mcache_obj_t ***plist, unsigned int num) clsp = slab_get(cl); VERIFY(m->m_flags == M_EXT && cl != NULL); VERIFY(MEXT_RFA(m) != NULL && MBUF_IS_COMPOSITE(m)); - VERIFY(clsp->sl_refcnt == 1); - if (class == MC_MBUF_BIGCL) { - nsp = clsp->sl_next; - /* Next slab must already be present */ - VERIFY(nsp != NULL); - VERIFY(nsp->sl_refcnt == 1); - } else if (class == MC_MBUF_16KCL) { + + if (class == MC_MBUF_CL) { + VERIFY(clsp->sl_refcnt >= 1 && + clsp->sl_refcnt <= NCLPBG); + } else { + VERIFY(clsp->sl_refcnt == 1); + } + + if (class == MC_MBUF_16KCL) { int k; - for (nsp = clsp, k = 1; - k < (M16KCLBYTES / MCLBYTES); k++) { + for (nsp = clsp, k = 1; k < NSLABSP16KB; k++) { nsp = nsp->sl_next; /* Next slab must already be present */ VERIFY(nsp != NULL); @@ -1665,11 +2175,21 @@ cslab_free(mbuf_class_t class, mcache_obj_t *list, int purged) mcache_obj_t *ref_list = NULL; mcl_slab_t *clsp, *nsp; void *cl; + mbuf_class_t cl_class; ASSERT(MBUF_CLASS_VALID(class) && MBUF_CLASS_COMPOSITE(class)); VERIFY(class != MC_MBUF_16KCL || njcl > 0); lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + if (class == MC_MBUF_CL) { + cl_class = MC_CL; + } else if (class == MC_MBUF_BIGCL) { + cl_class = MC_BIGCL; + } else { + VERIFY(class == MC_MBUF_16KCL); + cl_class = MC_16KCL; + } + o = tail = list; while ((m = ms = (struct mbuf *)o) != NULL) { @@ -1678,37 +2198,33 @@ cslab_free(mbuf_class_t class, mcache_obj_t *list, int purged) /* Do the mbuf sanity checks */ if (mclaudit != NULL) { mca = mcl_audit_buf2mca(MC_MBUF, (mcache_obj_t *)m); - mcache_audit_free_verify(mca, m, 0, m_maxsize(MC_MBUF)); + if (mclverify) { + mcache_audit_free_verify(mca, m, 0, + m_maxsize(MC_MBUF)); + } ms = (struct mbuf *)mca->mca_contents; } /* Do the cluster sanity checks */ cl = ms->m_ext.ext_buf; clsp = slab_get(cl); - if (mclaudit != NULL) { - size_t size; - if (class == MC_MBUF_CL) - size = m_maxsize(MC_CL); - else if (class == MC_MBUF_BIGCL) - size = m_maxsize(MC_BIGCL); - else - size = m_maxsize(MC_16KCL); - mcache_audit_free_verify(mcl_audit_buf2mca(MC_CL, + if (mclverify) { + size_t size = m_maxsize(cl_class); + mcache_audit_free_verify(mcl_audit_buf2mca(cl_class, (mcache_obj_t *)cl), cl, 0, size); } VERIFY(ms->m_type == MT_FREE); VERIFY(ms->m_flags == M_EXT); VERIFY(MEXT_RFA(ms) != NULL && MBUF_IS_COMPOSITE(ms)); - VERIFY(clsp->sl_refcnt == 1); - if (class == MC_MBUF_BIGCL) { - nsp = clsp->sl_next; - /* Next slab must already be present */ - VERIFY(nsp != NULL); - VERIFY(nsp->sl_refcnt == 1); - } else if (class == MC_MBUF_16KCL) { + if (cl_class == MC_CL) { + VERIFY(clsp->sl_refcnt >= 1 && + clsp->sl_refcnt <= NCLPBG); + } else { + VERIFY(clsp->sl_refcnt == 1); + } + if (cl_class == MC_16KCL) { int k; - for (nsp = clsp, k = 1; - k < (M16KCLBYTES / MCLBYTES); k++) { + for (nsp = clsp, k = 1; k < NSLABSP16KB; k++) { nsp = nsp->sl_next; /* Next slab must already be present */ VERIFY(nsp != NULL); @@ -1730,7 +2246,7 @@ cslab_free(mbuf_class_t class, mcache_obj_t *list, int purged) MEXT_REF(m) = 0; MEXT_FLAGS(m) = 0; - rfa = (mcache_obj_t *)MEXT_RFA(m); + rfa = (mcache_obj_t *)(void *)MEXT_RFA(m); rfa->obj_next = ref_list; ref_list = rfa; MEXT_RFA(m) = NULL; @@ -1789,7 +2305,7 @@ mbuf_cslab_alloc(void *arg, mcache_obj_t ***plist, unsigned int needed, int wait) { mbuf_class_t class = (mbuf_class_t)arg; - mcache_t *cp = NULL; + mbuf_class_t cl_class = 0; unsigned int num = 0, cnum = 0, want = needed; mcache_obj_t *ref_list = NULL; mcache_obj_t *mp_list = NULL; @@ -1840,22 +2356,28 @@ mbuf_cslab_alloc(void *arg, mcache_obj_t ***plist, unsigned int needed, if (!(wait & MCR_NOSLEEP)) wait |= MCR_FAILOK; + /* allocate mbufs */ needed = mcache_alloc_ext(m_cache(MC_MBUF), &mp_list, needed, wait); if (needed == 0) { ASSERT(mp_list == NULL); goto fail; } - if (class == MC_MBUF_CL) - cp = m_cache(MC_CL); - else if (class == MC_MBUF_BIGCL) - cp = m_cache(MC_BIGCL); - else - cp = m_cache(MC_16KCL); - needed = mcache_alloc_ext(cp, &clp_list, needed, wait); + + /* allocate clusters */ + if (class == MC_MBUF_CL) { + cl_class = MC_CL; + } else if (class == MC_MBUF_BIGCL) { + cl_class = MC_BIGCL; + } else { + VERIFY(class == MC_MBUF_16KCL); + cl_class = MC_16KCL; + } + needed = mcache_alloc_ext(m_cache(cl_class), &clp_list, needed, wait); if (needed == 0) { ASSERT(clp_list == NULL); goto fail; } + needed = mcache_alloc_ext(ref_cache, &ref_list, needed, wait); if (needed == 0) { ASSERT(ref_list == NULL); @@ -1878,7 +2400,7 @@ mbuf_cslab_alloc(void *arg, mcache_obj_t ***plist, unsigned int needed, rfa = (struct ext_ref *)ref_list; ref_list = ref_list->obj_next; - ((mcache_obj_t *)rfa)->obj_next = NULL; + ((mcache_obj_t *)(void *)rfa)->obj_next = NULL; /* * If auditing is enabled, construct the shadow mbuf @@ -1888,7 +2410,6 @@ mbuf_cslab_alloc(void *arg, mcache_obj_t ***plist, unsigned int needed, */ if (mclaudit != NULL) { mcache_audit_t *mca, *cl_mca; - size_t size; lck_mtx_lock(mbuf_mlock); mca = mcl_audit_buf2mca(MC_MBUF, (mcache_obj_t *)m); @@ -1911,15 +2432,22 @@ mbuf_cslab_alloc(void *arg, mcache_obj_t ***plist, unsigned int needed, lck_mtx_unlock(mbuf_mlock); /* Technically, they are in the freelist */ - mcache_set_pattern(MCACHE_FREE_PATTERN, m, - m_maxsize(MC_MBUF)); - if (class == MC_MBUF_CL) - size = m_maxsize(MC_CL); - else if (class == MC_MBUF_BIGCL) - size = m_maxsize(MC_BIGCL); - else - size = m_maxsize(MC_16KCL); - mcache_set_pattern(MCACHE_FREE_PATTERN, cl, size); + if (mclverify) { + size_t size; + + mcache_set_pattern(MCACHE_FREE_PATTERN, m, + m_maxsize(MC_MBUF)); + + if (class == MC_MBUF_CL) + size = m_maxsize(MC_CL); + else if (class == MC_MBUF_BIGCL) + size = m_maxsize(MC_BIGCL); + else + size = m_maxsize(MC_16KCL); + + mcache_set_pattern(MCACHE_FREE_PATTERN, cl, + size); + } } MBUF_INIT(ms, 0, MT_FREE); @@ -1945,7 +2473,7 @@ fail: if (mp_list != NULL) mcache_free_ext(m_cache(MC_MBUF), mp_list); if (clp_list != NULL) - mcache_free_ext(cp, clp_list); + mcache_free_ext(m_cache(cl_class), clp_list); if (ref_list != NULL) mcache_free_ext(ref_cache, ref_list); @@ -2015,7 +2543,9 @@ mbuf_cslab_audit(void *arg, mcache_obj_t *list, boolean_t alloc) /* Do the mbuf sanity checks and record its transaction */ mca = mcl_audit_buf2mca(MC_MBUF, (mcache_obj_t *)m); mcl_audit_mbuf(mca, m, TRUE, alloc); - mcache_buffer_log(mca, m, m_cache(class)); + if (mcltrace) + mcache_buffer_log(mca, m, m_cache(class)); + if (alloc) mca->mca_uflags |= MB_COMP_INUSE; else @@ -2026,7 +2556,7 @@ mbuf_cslab_audit(void *arg, mcache_obj_t *list, boolean_t alloc) * freeing, since the contents of the actual mbuf has been * pattern-filled by the above call to mcl_audit_mbuf(). */ - if (!alloc) + if (!alloc && mclverify) ms = (struct mbuf *)mca->mca_contents; /* Do the cluster sanity checks and record its transaction */ @@ -2034,16 +2564,15 @@ mbuf_cslab_audit(void *arg, mcache_obj_t *list, boolean_t alloc) clsp = slab_get(cl); VERIFY(ms->m_flags == M_EXT && cl != NULL); VERIFY(MEXT_RFA(ms) != NULL && MBUF_IS_COMPOSITE(ms)); - VERIFY(clsp->sl_refcnt == 1); - if (class == MC_MBUF_BIGCL) { - nsp = clsp->sl_next; - /* Next slab must already be present */ - VERIFY(nsp != NULL); - VERIFY(nsp->sl_refcnt == 1); - } else if (class == MC_MBUF_16KCL) { + if (class == MC_MBUF_CL) + VERIFY(clsp->sl_refcnt >= 1 && + clsp->sl_refcnt <= NCLPBG); + else + VERIFY(clsp->sl_refcnt == 1); + + if (class == MC_MBUF_16KCL) { int k; - for (nsp = clsp, k = 1; - k < (M16KCLBYTES / MCLBYTES); k++) { + for (nsp = clsp, k = 1; k < NSLABSP16KB; k++) { nsp = nsp->sl_next; /* Next slab must already be present */ VERIFY(nsp != NULL); @@ -2059,7 +2588,9 @@ mbuf_cslab_audit(void *arg, mcache_obj_t *list, boolean_t alloc) else size = m_maxsize(MC_16KCL); mcl_audit_cluster(mca, cl, size, alloc, FALSE); - mcache_buffer_log(mca, cl, m_cache(class)); + if (mcltrace) + mcache_buffer_log(mca, cl, m_cache(class)); + if (alloc) mca->mca_uflags |= MB_COMP_INUSE; else @@ -2078,14 +2609,14 @@ m_clalloc(const u_int32_t num, const int wait, const u_int32_t bufsize) { int i; vm_size_t size = 0; - int numpages = 0; + int numpages = 0, large_buffer = (bufsize == m_maxsize(MC_16KCL)); vm_offset_t page = 0; mcache_audit_t *mca_list = NULL; mcache_obj_t *con_list = NULL; mcl_slab_t *sp; - VERIFY(bufsize == m_maxsize(MC_CL) || - bufsize == m_maxsize(MC_BIGCL) || bufsize == m_maxsize(MC_16KCL)); + VERIFY(bufsize == m_maxsize(MC_BIGCL) || + bufsize == m_maxsize(MC_16KCL)); lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); @@ -2117,14 +2648,22 @@ m_clalloc(const u_int32_t num, const int wait, const u_int32_t bufsize) lck_mtx_unlock(mbuf_mlock); - size = round_page_32(i * bufsize); - page = kmem_mb_alloc(mb_map, size); + size = round_page(i * bufsize); + page = kmem_mb_alloc(mb_map, size, large_buffer); + + /* + * If we did ask for "n" 16KB physically contiguous chunks + * and didn't get them, then please try again without this + * restriction. + */ + if (large_buffer && page == 0) + page = kmem_mb_alloc(mb_map, size, 0); if (page == 0) { - if (bufsize <= m_maxsize(MC_BIGCL)) { - /* Try for 1 page if failed, only for 2KB/4KB request */ + if (bufsize == m_maxsize(MC_BIGCL)) { + /* Try for 1 page if failed, only 4KB request */ size = NBPG; - page = kmem_mb_alloc(mb_map, size); + page = kmem_mb_alloc(mb_map, size, 0); } if (page == 0) { @@ -2143,24 +2682,20 @@ m_clalloc(const u_int32_t num, const int wait, const u_int32_t bufsize) /* * Yes, I realize this is a waste of memory for clusters * that never get transformed into mbufs, as we may end - * up with NMBPCL-1 unused audit structures per cluster. + * up with NMBPBG-1 unused audit structures per cluster. * But doing so tremendously simplifies the allocation * strategy, since at this point we are not holding the - * mbuf lock and the caller is okay to be blocked. For - * the case of big clusters, we allocate one structure - * for each as we never turn them into mbufs. + * mbuf lock and the caller is okay to be blocked. */ - if (bufsize == m_maxsize(MC_CL)) { - needed = numpages * 2 * NMBPCL; + if (bufsize == m_maxsize(MC_BIGCL)) { + needed = numpages * NMBPBG; i = mcache_alloc_ext(mcl_audit_con_cache, &con_list, needed, MCR_SLEEP); VERIFY(con_list != NULL && i == needed); - } else if (bufsize == m_maxsize(MC_BIGCL)) { - needed = numpages; } else { - needed = numpages / (M16KCLBYTES / NBPG); + needed = numpages / NSLABSP16KB; } i = mcache_alloc_ext(mcache_audit_cache, @@ -2173,80 +2708,38 @@ m_clalloc(const u_int32_t num, const int wait, const u_int32_t bufsize) for (i = 0; i < numpages; i++, page += NBPG) { ppnum_t offset = ((char *)page - (char *)mbutl) / NBPG; - ppnum_t new_page = pmap_find_phys(kernel_pmap, - (vm_address_t)page); + ppnum_t new_page = pmap_find_phys(kernel_pmap, page); /* * In the case of no mapper being available the following * code noops and returns the input page; if there is a * mapper the appropriate I/O page is returned. */ - new_page = IOMapperInsertPage(mcl_paddr_base, offset, new_page); + VERIFY(offset < mcl_pages); + if (mcl_paddr_base) { + bzero((void *)(uintptr_t) page, page_size); + new_page = IOMapperInsertPage(mcl_paddr_base, offset, new_page); + } mcl_paddr[offset] = new_page << PGSHIFT; /* Pattern-fill this fresh page */ - if (mclaudit != NULL) + if (mclverify) { mcache_set_pattern(MCACHE_FREE_PATTERN, (caddr_t)page, NBPG); - - if (bufsize == m_maxsize(MC_CL)) { - union mcluster *mcl = (union mcluster *)page; - - /* 1st cluster in the page */ - sp = slab_get(mcl); - if (mclaudit != NULL) - mcl_audit_init(mcl, &mca_list, &con_list, - AUDIT_CONTENTS_SIZE, NMBPCL); - - VERIFY(sp->sl_refcnt == 0 && sp->sl_flags == 0); - slab_init(sp, MC_CL, SLF_MAPPED, - mcl, mcl, bufsize, 0, 1); - - /* Insert this slab */ - slab_insert(sp, MC_CL); - - /* Update stats now since slab_get() drops the lock */ - mbstat.m_clfree = ++m_infree(MC_CL) + - m_infree(MC_MBUF_CL); - mbstat.m_clusters = ++m_total(MC_CL); - VERIFY(m_total(MC_CL) <= m_maxlimit(MC_CL)); - - /* 2nd cluster in the page */ - sp = slab_get(++mcl); - if (mclaudit != NULL) - mcl_audit_init(mcl, &mca_list, &con_list, - AUDIT_CONTENTS_SIZE, NMBPCL); - - VERIFY(sp->sl_refcnt == 0 && sp->sl_flags == 0); - slab_init(sp, MC_CL, SLF_MAPPED, - mcl, mcl, bufsize, 0, 1); - - /* Insert this slab */ - slab_insert(sp, MC_CL); - - /* Update stats now since slab_get() drops the lock */ - mbstat.m_clfree = ++m_infree(MC_CL) + - m_infree(MC_MBUF_CL); - mbstat.m_clusters = ++m_total(MC_CL); - VERIFY(m_total(MC_CL) <= m_maxlimit(MC_CL)); - } else if (bufsize == m_maxsize(MC_BIGCL)) { + } + if (bufsize == m_maxsize(MC_BIGCL)) { union mbigcluster *mbc = (union mbigcluster *)page; - mcl_slab_t *nsp; /* One for the entire page */ sp = slab_get(mbc); - if (mclaudit != NULL) - mcl_audit_init(mbc, &mca_list, NULL, 0, 1); - + if (mclaudit != NULL) { + mcl_audit_init(mbc, &mca_list, &con_list, + AUDIT_CONTENTS_SIZE, NMBPBG); + } VERIFY(sp->sl_refcnt == 0 && sp->sl_flags == 0); slab_init(sp, MC_BIGCL, SLF_MAPPED, mbc, mbc, bufsize, 0, 1); - /* 2nd cluster's slab is part of the previous one */ - nsp = slab_get(((union mcluster *)page) + 1); - slab_init(nsp, MC_BIGCL, SLF_MAPPED | SLF_PARTIAL, - mbc, NULL, 0, 0, 0); - /* Insert this slab */ slab_insert(sp, MC_BIGCL); @@ -2255,7 +2748,7 @@ m_clalloc(const u_int32_t num, const int wait, const u_int32_t bufsize) m_infree(MC_MBUF_BIGCL); mbstat.m_bigclusters = ++m_total(MC_BIGCL); VERIFY(m_total(MC_BIGCL) <= m_maxlimit(MC_BIGCL)); - } else if ((i % (M16KCLBYTES / NBPG)) == 0) { + } else if ((i % NSLABSP16KB) == 0) { union m16kcluster *m16kcl = (union m16kcluster *)page; mcl_slab_t *nsp; int k; @@ -2270,9 +2763,12 @@ m_clalloc(const u_int32_t num, const int wait, const u_int32_t bufsize) slab_init(sp, MC_16KCL, SLF_MAPPED, m16kcl, m16kcl, bufsize, 0, 1); - /* 2nd-8th cluster's slab is part of the first one */ - for (k = 1; k < (M16KCLBYTES / MCLBYTES); k++) { - nsp = slab_get(((union mcluster *)page) + k); + /* + * 2nd-Nth page's slab is part of the first one, + * where N is NSLABSP16KB. + */ + for (k = 1; k < NSLABSP16KB; k++) { + nsp = slab_get(((union mbigcluster *)page) + k); VERIFY(nsp->sl_refcnt == 0 && nsp->sl_flags == 0); slab_init(nsp, MC_16KCL, @@ -2298,13 +2794,11 @@ m_clalloc(const u_int32_t num, const int wait, const u_int32_t bufsize) wakeup(mb_clalloc_waitchan); } - if (bufsize == m_maxsize(MC_CL)) - return (numpages << 1); - else if (bufsize == m_maxsize(MC_BIGCL)) + if (bufsize == m_maxsize(MC_BIGCL)) return (numpages); VERIFY(bufsize == m_maxsize(MC_16KCL)); - return (numpages / (M16KCLBYTES / NBPG)); + return (numpages / NSLABSP16KB); out: lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); @@ -2320,23 +2814,7 @@ out: * When non-blocking we kick a thread if we have to grow the * pool or if the number of free clusters is less than requested. */ - if (bufsize == m_maxsize(MC_CL)) { - if (i > 0) { - /* - * Remember total number of clusters needed - * at this time. - */ - i += m_total(MC_CL); - if (i > mbuf_expand_mcl) { - mbuf_expand_mcl = i; - if (mbuf_worker_ready) - wakeup((caddr_t)&mbuf_worker_run); - } - } - - if (m_infree(MC_CL) >= num) - return (1); - } else if (bufsize == m_maxsize(MC_BIGCL)) { + if (bufsize == m_maxsize(MC_BIGCL)) { if (i > 0) { /* * Remember total number of 4KB clusters needed @@ -2379,44 +2857,30 @@ static int freelist_populate(mbuf_class_t class, unsigned int num, int wait) { mcache_obj_t *o = NULL; - int i; + int i, numpages = 0, count; VERIFY(class == MC_MBUF || class == MC_CL || class == MC_BIGCL || class == MC_16KCL); -#if CONFIG_MBUF_NOEXPAND - if ((mbstat.m_mbufs / NMBPCL) >= maxmbufcl) { -#if DEBUG - static int printonce = 1; - if (printonce == 1) { - printonce = 0; - printf("m_expand failed, allocated %ld out of %d " - "clusters\n", mbstat.m_mbufs / NMBPCL, - nmbclusters); - } -#endif /* DEBUG */ - return (0); - } -#endif /* CONFIG_MBUF_NOEXPAND */ - lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); switch (class) { case MC_MBUF: case MC_CL: - i = m_clalloc(num, wait, m_maxsize(MC_CL)); + case MC_BIGCL: + numpages = (num * m_size(class) + NBPG - 1) / NBPG; + i = m_clalloc(numpages, wait, m_maxsize(MC_BIGCL)); - /* Respect the 2K clusters minimum limit */ - if (m_total(MC_CL) == m_maxlimit(MC_CL) && - m_infree(MC_CL) <= m_minlimit(MC_CL)) { - if (class != MC_CL || (wait & MCR_COMP)) + /* Respect the 4KB clusters minimum limit */ + if (m_total(MC_BIGCL) == m_maxlimit(MC_BIGCL) && + m_infree(MC_BIGCL) <= m_minlimit(MC_BIGCL)) { + if (class != MC_BIGCL || (wait & MCR_COMP)) return (0); } - if (class == MC_CL) + if (class == MC_BIGCL) return (i != 0); break; - case MC_BIGCL: case MC_16KCL: return (m_clalloc(num, wait, m_maxsize(class)) != 0); /* NOTREACHED */ @@ -2426,66 +2890,119 @@ freelist_populate(mbuf_class_t class, unsigned int num, int wait) /* NOTREACHED */ } - /* Steal a cluster and cut it up to create NMBPCL mbufs */ - if ((o = slab_alloc(MC_CL, wait)) != NULL) { + VERIFY(class == MC_MBUF || class == MC_CL); + + /* how many objects will we cut the page into? */ + int numobj = (class == MC_MBUF ? NMBPBG : NCLPBG); + + for (count = 0; count < numpages; count++) { + + /* respect totals, minlimit, maxlimit */ + if (m_total(MC_BIGCL) <= m_minlimit(MC_BIGCL) || + m_total(class) >= m_maxlimit(class)) + break; + + if ((o = slab_alloc(MC_BIGCL, wait)) == NULL) + break; + struct mbuf *m = (struct mbuf *)o; - mcache_audit_t *mca = NULL; + union mcluster *c = (union mcluster *)o; mcl_slab_t *sp = slab_get(o); + mcache_audit_t *mca = NULL; VERIFY(slab_is_detached(sp) && (sp->sl_flags & (SLF_MAPPED | SLF_PARTIAL)) == SLF_MAPPED); - /* Make sure that the cluster is unmolested while in freelist */ - if (mclaudit != NULL) { - mca = mcl_audit_buf2mca(MC_CL, o); - mcache_audit_free_verify(mca, o, 0, m_maxsize(MC_CL)); + /* + * Make sure that the cluster is unmolested + * while in freelist + */ + if (mclverify) { + mca = mcl_audit_buf2mca(MC_BIGCL, o); + mcache_audit_free_verify(mca, o, 0, + m_maxsize(MC_BIGCL)); } - /* Reinitialize it as an mbuf slab */ - slab_init(sp, MC_MBUF, sp->sl_flags, sp->sl_base, NULL, - sp->sl_len, 0, NMBPCL); + /* Reinitialize it as an mbuf or 2K slab */ + slab_init(sp, class, sp->sl_flags, + sp->sl_base, NULL, sp->sl_len, 0, numobj); - VERIFY(m == (struct mbuf *)sp->sl_base); + VERIFY(o == (mcache_obj_t *)sp->sl_base); VERIFY(sp->sl_head == NULL); - m_total(MC_MBUF) += NMBPCL; - mbstat.m_mbufs = m_total(MC_MBUF); - m_infree(MC_MBUF) += NMBPCL; - mtype_stat_add(MT_FREE, NMBPCL); + VERIFY(m_total(MC_BIGCL) > 0); + m_total(MC_BIGCL)--; + mbstat.m_bigclusters = m_total(MC_BIGCL); - i = NMBPCL; - while (i--) { - /* - * If auditing is enabled, construct the shadow mbuf - * in the audit structure instead of the actual one. - * mbuf_slab_audit() will take care of restoring the - * contents after the integrity check. - */ - if (mclaudit != NULL) { - struct mbuf *ms; - mca = mcl_audit_buf2mca(MC_MBUF, - (mcache_obj_t *)m); - ms = ((struct mbuf *)mca->mca_contents); - ms->m_type = MT_FREE; - } else { - m->m_type = MT_FREE; + m_total(class) += numobj; + m_infree(class) += numobj; + + VERIFY(m_total(MC_BIGCL) >= m_minlimit(MC_BIGCL)); + VERIFY(m_total(class) <= m_maxlimit(class)); + + i = numobj; + if (class == MC_MBUF) { + mbstat.m_mbufs = m_total(MC_MBUF); + mtype_stat_add(MT_FREE, NMBPBG); + while (i--) { + /* + * If auditing is enabled, construct the + * shadow mbuf in the audit structure + * instead of the actual one. + * mbuf_slab_audit() will take care of + * restoring the contents after the + * integrity check. + */ + if (mclaudit != NULL) { + struct mbuf *ms; + mca = mcl_audit_buf2mca(MC_MBUF, + (mcache_obj_t *)m); + ms = ((struct mbuf *) + mca->mca_contents); + ms->m_type = MT_FREE; + } else { + m->m_type = MT_FREE; + } + m->m_next = sp->sl_head; + sp->sl_head = (void *)m++; + } + } else { /* MC_CL */ + mbstat.m_clfree = + m_infree(MC_CL) + m_infree(MC_MBUF_CL); + mbstat.m_clusters = m_total(MC_CL); + while (i--) { + c->mcl_next = sp->sl_head; + sp->sl_head = (void *)c++; } - m->m_next = sp->sl_head; - sp->sl_head = (void *)m++; } - /* Insert it into the mbuf class's slab list */ - slab_insert(sp, MC_MBUF); + /* Insert into the mbuf or 2k slab list */ + slab_insert(sp, class); if ((i = mb_waiters) > 0) mb_waiters = 0; if (i != 0) wakeup(mb_waitchan); - - return (1); } + return (count != 0); +} - return (0); +/* + * For each class, initialize the freelist to hold m_minlimit() objects. + */ +static void +freelist_init(mbuf_class_t class) +{ + lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + + VERIFY(class == MC_CL || class == MC_BIGCL); + VERIFY(m_total(class) == 0); + VERIFY(m_minlimit(class) > 0); + + while (m_total(class) < m_minlimit(class)) + (void) freelist_populate(class, m_minlimit(class), M_WAIT); + + VERIFY(m_total(class) >= m_minlimit(class)); } /* @@ -2590,17 +3107,23 @@ m_reclaim(mbuf_class_t class, unsigned int num, boolean_t comp) switch (class) { case MC_MBUF: m_wantpurge(MC_CL)++; + m_wantpurge(MC_BIGCL)++; m_wantpurge(MC_MBUF_CL)++; m_wantpurge(MC_MBUF_BIGCL)++; break; case MC_CL: m_wantpurge(MC_MBUF)++; + m_wantpurge(MC_BIGCL)++; + m_wantpurge(MC_MBUF_BIGCL)++; if (!comp) m_wantpurge(MC_MBUF_CL)++; break; case MC_BIGCL: + m_wantpurge(MC_MBUF)++; + m_wantpurge(MC_CL)++; + m_wantpurge(MC_MBUF_CL)++; if (!comp) m_wantpurge(MC_MBUF_BIGCL)++; break; @@ -2748,11 +3271,11 @@ m_free(struct mbuf *m) if (m->m_flags & M_EXT) { u_int32_t refcnt; - u_int32_t flags; + u_int32_t composite; refcnt = m_decref(m); - flags = MEXT_FLAGS(m); - if (refcnt == 0 && flags == 0) { + composite = (MEXT_FLAGS(m) & EXTF_COMPOSITE); + if (refcnt == 0 && !composite) { if (m->m_ext.ext_free == NULL) { mcache_free(m_cache(MC_CL), m->m_ext.ext_buf); } else if (m->m_ext.ext_free == m_bigfree) { @@ -2767,7 +3290,7 @@ m_free(struct mbuf *m) } mcache_free(ref_cache, MEXT_RFA(m)); MEXT_RFA(m) = NULL; - } else if (refcnt == 0 && (flags & EXTF_COMPOSITE)) { + } else if (refcnt == 0 && composite) { VERIFY(m->m_type != MT_FREE); mtype_stat_dec(m->m_type); @@ -2778,6 +3301,8 @@ m_free(struct mbuf *m) m->m_len = 0; m->m_next = m->m_nextpkt = NULL; + MEXT_FLAGS(m) &= ~EXTF_READONLY; + /* "Free" into the intermediate cache */ if (m->m_ext.ext_free == NULL) { mcache_free(m_cache(MC_MBUF_CL), m); @@ -2817,11 +3342,11 @@ m_clattach(struct mbuf *m, int type, caddr_t extbuf, if (m->m_flags & M_EXT) { u_int32_t refcnt; - u_int32_t flags; + u_int32_t composite; refcnt = m_decref(m); - flags = MEXT_FLAGS(m); - if (refcnt == 0 && flags == 0) { + composite = (MEXT_FLAGS(m) & EXTF_COMPOSITE); + if (refcnt == 0 && !composite) { if (m->m_ext.ext_free == NULL) { mcache_free(m_cache(MC_CL), m->m_ext.ext_buf); } else if (m->m_ext.ext_free == m_bigfree) { @@ -2836,7 +3361,7 @@ m_clattach(struct mbuf *m, int type, caddr_t extbuf, } /* Re-use the reference structure */ rfa = MEXT_RFA(m); - } else if (refcnt == 0 && (flags & EXTF_COMPOSITE)) { + } else if (refcnt == 0 && composite) { VERIFY(m->m_type != MT_FREE); mtype_stat_dec(m->m_type); @@ -2846,6 +3371,9 @@ m_clattach(struct mbuf *m, int type, caddr_t extbuf, m->m_flags = M_EXT; m->m_len = 0; m->m_next = m->m_nextpkt = NULL; + + MEXT_FLAGS(m) &= ~EXTF_READONLY; + /* "Free" into the intermediate cache */ if (m->m_ext.ext_free == NULL) { mcache_free(m_cache(MC_MBUF_CL), m); @@ -2875,6 +3403,51 @@ m_clattach(struct mbuf *m, int type, caddr_t extbuf, return (m); } +/* + * Perform `fast' allocation mbuf clusters from a cache of recently-freed + * clusters. (If the cache is empty, new clusters are allocated en-masse.) + */ +struct mbuf * +m_getcl(int wait, int type, int flags) +{ + struct mbuf *m; + int mcflags = MSLEEPF(wait); + int hdr = (flags & M_PKTHDR); + + /* Is this due to a non-blocking retry? If so, then try harder */ + if (mcflags & MCR_NOSLEEP) + mcflags |= MCR_TRYHARD; + + m = mcache_alloc(m_cache(MC_MBUF_CL), mcflags); + if (m != NULL) { + u_int32_t flag; + struct ext_ref *rfa; + void *cl; + + VERIFY(m->m_type == MT_FREE && m->m_flags == M_EXT); + cl = m->m_ext.ext_buf; + rfa = MEXT_RFA(m); + + ASSERT(cl != NULL && rfa != NULL); + VERIFY(MBUF_IS_COMPOSITE(m) && m->m_ext.ext_free == NULL); + + flag = MEXT_FLAGS(m); + + MBUF_INIT(m, hdr, type); + MBUF_CL_INIT(m, cl, rfa, 1, flag); + + mtype_stat_inc(type); + mtype_stat_dec(MT_FREE); +#if CONFIG_MACF_NET + if (hdr && mac_init_mbuf(m, wait) != 0) { + m_freem(m); + return (NULL); + } +#endif /* MAC_NET */ + } + return (m); +} + /* m_mclget() add an mbuf cluster to a normal mbuf */ struct mbuf * m_mclget(struct mbuf *m, int wait) @@ -2915,7 +3488,7 @@ m_mclfree(caddr_t p) /* * mcl_hasreference() checks if a cluster of an mbuf is referenced by - * another mbuf + * another mbuf; see comments in m_incref() regarding EXTF_READONLY. */ int m_mclhasreference(struct mbuf *m) @@ -2925,7 +3498,7 @@ m_mclhasreference(struct mbuf *m) ASSERT(MEXT_RFA(m) != NULL); - return (MEXT_REF(m) > 1); + return ((MEXT_FLAGS(m) & EXTF_READONLY) ? 1 : 0); } __private_extern__ caddr_t @@ -3000,19 +3573,23 @@ m_m16kget(struct mbuf *m, int wait) return (m); } -/* */ +/* + * "Move" mbuf pkthdr from "from" to "to". + * "from" must have M_PKTHDR set, and "to" must be empty. + */ void m_copy_pkthdr(struct mbuf *to, struct mbuf *from) { -#if CONFIG_MACF_NET /* We will be taking over the tags of 'to' */ if (to->m_flags & M_PKTHDR) m_tag_delete_chain(to, NULL); -#endif /* MAC_NET */ to->m_pkthdr = from->m_pkthdr; /* especially tags */ m_tag_init(from); /* purge tags from src */ - to->m_flags = from->m_flags & M_COPYFLAGS; - to->m_data = (to)->m_pktdat; + m_service_class_init(from); /* reset svc class from src */ + from->m_pkthdr.aux_flags = 0; /* clear aux flags from src */ + to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT); + if ((to->m_flags & M_EXT) == 0) + to->m_data = to->m_pktdat; } /* @@ -3023,10 +3600,8 @@ m_copy_pkthdr(struct mbuf *to, struct mbuf *from) static int m_dup_pkthdr(struct mbuf *to, struct mbuf *from, int how) { -#if CONFIG_MACF_NET if (to->m_flags & M_PKTHDR) m_tag_delete_chain(to, NULL); -#endif /* MAC_NET */ to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT); if ((to->m_flags & M_EXT) == 0) to->m_data = to->m_pktdat; @@ -3035,6 +3610,14 @@ m_dup_pkthdr(struct mbuf *to, struct mbuf *from, int how) return (m_tag_copy_chain(to, from, how)); } +void +m_copy_pftag(struct mbuf *to, struct mbuf *from) +{ + to->m_pkthdr.pf_mtag = from->m_pkthdr.pf_mtag; + to->m_pkthdr.pf_mtag.pftag_hdr = NULL; + to->m_pkthdr.pf_mtag.pftag_flags &= ~(PF_TAG_HDR_INET|PF_TAG_HDR_INET6); +} + /* * Return a list of mbuf hdrs that point to clusters. Try for num_needed; * if wantall is not set, return whatever number were available. Set up the @@ -3115,7 +3698,7 @@ m_getpackets_internal(unsigned int *num_needed, int num_with_pkthdrs, --num_with_pkthdrs; #if CONFIG_MACF_NET if (mac_mbuf_label_init(m, wait) != 0) { - m_free(m); + m_freem(m); break; } #endif /* MAC_NET */ @@ -3142,6 +3725,12 @@ m_getpackets_internal(unsigned int *num_needed, int num_with_pkthdrs, return (NULL); } + if (pnum > *num_needed) { + printf("%s: File a radar related to . \ + needed = %u, pnum = %u, num_needed = %u \n", + __func__, needed, pnum, *num_needed); + } + *num_needed = pnum; return (top); } @@ -3431,7 +4020,7 @@ m_allocpacket_internal(unsigned int *numlist, size_t packetlen, #if CONFIG_MACF_NET if (pkthdr && mac_init_mbuf(m, wait) != 0) { --num; - m_free(m); + m_freem(m); break; } #endif /* MAC_NET */ @@ -3568,7 +4157,7 @@ m_freem_list(struct mbuf *m) while (m != NULL) { struct mbuf *next = m->m_next; mcache_obj_t *o, *rfa; - u_int32_t refcnt, flags; + u_int32_t refcnt, composite; if (m->m_type == MT_FREE) panic("m_free: freeing an already freed mbuf"); @@ -3583,10 +4172,10 @@ m_freem_list(struct mbuf *m) if (!(m->m_flags & M_EXT)) goto simple_free; - o = (mcache_obj_t *)m->m_ext.ext_buf; + o = (mcache_obj_t *)(void *)m->m_ext.ext_buf; refcnt = m_decref(m); - flags = MEXT_FLAGS(m); - if (refcnt == 0 && flags == 0) { + composite = (MEXT_FLAGS(m) & EXTF_COMPOSITE); + if (refcnt == 0 && !composite) { if (m->m_ext.ext_free == NULL) { o->obj_next = mcl_list; mcl_list = o; @@ -3601,11 +4190,11 @@ m_freem_list(struct mbuf *m) m->m_ext.ext_size, m->m_ext.ext_arg); } - rfa = (mcache_obj_t *)MEXT_RFA(m); + rfa = (mcache_obj_t *)(void *)MEXT_RFA(m); rfa->obj_next = ref_list; ref_list = rfa; MEXT_RFA(m) = NULL; - } else if (refcnt == 0 && (flags & EXTF_COMPOSITE)) { + } else if (refcnt == 0 && composite) { VERIFY(m->m_type != MT_FREE); /* * Amortize the costs of atomic operations @@ -3627,6 +4216,8 @@ m_freem_list(struct mbuf *m) m->m_len = 0; m->m_next = m->m_nextpkt = NULL; + MEXT_FLAGS(m) &= ~EXTF_READONLY; + /* "Free" into the intermediate cache */ o = (mcache_obj_t *)m; if (m->m_ext.ext_free == NULL) { @@ -3845,7 +4436,7 @@ m_copym(struct mbuf *m, int off0, int len, int wait) } if (len == M_COPYALL) { if (MIN(len, (m->m_len - off)) == len) { - printf("m->m_len %ld - off %d = %ld, %ld\n", + printf("m->m_len %d - off %d = %d, %d\n", m->m_len, off, m->m_len - off, MIN(len, (m->m_len - off))); } @@ -3890,7 +4481,7 @@ nospace: */ struct mbuf * m_copym_with_hdrs(struct mbuf *m, int off0, int len0, int wait, - struct mbuf **m_last, int *m_off) + struct mbuf **m_lastm, int *m_off) { struct mbuf *n, **np = NULL; int off = off0, len = len0; @@ -3904,8 +4495,8 @@ m_copym_with_hdrs(struct mbuf *m, int off0, int len0, int wait, if (off == 0 && (m->m_flags & M_PKTHDR)) copyhdr = 1; - if (*m_last != NULL) { - m = *m_last; + if (*m_lastm != NULL) { + m = *m_lastm; off = *m_off; } else { while (off >= m->m_len) { @@ -3982,10 +4573,10 @@ m_copym_with_hdrs(struct mbuf *m, int off0, int len0, int wait, if (len == 0) { if ((off + n->m_len) == m->m_len) { - *m_last = m->m_next; + *m_lastm = m->m_next; *m_off = 0; } else { - *m_last = m; + *m_lastm = m; *m_off = off + n->m_len; } break; @@ -4016,9 +4607,10 @@ nospace: * continuing for "len" bytes, into the indicated buffer. */ void -m_copydata(struct mbuf *m, int off, int len, caddr_t cp) +m_copydata(struct mbuf *m, int off, int len, void *vp) { unsigned count; + char *cp = vp; if (off < 0 || len < 0) panic("m_copydata: invalid offset %d or len %d", off, len); @@ -4207,6 +4799,56 @@ bad: return (0); } +/* + * Like m_pullup(), except a new mbuf is always allocated, and we allow + * the amount of empty space before the data in the new mbuf to be specified + * (in the event that the caller expects to prepend later). + */ +__private_extern__ int MSFail = 0; + +__private_extern__ struct mbuf * +m_copyup(struct mbuf *n, int len, int dstoff) +{ + struct mbuf *m; + int count, space; + + if (len > (MHLEN - dstoff)) + goto bad; + MGET(m, M_DONTWAIT, n->m_type); + if (m == NULL) + goto bad; + m->m_len = 0; + if (n->m_flags & M_PKTHDR) { + m_copy_pkthdr(m, n); + n->m_flags &= ~M_PKTHDR; + } + m->m_data += dstoff; + space = &m->m_dat[MLEN] - (m->m_data + m->m_len); + do { + count = min(min(max(len, max_protohdr), space), n->m_len); + memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t), + (unsigned)count); + len -= count; + m->m_len += count; + n->m_len -= count; + space -= count; + if (n->m_len) + n->m_data += count; + else + n = m_free(n); + } while (len > 0 && n); + if (len > 0) { + (void) m_free(m); + goto bad; + } + m->m_next = n; + return (m); +bad: + m_freem(n); + MSFail++; + return (NULL); +} + /* * Partition an mbuf chain in two pieces, returning the tail -- * all but the first len0 bytes. In case of failure, it returns NULL and @@ -4214,6 +4856,12 @@ bad: */ struct mbuf * m_split(struct mbuf *m0, int len0, int wait) +{ + return (m_split0(m0, len0, wait, 1)); +} + +static struct mbuf * +m_split0(struct mbuf *m0, int len0, int wait, int copyhdr) { struct mbuf *m, *n; unsigned len = len0, remain; @@ -4223,7 +4871,7 @@ m_split(struct mbuf *m0, int len0, int wait) if (m == NULL) return (NULL); remain = m->m_len - len; - if (m0->m_flags & M_PKTHDR) { + if (copyhdr && (m0->m_flags & M_PKTHDR)) { _MGETHDR(n, wait, m0->m_type); if (n == NULL) return (NULL); @@ -4347,6 +4995,10 @@ m_devget(char *buf, int totlen, int off0, struct ifnet *ifp, return (top); } +#ifndef MBUF_GROWTH_NORMAL_THRESH +#define MBUF_GROWTH_NORMAL_THRESH 25 +#endif + /* * Cluster freelist allocation check. */ @@ -4354,88 +5006,134 @@ static int m_howmany(int num, size_t bufsize) { int i = 0, j = 0; - u_int32_t m_clusters, m_bigclusters, m_16kclusters; - u_int32_t m_clfree, m_bigclfree, m_16kclfree; + u_int32_t m_mbclusters, m_clusters, m_bigclusters, m_16kclusters; + u_int32_t m_mbfree, m_clfree, m_bigclfree, m_16kclfree; + u_int32_t sumclusters, freeclusters; + u_int32_t percent_pool, percent_kmem; + u_int32_t mb_growth, mb_growth_thresh; + + VERIFY(bufsize == m_maxsize(MC_BIGCL) || + bufsize == m_maxsize(MC_16KCL)); lck_mtx_assert(mbuf_mlock, LCK_MTX_ASSERT_OWNED); + /* Numbers in 2K cluster units */ + m_mbclusters = m_total(MC_MBUF) >> NMBPCLSHIFT; m_clusters = m_total(MC_CL); - m_bigclusters = m_total(MC_BIGCL); + m_bigclusters = m_total(MC_BIGCL) << NCLPBGSHIFT; m_16kclusters = m_total(MC_16KCL); + sumclusters = m_mbclusters + m_clusters + m_bigclusters; + + m_mbfree = m_infree(MC_MBUF) >> NMBPCLSHIFT; m_clfree = m_infree(MC_CL); - m_bigclfree = m_infree(MC_BIGCL); + m_bigclfree = m_infree(MC_BIGCL) << NCLPBGSHIFT; m_16kclfree = m_infree(MC_16KCL); + freeclusters = m_mbfree + m_clfree + m_bigclfree; /* Bail if we've maxed out the mbuf memory map */ - if ((bufsize != m_maxsize(MC_16KCL) && - (m_clusters + (m_bigclusters << 1) >= nclusters)) || + if ((bufsize == m_maxsize(MC_BIGCL) && sumclusters >= nclusters) || (njcl > 0 && bufsize == m_maxsize(MC_16KCL) && - (m_16kclusters << 3) >= njcl)) { -#if DEBUG - if (bufsize == MCLBYTES && num > m_clfree) { - printf("m_howmany - out of small clusters, " - "%d short\n", num - mbstat.m_clfree); - } -#endif /* DEBUG */ + (m_16kclusters << NCLPJCLSHIFT) >= njcl)) { return (0); } - if (bufsize == m_maxsize(MC_CL)) { + if (bufsize == m_maxsize(MC_BIGCL)) { /* Under minimum */ - if (m_clusters < MINCL) - return (MINCL - m_clusters); - /* Too few (free < 1/16 total) and not over maximum */ - if (m_clusters < m_maxlimit(MC_CL)) { - if (m_clfree >= MCL_LOWAT) + if (m_bigclusters < m_minlimit(MC_BIGCL)) + return (m_minlimit(MC_BIGCL) - m_bigclusters); + + percent_pool = + ((sumclusters - freeclusters) * 100) / sumclusters; + percent_kmem = (sumclusters * 100) / nclusters; + + /* + * If a light/normal user, grow conservatively (75%) + * If a heavy user, grow aggressively (50%) + */ + if (percent_kmem < MBUF_GROWTH_NORMAL_THRESH) + mb_growth = MB_GROWTH_NORMAL; + else + mb_growth = MB_GROWTH_AGGRESSIVE; + + if (percent_kmem < 5) { + /* For initial allocations */ + i = num; + } else { + /* Return if >= MBIGCL_LOWAT clusters available */ + if (m_infree(MC_BIGCL) >= MBIGCL_LOWAT && + m_total(MC_BIGCL) >= + MBIGCL_LOWAT + m_minlimit(MC_BIGCL)) return (0); - if (num >= m_clfree) - i = num - m_clfree; - if (((m_clusters + num) >> 4) > m_clfree) - j = ((m_clusters + num) >> 4) - m_clfree; + + /* Ensure at least num clusters are accessible */ + if (num >= m_infree(MC_BIGCL)) + i = num - m_infree(MC_BIGCL); + if (num > m_total(MC_BIGCL) - m_minlimit(MC_BIGCL)) + j = num - (m_total(MC_BIGCL) - + m_minlimit(MC_BIGCL)); + i = MAX(i, j); - if (i + m_clusters >= m_maxlimit(MC_CL)) - i = m_maxlimit(MC_CL) - m_clusters; - } - VERIFY((m_total(MC_CL) + i) <= m_maxlimit(MC_CL)); - } else if (bufsize == m_maxsize(MC_BIGCL)) { - /* Under minimum */ - if (m_bigclusters < MINBIGCL) - return (MINBIGCL - m_bigclusters); - /* Too few (free < 1/16 total) and not over maximum */ - if (m_bigclusters < m_maxlimit(MC_BIGCL)) { - if (m_bigclfree >= MBIGCL_LOWAT) - return (0); - if (num >= m_bigclfree) - i = num - m_bigclfree; - if (((m_bigclusters + num) >> 4) > m_bigclfree) - j = ((m_bigclusters + num) >> 4) - m_bigclfree; + + /* + * Grow pool if percent_pool > 75 (normal growth) + * or percent_pool > 50 (aggressive growth). + */ + mb_growth_thresh = 100 - (100 / (1 << mb_growth)); + if (percent_pool > mb_growth_thresh) + j = ((sumclusters + num) >> mb_growth) - + freeclusters; i = MAX(i, j); - if (i + m_bigclusters >= m_maxlimit(MC_BIGCL)) - i = m_maxlimit(MC_BIGCL) - m_bigclusters; } + + /* Check to ensure we didn't go over limits */ + if (i + m_bigclusters >= m_maxlimit(MC_BIGCL)) + i = m_maxlimit(MC_BIGCL) - m_bigclusters; + if ((i << 1) + sumclusters >= nclusters) + i = (nclusters - sumclusters) >> 1; VERIFY((m_total(MC_BIGCL) + i) <= m_maxlimit(MC_BIGCL)); - } else { + VERIFY(sumclusters + (i << 1) <= nclusters); + + } else { /* 16K CL */ VERIFY(njcl > 0); /* Under minimum */ if (m_16kclusters < MIN16KCL) return (MIN16KCL - m_16kclusters); - /* Too few (free < 1/16 total) and not over maximum */ - if (m_16kclusters < m_maxlimit(MC_16KCL)) { - if (m_16kclfree >= M16KCL_LOWAT) - return (0); - if (num >= m_16kclfree) - i = num - m_16kclfree; - if (((m_16kclusters + num) >> 4) > m_16kclfree) - j = ((m_16kclusters + num) >> 4) - m_16kclfree; - i = MAX(i, j); - if (i + m_16kclusters >= m_maxlimit(MC_16KCL)) - i = m_maxlimit(MC_16KCL) - m_16kclusters; - } + if (m_16kclfree >= M16KCL_LOWAT) + return (0); + + /* Ensure at least num clusters are available */ + if (num >= m_16kclfree) + i = num - m_16kclfree; + + /* Always grow 16KCL pool aggressively */ + if (((m_16kclusters + num) >> 1) > m_16kclfree) + j = ((m_16kclusters + num) >> 1) - m_16kclfree; + i = MAX(i, j); + + /* Check to ensure we don't go over limit */ + if (i + m_16kclusters >= m_maxlimit(MC_16KCL)) + i = m_maxlimit(MC_16KCL) - m_16kclusters; VERIFY((m_total(MC_16KCL) + i) <= m_maxlimit(MC_16KCL)); } - return (i); } +/* + * Return the number of bytes in the mbuf chain, m. + */ +unsigned int +m_length(struct mbuf *m) +{ + struct mbuf *m0; + unsigned int pktlen; + + if (m->m_flags & M_PKTHDR) + return (m->m_pkthdr.len); + + pktlen = 0; + for (m0 = m; m0 != NULL; m0 = m0->m_next) + pktlen += m0->m_len; + return (pktlen); +} /* * Copy data from a buffer back into the indicated mbuf chain, @@ -4443,30 +5141,258 @@ m_howmany(int num, size_t bufsize) * chain if necessary. */ void -m_copyback(struct mbuf *m0, int off, int len, caddr_t cp) +m_copyback(struct mbuf *m0, int off, int len, const void *cp) { - int mlen; - struct mbuf *m = m0, *n; - int totlen = 0; +#if DEBUG + struct mbuf *origm = m0; + int error; +#endif /* DEBUG */ if (m0 == NULL) return; + +#if DEBUG + error = +#endif /* DEBUG */ + m_copyback0(&m0, off, len, cp, + M_COPYBACK0_COPYBACK | M_COPYBACK0_EXTEND, M_DONTWAIT); + +#if DEBUG + if (error != 0 || (m0 != NULL && origm != m0)) + panic("m_copyback"); +#endif /* DEBUG */ +} + +struct mbuf * +m_copyback_cow(struct mbuf *m0, int off, int len, const void *cp, int how) +{ + int error; + + /* don't support chain expansion */ + VERIFY(off + len <= m_length(m0)); + + error = m_copyback0(&m0, off, len, cp, + M_COPYBACK0_COPYBACK | M_COPYBACK0_COW, how); + if (error) { + /* + * no way to recover from partial success. + * just free the chain. + */ + m_freem(m0); + return (NULL); + } + return (m0); +} + +/* + * m_makewritable: ensure the specified range writable. + */ +int +m_makewritable(struct mbuf **mp, int off, int len, int how) +{ + int error; +#if DEBUG + struct mbuf *n; + int origlen, reslen; + + origlen = m_length(*mp); +#endif /* DEBUG */ + +#if 0 /* M_COPYALL is large enough */ + if (len == M_COPYALL) + len = m_length(*mp) - off; /* XXX */ +#endif + + error = m_copyback0(mp, off, len, NULL, + M_COPYBACK0_PRESERVE | M_COPYBACK0_COW, how); + +#if DEBUG + reslen = 0; + for (n = *mp; n; n = n->m_next) + reslen += n->m_len; + if (origlen != reslen) + panic("m_makewritable: length changed"); + if (((*mp)->m_flags & M_PKTHDR) && reslen != (*mp)->m_pkthdr.len) + panic("m_makewritable: inconsist"); +#endif /* DEBUG */ + + return (error); +} + +static int +m_copyback0(struct mbuf **mp0, int off, int len, const void *vp, int flags, + int how) +{ + int mlen; + struct mbuf *m, *n; + struct mbuf **mp; + int totlen = 0; + const char *cp = vp; + + VERIFY(mp0 != NULL); + VERIFY(*mp0 != NULL); + VERIFY((flags & M_COPYBACK0_PRESERVE) == 0 || cp == NULL); + VERIFY((flags & M_COPYBACK0_COPYBACK) == 0 || cp != NULL); + + /* + * we don't bother to update "totlen" in the case of M_COPYBACK0_COW, + * assuming that M_COPYBACK0_EXTEND and M_COPYBACK0_COW are exclusive. + */ + + VERIFY((~flags & (M_COPYBACK0_EXTEND|M_COPYBACK0_COW)) != 0); + + mp = mp0; + m = *mp; while (off > (mlen = m->m_len)) { off -= mlen; totlen += mlen; if (m->m_next == NULL) { - n = m_getclr(M_DONTWAIT, m->m_type); - if (n == NULL) + int tspace; +extend: + if (!(flags & M_COPYBACK0_EXTEND)) + goto out; + + /* + * try to make some space at the end of "m". + */ + + mlen = m->m_len; + if (off + len >= MINCLSIZE && + !(m->m_flags & M_EXT) && m->m_len == 0) { + MCLGET(m, how); + } + tspace = M_TRAILINGSPACE(m); + if (tspace > 0) { + tspace = MIN(tspace, off + len); + VERIFY(tspace > 0); + bzero(mtod(m, char *) + m->m_len, + MIN(off, tspace)); + m->m_len += tspace; + off += mlen; + totlen -= mlen; + continue; + } + + /* + * need to allocate an mbuf. + */ + + if (off + len >= MINCLSIZE) { + n = m_getcl(how, m->m_type, 0); + } else { + n = _M_GET(how, m->m_type); + } + if (n == NULL) { goto out; - n->m_len = MIN(MLEN, len + off); + } + n->m_len = 0; + n->m_len = MIN(M_TRAILINGSPACE(n), off + len); + bzero(mtod(n, char *), MIN(n->m_len, off)); m->m_next = n; } + mp = &m->m_next; m = m->m_next; } while (len > 0) { - mlen = MIN(m->m_len - off, len); - bcopy(cp, off + MTOD(m, caddr_t), (unsigned)mlen); - cp += mlen; + mlen = m->m_len - off; + if (mlen != 0 && m_mclhasreference(m)) { + char *datap; + int eatlen; + + /* + * this mbuf is read-only. + * allocate a new writable mbuf and try again. + */ + +#if defined(DIAGNOSTIC) + if (!(flags & M_COPYBACK0_COW)) + panic("m_copyback0: read-only"); +#endif /* defined(DIAGNOSTIC) */ + + /* + * if we're going to write into the middle of + * a mbuf, split it first. + */ + if (off > 0 && len < mlen) { + n = m_split0(m, off, how, 0); + if (n == NULL) + goto enobufs; + m->m_next = n; + mp = &m->m_next; + m = n; + off = 0; + continue; + } + + /* + * XXX TODO coalesce into the trailingspace of + * the previous mbuf when possible. + */ + + /* + * allocate a new mbuf. copy packet header if needed. + */ + n = _M_GET(how, m->m_type); + if (n == NULL) + goto enobufs; + if (off == 0 && (m->m_flags & M_PKTHDR)) { + M_COPY_PKTHDR(n, m); + n->m_len = MHLEN; + } else { + if (len >= MINCLSIZE) + MCLGET(n, M_DONTWAIT); + n->m_len = + (n->m_flags & M_EXT) ? MCLBYTES : MLEN; + } + if (n->m_len > len) + n->m_len = len; + + /* + * free the region which has been overwritten. + * copying data from old mbufs if requested. + */ + if (flags & M_COPYBACK0_PRESERVE) + datap = mtod(n, char *); + else + datap = NULL; + eatlen = n->m_len; + VERIFY(off == 0 || eatlen >= mlen); + if (off > 0) { + VERIFY(len >= mlen); + m->m_len = off; + m->m_next = n; + if (datap) { + m_copydata(m, off, mlen, datap); + datap += mlen; + } + eatlen -= mlen; + mp = &m->m_next; + m = m->m_next; + } + while (m != NULL && m_mclhasreference(m) && + n->m_type == m->m_type && eatlen > 0) { + mlen = MIN(eatlen, m->m_len); + if (datap) { + m_copydata(m, 0, mlen, datap); + datap += mlen; + } + m->m_data += mlen; + m->m_len -= mlen; + eatlen -= mlen; + if (m->m_len == 0) + *mp = m = m_free(m); + } + if (eatlen > 0) + n->m_len -= eatlen; + n->m_next = m; + *mp = m = n; + continue; + } + mlen = MIN(mlen, len); + if (flags & M_COPYBACK0_COPYBACK) { + bcopy(cp, mtod(m, caddr_t) + off, (unsigned)mlen); + cp += mlen; + } len -= mlen; mlen += off; off = 0; @@ -4474,23 +5400,27 @@ m_copyback(struct mbuf *m0, int off, int len, caddr_t cp) if (len == 0) break; if (m->m_next == NULL) { - n = _M_GET(M_DONTWAIT, m->m_type); - if (n == NULL) - break; - n->m_len = MIN(MLEN, len); - m->m_next = n; + goto extend; } + mp = &m->m_next; m = m->m_next; } out: - if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen)) + if (((m = *mp0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen)) { + VERIFY(flags & M_COPYBACK0_EXTEND); m->m_pkthdr.len = totlen; + } + + return (0); + +enobufs: + return (ENOBUFS); } char * mcl_to_paddr(char *addr) { - int base_phys; + vm_offset_t base_phys; if (!MBUF_IN_MAP(addr)) return (NULL); @@ -4498,7 +5428,7 @@ mcl_to_paddr(char *addr) if (base_phys == 0) return (NULL); - return ((char *)((int)base_phys | ((int)addr & PGOFSET))); + return ((char *)((uintptr_t)base_phys | ((uintptr_t)addr & PGOFSET))); } /* @@ -4698,6 +5628,178 @@ m_normalize(struct mbuf *m) return (top); } +/* + * Append the specified data to the indicated mbuf chain, + * Extend the mbuf chain if the new data does not fit in + * existing space. + * + * Return 1 if able to complete the job; otherwise 0. + */ +int +m_append(struct mbuf *m0, int len, caddr_t cp) +{ + struct mbuf *m, *n; + int remainder, space; + + for (m = m0; m->m_next != NULL; m = m->m_next) + ; + remainder = len; + space = M_TRAILINGSPACE(m); + if (space > 0) { + /* + * Copy into available space. + */ + if (space > remainder) + space = remainder; + bcopy(cp, mtod(m, caddr_t) + m->m_len, space); + m->m_len += space; + cp += space, remainder -= space; + } + while (remainder > 0) { + /* + * Allocate a new mbuf; could check space + * and allocate a cluster instead. + */ + n = m_get(M_WAITOK, m->m_type); + if (n == NULL) + break; + n->m_len = min(MLEN, remainder); + bcopy(cp, mtod(n, caddr_t), n->m_len); + cp += n->m_len; + remainder -= n->m_len; + m->m_next = n; + m = n; + } + if (m0->m_flags & M_PKTHDR) + m0->m_pkthdr.len += len - remainder; + return (remainder == 0); +} + +struct mbuf * +m_last(struct mbuf *m) +{ + while (m->m_next != NULL) + m = m->m_next; + return (m); +} + +unsigned int +m_fixhdr(struct mbuf *m0) +{ + u_int len; + + len = m_length2(m0, NULL); + m0->m_pkthdr.len = len; + return (len); +} + +unsigned int +m_length2(struct mbuf *m0, struct mbuf **last) +{ + struct mbuf *m; + u_int len; + + len = 0; + for (m = m0; m != NULL; m = m->m_next) { + len += m->m_len; + if (m->m_next == NULL) + break; + } + if (last != NULL) + *last = m; + return (len); +} + +/* + * Defragment a mbuf chain, returning the shortest possible chain of mbufs + * and clusters. If allocation fails and this cannot be completed, NULL will + * be returned, but the passed in chain will be unchanged. Upon success, + * the original chain will be freed, and the new chain will be returned. + * + * If a non-packet header is passed in, the original mbuf (chain?) will + * be returned unharmed. + * + * If offset is specfied, the first mbuf in the chain will have a leading + * space of the amount stated by the "off" parameter. + * + * This routine requires that the m_pkthdr.header field of the original + * mbuf chain is cleared by the caller. + */ +struct mbuf * +m_defrag_offset(struct mbuf *m0, u_int32_t off, int how) +{ + struct mbuf *m_new = NULL, *m_final = NULL; + int progress = 0, length, pktlen; + + if (!(m0->m_flags & M_PKTHDR)) + return (m0); + + VERIFY(off < MHLEN); + m_fixhdr(m0); /* Needed sanity check */ + + pktlen = m0->m_pkthdr.len + off; + if (pktlen > MHLEN) + m_final = m_getcl(how, MT_DATA, M_PKTHDR); + else + m_final = m_gethdr(how, MT_DATA); + + if (m_final == NULL) + goto nospace; + + if (off > 0) { + pktlen -= off; + m_final->m_len -= off; + m_final->m_data += off; + } + + /* + * Caller must have handled the contents pointed to by this + * pointer before coming here, as otherwise it will point to + * the original mbuf which will get freed upon success. + */ + VERIFY(m0->m_pkthdr.header == NULL); + + if (m_dup_pkthdr(m_final, m0, how) == 0) + goto nospace; + + m_new = m_final; + + while (progress < pktlen) { + length = pktlen - progress; + if (length > MCLBYTES) + length = MCLBYTES; + + if (m_new == NULL) { + if (length > MLEN) + m_new = m_getcl(how, MT_DATA, 0); + else + m_new = m_get(how, MT_DATA); + if (m_new == NULL) + goto nospace; + } + + m_copydata(m0, progress, length, mtod(m_new, caddr_t)); + progress += length; + m_new->m_len = length; + if (m_new != m_final) + m_cat(m_final, m_new); + m_new = NULL; + } + m_freem(m0); + m0 = m_final; + return (m0); +nospace: + if (m_final) + m_freem(m_final); + return (NULL); +} + +struct mbuf * +m_defrag(struct mbuf *m0, int how) +{ + return (m_defrag_offset(m0, 0, how)); +} + void m_mchtype(struct mbuf *m, int t) { @@ -4715,7 +5817,7 @@ m_mtod(struct mbuf *m) struct mbuf * m_dtom(void *x) { - return ((struct mbuf *)((u_long)(x) & ~(MSIZE-1))); + return ((struct mbuf *)((uintptr_t)(x) & ~(MSIZE-1))); } void @@ -4724,6 +5826,34 @@ m_mcheck(struct mbuf *m) _MCHECK(m); } +/* + * Return a pointer to mbuf/offset of location in mbuf chain. + */ +struct mbuf * +m_getptr(struct mbuf *m, int loc, int *off) +{ + + while (loc >= 0) { + /* Normal end of search. */ + if (m->m_len > loc) { + *off = loc; + return (m); + } else { + loc -= m->m_len; + if (m->m_next == NULL) { + if (loc == 0) { + /* Point at the end of valid data. */ + *off = m->m_len; + return (m); + } + return (NULL); + } + m = m->m_next; + } + } + return (NULL); +} + /* * Inform the corresponding mcache(s) that there's a waiter below. */ @@ -4766,6 +5896,29 @@ mbuf_waiter_dec(mbuf_class_t class, boolean_t comp) } } +/* + * Called during slab (blocking and non-blocking) allocation. If there + * is at least one waiter, and the time since the first waiter is blocked + * is greater than the watchdog timeout, panic the system. + */ +static void +mbuf_watchdog(void) +{ + struct timeval now; + unsigned int since; + + if (mb_waiters == 0 || !mb_watchdog) + return; + + microuptime(&now); + since = now.tv_sec - mb_wdtstart.tv_sec; + if (since >= MB_WDT_MAXTIME) { + panic_plain("%s: %d waiters stuck for %u secs\n%s", __func__, + mb_waiters, since, mbuf_dump()); + /* NOTREACHED */ + } +} + /* * Called during blocking allocation. Returns TRUE if one or more objects * are available at the per-CPU caches layer and that allocation should be @@ -4807,6 +5960,16 @@ mbuf_sleep(mbuf_class_t class, unsigned int num, int wait) mbuf_waiter_inc(class, (wait & MCR_COMP)); VERIFY(!(wait & MCR_NOSLEEP)); + + /* + * If this is the first waiter, arm the watchdog timer. Otherwise + * check if we need to panic the system due to watchdog timeout. + */ + if (mb_waiters == 0) + microuptime(&mb_wdtstart); + else + mbuf_watchdog(); + mb_waiters++; (void) msleep(mb_waitchan, mbuf_mlock, (PZERO-1), m_cname(class), NULL); @@ -4961,7 +6124,7 @@ slab_get(void *buf) } } - ix = MTOCL(buf) % NSLABSPMB; + ix = MTOBG(buf) % NSLABSPMB; VERIFY(ix < NSLABSPMB); return (&slg->slg_slab[ix]); @@ -4988,15 +6151,9 @@ slab_insert(mcl_slab_t *sp, mbuf_class_t class) m_slab_cnt(class)++; TAILQ_INSERT_TAIL(&m_slablist(class), sp, sl_link); sp->sl_flags &= ~SLF_DETACHED; - if (class == MC_BIGCL) { - sp = sp->sl_next; - /* Next slab must already be present */ - VERIFY(sp != NULL); - VERIFY(slab_is_detached(sp)); - sp->sl_flags &= ~SLF_DETACHED; - } else if (class == MC_16KCL) { + if (class == MC_16KCL) { int k; - for (k = 1; k < (M16KCLBYTES / MCLBYTES); k++) { + for (k = 1; k < NSLABSP16KB; k++) { sp = sp->sl_next; /* Next slab must already be present */ VERIFY(sp != NULL); @@ -5014,15 +6171,9 @@ slab_remove(mcl_slab_t *sp, mbuf_class_t class) m_slab_cnt(class)--; TAILQ_REMOVE(&m_slablist(class), sp, sl_link); slab_detach(sp); - if (class == MC_BIGCL) { - sp = sp->sl_next; - /* Next slab must already be present */ - VERIFY(sp != NULL); - VERIFY(!slab_is_detached(sp)); - slab_detach(sp); - } else if (class == MC_16KCL) { + if (class == MC_16KCL) { int k; - for (k = 1; k < (M16KCLBYTES / MCLBYTES); k++) { + for (k = 1; k < NSLABSP16KB; k++) { sp = sp->sl_next; /* Next slab must already be present */ VERIFY(sp != NULL); @@ -5039,7 +6190,7 @@ slab_inrange(mcl_slab_t *sp, void *buf) (uintptr_t)buf < ((uintptr_t)sp->sl_base + sp->sl_len)); } -#undef panic(...) +#undef panic static void slab_nextptr_panic(mcl_slab_t *sp, void *addr) @@ -5052,7 +6203,7 @@ slab_nextptr_panic(mcl_slab_t *sp, void *addr) void *next = ((mcache_obj_t *)buf)->obj_next; if (next != addr) continue; - if (mclaudit == NULL) { + if (!mclverify) { if (next != NULL && !MBUF_IN_MAP(next)) { mcache_t *cp = m_cache(sp->sl_class); panic("%s: %s buffer %p in slab %p modified " @@ -5094,12 +6245,14 @@ mcl_audit_init(void *buf, mcache_audit_t **mca_list, boolean_t save_contents = (con_list != NULL); unsigned int i, ix; - ASSERT(num <= NMBPCL); + ASSERT(num <= NMBPBG); ASSERT(con_list == NULL || con_size != 0); - ix = MTOCL(buf); + ix = MTOBG(buf); + VERIFY(ix < maxclaudit); + /* Make sure we haven't been here before */ - for (i = 0; i < NMBPCL; i++) + for (i = 0; i < NMBPBG; i++) VERIFY(mclaudit[ix].cl_audit[i] == NULL); mca = mca_tail = *mca_list; @@ -5135,31 +6288,39 @@ mcl_audit_init(void *buf, mcache_audit_t **mca_list, } /* - * Given an address of a buffer (mbuf/cluster/big cluster), return + * Given an address of a buffer (mbuf/2KB/4KB/16KB), return * the corresponding audit structure for that buffer. */ static mcache_audit_t * mcl_audit_buf2mca(mbuf_class_t class, mcache_obj_t *o) { mcache_audit_t *mca = NULL; - int ix = MTOCL(o); + int ix = MTOBG(o); + VERIFY(ix < maxclaudit); VERIFY(IS_P2ALIGNED(o, MIN(m_maxsize(class), NBPG))); switch (class) { case MC_MBUF: /* - * For the mbuf case, find the index of the cluster + * For the mbuf case, find the index of the page * used by the mbuf and use that index to locate the - * base address of the cluster. Then find out the - * mbuf index relative to the cluster base and use + * base address of the page. Then find out the + * mbuf index relative to the page base and use * it to locate the audit structure. */ - VERIFY(MCLIDX(CLTOM(ix), o) < (int)NMBPCL); - mca = mclaudit[ix].cl_audit[MCLIDX(CLTOM(ix), o)]; + VERIFY(MCLIDX(BGTOM(ix), o) < (int)NMBPBG); + mca = mclaudit[ix].cl_audit[MCLIDX(BGTOM(ix), o)]; break; case MC_CL: + /* + * Same thing as above, but for 2KB clusters in a page. + */ + VERIFY(CLBGIDX(BGTOM(ix), o) < (int)NCLPBG); + mca = mclaudit[ix].cl_audit[CLBGIDX(BGTOM(ix), o)]; + break; + case MC_BIGCL: case MC_16KCL: /* @@ -5186,19 +6347,24 @@ mcl_audit_mbuf(mcache_audit_t *mca, void *addr, boolean_t composite, VERIFY(mca->mca_contents != NULL && mca->mca_contents_size == AUDIT_CONTENTS_SIZE); - mcl_audit_verify_nextptr(next, mca); + if (mclverify) + mcl_audit_verify_nextptr(next, mca); if (!alloc) { /* Save constructed mbuf fields */ mcl_audit_save_mbuf(m, mca); - mcache_set_pattern(MCACHE_FREE_PATTERN, m, m_maxsize(MC_MBUF)); + if (mclverify) { + mcache_set_pattern(MCACHE_FREE_PATTERN, m, + m_maxsize(MC_MBUF)); + } ((mcache_obj_t *)m)->obj_next = next; return; } /* Check if the buffer has been corrupted while in freelist */ - mcache_audit_free_verify_set(mca, addr, 0, m_maxsize(MC_MBUF)); - + if (mclverify) { + mcache_audit_free_verify_set(mca, addr, 0, m_maxsize(MC_MBUF)); + } /* Restore constructed mbuf fields */ mcl_audit_restore_mbuf(m, mca, composite); } @@ -5245,12 +6411,14 @@ mcl_audit_cluster(mcache_audit_t *mca, void *addr, size_t size, boolean_t alloc, mcache_obj_t *next = ((mcache_obj_t *)addr)->obj_next; if (!alloc) { - mcache_set_pattern(MCACHE_FREE_PATTERN, addr, size); + if (mclverify) { + mcache_set_pattern(MCACHE_FREE_PATTERN, addr, size); + } if (save_next) { mcl_audit_verify_nextptr(next, mca); ((mcache_obj_t *)addr)->obj_next = next; } - } else { + } else if (mclverify) { /* Check if the buffer has been corrupted while in freelist */ mcl_audit_verify_nextptr(next, mca); mcache_audit_free_verify_set(mca, addr, 0, size); @@ -5273,8 +6441,8 @@ mcl_audit_mcheck_panic(struct mbuf *m) static void mcl_audit_verify_nextptr(void *next, mcache_audit_t *mca) { - if (next != NULL && next != (void *)MCACHE_FREE_PATTERN && - !MBUF_IN_MAP(next)) { + if (next != NULL && !MBUF_IN_MAP(next) && + (next != (void *)MCACHE_FREE_PATTERN || !mclverify)) { panic("mcl_audit: buffer %p modified after free at offset 0: " "%p out of range [%p-%p)\n%s\n", mca->mca_addr, next, mbutl, embutl, mcache_dump_mca(mca)); @@ -5282,10 +6450,497 @@ mcl_audit_verify_nextptr(void *next, mcache_audit_t *mca) } } +/* This function turns on mbuf leak detection */ +static void +mleak_activate(void) +{ + mleak_table.mleak_sample_factor = MLEAK_SAMPLE_FACTOR; + PE_parse_boot_argn("mleak_sample_factor", + &mleak_table.mleak_sample_factor, + sizeof (mleak_table.mleak_sample_factor)); + + if (mleak_table.mleak_sample_factor == 0) + mclfindleak = 0; + + if (mclfindleak == 0) + return; + + vm_size_t alloc_size = + mleak_alloc_buckets * sizeof (struct mallocation); + vm_size_t trace_size = mleak_trace_buckets * sizeof (struct mtrace); + + MALLOC(mleak_allocations, struct mallocation *, alloc_size, + M_TEMP, M_WAITOK | M_ZERO); + VERIFY(mleak_allocations != NULL); + + MALLOC(mleak_traces, struct mtrace *, trace_size, + M_TEMP, M_WAITOK | M_ZERO); + VERIFY(mleak_traces != NULL); + + MALLOC(mleak_stat, mleak_stat_t *, MLEAK_STAT_SIZE(MLEAK_NUM_TRACES), + M_TEMP, M_WAITOK | M_ZERO); + VERIFY(mleak_stat != NULL); + mleak_stat->ml_cnt = MLEAK_NUM_TRACES; +#ifdef __LP64__ + mleak_stat->ml_isaddr64 = 1; +#endif /* __LP64__ */ +} + +static void +mleak_logger(u_int32_t num, mcache_obj_t *addr, boolean_t alloc) +{ + int temp; + + if (mclfindleak == 0) + return; + + if (!alloc) + return (mleak_free(addr)); + + temp = atomic_add_32_ov(&mleak_table.mleak_capture, 1); + + if ((temp % mleak_table.mleak_sample_factor) == 0 && addr != NULL) { + uintptr_t bt[MLEAK_STACK_DEPTH]; + int logged = fastbacktrace(bt, MLEAK_STACK_DEPTH); + mleak_log(bt, addr, logged, num); + } +} + +/* + * This function records the allocation in the mleak_allocations table + * and the backtrace in the mleak_traces table; if allocation slot is in use, + * replace old allocation with new one if the trace slot is in use, return + * (or increment refcount if same trace). + */ +static boolean_t +mleak_log(uintptr_t *bt, mcache_obj_t *addr, uint32_t depth, int num) +{ + struct mallocation *allocation; + struct mtrace *trace; + uint32_t trace_index; + + /* Quit if someone else modifying the tables */ + if (!lck_mtx_try_lock_spin(mleak_lock)) { + mleak_table.total_conflicts++; + return (FALSE); + } + + allocation = &mleak_allocations[hashaddr((uintptr_t)addr, + mleak_alloc_buckets)]; + trace_index = hashbacktrace(bt, depth, mleak_trace_buckets); + trace = &mleak_traces[trace_index]; + + VERIFY(allocation <= &mleak_allocations[mleak_alloc_buckets - 1]); + VERIFY(trace <= &mleak_traces[mleak_trace_buckets - 1]); + + allocation->hitcount++; + trace->hitcount++; + + /* + * If the allocation bucket we want is occupied + * and the occupier has the same trace, just bail. + */ + if (allocation->element != NULL && + trace_index == allocation->trace_index) { + mleak_table.alloc_collisions++; + lck_mtx_unlock(mleak_lock); + return (TRUE); + } + + /* + * Store the backtrace in the traces array; + * Size of zero = trace bucket is free. + */ + if (trace->allocs > 0 && + bcmp(trace->addr, bt, (depth * sizeof (uintptr_t))) != 0) { + /* Different, unique trace, but the same hash! Bail out. */ + trace->collisions++; + mleak_table.trace_collisions++; + lck_mtx_unlock(mleak_lock); + return (TRUE); + } else if (trace->allocs > 0) { + /* Same trace, already added, so increment refcount */ + trace->allocs++; + } else { + /* Found an unused trace bucket, so record the trace here */ + if (trace->depth != 0) { + /* this slot previously used but not currently in use */ + mleak_table.trace_overwrites++; + } + mleak_table.trace_recorded++; + trace->allocs = 1; + memcpy(trace->addr, bt, (depth * sizeof (uintptr_t))); + trace->depth = depth; + trace->collisions = 0; + } + + /* Step 2: Store the allocation record in the allocations array */ + if (allocation->element != NULL) { + /* + * Replace an existing allocation. No need to preserve + * because only a subset of the allocations are being + * recorded anyway. + */ + mleak_table.alloc_collisions++; + } else if (allocation->trace_index != 0) { + mleak_table.alloc_overwrites++; + } + allocation->element = addr; + allocation->trace_index = trace_index; + allocation->count = num; + mleak_table.alloc_recorded++; + mleak_table.outstanding_allocs++; + + lck_mtx_unlock(mleak_lock); + return (TRUE); +} + +static void +mleak_free(mcache_obj_t *addr) +{ + while (addr != NULL) { + struct mallocation *allocation = &mleak_allocations + [hashaddr((uintptr_t)addr, mleak_alloc_buckets)]; + + if (allocation->element == addr && + allocation->trace_index < mleak_trace_buckets) { + lck_mtx_lock_spin(mleak_lock); + if (allocation->element == addr && + allocation->trace_index < mleak_trace_buckets) { + struct mtrace *trace; + trace = &mleak_traces[allocation->trace_index]; + /* allocs = 0 means trace bucket is unused */ + if (trace->allocs > 0) + trace->allocs--; + if (trace->allocs == 0) + trace->depth = 0; + /* NULL element means alloc bucket is unused */ + allocation->element = NULL; + mleak_table.outstanding_allocs--; + } + lck_mtx_unlock(mleak_lock); + } + addr = addr->obj_next; + } +} + +static void +mleak_sort_traces() +{ + int i, j, k; + struct mtrace *swap; + + for(i = 0; i < MLEAK_NUM_TRACES; i++) + mleak_top_trace[i] = NULL; + + for(i = 0, j = 0; j < MLEAK_NUM_TRACES && i < mleak_trace_buckets; i++) + { + if (mleak_traces[i].allocs <= 0) + continue; + + mleak_top_trace[j] = &mleak_traces[i]; + for (k = j; k > 0; k--) { + if (mleak_top_trace[k]->allocs <= + mleak_top_trace[k-1]->allocs) + break; + + swap = mleak_top_trace[k-1]; + mleak_top_trace[k-1] = mleak_top_trace[k]; + mleak_top_trace[k] = swap; + } + j++; + } + + j--; + for(; i < mleak_trace_buckets; i++) { + if (mleak_traces[i].allocs <= mleak_top_trace[j]->allocs) + continue; + + mleak_top_trace[j] = &mleak_traces[i]; + + for (k = j; k > 0; k--) { + if (mleak_top_trace[k]->allocs <= + mleak_top_trace[k-1]->allocs) + break; + + swap = mleak_top_trace[k-1]; + mleak_top_trace[k-1] = mleak_top_trace[k]; + mleak_top_trace[k] = swap; + } + } +} + +static void +mleak_update_stats() +{ + mleak_trace_stat_t *mltr; + int i; + + VERIFY(mleak_stat != NULL); +#ifdef __LP64__ + VERIFY(mleak_stat->ml_isaddr64); +#else + VERIFY(!mleak_stat->ml_isaddr64); +#endif /* !__LP64__ */ + VERIFY(mleak_stat->ml_cnt == MLEAK_NUM_TRACES); + + mleak_sort_traces(); + + mltr = &mleak_stat->ml_trace[0]; + bzero(mltr, sizeof (*mltr) * MLEAK_NUM_TRACES); + for (i = 0; i < MLEAK_NUM_TRACES; i++) { + int j; + + if (mleak_top_trace[i] == NULL || + mleak_top_trace[i]->allocs == 0) + continue; + + mltr->mltr_collisions = mleak_top_trace[i]->collisions; + mltr->mltr_hitcount = mleak_top_trace[i]->hitcount; + mltr->mltr_allocs = mleak_top_trace[i]->allocs; + mltr->mltr_depth = mleak_top_trace[i]->depth; + + VERIFY(mltr->mltr_depth <= MLEAK_STACK_DEPTH); + for (j = 0; j < mltr->mltr_depth; j++) + mltr->mltr_addr[j] = mleak_top_trace[i]->addr[j]; + + mltr++; + } +} + +static struct mbtypes { + int mt_type; + const char *mt_name; +} mbtypes[] = { + { MT_DATA, "data" }, + { MT_OOBDATA, "oob data" }, + { MT_CONTROL, "ancillary data" }, + { MT_HEADER, "packet headers" }, + { MT_SOCKET, "socket structures" }, + { MT_PCB, "protocol control blocks" }, + { MT_RTABLE, "routing table entries" }, + { MT_HTABLE, "IMP host table entries" }, + { MT_ATABLE, "address resolution tables" }, + { MT_FTABLE, "fragment reassembly queue headers" }, + { MT_SONAME, "socket names and addresses" }, + { MT_SOOPTS, "socket options" }, + { MT_RIGHTS, "access rights" }, + { MT_IFADDR, "interface addresses" }, + { MT_TAG, "packet tags" }, + { 0, NULL } +}; + +#define MBUF_DUMP_BUF_CHK() { \ + clen -= k; \ + if (clen < 1) \ + goto done; \ + c += k; \ +} + +static char * +mbuf_dump(void) +{ + unsigned long totmem = 0, totfree = 0, totmbufs, totused, totpct; + u_int32_t m_mbufs = 0, m_clfree = 0, m_bigclfree = 0; + u_int32_t m_mbufclfree = 0, m_mbufbigclfree = 0; + u_int32_t m_16kclusters = 0, m_16kclfree = 0, m_mbuf16kclfree = 0; + int nmbtypes = sizeof (mbstat.m_mtypes) / sizeof (short); + uint8_t seen[256]; + struct mbtypes *mp; + mb_class_stat_t *sp; + mleak_trace_stat_t *mltr; + char *c = mbuf_dump_buf; + int i, k, clen = MBUF_DUMP_BUF_SIZE; + + mbuf_dump_buf[0] = '\0'; + + /* synchronize all statistics in the mbuf table */ + mbuf_stat_sync(); + mbuf_mtypes_sync(TRUE); + + sp = &mb_stat->mbs_class[0]; + for (i = 0; i < mb_stat->mbs_cnt; i++, sp++) { + u_int32_t mem; + + if (m_class(i) == MC_MBUF) { + m_mbufs = sp->mbcl_active; + } else if (m_class(i) == MC_CL) { + m_clfree = sp->mbcl_total - sp->mbcl_active; + } else if (m_class(i) == MC_BIGCL) { + m_bigclfree = sp->mbcl_total - sp->mbcl_active; + } else if (njcl > 0 && m_class(i) == MC_16KCL) { + m_16kclfree = sp->mbcl_total - sp->mbcl_active; + m_16kclusters = sp->mbcl_total; + } else if (m_class(i) == MC_MBUF_CL) { + m_mbufclfree = sp->mbcl_total - sp->mbcl_active; + } else if (m_class(i) == MC_MBUF_BIGCL) { + m_mbufbigclfree = sp->mbcl_total - sp->mbcl_active; + } else if (njcl > 0 && m_class(i) == MC_MBUF_16KCL) { + m_mbuf16kclfree = sp->mbcl_total - sp->mbcl_active; + } + + mem = sp->mbcl_ctotal * sp->mbcl_size; + totmem += mem; + totfree += (sp->mbcl_mc_cached + sp->mbcl_infree) * + sp->mbcl_size; + + } + + /* adjust free counts to include composite caches */ + m_clfree += m_mbufclfree; + m_bigclfree += m_mbufbigclfree; + m_16kclfree += m_mbuf16kclfree; + + totmbufs = 0; + for (mp = mbtypes; mp->mt_name != NULL; mp++) + totmbufs += mbstat.m_mtypes[mp->mt_type]; + if (totmbufs > m_mbufs) + totmbufs = m_mbufs; + k = snprintf(c, clen, "%lu/%u mbufs in use:\n", totmbufs, m_mbufs); + MBUF_DUMP_BUF_CHK(); + + bzero(&seen, sizeof (seen)); + for (mp = mbtypes; mp->mt_name != NULL; mp++) { + if (mbstat.m_mtypes[mp->mt_type] != 0) { + seen[mp->mt_type] = 1; + k = snprintf(c, clen, "\t%u mbufs allocated to %s\n", + mbstat.m_mtypes[mp->mt_type], mp->mt_name); + MBUF_DUMP_BUF_CHK(); + } + } + seen[MT_FREE] = 1; + for (i = 0; i < nmbtypes; i++) + if (!seen[i] && mbstat.m_mtypes[i] != 0) { + k = snprintf(c, clen, "\t%u mbufs allocated to " + "\n", mbstat.m_mtypes[i], i); + MBUF_DUMP_BUF_CHK(); + } + if ((m_mbufs - totmbufs) > 0) { + k = snprintf(c, clen, "\t%lu mbufs allocated to caches\n", + m_mbufs - totmbufs); + MBUF_DUMP_BUF_CHK(); + } + k = snprintf(c, clen, "%u/%u mbuf 2KB clusters in use\n" + "%u/%u mbuf 4KB clusters in use\n", + (unsigned int)(mbstat.m_clusters - m_clfree), + (unsigned int)mbstat.m_clusters, + (unsigned int)(mbstat.m_bigclusters - m_bigclfree), + (unsigned int)mbstat.m_bigclusters); + MBUF_DUMP_BUF_CHK(); + + if (njcl > 0) { + k = snprintf(c, clen, "%u/%u mbuf %uKB clusters in use\n", + m_16kclusters - m_16kclfree, m_16kclusters, + njclbytes / 1024); + MBUF_DUMP_BUF_CHK(); + } + totused = totmem - totfree; + if (totmem == 0) { + totpct = 0; + } else if (totused < (ULONG_MAX / 100)) { + totpct = (totused * 100) / totmem; + } else { + u_long totmem1 = totmem / 100; + u_long totused1 = totused / 100; + totpct = (totused1 * 100) / totmem1; + } + k = snprintf(c, clen, "%lu KB allocated to network (approx. %lu%% " + "in use)\n", totmem / 1024, totpct); + MBUF_DUMP_BUF_CHK(); + + /* mbuf leak detection statistics */ + mleak_update_stats(); + + k = snprintf(c, clen, "\nmbuf leak detection table:\n"); + MBUF_DUMP_BUF_CHK(); + k = snprintf(c, clen, "\ttotal captured: %u (one per %u)\n", + mleak_table.mleak_capture / mleak_table.mleak_sample_factor, + mleak_table.mleak_sample_factor); + MBUF_DUMP_BUF_CHK(); + k = snprintf(c, clen, "\ttotal allocs outstanding: %llu\n", + mleak_table.outstanding_allocs); + MBUF_DUMP_BUF_CHK(); + k = snprintf(c, clen, "\tnew hash recorded: %llu allocs, %llu traces\n", + mleak_table.alloc_recorded, mleak_table.trace_recorded); + MBUF_DUMP_BUF_CHK(); + k = snprintf(c, clen, "\thash collisions: %llu allocs, %llu traces\n", + mleak_table.alloc_collisions, mleak_table.trace_collisions); + MBUF_DUMP_BUF_CHK(); + k = snprintf(c, clen, "\toverwrites: %llu allocs, %llu traces\n", + mleak_table.alloc_overwrites, mleak_table.trace_overwrites); + MBUF_DUMP_BUF_CHK(); + k = snprintf(c, clen, "\tlock conflicts: %llu\n\n", + mleak_table.total_conflicts); + MBUF_DUMP_BUF_CHK(); + + k = snprintf(c, clen, "top %d outstanding traces:\n", + mleak_stat->ml_cnt); + MBUF_DUMP_BUF_CHK(); + for (i = 0; i < mleak_stat->ml_cnt; i++) { + mltr = &mleak_stat->ml_trace[i]; + k = snprintf(c, clen, "[%d] %llu outstanding alloc(s), " + "%llu hit(s), %llu collision(s)\n", (i + 1), + mltr->mltr_allocs, mltr->mltr_hitcount, + mltr->mltr_collisions); + MBUF_DUMP_BUF_CHK(); + } + + if (mleak_stat->ml_isaddr64) + k = snprintf(c, clen, MB_LEAK_HDR_64); + else + k = snprintf(c, clen, MB_LEAK_HDR_32); + MBUF_DUMP_BUF_CHK(); + + for (i = 0; i < MLEAK_STACK_DEPTH; i++) { + int j; + k = snprintf(c, clen, "%2d: ", (i + 1)); + MBUF_DUMP_BUF_CHK(); + for (j = 0; j < mleak_stat->ml_cnt; j++) { + mltr = &mleak_stat->ml_trace[j]; + if (i < mltr->mltr_depth) { + if (mleak_stat->ml_isaddr64) { + k = snprintf(c, clen, "0x%0llx ", + mltr->mltr_addr[i]); + } else { + k = snprintf(c, clen, + "0x%08x ", + (u_int32_t)mltr->mltr_addr[i]); + } + } else { + if (mleak_stat->ml_isaddr64) + k = snprintf(c, clen, + MB_LEAK_SPACING_64); + else + k = snprintf(c, clen, + MB_LEAK_SPACING_32); + } + MBUF_DUMP_BUF_CHK(); + } + k = snprintf(c, clen, "\n"); + MBUF_DUMP_BUF_CHK(); + } +done: + return (mbuf_dump_buf); +} + +#undef MBUF_DUMP_BUF_CHK + SYSCTL_DECL(_kern_ipc); -SYSCTL_PROC(_kern_ipc, KIPC_MBSTAT, mbstat, CTLFLAG_RD | CTLFLAG_LOCKED, +SYSCTL_PROC(_kern_ipc, KIPC_MBSTAT, mbstat, + CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0, mbstat_sysctl, "S,mbstat", ""); -SYSCTL_PROC(_kern_ipc, OID_AUTO, mb_stat, CTLFLAG_RD | CTLFLAG_LOCKED, +SYSCTL_PROC(_kern_ipc, OID_AUTO, mb_stat, + CTLFLAG_RD | CTLFLAG_LOCKED, 0, 0, mb_stat_sysctl, "S,mb_stat", ""); -SYSCTL_INT(_kern_ipc, OID_AUTO, mb_normalized, CTLFLAG_RD | CTLFLAG_LOCKED, - &mb_normalized, 0, ""); +SYSCTL_PROC(_kern_ipc, OID_AUTO, mleak_top_trace, + CTLFLAG_RD | CTLFLAG_LOCKED, + 0, 0, mleak_top_trace_sysctl, "S,mb_top_trace", ""); +SYSCTL_PROC(_kern_ipc, OID_AUTO, mleak_table, + CTLFLAG_RD | CTLFLAG_LOCKED, + 0, 0, mleak_table_sysctl, "S,mleak_table", ""); +SYSCTL_INT(_kern_ipc, OID_AUTO, mleak_sample_factor, + CTLFLAG_RW | CTLFLAG_LOCKED, &mleak_table.mleak_sample_factor, 0, ""); +SYSCTL_INT(_kern_ipc, OID_AUTO, mb_normalized, + CTLFLAG_RD | CTLFLAG_LOCKED, &mb_normalized, 0, ""); +SYSCTL_INT(_kern_ipc, OID_AUTO, mb_watchdog, + CTLFLAG_RW | CTLFLAG_LOCKED, &mb_watchdog, 0, "");