+ /* We are busy now; tell everyone else to go away */
+ mb_clalloc_busy = TRUE;
+
+ sp = slab_get(buf);
+ VERIFY(sp->sl_class == class && slab_inrange(sp, buf) &&
+ (sp->sl_flags & (SLF_MAPPED | SLF_PARTIAL)) == SLF_MAPPED);
+
+ /* Decrement slab reference */
+ sp->sl_refcnt--;
+
+ if (class == MC_CL) {
+ VERIFY(IS_P2ALIGNED(buf, MCLBYTES));
+ /*
+ * A slab that has been splitted for 2KB clusters can have
+ * at most 1 outstanding reference at this point.
+ */
+ VERIFY(sp->sl_refcnt >= 0 && sp->sl_refcnt <= (NCLPG - 1) &&
+ sp->sl_chunks == NCLPG && sp->sl_len == PAGE_SIZE);
+ VERIFY(sp->sl_refcnt < (NCLPG - 1) ||
+ (slab_is_detached(sp) && sp->sl_head == NULL));
+ } else if (class == MC_BIGCL) {
+ VERIFY(IS_P2ALIGNED(buf, MBIGCLBYTES));
+
+ /* A 4KB cluster slab can have NBCLPG references at most */
+ VERIFY(sp->sl_refcnt >= 0 && sp->sl_chunks == NBCLPG);
+ VERIFY(sp->sl_refcnt < (NBCLPG - 1) ||
+ (slab_is_detached(sp) && sp->sl_head == NULL));
+ } else if (class == MC_16KCL) {
+ mcl_slab_t *nsp;
+ int k;
+ /*
+ * A 16KB cluster takes NSLABSP16KB slabs, all must
+ * now have 0 reference.
+ */
+ VERIFY(IS_P2ALIGNED(buf, PAGE_SIZE));
+ VERIFY(sp->sl_refcnt == 0 && sp->sl_chunks == 1 &&
+ sp->sl_len == m_maxsize(class) && sp->sl_head == NULL);
+ VERIFY(slab_is_detached(sp));
+ for (nsp = sp, k = 1; k < NSLABSP16KB; k++) {
+ nsp = nsp->sl_next;
+ /* Next slab must already be present */
+ VERIFY(nsp != NULL);
+ nsp->sl_refcnt--;
+ VERIFY(slab_is_detached(nsp));
+ VERIFY(nsp->sl_class == MC_16KCL &&
+ (nsp->sl_flags & (SLF_MAPPED | SLF_PARTIAL)) &&
+ nsp->sl_refcnt == 0 && nsp->sl_chunks == 0 &&
+ nsp->sl_len == 0 && nsp->sl_base == sp->sl_base &&
+ nsp->sl_head == NULL);
+ }
+ } else {
+ /*
+ * A slab that has been splitted for mbufs has at most
+ * NMBPG reference counts. Since we have decremented
+ * one reference above, it must now be between 0 and
+ * NMBPG-1.
+ */
+ VERIFY(class == MC_MBUF);
+ VERIFY(sp->sl_refcnt >= 0 &&
+ sp->sl_refcnt <= (NMBPG - 1) &&
+ sp->sl_chunks == NMBPG &&
+ sp->sl_len == PAGE_SIZE);
+ VERIFY(sp->sl_refcnt < (NMBPG - 1) ||
+ (slab_is_detached(sp) && sp->sl_head == NULL));
+ }
+
+ /*
+ * When auditing is enabled, ensure that the buffer still
+ * contains the free pattern. Otherwise it got corrupted
+ * while at the CPU cache layer.
+ */
+ if (mclaudit != NULL) {
+ mcache_audit_t *mca = mcl_audit_buf2mca(class, buf);
+ if (mclverify) {
+ mcache_audit_free_verify(mca, buf, 0,
+ m_maxsize(class));
+ }
+ mca->mca_uflags &= ~MB_SCVALID;
+ }
+
+ if (class == MC_CL) {
+ mbstat.m_clfree = (++m_infree(MC_CL)) + m_infree(MC_MBUF_CL);
+ buf->obj_next = sp->sl_head;
+ } else if (class == MC_BIGCL) {
+ mbstat.m_bigclfree = (++m_infree(MC_BIGCL)) +
+ m_infree(MC_MBUF_BIGCL);
+ buf->obj_next = sp->sl_head;
+ } else if (class == MC_16KCL) {
+ ++m_infree(MC_16KCL);
+ } else {
+ ++m_infree(MC_MBUF);
+ buf->obj_next = sp->sl_head;
+ }
+ sp->sl_head = buf;
+
+ /*
+ * If a slab has been split to either one which holds 2KB clusters,
+ * or one which holds mbufs, turn it back to one which holds a
+ * 4 or 16 KB cluster depending on the page size.
+ */
+ if (m_maxsize(MC_BIGCL) == PAGE_SIZE) {
+ super_class = MC_BIGCL;
+ } else {
+ VERIFY(PAGE_SIZE == m_maxsize(MC_16KCL));
+ super_class = MC_16KCL;
+ }
+ if (class == MC_MBUF && sp->sl_refcnt == 0 &&
+ m_total(class) >= (m_minlimit(class) + NMBPG) &&
+ m_total(super_class) < m_maxlimit(super_class)) {
+ int i = NMBPG;
+
+ m_total(MC_MBUF) -= NMBPG;
+ mbstat.m_mbufs = m_total(MC_MBUF);
+ m_infree(MC_MBUF) -= NMBPG;
+ mtype_stat_add(MT_FREE, -((unsigned)NMBPG));
+
+ while (i--) {
+ struct mbuf *m = sp->sl_head;
+ VERIFY(m != NULL);
+ sp->sl_head = m->m_next;
+ m->m_next = NULL;
+ }
+ reinit_supercl = true;
+ } else if (class == MC_CL && sp->sl_refcnt == 0 &&
+ m_total(class) >= (m_minlimit(class) + NCLPG) &&
+ m_total(super_class) < m_maxlimit(super_class)) {
+ int i = NCLPG;
+
+ m_total(MC_CL) -= NCLPG;
+ mbstat.m_clusters = m_total(MC_CL);
+ m_infree(MC_CL) -= NCLPG;
+
+ while (i--) {
+ union mcluster *c = sp->sl_head;
+ VERIFY(c != NULL);
+ sp->sl_head = c->mcl_next;
+ c->mcl_next = NULL;
+ }
+ reinit_supercl = true;
+ } else if (class == MC_BIGCL && super_class != MC_BIGCL &&
+ sp->sl_refcnt == 0 &&
+ m_total(class) >= (m_minlimit(class) + NBCLPG) &&
+ m_total(super_class) < m_maxlimit(super_class)) {
+ int i = NBCLPG;
+
+ VERIFY(super_class == MC_16KCL);
+ m_total(MC_BIGCL) -= NBCLPG;
+ mbstat.m_bigclusters = m_total(MC_BIGCL);
+ m_infree(MC_BIGCL) -= NBCLPG;
+
+ while (i--) {
+ union mbigcluster *bc = sp->sl_head;
+ VERIFY(bc != NULL);
+ sp->sl_head = bc->mbc_next;
+ bc->mbc_next = NULL;
+ }
+ reinit_supercl = true;
+ }
+
+ if (reinit_supercl) {
+ VERIFY(sp->sl_head == NULL);
+ VERIFY(m_total(class) >= m_minlimit(class));
+ slab_remove(sp, class);
+
+ /* Reinitialize it as a cluster for the super class */
+ m_total(super_class)++;
+ m_infree(super_class)++;
+ VERIFY(sp->sl_flags == (SLF_MAPPED | SLF_DETACHED) &&
+ sp->sl_len == PAGE_SIZE && sp->sl_refcnt == 0);
+
+ slab_init(sp, super_class, SLF_MAPPED, sp->sl_base,
+ sp->sl_base, PAGE_SIZE, 0, 1);
+ if (mclverify) {
+ mcache_set_pattern(MCACHE_FREE_PATTERN,
+ (caddr_t)sp->sl_base, sp->sl_len);
+ }
+ ((mcache_obj_t *)(sp->sl_base))->obj_next = NULL;
+
+ if (super_class == MC_BIGCL) {
+ mbstat.m_bigclusters = m_total(MC_BIGCL);
+ mbstat.m_bigclfree = m_infree(MC_BIGCL) +
+ m_infree(MC_MBUF_BIGCL);
+ }
+
+ VERIFY(slab_is_detached(sp));
+ VERIFY(m_total(super_class) <= m_maxlimit(super_class));
+
+ /* And finally switch class */
+ class = super_class;
+ }
+
+ /* Reinsert the slab to the class's slab list */
+ if (slab_is_detached(sp)) {
+ slab_insert(sp, class);
+ }
+
+ /* We're done; let others enter */
+ mb_clalloc_busy = FALSE;
+ if (mb_clalloc_waiters > 0) {
+ mb_clalloc_waiters = 0;
+ wakeup(mb_clalloc_waitchan);
+ }
+}
+
+/*
+ * Common allocator for rudimentary objects called by the CPU cache layer
+ * during an allocation request whenever there is no available element in the
+ * bucket layer. It returns one or more elements from the appropriate global
+ * freelist. If the freelist is empty, it will attempt to populate it and
+ * retry the allocation.
+ */
+static unsigned int
+mbuf_slab_alloc(void *arg, mcache_obj_t ***plist, unsigned int num, int wait)
+{
+ mbuf_class_t class = (mbuf_class_t)arg;
+ unsigned int need = num;
+ mcache_obj_t **list = *plist;
+
+ ASSERT(MBUF_CLASS_VALID(class) && !MBUF_CLASS_COMPOSITE(class));
+ ASSERT(need > 0);
+
+ lck_mtx_lock(mbuf_mlock);
+
+ for (;;) {
+ if ((*list = slab_alloc(class, wait)) != NULL) {
+ (*list)->obj_next = NULL;
+ list = *plist = &(*list)->obj_next;
+
+ if (--need == 0) {
+ /*
+ * If the number of elements in freelist has
+ * dropped below low watermark, asynchronously
+ * populate the freelist now rather than doing
+ * it later when we run out of elements.
+ */
+ if (!mbuf_cached_above(class, wait) &&
+ m_infree(class) < (m_total(class) >> 5)) {
+ (void) freelist_populate(class, 1,
+ M_DONTWAIT);
+ }
+ break;
+ }
+ } else {
+ VERIFY(m_infree(class) == 0 || class == MC_CL);
+
+ (void) freelist_populate(class, 1,
+ (wait & MCR_NOSLEEP) ? M_DONTWAIT : M_WAIT);
+
+ if (m_infree(class) > 0) {
+ continue;
+ }
+
+ /* Check if there's anything at the cache layer */
+ if (mbuf_cached_above(class, wait)) {
+ break;
+ }
+
+ /* watchdog checkpoint */
+ mbuf_watchdog();
+
+ /* We have nothing and cannot block; give up */
+ if (wait & MCR_NOSLEEP) {
+ if (!(wait & MCR_TRYHARD)) {
+ m_fail_cnt(class)++;
+ mbstat.m_drops++;
+ break;
+ }
+ }
+
+ /*
+ * If the freelist is still empty and the caller is
+ * willing to be blocked, sleep on the wait channel
+ * until an element is available. Otherwise, if
+ * MCR_TRYHARD is set, do our best to satisfy the
+ * request without having to go to sleep.
+ */
+ if (mbuf_worker_ready &&
+ mbuf_sleep(class, need, wait)) {
+ break;
+ }
+
+ LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
+ }
+ }
+
+ m_alloc_cnt(class) += num - need;
+ lck_mtx_unlock(mbuf_mlock);
+
+ return num - need;
+}
+
+/*
+ * Common de-allocator for rudimentary objects called by the CPU cache
+ * layer when one or more elements need to be returned to the appropriate
+ * global freelist.
+ */
+static void
+mbuf_slab_free(void *arg, mcache_obj_t *list, __unused int purged)
+{
+ mbuf_class_t class = (mbuf_class_t)arg;
+ mcache_obj_t *nlist;
+ unsigned int num = 0;
+ int w;
+
+ ASSERT(MBUF_CLASS_VALID(class) && !MBUF_CLASS_COMPOSITE(class));
+
+ lck_mtx_lock(mbuf_mlock);
+
+ for (;;) {
+ nlist = list->obj_next;
+ list->obj_next = NULL;
+ slab_free(class, list);
+ ++num;
+ if ((list = nlist) == NULL) {
+ break;
+ }
+ }
+ m_free_cnt(class) += num;
+
+ if ((w = mb_waiters) > 0) {
+ mb_waiters = 0;
+ }
+ if (w) {
+ mbwdog_logger("waking up all threads");
+ }
+ lck_mtx_unlock(mbuf_mlock);
+
+ if (w != 0) {
+ wakeup(mb_waitchan);
+ }
+}
+
+/*
+ * Common auditor for rudimentary objects called by the CPU cache layer
+ * during an allocation or free request. For the former, this is called
+ * after the objects are obtained from either the bucket or slab layer
+ * and before they are returned to the caller. For the latter, this is
+ * called immediately during free and before placing the objects into
+ * the bucket or slab layer.
+ */
+static void
+mbuf_slab_audit(void *arg, mcache_obj_t *list, boolean_t alloc)
+{
+ mbuf_class_t class = (mbuf_class_t)arg;
+ mcache_audit_t *mca;
+
+ ASSERT(MBUF_CLASS_VALID(class) && !MBUF_CLASS_COMPOSITE(class));
+
+ while (list != NULL) {
+ lck_mtx_lock(mbuf_mlock);
+ mca = mcl_audit_buf2mca(class, list);
+
+ /* Do the sanity checks */
+ if (class == MC_MBUF) {
+ mcl_audit_mbuf(mca, list, FALSE, alloc);
+ ASSERT(mca->mca_uflags & MB_SCVALID);
+ } else {
+ mcl_audit_cluster(mca, list, m_maxsize(class),
+ alloc, TRUE);
+ ASSERT(!(mca->mca_uflags & MB_SCVALID));
+ }
+ /* Record this transaction */
+ if (mcltrace) {
+ mcache_buffer_log(mca, list, m_cache(class), &mb_start);
+ }
+
+ if (alloc) {
+ mca->mca_uflags |= MB_INUSE;
+ } else {
+ mca->mca_uflags &= ~MB_INUSE;
+ }
+ /* Unpair the object (unconditionally) */
+ mca->mca_uptr = NULL;
+ lck_mtx_unlock(mbuf_mlock);
+
+ list = list->obj_next;
+ }
+}
+
+/*
+ * Common notify routine for all caches. It is called by mcache when
+ * one or more objects get freed. We use this indication to trigger
+ * the wakeup of any sleeping threads so that they can retry their
+ * allocation requests.
+ */
+static void
+mbuf_slab_notify(void *arg, u_int32_t reason)
+{
+ mbuf_class_t class = (mbuf_class_t)arg;
+ int w;
+
+ ASSERT(MBUF_CLASS_VALID(class));
+
+ if (reason != MCN_RETRYALLOC) {
+ return;
+ }
+
+ lck_mtx_lock(mbuf_mlock);
+ if ((w = mb_waiters) > 0) {
+ m_notified(class)++;
+ mb_waiters = 0;
+ }
+ if (w) {
+ mbwdog_logger("waking up all threads");
+ }
+ lck_mtx_unlock(mbuf_mlock);
+
+ if (w != 0) {
+ wakeup(mb_waitchan);
+ }
+}
+
+/*
+ * Obtain object(s) from the composite class's freelist.
+ */
+static unsigned int
+cslab_alloc(mbuf_class_t class, mcache_obj_t ***plist, unsigned int num)
+{
+ unsigned int need = num;
+ mcl_slab_t *sp, *clsp, *nsp;
+ struct mbuf *m;
+ mcache_obj_t **list = *plist;
+ void *cl;
+
+ VERIFY(need > 0);
+ VERIFY(class != MC_MBUF_16KCL || njcl > 0);
+ LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
+
+ /* Get what we can from the freelist */
+ while ((*list = m_cobjlist(class)) != NULL) {
+ MRANGE(*list);
+
+ m = (struct mbuf *)*list;
+ sp = slab_get(m);
+ cl = m->m_ext.ext_buf;
+ clsp = slab_get(cl);
+ VERIFY(m->m_flags == M_EXT && cl != NULL);
+ VERIFY(m_get_rfa(m) != NULL && MBUF_IS_COMPOSITE(m));
+
+ if (class == MC_MBUF_CL) {
+ VERIFY(clsp->sl_refcnt >= 1 &&
+ clsp->sl_refcnt <= NCLPG);
+ } else {
+ VERIFY(clsp->sl_refcnt >= 1 &&
+ clsp->sl_refcnt <= NBCLPG);
+ }
+
+ if (class == MC_MBUF_16KCL) {
+ int k;
+ for (nsp = clsp, k = 1; k < NSLABSP16KB; k++) {
+ nsp = nsp->sl_next;
+ /* Next slab must already be present */
+ VERIFY(nsp != NULL);
+ VERIFY(nsp->sl_refcnt == 1);
+ }
+ }
+
+ if ((m_cobjlist(class) = (*list)->obj_next) != NULL &&
+ !MBUF_IN_MAP(m_cobjlist(class))) {
+ slab_nextptr_panic(sp, m_cobjlist(class));
+ /* NOTREACHED */
+ }
+ (*list)->obj_next = NULL;
+ list = *plist = &(*list)->obj_next;
+
+ if (--need == 0) {
+ break;
+ }
+ }
+ m_infree(class) -= (num - need);
+
+ return num - need;
+}
+
+/*
+ * Place object(s) back into a composite class's freelist.
+ */
+static unsigned int
+cslab_free(mbuf_class_t class, mcache_obj_t *list, int purged)
+{
+ mcache_obj_t *o, *tail;
+ unsigned int num = 0;
+ struct mbuf *m, *ms;
+ mcache_audit_t *mca = NULL;
+ mcache_obj_t *ref_list = NULL;
+ mcl_slab_t *clsp, *nsp;
+ void *cl;
+ mbuf_class_t cl_class;
+
+ ASSERT(MBUF_CLASS_VALID(class) && MBUF_CLASS_COMPOSITE(class));
+ VERIFY(class != MC_MBUF_16KCL || njcl > 0);
+ LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
+
+ if (class == MC_MBUF_CL) {
+ cl_class = MC_CL;
+ } else if (class == MC_MBUF_BIGCL) {
+ cl_class = MC_BIGCL;
+ } else {
+ VERIFY(class == MC_MBUF_16KCL);
+ cl_class = MC_16KCL;
+ }
+
+ o = tail = list;
+
+ while ((m = ms = (struct mbuf *)o) != NULL) {
+ mcache_obj_t *rfa, *nexto = o->obj_next;
+
+ /* Do the mbuf sanity checks */
+ if (mclaudit != NULL) {
+ mca = mcl_audit_buf2mca(MC_MBUF, (mcache_obj_t *)m);
+ if (mclverify) {
+ mcache_audit_free_verify(mca, m, 0,
+ m_maxsize(MC_MBUF));
+ }
+ ms = MCA_SAVED_MBUF_PTR(mca);
+ }
+
+ /* Do the cluster sanity checks */
+ cl = ms->m_ext.ext_buf;
+ clsp = slab_get(cl);
+ if (mclverify) {
+ size_t size = m_maxsize(cl_class);
+ mcache_audit_free_verify(mcl_audit_buf2mca(cl_class,
+ (mcache_obj_t *)cl), cl, 0, size);
+ }
+ VERIFY(ms->m_type == MT_FREE);
+ VERIFY(ms->m_flags == M_EXT);
+ VERIFY(m_get_rfa(ms) != NULL && MBUF_IS_COMPOSITE(ms));
+ if (cl_class == MC_CL) {
+ VERIFY(clsp->sl_refcnt >= 1 &&
+ clsp->sl_refcnt <= NCLPG);
+ } else {
+ VERIFY(clsp->sl_refcnt >= 1 &&
+ clsp->sl_refcnt <= NBCLPG);
+ }
+ if (cl_class == MC_16KCL) {
+ int k;
+ for (nsp = clsp, k = 1; k < NSLABSP16KB; k++) {
+ nsp = nsp->sl_next;
+ /* Next slab must already be present */
+ VERIFY(nsp != NULL);
+ VERIFY(nsp->sl_refcnt == 1);
+ }
+ }
+
+ /*
+ * If we're asked to purge, restore the actual mbuf using
+ * contents of the shadow structure (if auditing is enabled)
+ * and clear EXTF_COMPOSITE flag from the mbuf, as we are
+ * about to free it and the attached cluster into their caches.
+ */
+ if (purged) {
+ /* Restore constructed mbuf fields */
+ if (mclaudit != NULL) {
+ mcl_audit_restore_mbuf(m, mca, TRUE);
+ }
+
+ MEXT_MINREF(m) = 0;
+ MEXT_REF(m) = 0;
+ MEXT_PREF(m) = 0;
+ MEXT_FLAGS(m) = 0;
+ MEXT_PRIV(m) = 0;
+ MEXT_PMBUF(m) = NULL;
+ MEXT_TOKEN(m) = 0;
+
+ rfa = (mcache_obj_t *)(void *)m_get_rfa(m);
+ m_set_ext(m, NULL, NULL, NULL);
+ rfa->obj_next = ref_list;
+ ref_list = rfa;
+
+ m->m_type = MT_FREE;
+ m->m_flags = m->m_len = 0;
+ m->m_next = m->m_nextpkt = NULL;
+
+ /* Save mbuf fields and make auditing happy */
+ if (mclaudit != NULL) {
+ mcl_audit_mbuf(mca, o, FALSE, FALSE);
+ }
+
+ VERIFY(m_total(class) > 0);
+ m_total(class)--;
+
+ /* Free the mbuf */
+ o->obj_next = NULL;
+ slab_free(MC_MBUF, o);
+
+ /* And free the cluster */
+ ((mcache_obj_t *)cl)->obj_next = NULL;
+ if (class == MC_MBUF_CL) {
+ slab_free(MC_CL, cl);
+ } else if (class == MC_MBUF_BIGCL) {
+ slab_free(MC_BIGCL, cl);
+ } else {
+ slab_free(MC_16KCL, cl);
+ }
+ }
+
+ ++num;
+ tail = o;
+ o = nexto;
+ }
+
+ if (!purged) {
+ tail->obj_next = m_cobjlist(class);
+ m_cobjlist(class) = list;
+ m_infree(class) += num;
+ } else if (ref_list != NULL) {
+ mcache_free_ext(ref_cache, ref_list);
+ }
+
+ return num;
+}
+
+/*
+ * Common allocator for composite objects called by the CPU cache layer
+ * during an allocation request whenever there is no available element in
+ * the bucket layer. It returns one or more composite elements from the
+ * appropriate global freelist. If the freelist is empty, it will attempt
+ * to obtain the rudimentary objects from their caches and construct them
+ * into composite mbuf + cluster objects.
+ */
+static unsigned int
+mbuf_cslab_alloc(void *arg, mcache_obj_t ***plist, unsigned int needed,
+ int wait)
+{
+ mbuf_class_t class = (mbuf_class_t)arg;
+ mbuf_class_t cl_class = 0;
+ unsigned int num = 0, cnum = 0, want = needed;
+ mcache_obj_t *ref_list = NULL;
+ mcache_obj_t *mp_list = NULL;
+ mcache_obj_t *clp_list = NULL;
+ mcache_obj_t **list;
+ struct ext_ref *rfa;
+ struct mbuf *m;
+ void *cl;
+
+ ASSERT(MBUF_CLASS_VALID(class) && MBUF_CLASS_COMPOSITE(class));
+ ASSERT(needed > 0);
+
+ VERIFY(class != MC_MBUF_16KCL || njcl > 0);
+
+ /* There should not be any slab for this class */
+ VERIFY(m_slab_cnt(class) == 0 &&
+ m_slablist(class).tqh_first == NULL &&
+ m_slablist(class).tqh_last == NULL);
+
+ lck_mtx_lock(mbuf_mlock);
+
+ /* Try using the freelist first */
+ num = cslab_alloc(class, plist, needed);
+ list = *plist;
+ if (num == needed) {
+ m_alloc_cnt(class) += num;
+ lck_mtx_unlock(mbuf_mlock);
+ return needed;
+ }
+
+ lck_mtx_unlock(mbuf_mlock);
+
+ /*
+ * We could not satisfy the request using the freelist alone;
+ * allocate from the appropriate rudimentary caches and use
+ * whatever we can get to construct the composite objects.
+ */
+ needed -= num;
+
+ /*
+ * Mark these allocation requests as coming from a composite cache.
+ * Also, if the caller is willing to be blocked, mark the request
+ * with MCR_FAILOK such that we don't end up sleeping at the mbuf
+ * slab layer waiting for the individual object when one or more
+ * of the already-constructed composite objects are available.
+ */
+ wait |= MCR_COMP;
+ if (!(wait & MCR_NOSLEEP)) {
+ wait |= MCR_FAILOK;
+ }
+
+ /* allocate mbufs */
+ needed = mcache_alloc_ext(m_cache(MC_MBUF), &mp_list, needed, wait);
+ if (needed == 0) {
+ ASSERT(mp_list == NULL);
+ goto fail;
+ }
+
+ /* allocate clusters */
+ if (class == MC_MBUF_CL) {
+ cl_class = MC_CL;
+ } else if (class == MC_MBUF_BIGCL) {
+ cl_class = MC_BIGCL;
+ } else {
+ VERIFY(class == MC_MBUF_16KCL);
+ cl_class = MC_16KCL;
+ }
+ needed = mcache_alloc_ext(m_cache(cl_class), &clp_list, needed, wait);
+ if (needed == 0) {
+ ASSERT(clp_list == NULL);
+ goto fail;
+ }
+
+ needed = mcache_alloc_ext(ref_cache, &ref_list, needed, wait);
+ if (needed == 0) {
+ ASSERT(ref_list == NULL);
+ goto fail;
+ }
+
+ /*
+ * By this time "needed" is MIN(mbuf, cluster, ref). Any left
+ * overs will get freed accordingly before we return to caller.
+ */
+ for (cnum = 0; cnum < needed; cnum++) {
+ struct mbuf *ms;
+
+ m = ms = (struct mbuf *)mp_list;
+ mp_list = mp_list->obj_next;
+
+ cl = clp_list;
+ clp_list = clp_list->obj_next;
+ ((mcache_obj_t *)cl)->obj_next = NULL;
+
+ rfa = (struct ext_ref *)ref_list;
+ ref_list = ref_list->obj_next;
+ ((mcache_obj_t *)(void *)rfa)->obj_next = NULL;
+
+ /*
+ * If auditing is enabled, construct the shadow mbuf
+ * in the audit structure instead of in the actual one.
+ * mbuf_cslab_audit() will take care of restoring the
+ * contents after the integrity check.
+ */
+ if (mclaudit != NULL) {
+ mcache_audit_t *mca, *cl_mca;
+
+ lck_mtx_lock(mbuf_mlock);
+ mca = mcl_audit_buf2mca(MC_MBUF, (mcache_obj_t *)m);
+ ms = MCA_SAVED_MBUF_PTR(mca);
+ cl_mca = mcl_audit_buf2mca(cl_class,
+ (mcache_obj_t *)cl);
+
+ /*
+ * Pair them up. Note that this is done at the time
+ * the mbuf+cluster objects are constructed. This
+ * information should be treated as "best effort"
+ * debugging hint since more than one mbufs can refer
+ * to a cluster. In that case, the cluster might not
+ * be freed along with the mbuf it was paired with.
+ */
+ mca->mca_uptr = cl_mca;
+ cl_mca->mca_uptr = mca;
+
+ ASSERT(mca->mca_uflags & MB_SCVALID);
+ ASSERT(!(cl_mca->mca_uflags & MB_SCVALID));
+ lck_mtx_unlock(mbuf_mlock);
+
+ /* Technically, they are in the freelist */
+ if (mclverify) {
+ size_t size;
+
+ mcache_set_pattern(MCACHE_FREE_PATTERN, m,
+ m_maxsize(MC_MBUF));
+
+ if (class == MC_MBUF_CL) {
+ size = m_maxsize(MC_CL);
+ } else if (class == MC_MBUF_BIGCL) {
+ size = m_maxsize(MC_BIGCL);
+ } else {
+ size = m_maxsize(MC_16KCL);
+ }
+
+ mcache_set_pattern(MCACHE_FREE_PATTERN, cl,
+ size);
+ }
+ }
+
+ MBUF_INIT(ms, 0, MT_FREE);
+ if (class == MC_MBUF_16KCL) {
+ MBUF_16KCL_INIT(ms, cl, rfa, 0, EXTF_COMPOSITE);
+ } else if (class == MC_MBUF_BIGCL) {
+ MBUF_BIGCL_INIT(ms, cl, rfa, 0, EXTF_COMPOSITE);
+ } else {
+ MBUF_CL_INIT(ms, cl, rfa, 0, EXTF_COMPOSITE);
+ }
+ VERIFY(ms->m_flags == M_EXT);
+ VERIFY(m_get_rfa(ms) != NULL && MBUF_IS_COMPOSITE(ms));
+
+ *list = (mcache_obj_t *)m;
+ (*list)->obj_next = NULL;
+ list = *plist = &(*list)->obj_next;
+ }
+
+fail:
+ /*
+ * Free up what's left of the above.
+ */
+ if (mp_list != NULL) {
+ mcache_free_ext(m_cache(MC_MBUF), mp_list);
+ }
+ if (clp_list != NULL) {
+ mcache_free_ext(m_cache(cl_class), clp_list);
+ }
+ if (ref_list != NULL) {
+ mcache_free_ext(ref_cache, ref_list);
+ }
+
+ lck_mtx_lock(mbuf_mlock);
+ if (num > 0 || cnum > 0) {
+ m_total(class) += cnum;
+ VERIFY(m_total(class) <= m_maxlimit(class));
+ m_alloc_cnt(class) += num + cnum;
+ }
+ if ((num + cnum) < want) {
+ m_fail_cnt(class) += (want - (num + cnum));
+ }
+ lck_mtx_unlock(mbuf_mlock);
+
+ return num + cnum;
+}
+
+/*
+ * Common de-allocator for composite objects called by the CPU cache
+ * layer when one or more elements need to be returned to the appropriate
+ * global freelist.
+ */
+static void
+mbuf_cslab_free(void *arg, mcache_obj_t *list, int purged)
+{
+ mbuf_class_t class = (mbuf_class_t)arg;
+ unsigned int num;
+ int w;
+
+ ASSERT(MBUF_CLASS_VALID(class) && MBUF_CLASS_COMPOSITE(class));
+
+ lck_mtx_lock(mbuf_mlock);
+
+ num = cslab_free(class, list, purged);
+ m_free_cnt(class) += num;
+
+ if ((w = mb_waiters) > 0) {
+ mb_waiters = 0;
+ }
+ if (w) {
+ mbwdog_logger("waking up all threads");
+ }
+
+ lck_mtx_unlock(mbuf_mlock);
+
+ if (w != 0) {
+ wakeup(mb_waitchan);
+ }
+}
+
+/*
+ * Common auditor for composite objects called by the CPU cache layer
+ * during an allocation or free request. For the former, this is called
+ * after the objects are obtained from either the bucket or slab layer
+ * and before they are returned to the caller. For the latter, this is
+ * called immediately during free and before placing the objects into
+ * the bucket or slab layer.
+ */
+static void
+mbuf_cslab_audit(void *arg, mcache_obj_t *list, boolean_t alloc)
+{
+ mbuf_class_t class = (mbuf_class_t)arg, cl_class;
+ mcache_audit_t *mca;
+ struct mbuf *m, *ms;
+ mcl_slab_t *clsp, *nsp;
+ size_t cl_size;
+ void *cl;
+
+ ASSERT(MBUF_CLASS_VALID(class) && MBUF_CLASS_COMPOSITE(class));
+ if (class == MC_MBUF_CL) {
+ cl_class = MC_CL;
+ } else if (class == MC_MBUF_BIGCL) {
+ cl_class = MC_BIGCL;
+ } else {
+ cl_class = MC_16KCL;
+ }
+ cl_size = m_maxsize(cl_class);
+
+ while ((m = ms = (struct mbuf *)list) != NULL) {
+ lck_mtx_lock(mbuf_mlock);
+ /* Do the mbuf sanity checks and record its transaction */
+ mca = mcl_audit_buf2mca(MC_MBUF, (mcache_obj_t *)m);
+ mcl_audit_mbuf(mca, m, TRUE, alloc);
+ if (mcltrace) {
+ mcache_buffer_log(mca, m, m_cache(class), &mb_start);
+ }
+
+ if (alloc) {
+ mca->mca_uflags |= MB_COMP_INUSE;
+ } else {
+ mca->mca_uflags &= ~MB_COMP_INUSE;
+ }
+
+ /*
+ * Use the shadow mbuf in the audit structure if we are
+ * freeing, since the contents of the actual mbuf has been
+ * pattern-filled by the above call to mcl_audit_mbuf().
+ */
+ if (!alloc && mclverify) {
+ ms = MCA_SAVED_MBUF_PTR(mca);
+ }
+
+ /* Do the cluster sanity checks and record its transaction */
+ cl = ms->m_ext.ext_buf;
+ clsp = slab_get(cl);
+ VERIFY(ms->m_flags == M_EXT && cl != NULL);
+ VERIFY(m_get_rfa(ms) != NULL && MBUF_IS_COMPOSITE(ms));
+ if (class == MC_MBUF_CL) {
+ VERIFY(clsp->sl_refcnt >= 1 &&
+ clsp->sl_refcnt <= NCLPG);
+ } else {
+ VERIFY(clsp->sl_refcnt >= 1 &&
+ clsp->sl_refcnt <= NBCLPG);
+ }
+
+ if (class == MC_MBUF_16KCL) {
+ int k;
+ for (nsp = clsp, k = 1; k < NSLABSP16KB; k++) {
+ nsp = nsp->sl_next;
+ /* Next slab must already be present */
+ VERIFY(nsp != NULL);
+ VERIFY(nsp->sl_refcnt == 1);
+ }
+ }
+
+
+ mca = mcl_audit_buf2mca(cl_class, cl);
+ mcl_audit_cluster(mca, cl, cl_size, alloc, FALSE);
+ if (mcltrace) {
+ mcache_buffer_log(mca, cl, m_cache(class), &mb_start);
+ }
+
+ if (alloc) {
+ mca->mca_uflags |= MB_COMP_INUSE;
+ } else {
+ mca->mca_uflags &= ~MB_COMP_INUSE;
+ }
+ lck_mtx_unlock(mbuf_mlock);
+
+ list = list->obj_next;
+ }
+}
+
+static void
+m_vm_error_stats(uint32_t *cnt, uint64_t *ts, uint64_t *size,
+ uint64_t alloc_size, kern_return_t error)
+{
+ *cnt = *cnt + 1;
+ *ts = net_uptime();
+ if (size) {
+ *size = alloc_size;
+ }
+ _CASSERT(sizeof(mb_kmem_stats) / sizeof(mb_kmem_stats[0]) ==
+ sizeof(mb_kmem_stats_labels) / sizeof(mb_kmem_stats_labels[0]));
+ switch (error) {
+ case KERN_SUCCESS:
+ break;
+ case KERN_INVALID_ARGUMENT:
+ mb_kmem_stats[0]++;
+ break;
+ case KERN_INVALID_ADDRESS:
+ mb_kmem_stats[1]++;
+ break;
+ case KERN_RESOURCE_SHORTAGE:
+ mb_kmem_stats[2]++;
+ break;
+ case KERN_NO_SPACE:
+ mb_kmem_stats[3]++;
+ break;
+ case KERN_FAILURE:
+ mb_kmem_stats[4]++;
+ break;
+ default:
+ mb_kmem_stats[5]++;
+ break;
+ }
+}
+
+/*
+ * Allocate some number of mbuf clusters and place on cluster freelist.
+ */
+static int
+m_clalloc(const u_int32_t num, const int wait, const u_int32_t bufsize)
+{
+ int i, count = 0;
+ vm_size_t size = 0;
+ int numpages = 0, large_buffer;
+ vm_offset_t page = 0;
+ mcache_audit_t *mca_list = NULL;
+ mcache_obj_t *con_list = NULL;
+ mcl_slab_t *sp;
+ mbuf_class_t class;
+ kern_return_t error;
+
+ /* Set if a buffer allocation needs allocation of multiple pages */
+ large_buffer = ((bufsize == m_maxsize(MC_16KCL)) &&
+ PAGE_SIZE < M16KCLBYTES);
+ VERIFY(bufsize == m_maxsize(MC_BIGCL) ||
+ bufsize == m_maxsize(MC_16KCL));
+
+ VERIFY((bufsize == PAGE_SIZE) ||
+ (bufsize > PAGE_SIZE && bufsize == m_maxsize(MC_16KCL)));
+
+ if (bufsize == m_size(MC_BIGCL)) {
+ class = MC_BIGCL;
+ } else {
+ class = MC_16KCL;
+ }
+
+ LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
+
+ /*
+ * Multiple threads may attempt to populate the cluster map one
+ * after another. Since we drop the lock below prior to acquiring
+ * the physical page(s), our view of the cluster map may no longer
+ * be accurate, and we could end up over-committing the pages beyond
+ * the maximum allowed for each class. To prevent it, this entire
+ * operation (including the page mapping) is serialized.
+ */
+ while (mb_clalloc_busy) {
+ mb_clalloc_waiters++;
+ (void) msleep(mb_clalloc_waitchan, mbuf_mlock,
+ (PZERO - 1), "m_clalloc", NULL);
+ LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
+ }
+
+ /* We are busy now; tell everyone else to go away */
+ mb_clalloc_busy = TRUE;
+
+ /*
+ * Honor the caller's wish to block or not block. We have a way
+ * to grow the pool asynchronously using the mbuf worker thread.
+ */
+ i = m_howmany(num, bufsize);
+ if (i <= 0 || (wait & M_DONTWAIT)) {
+ goto out;
+ }
+
+ lck_mtx_unlock(mbuf_mlock);
+
+ size = round_page(i * bufsize);
+ page = kmem_mb_alloc(mb_map, size, large_buffer, &error);
+
+ /*
+ * If we did ask for "n" 16KB physically contiguous chunks
+ * and didn't get them, then please try again without this
+ * restriction.
+ */
+ net_update_uptime();
+ if (large_buffer && page == 0) {
+ m_vm_error_stats(&mb_kmem_contig_failed,
+ &mb_kmem_contig_failed_ts,
+ &mb_kmem_contig_failed_size,
+ size, error);
+ page = kmem_mb_alloc(mb_map, size, 0, &error);
+ }
+
+ if (page == 0) {
+ m_vm_error_stats(&mb_kmem_failed,
+ &mb_kmem_failed_ts,
+ &mb_kmem_failed_size,
+ size, error);
+#if PAGE_SIZE == 4096
+ if (bufsize == m_maxsize(MC_BIGCL)) {
+#else
+ if (bufsize >= m_maxsize(MC_BIGCL)) {
+#endif
+ /* Try for 1 page if failed */
+ size = PAGE_SIZE;
+ page = kmem_mb_alloc(mb_map, size, 0, &error);
+ if (page == 0) {
+ m_vm_error_stats(&mb_kmem_one_failed,
+ &mb_kmem_one_failed_ts,
+ NULL, size, error);
+ }
+ }
+
+ if (page == 0) {
+ lck_mtx_lock(mbuf_mlock);
+ goto out;
+ }
+ }
+
+ VERIFY(IS_P2ALIGNED(page, PAGE_SIZE));
+ numpages = size / PAGE_SIZE;
+
+ /* If auditing is enabled, allocate the audit structures now */
+ if (mclaudit != NULL) {
+ int needed;
+
+ /*
+ * Yes, I realize this is a waste of memory for clusters
+ * that never get transformed into mbufs, as we may end
+ * up with NMBPG-1 unused audit structures per cluster.
+ * But doing so tremendously simplifies the allocation
+ * strategy, since at this point we are not holding the
+ * mbuf lock and the caller is okay to be blocked.
+ */
+ if (bufsize == PAGE_SIZE) {
+ needed = numpages * NMBPG;
+
+ i = mcache_alloc_ext(mcl_audit_con_cache,
+ &con_list, needed, MCR_SLEEP);
+
+ VERIFY(con_list != NULL && i == needed);
+ } else {
+ /*
+ * if multiple 4K pages are being used for a
+ * 16K cluster
+ */
+ needed = numpages / NSLABSP16KB;
+ }
+
+ i = mcache_alloc_ext(mcache_audit_cache,
+ (mcache_obj_t **)&mca_list, needed, MCR_SLEEP);
+
+ VERIFY(mca_list != NULL && i == needed);
+ }
+
+ lck_mtx_lock(mbuf_mlock);
+
+ for (i = 0; i < numpages; i++, page += PAGE_SIZE) {
+ ppnum_t offset =
+ ((unsigned char *)page - mbutl) >> PAGE_SHIFT;
+ ppnum_t new_page = pmap_find_phys(kernel_pmap, page);
+
+ /*
+ * If there is a mapper the appropriate I/O page is
+ * returned; zero out the page to discard its past
+ * contents to prevent exposing leftover kernel memory.
+ */
+ VERIFY(offset < mcl_pages);
+ if (mcl_paddr_base != 0) {
+ bzero((void *)(uintptr_t) page, PAGE_SIZE);
+ new_page = IOMapperInsertPage(mcl_paddr_base,
+ offset, new_page);
+ }
+ mcl_paddr[offset] = new_page;
+
+ /* Pattern-fill this fresh page */
+ if (mclverify) {
+ mcache_set_pattern(MCACHE_FREE_PATTERN,
+ (caddr_t)page, PAGE_SIZE);
+ }
+ if (bufsize == PAGE_SIZE) {
+ mcache_obj_t *buf;
+ /* One for the entire page */
+ sp = slab_get((void *)page);
+ if (mclaudit != NULL) {
+ mcl_audit_init((void *)page,
+ &mca_list, &con_list,
+ AUDIT_CONTENTS_SIZE, NMBPG);
+ }
+ VERIFY(sp->sl_refcnt == 0 && sp->sl_flags == 0);
+ slab_init(sp, class, SLF_MAPPED, (void *)page,
+ (void *)page, PAGE_SIZE, 0, 1);
+ buf = (mcache_obj_t *)page;
+ buf->obj_next = NULL;
+
+ /* Insert this slab */
+ slab_insert(sp, class);
+
+ /* Update stats now since slab_get drops the lock */
+ ++m_infree(class);
+ ++m_total(class);
+ VERIFY(m_total(class) <= m_maxlimit(class));
+ if (class == MC_BIGCL) {
+ mbstat.m_bigclfree = m_infree(MC_BIGCL) +
+ m_infree(MC_MBUF_BIGCL);
+ mbstat.m_bigclusters = m_total(MC_BIGCL);
+ }
+ ++count;
+ } else if ((bufsize > PAGE_SIZE) &&
+ (i % NSLABSP16KB) == 0) {
+ union m16kcluster *m16kcl = (union m16kcluster *)page;
+ mcl_slab_t *nsp;
+ int k;
+
+ /* One for the entire 16KB */
+ sp = slab_get(m16kcl);
+ if (mclaudit != NULL) {
+ mcl_audit_init(m16kcl, &mca_list, NULL, 0, 1);
+ }
+
+ VERIFY(sp->sl_refcnt == 0 && sp->sl_flags == 0);
+ slab_init(sp, MC_16KCL, SLF_MAPPED,
+ m16kcl, m16kcl, bufsize, 0, 1);
+ m16kcl->m16kcl_next = NULL;
+
+ /*
+ * 2nd-Nth page's slab is part of the first one,
+ * where N is NSLABSP16KB.
+ */
+ for (k = 1; k < NSLABSP16KB; k++) {
+ nsp = slab_get(((union mbigcluster *)page) + k);
+ VERIFY(nsp->sl_refcnt == 0 &&
+ nsp->sl_flags == 0);
+ slab_init(nsp, MC_16KCL,
+ SLF_MAPPED | SLF_PARTIAL,
+ m16kcl, NULL, 0, 0, 0);
+ }
+ /* Insert this slab */
+ slab_insert(sp, MC_16KCL);
+
+ /* Update stats now since slab_get drops the lock */
+ ++m_infree(MC_16KCL);
+ ++m_total(MC_16KCL);
+ VERIFY(m_total(MC_16KCL) <= m_maxlimit(MC_16KCL));
+ ++count;
+ }
+ }
+ VERIFY(mca_list == NULL && con_list == NULL);
+
+ if (!mb_peak_newreport && mbuf_report_usage(class)) {
+ mb_peak_newreport = TRUE;
+ }
+
+ /* We're done; let others enter */
+ mb_clalloc_busy = FALSE;
+ if (mb_clalloc_waiters > 0) {
+ mb_clalloc_waiters = 0;
+ wakeup(mb_clalloc_waitchan);
+ }
+
+ return count;
+out:
+ LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
+
+ mtracelarge_register(size);
+
+ /* We're done; let others enter */
+ mb_clalloc_busy = FALSE;
+ if (mb_clalloc_waiters > 0) {
+ mb_clalloc_waiters = 0;
+ wakeup(mb_clalloc_waitchan);
+ }
+
+ /*
+ * When non-blocking we kick a thread if we have to grow the
+ * pool or if the number of free clusters is less than requested.
+ */
+ if (i > 0 && mbuf_worker_ready && mbuf_worker_needs_wakeup) {
+ mbwdog_logger("waking up the worker thread to to grow %s by %d",
+ m_cname(class), i);
+ wakeup((caddr_t)&mbuf_worker_needs_wakeup);
+ mbuf_worker_needs_wakeup = FALSE;
+ }
+ if (class == MC_BIGCL) {
+ if (i > 0) {
+ /*
+ * Remember total number of 4KB clusters needed
+ * at this time.
+ */
+ i += m_total(MC_BIGCL);
+ if (i > m_region_expand(MC_BIGCL)) {
+ m_region_expand(MC_BIGCL) = i;
+ }
+ }
+ if (m_infree(MC_BIGCL) >= num) {
+ return 1;
+ }
+ } else {
+ if (i > 0) {
+ /*
+ * Remember total number of 16KB clusters needed
+ * at this time.
+ */
+ i += m_total(MC_16KCL);
+ if (i > m_region_expand(MC_16KCL)) {
+ m_region_expand(MC_16KCL) = i;
+ }
+ }
+ if (m_infree(MC_16KCL) >= num) {
+ return 1;
+ }
+ }
+ return 0;
+}
+
+/*
+ * Populate the global freelist of the corresponding buffer class.
+ */
+static int
+freelist_populate(mbuf_class_t class, unsigned int num, int wait)
+{
+ mcache_obj_t *o = NULL;
+ int i, numpages = 0, count;
+ mbuf_class_t super_class;
+
+ VERIFY(class == MC_MBUF || class == MC_CL || class == MC_BIGCL ||
+ class == MC_16KCL);
+
+ LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
+
+ VERIFY(PAGE_SIZE == m_maxsize(MC_BIGCL) ||
+ PAGE_SIZE == m_maxsize(MC_16KCL));
+
+ if (m_maxsize(class) >= PAGE_SIZE) {
+ return m_clalloc(num, wait, m_maxsize(class)) != 0;
+ }
+
+ /*
+ * The rest of the function will allocate pages and will slice
+ * them up into the right size
+ */
+
+ numpages = (num * m_size(class) + PAGE_SIZE - 1) / PAGE_SIZE;
+
+ /* Currently assume that pages are 4K or 16K */
+ if (PAGE_SIZE == m_maxsize(MC_BIGCL)) {
+ super_class = MC_BIGCL;
+ } else {
+ super_class = MC_16KCL;
+ }
+
+ i = m_clalloc(numpages, wait, m_maxsize(super_class));
+
+ /* how many objects will we cut the page into? */
+ int numobj = PAGE_SIZE / m_maxsize(class);
+
+ for (count = 0; count < numpages; count++) {
+ /* respect totals, minlimit, maxlimit */
+ if (m_total(super_class) <= m_minlimit(super_class) ||
+ m_total(class) >= m_maxlimit(class)) {
+ break;
+ }
+
+ if ((o = slab_alloc(super_class, wait)) == NULL) {
+ break;
+ }
+
+ struct mbuf *m = (struct mbuf *)o;
+ union mcluster *c = (union mcluster *)o;
+ union mbigcluster *mbc = (union mbigcluster *)o;
+ mcl_slab_t *sp = slab_get(o);
+ mcache_audit_t *mca = NULL;
+
+ /*
+ * since one full page will be converted to MC_MBUF or
+ * MC_CL, verify that the reference count will match that
+ * assumption
+ */
+ VERIFY(sp->sl_refcnt == 1 && slab_is_detached(sp));
+ VERIFY((sp->sl_flags & (SLF_MAPPED | SLF_PARTIAL)) == SLF_MAPPED);
+ /*
+ * Make sure that the cluster is unmolested
+ * while in freelist
+ */
+ if (mclverify) {
+ mca = mcl_audit_buf2mca(super_class,
+ (mcache_obj_t *)o);
+ mcache_audit_free_verify(mca,
+ (mcache_obj_t *)o, 0, m_maxsize(super_class));
+ }
+
+ /* Reinitialize it as an mbuf or 2K or 4K slab */
+ slab_init(sp, class, sp->sl_flags,
+ sp->sl_base, NULL, PAGE_SIZE, 0, numobj);
+
+ VERIFY(sp->sl_head == NULL);
+
+ VERIFY(m_total(super_class) >= 1);
+ m_total(super_class)--;
+
+ if (super_class == MC_BIGCL) {
+ mbstat.m_bigclusters = m_total(MC_BIGCL);
+ }
+
+ m_total(class) += numobj;
+ VERIFY(m_total(class) <= m_maxlimit(class));
+ m_infree(class) += numobj;
+
+ if (!mb_peak_newreport && mbuf_report_usage(class)) {
+ mb_peak_newreport = TRUE;
+ }
+
+ i = numobj;
+ if (class == MC_MBUF) {
+ mbstat.m_mbufs = m_total(MC_MBUF);
+ mtype_stat_add(MT_FREE, NMBPG);
+ while (i--) {
+ /*
+ * If auditing is enabled, construct the
+ * shadow mbuf in the audit structure
+ * instead of the actual one.
+ * mbuf_slab_audit() will take care of
+ * restoring the contents after the
+ * integrity check.
+ */
+ if (mclaudit != NULL) {
+ struct mbuf *ms;
+ mca = mcl_audit_buf2mca(MC_MBUF,
+ (mcache_obj_t *)m);
+ ms = MCA_SAVED_MBUF_PTR(mca);
+ ms->m_type = MT_FREE;
+ } else {
+ m->m_type = MT_FREE;
+ }
+ m->m_next = sp->sl_head;
+ sp->sl_head = (void *)m++;
+ }
+ } else if (class == MC_CL) { /* MC_CL */
+ mbstat.m_clfree =
+ m_infree(MC_CL) + m_infree(MC_MBUF_CL);
+ mbstat.m_clusters = m_total(MC_CL);
+ while (i--) {
+ c->mcl_next = sp->sl_head;
+ sp->sl_head = (void *)c++;
+ }
+ } else {
+ VERIFY(class == MC_BIGCL);
+ mbstat.m_bigclusters = m_total(MC_BIGCL);
+ mbstat.m_bigclfree = m_infree(MC_BIGCL) +
+ m_infree(MC_MBUF_BIGCL);
+ while (i--) {
+ mbc->mbc_next = sp->sl_head;
+ sp->sl_head = (void *)mbc++;
+ }
+ }
+
+ /* Insert into the mbuf or 2k or 4k slab list */
+ slab_insert(sp, class);
+
+ if ((i = mb_waiters) > 0) {
+ mb_waiters = 0;
+ }
+ if (i != 0) {
+ mbwdog_logger("waking up all threads");
+ wakeup(mb_waitchan);
+ }
+ }
+ return count != 0;
+}
+
+/*
+ * For each class, initialize the freelist to hold m_minlimit() objects.
+ */
+static void
+freelist_init(mbuf_class_t class)
+{
+ LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
+
+ VERIFY(class == MC_CL || class == MC_BIGCL);
+ VERIFY(m_total(class) == 0);
+ VERIFY(m_minlimit(class) > 0);
+
+ while (m_total(class) < m_minlimit(class)) {
+ (void) freelist_populate(class, m_minlimit(class), M_WAIT);
+ }
+
+ VERIFY(m_total(class) >= m_minlimit(class));
+}
+
+/*
+ * (Inaccurately) check if it might be worth a trip back to the
+ * mcache layer due the availability of objects there. We'll
+ * end up back here if there's nothing up there.
+ */
+static boolean_t
+mbuf_cached_above(mbuf_class_t class, int wait)
+{
+ switch (class) {
+ case MC_MBUF:
+ if (wait & MCR_COMP) {
+ return !mcache_bkt_isempty(m_cache(MC_MBUF_CL)) ||
+ !mcache_bkt_isempty(m_cache(MC_MBUF_BIGCL));
+ }
+ break;
+
+ case MC_CL:
+ if (wait & MCR_COMP) {
+ return !mcache_bkt_isempty(m_cache(MC_MBUF_CL));
+ }
+ break;
+
+ case MC_BIGCL:
+ if (wait & MCR_COMP) {
+ return !mcache_bkt_isempty(m_cache(MC_MBUF_BIGCL));
+ }
+ break;
+
+ case MC_16KCL:
+ if (wait & MCR_COMP) {
+ return !mcache_bkt_isempty(m_cache(MC_MBUF_16KCL));
+ }
+ break;
+
+ case MC_MBUF_CL:
+ case MC_MBUF_BIGCL:
+ case MC_MBUF_16KCL:
+ break;
+
+ default:
+ VERIFY(0);
+ /* NOTREACHED */
+ }
+
+ return !mcache_bkt_isempty(m_cache(class));
+}
+
+/*
+ * If possible, convert constructed objects to raw ones.
+ */
+static boolean_t
+mbuf_steal(mbuf_class_t class, unsigned int num)
+{
+ mcache_obj_t *top = NULL;
+ mcache_obj_t **list = ⊤
+ unsigned int tot = 0;
+
+ LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
+
+ switch (class) {
+ case MC_MBUF:
+ case MC_CL:
+ case MC_BIGCL:
+ case MC_16KCL:
+ return FALSE;
+
+ case MC_MBUF_CL:
+ case MC_MBUF_BIGCL:
+ case MC_MBUF_16KCL:
+ /* Get the required number of constructed objects if possible */
+ if (m_infree(class) > m_minlimit(class)) {
+ tot = cslab_alloc(class, &list,
+ MIN(num, m_infree(class)));
+ }
+
+ /* And destroy them to get back the raw objects */
+ if (top != NULL) {
+ (void) cslab_free(class, top, 1);
+ }
+ break;
+
+ default:
+ VERIFY(0);
+ /* NOTREACHED */
+ }
+
+ return tot == num;
+}
+
+static void
+m_reclaim(mbuf_class_t class, unsigned int num, boolean_t comp)
+{
+ int m, bmap = 0;
+
+ LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
+
+ VERIFY(m_total(MC_CL) <= m_maxlimit(MC_CL));
+ VERIFY(m_total(MC_BIGCL) <= m_maxlimit(MC_BIGCL));
+ VERIFY(m_total(MC_16KCL) <= m_maxlimit(MC_16KCL));
+
+ /*
+ * This logic can be made smarter; for now, simply mark
+ * all other related classes as potential victims.
+ */
+ switch (class) {
+ case MC_MBUF:
+ m_wantpurge(MC_CL)++;
+ m_wantpurge(MC_BIGCL)++;
+ m_wantpurge(MC_MBUF_CL)++;
+ m_wantpurge(MC_MBUF_BIGCL)++;
+ break;
+
+ case MC_CL:
+ m_wantpurge(MC_MBUF)++;
+ m_wantpurge(MC_BIGCL)++;
+ m_wantpurge(MC_MBUF_BIGCL)++;
+ if (!comp) {
+ m_wantpurge(MC_MBUF_CL)++;
+ }
+ break;
+
+ case MC_BIGCL:
+ m_wantpurge(MC_MBUF)++;
+ m_wantpurge(MC_CL)++;
+ m_wantpurge(MC_MBUF_CL)++;
+ if (!comp) {
+ m_wantpurge(MC_MBUF_BIGCL)++;
+ }
+ break;
+
+ case MC_16KCL:
+ if (!comp) {
+ m_wantpurge(MC_MBUF_16KCL)++;
+ }
+ break;
+
+ default:
+ VERIFY(0);
+ /* NOTREACHED */
+ }
+
+ /*
+ * Run through each marked class and check if we really need to
+ * purge (and therefore temporarily disable) the per-CPU caches
+ * layer used by the class. If so, remember the classes since
+ * we are going to drop the lock below prior to purging.
+ */
+ for (m = 0; m < NELEM(mbuf_table); m++) {
+ if (m_wantpurge(m) > 0) {
+ m_wantpurge(m) = 0;
+ /*
+ * Try hard to steal the required number of objects
+ * from the freelist of other mbuf classes. Only
+ * purge and disable the per-CPU caches layer when
+ * we don't have enough; it's the last resort.
+ */
+ if (!mbuf_steal(m, num)) {
+ bmap |= (1 << m);
+ }
+ }
+ }
+
+ lck_mtx_unlock(mbuf_mlock);
+
+ if (bmap != 0) {
+ /* signal the domains to drain */
+ net_drain_domains();
+
+ /* Sigh; we have no other choices but to ask mcache to purge */
+ for (m = 0; m < NELEM(mbuf_table); m++) {
+ if ((bmap & (1 << m)) &&
+ mcache_purge_cache(m_cache(m), TRUE)) {
+ lck_mtx_lock(mbuf_mlock);
+ m_purge_cnt(m)++;
+ mbstat.m_drain++;
+ lck_mtx_unlock(mbuf_mlock);
+ }
+ }
+ } else {
+ /*
+ * Request mcache to reap extra elements from all of its caches;
+ * note that all reaps are serialized and happen only at a fixed
+ * interval.
+ */
+ mcache_reap();
+ }
+ lck_mtx_lock(mbuf_mlock);
+}
+
+static inline struct mbuf *
+m_get_common(int wait, short type, int hdr)
+{
+ struct mbuf *m;
+ int mcflags = MSLEEPF(wait);
+
+ /* Is this due to a non-blocking retry? If so, then try harder */
+ if (mcflags & MCR_NOSLEEP) {
+ mcflags |= MCR_TRYHARD;
+ }
+
+ m = mcache_alloc(m_cache(MC_MBUF), mcflags);
+ if (m != NULL) {
+ MBUF_INIT(m, hdr, type);
+ mtype_stat_inc(type);
+ mtype_stat_dec(MT_FREE);
+#if CONFIG_MACF_NET
+ if (hdr && mac_init_mbuf(m, wait) != 0) {
+ m_free(m);
+ return NULL;
+ }
+#endif /* MAC_NET */
+ }
+ return m;
+}
+
+/*
+ * Space allocation routines; these are also available as macros
+ * for critical paths.
+ */
+#define _M_GET(wait, type) m_get_common(wait, type, 0)
+#define _M_GETHDR(wait, type) m_get_common(wait, type, 1)
+#define _M_RETRY(wait, type) _M_GET(wait, type)
+#define _M_RETRYHDR(wait, type) _M_GETHDR(wait, type)
+#define _MGET(m, how, type) ((m) = _M_GET(how, type))
+#define _MGETHDR(m, how, type) ((m) = _M_GETHDR(how, type))
+
+struct mbuf *
+m_get(int wait, int type)
+{
+ return _M_GET(wait, type);
+}
+
+struct mbuf *
+m_gethdr(int wait, int type)
+{
+ return _M_GETHDR(wait, type);
+}
+
+struct mbuf *
+m_retry(int wait, int type)
+{
+ return _M_RETRY(wait, type);
+}
+
+struct mbuf *
+m_retryhdr(int wait, int type)
+{
+ return _M_RETRYHDR(wait, type);
+}
+
+struct mbuf *
+m_getclr(int wait, int type)
+{
+ struct mbuf *m;
+
+ _MGET(m, wait, type);
+ if (m != NULL) {
+ bzero(MTOD(m, caddr_t), MLEN);
+ }
+ return m;
+}
+
+static int
+m_free_paired(struct mbuf *m)
+{
+ VERIFY((m->m_flags & M_EXT) && (MEXT_FLAGS(m) & EXTF_PAIRED));
+
+ membar_sync();
+ if (MEXT_PMBUF(m) == m) {
+ volatile UInt16 *addr = (volatile UInt16 *)&MEXT_PREF(m);
+ int16_t oprefcnt, prefcnt;
+
+ /*
+ * Paired ref count might be negative in case we lose
+ * against another thread clearing MEXT_PMBUF, in the
+ * event it occurs after the above memory barrier sync.
+ * In that case just ignore as things have been unpaired.
+ */
+ do {
+ oprefcnt = *addr;
+ prefcnt = oprefcnt - 1;
+ } while (!OSCompareAndSwap16(oprefcnt, prefcnt, addr));
+
+ if (prefcnt > 1) {
+ return 1;
+ } else if (prefcnt == 1) {
+ (*(m_get_ext_free(m)))(m->m_ext.ext_buf,
+ m->m_ext.ext_size, m_get_ext_arg(m));
+ return 1;
+ } else if (prefcnt == 0) {
+ VERIFY(MBUF_IS_PAIRED(m));
+
+ /*
+ * Restore minref to its natural value, so that
+ * the caller will be able to free the cluster
+ * as appropriate.
+ */
+ MEXT_MINREF(m) = 0;
+
+ /*
+ * Clear MEXT_PMBUF, but leave EXTF_PAIRED intact
+ * as it is immutable. atomic_set_ptr also causes
+ * memory barrier sync.
+ */
+ atomic_set_ptr(&MEXT_PMBUF(m), NULL);
+
+ switch (m->m_ext.ext_size) {
+ case MCLBYTES:
+ m_set_ext(m, m_get_rfa(m), NULL, NULL);
+ break;
+
+ case MBIGCLBYTES:
+ m_set_ext(m, m_get_rfa(m), m_bigfree, NULL);
+ break;
+
+ case M16KCLBYTES:
+ m_set_ext(m, m_get_rfa(m), m_16kfree, NULL);
+ break;
+
+ default:
+ VERIFY(0);
+ /* NOTREACHED */
+ }
+ }
+ }
+
+ /*
+ * Tell caller the unpair has occurred, and that the reference
+ * count on the external cluster held for the paired mbuf should
+ * now be dropped.
+ */
+ return 0;
+}
+
+struct mbuf *
+m_free(struct mbuf *m)
+{
+ struct mbuf *n = m->m_next;
+
+ if (m->m_type == MT_FREE) {
+ panic("m_free: freeing an already freed mbuf");
+ }
+
+ if (m->m_flags & M_PKTHDR) {
+ /* Check for scratch area overflow */
+ m_redzone_verify(m);
+ /* Free the aux data and tags if there is any */
+ m_tag_delete_chain(m, NULL);
+
+ m_do_tx_compl_callback(m, NULL);
+ }
+
+ if (m->m_flags & M_EXT) {
+ u_int16_t refcnt;
+ u_int32_t composite;
+ m_ext_free_func_t m_free_func;
+
+ if (MBUF_IS_PAIRED(m) && m_free_paired(m)) {
+ return n;
+ }
+
+ refcnt = m_decref(m);
+ composite = (MEXT_FLAGS(m) & EXTF_COMPOSITE);
+ m_free_func = m_get_ext_free(m);
+
+ if (refcnt == MEXT_MINREF(m) && !composite) {
+ if (m_free_func == NULL) {
+ mcache_free(m_cache(MC_CL), m->m_ext.ext_buf);
+ } else if (m_free_func == m_bigfree) {
+ mcache_free(m_cache(MC_BIGCL),
+ m->m_ext.ext_buf);
+ } else if (m_free_func == m_16kfree) {
+ mcache_free(m_cache(MC_16KCL),
+ m->m_ext.ext_buf);
+ } else {
+ (*m_free_func)(m->m_ext.ext_buf,
+ m->m_ext.ext_size, m_get_ext_arg(m));
+ }
+ mcache_free(ref_cache, m_get_rfa(m));
+ m_set_ext(m, NULL, NULL, NULL);
+ } else if (refcnt == MEXT_MINREF(m) && composite) {
+ VERIFY(!(MEXT_FLAGS(m) & EXTF_PAIRED));
+ VERIFY(m->m_type != MT_FREE);
+
+ mtype_stat_dec(m->m_type);
+ mtype_stat_inc(MT_FREE);
+
+ m->m_type = MT_FREE;
+ m->m_flags = M_EXT;
+ m->m_len = 0;
+ m->m_next = m->m_nextpkt = NULL;
+
+ MEXT_FLAGS(m) &= ~EXTF_READONLY;
+
+ /* "Free" into the intermediate cache */
+ if (m_free_func == NULL) {
+ mcache_free(m_cache(MC_MBUF_CL), m);
+ } else if (m_free_func == m_bigfree) {
+ mcache_free(m_cache(MC_MBUF_BIGCL), m);
+ } else {
+ VERIFY(m_free_func == m_16kfree);
+ mcache_free(m_cache(MC_MBUF_16KCL), m);
+ }
+ return n;
+ }
+ }
+
+ if (m->m_type != MT_FREE) {
+ mtype_stat_dec(m->m_type);
+ mtype_stat_inc(MT_FREE);
+ }
+
+ m->m_type = MT_FREE;
+ m->m_flags = m->m_len = 0;
+ m->m_next = m->m_nextpkt = NULL;
+
+ mcache_free(m_cache(MC_MBUF), m);
+
+ return n;
+}
+
+__private_extern__ struct mbuf *
+m_clattach(struct mbuf *m, int type, caddr_t extbuf,
+ void (*extfree)(caddr_t, u_int, caddr_t), u_int extsize, caddr_t extarg,
+ int wait, int pair)
+{
+ struct ext_ref *rfa = NULL;
+
+ /*
+ * If pairing is requested and an existing mbuf is provided, reject
+ * it if it's already been paired to another cluster. Otherwise,
+ * allocate a new one or free any existing below.
+ */
+ if ((m != NULL && MBUF_IS_PAIRED(m)) ||
+ (m == NULL && (m = _M_GETHDR(wait, type)) == NULL)) {
+ return NULL;
+ }
+
+ if (m->m_flags & M_EXT) {
+ u_int16_t refcnt;
+ u_int32_t composite;
+ m_ext_free_func_t m_free_func;
+
+ refcnt = m_decref(m);
+ composite = (MEXT_FLAGS(m) & EXTF_COMPOSITE);
+ VERIFY(!(MEXT_FLAGS(m) & EXTF_PAIRED) && MEXT_PMBUF(m) == NULL);
+ m_free_func = m_get_ext_free(m);
+ if (refcnt == MEXT_MINREF(m) && !composite) {
+ if (m_free_func == NULL) {
+ mcache_free(m_cache(MC_CL), m->m_ext.ext_buf);
+ } else if (m_free_func == m_bigfree) {
+ mcache_free(m_cache(MC_BIGCL),
+ m->m_ext.ext_buf);
+ } else if (m_free_func == m_16kfree) {
+ mcache_free(m_cache(MC_16KCL),
+ m->m_ext.ext_buf);
+ } else {
+ (*m_free_func)(m->m_ext.ext_buf,
+ m->m_ext.ext_size, m_get_ext_arg(m));
+ }
+ /* Re-use the reference structure */
+ rfa = m_get_rfa(m);
+ } else if (refcnt == MEXT_MINREF(m) && composite) {
+ VERIFY(m->m_type != MT_FREE);
+
+ mtype_stat_dec(m->m_type);
+ mtype_stat_inc(MT_FREE);
+
+ m->m_type = MT_FREE;
+ m->m_flags = M_EXT;
+ m->m_len = 0;
+ m->m_next = m->m_nextpkt = NULL;
+
+ MEXT_FLAGS(m) &= ~EXTF_READONLY;
+
+ /* "Free" into the intermediate cache */
+ if (m_free_func == NULL) {
+ mcache_free(m_cache(MC_MBUF_CL), m);
+ } else if (m_free_func == m_bigfree) {
+ mcache_free(m_cache(MC_MBUF_BIGCL), m);
+ } else {
+ VERIFY(m_free_func == m_16kfree);
+ mcache_free(m_cache(MC_MBUF_16KCL), m);
+ }
+ /*
+ * Allocate a new mbuf, since we didn't divorce
+ * the composite mbuf + cluster pair above.
+ */
+ if ((m = _M_GETHDR(wait, type)) == NULL) {
+ return NULL;
+ }
+ }
+ }
+
+ if (rfa == NULL &&
+ (rfa = mcache_alloc(ref_cache, MSLEEPF(wait))) == NULL) {
+ m_free(m);
+ return NULL;
+ }
+
+ if (!pair) {
+ MEXT_INIT(m, extbuf, extsize, extfree, extarg, rfa,
+ 0, 1, 0, 0, 0, NULL);
+ } else {
+ MEXT_INIT(m, extbuf, extsize, extfree, (caddr_t)m, rfa,
+ 1, 1, 1, EXTF_PAIRED, 0, m);
+ }
+
+ return m;
+}
+
+/*
+ * Perform `fast' allocation mbuf clusters from a cache of recently-freed
+ * clusters. (If the cache is empty, new clusters are allocated en-masse.)
+ */
+struct mbuf *
+m_getcl(int wait, int type, int flags)
+{
+ struct mbuf *m;
+ int mcflags = MSLEEPF(wait);
+ int hdr = (flags & M_PKTHDR);
+
+ /* Is this due to a non-blocking retry? If so, then try harder */
+ if (mcflags & MCR_NOSLEEP) {
+ mcflags |= MCR_TRYHARD;
+ }
+
+ m = mcache_alloc(m_cache(MC_MBUF_CL), mcflags);
+ if (m != NULL) {
+ u_int16_t flag;
+ struct ext_ref *rfa;
+ void *cl;
+
+ VERIFY(m->m_type == MT_FREE && m->m_flags == M_EXT);
+ cl = m->m_ext.ext_buf;
+ rfa = m_get_rfa(m);
+
+ ASSERT(cl != NULL && rfa != NULL);
+ VERIFY(MBUF_IS_COMPOSITE(m) && m_get_ext_free(m) == NULL);
+
+ flag = MEXT_FLAGS(m);
+
+ MBUF_INIT(m, hdr, type);
+ MBUF_CL_INIT(m, cl, rfa, 1, flag);
+
+ mtype_stat_inc(type);
+ mtype_stat_dec(MT_FREE);
+#if CONFIG_MACF_NET
+ if (hdr && mac_init_mbuf(m, wait) != 0) {
+ m_freem(m);
+ return NULL;
+ }
+#endif /* MAC_NET */
+ }
+ return m;
+}
+
+/* m_mclget() add an mbuf cluster to a normal mbuf */
+struct mbuf *
+m_mclget(struct mbuf *m, int wait)
+{
+ struct ext_ref *rfa;
+
+ if ((rfa = mcache_alloc(ref_cache, MSLEEPF(wait))) == NULL) {
+ return m;
+ }
+
+ m->m_ext.ext_buf = m_mclalloc(wait);
+ if (m->m_ext.ext_buf != NULL) {
+ MBUF_CL_INIT(m, m->m_ext.ext_buf, rfa, 1, 0);
+ } else {
+ mcache_free(ref_cache, rfa);
+ }
+ return m;
+}
+
+/* Allocate an mbuf cluster */
+caddr_t
+m_mclalloc(int wait)
+{
+ int mcflags = MSLEEPF(wait);
+
+ /* Is this due to a non-blocking retry? If so, then try harder */
+ if (mcflags & MCR_NOSLEEP) {
+ mcflags |= MCR_TRYHARD;
+ }
+
+ return mcache_alloc(m_cache(MC_CL), mcflags);
+}
+
+/* Free an mbuf cluster */
+void
+m_mclfree(caddr_t p)
+{
+ mcache_free(m_cache(MC_CL), p);
+}
+
+/*
+ * mcl_hasreference() checks if a cluster of an mbuf is referenced by
+ * another mbuf; see comments in m_incref() regarding EXTF_READONLY.
+ */
+int
+m_mclhasreference(struct mbuf *m)
+{
+ if (!(m->m_flags & M_EXT)) {
+ return 0;
+ }
+
+ ASSERT(m_get_rfa(m) != NULL);
+
+ return (MEXT_FLAGS(m) & EXTF_READONLY) ? 1 : 0;
+}
+
+__private_extern__ caddr_t
+m_bigalloc(int wait)
+{
+ int mcflags = MSLEEPF(wait);
+
+ /* Is this due to a non-blocking retry? If so, then try harder */
+ if (mcflags & MCR_NOSLEEP) {
+ mcflags |= MCR_TRYHARD;
+ }
+
+ return mcache_alloc(m_cache(MC_BIGCL), mcflags);
+}
+
+__private_extern__ void
+m_bigfree(caddr_t p, __unused u_int size, __unused caddr_t arg)
+{
+ mcache_free(m_cache(MC_BIGCL), p);
+}
+
+/* m_mbigget() add an 4KB mbuf cluster to a normal mbuf */
+__private_extern__ struct mbuf *
+m_mbigget(struct mbuf *m, int wait)
+{
+ struct ext_ref *rfa;
+
+ if ((rfa = mcache_alloc(ref_cache, MSLEEPF(wait))) == NULL) {
+ return m;
+ }
+
+ m->m_ext.ext_buf = m_bigalloc(wait);
+ if (m->m_ext.ext_buf != NULL) {
+ MBUF_BIGCL_INIT(m, m->m_ext.ext_buf, rfa, 1, 0);
+ } else {
+ mcache_free(ref_cache, rfa);
+ }
+ return m;
+}
+
+__private_extern__ caddr_t
+m_16kalloc(int wait)
+{
+ int mcflags = MSLEEPF(wait);
+
+ /* Is this due to a non-blocking retry? If so, then try harder */
+ if (mcflags & MCR_NOSLEEP) {
+ mcflags |= MCR_TRYHARD;
+ }
+
+ return mcache_alloc(m_cache(MC_16KCL), mcflags);
+}
+
+__private_extern__ void
+m_16kfree(caddr_t p, __unused u_int size, __unused caddr_t arg)
+{
+ mcache_free(m_cache(MC_16KCL), p);
+}
+
+/* m_m16kget() add a 16KB mbuf cluster to a normal mbuf */
+__private_extern__ struct mbuf *
+m_m16kget(struct mbuf *m, int wait)
+{
+ struct ext_ref *rfa;
+
+ if ((rfa = mcache_alloc(ref_cache, MSLEEPF(wait))) == NULL) {
+ return m;
+ }
+
+ m->m_ext.ext_buf = m_16kalloc(wait);
+ if (m->m_ext.ext_buf != NULL) {
+ MBUF_16KCL_INIT(m, m->m_ext.ext_buf, rfa, 1, 0);
+ } else {
+ mcache_free(ref_cache, rfa);
+ }
+ return m;
+}
+
+/*
+ * "Move" mbuf pkthdr from "from" to "to".
+ * "from" must have M_PKTHDR set, and "to" must be empty.
+ */
+void
+m_copy_pkthdr(struct mbuf *to, struct mbuf *from)
+{
+ VERIFY(from->m_flags & M_PKTHDR);
+
+ /* Check for scratch area overflow */
+ m_redzone_verify(from);
+
+ if (to->m_flags & M_PKTHDR) {
+ /* Check for scratch area overflow */
+ m_redzone_verify(to);
+ /* We will be taking over the tags of 'to' */
+ m_tag_delete_chain(to, NULL);
+ }
+ to->m_pkthdr = from->m_pkthdr; /* especially tags */
+ m_classifier_init(from, 0); /* purge classifier info */
+ m_tag_init(from, 1); /* purge all tags from src */
+ m_scratch_init(from); /* clear src scratch area */
+ to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
+ if ((to->m_flags & M_EXT) == 0) {
+ to->m_data = to->m_pktdat;
+ }
+ m_redzone_init(to); /* setup red zone on dst */
+}
+
+/*
+ * Duplicate "from"'s mbuf pkthdr in "to".
+ * "from" must have M_PKTHDR set, and "to" must be empty.
+ * In particular, this does a deep copy of the packet tags.
+ */
+static int
+m_dup_pkthdr(struct mbuf *to, struct mbuf *from, int how)
+{
+ VERIFY(from->m_flags & M_PKTHDR);
+
+ /* Check for scratch area overflow */
+ m_redzone_verify(from);
+
+ if (to->m_flags & M_PKTHDR) {
+ /* Check for scratch area overflow */
+ m_redzone_verify(to);
+ /* We will be taking over the tags of 'to' */
+ m_tag_delete_chain(to, NULL);
+ }
+ to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
+ if ((to->m_flags & M_EXT) == 0) {
+ to->m_data = to->m_pktdat;
+ }
+ to->m_pkthdr = from->m_pkthdr;
+ m_redzone_init(to); /* setup red zone on dst */
+ m_tag_init(to, 0); /* preserve dst static tags */
+ return m_tag_copy_chain(to, from, how);
+}
+
+void
+m_copy_pftag(struct mbuf *to, struct mbuf *from)
+{
+ memcpy(m_pftag(to), m_pftag(from), sizeof(struct pf_mtag));
+#if PF_ECN
+ m_pftag(to)->pftag_hdr = NULL;
+ m_pftag(to)->pftag_flags &= ~(PF_TAG_HDR_INET | PF_TAG_HDR_INET6);
+#endif /* PF_ECN */
+}
+
+void
+m_classifier_init(struct mbuf *m, uint32_t pktf_mask)
+{
+ VERIFY(m->m_flags & M_PKTHDR);
+
+ m->m_pkthdr.pkt_proto = 0;
+ m->m_pkthdr.pkt_flowsrc = 0;
+ m->m_pkthdr.pkt_flowid = 0;
+ m->m_pkthdr.pkt_flags &= pktf_mask; /* caller-defined mask */
+ /* preserve service class and interface info for loopback packets */
+ if (!(m->m_pkthdr.pkt_flags & PKTF_LOOP)) {
+ (void) m_set_service_class(m, MBUF_SC_BE);
+ }
+ if (!(m->m_pkthdr.pkt_flags & PKTF_IFAINFO)) {
+ m->m_pkthdr.pkt_ifainfo = 0;
+ }
+ /*
+ * Preserve timestamp if requested
+ */
+ if (!(m->m_pkthdr.pkt_flags & PKTF_TS_VALID)) {
+ m->m_pkthdr.pkt_timestamp = 0;
+ }
+}
+
+void
+m_copy_classifier(struct mbuf *to, struct mbuf *from)
+{
+ VERIFY(to->m_flags & M_PKTHDR);
+ VERIFY(from->m_flags & M_PKTHDR);
+
+ to->m_pkthdr.pkt_proto = from->m_pkthdr.pkt_proto;
+ to->m_pkthdr.pkt_flowsrc = from->m_pkthdr.pkt_flowsrc;
+ to->m_pkthdr.pkt_flowid = from->m_pkthdr.pkt_flowid;
+ to->m_pkthdr.pkt_flags = from->m_pkthdr.pkt_flags;
+ (void) m_set_service_class(to, from->m_pkthdr.pkt_svc);
+ to->m_pkthdr.pkt_ifainfo = from->m_pkthdr.pkt_ifainfo;
+}
+
+/*
+ * Return a list of mbuf hdrs that point to clusters. Try for num_needed;
+ * if wantall is not set, return whatever number were available. Set up the
+ * first num_with_pkthdrs with mbuf hdrs configured as packet headers; these
+ * are chained on the m_nextpkt field. Any packets requested beyond this
+ * are chained onto the last packet header's m_next field. The size of
+ * the cluster is controlled by the parameter bufsize.
+ */
+__private_extern__ struct mbuf *
+m_getpackets_internal(unsigned int *num_needed, int num_with_pkthdrs,
+ int wait, int wantall, size_t bufsize)
+{
+ struct mbuf *m;
+ struct mbuf **np, *top;
+ unsigned int pnum, needed = *num_needed;
+ mcache_obj_t *mp_list = NULL;
+ int mcflags = MSLEEPF(wait);
+ u_int16_t flag;
+ struct ext_ref *rfa;
+ mcache_t *cp;
+ void *cl;
+
+ ASSERT(bufsize == m_maxsize(MC_CL) ||
+ bufsize == m_maxsize(MC_BIGCL) ||
+ bufsize == m_maxsize(MC_16KCL));
+
+ /*
+ * Caller must first check for njcl because this
+ * routine is internal and not exposed/used via KPI.
+ */
+ VERIFY(bufsize != m_maxsize(MC_16KCL) || njcl > 0);
+
+ top = NULL;
+ np = ⊤
+ pnum = 0;
+
+ /*
+ * The caller doesn't want all the requested buffers; only some.
+ * Try hard to get what we can, but don't block. This effectively
+ * overrides MCR_SLEEP, since this thread will not go to sleep
+ * if we can't get all the buffers.
+ */
+ if (!wantall || (mcflags & MCR_NOSLEEP)) {
+ mcflags |= MCR_TRYHARD;
+ }
+
+ /* Allocate the composite mbuf + cluster elements from the cache */
+ if (bufsize == m_maxsize(MC_CL)) {
+ cp = m_cache(MC_MBUF_CL);
+ } else if (bufsize == m_maxsize(MC_BIGCL)) {
+ cp = m_cache(MC_MBUF_BIGCL);
+ } else {
+ cp = m_cache(MC_MBUF_16KCL);
+ }
+ needed = mcache_alloc_ext(cp, &mp_list, needed, mcflags);
+
+ for (pnum = 0; pnum < needed; pnum++) {
+ m = (struct mbuf *)mp_list;
+ mp_list = mp_list->obj_next;
+
+ VERIFY(m->m_type == MT_FREE && m->m_flags == M_EXT);
+ cl = m->m_ext.ext_buf;
+ rfa = m_get_rfa(m);
+
+ ASSERT(cl != NULL && rfa != NULL);
+ VERIFY(MBUF_IS_COMPOSITE(m));
+
+ flag = MEXT_FLAGS(m);
+
+ MBUF_INIT(m, num_with_pkthdrs, MT_DATA);
+ if (bufsize == m_maxsize(MC_16KCL)) {
+ MBUF_16KCL_INIT(m, cl, rfa, 1, flag);
+ } else if (bufsize == m_maxsize(MC_BIGCL)) {
+ MBUF_BIGCL_INIT(m, cl, rfa, 1, flag);
+ } else {
+ MBUF_CL_INIT(m, cl, rfa, 1, flag);
+ }
+
+ if (num_with_pkthdrs > 0) {
+ --num_with_pkthdrs;
+#if CONFIG_MACF_NET
+ if (mac_mbuf_label_init(m, wait) != 0) {
+ m_freem(m);
+ break;
+ }
+#endif /* MAC_NET */
+ }
+
+ *np = m;
+ if (num_with_pkthdrs > 0) {
+ np = &m->m_nextpkt;
+ } else {
+ np = &m->m_next;
+ }
+ }
+ ASSERT(pnum != *num_needed || mp_list == NULL);
+ if (mp_list != NULL) {
+ mcache_free_ext(cp, mp_list);
+ }
+
+ if (pnum > 0) {
+ mtype_stat_add(MT_DATA, pnum);
+ mtype_stat_sub(MT_FREE, pnum);
+ }
+
+ if (wantall && (pnum != *num_needed)) {
+ if (top != NULL) {
+ m_freem_list(top);
+ }
+ return NULL;
+ }
+
+ if (pnum > *num_needed) {
+ printf("%s: File a radar related to <rdar://10146739>. \
+ needed = %u, pnum = %u, num_needed = %u \n",
+ __func__, needed, pnum, *num_needed);
+ }
+
+ *num_needed = pnum;
+ return top;
+}
+
+/*
+ * Return list of mbuf linked by m_nextpkt. Try for numlist, and if
+ * wantall is not set, return whatever number were available. The size of
+ * each mbuf in the list is controlled by the parameter packetlen. Each
+ * mbuf of the list may have a chain of mbufs linked by m_next. Each mbuf
+ * in the chain is called a segment. If maxsegments is not null and the
+ * value pointed to is not null, this specify the maximum number of segments
+ * for a chain of mbufs. If maxsegments is zero or the value pointed to
+ * is zero the caller does not have any restriction on the number of segments.
+ * The actual number of segments of a mbuf chain is return in the value
+ * pointed to by maxsegments.
+ */
+__private_extern__ struct mbuf *
+m_allocpacket_internal(unsigned int *numlist, size_t packetlen,
+ unsigned int *maxsegments, int wait, int wantall, size_t wantsize)
+{
+ struct mbuf **np, *top, *first = NULL;
+ size_t bufsize, r_bufsize;
+ unsigned int num = 0;
+ unsigned int nsegs = 0;
+ unsigned int needed, resid;
+ int mcflags = MSLEEPF(wait);
+ mcache_obj_t *mp_list = NULL, *rmp_list = NULL;
+ mcache_t *cp = NULL, *rcp = NULL;
+
+ if (*numlist == 0) {
+ return NULL;
+ }
+
+ top = NULL;
+ np = ⊤
+
+ if (wantsize == 0) {
+ if (packetlen <= MINCLSIZE) {
+ bufsize = packetlen;
+ } else if (packetlen > m_maxsize(MC_CL)) {
+ /* Use 4KB if jumbo cluster pool isn't available */
+ if (packetlen <= m_maxsize(MC_BIGCL) || njcl == 0) {
+ bufsize = m_maxsize(MC_BIGCL);
+ } else {
+ bufsize = m_maxsize(MC_16KCL);
+ }
+ } else {
+ bufsize = m_maxsize(MC_CL);
+ }
+ } else if (wantsize == m_maxsize(MC_CL) ||
+ wantsize == m_maxsize(MC_BIGCL) ||
+ (wantsize == m_maxsize(MC_16KCL) && njcl > 0)) {
+ bufsize = wantsize;
+ } else {
+ return NULL;
+ }
+
+ if (bufsize <= MHLEN) {
+ nsegs = 1;
+ } else if (bufsize <= MINCLSIZE) {
+ if (maxsegments != NULL && *maxsegments == 1) {
+ bufsize = m_maxsize(MC_CL);
+ nsegs = 1;
+ } else {
+ nsegs = 2;
+ }
+ } else if (bufsize == m_maxsize(MC_16KCL)) {
+ VERIFY(njcl > 0);
+ nsegs = ((packetlen - 1) >> M16KCLSHIFT) + 1;
+ } else if (bufsize == m_maxsize(MC_BIGCL)) {
+ nsegs = ((packetlen - 1) >> MBIGCLSHIFT) + 1;
+ } else {
+ nsegs = ((packetlen - 1) >> MCLSHIFT) + 1;
+ }
+ if (maxsegments != NULL) {
+ if (*maxsegments && nsegs > *maxsegments) {
+ *maxsegments = nsegs;
+ return NULL;
+ }
+ *maxsegments = nsegs;
+ }
+
+ /*
+ * The caller doesn't want all the requested buffers; only some.
+ * Try hard to get what we can, but don't block. This effectively
+ * overrides MCR_SLEEP, since this thread will not go to sleep
+ * if we can't get all the buffers.
+ */
+ if (!wantall || (mcflags & MCR_NOSLEEP)) {
+ mcflags |= MCR_TRYHARD;
+ }
+
+ /*
+ * Simple case where all elements in the lists/chains are mbufs.
+ * Unless bufsize is greater than MHLEN, each segment chain is made
+ * up of exactly 1 mbuf. Otherwise, each segment chain is made up
+ * of 2 mbufs; the second one is used for the residual data, i.e.
+ * the remaining data that cannot fit into the first mbuf.
+ */
+ if (bufsize <= MINCLSIZE) {
+ /* Allocate the elements in one shot from the mbuf cache */
+ ASSERT(bufsize <= MHLEN || nsegs == 2);
+ cp = m_cache(MC_MBUF);
+ needed = mcache_alloc_ext(cp, &mp_list,
+ (*numlist) * nsegs, mcflags);
+
+ /*
+ * The number of elements must be even if we are to use an
+ * mbuf (instead of a cluster) to store the residual data.
+ * If we couldn't allocate the requested number of mbufs,
+ * trim the number down (if it's odd) in order to avoid
+ * creating a partial segment chain.
+ */
+ if (bufsize > MHLEN && (needed & 0x1)) {
+ needed--;
+ }
+
+ while (num < needed) {
+ struct mbuf *m;
+
+ m = (struct mbuf *)mp_list;
+ mp_list = mp_list->obj_next;
+ ASSERT(m != NULL);
+
+ MBUF_INIT(m, 1, MT_DATA);
+#if CONFIG_MACF_NET
+ if (mac_init_mbuf(m, wait) != 0) {
+ m_free(m);
+ break;
+ }
+#endif /* MAC_NET */
+ num++;
+ if (bufsize > MHLEN) {
+ /* A second mbuf for this segment chain */
+ m->m_next = (struct mbuf *)mp_list;
+ mp_list = mp_list->obj_next;
+ ASSERT(m->m_next != NULL);
+
+ MBUF_INIT(m->m_next, 0, MT_DATA);
+ num++;
+ }
+ *np = m;
+ np = &m->m_nextpkt;
+ }
+ ASSERT(num != *numlist || mp_list == NULL);
+
+ if (num > 0) {
+ mtype_stat_add(MT_DATA, num);
+ mtype_stat_sub(MT_FREE, num);
+ }
+ num /= nsegs;
+
+ /* We've got them all; return to caller */
+ if (num == *numlist) {
+ return top;
+ }
+
+ goto fail;
+ }
+
+ /*
+ * Complex cases where elements are made up of one or more composite
+ * mbufs + cluster, depending on packetlen. Each N-segment chain can
+ * be illustrated as follows:
+ *
+ * [mbuf + cluster 1] [mbuf + cluster 2] ... [mbuf + cluster N]
+ *
+ * Every composite mbuf + cluster element comes from the intermediate
+ * cache (either MC_MBUF_CL or MC_MBUF_BIGCL). For space efficiency,
+ * the last composite element will come from the MC_MBUF_CL cache,
+ * unless the residual data is larger than 2KB where we use the
+ * big cluster composite cache (MC_MBUF_BIGCL) instead. Residual
+ * data is defined as extra data beyond the first element that cannot
+ * fit into the previous element, i.e. there is no residual data if
+ * the chain only has 1 segment.
+ */
+ r_bufsize = bufsize;
+ resid = packetlen > bufsize ? packetlen % bufsize : 0;
+ if (resid > 0) {
+ /* There is residual data; figure out the cluster size */
+ if (wantsize == 0 && packetlen > MINCLSIZE) {
+ /*
+ * Caller didn't request that all of the segments
+ * in the chain use the same cluster size; use the
+ * smaller of the cluster sizes.
+ */
+ if (njcl > 0 && resid > m_maxsize(MC_BIGCL)) {
+ r_bufsize = m_maxsize(MC_16KCL);
+ } else if (resid > m_maxsize(MC_CL)) {
+ r_bufsize = m_maxsize(MC_BIGCL);
+ } else {
+ r_bufsize = m_maxsize(MC_CL);
+ }
+ } else {
+ /* Use the same cluster size as the other segments */
+ resid = 0;
+ }
+ }
+
+ needed = *numlist;
+ if (resid > 0) {
+ /*
+ * Attempt to allocate composite mbuf + cluster elements for
+ * the residual data in each chain; record the number of such
+ * elements that can be allocated so that we know how many
+ * segment chains we can afford to create.
+ */
+ if (r_bufsize <= m_maxsize(MC_CL)) {
+ rcp = m_cache(MC_MBUF_CL);
+ } else if (r_bufsize <= m_maxsize(MC_BIGCL)) {
+ rcp = m_cache(MC_MBUF_BIGCL);
+ } else {
+ rcp = m_cache(MC_MBUF_16KCL);
+ }
+ needed = mcache_alloc_ext(rcp, &rmp_list, *numlist, mcflags);
+
+ if (needed == 0) {
+ goto fail;
+ }
+
+ /* This is temporarily reduced for calculation */
+ ASSERT(nsegs > 1);
+ nsegs--;
+ }
+
+ /*
+ * Attempt to allocate the rest of the composite mbuf + cluster
+ * elements for the number of segment chains that we need.
+ */
+ if (bufsize <= m_maxsize(MC_CL)) {
+ cp = m_cache(MC_MBUF_CL);
+ } else if (bufsize <= m_maxsize(MC_BIGCL)) {
+ cp = m_cache(MC_MBUF_BIGCL);
+ } else {
+ cp = m_cache(MC_MBUF_16KCL);
+ }
+ needed = mcache_alloc_ext(cp, &mp_list, needed * nsegs, mcflags);
+
+ /* Round it down to avoid creating a partial segment chain */
+ needed = (needed / nsegs) * nsegs;
+ if (needed == 0) {
+ goto fail;
+ }
+
+ if (resid > 0) {
+ /*
+ * We're about to construct the chain(s); take into account
+ * the number of segments we have created above to hold the
+ * residual data for each chain, as well as restore the
+ * original count of segments per chain.
+ */
+ ASSERT(nsegs > 0);
+ needed += needed / nsegs;
+ nsegs++;
+ }
+
+ for (;;) {
+ struct mbuf *m;
+ u_int16_t flag;
+ struct ext_ref *rfa;
+ void *cl;
+ int pkthdr;
+ m_ext_free_func_t m_free_func;
+
+ ++num;
+ if (nsegs == 1 || (num % nsegs) != 0 || resid == 0) {
+ m = (struct mbuf *)mp_list;
+ mp_list = mp_list->obj_next;
+ } else {
+ m = (struct mbuf *)rmp_list;
+ rmp_list = rmp_list->obj_next;
+ }
+ m_free_func = m_get_ext_free(m);
+ ASSERT(m != NULL);
+ VERIFY(m->m_type == MT_FREE && m->m_flags == M_EXT);
+ VERIFY(m_free_func == NULL || m_free_func == m_bigfree ||
+ m_free_func == m_16kfree);
+
+ cl = m->m_ext.ext_buf;
+ rfa = m_get_rfa(m);
+
+ ASSERT(cl != NULL && rfa != NULL);
+ VERIFY(MBUF_IS_COMPOSITE(m));
+
+ flag = MEXT_FLAGS(m);
+
+ pkthdr = (nsegs == 1 || (num % nsegs) == 1);
+ if (pkthdr) {
+ first = m;
+ }
+ MBUF_INIT(m, pkthdr, MT_DATA);
+ if (m_free_func == m_16kfree) {
+ MBUF_16KCL_INIT(m, cl, rfa, 1, flag);
+ } else if (m_free_func == m_bigfree) {
+ MBUF_BIGCL_INIT(m, cl, rfa, 1, flag);
+ } else {
+ MBUF_CL_INIT(m, cl, rfa, 1, flag);
+ }
+#if CONFIG_MACF_NET
+ if (pkthdr && mac_init_mbuf(m, wait) != 0) {
+ --num;
+ m_freem(m);
+ break;
+ }
+#endif /* MAC_NET */
+
+ *np = m;
+ if ((num % nsegs) == 0) {
+ np = &first->m_nextpkt;
+ } else {
+ np = &m->m_next;
+ }
+
+ if (num == needed) {
+ break;
+ }
+ }
+
+ if (num > 0) {
+ mtype_stat_add(MT_DATA, num);
+ mtype_stat_sub(MT_FREE, num);
+ }
+
+ num /= nsegs;
+
+ /* We've got them all; return to caller */
+ if (num == *numlist) {
+ ASSERT(mp_list == NULL && rmp_list == NULL);
+ return top;
+ }
+
+fail:
+ /* Free up what's left of the above */
+ if (mp_list != NULL) {
+ mcache_free_ext(cp, mp_list);
+ }
+ if (rmp_list != NULL) {
+ mcache_free_ext(rcp, rmp_list);
+ }
+ if (wantall && top != NULL) {
+ m_freem_list(top);
+ return NULL;
+ }
+ *numlist = num;
+ return top;
+}
+
+/*
+ * Best effort to get a mbuf cluster + pkthdr. Used by drivers to allocated
+ * packets on receive ring.
+ */
+__private_extern__ struct mbuf *
+m_getpacket_how(int wait)
+{
+ unsigned int num_needed = 1;
+
+ return m_getpackets_internal(&num_needed, 1, wait, 1,
+ m_maxsize(MC_CL));
+}
+
+/*
+ * Best effort to get a mbuf cluster + pkthdr. Used by drivers to allocated
+ * packets on receive ring.
+ */
+struct mbuf *
+m_getpacket(void)
+{
+ unsigned int num_needed = 1;
+
+ return m_getpackets_internal(&num_needed, 1, M_WAIT, 1,
+ m_maxsize(MC_CL));
+}
+
+/*
+ * Return a list of mbuf hdrs that point to clusters. Try for num_needed;
+ * if this can't be met, return whatever number were available. Set up the
+ * first num_with_pkthdrs with mbuf hdrs configured as packet headers. These
+ * are chained on the m_nextpkt field. Any packets requested beyond this are
+ * chained onto the last packet header's m_next field.
+ */
+struct mbuf *
+m_getpackets(int num_needed, int num_with_pkthdrs, int how)
+{
+ unsigned int n = num_needed;
+
+ return m_getpackets_internal(&n, num_with_pkthdrs, how, 0,
+ m_maxsize(MC_CL));
+}
+
+/*
+ * Return a list of mbuf hdrs set up as packet hdrs chained together
+ * on the m_nextpkt field
+ */
+struct mbuf *
+m_getpackethdrs(int num_needed, int how)
+{
+ struct mbuf *m;
+ struct mbuf **np, *top;
+
+ top = NULL;
+ np = ⊤
+
+ while (num_needed--) {
+ m = _M_RETRYHDR(how, MT_DATA);
+ if (m == NULL) {
+ break;
+ }
+
+ *np = m;
+ np = &m->m_nextpkt;
+ }
+
+ return top;
+}
+
+/*
+ * Free an mbuf list (m_nextpkt) while following m_next. Returns the count
+ * for mbufs packets freed. Used by the drivers.
+ */
+int
+m_freem_list(struct mbuf *m)
+{
+ struct mbuf *nextpkt;
+ mcache_obj_t *mp_list = NULL;
+ mcache_obj_t *mcl_list = NULL;
+ mcache_obj_t *mbc_list = NULL;
+ mcache_obj_t *m16k_list = NULL;
+ mcache_obj_t *m_mcl_list = NULL;
+ mcache_obj_t *m_mbc_list = NULL;
+ mcache_obj_t *m_m16k_list = NULL;
+ mcache_obj_t *ref_list = NULL;
+ int pktcount = 0;
+ int mt_free = 0, mt_data = 0, mt_header = 0, mt_soname = 0, mt_tag = 0;
+
+ while (m != NULL) {
+ pktcount++;
+
+ nextpkt = m->m_nextpkt;
+ m->m_nextpkt = NULL;
+
+ while (m != NULL) {
+ struct mbuf *next = m->m_next;
+ mcache_obj_t *o, *rfa;
+ u_int32_t composite;
+ u_int16_t refcnt;
+ m_ext_free_func_t m_free_func;
+
+ if (m->m_type == MT_FREE) {
+ panic("m_free: freeing an already freed mbuf");
+ }
+
+ if (m->m_flags & M_PKTHDR) {
+ /* Check for scratch area overflow */
+ m_redzone_verify(m);
+ /* Free the aux data and tags if there is any */
+ m_tag_delete_chain(m, NULL);
+ }
+
+ if (!(m->m_flags & M_EXT)) {
+ mt_free++;
+ goto simple_free;
+ }
+
+ if (MBUF_IS_PAIRED(m) && m_free_paired(m)) {
+ m = next;
+ continue;
+ }
+
+ mt_free++;
+
+ o = (mcache_obj_t *)(void *)m->m_ext.ext_buf;
+ refcnt = m_decref(m);
+ composite = (MEXT_FLAGS(m) & EXTF_COMPOSITE);
+ m_free_func = m_get_ext_free(m);
+ if (refcnt == MEXT_MINREF(m) && !composite) {
+ if (m_free_func == NULL) {
+ o->obj_next = mcl_list;
+ mcl_list = o;
+ } else if (m_free_func == m_bigfree) {
+ o->obj_next = mbc_list;
+ mbc_list = o;
+ } else if (m_free_func == m_16kfree) {
+ o->obj_next = m16k_list;
+ m16k_list = o;
+ } else {
+ (*(m_free_func))((caddr_t)o,
+ m->m_ext.ext_size,
+ m_get_ext_arg(m));
+ }
+ rfa = (mcache_obj_t *)(void *)m_get_rfa(m);
+ rfa->obj_next = ref_list;
+ ref_list = rfa;
+ m_set_ext(m, NULL, NULL, NULL);
+ } else if (refcnt == MEXT_MINREF(m) && composite) {
+ VERIFY(!(MEXT_FLAGS(m) & EXTF_PAIRED));
+ VERIFY(m->m_type != MT_FREE);
+ /*
+ * Amortize the costs of atomic operations
+ * by doing them at the end, if possible.
+ */
+ if (m->m_type == MT_DATA) {
+ mt_data++;
+ } else if (m->m_type == MT_HEADER) {
+ mt_header++;
+ } else if (m->m_type == MT_SONAME) {
+ mt_soname++;
+ } else if (m->m_type == MT_TAG) {
+ mt_tag++;
+ } else {
+ mtype_stat_dec(m->m_type);
+ }
+
+ m->m_type = MT_FREE;
+ m->m_flags = M_EXT;
+ m->m_len = 0;
+ m->m_next = m->m_nextpkt = NULL;
+
+ MEXT_FLAGS(m) &= ~EXTF_READONLY;
+
+ /* "Free" into the intermediate cache */
+ o = (mcache_obj_t *)m;
+ if (m_free_func == NULL) {
+ o->obj_next = m_mcl_list;
+ m_mcl_list = o;
+ } else if (m_free_func == m_bigfree) {
+ o->obj_next = m_mbc_list;
+ m_mbc_list = o;
+ } else {
+ VERIFY(m_free_func == m_16kfree);
+ o->obj_next = m_m16k_list;
+ m_m16k_list = o;
+ }
+ m = next;
+ continue;
+ }
+simple_free:
+ /*
+ * Amortize the costs of atomic operations
+ * by doing them at the end, if possible.
+ */
+ if (m->m_type == MT_DATA) {
+ mt_data++;
+ } else if (m->m_type == MT_HEADER) {
+ mt_header++;
+ } else if (m->m_type == MT_SONAME) {
+ mt_soname++;
+ } else if (m->m_type == MT_TAG) {
+ mt_tag++;
+ } else if (m->m_type != MT_FREE) {
+ mtype_stat_dec(m->m_type);
+ }
+
+ m->m_type = MT_FREE;
+ m->m_flags = m->m_len = 0;
+ m->m_next = m->m_nextpkt = NULL;
+
+ ((mcache_obj_t *)m)->obj_next = mp_list;
+ mp_list = (mcache_obj_t *)m;
+
+ m = next;
+ }
+
+ m = nextpkt;
+ }
+
+ if (mt_free > 0) {
+ mtype_stat_add(MT_FREE, mt_free);
+ }
+ if (mt_data > 0) {
+ mtype_stat_sub(MT_DATA, mt_data);
+ }
+ if (mt_header > 0) {
+ mtype_stat_sub(MT_HEADER, mt_header);
+ }
+ if (mt_soname > 0) {
+ mtype_stat_sub(MT_SONAME, mt_soname);
+ }
+ if (mt_tag > 0) {
+ mtype_stat_sub(MT_TAG, mt_tag);
+ }
+
+ if (mp_list != NULL) {
+ mcache_free_ext(m_cache(MC_MBUF), mp_list);
+ }
+ if (mcl_list != NULL) {
+ mcache_free_ext(m_cache(MC_CL), mcl_list);
+ }
+ if (mbc_list != NULL) {
+ mcache_free_ext(m_cache(MC_BIGCL), mbc_list);
+ }
+ if (m16k_list != NULL) {
+ mcache_free_ext(m_cache(MC_16KCL), m16k_list);
+ }
+ if (m_mcl_list != NULL) {
+ mcache_free_ext(m_cache(MC_MBUF_CL), m_mcl_list);
+ }
+ if (m_mbc_list != NULL) {
+ mcache_free_ext(m_cache(MC_MBUF_BIGCL), m_mbc_list);
+ }
+ if (m_m16k_list != NULL) {
+ mcache_free_ext(m_cache(MC_MBUF_16KCL), m_m16k_list);
+ }
+ if (ref_list != NULL) {
+ mcache_free_ext(ref_cache, ref_list);
+ }
+
+ return pktcount;
+}
+
+void
+m_freem(struct mbuf *m)
+{
+ while (m != NULL) {
+ m = m_free(m);
+ }
+}
+
+/*
+ * Mbuffer utility routines.
+ */
+/*
+ * Set the m_data pointer of a newly allocated mbuf to place an object of the
+ * specified size at the end of the mbuf, longword aligned.
+ *
+ * NB: Historically, we had M_ALIGN(), MH_ALIGN(), and MEXT_ALIGN() as
+ * separate macros, each asserting that it was called at the proper moment.
+ * This required callers to themselves test the storage type and call the
+ * right one. Rather than require callers to be aware of those layout
+ * decisions, we centralize here.
+ */
+void
+m_align(struct mbuf *m, int len)
+{
+ int adjust = 0;
+
+ /* At this point data must point to start */
+ VERIFY(m->m_data == M_START(m));
+ VERIFY(len >= 0);
+ VERIFY(len <= M_SIZE(m));
+ adjust = M_SIZE(m) - len;
+ m->m_data += adjust & ~(sizeof(long) - 1);
+}
+
+/*
+ * Lesser-used path for M_PREPEND: allocate new mbuf to prepend to chain,
+ * copy junk along. Does not adjust packet header length.
+ */
+struct mbuf *
+m_prepend(struct mbuf *m, int len, int how)
+{
+ struct mbuf *mn;
+
+ _MGET(mn, how, m->m_type);
+ if (mn == NULL) {
+ m_freem(m);
+ return NULL;
+ }
+ if (m->m_flags & M_PKTHDR) {
+ M_COPY_PKTHDR(mn, m);
+ m->m_flags &= ~M_PKTHDR;
+ }
+ mn->m_next = m;
+ m = mn;
+ if (m->m_flags & M_PKTHDR) {
+ VERIFY(len <= MHLEN);
+ MH_ALIGN(m, len);
+ } else {
+ VERIFY(len <= MLEN);
+ M_ALIGN(m, len);
+ }
+ m->m_len = len;
+ return m;
+}
+
+/*
+ * Replacement for old M_PREPEND macro: allocate new mbuf to prepend to
+ * chain, copy junk along, and adjust length.
+ */
+struct mbuf *
+m_prepend_2(struct mbuf *m, int len, int how, int align)
+{
+ if (M_LEADINGSPACE(m) >= len &&
+ (!align || IS_P2ALIGNED((m->m_data - len), sizeof(u_int32_t)))) {
+ m->m_data -= len;
+ m->m_len += len;
+ } else {
+ m = m_prepend(m, len, how);
+ }
+ if ((m) && (m->m_flags & M_PKTHDR)) {
+ m->m_pkthdr.len += len;
+ }
+ return m;
+}
+
+/*
+ * Make a copy of an mbuf chain starting "off0" bytes from the beginning,
+ * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf.
+ * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller.
+ */
+int MCFail;
+
+struct mbuf *
+m_copym_mode(struct mbuf *m, int off0, int len, int wait, uint32_t mode)
+{
+ struct mbuf *n, *mhdr = NULL, **np;
+ int off = off0;
+ struct mbuf *top;
+ int copyhdr = 0;
+
+ if (off < 0 || len < 0) {
+ panic("m_copym: invalid offset %d or len %d", off, len);
+ }
+
+ VERIFY((mode != M_COPYM_MUST_COPY_HDR &&
+ mode != M_COPYM_MUST_MOVE_HDR) || (m->m_flags & M_PKTHDR));
+
+ if ((off == 0 && (m->m_flags & M_PKTHDR)) ||
+ mode == M_COPYM_MUST_COPY_HDR || mode == M_COPYM_MUST_MOVE_HDR) {
+ mhdr = m;
+ copyhdr = 1;
+ }
+
+ while (off >= m->m_len) {
+ if (m->m_next == NULL) {
+ panic("m_copym: invalid mbuf chain");
+ }
+ off -= m->m_len;
+ m = m->m_next;
+ }
+ np = ⊤
+ top = NULL;
+
+ while (len > 0) {
+ if (m == NULL) {
+ if (len != M_COPYALL) {
+ panic("m_copym: len != M_COPYALL");
+ }
+ break;
+ }
+
+ if (copyhdr) {
+ n = _M_RETRYHDR(wait, m->m_type);
+ } else {
+ n = _M_RETRY(wait, m->m_type);
+ }
+ *np = n;
+
+ if (n == NULL) {
+ goto nospace;
+ }
+
+ if (copyhdr != 0) {
+ if ((mode == M_COPYM_MOVE_HDR) ||
+ (mode == M_COPYM_MUST_MOVE_HDR)) {
+ M_COPY_PKTHDR(n, mhdr);
+ } else if ((mode == M_COPYM_COPY_HDR) ||
+ (mode == M_COPYM_MUST_COPY_HDR)) {
+ if (m_dup_pkthdr(n, mhdr, wait) == 0) {
+ goto nospace;
+ }
+ }
+ if (len == M_COPYALL) {
+ n->m_pkthdr.len -= off0;
+ } else {
+ n->m_pkthdr.len = len;
+ }
+ copyhdr = 0;
+ /*
+ * There is data to copy from the packet header mbuf
+ * if it is empty or it is before the starting offset
+ */
+ if (mhdr != m) {
+ np = &n->m_next;
+ continue;
+ }
+ }
+ n->m_len = MIN(len, (m->m_len - off));
+ if (m->m_flags & M_EXT) {
+ n->m_ext = m->m_ext;
+ m_incref(m);
+ n->m_data = m->m_data + off;
+ n->m_flags |= M_EXT;
+ } else {
+ /*
+ * Limit to the capacity of the destination
+ */
+ if (n->m_flags & M_PKTHDR) {
+ n->m_len = MIN(n->m_len, MHLEN);
+ } else {
+ n->m_len = MIN(n->m_len, MLEN);
+ }
+
+ if (MTOD(n, char *) + n->m_len > ((char *)n) + MSIZE) {
+ panic("%s n %p copy overflow",
+ __func__, n);
+ }
+
+ bcopy(MTOD(m, caddr_t) + off, MTOD(n, caddr_t),
+ (unsigned)n->m_len);
+ }
+ if (len != M_COPYALL) {
+ len -= n->m_len;
+ }
+ off = 0;
+ m = m->m_next;
+ np = &n->m_next;
+ }
+
+ if (top == NULL) {
+ MCFail++;
+ }
+
+ return top;
+nospace:
+
+ m_freem(top);
+ MCFail++;
+ return NULL;
+}
+
+
+struct mbuf *
+m_copym(struct mbuf *m, int off0, int len, int wait)
+{
+ return m_copym_mode(m, off0, len, wait, M_COPYM_MOVE_HDR);
+}
+
+/*
+ * Equivalent to m_copym except that all necessary mbuf hdrs are allocated
+ * within this routine also, the last mbuf and offset accessed are passed
+ * out and can be passed back in to avoid having to rescan the entire mbuf
+ * list (normally hung off of the socket)
+ */
+struct mbuf *
+m_copym_with_hdrs(struct mbuf *m0, int off0, int len0, int wait,
+ struct mbuf **m_lastm, int *m_off, uint32_t mode)
+{
+ struct mbuf *m = m0, *n, **np = NULL;
+ int off = off0, len = len0;
+ struct mbuf *top = NULL;
+ int mcflags = MSLEEPF(wait);
+ int copyhdr = 0;
+ int type = 0;
+ mcache_obj_t *list = NULL;
+ int needed = 0;
+
+ if (off == 0 && (m->m_flags & M_PKTHDR)) {
+ copyhdr = 1;
+ }
+
+ if (m_lastm != NULL && *m_lastm != NULL) {
+ m = *m_lastm;
+ off = *m_off;
+ } else {
+ while (off >= m->m_len) {
+ off -= m->m_len;
+ m = m->m_next;
+ }
+ }
+
+ n = m;
+ while (len > 0) {
+ needed++;
+ ASSERT(n != NULL);
+ len -= MIN(len, (n->m_len - ((needed == 1) ? off : 0)));
+ n = n->m_next;
+ }
+ needed++;
+ len = len0;
+
+ /*
+ * If the caller doesn't want to be put to sleep, mark it with
+ * MCR_TRYHARD so that we may reclaim buffers from other places
+ * before giving up.
+ */
+ if (mcflags & MCR_NOSLEEP) {
+ mcflags |= MCR_TRYHARD;
+ }
+
+ if (mcache_alloc_ext(m_cache(MC_MBUF), &list, needed,
+ mcflags) != needed) {
+ goto nospace;
+ }
+
+ needed = 0;
+ while (len > 0) {
+ n = (struct mbuf *)list;
+ list = list->obj_next;
+ ASSERT(n != NULL && m != NULL);
+
+ type = (top == NULL) ? MT_HEADER : m->m_type;
+ MBUF_INIT(n, (top == NULL), type);
+#if CONFIG_MACF_NET
+ if (top == NULL && mac_mbuf_label_init(n, wait) != 0) {
+ mtype_stat_inc(MT_HEADER);
+ mtype_stat_dec(MT_FREE);
+ m_free(n);
+ goto nospace;
+ }
+#endif /* MAC_NET */
+
+ if (top == NULL) {
+ top = n;
+ np = &top->m_next;
+ continue;
+ } else {
+ needed++;
+ *np = n;
+ }
+
+ if (copyhdr) {
+ if ((mode == M_COPYM_MOVE_HDR) ||
+ (mode == M_COPYM_MUST_MOVE_HDR)) {
+ M_COPY_PKTHDR(n, m);
+ } else if ((mode == M_COPYM_COPY_HDR) ||
+ (mode == M_COPYM_MUST_COPY_HDR)) {
+ if (m_dup_pkthdr(n, m, wait) == 0) {
+ goto nospace;
+ }
+ }
+ n->m_pkthdr.len = len;
+ copyhdr = 0;
+ }
+ n->m_len = MIN(len, (m->m_len - off));
+
+ if (m->m_flags & M_EXT) {
+ n->m_ext = m->m_ext;
+ m_incref(m);
+ n->m_data = m->m_data + off;
+ n->m_flags |= M_EXT;
+ } else {
+ if (MTOD(n, char *) + n->m_len > ((char *)n) + MSIZE) {
+ panic("%s n %p copy overflow",
+ __func__, n);
+ }
+
+ bcopy(MTOD(m, caddr_t) + off, MTOD(n, caddr_t),
+ (unsigned)n->m_len);
+ }
+ len -= n->m_len;
+
+ if (len == 0) {
+ if (m_lastm != NULL && m_off != NULL) {
+ if ((off + n->m_len) == m->m_len) {
+ *m_lastm = m->m_next;
+ *m_off = 0;
+ } else {
+ *m_lastm = m;
+ *m_off = off + n->m_len;
+ }
+ }
+ break;
+ }
+ off = 0;
+ m = m->m_next;
+ np = &n->m_next;
+ }
+
+ mtype_stat_inc(MT_HEADER);
+ mtype_stat_add(type, needed);
+ mtype_stat_sub(MT_FREE, needed + 1);
+
+ ASSERT(list == NULL);
+ return top;
+
+nospace:
+ if (list != NULL) {
+ mcache_free_ext(m_cache(MC_MBUF), list);
+ }
+ if (top != NULL) {
+ m_freem(top);
+ }
+ MCFail++;
+ return NULL;
+}
+
+/*
+ * Copy data from an mbuf chain starting "off" bytes from the beginning,
+ * continuing for "len" bytes, into the indicated buffer.
+ */
+void
+m_copydata(struct mbuf *m, int off, int len, void *vp)
+{
+ int off0 = off, len0 = len;
+ struct mbuf *m0 = m;
+ unsigned count;
+ char *cp = vp;
+
+ if (__improbable(off < 0 || len < 0)) {
+ panic("%s: invalid offset %d or len %d", __func__, off, len);
+ /* NOTREACHED */
+ }
+
+ while (off > 0) {
+ if (__improbable(m == NULL)) {
+ panic("%s: invalid mbuf chain %p [off %d, len %d]",
+ __func__, m0, off0, len0);
+ /* NOTREACHED */
+ }
+ if (off < m->m_len) {
+ break;
+ }
+ off -= m->m_len;
+ m = m->m_next;
+ }
+ while (len > 0) {
+ if (__improbable(m == NULL)) {
+ panic("%s: invalid mbuf chain %p [off %d, len %d]",
+ __func__, m0, off0, len0);
+ /* NOTREACHED */
+ }
+ count = MIN(m->m_len - off, len);
+ bcopy(MTOD(m, caddr_t) + off, cp, count);
+ len -= count;
+ cp += count;
+ off = 0;
+ m = m->m_next;
+ }
+}
+
+/*
+ * Concatenate mbuf chain n to m. Both chains must be of the same type
+ * (e.g. MT_DATA). Any m_pkthdr is not updated.
+ */
+void
+m_cat(struct mbuf *m, struct mbuf *n)
+{
+ while (m->m_next) {
+ m = m->m_next;
+ }
+ while (n) {
+ if ((m->m_flags & M_EXT) ||
+ m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) {
+ /* just join the two chains */
+ m->m_next = n;
+ return;
+ }
+ /* splat the data from one into the other */
+ bcopy(MTOD(n, caddr_t), MTOD(m, caddr_t) + m->m_len,
+ (u_int)n->m_len);
+ m->m_len += n->m_len;
+ n = m_free(n);
+ }
+}
+
+void
+m_adj(struct mbuf *mp, int req_len)
+{
+ int len = req_len;
+ struct mbuf *m;
+ int count;
+
+ if ((m = mp) == NULL) {
+ return;
+ }
+ if (len >= 0) {
+ /*
+ * Trim from head.
+ */
+ while (m != NULL && len > 0) {
+ if (m->m_len <= len) {
+ len -= m->m_len;
+ m->m_len = 0;
+ m = m->m_next;
+ } else {
+ m->m_len -= len;
+ m->m_data += len;
+ len = 0;
+ }
+ }
+ m = mp;
+ if (m->m_flags & M_PKTHDR) {
+ m->m_pkthdr.len -= (req_len - len);
+ }
+ } else {
+ /*
+ * Trim from tail. Scan the mbuf chain,
+ * calculating its length and finding the last mbuf.
+ * If the adjustment only affects this mbuf, then just
+ * adjust and return. Otherwise, rescan and truncate
+ * after the remaining size.
+ */
+ len = -len;
+ count = 0;
+ for (;;) {
+ count += m->m_len;
+ if (m->m_next == (struct mbuf *)0) {
+ break;
+ }
+ m = m->m_next;
+ }
+ if (m->m_len >= len) {
+ m->m_len -= len;
+ m = mp;
+ if (m->m_flags & M_PKTHDR) {
+ m->m_pkthdr.len -= len;
+ }
+ return;
+ }
+ count -= len;
+ if (count < 0) {
+ count = 0;
+ }
+ /*
+ * Correct length for chain is "count".
+ * Find the mbuf with last data, adjust its length,
+ * and toss data from remaining mbufs on chain.
+ */
+ m = mp;
+ if (m->m_flags & M_PKTHDR) {
+ m->m_pkthdr.len = count;
+ }
+ for (; m; m = m->m_next) {
+ if (m->m_len >= count) {
+ m->m_len = count;
+ break;
+ }
+ count -= m->m_len;
+ }
+ while ((m = m->m_next)) {
+ m->m_len = 0;
+ }
+ }
+}
+
+/*
+ * Rearange an mbuf chain so that len bytes are contiguous
+ * and in the data area of an mbuf (so that mtod and dtom
+ * will work for a structure of size len). Returns the resulting
+ * mbuf chain on success, frees it and returns null on failure.
+ * If there is room, it will add up to max_protohdr-len extra bytes to the
+ * contiguous region in an attempt to avoid being called next time.
+ */
+int MPFail;
+
+struct mbuf *
+m_pullup(struct mbuf *n, int len)
+{
+ struct mbuf *m;
+ int count;
+ int space;
+
+ /* check invalid arguments */
+ if (n == NULL) {
+ panic("%s: n == NULL", __func__);
+ }
+ if (len < 0) {
+ os_log_info(OS_LOG_DEFAULT, "%s: failed negative len %d",
+ __func__, len);
+ goto bad;
+ }
+ if (len > MLEN) {
+ os_log_info(OS_LOG_DEFAULT, "%s: failed len %d too big",
+ __func__, len);
+ goto bad;
+ }
+ if ((n->m_flags & M_EXT) == 0 &&
+ n->m_data >= &n->m_dat[MLEN]) {
+ os_log_info(OS_LOG_DEFAULT, "%s: m_data out of bounds",
+ __func__);
+ goto bad;
+ }
+
+ /*
+ * If first mbuf has no cluster, and has room for len bytes
+ * without shifting current data, pullup into it,
+ * otherwise allocate a new mbuf to prepend to the chain.
+ */
+ if ((n->m_flags & M_EXT) == 0 &&
+ len < &n->m_dat[MLEN] - n->m_data && n->m_next != NULL) {
+ if (n->m_len >= len) {
+ return n;
+ }
+ m = n;
+ n = n->m_next;
+ len -= m->m_len;
+ } else {
+ if (len > MHLEN) {
+ goto bad;
+ }
+ _MGET(m, M_DONTWAIT, n->m_type);
+ if (m == 0) {
+ goto bad;
+ }
+ m->m_len = 0;
+ if (n->m_flags & M_PKTHDR) {
+ M_COPY_PKTHDR(m, n);
+ n->m_flags &= ~M_PKTHDR;
+ }
+ }
+ space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
+ do {
+ count = MIN(MIN(MAX(len, max_protohdr), space), n->m_len);
+ bcopy(MTOD(n, caddr_t), MTOD(m, caddr_t) + m->m_len,
+ (unsigned)count);
+ len -= count;
+ m->m_len += count;
+ n->m_len -= count;
+ space -= count;
+ if (n->m_len != 0) {
+ n->m_data += count;
+ } else {
+ n = m_free(n);
+ }
+ } while (len > 0 && n != NULL);
+ if (len > 0) {
+ (void) m_free(m);
+ goto bad;
+ }
+ m->m_next = n;
+ return m;
+bad:
+ m_freem(n);
+ MPFail++;
+ return 0;
+}
+
+/*
+ * Like m_pullup(), except a new mbuf is always allocated, and we allow
+ * the amount of empty space before the data in the new mbuf to be specified
+ * (in the event that the caller expects to prepend later).
+ */
+__private_extern__ int MSFail = 0;
+
+__private_extern__ struct mbuf *
+m_copyup(struct mbuf *n, int len, int dstoff)
+{
+ struct mbuf *m;
+ int count, space;
+
+ VERIFY(len >= 0 && dstoff >= 0);
+
+ if (len > (MHLEN - dstoff)) {
+ goto bad;
+ }
+ MGET(m, M_DONTWAIT, n->m_type);
+ if (m == NULL) {
+ goto bad;
+ }
+ m->m_len = 0;
+ if (n->m_flags & M_PKTHDR) {
+ m_copy_pkthdr(m, n);
+ n->m_flags &= ~M_PKTHDR;
+ }
+ m->m_data += dstoff;
+ space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
+ do {
+ count = min(min(max(len, max_protohdr), space), n->m_len);
+ memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t),
+ (unsigned)count);
+ len -= count;
+ m->m_len += count;
+ n->m_len -= count;
+ space -= count;
+ if (n->m_len) {
+ n->m_data += count;
+ } else {
+ n = m_free(n);
+ }
+ } while (len > 0 && n);
+ if (len > 0) {
+ (void) m_free(m);
+ goto bad;
+ }
+ m->m_next = n;
+ return m;
+bad:
+ m_freem(n);
+ MSFail++;
+ return NULL;
+}
+
+/*
+ * Partition an mbuf chain in two pieces, returning the tail --
+ * all but the first len0 bytes. In case of failure, it returns NULL and
+ * attempts to restore the chain to its original state.
+ */
+struct mbuf *
+m_split(struct mbuf *m0, int len0, int wait)
+{
+ return m_split0(m0, len0, wait, 1);
+}
+
+static struct mbuf *
+m_split0(struct mbuf *m0, int len0, int wait, int copyhdr)
+{
+ struct mbuf *m, *n;
+ unsigned len = len0, remain;
+
+ /*
+ * First iterate to the mbuf which contains the first byte of
+ * data at offset len0
+ */
+ for (m = m0; m && len > m->m_len; m = m->m_next) {
+ len -= m->m_len;
+ }
+ if (m == NULL) {
+ return NULL;
+ }
+ /*
+ * len effectively is now the offset in the current
+ * mbuf where we have to perform split.
+ *
+ * remain becomes the tail length.
+ * Note that len can also be == m->m_len
+ */
+ remain = m->m_len - len;
+
+ /*
+ * If current mbuf len contains the entire remaining offset len,
+ * just make the second mbuf chain pointing to next mbuf onwards
+ * and return after making necessary adjustments
+ */
+ if (copyhdr && (m0->m_flags & M_PKTHDR) && remain == 0) {
+ _MGETHDR(n, wait, m0->m_type);
+ if (n == NULL) {
+ return NULL;
+ }
+ n->m_next = m->m_next;
+ m->m_next = NULL;
+ n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
+ n->m_pkthdr.len = m0->m_pkthdr.len - len0;
+ m0->m_pkthdr.len = len0;
+ return n;
+ }
+ if (copyhdr && (m0->m_flags & M_PKTHDR)) {
+ _MGETHDR(n, wait, m0->m_type);
+ if (n == NULL) {
+ return NULL;
+ }
+ n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
+ n->m_pkthdr.len = m0->m_pkthdr.len - len0;
+ m0->m_pkthdr.len = len0;
+
+ /*
+ * If current points to external storage
+ * then it can be shared by making last mbuf
+ * of head chain and first mbuf of current chain
+ * pointing to different data offsets
+ */
+ if (m->m_flags & M_EXT) {
+ goto extpacket;
+ }
+ if (remain > MHLEN) {
+ /* m can't be the lead packet */
+ MH_ALIGN(n, 0);
+ n->m_next = m_split(m, len, wait);
+ if (n->m_next == NULL) {
+ (void) m_free(n);
+ return NULL;
+ } else {
+ return n;
+ }
+ } else {
+ MH_ALIGN(n, remain);
+ }
+ } else if (remain == 0) {
+ n = m->m_next;
+ m->m_next = NULL;
+ return n;
+ } else {
+ _MGET(n, wait, m->m_type);
+ if (n == NULL) {
+ return NULL;
+ }
+
+ if ((m->m_flags & M_EXT) == 0) {
+ VERIFY(remain <= MLEN);
+ M_ALIGN(n, remain);
+ }
+ }
+extpacket:
+ if (m->m_flags & M_EXT) {
+ n->m_flags |= M_EXT;
+ n->m_ext = m->m_ext;
+ m_incref(m);
+ n->m_data = m->m_data + len;
+ } else {
+ bcopy(MTOD(m, caddr_t) + len, MTOD(n, caddr_t), remain);
+ }
+ n->m_len = remain;
+ m->m_len = len;
+ n->m_next = m->m_next;
+ m->m_next = NULL;
+ return n;
+}
+
+/*
+ * Routine to copy from device local memory into mbufs.
+ */
+struct mbuf *
+m_devget(char *buf, int totlen, int off0, struct ifnet *ifp,
+ void (*copy)(const void *, void *, size_t))
+{
+ struct mbuf *m;
+ struct mbuf *top = NULL, **mp = ⊤
+ int off = off0, len;
+ char *cp;
+ char *epkt;
+
+ cp = buf;
+ epkt = cp + totlen;
+ if (off) {
+ /*
+ * If 'off' is non-zero, packet is trailer-encapsulated,
+ * so we have to skip the type and length fields.
+ */
+ cp += off + 2 * sizeof(u_int16_t);
+ totlen -= 2 * sizeof(u_int16_t);
+ }
+ _MGETHDR(m, M_DONTWAIT, MT_DATA);
+ if (m == NULL) {
+ return NULL;
+ }
+ m->m_pkthdr.rcvif = ifp;
+ m->m_pkthdr.len = totlen;
+ m->m_len = MHLEN;
+
+ while (totlen > 0) {
+ if (top != NULL) {
+ _MGET(m, M_DONTWAIT, MT_DATA);
+ if (m == NULL) {
+ m_freem(top);
+ return NULL;
+ }
+ m->m_len = MLEN;
+ }
+ len = MIN(totlen, epkt - cp);
+ if (len >= MINCLSIZE) {
+ MCLGET(m, M_DONTWAIT);
+ if (m->m_flags & M_EXT) {
+ m->m_len = len = MIN(len, m_maxsize(MC_CL));
+ } else {
+ /* give up when it's out of cluster mbufs */
+ if (top != NULL) {
+ m_freem(top);
+ }
+ m_freem(m);
+ return NULL;
+ }
+ } else {
+ /*
+ * Place initial small packet/header at end of mbuf.
+ */
+ if (len < m->m_len) {
+ if (top == NULL &&
+ len + max_linkhdr <= m->m_len) {
+ m->m_data += max_linkhdr;
+ }
+ m->m_len = len;
+ } else {
+ len = m->m_len;
+ }
+ }
+ if (copy) {
+ copy(cp, MTOD(m, caddr_t), (unsigned)len);
+ } else {
+ bcopy(cp, MTOD(m, caddr_t), (unsigned)len);
+ }
+ cp += len;
+ *mp = m;
+ mp = &m->m_next;
+ totlen -= len;
+ if (cp == epkt) {
+ cp = buf;
+ }
+ }
+ return top;
+}
+
+#ifndef MBUF_GROWTH_NORMAL_THRESH
+#define MBUF_GROWTH_NORMAL_THRESH 25
+#endif
+
+/*
+ * Cluster freelist allocation check.
+ */
+static int
+m_howmany(int num, size_t bufsize)
+{
+ int i = 0, j = 0;
+ u_int32_t m_mbclusters, m_clusters, m_bigclusters, m_16kclusters;
+ u_int32_t m_mbfree, m_clfree, m_bigclfree, m_16kclfree;
+ u_int32_t sumclusters, freeclusters;
+ u_int32_t percent_pool, percent_kmem;
+ u_int32_t mb_growth, mb_growth_thresh;
+
+ VERIFY(bufsize == m_maxsize(MC_BIGCL) ||
+ bufsize == m_maxsize(MC_16KCL));
+
+ LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
+
+ /* Numbers in 2K cluster units */
+ m_mbclusters = m_total(MC_MBUF) >> NMBPCLSHIFT;
+ m_clusters = m_total(MC_CL);
+ m_bigclusters = m_total(MC_BIGCL) << NCLPBGSHIFT;
+ m_16kclusters = m_total(MC_16KCL);
+ sumclusters = m_mbclusters + m_clusters + m_bigclusters;
+
+ m_mbfree = m_infree(MC_MBUF) >> NMBPCLSHIFT;
+ m_clfree = m_infree(MC_CL);
+ m_bigclfree = m_infree(MC_BIGCL) << NCLPBGSHIFT;
+ m_16kclfree = m_infree(MC_16KCL);
+ freeclusters = m_mbfree + m_clfree + m_bigclfree;
+
+ /* Bail if we've maxed out the mbuf memory map */
+ if ((bufsize == m_maxsize(MC_BIGCL) && sumclusters >= nclusters) ||
+ (njcl > 0 && bufsize == m_maxsize(MC_16KCL) &&
+ (m_16kclusters << NCLPJCLSHIFT) >= njcl)) {
+ mbwdog_logger("maxed out nclusters (%u >= %u) or njcl (%u >= %u)",
+ sumclusters, nclusters,
+ (m_16kclusters << NCLPJCLSHIFT), njcl);
+ return 0;
+ }
+
+ if (bufsize == m_maxsize(MC_BIGCL)) {
+ /* Under minimum */
+ if (m_bigclusters < m_minlimit(MC_BIGCL)) {
+ return m_minlimit(MC_BIGCL) - m_bigclusters;
+ }
+
+ percent_pool =
+ ((sumclusters - freeclusters) * 100) / sumclusters;
+ percent_kmem = (sumclusters * 100) / nclusters;
+
+ /*
+ * If a light/normal user, grow conservatively (75%)
+ * If a heavy user, grow aggressively (50%)
+ */
+ if (percent_kmem < MBUF_GROWTH_NORMAL_THRESH) {
+ mb_growth = MB_GROWTH_NORMAL;
+ } else {
+ mb_growth = MB_GROWTH_AGGRESSIVE;
+ }
+
+ if (percent_kmem < 5) {
+ /* For initial allocations */
+ i = num;
+ } else {
+ /* Return if >= MBIGCL_LOWAT clusters available */
+ if (m_infree(MC_BIGCL) >= MBIGCL_LOWAT &&
+ m_total(MC_BIGCL) >=
+ MBIGCL_LOWAT + m_minlimit(MC_BIGCL)) {
+ return 0;
+ }
+
+ /* Ensure at least num clusters are accessible */
+ if (num >= m_infree(MC_BIGCL)) {
+ i = num - m_infree(MC_BIGCL);
+ }
+ if (num > m_total(MC_BIGCL) - m_minlimit(MC_BIGCL)) {
+ j = num - (m_total(MC_BIGCL) -
+ m_minlimit(MC_BIGCL));
+ }
+
+ i = MAX(i, j);
+
+ /*
+ * Grow pool if percent_pool > 75 (normal growth)
+ * or percent_pool > 50 (aggressive growth).
+ */
+ mb_growth_thresh = 100 - (100 / (1 << mb_growth));
+ if (percent_pool > mb_growth_thresh) {
+ j = ((sumclusters + num) >> mb_growth) -
+ freeclusters;
+ }
+ i = MAX(i, j);
+ }
+
+ /* Check to ensure we didn't go over limits */
+ if (i + m_bigclusters >= m_maxlimit(MC_BIGCL)) {
+ i = m_maxlimit(MC_BIGCL) - m_bigclusters;
+ }
+ if ((i << 1) + sumclusters >= nclusters) {
+ i = (nclusters - sumclusters) >> 1;
+ }
+ VERIFY((m_total(MC_BIGCL) + i) <= m_maxlimit(MC_BIGCL));
+ VERIFY(sumclusters + (i << 1) <= nclusters);
+ } else { /* 16K CL */
+ VERIFY(njcl > 0);
+ /* Ensure at least num clusters are available */
+ if (num >= m_16kclfree) {
+ i = num - m_16kclfree;
+ }
+
+ /* Always grow 16KCL pool aggressively */
+ if (((m_16kclusters + num) >> 1) > m_16kclfree) {
+ j = ((m_16kclusters + num) >> 1) - m_16kclfree;
+ }
+ i = MAX(i, j);
+
+ /* Check to ensure we don't go over limit */
+ if ((i + m_total(MC_16KCL)) >= m_maxlimit(MC_16KCL)) {
+ i = m_maxlimit(MC_16KCL) - m_total(MC_16KCL);
+ }
+ }
+ return i;
+}
+/*
+ * Return the number of bytes in the mbuf chain, m.
+ */
+unsigned int
+m_length(struct mbuf *m)
+{
+ struct mbuf *m0;
+ unsigned int pktlen;
+
+ if (m->m_flags & M_PKTHDR) {
+ return m->m_pkthdr.len;
+ }
+
+ pktlen = 0;
+ for (m0 = m; m0 != NULL; m0 = m0->m_next) {
+ pktlen += m0->m_len;
+ }
+ return pktlen;
+}
+
+/*
+ * Copy data from a buffer back into the indicated mbuf chain,
+ * starting "off" bytes from the beginning, extending the mbuf
+ * chain if necessary.
+ */
+void
+m_copyback(struct mbuf *m0, int off, int len, const void *cp)
+{
+#if DEBUG
+ struct mbuf *origm = m0;
+ int error;
+#endif /* DEBUG */
+
+ if (m0 == NULL) {
+ return;
+ }
+
+#if DEBUG
+ error =
+#endif /* DEBUG */
+ m_copyback0(&m0, off, len, cp,
+ M_COPYBACK0_COPYBACK | M_COPYBACK0_EXTEND, M_DONTWAIT);
+
+#if DEBUG
+ if (error != 0 || (m0 != NULL && origm != m0)) {
+ panic("m_copyback");
+ }
+#endif /* DEBUG */
+}
+
+struct mbuf *
+m_copyback_cow(struct mbuf *m0, int off, int len, const void *cp, int how)
+{
+ int error;
+
+ /* don't support chain expansion */
+ VERIFY(off + len <= m_length(m0));
+
+ error = m_copyback0(&m0, off, len, cp,
+ M_COPYBACK0_COPYBACK | M_COPYBACK0_COW, how);
+ if (error) {
+ /*
+ * no way to recover from partial success.
+ * just free the chain.
+ */
+ m_freem(m0);
+ return NULL;
+ }
+ return m0;
+}
+
+/*
+ * m_makewritable: ensure the specified range writable.
+ */
+int
+m_makewritable(struct mbuf **mp, int off, int len, int how)
+{
+ int error;
+#if DEBUG
+ struct mbuf *n;
+ int origlen, reslen;
+
+ origlen = m_length(*mp);
+#endif /* DEBUG */
+
+#if 0 /* M_COPYALL is large enough */
+ if (len == M_COPYALL) {
+ len = m_length(*mp) - off; /* XXX */
+ }
+#endif
+
+ error = m_copyback0(mp, off, len, NULL,
+ M_COPYBACK0_PRESERVE | M_COPYBACK0_COW, how);
+
+#if DEBUG
+ reslen = 0;
+ for (n = *mp; n; n = n->m_next) {
+ reslen += n->m_len;
+ }
+ if (origlen != reslen) {
+ panic("m_makewritable: length changed");
+ }
+ if (((*mp)->m_flags & M_PKTHDR) && reslen != (*mp)->m_pkthdr.len) {
+ panic("m_makewritable: inconsist");
+ }
+#endif /* DEBUG */
+
+ return error;
+}
+
+static int
+m_copyback0(struct mbuf **mp0, int off, int len, const void *vp, int flags,
+ int how)
+{
+ int mlen;
+ struct mbuf *m, *n;
+ struct mbuf **mp;
+ int totlen = 0;
+ const char *cp = vp;
+
+ VERIFY(mp0 != NULL);
+ VERIFY(*mp0 != NULL);
+ VERIFY((flags & M_COPYBACK0_PRESERVE) == 0 || cp == NULL);
+ VERIFY((flags & M_COPYBACK0_COPYBACK) == 0 || cp != NULL);
+
+ /*
+ * we don't bother to update "totlen" in the case of M_COPYBACK0_COW,
+ * assuming that M_COPYBACK0_EXTEND and M_COPYBACK0_COW are exclusive.
+ */
+
+ VERIFY((~flags & (M_COPYBACK0_EXTEND | M_COPYBACK0_COW)) != 0);
+
+ mp = mp0;
+ m = *mp;
+ while (off > (mlen = m->m_len)) {
+ off -= mlen;
+ totlen += mlen;
+ if (m->m_next == NULL) {
+ int tspace;
+extend:
+ if (!(flags & M_COPYBACK0_EXTEND)) {
+ goto out;
+ }
+
+ /*
+ * try to make some space at the end of "m".
+ */
+
+ mlen = m->m_len;
+ if (off + len >= MINCLSIZE &&
+ !(m->m_flags & M_EXT) && m->m_len == 0) {
+ MCLGET(m, how);
+ }
+ tspace = M_TRAILINGSPACE(m);
+ if (tspace > 0) {
+ tspace = MIN(tspace, off + len);
+ VERIFY(tspace > 0);
+ bzero(mtod(m, char *) + m->m_len,
+ MIN(off, tspace));
+ m->m_len += tspace;
+ off += mlen;
+ totlen -= mlen;
+ continue;
+ }
+
+ /*
+ * need to allocate an mbuf.
+ */
+
+ if (off + len >= MINCLSIZE) {
+ n = m_getcl(how, m->m_type, 0);
+ } else {
+ n = _M_GET(how, m->m_type);
+ }
+ if (n == NULL) {
+ goto out;
+ }
+ n->m_len = 0;
+ n->m_len = MIN(M_TRAILINGSPACE(n), off + len);
+ bzero(mtod(n, char *), MIN(n->m_len, off));
+ m->m_next = n;
+ }
+ mp = &m->m_next;
+ m = m->m_next;
+ }
+ while (len > 0) {
+ mlen = m->m_len - off;
+ if (mlen != 0 && m_mclhasreference(m)) {
+ char *datap;
+ int eatlen;
+
+ /*
+ * this mbuf is read-only.
+ * allocate a new writable mbuf and try again.
+ */
+
+#if DIAGNOSTIC
+ if (!(flags & M_COPYBACK0_COW)) {
+ panic("m_copyback0: read-only");
+ }
+#endif /* DIAGNOSTIC */
+
+ /*
+ * if we're going to write into the middle of
+ * a mbuf, split it first.
+ */
+ if (off > 0 && len < mlen) {
+ n = m_split0(m, off, how, 0);
+ if (n == NULL) {
+ goto enobufs;
+ }
+ m->m_next = n;
+ mp = &m->m_next;
+ m = n;
+ off = 0;
+ continue;
+ }
+
+ /*
+ * XXX TODO coalesce into the trailingspace of
+ * the previous mbuf when possible.
+ */
+
+ /*
+ * allocate a new mbuf. copy packet header if needed.
+ */
+ n = _M_GET(how, m->m_type);
+ if (n == NULL) {
+ goto enobufs;
+ }
+ if (off == 0 && (m->m_flags & M_PKTHDR)) {
+ M_COPY_PKTHDR(n, m);
+ n->m_len = MHLEN;
+ } else {
+ if (len >= MINCLSIZE) {
+ MCLGET(n, M_DONTWAIT);
+ }
+ n->m_len =
+ (n->m_flags & M_EXT) ? MCLBYTES : MLEN;
+ }
+ if (n->m_len > len) {
+ n->m_len = len;
+ }
+
+ /*
+ * free the region which has been overwritten.
+ * copying data from old mbufs if requested.
+ */
+ if (flags & M_COPYBACK0_PRESERVE) {
+ datap = mtod(n, char *);
+ } else {
+ datap = NULL;
+ }
+ eatlen = n->m_len;
+ VERIFY(off == 0 || eatlen >= mlen);
+ if (off > 0) {
+ VERIFY(len >= mlen);
+ m->m_len = off;
+ m->m_next = n;
+ if (datap) {
+ m_copydata(m, off, mlen, datap);
+ datap += mlen;
+ }
+ eatlen -= mlen;
+ mp = &m->m_next;
+ m = m->m_next;
+ }
+ while (m != NULL && m_mclhasreference(m) &&
+ n->m_type == m->m_type && eatlen > 0) {
+ mlen = MIN(eatlen, m->m_len);
+ if (datap) {
+ m_copydata(m, 0, mlen, datap);
+ datap += mlen;
+ }
+ m->m_data += mlen;
+ m->m_len -= mlen;
+ eatlen -= mlen;
+ if (m->m_len == 0) {
+ *mp = m = m_free(m);
+ }
+ }
+ if (eatlen > 0) {
+ n->m_len -= eatlen;
+ }
+ n->m_next = m;
+ *mp = m = n;
+ continue;
+ }
+ mlen = MIN(mlen, len);
+ if (flags & M_COPYBACK0_COPYBACK) {
+ bcopy(cp, mtod(m, caddr_t) + off, (unsigned)mlen);
+ cp += mlen;
+ }
+ len -= mlen;
+ mlen += off;
+ off = 0;
+ totlen += mlen;
+ if (len == 0) {
+ break;
+ }
+ if (m->m_next == NULL) {
+ goto extend;
+ }
+ mp = &m->m_next;
projects / apple / xnu.git / blobdiff