]> git.saurik.com Git - apple/xnu.git/blob - bsd/kern/uipc_mbuf.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-792.1.5.tar.gz
[apple/xnu.git] / bsd / kern / uipc_mbuf.c
1 /*
2 * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * The contents of this file constitute Original Code as defined in and
7 * are subject to the Apple Public Source License Version 1.1 (the
8 * "License"). You may not use this file except in compliance with the
9 * License. Please obtain a copy of the License at
10 * http://www.apple.com/publicsource and read it before using this file.
11 *
12 * This Original Code and all software distributed under the License are
13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17 * License for the specific language governing rights and limitations
18 * under the License.
19 *
20 * @APPLE_LICENSE_HEADER_END@
21 */
22 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
23 /*
24 * Copyright (c) 1982, 1986, 1988, 1991, 1993
25 * The Regents of the University of California. All rights reserved.
26 *
27 * Redistribution and use in source and binary forms, with or without
28 * modification, are permitted provided that the following conditions
29 * are met:
30 * 1. Redistributions of source code must retain the above copyright
31 * notice, this list of conditions and the following disclaimer.
32 * 2. Redistributions in binary form must reproduce the above copyright
33 * notice, this list of conditions and the following disclaimer in the
34 * documentation and/or other materials provided with the distribution.
35 * 3. All advertising materials mentioning features or use of this software
36 * must display the following acknowledgement:
37 * This product includes software developed by the University of
38 * California, Berkeley and its contributors.
39 * 4. Neither the name of the University nor the names of its contributors
40 * may be used to endorse or promote products derived from this software
41 * without specific prior written permission.
42 *
43 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
44 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
45 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
46 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
47 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
48 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
49 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
50 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
51 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
52 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
53 * SUCH DAMAGE.
54 *
55 * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94
56 */
57 /* HISTORY
58 *
59 * 10/15/97 Annette DeSchon (deschon@apple.com)
60 * Fixed bug in which all cluster mbufs were broken up
61 * into regular mbufs: Some clusters are now reserved.
62 * When a cluster is needed, regular mbufs are no longer
63 * used. (Radar 1683621)
64 * 20-May-95 Mac Gillon (mgillon) at NeXT
65 * New version based on 4.4
66 */
67
68 #include <sys/param.h>
69 #include <sys/systm.h>
70 #include <sys/malloc.h>
71 #include <sys/mbuf.h>
72 #include <sys/kernel.h>
73 #include <sys/sysctl.h>
74 #include <sys/syslog.h>
75 #include <sys/protosw.h>
76 #include <sys/domain.h>
77
78 #include <kern/queue.h>
79 #include <kern/kern_types.h>
80 #include <kern/sched_prim.h>
81
82 #include <IOKit/IOMapper.h>
83
84 extern vm_offset_t kmem_mb_alloc(vm_map_t , int );
85 extern boolean_t PE_parse_boot_arg(const char *, void *);
86
87 #define _MCLREF(p) (++mclrefcnt[mtocl(p)])
88 #define _MCLUNREF(p) (--mclrefcnt[mtocl(p)] == 0)
89 #define _M_CLEAR_PKTHDR(mbuf_ptr) (mbuf_ptr)->m_pkthdr.rcvif = NULL; \
90 (mbuf_ptr)->m_pkthdr.len = 0; \
91 (mbuf_ptr)->m_pkthdr.header = NULL; \
92 (mbuf_ptr)->m_pkthdr.csum_flags = 0; \
93 (mbuf_ptr)->m_pkthdr.csum_data = 0; \
94 (mbuf_ptr)->m_pkthdr.aux = (struct mbuf*)NULL; \
95 (mbuf_ptr)->m_pkthdr.vlan_tag = 0; \
96 (mbuf_ptr)->m_pkthdr.socket_id = 0; \
97 SLIST_INIT(&(mbuf_ptr)->m_pkthdr.tags);
98
99 /* kernel translater */
100 extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
101
102 lck_mtx_t * mbuf_mlock;
103 lck_grp_t * mbuf_mlock_grp;
104 lck_grp_attr_t * mbuf_mlock_grp_attr;
105 lck_attr_t * mbuf_mlock_attr;
106 extern lck_mtx_t *domain_proto_mtx;
107
108 struct mbuf *mfree; /* mbuf free list */
109 struct mbuf *mfreelater; /* mbuf deallocation list */
110 extern vm_map_t mb_map; /* special map */
111 int m_want; /* sleepers on mbufs */
112 short *mclrefcnt; /* mapped cluster reference counts */
113 int *mcl_paddr;
114 static ppnum_t mcl_paddr_base; /* Handle returned by IOMapper::iovmAlloc() */
115 union mcluster *mclfree; /* mapped cluster free list */
116 union mbigcluster *mbigfree; /* mapped cluster free list */
117 int max_linkhdr; /* largest link-level header */
118 int max_protohdr; /* largest protocol header */
119 int max_hdr; /* largest link+protocol header */
120 int max_datalen; /* MHLEN - max_hdr */
121 struct mbstat mbstat; /* statistics */
122 union mcluster *mbutl; /* first mapped cluster address */
123 union mcluster *embutl; /* ending virtual address of mclusters */
124
125 static int nclpp; /* # clusters per physical page */
126
127 static int m_howmany(int, size_t );
128 void m_reclaim(void);
129 static int m_clalloc(const int , const int, const size_t, int);
130 int do_reclaim = 0;
131
132 #define MF_NOWAIT 0x1
133 #define MF_BIG 0x2
134
135 /* The number of cluster mbufs that are allocated, to start. */
136 #define MINCL max(16, 2)
137
138 static int mbuf_expand_thread_wakeup = 0;
139 static int mbuf_expand_mcl = 0;
140 static int mbuf_expand_big = 0;
141 static int mbuf_expand_thread_initialized = 0;
142
143 static void mbuf_expand_thread_init(void);
144 static void mbuf_expand_thread(void);
145 static int m_expand(int );
146 static caddr_t m_bigalloc(int );
147 static void m_bigfree(caddr_t , u_int , caddr_t );
148 static struct mbuf * m_mbigget(struct mbuf *, int );
149 void mbinit(void);
150 static void m_range_check(void *addr);
151
152
153 #if 0
154 static int mfree_munge = 0;
155 #if 0
156 #define _MFREE_MUNGE(m) { \
157 if (mfree_munge) \
158 { int i; \
159 vm_offset_t *element = (vm_offset_t *)(m); \
160 for (i = 0; \
161 i < sizeof(struct mbuf)/sizeof(vm_offset_t); \
162 i++) \
163 (element)[i] = 0xdeadbeef; \
164 } \
165 }
166 #else
167 void
168 munge_mbuf(struct mbuf *m)
169 {
170 int i;
171 vm_offset_t *element = (vm_offset_t *)(m);
172 for (i = 0;
173 i < sizeof(struct mbuf)/sizeof(vm_offset_t);
174 i++)
175 (element)[i] = 0xdeadbeef;
176 }
177 #define _MFREE_MUNGE(m) { \
178 if (mfree_munge) \
179 munge_mbuf(m); \
180 }
181 #endif
182 #else
183 #define _MFREE_MUNGE(m)
184 #endif
185
186
187 #define _MINTGET(m, type) { \
188 MBUF_LOCK(); \
189 if (((m) = mfree) != 0) { \
190 MCHECK(m); \
191 ++mclrefcnt[mtocl(m)]; \
192 mbstat.m_mtypes[MT_FREE]--; \
193 mbstat.m_mtypes[(type)]++; \
194 mfree = (m)->m_next; \
195 } \
196 MBUF_UNLOCK(); \
197 }
198
199
200 static void
201 m_range_check(void *addr)
202 {
203 if (addr && (addr < (void *)mbutl || addr >= (void *)embutl))
204 panic("mbuf address out of range 0x%x", addr);
205 }
206
207 __private_extern__ void
208 mbinit(void)
209 {
210 int m;
211 int initmcl = 32;
212 int mcl_pages;
213
214 if (nclpp)
215 return;
216 nclpp = round_page_32(MCLBYTES) / MCLBYTES; /* see mbufgc() */
217 if (nclpp < 1) nclpp = 1;
218 mbuf_mlock_grp_attr = lck_grp_attr_alloc_init();
219 lck_grp_attr_setdefault(mbuf_mlock_grp_attr);
220
221 mbuf_mlock_grp = lck_grp_alloc_init("mbuf", mbuf_mlock_grp_attr);
222 mbuf_mlock_attr = lck_attr_alloc_init();
223 lck_attr_setdefault(mbuf_mlock_attr);
224
225 mbuf_mlock = lck_mtx_alloc_init(mbuf_mlock_grp, mbuf_mlock_attr);
226
227 mbstat.m_msize = MSIZE;
228 mbstat.m_mclbytes = MCLBYTES;
229 mbstat.m_minclsize = MINCLSIZE;
230 mbstat.m_mlen = MLEN;
231 mbstat.m_mhlen = MHLEN;
232 mbstat.m_bigmclbytes = NBPG;
233
234 if (nmbclusters == 0)
235 nmbclusters = NMBCLUSTERS;
236 MALLOC(mclrefcnt, short *, nmbclusters * sizeof (short),
237 M_TEMP, M_WAITOK);
238 if (mclrefcnt == 0)
239 panic("mbinit");
240 for (m = 0; m < nmbclusters; m++)
241 mclrefcnt[m] = -1;
242
243 /* Calculate the number of pages assigned to the cluster pool */
244 mcl_pages = nmbclusters/(NBPG/CLBYTES);
245 MALLOC(mcl_paddr, int *, mcl_pages * sizeof(int), M_TEMP, M_WAITOK);
246 if (mcl_paddr == 0)
247 panic("mbinit1");
248 /* Register with the I/O Bus mapper */
249 mcl_paddr_base = IOMapperIOVMAlloc(mcl_pages);
250 bzero((char *)mcl_paddr, mcl_pages * sizeof(int));
251
252 embutl = (union mcluster *)((unsigned char *)mbutl + (nmbclusters * MCLBYTES));
253
254 PE_parse_boot_arg("initmcl", &initmcl);
255
256 if (m_clalloc(max(NBPG/CLBYTES, 1) * initmcl, M_WAIT, MCLBYTES, 0) == 0)
257 goto bad;
258 MBUF_UNLOCK();
259
260 (void) kernel_thread(kernel_task, mbuf_expand_thread_init);
261
262 return;
263 bad:
264 panic("mbinit");
265 }
266
267 /*
268 * Allocate some number of mbuf clusters
269 * and place on cluster free list.
270 * Take the mbuf lock (if not already locked) and do not release it
271 */
272 /* ARGSUSED */
273 static int
274 m_clalloc(
275 const int num,
276 const int nowait,
277 const size_t bufsize,
278 int locked)
279 {
280 int i;
281 vm_size_t size = 0;
282 int numpages = 0;
283 vm_offset_t page = 0;
284
285 if (locked == 0)
286 MBUF_LOCK();
287 /*
288 * Honor the caller's wish to block or not block.
289 * We have a way to grow the pool asynchronously,
290 * by kicking the dlil_input_thread.
291 */
292 i = m_howmany(num, bufsize);
293 if (i == 0 || nowait == M_DONTWAIT)
294 goto out;
295
296 MBUF_UNLOCK();
297 size = round_page_32(i * bufsize);
298 page = kmem_mb_alloc(mb_map, size);
299
300 if (page == 0) {
301 size = NBPG; /* Try for 1 if failed */
302 page = kmem_mb_alloc(mb_map, size);
303 }
304 MBUF_LOCK();
305
306 if (page) {
307 numpages = size / NBPG;
308 for (i = 0; i < numpages; i++, page += NBPG) {
309 if (((int)page & PGOFSET) == 0) {
310 ppnum_t offset = ((char *)page - (char *)mbutl)/NBPG;
311 ppnum_t new_page = pmap_find_phys(kernel_pmap, (vm_address_t) page);
312
313 /*
314 * In the case of no mapper being available
315 * the following code nops and returns the
316 * input page, if there is a mapper the I/O
317 * page appropriate is returned.
318 */
319 new_page = IOMapperInsertPage(mcl_paddr_base, offset, new_page);
320 mcl_paddr[offset] = new_page << 12;
321 }
322 if (bufsize == MCLBYTES) {
323 union mcluster *mcl = (union mcluster *)page;
324
325 if (++mclrefcnt[mtocl(mcl)] != 0)
326 panic("m_clalloc already there");
327 mcl->mcl_next = mclfree;
328 mclfree = mcl++;
329 if (++mclrefcnt[mtocl(mcl)] != 0)
330 panic("m_clalloc already there");
331 mcl->mcl_next = mclfree;
332 mclfree = mcl++;
333 } else {
334 union mbigcluster *mbc = (union mbigcluster *)page;
335
336 if (++mclrefcnt[mtocl(mbc)] != 0)
337 panic("m_clalloc already there");
338 if (++mclrefcnt[mtocl(mbc) + 1] != 0)
339 panic("m_clalloc already there");
340
341 mbc->mbc_next = mbigfree;
342 mbigfree = mbc;
343 }
344 }
345 if (bufsize == MCLBYTES) {
346 int numcl = numpages << 1;
347 mbstat.m_clfree += numcl;
348 mbstat.m_clusters += numcl;
349 return (numcl);
350 } else {
351 mbstat.m_bigclfree += numpages;
352 mbstat.m_bigclusters += numpages;
353 return (numpages);
354 }
355 } /* else ... */
356 out:
357 /*
358 * When non-blocking we kick a thread if we havve to grow the
359 * pool or if the number of free clusters is less than requested.
360 */
361 if (bufsize == MCLBYTES) {
362 if (i > 0) {
363 /* Remember total number of clusters needed at this time */
364 i += mbstat.m_clusters;
365 if (i > mbuf_expand_mcl) {
366 mbuf_expand_mcl = i;
367 if (mbuf_expand_thread_initialized)
368 wakeup((caddr_t)&mbuf_expand_thread_wakeup);
369 }
370 }
371
372 if (mbstat.m_clfree >= num)
373 return 1;
374 } else {
375 if (i > 0) {
376 /* Remember total number of 4KB clusters needed at this time */
377 i += mbstat.m_bigclusters;
378 if (i > mbuf_expand_big) {
379 mbuf_expand_big = i;
380 if (mbuf_expand_thread_initialized)
381 wakeup((caddr_t)&mbuf_expand_thread_wakeup);
382 }
383 }
384
385 if (mbstat.m_bigclfree >= num)
386 return 1;
387 }
388 return 0;
389 }
390
391 /*
392 * Add more free mbufs by cutting up a cluster.
393 */
394 static int
395 m_expand(int canwait)
396 {
397 caddr_t mcl;
398
399 if (mbstat.m_clfree < (mbstat.m_clusters >> 4)) {
400 /*
401 * 1/16th of the total number of cluster mbufs allocated is
402 * reserved for large packets. The number reserved must
403 * always be < 1/2, or future allocation will be prevented.
404 */
405 (void)m_clalloc(1, canwait, MCLBYTES, 0);
406 MBUF_UNLOCK();
407 if (mbstat.m_clfree < (mbstat.m_clusters >> 4))
408 return 0;
409 }
410
411 MCLALLOC(mcl, canwait);
412 if (mcl) {
413 struct mbuf *m = (struct mbuf *)mcl;
414 int i = NMBPCL;
415 MBUF_LOCK();
416 mbstat.m_mtypes[MT_FREE] += i;
417 mbstat.m_mbufs += i;
418 while (i--) {
419 _MFREE_MUNGE(m);
420 m->m_type = MT_FREE;
421 m->m_next = mfree;
422 mfree = m++;
423 }
424 i = m_want;
425 m_want = 0;
426 MBUF_UNLOCK();
427 if (i) wakeup((caddr_t)&mfree);
428 return 1;
429 }
430 return 0;
431 }
432
433 /*
434 * When MGET failes, ask protocols to free space when short of memory,
435 * then re-attempt to allocate an mbuf.
436 */
437 struct mbuf *
438 m_retry(
439 int canwait,
440 int type)
441 {
442 struct mbuf *m;
443 int wait;
444
445 for (;;) {
446 (void) m_expand(canwait);
447 _MINTGET(m, type);
448 if (m) {
449 (m)->m_next = (m)->m_nextpkt = 0;
450 (m)->m_type = (type);
451 (m)->m_data = (m)->m_dat;
452 (m)->m_flags = 0;
453 (m)->m_len = 0;
454 }
455 if (m || canwait == M_DONTWAIT)
456 break;
457 MBUF_LOCK();
458 wait = m_want++;
459 mbuf_expand_mcl++;
460 if (wait == 0)
461 mbstat.m_drain++;
462 else
463 mbstat.m_wait++;
464 MBUF_UNLOCK();
465
466 if (mbuf_expand_thread_initialized)
467 wakeup((caddr_t)&mbuf_expand_thread_wakeup);
468
469 if (wait == 0) {
470 m_reclaim();
471 } else {
472 struct timespec ts;
473 ts.tv_sec = 1;
474 ts.tv_nsec = 0;
475 (void) msleep((caddr_t)&mfree, 0, (PZERO-1) | PDROP, "m_retry", &ts);
476 }
477 }
478 if (m == 0)
479 mbstat.m_drops++;
480 return (m);
481 }
482
483 /*
484 * As above; retry an MGETHDR.
485 */
486 struct mbuf *
487 m_retryhdr(
488 int canwait,
489 int type)
490 {
491 struct mbuf *m;
492
493 if ((m = m_retry(canwait, type))) {
494 m->m_next = m->m_nextpkt = 0;
495 m->m_flags |= M_PKTHDR;
496 m->m_data = m->m_pktdat;
497 _M_CLEAR_PKTHDR(m);
498 }
499 return (m);
500 }
501
502 void
503 m_reclaim(void)
504 {
505 do_reclaim = 1; /* drain is performed in pfslowtimo(), to avoid deadlocks */
506 mbstat.m_drain++;
507 }
508
509 /*
510 * Space allocation routines.
511 * These are also available as macros
512 * for critical paths.
513 */
514 struct mbuf *
515 m_get(
516 int nowait,
517 int type)
518 {
519 struct mbuf *m;
520
521 m_range_check(mfree);
522 m_range_check(mclfree);
523 m_range_check(mbigfree);
524
525 _MINTGET(m, type);
526 if (m) {
527 m->m_next = m->m_nextpkt = 0;
528 m->m_type = type;
529 m->m_data = m->m_dat;
530 m->m_flags = 0;
531 m->m_len = 0;
532 } else
533 (m) = m_retry(nowait, type);
534
535 m_range_check(mfree);
536 m_range_check(mclfree);
537 m_range_check(mbigfree);
538
539
540 return (m);
541 }
542
543 struct mbuf *
544 m_gethdr(
545 int nowait,
546 int type)
547 {
548 struct mbuf *m;
549
550 m_range_check(mfree);
551 m_range_check(mclfree);
552 m_range_check(mbigfree);
553
554
555 _MINTGET(m, type);
556 if (m) {
557 m->m_next = m->m_nextpkt = 0;
558 m->m_type = type;
559 m->m_data = m->m_pktdat;
560 m->m_flags = M_PKTHDR;
561 m->m_len = 0;
562 _M_CLEAR_PKTHDR(m)
563 } else
564 m = m_retryhdr(nowait, type);
565
566 m_range_check(mfree);
567 m_range_check(mclfree);
568 m_range_check(mbigfree);
569
570
571 return m;
572 }
573
574 struct mbuf *
575 m_getclr(
576 int nowait,
577 int type)
578 {
579 struct mbuf *m;
580
581 MGET(m, nowait, type);
582 if (m == 0)
583 return (0);
584 bzero(mtod(m, caddr_t), MLEN);
585 return (m);
586 }
587
588 struct mbuf *
589 m_free(
590 struct mbuf *m)
591 {
592 struct mbuf *n = m->m_next;
593 int i;
594
595 m_range_check(m);
596 m_range_check(mfree);
597 m_range_check(mclfree);
598
599 if (m->m_type == MT_FREE)
600 panic("freeing free mbuf");
601
602 /* Free the aux data if there is any */
603 if ((m->m_flags & M_PKTHDR) && m->m_pkthdr.aux)
604 {
605 m_freem(m->m_pkthdr.aux);
606 }
607 if ((m->m_flags & M_PKTHDR) != 0)
608 m_tag_delete_chain(m, NULL);
609
610 MBUF_LOCK();
611 if ((m->m_flags & M_EXT))
612 {
613 if (MCLHASREFERENCE(m)) {
614 remque((queue_t)&m->m_ext.ext_refs);
615 } else if (m->m_ext.ext_free == NULL) {
616 union mcluster *mcl= (union mcluster *)m->m_ext.ext_buf;
617
618 m_range_check(mcl);
619
620 if (_MCLUNREF(mcl)) {
621 mcl->mcl_next = mclfree;
622 mclfree = mcl;
623 ++mbstat.m_clfree;
624 }
625 #ifdef COMMENT_OUT
626 /* *** Since m_split() increments "mclrefcnt[mtocl(m->m_ext.ext_buf)]",
627 and AppleTalk ADSP uses m_split(), this incorrect sanity check
628 caused a panic.
629 *** */
630 else /* sanity check - not referenced this way */
631 panic("m_free m_ext cluster not free");
632 #endif
633 } else {
634 (*(m->m_ext.ext_free))(m->m_ext.ext_buf,
635 m->m_ext.ext_size, m->m_ext.ext_arg);
636 }
637 }
638 mbstat.m_mtypes[m->m_type]--;
639 (void) _MCLUNREF(m);
640 _MFREE_MUNGE(m);
641 m->m_type = MT_FREE;
642 mbstat.m_mtypes[m->m_type]++;
643 m->m_flags = 0;
644 m->m_next = mfree;
645 m->m_len = 0;
646 mfree = m;
647 i = m_want;
648 m_want = 0;
649 MBUF_UNLOCK();
650 if (i) wakeup((caddr_t)&mfree);
651 return (n);
652 }
653
654 /* m_mclget() add an mbuf cluster to a normal mbuf */
655 struct mbuf *
656 m_mclget(
657 struct mbuf *m,
658 int nowait)
659 {
660 MCLALLOC(m->m_ext.ext_buf, nowait);
661 if (m->m_ext.ext_buf) {
662 m->m_data = m->m_ext.ext_buf;
663 m->m_flags |= M_EXT;
664 m->m_ext.ext_size = MCLBYTES;
665 m->m_ext.ext_free = 0;
666 m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward =
667 &m->m_ext.ext_refs;
668 }
669
670 return m;
671 }
672
673 /* m_mclalloc() allocate an mbuf cluster */
674 caddr_t
675 m_mclalloc(
676 int nowait)
677 {
678 caddr_t p;
679
680 (void)m_clalloc(1, nowait, MCLBYTES, 0);
681 if ((p = (caddr_t)mclfree)) {
682 ++mclrefcnt[mtocl(p)];
683 mbstat.m_clfree--;
684 mclfree = ((union mcluster *)p)->mcl_next;
685 } else {
686 mbstat.m_drops++;
687 }
688 MBUF_UNLOCK();
689
690 return p;
691 }
692
693 /* m_mclfree() releases a reference to a cluster allocated by MCLALLOC,
694 * freeing the cluster if the reference count has reached 0. */
695 void
696 m_mclfree(
697 caddr_t p)
698 {
699 MBUF_LOCK();
700
701 m_range_check(p);
702
703 if (--mclrefcnt[mtocl(p)] == 0) {
704 ((union mcluster *)(p))->mcl_next = mclfree;
705 mclfree = (union mcluster *)(p);
706 mbstat.m_clfree++;
707 }
708 MBUF_UNLOCK();
709 }
710
711 /* mcl_hasreference() checks if a cluster of an mbuf is referenced by another mbuf */
712 int
713 m_mclhasreference(
714 struct mbuf *m)
715 {
716 return (m->m_ext.ext_refs.forward != &(m->m_ext.ext_refs));
717 }
718
719 __private_extern__ caddr_t
720 m_bigalloc(int nowait)
721 {
722 caddr_t p;
723
724 (void)m_clalloc(1, nowait, NBPG, 0);
725 if ((p = (caddr_t)mbigfree)) {
726 if (mclrefcnt[mtocl(p)] != mclrefcnt[mtocl(p) + 1])
727 panic("m_bigalloc mclrefcnt %x mismatch %d != %d",
728 p, mclrefcnt[mtocl(p)], mclrefcnt[mtocl(p) + 1]);
729 if (mclrefcnt[mtocl(p)] || mclrefcnt[mtocl(p) + 1])
730 panic("m_bigalloc mclrefcnt %x not null %d != %d",
731 p, mclrefcnt[mtocl(p)], mclrefcnt[mtocl(p) + 1]);
732 ++mclrefcnt[mtocl(p)];
733 ++mclrefcnt[mtocl(p) + 1];
734 mbstat.m_bigclfree--;
735 mbigfree = ((union mbigcluster *)p)->mbc_next;
736 } else {
737 mbstat.m_drops++;
738 }
739 MBUF_UNLOCK();
740 return p;
741 }
742
743 __private_extern__ void
744 m_bigfree(caddr_t p, __unused u_int size, __unused caddr_t arg)
745 {
746 m_range_check(p);
747
748 if (mclrefcnt[mtocl(p)] != mclrefcnt[mtocl(p) + 1])
749 panic("m_bigfree mclrefcnt %x mismatch %d != %d",
750 p, mclrefcnt[mtocl(p)], mclrefcnt[mtocl(p) + 1]);
751 --mclrefcnt[mtocl(p)];
752 --mclrefcnt[mtocl(p) + 1];
753 if (mclrefcnt[mtocl(p)] == 0) {
754 ((union mbigcluster *)(p))->mbc_next = mbigfree;
755 mbigfree = (union mbigcluster *)(p);
756 mbstat.m_bigclfree++;
757 }
758 }
759
760 /* m_mbigget() add an 4KB mbuf cluster to a normal mbuf */
761 __private_extern__ struct mbuf *
762 m_mbigget(struct mbuf *m, int nowait)
763 {
764 m->m_ext.ext_buf = m_bigalloc(nowait);
765 if (m->m_ext.ext_buf) {
766 m->m_data = m->m_ext.ext_buf;
767 m->m_flags |= M_EXT;
768 m->m_ext.ext_size = NBPG;
769 m->m_ext.ext_free = m_bigfree;
770 m->m_ext.ext_arg = 0;
771 m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward =
772 &m->m_ext.ext_refs;
773 }
774
775 return m;
776 }
777
778
779 /* */
780 void
781 m_copy_pkthdr(
782 struct mbuf *to,
783 struct mbuf *from)
784 {
785 to->m_pkthdr = from->m_pkthdr;
786 from->m_pkthdr.aux = (struct mbuf *)NULL;
787 SLIST_INIT(&from->m_pkthdr.tags); /* purge tags from src */
788 to->m_flags = from->m_flags & M_COPYFLAGS;
789 to->m_data = (to)->m_pktdat;
790 }
791
792 /*
793 * "Move" mbuf pkthdr from "from" to "to".
794 * "from" must have M_PKTHDR set, and "to" must be empty.
795 */
796 #ifndef __APPLE__
797 void
798 m_move_pkthdr(struct mbuf *to, struct mbuf *from)
799 {
800 KASSERT((to->m_flags & M_EXT) == 0, ("m_move_pkthdr: to has cluster"));
801
802 to->m_flags = from->m_flags & M_COPYFLAGS;
803 to->m_data = to->m_pktdat;
804 to->m_pkthdr = from->m_pkthdr; /* especially tags */
805 SLIST_INIT(&from->m_pkthdr.tags); /* purge tags from src */
806 from->m_flags &= ~M_PKTHDR;
807 }
808 #endif
809
810 /*
811 * Duplicate "from"'s mbuf pkthdr in "to".
812 * "from" must have M_PKTHDR set, and "to" must be empty.
813 * In particular, this does a deep copy of the packet tags.
814 */
815 #ifndef __APPLE__
816 int
817 m_dup_pkthdr(struct mbuf *to, struct mbuf *from, int how)
818 {
819 to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
820 if ((to->m_flags & M_EXT) == 0)
821 to->m_data = to->m_pktdat;
822 to->m_pkthdr = from->m_pkthdr;
823 SLIST_INIT(&to->m_pkthdr.tags);
824 return (m_tag_copy_chain(to, from, how));
825 }
826 #endif
827
828 /*
829 * return a list of mbuf hdrs that point to clusters...
830 * try for num_needed, if wantall is not set, return whatever
831 * number were available... set up the first num_with_pkthdrs
832 * with mbuf hdrs configured as packet headers... these are
833 * chained on the m_nextpkt field... any packets requested beyond
834 * this are chained onto the last packet header's m_next field.
835 * The size of the cluster is controlled by the paramter bufsize.
836 */
837 __private_extern__ struct mbuf *
838 m_getpackets_internal(unsigned int *num_needed, int num_with_pkthdrs, int how, int wantall, size_t bufsize)
839 {
840 struct mbuf *m;
841 struct mbuf **np, *top;
842 unsigned int num, needed = *num_needed;
843
844 if (bufsize != MCLBYTES && bufsize != NBPG)
845 return 0;
846
847 top = NULL;
848 np = &top;
849
850 (void)m_clalloc(needed, how, bufsize, 0); /* takes the MBUF_LOCK, but doesn't release it... */
851
852 for (num = 0; num < needed; num++) {
853 m_range_check(mfree);
854 m_range_check(mclfree);
855 m_range_check(mbigfree);
856
857 if (mfree && ((bufsize == NBPG && mbigfree) || (bufsize == MCLBYTES && mclfree))) {
858 /* mbuf + cluster are available */
859 m = mfree;
860 MCHECK(m);
861 mfree = m->m_next;
862 ++mclrefcnt[mtocl(m)];
863 mbstat.m_mtypes[MT_FREE]--;
864 mbstat.m_mtypes[MT_DATA]++;
865 if (bufsize == NBPG) {
866 m->m_ext.ext_buf = (caddr_t)mbigfree; /* get the big cluster */
867 ++mclrefcnt[mtocl(m->m_ext.ext_buf)];
868 ++mclrefcnt[mtocl(m->m_ext.ext_buf) + 1];
869 mbstat.m_bigclfree--;
870 mbigfree = ((union mbigcluster *)(m->m_ext.ext_buf))->mbc_next;
871 m->m_ext.ext_free = m_bigfree;
872 m->m_ext.ext_size = NBPG;
873 } else {
874 m->m_ext.ext_buf = (caddr_t)mclfree; /* get the cluster */
875 ++mclrefcnt[mtocl(m->m_ext.ext_buf)];
876 mbstat.m_clfree--;
877 mclfree = ((union mcluster *)(m->m_ext.ext_buf))->mcl_next;
878 m->m_ext.ext_free = 0;
879 m->m_ext.ext_size = MCLBYTES;
880 }
881 m->m_ext.ext_arg = 0;
882 m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward = &m->m_ext.ext_refs;
883 m->m_next = m->m_nextpkt = 0;
884 m->m_type = MT_DATA;
885 m->m_data = m->m_ext.ext_buf;
886 m->m_len = 0;
887
888 if (num_with_pkthdrs == 0)
889 m->m_flags = M_EXT;
890 else {
891 m->m_flags = M_PKTHDR | M_EXT;
892 _M_CLEAR_PKTHDR(m);
893
894 num_with_pkthdrs--;
895 }
896 } else {
897 MBUF_UNLOCK();
898
899 if (num_with_pkthdrs == 0) {
900 MGET(m, how, MT_DATA );
901 } else {
902 MGETHDR(m, how, MT_DATA);
903
904 num_with_pkthdrs--;
905 }
906 if (m == 0)
907 goto fail;
908
909 if (bufsize == NBPG)
910 m = m_mbigget(m, how);
911 else
912 m = m_mclget(m, how);
913 if ((m->m_flags & M_EXT) == 0) {
914 m_free(m);
915 goto fail;
916 }
917 MBUF_LOCK();
918 }
919 *np = m;
920
921 if (num_with_pkthdrs)
922 np = &m->m_nextpkt;
923 else
924 np = &m->m_next;
925 }
926 MBUF_UNLOCK();
927
928 *num_needed = num;
929 return (top);
930 fail:
931 if (wantall && top) {
932 m_freem(top);
933 return 0;
934 }
935 return top;
936 }
937
938
939 /*
940 * Return list of mbuf linked by m_nextpkt
941 * Try for num_needed, and if wantall is not set, return whatever
942 * number were available
943 * The size of each mbuf in the list is controlled by the parameter packetlen.
944 * Each mbuf of the list may have a chain of mbufs linked by m_next. Each mbuf in
945 * the chain is called a segment.
946 * If maxsegments is not null and the value pointed to is not null, this specify
947 * the maximum number of segments for a chain of mbufs.
948 * If maxsegments is zero or the value pointed to is zero the
949 * caller does not have any restriction on the number of segments.
950 * The actual number of segments of a mbuf chain is return in the value pointed
951 * to by maxsegments.
952 * When possible the allocation is done under a single lock.
953 */
954
955 __private_extern__ struct mbuf *
956 m_allocpacket_internal(unsigned int *num_needed, size_t packetlen, unsigned int * maxsegments,
957 int how, int wantall, size_t wantsize)
958 {
959 struct mbuf **np, *top;
960 size_t bufsize;
961 unsigned int num;
962 unsigned int numchunks = 0;
963
964 top = NULL;
965 np = &top;
966
967 if (wantsize == 0) {
968 if (packetlen <= MINCLSIZE)
969 bufsize = packetlen;
970 else if (packetlen > MCLBYTES)
971 bufsize = NBPG;
972 else
973 bufsize = MCLBYTES;
974 } else if (wantsize == MCLBYTES || wantsize == NBPG)
975 bufsize = wantsize;
976 else
977 return 0;
978
979 if (bufsize <= MHLEN) {
980 numchunks = 1;
981 } else if (bufsize <= MINCLSIZE) {
982 if (maxsegments != NULL && *maxsegments == 1) {
983 bufsize = MCLBYTES;
984 numchunks = 1;
985 } else {
986 numchunks = 2;
987 }
988 } else if (bufsize == NBPG) {
989 numchunks = ((packetlen - 1) >> PGSHIFT) + 1;
990 } else {
991 numchunks = ((packetlen - 1) >> MCLSHIFT) + 1;
992 }
993 if (maxsegments != NULL) {
994 if (*maxsegments && numchunks > *maxsegments) {
995 *maxsegments = numchunks;
996 return 0;
997 }
998 *maxsegments = numchunks;
999 }
1000 /* m_clalloc takes the MBUF_LOCK, but do not release it */
1001 (void)m_clalloc(numchunks, how, (bufsize == NBPG) ? NBPG : MCLBYTES, 0);
1002 for (num = 0; num < *num_needed; num++) {
1003 struct mbuf **nm, *pkt = 0;
1004 size_t len;
1005
1006 nm = &pkt;
1007
1008 m_range_check(mfree);
1009 m_range_check(mclfree);
1010 m_range_check(mbigfree);
1011
1012 for (len = 0; len < packetlen; ) {
1013 struct mbuf *m = NULL;
1014
1015 if (wantsize == 0 && packetlen > MINCLSIZE) {
1016 if (packetlen - len > MCLBYTES)
1017 bufsize = NBPG;
1018 else
1019 bufsize = MCLBYTES;
1020 }
1021 len += bufsize;
1022
1023 if (mfree && ((bufsize == NBPG && mbigfree) || (bufsize == MCLBYTES && mclfree))) {
1024 /* mbuf + cluster are available */
1025 m = mfree;
1026 MCHECK(m);
1027 mfree = m->m_next;
1028 ++mclrefcnt[mtocl(m)];
1029 mbstat.m_mtypes[MT_FREE]--;
1030 mbstat.m_mtypes[MT_DATA]++;
1031 if (bufsize == NBPG) {
1032 m->m_ext.ext_buf = (caddr_t)mbigfree; /* get the big cluster */
1033 ++mclrefcnt[mtocl(m->m_ext.ext_buf)];
1034 ++mclrefcnt[mtocl(m->m_ext.ext_buf) + 1];
1035 mbstat.m_bigclfree--;
1036 mbigfree = ((union mbigcluster *)(m->m_ext.ext_buf))->mbc_next;
1037 m->m_ext.ext_free = m_bigfree;
1038 m->m_ext.ext_size = NBPG;
1039 } else {
1040 m->m_ext.ext_buf = (caddr_t)mclfree; /* get the cluster */
1041 ++mclrefcnt[mtocl(m->m_ext.ext_buf)];
1042 mbstat.m_clfree--;
1043 mclfree = ((union mcluster *)(m->m_ext.ext_buf))->mcl_next;
1044 m->m_ext.ext_free = 0;
1045 m->m_ext.ext_size = MCLBYTES;
1046 }
1047 m->m_ext.ext_arg = 0;
1048 m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward = &m->m_ext.ext_refs;
1049 m->m_next = m->m_nextpkt = 0;
1050 m->m_type = MT_DATA;
1051 m->m_data = m->m_ext.ext_buf;
1052 m->m_len = 0;
1053
1054 if (pkt == 0) {
1055 pkt = m;
1056 m->m_flags = M_PKTHDR | M_EXT;
1057 _M_CLEAR_PKTHDR(m);
1058 } else {
1059 m->m_flags = M_EXT;
1060 }
1061 } else {
1062 MBUF_UNLOCK();
1063
1064 if (pkt == 0) {
1065 MGETHDR(m, how, MT_DATA);
1066 } else {
1067 MGET(m, how, MT_DATA );
1068 }
1069 if (m == 0) {
1070 m_freem(pkt);
1071 goto fail;
1072 }
1073 if (bufsize <= MINCLSIZE) {
1074 if (bufsize > MHLEN) {
1075 MGET(m->m_next, how, MT_DATA);
1076 if (m->m_next == 0) {
1077 m_free(m);
1078 m_freem(pkt);
1079 goto fail;
1080 }
1081 }
1082 } else {
1083 if (bufsize == NBPG)
1084 m = m_mbigget(m, how);
1085 else
1086 m = m_mclget(m, how);
1087 if ((m->m_flags & M_EXT) == 0) {
1088 m_free(m);
1089 m_freem(pkt);
1090 goto fail;
1091 }
1092 }
1093 MBUF_LOCK();
1094 }
1095 *nm = m;
1096 nm = &m->m_next;
1097 }
1098 *np = pkt;
1099 np = &pkt->m_nextpkt;
1100 }
1101 MBUF_UNLOCK();
1102 *num_needed = num;
1103
1104 return top;
1105 fail:
1106 if (wantall && top) {
1107 m_freem(top);
1108 return 0;
1109 }
1110 *num_needed = num;
1111
1112 return top;
1113 }
1114
1115
1116 /* Best effort to get a mbuf cluster + pkthdr under one lock.
1117 * If we don't have them avail, just bail out and use the regular
1118 * path.
1119 * Used by drivers to allocated packets on receive ring.
1120 */
1121 __private_extern__ struct mbuf *
1122 m_getpacket_how(int how)
1123 {
1124 unsigned int num_needed = 1;
1125
1126 return m_getpackets_internal(&num_needed, 1, how, 1, MCLBYTES);
1127 }
1128
1129 /* Best effort to get a mbuf cluster + pkthdr under one lock.
1130 * If we don't have them avail, just bail out and use the regular
1131 * path.
1132 * Used by drivers to allocated packets on receive ring.
1133 */
1134 struct mbuf *
1135 m_getpacket(void)
1136 {
1137 unsigned int num_needed = 1;
1138
1139 return m_getpackets_internal(&num_needed, 1, M_WAITOK, 1, MCLBYTES);
1140 }
1141
1142
1143 /*
1144 * return a list of mbuf hdrs that point to clusters...
1145 * try for num_needed, if this can't be met, return whatever
1146 * number were available... set up the first num_with_pkthdrs
1147 * with mbuf hdrs configured as packet headers... these are
1148 * chained on the m_nextpkt field... any packets requested beyond
1149 * this are chained onto the last packet header's m_next field.
1150 */
1151 struct mbuf *
1152 m_getpackets(int num_needed, int num_with_pkthdrs, int how)
1153 {
1154 unsigned int n = num_needed;
1155
1156 return m_getpackets_internal(&n, num_with_pkthdrs, how, 0, MCLBYTES);
1157 }
1158
1159
1160 /*
1161 * return a list of mbuf hdrs set up as packet hdrs
1162 * chained together on the m_nextpkt field
1163 */
1164 struct mbuf *
1165 m_getpackethdrs(int num_needed, int how)
1166 {
1167 struct mbuf *m;
1168 struct mbuf **np, *top;
1169
1170 top = NULL;
1171 np = &top;
1172
1173 MBUF_LOCK();
1174
1175 while (num_needed--) {
1176 m_range_check(mfree);
1177 m_range_check(mclfree);
1178 m_range_check(mbigfree);
1179
1180 if ((m = mfree)) { /* mbufs are available */
1181 MCHECK(m);
1182 mfree = m->m_next;
1183 ++mclrefcnt[mtocl(m)];
1184 mbstat.m_mtypes[MT_FREE]--;
1185 mbstat.m_mtypes[MT_DATA]++;
1186
1187 m->m_next = m->m_nextpkt = 0;
1188 m->m_type = MT_DATA;
1189 m->m_flags = M_PKTHDR;
1190 m->m_len = 0;
1191 m->m_data = m->m_pktdat;
1192 _M_CLEAR_PKTHDR(m);
1193
1194 } else {
1195
1196 MBUF_UNLOCK();
1197 m = m_retryhdr(how, MT_DATA);
1198 if (m == 0)
1199 return(top);
1200 MBUF_LOCK();
1201 }
1202 *np = m;
1203 np = &m->m_nextpkt;
1204 }
1205 MBUF_UNLOCK();
1206
1207 return (top);
1208 }
1209
1210
1211 /* free and mbuf list (m_nextpkt) while following m_next under one lock.
1212 * returns the count for mbufs packets freed. Used by the drivers.
1213 */
1214 int
1215 m_freem_list(
1216 struct mbuf *m)
1217 {
1218 struct mbuf *nextpkt;
1219 int i, count=0;
1220
1221 MBUF_LOCK();
1222
1223 while (m) {
1224 if (m)
1225 nextpkt = m->m_nextpkt; /* chain of linked mbufs from driver */
1226 else
1227 nextpkt = 0;
1228
1229 count++;
1230
1231 while (m) { /* free the mbuf chain (like mfreem) */
1232
1233 struct mbuf *n;
1234
1235 m_range_check(m);
1236 m_range_check(mfree);
1237 m_range_check(mclfree);
1238 m_range_check(mbigfree);
1239
1240
1241 /* Free the aux data if there is any */
1242 if ((m->m_flags & M_PKTHDR) && m->m_pkthdr.aux) {
1243 /*
1244 * Treat the current m as the nextpkt and set m
1245 * to the aux data. Preserve nextpkt in m->m_nextpkt.
1246 * This lets us free the aux data in this loop
1247 * without having to call m_freem recursively,
1248 * which wouldn't work because we've still got
1249 * the lock.
1250 */
1251 m->m_nextpkt = nextpkt;
1252 nextpkt = m;
1253 m = nextpkt->m_pkthdr.aux;
1254 nextpkt->m_pkthdr.aux = NULL;
1255 }
1256
1257 if ((m->m_flags & M_PKTHDR) != 0 && !SLIST_EMPTY(&m->m_pkthdr.tags)) {
1258 /* A quick (albeit inefficient) expedient */
1259 MBUF_UNLOCK();
1260 m_tag_delete_chain(m, NULL);
1261 MBUF_LOCK();
1262 }
1263
1264 n = m->m_next;
1265
1266 if (n && n->m_nextpkt)
1267 panic("m_freem_list: m_nextpkt of m_next != NULL");
1268 if (m->m_type == MT_FREE)
1269 panic("freeing free mbuf");
1270
1271 if (m->m_flags & M_EXT) {
1272 if (MCLHASREFERENCE(m)) {
1273 remque((queue_t)&m->m_ext.ext_refs);
1274 } else if (m->m_ext.ext_free == NULL) {
1275 union mcluster *mcl= (union mcluster *)m->m_ext.ext_buf;
1276
1277 m_range_check(mcl);
1278
1279 if (_MCLUNREF(mcl)) {
1280 mcl->mcl_next = mclfree;
1281 mclfree = mcl;
1282 ++mbstat.m_clfree;
1283 }
1284 } else {
1285 (*(m->m_ext.ext_free))(m->m_ext.ext_buf,
1286 m->m_ext.ext_size, m->m_ext.ext_arg);
1287 }
1288 }
1289 mbstat.m_mtypes[m->m_type]--;
1290 (void) _MCLUNREF(m);
1291 _MFREE_MUNGE(m);
1292 mbstat.m_mtypes[MT_FREE]++;
1293 m->m_type = MT_FREE;
1294 m->m_flags = 0;
1295 m->m_len = 0;
1296 m->m_next = mfree;
1297 mfree = m;
1298 m = n;
1299 }
1300 m = nextpkt; /* bump m with saved nextpkt if any */
1301 }
1302 if ((i = m_want))
1303 m_want = 0;
1304
1305 MBUF_UNLOCK();
1306
1307 if (i)
1308 wakeup((caddr_t)&mfree);
1309
1310 return (count);
1311 }
1312
1313 void
1314 m_freem(
1315 struct mbuf *m)
1316 {
1317 while (m)
1318 m = m_free(m);
1319 }
1320
1321 /*
1322 * Mbuffer utility routines.
1323 */
1324 /*
1325 * Compute the amount of space available
1326 * before the current start of data in an mbuf.
1327 */
1328 int
1329 m_leadingspace(
1330 struct mbuf *m)
1331 {
1332 if (m->m_flags & M_EXT) {
1333 if (MCLHASREFERENCE(m))
1334 return(0);
1335 return (m->m_data - m->m_ext.ext_buf);
1336 }
1337 if (m->m_flags & M_PKTHDR)
1338 return (m->m_data - m->m_pktdat);
1339 return (m->m_data - m->m_dat);
1340 }
1341
1342 /*
1343 * Compute the amount of space available
1344 * after the end of data in an mbuf.
1345 */
1346 int
1347 m_trailingspace(
1348 struct mbuf *m)
1349 {
1350 if (m->m_flags & M_EXT) {
1351 if (MCLHASREFERENCE(m))
1352 return(0);
1353 return (m->m_ext.ext_buf + m->m_ext.ext_size -
1354 (m->m_data + m->m_len));
1355 }
1356 return (&m->m_dat[MLEN] - (m->m_data + m->m_len));
1357 }
1358
1359 /*
1360 * Lesser-used path for M_PREPEND:
1361 * allocate new mbuf to prepend to chain,
1362 * copy junk along.
1363 * Does not adjust packet header length.
1364 */
1365 struct mbuf *
1366 m_prepend(
1367 struct mbuf *m,
1368 int len,
1369 int how)
1370 {
1371 struct mbuf *mn;
1372
1373 MGET(mn, how, m->m_type);
1374 if (mn == (struct mbuf *)NULL) {
1375 m_freem(m);
1376 return ((struct mbuf *)NULL);
1377 }
1378 if (m->m_flags & M_PKTHDR) {
1379 M_COPY_PKTHDR(mn, m);
1380 m->m_flags &= ~M_PKTHDR;
1381 }
1382 mn->m_next = m;
1383 m = mn;
1384 if (len < MHLEN)
1385 MH_ALIGN(m, len);
1386 m->m_len = len;
1387 return (m);
1388 }
1389
1390 /*
1391 * Replacement for old M_PREPEND macro:
1392 * allocate new mbuf to prepend to chain,
1393 * copy junk along, and adjust length.
1394 *
1395 */
1396 struct mbuf *
1397 m_prepend_2(
1398 struct mbuf *m,
1399 int len,
1400 int how)
1401 {
1402 if (M_LEADINGSPACE(m) >= len) {
1403 m->m_data -= len;
1404 m->m_len += len;
1405 } else {
1406 m = m_prepend(m, len, how);
1407 }
1408 if ((m) && (m->m_flags & M_PKTHDR))
1409 m->m_pkthdr.len += len;
1410 return (m);
1411 }
1412
1413 /*
1414 * Make a copy of an mbuf chain starting "off0" bytes from the beginning,
1415 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf.
1416 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller.
1417 */
1418 int MCFail;
1419
1420 struct mbuf *
1421 m_copym(
1422 struct mbuf *m,
1423 int off0,
1424 int len,
1425 int wait)
1426 {
1427 struct mbuf *n, **np;
1428 int off = off0;
1429 struct mbuf *top;
1430 int copyhdr = 0;
1431
1432 if (off < 0 || len < 0)
1433 panic("m_copym");
1434 if (off == 0 && m->m_flags & M_PKTHDR)
1435 copyhdr = 1;
1436
1437 while (off >= m->m_len) {
1438 if (m == 0)
1439 panic("m_copym");
1440 off -= m->m_len;
1441 m = m->m_next;
1442 }
1443 np = &top;
1444 top = 0;
1445
1446 MBUF_LOCK();
1447
1448 while (len > 0) {
1449 m_range_check(mfree);
1450 m_range_check(mclfree);
1451 m_range_check(mbigfree);
1452
1453 if (m == 0) {
1454 if (len != M_COPYALL)
1455 panic("m_copym");
1456 break;
1457 }
1458 if ((n = mfree)) {
1459 MCHECK(n);
1460 ++mclrefcnt[mtocl(n)];
1461 mbstat.m_mtypes[MT_FREE]--;
1462 mbstat.m_mtypes[m->m_type]++;
1463 mfree = n->m_next;
1464 n->m_next = n->m_nextpkt = 0;
1465 n->m_type = m->m_type;
1466 n->m_data = n->m_dat;
1467 n->m_flags = 0;
1468 } else {
1469 MBUF_UNLOCK();
1470 n = m_retry(wait, m->m_type);
1471 MBUF_LOCK();
1472 }
1473 *np = n;
1474
1475 if (n == 0)
1476 goto nospace;
1477 if (copyhdr) {
1478 M_COPY_PKTHDR(n, m);
1479 if (len == M_COPYALL)
1480 n->m_pkthdr.len -= off0;
1481 else
1482 n->m_pkthdr.len = len;
1483 copyhdr = 0;
1484 }
1485 if (len == M_COPYALL) {
1486 if (min(len, (m->m_len - off)) == len) {
1487 printf("m->m_len %d - off %d = %d, %d\n",
1488 m->m_len, off, m->m_len - off,
1489 min(len, (m->m_len - off)));
1490 }
1491 }
1492 n->m_len = min(len, (m->m_len - off));
1493 if (n->m_len == M_COPYALL) {
1494 printf("n->m_len == M_COPYALL, fixing\n");
1495 n->m_len = MHLEN;
1496 }
1497 if (m->m_flags & M_EXT) {
1498 n->m_ext = m->m_ext;
1499 insque((queue_t)&n->m_ext.ext_refs, (queue_t)&m->m_ext.ext_refs);
1500 n->m_data = m->m_data + off;
1501 n->m_flags |= M_EXT;
1502 } else {
1503 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
1504 (unsigned)n->m_len);
1505 }
1506 if (len != M_COPYALL)
1507 len -= n->m_len;
1508 off = 0;
1509 m = m->m_next;
1510 np = &n->m_next;
1511 }
1512 MBUF_UNLOCK();
1513
1514 if (top == 0)
1515 MCFail++;
1516
1517 return (top);
1518 nospace:
1519 MBUF_UNLOCK();
1520
1521 m_freem(top);
1522 MCFail++;
1523 return (0);
1524 }
1525
1526
1527 /*
1528 * equivilent to m_copym except that all necessary
1529 * mbuf hdrs are allocated within this routine
1530 * also, the last mbuf and offset accessed are passed
1531 * out and can be passed back in to avoid having to
1532 * rescan the entire mbuf list (normally hung off of the socket)
1533 */
1534 struct mbuf *
1535 m_copym_with_hdrs(
1536 struct mbuf *m,
1537 int off0,
1538 int len,
1539 int wait,
1540 struct mbuf **m_last,
1541 int *m_off)
1542 {
1543 struct mbuf *n, **np = 0;
1544 int off = off0;
1545 struct mbuf *top = 0;
1546 int copyhdr = 0;
1547 int type;
1548
1549 if (off == 0 && m->m_flags & M_PKTHDR)
1550 copyhdr = 1;
1551
1552 if (*m_last) {
1553 m = *m_last;
1554 off = *m_off;
1555 } else {
1556 while (off >= m->m_len) {
1557 off -= m->m_len;
1558 m = m->m_next;
1559 }
1560 }
1561
1562 MBUF_LOCK();
1563
1564 while (len > 0) {
1565 m_range_check(mfree);
1566 m_range_check(mclfree);
1567 m_range_check(mbigfree);
1568
1569 if (top == 0)
1570 type = MT_HEADER;
1571 else {
1572 if (m == 0)
1573 panic("m_gethdr_and_copym");
1574 type = m->m_type;
1575 }
1576 if ((n = mfree)) {
1577 MCHECK(n);
1578 ++mclrefcnt[mtocl(n)];
1579 mbstat.m_mtypes[MT_FREE]--;
1580 mbstat.m_mtypes[type]++;
1581 mfree = n->m_next;
1582 n->m_next = n->m_nextpkt = 0;
1583 n->m_type = type;
1584
1585 if (top) {
1586 n->m_data = n->m_dat;
1587 n->m_flags = 0;
1588 } else {
1589 n->m_data = n->m_pktdat;
1590 n->m_flags = M_PKTHDR;
1591 _M_CLEAR_PKTHDR(n);
1592 }
1593 } else {
1594 MBUF_UNLOCK();
1595 if (top)
1596 n = m_retry(wait, type);
1597 else
1598 n = m_retryhdr(wait, type);
1599 MBUF_LOCK();
1600 }
1601 if (n == 0)
1602 goto nospace;
1603 if (top == 0) {
1604 top = n;
1605 np = &top->m_next;
1606 continue;
1607 } else
1608 *np = n;
1609
1610 if (copyhdr) {
1611 M_COPY_PKTHDR(n, m);
1612 n->m_pkthdr.len = len;
1613 copyhdr = 0;
1614 }
1615 n->m_len = min(len, (m->m_len - off));
1616
1617 if (m->m_flags & M_EXT) {
1618 n->m_ext = m->m_ext;
1619 insque((queue_t)&n->m_ext.ext_refs, (queue_t)&m->m_ext.ext_refs);
1620 n->m_data = m->m_data + off;
1621 n->m_flags |= M_EXT;
1622 } else {
1623 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
1624 (unsigned)n->m_len);
1625 }
1626 len -= n->m_len;
1627
1628 if (len == 0) {
1629 if ((off + n->m_len) == m->m_len) {
1630 *m_last = m->m_next;
1631 *m_off = 0;
1632 } else {
1633 *m_last = m;
1634 *m_off = off + n->m_len;
1635 }
1636 break;
1637 }
1638 off = 0;
1639 m = m->m_next;
1640 np = &n->m_next;
1641 }
1642 MBUF_UNLOCK();
1643
1644 return (top);
1645 nospace:
1646 MBUF_UNLOCK();
1647
1648 if (top)
1649 m_freem(top);
1650 MCFail++;
1651 return (0);
1652 }
1653
1654
1655 /*
1656 * Copy data from an mbuf chain starting "off" bytes from the beginning,
1657 * continuing for "len" bytes, into the indicated buffer.
1658 */
1659 void m_copydata(
1660 struct mbuf *m,
1661 int off,
1662 int len,
1663 caddr_t cp)
1664 {
1665 unsigned count;
1666
1667 if (off < 0 || len < 0)
1668 panic("m_copydata");
1669 while (off > 0) {
1670 if (m == 0)
1671 panic("m_copydata");
1672 if (off < m->m_len)
1673 break;
1674 off -= m->m_len;
1675 m = m->m_next;
1676 }
1677 while (len > 0) {
1678 if (m == 0)
1679 panic("m_copydata");
1680 count = min(m->m_len - off, len);
1681 bcopy(mtod(m, caddr_t) + off, cp, count);
1682 len -= count;
1683 cp += count;
1684 off = 0;
1685 m = m->m_next;
1686 }
1687 }
1688
1689 /*
1690 * Concatenate mbuf chain n to m.
1691 * Both chains must be of the same type (e.g. MT_DATA).
1692 * Any m_pkthdr is not updated.
1693 */
1694 void m_cat(
1695 struct mbuf *m, struct mbuf *n)
1696 {
1697 while (m->m_next)
1698 m = m->m_next;
1699 while (n) {
1700 if (m->m_flags & M_EXT ||
1701 m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) {
1702 /* just join the two chains */
1703 m->m_next = n;
1704 return;
1705 }
1706 /* splat the data from one into the other */
1707 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
1708 (u_int)n->m_len);
1709 m->m_len += n->m_len;
1710 n = m_free(n);
1711 }
1712 }
1713
1714 void
1715 m_adj(
1716 struct mbuf *mp,
1717 int req_len)
1718 {
1719 int len = req_len;
1720 struct mbuf *m;
1721 int count;
1722
1723 if ((m = mp) == NULL)
1724 return;
1725 if (len >= 0) {
1726 /*
1727 * Trim from head.
1728 */
1729 while (m != NULL && len > 0) {
1730 if (m->m_len <= len) {
1731 len -= m->m_len;
1732 m->m_len = 0;
1733 m = m->m_next;
1734 } else {
1735 m->m_len -= len;
1736 m->m_data += len;
1737 len = 0;
1738 }
1739 }
1740 m = mp;
1741 if (m->m_flags & M_PKTHDR)
1742 m->m_pkthdr.len -= (req_len - len);
1743 } else {
1744 /*
1745 * Trim from tail. Scan the mbuf chain,
1746 * calculating its length and finding the last mbuf.
1747 * If the adjustment only affects this mbuf, then just
1748 * adjust and return. Otherwise, rescan and truncate
1749 * after the remaining size.
1750 */
1751 len = -len;
1752 count = 0;
1753 for (;;) {
1754 count += m->m_len;
1755 if (m->m_next == (struct mbuf *)0)
1756 break;
1757 m = m->m_next;
1758 }
1759 if (m->m_len >= len) {
1760 m->m_len -= len;
1761 m = mp;
1762 if (m->m_flags & M_PKTHDR)
1763 m->m_pkthdr.len -= len;
1764 return;
1765 }
1766 count -= len;
1767 if (count < 0)
1768 count = 0;
1769 /*
1770 * Correct length for chain is "count".
1771 * Find the mbuf with last data, adjust its length,
1772 * and toss data from remaining mbufs on chain.
1773 */
1774 m = mp;
1775 if (m->m_flags & M_PKTHDR)
1776 m->m_pkthdr.len = count;
1777 for (; m; m = m->m_next) {
1778 if (m->m_len >= count) {
1779 m->m_len = count;
1780 break;
1781 }
1782 count -= m->m_len;
1783 }
1784 while ((m = m->m_next))
1785 m->m_len = 0;
1786 }
1787 }
1788
1789 /*
1790 * Rearange an mbuf chain so that len bytes are contiguous
1791 * and in the data area of an mbuf (so that mtod and dtom
1792 * will work for a structure of size len). Returns the resulting
1793 * mbuf chain on success, frees it and returns null on failure.
1794 * If there is room, it will add up to max_protohdr-len extra bytes to the
1795 * contiguous region in an attempt to avoid being called next time.
1796 */
1797 int MPFail;
1798
1799 struct mbuf *
1800 m_pullup(
1801 struct mbuf *n,
1802 int len)
1803 {
1804 struct mbuf *m;
1805 int count;
1806 int space;
1807
1808 /*
1809 * If first mbuf has no cluster, and has room for len bytes
1810 * without shifting current data, pullup into it,
1811 * otherwise allocate a new mbuf to prepend to the chain.
1812 */
1813 if ((n->m_flags & M_EXT) == 0 &&
1814 n->m_data + len < &n->m_dat[MLEN] && n->m_next) {
1815 if (n->m_len >= len)
1816 return (n);
1817 m = n;
1818 n = n->m_next;
1819 len -= m->m_len;
1820 } else {
1821 if (len > MHLEN)
1822 goto bad;
1823 MGET(m, M_DONTWAIT, n->m_type);
1824 if (m == 0)
1825 goto bad;
1826 m->m_len = 0;
1827 if (n->m_flags & M_PKTHDR) {
1828 M_COPY_PKTHDR(m, n);
1829 n->m_flags &= ~M_PKTHDR;
1830 }
1831 }
1832 space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
1833 do {
1834 count = min(min(max(len, max_protohdr), space), n->m_len);
1835 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
1836 (unsigned)count);
1837 len -= count;
1838 m->m_len += count;
1839 n->m_len -= count;
1840 space -= count;
1841 if (n->m_len)
1842 n->m_data += count;
1843 else
1844 n = m_free(n);
1845 } while (len > 0 && n);
1846 if (len > 0) {
1847 (void) m_free(m);
1848 goto bad;
1849 }
1850 m->m_next = n;
1851 return (m);
1852 bad:
1853 m_freem(n);
1854 MPFail++;
1855 return (0);
1856 }
1857
1858 /*
1859 * Partition an mbuf chain in two pieces, returning the tail --
1860 * all but the first len0 bytes. In case of failure, it returns NULL and
1861 * attempts to restore the chain to its original state.
1862 */
1863 struct mbuf *
1864 m_split(
1865 struct mbuf *m0,
1866 int len0,
1867 int wait)
1868 {
1869 struct mbuf *m, *n;
1870 unsigned len = len0, remain;
1871
1872 for (m = m0; m && len > m->m_len; m = m->m_next)
1873 len -= m->m_len;
1874 if (m == 0)
1875 return (0);
1876 remain = m->m_len - len;
1877 if (m0->m_flags & M_PKTHDR) {
1878 MGETHDR(n, wait, m0->m_type);
1879 if (n == 0)
1880 return (0);
1881 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
1882 n->m_pkthdr.len = m0->m_pkthdr.len - len0;
1883 m0->m_pkthdr.len = len0;
1884 if (m->m_flags & M_EXT)
1885 goto extpacket;
1886 if (remain > MHLEN) {
1887 /* m can't be the lead packet */
1888 MH_ALIGN(n, 0);
1889 n->m_next = m_split(m, len, wait);
1890 if (n->m_next == 0) {
1891 (void) m_free(n);
1892 return (0);
1893 } else
1894 return (n);
1895 } else
1896 MH_ALIGN(n, remain);
1897 } else if (remain == 0) {
1898 n = m->m_next;
1899 m->m_next = 0;
1900 return (n);
1901 } else {
1902 MGET(n, wait, m->m_type);
1903 if (n == 0)
1904 return (0);
1905 M_ALIGN(n, remain);
1906 }
1907 extpacket:
1908 if (m->m_flags & M_EXT) {
1909 n->m_flags |= M_EXT;
1910 MBUF_LOCK();
1911 n->m_ext = m->m_ext;
1912 insque((queue_t)&n->m_ext.ext_refs, (queue_t)&m->m_ext.ext_refs);
1913 MBUF_UNLOCK();
1914 n->m_data = m->m_data + len;
1915 } else {
1916 bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain);
1917 }
1918 n->m_len = remain;
1919 m->m_len = len;
1920 n->m_next = m->m_next;
1921 m->m_next = 0;
1922 return (n);
1923 }
1924 /*
1925 * Routine to copy from device local memory into mbufs.
1926 */
1927 struct mbuf *
1928 m_devget(
1929 char *buf,
1930 int totlen,
1931 int off0,
1932 struct ifnet *ifp,
1933 void (*copy)(const void *, void *, size_t))
1934 {
1935 struct mbuf *m;
1936 struct mbuf *top = 0, **mp = &top;
1937 int off = off0, len;
1938 char *cp;
1939 char *epkt;
1940
1941 cp = buf;
1942 epkt = cp + totlen;
1943 if (off) {
1944 /*
1945 * If 'off' is non-zero, packet is trailer-encapsulated,
1946 * so we have to skip the type and length fields.
1947 */
1948 cp += off + 2 * sizeof(u_int16_t);
1949 totlen -= 2 * sizeof(u_int16_t);
1950 }
1951 MGETHDR(m, M_DONTWAIT, MT_DATA);
1952 if (m == 0)
1953 return (0);
1954 m->m_pkthdr.rcvif = ifp;
1955 m->m_pkthdr.len = totlen;
1956 m->m_len = MHLEN;
1957
1958 while (totlen > 0) {
1959 if (top) {
1960 MGET(m, M_DONTWAIT, MT_DATA);
1961 if (m == 0) {
1962 m_freem(top);
1963 return (0);
1964 }
1965 m->m_len = MLEN;
1966 }
1967 len = min(totlen, epkt - cp);
1968 if (len >= MINCLSIZE) {
1969 MCLGET(m, M_DONTWAIT);
1970 if (m->m_flags & M_EXT)
1971 m->m_len = len = min(len, MCLBYTES);
1972 else {
1973 /* give up when it's out of cluster mbufs */
1974 if (top)
1975 m_freem(top);
1976 m_freem(m);
1977 return (0);
1978 }
1979 } else {
1980 /*
1981 * Place initial small packet/header at end of mbuf.
1982 */
1983 if (len < m->m_len) {
1984 if (top == 0 && len + max_linkhdr <= m->m_len)
1985 m->m_data += max_linkhdr;
1986 m->m_len = len;
1987 } else
1988 len = m->m_len;
1989 }
1990 if (copy)
1991 copy(cp, mtod(m, caddr_t), (unsigned)len);
1992 else
1993 bcopy(cp, mtod(m, caddr_t), (unsigned)len);
1994 cp += len;
1995 *mp = m;
1996 mp = &m->m_next;
1997 totlen -= len;
1998 if (cp == epkt)
1999 cp = buf;
2000 }
2001 return (top);
2002 }
2003
2004 /*
2005 * Cluster freelist allocation check. The mbuf lock must be held.
2006 * Ensure hysteresis between hi/lo.
2007 */
2008 static int
2009 m_howmany(int num, size_t bufsize)
2010 {
2011 int i = 0;
2012
2013 /* Bail if we've maxed out the mbuf memory map */
2014 if (mbstat.m_clusters + (mbstat.m_bigclusters << 1) < nmbclusters) {
2015 int j = 0;
2016
2017 if (bufsize == MCLBYTES) {
2018 /* Under minimum */
2019 if (mbstat.m_clusters < MINCL)
2020 return (MINCL - mbstat.m_clusters);
2021 /* Too few (free < 1/2 total) and not over maximum */
2022 if (mbstat.m_clusters < (nmbclusters >> 1)) {
2023 if (num >= mbstat.m_clfree)
2024 i = num - mbstat.m_clfree;
2025 if (((mbstat.m_clusters + num) >> 1) > mbstat.m_clfree)
2026 j = ((mbstat.m_clusters + num) >> 1) - mbstat.m_clfree;
2027 i = max(i, j);
2028 if (i + mbstat.m_clusters >= (nmbclusters >> 1))
2029 i = (nmbclusters >> 1) - mbstat.m_clusters;
2030 }
2031 } else {
2032 /* Under minimum */
2033 if (mbstat.m_bigclusters < MINCL)
2034 return (MINCL - mbstat.m_bigclusters);
2035 /* Too few (free < 1/2 total) and not over maximum */
2036 if (mbstat.m_bigclusters < (nmbclusters >> 2)) {
2037 if (num >= mbstat.m_bigclfree)
2038 i = num - mbstat.m_bigclfree;
2039 if (((mbstat.m_bigclusters + num) >> 1) > mbstat.m_bigclfree)
2040 j = ((mbstat.m_bigclusters + num) >> 1) - mbstat.m_bigclfree;
2041 i = max(i, j);
2042 if (i + mbstat.m_bigclusters >= (nmbclusters >> 2))
2043 i = (nmbclusters >> 2) - mbstat.m_bigclusters;
2044 }
2045 }
2046 }
2047 return i;
2048 }
2049
2050 /*
2051 * Copy data from a buffer back into the indicated mbuf chain,
2052 * starting "off" bytes from the beginning, extending the mbuf
2053 * chain if necessary.
2054 */
2055 void
2056 m_copyback(
2057 struct mbuf *m0,
2058 int off,
2059 int len,
2060 caddr_t cp)
2061 {
2062 int mlen;
2063 struct mbuf *m = m0, *n;
2064 int totlen = 0;
2065
2066 if (m0 == 0)
2067 return;
2068 while (off > (mlen = m->m_len)) {
2069 off -= mlen;
2070 totlen += mlen;
2071 if (m->m_next == 0) {
2072 n = m_getclr(M_DONTWAIT, m->m_type);
2073 if (n == 0)
2074 goto out;
2075 n->m_len = min(MLEN, len + off);
2076 m->m_next = n;
2077 }
2078 m = m->m_next;
2079 }
2080 while (len > 0) {
2081 mlen = min (m->m_len - off, len);
2082 bcopy(cp, off + mtod(m, caddr_t), (unsigned)mlen);
2083 cp += mlen;
2084 len -= mlen;
2085 mlen += off;
2086 off = 0;
2087 totlen += mlen;
2088 if (len == 0)
2089 break;
2090 if (m->m_next == 0) {
2091 n = m_get(M_DONTWAIT, m->m_type);
2092 if (n == 0)
2093 break;
2094 n->m_len = min(MLEN, len);
2095 m->m_next = n;
2096 }
2097 m = m->m_next;
2098 }
2099 out: if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen))
2100 m->m_pkthdr.len = totlen;
2101 }
2102
2103
2104 char *mcl_to_paddr(char *addr) {
2105 int base_phys;
2106
2107 if (addr < (char *)mbutl || addr >= (char *)embutl)
2108 return (0);
2109 base_phys = mcl_paddr[(addr - (char *)mbutl) >> PGSHIFT];
2110
2111 if (base_phys == 0)
2112 return (0);
2113 return ((char *)((int)base_phys | ((int)addr & PGOFSET)));
2114 }
2115
2116 /*
2117 * Dup the mbuf chain passed in. The whole thing. No cute additional cruft.
2118 * And really copy the thing. That way, we don't "precompute" checksums
2119 * for unsuspecting consumers.
2120 * Assumption: m->m_nextpkt == 0.
2121 * Trick: for small packets, don't dup into a cluster. That way received
2122 * packets don't take up too much room in the sockbuf (cf. sbspace()).
2123 */
2124 int MDFail;
2125
2126 struct mbuf *
2127 m_dup(struct mbuf *m, int how)
2128 {
2129 struct mbuf *n, **np;
2130 struct mbuf *top;
2131 int copyhdr = 0;
2132
2133 np = &top;
2134 top = 0;
2135 if (m->m_flags & M_PKTHDR)
2136 copyhdr = 1;
2137
2138 /*
2139 * Quick check: if we have one mbuf and its data fits in an
2140 * mbuf with packet header, just copy and go.
2141 */
2142 if (m->m_next == NULL)
2143 { /* Then just move the data into an mbuf and be done... */
2144 if (copyhdr)
2145 { if (m->m_pkthdr.len <= MHLEN)
2146 { if ((n = m_gethdr(how, m->m_type)) == NULL)
2147 return(NULL);
2148 n->m_len = m->m_len;
2149 n->m_flags |= (m->m_flags & M_COPYFLAGS);
2150 n->m_pkthdr.len = m->m_pkthdr.len;
2151 n->m_pkthdr.rcvif = m->m_pkthdr.rcvif;
2152 n->m_pkthdr.header = NULL;
2153 n->m_pkthdr.csum_flags = 0;
2154 n->m_pkthdr.csum_data = 0;
2155 n->m_pkthdr.aux = NULL;
2156 n->m_pkthdr.vlan_tag = 0;
2157 n->m_pkthdr.socket_id = 0;
2158 SLIST_INIT(&n->m_pkthdr.tags);
2159 bcopy(m->m_data, n->m_data, m->m_pkthdr.len);
2160 return(n);
2161 }
2162 } else if (m->m_len <= MLEN)
2163 { if ((n = m_get(how, m->m_type)) == NULL)
2164 return(NULL);
2165 bcopy(m->m_data, n->m_data, m->m_len);
2166 n->m_len = m->m_len;
2167 return(n);
2168 }
2169 }
2170 while (m)
2171 {
2172 #if BLUE_DEBUG
2173 kprintf("<%x: %x, %x, %x\n", m, m->m_flags, m->m_len,
2174 m->m_data);
2175 #endif
2176 if (copyhdr)
2177 n = m_gethdr(how, m->m_type);
2178 else
2179 n = m_get(how, m->m_type);
2180 if (n == 0)
2181 goto nospace;
2182 if (m->m_flags & M_EXT)
2183 { MCLGET(n, how);
2184 if ((n->m_flags & M_EXT) == 0)
2185 goto nospace;
2186 }
2187 *np = n;
2188 if (copyhdr)
2189 { /* Don't use M_COPY_PKTHDR: preserve m_data */
2190 n->m_pkthdr = m->m_pkthdr;
2191 n->m_flags |= (m->m_flags & M_COPYFLAGS);
2192 copyhdr = 0;
2193 if ((n->m_flags & M_EXT) == 0)
2194 n->m_data = n->m_pktdat;
2195 }
2196 n->m_len = m->m_len;
2197 /*
2198 * Get the dup on the same bdry as the original
2199 * Assume that the two mbufs have the same offset to data area
2200 * (up to word bdries)
2201 */
2202 bcopy(mtod(m, caddr_t), mtod(n, caddr_t), (unsigned)n->m_len);
2203 m = m->m_next;
2204 np = &n->m_next;
2205 #if BLUE_DEBUG
2206 kprintf(">%x: %x, %x, %x\n", n, n->m_flags, n->m_len,
2207 n->m_data);
2208 #endif
2209 }
2210
2211 if (top == 0)
2212 MDFail++;
2213 return (top);
2214 nospace:
2215 m_freem(top);
2216 MDFail++;
2217 return (0);
2218 }
2219
2220 int
2221 m_mclref(struct mbuf *p)
2222 {
2223 return (_MCLREF(p));
2224 }
2225
2226 int
2227 m_mclunref(struct mbuf *p)
2228 {
2229 return (_MCLUNREF(p));
2230 }
2231
2232 /* change mbuf to new type */
2233 void
2234 m_mchtype(struct mbuf *m, int t)
2235 {
2236 MBUF_LOCK();
2237 mbstat.m_mtypes[(m)->m_type]--;
2238 mbstat.m_mtypes[t]++;
2239 (m)->m_type = t;
2240 MBUF_UNLOCK();
2241 }
2242
2243 void *m_mtod(struct mbuf *m)
2244 {
2245 return ((m)->m_data);
2246 }
2247
2248 struct mbuf *m_dtom(void *x)
2249 {
2250 return ((struct mbuf *)((u_long)(x) & ~(MSIZE-1)));
2251 }
2252
2253 int m_mtocl(void *x)
2254 {
2255 return (((char *)(x) - (char *)mbutl) / sizeof(union mcluster));
2256 }
2257
2258 union mcluster *m_cltom(int x)
2259 {
2260 return ((union mcluster *)(mbutl + (x)));
2261 }
2262
2263
2264 void m_mcheck(struct mbuf *m)
2265 {
2266 if (m->m_type != MT_FREE)
2267 panic("mget MCHECK: m_type=%x m=%x", m->m_type, m);
2268 }
2269
2270 static void
2271 mbuf_expand_thread(void)
2272 {
2273 while (1) {
2274 MBUF_LOCK();
2275 if (mbuf_expand_mcl) {
2276 int n;
2277
2278 /* Adjust to the current number of cluster in use */
2279 n = mbuf_expand_mcl - (mbstat.m_clusters - mbstat.m_clfree);
2280 mbuf_expand_mcl = 0;
2281
2282 if (n > 0)
2283 (void)m_clalloc(n, M_WAIT, MCLBYTES, 1);
2284 }
2285 if (mbuf_expand_big) {
2286 int n;
2287
2288 /* Adjust to the current number of 4 KB cluster in use */
2289 n = mbuf_expand_big - (mbstat.m_bigclusters - mbstat.m_bigclfree);
2290 mbuf_expand_big = 0;
2291
2292 if (n > 0)
2293 (void)m_clalloc(n, M_WAIT, NBPG, 1);
2294 }
2295 MBUF_UNLOCK();
2296 /*
2297 * Because we can run out of memory before filling the mbuf map, we
2298 * should not allocate more clusters than they are mbufs -- otherwise
2299 * we could have a large number of useless clusters allocated.
2300 */
2301 while (mbstat.m_mbufs < mbstat.m_bigclusters + mbstat.m_clusters) {
2302 if (m_expand(M_WAIT) == 0)
2303 break;
2304 }
2305
2306 assert_wait(&mbuf_expand_thread_wakeup, THREAD_UNINT);
2307 (void) thread_block((thread_continue_t)mbuf_expand_thread);
2308 }
2309 }
2310
2311 static void
2312 mbuf_expand_thread_init(void)
2313 {
2314 mbuf_expand_thread_initialized++;
2315 mbuf_expand_thread();
2316 }
2317
2318 SYSCTL_DECL(_kern_ipc);
2319 SYSCTL_STRUCT(_kern_ipc, KIPC_MBSTAT, mbstat, CTLFLAG_RW, &mbstat, mbstat, "");
2320
mirror of XNU