]> git.saurik.com Git - apple/xnu.git/blob - bsd/kern/uipc_mbuf.c
projects / apple / xnu.git / blob
? search:


5d910167b4791fe17b6447938bd51543dbdc704a
[apple/xnu.git] / bsd / kern / uipc_mbuf.c
1 /*
2 * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * The contents of this file constitute Original Code as defined in and
7 * are subject to the Apple Public Source License Version 1.1 (the
8 * "License"). You may not use this file except in compliance with the
9 * License. Please obtain a copy of the License at
10 * http://www.apple.com/publicsource and read it before using this file.
11 *
12 * This Original Code and all software distributed under the License are
13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17 * License for the specific language governing rights and limitations
18 * under the License.
19 *
20 * @APPLE_LICENSE_HEADER_END@
21 */
22 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
23 /*
24 * Copyright (c) 1982, 1986, 1988, 1991, 1993
25 * The Regents of the University of California. All rights reserved.
26 *
27 * Redistribution and use in source and binary forms, with or without
28 * modification, are permitted provided that the following conditions
29 * are met:
30 * 1. Redistributions of source code must retain the above copyright
31 * notice, this list of conditions and the following disclaimer.
32 * 2. Redistributions in binary form must reproduce the above copyright
33 * notice, this list of conditions and the following disclaimer in the
34 * documentation and/or other materials provided with the distribution.
35 * 3. All advertising materials mentioning features or use of this software
36 * must display the following acknowledgement:
37 * This product includes software developed by the University of
38 * California, Berkeley and its contributors.
39 * 4. Neither the name of the University nor the names of its contributors
40 * may be used to endorse or promote products derived from this software
41 * without specific prior written permission.
42 *
43 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
44 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
45 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
46 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
47 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
48 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
49 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
50 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
51 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
52 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
53 * SUCH DAMAGE.
54 *
55 * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94
56 */
57 /* HISTORY
58 *
59 * 10/15/97 Annette DeSchon (deschon@apple.com)
60 * Fixed bug in which all cluster mbufs were broken up
61 * into regular mbufs: Some clusters are now reserved.
62 * When a cluster is needed, regular mbufs are no longer
63 * used. (Radar 1683621)
64 * 20-May-95 Mac Gillon (mgillon) at NeXT
65 * New version based on 4.4
66 */
67
68 #include <sys/param.h>
69 #include <sys/systm.h>
70 #include <sys/malloc.h>
71 #include <sys/mbuf.h>
72 #include <sys/kernel.h>
73 #include <sys/sysctl.h>
74 #include <sys/syslog.h>
75 #include <sys/protosw.h>
76 #include <sys/domain.h>
77
78 #include <kern/queue.h>
79 #include <kern/kern_types.h>
80 #include <kern/sched_prim.h>
81
82 #include <IOKit/IOMapper.h>
83
84 extern vm_offset_t kmem_mb_alloc(vm_map_t , int );
85 extern boolean_t PE_parse_boot_arg(const char *, void *);
86
87 #define _MCLREF(p) (++mclrefcnt[mtocl(p)])
88 #define _MCLUNREF(p) (--mclrefcnt[mtocl(p)] == 0)
89 #define _M_CLEAR_PKTHDR(mbuf_ptr) (mbuf_ptr)->m_pkthdr.rcvif = NULL; \
90 (mbuf_ptr)->m_pkthdr.len = 0; \
91 (mbuf_ptr)->m_pkthdr.header = NULL; \
92 (mbuf_ptr)->m_pkthdr.csum_flags = 0; \
93 (mbuf_ptr)->m_pkthdr.csum_data = 0; \
94 (mbuf_ptr)->m_pkthdr.aux = (struct mbuf*)NULL; \
95 (mbuf_ptr)->m_pkthdr.vlan_tag = 0; \
96 (mbuf_ptr)->m_pkthdr.socket_id = 0; \
97 SLIST_INIT(&(mbuf_ptr)->m_pkthdr.tags);
98
99 /* kernel translater */
100 extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
101
102 lck_mtx_t * mbuf_mlock;
103 lck_grp_t * mbuf_mlock_grp;
104 lck_grp_attr_t * mbuf_mlock_grp_attr;
105 lck_attr_t * mbuf_mlock_attr;
106 extern lck_mtx_t *domain_proto_mtx;
107
108 struct mbuf *mfree; /* mbuf free list */
109 struct mbuf *mfreelater; /* mbuf deallocation list */
110 extern vm_map_t mb_map; /* special map */
111 int m_want; /* sleepers on mbufs */
112 short *mclrefcnt; /* mapped cluster reference counts */
113 int *mcl_paddr;
114 static ppnum_t mcl_paddr_base; /* Handle returned by IOMapper::iovmAlloc() */
115 union mcluster *mclfree; /* mapped cluster free list */
116 union mbigcluster *mbigfree; /* mapped cluster free list */
117 int max_linkhdr; /* largest link-level header */
118 int max_protohdr; /* largest protocol header */
119 int max_hdr; /* largest link+protocol header */
120 int max_datalen; /* MHLEN - max_hdr */
121 struct mbstat mbstat; /* statistics */
122 union mcluster *mbutl; /* first mapped cluster address */
123 union mcluster *embutl; /* ending virtual address of mclusters */
124
125 static int nclpp; /* # clusters per physical page */
126
127 static int m_howmany(int, size_t );
128 void m_reclaim(void);
129 static int m_clalloc(const int , const int, const size_t, int);
130 int do_reclaim = 0;
131
132 #define MF_NOWAIT 0x1
133 #define MF_BIG 0x2
134
135 /* The number of cluster mbufs that are allocated, to start. */
136 #define MINCL max(16, 2)
137
138 static int mbuf_expand_thread_wakeup = 0;
139 static int mbuf_expand_mcl = 0;
140 static int mbuf_expand_big = 0;
141 static int mbuf_expand_thread_initialized = 0;
142
143 static void mbuf_expand_thread_init(void);
144 static void mbuf_expand_thread(void);
145 static int m_expand(int );
146 static caddr_t m_bigalloc(int );
147 static void m_bigfree(caddr_t , u_int , caddr_t );
148 static struct mbuf * m_mbigget(struct mbuf *, int );
149 void mbinit(void);
150 static void m_range_check(void *addr);
151
152
153 #if 0
154 static int mfree_munge = 0;
155 #if 0
156 #define _MFREE_MUNGE(m) { \
157 if (mfree_munge) \
158 { int i; \
159 vm_offset_t *element = (vm_offset_t *)(m); \
160 for (i = 0; \
161 i < sizeof(struct mbuf)/sizeof(vm_offset_t); \
162 i++) \
163 (element)[i] = 0xdeadbeef; \
164 } \
165 }
166 #else
167 void
168 munge_mbuf(struct mbuf *m)
169 {
170 int i;
171 vm_offset_t *element = (vm_offset_t *)(m);
172 for (i = 0;
173 i < sizeof(struct mbuf)/sizeof(vm_offset_t);
174 i++)
175 (element)[i] = 0xdeadbeef;
176 }
177 #define _MFREE_MUNGE(m) { \
178 if (mfree_munge) \
179 munge_mbuf(m); \
180 }
181 #endif
182 #else
183 #define _MFREE_MUNGE(m)
184 #endif
185
186
187 #define _MINTGET(m, type) { \
188 MBUF_LOCK(); \
189 if (((m) = mfree) != 0) { \
190 MCHECK(m); \
191 ++mclrefcnt[mtocl(m)]; \
192 mbstat.m_mtypes[MT_FREE]--; \
193 mbstat.m_mtypes[(type)]++; \
194 mfree = (m)->m_next; \
195 } \
196 MBUF_UNLOCK(); \
197 }
198
199
200 static void
201 m_range_check(void *addr)
202 {
203 if (addr && (addr < (void *)mbutl || addr >= (void *)embutl))
204 panic("mbuf address out of range 0x%x", addr);
205 }
206
207 __private_extern__ void
208 mbinit(void)
209 {
210 int m;
211 int initmcl = 32;
212 int mcl_pages;
213
214 if (nclpp)
215 return;
216 nclpp = round_page_32(MCLBYTES) / MCLBYTES; /* see mbufgc() */
217 if (nclpp < 1) nclpp = 1;
218 mbuf_mlock_grp_attr = lck_grp_attr_alloc_init();
219 lck_grp_attr_setdefault(mbuf_mlock_grp_attr);
220
221 mbuf_mlock_grp = lck_grp_alloc_init("mbuf", mbuf_mlock_grp_attr);
222 mbuf_mlock_attr = lck_attr_alloc_init();
223 lck_attr_setdefault(mbuf_mlock_attr);
224
225 mbuf_mlock = lck_mtx_alloc_init(mbuf_mlock_grp, mbuf_mlock_attr);
226
227 mbstat.m_msize = MSIZE;
228 mbstat.m_mclbytes = MCLBYTES;
229 mbstat.m_minclsize = MINCLSIZE;
230 mbstat.m_mlen = MLEN;
231 mbstat.m_mhlen = MHLEN;
232 mbstat.m_bigmclbytes = NBPG;
233
234 if (nmbclusters == 0)
235 nmbclusters = NMBCLUSTERS;
236 MALLOC(mclrefcnt, short *, nmbclusters * sizeof (short),
237 M_TEMP, M_WAITOK);
238 if (mclrefcnt == 0)
239 panic("mbinit");
240 for (m = 0; m < nmbclusters; m++)
241 mclrefcnt[m] = -1;
242
243 /* Calculate the number of pages assigned to the cluster pool */
244 mcl_pages = nmbclusters/(NBPG/CLBYTES);
245 MALLOC(mcl_paddr, int *, mcl_pages * sizeof(int), M_TEMP, M_WAITOK);
246 if (mcl_paddr == 0)
247 panic("mbinit1");
248 /* Register with the I/O Bus mapper */
249 mcl_paddr_base = IOMapperIOVMAlloc(mcl_pages);
250 bzero((char *)mcl_paddr, mcl_pages * sizeof(int));
251
252 embutl = (union mcluster *)((unsigned char *)mbutl + (nmbclusters * MCLBYTES));
253
254 PE_parse_boot_arg("initmcl", &initmcl);
255
256 if (m_clalloc(max(NBPG/CLBYTES, 1) * initmcl, M_WAIT, MCLBYTES, 0) == 0)
257 goto bad;
258 MBUF_UNLOCK();
259
260 (void) kernel_thread(kernel_task, mbuf_expand_thread_init);
261
262 return;
263 bad:
264 panic("mbinit");
265 }
266
267 /*
268 * Allocate some number of mbuf clusters
269 * and place on cluster free list.
270 * Take the mbuf lock (if not already locked) and do not release it
271 */
272 /* ARGSUSED */
273 static int
274 m_clalloc(
275 const int num,
276 const int nowait,
277 const size_t bufsize,
278 int locked)
279 {
280 int i;
281 vm_size_t size = 0;
282 int numpages = 0;
283 vm_offset_t page = 0;
284
285 if (locked == 0)
286 MBUF_LOCK();
287 /*
288 * Honor the caller's wish to block or not block.
289 * We have a way to grow the pool asynchronously,
290 * by kicking the dlil_input_thread.
291 */
292 i = m_howmany(num, bufsize);
293 if (i == 0 || nowait == M_DONTWAIT)
294 goto out;
295
296 MBUF_UNLOCK();
297 size = round_page_32(i * bufsize);
298 page = kmem_mb_alloc(mb_map, size);
299
300 if (page == 0) {
301 size = NBPG; /* Try for 1 if failed */
302 page = kmem_mb_alloc(mb_map, size);
303 }
304 MBUF_LOCK();
305
306 if (page) {
307 numpages = size / NBPG;
308 for (i = 0; i < numpages; i++, page += NBPG) {
309 if (((int)page & PGOFSET) == 0) {
310 ppnum_t offset = ((char *)page - (char *)mbutl)/NBPG;
311 ppnum_t new_page = pmap_find_phys(kernel_pmap, (vm_address_t) page);
312
313 /*
314 * In the case of no mapper being available
315 * the following code nops and returns the
316 * input page, if there is a mapper the I/O
317 * page appropriate is returned.
318 */
319 new_page = IOMapperInsertPage(mcl_paddr_base, offset, new_page);
320 mcl_paddr[offset] = new_page << 12;
321 }
322 if (bufsize == MCLBYTES) {
323 union mcluster *mcl = (union mcluster *)page;
324
325 if (++mclrefcnt[mtocl(mcl)] != 0)
326 panic("m_clalloc already there");
327 mcl->mcl_next = mclfree;
328 mclfree = mcl++;
329 if (++mclrefcnt[mtocl(mcl)] != 0)
330 panic("m_clalloc already there");
331 mcl->mcl_next = mclfree;
332 mclfree = mcl++;
333 } else {
334 union mbigcluster *mbc = (union mbigcluster *)page;
335
336 if (++mclrefcnt[mtocl(mbc)] != 0)
337 panic("m_clalloc already there");
338 if (++mclrefcnt[mtocl(mbc) + 1] != 0)
339 panic("m_clalloc already there");
340
341 mbc->mbc_next = mbigfree;
342 mbigfree = mbc;
343 }
344 }
345 if (bufsize == MCLBYTES) {
346 int numcl = numpages << 1;
347 mbstat.m_clfree += numcl;
348 mbstat.m_clusters += numcl;
349 return (numcl);
350 } else {
351 mbstat.m_bigclfree += numpages;
352 mbstat.m_bigclusters += numpages;
353 return (numpages);
354 }
355 } /* else ... */
356 out:
357 /*
358 * When non-blocking we kick a thread if we havve to grow the
359 * pool or if the number of free clusters is less than requested.
360 */
361 if (bufsize == MCLBYTES) {
362 if (i > 0) {
363 /* Remember total number of clusters needed at this time */
364 i += mbstat.m_clusters;
365 if (i > mbuf_expand_mcl) {
366 mbuf_expand_mcl = i;
367 if (mbuf_expand_thread_initialized)
368 wakeup((caddr_t)&mbuf_expand_thread_wakeup);
369 }
370 }
371
372 if (mbstat.m_clfree >= num)
373 return 1;
374 } else {
375 if (i > 0) {
376 /* Remember total number of 4KB clusters needed at this time */
377 i += mbstat.m_bigclusters;
378 if (i > mbuf_expand_big) {
379 mbuf_expand_big = i;
380 if (mbuf_expand_thread_initialized)
381 wakeup((caddr_t)&mbuf_expand_thread_wakeup);
382 }
383 }
384
385 if (mbstat.m_bigclfree >= num)
386 return 1;
387 }
388 return 0;
389 }
390
391 /*
392 * Add more free mbufs by cutting up a cluster.
393 */
394 static int
395 m_expand(int canwait)
396 {
397 caddr_t mcl;
398
399 if (mbstat.m_clfree < (mbstat.m_clusters >> 4)) {
400 /*
401 * 1/16th of the total number of cluster mbufs allocated is
402 * reserved for large packets. The number reserved must
403 * always be < 1/2, or future allocation will be prevented.
404 */
405 (void)m_clalloc(1, canwait, MCLBYTES, 0);
406 MBUF_UNLOCK();
407 if (mbstat.m_clfree < (mbstat.m_clusters >> 4))
408 return 0;
409 }
410
411 MCLALLOC(mcl, canwait);
412 if (mcl) {
413 struct mbuf *m = (struct mbuf *)mcl;
414 int i = NMBPCL;
415 MBUF_LOCK();
416 mbstat.m_mtypes[MT_FREE] += i;
417 mbstat.m_mbufs += i;
418 while (i--) {
419 _MFREE_MUNGE(m);
420 m->m_type = MT_FREE;
421 m->m_next = mfree;
422 mfree = m++;
423 }
424 i = m_want;
425 m_want = 0;
426 MBUF_UNLOCK();
427 if (i) wakeup((caddr_t)&mfree);
428 return 1;
429 }
430 return 0;
431 }
432
433 /*
434 * When MGET failes, ask protocols to free space when short of memory,
435 * then re-attempt to allocate an mbuf.
436 */
437 struct mbuf *
438 m_retry(
439 int canwait,
440 int type)
441 {
442 struct mbuf *m;
443 int wait;
444
445 for (;;) {
446 (void) m_expand(canwait);
447 _MINTGET(m, type);
448 if (m) {
449 (m)->m_next = (m)->m_nextpkt = 0;
450 (m)->m_type = (type);
451 (m)->m_data = (m)->m_dat;
452 (m)->m_flags = 0;
453 (m)->m_len = 0;
454 }
455 if (m || canwait == M_DONTWAIT)
456 break;
457 MBUF_LOCK();
458 wait = m_want++;
459 mbuf_expand_mcl++;
460 if (wait == 0)
461 mbstat.m_drain++;
462 else
463 mbstat.m_wait++;
464 MBUF_UNLOCK();
465
466 if (mbuf_expand_thread_initialized)
467 wakeup((caddr_t)&mbuf_expand_thread_wakeup);
468
469 if (wait == 0) {
470 m_reclaim();
471 } else {
472 struct timespec ts;
473 ts.tv_sec = 1;
474 ts.tv_nsec = 0;
475 (void) msleep((caddr_t)&mfree, 0, (PZERO-1) | PDROP, "m_retry", &ts);
476 }
477 }
478 if (m == 0)
479 mbstat.m_drops++;
480 return (m);
481 }
482
483 /*
484 * As above; retry an MGETHDR.
485 */
486 struct mbuf *
487 m_retryhdr(
488 int canwait,
489 int type)
490 {
491 struct mbuf *m;
492
493 if ((m = m_retry(canwait, type))) {
494 m->m_next = m->m_nextpkt = 0;
495 m->m_flags |= M_PKTHDR;
496 m->m_data = m->m_pktdat;
497 _M_CLEAR_PKTHDR(m);
498 }
499 return (m);
500 }
501
502 void
503 m_reclaim(void)
504 {
505 do_reclaim = 1; /* drain is performed in pfslowtimo(), to avoid deadlocks */
506 mbstat.m_drain++;
507 }
508
509 /*
510 * Space allocation routines.
511 * These are also available as macros
512 * for critical paths.
513 */
514 struct mbuf *
515 m_get(
516 int nowait,
517 int type)
518 {
519 struct mbuf *m;
520
521 m_range_check(mfree);
522 m_range_check(mclfree);
523 m_range_check(mbigfree);
524
525 _MINTGET(m, type);
526 if (m) {
527 m->m_next = m->m_nextpkt = 0;
528 m->m_type = type;
529 m->m_data = m->m_dat;
530 m->m_flags = 0;
531 m->m_len = 0;
532 } else
533 (m) = m_retry(nowait, type);
534
535 m_range_check(mfree);
536 m_range_check(mclfree);
537 m_range_check(mbigfree);
538
539
540 return (m);
541 }
542
543 struct mbuf *
544 m_gethdr(
545 int nowait,
546 int type)
547 {
548 struct mbuf *m;
549
550 m_range_check(mfree);
551 m_range_check(mclfree);
552 m_range_check(mbigfree);
553
554
555 _MINTGET(m, type);
556 if (m) {
557 m->m_next = m->m_nextpkt = 0;
558 m->m_type = type;
559 m->m_data = m->m_pktdat;
560 m->m_flags = M_PKTHDR;
561 m->m_len = 0;
562 _M_CLEAR_PKTHDR(m)
563 } else
564 m = m_retryhdr(nowait, type);
565
566 m_range_check(mfree);
567 m_range_check(mclfree);
568 m_range_check(mbigfree);
569
570
571 return m;
572 }
573
574 struct mbuf *
575 m_getclr(
576 int nowait,
577 int type)
578 {
579 struct mbuf *m;
580
581 MGET(m, nowait, type);
582 if (m == 0)
583 return (0);
584 bzero(mtod(m, caddr_t), MLEN);
585 return (m);
586 }
587
588 struct mbuf *
589 m_free(
590 struct mbuf *m)
591 {
592 struct mbuf *n = m->m_next;
593 int i;
594
595 m_range_check(m);
596 m_range_check(mfree);
597 m_range_check(mclfree);
598
599 if (m->m_type == MT_FREE)
600 panic("freeing free mbuf");
601
602 /* Free the aux data if there is any */
603 if ((m->m_flags & M_PKTHDR) && m->m_pkthdr.aux)
604 {
605 m_freem(m->m_pkthdr.aux);
606 }
607 if ((m->m_flags & M_PKTHDR) != 0)
608 m_tag_delete_chain(m, NULL);
609
610 MBUF_LOCK();
611 if ((m->m_flags & M_EXT))
612 {
613 if (MCLHASREFERENCE(m)) {
614 remque((queue_t)&m->m_ext.ext_refs);
615 } else if (m->m_ext.ext_free == NULL) {
616 union mcluster *mcl= (union mcluster *)m->m_ext.ext_buf;
617
618 m_range_check(mcl);
619
620 if (_MCLUNREF(mcl)) {
621 mcl->mcl_next = mclfree;
622 mclfree = mcl;
623 ++mbstat.m_clfree;
624 }
625 #ifdef COMMENT_OUT
626 /* *** Since m_split() increments "mclrefcnt[mtocl(m->m_ext.ext_buf)]",
627 and AppleTalk ADSP uses m_split(), this incorrect sanity check
628 caused a panic.
629 *** */
630 else /* sanity check - not referenced this way */
631 panic("m_free m_ext cluster not free");
632 #endif
633 } else {
634 (*(m->m_ext.ext_free))(m->m_ext.ext_buf,
635 m->m_ext.ext_size, m->m_ext.ext_arg);
636 }
637 }
638 mbstat.m_mtypes[m->m_type]--;
639 (void) _MCLUNREF(m);
640 _MFREE_MUNGE(m);
641 m->m_type = MT_FREE;
642 mbstat.m_mtypes[m->m_type]++;
643 m->m_flags = 0;
644 m->m_next = mfree;
645 m->m_len = 0;
646 mfree = m;
647 i = m_want;
648 m_want = 0;
649 MBUF_UNLOCK();
650 if (i) wakeup((caddr_t)&mfree);
651 return (n);
652 }
653
654 /* m_mclget() add an mbuf cluster to a normal mbuf */
655 struct mbuf *
656 m_mclget(
657 struct mbuf *m,
658 int nowait)
659 {
660 MCLALLOC(m->m_ext.ext_buf, nowait);
661 if (m->m_ext.ext_buf) {
662 m->m_data = m->m_ext.ext_buf;
663 m->m_flags |= M_EXT;
664 m->m_ext.ext_size = MCLBYTES;
665 m->m_ext.ext_free = 0;
666 m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward =
667 &m->m_ext.ext_refs;
668 }
669
670 return m;
671 }
672
673 /* m_mclalloc() allocate an mbuf cluster */
674 caddr_t
675 m_mclalloc(
676 int nowait)
677 {
678 caddr_t p;
679
680 (void)m_clalloc(1, nowait, MCLBYTES, 0);
681 if ((p = (caddr_t)mclfree)) {
682 ++mclrefcnt[mtocl(p)];
683 mbstat.m_clfree--;
684 mclfree = ((union mcluster *)p)->mcl_next;
685 } else {
686 mbstat.m_drops++;
687 }
688 MBUF_UNLOCK();
689
690 return p;
691 }
692
693 /* m_mclfree() releases a reference to a cluster allocated by MCLALLOC,
694 * freeing the cluster if the reference count has reached 0. */
695 void
696 m_mclfree(
697 caddr_t p)
698 {
699 MBUF_LOCK();
700
701 m_range_check(p);
702
703 if (--mclrefcnt[mtocl(p)] == 0) {
704 ((union mcluster *)(p))->mcl_next = mclfree;
705 mclfree = (union mcluster *)(p);
706 mbstat.m_clfree++;
707 }
708 MBUF_UNLOCK();
709 }
710
711 /* mcl_hasreference() checks if a cluster of an mbuf is referenced by another mbuf */
712 int
713 m_mclhasreference(
714 struct mbuf *m)
715 {
716 return (m->m_ext.ext_refs.forward != &(m->m_ext.ext_refs));
717 }
718
719 __private_extern__ caddr_t
720 m_bigalloc(int nowait)
721 {
722 caddr_t p;
723
724 (void)m_clalloc(1, nowait, NBPG, 0);
725 if ((p = (caddr_t)mbigfree)) {
726 if (mclrefcnt[mtocl(p)] != mclrefcnt[mtocl(p) + 1])
727 panic("m_bigalloc mclrefcnt %x mismatch %d != %d",
728 p, mclrefcnt[mtocl(p)], mclrefcnt[mtocl(p) + 1]);
729 if (mclrefcnt[mtocl(p)] || mclrefcnt[mtocl(p) + 1])
730 panic("m_bigalloc mclrefcnt %x not null %d != %d",
731 p, mclrefcnt[mtocl(p)], mclrefcnt[mtocl(p) + 1]);
732 ++mclrefcnt[mtocl(p)];
733 ++mclrefcnt[mtocl(p) + 1];
734 mbstat.m_bigclfree--;
735 mbigfree = ((union mbigcluster *)p)->mbc_next;
736 } else {
737 mbstat.m_drops++;
738 }
739 MBUF_UNLOCK();
740 return p;
741 }
742
743 __private_extern__ void
744 m_bigfree(caddr_t p, __unused u_int size, __unused caddr_t arg)
745 {
746 m_range_check(p);
747
748 if (mclrefcnt[mtocl(p)] != mclrefcnt[mtocl(p) + 1])
749 panic("m_bigfree mclrefcnt %x mismatch %d != %d",
750 p, mclrefcnt[mtocl(p)], mclrefcnt[mtocl(p) + 1]);
751 --mclrefcnt[mtocl(p)];
752 --mclrefcnt[mtocl(p) + 1];
753 if (mclrefcnt[mtocl(p)] == 0) {
754 ((union mbigcluster *)(p))->mbc_next = mbigfree;
755 mbigfree = (union mbigcluster *)(p);
756 mbstat.m_bigclfree++;
757 }
758 }
759
760 /* m_mbigget() add an 4KB mbuf cluster to a normal mbuf */
761 __private_extern__ struct mbuf *
762 m_mbigget(struct mbuf *m, int nowait)
763 {
764 m->m_ext.ext_buf = m_bigalloc(nowait);
765 if (m->m_ext.ext_buf) {
766 m->m_data = m->m_ext.ext_buf;
767 m->m_flags |= M_EXT;
768 m->m_ext.ext_size = NBPG;
769 m->m_ext.ext_free = m_bigfree;
770 m->m_ext.ext_arg = 0;
771 m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward =
772 &m->m_ext.ext_refs;
773 }
774
775 return m;
776 }
777
778
779 /* */
780 void
781 m_copy_pkthdr(
782 struct mbuf *to,
783 struct mbuf *from)
784 {
785 to->m_pkthdr = from->m_pkthdr;
786 from->m_pkthdr.aux = (struct mbuf *)NULL;
787 SLIST_INIT(&from->m_pkthdr.tags); /* purge tags from src */
788 to->m_flags = from->m_flags & M_COPYFLAGS;
789 to->m_data = (to)->m_pktdat;
790 }
791
792 /*
793 * "Move" mbuf pkthdr from "from" to "to".
794 * "from" must have M_PKTHDR set, and "to" must be empty.
795 */
796 #ifndef __APPLE__
797 void
798 m_move_pkthdr(struct mbuf *to, struct mbuf *from)
799 {
800 KASSERT((to->m_flags & M_EXT) == 0, ("m_move_pkthdr: to has cluster"));
801
802 to->m_flags = from->m_flags & M_COPYFLAGS;
803 to->m_data = to->m_pktdat;
804 to->m_pkthdr = from->m_pkthdr; /* especially tags */
805 SLIST_INIT(&from->m_pkthdr.tags); /* purge tags from src */
806 from->m_flags &= ~M_PKTHDR;
807 }
808 #endif
809
810 /*
811 * Duplicate "from"'s mbuf pkthdr in "to".
812 * "from" must have M_PKTHDR set, and "to" must be empty.
813 * In particular, this does a deep copy of the packet tags.
814 */
815 static int
816 m_dup_pkthdr(struct mbuf *to, struct mbuf *from, int how)
817 {
818 to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
819 if ((to->m_flags & M_EXT) == 0)
820 to->m_data = to->m_pktdat;
821 if (to->m_pkthdr.aux != NULL)
822 m_freem(to->m_pkthdr.aux);
823 to->m_pkthdr = from->m_pkthdr;
824 to->m_pkthdr.aux = NULL;
825 (void) m_aux_copy(to, from);
826 SLIST_INIT(&to->m_pkthdr.tags);
827 return (m_tag_copy_chain(to, from, how));
828 }
829
830 /*
831 * return a list of mbuf hdrs that point to clusters...
832 * try for num_needed, if wantall is not set, return whatever
833 * number were available... set up the first num_with_pkthdrs
834 * with mbuf hdrs configured as packet headers... these are
835 * chained on the m_nextpkt field... any packets requested beyond
836 * this are chained onto the last packet header's m_next field.
837 * The size of the cluster is controlled by the paramter bufsize.
838 */
839 __private_extern__ struct mbuf *
840 m_getpackets_internal(unsigned int *num_needed, int num_with_pkthdrs, int how, int wantall, size_t bufsize)
841 {
842 struct mbuf *m;
843 struct mbuf **np, *top;
844 unsigned int num, needed = *num_needed;
845
846 if (bufsize != MCLBYTES && bufsize != NBPG)
847 return 0;
848
849 top = NULL;
850 np = &top;
851
852 (void)m_clalloc(needed, how, bufsize, 0); /* takes the MBUF_LOCK, but doesn't release it... */
853
854 for (num = 0; num < needed; num++) {
855 m_range_check(mfree);
856 m_range_check(mclfree);
857 m_range_check(mbigfree);
858
859 if (mfree && ((bufsize == NBPG && mbigfree) || (bufsize == MCLBYTES && mclfree))) {
860 /* mbuf + cluster are available */
861 m = mfree;
862 MCHECK(m);
863 mfree = m->m_next;
864 ++mclrefcnt[mtocl(m)];
865 mbstat.m_mtypes[MT_FREE]--;
866 mbstat.m_mtypes[MT_DATA]++;
867 if (bufsize == NBPG) {
868 m->m_ext.ext_buf = (caddr_t)mbigfree; /* get the big cluster */
869 ++mclrefcnt[mtocl(m->m_ext.ext_buf)];
870 ++mclrefcnt[mtocl(m->m_ext.ext_buf) + 1];
871 mbstat.m_bigclfree--;
872 mbigfree = ((union mbigcluster *)(m->m_ext.ext_buf))->mbc_next;
873 m->m_ext.ext_free = m_bigfree;
874 m->m_ext.ext_size = NBPG;
875 } else {
876 m->m_ext.ext_buf = (caddr_t)mclfree; /* get the cluster */
877 ++mclrefcnt[mtocl(m->m_ext.ext_buf)];
878 mbstat.m_clfree--;
879 mclfree = ((union mcluster *)(m->m_ext.ext_buf))->mcl_next;
880 m->m_ext.ext_free = 0;
881 m->m_ext.ext_size = MCLBYTES;
882 }
883 m->m_ext.ext_arg = 0;
884 m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward = &m->m_ext.ext_refs;
885 m->m_next = m->m_nextpkt = 0;
886 m->m_type = MT_DATA;
887 m->m_data = m->m_ext.ext_buf;
888 m->m_len = 0;
889
890 if (num_with_pkthdrs == 0)
891 m->m_flags = M_EXT;
892 else {
893 m->m_flags = M_PKTHDR | M_EXT;
894 _M_CLEAR_PKTHDR(m);
895
896 num_with_pkthdrs--;
897 }
898 } else {
899 MBUF_UNLOCK();
900
901 if (num_with_pkthdrs == 0) {
902 MGET(m, how, MT_DATA );
903 } else {
904 MGETHDR(m, how, MT_DATA);
905
906 num_with_pkthdrs--;
907 }
908 if (m == 0)
909 goto fail;
910
911 if (bufsize == NBPG)
912 m = m_mbigget(m, how);
913 else
914 m = m_mclget(m, how);
915 if ((m->m_flags & M_EXT) == 0) {
916 m_free(m);
917 goto fail;
918 }
919 MBUF_LOCK();
920 }
921 *np = m;
922
923 if (num_with_pkthdrs)
924 np = &m->m_nextpkt;
925 else
926 np = &m->m_next;
927 }
928 MBUF_UNLOCK();
929
930 *num_needed = num;
931 return (top);
932 fail:
933 if (wantall && top) {
934 m_freem(top);
935 return 0;
936 }
937 return top;
938 }
939
940
941 /*
942 * Return list of mbuf linked by m_nextpkt
943 * Try for num_needed, and if wantall is not set, return whatever
944 * number were available
945 * The size of each mbuf in the list is controlled by the parameter packetlen.
946 * Each mbuf of the list may have a chain of mbufs linked by m_next. Each mbuf in
947 * the chain is called a segment.
948 * If maxsegments is not null and the value pointed to is not null, this specify
949 * the maximum number of segments for a chain of mbufs.
950 * If maxsegments is zero or the value pointed to is zero the
951 * caller does not have any restriction on the number of segments.
952 * The actual number of segments of a mbuf chain is return in the value pointed
953 * to by maxsegments.
954 * When possible the allocation is done under a single lock.
955 */
956
957 __private_extern__ struct mbuf *
958 m_allocpacket_internal(unsigned int *num_needed, size_t packetlen, unsigned int * maxsegments,
959 int how, int wantall, size_t wantsize)
960 {
961 struct mbuf **np, *top;
962 size_t bufsize;
963 unsigned int num;
964 unsigned int numchunks = 0;
965
966 top = NULL;
967 np = &top;
968
969 if (wantsize == 0) {
970 if (packetlen <= MINCLSIZE)
971 bufsize = packetlen;
972 else if (packetlen > MCLBYTES)
973 bufsize = NBPG;
974 else
975 bufsize = MCLBYTES;
976 } else if (wantsize == MCLBYTES || wantsize == NBPG)
977 bufsize = wantsize;
978 else
979 return 0;
980
981 if (bufsize <= MHLEN) {
982 numchunks = 1;
983 } else if (bufsize <= MINCLSIZE) {
984 if (maxsegments != NULL && *maxsegments == 1) {
985 bufsize = MCLBYTES;
986 numchunks = 1;
987 } else {
988 numchunks = 2;
989 }
990 } else if (bufsize == NBPG) {
991 numchunks = ((packetlen - 1) >> PGSHIFT) + 1;
992 } else {
993 numchunks = ((packetlen - 1) >> MCLSHIFT) + 1;
994 }
995 if (maxsegments != NULL) {
996 if (*maxsegments && numchunks > *maxsegments) {
997 *maxsegments = numchunks;
998 return 0;
999 }
1000 *maxsegments = numchunks;
1001 }
1002 /* m_clalloc takes the MBUF_LOCK, but do not release it */
1003 (void)m_clalloc(numchunks, how, (bufsize == NBPG) ? NBPG : MCLBYTES, 0);
1004 for (num = 0; num < *num_needed; num++) {
1005 struct mbuf **nm, *pkt = 0;
1006 size_t len;
1007
1008 nm = &pkt;
1009
1010 m_range_check(mfree);
1011 m_range_check(mclfree);
1012 m_range_check(mbigfree);
1013
1014 for (len = 0; len < packetlen; ) {
1015 struct mbuf *m = NULL;
1016
1017 if (wantsize == 0 && packetlen > MINCLSIZE) {
1018 if (packetlen - len > MCLBYTES)
1019 bufsize = NBPG;
1020 else
1021 bufsize = MCLBYTES;
1022 }
1023 len += bufsize;
1024
1025 if (mfree && ((bufsize == NBPG && mbigfree) || (bufsize == MCLBYTES && mclfree))) {
1026 /* mbuf + cluster are available */
1027 m = mfree;
1028 MCHECK(m);
1029 mfree = m->m_next;
1030 ++mclrefcnt[mtocl(m)];
1031 mbstat.m_mtypes[MT_FREE]--;
1032 mbstat.m_mtypes[MT_DATA]++;
1033 if (bufsize == NBPG) {
1034 m->m_ext.ext_buf = (caddr_t)mbigfree; /* get the big cluster */
1035 ++mclrefcnt[mtocl(m->m_ext.ext_buf)];
1036 ++mclrefcnt[mtocl(m->m_ext.ext_buf) + 1];
1037 mbstat.m_bigclfree--;
1038 mbigfree = ((union mbigcluster *)(m->m_ext.ext_buf))->mbc_next;
1039 m->m_ext.ext_free = m_bigfree;
1040 m->m_ext.ext_size = NBPG;
1041 } else {
1042 m->m_ext.ext_buf = (caddr_t)mclfree; /* get the cluster */
1043 ++mclrefcnt[mtocl(m->m_ext.ext_buf)];
1044 mbstat.m_clfree--;
1045 mclfree = ((union mcluster *)(m->m_ext.ext_buf))->mcl_next;
1046 m->m_ext.ext_free = 0;
1047 m->m_ext.ext_size = MCLBYTES;
1048 }
1049 m->m_ext.ext_arg = 0;
1050 m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward = &m->m_ext.ext_refs;
1051 m->m_next = m->m_nextpkt = 0;
1052 m->m_type = MT_DATA;
1053 m->m_data = m->m_ext.ext_buf;
1054 m->m_len = 0;
1055
1056 if (pkt == 0) {
1057 pkt = m;
1058 m->m_flags = M_PKTHDR | M_EXT;
1059 _M_CLEAR_PKTHDR(m);
1060 } else {
1061 m->m_flags = M_EXT;
1062 }
1063 } else {
1064 MBUF_UNLOCK();
1065
1066 if (pkt == 0) {
1067 MGETHDR(m, how, MT_DATA);
1068 } else {
1069 MGET(m, how, MT_DATA );
1070 }
1071 if (m == 0) {
1072 m_freem(pkt);
1073 goto fail;
1074 }
1075 if (bufsize <= MINCLSIZE) {
1076 if (bufsize > MHLEN) {
1077 MGET(m->m_next, how, MT_DATA);
1078 if (m->m_next == 0) {
1079 m_free(m);
1080 m_freem(pkt);
1081 goto fail;
1082 }
1083 }
1084 } else {
1085 if (bufsize == NBPG)
1086 m = m_mbigget(m, how);
1087 else
1088 m = m_mclget(m, how);
1089 if ((m->m_flags & M_EXT) == 0) {
1090 m_free(m);
1091 m_freem(pkt);
1092 goto fail;
1093 }
1094 }
1095 MBUF_LOCK();
1096 }
1097 *nm = m;
1098 nm = &m->m_next;
1099 }
1100 *np = pkt;
1101 np = &pkt->m_nextpkt;
1102 }
1103 MBUF_UNLOCK();
1104 *num_needed = num;
1105
1106 return top;
1107 fail:
1108 if (wantall && top) {
1109 m_freem(top);
1110 return 0;
1111 }
1112 *num_needed = num;
1113
1114 return top;
1115 }
1116
1117
1118 /* Best effort to get a mbuf cluster + pkthdr under one lock.
1119 * If we don't have them avail, just bail out and use the regular
1120 * path.
1121 * Used by drivers to allocated packets on receive ring.
1122 */
1123 __private_extern__ struct mbuf *
1124 m_getpacket_how(int how)
1125 {
1126 unsigned int num_needed = 1;
1127
1128 return m_getpackets_internal(&num_needed, 1, how, 1, MCLBYTES);
1129 }
1130
1131 /* Best effort to get a mbuf cluster + pkthdr under one lock.
1132 * If we don't have them avail, just bail out and use the regular
1133 * path.
1134 * Used by drivers to allocated packets on receive ring.
1135 */
1136 struct mbuf *
1137 m_getpacket(void)
1138 {
1139 unsigned int num_needed = 1;
1140
1141 return m_getpackets_internal(&num_needed, 1, M_WAITOK, 1, MCLBYTES);
1142 }
1143
1144
1145 /*
1146 * return a list of mbuf hdrs that point to clusters...
1147 * try for num_needed, if this can't be met, return whatever
1148 * number were available... set up the first num_with_pkthdrs
1149 * with mbuf hdrs configured as packet headers... these are
1150 * chained on the m_nextpkt field... any packets requested beyond
1151 * this are chained onto the last packet header's m_next field.
1152 */
1153 struct mbuf *
1154 m_getpackets(int num_needed, int num_with_pkthdrs, int how)
1155 {
1156 unsigned int n = num_needed;
1157
1158 return m_getpackets_internal(&n, num_with_pkthdrs, how, 0, MCLBYTES);
1159 }
1160
1161
1162 /*
1163 * return a list of mbuf hdrs set up as packet hdrs
1164 * chained together on the m_nextpkt field
1165 */
1166 struct mbuf *
1167 m_getpackethdrs(int num_needed, int how)
1168 {
1169 struct mbuf *m;
1170 struct mbuf **np, *top;
1171
1172 top = NULL;
1173 np = &top;
1174
1175 MBUF_LOCK();
1176
1177 while (num_needed--) {
1178 m_range_check(mfree);
1179 m_range_check(mclfree);
1180 m_range_check(mbigfree);
1181
1182 if ((m = mfree)) { /* mbufs are available */
1183 MCHECK(m);
1184 mfree = m->m_next;
1185 ++mclrefcnt[mtocl(m)];
1186 mbstat.m_mtypes[MT_FREE]--;
1187 mbstat.m_mtypes[MT_DATA]++;
1188
1189 m->m_next = m->m_nextpkt = 0;
1190 m->m_type = MT_DATA;
1191 m->m_flags = M_PKTHDR;
1192 m->m_len = 0;
1193 m->m_data = m->m_pktdat;
1194 _M_CLEAR_PKTHDR(m);
1195
1196 } else {
1197
1198 MBUF_UNLOCK();
1199 m = m_retryhdr(how, MT_DATA);
1200 if (m == 0)
1201 return(top);
1202 MBUF_LOCK();
1203 }
1204 *np = m;
1205 np = &m->m_nextpkt;
1206 }
1207 MBUF_UNLOCK();
1208
1209 return (top);
1210 }
1211
1212
1213 /* free and mbuf list (m_nextpkt) while following m_next under one lock.
1214 * returns the count for mbufs packets freed. Used by the drivers.
1215 */
1216 int
1217 m_freem_list(
1218 struct mbuf *m)
1219 {
1220 struct mbuf *nextpkt;
1221 int i, count=0;
1222
1223 MBUF_LOCK();
1224
1225 while (m) {
1226 if (m)
1227 nextpkt = m->m_nextpkt; /* chain of linked mbufs from driver */
1228 else
1229 nextpkt = 0;
1230
1231 count++;
1232
1233 while (m) { /* free the mbuf chain (like mfreem) */
1234
1235 struct mbuf *n;
1236
1237 m_range_check(m);
1238 m_range_check(mfree);
1239 m_range_check(mclfree);
1240 m_range_check(mbigfree);
1241
1242
1243 /* Free the aux data if there is any */
1244 if ((m->m_flags & M_PKTHDR) && m->m_pkthdr.aux) {
1245 /*
1246 * Treat the current m as the nextpkt and set m
1247 * to the aux data. Preserve nextpkt in m->m_nextpkt.
1248 * This lets us free the aux data in this loop
1249 * without having to call m_freem recursively,
1250 * which wouldn't work because we've still got
1251 * the lock.
1252 */
1253 m->m_nextpkt = nextpkt;
1254 nextpkt = m;
1255 m = nextpkt->m_pkthdr.aux;
1256 nextpkt->m_pkthdr.aux = NULL;
1257 }
1258
1259 if ((m->m_flags & M_PKTHDR) != 0 && !SLIST_EMPTY(&m->m_pkthdr.tags)) {
1260 /* A quick (albeit inefficient) expedient */
1261 MBUF_UNLOCK();
1262 m_tag_delete_chain(m, NULL);
1263 MBUF_LOCK();
1264 }
1265
1266 n = m->m_next;
1267
1268 if (n && n->m_nextpkt)
1269 panic("m_freem_list: m_nextpkt of m_next != NULL");
1270 if (m->m_type == MT_FREE)
1271 panic("freeing free mbuf");
1272
1273 if (m->m_flags & M_EXT) {
1274 if (MCLHASREFERENCE(m)) {
1275 remque((queue_t)&m->m_ext.ext_refs);
1276 } else if (m->m_ext.ext_free == NULL) {
1277 union mcluster *mcl= (union mcluster *)m->m_ext.ext_buf;
1278
1279 m_range_check(mcl);
1280
1281 if (_MCLUNREF(mcl)) {
1282 mcl->mcl_next = mclfree;
1283 mclfree = mcl;
1284 ++mbstat.m_clfree;
1285 }
1286 } else {
1287 (*(m->m_ext.ext_free))(m->m_ext.ext_buf,
1288 m->m_ext.ext_size, m->m_ext.ext_arg);
1289 }
1290 }
1291 mbstat.m_mtypes[m->m_type]--;
1292 (void) _MCLUNREF(m);
1293 _MFREE_MUNGE(m);
1294 mbstat.m_mtypes[MT_FREE]++;
1295 m->m_type = MT_FREE;
1296 m->m_flags = 0;
1297 m->m_len = 0;
1298 m->m_next = mfree;
1299 mfree = m;
1300 m = n;
1301 }
1302 m = nextpkt; /* bump m with saved nextpkt if any */
1303 }
1304 if ((i = m_want))
1305 m_want = 0;
1306
1307 MBUF_UNLOCK();
1308
1309 if (i)
1310 wakeup((caddr_t)&mfree);
1311
1312 return (count);
1313 }
1314
1315 void
1316 m_freem(
1317 struct mbuf *m)
1318 {
1319 while (m)
1320 m = m_free(m);
1321 }
1322
1323 /*
1324 * Mbuffer utility routines.
1325 */
1326 /*
1327 * Compute the amount of space available
1328 * before the current start of data in an mbuf.
1329 */
1330 int
1331 m_leadingspace(
1332 struct mbuf *m)
1333 {
1334 if (m->m_flags & M_EXT) {
1335 if (MCLHASREFERENCE(m))
1336 return(0);
1337 return (m->m_data - m->m_ext.ext_buf);
1338 }
1339 if (m->m_flags & M_PKTHDR)
1340 return (m->m_data - m->m_pktdat);
1341 return (m->m_data - m->m_dat);
1342 }
1343
1344 /*
1345 * Compute the amount of space available
1346 * after the end of data in an mbuf.
1347 */
1348 int
1349 m_trailingspace(
1350 struct mbuf *m)
1351 {
1352 if (m->m_flags & M_EXT) {
1353 if (MCLHASREFERENCE(m))
1354 return(0);
1355 return (m->m_ext.ext_buf + m->m_ext.ext_size -
1356 (m->m_data + m->m_len));
1357 }
1358 return (&m->m_dat[MLEN] - (m->m_data + m->m_len));
1359 }
1360
1361 /*
1362 * Lesser-used path for M_PREPEND:
1363 * allocate new mbuf to prepend to chain,
1364 * copy junk along.
1365 * Does not adjust packet header length.
1366 */
1367 struct mbuf *
1368 m_prepend(
1369 struct mbuf *m,
1370 int len,
1371 int how)
1372 {
1373 struct mbuf *mn;
1374
1375 MGET(mn, how, m->m_type);
1376 if (mn == (struct mbuf *)NULL) {
1377 m_freem(m);
1378 return ((struct mbuf *)NULL);
1379 }
1380 if (m->m_flags & M_PKTHDR) {
1381 M_COPY_PKTHDR(mn, m);
1382 m->m_flags &= ~M_PKTHDR;
1383 }
1384 mn->m_next = m;
1385 m = mn;
1386 if (len < MHLEN)
1387 MH_ALIGN(m, len);
1388 m->m_len = len;
1389 return (m);
1390 }
1391
1392 /*
1393 * Replacement for old M_PREPEND macro:
1394 * allocate new mbuf to prepend to chain,
1395 * copy junk along, and adjust length.
1396 *
1397 */
1398 struct mbuf *
1399 m_prepend_2(
1400 struct mbuf *m,
1401 int len,
1402 int how)
1403 {
1404 if (M_LEADINGSPACE(m) >= len) {
1405 m->m_data -= len;
1406 m->m_len += len;
1407 } else {
1408 m = m_prepend(m, len, how);
1409 }
1410 if ((m) && (m->m_flags & M_PKTHDR))
1411 m->m_pkthdr.len += len;
1412 return (m);
1413 }
1414
1415 /*
1416 * Make a copy of an mbuf chain starting "off0" bytes from the beginning,
1417 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf.
1418 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller.
1419 */
1420 int MCFail;
1421
1422 struct mbuf *
1423 m_copym(
1424 struct mbuf *m,
1425 int off0,
1426 int len,
1427 int wait)
1428 {
1429 struct mbuf *n, **np;
1430 int off = off0;
1431 struct mbuf *top;
1432 int copyhdr = 0;
1433
1434 if (off < 0 || len < 0)
1435 panic("m_copym");
1436 if (off == 0 && m->m_flags & M_PKTHDR)
1437 copyhdr = 1;
1438
1439 while (off >= m->m_len) {
1440 if (m == 0)
1441 panic("m_copym");
1442 off -= m->m_len;
1443 m = m->m_next;
1444 }
1445 np = &top;
1446 top = 0;
1447
1448 MBUF_LOCK();
1449
1450 while (len > 0) {
1451 m_range_check(mfree);
1452 m_range_check(mclfree);
1453 m_range_check(mbigfree);
1454
1455 if (m == 0) {
1456 if (len != M_COPYALL)
1457 panic("m_copym");
1458 break;
1459 }
1460 if ((n = mfree)) {
1461 MCHECK(n);
1462 ++mclrefcnt[mtocl(n)];
1463 mbstat.m_mtypes[MT_FREE]--;
1464 mbstat.m_mtypes[m->m_type]++;
1465 mfree = n->m_next;
1466 n->m_next = n->m_nextpkt = 0;
1467 n->m_type = m->m_type;
1468 n->m_data = n->m_dat;
1469 n->m_flags = 0;
1470 } else {
1471 MBUF_UNLOCK();
1472 n = m_retry(wait, m->m_type);
1473 MBUF_LOCK();
1474 }
1475 *np = n;
1476
1477 if (n == 0)
1478 goto nospace;
1479 if (copyhdr) {
1480 M_COPY_PKTHDR(n, m);
1481 if (len == M_COPYALL)
1482 n->m_pkthdr.len -= off0;
1483 else
1484 n->m_pkthdr.len = len;
1485 copyhdr = 0;
1486 }
1487 if (len == M_COPYALL) {
1488 if (min(len, (m->m_len - off)) == len) {
1489 printf("m->m_len %d - off %d = %d, %d\n",
1490 m->m_len, off, m->m_len - off,
1491 min(len, (m->m_len - off)));
1492 }
1493 }
1494 n->m_len = min(len, (m->m_len - off));
1495 if (n->m_len == M_COPYALL) {
1496 printf("n->m_len == M_COPYALL, fixing\n");
1497 n->m_len = MHLEN;
1498 }
1499 if (m->m_flags & M_EXT) {
1500 n->m_ext = m->m_ext;
1501 insque((queue_t)&n->m_ext.ext_refs, (queue_t)&m->m_ext.ext_refs);
1502 n->m_data = m->m_data + off;
1503 n->m_flags |= M_EXT;
1504 } else {
1505 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
1506 (unsigned)n->m_len);
1507 }
1508 if (len != M_COPYALL)
1509 len -= n->m_len;
1510 off = 0;
1511 m = m->m_next;
1512 np = &n->m_next;
1513 }
1514 MBUF_UNLOCK();
1515
1516 if (top == 0)
1517 MCFail++;
1518
1519 return (top);
1520 nospace:
1521 MBUF_UNLOCK();
1522
1523 m_freem(top);
1524 MCFail++;
1525 return (0);
1526 }
1527
1528
1529 /*
1530 * equivilent to m_copym except that all necessary
1531 * mbuf hdrs are allocated within this routine
1532 * also, the last mbuf and offset accessed are passed
1533 * out and can be passed back in to avoid having to
1534 * rescan the entire mbuf list (normally hung off of the socket)
1535 */
1536 struct mbuf *
1537 m_copym_with_hdrs(
1538 struct mbuf *m,
1539 int off0,
1540 int len,
1541 int wait,
1542 struct mbuf **m_last,
1543 int *m_off)
1544 {
1545 struct mbuf *n, **np = 0;
1546 int off = off0;
1547 struct mbuf *top = 0;
1548 int copyhdr = 0;
1549 int type;
1550
1551 if (off == 0 && m->m_flags & M_PKTHDR)
1552 copyhdr = 1;
1553
1554 if (*m_last) {
1555 m = *m_last;
1556 off = *m_off;
1557 } else {
1558 while (off >= m->m_len) {
1559 off -= m->m_len;
1560 m = m->m_next;
1561 }
1562 }
1563
1564 MBUF_LOCK();
1565
1566 while (len > 0) {
1567 m_range_check(mfree);
1568 m_range_check(mclfree);
1569 m_range_check(mbigfree);
1570
1571 if (top == 0)
1572 type = MT_HEADER;
1573 else {
1574 if (m == 0)
1575 panic("m_gethdr_and_copym");
1576 type = m->m_type;
1577 }
1578 if ((n = mfree)) {
1579 MCHECK(n);
1580 ++mclrefcnt[mtocl(n)];
1581 mbstat.m_mtypes[MT_FREE]--;
1582 mbstat.m_mtypes[type]++;
1583 mfree = n->m_next;
1584 n->m_next = n->m_nextpkt = 0;
1585 n->m_type = type;
1586
1587 if (top) {
1588 n->m_data = n->m_dat;
1589 n->m_flags = 0;
1590 } else {
1591 n->m_data = n->m_pktdat;
1592 n->m_flags = M_PKTHDR;
1593 _M_CLEAR_PKTHDR(n);
1594 }
1595 } else {
1596 MBUF_UNLOCK();
1597 if (top)
1598 n = m_retry(wait, type);
1599 else
1600 n = m_retryhdr(wait, type);
1601 MBUF_LOCK();
1602 }
1603 if (n == 0)
1604 goto nospace;
1605 if (top == 0) {
1606 top = n;
1607 np = &top->m_next;
1608 continue;
1609 } else
1610 *np = n;
1611
1612 if (copyhdr) {
1613 M_COPY_PKTHDR(n, m);
1614 n->m_pkthdr.len = len;
1615 copyhdr = 0;
1616 }
1617 n->m_len = min(len, (m->m_len - off));
1618
1619 if (m->m_flags & M_EXT) {
1620 n->m_ext = m->m_ext;
1621 insque((queue_t)&n->m_ext.ext_refs, (queue_t)&m->m_ext.ext_refs);
1622 n->m_data = m->m_data + off;
1623 n->m_flags |= M_EXT;
1624 } else {
1625 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
1626 (unsigned)n->m_len);
1627 }
1628 len -= n->m_len;
1629
1630 if (len == 0) {
1631 if ((off + n->m_len) == m->m_len) {
1632 *m_last = m->m_next;
1633 *m_off = 0;
1634 } else {
1635 *m_last = m;
1636 *m_off = off + n->m_len;
1637 }
1638 break;
1639 }
1640 off = 0;
1641 m = m->m_next;
1642 np = &n->m_next;
1643 }
1644 MBUF_UNLOCK();
1645
1646 return (top);
1647 nospace:
1648 MBUF_UNLOCK();
1649
1650 if (top)
1651 m_freem(top);
1652 MCFail++;
1653 return (0);
1654 }
1655
1656
1657 /*
1658 * Copy data from an mbuf chain starting "off" bytes from the beginning,
1659 * continuing for "len" bytes, into the indicated buffer.
1660 */
1661 void m_copydata(
1662 struct mbuf *m,
1663 int off,
1664 int len,
1665 caddr_t cp)
1666 {
1667 unsigned count;
1668
1669 if (off < 0 || len < 0)
1670 panic("m_copydata");
1671 while (off > 0) {
1672 if (m == 0)
1673 panic("m_copydata");
1674 if (off < m->m_len)
1675 break;
1676 off -= m->m_len;
1677 m = m->m_next;
1678 }
1679 while (len > 0) {
1680 if (m == 0)
1681 panic("m_copydata");
1682 count = min(m->m_len - off, len);
1683 bcopy(mtod(m, caddr_t) + off, cp, count);
1684 len -= count;
1685 cp += count;
1686 off = 0;
1687 m = m->m_next;
1688 }
1689 }
1690
1691 /*
1692 * Concatenate mbuf chain n to m.
1693 * Both chains must be of the same type (e.g. MT_DATA).
1694 * Any m_pkthdr is not updated.
1695 */
1696 void m_cat(
1697 struct mbuf *m, struct mbuf *n)
1698 {
1699 while (m->m_next)
1700 m = m->m_next;
1701 while (n) {
1702 if (m->m_flags & M_EXT ||
1703 m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) {
1704 /* just join the two chains */
1705 m->m_next = n;
1706 return;
1707 }
1708 /* splat the data from one into the other */
1709 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
1710 (u_int)n->m_len);
1711 m->m_len += n->m_len;
1712 n = m_free(n);
1713 }
1714 }
1715
1716 void
1717 m_adj(
1718 struct mbuf *mp,
1719 int req_len)
1720 {
1721 int len = req_len;
1722 struct mbuf *m;
1723 int count;
1724
1725 if ((m = mp) == NULL)
1726 return;
1727 if (len >= 0) {
1728 /*
1729 * Trim from head.
1730 */
1731 while (m != NULL && len > 0) {
1732 if (m->m_len <= len) {
1733 len -= m->m_len;
1734 m->m_len = 0;
1735 m = m->m_next;
1736 } else {
1737 m->m_len -= len;
1738 m->m_data += len;
1739 len = 0;
1740 }
1741 }
1742 m = mp;
1743 if (m->m_flags & M_PKTHDR)
1744 m->m_pkthdr.len -= (req_len - len);
1745 } else {
1746 /*
1747 * Trim from tail. Scan the mbuf chain,
1748 * calculating its length and finding the last mbuf.
1749 * If the adjustment only affects this mbuf, then just
1750 * adjust and return. Otherwise, rescan and truncate
1751 * after the remaining size.
1752 */
1753 len = -len;
1754 count = 0;
1755 for (;;) {
1756 count += m->m_len;
1757 if (m->m_next == (struct mbuf *)0)
1758 break;
1759 m = m->m_next;
1760 }
1761 if (m->m_len >= len) {
1762 m->m_len -= len;
1763 m = mp;
1764 if (m->m_flags & M_PKTHDR)
1765 m->m_pkthdr.len -= len;
1766 return;
1767 }
1768 count -= len;
1769 if (count < 0)
1770 count = 0;
1771 /*
1772 * Correct length for chain is "count".
1773 * Find the mbuf with last data, adjust its length,
1774 * and toss data from remaining mbufs on chain.
1775 */
1776 m = mp;
1777 if (m->m_flags & M_PKTHDR)
1778 m->m_pkthdr.len = count;
1779 for (; m; m = m->m_next) {
1780 if (m->m_len >= count) {
1781 m->m_len = count;
1782 break;
1783 }
1784 count -= m->m_len;
1785 }
1786 while ((m = m->m_next))
1787 m->m_len = 0;
1788 }
1789 }
1790
1791 /*
1792 * Rearange an mbuf chain so that len bytes are contiguous
1793 * and in the data area of an mbuf (so that mtod and dtom
1794 * will work for a structure of size len). Returns the resulting
1795 * mbuf chain on success, frees it and returns null on failure.
1796 * If there is room, it will add up to max_protohdr-len extra bytes to the
1797 * contiguous region in an attempt to avoid being called next time.
1798 */
1799 int MPFail;
1800
1801 struct mbuf *
1802 m_pullup(
1803 struct mbuf *n,
1804 int len)
1805 {
1806 struct mbuf *m;
1807 int count;
1808 int space;
1809
1810 /*
1811 * If first mbuf has no cluster, and has room for len bytes
1812 * without shifting current data, pullup into it,
1813 * otherwise allocate a new mbuf to prepend to the chain.
1814 */
1815 if ((n->m_flags & M_EXT) == 0 &&
1816 n->m_data + len < &n->m_dat[MLEN] && n->m_next) {
1817 if (n->m_len >= len)
1818 return (n);
1819 m = n;
1820 n = n->m_next;
1821 len -= m->m_len;
1822 } else {
1823 if (len > MHLEN)
1824 goto bad;
1825 MGET(m, M_DONTWAIT, n->m_type);
1826 if (m == 0)
1827 goto bad;
1828 m->m_len = 0;
1829 if (n->m_flags & M_PKTHDR) {
1830 M_COPY_PKTHDR(m, n);
1831 n->m_flags &= ~M_PKTHDR;
1832 }
1833 }
1834 space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
1835 do {
1836 count = min(min(max(len, max_protohdr), space), n->m_len);
1837 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
1838 (unsigned)count);
1839 len -= count;
1840 m->m_len += count;
1841 n->m_len -= count;
1842 space -= count;
1843 if (n->m_len)
1844 n->m_data += count;
1845 else
1846 n = m_free(n);
1847 } while (len > 0 && n);
1848 if (len > 0) {
1849 (void) m_free(m);
1850 goto bad;
1851 }
1852 m->m_next = n;
1853 return (m);
1854 bad:
1855 m_freem(n);
1856 MPFail++;
1857 return (0);
1858 }
1859
1860 /*
1861 * Partition an mbuf chain in two pieces, returning the tail --
1862 * all but the first len0 bytes. In case of failure, it returns NULL and
1863 * attempts to restore the chain to its original state.
1864 */
1865 struct mbuf *
1866 m_split(
1867 struct mbuf *m0,
1868 int len0,
1869 int wait)
1870 {
1871 struct mbuf *m, *n;
1872 unsigned len = len0, remain;
1873
1874 for (m = m0; m && len > m->m_len; m = m->m_next)
1875 len -= m->m_len;
1876 if (m == 0)
1877 return (0);
1878 remain = m->m_len - len;
1879 if (m0->m_flags & M_PKTHDR) {
1880 MGETHDR(n, wait, m0->m_type);
1881 if (n == 0)
1882 return (0);
1883 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
1884 n->m_pkthdr.len = m0->m_pkthdr.len - len0;
1885 m0->m_pkthdr.len = len0;
1886 if (m->m_flags & M_EXT)
1887 goto extpacket;
1888 if (remain > MHLEN) {
1889 /* m can't be the lead packet */
1890 MH_ALIGN(n, 0);
1891 n->m_next = m_split(m, len, wait);
1892 if (n->m_next == 0) {
1893 (void) m_free(n);
1894 return (0);
1895 } else
1896 return (n);
1897 } else
1898 MH_ALIGN(n, remain);
1899 } else if (remain == 0) {
1900 n = m->m_next;
1901 m->m_next = 0;
1902 return (n);
1903 } else {
1904 MGET(n, wait, m->m_type);
1905 if (n == 0)
1906 return (0);
1907 M_ALIGN(n, remain);
1908 }
1909 extpacket:
1910 if (m->m_flags & M_EXT) {
1911 n->m_flags |= M_EXT;
1912 MBUF_LOCK();
1913 n->m_ext = m->m_ext;
1914 insque((queue_t)&n->m_ext.ext_refs, (queue_t)&m->m_ext.ext_refs);
1915 MBUF_UNLOCK();
1916 n->m_data = m->m_data + len;
1917 } else {
1918 bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain);
1919 }
1920 n->m_len = remain;
1921 m->m_len = len;
1922 n->m_next = m->m_next;
1923 m->m_next = 0;
1924 return (n);
1925 }
1926 /*
1927 * Routine to copy from device local memory into mbufs.
1928 */
1929 struct mbuf *
1930 m_devget(
1931 char *buf,
1932 int totlen,
1933 int off0,
1934 struct ifnet *ifp,
1935 void (*copy)(const void *, void *, size_t))
1936 {
1937 struct mbuf *m;
1938 struct mbuf *top = 0, **mp = &top;
1939 int off = off0, len;
1940 char *cp;
1941 char *epkt;
1942
1943 cp = buf;
1944 epkt = cp + totlen;
1945 if (off) {
1946 /*
1947 * If 'off' is non-zero, packet is trailer-encapsulated,
1948 * so we have to skip the type and length fields.
1949 */
1950 cp += off + 2 * sizeof(u_int16_t);
1951 totlen -= 2 * sizeof(u_int16_t);
1952 }
1953 MGETHDR(m, M_DONTWAIT, MT_DATA);
1954 if (m == 0)
1955 return (0);
1956 m->m_pkthdr.rcvif = ifp;
1957 m->m_pkthdr.len = totlen;
1958 m->m_len = MHLEN;
1959
1960 while (totlen > 0) {
1961 if (top) {
1962 MGET(m, M_DONTWAIT, MT_DATA);
1963 if (m == 0) {
1964 m_freem(top);
1965 return (0);
1966 }
1967 m->m_len = MLEN;
1968 }
1969 len = min(totlen, epkt - cp);
1970 if (len >= MINCLSIZE) {
1971 MCLGET(m, M_DONTWAIT);
1972 if (m->m_flags & M_EXT)
1973 m->m_len = len = min(len, MCLBYTES);
1974 else {
1975 /* give up when it's out of cluster mbufs */
1976 if (top)
1977 m_freem(top);
1978 m_freem(m);
1979 return (0);
1980 }
1981 } else {
1982 /*
1983 * Place initial small packet/header at end of mbuf.
1984 */
1985 if (len < m->m_len) {
1986 if (top == 0 && len + max_linkhdr <= m->m_len)
1987 m->m_data += max_linkhdr;
1988 m->m_len = len;
1989 } else
1990 len = m->m_len;
1991 }
1992 if (copy)
1993 copy(cp, mtod(m, caddr_t), (unsigned)len);
1994 else
1995 bcopy(cp, mtod(m, caddr_t), (unsigned)len);
1996 cp += len;
1997 *mp = m;
1998 mp = &m->m_next;
1999 totlen -= len;
2000 if (cp == epkt)
2001 cp = buf;
2002 }
2003 return (top);
2004 }
2005
2006 /*
2007 * Cluster freelist allocation check. The mbuf lock must be held.
2008 * Ensure hysteresis between hi/lo.
2009 */
2010 static int
2011 m_howmany(int num, size_t bufsize)
2012 {
2013 int i = 0;
2014
2015 /* Bail if we've maxed out the mbuf memory map */
2016 if (mbstat.m_clusters + (mbstat.m_bigclusters << 1) < nmbclusters) {
2017 int j = 0;
2018
2019 if (bufsize == MCLBYTES) {
2020 /* Under minimum */
2021 if (mbstat.m_clusters < MINCL)
2022 return (MINCL - mbstat.m_clusters);
2023 /* Too few (free < 1/2 total) and not over maximum */
2024 if (mbstat.m_clusters < (nmbclusters >> 1)) {
2025 if (num >= mbstat.m_clfree)
2026 i = num - mbstat.m_clfree;
2027 if (((mbstat.m_clusters + num) >> 1) > mbstat.m_clfree)
2028 j = ((mbstat.m_clusters + num) >> 1) - mbstat.m_clfree;
2029 i = max(i, j);
2030 if (i + mbstat.m_clusters >= (nmbclusters >> 1))
2031 i = (nmbclusters >> 1) - mbstat.m_clusters;
2032 }
2033 } else {
2034 /* Under minimum */
2035 if (mbstat.m_bigclusters < MINCL)
2036 return (MINCL - mbstat.m_bigclusters);
2037 /* Too few (free < 1/2 total) and not over maximum */
2038 if (mbstat.m_bigclusters < (nmbclusters >> 2)) {
2039 if (num >= mbstat.m_bigclfree)
2040 i = num - mbstat.m_bigclfree;
2041 if (((mbstat.m_bigclusters + num) >> 1) > mbstat.m_bigclfree)
2042 j = ((mbstat.m_bigclusters + num) >> 1) - mbstat.m_bigclfree;
2043 i = max(i, j);
2044 if (i + mbstat.m_bigclusters >= (nmbclusters >> 2))
2045 i = (nmbclusters >> 2) - mbstat.m_bigclusters;
2046 }
2047 }
2048 }
2049 return i;
2050 }
2051
2052 /*
2053 * Copy data from a buffer back into the indicated mbuf chain,
2054 * starting "off" bytes from the beginning, extending the mbuf
2055 * chain if necessary.
2056 */
2057 void
2058 m_copyback(
2059 struct mbuf *m0,
2060 int off,
2061 int len,
2062 caddr_t cp)
2063 {
2064 int mlen;
2065 struct mbuf *m = m0, *n;
2066 int totlen = 0;
2067
2068 if (m0 == 0)
2069 return;
2070 while (off > (mlen = m->m_len)) {
2071 off -= mlen;
2072 totlen += mlen;
2073 if (m->m_next == 0) {
2074 n = m_getclr(M_DONTWAIT, m->m_type);
2075 if (n == 0)
2076 goto out;
2077 n->m_len = min(MLEN, len + off);
2078 m->m_next = n;
2079 }
2080 m = m->m_next;
2081 }
2082 while (len > 0) {
2083 mlen = min (m->m_len - off, len);
2084 bcopy(cp, off + mtod(m, caddr_t), (unsigned)mlen);
2085 cp += mlen;
2086 len -= mlen;
2087 mlen += off;
2088 off = 0;
2089 totlen += mlen;
2090 if (len == 0)
2091 break;
2092 if (m->m_next == 0) {
2093 n = m_get(M_DONTWAIT, m->m_type);
2094 if (n == 0)
2095 break;
2096 n->m_len = min(MLEN, len);
2097 m->m_next = n;
2098 }
2099 m = m->m_next;
2100 }
2101 out: if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen))
2102 m->m_pkthdr.len = totlen;
2103 }
2104
2105
2106 char *mcl_to_paddr(char *addr) {
2107 int base_phys;
2108
2109 if (addr < (char *)mbutl || addr >= (char *)embutl)
2110 return (0);
2111 base_phys = mcl_paddr[(addr - (char *)mbutl) >> PGSHIFT];
2112
2113 if (base_phys == 0)
2114 return (0);
2115 return ((char *)((int)base_phys | ((int)addr & PGOFSET)));
2116 }
2117
2118 /*
2119 * Dup the mbuf chain passed in. The whole thing. No cute additional cruft.
2120 * And really copy the thing. That way, we don't "precompute" checksums
2121 * for unsuspecting consumers.
2122 * Assumption: m->m_nextpkt == 0.
2123 * Trick: for small packets, don't dup into a cluster. That way received
2124 * packets don't take up too much room in the sockbuf (cf. sbspace()).
2125 */
2126 int MDFail;
2127
2128 struct mbuf *
2129 m_dup(struct mbuf *m, int how)
2130 {
2131 struct mbuf *n, **np;
2132 struct mbuf *top;
2133 int copyhdr = 0;
2134
2135 np = &top;
2136 top = 0;
2137 if (m->m_flags & M_PKTHDR)
2138 copyhdr = 1;
2139
2140 /*
2141 * Quick check: if we have one mbuf and its data fits in an
2142 * mbuf with packet header, just copy and go.
2143 */
2144 if (m->m_next == NULL)
2145 { /* Then just move the data into an mbuf and be done... */
2146 if (copyhdr)
2147 { if (m->m_pkthdr.len <= MHLEN)
2148 { if ((n = m_gethdr(how, m->m_type)) == NULL)
2149 return(NULL);
2150 n->m_len = m->m_len;
2151 m_dup_pkthdr(n, m, how);
2152 bcopy(m->m_data, n->m_data, m->m_len);
2153 return(n);
2154 }
2155 } else if (m->m_len <= MLEN)
2156 { if ((n = m_get(how, m->m_type)) == NULL)
2157 return(NULL);
2158 bcopy(m->m_data, n->m_data, m->m_len);
2159 n->m_len = m->m_len;
2160 return(n);
2161 }
2162 }
2163 while (m)
2164 {
2165 #if BLUE_DEBUG
2166 kprintf("<%x: %x, %x, %x\n", m, m->m_flags, m->m_len,
2167 m->m_data);
2168 #endif
2169 if (copyhdr)
2170 n = m_gethdr(how, m->m_type);
2171 else
2172 n = m_get(how, m->m_type);
2173 if (n == 0)
2174 goto nospace;
2175 if (m->m_flags & M_EXT)
2176 { MCLGET(n, how);
2177 if ((n->m_flags & M_EXT) == 0)
2178 goto nospace;
2179 }
2180 *np = n;
2181 if (copyhdr)
2182 { /* Don't use M_COPY_PKTHDR: preserve m_data */
2183 m_dup_pkthdr(n, m, how);
2184 copyhdr = 0;
2185 if ((n->m_flags & M_EXT) == 0)
2186 n->m_data = n->m_pktdat;
2187 }
2188 n->m_len = m->m_len;
2189 /*
2190 * Get the dup on the same bdry as the original
2191 * Assume that the two mbufs have the same offset to data area
2192 * (up to word bdries)
2193 */
2194 bcopy(mtod(m, caddr_t), mtod(n, caddr_t), (unsigned)n->m_len);
2195 m = m->m_next;
2196 np = &n->m_next;
2197 #if BLUE_DEBUG
2198 kprintf(">%x: %x, %x, %x\n", n, n->m_flags, n->m_len,
2199 n->m_data);
2200 #endif
2201 }
2202
2203 if (top == 0)
2204 MDFail++;
2205 return (top);
2206 nospace:
2207 m_freem(top);
2208 MDFail++;
2209 return (0);
2210 }
2211
2212 int
2213 m_mclref(struct mbuf *p)
2214 {
2215 return (_MCLREF(p));
2216 }
2217
2218 int
2219 m_mclunref(struct mbuf *p)
2220 {
2221 return (_MCLUNREF(p));
2222 }
2223
2224 /* change mbuf to new type */
2225 void
2226 m_mchtype(struct mbuf *m, int t)
2227 {
2228 MBUF_LOCK();
2229 mbstat.m_mtypes[(m)->m_type]--;
2230 mbstat.m_mtypes[t]++;
2231 (m)->m_type = t;
2232 MBUF_UNLOCK();
2233 }
2234
2235 void *m_mtod(struct mbuf *m)
2236 {
2237 return ((m)->m_data);
2238 }
2239
2240 struct mbuf *m_dtom(void *x)
2241 {
2242 return ((struct mbuf *)((u_long)(x) & ~(MSIZE-1)));
2243 }
2244
2245 int m_mtocl(void *x)
2246 {
2247 return (((char *)(x) - (char *)mbutl) / sizeof(union mcluster));
2248 }
2249
2250 union mcluster *m_cltom(int x)
2251 {
2252 return ((union mcluster *)(mbutl + (x)));
2253 }
2254
2255
2256 void m_mcheck(struct mbuf *m)
2257 {
2258 if (m->m_type != MT_FREE)
2259 panic("mget MCHECK: m_type=%x m=%x", m->m_type, m);
2260 }
2261
2262 static void
2263 mbuf_expand_thread(void)
2264 {
2265 while (1) {
2266 MBUF_LOCK();
2267 if (mbuf_expand_mcl) {
2268 int n;
2269
2270 /* Adjust to the current number of cluster in use */
2271 n = mbuf_expand_mcl - (mbstat.m_clusters - mbstat.m_clfree);
2272 mbuf_expand_mcl = 0;
2273
2274 if (n > 0)
2275 (void)m_clalloc(n, M_WAIT, MCLBYTES, 1);
2276 }
2277 if (mbuf_expand_big) {
2278 int n;
2279
2280 /* Adjust to the current number of 4 KB cluster in use */
2281 n = mbuf_expand_big - (mbstat.m_bigclusters - mbstat.m_bigclfree);
2282 mbuf_expand_big = 0;
2283
2284 if (n > 0)
2285 (void)m_clalloc(n, M_WAIT, NBPG, 1);
2286 }
2287 MBUF_UNLOCK();
2288 /*
2289 * Because we can run out of memory before filling the mbuf map, we
2290 * should not allocate more clusters than they are mbufs -- otherwise
2291 * we could have a large number of useless clusters allocated.
2292 */
2293 while (mbstat.m_mbufs < mbstat.m_bigclusters + mbstat.m_clusters) {
2294 if (m_expand(M_WAIT) == 0)
2295 break;
2296 }
2297
2298 assert_wait(&mbuf_expand_thread_wakeup, THREAD_UNINT);
2299 (void) thread_block((thread_continue_t)mbuf_expand_thread);
2300 }
2301 }
2302
2303 static void
2304 mbuf_expand_thread_init(void)
2305 {
2306 mbuf_expand_thread_initialized++;
2307 mbuf_expand_thread();
2308 }
2309
2310 SYSCTL_DECL(_kern_ipc);
2311 SYSCTL_STRUCT(_kern_ipc, KIPC_MBSTAT, mbstat, CTLFLAG_RW, &mbstat, mbstat, "");
2312
mirror of XNU