]> git.saurik.com Git - apple/xnu.git/blob - bsd/kern/uipc_mbuf.c
projects / apple / xnu.git / blob
? search:


c4fac31f33dba8a19b536a71fdcd793e27fa9d49
[apple/xnu.git] / bsd / kern / uipc_mbuf.c
1 /*
2 * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * The contents of this file constitute Original Code as defined in and
7 * are subject to the Apple Public Source License Version 1.1 (the
8 * "License"). You may not use this file except in compliance with the
9 * License. Please obtain a copy of the License at
10 * http://www.apple.com/publicsource and read it before using this file.
11 *
12 * This Original Code and all software distributed under the License are
13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17 * License for the specific language governing rights and limitations
18 * under the License.
19 *
20 * @APPLE_LICENSE_HEADER_END@
21 */
22 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
23 /*
24 * Copyright (c) 1982, 1986, 1988, 1991, 1993
25 * The Regents of the University of California. All rights reserved.
26 *
27 * Redistribution and use in source and binary forms, with or without
28 * modification, are permitted provided that the following conditions
29 * are met:
30 * 1. Redistributions of source code must retain the above copyright
31 * notice, this list of conditions and the following disclaimer.
32 * 2. Redistributions in binary form must reproduce the above copyright
33 * notice, this list of conditions and the following disclaimer in the
34 * documentation and/or other materials provided with the distribution.
35 * 3. All advertising materials mentioning features or use of this software
36 * must display the following acknowledgement:
37 * This product includes software developed by the University of
38 * California, Berkeley and its contributors.
39 * 4. Neither the name of the University nor the names of its contributors
40 * may be used to endorse or promote products derived from this software
41 * without specific prior written permission.
42 *
43 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
44 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
45 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
46 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
47 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
48 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
49 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
50 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
51 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
52 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
53 * SUCH DAMAGE.
54 *
55 * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94
56 */
57 /* HISTORY
58 *
59 * 10/15/97 Annette DeSchon (deschon@apple.com)
60 * Fixed bug in which all cluster mbufs were broken up
61 * into regular mbufs: Some clusters are now reserved.
62 * When a cluster is needed, regular mbufs are no longer
63 * used. (Radar 1683621)
64 * 20-May-95 Mac Gillon (mgillon) at NeXT
65 * New version based on 4.4
66 */
67
68 #include <sys/param.h>
69 #include <sys/systm.h>
70 #include <sys/malloc.h>
71 #include <sys/mbuf.h>
72 #include <sys/kernel.h>
73 #include <sys/sysctl.h>
74 #include <sys/syslog.h>
75 #include <sys/protosw.h>
76 #include <sys/domain.h>
77
78 #include <kern/queue.h>
79 #include <kern/kern_types.h>
80 #include <kern/sched_prim.h>
81
82 #include <IOKit/IOMapper.h>
83
84 extern vm_offset_t kmem_mb_alloc(vm_map_t , int );
85 extern boolean_t PE_parse_boot_arg(const char *, void *);
86
87 #define _MCLREF(p) (++mclrefcnt[mtocl(p)])
88 #define _MCLUNREF(p) (--mclrefcnt[mtocl(p)] == 0)
89 #define _M_CLEAR_PKTHDR(mbuf_ptr) (mbuf_ptr)->m_pkthdr.rcvif = NULL; \
90 (mbuf_ptr)->m_pkthdr.len = 0; \
91 (mbuf_ptr)->m_pkthdr.header = NULL; \
92 (mbuf_ptr)->m_pkthdr.csum_flags = 0; \
93 (mbuf_ptr)->m_pkthdr.csum_data = 0; \
94 (mbuf_ptr)->m_pkthdr.aux = (struct mbuf*)NULL; \
95 (mbuf_ptr)->m_pkthdr.vlan_tag = 0; \
96 (mbuf_ptr)->m_pkthdr.socket_id = 0; \
97 SLIST_INIT(&(mbuf_ptr)->m_pkthdr.tags);
98
99 /* kernel translater */
100 extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
101
102 lck_mtx_t * mbuf_mlock;
103 lck_grp_t * mbuf_mlock_grp;
104 lck_grp_attr_t * mbuf_mlock_grp_attr;
105 lck_attr_t * mbuf_mlock_attr;
106 extern lck_mtx_t *domain_proto_mtx;
107
108 struct mbuf *mfree; /* mbuf free list */
109 struct mbuf *mfreelater; /* mbuf deallocation list */
110 extern vm_map_t mb_map; /* special map */
111 int m_want; /* sleepers on mbufs */
112 short *mclrefcnt; /* mapped cluster reference counts */
113 int *mcl_paddr;
114 static ppnum_t mcl_paddr_base; /* Handle returned by IOMapper::iovmAlloc() */
115 union mcluster *mclfree; /* mapped cluster free list */
116 union mbigcluster *mbigfree; /* mapped cluster free list */
117 int max_linkhdr; /* largest link-level header */
118 int max_protohdr; /* largest protocol header */
119 int max_hdr; /* largest link+protocol header */
120 int max_datalen; /* MHLEN - max_hdr */
121 struct mbstat mbstat; /* statistics */
122 union mcluster *mbutl; /* first mapped cluster address */
123 union mcluster *embutl; /* ending virtual address of mclusters */
124
125 static int nclpp; /* # clusters per physical page */
126
127 static int m_howmany(int, size_t );
128 void m_reclaim(void);
129 static int m_clalloc(const int , const int, const size_t, int);
130 int do_reclaim = 0;
131
132 #define MF_NOWAIT 0x1
133 #define MF_BIG 0x2
134
135 /* The number of cluster mbufs that are allocated, to start. */
136 #define MINCL max(16, 2)
137
138 static int mbuf_expand_thread_wakeup = 0;
139 static int mbuf_expand_mcl = 0;
140 static int mbuf_expand_big = 0;
141 static int mbuf_expand_thread_initialized = 0;
142
143 static void mbuf_expand_thread_init(void);
144 static void mbuf_expand_thread(void);
145 static int m_expand(int );
146 static caddr_t m_bigalloc(int );
147 static void m_bigfree(caddr_t , u_int , caddr_t );
148 __private_extern__ struct mbuf * m_mbigget(struct mbuf *, int );
149 void mbinit(void);
150 static void m_range_check(void *addr);
151
152
153 #if 0
154 static int mfree_munge = 0;
155 #if 0
156 #define _MFREE_MUNGE(m) { \
157 if (mfree_munge) \
158 { int i; \
159 vm_offset_t *element = (vm_offset_t *)(m); \
160 for (i = 0; \
161 i < sizeof(struct mbuf)/sizeof(vm_offset_t); \
162 i++) \
163 (element)[i] = 0xdeadbeef; \
164 } \
165 }
166 #else
167 void
168 munge_mbuf(struct mbuf *m)
169 {
170 int i;
171 vm_offset_t *element = (vm_offset_t *)(m);
172 for (i = 0;
173 i < sizeof(struct mbuf)/sizeof(vm_offset_t);
174 i++)
175 (element)[i] = 0xdeadbeef;
176 }
177 #define _MFREE_MUNGE(m) { \
178 if (mfree_munge) \
179 munge_mbuf(m); \
180 }
181 #endif
182 #else
183 #define _MFREE_MUNGE(m)
184 #endif
185
186
187 #define _MINTGET(m, type) { \
188 MBUF_LOCK(); \
189 if (((m) = mfree) != 0) { \
190 MCHECK(m); \
191 ++mclrefcnt[mtocl(m)]; \
192 mbstat.m_mtypes[MT_FREE]--; \
193 mbstat.m_mtypes[(type)]++; \
194 mfree = (m)->m_next; \
195 } \
196 MBUF_UNLOCK(); \
197 }
198
199
200 static void
201 m_range_check(void *addr)
202 {
203 if (addr && (addr < (void *)mbutl || addr >= (void *)embutl))
204 panic("mbuf address out of range 0x%x", addr);
205 }
206
207 __private_extern__ void
208 mbinit(void)
209 {
210 int m;
211 int initmcl = 32;
212 int mcl_pages;
213
214 if (nclpp)
215 return;
216 nclpp = round_page_32(MCLBYTES) / MCLBYTES; /* see mbufgc() */
217 if (nclpp < 1) nclpp = 1;
218 mbuf_mlock_grp_attr = lck_grp_attr_alloc_init();
219
220 mbuf_mlock_grp = lck_grp_alloc_init("mbuf", mbuf_mlock_grp_attr);
221 mbuf_mlock_attr = lck_attr_alloc_init();
222
223 mbuf_mlock = lck_mtx_alloc_init(mbuf_mlock_grp, mbuf_mlock_attr);
224
225 mbstat.m_msize = MSIZE;
226 mbstat.m_mclbytes = MCLBYTES;
227 mbstat.m_minclsize = MINCLSIZE;
228 mbstat.m_mlen = MLEN;
229 mbstat.m_mhlen = MHLEN;
230 mbstat.m_bigmclbytes = NBPG;
231
232 if (nmbclusters == 0)
233 nmbclusters = NMBCLUSTERS;
234 MALLOC(mclrefcnt, short *, nmbclusters * sizeof (short),
235 M_TEMP, M_WAITOK);
236 if (mclrefcnt == 0)
237 panic("mbinit");
238 for (m = 0; m < nmbclusters; m++)
239 mclrefcnt[m] = -1;
240
241 /* Calculate the number of pages assigned to the cluster pool */
242 mcl_pages = nmbclusters/(NBPG/CLBYTES);
243 MALLOC(mcl_paddr, int *, mcl_pages * sizeof(int), M_TEMP, M_WAITOK);
244 if (mcl_paddr == 0)
245 panic("mbinit1");
246 /* Register with the I/O Bus mapper */
247 mcl_paddr_base = IOMapperIOVMAlloc(mcl_pages);
248 bzero((char *)mcl_paddr, mcl_pages * sizeof(int));
249
250 embutl = (union mcluster *)((unsigned char *)mbutl + (nmbclusters * MCLBYTES));
251
252 PE_parse_boot_arg("initmcl", &initmcl);
253
254 if (m_clalloc(max(NBPG/CLBYTES, 1) * initmcl, M_WAIT, MCLBYTES, 0) == 0)
255 goto bad;
256 MBUF_UNLOCK();
257
258 (void) kernel_thread(kernel_task, mbuf_expand_thread_init);
259
260 return;
261 bad:
262 panic("mbinit");
263 }
264
265 /*
266 * Allocate some number of mbuf clusters
267 * and place on cluster free list.
268 * Take the mbuf lock (if not already locked) and do not release it
269 */
270 /* ARGSUSED */
271 static int
272 m_clalloc(
273 const int num,
274 const int nowait,
275 const size_t bufsize,
276 int locked)
277 {
278 int i;
279 vm_size_t size = 0;
280 int numpages = 0;
281 vm_offset_t page = 0;
282
283 if (locked == 0)
284 MBUF_LOCK();
285 /*
286 * Honor the caller's wish to block or not block.
287 * We have a way to grow the pool asynchronously,
288 * by kicking the dlil_input_thread.
289 */
290 i = m_howmany(num, bufsize);
291 if (i == 0 || nowait == M_DONTWAIT)
292 goto out;
293
294 MBUF_UNLOCK();
295 size = round_page_32(i * bufsize);
296 page = kmem_mb_alloc(mb_map, size);
297
298 if (page == 0) {
299 size = NBPG; /* Try for 1 if failed */
300 page = kmem_mb_alloc(mb_map, size);
301 }
302 MBUF_LOCK();
303
304 if (page) {
305 numpages = size / NBPG;
306 for (i = 0; i < numpages; i++, page += NBPG) {
307 if (((int)page & PGOFSET) == 0) {
308 ppnum_t offset = ((char *)page - (char *)mbutl)/NBPG;
309 ppnum_t new_page = pmap_find_phys(kernel_pmap, (vm_address_t) page);
310
311 /*
312 * In the case of no mapper being available
313 * the following code nops and returns the
314 * input page, if there is a mapper the I/O
315 * page appropriate is returned.
316 */
317 new_page = IOMapperInsertPage(mcl_paddr_base, offset, new_page);
318 mcl_paddr[offset] = new_page << 12;
319 }
320 if (bufsize == MCLBYTES) {
321 union mcluster *mcl = (union mcluster *)page;
322
323 if (++mclrefcnt[mtocl(mcl)] != 0)
324 panic("m_clalloc already there");
325 mcl->mcl_next = mclfree;
326 mclfree = mcl++;
327 if (++mclrefcnt[mtocl(mcl)] != 0)
328 panic("m_clalloc already there");
329 mcl->mcl_next = mclfree;
330 mclfree = mcl++;
331 } else {
332 union mbigcluster *mbc = (union mbigcluster *)page;
333
334 if (++mclrefcnt[mtocl(mbc)] != 0)
335 panic("m_clalloc already there");
336 if (++mclrefcnt[mtocl(mbc) + 1] != 0)
337 panic("m_clalloc already there");
338
339 mbc->mbc_next = mbigfree;
340 mbigfree = mbc;
341 }
342 }
343 if (bufsize == MCLBYTES) {
344 int numcl = numpages << 1;
345 mbstat.m_clfree += numcl;
346 mbstat.m_clusters += numcl;
347 return (numcl);
348 } else {
349 mbstat.m_bigclfree += numpages;
350 mbstat.m_bigclusters += numpages;
351 return (numpages);
352 }
353 } /* else ... */
354 out:
355 /*
356 * When non-blocking we kick a thread if we havve to grow the
357 * pool or if the number of free clusters is less than requested.
358 */
359 if (bufsize == MCLBYTES) {
360 if (i > 0) {
361 /* Remember total number of clusters needed at this time */
362 i += mbstat.m_clusters;
363 if (i > mbuf_expand_mcl) {
364 mbuf_expand_mcl = i;
365 if (mbuf_expand_thread_initialized)
366 wakeup((caddr_t)&mbuf_expand_thread_wakeup);
367 }
368 }
369
370 if (mbstat.m_clfree >= num)
371 return 1;
372 } else {
373 if (i > 0) {
374 /* Remember total number of 4KB clusters needed at this time */
375 i += mbstat.m_bigclusters;
376 if (i > mbuf_expand_big) {
377 mbuf_expand_big = i;
378 if (mbuf_expand_thread_initialized)
379 wakeup((caddr_t)&mbuf_expand_thread_wakeup);
380 }
381 }
382
383 if (mbstat.m_bigclfree >= num)
384 return 1;
385 }
386 return 0;
387 }
388
389 /*
390 * Add more free mbufs by cutting up a cluster.
391 */
392 static int
393 m_expand(int canwait)
394 {
395 caddr_t mcl;
396
397 if (mbstat.m_clfree < (mbstat.m_clusters >> 4)) {
398 /*
399 * 1/16th of the total number of cluster mbufs allocated is
400 * reserved for large packets. The number reserved must
401 * always be < 1/2, or future allocation will be prevented.
402 */
403 (void)m_clalloc(1, canwait, MCLBYTES, 0);
404 MBUF_UNLOCK();
405 if (mbstat.m_clfree < (mbstat.m_clusters >> 4))
406 return 0;
407 }
408
409 MCLALLOC(mcl, canwait);
410 if (mcl) {
411 struct mbuf *m = (struct mbuf *)mcl;
412 int i = NMBPCL;
413 MBUF_LOCK();
414 mbstat.m_mtypes[MT_FREE] += i;
415 mbstat.m_mbufs += i;
416 while (i--) {
417 _MFREE_MUNGE(m);
418 m->m_type = MT_FREE;
419 m->m_next = mfree;
420 mfree = m++;
421 }
422 i = m_want;
423 m_want = 0;
424 MBUF_UNLOCK();
425 if (i) wakeup((caddr_t)&mfree);
426 return 1;
427 }
428 return 0;
429 }
430
431 /*
432 * When MGET failes, ask protocols to free space when short of memory,
433 * then re-attempt to allocate an mbuf.
434 */
435 struct mbuf *
436 m_retry(
437 int canwait,
438 int type)
439 {
440 struct mbuf *m;
441 int wait;
442
443 for (;;) {
444 (void) m_expand(canwait);
445 _MINTGET(m, type);
446 if (m) {
447 (m)->m_next = (m)->m_nextpkt = 0;
448 (m)->m_type = (type);
449 (m)->m_data = (m)->m_dat;
450 (m)->m_flags = 0;
451 (m)->m_len = 0;
452 }
453 if (m || canwait == M_DONTWAIT)
454 break;
455 MBUF_LOCK();
456 wait = m_want++;
457 mbuf_expand_mcl++;
458 if (wait == 0)
459 mbstat.m_drain++;
460 else
461 mbstat.m_wait++;
462 MBUF_UNLOCK();
463
464 if (mbuf_expand_thread_initialized)
465 wakeup((caddr_t)&mbuf_expand_thread_wakeup);
466
467 if (wait == 0) {
468 m_reclaim();
469 } else {
470 struct timespec ts;
471 ts.tv_sec = 1;
472 ts.tv_nsec = 0;
473 (void) msleep((caddr_t)&mfree, 0, (PZERO-1) | PDROP, "m_retry", &ts);
474 }
475 }
476 if (m == 0)
477 mbstat.m_drops++;
478 return (m);
479 }
480
481 /*
482 * As above; retry an MGETHDR.
483 */
484 struct mbuf *
485 m_retryhdr(
486 int canwait,
487 int type)
488 {
489 struct mbuf *m;
490
491 if ((m = m_retry(canwait, type))) {
492 m->m_next = m->m_nextpkt = 0;
493 m->m_flags |= M_PKTHDR;
494 m->m_data = m->m_pktdat;
495 _M_CLEAR_PKTHDR(m);
496 }
497 return (m);
498 }
499
500 void
501 m_reclaim(void)
502 {
503 do_reclaim = 1; /* drain is performed in pfslowtimo(), to avoid deadlocks */
504 mbstat.m_drain++;
505 }
506
507 /*
508 * Space allocation routines.
509 * These are also available as macros
510 * for critical paths.
511 */
512 struct mbuf *
513 m_get(
514 int nowait,
515 int type)
516 {
517 struct mbuf *m;
518
519 m_range_check(mfree);
520 m_range_check(mclfree);
521 m_range_check(mbigfree);
522
523 _MINTGET(m, type);
524 if (m) {
525 m->m_next = m->m_nextpkt = 0;
526 m->m_type = type;
527 m->m_data = m->m_dat;
528 m->m_flags = 0;
529 m->m_len = 0;
530 } else
531 (m) = m_retry(nowait, type);
532
533 m_range_check(mfree);
534 m_range_check(mclfree);
535 m_range_check(mbigfree);
536
537
538 return (m);
539 }
540
541 struct mbuf *
542 m_gethdr(
543 int nowait,
544 int type)
545 {
546 struct mbuf *m;
547
548 m_range_check(mfree);
549 m_range_check(mclfree);
550 m_range_check(mbigfree);
551
552
553 _MINTGET(m, type);
554 if (m) {
555 m->m_next = m->m_nextpkt = 0;
556 m->m_type = type;
557 m->m_data = m->m_pktdat;
558 m->m_flags = M_PKTHDR;
559 m->m_len = 0;
560 _M_CLEAR_PKTHDR(m)
561 } else
562 m = m_retryhdr(nowait, type);
563
564 m_range_check(mfree);
565 m_range_check(mclfree);
566 m_range_check(mbigfree);
567
568
569 return m;
570 }
571
572 struct mbuf *
573 m_getclr(
574 int nowait,
575 int type)
576 {
577 struct mbuf *m;
578
579 MGET(m, nowait, type);
580 if (m == 0)
581 return (0);
582 bzero(mtod(m, caddr_t), MLEN);
583 return (m);
584 }
585
586 struct mbuf *
587 m_free(
588 struct mbuf *m)
589 {
590 struct mbuf *n = m->m_next;
591 int i;
592
593 m_range_check(m);
594 m_range_check(mfree);
595 m_range_check(mclfree);
596
597 if (m->m_type == MT_FREE)
598 panic("freeing free mbuf");
599
600 /* Free the aux data if there is any */
601 if ((m->m_flags & M_PKTHDR) && m->m_pkthdr.aux)
602 {
603 m_freem(m->m_pkthdr.aux);
604 }
605 if ((m->m_flags & M_PKTHDR) != 0)
606 m_tag_delete_chain(m, NULL);
607
608 MBUF_LOCK();
609 if ((m->m_flags & M_EXT))
610 {
611 if (MCLHASREFERENCE(m)) {
612 remque((queue_t)&m->m_ext.ext_refs);
613 } else if (m->m_ext.ext_free == NULL) {
614 union mcluster *mcl= (union mcluster *)m->m_ext.ext_buf;
615
616 m_range_check(mcl);
617
618 if (_MCLUNREF(mcl)) {
619 mcl->mcl_next = mclfree;
620 mclfree = mcl;
621 ++mbstat.m_clfree;
622 }
623 #ifdef COMMENT_OUT
624 /* *** Since m_split() increments "mclrefcnt[mtocl(m->m_ext.ext_buf)]",
625 and AppleTalk ADSP uses m_split(), this incorrect sanity check
626 caused a panic.
627 *** */
628 else /* sanity check - not referenced this way */
629 panic("m_free m_ext cluster not free");
630 #endif
631 } else {
632 (*(m->m_ext.ext_free))(m->m_ext.ext_buf,
633 m->m_ext.ext_size, m->m_ext.ext_arg);
634 }
635 }
636 mbstat.m_mtypes[m->m_type]--;
637 (void) _MCLUNREF(m);
638 _MFREE_MUNGE(m);
639 m->m_type = MT_FREE;
640 mbstat.m_mtypes[m->m_type]++;
641 m->m_flags = 0;
642 m->m_next = mfree;
643 m->m_len = 0;
644 mfree = m;
645 i = m_want;
646 m_want = 0;
647 MBUF_UNLOCK();
648 if (i) wakeup((caddr_t)&mfree);
649 return (n);
650 }
651
652 /* m_mclget() add an mbuf cluster to a normal mbuf */
653 struct mbuf *
654 m_mclget(
655 struct mbuf *m,
656 int nowait)
657 {
658 MCLALLOC(m->m_ext.ext_buf, nowait);
659 if (m->m_ext.ext_buf) {
660 m->m_data = m->m_ext.ext_buf;
661 m->m_flags |= M_EXT;
662 m->m_ext.ext_size = MCLBYTES;
663 m->m_ext.ext_free = 0;
664 m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward =
665 &m->m_ext.ext_refs;
666 }
667
668 return m;
669 }
670
671 /* m_mclalloc() allocate an mbuf cluster */
672 caddr_t
673 m_mclalloc(
674 int nowait)
675 {
676 caddr_t p;
677
678 (void)m_clalloc(1, nowait, MCLBYTES, 0);
679 if ((p = (caddr_t)mclfree)) {
680 ++mclrefcnt[mtocl(p)];
681 mbstat.m_clfree--;
682 mclfree = ((union mcluster *)p)->mcl_next;
683 } else {
684 mbstat.m_drops++;
685 }
686 MBUF_UNLOCK();
687
688 return p;
689 }
690
691 /* m_mclfree() releases a reference to a cluster allocated by MCLALLOC,
692 * freeing the cluster if the reference count has reached 0. */
693 void
694 m_mclfree(
695 caddr_t p)
696 {
697 MBUF_LOCK();
698
699 m_range_check(p);
700
701 if (--mclrefcnt[mtocl(p)] == 0) {
702 ((union mcluster *)(p))->mcl_next = mclfree;
703 mclfree = (union mcluster *)(p);
704 mbstat.m_clfree++;
705 }
706 MBUF_UNLOCK();
707 }
708
709 /* mcl_hasreference() checks if a cluster of an mbuf is referenced by another mbuf */
710 int
711 m_mclhasreference(
712 struct mbuf *m)
713 {
714 return (m->m_ext.ext_refs.forward != &(m->m_ext.ext_refs));
715 }
716
717 __private_extern__ caddr_t
718 m_bigalloc(int nowait)
719 {
720 caddr_t p;
721
722 (void)m_clalloc(1, nowait, NBPG, 0);
723 if ((p = (caddr_t)mbigfree)) {
724 if (mclrefcnt[mtocl(p)] != mclrefcnt[mtocl(p) + 1])
725 panic("m_bigalloc mclrefcnt %x mismatch %d != %d",
726 p, mclrefcnt[mtocl(p)], mclrefcnt[mtocl(p) + 1]);
727 if (mclrefcnt[mtocl(p)] || mclrefcnt[mtocl(p) + 1])
728 panic("m_bigalloc mclrefcnt %x not null %d != %d",
729 p, mclrefcnt[mtocl(p)], mclrefcnt[mtocl(p) + 1]);
730 ++mclrefcnt[mtocl(p)];
731 ++mclrefcnt[mtocl(p) + 1];
732 mbstat.m_bigclfree--;
733 mbigfree = ((union mbigcluster *)p)->mbc_next;
734 } else {
735 mbstat.m_drops++;
736 }
737 MBUF_UNLOCK();
738 return p;
739 }
740
741 __private_extern__ void
742 m_bigfree(caddr_t p, __unused u_int size, __unused caddr_t arg)
743 {
744 m_range_check(p);
745
746 if (mclrefcnt[mtocl(p)] != mclrefcnt[mtocl(p) + 1])
747 panic("m_bigfree mclrefcnt %x mismatch %d != %d",
748 p, mclrefcnt[mtocl(p)], mclrefcnt[mtocl(p) + 1]);
749 --mclrefcnt[mtocl(p)];
750 --mclrefcnt[mtocl(p) + 1];
751 if (mclrefcnt[mtocl(p)] == 0) {
752 ((union mbigcluster *)(p))->mbc_next = mbigfree;
753 mbigfree = (union mbigcluster *)(p);
754 mbstat.m_bigclfree++;
755 }
756 }
757
758 /* m_mbigget() add an 4KB mbuf cluster to a normal mbuf */
759 __private_extern__ struct mbuf *
760 m_mbigget(struct mbuf *m, int nowait)
761 {
762 m->m_ext.ext_buf = m_bigalloc(nowait);
763 if (m->m_ext.ext_buf) {
764 m->m_data = m->m_ext.ext_buf;
765 m->m_flags |= M_EXT;
766 m->m_ext.ext_size = NBPG;
767 m->m_ext.ext_free = m_bigfree;
768 m->m_ext.ext_arg = 0;
769 m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward =
770 &m->m_ext.ext_refs;
771 }
772
773 return m;
774 }
775
776
777 /* */
778 void
779 m_copy_pkthdr(
780 struct mbuf *to,
781 struct mbuf *from)
782 {
783 to->m_pkthdr = from->m_pkthdr;
784 from->m_pkthdr.aux = (struct mbuf *)NULL;
785 SLIST_INIT(&from->m_pkthdr.tags); /* purge tags from src */
786 to->m_flags = from->m_flags & M_COPYFLAGS;
787 to->m_data = (to)->m_pktdat;
788 }
789
790 /*
791 * "Move" mbuf pkthdr from "from" to "to".
792 * "from" must have M_PKTHDR set, and "to" must be empty.
793 */
794 #ifndef __APPLE__
795 void
796 m_move_pkthdr(struct mbuf *to, struct mbuf *from)
797 {
798 KASSERT((to->m_flags & M_EXT) == 0, ("m_move_pkthdr: to has cluster"));
799
800 to->m_flags = from->m_flags & M_COPYFLAGS;
801 to->m_data = to->m_pktdat;
802 to->m_pkthdr = from->m_pkthdr; /* especially tags */
803 SLIST_INIT(&from->m_pkthdr.tags); /* purge tags from src */
804 from->m_flags &= ~M_PKTHDR;
805 }
806 #endif
807
808 /*
809 * Duplicate "from"'s mbuf pkthdr in "to".
810 * "from" must have M_PKTHDR set, and "to" must be empty.
811 * In particular, this does a deep copy of the packet tags.
812 */
813 static int
814 m_dup_pkthdr(struct mbuf *to, struct mbuf *from, int how)
815 {
816 to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
817 if ((to->m_flags & M_EXT) == 0)
818 to->m_data = to->m_pktdat;
819 to->m_pkthdr = from->m_pkthdr;
820 SLIST_INIT(&to->m_pkthdr.tags);
821 return (m_tag_copy_chain(to, from, how));
822 }
823
824 /*
825 * return a list of mbuf hdrs that point to clusters...
826 * try for num_needed, if wantall is not set, return whatever
827 * number were available... set up the first num_with_pkthdrs
828 * with mbuf hdrs configured as packet headers... these are
829 * chained on the m_nextpkt field... any packets requested beyond
830 * this are chained onto the last packet header's m_next field.
831 * The size of the cluster is controlled by the paramter bufsize.
832 */
833 __private_extern__ struct mbuf *
834 m_getpackets_internal(unsigned int *num_needed, int num_with_pkthdrs, int how, int wantall, size_t bufsize)
835 {
836 struct mbuf *m;
837 struct mbuf **np, *top;
838 unsigned int num, needed = *num_needed;
839
840 if (bufsize != MCLBYTES && bufsize != NBPG)
841 return 0;
842
843 top = NULL;
844 np = &top;
845
846 (void)m_clalloc(needed, how, bufsize, 0); /* takes the MBUF_LOCK, but doesn't release it... */
847
848 for (num = 0; num < needed; num++) {
849 m_range_check(mfree);
850 m_range_check(mclfree);
851 m_range_check(mbigfree);
852
853 if (mfree && ((bufsize == NBPG && mbigfree) || (bufsize == MCLBYTES && mclfree))) {
854 /* mbuf + cluster are available */
855 m = mfree;
856 MCHECK(m);
857 mfree = m->m_next;
858 ++mclrefcnt[mtocl(m)];
859 mbstat.m_mtypes[MT_FREE]--;
860 mbstat.m_mtypes[MT_DATA]++;
861 if (bufsize == NBPG) {
862 m->m_ext.ext_buf = (caddr_t)mbigfree; /* get the big cluster */
863 ++mclrefcnt[mtocl(m->m_ext.ext_buf)];
864 ++mclrefcnt[mtocl(m->m_ext.ext_buf) + 1];
865 mbstat.m_bigclfree--;
866 mbigfree = ((union mbigcluster *)(m->m_ext.ext_buf))->mbc_next;
867 m->m_ext.ext_free = m_bigfree;
868 m->m_ext.ext_size = NBPG;
869 } else {
870 m->m_ext.ext_buf = (caddr_t)mclfree; /* get the cluster */
871 ++mclrefcnt[mtocl(m->m_ext.ext_buf)];
872 mbstat.m_clfree--;
873 mclfree = ((union mcluster *)(m->m_ext.ext_buf))->mcl_next;
874 m->m_ext.ext_free = 0;
875 m->m_ext.ext_size = MCLBYTES;
876 }
877 m->m_ext.ext_arg = 0;
878 m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward = &m->m_ext.ext_refs;
879 m->m_next = m->m_nextpkt = 0;
880 m->m_type = MT_DATA;
881 m->m_data = m->m_ext.ext_buf;
882 m->m_len = 0;
883
884 if (num_with_pkthdrs == 0)
885 m->m_flags = M_EXT;
886 else {
887 m->m_flags = M_PKTHDR | M_EXT;
888 _M_CLEAR_PKTHDR(m);
889
890 num_with_pkthdrs--;
891 }
892 } else {
893 MBUF_UNLOCK();
894
895 if (num_with_pkthdrs == 0) {
896 MGET(m, how, MT_DATA );
897 } else {
898 MGETHDR(m, how, MT_DATA);
899
900 num_with_pkthdrs--;
901 }
902 if (m == 0)
903 goto fail;
904
905 if (bufsize == NBPG)
906 m = m_mbigget(m, how);
907 else
908 m = m_mclget(m, how);
909 if ((m->m_flags & M_EXT) == 0) {
910 m_free(m);
911 goto fail;
912 }
913 MBUF_LOCK();
914 }
915 *np = m;
916
917 if (num_with_pkthdrs)
918 np = &m->m_nextpkt;
919 else
920 np = &m->m_next;
921 }
922 MBUF_UNLOCK();
923
924 *num_needed = num;
925 return (top);
926 fail:
927 if (wantall && top) {
928 m_freem(top);
929 return 0;
930 }
931 return top;
932 }
933
934
935 /*
936 * Return list of mbuf linked by m_nextpkt
937 * Try for num_needed, and if wantall is not set, return whatever
938 * number were available
939 * The size of each mbuf in the list is controlled by the parameter packetlen.
940 * Each mbuf of the list may have a chain of mbufs linked by m_next. Each mbuf in
941 * the chain is called a segment.
942 * If maxsegments is not null and the value pointed to is not null, this specify
943 * the maximum number of segments for a chain of mbufs.
944 * If maxsegments is zero or the value pointed to is zero the
945 * caller does not have any restriction on the number of segments.
946 * The actual number of segments of a mbuf chain is return in the value pointed
947 * to by maxsegments.
948 * When possible the allocation is done under a single lock.
949 */
950
951 __private_extern__ struct mbuf *
952 m_allocpacket_internal(unsigned int *num_needed, size_t packetlen, unsigned int * maxsegments,
953 int how, int wantall, size_t wantsize)
954 {
955 struct mbuf **np, *top;
956 size_t bufsize;
957 unsigned int num;
958 unsigned int numchunks = 0;
959
960 top = NULL;
961 np = &top;
962
963 if (wantsize == 0) {
964 if (packetlen <= MINCLSIZE)
965 bufsize = packetlen;
966 else if (packetlen > MCLBYTES)
967 bufsize = NBPG;
968 else
969 bufsize = MCLBYTES;
970 } else if (wantsize == MCLBYTES || wantsize == NBPG)
971 bufsize = wantsize;
972 else
973 return 0;
974
975 if (bufsize <= MHLEN) {
976 numchunks = 1;
977 } else if (bufsize <= MINCLSIZE) {
978 if (maxsegments != NULL && *maxsegments == 1) {
979 bufsize = MCLBYTES;
980 numchunks = 1;
981 } else {
982 numchunks = 2;
983 }
984 } else if (bufsize == NBPG) {
985 numchunks = ((packetlen - 1) >> PGSHIFT) + 1;
986 } else {
987 numchunks = ((packetlen - 1) >> MCLSHIFT) + 1;
988 }
989 if (maxsegments != NULL) {
990 if (*maxsegments && numchunks > *maxsegments) {
991 *maxsegments = numchunks;
992 return 0;
993 }
994 *maxsegments = numchunks;
995 }
996 /* m_clalloc takes the MBUF_LOCK, but do not release it */
997 (void)m_clalloc(numchunks, how, (bufsize == NBPG) ? NBPG : MCLBYTES, 0);
998 for (num = 0; num < *num_needed; num++) {
999 struct mbuf **nm, *pkt = 0;
1000 size_t len;
1001
1002 nm = &pkt;
1003
1004 m_range_check(mfree);
1005 m_range_check(mclfree);
1006 m_range_check(mbigfree);
1007
1008 for (len = 0; len < packetlen; ) {
1009 struct mbuf *m = NULL;
1010
1011 if (wantsize == 0 && packetlen > MINCLSIZE) {
1012 if (packetlen - len > MCLBYTES)
1013 bufsize = NBPG;
1014 else
1015 bufsize = MCLBYTES;
1016 }
1017 len += bufsize;
1018
1019 if (mfree && ((bufsize == NBPG && mbigfree) || (bufsize == MCLBYTES && mclfree))) {
1020 /* mbuf + cluster are available */
1021 m = mfree;
1022 MCHECK(m);
1023 mfree = m->m_next;
1024 ++mclrefcnt[mtocl(m)];
1025 mbstat.m_mtypes[MT_FREE]--;
1026 mbstat.m_mtypes[MT_DATA]++;
1027 if (bufsize == NBPG) {
1028 m->m_ext.ext_buf = (caddr_t)mbigfree; /* get the big cluster */
1029 ++mclrefcnt[mtocl(m->m_ext.ext_buf)];
1030 ++mclrefcnt[mtocl(m->m_ext.ext_buf) + 1];
1031 mbstat.m_bigclfree--;
1032 mbigfree = ((union mbigcluster *)(m->m_ext.ext_buf))->mbc_next;
1033 m->m_ext.ext_free = m_bigfree;
1034 m->m_ext.ext_size = NBPG;
1035 } else {
1036 m->m_ext.ext_buf = (caddr_t)mclfree; /* get the cluster */
1037 ++mclrefcnt[mtocl(m->m_ext.ext_buf)];
1038 mbstat.m_clfree--;
1039 mclfree = ((union mcluster *)(m->m_ext.ext_buf))->mcl_next;
1040 m->m_ext.ext_free = 0;
1041 m->m_ext.ext_size = MCLBYTES;
1042 }
1043 m->m_ext.ext_arg = 0;
1044 m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward = &m->m_ext.ext_refs;
1045 m->m_next = m->m_nextpkt = 0;
1046 m->m_type = MT_DATA;
1047 m->m_data = m->m_ext.ext_buf;
1048 m->m_len = 0;
1049
1050 if (pkt == 0) {
1051 pkt = m;
1052 m->m_flags = M_PKTHDR | M_EXT;
1053 _M_CLEAR_PKTHDR(m);
1054 } else {
1055 m->m_flags = M_EXT;
1056 }
1057 } else {
1058 MBUF_UNLOCK();
1059
1060 if (pkt == 0) {
1061 MGETHDR(m, how, MT_DATA);
1062 } else {
1063 MGET(m, how, MT_DATA );
1064 }
1065 if (m == 0) {
1066 m_freem(pkt);
1067 goto fail;
1068 }
1069 if (bufsize <= MINCLSIZE) {
1070 if (bufsize > MHLEN) {
1071 MGET(m->m_next, how, MT_DATA);
1072 if (m->m_next == 0) {
1073 m_free(m);
1074 m_freem(pkt);
1075 goto fail;
1076 }
1077 }
1078 } else {
1079 if (bufsize == NBPG)
1080 m = m_mbigget(m, how);
1081 else
1082 m = m_mclget(m, how);
1083 if ((m->m_flags & M_EXT) == 0) {
1084 m_free(m);
1085 m_freem(pkt);
1086 goto fail;
1087 }
1088 }
1089 MBUF_LOCK();
1090 }
1091 *nm = m;
1092 nm = &m->m_next;
1093 }
1094 *np = pkt;
1095 np = &pkt->m_nextpkt;
1096 }
1097 MBUF_UNLOCK();
1098 *num_needed = num;
1099
1100 return top;
1101 fail:
1102 if (wantall && top) {
1103 m_freem(top);
1104 return 0;
1105 }
1106 *num_needed = num;
1107
1108 return top;
1109 }
1110
1111
1112 /* Best effort to get a mbuf cluster + pkthdr under one lock.
1113 * If we don't have them avail, just bail out and use the regular
1114 * path.
1115 * Used by drivers to allocated packets on receive ring.
1116 */
1117 __private_extern__ struct mbuf *
1118 m_getpacket_how(int how)
1119 {
1120 unsigned int num_needed = 1;
1121
1122 return m_getpackets_internal(&num_needed, 1, how, 1, MCLBYTES);
1123 }
1124
1125 /* Best effort to get a mbuf cluster + pkthdr under one lock.
1126 * If we don't have them avail, just bail out and use the regular
1127 * path.
1128 * Used by drivers to allocated packets on receive ring.
1129 */
1130 struct mbuf *
1131 m_getpacket(void)
1132 {
1133 unsigned int num_needed = 1;
1134
1135 return m_getpackets_internal(&num_needed, 1, M_WAITOK, 1, MCLBYTES);
1136 }
1137
1138
1139 /*
1140 * return a list of mbuf hdrs that point to clusters...
1141 * try for num_needed, if this can't be met, return whatever
1142 * number were available... set up the first num_with_pkthdrs
1143 * with mbuf hdrs configured as packet headers... these are
1144 * chained on the m_nextpkt field... any packets requested beyond
1145 * this are chained onto the last packet header's m_next field.
1146 */
1147 struct mbuf *
1148 m_getpackets(int num_needed, int num_with_pkthdrs, int how)
1149 {
1150 unsigned int n = num_needed;
1151
1152 return m_getpackets_internal(&n, num_with_pkthdrs, how, 0, MCLBYTES);
1153 }
1154
1155
1156 /*
1157 * return a list of mbuf hdrs set up as packet hdrs
1158 * chained together on the m_nextpkt field
1159 */
1160 struct mbuf *
1161 m_getpackethdrs(int num_needed, int how)
1162 {
1163 struct mbuf *m;
1164 struct mbuf **np, *top;
1165
1166 top = NULL;
1167 np = &top;
1168
1169 MBUF_LOCK();
1170
1171 while (num_needed--) {
1172 m_range_check(mfree);
1173 m_range_check(mclfree);
1174 m_range_check(mbigfree);
1175
1176 if ((m = mfree)) { /* mbufs are available */
1177 MCHECK(m);
1178 mfree = m->m_next;
1179 ++mclrefcnt[mtocl(m)];
1180 mbstat.m_mtypes[MT_FREE]--;
1181 mbstat.m_mtypes[MT_DATA]++;
1182
1183 m->m_next = m->m_nextpkt = 0;
1184 m->m_type = MT_DATA;
1185 m->m_flags = M_PKTHDR;
1186 m->m_len = 0;
1187 m->m_data = m->m_pktdat;
1188 _M_CLEAR_PKTHDR(m);
1189
1190 } else {
1191
1192 MBUF_UNLOCK();
1193 m = m_retryhdr(how, MT_DATA);
1194 if (m == 0)
1195 return(top);
1196 MBUF_LOCK();
1197 }
1198 *np = m;
1199 np = &m->m_nextpkt;
1200 }
1201 MBUF_UNLOCK();
1202
1203 return (top);
1204 }
1205
1206
1207 /* free and mbuf list (m_nextpkt) while following m_next under one lock.
1208 * returns the count for mbufs packets freed. Used by the drivers.
1209 */
1210 int
1211 m_freem_list(
1212 struct mbuf *m)
1213 {
1214 struct mbuf *nextpkt;
1215 int i, count=0;
1216
1217 MBUF_LOCK();
1218
1219 while (m) {
1220 if (m)
1221 nextpkt = m->m_nextpkt; /* chain of linked mbufs from driver */
1222 else
1223 nextpkt = 0;
1224
1225 count++;
1226
1227 while (m) { /* free the mbuf chain (like mfreem) */
1228
1229 struct mbuf *n;
1230
1231 m_range_check(m);
1232 m_range_check(mfree);
1233 m_range_check(mclfree);
1234 m_range_check(mbigfree);
1235
1236
1237 /* Free the aux data if there is any */
1238 if ((m->m_flags & M_PKTHDR) && m->m_pkthdr.aux) {
1239 /*
1240 * Treat the current m as the nextpkt and set m
1241 * to the aux data. Preserve nextpkt in m->m_nextpkt.
1242 * This lets us free the aux data in this loop
1243 * without having to call m_freem recursively,
1244 * which wouldn't work because we've still got
1245 * the lock.
1246 */
1247 m->m_nextpkt = nextpkt;
1248 nextpkt = m;
1249 m = nextpkt->m_pkthdr.aux;
1250 nextpkt->m_pkthdr.aux = NULL;
1251 }
1252
1253 if ((m->m_flags & M_PKTHDR) != 0 && !SLIST_EMPTY(&m->m_pkthdr.tags)) {
1254 /* A quick (albeit inefficient) expedient */
1255 MBUF_UNLOCK();
1256 m_tag_delete_chain(m, NULL);
1257 MBUF_LOCK();
1258 }
1259
1260 n = m->m_next;
1261
1262 if (n && n->m_nextpkt)
1263 panic("m_freem_list: m_nextpkt of m_next != NULL");
1264 if (m->m_type == MT_FREE)
1265 panic("freeing free mbuf");
1266
1267 if (m->m_flags & M_EXT) {
1268 if (MCLHASREFERENCE(m)) {
1269 remque((queue_t)&m->m_ext.ext_refs);
1270 } else if (m->m_ext.ext_free == NULL) {
1271 union mcluster *mcl= (union mcluster *)m->m_ext.ext_buf;
1272
1273 m_range_check(mcl);
1274
1275 if (_MCLUNREF(mcl)) {
1276 mcl->mcl_next = mclfree;
1277 mclfree = mcl;
1278 ++mbstat.m_clfree;
1279 }
1280 } else {
1281 (*(m->m_ext.ext_free))(m->m_ext.ext_buf,
1282 m->m_ext.ext_size, m->m_ext.ext_arg);
1283 }
1284 }
1285 mbstat.m_mtypes[m->m_type]--;
1286 (void) _MCLUNREF(m);
1287 _MFREE_MUNGE(m);
1288 mbstat.m_mtypes[MT_FREE]++;
1289 m->m_type = MT_FREE;
1290 m->m_flags = 0;
1291 m->m_len = 0;
1292 m->m_next = mfree;
1293 mfree = m;
1294 m = n;
1295 }
1296 m = nextpkt; /* bump m with saved nextpkt if any */
1297 }
1298 if ((i = m_want))
1299 m_want = 0;
1300
1301 MBUF_UNLOCK();
1302
1303 if (i)
1304 wakeup((caddr_t)&mfree);
1305
1306 return (count);
1307 }
1308
1309 void
1310 m_freem(
1311 struct mbuf *m)
1312 {
1313 while (m)
1314 m = m_free(m);
1315 }
1316
1317 /*
1318 * Mbuffer utility routines.
1319 */
1320 /*
1321 * Compute the amount of space available
1322 * before the current start of data in an mbuf.
1323 */
1324 int
1325 m_leadingspace(
1326 struct mbuf *m)
1327 {
1328 if (m->m_flags & M_EXT) {
1329 if (MCLHASREFERENCE(m))
1330 return(0);
1331 return (m->m_data - m->m_ext.ext_buf);
1332 }
1333 if (m->m_flags & M_PKTHDR)
1334 return (m->m_data - m->m_pktdat);
1335 return (m->m_data - m->m_dat);
1336 }
1337
1338 /*
1339 * Compute the amount of space available
1340 * after the end of data in an mbuf.
1341 */
1342 int
1343 m_trailingspace(
1344 struct mbuf *m)
1345 {
1346 if (m->m_flags & M_EXT) {
1347 if (MCLHASREFERENCE(m))
1348 return(0);
1349 return (m->m_ext.ext_buf + m->m_ext.ext_size -
1350 (m->m_data + m->m_len));
1351 }
1352 return (&m->m_dat[MLEN] - (m->m_data + m->m_len));
1353 }
1354
1355 /*
1356 * Lesser-used path for M_PREPEND:
1357 * allocate new mbuf to prepend to chain,
1358 * copy junk along.
1359 * Does not adjust packet header length.
1360 */
1361 struct mbuf *
1362 m_prepend(
1363 struct mbuf *m,
1364 int len,
1365 int how)
1366 {
1367 struct mbuf *mn;
1368
1369 MGET(mn, how, m->m_type);
1370 if (mn == (struct mbuf *)NULL) {
1371 m_freem(m);
1372 return ((struct mbuf *)NULL);
1373 }
1374 if (m->m_flags & M_PKTHDR) {
1375 M_COPY_PKTHDR(mn, m);
1376 m->m_flags &= ~M_PKTHDR;
1377 }
1378 mn->m_next = m;
1379 m = mn;
1380 if (len < MHLEN)
1381 MH_ALIGN(m, len);
1382 m->m_len = len;
1383 return (m);
1384 }
1385
1386 /*
1387 * Replacement for old M_PREPEND macro:
1388 * allocate new mbuf to prepend to chain,
1389 * copy junk along, and adjust length.
1390 *
1391 */
1392 struct mbuf *
1393 m_prepend_2(
1394 struct mbuf *m,
1395 int len,
1396 int how)
1397 {
1398 if (M_LEADINGSPACE(m) >= len) {
1399 m->m_data -= len;
1400 m->m_len += len;
1401 } else {
1402 m = m_prepend(m, len, how);
1403 }
1404 if ((m) && (m->m_flags & M_PKTHDR))
1405 m->m_pkthdr.len += len;
1406 return (m);
1407 }
1408
1409 /*
1410 * Make a copy of an mbuf chain starting "off0" bytes from the beginning,
1411 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf.
1412 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller.
1413 */
1414 int MCFail;
1415
1416 struct mbuf *
1417 m_copym(
1418 struct mbuf *m,
1419 int off0,
1420 int len,
1421 int wait)
1422 {
1423 struct mbuf *n, **np;
1424 int off = off0;
1425 struct mbuf *top;
1426 int copyhdr = 0;
1427
1428 if (off < 0 || len < 0)
1429 panic("m_copym");
1430 if (off == 0 && m->m_flags & M_PKTHDR)
1431 copyhdr = 1;
1432
1433 while (off >= m->m_len) {
1434 if (m == 0)
1435 panic("m_copym");
1436 off -= m->m_len;
1437 m = m->m_next;
1438 }
1439 np = &top;
1440 top = 0;
1441
1442 MBUF_LOCK();
1443
1444 while (len > 0) {
1445 m_range_check(mfree);
1446 m_range_check(mclfree);
1447 m_range_check(mbigfree);
1448
1449 if (m == 0) {
1450 if (len != M_COPYALL)
1451 panic("m_copym");
1452 break;
1453 }
1454 if ((n = mfree)) {
1455 MCHECK(n);
1456 ++mclrefcnt[mtocl(n)];
1457 mbstat.m_mtypes[MT_FREE]--;
1458 mbstat.m_mtypes[m->m_type]++;
1459 mfree = n->m_next;
1460 n->m_next = n->m_nextpkt = 0;
1461 n->m_type = m->m_type;
1462 n->m_data = n->m_dat;
1463 n->m_flags = 0;
1464 } else {
1465 MBUF_UNLOCK();
1466 n = m_retry(wait, m->m_type);
1467 MBUF_LOCK();
1468 }
1469 *np = n;
1470
1471 if (n == 0)
1472 goto nospace;
1473 if (copyhdr) {
1474 M_COPY_PKTHDR(n, m);
1475 if (len == M_COPYALL)
1476 n->m_pkthdr.len -= off0;
1477 else
1478 n->m_pkthdr.len = len;
1479 copyhdr = 0;
1480 }
1481 if (len == M_COPYALL) {
1482 if (min(len, (m->m_len - off)) == len) {
1483 printf("m->m_len %d - off %d = %d, %d\n",
1484 m->m_len, off, m->m_len - off,
1485 min(len, (m->m_len - off)));
1486 }
1487 }
1488 n->m_len = min(len, (m->m_len - off));
1489 if (n->m_len == M_COPYALL) {
1490 printf("n->m_len == M_COPYALL, fixing\n");
1491 n->m_len = MHLEN;
1492 }
1493 if (m->m_flags & M_EXT) {
1494 n->m_ext = m->m_ext;
1495 insque((queue_t)&n->m_ext.ext_refs, (queue_t)&m->m_ext.ext_refs);
1496 n->m_data = m->m_data + off;
1497 n->m_flags |= M_EXT;
1498 } else {
1499 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
1500 (unsigned)n->m_len);
1501 }
1502 if (len != M_COPYALL)
1503 len -= n->m_len;
1504 off = 0;
1505 m = m->m_next;
1506 np = &n->m_next;
1507 }
1508 MBUF_UNLOCK();
1509
1510 if (top == 0)
1511 MCFail++;
1512
1513 return (top);
1514 nospace:
1515 MBUF_UNLOCK();
1516
1517 m_freem(top);
1518 MCFail++;
1519 return (0);
1520 }
1521
1522
1523 /*
1524 * equivilent to m_copym except that all necessary
1525 * mbuf hdrs are allocated within this routine
1526 * also, the last mbuf and offset accessed are passed
1527 * out and can be passed back in to avoid having to
1528 * rescan the entire mbuf list (normally hung off of the socket)
1529 */
1530 struct mbuf *
1531 m_copym_with_hdrs(
1532 struct mbuf *m,
1533 int off0,
1534 int len,
1535 int wait,
1536 struct mbuf **m_last,
1537 int *m_off)
1538 {
1539 struct mbuf *n, **np = 0;
1540 int off = off0;
1541 struct mbuf *top = 0;
1542 int copyhdr = 0;
1543 int type;
1544
1545 if (off == 0 && m->m_flags & M_PKTHDR)
1546 copyhdr = 1;
1547
1548 if (*m_last) {
1549 m = *m_last;
1550 off = *m_off;
1551 } else {
1552 while (off >= m->m_len) {
1553 off -= m->m_len;
1554 m = m->m_next;
1555 }
1556 }
1557
1558 MBUF_LOCK();
1559
1560 while (len > 0) {
1561 m_range_check(mfree);
1562 m_range_check(mclfree);
1563 m_range_check(mbigfree);
1564
1565 if (top == 0)
1566 type = MT_HEADER;
1567 else {
1568 if (m == 0)
1569 panic("m_gethdr_and_copym");
1570 type = m->m_type;
1571 }
1572 if ((n = mfree)) {
1573 MCHECK(n);
1574 ++mclrefcnt[mtocl(n)];
1575 mbstat.m_mtypes[MT_FREE]--;
1576 mbstat.m_mtypes[type]++;
1577 mfree = n->m_next;
1578 n->m_next = n->m_nextpkt = 0;
1579 n->m_type = type;
1580
1581 if (top) {
1582 n->m_data = n->m_dat;
1583 n->m_flags = 0;
1584 } else {
1585 n->m_data = n->m_pktdat;
1586 n->m_flags = M_PKTHDR;
1587 _M_CLEAR_PKTHDR(n);
1588 }
1589 } else {
1590 MBUF_UNLOCK();
1591 if (top)
1592 n = m_retry(wait, type);
1593 else
1594 n = m_retryhdr(wait, type);
1595 MBUF_LOCK();
1596 }
1597 if (n == 0)
1598 goto nospace;
1599 if (top == 0) {
1600 top = n;
1601 np = &top->m_next;
1602 continue;
1603 } else
1604 *np = n;
1605
1606 if (copyhdr) {
1607 M_COPY_PKTHDR(n, m);
1608 n->m_pkthdr.len = len;
1609 copyhdr = 0;
1610 }
1611 n->m_len = min(len, (m->m_len - off));
1612
1613 if (m->m_flags & M_EXT) {
1614 n->m_ext = m->m_ext;
1615 insque((queue_t)&n->m_ext.ext_refs, (queue_t)&m->m_ext.ext_refs);
1616 n->m_data = m->m_data + off;
1617 n->m_flags |= M_EXT;
1618 } else {
1619 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
1620 (unsigned)n->m_len);
1621 }
1622 len -= n->m_len;
1623
1624 if (len == 0) {
1625 if ((off + n->m_len) == m->m_len) {
1626 *m_last = m->m_next;
1627 *m_off = 0;
1628 } else {
1629 *m_last = m;
1630 *m_off = off + n->m_len;
1631 }
1632 break;
1633 }
1634 off = 0;
1635 m = m->m_next;
1636 np = &n->m_next;
1637 }
1638 MBUF_UNLOCK();
1639
1640 return (top);
1641 nospace:
1642 MBUF_UNLOCK();
1643
1644 if (top)
1645 m_freem(top);
1646 MCFail++;
1647 return (0);
1648 }
1649
1650
1651 /*
1652 * Copy data from an mbuf chain starting "off" bytes from the beginning,
1653 * continuing for "len" bytes, into the indicated buffer.
1654 */
1655 void m_copydata(
1656 struct mbuf *m,
1657 int off,
1658 int len,
1659 caddr_t cp)
1660 {
1661 unsigned count;
1662
1663 if (off < 0 || len < 0)
1664 panic("m_copydata");
1665 while (off > 0) {
1666 if (m == 0)
1667 panic("m_copydata");
1668 if (off < m->m_len)
1669 break;
1670 off -= m->m_len;
1671 m = m->m_next;
1672 }
1673 while (len > 0) {
1674 if (m == 0)
1675 panic("m_copydata");
1676 count = min(m->m_len - off, len);
1677 bcopy(mtod(m, caddr_t) + off, cp, count);
1678 len -= count;
1679 cp += count;
1680 off = 0;
1681 m = m->m_next;
1682 }
1683 }
1684
1685 /*
1686 * Concatenate mbuf chain n to m.
1687 * Both chains must be of the same type (e.g. MT_DATA).
1688 * Any m_pkthdr is not updated.
1689 */
1690 void m_cat(
1691 struct mbuf *m, struct mbuf *n)
1692 {
1693 while (m->m_next)
1694 m = m->m_next;
1695 while (n) {
1696 if (m->m_flags & M_EXT ||
1697 m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) {
1698 /* just join the two chains */
1699 m->m_next = n;
1700 return;
1701 }
1702 /* splat the data from one into the other */
1703 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
1704 (u_int)n->m_len);
1705 m->m_len += n->m_len;
1706 n = m_free(n);
1707 }
1708 }
1709
1710 void
1711 m_adj(
1712 struct mbuf *mp,
1713 int req_len)
1714 {
1715 int len = req_len;
1716 struct mbuf *m;
1717 int count;
1718
1719 if ((m = mp) == NULL)
1720 return;
1721 if (len >= 0) {
1722 /*
1723 * Trim from head.
1724 */
1725 while (m != NULL && len > 0) {
1726 if (m->m_len <= len) {
1727 len -= m->m_len;
1728 m->m_len = 0;
1729 m = m->m_next;
1730 } else {
1731 m->m_len -= len;
1732 m->m_data += len;
1733 len = 0;
1734 }
1735 }
1736 m = mp;
1737 if (m->m_flags & M_PKTHDR)
1738 m->m_pkthdr.len -= (req_len - len);
1739 } else {
1740 /*
1741 * Trim from tail. Scan the mbuf chain,
1742 * calculating its length and finding the last mbuf.
1743 * If the adjustment only affects this mbuf, then just
1744 * adjust and return. Otherwise, rescan and truncate
1745 * after the remaining size.
1746 */
1747 len = -len;
1748 count = 0;
1749 for (;;) {
1750 count += m->m_len;
1751 if (m->m_next == (struct mbuf *)0)
1752 break;
1753 m = m->m_next;
1754 }
1755 if (m->m_len >= len) {
1756 m->m_len -= len;
1757 m = mp;
1758 if (m->m_flags & M_PKTHDR)
1759 m->m_pkthdr.len -= len;
1760 return;
1761 }
1762 count -= len;
1763 if (count < 0)
1764 count = 0;
1765 /*
1766 * Correct length for chain is "count".
1767 * Find the mbuf with last data, adjust its length,
1768 * and toss data from remaining mbufs on chain.
1769 */
1770 m = mp;
1771 if (m->m_flags & M_PKTHDR)
1772 m->m_pkthdr.len = count;
1773 for (; m; m = m->m_next) {
1774 if (m->m_len >= count) {
1775 m->m_len = count;
1776 break;
1777 }
1778 count -= m->m_len;
1779 }
1780 while ((m = m->m_next))
1781 m->m_len = 0;
1782 }
1783 }
1784
1785 /*
1786 * Rearange an mbuf chain so that len bytes are contiguous
1787 * and in the data area of an mbuf (so that mtod and dtom
1788 * will work for a structure of size len). Returns the resulting
1789 * mbuf chain on success, frees it and returns null on failure.
1790 * If there is room, it will add up to max_protohdr-len extra bytes to the
1791 * contiguous region in an attempt to avoid being called next time.
1792 */
1793 int MPFail;
1794
1795 struct mbuf *
1796 m_pullup(
1797 struct mbuf *n,
1798 int len)
1799 {
1800 struct mbuf *m;
1801 int count;
1802 int space;
1803
1804 /*
1805 * If first mbuf has no cluster, and has room for len bytes
1806 * without shifting current data, pullup into it,
1807 * otherwise allocate a new mbuf to prepend to the chain.
1808 */
1809 if ((n->m_flags & M_EXT) == 0 &&
1810 n->m_data + len < &n->m_dat[MLEN] && n->m_next) {
1811 if (n->m_len >= len)
1812 return (n);
1813 m = n;
1814 n = n->m_next;
1815 len -= m->m_len;
1816 } else {
1817 if (len > MHLEN)
1818 goto bad;
1819 MGET(m, M_DONTWAIT, n->m_type);
1820 if (m == 0)
1821 goto bad;
1822 m->m_len = 0;
1823 if (n->m_flags & M_PKTHDR) {
1824 M_COPY_PKTHDR(m, n);
1825 n->m_flags &= ~M_PKTHDR;
1826 }
1827 }
1828 space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
1829 do {
1830 count = min(min(max(len, max_protohdr), space), n->m_len);
1831 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
1832 (unsigned)count);
1833 len -= count;
1834 m->m_len += count;
1835 n->m_len -= count;
1836 space -= count;
1837 if (n->m_len)
1838 n->m_data += count;
1839 else
1840 n = m_free(n);
1841 } while (len > 0 && n);
1842 if (len > 0) {
1843 (void) m_free(m);
1844 goto bad;
1845 }
1846 m->m_next = n;
1847 return (m);
1848 bad:
1849 m_freem(n);
1850 MPFail++;
1851 return (0);
1852 }
1853
1854 /*
1855 * Partition an mbuf chain in two pieces, returning the tail --
1856 * all but the first len0 bytes. In case of failure, it returns NULL and
1857 * attempts to restore the chain to its original state.
1858 */
1859 struct mbuf *
1860 m_split(
1861 struct mbuf *m0,
1862 int len0,
1863 int wait)
1864 {
1865 struct mbuf *m, *n;
1866 unsigned len = len0, remain;
1867
1868 for (m = m0; m && len > m->m_len; m = m->m_next)
1869 len -= m->m_len;
1870 if (m == 0)
1871 return (0);
1872 remain = m->m_len - len;
1873 if (m0->m_flags & M_PKTHDR) {
1874 MGETHDR(n, wait, m0->m_type);
1875 if (n == 0)
1876 return (0);
1877 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
1878 n->m_pkthdr.len = m0->m_pkthdr.len - len0;
1879 m0->m_pkthdr.len = len0;
1880 if (m->m_flags & M_EXT)
1881 goto extpacket;
1882 if (remain > MHLEN) {
1883 /* m can't be the lead packet */
1884 MH_ALIGN(n, 0);
1885 n->m_next = m_split(m, len, wait);
1886 if (n->m_next == 0) {
1887 (void) m_free(n);
1888 return (0);
1889 } else
1890 return (n);
1891 } else
1892 MH_ALIGN(n, remain);
1893 } else if (remain == 0) {
1894 n = m->m_next;
1895 m->m_next = 0;
1896 return (n);
1897 } else {
1898 MGET(n, wait, m->m_type);
1899 if (n == 0)
1900 return (0);
1901 M_ALIGN(n, remain);
1902 }
1903 extpacket:
1904 if (m->m_flags & M_EXT) {
1905 n->m_flags |= M_EXT;
1906 MBUF_LOCK();
1907 n->m_ext = m->m_ext;
1908 insque((queue_t)&n->m_ext.ext_refs, (queue_t)&m->m_ext.ext_refs);
1909 MBUF_UNLOCK();
1910 n->m_data = m->m_data + len;
1911 } else {
1912 bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain);
1913 }
1914 n->m_len = remain;
1915 m->m_len = len;
1916 n->m_next = m->m_next;
1917 m->m_next = 0;
1918 return (n);
1919 }
1920 /*
1921 * Routine to copy from device local memory into mbufs.
1922 */
1923 struct mbuf *
1924 m_devget(
1925 char *buf,
1926 int totlen,
1927 int off0,
1928 struct ifnet *ifp,
1929 void (*copy)(const void *, void *, size_t))
1930 {
1931 struct mbuf *m;
1932 struct mbuf *top = 0, **mp = &top;
1933 int off = off0, len;
1934 char *cp;
1935 char *epkt;
1936
1937 cp = buf;
1938 epkt = cp + totlen;
1939 if (off) {
1940 /*
1941 * If 'off' is non-zero, packet is trailer-encapsulated,
1942 * so we have to skip the type and length fields.
1943 */
1944 cp += off + 2 * sizeof(u_int16_t);
1945 totlen -= 2 * sizeof(u_int16_t);
1946 }
1947 MGETHDR(m, M_DONTWAIT, MT_DATA);
1948 if (m == 0)
1949 return (0);
1950 m->m_pkthdr.rcvif = ifp;
1951 m->m_pkthdr.len = totlen;
1952 m->m_len = MHLEN;
1953
1954 while (totlen > 0) {
1955 if (top) {
1956 MGET(m, M_DONTWAIT, MT_DATA);
1957 if (m == 0) {
1958 m_freem(top);
1959 return (0);
1960 }
1961 m->m_len = MLEN;
1962 }
1963 len = min(totlen, epkt - cp);
1964 if (len >= MINCLSIZE) {
1965 MCLGET(m, M_DONTWAIT);
1966 if (m->m_flags & M_EXT)
1967 m->m_len = len = min(len, MCLBYTES);
1968 else {
1969 /* give up when it's out of cluster mbufs */
1970 if (top)
1971 m_freem(top);
1972 m_freem(m);
1973 return (0);
1974 }
1975 } else {
1976 /*
1977 * Place initial small packet/header at end of mbuf.
1978 */
1979 if (len < m->m_len) {
1980 if (top == 0 && len + max_linkhdr <= m->m_len)
1981 m->m_data += max_linkhdr;
1982 m->m_len = len;
1983 } else
1984 len = m->m_len;
1985 }
1986 if (copy)
1987 copy(cp, mtod(m, caddr_t), (unsigned)len);
1988 else
1989 bcopy(cp, mtod(m, caddr_t), (unsigned)len);
1990 cp += len;
1991 *mp = m;
1992 mp = &m->m_next;
1993 totlen -= len;
1994 if (cp == epkt)
1995 cp = buf;
1996 }
1997 return (top);
1998 }
1999
2000 /*
2001 * Cluster freelist allocation check. The mbuf lock must be held.
2002 * Ensure hysteresis between hi/lo.
2003 */
2004 static int
2005 m_howmany(int num, size_t bufsize)
2006 {
2007 int i = 0;
2008
2009 /* Bail if we've maxed out the mbuf memory map */
2010 if (mbstat.m_clusters + (mbstat.m_bigclusters << 1) < nmbclusters) {
2011 int j = 0;
2012
2013 if (bufsize == MCLBYTES) {
2014 /* Under minimum */
2015 if (mbstat.m_clusters < MINCL)
2016 return (MINCL - mbstat.m_clusters);
2017 /* Too few (free < 1/2 total) and not over maximum */
2018 if (mbstat.m_clusters < (nmbclusters >> 1)) {
2019 if (num >= mbstat.m_clfree)
2020 i = num - mbstat.m_clfree;
2021 if (((mbstat.m_clusters + num) >> 1) > mbstat.m_clfree)
2022 j = ((mbstat.m_clusters + num) >> 1) - mbstat.m_clfree;
2023 i = max(i, j);
2024 if (i + mbstat.m_clusters >= (nmbclusters >> 1))
2025 i = (nmbclusters >> 1) - mbstat.m_clusters;
2026 }
2027 } else {
2028 /* Under minimum */
2029 if (mbstat.m_bigclusters < MINCL)
2030 return (MINCL - mbstat.m_bigclusters);
2031 /* Too few (free < 1/2 total) and not over maximum */
2032 if (mbstat.m_bigclusters < (nmbclusters >> 2)) {
2033 if (num >= mbstat.m_bigclfree)
2034 i = num - mbstat.m_bigclfree;
2035 if (((mbstat.m_bigclusters + num) >> 1) > mbstat.m_bigclfree)
2036 j = ((mbstat.m_bigclusters + num) >> 1) - mbstat.m_bigclfree;
2037 i = max(i, j);
2038 if (i + mbstat.m_bigclusters >= (nmbclusters >> 2))
2039 i = (nmbclusters >> 2) - mbstat.m_bigclusters;
2040 }
2041 }
2042 }
2043 return i;
2044 }
2045
2046 /*
2047 * Copy data from a buffer back into the indicated mbuf chain,
2048 * starting "off" bytes from the beginning, extending the mbuf
2049 * chain if necessary.
2050 */
2051 void
2052 m_copyback(
2053 struct mbuf *m0,
2054 int off,
2055 int len,
2056 caddr_t cp)
2057 {
2058 int mlen;
2059 struct mbuf *m = m0, *n;
2060 int totlen = 0;
2061
2062 if (m0 == 0)
2063 return;
2064 while (off > (mlen = m->m_len)) {
2065 off -= mlen;
2066 totlen += mlen;
2067 if (m->m_next == 0) {
2068 n = m_getclr(M_DONTWAIT, m->m_type);
2069 if (n == 0)
2070 goto out;
2071 n->m_len = min(MLEN, len + off);
2072 m->m_next = n;
2073 }
2074 m = m->m_next;
2075 }
2076 while (len > 0) {
2077 mlen = min (m->m_len - off, len);
2078 bcopy(cp, off + mtod(m, caddr_t), (unsigned)mlen);
2079 cp += mlen;
2080 len -= mlen;
2081 mlen += off;
2082 off = 0;
2083 totlen += mlen;
2084 if (len == 0)
2085 break;
2086 if (m->m_next == 0) {
2087 n = m_get(M_DONTWAIT, m->m_type);
2088 if (n == 0)
2089 break;
2090 n->m_len = min(MLEN, len);
2091 m->m_next = n;
2092 }
2093 m = m->m_next;
2094 }
2095 out: if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen))
2096 m->m_pkthdr.len = totlen;
2097 }
2098
2099
2100 char *mcl_to_paddr(char *addr) {
2101 int base_phys;
2102
2103 if (addr < (char *)mbutl || addr >= (char *)embutl)
2104 return (0);
2105 base_phys = mcl_paddr[(addr - (char *)mbutl) >> PGSHIFT];
2106
2107 if (base_phys == 0)
2108 return (0);
2109 return ((char *)((int)base_phys | ((int)addr & PGOFSET)));
2110 }
2111
2112 /*
2113 * Dup the mbuf chain passed in. The whole thing. No cute additional cruft.
2114 * And really copy the thing. That way, we don't "precompute" checksums
2115 * for unsuspecting consumers.
2116 * Assumption: m->m_nextpkt == 0.
2117 * Trick: for small packets, don't dup into a cluster. That way received
2118 * packets don't take up too much room in the sockbuf (cf. sbspace()).
2119 */
2120 int MDFail;
2121
2122 struct mbuf *
2123 m_dup(struct mbuf *m, int how)
2124 {
2125 struct mbuf *n, **np;
2126 struct mbuf *top;
2127 int copyhdr = 0;
2128
2129 np = &top;
2130 top = 0;
2131 if (m->m_flags & M_PKTHDR)
2132 copyhdr = 1;
2133
2134 /*
2135 * Quick check: if we have one mbuf and its data fits in an
2136 * mbuf with packet header, just copy and go.
2137 */
2138 if (m->m_next == NULL)
2139 { /* Then just move the data into an mbuf and be done... */
2140 if (copyhdr)
2141 { if (m->m_pkthdr.len <= MHLEN)
2142 { if ((n = m_gethdr(how, m->m_type)) == NULL)
2143 return(NULL);
2144 n->m_len = m->m_len;
2145 m_dup_pkthdr(n, m, how);
2146 bcopy(m->m_data, n->m_data, m->m_len);
2147 return(n);
2148 }
2149 } else if (m->m_len <= MLEN)
2150 { if ((n = m_get(how, m->m_type)) == NULL)
2151 return(NULL);
2152 bcopy(m->m_data, n->m_data, m->m_len);
2153 n->m_len = m->m_len;
2154 return(n);
2155 }
2156 }
2157 while (m)
2158 {
2159 #if BLUE_DEBUG
2160 kprintf("<%x: %x, %x, %x\n", m, m->m_flags, m->m_len,
2161 m->m_data);
2162 #endif
2163 if (copyhdr)
2164 n = m_gethdr(how, m->m_type);
2165 else
2166 n = m_get(how, m->m_type);
2167 if (n == 0)
2168 goto nospace;
2169 if (m->m_flags & M_EXT)
2170 { MCLGET(n, how);
2171 if ((n->m_flags & M_EXT) == 0)
2172 goto nospace;
2173 }
2174 *np = n;
2175 if (copyhdr)
2176 { /* Don't use M_COPY_PKTHDR: preserve m_data */
2177 m_dup_pkthdr(n, m, how);
2178 copyhdr = 0;
2179 if ((n->m_flags & M_EXT) == 0)
2180 n->m_data = n->m_pktdat;
2181 }
2182 n->m_len = m->m_len;
2183 /*
2184 * Get the dup on the same bdry as the original
2185 * Assume that the two mbufs have the same offset to data area
2186 * (up to word bdries)
2187 */
2188 bcopy(mtod(m, caddr_t), mtod(n, caddr_t), (unsigned)n->m_len);
2189 m = m->m_next;
2190 np = &n->m_next;
2191 #if BLUE_DEBUG
2192 kprintf(">%x: %x, %x, %x\n", n, n->m_flags, n->m_len,
2193 n->m_data);
2194 #endif
2195 }
2196
2197 if (top == 0)
2198 MDFail++;
2199 return (top);
2200 nospace:
2201 m_freem(top);
2202 MDFail++;
2203 return (0);
2204 }
2205
2206 int
2207 m_mclref(struct mbuf *p)
2208 {
2209 return (_MCLREF(p));
2210 }
2211
2212 int
2213 m_mclunref(struct mbuf *p)
2214 {
2215 return (_MCLUNREF(p));
2216 }
2217
2218 /* change mbuf to new type */
2219 void
2220 m_mchtype(struct mbuf *m, int t)
2221 {
2222 MBUF_LOCK();
2223 mbstat.m_mtypes[(m)->m_type]--;
2224 mbstat.m_mtypes[t]++;
2225 (m)->m_type = t;
2226 MBUF_UNLOCK();
2227 }
2228
2229 void *m_mtod(struct mbuf *m)
2230 {
2231 return ((m)->m_data);
2232 }
2233
2234 struct mbuf *m_dtom(void *x)
2235 {
2236 return ((struct mbuf *)((u_long)(x) & ~(MSIZE-1)));
2237 }
2238
2239 int m_mtocl(void *x)
2240 {
2241 return (((char *)(x) - (char *)mbutl) / sizeof(union mcluster));
2242 }
2243
2244 union mcluster *m_cltom(int x)
2245 {
2246 return ((union mcluster *)(mbutl + (x)));
2247 }
2248
2249
2250 void m_mcheck(struct mbuf *m)
2251 {
2252 if (m->m_type != MT_FREE)
2253 panic("mget MCHECK: m_type=%x m=%x", m->m_type, m);
2254 }
2255
2256 static void
2257 mbuf_expand_thread(void)
2258 {
2259 while (1) {
2260 MBUF_LOCK();
2261 if (mbuf_expand_mcl) {
2262 int n;
2263
2264 /* Adjust to the current number of cluster in use */
2265 n = mbuf_expand_mcl - (mbstat.m_clusters - mbstat.m_clfree);
2266 mbuf_expand_mcl = 0;
2267
2268 if (n > 0)
2269 (void)m_clalloc(n, M_WAIT, MCLBYTES, 1);
2270 }
2271 if (mbuf_expand_big) {
2272 int n;
2273
2274 /* Adjust to the current number of 4 KB cluster in use */
2275 n = mbuf_expand_big - (mbstat.m_bigclusters - mbstat.m_bigclfree);
2276 mbuf_expand_big = 0;
2277
2278 if (n > 0)
2279 (void)m_clalloc(n, M_WAIT, NBPG, 1);
2280 }
2281 MBUF_UNLOCK();
2282 /*
2283 * Because we can run out of memory before filling the mbuf map, we
2284 * should not allocate more clusters than they are mbufs -- otherwise
2285 * we could have a large number of useless clusters allocated.
2286 */
2287 while (mbstat.m_mbufs < mbstat.m_bigclusters + mbstat.m_clusters) {
2288 if (m_expand(M_WAIT) == 0)
2289 break;
2290 }
2291
2292 assert_wait(&mbuf_expand_thread_wakeup, THREAD_UNINT);
2293 (void) thread_block((thread_continue_t)mbuf_expand_thread);
2294 }
2295 }
2296
2297 static void
2298 mbuf_expand_thread_init(void)
2299 {
2300 mbuf_expand_thread_initialized++;
2301 mbuf_expand_thread();
2302 }
2303
2304 SYSCTL_DECL(_kern_ipc);
2305 SYSCTL_STRUCT(_kern_ipc, KIPC_MBSTAT, mbstat, CTLFLAG_RW, &mbstat, mbstat, "");
2306
mirror of XNU