projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-792.6.76.tar.gz
[apple/xnu.git] / bsd / kern / uipc_mbuf.c
1 /*
2 * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * The contents of this file constitute Original Code as defined in and
7 * are subject to the Apple Public Source License Version 1.1 (the
8 * "License"). You may not use this file except in compliance with the
9 * License. Please obtain a copy of the License at
10 * http://www.apple.com/publicsource and read it before using this file.
11 *
12 * This Original Code and all software distributed under the License are
13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17 * License for the specific language governing rights and limitations
18 * under the License.
19 *
20 * @APPLE_LICENSE_HEADER_END@
21 */
22 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
23 /*
24 * Copyright (c) 1982, 1986, 1988, 1991, 1993
25 * The Regents of the University of California. All rights reserved.
26 *
27 * Redistribution and use in source and binary forms, with or without
28 * modification, are permitted provided that the following conditions
29 * are met:
30 * 1. Redistributions of source code must retain the above copyright
31 * notice, this list of conditions and the following disclaimer.
32 * 2. Redistributions in binary form must reproduce the above copyright
33 * notice, this list of conditions and the following disclaimer in the
34 * documentation and/or other materials provided with the distribution.
35 * 3. All advertising materials mentioning features or use of this software
36 * must display the following acknowledgement:
37 * This product includes software developed by the University of
38 * California, Berkeley and its contributors.
39 * 4. Neither the name of the University nor the names of its contributors
40 * may be used to endorse or promote products derived from this software
41 * without specific prior written permission.
42 *
43 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
44 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
45 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
46 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
47 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
48 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
49 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
50 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
51 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
52 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
53 * SUCH DAMAGE.
54 *
55 * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94
56 */
57 /* HISTORY
58 *
59 * 10/15/97 Annette DeSchon (deschon@apple.com)
60 * Fixed bug in which all cluster mbufs were broken up
61 * into regular mbufs: Some clusters are now reserved.
62 * When a cluster is needed, regular mbufs are no longer
63 * used. (Radar 1683621)
64 * 20-May-95 Mac Gillon (mgillon) at NeXT
65 * New version based on 4.4
66 */
67
68 #include <sys/param.h>
69 #include <sys/systm.h>
70 #include <sys/malloc.h>
71 #include <sys/mbuf.h>
72 #include <sys/kernel.h>
73 #include <sys/sysctl.h>
74 #include <sys/syslog.h>
75 #include <sys/protosw.h>
76 #include <sys/domain.h>
77
78 #include <kern/queue.h>
79 #include <kern/kern_types.h>
80 #include <kern/sched_prim.h>
81
82 #include <IOKit/IOMapper.h>
83
84 extern vm_offset_t kmem_mb_alloc(vm_map_t , int );
85 extern boolean_t PE_parse_boot_arg(const char *, void *);
86
87 #define _MCLREF(p) (++mclrefcnt[mtocl(p)])
88 #define _MCLUNREF(p) (--mclrefcnt[mtocl(p)] == 0)
89 #define _M_CLEAR_PKTHDR(mbuf_ptr) (mbuf_ptr)->m_pkthdr.rcvif = NULL; \
90 (mbuf_ptr)->m_pkthdr.len = 0; \
91 (mbuf_ptr)->m_pkthdr.header = NULL; \
92 (mbuf_ptr)->m_pkthdr.csum_flags = 0; \
93 (mbuf_ptr)->m_pkthdr.csum_data = 0; \
94 (mbuf_ptr)->m_pkthdr.aux = (struct mbuf*)NULL; \
95 (mbuf_ptr)->m_pkthdr.vlan_tag = 0; \
96 (mbuf_ptr)->m_pkthdr.socket_id = 0; \
97 SLIST_INIT(&(mbuf_ptr)->m_pkthdr.tags);
98
99 /* kernel translater */
100 extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
101
102 lck_mtx_t * mbuf_mlock;
103 lck_grp_t * mbuf_mlock_grp;
104 lck_grp_attr_t * mbuf_mlock_grp_attr;
105 lck_attr_t * mbuf_mlock_attr;
106 extern lck_mtx_t *domain_proto_mtx;
107
108 struct mbuf *mfree; /* mbuf free list */
109 struct mbuf *mfreelater; /* mbuf deallocation list */
110 extern vm_map_t mb_map; /* special map */
111 int m_want; /* sleepers on mbufs */
112 short *mclrefcnt; /* mapped cluster reference counts */
113 int *mcl_paddr;
114 static ppnum_t mcl_paddr_base; /* Handle returned by IOMapper::iovmAlloc() */
115 union mcluster *mclfree; /* mapped cluster free list */
116 union mbigcluster *mbigfree; /* mapped cluster free list */
117 int max_linkhdr; /* largest link-level header */
118 int max_protohdr; /* largest protocol header */
119 int max_hdr; /* largest link+protocol header */
120 int max_datalen; /* MHLEN - max_hdr */
121 struct mbstat mbstat; /* statistics */
122 union mcluster *mbutl; /* first mapped cluster address */
123 union mcluster *embutl; /* ending virtual address of mclusters */
124
125 static int nclpp; /* # clusters per physical page */
126
127 static int m_howmany(int, size_t );
128 void m_reclaim(void);
129 static int m_clalloc(const int , const int, const size_t, int);
130 int do_reclaim = 0;
131
132 #define MF_NOWAIT 0x1
133 #define MF_BIG 0x2
134
135 /* The number of cluster mbufs that are allocated, to start. */
136 #define MINCL max(16, 2)
137
138 static int mbuf_expand_thread_wakeup = 0;
139 static int mbuf_expand_mcl = 0;
140 static int mbuf_expand_big = 0;
141 static int mbuf_expand_thread_initialized = 0;
142
143 static void mbuf_expand_thread_init(void);
144 static void mbuf_expand_thread(void);
145 static int m_expand(int );
146 static caddr_t m_bigalloc(int );
147 static void m_bigfree(caddr_t , u_int , caddr_t );
148 static struct mbuf * m_mbigget(struct mbuf *, int );
149 void mbinit(void);
150 static void m_range_check(void *addr);
151
152
153 #if 0
154 static int mfree_munge = 0;
155 #if 0
156 #define _MFREE_MUNGE(m) { \
157 if (mfree_munge) \
158 { int i; \
159 vm_offset_t *element = (vm_offset_t *)(m); \
160 for (i = 0; \
161 i < sizeof(struct mbuf)/sizeof(vm_offset_t); \
162 i++) \
163 (element)[i] = 0xdeadbeef; \
164 } \
165 }
166 #else
167 void
168 munge_mbuf(struct mbuf *m)
169 {
170 int i;
171 vm_offset_t *element = (vm_offset_t *)(m);
172 for (i = 0;
173 i < sizeof(struct mbuf)/sizeof(vm_offset_t);
174 i++)
175 (element)[i] = 0xdeadbeef;
176 }
177 #define _MFREE_MUNGE(m) { \
178 if (mfree_munge) \
179 munge_mbuf(m); \
180 }
181 #endif
182 #else
183 #define _MFREE_MUNGE(m)
184 #endif
185
186
187 #define _MINTGET(m, type) { \
188 MBUF_LOCK(); \
189 if (((m) = mfree) != 0) { \
190 MCHECK(m); \
191 ++mclrefcnt[mtocl(m)]; \
192 mbstat.m_mtypes[MT_FREE]--; \
193 mbstat.m_mtypes[(type)]++; \
194 mfree = (m)->m_next; \
195 } \
196 MBUF_UNLOCK(); \
197 }
198
199
200 static void
201 m_range_check(void *addr)
202 {
203 if (addr && (addr < (void *)mbutl || addr >= (void *)embutl))
204 panic("mbuf address out of range 0x%x", addr);
205 }
206
207 __private_extern__ void
208 mbinit(void)
209 {
210 int m;
211 int initmcl = 32;
212 int mcl_pages;
213
214 if (nclpp)
215 return;
216 nclpp = round_page_32(MCLBYTES) / MCLBYTES; /* see mbufgc() */
217 if (nclpp < 1) nclpp = 1;
218 mbuf_mlock_grp_attr = lck_grp_attr_alloc_init();
219 lck_grp_attr_setdefault(mbuf_mlock_grp_attr);
220
221 mbuf_mlock_grp = lck_grp_alloc_init("mbuf", mbuf_mlock_grp_attr);
222 mbuf_mlock_attr = lck_attr_alloc_init();
223 lck_attr_setdefault(mbuf_mlock_attr);
224
225 mbuf_mlock = lck_mtx_alloc_init(mbuf_mlock_grp, mbuf_mlock_attr);
226
227 mbstat.m_msize = MSIZE;
228 mbstat.m_mclbytes = MCLBYTES;
229 mbstat.m_minclsize = MINCLSIZE;
230 mbstat.m_mlen = MLEN;
231 mbstat.m_mhlen = MHLEN;
232 mbstat.m_bigmclbytes = NBPG;
233
234 if (nmbclusters == 0)
235 nmbclusters = NMBCLUSTERS;
236 MALLOC(mclrefcnt, short *, nmbclusters * sizeof (short),
237 M_TEMP, M_WAITOK);
238 if (mclrefcnt == 0)
239 panic("mbinit");
240 for (m = 0; m < nmbclusters; m++)
241 mclrefcnt[m] = -1;
242
243 /* Calculate the number of pages assigned to the cluster pool */
244 mcl_pages = nmbclusters/(NBPG/CLBYTES);
245 MALLOC(mcl_paddr, int *, mcl_pages * sizeof(int), M_TEMP, M_WAITOK);
246 if (mcl_paddr == 0)
247 panic("mbinit1");
248 /* Register with the I/O Bus mapper */
249 mcl_paddr_base = IOMapperIOVMAlloc(mcl_pages);
250 bzero((char *)mcl_paddr, mcl_pages * sizeof(int));
251
252 embutl = (union mcluster *)((unsigned char *)mbutl + (nmbclusters * MCLBYTES));
253
254 PE_parse_boot_arg("initmcl", &initmcl);
255
256 if (m_clalloc(max(NBPG/CLBYTES, 1) * initmcl, M_WAIT, MCLBYTES, 0) == 0)
257 goto bad;
258 MBUF_UNLOCK();
259
260 (void) kernel_thread(kernel_task, mbuf_expand_thread_init);
261
262 return;
263 bad:
264 panic("mbinit");
265 }
266
267 /*
268 * Allocate some number of mbuf clusters
269 * and place on cluster free list.
270 * Take the mbuf lock (if not already locked) and do not release it
271 */
272 /* ARGSUSED */
273 static int
274 m_clalloc(
275 const int num,
276 const int nowait,
277 const size_t bufsize,
278 int locked)
279 {
280 int i;
281 vm_size_t size = 0;
282 int numpages = 0;
283 vm_offset_t page = 0;
284
285 if (locked == 0)
286 MBUF_LOCK();
287 /*
288 * Honor the caller's wish to block or not block.
289 * We have a way to grow the pool asynchronously,
290 * by kicking the dlil_input_thread.
291 */
292 i = m_howmany(num, bufsize);
293 if (i == 0 || nowait == M_DONTWAIT)
294 goto out;
295
296 MBUF_UNLOCK();
297 size = round_page_32(i * bufsize);
298 page = kmem_mb_alloc(mb_map, size);
299
300 if (page == 0) {
301 size = NBPG; /* Try for 1 if failed */
302 page = kmem_mb_alloc(mb_map, size);
303 }
304 MBUF_LOCK();
305
306 if (page) {
307 numpages = size / NBPG;
308 for (i = 0; i < numpages; i++, page += NBPG) {
309 if (((int)page & PGOFSET) == 0) {
310 ppnum_t offset = ((char *)page - (char *)mbutl)/NBPG;
311 ppnum_t new_page = pmap_find_phys(kernel_pmap, (vm_address_t) page);
312
313 /*
314 * In the case of no mapper being available
315 * the following code nops and returns the
316 * input page, if there is a mapper the I/O
317 * page appropriate is returned.
318 */
319 new_page = IOMapperInsertPage(mcl_paddr_base, offset, new_page);
320 mcl_paddr[offset] = new_page << 12;
321 }
322 if (bufsize == MCLBYTES) {
323 union mcluster *mcl = (union mcluster *)page;
324
325 if (++mclrefcnt[mtocl(mcl)] != 0)
326 panic("m_clalloc already there");
327 mcl->mcl_next = mclfree;
328 mclfree = mcl++;
329 if (++mclrefcnt[mtocl(mcl)] != 0)
330 panic("m_clalloc already there");
331 mcl->mcl_next = mclfree;
332 mclfree = mcl++;
333 } else {
334 union mbigcluster *mbc = (union mbigcluster *)page;
335
336 if (++mclrefcnt[mtocl(mbc)] != 0)
337 panic("m_clalloc already there");
338 if (++mclrefcnt[mtocl(mbc) + 1] != 0)
339 panic("m_clalloc already there");
340
341 mbc->mbc_next = mbigfree;
342 mbigfree = mbc;
343 }
344 }
345 if (bufsize == MCLBYTES) {
346 int numcl = numpages << 1;
347 mbstat.m_clfree += numcl;
348 mbstat.m_clusters += numcl;
349 return (numcl);
350 } else {
351 mbstat.m_bigclfree += numpages;
352 mbstat.m_bigclusters += numpages;
353 return (numpages);
354 }
355 } /* else ... */
356 out:
357 /*
358 * When non-blocking we kick a thread if we havve to grow the
359 * pool or if the number of free clusters is less than requested.
360 */
361 if (bufsize == MCLBYTES) {
362 if (i > 0) {
363 /* Remember total number of clusters needed at this time */
364 i += mbstat.m_clusters;
365 if (i > mbuf_expand_mcl) {
366 mbuf_expand_mcl = i;
367 if (mbuf_expand_thread_initialized)
368 wakeup((caddr_t)&mbuf_expand_thread_wakeup);
369 }
370 }
371
372 if (mbstat.m_clfree >= num)
373 return 1;
374 } else {
375 if (i > 0) {
376 /* Remember total number of 4KB clusters needed at this time */
377 i += mbstat.m_bigclusters;
378 if (i > mbuf_expand_big) {
379 mbuf_expand_big = i;
380 if (mbuf_expand_thread_initialized)
381 wakeup((caddr_t)&mbuf_expand_thread_wakeup);
382 }
383 }
384
385 if (mbstat.m_bigclfree >= num)
386 return 1;
387 }
388 return 0;
389 }
390
391 /*
392 * Add more free mbufs by cutting up a cluster.
393 */
394 static int
395 m_expand(int canwait)
396 {
397 caddr_t mcl;
398
399 if (mbstat.m_clfree < (mbstat.m_clusters >> 4)) {
400 /*
401 * 1/16th of the total number of cluster mbufs allocated is
402 * reserved for large packets. The number reserved must
403 * always be < 1/2, or future allocation will be prevented.
404 */
405 (void)m_clalloc(1, canwait, MCLBYTES, 0);
406 MBUF_UNLOCK();
407 if (mbstat.m_clfree < (mbstat.m_clusters >> 4))
408 return 0;
409 }
410
411 MCLALLOC(mcl, canwait);
412 if (mcl) {
413 struct mbuf *m = (struct mbuf *)mcl;
414 int i = NMBPCL;
415 MBUF_LOCK();
416 mbstat.m_mtypes[MT_FREE] += i;
417 mbstat.m_mbufs += i;
418 while (i--) {
419 _MFREE_MUNGE(m);
420 m->m_type = MT_FREE;
421 m->m_next = mfree;
422 mfree = m++;
423 }
424 i = m_want;
425 m_want = 0;
426 MBUF_UNLOCK();
427 if (i) wakeup((caddr_t)&mfree);
428 return 1;
429 }
430 return 0;
431 }
432
433 /*
434 * When MGET failes, ask protocols to free space when short of memory,
435 * then re-attempt to allocate an mbuf.
436 */
437 struct mbuf *
438 m_retry(
439 int canwait,
440 int type)
441 {
442 struct mbuf *m;
443 int wait;
444
445 for (;;) {
446 (void) m_expand(canwait);
447 _MINTGET(m, type);
448 if (m) {
449 (m)->m_next = (m)->m_nextpkt = 0;
450 (m)->m_type = (type);
451 (m)->m_data = (m)->m_dat;
452 (m)->m_flags = 0;
453 (m)->m_len = 0;
454 }
455 if (m || canwait == M_DONTWAIT)
456 break;
457 MBUF_LOCK();
458 wait = m_want++;
459 mbuf_expand_mcl++;
460 if (wait == 0)
461 mbstat.m_drain++;
462 else
463 mbstat.m_wait++;
464 MBUF_UNLOCK();
465
466 if (mbuf_expand_thread_initialized)
467 wakeup((caddr_t)&mbuf_expand_thread_wakeup);
468
469 if (wait == 0) {
470 m_reclaim();
471 } else {
472 struct timespec ts;
473 ts.tv_sec = 1;
474 ts.tv_nsec = 0;
475 (void) msleep((caddr_t)&mfree, 0, (PZERO-1) | PDROP, "m_retry", &ts);
476 }
477 }
478 if (m == 0)
479 mbstat.m_drops++;
480 return (m);
481 }
482
483 /*
484 * As above; retry an MGETHDR.
485 */
486 struct mbuf *
487 m_retryhdr(
488 int canwait,
489 int type)
490 {
491 struct mbuf *m;
492
493 if ((m = m_retry(canwait, type))) {
494 m->m_next = m->m_nextpkt = 0;
495 m->m_flags |= M_PKTHDR;
496 m->m_data = m->m_pktdat;
497 _M_CLEAR_PKTHDR(m);
498 }
499 return (m);
500 }
501
502 void
503 m_reclaim(void)
504 {
505 do_reclaim = 1; /* drain is performed in pfslowtimo(), to avoid deadlocks */
506 mbstat.m_drain++;
507 }
508
509 /*
510 * Space allocation routines.
511 * These are also available as macros
512 * for critical paths.
513 */
514 struct mbuf *
515 m_get(
516 int nowait,
517 int type)
518 {
519 struct mbuf *m;
520
521 m_range_check(mfree);
522 m_range_check(mclfree);
523 m_range_check(mbigfree);
524
525 _MINTGET(m, type);
526 if (m) {
527 m->m_next = m->m_nextpkt = 0;
528 m->m_type = type;
529 m->m_data = m->m_dat;
530 m->m_flags = 0;
531 m->m_len = 0;
532 } else
533 (m) = m_retry(nowait, type);
534
535 m_range_check(mfree);
536 m_range_check(mclfree);
537 m_range_check(mbigfree);
538
539
540 return (m);
541 }
542
543 struct mbuf *
544 m_gethdr(
545 int nowait,
546 int type)
547 {
548 struct mbuf *m;
549
550 m_range_check(mfree);
551 m_range_check(mclfree);
552 m_range_check(mbigfree);
553
554
555 _MINTGET(m, type);
556 if (m) {
557 m->m_next = m->m_nextpkt = 0;
558 m->m_type = type;
559 m->m_data = m->m_pktdat;
560 m->m_flags = M_PKTHDR;
561 m->m_len = 0;
562 _M_CLEAR_PKTHDR(m)
563 } else
564 m = m_retryhdr(nowait, type);
565
566 m_range_check(mfree);
567 m_range_check(mclfree);
568 m_range_check(mbigfree);
569
570
571 return m;
572 }
573
574 struct mbuf *
575 m_getclr(
576 int nowait,
577 int type)
578 {
579 struct mbuf *m;
580
581 MGET(m, nowait, type);
582 if (m == 0)
583 return (0);
584 bzero(mtod(m, caddr_t), MLEN);
585 return (m);
586 }
587
588 struct mbuf *
589 m_free(
590 struct mbuf *m)
591 {
592 struct mbuf *n = m->m_next;
593 int i;
594
595 m_range_check(m);
596 m_range_check(mfree);
597 m_range_check(mclfree);
598
599 if (m->m_type == MT_FREE)
600 panic("freeing free mbuf");
601
602 /* Free the aux data if there is any */
603 if ((m->m_flags & M_PKTHDR) && m->m_pkthdr.aux)
604 {
605 m_freem(m->m_pkthdr.aux);
606 }
607 if ((m->m_flags & M_PKTHDR) != 0)
608 m_tag_delete_chain(m, NULL);
609
610 MBUF_LOCK();
611 if ((m->m_flags & M_EXT))
612 {
613 if (MCLHASREFERENCE(m)) {
614 remque((queue_t)&m->m_ext.ext_refs);
615 } else if (m->m_ext.ext_free == NULL) {
616 union mcluster *mcl= (union mcluster *)m->m_ext.ext_buf;
617
618 m_range_check(mcl);
619
620 if (_MCLUNREF(mcl)) {
621 mcl->mcl_next = mclfree;
622 mclfree = mcl;
623 ++mbstat.m_clfree;
624 }
625 #ifdef COMMENT_OUT
626 /* *** Since m_split() increments "mclrefcnt[mtocl(m->m_ext.ext_buf)]",
627 and AppleTalk ADSP uses m_split(), this incorrect sanity check
628 caused a panic.
629 *** */
630 else /* sanity check - not referenced this way */
631 panic("m_free m_ext cluster not free");
632 #endif
633 } else {
634 (*(m->m_ext.ext_free))(m->m_ext.ext_buf,
635 m->m_ext.ext_size, m->m_ext.ext_arg);
636 }
637 }
638 mbstat.m_mtypes[m->m_type]--;
639 (void) _MCLUNREF(m);
640 _MFREE_MUNGE(m);
641 m->m_type = MT_FREE;
642 mbstat.m_mtypes[m->m_type]++;
643 m->m_flags = 0;
644 m->m_next = mfree;
645 m->m_len = 0;
646 mfree = m;
647 i = m_want;
648 m_want = 0;
649 MBUF_UNLOCK();
650 if (i) wakeup((caddr_t)&mfree);
651 return (n);
652 }
653
654 /* m_mclget() add an mbuf cluster to a normal mbuf */
655 struct mbuf *
656 m_mclget(
657 struct mbuf *m,
658 int nowait)
659 {
660 MCLALLOC(m->m_ext.ext_buf, nowait);
661 if (m->m_ext.ext_buf) {
662 m->m_data = m->m_ext.ext_buf;
663 m->m_flags |= M_EXT;
664 m->m_ext.ext_size = MCLBYTES;
665 m->m_ext.ext_free = 0;
666 m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward =
667 &m->m_ext.ext_refs;
668 }
669
670 return m;
671 }
672
673 /* m_mclalloc() allocate an mbuf cluster */
674 caddr_t
675 m_mclalloc(
676 int nowait)
677 {
678 caddr_t p;
679
680 (void)m_clalloc(1, nowait, MCLBYTES, 0);
681 if ((p = (caddr_t)mclfree)) {
682 ++mclrefcnt[mtocl(p)];
683 mbstat.m_clfree--;
684 mclfree = ((union mcluster *)p)->mcl_next;
685 } else {
686 mbstat.m_drops++;
687 }
688 MBUF_UNLOCK();
689
690 return p;
691 }
692
693 /* m_mclfree() releases a reference to a cluster allocated by MCLALLOC,
694 * freeing the cluster if the reference count has reached 0. */
695 void
696 m_mclfree(
697 caddr_t p)
698 {
699 MBUF_LOCK();
700
701 m_range_check(p);
702
703 if (--mclrefcnt[mtocl(p)] == 0) {
704 ((union mcluster *)(p))->mcl_next = mclfree;
705 mclfree = (union mcluster *)(p);
706 mbstat.m_clfree++;
707 }
708 MBUF_UNLOCK();
709 }
710
711 /* mcl_hasreference() checks if a cluster of an mbuf is referenced by another mbuf */
712 int
713 m_mclhasreference(
714 struct mbuf *m)
715 {
716 return (m->m_ext.ext_refs.forward != &(m->m_ext.ext_refs));
717 }
718
719 __private_extern__ caddr_t
720 m_bigalloc(int nowait)
721 {
722 caddr_t p;
723
724 (void)m_clalloc(1, nowait, NBPG, 0);
725 if ((p = (caddr_t)mbigfree)) {
726 if (mclrefcnt[mtocl(p)] != mclrefcnt[mtocl(p) + 1])
727 panic("m_bigalloc mclrefcnt %x mismatch %d != %d",
728 p, mclrefcnt[mtocl(p)], mclrefcnt[mtocl(p) + 1]);
729 if (mclrefcnt[mtocl(p)] || mclrefcnt[mtocl(p) + 1])
730 panic("m_bigalloc mclrefcnt %x not null %d != %d",
731 p, mclrefcnt[mtocl(p)], mclrefcnt[mtocl(p) + 1]);
732 ++mclrefcnt[mtocl(p)];
733 ++mclrefcnt[mtocl(p) + 1];
734 mbstat.m_bigclfree--;
735 mbigfree = ((union mbigcluster *)p)->mbc_next;
736 } else {
737 mbstat.m_drops++;
738 }
739 MBUF_UNLOCK();
740 return p;
741 }
742
743 __private_extern__ void
744 m_bigfree(caddr_t p, __unused u_int size, __unused caddr_t arg)
745 {
746 m_range_check(p);
747
748 if (mclrefcnt[mtocl(p)] != mclrefcnt[mtocl(p) + 1])
749 panic("m_bigfree mclrefcnt %x mismatch %d != %d",
750 p, mclrefcnt[mtocl(p)], mclrefcnt[mtocl(p) + 1]);
751 --mclrefcnt[mtocl(p)];
752 --mclrefcnt[mtocl(p) + 1];
753 if (mclrefcnt[mtocl(p)] == 0) {
754 ((union mbigcluster *)(p))->mbc_next = mbigfree;
755 mbigfree = (union mbigcluster *)(p);
756 mbstat.m_bigclfree++;
757 }
758 }
759
760 /* m_mbigget() add an 4KB mbuf cluster to a normal mbuf */
761 __private_extern__ struct mbuf *
762 m_mbigget(struct mbuf *m, int nowait)
763 {
764 m->m_ext.ext_buf = m_bigalloc(nowait);
765 if (m->m_ext.ext_buf) {
766 m->m_data = m->m_ext.ext_buf;
767 m->m_flags |= M_EXT;
768 m->m_ext.ext_size = NBPG;
769 m->m_ext.ext_free = m_bigfree;
770 m->m_ext.ext_arg = 0;
771 m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward =
772 &m->m_ext.ext_refs;
773 }
774
775 return m;
776 }
777
778
779 /* */
780 void
781 m_copy_pkthdr(
782 struct mbuf *to,
783 struct mbuf *from)
784 {
785 to->m_pkthdr = from->m_pkthdr;
786 from->m_pkthdr.aux = (struct mbuf *)NULL;
787 SLIST_INIT(&from->m_pkthdr.tags); /* purge tags from src */
788 to->m_flags = from->m_flags & M_COPYFLAGS;
789 to->m_data = (to)->m_pktdat;
790 }
791
792 /*
793 * "Move" mbuf pkthdr from "from" to "to".
794 * "from" must have M_PKTHDR set, and "to" must be empty.
795 */
796 #ifndef __APPLE__
797 void
798 m_move_pkthdr(struct mbuf *to, struct mbuf *from)
799 {
800 KASSERT((to->m_flags & M_EXT) == 0, ("m_move_pkthdr: to has cluster"));
801
802 to->m_flags = from->m_flags & M_COPYFLAGS;
803 to->m_data = to->m_pktdat;
804 to->m_pkthdr = from->m_pkthdr; /* especially tags */
805 SLIST_INIT(&from->m_pkthdr.tags); /* purge tags from src */
806 from->m_flags &= ~M_PKTHDR;
807 }
808 #endif
809
810 /*
811 * Duplicate "from"'s mbuf pkthdr in "to".
812 * "from" must have M_PKTHDR set, and "to" must be empty.
813 * In particular, this does a deep copy of the packet tags.
814 */
815 static int
816 m_dup_pkthdr(struct mbuf *to, struct mbuf *from, int how)
817 {
818 to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
819 if ((to->m_flags & M_EXT) == 0)
820 to->m_data = to->m_pktdat;
821 to->m_pkthdr = from->m_pkthdr;
822 SLIST_INIT(&to->m_pkthdr.tags);
823 return (m_tag_copy_chain(to, from, how));
824 }
825
826 /*
827 * return a list of mbuf hdrs that point to clusters...
828 * try for num_needed, if wantall is not set, return whatever
829 * number were available... set up the first num_with_pkthdrs
830 * with mbuf hdrs configured as packet headers... these are
831 * chained on the m_nextpkt field... any packets requested beyond
832 * this are chained onto the last packet header's m_next field.
833 * The size of the cluster is controlled by the paramter bufsize.
834 */
835 __private_extern__ struct mbuf *
836 m_getpackets_internal(unsigned int *num_needed, int num_with_pkthdrs, int how, int wantall, size_t bufsize)
837 {
838 struct mbuf *m;
839 struct mbuf **np, *top;
840 unsigned int num, needed = *num_needed;
841
842 if (bufsize != MCLBYTES && bufsize != NBPG)
843 return 0;
844
845 top = NULL;
846 np = &top;
847
848 (void)m_clalloc(needed, how, bufsize, 0); /* takes the MBUF_LOCK, but doesn't release it... */
849
850 for (num = 0; num < needed; num++) {
851 m_range_check(mfree);
852 m_range_check(mclfree);
853 m_range_check(mbigfree);
854
855 if (mfree && ((bufsize == NBPG && mbigfree) || (bufsize == MCLBYTES && mclfree))) {
856 /* mbuf + cluster are available */
857 m = mfree;
858 MCHECK(m);
859 mfree = m->m_next;
860 ++mclrefcnt[mtocl(m)];
861 mbstat.m_mtypes[MT_FREE]--;
862 mbstat.m_mtypes[MT_DATA]++;
863 if (bufsize == NBPG) {
864 m->m_ext.ext_buf = (caddr_t)mbigfree; /* get the big cluster */
865 ++mclrefcnt[mtocl(m->m_ext.ext_buf)];
866 ++mclrefcnt[mtocl(m->m_ext.ext_buf) + 1];
867 mbstat.m_bigclfree--;
868 mbigfree = ((union mbigcluster *)(m->m_ext.ext_buf))->mbc_next;
869 m->m_ext.ext_free = m_bigfree;
870 m->m_ext.ext_size = NBPG;
871 } else {
872 m->m_ext.ext_buf = (caddr_t)mclfree; /* get the cluster */
873 ++mclrefcnt[mtocl(m->m_ext.ext_buf)];
874 mbstat.m_clfree--;
875 mclfree = ((union mcluster *)(m->m_ext.ext_buf))->mcl_next;
876 m->m_ext.ext_free = 0;
877 m->m_ext.ext_size = MCLBYTES;
878 }
879 m->m_ext.ext_arg = 0;
880 m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward = &m->m_ext.ext_refs;
881 m->m_next = m->m_nextpkt = 0;
882 m->m_type = MT_DATA;
883 m->m_data = m->m_ext.ext_buf;
884 m->m_len = 0;
885
886 if (num_with_pkthdrs == 0)
887 m->m_flags = M_EXT;
888 else {
889 m->m_flags = M_PKTHDR | M_EXT;
890 _M_CLEAR_PKTHDR(m);
891
892 num_with_pkthdrs--;
893 }
894 } else {
895 MBUF_UNLOCK();
896
897 if (num_with_pkthdrs == 0) {
898 MGET(m, how, MT_DATA );
899 } else {
900 MGETHDR(m, how, MT_DATA);
901
902 num_with_pkthdrs--;
903 }
904 if (m == 0)
905 goto fail;
906
907 if (bufsize == NBPG)
908 m = m_mbigget(m, how);
909 else
910 m = m_mclget(m, how);
911 if ((m->m_flags & M_EXT) == 0) {
912 m_free(m);
913 goto fail;
914 }
915 MBUF_LOCK();
916 }
917 *np = m;
918
919 if (num_with_pkthdrs)
920 np = &m->m_nextpkt;
921 else
922 np = &m->m_next;
923 }
924 MBUF_UNLOCK();
925
926 *num_needed = num;
927 return (top);
928 fail:
929 if (wantall && top) {
930 m_freem(top);
931 return 0;
932 }
933 return top;
934 }
935
936
937 /*
938 * Return list of mbuf linked by m_nextpkt
939 * Try for num_needed, and if wantall is not set, return whatever
940 * number were available
941 * The size of each mbuf in the list is controlled by the parameter packetlen.
942 * Each mbuf of the list may have a chain of mbufs linked by m_next. Each mbuf in
943 * the chain is called a segment.
944 * If maxsegments is not null and the value pointed to is not null, this specify
945 * the maximum number of segments for a chain of mbufs.
946 * If maxsegments is zero or the value pointed to is zero the
947 * caller does not have any restriction on the number of segments.
948 * The actual number of segments of a mbuf chain is return in the value pointed
949 * to by maxsegments.
950 * When possible the allocation is done under a single lock.
951 */
952
953 __private_extern__ struct mbuf *
954 m_allocpacket_internal(unsigned int *num_needed, size_t packetlen, unsigned int * maxsegments,
955 int how, int wantall, size_t wantsize)
956 {
957 struct mbuf **np, *top;
958 size_t bufsize;
959 unsigned int num;
960 unsigned int numchunks = 0;
961
962 top = NULL;
963 np = &top;
964
965 if (wantsize == 0) {
966 if (packetlen <= MINCLSIZE)
967 bufsize = packetlen;
968 else if (packetlen > MCLBYTES)
969 bufsize = NBPG;
970 else
971 bufsize = MCLBYTES;
972 } else if (wantsize == MCLBYTES || wantsize == NBPG)
973 bufsize = wantsize;
974 else
975 return 0;
976
977 if (bufsize <= MHLEN) {
978 numchunks = 1;
979 } else if (bufsize <= MINCLSIZE) {
980 if (maxsegments != NULL && *maxsegments == 1) {
981 bufsize = MCLBYTES;
982 numchunks = 1;
983 } else {
984 numchunks = 2;
985 }
986 } else if (bufsize == NBPG) {
987 numchunks = ((packetlen - 1) >> PGSHIFT) + 1;
988 } else {
989 numchunks = ((packetlen - 1) >> MCLSHIFT) + 1;
990 }
991 if (maxsegments != NULL) {
992 if (*maxsegments && numchunks > *maxsegments) {
993 *maxsegments = numchunks;
994 return 0;
995 }
996 *maxsegments = numchunks;
997 }
998 /* m_clalloc takes the MBUF_LOCK, but do not release it */
999 (void)m_clalloc(numchunks, how, (bufsize == NBPG) ? NBPG : MCLBYTES, 0);
1000 for (num = 0; num < *num_needed; num++) {
1001 struct mbuf **nm, *pkt = 0;
1002 size_t len;
1003
1004 nm = &pkt;
1005
1006 m_range_check(mfree);
1007 m_range_check(mclfree);
1008 m_range_check(mbigfree);
1009
1010 for (len = 0; len < packetlen; ) {
1011 struct mbuf *m = NULL;
1012
1013 if (wantsize == 0 && packetlen > MINCLSIZE) {
1014 if (packetlen - len > MCLBYTES)
1015 bufsize = NBPG;
1016 else
1017 bufsize = MCLBYTES;
1018 }
1019 len += bufsize;
1020
1021 if (mfree && ((bufsize == NBPG && mbigfree) || (bufsize == MCLBYTES && mclfree))) {
1022 /* mbuf + cluster are available */
1023 m = mfree;
1024 MCHECK(m);
1025 mfree = m->m_next;
1026 ++mclrefcnt[mtocl(m)];
1027 mbstat.m_mtypes[MT_FREE]--;
1028 mbstat.m_mtypes[MT_DATA]++;
1029 if (bufsize == NBPG) {
1030 m->m_ext.ext_buf = (caddr_t)mbigfree; /* get the big cluster */
1031 ++mclrefcnt[mtocl(m->m_ext.ext_buf)];
1032 ++mclrefcnt[mtocl(m->m_ext.ext_buf) + 1];
1033 mbstat.m_bigclfree--;
1034 mbigfree = ((union mbigcluster *)(m->m_ext.ext_buf))->mbc_next;
1035 m->m_ext.ext_free = m_bigfree;
1036 m->m_ext.ext_size = NBPG;
1037 } else {
1038 m->m_ext.ext_buf = (caddr_t)mclfree; /* get the cluster */
1039 ++mclrefcnt[mtocl(m->m_ext.ext_buf)];
1040 mbstat.m_clfree--;
1041 mclfree = ((union mcluster *)(m->m_ext.ext_buf))->mcl_next;
1042 m->m_ext.ext_free = 0;
1043 m->m_ext.ext_size = MCLBYTES;
1044 }
1045 m->m_ext.ext_arg = 0;
1046 m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward = &m->m_ext.ext_refs;
1047 m->m_next = m->m_nextpkt = 0;
1048 m->m_type = MT_DATA;
1049 m->m_data = m->m_ext.ext_buf;
1050 m->m_len = 0;
1051
1052 if (pkt == 0) {
1053 pkt = m;
1054 m->m_flags = M_PKTHDR | M_EXT;
1055 _M_CLEAR_PKTHDR(m);
1056 } else {
1057 m->m_flags = M_EXT;
1058 }
1059 } else {
1060 MBUF_UNLOCK();
1061
1062 if (pkt == 0) {
1063 MGETHDR(m, how, MT_DATA);
1064 } else {
1065 MGET(m, how, MT_DATA );
1066 }
1067 if (m == 0) {
1068 m_freem(pkt);
1069 goto fail;
1070 }
1071 if (bufsize <= MINCLSIZE) {
1072 if (bufsize > MHLEN) {
1073 MGET(m->m_next, how, MT_DATA);
1074 if (m->m_next == 0) {
1075 m_free(m);
1076 m_freem(pkt);
1077 goto fail;
1078 }
1079 }
1080 } else {
1081 if (bufsize == NBPG)
1082 m = m_mbigget(m, how);
1083 else
1084 m = m_mclget(m, how);
1085 if ((m->m_flags & M_EXT) == 0) {
1086 m_free(m);
1087 m_freem(pkt);
1088 goto fail;
1089 }
1090 }
1091 MBUF_LOCK();
1092 }
1093 *nm = m;
1094 nm = &m->m_next;
1095 }
1096 *np = pkt;
1097 np = &pkt->m_nextpkt;
1098 }
1099 MBUF_UNLOCK();
1100 *num_needed = num;
1101
1102 return top;
1103 fail:
1104 if (wantall && top) {
1105 m_freem(top);
1106 return 0;
1107 }
1108 *num_needed = num;
1109
1110 return top;
1111 }
1112
1113
1114 /* Best effort to get a mbuf cluster + pkthdr under one lock.
1115 * If we don't have them avail, just bail out and use the regular
1116 * path.
1117 * Used by drivers to allocated packets on receive ring.
1118 */
1119 __private_extern__ struct mbuf *
1120 m_getpacket_how(int how)
1121 {
1122 unsigned int num_needed = 1;
1123
1124 return m_getpackets_internal(&num_needed, 1, how, 1, MCLBYTES);
1125 }
1126
1127 /* Best effort to get a mbuf cluster + pkthdr under one lock.
1128 * If we don't have them avail, just bail out and use the regular
1129 * path.
1130 * Used by drivers to allocated packets on receive ring.
1131 */
1132 struct mbuf *
1133 m_getpacket(void)
1134 {
1135 unsigned int num_needed = 1;
1136
1137 return m_getpackets_internal(&num_needed, 1, M_WAITOK, 1, MCLBYTES);
1138 }
1139
1140
1141 /*
1142 * return a list of mbuf hdrs that point to clusters...
1143 * try for num_needed, if this can't be met, return whatever
1144 * number were available... set up the first num_with_pkthdrs
1145 * with mbuf hdrs configured as packet headers... these are
1146 * chained on the m_nextpkt field... any packets requested beyond
1147 * this are chained onto the last packet header's m_next field.
1148 */
1149 struct mbuf *
1150 m_getpackets(int num_needed, int num_with_pkthdrs, int how)
1151 {
1152 unsigned int n = num_needed;
1153
1154 return m_getpackets_internal(&n, num_with_pkthdrs, how, 0, MCLBYTES);
1155 }
1156
1157
1158 /*
1159 * return a list of mbuf hdrs set up as packet hdrs
1160 * chained together on the m_nextpkt field
1161 */
1162 struct mbuf *
1163 m_getpackethdrs(int num_needed, int how)
1164 {
1165 struct mbuf *m;
1166 struct mbuf **np, *top;
1167
1168 top = NULL;
1169 np = &top;
1170
1171 MBUF_LOCK();
1172
1173 while (num_needed--) {
1174 m_range_check(mfree);
1175 m_range_check(mclfree);
1176 m_range_check(mbigfree);
1177
1178 if ((m = mfree)) { /* mbufs are available */
1179 MCHECK(m);
1180 mfree = m->m_next;
1181 ++mclrefcnt[mtocl(m)];
1182 mbstat.m_mtypes[MT_FREE]--;
1183 mbstat.m_mtypes[MT_DATA]++;
1184
1185 m->m_next = m->m_nextpkt = 0;
1186 m->m_type = MT_DATA;
1187 m->m_flags = M_PKTHDR;
1188 m->m_len = 0;
1189 m->m_data = m->m_pktdat;
1190 _M_CLEAR_PKTHDR(m);
1191
1192 } else {
1193
1194 MBUF_UNLOCK();
1195 m = m_retryhdr(how, MT_DATA);
1196 if (m == 0)
1197 return(top);
1198 MBUF_LOCK();
1199 }
1200 *np = m;
1201 np = &m->m_nextpkt;
1202 }
1203 MBUF_UNLOCK();
1204
1205 return (top);
1206 }
1207
1208
1209 /* free and mbuf list (m_nextpkt) while following m_next under one lock.
1210 * returns the count for mbufs packets freed. Used by the drivers.
1211 */
1212 int
1213 m_freem_list(
1214 struct mbuf *m)
1215 {
1216 struct mbuf *nextpkt;
1217 int i, count=0;
1218
1219 MBUF_LOCK();
1220
1221 while (m) {
1222 if (m)
1223 nextpkt = m->m_nextpkt; /* chain of linked mbufs from driver */
1224 else
1225 nextpkt = 0;
1226
1227 count++;
1228
1229 while (m) { /* free the mbuf chain (like mfreem) */
1230
1231 struct mbuf *n;
1232
1233 m_range_check(m);
1234 m_range_check(mfree);
1235 m_range_check(mclfree);
1236 m_range_check(mbigfree);
1237
1238
1239 /* Free the aux data if there is any */
1240 if ((m->m_flags & M_PKTHDR) && m->m_pkthdr.aux) {
1241 /*
1242 * Treat the current m as the nextpkt and set m
1243 * to the aux data. Preserve nextpkt in m->m_nextpkt.
1244 * This lets us free the aux data in this loop
1245 * without having to call m_freem recursively,
1246 * which wouldn't work because we've still got
1247 * the lock.
1248 */
1249 m->m_nextpkt = nextpkt;
1250 nextpkt = m;
1251 m = nextpkt->m_pkthdr.aux;
1252 nextpkt->m_pkthdr.aux = NULL;
1253 }
1254
1255 if ((m->m_flags & M_PKTHDR) != 0 && !SLIST_EMPTY(&m->m_pkthdr.tags)) {
1256 /* A quick (albeit inefficient) expedient */
1257 MBUF_UNLOCK();
1258 m_tag_delete_chain(m, NULL);
1259 MBUF_LOCK();
1260 }
1261
1262 n = m->m_next;
1263
1264 if (n && n->m_nextpkt)
1265 panic("m_freem_list: m_nextpkt of m_next != NULL");
1266 if (m->m_type == MT_FREE)
1267 panic("freeing free mbuf");
1268
1269 if (m->m_flags & M_EXT) {
1270 if (MCLHASREFERENCE(m)) {
1271 remque((queue_t)&m->m_ext.ext_refs);
1272 } else if (m->m_ext.ext_free == NULL) {
1273 union mcluster *mcl= (union mcluster *)m->m_ext.ext_buf;
1274
1275 m_range_check(mcl);
1276
1277 if (_MCLUNREF(mcl)) {
1278 mcl->mcl_next = mclfree;
1279 mclfree = mcl;
1280 ++mbstat.m_clfree;
1281 }
1282 } else {
1283 (*(m->m_ext.ext_free))(m->m_ext.ext_buf,
1284 m->m_ext.ext_size, m->m_ext.ext_arg);
1285 }
1286 }
1287 mbstat.m_mtypes[m->m_type]--;
1288 (void) _MCLUNREF(m);
1289 _MFREE_MUNGE(m);
1290 mbstat.m_mtypes[MT_FREE]++;
1291 m->m_type = MT_FREE;
1292 m->m_flags = 0;
1293 m->m_len = 0;
1294 m->m_next = mfree;
1295 mfree = m;
1296 m = n;
1297 }
1298 m = nextpkt; /* bump m with saved nextpkt if any */
1299 }
1300 if ((i = m_want))
1301 m_want = 0;
1302
1303 MBUF_UNLOCK();
1304
1305 if (i)
1306 wakeup((caddr_t)&mfree);
1307
1308 return (count);
1309 }
1310
1311 void
1312 m_freem(
1313 struct mbuf *m)
1314 {
1315 while (m)
1316 m = m_free(m);
1317 }
1318
1319 /*
1320 * Mbuffer utility routines.
1321 */
1322 /*
1323 * Compute the amount of space available
1324 * before the current start of data in an mbuf.
1325 */
1326 int
1327 m_leadingspace(
1328 struct mbuf *m)
1329 {
1330 if (m->m_flags & M_EXT) {
1331 if (MCLHASREFERENCE(m))
1332 return(0);
1333 return (m->m_data - m->m_ext.ext_buf);
1334 }
1335 if (m->m_flags & M_PKTHDR)
1336 return (m->m_data - m->m_pktdat);
1337 return (m->m_data - m->m_dat);
1338 }
1339
1340 /*
1341 * Compute the amount of space available
1342 * after the end of data in an mbuf.
1343 */
1344 int
1345 m_trailingspace(
1346 struct mbuf *m)
1347 {
1348 if (m->m_flags & M_EXT) {
1349 if (MCLHASREFERENCE(m))
1350 return(0);
1351 return (m->m_ext.ext_buf + m->m_ext.ext_size -
1352 (m->m_data + m->m_len));
1353 }
1354 return (&m->m_dat[MLEN] - (m->m_data + m->m_len));
1355 }
1356
1357 /*
1358 * Lesser-used path for M_PREPEND:
1359 * allocate new mbuf to prepend to chain,
1360 * copy junk along.
1361 * Does not adjust packet header length.
1362 */
1363 struct mbuf *
1364 m_prepend(
1365 struct mbuf *m,
1366 int len,
1367 int how)
1368 {
1369 struct mbuf *mn;
1370
1371 MGET(mn, how, m->m_type);
1372 if (mn == (struct mbuf *)NULL) {
1373 m_freem(m);
1374 return ((struct mbuf *)NULL);
1375 }
1376 if (m->m_flags & M_PKTHDR) {
1377 M_COPY_PKTHDR(mn, m);
1378 m->m_flags &= ~M_PKTHDR;
1379 }
1380 mn->m_next = m;
1381 m = mn;
1382 if (len < MHLEN)
1383 MH_ALIGN(m, len);
1384 m->m_len = len;
1385 return (m);
1386 }
1387
1388 /*
1389 * Replacement for old M_PREPEND macro:
1390 * allocate new mbuf to prepend to chain,
1391 * copy junk along, and adjust length.
1392 *
1393 */
1394 struct mbuf *
1395 m_prepend_2(
1396 struct mbuf *m,
1397 int len,
1398 int how)
1399 {
1400 if (M_LEADINGSPACE(m) >= len) {
1401 m->m_data -= len;
1402 m->m_len += len;
1403 } else {
1404 m = m_prepend(m, len, how);
1405 }
1406 if ((m) && (m->m_flags & M_PKTHDR))
1407 m->m_pkthdr.len += len;
1408 return (m);
1409 }
1410
1411 /*
1412 * Make a copy of an mbuf chain starting "off0" bytes from the beginning,
1413 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf.
1414 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller.
1415 */
1416 int MCFail;
1417
1418 struct mbuf *
1419 m_copym(
1420 struct mbuf *m,
1421 int off0,
1422 int len,
1423 int wait)
1424 {
1425 struct mbuf *n, **np;
1426 int off = off0;
1427 struct mbuf *top;
1428 int copyhdr = 0;
1429
1430 if (off < 0 || len < 0)
1431 panic("m_copym");
1432 if (off == 0 && m->m_flags & M_PKTHDR)
1433 copyhdr = 1;
1434
1435 while (off >= m->m_len) {
1436 if (m == 0)
1437 panic("m_copym");
1438 off -= m->m_len;
1439 m = m->m_next;
1440 }
1441 np = &top;
1442 top = 0;
1443
1444 MBUF_LOCK();
1445
1446 while (len > 0) {
1447 m_range_check(mfree);
1448 m_range_check(mclfree);
1449 m_range_check(mbigfree);
1450
1451 if (m == 0) {
1452 if (len != M_COPYALL)
1453 panic("m_copym");
1454 break;
1455 }
1456 if ((n = mfree)) {
1457 MCHECK(n);
1458 ++mclrefcnt[mtocl(n)];
1459 mbstat.m_mtypes[MT_FREE]--;
1460 mbstat.m_mtypes[m->m_type]++;
1461 mfree = n->m_next;
1462 n->m_next = n->m_nextpkt = 0;
1463 n->m_type = m->m_type;
1464 n->m_data = n->m_dat;
1465 n->m_flags = 0;
1466 } else {
1467 MBUF_UNLOCK();
1468 n = m_retry(wait, m->m_type);
1469 MBUF_LOCK();
1470 }
1471 *np = n;
1472
1473 if (n == 0)
1474 goto nospace;
1475 if (copyhdr) {
1476 M_COPY_PKTHDR(n, m);
1477 if (len == M_COPYALL)
1478 n->m_pkthdr.len -= off0;
1479 else
1480 n->m_pkthdr.len = len;
1481 copyhdr = 0;
1482 }
1483 if (len == M_COPYALL) {
1484 if (min(len, (m->m_len - off)) == len) {
1485 printf("m->m_len %d - off %d = %d, %d\n",
1486 m->m_len, off, m->m_len - off,
1487 min(len, (m->m_len - off)));
1488 }
1489 }
1490 n->m_len = min(len, (m->m_len - off));
1491 if (n->m_len == M_COPYALL) {
1492 printf("n->m_len == M_COPYALL, fixing\n");
1493 n->m_len = MHLEN;
1494 }
1495 if (m->m_flags & M_EXT) {
1496 n->m_ext = m->m_ext;
1497 insque((queue_t)&n->m_ext.ext_refs, (queue_t)&m->m_ext.ext_refs);
1498 n->m_data = m->m_data + off;
1499 n->m_flags |= M_EXT;
1500 } else {
1501 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
1502 (unsigned)n->m_len);
1503 }
1504 if (len != M_COPYALL)
1505 len -= n->m_len;
1506 off = 0;
1507 m = m->m_next;
1508 np = &n->m_next;
1509 }
1510 MBUF_UNLOCK();
1511
1512 if (top == 0)
1513 MCFail++;
1514
1515 return (top);
1516 nospace:
1517 MBUF_UNLOCK();
1518
1519 m_freem(top);
1520 MCFail++;
1521 return (0);
1522 }
1523
1524
1525 /*
1526 * equivilent to m_copym except that all necessary
1527 * mbuf hdrs are allocated within this routine
1528 * also, the last mbuf and offset accessed are passed
1529 * out and can be passed back in to avoid having to
1530 * rescan the entire mbuf list (normally hung off of the socket)
1531 */
1532 struct mbuf *
1533 m_copym_with_hdrs(
1534 struct mbuf *m,
1535 int off0,
1536 int len,
1537 int wait,
1538 struct mbuf **m_last,
1539 int *m_off)
1540 {
1541 struct mbuf *n, **np = 0;
1542 int off = off0;
1543 struct mbuf *top = 0;
1544 int copyhdr = 0;
1545 int type;
1546
1547 if (off == 0 && m->m_flags & M_PKTHDR)
1548 copyhdr = 1;
1549
1550 if (*m_last) {
1551 m = *m_last;
1552 off = *m_off;
1553 } else {
1554 while (off >= m->m_len) {
1555 off -= m->m_len;
1556 m = m->m_next;
1557 }
1558 }
1559
1560 MBUF_LOCK();
1561
1562 while (len > 0) {
1563 m_range_check(mfree);
1564 m_range_check(mclfree);
1565 m_range_check(mbigfree);
1566
1567 if (top == 0)
1568 type = MT_HEADER;
1569 else {
1570 if (m == 0)
1571 panic("m_gethdr_and_copym");
1572 type = m->m_type;
1573 }
1574 if ((n = mfree)) {
1575 MCHECK(n);
1576 ++mclrefcnt[mtocl(n)];
1577 mbstat.m_mtypes[MT_FREE]--;
1578 mbstat.m_mtypes[type]++;
1579 mfree = n->m_next;
1580 n->m_next = n->m_nextpkt = 0;
1581 n->m_type = type;
1582
1583 if (top) {
1584 n->m_data = n->m_dat;
1585 n->m_flags = 0;
1586 } else {
1587 n->m_data = n->m_pktdat;
1588 n->m_flags = M_PKTHDR;
1589 _M_CLEAR_PKTHDR(n);
1590 }
1591 } else {
1592 MBUF_UNLOCK();
1593 if (top)
1594 n = m_retry(wait, type);
1595 else
1596 n = m_retryhdr(wait, type);
1597 MBUF_LOCK();
1598 }
1599 if (n == 0)
1600 goto nospace;
1601 if (top == 0) {
1602 top = n;
1603 np = &top->m_next;
1604 continue;
1605 } else
1606 *np = n;
1607
1608 if (copyhdr) {
1609 M_COPY_PKTHDR(n, m);
1610 n->m_pkthdr.len = len;
1611 copyhdr = 0;
1612 }
1613 n->m_len = min(len, (m->m_len - off));
1614
1615 if (m->m_flags & M_EXT) {
1616 n->m_ext = m->m_ext;
1617 insque((queue_t)&n->m_ext.ext_refs, (queue_t)&m->m_ext.ext_refs);
1618 n->m_data = m->m_data + off;
1619 n->m_flags |= M_EXT;
1620 } else {
1621 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
1622 (unsigned)n->m_len);
1623 }
1624 len -= n->m_len;
1625
1626 if (len == 0) {
1627 if ((off + n->m_len) == m->m_len) {
1628 *m_last = m->m_next;
1629 *m_off = 0;
1630 } else {
1631 *m_last = m;
1632 *m_off = off + n->m_len;
1633 }
1634 break;
1635 }
1636 off = 0;
1637 m = m->m_next;
1638 np = &n->m_next;
1639 }
1640 MBUF_UNLOCK();
1641
1642 return (top);
1643 nospace:
1644 MBUF_UNLOCK();
1645
1646 if (top)
1647 m_freem(top);
1648 MCFail++;
1649 return (0);
1650 }
1651
1652
1653 /*
1654 * Copy data from an mbuf chain starting "off" bytes from the beginning,
1655 * continuing for "len" bytes, into the indicated buffer.
1656 */
1657 void m_copydata(
1658 struct mbuf *m,
1659 int off,
1660 int len,
1661 caddr_t cp)
1662 {
1663 unsigned count;
1664
1665 if (off < 0 || len < 0)
1666 panic("m_copydata");
1667 while (off > 0) {
1668 if (m == 0)
1669 panic("m_copydata");
1670 if (off < m->m_len)
1671 break;
1672 off -= m->m_len;
1673 m = m->m_next;
1674 }
1675 while (len > 0) {
1676 if (m == 0)
1677 panic("m_copydata");
1678 count = min(m->m_len - off, len);
1679 bcopy(mtod(m, caddr_t) + off, cp, count);
1680 len -= count;
1681 cp += count;
1682 off = 0;
1683 m = m->m_next;
1684 }
1685 }
1686
1687 /*
1688 * Concatenate mbuf chain n to m.
1689 * Both chains must be of the same type (e.g. MT_DATA).
1690 * Any m_pkthdr is not updated.
1691 */
1692 void m_cat(
1693 struct mbuf *m, struct mbuf *n)
1694 {
1695 while (m->m_next)
1696 m = m->m_next;
1697 while (n) {
1698 if (m->m_flags & M_EXT ||
1699 m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) {
1700 /* just join the two chains */
1701 m->m_next = n;
1702 return;
1703 }
1704 /* splat the data from one into the other */
1705 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
1706 (u_int)n->m_len);
1707 m->m_len += n->m_len;
1708 n = m_free(n);
1709 }
1710 }
1711
1712 void
1713 m_adj(
1714 struct mbuf *mp,
1715 int req_len)
1716 {
1717 int len = req_len;
1718 struct mbuf *m;
1719 int count;
1720
1721 if ((m = mp) == NULL)
1722 return;
1723 if (len >= 0) {
1724 /*
1725 * Trim from head.
1726 */
1727 while (m != NULL && len > 0) {
1728 if (m->m_len <= len) {
1729 len -= m->m_len;
1730 m->m_len = 0;
1731 m = m->m_next;
1732 } else {
1733 m->m_len -= len;
1734 m->m_data += len;
1735 len = 0;
1736 }
1737 }
1738 m = mp;
1739 if (m->m_flags & M_PKTHDR)
1740 m->m_pkthdr.len -= (req_len - len);
1741 } else {
1742 /*
1743 * Trim from tail. Scan the mbuf chain,
1744 * calculating its length and finding the last mbuf.
1745 * If the adjustment only affects this mbuf, then just
1746 * adjust and return. Otherwise, rescan and truncate
1747 * after the remaining size.
1748 */
1749 len = -len;
1750 count = 0;
1751 for (;;) {
1752 count += m->m_len;
1753 if (m->m_next == (struct mbuf *)0)
1754 break;
1755 m = m->m_next;
1756 }
1757 if (m->m_len >= len) {
1758 m->m_len -= len;
1759 m = mp;
1760 if (m->m_flags & M_PKTHDR)
1761 m->m_pkthdr.len -= len;
1762 return;
1763 }
1764 count -= len;
1765 if (count < 0)
1766 count = 0;
1767 /*
1768 * Correct length for chain is "count".
1769 * Find the mbuf with last data, adjust its length,
1770 * and toss data from remaining mbufs on chain.
1771 */
1772 m = mp;
1773 if (m->m_flags & M_PKTHDR)
1774 m->m_pkthdr.len = count;
1775 for (; m; m = m->m_next) {
1776 if (m->m_len >= count) {
1777 m->m_len = count;
1778 break;
1779 }
1780 count -= m->m_len;
1781 }
1782 while ((m = m->m_next))
1783 m->m_len = 0;
1784 }
1785 }
1786
1787 /*
1788 * Rearange an mbuf chain so that len bytes are contiguous
1789 * and in the data area of an mbuf (so that mtod and dtom
1790 * will work for a structure of size len). Returns the resulting
1791 * mbuf chain on success, frees it and returns null on failure.
1792 * If there is room, it will add up to max_protohdr-len extra bytes to the
1793 * contiguous region in an attempt to avoid being called next time.
1794 */
1795 int MPFail;
1796
1797 struct mbuf *
1798 m_pullup(
1799 struct mbuf *n,
1800 int len)
1801 {
1802 struct mbuf *m;
1803 int count;
1804 int space;
1805
1806 /*
1807 * If first mbuf has no cluster, and has room for len bytes
1808 * without shifting current data, pullup into it,
1809 * otherwise allocate a new mbuf to prepend to the chain.
1810 */
1811 if ((n->m_flags & M_EXT) == 0 &&
1812 n->m_data + len < &n->m_dat[MLEN] && n->m_next) {
1813 if (n->m_len >= len)
1814 return (n);
1815 m = n;
1816 n = n->m_next;
1817 len -= m->m_len;
1818 } else {
1819 if (len > MHLEN)
1820 goto bad;
1821 MGET(m, M_DONTWAIT, n->m_type);
1822 if (m == 0)
1823 goto bad;
1824 m->m_len = 0;
1825 if (n->m_flags & M_PKTHDR) {
1826 M_COPY_PKTHDR(m, n);
1827 n->m_flags &= ~M_PKTHDR;
1828 }
1829 }
1830 space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
1831 do {
1832 count = min(min(max(len, max_protohdr), space), n->m_len);
1833 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
1834 (unsigned)count);
1835 len -= count;
1836 m->m_len += count;
1837 n->m_len -= count;
1838 space -= count;
1839 if (n->m_len)
1840 n->m_data += count;
1841 else
1842 n = m_free(n);
1843 } while (len > 0 && n);
1844 if (len > 0) {
1845 (void) m_free(m);
1846 goto bad;
1847 }
1848 m->m_next = n;
1849 return (m);
1850 bad:
1851 m_freem(n);
1852 MPFail++;
1853 return (0);
1854 }
1855
1856 /*
1857 * Partition an mbuf chain in two pieces, returning the tail --
1858 * all but the first len0 bytes. In case of failure, it returns NULL and
1859 * attempts to restore the chain to its original state.
1860 */
1861 struct mbuf *
1862 m_split(
1863 struct mbuf *m0,
1864 int len0,
1865 int wait)
1866 {
1867 struct mbuf *m, *n;
1868 unsigned len = len0, remain;
1869
1870 for (m = m0; m && len > m->m_len; m = m->m_next)
1871 len -= m->m_len;
1872 if (m == 0)
1873 return (0);
1874 remain = m->m_len - len;
1875 if (m0->m_flags & M_PKTHDR) {
1876 MGETHDR(n, wait, m0->m_type);
1877 if (n == 0)
1878 return (0);
1879 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
1880 n->m_pkthdr.len = m0->m_pkthdr.len - len0;
1881 m0->m_pkthdr.len = len0;
1882 if (m->m_flags & M_EXT)
1883 goto extpacket;
1884 if (remain > MHLEN) {
1885 /* m can't be the lead packet */
1886 MH_ALIGN(n, 0);
1887 n->m_next = m_split(m, len, wait);
1888 if (n->m_next == 0) {
1889 (void) m_free(n);
1890 return (0);
1891 } else
1892 return (n);
1893 } else
1894 MH_ALIGN(n, remain);
1895 } else if (remain == 0) {
1896 n = m->m_next;
1897 m->m_next = 0;
1898 return (n);
1899 } else {
1900 MGET(n, wait, m->m_type);
1901 if (n == 0)
1902 return (0);
1903 M_ALIGN(n, remain);
1904 }
1905 extpacket:
1906 if (m->m_flags & M_EXT) {
1907 n->m_flags |= M_EXT;
1908 MBUF_LOCK();
1909 n->m_ext = m->m_ext;
1910 insque((queue_t)&n->m_ext.ext_refs, (queue_t)&m->m_ext.ext_refs);
1911 MBUF_UNLOCK();
1912 n->m_data = m->m_data + len;
1913 } else {
1914 bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain);
1915 }
1916 n->m_len = remain;
1917 m->m_len = len;
1918 n->m_next = m->m_next;
1919 m->m_next = 0;
1920 return (n);
1921 }
1922 /*
1923 * Routine to copy from device local memory into mbufs.
1924 */
1925 struct mbuf *
1926 m_devget(
1927 char *buf,
1928 int totlen,
1929 int off0,
1930 struct ifnet *ifp,
1931 void (*copy)(const void *, void *, size_t))
1932 {
1933 struct mbuf *m;
1934 struct mbuf *top = 0, **mp = &top;
1935 int off = off0, len;
1936 char *cp;
1937 char *epkt;
1938
1939 cp = buf;
1940 epkt = cp + totlen;
1941 if (off) {
1942 /*
1943 * If 'off' is non-zero, packet is trailer-encapsulated,
1944 * so we have to skip the type and length fields.
1945 */
1946 cp += off + 2 * sizeof(u_int16_t);
1947 totlen -= 2 * sizeof(u_int16_t);
1948 }
1949 MGETHDR(m, M_DONTWAIT, MT_DATA);
1950 if (m == 0)
1951 return (0);
1952 m->m_pkthdr.rcvif = ifp;
1953 m->m_pkthdr.len = totlen;
1954 m->m_len = MHLEN;
1955
1956 while (totlen > 0) {
1957 if (top) {
1958 MGET(m, M_DONTWAIT, MT_DATA);
1959 if (m == 0) {
1960 m_freem(top);
1961 return (0);
1962 }
1963 m->m_len = MLEN;
1964 }
1965 len = min(totlen, epkt - cp);
1966 if (len >= MINCLSIZE) {
1967 MCLGET(m, M_DONTWAIT);
1968 if (m->m_flags & M_EXT)
1969 m->m_len = len = min(len, MCLBYTES);
1970 else {
1971 /* give up when it's out of cluster mbufs */
1972 if (top)
1973 m_freem(top);
1974 m_freem(m);
1975 return (0);
1976 }
1977 } else {
1978 /*
1979 * Place initial small packet/header at end of mbuf.
1980 */
1981 if (len < m->m_len) {
1982 if (top == 0 && len + max_linkhdr <= m->m_len)
1983 m->m_data += max_linkhdr;
1984 m->m_len = len;
1985 } else
1986 len = m->m_len;
1987 }
1988 if (copy)
1989 copy(cp, mtod(m, caddr_t), (unsigned)len);
1990 else
1991 bcopy(cp, mtod(m, caddr_t), (unsigned)len);
1992 cp += len;
1993 *mp = m;
1994 mp = &m->m_next;
1995 totlen -= len;
1996 if (cp == epkt)
1997 cp = buf;
1998 }
1999 return (top);
2000 }
2001
2002 /*
2003 * Cluster freelist allocation check. The mbuf lock must be held.
2004 * Ensure hysteresis between hi/lo.
2005 */
2006 static int
2007 m_howmany(int num, size_t bufsize)
2008 {
2009 int i = 0;
2010
2011 /* Bail if we've maxed out the mbuf memory map */
2012 if (mbstat.m_clusters + (mbstat.m_bigclusters << 1) < nmbclusters) {
2013 int j = 0;
2014
2015 if (bufsize == MCLBYTES) {
2016 /* Under minimum */
2017 if (mbstat.m_clusters < MINCL)
2018 return (MINCL - mbstat.m_clusters);
2019 /* Too few (free < 1/2 total) and not over maximum */
2020 if (mbstat.m_clusters < (nmbclusters >> 1)) {
2021 if (num >= mbstat.m_clfree)
2022 i = num - mbstat.m_clfree;
2023 if (((mbstat.m_clusters + num) >> 1) > mbstat.m_clfree)
2024 j = ((mbstat.m_clusters + num) >> 1) - mbstat.m_clfree;
2025 i = max(i, j);
2026 if (i + mbstat.m_clusters >= (nmbclusters >> 1))
2027 i = (nmbclusters >> 1) - mbstat.m_clusters;
2028 }
2029 } else {
2030 /* Under minimum */
2031 if (mbstat.m_bigclusters < MINCL)
2032 return (MINCL - mbstat.m_bigclusters);
2033 /* Too few (free < 1/2 total) and not over maximum */
2034 if (mbstat.m_bigclusters < (nmbclusters >> 2)) {
2035 if (num >= mbstat.m_bigclfree)
2036 i = num - mbstat.m_bigclfree;
2037 if (((mbstat.m_bigclusters + num) >> 1) > mbstat.m_bigclfree)
2038 j = ((mbstat.m_bigclusters + num) >> 1) - mbstat.m_bigclfree;
2039 i = max(i, j);
2040 if (i + mbstat.m_bigclusters >= (nmbclusters >> 2))
2041 i = (nmbclusters >> 2) - mbstat.m_bigclusters;
2042 }
2043 }
2044 }
2045 return i;
2046 }
2047
2048 /*
2049 * Copy data from a buffer back into the indicated mbuf chain,
2050 * starting "off" bytes from the beginning, extending the mbuf
2051 * chain if necessary.
2052 */
2053 void
2054 m_copyback(
2055 struct mbuf *m0,
2056 int off,
2057 int len,
2058 caddr_t cp)
2059 {
2060 int mlen;
2061 struct mbuf *m = m0, *n;
2062 int totlen = 0;
2063
2064 if (m0 == 0)
2065 return;
2066 while (off > (mlen = m->m_len)) {
2067 off -= mlen;
2068 totlen += mlen;
2069 if (m->m_next == 0) {
2070 n = m_getclr(M_DONTWAIT, m->m_type);
2071 if (n == 0)
2072 goto out;
2073 n->m_len = min(MLEN, len + off);
2074 m->m_next = n;
2075 }
2076 m = m->m_next;
2077 }
2078 while (len > 0) {
2079 mlen = min (m->m_len - off, len);
2080 bcopy(cp, off + mtod(m, caddr_t), (unsigned)mlen);
2081 cp += mlen;
2082 len -= mlen;
2083 mlen += off;
2084 off = 0;
2085 totlen += mlen;
2086 if (len == 0)
2087 break;
2088 if (m->m_next == 0) {
2089 n = m_get(M_DONTWAIT, m->m_type);
2090 if (n == 0)
2091 break;
2092 n->m_len = min(MLEN, len);
2093 m->m_next = n;
2094 }
2095 m = m->m_next;
2096 }
2097 out: if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen))
2098 m->m_pkthdr.len = totlen;
2099 }
2100
2101
2102 char *mcl_to_paddr(char *addr) {
2103 int base_phys;
2104
2105 if (addr < (char *)mbutl || addr >= (char *)embutl)
2106 return (0);
2107 base_phys = mcl_paddr[(addr - (char *)mbutl) >> PGSHIFT];
2108
2109 if (base_phys == 0)
2110 return (0);
2111 return ((char *)((int)base_phys | ((int)addr & PGOFSET)));
2112 }
2113
2114 /*
2115 * Dup the mbuf chain passed in. The whole thing. No cute additional cruft.
2116 * And really copy the thing. That way, we don't "precompute" checksums
2117 * for unsuspecting consumers.
2118 * Assumption: m->m_nextpkt == 0.
2119 * Trick: for small packets, don't dup into a cluster. That way received
2120 * packets don't take up too much room in the sockbuf (cf. sbspace()).
2121 */
2122 int MDFail;
2123
2124 struct mbuf *
2125 m_dup(struct mbuf *m, int how)
2126 {
2127 struct mbuf *n, **np;
2128 struct mbuf *top;
2129 int copyhdr = 0;
2130
2131 np = &top;
2132 top = 0;
2133 if (m->m_flags & M_PKTHDR)
2134 copyhdr = 1;
2135
2136 /*
2137 * Quick check: if we have one mbuf and its data fits in an
2138 * mbuf with packet header, just copy and go.
2139 */
2140 if (m->m_next == NULL)
2141 { /* Then just move the data into an mbuf and be done... */
2142 if (copyhdr)
2143 { if (m->m_pkthdr.len <= MHLEN)
2144 { if ((n = m_gethdr(how, m->m_type)) == NULL)
2145 return(NULL);
2146 n->m_len = m->m_len;
2147 m_dup_pkthdr(n, m, how);
2148 bcopy(m->m_data, n->m_data, m->m_len);
2149 return(n);
2150 }
2151 } else if (m->m_len <= MLEN)
2152 { if ((n = m_get(how, m->m_type)) == NULL)
2153 return(NULL);
2154 bcopy(m->m_data, n->m_data, m->m_len);
2155 n->m_len = m->m_len;
2156 return(n);
2157 }
2158 }
2159 while (m)
2160 {
2161 #if BLUE_DEBUG
2162 kprintf("<%x: %x, %x, %x\n", m, m->m_flags, m->m_len,
2163 m->m_data);
2164 #endif
2165 if (copyhdr)
2166 n = m_gethdr(how, m->m_type);
2167 else
2168 n = m_get(how, m->m_type);
2169 if (n == 0)
2170 goto nospace;
2171 if (m->m_flags & M_EXT)
2172 { MCLGET(n, how);
2173 if ((n->m_flags & M_EXT) == 0)
2174 goto nospace;
2175 }
2176 *np = n;
2177 if (copyhdr)
2178 { /* Don't use M_COPY_PKTHDR: preserve m_data */
2179 m_dup_pkthdr(n, m, how);
2180 copyhdr = 0;
2181 if ((n->m_flags & M_EXT) == 0)
2182 n->m_data = n->m_pktdat;
2183 }
2184 n->m_len = m->m_len;
2185 /*
2186 * Get the dup on the same bdry as the original
2187 * Assume that the two mbufs have the same offset to data area
2188 * (up to word bdries)
2189 */
2190 bcopy(mtod(m, caddr_t), mtod(n, caddr_t), (unsigned)n->m_len);
2191 m = m->m_next;
2192 np = &n->m_next;
2193 #if BLUE_DEBUG
2194 kprintf(">%x: %x, %x, %x\n", n, n->m_flags, n->m_len,
2195 n->m_data);
2196 #endif
2197 }
2198
2199 if (top == 0)
2200 MDFail++;
2201 return (top);
2202 nospace:
2203 m_freem(top);
2204 MDFail++;
2205 return (0);
2206 }
2207
2208 int
2209 m_mclref(struct mbuf *p)
2210 {
2211 return (_MCLREF(p));
2212 }
2213
2214 int
2215 m_mclunref(struct mbuf *p)
2216 {
2217 return (_MCLUNREF(p));
2218 }
2219
2220 /* change mbuf to new type */
2221 void
2222 m_mchtype(struct mbuf *m, int t)
2223 {
2224 MBUF_LOCK();
2225 mbstat.m_mtypes[(m)->m_type]--;
2226 mbstat.m_mtypes[t]++;
2227 (m)->m_type = t;
2228 MBUF_UNLOCK();
2229 }
2230
2231 void *m_mtod(struct mbuf *m)
2232 {
2233 return ((m)->m_data);
2234 }
2235
2236 struct mbuf *m_dtom(void *x)
2237 {
2238 return ((struct mbuf *)((u_long)(x) & ~(MSIZE-1)));
2239 }
2240
2241 int m_mtocl(void *x)
2242 {
2243 return (((char *)(x) - (char *)mbutl) / sizeof(union mcluster));
2244 }
2245
2246 union mcluster *m_cltom(int x)
2247 {
2248 return ((union mcluster *)(mbutl + (x)));
2249 }
2250
2251
2252 void m_mcheck(struct mbuf *m)
2253 {
2254 if (m->m_type != MT_FREE)
2255 panic("mget MCHECK: m_type=%x m=%x", m->m_type, m);
2256 }
2257
2258 static void
2259 mbuf_expand_thread(void)
2260 {
2261 while (1) {
2262 MBUF_LOCK();
2263 if (mbuf_expand_mcl) {
2264 int n;
2265
2266 /* Adjust to the current number of cluster in use */
2267 n = mbuf_expand_mcl - (mbstat.m_clusters - mbstat.m_clfree);
2268 mbuf_expand_mcl = 0;
2269
2270 if (n > 0)
2271 (void)m_clalloc(n, M_WAIT, MCLBYTES, 1);
2272 }
2273 if (mbuf_expand_big) {
2274 int n;
2275
2276 /* Adjust to the current number of 4 KB cluster in use */
2277 n = mbuf_expand_big - (mbstat.m_bigclusters - mbstat.m_bigclfree);
2278 mbuf_expand_big = 0;
2279
2280 if (n > 0)
2281 (void)m_clalloc(n, M_WAIT, NBPG, 1);
2282 }
2283 MBUF_UNLOCK();
2284 /*
2285 * Because we can run out of memory before filling the mbuf map, we
2286 * should not allocate more clusters than they are mbufs -- otherwise
2287 * we could have a large number of useless clusters allocated.
2288 */
2289 while (mbstat.m_mbufs < mbstat.m_bigclusters + mbstat.m_clusters) {
2290 if (m_expand(M_WAIT) == 0)
2291 break;
2292 }
2293
2294 assert_wait(&mbuf_expand_thread_wakeup, THREAD_UNINT);
2295 (void) thread_block((thread_continue_t)mbuf_expand_thread);
2296 }
2297 }
2298
2299 static void
2300 mbuf_expand_thread_init(void)
2301 {
2302 mbuf_expand_thread_initialized++;
2303 mbuf_expand_thread();
2304 }
2305
2306 SYSCTL_DECL(_kern_ipc);
2307 SYSCTL_STRUCT(_kern_ipc, KIPC_MBSTAT, mbstat, CTLFLAG_RW, &mbstat, mbstat, "");
2308
mirror of XNU