]> git.saurik.com Git - apple/xnu.git/blob - bsd/kern/uipc_mbuf.c
projects / apple / xnu.git / blob
? search:


d73d7a4dacbb4aa12f71fcfc74bf5efb275e7934
[apple/xnu.git] / bsd / kern / uipc_mbuf.c
1 /*
2 * Copyright (c) 2000-2005 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
11 * file.
12 *
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
20 *
21 * @APPLE_LICENSE_HEADER_END@
22 */
23 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
24 /*
25 * Copyright (c) 1982, 1986, 1988, 1991, 1993
26 * The Regents of the University of California. All rights reserved.
27 *
28 * Redistribution and use in source and binary forms, with or without
29 * modification, are permitted provided that the following conditions
30 * are met:
31 * 1. Redistributions of source code must retain the above copyright
32 * notice, this list of conditions and the following disclaimer.
33 * 2. Redistributions in binary form must reproduce the above copyright
34 * notice, this list of conditions and the following disclaimer in the
35 * documentation and/or other materials provided with the distribution.
36 * 3. All advertising materials mentioning features or use of this software
37 * must display the following acknowledgement:
38 * This product includes software developed by the University of
39 * California, Berkeley and its contributors.
40 * 4. Neither the name of the University nor the names of its contributors
41 * may be used to endorse or promote products derived from this software
42 * without specific prior written permission.
43 *
44 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
45 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
46 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
47 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
48 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
49 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
50 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
51 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
52 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
53 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
54 * SUCH DAMAGE.
55 *
56 * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94
57 */
58 /* HISTORY
59 *
60 * 10/15/97 Annette DeSchon (deschon@apple.com)
61 * Fixed bug in which all cluster mbufs were broken up
62 * into regular mbufs: Some clusters are now reserved.
63 * When a cluster is needed, regular mbufs are no longer
64 * used. (Radar 1683621)
65 * 20-May-95 Mac Gillon (mgillon) at NeXT
66 * New version based on 4.4
67 */
68
69 #include <sys/param.h>
70 #include <sys/systm.h>
71 #include <sys/malloc.h>
72 #include <sys/mbuf.h>
73 #include <sys/kernel.h>
74 #include <sys/sysctl.h>
75 #include <sys/syslog.h>
76 #include <sys/protosw.h>
77 #include <sys/domain.h>
78
79 #include <kern/queue.h>
80 #include <kern/kern_types.h>
81 #include <kern/sched_prim.h>
82
83 #include <IOKit/IOMapper.h>
84
85 extern vm_offset_t kmem_mb_alloc(vm_map_t , int );
86 extern boolean_t PE_parse_boot_arg(const char *, void *);
87
88 #define _MCLREF(p) (++mclrefcnt[mtocl(p)])
89 #define _MCLUNREF(p) (--mclrefcnt[mtocl(p)] == 0)
90 #define _M_CLEAR_PKTHDR(mbuf_ptr) (mbuf_ptr)->m_pkthdr.rcvif = NULL; \
91 (mbuf_ptr)->m_pkthdr.len = 0; \
92 (mbuf_ptr)->m_pkthdr.header = NULL; \
93 (mbuf_ptr)->m_pkthdr.csum_flags = 0; \
94 (mbuf_ptr)->m_pkthdr.csum_data = 0; \
95 (mbuf_ptr)->m_pkthdr.aux = (struct mbuf*)NULL; \
96 (mbuf_ptr)->m_pkthdr.vlan_tag = 0; \
97 (mbuf_ptr)->m_pkthdr.socket_id = 0; \
98 SLIST_INIT(&(mbuf_ptr)->m_pkthdr.tags);
99
100 /* kernel translater */
101 extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
102
103 lck_mtx_t * mbuf_mlock;
104 lck_grp_t * mbuf_mlock_grp;
105 lck_grp_attr_t * mbuf_mlock_grp_attr;
106 lck_attr_t * mbuf_mlock_attr;
107 extern lck_mtx_t *domain_proto_mtx;
108
109 struct mbuf *mfree; /* mbuf free list */
110 struct mbuf *mfreelater; /* mbuf deallocation list */
111 extern vm_map_t mb_map; /* special map */
112 int m_want; /* sleepers on mbufs */
113 short *mclrefcnt; /* mapped cluster reference counts */
114 int *mcl_paddr;
115 static ppnum_t mcl_paddr_base; /* Handle returned by IOMapper::iovmAlloc() */
116 union mcluster *mclfree; /* mapped cluster free list */
117 union mbigcluster *mbigfree; /* mapped cluster free list */
118 int max_linkhdr; /* largest link-level header */
119 int max_protohdr; /* largest protocol header */
120 int max_hdr; /* largest link+protocol header */
121 int max_datalen; /* MHLEN - max_hdr */
122 struct mbstat mbstat; /* statistics */
123 union mcluster *mbutl; /* first mapped cluster address */
124 union mcluster *embutl; /* ending virtual address of mclusters */
125
126 static int nclpp; /* # clusters per physical page */
127
128 static int m_howmany(int, size_t );
129 void m_reclaim(void);
130 static int m_clalloc(const int , const int, const size_t, int);
131 int do_reclaim = 0;
132
133 #define MF_NOWAIT 0x1
134 #define MF_BIG 0x2
135
136 /* The number of cluster mbufs that are allocated, to start. */
137 #define MINCL max(16, 2)
138
139 static int mbuf_expand_thread_wakeup = 0;
140 static int mbuf_expand_mcl = 0;
141 static int mbuf_expand_big = 0;
142 static int mbuf_expand_thread_initialized = 0;
143
144 static void mbuf_expand_thread_init(void);
145 static void mbuf_expand_thread(void);
146 static int m_expand(int );
147 static caddr_t m_bigalloc(int );
148 static void m_bigfree(caddr_t , u_int , caddr_t );
149 static struct mbuf * m_mbigget(struct mbuf *, int );
150 void mbinit(void);
151 static void m_range_check(void *addr);
152
153
154 #if 0
155 static int mfree_munge = 0;
156 #if 0
157 #define _MFREE_MUNGE(m) { \
158 if (mfree_munge) \
159 { int i; \
160 vm_offset_t *element = (vm_offset_t *)(m); \
161 for (i = 0; \
162 i < sizeof(struct mbuf)/sizeof(vm_offset_t); \
163 i++) \
164 (element)[i] = 0xdeadbeef; \
165 } \
166 }
167 #else
168 void
169 munge_mbuf(struct mbuf *m)
170 {
171 int i;
172 vm_offset_t *element = (vm_offset_t *)(m);
173 for (i = 0;
174 i < sizeof(struct mbuf)/sizeof(vm_offset_t);
175 i++)
176 (element)[i] = 0xdeadbeef;
177 }
178 #define _MFREE_MUNGE(m) { \
179 if (mfree_munge) \
180 munge_mbuf(m); \
181 }
182 #endif
183 #else
184 #define _MFREE_MUNGE(m)
185 #endif
186
187
188 #define _MINTGET(m, type) { \
189 MBUF_LOCK(); \
190 if (((m) = mfree) != 0) { \
191 MCHECK(m); \
192 ++mclrefcnt[mtocl(m)]; \
193 mbstat.m_mtypes[MT_FREE]--; \
194 mbstat.m_mtypes[(type)]++; \
195 mfree = (m)->m_next; \
196 } \
197 MBUF_UNLOCK(); \
198 }
199
200
201 static void
202 m_range_check(void *addr)
203 {
204 if (addr && (addr < (void *)mbutl || addr >= (void *)embutl))
205 panic("mbuf address out of range 0x%x", addr);
206 }
207
208 __private_extern__ void
209 mbinit(void)
210 {
211 int m;
212 int initmcl = 32;
213 int mcl_pages;
214
215 if (nclpp)
216 return;
217 nclpp = round_page_32(MCLBYTES) / MCLBYTES; /* see mbufgc() */
218 if (nclpp < 1) nclpp = 1;
219 mbuf_mlock_grp_attr = lck_grp_attr_alloc_init();
220 lck_grp_attr_setdefault(mbuf_mlock_grp_attr);
221
222 mbuf_mlock_grp = lck_grp_alloc_init("mbuf", mbuf_mlock_grp_attr);
223 mbuf_mlock_attr = lck_attr_alloc_init();
224 lck_attr_setdefault(mbuf_mlock_attr);
225
226 mbuf_mlock = lck_mtx_alloc_init(mbuf_mlock_grp, mbuf_mlock_attr);
227
228 mbstat.m_msize = MSIZE;
229 mbstat.m_mclbytes = MCLBYTES;
230 mbstat.m_minclsize = MINCLSIZE;
231 mbstat.m_mlen = MLEN;
232 mbstat.m_mhlen = MHLEN;
233 mbstat.m_bigmclbytes = NBPG;
234
235 if (nmbclusters == 0)
236 nmbclusters = NMBCLUSTERS;
237 MALLOC(mclrefcnt, short *, nmbclusters * sizeof (short),
238 M_TEMP, M_WAITOK);
239 if (mclrefcnt == 0)
240 panic("mbinit");
241 for (m = 0; m < nmbclusters; m++)
242 mclrefcnt[m] = -1;
243
244 /* Calculate the number of pages assigned to the cluster pool */
245 mcl_pages = nmbclusters/(NBPG/CLBYTES);
246 MALLOC(mcl_paddr, int *, mcl_pages * sizeof(int), M_TEMP, M_WAITOK);
247 if (mcl_paddr == 0)
248 panic("mbinit1");
249 /* Register with the I/O Bus mapper */
250 mcl_paddr_base = IOMapperIOVMAlloc(mcl_pages);
251 bzero((char *)mcl_paddr, mcl_pages * sizeof(int));
252
253 embutl = (union mcluster *)((unsigned char *)mbutl + (nmbclusters * MCLBYTES));
254
255 PE_parse_boot_arg("initmcl", &initmcl);
256
257 if (m_clalloc(max(NBPG/CLBYTES, 1) * initmcl, M_WAIT, MCLBYTES, 0) == 0)
258 goto bad;
259 MBUF_UNLOCK();
260
261 (void) kernel_thread(kernel_task, mbuf_expand_thread_init);
262
263 return;
264 bad:
265 panic("mbinit");
266 }
267
268 /*
269 * Allocate some number of mbuf clusters
270 * and place on cluster free list.
271 * Take the mbuf lock (if not already locked) and do not release it
272 */
273 /* ARGSUSED */
274 static int
275 m_clalloc(
276 const int num,
277 const int nowait,
278 const size_t bufsize,
279 int locked)
280 {
281 int i;
282 vm_size_t size = 0;
283 int numpages = 0;
284 vm_offset_t page = 0;
285
286 if (locked == 0)
287 MBUF_LOCK();
288 /*
289 * Honor the caller's wish to block or not block.
290 * We have a way to grow the pool asynchronously,
291 * by kicking the dlil_input_thread.
292 */
293 i = m_howmany(num, bufsize);
294 if (i == 0 || nowait == M_DONTWAIT)
295 goto out;
296
297 MBUF_UNLOCK();
298 size = round_page_32(i * bufsize);
299 page = kmem_mb_alloc(mb_map, size);
300
301 if (page == 0) {
302 size = NBPG; /* Try for 1 if failed */
303 page = kmem_mb_alloc(mb_map, size);
304 }
305 MBUF_LOCK();
306
307 if (page) {
308 numpages = size / NBPG;
309 for (i = 0; i < numpages; i++, page += NBPG) {
310 if (((int)page & PGOFSET) == 0) {
311 ppnum_t offset = ((char *)page - (char *)mbutl)/NBPG;
312 ppnum_t new_page = pmap_find_phys(kernel_pmap, (vm_address_t) page);
313
314 /*
315 * In the case of no mapper being available
316 * the following code nops and returns the
317 * input page, if there is a mapper the I/O
318 * page appropriate is returned.
319 */
320 new_page = IOMapperInsertPage(mcl_paddr_base, offset, new_page);
321 mcl_paddr[offset] = new_page << 12;
322 }
323 if (bufsize == MCLBYTES) {
324 union mcluster *mcl = (union mcluster *)page;
325
326 if (++mclrefcnt[mtocl(mcl)] != 0)
327 panic("m_clalloc already there");
328 mcl->mcl_next = mclfree;
329 mclfree = mcl++;
330 if (++mclrefcnt[mtocl(mcl)] != 0)
331 panic("m_clalloc already there");
332 mcl->mcl_next = mclfree;
333 mclfree = mcl++;
334 } else {
335 union mbigcluster *mbc = (union mbigcluster *)page;
336
337 if (++mclrefcnt[mtocl(mbc)] != 0)
338 panic("m_clalloc already there");
339 if (++mclrefcnt[mtocl(mbc) + 1] != 0)
340 panic("m_clalloc already there");
341
342 mbc->mbc_next = mbigfree;
343 mbigfree = mbc;
344 }
345 }
346 if (bufsize == MCLBYTES) {
347 int numcl = numpages << 1;
348 mbstat.m_clfree += numcl;
349 mbstat.m_clusters += numcl;
350 return (numcl);
351 } else {
352 mbstat.m_bigclfree += numpages;
353 mbstat.m_bigclusters += numpages;
354 return (numpages);
355 }
356 } /* else ... */
357 out:
358 /*
359 * When non-blocking we kick a thread if we havve to grow the
360 * pool or if the number of free clusters is less than requested.
361 */
362 if (bufsize == MCLBYTES) {
363 if (i > 0) {
364 /* Remember total number of clusters needed at this time */
365 i += mbstat.m_clusters;
366 if (i > mbuf_expand_mcl) {
367 mbuf_expand_mcl = i;
368 if (mbuf_expand_thread_initialized)
369 wakeup((caddr_t)&mbuf_expand_thread_wakeup);
370 }
371 }
372
373 if (mbstat.m_clfree >= num)
374 return 1;
375 } else {
376 if (i > 0) {
377 /* Remember total number of 4KB clusters needed at this time */
378 i += mbstat.m_bigclusters;
379 if (i > mbuf_expand_big) {
380 mbuf_expand_big = i;
381 if (mbuf_expand_thread_initialized)
382 wakeup((caddr_t)&mbuf_expand_thread_wakeup);
383 }
384 }
385
386 if (mbstat.m_bigclfree >= num)
387 return 1;
388 }
389 return 0;
390 }
391
392 /*
393 * Add more free mbufs by cutting up a cluster.
394 */
395 static int
396 m_expand(int canwait)
397 {
398 caddr_t mcl;
399
400 if (mbstat.m_clfree < (mbstat.m_clusters >> 4)) {
401 /*
402 * 1/16th of the total number of cluster mbufs allocated is
403 * reserved for large packets. The number reserved must
404 * always be < 1/2, or future allocation will be prevented.
405 */
406 (void)m_clalloc(1, canwait, MCLBYTES, 0);
407 MBUF_UNLOCK();
408 if (mbstat.m_clfree < (mbstat.m_clusters >> 4))
409 return 0;
410 }
411
412 MCLALLOC(mcl, canwait);
413 if (mcl) {
414 struct mbuf *m = (struct mbuf *)mcl;
415 int i = NMBPCL;
416 MBUF_LOCK();
417 mbstat.m_mtypes[MT_FREE] += i;
418 mbstat.m_mbufs += i;
419 while (i--) {
420 _MFREE_MUNGE(m);
421 m->m_type = MT_FREE;
422 m->m_next = mfree;
423 mfree = m++;
424 }
425 i = m_want;
426 m_want = 0;
427 MBUF_UNLOCK();
428 if (i) wakeup((caddr_t)&mfree);
429 return 1;
430 }
431 return 0;
432 }
433
434 /*
435 * When MGET failes, ask protocols to free space when short of memory,
436 * then re-attempt to allocate an mbuf.
437 */
438 struct mbuf *
439 m_retry(
440 int canwait,
441 int type)
442 {
443 struct mbuf *m;
444 int wait;
445
446 for (;;) {
447 (void) m_expand(canwait);
448 _MINTGET(m, type);
449 if (m) {
450 (m)->m_next = (m)->m_nextpkt = 0;
451 (m)->m_type = (type);
452 (m)->m_data = (m)->m_dat;
453 (m)->m_flags = 0;
454 (m)->m_len = 0;
455 }
456 if (m || canwait == M_DONTWAIT)
457 break;
458 MBUF_LOCK();
459 wait = m_want++;
460 mbuf_expand_mcl++;
461 if (wait == 0)
462 mbstat.m_drain++;
463 else
464 mbstat.m_wait++;
465 MBUF_UNLOCK();
466
467 if (mbuf_expand_thread_initialized)
468 wakeup((caddr_t)&mbuf_expand_thread_wakeup);
469
470 if (wait == 0) {
471 m_reclaim();
472 } else {
473 struct timespec ts;
474 ts.tv_sec = 1;
475 ts.tv_nsec = 0;
476 (void) msleep((caddr_t)&mfree, 0, (PZERO-1) | PDROP, "m_retry", &ts);
477 }
478 }
479 if (m == 0)
480 mbstat.m_drops++;
481 return (m);
482 }
483
484 /*
485 * As above; retry an MGETHDR.
486 */
487 struct mbuf *
488 m_retryhdr(
489 int canwait,
490 int type)
491 {
492 struct mbuf *m;
493
494 if ((m = m_retry(canwait, type))) {
495 m->m_next = m->m_nextpkt = 0;
496 m->m_flags |= M_PKTHDR;
497 m->m_data = m->m_pktdat;
498 _M_CLEAR_PKTHDR(m);
499 }
500 return (m);
501 }
502
503 void
504 m_reclaim(void)
505 {
506 do_reclaim = 1; /* drain is performed in pfslowtimo(), to avoid deadlocks */
507 mbstat.m_drain++;
508 }
509
510 /*
511 * Space allocation routines.
512 * These are also available as macros
513 * for critical paths.
514 */
515 struct mbuf *
516 m_get(
517 int nowait,
518 int type)
519 {
520 struct mbuf *m;
521
522 m_range_check(mfree);
523 m_range_check(mclfree);
524 m_range_check(mbigfree);
525
526 _MINTGET(m, type);
527 if (m) {
528 m->m_next = m->m_nextpkt = 0;
529 m->m_type = type;
530 m->m_data = m->m_dat;
531 m->m_flags = 0;
532 m->m_len = 0;
533 } else
534 (m) = m_retry(nowait, type);
535
536 m_range_check(mfree);
537 m_range_check(mclfree);
538 m_range_check(mbigfree);
539
540
541 return (m);
542 }
543
544 struct mbuf *
545 m_gethdr(
546 int nowait,
547 int type)
548 {
549 struct mbuf *m;
550
551 m_range_check(mfree);
552 m_range_check(mclfree);
553 m_range_check(mbigfree);
554
555
556 _MINTGET(m, type);
557 if (m) {
558 m->m_next = m->m_nextpkt = 0;
559 m->m_type = type;
560 m->m_data = m->m_pktdat;
561 m->m_flags = M_PKTHDR;
562 m->m_len = 0;
563 _M_CLEAR_PKTHDR(m)
564 } else
565 m = m_retryhdr(nowait, type);
566
567 m_range_check(mfree);
568 m_range_check(mclfree);
569 m_range_check(mbigfree);
570
571
572 return m;
573 }
574
575 struct mbuf *
576 m_getclr(
577 int nowait,
578 int type)
579 {
580 struct mbuf *m;
581
582 MGET(m, nowait, type);
583 if (m == 0)
584 return (0);
585 bzero(mtod(m, caddr_t), MLEN);
586 return (m);
587 }
588
589 struct mbuf *
590 m_free(
591 struct mbuf *m)
592 {
593 struct mbuf *n = m->m_next;
594 int i;
595
596 m_range_check(m);
597 m_range_check(mfree);
598 m_range_check(mclfree);
599
600 if (m->m_type == MT_FREE)
601 panic("freeing free mbuf");
602
603 /* Free the aux data if there is any */
604 if ((m->m_flags & M_PKTHDR) && m->m_pkthdr.aux)
605 {
606 m_freem(m->m_pkthdr.aux);
607 }
608 if ((m->m_flags & M_PKTHDR) != 0)
609 m_tag_delete_chain(m, NULL);
610
611 MBUF_LOCK();
612 if ((m->m_flags & M_EXT))
613 {
614 if (MCLHASREFERENCE(m)) {
615 remque((queue_t)&m->m_ext.ext_refs);
616 } else if (m->m_ext.ext_free == NULL) {
617 union mcluster *mcl= (union mcluster *)m->m_ext.ext_buf;
618
619 m_range_check(mcl);
620
621 if (_MCLUNREF(mcl)) {
622 mcl->mcl_next = mclfree;
623 mclfree = mcl;
624 ++mbstat.m_clfree;
625 }
626 #ifdef COMMENT_OUT
627 /* *** Since m_split() increments "mclrefcnt[mtocl(m->m_ext.ext_buf)]",
628 and AppleTalk ADSP uses m_split(), this incorrect sanity check
629 caused a panic.
630 *** */
631 else /* sanity check - not referenced this way */
632 panic("m_free m_ext cluster not free");
633 #endif
634 } else {
635 (*(m->m_ext.ext_free))(m->m_ext.ext_buf,
636 m->m_ext.ext_size, m->m_ext.ext_arg);
637 }
638 }
639 mbstat.m_mtypes[m->m_type]--;
640 (void) _MCLUNREF(m);
641 _MFREE_MUNGE(m);
642 m->m_type = MT_FREE;
643 mbstat.m_mtypes[m->m_type]++;
644 m->m_flags = 0;
645 m->m_next = mfree;
646 m->m_len = 0;
647 mfree = m;
648 i = m_want;
649 m_want = 0;
650 MBUF_UNLOCK();
651 if (i) wakeup((caddr_t)&mfree);
652 return (n);
653 }
654
655 /* m_mclget() add an mbuf cluster to a normal mbuf */
656 struct mbuf *
657 m_mclget(
658 struct mbuf *m,
659 int nowait)
660 {
661 MCLALLOC(m->m_ext.ext_buf, nowait);
662 if (m->m_ext.ext_buf) {
663 m->m_data = m->m_ext.ext_buf;
664 m->m_flags |= M_EXT;
665 m->m_ext.ext_size = MCLBYTES;
666 m->m_ext.ext_free = 0;
667 m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward =
668 &m->m_ext.ext_refs;
669 }
670
671 return m;
672 }
673
674 /* m_mclalloc() allocate an mbuf cluster */
675 caddr_t
676 m_mclalloc(
677 int nowait)
678 {
679 caddr_t p;
680
681 (void)m_clalloc(1, nowait, MCLBYTES, 0);
682 if ((p = (caddr_t)mclfree)) {
683 ++mclrefcnt[mtocl(p)];
684 mbstat.m_clfree--;
685 mclfree = ((union mcluster *)p)->mcl_next;
686 } else {
687 mbstat.m_drops++;
688 }
689 MBUF_UNLOCK();
690
691 return p;
692 }
693
694 /* m_mclfree() releases a reference to a cluster allocated by MCLALLOC,
695 * freeing the cluster if the reference count has reached 0. */
696 void
697 m_mclfree(
698 caddr_t p)
699 {
700 MBUF_LOCK();
701
702 m_range_check(p);
703
704 if (--mclrefcnt[mtocl(p)] == 0) {
705 ((union mcluster *)(p))->mcl_next = mclfree;
706 mclfree = (union mcluster *)(p);
707 mbstat.m_clfree++;
708 }
709 MBUF_UNLOCK();
710 }
711
712 /* mcl_hasreference() checks if a cluster of an mbuf is referenced by another mbuf */
713 int
714 m_mclhasreference(
715 struct mbuf *m)
716 {
717 return (m->m_ext.ext_refs.forward != &(m->m_ext.ext_refs));
718 }
719
720 __private_extern__ caddr_t
721 m_bigalloc(int nowait)
722 {
723 caddr_t p;
724
725 (void)m_clalloc(1, nowait, NBPG, 0);
726 if ((p = (caddr_t)mbigfree)) {
727 if (mclrefcnt[mtocl(p)] != mclrefcnt[mtocl(p) + 1])
728 panic("m_bigalloc mclrefcnt %x mismatch %d != %d",
729 p, mclrefcnt[mtocl(p)], mclrefcnt[mtocl(p) + 1]);
730 if (mclrefcnt[mtocl(p)] || mclrefcnt[mtocl(p) + 1])
731 panic("m_bigalloc mclrefcnt %x not null %d != %d",
732 p, mclrefcnt[mtocl(p)], mclrefcnt[mtocl(p) + 1]);
733 ++mclrefcnt[mtocl(p)];
734 ++mclrefcnt[mtocl(p) + 1];
735 mbstat.m_bigclfree--;
736 mbigfree = ((union mbigcluster *)p)->mbc_next;
737 } else {
738 mbstat.m_drops++;
739 }
740 MBUF_UNLOCK();
741 return p;
742 }
743
744 __private_extern__ void
745 m_bigfree(caddr_t p, __unused u_int size, __unused caddr_t arg)
746 {
747 m_range_check(p);
748
749 if (mclrefcnt[mtocl(p)] != mclrefcnt[mtocl(p) + 1])
750 panic("m_bigfree mclrefcnt %x mismatch %d != %d",
751 p, mclrefcnt[mtocl(p)], mclrefcnt[mtocl(p) + 1]);
752 --mclrefcnt[mtocl(p)];
753 --mclrefcnt[mtocl(p) + 1];
754 if (mclrefcnt[mtocl(p)] == 0) {
755 ((union mbigcluster *)(p))->mbc_next = mbigfree;
756 mbigfree = (union mbigcluster *)(p);
757 mbstat.m_bigclfree++;
758 }
759 }
760
761 /* m_mbigget() add an 4KB mbuf cluster to a normal mbuf */
762 __private_extern__ struct mbuf *
763 m_mbigget(struct mbuf *m, int nowait)
764 {
765 m->m_ext.ext_buf = m_bigalloc(nowait);
766 if (m->m_ext.ext_buf) {
767 m->m_data = m->m_ext.ext_buf;
768 m->m_flags |= M_EXT;
769 m->m_ext.ext_size = NBPG;
770 m->m_ext.ext_free = m_bigfree;
771 m->m_ext.ext_arg = 0;
772 m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward =
773 &m->m_ext.ext_refs;
774 }
775
776 return m;
777 }
778
779
780 /* */
781 void
782 m_copy_pkthdr(
783 struct mbuf *to,
784 struct mbuf *from)
785 {
786 to->m_pkthdr = from->m_pkthdr;
787 from->m_pkthdr.aux = (struct mbuf *)NULL;
788 SLIST_INIT(&from->m_pkthdr.tags); /* purge tags from src */
789 to->m_flags = from->m_flags & M_COPYFLAGS;
790 to->m_data = (to)->m_pktdat;
791 }
792
793 /*
794 * "Move" mbuf pkthdr from "from" to "to".
795 * "from" must have M_PKTHDR set, and "to" must be empty.
796 */
797 #ifndef __APPLE__
798 void
799 m_move_pkthdr(struct mbuf *to, struct mbuf *from)
800 {
801 KASSERT((to->m_flags & M_EXT) == 0, ("m_move_pkthdr: to has cluster"));
802
803 to->m_flags = from->m_flags & M_COPYFLAGS;
804 to->m_data = to->m_pktdat;
805 to->m_pkthdr = from->m_pkthdr; /* especially tags */
806 SLIST_INIT(&from->m_pkthdr.tags); /* purge tags from src */
807 from->m_flags &= ~M_PKTHDR;
808 }
809 #endif
810
811 /*
812 * Duplicate "from"'s mbuf pkthdr in "to".
813 * "from" must have M_PKTHDR set, and "to" must be empty.
814 * In particular, this does a deep copy of the packet tags.
815 */
816 static int
817 m_dup_pkthdr(struct mbuf *to, struct mbuf *from, int how)
818 {
819 to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
820 if ((to->m_flags & M_EXT) == 0)
821 to->m_data = to->m_pktdat;
822 to->m_pkthdr = from->m_pkthdr;
823 SLIST_INIT(&to->m_pkthdr.tags);
824 return (m_tag_copy_chain(to, from, how));
825 }
826
827 /*
828 * return a list of mbuf hdrs that point to clusters...
829 * try for num_needed, if wantall is not set, return whatever
830 * number were available... set up the first num_with_pkthdrs
831 * with mbuf hdrs configured as packet headers... these are
832 * chained on the m_nextpkt field... any packets requested beyond
833 * this are chained onto the last packet header's m_next field.
834 * The size of the cluster is controlled by the paramter bufsize.
835 */
836 __private_extern__ struct mbuf *
837 m_getpackets_internal(unsigned int *num_needed, int num_with_pkthdrs, int how, int wantall, size_t bufsize)
838 {
839 struct mbuf *m;
840 struct mbuf **np, *top;
841 unsigned int num, needed = *num_needed;
842
843 if (bufsize != MCLBYTES && bufsize != NBPG)
844 return 0;
845
846 top = NULL;
847 np = &top;
848
849 (void)m_clalloc(needed, how, bufsize, 0); /* takes the MBUF_LOCK, but doesn't release it... */
850
851 for (num = 0; num < needed; num++) {
852 m_range_check(mfree);
853 m_range_check(mclfree);
854 m_range_check(mbigfree);
855
856 if (mfree && ((bufsize == NBPG && mbigfree) || (bufsize == MCLBYTES && mclfree))) {
857 /* mbuf + cluster are available */
858 m = mfree;
859 MCHECK(m);
860 mfree = m->m_next;
861 ++mclrefcnt[mtocl(m)];
862 mbstat.m_mtypes[MT_FREE]--;
863 mbstat.m_mtypes[MT_DATA]++;
864 if (bufsize == NBPG) {
865 m->m_ext.ext_buf = (caddr_t)mbigfree; /* get the big cluster */
866 ++mclrefcnt[mtocl(m->m_ext.ext_buf)];
867 ++mclrefcnt[mtocl(m->m_ext.ext_buf) + 1];
868 mbstat.m_bigclfree--;
869 mbigfree = ((union mbigcluster *)(m->m_ext.ext_buf))->mbc_next;
870 m->m_ext.ext_free = m_bigfree;
871 m->m_ext.ext_size = NBPG;
872 } else {
873 m->m_ext.ext_buf = (caddr_t)mclfree; /* get the cluster */
874 ++mclrefcnt[mtocl(m->m_ext.ext_buf)];
875 mbstat.m_clfree--;
876 mclfree = ((union mcluster *)(m->m_ext.ext_buf))->mcl_next;
877 m->m_ext.ext_free = 0;
878 m->m_ext.ext_size = MCLBYTES;
879 }
880 m->m_ext.ext_arg = 0;
881 m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward = &m->m_ext.ext_refs;
882 m->m_next = m->m_nextpkt = 0;
883 m->m_type = MT_DATA;
884 m->m_data = m->m_ext.ext_buf;
885 m->m_len = 0;
886
887 if (num_with_pkthdrs == 0)
888 m->m_flags = M_EXT;
889 else {
890 m->m_flags = M_PKTHDR | M_EXT;
891 _M_CLEAR_PKTHDR(m);
892
893 num_with_pkthdrs--;
894 }
895 } else {
896 MBUF_UNLOCK();
897
898 if (num_with_pkthdrs == 0) {
899 MGET(m, how, MT_DATA );
900 } else {
901 MGETHDR(m, how, MT_DATA);
902
903 num_with_pkthdrs--;
904 }
905 if (m == 0)
906 goto fail;
907
908 if (bufsize == NBPG)
909 m = m_mbigget(m, how);
910 else
911 m = m_mclget(m, how);
912 if ((m->m_flags & M_EXT) == 0) {
913 m_free(m);
914 goto fail;
915 }
916 MBUF_LOCK();
917 }
918 *np = m;
919
920 if (num_with_pkthdrs)
921 np = &m->m_nextpkt;
922 else
923 np = &m->m_next;
924 }
925 MBUF_UNLOCK();
926
927 *num_needed = num;
928 return (top);
929 fail:
930 if (wantall && top) {
931 m_freem(top);
932 return 0;
933 }
934 return top;
935 }
936
937
938 /*
939 * Return list of mbuf linked by m_nextpkt
940 * Try for num_needed, and if wantall is not set, return whatever
941 * number were available
942 * The size of each mbuf in the list is controlled by the parameter packetlen.
943 * Each mbuf of the list may have a chain of mbufs linked by m_next. Each mbuf in
944 * the chain is called a segment.
945 * If maxsegments is not null and the value pointed to is not null, this specify
946 * the maximum number of segments for a chain of mbufs.
947 * If maxsegments is zero or the value pointed to is zero the
948 * caller does not have any restriction on the number of segments.
949 * The actual number of segments of a mbuf chain is return in the value pointed
950 * to by maxsegments.
951 * When possible the allocation is done under a single lock.
952 */
953
954 __private_extern__ struct mbuf *
955 m_allocpacket_internal(unsigned int *num_needed, size_t packetlen, unsigned int * maxsegments,
956 int how, int wantall, size_t wantsize)
957 {
958 struct mbuf **np, *top;
959 size_t bufsize;
960 unsigned int num;
961 unsigned int numchunks = 0;
962
963 top = NULL;
964 np = &top;
965
966 if (wantsize == 0) {
967 if (packetlen <= MINCLSIZE)
968 bufsize = packetlen;
969 else if (packetlen > MCLBYTES)
970 bufsize = NBPG;
971 else
972 bufsize = MCLBYTES;
973 } else if (wantsize == MCLBYTES || wantsize == NBPG)
974 bufsize = wantsize;
975 else
976 return 0;
977
978 if (bufsize <= MHLEN) {
979 numchunks = 1;
980 } else if (bufsize <= MINCLSIZE) {
981 if (maxsegments != NULL && *maxsegments == 1) {
982 bufsize = MCLBYTES;
983 numchunks = 1;
984 } else {
985 numchunks = 2;
986 }
987 } else if (bufsize == NBPG) {
988 numchunks = ((packetlen - 1) >> PGSHIFT) + 1;
989 } else {
990 numchunks = ((packetlen - 1) >> MCLSHIFT) + 1;
991 }
992 if (maxsegments != NULL) {
993 if (*maxsegments && numchunks > *maxsegments) {
994 *maxsegments = numchunks;
995 return 0;
996 }
997 *maxsegments = numchunks;
998 }
999 /* m_clalloc takes the MBUF_LOCK, but do not release it */
1000 (void)m_clalloc(numchunks, how, (bufsize == NBPG) ? NBPG : MCLBYTES, 0);
1001 for (num = 0; num < *num_needed; num++) {
1002 struct mbuf **nm, *pkt = 0;
1003 size_t len;
1004
1005 nm = &pkt;
1006
1007 m_range_check(mfree);
1008 m_range_check(mclfree);
1009 m_range_check(mbigfree);
1010
1011 for (len = 0; len < packetlen; ) {
1012 struct mbuf *m = NULL;
1013
1014 if (wantsize == 0 && packetlen > MINCLSIZE) {
1015 if (packetlen - len > MCLBYTES)
1016 bufsize = NBPG;
1017 else
1018 bufsize = MCLBYTES;
1019 }
1020 len += bufsize;
1021
1022 if (mfree && ((bufsize == NBPG && mbigfree) || (bufsize == MCLBYTES && mclfree))) {
1023 /* mbuf + cluster are available */
1024 m = mfree;
1025 MCHECK(m);
1026 mfree = m->m_next;
1027 ++mclrefcnt[mtocl(m)];
1028 mbstat.m_mtypes[MT_FREE]--;
1029 mbstat.m_mtypes[MT_DATA]++;
1030 if (bufsize == NBPG) {
1031 m->m_ext.ext_buf = (caddr_t)mbigfree; /* get the big cluster */
1032 ++mclrefcnt[mtocl(m->m_ext.ext_buf)];
1033 ++mclrefcnt[mtocl(m->m_ext.ext_buf) + 1];
1034 mbstat.m_bigclfree--;
1035 mbigfree = ((union mbigcluster *)(m->m_ext.ext_buf))->mbc_next;
1036 m->m_ext.ext_free = m_bigfree;
1037 m->m_ext.ext_size = NBPG;
1038 } else {
1039 m->m_ext.ext_buf = (caddr_t)mclfree; /* get the cluster */
1040 ++mclrefcnt[mtocl(m->m_ext.ext_buf)];
1041 mbstat.m_clfree--;
1042 mclfree = ((union mcluster *)(m->m_ext.ext_buf))->mcl_next;
1043 m->m_ext.ext_free = 0;
1044 m->m_ext.ext_size = MCLBYTES;
1045 }
1046 m->m_ext.ext_arg = 0;
1047 m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward = &m->m_ext.ext_refs;
1048 m->m_next = m->m_nextpkt = 0;
1049 m->m_type = MT_DATA;
1050 m->m_data = m->m_ext.ext_buf;
1051 m->m_len = 0;
1052
1053 if (pkt == 0) {
1054 pkt = m;
1055 m->m_flags = M_PKTHDR | M_EXT;
1056 _M_CLEAR_PKTHDR(m);
1057 } else {
1058 m->m_flags = M_EXT;
1059 }
1060 } else {
1061 MBUF_UNLOCK();
1062
1063 if (pkt == 0) {
1064 MGETHDR(m, how, MT_DATA);
1065 } else {
1066 MGET(m, how, MT_DATA );
1067 }
1068 if (m == 0) {
1069 m_freem(pkt);
1070 goto fail;
1071 }
1072 if (bufsize <= MINCLSIZE) {
1073 if (bufsize > MHLEN) {
1074 MGET(m->m_next, how, MT_DATA);
1075 if (m->m_next == 0) {
1076 m_free(m);
1077 m_freem(pkt);
1078 goto fail;
1079 }
1080 }
1081 } else {
1082 if (bufsize == NBPG)
1083 m = m_mbigget(m, how);
1084 else
1085 m = m_mclget(m, how);
1086 if ((m->m_flags & M_EXT) == 0) {
1087 m_free(m);
1088 m_freem(pkt);
1089 goto fail;
1090 }
1091 }
1092 MBUF_LOCK();
1093 }
1094 *nm = m;
1095 nm = &m->m_next;
1096 }
1097 *np = pkt;
1098 np = &pkt->m_nextpkt;
1099 }
1100 MBUF_UNLOCK();
1101 *num_needed = num;
1102
1103 return top;
1104 fail:
1105 if (wantall && top) {
1106 m_freem(top);
1107 return 0;
1108 }
1109 *num_needed = num;
1110
1111 return top;
1112 }
1113
1114
1115 /* Best effort to get a mbuf cluster + pkthdr under one lock.
1116 * If we don't have them avail, just bail out and use the regular
1117 * path.
1118 * Used by drivers to allocated packets on receive ring.
1119 */
1120 __private_extern__ struct mbuf *
1121 m_getpacket_how(int how)
1122 {
1123 unsigned int num_needed = 1;
1124
1125 return m_getpackets_internal(&num_needed, 1, how, 1, MCLBYTES);
1126 }
1127
1128 /* Best effort to get a mbuf cluster + pkthdr under one lock.
1129 * If we don't have them avail, just bail out and use the regular
1130 * path.
1131 * Used by drivers to allocated packets on receive ring.
1132 */
1133 struct mbuf *
1134 m_getpacket(void)
1135 {
1136 unsigned int num_needed = 1;
1137
1138 return m_getpackets_internal(&num_needed, 1, M_WAITOK, 1, MCLBYTES);
1139 }
1140
1141
1142 /*
1143 * return a list of mbuf hdrs that point to clusters...
1144 * try for num_needed, if this can't be met, return whatever
1145 * number were available... set up the first num_with_pkthdrs
1146 * with mbuf hdrs configured as packet headers... these are
1147 * chained on the m_nextpkt field... any packets requested beyond
1148 * this are chained onto the last packet header's m_next field.
1149 */
1150 struct mbuf *
1151 m_getpackets(int num_needed, int num_with_pkthdrs, int how)
1152 {
1153 unsigned int n = num_needed;
1154
1155 return m_getpackets_internal(&n, num_with_pkthdrs, how, 0, MCLBYTES);
1156 }
1157
1158
1159 /*
1160 * return a list of mbuf hdrs set up as packet hdrs
1161 * chained together on the m_nextpkt field
1162 */
1163 struct mbuf *
1164 m_getpackethdrs(int num_needed, int how)
1165 {
1166 struct mbuf *m;
1167 struct mbuf **np, *top;
1168
1169 top = NULL;
1170 np = &top;
1171
1172 MBUF_LOCK();
1173
1174 while (num_needed--) {
1175 m_range_check(mfree);
1176 m_range_check(mclfree);
1177 m_range_check(mbigfree);
1178
1179 if ((m = mfree)) { /* mbufs are available */
1180 MCHECK(m);
1181 mfree = m->m_next;
1182 ++mclrefcnt[mtocl(m)];
1183 mbstat.m_mtypes[MT_FREE]--;
1184 mbstat.m_mtypes[MT_DATA]++;
1185
1186 m->m_next = m->m_nextpkt = 0;
1187 m->m_type = MT_DATA;
1188 m->m_flags = M_PKTHDR;
1189 m->m_len = 0;
1190 m->m_data = m->m_pktdat;
1191 _M_CLEAR_PKTHDR(m);
1192
1193 } else {
1194
1195 MBUF_UNLOCK();
1196 m = m_retryhdr(how, MT_DATA);
1197 if (m == 0)
1198 return(top);
1199 MBUF_LOCK();
1200 }
1201 *np = m;
1202 np = &m->m_nextpkt;
1203 }
1204 MBUF_UNLOCK();
1205
1206 return (top);
1207 }
1208
1209
1210 /* free and mbuf list (m_nextpkt) while following m_next under one lock.
1211 * returns the count for mbufs packets freed. Used by the drivers.
1212 */
1213 int
1214 m_freem_list(
1215 struct mbuf *m)
1216 {
1217 struct mbuf *nextpkt;
1218 int i, count=0;
1219
1220 MBUF_LOCK();
1221
1222 while (m) {
1223 if (m)
1224 nextpkt = m->m_nextpkt; /* chain of linked mbufs from driver */
1225 else
1226 nextpkt = 0;
1227
1228 count++;
1229
1230 while (m) { /* free the mbuf chain (like mfreem) */
1231
1232 struct mbuf *n;
1233
1234 m_range_check(m);
1235 m_range_check(mfree);
1236 m_range_check(mclfree);
1237 m_range_check(mbigfree);
1238
1239
1240 /* Free the aux data if there is any */
1241 if ((m->m_flags & M_PKTHDR) && m->m_pkthdr.aux) {
1242 /*
1243 * Treat the current m as the nextpkt and set m
1244 * to the aux data. Preserve nextpkt in m->m_nextpkt.
1245 * This lets us free the aux data in this loop
1246 * without having to call m_freem recursively,
1247 * which wouldn't work because we've still got
1248 * the lock.
1249 */
1250 m->m_nextpkt = nextpkt;
1251 nextpkt = m;
1252 m = nextpkt->m_pkthdr.aux;
1253 nextpkt->m_pkthdr.aux = NULL;
1254 }
1255
1256 if ((m->m_flags & M_PKTHDR) != 0 && !SLIST_EMPTY(&m->m_pkthdr.tags)) {
1257 /* A quick (albeit inefficient) expedient */
1258 MBUF_UNLOCK();
1259 m_tag_delete_chain(m, NULL);
1260 MBUF_LOCK();
1261 }
1262
1263 n = m->m_next;
1264
1265 if (n && n->m_nextpkt)
1266 panic("m_freem_list: m_nextpkt of m_next != NULL");
1267 if (m->m_type == MT_FREE)
1268 panic("freeing free mbuf");
1269
1270 if (m->m_flags & M_EXT) {
1271 if (MCLHASREFERENCE(m)) {
1272 remque((queue_t)&m->m_ext.ext_refs);
1273 } else if (m->m_ext.ext_free == NULL) {
1274 union mcluster *mcl= (union mcluster *)m->m_ext.ext_buf;
1275
1276 m_range_check(mcl);
1277
1278 if (_MCLUNREF(mcl)) {
1279 mcl->mcl_next = mclfree;
1280 mclfree = mcl;
1281 ++mbstat.m_clfree;
1282 }
1283 } else {
1284 (*(m->m_ext.ext_free))(m->m_ext.ext_buf,
1285 m->m_ext.ext_size, m->m_ext.ext_arg);
1286 }
1287 }
1288 mbstat.m_mtypes[m->m_type]--;
1289 (void) _MCLUNREF(m);
1290 _MFREE_MUNGE(m);
1291 mbstat.m_mtypes[MT_FREE]++;
1292 m->m_type = MT_FREE;
1293 m->m_flags = 0;
1294 m->m_len = 0;
1295 m->m_next = mfree;
1296 mfree = m;
1297 m = n;
1298 }
1299 m = nextpkt; /* bump m with saved nextpkt if any */
1300 }
1301 if ((i = m_want))
1302 m_want = 0;
1303
1304 MBUF_UNLOCK();
1305
1306 if (i)
1307 wakeup((caddr_t)&mfree);
1308
1309 return (count);
1310 }
1311
1312 void
1313 m_freem(
1314 struct mbuf *m)
1315 {
1316 while (m)
1317 m = m_free(m);
1318 }
1319
1320 /*
1321 * Mbuffer utility routines.
1322 */
1323 /*
1324 * Compute the amount of space available
1325 * before the current start of data in an mbuf.
1326 */
1327 int
1328 m_leadingspace(
1329 struct mbuf *m)
1330 {
1331 if (m->m_flags & M_EXT) {
1332 if (MCLHASREFERENCE(m))
1333 return(0);
1334 return (m->m_data - m->m_ext.ext_buf);
1335 }
1336 if (m->m_flags & M_PKTHDR)
1337 return (m->m_data - m->m_pktdat);
1338 return (m->m_data - m->m_dat);
1339 }
1340
1341 /*
1342 * Compute the amount of space available
1343 * after the end of data in an mbuf.
1344 */
1345 int
1346 m_trailingspace(
1347 struct mbuf *m)
1348 {
1349 if (m->m_flags & M_EXT) {
1350 if (MCLHASREFERENCE(m))
1351 return(0);
1352 return (m->m_ext.ext_buf + m->m_ext.ext_size -
1353 (m->m_data + m->m_len));
1354 }
1355 return (&m->m_dat[MLEN] - (m->m_data + m->m_len));
1356 }
1357
1358 /*
1359 * Lesser-used path for M_PREPEND:
1360 * allocate new mbuf to prepend to chain,
1361 * copy junk along.
1362 * Does not adjust packet header length.
1363 */
1364 struct mbuf *
1365 m_prepend(
1366 struct mbuf *m,
1367 int len,
1368 int how)
1369 {
1370 struct mbuf *mn;
1371
1372 MGET(mn, how, m->m_type);
1373 if (mn == (struct mbuf *)NULL) {
1374 m_freem(m);
1375 return ((struct mbuf *)NULL);
1376 }
1377 if (m->m_flags & M_PKTHDR) {
1378 M_COPY_PKTHDR(mn, m);
1379 m->m_flags &= ~M_PKTHDR;
1380 }
1381 mn->m_next = m;
1382 m = mn;
1383 if (len < MHLEN)
1384 MH_ALIGN(m, len);
1385 m->m_len = len;
1386 return (m);
1387 }
1388
1389 /*
1390 * Replacement for old M_PREPEND macro:
1391 * allocate new mbuf to prepend to chain,
1392 * copy junk along, and adjust length.
1393 *
1394 */
1395 struct mbuf *
1396 m_prepend_2(
1397 struct mbuf *m,
1398 int len,
1399 int how)
1400 {
1401 if (M_LEADINGSPACE(m) >= len) {
1402 m->m_data -= len;
1403 m->m_len += len;
1404 } else {
1405 m = m_prepend(m, len, how);
1406 }
1407 if ((m) && (m->m_flags & M_PKTHDR))
1408 m->m_pkthdr.len += len;
1409 return (m);
1410 }
1411
1412 /*
1413 * Make a copy of an mbuf chain starting "off0" bytes from the beginning,
1414 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf.
1415 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller.
1416 */
1417 int MCFail;
1418
1419 struct mbuf *
1420 m_copym(
1421 struct mbuf *m,
1422 int off0,
1423 int len,
1424 int wait)
1425 {
1426 struct mbuf *n, **np;
1427 int off = off0;
1428 struct mbuf *top;
1429 int copyhdr = 0;
1430
1431 if (off < 0 || len < 0)
1432 panic("m_copym");
1433 if (off == 0 && m->m_flags & M_PKTHDR)
1434 copyhdr = 1;
1435
1436 while (off >= m->m_len) {
1437 if (m == 0)
1438 panic("m_copym");
1439 off -= m->m_len;
1440 m = m->m_next;
1441 }
1442 np = &top;
1443 top = 0;
1444
1445 MBUF_LOCK();
1446
1447 while (len > 0) {
1448 m_range_check(mfree);
1449 m_range_check(mclfree);
1450 m_range_check(mbigfree);
1451
1452 if (m == 0) {
1453 if (len != M_COPYALL)
1454 panic("m_copym");
1455 break;
1456 }
1457 if ((n = mfree)) {
1458 MCHECK(n);
1459 ++mclrefcnt[mtocl(n)];
1460 mbstat.m_mtypes[MT_FREE]--;
1461 mbstat.m_mtypes[m->m_type]++;
1462 mfree = n->m_next;
1463 n->m_next = n->m_nextpkt = 0;
1464 n->m_type = m->m_type;
1465 n->m_data = n->m_dat;
1466 n->m_flags = 0;
1467 } else {
1468 MBUF_UNLOCK();
1469 n = m_retry(wait, m->m_type);
1470 MBUF_LOCK();
1471 }
1472 *np = n;
1473
1474 if (n == 0)
1475 goto nospace;
1476 if (copyhdr) {
1477 M_COPY_PKTHDR(n, m);
1478 if (len == M_COPYALL)
1479 n->m_pkthdr.len -= off0;
1480 else
1481 n->m_pkthdr.len = len;
1482 copyhdr = 0;
1483 }
1484 if (len == M_COPYALL) {
1485 if (min(len, (m->m_len - off)) == len) {
1486 printf("m->m_len %d - off %d = %d, %d\n",
1487 m->m_len, off, m->m_len - off,
1488 min(len, (m->m_len - off)));
1489 }
1490 }
1491 n->m_len = min(len, (m->m_len - off));
1492 if (n->m_len == M_COPYALL) {
1493 printf("n->m_len == M_COPYALL, fixing\n");
1494 n->m_len = MHLEN;
1495 }
1496 if (m->m_flags & M_EXT) {
1497 n->m_ext = m->m_ext;
1498 insque((queue_t)&n->m_ext.ext_refs, (queue_t)&m->m_ext.ext_refs);
1499 n->m_data = m->m_data + off;
1500 n->m_flags |= M_EXT;
1501 } else {
1502 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
1503 (unsigned)n->m_len);
1504 }
1505 if (len != M_COPYALL)
1506 len -= n->m_len;
1507 off = 0;
1508 m = m->m_next;
1509 np = &n->m_next;
1510 }
1511 MBUF_UNLOCK();
1512
1513 if (top == 0)
1514 MCFail++;
1515
1516 return (top);
1517 nospace:
1518 MBUF_UNLOCK();
1519
1520 m_freem(top);
1521 MCFail++;
1522 return (0);
1523 }
1524
1525
1526 /*
1527 * equivilent to m_copym except that all necessary
1528 * mbuf hdrs are allocated within this routine
1529 * also, the last mbuf and offset accessed are passed
1530 * out and can be passed back in to avoid having to
1531 * rescan the entire mbuf list (normally hung off of the socket)
1532 */
1533 struct mbuf *
1534 m_copym_with_hdrs(
1535 struct mbuf *m,
1536 int off0,
1537 int len,
1538 int wait,
1539 struct mbuf **m_last,
1540 int *m_off)
1541 {
1542 struct mbuf *n, **np = 0;
1543 int off = off0;
1544 struct mbuf *top = 0;
1545 int copyhdr = 0;
1546 int type;
1547
1548 if (off == 0 && m->m_flags & M_PKTHDR)
1549 copyhdr = 1;
1550
1551 if (*m_last) {
1552 m = *m_last;
1553 off = *m_off;
1554 } else {
1555 while (off >= m->m_len) {
1556 off -= m->m_len;
1557 m = m->m_next;
1558 }
1559 }
1560
1561 MBUF_LOCK();
1562
1563 while (len > 0) {
1564 m_range_check(mfree);
1565 m_range_check(mclfree);
1566 m_range_check(mbigfree);
1567
1568 if (top == 0)
1569 type = MT_HEADER;
1570 else {
1571 if (m == 0)
1572 panic("m_gethdr_and_copym");
1573 type = m->m_type;
1574 }
1575 if ((n = mfree)) {
1576 MCHECK(n);
1577 ++mclrefcnt[mtocl(n)];
1578 mbstat.m_mtypes[MT_FREE]--;
1579 mbstat.m_mtypes[type]++;
1580 mfree = n->m_next;
1581 n->m_next = n->m_nextpkt = 0;
1582 n->m_type = type;
1583
1584 if (top) {
1585 n->m_data = n->m_dat;
1586 n->m_flags = 0;
1587 } else {
1588 n->m_data = n->m_pktdat;
1589 n->m_flags = M_PKTHDR;
1590 _M_CLEAR_PKTHDR(n);
1591 }
1592 } else {
1593 MBUF_UNLOCK();
1594 if (top)
1595 n = m_retry(wait, type);
1596 else
1597 n = m_retryhdr(wait, type);
1598 MBUF_LOCK();
1599 }
1600 if (n == 0)
1601 goto nospace;
1602 if (top == 0) {
1603 top = n;
1604 np = &top->m_next;
1605 continue;
1606 } else
1607 *np = n;
1608
1609 if (copyhdr) {
1610 M_COPY_PKTHDR(n, m);
1611 n->m_pkthdr.len = len;
1612 copyhdr = 0;
1613 }
1614 n->m_len = min(len, (m->m_len - off));
1615
1616 if (m->m_flags & M_EXT) {
1617 n->m_ext = m->m_ext;
1618 insque((queue_t)&n->m_ext.ext_refs, (queue_t)&m->m_ext.ext_refs);
1619 n->m_data = m->m_data + off;
1620 n->m_flags |= M_EXT;
1621 } else {
1622 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
1623 (unsigned)n->m_len);
1624 }
1625 len -= n->m_len;
1626
1627 if (len == 0) {
1628 if ((off + n->m_len) == m->m_len) {
1629 *m_last = m->m_next;
1630 *m_off = 0;
1631 } else {
1632 *m_last = m;
1633 *m_off = off + n->m_len;
1634 }
1635 break;
1636 }
1637 off = 0;
1638 m = m->m_next;
1639 np = &n->m_next;
1640 }
1641 MBUF_UNLOCK();
1642
1643 return (top);
1644 nospace:
1645 MBUF_UNLOCK();
1646
1647 if (top)
1648 m_freem(top);
1649 MCFail++;
1650 return (0);
1651 }
1652
1653
1654 /*
1655 * Copy data from an mbuf chain starting "off" bytes from the beginning,
1656 * continuing for "len" bytes, into the indicated buffer.
1657 */
1658 void m_copydata(
1659 struct mbuf *m,
1660 int off,
1661 int len,
1662 caddr_t cp)
1663 {
1664 unsigned count;
1665
1666 if (off < 0 || len < 0)
1667 panic("m_copydata");
1668 while (off > 0) {
1669 if (m == 0)
1670 panic("m_copydata");
1671 if (off < m->m_len)
1672 break;
1673 off -= m->m_len;
1674 m = m->m_next;
1675 }
1676 while (len > 0) {
1677 if (m == 0)
1678 panic("m_copydata");
1679 count = min(m->m_len - off, len);
1680 bcopy(mtod(m, caddr_t) + off, cp, count);
1681 len -= count;
1682 cp += count;
1683 off = 0;
1684 m = m->m_next;
1685 }
1686 }
1687
1688 /*
1689 * Concatenate mbuf chain n to m.
1690 * Both chains must be of the same type (e.g. MT_DATA).
1691 * Any m_pkthdr is not updated.
1692 */
1693 void m_cat(
1694 struct mbuf *m, struct mbuf *n)
1695 {
1696 while (m->m_next)
1697 m = m->m_next;
1698 while (n) {
1699 if (m->m_flags & M_EXT ||
1700 m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) {
1701 /* just join the two chains */
1702 m->m_next = n;
1703 return;
1704 }
1705 /* splat the data from one into the other */
1706 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
1707 (u_int)n->m_len);
1708 m->m_len += n->m_len;
1709 n = m_free(n);
1710 }
1711 }
1712
1713 void
1714 m_adj(
1715 struct mbuf *mp,
1716 int req_len)
1717 {
1718 int len = req_len;
1719 struct mbuf *m;
1720 int count;
1721
1722 if ((m = mp) == NULL)
1723 return;
1724 if (len >= 0) {
1725 /*
1726 * Trim from head.
1727 */
1728 while (m != NULL && len > 0) {
1729 if (m->m_len <= len) {
1730 len -= m->m_len;
1731 m->m_len = 0;
1732 m = m->m_next;
1733 } else {
1734 m->m_len -= len;
1735 m->m_data += len;
1736 len = 0;
1737 }
1738 }
1739 m = mp;
1740 if (m->m_flags & M_PKTHDR)
1741 m->m_pkthdr.len -= (req_len - len);
1742 } else {
1743 /*
1744 * Trim from tail. Scan the mbuf chain,
1745 * calculating its length and finding the last mbuf.
1746 * If the adjustment only affects this mbuf, then just
1747 * adjust and return. Otherwise, rescan and truncate
1748 * after the remaining size.
1749 */
1750 len = -len;
1751 count = 0;
1752 for (;;) {
1753 count += m->m_len;
1754 if (m->m_next == (struct mbuf *)0)
1755 break;
1756 m = m->m_next;
1757 }
1758 if (m->m_len >= len) {
1759 m->m_len -= len;
1760 m = mp;
1761 if (m->m_flags & M_PKTHDR)
1762 m->m_pkthdr.len -= len;
1763 return;
1764 }
1765 count -= len;
1766 if (count < 0)
1767 count = 0;
1768 /*
1769 * Correct length for chain is "count".
1770 * Find the mbuf with last data, adjust its length,
1771 * and toss data from remaining mbufs on chain.
1772 */
1773 m = mp;
1774 if (m->m_flags & M_PKTHDR)
1775 m->m_pkthdr.len = count;
1776 for (; m; m = m->m_next) {
1777 if (m->m_len >= count) {
1778 m->m_len = count;
1779 break;
1780 }
1781 count -= m->m_len;
1782 }
1783 while ((m = m->m_next))
1784 m->m_len = 0;
1785 }
1786 }
1787
1788 /*
1789 * Rearange an mbuf chain so that len bytes are contiguous
1790 * and in the data area of an mbuf (so that mtod and dtom
1791 * will work for a structure of size len). Returns the resulting
1792 * mbuf chain on success, frees it and returns null on failure.
1793 * If there is room, it will add up to max_protohdr-len extra bytes to the
1794 * contiguous region in an attempt to avoid being called next time.
1795 */
1796 int MPFail;
1797
1798 struct mbuf *
1799 m_pullup(
1800 struct mbuf *n,
1801 int len)
1802 {
1803 struct mbuf *m;
1804 int count;
1805 int space;
1806
1807 /*
1808 * If first mbuf has no cluster, and has room for len bytes
1809 * without shifting current data, pullup into it,
1810 * otherwise allocate a new mbuf to prepend to the chain.
1811 */
1812 if ((n->m_flags & M_EXT) == 0 &&
1813 n->m_data + len < &n->m_dat[MLEN] && n->m_next) {
1814 if (n->m_len >= len)
1815 return (n);
1816 m = n;
1817 n = n->m_next;
1818 len -= m->m_len;
1819 } else {
1820 if (len > MHLEN)
1821 goto bad;
1822 MGET(m, M_DONTWAIT, n->m_type);
1823 if (m == 0)
1824 goto bad;
1825 m->m_len = 0;
1826 if (n->m_flags & M_PKTHDR) {
1827 M_COPY_PKTHDR(m, n);
1828 n->m_flags &= ~M_PKTHDR;
1829 }
1830 }
1831 space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
1832 do {
1833 count = min(min(max(len, max_protohdr), space), n->m_len);
1834 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
1835 (unsigned)count);
1836 len -= count;
1837 m->m_len += count;
1838 n->m_len -= count;
1839 space -= count;
1840 if (n->m_len)
1841 n->m_data += count;
1842 else
1843 n = m_free(n);
1844 } while (len > 0 && n);
1845 if (len > 0) {
1846 (void) m_free(m);
1847 goto bad;
1848 }
1849 m->m_next = n;
1850 return (m);
1851 bad:
1852 m_freem(n);
1853 MPFail++;
1854 return (0);
1855 }
1856
1857 /*
1858 * Partition an mbuf chain in two pieces, returning the tail --
1859 * all but the first len0 bytes. In case of failure, it returns NULL and
1860 * attempts to restore the chain to its original state.
1861 */
1862 struct mbuf *
1863 m_split(
1864 struct mbuf *m0,
1865 int len0,
1866 int wait)
1867 {
1868 struct mbuf *m, *n;
1869 unsigned len = len0, remain;
1870
1871 for (m = m0; m && len > m->m_len; m = m->m_next)
1872 len -= m->m_len;
1873 if (m == 0)
1874 return (0);
1875 remain = m->m_len - len;
1876 if (m0->m_flags & M_PKTHDR) {
1877 MGETHDR(n, wait, m0->m_type);
1878 if (n == 0)
1879 return (0);
1880 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
1881 n->m_pkthdr.len = m0->m_pkthdr.len - len0;
1882 m0->m_pkthdr.len = len0;
1883 if (m->m_flags & M_EXT)
1884 goto extpacket;
1885 if (remain > MHLEN) {
1886 /* m can't be the lead packet */
1887 MH_ALIGN(n, 0);
1888 n->m_next = m_split(m, len, wait);
1889 if (n->m_next == 0) {
1890 (void) m_free(n);
1891 return (0);
1892 } else
1893 return (n);
1894 } else
1895 MH_ALIGN(n, remain);
1896 } else if (remain == 0) {
1897 n = m->m_next;
1898 m->m_next = 0;
1899 return (n);
1900 } else {
1901 MGET(n, wait, m->m_type);
1902 if (n == 0)
1903 return (0);
1904 M_ALIGN(n, remain);
1905 }
1906 extpacket:
1907 if (m->m_flags & M_EXT) {
1908 n->m_flags |= M_EXT;
1909 MBUF_LOCK();
1910 n->m_ext = m->m_ext;
1911 insque((queue_t)&n->m_ext.ext_refs, (queue_t)&m->m_ext.ext_refs);
1912 MBUF_UNLOCK();
1913 n->m_data = m->m_data + len;
1914 } else {
1915 bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain);
1916 }
1917 n->m_len = remain;
1918 m->m_len = len;
1919 n->m_next = m->m_next;
1920 m->m_next = 0;
1921 return (n);
1922 }
1923 /*
1924 * Routine to copy from device local memory into mbufs.
1925 */
1926 struct mbuf *
1927 m_devget(
1928 char *buf,
1929 int totlen,
1930 int off0,
1931 struct ifnet *ifp,
1932 void (*copy)(const void *, void *, size_t))
1933 {
1934 struct mbuf *m;
1935 struct mbuf *top = 0, **mp = &top;
1936 int off = off0, len;
1937 char *cp;
1938 char *epkt;
1939
1940 cp = buf;
1941 epkt = cp + totlen;
1942 if (off) {
1943 /*
1944 * If 'off' is non-zero, packet is trailer-encapsulated,
1945 * so we have to skip the type and length fields.
1946 */
1947 cp += off + 2 * sizeof(u_int16_t);
1948 totlen -= 2 * sizeof(u_int16_t);
1949 }
1950 MGETHDR(m, M_DONTWAIT, MT_DATA);
1951 if (m == 0)
1952 return (0);
1953 m->m_pkthdr.rcvif = ifp;
1954 m->m_pkthdr.len = totlen;
1955 m->m_len = MHLEN;
1956
1957 while (totlen > 0) {
1958 if (top) {
1959 MGET(m, M_DONTWAIT, MT_DATA);
1960 if (m == 0) {
1961 m_freem(top);
1962 return (0);
1963 }
1964 m->m_len = MLEN;
1965 }
1966 len = min(totlen, epkt - cp);
1967 if (len >= MINCLSIZE) {
1968 MCLGET(m, M_DONTWAIT);
1969 if (m->m_flags & M_EXT)
1970 m->m_len = len = min(len, MCLBYTES);
1971 else {
1972 /* give up when it's out of cluster mbufs */
1973 if (top)
1974 m_freem(top);
1975 m_freem(m);
1976 return (0);
1977 }
1978 } else {
1979 /*
1980 * Place initial small packet/header at end of mbuf.
1981 */
1982 if (len < m->m_len) {
1983 if (top == 0 && len + max_linkhdr <= m->m_len)
1984 m->m_data += max_linkhdr;
1985 m->m_len = len;
1986 } else
1987 len = m->m_len;
1988 }
1989 if (copy)
1990 copy(cp, mtod(m, caddr_t), (unsigned)len);
1991 else
1992 bcopy(cp, mtod(m, caddr_t), (unsigned)len);
1993 cp += len;
1994 *mp = m;
1995 mp = &m->m_next;
1996 totlen -= len;
1997 if (cp == epkt)
1998 cp = buf;
1999 }
2000 return (top);
2001 }
2002
2003 /*
2004 * Cluster freelist allocation check. The mbuf lock must be held.
2005 * Ensure hysteresis between hi/lo.
2006 */
2007 static int
2008 m_howmany(int num, size_t bufsize)
2009 {
2010 int i = 0;
2011
2012 /* Bail if we've maxed out the mbuf memory map */
2013 if (mbstat.m_clusters + (mbstat.m_bigclusters << 1) < nmbclusters) {
2014 int j = 0;
2015
2016 if (bufsize == MCLBYTES) {
2017 /* Under minimum */
2018 if (mbstat.m_clusters < MINCL)
2019 return (MINCL - mbstat.m_clusters);
2020 /* Too few (free < 1/2 total) and not over maximum */
2021 if (mbstat.m_clusters < (nmbclusters >> 1)) {
2022 if (num >= mbstat.m_clfree)
2023 i = num - mbstat.m_clfree;
2024 if (((mbstat.m_clusters + num) >> 1) > mbstat.m_clfree)
2025 j = ((mbstat.m_clusters + num) >> 1) - mbstat.m_clfree;
2026 i = max(i, j);
2027 if (i + mbstat.m_clusters >= (nmbclusters >> 1))
2028 i = (nmbclusters >> 1) - mbstat.m_clusters;
2029 }
2030 } else {
2031 /* Under minimum */
2032 if (mbstat.m_bigclusters < MINCL)
2033 return (MINCL - mbstat.m_bigclusters);
2034 /* Too few (free < 1/2 total) and not over maximum */
2035 if (mbstat.m_bigclusters < (nmbclusters >> 2)) {
2036 if (num >= mbstat.m_bigclfree)
2037 i = num - mbstat.m_bigclfree;
2038 if (((mbstat.m_bigclusters + num) >> 1) > mbstat.m_bigclfree)
2039 j = ((mbstat.m_bigclusters + num) >> 1) - mbstat.m_bigclfree;
2040 i = max(i, j);
2041 if (i + mbstat.m_bigclusters >= (nmbclusters >> 2))
2042 i = (nmbclusters >> 2) - mbstat.m_bigclusters;
2043 }
2044 }
2045 }
2046 return i;
2047 }
2048
2049 /*
2050 * Copy data from a buffer back into the indicated mbuf chain,
2051 * starting "off" bytes from the beginning, extending the mbuf
2052 * chain if necessary.
2053 */
2054 void
2055 m_copyback(
2056 struct mbuf *m0,
2057 int off,
2058 int len,
2059 caddr_t cp)
2060 {
2061 int mlen;
2062 struct mbuf *m = m0, *n;
2063 int totlen = 0;
2064
2065 if (m0 == 0)
2066 return;
2067 while (off > (mlen = m->m_len)) {
2068 off -= mlen;
2069 totlen += mlen;
2070 if (m->m_next == 0) {
2071 n = m_getclr(M_DONTWAIT, m->m_type);
2072 if (n == 0)
2073 goto out;
2074 n->m_len = min(MLEN, len + off);
2075 m->m_next = n;
2076 }
2077 m = m->m_next;
2078 }
2079 while (len > 0) {
2080 mlen = min (m->m_len - off, len);
2081 bcopy(cp, off + mtod(m, caddr_t), (unsigned)mlen);
2082 cp += mlen;
2083 len -= mlen;
2084 mlen += off;
2085 off = 0;
2086 totlen += mlen;
2087 if (len == 0)
2088 break;
2089 if (m->m_next == 0) {
2090 n = m_get(M_DONTWAIT, m->m_type);
2091 if (n == 0)
2092 break;
2093 n->m_len = min(MLEN, len);
2094 m->m_next = n;
2095 }
2096 m = m->m_next;
2097 }
2098 out: if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen))
2099 m->m_pkthdr.len = totlen;
2100 }
2101
2102
2103 char *mcl_to_paddr(char *addr) {
2104 int base_phys;
2105
2106 if (addr < (char *)mbutl || addr >= (char *)embutl)
2107 return (0);
2108 base_phys = mcl_paddr[(addr - (char *)mbutl) >> PGSHIFT];
2109
2110 if (base_phys == 0)
2111 return (0);
2112 return ((char *)((int)base_phys | ((int)addr & PGOFSET)));
2113 }
2114
2115 /*
2116 * Dup the mbuf chain passed in. The whole thing. No cute additional cruft.
2117 * And really copy the thing. That way, we don't "precompute" checksums
2118 * for unsuspecting consumers.
2119 * Assumption: m->m_nextpkt == 0.
2120 * Trick: for small packets, don't dup into a cluster. That way received
2121 * packets don't take up too much room in the sockbuf (cf. sbspace()).
2122 */
2123 int MDFail;
2124
2125 struct mbuf *
2126 m_dup(struct mbuf *m, int how)
2127 {
2128 struct mbuf *n, **np;
2129 struct mbuf *top;
2130 int copyhdr = 0;
2131
2132 np = &top;
2133 top = 0;
2134 if (m->m_flags & M_PKTHDR)
2135 copyhdr = 1;
2136
2137 /*
2138 * Quick check: if we have one mbuf and its data fits in an
2139 * mbuf with packet header, just copy and go.
2140 */
2141 if (m->m_next == NULL)
2142 { /* Then just move the data into an mbuf and be done... */
2143 if (copyhdr)
2144 { if (m->m_pkthdr.len <= MHLEN)
2145 { if ((n = m_gethdr(how, m->m_type)) == NULL)
2146 return(NULL);
2147 n->m_len = m->m_len;
2148 m_dup_pkthdr(n, m, how);
2149 bcopy(m->m_data, n->m_data, m->m_len);
2150 return(n);
2151 }
2152 } else if (m->m_len <= MLEN)
2153 { if ((n = m_get(how, m->m_type)) == NULL)
2154 return(NULL);
2155 bcopy(m->m_data, n->m_data, m->m_len);
2156 n->m_len = m->m_len;
2157 return(n);
2158 }
2159 }
2160 while (m)
2161 {
2162 #if BLUE_DEBUG
2163 kprintf("<%x: %x, %x, %x\n", m, m->m_flags, m->m_len,
2164 m->m_data);
2165 #endif
2166 if (copyhdr)
2167 n = m_gethdr(how, m->m_type);
2168 else
2169 n = m_get(how, m->m_type);
2170 if (n == 0)
2171 goto nospace;
2172 if (m->m_flags & M_EXT)
2173 { MCLGET(n, how);
2174 if ((n->m_flags & M_EXT) == 0)
2175 goto nospace;
2176 }
2177 *np = n;
2178 if (copyhdr)
2179 { /* Don't use M_COPY_PKTHDR: preserve m_data */
2180 m_dup_pkthdr(n, m, how);
2181 copyhdr = 0;
2182 if ((n->m_flags & M_EXT) == 0)
2183 n->m_data = n->m_pktdat;
2184 }
2185 n->m_len = m->m_len;
2186 /*
2187 * Get the dup on the same bdry as the original
2188 * Assume that the two mbufs have the same offset to data area
2189 * (up to word bdries)
2190 */
2191 bcopy(mtod(m, caddr_t), mtod(n, caddr_t), (unsigned)n->m_len);
2192 m = m->m_next;
2193 np = &n->m_next;
2194 #if BLUE_DEBUG
2195 kprintf(">%x: %x, %x, %x\n", n, n->m_flags, n->m_len,
2196 n->m_data);
2197 #endif
2198 }
2199
2200 if (top == 0)
2201 MDFail++;
2202 return (top);
2203 nospace:
2204 m_freem(top);
2205 MDFail++;
2206 return (0);
2207 }
2208
2209 int
2210 m_mclref(struct mbuf *p)
2211 {
2212 return (_MCLREF(p));
2213 }
2214
2215 int
2216 m_mclunref(struct mbuf *p)
2217 {
2218 return (_MCLUNREF(p));
2219 }
2220
2221 /* change mbuf to new type */
2222 void
2223 m_mchtype(struct mbuf *m, int t)
2224 {
2225 MBUF_LOCK();
2226 mbstat.m_mtypes[(m)->m_type]--;
2227 mbstat.m_mtypes[t]++;
2228 (m)->m_type = t;
2229 MBUF_UNLOCK();
2230 }
2231
2232 void *m_mtod(struct mbuf *m)
2233 {
2234 return ((m)->m_data);
2235 }
2236
2237 struct mbuf *m_dtom(void *x)
2238 {
2239 return ((struct mbuf *)((u_long)(x) & ~(MSIZE-1)));
2240 }
2241
2242 int m_mtocl(void *x)
2243 {
2244 return (((char *)(x) - (char *)mbutl) / sizeof(union mcluster));
2245 }
2246
2247 union mcluster *m_cltom(int x)
2248 {
2249 return ((union mcluster *)(mbutl + (x)));
2250 }
2251
2252
2253 void m_mcheck(struct mbuf *m)
2254 {
2255 if (m->m_type != MT_FREE)
2256 panic("mget MCHECK: m_type=%x m=%x", m->m_type, m);
2257 }
2258
2259 static void
2260 mbuf_expand_thread(void)
2261 {
2262 while (1) {
2263 MBUF_LOCK();
2264 if (mbuf_expand_mcl) {
2265 int n;
2266
2267 /* Adjust to the current number of cluster in use */
2268 n = mbuf_expand_mcl - (mbstat.m_clusters - mbstat.m_clfree);
2269 mbuf_expand_mcl = 0;
2270
2271 if (n > 0)
2272 (void)m_clalloc(n, M_WAIT, MCLBYTES, 1);
2273 }
2274 if (mbuf_expand_big) {
2275 int n;
2276
2277 /* Adjust to the current number of 4 KB cluster in use */
2278 n = mbuf_expand_big - (mbstat.m_bigclusters - mbstat.m_bigclfree);
2279 mbuf_expand_big = 0;
2280
2281 if (n > 0)
2282 (void)m_clalloc(n, M_WAIT, NBPG, 1);
2283 }
2284 MBUF_UNLOCK();
2285 /*
2286 * Because we can run out of memory before filling the mbuf map, we
2287 * should not allocate more clusters than they are mbufs -- otherwise
2288 * we could have a large number of useless clusters allocated.
2289 */
2290 while (mbstat.m_mbufs < mbstat.m_bigclusters + mbstat.m_clusters) {
2291 if (m_expand(M_WAIT) == 0)
2292 break;
2293 }
2294
2295 assert_wait(&mbuf_expand_thread_wakeup, THREAD_UNINT);
2296 (void) thread_block((thread_continue_t)mbuf_expand_thread);
2297 }
2298 }
2299
2300 static void
2301 mbuf_expand_thread_init(void)
2302 {
2303 mbuf_expand_thread_initialized++;
2304 mbuf_expand_thread();
2305 }
2306
2307 SYSCTL_DECL(_kern_ipc);
2308 SYSCTL_STRUCT(_kern_ipc, KIPC_MBSTAT, mbstat, CTLFLAG_RW, &mbstat, mbstat, "");
2309
mirror of XNU