]> git.saurik.com Git - apple/xnu.git/blob - bsd/kern/uipc_mbuf.c
xnu-517.11.1.tar.gz
[apple/xnu.git] / bsd / kern / uipc_mbuf.c
1 /*
2 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * The contents of this file constitute Original Code as defined in and
7 * are subject to the Apple Public Source License Version 1.1 (the
8 * "License"). You may not use this file except in compliance with the
9 * License. Please obtain a copy of the License at
10 * http://www.apple.com/publicsource and read it before using this file.
11 *
12 * This Original Code and all software distributed under the License are
13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17 * License for the specific language governing rights and limitations
18 * under the License.
19 *
20 * @APPLE_LICENSE_HEADER_END@
21 */
22 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
23 /*
24 * Copyright (c) 1982, 1986, 1988, 1991, 1993
25 * The Regents of the University of California. All rights reserved.
26 *
27 * Redistribution and use in source and binary forms, with or without
28 * modification, are permitted provided that the following conditions
29 * are met:
30 * 1. Redistributions of source code must retain the above copyright
31 * notice, this list of conditions and the following disclaimer.
32 * 2. Redistributions in binary form must reproduce the above copyright
33 * notice, this list of conditions and the following disclaimer in the
34 * documentation and/or other materials provided with the distribution.
35 * 3. All advertising materials mentioning features or use of this software
36 * must display the following acknowledgement:
37 * This product includes software developed by the University of
38 * California, Berkeley and its contributors.
39 * 4. Neither the name of the University nor the names of its contributors
40 * may be used to endorse or promote products derived from this software
41 * without specific prior written permission.
42 *
43 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
44 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
45 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
46 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
47 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
48 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
49 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
50 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
51 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
52 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
53 * SUCH DAMAGE.
54 *
55 * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94
56 */
57 /* HISTORY
58 *
59 * 10/15/97 Annette DeSchon (deschon@apple.com)
60 * Fixed bug in which all cluster mbufs were broken up
61 * into regular mbufs: Some clusters are now reserved.
62 * When a cluster is needed, regular mbufs are no longer
63 * used. (Radar 1683621)
64 * 20-May-95 Mac Gillon (mgillon) at NeXT
65 * New version based on 4.4
66 */
67
68 #include <sys/param.h>
69 #include <sys/systm.h>
70 #include <sys/malloc.h>
71 #include <sys/mbuf.h>
72 #include <sys/kernel.h>
73 #include <sys/syslog.h>
74 #include <sys/protosw.h>
75 #include <sys/domain.h>
76 #include <net/netisr.h>
77
78 #include <kern/queue.h>
79 #include <kern/kern_types.h>
80 #include <kern/sched_prim.h>
81
82 #include <IOKit/IOMapper.h>
83
84 #define _MCLREF(p) (++mclrefcnt[mtocl(p)])
85 #define _MCLUNREF(p) (--mclrefcnt[mtocl(p)] == 0)
86 #define _M_CLEAR_PKTHDR(mbuf_ptr) (mbuf_ptr)->m_pkthdr.rcvif = NULL; \
87 (mbuf_ptr)->m_pkthdr.len = 0; \
88 (mbuf_ptr)->m_pkthdr.header = NULL; \
89 (mbuf_ptr)->m_pkthdr.csum_flags = 0; \
90 (mbuf_ptr)->m_pkthdr.csum_data = 0; \
91 (mbuf_ptr)->m_pkthdr.aux = (struct mbuf*)NULL; \
92 (mbuf_ptr)->m_pkthdr.reserved_1 = 0; \
93 (mbuf_ptr)->m_pkthdr.vlan_tag = 0; \
94 (mbuf_ptr)->m_pkthdr.reserved2 = NULL;
95
96 extern pmap_t kernel_pmap; /* The kernel's pmap */
97 /* kernel translater */
98 extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
99
100 decl_simple_lock_data(, mbuf_slock);
101 struct mbuf *mfree; /* mbuf free list */
102 struct mbuf *mfreelater; /* mbuf deallocation list */
103 extern vm_map_t mb_map; /* special map */
104 int m_want; /* sleepers on mbufs */
105 extern int nmbclusters; /* max number of mapped clusters */
106 short *mclrefcnt; /* mapped cluster reference counts */
107 int *mcl_paddr;
108 static ppnum_t mcl_paddr_base; /* Handle returned by IOMapper::iovmAlloc() */
109 union mcluster *mclfree; /* mapped cluster free list */
110 int max_linkhdr; /* largest link-level header */
111 int max_protohdr; /* largest protocol header */
112 int max_hdr; /* largest link+protocol header */
113 int max_datalen; /* MHLEN - max_hdr */
114 struct mbstat mbstat; /* statistics */
115 union mcluster *mbutl; /* first mapped cluster address */
116 union mcluster *embutl; /* ending virtual address of mclusters */
117
118 static int nclpp; /* # clusters per physical page */
119 static char mbfail[] = "mbuf not mapped";
120
121 static int m_howmany();
122
123 /* The number of cluster mbufs that are allocated, to start. */
124 #define MINCL max(16, 2)
125
126 static int mbuf_expand_thread_wakeup = 0;
127 static int mbuf_expand_mcl = 0;
128 static int mbuf_expand_thread_initialized = 0;
129
130 static void mbuf_expand_thread_init(void);
131
132 #if 0
133 static int mfree_munge = 0;
134 #if 0
135 #define _MFREE_MUNGE(m) { \
136 if (mfree_munge) \
137 { int i; \
138 vm_offset_t *element = (vm_offset_t *)(m); \
139 for (i = 0; \
140 i < sizeof(struct mbuf)/sizeof(vm_offset_t); \
141 i++) \
142 (element)[i] = 0xdeadbeef; \
143 } \
144 }
145 #else
146 void
147 munge_mbuf(struct mbuf *m)
148 {
149 int i;
150 vm_offset_t *element = (vm_offset_t *)(m);
151 for (i = 0;
152 i < sizeof(struct mbuf)/sizeof(vm_offset_t);
153 i++)
154 (element)[i] = 0xdeadbeef;
155 }
156 #define _MFREE_MUNGE(m) { \
157 if (mfree_munge) \
158 munge_mbuf(m); \
159 }
160 #endif
161 #else
162 #define _MFREE_MUNGE(m)
163 #endif
164
165
166 #define _MINTGET(m, type) { \
167 MBUF_LOCK(); \
168 if (((m) = mfree) != 0) { \
169 MCHECK(m); \
170 ++mclrefcnt[mtocl(m)]; \
171 mbstat.m_mtypes[MT_FREE]--; \
172 mbstat.m_mtypes[(type)]++; \
173 mfree = (m)->m_next; \
174 } \
175 MBUF_UNLOCK(); \
176 }
177
178
179 void
180 mbinit()
181 {
182 int s,m;
183 int initmcl = 32;
184 int mcl_pages;
185
186 if (nclpp)
187 return;
188 nclpp = round_page_32(MCLBYTES) / MCLBYTES; /* see mbufgc() */
189 if (nclpp < 1) nclpp = 1;
190 MBUF_LOCKINIT();
191 // NETISR_LOCKINIT();
192
193 mbstat.m_msize = MSIZE;
194 mbstat.m_mclbytes = MCLBYTES;
195 mbstat.m_minclsize = MINCLSIZE;
196 mbstat.m_mlen = MLEN;
197 mbstat.m_mhlen = MHLEN;
198
199 if (nmbclusters == 0)
200 nmbclusters = NMBCLUSTERS;
201 MALLOC(mclrefcnt, short *, nmbclusters * sizeof (short),
202 M_TEMP, M_WAITOK);
203 if (mclrefcnt == 0)
204 panic("mbinit");
205 for (m = 0; m < nmbclusters; m++)
206 mclrefcnt[m] = -1;
207
208 /* Calculate the number of pages assigned to the cluster pool */
209 mcl_pages = nmbclusters/(PAGE_SIZE/CLBYTES);
210 MALLOC(mcl_paddr, int *, mcl_pages * sizeof(int), M_TEMP, M_WAITOK);
211 if (mcl_paddr == 0)
212 panic("mbinit1");
213 /* Register with the I/O Bus mapper */
214 mcl_paddr_base = IOMapperIOVMAlloc(mcl_pages);
215 bzero((char *)mcl_paddr, mcl_pages * sizeof(int));
216
217 embutl = (union mcluster *)((unsigned char *)mbutl + (nmbclusters * MCLBYTES));
218
219 PE_parse_boot_arg("initmcl", &initmcl);
220
221 if (m_clalloc(max(PAGE_SIZE/CLBYTES, 1) * initmcl, M_WAIT) == 0)
222 goto bad;
223 MBUF_UNLOCK();
224
225 (void) kernel_thread(kernel_task, mbuf_expand_thread_init);
226
227 return;
228 bad:
229 panic("mbinit");
230 }
231
232 /*
233 * Allocate some number of mbuf clusters
234 * and place on cluster free list.
235 */
236 /* ARGSUSED */
237 m_clalloc(ncl, nowait)
238 register int ncl;
239 int nowait;
240 {
241 register union mcluster *mcl;
242 register int i;
243 vm_size_t size;
244 static char doing_alloc;
245
246 /*
247 * Honor the caller's wish to block or not block.
248 * We have a way to grow the pool asynchronously,
249 * by kicking the dlil_input_thread.
250 */
251 if ((i = m_howmany()) <= 0)
252 goto out;
253
254 if ((nowait == M_DONTWAIT))
255 goto out;
256
257 if (ncl < i)
258 ncl = i;
259 size = round_page_32(ncl * MCLBYTES);
260 mcl = (union mcluster *)kmem_mb_alloc(mb_map, size);
261
262 if (mcl == 0 && ncl > 1) {
263 size = round_page_32(MCLBYTES); /* Try for 1 if failed */
264 mcl = (union mcluster *)kmem_mb_alloc(mb_map, size);
265 }
266
267 if (mcl) {
268 MBUF_LOCK();
269 ncl = size / MCLBYTES;
270 for (i = 0; i < ncl; i++) {
271 if (++mclrefcnt[mtocl(mcl)] != 0)
272 panic("m_clalloc already there");
273 if (((int)mcl & PAGE_MASK) == 0) {
274 ppnum_t offset = ((char *)mcl - (char *)mbutl)/PAGE_SIZE;
275 ppnum_t new_page = pmap_find_phys(kernel_pmap, (vm_address_t) mcl);
276
277 /*
278 * In the case of no mapper being available
279 * the following code nops and returns the
280 * input page, if there is a mapper the I/O
281 * page appropriate is returned.
282 */
283 new_page = IOMapperInsertPage(mcl_paddr_base, offset, new_page);
284 mcl_paddr[offset] = new_page << 12;
285 }
286
287 mcl->mcl_next = mclfree;
288 mclfree = mcl++;
289 }
290 mbstat.m_clfree += ncl;
291 mbstat.m_clusters += ncl;
292 return (ncl);
293 } /* else ... */
294 out:
295 MBUF_LOCK();
296
297 /*
298 * When non-blocking we kick the dlil thread if we havve to grow the
299 * pool or if the number of free clusters is less than requested.
300 */
301 if ((nowait == M_DONTWAIT) && (i > 0 || ncl >= mbstat.m_clfree)) {
302 mbuf_expand_mcl = 1;
303 if (mbuf_expand_thread_initialized)
304 wakeup((caddr_t)&mbuf_expand_thread_wakeup);
305 }
306
307 if (mbstat.m_clfree >= ncl)
308 return 1;
309
310 return 0;
311 }
312
313 /*
314 * Add more free mbufs by cutting up a cluster.
315 */
316 m_expand(canwait)
317 int canwait;
318 {
319 register caddr_t mcl;
320
321 if (mbstat.m_clfree < (mbstat.m_clusters >> 4))
322 /* 1/16th of the total number of cluster mbufs allocated is
323 reserved for large packets. The number reserved must
324 always be < 1/2, or future allocation will be prevented.
325 */
326 return 0;
327
328 MCLALLOC(mcl, canwait);
329 if (mcl) {
330 register struct mbuf *m = (struct mbuf *)mcl;
331 register int i = NMBPCL;
332 MBUF_LOCK();
333 mbstat.m_mtypes[MT_FREE] += i;
334 mbstat.m_mbufs += i;
335 while (i--) {
336 _MFREE_MUNGE(m);
337 m->m_type = MT_FREE;
338 m->m_next = mfree;
339 mfree = m++;
340 }
341 i = m_want;
342 m_want = 0;
343 MBUF_UNLOCK();
344 if (i) wakeup((caddr_t)&mfree);
345 return 1;
346 }
347 return 0;
348 }
349
350 /*
351 * When MGET failes, ask protocols to free space when short of memory,
352 * then re-attempt to allocate an mbuf.
353 */
354 struct mbuf *
355 m_retry(canwait, type)
356 int canwait, type;
357 {
358 register struct mbuf *m;
359 int wait, s;
360 funnel_t * fnl;
361 int fnl_switch = 0;
362 boolean_t funnel_state;
363
364 for (;;) {
365 (void) m_expand(canwait);
366 _MINTGET(m, type);
367 if (m) {
368 (m)->m_next = (m)->m_nextpkt = 0;
369 (m)->m_type = (type);
370 (m)->m_data = (m)->m_dat;
371 (m)->m_flags = 0;
372 }
373 if (m || canwait == M_DONTWAIT)
374 break;
375 MBUF_LOCK();
376 wait = m_want++;
377 mbuf_expand_mcl = 1;
378 if (wait == 0)
379 mbstat.m_drain++;
380 else
381 mbstat.m_wait++;
382 MBUF_UNLOCK();
383
384 if (mbuf_expand_thread_initialized)
385 wakeup((caddr_t)&mbuf_expand_thread_wakeup);
386
387 /*
388 * Need to be inside network funnel for m_reclaim because it calls into the
389 * socket domains and tsleep end-up calling splhigh
390 */
391 fnl = thread_funnel_get();
392 if (wait == 0 && fnl == network_flock) {
393 m_reclaim();
394 } else if (fnl != THR_FUNNEL_NULL) {
395 /* Sleep with a small timeout as insurance */
396 (void) tsleep((caddr_t)&mfree, PZERO-1, "m_retry", hz);
397 } else {
398 /* We are called from a non-BSD context: use mach primitives */
399 u_int64_t abstime = 0;
400
401 assert_wait((event_t)&mfree, THREAD_UNINT);
402 clock_interval_to_deadline(hz, NSEC_PER_SEC / hz, &abstime);
403 thread_set_timer_deadline(abstime);
404 if (thread_block(THREAD_CONTINUE_NULL) != THREAD_TIMED_OUT)
405 thread_cancel_timer();
406 }
407 }
408 if (m == 0)
409 mbstat.m_drops++;
410 return (m);
411 }
412
413 /*
414 * As above; retry an MGETHDR.
415 */
416 struct mbuf *
417 m_retryhdr(canwait, type)
418 int canwait, type;
419 {
420 register struct mbuf *m;
421
422 if (m = m_retry(canwait, type)) {
423 m->m_flags |= M_PKTHDR;
424 m->m_data = m->m_pktdat;
425 _M_CLEAR_PKTHDR(m);
426 }
427 return (m);
428 }
429
430 m_reclaim()
431 {
432 register struct domain *dp;
433 register struct protosw *pr;
434
435 for (dp = domains; dp; dp = dp->dom_next)
436 for (pr = dp->dom_protosw; pr; pr = pr->pr_next)
437 if (pr->pr_drain)
438 (*pr->pr_drain)();
439 mbstat.m_drain++;
440 }
441
442 /*
443 * Space allocation routines.
444 * These are also available as macros
445 * for critical paths.
446 */
447 struct mbuf *
448 m_get(nowait, type)
449 int nowait, type;
450 {
451 register struct mbuf *m;
452
453 _MINTGET(m, type);
454 if (m) {
455 m->m_next = m->m_nextpkt = 0;
456 m->m_type = type;
457 m->m_data = m->m_dat;
458 m->m_flags = 0;
459 } else
460 (m) = m_retry(nowait, type);
461
462 return (m);
463 }
464
465 struct mbuf *
466 m_gethdr(nowait, type)
467 int nowait, type;
468 {
469 register struct mbuf *m;
470
471 _MINTGET(m, type);
472 if (m) {
473 m->m_next = m->m_nextpkt = 0;
474 m->m_type = type;
475 m->m_data = m->m_pktdat;
476 m->m_flags = M_PKTHDR;
477 _M_CLEAR_PKTHDR(m)
478 } else
479 m = m_retryhdr(nowait, type);
480
481 return m;
482 }
483
484 struct mbuf *
485 m_getclr(nowait, type)
486 int nowait, type;
487 {
488 register struct mbuf *m;
489
490 MGET(m, nowait, type);
491 if (m == 0)
492 return (0);
493 bzero(mtod(m, caddr_t), MLEN);
494 return (m);
495 }
496
497 struct mbuf *
498 m_free(m)
499 struct mbuf *m;
500 {
501 struct mbuf *n = m->m_next;
502 int i, s;
503
504 if (m->m_type == MT_FREE)
505 panic("freeing free mbuf");
506
507 /* Free the aux data if there is any */
508 if ((m->m_flags & M_PKTHDR) && m->m_pkthdr.aux)
509 {
510 m_freem(m->m_pkthdr.aux);
511 }
512
513 MBUF_LOCK();
514 if ((m->m_flags & M_EXT))
515 {
516 if (MCLHASREFERENCE(m)) {
517 remque((queue_t)&m->m_ext.ext_refs);
518 } else if (m->m_ext.ext_free == NULL) {
519 union mcluster *mcl= (union mcluster *)m->m_ext.ext_buf;
520 if (_MCLUNREF(mcl)) {
521 mcl->mcl_next = mclfree;
522 mclfree = mcl;
523 ++mbstat.m_clfree;
524 }
525 #ifdef COMMENT_OUT
526 /* *** Since m_split() increments "mclrefcnt[mtocl(m->m_ext.ext_buf)]",
527 and AppleTalk ADSP uses m_split(), this incorrect sanity check
528 caused a panic.
529 *** */
530 else /* sanity check - not referenced this way */
531 panic("m_free m_ext cluster not free");
532 #endif
533 } else {
534 (*(m->m_ext.ext_free))(m->m_ext.ext_buf,
535 m->m_ext.ext_size, m->m_ext.ext_arg);
536 }
537 }
538 mbstat.m_mtypes[m->m_type]--;
539 (void) _MCLUNREF(m);
540 _MFREE_MUNGE(m);
541 m->m_type = MT_FREE;
542 mbstat.m_mtypes[m->m_type]++;
543 m->m_flags = 0;
544 m->m_next = mfree;
545 m->m_len = 0;
546 mfree = m;
547 i = m_want;
548 m_want = 0;
549 MBUF_UNLOCK();
550 if (i) wakeup((caddr_t)&mfree);
551 return (n);
552 }
553
554 /* m_mclget() add an mbuf cluster to a normal mbuf */
555 struct mbuf *
556 m_mclget(m, nowait)
557 struct mbuf *m;
558 int nowait;
559 {
560 MCLALLOC(m->m_ext.ext_buf, nowait);
561 if (m->m_ext.ext_buf) {
562 m->m_data = m->m_ext.ext_buf;
563 m->m_flags |= M_EXT;
564 m->m_ext.ext_size = MCLBYTES;
565 m->m_ext.ext_free = 0;
566 m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward =
567 &m->m_ext.ext_refs;
568 }
569
570 return m;
571 }
572
573 /* m_mclalloc() allocate an mbuf cluster */
574 caddr_t
575 m_mclalloc( nowait)
576 int nowait;
577 {
578 caddr_t p;
579
580 (void)m_clalloc(1, nowait);
581 if ((p = (caddr_t)mclfree)) {
582 ++mclrefcnt[mtocl(p)];
583 mbstat.m_clfree--;
584 mclfree = ((union mcluster *)p)->mcl_next;
585 } else {
586 mbstat.m_drops++;
587 }
588 MBUF_UNLOCK();
589
590 return p;
591 }
592
593 /* m_mclfree() releases a reference to a cluster allocated by MCLALLOC,
594 * freeing the cluster if the reference count has reached 0. */
595 void
596 m_mclfree(p)
597 caddr_t p;
598 {
599 MBUF_LOCK();
600 if (--mclrefcnt[mtocl(p)] == 0) {
601 ((union mcluster *)(p))->mcl_next = mclfree;
602 mclfree = (union mcluster *)(p);
603 mbstat.m_clfree++;
604 }
605 MBUF_UNLOCK();
606 }
607
608 /* mcl_hasreference() checks if a cluster of an mbuf is referenced by another mbuf */
609 int
610 m_mclhasreference(m)
611 struct mbuf *m;
612 {
613 return (m->m_ext.ext_refs.forward != &(m->m_ext.ext_refs));
614 }
615
616 /* */
617 void
618 m_copy_pkthdr(to, from)
619 struct mbuf *to, *from;
620 {
621 to->m_pkthdr = from->m_pkthdr;
622 from->m_pkthdr.aux = (struct mbuf *)NULL;
623 to->m_flags = from->m_flags & M_COPYFLAGS;
624 to->m_data = (to)->m_pktdat;
625 }
626
627 /* Best effort to get a mbuf cluster + pkthdr under one lock.
628 * If we don't have them avail, just bail out and use the regular
629 * path.
630 * Used by drivers to allocated packets on receive ring.
631 */
632 struct mbuf *
633 m_getpacket(void)
634 {
635 struct mbuf *m;
636 m_clalloc(1, M_DONTWAIT); /* takes the MBUF_LOCK, but doesn't release it... */
637 if ((mfree != 0) && (mclfree != 0)) { /* mbuf + cluster are available */
638 m = mfree;
639 mfree = m->m_next;
640 MCHECK(m);
641 ++mclrefcnt[mtocl(m)];
642 mbstat.m_mtypes[MT_FREE]--;
643 mbstat.m_mtypes[MT_DATA]++;
644 m->m_ext.ext_buf = (caddr_t)mclfree; /* get the cluster */
645 ++mclrefcnt[mtocl(m->m_ext.ext_buf)];
646 mbstat.m_clfree--;
647 mclfree = ((union mcluster *)(m->m_ext.ext_buf))->mcl_next;
648
649 m->m_next = m->m_nextpkt = 0;
650 m->m_type = MT_DATA;
651 m->m_data = m->m_ext.ext_buf;
652 m->m_flags = M_PKTHDR | M_EXT;
653 _M_CLEAR_PKTHDR(m)
654 m->m_ext.ext_free = 0;
655 m->m_ext.ext_size = MCLBYTES;
656 m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward =
657 &m->m_ext.ext_refs;
658 MBUF_UNLOCK();
659 }
660 else { /* slow path: either mbuf or cluster need to be allocated anyway */
661 MBUF_UNLOCK();
662
663 MGETHDR(m, M_WAITOK, MT_DATA );
664
665 if ( m == 0 )
666 return (NULL);
667
668 MCLGET( m, M_WAITOK );
669 if ( ( m->m_flags & M_EXT ) == 0 )
670 {
671 m_free(m); m = 0;
672 }
673 }
674 return (m);
675 }
676
677
678 /*
679 * return a list of mbuf hdrs that point to clusters...
680 * try for num_needed, if this can't be met, return whatever
681 * number were available... set up the first num_with_pkthdrs
682 * with mbuf hdrs configured as packet headers... these are
683 * chained on the m_nextpkt field... any packets requested beyond
684 * this are chained onto the last packet header's m_next field.
685 */
686 struct mbuf *
687 m_getpackets(int num_needed, int num_with_pkthdrs, int how)
688 {
689 struct mbuf *m;
690 struct mbuf **np, *top;
691
692 top = NULL;
693 np = &top;
694
695 m_clalloc(num_needed, how); /* takes the MBUF_LOCK, but doesn't release it... */
696
697 while (num_needed--) {
698 if (mfree && mclfree) { /* mbuf + cluster are available */
699 m = mfree;
700 MCHECK(m);
701 mfree = m->m_next;
702 ++mclrefcnt[mtocl(m)];
703 mbstat.m_mtypes[MT_FREE]--;
704 mbstat.m_mtypes[MT_DATA]++;
705 m->m_ext.ext_buf = (caddr_t)mclfree; /* get the cluster */
706 ++mclrefcnt[mtocl(m->m_ext.ext_buf)];
707 mbstat.m_clfree--;
708 mclfree = ((union mcluster *)(m->m_ext.ext_buf))->mcl_next;
709
710 m->m_next = m->m_nextpkt = 0;
711 m->m_type = MT_DATA;
712 m->m_data = m->m_ext.ext_buf;
713 m->m_ext.ext_free = 0;
714 m->m_ext.ext_size = MCLBYTES;
715 m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward = &m->m_ext.ext_refs;
716
717 if (num_with_pkthdrs == 0)
718 m->m_flags = M_EXT;
719 else {
720 m->m_flags = M_PKTHDR | M_EXT;
721 _M_CLEAR_PKTHDR(m);
722
723 num_with_pkthdrs--;
724 }
725
726 } else {
727
728 MBUF_UNLOCK();
729
730 if (num_with_pkthdrs == 0) {
731 MGET(m, how, MT_DATA );
732 } else {
733 MGETHDR(m, how, MT_DATA);
734
735 num_with_pkthdrs--;
736 }
737 if (m == 0)
738 return(top);
739
740 MCLGET(m, how);
741 if ((m->m_flags & M_EXT) == 0) {
742 m_free(m);
743 return(top);
744 }
745 MBUF_LOCK();
746 }
747 *np = m;
748
749 if (num_with_pkthdrs)
750 np = &m->m_nextpkt;
751 else
752 np = &m->m_next;
753 }
754 MBUF_UNLOCK();
755
756 return (top);
757 }
758
759
760 /*
761 * return a list of mbuf hdrs set up as packet hdrs
762 * chained together on the m_nextpkt field
763 */
764 struct mbuf *
765 m_getpackethdrs(int num_needed, int how)
766 {
767 struct mbuf *m;
768 struct mbuf **np, *top;
769
770 top = NULL;
771 np = &top;
772
773 MBUF_LOCK();
774
775 while (num_needed--) {
776 if (m = mfree) { /* mbufs are available */
777 MCHECK(m);
778 mfree = m->m_next;
779 ++mclrefcnt[mtocl(m)];
780 mbstat.m_mtypes[MT_FREE]--;
781 mbstat.m_mtypes[MT_DATA]++;
782
783 m->m_next = m->m_nextpkt = 0;
784 m->m_type = MT_DATA;
785 m->m_flags = M_PKTHDR;
786 m->m_data = m->m_pktdat;
787 _M_CLEAR_PKTHDR(m);
788
789 } else {
790
791 MBUF_UNLOCK();
792
793 m = m_retryhdr(how, MT_DATA);
794
795 if (m == 0)
796 return(top);
797
798 MBUF_LOCK();
799 }
800 *np = m;
801 np = &m->m_nextpkt;
802 }
803 MBUF_UNLOCK();
804
805 return (top);
806 }
807
808
809 /* free and mbuf list (m_nextpkt) while following m_next under one lock.
810 * returns the count for mbufs packets freed. Used by the drivers.
811 */
812 int
813 m_freem_list(m)
814 struct mbuf *m;
815 {
816 struct mbuf *nextpkt;
817 int i, count=0;
818
819 MBUF_LOCK();
820
821 while (m) {
822 if (m)
823 nextpkt = m->m_nextpkt; /* chain of linked mbufs from driver */
824 else
825 nextpkt = 0;
826
827 count++;
828
829 while (m) { /* free the mbuf chain (like mfreem) */
830
831 struct mbuf *n;
832
833 /* Free the aux data if there is any */
834 if ((m->m_flags & M_PKTHDR) && m->m_pkthdr.aux) {
835 /*
836 * Treat the current m as the nextpkt and set m
837 * to the aux data. Preserve nextpkt in m->m_nextpkt.
838 * This lets us free the aux data in this loop
839 * without having to call m_freem recursively,
840 * which wouldn't work because we've still got
841 * the lock.
842 */
843 m->m_nextpkt = nextpkt;
844 nextpkt = m;
845 m = nextpkt->m_pkthdr.aux;
846 nextpkt->m_pkthdr.aux = NULL;
847 }
848
849 n = m->m_next;
850
851 if (n && n->m_nextpkt)
852 panic("m_freem_list: m_nextpkt of m_next != NULL");
853 if (m->m_type == MT_FREE)
854 panic("freeing free mbuf");
855
856 if (m->m_flags & M_EXT) {
857 if (MCLHASREFERENCE(m)) {
858 remque((queue_t)&m->m_ext.ext_refs);
859 } else if (m->m_ext.ext_free == NULL) {
860 union mcluster *mcl= (union mcluster *)m->m_ext.ext_buf;
861 if (_MCLUNREF(mcl)) {
862 mcl->mcl_next = mclfree;
863 mclfree = mcl;
864 ++mbstat.m_clfree;
865 }
866 } else {
867 (*(m->m_ext.ext_free))(m->m_ext.ext_buf,
868 m->m_ext.ext_size, m->m_ext.ext_arg);
869 }
870 }
871 mbstat.m_mtypes[m->m_type]--;
872 (void) _MCLUNREF(m);
873 _MFREE_MUNGE(m);
874 mbstat.m_mtypes[MT_FREE]++;
875 m->m_type = MT_FREE;
876 m->m_flags = 0;
877 m->m_len = 0;
878 m->m_next = mfree;
879 mfree = m;
880 m = n;
881 }
882 m = nextpkt; /* bump m with saved nextpkt if any */
883 }
884 if (i = m_want)
885 m_want = 0;
886
887 MBUF_UNLOCK();
888
889 if (i)
890 wakeup((caddr_t)&mfree);
891
892 return (count);
893 }
894
895 void
896 m_freem(m)
897 register struct mbuf *m;
898 {
899 while (m)
900 m = m_free(m);
901 }
902
903 /*
904 * Mbuffer utility routines.
905 */
906 /*
907 * Compute the amount of space available
908 * before the current start of data in an mbuf.
909 */
910 m_leadingspace(m)
911 register struct mbuf *m;
912 {
913 if (m->m_flags & M_EXT) {
914 if (MCLHASREFERENCE(m))
915 return(0);
916 return (m->m_data - m->m_ext.ext_buf);
917 }
918 if (m->m_flags & M_PKTHDR)
919 return (m->m_data - m->m_pktdat);
920 return (m->m_data - m->m_dat);
921 }
922
923 /*
924 * Compute the amount of space available
925 * after the end of data in an mbuf.
926 */
927 m_trailingspace(m)
928 register struct mbuf *m;
929 {
930 if (m->m_flags & M_EXT) {
931 if (MCLHASREFERENCE(m))
932 return(0);
933 return (m->m_ext.ext_buf + m->m_ext.ext_size -
934 (m->m_data + m->m_len));
935 }
936 return (&m->m_dat[MLEN] - (m->m_data + m->m_len));
937 }
938
939 /*
940 * Lesser-used path for M_PREPEND:
941 * allocate new mbuf to prepend to chain,
942 * copy junk along.
943 * Does not adjust packet header length.
944 */
945 struct mbuf *
946 m_prepend(m, len, how)
947 register struct mbuf *m;
948 int len, how;
949 {
950 struct mbuf *mn;
951
952 MGET(mn, how, m->m_type);
953 if (mn == (struct mbuf *)NULL) {
954 m_freem(m);
955 return ((struct mbuf *)NULL);
956 }
957 if (m->m_flags & M_PKTHDR) {
958 M_COPY_PKTHDR(mn, m);
959 m->m_flags &= ~M_PKTHDR;
960 }
961 mn->m_next = m;
962 m = mn;
963 if (len < MHLEN)
964 MH_ALIGN(m, len);
965 m->m_len = len;
966 return (m);
967 }
968
969 /*
970 * Replacement for old M_PREPEND macro:
971 * allocate new mbuf to prepend to chain,
972 * copy junk along, and adjust length.
973 *
974 */
975 struct mbuf *
976 m_prepend_2(m, len, how)
977 register struct mbuf *m;
978 int len, how;
979 {
980 if (M_LEADINGSPACE(m) >= len) {
981 m->m_data -= len;
982 m->m_len += len;
983 } else {
984 m = m_prepend(m, len, how);
985 }
986 if ((m) && (m->m_flags & M_PKTHDR))
987 m->m_pkthdr.len += len;
988 return (m);
989 }
990
991 /*
992 * Make a copy of an mbuf chain starting "off0" bytes from the beginning,
993 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf.
994 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller.
995 */
996 int MCFail;
997
998 struct mbuf *
999 m_copym(m, off0, len, wait)
1000 register struct mbuf *m;
1001 int off0, wait;
1002 register int len;
1003 {
1004 register struct mbuf *n, **np;
1005 register int off = off0;
1006 struct mbuf *top;
1007 int copyhdr = 0;
1008
1009 if (off < 0 || len < 0)
1010 panic("m_copym");
1011 if (off == 0 && m->m_flags & M_PKTHDR)
1012 copyhdr = 1;
1013
1014 while (off >= m->m_len) {
1015 if (m == 0)
1016 panic("m_copym");
1017 off -= m->m_len;
1018 m = m->m_next;
1019 }
1020 np = &top;
1021 top = 0;
1022
1023 MBUF_LOCK();
1024
1025 while (len > 0) {
1026 if (m == 0) {
1027 if (len != M_COPYALL)
1028 panic("m_copym");
1029 break;
1030 }
1031 if (n = mfree) {
1032 MCHECK(n);
1033 ++mclrefcnt[mtocl(n)];
1034 mbstat.m_mtypes[MT_FREE]--;
1035 mbstat.m_mtypes[m->m_type]++;
1036 mfree = n->m_next;
1037 n->m_next = n->m_nextpkt = 0;
1038 n->m_type = m->m_type;
1039 n->m_data = n->m_dat;
1040 n->m_flags = 0;
1041 } else {
1042 MBUF_UNLOCK();
1043 n = m_retry(wait, m->m_type);
1044 MBUF_LOCK();
1045 }
1046 *np = n;
1047
1048 if (n == 0)
1049 goto nospace;
1050 if (copyhdr) {
1051 M_COPY_PKTHDR(n, m);
1052 if (len == M_COPYALL)
1053 n->m_pkthdr.len -= off0;
1054 else
1055 n->m_pkthdr.len = len;
1056 copyhdr = 0;
1057 }
1058 if (len == M_COPYALL) {
1059 if (min(len, (m->m_len - off)) == len) {
1060 printf("m->m_len %d - off %d = %d, %d\n",
1061 m->m_len, off, m->m_len - off,
1062 min(len, (m->m_len - off)));
1063 }
1064 }
1065 n->m_len = min(len, (m->m_len - off));
1066 if (n->m_len == M_COPYALL) {
1067 printf("n->m_len == M_COPYALL, fixing\n");
1068 n->m_len = MHLEN;
1069 }
1070 if (m->m_flags & M_EXT) {
1071 n->m_ext = m->m_ext;
1072 insque((queue_t)&n->m_ext.ext_refs, (queue_t)&m->m_ext.ext_refs);
1073 n->m_data = m->m_data + off;
1074 n->m_flags |= M_EXT;
1075 } else {
1076 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
1077 (unsigned)n->m_len);
1078 }
1079 if (len != M_COPYALL)
1080 len -= n->m_len;
1081 off = 0;
1082 m = m->m_next;
1083 np = &n->m_next;
1084 }
1085 MBUF_UNLOCK();
1086
1087 if (top == 0)
1088 MCFail++;
1089
1090 return (top);
1091 nospace:
1092 MBUF_UNLOCK();
1093
1094 m_freem(top);
1095 MCFail++;
1096 return (0);
1097 }
1098
1099
1100 /*
1101 * equivilent to m_copym except that all necessary
1102 * mbuf hdrs are allocated within this routine
1103 * also, the last mbuf and offset accessed are passed
1104 * out and can be passed back in to avoid having to
1105 * rescan the entire mbuf list (normally hung off of the socket)
1106 */
1107 struct mbuf *
1108 m_copym_with_hdrs(m, off0, len, wait, m_last, m_off)
1109 register struct mbuf *m;
1110 int off0, wait;
1111 register int len;
1112 struct mbuf **m_last;
1113 int *m_off;
1114 {
1115 register struct mbuf *n, **np;
1116 register int off = off0;
1117 struct mbuf *top = 0;
1118 int copyhdr = 0;
1119 int type;
1120
1121 if (off == 0 && m->m_flags & M_PKTHDR)
1122 copyhdr = 1;
1123
1124 if (*m_last) {
1125 m = *m_last;
1126 off = *m_off;
1127 } else {
1128 while (off >= m->m_len) {
1129 off -= m->m_len;
1130 m = m->m_next;
1131 }
1132 }
1133 MBUF_LOCK();
1134
1135 while (len > 0) {
1136 if (top == 0)
1137 type = MT_HEADER;
1138 else {
1139 if (m == 0)
1140 panic("m_gethdr_and_copym");
1141 type = m->m_type;
1142 }
1143 if (n = mfree) {
1144 MCHECK(n);
1145 ++mclrefcnt[mtocl(n)];
1146 mbstat.m_mtypes[MT_FREE]--;
1147 mbstat.m_mtypes[type]++;
1148 mfree = n->m_next;
1149 n->m_next = n->m_nextpkt = 0;
1150 n->m_type = type;
1151
1152 if (top) {
1153 n->m_data = n->m_dat;
1154 n->m_flags = 0;
1155 } else {
1156 n->m_data = n->m_pktdat;
1157 n->m_flags = M_PKTHDR;
1158 _M_CLEAR_PKTHDR(n);
1159 }
1160 } else {
1161 MBUF_UNLOCK();
1162 if (top)
1163 n = m_retry(wait, type);
1164 else
1165 n = m_retryhdr(wait, type);
1166 MBUF_LOCK();
1167 }
1168 if (n == 0)
1169 goto nospace;
1170 if (top == 0) {
1171 top = n;
1172 np = &top->m_next;
1173 continue;
1174 } else
1175 *np = n;
1176
1177 if (copyhdr) {
1178 M_COPY_PKTHDR(n, m);
1179 n->m_pkthdr.len = len;
1180 copyhdr = 0;
1181 }
1182 n->m_len = min(len, (m->m_len - off));
1183
1184 if (m->m_flags & M_EXT) {
1185 n->m_ext = m->m_ext;
1186 insque((queue_t)&n->m_ext.ext_refs, (queue_t)&m->m_ext.ext_refs);
1187 n->m_data = m->m_data + off;
1188 n->m_flags |= M_EXT;
1189 } else {
1190 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
1191 (unsigned)n->m_len);
1192 }
1193 len -= n->m_len;
1194
1195 if (len == 0) {
1196 if ((off + n->m_len) == m->m_len) {
1197 *m_last = m->m_next;
1198 *m_off = 0;
1199 } else {
1200 *m_last = m;
1201 *m_off = off + n->m_len;
1202 }
1203 break;
1204 }
1205 off = 0;
1206 m = m->m_next;
1207 np = &n->m_next;
1208 }
1209 MBUF_UNLOCK();
1210
1211 return (top);
1212 nospace:
1213 MBUF_UNLOCK();
1214
1215 if (top)
1216 m_freem(top);
1217 MCFail++;
1218 return (0);
1219 }
1220
1221
1222 /*
1223 * Copy data from an mbuf chain starting "off" bytes from the beginning,
1224 * continuing for "len" bytes, into the indicated buffer.
1225 */
1226 void m_copydata(m, off, len, cp)
1227 register struct mbuf *m;
1228 register int off;
1229 register int len;
1230 caddr_t cp;
1231 {
1232 register unsigned count;
1233
1234 if (off < 0 || len < 0)
1235 panic("m_copydata");
1236 while (off > 0) {
1237 if (m == 0)
1238 panic("m_copydata");
1239 if (off < m->m_len)
1240 break;
1241 off -= m->m_len;
1242 m = m->m_next;
1243 }
1244 while (len > 0) {
1245 if (m == 0)
1246 panic("m_copydata");
1247 count = min(m->m_len - off, len);
1248 bcopy(mtod(m, caddr_t) + off, cp, count);
1249 len -= count;
1250 cp += count;
1251 off = 0;
1252 m = m->m_next;
1253 }
1254 }
1255
1256 /*
1257 * Concatenate mbuf chain n to m.
1258 * Both chains must be of the same type (e.g. MT_DATA).
1259 * Any m_pkthdr is not updated.
1260 */
1261 void m_cat(m, n)
1262 register struct mbuf *m, *n;
1263 {
1264 while (m->m_next)
1265 m = m->m_next;
1266 while (n) {
1267 if (m->m_flags & M_EXT ||
1268 m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) {
1269 /* just join the two chains */
1270 m->m_next = n;
1271 return;
1272 }
1273 /* splat the data from one into the other */
1274 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
1275 (u_int)n->m_len);
1276 m->m_len += n->m_len;
1277 n = m_free(n);
1278 }
1279 }
1280
1281 void
1282 m_adj(mp, req_len)
1283 struct mbuf *mp;
1284 int req_len;
1285 {
1286 register int len = req_len;
1287 register struct mbuf *m;
1288 register count;
1289
1290 if ((m = mp) == NULL)
1291 return;
1292 if (len >= 0) {
1293 /*
1294 * Trim from head.
1295 */
1296 while (m != NULL && len > 0) {
1297 if (m->m_len <= len) {
1298 len -= m->m_len;
1299 m->m_len = 0;
1300 m = m->m_next;
1301 } else {
1302 m->m_len -= len;
1303 m->m_data += len;
1304 len = 0;
1305 }
1306 }
1307 m = mp;
1308 if (m->m_flags & M_PKTHDR)
1309 m->m_pkthdr.len -= (req_len - len);
1310 } else {
1311 /*
1312 * Trim from tail. Scan the mbuf chain,
1313 * calculating its length and finding the last mbuf.
1314 * If the adjustment only affects this mbuf, then just
1315 * adjust and return. Otherwise, rescan and truncate
1316 * after the remaining size.
1317 */
1318 len = -len;
1319 count = 0;
1320 for (;;) {
1321 count += m->m_len;
1322 if (m->m_next == (struct mbuf *)0)
1323 break;
1324 m = m->m_next;
1325 }
1326 if (m->m_len >= len) {
1327 m->m_len -= len;
1328 m = mp;
1329 if (m->m_flags & M_PKTHDR)
1330 m->m_pkthdr.len -= len;
1331 return;
1332 }
1333 count -= len;
1334 if (count < 0)
1335 count = 0;
1336 /*
1337 * Correct length for chain is "count".
1338 * Find the mbuf with last data, adjust its length,
1339 * and toss data from remaining mbufs on chain.
1340 */
1341 m = mp;
1342 if (m->m_flags & M_PKTHDR)
1343 m->m_pkthdr.len = count;
1344 for (; m; m = m->m_next) {
1345 if (m->m_len >= count) {
1346 m->m_len = count;
1347 break;
1348 }
1349 count -= m->m_len;
1350 }
1351 while (m = m->m_next)
1352 m->m_len = 0;
1353 }
1354 }
1355
1356 /*
1357 * Rearange an mbuf chain so that len bytes are contiguous
1358 * and in the data area of an mbuf (so that mtod and dtom
1359 * will work for a structure of size len). Returns the resulting
1360 * mbuf chain on success, frees it and returns null on failure.
1361 * If there is room, it will add up to max_protohdr-len extra bytes to the
1362 * contiguous region in an attempt to avoid being called next time.
1363 */
1364 int MPFail;
1365
1366 struct mbuf *
1367 m_pullup(n, len)
1368 register struct mbuf *n;
1369 int len;
1370 {
1371 register struct mbuf *m;
1372 register int count;
1373 int space;
1374
1375 /*
1376 * If first mbuf has no cluster, and has room for len bytes
1377 * without shifting current data, pullup into it,
1378 * otherwise allocate a new mbuf to prepend to the chain.
1379 */
1380 if ((n->m_flags & M_EXT) == 0 &&
1381 n->m_data + len < &n->m_dat[MLEN] && n->m_next) {
1382 if (n->m_len >= len)
1383 return (n);
1384 m = n;
1385 n = n->m_next;
1386 len -= m->m_len;
1387 } else {
1388 if (len > MHLEN)
1389 goto bad;
1390 MGET(m, M_DONTWAIT, n->m_type);
1391 if (m == 0)
1392 goto bad;
1393 m->m_len = 0;
1394 if (n->m_flags & M_PKTHDR) {
1395 M_COPY_PKTHDR(m, n);
1396 n->m_flags &= ~M_PKTHDR;
1397 }
1398 }
1399 space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
1400 do {
1401 count = min(min(max(len, max_protohdr), space), n->m_len);
1402 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
1403 (unsigned)count);
1404 len -= count;
1405 m->m_len += count;
1406 n->m_len -= count;
1407 space -= count;
1408 if (n->m_len)
1409 n->m_data += count;
1410 else
1411 n = m_free(n);
1412 } while (len > 0 && n);
1413 if (len > 0) {
1414 (void) m_free(m);
1415 goto bad;
1416 }
1417 m->m_next = n;
1418 return (m);
1419 bad:
1420 m_freem(n);
1421 MPFail++;
1422 return (0);
1423 }
1424
1425 /*
1426 * Partition an mbuf chain in two pieces, returning the tail --
1427 * all but the first len0 bytes. In case of failure, it returns NULL and
1428 * attempts to restore the chain to its original state.
1429 */
1430 struct mbuf *
1431 m_split(m0, len0, wait)
1432 register struct mbuf *m0;
1433 int len0, wait;
1434 {
1435 register struct mbuf *m, *n;
1436 unsigned len = len0, remain;
1437
1438 for (m = m0; m && len > m->m_len; m = m->m_next)
1439 len -= m->m_len;
1440 if (m == 0)
1441 return (0);
1442 remain = m->m_len - len;
1443 if (m0->m_flags & M_PKTHDR) {
1444 MGETHDR(n, wait, m0->m_type);
1445 if (n == 0)
1446 return (0);
1447 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
1448 n->m_pkthdr.len = m0->m_pkthdr.len - len0;
1449 m0->m_pkthdr.len = len0;
1450 if (m->m_flags & M_EXT)
1451 goto extpacket;
1452 if (remain > MHLEN) {
1453 /* m can't be the lead packet */
1454 MH_ALIGN(n, 0);
1455 n->m_next = m_split(m, len, wait);
1456 if (n->m_next == 0) {
1457 (void) m_free(n);
1458 return (0);
1459 } else
1460 return (n);
1461 } else
1462 MH_ALIGN(n, remain);
1463 } else if (remain == 0) {
1464 n = m->m_next;
1465 m->m_next = 0;
1466 return (n);
1467 } else {
1468 MGET(n, wait, m->m_type);
1469 if (n == 0)
1470 return (0);
1471 M_ALIGN(n, remain);
1472 }
1473 extpacket:
1474 if (m->m_flags & M_EXT) {
1475 n->m_flags |= M_EXT;
1476 MBUF_LOCK();
1477 n->m_ext = m->m_ext;
1478 insque((queue_t)&n->m_ext.ext_refs, (queue_t)&m->m_ext.ext_refs);
1479 MBUF_UNLOCK();
1480 n->m_data = m->m_data + len;
1481 } else {
1482 bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain);
1483 }
1484 n->m_len = remain;
1485 m->m_len = len;
1486 n->m_next = m->m_next;
1487 m->m_next = 0;
1488 return (n);
1489 }
1490 /*
1491 * Routine to copy from device local memory into mbufs.
1492 */
1493 struct mbuf *
1494 m_devget(buf, totlen, off0, ifp, copy)
1495 char *buf;
1496 int totlen, off0;
1497 struct ifnet *ifp;
1498 void (*copy)();
1499 {
1500 register struct mbuf *m;
1501 struct mbuf *top = 0, **mp = &top;
1502 register int off = off0, len;
1503 register char *cp;
1504 char *epkt;
1505
1506 cp = buf;
1507 epkt = cp + totlen;
1508 if (off) {
1509 /*
1510 * If 'off' is non-zero, packet is trailer-encapsulated,
1511 * so we have to skip the type and length fields.
1512 */
1513 cp += off + 2 * sizeof(u_int16_t);
1514 totlen -= 2 * sizeof(u_int16_t);
1515 }
1516 MGETHDR(m, M_DONTWAIT, MT_DATA);
1517 if (m == 0)
1518 return (0);
1519 m->m_pkthdr.rcvif = ifp;
1520 m->m_pkthdr.len = totlen;
1521 m->m_len = MHLEN;
1522
1523 while (totlen > 0) {
1524 if (top) {
1525 MGET(m, M_DONTWAIT, MT_DATA);
1526 if (m == 0) {
1527 m_freem(top);
1528 return (0);
1529 }
1530 m->m_len = MLEN;
1531 }
1532 len = min(totlen, epkt - cp);
1533 if (len >= MINCLSIZE) {
1534 MCLGET(m, M_DONTWAIT);
1535 if (m->m_flags & M_EXT)
1536 m->m_len = len = min(len, MCLBYTES);
1537 else {
1538 /* give up when it's out of cluster mbufs */
1539 if (top)
1540 m_freem(top);
1541 m_freem(m);
1542 return (0);
1543 }
1544 } else {
1545 /*
1546 * Place initial small packet/header at end of mbuf.
1547 */
1548 if (len < m->m_len) {
1549 if (top == 0 && len + max_linkhdr <= m->m_len)
1550 m->m_data += max_linkhdr;
1551 m->m_len = len;
1552 } else
1553 len = m->m_len;
1554 }
1555 if (copy)
1556 copy(cp, mtod(m, caddr_t), (unsigned)len);
1557 else
1558 bcopy(cp, mtod(m, caddr_t), (unsigned)len);
1559 cp += len;
1560 *mp = m;
1561 mp = &m->m_next;
1562 totlen -= len;
1563 if (cp == epkt)
1564 cp = buf;
1565 }
1566 return (top);
1567 }
1568
1569 /*
1570 * Cluster freelist allocation check. The mbuf lock must be held.
1571 * Ensure hysteresis between hi/lo.
1572 */
1573 static int
1574 m_howmany()
1575 {
1576 register int i;
1577
1578 /* Under minimum */
1579 if (mbstat.m_clusters < MINCL)
1580 return (MINCL - mbstat.m_clusters);
1581 /* Too few (free < 1/2 total) and not over maximum */
1582 if (mbstat.m_clusters < nmbclusters &&
1583 (i = ((mbstat.m_clusters >> 1) - mbstat.m_clfree)) > 0)
1584 return i;
1585 return 0;
1586 }
1587
1588
1589 /*
1590 * Copy data from a buffer back into the indicated mbuf chain,
1591 * starting "off" bytes from the beginning, extending the mbuf
1592 * chain if necessary.
1593 */
1594 void
1595 m_copyback(m0, off, len, cp)
1596 struct mbuf *m0;
1597 register int off;
1598 register int len;
1599 caddr_t cp;
1600 {
1601 register int mlen;
1602 register struct mbuf *m = m0, *n;
1603 int totlen = 0;
1604
1605 if (m0 == 0)
1606 return;
1607 while (off > (mlen = m->m_len)) {
1608 off -= mlen;
1609 totlen += mlen;
1610 if (m->m_next == 0) {
1611 n = m_getclr(M_DONTWAIT, m->m_type);
1612 if (n == 0)
1613 goto out;
1614 n->m_len = min(MLEN, len + off);
1615 m->m_next = n;
1616 }
1617 m = m->m_next;
1618 }
1619 while (len > 0) {
1620 mlen = min (m->m_len - off, len);
1621 bcopy(cp, off + mtod(m, caddr_t), (unsigned)mlen);
1622 cp += mlen;
1623 len -= mlen;
1624 mlen += off;
1625 off = 0;
1626 totlen += mlen;
1627 if (len == 0)
1628 break;
1629 if (m->m_next == 0) {
1630 n = m_get(M_DONTWAIT, m->m_type);
1631 if (n == 0)
1632 break;
1633 n->m_len = min(MLEN, len);
1634 m->m_next = n;
1635 }
1636 m = m->m_next;
1637 }
1638 out: if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen))
1639 m->m_pkthdr.len = totlen;
1640 }
1641
1642
1643 char *mcl_to_paddr(register char *addr) {
1644 register int base_phys;
1645
1646 if (addr < (char *)mbutl || addr >= (char *)embutl)
1647 return (0);
1648 base_phys = mcl_paddr[(addr - (char *)mbutl) >> PAGE_SHIFT];
1649
1650 if (base_phys == 0)
1651 return (0);
1652 return ((char *)((int)base_phys | ((int)addr & PAGE_MASK)));
1653 }
1654
1655 /*
1656 * Dup the mbuf chain passed in. The whole thing. No cute additional cruft.
1657 * And really copy the thing. That way, we don't "precompute" checksums
1658 * for unsuspecting consumers.
1659 * Assumption: m->m_nextpkt == 0.
1660 * Trick: for small packets, don't dup into a cluster. That way received
1661 * packets don't take up too much room in the sockbuf (cf. sbspace()).
1662 */
1663 int MDFail;
1664
1665 struct mbuf *
1666 m_dup(register struct mbuf *m, int how)
1667 { register struct mbuf *n, **np;
1668 struct mbuf *top;
1669 int copyhdr = 0;
1670
1671 np = &top;
1672 top = 0;
1673 if (m->m_flags & M_PKTHDR)
1674 copyhdr = 1;
1675
1676 /*
1677 * Quick check: if we have one mbuf and its data fits in an
1678 * mbuf with packet header, just copy and go.
1679 */
1680 if (m->m_next == NULL)
1681 { /* Then just move the data into an mbuf and be done... */
1682 if (copyhdr)
1683 { if (m->m_pkthdr.len <= MHLEN)
1684 { if ((n = m_gethdr(how, m->m_type)) == NULL)
1685 return(NULL);
1686 n->m_len = m->m_len;
1687 n->m_flags |= (m->m_flags & M_COPYFLAGS);
1688 n->m_pkthdr.len = m->m_pkthdr.len;
1689 n->m_pkthdr.rcvif = m->m_pkthdr.rcvif;
1690 n->m_pkthdr.header = NULL;
1691 n->m_pkthdr.csum_flags = 0;
1692 n->m_pkthdr.csum_data = 0;
1693 n->m_pkthdr.aux = NULL;
1694 n->m_pkthdr.vlan_tag = 0;
1695 n->m_pkthdr.reserved_1 = 0;
1696 n->m_pkthdr.reserved2 = 0;
1697 bcopy(m->m_data, n->m_data, m->m_pkthdr.len);
1698 return(n);
1699 }
1700 } else if (m->m_len <= MLEN)
1701 { if ((n = m_get(how, m->m_type)) == NULL)
1702 return(NULL);
1703 bcopy(m->m_data, n->m_data, m->m_len);
1704 n->m_len = m->m_len;
1705 return(n);
1706 }
1707 }
1708 while (m)
1709 {
1710 #if BLUE_DEBUG
1711 kprintf("<%x: %x, %x, %x\n", m, m->m_flags, m->m_len,
1712 m->m_data);
1713 #endif
1714 if (copyhdr)
1715 n = m_gethdr(how, m->m_type);
1716 else
1717 n = m_get(how, m->m_type);
1718 if (n == 0)
1719 goto nospace;
1720 if (m->m_flags & M_EXT)
1721 { MCLGET(n, how);
1722 if ((n->m_flags & M_EXT) == 0)
1723 goto nospace;
1724 }
1725 *np = n;
1726 if (copyhdr)
1727 { /* Don't use M_COPY_PKTHDR: preserve m_data */
1728 n->m_pkthdr = m->m_pkthdr;
1729 n->m_flags |= (m->m_flags & M_COPYFLAGS);
1730 copyhdr = 0;
1731 if ((n->m_flags & M_EXT) == 0)
1732 n->m_data = n->m_pktdat;
1733 }
1734 n->m_len = m->m_len;
1735 /*
1736 * Get the dup on the same bdry as the original
1737 * Assume that the two mbufs have the same offset to data area
1738 * (up to word bdries)
1739 */
1740 bcopy(mtod(m, caddr_t), mtod(n, caddr_t), (unsigned)n->m_len);
1741 m = m->m_next;
1742 np = &n->m_next;
1743 #if BLUE_DEBUG
1744 kprintf(">%x: %x, %x, %x\n", n, n->m_flags, n->m_len,
1745 n->m_data);
1746 #endif
1747 }
1748
1749 if (top == 0)
1750 MDFail++;
1751 return (top);
1752 nospace:
1753 m_freem(top);
1754 MDFail++;
1755 return (0);
1756 }
1757
1758 int
1759 m_mclref(struct mbuf *p)
1760 {
1761 return (_MCLREF(p));
1762 }
1763
1764 int
1765 m_mclunref(struct mbuf *p)
1766 {
1767 return (_MCLUNREF(p));
1768 }
1769
1770 /* change mbuf to new type */
1771 void
1772 m_mchtype(struct mbuf *m, int t)
1773 {
1774 MBUF_LOCK();
1775 mbstat.m_mtypes[(m)->m_type]--;
1776 mbstat.m_mtypes[t]++;
1777 (m)->m_type = t;
1778 MBUF_UNLOCK();
1779 }
1780
1781 void *m_mtod(struct mbuf *m)
1782 {
1783 return ((m)->m_data);
1784 }
1785
1786 struct mbuf *m_dtom(void *x)
1787 {
1788 return ((struct mbuf *)((u_long)(x) & ~(MSIZE-1)));
1789 }
1790
1791 int m_mtocl(void *x)
1792 {
1793 return (((char *)(x) - (char *)mbutl) / sizeof(union mcluster));
1794 }
1795
1796 union mcluster *m_cltom(int x)
1797 {
1798 return ((union mcluster *)(mbutl + (x)));
1799 }
1800
1801
1802 void m_mcheck(struct mbuf *m)
1803 {
1804 if (m->m_type != MT_FREE)
1805 panic("mget MCHECK: m_type=%x m=%x", m->m_type, m);
1806 }
1807
1808 void
1809 mbuf_expand_thread(void)
1810 {
1811 while (1) {
1812 int expand_mcl;
1813 MBUF_LOCK();
1814 expand_mcl = mbuf_expand_mcl;
1815 mbuf_expand_mcl = 0;
1816 MBUF_UNLOCK();
1817 if (expand_mcl) {
1818 caddr_t p;
1819 MCLALLOC(p, M_WAIT);
1820 if (p) MCLFREE(p);
1821 }
1822 assert_wait(&mbuf_expand_thread_wakeup, THREAD_UNINT);
1823 (void) thread_block(mbuf_expand_thread);
1824 }
1825 }
1826
1827 void
1828 mbuf_expand_thread_init(void)
1829 {
1830 mbuf_expand_thread_initialized++;
1831 mbuf_expand_thread();
1832 }
1833