]> git.saurik.com Git - apple/xnu.git/blob - bsd/kern/uipc_mbuf.c
projects / apple / xnu.git / blob
? search:


6a914e6896ddccdff845629f0091123543c7621c
[apple/xnu.git] / bsd / kern / uipc_mbuf.c
1 /*
2 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * Copyright (c) 1999-2003 Apple Computer, Inc. All Rights Reserved.
7 *
8 * This file contains Original Code and/or Modifications of Original Code
9 * as defined in and that are subject to the Apple Public Source License
10 * Version 2.0 (the 'License'). You may not use this file except in
11 * compliance with the License. Please obtain a copy of the License at
12 * http://www.opensource.apple.com/apsl/ and read it before using this
13 * file.
14 *
15 * The Original Code and all software distributed under the License are
16 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
17 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
18 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
20 * Please see the License for the specific language governing rights and
21 * limitations under the License.
22 *
23 * @APPLE_LICENSE_HEADER_END@
24 */
25 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
26 /*
27 * Copyright (c) 1982, 1986, 1988, 1991, 1993
28 * The Regents of the University of California. All rights reserved.
29 *
30 * Redistribution and use in source and binary forms, with or without
31 * modification, are permitted provided that the following conditions
32 * are met:
33 * 1. Redistributions of source code must retain the above copyright
34 * notice, this list of conditions and the following disclaimer.
35 * 2. Redistributions in binary form must reproduce the above copyright
36 * notice, this list of conditions and the following disclaimer in the
37 * documentation and/or other materials provided with the distribution.
38 * 3. All advertising materials mentioning features or use of this software
39 * must display the following acknowledgement:
40 * This product includes software developed by the University of
41 * California, Berkeley and its contributors.
42 * 4. Neither the name of the University nor the names of its contributors
43 * may be used to endorse or promote products derived from this software
44 * without specific prior written permission.
45 *
46 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
47 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
48 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
49 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
50 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
51 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
52 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
53 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
54 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
55 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
56 * SUCH DAMAGE.
57 *
58 * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94
59 */
60 /* HISTORY
61 *
62 * 10/15/97 Annette DeSchon (deschon@apple.com)
63 * Fixed bug in which all cluster mbufs were broken up
64 * into regular mbufs: Some clusters are now reserved.
65 * When a cluster is needed, regular mbufs are no longer
66 * used. (Radar 1683621)
67 * 20-May-95 Mac Gillon (mgillon) at NeXT
68 * New version based on 4.4
69 */
70
71 #include <sys/param.h>
72 #include <sys/systm.h>
73 #include <sys/malloc.h>
74 #include <sys/mbuf.h>
75 #include <sys/kernel.h>
76 #include <sys/syslog.h>
77 #include <sys/protosw.h>
78 #include <sys/domain.h>
79 #include <net/netisr.h>
80
81 #include <kern/queue.h>
82 #include <kern/kern_types.h>
83 #include <kern/sched_prim.h>
84
85 #define _MCLREF(p) (++mclrefcnt[mtocl(p)])
86 #define _MCLUNREF(p) (--mclrefcnt[mtocl(p)] == 0)
87
88 extern kernel_pmap; /* The kernel's pmap */
89
90 decl_simple_lock_data(, mbuf_slock);
91 struct mbuf *mfree; /* mbuf free list */
92 struct mbuf *mfreelater; /* mbuf deallocation list */
93 extern vm_map_t mb_map; /* special map */
94 int m_want; /* sleepers on mbufs */
95 extern int nmbclusters; /* max number of mapped clusters */
96 short *mclrefcnt; /* mapped cluster reference counts */
97 int *mcl_paddr;
98 union mcluster *mclfree; /* mapped cluster free list */
99 int max_linkhdr; /* largest link-level header */
100 int max_protohdr; /* largest protocol header */
101 int max_hdr; /* largest link+protocol header */
102 int max_datalen; /* MHLEN - max_hdr */
103 struct mbstat mbstat; /* statistics */
104 union mcluster *mbutl; /* first mapped cluster address */
105 union mcluster *embutl; /* ending virtual address of mclusters */
106
107 static int nclpp; /* # clusters per physical page */
108 static char mbfail[] = "mbuf not mapped";
109
110 static int m_howmany();
111
112 /* The number of cluster mbufs that are allocated, to start. */
113 #define MINCL max(16, 2)
114
115 extern int dlil_input_thread_wakeup;
116 extern int dlil_expand_mcl;
117 extern int dlil_initialized;
118
119 #if 0
120 static int mfree_munge = 0;
121 #if 0
122 #define _MFREE_MUNGE(m) { \
123 if (mfree_munge) \
124 { int i; \
125 vm_offset_t *element = (vm_offset_t *)(m); \
126 for (i = 0; \
127 i < sizeof(struct mbuf)/sizeof(vm_offset_t); \
128 i++) \
129 (element)[i] = 0xdeadbeef; \
130 } \
131 }
132 #else
133 void
134 munge_mbuf(struct mbuf *m)
135 {
136 int i;
137 vm_offset_t *element = (vm_offset_t *)(m);
138 for (i = 0;
139 i < sizeof(struct mbuf)/sizeof(vm_offset_t);
140 i++)
141 (element)[i] = 0xdeadbeef;
142 }
143 #define _MFREE_MUNGE(m) { \
144 if (mfree_munge) \
145 munge_mbuf(m); \
146 }
147 #endif
148 #else
149 #define _MFREE_MUNGE(m)
150 #endif
151
152
153 #define _MINTGET(m, type) { \
154 MBUF_LOCK(); \
155 if (((m) = mfree) != 0) { \
156 MCHECK(m); \
157 ++mclrefcnt[mtocl(m)]; \
158 mbstat.m_mtypes[MT_FREE]--; \
159 mbstat.m_mtypes[(type)]++; \
160 mfree = (m)->m_next; \
161 } \
162 MBUF_UNLOCK(); \
163 }
164
165
166 void
167 mbinit()
168 {
169 int s,m;
170 int initmcl = 32;
171
172 if (nclpp)
173 return;
174 nclpp = round_page(MCLBYTES) / MCLBYTES; /* see mbufgc() */
175 if (nclpp < 1) nclpp = 1;
176 MBUF_LOCKINIT();
177 // NETISR_LOCKINIT();
178
179 mbstat.m_msize = MSIZE;
180 mbstat.m_mclbytes = MCLBYTES;
181 mbstat.m_minclsize = MINCLSIZE;
182 mbstat.m_mlen = MLEN;
183 mbstat.m_mhlen = MHLEN;
184
185 if (nmbclusters == 0)
186 nmbclusters = NMBCLUSTERS;
187 MALLOC(mclrefcnt, short *, nmbclusters * sizeof (short),
188 M_TEMP, M_WAITOK);
189 if (mclrefcnt == 0)
190 panic("mbinit");
191 for (m = 0; m < nmbclusters; m++)
192 mclrefcnt[m] = -1;
193
194 MALLOC(mcl_paddr, int *, (nmbclusters/(PAGE_SIZE/CLBYTES)) * sizeof (int),
195 M_TEMP, M_WAITOK);
196 if (mcl_paddr == 0)
197 panic("mbinit1");
198 bzero((char *)mcl_paddr, (nmbclusters/(PAGE_SIZE/CLBYTES)) * sizeof (int));
199
200 embutl = (union mcluster *)((unsigned char *)mbutl + (nmbclusters * MCLBYTES));
201
202 PE_parse_boot_arg("initmcl", &initmcl);
203
204 if (m_clalloc(max(PAGE_SIZE/CLBYTES, 1) * initmcl, M_WAIT) == 0)
205 goto bad;
206 MBUF_UNLOCK();
207 return;
208 bad:
209 panic("mbinit");
210 }
211
212 /*
213 * Allocate some number of mbuf clusters
214 * and place on cluster free list.
215 */
216 /* ARGSUSED */
217 m_clalloc(ncl, nowait)
218 register int ncl;
219 int nowait;
220 {
221 register union mcluster *mcl;
222 register int i;
223 vm_size_t size;
224 static char doing_alloc;
225
226 /*
227 * Honor the caller's wish to block or not block.
228 * We have a way to grow the pool asynchronously,
229 * by kicking the dlil_input_thread.
230 */
231 if ((i = m_howmany()) <= 0)
232 goto out;
233
234 if ((nowait == M_DONTWAIT))
235 goto out;
236
237 if (ncl < i)
238 ncl = i;
239 size = round_page(ncl * MCLBYTES);
240 mcl = (union mcluster *)kmem_mb_alloc(mb_map, size);
241
242 if (mcl == 0 && ncl > 1) {
243 size = round_page(MCLBYTES); /* Try for 1 if failed */
244 mcl = (union mcluster *)kmem_mb_alloc(mb_map, size);
245 }
246
247 if (mcl) {
248 MBUF_LOCK();
249 ncl = size / MCLBYTES;
250 for (i = 0; i < ncl; i++) {
251 if (++mclrefcnt[mtocl(mcl)] != 0)
252 panic("m_clalloc already there");
253 if (((int)mcl & PAGE_MASK) == 0)
254 mcl_paddr[((char *)mcl - (char *)mbutl)/PAGE_SIZE] = pmap_extract(kernel_pmap, (char *)mcl);
255
256 mcl->mcl_next = mclfree;
257 mclfree = mcl++;
258 }
259 mbstat.m_clfree += ncl;
260 mbstat.m_clusters += ncl;
261 return (ncl);
262 } /* else ... */
263 out:
264 MBUF_LOCK();
265
266 /*
267 * When non-blocking we kick the dlil thread if we havve to grow the
268 * pool or if the number of free clusters is less than requested.
269 */
270 if ((nowait == M_DONTWAIT) && (i > 0 || ncl >= mbstat.m_clfree)) {
271 dlil_expand_mcl = 1;
272 if (dlil_initialized)
273 wakeup((caddr_t)&dlil_input_thread_wakeup);
274 }
275
276 if (mbstat.m_clfree >= ncl)
277 return 1;
278
279 mbstat.m_drops++;
280
281 return 0;
282 }
283
284 /*
285 * Add more free mbufs by cutting up a cluster.
286 */
287 m_expand(canwait)
288 int canwait;
289 {
290 register caddr_t mcl;
291
292 if (mbstat.m_clfree < (mbstat.m_clusters >> 4))
293 /* 1/16th of the total number of cluster mbufs allocated is
294 reserved for large packets. The number reserved must
295 always be < 1/2, or future allocation will be prevented.
296 */
297 return 0;
298
299 MCLALLOC(mcl, canwait);
300 if (mcl) {
301 register struct mbuf *m = (struct mbuf *)mcl;
302 register int i = NMBPCL;
303 MBUF_LOCK();
304 mbstat.m_mtypes[MT_FREE] += i;
305 mbstat.m_mbufs += i;
306 while (i--) {
307 _MFREE_MUNGE(m);
308 m->m_type = MT_FREE;
309 m->m_next = mfree;
310 mfree = m++;
311 }
312 i = m_want;
313 m_want = 0;
314 MBUF_UNLOCK();
315 if (i) wakeup((caddr_t)&mfree);
316 return 1;
317 }
318 return 0;
319 }
320
321 /*
322 * When MGET failes, ask protocols to free space when short of memory,
323 * then re-attempt to allocate an mbuf.
324 */
325 struct mbuf *
326 m_retry(canwait, type)
327 int canwait, type;
328 {
329 register struct mbuf *m;
330 int wait, s;
331 funnel_t * fnl;
332 int fnl_switch = 0;
333 boolean_t funnel_state;
334
335 for (;;) {
336 (void) m_expand(canwait);
337 _MINTGET(m, type);
338 if (m) {
339 (m)->m_next = (m)->m_nextpkt = 0;
340 (m)->m_type = (type);
341 (m)->m_data = (m)->m_dat;
342 (m)->m_flags = 0;
343 }
344 if (m || canwait == M_DONTWAIT)
345 break;
346 MBUF_LOCK();
347 wait = m_want++;
348 dlil_expand_mcl = 1;
349 if (wait == 0)
350 mbstat.m_drain++;
351 else
352 mbstat.m_wait++;
353 MBUF_UNLOCK();
354
355 if (dlil_initialized)
356 wakeup((caddr_t)&dlil_input_thread_wakeup);
357
358 /*
359 * Grab network funnel because m_reclaim calls into the
360 * socket domains and tsleep end-up calling splhigh
361 */
362 fnl = thread_funnel_get();
363 if (fnl && (fnl == kernel_flock)) {
364 fnl_switch = 1;
365 thread_funnel_switch(KERNEL_FUNNEL, NETWORK_FUNNEL);
366 } else
367 funnel_state = thread_funnel_set(network_flock, TRUE);
368 if (wait == 0) {
369 m_reclaim();
370 } else {
371 /* Sleep with a small timeout as insurance */
372 (void) tsleep((caddr_t)&mfree, PZERO-1, "m_retry", hz);
373 }
374 if (fnl_switch)
375 thread_funnel_switch(NETWORK_FUNNEL, KERNEL_FUNNEL);
376 else
377 thread_funnel_set(network_flock, funnel_state);
378 }
379 return (m);
380 }
381
382 /*
383 * As above; retry an MGETHDR.
384 */
385 struct mbuf *
386 m_retryhdr(canwait, type)
387 int canwait, type;
388 {
389 register struct mbuf *m;
390
391 if (m = m_retry(canwait, type)) {
392 m->m_flags |= M_PKTHDR;
393 m->m_data = m->m_pktdat;
394 m->m_pkthdr.rcvif = NULL;
395 m->m_pkthdr.len = 0;
396 m->m_pkthdr.header = NULL;
397 m->m_pkthdr.csum_flags = 0;
398 m->m_pkthdr.csum_data = 0;
399 m->m_pkthdr.aux = (struct mbuf *)NULL;
400 m->m_pkthdr.reserved1 = NULL;
401 m->m_pkthdr.reserved2 = NULL;
402 }
403 return (m);
404 }
405
406 m_reclaim()
407 {
408 register struct domain *dp;
409 register struct protosw *pr;
410
411 for (dp = domains; dp; dp = dp->dom_next)
412 for (pr = dp->dom_protosw; pr; pr = pr->pr_next)
413 if (pr->pr_drain)
414 (*pr->pr_drain)();
415 mbstat.m_drain++;
416 }
417
418 /*
419 * Space allocation routines.
420 * These are also available as macros
421 * for critical paths.
422 */
423 struct mbuf *
424 m_get(nowait, type)
425 int nowait, type;
426 {
427 register struct mbuf *m;
428
429 _MINTGET(m, type);
430 if (m) {
431 m->m_next = m->m_nextpkt = 0;
432 m->m_type = type;
433 m->m_data = m->m_dat;
434 m->m_flags = 0;
435 } else
436 (m) = m_retry(nowait, type);
437
438 return (m);
439 }
440
441 struct mbuf *
442 m_gethdr(nowait, type)
443 int nowait, type;
444 {
445 register struct mbuf *m;
446
447 _MINTGET(m, type);
448 if (m) {
449 m->m_next = m->m_nextpkt = 0;
450 m->m_type = type;
451 m->m_data = m->m_pktdat;
452 m->m_flags = M_PKTHDR;
453 m->m_pkthdr.rcvif = NULL;
454 m->m_pkthdr.header = NULL;
455 m->m_pkthdr.csum_flags = 0;
456 m->m_pkthdr.csum_data = 0;
457 m->m_pkthdr.aux = (struct mbuf *)NULL;
458 m->m_pkthdr.reserved1 = NULL;
459 m->m_pkthdr.reserved2 = NULL;
460 } else
461 m = m_retryhdr(nowait, type);
462
463 return m;
464 }
465
466 struct mbuf *
467 m_getclr(nowait, type)
468 int nowait, type;
469 {
470 register struct mbuf *m;
471
472 MGET(m, nowait, type);
473 if (m == 0)
474 return (0);
475 bzero(mtod(m, caddr_t), MLEN);
476 return (m);
477 }
478
479 struct mbuf *
480 m_free(m)
481 struct mbuf *m;
482 {
483 struct mbuf *n = m->m_next;
484 int i, s;
485
486 if (m->m_type == MT_FREE)
487 panic("freeing free mbuf");
488
489 /* Free the aux data if there is any */
490 if ((m->m_flags & M_PKTHDR) && m->m_pkthdr.aux)
491 {
492 m_freem(m->m_pkthdr.aux);
493 }
494
495 MBUF_LOCK();
496 if ((m->m_flags & M_EXT))
497 {
498 if (MCLHASREFERENCE(m)) {
499 remque((queue_t)&m->m_ext.ext_refs);
500 } else if (m->m_ext.ext_free == NULL) {
501 union mcluster *mcl= (union mcluster *)m->m_ext.ext_buf;
502 if (_MCLUNREF(mcl)) {
503 mcl->mcl_next = mclfree;
504 mclfree = mcl;
505 ++mbstat.m_clfree;
506 }
507 #ifdef COMMENT_OUT
508 /* *** Since m_split() increments "mclrefcnt[mtocl(m->m_ext.ext_buf)]",
509 and AppleTalk ADSP uses m_split(), this incorrect sanity check
510 caused a panic.
511 *** */
512 else /* sanity check - not referenced this way */
513 panic("m_free m_ext cluster not free");
514 #endif
515 } else {
516 (*(m->m_ext.ext_free))(m->m_ext.ext_buf,
517 m->m_ext.ext_size, m->m_ext.ext_arg);
518 }
519 }
520 mbstat.m_mtypes[m->m_type]--;
521 (void) _MCLUNREF(m);
522 _MFREE_MUNGE(m);
523 m->m_type = MT_FREE;
524 mbstat.m_mtypes[m->m_type]++;
525 m->m_flags = 0;
526 m->m_next = mfree;
527 m->m_len = 0;
528 mfree = m;
529 i = m_want;
530 m_want = 0;
531 MBUF_UNLOCK();
532 if (i) wakeup((caddr_t)&mfree);
533 return (n);
534 }
535
536 /* m_mclget() add an mbuf cluster to a normal mbuf */
537 struct mbuf *
538 m_mclget(m, nowait)
539 struct mbuf *m;
540 int nowait;
541 {
542 MCLALLOC(m->m_ext.ext_buf, nowait);
543 if (m->m_ext.ext_buf) {
544 m->m_data = m->m_ext.ext_buf;
545 m->m_flags |= M_EXT;
546 m->m_ext.ext_size = MCLBYTES;
547 m->m_ext.ext_free = 0;
548 m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward =
549 &m->m_ext.ext_refs;
550 }
551
552 return m;
553 }
554
555 /* m_mclalloc() allocate an mbuf cluster */
556 caddr_t
557 m_mclalloc( nowait)
558 int nowait;
559 {
560 caddr_t p;
561
562 (void)m_clalloc(1, nowait);
563 if ((p = (caddr_t)mclfree)) {
564 ++mclrefcnt[mtocl(p)];
565 mbstat.m_clfree--;
566 mclfree = ((union mcluster *)p)->mcl_next;
567 }
568 MBUF_UNLOCK();
569
570 return p;
571 }
572
573 /* m_mclfree() releases a reference to a cluster allocated by MCLALLOC,
574 * freeing the cluster if the reference count has reached 0. */
575 void
576 m_mclfree(p)
577 caddr_t p;
578 {
579 MBUF_LOCK();
580 if (--mclrefcnt[mtocl(p)] == 0) {
581 ((union mcluster *)(p))->mcl_next = mclfree;
582 mclfree = (union mcluster *)(p);
583 mbstat.m_clfree++;
584 }
585 MBUF_UNLOCK();
586 }
587
588 /* mcl_hasreference() checks if a cluster of an mbuf is referenced by another mbuf */
589 int
590 m_mclhasreference(m)
591 struct mbuf *m;
592 {
593 return (m->m_ext.ext_refs.forward != &(m->m_ext.ext_refs));
594 }
595
596 /* */
597 void
598 m_copy_pkthdr(to, from)
599 struct mbuf *to, *from;
600 {
601 to->m_pkthdr = from->m_pkthdr;
602 from->m_pkthdr.aux = (struct mbuf *)NULL;
603 to->m_flags = from->m_flags & M_COPYFLAGS;
604 to->m_data = (to)->m_pktdat;
605 }
606
607 /* Best effort to get a mbuf cluster + pkthdr under one lock.
608 * If we don't have them avail, just bail out and use the regular
609 * path.
610 * Used by drivers to allocated packets on receive ring.
611 */
612 struct mbuf *
613 m_getpacket(void)
614 {
615 struct mbuf *m;
616 m_clalloc(1, M_DONTWAIT); /* takes the MBUF_LOCK, but doesn't release it... */
617 if ((mfree != 0) && (mclfree != 0)) { /* mbuf + cluster are available */
618 m = mfree;
619 mfree = m->m_next;
620 MCHECK(m);
621 ++mclrefcnt[mtocl(m)];
622 mbstat.m_mtypes[MT_FREE]--;
623 mbstat.m_mtypes[MT_DATA]++;
624 m->m_ext.ext_buf = (caddr_t)mclfree; /* get the cluster */
625 ++mclrefcnt[mtocl(m->m_ext.ext_buf)];
626 mbstat.m_clfree--;
627 mclfree = ((union mcluster *)(m->m_ext.ext_buf))->mcl_next;
628
629 m->m_next = m->m_nextpkt = 0;
630 m->m_type = MT_DATA;
631 m->m_data = m->m_ext.ext_buf;
632 m->m_flags = M_PKTHDR | M_EXT;
633 m->m_pkthdr.len = 0;
634 m->m_pkthdr.rcvif = NULL;
635 m->m_pkthdr.header = NULL;
636 m->m_pkthdr.csum_data = 0;
637 m->m_pkthdr.csum_flags = 0;
638 m->m_pkthdr.aux = (struct mbuf *)NULL;
639 m->m_pkthdr.reserved1 = 0;
640 m->m_pkthdr.reserved2 = 0;
641 m->m_ext.ext_free = 0;
642 m->m_ext.ext_size = MCLBYTES;
643 m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward =
644 &m->m_ext.ext_refs;
645 MBUF_UNLOCK();
646 }
647 else { /* slow path: either mbuf or cluster need to be allocated anyway */
648 MBUF_UNLOCK();
649
650 MGETHDR(m, M_WAITOK, MT_DATA );
651
652 if ( m == 0 )
653 return (NULL);
654
655 MCLGET( m, M_WAITOK );
656 if ( ( m->m_flags & M_EXT ) == 0 )
657 {
658 m_free(m); m = 0;
659 }
660 }
661 return (m);
662 }
663
664
665 /*
666 * return a list of mbuf hdrs that point to clusters...
667 * try for num_needed, if this can't be met, return whatever
668 * number were available... set up the first num_with_pkthdrs
669 * with mbuf hdrs configured as packet headers... these are
670 * chained on the m_nextpkt field... any packets requested beyond
671 * this are chained onto the last packet header's m_next field.
672 */
673 struct mbuf *
674 m_getpackets(int num_needed, int num_with_pkthdrs, int how)
675 {
676 struct mbuf *m;
677 struct mbuf **np, *top;
678
679 top = NULL;
680 np = &top;
681
682 m_clalloc(num_needed, how); /* takes the MBUF_LOCK, but doesn't release it... */
683
684 while (num_needed--) {
685 if (mfree && mclfree) { /* mbuf + cluster are available */
686 m = mfree;
687 MCHECK(m);
688 mfree = m->m_next;
689 ++mclrefcnt[mtocl(m)];
690 mbstat.m_mtypes[MT_FREE]--;
691 mbstat.m_mtypes[MT_DATA]++;
692 m->m_ext.ext_buf = (caddr_t)mclfree; /* get the cluster */
693 ++mclrefcnt[mtocl(m->m_ext.ext_buf)];
694 mbstat.m_clfree--;
695 mclfree = ((union mcluster *)(m->m_ext.ext_buf))->mcl_next;
696
697 m->m_next = m->m_nextpkt = 0;
698 m->m_type = MT_DATA;
699 m->m_data = m->m_ext.ext_buf;
700 m->m_ext.ext_free = 0;
701 m->m_ext.ext_size = MCLBYTES;
702 m->m_ext.ext_refs.forward = m->m_ext.ext_refs.backward = &m->m_ext.ext_refs;
703
704 if (num_with_pkthdrs == 0)
705 m->m_flags = M_EXT;
706 else {
707 m->m_flags = M_PKTHDR | M_EXT;
708 m->m_pkthdr.len = 0;
709 m->m_pkthdr.rcvif = NULL;
710 m->m_pkthdr.header = NULL;
711 m->m_pkthdr.csum_flags = 0;
712 m->m_pkthdr.csum_data = 0;
713 m->m_pkthdr.aux = (struct mbuf *)NULL;
714 m->m_pkthdr.reserved1 = NULL;
715 m->m_pkthdr.reserved2 = NULL;
716
717 num_with_pkthdrs--;
718 }
719
720 } else {
721
722 MBUF_UNLOCK();
723
724 if (num_with_pkthdrs == 0) {
725 MGET(m, how, MT_DATA );
726 } else {
727 MGETHDR(m, how, MT_DATA);
728
729 num_with_pkthdrs--;
730 }
731 if (m == 0)
732 return(top);
733
734 MCLGET(m, how);
735 if ((m->m_flags & M_EXT) == 0) {
736 m_free(m);
737 return(top);
738 }
739 MBUF_LOCK();
740 }
741 *np = m;
742
743 if (num_with_pkthdrs)
744 np = &m->m_nextpkt;
745 else
746 np = &m->m_next;
747 }
748 MBUF_UNLOCK();
749
750 return (top);
751 }
752
753
754 /*
755 * return a list of mbuf hdrs set up as packet hdrs
756 * chained together on the m_nextpkt field
757 */
758 struct mbuf *
759 m_getpackethdrs(int num_needed, int how)
760 {
761 struct mbuf *m;
762 struct mbuf **np, *top;
763
764 top = NULL;
765 np = &top;
766
767 MBUF_LOCK();
768
769 while (num_needed--) {
770 if (m = mfree) { /* mbufs are available */
771 MCHECK(m);
772 mfree = m->m_next;
773 ++mclrefcnt[mtocl(m)];
774 mbstat.m_mtypes[MT_FREE]--;
775 mbstat.m_mtypes[MT_DATA]++;
776
777 m->m_next = m->m_nextpkt = 0;
778 m->m_type = MT_DATA;
779 m->m_flags = M_PKTHDR;
780 m->m_data = m->m_pktdat;
781 m->m_pkthdr.len = 0;
782 m->m_pkthdr.rcvif = NULL;
783 m->m_pkthdr.header = NULL;
784 m->m_pkthdr.csum_flags = 0;
785 m->m_pkthdr.csum_data = 0;
786 m->m_pkthdr.aux = (struct mbuf *)NULL;
787 m->m_pkthdr.reserved1 = NULL;
788 m->m_pkthdr.reserved2 = NULL;
789
790 } else {
791
792 MBUF_UNLOCK();
793
794 m = m_retryhdr(how, MT_DATA);
795
796 if (m == 0)
797 return(top);
798
799 MBUF_LOCK();
800 }
801 *np = m;
802 np = &m->m_nextpkt;
803 }
804 MBUF_UNLOCK();
805
806 return (top);
807 }
808
809
810 /* free and mbuf list (m_nextpkt) while following m_next under one lock.
811 * returns the count for mbufs packets freed. Used by the drivers.
812 */
813 int
814 m_freem_list(m)
815 struct mbuf *m;
816 {
817 struct mbuf *nextpkt;
818 int i, count=0;
819
820 MBUF_LOCK();
821
822 while (m) {
823 if (m)
824 nextpkt = m->m_nextpkt; /* chain of linked mbufs from driver */
825 else
826 nextpkt = 0;
827
828 count++;
829
830 while (m) { /* free the mbuf chain (like mfreem) */
831
832 struct mbuf *n;
833
834 /* Free the aux data if there is any */
835 if ((m->m_flags & M_PKTHDR) && m->m_pkthdr.aux) {
836 /*
837 * Treat the current m as the nextpkt and set m
838 * to the aux data. This lets us free the aux
839 * data in this loop without having to call
840 * m_freem recursively, which wouldn't work
841 * because we've still got the lock.
842 */
843 nextpkt = m;
844 m = nextpkt->m_pkthdr.aux;
845 nextpkt->m_pkthdr.aux = NULL;
846 }
847
848 n = m->m_next;
849
850 if (n && n->m_nextpkt)
851 panic("m_freem_list: m_nextpkt of m_next != NULL");
852 if (m->m_type == MT_FREE)
853 panic("freeing free mbuf");
854
855 if (m->m_flags & M_EXT) {
856 if (MCLHASREFERENCE(m)) {
857 remque((queue_t)&m->m_ext.ext_refs);
858 } else if (m->m_ext.ext_free == NULL) {
859 union mcluster *mcl= (union mcluster *)m->m_ext.ext_buf;
860 if (_MCLUNREF(mcl)) {
861 mcl->mcl_next = mclfree;
862 mclfree = mcl;
863 ++mbstat.m_clfree;
864 }
865 } else {
866 (*(m->m_ext.ext_free))(m->m_ext.ext_buf,
867 m->m_ext.ext_size, m->m_ext.ext_arg);
868 }
869 }
870 mbstat.m_mtypes[m->m_type]--;
871 (void) _MCLUNREF(m);
872 _MFREE_MUNGE(m);
873 mbstat.m_mtypes[MT_FREE]++;
874 m->m_type = MT_FREE;
875 m->m_flags = 0;
876 m->m_len = 0;
877 m->m_next = mfree;
878 mfree = m;
879 m = n;
880 }
881 m = nextpkt; /* bump m with saved nextpkt if any */
882 }
883 if (i = m_want)
884 m_want = 0;
885
886 MBUF_UNLOCK();
887
888 if (i)
889 wakeup((caddr_t)&mfree);
890
891 return (count);
892 }
893
894 void
895 m_freem(m)
896 register struct mbuf *m;
897 {
898 while (m)
899 m = m_free(m);
900 }
901
902 /*
903 * Mbuffer utility routines.
904 */
905 /*
906 * Compute the amount of space available
907 * before the current start of data in an mbuf.
908 */
909 m_leadingspace(m)
910 register struct mbuf *m;
911 {
912 if (m->m_flags & M_EXT) {
913 if (MCLHASREFERENCE(m))
914 return(0);
915 return (m->m_data - m->m_ext.ext_buf);
916 }
917 if (m->m_flags & M_PKTHDR)
918 return (m->m_data - m->m_pktdat);
919 return (m->m_data - m->m_dat);
920 }
921
922 /*
923 * Compute the amount of space available
924 * after the end of data in an mbuf.
925 */
926 m_trailingspace(m)
927 register struct mbuf *m;
928 {
929 if (m->m_flags & M_EXT) {
930 if (MCLHASREFERENCE(m))
931 return(0);
932 return (m->m_ext.ext_buf + m->m_ext.ext_size -
933 (m->m_data + m->m_len));
934 }
935 return (&m->m_dat[MLEN] - (m->m_data + m->m_len));
936 }
937
938 /*
939 * Lesser-used path for M_PREPEND:
940 * allocate new mbuf to prepend to chain,
941 * copy junk along.
942 * Does not adjust packet header length.
943 */
944 struct mbuf *
945 m_prepend(m, len, how)
946 register struct mbuf *m;
947 int len, how;
948 {
949 struct mbuf *mn;
950
951 MGET(mn, how, m->m_type);
952 if (mn == (struct mbuf *)NULL) {
953 m_freem(m);
954 return ((struct mbuf *)NULL);
955 }
956 if (m->m_flags & M_PKTHDR) {
957 M_COPY_PKTHDR(mn, m);
958 m->m_flags &= ~M_PKTHDR;
959 }
960 mn->m_next = m;
961 m = mn;
962 if (len < MHLEN)
963 MH_ALIGN(m, len);
964 m->m_len = len;
965 return (m);
966 }
967
968 /*
969 * Replacement for old M_PREPEND macro:
970 * allocate new mbuf to prepend to chain,
971 * copy junk along, and adjust length.
972 *
973 */
974 struct mbuf *
975 m_prepend_2(m, len, how)
976 register struct mbuf *m;
977 int len, how;
978 {
979 if (M_LEADINGSPACE(m) >= len) {
980 m->m_data -= len;
981 m->m_len += len;
982 } else {
983 m = m_prepend(m, len, how);
984 }
985 if ((m) && (m->m_flags & M_PKTHDR))
986 m->m_pkthdr.len += len;
987 return (m);
988 }
989
990 /*
991 * Make a copy of an mbuf chain starting "off0" bytes from the beginning,
992 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf.
993 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller.
994 */
995 int MCFail;
996
997 struct mbuf *
998 m_copym(m, off0, len, wait)
999 register struct mbuf *m;
1000 int off0, wait;
1001 register int len;
1002 {
1003 register struct mbuf *n, **np;
1004 register int off = off0;
1005 struct mbuf *top;
1006 int copyhdr = 0;
1007
1008 if (off < 0 || len < 0)
1009 panic("m_copym");
1010 if (off == 0 && m->m_flags & M_PKTHDR)
1011 copyhdr = 1;
1012
1013 while (off >= m->m_len) {
1014 if (m == 0)
1015 panic("m_copym");
1016 off -= m->m_len;
1017 m = m->m_next;
1018 }
1019 np = &top;
1020 top = 0;
1021
1022 MBUF_LOCK();
1023
1024 while (len > 0) {
1025 if (m == 0) {
1026 if (len != M_COPYALL)
1027 panic("m_copym");
1028 break;
1029 }
1030 if (n = mfree) {
1031 MCHECK(n);
1032 ++mclrefcnt[mtocl(n)];
1033 mbstat.m_mtypes[MT_FREE]--;
1034 mbstat.m_mtypes[m->m_type]++;
1035 mfree = n->m_next;
1036 n->m_next = n->m_nextpkt = 0;
1037 n->m_type = m->m_type;
1038 n->m_data = n->m_dat;
1039 n->m_flags = 0;
1040 } else {
1041 MBUF_UNLOCK();
1042 n = m_retry(wait, m->m_type);
1043 MBUF_LOCK();
1044 }
1045 *np = n;
1046
1047 if (n == 0)
1048 goto nospace;
1049 if (copyhdr) {
1050 M_COPY_PKTHDR(n, m);
1051 if (len == M_COPYALL)
1052 n->m_pkthdr.len -= off0;
1053 else
1054 n->m_pkthdr.len = len;
1055 copyhdr = 0;
1056 }
1057 if (len == M_COPYALL) {
1058 if (min(len, (m->m_len - off)) == len) {
1059 printf("m->m_len %d - off %d = %d, %d\n",
1060 m->m_len, off, m->m_len - off,
1061 min(len, (m->m_len - off)));
1062 }
1063 }
1064 n->m_len = min(len, (m->m_len - off));
1065 if (n->m_len == M_COPYALL) {
1066 printf("n->m_len == M_COPYALL, fixing\n");
1067 n->m_len = MHLEN;
1068 }
1069 if (m->m_flags & M_EXT) {
1070 n->m_ext = m->m_ext;
1071 insque((queue_t)&n->m_ext.ext_refs, (queue_t)&m->m_ext.ext_refs);
1072 n->m_data = m->m_data + off;
1073 n->m_flags |= M_EXT;
1074 } else {
1075 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
1076 (unsigned)n->m_len);
1077 }
1078 if (len != M_COPYALL)
1079 len -= n->m_len;
1080 off = 0;
1081 m = m->m_next;
1082 np = &n->m_next;
1083 }
1084 MBUF_UNLOCK();
1085
1086 if (top == 0)
1087 MCFail++;
1088
1089 return (top);
1090 nospace:
1091 MBUF_UNLOCK();
1092
1093 m_freem(top);
1094 MCFail++;
1095 return (0);
1096 }
1097
1098
1099 /*
1100 * equivilent to m_copym except that all necessary
1101 * mbuf hdrs are allocated within this routine
1102 * also, the last mbuf and offset accessed are passed
1103 * out and can be passed back in to avoid having to
1104 * rescan the entire mbuf list (normally hung off of the socket)
1105 */
1106 struct mbuf *
1107 m_copym_with_hdrs(m, off0, len, wait, m_last, m_off)
1108 register struct mbuf *m;
1109 int off0, wait;
1110 register int len;
1111 struct mbuf **m_last;
1112 int *m_off;
1113 {
1114 register struct mbuf *n, **np;
1115 register int off = off0;
1116 struct mbuf *top = 0;
1117 int copyhdr = 0;
1118 int type;
1119
1120 if (off == 0 && m->m_flags & M_PKTHDR)
1121 copyhdr = 1;
1122
1123 if (*m_last) {
1124 m = *m_last;
1125 off = *m_off;
1126 } else {
1127 while (off >= m->m_len) {
1128 off -= m->m_len;
1129 m = m->m_next;
1130 }
1131 }
1132 MBUF_LOCK();
1133
1134 while (len > 0) {
1135 if (top == 0)
1136 type = MT_HEADER;
1137 else {
1138 if (m == 0)
1139 panic("m_gethdr_and_copym");
1140 type = m->m_type;
1141 }
1142 if (n = mfree) {
1143 MCHECK(n);
1144 ++mclrefcnt[mtocl(n)];
1145 mbstat.m_mtypes[MT_FREE]--;
1146 mbstat.m_mtypes[type]++;
1147 mfree = n->m_next;
1148 n->m_next = n->m_nextpkt = 0;
1149 n->m_type = type;
1150
1151 if (top) {
1152 n->m_data = n->m_dat;
1153 n->m_flags = 0;
1154 } else {
1155 n->m_data = n->m_pktdat;
1156 n->m_flags = M_PKTHDR;
1157 n->m_pkthdr.len = 0;
1158 n->m_pkthdr.rcvif = NULL;
1159 n->m_pkthdr.header = NULL;
1160 n->m_pkthdr.csum_flags = 0;
1161 n->m_pkthdr.csum_data = 0;
1162 n->m_pkthdr.aux = (struct mbuf *)NULL;
1163 n->m_pkthdr.reserved1 = NULL;
1164 n->m_pkthdr.reserved2 = NULL;
1165 }
1166 } else {
1167 MBUF_UNLOCK();
1168 if (top)
1169 n = m_retry(wait, type);
1170 else
1171 n = m_retryhdr(wait, type);
1172 MBUF_LOCK();
1173 }
1174 if (n == 0)
1175 goto nospace;
1176 if (top == 0) {
1177 top = n;
1178 np = &top->m_next;
1179 continue;
1180 } else
1181 *np = n;
1182
1183 if (copyhdr) {
1184 M_COPY_PKTHDR(n, m);
1185 n->m_pkthdr.len = len;
1186 copyhdr = 0;
1187 }
1188 n->m_len = min(len, (m->m_len - off));
1189
1190 if (m->m_flags & M_EXT) {
1191 n->m_ext = m->m_ext;
1192 insque((queue_t)&n->m_ext.ext_refs, (queue_t)&m->m_ext.ext_refs);
1193 n->m_data = m->m_data + off;
1194 n->m_flags |= M_EXT;
1195 } else {
1196 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
1197 (unsigned)n->m_len);
1198 }
1199 len -= n->m_len;
1200
1201 if (len == 0) {
1202 if ((off + n->m_len) == m->m_len) {
1203 *m_last = m->m_next;
1204 *m_off = 0;
1205 } else {
1206 *m_last = m;
1207 *m_off = off + n->m_len;
1208 }
1209 break;
1210 }
1211 off = 0;
1212 m = m->m_next;
1213 np = &n->m_next;
1214 }
1215 MBUF_UNLOCK();
1216
1217 return (top);
1218 nospace:
1219 MBUF_UNLOCK();
1220
1221 if (top)
1222 m_freem(top);
1223 MCFail++;
1224 return (0);
1225 }
1226
1227
1228 /*
1229 * Copy data from an mbuf chain starting "off" bytes from the beginning,
1230 * continuing for "len" bytes, into the indicated buffer.
1231 */
1232 void m_copydata(m, off, len, cp)
1233 register struct mbuf *m;
1234 register int off;
1235 register int len;
1236 caddr_t cp;
1237 {
1238 register unsigned count;
1239
1240 if (off < 0 || len < 0)
1241 panic("m_copydata");
1242 while (off > 0) {
1243 if (m == 0)
1244 panic("m_copydata");
1245 if (off < m->m_len)
1246 break;
1247 off -= m->m_len;
1248 m = m->m_next;
1249 }
1250 while (len > 0) {
1251 if (m == 0)
1252 panic("m_copydata");
1253 count = min(m->m_len - off, len);
1254 bcopy(mtod(m, caddr_t) + off, cp, count);
1255 len -= count;
1256 cp += count;
1257 off = 0;
1258 m = m->m_next;
1259 }
1260 }
1261
1262 /*
1263 * Concatenate mbuf chain n to m.
1264 * Both chains must be of the same type (e.g. MT_DATA).
1265 * Any m_pkthdr is not updated.
1266 */
1267 void m_cat(m, n)
1268 register struct mbuf *m, *n;
1269 {
1270 while (m->m_next)
1271 m = m->m_next;
1272 while (n) {
1273 if (m->m_flags & M_EXT ||
1274 m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) {
1275 /* just join the two chains */
1276 m->m_next = n;
1277 return;
1278 }
1279 /* splat the data from one into the other */
1280 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
1281 (u_int)n->m_len);
1282 m->m_len += n->m_len;
1283 n = m_free(n);
1284 }
1285 }
1286
1287 void
1288 m_adj(mp, req_len)
1289 struct mbuf *mp;
1290 int req_len;
1291 {
1292 register int len = req_len;
1293 register struct mbuf *m;
1294 register count;
1295
1296 if ((m = mp) == NULL)
1297 return;
1298 if (len >= 0) {
1299 /*
1300 * Trim from head.
1301 */
1302 while (m != NULL && len > 0) {
1303 if (m->m_len <= len) {
1304 len -= m->m_len;
1305 m->m_len = 0;
1306 m = m->m_next;
1307 } else {
1308 m->m_len -= len;
1309 m->m_data += len;
1310 len = 0;
1311 }
1312 }
1313 m = mp;
1314 if (m->m_flags & M_PKTHDR)
1315 m->m_pkthdr.len -= (req_len - len);
1316 } else {
1317 /*
1318 * Trim from tail. Scan the mbuf chain,
1319 * calculating its length and finding the last mbuf.
1320 * If the adjustment only affects this mbuf, then just
1321 * adjust and return. Otherwise, rescan and truncate
1322 * after the remaining size.
1323 */
1324 len = -len;
1325 count = 0;
1326 for (;;) {
1327 count += m->m_len;
1328 if (m->m_next == (struct mbuf *)0)
1329 break;
1330 m = m->m_next;
1331 }
1332 if (m->m_len >= len) {
1333 m->m_len -= len;
1334 m = mp;
1335 if (m->m_flags & M_PKTHDR)
1336 m->m_pkthdr.len -= len;
1337 return;
1338 }
1339 count -= len;
1340 if (count < 0)
1341 count = 0;
1342 /*
1343 * Correct length for chain is "count".
1344 * Find the mbuf with last data, adjust its length,
1345 * and toss data from remaining mbufs on chain.
1346 */
1347 m = mp;
1348 if (m->m_flags & M_PKTHDR)
1349 m->m_pkthdr.len = count;
1350 for (; m; m = m->m_next) {
1351 if (m->m_len >= count) {
1352 m->m_len = count;
1353 break;
1354 }
1355 count -= m->m_len;
1356 }
1357 while (m = m->m_next)
1358 m->m_len = 0;
1359 }
1360 }
1361
1362 /*
1363 * Rearange an mbuf chain so that len bytes are contiguous
1364 * and in the data area of an mbuf (so that mtod and dtom
1365 * will work for a structure of size len). Returns the resulting
1366 * mbuf chain on success, frees it and returns null on failure.
1367 * If there is room, it will add up to max_protohdr-len extra bytes to the
1368 * contiguous region in an attempt to avoid being called next time.
1369 */
1370 int MPFail;
1371
1372 struct mbuf *
1373 m_pullup(n, len)
1374 register struct mbuf *n;
1375 int len;
1376 {
1377 register struct mbuf *m;
1378 register int count;
1379 int space;
1380
1381 /*
1382 * If first mbuf has no cluster, and has room for len bytes
1383 * without shifting current data, pullup into it,
1384 * otherwise allocate a new mbuf to prepend to the chain.
1385 */
1386 if ((n->m_flags & M_EXT) == 0 &&
1387 n->m_data + len < &n->m_dat[MLEN] && n->m_next) {
1388 if (n->m_len >= len)
1389 return (n);
1390 m = n;
1391 n = n->m_next;
1392 len -= m->m_len;
1393 } else {
1394 if (len > MHLEN)
1395 goto bad;
1396 MGET(m, M_DONTWAIT, n->m_type);
1397 if (m == 0)
1398 goto bad;
1399 m->m_len = 0;
1400 if (n->m_flags & M_PKTHDR) {
1401 M_COPY_PKTHDR(m, n);
1402 n->m_flags &= ~M_PKTHDR;
1403 }
1404 }
1405 space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
1406 do {
1407 count = min(min(max(len, max_protohdr), space), n->m_len);
1408 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
1409 (unsigned)count);
1410 len -= count;
1411 m->m_len += count;
1412 n->m_len -= count;
1413 space -= count;
1414 if (n->m_len)
1415 n->m_data += count;
1416 else
1417 n = m_free(n);
1418 } while (len > 0 && n);
1419 if (len > 0) {
1420 (void) m_free(m);
1421 goto bad;
1422 }
1423 m->m_next = n;
1424 return (m);
1425 bad:
1426 m_freem(n);
1427 MPFail++;
1428 return (0);
1429 }
1430
1431 /*
1432 * Partition an mbuf chain in two pieces, returning the tail --
1433 * all but the first len0 bytes. In case of failure, it returns NULL and
1434 * attempts to restore the chain to its original state.
1435 */
1436 struct mbuf *
1437 m_split(m0, len0, wait)
1438 register struct mbuf *m0;
1439 int len0, wait;
1440 {
1441 register struct mbuf *m, *n;
1442 unsigned len = len0, remain;
1443
1444 for (m = m0; m && len > m->m_len; m = m->m_next)
1445 len -= m->m_len;
1446 if (m == 0)
1447 return (0);
1448 remain = m->m_len - len;
1449 if (m0->m_flags & M_PKTHDR) {
1450 MGETHDR(n, wait, m0->m_type);
1451 if (n == 0)
1452 return (0);
1453 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
1454 n->m_pkthdr.len = m0->m_pkthdr.len - len0;
1455 m0->m_pkthdr.len = len0;
1456 if (m->m_flags & M_EXT)
1457 goto extpacket;
1458 if (remain > MHLEN) {
1459 /* m can't be the lead packet */
1460 MH_ALIGN(n, 0);
1461 n->m_next = m_split(m, len, wait);
1462 if (n->m_next == 0) {
1463 (void) m_free(n);
1464 return (0);
1465 } else
1466 return (n);
1467 } else
1468 MH_ALIGN(n, remain);
1469 } else if (remain == 0) {
1470 n = m->m_next;
1471 m->m_next = 0;
1472 return (n);
1473 } else {
1474 MGET(n, wait, m->m_type);
1475 if (n == 0)
1476 return (0);
1477 M_ALIGN(n, remain);
1478 }
1479 extpacket:
1480 if (m->m_flags & M_EXT) {
1481 n->m_flags |= M_EXT;
1482 MBUF_LOCK();
1483 n->m_ext = m->m_ext;
1484 insque((queue_t)&n->m_ext.ext_refs, (queue_t)&m->m_ext.ext_refs);
1485 MBUF_UNLOCK();
1486 n->m_data = m->m_data + len;
1487 } else {
1488 bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain);
1489 }
1490 n->m_len = remain;
1491 m->m_len = len;
1492 n->m_next = m->m_next;
1493 m->m_next = 0;
1494 return (n);
1495 }
1496 /*
1497 * Routine to copy from device local memory into mbufs.
1498 */
1499 struct mbuf *
1500 m_devget(buf, totlen, off0, ifp, copy)
1501 char *buf;
1502 int totlen, off0;
1503 struct ifnet *ifp;
1504 void (*copy)();
1505 {
1506 register struct mbuf *m;
1507 struct mbuf *top = 0, **mp = &top;
1508 register int off = off0, len;
1509 register char *cp;
1510 char *epkt;
1511
1512 cp = buf;
1513 epkt = cp + totlen;
1514 if (off) {
1515 /*
1516 * If 'off' is non-zero, packet is trailer-encapsulated,
1517 * so we have to skip the type and length fields.
1518 */
1519 cp += off + 2 * sizeof(u_int16_t);
1520 totlen -= 2 * sizeof(u_int16_t);
1521 }
1522 MGETHDR(m, M_DONTWAIT, MT_DATA);
1523 if (m == 0)
1524 return (0);
1525 m->m_pkthdr.rcvif = ifp;
1526 m->m_pkthdr.len = totlen;
1527 m->m_len = MHLEN;
1528
1529 while (totlen > 0) {
1530 if (top) {
1531 MGET(m, M_DONTWAIT, MT_DATA);
1532 if (m == 0) {
1533 m_freem(top);
1534 return (0);
1535 }
1536 m->m_len = MLEN;
1537 }
1538 len = min(totlen, epkt - cp);
1539 if (len >= MINCLSIZE) {
1540 MCLGET(m, M_DONTWAIT);
1541 if (m->m_flags & M_EXT)
1542 m->m_len = len = min(len, MCLBYTES);
1543 else {
1544 /* give up when it's out of cluster mbufs */
1545 if (top)
1546 m_freem(top);
1547 m_freem(m);
1548 return (0);
1549 }
1550 } else {
1551 /*
1552 * Place initial small packet/header at end of mbuf.
1553 */
1554 if (len < m->m_len) {
1555 if (top == 0 && len + max_linkhdr <= m->m_len)
1556 m->m_data += max_linkhdr;
1557 m->m_len = len;
1558 } else
1559 len = m->m_len;
1560 }
1561 if (copy)
1562 copy(cp, mtod(m, caddr_t), (unsigned)len);
1563 else
1564 bcopy(cp, mtod(m, caddr_t), (unsigned)len);
1565 cp += len;
1566 *mp = m;
1567 mp = &m->m_next;
1568 totlen -= len;
1569 if (cp == epkt)
1570 cp = buf;
1571 }
1572 return (top);
1573 }
1574
1575 /*
1576 * Cluster freelist allocation check. The mbuf lock must be held.
1577 * Ensure hysteresis between hi/lo.
1578 */
1579 static int
1580 m_howmany()
1581 {
1582 register int i;
1583
1584 /* Under minimum */
1585 if (mbstat.m_clusters < MINCL)
1586 return (MINCL - mbstat.m_clusters);
1587 /* Too few (free < 1/2 total) and not over maximum */
1588 if (mbstat.m_clusters < nmbclusters &&
1589 (i = ((mbstat.m_clusters >> 1) - mbstat.m_clfree)) > 0)
1590 return i;
1591 return 0;
1592 }
1593
1594
1595 /*
1596 * Copy data from a buffer back into the indicated mbuf chain,
1597 * starting "off" bytes from the beginning, extending the mbuf
1598 * chain if necessary.
1599 */
1600 void
1601 m_copyback(m0, off, len, cp)
1602 struct mbuf *m0;
1603 register int off;
1604 register int len;
1605 caddr_t cp;
1606 {
1607 register int mlen;
1608 register struct mbuf *m = m0, *n;
1609 int totlen = 0;
1610
1611 if (m0 == 0)
1612 return;
1613 while (off > (mlen = m->m_len)) {
1614 off -= mlen;
1615 totlen += mlen;
1616 if (m->m_next == 0) {
1617 n = m_getclr(M_DONTWAIT, m->m_type);
1618 if (n == 0)
1619 goto out;
1620 n->m_len = min(MLEN, len + off);
1621 m->m_next = n;
1622 }
1623 m = m->m_next;
1624 }
1625 while (len > 0) {
1626 mlen = min (m->m_len - off, len);
1627 bcopy(cp, off + mtod(m, caddr_t), (unsigned)mlen);
1628 cp += mlen;
1629 len -= mlen;
1630 mlen += off;
1631 off = 0;
1632 totlen += mlen;
1633 if (len == 0)
1634 break;
1635 if (m->m_next == 0) {
1636 n = m_get(M_DONTWAIT, m->m_type);
1637 if (n == 0)
1638 break;
1639 n->m_len = min(MLEN, len);
1640 m->m_next = n;
1641 }
1642 m = m->m_next;
1643 }
1644 out: if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen))
1645 m->m_pkthdr.len = totlen;
1646 }
1647
1648
1649 char *mcl_to_paddr(register char *addr) {
1650 register int base_phys;
1651
1652 if (addr < (char *)mbutl || addr >= (char *)embutl)
1653 return (0);
1654 base_phys = mcl_paddr[(addr - (char *)mbutl) >> PAGE_SHIFT];
1655
1656 if (base_phys == 0)
1657 return (0);
1658 return ((char *)((int)base_phys | ((int)addr & PAGE_MASK)));
1659 }
1660
1661 /*
1662 * Dup the mbuf chain passed in. The whole thing. No cute additional cruft.
1663 * And really copy the thing. That way, we don't "precompute" checksums
1664 * for unsuspecting consumers.
1665 * Assumption: m->m_nextpkt == 0.
1666 * Trick: for small packets, don't dup into a cluster. That way received
1667 * packets don't take up too much room in the sockbuf (cf. sbspace()).
1668 */
1669 int MDFail;
1670
1671 struct mbuf *
1672 m_dup(register struct mbuf *m, int how)
1673 { register struct mbuf *n, **np;
1674 struct mbuf *top;
1675 int copyhdr = 0;
1676
1677 np = &top;
1678 top = 0;
1679 if (m->m_flags & M_PKTHDR)
1680 copyhdr = 1;
1681
1682 /*
1683 * Quick check: if we have one mbuf and its data fits in an
1684 * mbuf with packet header, just copy and go.
1685 */
1686 if (m->m_next == NULL)
1687 { /* Then just move the data into an mbuf and be done... */
1688 if (copyhdr)
1689 { if (m->m_pkthdr.len <= MHLEN)
1690 { if ((n = m_gethdr(how, m->m_type)) == NULL)
1691 return(NULL);
1692 n->m_len = m->m_len;
1693 n->m_flags |= (m->m_flags & M_COPYFLAGS);
1694 n->m_pkthdr.len = m->m_pkthdr.len;
1695 n->m_pkthdr.rcvif = m->m_pkthdr.rcvif;
1696 n->m_pkthdr.header = NULL;
1697 n->m_pkthdr.csum_flags = 0;
1698 n->m_pkthdr.csum_data = 0;
1699 n->m_pkthdr.aux = NULL;
1700 n->m_pkthdr.reserved1 = 0;
1701 n->m_pkthdr.reserved2 = 0;
1702 bcopy(m->m_data, n->m_data, m->m_pkthdr.len);
1703 return(n);
1704 }
1705 } else if (m->m_len <= MLEN)
1706 { if ((n = m_get(how, m->m_type)) == NULL)
1707 return(NULL);
1708 bcopy(m->m_data, n->m_data, m->m_len);
1709 n->m_len = m->m_len;
1710 return(n);
1711 }
1712 }
1713 while (m)
1714 {
1715 #if BLUE_DEBUG
1716 kprintf("<%x: %x, %x, %x\n", m, m->m_flags, m->m_len,
1717 m->m_data);
1718 #endif
1719 if (copyhdr)
1720 n = m_gethdr(how, m->m_type);
1721 else
1722 n = m_get(how, m->m_type);
1723 if (n == 0)
1724 goto nospace;
1725 if (m->m_flags & M_EXT)
1726 { MCLGET(n, how);
1727 if ((n->m_flags & M_EXT) == 0)
1728 goto nospace;
1729 }
1730 *np = n;
1731 if (copyhdr)
1732 { /* Don't use M_COPY_PKTHDR: preserve m_data */
1733 n->m_pkthdr = m->m_pkthdr;
1734 n->m_flags |= (m->m_flags & M_COPYFLAGS);
1735 copyhdr = 0;
1736 if ((n->m_flags & M_EXT) == 0)
1737 n->m_data = n->m_pktdat;
1738 }
1739 n->m_len = m->m_len;
1740 /*
1741 * Get the dup on the same bdry as the original
1742 * Assume that the two mbufs have the same offset to data area
1743 * (up to word bdries)
1744 */
1745 bcopy(mtod(m, caddr_t), mtod(n, caddr_t), (unsigned)n->m_len);
1746 m = m->m_next;
1747 np = &n->m_next;
1748 #if BLUE_DEBUG
1749 kprintf(">%x: %x, %x, %x\n", n, n->m_flags, n->m_len,
1750 n->m_data);
1751 #endif
1752 }
1753
1754 if (top == 0)
1755 MDFail++;
1756 return (top);
1757 nospace:
1758 m_freem(top);
1759 MDFail++;
1760 return (0);
1761 }
1762
1763 int
1764 m_mclref(struct mbuf *p)
1765 {
1766 return (_MCLREF(p));
1767 }
1768
1769 int
1770 m_mclunref(struct mbuf *p)
1771 {
1772 return (_MCLUNREF(p));
1773 }
1774
1775 /* change mbuf to new type */
1776 void
1777 m_mchtype(struct mbuf *m, int t)
1778 {
1779 MBUF_LOCK();
1780 mbstat.m_mtypes[(m)->m_type]--;
1781 mbstat.m_mtypes[t]++;
1782 (m)->m_type = t;
1783 MBUF_UNLOCK();
1784 }
1785
1786 void *m_mtod(struct mbuf *m)
1787 {
1788 return ((m)->m_data);
1789 }
1790
1791 struct mbuf *m_dtom(void *x)
1792 {
1793 return ((struct mbuf *)((u_long)(x) & ~(MSIZE-1)));
1794 }
1795
1796 int m_mtocl(void *x)
1797 {
1798 return (((char *)(x) - (char *)mbutl) / sizeof(union mcluster));
1799 }
1800
1801 union mcluster *m_cltom(int x)
1802 {
1803 return ((union mcluster *)(mbutl + (x)));
1804 }
1805
1806
1807 void m_mcheck(struct mbuf *m)
1808 {
1809 if (m->m_type != MT_FREE)
1810 panic("mget MCHECK: m_type=%x m=%x", m->m_type, m);
1811 }
1812
1813 #if 0
1814 #include <sys/sysctl.h>
1815
1816 static int mhog_num = 0;
1817 static struct mbuf *mhog_chain = 0;
1818 static int mhog_wait = 1;
1819
1820 static int
1821 sysctl_mhog_num SYSCTL_HANDLER_ARGS
1822 {
1823 int old = mhog_num;
1824 int error;
1825
1826 error = sysctl_handle_int(oidp, oidp->oid_arg1, oidp->oid_arg2, req);
1827 if (!error && req->newptr) {
1828 int i;
1829 struct mbuf *m;
1830
1831 if (mhog_chain) {
1832 m_freem(mhog_chain);
1833 mhog_chain = 0;
1834 }
1835
1836 for (i = 0; i < mhog_num; i++) {
1837 MGETHDR(m, mhog_wait ? M_WAIT : M_DONTWAIT, MT_DATA);
1838 if (m == 0)
1839 break;
1840
1841 MCLGET(m, mhog_wait ? M_WAIT : M_DONTWAIT);
1842 if ((m->m_flags & M_EXT) == 0) {
1843 m_free(m);
1844 m = 0;
1845 break;
1846 }
1847 m->m_next = mhog_chain;
1848 mhog_chain = m;
1849 }
1850 mhog_num = i;
1851 }
1852
1853 return error;
1854 }
1855
1856 SYSCTL_NODE(_kern_ipc, OID_AUTO, mhog, CTLFLAG_RW, 0, "mbuf hog");
1857
1858 SYSCTL_PROC(_kern_ipc_mhog, OID_AUTO, cluster, CTLTYPE_INT|CTLFLAG_RW,
1859 &mhog_num, 0, &sysctl_mhog_num, "I", "");
1860 SYSCTL_INT(_kern_ipc_mhog, OID_AUTO, wait, CTLFLAG_RW, &mhog_wait,
1861 0, "");
1862 #endif
1863
mirror of XNU