2 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
4 * @APPLE_LICENSE_HEADER_START@
6 * The contents of this file constitute Original Code as defined in and
7 * are subject to the Apple Public Source License Version 1.1 (the
8 * "License"). You may not use this file except in compliance with the
9 * License. Please obtain a copy of the License at
10 * http://www.apple.com/publicsource and read it before using this file.
12 * This Original Code and all software distributed under the License are
13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17 * License for the specific language governing rights and limitations
20 * @APPLE_LICENSE_HEADER_END@
22 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
24 * Copyright (c) 1982, 1986, 1988, 1991, 1993
25 * The Regents of the University of California. All rights reserved.
27 * Redistribution and use in source and binary forms, with or without
28 * modification, are permitted provided that the following conditions
30 * 1. Redistributions of source code must retain the above copyright
31 * notice, this list of conditions and the following disclaimer.
32 * 2. Redistributions in binary form must reproduce the above copyright
33 * notice, this list of conditions and the following disclaimer in the
34 * documentation and/or other materials provided with the distribution.
35 * 3. All advertising materials mentioning features or use of this software
36 * must display the following acknowledgement:
37 * This product includes software developed by the University of
38 * California, Berkeley and its contributors.
39 * 4. Neither the name of the University nor the names of its contributors
40 * may be used to endorse or promote products derived from this software
41 * without specific prior written permission.
43 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
44 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
45 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
46 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
47 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
48 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
49 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
50 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
51 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
52 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
55 * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94
59 * 10/15/97 Annette DeSchon (deschon@apple.com)
60 * Fixed bug in which all cluster mbufs were broken up
61 * into regular mbufs: Some clusters are now reserved.
62 * When a cluster is needed, regular mbufs are no longer
63 * used. (Radar 1683621)
64 * 20-May-95 Mac Gillon (mgillon) at NeXT
65 * New version based on 4.4
68 #include <sys/param.h>
69 #include <sys/systm.h>
70 #include <sys/malloc.h>
72 #include <sys/kernel.h>
73 #include <sys/syslog.h>
74 #include <sys/protosw.h>
75 #include <sys/domain.h>
76 #include <net/netisr.h>
78 #include <kern/queue.h>
79 #include <kern/kern_types.h>
80 #include <kern/sched_prim.h>
82 #include <IOKit/IOMapper.h>
84 #define _MCLREF(p) (++mclrefcnt[mtocl(p)])
85 #define _MCLUNREF(p) (--mclrefcnt[mtocl(p)] == 0)
86 #define _M_CLEAR_PKTHDR(mbuf_ptr) (mbuf_ptr)->m_pkthdr.rcvif = NULL; \
87 (mbuf_ptr)->m_pkthdr.len = 0; \
88 (mbuf_ptr)->m_pkthdr.header = NULL; \
89 (mbuf_ptr)->m_pkthdr.csum_flags = 0; \
90 (mbuf_ptr)->m_pkthdr.csum_data = 0; \
91 (mbuf_ptr)->m_pkthdr.aux = (struct mbuf*)NULL; \
92 (mbuf_ptr)->m_pkthdr.reserved_1 = 0; \
93 (mbuf_ptr)->m_pkthdr.vlan_tag = 0; \
94 (mbuf_ptr)->m_pkthdr.reserved2 = NULL;
96 extern pmap_t kernel_pmap
; /* The kernel's pmap */
97 /* kernel translater */
98 extern ppnum_t
pmap_find_phys(pmap_t pmap
, addr64_t va
);
100 decl_simple_lock_data(, mbuf_slock
);
101 struct mbuf
*mfree
; /* mbuf free list */
102 struct mbuf
*mfreelater
; /* mbuf deallocation list */
103 extern vm_map_t mb_map
; /* special map */
104 int m_want
; /* sleepers on mbufs */
105 extern int nmbclusters
; /* max number of mapped clusters */
106 short *mclrefcnt
; /* mapped cluster reference counts */
108 static ppnum_t mcl_paddr_base
; /* Handle returned by IOMapper::iovmAlloc() */
109 union mcluster
*mclfree
; /* mapped cluster free list */
110 int max_linkhdr
; /* largest link-level header */
111 int max_protohdr
; /* largest protocol header */
112 int max_hdr
; /* largest link+protocol header */
113 int max_datalen
; /* MHLEN - max_hdr */
114 struct mbstat mbstat
; /* statistics */
115 union mcluster
*mbutl
; /* first mapped cluster address */
116 union mcluster
*embutl
; /* ending virtual address of mclusters */
118 static int nclpp
; /* # clusters per physical page */
119 static char mbfail
[] = "mbuf not mapped";
121 static int m_howmany();
123 /* The number of cluster mbufs that are allocated, to start. */
124 #define MINCL max(16, 2)
126 static int mbuf_expand_thread_wakeup
= 0;
127 static int mbuf_expand_mcl
= 0;
128 static int mbuf_expand_thread_initialized
= 0;
130 static void mbuf_expand_thread_init(void);
133 static int mfree_munge
= 0;
135 #define _MFREE_MUNGE(m) { \
138 vm_offset_t *element = (vm_offset_t *)(m); \
140 i < sizeof(struct mbuf)/sizeof(vm_offset_t); \
142 (element)[i] = 0xdeadbeef; \
147 munge_mbuf(struct mbuf
*m
)
150 vm_offset_t
*element
= (vm_offset_t
*)(m
);
152 i
< sizeof(struct mbuf
)/sizeof(vm_offset_t
);
154 (element
)[i
] = 0xdeadbeef;
156 #define _MFREE_MUNGE(m) { \
162 #define _MFREE_MUNGE(m)
166 #define _MINTGET(m, type) { \
168 if (((m) = mfree) != 0) { \
170 ++mclrefcnt[mtocl(m)]; \
171 mbstat.m_mtypes[MT_FREE]--; \
172 mbstat.m_mtypes[(type)]++; \
173 mfree = (m)->m_next; \
188 nclpp
= round_page_32(MCLBYTES
) / MCLBYTES
; /* see mbufgc() */
189 if (nclpp
< 1) nclpp
= 1;
191 // NETISR_LOCKINIT();
193 mbstat
.m_msize
= MSIZE
;
194 mbstat
.m_mclbytes
= MCLBYTES
;
195 mbstat
.m_minclsize
= MINCLSIZE
;
196 mbstat
.m_mlen
= MLEN
;
197 mbstat
.m_mhlen
= MHLEN
;
199 if (nmbclusters
== 0)
200 nmbclusters
= NMBCLUSTERS
;
201 MALLOC(mclrefcnt
, short *, nmbclusters
* sizeof (short),
205 for (m
= 0; m
< nmbclusters
; m
++)
208 /* Calculate the number of pages assigned to the cluster pool */
209 mcl_pages
= nmbclusters
/(PAGE_SIZE
/CLBYTES
);
210 MALLOC(mcl_paddr
, int *, mcl_pages
* sizeof(int), M_TEMP
, M_WAITOK
);
213 /* Register with the I/O Bus mapper */
214 mcl_paddr_base
= IOMapperIOVMAlloc(mcl_pages
);
215 bzero((char *)mcl_paddr
, mcl_pages
* sizeof(int));
217 embutl
= (union mcluster
*)((unsigned char *)mbutl
+ (nmbclusters
* MCLBYTES
));
219 PE_parse_boot_arg("initmcl", &initmcl
);
221 if (m_clalloc(max(PAGE_SIZE
/CLBYTES
, 1) * initmcl
, M_WAIT
) == 0)
225 (void) kernel_thread(kernel_task
, mbuf_expand_thread_init
);
233 * Allocate some number of mbuf clusters
234 * and place on cluster free list.
237 m_clalloc(ncl
, nowait
)
241 register union mcluster
*mcl
;
244 static char doing_alloc
;
247 * Honor the caller's wish to block or not block.
248 * We have a way to grow the pool asynchronously,
249 * by kicking the dlil_input_thread.
251 if ((i
= m_howmany()) <= 0)
254 if ((nowait
== M_DONTWAIT
))
259 size
= round_page_32(ncl
* MCLBYTES
);
260 mcl
= (union mcluster
*)kmem_mb_alloc(mb_map
, size
);
262 if (mcl
== 0 && ncl
> 1) {
263 size
= round_page_32(MCLBYTES
); /* Try for 1 if failed */
264 mcl
= (union mcluster
*)kmem_mb_alloc(mb_map
, size
);
269 ncl
= size
/ MCLBYTES
;
270 for (i
= 0; i
< ncl
; i
++) {
271 if (++mclrefcnt
[mtocl(mcl
)] != 0)
272 panic("m_clalloc already there");
273 if (((int)mcl
& PAGE_MASK
) == 0) {
274 ppnum_t offset
= ((char *)mcl
- (char *)mbutl
)/PAGE_SIZE
;
275 ppnum_t new_page
= pmap_find_phys(kernel_pmap
, (vm_address_t
) mcl
);
278 * In the case of no mapper being available
279 * the following code nops and returns the
280 * input page, if there is a mapper the I/O
281 * page appropriate is returned.
283 new_page
= IOMapperInsertPage(mcl_paddr_base
, offset
, new_page
);
284 mcl_paddr
[offset
] = new_page
<< 12;
287 mcl
->mcl_next
= mclfree
;
290 mbstat
.m_clfree
+= ncl
;
291 mbstat
.m_clusters
+= ncl
;
298 * When non-blocking we kick the dlil thread if we havve to grow the
299 * pool or if the number of free clusters is less than requested.
301 if ((nowait
== M_DONTWAIT
) && (i
> 0 || ncl
>= mbstat
.m_clfree
)) {
303 if (mbuf_expand_thread_initialized
)
304 wakeup((caddr_t
)&mbuf_expand_thread_wakeup
);
307 if (mbstat
.m_clfree
>= ncl
)
314 * Add more free mbufs by cutting up a cluster.
319 register caddr_t mcl
;
321 if (mbstat
.m_clfree
< (mbstat
.m_clusters
>> 4))
322 /* 1/16th of the total number of cluster mbufs allocated is
323 reserved for large packets. The number reserved must
324 always be < 1/2, or future allocation will be prevented.
328 MCLALLOC(mcl
, canwait
);
330 register struct mbuf
*m
= (struct mbuf
*)mcl
;
331 register int i
= NMBPCL
;
333 mbstat
.m_mtypes
[MT_FREE
] += i
;
344 if (i
) wakeup((caddr_t
)&mfree
);
351 * When MGET failes, ask protocols to free space when short of memory,
352 * then re-attempt to allocate an mbuf.
355 m_retry(canwait
, type
)
358 register struct mbuf
*m
;
362 boolean_t funnel_state
;
365 (void) m_expand(canwait
);
368 (m
)->m_next
= (m
)->m_nextpkt
= 0;
369 (m
)->m_type
= (type
);
370 (m
)->m_data
= (m
)->m_dat
;
373 if (m
|| canwait
== M_DONTWAIT
)
384 if (mbuf_expand_thread_initialized
)
385 wakeup((caddr_t
)&mbuf_expand_thread_wakeup
);
388 * Need to be inside network funnel for m_reclaim because it calls into the
389 * socket domains and tsleep end-up calling splhigh
391 fnl
= thread_funnel_get();
392 if (wait
== 0 && fnl
== network_flock
) {
394 } else if (fnl
!= THR_FUNNEL_NULL
) {
395 /* Sleep with a small timeout as insurance */
396 (void) tsleep((caddr_t
)&mfree
, PZERO
-1, "m_retry", hz
);
398 /* We are called from a non-BSD context: use mach primitives */
399 u_int64_t abstime
= 0;
401 assert_wait((event_t
)&mfree
, THREAD_UNINT
);
402 clock_interval_to_deadline(hz
, NSEC_PER_SEC
/ hz
, &abstime
);
403 thread_set_timer_deadline(abstime
);
404 if (thread_block(THREAD_CONTINUE_NULL
) != THREAD_TIMED_OUT
)
405 thread_cancel_timer();
414 * As above; retry an MGETHDR.
417 m_retryhdr(canwait
, type
)
420 register struct mbuf
*m
;
422 if (m
= m_retry(canwait
, type
)) {
423 m
->m_flags
|= M_PKTHDR
;
424 m
->m_data
= m
->m_pktdat
;
432 register struct domain
*dp
;
433 register struct protosw
*pr
;
435 for (dp
= domains
; dp
; dp
= dp
->dom_next
)
436 for (pr
= dp
->dom_protosw
; pr
; pr
= pr
->pr_next
)
443 * Space allocation routines.
444 * These are also available as macros
445 * for critical paths.
451 register struct mbuf
*m
;
455 m
->m_next
= m
->m_nextpkt
= 0;
457 m
->m_data
= m
->m_dat
;
460 (m
) = m_retry(nowait
, type
);
466 m_gethdr(nowait
, type
)
469 register struct mbuf
*m
;
473 m
->m_next
= m
->m_nextpkt
= 0;
475 m
->m_data
= m
->m_pktdat
;
476 m
->m_flags
= M_PKTHDR
;
479 m
= m_retryhdr(nowait
, type
);
485 m_getclr(nowait
, type
)
488 register struct mbuf
*m
;
490 MGET(m
, nowait
, type
);
493 bzero(mtod(m
, caddr_t
), MLEN
);
501 struct mbuf
*n
= m
->m_next
;
504 if (m
->m_type
== MT_FREE
)
505 panic("freeing free mbuf");
507 /* Free the aux data if there is any */
508 if ((m
->m_flags
& M_PKTHDR
) && m
->m_pkthdr
.aux
)
510 m_freem(m
->m_pkthdr
.aux
);
514 if ((m
->m_flags
& M_EXT
))
516 if (MCLHASREFERENCE(m
)) {
517 remque((queue_t
)&m
->m_ext
.ext_refs
);
518 } else if (m
->m_ext
.ext_free
== NULL
) {
519 union mcluster
*mcl
= (union mcluster
*)m
->m_ext
.ext_buf
;
520 if (_MCLUNREF(mcl
)) {
521 mcl
->mcl_next
= mclfree
;
526 /* *** Since m_split() increments "mclrefcnt[mtocl(m->m_ext.ext_buf)]",
527 and AppleTalk ADSP uses m_split(), this incorrect sanity check
530 else /* sanity check - not referenced this way */
531 panic("m_free m_ext cluster not free");
534 (*(m
->m_ext
.ext_free
))(m
->m_ext
.ext_buf
,
535 m
->m_ext
.ext_size
, m
->m_ext
.ext_arg
);
538 mbstat
.m_mtypes
[m
->m_type
]--;
542 mbstat
.m_mtypes
[m
->m_type
]++;
550 if (i
) wakeup((caddr_t
)&mfree
);
554 /* m_mclget() add an mbuf cluster to a normal mbuf */
560 MCLALLOC(m
->m_ext
.ext_buf
, nowait
);
561 if (m
->m_ext
.ext_buf
) {
562 m
->m_data
= m
->m_ext
.ext_buf
;
564 m
->m_ext
.ext_size
= MCLBYTES
;
565 m
->m_ext
.ext_free
= 0;
566 m
->m_ext
.ext_refs
.forward
= m
->m_ext
.ext_refs
.backward
=
573 /* m_mclalloc() allocate an mbuf cluster */
580 (void)m_clalloc(1, nowait
);
581 if ((p
= (caddr_t
)mclfree
)) {
582 ++mclrefcnt
[mtocl(p
)];
584 mclfree
= ((union mcluster
*)p
)->mcl_next
;
593 /* m_mclfree() releases a reference to a cluster allocated by MCLALLOC,
594 * freeing the cluster if the reference count has reached 0. */
600 if (--mclrefcnt
[mtocl(p
)] == 0) {
601 ((union mcluster
*)(p
))->mcl_next
= mclfree
;
602 mclfree
= (union mcluster
*)(p
);
608 /* mcl_hasreference() checks if a cluster of an mbuf is referenced by another mbuf */
613 return (m
->m_ext
.ext_refs
.forward
!= &(m
->m_ext
.ext_refs
));
618 m_copy_pkthdr(to
, from
)
619 struct mbuf
*to
, *from
;
621 to
->m_pkthdr
= from
->m_pkthdr
;
622 from
->m_pkthdr
.aux
= (struct mbuf
*)NULL
;
623 to
->m_flags
= from
->m_flags
& M_COPYFLAGS
;
624 to
->m_data
= (to
)->m_pktdat
;
627 /* Best effort to get a mbuf cluster + pkthdr under one lock.
628 * If we don't have them avail, just bail out and use the regular
630 * Used by drivers to allocated packets on receive ring.
636 m_clalloc(1, M_DONTWAIT
); /* takes the MBUF_LOCK, but doesn't release it... */
637 if ((mfree
!= 0) && (mclfree
!= 0)) { /* mbuf + cluster are available */
641 ++mclrefcnt
[mtocl(m
)];
642 mbstat
.m_mtypes
[MT_FREE
]--;
643 mbstat
.m_mtypes
[MT_DATA
]++;
644 m
->m_ext
.ext_buf
= (caddr_t
)mclfree
; /* get the cluster */
645 ++mclrefcnt
[mtocl(m
->m_ext
.ext_buf
)];
647 mclfree
= ((union mcluster
*)(m
->m_ext
.ext_buf
))->mcl_next
;
649 m
->m_next
= m
->m_nextpkt
= 0;
651 m
->m_data
= m
->m_ext
.ext_buf
;
652 m
->m_flags
= M_PKTHDR
| M_EXT
;
654 m
->m_ext
.ext_free
= 0;
655 m
->m_ext
.ext_size
= MCLBYTES
;
656 m
->m_ext
.ext_refs
.forward
= m
->m_ext
.ext_refs
.backward
=
660 else { /* slow path: either mbuf or cluster need to be allocated anyway */
663 MGETHDR(m
, M_WAITOK
, MT_DATA
);
668 MCLGET( m
, M_WAITOK
);
669 if ( ( m
->m_flags
& M_EXT
) == 0 )
679 * return a list of mbuf hdrs that point to clusters...
680 * try for num_needed, if this can't be met, return whatever
681 * number were available... set up the first num_with_pkthdrs
682 * with mbuf hdrs configured as packet headers... these are
683 * chained on the m_nextpkt field... any packets requested beyond
684 * this are chained onto the last packet header's m_next field.
687 m_getpackets(int num_needed
, int num_with_pkthdrs
, int how
)
690 struct mbuf
**np
, *top
;
695 m_clalloc(num_needed
, how
); /* takes the MBUF_LOCK, but doesn't release it... */
697 while (num_needed
--) {
698 if (mfree
&& mclfree
) { /* mbuf + cluster are available */
702 ++mclrefcnt
[mtocl(m
)];
703 mbstat
.m_mtypes
[MT_FREE
]--;
704 mbstat
.m_mtypes
[MT_DATA
]++;
705 m
->m_ext
.ext_buf
= (caddr_t
)mclfree
; /* get the cluster */
706 ++mclrefcnt
[mtocl(m
->m_ext
.ext_buf
)];
708 mclfree
= ((union mcluster
*)(m
->m_ext
.ext_buf
))->mcl_next
;
710 m
->m_next
= m
->m_nextpkt
= 0;
712 m
->m_data
= m
->m_ext
.ext_buf
;
713 m
->m_ext
.ext_free
= 0;
714 m
->m_ext
.ext_size
= MCLBYTES
;
715 m
->m_ext
.ext_refs
.forward
= m
->m_ext
.ext_refs
.backward
= &m
->m_ext
.ext_refs
;
717 if (num_with_pkthdrs
== 0)
720 m
->m_flags
= M_PKTHDR
| M_EXT
;
730 if (num_with_pkthdrs
== 0) {
731 MGET(m
, how
, MT_DATA
);
733 MGETHDR(m
, how
, MT_DATA
);
741 if ((m
->m_flags
& M_EXT
) == 0) {
749 if (num_with_pkthdrs
)
761 * return a list of mbuf hdrs set up as packet hdrs
762 * chained together on the m_nextpkt field
765 m_getpackethdrs(int num_needed
, int how
)
768 struct mbuf
**np
, *top
;
775 while (num_needed
--) {
776 if (m
= mfree
) { /* mbufs are available */
779 ++mclrefcnt
[mtocl(m
)];
780 mbstat
.m_mtypes
[MT_FREE
]--;
781 mbstat
.m_mtypes
[MT_DATA
]++;
783 m
->m_next
= m
->m_nextpkt
= 0;
785 m
->m_flags
= M_PKTHDR
;
786 m
->m_data
= m
->m_pktdat
;
793 m
= m_retryhdr(how
, MT_DATA
);
809 /* free and mbuf list (m_nextpkt) while following m_next under one lock.
810 * returns the count for mbufs packets freed. Used by the drivers.
816 struct mbuf
*nextpkt
;
823 nextpkt
= m
->m_nextpkt
; /* chain of linked mbufs from driver */
829 while (m
) { /* free the mbuf chain (like mfreem) */
833 /* Free the aux data if there is any */
834 if ((m
->m_flags
& M_PKTHDR
) && m
->m_pkthdr
.aux
) {
836 * Treat the current m as the nextpkt and set m
837 * to the aux data. Preserve nextpkt in m->m_nextpkt.
838 * This lets us free the aux data in this loop
839 * without having to call m_freem recursively,
840 * which wouldn't work because we've still got
843 m
->m_nextpkt
= nextpkt
;
845 m
= nextpkt
->m_pkthdr
.aux
;
846 nextpkt
->m_pkthdr
.aux
= NULL
;
851 if (n
&& n
->m_nextpkt
)
852 panic("m_freem_list: m_nextpkt of m_next != NULL");
853 if (m
->m_type
== MT_FREE
)
854 panic("freeing free mbuf");
856 if (m
->m_flags
& M_EXT
) {
857 if (MCLHASREFERENCE(m
)) {
858 remque((queue_t
)&m
->m_ext
.ext_refs
);
859 } else if (m
->m_ext
.ext_free
== NULL
) {
860 union mcluster
*mcl
= (union mcluster
*)m
->m_ext
.ext_buf
;
861 if (_MCLUNREF(mcl
)) {
862 mcl
->mcl_next
= mclfree
;
867 (*(m
->m_ext
.ext_free
))(m
->m_ext
.ext_buf
,
868 m
->m_ext
.ext_size
, m
->m_ext
.ext_arg
);
871 mbstat
.m_mtypes
[m
->m_type
]--;
874 mbstat
.m_mtypes
[MT_FREE
]++;
882 m
= nextpkt
; /* bump m with saved nextpkt if any */
890 wakeup((caddr_t
)&mfree
);
897 register struct mbuf
*m
;
904 * Mbuffer utility routines.
907 * Compute the amount of space available
908 * before the current start of data in an mbuf.
911 register struct mbuf
*m
;
913 if (m
->m_flags
& M_EXT
) {
914 if (MCLHASREFERENCE(m
))
916 return (m
->m_data
- m
->m_ext
.ext_buf
);
918 if (m
->m_flags
& M_PKTHDR
)
919 return (m
->m_data
- m
->m_pktdat
);
920 return (m
->m_data
- m
->m_dat
);
924 * Compute the amount of space available
925 * after the end of data in an mbuf.
928 register struct mbuf
*m
;
930 if (m
->m_flags
& M_EXT
) {
931 if (MCLHASREFERENCE(m
))
933 return (m
->m_ext
.ext_buf
+ m
->m_ext
.ext_size
-
934 (m
->m_data
+ m
->m_len
));
936 return (&m
->m_dat
[MLEN
] - (m
->m_data
+ m
->m_len
));
940 * Lesser-used path for M_PREPEND:
941 * allocate new mbuf to prepend to chain,
943 * Does not adjust packet header length.
946 m_prepend(m
, len
, how
)
947 register struct mbuf
*m
;
952 MGET(mn
, how
, m
->m_type
);
953 if (mn
== (struct mbuf
*)NULL
) {
955 return ((struct mbuf
*)NULL
);
957 if (m
->m_flags
& M_PKTHDR
) {
958 M_COPY_PKTHDR(mn
, m
);
959 m
->m_flags
&= ~M_PKTHDR
;
970 * Replacement for old M_PREPEND macro:
971 * allocate new mbuf to prepend to chain,
972 * copy junk along, and adjust length.
976 m_prepend_2(m
, len
, how
)
977 register struct mbuf
*m
;
980 if (M_LEADINGSPACE(m
) >= len
) {
984 m
= m_prepend(m
, len
, how
);
986 if ((m
) && (m
->m_flags
& M_PKTHDR
))
987 m
->m_pkthdr
.len
+= len
;
992 * Make a copy of an mbuf chain starting "off0" bytes from the beginning,
993 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf.
994 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller.
999 m_copym(m
, off0
, len
, wait
)
1000 register struct mbuf
*m
;
1004 register struct mbuf
*n
, **np
;
1005 register int off
= off0
;
1009 if (off
< 0 || len
< 0)
1011 if (off
== 0 && m
->m_flags
& M_PKTHDR
)
1014 while (off
>= m
->m_len
) {
1027 if (len
!= M_COPYALL
)
1033 ++mclrefcnt
[mtocl(n
)];
1034 mbstat
.m_mtypes
[MT_FREE
]--;
1035 mbstat
.m_mtypes
[m
->m_type
]++;
1037 n
->m_next
= n
->m_nextpkt
= 0;
1038 n
->m_type
= m
->m_type
;
1039 n
->m_data
= n
->m_dat
;
1043 n
= m_retry(wait
, m
->m_type
);
1051 M_COPY_PKTHDR(n
, m
);
1052 if (len
== M_COPYALL
)
1053 n
->m_pkthdr
.len
-= off0
;
1055 n
->m_pkthdr
.len
= len
;
1058 if (len
== M_COPYALL
) {
1059 if (min(len
, (m
->m_len
- off
)) == len
) {
1060 printf("m->m_len %d - off %d = %d, %d\n",
1061 m
->m_len
, off
, m
->m_len
- off
,
1062 min(len
, (m
->m_len
- off
)));
1065 n
->m_len
= min(len
, (m
->m_len
- off
));
1066 if (n
->m_len
== M_COPYALL
) {
1067 printf("n->m_len == M_COPYALL, fixing\n");
1070 if (m
->m_flags
& M_EXT
) {
1071 n
->m_ext
= m
->m_ext
;
1072 insque((queue_t
)&n
->m_ext
.ext_refs
, (queue_t
)&m
->m_ext
.ext_refs
);
1073 n
->m_data
= m
->m_data
+ off
;
1074 n
->m_flags
|= M_EXT
;
1076 bcopy(mtod(m
, caddr_t
)+off
, mtod(n
, caddr_t
),
1077 (unsigned)n
->m_len
);
1079 if (len
!= M_COPYALL
)
1101 * equivilent to m_copym except that all necessary
1102 * mbuf hdrs are allocated within this routine
1103 * also, the last mbuf and offset accessed are passed
1104 * out and can be passed back in to avoid having to
1105 * rescan the entire mbuf list (normally hung off of the socket)
1108 m_copym_with_hdrs(m
, off0
, len
, wait
, m_last
, m_off
)
1109 register struct mbuf
*m
;
1112 struct mbuf
**m_last
;
1115 register struct mbuf
*n
, **np
;
1116 register int off
= off0
;
1117 struct mbuf
*top
= 0;
1121 if (off
== 0 && m
->m_flags
& M_PKTHDR
)
1128 while (off
>= m
->m_len
) {
1140 panic("m_gethdr_and_copym");
1145 ++mclrefcnt
[mtocl(n
)];
1146 mbstat
.m_mtypes
[MT_FREE
]--;
1147 mbstat
.m_mtypes
[type
]++;
1149 n
->m_next
= n
->m_nextpkt
= 0;
1153 n
->m_data
= n
->m_dat
;
1156 n
->m_data
= n
->m_pktdat
;
1157 n
->m_flags
= M_PKTHDR
;
1163 n
= m_retry(wait
, type
);
1165 n
= m_retryhdr(wait
, type
);
1178 M_COPY_PKTHDR(n
, m
);
1179 n
->m_pkthdr
.len
= len
;
1182 n
->m_len
= min(len
, (m
->m_len
- off
));
1184 if (m
->m_flags
& M_EXT
) {
1185 n
->m_ext
= m
->m_ext
;
1186 insque((queue_t
)&n
->m_ext
.ext_refs
, (queue_t
)&m
->m_ext
.ext_refs
);
1187 n
->m_data
= m
->m_data
+ off
;
1188 n
->m_flags
|= M_EXT
;
1190 bcopy(mtod(m
, caddr_t
)+off
, mtod(n
, caddr_t
),
1191 (unsigned)n
->m_len
);
1196 if ((off
+ n
->m_len
) == m
->m_len
) {
1197 *m_last
= m
->m_next
;
1201 *m_off
= off
+ n
->m_len
;
1223 * Copy data from an mbuf chain starting "off" bytes from the beginning,
1224 * continuing for "len" bytes, into the indicated buffer.
1226 void m_copydata(m
, off
, len
, cp
)
1227 register struct mbuf
*m
;
1232 register unsigned count
;
1234 if (off
< 0 || len
< 0)
1235 panic("m_copydata");
1238 panic("m_copydata");
1246 panic("m_copydata");
1247 count
= min(m
->m_len
- off
, len
);
1248 bcopy(mtod(m
, caddr_t
) + off
, cp
, count
);
1257 * Concatenate mbuf chain n to m.
1258 * Both chains must be of the same type (e.g. MT_DATA).
1259 * Any m_pkthdr is not updated.
1262 register struct mbuf
*m
, *n
;
1267 if (m
->m_flags
& M_EXT
||
1268 m
->m_data
+ m
->m_len
+ n
->m_len
>= &m
->m_dat
[MLEN
]) {
1269 /* just join the two chains */
1273 /* splat the data from one into the other */
1274 bcopy(mtod(n
, caddr_t
), mtod(m
, caddr_t
) + m
->m_len
,
1276 m
->m_len
+= n
->m_len
;
1286 register int len
= req_len
;
1287 register struct mbuf
*m
;
1290 if ((m
= mp
) == NULL
)
1296 while (m
!= NULL
&& len
> 0) {
1297 if (m
->m_len
<= len
) {
1308 if (m
->m_flags
& M_PKTHDR
)
1309 m
->m_pkthdr
.len
-= (req_len
- len
);
1312 * Trim from tail. Scan the mbuf chain,
1313 * calculating its length and finding the last mbuf.
1314 * If the adjustment only affects this mbuf, then just
1315 * adjust and return. Otherwise, rescan and truncate
1316 * after the remaining size.
1322 if (m
->m_next
== (struct mbuf
*)0)
1326 if (m
->m_len
>= len
) {
1329 if (m
->m_flags
& M_PKTHDR
)
1330 m
->m_pkthdr
.len
-= len
;
1337 * Correct length for chain is "count".
1338 * Find the mbuf with last data, adjust its length,
1339 * and toss data from remaining mbufs on chain.
1342 if (m
->m_flags
& M_PKTHDR
)
1343 m
->m_pkthdr
.len
= count
;
1344 for (; m
; m
= m
->m_next
) {
1345 if (m
->m_len
>= count
) {
1351 while (m
= m
->m_next
)
1357 * Rearange an mbuf chain so that len bytes are contiguous
1358 * and in the data area of an mbuf (so that mtod and dtom
1359 * will work for a structure of size len). Returns the resulting
1360 * mbuf chain on success, frees it and returns null on failure.
1361 * If there is room, it will add up to max_protohdr-len extra bytes to the
1362 * contiguous region in an attempt to avoid being called next time.
1368 register struct mbuf
*n
;
1371 register struct mbuf
*m
;
1376 * If first mbuf has no cluster, and has room for len bytes
1377 * without shifting current data, pullup into it,
1378 * otherwise allocate a new mbuf to prepend to the chain.
1380 if ((n
->m_flags
& M_EXT
) == 0 &&
1381 n
->m_data
+ len
< &n
->m_dat
[MLEN
] && n
->m_next
) {
1382 if (n
->m_len
>= len
)
1390 MGET(m
, M_DONTWAIT
, n
->m_type
);
1394 if (n
->m_flags
& M_PKTHDR
) {
1395 M_COPY_PKTHDR(m
, n
);
1396 n
->m_flags
&= ~M_PKTHDR
;
1399 space
= &m
->m_dat
[MLEN
] - (m
->m_data
+ m
->m_len
);
1401 count
= min(min(max(len
, max_protohdr
), space
), n
->m_len
);
1402 bcopy(mtod(n
, caddr_t
), mtod(m
, caddr_t
) + m
->m_len
,
1412 } while (len
> 0 && n
);
1426 * Partition an mbuf chain in two pieces, returning the tail --
1427 * all but the first len0 bytes. In case of failure, it returns NULL and
1428 * attempts to restore the chain to its original state.
1431 m_split(m0
, len0
, wait
)
1432 register struct mbuf
*m0
;
1435 register struct mbuf
*m
, *n
;
1436 unsigned len
= len0
, remain
;
1438 for (m
= m0
; m
&& len
> m
->m_len
; m
= m
->m_next
)
1442 remain
= m
->m_len
- len
;
1443 if (m0
->m_flags
& M_PKTHDR
) {
1444 MGETHDR(n
, wait
, m0
->m_type
);
1447 n
->m_pkthdr
.rcvif
= m0
->m_pkthdr
.rcvif
;
1448 n
->m_pkthdr
.len
= m0
->m_pkthdr
.len
- len0
;
1449 m0
->m_pkthdr
.len
= len0
;
1450 if (m
->m_flags
& M_EXT
)
1452 if (remain
> MHLEN
) {
1453 /* m can't be the lead packet */
1455 n
->m_next
= m_split(m
, len
, wait
);
1456 if (n
->m_next
== 0) {
1462 MH_ALIGN(n
, remain
);
1463 } else if (remain
== 0) {
1468 MGET(n
, wait
, m
->m_type
);
1474 if (m
->m_flags
& M_EXT
) {
1475 n
->m_flags
|= M_EXT
;
1477 n
->m_ext
= m
->m_ext
;
1478 insque((queue_t
)&n
->m_ext
.ext_refs
, (queue_t
)&m
->m_ext
.ext_refs
);
1480 n
->m_data
= m
->m_data
+ len
;
1482 bcopy(mtod(m
, caddr_t
) + len
, mtod(n
, caddr_t
), remain
);
1486 n
->m_next
= m
->m_next
;
1491 * Routine to copy from device local memory into mbufs.
1494 m_devget(buf
, totlen
, off0
, ifp
, copy
)
1500 register struct mbuf
*m
;
1501 struct mbuf
*top
= 0, **mp
= &top
;
1502 register int off
= off0
, len
;
1510 * If 'off' is non-zero, packet is trailer-encapsulated,
1511 * so we have to skip the type and length fields.
1513 cp
+= off
+ 2 * sizeof(u_int16_t
);
1514 totlen
-= 2 * sizeof(u_int16_t
);
1516 MGETHDR(m
, M_DONTWAIT
, MT_DATA
);
1519 m
->m_pkthdr
.rcvif
= ifp
;
1520 m
->m_pkthdr
.len
= totlen
;
1523 while (totlen
> 0) {
1525 MGET(m
, M_DONTWAIT
, MT_DATA
);
1532 len
= min(totlen
, epkt
- cp
);
1533 if (len
>= MINCLSIZE
) {
1534 MCLGET(m
, M_DONTWAIT
);
1535 if (m
->m_flags
& M_EXT
)
1536 m
->m_len
= len
= min(len
, MCLBYTES
);
1538 /* give up when it's out of cluster mbufs */
1546 * Place initial small packet/header at end of mbuf.
1548 if (len
< m
->m_len
) {
1549 if (top
== 0 && len
+ max_linkhdr
<= m
->m_len
)
1550 m
->m_data
+= max_linkhdr
;
1556 copy(cp
, mtod(m
, caddr_t
), (unsigned)len
);
1558 bcopy(cp
, mtod(m
, caddr_t
), (unsigned)len
);
1570 * Cluster freelist allocation check. The mbuf lock must be held.
1571 * Ensure hysteresis between hi/lo.
1579 if (mbstat
.m_clusters
< MINCL
)
1580 return (MINCL
- mbstat
.m_clusters
);
1581 /* Too few (free < 1/2 total) and not over maximum */
1582 if (mbstat
.m_clusters
< nmbclusters
&&
1583 (i
= ((mbstat
.m_clusters
>> 1) - mbstat
.m_clfree
)) > 0)
1590 * Copy data from a buffer back into the indicated mbuf chain,
1591 * starting "off" bytes from the beginning, extending the mbuf
1592 * chain if necessary.
1595 m_copyback(m0
, off
, len
, cp
)
1602 register struct mbuf
*m
= m0
, *n
;
1607 while (off
> (mlen
= m
->m_len
)) {
1610 if (m
->m_next
== 0) {
1611 n
= m_getclr(M_DONTWAIT
, m
->m_type
);
1614 n
->m_len
= min(MLEN
, len
+ off
);
1620 mlen
= min (m
->m_len
- off
, len
);
1621 bcopy(cp
, off
+ mtod(m
, caddr_t
), (unsigned)mlen
);
1629 if (m
->m_next
== 0) {
1630 n
= m_get(M_DONTWAIT
, m
->m_type
);
1633 n
->m_len
= min(MLEN
, len
);
1638 out
: if (((m
= m0
)->m_flags
& M_PKTHDR
) && (m
->m_pkthdr
.len
< totlen
))
1639 m
->m_pkthdr
.len
= totlen
;
1643 char *mcl_to_paddr(register char *addr
) {
1644 register int base_phys
;
1646 if (addr
< (char *)mbutl
|| addr
>= (char *)embutl
)
1648 base_phys
= mcl_paddr
[(addr
- (char *)mbutl
) >> PAGE_SHIFT
];
1652 return ((char *)((int)base_phys
| ((int)addr
& PAGE_MASK
)));
1656 * Dup the mbuf chain passed in. The whole thing. No cute additional cruft.
1657 * And really copy the thing. That way, we don't "precompute" checksums
1658 * for unsuspecting consumers.
1659 * Assumption: m->m_nextpkt == 0.
1660 * Trick: for small packets, don't dup into a cluster. That way received
1661 * packets don't take up too much room in the sockbuf (cf. sbspace()).
1666 m_dup(register struct mbuf
*m
, int how
)
1667 { register struct mbuf
*n
, **np
;
1673 if (m
->m_flags
& M_PKTHDR
)
1677 * Quick check: if we have one mbuf and its data fits in an
1678 * mbuf with packet header, just copy and go.
1680 if (m
->m_next
== NULL
)
1681 { /* Then just move the data into an mbuf and be done... */
1683 { if (m
->m_pkthdr
.len
<= MHLEN
)
1684 { if ((n
= m_gethdr(how
, m
->m_type
)) == NULL
)
1686 n
->m_len
= m
->m_len
;
1687 n
->m_flags
|= (m
->m_flags
& M_COPYFLAGS
);
1688 n
->m_pkthdr
.len
= m
->m_pkthdr
.len
;
1689 n
->m_pkthdr
.rcvif
= m
->m_pkthdr
.rcvif
;
1690 n
->m_pkthdr
.header
= NULL
;
1691 n
->m_pkthdr
.csum_flags
= 0;
1692 n
->m_pkthdr
.csum_data
= 0;
1693 n
->m_pkthdr
.aux
= NULL
;
1694 n
->m_pkthdr
.vlan_tag
= 0;
1695 n
->m_pkthdr
.reserved_1
= 0;
1696 n
->m_pkthdr
.reserved2
= 0;
1697 bcopy(m
->m_data
, n
->m_data
, m
->m_pkthdr
.len
);
1700 } else if (m
->m_len
<= MLEN
)
1701 { if ((n
= m_get(how
, m
->m_type
)) == NULL
)
1703 bcopy(m
->m_data
, n
->m_data
, m
->m_len
);
1704 n
->m_len
= m
->m_len
;
1711 kprintf("<%x: %x, %x, %x\n", m
, m
->m_flags
, m
->m_len
,
1715 n
= m_gethdr(how
, m
->m_type
);
1717 n
= m_get(how
, m
->m_type
);
1720 if (m
->m_flags
& M_EXT
)
1722 if ((n
->m_flags
& M_EXT
) == 0)
1727 { /* Don't use M_COPY_PKTHDR: preserve m_data */
1728 n
->m_pkthdr
= m
->m_pkthdr
;
1729 n
->m_flags
|= (m
->m_flags
& M_COPYFLAGS
);
1731 if ((n
->m_flags
& M_EXT
) == 0)
1732 n
->m_data
= n
->m_pktdat
;
1734 n
->m_len
= m
->m_len
;
1736 * Get the dup on the same bdry as the original
1737 * Assume that the two mbufs have the same offset to data area
1738 * (up to word bdries)
1740 bcopy(mtod(m
, caddr_t
), mtod(n
, caddr_t
), (unsigned)n
->m_len
);
1744 kprintf(">%x: %x, %x, %x\n", n
, n
->m_flags
, n
->m_len
,
1759 m_mclref(struct mbuf
*p
)
1761 return (_MCLREF(p
));
1765 m_mclunref(struct mbuf
*p
)
1767 return (_MCLUNREF(p
));
1770 /* change mbuf to new type */
1772 m_mchtype(struct mbuf
*m
, int t
)
1775 mbstat
.m_mtypes
[(m
)->m_type
]--;
1776 mbstat
.m_mtypes
[t
]++;
1781 void *m_mtod(struct mbuf
*m
)
1783 return ((m
)->m_data
);
1786 struct mbuf
*m_dtom(void *x
)
1788 return ((struct mbuf
*)((u_long
)(x
) & ~(MSIZE
-1)));
1791 int m_mtocl(void *x
)
1793 return (((char *)(x
) - (char *)mbutl
) / sizeof(union mcluster
));
1796 union mcluster
*m_cltom(int x
)
1798 return ((union mcluster
*)(mbutl
+ (x
)));
1802 void m_mcheck(struct mbuf
*m
)
1804 if (m
->m_type
!= MT_FREE
)
1805 panic("mget MCHECK: m_type=%x m=%x", m
->m_type
, m
);
1809 mbuf_expand_thread(void)
1814 expand_mcl
= mbuf_expand_mcl
;
1815 mbuf_expand_mcl
= 0;
1819 MCLALLOC(p
, M_WAIT
);
1822 assert_wait(&mbuf_expand_thread_wakeup
, THREAD_UNINT
);
1823 (void) thread_block(mbuf_expand_thread
);
1828 mbuf_expand_thread_init(void)
1830 mbuf_expand_thread_initialized
++;
1831 mbuf_expand_thread();