X-Git-Url: https://git.saurik.com/apple/xnu.git/blobdiff_plain/21362eb3e66fd2c787aee132bce100a44d71a99c..0b4c1975fb5e4eccf1012a35081f7e7799b81046:/bsd/kern/kpi_mbuf.c diff --git a/bsd/kern/kpi_mbuf.c b/bsd/kern/kpi_mbuf.c index 61ac09968..568835be1 100644 --- a/bsd/kern/kpi_mbuf.c +++ b/bsd/kern/kpi_mbuf.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2004 Apple Computer, Inc. All rights reserved. + * Copyright (c) 2004-2009 Apple Inc. All rights reserved. * * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ * @@ -36,9 +36,9 @@ #include #include #include +#include -void mbuf_tag_id_first_last(u_long *first, u_long *last); -errno_t mbuf_tag_id_find_internal(const char *string, u_long *out_id, int create); +#include "net/net_str_id.h" static const mbuf_flags_t mbuf_flags_mask = MBUF_EXT | MBUF_PKTHDR | MBUF_EOR | MBUF_BCAST | MBUF_MCAST | MBUF_FRAG | MBUF_FIRSTFRAG | @@ -46,7 +46,7 @@ static const mbuf_flags_t mbuf_flags_mask = MBUF_EXT | MBUF_PKTHDR | MBUF_EOR | void* mbuf_data(mbuf_t mbuf) { - return m_mtod(mbuf); + return mbuf->m_data; } void* mbuf_datastart(mbuf_t mbuf) @@ -83,7 +83,7 @@ errno_t mbuf_align_32(mbuf_t mbuf, size_t len) addr64_t mbuf_data_to_physical(void* ptr) { - return (addr64_t)mcl_to_paddr(ptr); + return (addr64_t)(uintptr_t)mcl_to_paddr(ptr); } errno_t mbuf_get(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf) @@ -102,13 +102,73 @@ errno_t mbuf_gethdr(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf) return (*mbuf == NULL) ? ENOMEM : 0; } -extern struct mbuf * m_mbigget(struct mbuf *m, int nowait); +errno_t +mbuf_attachcluster(mbuf_how_t how, mbuf_type_t type, mbuf_t *mbuf, + caddr_t extbuf, void (*extfree)(caddr_t , u_int, caddr_t), + size_t extsize, caddr_t extarg) +{ + if (extbuf == NULL || extfree == NULL || extsize == 0) + return (EINVAL); + + if ((*mbuf = m_clattach(mbuf != NULL ? *mbuf : NULL, type, extbuf, + extfree, extsize, extarg, how)) == NULL) + return (ENOMEM); -errno_t mbuf_getcluster(mbuf_how_t how, mbuf_type_t type, size_t size, mbuf_t* mbuf) + return (0); +} + +errno_t +mbuf_alloccluster(mbuf_how_t how, size_t *size, caddr_t *addr) +{ + if (size == NULL || *size == 0 || addr == NULL) + return (EINVAL); + + *addr = NULL; + + /* Jumbo cluster pool not available? */ + if (*size > NBPG && njcl == 0) + return (ENOTSUP); + + if (*size <= MCLBYTES && (*addr = m_mclalloc(how)) != NULL) + *size = MCLBYTES; + else if (*size > MCLBYTES && *size <= NBPG && + (*addr = m_bigalloc(how)) != NULL) + *size = NBPG; + else if (*size > NBPG && *size <= M16KCLBYTES && + (*addr = m_16kalloc(how)) != NULL) + *size = M16KCLBYTES; + else + *size = 0; + + if (*addr == NULL) + return (ENOMEM); + + return (0); +} + +void +mbuf_freecluster(caddr_t addr, size_t size) +{ + if (size != MCLBYTES && size != NBPG && size != M16KCLBYTES) + panic("%s: invalid size (%ld) for cluster %p", __func__, + size, (void *)addr); + + if (size == MCLBYTES) + m_mclfree(addr); + else if (size == NBPG) + m_bigfree(addr, NBPG, NULL); + else if (njcl > 0) + m_16kfree(addr, M16KCLBYTES, NULL); + else + panic("%s: freeing jumbo cluster to an empty pool", __func__); +} + +errno_t +mbuf_getcluster(mbuf_how_t how, mbuf_type_t type, size_t size, mbuf_t* mbuf) { /* Must set *mbuf to NULL in failure case */ errno_t error = 0; - int created = 0; + int created = 0; if (mbuf == NULL) return EINVAL; @@ -119,13 +179,21 @@ errno_t mbuf_getcluster(mbuf_how_t how, mbuf_type_t type, size_t size, mbuf_t* m created = 1; } /* - * At the time this code was written, m_mclget and m_mbigget would always - * return the same value that was passed in to it. + * At the time this code was written, m_{mclget,mbigget,m16kget} + * would always return the same value that was passed in to it. */ if (size == MCLBYTES) { *mbuf = m_mclget(*mbuf, how); } else if (size == NBPG) { *mbuf = m_mbigget(*mbuf, how); + } else if (size == M16KCLBYTES) { + if (njcl > 0) { + *mbuf = m_m16kget(*mbuf, how); + } else { + /* Jumbo cluster pool not available? */ + error = ENOTSUP; + goto out; + } } else { error = EINVAL; goto out; @@ -134,7 +202,6 @@ errno_t mbuf_getcluster(mbuf_how_t how, mbuf_type_t type, size_t size, mbuf_t* m error = ENOMEM; out: if (created && error != 0) { - error = ENOMEM; mbuf_free(*mbuf); *mbuf = NULL; } @@ -202,18 +269,18 @@ int mbuf_freem_list(mbuf_t mbuf) return m_freem_list(mbuf); } -size_t mbuf_leadingspace(mbuf_t mbuf) +size_t mbuf_leadingspace(const mbuf_t mbuf) { return m_leadingspace(mbuf); } -size_t mbuf_trailingspace(mbuf_t mbuf) +size_t mbuf_trailingspace(const mbuf_t mbuf) { return m_trailingspace(mbuf); } /* Manipulation */ -errno_t mbuf_copym(mbuf_t src, size_t offset, size_t len, +errno_t mbuf_copym(const mbuf_t src, size_t offset, size_t len, mbuf_how_t how, mbuf_t *new_mbuf) { /* Must set *mbuf to NULL in failure case */ @@ -222,7 +289,7 @@ errno_t mbuf_copym(mbuf_t src, size_t offset, size_t len, return (*new_mbuf == NULL) ? ENOMEM : 0; } -errno_t mbuf_dup(mbuf_t src, mbuf_how_t how, mbuf_t *new_mbuf) +errno_t mbuf_dup(const mbuf_t src, mbuf_how_t how, mbuf_t *new_mbuf) { /* Must set *new_mbuf to NULL in failure case */ *new_mbuf = m_dup(src, how); @@ -270,10 +337,40 @@ void mbuf_adj(mbuf_t mbuf, int len) m_adj(mbuf, len); } -errno_t mbuf_copydata(mbuf_t m, size_t off, size_t len, void* out_data) +errno_t mbuf_adjustlen(mbuf_t m, int amount) +{ + /* Verify m_len will be valid after adding amount */ + if (amount > 0) { + int used = (size_t)mbuf_data(m) - (size_t)mbuf_datastart(m) + + m->m_len; + + if ((size_t)(amount + used) > mbuf_maxlen(m)) + return EINVAL; + } + else if (-amount > m->m_len) { + return EINVAL; + } + + m->m_len += amount; + return 0; +} + +mbuf_t +mbuf_concatenate(mbuf_t dst, mbuf_t src) +{ + if (dst == NULL) + return (NULL); + + m_cat(dst, src); + + /* return dst as is in the current implementation */ + return (dst); +} +errno_t mbuf_copydata(const mbuf_t m0, size_t off, size_t len, void* out_data) { /* Copied m_copydata, added error handling (don't just panic) */ int count; + mbuf_t m = m0; while (off > 0) { if (m == 0) @@ -297,16 +394,6 @@ errno_t mbuf_copydata(mbuf_t m, size_t off, size_t len, void* out_data) return 0; } -int mbuf_mclref(mbuf_t mbuf) -{ - return m_mclref(mbuf); -} - -int mbuf_mclunref(mbuf_t mbuf) -{ - return m_mclunref(mbuf); -} - int mbuf_mclhasreference(mbuf_t mbuf) { if ((mbuf->m_flags & M_EXT)) @@ -317,7 +404,7 @@ int mbuf_mclhasreference(mbuf_t mbuf) /* mbuf header */ -mbuf_t mbuf_next(mbuf_t mbuf) +mbuf_t mbuf_next(const mbuf_t mbuf) { return mbuf->m_next; } @@ -331,7 +418,7 @@ errno_t mbuf_setnext(mbuf_t mbuf, mbuf_t next) return 0; } -mbuf_t mbuf_nextpkt(mbuf_t mbuf) +mbuf_t mbuf_nextpkt(const mbuf_t mbuf) { return mbuf->m_nextpkt; } @@ -341,7 +428,7 @@ void mbuf_setnextpkt(mbuf_t mbuf, mbuf_t nextpkt) mbuf->m_nextpkt = nextpkt; } -size_t mbuf_len(mbuf_t mbuf) +size_t mbuf_len(const mbuf_t mbuf) { return mbuf->m_len; } @@ -351,14 +438,14 @@ void mbuf_setlen(mbuf_t mbuf, size_t len) mbuf->m_len = len; } -size_t mbuf_maxlen(mbuf_t mbuf) +size_t mbuf_maxlen(const mbuf_t mbuf) { if (mbuf->m_flags & M_EXT) return mbuf->m_ext.ext_size; return &mbuf->m_dat[MLEN] - ((char*)mbuf_datastart(mbuf)); } -mbuf_type_t mbuf_type(mbuf_t mbuf) +mbuf_type_t mbuf_type(const mbuf_t mbuf) { return mbuf->m_type; } @@ -372,7 +459,7 @@ errno_t mbuf_settype(mbuf_t mbuf, mbuf_type_t new_type) return 0; } -mbuf_flags_t mbuf_flags(mbuf_t mbuf) +mbuf_flags_t mbuf_flags(const mbuf_t mbuf) { return mbuf->m_flags & mbuf_flags_mask; } @@ -395,7 +482,7 @@ errno_t mbuf_setflags_mask(mbuf_t mbuf, mbuf_flags_t flags, mbuf_flags_t mask) return 0; } -errno_t mbuf_copy_pkthdr(mbuf_t dest, mbuf_t src) +errno_t mbuf_copy_pkthdr(mbuf_t dest, const mbuf_t src) { if (((src)->m_flags & M_PKTHDR) == 0) return EINVAL; @@ -405,7 +492,7 @@ errno_t mbuf_copy_pkthdr(mbuf_t dest, mbuf_t src) return 0; } -size_t mbuf_pkthdr_len(mbuf_t mbuf) +size_t mbuf_pkthdr_len(const mbuf_t mbuf) { return mbuf->m_pkthdr.len; } @@ -415,7 +502,12 @@ void mbuf_pkthdr_setlen(mbuf_t mbuf, size_t len) mbuf->m_pkthdr.len = len; } -ifnet_t mbuf_pkthdr_rcvif(mbuf_t mbuf) +void mbuf_pkthdr_adjustlen(mbuf_t mbuf, int amount) +{ + mbuf->m_pkthdr.len += amount; +} + +ifnet_t mbuf_pkthdr_rcvif(const mbuf_t mbuf) { // If we reference count ifnets, we should take a reference here before returning return mbuf->m_pkthdr.rcvif; @@ -428,7 +520,7 @@ errno_t mbuf_pkthdr_setrcvif(mbuf_t mbuf, ifnet_t ifnet) return 0; } -void* mbuf_pkthdr_header(mbuf_t mbuf) +void* mbuf_pkthdr_header(const mbuf_t mbuf) { return mbuf->m_pkthdr.header; } @@ -438,23 +530,6 @@ void mbuf_pkthdr_setheader(mbuf_t mbuf, void *header) mbuf->m_pkthdr.header = (void*)header; } -/* mbuf aux data */ -errno_t mbuf_aux_add(mbuf_t mbuf, int family, mbuf_type_t type, mbuf_t *aux_mbuf) -{ - *aux_mbuf = m_aux_add(mbuf, family, type); - return (*aux_mbuf == NULL) ? ENOMEM : 0; -} - -mbuf_t mbuf_aux_find(mbuf_t mbuf, int family, mbuf_type_t type) -{ - return m_aux_find(mbuf, family, type); -} - -void mbuf_aux_delete(mbuf_t mbuf, mbuf_t aux) -{ - m_aux_delete(mbuf, aux); -} - void mbuf_inbound_modified(mbuf_t mbuf) { @@ -466,7 +541,7 @@ extern void in_cksum_offset(struct mbuf* m, size_t ip_offset); extern void in_delayed_cksum_offset(struct mbuf *m, int ip_offset); void -mbuf_outbound_finalize(mbuf_t mbuf, u_long protocol_family, size_t protocol_offset) +mbuf_outbound_finalize(mbuf_t mbuf, u_int32_t protocol_family, size_t protocol_offset) { if ((mbuf->m_pkthdr.csum_flags & (CSUM_DELAY_DATA | CSUM_DELAY_IP | CSUM_TCP_SUM16)) == 0) @@ -505,7 +580,7 @@ mbuf_outbound_finalize(mbuf_t mbuf, u_long protocol_family, size_t protocol_offs * Hardware checksum code looked pretty IPv4 specific. */ if ((mbuf->m_pkthdr.csum_flags & (CSUM_DELAY_DATA | CSUM_DELAY_IP)) != 0) - panic("mbuf_outbound_finalize - CSUM flags set for non-IPv4 packet (%d)!\n", protocol_family); + panic("mbuf_outbound_finalize - CSUM flags set for non-IPv4 packet (%u)!\n", protocol_family); } } @@ -559,6 +634,27 @@ mbuf_set_csum_requested( return 0; } +static const mbuf_tso_request_flags_t mbuf_valid_tso_request_flags = + MBUF_TSO_IPV4 | MBUF_TSO_IPV6; + +errno_t +mbuf_get_tso_requested( + mbuf_t mbuf, + mbuf_tso_request_flags_t *request, + u_int32_t *value) +{ + if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || + request == NULL || value == NULL) + return EINVAL; + + *request = mbuf->m_pkthdr.csum_flags; + *request &= mbuf_valid_tso_request_flags; + if (*request && value != NULL) + *value = mbuf->m_pkthdr.tso_segsz; + + return 0; +} + errno_t mbuf_get_csum_requested( mbuf_t mbuf, @@ -623,117 +719,82 @@ mbuf_clear_csum_performed( return 0; } -/* - * Mbuf tag KPIs - */ +errno_t +mbuf_inet_cksum(mbuf_t mbuf, int protocol, u_int32_t offset, u_int32_t length, + u_int16_t *csum) +{ + if (mbuf == NULL || length == 0 || csum == NULL || + (u_int32_t)mbuf->m_pkthdr.len < (offset + length)) + return (EINVAL); -struct mbuf_tag_id_entry { - SLIST_ENTRY(mbuf_tag_id_entry) next; - mbuf_tag_id_t id; - char string[]; -}; + *csum = inet_cksum(mbuf, protocol, offset, length); + return (0); +} -#define MBUF_TAG_ID_ENTRY_SIZE(__str) \ - ((size_t)&(((struct mbuf_tag_id_entry*)0)->string[0]) + \ - strlen(__str) + 1) +#if INET6 +errno_t +mbuf_inet6_cksum(mbuf_t mbuf, int protocol, u_int32_t offset, u_int32_t length, + u_int16_t *csum) +{ + if (mbuf == NULL || length == 0 || csum == NULL || + (u_int32_t)mbuf->m_pkthdr.len < (offset + length)) + return (EINVAL); -#define MTAG_FIRST_ID 1000 -static u_long mtag_id_next = MTAG_FIRST_ID; -static SLIST_HEAD(,mbuf_tag_id_entry) mtag_id_list = {NULL}; -static lck_mtx_t *mtag_id_lock = NULL; + *csum = inet6_cksum(mbuf, protocol, offset, length); + return (0); +} +#else /* INET6 */ +errno_t +mbuf_inet6_cksum(__unused mbuf_t mbuf, __unused int protocol, + __unused u_int32_t offset, __unused u_int32_t length, + __unused u_int16_t *csum) +{ + panic("mbuf_inet6_cksum() doesn't exist on this platform\n"); + return (0); +} -__private_extern__ void -mbuf_tag_id_first_last( - u_long *first, - u_long *last) +u_int16_t +inet6_cksum(__unused struct mbuf *m, __unused unsigned int nxt, + __unused unsigned int off, __unused unsigned int len) { - *first = MTAG_FIRST_ID; - *last = mtag_id_next - 1; + panic("inet6_cksum() doesn't exist on this platform\n"); + return (0); } -__private_extern__ errno_t -mbuf_tag_id_find_internal( - const char *string, - u_long *out_id, - int create) +void nd6_lookup_ipv6(void); +void +nd6_lookup_ipv6(void) { - struct mbuf_tag_id_entry *entry = NULL; - - - *out_id = 0; - - if (string == NULL || out_id == NULL) { - return EINVAL; - } - - /* Don't bother allocating the lock if we're only doing a lookup */ - if (create == 0 && mtag_id_lock == NULL) - return ENOENT; - - /* Allocate lock if necessary */ - if (mtag_id_lock == NULL) { - lck_grp_attr_t *grp_attrib = NULL; - lck_attr_t *lck_attrb = NULL; - lck_grp_t *lck_group = NULL; - lck_mtx_t *new_lock = NULL; - - grp_attrib = lck_grp_attr_alloc_init(); - lck_grp_attr_setdefault(grp_attrib); - lck_group = lck_grp_alloc_init("mbuf_tag_allocate_id", grp_attrib); - lck_grp_attr_free(grp_attrib); - lck_attrb = lck_attr_alloc_init(); - lck_attr_setdefault(lck_attrb); - lck_attr_setdebug(lck_attrb); - new_lock = lck_mtx_alloc_init(lck_group, lck_attrb); - if (!OSCompareAndSwap((UInt32)0, (UInt32)new_lock, (UInt32*)&mtag_id_lock)) { - /* - * If the atomic swap fails, someone else has already - * done this work. We can free the stuff we allocated. - */ - lck_mtx_free(new_lock, lck_group); - lck_grp_free(lck_group); - } - lck_attr_free(lck_attrb); - } - - /* Look for an existing entry */ - lck_mtx_lock(mtag_id_lock); - SLIST_FOREACH(entry, &mtag_id_list, next) { - if (strcmp(string, entry->string) == 0) { - break; - } - } - - if (entry == NULL) { - if (create == 0) { - lck_mtx_unlock(mtag_id_lock); - return ENOENT; - } - - entry = kalloc(MBUF_TAG_ID_ENTRY_SIZE(string)); - if (entry == NULL) { - lck_mtx_unlock(mtag_id_lock); - return ENOMEM; - } - - strcpy(entry->string, string); - entry->id = mtag_id_next; - mtag_id_next++; - SLIST_INSERT_HEAD(&mtag_id_list, entry, next); - } - lck_mtx_unlock(mtag_id_lock); - - *out_id = entry->id; - - return 0; + panic("nd6_lookup_ipv6() doesn't exist on this platform\n"); +} + +int +in6addr_local(__unused struct in6_addr *a) +{ + panic("in6addr_local() doesn't exist on this platform\n"); + return (0); } +void nd6_storelladdr(void); +void +nd6_storelladdr(void) +{ + panic("nd6_storelladdr() doesn't exist on this platform\n"); +} +#endif /* INET6 */ + +/* + * Mbuf tag KPIs + */ + +#define MTAG_FIRST_ID FIRST_KPI_STR_ID + errno_t mbuf_tag_id_find( const char *string, mbuf_tag_id_t *out_id) { - return mbuf_tag_id_find_internal(string, (u_long*)out_id, 1); + return net_str_id_find_internal(string, out_id, NSI_MBUF_TAG, 1); } errno_t @@ -746,13 +807,15 @@ mbuf_tag_allocate( void** data_p) { struct m_tag *tag; + u_int32_t mtag_id_first, mtag_id_last; if (data_p != NULL) *data_p = NULL; /* Sanity check parameters */ - if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || id < MTAG_FIRST_ID || - id >= mtag_id_next || length < 1 || (length & 0xffff0000) != 0 || + (void) net_str_id_first_last(&mtag_id_first, &mtag_id_last, NSI_MBUF_TAG); + if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || id < mtag_id_first || + id > mtag_id_last || length < 1 || (length & 0xffff0000) != 0 || data_p == NULL) { return EINVAL; } @@ -785,6 +848,7 @@ mbuf_tag_find( void** data_p) { struct m_tag *tag; + u_int32_t mtag_id_first, mtag_id_last; if (length != NULL) *length = 0; @@ -792,8 +856,9 @@ mbuf_tag_find( *data_p = NULL; /* Sanity check parameters */ - if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || id < MTAG_FIRST_ID || - id >= mtag_id_next || length == NULL || data_p == NULL) { + (void) net_str_id_first_last(&mtag_id_first, &mtag_id_last, NSI_MBUF_TAG); + if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || id < mtag_id_first || + id > mtag_id_last || length == NULL || data_p == NULL) { return EINVAL; } @@ -817,9 +882,12 @@ mbuf_tag_free( mbuf_tag_type_t type) { struct m_tag *tag; + u_int32_t mtag_id_first, mtag_id_last; - if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || id < MTAG_FIRST_ID || - id >= mtag_id_next) + /* Sanity check parameters */ + (void) net_str_id_first_last(&mtag_id_first, &mtag_id_last, NSI_MBUF_TAG); + if (mbuf == NULL || (mbuf->m_flags & M_PKTHDR) == 0 || id < mtag_id_first || + id > mtag_id_last) return; tag = m_tag_locate(mbuf, id, type, NULL); @@ -872,6 +940,8 @@ mbuf_allocpacket(mbuf_how_t how, size_t packetlen, unsigned int *maxchunks, mbuf else error = ENOMEM; } else { + if (maxchunks) + *maxchunks = numchunks; error = 0; *mbuf = m; } @@ -879,6 +949,38 @@ out: return error; } +errno_t +mbuf_allocpacket_list(unsigned int numpkts, mbuf_how_t how, size_t packetlen, unsigned int *maxchunks, mbuf_t *mbuf) +{ + errno_t error; + struct mbuf *m; + unsigned int numchunks = maxchunks ? *maxchunks : 0; + + if (numpkts == 0) { + error = EINVAL; + goto out; + } + if (packetlen == 0) { + error = EINVAL; + goto out; + } + m = m_allocpacket_internal(&numpkts, packetlen, maxchunks ? &numchunks : NULL, how, 1, 0); + if (m == 0) { + if (maxchunks && *maxchunks && numchunks > *maxchunks) + error = ENOBUFS; + else + error = ENOMEM; + } else { + if (maxchunks) + *maxchunks = numchunks; + error = 0; + *mbuf = m; + } +out: + return error; +} + + /* * mbuf_copyback differs from m_copyback in a few ways: @@ -957,3 +1059,35 @@ out: return result; } + +u_int32_t +mbuf_get_mlen(void) +{ + return (_MLEN); +} + +u_int32_t +mbuf_get_mhlen(void) +{ + return (_MHLEN); +} + +mbuf_priority_t +mbuf_get_priority(struct mbuf *m) +{ +#if !PKT_PRIORITY +#pragma unused(m) + return (MBUF_PRIORITY_NORMAL); +#else /* PKT_PRIORITY */ + mbuf_priority_t prio = MBUF_PRIORITY_NORMAL; + + if (m == NULL || !(m->m_flags & M_PKTHDR)) + return (prio); + + /* Defaults to normal; ignore anything else but background */ + if (m->m_pkthdr.prio == MBUF_PRIORITY_BACKGROUND) + prio = MBUF_PRIORITY_BACKGROUND; + + return (prio); +#endif /* PKT_PRIORITY */ +}