]> git.saurik.com Git - apple/xnu.git/blob - bsd/net/dlil.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-4903.241.1.tar.gz
[apple/xnu.git] / bsd / net / dlil.c
1 /*
2 * Copyright (c) 1999-2018 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
30 * support for mandatory and extensible security protections. This notice
31 * is included in support of clause 2.2 (b) of the Apple Public License,
32 * Version 2.0.
33 */
34 #include <stddef.h>
35
36 #include <sys/param.h>
37 #include <sys/systm.h>
38 #include <sys/kernel.h>
39 #include <sys/malloc.h>
40 #include <sys/mbuf.h>
41 #include <sys/socket.h>
42 #include <sys/domain.h>
43 #include <sys/user.h>
44 #include <sys/random.h>
45 #include <sys/socketvar.h>
46 #include <net/if_dl.h>
47 #include <net/if.h>
48 #include <net/route.h>
49 #include <net/if_var.h>
50 #include <net/dlil.h>
51 #include <net/if_arp.h>
52 #include <net/iptap.h>
53 #include <net/pktap.h>
54 #include <sys/kern_event.h>
55 #include <sys/kdebug.h>
56 #include <sys/mcache.h>
57 #include <sys/syslog.h>
58 #include <sys/protosw.h>
59 #include <sys/priv.h>
60
61 #include <kern/assert.h>
62 #include <kern/task.h>
63 #include <kern/thread.h>
64 #include <kern/sched_prim.h>
65 #include <kern/locks.h>
66 #include <kern/zalloc.h>
67
68 #include <net/kpi_protocol.h>
69 #include <net/if_types.h>
70 #include <net/if_ipsec.h>
71 #include <net/if_llreach.h>
72 #include <net/if_utun.h>
73 #include <net/kpi_interfacefilter.h>
74 #include <net/classq/classq.h>
75 #include <net/classq/classq_sfb.h>
76 #include <net/flowhash.h>
77 #include <net/ntstat.h>
78 #include <net/if_llatbl.h>
79 #include <net/net_api_stats.h>
80 #include <net/if_ports_used.h>
81 #include <netinet/in.h>
82 #if INET
83 #include <netinet/in_var.h>
84 #include <netinet/igmp_var.h>
85 #include <netinet/ip_var.h>
86 #include <netinet/tcp.h>
87 #include <netinet/tcp_var.h>
88 #include <netinet/udp.h>
89 #include <netinet/udp_var.h>
90 #include <netinet/if_ether.h>
91 #include <netinet/in_pcb.h>
92 #include <netinet/in_tclass.h>
93 #include <netinet/ip.h>
94 #include <netinet/ip_icmp.h>
95 #include <netinet/icmp_var.h>
96 #endif /* INET */
97
98 #if INET6
99 #include <net/nat464_utils.h>
100 #include <netinet6/in6_var.h>
101 #include <netinet6/nd6.h>
102 #include <netinet6/mld6_var.h>
103 #include <netinet6/scope6_var.h>
104 #include <netinet/ip6.h>
105 #include <netinet/icmp6.h>
106 #endif /* INET6 */
107 #include <net/pf_pbuf.h>
108 #include <libkern/OSAtomic.h>
109 #include <libkern/tree.h>
110
111 #include <dev/random/randomdev.h>
112 #include <machine/machine_routines.h>
113
114 #include <mach/thread_act.h>
115 #include <mach/sdt.h>
116
117 #if CONFIG_MACF
118 #include <sys/kauth.h>
119 #include <security/mac_framework.h>
120 #include <net/ethernet.h>
121 #include <net/firewire.h>
122 #endif
123
124 #if PF
125 #include <net/pfvar.h>
126 #endif /* PF */
127 #include <net/pktsched/pktsched.h>
128
129 #if NECP
130 #include <net/necp.h>
131 #endif /* NECP */
132
133
134 #define DBG_LAYER_BEG DLILDBG_CODE(DBG_DLIL_STATIC, 0)
135 #define DBG_LAYER_END DLILDBG_CODE(DBG_DLIL_STATIC, 2)
136 #define DBG_FNC_DLIL_INPUT DLILDBG_CODE(DBG_DLIL_STATIC, (1 << 8))
137 #define DBG_FNC_DLIL_OUTPUT DLILDBG_CODE(DBG_DLIL_STATIC, (2 << 8))
138 #define DBG_FNC_DLIL_IFOUT DLILDBG_CODE(DBG_DLIL_STATIC, (3 << 8))
139
140 #define MAX_FRAME_TYPE_SIZE 4 /* LONGWORDS */
141 #define MAX_LINKADDR 4 /* LONGWORDS */
142 #define M_NKE M_IFADDR
143
144 #if 1
145 #define DLIL_PRINTF printf
146 #else
147 #define DLIL_PRINTF kprintf
148 #endif
149
150 #define IF_DATA_REQUIRE_ALIGNED_64(f) \
151 _CASSERT(!(offsetof(struct if_data_internal, f) % sizeof (u_int64_t)))
152
153 #define IFNET_IF_DATA_REQUIRE_ALIGNED_64(f) \
154 _CASSERT(!(offsetof(struct ifnet, if_data.f) % sizeof (u_int64_t)))
155
156 enum {
157 kProtoKPI_v1 = 1,
158 kProtoKPI_v2 = 2
159 };
160
161 /*
162 * List of if_proto structures in if_proto_hash[] is protected by
163 * the ifnet lock. The rest of the fields are initialized at protocol
164 * attach time and never change, thus no lock required as long as
165 * a reference to it is valid, via if_proto_ref().
166 */
167 struct if_proto {
168 SLIST_ENTRY(if_proto) next_hash;
169 u_int32_t refcount;
170 u_int32_t detached;
171 struct ifnet *ifp;
172 protocol_family_t protocol_family;
173 int proto_kpi;
174 union {
175 struct {
176 proto_media_input input;
177 proto_media_preout pre_output;
178 proto_media_event event;
179 proto_media_ioctl ioctl;
180 proto_media_detached detached;
181 proto_media_resolve_multi resolve_multi;
182 proto_media_send_arp send_arp;
183 } v1;
184 struct {
185 proto_media_input_v2 input;
186 proto_media_preout pre_output;
187 proto_media_event event;
188 proto_media_ioctl ioctl;
189 proto_media_detached detached;
190 proto_media_resolve_multi resolve_multi;
191 proto_media_send_arp send_arp;
192 } v2;
193 } kpi;
194 };
195
196 SLIST_HEAD(proto_hash_entry, if_proto);
197
198 #define DLIL_SDLDATALEN \
199 (DLIL_SDLMAXLEN - offsetof(struct sockaddr_dl, sdl_data[0]))
200
201 struct dlil_ifnet {
202 struct ifnet dl_if; /* public ifnet */
203 /*
204 * DLIL private fields, protected by dl_if_lock
205 */
206 decl_lck_mtx_data(, dl_if_lock);
207 TAILQ_ENTRY(dlil_ifnet) dl_if_link; /* dlil_ifnet link */
208 u_int32_t dl_if_flags; /* flags (below) */
209 u_int32_t dl_if_refcnt; /* refcnt */
210 void (*dl_if_trace)(struct dlil_ifnet *, int); /* ref trace callback */
211 void *dl_if_uniqueid; /* unique interface id */
212 size_t dl_if_uniqueid_len; /* length of the unique id */
213 char dl_if_namestorage[IFNAMSIZ]; /* interface name storage */
214 char dl_if_xnamestorage[IFXNAMSIZ]; /* external name storage */
215 struct {
216 struct ifaddr ifa; /* lladdr ifa */
217 u_int8_t asdl[DLIL_SDLMAXLEN]; /* addr storage */
218 u_int8_t msdl[DLIL_SDLMAXLEN]; /* mask storage */
219 } dl_if_lladdr;
220 u_int8_t dl_if_descstorage[IF_DESCSIZE]; /* desc storage */
221 struct dlil_threading_info dl_if_inpstorage; /* input thread storage */
222 ctrace_t dl_if_attach; /* attach PC stacktrace */
223 ctrace_t dl_if_detach; /* detach PC stacktrace */
224 };
225
226 /* Values for dl_if_flags (private to DLIL) */
227 #define DLIF_INUSE 0x1 /* DLIL ifnet recycler, ifnet in use */
228 #define DLIF_REUSE 0x2 /* DLIL ifnet recycles, ifnet is not new */
229 #define DLIF_DEBUG 0x4 /* has debugging info */
230
231 #define IF_REF_TRACE_HIST_SIZE 8 /* size of ref trace history */
232
233 /* For gdb */
234 __private_extern__ unsigned int if_ref_trace_hist_size = IF_REF_TRACE_HIST_SIZE;
235
236 struct dlil_ifnet_dbg {
237 struct dlil_ifnet dldbg_dlif; /* dlil_ifnet */
238 u_int16_t dldbg_if_refhold_cnt; /* # ifnet references */
239 u_int16_t dldbg_if_refrele_cnt; /* # ifnet releases */
240 /*
241 * Circular lists of ifnet_{reference,release} callers.
242 */
243 ctrace_t dldbg_if_refhold[IF_REF_TRACE_HIST_SIZE];
244 ctrace_t dldbg_if_refrele[IF_REF_TRACE_HIST_SIZE];
245 };
246
247 #define DLIL_TO_IFP(s) (&s->dl_if)
248 #define IFP_TO_DLIL(s) ((struct dlil_ifnet *)s)
249
250 struct ifnet_filter {
251 TAILQ_ENTRY(ifnet_filter) filt_next;
252 u_int32_t filt_skip;
253 u_int32_t filt_flags;
254 ifnet_t filt_ifp;
255 const char *filt_name;
256 void *filt_cookie;
257 protocol_family_t filt_protocol;
258 iff_input_func filt_input;
259 iff_output_func filt_output;
260 iff_event_func filt_event;
261 iff_ioctl_func filt_ioctl;
262 iff_detached_func filt_detached;
263 };
264
265 struct proto_input_entry;
266
267 static TAILQ_HEAD(, dlil_ifnet) dlil_ifnet_head;
268 static lck_grp_t *dlil_lock_group;
269 lck_grp_t *ifnet_lock_group;
270 static lck_grp_t *ifnet_head_lock_group;
271 static lck_grp_t *ifnet_snd_lock_group;
272 static lck_grp_t *ifnet_rcv_lock_group;
273 lck_attr_t *ifnet_lock_attr;
274 decl_lck_rw_data(static, ifnet_head_lock);
275 decl_lck_mtx_data(static, dlil_ifnet_lock);
276 u_int32_t dlil_filter_disable_tso_count = 0;
277
278 #if DEBUG
279 static unsigned int ifnet_debug = 1; /* debugging (enabled) */
280 #else
281 static unsigned int ifnet_debug; /* debugging (disabled) */
282 #endif /* !DEBUG */
283 static unsigned int dlif_size; /* size of dlil_ifnet to allocate */
284 static unsigned int dlif_bufsize; /* size of dlif_size + headroom */
285 static struct zone *dlif_zone; /* zone for dlil_ifnet */
286
287 #define DLIF_ZONE_MAX IFNETS_MAX /* maximum elements in zone */
288 #define DLIF_ZONE_NAME "ifnet" /* zone name */
289
290 static unsigned int dlif_filt_size; /* size of ifnet_filter */
291 static struct zone *dlif_filt_zone; /* zone for ifnet_filter */
292
293 #define DLIF_FILT_ZONE_MAX 8 /* maximum elements in zone */
294 #define DLIF_FILT_ZONE_NAME "ifnet_filter" /* zone name */
295
296 static unsigned int dlif_phash_size; /* size of ifnet proto hash table */
297 static struct zone *dlif_phash_zone; /* zone for ifnet proto hash table */
298
299 #define DLIF_PHASH_ZONE_MAX DLIF_ZONE_MAX /* maximum elements in zone */
300 #define DLIF_PHASH_ZONE_NAME "ifnet_proto_hash" /* zone name */
301
302 static unsigned int dlif_proto_size; /* size of if_proto */
303 static struct zone *dlif_proto_zone; /* zone for if_proto */
304
305 #define DLIF_PROTO_ZONE_MAX (DLIF_ZONE_MAX*2) /* maximum elements in zone */
306 #define DLIF_PROTO_ZONE_NAME "ifnet_proto" /* zone name */
307
308 static unsigned int dlif_tcpstat_size; /* size of tcpstat_local to allocate */
309 static unsigned int dlif_tcpstat_bufsize; /* size of dlif_tcpstat_size + headroom */
310 static struct zone *dlif_tcpstat_zone; /* zone for tcpstat_local */
311
312 #define DLIF_TCPSTAT_ZONE_MAX 1 /* maximum elements in zone */
313 #define DLIF_TCPSTAT_ZONE_NAME "ifnet_tcpstat" /* zone name */
314
315 static unsigned int dlif_udpstat_size; /* size of udpstat_local to allocate */
316 static unsigned int dlif_udpstat_bufsize; /* size of dlif_udpstat_size + headroom */
317 static struct zone *dlif_udpstat_zone; /* zone for udpstat_local */
318
319 #define DLIF_UDPSTAT_ZONE_MAX 1 /* maximum elements in zone */
320 #define DLIF_UDPSTAT_ZONE_NAME "ifnet_udpstat" /* zone name */
321
322 static u_int32_t net_rtref;
323
324 static struct dlil_main_threading_info dlil_main_input_thread_info;
325 __private_extern__ struct dlil_threading_info *dlil_main_input_thread =
326 (struct dlil_threading_info *)&dlil_main_input_thread_info;
327
328 static int dlil_event_internal(struct ifnet *ifp, struct kev_msg *msg, bool update_generation);
329 static int dlil_detach_filter_internal(interface_filter_t filter, int detached);
330 static void dlil_if_trace(struct dlil_ifnet *, int);
331 static void if_proto_ref(struct if_proto *);
332 static void if_proto_free(struct if_proto *);
333 static struct if_proto *find_attached_proto(struct ifnet *, u_int32_t);
334 static u_int32_t dlil_ifp_protolist(struct ifnet *ifp, protocol_family_t *list,
335 u_int32_t list_count);
336 static void if_flt_monitor_busy(struct ifnet *);
337 static void if_flt_monitor_unbusy(struct ifnet *);
338 static void if_flt_monitor_enter(struct ifnet *);
339 static void if_flt_monitor_leave(struct ifnet *);
340 static int dlil_interface_filters_input(struct ifnet *, struct mbuf **,
341 char **, protocol_family_t);
342 static int dlil_interface_filters_output(struct ifnet *, struct mbuf **,
343 protocol_family_t);
344 static struct ifaddr *dlil_alloc_lladdr(struct ifnet *,
345 const struct sockaddr_dl *);
346 static int ifnet_lookup(struct ifnet *);
347 static void if_purgeaddrs(struct ifnet *);
348
349 static errno_t ifproto_media_input_v1(struct ifnet *, protocol_family_t,
350 struct mbuf *, char *);
351 static errno_t ifproto_media_input_v2(struct ifnet *, protocol_family_t,
352 struct mbuf *);
353 static errno_t ifproto_media_preout(struct ifnet *, protocol_family_t,
354 mbuf_t *, const struct sockaddr *, void *, char *, char *);
355 static void ifproto_media_event(struct ifnet *, protocol_family_t,
356 const struct kev_msg *);
357 static errno_t ifproto_media_ioctl(struct ifnet *, protocol_family_t,
358 unsigned long, void *);
359 static errno_t ifproto_media_resolve_multi(ifnet_t, const struct sockaddr *,
360 struct sockaddr_dl *, size_t);
361 static errno_t ifproto_media_send_arp(struct ifnet *, u_short,
362 const struct sockaddr_dl *, const struct sockaddr *,
363 const struct sockaddr_dl *, const struct sockaddr *);
364
365 static errno_t ifp_if_input(struct ifnet *ifp, struct mbuf *m_head,
366 struct mbuf *m_tail, const struct ifnet_stat_increment_param *s,
367 boolean_t poll, struct thread *tp);
368 static void ifp_if_input_poll(struct ifnet *, u_int32_t, u_int32_t,
369 struct mbuf **, struct mbuf **, u_int32_t *, u_int32_t *);
370 static errno_t ifp_if_ctl(struct ifnet *, ifnet_ctl_cmd_t, u_int32_t, void *);
371 static errno_t ifp_if_demux(struct ifnet *, struct mbuf *, char *,
372 protocol_family_t *);
373 static errno_t ifp_if_add_proto(struct ifnet *, protocol_family_t,
374 const struct ifnet_demux_desc *, u_int32_t);
375 static errno_t ifp_if_del_proto(struct ifnet *, protocol_family_t);
376 static errno_t ifp_if_check_multi(struct ifnet *, const struct sockaddr *);
377 #if CONFIG_EMBEDDED
378 static errno_t ifp_if_framer(struct ifnet *, struct mbuf **,
379 const struct sockaddr *, const char *, const char *,
380 u_int32_t *, u_int32_t *);
381 #else
382 static errno_t ifp_if_framer(struct ifnet *, struct mbuf **,
383 const struct sockaddr *, const char *, const char *);
384 #endif /* CONFIG_EMBEDDED */
385 static errno_t ifp_if_framer_extended(struct ifnet *, struct mbuf **,
386 const struct sockaddr *, const char *, const char *,
387 u_int32_t *, u_int32_t *);
388 static errno_t ifp_if_set_bpf_tap(struct ifnet *, bpf_tap_mode, bpf_packet_func);
389 static void ifp_if_free(struct ifnet *);
390 static void ifp_if_event(struct ifnet *, const struct kev_msg *);
391 static __inline void ifp_inc_traffic_class_in(struct ifnet *, struct mbuf *);
392 static __inline void ifp_inc_traffic_class_out(struct ifnet *, struct mbuf *);
393
394 static void dlil_main_input_thread_func(void *, wait_result_t);
395 static void dlil_input_thread_func(void *, wait_result_t);
396 static void dlil_rxpoll_input_thread_func(void *, wait_result_t);
397 static int dlil_create_input_thread(ifnet_t, struct dlil_threading_info *);
398 static void dlil_terminate_input_thread(struct dlil_threading_info *);
399 static void dlil_input_stats_add(const struct ifnet_stat_increment_param *,
400 struct dlil_threading_info *, boolean_t);
401 static void dlil_input_stats_sync(struct ifnet *, struct dlil_threading_info *);
402 static void dlil_input_packet_list_common(struct ifnet *, struct mbuf *,
403 u_int32_t, ifnet_model_t, boolean_t);
404 static errno_t ifnet_input_common(struct ifnet *, struct mbuf *, struct mbuf *,
405 const struct ifnet_stat_increment_param *, boolean_t, boolean_t);
406 static int dlil_is_clat_needed(protocol_family_t , mbuf_t );
407 static errno_t dlil_clat46(ifnet_t, protocol_family_t *, mbuf_t *);
408 static errno_t dlil_clat64(ifnet_t, protocol_family_t *, mbuf_t *);
409 #if DEBUG || DEVELOPMENT
410 static void dlil_verify_sum16(void);
411 #endif /* DEBUG || DEVELOPMENT */
412 static void dlil_output_cksum_dbg(struct ifnet *, struct mbuf *, uint32_t,
413 protocol_family_t);
414 static void dlil_input_cksum_dbg(struct ifnet *, struct mbuf *, char *,
415 protocol_family_t);
416
417 static void ifnet_detacher_thread_func(void *, wait_result_t);
418 static int ifnet_detacher_thread_cont(int);
419 static void ifnet_detach_final(struct ifnet *);
420 static void ifnet_detaching_enqueue(struct ifnet *);
421 static struct ifnet *ifnet_detaching_dequeue(void);
422
423 static void ifnet_start_thread_fn(void *, wait_result_t);
424 static void ifnet_poll_thread_fn(void *, wait_result_t);
425 static void ifnet_poll(struct ifnet *);
426 static errno_t ifnet_enqueue_common(struct ifnet *, void *,
427 classq_pkt_type_t, boolean_t, boolean_t *);
428
429 static void ifp_src_route_copyout(struct ifnet *, struct route *);
430 static void ifp_src_route_copyin(struct ifnet *, struct route *);
431 #if INET6
432 static void ifp_src_route6_copyout(struct ifnet *, struct route_in6 *);
433 static void ifp_src_route6_copyin(struct ifnet *, struct route_in6 *);
434 #endif /* INET6 */
435
436 static int sysctl_rxpoll SYSCTL_HANDLER_ARGS;
437 static int sysctl_rxpoll_mode_holdtime SYSCTL_HANDLER_ARGS;
438 static int sysctl_rxpoll_sample_holdtime SYSCTL_HANDLER_ARGS;
439 static int sysctl_rxpoll_interval_time SYSCTL_HANDLER_ARGS;
440 static int sysctl_rxpoll_wlowat SYSCTL_HANDLER_ARGS;
441 static int sysctl_rxpoll_whiwat SYSCTL_HANDLER_ARGS;
442 static int sysctl_sndq_maxlen SYSCTL_HANDLER_ARGS;
443 static int sysctl_rcvq_maxlen SYSCTL_HANDLER_ARGS;
444 static int sysctl_hwcksum_dbg_mode SYSCTL_HANDLER_ARGS;
445 static int sysctl_hwcksum_dbg_partial_rxoff_forced SYSCTL_HANDLER_ARGS;
446 static int sysctl_hwcksum_dbg_partial_rxoff_adj SYSCTL_HANDLER_ARGS;
447
448 struct chain_len_stats tx_chain_len_stats;
449 static int sysctl_tx_chain_len_stats SYSCTL_HANDLER_ARGS;
450
451 #if TEST_INPUT_THREAD_TERMINATION
452 static int sysctl_input_thread_termination_spin SYSCTL_HANDLER_ARGS;
453 #endif /* TEST_INPUT_THREAD_TERMINATION */
454
455 /* The following are protected by dlil_ifnet_lock */
456 static TAILQ_HEAD(, ifnet) ifnet_detaching_head;
457 static u_int32_t ifnet_detaching_cnt;
458 static void *ifnet_delayed_run; /* wait channel for detaching thread */
459
460 decl_lck_mtx_data(static, ifnet_fc_lock);
461
462 static uint32_t ifnet_flowhash_seed;
463
464 struct ifnet_flowhash_key {
465 char ifk_name[IFNAMSIZ];
466 uint32_t ifk_unit;
467 uint32_t ifk_flags;
468 uint32_t ifk_eflags;
469 uint32_t ifk_capabilities;
470 uint32_t ifk_capenable;
471 uint32_t ifk_output_sched_model;
472 uint32_t ifk_rand1;
473 uint32_t ifk_rand2;
474 };
475
476 /* Flow control entry per interface */
477 struct ifnet_fc_entry {
478 RB_ENTRY(ifnet_fc_entry) ifce_entry;
479 u_int32_t ifce_flowhash;
480 struct ifnet *ifce_ifp;
481 };
482
483 static uint32_t ifnet_calc_flowhash(struct ifnet *);
484 static int ifce_cmp(const struct ifnet_fc_entry *,
485 const struct ifnet_fc_entry *);
486 static int ifnet_fc_add(struct ifnet *);
487 static struct ifnet_fc_entry *ifnet_fc_get(u_int32_t);
488 static void ifnet_fc_entry_free(struct ifnet_fc_entry *);
489
490 /* protected by ifnet_fc_lock */
491 RB_HEAD(ifnet_fc_tree, ifnet_fc_entry) ifnet_fc_tree;
492 RB_PROTOTYPE(ifnet_fc_tree, ifnet_fc_entry, ifce_entry, ifce_cmp);
493 RB_GENERATE(ifnet_fc_tree, ifnet_fc_entry, ifce_entry, ifce_cmp);
494
495 static unsigned int ifnet_fc_zone_size; /* sizeof ifnet_fc_entry */
496 static struct zone *ifnet_fc_zone; /* ifnet_fc_entry zone */
497
498 #define IFNET_FC_ZONE_NAME "ifnet_fc_zone"
499 #define IFNET_FC_ZONE_MAX 32
500
501 extern void bpfdetach(struct ifnet *);
502 extern void proto_input_run(void);
503
504 extern uint32_t udp_count_opportunistic(unsigned int ifindex,
505 u_int32_t flags);
506 extern uint32_t tcp_count_opportunistic(unsigned int ifindex,
507 u_int32_t flags);
508
509 __private_extern__ void link_rtrequest(int, struct rtentry *, struct sockaddr *);
510
511 #if CONFIG_MACF
512 #ifdef CONFIG_EMBEDDED
513 int dlil_lladdr_ckreq = 1;
514 #else
515 int dlil_lladdr_ckreq = 0;
516 #endif
517 #endif
518
519 #if DEBUG
520 int dlil_verbose = 1;
521 #else
522 int dlil_verbose = 0;
523 #endif /* DEBUG */
524 #if IFNET_INPUT_SANITY_CHK
525 /* sanity checking of input packet lists received */
526 static u_int32_t dlil_input_sanity_check = 0;
527 #endif /* IFNET_INPUT_SANITY_CHK */
528 /* rate limit debug messages */
529 struct timespec dlil_dbgrate = { 1, 0 };
530
531 SYSCTL_DECL(_net_link_generic_system);
532
533 SYSCTL_INT(_net_link_generic_system, OID_AUTO, dlil_verbose,
534 CTLFLAG_RW | CTLFLAG_LOCKED, &dlil_verbose, 0, "Log DLIL error messages");
535
536 #define IF_SNDQ_MINLEN 32
537 u_int32_t if_sndq_maxlen = IFQ_MAXLEN;
538 SYSCTL_PROC(_net_link_generic_system, OID_AUTO, sndq_maxlen,
539 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &if_sndq_maxlen, IFQ_MAXLEN,
540 sysctl_sndq_maxlen, "I", "Default transmit queue max length");
541
542 #define IF_RCVQ_MINLEN 32
543 #define IF_RCVQ_MAXLEN 256
544 u_int32_t if_rcvq_maxlen = IF_RCVQ_MAXLEN;
545 SYSCTL_PROC(_net_link_generic_system, OID_AUTO, rcvq_maxlen,
546 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &if_rcvq_maxlen, IFQ_MAXLEN,
547 sysctl_rcvq_maxlen, "I", "Default receive queue max length");
548
549 #define IF_RXPOLL_DECAY 2 /* ilog2 of EWMA decay rate (4) */
550 static u_int32_t if_rxpoll_decay = IF_RXPOLL_DECAY;
551 SYSCTL_UINT(_net_link_generic_system, OID_AUTO, rxpoll_decay,
552 CTLFLAG_RW | CTLFLAG_LOCKED, &if_rxpoll_decay, IF_RXPOLL_DECAY,
553 "ilog2 of EWMA decay rate of avg inbound packets");
554
555 #define IF_RXPOLL_MODE_HOLDTIME_MIN (10ULL * 1000 * 1000) /* 10 ms */
556 #define IF_RXPOLL_MODE_HOLDTIME (1000ULL * 1000 * 1000) /* 1 sec */
557 static u_int64_t if_rxpoll_mode_holdtime = IF_RXPOLL_MODE_HOLDTIME;
558 SYSCTL_PROC(_net_link_generic_system, OID_AUTO, rxpoll_freeze_time,
559 CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED, &if_rxpoll_mode_holdtime,
560 IF_RXPOLL_MODE_HOLDTIME, sysctl_rxpoll_mode_holdtime,
561 "Q", "input poll mode freeze time");
562
563 #define IF_RXPOLL_SAMPLETIME_MIN (1ULL * 1000 * 1000) /* 1 ms */
564 #define IF_RXPOLL_SAMPLETIME (10ULL * 1000 * 1000) /* 10 ms */
565 static u_int64_t if_rxpoll_sample_holdtime = IF_RXPOLL_SAMPLETIME;
566 SYSCTL_PROC(_net_link_generic_system, OID_AUTO, rxpoll_sample_time,
567 CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED, &if_rxpoll_sample_holdtime,
568 IF_RXPOLL_SAMPLETIME, sysctl_rxpoll_sample_holdtime,
569 "Q", "input poll sampling time");
570
571 #define IF_RXPOLL_INTERVALTIME_MIN (1ULL * 1000) /* 1 us */
572 #define IF_RXPOLL_INTERVALTIME (1ULL * 1000 * 1000) /* 1 ms */
573 static u_int64_t if_rxpoll_interval_time = IF_RXPOLL_INTERVALTIME;
574 SYSCTL_PROC(_net_link_generic_system, OID_AUTO, rxpoll_interval_time,
575 CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED, &if_rxpoll_interval_time,
576 IF_RXPOLL_INTERVALTIME, sysctl_rxpoll_interval_time,
577 "Q", "input poll interval (time)");
578
579 #define IF_RXPOLL_INTERVAL_PKTS 0 /* 0 (disabled) */
580 static u_int32_t if_rxpoll_interval_pkts = IF_RXPOLL_INTERVAL_PKTS;
581 SYSCTL_UINT(_net_link_generic_system, OID_AUTO, rxpoll_interval_pkts,
582 CTLFLAG_RW | CTLFLAG_LOCKED, &if_rxpoll_interval_pkts,
583 IF_RXPOLL_INTERVAL_PKTS, "input poll interval (packets)");
584
585 #define IF_RXPOLL_WLOWAT 10
586 static u_int32_t if_rxpoll_wlowat = IF_RXPOLL_WLOWAT;
587 SYSCTL_PROC(_net_link_generic_system, OID_AUTO, rxpoll_wakeups_lowat,
588 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &if_rxpoll_wlowat,
589 IF_RXPOLL_WLOWAT, sysctl_rxpoll_wlowat,
590 "I", "input poll wakeup low watermark");
591
592 #define IF_RXPOLL_WHIWAT 100
593 static u_int32_t if_rxpoll_whiwat = IF_RXPOLL_WHIWAT;
594 SYSCTL_PROC(_net_link_generic_system, OID_AUTO, rxpoll_wakeups_hiwat,
595 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &if_rxpoll_whiwat,
596 IF_RXPOLL_WHIWAT, sysctl_rxpoll_whiwat,
597 "I", "input poll wakeup high watermark");
598
599 static u_int32_t if_rxpoll_max = 0; /* 0 (automatic) */
600 SYSCTL_UINT(_net_link_generic_system, OID_AUTO, rxpoll_max,
601 CTLFLAG_RW | CTLFLAG_LOCKED, &if_rxpoll_max, 0,
602 "max packets per poll call");
603
604 static u_int32_t if_rxpoll = 1;
605 SYSCTL_PROC(_net_link_generic_system, OID_AUTO, rxpoll,
606 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &if_rxpoll, 0,
607 sysctl_rxpoll, "I", "enable opportunistic input polling");
608
609 #if TEST_INPUT_THREAD_TERMINATION
610 static u_int32_t if_input_thread_termination_spin = 0;
611 SYSCTL_PROC(_net_link_generic_system, OID_AUTO, input_thread_termination_spin,
612 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
613 &if_input_thread_termination_spin, 0,
614 sysctl_input_thread_termination_spin,
615 "I", "input thread termination spin limit");
616 #endif /* TEST_INPUT_THREAD_TERMINATION */
617
618 static u_int32_t cur_dlil_input_threads = 0;
619 SYSCTL_UINT(_net_link_generic_system, OID_AUTO, dlil_input_threads,
620 CTLFLAG_RD | CTLFLAG_LOCKED, &cur_dlil_input_threads, 0,
621 "Current number of DLIL input threads");
622
623 #if IFNET_INPUT_SANITY_CHK
624 SYSCTL_UINT(_net_link_generic_system, OID_AUTO, dlil_input_sanity_check,
625 CTLFLAG_RW | CTLFLAG_LOCKED, &dlil_input_sanity_check, 0,
626 "Turn on sanity checking in DLIL input");
627 #endif /* IFNET_INPUT_SANITY_CHK */
628
629 static u_int32_t if_flowadv = 1;
630 SYSCTL_UINT(_net_link_generic_system, OID_AUTO, flow_advisory,
631 CTLFLAG_RW | CTLFLAG_LOCKED, &if_flowadv, 1,
632 "enable flow-advisory mechanism");
633
634 static u_int32_t if_delaybased_queue = 1;
635 SYSCTL_UINT(_net_link_generic_system, OID_AUTO, delaybased_queue,
636 CTLFLAG_RW | CTLFLAG_LOCKED, &if_delaybased_queue, 1,
637 "enable delay based dynamic queue sizing");
638
639 static uint64_t hwcksum_in_invalidated = 0;
640 SYSCTL_QUAD(_net_link_generic_system, OID_AUTO,
641 hwcksum_in_invalidated, CTLFLAG_RD | CTLFLAG_LOCKED,
642 &hwcksum_in_invalidated, "inbound packets with invalidated hardware cksum");
643
644 uint32_t hwcksum_dbg = 0;
645 SYSCTL_UINT(_net_link_generic_system, OID_AUTO, hwcksum_dbg,
646 CTLFLAG_RW | CTLFLAG_LOCKED, &hwcksum_dbg, 0,
647 "enable hardware cksum debugging");
648
649 u_int32_t ifnet_start_delayed = 0;
650 SYSCTL_UINT(_net_link_generic_system, OID_AUTO, start_delayed,
651 CTLFLAG_RW | CTLFLAG_LOCKED, &ifnet_start_delayed, 0,
652 "number of times start was delayed");
653
654 u_int32_t ifnet_delay_start_disabled = 0;
655 SYSCTL_UINT(_net_link_generic_system, OID_AUTO, start_delay_disabled,
656 CTLFLAG_RW | CTLFLAG_LOCKED, &ifnet_delay_start_disabled, 0,
657 "number of times start was delayed");
658
659 #define HWCKSUM_DBG_PARTIAL_FORCED 0x1 /* forced partial checksum */
660 #define HWCKSUM_DBG_PARTIAL_RXOFF_ADJ 0x2 /* adjust start offset */
661 #define HWCKSUM_DBG_FINALIZE_FORCED 0x10 /* forced finalize */
662 #define HWCKSUM_DBG_MASK \
663 (HWCKSUM_DBG_PARTIAL_FORCED | HWCKSUM_DBG_PARTIAL_RXOFF_ADJ | \
664 HWCKSUM_DBG_FINALIZE_FORCED)
665
666 static uint32_t hwcksum_dbg_mode = 0;
667 SYSCTL_PROC(_net_link_generic_system, OID_AUTO, hwcksum_dbg_mode,
668 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &hwcksum_dbg_mode,
669 0, sysctl_hwcksum_dbg_mode, "I", "hardware cksum debugging mode");
670
671 static uint64_t hwcksum_dbg_partial_forced = 0;
672 SYSCTL_QUAD(_net_link_generic_system, OID_AUTO,
673 hwcksum_dbg_partial_forced, CTLFLAG_RD | CTLFLAG_LOCKED,
674 &hwcksum_dbg_partial_forced, "packets forced using partial cksum");
675
676 static uint64_t hwcksum_dbg_partial_forced_bytes = 0;
677 SYSCTL_QUAD(_net_link_generic_system, OID_AUTO,
678 hwcksum_dbg_partial_forced_bytes, CTLFLAG_RD | CTLFLAG_LOCKED,
679 &hwcksum_dbg_partial_forced_bytes, "bytes forced using partial cksum");
680
681 static uint32_t hwcksum_dbg_partial_rxoff_forced = 0;
682 SYSCTL_PROC(_net_link_generic_system, OID_AUTO,
683 hwcksum_dbg_partial_rxoff_forced, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
684 &hwcksum_dbg_partial_rxoff_forced, 0,
685 sysctl_hwcksum_dbg_partial_rxoff_forced, "I",
686 "forced partial cksum rx offset");
687
688 static uint32_t hwcksum_dbg_partial_rxoff_adj = 0;
689 SYSCTL_PROC(_net_link_generic_system, OID_AUTO, hwcksum_dbg_partial_rxoff_adj,
690 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &hwcksum_dbg_partial_rxoff_adj,
691 0, sysctl_hwcksum_dbg_partial_rxoff_adj, "I",
692 "adjusted partial cksum rx offset");
693
694 static uint64_t hwcksum_dbg_verified = 0;
695 SYSCTL_QUAD(_net_link_generic_system, OID_AUTO,
696 hwcksum_dbg_verified, CTLFLAG_RD | CTLFLAG_LOCKED,
697 &hwcksum_dbg_verified, "packets verified for having good checksum");
698
699 static uint64_t hwcksum_dbg_bad_cksum = 0;
700 SYSCTL_QUAD(_net_link_generic_system, OID_AUTO,
701 hwcksum_dbg_bad_cksum, CTLFLAG_RD | CTLFLAG_LOCKED,
702 &hwcksum_dbg_bad_cksum, "packets with bad hardware calculated checksum");
703
704 static uint64_t hwcksum_dbg_bad_rxoff = 0;
705 SYSCTL_QUAD(_net_link_generic_system, OID_AUTO,
706 hwcksum_dbg_bad_rxoff, CTLFLAG_RD | CTLFLAG_LOCKED,
707 &hwcksum_dbg_bad_rxoff, "packets with invalid rxoff");
708
709 static uint64_t hwcksum_dbg_adjusted = 0;
710 SYSCTL_QUAD(_net_link_generic_system, OID_AUTO,
711 hwcksum_dbg_adjusted, CTLFLAG_RD | CTLFLAG_LOCKED,
712 &hwcksum_dbg_adjusted, "packets with rxoff adjusted");
713
714 static uint64_t hwcksum_dbg_finalized_hdr = 0;
715 SYSCTL_QUAD(_net_link_generic_system, OID_AUTO,
716 hwcksum_dbg_finalized_hdr, CTLFLAG_RD | CTLFLAG_LOCKED,
717 &hwcksum_dbg_finalized_hdr, "finalized headers");
718
719 static uint64_t hwcksum_dbg_finalized_data = 0;
720 SYSCTL_QUAD(_net_link_generic_system, OID_AUTO,
721 hwcksum_dbg_finalized_data, CTLFLAG_RD | CTLFLAG_LOCKED,
722 &hwcksum_dbg_finalized_data, "finalized payloads");
723
724 uint32_t hwcksum_tx = 1;
725 SYSCTL_UINT(_net_link_generic_system, OID_AUTO, hwcksum_tx,
726 CTLFLAG_RW | CTLFLAG_LOCKED, &hwcksum_tx, 0,
727 "enable transmit hardware checksum offload");
728
729 uint32_t hwcksum_rx = 1;
730 SYSCTL_UINT(_net_link_generic_system, OID_AUTO, hwcksum_rx,
731 CTLFLAG_RW | CTLFLAG_LOCKED, &hwcksum_rx, 0,
732 "enable receive hardware checksum offload");
733
734 SYSCTL_PROC(_net_link_generic_system, OID_AUTO, tx_chain_len_stats,
735 CTLFLAG_RD | CTLFLAG_LOCKED, 0, 9,
736 sysctl_tx_chain_len_stats, "S", "");
737
738 uint32_t tx_chain_len_count = 0;
739 SYSCTL_UINT(_net_link_generic_system, OID_AUTO, tx_chain_len_count,
740 CTLFLAG_RW | CTLFLAG_LOCKED, &tx_chain_len_count, 0, "");
741
742 static uint32_t threshold_notify = 1; /* enable/disable */
743 SYSCTL_UINT(_net_link_generic_system, OID_AUTO, threshold_notify,
744 CTLFLAG_RW | CTLFLAG_LOCKED, &threshold_notify, 0, "");
745
746 static uint32_t threshold_interval = 2; /* in seconds */
747 SYSCTL_UINT(_net_link_generic_system, OID_AUTO, threshold_interval,
748 CTLFLAG_RW | CTLFLAG_LOCKED, &threshold_interval, 0, "");
749
750 #if (DEVELOPMENT || DEBUG)
751 static int sysctl_get_kao_frames SYSCTL_HANDLER_ARGS;
752 SYSCTL_NODE(_net_link_generic_system, OID_AUTO, get_kao_frames,
753 CTLFLAG_RD | CTLFLAG_LOCKED, sysctl_get_kao_frames, "");
754 #endif /* DEVELOPMENT || DEBUG */
755
756 struct net_api_stats net_api_stats;
757 SYSCTL_STRUCT(_net, OID_AUTO, api_stats, CTLFLAG_RD|CTLFLAG_LOCKED,
758 &net_api_stats, net_api_stats, "");
759
760
761 unsigned int net_rxpoll = 1;
762 unsigned int net_affinity = 1;
763 static kern_return_t dlil_affinity_set(struct thread *, u_int32_t);
764
765 extern u_int32_t inject_buckets;
766
767 static lck_grp_attr_t *dlil_grp_attributes = NULL;
768 static lck_attr_t *dlil_lck_attributes = NULL;
769
770 /* DLIL data threshold thread call */
771 static void dlil_dt_tcall_fn(thread_call_param_t, thread_call_param_t);
772
773 static void dlil_mit_tcall_fn(thread_call_param_t, thread_call_param_t);
774
775 uint32_t dlil_rcv_mit_pkts_min = 5;
776 uint32_t dlil_rcv_mit_pkts_max = 64;
777 uint32_t dlil_rcv_mit_interval = (500 * 1000);
778
779 #if (DEVELOPMENT || DEBUG)
780 SYSCTL_UINT(_net_link_generic_system, OID_AUTO, rcv_mit_pkts_min,
781 CTLFLAG_RW | CTLFLAG_LOCKED, &dlil_rcv_mit_pkts_min, 0, "");
782 SYSCTL_UINT(_net_link_generic_system, OID_AUTO, rcv_mit_pkts_max,
783 CTLFLAG_RW | CTLFLAG_LOCKED, &dlil_rcv_mit_pkts_max, 0, "");
784 SYSCTL_UINT(_net_link_generic_system, OID_AUTO, rcv_mit_interval,
785 CTLFLAG_RW | CTLFLAG_LOCKED, &dlil_rcv_mit_interval, 0, "");
786 #endif /* DEVELOPMENT || DEBUG */
787
788
789 #define DLIL_INPUT_CHECK(m, ifp) { \
790 struct ifnet *_rcvif = mbuf_pkthdr_rcvif(m); \
791 if (_rcvif == NULL || (ifp != lo_ifp && _rcvif != ifp) || \
792 !(mbuf_flags(m) & MBUF_PKTHDR)) { \
793 panic_plain("%s: invalid mbuf %p\n", __func__, m); \
794 /* NOTREACHED */ \
795 } \
796 }
797
798 #define DLIL_EWMA(old, new, decay) do { \
799 u_int32_t _avg; \
800 if ((_avg = (old)) > 0) \
801 _avg = (((_avg << (decay)) - _avg) + (new)) >> (decay); \
802 else \
803 _avg = (new); \
804 (old) = _avg; \
805 } while (0)
806
807 #define MBPS (1ULL * 1000 * 1000)
808 #define GBPS (MBPS * 1000)
809
810 struct rxpoll_time_tbl {
811 u_int64_t speed; /* downlink speed */
812 u_int32_t plowat; /* packets low watermark */
813 u_int32_t phiwat; /* packets high watermark */
814 u_int32_t blowat; /* bytes low watermark */
815 u_int32_t bhiwat; /* bytes high watermark */
816 };
817
818 static struct rxpoll_time_tbl rxpoll_tbl[] = {
819 { 10 * MBPS, 2, 8, (1 * 1024), (6 * 1024) },
820 { 100 * MBPS, 10, 40, (4 * 1024), (64 * 1024) },
821 { 1 * GBPS, 10, 40, (4 * 1024), (64 * 1024) },
822 { 10 * GBPS, 10, 40, (4 * 1024), (64 * 1024) },
823 { 100 * GBPS, 10, 40, (4 * 1024), (64 * 1024) },
824 { 0, 0, 0, 0, 0 }
825 };
826
827 int
828 proto_hash_value(u_int32_t protocol_family)
829 {
830 /*
831 * dlil_proto_unplumb_all() depends on the mapping between
832 * the hash bucket index and the protocol family defined
833 * here; future changes must be applied there as well.
834 */
835 switch (protocol_family) {
836 case PF_INET:
837 return (0);
838 case PF_INET6:
839 return (1);
840 case PF_VLAN:
841 return (2);
842 case PF_UNSPEC:
843 default:
844 return (3);
845 }
846 }
847
848 /*
849 * Caller must already be holding ifnet lock.
850 */
851 static struct if_proto *
852 find_attached_proto(struct ifnet *ifp, u_int32_t protocol_family)
853 {
854 struct if_proto *proto = NULL;
855 u_int32_t i = proto_hash_value(protocol_family);
856
857 ifnet_lock_assert(ifp, IFNET_LCK_ASSERT_OWNED);
858
859 if (ifp->if_proto_hash != NULL)
860 proto = SLIST_FIRST(&ifp->if_proto_hash[i]);
861
862 while (proto != NULL && proto->protocol_family != protocol_family)
863 proto = SLIST_NEXT(proto, next_hash);
864
865 if (proto != NULL)
866 if_proto_ref(proto);
867
868 return (proto);
869 }
870
871 static void
872 if_proto_ref(struct if_proto *proto)
873 {
874 atomic_add_32(&proto->refcount, 1);
875 }
876
877 extern void if_rtproto_del(struct ifnet *ifp, int protocol);
878
879 static void
880 if_proto_free(struct if_proto *proto)
881 {
882 u_int32_t oldval;
883 struct ifnet *ifp = proto->ifp;
884 u_int32_t proto_family = proto->protocol_family;
885 struct kev_dl_proto_data ev_pr_data;
886
887 oldval = atomic_add_32_ov(&proto->refcount, -1);
888 if (oldval > 1)
889 return;
890
891 /* No more reference on this, protocol must have been detached */
892 VERIFY(proto->detached);
893
894 if (proto->proto_kpi == kProtoKPI_v1) {
895 if (proto->kpi.v1.detached)
896 proto->kpi.v1.detached(ifp, proto->protocol_family);
897 }
898 if (proto->proto_kpi == kProtoKPI_v2) {
899 if (proto->kpi.v2.detached)
900 proto->kpi.v2.detached(ifp, proto->protocol_family);
901 }
902
903 /*
904 * Cleanup routes that may still be in the routing table for that
905 * interface/protocol pair.
906 */
907 if_rtproto_del(ifp, proto_family);
908
909 /*
910 * The reserved field carries the number of protocol still attached
911 * (subject to change)
912 */
913 ifnet_lock_shared(ifp);
914 ev_pr_data.proto_family = proto_family;
915 ev_pr_data.proto_remaining_count = dlil_ifp_protolist(ifp, NULL, 0);
916 ifnet_lock_done(ifp);
917
918 dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_PROTO_DETACHED,
919 (struct net_event_data *)&ev_pr_data,
920 sizeof (struct kev_dl_proto_data));
921
922 if (ev_pr_data.proto_remaining_count == 0) {
923 /*
924 * The protocol count has gone to zero, mark the interface down.
925 * This used to be done by configd.KernelEventMonitor, but that
926 * is inherently prone to races (rdar://problem/30810208).
927 */
928 (void) ifnet_set_flags(ifp, 0, IFF_UP);
929 (void) ifnet_ioctl(ifp, 0, SIOCSIFFLAGS, NULL);
930 dlil_post_sifflags_msg(ifp);
931 }
932
933 zfree(dlif_proto_zone, proto);
934 }
935
936 __private_extern__ void
937 ifnet_lock_assert(struct ifnet *ifp, ifnet_lock_assert_t what)
938 {
939 #if !MACH_ASSERT
940 #pragma unused(ifp)
941 #endif
942 unsigned int type = 0;
943 int ass = 1;
944
945 switch (what) {
946 case IFNET_LCK_ASSERT_EXCLUSIVE:
947 type = LCK_RW_ASSERT_EXCLUSIVE;
948 break;
949
950 case IFNET_LCK_ASSERT_SHARED:
951 type = LCK_RW_ASSERT_SHARED;
952 break;
953
954 case IFNET_LCK_ASSERT_OWNED:
955 type = LCK_RW_ASSERT_HELD;
956 break;
957
958 case IFNET_LCK_ASSERT_NOTOWNED:
959 /* nothing to do here for RW lock; bypass assert */
960 ass = 0;
961 break;
962
963 default:
964 panic("bad ifnet assert type: %d", what);
965 /* NOTREACHED */
966 }
967 if (ass)
968 LCK_RW_ASSERT(&ifp->if_lock, type);
969 }
970
971 __private_extern__ void
972 ifnet_lock_shared(struct ifnet *ifp)
973 {
974 lck_rw_lock_shared(&ifp->if_lock);
975 }
976
977 __private_extern__ void
978 ifnet_lock_exclusive(struct ifnet *ifp)
979 {
980 lck_rw_lock_exclusive(&ifp->if_lock);
981 }
982
983 __private_extern__ void
984 ifnet_lock_done(struct ifnet *ifp)
985 {
986 lck_rw_done(&ifp->if_lock);
987 }
988
989 #if INET
990 __private_extern__ void
991 if_inetdata_lock_shared(struct ifnet *ifp)
992 {
993 lck_rw_lock_shared(&ifp->if_inetdata_lock);
994 }
995
996 __private_extern__ void
997 if_inetdata_lock_exclusive(struct ifnet *ifp)
998 {
999 lck_rw_lock_exclusive(&ifp->if_inetdata_lock);
1000 }
1001
1002 __private_extern__ void
1003 if_inetdata_lock_done(struct ifnet *ifp)
1004 {
1005 lck_rw_done(&ifp->if_inetdata_lock);
1006 }
1007 #endif
1008
1009 #if INET6
1010 __private_extern__ void
1011 if_inet6data_lock_shared(struct ifnet *ifp)
1012 {
1013 lck_rw_lock_shared(&ifp->if_inet6data_lock);
1014 }
1015
1016 __private_extern__ void
1017 if_inet6data_lock_exclusive(struct ifnet *ifp)
1018 {
1019 lck_rw_lock_exclusive(&ifp->if_inet6data_lock);
1020 }
1021
1022 __private_extern__ void
1023 if_inet6data_lock_done(struct ifnet *ifp)
1024 {
1025 lck_rw_done(&ifp->if_inet6data_lock);
1026 }
1027 #endif
1028
1029 __private_extern__ void
1030 ifnet_head_lock_shared(void)
1031 {
1032 lck_rw_lock_shared(&ifnet_head_lock);
1033 }
1034
1035 __private_extern__ void
1036 ifnet_head_lock_exclusive(void)
1037 {
1038 lck_rw_lock_exclusive(&ifnet_head_lock);
1039 }
1040
1041 __private_extern__ void
1042 ifnet_head_done(void)
1043 {
1044 lck_rw_done(&ifnet_head_lock);
1045 }
1046
1047 __private_extern__ void
1048 ifnet_head_assert_exclusive(void)
1049 {
1050 LCK_RW_ASSERT(&ifnet_head_lock, LCK_RW_ASSERT_EXCLUSIVE);
1051 }
1052
1053 /*
1054 * dlil_ifp_protolist
1055 * - get the list of protocols attached to the interface, or just the number
1056 * of attached protocols
1057 * - if the number returned is greater than 'list_count', truncation occurred
1058 *
1059 * Note:
1060 * - caller must already be holding ifnet lock.
1061 */
1062 static u_int32_t
1063 dlil_ifp_protolist(struct ifnet *ifp, protocol_family_t *list,
1064 u_int32_t list_count)
1065 {
1066 u_int32_t count = 0;
1067 int i;
1068
1069 ifnet_lock_assert(ifp, IFNET_LCK_ASSERT_OWNED);
1070
1071 if (ifp->if_proto_hash == NULL)
1072 goto done;
1073
1074 for (i = 0; i < PROTO_HASH_SLOTS; i++) {
1075 struct if_proto *proto;
1076 SLIST_FOREACH(proto, &ifp->if_proto_hash[i], next_hash) {
1077 if (list != NULL && count < list_count) {
1078 list[count] = proto->protocol_family;
1079 }
1080 count++;
1081 }
1082 }
1083 done:
1084 return (count);
1085 }
1086
1087 __private_extern__ u_int32_t
1088 if_get_protolist(struct ifnet * ifp, u_int32_t *protolist, u_int32_t count)
1089 {
1090 ifnet_lock_shared(ifp);
1091 count = dlil_ifp_protolist(ifp, protolist, count);
1092 ifnet_lock_done(ifp);
1093 return (count);
1094 }
1095
1096 __private_extern__ void
1097 if_free_protolist(u_int32_t *list)
1098 {
1099 _FREE(list, M_TEMP);
1100 }
1101
1102 __private_extern__ void
1103 dlil_post_msg(struct ifnet *ifp, u_int32_t event_subclass,
1104 u_int32_t event_code, struct net_event_data *event_data,
1105 u_int32_t event_data_len)
1106 {
1107 struct net_event_data ev_data;
1108 struct kev_msg ev_msg;
1109
1110 bzero(&ev_msg, sizeof (ev_msg));
1111 bzero(&ev_data, sizeof (ev_data));
1112 /*
1113 * a net event always starts with a net_event_data structure
1114 * but the caller can generate a simple net event or
1115 * provide a longer event structure to post
1116 */
1117 ev_msg.vendor_code = KEV_VENDOR_APPLE;
1118 ev_msg.kev_class = KEV_NETWORK_CLASS;
1119 ev_msg.kev_subclass = event_subclass;
1120 ev_msg.event_code = event_code;
1121
1122 if (event_data == NULL) {
1123 event_data = &ev_data;
1124 event_data_len = sizeof (struct net_event_data);
1125 }
1126
1127 strlcpy(&event_data->if_name[0], ifp->if_name, IFNAMSIZ);
1128 event_data->if_family = ifp->if_family;
1129 event_data->if_unit = (u_int32_t)ifp->if_unit;
1130
1131 ev_msg.dv[0].data_length = event_data_len;
1132 ev_msg.dv[0].data_ptr = event_data;
1133 ev_msg.dv[1].data_length = 0;
1134
1135 /* Don't update interface generation for quality and RRC state changess */
1136 bool update_generation = (event_subclass != KEV_DL_SUBCLASS ||
1137 (event_code != KEV_DL_LINK_QUALITY_METRIC_CHANGED &&
1138 event_code != KEV_DL_RRC_STATE_CHANGED));
1139
1140 dlil_event_internal(ifp, &ev_msg, update_generation);
1141 }
1142
1143 __private_extern__ int
1144 dlil_alloc_local_stats(struct ifnet *ifp)
1145 {
1146 int ret = EINVAL;
1147 void *buf, *base, **pbuf;
1148
1149 if (ifp == NULL)
1150 goto end;
1151
1152 if (ifp->if_tcp_stat == NULL && ifp->if_udp_stat == NULL) {
1153 /* allocate tcpstat_local structure */
1154 buf = zalloc(dlif_tcpstat_zone);
1155 if (buf == NULL) {
1156 ret = ENOMEM;
1157 goto end;
1158 }
1159 bzero(buf, dlif_tcpstat_bufsize);
1160
1161 /* Get the 64-bit aligned base address for this object */
1162 base = (void *)P2ROUNDUP((intptr_t)buf + sizeof (u_int64_t),
1163 sizeof (u_int64_t));
1164 VERIFY(((intptr_t)base + dlif_tcpstat_size) <=
1165 ((intptr_t)buf + dlif_tcpstat_bufsize));
1166
1167 /*
1168 * Wind back a pointer size from the aligned base and
1169 * save the original address so we can free it later.
1170 */
1171 pbuf = (void **)((intptr_t)base - sizeof (void *));
1172 *pbuf = buf;
1173 ifp->if_tcp_stat = base;
1174
1175 /* allocate udpstat_local structure */
1176 buf = zalloc(dlif_udpstat_zone);
1177 if (buf == NULL) {
1178 ret = ENOMEM;
1179 goto end;
1180 }
1181 bzero(buf, dlif_udpstat_bufsize);
1182
1183 /* Get the 64-bit aligned base address for this object */
1184 base = (void *)P2ROUNDUP((intptr_t)buf + sizeof (u_int64_t),
1185 sizeof (u_int64_t));
1186 VERIFY(((intptr_t)base + dlif_udpstat_size) <=
1187 ((intptr_t)buf + dlif_udpstat_bufsize));
1188
1189 /*
1190 * Wind back a pointer size from the aligned base and
1191 * save the original address so we can free it later.
1192 */
1193 pbuf = (void **)((intptr_t)base - sizeof (void *));
1194 *pbuf = buf;
1195 ifp->if_udp_stat = base;
1196
1197 VERIFY(IS_P2ALIGNED(ifp->if_tcp_stat, sizeof (u_int64_t)) &&
1198 IS_P2ALIGNED(ifp->if_udp_stat, sizeof (u_int64_t)));
1199
1200 ret = 0;
1201 }
1202
1203 if (ifp->if_ipv4_stat == NULL) {
1204 MALLOC(ifp->if_ipv4_stat, struct if_tcp_ecn_stat *,
1205 sizeof (struct if_tcp_ecn_stat), M_TEMP, M_WAITOK|M_ZERO);
1206 if (ifp->if_ipv4_stat == NULL) {
1207 ret = ENOMEM;
1208 goto end;
1209 }
1210 }
1211
1212 if (ifp->if_ipv6_stat == NULL) {
1213 MALLOC(ifp->if_ipv6_stat, struct if_tcp_ecn_stat *,
1214 sizeof (struct if_tcp_ecn_stat), M_TEMP, M_WAITOK|M_ZERO);
1215 if (ifp->if_ipv6_stat == NULL) {
1216 ret = ENOMEM;
1217 goto end;
1218 }
1219 }
1220 end:
1221 if (ret != 0) {
1222 if (ifp->if_tcp_stat != NULL) {
1223 pbuf = (void **)
1224 ((intptr_t)ifp->if_tcp_stat - sizeof (void *));
1225 zfree(dlif_tcpstat_zone, *pbuf);
1226 ifp->if_tcp_stat = NULL;
1227 }
1228 if (ifp->if_udp_stat != NULL) {
1229 pbuf = (void **)
1230 ((intptr_t)ifp->if_udp_stat - sizeof (void *));
1231 zfree(dlif_udpstat_zone, *pbuf);
1232 ifp->if_udp_stat = NULL;
1233 }
1234 if (ifp->if_ipv4_stat != NULL) {
1235 FREE(ifp->if_ipv4_stat, M_TEMP);
1236 ifp->if_ipv4_stat = NULL;
1237 }
1238 if (ifp->if_ipv6_stat != NULL) {
1239 FREE(ifp->if_ipv6_stat, M_TEMP);
1240 ifp->if_ipv6_stat = NULL;
1241 }
1242 }
1243
1244 return (ret);
1245 }
1246
1247 static int
1248 dlil_create_input_thread(ifnet_t ifp, struct dlil_threading_info *inp)
1249 {
1250 thread_continue_t func;
1251 u_int32_t limit;
1252 int error;
1253
1254 /* NULL ifp indicates the main input thread, called at dlil_init time */
1255 if (ifp == NULL) {
1256 func = dlil_main_input_thread_func;
1257 VERIFY(inp == dlil_main_input_thread);
1258 (void) strlcat(inp->input_name,
1259 "main_input", DLIL_THREADNAME_LEN);
1260 } else if (net_rxpoll && (ifp->if_eflags & IFEF_RXPOLL)) {
1261 func = dlil_rxpoll_input_thread_func;
1262 VERIFY(inp != dlil_main_input_thread);
1263 (void) snprintf(inp->input_name, DLIL_THREADNAME_LEN,
1264 "%s_input_poll", if_name(ifp));
1265 } else {
1266 func = dlil_input_thread_func;
1267 VERIFY(inp != dlil_main_input_thread);
1268 (void) snprintf(inp->input_name, DLIL_THREADNAME_LEN,
1269 "%s_input", if_name(ifp));
1270 }
1271 VERIFY(inp->input_thr == THREAD_NULL);
1272
1273 inp->lck_grp = lck_grp_alloc_init(inp->input_name, dlil_grp_attributes);
1274 lck_mtx_init(&inp->input_lck, inp->lck_grp, dlil_lck_attributes);
1275
1276 inp->mode = IFNET_MODEL_INPUT_POLL_OFF;
1277 inp->ifp = ifp; /* NULL for main input thread */
1278
1279 net_timerclear(&inp->mode_holdtime);
1280 net_timerclear(&inp->mode_lasttime);
1281 net_timerclear(&inp->sample_holdtime);
1282 net_timerclear(&inp->sample_lasttime);
1283 net_timerclear(&inp->dbg_lasttime);
1284
1285 /*
1286 * For interfaces that support opportunistic polling, set the
1287 * low and high watermarks for outstanding inbound packets/bytes.
1288 * Also define freeze times for transitioning between modes
1289 * and updating the average.
1290 */
1291 if (ifp != NULL && net_rxpoll && (ifp->if_eflags & IFEF_RXPOLL)) {
1292 limit = MAX(if_rcvq_maxlen, IF_RCVQ_MINLEN);
1293 (void) dlil_rxpoll_set_params(ifp, NULL, FALSE);
1294 } else {
1295 limit = (u_int32_t)-1;
1296 }
1297
1298 _qinit(&inp->rcvq_pkts, Q_DROPTAIL, limit, QP_MBUF);
1299 if (inp == dlil_main_input_thread) {
1300 struct dlil_main_threading_info *inpm =
1301 (struct dlil_main_threading_info *)inp;
1302 _qinit(&inpm->lo_rcvq_pkts, Q_DROPTAIL, limit, QP_MBUF);
1303 }
1304
1305 error = kernel_thread_start(func, inp, &inp->input_thr);
1306 if (error == KERN_SUCCESS) {
1307 ml_thread_policy(inp->input_thr, MACHINE_GROUP,
1308 (MACHINE_NETWORK_GROUP|MACHINE_NETWORK_NETISR));
1309 /*
1310 * We create an affinity set so that the matching workloop
1311 * thread or the starter thread (for loopback) can be
1312 * scheduled on the same processor set as the input thread.
1313 */
1314 if (net_affinity) {
1315 struct thread *tp = inp->input_thr;
1316 u_int32_t tag;
1317 /*
1318 * Randomize to reduce the probability
1319 * of affinity tag namespace collision.
1320 */
1321 read_frandom(&tag, sizeof (tag));
1322 if (dlil_affinity_set(tp, tag) == KERN_SUCCESS) {
1323 thread_reference(tp);
1324 inp->tag = tag;
1325 inp->net_affinity = TRUE;
1326 }
1327 }
1328 } else if (inp == dlil_main_input_thread) {
1329 panic_plain("%s: couldn't create main input thread", __func__);
1330 /* NOTREACHED */
1331 } else {
1332 panic_plain("%s: couldn't create %s input thread", __func__,
1333 if_name(ifp));
1334 /* NOTREACHED */
1335 }
1336 OSAddAtomic(1, &cur_dlil_input_threads);
1337
1338 return (error);
1339 }
1340
1341 #if TEST_INPUT_THREAD_TERMINATION
1342 static int
1343 sysctl_input_thread_termination_spin SYSCTL_HANDLER_ARGS
1344 {
1345 #pragma unused(arg1, arg2)
1346 uint32_t i;
1347 int err;
1348
1349 i = if_input_thread_termination_spin;
1350
1351 err = sysctl_handle_int(oidp, &i, 0, req);
1352 if (err != 0 || req->newptr == USER_ADDR_NULL)
1353 return (err);
1354
1355 if (net_rxpoll == 0)
1356 return (ENXIO);
1357
1358 if_input_thread_termination_spin = i;
1359 return (err);
1360 }
1361 #endif /* TEST_INPUT_THREAD_TERMINATION */
1362
1363 static void
1364 dlil_clean_threading_info(struct dlil_threading_info *inp)
1365 {
1366 lck_mtx_destroy(&inp->input_lck, inp->lck_grp);
1367 lck_grp_free(inp->lck_grp);
1368
1369 inp->input_waiting = 0;
1370 inp->wtot = 0;
1371 bzero(inp->input_name, sizeof (inp->input_name));
1372 inp->ifp = NULL;
1373 VERIFY(qhead(&inp->rcvq_pkts) == NULL && qempty(&inp->rcvq_pkts));
1374 qlimit(&inp->rcvq_pkts) = 0;
1375 bzero(&inp->stats, sizeof (inp->stats));
1376
1377 VERIFY(!inp->net_affinity);
1378 inp->input_thr = THREAD_NULL;
1379 VERIFY(inp->wloop_thr == THREAD_NULL);
1380 VERIFY(inp->poll_thr == THREAD_NULL);
1381 VERIFY(inp->tag == 0);
1382
1383 inp->mode = IFNET_MODEL_INPUT_POLL_OFF;
1384 bzero(&inp->tstats, sizeof (inp->tstats));
1385 bzero(&inp->pstats, sizeof (inp->pstats));
1386 bzero(&inp->sstats, sizeof (inp->sstats));
1387
1388 net_timerclear(&inp->mode_holdtime);
1389 net_timerclear(&inp->mode_lasttime);
1390 net_timerclear(&inp->sample_holdtime);
1391 net_timerclear(&inp->sample_lasttime);
1392 net_timerclear(&inp->dbg_lasttime);
1393
1394 #if IFNET_INPUT_SANITY_CHK
1395 inp->input_mbuf_cnt = 0;
1396 #endif /* IFNET_INPUT_SANITY_CHK */
1397 }
1398
1399 static void
1400 dlil_terminate_input_thread(struct dlil_threading_info *inp)
1401 {
1402 struct ifnet *ifp = inp->ifp;
1403
1404 VERIFY(current_thread() == inp->input_thr);
1405 VERIFY(inp != dlil_main_input_thread);
1406
1407 OSAddAtomic(-1, &cur_dlil_input_threads);
1408
1409 #if TEST_INPUT_THREAD_TERMINATION
1410 { /* do something useless that won't get optimized away */
1411 uint32_t v = 1;
1412 for (uint32_t i = 0;
1413 i < if_input_thread_termination_spin;
1414 i++) {
1415 v = (i + 1) * v;
1416 }
1417 printf("the value is %d\n", v);
1418 }
1419 #endif /* TEST_INPUT_THREAD_TERMINATION */
1420
1421 lck_mtx_lock_spin(&inp->input_lck);
1422 VERIFY((inp->input_waiting & DLIL_INPUT_TERMINATE) != 0);
1423 inp->input_waiting |= DLIL_INPUT_TERMINATE_COMPLETE;
1424 wakeup_one((caddr_t)&inp->input_waiting);
1425 lck_mtx_unlock(&inp->input_lck);
1426
1427 /* for the extra refcnt from kernel_thread_start() */
1428 thread_deallocate(current_thread());
1429
1430 if (dlil_verbose) {
1431 printf("%s: input thread terminated\n",
1432 if_name(ifp));
1433 }
1434
1435 /* this is the end */
1436 thread_terminate(current_thread());
1437 /* NOTREACHED */
1438 }
1439
1440 static kern_return_t
1441 dlil_affinity_set(struct thread *tp, u_int32_t tag)
1442 {
1443 thread_affinity_policy_data_t policy;
1444
1445 bzero(&policy, sizeof (policy));
1446 policy.affinity_tag = tag;
1447 return (thread_policy_set(tp, THREAD_AFFINITY_POLICY,
1448 (thread_policy_t)&policy, THREAD_AFFINITY_POLICY_COUNT));
1449 }
1450
1451 void
1452 dlil_init(void)
1453 {
1454 thread_t thread = THREAD_NULL;
1455
1456 /*
1457 * The following fields must be 64-bit aligned for atomic operations.
1458 */
1459 IF_DATA_REQUIRE_ALIGNED_64(ifi_ipackets);
1460 IF_DATA_REQUIRE_ALIGNED_64(ifi_ierrors);
1461 IF_DATA_REQUIRE_ALIGNED_64(ifi_opackets);
1462 IF_DATA_REQUIRE_ALIGNED_64(ifi_oerrors);
1463 IF_DATA_REQUIRE_ALIGNED_64(ifi_collisions);
1464 IF_DATA_REQUIRE_ALIGNED_64(ifi_ibytes);
1465 IF_DATA_REQUIRE_ALIGNED_64(ifi_obytes);
1466 IF_DATA_REQUIRE_ALIGNED_64(ifi_imcasts);
1467 IF_DATA_REQUIRE_ALIGNED_64(ifi_omcasts);
1468 IF_DATA_REQUIRE_ALIGNED_64(ifi_iqdrops);
1469 IF_DATA_REQUIRE_ALIGNED_64(ifi_noproto);
1470 IF_DATA_REQUIRE_ALIGNED_64(ifi_alignerrs);
1471 IF_DATA_REQUIRE_ALIGNED_64(ifi_dt_bytes);
1472 IF_DATA_REQUIRE_ALIGNED_64(ifi_fpackets);
1473 IF_DATA_REQUIRE_ALIGNED_64(ifi_fbytes);
1474
1475 IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_ipackets);
1476 IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_ierrors);
1477 IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_opackets);
1478 IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_oerrors);
1479 IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_collisions);
1480 IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_ibytes);
1481 IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_obytes);
1482 IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_imcasts);
1483 IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_omcasts);
1484 IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_iqdrops);
1485 IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_noproto);
1486 IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_alignerrs);
1487 IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_dt_bytes);
1488 IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_fpackets);
1489 IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_fbytes);
1490
1491 /*
1492 * These IF_HWASSIST_ flags must be equal to their IFNET_* counterparts.
1493 */
1494 _CASSERT(IF_HWASSIST_CSUM_IP == IFNET_CSUM_IP);
1495 _CASSERT(IF_HWASSIST_CSUM_TCP == IFNET_CSUM_TCP);
1496 _CASSERT(IF_HWASSIST_CSUM_UDP == IFNET_CSUM_UDP);
1497 _CASSERT(IF_HWASSIST_CSUM_IP_FRAGS == IFNET_CSUM_FRAGMENT);
1498 _CASSERT(IF_HWASSIST_CSUM_FRAGMENT == IFNET_IP_FRAGMENT);
1499 _CASSERT(IF_HWASSIST_CSUM_TCPIPV6 == IFNET_CSUM_TCPIPV6);
1500 _CASSERT(IF_HWASSIST_CSUM_UDPIPV6 == IFNET_CSUM_UDPIPV6);
1501 _CASSERT(IF_HWASSIST_CSUM_FRAGMENT_IPV6 == IFNET_IPV6_FRAGMENT);
1502 _CASSERT(IF_HWASSIST_CSUM_PARTIAL == IFNET_CSUM_PARTIAL);
1503 _CASSERT(IF_HWASSIST_CSUM_ZERO_INVERT == IFNET_CSUM_ZERO_INVERT);
1504 _CASSERT(IF_HWASSIST_VLAN_TAGGING == IFNET_VLAN_TAGGING);
1505 _CASSERT(IF_HWASSIST_VLAN_MTU == IFNET_VLAN_MTU);
1506 _CASSERT(IF_HWASSIST_TSO_V4 == IFNET_TSO_IPV4);
1507 _CASSERT(IF_HWASSIST_TSO_V6 == IFNET_TSO_IPV6);
1508
1509 /*
1510 * ... as well as the mbuf checksum flags counterparts.
1511 */
1512 _CASSERT(CSUM_IP == IF_HWASSIST_CSUM_IP);
1513 _CASSERT(CSUM_TCP == IF_HWASSIST_CSUM_TCP);
1514 _CASSERT(CSUM_UDP == IF_HWASSIST_CSUM_UDP);
1515 _CASSERT(CSUM_IP_FRAGS == IF_HWASSIST_CSUM_IP_FRAGS);
1516 _CASSERT(CSUM_FRAGMENT == IF_HWASSIST_CSUM_FRAGMENT);
1517 _CASSERT(CSUM_TCPIPV6 == IF_HWASSIST_CSUM_TCPIPV6);
1518 _CASSERT(CSUM_UDPIPV6 == IF_HWASSIST_CSUM_UDPIPV6);
1519 _CASSERT(CSUM_FRAGMENT_IPV6 == IF_HWASSIST_CSUM_FRAGMENT_IPV6);
1520 _CASSERT(CSUM_PARTIAL == IF_HWASSIST_CSUM_PARTIAL);
1521 _CASSERT(CSUM_ZERO_INVERT == IF_HWASSIST_CSUM_ZERO_INVERT);
1522 _CASSERT(CSUM_VLAN_TAG_VALID == IF_HWASSIST_VLAN_TAGGING);
1523
1524 /*
1525 * Make sure we have at least IF_LLREACH_MAXLEN in the llreach info.
1526 */
1527 _CASSERT(IF_LLREACH_MAXLEN <= IF_LLREACHINFO_ADDRLEN);
1528 _CASSERT(IFNET_LLREACHINFO_ADDRLEN == IF_LLREACHINFO_ADDRLEN);
1529
1530 _CASSERT(IFRLOGF_DLIL == IFNET_LOGF_DLIL);
1531 _CASSERT(IFRLOGF_FAMILY == IFNET_LOGF_FAMILY);
1532 _CASSERT(IFRLOGF_DRIVER == IFNET_LOGF_DRIVER);
1533 _CASSERT(IFRLOGF_FIRMWARE == IFNET_LOGF_FIRMWARE);
1534
1535 _CASSERT(IFRLOGCAT_CONNECTIVITY == IFNET_LOGCAT_CONNECTIVITY);
1536 _CASSERT(IFRLOGCAT_QUALITY == IFNET_LOGCAT_QUALITY);
1537 _CASSERT(IFRLOGCAT_PERFORMANCE == IFNET_LOGCAT_PERFORMANCE);
1538
1539 _CASSERT(IFRTYPE_FAMILY_ANY == IFNET_FAMILY_ANY);
1540 _CASSERT(IFRTYPE_FAMILY_LOOPBACK == IFNET_FAMILY_LOOPBACK);
1541 _CASSERT(IFRTYPE_FAMILY_ETHERNET == IFNET_FAMILY_ETHERNET);
1542 _CASSERT(IFRTYPE_FAMILY_SLIP == IFNET_FAMILY_SLIP);
1543 _CASSERT(IFRTYPE_FAMILY_TUN == IFNET_FAMILY_TUN);
1544 _CASSERT(IFRTYPE_FAMILY_VLAN == IFNET_FAMILY_VLAN);
1545 _CASSERT(IFRTYPE_FAMILY_PPP == IFNET_FAMILY_PPP);
1546 _CASSERT(IFRTYPE_FAMILY_PVC == IFNET_FAMILY_PVC);
1547 _CASSERT(IFRTYPE_FAMILY_DISC == IFNET_FAMILY_DISC);
1548 _CASSERT(IFRTYPE_FAMILY_MDECAP == IFNET_FAMILY_MDECAP);
1549 _CASSERT(IFRTYPE_FAMILY_GIF == IFNET_FAMILY_GIF);
1550 _CASSERT(IFRTYPE_FAMILY_FAITH == IFNET_FAMILY_FAITH);
1551 _CASSERT(IFRTYPE_FAMILY_STF == IFNET_FAMILY_STF);
1552 _CASSERT(IFRTYPE_FAMILY_FIREWIRE == IFNET_FAMILY_FIREWIRE);
1553 _CASSERT(IFRTYPE_FAMILY_BOND == IFNET_FAMILY_BOND);
1554 _CASSERT(IFRTYPE_FAMILY_CELLULAR == IFNET_FAMILY_CELLULAR);
1555
1556 _CASSERT(IFRTYPE_SUBFAMILY_ANY == IFNET_SUBFAMILY_ANY);
1557 _CASSERT(IFRTYPE_SUBFAMILY_USB == IFNET_SUBFAMILY_USB);
1558 _CASSERT(IFRTYPE_SUBFAMILY_BLUETOOTH == IFNET_SUBFAMILY_BLUETOOTH);
1559 _CASSERT(IFRTYPE_SUBFAMILY_WIFI == IFNET_SUBFAMILY_WIFI);
1560 _CASSERT(IFRTYPE_SUBFAMILY_THUNDERBOLT == IFNET_SUBFAMILY_THUNDERBOLT);
1561 _CASSERT(IFRTYPE_SUBFAMILY_RESERVED == IFNET_SUBFAMILY_RESERVED);
1562 _CASSERT(IFRTYPE_SUBFAMILY_INTCOPROC == IFNET_SUBFAMILY_INTCOPROC);
1563
1564 _CASSERT(DLIL_MODIDLEN == IFNET_MODIDLEN);
1565 _CASSERT(DLIL_MODARGLEN == IFNET_MODARGLEN);
1566
1567 PE_parse_boot_argn("net_affinity", &net_affinity,
1568 sizeof (net_affinity));
1569
1570 PE_parse_boot_argn("net_rxpoll", &net_rxpoll, sizeof (net_rxpoll));
1571
1572 PE_parse_boot_argn("net_rtref", &net_rtref, sizeof (net_rtref));
1573
1574 PE_parse_boot_argn("ifnet_debug", &ifnet_debug, sizeof (ifnet_debug));
1575
1576 dlif_size = (ifnet_debug == 0) ? sizeof (struct dlil_ifnet) :
1577 sizeof (struct dlil_ifnet_dbg);
1578 /* Enforce 64-bit alignment for dlil_ifnet structure */
1579 dlif_bufsize = dlif_size + sizeof (void *) + sizeof (u_int64_t);
1580 dlif_bufsize = P2ROUNDUP(dlif_bufsize, sizeof (u_int64_t));
1581 dlif_zone = zinit(dlif_bufsize, DLIF_ZONE_MAX * dlif_bufsize,
1582 0, DLIF_ZONE_NAME);
1583 if (dlif_zone == NULL) {
1584 panic_plain("%s: failed allocating %s", __func__,
1585 DLIF_ZONE_NAME);
1586 /* NOTREACHED */
1587 }
1588 zone_change(dlif_zone, Z_EXPAND, TRUE);
1589 zone_change(dlif_zone, Z_CALLERACCT, FALSE);
1590
1591 dlif_filt_size = sizeof (struct ifnet_filter);
1592 dlif_filt_zone = zinit(dlif_filt_size,
1593 DLIF_FILT_ZONE_MAX * dlif_filt_size, 0, DLIF_FILT_ZONE_NAME);
1594 if (dlif_filt_zone == NULL) {
1595 panic_plain("%s: failed allocating %s", __func__,
1596 DLIF_FILT_ZONE_NAME);
1597 /* NOTREACHED */
1598 }
1599 zone_change(dlif_filt_zone, Z_EXPAND, TRUE);
1600 zone_change(dlif_filt_zone, Z_CALLERACCT, FALSE);
1601
1602 dlif_phash_size = sizeof (struct proto_hash_entry) * PROTO_HASH_SLOTS;
1603 dlif_phash_zone = zinit(dlif_phash_size,
1604 DLIF_PHASH_ZONE_MAX * dlif_phash_size, 0, DLIF_PHASH_ZONE_NAME);
1605 if (dlif_phash_zone == NULL) {
1606 panic_plain("%s: failed allocating %s", __func__,
1607 DLIF_PHASH_ZONE_NAME);
1608 /* NOTREACHED */
1609 }
1610 zone_change(dlif_phash_zone, Z_EXPAND, TRUE);
1611 zone_change(dlif_phash_zone, Z_CALLERACCT, FALSE);
1612
1613 dlif_proto_size = sizeof (struct if_proto);
1614 dlif_proto_zone = zinit(dlif_proto_size,
1615 DLIF_PROTO_ZONE_MAX * dlif_proto_size, 0, DLIF_PROTO_ZONE_NAME);
1616 if (dlif_proto_zone == NULL) {
1617 panic_plain("%s: failed allocating %s", __func__,
1618 DLIF_PROTO_ZONE_NAME);
1619 /* NOTREACHED */
1620 }
1621 zone_change(dlif_proto_zone, Z_EXPAND, TRUE);
1622 zone_change(dlif_proto_zone, Z_CALLERACCT, FALSE);
1623
1624 dlif_tcpstat_size = sizeof (struct tcpstat_local);
1625 /* Enforce 64-bit alignment for tcpstat_local structure */
1626 dlif_tcpstat_bufsize =
1627 dlif_tcpstat_size + sizeof (void *) + sizeof (u_int64_t);
1628 dlif_tcpstat_bufsize =
1629 P2ROUNDUP(dlif_tcpstat_bufsize, sizeof (u_int64_t));
1630 dlif_tcpstat_zone = zinit(dlif_tcpstat_bufsize,
1631 DLIF_TCPSTAT_ZONE_MAX * dlif_tcpstat_bufsize, 0,
1632 DLIF_TCPSTAT_ZONE_NAME);
1633 if (dlif_tcpstat_zone == NULL) {
1634 panic_plain("%s: failed allocating %s", __func__,
1635 DLIF_TCPSTAT_ZONE_NAME);
1636 /* NOTREACHED */
1637 }
1638 zone_change(dlif_tcpstat_zone, Z_EXPAND, TRUE);
1639 zone_change(dlif_tcpstat_zone, Z_CALLERACCT, FALSE);
1640
1641 dlif_udpstat_size = sizeof (struct udpstat_local);
1642 /* Enforce 64-bit alignment for udpstat_local structure */
1643 dlif_udpstat_bufsize =
1644 dlif_udpstat_size + sizeof (void *) + sizeof (u_int64_t);
1645 dlif_udpstat_bufsize =
1646 P2ROUNDUP(dlif_udpstat_bufsize, sizeof (u_int64_t));
1647 dlif_udpstat_zone = zinit(dlif_udpstat_bufsize,
1648 DLIF_TCPSTAT_ZONE_MAX * dlif_udpstat_bufsize, 0,
1649 DLIF_UDPSTAT_ZONE_NAME);
1650 if (dlif_udpstat_zone == NULL) {
1651 panic_plain("%s: failed allocating %s", __func__,
1652 DLIF_UDPSTAT_ZONE_NAME);
1653 /* NOTREACHED */
1654 }
1655 zone_change(dlif_udpstat_zone, Z_EXPAND, TRUE);
1656 zone_change(dlif_udpstat_zone, Z_CALLERACCT, FALSE);
1657
1658 ifnet_llreach_init();
1659 eventhandler_lists_ctxt_init(&ifnet_evhdlr_ctxt);
1660
1661 TAILQ_INIT(&dlil_ifnet_head);
1662 TAILQ_INIT(&ifnet_head);
1663 TAILQ_INIT(&ifnet_detaching_head);
1664 TAILQ_INIT(&ifnet_ordered_head);
1665
1666 /* Setup the lock groups we will use */
1667 dlil_grp_attributes = lck_grp_attr_alloc_init();
1668
1669 dlil_lock_group = lck_grp_alloc_init("DLIL internal locks",
1670 dlil_grp_attributes);
1671 ifnet_lock_group = lck_grp_alloc_init("ifnet locks",
1672 dlil_grp_attributes);
1673 ifnet_head_lock_group = lck_grp_alloc_init("ifnet head lock",
1674 dlil_grp_attributes);
1675 ifnet_rcv_lock_group = lck_grp_alloc_init("ifnet rcv locks",
1676 dlil_grp_attributes);
1677 ifnet_snd_lock_group = lck_grp_alloc_init("ifnet snd locks",
1678 dlil_grp_attributes);
1679
1680 /* Setup the lock attributes we will use */
1681 dlil_lck_attributes = lck_attr_alloc_init();
1682
1683 ifnet_lock_attr = lck_attr_alloc_init();
1684
1685 lck_rw_init(&ifnet_head_lock, ifnet_head_lock_group,
1686 dlil_lck_attributes);
1687 lck_mtx_init(&dlil_ifnet_lock, dlil_lock_group, dlil_lck_attributes);
1688
1689 /* Setup interface flow control related items */
1690 lck_mtx_init(&ifnet_fc_lock, dlil_lock_group, dlil_lck_attributes);
1691
1692 ifnet_fc_zone_size = sizeof (struct ifnet_fc_entry);
1693 ifnet_fc_zone = zinit(ifnet_fc_zone_size,
1694 IFNET_FC_ZONE_MAX * ifnet_fc_zone_size, 0, IFNET_FC_ZONE_NAME);
1695 if (ifnet_fc_zone == NULL) {
1696 panic_plain("%s: failed allocating %s", __func__,
1697 IFNET_FC_ZONE_NAME);
1698 /* NOTREACHED */
1699 }
1700 zone_change(ifnet_fc_zone, Z_EXPAND, TRUE);
1701 zone_change(ifnet_fc_zone, Z_CALLERACCT, FALSE);
1702
1703 /* Initialize interface address subsystem */
1704 ifa_init();
1705
1706 #if PF
1707 /* Initialize the packet filter */
1708 pfinit();
1709 #endif /* PF */
1710
1711 /* Initialize queue algorithms */
1712 classq_init();
1713
1714 /* Initialize packet schedulers */
1715 pktsched_init();
1716
1717 /* Initialize flow advisory subsystem */
1718 flowadv_init();
1719
1720 /* Initialize the pktap virtual interface */
1721 pktap_init();
1722
1723 /* Initialize the service class to dscp map */
1724 net_qos_map_init();
1725
1726 /* Initialize the interface port list */
1727 if_ports_used_init();
1728
1729 /* Initialize the interface low power mode event handler */
1730 if_low_power_evhdlr_init();
1731
1732 #if DEBUG || DEVELOPMENT
1733 /* Run self-tests */
1734 dlil_verify_sum16();
1735 #endif /* DEBUG || DEVELOPMENT */
1736
1737 /* Initialize link layer table */
1738 lltable_glbl_init();
1739
1740 /*
1741 * Create and start up the main DLIL input thread and the interface
1742 * detacher threads once everything is initialized.
1743 */
1744 dlil_create_input_thread(NULL, dlil_main_input_thread);
1745
1746 if (kernel_thread_start(ifnet_detacher_thread_func,
1747 NULL, &thread) != KERN_SUCCESS) {
1748 panic_plain("%s: couldn't create detacher thread", __func__);
1749 /* NOTREACHED */
1750 }
1751 thread_deallocate(thread);
1752
1753 }
1754
1755 static void
1756 if_flt_monitor_busy(struct ifnet *ifp)
1757 {
1758 LCK_MTX_ASSERT(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED);
1759
1760 ++ifp->if_flt_busy;
1761 VERIFY(ifp->if_flt_busy != 0);
1762 }
1763
1764 static void
1765 if_flt_monitor_unbusy(struct ifnet *ifp)
1766 {
1767 if_flt_monitor_leave(ifp);
1768 }
1769
1770 static void
1771 if_flt_monitor_enter(struct ifnet *ifp)
1772 {
1773 LCK_MTX_ASSERT(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED);
1774
1775 while (ifp->if_flt_busy) {
1776 ++ifp->if_flt_waiters;
1777 (void) msleep(&ifp->if_flt_head, &ifp->if_flt_lock,
1778 (PZERO - 1), "if_flt_monitor", NULL);
1779 }
1780 if_flt_monitor_busy(ifp);
1781 }
1782
1783 static void
1784 if_flt_monitor_leave(struct ifnet *ifp)
1785 {
1786 LCK_MTX_ASSERT(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED);
1787
1788 VERIFY(ifp->if_flt_busy != 0);
1789 --ifp->if_flt_busy;
1790
1791 if (ifp->if_flt_busy == 0 && ifp->if_flt_waiters > 0) {
1792 ifp->if_flt_waiters = 0;
1793 wakeup(&ifp->if_flt_head);
1794 }
1795 }
1796
1797 __private_extern__ int
1798 dlil_attach_filter(struct ifnet *ifp, const struct iff_filter *if_filter,
1799 interface_filter_t *filter_ref, u_int32_t flags)
1800 {
1801 int retval = 0;
1802 struct ifnet_filter *filter = NULL;
1803
1804 ifnet_head_lock_shared();
1805 /* Check that the interface is in the global list */
1806 if (!ifnet_lookup(ifp)) {
1807 retval = ENXIO;
1808 goto done;
1809 }
1810
1811 filter = zalloc(dlif_filt_zone);
1812 if (filter == NULL) {
1813 retval = ENOMEM;
1814 goto done;
1815 }
1816 bzero(filter, dlif_filt_size);
1817
1818 /* refcnt held above during lookup */
1819 filter->filt_flags = flags;
1820 filter->filt_ifp = ifp;
1821 filter->filt_cookie = if_filter->iff_cookie;
1822 filter->filt_name = if_filter->iff_name;
1823 filter->filt_protocol = if_filter->iff_protocol;
1824 /*
1825 * Do not install filter callbacks for internal coproc interface
1826 */
1827 if (!IFNET_IS_INTCOPROC(ifp)) {
1828 filter->filt_input = if_filter->iff_input;
1829 filter->filt_output = if_filter->iff_output;
1830 filter->filt_event = if_filter->iff_event;
1831 filter->filt_ioctl = if_filter->iff_ioctl;
1832 }
1833 filter->filt_detached = if_filter->iff_detached;
1834
1835 lck_mtx_lock(&ifp->if_flt_lock);
1836 if_flt_monitor_enter(ifp);
1837
1838 LCK_MTX_ASSERT(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED);
1839 TAILQ_INSERT_TAIL(&ifp->if_flt_head, filter, filt_next);
1840
1841 if_flt_monitor_leave(ifp);
1842 lck_mtx_unlock(&ifp->if_flt_lock);
1843
1844 *filter_ref = filter;
1845
1846 /*
1847 * Bump filter count and route_generation ID to let TCP
1848 * know it shouldn't do TSO on this connection
1849 */
1850 if ((filter->filt_flags & DLIL_IFF_TSO) == 0) {
1851 OSAddAtomic(1, &dlil_filter_disable_tso_count);
1852 routegenid_update();
1853 }
1854 OSIncrementAtomic64(&net_api_stats.nas_iflt_attach_count);
1855 INC_ATOMIC_INT64_LIM(net_api_stats.nas_iflt_attach_total);
1856 if ((filter->filt_flags & DLIL_IFF_INTERNAL)) {
1857 INC_ATOMIC_INT64_LIM(net_api_stats.nas_iflt_attach_os_total);
1858 }
1859 if (dlil_verbose) {
1860 printf("%s: %s filter attached\n", if_name(ifp),
1861 if_filter->iff_name);
1862 }
1863 done:
1864 ifnet_head_done();
1865 if (retval != 0 && ifp != NULL) {
1866 DLIL_PRINTF("%s: failed to attach %s (err=%d)\n",
1867 if_name(ifp), if_filter->iff_name, retval);
1868 }
1869 if (retval != 0 && filter != NULL)
1870 zfree(dlif_filt_zone, filter);
1871
1872 return (retval);
1873 }
1874
1875 static int
1876 dlil_detach_filter_internal(interface_filter_t filter, int detached)
1877 {
1878 int retval = 0;
1879
1880 if (detached == 0) {
1881 ifnet_t ifp = NULL;
1882
1883 ifnet_head_lock_shared();
1884 TAILQ_FOREACH(ifp, &ifnet_head, if_link) {
1885 interface_filter_t entry = NULL;
1886
1887 lck_mtx_lock(&ifp->if_flt_lock);
1888 TAILQ_FOREACH(entry, &ifp->if_flt_head, filt_next) {
1889 if (entry != filter || entry->filt_skip)
1890 continue;
1891 /*
1892 * We've found a match; since it's possible
1893 * that the thread gets blocked in the monitor,
1894 * we do the lock dance. Interface should
1895 * not be detached since we still have a use
1896 * count held during filter attach.
1897 */
1898 entry->filt_skip = 1; /* skip input/output */
1899 lck_mtx_unlock(&ifp->if_flt_lock);
1900 ifnet_head_done();
1901
1902 lck_mtx_lock(&ifp->if_flt_lock);
1903 if_flt_monitor_enter(ifp);
1904 LCK_MTX_ASSERT(&ifp->if_flt_lock,
1905 LCK_MTX_ASSERT_OWNED);
1906
1907 /* Remove the filter from the list */
1908 TAILQ_REMOVE(&ifp->if_flt_head, filter,
1909 filt_next);
1910
1911 if_flt_monitor_leave(ifp);
1912 lck_mtx_unlock(&ifp->if_flt_lock);
1913 if (dlil_verbose) {
1914 printf("%s: %s filter detached\n",
1915 if_name(ifp), filter->filt_name);
1916 }
1917 goto destroy;
1918 }
1919 lck_mtx_unlock(&ifp->if_flt_lock);
1920 }
1921 ifnet_head_done();
1922
1923 /* filter parameter is not a valid filter ref */
1924 retval = EINVAL;
1925 goto done;
1926 }
1927
1928 if (dlil_verbose)
1929 printf("%s filter detached\n", filter->filt_name);
1930
1931 destroy:
1932
1933 /* Call the detached function if there is one */
1934 if (filter->filt_detached)
1935 filter->filt_detached(filter->filt_cookie, filter->filt_ifp);
1936
1937 /*
1938 * Decrease filter count and route_generation ID to let TCP
1939 * know it should reevalute doing TSO or not
1940 */
1941 if ((filter->filt_flags & DLIL_IFF_TSO) == 0) {
1942 OSAddAtomic(-1, &dlil_filter_disable_tso_count);
1943 routegenid_update();
1944 }
1945
1946 VERIFY(OSDecrementAtomic64(&net_api_stats.nas_iflt_attach_count) > 0);
1947
1948 /* Free the filter */
1949 zfree(dlif_filt_zone, filter);
1950 filter = NULL;
1951 done:
1952 if (retval != 0 && filter != NULL) {
1953 DLIL_PRINTF("failed to detach %s filter (err=%d)\n",
1954 filter->filt_name, retval);
1955 }
1956
1957 return (retval);
1958 }
1959
1960 __private_extern__ void
1961 dlil_detach_filter(interface_filter_t filter)
1962 {
1963 if (filter == NULL)
1964 return;
1965 dlil_detach_filter_internal(filter, 0);
1966 }
1967
1968 /*
1969 * Main input thread:
1970 *
1971 * a) handles all inbound packets for lo0
1972 * b) handles all inbound packets for interfaces with no dedicated
1973 * input thread (e.g. anything but Ethernet/PDP or those that support
1974 * opportunistic polling.)
1975 * c) protocol registrations
1976 * d) packet injections
1977 */
1978 __attribute__((noreturn))
1979 static void
1980 dlil_main_input_thread_func(void *v, wait_result_t w)
1981 {
1982 #pragma unused(w)
1983 struct dlil_main_threading_info *inpm = v;
1984 struct dlil_threading_info *inp = v;
1985
1986 VERIFY(inp == dlil_main_input_thread);
1987 VERIFY(inp->ifp == NULL);
1988 VERIFY(inp->mode == IFNET_MODEL_INPUT_POLL_OFF);
1989
1990 while (1) {
1991 struct mbuf *m = NULL, *m_loop = NULL;
1992 u_int32_t m_cnt, m_cnt_loop;
1993 boolean_t proto_req;
1994
1995 lck_mtx_lock_spin(&inp->input_lck);
1996
1997 /* Wait until there is work to be done */
1998 while (!(inp->input_waiting & ~DLIL_INPUT_RUNNING)) {
1999 inp->input_waiting &= ~DLIL_INPUT_RUNNING;
2000 (void) msleep(&inp->input_waiting, &inp->input_lck,
2001 (PZERO - 1) | PSPIN, inp->input_name, NULL);
2002 }
2003
2004 inp->input_waiting |= DLIL_INPUT_RUNNING;
2005 inp->input_waiting &= ~DLIL_INPUT_WAITING;
2006
2007 /* Main input thread cannot be terminated */
2008 VERIFY(!(inp->input_waiting & DLIL_INPUT_TERMINATE));
2009
2010 proto_req = (inp->input_waiting &
2011 (DLIL_PROTO_WAITING | DLIL_PROTO_REGISTER));
2012
2013 /* Packets for non-dedicated interfaces other than lo0 */
2014 m_cnt = qlen(&inp->rcvq_pkts);
2015 m = _getq_all(&inp->rcvq_pkts, NULL, NULL, NULL);
2016
2017 /* Packets exclusive to lo0 */
2018 m_cnt_loop = qlen(&inpm->lo_rcvq_pkts);
2019 m_loop = _getq_all(&inpm->lo_rcvq_pkts, NULL, NULL, NULL);
2020
2021 inp->wtot = 0;
2022
2023 lck_mtx_unlock(&inp->input_lck);
2024
2025 /*
2026 * NOTE warning %%% attention !!!!
2027 * We should think about putting some thread starvation
2028 * safeguards if we deal with long chains of packets.
2029 */
2030 if (m_loop != NULL)
2031 dlil_input_packet_list_extended(lo_ifp, m_loop,
2032 m_cnt_loop, inp->mode);
2033
2034 if (m != NULL)
2035 dlil_input_packet_list_extended(NULL, m,
2036 m_cnt, inp->mode);
2037
2038 if (proto_req)
2039 proto_input_run();
2040 }
2041
2042 /* NOTREACHED */
2043 VERIFY(0); /* we should never get here */
2044 }
2045
2046 /*
2047 * Input thread for interfaces with legacy input model.
2048 */
2049 static void
2050 dlil_input_thread_func(void *v, wait_result_t w)
2051 {
2052 #pragma unused(w)
2053 char thread_name[MAXTHREADNAMESIZE];
2054 struct dlil_threading_info *inp = v;
2055 struct ifnet *ifp = inp->ifp;
2056
2057 /* Construct the name for this thread, and then apply it. */
2058 bzero(thread_name, sizeof(thread_name));
2059 snprintf(thread_name, sizeof(thread_name), "dlil_input_%s", ifp->if_xname);
2060 thread_set_thread_name(inp->input_thr, thread_name);
2061
2062 VERIFY(inp != dlil_main_input_thread);
2063 VERIFY(ifp != NULL);
2064 VERIFY(!(ifp->if_eflags & IFEF_RXPOLL) || !net_rxpoll);
2065 VERIFY(inp->mode == IFNET_MODEL_INPUT_POLL_OFF);
2066
2067 while (1) {
2068 struct mbuf *m = NULL;
2069 u_int32_t m_cnt;
2070
2071 lck_mtx_lock_spin(&inp->input_lck);
2072
2073 /* Wait until there is work to be done */
2074 while (!(inp->input_waiting & ~DLIL_INPUT_RUNNING)) {
2075 inp->input_waiting &= ~DLIL_INPUT_RUNNING;
2076 (void) msleep(&inp->input_waiting, &inp->input_lck,
2077 (PZERO - 1) | PSPIN, inp->input_name, NULL);
2078 }
2079
2080 inp->input_waiting |= DLIL_INPUT_RUNNING;
2081 inp->input_waiting &= ~DLIL_INPUT_WAITING;
2082
2083 /*
2084 * Protocol registration and injection must always use
2085 * the main input thread; in theory the latter can utilize
2086 * the corresponding input thread where the packet arrived
2087 * on, but that requires our knowing the interface in advance
2088 * (and the benefits might not worth the trouble.)
2089 */
2090 VERIFY(!(inp->input_waiting &
2091 (DLIL_PROTO_WAITING|DLIL_PROTO_REGISTER)));
2092
2093 /* Packets for this interface */
2094 m_cnt = qlen(&inp->rcvq_pkts);
2095 m = _getq_all(&inp->rcvq_pkts, NULL, NULL, NULL);
2096
2097 if (inp->input_waiting & DLIL_INPUT_TERMINATE) {
2098 lck_mtx_unlock(&inp->input_lck);
2099
2100 /* Free up pending packets */
2101 if (m != NULL)
2102 mbuf_freem_list(m);
2103
2104 dlil_terminate_input_thread(inp);
2105 /* NOTREACHED */
2106 return;
2107 }
2108
2109 inp->wtot = 0;
2110
2111 dlil_input_stats_sync(ifp, inp);
2112
2113 lck_mtx_unlock(&inp->input_lck);
2114
2115 /*
2116 * NOTE warning %%% attention !!!!
2117 * We should think about putting some thread starvation
2118 * safeguards if we deal with long chains of packets.
2119 */
2120 if (m != NULL)
2121 dlil_input_packet_list_extended(NULL, m,
2122 m_cnt, inp->mode);
2123 }
2124
2125 /* NOTREACHED */
2126 VERIFY(0); /* we should never get here */
2127 }
2128
2129 /*
2130 * Input thread for interfaces with opportunistic polling input model.
2131 */
2132 static void
2133 dlil_rxpoll_input_thread_func(void *v, wait_result_t w)
2134 {
2135 #pragma unused(w)
2136 struct dlil_threading_info *inp = v;
2137 struct ifnet *ifp = inp->ifp;
2138 struct timespec ts;
2139
2140 VERIFY(inp != dlil_main_input_thread);
2141 VERIFY(ifp != NULL && (ifp->if_eflags & IFEF_RXPOLL));
2142
2143 while (1) {
2144 struct mbuf *m = NULL;
2145 u_int32_t m_cnt, m_size, poll_req = 0;
2146 ifnet_model_t mode;
2147 struct timespec now, delta;
2148 u_int64_t ival;
2149
2150 lck_mtx_lock_spin(&inp->input_lck);
2151
2152 if ((ival = inp->rxpoll_ival) < IF_RXPOLL_INTERVALTIME_MIN)
2153 ival = IF_RXPOLL_INTERVALTIME_MIN;
2154
2155 /* Link parameters changed? */
2156 if (ifp->if_poll_update != 0) {
2157 ifp->if_poll_update = 0;
2158 (void) dlil_rxpoll_set_params(ifp, NULL, TRUE);
2159 }
2160
2161 /* Current operating mode */
2162 mode = inp->mode;
2163
2164 /* Wait until there is work to be done */
2165 while (!(inp->input_waiting & ~DLIL_INPUT_RUNNING)) {
2166 inp->input_waiting &= ~DLIL_INPUT_RUNNING;
2167 (void) msleep(&inp->input_waiting, &inp->input_lck,
2168 (PZERO - 1) | PSPIN, inp->input_name, NULL);
2169 }
2170
2171 inp->input_waiting |= DLIL_INPUT_RUNNING;
2172 inp->input_waiting &= ~DLIL_INPUT_WAITING;
2173
2174 /*
2175 * Protocol registration and injection must always use
2176 * the main input thread; in theory the latter can utilize
2177 * the corresponding input thread where the packet arrived
2178 * on, but that requires our knowing the interface in advance
2179 * (and the benefits might not worth the trouble.)
2180 */
2181 VERIFY(!(inp->input_waiting &
2182 (DLIL_PROTO_WAITING|DLIL_PROTO_REGISTER)));
2183
2184 if (inp->input_waiting & DLIL_INPUT_TERMINATE) {
2185 /* Free up pending packets */
2186 lck_mtx_convert_spin(&inp->input_lck);
2187 _flushq(&inp->rcvq_pkts);
2188 if (inp->input_mit_tcall != NULL) {
2189 if (thread_call_isactive(inp->input_mit_tcall))
2190 thread_call_cancel(inp->input_mit_tcall);
2191 }
2192 lck_mtx_unlock(&inp->input_lck);
2193
2194 dlil_terminate_input_thread(inp);
2195 /* NOTREACHED */
2196 return;
2197 }
2198
2199 /* Total count of all packets */
2200 m_cnt = qlen(&inp->rcvq_pkts);
2201
2202 /* Total bytes of all packets */
2203 m_size = qsize(&inp->rcvq_pkts);
2204
2205 /* Packets for this interface */
2206 m = _getq_all(&inp->rcvq_pkts, NULL, NULL, NULL);
2207 VERIFY(m != NULL || m_cnt == 0);
2208
2209 nanouptime(&now);
2210 if (!net_timerisset(&inp->sample_lasttime))
2211 *(&inp->sample_lasttime) = *(&now);
2212
2213 net_timersub(&now, &inp->sample_lasttime, &delta);
2214 if (if_rxpoll && net_timerisset(&inp->sample_holdtime)) {
2215 u_int32_t ptot, btot;
2216
2217 /* Accumulate statistics for current sampling */
2218 PKTCNTR_ADD(&inp->sstats, m_cnt, m_size);
2219
2220 if (net_timercmp(&delta, &inp->sample_holdtime, <))
2221 goto skip;
2222
2223 *(&inp->sample_lasttime) = *(&now);
2224
2225 /* Calculate min/max of inbound bytes */
2226 btot = (u_int32_t)inp->sstats.bytes;
2227 if (inp->rxpoll_bmin == 0 || inp->rxpoll_bmin > btot)
2228 inp->rxpoll_bmin = btot;
2229 if (btot > inp->rxpoll_bmax)
2230 inp->rxpoll_bmax = btot;
2231
2232 /* Calculate EWMA of inbound bytes */
2233 DLIL_EWMA(inp->rxpoll_bavg, btot, if_rxpoll_decay);
2234
2235 /* Calculate min/max of inbound packets */
2236 ptot = (u_int32_t)inp->sstats.packets;
2237 if (inp->rxpoll_pmin == 0 || inp->rxpoll_pmin > ptot)
2238 inp->rxpoll_pmin = ptot;
2239 if (ptot > inp->rxpoll_pmax)
2240 inp->rxpoll_pmax = ptot;
2241
2242 /* Calculate EWMA of inbound packets */
2243 DLIL_EWMA(inp->rxpoll_pavg, ptot, if_rxpoll_decay);
2244
2245 /* Reset sampling statistics */
2246 PKTCNTR_CLEAR(&inp->sstats);
2247
2248 /* Calculate EWMA of wakeup requests */
2249 DLIL_EWMA(inp->rxpoll_wavg, inp->wtot, if_rxpoll_decay);
2250 inp->wtot = 0;
2251
2252 if (dlil_verbose) {
2253 if (!net_timerisset(&inp->dbg_lasttime))
2254 *(&inp->dbg_lasttime) = *(&now);
2255 net_timersub(&now, &inp->dbg_lasttime, &delta);
2256 if (net_timercmp(&delta, &dlil_dbgrate, >=)) {
2257 *(&inp->dbg_lasttime) = *(&now);
2258 printf("%s: [%s] pkts avg %d max %d "
2259 "limits [%d/%d], wreq avg %d "
2260 "limits [%d/%d], bytes avg %d "
2261 "limits [%d/%d]\n", if_name(ifp),
2262 (inp->mode ==
2263 IFNET_MODEL_INPUT_POLL_ON) ?
2264 "ON" : "OFF", inp->rxpoll_pavg,
2265 inp->rxpoll_pmax,
2266 inp->rxpoll_plowat,
2267 inp->rxpoll_phiwat,
2268 inp->rxpoll_wavg,
2269 inp->rxpoll_wlowat,
2270 inp->rxpoll_whiwat,
2271 inp->rxpoll_bavg,
2272 inp->rxpoll_blowat,
2273 inp->rxpoll_bhiwat);
2274 }
2275 }
2276
2277 /* Perform mode transition, if necessary */
2278 if (!net_timerisset(&inp->mode_lasttime))
2279 *(&inp->mode_lasttime) = *(&now);
2280
2281 net_timersub(&now, &inp->mode_lasttime, &delta);
2282 if (net_timercmp(&delta, &inp->mode_holdtime, <))
2283 goto skip;
2284
2285 if (inp->rxpoll_pavg <= inp->rxpoll_plowat &&
2286 inp->rxpoll_bavg <= inp->rxpoll_blowat &&
2287 inp->mode != IFNET_MODEL_INPUT_POLL_OFF) {
2288 mode = IFNET_MODEL_INPUT_POLL_OFF;
2289 } else if (inp->rxpoll_pavg >= inp->rxpoll_phiwat &&
2290 (inp->rxpoll_bavg >= inp->rxpoll_bhiwat ||
2291 inp->rxpoll_wavg >= inp->rxpoll_whiwat) &&
2292 inp->mode != IFNET_MODEL_INPUT_POLL_ON) {
2293 mode = IFNET_MODEL_INPUT_POLL_ON;
2294 }
2295
2296 if (mode != inp->mode) {
2297 inp->mode = mode;
2298 *(&inp->mode_lasttime) = *(&now);
2299 poll_req++;
2300 }
2301 }
2302 skip:
2303 dlil_input_stats_sync(ifp, inp);
2304
2305 lck_mtx_unlock(&inp->input_lck);
2306
2307 /*
2308 * If there's a mode change and interface is still attached,
2309 * perform a downcall to the driver for the new mode. Also
2310 * hold an IO refcnt on the interface to prevent it from
2311 * being detached (will be release below.)
2312 */
2313 if (poll_req != 0 && ifnet_is_attached(ifp, 1)) {
2314 struct ifnet_model_params p = { mode, { 0 } };
2315 errno_t err;
2316
2317 if (dlil_verbose) {
2318 printf("%s: polling is now %s, "
2319 "pkts avg %d max %d limits [%d/%d], "
2320 "wreq avg %d limits [%d/%d], "
2321 "bytes avg %d limits [%d/%d]\n",
2322 if_name(ifp),
2323 (mode == IFNET_MODEL_INPUT_POLL_ON) ?
2324 "ON" : "OFF", inp->rxpoll_pavg,
2325 inp->rxpoll_pmax, inp->rxpoll_plowat,
2326 inp->rxpoll_phiwat, inp->rxpoll_wavg,
2327 inp->rxpoll_wlowat, inp->rxpoll_whiwat,
2328 inp->rxpoll_bavg, inp->rxpoll_blowat,
2329 inp->rxpoll_bhiwat);
2330 }
2331
2332 if ((err = ((*ifp->if_input_ctl)(ifp,
2333 IFNET_CTL_SET_INPUT_MODEL, sizeof (p), &p))) != 0) {
2334 printf("%s: error setting polling mode "
2335 "to %s (%d)\n", if_name(ifp),
2336 (mode == IFNET_MODEL_INPUT_POLL_ON) ?
2337 "ON" : "OFF", err);
2338 }
2339
2340 switch (mode) {
2341 case IFNET_MODEL_INPUT_POLL_OFF:
2342 ifnet_set_poll_cycle(ifp, NULL);
2343 inp->rxpoll_offreq++;
2344 if (err != 0)
2345 inp->rxpoll_offerr++;
2346 break;
2347
2348 case IFNET_MODEL_INPUT_POLL_ON:
2349 net_nsectimer(&ival, &ts);
2350 ifnet_set_poll_cycle(ifp, &ts);
2351 ifnet_poll(ifp);
2352 inp->rxpoll_onreq++;
2353 if (err != 0)
2354 inp->rxpoll_onerr++;
2355 break;
2356
2357 default:
2358 VERIFY(0);
2359 /* NOTREACHED */
2360 }
2361
2362 /* Release the IO refcnt */
2363 ifnet_decr_iorefcnt(ifp);
2364 }
2365
2366 /*
2367 * NOTE warning %%% attention !!!!
2368 * We should think about putting some thread starvation
2369 * safeguards if we deal with long chains of packets.
2370 */
2371 if (m != NULL)
2372 dlil_input_packet_list_extended(NULL, m, m_cnt, mode);
2373 }
2374
2375 /* NOTREACHED */
2376 VERIFY(0); /* we should never get here */
2377 }
2378
2379 /*
2380 * Must be called on an attached ifnet (caller is expected to check.)
2381 * Caller may pass NULL for poll parameters to indicate "auto-tuning."
2382 */
2383 errno_t
2384 dlil_rxpoll_set_params(struct ifnet *ifp, struct ifnet_poll_params *p,
2385 boolean_t locked)
2386 {
2387 struct dlil_threading_info *inp;
2388 u_int64_t sample_holdtime, inbw;
2389
2390 VERIFY(ifp != NULL);
2391 if (!(ifp->if_eflags & IFEF_RXPOLL) || (inp = ifp->if_inp) == NULL)
2392 return (ENXIO);
2393
2394 if (p != NULL) {
2395 if ((p->packets_lowat == 0 && p->packets_hiwat != 0) ||
2396 (p->packets_lowat != 0 && p->packets_hiwat == 0))
2397 return (EINVAL);
2398 if (p->packets_lowat != 0 && /* hiwat must be non-zero */
2399 p->packets_lowat >= p->packets_hiwat)
2400 return (EINVAL);
2401 if ((p->bytes_lowat == 0 && p->bytes_hiwat != 0) ||
2402 (p->bytes_lowat != 0 && p->bytes_hiwat == 0))
2403 return (EINVAL);
2404 if (p->bytes_lowat != 0 && /* hiwat must be non-zero */
2405 p->bytes_lowat >= p->bytes_hiwat)
2406 return (EINVAL);
2407 if (p->interval_time != 0 &&
2408 p->interval_time < IF_RXPOLL_INTERVALTIME_MIN)
2409 p->interval_time = IF_RXPOLL_INTERVALTIME_MIN;
2410 }
2411
2412 if (!locked)
2413 lck_mtx_lock(&inp->input_lck);
2414
2415 LCK_MTX_ASSERT(&inp->input_lck, LCK_MTX_ASSERT_OWNED);
2416
2417 /*
2418 * Normally, we'd reset the parameters to the auto-tuned values
2419 * if the the input thread detects a change in link rate. If the
2420 * driver provides its own parameters right after a link rate
2421 * changes, but before the input thread gets to run, we want to
2422 * make sure to keep the driver's values. Clearing if_poll_update
2423 * will achieve that.
2424 */
2425 if (p != NULL && !locked && ifp->if_poll_update != 0)
2426 ifp->if_poll_update = 0;
2427
2428 if ((inbw = ifnet_input_linkrate(ifp)) == 0 && p == NULL) {
2429 sample_holdtime = 0; /* polling is disabled */
2430 inp->rxpoll_wlowat = inp->rxpoll_plowat =
2431 inp->rxpoll_blowat = 0;
2432 inp->rxpoll_whiwat = inp->rxpoll_phiwat =
2433 inp->rxpoll_bhiwat = (u_int32_t)-1;
2434 inp->rxpoll_plim = 0;
2435 inp->rxpoll_ival = IF_RXPOLL_INTERVALTIME_MIN;
2436 } else {
2437 u_int32_t plowat, phiwat, blowat, bhiwat, plim;
2438 u_int64_t ival;
2439 unsigned int n, i;
2440
2441 for (n = 0, i = 0; rxpoll_tbl[i].speed != 0; i++) {
2442 if (inbw < rxpoll_tbl[i].speed)
2443 break;
2444 n = i;
2445 }
2446 /* auto-tune if caller didn't specify a value */
2447 plowat = ((p == NULL || p->packets_lowat == 0) ?
2448 rxpoll_tbl[n].plowat : p->packets_lowat);
2449 phiwat = ((p == NULL || p->packets_hiwat == 0) ?
2450 rxpoll_tbl[n].phiwat : p->packets_hiwat);
2451 blowat = ((p == NULL || p->bytes_lowat == 0) ?
2452 rxpoll_tbl[n].blowat : p->bytes_lowat);
2453 bhiwat = ((p == NULL || p->bytes_hiwat == 0) ?
2454 rxpoll_tbl[n].bhiwat : p->bytes_hiwat);
2455 plim = ((p == NULL || p->packets_limit == 0) ?
2456 if_rxpoll_max : p->packets_limit);
2457 ival = ((p == NULL || p->interval_time == 0) ?
2458 if_rxpoll_interval_time : p->interval_time);
2459
2460 VERIFY(plowat != 0 && phiwat != 0);
2461 VERIFY(blowat != 0 && bhiwat != 0);
2462 VERIFY(ival >= IF_RXPOLL_INTERVALTIME_MIN);
2463
2464 sample_holdtime = if_rxpoll_sample_holdtime;
2465 inp->rxpoll_wlowat = if_rxpoll_wlowat;
2466 inp->rxpoll_whiwat = if_rxpoll_whiwat;
2467 inp->rxpoll_plowat = plowat;
2468 inp->rxpoll_phiwat = phiwat;
2469 inp->rxpoll_blowat = blowat;
2470 inp->rxpoll_bhiwat = bhiwat;
2471 inp->rxpoll_plim = plim;
2472 inp->rxpoll_ival = ival;
2473 }
2474
2475 net_nsectimer(&if_rxpoll_mode_holdtime, &inp->mode_holdtime);
2476 net_nsectimer(&sample_holdtime, &inp->sample_holdtime);
2477
2478 if (dlil_verbose) {
2479 printf("%s: speed %llu bps, sample per %llu nsec, "
2480 "poll interval %llu nsec, pkts per poll %u, "
2481 "pkt limits [%u/%u], wreq limits [%u/%u], "
2482 "bytes limits [%u/%u]\n", if_name(ifp),
2483 inbw, sample_holdtime, inp->rxpoll_ival, inp->rxpoll_plim,
2484 inp->rxpoll_plowat, inp->rxpoll_phiwat, inp->rxpoll_wlowat,
2485 inp->rxpoll_whiwat, inp->rxpoll_blowat, inp->rxpoll_bhiwat);
2486 }
2487
2488 if (!locked)
2489 lck_mtx_unlock(&inp->input_lck);
2490
2491 return (0);
2492 }
2493
2494 /*
2495 * Must be called on an attached ifnet (caller is expected to check.)
2496 */
2497 errno_t
2498 dlil_rxpoll_get_params(struct ifnet *ifp, struct ifnet_poll_params *p)
2499 {
2500 struct dlil_threading_info *inp;
2501
2502 VERIFY(ifp != NULL && p != NULL);
2503 if (!(ifp->if_eflags & IFEF_RXPOLL) || (inp = ifp->if_inp) == NULL)
2504 return (ENXIO);
2505
2506 bzero(p, sizeof (*p));
2507
2508 lck_mtx_lock(&inp->input_lck);
2509 p->packets_limit = inp->rxpoll_plim;
2510 p->packets_lowat = inp->rxpoll_plowat;
2511 p->packets_hiwat = inp->rxpoll_phiwat;
2512 p->bytes_lowat = inp->rxpoll_blowat;
2513 p->bytes_hiwat = inp->rxpoll_bhiwat;
2514 p->interval_time = inp->rxpoll_ival;
2515 lck_mtx_unlock(&inp->input_lck);
2516
2517 return (0);
2518 }
2519
2520 errno_t
2521 ifnet_input(struct ifnet *ifp, struct mbuf *m_head,
2522 const struct ifnet_stat_increment_param *s)
2523 {
2524 return (ifnet_input_common(ifp, m_head, NULL, s, FALSE, FALSE));
2525 }
2526
2527 errno_t
2528 ifnet_input_extended(struct ifnet *ifp, struct mbuf *m_head,
2529 struct mbuf *m_tail, const struct ifnet_stat_increment_param *s)
2530 {
2531 return (ifnet_input_common(ifp, m_head, m_tail, s, TRUE, FALSE));
2532 }
2533
2534 static errno_t
2535 ifnet_input_common(struct ifnet *ifp, struct mbuf *m_head, struct mbuf *m_tail,
2536 const struct ifnet_stat_increment_param *s, boolean_t ext, boolean_t poll)
2537 {
2538 dlil_input_func input_func;
2539 struct ifnet_stat_increment_param _s;
2540 u_int32_t m_cnt = 0, m_size = 0;
2541 struct mbuf *last;
2542 errno_t err = 0;
2543
2544 if ((m_head == NULL && !poll) || (s == NULL && ext)) {
2545 if (m_head != NULL)
2546 mbuf_freem_list(m_head);
2547 return (EINVAL);
2548 }
2549
2550 VERIFY(m_head != NULL || (s == NULL && m_tail == NULL && !ext && poll));
2551 VERIFY(m_tail == NULL || ext);
2552 VERIFY(s != NULL || !ext);
2553
2554 /*
2555 * Drop the packet(s) if the parameters are invalid, or if the
2556 * interface is no longer attached; else hold an IO refcnt to
2557 * prevent it from being detached (will be released below.)
2558 */
2559 if (ifp == NULL || (ifp != lo_ifp && !ifnet_is_attached(ifp, 1))) {
2560 if (m_head != NULL)
2561 mbuf_freem_list(m_head);
2562 return (EINVAL);
2563 }
2564
2565 input_func = ifp->if_input_dlil;
2566 VERIFY(input_func != NULL);
2567
2568 if (m_tail == NULL) {
2569 last = m_head;
2570 while (m_head != NULL) {
2571 #if IFNET_INPUT_SANITY_CHK
2572 if (dlil_input_sanity_check != 0)
2573 DLIL_INPUT_CHECK(last, ifp);
2574 #endif /* IFNET_INPUT_SANITY_CHK */
2575 m_cnt++;
2576 m_size += m_length(last);
2577 if (mbuf_nextpkt(last) == NULL)
2578 break;
2579 last = mbuf_nextpkt(last);
2580 }
2581 m_tail = last;
2582 } else {
2583 #if IFNET_INPUT_SANITY_CHK
2584 if (dlil_input_sanity_check != 0) {
2585 last = m_head;
2586 while (1) {
2587 DLIL_INPUT_CHECK(last, ifp);
2588 m_cnt++;
2589 m_size += m_length(last);
2590 if (mbuf_nextpkt(last) == NULL)
2591 break;
2592 last = mbuf_nextpkt(last);
2593 }
2594 } else {
2595 m_cnt = s->packets_in;
2596 m_size = s->bytes_in;
2597 last = m_tail;
2598 }
2599 #else
2600 m_cnt = s->packets_in;
2601 m_size = s->bytes_in;
2602 last = m_tail;
2603 #endif /* IFNET_INPUT_SANITY_CHK */
2604 }
2605
2606 if (last != m_tail) {
2607 panic_plain("%s: invalid input packet chain for %s, "
2608 "tail mbuf %p instead of %p\n", __func__, if_name(ifp),
2609 m_tail, last);
2610 }
2611
2612 /*
2613 * Assert packet count only for the extended variant, for backwards
2614 * compatibility, since this came directly from the device driver.
2615 * Relax this assertion for input bytes, as the driver may have
2616 * included the link-layer headers in the computation; hence
2617 * m_size is just an approximation.
2618 */
2619 if (ext && s->packets_in != m_cnt) {
2620 panic_plain("%s: input packet count mismatch for %s, "
2621 "%d instead of %d\n", __func__, if_name(ifp),
2622 s->packets_in, m_cnt);
2623 }
2624
2625 if (s == NULL) {
2626 bzero(&_s, sizeof (_s));
2627 s = &_s;
2628 } else {
2629 _s = *s;
2630 }
2631 _s.packets_in = m_cnt;
2632 _s.bytes_in = m_size;
2633
2634 err = (*input_func)(ifp, m_head, m_tail, s, poll, current_thread());
2635
2636 if (ifp != lo_ifp) {
2637 /* Release the IO refcnt */
2638 ifnet_decr_iorefcnt(ifp);
2639 }
2640
2641 return (err);
2642 }
2643
2644
2645 errno_t
2646 dlil_output_handler(struct ifnet *ifp, struct mbuf *m)
2647 {
2648 return (ifp->if_output(ifp, m));
2649 }
2650
2651 errno_t
2652 dlil_input_handler(struct ifnet *ifp, struct mbuf *m_head,
2653 struct mbuf *m_tail, const struct ifnet_stat_increment_param *s,
2654 boolean_t poll, struct thread *tp)
2655 {
2656 struct dlil_threading_info *inp;
2657 u_int32_t m_cnt = s->packets_in;
2658 u_int32_t m_size = s->bytes_in;
2659
2660 if ((inp = ifp->if_inp) == NULL)
2661 inp = dlil_main_input_thread;
2662
2663 /*
2664 * If there is a matching DLIL input thread associated with an
2665 * affinity set, associate this thread with the same set. We
2666 * will only do this once.
2667 */
2668 lck_mtx_lock_spin(&inp->input_lck);
2669 if (inp != dlil_main_input_thread && inp->net_affinity && tp != NULL &&
2670 ((!poll && inp->wloop_thr == THREAD_NULL) ||
2671 (poll && inp->poll_thr == THREAD_NULL))) {
2672 u_int32_t tag = inp->tag;
2673
2674 if (poll) {
2675 VERIFY(inp->poll_thr == THREAD_NULL);
2676 inp->poll_thr = tp;
2677 } else {
2678 VERIFY(inp->wloop_thr == THREAD_NULL);
2679 inp->wloop_thr = tp;
2680 }
2681 lck_mtx_unlock(&inp->input_lck);
2682
2683 /* Associate the current thread with the new affinity tag */
2684 (void) dlil_affinity_set(tp, tag);
2685
2686 /*
2687 * Take a reference on the current thread; during detach,
2688 * we will need to refer to it in order to tear down its
2689 * affinity.
2690 */
2691 thread_reference(tp);
2692 lck_mtx_lock_spin(&inp->input_lck);
2693 }
2694
2695 VERIFY(m_head != NULL || (m_tail == NULL && m_cnt == 0));
2696
2697 /*
2698 * Because of loopbacked multicast we cannot stuff the ifp in
2699 * the rcvif of the packet header: loopback (lo0) packets use a
2700 * dedicated list so that we can later associate them with lo_ifp
2701 * on their way up the stack. Packets for other interfaces without
2702 * dedicated input threads go to the regular list.
2703 */
2704 if (m_head != NULL) {
2705 if (inp == dlil_main_input_thread && ifp == lo_ifp) {
2706 struct dlil_main_threading_info *inpm =
2707 (struct dlil_main_threading_info *)inp;
2708 _addq_multi(&inpm->lo_rcvq_pkts, m_head, m_tail,
2709 m_cnt, m_size);
2710 } else {
2711 _addq_multi(&inp->rcvq_pkts, m_head, m_tail,
2712 m_cnt, m_size);
2713 }
2714 }
2715
2716 #if IFNET_INPUT_SANITY_CHK
2717 if (dlil_input_sanity_check != 0) {
2718 u_int32_t count;
2719 struct mbuf *m0;
2720
2721 for (m0 = m_head, count = 0; m0; m0 = mbuf_nextpkt(m0))
2722 count++;
2723
2724 if (count != m_cnt) {
2725 panic_plain("%s: invalid packet count %d "
2726 "(expected %d)\n", if_name(ifp),
2727 count, m_cnt);
2728 /* NOTREACHED */
2729 }
2730
2731 inp->input_mbuf_cnt += m_cnt;
2732 }
2733 #endif /* IFNET_INPUT_SANITY_CHK */
2734
2735 dlil_input_stats_add(s, inp, poll);
2736 /*
2737 * If we're using the main input thread, synchronize the
2738 * stats now since we have the interface context. All
2739 * other cases involving dedicated input threads will
2740 * have their stats synchronized there.
2741 */
2742 if (inp == dlil_main_input_thread)
2743 dlil_input_stats_sync(ifp, inp);
2744
2745 if (inp->input_mit_tcall &&
2746 qlen(&inp->rcvq_pkts) >= dlil_rcv_mit_pkts_min &&
2747 qlen(&inp->rcvq_pkts) < dlil_rcv_mit_pkts_max &&
2748 (ifp->if_family == IFNET_FAMILY_ETHERNET ||
2749 ifp->if_type == IFT_CELLULAR)
2750 ) {
2751 if (!thread_call_isactive(inp->input_mit_tcall)) {
2752 uint64_t deadline;
2753 clock_interval_to_deadline(dlil_rcv_mit_interval,
2754 1, &deadline);
2755 (void) thread_call_enter_delayed(
2756 inp->input_mit_tcall, deadline);
2757 }
2758 } else {
2759 inp->input_waiting |= DLIL_INPUT_WAITING;
2760 if (!(inp->input_waiting & DLIL_INPUT_RUNNING)) {
2761 inp->wtot++;
2762 wakeup_one((caddr_t)&inp->input_waiting);
2763 }
2764 }
2765 lck_mtx_unlock(&inp->input_lck);
2766
2767 return (0);
2768 }
2769
2770
2771 static void
2772 ifnet_start_common(struct ifnet *ifp, boolean_t resetfc)
2773 {
2774 if (!(ifp->if_eflags & IFEF_TXSTART))
2775 return;
2776 /*
2777 * If the starter thread is inactive, signal it to do work,
2778 * unless the interface is being flow controlled from below,
2779 * e.g. a virtual interface being flow controlled by a real
2780 * network interface beneath it, or it's been disabled via
2781 * a call to ifnet_disable_output().
2782 */
2783 lck_mtx_lock_spin(&ifp->if_start_lock);
2784 if (resetfc) {
2785 ifp->if_start_flags &= ~IFSF_FLOW_CONTROLLED;
2786 } else if (ifp->if_start_flags & IFSF_FLOW_CONTROLLED) {
2787 lck_mtx_unlock(&ifp->if_start_lock);
2788 return;
2789 }
2790 ifp->if_start_req++;
2791 if (!ifp->if_start_active && ifp->if_start_thread != THREAD_NULL &&
2792 (resetfc || !(ifp->if_eflags & IFEF_ENQUEUE_MULTI) ||
2793 IFCQ_LEN(&ifp->if_snd) >= ifp->if_start_delay_qlen ||
2794 ifp->if_start_delayed == 0)) {
2795 (void) thread_wakeup_thread((caddr_t)&ifp->if_start_thread,
2796 ifp->if_start_thread);
2797 }
2798 lck_mtx_unlock(&ifp->if_start_lock);
2799 }
2800
2801 void
2802 ifnet_start(struct ifnet *ifp)
2803 {
2804 ifnet_start_common(ifp, FALSE);
2805 }
2806
2807 static void
2808 ifnet_start_thread_fn(void *v, wait_result_t w)
2809 {
2810 #pragma unused(w)
2811 struct ifnet *ifp = v;
2812 char ifname[IFNAMSIZ + 1];
2813 char thread_name[MAXTHREADNAMESIZE];
2814 struct timespec *ts = NULL;
2815 struct ifclassq *ifq = &ifp->if_snd;
2816 struct timespec delay_start_ts;
2817
2818 /* Construct the name for this thread, and then apply it. */
2819 bzero(thread_name, sizeof(thread_name));
2820 (void) snprintf(thread_name, sizeof (thread_name),
2821 "ifnet_start_%s", ifp->if_xname);
2822 thread_set_thread_name(ifp->if_start_thread, thread_name);
2823
2824 /*
2825 * Treat the dedicated starter thread for lo0 as equivalent to
2826 * the driver workloop thread; if net_affinity is enabled for
2827 * the main input thread, associate this starter thread to it
2828 * by binding them with the same affinity tag. This is done
2829 * only once (as we only have one lo_ifp which never goes away.)
2830 */
2831 if (ifp == lo_ifp) {
2832 struct dlil_threading_info *inp = dlil_main_input_thread;
2833 struct thread *tp = current_thread();
2834
2835 lck_mtx_lock(&inp->input_lck);
2836 if (inp->net_affinity) {
2837 u_int32_t tag = inp->tag;
2838
2839 VERIFY(inp->wloop_thr == THREAD_NULL);
2840 VERIFY(inp->poll_thr == THREAD_NULL);
2841 inp->wloop_thr = tp;
2842 lck_mtx_unlock(&inp->input_lck);
2843
2844 /* Associate this thread with the affinity tag */
2845 (void) dlil_affinity_set(tp, tag);
2846 } else {
2847 lck_mtx_unlock(&inp->input_lck);
2848 }
2849 }
2850
2851 (void) snprintf(ifname, sizeof (ifname), "%s_starter", if_name(ifp));
2852
2853 lck_mtx_lock_spin(&ifp->if_start_lock);
2854
2855 for (;;) {
2856 if (ifp->if_start_thread != NULL) {
2857 (void) msleep(&ifp->if_start_thread,
2858 &ifp->if_start_lock,
2859 (PZERO - 1) | PSPIN, ifname, ts);
2860 }
2861 /* interface is detached? */
2862 if (ifp->if_start_thread == THREAD_NULL) {
2863 ifnet_set_start_cycle(ifp, NULL);
2864 lck_mtx_unlock(&ifp->if_start_lock);
2865 ifnet_purge(ifp);
2866
2867 if (dlil_verbose) {
2868 printf("%s: starter thread terminated\n",
2869 if_name(ifp));
2870 }
2871
2872 /* for the extra refcnt from kernel_thread_start() */
2873 thread_deallocate(current_thread());
2874 /* this is the end */
2875 thread_terminate(current_thread());
2876 /* NOTREACHED */
2877 return;
2878 }
2879
2880 ifp->if_start_active = 1;
2881
2882 for (;;) {
2883 u_int32_t req = ifp->if_start_req;
2884 if (!IFCQ_IS_EMPTY(ifq) &&
2885 (ifp->if_eflags & IFEF_ENQUEUE_MULTI) &&
2886 ifp->if_start_delayed == 0 &&
2887 IFCQ_LEN(ifq) < ifp->if_start_delay_qlen &&
2888 (ifp->if_eflags & IFEF_DELAY_START)) {
2889 ifp->if_start_delayed = 1;
2890 ifnet_start_delayed++;
2891 break;
2892 } else {
2893 ifp->if_start_delayed = 0;
2894 }
2895 lck_mtx_unlock(&ifp->if_start_lock);
2896
2897 /*
2898 * If no longer attached, don't call start because ifp
2899 * is being destroyed; else hold an IO refcnt to
2900 * prevent the interface from being detached (will be
2901 * released below.)
2902 */
2903 if (!ifnet_is_attached(ifp, 1)) {
2904 lck_mtx_lock_spin(&ifp->if_start_lock);
2905 break;
2906 }
2907
2908 /* invoke the driver's start routine */
2909 ((*ifp->if_start)(ifp));
2910
2911 /*
2912 * Release the io ref count taken by ifnet_is_attached.
2913 */
2914 ifnet_decr_iorefcnt(ifp);
2915
2916 lck_mtx_lock_spin(&ifp->if_start_lock);
2917
2918 /*
2919 * If there's no pending request or if the
2920 * interface has been disabled, we're done.
2921 */
2922 if (req == ifp->if_start_req ||
2923 (ifp->if_start_flags & IFSF_FLOW_CONTROLLED)) {
2924 break;
2925 }
2926 }
2927
2928 ifp->if_start_req = 0;
2929 ifp->if_start_active = 0;
2930
2931 /*
2932 * Wakeup N ns from now if rate-controlled by TBR, and if
2933 * there are still packets in the send queue which haven't
2934 * been dequeued so far; else sleep indefinitely (ts = NULL)
2935 * until ifnet_start() is called again.
2936 */
2937 ts = ((IFCQ_TBR_IS_ENABLED(ifq) && !IFCQ_IS_EMPTY(ifq)) ?
2938 &ifp->if_start_cycle : NULL);
2939
2940 if (ts == NULL && ifp->if_start_delayed == 1) {
2941 delay_start_ts.tv_sec = 0;
2942 delay_start_ts.tv_nsec = ifp->if_start_delay_timeout;
2943 ts = &delay_start_ts;
2944 }
2945
2946 if (ts != NULL && ts->tv_sec == 0 && ts->tv_nsec == 0)
2947 ts = NULL;
2948 }
2949
2950 /* NOTREACHED */
2951 }
2952
2953 void
2954 ifnet_set_start_cycle(struct ifnet *ifp, struct timespec *ts)
2955 {
2956 if (ts == NULL)
2957 bzero(&ifp->if_start_cycle, sizeof (ifp->if_start_cycle));
2958 else
2959 *(&ifp->if_start_cycle) = *ts;
2960
2961 if (ts != NULL && ts->tv_nsec != 0 && dlil_verbose)
2962 printf("%s: restart interval set to %lu nsec\n",
2963 if_name(ifp), ts->tv_nsec);
2964 }
2965
2966 static void
2967 ifnet_poll(struct ifnet *ifp)
2968 {
2969 /*
2970 * If the poller thread is inactive, signal it to do work.
2971 */
2972 lck_mtx_lock_spin(&ifp->if_poll_lock);
2973 ifp->if_poll_req++;
2974 if (!ifp->if_poll_active && ifp->if_poll_thread != THREAD_NULL) {
2975 wakeup_one((caddr_t)&ifp->if_poll_thread);
2976 }
2977 lck_mtx_unlock(&ifp->if_poll_lock);
2978 }
2979
2980 static void
2981 ifnet_poll_thread_fn(void *v, wait_result_t w)
2982 {
2983 #pragma unused(w)
2984 struct dlil_threading_info *inp;
2985 struct ifnet *ifp = v;
2986 char ifname[IFNAMSIZ + 1];
2987 struct timespec *ts = NULL;
2988 struct ifnet_stat_increment_param s;
2989
2990 snprintf(ifname, sizeof (ifname), "%s_poller",
2991 if_name(ifp));
2992 bzero(&s, sizeof (s));
2993
2994 lck_mtx_lock_spin(&ifp->if_poll_lock);
2995
2996 inp = ifp->if_inp;
2997 VERIFY(inp != NULL);
2998
2999 for (;;) {
3000 if (ifp->if_poll_thread != THREAD_NULL) {
3001 (void) msleep(&ifp->if_poll_thread, &ifp->if_poll_lock,
3002 (PZERO - 1) | PSPIN, ifname, ts);
3003 }
3004
3005 /* interface is detached (maybe while asleep)? */
3006 if (ifp->if_poll_thread == THREAD_NULL) {
3007 ifnet_set_poll_cycle(ifp, NULL);
3008 lck_mtx_unlock(&ifp->if_poll_lock);
3009
3010 if (dlil_verbose) {
3011 printf("%s: poller thread terminated\n",
3012 if_name(ifp));
3013 }
3014
3015 /* for the extra refcnt from kernel_thread_start() */
3016 thread_deallocate(current_thread());
3017 /* this is the end */
3018 thread_terminate(current_thread());
3019 /* NOTREACHED */
3020 return;
3021 }
3022
3023 ifp->if_poll_active = 1;
3024 for (;;) {
3025 struct mbuf *m_head, *m_tail;
3026 u_int32_t m_lim, m_cnt, m_totlen;
3027 u_int16_t req = ifp->if_poll_req;
3028
3029 lck_mtx_unlock(&ifp->if_poll_lock);
3030
3031 /*
3032 * If no longer attached, there's nothing to do;
3033 * else hold an IO refcnt to prevent the interface
3034 * from being detached (will be released below.)
3035 */
3036 if (!ifnet_is_attached(ifp, 1)) {
3037 lck_mtx_lock_spin(&ifp->if_poll_lock);
3038 break;
3039 }
3040
3041 m_lim = (inp->rxpoll_plim != 0) ? inp->rxpoll_plim :
3042 MAX((qlimit(&inp->rcvq_pkts)),
3043 (inp->rxpoll_phiwat << 2));
3044
3045 if (dlil_verbose > 1) {
3046 printf("%s: polling up to %d pkts, "
3047 "pkts avg %d max %d, wreq avg %d, "
3048 "bytes avg %d\n",
3049 if_name(ifp), m_lim,
3050 inp->rxpoll_pavg, inp->rxpoll_pmax,
3051 inp->rxpoll_wavg, inp->rxpoll_bavg);
3052 }
3053
3054 /* invoke the driver's input poll routine */
3055 ((*ifp->if_input_poll)(ifp, 0, m_lim, &m_head, &m_tail,
3056 &m_cnt, &m_totlen));
3057
3058 if (m_head != NULL) {
3059 VERIFY(m_tail != NULL && m_cnt > 0);
3060
3061 if (dlil_verbose > 1) {
3062 printf("%s: polled %d pkts, "
3063 "pkts avg %d max %d, wreq avg %d, "
3064 "bytes avg %d\n",
3065 if_name(ifp), m_cnt,
3066 inp->rxpoll_pavg, inp->rxpoll_pmax,
3067 inp->rxpoll_wavg, inp->rxpoll_bavg);
3068 }
3069
3070 /* stats are required for extended variant */
3071 s.packets_in = m_cnt;
3072 s.bytes_in = m_totlen;
3073
3074 (void) ifnet_input_common(ifp, m_head, m_tail,
3075 &s, TRUE, TRUE);
3076 } else {
3077 if (dlil_verbose > 1) {
3078 printf("%s: no packets, "
3079 "pkts avg %d max %d, wreq avg %d, "
3080 "bytes avg %d\n",
3081 if_name(ifp), inp->rxpoll_pavg,
3082 inp->rxpoll_pmax, inp->rxpoll_wavg,
3083 inp->rxpoll_bavg);
3084 }
3085
3086 (void) ifnet_input_common(ifp, NULL, NULL,
3087 NULL, FALSE, TRUE);
3088 }
3089
3090 /* Release the io ref count */
3091 ifnet_decr_iorefcnt(ifp);
3092
3093 lck_mtx_lock_spin(&ifp->if_poll_lock);
3094
3095 /* if there's no pending request, we're done */
3096 if (req == ifp->if_poll_req) {
3097 break;
3098 }
3099 }
3100 ifp->if_poll_req = 0;
3101 ifp->if_poll_active = 0;
3102
3103 /*
3104 * Wakeup N ns from now, else sleep indefinitely (ts = NULL)
3105 * until ifnet_poll() is called again.
3106 */
3107 ts = &ifp->if_poll_cycle;
3108 if (ts->tv_sec == 0 && ts->tv_nsec == 0)
3109 ts = NULL;
3110 }
3111
3112 /* NOTREACHED */
3113 }
3114
3115 void
3116 ifnet_set_poll_cycle(struct ifnet *ifp, struct timespec *ts)
3117 {
3118 if (ts == NULL)
3119 bzero(&ifp->if_poll_cycle, sizeof (ifp->if_poll_cycle));
3120 else
3121 *(&ifp->if_poll_cycle) = *ts;
3122
3123 if (ts != NULL && ts->tv_nsec != 0 && dlil_verbose)
3124 printf("%s: poll interval set to %lu nsec\n",
3125 if_name(ifp), ts->tv_nsec);
3126 }
3127
3128 void
3129 ifnet_purge(struct ifnet *ifp)
3130 {
3131 if (ifp != NULL && (ifp->if_eflags & IFEF_TXSTART))
3132 if_qflush(ifp, 0);
3133 }
3134
3135 void
3136 ifnet_update_sndq(struct ifclassq *ifq, cqev_t ev)
3137 {
3138 IFCQ_LOCK_ASSERT_HELD(ifq);
3139
3140 if (!(IFCQ_IS_READY(ifq)))
3141 return;
3142
3143 if (IFCQ_TBR_IS_ENABLED(ifq)) {
3144 struct tb_profile tb = { ifq->ifcq_tbr.tbr_rate_raw,
3145 ifq->ifcq_tbr.tbr_percent, 0 };
3146 (void) ifclassq_tbr_set(ifq, &tb, FALSE);
3147 }
3148
3149 ifclassq_update(ifq, ev);
3150 }
3151
3152 void
3153 ifnet_update_rcv(struct ifnet *ifp, cqev_t ev)
3154 {
3155 switch (ev) {
3156 case CLASSQ_EV_LINK_BANDWIDTH:
3157 if (net_rxpoll && (ifp->if_eflags & IFEF_RXPOLL))
3158 ifp->if_poll_update++;
3159 break;
3160
3161 default:
3162 break;
3163 }
3164 }
3165
3166 errno_t
3167 ifnet_set_output_sched_model(struct ifnet *ifp, u_int32_t model)
3168 {
3169 struct ifclassq *ifq;
3170 u_int32_t omodel;
3171 errno_t err;
3172
3173 if (ifp == NULL || model >= IFNET_SCHED_MODEL_MAX)
3174 return (EINVAL);
3175 else if (!(ifp->if_eflags & IFEF_TXSTART))
3176 return (ENXIO);
3177
3178 ifq = &ifp->if_snd;
3179 IFCQ_LOCK(ifq);
3180 omodel = ifp->if_output_sched_model;
3181 ifp->if_output_sched_model = model;
3182 if ((err = ifclassq_pktsched_setup(ifq)) != 0)
3183 ifp->if_output_sched_model = omodel;
3184 IFCQ_UNLOCK(ifq);
3185
3186 return (err);
3187 }
3188
3189 errno_t
3190 ifnet_set_sndq_maxlen(struct ifnet *ifp, u_int32_t maxqlen)
3191 {
3192 if (ifp == NULL)
3193 return (EINVAL);
3194 else if (!(ifp->if_eflags & IFEF_TXSTART))
3195 return (ENXIO);
3196
3197 ifclassq_set_maxlen(&ifp->if_snd, maxqlen);
3198
3199 return (0);
3200 }
3201
3202 errno_t
3203 ifnet_get_sndq_maxlen(struct ifnet *ifp, u_int32_t *maxqlen)
3204 {
3205 if (ifp == NULL || maxqlen == NULL)
3206 return (EINVAL);
3207 else if (!(ifp->if_eflags & IFEF_TXSTART))
3208 return (ENXIO);
3209
3210 *maxqlen = ifclassq_get_maxlen(&ifp->if_snd);
3211
3212 return (0);
3213 }
3214
3215 errno_t
3216 ifnet_get_sndq_len(struct ifnet *ifp, u_int32_t *pkts)
3217 {
3218 errno_t err;
3219
3220 if (ifp == NULL || pkts == NULL)
3221 err = EINVAL;
3222 else if (!(ifp->if_eflags & IFEF_TXSTART))
3223 err = ENXIO;
3224 else
3225 err = ifclassq_get_len(&ifp->if_snd, MBUF_SC_UNSPEC,
3226 pkts, NULL);
3227
3228 return (err);
3229 }
3230
3231 errno_t
3232 ifnet_get_service_class_sndq_len(struct ifnet *ifp, mbuf_svc_class_t sc,
3233 u_int32_t *pkts, u_int32_t *bytes)
3234 {
3235 errno_t err;
3236
3237 if (ifp == NULL || !MBUF_VALID_SC(sc) ||
3238 (pkts == NULL && bytes == NULL))
3239 err = EINVAL;
3240 else if (!(ifp->if_eflags & IFEF_TXSTART))
3241 err = ENXIO;
3242 else
3243 err = ifclassq_get_len(&ifp->if_snd, sc, pkts, bytes);
3244
3245 return (err);
3246 }
3247
3248 errno_t
3249 ifnet_set_rcvq_maxlen(struct ifnet *ifp, u_int32_t maxqlen)
3250 {
3251 struct dlil_threading_info *inp;
3252
3253 if (ifp == NULL)
3254 return (EINVAL);
3255 else if (!(ifp->if_eflags & IFEF_RXPOLL) || ifp->if_inp == NULL)
3256 return (ENXIO);
3257
3258 if (maxqlen == 0)
3259 maxqlen = if_rcvq_maxlen;
3260 else if (maxqlen < IF_RCVQ_MINLEN)
3261 maxqlen = IF_RCVQ_MINLEN;
3262
3263 inp = ifp->if_inp;
3264 lck_mtx_lock(&inp->input_lck);
3265 qlimit(&inp->rcvq_pkts) = maxqlen;
3266 lck_mtx_unlock(&inp->input_lck);
3267
3268 return (0);
3269 }
3270
3271 errno_t
3272 ifnet_get_rcvq_maxlen(struct ifnet *ifp, u_int32_t *maxqlen)
3273 {
3274 struct dlil_threading_info *inp;
3275
3276 if (ifp == NULL || maxqlen == NULL)
3277 return (EINVAL);
3278 else if (!(ifp->if_eflags & IFEF_RXPOLL) || ifp->if_inp == NULL)
3279 return (ENXIO);
3280
3281 inp = ifp->if_inp;
3282 lck_mtx_lock(&inp->input_lck);
3283 *maxqlen = qlimit(&inp->rcvq_pkts);
3284 lck_mtx_unlock(&inp->input_lck);
3285 return (0);
3286 }
3287
3288 void
3289 ifnet_enqueue_multi_setup(struct ifnet *ifp, uint16_t delay_qlen,
3290 uint16_t delay_timeout)
3291 {
3292 if (delay_qlen > 0 && delay_timeout > 0) {
3293 ifp->if_eflags |= IFEF_ENQUEUE_MULTI;
3294 ifp->if_start_delay_qlen = min(100, delay_qlen);
3295 ifp->if_start_delay_timeout = min(20000, delay_timeout);
3296 /* convert timeout to nanoseconds */
3297 ifp->if_start_delay_timeout *= 1000;
3298 kprintf("%s: forced IFEF_ENQUEUE_MULTI qlen %u timeout %u\n",
3299 ifp->if_xname, (uint32_t)delay_qlen,
3300 (uint32_t)delay_timeout);
3301 } else {
3302 ifp->if_eflags &= ~IFEF_ENQUEUE_MULTI;
3303 }
3304 }
3305
3306 static inline errno_t
3307 ifnet_enqueue_common(struct ifnet *ifp, void *p, classq_pkt_type_t ptype,
3308 boolean_t flush, boolean_t *pdrop)
3309 {
3310 volatile uint64_t *fg_ts = NULL;
3311 volatile uint64_t *rt_ts = NULL;
3312 struct mbuf *m = p;
3313 struct timespec now;
3314 u_int64_t now_nsec = 0;
3315 int error = 0;
3316
3317 ASSERT(ifp->if_eflags & IFEF_TXSTART);
3318
3319 /*
3320 * If packet already carries a timestamp, either from dlil_output()
3321 * or from flowswitch, use it here. Otherwise, record timestamp.
3322 * PKTF_TS_VALID is always cleared prior to entering classq, i.e.
3323 * the timestamp value is used internally there.
3324 */
3325 switch (ptype) {
3326 case QP_MBUF:
3327 ASSERT(m->m_flags & M_PKTHDR);
3328 ASSERT(m->m_nextpkt == NULL);
3329
3330 if (!(m->m_pkthdr.pkt_flags & PKTF_TS_VALID) ||
3331 m->m_pkthdr.pkt_timestamp == 0) {
3332 nanouptime(&now);
3333 net_timernsec(&now, &now_nsec);
3334 m->m_pkthdr.pkt_timestamp = now_nsec;
3335 }
3336 m->m_pkthdr.pkt_flags &= ~PKTF_TS_VALID;
3337 /*
3338 * If the packet service class is not background,
3339 * update the timestamp to indicate recent activity
3340 * on a foreground socket.
3341 */
3342 if ((m->m_pkthdr.pkt_flags & PKTF_FLOW_ID) &&
3343 m->m_pkthdr.pkt_flowsrc == FLOWSRC_INPCB) {
3344 if (!(m->m_pkthdr.pkt_flags & PKTF_SO_BACKGROUND)) {
3345 ifp->if_fg_sendts = _net_uptime;
3346 if (fg_ts != NULL)
3347 *fg_ts = _net_uptime;
3348 }
3349 if (m->m_pkthdr.pkt_flags & PKTF_SO_REALTIME) {
3350 ifp->if_rt_sendts = _net_uptime;
3351 if (rt_ts != NULL)
3352 *rt_ts = _net_uptime;
3353 }
3354 }
3355 break;
3356
3357
3358 default:
3359 VERIFY(0);
3360 /* NOTREACHED */
3361 }
3362
3363 if (ifp->if_eflags & IFEF_ENQUEUE_MULTI) {
3364 if (now_nsec == 0) {
3365 nanouptime(&now);
3366 net_timernsec(&now, &now_nsec);
3367 }
3368 /*
3369 * If the driver chose to delay start callback for
3370 * coalescing multiple packets, Then use the following
3371 * heuristics to make sure that start callback will
3372 * be delayed only when bulk data transfer is detected.
3373 * 1. number of packets enqueued in (delay_win * 2) is
3374 * greater than or equal to the delay qlen.
3375 * 2. If delay_start is enabled it will stay enabled for
3376 * another 10 idle windows. This is to take into account
3377 * variable RTT and burst traffic.
3378 * 3. If the time elapsed since last enqueue is more
3379 * than 200ms we disable delaying start callback. This is
3380 * is to take idle time into account.
3381 */
3382 u_int64_t dwin = (ifp->if_start_delay_timeout << 1);
3383 if (ifp->if_start_delay_swin > 0) {
3384 if ((ifp->if_start_delay_swin + dwin) > now_nsec) {
3385 ifp->if_start_delay_cnt++;
3386 } else if ((now_nsec - ifp->if_start_delay_swin)
3387 >= (200 * 1000 * 1000)) {
3388 ifp->if_start_delay_swin = now_nsec;
3389 ifp->if_start_delay_cnt = 1;
3390 ifp->if_start_delay_idle = 0;
3391 if (ifp->if_eflags & IFEF_DELAY_START) {
3392 ifp->if_eflags &=
3393 ~(IFEF_DELAY_START);
3394 ifnet_delay_start_disabled++;
3395 }
3396 } else {
3397 if (ifp->if_start_delay_cnt >=
3398 ifp->if_start_delay_qlen) {
3399 ifp->if_eflags |= IFEF_DELAY_START;
3400 ifp->if_start_delay_idle = 0;
3401 } else {
3402 if (ifp->if_start_delay_idle >= 10) {
3403 ifp->if_eflags &= ~(IFEF_DELAY_START);
3404 ifnet_delay_start_disabled++;
3405 } else {
3406 ifp->if_start_delay_idle++;
3407 }
3408 }
3409 ifp->if_start_delay_swin = now_nsec;
3410 ifp->if_start_delay_cnt = 1;
3411 }
3412 } else {
3413 ifp->if_start_delay_swin = now_nsec;
3414 ifp->if_start_delay_cnt = 1;
3415 ifp->if_start_delay_idle = 0;
3416 ifp->if_eflags &= ~(IFEF_DELAY_START);
3417 }
3418 } else {
3419 ifp->if_eflags &= ~(IFEF_DELAY_START);
3420 }
3421
3422 switch (ptype) {
3423 case QP_MBUF:
3424 /* enqueue the packet (caller consumes object) */
3425 error = ifclassq_enqueue(&ifp->if_snd, m, QP_MBUF, pdrop);
3426 m = NULL;
3427 break;
3428
3429
3430 default:
3431 break;
3432 }
3433
3434 /*
3435 * Tell the driver to start dequeueing; do this even when the queue
3436 * for the packet is suspended (EQSUSPENDED), as the driver could still
3437 * be dequeueing from other unsuspended queues.
3438 */
3439 if (!(ifp->if_eflags & IFEF_ENQUEUE_MULTI) &&
3440 ((error == 0 && flush) || error == EQFULL || error == EQSUSPENDED))
3441 ifnet_start(ifp);
3442
3443 return (error);
3444 }
3445
3446 errno_t
3447 ifnet_enqueue(struct ifnet *ifp, struct mbuf *m)
3448 {
3449 boolean_t pdrop;
3450 return (ifnet_enqueue_mbuf(ifp, m, TRUE, &pdrop));
3451 }
3452
3453 errno_t
3454 ifnet_enqueue_mbuf(struct ifnet *ifp, struct mbuf *m, boolean_t flush,
3455 boolean_t *pdrop)
3456 {
3457 if (ifp == NULL || m == NULL || !(m->m_flags & M_PKTHDR) ||
3458 m->m_nextpkt != NULL) {
3459 if (m != NULL) {
3460 m_freem_list(m);
3461 *pdrop = TRUE;
3462 }
3463 return (EINVAL);
3464 } else if (!(ifp->if_eflags & IFEF_TXSTART) ||
3465 !IF_FULLY_ATTACHED(ifp)) {
3466 /* flag tested without lock for performance */
3467 m_freem(m);
3468 *pdrop = TRUE;
3469 return (ENXIO);
3470 } else if (!(ifp->if_flags & IFF_UP)) {
3471 m_freem(m);
3472 *pdrop = TRUE;
3473 return (ENETDOWN);
3474 }
3475
3476 return (ifnet_enqueue_common(ifp, m, QP_MBUF, flush, pdrop));
3477 }
3478
3479
3480 errno_t
3481 ifnet_dequeue(struct ifnet *ifp, struct mbuf **mp)
3482 {
3483 errno_t rc;
3484 classq_pkt_type_t ptype;
3485 if (ifp == NULL || mp == NULL)
3486 return (EINVAL);
3487 else if (!(ifp->if_eflags & IFEF_TXSTART) ||
3488 ifp->if_output_sched_model >= IFNET_SCHED_MODEL_MAX)
3489 return (ENXIO);
3490 if (!ifnet_is_attached(ifp, 1))
3491 return (ENXIO);
3492
3493 rc = ifclassq_dequeue(&ifp->if_snd, 1, CLASSQ_DEQUEUE_MAX_BYTE_LIMIT,
3494 (void **)mp, NULL, NULL, NULL, &ptype);
3495 VERIFY((*mp == NULL) || (ptype == QP_MBUF));
3496 ifnet_decr_iorefcnt(ifp);
3497
3498 return (rc);
3499 }
3500
3501 errno_t
3502 ifnet_dequeue_service_class(struct ifnet *ifp, mbuf_svc_class_t sc,
3503 struct mbuf **mp)
3504 {
3505 errno_t rc;
3506 classq_pkt_type_t ptype;
3507 if (ifp == NULL || mp == NULL || !MBUF_VALID_SC(sc))
3508 return (EINVAL);
3509 else if (!(ifp->if_eflags & IFEF_TXSTART) ||
3510 ifp->if_output_sched_model >= IFNET_SCHED_MODEL_MAX)
3511 return (ENXIO);
3512 if (!ifnet_is_attached(ifp, 1))
3513 return (ENXIO);
3514
3515 rc = ifclassq_dequeue_sc(&ifp->if_snd, sc, 1,
3516 CLASSQ_DEQUEUE_MAX_BYTE_LIMIT, (void **)mp, NULL, NULL,
3517 NULL, &ptype);
3518 VERIFY((*mp == NULL) || (ptype == QP_MBUF));
3519 ifnet_decr_iorefcnt(ifp);
3520 return (rc);
3521 }
3522
3523 errno_t
3524 ifnet_dequeue_multi(struct ifnet *ifp, u_int32_t pkt_limit,
3525 struct mbuf **head, struct mbuf **tail, u_int32_t *cnt, u_int32_t *len)
3526 {
3527 errno_t rc;
3528 classq_pkt_type_t ptype;
3529 if (ifp == NULL || head == NULL || pkt_limit < 1)
3530 return (EINVAL);
3531 else if (!(ifp->if_eflags & IFEF_TXSTART) ||
3532 ifp->if_output_sched_model >= IFNET_SCHED_MODEL_MAX)
3533 return (ENXIO);
3534 if (!ifnet_is_attached(ifp, 1))
3535 return (ENXIO);
3536
3537 rc = ifclassq_dequeue(&ifp->if_snd, pkt_limit,
3538 CLASSQ_DEQUEUE_MAX_BYTE_LIMIT, (void **)head, (void **)tail, cnt,
3539 len, &ptype);
3540 VERIFY((*head == NULL) || (ptype == QP_MBUF));
3541 ifnet_decr_iorefcnt(ifp);
3542 return (rc);
3543 }
3544
3545 errno_t
3546 ifnet_dequeue_multi_bytes(struct ifnet *ifp, u_int32_t byte_limit,
3547 struct mbuf **head, struct mbuf **tail, u_int32_t *cnt, u_int32_t *len)
3548 {
3549 errno_t rc;
3550 classq_pkt_type_t ptype;
3551 if (ifp == NULL || head == NULL || byte_limit < 1)
3552 return (EINVAL);
3553 else if (!(ifp->if_eflags & IFEF_TXSTART) ||
3554 ifp->if_output_sched_model >= IFNET_SCHED_MODEL_MAX)
3555 return (ENXIO);
3556 if (!ifnet_is_attached(ifp, 1))
3557 return (ENXIO);
3558
3559 rc = ifclassq_dequeue(&ifp->if_snd, CLASSQ_DEQUEUE_MAX_PKT_LIMIT,
3560 byte_limit, (void **)head, (void **)tail, cnt, len, &ptype);
3561 VERIFY((*head == NULL) || (ptype == QP_MBUF));
3562 ifnet_decr_iorefcnt(ifp);
3563 return (rc);
3564 }
3565
3566 errno_t
3567 ifnet_dequeue_service_class_multi(struct ifnet *ifp, mbuf_svc_class_t sc,
3568 u_int32_t pkt_limit, struct mbuf **head, struct mbuf **tail, u_int32_t *cnt,
3569 u_int32_t *len)
3570 {
3571 errno_t rc;
3572 classq_pkt_type_t ptype;
3573 if (ifp == NULL || head == NULL || pkt_limit < 1 ||
3574 !MBUF_VALID_SC(sc))
3575 return (EINVAL);
3576 else if (!(ifp->if_eflags & IFEF_TXSTART) ||
3577 ifp->if_output_sched_model >= IFNET_SCHED_MODEL_MAX)
3578 return (ENXIO);
3579 if (!ifnet_is_attached(ifp, 1))
3580 return (ENXIO);
3581
3582 rc = ifclassq_dequeue_sc(&ifp->if_snd, sc, pkt_limit,
3583 CLASSQ_DEQUEUE_MAX_BYTE_LIMIT, (void **)head,
3584 (void **)tail, cnt, len, &ptype);
3585 VERIFY((*head == NULL) || (ptype == QP_MBUF));
3586 ifnet_decr_iorefcnt(ifp);
3587 return (rc);
3588 }
3589
3590 #if !CONFIG_EMBEDDED
3591 errno_t
3592 ifnet_framer_stub(struct ifnet *ifp, struct mbuf **m,
3593 const struct sockaddr *dest, const char *dest_linkaddr,
3594 const char *frame_type, u_int32_t *pre, u_int32_t *post)
3595 {
3596 if (pre != NULL)
3597 *pre = 0;
3598 if (post != NULL)
3599 *post = 0;
3600
3601 return (ifp->if_framer_legacy(ifp, m, dest, dest_linkaddr, frame_type));
3602 }
3603 #endif /* !CONFIG_EMBEDDED */
3604
3605 static int
3606 dlil_interface_filters_input(struct ifnet *ifp, struct mbuf **m_p,
3607 char **frame_header_p, protocol_family_t protocol_family)
3608 {
3609 struct ifnet_filter *filter;
3610
3611 /*
3612 * Pass the inbound packet to the interface filters
3613 */
3614 lck_mtx_lock_spin(&ifp->if_flt_lock);
3615 /* prevent filter list from changing in case we drop the lock */
3616 if_flt_monitor_busy(ifp);
3617 TAILQ_FOREACH(filter, &ifp->if_flt_head, filt_next) {
3618 int result;
3619
3620 if (!filter->filt_skip && filter->filt_input != NULL &&
3621 (filter->filt_protocol == 0 ||
3622 filter->filt_protocol == protocol_family)) {
3623 lck_mtx_unlock(&ifp->if_flt_lock);
3624
3625 result = (*filter->filt_input)(filter->filt_cookie,
3626 ifp, protocol_family, m_p, frame_header_p);
3627
3628 lck_mtx_lock_spin(&ifp->if_flt_lock);
3629 if (result != 0) {
3630 /* we're done with the filter list */
3631 if_flt_monitor_unbusy(ifp);
3632 lck_mtx_unlock(&ifp->if_flt_lock);
3633 return (result);
3634 }
3635 }
3636 }
3637 /* we're done with the filter list */
3638 if_flt_monitor_unbusy(ifp);
3639 lck_mtx_unlock(&ifp->if_flt_lock);
3640
3641 /*
3642 * Strip away M_PROTO1 bit prior to sending packet up the stack as
3643 * it is meant to be local to a subsystem -- if_bridge for M_PROTO1
3644 */
3645 if (*m_p != NULL)
3646 (*m_p)->m_flags &= ~M_PROTO1;
3647
3648 return (0);
3649 }
3650
3651 static int
3652 dlil_interface_filters_output(struct ifnet *ifp, struct mbuf **m_p,
3653 protocol_family_t protocol_family)
3654 {
3655 struct ifnet_filter *filter;
3656
3657 /*
3658 * Pass the outbound packet to the interface filters
3659 */
3660 lck_mtx_lock_spin(&ifp->if_flt_lock);
3661 /* prevent filter list from changing in case we drop the lock */
3662 if_flt_monitor_busy(ifp);
3663 TAILQ_FOREACH(filter, &ifp->if_flt_head, filt_next) {
3664 int result;
3665
3666 if (!filter->filt_skip && filter->filt_output != NULL &&
3667 (filter->filt_protocol == 0 ||
3668 filter->filt_protocol == protocol_family)) {
3669 lck_mtx_unlock(&ifp->if_flt_lock);
3670
3671 result = filter->filt_output(filter->filt_cookie, ifp,
3672 protocol_family, m_p);
3673
3674 lck_mtx_lock_spin(&ifp->if_flt_lock);
3675 if (result != 0) {
3676 /* we're done with the filter list */
3677 if_flt_monitor_unbusy(ifp);
3678 lck_mtx_unlock(&ifp->if_flt_lock);
3679 return (result);
3680 }
3681 }
3682 }
3683 /* we're done with the filter list */
3684 if_flt_monitor_unbusy(ifp);
3685 lck_mtx_unlock(&ifp->if_flt_lock);
3686
3687 return (0);
3688 }
3689
3690 static void
3691 dlil_ifproto_input(struct if_proto * ifproto, mbuf_t m)
3692 {
3693 int error;
3694
3695 if (ifproto->proto_kpi == kProtoKPI_v1) {
3696 /* Version 1 protocols get one packet at a time */
3697 while (m != NULL) {
3698 char * frame_header;
3699 mbuf_t next_packet;
3700
3701 next_packet = m->m_nextpkt;
3702 m->m_nextpkt = NULL;
3703 frame_header = m->m_pkthdr.pkt_hdr;
3704 m->m_pkthdr.pkt_hdr = NULL;
3705 error = (*ifproto->kpi.v1.input)(ifproto->ifp,
3706 ifproto->protocol_family, m, frame_header);
3707 if (error != 0 && error != EJUSTRETURN)
3708 m_freem(m);
3709 m = next_packet;
3710 }
3711 } else if (ifproto->proto_kpi == kProtoKPI_v2) {
3712 /* Version 2 protocols support packet lists */
3713 error = (*ifproto->kpi.v2.input)(ifproto->ifp,
3714 ifproto->protocol_family, m);
3715 if (error != 0 && error != EJUSTRETURN)
3716 m_freem_list(m);
3717 }
3718 }
3719
3720 static void
3721 dlil_input_stats_add(const struct ifnet_stat_increment_param *s,
3722 struct dlil_threading_info *inp, boolean_t poll)
3723 {
3724 struct ifnet_stat_increment_param *d = &inp->stats;
3725
3726 if (s->packets_in != 0)
3727 d->packets_in += s->packets_in;
3728 if (s->bytes_in != 0)
3729 d->bytes_in += s->bytes_in;
3730 if (s->errors_in != 0)
3731 d->errors_in += s->errors_in;
3732
3733 if (s->packets_out != 0)
3734 d->packets_out += s->packets_out;
3735 if (s->bytes_out != 0)
3736 d->bytes_out += s->bytes_out;
3737 if (s->errors_out != 0)
3738 d->errors_out += s->errors_out;
3739
3740 if (s->collisions != 0)
3741 d->collisions += s->collisions;
3742 if (s->dropped != 0)
3743 d->dropped += s->dropped;
3744
3745 if (poll)
3746 PKTCNTR_ADD(&inp->tstats, s->packets_in, s->bytes_in);
3747 }
3748
3749 static void
3750 dlil_input_stats_sync(struct ifnet *ifp, struct dlil_threading_info *inp)
3751 {
3752 struct ifnet_stat_increment_param *s = &inp->stats;
3753
3754 /*
3755 * Use of atomic operations is unavoidable here because
3756 * these stats may also be incremented elsewhere via KPIs.
3757 */
3758 if (s->packets_in != 0) {
3759 atomic_add_64(&ifp->if_data.ifi_ipackets, s->packets_in);
3760 s->packets_in = 0;
3761 }
3762 if (s->bytes_in != 0) {
3763 atomic_add_64(&ifp->if_data.ifi_ibytes, s->bytes_in);
3764 s->bytes_in = 0;
3765 }
3766 if (s->errors_in != 0) {
3767 atomic_add_64(&ifp->if_data.ifi_ierrors, s->errors_in);
3768 s->errors_in = 0;
3769 }
3770
3771 if (s->packets_out != 0) {
3772 atomic_add_64(&ifp->if_data.ifi_opackets, s->packets_out);
3773 s->packets_out = 0;
3774 }
3775 if (s->bytes_out != 0) {
3776 atomic_add_64(&ifp->if_data.ifi_obytes, s->bytes_out);
3777 s->bytes_out = 0;
3778 }
3779 if (s->errors_out != 0) {
3780 atomic_add_64(&ifp->if_data.ifi_oerrors, s->errors_out);
3781 s->errors_out = 0;
3782 }
3783
3784 if (s->collisions != 0) {
3785 atomic_add_64(&ifp->if_data.ifi_collisions, s->collisions);
3786 s->collisions = 0;
3787 }
3788 if (s->dropped != 0) {
3789 atomic_add_64(&ifp->if_data.ifi_iqdrops, s->dropped);
3790 s->dropped = 0;
3791 }
3792
3793 if (ifp->if_data_threshold != 0) {
3794 lck_mtx_convert_spin(&inp->input_lck);
3795 ifnet_notify_data_threshold(ifp);
3796 }
3797
3798 /*
3799 * No need for atomic operations as they are modified here
3800 * only from within the DLIL input thread context.
3801 */
3802 if (inp->tstats.packets != 0) {
3803 inp->pstats.ifi_poll_packets += inp->tstats.packets;
3804 inp->tstats.packets = 0;
3805 }
3806 if (inp->tstats.bytes != 0) {
3807 inp->pstats.ifi_poll_bytes += inp->tstats.bytes;
3808 inp->tstats.bytes = 0;
3809 }
3810 }
3811
3812 __private_extern__ void
3813 dlil_input_packet_list(struct ifnet *ifp, struct mbuf *m)
3814 {
3815 return (dlil_input_packet_list_common(ifp, m, 0,
3816 IFNET_MODEL_INPUT_POLL_OFF, FALSE));
3817 }
3818
3819 __private_extern__ void
3820 dlil_input_packet_list_extended(struct ifnet *ifp, struct mbuf *m,
3821 u_int32_t cnt, ifnet_model_t mode)
3822 {
3823 return (dlil_input_packet_list_common(ifp, m, cnt, mode, TRUE));
3824 }
3825
3826 static void
3827 dlil_input_packet_list_common(struct ifnet *ifp_param, struct mbuf *m,
3828 u_int32_t cnt, ifnet_model_t mode, boolean_t ext)
3829 {
3830 int error = 0;
3831 protocol_family_t protocol_family;
3832 mbuf_t next_packet;
3833 ifnet_t ifp = ifp_param;
3834 char *frame_header = NULL;
3835 struct if_proto *last_ifproto = NULL;
3836 mbuf_t pkt_first = NULL;
3837 mbuf_t *pkt_next = NULL;
3838 u_int32_t poll_thresh = 0, poll_ival = 0;
3839
3840 KERNEL_DEBUG(DBG_FNC_DLIL_INPUT | DBG_FUNC_START, 0, 0, 0, 0, 0);
3841
3842 if (ext && mode == IFNET_MODEL_INPUT_POLL_ON && cnt > 1 &&
3843 (poll_ival = if_rxpoll_interval_pkts) > 0)
3844 poll_thresh = cnt;
3845
3846 while (m != NULL) {
3847 struct if_proto *ifproto = NULL;
3848 int iorefcnt = 0;
3849 uint32_t pktf_mask; /* pkt flags to preserve */
3850
3851 if (ifp_param == NULL)
3852 ifp = m->m_pkthdr.rcvif;
3853
3854 if ((ifp->if_eflags & IFEF_RXPOLL) && poll_thresh != 0 &&
3855 poll_ival > 0 && (--poll_thresh % poll_ival) == 0)
3856 ifnet_poll(ifp);
3857
3858 /* Check if this mbuf looks valid */
3859 MBUF_INPUT_CHECK(m, ifp);
3860
3861 next_packet = m->m_nextpkt;
3862 m->m_nextpkt = NULL;
3863 frame_header = m->m_pkthdr.pkt_hdr;
3864 m->m_pkthdr.pkt_hdr = NULL;
3865
3866 /*
3867 * Get an IO reference count if the interface is not
3868 * loopback (lo0) and it is attached; lo0 never goes
3869 * away, so optimize for that.
3870 */
3871 if (ifp != lo_ifp) {
3872 if (!ifnet_is_attached(ifp, 1)) {
3873 m_freem(m);
3874 goto next;
3875 }
3876 iorefcnt = 1;
3877 /*
3878 * Preserve the time stamp if it was set.
3879 */
3880 pktf_mask = PKTF_TS_VALID;
3881 } else {
3882 /*
3883 * If this arrived on lo0, preserve interface addr
3884 * info to allow for connectivity between loopback
3885 * and local interface addresses.
3886 */
3887 pktf_mask = (PKTF_LOOP|PKTF_IFAINFO);
3888 }
3889
3890 /* make sure packet comes in clean */
3891 m_classifier_init(m, pktf_mask);
3892
3893 ifp_inc_traffic_class_in(ifp, m);
3894
3895 /* find which protocol family this packet is for */
3896 ifnet_lock_shared(ifp);
3897 error = (*ifp->if_demux)(ifp, m, frame_header,
3898 &protocol_family);
3899 ifnet_lock_done(ifp);
3900 if (error != 0) {
3901 if (error == EJUSTRETURN)
3902 goto next;
3903 protocol_family = 0;
3904 }
3905
3906 pktap_input(ifp, protocol_family, m, frame_header);
3907
3908 /* Drop v4 packets received on CLAT46 enabled interface */
3909 if (protocol_family == PF_INET && IS_INTF_CLAT46(ifp)) {
3910 m_freem(m);
3911 ip6stat.ip6s_clat464_in_v4_drop++;
3912 goto next;
3913 }
3914
3915 /* Translate the packet if it is received on CLAT interface */
3916 if (protocol_family == PF_INET6 && IS_INTF_CLAT46(ifp)
3917 && dlil_is_clat_needed(protocol_family, m)) {
3918 char *data = NULL;
3919 struct ether_header eh;
3920 struct ether_header *ehp = NULL;
3921
3922 if (ifp->if_type == IFT_ETHER) {
3923 ehp = (struct ether_header *)(void *)frame_header;
3924 /* Skip RX Ethernet packets if they are not IPV6 */
3925 if (ntohs(ehp->ether_type) != ETHERTYPE_IPV6)
3926 goto skip_clat;
3927
3928 /* Keep a copy of frame_header for Ethernet packets */
3929 bcopy(frame_header, (caddr_t)&eh, ETHER_HDR_LEN);
3930 }
3931 error = dlil_clat64(ifp, &protocol_family, &m);
3932 data = (char *) mbuf_data(m);
3933 if (error != 0) {
3934 m_freem(m);
3935 ip6stat.ip6s_clat464_in_drop++;
3936 goto next;
3937 }
3938 /* Native v6 should be No-op */
3939 if (protocol_family != PF_INET)
3940 goto skip_clat;
3941
3942 /* Do this only for translated v4 packets. */
3943 switch (ifp->if_type) {
3944 case IFT_CELLULAR:
3945 frame_header = data;
3946 break;
3947 case IFT_ETHER:
3948 /*
3949 * Drop if the mbuf doesn't have enough
3950 * space for Ethernet header
3951 */
3952 if (M_LEADINGSPACE(m) < ETHER_HDR_LEN) {
3953 m_free(m);
3954 ip6stat.ip6s_clat464_in_drop++;
3955 goto next;
3956 }
3957 /*
3958 * Set the frame_header ETHER_HDR_LEN bytes
3959 * preceeding the data pointer. Change
3960 * the ether_type too.
3961 */
3962 frame_header = data - ETHER_HDR_LEN;
3963 eh.ether_type = htons(ETHERTYPE_IP);
3964 bcopy((caddr_t)&eh, frame_header, ETHER_HDR_LEN);
3965 break;
3966 }
3967 }
3968 skip_clat:
3969 if (hwcksum_dbg != 0 && !(ifp->if_flags & IFF_LOOPBACK) &&
3970 !(m->m_pkthdr.pkt_flags & PKTF_LOOP))
3971 dlil_input_cksum_dbg(ifp, m, frame_header,
3972 protocol_family);
3973
3974 /*
3975 * For partial checksum offload, we expect the driver to
3976 * set the start offset indicating the start of the span
3977 * that is covered by the hardware-computed checksum;
3978 * adjust this start offset accordingly because the data
3979 * pointer has been advanced beyond the link-layer header.
3980 *
3981 * Don't adjust if the interface is a bridge member, as
3982 * the adjustment will occur from the context of the
3983 * bridge interface during input.
3984 */
3985 if (ifp->if_bridge == NULL && (m->m_pkthdr.csum_flags &
3986 (CSUM_DATA_VALID | CSUM_PARTIAL)) ==
3987 (CSUM_DATA_VALID | CSUM_PARTIAL)) {
3988 int adj;
3989 if (frame_header == NULL ||
3990 frame_header < (char *)mbuf_datastart(m) ||
3991 frame_header > (char *)m->m_data ||
3992 (adj = (m->m_data - frame_header)) >
3993 m->m_pkthdr.csum_rx_start) {
3994 m->m_pkthdr.csum_data = 0;
3995 m->m_pkthdr.csum_flags &= ~CSUM_DATA_VALID;
3996 hwcksum_in_invalidated++;
3997 } else {
3998 m->m_pkthdr.csum_rx_start -= adj;
3999 }
4000 }
4001
4002 if (clat_debug)
4003 pktap_input(ifp, protocol_family, m, frame_header);
4004
4005 if (m->m_flags & (M_BCAST|M_MCAST))
4006 atomic_add_64(&ifp->if_imcasts, 1);
4007
4008 /* run interface filters, exclude VLAN packets PR-3586856 */
4009 if ((m->m_pkthdr.csum_flags & CSUM_VLAN_TAG_VALID) == 0) {
4010 error = dlil_interface_filters_input(ifp, &m,
4011 &frame_header, protocol_family);
4012 if (error != 0) {
4013 if (error != EJUSTRETURN)
4014 m_freem(m);
4015 goto next;
4016 }
4017 }
4018 if (error != 0 || ((m->m_flags & M_PROMISC) != 0)) {
4019 m_freem(m);
4020 goto next;
4021 }
4022
4023 /* Lookup the protocol attachment to this interface */
4024 if (protocol_family == 0) {
4025 ifproto = NULL;
4026 } else if (last_ifproto != NULL && last_ifproto->ifp == ifp &&
4027 (last_ifproto->protocol_family == protocol_family)) {
4028 VERIFY(ifproto == NULL);
4029 ifproto = last_ifproto;
4030 if_proto_ref(last_ifproto);
4031 } else {
4032 VERIFY(ifproto == NULL);
4033 ifnet_lock_shared(ifp);
4034 /* callee holds a proto refcnt upon success */
4035 ifproto = find_attached_proto(ifp, protocol_family);
4036 ifnet_lock_done(ifp);
4037 }
4038 if (ifproto == NULL) {
4039 /* no protocol for this packet, discard */
4040 m_freem(m);
4041 goto next;
4042 }
4043 if (ifproto != last_ifproto) {
4044 if (last_ifproto != NULL) {
4045 /* pass up the list for the previous protocol */
4046 dlil_ifproto_input(last_ifproto, pkt_first);
4047 pkt_first = NULL;
4048 if_proto_free(last_ifproto);
4049 }
4050 last_ifproto = ifproto;
4051 if_proto_ref(ifproto);
4052 }
4053 /* extend the list */
4054 m->m_pkthdr.pkt_hdr = frame_header;
4055 if (pkt_first == NULL) {
4056 pkt_first = m;
4057 } else {
4058 *pkt_next = m;
4059 }
4060 pkt_next = &m->m_nextpkt;
4061
4062 next:
4063 if (next_packet == NULL && last_ifproto != NULL) {
4064 /* pass up the last list of packets */
4065 dlil_ifproto_input(last_ifproto, pkt_first);
4066 if_proto_free(last_ifproto);
4067 last_ifproto = NULL;
4068 }
4069 if (ifproto != NULL) {
4070 if_proto_free(ifproto);
4071 ifproto = NULL;
4072 }
4073
4074 m = next_packet;
4075
4076 /* update the driver's multicast filter, if needed */
4077 if (ifp->if_updatemcasts > 0 && if_mcasts_update(ifp) == 0)
4078 ifp->if_updatemcasts = 0;
4079 if (iorefcnt == 1)
4080 ifnet_decr_iorefcnt(ifp);
4081 }
4082
4083 KERNEL_DEBUG(DBG_FNC_DLIL_INPUT | DBG_FUNC_END, 0, 0, 0, 0, 0);
4084 }
4085
4086 errno_t
4087 if_mcasts_update(struct ifnet *ifp)
4088 {
4089 errno_t err;
4090
4091 err = ifnet_ioctl(ifp, 0, SIOCADDMULTI, NULL);
4092 if (err == EAFNOSUPPORT)
4093 err = 0;
4094 printf("%s: %s %d suspended link-layer multicast membership(s) "
4095 "(err=%d)\n", if_name(ifp),
4096 (err == 0 ? "successfully restored" : "failed to restore"),
4097 ifp->if_updatemcasts, err);
4098
4099 /* just return success */
4100 return (0);
4101 }
4102
4103 /* If ifp is set, we will increment the generation for the interface */
4104 int
4105 dlil_post_complete_msg(struct ifnet *ifp, struct kev_msg *event)
4106 {
4107 if (ifp != NULL) {
4108 ifnet_increment_generation(ifp);
4109 }
4110
4111 #if NECP
4112 necp_update_all_clients();
4113 #endif /* NECP */
4114
4115 return (kev_post_msg(event));
4116 }
4117
4118 __private_extern__ void
4119 dlil_post_sifflags_msg(struct ifnet * ifp)
4120 {
4121 struct kev_msg ev_msg;
4122 struct net_event_data ev_data;
4123
4124 bzero(&ev_data, sizeof (ev_data));
4125 bzero(&ev_msg, sizeof (ev_msg));
4126 ev_msg.vendor_code = KEV_VENDOR_APPLE;
4127 ev_msg.kev_class = KEV_NETWORK_CLASS;
4128 ev_msg.kev_subclass = KEV_DL_SUBCLASS;
4129 ev_msg.event_code = KEV_DL_SIFFLAGS;
4130 strlcpy(&ev_data.if_name[0], ifp->if_name, IFNAMSIZ);
4131 ev_data.if_family = ifp->if_family;
4132 ev_data.if_unit = (u_int32_t) ifp->if_unit;
4133 ev_msg.dv[0].data_length = sizeof(struct net_event_data);
4134 ev_msg.dv[0].data_ptr = &ev_data;
4135 ev_msg.dv[1].data_length = 0;
4136 dlil_post_complete_msg(ifp, &ev_msg);
4137 }
4138
4139 #define TMP_IF_PROTO_ARR_SIZE 10
4140 static int
4141 dlil_event_internal(struct ifnet *ifp, struct kev_msg *event, bool update_generation)
4142 {
4143 struct ifnet_filter *filter = NULL;
4144 struct if_proto *proto = NULL;
4145 int if_proto_count = 0;
4146 struct if_proto **tmp_ifproto_arr = NULL;
4147 struct if_proto *tmp_ifproto_stack_arr[TMP_IF_PROTO_ARR_SIZE] = {NULL};
4148 int tmp_ifproto_arr_idx = 0;
4149 bool tmp_malloc = false;
4150
4151 /*
4152 * Pass the event to the interface filters
4153 */
4154 lck_mtx_lock_spin(&ifp->if_flt_lock);
4155 /* prevent filter list from changing in case we drop the lock */
4156 if_flt_monitor_busy(ifp);
4157 TAILQ_FOREACH(filter, &ifp->if_flt_head, filt_next) {
4158 if (filter->filt_event != NULL) {
4159 lck_mtx_unlock(&ifp->if_flt_lock);
4160
4161 filter->filt_event(filter->filt_cookie, ifp,
4162 filter->filt_protocol, event);
4163
4164 lck_mtx_lock_spin(&ifp->if_flt_lock);
4165 }
4166 }
4167 /* we're done with the filter list */
4168 if_flt_monitor_unbusy(ifp);
4169 lck_mtx_unlock(&ifp->if_flt_lock);
4170
4171 /* Get an io ref count if the interface is attached */
4172 if (!ifnet_is_attached(ifp, 1))
4173 goto done;
4174
4175 /*
4176 * An embedded tmp_list_entry in if_proto may still get
4177 * over-written by another thread after giving up ifnet lock,
4178 * therefore we are avoiding embedded pointers here.
4179 */
4180 ifnet_lock_shared(ifp);
4181 if_proto_count = dlil_ifp_protolist(ifp, NULL, 0);
4182 if (if_proto_count) {
4183 int i;
4184 VERIFY(ifp->if_proto_hash != NULL);
4185 if (if_proto_count <= TMP_IF_PROTO_ARR_SIZE) {
4186 tmp_ifproto_arr = tmp_ifproto_stack_arr;
4187 } else {
4188 MALLOC(tmp_ifproto_arr, struct if_proto **,
4189 sizeof (*tmp_ifproto_arr) * if_proto_count,
4190 M_TEMP, M_ZERO);
4191 if (tmp_ifproto_arr == NULL) {
4192 ifnet_lock_done(ifp);
4193 goto cleanup;
4194 }
4195 tmp_malloc = true;
4196 }
4197
4198 for (i = 0; i < PROTO_HASH_SLOTS; i++) {
4199 SLIST_FOREACH(proto, &ifp->if_proto_hash[i],
4200 next_hash) {
4201 if_proto_ref(proto);
4202 tmp_ifproto_arr[tmp_ifproto_arr_idx] = proto;
4203 tmp_ifproto_arr_idx++;
4204 }
4205 }
4206 VERIFY(if_proto_count == tmp_ifproto_arr_idx);
4207 }
4208 ifnet_lock_done(ifp);
4209
4210 for (tmp_ifproto_arr_idx = 0; tmp_ifproto_arr_idx < if_proto_count;
4211 tmp_ifproto_arr_idx++) {
4212 proto = tmp_ifproto_arr[tmp_ifproto_arr_idx];
4213 VERIFY(proto != NULL);
4214 proto_media_event eventp =
4215 (proto->proto_kpi == kProtoKPI_v1 ?
4216 proto->kpi.v1.event :
4217 proto->kpi.v2.event);
4218
4219 if (eventp != NULL) {
4220 eventp(ifp, proto->protocol_family,
4221 event);
4222 }
4223 if_proto_free(proto);
4224 }
4225
4226 cleanup:
4227 if (tmp_malloc) {
4228 FREE(tmp_ifproto_arr, M_TEMP);
4229 }
4230
4231 /* Pass the event to the interface */
4232 if (ifp->if_event != NULL)
4233 ifp->if_event(ifp, event);
4234
4235 /* Release the io ref count */
4236 ifnet_decr_iorefcnt(ifp);
4237 done:
4238 return (dlil_post_complete_msg(update_generation ? ifp : NULL, event));
4239 }
4240
4241 errno_t
4242 ifnet_event(ifnet_t ifp, struct kern_event_msg *event)
4243 {
4244 struct kev_msg kev_msg;
4245 int result = 0;
4246
4247 if (ifp == NULL || event == NULL)
4248 return (EINVAL);
4249
4250 bzero(&kev_msg, sizeof (kev_msg));
4251 kev_msg.vendor_code = event->vendor_code;
4252 kev_msg.kev_class = event->kev_class;
4253 kev_msg.kev_subclass = event->kev_subclass;
4254 kev_msg.event_code = event->event_code;
4255 kev_msg.dv[0].data_ptr = &event->event_data[0];
4256 kev_msg.dv[0].data_length = event->total_size - KEV_MSG_HEADER_SIZE;
4257 kev_msg.dv[1].data_length = 0;
4258
4259 result = dlil_event_internal(ifp, &kev_msg, TRUE);
4260
4261 return (result);
4262 }
4263
4264 #if CONFIG_MACF_NET
4265 #include <netinet/ip6.h>
4266 #include <netinet/ip.h>
4267 static int
4268 dlil_get_socket_type(struct mbuf **mp, int family, int raw)
4269 {
4270 struct mbuf *m;
4271 struct ip *ip;
4272 struct ip6_hdr *ip6;
4273 int type = SOCK_RAW;
4274
4275 if (!raw) {
4276 switch (family) {
4277 case PF_INET:
4278 m = m_pullup(*mp, sizeof(struct ip));
4279 if (m == NULL)
4280 break;
4281 *mp = m;
4282 ip = mtod(m, struct ip *);
4283 if (ip->ip_p == IPPROTO_TCP)
4284 type = SOCK_STREAM;
4285 else if (ip->ip_p == IPPROTO_UDP)
4286 type = SOCK_DGRAM;
4287 break;
4288 case PF_INET6:
4289 m = m_pullup(*mp, sizeof(struct ip6_hdr));
4290 if (m == NULL)
4291 break;
4292 *mp = m;
4293 ip6 = mtod(m, struct ip6_hdr *);
4294 if (ip6->ip6_nxt == IPPROTO_TCP)
4295 type = SOCK_STREAM;
4296 else if (ip6->ip6_nxt == IPPROTO_UDP)
4297 type = SOCK_DGRAM;
4298 break;
4299 }
4300 }
4301
4302 return (type);
4303 }
4304 #endif
4305
4306 static void
4307 dlil_count_chain_len(mbuf_t m, struct chain_len_stats *cls)
4308 {
4309 mbuf_t n = m;
4310 int chainlen = 0;
4311
4312 while (n != NULL) {
4313 chainlen++;
4314 n = n->m_next;
4315 }
4316 switch (chainlen) {
4317 case 0:
4318 break;
4319 case 1:
4320 atomic_add_64(&cls->cls_one, 1);
4321 break;
4322 case 2:
4323 atomic_add_64(&cls->cls_two, 1);
4324 break;
4325 case 3:
4326 atomic_add_64(&cls->cls_three, 1);
4327 break;
4328 case 4:
4329 atomic_add_64(&cls->cls_four, 1);
4330 break;
4331 case 5:
4332 default:
4333 atomic_add_64(&cls->cls_five_or_more, 1);
4334 break;
4335 }
4336 }
4337
4338 /*
4339 * dlil_output
4340 *
4341 * Caller should have a lock on the protocol domain if the protocol
4342 * doesn't support finer grained locking. In most cases, the lock
4343 * will be held from the socket layer and won't be released until
4344 * we return back to the socket layer.
4345 *
4346 * This does mean that we must take a protocol lock before we take
4347 * an interface lock if we're going to take both. This makes sense
4348 * because a protocol is likely to interact with an ifp while it
4349 * is under the protocol lock.
4350 *
4351 * An advisory code will be returned if adv is not null. This
4352 * can be used to provide feedback about interface queues to the
4353 * application.
4354 */
4355 errno_t
4356 dlil_output(ifnet_t ifp, protocol_family_t proto_family, mbuf_t packetlist,
4357 void *route, const struct sockaddr *dest, int raw, struct flowadv *adv)
4358 {
4359 char *frame_type = NULL;
4360 char *dst_linkaddr = NULL;
4361 int retval = 0;
4362 char frame_type_buffer[MAX_FRAME_TYPE_SIZE * 4];
4363 char dst_linkaddr_buffer[MAX_LINKADDR * 4];
4364 struct if_proto *proto = NULL;
4365 mbuf_t m = NULL;
4366 mbuf_t send_head = NULL;
4367 mbuf_t *send_tail = &send_head;
4368 int iorefcnt = 0;
4369 u_int32_t pre = 0, post = 0;
4370 u_int32_t fpkts = 0, fbytes = 0;
4371 int32_t flen = 0;
4372 struct timespec now;
4373 u_int64_t now_nsec;
4374 boolean_t did_clat46 = FALSE;
4375 protocol_family_t old_proto_family = proto_family;
4376 struct rtentry *rt = NULL;
4377
4378 KERNEL_DEBUG(DBG_FNC_DLIL_OUTPUT | DBG_FUNC_START, 0, 0, 0, 0, 0);
4379
4380 /*
4381 * Get an io refcnt if the interface is attached to prevent ifnet_detach
4382 * from happening while this operation is in progress
4383 */
4384 if (!ifnet_is_attached(ifp, 1)) {
4385 retval = ENXIO;
4386 goto cleanup;
4387 }
4388 iorefcnt = 1;
4389
4390 VERIFY(ifp->if_output_dlil != NULL);
4391
4392 /* update the driver's multicast filter, if needed */
4393 if (ifp->if_updatemcasts > 0 && if_mcasts_update(ifp) == 0)
4394 ifp->if_updatemcasts = 0;
4395
4396 frame_type = frame_type_buffer;
4397 dst_linkaddr = dst_linkaddr_buffer;
4398
4399 if (raw == 0) {
4400 ifnet_lock_shared(ifp);
4401 /* callee holds a proto refcnt upon success */
4402 proto = find_attached_proto(ifp, proto_family);
4403 if (proto == NULL) {
4404 ifnet_lock_done(ifp);
4405 retval = ENXIO;
4406 goto cleanup;
4407 }
4408 ifnet_lock_done(ifp);
4409 }
4410
4411 preout_again:
4412 if (packetlist == NULL)
4413 goto cleanup;
4414
4415 m = packetlist;
4416 packetlist = packetlist->m_nextpkt;
4417 m->m_nextpkt = NULL;
4418
4419 /*
4420 * Perform address family translation for the first
4421 * packet outside the loop in order to perform address
4422 * lookup for the translated proto family.
4423 */
4424 if (proto_family == PF_INET && IS_INTF_CLAT46(ifp) &&
4425 (ifp->if_type == IFT_CELLULAR ||
4426 dlil_is_clat_needed(proto_family, m))) {
4427 retval = dlil_clat46(ifp, &proto_family, &m);
4428 /*
4429 * Go to the next packet if translation fails
4430 */
4431 if (retval != 0) {
4432 m_freem(m);
4433 m = NULL;
4434 ip6stat.ip6s_clat464_out_drop++;
4435 /* Make sure that the proto family is PF_INET */
4436 ASSERT(proto_family == PF_INET);
4437 goto preout_again;
4438 }
4439 /*
4440 * Free the old one and make it point to the IPv6 proto structure.
4441 *
4442 * Change proto for the first time we have successfully
4443 * performed address family translation.
4444 */
4445 if (!did_clat46 && proto_family == PF_INET6) {
4446 struct sockaddr_in6 dest6;
4447 did_clat46 = TRUE;
4448
4449 if (proto != NULL)
4450 if_proto_free(proto);
4451 ifnet_lock_shared(ifp);
4452 /* callee holds a proto refcnt upon success */
4453 proto = find_attached_proto(ifp, proto_family);
4454 if (proto == NULL) {
4455 ifnet_lock_done(ifp);
4456 retval = ENXIO;
4457 m_freem(m);
4458 m = NULL;
4459 goto cleanup;
4460 }
4461 ifnet_lock_done(ifp);
4462 if (ifp->if_type == IFT_ETHER) {
4463 /* Update the dest to translated v6 address */
4464 dest6.sin6_len = sizeof(struct sockaddr_in6);
4465 dest6.sin6_family = AF_INET6;
4466 dest6.sin6_addr = (mtod(m, struct ip6_hdr *))->ip6_dst;
4467 dest = (const struct sockaddr *)&dest6;
4468
4469 /*
4470 * Lookup route to the translated destination
4471 * Free this route ref during cleanup
4472 */
4473 rt = rtalloc1_scoped((struct sockaddr *)&dest6,
4474 0, 0, ifp->if_index);
4475
4476 route = rt;
4477 }
4478 }
4479 }
4480
4481 /*
4482 * This path gets packet chain going to the same destination.
4483 * The pre output routine is used to either trigger resolution of
4484 * the next hop or retreive the next hop's link layer addressing.
4485 * For ex: ether_inet(6)_pre_output routine.
4486 *
4487 * If the routine returns EJUSTRETURN, it implies that packet has
4488 * been queued, and therefore we have to call preout_again for the
4489 * following packet in the chain.
4490 *
4491 * For errors other than EJUSTRETURN, the current packet is freed
4492 * and the rest of the chain (pointed by packetlist is freed as
4493 * part of clean up.
4494 *
4495 * Else if there is no error the retrieved information is used for
4496 * all the packets in the chain.
4497 */
4498 if (raw == 0) {
4499 proto_media_preout preoutp = (proto->proto_kpi == kProtoKPI_v1 ?
4500 proto->kpi.v1.pre_output : proto->kpi.v2.pre_output);
4501 retval = 0;
4502 if (preoutp != NULL) {
4503 retval = preoutp(ifp, proto_family, &m, dest, route,
4504 frame_type, dst_linkaddr);
4505
4506 if (retval != 0) {
4507 if (retval == EJUSTRETURN)
4508 goto preout_again;
4509 m_freem(m);
4510 m = NULL;
4511 goto cleanup;
4512 }
4513 }
4514 }
4515
4516 #if CONFIG_MACF_NET
4517 retval = mac_ifnet_check_transmit(ifp, m, proto_family,
4518 dlil_get_socket_type(&m, proto_family, raw));
4519 if (retval != 0) {
4520 m_freem(m);
4521 goto cleanup;
4522 }
4523 #endif
4524
4525 do {
4526 /*
4527 * Perform address family translation if needed.
4528 * For now we only support stateless 4 to 6 translation
4529 * on the out path.
4530 *
4531 * The routine below translates IP header, updates protocol
4532 * checksum and also translates ICMP.
4533 *
4534 * We skip the first packet as it is already translated and
4535 * the proto family is set to PF_INET6.
4536 */
4537 if (proto_family == PF_INET && IS_INTF_CLAT46(ifp) &&
4538 (ifp->if_type == IFT_CELLULAR ||
4539 dlil_is_clat_needed(proto_family, m))) {
4540 retval = dlil_clat46(ifp, &proto_family, &m);
4541 /* Goto the next packet if the translation fails */
4542 if (retval != 0) {
4543 m_freem(m);
4544 m = NULL;
4545 ip6stat.ip6s_clat464_out_drop++;
4546 goto next;
4547 }
4548 }
4549
4550 #if CONFIG_DTRACE
4551 if (!raw && proto_family == PF_INET) {
4552 struct ip *ip = mtod(m, struct ip *);
4553 DTRACE_IP6(send, struct mbuf *, m, struct inpcb *, NULL,
4554 struct ip *, ip, struct ifnet *, ifp,
4555 struct ip *, ip, struct ip6_hdr *, NULL);
4556
4557 } else if (!raw && proto_family == PF_INET6) {
4558 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *);
4559 DTRACE_IP6(send, struct mbuf *, m, struct inpcb *, NULL,
4560 struct ip6_hdr *, ip6, struct ifnet *, ifp,
4561 struct ip *, NULL, struct ip6_hdr *, ip6);
4562 }
4563 #endif /* CONFIG_DTRACE */
4564
4565 if (raw == 0 && ifp->if_framer != NULL) {
4566 int rcvif_set = 0;
4567
4568 /*
4569 * If this is a broadcast packet that needs to be
4570 * looped back into the system, set the inbound ifp
4571 * to that of the outbound ifp. This will allow
4572 * us to determine that it is a legitimate packet
4573 * for the system. Only set the ifp if it's not
4574 * already set, just to be safe.
4575 */
4576 if ((m->m_flags & (M_BCAST | M_LOOP)) &&
4577 m->m_pkthdr.rcvif == NULL) {
4578 m->m_pkthdr.rcvif = ifp;
4579 rcvif_set = 1;
4580 }
4581
4582 retval = ifp->if_framer(ifp, &m, dest, dst_linkaddr,
4583 frame_type, &pre, &post);
4584 if (retval != 0) {
4585 if (retval != EJUSTRETURN)
4586 m_freem(m);
4587 goto next;
4588 }
4589
4590 /*
4591 * For partial checksum offload, adjust the start
4592 * and stuff offsets based on the prepended header.
4593 */
4594 if ((m->m_pkthdr.csum_flags &
4595 (CSUM_DATA_VALID | CSUM_PARTIAL)) ==
4596 (CSUM_DATA_VALID | CSUM_PARTIAL)) {
4597 m->m_pkthdr.csum_tx_stuff += pre;
4598 m->m_pkthdr.csum_tx_start += pre;
4599 }
4600
4601 if (hwcksum_dbg != 0 && !(ifp->if_flags & IFF_LOOPBACK))
4602 dlil_output_cksum_dbg(ifp, m, pre,
4603 proto_family);
4604
4605 /*
4606 * Clear the ifp if it was set above, and to be
4607 * safe, only if it is still the same as the
4608 * outbound ifp we have in context. If it was
4609 * looped back, then a copy of it was sent to the
4610 * loopback interface with the rcvif set, and we
4611 * are clearing the one that will go down to the
4612 * layer below.
4613 */
4614 if (rcvif_set && m->m_pkthdr.rcvif == ifp)
4615 m->m_pkthdr.rcvif = NULL;
4616 }
4617
4618 /*
4619 * Let interface filters (if any) do their thing ...
4620 */
4621 /* Do not pass VLAN tagged packets to filters PR-3586856 */
4622 if ((m->m_pkthdr.csum_flags & CSUM_VLAN_TAG_VALID) == 0) {
4623 retval = dlil_interface_filters_output(ifp,
4624 &m, proto_family);
4625 if (retval != 0) {
4626 if (retval != EJUSTRETURN)
4627 m_freem(m);
4628 goto next;
4629 }
4630 }
4631 /*
4632 * Strip away M_PROTO1 bit prior to sending packet
4633 * to the driver as this field may be used by the driver
4634 */
4635 m->m_flags &= ~M_PROTO1;
4636
4637 /*
4638 * If the underlying interface is not capable of handling a
4639 * packet whose data portion spans across physically disjoint
4640 * pages, we need to "normalize" the packet so that we pass
4641 * down a chain of mbufs where each mbuf points to a span that
4642 * resides in the system page boundary. If the packet does
4643 * not cross page(s), the following is a no-op.
4644 */
4645 if (!(ifp->if_hwassist & IFNET_MULTIPAGES)) {
4646 if ((m = m_normalize(m)) == NULL)
4647 goto next;
4648 }
4649
4650 /*
4651 * If this is a TSO packet, make sure the interface still
4652 * advertise TSO capability.
4653 */
4654 if (TSO_IPV4_NOTOK(ifp, m) || TSO_IPV6_NOTOK(ifp, m)) {
4655 retval = EMSGSIZE;
4656 m_freem(m);
4657 goto cleanup;
4658 }
4659
4660 ifp_inc_traffic_class_out(ifp, m);
4661 pktap_output(ifp, proto_family, m, pre, post);
4662
4663 /*
4664 * Count the number of elements in the mbuf chain
4665 */
4666 if (tx_chain_len_count) {
4667 dlil_count_chain_len(m, &tx_chain_len_stats);
4668 }
4669
4670 /*
4671 * Record timestamp; ifnet_enqueue() will use this info
4672 * rather than redoing the work. An optimization could
4673 * involve doing this just once at the top, if there are
4674 * no interface filters attached, but that's probably
4675 * not a big deal.
4676 */
4677 nanouptime(&now);
4678 net_timernsec(&now, &now_nsec);
4679 (void) mbuf_set_timestamp(m, now_nsec, TRUE);
4680
4681 /*
4682 * Discard partial sum information if this packet originated
4683 * from another interface; the packet would already have the
4684 * final checksum and we shouldn't recompute it.
4685 */
4686 if ((m->m_pkthdr.pkt_flags & PKTF_FORWARDED) &&
4687 (m->m_pkthdr.csum_flags & (CSUM_DATA_VALID|CSUM_PARTIAL)) ==
4688 (CSUM_DATA_VALID|CSUM_PARTIAL)) {
4689 m->m_pkthdr.csum_flags &= ~CSUM_TX_FLAGS;
4690 m->m_pkthdr.csum_data = 0;
4691 }
4692
4693 /*
4694 * Finally, call the driver.
4695 */
4696 if (ifp->if_eflags & (IFEF_SENDLIST | IFEF_ENQUEUE_MULTI)) {
4697 if (m->m_pkthdr.pkt_flags & PKTF_FORWARDED) {
4698 flen += (m_pktlen(m) - (pre + post));
4699 m->m_pkthdr.pkt_flags &= ~PKTF_FORWARDED;
4700 }
4701 *send_tail = m;
4702 send_tail = &m->m_nextpkt;
4703 } else {
4704 if (m->m_pkthdr.pkt_flags & PKTF_FORWARDED) {
4705 flen = (m_pktlen(m) - (pre + post));
4706 m->m_pkthdr.pkt_flags &= ~PKTF_FORWARDED;
4707 } else {
4708 flen = 0;
4709 }
4710 KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_START,
4711 0, 0, 0, 0, 0);
4712 retval = (*ifp->if_output_dlil)(ifp, m);
4713 if (retval == EQFULL || retval == EQSUSPENDED) {
4714 if (adv != NULL && adv->code == FADV_SUCCESS) {
4715 adv->code = (retval == EQFULL ?
4716 FADV_FLOW_CONTROLLED :
4717 FADV_SUSPENDED);
4718 }
4719 retval = 0;
4720 }
4721 if (retval == 0 && flen > 0) {
4722 fbytes += flen;
4723 fpkts++;
4724 }
4725 if (retval != 0 && dlil_verbose) {
4726 printf("%s: output error on %s retval = %d\n",
4727 __func__, if_name(ifp),
4728 retval);
4729 }
4730 KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_END,
4731 0, 0, 0, 0, 0);
4732 }
4733 KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_END, 0, 0, 0, 0, 0);
4734
4735 next:
4736 m = packetlist;
4737 if (m != NULL) {
4738 packetlist = packetlist->m_nextpkt;
4739 m->m_nextpkt = NULL;
4740 }
4741 /* Reset the proto family to old proto family for CLAT */
4742 if (did_clat46)
4743 proto_family = old_proto_family;
4744 } while (m != NULL);
4745
4746 if (send_head != NULL) {
4747 KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_START,
4748 0, 0, 0, 0, 0);
4749 if (ifp->if_eflags & IFEF_SENDLIST) {
4750 retval = (*ifp->if_output_dlil)(ifp, send_head);
4751 if (retval == EQFULL || retval == EQSUSPENDED) {
4752 if (adv != NULL) {
4753 adv->code = (retval == EQFULL ?
4754 FADV_FLOW_CONTROLLED :
4755 FADV_SUSPENDED);
4756 }
4757 retval = 0;
4758 }
4759 if (retval == 0 && flen > 0) {
4760 fbytes += flen;
4761 fpkts++;
4762 }
4763 if (retval != 0 && dlil_verbose) {
4764 printf("%s: output error on %s retval = %d\n",
4765 __func__, if_name(ifp), retval);
4766 }
4767 } else {
4768 struct mbuf *send_m;
4769 int enq_cnt = 0;
4770 VERIFY(ifp->if_eflags & IFEF_ENQUEUE_MULTI);
4771 while (send_head != NULL) {
4772 send_m = send_head;
4773 send_head = send_m->m_nextpkt;
4774 send_m->m_nextpkt = NULL;
4775 retval = (*ifp->if_output_dlil)(ifp, send_m);
4776 if (retval == EQFULL || retval == EQSUSPENDED) {
4777 if (adv != NULL) {
4778 adv->code = (retval == EQFULL ?
4779 FADV_FLOW_CONTROLLED :
4780 FADV_SUSPENDED);
4781 }
4782 retval = 0;
4783 }
4784 if (retval == 0) {
4785 enq_cnt++;
4786 if (flen > 0)
4787 fpkts++;
4788 }
4789 if (retval != 0 && dlil_verbose) {
4790 printf("%s: output error on %s "
4791 "retval = %d\n",
4792 __func__, if_name(ifp), retval);
4793 }
4794 }
4795 if (enq_cnt > 0) {
4796 fbytes += flen;
4797 ifnet_start(ifp);
4798 }
4799 }
4800 KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_END, 0, 0, 0, 0, 0);
4801 }
4802
4803 KERNEL_DEBUG(DBG_FNC_DLIL_OUTPUT | DBG_FUNC_END, 0, 0, 0, 0, 0);
4804
4805 cleanup:
4806 if (fbytes > 0)
4807 ifp->if_fbytes += fbytes;
4808 if (fpkts > 0)
4809 ifp->if_fpackets += fpkts;
4810 if (proto != NULL)
4811 if_proto_free(proto);
4812 if (packetlist) /* if any packets are left, clean up */
4813 mbuf_freem_list(packetlist);
4814 if (retval == EJUSTRETURN)
4815 retval = 0;
4816 if (iorefcnt == 1)
4817 ifnet_decr_iorefcnt(ifp);
4818 if (rt != NULL) {
4819 rtfree(rt);
4820 rt = NULL;
4821 }
4822
4823 return (retval);
4824 }
4825
4826 /*
4827 * This routine checks if the destination address is not a loopback, link-local,
4828 * multicast or broadcast address.
4829 */
4830 static int
4831 dlil_is_clat_needed(protocol_family_t proto_family, mbuf_t m)
4832 {
4833 int ret = 0;
4834 switch(proto_family) {
4835 case PF_INET: {
4836 struct ip *iph = mtod(m, struct ip *);
4837 if (CLAT46_NEEDED(ntohl(iph->ip_dst.s_addr)))
4838 ret = 1;
4839 break;
4840 }
4841 case PF_INET6: {
4842 struct ip6_hdr *ip6h = mtod(m, struct ip6_hdr *);
4843 if ((size_t)m_pktlen(m) >= sizeof(struct ip6_hdr) &&
4844 CLAT64_NEEDED(&ip6h->ip6_dst))
4845 ret = 1;
4846 break;
4847 }
4848 }
4849
4850 return (ret);
4851 }
4852 /*
4853 * @brief This routine translates IPv4 packet to IPv6 packet,
4854 * updates protocol checksum and also translates ICMP for code
4855 * along with inner header translation.
4856 *
4857 * @param ifp Pointer to the interface
4858 * @param proto_family pointer to protocol family. It is updated if function
4859 * performs the translation successfully.
4860 * @param m Pointer to the pointer pointing to the packet. Needed because this
4861 * routine can end up changing the mbuf to a different one.
4862 *
4863 * @return 0 on success or else a negative value.
4864 */
4865 static errno_t
4866 dlil_clat46(ifnet_t ifp, protocol_family_t *proto_family, mbuf_t *m)
4867 {
4868 VERIFY(*proto_family == PF_INET);
4869 VERIFY(IS_INTF_CLAT46(ifp));
4870
4871 pbuf_t pbuf_store, *pbuf = NULL;
4872 struct ip *iph = NULL;
4873 struct in_addr osrc, odst;
4874 uint8_t proto = 0;
4875 struct in6_ifaddr *ia6_clat_src = NULL;
4876 struct in6_addr *src = NULL;
4877 struct in6_addr dst;
4878 int error = 0;
4879 uint32_t off = 0;
4880 uint64_t tot_len = 0;
4881 uint16_t ip_id_val = 0;
4882 uint16_t ip_frag_off = 0;
4883
4884 boolean_t is_frag = FALSE;
4885 boolean_t is_first_frag = TRUE;
4886 boolean_t is_last_frag = TRUE;
4887
4888 pbuf_init_mbuf(&pbuf_store, *m, ifp);
4889 pbuf = &pbuf_store;
4890 iph = pbuf->pb_data;
4891
4892 osrc = iph->ip_src;
4893 odst = iph->ip_dst;
4894 proto = iph->ip_p;
4895 off = iph->ip_hl << 2;
4896 ip_id_val = iph->ip_id;
4897 ip_frag_off = ntohs(iph->ip_off) & IP_OFFMASK;
4898
4899 tot_len = ntohs(iph->ip_len);
4900
4901 /*
4902 * For packets that are not first frags
4903 * we only need to adjust CSUM.
4904 * For 4 to 6, Fragmentation header gets appended
4905 * after proto translation.
4906 */
4907 if (ntohs(iph->ip_off) & ~(IP_DF | IP_RF)) {
4908 is_frag = TRUE;
4909
4910 /* If the offset is not zero, it is not first frag */
4911 if (ip_frag_off != 0)
4912 is_first_frag = FALSE;
4913
4914 /* If IP_MF is set, then it is not last frag */
4915 if (ntohs(iph->ip_off) & IP_MF)
4916 is_last_frag = FALSE;
4917 }
4918
4919 /*
4920 * Retrive the local IPv6 CLAT46 address reserved for stateless
4921 * translation.
4922 */
4923 ia6_clat_src = in6ifa_ifpwithflag(ifp, IN6_IFF_CLAT46);
4924 if (ia6_clat_src == NULL) {
4925 ip6stat.ip6s_clat464_out_nov6addr_drop++;
4926 error = -1;
4927 goto cleanup;
4928 }
4929
4930 src = &ia6_clat_src->ia_addr.sin6_addr;
4931
4932 /*
4933 * Translate IPv4 destination to IPv6 destination by using the
4934 * prefixes learned through prior PLAT discovery.
4935 */
4936 if ((error = nat464_synthesize_ipv6(ifp, &odst, &dst)) != 0) {
4937 ip6stat.ip6s_clat464_out_v6synthfail_drop++;
4938 goto cleanup;
4939 }
4940
4941 /* Translate the IP header part first */
4942 error = (nat464_translate_46(pbuf, off, iph->ip_tos, iph->ip_p,
4943 iph->ip_ttl, *src, dst, tot_len) == NT_NAT64) ? 0 : -1;
4944
4945 iph = NULL; /* Invalidate iph as pbuf has been modified */
4946
4947 if (error != 0) {
4948 ip6stat.ip6s_clat464_out_46transfail_drop++;
4949 goto cleanup;
4950 }
4951
4952 /*
4953 * Translate protocol header, update checksum, checksum flags
4954 * and related fields.
4955 */
4956 error = (nat464_translate_proto(pbuf, (struct nat464_addr *)&osrc, (struct nat464_addr *)&odst,
4957 proto, PF_INET, PF_INET6, NT_OUT, !is_first_frag) == NT_NAT64) ? 0 : -1;
4958
4959 if (error != 0) {
4960 ip6stat.ip6s_clat464_out_46proto_transfail_drop++;
4961 goto cleanup;
4962 }
4963
4964 /* Now insert the IPv6 fragment header */
4965 if (is_frag) {
4966 error = nat464_insert_frag46(pbuf, ip_id_val, ip_frag_off, is_last_frag);
4967
4968 if (error != 0) {
4969 ip6stat.ip6s_clat464_out_46frag_transfail_drop++;
4970 goto cleanup;
4971 }
4972 }
4973
4974 cleanup:
4975 if (ia6_clat_src != NULL)
4976 IFA_REMREF(&ia6_clat_src->ia_ifa);
4977
4978 if (pbuf_is_valid(pbuf)) {
4979 *m = pbuf->pb_mbuf;
4980 pbuf->pb_mbuf = NULL;
4981 pbuf_destroy(pbuf);
4982 } else {
4983 error = -1;
4984 ip6stat.ip6s_clat464_out_invalpbuf_drop++;
4985 }
4986
4987 if (error == 0) {
4988 *proto_family = PF_INET6;
4989 ip6stat.ip6s_clat464_out_success++;
4990 }
4991
4992 return (error);
4993 }
4994
4995 /*
4996 * @brief This routine translates incoming IPv6 to IPv4 packet,
4997 * updates protocol checksum and also translates ICMPv6 outer
4998 * and inner headers
4999 *
5000 * @return 0 on success or else a negative value.
5001 */
5002 static errno_t
5003 dlil_clat64(ifnet_t ifp, protocol_family_t *proto_family, mbuf_t *m)
5004 {
5005 VERIFY(*proto_family == PF_INET6);
5006 VERIFY(IS_INTF_CLAT46(ifp));
5007
5008 struct ip6_hdr *ip6h = NULL;
5009 struct in6_addr osrc, odst;
5010 uint8_t proto = 0;
5011 struct in6_ifaddr *ia6_clat_dst = NULL;
5012 struct in_ifaddr *ia4_clat_dst = NULL;
5013 struct in_addr *dst = NULL;
5014 struct in_addr src;
5015 int error = 0;
5016 uint32_t off = 0;
5017 u_int64_t tot_len = 0;
5018 uint8_t tos = 0;
5019 boolean_t is_first_frag = TRUE;
5020
5021 /* Incoming mbuf does not contain valid IP6 header */
5022 if ((size_t)(*m)->m_pkthdr.len < sizeof(struct ip6_hdr) ||
5023 ((size_t)(*m)->m_len < sizeof(struct ip6_hdr) &&
5024 (*m = m_pullup(*m, sizeof(struct ip6_hdr))) == NULL)) {
5025 ip6stat.ip6s_clat464_in_tooshort_drop++;
5026 return (-1);
5027 }
5028
5029 ip6h = mtod(*m, struct ip6_hdr *);
5030 /* Validate that mbuf contains IP payload equal to ip6_plen */
5031 if ((size_t)(*m)->m_pkthdr.len < ntohs(ip6h->ip6_plen) + sizeof(struct ip6_hdr)) {
5032 ip6stat.ip6s_clat464_in_tooshort_drop++;
5033 return (-1);
5034 }
5035
5036 osrc = ip6h->ip6_src;
5037 odst = ip6h->ip6_dst;
5038
5039 /*
5040 * Retrieve the local CLAT46 reserved IPv6 address.
5041 * Let the packet pass if we don't find one, as the flag
5042 * may get set before IPv6 configuration has taken place.
5043 */
5044 ia6_clat_dst = in6ifa_ifpwithflag(ifp, IN6_IFF_CLAT46);
5045 if (ia6_clat_dst == NULL)
5046 goto done;
5047
5048 /*
5049 * Check if the original dest in the packet is same as the reserved
5050 * CLAT46 IPv6 address
5051 */
5052 if (IN6_ARE_ADDR_EQUAL(&odst, &ia6_clat_dst->ia_addr.sin6_addr)) {
5053 pbuf_t pbuf_store, *pbuf = NULL;
5054 pbuf_init_mbuf(&pbuf_store, *m, ifp);
5055 pbuf = &pbuf_store;
5056
5057 /*
5058 * Retrive the local CLAT46 IPv4 address reserved for stateless
5059 * translation.
5060 */
5061 ia4_clat_dst = inifa_ifpclatv4(ifp);
5062 if (ia4_clat_dst == NULL) {
5063 IFA_REMREF(&ia6_clat_dst->ia_ifa);
5064 ip6stat.ip6s_clat464_in_nov4addr_drop++;
5065 error = -1;
5066 goto cleanup;
5067 }
5068 IFA_REMREF(&ia6_clat_dst->ia_ifa);
5069
5070 /* Translate IPv6 src to IPv4 src by removing the NAT64 prefix */
5071 dst = &ia4_clat_dst->ia_addr.sin_addr;
5072 if ((error = nat464_synthesize_ipv4(ifp, &osrc, &src)) != 0) {
5073 ip6stat.ip6s_clat464_in_v4synthfail_drop++;
5074 error = -1;
5075 goto cleanup;
5076 }
5077
5078 ip6h = pbuf->pb_data;
5079 off = sizeof(struct ip6_hdr);
5080 proto = ip6h->ip6_nxt;
5081 tos = (ntohl(ip6h->ip6_flow) >> 20) & 0xff;
5082 tot_len = ntohs(ip6h->ip6_plen) + sizeof(struct ip6_hdr);
5083
5084 /*
5085 * Translate the IP header and update the fragmentation
5086 * header if needed
5087 */
5088 error = (nat464_translate_64(pbuf, off, tos, &proto,
5089 ip6h->ip6_hlim, src, *dst, tot_len, &is_first_frag) == NT_NAT64) ?
5090 0 : -1;
5091
5092 ip6h = NULL; /* Invalidate ip6h as pbuf has been changed */
5093
5094 if (error != 0) {
5095 ip6stat.ip6s_clat464_in_64transfail_drop++;
5096 goto cleanup;
5097 }
5098
5099 /*
5100 * Translate protocol header, update checksum, checksum flags
5101 * and related fields.
5102 */
5103 error = (nat464_translate_proto(pbuf, (struct nat464_addr *)&osrc,
5104 (struct nat464_addr *)&odst, proto, PF_INET6, PF_INET,
5105 NT_IN, !is_first_frag) == NT_NAT64) ? 0 : -1;
5106
5107 if (error != 0) {
5108 ip6stat.ip6s_clat464_in_64proto_transfail_drop++;
5109 goto cleanup;
5110 }
5111
5112 cleanup:
5113 if (ia4_clat_dst != NULL)
5114 IFA_REMREF(&ia4_clat_dst->ia_ifa);
5115
5116 if (pbuf_is_valid(pbuf)) {
5117 *m = pbuf->pb_mbuf;
5118 pbuf->pb_mbuf = NULL;
5119 pbuf_destroy(pbuf);
5120 } else {
5121 error = -1;
5122 ip6stat.ip6s_clat464_in_invalpbuf_drop++;
5123 }
5124
5125 if (error == 0) {
5126 *proto_family = PF_INET;
5127 ip6stat.ip6s_clat464_in_success++;
5128 }
5129 } /* CLAT traffic */
5130
5131 done:
5132 return (error);
5133 }
5134
5135 errno_t
5136 ifnet_ioctl(ifnet_t ifp, protocol_family_t proto_fam, u_long ioctl_code,
5137 void *ioctl_arg)
5138 {
5139 struct ifnet_filter *filter;
5140 int retval = EOPNOTSUPP;
5141 int result = 0;
5142
5143 if (ifp == NULL || ioctl_code == 0)
5144 return (EINVAL);
5145
5146 /* Get an io ref count if the interface is attached */
5147 if (!ifnet_is_attached(ifp, 1))
5148 return (EOPNOTSUPP);
5149
5150 /*
5151 * Run the interface filters first.
5152 * We want to run all filters before calling the protocol,
5153 * interface family, or interface.
5154 */
5155 lck_mtx_lock_spin(&ifp->if_flt_lock);
5156 /* prevent filter list from changing in case we drop the lock */
5157 if_flt_monitor_busy(ifp);
5158 TAILQ_FOREACH(filter, &ifp->if_flt_head, filt_next) {
5159 if (filter->filt_ioctl != NULL && (filter->filt_protocol == 0 ||
5160 filter->filt_protocol == proto_fam)) {
5161 lck_mtx_unlock(&ifp->if_flt_lock);
5162
5163 result = filter->filt_ioctl(filter->filt_cookie, ifp,
5164 proto_fam, ioctl_code, ioctl_arg);
5165
5166 lck_mtx_lock_spin(&ifp->if_flt_lock);
5167
5168 /* Only update retval if no one has handled the ioctl */
5169 if (retval == EOPNOTSUPP || result == EJUSTRETURN) {
5170 if (result == ENOTSUP)
5171 result = EOPNOTSUPP;
5172 retval = result;
5173 if (retval != 0 && retval != EOPNOTSUPP) {
5174 /* we're done with the filter list */
5175 if_flt_monitor_unbusy(ifp);
5176 lck_mtx_unlock(&ifp->if_flt_lock);
5177 goto cleanup;
5178 }
5179 }
5180 }
5181 }
5182 /* we're done with the filter list */
5183 if_flt_monitor_unbusy(ifp);
5184 lck_mtx_unlock(&ifp->if_flt_lock);
5185
5186 /* Allow the protocol to handle the ioctl */
5187 if (proto_fam != 0) {
5188 struct if_proto *proto;
5189
5190 /* callee holds a proto refcnt upon success */
5191 ifnet_lock_shared(ifp);
5192 proto = find_attached_proto(ifp, proto_fam);
5193 ifnet_lock_done(ifp);
5194 if (proto != NULL) {
5195 proto_media_ioctl ioctlp =
5196 (proto->proto_kpi == kProtoKPI_v1 ?
5197 proto->kpi.v1.ioctl : proto->kpi.v2.ioctl);
5198 result = EOPNOTSUPP;
5199 if (ioctlp != NULL)
5200 result = ioctlp(ifp, proto_fam, ioctl_code,
5201 ioctl_arg);
5202 if_proto_free(proto);
5203
5204 /* Only update retval if no one has handled the ioctl */
5205 if (retval == EOPNOTSUPP || result == EJUSTRETURN) {
5206 if (result == ENOTSUP)
5207 result = EOPNOTSUPP;
5208 retval = result;
5209 if (retval && retval != EOPNOTSUPP)
5210 goto cleanup;
5211 }
5212 }
5213 }
5214
5215 /* retval is either 0 or EOPNOTSUPP */
5216
5217 /*
5218 * Let the interface handle this ioctl.
5219 * If it returns EOPNOTSUPP, ignore that, we may have
5220 * already handled this in the protocol or family.
5221 */
5222 if (ifp->if_ioctl)
5223 result = (*ifp->if_ioctl)(ifp, ioctl_code, ioctl_arg);
5224
5225 /* Only update retval if no one has handled the ioctl */
5226 if (retval == EOPNOTSUPP || result == EJUSTRETURN) {
5227 if (result == ENOTSUP)
5228 result = EOPNOTSUPP;
5229 retval = result;
5230 if (retval && retval != EOPNOTSUPP) {
5231 goto cleanup;
5232 }
5233 }
5234
5235 cleanup:
5236 if (retval == EJUSTRETURN)
5237 retval = 0;
5238
5239 ifnet_decr_iorefcnt(ifp);
5240
5241 return (retval);
5242 }
5243
5244 __private_extern__ errno_t
5245 dlil_set_bpf_tap(ifnet_t ifp, bpf_tap_mode mode, bpf_packet_func callback)
5246 {
5247 errno_t error = 0;
5248
5249
5250 if (ifp->if_set_bpf_tap) {
5251 /* Get an io reference on the interface if it is attached */
5252 if (!ifnet_is_attached(ifp, 1))
5253 return (ENXIO);
5254 error = ifp->if_set_bpf_tap(ifp, mode, callback);
5255 ifnet_decr_iorefcnt(ifp);
5256 }
5257 return (error);
5258 }
5259
5260 errno_t
5261 dlil_resolve_multi(struct ifnet *ifp, const struct sockaddr *proto_addr,
5262 struct sockaddr *ll_addr, size_t ll_len)
5263 {
5264 errno_t result = EOPNOTSUPP;
5265 struct if_proto *proto;
5266 const struct sockaddr *verify;
5267 proto_media_resolve_multi resolvep;
5268
5269 if (!ifnet_is_attached(ifp, 1))
5270 return (result);
5271
5272 bzero(ll_addr, ll_len);
5273
5274 /* Call the protocol first; callee holds a proto refcnt upon success */
5275 ifnet_lock_shared(ifp);
5276 proto = find_attached_proto(ifp, proto_addr->sa_family);
5277 ifnet_lock_done(ifp);
5278 if (proto != NULL) {
5279 resolvep = (proto->proto_kpi == kProtoKPI_v1 ?
5280 proto->kpi.v1.resolve_multi : proto->kpi.v2.resolve_multi);
5281 if (resolvep != NULL)
5282 result = resolvep(ifp, proto_addr,
5283 (struct sockaddr_dl *)(void *)ll_addr, ll_len);
5284 if_proto_free(proto);
5285 }
5286
5287 /* Let the interface verify the multicast address */
5288 if ((result == EOPNOTSUPP || result == 0) && ifp->if_check_multi) {
5289 if (result == 0)
5290 verify = ll_addr;
5291 else
5292 verify = proto_addr;
5293 result = ifp->if_check_multi(ifp, verify);
5294 }
5295
5296 ifnet_decr_iorefcnt(ifp);
5297 return (result);
5298 }
5299
5300 __private_extern__ errno_t
5301 dlil_send_arp_internal(ifnet_t ifp, u_short arpop,
5302 const struct sockaddr_dl *sender_hw, const struct sockaddr *sender_proto,
5303 const struct sockaddr_dl *target_hw, const struct sockaddr *target_proto)
5304 {
5305 struct if_proto *proto;
5306 errno_t result = 0;
5307
5308 /* callee holds a proto refcnt upon success */
5309 ifnet_lock_shared(ifp);
5310 proto = find_attached_proto(ifp, target_proto->sa_family);
5311 ifnet_lock_done(ifp);
5312 if (proto == NULL) {
5313 result = ENOTSUP;
5314 } else {
5315 proto_media_send_arp arpp;
5316 arpp = (proto->proto_kpi == kProtoKPI_v1 ?
5317 proto->kpi.v1.send_arp : proto->kpi.v2.send_arp);
5318 if (arpp == NULL) {
5319 result = ENOTSUP;
5320 } else {
5321 switch (arpop) {
5322 case ARPOP_REQUEST:
5323 arpstat.txrequests++;
5324 if (target_hw != NULL)
5325 arpstat.txurequests++;
5326 break;
5327 case ARPOP_REPLY:
5328 arpstat.txreplies++;
5329 break;
5330 }
5331 result = arpp(ifp, arpop, sender_hw, sender_proto,
5332 target_hw, target_proto);
5333 }
5334 if_proto_free(proto);
5335 }
5336
5337 return (result);
5338 }
5339
5340 struct net_thread_marks { };
5341 static const struct net_thread_marks net_thread_marks_base = { };
5342
5343 __private_extern__ const net_thread_marks_t net_thread_marks_none =
5344 &net_thread_marks_base;
5345
5346 __private_extern__ net_thread_marks_t
5347 net_thread_marks_push(u_int32_t push)
5348 {
5349 static const char *const base = (const void*)&net_thread_marks_base;
5350 u_int32_t pop = 0;
5351
5352 if (push != 0) {
5353 struct uthread *uth = get_bsdthread_info(current_thread());
5354
5355 pop = push & ~uth->uu_network_marks;
5356 if (pop != 0)
5357 uth->uu_network_marks |= pop;
5358 }
5359
5360 return ((net_thread_marks_t)&base[pop]);
5361 }
5362
5363 __private_extern__ net_thread_marks_t
5364 net_thread_unmarks_push(u_int32_t unpush)
5365 {
5366 static const char *const base = (const void*)&net_thread_marks_base;
5367 u_int32_t unpop = 0;
5368
5369 if (unpush != 0) {
5370 struct uthread *uth = get_bsdthread_info(current_thread());
5371
5372 unpop = unpush & uth->uu_network_marks;
5373 if (unpop != 0)
5374 uth->uu_network_marks &= ~unpop;
5375 }
5376
5377 return ((net_thread_marks_t)&base[unpop]);
5378 }
5379
5380 __private_extern__ void
5381 net_thread_marks_pop(net_thread_marks_t popx)
5382 {
5383 static const char *const base = (const void*)&net_thread_marks_base;
5384 const ptrdiff_t pop = (const char *)popx - (const char *)base;
5385
5386 if (pop != 0) {
5387 static const ptrdiff_t ones = (ptrdiff_t)(u_int32_t)~0U;
5388 struct uthread *uth = get_bsdthread_info(current_thread());
5389
5390 VERIFY((pop & ones) == pop);
5391 VERIFY((ptrdiff_t)(uth->uu_network_marks & pop) == pop);
5392 uth->uu_network_marks &= ~pop;
5393 }
5394 }
5395
5396 __private_extern__ void
5397 net_thread_unmarks_pop(net_thread_marks_t unpopx)
5398 {
5399 static const char *const base = (const void*)&net_thread_marks_base;
5400 ptrdiff_t unpop = (const char *)unpopx - (const char *)base;
5401
5402 if (unpop != 0) {
5403 static const ptrdiff_t ones = (ptrdiff_t)(u_int32_t)~0U;
5404 struct uthread *uth = get_bsdthread_info(current_thread());
5405
5406 VERIFY((unpop & ones) == unpop);
5407 VERIFY((ptrdiff_t)(uth->uu_network_marks & unpop) == 0);
5408 uth->uu_network_marks |= unpop;
5409 }
5410 }
5411
5412 __private_extern__ u_int32_t
5413 net_thread_is_marked(u_int32_t check)
5414 {
5415 if (check != 0) {
5416 struct uthread *uth = get_bsdthread_info(current_thread());
5417 return (uth->uu_network_marks & check);
5418 }
5419 else
5420 return (0);
5421 }
5422
5423 __private_extern__ u_int32_t
5424 net_thread_is_unmarked(u_int32_t check)
5425 {
5426 if (check != 0) {
5427 struct uthread *uth = get_bsdthread_info(current_thread());
5428 return (~uth->uu_network_marks & check);
5429 }
5430 else
5431 return (0);
5432 }
5433
5434 static __inline__ int
5435 _is_announcement(const struct sockaddr_in * sender_sin,
5436 const struct sockaddr_in * target_sin)
5437 {
5438 if (sender_sin == NULL) {
5439 return (FALSE);
5440 }
5441 return (sender_sin->sin_addr.s_addr == target_sin->sin_addr.s_addr);
5442 }
5443
5444 __private_extern__ errno_t
5445 dlil_send_arp(ifnet_t ifp, u_short arpop, const struct sockaddr_dl *sender_hw,
5446 const struct sockaddr *sender_proto, const struct sockaddr_dl *target_hw,
5447 const struct sockaddr *target_proto0, u_int32_t rtflags)
5448 {
5449 errno_t result = 0;
5450 const struct sockaddr_in * sender_sin;
5451 const struct sockaddr_in * target_sin;
5452 struct sockaddr_inarp target_proto_sinarp;
5453 struct sockaddr *target_proto = (void *)(uintptr_t)target_proto0;
5454
5455 if (target_proto == NULL || (sender_proto != NULL &&
5456 sender_proto->sa_family != target_proto->sa_family))
5457 return (EINVAL);
5458
5459 /*
5460 * If the target is a (default) router, provide that
5461 * information to the send_arp callback routine.
5462 */
5463 if (rtflags & RTF_ROUTER) {
5464 bcopy(target_proto, &target_proto_sinarp,
5465 sizeof (struct sockaddr_in));
5466 target_proto_sinarp.sin_other |= SIN_ROUTER;
5467 target_proto = (struct sockaddr *)&target_proto_sinarp;
5468 }
5469
5470 /*
5471 * If this is an ARP request and the target IP is IPv4LL,
5472 * send the request on all interfaces. The exception is
5473 * an announcement, which must only appear on the specific
5474 * interface.
5475 */
5476 sender_sin = (struct sockaddr_in *)(void *)(uintptr_t)sender_proto;
5477 target_sin = (struct sockaddr_in *)(void *)(uintptr_t)target_proto;
5478 if (target_proto->sa_family == AF_INET &&
5479 IN_LINKLOCAL(ntohl(target_sin->sin_addr.s_addr)) &&
5480 ipv4_ll_arp_aware != 0 && arpop == ARPOP_REQUEST &&
5481 !_is_announcement(target_sin, sender_sin)) {
5482 ifnet_t *ifp_list;
5483 u_int32_t count;
5484 u_int32_t ifp_on;
5485
5486 result = ENOTSUP;
5487
5488 if (ifnet_list_get(IFNET_FAMILY_ANY, &ifp_list, &count) == 0) {
5489 for (ifp_on = 0; ifp_on < count; ifp_on++) {
5490 errno_t new_result;
5491 ifaddr_t source_hw = NULL;
5492 ifaddr_t source_ip = NULL;
5493 struct sockaddr_in source_ip_copy;
5494 struct ifnet *cur_ifp = ifp_list[ifp_on];
5495
5496 /*
5497 * Only arp on interfaces marked for IPv4LL
5498 * ARPing. This may mean that we don't ARP on
5499 * the interface the subnet route points to.
5500 */
5501 if (!(cur_ifp->if_eflags & IFEF_ARPLL))
5502 continue;
5503
5504 /* Find the source IP address */
5505 ifnet_lock_shared(cur_ifp);
5506 source_hw = cur_ifp->if_lladdr;
5507 TAILQ_FOREACH(source_ip, &cur_ifp->if_addrhead,
5508 ifa_link) {
5509 IFA_LOCK(source_ip);
5510 if (source_ip->ifa_addr != NULL &&
5511 source_ip->ifa_addr->sa_family ==
5512 AF_INET) {
5513 /* Copy the source IP address */
5514 source_ip_copy =
5515 *(struct sockaddr_in *)
5516 (void *)source_ip->ifa_addr;
5517 IFA_UNLOCK(source_ip);
5518 break;
5519 }
5520 IFA_UNLOCK(source_ip);
5521 }
5522
5523 /* No IP Source, don't arp */
5524 if (source_ip == NULL) {
5525 ifnet_lock_done(cur_ifp);
5526 continue;
5527 }
5528
5529 IFA_ADDREF(source_hw);
5530 ifnet_lock_done(cur_ifp);
5531
5532 /* Send the ARP */
5533 new_result = dlil_send_arp_internal(cur_ifp,
5534 arpop, (struct sockaddr_dl *)(void *)
5535 source_hw->ifa_addr,
5536 (struct sockaddr *)&source_ip_copy, NULL,
5537 target_proto);
5538
5539 IFA_REMREF(source_hw);
5540 if (result == ENOTSUP) {
5541 result = new_result;
5542 }
5543 }
5544 ifnet_list_free(ifp_list);
5545 }
5546 } else {
5547 result = dlil_send_arp_internal(ifp, arpop, sender_hw,
5548 sender_proto, target_hw, target_proto);
5549 }
5550
5551 return (result);
5552 }
5553
5554 /*
5555 * Caller must hold ifnet head lock.
5556 */
5557 static int
5558 ifnet_lookup(struct ifnet *ifp)
5559 {
5560 struct ifnet *_ifp;
5561
5562 LCK_RW_ASSERT(&ifnet_head_lock, LCK_RW_ASSERT_HELD);
5563 TAILQ_FOREACH(_ifp, &ifnet_head, if_link) {
5564 if (_ifp == ifp)
5565 break;
5566 }
5567 return (_ifp != NULL);
5568 }
5569
5570 /*
5571 * Caller has to pass a non-zero refio argument to get a
5572 * IO reference count. This will prevent ifnet_detach from
5573 * being called when there are outstanding io reference counts.
5574 */
5575 int
5576 ifnet_is_attached(struct ifnet *ifp, int refio)
5577 {
5578 int ret;
5579
5580 lck_mtx_lock_spin(&ifp->if_ref_lock);
5581 if ((ret = IF_FULLY_ATTACHED(ifp))) {
5582 if (refio > 0)
5583 ifp->if_refio++;
5584 }
5585 lck_mtx_unlock(&ifp->if_ref_lock);
5586
5587 return (ret);
5588 }
5589
5590 /*
5591 * Caller must ensure the interface is attached; the assumption is that
5592 * there is at least an outstanding IO reference count held already.
5593 * Most callers would call ifnet_is_attached() instead.
5594 */
5595 void
5596 ifnet_incr_iorefcnt(struct ifnet *ifp)
5597 {
5598 lck_mtx_lock_spin(&ifp->if_ref_lock);
5599 VERIFY(IF_FULLY_ATTACHED(ifp));
5600 VERIFY(ifp->if_refio > 0);
5601 ifp->if_refio++;
5602 lck_mtx_unlock(&ifp->if_ref_lock);
5603 }
5604
5605 void
5606 ifnet_decr_iorefcnt(struct ifnet *ifp)
5607 {
5608 lck_mtx_lock_spin(&ifp->if_ref_lock);
5609 VERIFY(ifp->if_refio > 0);
5610 VERIFY(ifp->if_refflags & (IFRF_ATTACHED | IFRF_DETACHING));
5611 ifp->if_refio--;
5612
5613 /*
5614 * if there are no more outstanding io references, wakeup the
5615 * ifnet_detach thread if detaching flag is set.
5616 */
5617 if (ifp->if_refio == 0 && (ifp->if_refflags & IFRF_DETACHING))
5618 wakeup(&(ifp->if_refio));
5619
5620 lck_mtx_unlock(&ifp->if_ref_lock);
5621 }
5622
5623 static void
5624 dlil_if_trace(struct dlil_ifnet *dl_if, int refhold)
5625 {
5626 struct dlil_ifnet_dbg *dl_if_dbg = (struct dlil_ifnet_dbg *)dl_if;
5627 ctrace_t *tr;
5628 u_int32_t idx;
5629 u_int16_t *cnt;
5630
5631 if (!(dl_if->dl_if_flags & DLIF_DEBUG)) {
5632 panic("%s: dl_if %p has no debug structure", __func__, dl_if);
5633 /* NOTREACHED */
5634 }
5635
5636 if (refhold) {
5637 cnt = &dl_if_dbg->dldbg_if_refhold_cnt;
5638 tr = dl_if_dbg->dldbg_if_refhold;
5639 } else {
5640 cnt = &dl_if_dbg->dldbg_if_refrele_cnt;
5641 tr = dl_if_dbg->dldbg_if_refrele;
5642 }
5643
5644 idx = atomic_add_16_ov(cnt, 1) % IF_REF_TRACE_HIST_SIZE;
5645 ctrace_record(&tr[idx]);
5646 }
5647
5648 errno_t
5649 dlil_if_ref(struct ifnet *ifp)
5650 {
5651 struct dlil_ifnet *dl_if = (struct dlil_ifnet *)ifp;
5652
5653 if (dl_if == NULL)
5654 return (EINVAL);
5655
5656 lck_mtx_lock_spin(&dl_if->dl_if_lock);
5657 ++dl_if->dl_if_refcnt;
5658 if (dl_if->dl_if_refcnt == 0) {
5659 panic("%s: wraparound refcnt for ifp=%p", __func__, ifp);
5660 /* NOTREACHED */
5661 }
5662 if (dl_if->dl_if_trace != NULL)
5663 (*dl_if->dl_if_trace)(dl_if, TRUE);
5664 lck_mtx_unlock(&dl_if->dl_if_lock);
5665
5666 return (0);
5667 }
5668
5669 errno_t
5670 dlil_if_free(struct ifnet *ifp)
5671 {
5672 struct dlil_ifnet *dl_if = (struct dlil_ifnet *)ifp;
5673 bool need_release = FALSE;
5674
5675 if (dl_if == NULL)
5676 return (EINVAL);
5677
5678 lck_mtx_lock_spin(&dl_if->dl_if_lock);
5679 switch (dl_if->dl_if_refcnt) {
5680 case 0:
5681 panic("%s: negative refcnt for ifp=%p", __func__, ifp);
5682 /* NOTREACHED */
5683 break;
5684 case 1:
5685 if ((ifp->if_refflags & IFRF_EMBRYONIC) != 0) {
5686 need_release = TRUE;
5687 }
5688 break;
5689 default:
5690 break;
5691 }
5692 --dl_if->dl_if_refcnt;
5693 if (dl_if->dl_if_trace != NULL)
5694 (*dl_if->dl_if_trace)(dl_if, FALSE);
5695 lck_mtx_unlock(&dl_if->dl_if_lock);
5696 if (need_release) {
5697 dlil_if_release(ifp);
5698 }
5699 return (0);
5700 }
5701
5702 static errno_t
5703 dlil_attach_protocol_internal(struct if_proto *proto,
5704 const struct ifnet_demux_desc *demux_list, u_int32_t demux_count,
5705 uint32_t * proto_count)
5706 {
5707 struct kev_dl_proto_data ev_pr_data;
5708 struct ifnet *ifp = proto->ifp;
5709 int retval = 0;
5710 u_int32_t hash_value = proto_hash_value(proto->protocol_family);
5711 struct if_proto *prev_proto;
5712 struct if_proto *_proto;
5713
5714 /* callee holds a proto refcnt upon success */
5715 ifnet_lock_exclusive(ifp);
5716 _proto = find_attached_proto(ifp, proto->protocol_family);
5717 if (_proto != NULL) {
5718 ifnet_lock_done(ifp);
5719 if_proto_free(_proto);
5720 return (EEXIST);
5721 }
5722
5723 /*
5724 * Call family module add_proto routine so it can refine the
5725 * demux descriptors as it wishes.
5726 */
5727 retval = ifp->if_add_proto(ifp, proto->protocol_family, demux_list,
5728 demux_count);
5729 if (retval) {
5730 ifnet_lock_done(ifp);
5731 return (retval);
5732 }
5733
5734 /*
5735 * Insert the protocol in the hash
5736 */
5737 prev_proto = SLIST_FIRST(&ifp->if_proto_hash[hash_value]);
5738 while (prev_proto != NULL && SLIST_NEXT(prev_proto, next_hash) != NULL)
5739 prev_proto = SLIST_NEXT(prev_proto, next_hash);
5740 if (prev_proto)
5741 SLIST_INSERT_AFTER(prev_proto, proto, next_hash);
5742 else
5743 SLIST_INSERT_HEAD(&ifp->if_proto_hash[hash_value],
5744 proto, next_hash);
5745
5746 /* hold a proto refcnt for attach */
5747 if_proto_ref(proto);
5748
5749 /*
5750 * The reserved field carries the number of protocol still attached
5751 * (subject to change)
5752 */
5753 ev_pr_data.proto_family = proto->protocol_family;
5754 ev_pr_data.proto_remaining_count = dlil_ifp_protolist(ifp, NULL, 0);
5755
5756 ifnet_lock_done(ifp);
5757
5758 dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_PROTO_ATTACHED,
5759 (struct net_event_data *)&ev_pr_data,
5760 sizeof (struct kev_dl_proto_data));
5761 if (proto_count != NULL) {
5762 *proto_count = ev_pr_data.proto_remaining_count;
5763 }
5764 return (retval);
5765 }
5766
5767 errno_t
5768 ifnet_attach_protocol(ifnet_t ifp, protocol_family_t protocol,
5769 const struct ifnet_attach_proto_param *proto_details)
5770 {
5771 int retval = 0;
5772 struct if_proto *ifproto = NULL;
5773 uint32_t proto_count = 0;
5774
5775 ifnet_head_lock_shared();
5776 if (ifp == NULL || protocol == 0 || proto_details == NULL) {
5777 retval = EINVAL;
5778 goto end;
5779 }
5780 /* Check that the interface is in the global list */
5781 if (!ifnet_lookup(ifp)) {
5782 retval = ENXIO;
5783 goto end;
5784 }
5785
5786 ifproto = zalloc(dlif_proto_zone);
5787 if (ifproto == NULL) {
5788 retval = ENOMEM;
5789 goto end;
5790 }
5791 bzero(ifproto, dlif_proto_size);
5792
5793 /* refcnt held above during lookup */
5794 ifproto->ifp = ifp;
5795 ifproto->protocol_family = protocol;
5796 ifproto->proto_kpi = kProtoKPI_v1;
5797 ifproto->kpi.v1.input = proto_details->input;
5798 ifproto->kpi.v1.pre_output = proto_details->pre_output;
5799 ifproto->kpi.v1.event = proto_details->event;
5800 ifproto->kpi.v1.ioctl = proto_details->ioctl;
5801 ifproto->kpi.v1.detached = proto_details->detached;
5802 ifproto->kpi.v1.resolve_multi = proto_details->resolve;
5803 ifproto->kpi.v1.send_arp = proto_details->send_arp;
5804
5805 retval = dlil_attach_protocol_internal(ifproto,
5806 proto_details->demux_list, proto_details->demux_count,
5807 &proto_count);
5808
5809 end:
5810 if (retval != 0 && retval != EEXIST && ifp != NULL) {
5811 DLIL_PRINTF("%s: failed to attach v1 protocol %d (err=%d)\n",
5812 if_name(ifp), protocol, retval);
5813 } else {
5814 if (dlil_verbose) {
5815 printf("%s: attached v1 protocol %d (count = %d)\n",
5816 if_name(ifp),
5817 protocol, proto_count);
5818 }
5819 }
5820 ifnet_head_done();
5821 if (retval == 0) {
5822 /*
5823 * A protocol has been attached, mark the interface up.
5824 * This used to be done by configd.KernelEventMonitor, but that
5825 * is inherently prone to races (rdar://problem/30810208).
5826 */
5827 (void) ifnet_set_flags(ifp, IFF_UP, IFF_UP);
5828 (void) ifnet_ioctl(ifp, 0, SIOCSIFFLAGS, NULL);
5829 dlil_post_sifflags_msg(ifp);
5830 } else if (ifproto != NULL) {
5831 zfree(dlif_proto_zone, ifproto);
5832 }
5833 return (retval);
5834 }
5835
5836 errno_t
5837 ifnet_attach_protocol_v2(ifnet_t ifp, protocol_family_t protocol,
5838 const struct ifnet_attach_proto_param_v2 *proto_details)
5839 {
5840 int retval = 0;
5841 struct if_proto *ifproto = NULL;
5842 uint32_t proto_count = 0;
5843
5844 ifnet_head_lock_shared();
5845 if (ifp == NULL || protocol == 0 || proto_details == NULL) {
5846 retval = EINVAL;
5847 goto end;
5848 }
5849 /* Check that the interface is in the global list */
5850 if (!ifnet_lookup(ifp)) {
5851 retval = ENXIO;
5852 goto end;
5853 }
5854
5855 ifproto = zalloc(dlif_proto_zone);
5856 if (ifproto == NULL) {
5857 retval = ENOMEM;
5858 goto end;
5859 }
5860 bzero(ifproto, sizeof(*ifproto));
5861
5862 /* refcnt held above during lookup */
5863 ifproto->ifp = ifp;
5864 ifproto->protocol_family = protocol;
5865 ifproto->proto_kpi = kProtoKPI_v2;
5866 ifproto->kpi.v2.input = proto_details->input;
5867 ifproto->kpi.v2.pre_output = proto_details->pre_output;
5868 ifproto->kpi.v2.event = proto_details->event;
5869 ifproto->kpi.v2.ioctl = proto_details->ioctl;
5870 ifproto->kpi.v2.detached = proto_details->detached;
5871 ifproto->kpi.v2.resolve_multi = proto_details->resolve;
5872 ifproto->kpi.v2.send_arp = proto_details->send_arp;
5873
5874 retval = dlil_attach_protocol_internal(ifproto,
5875 proto_details->demux_list, proto_details->demux_count,
5876 &proto_count);
5877
5878 end:
5879 if (retval != 0 && retval != EEXIST && ifp != NULL) {
5880 DLIL_PRINTF("%s: failed to attach v2 protocol %d (err=%d)\n",
5881 if_name(ifp), protocol, retval);
5882 } else {
5883 if (dlil_verbose) {
5884 printf("%s: attached v2 protocol %d (count = %d)\n",
5885 if_name(ifp),
5886 protocol, proto_count);
5887 }
5888 }
5889 ifnet_head_done();
5890 if (retval == 0) {
5891 /*
5892 * A protocol has been attached, mark the interface up.
5893 * This used to be done by configd.KernelEventMonitor, but that
5894 * is inherently prone to races (rdar://problem/30810208).
5895 */
5896 (void) ifnet_set_flags(ifp, IFF_UP, IFF_UP);
5897 (void) ifnet_ioctl(ifp, 0, SIOCSIFFLAGS, NULL);
5898 dlil_post_sifflags_msg(ifp);
5899 } else if (ifproto != NULL) {
5900 zfree(dlif_proto_zone, ifproto);
5901 }
5902 return (retval);
5903 }
5904
5905 errno_t
5906 ifnet_detach_protocol(ifnet_t ifp, protocol_family_t proto_family)
5907 {
5908 struct if_proto *proto = NULL;
5909 int retval = 0;
5910
5911 if (ifp == NULL || proto_family == 0) {
5912 retval = EINVAL;
5913 goto end;
5914 }
5915
5916 ifnet_lock_exclusive(ifp);
5917 /* callee holds a proto refcnt upon success */
5918 proto = find_attached_proto(ifp, proto_family);
5919 if (proto == NULL) {
5920 retval = ENXIO;
5921 ifnet_lock_done(ifp);
5922 goto end;
5923 }
5924
5925 /* call family module del_proto */
5926 if (ifp->if_del_proto)
5927 ifp->if_del_proto(ifp, proto->protocol_family);
5928
5929 SLIST_REMOVE(&ifp->if_proto_hash[proto_hash_value(proto_family)],
5930 proto, if_proto, next_hash);
5931
5932 if (proto->proto_kpi == kProtoKPI_v1) {
5933 proto->kpi.v1.input = ifproto_media_input_v1;
5934 proto->kpi.v1.pre_output = ifproto_media_preout;
5935 proto->kpi.v1.event = ifproto_media_event;
5936 proto->kpi.v1.ioctl = ifproto_media_ioctl;
5937 proto->kpi.v1.resolve_multi = ifproto_media_resolve_multi;
5938 proto->kpi.v1.send_arp = ifproto_media_send_arp;
5939 } else {
5940 proto->kpi.v2.input = ifproto_media_input_v2;
5941 proto->kpi.v2.pre_output = ifproto_media_preout;
5942 proto->kpi.v2.event = ifproto_media_event;
5943 proto->kpi.v2.ioctl = ifproto_media_ioctl;
5944 proto->kpi.v2.resolve_multi = ifproto_media_resolve_multi;
5945 proto->kpi.v2.send_arp = ifproto_media_send_arp;
5946 }
5947 proto->detached = 1;
5948 ifnet_lock_done(ifp);
5949
5950 if (dlil_verbose) {
5951 printf("%s: detached %s protocol %d\n", if_name(ifp),
5952 (proto->proto_kpi == kProtoKPI_v1) ?
5953 "v1" : "v2", proto_family);
5954 }
5955
5956 /* release proto refcnt held during protocol attach */
5957 if_proto_free(proto);
5958
5959 /*
5960 * Release proto refcnt held during lookup; the rest of
5961 * protocol detach steps will happen when the last proto
5962 * reference is released.
5963 */
5964 if_proto_free(proto);
5965
5966 end:
5967 return (retval);
5968 }
5969
5970
5971 static errno_t
5972 ifproto_media_input_v1(struct ifnet *ifp, protocol_family_t protocol,
5973 struct mbuf *packet, char *header)
5974 {
5975 #pragma unused(ifp, protocol, packet, header)
5976 return (ENXIO);
5977 }
5978
5979 static errno_t
5980 ifproto_media_input_v2(struct ifnet *ifp, protocol_family_t protocol,
5981 struct mbuf *packet)
5982 {
5983 #pragma unused(ifp, protocol, packet)
5984 return (ENXIO);
5985
5986 }
5987
5988 static errno_t
5989 ifproto_media_preout(struct ifnet *ifp, protocol_family_t protocol,
5990 mbuf_t *packet, const struct sockaddr *dest, void *route, char *frame_type,
5991 char *link_layer_dest)
5992 {
5993 #pragma unused(ifp, protocol, packet, dest, route, frame_type, link_layer_dest)
5994 return (ENXIO);
5995
5996 }
5997
5998 static void
5999 ifproto_media_event(struct ifnet *ifp, protocol_family_t protocol,
6000 const struct kev_msg *event)
6001 {
6002 #pragma unused(ifp, protocol, event)
6003 }
6004
6005 static errno_t
6006 ifproto_media_ioctl(struct ifnet *ifp, protocol_family_t protocol,
6007 unsigned long command, void *argument)
6008 {
6009 #pragma unused(ifp, protocol, command, argument)
6010 return (ENXIO);
6011 }
6012
6013 static errno_t
6014 ifproto_media_resolve_multi(ifnet_t ifp, const struct sockaddr *proto_addr,
6015 struct sockaddr_dl *out_ll, size_t ll_len)
6016 {
6017 #pragma unused(ifp, proto_addr, out_ll, ll_len)
6018 return (ENXIO);
6019 }
6020
6021 static errno_t
6022 ifproto_media_send_arp(struct ifnet *ifp, u_short arpop,
6023 const struct sockaddr_dl *sender_hw, const struct sockaddr *sender_proto,
6024 const struct sockaddr_dl *target_hw, const struct sockaddr *target_proto)
6025 {
6026 #pragma unused(ifp, arpop, sender_hw, sender_proto, target_hw, target_proto)
6027 return (ENXIO);
6028 }
6029
6030 extern int if_next_index(void);
6031 extern int tcp_ecn_outbound;
6032
6033 errno_t
6034 ifnet_attach(ifnet_t ifp, const struct sockaddr_dl *ll_addr)
6035 {
6036 struct ifnet *tmp_if;
6037 struct ifaddr *ifa;
6038 struct if_data_internal if_data_saved;
6039 struct dlil_ifnet *dl_if = (struct dlil_ifnet *)ifp;
6040 struct dlil_threading_info *dl_inp;
6041 u_int32_t sflags = 0;
6042 int err;
6043
6044 if (ifp == NULL)
6045 return (EINVAL);
6046
6047 /*
6048 * Serialize ifnet attach using dlil_ifnet_lock, in order to
6049 * prevent the interface from being configured while it is
6050 * embryonic, as ifnet_head_lock is dropped and reacquired
6051 * below prior to marking the ifnet with IFRF_ATTACHED.
6052 */
6053 dlil_if_lock();
6054 ifnet_head_lock_exclusive();
6055 /* Verify we aren't already on the list */
6056 TAILQ_FOREACH(tmp_if, &ifnet_head, if_link) {
6057 if (tmp_if == ifp) {
6058 ifnet_head_done();
6059 dlil_if_unlock();
6060 return (EEXIST);
6061 }
6062 }
6063
6064 lck_mtx_lock_spin(&ifp->if_ref_lock);
6065 if (!(ifp->if_refflags & IFRF_EMBRYONIC)) {
6066 panic_plain("%s: flags mismatch (embryonic not set) ifp=%p",
6067 __func__, ifp);
6068 /* NOTREACHED */
6069 }
6070 lck_mtx_unlock(&ifp->if_ref_lock);
6071
6072 ifnet_lock_exclusive(ifp);
6073
6074 /* Sanity check */
6075 VERIFY(ifp->if_detaching_link.tqe_next == NULL);
6076 VERIFY(ifp->if_detaching_link.tqe_prev == NULL);
6077
6078 if (ll_addr != NULL) {
6079 if (ifp->if_addrlen == 0) {
6080 ifp->if_addrlen = ll_addr->sdl_alen;
6081 } else if (ll_addr->sdl_alen != ifp->if_addrlen) {
6082 ifnet_lock_done(ifp);
6083 ifnet_head_done();
6084 dlil_if_unlock();
6085 return (EINVAL);
6086 }
6087 }
6088
6089 /*
6090 * Allow interfaces without protocol families to attach
6091 * only if they have the necessary fields filled out.
6092 */
6093 if (ifp->if_add_proto == NULL || ifp->if_del_proto == NULL) {
6094 DLIL_PRINTF("%s: Attempt to attach interface without "
6095 "family module - %d\n", __func__, ifp->if_family);
6096 ifnet_lock_done(ifp);
6097 ifnet_head_done();
6098 dlil_if_unlock();
6099 return (ENODEV);
6100 }
6101
6102 /* Allocate protocol hash table */
6103 VERIFY(ifp->if_proto_hash == NULL);
6104 ifp->if_proto_hash = zalloc(dlif_phash_zone);
6105 if (ifp->if_proto_hash == NULL) {
6106 ifnet_lock_done(ifp);
6107 ifnet_head_done();
6108 dlil_if_unlock();
6109 return (ENOBUFS);
6110 }
6111 bzero(ifp->if_proto_hash, dlif_phash_size);
6112
6113 lck_mtx_lock_spin(&ifp->if_flt_lock);
6114 VERIFY(TAILQ_EMPTY(&ifp->if_flt_head));
6115 TAILQ_INIT(&ifp->if_flt_head);
6116 VERIFY(ifp->if_flt_busy == 0);
6117 VERIFY(ifp->if_flt_waiters == 0);
6118 lck_mtx_unlock(&ifp->if_flt_lock);
6119
6120 if (!(dl_if->dl_if_flags & DLIF_REUSE)) {
6121 VERIFY(LIST_EMPTY(&ifp->if_multiaddrs));
6122 LIST_INIT(&ifp->if_multiaddrs);
6123 }
6124
6125 VERIFY(ifp->if_allhostsinm == NULL);
6126 VERIFY(TAILQ_EMPTY(&ifp->if_addrhead));
6127 TAILQ_INIT(&ifp->if_addrhead);
6128
6129 if (ifp->if_index == 0) {
6130 int idx = if_next_index();
6131
6132 if (idx == -1) {
6133 ifp->if_index = 0;
6134 ifnet_lock_done(ifp);
6135 ifnet_head_done();
6136 dlil_if_unlock();
6137 return (ENOBUFS);
6138 }
6139 ifp->if_index = idx;
6140 }
6141 /* There should not be anything occupying this slot */
6142 VERIFY(ifindex2ifnet[ifp->if_index] == NULL);
6143
6144 /* allocate (if needed) and initialize a link address */
6145 ifa = dlil_alloc_lladdr(ifp, ll_addr);
6146 if (ifa == NULL) {
6147 ifnet_lock_done(ifp);
6148 ifnet_head_done();
6149 dlil_if_unlock();
6150 return (ENOBUFS);
6151 }
6152
6153 VERIFY(ifnet_addrs[ifp->if_index - 1] == NULL);
6154 ifnet_addrs[ifp->if_index - 1] = ifa;
6155
6156 /* make this address the first on the list */
6157 IFA_LOCK(ifa);
6158 /* hold a reference for ifnet_addrs[] */
6159 IFA_ADDREF_LOCKED(ifa);
6160 /* if_attach_link_ifa() holds a reference for ifa_link */
6161 if_attach_link_ifa(ifp, ifa);
6162 IFA_UNLOCK(ifa);
6163
6164 #if CONFIG_MACF_NET
6165 mac_ifnet_label_associate(ifp);
6166 #endif
6167
6168 TAILQ_INSERT_TAIL(&ifnet_head, ifp, if_link);
6169 ifindex2ifnet[ifp->if_index] = ifp;
6170
6171 /* Hold a reference to the underlying dlil_ifnet */
6172 ifnet_reference(ifp);
6173
6174 /* Clear stats (save and restore other fields that we care) */
6175 if_data_saved = ifp->if_data;
6176 bzero(&ifp->if_data, sizeof (ifp->if_data));
6177 ifp->if_data.ifi_type = if_data_saved.ifi_type;
6178 ifp->if_data.ifi_typelen = if_data_saved.ifi_typelen;
6179 ifp->if_data.ifi_physical = if_data_saved.ifi_physical;
6180 ifp->if_data.ifi_addrlen = if_data_saved.ifi_addrlen;
6181 ifp->if_data.ifi_hdrlen = if_data_saved.ifi_hdrlen;
6182 ifp->if_data.ifi_mtu = if_data_saved.ifi_mtu;
6183 ifp->if_data.ifi_baudrate = if_data_saved.ifi_baudrate;
6184 ifp->if_data.ifi_hwassist = if_data_saved.ifi_hwassist;
6185 ifp->if_data.ifi_tso_v4_mtu = if_data_saved.ifi_tso_v4_mtu;
6186 ifp->if_data.ifi_tso_v6_mtu = if_data_saved.ifi_tso_v6_mtu;
6187 ifnet_touch_lastchange(ifp);
6188
6189 VERIFY(ifp->if_output_sched_model == IFNET_SCHED_MODEL_NORMAL ||
6190 ifp->if_output_sched_model == IFNET_SCHED_MODEL_DRIVER_MANAGED ||
6191 ifp->if_output_sched_model == IFNET_SCHED_MODEL_FQ_CODEL);
6192
6193 /* By default, use SFB and enable flow advisory */
6194 sflags = PKTSCHEDF_QALG_SFB;
6195 if (if_flowadv)
6196 sflags |= PKTSCHEDF_QALG_FLOWCTL;
6197
6198 if (if_delaybased_queue)
6199 sflags |= PKTSCHEDF_QALG_DELAYBASED;
6200
6201 if (ifp->if_output_sched_model ==
6202 IFNET_SCHED_MODEL_DRIVER_MANAGED)
6203 sflags |= PKTSCHEDF_QALG_DRIVER_MANAGED;
6204
6205 /* Initialize transmit queue(s) */
6206 err = ifclassq_setup(ifp, sflags, (dl_if->dl_if_flags & DLIF_REUSE));
6207 if (err != 0) {
6208 panic_plain("%s: ifp=%p couldn't initialize transmit queue; "
6209 "err=%d", __func__, ifp, err);
6210 /* NOTREACHED */
6211 }
6212
6213 /* Sanity checks on the input thread storage */
6214 dl_inp = &dl_if->dl_if_inpstorage;
6215 bzero(&dl_inp->stats, sizeof (dl_inp->stats));
6216 VERIFY(dl_inp->input_waiting == 0);
6217 VERIFY(dl_inp->wtot == 0);
6218 VERIFY(dl_inp->ifp == NULL);
6219 VERIFY(qhead(&dl_inp->rcvq_pkts) == NULL && qempty(&dl_inp->rcvq_pkts));
6220 VERIFY(qlimit(&dl_inp->rcvq_pkts) == 0);
6221 VERIFY(!dl_inp->net_affinity);
6222 VERIFY(ifp->if_inp == NULL);
6223 VERIFY(dl_inp->input_thr == THREAD_NULL);
6224 VERIFY(dl_inp->wloop_thr == THREAD_NULL);
6225 VERIFY(dl_inp->poll_thr == THREAD_NULL);
6226 VERIFY(dl_inp->tag == 0);
6227 VERIFY(dl_inp->mode == IFNET_MODEL_INPUT_POLL_OFF);
6228 bzero(&dl_inp->tstats, sizeof (dl_inp->tstats));
6229 bzero(&dl_inp->pstats, sizeof (dl_inp->pstats));
6230 bzero(&dl_inp->sstats, sizeof (dl_inp->sstats));
6231 #if IFNET_INPUT_SANITY_CHK
6232 VERIFY(dl_inp->input_mbuf_cnt == 0);
6233 #endif /* IFNET_INPUT_SANITY_CHK */
6234
6235 /*
6236 * A specific DLIL input thread is created per Ethernet/cellular
6237 * interface or for an interface which supports opportunistic
6238 * input polling. Pseudo interfaces or other types of interfaces
6239 * use the main input thread instead.
6240 */
6241 if ((net_rxpoll && (ifp->if_eflags & IFEF_RXPOLL)) ||
6242 ifp->if_type == IFT_ETHER || ifp->if_type == IFT_CELLULAR) {
6243 ifp->if_inp = dl_inp;
6244 err = dlil_create_input_thread(ifp, ifp->if_inp);
6245 if (err != 0) {
6246 panic_plain("%s: ifp=%p couldn't get an input thread; "
6247 "err=%d", __func__, ifp, err);
6248 /* NOTREACHED */
6249 }
6250 }
6251
6252 if (ifp->if_inp != NULL && ifp->if_inp->input_mit_tcall == NULL) {
6253 ifp->if_inp->input_mit_tcall =
6254 thread_call_allocate_with_priority(dlil_mit_tcall_fn,
6255 ifp, THREAD_CALL_PRIORITY_KERNEL);
6256 }
6257
6258 /*
6259 * If the driver supports the new transmit model, calculate flow hash
6260 * and create a workloop starter thread to invoke the if_start callback
6261 * where the packets may be dequeued and transmitted.
6262 */
6263 if (ifp->if_eflags & IFEF_TXSTART) {
6264 ifp->if_flowhash = ifnet_calc_flowhash(ifp);
6265 VERIFY(ifp->if_flowhash != 0);
6266 VERIFY(ifp->if_start_thread == THREAD_NULL);
6267
6268 ifnet_set_start_cycle(ifp, NULL);
6269 ifp->if_start_active = 0;
6270 ifp->if_start_req = 0;
6271 ifp->if_start_flags = 0;
6272 VERIFY(ifp->if_start != NULL);
6273 if ((err = kernel_thread_start(ifnet_start_thread_fn,
6274 ifp, &ifp->if_start_thread)) != KERN_SUCCESS) {
6275 panic_plain("%s: "
6276 "ifp=%p couldn't get a start thread; "
6277 "err=%d", __func__, ifp, err);
6278 /* NOTREACHED */
6279 }
6280 ml_thread_policy(ifp->if_start_thread, MACHINE_GROUP,
6281 (MACHINE_NETWORK_GROUP|MACHINE_NETWORK_WORKLOOP));
6282 } else {
6283 ifp->if_flowhash = 0;
6284 }
6285
6286 /*
6287 * If the driver supports the new receive model, create a poller
6288 * thread to invoke if_input_poll callback where the packets may
6289 * be dequeued from the driver and processed for reception.
6290 */
6291 if (ifp->if_eflags & IFEF_RXPOLL) {
6292 VERIFY(ifp->if_input_poll != NULL);
6293 VERIFY(ifp->if_input_ctl != NULL);
6294 VERIFY(ifp->if_poll_thread == THREAD_NULL);
6295
6296 ifnet_set_poll_cycle(ifp, NULL);
6297 ifp->if_poll_update = 0;
6298 ifp->if_poll_active = 0;
6299 ifp->if_poll_req = 0;
6300 if ((err = kernel_thread_start(ifnet_poll_thread_fn, ifp,
6301 &ifp->if_poll_thread)) != KERN_SUCCESS) {
6302 panic_plain("%s: ifp=%p couldn't get a poll thread; "
6303 "err=%d", __func__, ifp, err);
6304 /* NOTREACHED */
6305 }
6306 ml_thread_policy(ifp->if_poll_thread, MACHINE_GROUP,
6307 (MACHINE_NETWORK_GROUP|MACHINE_NETWORK_WORKLOOP));
6308 }
6309
6310 VERIFY(ifp->if_desc.ifd_maxlen == IF_DESCSIZE);
6311 VERIFY(ifp->if_desc.ifd_len == 0);
6312 VERIFY(ifp->if_desc.ifd_desc != NULL);
6313
6314 /* Record attach PC stacktrace */
6315 ctrace_record(&((struct dlil_ifnet *)ifp)->dl_if_attach);
6316
6317 ifp->if_updatemcasts = 0;
6318 if (!LIST_EMPTY(&ifp->if_multiaddrs)) {
6319 struct ifmultiaddr *ifma;
6320 LIST_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
6321 IFMA_LOCK(ifma);
6322 if (ifma->ifma_addr->sa_family == AF_LINK ||
6323 ifma->ifma_addr->sa_family == AF_UNSPEC)
6324 ifp->if_updatemcasts++;
6325 IFMA_UNLOCK(ifma);
6326 }
6327
6328 printf("%s: attached with %d suspended link-layer multicast "
6329 "membership(s)\n", if_name(ifp),
6330 ifp->if_updatemcasts);
6331 }
6332
6333 /* Clear logging parameters */
6334 bzero(&ifp->if_log, sizeof (ifp->if_log));
6335
6336 /* Clear foreground/realtime activity timestamps */
6337 ifp->if_fg_sendts = 0;
6338 ifp->if_rt_sendts = 0;
6339
6340 VERIFY(ifp->if_delegated.ifp == NULL);
6341 VERIFY(ifp->if_delegated.type == 0);
6342 VERIFY(ifp->if_delegated.family == 0);
6343 VERIFY(ifp->if_delegated.subfamily == 0);
6344 VERIFY(ifp->if_delegated.expensive == 0);
6345
6346 VERIFY(ifp->if_agentids == NULL);
6347 VERIFY(ifp->if_agentcount == 0);
6348
6349 /* Reset interface state */
6350 bzero(&ifp->if_interface_state, sizeof(ifp->if_interface_state));
6351 ifp->if_interface_state.valid_bitmask |=
6352 IF_INTERFACE_STATE_INTERFACE_AVAILABILITY_VALID;
6353 ifp->if_interface_state.interface_availability =
6354 IF_INTERFACE_STATE_INTERFACE_AVAILABLE;
6355
6356 /* Initialize Link Quality Metric (loopback [lo0] is always good) */
6357 if (ifp == lo_ifp) {
6358 ifp->if_interface_state.lqm_state = IFNET_LQM_THRESH_GOOD;
6359 ifp->if_interface_state.valid_bitmask |=
6360 IF_INTERFACE_STATE_LQM_STATE_VALID;
6361 } else {
6362 ifp->if_interface_state.lqm_state = IFNET_LQM_THRESH_UNKNOWN;
6363 }
6364
6365 /*
6366 * Enable ECN capability on this interface depending on the
6367 * value of ECN global setting
6368 */
6369 if (tcp_ecn_outbound == 2 && !IFNET_IS_CELLULAR(ifp)) {
6370 ifp->if_eflags |= IFEF_ECN_ENABLE;
6371 ifp->if_eflags &= ~IFEF_ECN_DISABLE;
6372 }
6373
6374 /*
6375 * Built-in Cyclops always on policy for WiFi infra
6376 */
6377 if (IFNET_IS_WIFI_INFRA(ifp) && net_qos_policy_wifi_enabled != 0) {
6378 errno_t error;
6379
6380 error = if_set_qosmarking_mode(ifp,
6381 IFRTYPE_QOSMARKING_FASTLANE);
6382 if (error != 0) {
6383 printf("%s if_set_qosmarking_mode(%s) error %d\n",
6384 __func__, ifp->if_xname, error);
6385 } else {
6386 ifp->if_eflags |= IFEF_QOSMARKING_ENABLED;
6387 #if (DEVELOPMENT || DEBUG)
6388 printf("%s fastlane enabled on %s\n",
6389 __func__, ifp->if_xname);
6390 #endif /* (DEVELOPMENT || DEBUG) */
6391 }
6392 }
6393
6394 ifnet_lock_done(ifp);
6395 ifnet_head_done();
6396
6397
6398 lck_mtx_lock(&ifp->if_cached_route_lock);
6399 /* Enable forwarding cached route */
6400 ifp->if_fwd_cacheok = 1;
6401 /* Clean up any existing cached routes */
6402 ROUTE_RELEASE(&ifp->if_fwd_route);
6403 bzero(&ifp->if_fwd_route, sizeof (ifp->if_fwd_route));
6404 ROUTE_RELEASE(&ifp->if_src_route);
6405 bzero(&ifp->if_src_route, sizeof (ifp->if_src_route));
6406 ROUTE_RELEASE(&ifp->if_src_route6);
6407 bzero(&ifp->if_src_route6, sizeof (ifp->if_src_route6));
6408 lck_mtx_unlock(&ifp->if_cached_route_lock);
6409
6410 ifnet_llreach_ifattach(ifp, (dl_if->dl_if_flags & DLIF_REUSE));
6411
6412 /*
6413 * Allocate and attach IGMPv3/MLDv2 interface specific variables
6414 * and trees; do this before the ifnet is marked as attached.
6415 * The ifnet keeps the reference to the info structures even after
6416 * the ifnet is detached, since the network-layer records still
6417 * refer to the info structures even after that. This also
6418 * makes it possible for them to still function after the ifnet
6419 * is recycled or reattached.
6420 */
6421 #if INET
6422 if (IGMP_IFINFO(ifp) == NULL) {
6423 IGMP_IFINFO(ifp) = igmp_domifattach(ifp, M_WAITOK);
6424 VERIFY(IGMP_IFINFO(ifp) != NULL);
6425 } else {
6426 VERIFY(IGMP_IFINFO(ifp)->igi_ifp == ifp);
6427 igmp_domifreattach(IGMP_IFINFO(ifp));
6428 }
6429 #endif /* INET */
6430 #if INET6
6431 if (MLD_IFINFO(ifp) == NULL) {
6432 MLD_IFINFO(ifp) = mld_domifattach(ifp, M_WAITOK);
6433 VERIFY(MLD_IFINFO(ifp) != NULL);
6434 } else {
6435 VERIFY(MLD_IFINFO(ifp)->mli_ifp == ifp);
6436 mld_domifreattach(MLD_IFINFO(ifp));
6437 }
6438 #endif /* INET6 */
6439
6440 VERIFY(ifp->if_data_threshold == 0);
6441 VERIFY(ifp->if_dt_tcall != NULL);
6442
6443 /*
6444 * Finally, mark this ifnet as attached.
6445 */
6446 lck_mtx_lock(rnh_lock);
6447 ifnet_lock_exclusive(ifp);
6448 lck_mtx_lock_spin(&ifp->if_ref_lock);
6449 ifp->if_refflags = IFRF_ATTACHED; /* clears embryonic */
6450 lck_mtx_unlock(&ifp->if_ref_lock);
6451 if (net_rtref) {
6452 /* boot-args override; enable idle notification */
6453 (void) ifnet_set_idle_flags_locked(ifp, IFRF_IDLE_NOTIFY,
6454 IFRF_IDLE_NOTIFY);
6455 } else {
6456 /* apply previous request(s) to set the idle flags, if any */
6457 (void) ifnet_set_idle_flags_locked(ifp, ifp->if_idle_new_flags,
6458 ifp->if_idle_new_flags_mask);
6459
6460 }
6461 ifnet_lock_done(ifp);
6462 lck_mtx_unlock(rnh_lock);
6463 dlil_if_unlock();
6464
6465 #if PF
6466 /*
6467 * Attach packet filter to this interface, if enabled.
6468 */
6469 pf_ifnet_hook(ifp, 1);
6470 #endif /* PF */
6471
6472 dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_IF_ATTACHED, NULL, 0);
6473
6474 if (dlil_verbose) {
6475 printf("%s: attached%s\n", if_name(ifp),
6476 (dl_if->dl_if_flags & DLIF_REUSE) ? " (recycled)" : "");
6477 }
6478
6479 return (0);
6480 }
6481
6482 /*
6483 * Prepare the storage for the first/permanent link address, which must
6484 * must have the same lifetime as the ifnet itself. Although the link
6485 * address gets removed from if_addrhead and ifnet_addrs[] at detach time,
6486 * its location in memory must never change as it may still be referred
6487 * to by some parts of the system afterwards (unfortunate implementation
6488 * artifacts inherited from BSD.)
6489 *
6490 * Caller must hold ifnet lock as writer.
6491 */
6492 static struct ifaddr *
6493 dlil_alloc_lladdr(struct ifnet *ifp, const struct sockaddr_dl *ll_addr)
6494 {
6495 struct ifaddr *ifa, *oifa;
6496 struct sockaddr_dl *asdl, *msdl;
6497 char workbuf[IFNAMSIZ*2];
6498 int namelen, masklen, socksize;
6499 struct dlil_ifnet *dl_if = (struct dlil_ifnet *)ifp;
6500
6501 ifnet_lock_assert(ifp, IFNET_LCK_ASSERT_EXCLUSIVE);
6502 VERIFY(ll_addr == NULL || ll_addr->sdl_alen == ifp->if_addrlen);
6503
6504 namelen = snprintf(workbuf, sizeof (workbuf), "%s",
6505 if_name(ifp));
6506 masklen = offsetof(struct sockaddr_dl, sdl_data[0])
6507 + ((namelen > 0) ? namelen : 0);
6508 socksize = masklen + ifp->if_addrlen;
6509 #define ROUNDUP(a) (1 + (((a) - 1) | (sizeof (u_int32_t) - 1)))
6510 if ((u_int32_t)socksize < sizeof (struct sockaddr_dl))
6511 socksize = sizeof(struct sockaddr_dl);
6512 socksize = ROUNDUP(socksize);
6513 #undef ROUNDUP
6514
6515 ifa = ifp->if_lladdr;
6516 if (socksize > DLIL_SDLMAXLEN ||
6517 (ifa != NULL && ifa != &dl_if->dl_if_lladdr.ifa)) {
6518 /*
6519 * Rare, but in the event that the link address requires
6520 * more storage space than DLIL_SDLMAXLEN, allocate the
6521 * largest possible storages for address and mask, such
6522 * that we can reuse the same space when if_addrlen grows.
6523 * This same space will be used when if_addrlen shrinks.
6524 */
6525 if (ifa == NULL || ifa == &dl_if->dl_if_lladdr.ifa) {
6526 int ifasize = sizeof (*ifa) + 2 * SOCK_MAXADDRLEN;
6527 ifa = _MALLOC(ifasize, M_IFADDR, M_WAITOK | M_ZERO);
6528 if (ifa == NULL)
6529 return (NULL);
6530 ifa_lock_init(ifa);
6531 /* Don't set IFD_ALLOC, as this is permanent */
6532 ifa->ifa_debug = IFD_LINK;
6533 }
6534 IFA_LOCK(ifa);
6535 /* address and mask sockaddr_dl locations */
6536 asdl = (struct sockaddr_dl *)(ifa + 1);
6537 bzero(asdl, SOCK_MAXADDRLEN);
6538 msdl = (struct sockaddr_dl *)(void *)
6539 ((char *)asdl + SOCK_MAXADDRLEN);
6540 bzero(msdl, SOCK_MAXADDRLEN);
6541 } else {
6542 VERIFY(ifa == NULL || ifa == &dl_if->dl_if_lladdr.ifa);
6543 /*
6544 * Use the storage areas for address and mask within the
6545 * dlil_ifnet structure. This is the most common case.
6546 */
6547 if (ifa == NULL) {
6548 ifa = &dl_if->dl_if_lladdr.ifa;
6549 ifa_lock_init(ifa);
6550 /* Don't set IFD_ALLOC, as this is permanent */
6551 ifa->ifa_debug = IFD_LINK;
6552 }
6553 IFA_LOCK(ifa);
6554 /* address and mask sockaddr_dl locations */
6555 asdl = (struct sockaddr_dl *)(void *)&dl_if->dl_if_lladdr.asdl;
6556 bzero(asdl, sizeof (dl_if->dl_if_lladdr.asdl));
6557 msdl = (struct sockaddr_dl *)(void *)&dl_if->dl_if_lladdr.msdl;
6558 bzero(msdl, sizeof (dl_if->dl_if_lladdr.msdl));
6559 }
6560
6561 /* hold a permanent reference for the ifnet itself */
6562 IFA_ADDREF_LOCKED(ifa);
6563 oifa = ifp->if_lladdr;
6564 ifp->if_lladdr = ifa;
6565
6566 VERIFY(ifa->ifa_debug == IFD_LINK);
6567 ifa->ifa_ifp = ifp;
6568 ifa->ifa_rtrequest = link_rtrequest;
6569 ifa->ifa_addr = (struct sockaddr *)asdl;
6570 asdl->sdl_len = socksize;
6571 asdl->sdl_family = AF_LINK;
6572 if (namelen > 0) {
6573 bcopy(workbuf, asdl->sdl_data, min(namelen,
6574 sizeof (asdl->sdl_data)));
6575 asdl->sdl_nlen = namelen;
6576 } else {
6577 asdl->sdl_nlen = 0;
6578 }
6579 asdl->sdl_index = ifp->if_index;
6580 asdl->sdl_type = ifp->if_type;
6581 if (ll_addr != NULL) {
6582 asdl->sdl_alen = ll_addr->sdl_alen;
6583 bcopy(CONST_LLADDR(ll_addr), LLADDR(asdl), asdl->sdl_alen);
6584 } else {
6585 asdl->sdl_alen = 0;
6586 }
6587 ifa->ifa_netmask = (struct sockaddr *)msdl;
6588 msdl->sdl_len = masklen;
6589 while (namelen > 0)
6590 msdl->sdl_data[--namelen] = 0xff;
6591 IFA_UNLOCK(ifa);
6592
6593 if (oifa != NULL)
6594 IFA_REMREF(oifa);
6595
6596 return (ifa);
6597 }
6598
6599 static void
6600 if_purgeaddrs(struct ifnet *ifp)
6601 {
6602 #if INET
6603 in_purgeaddrs(ifp);
6604 #endif /* INET */
6605 #if INET6
6606 in6_purgeaddrs(ifp);
6607 #endif /* INET6 */
6608 }
6609
6610 errno_t
6611 ifnet_detach(ifnet_t ifp)
6612 {
6613 struct ifnet *delegated_ifp;
6614 struct nd_ifinfo *ndi = NULL;
6615
6616 if (ifp == NULL)
6617 return (EINVAL);
6618
6619 ndi = ND_IFINFO(ifp);
6620 if (NULL != ndi)
6621 ndi->cga_initialized = FALSE;
6622
6623 lck_mtx_lock(rnh_lock);
6624 ifnet_head_lock_exclusive();
6625 ifnet_lock_exclusive(ifp);
6626
6627 /*
6628 * Check to see if this interface has previously triggered
6629 * aggressive protocol draining; if so, decrement the global
6630 * refcnt and clear PR_AGGDRAIN on the route domain if
6631 * there are no more of such an interface around.
6632 */
6633 (void) ifnet_set_idle_flags_locked(ifp, 0, ~0);
6634
6635 lck_mtx_lock_spin(&ifp->if_ref_lock);
6636 if (!(ifp->if_refflags & IFRF_ATTACHED)) {
6637 lck_mtx_unlock(&ifp->if_ref_lock);
6638 ifnet_lock_done(ifp);
6639 ifnet_head_done();
6640 lck_mtx_unlock(rnh_lock);
6641 return (EINVAL);
6642 } else if (ifp->if_refflags & IFRF_DETACHING) {
6643 /* Interface has already been detached */
6644 lck_mtx_unlock(&ifp->if_ref_lock);
6645 ifnet_lock_done(ifp);
6646 ifnet_head_done();
6647 lck_mtx_unlock(rnh_lock);
6648 return (ENXIO);
6649 }
6650 VERIFY(!(ifp->if_refflags & IFRF_EMBRYONIC));
6651 /* Indicate this interface is being detached */
6652 ifp->if_refflags &= ~IFRF_ATTACHED;
6653 ifp->if_refflags |= IFRF_DETACHING;
6654 lck_mtx_unlock(&ifp->if_ref_lock);
6655
6656 if (dlil_verbose) {
6657 printf("%s: detaching\n", if_name(ifp));
6658 }
6659
6660 /* clean up flow control entry object if there's any */
6661 if (ifp->if_eflags & IFEF_TXSTART) {
6662 ifnet_flowadv(ifp->if_flowhash);
6663 }
6664
6665 /* Reset ECN enable/disable flags */
6666 ifp->if_eflags &= ~IFEF_ECN_DISABLE;
6667 ifp->if_eflags &= ~IFEF_ECN_ENABLE;
6668
6669 /* Reset CLAT46 flag */
6670 ifp->if_eflags &= ~IFEF_CLAT46;
6671
6672 /*
6673 * Remove ifnet from the ifnet_head, ifindex2ifnet[]; it will
6674 * no longer be visible during lookups from this point.
6675 */
6676 VERIFY(ifindex2ifnet[ifp->if_index] == ifp);
6677 TAILQ_REMOVE(&ifnet_head, ifp, if_link);
6678 ifp->if_link.tqe_next = NULL;
6679 ifp->if_link.tqe_prev = NULL;
6680 if (ifp->if_ordered_link.tqe_next != NULL ||
6681 ifp->if_ordered_link.tqe_prev != NULL) {
6682 ifnet_remove_from_ordered_list(ifp);
6683 }
6684 ifindex2ifnet[ifp->if_index] = NULL;
6685
6686 /* 18717626 - reset IFEF_IPV4_ROUTER and IFEF_IPV6_ROUTER */
6687 ifp->if_eflags &= ~(IFEF_IPV4_ROUTER | IFEF_IPV6_ROUTER);
6688
6689 /* Record detach PC stacktrace */
6690 ctrace_record(&((struct dlil_ifnet *)ifp)->dl_if_detach);
6691
6692 /* Clear logging parameters */
6693 bzero(&ifp->if_log, sizeof (ifp->if_log));
6694
6695 /* Clear delegated interface info (reference released below) */
6696 delegated_ifp = ifp->if_delegated.ifp;
6697 bzero(&ifp->if_delegated, sizeof (ifp->if_delegated));
6698
6699 /* Reset interface state */
6700 bzero(&ifp->if_interface_state, sizeof(ifp->if_interface_state));
6701
6702 ifnet_lock_done(ifp);
6703 ifnet_head_done();
6704 lck_mtx_unlock(rnh_lock);
6705
6706
6707 /* Release reference held on the delegated interface */
6708 if (delegated_ifp != NULL)
6709 ifnet_release(delegated_ifp);
6710
6711 /* Reset Link Quality Metric (unless loopback [lo0]) */
6712 if (ifp != lo_ifp)
6713 if_lqm_update(ifp, IFNET_LQM_THRESH_OFF, 0);
6714
6715 /* Reset TCP local statistics */
6716 if (ifp->if_tcp_stat != NULL)
6717 bzero(ifp->if_tcp_stat, sizeof(*ifp->if_tcp_stat));
6718
6719 /* Reset UDP local statistics */
6720 if (ifp->if_udp_stat != NULL)
6721 bzero(ifp->if_udp_stat, sizeof(*ifp->if_udp_stat));
6722
6723 /* Reset ifnet IPv4 stats */
6724 if (ifp->if_ipv4_stat != NULL)
6725 bzero(ifp->if_ipv4_stat, sizeof(*ifp->if_ipv4_stat));
6726
6727 /* Reset ifnet IPv6 stats */
6728 if (ifp->if_ipv6_stat != NULL)
6729 bzero(ifp->if_ipv6_stat, sizeof(*ifp->if_ipv6_stat));
6730
6731 /* Release memory held for interface link status report */
6732 if (ifp->if_link_status != NULL) {
6733 FREE(ifp->if_link_status, M_TEMP);
6734 ifp->if_link_status = NULL;
6735 }
6736
6737 /* Clear agent IDs */
6738 if (ifp->if_agentids != NULL) {
6739 FREE(ifp->if_agentids, M_NETAGENT);
6740 ifp->if_agentids = NULL;
6741 }
6742 ifp->if_agentcount = 0;
6743
6744
6745 /* Let BPF know we're detaching */
6746 bpfdetach(ifp);
6747
6748 /* Mark the interface as DOWN */
6749 if_down(ifp);
6750
6751 /* Disable forwarding cached route */
6752 lck_mtx_lock(&ifp->if_cached_route_lock);
6753 ifp->if_fwd_cacheok = 0;
6754 lck_mtx_unlock(&ifp->if_cached_route_lock);
6755
6756 /* Disable data threshold and wait for any pending event posting */
6757 ifp->if_data_threshold = 0;
6758 VERIFY(ifp->if_dt_tcall != NULL);
6759 (void) thread_call_cancel_wait(ifp->if_dt_tcall);
6760
6761 /*
6762 * Drain any deferred IGMPv3/MLDv2 query responses, but keep the
6763 * references to the info structures and leave them attached to
6764 * this ifnet.
6765 */
6766 #if INET
6767 igmp_domifdetach(ifp);
6768 #endif /* INET */
6769 #if INET6
6770 mld_domifdetach(ifp);
6771 #endif /* INET6 */
6772
6773 dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_IF_DETACHING, NULL, 0);
6774
6775 /* Let worker thread take care of the rest, to avoid reentrancy */
6776 dlil_if_lock();
6777 ifnet_detaching_enqueue(ifp);
6778 dlil_if_unlock();
6779
6780 return (0);
6781 }
6782
6783 static void
6784 ifnet_detaching_enqueue(struct ifnet *ifp)
6785 {
6786 dlil_if_lock_assert();
6787
6788 ++ifnet_detaching_cnt;
6789 VERIFY(ifnet_detaching_cnt != 0);
6790 TAILQ_INSERT_TAIL(&ifnet_detaching_head, ifp, if_detaching_link);
6791 wakeup((caddr_t)&ifnet_delayed_run);
6792 }
6793
6794 static struct ifnet *
6795 ifnet_detaching_dequeue(void)
6796 {
6797 struct ifnet *ifp;
6798
6799 dlil_if_lock_assert();
6800
6801 ifp = TAILQ_FIRST(&ifnet_detaching_head);
6802 VERIFY(ifnet_detaching_cnt != 0 || ifp == NULL);
6803 if (ifp != NULL) {
6804 VERIFY(ifnet_detaching_cnt != 0);
6805 --ifnet_detaching_cnt;
6806 TAILQ_REMOVE(&ifnet_detaching_head, ifp, if_detaching_link);
6807 ifp->if_detaching_link.tqe_next = NULL;
6808 ifp->if_detaching_link.tqe_prev = NULL;
6809 }
6810 return (ifp);
6811 }
6812
6813 static int
6814 ifnet_detacher_thread_cont(int err)
6815 {
6816 #pragma unused(err)
6817 struct ifnet *ifp;
6818
6819 for (;;) {
6820 dlil_if_lock_assert();
6821 while (ifnet_detaching_cnt == 0) {
6822 (void) msleep0(&ifnet_delayed_run, &dlil_ifnet_lock,
6823 (PZERO - 1), "ifnet_detacher_cont", 0,
6824 ifnet_detacher_thread_cont);
6825 /* NOTREACHED */
6826 }
6827
6828 VERIFY(TAILQ_FIRST(&ifnet_detaching_head) != NULL);
6829
6830 /* Take care of detaching ifnet */
6831 ifp = ifnet_detaching_dequeue();
6832 if (ifp != NULL) {
6833 dlil_if_unlock();
6834 ifnet_detach_final(ifp);
6835 dlil_if_lock();
6836 }
6837 }
6838 }
6839
6840 static void
6841 ifnet_detacher_thread_func(void *v, wait_result_t w)
6842 {
6843 #pragma unused(v, w)
6844 dlil_if_lock();
6845 (void) msleep0(&ifnet_delayed_run, &dlil_ifnet_lock,
6846 (PZERO - 1), "ifnet_detacher", 0, ifnet_detacher_thread_cont);
6847 /*
6848 * msleep0() shouldn't have returned as PCATCH was not set;
6849 * therefore assert in this case.
6850 */
6851 dlil_if_unlock();
6852 VERIFY(0);
6853 }
6854
6855 static void
6856 ifnet_detach_final(struct ifnet *ifp)
6857 {
6858 struct ifnet_filter *filter, *filter_next;
6859 struct ifnet_filter_head fhead;
6860 struct dlil_threading_info *inp;
6861 struct ifaddr *ifa;
6862 ifnet_detached_func if_free;
6863 int i;
6864
6865 lck_mtx_lock(&ifp->if_ref_lock);
6866 if (!(ifp->if_refflags & IFRF_DETACHING)) {
6867 panic("%s: flags mismatch (detaching not set) ifp=%p",
6868 __func__, ifp);
6869 /* NOTREACHED */
6870 }
6871
6872 /*
6873 * Wait until the existing IO references get released
6874 * before we proceed with ifnet_detach. This is not a
6875 * common case, so block without using a continuation.
6876 */
6877 while (ifp->if_refio > 0) {
6878 printf("%s: Waiting for IO references on %s interface "
6879 "to be released\n", __func__, if_name(ifp));
6880 (void) msleep(&(ifp->if_refio), &ifp->if_ref_lock,
6881 (PZERO - 1), "ifnet_ioref_wait", NULL);
6882 }
6883 lck_mtx_unlock(&ifp->if_ref_lock);
6884
6885 /* Drain and destroy send queue */
6886 ifclassq_teardown(ifp);
6887
6888 /* Detach interface filters */
6889 lck_mtx_lock(&ifp->if_flt_lock);
6890 if_flt_monitor_enter(ifp);
6891
6892 LCK_MTX_ASSERT(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED);
6893 fhead = ifp->if_flt_head;
6894 TAILQ_INIT(&ifp->if_flt_head);
6895
6896 for (filter = TAILQ_FIRST(&fhead); filter; filter = filter_next) {
6897 filter_next = TAILQ_NEXT(filter, filt_next);
6898 lck_mtx_unlock(&ifp->if_flt_lock);
6899
6900 dlil_detach_filter_internal(filter, 1);
6901 lck_mtx_lock(&ifp->if_flt_lock);
6902 }
6903 if_flt_monitor_leave(ifp);
6904 lck_mtx_unlock(&ifp->if_flt_lock);
6905
6906 /* Tell upper layers to drop their network addresses */
6907 if_purgeaddrs(ifp);
6908
6909 ifnet_lock_exclusive(ifp);
6910
6911 /* Uplumb all protocols */
6912 for (i = 0; i < PROTO_HASH_SLOTS; i++) {
6913 struct if_proto *proto;
6914
6915 proto = SLIST_FIRST(&ifp->if_proto_hash[i]);
6916 while (proto != NULL) {
6917 protocol_family_t family = proto->protocol_family;
6918 ifnet_lock_done(ifp);
6919 proto_unplumb(family, ifp);
6920 ifnet_lock_exclusive(ifp);
6921 proto = SLIST_FIRST(&ifp->if_proto_hash[i]);
6922 }
6923 /* There should not be any protocols left */
6924 VERIFY(SLIST_EMPTY(&ifp->if_proto_hash[i]));
6925 }
6926 zfree(dlif_phash_zone, ifp->if_proto_hash);
6927 ifp->if_proto_hash = NULL;
6928
6929 /* Detach (permanent) link address from if_addrhead */
6930 ifa = TAILQ_FIRST(&ifp->if_addrhead);
6931 VERIFY(ifnet_addrs[ifp->if_index - 1] == ifa);
6932 IFA_LOCK(ifa);
6933 if_detach_link_ifa(ifp, ifa);
6934 IFA_UNLOCK(ifa);
6935
6936 /* Remove (permanent) link address from ifnet_addrs[] */
6937 IFA_REMREF(ifa);
6938 ifnet_addrs[ifp->if_index - 1] = NULL;
6939
6940 /* This interface should not be on {ifnet_head,detaching} */
6941 VERIFY(ifp->if_link.tqe_next == NULL);
6942 VERIFY(ifp->if_link.tqe_prev == NULL);
6943 VERIFY(ifp->if_detaching_link.tqe_next == NULL);
6944 VERIFY(ifp->if_detaching_link.tqe_prev == NULL);
6945 VERIFY(ifp->if_ordered_link.tqe_next == NULL);
6946 VERIFY(ifp->if_ordered_link.tqe_prev == NULL);
6947
6948 /* The slot should have been emptied */
6949 VERIFY(ifindex2ifnet[ifp->if_index] == NULL);
6950
6951 /* There should not be any addresses left */
6952 VERIFY(TAILQ_EMPTY(&ifp->if_addrhead));
6953
6954 /*
6955 * Signal the starter thread to terminate itself.
6956 */
6957 if (ifp->if_start_thread != THREAD_NULL) {
6958 lck_mtx_lock_spin(&ifp->if_start_lock);
6959 ifp->if_start_flags = 0;
6960 ifp->if_start_thread = THREAD_NULL;
6961 wakeup_one((caddr_t)&ifp->if_start_thread);
6962 lck_mtx_unlock(&ifp->if_start_lock);
6963 }
6964
6965 /*
6966 * Signal the poller thread to terminate itself.
6967 */
6968 if (ifp->if_poll_thread != THREAD_NULL) {
6969 lck_mtx_lock_spin(&ifp->if_poll_lock);
6970 ifp->if_poll_thread = THREAD_NULL;
6971 wakeup_one((caddr_t)&ifp->if_poll_thread);
6972 lck_mtx_unlock(&ifp->if_poll_lock);
6973 }
6974
6975 /*
6976 * If thread affinity was set for the workloop thread, we will need
6977 * to tear down the affinity and release the extra reference count
6978 * taken at attach time. Does not apply to lo0 or other interfaces
6979 * without dedicated input threads.
6980 */
6981 if ((inp = ifp->if_inp) != NULL) {
6982 VERIFY(inp != dlil_main_input_thread);
6983
6984 if (inp->net_affinity) {
6985 struct thread *tp, *wtp, *ptp;
6986
6987 lck_mtx_lock_spin(&inp->input_lck);
6988 wtp = inp->wloop_thr;
6989 inp->wloop_thr = THREAD_NULL;
6990 ptp = inp->poll_thr;
6991 inp->poll_thr = THREAD_NULL;
6992 tp = inp->input_thr; /* don't nullify now */
6993 inp->tag = 0;
6994 inp->net_affinity = FALSE;
6995 lck_mtx_unlock(&inp->input_lck);
6996
6997 /* Tear down poll thread affinity */
6998 if (ptp != NULL) {
6999 VERIFY(ifp->if_eflags & IFEF_RXPOLL);
7000 (void) dlil_affinity_set(ptp,
7001 THREAD_AFFINITY_TAG_NULL);
7002 thread_deallocate(ptp);
7003 }
7004
7005 /* Tear down workloop thread affinity */
7006 if (wtp != NULL) {
7007 (void) dlil_affinity_set(wtp,
7008 THREAD_AFFINITY_TAG_NULL);
7009 thread_deallocate(wtp);
7010 }
7011
7012 /* Tear down DLIL input thread affinity */
7013 (void) dlil_affinity_set(tp, THREAD_AFFINITY_TAG_NULL);
7014 thread_deallocate(tp);
7015 }
7016
7017 /* disassociate ifp DLIL input thread */
7018 ifp->if_inp = NULL;
7019
7020 /* tell the input thread to terminate */
7021 lck_mtx_lock_spin(&inp->input_lck);
7022 inp->input_waiting |= DLIL_INPUT_TERMINATE;
7023 if (!(inp->input_waiting & DLIL_INPUT_RUNNING)) {
7024 wakeup_one((caddr_t)&inp->input_waiting);
7025 }
7026 lck_mtx_unlock(&inp->input_lck);
7027 ifnet_lock_done(ifp);
7028
7029 /* wait for the input thread to terminate */
7030 lck_mtx_lock_spin(&inp->input_lck);
7031 while ((inp->input_waiting & DLIL_INPUT_TERMINATE_COMPLETE)
7032 == 0) {
7033 (void) msleep(&inp->input_waiting, &inp->input_lck,
7034 (PZERO - 1) | PSPIN, inp->input_name, NULL);
7035 }
7036 lck_mtx_unlock(&inp->input_lck);
7037 ifnet_lock_exclusive(ifp);
7038
7039 /* clean-up input thread state */
7040 dlil_clean_threading_info(inp);
7041
7042 }
7043
7044 /* The driver might unload, so point these to ourselves */
7045 if_free = ifp->if_free;
7046 ifp->if_output_dlil = ifp_if_output;
7047 ifp->if_output = ifp_if_output;
7048 ifp->if_pre_enqueue = ifp_if_output;
7049 ifp->if_start = ifp_if_start;
7050 ifp->if_output_ctl = ifp_if_ctl;
7051 ifp->if_input_dlil = ifp_if_input;
7052 ifp->if_input_poll = ifp_if_input_poll;
7053 ifp->if_input_ctl = ifp_if_ctl;
7054 ifp->if_ioctl = ifp_if_ioctl;
7055 ifp->if_set_bpf_tap = ifp_if_set_bpf_tap;
7056 ifp->if_free = ifp_if_free;
7057 ifp->if_demux = ifp_if_demux;
7058 ifp->if_event = ifp_if_event;
7059 ifp->if_framer_legacy = ifp_if_framer;
7060 ifp->if_framer = ifp_if_framer_extended;
7061 ifp->if_add_proto = ifp_if_add_proto;
7062 ifp->if_del_proto = ifp_if_del_proto;
7063 ifp->if_check_multi = ifp_if_check_multi;
7064
7065 /* wipe out interface description */
7066 VERIFY(ifp->if_desc.ifd_maxlen == IF_DESCSIZE);
7067 ifp->if_desc.ifd_len = 0;
7068 VERIFY(ifp->if_desc.ifd_desc != NULL);
7069 bzero(ifp->if_desc.ifd_desc, IF_DESCSIZE);
7070
7071 /* there shouldn't be any delegation by now */
7072 VERIFY(ifp->if_delegated.ifp == NULL);
7073 VERIFY(ifp->if_delegated.type == 0);
7074 VERIFY(ifp->if_delegated.family == 0);
7075 VERIFY(ifp->if_delegated.subfamily == 0);
7076 VERIFY(ifp->if_delegated.expensive == 0);
7077
7078 /* QoS marking get cleared */
7079 ifp->if_eflags &= ~IFEF_QOSMARKING_ENABLED;
7080 if_set_qosmarking_mode(ifp, IFRTYPE_QOSMARKING_MODE_NONE);
7081
7082
7083 ifnet_lock_done(ifp);
7084
7085 #if PF
7086 /*
7087 * Detach this interface from packet filter, if enabled.
7088 */
7089 pf_ifnet_hook(ifp, 0);
7090 #endif /* PF */
7091
7092 /* Filter list should be empty */
7093 lck_mtx_lock_spin(&ifp->if_flt_lock);
7094 VERIFY(TAILQ_EMPTY(&ifp->if_flt_head));
7095 VERIFY(ifp->if_flt_busy == 0);
7096 VERIFY(ifp->if_flt_waiters == 0);
7097 lck_mtx_unlock(&ifp->if_flt_lock);
7098
7099 /* Last chance to drain send queue */
7100 if_qflush(ifp, 0);
7101
7102 /* Last chance to cleanup any cached route */
7103 lck_mtx_lock(&ifp->if_cached_route_lock);
7104 VERIFY(!ifp->if_fwd_cacheok);
7105 ROUTE_RELEASE(&ifp->if_fwd_route);
7106 bzero(&ifp->if_fwd_route, sizeof (ifp->if_fwd_route));
7107 ROUTE_RELEASE(&ifp->if_src_route);
7108 bzero(&ifp->if_src_route, sizeof (ifp->if_src_route));
7109 ROUTE_RELEASE(&ifp->if_src_route6);
7110 bzero(&ifp->if_src_route6, sizeof (ifp->if_src_route6));
7111 lck_mtx_unlock(&ifp->if_cached_route_lock);
7112
7113 VERIFY(ifp->if_data_threshold == 0);
7114 VERIFY(ifp->if_dt_tcall != NULL);
7115 VERIFY(!thread_call_isactive(ifp->if_dt_tcall));
7116
7117 ifnet_llreach_ifdetach(ifp);
7118
7119 dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_IF_DETACHED, NULL, 0);
7120
7121 /*
7122 * Finally, mark this ifnet as detached.
7123 */
7124 lck_mtx_lock_spin(&ifp->if_ref_lock);
7125 if (!(ifp->if_refflags & IFRF_DETACHING)) {
7126 panic("%s: flags mismatch (detaching not set) ifp=%p",
7127 __func__, ifp);
7128 /* NOTREACHED */
7129 }
7130 ifp->if_refflags &= ~IFRF_DETACHING;
7131 lck_mtx_unlock(&ifp->if_ref_lock);
7132 if (if_free != NULL)
7133 if_free(ifp);
7134
7135 if (dlil_verbose)
7136 printf("%s: detached\n", if_name(ifp));
7137
7138 /* Release reference held during ifnet attach */
7139 ifnet_release(ifp);
7140 }
7141
7142 errno_t
7143 ifp_if_output(struct ifnet *ifp, struct mbuf *m)
7144 {
7145 #pragma unused(ifp)
7146 m_freem_list(m);
7147 return (0);
7148 }
7149
7150 void
7151 ifp_if_start(struct ifnet *ifp)
7152 {
7153 ifnet_purge(ifp);
7154 }
7155
7156 static errno_t
7157 ifp_if_input(struct ifnet *ifp, struct mbuf *m_head,
7158 struct mbuf *m_tail, const struct ifnet_stat_increment_param *s,
7159 boolean_t poll, struct thread *tp)
7160 {
7161 #pragma unused(ifp, m_tail, s, poll, tp)
7162 m_freem_list(m_head);
7163 return (ENXIO);
7164 }
7165
7166 static void
7167 ifp_if_input_poll(struct ifnet *ifp, u_int32_t flags, u_int32_t max_cnt,
7168 struct mbuf **m_head, struct mbuf **m_tail, u_int32_t *cnt, u_int32_t *len)
7169 {
7170 #pragma unused(ifp, flags, max_cnt)
7171 if (m_head != NULL)
7172 *m_head = NULL;
7173 if (m_tail != NULL)
7174 *m_tail = NULL;
7175 if (cnt != NULL)
7176 *cnt = 0;
7177 if (len != NULL)
7178 *len = 0;
7179 }
7180
7181 static errno_t
7182 ifp_if_ctl(struct ifnet *ifp, ifnet_ctl_cmd_t cmd, u_int32_t arglen, void *arg)
7183 {
7184 #pragma unused(ifp, cmd, arglen, arg)
7185 return (EOPNOTSUPP);
7186 }
7187
7188 static errno_t
7189 ifp_if_demux(struct ifnet *ifp, struct mbuf *m, char *fh, protocol_family_t *pf)
7190 {
7191 #pragma unused(ifp, fh, pf)
7192 m_freem(m);
7193 return (EJUSTRETURN);
7194 }
7195
7196 static errno_t
7197 ifp_if_add_proto(struct ifnet *ifp, protocol_family_t pf,
7198 const struct ifnet_demux_desc *da, u_int32_t dc)
7199 {
7200 #pragma unused(ifp, pf, da, dc)
7201 return (EINVAL);
7202 }
7203
7204 static errno_t
7205 ifp_if_del_proto(struct ifnet *ifp, protocol_family_t pf)
7206 {
7207 #pragma unused(ifp, pf)
7208 return (EINVAL);
7209 }
7210
7211 static errno_t
7212 ifp_if_check_multi(struct ifnet *ifp, const struct sockaddr *sa)
7213 {
7214 #pragma unused(ifp, sa)
7215 return (EOPNOTSUPP);
7216 }
7217
7218 #if CONFIG_EMBEDDED
7219 static errno_t
7220 ifp_if_framer(struct ifnet *ifp, struct mbuf **m,
7221 const struct sockaddr *sa, const char *ll, const char *t,
7222 u_int32_t *pre, u_int32_t *post)
7223 #else
7224 static errno_t
7225 ifp_if_framer(struct ifnet *ifp, struct mbuf **m,
7226 const struct sockaddr *sa, const char *ll, const char *t)
7227 #endif /* !CONFIG_EMBEDDED */
7228 {
7229 #pragma unused(ifp, m, sa, ll, t)
7230 #if CONFIG_EMBEDDED
7231 return (ifp_if_framer_extended(ifp, m, sa, ll, t, pre, post));
7232 #else
7233 return (ifp_if_framer_extended(ifp, m, sa, ll, t, NULL, NULL));
7234 #endif /* !CONFIG_EMBEDDED */
7235 }
7236
7237 static errno_t
7238 ifp_if_framer_extended(struct ifnet *ifp, struct mbuf **m,
7239 const struct sockaddr *sa, const char *ll, const char *t,
7240 u_int32_t *pre, u_int32_t *post)
7241 {
7242 #pragma unused(ifp, sa, ll, t)
7243 m_freem(*m);
7244 *m = NULL;
7245
7246 if (pre != NULL)
7247 *pre = 0;
7248 if (post != NULL)
7249 *post = 0;
7250
7251 return (EJUSTRETURN);
7252 }
7253
7254 errno_t
7255 ifp_if_ioctl(struct ifnet *ifp, unsigned long cmd, void *arg)
7256 {
7257 #pragma unused(ifp, cmd, arg)
7258 return (EOPNOTSUPP);
7259 }
7260
7261 static errno_t
7262 ifp_if_set_bpf_tap(struct ifnet *ifp, bpf_tap_mode tm, bpf_packet_func f)
7263 {
7264 #pragma unused(ifp, tm, f)
7265 /* XXX not sure what to do here */
7266 return (0);
7267 }
7268
7269 static void
7270 ifp_if_free(struct ifnet *ifp)
7271 {
7272 #pragma unused(ifp)
7273 }
7274
7275 static void
7276 ifp_if_event(struct ifnet *ifp, const struct kev_msg *e)
7277 {
7278 #pragma unused(ifp, e)
7279 }
7280
7281 int dlil_if_acquire(u_int32_t family, const void *uniqueid,
7282 size_t uniqueid_len, const char *ifxname, struct ifnet **ifp)
7283 {
7284 struct ifnet *ifp1 = NULL;
7285 struct dlil_ifnet *dlifp1 = NULL;
7286 void *buf, *base, **pbuf;
7287 int ret = 0;
7288
7289 VERIFY(*ifp == NULL);
7290 dlil_if_lock();
7291 /*
7292 * We absolutely can't have an interface with the same name
7293 * in in-use state.
7294 * To make sure of that list has to be traversed completely
7295 */
7296 TAILQ_FOREACH(dlifp1, &dlil_ifnet_head, dl_if_link) {
7297 ifp1 = (struct ifnet *)dlifp1;
7298
7299 if (ifp1->if_family != family)
7300 continue;
7301
7302 /*
7303 * If interface is in use, return EBUSY if either unique id
7304 * or interface extended names are the same
7305 */
7306 lck_mtx_lock(&dlifp1->dl_if_lock);
7307 if (strncmp(ifxname, ifp1->if_xname, IFXNAMSIZ) == 0) {
7308 if (dlifp1->dl_if_flags & DLIF_INUSE) {
7309 lck_mtx_unlock(&dlifp1->dl_if_lock);
7310 ret = EBUSY;
7311 goto end;
7312 }
7313 }
7314
7315 if (uniqueid_len) {
7316 if (uniqueid_len == dlifp1->dl_if_uniqueid_len &&
7317 bcmp(uniqueid, dlifp1->dl_if_uniqueid, uniqueid_len) == 0) {
7318 if (dlifp1->dl_if_flags & DLIF_INUSE) {
7319 lck_mtx_unlock(&dlifp1->dl_if_lock);
7320 ret = EBUSY;
7321 goto end;
7322 } else {
7323 dlifp1->dl_if_flags |= (DLIF_INUSE|DLIF_REUSE);
7324 /* Cache the first interface that can be recycled */
7325 if (*ifp == NULL)
7326 *ifp = ifp1;
7327 /*
7328 * XXX Do not break or jump to end as we have to traverse
7329 * the whole list to ensure there are no name collisions
7330 */
7331 }
7332 }
7333 }
7334 lck_mtx_unlock(&dlifp1->dl_if_lock);
7335 }
7336
7337 /* If there's an interface that can be recycled, use that */
7338 if (*ifp != NULL)
7339 goto end;
7340
7341 /* no interface found, allocate a new one */
7342 buf = zalloc(dlif_zone);
7343 if (buf == NULL) {
7344 ret = ENOMEM;
7345 goto end;
7346 }
7347 bzero(buf, dlif_bufsize);
7348
7349 /* Get the 64-bit aligned base address for this object */
7350 base = (void *)P2ROUNDUP((intptr_t)buf + sizeof (u_int64_t),
7351 sizeof (u_int64_t));
7352 VERIFY(((intptr_t)base + dlif_size) <= ((intptr_t)buf + dlif_bufsize));
7353
7354 /*
7355 * Wind back a pointer size from the aligned base and
7356 * save the original address so we can free it later.
7357 */
7358 pbuf = (void **)((intptr_t)base - sizeof (void *));
7359 *pbuf = buf;
7360 dlifp1 = base;
7361
7362 if (uniqueid_len) {
7363 MALLOC(dlifp1->dl_if_uniqueid, void *, uniqueid_len,
7364 M_NKE, M_WAITOK);
7365 if (dlifp1->dl_if_uniqueid == NULL) {
7366 zfree(dlif_zone, buf);
7367 ret = ENOMEM;
7368 goto end;
7369 }
7370 bcopy(uniqueid, dlifp1->dl_if_uniqueid, uniqueid_len);
7371 dlifp1->dl_if_uniqueid_len = uniqueid_len;
7372 }
7373
7374 ifp1 = (struct ifnet *)dlifp1;
7375 dlifp1->dl_if_flags = DLIF_INUSE;
7376 if (ifnet_debug) {
7377 dlifp1->dl_if_flags |= DLIF_DEBUG;
7378 dlifp1->dl_if_trace = dlil_if_trace;
7379 }
7380 ifp1->if_name = dlifp1->dl_if_namestorage;
7381 ifp1->if_xname = dlifp1->dl_if_xnamestorage;
7382
7383 /* initialize interface description */
7384 ifp1->if_desc.ifd_maxlen = IF_DESCSIZE;
7385 ifp1->if_desc.ifd_len = 0;
7386 ifp1->if_desc.ifd_desc = dlifp1->dl_if_descstorage;
7387
7388
7389 #if CONFIG_MACF_NET
7390 mac_ifnet_label_init(ifp1);
7391 #endif
7392
7393 if ((ret = dlil_alloc_local_stats(ifp1)) != 0) {
7394 DLIL_PRINTF("%s: failed to allocate if local stats, "
7395 "error: %d\n", __func__, ret);
7396 /* This probably shouldn't be fatal */
7397 ret = 0;
7398 }
7399
7400 lck_mtx_init(&dlifp1->dl_if_lock, ifnet_lock_group, ifnet_lock_attr);
7401 lck_rw_init(&ifp1->if_lock, ifnet_lock_group, ifnet_lock_attr);
7402 lck_mtx_init(&ifp1->if_ref_lock, ifnet_lock_group, ifnet_lock_attr);
7403 lck_mtx_init(&ifp1->if_flt_lock, ifnet_lock_group, ifnet_lock_attr);
7404 lck_mtx_init(&ifp1->if_addrconfig_lock, ifnet_lock_group,
7405 ifnet_lock_attr);
7406 lck_rw_init(&ifp1->if_llreach_lock, ifnet_lock_group, ifnet_lock_attr);
7407 #if INET
7408 lck_rw_init(&ifp1->if_inetdata_lock, ifnet_lock_group,
7409 ifnet_lock_attr);
7410 ifp1->if_inetdata = NULL;
7411 #endif
7412 #if INET6
7413 lck_rw_init(&ifp1->if_inet6data_lock, ifnet_lock_group,
7414 ifnet_lock_attr);
7415 ifp1->if_inet6data = NULL;
7416 #endif
7417 lck_rw_init(&ifp1->if_link_status_lock, ifnet_lock_group,
7418 ifnet_lock_attr);
7419 ifp1->if_link_status = NULL;
7420
7421 /* for send data paths */
7422 lck_mtx_init(&ifp1->if_start_lock, ifnet_snd_lock_group,
7423 ifnet_lock_attr);
7424 lck_mtx_init(&ifp1->if_cached_route_lock, ifnet_snd_lock_group,
7425 ifnet_lock_attr);
7426 lck_mtx_init(&ifp1->if_snd.ifcq_lock, ifnet_snd_lock_group,
7427 ifnet_lock_attr);
7428
7429 /* for receive data paths */
7430 lck_mtx_init(&ifp1->if_poll_lock, ifnet_rcv_lock_group,
7431 ifnet_lock_attr);
7432
7433 /* thread call allocation is done with sleeping zalloc */
7434 ifp1->if_dt_tcall = thread_call_allocate_with_options(dlil_dt_tcall_fn,
7435 ifp1, THREAD_CALL_PRIORITY_KERNEL, THREAD_CALL_OPTIONS_ONCE);
7436 if (ifp1->if_dt_tcall == NULL) {
7437 panic_plain("%s: couldn't create if_dt_tcall", __func__);
7438 /* NOTREACHED */
7439 }
7440
7441 TAILQ_INSERT_TAIL(&dlil_ifnet_head, dlifp1, dl_if_link);
7442
7443 *ifp = ifp1;
7444
7445 end:
7446 dlil_if_unlock();
7447
7448 VERIFY(dlifp1 == NULL || (IS_P2ALIGNED(dlifp1, sizeof (u_int64_t)) &&
7449 IS_P2ALIGNED(&ifp1->if_data, sizeof (u_int64_t))));
7450
7451 return (ret);
7452 }
7453
7454 __private_extern__ void
7455 dlil_if_release(ifnet_t ifp)
7456 {
7457 struct dlil_ifnet *dlifp = (struct dlil_ifnet *)ifp;
7458
7459 VERIFY(OSDecrementAtomic64(&net_api_stats.nas_ifnet_alloc_count) > 0);
7460 if (!(ifp->if_xflags & IFXF_ALLOC_KPI)) {
7461 VERIFY(OSDecrementAtomic64(&net_api_stats.nas_ifnet_alloc_os_count) > 0);
7462 }
7463
7464 ifnet_lock_exclusive(ifp);
7465 lck_mtx_lock(&dlifp->dl_if_lock);
7466 dlifp->dl_if_flags &= ~DLIF_INUSE;
7467 strlcpy(dlifp->dl_if_namestorage, ifp->if_name, IFNAMSIZ);
7468 ifp->if_name = dlifp->dl_if_namestorage;
7469 /* Reset external name (name + unit) */
7470 ifp->if_xname = dlifp->dl_if_xnamestorage;
7471 snprintf(__DECONST(char *, ifp->if_xname), IFXNAMSIZ,
7472 "%s?", ifp->if_name);
7473 lck_mtx_unlock(&dlifp->dl_if_lock);
7474 #if CONFIG_MACF_NET
7475 /*
7476 * We can either recycle the MAC label here or in dlil_if_acquire().
7477 * It seems logical to do it here but this means that anything that
7478 * still has a handle on ifp will now see it as unlabeled.
7479 * Since the interface is "dead" that may be OK. Revisit later.
7480 */
7481 mac_ifnet_label_recycle(ifp);
7482 #endif
7483 ifnet_lock_done(ifp);
7484 }
7485
7486 __private_extern__ void
7487 dlil_if_lock(void)
7488 {
7489 lck_mtx_lock(&dlil_ifnet_lock);
7490 }
7491
7492 __private_extern__ void
7493 dlil_if_unlock(void)
7494 {
7495 lck_mtx_unlock(&dlil_ifnet_lock);
7496 }
7497
7498 __private_extern__ void
7499 dlil_if_lock_assert(void)
7500 {
7501 LCK_MTX_ASSERT(&dlil_ifnet_lock, LCK_MTX_ASSERT_OWNED);
7502 }
7503
7504 __private_extern__ void
7505 dlil_proto_unplumb_all(struct ifnet *ifp)
7506 {
7507 /*
7508 * if_proto_hash[0-2] are for PF_INET, PF_INET6 and PF_VLAN, where
7509 * each bucket contains exactly one entry; PF_VLAN does not need an
7510 * explicit unplumb.
7511 *
7512 * if_proto_hash[3] is for other protocols; we expect anything
7513 * in this bucket to respond to the DETACHING event (which would
7514 * have happened by now) and do the unplumb then.
7515 */
7516 (void) proto_unplumb(PF_INET, ifp);
7517 #if INET6
7518 (void) proto_unplumb(PF_INET6, ifp);
7519 #endif /* INET6 */
7520 }
7521
7522 static void
7523 ifp_src_route_copyout(struct ifnet *ifp, struct route *dst)
7524 {
7525 lck_mtx_lock_spin(&ifp->if_cached_route_lock);
7526 lck_mtx_convert_spin(&ifp->if_cached_route_lock);
7527
7528 route_copyout(dst, &ifp->if_src_route, sizeof (*dst));
7529
7530 lck_mtx_unlock(&ifp->if_cached_route_lock);
7531 }
7532
7533 static void
7534 ifp_src_route_copyin(struct ifnet *ifp, struct route *src)
7535 {
7536 lck_mtx_lock_spin(&ifp->if_cached_route_lock);
7537 lck_mtx_convert_spin(&ifp->if_cached_route_lock);
7538
7539 if (ifp->if_fwd_cacheok) {
7540 route_copyin(src, &ifp->if_src_route, sizeof (*src));
7541 } else {
7542 ROUTE_RELEASE(src);
7543 }
7544 lck_mtx_unlock(&ifp->if_cached_route_lock);
7545 }
7546
7547 #if INET6
7548 static void
7549 ifp_src_route6_copyout(struct ifnet *ifp, struct route_in6 *dst)
7550 {
7551 lck_mtx_lock_spin(&ifp->if_cached_route_lock);
7552 lck_mtx_convert_spin(&ifp->if_cached_route_lock);
7553
7554 route_copyout((struct route *)dst, (struct route *)&ifp->if_src_route6,
7555 sizeof (*dst));
7556
7557 lck_mtx_unlock(&ifp->if_cached_route_lock);
7558 }
7559
7560 static void
7561 ifp_src_route6_copyin(struct ifnet *ifp, struct route_in6 *src)
7562 {
7563 lck_mtx_lock_spin(&ifp->if_cached_route_lock);
7564 lck_mtx_convert_spin(&ifp->if_cached_route_lock);
7565
7566 if (ifp->if_fwd_cacheok) {
7567 route_copyin((struct route *)src,
7568 (struct route *)&ifp->if_src_route6, sizeof (*src));
7569 } else {
7570 ROUTE_RELEASE(src);
7571 }
7572 lck_mtx_unlock(&ifp->if_cached_route_lock);
7573 }
7574 #endif /* INET6 */
7575
7576 struct rtentry *
7577 ifnet_cached_rtlookup_inet(struct ifnet *ifp, struct in_addr src_ip)
7578 {
7579 struct route src_rt;
7580 struct sockaddr_in *dst;
7581
7582 dst = (struct sockaddr_in *)(void *)(&src_rt.ro_dst);
7583
7584 ifp_src_route_copyout(ifp, &src_rt);
7585
7586 if (ROUTE_UNUSABLE(&src_rt) || src_ip.s_addr != dst->sin_addr.s_addr) {
7587 ROUTE_RELEASE(&src_rt);
7588 if (dst->sin_family != AF_INET) {
7589 bzero(&src_rt.ro_dst, sizeof (src_rt.ro_dst));
7590 dst->sin_len = sizeof (src_rt.ro_dst);
7591 dst->sin_family = AF_INET;
7592 }
7593 dst->sin_addr = src_ip;
7594
7595 VERIFY(src_rt.ro_rt == NULL);
7596 src_rt.ro_rt = rtalloc1_scoped((struct sockaddr *)dst,
7597 0, 0, ifp->if_index);
7598
7599 if (src_rt.ro_rt != NULL) {
7600 /* retain a ref, copyin consumes one */
7601 struct rtentry *rte = src_rt.ro_rt;
7602 RT_ADDREF(rte);
7603 ifp_src_route_copyin(ifp, &src_rt);
7604 src_rt.ro_rt = rte;
7605 }
7606 }
7607
7608 return (src_rt.ro_rt);
7609 }
7610
7611 #if INET6
7612 struct rtentry *
7613 ifnet_cached_rtlookup_inet6(struct ifnet *ifp, struct in6_addr *src_ip6)
7614 {
7615 struct route_in6 src_rt;
7616
7617 ifp_src_route6_copyout(ifp, &src_rt);
7618
7619 if (ROUTE_UNUSABLE(&src_rt) ||
7620 !IN6_ARE_ADDR_EQUAL(src_ip6, &src_rt.ro_dst.sin6_addr)) {
7621 ROUTE_RELEASE(&src_rt);
7622 if (src_rt.ro_dst.sin6_family != AF_INET6) {
7623 bzero(&src_rt.ro_dst, sizeof (src_rt.ro_dst));
7624 src_rt.ro_dst.sin6_len = sizeof (src_rt.ro_dst);
7625 src_rt.ro_dst.sin6_family = AF_INET6;
7626 }
7627 src_rt.ro_dst.sin6_scope_id = in6_addr2scopeid(ifp, src_ip6);
7628 bcopy(src_ip6, &src_rt.ro_dst.sin6_addr,
7629 sizeof (src_rt.ro_dst.sin6_addr));
7630
7631 if (src_rt.ro_rt == NULL) {
7632 src_rt.ro_rt = rtalloc1_scoped(
7633 (struct sockaddr *)&src_rt.ro_dst, 0, 0,
7634 ifp->if_index);
7635
7636 if (src_rt.ro_rt != NULL) {
7637 /* retain a ref, copyin consumes one */
7638 struct rtentry *rte = src_rt.ro_rt;
7639 RT_ADDREF(rte);
7640 ifp_src_route6_copyin(ifp, &src_rt);
7641 src_rt.ro_rt = rte;
7642 }
7643 }
7644 }
7645
7646 return (src_rt.ro_rt);
7647 }
7648 #endif /* INET6 */
7649
7650 void
7651 if_lqm_update(struct ifnet *ifp, int lqm, int locked)
7652 {
7653 struct kev_dl_link_quality_metric_data ev_lqm_data;
7654
7655 VERIFY(lqm >= IFNET_LQM_MIN && lqm <= IFNET_LQM_MAX);
7656
7657 /* Normalize to edge */
7658 if (lqm >= 0 && lqm <= IFNET_LQM_THRESH_ABORT) {
7659 lqm = IFNET_LQM_THRESH_ABORT;
7660 atomic_bitset_32(&tcbinfo.ipi_flags,
7661 INPCBINFO_HANDLE_LQM_ABORT);
7662 inpcb_timer_sched(&tcbinfo, INPCB_TIMER_FAST);
7663 } else if (lqm > IFNET_LQM_THRESH_ABORT &&
7664 lqm <= IFNET_LQM_THRESH_MINIMALLY_VIABLE) {
7665 lqm = IFNET_LQM_THRESH_MINIMALLY_VIABLE;
7666 } else if (lqm > IFNET_LQM_THRESH_MINIMALLY_VIABLE &&
7667 lqm <= IFNET_LQM_THRESH_POOR) {
7668 lqm = IFNET_LQM_THRESH_POOR;
7669 } else if (lqm > IFNET_LQM_THRESH_POOR &&
7670 lqm <= IFNET_LQM_THRESH_GOOD) {
7671 lqm = IFNET_LQM_THRESH_GOOD;
7672 }
7673
7674 /*
7675 * Take the lock if needed
7676 */
7677 if (!locked)
7678 ifnet_lock_exclusive(ifp);
7679
7680 if (lqm == ifp->if_interface_state.lqm_state &&
7681 (ifp->if_interface_state.valid_bitmask &
7682 IF_INTERFACE_STATE_LQM_STATE_VALID)) {
7683 /*
7684 * Release the lock if was not held by the caller
7685 */
7686 if (!locked)
7687 ifnet_lock_done(ifp);
7688 return; /* nothing to update */
7689 }
7690 ifp->if_interface_state.valid_bitmask |=
7691 IF_INTERFACE_STATE_LQM_STATE_VALID;
7692 ifp->if_interface_state.lqm_state = lqm;
7693
7694 /*
7695 * Don't want to hold the lock when issuing kernel events
7696 */
7697 ifnet_lock_done(ifp);
7698
7699 bzero(&ev_lqm_data, sizeof (ev_lqm_data));
7700 ev_lqm_data.link_quality_metric = lqm;
7701
7702 dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_LINK_QUALITY_METRIC_CHANGED,
7703 (struct net_event_data *)&ev_lqm_data, sizeof (ev_lqm_data));
7704
7705 /*
7706 * Reacquire the lock for the caller
7707 */
7708 if (locked)
7709 ifnet_lock_exclusive(ifp);
7710 }
7711
7712 static void
7713 if_rrc_state_update(struct ifnet *ifp, unsigned int rrc_state)
7714 {
7715 struct kev_dl_rrc_state kev;
7716
7717 if (rrc_state == ifp->if_interface_state.rrc_state &&
7718 (ifp->if_interface_state.valid_bitmask &
7719 IF_INTERFACE_STATE_RRC_STATE_VALID))
7720 return;
7721
7722 ifp->if_interface_state.valid_bitmask |=
7723 IF_INTERFACE_STATE_RRC_STATE_VALID;
7724
7725 ifp->if_interface_state.rrc_state = rrc_state;
7726
7727 /*
7728 * Don't want to hold the lock when issuing kernel events
7729 */
7730 ifnet_lock_done(ifp);
7731
7732 bzero(&kev, sizeof(struct kev_dl_rrc_state));
7733 kev.rrc_state = rrc_state;
7734
7735 dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_RRC_STATE_CHANGED,
7736 (struct net_event_data *)&kev, sizeof(struct kev_dl_rrc_state));
7737
7738 ifnet_lock_exclusive(ifp);
7739 }
7740
7741 errno_t
7742 if_state_update(struct ifnet *ifp,
7743 struct if_interface_state *if_interface_state)
7744 {
7745 u_short if_index_available = 0;
7746
7747 ifnet_lock_exclusive(ifp);
7748
7749 if ((ifp->if_type != IFT_CELLULAR) &&
7750 (if_interface_state->valid_bitmask &
7751 IF_INTERFACE_STATE_RRC_STATE_VALID)) {
7752 ifnet_lock_done(ifp);
7753 return (ENOTSUP);
7754 }
7755 if ((if_interface_state->valid_bitmask &
7756 IF_INTERFACE_STATE_LQM_STATE_VALID) &&
7757 (if_interface_state->lqm_state < IFNET_LQM_MIN ||
7758 if_interface_state->lqm_state > IFNET_LQM_MAX)) {
7759 ifnet_lock_done(ifp);
7760 return (EINVAL);
7761 }
7762 if ((if_interface_state->valid_bitmask &
7763 IF_INTERFACE_STATE_RRC_STATE_VALID) &&
7764 if_interface_state->rrc_state !=
7765 IF_INTERFACE_STATE_RRC_STATE_IDLE &&
7766 if_interface_state->rrc_state !=
7767 IF_INTERFACE_STATE_RRC_STATE_CONNECTED) {
7768 ifnet_lock_done(ifp);
7769 return (EINVAL);
7770 }
7771
7772 if (if_interface_state->valid_bitmask &
7773 IF_INTERFACE_STATE_LQM_STATE_VALID) {
7774 if_lqm_update(ifp, if_interface_state->lqm_state, 1);
7775 }
7776 if (if_interface_state->valid_bitmask &
7777 IF_INTERFACE_STATE_RRC_STATE_VALID) {
7778 if_rrc_state_update(ifp, if_interface_state->rrc_state);
7779 }
7780 if (if_interface_state->valid_bitmask &
7781 IF_INTERFACE_STATE_INTERFACE_AVAILABILITY_VALID) {
7782 ifp->if_interface_state.valid_bitmask |=
7783 IF_INTERFACE_STATE_INTERFACE_AVAILABILITY_VALID;
7784 ifp->if_interface_state.interface_availability =
7785 if_interface_state->interface_availability;
7786
7787 if (ifp->if_interface_state.interface_availability ==
7788 IF_INTERFACE_STATE_INTERFACE_AVAILABLE) {
7789 if_index_available = ifp->if_index;
7790 }
7791 }
7792 ifnet_lock_done(ifp);
7793
7794 /*
7795 * Check if the TCP connections going on this interface should be
7796 * forced to send probe packets instead of waiting for TCP timers
7797 * to fire. This will be done when there is an explicit
7798 * notification that the interface became available.
7799 */
7800 if (if_index_available > 0)
7801 tcp_interface_send_probe(if_index_available);
7802
7803 return (0);
7804 }
7805
7806 void
7807 if_get_state(struct ifnet *ifp,
7808 struct if_interface_state *if_interface_state)
7809 {
7810 ifnet_lock_shared(ifp);
7811
7812 if_interface_state->valid_bitmask = 0;
7813
7814 if (ifp->if_interface_state.valid_bitmask &
7815 IF_INTERFACE_STATE_RRC_STATE_VALID) {
7816 if_interface_state->valid_bitmask |=
7817 IF_INTERFACE_STATE_RRC_STATE_VALID;
7818 if_interface_state->rrc_state =
7819 ifp->if_interface_state.rrc_state;
7820 }
7821 if (ifp->if_interface_state.valid_bitmask &
7822 IF_INTERFACE_STATE_LQM_STATE_VALID) {
7823 if_interface_state->valid_bitmask |=
7824 IF_INTERFACE_STATE_LQM_STATE_VALID;
7825 if_interface_state->lqm_state =
7826 ifp->if_interface_state.lqm_state;
7827 }
7828 if (ifp->if_interface_state.valid_bitmask &
7829 IF_INTERFACE_STATE_INTERFACE_AVAILABILITY_VALID) {
7830 if_interface_state->valid_bitmask |=
7831 IF_INTERFACE_STATE_INTERFACE_AVAILABILITY_VALID;
7832 if_interface_state->interface_availability =
7833 ifp->if_interface_state.interface_availability;
7834 }
7835
7836 ifnet_lock_done(ifp);
7837 }
7838
7839 errno_t
7840 if_probe_connectivity(struct ifnet *ifp, u_int32_t conn_probe)
7841 {
7842 ifnet_lock_exclusive(ifp);
7843 if (conn_probe > 1) {
7844 ifnet_lock_done(ifp);
7845 return (EINVAL);
7846 }
7847 if (conn_probe == 0)
7848 ifp->if_eflags &= ~IFEF_PROBE_CONNECTIVITY;
7849 else
7850 ifp->if_eflags |= IFEF_PROBE_CONNECTIVITY;
7851 ifnet_lock_done(ifp);
7852
7853 #if NECP
7854 necp_update_all_clients();
7855 #endif /* NECP */
7856
7857 tcp_probe_connectivity(ifp, conn_probe);
7858 return (0);
7859 }
7860
7861 /* for uuid.c */
7862 int
7863 uuid_get_ethernet(u_int8_t *node)
7864 {
7865 struct ifnet *ifp;
7866 struct sockaddr_dl *sdl;
7867
7868 ifnet_head_lock_shared();
7869 TAILQ_FOREACH(ifp, &ifnet_head, if_link) {
7870 ifnet_lock_shared(ifp);
7871 IFA_LOCK_SPIN(ifp->if_lladdr);
7872 sdl = (struct sockaddr_dl *)(void *)ifp->if_lladdr->ifa_addr;
7873 if (sdl->sdl_type == IFT_ETHER) {
7874 memcpy(node, LLADDR(sdl), ETHER_ADDR_LEN);
7875 IFA_UNLOCK(ifp->if_lladdr);
7876 ifnet_lock_done(ifp);
7877 ifnet_head_done();
7878 return (0);
7879 }
7880 IFA_UNLOCK(ifp->if_lladdr);
7881 ifnet_lock_done(ifp);
7882 }
7883 ifnet_head_done();
7884
7885 return (-1);
7886 }
7887
7888 static int
7889 sysctl_rxpoll SYSCTL_HANDLER_ARGS
7890 {
7891 #pragma unused(arg1, arg2)
7892 uint32_t i;
7893 int err;
7894
7895 i = if_rxpoll;
7896
7897 err = sysctl_handle_int(oidp, &i, 0, req);
7898 if (err != 0 || req->newptr == USER_ADDR_NULL)
7899 return (err);
7900
7901 if (net_rxpoll == 0)
7902 return (ENXIO);
7903
7904 if_rxpoll = i;
7905 return (err);
7906 }
7907
7908 static int
7909 sysctl_rxpoll_mode_holdtime SYSCTL_HANDLER_ARGS
7910 {
7911 #pragma unused(arg1, arg2)
7912 uint64_t q;
7913 int err;
7914
7915 q = if_rxpoll_mode_holdtime;
7916
7917 err = sysctl_handle_quad(oidp, &q, 0, req);
7918 if (err != 0 || req->newptr == USER_ADDR_NULL)
7919 return (err);
7920
7921 if (q < IF_RXPOLL_MODE_HOLDTIME_MIN)
7922 q = IF_RXPOLL_MODE_HOLDTIME_MIN;
7923
7924 if_rxpoll_mode_holdtime = q;
7925
7926 return (err);
7927 }
7928
7929 static int
7930 sysctl_rxpoll_sample_holdtime SYSCTL_HANDLER_ARGS
7931 {
7932 #pragma unused(arg1, arg2)
7933 uint64_t q;
7934 int err;
7935
7936 q = if_rxpoll_sample_holdtime;
7937
7938 err = sysctl_handle_quad(oidp, &q, 0, req);
7939 if (err != 0 || req->newptr == USER_ADDR_NULL)
7940 return (err);
7941
7942 if (q < IF_RXPOLL_SAMPLETIME_MIN)
7943 q = IF_RXPOLL_SAMPLETIME_MIN;
7944
7945 if_rxpoll_sample_holdtime = q;
7946
7947 return (err);
7948 }
7949
7950 static int
7951 sysctl_rxpoll_interval_time SYSCTL_HANDLER_ARGS
7952 {
7953 #pragma unused(arg1, arg2)
7954 uint64_t q;
7955 int err;
7956
7957 q = if_rxpoll_interval_time;
7958
7959 err = sysctl_handle_quad(oidp, &q, 0, req);
7960 if (err != 0 || req->newptr == USER_ADDR_NULL)
7961 return (err);
7962
7963 if (q < IF_RXPOLL_INTERVALTIME_MIN)
7964 q = IF_RXPOLL_INTERVALTIME_MIN;
7965
7966 if_rxpoll_interval_time = q;
7967
7968 return (err);
7969 }
7970
7971 static int
7972 sysctl_rxpoll_wlowat SYSCTL_HANDLER_ARGS
7973 {
7974 #pragma unused(arg1, arg2)
7975 uint32_t i;
7976 int err;
7977
7978 i = if_rxpoll_wlowat;
7979
7980 err = sysctl_handle_int(oidp, &i, 0, req);
7981 if (err != 0 || req->newptr == USER_ADDR_NULL)
7982 return (err);
7983
7984 if (i == 0 || i >= if_rxpoll_whiwat)
7985 return (EINVAL);
7986
7987 if_rxpoll_wlowat = i;
7988 return (err);
7989 }
7990
7991 static int
7992 sysctl_rxpoll_whiwat SYSCTL_HANDLER_ARGS
7993 {
7994 #pragma unused(arg1, arg2)
7995 uint32_t i;
7996 int err;
7997
7998 i = if_rxpoll_whiwat;
7999
8000 err = sysctl_handle_int(oidp, &i, 0, req);
8001 if (err != 0 || req->newptr == USER_ADDR_NULL)
8002 return (err);
8003
8004 if (i <= if_rxpoll_wlowat)
8005 return (EINVAL);
8006
8007 if_rxpoll_whiwat = i;
8008 return (err);
8009 }
8010
8011 static int
8012 sysctl_sndq_maxlen SYSCTL_HANDLER_ARGS
8013 {
8014 #pragma unused(arg1, arg2)
8015 int i, err;
8016
8017 i = if_sndq_maxlen;
8018
8019 err = sysctl_handle_int(oidp, &i, 0, req);
8020 if (err != 0 || req->newptr == USER_ADDR_NULL)
8021 return (err);
8022
8023 if (i < IF_SNDQ_MINLEN)
8024 i = IF_SNDQ_MINLEN;
8025
8026 if_sndq_maxlen = i;
8027 return (err);
8028 }
8029
8030 static int
8031 sysctl_rcvq_maxlen SYSCTL_HANDLER_ARGS
8032 {
8033 #pragma unused(arg1, arg2)
8034 int i, err;
8035
8036 i = if_rcvq_maxlen;
8037
8038 err = sysctl_handle_int(oidp, &i, 0, req);
8039 if (err != 0 || req->newptr == USER_ADDR_NULL)
8040 return (err);
8041
8042 if (i < IF_RCVQ_MINLEN)
8043 i = IF_RCVQ_MINLEN;
8044
8045 if_rcvq_maxlen = i;
8046 return (err);
8047 }
8048
8049 void
8050 dlil_node_present(struct ifnet *ifp, struct sockaddr *sa,
8051 int32_t rssi, int lqm, int npm, u_int8_t srvinfo[48])
8052 {
8053 struct kev_dl_node_presence kev;
8054 struct sockaddr_dl *sdl;
8055 struct sockaddr_in6 *sin6;
8056
8057 VERIFY(ifp);
8058 VERIFY(sa);
8059 VERIFY(sa->sa_family == AF_LINK || sa->sa_family == AF_INET6);
8060
8061 bzero(&kev, sizeof (kev));
8062 sin6 = &kev.sin6_node_address;
8063 sdl = &kev.sdl_node_address;
8064 nd6_alt_node_addr_decompose(ifp, sa, sdl, sin6);
8065 kev.rssi = rssi;
8066 kev.link_quality_metric = lqm;
8067 kev.node_proximity_metric = npm;
8068 bcopy(srvinfo, kev.node_service_info, sizeof (kev.node_service_info));
8069
8070 nd6_alt_node_present(ifp, sin6, sdl, rssi, lqm, npm);
8071 dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_NODE_PRESENCE,
8072 &kev.link_data, sizeof (kev));
8073 }
8074
8075 void
8076 dlil_node_absent(struct ifnet *ifp, struct sockaddr *sa)
8077 {
8078 struct kev_dl_node_absence kev;
8079 struct sockaddr_in6 *sin6;
8080 struct sockaddr_dl *sdl;
8081
8082 VERIFY(ifp);
8083 VERIFY(sa);
8084 VERIFY(sa->sa_family == AF_LINK || sa->sa_family == AF_INET6);
8085
8086 bzero(&kev, sizeof (kev));
8087 sin6 = &kev.sin6_node_address;
8088 sdl = &kev.sdl_node_address;
8089 nd6_alt_node_addr_decompose(ifp, sa, sdl, sin6);
8090
8091 nd6_alt_node_absent(ifp, sin6);
8092 dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_NODE_ABSENCE,
8093 &kev.link_data, sizeof (kev));
8094 }
8095
8096 const void *
8097 dlil_ifaddr_bytes(const struct sockaddr_dl *sdl, size_t *sizep,
8098 kauth_cred_t *credp)
8099 {
8100 const u_int8_t *bytes;
8101 size_t size;
8102
8103 bytes = CONST_LLADDR(sdl);
8104 size = sdl->sdl_alen;
8105
8106 #if CONFIG_MACF
8107 if (dlil_lladdr_ckreq) {
8108 switch (sdl->sdl_type) {
8109 case IFT_ETHER:
8110 case IFT_IEEE1394:
8111 break;
8112 default:
8113 credp = NULL;
8114 break;
8115 };
8116
8117 if (credp && mac_system_check_info(*credp, "net.link.addr")) {
8118 static const u_int8_t unspec[FIREWIRE_EUI64_LEN] = {
8119 [0] = 2
8120 };
8121
8122 bytes = unspec;
8123 }
8124 }
8125 #else
8126 #pragma unused(credp)
8127 #endif
8128
8129 if (sizep != NULL) *sizep = size;
8130 return (bytes);
8131 }
8132
8133 void
8134 dlil_report_issues(struct ifnet *ifp, u_int8_t modid[DLIL_MODIDLEN],
8135 u_int8_t info[DLIL_MODARGLEN])
8136 {
8137 struct kev_dl_issues kev;
8138 struct timeval tv;
8139
8140 VERIFY(ifp != NULL);
8141 VERIFY(modid != NULL);
8142 _CASSERT(sizeof (kev.modid) == DLIL_MODIDLEN);
8143 _CASSERT(sizeof (kev.info) == DLIL_MODARGLEN);
8144
8145 bzero(&kev, sizeof (kev));
8146
8147 microtime(&tv);
8148 kev.timestamp = tv.tv_sec;
8149 bcopy(modid, &kev.modid, DLIL_MODIDLEN);
8150 if (info != NULL)
8151 bcopy(info, &kev.info, DLIL_MODARGLEN);
8152
8153 dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_ISSUES,
8154 &kev.link_data, sizeof (kev));
8155 }
8156
8157 errno_t
8158 ifnet_getset_opportunistic(ifnet_t ifp, u_long cmd, struct ifreq *ifr,
8159 struct proc *p)
8160 {
8161 u_int32_t level = IFNET_THROTTLE_OFF;
8162 errno_t result = 0;
8163
8164 VERIFY(cmd == SIOCSIFOPPORTUNISTIC || cmd == SIOCGIFOPPORTUNISTIC);
8165
8166 if (cmd == SIOCSIFOPPORTUNISTIC) {
8167 /*
8168 * XXX: Use priv_check_cred() instead of root check?
8169 */
8170 if ((result = proc_suser(p)) != 0)
8171 return (result);
8172
8173 if (ifr->ifr_opportunistic.ifo_flags ==
8174 IFRIFOF_BLOCK_OPPORTUNISTIC)
8175 level = IFNET_THROTTLE_OPPORTUNISTIC;
8176 else if (ifr->ifr_opportunistic.ifo_flags == 0)
8177 level = IFNET_THROTTLE_OFF;
8178 else
8179 result = EINVAL;
8180
8181 if (result == 0)
8182 result = ifnet_set_throttle(ifp, level);
8183 } else if ((result = ifnet_get_throttle(ifp, &level)) == 0) {
8184 ifr->ifr_opportunistic.ifo_flags = 0;
8185 if (level == IFNET_THROTTLE_OPPORTUNISTIC) {
8186 ifr->ifr_opportunistic.ifo_flags |=
8187 IFRIFOF_BLOCK_OPPORTUNISTIC;
8188 }
8189 }
8190
8191 /*
8192 * Return the count of current opportunistic connections
8193 * over the interface.
8194 */
8195 if (result == 0) {
8196 uint32_t flags = 0;
8197 flags |= (cmd == SIOCSIFOPPORTUNISTIC) ?
8198 INPCB_OPPORTUNISTIC_SETCMD : 0;
8199 flags |= (level == IFNET_THROTTLE_OPPORTUNISTIC) ?
8200 INPCB_OPPORTUNISTIC_THROTTLEON : 0;
8201 ifr->ifr_opportunistic.ifo_inuse =
8202 udp_count_opportunistic(ifp->if_index, flags) +
8203 tcp_count_opportunistic(ifp->if_index, flags);
8204 }
8205
8206 if (result == EALREADY)
8207 result = 0;
8208
8209 return (result);
8210 }
8211
8212 int
8213 ifnet_get_throttle(struct ifnet *ifp, u_int32_t *level)
8214 {
8215 struct ifclassq *ifq;
8216 int err = 0;
8217
8218 if (!(ifp->if_eflags & IFEF_TXSTART))
8219 return (ENXIO);
8220
8221 *level = IFNET_THROTTLE_OFF;
8222
8223 ifq = &ifp->if_snd;
8224 IFCQ_LOCK(ifq);
8225 /* Throttling works only for IFCQ, not ALTQ instances */
8226 if (IFCQ_IS_ENABLED(ifq))
8227 IFCQ_GET_THROTTLE(ifq, *level, err);
8228 IFCQ_UNLOCK(ifq);
8229
8230 return (err);
8231 }
8232
8233 int
8234 ifnet_set_throttle(struct ifnet *ifp, u_int32_t level)
8235 {
8236 struct ifclassq *ifq;
8237 int err = 0;
8238
8239 if (!(ifp->if_eflags & IFEF_TXSTART))
8240 return (ENXIO);
8241
8242 ifq = &ifp->if_snd;
8243
8244 switch (level) {
8245 case IFNET_THROTTLE_OFF:
8246 case IFNET_THROTTLE_OPPORTUNISTIC:
8247 break;
8248 default:
8249 return (EINVAL);
8250 }
8251
8252 IFCQ_LOCK(ifq);
8253 if (IFCQ_IS_ENABLED(ifq))
8254 IFCQ_SET_THROTTLE(ifq, level, err);
8255 IFCQ_UNLOCK(ifq);
8256
8257 if (err == 0) {
8258 printf("%s: throttling level set to %d\n", if_name(ifp),
8259 level);
8260 if (level == IFNET_THROTTLE_OFF)
8261 ifnet_start(ifp);
8262 }
8263
8264 return (err);
8265 }
8266
8267 errno_t
8268 ifnet_getset_log(ifnet_t ifp, u_long cmd, struct ifreq *ifr,
8269 struct proc *p)
8270 {
8271 #pragma unused(p)
8272 errno_t result = 0;
8273 uint32_t flags;
8274 int level, category, subcategory;
8275
8276 VERIFY(cmd == SIOCSIFLOG || cmd == SIOCGIFLOG);
8277
8278 if (cmd == SIOCSIFLOG) {
8279 if ((result = priv_check_cred(kauth_cred_get(),
8280 PRIV_NET_INTERFACE_CONTROL, 0)) != 0)
8281 return (result);
8282
8283 level = ifr->ifr_log.ifl_level;
8284 if (level < IFNET_LOG_MIN || level > IFNET_LOG_MAX)
8285 result = EINVAL;
8286
8287 flags = ifr->ifr_log.ifl_flags;
8288 if ((flags &= IFNET_LOGF_MASK) == 0)
8289 result = EINVAL;
8290
8291 category = ifr->ifr_log.ifl_category;
8292 subcategory = ifr->ifr_log.ifl_subcategory;
8293
8294 if (result == 0)
8295 result = ifnet_set_log(ifp, level, flags,
8296 category, subcategory);
8297 } else {
8298 result = ifnet_get_log(ifp, &level, &flags, &category,
8299 &subcategory);
8300 if (result == 0) {
8301 ifr->ifr_log.ifl_level = level;
8302 ifr->ifr_log.ifl_flags = flags;
8303 ifr->ifr_log.ifl_category = category;
8304 ifr->ifr_log.ifl_subcategory = subcategory;
8305 }
8306 }
8307
8308 return (result);
8309 }
8310
8311 int
8312 ifnet_set_log(struct ifnet *ifp, int32_t level, uint32_t flags,
8313 int32_t category, int32_t subcategory)
8314 {
8315 int err = 0;
8316
8317 VERIFY(level >= IFNET_LOG_MIN && level <= IFNET_LOG_MAX);
8318 VERIFY(flags & IFNET_LOGF_MASK);
8319
8320 /*
8321 * The logging level applies to all facilities; make sure to
8322 * update them all with the most current level.
8323 */
8324 flags |= ifp->if_log.flags;
8325
8326 if (ifp->if_output_ctl != NULL) {
8327 struct ifnet_log_params l;
8328
8329 bzero(&l, sizeof (l));
8330 l.level = level;
8331 l.flags = flags;
8332 l.flags &= ~IFNET_LOGF_DLIL;
8333 l.category = category;
8334 l.subcategory = subcategory;
8335
8336 /* Send this request to lower layers */
8337 if (l.flags != 0) {
8338 err = ifp->if_output_ctl(ifp, IFNET_CTL_SET_LOG,
8339 sizeof (l), &l);
8340 }
8341 } else if ((flags & ~IFNET_LOGF_DLIL) && ifp->if_output_ctl == NULL) {
8342 /*
8343 * If targeted to the lower layers without an output
8344 * control callback registered on the interface, just
8345 * silently ignore facilities other than ours.
8346 */
8347 flags &= IFNET_LOGF_DLIL;
8348 if (flags == 0 && (!(ifp->if_log.flags & IFNET_LOGF_DLIL)))
8349 level = 0;
8350 }
8351
8352 if (err == 0) {
8353 if ((ifp->if_log.level = level) == IFNET_LOG_DEFAULT)
8354 ifp->if_log.flags = 0;
8355 else
8356 ifp->if_log.flags |= flags;
8357
8358 log(LOG_INFO, "%s: logging level set to %d flags=%b "
8359 "arg=%b, category=%d subcategory=%d\n", if_name(ifp),
8360 ifp->if_log.level, ifp->if_log.flags,
8361 IFNET_LOGF_BITS, flags, IFNET_LOGF_BITS,
8362 category, subcategory);
8363 }
8364
8365 return (err);
8366 }
8367
8368 int
8369 ifnet_get_log(struct ifnet *ifp, int32_t *level, uint32_t *flags,
8370 int32_t *category, int32_t *subcategory)
8371 {
8372 if (level != NULL)
8373 *level = ifp->if_log.level;
8374 if (flags != NULL)
8375 *flags = ifp->if_log.flags;
8376 if (category != NULL)
8377 *category = ifp->if_log.category;
8378 if (subcategory != NULL)
8379 *subcategory = ifp->if_log.subcategory;
8380
8381 return (0);
8382 }
8383
8384 int
8385 ifnet_notify_address(struct ifnet *ifp, int af)
8386 {
8387 struct ifnet_notify_address_params na;
8388
8389 #if PF
8390 (void) pf_ifaddr_hook(ifp);
8391 #endif /* PF */
8392
8393 if (ifp->if_output_ctl == NULL)
8394 return (EOPNOTSUPP);
8395
8396 bzero(&na, sizeof (na));
8397 na.address_family = af;
8398
8399 return (ifp->if_output_ctl(ifp, IFNET_CTL_NOTIFY_ADDRESS,
8400 sizeof (na), &na));
8401 }
8402
8403 errno_t
8404 ifnet_flowid(struct ifnet *ifp, uint32_t *flowid)
8405 {
8406 if (ifp == NULL || flowid == NULL) {
8407 return (EINVAL);
8408 } else if (!(ifp->if_eflags & IFEF_TXSTART) ||
8409 !IF_FULLY_ATTACHED(ifp)) {
8410 return (ENXIO);
8411 }
8412
8413 *flowid = ifp->if_flowhash;
8414
8415 return (0);
8416 }
8417
8418 errno_t
8419 ifnet_disable_output(struct ifnet *ifp)
8420 {
8421 int err;
8422
8423 if (ifp == NULL) {
8424 return (EINVAL);
8425 } else if (!(ifp->if_eflags & IFEF_TXSTART) ||
8426 !IF_FULLY_ATTACHED(ifp)) {
8427 return (ENXIO);
8428 }
8429
8430 if ((err = ifnet_fc_add(ifp)) == 0) {
8431 lck_mtx_lock_spin(&ifp->if_start_lock);
8432 ifp->if_start_flags |= IFSF_FLOW_CONTROLLED;
8433 lck_mtx_unlock(&ifp->if_start_lock);
8434 }
8435 return (err);
8436 }
8437
8438 errno_t
8439 ifnet_enable_output(struct ifnet *ifp)
8440 {
8441 if (ifp == NULL) {
8442 return (EINVAL);
8443 } else if (!(ifp->if_eflags & IFEF_TXSTART) ||
8444 !IF_FULLY_ATTACHED(ifp)) {
8445 return (ENXIO);
8446 }
8447
8448 ifnet_start_common(ifp, TRUE);
8449 return (0);
8450 }
8451
8452 void
8453 ifnet_flowadv(uint32_t flowhash)
8454 {
8455 struct ifnet_fc_entry *ifce;
8456 struct ifnet *ifp;
8457
8458 ifce = ifnet_fc_get(flowhash);
8459 if (ifce == NULL)
8460 return;
8461
8462 VERIFY(ifce->ifce_ifp != NULL);
8463 ifp = ifce->ifce_ifp;
8464
8465 /* flow hash gets recalculated per attach, so check */
8466 if (ifnet_is_attached(ifp, 1)) {
8467 if (ifp->if_flowhash == flowhash)
8468 (void) ifnet_enable_output(ifp);
8469 ifnet_decr_iorefcnt(ifp);
8470 }
8471 ifnet_fc_entry_free(ifce);
8472 }
8473
8474 /*
8475 * Function to compare ifnet_fc_entries in ifnet flow control tree
8476 */
8477 static inline int
8478 ifce_cmp(const struct ifnet_fc_entry *fc1, const struct ifnet_fc_entry *fc2)
8479 {
8480 return (fc1->ifce_flowhash - fc2->ifce_flowhash);
8481 }
8482
8483 static int
8484 ifnet_fc_add(struct ifnet *ifp)
8485 {
8486 struct ifnet_fc_entry keyfc, *ifce;
8487 uint32_t flowhash;
8488
8489 VERIFY(ifp != NULL && (ifp->if_eflags & IFEF_TXSTART));
8490 VERIFY(ifp->if_flowhash != 0);
8491 flowhash = ifp->if_flowhash;
8492
8493 bzero(&keyfc, sizeof (keyfc));
8494 keyfc.ifce_flowhash = flowhash;
8495
8496 lck_mtx_lock_spin(&ifnet_fc_lock);
8497 ifce = RB_FIND(ifnet_fc_tree, &ifnet_fc_tree, &keyfc);
8498 if (ifce != NULL && ifce->ifce_ifp == ifp) {
8499 /* Entry is already in ifnet_fc_tree, return */
8500 lck_mtx_unlock(&ifnet_fc_lock);
8501 return (0);
8502 }
8503
8504 if (ifce != NULL) {
8505 /*
8506 * There is a different fc entry with the same flow hash
8507 * but different ifp pointer. There can be a collision
8508 * on flow hash but the probability is low. Let's just
8509 * avoid adding a second one when there is a collision.
8510 */
8511 lck_mtx_unlock(&ifnet_fc_lock);
8512 return (EAGAIN);
8513 }
8514
8515 /* become regular mutex */
8516 lck_mtx_convert_spin(&ifnet_fc_lock);
8517
8518 ifce = zalloc(ifnet_fc_zone);
8519 if (ifce == NULL) {
8520 /* memory allocation failed */
8521 lck_mtx_unlock(&ifnet_fc_lock);
8522 return (ENOMEM);
8523 }
8524 bzero(ifce, ifnet_fc_zone_size);
8525
8526 ifce->ifce_flowhash = flowhash;
8527 ifce->ifce_ifp = ifp;
8528
8529 RB_INSERT(ifnet_fc_tree, &ifnet_fc_tree, ifce);
8530 lck_mtx_unlock(&ifnet_fc_lock);
8531 return (0);
8532 }
8533
8534 static struct ifnet_fc_entry *
8535 ifnet_fc_get(uint32_t flowhash)
8536 {
8537 struct ifnet_fc_entry keyfc, *ifce;
8538 struct ifnet *ifp;
8539
8540 bzero(&keyfc, sizeof (keyfc));
8541 keyfc.ifce_flowhash = flowhash;
8542
8543 lck_mtx_lock_spin(&ifnet_fc_lock);
8544 ifce = RB_FIND(ifnet_fc_tree, &ifnet_fc_tree, &keyfc);
8545 if (ifce == NULL) {
8546 /* Entry is not present in ifnet_fc_tree, return */
8547 lck_mtx_unlock(&ifnet_fc_lock);
8548 return (NULL);
8549 }
8550
8551 RB_REMOVE(ifnet_fc_tree, &ifnet_fc_tree, ifce);
8552
8553 VERIFY(ifce->ifce_ifp != NULL);
8554 ifp = ifce->ifce_ifp;
8555
8556 /* become regular mutex */
8557 lck_mtx_convert_spin(&ifnet_fc_lock);
8558
8559 if (!ifnet_is_attached(ifp, 0)) {
8560 /*
8561 * This ifp is not attached or in the process of being
8562 * detached; just don't process it.
8563 */
8564 ifnet_fc_entry_free(ifce);
8565 ifce = NULL;
8566 }
8567 lck_mtx_unlock(&ifnet_fc_lock);
8568
8569 return (ifce);
8570 }
8571
8572 static void
8573 ifnet_fc_entry_free(struct ifnet_fc_entry *ifce)
8574 {
8575 zfree(ifnet_fc_zone, ifce);
8576 }
8577
8578 static uint32_t
8579 ifnet_calc_flowhash(struct ifnet *ifp)
8580 {
8581 struct ifnet_flowhash_key fh __attribute__((aligned(8)));
8582 uint32_t flowhash = 0;
8583
8584 if (ifnet_flowhash_seed == 0)
8585 ifnet_flowhash_seed = RandomULong();
8586
8587 bzero(&fh, sizeof (fh));
8588
8589 (void) snprintf(fh.ifk_name, sizeof (fh.ifk_name), "%s", ifp->if_name);
8590 fh.ifk_unit = ifp->if_unit;
8591 fh.ifk_flags = ifp->if_flags;
8592 fh.ifk_eflags = ifp->if_eflags;
8593 fh.ifk_capabilities = ifp->if_capabilities;
8594 fh.ifk_capenable = ifp->if_capenable;
8595 fh.ifk_output_sched_model = ifp->if_output_sched_model;
8596 fh.ifk_rand1 = RandomULong();
8597 fh.ifk_rand2 = RandomULong();
8598
8599 try_again:
8600 flowhash = net_flowhash(&fh, sizeof (fh), ifnet_flowhash_seed);
8601 if (flowhash == 0) {
8602 /* try to get a non-zero flowhash */
8603 ifnet_flowhash_seed = RandomULong();
8604 goto try_again;
8605 }
8606
8607 return (flowhash);
8608 }
8609
8610 int
8611 ifnet_set_netsignature(struct ifnet *ifp, uint8_t family, uint8_t len,
8612 uint16_t flags, uint8_t *data)
8613 {
8614 #pragma unused(flags)
8615 int error = 0;
8616
8617 switch (family) {
8618 case AF_INET:
8619 if_inetdata_lock_exclusive(ifp);
8620 if (IN_IFEXTRA(ifp) != NULL) {
8621 if (len == 0) {
8622 /* Allow clearing the signature */
8623 IN_IFEXTRA(ifp)->netsig_len = 0;
8624 bzero(IN_IFEXTRA(ifp)->netsig,
8625 sizeof (IN_IFEXTRA(ifp)->netsig));
8626 if_inetdata_lock_done(ifp);
8627 break;
8628 } else if (len > sizeof (IN_IFEXTRA(ifp)->netsig)) {
8629 error = EINVAL;
8630 if_inetdata_lock_done(ifp);
8631 break;
8632 }
8633 IN_IFEXTRA(ifp)->netsig_len = len;
8634 bcopy(data, IN_IFEXTRA(ifp)->netsig, len);
8635 } else {
8636 error = ENOMEM;
8637 }
8638 if_inetdata_lock_done(ifp);
8639 break;
8640
8641 case AF_INET6:
8642 if_inet6data_lock_exclusive(ifp);
8643 if (IN6_IFEXTRA(ifp) != NULL) {
8644 if (len == 0) {
8645 /* Allow clearing the signature */
8646 IN6_IFEXTRA(ifp)->netsig_len = 0;
8647 bzero(IN6_IFEXTRA(ifp)->netsig,
8648 sizeof (IN6_IFEXTRA(ifp)->netsig));
8649 if_inet6data_lock_done(ifp);
8650 break;
8651 } else if (len > sizeof (IN6_IFEXTRA(ifp)->netsig)) {
8652 error = EINVAL;
8653 if_inet6data_lock_done(ifp);
8654 break;
8655 }
8656 IN6_IFEXTRA(ifp)->netsig_len = len;
8657 bcopy(data, IN6_IFEXTRA(ifp)->netsig, len);
8658 } else {
8659 error = ENOMEM;
8660 }
8661 if_inet6data_lock_done(ifp);
8662 break;
8663
8664 default:
8665 error = EINVAL;
8666 break;
8667 }
8668
8669 return (error);
8670 }
8671
8672 int
8673 ifnet_get_netsignature(struct ifnet *ifp, uint8_t family, uint8_t *len,
8674 uint16_t *flags, uint8_t *data)
8675 {
8676 int error = 0;
8677
8678 if (ifp == NULL || len == NULL || data == NULL)
8679 return (EINVAL);
8680
8681 switch (family) {
8682 case AF_INET:
8683 if_inetdata_lock_shared(ifp);
8684 if (IN_IFEXTRA(ifp) != NULL) {
8685 if (*len == 0 || *len < IN_IFEXTRA(ifp)->netsig_len) {
8686 error = EINVAL;
8687 if_inetdata_lock_done(ifp);
8688 break;
8689 }
8690 if ((*len = IN_IFEXTRA(ifp)->netsig_len) > 0)
8691 bcopy(IN_IFEXTRA(ifp)->netsig, data, *len);
8692 else
8693 error = ENOENT;
8694 } else {
8695 error = ENOMEM;
8696 }
8697 if_inetdata_lock_done(ifp);
8698 break;
8699
8700 case AF_INET6:
8701 if_inet6data_lock_shared(ifp);
8702 if (IN6_IFEXTRA(ifp) != NULL) {
8703 if (*len == 0 || *len < IN6_IFEXTRA(ifp)->netsig_len) {
8704 error = EINVAL;
8705 if_inet6data_lock_done(ifp);
8706 break;
8707 }
8708 if ((*len = IN6_IFEXTRA(ifp)->netsig_len) > 0)
8709 bcopy(IN6_IFEXTRA(ifp)->netsig, data, *len);
8710 else
8711 error = ENOENT;
8712 } else {
8713 error = ENOMEM;
8714 }
8715 if_inet6data_lock_done(ifp);
8716 break;
8717
8718 default:
8719 error = EINVAL;
8720 break;
8721 }
8722
8723 if (error == 0 && flags != NULL)
8724 *flags = 0;
8725
8726 return (error);
8727 }
8728
8729 #if INET6
8730 int
8731 ifnet_set_nat64prefix(struct ifnet *ifp, struct ipv6_prefix *prefixes)
8732 {
8733 int i, error = 0, one_set = 0;
8734
8735 if_inet6data_lock_exclusive(ifp);
8736
8737 if (IN6_IFEXTRA(ifp) == NULL) {
8738 error = ENOMEM;
8739 goto out;
8740 }
8741
8742 for (i = 0; i < NAT64_MAX_NUM_PREFIXES; i++) {
8743 uint32_t prefix_len =
8744 prefixes[i].prefix_len;
8745 struct in6_addr *prefix =
8746 &prefixes[i].ipv6_prefix;
8747
8748 if (prefix_len == 0) {
8749 clat_log0((LOG_DEBUG,
8750 "NAT64 prefixes purged from Interface %s\n",
8751 if_name(ifp)));
8752 /* Allow clearing the signature */
8753 IN6_IFEXTRA(ifp)->nat64_prefixes[i].prefix_len = 0;
8754 bzero(&IN6_IFEXTRA(ifp)->nat64_prefixes[i].ipv6_prefix,
8755 sizeof(struct in6_addr));
8756
8757 continue;
8758 } else if (prefix_len != NAT64_PREFIX_LEN_32 &&
8759 prefix_len != NAT64_PREFIX_LEN_40 &&
8760 prefix_len != NAT64_PREFIX_LEN_48 &&
8761 prefix_len != NAT64_PREFIX_LEN_56 &&
8762 prefix_len != NAT64_PREFIX_LEN_64 &&
8763 prefix_len != NAT64_PREFIX_LEN_96) {
8764 clat_log0((LOG_DEBUG,
8765 "NAT64 prefixlen is incorrect %d\n", prefix_len));
8766 error = EINVAL;
8767 goto out;
8768 }
8769
8770 if (IN6_IS_SCOPE_EMBED(prefix)) {
8771 clat_log0((LOG_DEBUG,
8772 "NAT64 prefix has interface/link local scope.\n"));
8773 error = EINVAL;
8774 goto out;
8775 }
8776
8777 IN6_IFEXTRA(ifp)->nat64_prefixes[i].prefix_len = prefix_len;
8778 bcopy(prefix, &IN6_IFEXTRA(ifp)->nat64_prefixes[i].ipv6_prefix,
8779 sizeof(struct in6_addr));
8780 clat_log0((LOG_DEBUG,
8781 "NAT64 prefix set to %s with prefixlen: %d\n",
8782 ip6_sprintf(prefix), prefix_len));
8783 one_set = 1;
8784 }
8785
8786 out:
8787 if_inet6data_lock_done(ifp);
8788
8789 if (error == 0 && one_set != 0)
8790 necp_update_all_clients();
8791
8792 return (error);
8793 }
8794
8795 int
8796 ifnet_get_nat64prefix(struct ifnet *ifp, struct ipv6_prefix *prefixes)
8797 {
8798 int i, found_one = 0, error = 0;
8799
8800 if (ifp == NULL)
8801 return (EINVAL);
8802
8803 if_inet6data_lock_shared(ifp);
8804
8805 if (IN6_IFEXTRA(ifp) == NULL) {
8806 error = ENOMEM;
8807 goto out;
8808 }
8809
8810 for (i = 0; i < NAT64_MAX_NUM_PREFIXES; i++) {
8811 if (IN6_IFEXTRA(ifp)->nat64_prefixes[i].prefix_len != 0)
8812 found_one = 1;
8813 }
8814
8815 if (found_one == 0) {
8816 error = ENOENT;
8817 goto out;
8818 }
8819
8820 if (prefixes)
8821 bcopy(IN6_IFEXTRA(ifp)->nat64_prefixes, prefixes,
8822 sizeof(IN6_IFEXTRA(ifp)->nat64_prefixes));
8823
8824 out:
8825 if_inet6data_lock_done(ifp);
8826
8827 return (error);
8828 }
8829 #endif
8830
8831 static void
8832 dlil_output_cksum_dbg(struct ifnet *ifp, struct mbuf *m, uint32_t hoff,
8833 protocol_family_t pf)
8834 {
8835 #pragma unused(ifp)
8836 uint32_t did_sw;
8837
8838 if (!(hwcksum_dbg_mode & HWCKSUM_DBG_FINALIZE_FORCED) ||
8839 (m->m_pkthdr.csum_flags & (CSUM_TSO_IPV4|CSUM_TSO_IPV6)))
8840 return;
8841
8842 switch (pf) {
8843 case PF_INET:
8844 did_sw = in_finalize_cksum(m, hoff, m->m_pkthdr.csum_flags);
8845 if (did_sw & CSUM_DELAY_IP)
8846 hwcksum_dbg_finalized_hdr++;
8847 if (did_sw & CSUM_DELAY_DATA)
8848 hwcksum_dbg_finalized_data++;
8849 break;
8850 #if INET6
8851 case PF_INET6:
8852 /*
8853 * Checksum offload should not have been enabled when
8854 * extension headers exist; that also means that we
8855 * cannot force-finalize packets with extension headers.
8856 * Indicate to the callee should it skip such case by
8857 * setting optlen to -1.
8858 */
8859 did_sw = in6_finalize_cksum(m, hoff, -1, -1,
8860 m->m_pkthdr.csum_flags);
8861 if (did_sw & CSUM_DELAY_IPV6_DATA)
8862 hwcksum_dbg_finalized_data++;
8863 break;
8864 #endif /* INET6 */
8865 default:
8866 return;
8867 }
8868 }
8869
8870 static void
8871 dlil_input_cksum_dbg(struct ifnet *ifp, struct mbuf *m, char *frame_header,
8872 protocol_family_t pf)
8873 {
8874 uint16_t sum = 0;
8875 uint32_t hlen;
8876
8877 if (frame_header == NULL ||
8878 frame_header < (char *)mbuf_datastart(m) ||
8879 frame_header > (char *)m->m_data) {
8880 printf("%s: frame header pointer 0x%llx out of range "
8881 "[0x%llx,0x%llx] for mbuf 0x%llx\n", if_name(ifp),
8882 (uint64_t)VM_KERNEL_ADDRPERM(frame_header),
8883 (uint64_t)VM_KERNEL_ADDRPERM(mbuf_datastart(m)),
8884 (uint64_t)VM_KERNEL_ADDRPERM(m->m_data),
8885 (uint64_t)VM_KERNEL_ADDRPERM(m));
8886 return;
8887 }
8888 hlen = (m->m_data - frame_header);
8889
8890 switch (pf) {
8891 case PF_INET:
8892 #if INET6
8893 case PF_INET6:
8894 #endif /* INET6 */
8895 break;
8896 default:
8897 return;
8898 }
8899
8900 /*
8901 * Force partial checksum offload; useful to simulate cases
8902 * where the hardware does not support partial checksum offload,
8903 * in order to validate correctness throughout the layers above.
8904 */
8905 if (hwcksum_dbg_mode & HWCKSUM_DBG_PARTIAL_FORCED) {
8906 uint32_t foff = hwcksum_dbg_partial_rxoff_forced;
8907
8908 if (foff > (uint32_t)m->m_pkthdr.len)
8909 return;
8910
8911 m->m_pkthdr.csum_flags &= ~CSUM_RX_FLAGS;
8912
8913 /* Compute 16-bit 1's complement sum from forced offset */
8914 sum = m_sum16(m, foff, (m->m_pkthdr.len - foff));
8915
8916 m->m_pkthdr.csum_flags |= (CSUM_DATA_VALID | CSUM_PARTIAL);
8917 m->m_pkthdr.csum_rx_val = sum;
8918 m->m_pkthdr.csum_rx_start = (foff + hlen);
8919
8920 hwcksum_dbg_partial_forced++;
8921 hwcksum_dbg_partial_forced_bytes += m->m_pkthdr.len;
8922 }
8923
8924 /*
8925 * Partial checksum offload verification (and adjustment);
8926 * useful to validate and test cases where the hardware
8927 * supports partial checksum offload.
8928 */
8929 if ((m->m_pkthdr.csum_flags &
8930 (CSUM_DATA_VALID | CSUM_PARTIAL | CSUM_PSEUDO_HDR)) ==
8931 (CSUM_DATA_VALID | CSUM_PARTIAL)) {
8932 uint32_t rxoff;
8933
8934 /* Start offset must begin after frame header */
8935 rxoff = m->m_pkthdr.csum_rx_start;
8936 if (hlen > rxoff) {
8937 hwcksum_dbg_bad_rxoff++;
8938 if (dlil_verbose) {
8939 printf("%s: partial cksum start offset %d "
8940 "is less than frame header length %d for "
8941 "mbuf 0x%llx\n", if_name(ifp), rxoff, hlen,
8942 (uint64_t)VM_KERNEL_ADDRPERM(m));
8943 }
8944 return;
8945 }
8946 rxoff -= hlen;
8947
8948 if (!(hwcksum_dbg_mode & HWCKSUM_DBG_PARTIAL_FORCED)) {
8949 /*
8950 * Compute the expected 16-bit 1's complement sum;
8951 * skip this if we've already computed it above
8952 * when partial checksum offload is forced.
8953 */
8954 sum = m_sum16(m, rxoff, (m->m_pkthdr.len - rxoff));
8955
8956 /* Hardware or driver is buggy */
8957 if (sum != m->m_pkthdr.csum_rx_val) {
8958 hwcksum_dbg_bad_cksum++;
8959 if (dlil_verbose) {
8960 printf("%s: bad partial cksum value "
8961 "0x%x (expected 0x%x) for mbuf "
8962 "0x%llx [rx_start %d]\n",
8963 if_name(ifp),
8964 m->m_pkthdr.csum_rx_val, sum,
8965 (uint64_t)VM_KERNEL_ADDRPERM(m),
8966 m->m_pkthdr.csum_rx_start);
8967 }
8968 return;
8969 }
8970 }
8971 hwcksum_dbg_verified++;
8972
8973 /*
8974 * This code allows us to emulate various hardwares that
8975 * perform 16-bit 1's complement sum beginning at various
8976 * start offset values.
8977 */
8978 if (hwcksum_dbg_mode & HWCKSUM_DBG_PARTIAL_RXOFF_ADJ) {
8979 uint32_t aoff = hwcksum_dbg_partial_rxoff_adj;
8980
8981 if (aoff == rxoff || aoff > (uint32_t)m->m_pkthdr.len)
8982 return;
8983
8984 sum = m_adj_sum16(m, rxoff, aoff,
8985 m_pktlen(m) - aoff, sum);
8986
8987 m->m_pkthdr.csum_rx_val = sum;
8988 m->m_pkthdr.csum_rx_start = (aoff + hlen);
8989
8990 hwcksum_dbg_adjusted++;
8991 }
8992 }
8993 }
8994
8995 static int
8996 sysctl_hwcksum_dbg_mode SYSCTL_HANDLER_ARGS
8997 {
8998 #pragma unused(arg1, arg2)
8999 u_int32_t i;
9000 int err;
9001
9002 i = hwcksum_dbg_mode;
9003
9004 err = sysctl_handle_int(oidp, &i, 0, req);
9005 if (err != 0 || req->newptr == USER_ADDR_NULL)
9006 return (err);
9007
9008 if (hwcksum_dbg == 0)
9009 return (ENODEV);
9010
9011 if ((i & ~HWCKSUM_DBG_MASK) != 0)
9012 return (EINVAL);
9013
9014 hwcksum_dbg_mode = (i & HWCKSUM_DBG_MASK);
9015
9016 return (err);
9017 }
9018
9019 static int
9020 sysctl_hwcksum_dbg_partial_rxoff_forced SYSCTL_HANDLER_ARGS
9021 {
9022 #pragma unused(arg1, arg2)
9023 u_int32_t i;
9024 int err;
9025
9026 i = hwcksum_dbg_partial_rxoff_forced;
9027
9028 err = sysctl_handle_int(oidp, &i, 0, req);
9029 if (err != 0 || req->newptr == USER_ADDR_NULL)
9030 return (err);
9031
9032 if (!(hwcksum_dbg_mode & HWCKSUM_DBG_PARTIAL_FORCED))
9033 return (ENODEV);
9034
9035 hwcksum_dbg_partial_rxoff_forced = i;
9036
9037 return (err);
9038 }
9039
9040 static int
9041 sysctl_hwcksum_dbg_partial_rxoff_adj SYSCTL_HANDLER_ARGS
9042 {
9043 #pragma unused(arg1, arg2)
9044 u_int32_t i;
9045 int err;
9046
9047 i = hwcksum_dbg_partial_rxoff_adj;
9048
9049 err = sysctl_handle_int(oidp, &i, 0, req);
9050 if (err != 0 || req->newptr == USER_ADDR_NULL)
9051 return (err);
9052
9053 if (!(hwcksum_dbg_mode & HWCKSUM_DBG_PARTIAL_RXOFF_ADJ))
9054 return (ENODEV);
9055
9056 hwcksum_dbg_partial_rxoff_adj = i;
9057
9058 return (err);
9059 }
9060
9061 static int
9062 sysctl_tx_chain_len_stats SYSCTL_HANDLER_ARGS
9063 {
9064 #pragma unused(oidp, arg1, arg2)
9065 int err;
9066
9067 if (req->oldptr == USER_ADDR_NULL) {
9068
9069 }
9070 if (req->newptr != USER_ADDR_NULL) {
9071 return (EPERM);
9072 }
9073 err = SYSCTL_OUT(req, &tx_chain_len_stats,
9074 sizeof(struct chain_len_stats));
9075
9076 return (err);
9077 }
9078
9079
9080 #if DEBUG || DEVELOPMENT
9081 /* Blob for sum16 verification */
9082 static uint8_t sumdata[] = {
9083 0x1f, 0x8b, 0x08, 0x08, 0x4c, 0xe5, 0x9a, 0x4f, 0x00, 0x03,
9084 0x5f, 0x00, 0x5d, 0x91, 0x41, 0x4e, 0xc4, 0x30, 0x0c, 0x45,
9085 0xf7, 0x9c, 0xc2, 0x07, 0x18, 0xf5, 0x0e, 0xb0, 0xe2, 0x00,
9086 0x48, 0x88, 0xa5, 0xdb, 0xba, 0x49, 0x34, 0x69, 0xdc, 0x71,
9087 0x92, 0xa9, 0xc2, 0x8a, 0x6b, 0x70, 0x3d, 0x4e, 0x82, 0x93,
9088 0xb4, 0x08, 0xd8, 0xc5, 0xb1, 0xfd, 0xff, 0xb3, 0xfd, 0x4c,
9089 0x42, 0x5f, 0x1f, 0x9f, 0x11, 0x12, 0x43, 0xb2, 0x04, 0x93,
9090 0xe0, 0x7b, 0x01, 0x0e, 0x14, 0x07, 0x78, 0xd1, 0x78, 0x75,
9091 0x71, 0x71, 0xe9, 0x08, 0x84, 0x46, 0xf2, 0xc7, 0x3b, 0x09,
9092 0xe7, 0xd1, 0xd3, 0x8a, 0x57, 0x92, 0x33, 0xcd, 0x39, 0xcc,
9093 0xb0, 0x91, 0x89, 0xe0, 0x42, 0x53, 0x8b, 0xb7, 0x8c, 0x42,
9094 0x60, 0xd9, 0x9f, 0x7a, 0x55, 0x19, 0x76, 0xcb, 0x10, 0x49,
9095 0x35, 0xac, 0x0b, 0x5a, 0x3c, 0xbb, 0x65, 0x51, 0x8c, 0x90,
9096 0x7c, 0x69, 0x45, 0x45, 0x81, 0xb4, 0x2b, 0x70, 0x82, 0x85,
9097 0x55, 0x91, 0x17, 0x90, 0xdc, 0x14, 0x1e, 0x35, 0x52, 0xdd,
9098 0x02, 0x16, 0xef, 0xb5, 0x40, 0x89, 0xe2, 0x46, 0x53, 0xad,
9099 0x93, 0x6e, 0x98, 0x30, 0xe5, 0x08, 0xb7, 0xcc, 0x03, 0xbc,
9100 0x71, 0x86, 0x09, 0x43, 0x0d, 0x52, 0xf5, 0xa2, 0xf5, 0xa2,
9101 0x56, 0x11, 0x8d, 0xa8, 0xf5, 0xee, 0x92, 0x3d, 0xfe, 0x8c,
9102 0x67, 0x71, 0x8b, 0x0e, 0x2d, 0x70, 0x77, 0xbe, 0xbe, 0xea,
9103 0xbf, 0x9a, 0x8d, 0x9c, 0x53, 0x53, 0xe5, 0xe0, 0x4b, 0x87,
9104 0x85, 0xd2, 0x45, 0x95, 0x30, 0xc1, 0xcc, 0xe0, 0x74, 0x54,
9105 0x13, 0x58, 0xe8, 0xe8, 0x79, 0xa2, 0x09, 0x73, 0xa4, 0x0e,
9106 0x39, 0x59, 0x0c, 0xe6, 0x9c, 0xb2, 0x4f, 0x06, 0x5b, 0x8e,
9107 0xcd, 0x17, 0x6c, 0x5e, 0x95, 0x4d, 0x70, 0xa2, 0x0a, 0xbf,
9108 0xa3, 0xcc, 0x03, 0xbc, 0x5a, 0xe7, 0x75, 0x06, 0x5e, 0x75,
9109 0xef, 0x58, 0x8e, 0x15, 0xd1, 0x0a, 0x18, 0xff, 0xdd, 0xe6,
9110 0x02, 0x3b, 0xb5, 0xb4, 0xa1, 0xe0, 0x72, 0xfc, 0xe3, 0xab,
9111 0x07, 0xe0, 0x4d, 0x65, 0xea, 0x92, 0xeb, 0xf2, 0x7b, 0x17,
9112 0x05, 0xce, 0xc6, 0xf6, 0x2b, 0xbb, 0x70, 0x3d, 0x00, 0x95,
9113 0xe0, 0x07, 0x52, 0x3b, 0x58, 0xfc, 0x7c, 0x69, 0x4d, 0xe9,
9114 0xf7, 0xa9, 0x66, 0x1e, 0x1e, 0xbe, 0x01, 0x69, 0x98, 0xfe,
9115 0xc8, 0x28, 0x02, 0x00, 0x00
9116 };
9117
9118 /* Precomputed 16-bit 1's complement sums for various spans of the above data */
9119 static struct {
9120 boolean_t init;
9121 uint16_t len;
9122 uint16_t sumr; /* reference */
9123 uint16_t sumrp; /* reference, precomputed */
9124 } sumtbl[] = {
9125 { FALSE, 0, 0, 0x0000 },
9126 { FALSE, 1, 0, 0x001f },
9127 { FALSE, 2, 0, 0x8b1f },
9128 { FALSE, 3, 0, 0x8b27 },
9129 { FALSE, 7, 0, 0x790e },
9130 { FALSE, 11, 0, 0xcb6d },
9131 { FALSE, 20, 0, 0x20dd },
9132 { FALSE, 27, 0, 0xbabd },
9133 { FALSE, 32, 0, 0xf3e8 },
9134 { FALSE, 37, 0, 0x197d },
9135 { FALSE, 43, 0, 0x9eae },
9136 { FALSE, 64, 0, 0x4678 },
9137 { FALSE, 127, 0, 0x9399 },
9138 { FALSE, 256, 0, 0xd147 },
9139 { FALSE, 325, 0, 0x0358 },
9140 };
9141 #define SUMTBL_MAX ((int)sizeof (sumtbl) / (int)sizeof (sumtbl[0]))
9142
9143 static void
9144 dlil_verify_sum16(void)
9145 {
9146 struct mbuf *m;
9147 uint8_t *buf;
9148 int n;
9149
9150 /* Make sure test data plus extra room for alignment fits in cluster */
9151 _CASSERT((sizeof (sumdata) + (sizeof (uint64_t) * 2)) <= MCLBYTES);
9152
9153 kprintf("DLIL: running SUM16 self-tests ... ");
9154
9155 m = m_getcl(M_WAITOK, MT_DATA, M_PKTHDR);
9156 m_align(m, sizeof(sumdata) + (sizeof (uint64_t) * 2));
9157
9158 buf = mtod(m, uint8_t *); /* base address */
9159
9160 for (n = 0; n < SUMTBL_MAX; n++) {
9161 uint16_t len = sumtbl[n].len;
9162 int i;
9163
9164 /* Verify for all possible alignments */
9165 for (i = 0; i < (int)sizeof (uint64_t); i++) {
9166 uint16_t sum, sumr;
9167 uint8_t *c;
9168
9169 /* Copy over test data to mbuf */
9170 VERIFY(len <= sizeof (sumdata));
9171 c = buf + i;
9172 bcopy(sumdata, c, len);
9173
9174 /* Zero-offset test (align by data pointer) */
9175 m->m_data = (caddr_t)c;
9176 m->m_len = len;
9177 sum = m_sum16(m, 0, len);
9178
9179 if (!sumtbl[n].init) {
9180 sumr = in_cksum_mbuf_ref(m, len, 0, 0);
9181 sumtbl[n].sumr = sumr;
9182 sumtbl[n].init = TRUE;
9183 } else {
9184 sumr = sumtbl[n].sumr;
9185 }
9186
9187 /* Something is horribly broken; stop now */
9188 if (sumr != sumtbl[n].sumrp) {
9189 panic_plain("\n%s: broken in_cksum_mbuf_ref() "
9190 "for len=%d align=%d sum=0x%04x "
9191 "[expected=0x%04x]\n", __func__,
9192 len, i, sum, sumr);
9193 /* NOTREACHED */
9194 } else if (sum != sumr) {
9195 panic_plain("\n%s: broken m_sum16() for len=%d "
9196 "align=%d sum=0x%04x [expected=0x%04x]\n",
9197 __func__, len, i, sum, sumr);
9198 /* NOTREACHED */
9199 }
9200
9201 /* Alignment test by offset (fixed data pointer) */
9202 m->m_data = (caddr_t)buf;
9203 m->m_len = i + len;
9204 sum = m_sum16(m, i, len);
9205
9206 /* Something is horribly broken; stop now */
9207 if (sum != sumr) {
9208 panic_plain("\n%s: broken m_sum16() for len=%d "
9209 "offset=%d sum=0x%04x [expected=0x%04x]\n",
9210 __func__, len, i, sum, sumr);
9211 /* NOTREACHED */
9212 }
9213 #if INET
9214 /* Simple sum16 contiguous buffer test by aligment */
9215 sum = b_sum16(c, len);
9216
9217 /* Something is horribly broken; stop now */
9218 if (sum != sumr) {
9219 panic_plain("\n%s: broken b_sum16() for len=%d "
9220 "align=%d sum=0x%04x [expected=0x%04x]\n",
9221 __func__, len, i, sum, sumr);
9222 /* NOTREACHED */
9223 }
9224 #endif /* INET */
9225 }
9226 }
9227 m_freem(m);
9228
9229 kprintf("PASSED\n");
9230 }
9231 #endif /* DEBUG || DEVELOPMENT */
9232
9233 #define CASE_STRINGIFY(x) case x: return #x
9234
9235 __private_extern__ const char *
9236 dlil_kev_dl_code_str(u_int32_t event_code)
9237 {
9238 switch (event_code) {
9239 CASE_STRINGIFY(KEV_DL_SIFFLAGS);
9240 CASE_STRINGIFY(KEV_DL_SIFMETRICS);
9241 CASE_STRINGIFY(KEV_DL_SIFMTU);
9242 CASE_STRINGIFY(KEV_DL_SIFPHYS);
9243 CASE_STRINGIFY(KEV_DL_SIFMEDIA);
9244 CASE_STRINGIFY(KEV_DL_SIFGENERIC);
9245 CASE_STRINGIFY(KEV_DL_ADDMULTI);
9246 CASE_STRINGIFY(KEV_DL_DELMULTI);
9247 CASE_STRINGIFY(KEV_DL_IF_ATTACHED);
9248 CASE_STRINGIFY(KEV_DL_IF_DETACHING);
9249 CASE_STRINGIFY(KEV_DL_IF_DETACHED);
9250 CASE_STRINGIFY(KEV_DL_LINK_OFF);
9251 CASE_STRINGIFY(KEV_DL_LINK_ON);
9252 CASE_STRINGIFY(KEV_DL_PROTO_ATTACHED);
9253 CASE_STRINGIFY(KEV_DL_PROTO_DETACHED);
9254 CASE_STRINGIFY(KEV_DL_LINK_ADDRESS_CHANGED);
9255 CASE_STRINGIFY(KEV_DL_WAKEFLAGS_CHANGED);
9256 CASE_STRINGIFY(KEV_DL_IF_IDLE_ROUTE_REFCNT);
9257 CASE_STRINGIFY(KEV_DL_IFCAP_CHANGED);
9258 CASE_STRINGIFY(KEV_DL_LINK_QUALITY_METRIC_CHANGED);
9259 CASE_STRINGIFY(KEV_DL_NODE_PRESENCE);
9260 CASE_STRINGIFY(KEV_DL_NODE_ABSENCE);
9261 CASE_STRINGIFY(KEV_DL_MASTER_ELECTED);
9262 CASE_STRINGIFY(KEV_DL_ISSUES);
9263 CASE_STRINGIFY(KEV_DL_IFDELEGATE_CHANGED);
9264 default:
9265 break;
9266 }
9267 return ("");
9268 }
9269
9270 static void
9271 dlil_dt_tcall_fn(thread_call_param_t arg0, thread_call_param_t arg1)
9272 {
9273 #pragma unused(arg1)
9274 struct ifnet *ifp = arg0;
9275
9276 if (ifnet_is_attached(ifp, 1)) {
9277 nstat_ifnet_threshold_reached(ifp->if_index);
9278 ifnet_decr_iorefcnt(ifp);
9279 }
9280 }
9281
9282 void
9283 ifnet_notify_data_threshold(struct ifnet *ifp)
9284 {
9285 uint64_t bytes = (ifp->if_ibytes + ifp->if_obytes);
9286 uint64_t oldbytes = ifp->if_dt_bytes;
9287
9288 ASSERT(ifp->if_dt_tcall != NULL);
9289
9290 /*
9291 * If we went over the threshold, notify NetworkStatistics.
9292 * We rate-limit it based on the threshold interval value.
9293 */
9294 if (threshold_notify && (bytes - oldbytes) > ifp->if_data_threshold &&
9295 OSCompareAndSwap64(oldbytes, bytes, &ifp->if_dt_bytes) &&
9296 !thread_call_isactive(ifp->if_dt_tcall)) {
9297 uint64_t tival = (threshold_interval * NSEC_PER_SEC);
9298 uint64_t now = mach_absolute_time(), deadline = now;
9299 uint64_t ival;
9300
9301 if (tival != 0) {
9302 nanoseconds_to_absolutetime(tival, &ival);
9303 clock_deadline_for_periodic_event(ival, now, &deadline);
9304 (void) thread_call_enter_delayed(ifp->if_dt_tcall,
9305 deadline);
9306 } else {
9307 (void) thread_call_enter(ifp->if_dt_tcall);
9308 }
9309 }
9310 }
9311
9312 #if (DEVELOPMENT || DEBUG)
9313 /*
9314 * The sysctl variable name contains the input parameters of
9315 * ifnet_get_keepalive_offload_frames()
9316 * ifp (interface index): name[0]
9317 * frames_array_count: name[1]
9318 * frame_data_offset: name[2]
9319 * The return length gives used_frames_count
9320 */
9321 static int
9322 sysctl_get_kao_frames SYSCTL_HANDLER_ARGS
9323 {
9324 #pragma unused(oidp)
9325 int *name = (int *)arg1;
9326 u_int namelen = arg2;
9327 int idx;
9328 ifnet_t ifp = NULL;
9329 u_int32_t frames_array_count;
9330 size_t frame_data_offset;
9331 u_int32_t used_frames_count;
9332 struct ifnet_keepalive_offload_frame *frames_array = NULL;
9333 int error = 0;
9334 u_int32_t i;
9335
9336 /*
9337 * Only root can get look at other people TCP frames
9338 */
9339 error = proc_suser(current_proc());
9340 if (error != 0)
9341 goto done;
9342 /*
9343 * Validate the input parameters
9344 */
9345 if (req->newptr != USER_ADDR_NULL) {
9346 error = EPERM;
9347 goto done;
9348 }
9349 if (namelen != 3) {
9350 error = EINVAL;
9351 goto done;
9352 }
9353 if (req->oldptr == USER_ADDR_NULL) {
9354 error = EINVAL;
9355 goto done;
9356 }
9357 if (req->oldlen == 0) {
9358 error = EINVAL;
9359 goto done;
9360 }
9361 idx = name[0];
9362 frames_array_count = name[1];
9363 frame_data_offset = name[2];
9364
9365 /* Make sure the passed buffer is large enough */
9366 if (frames_array_count * sizeof(struct ifnet_keepalive_offload_frame) >
9367 req->oldlen) {
9368 error = ENOMEM;
9369 goto done;
9370 }
9371
9372 ifnet_head_lock_shared();
9373 if (!IF_INDEX_IN_RANGE(idx)) {
9374 ifnet_head_done();
9375 error = ENOENT;
9376 goto done;
9377 }
9378 ifp = ifindex2ifnet[idx];
9379 ifnet_head_done();
9380
9381 frames_array = _MALLOC(frames_array_count *
9382 sizeof(struct ifnet_keepalive_offload_frame), M_TEMP, M_WAITOK);
9383 if (frames_array == NULL) {
9384 error = ENOMEM;
9385 goto done;
9386 }
9387
9388 error = ifnet_get_keepalive_offload_frames(ifp, frames_array,
9389 frames_array_count, frame_data_offset, &used_frames_count);
9390 if (error != 0) {
9391 printf("%s: ifnet_get_keepalive_offload_frames error %d\n",
9392 __func__, error);
9393 goto done;
9394 }
9395
9396 for (i = 0; i < used_frames_count; i++) {
9397 error = SYSCTL_OUT(req, frames_array + i,
9398 sizeof(struct ifnet_keepalive_offload_frame));
9399 if (error != 0) {
9400 goto done;
9401 }
9402 }
9403 done:
9404 if (frames_array != NULL)
9405 _FREE(frames_array, M_TEMP);
9406 return (error);
9407 }
9408 #endif /* DEVELOPMENT || DEBUG */
9409
9410 void
9411 ifnet_update_stats_per_flow(struct ifnet_stats_per_flow *ifs,
9412 struct ifnet *ifp)
9413 {
9414 tcp_update_stats_per_flow(ifs, ifp);
9415 }
9416
9417 static void
9418 dlil_mit_tcall_fn(thread_call_param_t arg0, thread_call_param_t arg1)
9419 {
9420 #pragma unused(arg1)
9421 struct ifnet *ifp = (struct ifnet *)arg0;
9422 struct dlil_threading_info *inp = ifp->if_inp;
9423
9424 ifnet_lock_shared(ifp);
9425 if (!IF_FULLY_ATTACHED(ifp) || inp == NULL) {
9426 ifnet_lock_done(ifp);
9427 return;
9428 }
9429
9430 lck_mtx_lock_spin(&inp->input_lck);
9431 inp->input_waiting |= DLIL_INPUT_WAITING;
9432 if (!(inp->input_waiting & DLIL_INPUT_RUNNING) ||
9433 !qempty(&inp->rcvq_pkts)) {
9434 inp->wtot++;
9435 wakeup_one((caddr_t)&inp->input_waiting);
9436 }
9437 lck_mtx_unlock(&inp->input_lck);
9438 ifnet_lock_done(ifp);
9439 }
mirror of XNU