]> git.saurik.com Git - apple/xnu.git/blob - bsd/net/dlil.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-2422.90.20.tar.gz
[apple/xnu.git] / bsd / net / dlil.c
1 /*
2 * Copyright (c) 1999-2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
30 * support for mandatory and extensible security protections. This notice
31 * is included in support of clause 2.2 (b) of the Apple Public License,
32 * Version 2.0.
33 */
34 #include <stddef.h>
35
36 #include <sys/param.h>
37 #include <sys/systm.h>
38 #include <sys/kernel.h>
39 #include <sys/malloc.h>
40 #include <sys/mbuf.h>
41 #include <sys/socket.h>
42 #include <sys/domain.h>
43 #include <sys/user.h>
44 #include <sys/random.h>
45 #include <sys/socketvar.h>
46 #include <net/if_dl.h>
47 #include <net/if.h>
48 #include <net/route.h>
49 #include <net/if_var.h>
50 #include <net/dlil.h>
51 #include <net/if_arp.h>
52 #include <net/iptap.h>
53 #include <net/pktap.h>
54 #include <sys/kern_event.h>
55 #include <sys/kdebug.h>
56 #include <sys/mcache.h>
57 #include <sys/syslog.h>
58 #include <sys/protosw.h>
59 #include <sys/priv.h>
60
61 #include <kern/assert.h>
62 #include <kern/task.h>
63 #include <kern/thread.h>
64 #include <kern/sched_prim.h>
65 #include <kern/locks.h>
66 #include <kern/zalloc.h>
67
68 #include <net/kpi_protocol.h>
69 #include <net/if_types.h>
70 #include <net/if_llreach.h>
71 #include <net/kpi_interfacefilter.h>
72 #include <net/classq/classq.h>
73 #include <net/classq/classq_sfb.h>
74 #include <net/flowhash.h>
75 #include <net/ntstat.h>
76
77 #if INET
78 #include <netinet/in_var.h>
79 #include <netinet/igmp_var.h>
80 #include <netinet/ip_var.h>
81 #include <netinet/tcp.h>
82 #include <netinet/tcp_var.h>
83 #include <netinet/udp.h>
84 #include <netinet/udp_var.h>
85 #include <netinet/if_ether.h>
86 #include <netinet/in_pcb.h>
87 #endif /* INET */
88
89 #if INET6
90 #include <netinet6/in6_var.h>
91 #include <netinet6/nd6.h>
92 #include <netinet6/mld6_var.h>
93 #include <netinet6/scope6_var.h>
94 #endif /* INET6 */
95
96 #include <libkern/OSAtomic.h>
97 #include <libkern/tree.h>
98
99 #include <dev/random/randomdev.h>
100 #include <machine/machine_routines.h>
101
102 #include <mach/thread_act.h>
103 #include <mach/sdt.h>
104
105 #if CONFIG_MACF
106 #include <sys/kauth.h>
107 #include <security/mac_framework.h>
108 #include <net/ethernet.h>
109 #include <net/firewire.h>
110 #endif
111
112 #if PF
113 #include <net/pfvar.h>
114 #endif /* PF */
115 #if PF_ALTQ
116 #include <net/altq/altq.h>
117 #endif /* PF_ALTQ */
118 #include <net/pktsched/pktsched.h>
119
120 #define DBG_LAYER_BEG DLILDBG_CODE(DBG_DLIL_STATIC, 0)
121 #define DBG_LAYER_END DLILDBG_CODE(DBG_DLIL_STATIC, 2)
122 #define DBG_FNC_DLIL_INPUT DLILDBG_CODE(DBG_DLIL_STATIC, (1 << 8))
123 #define DBG_FNC_DLIL_OUTPUT DLILDBG_CODE(DBG_DLIL_STATIC, (2 << 8))
124 #define DBG_FNC_DLIL_IFOUT DLILDBG_CODE(DBG_DLIL_STATIC, (3 << 8))
125
126 #define MAX_FRAME_TYPE_SIZE 4 /* LONGWORDS */
127 #define MAX_LINKADDR 4 /* LONGWORDS */
128 #define M_NKE M_IFADDR
129
130 #if 1
131 #define DLIL_PRINTF printf
132 #else
133 #define DLIL_PRINTF kprintf
134 #endif
135
136 #define IF_DATA_REQUIRE_ALIGNED_64(f) \
137 _CASSERT(!(offsetof(struct if_data_internal, f) % sizeof (u_int64_t)))
138
139 #define IFNET_IF_DATA_REQUIRE_ALIGNED_64(f) \
140 _CASSERT(!(offsetof(struct ifnet, if_data.f) % sizeof (u_int64_t)))
141
142 enum {
143 kProtoKPI_v1 = 1,
144 kProtoKPI_v2 = 2
145 };
146
147 /*
148 * List of if_proto structures in if_proto_hash[] is protected by
149 * the ifnet lock. The rest of the fields are initialized at protocol
150 * attach time and never change, thus no lock required as long as
151 * a reference to it is valid, via if_proto_ref().
152 */
153 struct if_proto {
154 SLIST_ENTRY(if_proto) next_hash;
155 u_int32_t refcount;
156 u_int32_t detached;
157 struct ifnet *ifp;
158 protocol_family_t protocol_family;
159 int proto_kpi;
160 union {
161 struct {
162 proto_media_input input;
163 proto_media_preout pre_output;
164 proto_media_event event;
165 proto_media_ioctl ioctl;
166 proto_media_detached detached;
167 proto_media_resolve_multi resolve_multi;
168 proto_media_send_arp send_arp;
169 } v1;
170 struct {
171 proto_media_input_v2 input;
172 proto_media_preout pre_output;
173 proto_media_event event;
174 proto_media_ioctl ioctl;
175 proto_media_detached detached;
176 proto_media_resolve_multi resolve_multi;
177 proto_media_send_arp send_arp;
178 } v2;
179 } kpi;
180 };
181
182 SLIST_HEAD(proto_hash_entry, if_proto);
183
184 #define DLIL_SDLMAXLEN 64
185 #define DLIL_SDLDATALEN \
186 (DLIL_SDLMAXLEN - offsetof(struct sockaddr_dl, sdl_data[0]))
187
188 struct dlil_ifnet {
189 struct ifnet dl_if; /* public ifnet */
190 /*
191 * DLIL private fields, protected by dl_if_lock
192 */
193 decl_lck_mtx_data(, dl_if_lock);
194 TAILQ_ENTRY(dlil_ifnet) dl_if_link; /* dlil_ifnet link */
195 u_int32_t dl_if_flags; /* flags (below) */
196 u_int32_t dl_if_refcnt; /* refcnt */
197 void (*dl_if_trace)(struct dlil_ifnet *, int); /* ref trace callback */
198 void *dl_if_uniqueid; /* unique interface id */
199 size_t dl_if_uniqueid_len; /* length of the unique id */
200 char dl_if_namestorage[IFNAMSIZ]; /* interface name storage */
201 char dl_if_xnamestorage[IFXNAMSIZ]; /* external name storage */
202 struct {
203 struct ifaddr ifa; /* lladdr ifa */
204 u_int8_t asdl[DLIL_SDLMAXLEN]; /* addr storage */
205 u_int8_t msdl[DLIL_SDLMAXLEN]; /* mask storage */
206 } dl_if_lladdr;
207 u_int8_t dl_if_descstorage[IF_DESCSIZE]; /* desc storage */
208 struct dlil_threading_info dl_if_inpstorage; /* input thread storage */
209 ctrace_t dl_if_attach; /* attach PC stacktrace */
210 ctrace_t dl_if_detach; /* detach PC stacktrace */
211 };
212
213 /* Values for dl_if_flags (private to DLIL) */
214 #define DLIF_INUSE 0x1 /* DLIL ifnet recycler, ifnet in use */
215 #define DLIF_REUSE 0x2 /* DLIL ifnet recycles, ifnet is not new */
216 #define DLIF_DEBUG 0x4 /* has debugging info */
217
218 #define IF_REF_TRACE_HIST_SIZE 8 /* size of ref trace history */
219
220 /* For gdb */
221 __private_extern__ unsigned int if_ref_trace_hist_size = IF_REF_TRACE_HIST_SIZE;
222
223 struct dlil_ifnet_dbg {
224 struct dlil_ifnet dldbg_dlif; /* dlil_ifnet */
225 u_int16_t dldbg_if_refhold_cnt; /* # ifnet references */
226 u_int16_t dldbg_if_refrele_cnt; /* # ifnet releases */
227 /*
228 * Circular lists of ifnet_{reference,release} callers.
229 */
230 ctrace_t dldbg_if_refhold[IF_REF_TRACE_HIST_SIZE];
231 ctrace_t dldbg_if_refrele[IF_REF_TRACE_HIST_SIZE];
232 };
233
234 #define DLIL_TO_IFP(s) (&s->dl_if)
235 #define IFP_TO_DLIL(s) ((struct dlil_ifnet *)s)
236
237 struct ifnet_filter {
238 TAILQ_ENTRY(ifnet_filter) filt_next;
239 u_int32_t filt_skip;
240 u_int32_t filt_flags;
241 ifnet_t filt_ifp;
242 const char *filt_name;
243 void *filt_cookie;
244 protocol_family_t filt_protocol;
245 iff_input_func filt_input;
246 iff_output_func filt_output;
247 iff_event_func filt_event;
248 iff_ioctl_func filt_ioctl;
249 iff_detached_func filt_detached;
250 };
251
252 struct proto_input_entry;
253
254 static TAILQ_HEAD(, dlil_ifnet) dlil_ifnet_head;
255 static lck_grp_t *dlil_lock_group;
256 lck_grp_t *ifnet_lock_group;
257 static lck_grp_t *ifnet_head_lock_group;
258 static lck_grp_t *ifnet_snd_lock_group;
259 static lck_grp_t *ifnet_rcv_lock_group;
260 lck_attr_t *ifnet_lock_attr;
261 decl_lck_rw_data(static, ifnet_head_lock);
262 decl_lck_mtx_data(static, dlil_ifnet_lock);
263 u_int32_t dlil_filter_disable_tso_count = 0;
264
265 #if DEBUG
266 static unsigned int ifnet_debug = 1; /* debugging (enabled) */
267 #else
268 static unsigned int ifnet_debug; /* debugging (disabled) */
269 #endif /* !DEBUG */
270 static unsigned int dlif_size; /* size of dlil_ifnet to allocate */
271 static unsigned int dlif_bufsize; /* size of dlif_size + headroom */
272 static struct zone *dlif_zone; /* zone for dlil_ifnet */
273
274 #define DLIF_ZONE_MAX 64 /* maximum elements in zone */
275 #define DLIF_ZONE_NAME "ifnet" /* zone name */
276
277 static unsigned int dlif_filt_size; /* size of ifnet_filter */
278 static struct zone *dlif_filt_zone; /* zone for ifnet_filter */
279
280 #define DLIF_FILT_ZONE_MAX 8 /* maximum elements in zone */
281 #define DLIF_FILT_ZONE_NAME "ifnet_filter" /* zone name */
282
283 static unsigned int dlif_phash_size; /* size of ifnet proto hash table */
284 static struct zone *dlif_phash_zone; /* zone for ifnet proto hash table */
285
286 #define DLIF_PHASH_ZONE_MAX DLIF_ZONE_MAX /* maximum elements in zone */
287 #define DLIF_PHASH_ZONE_NAME "ifnet_proto_hash" /* zone name */
288
289 static unsigned int dlif_proto_size; /* size of if_proto */
290 static struct zone *dlif_proto_zone; /* zone for if_proto */
291
292 #define DLIF_PROTO_ZONE_MAX (DLIF_ZONE_MAX*2) /* maximum elements in zone */
293 #define DLIF_PROTO_ZONE_NAME "ifnet_proto" /* zone name */
294
295 static unsigned int dlif_tcpstat_size; /* size of tcpstat_local to allocate */
296 static unsigned int dlif_tcpstat_bufsize; /* size of dlif_tcpstat_size + headroom */
297 static struct zone *dlif_tcpstat_zone; /* zone for tcpstat_local */
298
299 #define DLIF_TCPSTAT_ZONE_MAX 1 /* maximum elements in zone */
300 #define DLIF_TCPSTAT_ZONE_NAME "ifnet_tcpstat" /* zone name */
301
302 static unsigned int dlif_udpstat_size; /* size of udpstat_local to allocate */
303 static unsigned int dlif_udpstat_bufsize; /* size of dlif_udpstat_size + headroom */
304 static struct zone *dlif_udpstat_zone; /* zone for udpstat_local */
305
306 #define DLIF_UDPSTAT_ZONE_MAX 1 /* maximum elements in zone */
307 #define DLIF_UDPSTAT_ZONE_NAME "ifnet_udpstat" /* zone name */
308
309 /*
310 * Updating this variable should be done by first acquiring the global
311 * radix node head (rnh_lock), in tandem with settting/clearing the
312 * PR_AGGDRAIN for routedomain.
313 */
314 u_int32_t ifnet_aggressive_drainers;
315 static u_int32_t net_rtref;
316
317 static struct dlil_main_threading_info dlil_main_input_thread_info;
318 __private_extern__ struct dlil_threading_info *dlil_main_input_thread =
319 (struct dlil_threading_info *)&dlil_main_input_thread_info;
320
321 static int dlil_event_internal(struct ifnet *ifp, struct kev_msg *msg);
322 static int dlil_detach_filter_internal(interface_filter_t filter, int detached);
323 static void dlil_if_trace(struct dlil_ifnet *, int);
324 static void if_proto_ref(struct if_proto *);
325 static void if_proto_free(struct if_proto *);
326 static struct if_proto *find_attached_proto(struct ifnet *, u_int32_t);
327 static int dlil_ifp_proto_count(struct ifnet *);
328 static void if_flt_monitor_busy(struct ifnet *);
329 static void if_flt_monitor_unbusy(struct ifnet *);
330 static void if_flt_monitor_enter(struct ifnet *);
331 static void if_flt_monitor_leave(struct ifnet *);
332 static int dlil_interface_filters_input(struct ifnet *, struct mbuf **,
333 char **, protocol_family_t);
334 static int dlil_interface_filters_output(struct ifnet *, struct mbuf **,
335 protocol_family_t);
336 static struct ifaddr *dlil_alloc_lladdr(struct ifnet *,
337 const struct sockaddr_dl *);
338 static int ifnet_lookup(struct ifnet *);
339 static void if_purgeaddrs(struct ifnet *);
340
341 static errno_t ifproto_media_input_v1(struct ifnet *, protocol_family_t,
342 struct mbuf *, char *);
343 static errno_t ifproto_media_input_v2(struct ifnet *, protocol_family_t,
344 struct mbuf *);
345 static errno_t ifproto_media_preout(struct ifnet *, protocol_family_t,
346 mbuf_t *, const struct sockaddr *, void *, char *, char *);
347 static void ifproto_media_event(struct ifnet *, protocol_family_t,
348 const struct kev_msg *);
349 static errno_t ifproto_media_ioctl(struct ifnet *, protocol_family_t,
350 unsigned long, void *);
351 static errno_t ifproto_media_resolve_multi(ifnet_t, const struct sockaddr *,
352 struct sockaddr_dl *, size_t);
353 static errno_t ifproto_media_send_arp(struct ifnet *, u_short,
354 const struct sockaddr_dl *, const struct sockaddr *,
355 const struct sockaddr_dl *, const struct sockaddr *);
356
357 static errno_t ifp_if_output(struct ifnet *, struct mbuf *);
358 static void ifp_if_start(struct ifnet *);
359 static void ifp_if_input_poll(struct ifnet *, u_int32_t, u_int32_t,
360 struct mbuf **, struct mbuf **, u_int32_t *, u_int32_t *);
361 static errno_t ifp_if_ctl(struct ifnet *, ifnet_ctl_cmd_t, u_int32_t, void *);
362 static errno_t ifp_if_demux(struct ifnet *, struct mbuf *, char *,
363 protocol_family_t *);
364 static errno_t ifp_if_add_proto(struct ifnet *, protocol_family_t,
365 const struct ifnet_demux_desc *, u_int32_t);
366 static errno_t ifp_if_del_proto(struct ifnet *, protocol_family_t);
367 static errno_t ifp_if_check_multi(struct ifnet *, const struct sockaddr *);
368 static errno_t ifp_if_framer(struct ifnet *, struct mbuf **,
369 const struct sockaddr *, const char *, const char *);
370 static errno_t ifp_if_framer_extended(struct ifnet *, struct mbuf **,
371 const struct sockaddr *, const char *, const char *,
372 u_int32_t *, u_int32_t *);
373 static errno_t ifp_if_set_bpf_tap(struct ifnet *, bpf_tap_mode, bpf_packet_func);
374 static void ifp_if_free(struct ifnet *);
375 static void ifp_if_event(struct ifnet *, const struct kev_msg *);
376 static __inline void ifp_inc_traffic_class_in(struct ifnet *, struct mbuf *);
377 static __inline void ifp_inc_traffic_class_out(struct ifnet *, struct mbuf *);
378
379 static void dlil_main_input_thread_func(void *, wait_result_t);
380 static void dlil_input_thread_func(void *, wait_result_t);
381 static void dlil_rxpoll_input_thread_func(void *, wait_result_t);
382 static int dlil_create_input_thread(ifnet_t, struct dlil_threading_info *);
383 static void dlil_terminate_input_thread(struct dlil_threading_info *);
384 static void dlil_input_stats_add(const struct ifnet_stat_increment_param *,
385 struct dlil_threading_info *, boolean_t);
386 static void dlil_input_stats_sync(struct ifnet *, struct dlil_threading_info *);
387 static void dlil_input_packet_list_common(struct ifnet *, struct mbuf *,
388 u_int32_t, ifnet_model_t, boolean_t);
389 static errno_t ifnet_input_common(struct ifnet *, struct mbuf *, struct mbuf *,
390 const struct ifnet_stat_increment_param *, boolean_t, boolean_t);
391
392 #if DEBUG
393 static void dlil_verify_sum16(void);
394 #endif /* DEBUG */
395 static void dlil_output_cksum_dbg(struct ifnet *, struct mbuf *, uint32_t,
396 protocol_family_t);
397 static void dlil_input_cksum_dbg(struct ifnet *, struct mbuf *, char *,
398 protocol_family_t);
399
400 static void ifnet_detacher_thread_func(void *, wait_result_t);
401 static int ifnet_detacher_thread_cont(int);
402 static void ifnet_detach_final(struct ifnet *);
403 static void ifnet_detaching_enqueue(struct ifnet *);
404 static struct ifnet *ifnet_detaching_dequeue(void);
405
406 static void ifnet_start_thread_fn(void *, wait_result_t);
407 static void ifnet_poll_thread_fn(void *, wait_result_t);
408 static void ifnet_poll(struct ifnet *);
409
410 static void ifp_src_route_copyout(struct ifnet *, struct route *);
411 static void ifp_src_route_copyin(struct ifnet *, struct route *);
412 #if INET6
413 static void ifp_src_route6_copyout(struct ifnet *, struct route_in6 *);
414 static void ifp_src_route6_copyin(struct ifnet *, struct route_in6 *);
415 #endif /* INET6 */
416
417 static int sysctl_rxpoll SYSCTL_HANDLER_ARGS;
418 static int sysctl_rxpoll_mode_holdtime SYSCTL_HANDLER_ARGS;
419 static int sysctl_rxpoll_sample_holdtime SYSCTL_HANDLER_ARGS;
420 static int sysctl_rxpoll_interval_time SYSCTL_HANDLER_ARGS;
421 static int sysctl_rxpoll_wlowat SYSCTL_HANDLER_ARGS;
422 static int sysctl_rxpoll_whiwat SYSCTL_HANDLER_ARGS;
423 static int sysctl_sndq_maxlen SYSCTL_HANDLER_ARGS;
424 static int sysctl_rcvq_maxlen SYSCTL_HANDLER_ARGS;
425 static int sysctl_hwcksum_dbg_mode SYSCTL_HANDLER_ARGS;
426 static int sysctl_hwcksum_dbg_partial_rxoff_forced SYSCTL_HANDLER_ARGS;
427 static int sysctl_hwcksum_dbg_partial_rxoff_adj SYSCTL_HANDLER_ARGS;
428
429 /* The following are protected by dlil_ifnet_lock */
430 static TAILQ_HEAD(, ifnet) ifnet_detaching_head;
431 static u_int32_t ifnet_detaching_cnt;
432 static void *ifnet_delayed_run; /* wait channel for detaching thread */
433
434 decl_lck_mtx_data(static, ifnet_fc_lock);
435
436 static uint32_t ifnet_flowhash_seed;
437
438 struct ifnet_flowhash_key {
439 char ifk_name[IFNAMSIZ];
440 uint32_t ifk_unit;
441 uint32_t ifk_flags;
442 uint32_t ifk_eflags;
443 uint32_t ifk_capabilities;
444 uint32_t ifk_capenable;
445 uint32_t ifk_output_sched_model;
446 uint32_t ifk_rand1;
447 uint32_t ifk_rand2;
448 };
449
450 /* Flow control entry per interface */
451 struct ifnet_fc_entry {
452 RB_ENTRY(ifnet_fc_entry) ifce_entry;
453 u_int32_t ifce_flowhash;
454 struct ifnet *ifce_ifp;
455 };
456
457 static uint32_t ifnet_calc_flowhash(struct ifnet *);
458 static int ifce_cmp(const struct ifnet_fc_entry *,
459 const struct ifnet_fc_entry *);
460 static int ifnet_fc_add(struct ifnet *);
461 static struct ifnet_fc_entry *ifnet_fc_get(u_int32_t);
462 static void ifnet_fc_entry_free(struct ifnet_fc_entry *);
463
464 /* protected by ifnet_fc_lock */
465 RB_HEAD(ifnet_fc_tree, ifnet_fc_entry) ifnet_fc_tree;
466 RB_PROTOTYPE(ifnet_fc_tree, ifnet_fc_entry, ifce_entry, ifce_cmp);
467 RB_GENERATE(ifnet_fc_tree, ifnet_fc_entry, ifce_entry, ifce_cmp);
468
469 static unsigned int ifnet_fc_zone_size; /* sizeof ifnet_fc_entry */
470 static struct zone *ifnet_fc_zone; /* ifnet_fc_entry zone */
471
472 #define IFNET_FC_ZONE_NAME "ifnet_fc_zone"
473 #define IFNET_FC_ZONE_MAX 32
474
475 extern void bpfdetach(struct ifnet*);
476 extern void proto_input_run(void);
477
478 extern uint32_t udp_count_opportunistic(unsigned int ifindex,
479 u_int32_t flags);
480 extern uint32_t tcp_count_opportunistic(unsigned int ifindex,
481 u_int32_t flags);
482
483 __private_extern__ void link_rtrequest(int, struct rtentry *, struct sockaddr *);
484
485 #if CONFIG_MACF
486 int dlil_lladdr_ckreq = 0;
487 #endif
488
489 #if DEBUG
490 int dlil_verbose = 1;
491 #else
492 int dlil_verbose = 0;
493 #endif /* DEBUG */
494 #if IFNET_INPUT_SANITY_CHK
495 /* sanity checking of input packet lists received */
496 static u_int32_t dlil_input_sanity_check = 0;
497 #endif /* IFNET_INPUT_SANITY_CHK */
498 /* rate limit debug messages */
499 struct timespec dlil_dbgrate = { 1, 0 };
500
501 SYSCTL_DECL(_net_link_generic_system);
502
503 #if CONFIG_MACF
504 SYSCTL_INT(_net_link_generic_system, OID_AUTO, dlil_lladdr_ckreq,
505 CTLFLAG_RW | CTLFLAG_LOCKED, &dlil_lladdr_ckreq, 0,
506 "Require MACF system info check to expose link-layer address");
507 #endif
508
509 SYSCTL_INT(_net_link_generic_system, OID_AUTO, dlil_verbose,
510 CTLFLAG_RW | CTLFLAG_LOCKED, &dlil_verbose, 0, "Log DLIL error messages");
511
512 #define IF_SNDQ_MINLEN 32
513 u_int32_t if_sndq_maxlen = IFQ_MAXLEN;
514 SYSCTL_PROC(_net_link_generic_system, OID_AUTO, sndq_maxlen,
515 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &if_sndq_maxlen, IFQ_MAXLEN,
516 sysctl_sndq_maxlen, "I", "Default transmit queue max length");
517
518 #define IF_RCVQ_MINLEN 32
519 #define IF_RCVQ_MAXLEN 256
520 u_int32_t if_rcvq_maxlen = IF_RCVQ_MAXLEN;
521 SYSCTL_PROC(_net_link_generic_system, OID_AUTO, rcvq_maxlen,
522 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &if_rcvq_maxlen, IFQ_MAXLEN,
523 sysctl_rcvq_maxlen, "I", "Default receive queue max length");
524
525 #define IF_RXPOLL_DECAY 2 /* ilog2 of EWMA decay rate (4) */
526 static u_int32_t if_rxpoll_decay = IF_RXPOLL_DECAY;
527 SYSCTL_UINT(_net_link_generic_system, OID_AUTO, rxpoll_decay,
528 CTLFLAG_RW | CTLFLAG_LOCKED, &if_rxpoll_decay, IF_RXPOLL_DECAY,
529 "ilog2 of EWMA decay rate of avg inbound packets");
530
531 #define IF_RXPOLL_MODE_HOLDTIME_MIN (10ULL * 1000 * 1000) /* 10 ms */
532 #define IF_RXPOLL_MODE_HOLDTIME (1000ULL * 1000 * 1000) /* 1 sec */
533 static u_int64_t if_rxpoll_mode_holdtime = IF_RXPOLL_MODE_HOLDTIME;
534 SYSCTL_PROC(_net_link_generic_system, OID_AUTO, rxpoll_freeze_time,
535 CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED, &if_rxpoll_mode_holdtime,
536 IF_RXPOLL_MODE_HOLDTIME, sysctl_rxpoll_mode_holdtime,
537 "Q", "input poll mode freeze time");
538
539 #define IF_RXPOLL_SAMPLETIME_MIN (1ULL * 1000 * 1000) /* 1 ms */
540 #define IF_RXPOLL_SAMPLETIME (10ULL * 1000 * 1000) /* 10 ms */
541 static u_int64_t if_rxpoll_sample_holdtime = IF_RXPOLL_SAMPLETIME;
542 SYSCTL_PROC(_net_link_generic_system, OID_AUTO, rxpoll_sample_time,
543 CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED, &if_rxpoll_sample_holdtime,
544 IF_RXPOLL_SAMPLETIME, sysctl_rxpoll_sample_holdtime,
545 "Q", "input poll sampling time");
546
547 #define IF_RXPOLL_INTERVALTIME_MIN (1ULL * 1000) /* 1 us */
548 #define IF_RXPOLL_INTERVALTIME (1ULL * 1000 * 1000) /* 1 ms */
549 static u_int64_t if_rxpoll_interval_time = IF_RXPOLL_INTERVALTIME;
550 SYSCTL_PROC(_net_link_generic_system, OID_AUTO, rxpoll_interval_time,
551 CTLTYPE_QUAD | CTLFLAG_RW | CTLFLAG_LOCKED, &if_rxpoll_interval_time,
552 IF_RXPOLL_INTERVALTIME, sysctl_rxpoll_interval_time,
553 "Q", "input poll interval (time)");
554
555 #define IF_RXPOLL_INTERVAL_PKTS 0 /* 0 (disabled) */
556 static u_int32_t if_rxpoll_interval_pkts = IF_RXPOLL_INTERVAL_PKTS;
557 SYSCTL_UINT(_net_link_generic_system, OID_AUTO, rxpoll_interval_pkts,
558 CTLFLAG_RW | CTLFLAG_LOCKED, &if_rxpoll_interval_pkts,
559 IF_RXPOLL_INTERVAL_PKTS, "input poll interval (packets)");
560
561 #define IF_RXPOLL_WLOWAT 10
562 static u_int32_t if_rxpoll_wlowat = IF_RXPOLL_WLOWAT;
563 SYSCTL_PROC(_net_link_generic_system, OID_AUTO, rxpoll_wakeups_lowat,
564 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &if_rxpoll_wlowat,
565 IF_RXPOLL_WLOWAT, sysctl_rxpoll_wlowat,
566 "I", "input poll wakeup low watermark");
567
568 #define IF_RXPOLL_WHIWAT 100
569 static u_int32_t if_rxpoll_whiwat = IF_RXPOLL_WHIWAT;
570 SYSCTL_PROC(_net_link_generic_system, OID_AUTO, rxpoll_wakeups_hiwat,
571 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &if_rxpoll_whiwat,
572 IF_RXPOLL_WHIWAT, sysctl_rxpoll_whiwat,
573 "I", "input poll wakeup high watermark");
574
575 static u_int32_t if_rxpoll_max = 0; /* 0 (automatic) */
576 SYSCTL_UINT(_net_link_generic_system, OID_AUTO, rxpoll_max,
577 CTLFLAG_RW | CTLFLAG_LOCKED, &if_rxpoll_max, 0,
578 "max packets per poll call");
579
580 static u_int32_t if_rxpoll = 1;
581 SYSCTL_PROC(_net_link_generic_system, OID_AUTO, rxpoll,
582 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &if_rxpoll, 0,
583 sysctl_rxpoll, "I", "enable opportunistic input polling");
584
585 u_int32_t if_bw_smoothing_val = 3;
586 SYSCTL_UINT(_net_link_generic_system, OID_AUTO, if_bw_smoothing_val,
587 CTLFLAG_RW | CTLFLAG_LOCKED, &if_bw_smoothing_val, 0, "");
588
589 u_int32_t if_bw_measure_size = 10;
590 SYSCTL_INT(_net_link_generic_system, OID_AUTO, if_bw_measure_size,
591 CTLFLAG_RW | CTLFLAG_LOCKED, &if_bw_measure_size, 0, "");
592
593 static u_int32_t cur_dlil_input_threads = 0;
594 SYSCTL_UINT(_net_link_generic_system, OID_AUTO, dlil_input_threads,
595 CTLFLAG_RD | CTLFLAG_LOCKED, &cur_dlil_input_threads , 0,
596 "Current number of DLIL input threads");
597
598 #if IFNET_INPUT_SANITY_CHK
599 SYSCTL_UINT(_net_link_generic_system, OID_AUTO, dlil_input_sanity_check,
600 CTLFLAG_RW | CTLFLAG_LOCKED, &dlil_input_sanity_check , 0,
601 "Turn on sanity checking in DLIL input");
602 #endif /* IFNET_INPUT_SANITY_CHK */
603
604 static u_int32_t if_flowadv = 1;
605 SYSCTL_UINT(_net_link_generic_system, OID_AUTO, flow_advisory,
606 CTLFLAG_RW | CTLFLAG_LOCKED, &if_flowadv, 1,
607 "enable flow-advisory mechanism");
608
609 static uint64_t hwcksum_in_invalidated = 0;
610 SYSCTL_QUAD(_net_link_generic_system, OID_AUTO,
611 hwcksum_in_invalidated, CTLFLAG_RD | CTLFLAG_LOCKED,
612 &hwcksum_in_invalidated, "inbound packets with invalidated hardware cksum");
613
614 uint32_t hwcksum_dbg = 0;
615 SYSCTL_UINT(_net_link_generic_system, OID_AUTO, hwcksum_dbg,
616 CTLFLAG_RW | CTLFLAG_LOCKED, &hwcksum_dbg, 0,
617 "enable hardware cksum debugging");
618
619 #define HWCKSUM_DBG_PARTIAL_FORCED 0x1 /* forced partial checksum */
620 #define HWCKSUM_DBG_PARTIAL_RXOFF_ADJ 0x2 /* adjust start offset */
621 #define HWCKSUM_DBG_FINALIZE_FORCED 0x10 /* forced finalize */
622 #define HWCKSUM_DBG_MASK \
623 (HWCKSUM_DBG_PARTIAL_FORCED | HWCKSUM_DBG_PARTIAL_RXOFF_ADJ | \
624 HWCKSUM_DBG_FINALIZE_FORCED)
625
626 static uint32_t hwcksum_dbg_mode = 0;
627 SYSCTL_PROC(_net_link_generic_system, OID_AUTO, hwcksum_dbg_mode,
628 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &hwcksum_dbg_mode,
629 0, sysctl_hwcksum_dbg_mode, "I", "hardware cksum debugging mode");
630
631 static uint64_t hwcksum_dbg_partial_forced = 0;
632 SYSCTL_QUAD(_net_link_generic_system, OID_AUTO,
633 hwcksum_dbg_partial_forced, CTLFLAG_RD | CTLFLAG_LOCKED,
634 &hwcksum_dbg_partial_forced, "packets forced using partial cksum");
635
636 static uint64_t hwcksum_dbg_partial_forced_bytes = 0;
637 SYSCTL_QUAD(_net_link_generic_system, OID_AUTO,
638 hwcksum_dbg_partial_forced_bytes, CTLFLAG_RD | CTLFLAG_LOCKED,
639 &hwcksum_dbg_partial_forced_bytes, "bytes forced using partial cksum");
640
641 static uint32_t hwcksum_dbg_partial_rxoff_forced = 0;
642 SYSCTL_PROC(_net_link_generic_system, OID_AUTO,
643 hwcksum_dbg_partial_rxoff_forced, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
644 &hwcksum_dbg_partial_rxoff_forced, 0,
645 sysctl_hwcksum_dbg_partial_rxoff_forced, "I",
646 "forced partial cksum rx offset");
647
648 static uint32_t hwcksum_dbg_partial_rxoff_adj = 0;
649 SYSCTL_PROC(_net_link_generic_system, OID_AUTO, hwcksum_dbg_partial_rxoff_adj,
650 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &hwcksum_dbg_partial_rxoff_adj,
651 0, sysctl_hwcksum_dbg_partial_rxoff_adj, "I",
652 "adjusted partial cksum rx offset");
653
654 static uint64_t hwcksum_dbg_verified = 0;
655 SYSCTL_QUAD(_net_link_generic_system, OID_AUTO,
656 hwcksum_dbg_verified, CTLFLAG_RD | CTLFLAG_LOCKED,
657 &hwcksum_dbg_verified, "packets verified for having good checksum");
658
659 static uint64_t hwcksum_dbg_bad_cksum = 0;
660 SYSCTL_QUAD(_net_link_generic_system, OID_AUTO,
661 hwcksum_dbg_bad_cksum, CTLFLAG_RD | CTLFLAG_LOCKED,
662 &hwcksum_dbg_bad_cksum, "packets with bad hardware calculated checksum");
663
664 static uint64_t hwcksum_dbg_bad_rxoff = 0;
665 SYSCTL_QUAD(_net_link_generic_system, OID_AUTO,
666 hwcksum_dbg_bad_rxoff, CTLFLAG_RD | CTLFLAG_LOCKED,
667 &hwcksum_dbg_bad_rxoff, "packets with invalid rxoff");
668
669 static uint64_t hwcksum_dbg_adjusted = 0;
670 SYSCTL_QUAD(_net_link_generic_system, OID_AUTO,
671 hwcksum_dbg_adjusted, CTLFLAG_RD | CTLFLAG_LOCKED,
672 &hwcksum_dbg_adjusted, "packets with rxoff adjusted");
673
674 static uint64_t hwcksum_dbg_finalized_hdr = 0;
675 SYSCTL_QUAD(_net_link_generic_system, OID_AUTO,
676 hwcksum_dbg_finalized_hdr, CTLFLAG_RD | CTLFLAG_LOCKED,
677 &hwcksum_dbg_finalized_hdr, "finalized headers");
678
679 static uint64_t hwcksum_dbg_finalized_data = 0;
680 SYSCTL_QUAD(_net_link_generic_system, OID_AUTO,
681 hwcksum_dbg_finalized_data, CTLFLAG_RD | CTLFLAG_LOCKED,
682 &hwcksum_dbg_finalized_data, "finalized payloads");
683
684 uint32_t hwcksum_tx = 1;
685 SYSCTL_UINT(_net_link_generic_system, OID_AUTO, hwcksum_tx,
686 CTLFLAG_RW | CTLFLAG_LOCKED, &hwcksum_tx, 0,
687 "enable transmit hardware checksum offload");
688
689 uint32_t hwcksum_rx = 1;
690 SYSCTL_UINT(_net_link_generic_system, OID_AUTO, hwcksum_rx,
691 CTLFLAG_RW | CTLFLAG_LOCKED, &hwcksum_rx, 0,
692 "enable receive hardware checksum offload");
693
694 unsigned int net_rxpoll = 1;
695 unsigned int net_affinity = 1;
696 static kern_return_t dlil_affinity_set(struct thread *, u_int32_t);
697
698 extern u_int32_t inject_buckets;
699
700 static lck_grp_attr_t *dlil_grp_attributes = NULL;
701 static lck_attr_t *dlil_lck_attributes = NULL;
702
703
704 #define DLIL_INPUT_CHECK(m, ifp) { \
705 struct ifnet *_rcvif = mbuf_pkthdr_rcvif(m); \
706 if (_rcvif == NULL || (ifp != lo_ifp && _rcvif != ifp) || \
707 !(mbuf_flags(m) & MBUF_PKTHDR)) { \
708 panic_plain("%s: invalid mbuf %p\n", __func__, m); \
709 /* NOTREACHED */ \
710 } \
711 }
712
713 #define DLIL_EWMA(old, new, decay) do { \
714 u_int32_t _avg; \
715 if ((_avg = (old)) > 0) \
716 _avg = (((_avg << (decay)) - _avg) + (new)) >> (decay); \
717 else \
718 _avg = (new); \
719 (old) = _avg; \
720 } while (0)
721
722 #define MBPS (1ULL * 1000 * 1000)
723 #define GBPS (MBPS * 1000)
724
725 struct rxpoll_time_tbl {
726 u_int64_t speed; /* downlink speed */
727 u_int32_t plowat; /* packets low watermark */
728 u_int32_t phiwat; /* packets high watermark */
729 u_int32_t blowat; /* bytes low watermark */
730 u_int32_t bhiwat; /* bytes high watermark */
731 };
732
733 static struct rxpoll_time_tbl rxpoll_tbl[] = {
734 { 10 * MBPS, 2, 8, (1 * 1024), (6 * 1024) },
735 { 100 * MBPS, 10, 40, (4 * 1024), (64 * 1024) },
736 { 1 * GBPS, 10, 40, (4 * 1024), (64 * 1024) },
737 { 10 * GBPS, 10, 40, (4 * 1024), (64 * 1024) },
738 { 100 * GBPS, 10, 40, (4 * 1024), (64 * 1024) },
739 { 0, 0, 0, 0, 0 }
740 };
741
742 int
743 proto_hash_value(u_int32_t protocol_family)
744 {
745 /*
746 * dlil_proto_unplumb_all() depends on the mapping between
747 * the hash bucket index and the protocol family defined
748 * here; future changes must be applied there as well.
749 */
750 switch(protocol_family) {
751 case PF_INET:
752 return (0);
753 case PF_INET6:
754 return (1);
755 case PF_VLAN:
756 return (2);
757 case PF_UNSPEC:
758 default:
759 return (3);
760 }
761 }
762
763 /*
764 * Caller must already be holding ifnet lock.
765 */
766 static struct if_proto *
767 find_attached_proto(struct ifnet *ifp, u_int32_t protocol_family)
768 {
769 struct if_proto *proto = NULL;
770 u_int32_t i = proto_hash_value(protocol_family);
771
772 ifnet_lock_assert(ifp, IFNET_LCK_ASSERT_OWNED);
773
774 if (ifp->if_proto_hash != NULL)
775 proto = SLIST_FIRST(&ifp->if_proto_hash[i]);
776
777 while (proto != NULL && proto->protocol_family != protocol_family)
778 proto = SLIST_NEXT(proto, next_hash);
779
780 if (proto != NULL)
781 if_proto_ref(proto);
782
783 return (proto);
784 }
785
786 static void
787 if_proto_ref(struct if_proto *proto)
788 {
789 atomic_add_32(&proto->refcount, 1);
790 }
791
792 extern void if_rtproto_del(struct ifnet *ifp, int protocol);
793
794 static void
795 if_proto_free(struct if_proto *proto)
796 {
797 u_int32_t oldval;
798 struct ifnet *ifp = proto->ifp;
799 u_int32_t proto_family = proto->protocol_family;
800 struct kev_dl_proto_data ev_pr_data;
801
802 oldval = atomic_add_32_ov(&proto->refcount, -1);
803 if (oldval > 1)
804 return;
805
806 /* No more reference on this, protocol must have been detached */
807 VERIFY(proto->detached);
808
809 if (proto->proto_kpi == kProtoKPI_v1) {
810 if (proto->kpi.v1.detached)
811 proto->kpi.v1.detached(ifp, proto->protocol_family);
812 }
813 if (proto->proto_kpi == kProtoKPI_v2) {
814 if (proto->kpi.v2.detached)
815 proto->kpi.v2.detached(ifp, proto->protocol_family);
816 }
817
818 /*
819 * Cleanup routes that may still be in the routing table for that
820 * interface/protocol pair.
821 */
822 if_rtproto_del(ifp, proto_family);
823
824 /*
825 * The reserved field carries the number of protocol still attached
826 * (subject to change)
827 */
828 ifnet_lock_shared(ifp);
829 ev_pr_data.proto_family = proto_family;
830 ev_pr_data.proto_remaining_count = dlil_ifp_proto_count(ifp);
831 ifnet_lock_done(ifp);
832
833 dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_PROTO_DETACHED,
834 (struct net_event_data *)&ev_pr_data,
835 sizeof(struct kev_dl_proto_data));
836
837 zfree(dlif_proto_zone, proto);
838 }
839
840 __private_extern__ void
841 ifnet_lock_assert(struct ifnet *ifp, ifnet_lock_assert_t what)
842 {
843 unsigned int type = 0;
844 int ass = 1;
845
846 switch (what) {
847 case IFNET_LCK_ASSERT_EXCLUSIVE:
848 type = LCK_RW_ASSERT_EXCLUSIVE;
849 break;
850
851 case IFNET_LCK_ASSERT_SHARED:
852 type = LCK_RW_ASSERT_SHARED;
853 break;
854
855 case IFNET_LCK_ASSERT_OWNED:
856 type = LCK_RW_ASSERT_HELD;
857 break;
858
859 case IFNET_LCK_ASSERT_NOTOWNED:
860 /* nothing to do here for RW lock; bypass assert */
861 ass = 0;
862 break;
863
864 default:
865 panic("bad ifnet assert type: %d", what);
866 /* NOTREACHED */
867 }
868 if (ass)
869 lck_rw_assert(&ifp->if_lock, type);
870 }
871
872 __private_extern__ void
873 ifnet_lock_shared(struct ifnet *ifp)
874 {
875 lck_rw_lock_shared(&ifp->if_lock);
876 }
877
878 __private_extern__ void
879 ifnet_lock_exclusive(struct ifnet *ifp)
880 {
881 lck_rw_lock_exclusive(&ifp->if_lock);
882 }
883
884 __private_extern__ void
885 ifnet_lock_done(struct ifnet *ifp)
886 {
887 lck_rw_done(&ifp->if_lock);
888 }
889
890 #if INET6
891 __private_extern__ void
892 if_inet6data_lock_shared(struct ifnet *ifp)
893 {
894 lck_rw_lock_shared(&ifp->if_inet6data_lock);
895 }
896
897 __private_extern__ void
898 if_inet6data_lock_exclusive(struct ifnet *ifp)
899 {
900 lck_rw_lock_exclusive(&ifp->if_inet6data_lock);
901 }
902
903 __private_extern__ void
904 if_inet6data_lock_done(struct ifnet *ifp)
905 {
906 lck_rw_done(&ifp->if_inet6data_lock);
907 }
908 #endif
909
910 __private_extern__ void
911 ifnet_head_lock_shared(void)
912 {
913 lck_rw_lock_shared(&ifnet_head_lock);
914 }
915
916 __private_extern__ void
917 ifnet_head_lock_exclusive(void)
918 {
919 lck_rw_lock_exclusive(&ifnet_head_lock);
920 }
921
922 __private_extern__ void
923 ifnet_head_done(void)
924 {
925 lck_rw_done(&ifnet_head_lock);
926 }
927
928 /*
929 * Caller must already be holding ifnet lock.
930 */
931 static int
932 dlil_ifp_proto_count(struct ifnet * ifp)
933 {
934 int i, count = 0;
935
936 ifnet_lock_assert(ifp, IFNET_LCK_ASSERT_OWNED);
937
938 if (ifp->if_proto_hash == NULL)
939 goto done;
940
941 for (i = 0; i < PROTO_HASH_SLOTS; i++) {
942 struct if_proto *proto;
943 SLIST_FOREACH(proto, &ifp->if_proto_hash[i], next_hash) {
944 count++;
945 }
946 }
947 done:
948 return (count);
949 }
950
951 __private_extern__ void
952 dlil_post_msg(struct ifnet *ifp, u_int32_t event_subclass,
953 u_int32_t event_code, struct net_event_data *event_data,
954 u_int32_t event_data_len)
955 {
956 struct net_event_data ev_data;
957 struct kev_msg ev_msg;
958
959 bzero(&ev_msg, sizeof (ev_msg));
960 bzero(&ev_data, sizeof (ev_data));
961 /*
962 * a net event always starts with a net_event_data structure
963 * but the caller can generate a simple net event or
964 * provide a longer event structure to post
965 */
966 ev_msg.vendor_code = KEV_VENDOR_APPLE;
967 ev_msg.kev_class = KEV_NETWORK_CLASS;
968 ev_msg.kev_subclass = event_subclass;
969 ev_msg.event_code = event_code;
970
971 if (event_data == NULL) {
972 event_data = &ev_data;
973 event_data_len = sizeof(struct net_event_data);
974 }
975
976 strncpy(&event_data->if_name[0], ifp->if_name, IFNAMSIZ);
977 event_data->if_family = ifp->if_family;
978 event_data->if_unit = (u_int32_t) ifp->if_unit;
979
980 ev_msg.dv[0].data_length = event_data_len;
981 ev_msg.dv[0].data_ptr = event_data;
982 ev_msg.dv[1].data_length = 0;
983
984 dlil_event_internal(ifp, &ev_msg);
985 }
986
987 __private_extern__ int
988 dlil_alloc_local_stats(struct ifnet *ifp)
989 {
990 int ret = EINVAL;
991 void *buf, *base, **pbuf;
992
993 if (ifp == NULL)
994 goto end;
995
996 if (ifp->if_tcp_stat == NULL && ifp->if_udp_stat == NULL) {
997 /* allocate tcpstat_local structure */
998 buf = zalloc(dlif_tcpstat_zone);
999 if (buf == NULL) {
1000 ret = ENOMEM;
1001 goto end;
1002 }
1003 bzero(buf, dlif_tcpstat_bufsize);
1004
1005 /* Get the 64-bit aligned base address for this object */
1006 base = (void *)P2ROUNDUP((intptr_t)buf + sizeof (u_int64_t),
1007 sizeof (u_int64_t));
1008 VERIFY(((intptr_t)base + dlif_tcpstat_size) <=
1009 ((intptr_t)buf + dlif_tcpstat_bufsize));
1010
1011 /*
1012 * Wind back a pointer size from the aligned base and
1013 * save the original address so we can free it later.
1014 */
1015 pbuf = (void **)((intptr_t)base - sizeof (void *));
1016 *pbuf = buf;
1017 ifp->if_tcp_stat = base;
1018
1019 /* allocate udpstat_local structure */
1020 buf = zalloc(dlif_udpstat_zone);
1021 if (buf == NULL) {
1022 ret = ENOMEM;
1023 goto end;
1024 }
1025 bzero(buf, dlif_udpstat_bufsize);
1026
1027 /* Get the 64-bit aligned base address for this object */
1028 base = (void *)P2ROUNDUP((intptr_t)buf + sizeof (u_int64_t),
1029 sizeof (u_int64_t));
1030 VERIFY(((intptr_t)base + dlif_udpstat_size) <=
1031 ((intptr_t)buf + dlif_udpstat_bufsize));
1032
1033 /*
1034 * Wind back a pointer size from the aligned base and
1035 * save the original address so we can free it later.
1036 */
1037 pbuf = (void **)((intptr_t)base - sizeof (void *));
1038 *pbuf = buf;
1039 ifp->if_udp_stat = base;
1040
1041 VERIFY(IS_P2ALIGNED(ifp->if_tcp_stat, sizeof (u_int64_t)) &&
1042 IS_P2ALIGNED(ifp->if_udp_stat, sizeof (u_int64_t)));
1043
1044 ret = 0;
1045 }
1046
1047 end:
1048 if (ret != 0) {
1049 if (ifp->if_tcp_stat != NULL) {
1050 pbuf = (void **)
1051 ((intptr_t)ifp->if_tcp_stat - sizeof (void *));
1052 zfree(dlif_tcpstat_zone, *pbuf);
1053 ifp->if_tcp_stat = NULL;
1054 }
1055 if (ifp->if_udp_stat != NULL) {
1056 pbuf = (void **)
1057 ((intptr_t)ifp->if_udp_stat - sizeof (void *));
1058 zfree(dlif_udpstat_zone, *pbuf);
1059 ifp->if_udp_stat = NULL;
1060 }
1061 }
1062
1063 return (ret);
1064 }
1065
1066 static int
1067 dlil_create_input_thread(ifnet_t ifp, struct dlil_threading_info *inp)
1068 {
1069 thread_continue_t func;
1070 u_int32_t limit;
1071 int error;
1072
1073 /* NULL ifp indicates the main input thread, called at dlil_init time */
1074 if (ifp == NULL) {
1075 func = dlil_main_input_thread_func;
1076 VERIFY(inp == dlil_main_input_thread);
1077 (void) strlcat(inp->input_name,
1078 "main_input", DLIL_THREADNAME_LEN);
1079 } else if (net_rxpoll && (ifp->if_eflags & IFEF_RXPOLL)) {
1080 func = dlil_rxpoll_input_thread_func;
1081 VERIFY(inp != dlil_main_input_thread);
1082 (void) snprintf(inp->input_name, DLIL_THREADNAME_LEN,
1083 "%s_input_poll", if_name(ifp));
1084 } else {
1085 func = dlil_input_thread_func;
1086 VERIFY(inp != dlil_main_input_thread);
1087 (void) snprintf(inp->input_name, DLIL_THREADNAME_LEN,
1088 "%s_input", if_name(ifp));
1089 }
1090 VERIFY(inp->input_thr == THREAD_NULL);
1091
1092 inp->lck_grp = lck_grp_alloc_init(inp->input_name, dlil_grp_attributes);
1093 lck_mtx_init(&inp->input_lck, inp->lck_grp, dlil_lck_attributes);
1094
1095 inp->mode = IFNET_MODEL_INPUT_POLL_OFF;
1096 inp->ifp = ifp; /* NULL for main input thread */
1097
1098 net_timerclear(&inp->mode_holdtime);
1099 net_timerclear(&inp->mode_lasttime);
1100 net_timerclear(&inp->sample_holdtime);
1101 net_timerclear(&inp->sample_lasttime);
1102 net_timerclear(&inp->dbg_lasttime);
1103
1104 /*
1105 * For interfaces that support opportunistic polling, set the
1106 * low and high watermarks for outstanding inbound packets/bytes.
1107 * Also define freeze times for transitioning between modes
1108 * and updating the average.
1109 */
1110 if (ifp != NULL && net_rxpoll && (ifp->if_eflags & IFEF_RXPOLL)) {
1111 limit = MAX(if_rcvq_maxlen, IF_RCVQ_MINLEN);
1112 (void) dlil_rxpoll_set_params(ifp, NULL, FALSE);
1113 } else {
1114 limit = (u_int32_t)-1;
1115 }
1116
1117 _qinit(&inp->rcvq_pkts, Q_DROPTAIL, limit);
1118 if (inp == dlil_main_input_thread) {
1119 struct dlil_main_threading_info *inpm =
1120 (struct dlil_main_threading_info *)inp;
1121 _qinit(&inpm->lo_rcvq_pkts, Q_DROPTAIL, limit);
1122 }
1123
1124 error = kernel_thread_start(func, inp, &inp->input_thr);
1125 if (error == KERN_SUCCESS) {
1126 ml_thread_policy(inp->input_thr, MACHINE_GROUP,
1127 (MACHINE_NETWORK_GROUP|MACHINE_NETWORK_NETISR));
1128 /*
1129 * We create an affinity set so that the matching workloop
1130 * thread or the starter thread (for loopback) can be
1131 * scheduled on the same processor set as the input thread.
1132 */
1133 if (net_affinity) {
1134 struct thread *tp = inp->input_thr;
1135 u_int32_t tag;
1136 /*
1137 * Randomize to reduce the probability
1138 * of affinity tag namespace collision.
1139 */
1140 read_random(&tag, sizeof (tag));
1141 if (dlil_affinity_set(tp, tag) == KERN_SUCCESS) {
1142 thread_reference(tp);
1143 inp->tag = tag;
1144 inp->net_affinity = TRUE;
1145 }
1146 }
1147 } else if (inp == dlil_main_input_thread) {
1148 panic_plain("%s: couldn't create main input thread", __func__);
1149 /* NOTREACHED */
1150 } else {
1151 panic_plain("%s: couldn't create %s input thread", __func__,
1152 if_name(ifp));
1153 /* NOTREACHED */
1154 }
1155 OSAddAtomic(1, &cur_dlil_input_threads);
1156
1157 return (error);
1158 }
1159
1160 static void
1161 dlil_terminate_input_thread(struct dlil_threading_info *inp)
1162 {
1163 struct ifnet *ifp;
1164
1165 VERIFY(current_thread() == inp->input_thr);
1166 VERIFY(inp != dlil_main_input_thread);
1167
1168 OSAddAtomic(-1, &cur_dlil_input_threads);
1169
1170 lck_mtx_destroy(&inp->input_lck, inp->lck_grp);
1171 lck_grp_free(inp->lck_grp);
1172
1173 inp->input_waiting = 0;
1174 inp->wtot = 0;
1175 bzero(inp->input_name, sizeof (inp->input_name));
1176 ifp = inp->ifp;
1177 inp->ifp = NULL;
1178 VERIFY(qhead(&inp->rcvq_pkts) == NULL && qempty(&inp->rcvq_pkts));
1179 qlimit(&inp->rcvq_pkts) = 0;
1180 bzero(&inp->stats, sizeof (inp->stats));
1181
1182 VERIFY(!inp->net_affinity);
1183 inp->input_thr = THREAD_NULL;
1184 VERIFY(inp->wloop_thr == THREAD_NULL);
1185 VERIFY(inp->poll_thr == THREAD_NULL);
1186 VERIFY(inp->tag == 0);
1187
1188 inp->mode = IFNET_MODEL_INPUT_POLL_OFF;
1189 bzero(&inp->tstats, sizeof (inp->tstats));
1190 bzero(&inp->pstats, sizeof (inp->pstats));
1191 bzero(&inp->sstats, sizeof (inp->sstats));
1192
1193 net_timerclear(&inp->mode_holdtime);
1194 net_timerclear(&inp->mode_lasttime);
1195 net_timerclear(&inp->sample_holdtime);
1196 net_timerclear(&inp->sample_lasttime);
1197 net_timerclear(&inp->dbg_lasttime);
1198
1199 #if IFNET_INPUT_SANITY_CHK
1200 inp->input_mbuf_cnt = 0;
1201 #endif /* IFNET_INPUT_SANITY_CHK */
1202
1203 if (dlil_verbose) {
1204 printf("%s: input thread terminated\n",
1205 if_name(ifp));
1206 }
1207
1208 /* for the extra refcnt from kernel_thread_start() */
1209 thread_deallocate(current_thread());
1210
1211 /* this is the end */
1212 thread_terminate(current_thread());
1213 /* NOTREACHED */
1214 }
1215
1216 static kern_return_t
1217 dlil_affinity_set(struct thread *tp, u_int32_t tag)
1218 {
1219 thread_affinity_policy_data_t policy;
1220
1221 bzero(&policy, sizeof (policy));
1222 policy.affinity_tag = tag;
1223 return (thread_policy_set(tp, THREAD_AFFINITY_POLICY,
1224 (thread_policy_t)&policy, THREAD_AFFINITY_POLICY_COUNT));
1225 }
1226
1227 void
1228 dlil_init(void)
1229 {
1230 thread_t thread = THREAD_NULL;
1231
1232 /*
1233 * The following fields must be 64-bit aligned for atomic operations.
1234 */
1235 IF_DATA_REQUIRE_ALIGNED_64(ifi_ipackets);
1236 IF_DATA_REQUIRE_ALIGNED_64(ifi_ierrors)
1237 IF_DATA_REQUIRE_ALIGNED_64(ifi_opackets);
1238 IF_DATA_REQUIRE_ALIGNED_64(ifi_oerrors);
1239 IF_DATA_REQUIRE_ALIGNED_64(ifi_collisions);
1240 IF_DATA_REQUIRE_ALIGNED_64(ifi_ibytes);
1241 IF_DATA_REQUIRE_ALIGNED_64(ifi_obytes);
1242 IF_DATA_REQUIRE_ALIGNED_64(ifi_imcasts);
1243 IF_DATA_REQUIRE_ALIGNED_64(ifi_omcasts);
1244 IF_DATA_REQUIRE_ALIGNED_64(ifi_iqdrops);
1245 IF_DATA_REQUIRE_ALIGNED_64(ifi_noproto);
1246 IF_DATA_REQUIRE_ALIGNED_64(ifi_alignerrs);
1247 IF_DATA_REQUIRE_ALIGNED_64(ifi_dt_bytes);
1248 IF_DATA_REQUIRE_ALIGNED_64(ifi_fpackets);
1249 IF_DATA_REQUIRE_ALIGNED_64(ifi_fbytes);
1250
1251 IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_ipackets);
1252 IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_ierrors)
1253 IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_opackets);
1254 IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_oerrors);
1255 IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_collisions);
1256 IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_ibytes);
1257 IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_obytes);
1258 IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_imcasts);
1259 IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_omcasts);
1260 IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_iqdrops);
1261 IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_noproto);
1262 IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_alignerrs);
1263 IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_dt_bytes);
1264 IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_fpackets);
1265 IFNET_IF_DATA_REQUIRE_ALIGNED_64(ifi_fbytes);
1266
1267 /*
1268 * These IF_HWASSIST_ flags must be equal to their IFNET_* counterparts.
1269 */
1270 _CASSERT(IF_HWASSIST_CSUM_IP == IFNET_CSUM_IP);
1271 _CASSERT(IF_HWASSIST_CSUM_TCP == IFNET_CSUM_TCP);
1272 _CASSERT(IF_HWASSIST_CSUM_UDP == IFNET_CSUM_UDP);
1273 _CASSERT(IF_HWASSIST_CSUM_IP_FRAGS == IFNET_CSUM_FRAGMENT);
1274 _CASSERT(IF_HWASSIST_CSUM_FRAGMENT == IFNET_IP_FRAGMENT);
1275 _CASSERT(IF_HWASSIST_CSUM_TCPIPV6 == IFNET_CSUM_TCPIPV6);
1276 _CASSERT(IF_HWASSIST_CSUM_UDPIPV6 == IFNET_CSUM_UDPIPV6);
1277 _CASSERT(IF_HWASSIST_CSUM_FRAGMENT_IPV6 == IFNET_IPV6_FRAGMENT);
1278 _CASSERT(IF_HWASSIST_CSUM_PARTIAL == IFNET_CSUM_PARTIAL);
1279 _CASSERT(IF_HWASSIST_VLAN_TAGGING == IFNET_VLAN_TAGGING);
1280 _CASSERT(IF_HWASSIST_VLAN_MTU == IFNET_VLAN_MTU);
1281 _CASSERT(IF_HWASSIST_TSO_V4 == IFNET_TSO_IPV4);
1282 _CASSERT(IF_HWASSIST_TSO_V6 == IFNET_TSO_IPV6);
1283
1284 /*
1285 * ... as well as the mbuf checksum flags counterparts.
1286 */
1287 _CASSERT(CSUM_IP == IF_HWASSIST_CSUM_IP);
1288 _CASSERT(CSUM_TCP == IF_HWASSIST_CSUM_TCP);
1289 _CASSERT(CSUM_UDP == IF_HWASSIST_CSUM_UDP);
1290 _CASSERT(CSUM_IP_FRAGS == IF_HWASSIST_CSUM_IP_FRAGS);
1291 _CASSERT(CSUM_FRAGMENT == IF_HWASSIST_CSUM_FRAGMENT);
1292 _CASSERT(CSUM_TCPIPV6 == IF_HWASSIST_CSUM_TCPIPV6);
1293 _CASSERT(CSUM_UDPIPV6 == IF_HWASSIST_CSUM_UDPIPV6);
1294 _CASSERT(CSUM_FRAGMENT_IPV6 == IF_HWASSIST_CSUM_FRAGMENT_IPV6);
1295 _CASSERT(CSUM_PARTIAL == IF_HWASSIST_CSUM_PARTIAL);
1296 _CASSERT(CSUM_VLAN_TAG_VALID == IF_HWASSIST_VLAN_TAGGING);
1297
1298 /*
1299 * Make sure we have at least IF_LLREACH_MAXLEN in the llreach info.
1300 */
1301 _CASSERT(IF_LLREACH_MAXLEN <= IF_LLREACHINFO_ADDRLEN);
1302 _CASSERT(IFNET_LLREACHINFO_ADDRLEN == IF_LLREACHINFO_ADDRLEN);
1303
1304 _CASSERT(IFRLOGF_DLIL == IFNET_LOGF_DLIL);
1305 _CASSERT(IFRLOGF_FAMILY == IFNET_LOGF_FAMILY);
1306 _CASSERT(IFRLOGF_DRIVER == IFNET_LOGF_DRIVER);
1307 _CASSERT(IFRLOGF_FIRMWARE == IFNET_LOGF_FIRMWARE);
1308
1309 _CASSERT(IFRLOGCAT_CONNECTIVITY == IFNET_LOGCAT_CONNECTIVITY);
1310 _CASSERT(IFRLOGCAT_QUALITY == IFNET_LOGCAT_QUALITY);
1311 _CASSERT(IFRLOGCAT_PERFORMANCE == IFNET_LOGCAT_PERFORMANCE);
1312
1313 _CASSERT(IFRTYPE_FAMILY_ANY == IFNET_FAMILY_ANY);
1314 _CASSERT(IFRTYPE_FAMILY_LOOPBACK == IFNET_FAMILY_LOOPBACK);
1315 _CASSERT(IFRTYPE_FAMILY_ETHERNET == IFNET_FAMILY_ETHERNET);
1316 _CASSERT(IFRTYPE_FAMILY_SLIP == IFNET_FAMILY_SLIP);
1317 _CASSERT(IFRTYPE_FAMILY_TUN == IFNET_FAMILY_TUN);
1318 _CASSERT(IFRTYPE_FAMILY_VLAN == IFNET_FAMILY_VLAN);
1319 _CASSERT(IFRTYPE_FAMILY_PPP == IFNET_FAMILY_PPP);
1320 _CASSERT(IFRTYPE_FAMILY_PVC == IFNET_FAMILY_PVC);
1321 _CASSERT(IFRTYPE_FAMILY_DISC == IFNET_FAMILY_DISC);
1322 _CASSERT(IFRTYPE_FAMILY_MDECAP == IFNET_FAMILY_MDECAP);
1323 _CASSERT(IFRTYPE_FAMILY_GIF == IFNET_FAMILY_GIF);
1324 _CASSERT(IFRTYPE_FAMILY_FAITH == IFNET_FAMILY_FAITH);
1325 _CASSERT(IFRTYPE_FAMILY_STF == IFNET_FAMILY_STF);
1326 _CASSERT(IFRTYPE_FAMILY_FIREWIRE == IFNET_FAMILY_FIREWIRE);
1327 _CASSERT(IFRTYPE_FAMILY_BOND == IFNET_FAMILY_BOND);
1328 _CASSERT(IFRTYPE_FAMILY_CELLULAR == IFNET_FAMILY_CELLULAR);
1329
1330 _CASSERT(IFRTYPE_SUBFAMILY_ANY == IFNET_SUBFAMILY_ANY);
1331 _CASSERT(IFRTYPE_SUBFAMILY_USB == IFNET_SUBFAMILY_USB);
1332 _CASSERT(IFRTYPE_SUBFAMILY_BLUETOOTH == IFNET_SUBFAMILY_BLUETOOTH);
1333 _CASSERT(IFRTYPE_SUBFAMILY_WIFI == IFNET_SUBFAMILY_WIFI);
1334 _CASSERT(IFRTYPE_SUBFAMILY_THUNDERBOLT == IFNET_SUBFAMILY_THUNDERBOLT);
1335
1336 _CASSERT(DLIL_MODIDLEN == IFNET_MODIDLEN);
1337 _CASSERT(DLIL_MODARGLEN == IFNET_MODARGLEN);
1338
1339 PE_parse_boot_argn("net_affinity", &net_affinity,
1340 sizeof (net_affinity));
1341
1342 PE_parse_boot_argn("net_rxpoll", &net_rxpoll, sizeof (net_rxpoll));
1343
1344 PE_parse_boot_argn("net_rtref", &net_rtref, sizeof (net_rtref));
1345
1346 PE_parse_boot_argn("ifnet_debug", &ifnet_debug, sizeof (ifnet_debug));
1347
1348 dlif_size = (ifnet_debug == 0) ? sizeof (struct dlil_ifnet) :
1349 sizeof (struct dlil_ifnet_dbg);
1350 /* Enforce 64-bit alignment for dlil_ifnet structure */
1351 dlif_bufsize = dlif_size + sizeof (void *) + sizeof (u_int64_t);
1352 dlif_bufsize = P2ROUNDUP(dlif_bufsize, sizeof (u_int64_t));
1353 dlif_zone = zinit(dlif_bufsize, DLIF_ZONE_MAX * dlif_bufsize,
1354 0, DLIF_ZONE_NAME);
1355 if (dlif_zone == NULL) {
1356 panic_plain("%s: failed allocating %s", __func__,
1357 DLIF_ZONE_NAME);
1358 /* NOTREACHED */
1359 }
1360 zone_change(dlif_zone, Z_EXPAND, TRUE);
1361 zone_change(dlif_zone, Z_CALLERACCT, FALSE);
1362
1363 dlif_filt_size = sizeof (struct ifnet_filter);
1364 dlif_filt_zone = zinit(dlif_filt_size,
1365 DLIF_FILT_ZONE_MAX * dlif_filt_size, 0, DLIF_FILT_ZONE_NAME);
1366 if (dlif_filt_zone == NULL) {
1367 panic_plain("%s: failed allocating %s", __func__,
1368 DLIF_FILT_ZONE_NAME);
1369 /* NOTREACHED */
1370 }
1371 zone_change(dlif_filt_zone, Z_EXPAND, TRUE);
1372 zone_change(dlif_filt_zone, Z_CALLERACCT, FALSE);
1373
1374 dlif_phash_size = sizeof (struct proto_hash_entry) * PROTO_HASH_SLOTS;
1375 dlif_phash_zone = zinit(dlif_phash_size,
1376 DLIF_PHASH_ZONE_MAX * dlif_phash_size, 0, DLIF_PHASH_ZONE_NAME);
1377 if (dlif_phash_zone == NULL) {
1378 panic_plain("%s: failed allocating %s", __func__,
1379 DLIF_PHASH_ZONE_NAME);
1380 /* NOTREACHED */
1381 }
1382 zone_change(dlif_phash_zone, Z_EXPAND, TRUE);
1383 zone_change(dlif_phash_zone, Z_CALLERACCT, FALSE);
1384
1385 dlif_proto_size = sizeof (struct if_proto);
1386 dlif_proto_zone = zinit(dlif_proto_size,
1387 DLIF_PROTO_ZONE_MAX * dlif_proto_size, 0, DLIF_PROTO_ZONE_NAME);
1388 if (dlif_proto_zone == NULL) {
1389 panic_plain("%s: failed allocating %s", __func__,
1390 DLIF_PROTO_ZONE_NAME);
1391 /* NOTREACHED */
1392 }
1393 zone_change(dlif_proto_zone, Z_EXPAND, TRUE);
1394 zone_change(dlif_proto_zone, Z_CALLERACCT, FALSE);
1395
1396 dlif_tcpstat_size = sizeof (struct tcpstat_local);
1397 /* Enforce 64-bit alignment for tcpstat_local structure */
1398 dlif_tcpstat_bufsize =
1399 dlif_tcpstat_size + sizeof (void *) + sizeof (u_int64_t);
1400 dlif_tcpstat_bufsize =
1401 P2ROUNDUP(dlif_tcpstat_bufsize, sizeof (u_int64_t));
1402 dlif_tcpstat_zone = zinit(dlif_tcpstat_bufsize,
1403 DLIF_TCPSTAT_ZONE_MAX * dlif_tcpstat_bufsize, 0,
1404 DLIF_TCPSTAT_ZONE_NAME);
1405 if (dlif_tcpstat_zone == NULL) {
1406 panic_plain("%s: failed allocating %s", __func__,
1407 DLIF_TCPSTAT_ZONE_NAME);
1408 /* NOTREACHED */
1409 }
1410 zone_change(dlif_tcpstat_zone, Z_EXPAND, TRUE);
1411 zone_change(dlif_tcpstat_zone, Z_CALLERACCT, FALSE);
1412
1413 dlif_udpstat_size = sizeof (struct udpstat_local);
1414 /* Enforce 64-bit alignment for udpstat_local structure */
1415 dlif_udpstat_bufsize =
1416 dlif_udpstat_size + sizeof (void *) + sizeof (u_int64_t);
1417 dlif_udpstat_bufsize =
1418 P2ROUNDUP(dlif_udpstat_bufsize, sizeof (u_int64_t));
1419 dlif_udpstat_zone = zinit(dlif_udpstat_bufsize,
1420 DLIF_TCPSTAT_ZONE_MAX * dlif_udpstat_bufsize, 0,
1421 DLIF_UDPSTAT_ZONE_NAME);
1422 if (dlif_udpstat_zone == NULL) {
1423 panic_plain("%s: failed allocating %s", __func__,
1424 DLIF_UDPSTAT_ZONE_NAME);
1425 /* NOTREACHED */
1426 }
1427 zone_change(dlif_udpstat_zone, Z_EXPAND, TRUE);
1428 zone_change(dlif_udpstat_zone, Z_CALLERACCT, FALSE);
1429
1430 ifnet_llreach_init();
1431
1432 TAILQ_INIT(&dlil_ifnet_head);
1433 TAILQ_INIT(&ifnet_head);
1434 TAILQ_INIT(&ifnet_detaching_head);
1435
1436 /* Setup the lock groups we will use */
1437 dlil_grp_attributes = lck_grp_attr_alloc_init();
1438
1439 dlil_lock_group = lck_grp_alloc_init("DLIL internal locks",
1440 dlil_grp_attributes);
1441 ifnet_lock_group = lck_grp_alloc_init("ifnet locks",
1442 dlil_grp_attributes);
1443 ifnet_head_lock_group = lck_grp_alloc_init("ifnet head lock",
1444 dlil_grp_attributes);
1445 ifnet_rcv_lock_group = lck_grp_alloc_init("ifnet rcv locks",
1446 dlil_grp_attributes);
1447 ifnet_snd_lock_group = lck_grp_alloc_init("ifnet snd locks",
1448 dlil_grp_attributes);
1449
1450 /* Setup the lock attributes we will use */
1451 dlil_lck_attributes = lck_attr_alloc_init();
1452
1453 ifnet_lock_attr = lck_attr_alloc_init();
1454
1455 lck_rw_init(&ifnet_head_lock, ifnet_head_lock_group,
1456 dlil_lck_attributes);
1457 lck_mtx_init(&dlil_ifnet_lock, dlil_lock_group, dlil_lck_attributes);
1458
1459 /* Setup interface flow control related items */
1460 lck_mtx_init(&ifnet_fc_lock, dlil_lock_group, dlil_lck_attributes);
1461
1462 ifnet_fc_zone_size = sizeof (struct ifnet_fc_entry);
1463 ifnet_fc_zone = zinit(ifnet_fc_zone_size,
1464 IFNET_FC_ZONE_MAX * ifnet_fc_zone_size, 0, IFNET_FC_ZONE_NAME);
1465 if (ifnet_fc_zone == NULL) {
1466 panic_plain("%s: failed allocating %s", __func__,
1467 IFNET_FC_ZONE_NAME);
1468 /* NOTREACHED */
1469 }
1470 zone_change(ifnet_fc_zone, Z_EXPAND, TRUE);
1471 zone_change(ifnet_fc_zone, Z_CALLERACCT, FALSE);
1472
1473 /* Initialize interface address subsystem */
1474 ifa_init();
1475
1476 #if PF
1477 /* Initialize the packet filter */
1478 pfinit();
1479 #endif /* PF */
1480
1481 /* Initialize queue algorithms */
1482 classq_init();
1483
1484 /* Initialize packet schedulers */
1485 pktsched_init();
1486
1487 /* Initialize flow advisory subsystem */
1488 flowadv_init();
1489
1490 /* Initialize the pktap virtual interface */
1491 pktap_init();
1492
1493 #if DEBUG
1494 /* Run self-tests */
1495 dlil_verify_sum16();
1496 #endif /* DEBUG */
1497
1498 /*
1499 * Create and start up the main DLIL input thread and the interface
1500 * detacher threads once everything is initialized.
1501 */
1502 dlil_create_input_thread(NULL, dlil_main_input_thread);
1503
1504 if (kernel_thread_start(ifnet_detacher_thread_func,
1505 NULL, &thread) != KERN_SUCCESS) {
1506 panic_plain("%s: couldn't create detacher thread", __func__);
1507 /* NOTREACHED */
1508 }
1509 thread_deallocate(thread);
1510 }
1511
1512 static void
1513 if_flt_monitor_busy(struct ifnet *ifp)
1514 {
1515 lck_mtx_assert(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED);
1516
1517 ++ifp->if_flt_busy;
1518 VERIFY(ifp->if_flt_busy != 0);
1519 }
1520
1521 static void
1522 if_flt_monitor_unbusy(struct ifnet *ifp)
1523 {
1524 if_flt_monitor_leave(ifp);
1525 }
1526
1527 static void
1528 if_flt_monitor_enter(struct ifnet *ifp)
1529 {
1530 lck_mtx_assert(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED);
1531
1532 while (ifp->if_flt_busy) {
1533 ++ifp->if_flt_waiters;
1534 (void) msleep(&ifp->if_flt_head, &ifp->if_flt_lock,
1535 (PZERO - 1), "if_flt_monitor", NULL);
1536 }
1537 if_flt_monitor_busy(ifp);
1538 }
1539
1540 static void
1541 if_flt_monitor_leave(struct ifnet *ifp)
1542 {
1543 lck_mtx_assert(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED);
1544
1545 VERIFY(ifp->if_flt_busy != 0);
1546 --ifp->if_flt_busy;
1547
1548 if (ifp->if_flt_busy == 0 && ifp->if_flt_waiters > 0) {
1549 ifp->if_flt_waiters = 0;
1550 wakeup(&ifp->if_flt_head);
1551 }
1552 }
1553
1554 __private_extern__ int
1555 dlil_attach_filter(struct ifnet *ifp, const struct iff_filter *if_filter,
1556 interface_filter_t *filter_ref, u_int32_t flags)
1557 {
1558 int retval = 0;
1559 struct ifnet_filter *filter = NULL;
1560
1561 ifnet_head_lock_shared();
1562 /* Check that the interface is in the global list */
1563 if (!ifnet_lookup(ifp)) {
1564 retval = ENXIO;
1565 goto done;
1566 }
1567
1568 filter = zalloc(dlif_filt_zone);
1569 if (filter == NULL) {
1570 retval = ENOMEM;
1571 goto done;
1572 }
1573 bzero(filter, dlif_filt_size);
1574
1575 /* refcnt held above during lookup */
1576 filter->filt_flags = flags;
1577 filter->filt_ifp = ifp;
1578 filter->filt_cookie = if_filter->iff_cookie;
1579 filter->filt_name = if_filter->iff_name;
1580 filter->filt_protocol = if_filter->iff_protocol;
1581 filter->filt_input = if_filter->iff_input;
1582 filter->filt_output = if_filter->iff_output;
1583 filter->filt_event = if_filter->iff_event;
1584 filter->filt_ioctl = if_filter->iff_ioctl;
1585 filter->filt_detached = if_filter->iff_detached;
1586
1587 lck_mtx_lock(&ifp->if_flt_lock);
1588 if_flt_monitor_enter(ifp);
1589
1590 lck_mtx_assert(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED);
1591 TAILQ_INSERT_TAIL(&ifp->if_flt_head, filter, filt_next);
1592
1593 if_flt_monitor_leave(ifp);
1594 lck_mtx_unlock(&ifp->if_flt_lock);
1595
1596 *filter_ref = filter;
1597
1598 /*
1599 * Bump filter count and route_generation ID to let TCP
1600 * know it shouldn't do TSO on this connection
1601 */
1602 if ((filter->filt_flags & DLIL_IFF_TSO) == 0) {
1603 OSAddAtomic(1, &dlil_filter_disable_tso_count);
1604 routegenid_update();
1605 }
1606 if (dlil_verbose) {
1607 printf("%s: %s filter attached\n", if_name(ifp),
1608 if_filter->iff_name);
1609 }
1610 done:
1611 ifnet_head_done();
1612 if (retval != 0 && ifp != NULL) {
1613 DLIL_PRINTF("%s: failed to attach %s (err=%d)\n",
1614 if_name(ifp), if_filter->iff_name, retval);
1615 }
1616 if (retval != 0 && filter != NULL)
1617 zfree(dlif_filt_zone, filter);
1618
1619 return (retval);
1620 }
1621
1622 static int
1623 dlil_detach_filter_internal(interface_filter_t filter, int detached)
1624 {
1625 int retval = 0;
1626
1627 if (detached == 0) {
1628 ifnet_t ifp = NULL;
1629
1630 ifnet_head_lock_shared();
1631 TAILQ_FOREACH(ifp, &ifnet_head, if_link) {
1632 interface_filter_t entry = NULL;
1633
1634 lck_mtx_lock(&ifp->if_flt_lock);
1635 TAILQ_FOREACH(entry, &ifp->if_flt_head, filt_next) {
1636 if (entry != filter || entry->filt_skip)
1637 continue;
1638 /*
1639 * We've found a match; since it's possible
1640 * that the thread gets blocked in the monitor,
1641 * we do the lock dance. Interface should
1642 * not be detached since we still have a use
1643 * count held during filter attach.
1644 */
1645 entry->filt_skip = 1; /* skip input/output */
1646 lck_mtx_unlock(&ifp->if_flt_lock);
1647 ifnet_head_done();
1648
1649 lck_mtx_lock(&ifp->if_flt_lock);
1650 if_flt_monitor_enter(ifp);
1651 lck_mtx_assert(&ifp->if_flt_lock,
1652 LCK_MTX_ASSERT_OWNED);
1653
1654 /* Remove the filter from the list */
1655 TAILQ_REMOVE(&ifp->if_flt_head, filter,
1656 filt_next);
1657
1658 if_flt_monitor_leave(ifp);
1659 lck_mtx_unlock(&ifp->if_flt_lock);
1660 if (dlil_verbose) {
1661 printf("%s: %s filter detached\n",
1662 if_name(ifp), filter->filt_name);
1663 }
1664 goto destroy;
1665 }
1666 lck_mtx_unlock(&ifp->if_flt_lock);
1667 }
1668 ifnet_head_done();
1669
1670 /* filter parameter is not a valid filter ref */
1671 retval = EINVAL;
1672 goto done;
1673 }
1674
1675 if (dlil_verbose)
1676 printf("%s filter detached\n", filter->filt_name);
1677
1678 destroy:
1679
1680 /* Call the detached function if there is one */
1681 if (filter->filt_detached)
1682 filter->filt_detached(filter->filt_cookie, filter->filt_ifp);
1683
1684 /* Free the filter */
1685 zfree(dlif_filt_zone, filter);
1686
1687 /*
1688 * Decrease filter count and route_generation ID to let TCP
1689 * know it should reevalute doing TSO or not
1690 */
1691 if ((filter->filt_flags & DLIL_IFF_TSO) == 0) {
1692 OSAddAtomic(-1, &dlil_filter_disable_tso_count);
1693 routegenid_update();
1694 }
1695 done:
1696 if (retval != 0) {
1697 DLIL_PRINTF("failed to detach %s filter (err=%d)\n",
1698 filter->filt_name, retval);
1699 }
1700 return (retval);
1701 }
1702
1703 __private_extern__ void
1704 dlil_detach_filter(interface_filter_t filter)
1705 {
1706 if (filter == NULL)
1707 return;
1708 dlil_detach_filter_internal(filter, 0);
1709 }
1710
1711 /*
1712 * Main input thread:
1713 *
1714 * a) handles all inbound packets for lo0
1715 * b) handles all inbound packets for interfaces with no dedicated
1716 * input thread (e.g. anything but Ethernet/PDP or those that support
1717 * opportunistic polling.)
1718 * c) protocol registrations
1719 * d) packet injections
1720 */
1721 static void
1722 dlil_main_input_thread_func(void *v, wait_result_t w)
1723 {
1724 #pragma unused(w)
1725 struct dlil_main_threading_info *inpm = v;
1726 struct dlil_threading_info *inp = v;
1727
1728 VERIFY(inp == dlil_main_input_thread);
1729 VERIFY(inp->ifp == NULL);
1730 VERIFY(inp->mode == IFNET_MODEL_INPUT_POLL_OFF);
1731
1732 while (1) {
1733 struct mbuf *m = NULL, *m_loop = NULL;
1734 u_int32_t m_cnt, m_cnt_loop;
1735 boolean_t proto_req;
1736
1737 lck_mtx_lock_spin(&inp->input_lck);
1738
1739 /* Wait until there is work to be done */
1740 while (!(inp->input_waiting & ~DLIL_INPUT_RUNNING)) {
1741 inp->input_waiting &= ~DLIL_INPUT_RUNNING;
1742 (void) msleep(&inp->input_waiting, &inp->input_lck,
1743 (PZERO - 1) | PSPIN, inp->input_name, NULL);
1744 }
1745
1746 inp->input_waiting |= DLIL_INPUT_RUNNING;
1747 inp->input_waiting &= ~DLIL_INPUT_WAITING;
1748
1749 /* Main input thread cannot be terminated */
1750 VERIFY(!(inp->input_waiting & DLIL_INPUT_TERMINATE));
1751
1752 proto_req = (inp->input_waiting &
1753 (DLIL_PROTO_WAITING | DLIL_PROTO_REGISTER));
1754
1755 /* Packets for non-dedicated interfaces other than lo0 */
1756 m_cnt = qlen(&inp->rcvq_pkts);
1757 m = _getq_all(&inp->rcvq_pkts);
1758
1759 /* Packets exclusive to lo0 */
1760 m_cnt_loop = qlen(&inpm->lo_rcvq_pkts);
1761 m_loop = _getq_all(&inpm->lo_rcvq_pkts);
1762
1763 inp->wtot = 0;
1764
1765 lck_mtx_unlock(&inp->input_lck);
1766
1767 /*
1768 * NOTE warning %%% attention !!!!
1769 * We should think about putting some thread starvation
1770 * safeguards if we deal with long chains of packets.
1771 */
1772 if (m_loop != NULL)
1773 dlil_input_packet_list_extended(lo_ifp, m_loop,
1774 m_cnt_loop, inp->mode);
1775
1776 if (m != NULL)
1777 dlil_input_packet_list_extended(NULL, m,
1778 m_cnt, inp->mode);
1779
1780 if (proto_req)
1781 proto_input_run();
1782 }
1783
1784 /* NOTREACHED */
1785 VERIFY(0); /* we should never get here */
1786 }
1787
1788 /*
1789 * Input thread for interfaces with legacy input model.
1790 */
1791 static void
1792 dlil_input_thread_func(void *v, wait_result_t w)
1793 {
1794 #pragma unused(w)
1795 struct dlil_threading_info *inp = v;
1796 struct ifnet *ifp = inp->ifp;
1797
1798 VERIFY(inp != dlil_main_input_thread);
1799 VERIFY(ifp != NULL);
1800 VERIFY(!(ifp->if_eflags & IFEF_RXPOLL) || !net_rxpoll);
1801 VERIFY(inp->mode == IFNET_MODEL_INPUT_POLL_OFF);
1802
1803 while (1) {
1804 struct mbuf *m = NULL;
1805 u_int32_t m_cnt;
1806
1807 lck_mtx_lock_spin(&inp->input_lck);
1808
1809 /* Wait until there is work to be done */
1810 while (!(inp->input_waiting & ~DLIL_INPUT_RUNNING)) {
1811 inp->input_waiting &= ~DLIL_INPUT_RUNNING;
1812 (void) msleep(&inp->input_waiting, &inp->input_lck,
1813 (PZERO - 1) | PSPIN, inp->input_name, NULL);
1814 }
1815
1816 inp->input_waiting |= DLIL_INPUT_RUNNING;
1817 inp->input_waiting &= ~DLIL_INPUT_WAITING;
1818
1819 /*
1820 * Protocol registration and injection must always use
1821 * the main input thread; in theory the latter can utilize
1822 * the corresponding input thread where the packet arrived
1823 * on, but that requires our knowing the interface in advance
1824 * (and the benefits might not worth the trouble.)
1825 */
1826 VERIFY(!(inp->input_waiting &
1827 (DLIL_PROTO_WAITING|DLIL_PROTO_REGISTER)));
1828
1829 /* Packets for this interface */
1830 m_cnt = qlen(&inp->rcvq_pkts);
1831 m = _getq_all(&inp->rcvq_pkts);
1832
1833 if (inp->input_waiting & DLIL_INPUT_TERMINATE) {
1834 lck_mtx_unlock(&inp->input_lck);
1835
1836 /* Free up pending packets */
1837 if (m != NULL)
1838 mbuf_freem_list(m);
1839
1840 dlil_terminate_input_thread(inp);
1841 /* NOTREACHED */
1842 return;
1843 }
1844
1845 inp->wtot = 0;
1846
1847 dlil_input_stats_sync(ifp, inp);
1848
1849 lck_mtx_unlock(&inp->input_lck);
1850
1851 /*
1852 * NOTE warning %%% attention !!!!
1853 * We should think about putting some thread starvation
1854 * safeguards if we deal with long chains of packets.
1855 */
1856 if (m != NULL)
1857 dlil_input_packet_list_extended(NULL, m,
1858 m_cnt, inp->mode);
1859 }
1860
1861 /* NOTREACHED */
1862 VERIFY(0); /* we should never get here */
1863 }
1864
1865 /*
1866 * Input thread for interfaces with opportunistic polling input model.
1867 */
1868 static void
1869 dlil_rxpoll_input_thread_func(void *v, wait_result_t w)
1870 {
1871 #pragma unused(w)
1872 struct dlil_threading_info *inp = v;
1873 struct ifnet *ifp = inp->ifp;
1874 struct timespec ts;
1875
1876 VERIFY(inp != dlil_main_input_thread);
1877 VERIFY(ifp != NULL && (ifp->if_eflags & IFEF_RXPOLL));
1878
1879 while (1) {
1880 struct mbuf *m = NULL;
1881 u_int32_t m_cnt, m_size, poll_req = 0;
1882 ifnet_model_t mode;
1883 struct timespec now, delta;
1884 u_int64_t ival;
1885
1886 lck_mtx_lock_spin(&inp->input_lck);
1887
1888 if ((ival = inp->rxpoll_ival) < IF_RXPOLL_INTERVALTIME_MIN)
1889 ival = IF_RXPOLL_INTERVALTIME_MIN;
1890
1891 /* Link parameters changed? */
1892 if (ifp->if_poll_update != 0) {
1893 ifp->if_poll_update = 0;
1894 (void) dlil_rxpoll_set_params(ifp, NULL, TRUE);
1895 }
1896
1897 /* Current operating mode */
1898 mode = inp->mode;
1899
1900 /* Wait until there is work to be done */
1901 while (!(inp->input_waiting & ~DLIL_INPUT_RUNNING)) {
1902 inp->input_waiting &= ~DLIL_INPUT_RUNNING;
1903 (void) msleep(&inp->input_waiting, &inp->input_lck,
1904 (PZERO - 1) | PSPIN, inp->input_name, NULL);
1905 }
1906
1907 inp->input_waiting |= DLIL_INPUT_RUNNING;
1908 inp->input_waiting &= ~DLIL_INPUT_WAITING;
1909
1910 /*
1911 * Protocol registration and injection must always use
1912 * the main input thread; in theory the latter can utilize
1913 * the corresponding input thread where the packet arrived
1914 * on, but that requires our knowing the interface in advance
1915 * (and the benefits might not worth the trouble.)
1916 */
1917 VERIFY(!(inp->input_waiting &
1918 (DLIL_PROTO_WAITING|DLIL_PROTO_REGISTER)));
1919
1920 if (inp->input_waiting & DLIL_INPUT_TERMINATE) {
1921 /* Free up pending packets */
1922 _flushq(&inp->rcvq_pkts);
1923 lck_mtx_unlock(&inp->input_lck);
1924
1925 dlil_terminate_input_thread(inp);
1926 /* NOTREACHED */
1927 return;
1928 }
1929
1930 /* Total count of all packets */
1931 m_cnt = qlen(&inp->rcvq_pkts);
1932
1933 /* Total bytes of all packets */
1934 m_size = qsize(&inp->rcvq_pkts);
1935
1936 /* Packets for this interface */
1937 m = _getq_all(&inp->rcvq_pkts);
1938 VERIFY(m != NULL || m_cnt == 0);
1939
1940 nanouptime(&now);
1941 if (!net_timerisset(&inp->sample_lasttime))
1942 *(&inp->sample_lasttime) = *(&now);
1943
1944 net_timersub(&now, &inp->sample_lasttime, &delta);
1945 if (if_rxpoll && net_timerisset(&inp->sample_holdtime)) {
1946 u_int32_t ptot, btot;
1947
1948 /* Accumulate statistics for current sampling */
1949 PKTCNTR_ADD(&inp->sstats, m_cnt, m_size);
1950
1951 if (net_timercmp(&delta, &inp->sample_holdtime, <))
1952 goto skip;
1953
1954 *(&inp->sample_lasttime) = *(&now);
1955
1956 /* Calculate min/max of inbound bytes */
1957 btot = (u_int32_t)inp->sstats.bytes;
1958 if (inp->rxpoll_bmin == 0 || inp->rxpoll_bmin > btot)
1959 inp->rxpoll_bmin = btot;
1960 if (btot > inp->rxpoll_bmax)
1961 inp->rxpoll_bmax = btot;
1962
1963 /* Calculate EWMA of inbound bytes */
1964 DLIL_EWMA(inp->rxpoll_bavg, btot, if_rxpoll_decay);
1965
1966 /* Calculate min/max of inbound packets */
1967 ptot = (u_int32_t)inp->sstats.packets;
1968 if (inp->rxpoll_pmin == 0 || inp->rxpoll_pmin > ptot)
1969 inp->rxpoll_pmin = ptot;
1970 if (ptot > inp->rxpoll_pmax)
1971 inp->rxpoll_pmax = ptot;
1972
1973 /* Calculate EWMA of inbound packets */
1974 DLIL_EWMA(inp->rxpoll_pavg, ptot, if_rxpoll_decay);
1975
1976 /* Reset sampling statistics */
1977 PKTCNTR_CLEAR(&inp->sstats);
1978
1979 /* Calculate EWMA of wakeup requests */
1980 DLIL_EWMA(inp->rxpoll_wavg, inp->wtot, if_rxpoll_decay);
1981 inp->wtot = 0;
1982
1983 if (dlil_verbose) {
1984 if (!net_timerisset(&inp->dbg_lasttime))
1985 *(&inp->dbg_lasttime) = *(&now);
1986 net_timersub(&now, &inp->dbg_lasttime, &delta);
1987 if (net_timercmp(&delta, &dlil_dbgrate, >=)) {
1988 *(&inp->dbg_lasttime) = *(&now);
1989 printf("%s: [%s] pkts avg %d max %d "
1990 "limits [%d/%d], wreq avg %d "
1991 "limits [%d/%d], bytes avg %d "
1992 "limits [%d/%d]\n", if_name(ifp),
1993 (inp->mode ==
1994 IFNET_MODEL_INPUT_POLL_ON) ?
1995 "ON" : "OFF", inp->rxpoll_pavg,
1996 inp->rxpoll_pmax,
1997 inp->rxpoll_plowat,
1998 inp->rxpoll_phiwat,
1999 inp->rxpoll_wavg,
2000 inp->rxpoll_wlowat,
2001 inp->rxpoll_whiwat,
2002 inp->rxpoll_bavg,
2003 inp->rxpoll_blowat,
2004 inp->rxpoll_bhiwat);
2005 }
2006 }
2007
2008 /* Perform mode transition, if necessary */
2009 if (!net_timerisset(&inp->mode_lasttime))
2010 *(&inp->mode_lasttime) = *(&now);
2011
2012 net_timersub(&now, &inp->mode_lasttime, &delta);
2013 if (net_timercmp(&delta, &inp->mode_holdtime, <))
2014 goto skip;
2015
2016 if (inp->rxpoll_pavg <= inp->rxpoll_plowat &&
2017 inp->rxpoll_bavg <= inp->rxpoll_blowat &&
2018 inp->mode != IFNET_MODEL_INPUT_POLL_OFF) {
2019 mode = IFNET_MODEL_INPUT_POLL_OFF;
2020 } else if (inp->rxpoll_pavg >= inp->rxpoll_phiwat &&
2021 (inp->rxpoll_bavg >= inp->rxpoll_bhiwat ||
2022 inp->rxpoll_wavg >= inp->rxpoll_whiwat) &&
2023 inp->mode != IFNET_MODEL_INPUT_POLL_ON) {
2024 mode = IFNET_MODEL_INPUT_POLL_ON;
2025 }
2026
2027 if (mode != inp->mode) {
2028 inp->mode = mode;
2029 *(&inp->mode_lasttime) = *(&now);
2030 poll_req++;
2031 }
2032 }
2033 skip:
2034 dlil_input_stats_sync(ifp, inp);
2035
2036 lck_mtx_unlock(&inp->input_lck);
2037
2038 /*
2039 * If there's a mode change and interface is still attached,
2040 * perform a downcall to the driver for the new mode. Also
2041 * hold an IO refcnt on the interface to prevent it from
2042 * being detached (will be release below.)
2043 */
2044 if (poll_req != 0 && ifnet_is_attached(ifp, 1)) {
2045 struct ifnet_model_params p = { mode, { 0 } };
2046 errno_t err;
2047
2048 if (dlil_verbose) {
2049 printf("%s: polling is now %s, "
2050 "pkts avg %d max %d limits [%d/%d], "
2051 "wreq avg %d limits [%d/%d], "
2052 "bytes avg %d limits [%d/%d]\n",
2053 if_name(ifp),
2054 (mode == IFNET_MODEL_INPUT_POLL_ON) ?
2055 "ON" : "OFF", inp->rxpoll_pavg,
2056 inp->rxpoll_pmax, inp->rxpoll_plowat,
2057 inp->rxpoll_phiwat, inp->rxpoll_wavg,
2058 inp->rxpoll_wlowat, inp->rxpoll_whiwat,
2059 inp->rxpoll_bavg, inp->rxpoll_blowat,
2060 inp->rxpoll_bhiwat);
2061 }
2062
2063 if ((err = ((*ifp->if_input_ctl)(ifp,
2064 IFNET_CTL_SET_INPUT_MODEL, sizeof (p), &p))) != 0) {
2065 printf("%s: error setting polling mode "
2066 "to %s (%d)\n", if_name(ifp),
2067 (mode == IFNET_MODEL_INPUT_POLL_ON) ?
2068 "ON" : "OFF", err);
2069 }
2070
2071 switch (mode) {
2072 case IFNET_MODEL_INPUT_POLL_OFF:
2073 ifnet_set_poll_cycle(ifp, NULL);
2074 inp->rxpoll_offreq++;
2075 if (err != 0)
2076 inp->rxpoll_offerr++;
2077 break;
2078
2079 case IFNET_MODEL_INPUT_POLL_ON:
2080 net_nsectimer(&ival, &ts);
2081 ifnet_set_poll_cycle(ifp, &ts);
2082 ifnet_poll(ifp);
2083 inp->rxpoll_onreq++;
2084 if (err != 0)
2085 inp->rxpoll_onerr++;
2086 break;
2087
2088 default:
2089 VERIFY(0);
2090 /* NOTREACHED */
2091 }
2092
2093 /* Release the IO refcnt */
2094 ifnet_decr_iorefcnt(ifp);
2095 }
2096
2097 /*
2098 * NOTE warning %%% attention !!!!
2099 * We should think about putting some thread starvation
2100 * safeguards if we deal with long chains of packets.
2101 */
2102 if (m != NULL)
2103 dlil_input_packet_list_extended(NULL, m, m_cnt, mode);
2104 }
2105
2106 /* NOTREACHED */
2107 VERIFY(0); /* we should never get here */
2108 }
2109
2110 /*
2111 * Must be called on an attached ifnet (caller is expected to check.)
2112 * Caller may pass NULL for poll parameters to indicate "auto-tuning."
2113 */
2114 errno_t
2115 dlil_rxpoll_set_params(struct ifnet *ifp, struct ifnet_poll_params *p,
2116 boolean_t locked)
2117 {
2118 struct dlil_threading_info *inp;
2119 u_int64_t sample_holdtime, inbw;
2120
2121 VERIFY(ifp != NULL);
2122 if (!(ifp->if_eflags & IFEF_RXPOLL) || (inp = ifp->if_inp) == NULL)
2123 return (ENXIO);
2124
2125 if (p != NULL) {
2126 if ((p->packets_lowat == 0 && p->packets_hiwat != 0) ||
2127 (p->packets_lowat != 0 && p->packets_hiwat == 0))
2128 return (EINVAL);
2129 if (p->packets_lowat != 0 && /* hiwat must be non-zero */
2130 p->packets_lowat >= p->packets_hiwat)
2131 return (EINVAL);
2132 if ((p->bytes_lowat == 0 && p->bytes_hiwat != 0) ||
2133 (p->bytes_lowat != 0 && p->bytes_hiwat == 0))
2134 return (EINVAL);
2135 if (p->bytes_lowat != 0 && /* hiwat must be non-zero */
2136 p->bytes_lowat >= p->bytes_hiwat)
2137 return (EINVAL);
2138 if (p->interval_time != 0 &&
2139 p->interval_time < IF_RXPOLL_INTERVALTIME_MIN)
2140 p->interval_time = IF_RXPOLL_INTERVALTIME_MIN;
2141 }
2142
2143 if (!locked)
2144 lck_mtx_lock(&inp->input_lck);
2145
2146 lck_mtx_assert(&inp->input_lck, LCK_MTX_ASSERT_OWNED);
2147
2148 /*
2149 * Normally, we'd reset the parameters to the auto-tuned values
2150 * if the the input thread detects a change in link rate. If the
2151 * driver provides its own parameters right after a link rate
2152 * changes, but before the input thread gets to run, we want to
2153 * make sure to keep the driver's values. Clearing if_poll_update
2154 * will achieve that.
2155 */
2156 if (p != NULL && !locked && ifp->if_poll_update != 0)
2157 ifp->if_poll_update = 0;
2158
2159 if ((inbw = ifnet_input_linkrate(ifp)) == 0 && p == NULL) {
2160 sample_holdtime = 0; /* polling is disabled */
2161 inp->rxpoll_wlowat = inp->rxpoll_plowat =
2162 inp->rxpoll_blowat = 0;
2163 inp->rxpoll_whiwat = inp->rxpoll_phiwat =
2164 inp->rxpoll_bhiwat = (u_int32_t)-1;
2165 inp->rxpoll_plim = 0;
2166 inp->rxpoll_ival = IF_RXPOLL_INTERVALTIME_MIN;
2167 } else {
2168 u_int32_t plowat, phiwat, blowat, bhiwat, plim;
2169 u_int64_t ival;
2170 unsigned int n, i;
2171
2172 for (n = 0, i = 0; rxpoll_tbl[i].speed != 0; i++) {
2173 if (inbw < rxpoll_tbl[i].speed)
2174 break;
2175 n = i;
2176 }
2177 /* auto-tune if caller didn't specify a value */
2178 plowat = ((p == NULL || p->packets_lowat == 0) ?
2179 rxpoll_tbl[n].plowat : p->packets_lowat);
2180 phiwat = ((p == NULL || p->packets_hiwat == 0) ?
2181 rxpoll_tbl[n].phiwat : p->packets_hiwat);
2182 blowat = ((p == NULL || p->bytes_lowat == 0) ?
2183 rxpoll_tbl[n].blowat : p->bytes_lowat);
2184 bhiwat = ((p == NULL || p->bytes_hiwat == 0) ?
2185 rxpoll_tbl[n].bhiwat : p->bytes_hiwat);
2186 plim = ((p == NULL || p->packets_limit == 0) ?
2187 if_rxpoll_max : p->packets_limit);
2188 ival = ((p == NULL || p->interval_time == 0) ?
2189 if_rxpoll_interval_time : p->interval_time);
2190
2191 VERIFY(plowat != 0 && phiwat != 0);
2192 VERIFY(blowat != 0 && bhiwat != 0);
2193 VERIFY(ival >= IF_RXPOLL_INTERVALTIME_MIN);
2194
2195 sample_holdtime = if_rxpoll_sample_holdtime;
2196 inp->rxpoll_wlowat = if_rxpoll_wlowat;
2197 inp->rxpoll_whiwat = if_rxpoll_whiwat;
2198 inp->rxpoll_plowat = plowat;
2199 inp->rxpoll_phiwat = phiwat;
2200 inp->rxpoll_blowat = blowat;
2201 inp->rxpoll_bhiwat = bhiwat;
2202 inp->rxpoll_plim = plim;
2203 inp->rxpoll_ival = ival;
2204 }
2205
2206 net_nsectimer(&if_rxpoll_mode_holdtime, &inp->mode_holdtime);
2207 net_nsectimer(&sample_holdtime, &inp->sample_holdtime);
2208
2209 if (dlil_verbose) {
2210 printf("%s: speed %llu bps, sample per %llu nsec, "
2211 "poll interval %llu nsec, pkts per poll %u, "
2212 "pkt limits [%u/%u], wreq limits [%u/%u], "
2213 "bytes limits [%u/%u]\n", if_name(ifp),
2214 inbw, sample_holdtime, inp->rxpoll_ival, inp->rxpoll_plim,
2215 inp->rxpoll_plowat, inp->rxpoll_phiwat, inp->rxpoll_wlowat,
2216 inp->rxpoll_whiwat, inp->rxpoll_blowat, inp->rxpoll_bhiwat);
2217 }
2218
2219 if (!locked)
2220 lck_mtx_unlock(&inp->input_lck);
2221
2222 return (0);
2223 }
2224
2225 /*
2226 * Must be called on an attached ifnet (caller is expected to check.)
2227 */
2228 errno_t
2229 dlil_rxpoll_get_params(struct ifnet *ifp, struct ifnet_poll_params *p)
2230 {
2231 struct dlil_threading_info *inp;
2232
2233 VERIFY(ifp != NULL && p != NULL);
2234 if (!(ifp->if_eflags & IFEF_RXPOLL) || (inp = ifp->if_inp) == NULL)
2235 return (ENXIO);
2236
2237 bzero(p, sizeof (*p));
2238
2239 lck_mtx_lock(&inp->input_lck);
2240 p->packets_limit = inp->rxpoll_plim;
2241 p->packets_lowat = inp->rxpoll_plowat;
2242 p->packets_hiwat = inp->rxpoll_phiwat;
2243 p->bytes_lowat = inp->rxpoll_blowat;
2244 p->bytes_hiwat = inp->rxpoll_bhiwat;
2245 p->interval_time = inp->rxpoll_ival;
2246 lck_mtx_unlock(&inp->input_lck);
2247
2248 return (0);
2249 }
2250
2251 errno_t
2252 ifnet_input(struct ifnet *ifp, struct mbuf *m_head,
2253 const struct ifnet_stat_increment_param *s)
2254 {
2255 return (ifnet_input_common(ifp, m_head, NULL, s, FALSE, FALSE));
2256 }
2257
2258 errno_t
2259 ifnet_input_extended(struct ifnet *ifp, struct mbuf *m_head,
2260 struct mbuf *m_tail, const struct ifnet_stat_increment_param *s)
2261 {
2262 return (ifnet_input_common(ifp, m_head, m_tail, s, TRUE, FALSE));
2263 }
2264
2265 static errno_t
2266 ifnet_input_common(struct ifnet *ifp, struct mbuf *m_head, struct mbuf *m_tail,
2267 const struct ifnet_stat_increment_param *s, boolean_t ext, boolean_t poll)
2268 {
2269 struct thread *tp = current_thread();
2270 struct mbuf *last;
2271 struct dlil_threading_info *inp;
2272 u_int32_t m_cnt = 0, m_size = 0;
2273
2274 if ((m_head == NULL && !poll) || (s == NULL && ext)) {
2275 if (m_head != NULL)
2276 mbuf_freem_list(m_head);
2277 return (EINVAL);
2278 }
2279
2280 VERIFY(m_head != NULL || (s == NULL && m_tail == NULL && !ext && poll));
2281 VERIFY(m_tail == NULL || ext);
2282 VERIFY(s != NULL || !ext);
2283
2284 /*
2285 * Drop the packet(s) if the parameters are invalid, or if the
2286 * interface is no longer attached; else hold an IO refcnt to
2287 * prevent it from being detached (will be released below.)
2288 */
2289 if (ifp == NULL || (ifp != lo_ifp && !ifnet_is_attached(ifp, 1))) {
2290 if (m_head != NULL)
2291 mbuf_freem_list(m_head);
2292 return (EINVAL);
2293 }
2294
2295 if (m_tail == NULL) {
2296 last = m_head;
2297 while (m_head != NULL) {
2298 #if IFNET_INPUT_SANITY_CHK
2299 if (dlil_input_sanity_check != 0)
2300 DLIL_INPUT_CHECK(last, ifp);
2301 #endif /* IFNET_INPUT_SANITY_CHK */
2302 m_cnt++;
2303 m_size += m_length(last);
2304 if (mbuf_nextpkt(last) == NULL)
2305 break;
2306 last = mbuf_nextpkt(last);
2307 }
2308 m_tail = last;
2309 } else {
2310 #if IFNET_INPUT_SANITY_CHK
2311 if (dlil_input_sanity_check != 0) {
2312 last = m_head;
2313 while (1) {
2314 DLIL_INPUT_CHECK(last, ifp);
2315 m_cnt++;
2316 m_size += m_length(last);
2317 if (mbuf_nextpkt(last) == NULL)
2318 break;
2319 last = mbuf_nextpkt(last);
2320 }
2321 } else {
2322 m_cnt = s->packets_in;
2323 m_size = s->bytes_in;
2324 last = m_tail;
2325 }
2326 #else
2327 m_cnt = s->packets_in;
2328 m_size = s->bytes_in;
2329 last = m_tail;
2330 #endif /* IFNET_INPUT_SANITY_CHK */
2331 }
2332
2333 if (last != m_tail) {
2334 panic_plain("%s: invalid input packet chain for %s, "
2335 "tail mbuf %p instead of %p\n", __func__, if_name(ifp),
2336 m_tail, last);
2337 }
2338
2339 /*
2340 * Assert packet count only for the extended variant, for backwards
2341 * compatibility, since this came directly from the device driver.
2342 * Relax this assertion for input bytes, as the driver may have
2343 * included the link-layer headers in the computation; hence
2344 * m_size is just an approximation.
2345 */
2346 if (ext && s->packets_in != m_cnt) {
2347 panic_plain("%s: input packet count mismatch for %s, "
2348 "%d instead of %d\n", __func__, if_name(ifp),
2349 s->packets_in, m_cnt);
2350 }
2351
2352 if ((inp = ifp->if_inp) == NULL)
2353 inp = dlil_main_input_thread;
2354
2355 /*
2356 * If there is a matching DLIL input thread associated with an
2357 * affinity set, associate this thread with the same set. We
2358 * will only do this once.
2359 */
2360 lck_mtx_lock_spin(&inp->input_lck);
2361 if (inp != dlil_main_input_thread && inp->net_affinity &&
2362 ((!poll && inp->wloop_thr == THREAD_NULL) ||
2363 (poll && inp->poll_thr == THREAD_NULL))) {
2364 u_int32_t tag = inp->tag;
2365
2366 if (poll) {
2367 VERIFY(inp->poll_thr == THREAD_NULL);
2368 inp->poll_thr = tp;
2369 } else {
2370 VERIFY(inp->wloop_thr == THREAD_NULL);
2371 inp->wloop_thr = tp;
2372 }
2373 lck_mtx_unlock(&inp->input_lck);
2374
2375 /* Associate the current thread with the new affinity tag */
2376 (void) dlil_affinity_set(tp, tag);
2377
2378 /*
2379 * Take a reference on the current thread; during detach,
2380 * we will need to refer to it in order ot tear down its
2381 * affinity.
2382 */
2383 thread_reference(tp);
2384 lck_mtx_lock_spin(&inp->input_lck);
2385 }
2386
2387 VERIFY(m_head != NULL || (m_tail == NULL && m_cnt == 0));
2388
2389 /*
2390 * Because of loopbacked multicast we cannot stuff the ifp in
2391 * the rcvif of the packet header: loopback (lo0) packets use a
2392 * dedicated list so that we can later associate them with lo_ifp
2393 * on their way up the stack. Packets for other interfaces without
2394 * dedicated input threads go to the regular list.
2395 */
2396 if (m_head != NULL) {
2397 if (inp == dlil_main_input_thread && ifp == lo_ifp) {
2398 struct dlil_main_threading_info *inpm =
2399 (struct dlil_main_threading_info *)inp;
2400 _addq_multi(&inpm->lo_rcvq_pkts, m_head, m_tail,
2401 m_cnt, m_size);
2402 } else {
2403 _addq_multi(&inp->rcvq_pkts, m_head, m_tail,
2404 m_cnt, m_size);
2405 }
2406 }
2407
2408 #if IFNET_INPUT_SANITY_CHK
2409 if (dlil_input_sanity_check != 0) {
2410 u_int32_t count;
2411 struct mbuf *m0;
2412
2413 for (m0 = m_head, count = 0; m0; m0 = mbuf_nextpkt(m0))
2414 count++;
2415
2416 if (count != m_cnt) {
2417 panic_plain("%s: invalid packet count %d "
2418 "(expected %d)\n", if_name(ifp),
2419 count, m_cnt);
2420 /* NOTREACHED */
2421 }
2422
2423 inp->input_mbuf_cnt += m_cnt;
2424 }
2425 #endif /* IFNET_INPUT_SANITY_CHK */
2426
2427 if (s != NULL) {
2428 dlil_input_stats_add(s, inp, poll);
2429 /*
2430 * If we're using the main input thread, synchronize the
2431 * stats now since we have the interface context. All
2432 * other cases involving dedicated input threads will
2433 * have their stats synchronized there.
2434 */
2435 if (inp == dlil_main_input_thread)
2436 dlil_input_stats_sync(ifp, inp);
2437 }
2438
2439 inp->input_waiting |= DLIL_INPUT_WAITING;
2440 if (!(inp->input_waiting & DLIL_INPUT_RUNNING)) {
2441 inp->wtot++;
2442 wakeup_one((caddr_t)&inp->input_waiting);
2443 }
2444 lck_mtx_unlock(&inp->input_lck);
2445
2446 if (ifp != lo_ifp) {
2447 /* Release the IO refcnt */
2448 ifnet_decr_iorefcnt(ifp);
2449 }
2450
2451 return (0);
2452 }
2453
2454 static void
2455 ifnet_start_common(struct ifnet *ifp, int resetfc)
2456 {
2457 if (!(ifp->if_eflags & IFEF_TXSTART))
2458 return;
2459 /*
2460 * If the starter thread is inactive, signal it to do work,
2461 * unless the interface is being flow controlled from below,
2462 * e.g. a virtual interface being flow controlled by a real
2463 * network interface beneath it.
2464 */
2465 lck_mtx_lock_spin(&ifp->if_start_lock);
2466 if (resetfc) {
2467 ifp->if_start_flags &= ~IFSF_FLOW_CONTROLLED;
2468 } else if (ifp->if_start_flags & IFSF_FLOW_CONTROLLED) {
2469 lck_mtx_unlock(&ifp->if_start_lock);
2470 return;
2471 }
2472 ifp->if_start_req++;
2473 if (!ifp->if_start_active && ifp->if_start_thread != THREAD_NULL) {
2474 wakeup_one((caddr_t)&ifp->if_start_thread);
2475 }
2476 lck_mtx_unlock(&ifp->if_start_lock);
2477 }
2478
2479 void
2480 ifnet_start(struct ifnet *ifp)
2481 {
2482 ifnet_start_common(ifp, 0);
2483 }
2484
2485 static void
2486 ifnet_start_thread_fn(void *v, wait_result_t w)
2487 {
2488 #pragma unused(w)
2489 struct ifnet *ifp = v;
2490 char ifname[IFNAMSIZ + 1];
2491 struct timespec *ts = NULL;
2492 struct ifclassq *ifq = &ifp->if_snd;
2493
2494 /*
2495 * Treat the dedicated starter thread for lo0 as equivalent to
2496 * the driver workloop thread; if net_affinity is enabled for
2497 * the main input thread, associate this starter thread to it
2498 * by binding them with the same affinity tag. This is done
2499 * only once (as we only have one lo_ifp which never goes away.)
2500 */
2501 if (ifp == lo_ifp) {
2502 struct dlil_threading_info *inp = dlil_main_input_thread;
2503 struct thread *tp = current_thread();
2504
2505 lck_mtx_lock(&inp->input_lck);
2506 if (inp->net_affinity) {
2507 u_int32_t tag = inp->tag;
2508
2509 VERIFY(inp->wloop_thr == THREAD_NULL);
2510 VERIFY(inp->poll_thr == THREAD_NULL);
2511 inp->wloop_thr = tp;
2512 lck_mtx_unlock(&inp->input_lck);
2513
2514 /* Associate this thread with the affinity tag */
2515 (void) dlil_affinity_set(tp, tag);
2516 } else {
2517 lck_mtx_unlock(&inp->input_lck);
2518 }
2519 }
2520
2521 snprintf(ifname, sizeof (ifname), "%s_starter",
2522 if_name(ifp));
2523
2524 lck_mtx_lock_spin(&ifp->if_start_lock);
2525
2526 for (;;) {
2527 (void) msleep(&ifp->if_start_thread, &ifp->if_start_lock,
2528 (PZERO - 1) | PSPIN, ifname, ts);
2529
2530 /* interface is detached? */
2531 if (ifp->if_start_thread == THREAD_NULL) {
2532 ifnet_set_start_cycle(ifp, NULL);
2533 lck_mtx_unlock(&ifp->if_start_lock);
2534 ifnet_purge(ifp);
2535
2536 if (dlil_verbose) {
2537 printf("%s: starter thread terminated\n",
2538 if_name(ifp));
2539 }
2540
2541 /* for the extra refcnt from kernel_thread_start() */
2542 thread_deallocate(current_thread());
2543 /* this is the end */
2544 thread_terminate(current_thread());
2545 /* NOTREACHED */
2546 return;
2547 }
2548
2549 ifp->if_start_active = 1;
2550 for (;;) {
2551 u_int32_t req = ifp->if_start_req;
2552
2553 lck_mtx_unlock(&ifp->if_start_lock);
2554 /* invoke the driver's start routine */
2555 ((*ifp->if_start)(ifp));
2556 lck_mtx_lock_spin(&ifp->if_start_lock);
2557
2558 /* if there's no pending request, we're done */
2559 if (req == ifp->if_start_req)
2560 break;
2561 }
2562 ifp->if_start_req = 0;
2563 ifp->if_start_active = 0;
2564 /*
2565 * Wakeup N ns from now if rate-controlled by TBR, and if
2566 * there are still packets in the send queue which haven't
2567 * been dequeued so far; else sleep indefinitely (ts = NULL)
2568 * until ifnet_start() is called again.
2569 */
2570 ts = ((IFCQ_TBR_IS_ENABLED(ifq) && !IFCQ_IS_EMPTY(ifq)) ?
2571 &ifp->if_start_cycle : NULL);
2572
2573 if (ts != NULL && ts->tv_sec == 0 && ts->tv_nsec == 0)
2574 ts = NULL;
2575 }
2576
2577 /* NOTREACHED */
2578 lck_mtx_unlock(&ifp->if_start_lock);
2579 VERIFY(0); /* we should never get here */
2580 }
2581
2582 void
2583 ifnet_set_start_cycle(struct ifnet *ifp, struct timespec *ts)
2584 {
2585 if (ts == NULL)
2586 bzero(&ifp->if_start_cycle, sizeof (ifp->if_start_cycle));
2587 else
2588 *(&ifp->if_start_cycle) = *ts;
2589
2590 if (ts != NULL && ts->tv_nsec != 0 && dlil_verbose)
2591 printf("%s: restart interval set to %lu nsec\n",
2592 if_name(ifp), ts->tv_nsec);
2593 }
2594
2595 static void
2596 ifnet_poll(struct ifnet *ifp)
2597 {
2598 /*
2599 * If the poller thread is inactive, signal it to do work.
2600 */
2601 lck_mtx_lock_spin(&ifp->if_poll_lock);
2602 ifp->if_poll_req++;
2603 if (!ifp->if_poll_active && ifp->if_poll_thread != THREAD_NULL) {
2604 wakeup_one((caddr_t)&ifp->if_poll_thread);
2605 }
2606 lck_mtx_unlock(&ifp->if_poll_lock);
2607 }
2608
2609 static void
2610 ifnet_poll_thread_fn(void *v, wait_result_t w)
2611 {
2612 #pragma unused(w)
2613 struct dlil_threading_info *inp;
2614 struct ifnet *ifp = v;
2615 char ifname[IFNAMSIZ + 1];
2616 struct timespec *ts = NULL;
2617 struct ifnet_stat_increment_param s;
2618
2619 snprintf(ifname, sizeof (ifname), "%s_poller",
2620 if_name(ifp));
2621 bzero(&s, sizeof (s));
2622
2623 lck_mtx_lock_spin(&ifp->if_poll_lock);
2624
2625 inp = ifp->if_inp;
2626 VERIFY(inp != NULL);
2627
2628 for (;;) {
2629 if (ifp->if_poll_thread != THREAD_NULL) {
2630 (void) msleep(&ifp->if_poll_thread, &ifp->if_poll_lock,
2631 (PZERO - 1) | PSPIN, ifname, ts);
2632 }
2633
2634 /* interface is detached (maybe while asleep)? */
2635 if (ifp->if_poll_thread == THREAD_NULL) {
2636 ifnet_set_poll_cycle(ifp, NULL);
2637 lck_mtx_unlock(&ifp->if_poll_lock);
2638
2639 if (dlil_verbose) {
2640 printf("%s: poller thread terminated\n",
2641 if_name(ifp));
2642 }
2643
2644 /* for the extra refcnt from kernel_thread_start() */
2645 thread_deallocate(current_thread());
2646 /* this is the end */
2647 thread_terminate(current_thread());
2648 /* NOTREACHED */
2649 return;
2650 }
2651
2652 ifp->if_poll_active = 1;
2653 for (;;) {
2654 struct mbuf *m_head, *m_tail;
2655 u_int32_t m_lim, m_cnt, m_totlen;
2656 u_int16_t req = ifp->if_poll_req;
2657
2658 lck_mtx_unlock(&ifp->if_poll_lock);
2659
2660 /*
2661 * If no longer attached, there's nothing to do;
2662 * else hold an IO refcnt to prevent the interface
2663 * from being detached (will be released below.)
2664 */
2665 if (!ifnet_is_attached(ifp, 1)) {
2666 lck_mtx_lock_spin(&ifp->if_poll_lock);
2667 break;
2668 }
2669
2670 m_lim = (inp->rxpoll_plim != 0) ? inp->rxpoll_plim :
2671 MAX((qlimit(&inp->rcvq_pkts)),
2672 (inp->rxpoll_phiwat << 2));
2673
2674 if (dlil_verbose > 1) {
2675 printf("%s: polling up to %d pkts, "
2676 "pkts avg %d max %d, wreq avg %d, "
2677 "bytes avg %d\n",
2678 if_name(ifp), m_lim,
2679 inp->rxpoll_pavg, inp->rxpoll_pmax,
2680 inp->rxpoll_wavg, inp->rxpoll_bavg);
2681 }
2682
2683 /* invoke the driver's input poll routine */
2684 ((*ifp->if_input_poll)(ifp, 0, m_lim, &m_head, &m_tail,
2685 &m_cnt, &m_totlen));
2686
2687 if (m_head != NULL) {
2688 VERIFY(m_tail != NULL && m_cnt > 0);
2689
2690 if (dlil_verbose > 1) {
2691 printf("%s: polled %d pkts, "
2692 "pkts avg %d max %d, wreq avg %d, "
2693 "bytes avg %d\n",
2694 if_name(ifp), m_cnt,
2695 inp->rxpoll_pavg, inp->rxpoll_pmax,
2696 inp->rxpoll_wavg, inp->rxpoll_bavg);
2697 }
2698
2699 /* stats are required for extended variant */
2700 s.packets_in = m_cnt;
2701 s.bytes_in = m_totlen;
2702
2703 (void) ifnet_input_common(ifp, m_head, m_tail,
2704 &s, TRUE, TRUE);
2705 } else {
2706 if (dlil_verbose > 1) {
2707 printf("%s: no packets, "
2708 "pkts avg %d max %d, wreq avg %d, "
2709 "bytes avg %d\n",
2710 if_name(ifp), inp->rxpoll_pavg,
2711 inp->rxpoll_pmax, inp->rxpoll_wavg,
2712 inp->rxpoll_bavg);
2713 }
2714
2715 (void) ifnet_input_common(ifp, NULL, NULL,
2716 NULL, FALSE, TRUE);
2717 }
2718
2719 /* Release the io ref count */
2720 ifnet_decr_iorefcnt(ifp);
2721
2722 lck_mtx_lock_spin(&ifp->if_poll_lock);
2723
2724 /* if there's no pending request, we're done */
2725 if (req == ifp->if_poll_req)
2726 break;
2727 }
2728 ifp->if_poll_req = 0;
2729 ifp->if_poll_active = 0;
2730
2731 /*
2732 * Wakeup N ns from now, else sleep indefinitely (ts = NULL)
2733 * until ifnet_poll() is called again.
2734 */
2735 ts = &ifp->if_poll_cycle;
2736 if (ts->tv_sec == 0 && ts->tv_nsec == 0)
2737 ts = NULL;
2738 }
2739
2740 /* NOTREACHED */
2741 lck_mtx_unlock(&ifp->if_poll_lock);
2742 VERIFY(0); /* we should never get here */
2743 }
2744
2745 void
2746 ifnet_set_poll_cycle(struct ifnet *ifp, struct timespec *ts)
2747 {
2748 if (ts == NULL)
2749 bzero(&ifp->if_poll_cycle, sizeof (ifp->if_poll_cycle));
2750 else
2751 *(&ifp->if_poll_cycle) = *ts;
2752
2753 if (ts != NULL && ts->tv_nsec != 0 && dlil_verbose)
2754 printf("%s: poll interval set to %lu nsec\n",
2755 if_name(ifp), ts->tv_nsec);
2756 }
2757
2758 void
2759 ifnet_purge(struct ifnet *ifp)
2760 {
2761 if (ifp != NULL && (ifp->if_eflags & IFEF_TXSTART))
2762 if_qflush(ifp, 0);
2763 }
2764
2765 void
2766 ifnet_update_sndq(struct ifclassq *ifq, cqev_t ev)
2767 {
2768 IFCQ_LOCK_ASSERT_HELD(ifq);
2769
2770 if (!(IFCQ_IS_READY(ifq)))
2771 return;
2772
2773 if (IFCQ_TBR_IS_ENABLED(ifq)) {
2774 struct tb_profile tb = { ifq->ifcq_tbr.tbr_rate_raw,
2775 ifq->ifcq_tbr.tbr_percent, 0 };
2776 (void) ifclassq_tbr_set(ifq, &tb, FALSE);
2777 }
2778
2779 ifclassq_update(ifq, ev);
2780 }
2781
2782 void
2783 ifnet_update_rcv(struct ifnet *ifp, cqev_t ev)
2784 {
2785 switch (ev) {
2786 case CLASSQ_EV_LINK_BANDWIDTH:
2787 if (net_rxpoll && (ifp->if_eflags & IFEF_RXPOLL))
2788 ifp->if_poll_update++;
2789 break;
2790
2791 default:
2792 break;
2793 }
2794 }
2795
2796 errno_t
2797 ifnet_set_output_sched_model(struct ifnet *ifp, u_int32_t model)
2798 {
2799 struct ifclassq *ifq;
2800 u_int32_t omodel;
2801 errno_t err;
2802
2803 if (ifp == NULL || (model != IFNET_SCHED_MODEL_DRIVER_MANAGED &&
2804 model != IFNET_SCHED_MODEL_NORMAL))
2805 return (EINVAL);
2806 else if (!(ifp->if_eflags & IFEF_TXSTART))
2807 return (ENXIO);
2808
2809 ifq = &ifp->if_snd;
2810 IFCQ_LOCK(ifq);
2811 omodel = ifp->if_output_sched_model;
2812 ifp->if_output_sched_model = model;
2813 if ((err = ifclassq_pktsched_setup(ifq)) != 0)
2814 ifp->if_output_sched_model = omodel;
2815 IFCQ_UNLOCK(ifq);
2816
2817 return (err);
2818 }
2819
2820 errno_t
2821 ifnet_set_sndq_maxlen(struct ifnet *ifp, u_int32_t maxqlen)
2822 {
2823 if (ifp == NULL)
2824 return (EINVAL);
2825 else if (!(ifp->if_eflags & IFEF_TXSTART))
2826 return (ENXIO);
2827
2828 ifclassq_set_maxlen(&ifp->if_snd, maxqlen);
2829
2830 return (0);
2831 }
2832
2833 errno_t
2834 ifnet_get_sndq_maxlen(struct ifnet *ifp, u_int32_t *maxqlen)
2835 {
2836 if (ifp == NULL || maxqlen == NULL)
2837 return (EINVAL);
2838 else if (!(ifp->if_eflags & IFEF_TXSTART))
2839 return (ENXIO);
2840
2841 *maxqlen = ifclassq_get_maxlen(&ifp->if_snd);
2842
2843 return (0);
2844 }
2845
2846 errno_t
2847 ifnet_get_sndq_len(struct ifnet *ifp, u_int32_t *pkts)
2848 {
2849 errno_t err;
2850
2851 if (ifp == NULL || pkts == NULL)
2852 err = EINVAL;
2853 else if (!(ifp->if_eflags & IFEF_TXSTART))
2854 err = ENXIO;
2855 else
2856 err = ifclassq_get_len(&ifp->if_snd, MBUF_SC_UNSPEC,
2857 pkts, NULL);
2858
2859 return (err);
2860 }
2861
2862 errno_t
2863 ifnet_get_service_class_sndq_len(struct ifnet *ifp, mbuf_svc_class_t sc,
2864 u_int32_t *pkts, u_int32_t *bytes)
2865 {
2866 errno_t err;
2867
2868 if (ifp == NULL || !MBUF_VALID_SC(sc) ||
2869 (pkts == NULL && bytes == NULL))
2870 err = EINVAL;
2871 else if (!(ifp->if_eflags & IFEF_TXSTART))
2872 err = ENXIO;
2873 else
2874 err = ifclassq_get_len(&ifp->if_snd, sc, pkts, bytes);
2875
2876 return (err);
2877 }
2878
2879 errno_t
2880 ifnet_set_rcvq_maxlen(struct ifnet *ifp, u_int32_t maxqlen)
2881 {
2882 struct dlil_threading_info *inp;
2883
2884 if (ifp == NULL)
2885 return (EINVAL);
2886 else if (!(ifp->if_eflags & IFEF_RXPOLL) || ifp->if_inp == NULL)
2887 return (ENXIO);
2888
2889 if (maxqlen == 0)
2890 maxqlen = if_rcvq_maxlen;
2891 else if (maxqlen < IF_RCVQ_MINLEN)
2892 maxqlen = IF_RCVQ_MINLEN;
2893
2894 inp = ifp->if_inp;
2895 lck_mtx_lock(&inp->input_lck);
2896 qlimit(&inp->rcvq_pkts) = maxqlen;
2897 lck_mtx_unlock(&inp->input_lck);
2898
2899 return (0);
2900 }
2901
2902 errno_t
2903 ifnet_get_rcvq_maxlen(struct ifnet *ifp, u_int32_t *maxqlen)
2904 {
2905 struct dlil_threading_info *inp;
2906
2907 if (ifp == NULL || maxqlen == NULL)
2908 return (EINVAL);
2909 else if (!(ifp->if_eflags & IFEF_RXPOLL) || ifp->if_inp == NULL)
2910 return (ENXIO);
2911
2912 inp = ifp->if_inp;
2913 lck_mtx_lock(&inp->input_lck);
2914 *maxqlen = qlimit(&inp->rcvq_pkts);
2915 lck_mtx_unlock(&inp->input_lck);
2916 return (0);
2917 }
2918
2919 errno_t
2920 ifnet_enqueue(struct ifnet *ifp, struct mbuf *m)
2921 {
2922 int error;
2923
2924 if (ifp == NULL || m == NULL || !(m->m_flags & M_PKTHDR) ||
2925 m->m_nextpkt != NULL) {
2926 if (m != NULL)
2927 m_freem_list(m);
2928 return (EINVAL);
2929 } else if (!(ifp->if_eflags & IFEF_TXSTART) ||
2930 !(ifp->if_refflags & IFRF_ATTACHED)) {
2931 /* flag tested without lock for performance */
2932 m_freem(m);
2933 return (ENXIO);
2934 } else if (!(ifp->if_flags & IFF_UP)) {
2935 m_freem(m);
2936 return (ENETDOWN);
2937 }
2938
2939 /* enqueue the packet */
2940 error = ifclassq_enqueue(&ifp->if_snd, m);
2941
2942 /*
2943 * Tell the driver to start dequeueing; do this even when the queue
2944 * for the packet is suspended (EQSUSPENDED), as the driver could still
2945 * be dequeueing from other unsuspended queues.
2946 */
2947 if (error == 0 || error == EQFULL || error == EQSUSPENDED)
2948 ifnet_start(ifp);
2949
2950 return (error);
2951 }
2952
2953 errno_t
2954 ifnet_dequeue(struct ifnet *ifp, struct mbuf **mp)
2955 {
2956 if (ifp == NULL || mp == NULL)
2957 return (EINVAL);
2958 else if (!(ifp->if_eflags & IFEF_TXSTART) ||
2959 (ifp->if_output_sched_model != IFNET_SCHED_MODEL_NORMAL))
2960 return (ENXIO);
2961
2962 return (ifclassq_dequeue(&ifp->if_snd, 1, mp, NULL, NULL, NULL));
2963 }
2964
2965 errno_t
2966 ifnet_dequeue_service_class(struct ifnet *ifp, mbuf_svc_class_t sc,
2967 struct mbuf **mp)
2968 {
2969 if (ifp == NULL || mp == NULL || !MBUF_VALID_SC(sc))
2970 return (EINVAL);
2971 else if (!(ifp->if_eflags & IFEF_TXSTART) ||
2972 (ifp->if_output_sched_model != IFNET_SCHED_MODEL_DRIVER_MANAGED))
2973 return (ENXIO);
2974
2975 return (ifclassq_dequeue_sc(&ifp->if_snd, sc, 1, mp, NULL, NULL, NULL));
2976 }
2977
2978 errno_t
2979 ifnet_dequeue_multi(struct ifnet *ifp, u_int32_t limit, struct mbuf **head,
2980 struct mbuf **tail, u_int32_t *cnt, u_int32_t *len)
2981 {
2982 if (ifp == NULL || head == NULL || limit < 1)
2983 return (EINVAL);
2984 else if (!(ifp->if_eflags & IFEF_TXSTART) ||
2985 (ifp->if_output_sched_model != IFNET_SCHED_MODEL_NORMAL))
2986 return (ENXIO);
2987
2988 return (ifclassq_dequeue(&ifp->if_snd, limit, head, tail, cnt, len));
2989 }
2990
2991 errno_t
2992 ifnet_dequeue_service_class_multi(struct ifnet *ifp, mbuf_svc_class_t sc,
2993 u_int32_t limit, struct mbuf **head, struct mbuf **tail, u_int32_t *cnt,
2994 u_int32_t *len)
2995 {
2996
2997 if (ifp == NULL || head == NULL || limit < 1 || !MBUF_VALID_SC(sc))
2998 return (EINVAL);
2999 else if (!(ifp->if_eflags & IFEF_TXSTART) ||
3000 (ifp->if_output_sched_model != IFNET_SCHED_MODEL_DRIVER_MANAGED))
3001 return (ENXIO);
3002
3003 return (ifclassq_dequeue_sc(&ifp->if_snd, sc, limit, head,
3004 tail, cnt, len));
3005 }
3006
3007 errno_t
3008 ifnet_framer_stub(struct ifnet *ifp, struct mbuf **m,
3009 const struct sockaddr *dest, const char *dest_linkaddr,
3010 const char *frame_type, u_int32_t *pre, u_int32_t *post)
3011 {
3012 if (pre != NULL)
3013 *pre = 0;
3014 if (post != NULL)
3015 *post = 0;
3016
3017 return (ifp->if_framer_legacy(ifp, m, dest, dest_linkaddr, frame_type));
3018 }
3019
3020 static int
3021 dlil_interface_filters_input(struct ifnet *ifp, struct mbuf **m_p,
3022 char **frame_header_p, protocol_family_t protocol_family)
3023 {
3024 struct ifnet_filter *filter;
3025
3026 /*
3027 * Pass the inbound packet to the interface filters
3028 */
3029 lck_mtx_lock_spin(&ifp->if_flt_lock);
3030 /* prevent filter list from changing in case we drop the lock */
3031 if_flt_monitor_busy(ifp);
3032 TAILQ_FOREACH(filter, &ifp->if_flt_head, filt_next) {
3033 int result;
3034
3035 if (!filter->filt_skip && filter->filt_input != NULL &&
3036 (filter->filt_protocol == 0 ||
3037 filter->filt_protocol == protocol_family)) {
3038 lck_mtx_unlock(&ifp->if_flt_lock);
3039
3040 result = (*filter->filt_input)(filter->filt_cookie,
3041 ifp, protocol_family, m_p, frame_header_p);
3042
3043 lck_mtx_lock_spin(&ifp->if_flt_lock);
3044 if (result != 0) {
3045 /* we're done with the filter list */
3046 if_flt_monitor_unbusy(ifp);
3047 lck_mtx_unlock(&ifp->if_flt_lock);
3048 return (result);
3049 }
3050 }
3051 }
3052 /* we're done with the filter list */
3053 if_flt_monitor_unbusy(ifp);
3054 lck_mtx_unlock(&ifp->if_flt_lock);
3055
3056 /*
3057 * Strip away M_PROTO1 bit prior to sending packet up the stack as
3058 * it is meant to be local to a subsystem -- if_bridge for M_PROTO1
3059 */
3060 if (*m_p != NULL)
3061 (*m_p)->m_flags &= ~M_PROTO1;
3062
3063 return (0);
3064 }
3065
3066 static int
3067 dlil_interface_filters_output(struct ifnet *ifp, struct mbuf **m_p,
3068 protocol_family_t protocol_family)
3069 {
3070 struct ifnet_filter *filter;
3071
3072 /*
3073 * Pass the outbound packet to the interface filters
3074 */
3075 lck_mtx_lock_spin(&ifp->if_flt_lock);
3076 /* prevent filter list from changing in case we drop the lock */
3077 if_flt_monitor_busy(ifp);
3078 TAILQ_FOREACH(filter, &ifp->if_flt_head, filt_next) {
3079 int result;
3080
3081 if (!filter->filt_skip && filter->filt_output != NULL &&
3082 (filter->filt_protocol == 0 ||
3083 filter->filt_protocol == protocol_family)) {
3084 lck_mtx_unlock(&ifp->if_flt_lock);
3085
3086 result = filter->filt_output(filter->filt_cookie, ifp,
3087 protocol_family, m_p);
3088
3089 lck_mtx_lock_spin(&ifp->if_flt_lock);
3090 if (result != 0) {
3091 /* we're done with the filter list */
3092 if_flt_monitor_unbusy(ifp);
3093 lck_mtx_unlock(&ifp->if_flt_lock);
3094 return (result);
3095 }
3096 }
3097 }
3098 /* we're done with the filter list */
3099 if_flt_monitor_unbusy(ifp);
3100 lck_mtx_unlock(&ifp->if_flt_lock);
3101
3102 return (0);
3103 }
3104
3105 static void
3106 dlil_ifproto_input(struct if_proto * ifproto, mbuf_t m)
3107 {
3108 int error;
3109
3110 if (ifproto->proto_kpi == kProtoKPI_v1) {
3111 /* Version 1 protocols get one packet at a time */
3112 while (m != NULL) {
3113 char * frame_header;
3114 mbuf_t next_packet;
3115
3116 next_packet = m->m_nextpkt;
3117 m->m_nextpkt = NULL;
3118 frame_header = m->m_pkthdr.pkt_hdr;
3119 m->m_pkthdr.pkt_hdr = NULL;
3120 error = (*ifproto->kpi.v1.input)(ifproto->ifp,
3121 ifproto->protocol_family, m, frame_header);
3122 if (error != 0 && error != EJUSTRETURN)
3123 m_freem(m);
3124 m = next_packet;
3125 }
3126 } else if (ifproto->proto_kpi == kProtoKPI_v2) {
3127 /* Version 2 protocols support packet lists */
3128 error = (*ifproto->kpi.v2.input)(ifproto->ifp,
3129 ifproto->protocol_family, m);
3130 if (error != 0 && error != EJUSTRETURN)
3131 m_freem_list(m);
3132 }
3133 return;
3134 }
3135
3136 static void
3137 dlil_input_stats_add(const struct ifnet_stat_increment_param *s,
3138 struct dlil_threading_info *inp, boolean_t poll)
3139 {
3140 struct ifnet_stat_increment_param *d = &inp->stats;
3141
3142 if (s->packets_in != 0)
3143 d->packets_in += s->packets_in;
3144 if (s->bytes_in != 0)
3145 d->bytes_in += s->bytes_in;
3146 if (s->errors_in != 0)
3147 d->errors_in += s->errors_in;
3148
3149 if (s->packets_out != 0)
3150 d->packets_out += s->packets_out;
3151 if (s->bytes_out != 0)
3152 d->bytes_out += s->bytes_out;
3153 if (s->errors_out != 0)
3154 d->errors_out += s->errors_out;
3155
3156 if (s->collisions != 0)
3157 d->collisions += s->collisions;
3158 if (s->dropped != 0)
3159 d->dropped += s->dropped;
3160
3161 if (poll)
3162 PKTCNTR_ADD(&inp->tstats, s->packets_in, s->bytes_in);
3163 }
3164
3165 static void
3166 dlil_input_stats_sync(struct ifnet *ifp, struct dlil_threading_info *inp)
3167 {
3168 struct ifnet_stat_increment_param *s = &inp->stats;
3169
3170 /*
3171 * Use of atomic operations is unavoidable here because
3172 * these stats may also be incremented elsewhere via KPIs.
3173 */
3174 if (s->packets_in != 0) {
3175 atomic_add_64(&ifp->if_data.ifi_ipackets, s->packets_in);
3176 s->packets_in = 0;
3177 }
3178 if (s->bytes_in != 0) {
3179 atomic_add_64(&ifp->if_data.ifi_ibytes, s->bytes_in);
3180 s->bytes_in = 0;
3181 }
3182 if (s->errors_in != 0) {
3183 atomic_add_64(&ifp->if_data.ifi_ierrors, s->errors_in);
3184 s->errors_in = 0;
3185 }
3186
3187 if (s->packets_out != 0) {
3188 atomic_add_64(&ifp->if_data.ifi_opackets, s->packets_out);
3189 s->packets_out = 0;
3190 }
3191 if (s->bytes_out != 0) {
3192 atomic_add_64(&ifp->if_data.ifi_obytes, s->bytes_out);
3193 s->bytes_out = 0;
3194 }
3195 if (s->errors_out != 0) {
3196 atomic_add_64(&ifp->if_data.ifi_oerrors, s->errors_out);
3197 s->errors_out = 0;
3198 }
3199
3200 if (s->collisions != 0) {
3201 atomic_add_64(&ifp->if_data.ifi_collisions, s->collisions);
3202 s->collisions = 0;
3203 }
3204 if (s->dropped != 0) {
3205 atomic_add_64(&ifp->if_data.ifi_iqdrops, s->dropped);
3206 s->dropped = 0;
3207 }
3208 /*
3209 * If we went over the threshold, notify NetworkStatistics.
3210 */
3211 if (ifp->if_data_threshold &&
3212 (ifp->if_ibytes + ifp->if_obytes) - ifp->if_dt_bytes >
3213 ifp->if_data_threshold) {
3214 ifp->if_dt_bytes = ifp->if_ibytes + ifp->if_obytes;
3215 nstat_ifnet_threshold_reached(ifp->if_index);
3216 }
3217 /*
3218 * No need for atomic operations as they are modified here
3219 * only from within the DLIL input thread context.
3220 */
3221 if (inp->tstats.packets != 0) {
3222 inp->pstats.ifi_poll_packets += inp->tstats.packets;
3223 inp->tstats.packets = 0;
3224 }
3225 if (inp->tstats.bytes != 0) {
3226 inp->pstats.ifi_poll_bytes += inp->tstats.bytes;
3227 inp->tstats.bytes = 0;
3228 }
3229 }
3230
3231 __private_extern__ void
3232 dlil_input_packet_list(struct ifnet *ifp, struct mbuf *m)
3233 {
3234 return (dlil_input_packet_list_common(ifp, m, 0,
3235 IFNET_MODEL_INPUT_POLL_OFF, FALSE));
3236 }
3237
3238 __private_extern__ void
3239 dlil_input_packet_list_extended(struct ifnet *ifp, struct mbuf *m,
3240 u_int32_t cnt, ifnet_model_t mode)
3241 {
3242 return (dlil_input_packet_list_common(ifp, m, cnt, mode, TRUE));
3243 }
3244
3245 static void
3246 dlil_input_packet_list_common(struct ifnet *ifp_param, struct mbuf *m,
3247 u_int32_t cnt, ifnet_model_t mode, boolean_t ext)
3248 {
3249 int error = 0;
3250 protocol_family_t protocol_family;
3251 mbuf_t next_packet;
3252 ifnet_t ifp = ifp_param;
3253 char * frame_header;
3254 struct if_proto * last_ifproto = NULL;
3255 mbuf_t pkt_first = NULL;
3256 mbuf_t * pkt_next = NULL;
3257 u_int32_t poll_thresh = 0, poll_ival = 0;
3258
3259 KERNEL_DEBUG(DBG_FNC_DLIL_INPUT | DBG_FUNC_START,0,0,0,0,0);
3260
3261 if (ext && mode == IFNET_MODEL_INPUT_POLL_ON && cnt > 1 &&
3262 (poll_ival = if_rxpoll_interval_pkts) > 0)
3263 poll_thresh = cnt;
3264
3265 while (m != NULL) {
3266 struct if_proto *ifproto = NULL;
3267 int iorefcnt = 0;
3268 uint32_t pktf_mask; /* pkt flags to preserve */
3269
3270 if (ifp_param == NULL)
3271 ifp = m->m_pkthdr.rcvif;
3272
3273 if ((ifp->if_eflags & IFEF_RXPOLL) && poll_thresh != 0 &&
3274 poll_ival > 0 && (--poll_thresh % poll_ival) == 0)
3275 ifnet_poll(ifp);
3276
3277 /* Check if this mbuf looks valid */
3278 MBUF_INPUT_CHECK(m, ifp);
3279
3280 next_packet = m->m_nextpkt;
3281 m->m_nextpkt = NULL;
3282 frame_header = m->m_pkthdr.pkt_hdr;
3283 m->m_pkthdr.pkt_hdr = NULL;
3284
3285 /*
3286 * Get an IO reference count if the interface is not
3287 * loopback (lo0) and it is attached; lo0 never goes
3288 * away, so optimize for that.
3289 */
3290 if (ifp != lo_ifp) {
3291 if (!ifnet_is_attached(ifp, 1)) {
3292 m_freem(m);
3293 goto next;
3294 }
3295 iorefcnt = 1;
3296 pktf_mask = 0;
3297 } else {
3298 /*
3299 * If this arrived on lo0, preserve interface addr
3300 * info to allow for connectivity between loopback
3301 * and local interface addresses.
3302 */
3303 pktf_mask = (PKTF_LOOP|PKTF_IFAINFO);
3304 }
3305
3306 /* make sure packet comes in clean */
3307 m_classifier_init(m, pktf_mask);
3308
3309 ifp_inc_traffic_class_in(ifp, m);
3310
3311 /* find which protocol family this packet is for */
3312 ifnet_lock_shared(ifp);
3313 error = (*ifp->if_demux)(ifp, m, frame_header,
3314 &protocol_family);
3315 ifnet_lock_done(ifp);
3316 if (error != 0) {
3317 if (error == EJUSTRETURN)
3318 goto next;
3319 protocol_family = 0;
3320 }
3321
3322 if (hwcksum_dbg != 0 && !(ifp->if_flags & IFF_LOOPBACK) &&
3323 !(m->m_pkthdr.pkt_flags & PKTF_LOOP))
3324 dlil_input_cksum_dbg(ifp, m, frame_header,
3325 protocol_family);
3326
3327 /*
3328 * For partial checksum offload, we expect the driver to
3329 * set the start offset indicating the start of the span
3330 * that is covered by the hardware-computed checksum;
3331 * adjust this start offset accordingly because the data
3332 * pointer has been advanced beyond the link-layer header.
3333 *
3334 * Don't adjust if the interface is a bridge member, as
3335 * the adjustment will occur from the context of the
3336 * bridge interface during input.
3337 */
3338 if (ifp->if_bridge == NULL && (m->m_pkthdr.csum_flags &
3339 (CSUM_DATA_VALID | CSUM_PARTIAL)) ==
3340 (CSUM_DATA_VALID | CSUM_PARTIAL)) {
3341 int adj;
3342
3343 if (frame_header == NULL ||
3344 frame_header < (char *)mbuf_datastart(m) ||
3345 frame_header > (char *)m->m_data ||
3346 (adj = (m->m_data - frame_header)) >
3347 m->m_pkthdr.csum_rx_start) {
3348 m->m_pkthdr.csum_data = 0;
3349 m->m_pkthdr.csum_flags &= ~CSUM_DATA_VALID;
3350 hwcksum_in_invalidated++;
3351 } else {
3352 m->m_pkthdr.csum_rx_start -= adj;
3353 }
3354 }
3355
3356 pktap_input(ifp, protocol_family, m, frame_header);
3357
3358 if (m->m_flags & (M_BCAST|M_MCAST))
3359 atomic_add_64(&ifp->if_imcasts, 1);
3360
3361 /* run interface filters, exclude VLAN packets PR-3586856 */
3362 if ((m->m_pkthdr.csum_flags & CSUM_VLAN_TAG_VALID) == 0) {
3363 error = dlil_interface_filters_input(ifp, &m,
3364 &frame_header, protocol_family);
3365 if (error != 0) {
3366 if (error != EJUSTRETURN)
3367 m_freem(m);
3368 goto next;
3369 }
3370 }
3371 if (error != 0 || ((m->m_flags & M_PROMISC) != 0) ) {
3372 m_freem(m);
3373 goto next;
3374 }
3375
3376 /* Lookup the protocol attachment to this interface */
3377 if (protocol_family == 0) {
3378 ifproto = NULL;
3379 } else if (last_ifproto != NULL && last_ifproto->ifp == ifp &&
3380 (last_ifproto->protocol_family == protocol_family)) {
3381 VERIFY(ifproto == NULL);
3382 ifproto = last_ifproto;
3383 if_proto_ref(last_ifproto);
3384 } else {
3385 VERIFY(ifproto == NULL);
3386 ifnet_lock_shared(ifp);
3387 /* callee holds a proto refcnt upon success */
3388 ifproto = find_attached_proto(ifp, protocol_family);
3389 ifnet_lock_done(ifp);
3390 }
3391 if (ifproto == NULL) {
3392 /* no protocol for this packet, discard */
3393 m_freem(m);
3394 goto next;
3395 }
3396 if (ifproto != last_ifproto) {
3397 if (last_ifproto != NULL) {
3398 /* pass up the list for the previous protocol */
3399 dlil_ifproto_input(last_ifproto, pkt_first);
3400 pkt_first = NULL;
3401 if_proto_free(last_ifproto);
3402 }
3403 last_ifproto = ifproto;
3404 if_proto_ref(ifproto);
3405 }
3406 /* extend the list */
3407 m->m_pkthdr.pkt_hdr = frame_header;
3408 if (pkt_first == NULL) {
3409 pkt_first = m;
3410 } else {
3411 *pkt_next = m;
3412 }
3413 pkt_next = &m->m_nextpkt;
3414
3415 next:
3416 if (next_packet == NULL && last_ifproto != NULL) {
3417 /* pass up the last list of packets */
3418 dlil_ifproto_input(last_ifproto, pkt_first);
3419 if_proto_free(last_ifproto);
3420 last_ifproto = NULL;
3421 }
3422 if (ifproto != NULL) {
3423 if_proto_free(ifproto);
3424 ifproto = NULL;
3425 }
3426
3427 m = next_packet;
3428
3429 /* update the driver's multicast filter, if needed */
3430 if (ifp->if_updatemcasts > 0 && if_mcasts_update(ifp) == 0)
3431 ifp->if_updatemcasts = 0;
3432 if (iorefcnt == 1)
3433 ifnet_decr_iorefcnt(ifp);
3434 }
3435
3436 KERNEL_DEBUG(DBG_FNC_DLIL_INPUT | DBG_FUNC_END,0,0,0,0,0);
3437 }
3438
3439 errno_t
3440 if_mcasts_update(struct ifnet *ifp)
3441 {
3442 errno_t err;
3443
3444 err = ifnet_ioctl(ifp, 0, SIOCADDMULTI, NULL);
3445 if (err == EAFNOSUPPORT)
3446 err = 0;
3447 printf("%s: %s %d suspended link-layer multicast membership(s) "
3448 "(err=%d)\n", if_name(ifp),
3449 (err == 0 ? "successfully restored" : "failed to restore"),
3450 ifp->if_updatemcasts, err);
3451
3452 /* just return success */
3453 return (0);
3454 }
3455
3456 static int
3457 dlil_event_internal(struct ifnet *ifp, struct kev_msg *event)
3458 {
3459 struct ifnet_filter *filter;
3460
3461 /* Get an io ref count if the interface is attached */
3462 if (!ifnet_is_attached(ifp, 1))
3463 goto done;
3464
3465 /*
3466 * Pass the event to the interface filters
3467 */
3468 lck_mtx_lock_spin(&ifp->if_flt_lock);
3469 /* prevent filter list from changing in case we drop the lock */
3470 if_flt_monitor_busy(ifp);
3471 TAILQ_FOREACH(filter, &ifp->if_flt_head, filt_next) {
3472 if (filter->filt_event != NULL) {
3473 lck_mtx_unlock(&ifp->if_flt_lock);
3474
3475 filter->filt_event(filter->filt_cookie, ifp,
3476 filter->filt_protocol, event);
3477
3478 lck_mtx_lock_spin(&ifp->if_flt_lock);
3479 }
3480 }
3481 /* we're done with the filter list */
3482 if_flt_monitor_unbusy(ifp);
3483 lck_mtx_unlock(&ifp->if_flt_lock);
3484
3485 ifnet_lock_shared(ifp);
3486 if (ifp->if_proto_hash != NULL) {
3487 int i;
3488
3489 for (i = 0; i < PROTO_HASH_SLOTS; i++) {
3490 struct if_proto *proto;
3491
3492 SLIST_FOREACH(proto, &ifp->if_proto_hash[i],
3493 next_hash) {
3494 proto_media_event eventp =
3495 (proto->proto_kpi == kProtoKPI_v1 ?
3496 proto->kpi.v1.event :
3497 proto->kpi.v2.event);
3498
3499 if (eventp != NULL) {
3500 if_proto_ref(proto);
3501 ifnet_lock_done(ifp);
3502
3503 eventp(ifp, proto->protocol_family,
3504 event);
3505
3506 ifnet_lock_shared(ifp);
3507 if_proto_free(proto);
3508 }
3509 }
3510 }
3511 }
3512 ifnet_lock_done(ifp);
3513
3514 /* Pass the event to the interface */
3515 if (ifp->if_event != NULL)
3516 ifp->if_event(ifp, event);
3517
3518 /* Release the io ref count */
3519 ifnet_decr_iorefcnt(ifp);
3520
3521 done:
3522 return (kev_post_msg(event));
3523 }
3524
3525 errno_t
3526 ifnet_event(ifnet_t ifp, struct kern_event_msg *event)
3527 {
3528 struct kev_msg kev_msg;
3529 int result = 0;
3530
3531 if (ifp == NULL || event == NULL)
3532 return (EINVAL);
3533
3534 bzero(&kev_msg, sizeof (kev_msg));
3535 kev_msg.vendor_code = event->vendor_code;
3536 kev_msg.kev_class = event->kev_class;
3537 kev_msg.kev_subclass = event->kev_subclass;
3538 kev_msg.event_code = event->event_code;
3539 kev_msg.dv[0].data_ptr = &event->event_data[0];
3540 kev_msg.dv[0].data_length = event->total_size - KEV_MSG_HEADER_SIZE;
3541 kev_msg.dv[1].data_length = 0;
3542
3543 result = dlil_event_internal(ifp, &kev_msg);
3544
3545 return (result);
3546 }
3547
3548 #if CONFIG_MACF_NET
3549 #include <netinet/ip6.h>
3550 #include <netinet/ip.h>
3551 static int
3552 dlil_get_socket_type(struct mbuf **mp, int family, int raw)
3553 {
3554 struct mbuf *m;
3555 struct ip *ip;
3556 struct ip6_hdr *ip6;
3557 int type = SOCK_RAW;
3558
3559 if (!raw) {
3560 switch (family) {
3561 case PF_INET:
3562 m = m_pullup(*mp, sizeof(struct ip));
3563 if (m == NULL)
3564 break;
3565 *mp = m;
3566 ip = mtod(m, struct ip *);
3567 if (ip->ip_p == IPPROTO_TCP)
3568 type = SOCK_STREAM;
3569 else if (ip->ip_p == IPPROTO_UDP)
3570 type = SOCK_DGRAM;
3571 break;
3572 case PF_INET6:
3573 m = m_pullup(*mp, sizeof(struct ip6_hdr));
3574 if (m == NULL)
3575 break;
3576 *mp = m;
3577 ip6 = mtod(m, struct ip6_hdr *);
3578 if (ip6->ip6_nxt == IPPROTO_TCP)
3579 type = SOCK_STREAM;
3580 else if (ip6->ip6_nxt == IPPROTO_UDP)
3581 type = SOCK_DGRAM;
3582 break;
3583 }
3584 }
3585
3586 return (type);
3587 }
3588 #endif
3589
3590 /*
3591 * This is mostly called from the context of the DLIL input thread;
3592 * because of that there is no need for atomic operations.
3593 */
3594 static __inline void
3595 ifp_inc_traffic_class_in(struct ifnet *ifp, struct mbuf *m)
3596 {
3597 if (!(m->m_flags & M_PKTHDR))
3598 return;
3599
3600 switch (m_get_traffic_class(m)) {
3601 case MBUF_TC_BE:
3602 ifp->if_tc.ifi_ibepackets++;
3603 ifp->if_tc.ifi_ibebytes += m->m_pkthdr.len;
3604 break;
3605 case MBUF_TC_BK:
3606 ifp->if_tc.ifi_ibkpackets++;
3607 ifp->if_tc.ifi_ibkbytes += m->m_pkthdr.len;
3608 break;
3609 case MBUF_TC_VI:
3610 ifp->if_tc.ifi_ivipackets++;
3611 ifp->if_tc.ifi_ivibytes += m->m_pkthdr.len;
3612 break;
3613 case MBUF_TC_VO:
3614 ifp->if_tc.ifi_ivopackets++;
3615 ifp->if_tc.ifi_ivobytes += m->m_pkthdr.len;
3616 break;
3617 default:
3618 break;
3619 }
3620
3621 if (mbuf_is_traffic_class_privileged(m)) {
3622 ifp->if_tc.ifi_ipvpackets++;
3623 ifp->if_tc.ifi_ipvbytes += m->m_pkthdr.len;
3624 }
3625 }
3626
3627 /*
3628 * This is called from DLIL output, hence multiple threads could end
3629 * up modifying the statistics. We trade off acccuracy for performance
3630 * by not using atomic operations here.
3631 */
3632 static __inline void
3633 ifp_inc_traffic_class_out(struct ifnet *ifp, struct mbuf *m)
3634 {
3635 if (!(m->m_flags & M_PKTHDR))
3636 return;
3637
3638 switch (m_get_traffic_class(m)) {
3639 case MBUF_TC_BE:
3640 ifp->if_tc.ifi_obepackets++;
3641 ifp->if_tc.ifi_obebytes += m->m_pkthdr.len;
3642 break;
3643 case MBUF_TC_BK:
3644 ifp->if_tc.ifi_obkpackets++;
3645 ifp->if_tc.ifi_obkbytes += m->m_pkthdr.len;
3646 break;
3647 case MBUF_TC_VI:
3648 ifp->if_tc.ifi_ovipackets++;
3649 ifp->if_tc.ifi_ovibytes += m->m_pkthdr.len;
3650 break;
3651 case MBUF_TC_VO:
3652 ifp->if_tc.ifi_ovopackets++;
3653 ifp->if_tc.ifi_ovobytes += m->m_pkthdr.len;
3654 break;
3655 default:
3656 break;
3657 }
3658
3659 if (mbuf_is_traffic_class_privileged(m)) {
3660 ifp->if_tc.ifi_opvpackets++;
3661 ifp->if_tc.ifi_opvbytes += m->m_pkthdr.len;
3662 }
3663 }
3664
3665 /*
3666 * dlil_output
3667 *
3668 * Caller should have a lock on the protocol domain if the protocol
3669 * doesn't support finer grained locking. In most cases, the lock
3670 * will be held from the socket layer and won't be released until
3671 * we return back to the socket layer.
3672 *
3673 * This does mean that we must take a protocol lock before we take
3674 * an interface lock if we're going to take both. This makes sense
3675 * because a protocol is likely to interact with an ifp while it
3676 * is under the protocol lock.
3677 *
3678 * An advisory code will be returned if adv is not null. This
3679 * can be used to provide feedback about interface queues to the
3680 * application.
3681 */
3682 errno_t
3683 dlil_output(ifnet_t ifp, protocol_family_t proto_family, mbuf_t packetlist,
3684 void *route, const struct sockaddr *dest, int raw, struct flowadv *adv)
3685 {
3686 char *frame_type = NULL;
3687 char *dst_linkaddr = NULL;
3688 int retval = 0;
3689 char frame_type_buffer[MAX_FRAME_TYPE_SIZE * 4];
3690 char dst_linkaddr_buffer[MAX_LINKADDR * 4];
3691 struct if_proto *proto = NULL;
3692 mbuf_t m;
3693 mbuf_t send_head = NULL;
3694 mbuf_t *send_tail = &send_head;
3695 int iorefcnt = 0;
3696 u_int32_t pre = 0, post = 0;
3697 u_int32_t fpkts = 0, fbytes = 0;
3698 int32_t flen = 0;
3699
3700 KERNEL_DEBUG(DBG_FNC_DLIL_OUTPUT | DBG_FUNC_START, 0, 0, 0, 0, 0);
3701
3702 /* Get an io refcnt if the interface is attached to prevent ifnet_detach
3703 * from happening while this operation is in progress */
3704 if (!ifnet_is_attached(ifp, 1)) {
3705 retval = ENXIO;
3706 goto cleanup;
3707 }
3708 iorefcnt = 1;
3709
3710 /* update the driver's multicast filter, if needed */
3711 if (ifp->if_updatemcasts > 0 && if_mcasts_update(ifp) == 0)
3712 ifp->if_updatemcasts = 0;
3713
3714 frame_type = frame_type_buffer;
3715 dst_linkaddr = dst_linkaddr_buffer;
3716
3717 if (raw == 0) {
3718 ifnet_lock_shared(ifp);
3719 /* callee holds a proto refcnt upon success */
3720 proto = find_attached_proto(ifp, proto_family);
3721 if (proto == NULL) {
3722 ifnet_lock_done(ifp);
3723 retval = ENXIO;
3724 goto cleanup;
3725 }
3726 ifnet_lock_done(ifp);
3727 }
3728
3729 preout_again:
3730 if (packetlist == NULL)
3731 goto cleanup;
3732
3733 m = packetlist;
3734 packetlist = packetlist->m_nextpkt;
3735 m->m_nextpkt = NULL;
3736
3737 if (raw == 0) {
3738 proto_media_preout preoutp = (proto->proto_kpi == kProtoKPI_v1 ?
3739 proto->kpi.v1.pre_output : proto->kpi.v2.pre_output);
3740 retval = 0;
3741 if (preoutp != NULL) {
3742 retval = preoutp(ifp, proto_family, &m, dest, route,
3743 frame_type, dst_linkaddr);
3744
3745 if (retval != 0) {
3746 if (retval == EJUSTRETURN)
3747 goto preout_again;
3748 m_freem(m);
3749 goto cleanup;
3750 }
3751 }
3752 }
3753
3754 #if CONFIG_MACF_NET
3755 retval = mac_ifnet_check_transmit(ifp, m, proto_family,
3756 dlil_get_socket_type(&m, proto_family, raw));
3757 if (retval != 0) {
3758 m_freem(m);
3759 goto cleanup;
3760 }
3761 #endif
3762
3763 do {
3764 #if CONFIG_DTRACE
3765 if (!raw && proto_family == PF_INET) {
3766 struct ip *ip = mtod(m, struct ip*);
3767 DTRACE_IP6(send, struct mbuf *, m, struct inpcb *, NULL,
3768 struct ip *, ip, struct ifnet *, ifp,
3769 struct ip *, ip, struct ip6_hdr *, NULL);
3770
3771 } else if (!raw && proto_family == PF_INET6) {
3772 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr*);
3773 DTRACE_IP6(send, struct mbuf*, m, struct inpcb *, NULL,
3774 struct ip6_hdr *, ip6, struct ifnet*, ifp,
3775 struct ip*, NULL, struct ip6_hdr *, ip6);
3776 }
3777 #endif /* CONFIG_DTRACE */
3778
3779 if (raw == 0 && ifp->if_framer != NULL) {
3780 int rcvif_set = 0;
3781
3782 /*
3783 * If this is a broadcast packet that needs to be
3784 * looped back into the system, set the inbound ifp
3785 * to that of the outbound ifp. This will allow
3786 * us to determine that it is a legitimate packet
3787 * for the system. Only set the ifp if it's not
3788 * already set, just to be safe.
3789 */
3790 if ((m->m_flags & (M_BCAST | M_LOOP)) &&
3791 m->m_pkthdr.rcvif == NULL) {
3792 m->m_pkthdr.rcvif = ifp;
3793 rcvif_set = 1;
3794 }
3795
3796 retval = ifp->if_framer(ifp, &m, dest, dst_linkaddr,
3797 frame_type, &pre, &post);
3798 if (retval != 0) {
3799 if (retval != EJUSTRETURN)
3800 m_freem(m);
3801 goto next;
3802 }
3803
3804 /*
3805 * For partial checksum offload, adjust the start
3806 * and stuff offsets based on the prepended header.
3807 */
3808 if ((m->m_pkthdr.csum_flags &
3809 (CSUM_DATA_VALID | CSUM_PARTIAL)) ==
3810 (CSUM_DATA_VALID | CSUM_PARTIAL)) {
3811 m->m_pkthdr.csum_tx_stuff += pre;
3812 m->m_pkthdr.csum_tx_start += pre;
3813 }
3814
3815 if (hwcksum_dbg != 0 && !(ifp->if_flags & IFF_LOOPBACK))
3816 dlil_output_cksum_dbg(ifp, m, pre,
3817 proto_family);
3818
3819 /*
3820 * Clear the ifp if it was set above, and to be
3821 * safe, only if it is still the same as the
3822 * outbound ifp we have in context. If it was
3823 * looped back, then a copy of it was sent to the
3824 * loopback interface with the rcvif set, and we
3825 * are clearing the one that will go down to the
3826 * layer below.
3827 */
3828 if (rcvif_set && m->m_pkthdr.rcvif == ifp)
3829 m->m_pkthdr.rcvif = NULL;
3830 }
3831
3832 /*
3833 * Let interface filters (if any) do their thing ...
3834 */
3835 /* Do not pass VLAN tagged packets to filters PR-3586856 */
3836 if ((m->m_pkthdr.csum_flags & CSUM_VLAN_TAG_VALID) == 0) {
3837 retval = dlil_interface_filters_output(ifp,
3838 &m, proto_family);
3839 if (retval != 0) {
3840 if (retval != EJUSTRETURN)
3841 m_freem(m);
3842 goto next;
3843 }
3844 }
3845 /*
3846 * Strip away M_PROTO1 bit prior to sending packet
3847 * to the driver as this field may be used by the driver
3848 */
3849 m->m_flags &= ~M_PROTO1;
3850
3851 /*
3852 * If the underlying interface is not capable of handling a
3853 * packet whose data portion spans across physically disjoint
3854 * pages, we need to "normalize" the packet so that we pass
3855 * down a chain of mbufs where each mbuf points to a span that
3856 * resides in the system page boundary. If the packet does
3857 * not cross page(s), the following is a no-op.
3858 */
3859 if (!(ifp->if_hwassist & IFNET_MULTIPAGES)) {
3860 if ((m = m_normalize(m)) == NULL)
3861 goto next;
3862 }
3863
3864 /*
3865 * If this is a TSO packet, make sure the interface still
3866 * advertise TSO capability.
3867 */
3868 if (TSO_IPV4_NOTOK(ifp, m) || TSO_IPV6_NOTOK(ifp, m)) {
3869 retval = EMSGSIZE;
3870 m_freem(m);
3871 goto cleanup;
3872 }
3873
3874 /*
3875 * If the packet service class is not background,
3876 * update the timestamp to indicate recent activity
3877 * on a foreground socket.
3878 */
3879 if (!(m->m_pkthdr.pkt_flags & PKTF_SO_BACKGROUND) &&
3880 (m->m_pkthdr.pkt_flags & PKTF_FLOW_ID) &&
3881 m->m_pkthdr.pkt_flowsrc == FLOWSRC_INPCB)
3882 ifp->if_fg_sendts = net_uptime();
3883
3884 ifp_inc_traffic_class_out(ifp, m);
3885 pktap_output(ifp, proto_family, m, pre, post);
3886
3887 /*
3888 * Finally, call the driver.
3889 */
3890 if (ifp->if_eflags & IFEF_SENDLIST) {
3891 if (m->m_pkthdr.pkt_flags & PKTF_FORWARDED) {
3892 flen += (m_pktlen(m) - (pre + post));
3893 m->m_pkthdr.pkt_flags &= ~PKTF_FORWARDED;
3894 }
3895 *send_tail = m;
3896 send_tail = &m->m_nextpkt;
3897 } else {
3898 if (m->m_pkthdr.pkt_flags & PKTF_FORWARDED) {
3899 flen = (m_pktlen(m) - (pre + post));
3900 m->m_pkthdr.pkt_flags &= ~PKTF_FORWARDED;
3901 } else {
3902 flen = 0;
3903 }
3904 KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_START,
3905 0, 0, 0, 0, 0);
3906 retval = (*ifp->if_output)(ifp, m);
3907 if (retval == EQFULL || retval == EQSUSPENDED) {
3908 if (adv != NULL && adv->code == FADV_SUCCESS) {
3909 adv->code = (retval == EQFULL ?
3910 FADV_FLOW_CONTROLLED :
3911 FADV_SUSPENDED);
3912 }
3913 retval = 0;
3914 }
3915 if (retval == 0 && flen > 0) {
3916 fbytes += flen;
3917 fpkts++;
3918 }
3919 if (retval != 0 && dlil_verbose) {
3920 printf("%s: output error on %s retval = %d\n",
3921 __func__, if_name(ifp),
3922 retval);
3923 }
3924 KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_END,
3925 0, 0, 0, 0, 0);
3926 }
3927 KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_END, 0, 0, 0, 0, 0);
3928
3929 next:
3930 m = packetlist;
3931 if (m != NULL) {
3932 packetlist = packetlist->m_nextpkt;
3933 m->m_nextpkt = NULL;
3934 }
3935 } while (m != NULL);
3936
3937 if (send_head != NULL) {
3938 VERIFY(ifp->if_eflags & IFEF_SENDLIST);
3939 KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_START,
3940 0, 0, 0, 0, 0);
3941 retval = (*ifp->if_output)(ifp, send_head);
3942 if (retval == EQFULL || retval == EQSUSPENDED) {
3943 if (adv != NULL) {
3944 adv->code = (retval == EQFULL ?
3945 FADV_FLOW_CONTROLLED : FADV_SUSPENDED);
3946 }
3947 retval = 0;
3948 }
3949 if (retval == 0 && flen > 0) {
3950 fbytes += flen;
3951 fpkts++;
3952 }
3953 if (retval != 0 && dlil_verbose) {
3954 printf("%s: output error on %s retval = %d\n",
3955 __func__, if_name(ifp), retval);
3956 }
3957 KERNEL_DEBUG(DBG_FNC_DLIL_IFOUT | DBG_FUNC_END, 0, 0, 0, 0, 0);
3958 }
3959
3960 KERNEL_DEBUG(DBG_FNC_DLIL_OUTPUT | DBG_FUNC_END, 0, 0, 0, 0, 0);
3961
3962 cleanup:
3963 if (fbytes > 0)
3964 ifp->if_fbytes += fbytes;
3965 if (fpkts > 0)
3966 ifp->if_fpackets += fpkts;
3967 if (proto != NULL)
3968 if_proto_free(proto);
3969 if (packetlist) /* if any packets are left, clean up */
3970 mbuf_freem_list(packetlist);
3971 if (retval == EJUSTRETURN)
3972 retval = 0;
3973 if (iorefcnt == 1)
3974 ifnet_decr_iorefcnt(ifp);
3975
3976 return (retval);
3977 }
3978
3979 errno_t
3980 ifnet_ioctl(ifnet_t ifp, protocol_family_t proto_fam, u_long ioctl_code,
3981 void *ioctl_arg)
3982 {
3983 struct ifnet_filter *filter;
3984 int retval = EOPNOTSUPP;
3985 int result = 0;
3986
3987 if (ifp == NULL || ioctl_code == 0)
3988 return (EINVAL);
3989
3990 /* Get an io ref count if the interface is attached */
3991 if (!ifnet_is_attached(ifp, 1))
3992 return (EOPNOTSUPP);
3993
3994 /* Run the interface filters first.
3995 * We want to run all filters before calling the protocol,
3996 * interface family, or interface.
3997 */
3998 lck_mtx_lock_spin(&ifp->if_flt_lock);
3999 /* prevent filter list from changing in case we drop the lock */
4000 if_flt_monitor_busy(ifp);
4001 TAILQ_FOREACH(filter, &ifp->if_flt_head, filt_next) {
4002 if (filter->filt_ioctl != NULL && (filter->filt_protocol == 0 ||
4003 filter->filt_protocol == proto_fam)) {
4004 lck_mtx_unlock(&ifp->if_flt_lock);
4005
4006 result = filter->filt_ioctl(filter->filt_cookie, ifp,
4007 proto_fam, ioctl_code, ioctl_arg);
4008
4009 lck_mtx_lock_spin(&ifp->if_flt_lock);
4010
4011 /* Only update retval if no one has handled the ioctl */
4012 if (retval == EOPNOTSUPP || result == EJUSTRETURN) {
4013 if (result == ENOTSUP)
4014 result = EOPNOTSUPP;
4015 retval = result;
4016 if (retval != 0 && retval != EOPNOTSUPP) {
4017 /* we're done with the filter list */
4018 if_flt_monitor_unbusy(ifp);
4019 lck_mtx_unlock(&ifp->if_flt_lock);
4020 goto cleanup;
4021 }
4022 }
4023 }
4024 }
4025 /* we're done with the filter list */
4026 if_flt_monitor_unbusy(ifp);
4027 lck_mtx_unlock(&ifp->if_flt_lock);
4028
4029 /* Allow the protocol to handle the ioctl */
4030 if (proto_fam != 0) {
4031 struct if_proto *proto;
4032
4033 /* callee holds a proto refcnt upon success */
4034 ifnet_lock_shared(ifp);
4035 proto = find_attached_proto(ifp, proto_fam);
4036 ifnet_lock_done(ifp);
4037 if (proto != NULL) {
4038 proto_media_ioctl ioctlp =
4039 (proto->proto_kpi == kProtoKPI_v1 ?
4040 proto->kpi.v1.ioctl : proto->kpi.v2.ioctl);
4041 result = EOPNOTSUPP;
4042 if (ioctlp != NULL)
4043 result = ioctlp(ifp, proto_fam, ioctl_code,
4044 ioctl_arg);
4045 if_proto_free(proto);
4046
4047 /* Only update retval if no one has handled the ioctl */
4048 if (retval == EOPNOTSUPP || result == EJUSTRETURN) {
4049 if (result == ENOTSUP)
4050 result = EOPNOTSUPP;
4051 retval = result;
4052 if (retval && retval != EOPNOTSUPP)
4053 goto cleanup;
4054 }
4055 }
4056 }
4057
4058 /* retval is either 0 or EOPNOTSUPP */
4059
4060 /*
4061 * Let the interface handle this ioctl.
4062 * If it returns EOPNOTSUPP, ignore that, we may have
4063 * already handled this in the protocol or family.
4064 */
4065 if (ifp->if_ioctl)
4066 result = (*ifp->if_ioctl)(ifp, ioctl_code, ioctl_arg);
4067
4068 /* Only update retval if no one has handled the ioctl */
4069 if (retval == EOPNOTSUPP || result == EJUSTRETURN) {
4070 if (result == ENOTSUP)
4071 result = EOPNOTSUPP;
4072 retval = result;
4073 if (retval && retval != EOPNOTSUPP) {
4074 goto cleanup;
4075 }
4076 }
4077
4078 cleanup:
4079 if (retval == EJUSTRETURN)
4080 retval = 0;
4081
4082 ifnet_decr_iorefcnt(ifp);
4083
4084 return (retval);
4085 }
4086
4087 __private_extern__ errno_t
4088 dlil_set_bpf_tap(ifnet_t ifp, bpf_tap_mode mode, bpf_packet_func callback)
4089 {
4090 errno_t error = 0;
4091
4092
4093 if (ifp->if_set_bpf_tap) {
4094 /* Get an io reference on the interface if it is attached */
4095 if (!ifnet_is_attached(ifp, 1))
4096 return ENXIO;
4097 error = ifp->if_set_bpf_tap(ifp, mode, callback);
4098 ifnet_decr_iorefcnt(ifp);
4099 }
4100 return (error);
4101 }
4102
4103 errno_t
4104 dlil_resolve_multi(struct ifnet *ifp, const struct sockaddr *proto_addr,
4105 struct sockaddr *ll_addr, size_t ll_len)
4106 {
4107 errno_t result = EOPNOTSUPP;
4108 struct if_proto *proto;
4109 const struct sockaddr *verify;
4110 proto_media_resolve_multi resolvep;
4111
4112 if (!ifnet_is_attached(ifp, 1))
4113 return result;
4114
4115 bzero(ll_addr, ll_len);
4116
4117 /* Call the protocol first; callee holds a proto refcnt upon success */
4118 ifnet_lock_shared(ifp);
4119 proto = find_attached_proto(ifp, proto_addr->sa_family);
4120 ifnet_lock_done(ifp);
4121 if (proto != NULL) {
4122 resolvep = (proto->proto_kpi == kProtoKPI_v1 ?
4123 proto->kpi.v1.resolve_multi : proto->kpi.v2.resolve_multi);
4124 if (resolvep != NULL)
4125 result = resolvep(ifp, proto_addr,
4126 (struct sockaddr_dl*)(void *)ll_addr, ll_len);
4127 if_proto_free(proto);
4128 }
4129
4130 /* Let the interface verify the multicast address */
4131 if ((result == EOPNOTSUPP || result == 0) && ifp->if_check_multi) {
4132 if (result == 0)
4133 verify = ll_addr;
4134 else
4135 verify = proto_addr;
4136 result = ifp->if_check_multi(ifp, verify);
4137 }
4138
4139 ifnet_decr_iorefcnt(ifp);
4140 return (result);
4141 }
4142
4143 __private_extern__ errno_t
4144 dlil_send_arp_internal(ifnet_t ifp, u_short arpop,
4145 const struct sockaddr_dl* sender_hw, const struct sockaddr* sender_proto,
4146 const struct sockaddr_dl* target_hw, const struct sockaddr* target_proto)
4147 {
4148 struct if_proto *proto;
4149 errno_t result = 0;
4150
4151 /* callee holds a proto refcnt upon success */
4152 ifnet_lock_shared(ifp);
4153 proto = find_attached_proto(ifp, target_proto->sa_family);
4154 ifnet_lock_done(ifp);
4155 if (proto == NULL) {
4156 result = ENOTSUP;
4157 } else {
4158 proto_media_send_arp arpp;
4159 arpp = (proto->proto_kpi == kProtoKPI_v1 ?
4160 proto->kpi.v1.send_arp : proto->kpi.v2.send_arp);
4161 if (arpp == NULL) {
4162 result = ENOTSUP;
4163 } else {
4164 switch (arpop) {
4165 case ARPOP_REQUEST:
4166 arpstat.txrequests++;
4167 if (target_hw != NULL)
4168 arpstat.txurequests++;
4169 break;
4170 case ARPOP_REPLY:
4171 arpstat.txreplies++;
4172 break;
4173 }
4174 result = arpp(ifp, arpop, sender_hw, sender_proto,
4175 target_hw, target_proto);
4176 }
4177 if_proto_free(proto);
4178 }
4179
4180 return (result);
4181 }
4182
4183 struct net_thread_marks { };
4184 static const struct net_thread_marks net_thread_marks_base = { };
4185
4186 __private_extern__ const net_thread_marks_t net_thread_marks_none =
4187 &net_thread_marks_base;
4188
4189 __private_extern__ net_thread_marks_t
4190 net_thread_marks_push(u_int32_t push)
4191 {
4192 static const char *const base = (const void*)&net_thread_marks_base;
4193 u_int32_t pop = 0;
4194
4195 if (push != 0) {
4196 struct uthread *uth = get_bsdthread_info(current_thread());
4197
4198 pop = push & ~uth->uu_network_marks;
4199 if (pop != 0)
4200 uth->uu_network_marks |= pop;
4201 }
4202
4203 return ((net_thread_marks_t)&base[pop]);
4204 }
4205
4206 __private_extern__ net_thread_marks_t
4207 net_thread_unmarks_push(u_int32_t unpush)
4208 {
4209 static const char *const base = (const void*)&net_thread_marks_base;
4210 u_int32_t unpop = 0;
4211
4212 if (unpush != 0) {
4213 struct uthread *uth = get_bsdthread_info(current_thread());
4214
4215 unpop = unpush & uth->uu_network_marks;
4216 if (unpop != 0)
4217 uth->uu_network_marks &= ~unpop;
4218 }
4219
4220 return ((net_thread_marks_t)&base[unpop]);
4221 }
4222
4223 __private_extern__ void
4224 net_thread_marks_pop(net_thread_marks_t popx)
4225 {
4226 static const char *const base = (const void*)&net_thread_marks_base;
4227 ptrdiff_t pop = (caddr_t)popx - (caddr_t)base;
4228
4229 if (pop != 0) {
4230 static const ptrdiff_t ones = (ptrdiff_t)(u_int32_t)~0U;
4231 struct uthread *uth = get_bsdthread_info(current_thread());
4232
4233 VERIFY((pop & ones) == pop);
4234 VERIFY((ptrdiff_t)(uth->uu_network_marks & pop) == pop);
4235 uth->uu_network_marks &= ~pop;
4236 }
4237 }
4238
4239 __private_extern__ void
4240 net_thread_unmarks_pop(net_thread_marks_t unpopx)
4241 {
4242 static const char *const base = (const void*)&net_thread_marks_base;
4243 ptrdiff_t unpop = (caddr_t)unpopx - (caddr_t)base;
4244
4245 if (unpop != 0) {
4246 static const ptrdiff_t ones = (ptrdiff_t)(u_int32_t)~0U;
4247 struct uthread *uth = get_bsdthread_info(current_thread());
4248
4249 VERIFY((unpop & ones) == unpop);
4250 VERIFY((ptrdiff_t)(uth->uu_network_marks & unpop) == 0);
4251 uth->uu_network_marks |= unpop;
4252 }
4253 }
4254
4255 __private_extern__ u_int32_t
4256 net_thread_is_marked(u_int32_t check)
4257 {
4258 if (check != 0) {
4259 struct uthread *uth = get_bsdthread_info(current_thread());
4260 return (uth->uu_network_marks & check);
4261 }
4262 else
4263 return (0);
4264 }
4265
4266 __private_extern__ u_int32_t
4267 net_thread_is_unmarked(u_int32_t check)
4268 {
4269 if (check != 0) {
4270 struct uthread *uth = get_bsdthread_info(current_thread());
4271 return (~uth->uu_network_marks & check);
4272 }
4273 else
4274 return (0);
4275 }
4276
4277 static __inline__ int
4278 _is_announcement(const struct sockaddr_in * sender_sin,
4279 const struct sockaddr_in * target_sin)
4280 {
4281 if (sender_sin == NULL) {
4282 return (FALSE);
4283 }
4284 return (sender_sin->sin_addr.s_addr == target_sin->sin_addr.s_addr);
4285 }
4286
4287 __private_extern__ errno_t
4288 dlil_send_arp(ifnet_t ifp, u_short arpop, const struct sockaddr_dl* sender_hw,
4289 const struct sockaddr* sender_proto, const struct sockaddr_dl* target_hw,
4290 const struct sockaddr* target_proto0, u_int32_t rtflags)
4291 {
4292 errno_t result = 0;
4293 const struct sockaddr_in * sender_sin;
4294 const struct sockaddr_in * target_sin;
4295 struct sockaddr_inarp target_proto_sinarp;
4296 struct sockaddr *target_proto = (void *)(uintptr_t)target_proto0;
4297
4298 if (target_proto == NULL || (sender_proto != NULL &&
4299 sender_proto->sa_family != target_proto->sa_family))
4300 return (EINVAL);
4301
4302 /*
4303 * If the target is a (default) router, provide that
4304 * information to the send_arp callback routine.
4305 */
4306 if (rtflags & RTF_ROUTER) {
4307 bcopy(target_proto, &target_proto_sinarp,
4308 sizeof (struct sockaddr_in));
4309 target_proto_sinarp.sin_other |= SIN_ROUTER;
4310 target_proto = (struct sockaddr *)&target_proto_sinarp;
4311 }
4312
4313 /*
4314 * If this is an ARP request and the target IP is IPv4LL,
4315 * send the request on all interfaces. The exception is
4316 * an announcement, which must only appear on the specific
4317 * interface.
4318 */
4319 sender_sin = (struct sockaddr_in *)(void *)(uintptr_t)sender_proto;
4320 target_sin = (struct sockaddr_in *)(void *)(uintptr_t)target_proto;
4321 if (target_proto->sa_family == AF_INET &&
4322 IN_LINKLOCAL(ntohl(target_sin->sin_addr.s_addr)) &&
4323 ipv4_ll_arp_aware != 0 && arpop == ARPOP_REQUEST &&
4324 !_is_announcement(target_sin, sender_sin)) {
4325 ifnet_t *ifp_list;
4326 u_int32_t count;
4327 u_int32_t ifp_on;
4328
4329 result = ENOTSUP;
4330
4331 if (ifnet_list_get(IFNET_FAMILY_ANY, &ifp_list, &count) == 0) {
4332 for (ifp_on = 0; ifp_on < count; ifp_on++) {
4333 errno_t new_result;
4334 ifaddr_t source_hw = NULL;
4335 ifaddr_t source_ip = NULL;
4336 struct sockaddr_in source_ip_copy;
4337 struct ifnet *cur_ifp = ifp_list[ifp_on];
4338
4339 /*
4340 * Only arp on interfaces marked for IPv4LL
4341 * ARPing. This may mean that we don't ARP on
4342 * the interface the subnet route points to.
4343 */
4344 if (!(cur_ifp->if_eflags & IFEF_ARPLL))
4345 continue;
4346
4347 /* Find the source IP address */
4348 ifnet_lock_shared(cur_ifp);
4349 source_hw = cur_ifp->if_lladdr;
4350 TAILQ_FOREACH(source_ip, &cur_ifp->if_addrhead,
4351 ifa_link) {
4352 IFA_LOCK(source_ip);
4353 if (source_ip->ifa_addr != NULL &&
4354 source_ip->ifa_addr->sa_family ==
4355 AF_INET) {
4356 /* Copy the source IP address */
4357 source_ip_copy =
4358 *(struct sockaddr_in *)
4359 (void *)source_ip->ifa_addr;
4360 IFA_UNLOCK(source_ip);
4361 break;
4362 }
4363 IFA_UNLOCK(source_ip);
4364 }
4365
4366 /* No IP Source, don't arp */
4367 if (source_ip == NULL) {
4368 ifnet_lock_done(cur_ifp);
4369 continue;
4370 }
4371
4372 IFA_ADDREF(source_hw);
4373 ifnet_lock_done(cur_ifp);
4374
4375 /* Send the ARP */
4376 new_result = dlil_send_arp_internal(cur_ifp,
4377 arpop, (struct sockaddr_dl *)(void *)
4378 source_hw->ifa_addr,
4379 (struct sockaddr *)&source_ip_copy, NULL,
4380 target_proto);
4381
4382 IFA_REMREF(source_hw);
4383 if (result == ENOTSUP) {
4384 result = new_result;
4385 }
4386 }
4387 ifnet_list_free(ifp_list);
4388 }
4389 } else {
4390 result = dlil_send_arp_internal(ifp, arpop, sender_hw,
4391 sender_proto, target_hw, target_proto);
4392 }
4393
4394 return (result);
4395 }
4396
4397 /*
4398 * Caller must hold ifnet head lock.
4399 */
4400 static int
4401 ifnet_lookup(struct ifnet *ifp)
4402 {
4403 struct ifnet *_ifp;
4404
4405 lck_rw_assert(&ifnet_head_lock, LCK_RW_ASSERT_HELD);
4406 TAILQ_FOREACH(_ifp, &ifnet_head, if_link) {
4407 if (_ifp == ifp)
4408 break;
4409 }
4410 return (_ifp != NULL);
4411 }
4412 /*
4413 * Caller has to pass a non-zero refio argument to get a
4414 * IO reference count. This will prevent ifnet_detach from
4415 * being called when there are outstanding io reference counts.
4416 */
4417 int
4418 ifnet_is_attached(struct ifnet *ifp, int refio)
4419 {
4420 int ret;
4421
4422 lck_mtx_lock_spin(&ifp->if_ref_lock);
4423 if ((ret = ((ifp->if_refflags & (IFRF_ATTACHED | IFRF_DETACHING)) ==
4424 IFRF_ATTACHED))) {
4425 if (refio > 0)
4426 ifp->if_refio++;
4427 }
4428 lck_mtx_unlock(&ifp->if_ref_lock);
4429
4430 return (ret);
4431 }
4432
4433 void
4434 ifnet_decr_iorefcnt(struct ifnet *ifp)
4435 {
4436 lck_mtx_lock_spin(&ifp->if_ref_lock);
4437 VERIFY(ifp->if_refio > 0);
4438 VERIFY((ifp->if_refflags & (IFRF_ATTACHED | IFRF_DETACHING)) != 0);
4439 ifp->if_refio--;
4440
4441 /* if there are no more outstanding io references, wakeup the
4442 * ifnet_detach thread if detaching flag is set.
4443 */
4444 if (ifp->if_refio == 0 &&
4445 (ifp->if_refflags & IFRF_DETACHING) != 0) {
4446 wakeup(&(ifp->if_refio));
4447 }
4448 lck_mtx_unlock(&ifp->if_ref_lock);
4449 }
4450
4451 static void
4452 dlil_if_trace(struct dlil_ifnet *dl_if, int refhold)
4453 {
4454 struct dlil_ifnet_dbg *dl_if_dbg = (struct dlil_ifnet_dbg *)dl_if;
4455 ctrace_t *tr;
4456 u_int32_t idx;
4457 u_int16_t *cnt;
4458
4459 if (!(dl_if->dl_if_flags & DLIF_DEBUG)) {
4460 panic("%s: dl_if %p has no debug structure", __func__, dl_if);
4461 /* NOTREACHED */
4462 }
4463
4464 if (refhold) {
4465 cnt = &dl_if_dbg->dldbg_if_refhold_cnt;
4466 tr = dl_if_dbg->dldbg_if_refhold;
4467 } else {
4468 cnt = &dl_if_dbg->dldbg_if_refrele_cnt;
4469 tr = dl_if_dbg->dldbg_if_refrele;
4470 }
4471
4472 idx = atomic_add_16_ov(cnt, 1) % IF_REF_TRACE_HIST_SIZE;
4473 ctrace_record(&tr[idx]);
4474 }
4475
4476 errno_t
4477 dlil_if_ref(struct ifnet *ifp)
4478 {
4479 struct dlil_ifnet *dl_if = (struct dlil_ifnet *)ifp;
4480
4481 if (dl_if == NULL)
4482 return (EINVAL);
4483
4484 lck_mtx_lock_spin(&dl_if->dl_if_lock);
4485 ++dl_if->dl_if_refcnt;
4486 if (dl_if->dl_if_refcnt == 0) {
4487 panic("%s: wraparound refcnt for ifp=%p", __func__, ifp);
4488 /* NOTREACHED */
4489 }
4490 if (dl_if->dl_if_trace != NULL)
4491 (*dl_if->dl_if_trace)(dl_if, TRUE);
4492 lck_mtx_unlock(&dl_if->dl_if_lock);
4493
4494 return (0);
4495 }
4496
4497 errno_t
4498 dlil_if_free(struct ifnet *ifp)
4499 {
4500 struct dlil_ifnet *dl_if = (struct dlil_ifnet *)ifp;
4501
4502 if (dl_if == NULL)
4503 return (EINVAL);
4504
4505 lck_mtx_lock_spin(&dl_if->dl_if_lock);
4506 if (dl_if->dl_if_refcnt == 0) {
4507 panic("%s: negative refcnt for ifp=%p", __func__, ifp);
4508 /* NOTREACHED */
4509 }
4510 --dl_if->dl_if_refcnt;
4511 if (dl_if->dl_if_trace != NULL)
4512 (*dl_if->dl_if_trace)(dl_if, FALSE);
4513 lck_mtx_unlock(&dl_if->dl_if_lock);
4514
4515 return (0);
4516 }
4517
4518 static errno_t
4519 dlil_attach_protocol_internal(struct if_proto *proto,
4520 const struct ifnet_demux_desc *demux_list, u_int32_t demux_count)
4521 {
4522 struct kev_dl_proto_data ev_pr_data;
4523 struct ifnet *ifp = proto->ifp;
4524 int retval = 0;
4525 u_int32_t hash_value = proto_hash_value(proto->protocol_family);
4526 struct if_proto *prev_proto;
4527 struct if_proto *_proto;
4528
4529 /* callee holds a proto refcnt upon success */
4530 ifnet_lock_exclusive(ifp);
4531 _proto = find_attached_proto(ifp, proto->protocol_family);
4532 if (_proto != NULL) {
4533 ifnet_lock_done(ifp);
4534 if_proto_free(_proto);
4535 return (EEXIST);
4536 }
4537
4538 /*
4539 * Call family module add_proto routine so it can refine the
4540 * demux descriptors as it wishes.
4541 */
4542 retval = ifp->if_add_proto(ifp, proto->protocol_family, demux_list,
4543 demux_count);
4544 if (retval) {
4545 ifnet_lock_done(ifp);
4546 return (retval);
4547 }
4548
4549 /*
4550 * Insert the protocol in the hash
4551 */
4552 prev_proto = SLIST_FIRST(&ifp->if_proto_hash[hash_value]);
4553 while (prev_proto != NULL && SLIST_NEXT(prev_proto, next_hash) != NULL)
4554 prev_proto = SLIST_NEXT(prev_proto, next_hash);
4555 if (prev_proto)
4556 SLIST_INSERT_AFTER(prev_proto, proto, next_hash);
4557 else
4558 SLIST_INSERT_HEAD(&ifp->if_proto_hash[hash_value],
4559 proto, next_hash);
4560
4561 /* hold a proto refcnt for attach */
4562 if_proto_ref(proto);
4563
4564 /*
4565 * The reserved field carries the number of protocol still attached
4566 * (subject to change)
4567 */
4568 ev_pr_data.proto_family = proto->protocol_family;
4569 ev_pr_data.proto_remaining_count = dlil_ifp_proto_count(ifp);
4570 ifnet_lock_done(ifp);
4571
4572 dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_PROTO_ATTACHED,
4573 (struct net_event_data *)&ev_pr_data,
4574 sizeof (struct kev_dl_proto_data));
4575 return (retval);
4576 }
4577
4578 errno_t
4579 ifnet_attach_protocol(ifnet_t ifp, protocol_family_t protocol,
4580 const struct ifnet_attach_proto_param *proto_details)
4581 {
4582 int retval = 0;
4583 struct if_proto *ifproto = NULL;
4584
4585 ifnet_head_lock_shared();
4586 if (ifp == NULL || protocol == 0 || proto_details == NULL) {
4587 retval = EINVAL;
4588 goto end;
4589 }
4590 /* Check that the interface is in the global list */
4591 if (!ifnet_lookup(ifp)) {
4592 retval = ENXIO;
4593 goto end;
4594 }
4595
4596 ifproto = zalloc(dlif_proto_zone);
4597 if (ifproto == NULL) {
4598 retval = ENOMEM;
4599 goto end;
4600 }
4601 bzero(ifproto, dlif_proto_size);
4602
4603 /* refcnt held above during lookup */
4604 ifproto->ifp = ifp;
4605 ifproto->protocol_family = protocol;
4606 ifproto->proto_kpi = kProtoKPI_v1;
4607 ifproto->kpi.v1.input = proto_details->input;
4608 ifproto->kpi.v1.pre_output = proto_details->pre_output;
4609 ifproto->kpi.v1.event = proto_details->event;
4610 ifproto->kpi.v1.ioctl = proto_details->ioctl;
4611 ifproto->kpi.v1.detached = proto_details->detached;
4612 ifproto->kpi.v1.resolve_multi = proto_details->resolve;
4613 ifproto->kpi.v1.send_arp = proto_details->send_arp;
4614
4615 retval = dlil_attach_protocol_internal(ifproto,
4616 proto_details->demux_list, proto_details->demux_count);
4617
4618 if (dlil_verbose) {
4619 printf("%s: attached v1 protocol %d\n", if_name(ifp),
4620 protocol);
4621 }
4622
4623 end:
4624 if (retval != 0 && retval != EEXIST && ifp != NULL) {
4625 DLIL_PRINTF("%s: failed to attach v1 protocol %d (err=%d)\n",
4626 if_name(ifp), protocol, retval);
4627 }
4628 ifnet_head_done();
4629 if (retval != 0 && ifproto != NULL)
4630 zfree(dlif_proto_zone, ifproto);
4631 return (retval);
4632 }
4633
4634 errno_t
4635 ifnet_attach_protocol_v2(ifnet_t ifp, protocol_family_t protocol,
4636 const struct ifnet_attach_proto_param_v2 *proto_details)
4637 {
4638 int retval = 0;
4639 struct if_proto *ifproto = NULL;
4640
4641 ifnet_head_lock_shared();
4642 if (ifp == NULL || protocol == 0 || proto_details == NULL) {
4643 retval = EINVAL;
4644 goto end;
4645 }
4646 /* Check that the interface is in the global list */
4647 if (!ifnet_lookup(ifp)) {
4648 retval = ENXIO;
4649 goto end;
4650 }
4651
4652 ifproto = zalloc(dlif_proto_zone);
4653 if (ifproto == NULL) {
4654 retval = ENOMEM;
4655 goto end;
4656 }
4657 bzero(ifproto, sizeof(*ifproto));
4658
4659 /* refcnt held above during lookup */
4660 ifproto->ifp = ifp;
4661 ifproto->protocol_family = protocol;
4662 ifproto->proto_kpi = kProtoKPI_v2;
4663 ifproto->kpi.v2.input = proto_details->input;
4664 ifproto->kpi.v2.pre_output = proto_details->pre_output;
4665 ifproto->kpi.v2.event = proto_details->event;
4666 ifproto->kpi.v2.ioctl = proto_details->ioctl;
4667 ifproto->kpi.v2.detached = proto_details->detached;
4668 ifproto->kpi.v2.resolve_multi = proto_details->resolve;
4669 ifproto->kpi.v2.send_arp = proto_details->send_arp;
4670
4671 retval = dlil_attach_protocol_internal(ifproto,
4672 proto_details->demux_list, proto_details->demux_count);
4673
4674 if (dlil_verbose) {
4675 printf("%s: attached v2 protocol %d\n", if_name(ifp),
4676 protocol);
4677 }
4678
4679 end:
4680 if (retval != 0 && retval != EEXIST && ifp != NULL) {
4681 DLIL_PRINTF("%s: failed to attach v2 protocol %d (err=%d)\n",
4682 if_name(ifp), protocol, retval);
4683 }
4684 ifnet_head_done();
4685 if (retval != 0 && ifproto != NULL)
4686 zfree(dlif_proto_zone, ifproto);
4687 return (retval);
4688 }
4689
4690 errno_t
4691 ifnet_detach_protocol(ifnet_t ifp, protocol_family_t proto_family)
4692 {
4693 struct if_proto *proto = NULL;
4694 int retval = 0;
4695
4696 if (ifp == NULL || proto_family == 0) {
4697 retval = EINVAL;
4698 goto end;
4699 }
4700
4701 ifnet_lock_exclusive(ifp);
4702 /* callee holds a proto refcnt upon success */
4703 proto = find_attached_proto(ifp, proto_family);
4704 if (proto == NULL) {
4705 retval = ENXIO;
4706 ifnet_lock_done(ifp);
4707 goto end;
4708 }
4709
4710 /* call family module del_proto */
4711 if (ifp->if_del_proto)
4712 ifp->if_del_proto(ifp, proto->protocol_family);
4713
4714 SLIST_REMOVE(&ifp->if_proto_hash[proto_hash_value(proto_family)],
4715 proto, if_proto, next_hash);
4716
4717 if (proto->proto_kpi == kProtoKPI_v1) {
4718 proto->kpi.v1.input = ifproto_media_input_v1;
4719 proto->kpi.v1.pre_output= ifproto_media_preout;
4720 proto->kpi.v1.event = ifproto_media_event;
4721 proto->kpi.v1.ioctl = ifproto_media_ioctl;
4722 proto->kpi.v1.resolve_multi = ifproto_media_resolve_multi;
4723 proto->kpi.v1.send_arp = ifproto_media_send_arp;
4724 } else {
4725 proto->kpi.v2.input = ifproto_media_input_v2;
4726 proto->kpi.v2.pre_output = ifproto_media_preout;
4727 proto->kpi.v2.event = ifproto_media_event;
4728 proto->kpi.v2.ioctl = ifproto_media_ioctl;
4729 proto->kpi.v2.resolve_multi = ifproto_media_resolve_multi;
4730 proto->kpi.v2.send_arp = ifproto_media_send_arp;
4731 }
4732 proto->detached = 1;
4733 ifnet_lock_done(ifp);
4734
4735 if (dlil_verbose) {
4736 printf("%s: detached %s protocol %d\n", if_name(ifp),
4737 (proto->proto_kpi == kProtoKPI_v1) ?
4738 "v1" : "v2", proto_family);
4739 }
4740
4741 /* release proto refcnt held during protocol attach */
4742 if_proto_free(proto);
4743
4744 /*
4745 * Release proto refcnt held during lookup; the rest of
4746 * protocol detach steps will happen when the last proto
4747 * reference is released.
4748 */
4749 if_proto_free(proto);
4750
4751 end:
4752 return (retval);
4753 }
4754
4755
4756 static errno_t
4757 ifproto_media_input_v1(struct ifnet *ifp, protocol_family_t protocol,
4758 struct mbuf *packet, char *header)
4759 {
4760 #pragma unused(ifp, protocol, packet, header)
4761 return (ENXIO);
4762 }
4763
4764 static errno_t
4765 ifproto_media_input_v2(struct ifnet *ifp, protocol_family_t protocol,
4766 struct mbuf *packet)
4767 {
4768 #pragma unused(ifp, protocol, packet)
4769 return (ENXIO);
4770
4771 }
4772
4773 static errno_t
4774 ifproto_media_preout(struct ifnet *ifp, protocol_family_t protocol,
4775 mbuf_t *packet, const struct sockaddr *dest, void *route, char *frame_type,
4776 char *link_layer_dest)
4777 {
4778 #pragma unused(ifp, protocol, packet, dest, route, frame_type, link_layer_dest)
4779 return (ENXIO);
4780
4781 }
4782
4783 static void
4784 ifproto_media_event(struct ifnet *ifp, protocol_family_t protocol,
4785 const struct kev_msg *event)
4786 {
4787 #pragma unused(ifp, protocol, event)
4788 }
4789
4790 static errno_t
4791 ifproto_media_ioctl(struct ifnet *ifp, protocol_family_t protocol,
4792 unsigned long command, void *argument)
4793 {
4794 #pragma unused(ifp, protocol, command, argument)
4795 return (ENXIO);
4796 }
4797
4798 static errno_t
4799 ifproto_media_resolve_multi(ifnet_t ifp, const struct sockaddr *proto_addr,
4800 struct sockaddr_dl *out_ll, size_t ll_len)
4801 {
4802 #pragma unused(ifp, proto_addr, out_ll, ll_len)
4803 return (ENXIO);
4804 }
4805
4806 static errno_t
4807 ifproto_media_send_arp(struct ifnet *ifp, u_short arpop,
4808 const struct sockaddr_dl *sender_hw, const struct sockaddr *sender_proto,
4809 const struct sockaddr_dl *target_hw, const struct sockaddr *target_proto)
4810 {
4811 #pragma unused(ifp, arpop, sender_hw, sender_proto, target_hw, target_proto)
4812 return (ENXIO);
4813 }
4814
4815 extern int if_next_index(void);
4816
4817 errno_t
4818 ifnet_attach(ifnet_t ifp, const struct sockaddr_dl *ll_addr)
4819 {
4820 struct ifnet *tmp_if;
4821 struct ifaddr *ifa;
4822 struct if_data_internal if_data_saved;
4823 struct dlil_ifnet *dl_if = (struct dlil_ifnet *)ifp;
4824 struct dlil_threading_info *dl_inp;
4825 u_int32_t sflags = 0;
4826 int err;
4827
4828 if (ifp == NULL)
4829 return (EINVAL);
4830
4831 /*
4832 * Serialize ifnet attach using dlil_ifnet_lock, in order to
4833 * prevent the interface from being configured while it is
4834 * embryonic, as ifnet_head_lock is dropped and reacquired
4835 * below prior to marking the ifnet with IFRF_ATTACHED.
4836 */
4837 dlil_if_lock();
4838 ifnet_head_lock_exclusive();
4839 /* Verify we aren't already on the list */
4840 TAILQ_FOREACH(tmp_if, &ifnet_head, if_link) {
4841 if (tmp_if == ifp) {
4842 ifnet_head_done();
4843 dlil_if_unlock();
4844 return (EEXIST);
4845 }
4846 }
4847
4848 lck_mtx_lock_spin(&ifp->if_ref_lock);
4849 if (ifp->if_refflags & IFRF_ATTACHED) {
4850 panic_plain("%s: flags mismatch (attached set) ifp=%p",
4851 __func__, ifp);
4852 /* NOTREACHED */
4853 }
4854 lck_mtx_unlock(&ifp->if_ref_lock);
4855
4856 ifnet_lock_exclusive(ifp);
4857
4858 /* Sanity check */
4859 VERIFY(ifp->if_detaching_link.tqe_next == NULL);
4860 VERIFY(ifp->if_detaching_link.tqe_prev == NULL);
4861
4862 if (ll_addr != NULL) {
4863 if (ifp->if_addrlen == 0) {
4864 ifp->if_addrlen = ll_addr->sdl_alen;
4865 } else if (ll_addr->sdl_alen != ifp->if_addrlen) {
4866 ifnet_lock_done(ifp);
4867 ifnet_head_done();
4868 dlil_if_unlock();
4869 return (EINVAL);
4870 }
4871 }
4872
4873 /*
4874 * Allow interfaces without protocol families to attach
4875 * only if they have the necessary fields filled out.
4876 */
4877 if (ifp->if_add_proto == NULL || ifp->if_del_proto == NULL) {
4878 DLIL_PRINTF("%s: Attempt to attach interface without "
4879 "family module - %d\n", __func__, ifp->if_family);
4880 ifnet_lock_done(ifp);
4881 ifnet_head_done();
4882 dlil_if_unlock();
4883 return (ENODEV);
4884 }
4885
4886 /* Allocate protocol hash table */
4887 VERIFY(ifp->if_proto_hash == NULL);
4888 ifp->if_proto_hash = zalloc(dlif_phash_zone);
4889 if (ifp->if_proto_hash == NULL) {
4890 ifnet_lock_done(ifp);
4891 ifnet_head_done();
4892 dlil_if_unlock();
4893 return (ENOBUFS);
4894 }
4895 bzero(ifp->if_proto_hash, dlif_phash_size);
4896
4897 lck_mtx_lock_spin(&ifp->if_flt_lock);
4898 VERIFY(TAILQ_EMPTY(&ifp->if_flt_head));
4899 TAILQ_INIT(&ifp->if_flt_head);
4900 VERIFY(ifp->if_flt_busy == 0);
4901 VERIFY(ifp->if_flt_waiters == 0);
4902 lck_mtx_unlock(&ifp->if_flt_lock);
4903
4904 VERIFY(TAILQ_EMPTY(&ifp->if_prefixhead));
4905 TAILQ_INIT(&ifp->if_prefixhead);
4906
4907 if (!(dl_if->dl_if_flags & DLIF_REUSE)) {
4908 VERIFY(LIST_EMPTY(&ifp->if_multiaddrs));
4909 LIST_INIT(&ifp->if_multiaddrs);
4910 }
4911
4912 VERIFY(ifp->if_allhostsinm == NULL);
4913 VERIFY(TAILQ_EMPTY(&ifp->if_addrhead));
4914 TAILQ_INIT(&ifp->if_addrhead);
4915
4916 if (ifp->if_index == 0) {
4917 int idx = if_next_index();
4918
4919 if (idx == -1) {
4920 ifp->if_index = 0;
4921 ifnet_lock_done(ifp);
4922 ifnet_head_done();
4923 dlil_if_unlock();
4924 return (ENOBUFS);
4925 }
4926 ifp->if_index = idx;
4927 }
4928 /* There should not be anything occupying this slot */
4929 VERIFY(ifindex2ifnet[ifp->if_index] == NULL);
4930
4931 /* allocate (if needed) and initialize a link address */
4932 VERIFY(!(dl_if->dl_if_flags & DLIF_REUSE) || ifp->if_lladdr != NULL);
4933 ifa = dlil_alloc_lladdr(ifp, ll_addr);
4934 if (ifa == NULL) {
4935 ifnet_lock_done(ifp);
4936 ifnet_head_done();
4937 dlil_if_unlock();
4938 return (ENOBUFS);
4939 }
4940
4941 VERIFY(ifnet_addrs[ifp->if_index - 1] == NULL);
4942 ifnet_addrs[ifp->if_index - 1] = ifa;
4943
4944 /* make this address the first on the list */
4945 IFA_LOCK(ifa);
4946 /* hold a reference for ifnet_addrs[] */
4947 IFA_ADDREF_LOCKED(ifa);
4948 /* if_attach_link_ifa() holds a reference for ifa_link */
4949 if_attach_link_ifa(ifp, ifa);
4950 IFA_UNLOCK(ifa);
4951
4952 #if CONFIG_MACF_NET
4953 mac_ifnet_label_associate(ifp);
4954 #endif
4955
4956 TAILQ_INSERT_TAIL(&ifnet_head, ifp, if_link);
4957 ifindex2ifnet[ifp->if_index] = ifp;
4958
4959 /* Hold a reference to the underlying dlil_ifnet */
4960 ifnet_reference(ifp);
4961
4962 /* Clear stats (save and restore other fields that we care) */
4963 if_data_saved = ifp->if_data;
4964 bzero(&ifp->if_data, sizeof (ifp->if_data));
4965 ifp->if_data.ifi_type = if_data_saved.ifi_type;
4966 ifp->if_data.ifi_typelen = if_data_saved.ifi_typelen;
4967 ifp->if_data.ifi_physical = if_data_saved.ifi_physical;
4968 ifp->if_data.ifi_addrlen = if_data_saved.ifi_addrlen;
4969 ifp->if_data.ifi_hdrlen = if_data_saved.ifi_hdrlen;
4970 ifp->if_data.ifi_mtu = if_data_saved.ifi_mtu;
4971 ifp->if_data.ifi_baudrate = if_data_saved.ifi_baudrate;
4972 ifp->if_data.ifi_hwassist = if_data_saved.ifi_hwassist;
4973 ifp->if_data.ifi_tso_v4_mtu = if_data_saved.ifi_tso_v4_mtu;
4974 ifp->if_data.ifi_tso_v6_mtu = if_data_saved.ifi_tso_v6_mtu;
4975 ifnet_touch_lastchange(ifp);
4976
4977 VERIFY(ifp->if_output_sched_model == IFNET_SCHED_MODEL_NORMAL ||
4978 ifp->if_output_sched_model == IFNET_SCHED_MODEL_DRIVER_MANAGED);
4979
4980 /* By default, use SFB and enable flow advisory */
4981 sflags = PKTSCHEDF_QALG_SFB;
4982 if (if_flowadv)
4983 sflags |= PKTSCHEDF_QALG_FLOWCTL;
4984
4985 /* Initialize transmit queue(s) */
4986 err = ifclassq_setup(ifp, sflags, (dl_if->dl_if_flags & DLIF_REUSE));
4987 if (err != 0) {
4988 panic_plain("%s: ifp=%p couldn't initialize transmit queue; "
4989 "err=%d", __func__, ifp, err);
4990 /* NOTREACHED */
4991 }
4992
4993 /* Sanity checks on the input thread storage */
4994 dl_inp = &dl_if->dl_if_inpstorage;
4995 bzero(&dl_inp->stats, sizeof (dl_inp->stats));
4996 VERIFY(dl_inp->input_waiting == 0);
4997 VERIFY(dl_inp->wtot == 0);
4998 VERIFY(dl_inp->ifp == NULL);
4999 VERIFY(qhead(&dl_inp->rcvq_pkts) == NULL && qempty(&dl_inp->rcvq_pkts));
5000 VERIFY(qlimit(&dl_inp->rcvq_pkts) == 0);
5001 VERIFY(!dl_inp->net_affinity);
5002 VERIFY(ifp->if_inp == NULL);
5003 VERIFY(dl_inp->input_thr == THREAD_NULL);
5004 VERIFY(dl_inp->wloop_thr == THREAD_NULL);
5005 VERIFY(dl_inp->poll_thr == THREAD_NULL);
5006 VERIFY(dl_inp->tag == 0);
5007 VERIFY(dl_inp->mode == IFNET_MODEL_INPUT_POLL_OFF);
5008 bzero(&dl_inp->tstats, sizeof (dl_inp->tstats));
5009 bzero(&dl_inp->pstats, sizeof (dl_inp->pstats));
5010 bzero(&dl_inp->sstats, sizeof (dl_inp->sstats));
5011 #if IFNET_INPUT_SANITY_CHK
5012 VERIFY(dl_inp->input_mbuf_cnt == 0);
5013 #endif /* IFNET_INPUT_SANITY_CHK */
5014
5015 /*
5016 * A specific DLIL input thread is created per Ethernet/cellular
5017 * interface or for an interface which supports opportunistic
5018 * input polling. Pseudo interfaces or other types of interfaces
5019 * use the main input thread instead.
5020 */
5021 if ((net_rxpoll && (ifp->if_eflags & IFEF_RXPOLL)) ||
5022 ifp->if_type == IFT_ETHER || ifp->if_type == IFT_CELLULAR) {
5023 ifp->if_inp = dl_inp;
5024 err = dlil_create_input_thread(ifp, ifp->if_inp);
5025 if (err != 0) {
5026 panic_plain("%s: ifp=%p couldn't get an input thread; "
5027 "err=%d", __func__, ifp, err);
5028 /* NOTREACHED */
5029 }
5030 }
5031
5032 /*
5033 * If the driver supports the new transmit model, calculate flow hash
5034 * and create a workloop starter thread to invoke the if_start callback
5035 * where the packets may be dequeued and transmitted.
5036 */
5037 if (ifp->if_eflags & IFEF_TXSTART) {
5038 ifp->if_flowhash = ifnet_calc_flowhash(ifp);
5039 VERIFY(ifp->if_flowhash != 0);
5040
5041 VERIFY(ifp->if_start != NULL);
5042 VERIFY(ifp->if_start_thread == THREAD_NULL);
5043
5044 ifnet_set_start_cycle(ifp, NULL);
5045 ifp->if_start_active = 0;
5046 ifp->if_start_req = 0;
5047 ifp->if_start_flags = 0;
5048 if ((err = kernel_thread_start(ifnet_start_thread_fn, ifp,
5049 &ifp->if_start_thread)) != KERN_SUCCESS) {
5050 panic_plain("%s: ifp=%p couldn't get a start thread; "
5051 "err=%d", __func__, ifp, err);
5052 /* NOTREACHED */
5053 }
5054 ml_thread_policy(ifp->if_start_thread, MACHINE_GROUP,
5055 (MACHINE_NETWORK_GROUP|MACHINE_NETWORK_WORKLOOP));
5056 } else {
5057 ifp->if_flowhash = 0;
5058 }
5059
5060 /*
5061 * If the driver supports the new receive model, create a poller
5062 * thread to invoke if_input_poll callback where the packets may
5063 * be dequeued from the driver and processed for reception.
5064 */
5065 if (ifp->if_eflags & IFEF_RXPOLL) {
5066 VERIFY(ifp->if_input_poll != NULL);
5067 VERIFY(ifp->if_input_ctl != NULL);
5068 VERIFY(ifp->if_poll_thread == THREAD_NULL);
5069
5070 ifnet_set_poll_cycle(ifp, NULL);
5071 ifp->if_poll_update = 0;
5072 ifp->if_poll_active = 0;
5073 ifp->if_poll_req = 0;
5074 if ((err = kernel_thread_start(ifnet_poll_thread_fn, ifp,
5075 &ifp->if_poll_thread)) != KERN_SUCCESS) {
5076 panic_plain("%s: ifp=%p couldn't get a poll thread; "
5077 "err=%d", __func__, ifp, err);
5078 /* NOTREACHED */
5079 }
5080 ml_thread_policy(ifp->if_poll_thread, MACHINE_GROUP,
5081 (MACHINE_NETWORK_GROUP|MACHINE_NETWORK_WORKLOOP));
5082 }
5083
5084 VERIFY(ifp->if_desc.ifd_maxlen == IF_DESCSIZE);
5085 VERIFY(ifp->if_desc.ifd_len == 0);
5086 VERIFY(ifp->if_desc.ifd_desc != NULL);
5087
5088 /* Record attach PC stacktrace */
5089 ctrace_record(&((struct dlil_ifnet *)ifp)->dl_if_attach);
5090
5091 ifp->if_updatemcasts = 0;
5092 if (!LIST_EMPTY(&ifp->if_multiaddrs)) {
5093 struct ifmultiaddr *ifma;
5094 LIST_FOREACH(ifma, &ifp->if_multiaddrs, ifma_link) {
5095 IFMA_LOCK(ifma);
5096 if (ifma->ifma_addr->sa_family == AF_LINK ||
5097 ifma->ifma_addr->sa_family == AF_UNSPEC)
5098 ifp->if_updatemcasts++;
5099 IFMA_UNLOCK(ifma);
5100 }
5101
5102 printf("%s: attached with %d suspended link-layer multicast "
5103 "membership(s)\n", if_name(ifp),
5104 ifp->if_updatemcasts);
5105 }
5106
5107 /* Clear logging parameters */
5108 bzero(&ifp->if_log, sizeof (ifp->if_log));
5109 ifp->if_fg_sendts = 0;
5110
5111 VERIFY(ifp->if_delegated.ifp == NULL);
5112 VERIFY(ifp->if_delegated.type == 0);
5113 VERIFY(ifp->if_delegated.family == 0);
5114 VERIFY(ifp->if_delegated.subfamily == 0);
5115
5116 ifnet_lock_done(ifp);
5117 ifnet_head_done();
5118
5119 lck_mtx_lock(&ifp->if_cached_route_lock);
5120 /* Enable forwarding cached route */
5121 ifp->if_fwd_cacheok = 1;
5122 /* Clean up any existing cached routes */
5123 ROUTE_RELEASE(&ifp->if_fwd_route);
5124 bzero(&ifp->if_fwd_route, sizeof (ifp->if_fwd_route));
5125 ROUTE_RELEASE(&ifp->if_src_route);
5126 bzero(&ifp->if_src_route, sizeof (ifp->if_src_route));
5127 ROUTE_RELEASE(&ifp->if_src_route6);
5128 bzero(&ifp->if_src_route6, sizeof (ifp->if_src_route6));
5129 lck_mtx_unlock(&ifp->if_cached_route_lock);
5130
5131 ifnet_llreach_ifattach(ifp, (dl_if->dl_if_flags & DLIF_REUSE));
5132
5133 /*
5134 * Allocate and attach IGMPv3/MLDv2 interface specific variables
5135 * and trees; do this before the ifnet is marked as attached.
5136 * The ifnet keeps the reference to the info structures even after
5137 * the ifnet is detached, since the network-layer records still
5138 * refer to the info structures even after that. This also
5139 * makes it possible for them to still function after the ifnet
5140 * is recycled or reattached.
5141 */
5142 #if INET
5143 if (IGMP_IFINFO(ifp) == NULL) {
5144 IGMP_IFINFO(ifp) = igmp_domifattach(ifp, M_WAITOK);
5145 VERIFY(IGMP_IFINFO(ifp) != NULL);
5146 } else {
5147 VERIFY(IGMP_IFINFO(ifp)->igi_ifp == ifp);
5148 igmp_domifreattach(IGMP_IFINFO(ifp));
5149 }
5150 #endif /* INET */
5151 #if INET6
5152 if (MLD_IFINFO(ifp) == NULL) {
5153 MLD_IFINFO(ifp) = mld_domifattach(ifp, M_WAITOK);
5154 VERIFY(MLD_IFINFO(ifp) != NULL);
5155 } else {
5156 VERIFY(MLD_IFINFO(ifp)->mli_ifp == ifp);
5157 mld_domifreattach(MLD_IFINFO(ifp));
5158 }
5159 #endif /* INET6 */
5160
5161 VERIFY(ifp->if_data_threshold == 0);
5162
5163 /*
5164 * Finally, mark this ifnet as attached.
5165 */
5166 lck_mtx_lock(rnh_lock);
5167 ifnet_lock_exclusive(ifp);
5168 /* Initialize Link Quality Metric (loopback [lo0] is always good) */
5169 ifp->if_lqm = (ifp == lo_ifp) ? IFNET_LQM_THRESH_GOOD :
5170 IFNET_LQM_THRESH_UNKNOWN;
5171 lck_mtx_lock_spin(&ifp->if_ref_lock);
5172 ifp->if_refflags = IFRF_ATTACHED;
5173 lck_mtx_unlock(&ifp->if_ref_lock);
5174 if (net_rtref) {
5175 /* boot-args override; enable idle notification */
5176 (void) ifnet_set_idle_flags_locked(ifp, IFRF_IDLE_NOTIFY,
5177 IFRF_IDLE_NOTIFY);
5178 } else {
5179 /* apply previous request(s) to set the idle flags, if any */
5180 (void) ifnet_set_idle_flags_locked(ifp, ifp->if_idle_new_flags,
5181 ifp->if_idle_new_flags_mask);
5182
5183 }
5184 ifnet_lock_done(ifp);
5185 lck_mtx_unlock(rnh_lock);
5186 dlil_if_unlock();
5187
5188 #if PF
5189 /*
5190 * Attach packet filter to this interface, if enabled.
5191 */
5192 pf_ifnet_hook(ifp, 1);
5193 #endif /* PF */
5194
5195 dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_IF_ATTACHED, NULL, 0);
5196
5197 if (dlil_verbose) {
5198 printf("%s: attached%s\n", if_name(ifp),
5199 (dl_if->dl_if_flags & DLIF_REUSE) ? " (recycled)" : "");
5200 }
5201
5202 return (0);
5203 }
5204
5205 /*
5206 * Prepare the storage for the first/permanent link address, which must
5207 * must have the same lifetime as the ifnet itself. Although the link
5208 * address gets removed from if_addrhead and ifnet_addrs[] at detach time,
5209 * its location in memory must never change as it may still be referred
5210 * to by some parts of the system afterwards (unfortunate implementation
5211 * artifacts inherited from BSD.)
5212 *
5213 * Caller must hold ifnet lock as writer.
5214 */
5215 static struct ifaddr *
5216 dlil_alloc_lladdr(struct ifnet *ifp, const struct sockaddr_dl *ll_addr)
5217 {
5218 struct ifaddr *ifa, *oifa;
5219 struct sockaddr_dl *asdl, *msdl;
5220 char workbuf[IFNAMSIZ*2];
5221 int namelen, masklen, socksize;
5222 struct dlil_ifnet *dl_if = (struct dlil_ifnet *)ifp;
5223
5224 ifnet_lock_assert(ifp, IFNET_LCK_ASSERT_EXCLUSIVE);
5225 VERIFY(ll_addr == NULL || ll_addr->sdl_alen == ifp->if_addrlen);
5226
5227 namelen = snprintf(workbuf, sizeof (workbuf), "%s",
5228 if_name(ifp));
5229 masklen = offsetof(struct sockaddr_dl, sdl_data[0]) + namelen;
5230 socksize = masklen + ifp->if_addrlen;
5231 #define ROUNDUP(a) (1 + (((a) - 1) | (sizeof (u_int32_t) - 1)))
5232 if ((u_int32_t)socksize < sizeof (struct sockaddr_dl))
5233 socksize = sizeof(struct sockaddr_dl);
5234 socksize = ROUNDUP(socksize);
5235 #undef ROUNDUP
5236
5237 ifa = ifp->if_lladdr;
5238 if (socksize > DLIL_SDLMAXLEN ||
5239 (ifa != NULL && ifa != &dl_if->dl_if_lladdr.ifa)) {
5240 /*
5241 * Rare, but in the event that the link address requires
5242 * more storage space than DLIL_SDLMAXLEN, allocate the
5243 * largest possible storages for address and mask, such
5244 * that we can reuse the same space when if_addrlen grows.
5245 * This same space will be used when if_addrlen shrinks.
5246 */
5247 if (ifa == NULL || ifa == &dl_if->dl_if_lladdr.ifa) {
5248 int ifasize = sizeof (*ifa) + 2 * SOCK_MAXADDRLEN;
5249 ifa = _MALLOC(ifasize, M_IFADDR, M_WAITOK | M_ZERO);
5250 if (ifa == NULL)
5251 return (NULL);
5252 ifa_lock_init(ifa);
5253 /* Don't set IFD_ALLOC, as this is permanent */
5254 ifa->ifa_debug = IFD_LINK;
5255 }
5256 IFA_LOCK(ifa);
5257 /* address and mask sockaddr_dl locations */
5258 asdl = (struct sockaddr_dl *)(ifa + 1);
5259 bzero(asdl, SOCK_MAXADDRLEN);
5260 msdl = (struct sockaddr_dl *)(void *)
5261 ((char *)asdl + SOCK_MAXADDRLEN);
5262 bzero(msdl, SOCK_MAXADDRLEN);
5263 } else {
5264 VERIFY(ifa == NULL || ifa == &dl_if->dl_if_lladdr.ifa);
5265 /*
5266 * Use the storage areas for address and mask within the
5267 * dlil_ifnet structure. This is the most common case.
5268 */
5269 if (ifa == NULL) {
5270 ifa = &dl_if->dl_if_lladdr.ifa;
5271 ifa_lock_init(ifa);
5272 /* Don't set IFD_ALLOC, as this is permanent */
5273 ifa->ifa_debug = IFD_LINK;
5274 }
5275 IFA_LOCK(ifa);
5276 /* address and mask sockaddr_dl locations */
5277 asdl = (struct sockaddr_dl *)(void *)&dl_if->dl_if_lladdr.asdl;
5278 bzero(asdl, sizeof (dl_if->dl_if_lladdr.asdl));
5279 msdl = (struct sockaddr_dl *)(void *)&dl_if->dl_if_lladdr.msdl;
5280 bzero(msdl, sizeof (dl_if->dl_if_lladdr.msdl));
5281 }
5282
5283 /* hold a permanent reference for the ifnet itself */
5284 IFA_ADDREF_LOCKED(ifa);
5285 oifa = ifp->if_lladdr;
5286 ifp->if_lladdr = ifa;
5287
5288 VERIFY(ifa->ifa_debug == IFD_LINK);
5289 ifa->ifa_ifp = ifp;
5290 ifa->ifa_rtrequest = link_rtrequest;
5291 ifa->ifa_addr = (struct sockaddr *)asdl;
5292 asdl->sdl_len = socksize;
5293 asdl->sdl_family = AF_LINK;
5294 bcopy(workbuf, asdl->sdl_data, namelen);
5295 asdl->sdl_nlen = namelen;
5296 asdl->sdl_index = ifp->if_index;
5297 asdl->sdl_type = ifp->if_type;
5298 if (ll_addr != NULL) {
5299 asdl->sdl_alen = ll_addr->sdl_alen;
5300 bcopy(CONST_LLADDR(ll_addr), LLADDR(asdl), asdl->sdl_alen);
5301 } else {
5302 asdl->sdl_alen = 0;
5303 }
5304 ifa->ifa_netmask = (struct sockaddr*)msdl;
5305 msdl->sdl_len = masklen;
5306 while (namelen != 0)
5307 msdl->sdl_data[--namelen] = 0xff;
5308 IFA_UNLOCK(ifa);
5309
5310 if (oifa != NULL)
5311 IFA_REMREF(oifa);
5312
5313 return (ifa);
5314 }
5315
5316 static void
5317 if_purgeaddrs(struct ifnet *ifp)
5318 {
5319 #if INET
5320 in_purgeaddrs(ifp);
5321 #endif /* INET */
5322 #if INET6
5323 in6_purgeaddrs(ifp);
5324 #endif /* INET6 */
5325 }
5326
5327 errno_t
5328 ifnet_detach(ifnet_t ifp)
5329 {
5330 struct ifnet *delegated_ifp;
5331
5332 if (ifp == NULL)
5333 return (EINVAL);
5334
5335 lck_mtx_lock(rnh_lock);
5336 ifnet_head_lock_exclusive();
5337 ifnet_lock_exclusive(ifp);
5338
5339 /*
5340 * Check to see if this interface has previously triggered
5341 * aggressive protocol draining; if so, decrement the global
5342 * refcnt and clear PR_AGGDRAIN on the route domain if
5343 * there are no more of such an interface around.
5344 */
5345 (void) ifnet_set_idle_flags_locked(ifp, 0, ~0);
5346
5347 lck_mtx_lock_spin(&ifp->if_ref_lock);
5348 if (!(ifp->if_refflags & IFRF_ATTACHED)) {
5349 lck_mtx_unlock(&ifp->if_ref_lock);
5350 ifnet_lock_done(ifp);
5351 ifnet_head_done();
5352 lck_mtx_unlock(rnh_lock);
5353 return (EINVAL);
5354 } else if (ifp->if_refflags & IFRF_DETACHING) {
5355 /* Interface has already been detached */
5356 lck_mtx_unlock(&ifp->if_ref_lock);
5357 ifnet_lock_done(ifp);
5358 ifnet_head_done();
5359 lck_mtx_unlock(rnh_lock);
5360 return (ENXIO);
5361 }
5362 /* Indicate this interface is being detached */
5363 ifp->if_refflags &= ~IFRF_ATTACHED;
5364 ifp->if_refflags |= IFRF_DETACHING;
5365 lck_mtx_unlock(&ifp->if_ref_lock);
5366
5367 if (dlil_verbose)
5368 printf("%s: detaching\n", if_name(ifp));
5369
5370 /*
5371 * Remove ifnet from the ifnet_head, ifindex2ifnet[]; it will
5372 * no longer be visible during lookups from this point.
5373 */
5374 VERIFY(ifindex2ifnet[ifp->if_index] == ifp);
5375 TAILQ_REMOVE(&ifnet_head, ifp, if_link);
5376 ifp->if_link.tqe_next = NULL;
5377 ifp->if_link.tqe_prev = NULL;
5378 ifindex2ifnet[ifp->if_index] = NULL;
5379
5380 /* Record detach PC stacktrace */
5381 ctrace_record(&((struct dlil_ifnet *)ifp)->dl_if_detach);
5382
5383 /* Clear logging parameters */
5384 bzero(&ifp->if_log, sizeof (ifp->if_log));
5385
5386 /* Clear delegated interface info (reference released below) */
5387 delegated_ifp = ifp->if_delegated.ifp;
5388 bzero(&ifp->if_delegated, sizeof (ifp->if_delegated));
5389
5390 ifnet_lock_done(ifp);
5391 ifnet_head_done();
5392 lck_mtx_unlock(rnh_lock);
5393
5394 /* Release reference held on the delegated interface */
5395 if (delegated_ifp != NULL)
5396 ifnet_release(delegated_ifp);
5397
5398 /* Reset Link Quality Metric (unless loopback [lo0]) */
5399 if (ifp != lo_ifp)
5400 if_lqm_update(ifp, IFNET_LQM_THRESH_OFF);
5401
5402 /* Reset TCP local statistics */
5403 if (ifp->if_tcp_stat != NULL)
5404 bzero(ifp->if_tcp_stat, sizeof(*ifp->if_tcp_stat));
5405
5406 /* Reset UDP local statistics */
5407 if (ifp->if_udp_stat != NULL)
5408 bzero(ifp->if_udp_stat, sizeof(*ifp->if_udp_stat));
5409
5410 /* Let BPF know we're detaching */
5411 bpfdetach(ifp);
5412
5413 /* Mark the interface as DOWN */
5414 if_down(ifp);
5415
5416 /* Drain send queue */
5417 ifclassq_teardown(ifp);
5418
5419 /* Disable forwarding cached route */
5420 lck_mtx_lock(&ifp->if_cached_route_lock);
5421 ifp->if_fwd_cacheok = 0;
5422 lck_mtx_unlock(&ifp->if_cached_route_lock);
5423
5424 ifp->if_data_threshold = 0;
5425 /*
5426 * Drain any deferred IGMPv3/MLDv2 query responses, but keep the
5427 * references to the info structures and leave them attached to
5428 * this ifnet.
5429 */
5430 #if INET
5431 igmp_domifdetach(ifp);
5432 #endif /* INET */
5433 #if INET6
5434 mld_domifdetach(ifp);
5435 #endif /* INET6 */
5436
5437 dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_IF_DETACHING, NULL, 0);
5438
5439 /* Let worker thread take care of the rest, to avoid reentrancy */
5440 dlil_if_lock();
5441 ifnet_detaching_enqueue(ifp);
5442 dlil_if_unlock();
5443
5444 return (0);
5445 }
5446
5447 static void
5448 ifnet_detaching_enqueue(struct ifnet *ifp)
5449 {
5450 dlil_if_lock_assert();
5451
5452 ++ifnet_detaching_cnt;
5453 VERIFY(ifnet_detaching_cnt != 0);
5454 TAILQ_INSERT_TAIL(&ifnet_detaching_head, ifp, if_detaching_link);
5455 wakeup((caddr_t)&ifnet_delayed_run);
5456 }
5457
5458 static struct ifnet *
5459 ifnet_detaching_dequeue(void)
5460 {
5461 struct ifnet *ifp;
5462
5463 dlil_if_lock_assert();
5464
5465 ifp = TAILQ_FIRST(&ifnet_detaching_head);
5466 VERIFY(ifnet_detaching_cnt != 0 || ifp == NULL);
5467 if (ifp != NULL) {
5468 VERIFY(ifnet_detaching_cnt != 0);
5469 --ifnet_detaching_cnt;
5470 TAILQ_REMOVE(&ifnet_detaching_head, ifp, if_detaching_link);
5471 ifp->if_detaching_link.tqe_next = NULL;
5472 ifp->if_detaching_link.tqe_prev = NULL;
5473 }
5474 return (ifp);
5475 }
5476
5477 static int
5478 ifnet_detacher_thread_cont(int err)
5479 {
5480 #pragma unused(err)
5481 struct ifnet *ifp;
5482
5483 for (;;) {
5484 dlil_if_lock_assert();
5485 while (ifnet_detaching_cnt == 0) {
5486 (void) msleep0(&ifnet_delayed_run, &dlil_ifnet_lock,
5487 (PZERO - 1), "ifnet_detacher_cont", 0,
5488 ifnet_detacher_thread_cont);
5489 /* NOTREACHED */
5490 }
5491
5492 VERIFY(TAILQ_FIRST(&ifnet_detaching_head) != NULL);
5493
5494 /* Take care of detaching ifnet */
5495 ifp = ifnet_detaching_dequeue();
5496 if (ifp != NULL) {
5497 dlil_if_unlock();
5498 ifnet_detach_final(ifp);
5499 dlil_if_lock();
5500 }
5501 }
5502 /* NOTREACHED */
5503 return (0);
5504 }
5505
5506 static void
5507 ifnet_detacher_thread_func(void *v, wait_result_t w)
5508 {
5509 #pragma unused(v, w)
5510 dlil_if_lock();
5511 (void) msleep0(&ifnet_delayed_run, &dlil_ifnet_lock,
5512 (PZERO - 1), "ifnet_detacher", 0, ifnet_detacher_thread_cont);
5513 /*
5514 * msleep0() shouldn't have returned as PCATCH was not set;
5515 * therefore assert in this case.
5516 */
5517 dlil_if_unlock();
5518 VERIFY(0);
5519 }
5520
5521 static void
5522 ifnet_detach_final(struct ifnet *ifp)
5523 {
5524 struct ifnet_filter *filter, *filter_next;
5525 struct ifnet_filter_head fhead;
5526 struct dlil_threading_info *inp;
5527 struct ifaddr *ifa;
5528 ifnet_detached_func if_free;
5529 int i;
5530
5531 lck_mtx_lock(&ifp->if_ref_lock);
5532 if (!(ifp->if_refflags & IFRF_DETACHING)) {
5533 panic("%s: flags mismatch (detaching not set) ifp=%p",
5534 __func__, ifp);
5535 /* NOTREACHED */
5536 }
5537
5538 /*
5539 * Wait until the existing IO references get released
5540 * before we proceed with ifnet_detach. This is not a
5541 * common case, so block without using a continuation.
5542 */
5543 while (ifp->if_refio > 0) {
5544 printf("%s: Waiting for IO references on %s interface "
5545 "to be released\n", __func__, if_name(ifp));
5546 (void) msleep(&(ifp->if_refio), &ifp->if_ref_lock,
5547 (PZERO - 1), "ifnet_ioref_wait", NULL);
5548 }
5549 lck_mtx_unlock(&ifp->if_ref_lock);
5550
5551 /* Detach interface filters */
5552 lck_mtx_lock(&ifp->if_flt_lock);
5553 if_flt_monitor_enter(ifp);
5554
5555 lck_mtx_assert(&ifp->if_flt_lock, LCK_MTX_ASSERT_OWNED);
5556 fhead = ifp->if_flt_head;
5557 TAILQ_INIT(&ifp->if_flt_head);
5558
5559 for (filter = TAILQ_FIRST(&fhead); filter; filter = filter_next) {
5560 filter_next = TAILQ_NEXT(filter, filt_next);
5561 lck_mtx_unlock(&ifp->if_flt_lock);
5562
5563 dlil_detach_filter_internal(filter, 1);
5564 lck_mtx_lock(&ifp->if_flt_lock);
5565 }
5566 if_flt_monitor_leave(ifp);
5567 lck_mtx_unlock(&ifp->if_flt_lock);
5568
5569 /* Tell upper layers to drop their network addresses */
5570 if_purgeaddrs(ifp);
5571
5572 ifnet_lock_exclusive(ifp);
5573
5574 /* Uplumb all protocols */
5575 for (i = 0; i < PROTO_HASH_SLOTS; i++) {
5576 struct if_proto *proto;
5577
5578 proto = SLIST_FIRST(&ifp->if_proto_hash[i]);
5579 while (proto != NULL) {
5580 protocol_family_t family = proto->protocol_family;
5581 ifnet_lock_done(ifp);
5582 proto_unplumb(family, ifp);
5583 ifnet_lock_exclusive(ifp);
5584 proto = SLIST_FIRST(&ifp->if_proto_hash[i]);
5585 }
5586 /* There should not be any protocols left */
5587 VERIFY(SLIST_EMPTY(&ifp->if_proto_hash[i]));
5588 }
5589 zfree(dlif_phash_zone, ifp->if_proto_hash);
5590 ifp->if_proto_hash = NULL;
5591
5592 /* Detach (permanent) link address from if_addrhead */
5593 ifa = TAILQ_FIRST(&ifp->if_addrhead);
5594 VERIFY(ifnet_addrs[ifp->if_index - 1] == ifa);
5595 IFA_LOCK(ifa);
5596 if_detach_link_ifa(ifp, ifa);
5597 IFA_UNLOCK(ifa);
5598
5599 /* Remove (permanent) link address from ifnet_addrs[] */
5600 IFA_REMREF(ifa);
5601 ifnet_addrs[ifp->if_index - 1] = NULL;
5602
5603 /* This interface should not be on {ifnet_head,detaching} */
5604 VERIFY(ifp->if_link.tqe_next == NULL);
5605 VERIFY(ifp->if_link.tqe_prev == NULL);
5606 VERIFY(ifp->if_detaching_link.tqe_next == NULL);
5607 VERIFY(ifp->if_detaching_link.tqe_prev == NULL);
5608
5609 /* Prefix list should be empty by now */
5610 VERIFY(TAILQ_EMPTY(&ifp->if_prefixhead));
5611
5612 /* The slot should have been emptied */
5613 VERIFY(ifindex2ifnet[ifp->if_index] == NULL);
5614
5615 /* There should not be any addresses left */
5616 VERIFY(TAILQ_EMPTY(&ifp->if_addrhead));
5617
5618 /*
5619 * Signal the starter thread to terminate itself.
5620 */
5621 if (ifp->if_start_thread != THREAD_NULL) {
5622 lck_mtx_lock_spin(&ifp->if_start_lock);
5623 ifp->if_start_flags = 0;
5624 ifp->if_start_thread = THREAD_NULL;
5625 wakeup_one((caddr_t)&ifp->if_start_thread);
5626 lck_mtx_unlock(&ifp->if_start_lock);
5627 }
5628
5629 /*
5630 * Signal the poller thread to terminate itself.
5631 */
5632 if (ifp->if_poll_thread != THREAD_NULL) {
5633 lck_mtx_lock_spin(&ifp->if_poll_lock);
5634 ifp->if_poll_thread = THREAD_NULL;
5635 wakeup_one((caddr_t)&ifp->if_poll_thread);
5636 lck_mtx_unlock(&ifp->if_poll_lock);
5637 }
5638
5639 /*
5640 * If thread affinity was set for the workloop thread, we will need
5641 * to tear down the affinity and release the extra reference count
5642 * taken at attach time. Does not apply to lo0 or other interfaces
5643 * without dedicated input threads.
5644 */
5645 if ((inp = ifp->if_inp) != NULL) {
5646 VERIFY(inp != dlil_main_input_thread);
5647
5648 if (inp->net_affinity) {
5649 struct thread *tp, *wtp, *ptp;
5650
5651 lck_mtx_lock_spin(&inp->input_lck);
5652 wtp = inp->wloop_thr;
5653 inp->wloop_thr = THREAD_NULL;
5654 ptp = inp->poll_thr;
5655 inp->poll_thr = THREAD_NULL;
5656 tp = inp->input_thr; /* don't nullify now */
5657 inp->tag = 0;
5658 inp->net_affinity = FALSE;
5659 lck_mtx_unlock(&inp->input_lck);
5660
5661 /* Tear down poll thread affinity */
5662 if (ptp != NULL) {
5663 VERIFY(ifp->if_eflags & IFEF_RXPOLL);
5664 (void) dlil_affinity_set(ptp,
5665 THREAD_AFFINITY_TAG_NULL);
5666 thread_deallocate(ptp);
5667 }
5668
5669 /* Tear down workloop thread affinity */
5670 if (wtp != NULL) {
5671 (void) dlil_affinity_set(wtp,
5672 THREAD_AFFINITY_TAG_NULL);
5673 thread_deallocate(wtp);
5674 }
5675
5676 /* Tear down DLIL input thread affinity */
5677 (void) dlil_affinity_set(tp, THREAD_AFFINITY_TAG_NULL);
5678 thread_deallocate(tp);
5679 }
5680
5681 /* disassociate ifp DLIL input thread */
5682 ifp->if_inp = NULL;
5683
5684 lck_mtx_lock_spin(&inp->input_lck);
5685 inp->input_waiting |= DLIL_INPUT_TERMINATE;
5686 if (!(inp->input_waiting & DLIL_INPUT_RUNNING)) {
5687 wakeup_one((caddr_t)&inp->input_waiting);
5688 }
5689 lck_mtx_unlock(&inp->input_lck);
5690 }
5691
5692 /* The driver might unload, so point these to ourselves */
5693 if_free = ifp->if_free;
5694 ifp->if_output = ifp_if_output;
5695 ifp->if_pre_enqueue = ifp_if_output;
5696 ifp->if_start = ifp_if_start;
5697 ifp->if_output_ctl = ifp_if_ctl;
5698 ifp->if_input_poll = ifp_if_input_poll;
5699 ifp->if_input_ctl = ifp_if_ctl;
5700 ifp->if_ioctl = ifp_if_ioctl;
5701 ifp->if_set_bpf_tap = ifp_if_set_bpf_tap;
5702 ifp->if_free = ifp_if_free;
5703 ifp->if_demux = ifp_if_demux;
5704 ifp->if_event = ifp_if_event;
5705 ifp->if_framer_legacy = ifp_if_framer;
5706 ifp->if_framer = ifp_if_framer_extended;
5707 ifp->if_add_proto = ifp_if_add_proto;
5708 ifp->if_del_proto = ifp_if_del_proto;
5709 ifp->if_check_multi = ifp_if_check_multi;
5710
5711 /* wipe out interface description */
5712 VERIFY(ifp->if_desc.ifd_maxlen == IF_DESCSIZE);
5713 ifp->if_desc.ifd_len = 0;
5714 VERIFY(ifp->if_desc.ifd_desc != NULL);
5715 bzero(ifp->if_desc.ifd_desc, IF_DESCSIZE);
5716
5717 /* there shouldn't be any delegation by now */
5718 VERIFY(ifp->if_delegated.ifp == NULL);
5719 VERIFY(ifp->if_delegated.type == 0);
5720 VERIFY(ifp->if_delegated.family == 0);
5721 VERIFY(ifp->if_delegated.subfamily == 0);
5722
5723 ifnet_lock_done(ifp);
5724
5725 #if PF
5726 /*
5727 * Detach this interface from packet filter, if enabled.
5728 */
5729 pf_ifnet_hook(ifp, 0);
5730 #endif /* PF */
5731
5732 /* Filter list should be empty */
5733 lck_mtx_lock_spin(&ifp->if_flt_lock);
5734 VERIFY(TAILQ_EMPTY(&ifp->if_flt_head));
5735 VERIFY(ifp->if_flt_busy == 0);
5736 VERIFY(ifp->if_flt_waiters == 0);
5737 lck_mtx_unlock(&ifp->if_flt_lock);
5738
5739 /* Last chance to drain send queue */
5740 if_qflush(ifp, 0);
5741
5742 /* Last chance to cleanup any cached route */
5743 lck_mtx_lock(&ifp->if_cached_route_lock);
5744 VERIFY(!ifp->if_fwd_cacheok);
5745 ROUTE_RELEASE(&ifp->if_fwd_route);
5746 bzero(&ifp->if_fwd_route, sizeof (ifp->if_fwd_route));
5747 ROUTE_RELEASE(&ifp->if_src_route);
5748 bzero(&ifp->if_src_route, sizeof (ifp->if_src_route));
5749 ROUTE_RELEASE(&ifp->if_src_route6);
5750 bzero(&ifp->if_src_route6, sizeof (ifp->if_src_route6));
5751 lck_mtx_unlock(&ifp->if_cached_route_lock);
5752
5753 VERIFY(ifp->if_data_threshold == 0);
5754
5755 ifnet_llreach_ifdetach(ifp);
5756
5757 dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_IF_DETACHED, NULL, 0);
5758
5759 if (if_free != NULL)
5760 if_free(ifp);
5761
5762 /*
5763 * Finally, mark this ifnet as detached.
5764 */
5765 lck_mtx_lock_spin(&ifp->if_ref_lock);
5766 if (!(ifp->if_refflags & IFRF_DETACHING)) {
5767 panic("%s: flags mismatch (detaching not set) ifp=%p",
5768 __func__, ifp);
5769 /* NOTREACHED */
5770 }
5771 ifp->if_refflags &= ~IFRF_DETACHING;
5772 lck_mtx_unlock(&ifp->if_ref_lock);
5773
5774 if (dlil_verbose)
5775 printf("%s: detached\n", if_name(ifp));
5776
5777 /* Release reference held during ifnet attach */
5778 ifnet_release(ifp);
5779 }
5780
5781 static errno_t
5782 ifp_if_output(struct ifnet *ifp, struct mbuf *m)
5783 {
5784 #pragma unused(ifp)
5785 m_freem(m);
5786 return (0);
5787 }
5788
5789 static void
5790 ifp_if_start(struct ifnet *ifp)
5791 {
5792 ifnet_purge(ifp);
5793 }
5794
5795 static void
5796 ifp_if_input_poll(struct ifnet *ifp, u_int32_t flags, u_int32_t max_cnt,
5797 struct mbuf **m_head, struct mbuf **m_tail, u_int32_t *cnt, u_int32_t *len)
5798 {
5799 #pragma unused(ifp, flags, max_cnt)
5800 if (m_head != NULL)
5801 *m_head = NULL;
5802 if (m_tail != NULL)
5803 *m_tail = NULL;
5804 if (cnt != NULL)
5805 *cnt = 0;
5806 if (len != NULL)
5807 *len = 0;
5808 }
5809
5810 static errno_t
5811 ifp_if_ctl(struct ifnet *ifp, ifnet_ctl_cmd_t cmd, u_int32_t arglen, void *arg)
5812 {
5813 #pragma unused(ifp, cmd, arglen, arg)
5814 return (EOPNOTSUPP);
5815 }
5816
5817 static errno_t
5818 ifp_if_demux(struct ifnet *ifp, struct mbuf *m, char *fh, protocol_family_t *pf)
5819 {
5820 #pragma unused(ifp, fh, pf)
5821 m_freem(m);
5822 return (EJUSTRETURN);
5823 }
5824
5825 static errno_t
5826 ifp_if_add_proto(struct ifnet *ifp, protocol_family_t pf,
5827 const struct ifnet_demux_desc *da, u_int32_t dc)
5828 {
5829 #pragma unused(ifp, pf, da, dc)
5830 return (EINVAL);
5831 }
5832
5833 static errno_t
5834 ifp_if_del_proto(struct ifnet *ifp, protocol_family_t pf)
5835 {
5836 #pragma unused(ifp, pf)
5837 return (EINVAL);
5838 }
5839
5840 static errno_t
5841 ifp_if_check_multi(struct ifnet *ifp, const struct sockaddr *sa)
5842 {
5843 #pragma unused(ifp, sa)
5844 return (EOPNOTSUPP);
5845 }
5846
5847 static errno_t
5848 ifp_if_framer(struct ifnet *ifp, struct mbuf **m,
5849 const struct sockaddr *sa, const char *ll, const char *t)
5850 {
5851 #pragma unused(ifp, m, sa, ll, t)
5852 return (ifp_if_framer_extended(ifp, m, sa, ll, t, NULL, NULL));
5853 }
5854
5855 static errno_t
5856 ifp_if_framer_extended(struct ifnet *ifp, struct mbuf **m,
5857 const struct sockaddr *sa, const char *ll, const char *t,
5858 u_int32_t *pre, u_int32_t *post)
5859 {
5860 #pragma unused(ifp, sa, ll, t)
5861 m_freem(*m);
5862 *m = NULL;
5863
5864 if (pre != NULL)
5865 *pre = 0;
5866 if (post != NULL)
5867 *post = 0;
5868
5869 return (EJUSTRETURN);
5870 }
5871
5872 errno_t
5873 ifp_if_ioctl(struct ifnet *ifp, unsigned long cmd, void *arg)
5874 {
5875 #pragma unused(ifp, cmd, arg)
5876 return (EOPNOTSUPP);
5877 }
5878
5879 static errno_t
5880 ifp_if_set_bpf_tap(struct ifnet *ifp, bpf_tap_mode tm, bpf_packet_func f)
5881 {
5882 #pragma unused(ifp, tm, f)
5883 /* XXX not sure what to do here */
5884 return (0);
5885 }
5886
5887 static void
5888 ifp_if_free(struct ifnet *ifp)
5889 {
5890 #pragma unused(ifp)
5891 }
5892
5893 static void
5894 ifp_if_event(struct ifnet *ifp, const struct kev_msg *e)
5895 {
5896 #pragma unused(ifp, e)
5897 }
5898
5899 __private_extern__
5900 int dlil_if_acquire(u_int32_t family, const void *uniqueid,
5901 size_t uniqueid_len, struct ifnet **ifp)
5902 {
5903 struct ifnet *ifp1 = NULL;
5904 struct dlil_ifnet *dlifp1 = NULL;
5905 void *buf, *base, **pbuf;
5906 int ret = 0;
5907
5908 dlil_if_lock();
5909 TAILQ_FOREACH(dlifp1, &dlil_ifnet_head, dl_if_link) {
5910 ifp1 = (struct ifnet *)dlifp1;
5911
5912 if (ifp1->if_family != family)
5913 continue;
5914
5915 lck_mtx_lock(&dlifp1->dl_if_lock);
5916 /* same uniqueid and same len or no unique id specified */
5917 if ((uniqueid_len == dlifp1->dl_if_uniqueid_len) &&
5918 !bcmp(uniqueid, dlifp1->dl_if_uniqueid, uniqueid_len)) {
5919 /* check for matching interface in use */
5920 if (dlifp1->dl_if_flags & DLIF_INUSE) {
5921 if (uniqueid_len) {
5922 ret = EBUSY;
5923 lck_mtx_unlock(&dlifp1->dl_if_lock);
5924 goto end;
5925 }
5926 } else {
5927 dlifp1->dl_if_flags |= (DLIF_INUSE|DLIF_REUSE);
5928 lck_mtx_unlock(&dlifp1->dl_if_lock);
5929 *ifp = ifp1;
5930 goto end;
5931 }
5932 }
5933 lck_mtx_unlock(&dlifp1->dl_if_lock);
5934 }
5935
5936 /* no interface found, allocate a new one */
5937 buf = zalloc(dlif_zone);
5938 if (buf == NULL) {
5939 ret = ENOMEM;
5940 goto end;
5941 }
5942 bzero(buf, dlif_bufsize);
5943
5944 /* Get the 64-bit aligned base address for this object */
5945 base = (void *)P2ROUNDUP((intptr_t)buf + sizeof (u_int64_t),
5946 sizeof (u_int64_t));
5947 VERIFY(((intptr_t)base + dlif_size) <= ((intptr_t)buf + dlif_bufsize));
5948
5949 /*
5950 * Wind back a pointer size from the aligned base and
5951 * save the original address so we can free it later.
5952 */
5953 pbuf = (void **)((intptr_t)base - sizeof (void *));
5954 *pbuf = buf;
5955 dlifp1 = base;
5956
5957 if (uniqueid_len) {
5958 MALLOC(dlifp1->dl_if_uniqueid, void *, uniqueid_len,
5959 M_NKE, M_WAITOK);
5960 if (dlifp1->dl_if_uniqueid == NULL) {
5961 zfree(dlif_zone, dlifp1);
5962 ret = ENOMEM;
5963 goto end;
5964 }
5965 bcopy(uniqueid, dlifp1->dl_if_uniqueid, uniqueid_len);
5966 dlifp1->dl_if_uniqueid_len = uniqueid_len;
5967 }
5968
5969 ifp1 = (struct ifnet *)dlifp1;
5970 dlifp1->dl_if_flags = DLIF_INUSE;
5971 if (ifnet_debug) {
5972 dlifp1->dl_if_flags |= DLIF_DEBUG;
5973 dlifp1->dl_if_trace = dlil_if_trace;
5974 }
5975 ifp1->if_name = dlifp1->dl_if_namestorage;
5976 ifp1->if_xname = dlifp1->dl_if_xnamestorage;
5977
5978 /* initialize interface description */
5979 ifp1->if_desc.ifd_maxlen = IF_DESCSIZE;
5980 ifp1->if_desc.ifd_len = 0;
5981 ifp1->if_desc.ifd_desc = dlifp1->dl_if_descstorage;
5982
5983 #if CONFIG_MACF_NET
5984 mac_ifnet_label_init(ifp1);
5985 #endif
5986
5987 if ((ret = dlil_alloc_local_stats(ifp1)) != 0) {
5988 DLIL_PRINTF("%s: failed to allocate if local stats, "
5989 "error: %d\n", __func__, ret);
5990 /* This probably shouldn't be fatal */
5991 ret = 0;
5992 }
5993
5994 lck_mtx_init(&dlifp1->dl_if_lock, ifnet_lock_group, ifnet_lock_attr);
5995 lck_rw_init(&ifp1->if_lock, ifnet_lock_group, ifnet_lock_attr);
5996 lck_mtx_init(&ifp1->if_ref_lock, ifnet_lock_group, ifnet_lock_attr);
5997 lck_mtx_init(&ifp1->if_flt_lock, ifnet_lock_group, ifnet_lock_attr);
5998 lck_mtx_init(&ifp1->if_addrconfig_lock, ifnet_lock_group,
5999 ifnet_lock_attr);
6000 lck_rw_init(&ifp1->if_llreach_lock, ifnet_lock_group, ifnet_lock_attr);
6001 #if INET6
6002 lck_rw_init(&ifp1->if_inet6data_lock, ifnet_lock_group, ifnet_lock_attr);
6003 ifp1->if_inet6data = NULL;
6004 #endif
6005
6006 /* for send data paths */
6007 lck_mtx_init(&ifp1->if_start_lock, ifnet_snd_lock_group,
6008 ifnet_lock_attr);
6009 lck_mtx_init(&ifp1->if_cached_route_lock, ifnet_snd_lock_group,
6010 ifnet_lock_attr);
6011 lck_mtx_init(&ifp1->if_snd.ifcq_lock, ifnet_snd_lock_group,
6012 ifnet_lock_attr);
6013
6014 /* for receive data paths */
6015 lck_mtx_init(&ifp1->if_poll_lock, ifnet_rcv_lock_group,
6016 ifnet_lock_attr);
6017
6018 TAILQ_INSERT_TAIL(&dlil_ifnet_head, dlifp1, dl_if_link);
6019
6020 *ifp = ifp1;
6021
6022 end:
6023 dlil_if_unlock();
6024
6025 VERIFY(dlifp1 == NULL || (IS_P2ALIGNED(dlifp1, sizeof (u_int64_t)) &&
6026 IS_P2ALIGNED(&ifp1->if_data, sizeof (u_int64_t))));
6027
6028 return (ret);
6029 }
6030
6031 __private_extern__ void
6032 dlil_if_release(ifnet_t ifp)
6033 {
6034 struct dlil_ifnet *dlifp = (struct dlil_ifnet *)ifp;
6035
6036 ifnet_lock_exclusive(ifp);
6037 lck_mtx_lock(&dlifp->dl_if_lock);
6038 dlifp->dl_if_flags &= ~DLIF_INUSE;
6039 strncpy(dlifp->dl_if_namestorage, ifp->if_name, IFNAMSIZ);
6040 ifp->if_name = dlifp->dl_if_namestorage;
6041 /* Reset external name (name + unit) */
6042 ifp->if_xname = dlifp->dl_if_xnamestorage;
6043 snprintf(__DECONST(char *, ifp->if_xname), IFXNAMSIZ,
6044 "%s?", ifp->if_name);
6045 lck_mtx_unlock(&dlifp->dl_if_lock);
6046 #if CONFIG_MACF_NET
6047 /*
6048 * We can either recycle the MAC label here or in dlil_if_acquire().
6049 * It seems logical to do it here but this means that anything that
6050 * still has a handle on ifp will now see it as unlabeled.
6051 * Since the interface is "dead" that may be OK. Revisit later.
6052 */
6053 mac_ifnet_label_recycle(ifp);
6054 #endif
6055 ifnet_lock_done(ifp);
6056 }
6057
6058 __private_extern__ void
6059 dlil_if_lock(void)
6060 {
6061 lck_mtx_lock(&dlil_ifnet_lock);
6062 }
6063
6064 __private_extern__ void
6065 dlil_if_unlock(void)
6066 {
6067 lck_mtx_unlock(&dlil_ifnet_lock);
6068 }
6069
6070 __private_extern__ void
6071 dlil_if_lock_assert(void)
6072 {
6073 lck_mtx_assert(&dlil_ifnet_lock, LCK_MTX_ASSERT_OWNED);
6074 }
6075
6076 __private_extern__ void
6077 dlil_proto_unplumb_all(struct ifnet *ifp)
6078 {
6079 /*
6080 * if_proto_hash[0-2] are for PF_INET, PF_INET6 and PF_VLAN, where
6081 * each bucket contains exactly one entry; PF_VLAN does not need an
6082 * explicit unplumb.
6083 *
6084 * if_proto_hash[3] is for other protocols; we expect anything
6085 * in this bucket to respond to the DETACHING event (which would
6086 * have happened by now) and do the unplumb then.
6087 */
6088 (void) proto_unplumb(PF_INET, ifp);
6089 #if INET6
6090 (void) proto_unplumb(PF_INET6, ifp);
6091 #endif /* INET6 */
6092 }
6093
6094 static void
6095 ifp_src_route_copyout(struct ifnet *ifp, struct route *dst)
6096 {
6097 lck_mtx_lock_spin(&ifp->if_cached_route_lock);
6098 lck_mtx_convert_spin(&ifp->if_cached_route_lock);
6099
6100 route_copyout(dst, &ifp->if_src_route, sizeof (*dst));
6101
6102 lck_mtx_unlock(&ifp->if_cached_route_lock);
6103 }
6104
6105 static void
6106 ifp_src_route_copyin(struct ifnet *ifp, struct route *src)
6107 {
6108 lck_mtx_lock_spin(&ifp->if_cached_route_lock);
6109 lck_mtx_convert_spin(&ifp->if_cached_route_lock);
6110
6111 if (ifp->if_fwd_cacheok) {
6112 route_copyin(src, &ifp->if_src_route, sizeof (*src));
6113 } else {
6114 ROUTE_RELEASE(src);
6115 }
6116 lck_mtx_unlock(&ifp->if_cached_route_lock);
6117 }
6118
6119 #if INET6
6120 static void
6121 ifp_src_route6_copyout(struct ifnet *ifp, struct route_in6 *dst)
6122 {
6123 lck_mtx_lock_spin(&ifp->if_cached_route_lock);
6124 lck_mtx_convert_spin(&ifp->if_cached_route_lock);
6125
6126 route_copyout((struct route *)dst, (struct route *)&ifp->if_src_route6,
6127 sizeof (*dst));
6128
6129 lck_mtx_unlock(&ifp->if_cached_route_lock);
6130 }
6131
6132 static void
6133 ifp_src_route6_copyin(struct ifnet *ifp, struct route_in6 *src)
6134 {
6135 lck_mtx_lock_spin(&ifp->if_cached_route_lock);
6136 lck_mtx_convert_spin(&ifp->if_cached_route_lock);
6137
6138 if (ifp->if_fwd_cacheok) {
6139 route_copyin((struct route *)src,
6140 (struct route *)&ifp->if_src_route6, sizeof (*src));
6141 } else {
6142 ROUTE_RELEASE(src);
6143 }
6144 lck_mtx_unlock(&ifp->if_cached_route_lock);
6145 }
6146 #endif /* INET6 */
6147
6148 struct rtentry *
6149 ifnet_cached_rtlookup_inet(struct ifnet *ifp, struct in_addr src_ip)
6150 {
6151 struct route src_rt;
6152 struct sockaddr_in *dst;
6153
6154 dst = (struct sockaddr_in *)(void *)(&src_rt.ro_dst);
6155
6156 ifp_src_route_copyout(ifp, &src_rt);
6157
6158 if (ROUTE_UNUSABLE(&src_rt) || src_ip.s_addr != dst->sin_addr.s_addr) {
6159 ROUTE_RELEASE(&src_rt);
6160 if (dst->sin_family != AF_INET) {
6161 bzero(&src_rt.ro_dst, sizeof (src_rt.ro_dst));
6162 dst->sin_len = sizeof (src_rt.ro_dst);
6163 dst->sin_family = AF_INET;
6164 }
6165 dst->sin_addr = src_ip;
6166
6167 if (src_rt.ro_rt == NULL) {
6168 src_rt.ro_rt = rtalloc1_scoped((struct sockaddr *)dst,
6169 0, 0, ifp->if_index);
6170
6171 if (src_rt.ro_rt != NULL) {
6172 /* retain a ref, copyin consumes one */
6173 struct rtentry *rte = src_rt.ro_rt;
6174 RT_ADDREF(rte);
6175 ifp_src_route_copyin(ifp, &src_rt);
6176 src_rt.ro_rt = rte;
6177 }
6178 }
6179 }
6180
6181 return (src_rt.ro_rt);
6182 }
6183
6184 #if INET6
6185 struct rtentry*
6186 ifnet_cached_rtlookup_inet6(struct ifnet *ifp, struct in6_addr *src_ip6)
6187 {
6188 struct route_in6 src_rt;
6189
6190 ifp_src_route6_copyout(ifp, &src_rt);
6191
6192 if (ROUTE_UNUSABLE(&src_rt) ||
6193 !IN6_ARE_ADDR_EQUAL(src_ip6, &src_rt.ro_dst.sin6_addr)) {
6194 ROUTE_RELEASE(&src_rt);
6195 if (src_rt.ro_dst.sin6_family != AF_INET6) {
6196 bzero(&src_rt.ro_dst, sizeof (src_rt.ro_dst));
6197 src_rt.ro_dst.sin6_len = sizeof (src_rt.ro_dst);
6198 src_rt.ro_dst.sin6_family = AF_INET6;
6199 }
6200 src_rt.ro_dst.sin6_scope_id = in6_addr2scopeid(ifp, src_ip6);
6201 bcopy(src_ip6, &src_rt.ro_dst.sin6_addr,
6202 sizeof (src_rt.ro_dst.sin6_addr));
6203
6204 if (src_rt.ro_rt == NULL) {
6205 src_rt.ro_rt = rtalloc1_scoped(
6206 (struct sockaddr *)&src_rt.ro_dst, 0, 0,
6207 ifp->if_index);
6208
6209 if (src_rt.ro_rt != NULL) {
6210 /* retain a ref, copyin consumes one */
6211 struct rtentry *rte = src_rt.ro_rt;
6212 RT_ADDREF(rte);
6213 ifp_src_route6_copyin(ifp, &src_rt);
6214 src_rt.ro_rt = rte;
6215 }
6216 }
6217 }
6218
6219 return (src_rt.ro_rt);
6220 }
6221 #endif /* INET6 */
6222
6223 void
6224 if_lqm_update(struct ifnet *ifp, int lqm)
6225 {
6226 struct kev_dl_link_quality_metric_data ev_lqm_data;
6227
6228 VERIFY(lqm >= IFNET_LQM_MIN && lqm <= IFNET_LQM_MAX);
6229
6230 /* Normalize to edge */
6231 if (lqm > IFNET_LQM_THRESH_UNKNOWN && lqm <= IFNET_LQM_THRESH_POOR)
6232 lqm = IFNET_LQM_THRESH_POOR;
6233 else if (lqm > IFNET_LQM_THRESH_POOR && lqm <= IFNET_LQM_THRESH_GOOD)
6234 lqm = IFNET_LQM_THRESH_GOOD;
6235
6236 ifnet_lock_exclusive(ifp);
6237 if (lqm == ifp->if_lqm) {
6238 ifnet_lock_done(ifp);
6239 return; /* nothing to update */
6240 }
6241 ifp->if_lqm = lqm;
6242 ifnet_lock_done(ifp);
6243
6244 bzero(&ev_lqm_data, sizeof (ev_lqm_data));
6245 ev_lqm_data.link_quality_metric = lqm;
6246
6247 dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_LINK_QUALITY_METRIC_CHANGED,
6248 (struct net_event_data *)&ev_lqm_data, sizeof (ev_lqm_data));
6249 }
6250
6251 /* for uuid.c */
6252 int
6253 uuid_get_ethernet(u_int8_t *node)
6254 {
6255 struct ifnet *ifp;
6256 struct sockaddr_dl *sdl;
6257
6258 ifnet_head_lock_shared();
6259 TAILQ_FOREACH(ifp, &ifnet_head, if_link) {
6260 ifnet_lock_shared(ifp);
6261 IFA_LOCK_SPIN(ifp->if_lladdr);
6262 sdl = (struct sockaddr_dl *)(void *)ifp->if_lladdr->ifa_addr;
6263 if (sdl->sdl_type == IFT_ETHER) {
6264 memcpy(node, LLADDR(sdl), ETHER_ADDR_LEN);
6265 IFA_UNLOCK(ifp->if_lladdr);
6266 ifnet_lock_done(ifp);
6267 ifnet_head_done();
6268 return (0);
6269 }
6270 IFA_UNLOCK(ifp->if_lladdr);
6271 ifnet_lock_done(ifp);
6272 }
6273 ifnet_head_done();
6274
6275 return (-1);
6276 }
6277
6278 static int
6279 sysctl_rxpoll SYSCTL_HANDLER_ARGS
6280 {
6281 #pragma unused(arg1, arg2)
6282 uint32_t i;
6283 int err;
6284
6285 i = if_rxpoll;
6286
6287 err = sysctl_handle_int(oidp, &i, 0, req);
6288 if (err != 0 || req->newptr == USER_ADDR_NULL)
6289 return (err);
6290
6291 if (net_rxpoll == 0)
6292 return (ENXIO);
6293
6294 if_rxpoll = i;
6295 return (err);
6296 }
6297
6298 static int
6299 sysctl_rxpoll_mode_holdtime SYSCTL_HANDLER_ARGS
6300 {
6301 #pragma unused(arg1, arg2)
6302 uint64_t q;
6303 int err;
6304
6305 q = if_rxpoll_mode_holdtime;
6306
6307 err = sysctl_handle_quad(oidp, &q, 0, req);
6308 if (err != 0 || req->newptr == USER_ADDR_NULL)
6309 return (err);
6310
6311 if (q < IF_RXPOLL_MODE_HOLDTIME_MIN)
6312 q = IF_RXPOLL_MODE_HOLDTIME_MIN;
6313
6314 if_rxpoll_mode_holdtime = q;
6315
6316 return (err);
6317 }
6318
6319 static int
6320 sysctl_rxpoll_sample_holdtime SYSCTL_HANDLER_ARGS
6321 {
6322 #pragma unused(arg1, arg2)
6323 uint64_t q;
6324 int err;
6325
6326 q = if_rxpoll_sample_holdtime;
6327
6328 err = sysctl_handle_quad(oidp, &q, 0, req);
6329 if (err != 0 || req->newptr == USER_ADDR_NULL)
6330 return (err);
6331
6332 if (q < IF_RXPOLL_SAMPLETIME_MIN)
6333 q = IF_RXPOLL_SAMPLETIME_MIN;
6334
6335 if_rxpoll_sample_holdtime = q;
6336
6337 return (err);
6338 }
6339
6340 static int
6341 sysctl_rxpoll_interval_time SYSCTL_HANDLER_ARGS
6342 {
6343 #pragma unused(arg1, arg2)
6344 uint64_t q;
6345 int err;
6346
6347 q = if_rxpoll_interval_time;
6348
6349 err = sysctl_handle_quad(oidp, &q, 0, req);
6350 if (err != 0 || req->newptr == USER_ADDR_NULL)
6351 return (err);
6352
6353 if (q < IF_RXPOLL_INTERVALTIME_MIN)
6354 q = IF_RXPOLL_INTERVALTIME_MIN;
6355
6356 if_rxpoll_interval_time = q;
6357
6358 return (err);
6359 }
6360
6361 static int
6362 sysctl_rxpoll_wlowat SYSCTL_HANDLER_ARGS
6363 {
6364 #pragma unused(arg1, arg2)
6365 uint32_t i;
6366 int err;
6367
6368 i = if_rxpoll_wlowat;
6369
6370 err = sysctl_handle_int(oidp, &i, 0, req);
6371 if (err != 0 || req->newptr == USER_ADDR_NULL)
6372 return (err);
6373
6374 if (i == 0 || i >= if_rxpoll_whiwat)
6375 return (EINVAL);
6376
6377 if_rxpoll_wlowat = i;
6378 return (err);
6379 }
6380
6381 static int
6382 sysctl_rxpoll_whiwat SYSCTL_HANDLER_ARGS
6383 {
6384 #pragma unused(arg1, arg2)
6385 uint32_t i;
6386 int err;
6387
6388 i = if_rxpoll_whiwat;
6389
6390 err = sysctl_handle_int(oidp, &i, 0, req);
6391 if (err != 0 || req->newptr == USER_ADDR_NULL)
6392 return (err);
6393
6394 if (i <= if_rxpoll_wlowat)
6395 return (EINVAL);
6396
6397 if_rxpoll_whiwat = i;
6398 return (err);
6399 }
6400
6401 static int
6402 sysctl_sndq_maxlen SYSCTL_HANDLER_ARGS
6403 {
6404 #pragma unused(arg1, arg2)
6405 int i, err;
6406
6407 i = if_sndq_maxlen;
6408
6409 err = sysctl_handle_int(oidp, &i, 0, req);
6410 if (err != 0 || req->newptr == USER_ADDR_NULL)
6411 return (err);
6412
6413 if (i < IF_SNDQ_MINLEN)
6414 i = IF_SNDQ_MINLEN;
6415
6416 if_sndq_maxlen = i;
6417 return (err);
6418 }
6419
6420 static int
6421 sysctl_rcvq_maxlen SYSCTL_HANDLER_ARGS
6422 {
6423 #pragma unused(arg1, arg2)
6424 int i, err;
6425
6426 i = if_rcvq_maxlen;
6427
6428 err = sysctl_handle_int(oidp, &i, 0, req);
6429 if (err != 0 || req->newptr == USER_ADDR_NULL)
6430 return (err);
6431
6432 if (i < IF_RCVQ_MINLEN)
6433 i = IF_RCVQ_MINLEN;
6434
6435 if_rcvq_maxlen = i;
6436 return (err);
6437 }
6438
6439 void
6440 dlil_node_present(struct ifnet *ifp, struct sockaddr *sa,
6441 int32_t rssi, int lqm, int npm, u_int8_t srvinfo[48])
6442 {
6443 struct kev_dl_node_presence kev;
6444 struct sockaddr_dl *sdl;
6445 struct sockaddr_in6 *sin6;
6446
6447 VERIFY(ifp);
6448 VERIFY(sa);
6449 VERIFY(sa->sa_family == AF_LINK || sa->sa_family == AF_INET6);
6450
6451 bzero(&kev, sizeof (kev));
6452 sin6 = &kev.sin6_node_address;
6453 sdl = &kev.sdl_node_address;
6454 nd6_alt_node_addr_decompose(ifp, sa, sdl, sin6);
6455 kev.rssi = rssi;
6456 kev.link_quality_metric = lqm;
6457 kev.node_proximity_metric = npm;
6458 bcopy(srvinfo, kev.node_service_info, sizeof (kev.node_service_info));
6459
6460 nd6_alt_node_present(ifp, sin6, sdl, rssi, lqm, npm);
6461 dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_NODE_PRESENCE,
6462 &kev.link_data, sizeof (kev));
6463 }
6464
6465 void
6466 dlil_node_absent(struct ifnet *ifp, struct sockaddr *sa)
6467 {
6468 struct kev_dl_node_absence kev;
6469 struct sockaddr_in6 *sin6;
6470 struct sockaddr_dl *sdl;
6471
6472 VERIFY(ifp);
6473 VERIFY(sa);
6474 VERIFY(sa->sa_family == AF_LINK || sa->sa_family == AF_INET6);
6475
6476 bzero(&kev, sizeof (kev));
6477 sin6 = &kev.sin6_node_address;
6478 sdl = &kev.sdl_node_address;
6479 nd6_alt_node_addr_decompose(ifp, sa, sdl, sin6);
6480
6481 nd6_alt_node_absent(ifp, sin6);
6482 dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_NODE_ABSENCE,
6483 &kev.link_data, sizeof (kev));
6484 }
6485
6486 const void *
6487 dlil_ifaddr_bytes(const struct sockaddr_dl *sdl, size_t *sizep,
6488 kauth_cred_t *credp)
6489 {
6490 const u_int8_t *bytes;
6491 size_t size;
6492
6493 bytes = CONST_LLADDR(sdl);
6494 size = sdl->sdl_alen;
6495
6496 #if CONFIG_MACF
6497 if (dlil_lladdr_ckreq) {
6498 switch (sdl->sdl_type) {
6499 case IFT_ETHER:
6500 case IFT_BRIDGE:
6501 case IFT_IEEE1394:
6502 case IFT_IEEE8023ADLAG:
6503 case IFT_L2VLAN:
6504 break;
6505 default:
6506 credp = NULL;
6507 break;
6508 };
6509
6510 if (credp && mac_system_check_info(*credp, "net.link.addr")) {
6511 static const u_int8_t unspec[FIREWIRE_EUI64_LEN] = {
6512 [0] = 2
6513 };
6514
6515 switch (sdl->sdl_type) {
6516 case IFT_ETHER:
6517 case IFT_BRIDGE:
6518 case IFT_IEEE8023ADLAG:
6519 case IFT_L2VLAN:
6520 VERIFY(size == ETHER_ADDR_LEN);
6521 bytes = unspec;
6522 break;
6523 case IFT_IEEE1394:
6524 VERIFY(size == FIREWIRE_EUI64_LEN);
6525 bytes = unspec;
6526 break;
6527 default:
6528 VERIFY(FALSE);
6529 break;
6530 };
6531 }
6532 }
6533 #else
6534 #pragma unused(credp)
6535 #endif
6536
6537 if (sizep != NULL) *sizep = size;
6538 return (bytes);
6539 }
6540
6541 void
6542 dlil_report_issues(struct ifnet *ifp, u_int8_t modid[DLIL_MODIDLEN],
6543 u_int8_t info[DLIL_MODARGLEN])
6544 {
6545 struct kev_dl_issues kev;
6546 struct timeval tv;
6547
6548 VERIFY(ifp != NULL);
6549 VERIFY(modid != NULL);
6550 _CASSERT(sizeof (kev.modid) == DLIL_MODIDLEN);
6551 _CASSERT(sizeof (kev.info) == DLIL_MODARGLEN);
6552
6553 bzero(&kev, sizeof (&kev));
6554
6555 microtime(&tv);
6556 kev.timestamp = tv.tv_sec;
6557 bcopy(modid, &kev.modid, DLIL_MODIDLEN);
6558 if (info != NULL)
6559 bcopy(info, &kev.info, DLIL_MODARGLEN);
6560
6561 dlil_post_msg(ifp, KEV_DL_SUBCLASS, KEV_DL_ISSUES,
6562 &kev.link_data, sizeof (kev));
6563 }
6564
6565 errno_t
6566 ifnet_getset_opportunistic(ifnet_t ifp, u_long cmd, struct ifreq *ifr,
6567 struct proc *p)
6568 {
6569 u_int32_t level = IFNET_THROTTLE_OFF;
6570 errno_t result = 0;
6571
6572 VERIFY(cmd == SIOCSIFOPPORTUNISTIC || cmd == SIOCGIFOPPORTUNISTIC);
6573
6574 if (cmd == SIOCSIFOPPORTUNISTIC) {
6575 /*
6576 * XXX: Use priv_check_cred() instead of root check?
6577 */
6578 if ((result = proc_suser(p)) != 0)
6579 return (result);
6580
6581 if (ifr->ifr_opportunistic.ifo_flags ==
6582 IFRIFOF_BLOCK_OPPORTUNISTIC)
6583 level = IFNET_THROTTLE_OPPORTUNISTIC;
6584 else if (ifr->ifr_opportunistic.ifo_flags == 0)
6585 level = IFNET_THROTTLE_OFF;
6586 else
6587 result = EINVAL;
6588
6589 if (result == 0)
6590 result = ifnet_set_throttle(ifp, level);
6591 } else if ((result = ifnet_get_throttle(ifp, &level)) == 0) {
6592 ifr->ifr_opportunistic.ifo_flags = 0;
6593 if (level == IFNET_THROTTLE_OPPORTUNISTIC) {
6594 ifr->ifr_opportunistic.ifo_flags |=
6595 IFRIFOF_BLOCK_OPPORTUNISTIC;
6596 }
6597 }
6598
6599 /*
6600 * Return the count of current opportunistic connections
6601 * over the interface.
6602 */
6603 if (result == 0) {
6604 uint32_t flags = 0;
6605 flags |= (cmd == SIOCSIFOPPORTUNISTIC) ?
6606 INPCB_OPPORTUNISTIC_SETCMD : 0;
6607 flags |= (level == IFNET_THROTTLE_OPPORTUNISTIC) ?
6608 INPCB_OPPORTUNISTIC_THROTTLEON : 0;
6609 ifr->ifr_opportunistic.ifo_inuse =
6610 udp_count_opportunistic(ifp->if_index, flags) +
6611 tcp_count_opportunistic(ifp->if_index, flags);
6612 }
6613
6614 if (result == EALREADY)
6615 result = 0;
6616
6617 return (result);
6618 }
6619
6620 int
6621 ifnet_get_throttle(struct ifnet *ifp, u_int32_t *level)
6622 {
6623 struct ifclassq *ifq;
6624 int err = 0;
6625
6626 if (!(ifp->if_eflags & IFEF_TXSTART))
6627 return (ENXIO);
6628
6629 *level = IFNET_THROTTLE_OFF;
6630
6631 ifq = &ifp->if_snd;
6632 IFCQ_LOCK(ifq);
6633 /* Throttling works only for IFCQ, not ALTQ instances */
6634 if (IFCQ_IS_ENABLED(ifq))
6635 IFCQ_GET_THROTTLE(ifq, *level, err);
6636 IFCQ_UNLOCK(ifq);
6637
6638 return (err);
6639 }
6640
6641 int
6642 ifnet_set_throttle(struct ifnet *ifp, u_int32_t level)
6643 {
6644 struct ifclassq *ifq;
6645 int err = 0;
6646
6647 if (!(ifp->if_eflags & IFEF_TXSTART))
6648 return (ENXIO);
6649
6650 ifq = &ifp->if_snd;
6651
6652 switch (level) {
6653 case IFNET_THROTTLE_OFF:
6654 case IFNET_THROTTLE_OPPORTUNISTIC:
6655 #if PF_ALTQ
6656 /* Throttling works only for IFCQ, not ALTQ instances */
6657 if (ALTQ_IS_ENABLED(IFCQ_ALTQ(ifq)))
6658 return (ENXIO);
6659 #endif /* PF_ALTQ */
6660 break;
6661 default:
6662 return (EINVAL);
6663 }
6664
6665 IFCQ_LOCK(ifq);
6666 if (IFCQ_IS_ENABLED(ifq))
6667 IFCQ_SET_THROTTLE(ifq, level, err);
6668 IFCQ_UNLOCK(ifq);
6669
6670 if (err == 0) {
6671 printf("%s: throttling level set to %d\n", if_name(ifp),
6672 level);
6673 if (level == IFNET_THROTTLE_OFF)
6674 ifnet_start(ifp);
6675 }
6676
6677 return (err);
6678 }
6679
6680 errno_t
6681 ifnet_getset_log(ifnet_t ifp, u_long cmd, struct ifreq *ifr,
6682 struct proc *p)
6683 {
6684 #pragma unused(p)
6685 errno_t result = 0;
6686 uint32_t flags;
6687 int level, category, subcategory;
6688
6689 VERIFY(cmd == SIOCSIFLOG || cmd == SIOCGIFLOG);
6690
6691 if (cmd == SIOCSIFLOG) {
6692 if ((result = priv_check_cred(kauth_cred_get(),
6693 PRIV_NET_INTERFACE_CONTROL, 0)) != 0)
6694 return (result);
6695
6696 level = ifr->ifr_log.ifl_level;
6697 if (level < IFNET_LOG_MIN || level > IFNET_LOG_MAX)
6698 result = EINVAL;
6699
6700 flags = ifr->ifr_log.ifl_flags;
6701 if ((flags &= IFNET_LOGF_MASK) == 0)
6702 result = EINVAL;
6703
6704 category = ifr->ifr_log.ifl_category;
6705 subcategory = ifr->ifr_log.ifl_subcategory;
6706
6707 if (result == 0)
6708 result = ifnet_set_log(ifp, level, flags,
6709 category, subcategory);
6710 } else {
6711 result = ifnet_get_log(ifp, &level, &flags, &category,
6712 &subcategory);
6713 if (result == 0) {
6714 ifr->ifr_log.ifl_level = level;
6715 ifr->ifr_log.ifl_flags = flags;
6716 ifr->ifr_log.ifl_category = category;
6717 ifr->ifr_log.ifl_subcategory = subcategory;
6718 }
6719 }
6720
6721 return (result);
6722 }
6723
6724 int
6725 ifnet_set_log(struct ifnet *ifp, int32_t level, uint32_t flags,
6726 int32_t category, int32_t subcategory)
6727 {
6728 int err = 0;
6729
6730 VERIFY(level >= IFNET_LOG_MIN && level <= IFNET_LOG_MAX);
6731 VERIFY(flags & IFNET_LOGF_MASK);
6732
6733 /*
6734 * The logging level applies to all facilities; make sure to
6735 * update them all with the most current level.
6736 */
6737 flags |= ifp->if_log.flags;
6738
6739 if (ifp->if_output_ctl != NULL) {
6740 struct ifnet_log_params l;
6741
6742 bzero(&l, sizeof (l));
6743 l.level = level;
6744 l.flags = flags;
6745 l.flags &= ~IFNET_LOGF_DLIL;
6746 l.category = category;
6747 l.subcategory = subcategory;
6748
6749 /* Send this request to lower layers */
6750 if (l.flags != 0) {
6751 err = ifp->if_output_ctl(ifp, IFNET_CTL_SET_LOG,
6752 sizeof (l), &l);
6753 }
6754 } else if ((flags & ~IFNET_LOGF_DLIL) && ifp->if_output_ctl == NULL) {
6755 /*
6756 * If targeted to the lower layers without an output
6757 * control callback registered on the interface, just
6758 * silently ignore facilities other than ours.
6759 */
6760 flags &= IFNET_LOGF_DLIL;
6761 if (flags == 0 && (!ifp->if_log.flags & IFNET_LOGF_DLIL))
6762 level = 0;
6763 }
6764
6765 if (err == 0) {
6766 if ((ifp->if_log.level = level) == IFNET_LOG_DEFAULT)
6767 ifp->if_log.flags = 0;
6768 else
6769 ifp->if_log.flags |= flags;
6770
6771 log(LOG_INFO, "%s: logging level set to %d flags=%b "
6772 "arg=%b, category=%d subcategory=%d\n", if_name(ifp),
6773 ifp->if_log.level, ifp->if_log.flags,
6774 IFNET_LOGF_BITS, flags, IFNET_LOGF_BITS,
6775 category, subcategory);
6776 }
6777
6778 return (err);
6779 }
6780
6781 int
6782 ifnet_get_log(struct ifnet *ifp, int32_t *level, uint32_t *flags,
6783 int32_t *category, int32_t *subcategory)
6784 {
6785 if (level != NULL)
6786 *level = ifp->if_log.level;
6787 if (flags != NULL)
6788 *flags = ifp->if_log.flags;
6789 if (category != NULL)
6790 *category = ifp->if_log.category;
6791 if (subcategory != NULL)
6792 *subcategory = ifp->if_log.subcategory;
6793
6794 return (0);
6795 }
6796
6797 int
6798 ifnet_notify_address(struct ifnet *ifp, int af)
6799 {
6800 struct ifnet_notify_address_params na;
6801
6802 #if PF
6803 (void) pf_ifaddr_hook(ifp);
6804 #endif /* PF */
6805
6806 if (ifp->if_output_ctl == NULL)
6807 return (EOPNOTSUPP);
6808
6809 bzero(&na, sizeof (na));
6810 na.address_family = af;
6811
6812 return (ifp->if_output_ctl(ifp, IFNET_CTL_NOTIFY_ADDRESS,
6813 sizeof (na), &na));
6814 }
6815
6816 errno_t
6817 ifnet_flowid(struct ifnet *ifp, uint32_t *flowid)
6818 {
6819 if (ifp == NULL || flowid == NULL) {
6820 return (EINVAL);
6821 } else if (!(ifp->if_eflags & IFEF_TXSTART) ||
6822 !(ifp->if_refflags & IFRF_ATTACHED)) {
6823 return (ENXIO);
6824 }
6825
6826 *flowid = ifp->if_flowhash;
6827
6828 return (0);
6829 }
6830
6831 errno_t
6832 ifnet_disable_output(struct ifnet *ifp)
6833 {
6834 int err;
6835
6836 if (ifp == NULL) {
6837 return (EINVAL);
6838 } else if (!(ifp->if_eflags & IFEF_TXSTART) ||
6839 !(ifp->if_refflags & IFRF_ATTACHED)) {
6840 return (ENXIO);
6841 }
6842
6843 if ((err = ifnet_fc_add(ifp)) == 0) {
6844 lck_mtx_lock_spin(&ifp->if_start_lock);
6845 ifp->if_start_flags |= IFSF_FLOW_CONTROLLED;
6846 lck_mtx_unlock(&ifp->if_start_lock);
6847 }
6848 return (err);
6849 }
6850
6851 errno_t
6852 ifnet_enable_output(struct ifnet *ifp)
6853 {
6854 if (ifp == NULL) {
6855 return (EINVAL);
6856 } else if (!(ifp->if_eflags & IFEF_TXSTART) ||
6857 !(ifp->if_refflags & IFRF_ATTACHED)) {
6858 return (ENXIO);
6859 }
6860
6861 ifnet_start_common(ifp, 1);
6862 return (0);
6863 }
6864
6865 void
6866 ifnet_flowadv(uint32_t flowhash)
6867 {
6868 struct ifnet_fc_entry *ifce;
6869 struct ifnet *ifp;
6870
6871 ifce = ifnet_fc_get(flowhash);
6872 if (ifce == NULL)
6873 return;
6874
6875 VERIFY(ifce->ifce_ifp != NULL);
6876 ifp = ifce->ifce_ifp;
6877
6878 /* flow hash gets recalculated per attach, so check */
6879 if (ifnet_is_attached(ifp, 1)) {
6880 if (ifp->if_flowhash == flowhash)
6881 (void) ifnet_enable_output(ifp);
6882 ifnet_decr_iorefcnt(ifp);
6883 }
6884 ifnet_fc_entry_free(ifce);
6885 }
6886
6887 /*
6888 * Function to compare ifnet_fc_entries in ifnet flow control tree
6889 */
6890 static inline int
6891 ifce_cmp(const struct ifnet_fc_entry *fc1, const struct ifnet_fc_entry *fc2)
6892 {
6893 return (fc1->ifce_flowhash - fc2->ifce_flowhash);
6894 }
6895
6896 static int
6897 ifnet_fc_add(struct ifnet *ifp)
6898 {
6899 struct ifnet_fc_entry keyfc, *ifce;
6900 uint32_t flowhash;
6901
6902 VERIFY(ifp != NULL && (ifp->if_eflags & IFEF_TXSTART));
6903 VERIFY(ifp->if_flowhash != 0);
6904 flowhash = ifp->if_flowhash;
6905
6906 bzero(&keyfc, sizeof (keyfc));
6907 keyfc.ifce_flowhash = flowhash;
6908
6909 lck_mtx_lock_spin(&ifnet_fc_lock);
6910 ifce = RB_FIND(ifnet_fc_tree, &ifnet_fc_tree, &keyfc);
6911 if (ifce != NULL && ifce->ifce_ifp == ifp) {
6912 /* Entry is already in ifnet_fc_tree, return */
6913 lck_mtx_unlock(&ifnet_fc_lock);
6914 return (0);
6915 }
6916
6917 if (ifce != NULL) {
6918 /*
6919 * There is a different fc entry with the same flow hash
6920 * but different ifp pointer. There can be a collision
6921 * on flow hash but the probability is low. Let's just
6922 * avoid adding a second one when there is a collision.
6923 */
6924 lck_mtx_unlock(&ifnet_fc_lock);
6925 return (EAGAIN);
6926 }
6927
6928 /* become regular mutex */
6929 lck_mtx_convert_spin(&ifnet_fc_lock);
6930
6931 ifce = zalloc_noblock(ifnet_fc_zone);
6932 if (ifce == NULL) {
6933 /* memory allocation failed */
6934 lck_mtx_unlock(&ifnet_fc_lock);
6935 return (ENOMEM);
6936 }
6937 bzero(ifce, ifnet_fc_zone_size);
6938
6939 ifce->ifce_flowhash = flowhash;
6940 ifce->ifce_ifp = ifp;
6941
6942 RB_INSERT(ifnet_fc_tree, &ifnet_fc_tree, ifce);
6943 lck_mtx_unlock(&ifnet_fc_lock);
6944 return (0);
6945 }
6946
6947 static struct ifnet_fc_entry *
6948 ifnet_fc_get(uint32_t flowhash)
6949 {
6950 struct ifnet_fc_entry keyfc, *ifce;
6951 struct ifnet *ifp;
6952
6953 bzero(&keyfc, sizeof (keyfc));
6954 keyfc.ifce_flowhash = flowhash;
6955
6956 lck_mtx_lock_spin(&ifnet_fc_lock);
6957 ifce = RB_FIND(ifnet_fc_tree, &ifnet_fc_tree, &keyfc);
6958 if (ifce == NULL) {
6959 /* Entry is not present in ifnet_fc_tree, return */
6960 lck_mtx_unlock(&ifnet_fc_lock);
6961 return (NULL);
6962 }
6963
6964 RB_REMOVE(ifnet_fc_tree, &ifnet_fc_tree, ifce);
6965
6966 VERIFY(ifce->ifce_ifp != NULL);
6967 ifp = ifce->ifce_ifp;
6968
6969 /* become regular mutex */
6970 lck_mtx_convert_spin(&ifnet_fc_lock);
6971
6972 if (!ifnet_is_attached(ifp, 0)) {
6973 /*
6974 * This ifp is not attached or in the process of being
6975 * detached; just don't process it.
6976 */
6977 ifnet_fc_entry_free(ifce);
6978 ifce = NULL;
6979 }
6980 lck_mtx_unlock(&ifnet_fc_lock);
6981
6982 return (ifce);
6983 }
6984
6985 static void
6986 ifnet_fc_entry_free(struct ifnet_fc_entry *ifce)
6987 {
6988 zfree(ifnet_fc_zone, ifce);
6989 }
6990
6991 static uint32_t
6992 ifnet_calc_flowhash(struct ifnet *ifp)
6993 {
6994 struct ifnet_flowhash_key fh __attribute__((aligned(8)));
6995 uint32_t flowhash = 0;
6996
6997 if (ifnet_flowhash_seed == 0)
6998 ifnet_flowhash_seed = RandomULong();
6999
7000 bzero(&fh, sizeof (fh));
7001
7002 (void) snprintf(fh.ifk_name, sizeof (fh.ifk_name), "%s", ifp->if_name);
7003 fh.ifk_unit = ifp->if_unit;
7004 fh.ifk_flags = ifp->if_flags;
7005 fh.ifk_eflags = ifp->if_eflags;
7006 fh.ifk_capabilities = ifp->if_capabilities;
7007 fh.ifk_capenable = ifp->if_capenable;
7008 fh.ifk_output_sched_model = ifp->if_output_sched_model;
7009 fh.ifk_rand1 = RandomULong();
7010 fh.ifk_rand2 = RandomULong();
7011
7012 try_again:
7013 flowhash = net_flowhash(&fh, sizeof (fh), ifnet_flowhash_seed);
7014 if (flowhash == 0) {
7015 /* try to get a non-zero flowhash */
7016 ifnet_flowhash_seed = RandomULong();
7017 goto try_again;
7018 }
7019
7020 return (flowhash);
7021 }
7022
7023 static void
7024 dlil_output_cksum_dbg(struct ifnet *ifp, struct mbuf *m, uint32_t hoff,
7025 protocol_family_t pf)
7026 {
7027 #pragma unused(ifp)
7028 uint32_t did_sw;
7029
7030 if (!(hwcksum_dbg_mode & HWCKSUM_DBG_FINALIZE_FORCED) ||
7031 (m->m_pkthdr.csum_flags & (CSUM_TSO_IPV4|CSUM_TSO_IPV6)))
7032 return;
7033
7034 switch (pf) {
7035 case PF_INET:
7036 did_sw = in_finalize_cksum(m, hoff, m->m_pkthdr.csum_flags);
7037 if (did_sw & CSUM_DELAY_IP)
7038 hwcksum_dbg_finalized_hdr++;
7039 if (did_sw & CSUM_DELAY_DATA)
7040 hwcksum_dbg_finalized_data++;
7041 break;
7042 #if INET6
7043 case PF_INET6:
7044 /*
7045 * Checksum offload should not have been enabled when
7046 * extension headers exist; that also means that we
7047 * cannot force-finalize packets with extension headers.
7048 * Indicate to the callee should it skip such case by
7049 * setting optlen to -1.
7050 */
7051 did_sw = in6_finalize_cksum(m, hoff, -1, -1,
7052 m->m_pkthdr.csum_flags);
7053 if (did_sw & CSUM_DELAY_IPV6_DATA)
7054 hwcksum_dbg_finalized_data++;
7055 break;
7056 #endif /* INET6 */
7057 default:
7058 return;
7059 }
7060 }
7061
7062 static void
7063 dlil_input_cksum_dbg(struct ifnet *ifp, struct mbuf *m, char *frame_header,
7064 protocol_family_t pf)
7065 {
7066 uint16_t sum;
7067 uint32_t hlen;
7068
7069 if (frame_header == NULL ||
7070 frame_header < (char *)mbuf_datastart(m) ||
7071 frame_header > (char *)m->m_data) {
7072 printf("%s: frame header pointer 0x%llx out of range "
7073 "[0x%llx,0x%llx] for mbuf 0x%llx\n", if_name(ifp),
7074 (uint64_t)VM_KERNEL_ADDRPERM(frame_header),
7075 (uint64_t)VM_KERNEL_ADDRPERM(mbuf_datastart(m)),
7076 (uint64_t)VM_KERNEL_ADDRPERM(m->m_data),
7077 (uint64_t)VM_KERNEL_ADDRPERM(m));
7078 return;
7079 }
7080 hlen = (m->m_data - frame_header);
7081
7082 switch (pf) {
7083 case PF_INET:
7084 #if INET6
7085 case PF_INET6:
7086 #endif /* INET6 */
7087 break;
7088 default:
7089 return;
7090 }
7091
7092 /*
7093 * Force partial checksum offload; useful to simulate cases
7094 * where the hardware does not support partial checksum offload,
7095 * in order to validate correctness throughout the layers above.
7096 */
7097 if (hwcksum_dbg_mode & HWCKSUM_DBG_PARTIAL_FORCED) {
7098 uint32_t foff = hwcksum_dbg_partial_rxoff_forced;
7099
7100 if (foff > (uint32_t)m->m_pkthdr.len)
7101 return;
7102
7103 m->m_pkthdr.csum_flags &= ~CSUM_RX_FLAGS;
7104
7105 /* Compute 16-bit 1's complement sum from forced offset */
7106 sum = m_sum16(m, foff, (m->m_pkthdr.len - foff));
7107
7108 m->m_pkthdr.csum_flags |= (CSUM_DATA_VALID | CSUM_PARTIAL);
7109 m->m_pkthdr.csum_rx_val = sum;
7110 m->m_pkthdr.csum_rx_start = (foff + hlen);
7111
7112 hwcksum_dbg_partial_forced++;
7113 hwcksum_dbg_partial_forced_bytes += m->m_pkthdr.len;
7114 }
7115
7116 /*
7117 * Partial checksum offload verification (and adjustment);
7118 * useful to validate and test cases where the hardware
7119 * supports partial checksum offload.
7120 */
7121 if ((m->m_pkthdr.csum_flags &
7122 (CSUM_DATA_VALID | CSUM_PARTIAL | CSUM_PSEUDO_HDR)) ==
7123 (CSUM_DATA_VALID | CSUM_PARTIAL)) {
7124 uint32_t rxoff;
7125
7126 /* Start offset must begin after frame header */
7127 rxoff = m->m_pkthdr.csum_rx_start;
7128 if (hlen > rxoff) {
7129 hwcksum_dbg_bad_rxoff++;
7130 if (dlil_verbose) {
7131 printf("%s: partial cksum start offset %d "
7132 "is less than frame header length %d for "
7133 "mbuf 0x%llx\n", if_name(ifp), rxoff, hlen,
7134 (uint64_t)VM_KERNEL_ADDRPERM(m));
7135 }
7136 return;
7137 }
7138 rxoff -=hlen;
7139
7140 if (!(hwcksum_dbg_mode & HWCKSUM_DBG_PARTIAL_FORCED)) {
7141 /*
7142 * Compute the expected 16-bit 1's complement sum;
7143 * skip this if we've already computed it above
7144 * when partial checksum offload is forced.
7145 */
7146 sum = m_sum16(m, rxoff, (m->m_pkthdr.len - rxoff));
7147
7148 /* Hardware or driver is buggy */
7149 if (sum != m->m_pkthdr.csum_rx_val) {
7150 hwcksum_dbg_bad_cksum++;
7151 if (dlil_verbose) {
7152 printf("%s: bad partial cksum value "
7153 "0x%x (expected 0x%x) for mbuf "
7154 "0x%llx [rx_start %d]\n",
7155 if_name(ifp),
7156 m->m_pkthdr.csum_rx_val, sum,
7157 (uint64_t)VM_KERNEL_ADDRPERM(m),
7158 m->m_pkthdr.csum_rx_start);
7159 }
7160 return;
7161 }
7162 }
7163 hwcksum_dbg_verified++;
7164
7165 /*
7166 * This code allows us to emulate various hardwares that
7167 * perform 16-bit 1's complement sum beginning at various
7168 * start offset values.
7169 */
7170 if (hwcksum_dbg_mode & HWCKSUM_DBG_PARTIAL_RXOFF_ADJ) {
7171 uint32_t aoff = hwcksum_dbg_partial_rxoff_adj;
7172
7173 if (aoff == rxoff || aoff > (uint32_t)m->m_pkthdr.len)
7174 return;
7175
7176 sum = m_adj_sum16(m, rxoff, aoff, sum);
7177
7178 m->m_pkthdr.csum_rx_val = sum;
7179 m->m_pkthdr.csum_rx_start = (aoff + hlen);
7180
7181 hwcksum_dbg_adjusted++;
7182 }
7183 }
7184 }
7185
7186 static int
7187 sysctl_hwcksum_dbg_mode SYSCTL_HANDLER_ARGS
7188 {
7189 #pragma unused(arg1, arg2)
7190 u_int32_t i;
7191 int err;
7192
7193 i = hwcksum_dbg_mode;
7194
7195 err = sysctl_handle_int(oidp, &i, 0, req);
7196 if (err != 0 || req->newptr == USER_ADDR_NULL)
7197 return (err);
7198
7199 if (hwcksum_dbg == 0)
7200 return (ENODEV);
7201
7202 if ((i & ~HWCKSUM_DBG_MASK) != 0)
7203 return (EINVAL);
7204
7205 hwcksum_dbg_mode = (i & HWCKSUM_DBG_MASK);
7206
7207 return (err);
7208 }
7209
7210 static int
7211 sysctl_hwcksum_dbg_partial_rxoff_forced SYSCTL_HANDLER_ARGS
7212 {
7213 #pragma unused(arg1, arg2)
7214 u_int32_t i;
7215 int err;
7216
7217 i = hwcksum_dbg_partial_rxoff_forced;
7218
7219 err = sysctl_handle_int(oidp, &i, 0, req);
7220 if (err != 0 || req->newptr == USER_ADDR_NULL)
7221 return (err);
7222
7223 if (!(hwcksum_dbg_mode & HWCKSUM_DBG_PARTIAL_FORCED))
7224 return (ENODEV);
7225
7226 hwcksum_dbg_partial_rxoff_forced = i;
7227
7228 return (err);
7229 }
7230
7231 static int
7232 sysctl_hwcksum_dbg_partial_rxoff_adj SYSCTL_HANDLER_ARGS
7233 {
7234 #pragma unused(arg1, arg2)
7235 u_int32_t i;
7236 int err;
7237
7238 i = hwcksum_dbg_partial_rxoff_adj;
7239
7240 err = sysctl_handle_int(oidp, &i, 0, req);
7241 if (err != 0 || req->newptr == USER_ADDR_NULL)
7242 return (err);
7243
7244 if (!(hwcksum_dbg_mode & HWCKSUM_DBG_PARTIAL_RXOFF_ADJ))
7245 return (ENODEV);
7246
7247 hwcksum_dbg_partial_rxoff_adj = i;
7248
7249 return (err);
7250 }
7251
7252 #if DEBUG
7253 /* Blob for sum16 verification */
7254 static uint8_t sumdata[] = {
7255 0x1f, 0x8b, 0x08, 0x08, 0x4c, 0xe5, 0x9a, 0x4f, 0x00, 0x03,
7256 0x5f, 0x00, 0x5d, 0x91, 0x41, 0x4e, 0xc4, 0x30, 0x0c, 0x45,
7257 0xf7, 0x9c, 0xc2, 0x07, 0x18, 0xf5, 0x0e, 0xb0, 0xe2, 0x00,
7258 0x48, 0x88, 0xa5, 0xdb, 0xba, 0x49, 0x34, 0x69, 0xdc, 0x71,
7259 0x92, 0xa9, 0xc2, 0x8a, 0x6b, 0x70, 0x3d, 0x4e, 0x82, 0x93,
7260 0xb4, 0x08, 0xd8, 0xc5, 0xb1, 0xfd, 0xff, 0xb3, 0xfd, 0x4c,
7261 0x42, 0x5f, 0x1f, 0x9f, 0x11, 0x12, 0x43, 0xb2, 0x04, 0x93,
7262 0xe0, 0x7b, 0x01, 0x0e, 0x14, 0x07, 0x78, 0xd1, 0x78, 0x75,
7263 0x71, 0x71, 0xe9, 0x08, 0x84, 0x46, 0xf2, 0xc7, 0x3b, 0x09,
7264 0xe7, 0xd1, 0xd3, 0x8a, 0x57, 0x92, 0x33, 0xcd, 0x39, 0xcc,
7265 0xb0, 0x91, 0x89, 0xe0, 0x42, 0x53, 0x8b, 0xb7, 0x8c, 0x42,
7266 0x60, 0xd9, 0x9f, 0x7a, 0x55, 0x19, 0x76, 0xcb, 0x10, 0x49,
7267 0x35, 0xac, 0x0b, 0x5a, 0x3c, 0xbb, 0x65, 0x51, 0x8c, 0x90,
7268 0x7c, 0x69, 0x45, 0x45, 0x81, 0xb4, 0x2b, 0x70, 0x82, 0x85,
7269 0x55, 0x91, 0x17, 0x90, 0xdc, 0x14, 0x1e, 0x35, 0x52, 0xdd,
7270 0x02, 0x16, 0xef, 0xb5, 0x40, 0x89, 0xe2, 0x46, 0x53, 0xad,
7271 0x93, 0x6e, 0x98, 0x30, 0xe5, 0x08, 0xb7, 0xcc, 0x03, 0xbc,
7272 0x71, 0x86, 0x09, 0x43, 0x0d, 0x52, 0xf5, 0xa2, 0xf5, 0xa2,
7273 0x56, 0x11, 0x8d, 0xa8, 0xf5, 0xee, 0x92, 0x3d, 0xfe, 0x8c,
7274 0x67, 0x71, 0x8b, 0x0e, 0x2d, 0x70, 0x77, 0xbe, 0xbe, 0xea,
7275 0xbf, 0x9a, 0x8d, 0x9c, 0x53, 0x53, 0xe5, 0xe0, 0x4b, 0x87,
7276 0x85, 0xd2, 0x45, 0x95, 0x30, 0xc1, 0xcc, 0xe0, 0x74, 0x54,
7277 0x13, 0x58, 0xe8, 0xe8, 0x79, 0xa2, 0x09, 0x73, 0xa4, 0x0e,
7278 0x39, 0x59, 0x0c, 0xe6, 0x9c, 0xb2, 0x4f, 0x06, 0x5b, 0x8e,
7279 0xcd, 0x17, 0x6c, 0x5e, 0x95, 0x4d, 0x70, 0xa2, 0x0a, 0xbf,
7280 0xa3, 0xcc, 0x03, 0xbc, 0x5a, 0xe7, 0x75, 0x06, 0x5e, 0x75,
7281 0xef, 0x58, 0x8e, 0x15, 0xd1, 0x0a, 0x18, 0xff, 0xdd, 0xe6,
7282 0x02, 0x3b, 0xb5, 0xb4, 0xa1, 0xe0, 0x72, 0xfc, 0xe3, 0xab,
7283 0x07, 0xe0, 0x4d, 0x65, 0xea, 0x92, 0xeb, 0xf2, 0x7b, 0x17,
7284 0x05, 0xce, 0xc6, 0xf6, 0x2b, 0xbb, 0x70, 0x3d, 0x00, 0x95,
7285 0xe0, 0x07, 0x52, 0x3b, 0x58, 0xfc, 0x7c, 0x69, 0x4d, 0xe9,
7286 0xf7, 0xa9, 0x66, 0x1e, 0x1e, 0xbe, 0x01, 0x69, 0x98, 0xfe,
7287 0xc8, 0x28, 0x02, 0x00, 0x00
7288 };
7289
7290 /* Precomputed 16-bit 1's complement sums for various spans of the above data */
7291 static struct {
7292 int len;
7293 uint16_t sum;
7294 } sumtbl[] = {
7295 { 11, 0xcb6d },
7296 { 20, 0x20dd },
7297 { 27, 0xbabd },
7298 { 32, 0xf3e8 },
7299 { 37, 0x197d },
7300 { 43, 0x9eae },
7301 { 64, 0x4678 },
7302 { 127, 0x9399 },
7303 { 256, 0xd147 },
7304 { 325, 0x0358 }
7305 };
7306 #define SUMTBL_MAX ((int)sizeof (sumtbl) / (int)sizeof (sumtbl[0]))
7307
7308 static void
7309 dlil_verify_sum16(void)
7310 {
7311 struct mbuf *m;
7312 uint8_t *buf;
7313 int n;
7314
7315 /* Make sure test data plus extra room for alignment fits in cluster */
7316 _CASSERT((sizeof (sumdata) + (sizeof (uint64_t) * 2)) <= MCLBYTES);
7317
7318 m = m_getcl(M_WAITOK, MT_DATA, M_PKTHDR);
7319 MH_ALIGN(m, sizeof (uint32_t)); /* 32-bit starting alignment */
7320 buf = mtod(m, uint8_t *); /* base address */
7321
7322 for (n = 0; n < SUMTBL_MAX; n++) {
7323 uint16_t len = sumtbl[n].len;
7324 int i;
7325
7326 /* Verify for all possible alignments */
7327 for (i = 0; i < (int)sizeof (uint64_t); i++) {
7328 uint16_t sum;
7329 uint8_t *c;
7330
7331 /* Copy over test data to mbuf */
7332 VERIFY(len <= sizeof (sumdata));
7333 c = buf + i;
7334 bcopy(sumdata, c, len);
7335
7336 /* Zero-offset test (align by data pointer) */
7337 m->m_data = (caddr_t)c;
7338 m->m_len = len;
7339 sum = m_sum16(m, 0, len);
7340
7341 /* Something is horribly broken; stop now */
7342 if (sum != sumtbl[n].sum) {
7343 panic("%s: broken m_sum16 for len=%d align=%d "
7344 "sum=0x%04x [expected=0x%04x]\n", __func__,
7345 len, i, sum, sumtbl[n].sum);
7346 /* NOTREACHED */
7347 }
7348
7349 /* Alignment test by offset (fixed data pointer) */
7350 m->m_data = (caddr_t)buf;
7351 m->m_len = i + len;
7352 sum = m_sum16(m, i, len);
7353
7354 /* Something is horribly broken; stop now */
7355 if (sum != sumtbl[n].sum) {
7356 panic("%s: broken m_sum16 for len=%d offset=%d "
7357 "sum=0x%04x [expected=0x%04x]\n", __func__,
7358 len, i, sum, sumtbl[n].sum);
7359 /* NOTREACHED */
7360 }
7361 #if INET
7362 /* Simple sum16 contiguous buffer test by aligment */
7363 sum = b_sum16(c, len);
7364
7365 /* Something is horribly broken; stop now */
7366 if (sum != sumtbl[n].sum) {
7367 panic("%s: broken b_sum16 for len=%d align=%d "
7368 "sum=0x%04x [expected=0x%04x]\n", __func__,
7369 len, i, sum, sumtbl[n].sum);
7370 /* NOTREACHED */
7371 }
7372 #endif /* INET */
7373 }
7374 }
7375 m_freem(m);
7376
7377 printf("DLIL: SUM16 self-tests PASSED\n");
7378 }
7379 #endif /* DEBUG */
7380
7381 #define CASE_STRINGIFY(x) case x: return #x
7382
7383 __private_extern__ const char *
7384 dlil_kev_dl_code_str(u_int32_t event_code)
7385 {
7386 switch (event_code) {
7387 CASE_STRINGIFY(KEV_DL_SIFFLAGS);
7388 CASE_STRINGIFY(KEV_DL_SIFMETRICS);
7389 CASE_STRINGIFY(KEV_DL_SIFMTU);
7390 CASE_STRINGIFY(KEV_DL_SIFPHYS);
7391 CASE_STRINGIFY(KEV_DL_SIFMEDIA);
7392 CASE_STRINGIFY(KEV_DL_SIFGENERIC);
7393 CASE_STRINGIFY(KEV_DL_ADDMULTI);
7394 CASE_STRINGIFY(KEV_DL_DELMULTI);
7395 CASE_STRINGIFY(KEV_DL_IF_ATTACHED);
7396 CASE_STRINGIFY(KEV_DL_IF_DETACHING);
7397 CASE_STRINGIFY(KEV_DL_IF_DETACHED);
7398 CASE_STRINGIFY(KEV_DL_LINK_OFF);
7399 CASE_STRINGIFY(KEV_DL_LINK_ON);
7400 CASE_STRINGIFY(KEV_DL_PROTO_ATTACHED);
7401 CASE_STRINGIFY(KEV_DL_PROTO_DETACHED);
7402 CASE_STRINGIFY(KEV_DL_LINK_ADDRESS_CHANGED);
7403 CASE_STRINGIFY(KEV_DL_WAKEFLAGS_CHANGED);
7404 CASE_STRINGIFY(KEV_DL_IF_IDLE_ROUTE_REFCNT);
7405 CASE_STRINGIFY(KEV_DL_IFCAP_CHANGED);
7406 CASE_STRINGIFY(KEV_DL_LINK_QUALITY_METRIC_CHANGED);
7407 CASE_STRINGIFY(KEV_DL_NODE_PRESENCE);
7408 CASE_STRINGIFY(KEV_DL_NODE_ABSENCE);
7409 CASE_STRINGIFY(KEV_DL_MASTER_ELECTED);
7410 CASE_STRINGIFY(KEV_DL_ISSUES);
7411 CASE_STRINGIFY(KEV_DL_IFDELEGATE_CHANGED);
7412 default:
7413 break;
7414 }
7415 return ("");
7416 }
mirror of XNU