/*
- * Copyright (c) 2003-2012 Apple Inc. All rights reserved.
+ * Copyright (c) 2003-2017 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
#include <sys/protosw.h>
#include <sys/domain.h>
#include <sys/mbuf.h>
+#include <sys/mcache.h>
#include <sys/fcntl.h>
#include <sys/filio.h>
#include <sys/uio_internal.h>
-#include <kern/lock.h>
+#include <kern/locks.h>
+#include <net/net_api_stats.h>
#include <netinet/in.h>
#include <libkern/OSAtomic.h>
+#include <stdbool.h>
static errno_t sock_send_internal(socket_t, const struct msghdr *,
mbuf_t, int, size_t *);
-static void sock_setupcalls_common(socket_t, sock_upcall, void *,
- sock_upcall, void *);
+
+#undef sock_accept
+#undef sock_socket
+errno_t sock_accept(socket_t so, struct sockaddr *from, int fromlen,
+ int flags, sock_upcall callback, void *cookie, socket_t *new_so);
+errno_t sock_socket(int domain, int type, int protocol, sock_upcall callback,
+ void *context, socket_t *new_so);
+
+static errno_t sock_accept_common(socket_t sock, struct sockaddr *from,
+ int fromlen, int flags, sock_upcall callback, void *cookie,
+ socket_t *new_sock, bool is_internal);
+static errno_t sock_socket_common(int domain, int type, int protocol,
+ sock_upcall callback, void *context, socket_t *new_so, bool is_internal);
errno_t
-sock_accept(socket_t sock, struct sockaddr *from, int fromlen, int flags,
- sock_upcall callback, void *cookie, socket_t *new_sock)
+sock_accept_common(socket_t sock, struct sockaddr *from, int fromlen, int flags,
+ sock_upcall callback, void *cookie, socket_t *new_sock, bool is_internal)
{
struct sockaddr *sa;
struct socket *new_so;
socket_unlock(sock, 1);
return (ENOTSUP);
}
+check_again:
if (((flags & MSG_DONTWAIT) != 0 || (sock->so_state & SS_NBIO) != 0) &&
sock->so_comp.tqh_first == NULL) {
socket_unlock(sock, 1);
}
if (sock->so_proto->pr_getlock != NULL) {
- mutex_held = (*sock->so_proto->pr_getlock)(sock, 0);
+ mutex_held = (*sock->so_proto->pr_getlock)(sock, PR_F_WILLUNLOCK);
dosocklock = 1;
} else {
mutex_held = sock->so_proto->pr_domain->dom_mtx;
return (error);
}
+ so_acquire_accept_list(sock, NULL);
+ if (TAILQ_EMPTY(&sock->so_comp)) {
+ so_release_accept_list(sock);
+ goto check_again;
+ }
new_so = TAILQ_FIRST(&sock->so_comp);
TAILQ_REMOVE(&sock->so_comp, new_so, so_list);
+ new_so->so_state &= ~SS_COMP;
+ new_so->so_head = NULL;
sock->so_qlen--;
+ so_release_accept_list(sock);
+
+ /*
+ * Count the accepted socket as an in-kernel socket
+ */
+ new_so->so_flags1 |= SOF1_IN_KERNEL_SOCKET;
+ INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_in_kernel_total);
+ if (is_internal) {
+ INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_in_kernel_os_total);
+ }
+
/*
* Pass the pre-accepted socket to any interested socket filter(s).
* Upon failure, the socket would have been closed by the callee.
* again once we're done with the filter(s).
*/
socket_unlock(sock, 0);
- if ((error = soacceptfilter(new_so)) != 0) {
+ if ((error = soacceptfilter(new_so, sock)) != 0) {
/* Drop reference on listening socket */
sodereference(sock);
return (error);
}
if (dosocklock) {
- lck_mtx_assert(new_so->so_proto->pr_getlock(new_so, 0),
+ LCK_MTX_ASSERT(new_so->so_proto->pr_getlock(new_so, 0),
LCK_MTX_ASSERT_NOTOWNED);
socket_lock(new_so, 1);
}
- new_so->so_state &= ~SS_COMP;
- new_so->so_head = NULL;
(void) soacceptlock(new_so, &sa, 0);
socket_unlock(sock, 1); /* release the head */
/* see comments in sock_setupcall() */
if (callback != NULL) {
- sock_setupcalls_common(new_so, callback, cookie, NULL, NULL);
+#if CONFIG_EMBEDDED
+ sock_setupcalls_locked(new_so, callback, cookie, callback, cookie, 0);
+#else
+ sock_setupcalls_locked(new_so, callback, cookie, NULL, NULL, 0);
+#endif /* !CONFIG_EMBEDDED */
}
if (sa != NULL && from != NULL) {
return (error);
}
+errno_t
+sock_accept(socket_t sock, struct sockaddr *from, int fromlen, int flags,
+ sock_upcall callback, void *cookie, socket_t *new_sock)
+{
+ return (sock_accept_common(sock, from, fromlen, flags,
+ callback, cookie, new_sock, false));
+}
+
+errno_t
+sock_accept_internal(socket_t sock, struct sockaddr *from, int fromlen, int flags,
+ sock_upcall callback, void *cookie, socket_t *new_sock)
+{
+ return (sock_accept_common(sock, from, fromlen, flags,
+ callback, cookie, new_sock, true));
+}
+
errno_t
sock_bind(socket_t sock, const struct sockaddr *to)
{
}
if (sock->so_proto->pr_getlock != NULL)
- mutex_held = (*sock->so_proto->pr_getlock)(sock, 0);
+ mutex_held = (*sock->so_proto->pr_getlock)(sock, PR_F_WILLUNLOCK);
else
mutex_held = sock->so_proto->pr_domain->dom_mtx;
}
if (sock->so_proto->pr_getlock != NULL)
- mutex_held = (*sock->so_proto->pr_getlock)(sock, 0);
+ mutex_held = (*sock->so_proto->pr_getlock)(sock, PR_F_WILLUNLOCK);
else
mutex_held = sock->so_proto->pr_domain->dom_mtx;
else if (dscp >= 0x20 && dscp <= 0x2f)
tc = SO_TC_VI;
else if (dscp >= 0x08 && dscp <= 0x17)
- tc = SO_TC_BK;
+ tc = SO_TC_BK_SYS;
else
tc = SO_TC_BE;
return (soshutdown(sock, how));
}
-
errno_t
-sock_socket(int domain, int type, int protocol, sock_upcall callback,
- void *context, socket_t *new_so)
+sock_socket_common(int domain, int type, int protocol, sock_upcall callback,
+ void *context, socket_t *new_so, bool is_internal)
{
int error = 0;
/* socreate will create an initial so_count */
error = socreate(domain, new_so, type, protocol);
if (error == 0) {
+ /*
+ * This is an in-kernel socket
+ */
+ (*new_so)->so_flags1 |= SOF1_IN_KERNEL_SOCKET;
+ INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_in_kernel_total);
+ if (is_internal) {
+ INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_in_kernel_os_total);
+ }
+
/* see comments in sock_setupcall() */
if (callback != NULL) {
- sock_setupcalls_common(*new_so, callback, context,
- NULL, NULL);
+ sock_setupcall(*new_so, callback, context);
}
/*
* last_pid and last_upid should be zero for sockets
return (error);
}
+errno_t
+sock_socket_internal(int domain, int type, int protocol, sock_upcall callback,
+ void *context, socket_t *new_so)
+{
+ return (sock_socket_common(domain, type, protocol, callback,
+ context, new_so, true));
+}
+
+errno_t
+sock_socket(int domain, int type, int protocol, sock_upcall callback,
+ void *context, socket_t *new_so)
+{
+ return (sock_socket_common(domain, type, protocol, callback,
+ context, new_so, false));
+}
+
void
sock_close(socket_t sock)
{
__func__, sock->so_retaincnt, sock);
/* NOTREACHED */
}
- if ((sock->so_retaincnt == 0) && (sock->so_usecount == 2)) {
+ /*
+ * Check SS_NOFDREF in case a close happened as sock_retain()
+ * was grabbing the lock
+ */
+ if ((sock->so_retaincnt == 0) && (sock->so_usecount == 2) &&
+ (!(sock->so_state & SS_NOFDREF) ||
+ (sock->so_flags & SOF_MP_SUBFLOW))) {
/* close socket only if the FD is not holding it */
soclose_locked(sock);
} else {
/* remove extra reference holding the socket */
+ VERIFY(sock->so_usecount > 1);
sock->so_usecount--;
}
socket_unlock(sock, 1);
* Caller must have ensured socket is valid and won't be going away.
*/
void
-socket_set_traffic_mgt_flags_locked(socket_t sock, u_int32_t flags)
+socket_set_traffic_mgt_flags_locked(socket_t sock, u_int8_t flags)
{
- (void) OSBitOrAtomic(flags, &sock->so_traffic_mgt_flags);
+ u_int32_t soflags1 = 0;
+
+ if ((flags & TRAFFIC_MGT_SO_BACKGROUND))
+ soflags1 |= SOF1_TRAFFIC_MGT_SO_BACKGROUND;
+ if ((flags & TRAFFIC_MGT_TCP_RECVBG))
+ soflags1 |= SOF1_TRAFFIC_MGT_TCP_RECVBG;
+
+ (void) OSBitOrAtomic(soflags1, &sock->so_flags1);
+
sock_set_tcp_stream_priority(sock);
}
void
-socket_set_traffic_mgt_flags(socket_t sock, u_int32_t flags)
+socket_set_traffic_mgt_flags(socket_t sock, u_int8_t flags)
{
socket_lock(sock, 1);
socket_set_traffic_mgt_flags_locked(sock, flags);
* Caller must have ensured socket is valid and won't be going away.
*/
void
-socket_clear_traffic_mgt_flags_locked(socket_t sock, u_int32_t flags)
+socket_clear_traffic_mgt_flags_locked(socket_t sock, u_int8_t flags)
{
- (void) OSBitAndAtomic(~flags, &sock->so_traffic_mgt_flags);
+ u_int32_t soflags1 = 0;
+
+ if ((flags & TRAFFIC_MGT_SO_BACKGROUND))
+ soflags1 |= SOF1_TRAFFIC_MGT_SO_BACKGROUND;
+ if ((flags & TRAFFIC_MGT_TCP_RECVBG))
+ soflags1 |= SOF1_TRAFFIC_MGT_TCP_RECVBG;
+
+ (void) OSBitAndAtomic(~soflags1, &sock->so_flags1);
+
sock_set_tcp_stream_priority(sock);
}
void
-socket_clear_traffic_mgt_flags(socket_t sock, u_int32_t flags)
+socket_clear_traffic_mgt_flags(socket_t sock, u_int8_t flags)
{
socket_lock(sock, 1);
socket_clear_traffic_mgt_flags_locked(sock, flags);
return (retval);
}
-static void
-sock_setupcalls_common(socket_t sock, sock_upcall rcallback, void *rcontext,
- sock_upcall wcallback, void *wcontext)
+void
+sock_setupcalls_locked(socket_t sock, sock_upcall rcallback, void *rcontext,
+ sock_upcall wcallback, void *wcontext, int locked)
{
if (rcallback != NULL) {
sock->so_rcv.sb_flags |= SB_UPCALL;
+ if (locked)
+ sock->so_rcv.sb_flags |= SB_UPCALL_LOCK;
sock->so_rcv.sb_upcall = rcallback;
sock->so_rcv.sb_upcallarg = rcontext;
} else {
- sock->so_rcv.sb_flags &= ~SB_UPCALL;
+ sock->so_rcv.sb_flags &= ~(SB_UPCALL | SB_UPCALL_LOCK);
sock->so_rcv.sb_upcall = NULL;
sock->so_rcv.sb_upcallarg = NULL;
}
if (wcallback != NULL) {
sock->so_snd.sb_flags |= SB_UPCALL;
+ if (locked)
+ sock->so_snd.sb_flags |= SB_UPCALL_LOCK;
sock->so_snd.sb_upcall = wcallback;
sock->so_snd.sb_upcallarg = wcontext;
} else {
- sock->so_snd.sb_flags &= ~SB_UPCALL;
+ sock->so_snd.sb_flags &= ~(SB_UPCALL | SB_UPCALL_LOCK);
sock->so_snd.sb_upcall = NULL;
sock->so_snd.sb_upcallarg = NULL;
}
* the read and write callbacks and their respective parameters.
*/
socket_lock(sock, 1);
- sock_setupcalls_common(sock, callback, context, NULL, NULL);
+#if CONFIG_EMBEDDED
+ sock_setupcalls_locked(sock, callback, context, callback, context, 0);
+#else
+ sock_setupcalls_locked(sock, callback, context, NULL, NULL, 0);
+#endif /* !CONFIG_EMBEDDED */
socket_unlock(sock, 1);
return (0);
* Note that we don't wait for any in progress upcall to complete.
*/
socket_lock(sock, 1);
- sock_setupcalls_common(sock, rcallback, rcontext, wcallback, wcontext);
+ sock_setupcalls_locked(sock, rcallback, rcontext, wcallback, wcontext, 0);
socket_unlock(sock, 1);
return (0);
}
-errno_t
-sock_catchevents(socket_t sock, sock_evupcall ecallback, void *econtext,
+void
+sock_catchevents_locked(socket_t sock, sock_evupcall ecallback, void *econtext,
u_int32_t emask)
{
- if (sock == NULL)
- return (EINVAL);
+ socket_lock_assert_owned(sock);
/*
* Note that we don't wait for any in progress upcall to complete.
*/
- socket_lock(sock, 1);
if (ecallback != NULL) {
sock->so_event = ecallback;
sock->so_eventarg = econtext;
sock->so_eventmask = emask;
} else {
- sock->so_event = NULL;
+ sock->so_event = sonullevent;
sock->so_eventarg = NULL;
sock->so_eventmask = 0;
}
+}
+
+errno_t
+sock_catchevents(socket_t sock, sock_evupcall ecallback, void *econtext,
+ u_int32_t emask)
+{
+ if (sock == NULL)
+ return (EINVAL);
+
+ socket_lock(sock, 1);
+ sock_catchevents_locked(sock, ecallback, econtext, emask);
socket_unlock(sock, 1);
return (0);
}
+
+/*
+ * Returns true whether or not a socket belongs to the kernel.
+ */
+int
+sock_iskernel(socket_t so)
+{
+ return (so && so->last_pid == 0);
+}
projects / apple / xnu.git / blobdiff