xnu-4570.41.2.tar.gz

[apple/xnu.git] / bsd / kern / kpi_socket.c

diff --git a/bsd/kern/kpi_socket.c b/bsd/kern/kpi_socket.c
index 1db81b353be1396878f06244619dd3d3c5db398e..a7b17264d5ec4ee8b28f9cb086549de36074b3d6 100644 (file)
--- a/bsd/kern/kpi_socket.c
+++ b/bsd/kern/kpi_socket.c
@@ -1,5 +1,5 @@
 /*
 /*

- * Copyright (c) 2003-2012 Apple Inc. All rights reserved.
+ * Copyright (c) 2003-2017 Apple Inc. All rights reserved.

  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *
  *
  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
  *


@@ -39,21 +39,35 @@
 #include <sys/protosw.h>
 #include <sys/domain.h>
 #include <sys/mbuf.h>
 #include <sys/protosw.h>
 #include <sys/domain.h>
 #include <sys/mbuf.h>

+#include <sys/mcache.h>

 #include <sys/fcntl.h>
 #include <sys/filio.h>
 #include <sys/uio_internal.h>
 #include <sys/fcntl.h>
 #include <sys/filio.h>
 #include <sys/uio_internal.h>

-#include <kern/lock.h>
+#include <kern/locks.h>
+#include <net/net_api_stats.h>

 #include <netinet/in.h>
 #include <libkern/OSAtomic.h>
 #include <netinet/in.h>
 #include <libkern/OSAtomic.h>

+#include <stdbool.h>

 
 static errno_t sock_send_internal(socket_t, const struct msghdr        *,
     mbuf_t, int, size_t        *);
 
 static errno_t sock_send_internal(socket_t, const struct msghdr        *,
     mbuf_t, int, size_t        *);

-static void sock_setupcalls_common(socket_t, sock_upcall, void *,
-    sock_upcall, void *);
+
+#undef sock_accept
+#undef sock_socket
+errno_t sock_accept(socket_t so, struct sockaddr *from, int fromlen,
+    int flags, sock_upcall callback, void *cookie, socket_t *new_so);
+errno_t sock_socket(int domain, int type, int protocol, sock_upcall callback,
+    void *context, socket_t *new_so);
+
+static errno_t sock_accept_common(socket_t sock, struct sockaddr *from,
+    int fromlen, int flags, sock_upcall callback, void *cookie,
+    socket_t *new_sock, bool is_internal);
+static errno_t sock_socket_common(int domain, int type, int protocol,
+    sock_upcall callback, void *context, socket_t *new_so, bool is_internal);

 
 errno_t
 
 errno_t

-sock_accept(socket_t sock, struct sockaddr *from, int fromlen, int flags,
-    sock_upcall callback, void *cookie, socket_t *new_sock)
+sock_accept_common(socket_t sock, struct sockaddr *from, int fromlen, int flags,
+    sock_upcall callback, void *cookie, socket_t *new_sock, bool is_internal)

 {
        struct sockaddr *sa;
        struct socket *new_so;
 {
        struct sockaddr *sa;
        struct socket *new_so;


@@ -73,6 +87,7 @@ sock_accept(socket_t sock, struct sockaddr *from, int fromlen, int flags,
                socket_unlock(sock, 1);
                return (ENOTSUP);
        }
                socket_unlock(sock, 1);
                return (ENOTSUP);
        }

+check_again:

        if (((flags & MSG_DONTWAIT) != 0 || (sock->so_state & SS_NBIO) != 0) &&
            sock->so_comp.tqh_first == NULL) {
                socket_unlock(sock, 1);
        if (((flags & MSG_DONTWAIT) != 0 || (sock->so_state & SS_NBIO) != 0) &&
            sock->so_comp.tqh_first == NULL) {
                socket_unlock(sock, 1);


@@ -80,7 +95,7 @@ sock_accept(socket_t sock, struct sockaddr *from, int fromlen, int flags,
        }
 
        if (sock->so_proto->pr_getlock != NULL)  {
        }
 
        if (sock->so_proto->pr_getlock != NULL)  {

-               mutex_held = (*sock->so_proto->pr_getlock)(sock, 0);
+               mutex_held = (*sock->so_proto->pr_getlock)(sock, PR_F_WILLUNLOCK);

                dosocklock = 1;
        } else {
                mutex_held = sock->so_proto->pr_domain->dom_mtx;
                dosocklock = 1;
        } else {
                mutex_held = sock->so_proto->pr_domain->dom_mtx;


@@ -106,10 +121,28 @@ sock_accept(socket_t sock, struct sockaddr *from, int fromlen, int flags,
                return (error);
        }
 
                return (error);
        }
 

+       so_acquire_accept_list(sock, NULL);
+       if (TAILQ_EMPTY(&sock->so_comp)) {
+               so_release_accept_list(sock);
+               goto check_again;
+       }

        new_so = TAILQ_FIRST(&sock->so_comp);
        TAILQ_REMOVE(&sock->so_comp, new_so, so_list);
        new_so = TAILQ_FIRST(&sock->so_comp);
        TAILQ_REMOVE(&sock->so_comp, new_so, so_list);

+       new_so->so_state &= ~SS_COMP;
+       new_so->so_head = NULL;

        sock->so_qlen--;
 
        sock->so_qlen--;
 

+       so_release_accept_list(sock);
+
+       /*
+        * Count the accepted socket as an in-kernel socket
+        */
+       new_so->so_flags1 |= SOF1_IN_KERNEL_SOCKET;
+       INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_in_kernel_total);
+       if (is_internal) {
+               INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_in_kernel_os_total);
+       }
+

        /*
         * Pass the pre-accepted socket to any interested socket filter(s).
         * Upon failure, the socket would have been closed by the callee.
        /*
         * Pass the pre-accepted socket to any interested socket filter(s).
         * Upon failure, the socket would have been closed by the callee.


@@ -122,7 +155,7 @@ sock_accept(socket_t sock, struct sockaddr *from, int fromlen, int flags,
                 * again once we're done with the filter(s).
                 */
                socket_unlock(sock, 0);
                 * again once we're done with the filter(s).
                 */
                socket_unlock(sock, 0);

-               if ((error = soacceptfilter(new_so)) != 0) {
+               if ((error = soacceptfilter(new_so, sock)) != 0) {

                        /* Drop reference on listening socket */
                        sodereference(sock);
                        return (error);
                        /* Drop reference on listening socket */
                        sodereference(sock);
                        return (error);


@@ -131,20 +164,22 @@ sock_accept(socket_t sock, struct sockaddr *from, int fromlen, int flags,
        }
 
        if (dosocklock) {
        }
 
        if (dosocklock) {

-               lck_mtx_assert(new_so->so_proto->pr_getlock(new_so, 0),
+               LCK_MTX_ASSERT(new_so->so_proto->pr_getlock(new_so, 0),

                    LCK_MTX_ASSERT_NOTOWNED);
                socket_lock(new_so, 1);
        }
 
                    LCK_MTX_ASSERT_NOTOWNED);
                socket_lock(new_so, 1);
        }
 

-       new_so->so_state &= ~SS_COMP;
-       new_so->so_head = NULL;

        (void) soacceptlock(new_so, &sa, 0);
 
        socket_unlock(sock, 1); /* release the head */
 
        /* see comments in sock_setupcall() */
        if (callback != NULL) {
        (void) soacceptlock(new_so, &sa, 0);
 
        socket_unlock(sock, 1); /* release the head */
 
        /* see comments in sock_setupcall() */
        if (callback != NULL) {

-               sock_setupcalls_common(new_so, callback, cookie, NULL, NULL);
+#if CONFIG_EMBEDDED
+               sock_setupcalls_locked(new_so, callback, cookie, callback, cookie, 0);
+#else
+               sock_setupcalls_locked(new_so, callback, cookie, NULL, NULL, 0);
+#endif /* !CONFIG_EMBEDDED */

        }
 
        if (sa != NULL && from != NULL) {
        }
 
        if (sa != NULL && from != NULL) {


@@ -169,6 +204,22 @@ sock_accept(socket_t sock, struct sockaddr *from, int fromlen, int flags,
        return (error);
 }
 
        return (error);
 }
 

+errno_t
+sock_accept(socket_t sock, struct sockaddr *from, int fromlen, int flags,
+    sock_upcall callback, void *cookie, socket_t *new_sock)
+{
+       return (sock_accept_common(sock, from, fromlen, flags,
+           callback, cookie, new_sock, false));
+}
+
+errno_t
+sock_accept_internal(socket_t sock, struct sockaddr *from, int fromlen, int flags,
+    sock_upcall callback, void *cookie, socket_t *new_sock)
+{
+       return (sock_accept_common(sock, from, fromlen, flags,
+           callback, cookie, new_sock, true));
+}
+

 errno_t
 sock_bind(socket_t sock, const struct sockaddr *to)
 {
 errno_t
 sock_bind(socket_t sock, const struct sockaddr *to)
 {


@@ -238,7 +289,7 @@ sock_connect(socket_t sock, const struct sockaddr *to, int flags)
                }
 
                if (sock->so_proto->pr_getlock != NULL)
                }
 
                if (sock->so_proto->pr_getlock != NULL)

-                       mutex_held = (*sock->so_proto->pr_getlock)(sock, 0);
+                       mutex_held = (*sock->so_proto->pr_getlock)(sock, PR_F_WILLUNLOCK);

                else
                        mutex_held = sock->so_proto->pr_domain->dom_mtx;
 
                else
                        mutex_held = sock->so_proto->pr_domain->dom_mtx;
 


@@ -304,7 +355,7 @@ sock_connectwait(socket_t sock, const struct timeval *tv)
        }
 
        if (sock->so_proto->pr_getlock != NULL)
        }
 
        if (sock->so_proto->pr_getlock != NULL)

-               mutex_held = (*sock->so_proto->pr_getlock)(sock, 0);
+               mutex_held = (*sock->so_proto->pr_getlock)(sock, PR_F_WILLUNLOCK);

        else
                mutex_held = sock->so_proto->pr_domain->dom_mtx;
 
        else
                mutex_held = sock->so_proto->pr_domain->dom_mtx;
 


@@ -495,7 +546,7 @@ so_tc_from_dscp(u_int8_t dscp)
        else if (dscp >= 0x20 && dscp <= 0x2f)
                tc = SO_TC_VI;
        else if (dscp >= 0x08 && dscp <= 0x17)
        else if (dscp >= 0x20 && dscp <= 0x2f)
                tc = SO_TC_VI;
        else if (dscp >= 0x08 && dscp <= 0x17)

-               tc = SO_TC_BK;
+               tc = SO_TC_BK_SYS;

        else
                tc = SO_TC_BE;
 
        else
                tc = SO_TC_BE;
 


@@ -883,10 +934,9 @@ sock_shutdown(socket_t sock, int how)
        return (soshutdown(sock, how));
 }
 
        return (soshutdown(sock, how));
 }
 

-

 errno_t
 errno_t

-sock_socket(int        domain, int type, int protocol, sock_upcall callback,
-    void *context, socket_t *new_so)
+sock_socket_common(int domain, int type, int protocol, sock_upcall callback,
+    void *context, socket_t *new_so, bool is_internal)

 {
        int error = 0;
 
 {
        int error = 0;
 


@@ -896,10 +946,18 @@ sock_socket(int   domain, int type, int protocol, sock_upcall callback,
        /* socreate will create an initial so_count */
        error = socreate(domain, new_so, type, protocol);
        if (error == 0) {
        /* socreate will create an initial so_count */
        error = socreate(domain, new_so, type, protocol);
        if (error == 0) {

+               /*
+                * This is an in-kernel socket
+                */
+               (*new_so)->so_flags1 |= SOF1_IN_KERNEL_SOCKET;
+               INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_in_kernel_total);
+               if (is_internal) {
+                       INC_ATOMIC_INT64_LIM(net_api_stats.nas_socket_in_kernel_os_total);
+               }
+

                /* see comments in sock_setupcall() */
                if (callback != NULL) {
                /* see comments in sock_setupcall() */
                if (callback != NULL) {

-                       sock_setupcalls_common(*new_so, callback, context,
-                           NULL, NULL);
+                       sock_setupcall(*new_so, callback, context);

                }
                /* 
                 * last_pid and last_upid should be zero for sockets
                }
                /* 
                 * last_pid and last_upid should be zero for sockets


@@ -911,6 +969,22 @@ sock_socket(int    domain, int type, int protocol, sock_upcall callback,
        return (error);
 }
 
        return (error);
 }
 

+errno_t
+sock_socket_internal(int domain, int type, int protocol, sock_upcall callback,
+    void *context, socket_t *new_so)
+{
+       return (sock_socket_common(domain, type, protocol, callback,
+           context, new_so, true));
+}
+
+errno_t
+sock_socket(int domain, int type, int protocol, sock_upcall callback,
+    void *context, socket_t *new_so)
+{
+       return (sock_socket_common(domain, type, protocol, callback,
+           context, new_so, false));
+}
+

 void
 sock_close(socket_t sock)
 {
 void
 sock_close(socket_t sock)
 {


@@ -950,11 +1024,18 @@ sock_release(socket_t sock)
                    __func__, sock->so_retaincnt, sock);
                /* NOTREACHED */
        }
                    __func__, sock->so_retaincnt, sock);
                /* NOTREACHED */
        }

-       if ((sock->so_retaincnt == 0) && (sock->so_usecount == 2)) {
+       /*
+        * Check SS_NOFDREF in case a close happened as sock_retain()
+        * was grabbing the lock
+        */
+       if ((sock->so_retaincnt == 0) && (sock->so_usecount == 2) &&
+           (!(sock->so_state & SS_NOFDREF) ||
+           (sock->so_flags & SOF_MP_SUBFLOW))) {

                /* close socket only if the FD is not holding it */
                soclose_locked(sock);
        } else {
                /* remove extra reference holding the socket */
                /* close socket only if the FD is not holding it */
                soclose_locked(sock);
        } else {
                /* remove extra reference holding the socket */

+               VERIFY(sock->so_usecount > 1);

                sock->so_usecount--;
        }
        socket_unlock(sock, 1);
                sock->so_usecount--;
        }
        socket_unlock(sock, 1);


@@ -1038,14 +1119,22 @@ sock_set_tcp_stream_priority(socket_t sock)
  * Caller must have ensured socket is valid and won't be going away.
  */
 void
  * Caller must have ensured socket is valid and won't be going away.
  */
 void

-socket_set_traffic_mgt_flags_locked(socket_t sock, u_int32_t flags)
+socket_set_traffic_mgt_flags_locked(socket_t sock, u_int8_t flags)

 {
 {

-       (void) OSBitOrAtomic(flags, &sock->so_traffic_mgt_flags);
+       u_int32_t soflags1 = 0;
+       
+       if ((flags & TRAFFIC_MGT_SO_BACKGROUND))
+               soflags1 |= SOF1_TRAFFIC_MGT_SO_BACKGROUND;
+       if ((flags & TRAFFIC_MGT_TCP_RECVBG))
+               soflags1 |= SOF1_TRAFFIC_MGT_TCP_RECVBG;
+       
+       (void) OSBitOrAtomic(soflags1, &sock->so_flags1);
+

        sock_set_tcp_stream_priority(sock);
 }
 
 void
        sock_set_tcp_stream_priority(sock);
 }
 
 void

-socket_set_traffic_mgt_flags(socket_t sock, u_int32_t flags)
+socket_set_traffic_mgt_flags(socket_t sock, u_int8_t flags)

 {
        socket_lock(sock, 1);
        socket_set_traffic_mgt_flags_locked(sock, flags);
 {
        socket_lock(sock, 1);
        socket_set_traffic_mgt_flags_locked(sock, flags);


@@ -1056,14 +1145,22 @@ socket_set_traffic_mgt_flags(socket_t sock, u_int32_t flags)
  * Caller must have ensured socket is valid and won't be going away.
  */
 void
  * Caller must have ensured socket is valid and won't be going away.
  */
 void

-socket_clear_traffic_mgt_flags_locked(socket_t sock, u_int32_t flags)
+socket_clear_traffic_mgt_flags_locked(socket_t sock, u_int8_t flags)

 {
 {

-       (void) OSBitAndAtomic(~flags, &sock->so_traffic_mgt_flags);
+       u_int32_t soflags1 = 0;
+
+       if ((flags & TRAFFIC_MGT_SO_BACKGROUND))
+               soflags1 |= SOF1_TRAFFIC_MGT_SO_BACKGROUND;
+       if ((flags & TRAFFIC_MGT_TCP_RECVBG))
+               soflags1 |= SOF1_TRAFFIC_MGT_TCP_RECVBG;
+       
+       (void) OSBitAndAtomic(~soflags1, &sock->so_flags1);
+

        sock_set_tcp_stream_priority(sock);
 }
 
 void
        sock_set_tcp_stream_priority(sock);
 }
 
 void

-socket_clear_traffic_mgt_flags(socket_t sock, u_int32_t flags)
+socket_clear_traffic_mgt_flags(socket_t sock, u_int8_t flags)

 {
        socket_lock(sock, 1);
        socket_clear_traffic_mgt_flags_locked(sock, flags);
 {
        socket_lock(sock, 1);
        socket_clear_traffic_mgt_flags_locked(sock, flags);


@@ -1103,26 +1200,30 @@ socket_defunct(struct proc *p, socket_t so, int level)
        return (retval);
 }
 
        return (retval);
 }
 

-static void
-sock_setupcalls_common(socket_t sock, sock_upcall rcallback, void *rcontext,
-    sock_upcall wcallback, void *wcontext)
+void
+sock_setupcalls_locked(socket_t sock, sock_upcall rcallback, void *rcontext,
+    sock_upcall wcallback, void *wcontext, int locked)

 {
        if (rcallback != NULL) {
                sock->so_rcv.sb_flags |= SB_UPCALL;
 {
        if (rcallback != NULL) {
                sock->so_rcv.sb_flags |= SB_UPCALL;

+               if (locked)
+                       sock->so_rcv.sb_flags |= SB_UPCALL_LOCK;

                sock->so_rcv.sb_upcall = rcallback;
                sock->so_rcv.sb_upcallarg = rcontext;
        } else {
                sock->so_rcv.sb_upcall = rcallback;
                sock->so_rcv.sb_upcallarg = rcontext;
        } else {

-               sock->so_rcv.sb_flags &= ~SB_UPCALL;
+               sock->so_rcv.sb_flags &= ~(SB_UPCALL | SB_UPCALL_LOCK);

                sock->so_rcv.sb_upcall = NULL;
                sock->so_rcv.sb_upcallarg = NULL;
        }
 
        if (wcallback != NULL) {
                sock->so_snd.sb_flags |= SB_UPCALL;
                sock->so_rcv.sb_upcall = NULL;
                sock->so_rcv.sb_upcallarg = NULL;
        }
 
        if (wcallback != NULL) {
                sock->so_snd.sb_flags |= SB_UPCALL;

+               if (locked)
+                       sock->so_snd.sb_flags |= SB_UPCALL_LOCK;

                sock->so_snd.sb_upcall = wcallback;
                sock->so_snd.sb_upcallarg = wcontext;
        } else {
                sock->so_snd.sb_upcall = wcallback;
                sock->so_snd.sb_upcallarg = wcontext;
        } else {

-               sock->so_snd.sb_flags &= ~SB_UPCALL;
+               sock->so_snd.sb_flags &= ~(SB_UPCALL | SB_UPCALL_LOCK);

                sock->so_snd.sb_upcall = NULL;
                sock->so_snd.sb_upcallarg = NULL;
        }
                sock->so_snd.sb_upcall = NULL;
                sock->so_snd.sb_upcallarg = NULL;
        }


@@ -1144,7 +1245,11 @@ sock_setupcall(socket_t sock, sock_upcall callback, void *context)
         * the read and write callbacks and their respective parameters.
         */
        socket_lock(sock, 1);
         * the read and write callbacks and their respective parameters.
         */
        socket_lock(sock, 1);

-       sock_setupcalls_common(sock, callback, context, NULL, NULL);
+#if CONFIG_EMBEDDED
+       sock_setupcalls_locked(sock, callback, context, callback, context, 0);
+#else
+       sock_setupcalls_locked(sock, callback, context, NULL, NULL, 0);
+#endif /* !CONFIG_EMBEDDED */

        socket_unlock(sock, 1);
 
        return (0);
        socket_unlock(sock, 1);
 
        return (0);


@@ -1161,33 +1266,51 @@ sock_setupcalls(socket_t sock, sock_upcall rcallback, void *rcontext,
         * Note that we don't wait for any in progress upcall to complete.
         */
        socket_lock(sock, 1);
         * Note that we don't wait for any in progress upcall to complete.
         */
        socket_lock(sock, 1);

-       sock_setupcalls_common(sock, rcallback, rcontext, wcallback, wcontext);
+       sock_setupcalls_locked(sock, rcallback, rcontext, wcallback, wcontext, 0);

        socket_unlock(sock, 1);
 
        return (0);
 }
 
        socket_unlock(sock, 1);
 
        return (0);
 }
 

-errno_t
-sock_catchevents(socket_t sock, sock_evupcall ecallback, void *econtext,
+void
+sock_catchevents_locked(socket_t sock, sock_evupcall ecallback, void *econtext,

     u_int32_t emask)
 {
     u_int32_t emask)
 {

-       if (sock == NULL)
-               return (EINVAL);
+       socket_lock_assert_owned(sock);

 
        /*
         * Note that we don't wait for any in progress upcall to complete.
         */
 
        /*
         * Note that we don't wait for any in progress upcall to complete.
         */

-       socket_lock(sock, 1);

        if (ecallback != NULL) {
                sock->so_event = ecallback;
                sock->so_eventarg = econtext;
                sock->so_eventmask = emask;
        } else {
        if (ecallback != NULL) {
                sock->so_event = ecallback;
                sock->so_eventarg = econtext;
                sock->so_eventmask = emask;
        } else {

-               sock->so_event = NULL;
+               sock->so_event = sonullevent;

                sock->so_eventarg = NULL;
                sock->so_eventmask = 0;
        }
                sock->so_eventarg = NULL;
                sock->so_eventmask = 0;
        }

+}
+
+errno_t
+sock_catchevents(socket_t sock, sock_evupcall ecallback, void *econtext,
+    u_int32_t emask)
+{
+       if (sock == NULL)
+               return (EINVAL);
+
+       socket_lock(sock, 1);
+       sock_catchevents_locked(sock, ecallback, econtext, emask);

        socket_unlock(sock, 1);
 
        return (0);
 }
        socket_unlock(sock, 1);
 
        return (0);
 }

+
+/*
+ * Returns true whether or not a socket belongs to the kernel.
+ */
+int
+sock_iskernel(socket_t so)
+{
+       return (so && so->last_pid == 0);
+}