2 * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
4 * @APPLE_LICENSE_HEADER_START@
6 * The contents of this file constitute Original Code as defined in and
7 * are subject to the Apple Public Source License Version 1.1 (the
8 * "License"). You may not use this file except in compliance with the
9 * License. Please obtain a copy of the License at
10 * http://www.apple.com/publicsource and read it before using this file.
12 * This Original Code and all software distributed under the License are
13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17 * License for the specific language governing rights and limitations
20 * @APPLE_LICENSE_HEADER_END@
23 * Copyright (c) 1982, 1986, 1989, 1990, 1993
24 * The Regents of the University of California. All rights reserved.
26 * sendfile(2) and related extensions:
27 * Copyright (c) 1998, David Greenman. All rights reserved.
29 * Redistribution and use in source and binary forms, with or without
30 * modification, are permitted provided that the following conditions
32 * 1. Redistributions of source code must retain the above copyright
33 * notice, this list of conditions and the following disclaimer.
34 * 2. Redistributions in binary form must reproduce the above copyright
35 * notice, this list of conditions and the following disclaimer in the
36 * documentation and/or other materials provided with the distribution.
37 * 3. All advertising materials mentioning features or use of this software
38 * must display the following acknowledgement:
39 * This product includes software developed by the University of
40 * California, Berkeley and its contributors.
41 * 4. Neither the name of the University nor the names of its contributors
42 * may be used to endorse or promote products derived from this software
43 * without specific prior written permission.
45 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
46 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
47 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
48 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
49 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
50 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
51 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
52 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
53 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
54 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
57 * @(#)uipc_syscalls.c 8.4 (Berkeley) 2/21/94
62 #include <sys/param.h>
63 #include <sys/systm.h>
64 #include <sys/filedesc.h>
65 #include <sys/proc_internal.h>
66 #include <sys/file_internal.h>
67 #include <sys/malloc.h>
69 #include <kern/lock.h>
70 #include <sys/domain.h>
71 #include <sys/protosw.h>
72 #include <sys/signalvar.h>
73 #include <sys/socket.h>
74 #include <sys/socketvar.h>
76 #include <sys/ktrace.h>
78 #include <sys/kernel.h>
79 #include <sys/uio_internal.h>
81 #include <bsm/audit_kernel.h>
83 #include <sys/kdebug.h>
84 #include <sys/sysproto.h>
86 #define f_flag f_fglob->fg_flag
87 #define f_type f_fglob->fg_type
88 #define f_msgcount f_fglob->fg_msgcount
89 #define f_cred f_fglob->fg_cred
90 #define f_ops f_fglob->fg_ops
91 #define f_offset f_fglob->fg_offset
92 #define f_data f_fglob->fg_data
95 #define DBG_LAYER_IN_BEG NETDBG_CODE(DBG_NETSOCK, 0)
96 #define DBG_LAYER_IN_END NETDBG_CODE(DBG_NETSOCK, 2)
97 #define DBG_LAYER_OUT_BEG NETDBG_CODE(DBG_NETSOCK, 1)
98 #define DBG_LAYER_OUT_END NETDBG_CODE(DBG_NETSOCK, 3)
99 #define DBG_FNC_SENDMSG NETDBG_CODE(DBG_NETSOCK, (1 << 8) | 1)
100 #define DBG_FNC_SENDTO NETDBG_CODE(DBG_NETSOCK, (2 << 8) | 1)
101 #define DBG_FNC_SENDIT NETDBG_CODE(DBG_NETSOCK, (3 << 8) | 1)
102 #define DBG_FNC_RECVFROM NETDBG_CODE(DBG_NETSOCK, (5 << 8))
103 #define DBG_FNC_RECVMSG NETDBG_CODE(DBG_NETSOCK, (6 << 8))
104 #define DBG_FNC_RECVIT NETDBG_CODE(DBG_NETSOCK, (7 << 8))
109 #define HACK_FOR_4056224 1
111 static pid_t last_pid_4056224
= 0;
112 #endif /* HACK_FOR_4056224 */
116 static void sf_buf_init(void *arg
);
117 SYSINIT(sock_sf
, SI_SUB_MBUF
, SI_ORDER_ANY
, sf_buf_init
, NULL
)
118 static struct sf_buf
*sf_buf_alloc(void);
119 static void sf_buf_ref(caddr_t addr
, u_int size
);
120 static void sf_buf_free(caddr_t addr
, u_int size
);
122 static SLIST_HEAD(, sf_buf
) sf_freelist
;
123 static vm_offset_t sf_base
;
124 static struct sf_buf
*sf_bufs
;
125 static int sf_buf_alloc_want
;
128 static int sendit(struct proc
*p
, int s
, struct user_msghdr
*mp
, uio_t uiop
,
129 int flags
, register_t
*retval
);
130 static int recvit(struct proc
*p
, int s
, struct user_msghdr
*mp
, uio_t uiop
,
131 user_addr_t namelenp
, register_t
*retval
);
133 static int accept1(struct proc
*p
, struct accept_args
*uap
, register_t
*retval
, int compat
);
134 static int getsockname1(struct proc
*p
, struct getsockname_args
*uap
,
135 register_t
*retval
, int compat
);
136 static int getpeername1(struct proc
*p
, struct getpeername_args
*uap
,
137 register_t
*retval
, int compat
);
141 struct orecvmsg_args
{
146 struct osendmsg_args
{
164 int oaccept(struct proc
*p
, struct accept_args
*uap
, register_t
*retval
);
165 int ogetpeername(struct proc
*p
, struct getpeername_args
*uap
, register_t
*retval
);
166 int ogetsockname(struct proc
*p
, struct getsockname_args
*uap
, register_t
*retval
);
167 int orecv(struct proc
*p
, struct orecv_args
*uap
, register_t
*retval
);
168 int orecvfrom(struct proc
*p
, struct recvfrom_args
*uap
, register_t
*retval
);
169 int orecvmsg(struct proc
*p
, struct orecvmsg_args
*uap
, register_t
*retval
);
170 int osend(struct proc
*p
, struct osend_args
*uap
, register_t
*retval
);
171 int osendmsg(struct proc
*p
, struct osendmsg_args
*uap
, register_t
*retval
);
172 #endif // COMPAT_43_SOCKET
175 * System call interface to the socket abstraction.
178 extern struct fileops socketops
;
181 socket(p
, uap
, retval
)
183 register struct socket_args
*uap
;
190 AUDIT_ARG(socket
, uap
->domain
, uap
->type
, uap
->protocol
);
192 error
= falloc(p
, &fp
, &fd
);
196 fp
->f_flag
= FREAD
|FWRITE
;
197 fp
->f_type
= DTYPE_SOCKET
;
198 fp
->f_ops
= &socketops
;
200 error
= socreate(uap
->domain
, &so
, uap
->type
, uap
->protocol
);
204 fp
->f_data
= (caddr_t
)so
;
207 *fdflags(p
, fd
) &= ~UF_RESERVED
;
209 fp_drop(p
, fd
, fp
, 1);
219 bind(struct proc
*p
, struct bind_args
*uap
, __unused register_t
*retval
)
225 AUDIT_ARG(fd
, uap
->s
);
226 error
= file_socket(uap
->s
, &so
);
229 error
= getsockaddr(&sa
, uap
->name
, uap
->namelen
);
232 AUDIT_ARG(sockaddr
, p
, sa
);
234 error
= sobind(so
, sa
);
245 listen(__unused
struct proc
*p
, register struct listen_args
*uap
,
246 __unused register_t
*retval
)
251 AUDIT_ARG(fd
, uap
->s
);
252 error
= file_socket(uap
->s
, &so
);
256 error
= solisten(so
, uap
->backlog
);
263 #if !COMPAT_43_SOCKET
264 #define accept1 accept
270 accept1(struct proc
*p
, struct accept_args
*uap
, register_t
*retval
, int compat
)
276 struct socket
*head
, *so
= NULL
;
277 lck_mtx_t
*mutex_held
;
280 short fflag
; /* type must match fp->f_flag */
283 AUDIT_ARG(fd
, uap
->s
);
285 error
= copyin(uap
->anamelen
, (caddr_t
)&namelen
,
290 error
= fp_getfsock(p
, fd
, &fp
, &head
);
292 if (error
== EOPNOTSUPP
)
301 socket_lock(head
, 1);
303 if (head
->so_proto
->pr_getlock
!= NULL
) {
304 mutex_held
= (*head
->so_proto
->pr_getlock
)(head
, 0);
308 mutex_held
= head
->so_proto
->pr_domain
->dom_mtx
;
313 if ((head
->so_options
& SO_ACCEPTCONN
) == 0) {
314 socket_unlock(head
, 1);
318 if ((head
->so_state
& SS_NBIO
) && head
->so_comp
.tqh_first
== NULL
) {
319 socket_unlock(head
, 1);
323 while (TAILQ_EMPTY(&head
->so_comp
) && head
->so_error
== 0) {
324 if (head
->so_state
& SS_CANTRCVMORE
) {
325 head
->so_error
= ECONNABORTED
;
328 if (head
->so_usecount
< 1)
329 panic("accept1: head=%x refcount=%d\n", head
, head
->so_usecount
);
330 error
= msleep((caddr_t
)&head
->so_timeo
, mutex_held
, PSOCK
| PCATCH
,
332 if (head
->so_usecount
< 1)
333 panic("accept1: 2 head=%x refcount=%d\n", head
, head
->so_usecount
);
334 if ((head
->so_state
& SS_DRAINING
)) {
335 error
= ECONNABORTED
;
338 socket_unlock(head
, 1);
342 if (head
->so_error
) {
343 error
= head
->so_error
;
345 socket_unlock(head
, 1);
351 * At this point we know that there is at least one connection
352 * ready to be accepted. Remove it from the queue prior to
353 * allocating the file descriptor for it since falloc() may
354 * block allowing another process to accept the connection
357 lck_mtx_assert(mutex_held
, LCK_MTX_ASSERT_OWNED
);
358 so
= TAILQ_FIRST(&head
->so_comp
);
359 TAILQ_REMOVE(&head
->so_comp
, so
, so_list
);
361 socket_unlock(head
, 0); /* unlock head to avoid deadlock with select, keep a ref on head */
364 error
= falloc_locked(p
, &fp
, &newfd
, 1);
367 * Probably ran out of file descriptors. Put the
368 * unaccepted connection back onto the queue and
369 * do another wakeup so some other process might
370 * have a chance at it.
373 socket_lock(head
, 0);
374 TAILQ_INSERT_HEAD(&head
->so_comp
, so
, so_list
);
376 wakeup_one((caddr_t
)&head
->so_timeo
);
377 socket_unlock(head
, 1);
380 *fdflags(p
, newfd
) &= ~UF_RESERVED
;
382 fp
->f_type
= DTYPE_SOCKET
;
384 fp
->f_ops
= &socketops
;
385 fp
->f_data
= (caddr_t
)so
;
386 fp_drop(p
, newfd
, fp
, 1);
388 socket_lock(head
, 0);
391 so
->so_state
&= ~SS_COMP
;
394 (void) soacceptlock(so
, &sa
, 0);
395 socket_unlock(head
, 1);
401 socket_unlock(so
, 1);
405 AUDIT_ARG(sockaddr
, p
, sa
);
407 /* check sa_len before it is destroyed */
408 if (namelen
> sa
->sa_len
)
409 namelen
= sa
->sa_len
;
412 ((struct osockaddr
*)sa
)->sa_family
=
415 error
= copyout(sa
, uap
->name
, namelen
);
418 error
= copyout((caddr_t
)&namelen
, uap
->anamelen
,
423 socket_unlock(so
, 1);
430 accept(struct proc
*p
, struct accept_args
*uap
, register_t
*retval
)
433 return (accept1(p
, uap
, retval
, 0));
438 oaccept(struct proc
*p
, struct accept_args
*uap
, register_t
*retval
)
441 return (accept1(p
, uap
, retval
, 1));
443 #endif /* COMPAT_43_SOCKET */
447 connect(struct proc
*p
, struct connect_args
*uap
, __unused register_t
*retval
)
451 lck_mtx_t
*mutex_held
;
455 AUDIT_ARG(fd
, uap
->s
);
456 error
= file_socket( fd
, &so
);
466 if ((so
->so_state
& SS_NBIO
) && (so
->so_state
& SS_ISCONNECTING
)) {
467 socket_unlock(so
, 1);
471 error
= getsockaddr(&sa
, uap
->name
, uap
->namelen
);
473 socket_unlock(so
, 1);
476 AUDIT_ARG(sockaddr
, p
, sa
);
477 error
= soconnectlock(so
, sa
, 0);
480 if ((so
->so_state
& SS_NBIO
) && (so
->so_state
& SS_ISCONNECTING
)) {
482 socket_unlock(so
, 1);
486 while ((so
->so_state
& SS_ISCONNECTING
) && so
->so_error
== 0) {
487 if (so
->so_proto
->pr_getlock
!= NULL
)
488 mutex_held
= (*so
->so_proto
->pr_getlock
)(so
, 0);
490 mutex_held
= so
->so_proto
->pr_domain
->dom_mtx
;
491 error
= msleep((caddr_t
)&so
->so_timeo
, mutex_held
, PSOCK
| PCATCH
,
493 if ((so
->so_state
& SS_DRAINING
)) {
494 error
= ECONNABORTED
;
500 error
= so
->so_error
;
504 so
->so_state
&= ~SS_ISCONNECTING
;
505 socket_unlock(so
, 1);
507 if (error
== ERESTART
)
515 socketpair(struct proc
*p
, struct socketpair_args
*uap
, __unused register_t
*retval
)
517 struct fileproc
*fp1
, *fp2
;
518 struct socket
*so1
, *so2
;
519 int fd
, error
, sv
[2];
521 AUDIT_ARG(socket
, uap
->domain
, uap
->type
, uap
->protocol
);
522 error
= socreate(uap
->domain
, &so1
, uap
->type
, uap
->protocol
);
525 error
= socreate(uap
->domain
, &so2
, uap
->type
, uap
->protocol
);
529 error
= falloc(p
, &fp1
, &fd
);
533 fp1
->f_flag
= FREAD
|FWRITE
;
534 fp1
->f_type
= DTYPE_SOCKET
;
535 fp1
->f_ops
= &socketops
;
536 fp1
->f_data
= (caddr_t
)so1
;
539 error
= falloc(p
, &fp2
, &fd
);
543 fp2
->f_flag
= FREAD
|FWRITE
;
544 fp2
->f_type
= DTYPE_SOCKET
;
545 fp2
->f_ops
= &socketops
;
546 fp2
->f_data
= (caddr_t
)so2
;
549 error
= soconnect2(so1
, so2
);
553 if (uap
->type
== SOCK_DGRAM
) {
555 * Datagram socket connection is asymmetric.
557 error
= soconnect2(so2
, so1
);
564 *fdflags(p
, sv
[0]) &= ~UF_RESERVED
;
565 *fdflags(p
, sv
[1]) &= ~UF_RESERVED
;
566 fp_drop(p
, sv
[0], fp1
, 1);
567 fp_drop(p
, sv
[1], fp2
, 1);
570 error
= copyout((caddr_t
)sv
, uap
->rsv
, 2 * sizeof(int));
571 #if 0 /* old pipe(2) syscall compatability, unused these days */
572 retval
[0] = sv
[0]; /* XXX ??? */
573 retval
[1] = sv
[1]; /* XXX ??? */
577 fp_free(p
, sv
[1], fp2
);
579 fp_free(p
, sv
[0], fp1
);
588 sendit(struct proc
*p
, int s
, struct user_msghdr
*mp
, uio_t uiop
,
589 int flags
, register_t
*retval
)
591 struct mbuf
*control
;
600 KERNEL_DEBUG(DBG_FNC_SENDIT
| DBG_FUNC_START
, 0,0,0,0,0);
602 error
= file_socket(s
, &so
);
605 KERNEL_DEBUG(DBG_FNC_SENDIT
| DBG_FUNC_END
, error
,0,0,0,0);
610 error
= getsockaddr(&to
, mp
->msg_name
, mp
->msg_namelen
);
612 KERNEL_DEBUG(DBG_FNC_SENDIT
| DBG_FUNC_END
, error
,0,0,0,0);
615 AUDIT_ARG(sockaddr
, p
, to
);
619 if (mp
->msg_control
) {
620 if (mp
->msg_controllen
< ((socklen_t
)sizeof(struct cmsghdr
))
622 && !(mp
->msg_flags
& MSG_COMPAT
)
628 error
= sockargs(&control
, mp
->msg_control
,
629 mp
->msg_controllen
, MT_CONTROL
);
633 if (mp
->msg_flags
& MSG_COMPAT
) {
634 register struct cmsghdr
*cm
;
636 M_PREPEND(control
, sizeof(*cm
), M_WAIT
);
641 cm
= mtod(control
, struct cmsghdr
*);
642 cm
->cmsg_len
= control
->m_len
;
643 cm
->cmsg_level
= SOL_SOCKET
;
644 cm
->cmsg_type
= SCM_RIGHTS
;
653 if (KTRPOINT(p
, KTR_GENIO
)) {
654 ktruio
= uio_duplicate(uiop
);
658 len
= uio_resid(uiop
);
662 error
= so
->so_proto
->pr_usrreqs
->pru_sosend(so
, to
, uiop
, 0, control
,
665 if (uio_resid(uiop
) != len
&& (error
== ERESTART
||
666 error
== EINTR
|| error
== EWOULDBLOCK
))
668 /* Generation of SIGPIPE can be controlled per socket */
669 if (error
== EPIPE
&& !(so
->so_flags
& SOF_NOSIGPIPE
))
673 *retval
= (int)(len
- uio_resid(uiop
));
676 if (ktruio
!= NULL
) {
678 uio_setresid(ktruio
, retval
[0]);
679 ktrgenio(p
->p_tracep
, s
, UIO_WRITE
, ktruio
, error
);
686 KERNEL_DEBUG(DBG_FNC_SENDIT
| DBG_FUNC_END
, error
,0,0,0,0);
694 sendto(struct proc
*p
, struct sendto_args
*uap
, register_t
*retval
)
696 struct user_msghdr msg
;
700 KERNEL_DEBUG(DBG_FNC_SENDTO
| DBG_FUNC_START
, 0,0,0,0,0);
701 AUDIT_ARG(fd
, uap
->s
);
703 auio
= uio_create(1, 0,
704 (IS_64BIT_PROCESS(p
) ? UIO_USERSPACE64
: UIO_USERSPACE32
),
709 uio_addiov(auio
, uap
->buf
, uap
->len
);
711 msg
.msg_name
= uap
->to
;
712 msg
.msg_namelen
= uap
->tolen
;
713 /* no need to set up msg_iov. sendit uses uio_t we send it */
719 error
= sendit(p
, uap
->s
, &msg
, auio
, uap
->flags
, retval
);
728 * Temporary workaround to let send() and recv() work over a pipe for binary compatibility
729 * This will be removed in the release following Tiger
731 if (error
== ENOTSOCK
) {
734 if (fp_lookup(p
, uap
->s
, &fp
, 0) == 0) {
735 (void) fp_drop(p
, uap
->s
, fp
,0);
737 if (fp
->f_type
== DTYPE_PIPE
) {
738 struct write_args write_uap
;
739 user_ssize_t write_retval
;
741 if (p
->p_pid
> last_pid_4056224
) {
742 last_pid_4056224
= p
->p_pid
;
744 printf("%s[%d] uses send/recv on a pipe\n",
745 p
->p_comm
, p
->p_pid
);
748 bzero(&write_uap
, sizeof(struct write_args
));
749 write_uap
.fd
= uap
->s
;
750 write_uap
.cbuf
= uap
->buf
;
751 write_uap
.nbyte
= uap
->len
;
753 error
= write(p
, &write_uap
, &write_retval
);
754 *retval
= (int)write_retval
;
758 #endif /* HACK_FOR_4056224 */
760 KERNEL_DEBUG(DBG_FNC_SENDTO
| DBG_FUNC_END
, error
, *retval
,0,0,0);
767 osend(__unused
struct proc
*p
,
768 __unused
struct osend_args
*uap
,
769 __unused register_t
*retval
)
771 /* these are no longer supported and in fact
772 * there is no way to call it directly.
773 * LP64todo - remove this once we're sure there are no clients
779 osendmsg(__unused
struct proc
*p
,
780 __unused
struct osendmsg_args
*uap
,
781 __unused register_t
*retval
)
783 /* these are no longer supported and in fact
784 * there is no way to call it directly.
785 * LP64todo - remove this once we're sure there are no clients
793 sendmsg(struct proc
*p
, register struct sendmsg_args
*uap
, register_t
*retval
)
796 struct user_msghdr user_msg
;
802 struct user_iovec
*iovp
;
804 KERNEL_DEBUG(DBG_FNC_SENDMSG
| DBG_FUNC_START
, 0,0,0,0,0);
805 AUDIT_ARG(fd
, uap
->s
);
806 if (IS_64BIT_PROCESS(p
)) {
807 msghdrp
= (caddr_t
) &user_msg
;
808 size_of_msghdr
= sizeof(user_msg
);
809 size_of_iovec
= sizeof(struct user_iovec
);
812 msghdrp
= (caddr_t
) &msg
;
813 size_of_msghdr
= sizeof(msg
);
814 size_of_iovec
= sizeof(struct iovec
);
816 error
= copyin(uap
->msg
, msghdrp
, size_of_msghdr
);
819 KERNEL_DEBUG(DBG_FNC_SENDMSG
| DBG_FUNC_END
, error
,0,0,0,0);
823 /* only need to copy if user process is not 64-bit */
824 if (!IS_64BIT_PROCESS(p
)) {
825 user_msg
.msg_flags
= msg
.msg_flags
;
826 user_msg
.msg_controllen
= msg
.msg_controllen
;
827 user_msg
.msg_control
= CAST_USER_ADDR_T(msg
.msg_control
);
828 user_msg
.msg_iovlen
= msg
.msg_iovlen
;
829 user_msg
.msg_iov
= CAST_USER_ADDR_T(msg
.msg_iov
);
830 user_msg
.msg_namelen
= msg
.msg_namelen
;
831 user_msg
.msg_name
= CAST_USER_ADDR_T(msg
.msg_name
);
834 if (user_msg
.msg_iovlen
<= 0 || user_msg
.msg_iovlen
> UIO_MAXIOV
) {
835 KERNEL_DEBUG(DBG_FNC_SENDMSG
| DBG_FUNC_END
, EMSGSIZE
,0,0,0,0);
839 /* allocate a uio large enough to hold the number of iovecs passed */
840 auio
= uio_create(user_msg
.msg_iovlen
, 0,
841 (IS_64BIT_PROCESS(p
) ? UIO_USERSPACE64
: UIO_USERSPACE32
),
848 if (user_msg
.msg_iovlen
) {
849 /* get location of iovecs within the uio. then copyin the iovecs from
852 iovp
= uio_iovsaddr(auio
);
857 error
= copyin(user_msg
.msg_iov
, (caddr_t
)iovp
, (user_msg
.msg_iovlen
* size_of_iovec
));
860 user_msg
.msg_iov
= CAST_USER_ADDR_T(iovp
);
862 /* finish setup of uio_t */
863 uio_calculateresid(auio
);
866 user_msg
.msg_iov
= 0;
870 user_msg
.msg_flags
= 0;
872 error
= sendit(p
, uap
->s
, &user_msg
, auio
, uap
->flags
, retval
);
877 KERNEL_DEBUG(DBG_FNC_SENDMSG
| DBG_FUNC_END
, error
,0,0,0,0);
883 recvit(p
, s
, mp
, uiop
, namelenp
, retval
)
884 register struct proc
*p
;
886 register struct user_msghdr
*mp
;
888 user_addr_t namelenp
;
892 struct mbuf
*m
, *control
= 0;
895 struct sockaddr
*fromsa
= 0;
901 KERNEL_DEBUG(DBG_FNC_RECVIT
| DBG_FUNC_START
, 0,0,0,0,0);
903 if ( (error
= fp_lookup(p
, s
, &fp
, 1)) ) {
904 KERNEL_DEBUG(DBG_FNC_RECVIT
| DBG_FUNC_END
, error
,0,0,0,0);
908 if (fp
->f_type
!= DTYPE_SOCKET
) {
914 so
= (struct socket
*)fp
->f_data
;
917 if (uio_resid(uiop
) < 0) {
918 KERNEL_DEBUG(DBG_FNC_RECVIT
| DBG_FUNC_END
, EINVAL
,0,0,0,0);
923 if (KTRPOINT(p
, KTR_GENIO
)) {
924 ktruio
= uio_duplicate(uiop
);
928 len
= uio_resid(uiop
);
932 error
= so
->so_proto
->pr_usrreqs
->pru_soreceive(so
, &fromsa
, uiop
,
933 (struct mbuf
**)0, mp
->msg_control
? &control
: (struct mbuf
**)0,
936 AUDIT_ARG(sockaddr
, p
, fromsa
);
938 if (uio_resid(uiop
) != len
&& (error
== ERESTART
||
939 error
== EINTR
|| error
== EWOULDBLOCK
))
943 if (ktruio
!= NULL
) {
945 uio_setresid(ktruio
, len
- uio_resid(uiop
));
946 ktrgenio(p
->p_tracep
, s
, UIO_WRITE
, ktruio
, error
);
953 *retval
= len
- uio_resid(uiop
);
955 len
= mp
->msg_namelen
;
956 if (len
<= 0 || fromsa
== 0)
960 #define MIN(a,b) ((a)>(b)?(b):(a))
962 /* save sa_len before it is destroyed by MSG_COMPAT */
963 len
= MIN(len
, fromsa
->sa_len
);
965 if (mp
->msg_flags
& MSG_COMPAT
)
966 ((struct osockaddr
*)fromsa
)->sa_family
=
969 error
= copyout(fromsa
, mp
->msg_name
, (unsigned)len
);
973 mp
->msg_namelen
= len
;
975 (error
= copyout((caddr_t
)&len
, namelenp
, sizeof (int)))) {
977 if (mp
->msg_flags
& MSG_COMPAT
)
978 error
= 0; /* old recvfrom didn't check */
984 if (mp
->msg_control
) {
987 * We assume that old recvmsg calls won't receive access
988 * rights and other control info, esp. as control info
989 * is always optional and those options didn't exist in 4.3.
990 * If we receive rights, trim the cmsghdr; anything else
993 if (control
&& mp
->msg_flags
& MSG_COMPAT
) {
994 if (mtod(control
, struct cmsghdr
*)->cmsg_level
!=
996 mtod(control
, struct cmsghdr
*)->cmsg_type
!=
998 mp
->msg_controllen
= 0;
1001 control
->m_len
-= sizeof (struct cmsghdr
);
1002 control
->m_data
+= sizeof (struct cmsghdr
);
1005 len
= mp
->msg_controllen
;
1007 mp
->msg_controllen
= 0;
1008 ctlbuf
= mp
->msg_control
;
1010 while (m
&& len
> 0) {
1011 unsigned int tocopy
;
1013 if (len
>= m
->m_len
)
1016 mp
->msg_flags
|= MSG_CTRUNC
;
1020 error
= copyout((caddr_t
)mtod(m
, caddr_t
), ctlbuf
, tocopy
);
1028 mp
->msg_controllen
= ctlbuf
- mp
->msg_control
;
1032 FREE(fromsa
, M_SONAME
);
1035 KERNEL_DEBUG(DBG_FNC_RECVIT
| DBG_FUNC_END
, error
,0,0,0,0);
1037 fp_drop(p
, s
, fp
, 0);
1043 recvfrom(p
, uap
, retval
)
1045 register struct recvfrom_args
/* {
1055 struct user_msghdr msg
;
1059 KERNEL_DEBUG(DBG_FNC_RECVFROM
| DBG_FUNC_START
, 0,0,0,0,0);
1060 AUDIT_ARG(fd
, uap
->s
);
1062 if (uap
->fromlenaddr
) {
1063 error
= copyin(uap
->fromlenaddr
,
1064 (caddr_t
)&msg
.msg_namelen
, sizeof (msg
.msg_namelen
));
1068 msg
.msg_namelen
= 0;
1069 msg
.msg_name
= uap
->from
;
1070 auio
= uio_create(1, 0,
1071 (IS_64BIT_PROCESS(p
) ? UIO_USERSPACE64
: UIO_USERSPACE32
),
1077 uio_addiov(auio
, uap
->buf
, uap
->len
);
1078 /* no need to set up msg_iov. recvit uses uio_t we send it */
1081 msg
.msg_control
= 0;
1082 msg
.msg_controllen
= 0;
1083 msg
.msg_flags
= uap
->flags
;
1084 error
= recvit(p
, uap
->s
, &msg
, auio
, uap
->fromlenaddr
, retval
);
1089 #if HACK_FOR_4056224
1092 * Temporary workaround to let send() and recv() work over a pipe for binary compatibility
1093 * This will be removed in the release following Tiger
1095 if (error
== ENOTSOCK
&& proc_is64bit(p
) == 0) {
1096 struct fileproc
*fp
;
1098 if (fp_lookup(p
, uap
->s
, &fp
, 0) == 0) {
1099 (void) fp_drop(p
, uap
->s
, fp
,0);
1101 if (fp
->f_type
== DTYPE_PIPE
) {
1102 struct read_args read_uap
;
1103 user_ssize_t read_retval
;
1105 if (p
->p_pid
> last_pid_4056224
) {
1106 last_pid_4056224
= p
->p_pid
;
1108 printf("%s[%d] uses send/recv on a pipe\n",
1109 p
->p_comm
, p
->p_pid
);
1112 bzero(&read_uap
, sizeof(struct read_args
));
1113 read_uap
.fd
= uap
->s
;
1114 read_uap
.cbuf
= uap
->buf
;
1115 read_uap
.nbyte
= uap
->len
;
1117 error
= read(p
, &read_uap
, &read_retval
);
1118 *retval
= (int)read_retval
;
1122 #endif /* HACK_FOR_4056224 */
1124 KERNEL_DEBUG(DBG_FNC_RECVFROM
| DBG_FUNC_END
, error
,0,0,0,0);
1129 #if COMPAT_43_SOCKET
1131 orecvfrom(struct proc
*p
, struct recvfrom_args
*uap
, register_t
*retval
)
1134 uap
->flags
|= MSG_COMPAT
;
1135 return (recvfrom(p
, uap
, retval
));
1140 #if COMPAT_43_SOCKET
1142 orecv(__unused
struct proc
*p
, __unused
struct orecv_args
*uap
,
1143 __unused register_t
*retval
)
1145 /* these are no longer supported and in fact
1146 * there is no way to call it directly.
1147 * LP64todo - remove this once we're sure there are no clients
1154 * Old recvmsg. This code takes advantage of the fact that the old msghdr
1155 * overlays the new one, missing only the flags, and with the (old) access
1156 * rights where the control fields are now.
1159 orecvmsg(__unused
struct proc
*p
, __unused
struct orecvmsg_args
*uap
,
1160 __unused register_t
*retval
)
1162 /* these are no longer supported and in fact
1163 * there is no way to call it directly.
1164 * LP64todo - remove this once we're sure there are no clients
1173 recvmsg(p
, uap
, retval
)
1175 struct recvmsg_args
*uap
;
1179 struct user_msghdr user_msg
;
1186 struct user_iovec
*iovp
;
1188 KERNEL_DEBUG(DBG_FNC_RECVMSG
| DBG_FUNC_START
, 0,0,0,0,0);
1189 AUDIT_ARG(fd
, uap
->s
);
1190 if (IS_64BIT_PROCESS(p
)) {
1191 msghdrp
= (caddr_t
) &user_msg
;
1192 size_of_msghdr
= sizeof(user_msg
);
1193 size_of_iovec
= sizeof(struct user_iovec
);
1196 msghdrp
= (caddr_t
) &msg
;
1197 size_of_msghdr
= sizeof(msg
);
1198 size_of_iovec
= sizeof(struct iovec
);
1200 error
= copyin(uap
->msg
, msghdrp
, size_of_msghdr
);
1203 KERNEL_DEBUG(DBG_FNC_RECVMSG
| DBG_FUNC_END
, error
,0,0,0,0);
1207 /* only need to copy if user process is not 64-bit */
1208 if (!IS_64BIT_PROCESS(p
)) {
1209 user_msg
.msg_flags
= msg
.msg_flags
;
1210 user_msg
.msg_controllen
= msg
.msg_controllen
;
1211 user_msg
.msg_control
= CAST_USER_ADDR_T(msg
.msg_control
);
1212 user_msg
.msg_iovlen
= msg
.msg_iovlen
;
1213 user_msg
.msg_iov
= CAST_USER_ADDR_T(msg
.msg_iov
);
1214 user_msg
.msg_namelen
= msg
.msg_namelen
;
1215 user_msg
.msg_name
= CAST_USER_ADDR_T(msg
.msg_name
);
1218 if (user_msg
.msg_iovlen
<= 0 || user_msg
.msg_iovlen
> UIO_MAXIOV
) {
1219 KERNEL_DEBUG(DBG_FNC_RECVMSG
| DBG_FUNC_END
, EMSGSIZE
,0,0,0,0);
1223 #if COMPAT_43_SOCKET
1224 user_msg
.msg_flags
= uap
->flags
&~ MSG_COMPAT
;
1226 user_msg
.msg_flags
= uap
->flags
;
1229 /* allocate a uio large enough to hold the number of iovecs passed */
1230 auio
= uio_create(user_msg
.msg_iovlen
, 0,
1231 (IS_64BIT_PROCESS(p
) ? UIO_USERSPACE64
: UIO_USERSPACE32
),
1238 /* get location of iovecs within the uio. then copyin the iovecs from
1241 iovp
= uio_iovsaddr(auio
);
1246 uiov
= user_msg
.msg_iov
;
1247 user_msg
.msg_iov
= CAST_USER_ADDR_T(iovp
);
1248 error
= copyin(uiov
, (caddr_t
)iovp
, (user_msg
.msg_iovlen
* size_of_iovec
));
1252 /* finish setup of uio_t */
1253 uio_calculateresid(auio
);
1255 error
= recvit(p
, uap
->s
, &user_msg
, auio
, 0, retval
);
1257 user_msg
.msg_iov
= uiov
;
1258 /* only need to copy if user process is not 64-bit */
1259 if (!IS_64BIT_PROCESS(p
)) {
1260 // LP64todo - do all these change? if not, then no need to copy all of them!
1261 msg
.msg_flags
= user_msg
.msg_flags
;
1262 msg
.msg_controllen
= user_msg
.msg_controllen
;
1263 msg
.msg_control
= CAST_DOWN(caddr_t
, user_msg
.msg_control
);
1264 msg
.msg_iovlen
= user_msg
.msg_iovlen
;
1265 msg
.msg_iov
= (struct iovec
*) CAST_DOWN(caddr_t
, user_msg
.msg_iov
);
1266 msg
.msg_namelen
= user_msg
.msg_namelen
;
1267 msg
.msg_name
= CAST_DOWN(caddr_t
, user_msg
.msg_name
);
1269 error
= copyout(msghdrp
, uap
->msg
, size_of_msghdr
);
1275 KERNEL_DEBUG(DBG_FNC_RECVMSG
| DBG_FUNC_END
, error
,0,0,0,0);
1281 shutdown(__unused
struct proc
*p
, struct shutdown_args
*uap
, __unused register_t
*retval
)
1286 AUDIT_ARG(fd
, uap
->s
);
1287 error
= file_socket(uap
->s
, &so
);
1294 error
= soshutdown((struct socket
*)so
, uap
->how
);
1306 setsockopt(struct proc
*p
, struct setsockopt_args
*uap
, __unused register_t
*retval
)
1309 struct sockopt sopt
;
1312 AUDIT_ARG(fd
, uap
->s
);
1313 if (uap
->val
== 0 && uap
->valsize
!= 0)
1315 if (uap
->valsize
< 0)
1318 error
= file_socket(uap
->s
, &so
);
1322 sopt
.sopt_dir
= SOPT_SET
;
1323 sopt
.sopt_level
= uap
->level
;
1324 sopt
.sopt_name
= uap
->name
;
1325 sopt
.sopt_val
= uap
->val
;
1326 sopt
.sopt_valsize
= uap
->valsize
;
1333 error
= sosetopt(so
, &sopt
);
1342 getsockopt(struct proc
*p
, struct getsockopt_args
*uap
, __unused register_t
*retval
)
1346 struct sockopt sopt
;
1349 error
= file_socket(uap
->s
, &so
);
1353 error
= copyin(uap
->avalsize
, (caddr_t
)&valsize
, sizeof (valsize
));
1363 sopt
.sopt_dir
= SOPT_GET
;
1364 sopt
.sopt_level
= uap
->level
;
1365 sopt
.sopt_name
= uap
->name
;
1366 sopt
.sopt_val
= uap
->val
;
1367 sopt
.sopt_valsize
= (size_t)valsize
; /* checked non-negative above */
1374 error
= sogetopt((struct socket
*)so
, &sopt
);
1376 valsize
= sopt
.sopt_valsize
;
1377 error
= copyout((caddr_t
)&valsize
, uap
->avalsize
, sizeof (valsize
));
1390 getsockname1(__unused
struct proc
*p
, struct getsockname_args
*uap
, __unused register_t
*retval
,
1394 struct sockaddr
*sa
;
1398 error
= file_socket(uap
->fdes
, &so
);
1401 error
= copyin(uap
->alen
, (caddr_t
)&len
, sizeof(socklen_t
));
1410 error
= (*so
->so_proto
->pr_usrreqs
->pru_sockaddr
)(so
, &sa
);
1413 struct socket_filter_entry
*filter
;
1415 for (filter
= so
->so_filt
; filter
&& error
== 0;
1416 filter
= filter
->sfe_next_onsocket
) {
1417 if (filter
->sfe_filter
->sf_filter
.sf_getsockname
) {
1421 socket_unlock(so
, 0);
1423 error
= filter
->sfe_filter
->sf_filter
.sf_getsockname(filter
->sfe_cookie
,
1428 if (error
== EJUSTRETURN
)
1436 socket_unlock(so
, 1);
1444 len
= MIN(len
, sa
->sa_len
);
1445 #if COMPAT_43_SOCKET
1447 ((struct osockaddr
*)sa
)->sa_family
= sa
->sa_family
;
1449 error
= copyout((caddr_t
)sa
, uap
->asa
, len
);
1452 error
= copyout((caddr_t
)&len
, uap
->alen
, sizeof(socklen_t
));
1457 file_drop(uap
->fdes
);
1462 getsockname(struct proc
*p
, struct getsockname_args
*uap
, register_t
*retval
)
1464 return (getsockname1(p
, uap
, retval
, 0));
1467 #if COMPAT_43_SOCKET
1469 ogetsockname(struct proc
*p
, struct getsockname_args
*uap
, register_t
*retval
)
1471 return (getsockname1(p
, uap
, retval
, 1));
1473 #endif /* COMPAT_43_SOCKET */
1476 * Get name of peer for connected socket.
1480 getpeername1(__unused
struct proc
*p
, struct getpeername_args
*uap
, __unused register_t
*retval
,
1484 struct sockaddr
*sa
;
1488 error
= file_socket(uap
->fdes
, &so
);
1498 if ((so
->so_state
& (SS_ISCONNECTED
|SS_ISCONFIRMING
)) == 0) {
1499 socket_unlock(so
, 1);
1503 error
= copyin(uap
->alen
, (caddr_t
)&len
, sizeof(socklen_t
));
1505 socket_unlock(so
, 1);
1509 error
= (*so
->so_proto
->pr_usrreqs
->pru_peeraddr
)(so
, &sa
);
1512 struct socket_filter_entry
*filter
;
1514 for (filter
= so
->so_filt
; filter
&& error
== 0;
1515 filter
= filter
->sfe_next_onsocket
) {
1516 if (filter
->sfe_filter
->sf_filter
.sf_getpeername
) {
1520 socket_unlock(so
, 0);
1522 error
= filter
->sfe_filter
->sf_filter
.sf_getpeername(filter
->sfe_cookie
,
1527 if (error
== EJUSTRETURN
)
1535 socket_unlock(so
, 1);
1542 len
= MIN(len
, sa
->sa_len
);
1543 #if COMPAT_43_SOCKET
1545 ((struct osockaddr
*)sa
)->sa_family
=
1548 error
= copyout(sa
, uap
->asa
, len
);
1552 error
= copyout((caddr_t
)&len
, uap
->alen
, sizeof(socklen_t
));
1554 if (sa
) FREE(sa
, M_SONAME
);
1556 file_drop(uap
->fdes
);
1561 getpeername(struct proc
*p
, struct getpeername_args
*uap
, register_t
*retval
)
1564 return (getpeername1(p
, uap
, retval
, 0));
1567 #if COMPAT_43_SOCKET
1569 ogetpeername(struct proc
*p
, struct getpeername_args
*uap
, register_t
*retval
)
1572 return (getpeername1(p
, uap
, retval
, 1));
1574 #endif /* COMPAT_43_SOCKET */
1577 sockargs(mp
, data
, buflen
, type
)
1582 register struct sockaddr
*sa
;
1583 register struct mbuf
*m
;
1586 if ((u_int
)buflen
> MLEN
) {
1587 #if COMPAT_43_SOCKET
1588 if (type
== MT_SONAME
&& (u_int
)buflen
<= 112)
1589 buflen
= MLEN
; /* unix domain compat. hack */
1592 if ((u_int
)buflen
> MCLBYTES
)
1595 m
= m_get(M_WAIT
, type
);
1598 if ((u_int
)buflen
> MLEN
) {
1600 if ((m
->m_flags
& M_EXT
) == 0) {
1606 error
= copyin(data
, mtod(m
, caddr_t
), (u_int
)buflen
);
1611 if (type
== MT_SONAME
) {
1612 sa
= mtod(m
, struct sockaddr
*);
1614 #if COMPAT_43_SOCKET && BYTE_ORDER != BIG_ENDIAN
1615 if (sa
->sa_family
== 0 && sa
->sa_len
< AF_MAX
)
1616 sa
->sa_family
= sa
->sa_len
;
1618 sa
->sa_len
= buflen
;
1625 * Given a user_addr_t of length len, allocate and fill out a *sa.
1628 getsockaddr(struct sockaddr
**namp
, user_addr_t uaddr
, size_t len
)
1630 struct sockaddr
*sa
;
1633 if (len
> SOCK_MAXADDRLEN
)
1634 return ENAMETOOLONG
;
1639 MALLOC(sa
, struct sockaddr
*, len
, M_SONAME
, M_WAITOK
);
1643 error
= copyin(uaddr
, (caddr_t
)sa
, len
);
1647 #if COMPAT_43_SOCKET && BYTE_ORDER != BIG_ENDIAN
1648 if (sa
->sa_family
== 0 && sa
->sa_len
< AF_MAX
)
1649 sa
->sa_family
= sa
->sa_len
;
1660 * Allocate a pool of sf_bufs (sendfile(2) or "super-fast" if you prefer. :-))
1661 * XXX - The sf_buf functions are currently private to sendfile(2), so have
1662 * been made static, but may be useful in the future for doing zero-copy in
1663 * other parts of the networking code.
1666 sf_buf_init(void *arg
)
1670 SLIST_INIT(&sf_freelist
);
1671 kmem_alloc_pageable(kernel_map
, &sf_base
, nsfbufs
* PAGE_SIZE
);
1672 MALLOC(sf_bufs
, struct sf_buf
*, nsfbufs
* sizeof(struct sf_buf
), M_TEMP
, M_NOWAIT
|M_ZERO
);
1673 if (sf_bufs
== NULL
)
1674 return; /* XXX silently fail leaving sf_bufs NULL */
1676 for (i
= 0; i
< nsfbufs
; i
++) {
1677 sf_bufs
[i
].kva
= sf_base
+ i
* PAGE_SIZE
;
1678 SLIST_INSERT_HEAD(&sf_freelist
, &sf_bufs
[i
], free_list
);
1683 * Get an sf_buf from the freelist. Will block if none are available.
1685 static struct sf_buf
*
1690 while ((sf
= SLIST_FIRST(&sf_freelist
)) == NULL
) {
1691 sf_buf_alloc_want
= 1;
1692 tsleep(&sf_freelist
, PVM
, "sfbufa", 0);
1694 SLIST_REMOVE_HEAD(&sf_freelist
, free_list
);
1699 #define dtosf(x) (&sf_bufs[((uintptr_t)(x) - (uintptr_t)sf_base) >> PAGE_SHIFT])
1701 sf_buf_ref(caddr_t addr
, u_int size
)
1706 if (sf
->refcnt
== 0)
1707 panic("sf_buf_ref: referencing a free sf_buf");
1712 * Lose a reference to an sf_buf. When none left, detach mapped page
1713 * and release resources back to the system.
1715 * Must be called at splimp.
1718 sf_buf_free(caddr_t addr
, u_int size
)
1724 if (sf
->refcnt
== 0)
1725 panic("sf_buf_free: freeing free sf_buf");
1727 if (sf
->refcnt
== 0) {
1728 pmap_qremove((vm_offset_t
)addr
, 1);
1730 vm_page_unwire(m
, 0);
1732 * Check for the object going away on us. This can
1733 * happen since we don't hold a reference to it.
1734 * If so, we're responsible for freeing the page.
1736 if (m
->wire_count
== 0 && m
->object
== NULL
)
1737 vm_page_lock_queues();
1739 vm_page_unlock_queues();
1741 SLIST_INSERT_HEAD(&sf_freelist
, sf
, free_list
);
1742 if (sf_buf_alloc_want
) {
1743 sf_buf_alloc_want
= 0;
1744 wakeup(&sf_freelist
);
1751 * int sendfile(int fd, int s, off_t offset, size_t nbytes,
1752 * struct sf_hdtr *hdtr, off_t *sbytes, int flags)
1754 * Send a file specified by 'fd' and starting at 'offset' to a socket
1755 * specified by 's'. Send only 'nbytes' of the file or until EOF if
1756 * nbytes == 0. Optionally add a header and/or trailer to the socket
1757 * output. If specified, write the total number of bytes sent into *sbytes.
1760 sendfile(struct proc
*p
, struct sendfile_args
*uap
)
1762 struct fileproc
*fp
;
1764 struct vm_object
*obj
;
1769 struct writev_args nuap
;
1770 struct sf_hdtr hdtr
;
1771 off_t off
, xfsize
, sbytes
= 0;
1774 if (sf_bufs
== NULL
) {
1775 /* Fail if initialization failed */
1780 * Do argument checking. Must be a regular file in, stream
1781 * type and connected socket out, positive offset.
1783 if (error
= fp_getfvp(p
, uap
->fd
, &fp
, &vp
))
1785 if (fp
->f_flag
& FREAD
) == 0) {
1790 if (vp
->v_type
!= VREG
|| obj
== NULL
) {
1794 error
= file_socket(uap
->s
, &so
);
1804 if (so
->so_type
!= SOCK_STREAM
) {
1808 if ((so
->so_state
& SS_ISCONNECTED
) == 0) {
1812 if (uap
->offset
< 0) {
1818 * If specified, get the pointer to the sf_hdtr struct for
1819 * any headers/trailers.
1821 if (uap
->hdtr
!= NULL
) {
1822 error
= copyin(CAST_USER_ADDR_T(uap
->hdtr
), &hdtr
, sizeof(hdtr
));
1826 * Send any headers. Wimp out and use writev(2).
1828 if (hdtr
.headers
!= NULL
) {
1830 nuap
.iovp
= hdtr
.headers
;
1831 nuap
.iovcnt
= hdtr
.hdr_cnt
;
1832 error
= writev(p
, &nuap
);
1835 sbytes
+= p
->p_retval
[0];
1840 * Protect against multiple writers to the socket.
1842 (void) sblock(&so
->so_snd
, M_WAIT
);
1845 * Loop through the pages in the file, starting with the requested
1846 * offset. Get a file page (do I/O if necessary), map the file page
1847 * into an sf_buf, attach an mbuf header to the sf_buf, and queue
1850 for (off
= uap
->offset
; ; off
+= xfsize
, sbytes
+= xfsize
) {
1851 vm_object_offset_t pindex
;
1852 vm_object_offset_t pgoff
;
1854 pindex
= OFF_TO_IDX(off
);
1857 * Calculate the amount to transfer. Not to exceed a page,
1858 * the EOF, or the passed in nbytes.
1860 xfsize
= obj
->un_pager
.vnp
.vnp_size
- off
;
1861 if (xfsize
> PAGE_SIZE_64
)
1863 pgoff
= (vm_object_offset_t
)(off
& PAGE_MASK_64
);
1864 if (PAGE_SIZE
- pgoff
< xfsize
)
1865 xfsize
= PAGE_SIZE_64
- pgoff
;
1866 if (uap
->nbytes
&& xfsize
> (uap
->nbytes
- sbytes
))
1867 xfsize
= uap
->nbytes
- sbytes
;
1871 * Optimize the non-blocking case by looking at the socket space
1872 * before going to the extra work of constituting the sf_buf.
1874 if ((so
->so_state
& SS_NBIO
) && sbspace(&so
->so_snd
) <= 0) {
1875 if (so
->so_state
& SS_CANTSENDMORE
)
1879 sbunlock(&so
->so_snd
, 0); /* will release lock */
1883 * Attempt to look up the page. If the page doesn't exist or the
1884 * part we're interested in isn't valid, then read it from disk.
1885 * If some other part of the kernel has this page (i.e. it's busy),
1886 * then disk I/O may be occuring on it, so wait and retry.
1888 pg
= vm_page_lookup(obj
, pindex
);
1889 if (pg
== NULL
|| (!(pg
->flags
& PG_BUSY
) && !pg
->busy
&&
1890 !vm_page_is_valid(pg
, pgoff
, xfsize
))) {
1896 pg
= vm_page_alloc(obj
, pindex
, VM_ALLOC_NORMAL
);
1902 * don't just clear PG_BUSY manually -
1903 * vm_page_alloc() should be considered opaque,
1904 * use the VM routine provided to clear
1911 * Ensure that our page is still around when the I/O completes.
1913 vm_page_io_start(pg
);
1916 * Get the page from backing store.
1918 bsize
= vp
->v_mount
->mnt_vfsstat
.f_iosize
;
1919 auio
.uio_iov
= &aiov
;
1920 auio
.uio_iovcnt
= 1;
1922 aiov
.iov_len
= MAXBSIZE
;
1923 auio
.uio_offset
= trunc_page(off
);
1924 auio
.uio_segflg
= UIO_NOCOPY
;
1925 auio
.uio_rw
= UIO_READ
;
1926 uio_setresid(&auio
, MAXBSIZE
);
1927 error
= VOP_READ(vp
, &auio
, IO_VMIO
| ((MAXBSIZE
/ bsize
) << 16),
1929 vm_page_flag_clear(pg
, PG_ZERO
);
1930 vm_page_io_finish(pg
);
1932 vm_page_unwire(pg
, 0);
1934 * See if anyone else might know about this page.
1935 * If not and it is not valid, then free it.
1937 if (pg
->wire_count
== 0 && pg
->valid
== 0 &&
1938 pg
->busy
== 0 && !(pg
->flags
& PG_BUSY
) &&
1939 pg
->hold_count
== 0)
1940 vm_page_lock_queues();
1942 vm_page_unlock_queues();
1943 sbunlock(&so
->so_snd
, 0); /* will release socket lock */
1947 if ((pg
->flags
& PG_BUSY
) || pg
->busy
) {
1949 if ((pg
->flags
& PG_BUSY
) || pg
->busy
) {
1951 * Page is busy. Wait and retry.
1953 vm_page_flag_set(pg
, PG_WANTED
);
1954 tsleep(pg
, PVM
, "sfpbsy", 0);
1959 * Protect from having the page ripped out from beneath us.
1964 * Allocate a kernel virtual page and insert the physical page
1967 sf
= sf_buf_alloc();
1969 pmap_qenter(sf
->kva
, &pg
, 1);
1971 * Get an mbuf header and set it up as having external storage.
1973 MGETHDR(m
, M_WAIT
, MT_DATA
);
1976 sbunlock(&so
->so_snd
, 0); /* will release socket lock */
1979 m
->m_ext
.ext_free
= sf_buf_free
;
1980 m
->m_ext
.ext_ref
= sf_buf_ref
;
1981 m
->m_ext
.ext_buf
= (void *)sf
->kva
;
1982 m
->m_ext
.ext_size
= PAGE_SIZE
;
1983 m
->m_data
= (char *) sf
->kva
+ pgoff
;
1984 m
->m_flags
|= M_EXT
;
1985 m
->m_pkthdr
.len
= m
->m_len
= xfsize
;
1987 * Add the buffer to the socket buffer chain.
1991 * Make sure that the socket is still able to take more data.
1992 * CANTSENDMORE being true usually means that the connection
1993 * was closed. so_error is true when an error was sensed after
1995 * The state is checked after the page mapping and buffer
1996 * allocation above since those operations may block and make
1997 * any socket checks stale. From this point forward, nothing
1998 * blocks before the pru_send (or more accurately, any blocking
1999 * results in a loop back to here to re-check).
2001 if ((so
->so_state
& SS_CANTSENDMORE
) || so
->so_error
) {
2002 if (so
->so_state
& SS_CANTSENDMORE
) {
2005 error
= so
->so_error
;
2009 sbunlock(&so
->so_snd
, 0); /* will release socket lock */
2013 * Wait for socket space to become available. We do this just
2014 * after checking the connection state above in order to avoid
2015 * a race condition with sbwait().
2017 if (sbspace(&so
->so_snd
) < so
->so_snd
.sb_lowat
) {
2018 if (so
->so_state
& SS_NBIO
) {
2020 sbunlock(&so
->so_snd
, 0); /* will release socket lock */
2024 error
= sbwait(&so
->so_snd
);
2026 * An error from sbwait usually indicates that we've
2027 * been interrupted by a signal. If we've sent anything
2028 * then return bytes sent, otherwise return the error.
2032 sbunlock(&so
->so_snd
, 0);
2037 error
= (*so
->so_proto
->pr_usrreqs
->pru_send
)(so
, 0, m
, 0, 0, p
);
2040 sbunlock(&so
->so_snd
, 0); /* will release socket lock */
2044 sbunlock(&so
->so_snd
, 0); /* will release socket lock */
2047 * Send trailers. Wimp out and use writev(2).
2049 if (uap
->hdtr
!= NULL
&& hdtr
.trailers
!= NULL
) {
2051 nuap
.iovp
= hdtr
.trailers
;
2052 nuap
.iovcnt
= hdtr
.trl_cnt
;
2053 error
= writev(p
, &nuap
);
2056 sbytes
+= p
->p_retval
[0];
2063 if (uap
->sbytes
!= NULL
) {
2064 /* XXX this appears bogus for some early failure conditions */
2065 copyout(&sbytes
, CAST_USER_ADDR_T(uap
->sbytes
), sizeof(off_t
));
2069 socket_unlock(so
, 1);