2 * Copyright (c) 1996 John S. Dyson
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice immediately at the beginning of the file, without modification,
10 * this list of conditions, and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * 3. Absolutely no warranty of function or purpose is made by the author
16 * 4. Modifications may be freely made to this file if the above conditions
20 * Copyright (c) 2003-2004 Apple Computer, Inc. All rights reserved.
22 * @APPLE_LICENSE_HEADER_START@
24 * The contents of this file constitute Original Code as defined in and
25 * are subject to the Apple Public Source License Version 1.1 (the
26 * "License"). You may not use this file except in compliance with the
27 * License. Please obtain a copy of the License at
28 * http://www.apple.com/publicsource and read it before using this file.
30 * This Original Code and all software distributed under the License are
31 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
32 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
33 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
34 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
35 * License for the specific language governing rights and limitations
38 * @APPLE_LICENSE_HEADER_END@
42 * This file contains a high-performance replacement for the socket-based
43 * pipes scheme originally used in FreeBSD/4.4Lite. It does not support
44 * all features of sockets, but does do everything that pipes normally
49 * This code has two modes of operation, a small write mode and a large
50 * write mode. The small write mode acts like conventional pipes with
51 * a kernel buffer. If the buffer is less than PIPE_MINDIRECT, then the
52 * "normal" pipe buffering is done. If the buffer is between PIPE_MINDIRECT
53 * and PIPE_SIZE in size, it is fully mapped and wired into the kernel, and
54 * the receiving process can copy it directly from the pages in the sending
57 * If the sending process receives a signal, it is possible that it will
58 * go away, and certainly its address space can change, because control
59 * is returned back to the user-mode side. In that case, the pipe code
60 * arranges to copy the buffer supplied by the user process, to a pageable
61 * kernel buffer, and the receiving process will grab the data from the
62 * pageable kernel buffer. Since signals don't happen all that often,
63 * the copy operation is normally eliminated.
65 * The constant PIPE_MINDIRECT is chosen to make sure that buffering will
66 * happen for small transfers so that the system will not spend all of
67 * its time context switching.
69 * In order to limit the resource use of pipes, two sysctls exist:
71 * kern.ipc.maxpipekva - This is a hard limit on the amount of pageable
72 * address space available to us in pipe_map. Whenever the amount in use
73 * exceeds half of this value, all new pipes will be created with size
74 * SMALL_PIPE_SIZE, rather than PIPE_SIZE. Big pipe creation will be limited
75 * as well. This value is loader tunable only.
77 * kern.ipc.maxpipekvawired - This value limits the amount of memory that may
78 * be wired in order to facilitate direct copies using page flipping.
79 * Whenever this value is exceeded, pipes will fall back to using regular
80 * copies. This value is sysctl controllable at all times.
82 * These values are autotuned in subr_param.c.
84 * Memory usage may be monitored through the sysctls
85 * kern.ipc.pipes, kern.ipc.pipekva and kern.ipc.pipekvawired.
89 #include <sys/param.h>
90 #include <sys/systm.h>
91 #include <sys/filedesc.h>
92 #include <sys/kernel.h>
93 #include <sys/vnode.h>
94 #include <sys/proc_internal.h>
95 #include <sys/kauth.h>
96 #include <sys/file_internal.h>
98 #include <sys/ioctl.h>
99 #include <sys/fcntl.h>
100 #include <sys/malloc.h>
101 #include <sys/syslog.h>
102 #include <sys/unistd.h>
103 #include <sys/resourcevar.h>
104 #include <sys/aio_kern.h>
105 #include <sys/signalvar.h>
106 #include <sys/pipe.h>
107 #include <sys/sysproto.h>
108 #include <sys/proc_info.h>
110 #include <bsm/audit_kernel.h>
112 #include <sys/kdebug.h>
114 #include <kern/zalloc.h>
115 #include <vm/vm_kern.h>
116 #include <libkern/OSAtomic.h>
118 #define f_flag f_fglob->fg_flag
119 #define f_type f_fglob->fg_type
120 #define f_msgcount f_fglob->fg_msgcount
121 #define f_cred f_fglob->fg_cred
122 #define f_ops f_fglob->fg_ops
123 #define f_offset f_fglob->fg_offset
124 #define f_data f_fglob->fg_data
126 * Use this define if you want to disable *fancy* VM things. Expect an
127 * approx 30% decrease in transfer rate. This could be useful for
130 * this needs to be ported to X and the performance measured
131 * before committing to supporting it
133 #define PIPE_NODIRECT 1
135 #ifndef PIPE_NODIRECT
138 #include <vm/vm_param.h>
139 #include <vm/vm_object.h>
140 #include <vm/vm_kern.h>
141 #include <vm/vm_extern.h>
143 #include <vm/vm_map.h>
144 #include <vm/vm_page.h>
151 * interfaces to the outside world
153 static int pipe_read(struct fileproc
*fp
, struct uio
*uio
,
154 kauth_cred_t cred
, int flags
, struct proc
*p
);
156 static int pipe_write(struct fileproc
*fp
, struct uio
*uio
,
157 kauth_cred_t cred
, int flags
, struct proc
*p
);
159 static int pipe_close(struct fileglob
*fg
, struct proc
*p
);
161 static int pipe_select(struct fileproc
*fp
, int which
, void * wql
, struct proc
*p
);
163 static int pipe_kqfilter(struct fileproc
*fp
, struct knote
*kn
, struct proc
*p
);
165 static int pipe_ioctl(struct fileproc
*fp
, u_long cmd
, caddr_t data
, struct proc
*p
);
168 struct fileops pipeops
=
178 static void filt_pipedetach(struct knote
*kn
);
179 static int filt_piperead(struct knote
*kn
, long hint
);
180 static int filt_pipewrite(struct knote
*kn
, long hint
);
182 static struct filterops pipe_rfiltops
=
183 { 1, NULL
, filt_pipedetach
, filt_piperead
};
184 static struct filterops pipe_wfiltops
=
185 { 1, NULL
, filt_pipedetach
, filt_pipewrite
};
188 * Default pipe buffer size(s), this can be kind-of large now because pipe
189 * space is pageable. The pipe code will try to maintain locality of
190 * reference for performance reasons, so small amounts of outstanding I/O
191 * will not wipe the cache.
193 #define MINPIPESIZE (PIPE_SIZE/3)
196 * Limit the number of "big" pipes
198 #define LIMITBIGPIPES 32
201 static int amountpipes
;
202 static int amountpipekva
;
204 #ifndef PIPE_NODIRECT
205 static int amountpipekvawired
;
207 int maxpipekva
= 1024 * 1024 * 16;
210 SYSCTL_DECL(_kern_ipc
);
212 SYSCTL_INT(_kern_ipc
, OID_AUTO
, maxpipekva
, CTLFLAG_RD
,
213 &maxpipekva
, 0, "Pipe KVA limit");
214 SYSCTL_INT(_kern_ipc
, OID_AUTO
, maxpipekvawired
, CTLFLAG_RW
,
215 &maxpipekvawired
, 0, "Pipe KVA wired limit");
216 SYSCTL_INT(_kern_ipc
, OID_AUTO
, pipes
, CTLFLAG_RD
,
217 &amountpipes
, 0, "Current # of pipes");
218 SYSCTL_INT(_kern_ipc
, OID_AUTO
, bigpipes
, CTLFLAG_RD
,
219 &nbigpipe
, 0, "Current # of big pipes");
220 SYSCTL_INT(_kern_ipc
, OID_AUTO
, pipekva
, CTLFLAG_RD
,
221 &amountpipekva
, 0, "Pipe KVA usage");
222 SYSCTL_INT(_kern_ipc
, OID_AUTO
, pipekvawired
, CTLFLAG_RD
,
223 &amountpipekvawired
, 0, "Pipe wired KVA usage");
226 void pipeinit(void *dummy __unused
);
227 static void pipeclose(struct pipe
*cpipe
);
228 static void pipe_free_kmem(struct pipe
*cpipe
);
229 static int pipe_create(struct pipe
**cpipep
);
230 static void pipeselwakeup(struct pipe
*cpipe
, struct pipe
*spipe
);
231 static __inline
int pipelock(struct pipe
*cpipe
, int catch);
232 static __inline
void pipeunlock(struct pipe
*cpipe
);
234 #ifndef PIPE_NODIRECT
235 static int pipe_build_write_buffer(struct pipe
*wpipe
, struct uio
*uio
);
236 static void pipe_destroy_write_buffer(struct pipe
*wpipe
);
237 static int pipe_direct_write(struct pipe
*wpipe
, struct uio
*uio
);
238 static void pipe_clone_write_buffer(struct pipe
*wpipe
);
241 extern int postpipeevent(struct pipe
*, int);
242 extern void evpipefree(struct pipe
*cpipe
);
245 static int pipespace(struct pipe
*cpipe
, int size
);
247 static lck_grp_t
*pipe_mtx_grp
;
248 static lck_attr_t
*pipe_mtx_attr
;
249 static lck_grp_attr_t
*pipe_mtx_grp_attr
;
251 static zone_t pipe_zone
;
253 SYSINIT(vfs
, SI_SUB_VFS
, SI_ORDER_ANY
, pipeinit
, NULL
);
256 pipeinit(void *dummy __unused
)
258 pipe_zone
= (zone_t
)zinit(sizeof(struct pipe
), 8192 * sizeof(struct pipe
), 4096, "pipe zone");
261 * allocate lock group attribute and group for pipe mutexes
263 pipe_mtx_grp_attr
= lck_grp_attr_alloc_init();
264 pipe_mtx_grp
= lck_grp_alloc_init("pipe", pipe_mtx_grp_attr
);
267 * allocate the lock attribute for pipe mutexes
269 pipe_mtx_attr
= lck_attr_alloc_init();
275 * The pipe system call for the DTYPE_PIPE type of pipes
280 pipe(struct proc
*p
, __unused
struct pipe_args
*uap
, register_t
*retval
)
282 struct fileproc
*rf
, *wf
;
283 struct pipe
*rpipe
, *wpipe
;
287 if ((pmtx
= lck_mtx_alloc_init(pipe_mtx_grp
, pipe_mtx_attr
)) == NULL
)
290 rpipe
= wpipe
= NULL
;
291 if (pipe_create(&rpipe
) || pipe_create(&wpipe
)) {
296 * allocate the space for the normal I/O direction up
297 * front... we'll delay the allocation for the other
298 * direction until a write actually occurs (most
299 * likely it won't)...
301 * Reduce to 1/4th pipe size if we're over our global max.
303 if (amountpipekva
> maxpipekva
/ 2)
304 error
= pipespace(rpipe
, SMALL_PIPE_SIZE
);
306 error
= pipespace(rpipe
, PIPE_SIZE
);
310 #ifndef PIPE_NODIRECT
311 rpipe
->pipe_state
|= PIPE_DIRECTOK
;
312 wpipe
->pipe_state
|= PIPE_DIRECTOK
;
314 TAILQ_INIT(&rpipe
->pipe_evlist
);
315 TAILQ_INIT(&wpipe
->pipe_evlist
);
317 error
= falloc(p
, &rf
, &fd
);
324 * for now we'll create half-duplex
325 * pipes... this is what we've always
329 rf
->f_type
= DTYPE_PIPE
;
330 rf
->f_data
= (caddr_t
)rpipe
;
331 rf
->f_ops
= &pipeops
;
333 error
= falloc(p
, &wf
, &fd
);
335 fp_free(p
, retval
[0], rf
);
339 wf
->f_type
= DTYPE_PIPE
;
340 wf
->f_data
= (caddr_t
)wpipe
;
341 wf
->f_ops
= &pipeops
;
346 * XXXXXXXX SHOULD NOT HOLD FILE_LOCK() XXXXXXXXXXXX
348 * struct pipe represents a pipe endpoint. The MAC label is shared
349 * between the connected endpoints. As a result mac_init_pipe() and
350 * mac_create_pipe() should only be called on one of the endpoints
351 * after they have been connected.
353 mac_init_pipe(rpipe
);
354 mac_create_pipe(td
->td_ucred
, rpipe
);
357 *fdflags(p
, retval
[0]) &= ~UF_RESERVED
;
358 *fdflags(p
, retval
[1]) &= ~UF_RESERVED
;
359 fp_drop(p
, retval
[0], rf
, 1);
360 fp_drop(p
, retval
[1], wf
, 1);
363 rpipe
->pipe_peer
= wpipe
;
364 wpipe
->pipe_peer
= rpipe
;
366 rpipe
->pipe_mtxp
= wpipe
->pipe_mtxp
= pmtx
;
373 lck_mtx_free(pmtx
, pipe_mtx_grp
);
380 pipe_stat(struct pipe
*cpipe
, struct stat
*ub
)
391 error
= mac_check_pipe_stat(active_cred
, cpipe
);
396 if (cpipe
->pipe_buffer
.buffer
== 0) {
398 * must be stat'ing the write fd
400 cpipe
= cpipe
->pipe_peer
;
405 bzero(ub
, sizeof(*ub
));
406 ub
->st_mode
= S_IFIFO
| S_IRUSR
| S_IWUSR
| S_IRGRP
| S_IWGRP
;
407 ub
->st_blksize
= cpipe
->pipe_buffer
.size
;
408 ub
->st_size
= cpipe
->pipe_buffer
.cnt
;
409 ub
->st_blocks
= (ub
->st_size
+ ub
->st_blksize
- 1) / ub
->st_blksize
;
412 ub
->st_uid
= kauth_getuid();
413 ub
->st_gid
= kauth_getgid();
416 ub
->st_atimespec
.tv_sec
= now
.tv_sec
;
417 ub
->st_atimespec
.tv_nsec
= now
.tv_usec
* 1000;
419 ub
->st_mtimespec
.tv_sec
= now
.tv_sec
;
420 ub
->st_mtimespec
.tv_nsec
= now
.tv_usec
* 1000;
422 ub
->st_ctimespec
.tv_sec
= now
.tv_sec
;
423 ub
->st_ctimespec
.tv_nsec
= now
.tv_usec
* 1000;
426 * Left as 0: st_dev, st_ino, st_nlink, st_rdev, st_flags, st_gen, st_uid, st_gid.
427 * XXX (st_dev, st_ino) should be unique.
434 * Allocate kva for pipe circular buffer, the space is pageable
435 * This routine will 'realloc' the size of a pipe safely, if it fails
436 * it will retain the old buffer.
437 * If it fails it will return ENOMEM.
440 pipespace(struct pipe
*cpipe
, int size
)
444 size
= round_page(size
);
446 if (kmem_alloc(kernel_map
, &buffer
, size
) != KERN_SUCCESS
)
449 /* free old resources if we're resizing */
450 pipe_free_kmem(cpipe
);
451 cpipe
->pipe_buffer
.buffer
= (caddr_t
)buffer
;
452 cpipe
->pipe_buffer
.size
= size
;
453 cpipe
->pipe_buffer
.in
= 0;
454 cpipe
->pipe_buffer
.out
= 0;
455 cpipe
->pipe_buffer
.cnt
= 0;
457 OSAddAtomic(1, (SInt32
*)&amountpipes
);
458 OSAddAtomic(cpipe
->pipe_buffer
.size
, (SInt32
*)&amountpipekva
);
464 * initialize and allocate VM and memory for pipe
467 pipe_create(struct pipe
**cpipep
)
471 cpipe
= (struct pipe
*)zalloc(pipe_zone
);
473 if ((*cpipep
= cpipe
) == NULL
)
477 * protect so pipespace or pipeclose don't follow a junk pointer
478 * if pipespace() fails.
480 bzero(cpipe
, sizeof *cpipe
);
487 * lock a pipe for I/O, blocking other access
490 pipelock(cpipe
, catch)
496 while (cpipe
->pipe_state
& PIPE_LOCKFL
) {
497 cpipe
->pipe_state
|= PIPE_LWANT
;
499 error
= msleep(cpipe
, PIPE_MTX(cpipe
), catch ? (PRIBIO
| PCATCH
) : PRIBIO
,
504 cpipe
->pipe_state
|= PIPE_LOCKFL
;
510 * unlock a pipe I/O lock
517 cpipe
->pipe_state
&= ~PIPE_LOCKFL
;
519 if (cpipe
->pipe_state
& PIPE_LWANT
) {
520 cpipe
->pipe_state
&= ~PIPE_LWANT
;
526 pipeselwakeup(cpipe
, spipe
)
531 if (cpipe
->pipe_state
& PIPE_SEL
) {
532 cpipe
->pipe_state
&= ~PIPE_SEL
;
533 selwakeup(&cpipe
->pipe_sel
);
535 if (cpipe
->pipe_state
& PIPE_KNOTE
)
536 KNOTE(&cpipe
->pipe_sel
.si_note
, 1);
538 postpipeevent(cpipe
, EV_RWBYTES
);
540 if (spipe
&& (spipe
->pipe_state
& PIPE_ASYNC
) && spipe
->pipe_pgid
) {
543 if (spipe
->pipe_pgid
< 0)
544 gsignal(-spipe
->pipe_pgid
, SIGIO
);
545 else if ((p
= pfind(spipe
->pipe_pgid
)) != (struct proc
*)0)
552 pipe_read(struct fileproc
*fp
, struct uio
*uio
, __unused kauth_cred_t active_cred
, __unused
int flags
, __unused
struct proc
*p
)
554 struct pipe
*rpipe
= (struct pipe
*)fp
->f_data
;
562 error
= pipelock(rpipe
, 1);
567 error
= mac_check_pipe_read(active_cred
, rpipe
);
572 while (uio_resid(uio
)) {
574 * normal pipe buffer receive
576 if (rpipe
->pipe_buffer
.cnt
> 0) {
577 size
= rpipe
->pipe_buffer
.size
- rpipe
->pipe_buffer
.out
;
578 if (size
> rpipe
->pipe_buffer
.cnt
)
579 size
= rpipe
->pipe_buffer
.cnt
;
580 // LP64todo - fix this!
581 if (size
> (u_int
) uio_resid(uio
))
582 size
= (u_int
) uio_resid(uio
);
586 &rpipe
->pipe_buffer
.buffer
[rpipe
->pipe_buffer
.out
],
592 rpipe
->pipe_buffer
.out
+= size
;
593 if (rpipe
->pipe_buffer
.out
>= rpipe
->pipe_buffer
.size
)
594 rpipe
->pipe_buffer
.out
= 0;
596 rpipe
->pipe_buffer
.cnt
-= size
;
599 * If there is no more to read in the pipe, reset
600 * its pointers to the beginning. This improves
603 if (rpipe
->pipe_buffer
.cnt
== 0) {
604 rpipe
->pipe_buffer
.in
= 0;
605 rpipe
->pipe_buffer
.out
= 0;
608 #ifndef PIPE_NODIRECT
610 * Direct copy, bypassing a kernel buffer.
612 } else if ((size
= rpipe
->pipe_map
.cnt
) &&
613 (rpipe
->pipe_state
& PIPE_DIRECTW
)) {
615 // LP64todo - fix this!
616 if (size
> (u_int
) uio_resid(uio
))
617 size
= (u_int
) uio_resid(uio
);
619 va
= (caddr_t
) rpipe
->pipe_map
.kva
+
622 error
= uiomove(va
, size
, uio
);
627 rpipe
->pipe_map
.pos
+= size
;
628 rpipe
->pipe_map
.cnt
-= size
;
629 if (rpipe
->pipe_map
.cnt
== 0) {
630 rpipe
->pipe_state
&= ~PIPE_DIRECTW
;
636 * detect EOF condition
637 * read returns 0 on EOF, no need to set error
639 if (rpipe
->pipe_state
& PIPE_EOF
)
643 * If the "write-side" has been blocked, wake it up now.
645 if (rpipe
->pipe_state
& PIPE_WANTW
) {
646 rpipe
->pipe_state
&= ~PIPE_WANTW
;
651 * Break if some data was read.
657 * Unlock the pipe buffer for our remaining processing.
658 * We will either break out with an error or we will
659 * sleep and relock to loop.
664 * Handle non-blocking mode operation or
665 * wait for more data.
667 if (fp
->f_flag
& FNONBLOCK
) {
670 rpipe
->pipe_state
|= PIPE_WANTR
;
672 error
= msleep(rpipe
, PIPE_MTX(rpipe
), PRIBIO
| PCATCH
, "piperd", 0);
675 error
= pipelock(rpipe
, 1);
690 * PIPE_WANT processing only makes sense if pipe_busy is 0.
692 if ((rpipe
->pipe_busy
== 0) && (rpipe
->pipe_state
& PIPE_WANT
)) {
693 rpipe
->pipe_state
&= ~(PIPE_WANT
|PIPE_WANTW
);
695 } else if (rpipe
->pipe_buffer
.cnt
< MINPIPESIZE
) {
697 * Handle write blocking hysteresis.
699 if (rpipe
->pipe_state
& PIPE_WANTW
) {
700 rpipe
->pipe_state
&= ~PIPE_WANTW
;
705 if ((rpipe
->pipe_buffer
.size
- rpipe
->pipe_buffer
.cnt
) >= PIPE_BUF
)
706 pipeselwakeup(rpipe
, rpipe
->pipe_peer
);
715 #ifndef PIPE_NODIRECT
717 * Map the sending processes' buffer into kernel space and wire it.
718 * This is similar to a physical write operation.
721 pipe_build_write_buffer(wpipe
, uio
)
728 vm_offset_t addr
, endaddr
;
731 size
= (u_int
) uio
->uio_iov
->iov_len
;
732 if (size
> wpipe
->pipe_buffer
.size
)
733 size
= wpipe
->pipe_buffer
.size
;
735 pmap
= vmspace_pmap(curproc
->p_vmspace
);
736 endaddr
= round_page((vm_offset_t
)uio
->uio_iov
->iov_base
+ size
);
737 addr
= trunc_page((vm_offset_t
)uio
->uio_iov
->iov_base
);
738 for (i
= 0; addr
< endaddr
; addr
+= PAGE_SIZE
, i
++) {
740 * vm_fault_quick() can sleep. Consequently,
741 * vm_page_lock_queue() and vm_page_unlock_queue()
742 * should not be performed outside of this loop.
745 if (vm_fault_quick((caddr_t
)addr
, VM_PROT_READ
) < 0) {
746 vm_page_lock_queues();
747 for (j
= 0; j
< i
; j
++)
748 vm_page_unhold(wpipe
->pipe_map
.ms
[j
]);
749 vm_page_unlock_queues();
752 wpipe
->pipe_map
.ms
[i
] = pmap_extract_and_hold(pmap
, addr
,
754 if (wpipe
->pipe_map
.ms
[i
] == NULL
)
759 * set up the control block
761 wpipe
->pipe_map
.npages
= i
;
762 wpipe
->pipe_map
.pos
=
763 ((vm_offset_t
) uio
->uio_iov
->iov_base
) & PAGE_MASK
;
764 wpipe
->pipe_map
.cnt
= size
;
769 if (wpipe
->pipe_map
.kva
== 0) {
771 * We need to allocate space for an extra page because the
772 * address range might (will) span pages at times.
774 wpipe
->pipe_map
.kva
= kmem_alloc_nofault(kernel_map
,
775 wpipe
->pipe_buffer
.size
+ PAGE_SIZE
);
776 atomic_add_int(&amountpipekvawired
,
777 wpipe
->pipe_buffer
.size
+ PAGE_SIZE
);
779 pmap_qenter(wpipe
->pipe_map
.kva
, wpipe
->pipe_map
.ms
,
780 wpipe
->pipe_map
.npages
);
783 * and update the uio data
786 uio
->uio_iov
->iov_len
-= size
;
787 uio
->uio_iov
->iov_base
= (char *)uio
->uio_iov
->iov_base
+ size
;
788 if (uio
->uio_iov
->iov_len
== 0)
790 uio_setresid(uio
, (uio_resid(uio
) - size
));
791 uio
->uio_offset
+= size
;
796 * unmap and unwire the process buffer
799 pipe_destroy_write_buffer(wpipe
)
804 if (wpipe
->pipe_map
.kva
) {
805 pmap_qremove(wpipe
->pipe_map
.kva
, wpipe
->pipe_map
.npages
);
807 if (amountpipekvawired
> maxpipekvawired
/ 2) {
808 /* Conserve address space */
809 vm_offset_t kva
= wpipe
->pipe_map
.kva
;
810 wpipe
->pipe_map
.kva
= 0;
811 kmem_free(kernel_map
, kva
,
812 wpipe
->pipe_buffer
.size
+ PAGE_SIZE
);
813 atomic_subtract_int(&amountpipekvawired
,
814 wpipe
->pipe_buffer
.size
+ PAGE_SIZE
);
817 vm_page_lock_queues();
818 for (i
= 0; i
< wpipe
->pipe_map
.npages
; i
++) {
819 vm_page_unhold(wpipe
->pipe_map
.ms
[i
]);
821 vm_page_unlock_queues();
822 wpipe
->pipe_map
.npages
= 0;
826 * In the case of a signal, the writing process might go away. This
827 * code copies the data into the circular buffer so that the source
828 * pages can be freed without loss of data.
831 pipe_clone_write_buffer(wpipe
)
837 size
= wpipe
->pipe_map
.cnt
;
838 pos
= wpipe
->pipe_map
.pos
;
840 wpipe
->pipe_buffer
.in
= size
;
841 wpipe
->pipe_buffer
.out
= 0;
842 wpipe
->pipe_buffer
.cnt
= size
;
843 wpipe
->pipe_state
&= ~PIPE_DIRECTW
;
846 bcopy((caddr_t
) wpipe
->pipe_map
.kva
+ pos
,
847 wpipe
->pipe_buffer
.buffer
, size
);
848 pipe_destroy_write_buffer(wpipe
);
853 * This implements the pipe buffer write mechanism. Note that only
854 * a direct write OR a normal pipe write can be pending at any given time.
855 * If there are any characters in the pipe buffer, the direct write will
856 * be deferred until the receiving process grabs all of the bytes from
857 * the pipe buffer. Then the direct mapping write is set-up.
860 pipe_direct_write(wpipe
, uio
)
867 while (wpipe
->pipe_state
& PIPE_DIRECTW
) {
868 if (wpipe
->pipe_state
& PIPE_WANTR
) {
869 wpipe
->pipe_state
&= ~PIPE_WANTR
;
872 wpipe
->pipe_state
|= PIPE_WANTW
;
873 error
= msleep(wpipe
, PIPE_MTX(wpipe
),
874 PRIBIO
| PCATCH
, "pipdww", 0);
877 if (wpipe
->pipe_state
& PIPE_EOF
) {
882 wpipe
->pipe_map
.cnt
= 0; /* transfer not ready yet */
883 if (wpipe
->pipe_buffer
.cnt
> 0) {
884 if (wpipe
->pipe_state
& PIPE_WANTR
) {
885 wpipe
->pipe_state
&= ~PIPE_WANTR
;
889 wpipe
->pipe_state
|= PIPE_WANTW
;
890 error
= msleep(wpipe
, PIPE_MTX(wpipe
),
891 PRIBIO
| PCATCH
, "pipdwc", 0);
894 if (wpipe
->pipe_state
& PIPE_EOF
) {
901 wpipe
->pipe_state
|= PIPE_DIRECTW
;
905 error
= pipe_build_write_buffer(wpipe
, uio
);
909 wpipe
->pipe_state
&= ~PIPE_DIRECTW
;
914 while (!error
&& (wpipe
->pipe_state
& PIPE_DIRECTW
)) {
915 if (wpipe
->pipe_state
& PIPE_EOF
) {
918 pipe_destroy_write_buffer(wpipe
);
920 pipeselwakeup(wpipe
, wpipe
);
925 if (wpipe
->pipe_state
& PIPE_WANTR
) {
926 wpipe
->pipe_state
&= ~PIPE_WANTR
;
929 pipeselwakeup(wpipe
, wpipe
);
930 error
= msleep(wpipe
, PIPE_MTX(wpipe
), PRIBIO
| PCATCH
,
935 if (wpipe
->pipe_state
& PIPE_DIRECTW
) {
937 * this bit of trickery substitutes a kernel buffer for
938 * the process that might be going away.
940 pipe_clone_write_buffer(wpipe
);
943 pipe_destroy_write_buffer(wpipe
);
958 pipe_write(struct fileproc
*fp
, struct uio
*uio
, __unused kauth_cred_t active_cred
, __unused
int flags
, __unused
struct proc
*p
)
963 struct pipe
*wpipe
, *rpipe
;
965 rpipe
= (struct pipe
*)fp
->f_data
;
968 wpipe
= rpipe
->pipe_peer
;
971 * detect loss of pipe read side, issue SIGPIPE if lost.
973 if (wpipe
== NULL
|| (wpipe
->pipe_state
& PIPE_EOF
)) {
978 error
= mac_check_pipe_write(active_cred
, wpipe
);
988 if (wpipe
->pipe_buffer
.buffer
== 0) {
990 * need to allocate some storage... we delay the allocation
991 * until the first write on fd[0] to avoid allocating storage for both
992 * 'pipe ends'... most pipes are half-duplex with the writes targeting
993 * fd[1], so allocating space for both ends is a waste...
995 * Reduce to 1/4th pipe size if we're over our global max.
997 if (amountpipekva
> maxpipekva
/ 2)
998 pipe_size
= SMALL_PIPE_SIZE
;
1000 pipe_size
= PIPE_SIZE
;
1004 * If it is advantageous to resize the pipe buffer, do
1007 if ((uio_resid(uio
) > PIPE_SIZE
) &&
1008 (wpipe
->pipe_buffer
.size
<= PIPE_SIZE
) &&
1009 (amountpipekva
< maxpipekva
/ 2) &&
1010 (nbigpipe
< LIMITBIGPIPES
) &&
1011 #ifndef PIPE_NODIRECT
1012 (wpipe
->pipe_state
& PIPE_DIRECTW
) == 0 &&
1014 (wpipe
->pipe_buffer
.cnt
== 0)) {
1016 pipe_size
= BIG_PIPE_SIZE
;
1021 * need to do initial allocation or resizing of pipe
1023 if ((error
= pipelock(wpipe
, 1)) == 0) {
1025 if (pipespace(wpipe
, pipe_size
) == 0)
1026 OSAddAtomic(1, (SInt32
*)&nbigpipe
);
1030 if (wpipe
->pipe_buffer
.buffer
== 0) {
1032 * initial allocation failed
1039 * If an error occurred unbusy and return, waking up any pending
1043 if ((wpipe
->pipe_busy
== 0) &&
1044 (wpipe
->pipe_state
& PIPE_WANT
)) {
1045 wpipe
->pipe_state
&= ~(PIPE_WANT
| PIPE_WANTR
);
1052 // LP64todo - fix this!
1053 orig_resid
= uio_resid(uio
);
1055 while (uio_resid(uio
)) {
1058 #ifndef PIPE_NODIRECT
1060 * If the transfer is large, we can gain performance if
1061 * we do process-to-process copies directly.
1062 * If the write is non-blocking, we don't use the
1063 * direct write mechanism.
1065 * The direct write mechanism will detect the reader going
1068 if ((uio
->uio_iov
->iov_len
>= PIPE_MINDIRECT
) &&
1069 (fp
->f_flag
& FNONBLOCK
) == 0 &&
1070 amountpipekvawired
+ uio
->uio_resid
< maxpipekvawired
) {
1071 error
= pipe_direct_write(wpipe
, uio
);
1078 * Pipe buffered writes cannot be coincidental with
1079 * direct writes. We wait until the currently executing
1080 * direct write is completed before we start filling the
1081 * pipe buffer. We break out if a signal occurs or the
1085 while (wpipe
->pipe_state
& PIPE_DIRECTW
) {
1086 if (wpipe
->pipe_state
& PIPE_WANTR
) {
1087 wpipe
->pipe_state
&= ~PIPE_WANTR
;
1090 error
= msleep(wpipe
, PIPE_MTX(wpipe
), PRIBIO
| PCATCH
, "pipbww", 0);
1092 if (wpipe
->pipe_state
& PIPE_EOF
)
1100 space
= wpipe
->pipe_buffer
.size
- wpipe
->pipe_buffer
.cnt
;
1103 * Writes of size <= PIPE_BUF must be atomic.
1105 if ((space
< uio_resid(uio
)) && (orig_resid
<= PIPE_BUF
))
1110 if ((error
= pipelock(wpipe
,1)) == 0) {
1111 int size
; /* Transfer size */
1112 int segsize
; /* first segment to transfer */
1114 if (wpipe
->pipe_state
& PIPE_EOF
) {
1119 #ifndef PIPE_NODIRECT
1121 * It is possible for a direct write to
1122 * slip in on us... handle it here...
1124 if (wpipe
->pipe_state
& PIPE_DIRECTW
) {
1130 * If a process blocked in pipelock, our
1131 * value for space might be bad... the mutex
1132 * is dropped while we're blocked
1134 if (space
> (int)(wpipe
->pipe_buffer
.size
-
1135 wpipe
->pipe_buffer
.cnt
)) {
1141 * Transfer size is minimum of uio transfer
1142 * and free space in pipe buffer.
1144 // LP64todo - fix this!
1145 if (space
> uio_resid(uio
))
1146 size
= uio_resid(uio
);
1150 * First segment to transfer is minimum of
1151 * transfer size and contiguous space in
1152 * pipe buffer. If first segment to transfer
1153 * is less than the transfer size, we've got
1154 * a wraparound in the buffer.
1156 segsize
= wpipe
->pipe_buffer
.size
-
1157 wpipe
->pipe_buffer
.in
;
1161 /* Transfer first segment */
1164 error
= uiomove(&wpipe
->pipe_buffer
.buffer
[wpipe
->pipe_buffer
.in
],
1168 if (error
== 0 && segsize
< size
) {
1170 * Transfer remaining part now, to
1171 * support atomic writes. Wraparound
1174 if (wpipe
->pipe_buffer
.in
+ segsize
!=
1175 wpipe
->pipe_buffer
.size
)
1176 panic("Expected pipe buffer "
1177 "wraparound disappeared");
1181 &wpipe
->pipe_buffer
.buffer
[0],
1182 size
- segsize
, uio
);
1186 wpipe
->pipe_buffer
.in
+= size
;
1187 if (wpipe
->pipe_buffer
.in
>=
1188 wpipe
->pipe_buffer
.size
) {
1189 if (wpipe
->pipe_buffer
.in
!=
1191 wpipe
->pipe_buffer
.size
)
1194 wpipe
->pipe_buffer
.in
= size
-
1198 wpipe
->pipe_buffer
.cnt
+= size
;
1199 if (wpipe
->pipe_buffer
.cnt
>
1200 wpipe
->pipe_buffer
.size
)
1201 panic("Pipe buffer overflow");
1211 * If the "read-side" has been blocked, wake it up now.
1213 if (wpipe
->pipe_state
& PIPE_WANTR
) {
1214 wpipe
->pipe_state
&= ~PIPE_WANTR
;
1218 * don't block on non-blocking I/O
1219 * we'll do the pipeselwakeup on the way out
1221 if (fp
->f_flag
& FNONBLOCK
) {
1226 * We have no more space and have something to offer,
1227 * wake up select/poll.
1229 pipeselwakeup(wpipe
, wpipe
);
1231 wpipe
->pipe_state
|= PIPE_WANTW
;
1233 error
= msleep(wpipe
, PIPE_MTX(wpipe
), PRIBIO
| PCATCH
, "pipewr", 0);
1238 * If read side wants to go away, we just issue a signal
1241 if (wpipe
->pipe_state
& PIPE_EOF
) {
1249 if ((wpipe
->pipe_busy
== 0) && (wpipe
->pipe_state
& PIPE_WANT
)) {
1250 wpipe
->pipe_state
&= ~(PIPE_WANT
| PIPE_WANTR
);
1253 if (wpipe
->pipe_buffer
.cnt
> 0) {
1255 * If there are any characters in the buffer, we wake up
1256 * the reader if it was blocked waiting for data.
1258 if (wpipe
->pipe_state
& PIPE_WANTR
) {
1259 wpipe
->pipe_state
&= ~PIPE_WANTR
;
1263 * wake up thread blocked in select/poll or post the notification
1265 pipeselwakeup(wpipe
, wpipe
);
1273 * we implement a very minimal set of ioctls for compatibility with sockets.
1277 pipe_ioctl(struct fileproc
*fp
, u_long cmd
, caddr_t data
, __unused
struct proc
*p
)
1279 struct pipe
*mpipe
= (struct pipe
*)fp
->f_data
;
1287 error
= mac_check_pipe_ioctl(active_cred
, mpipe
, cmd
, data
);
1303 mpipe
->pipe_state
|= PIPE_ASYNC
;
1305 mpipe
->pipe_state
&= ~PIPE_ASYNC
;
1311 #ifndef PIPE_NODIRECT
1312 if (mpipe
->pipe_state
& PIPE_DIRECTW
)
1313 *(int *)data
= mpipe
->pipe_map
.cnt
;
1316 *(int *)data
= mpipe
->pipe_buffer
.cnt
;
1321 mpipe
->pipe_pgid
= *(int *)data
;
1327 *(int *)data
= mpipe
->pipe_pgid
;
1339 pipe_select(struct fileproc
*fp
, int which
, void *wql
, struct proc
*p
)
1341 struct pipe
*rpipe
= (struct pipe
*)fp
->f_data
;
1345 if (rpipe
== NULL
|| rpipe
== (struct pipe
*)-1)
1350 wpipe
= rpipe
->pipe_peer
;
1355 if ((rpipe
->pipe_state
& PIPE_DIRECTW
) ||
1356 (rpipe
->pipe_buffer
.cnt
> 0) ||
1357 (rpipe
->pipe_state
& PIPE_EOF
)) {
1361 rpipe
->pipe_state
|= PIPE_SEL
;
1362 selrecord(p
, &rpipe
->pipe_sel
, wql
);
1367 if (wpipe
== NULL
|| (wpipe
->pipe_state
& PIPE_EOF
) ||
1368 (((wpipe
->pipe_state
& PIPE_DIRECTW
) == 0) &&
1369 (wpipe
->pipe_buffer
.size
- wpipe
->pipe_buffer
.cnt
) >= PIPE_BUF
)) {
1373 wpipe
->pipe_state
|= PIPE_SEL
;
1374 selrecord(p
, &wpipe
->pipe_sel
, wql
);
1378 rpipe
->pipe_state
|= PIPE_SEL
;
1379 selrecord(p
, &rpipe
->pipe_sel
, wql
);
1390 pipe_close(struct fileglob
*fg
, __unused
struct proc
*p
)
1395 cpipe
= (struct pipe
*)fg
->fg_data
;
1406 pipe_free_kmem(struct pipe
*cpipe
)
1409 if (cpipe
->pipe_buffer
.buffer
!= NULL
) {
1410 if (cpipe
->pipe_buffer
.size
> PIPE_SIZE
)
1411 OSAddAtomic(-1, (SInt32
*)&nbigpipe
);
1412 OSAddAtomic(-(cpipe
->pipe_buffer
.size
), (SInt32
*)&amountpipekva
);
1413 OSAddAtomic(-1, (SInt32
*)&amountpipes
);
1415 kmem_free(kernel_map
, (vm_offset_t
)cpipe
->pipe_buffer
.buffer
,
1416 cpipe
->pipe_buffer
.size
);
1417 cpipe
->pipe_buffer
.buffer
= NULL
;
1419 #ifndef PIPE_NODIRECT
1420 if (cpipe
->pipe_map
.kva
!= 0) {
1421 atomic_subtract_int(&amountpipekvawired
,
1422 cpipe
->pipe_buffer
.size
+ PAGE_SIZE
);
1423 kmem_free(kernel_map
,
1424 cpipe
->pipe_map
.kva
,
1425 cpipe
->pipe_buffer
.size
+ PAGE_SIZE
);
1426 cpipe
->pipe_map
.cnt
= 0;
1427 cpipe
->pipe_map
.kva
= 0;
1428 cpipe
->pipe_map
.pos
= 0;
1429 cpipe
->pipe_map
.npages
= 0;
1438 pipeclose(struct pipe
*cpipe
)
1445 /* partially created pipes won't have a valid mutex. */
1446 if (PIPE_MTX(cpipe
) != NULL
)
1449 pipeselwakeup(cpipe
, cpipe
);
1452 * If the other side is blocked, wake it up saying that
1453 * we want to close it down.
1455 while (cpipe
->pipe_busy
) {
1456 cpipe
->pipe_state
|= PIPE_WANT
| PIPE_EOF
;
1460 msleep(cpipe
, PIPE_MTX(cpipe
), PRIBIO
, "pipecl", 0);
1464 if (cpipe
->pipe_label
!= NULL
&& cpipe
->pipe_peer
== NULL
)
1465 mac_destroy_pipe(cpipe
);
1469 * Disconnect from peer
1471 if ((ppipe
= cpipe
->pipe_peer
) != NULL
) {
1473 ppipe
->pipe_state
|= PIPE_EOF
;
1475 pipeselwakeup(ppipe
, ppipe
);
1478 if (cpipe
->pipe_state
& PIPE_KNOTE
)
1479 KNOTE(&ppipe
->pipe_sel
.si_note
, 1);
1481 postpipeevent(ppipe
, EV_RCLOSED
);
1483 ppipe
->pipe_peer
= NULL
;
1490 if (PIPE_MTX(cpipe
) != NULL
) {
1491 if (ppipe
!= NULL
) {
1493 * since the mutex is shared and the peer is still
1494 * alive, we need to release the mutex, not free it
1499 * peer is gone, so we're the sole party left with
1500 * interest in this mutex... we can just free it
1502 lck_mtx_free(PIPE_MTX(cpipe
), pipe_mtx_grp
);
1505 pipe_free_kmem(cpipe
);
1507 zfree(pipe_zone
, cpipe
);
1513 pipe_kqfilter(__unused
struct fileproc
*fp
, struct knote
*kn
, __unused
struct proc
*p
)
1517 cpipe
= (struct pipe
*)kn
->kn_fp
->f_data
;
1521 switch (kn
->kn_filter
) {
1523 kn
->kn_fop
= &pipe_rfiltops
;
1526 kn
->kn_fop
= &pipe_wfiltops
;
1528 if (cpipe
->pipe_peer
== NULL
) {
1530 * other end of pipe has been closed
1535 cpipe
= cpipe
->pipe_peer
;
1542 if (KNOTE_ATTACH(&cpipe
->pipe_sel
.si_note
, kn
))
1543 cpipe
->pipe_state
|= PIPE_KNOTE
;
1550 filt_pipedetach(struct knote
*kn
)
1552 struct pipe
*cpipe
= (struct pipe
*)kn
->kn_fp
->f_data
;
1556 if (kn
->kn_filter
== EVFILT_WRITE
) {
1557 if (cpipe
->pipe_peer
== NULL
) {
1561 cpipe
= cpipe
->pipe_peer
;
1563 if (cpipe
->pipe_state
& PIPE_KNOTE
) {
1564 if (KNOTE_DETACH(&cpipe
->pipe_sel
.si_note
, kn
))
1565 cpipe
->pipe_state
&= ~PIPE_KNOTE
;
1572 filt_piperead(struct knote
*kn
, long hint
)
1574 struct pipe
*rpipe
= (struct pipe
*)kn
->kn_fp
->f_data
;
1579 * if hint == 0, then we've been called from the kevent
1580 * world directly and do not currently hold the pipe mutex...
1581 * if hint == 1, we're being called back via the KNOTE post
1582 * we made in pipeselwakeup, and we already hold the mutex...
1587 wpipe
= rpipe
->pipe_peer
;
1588 kn
->kn_data
= rpipe
->pipe_buffer
.cnt
;
1590 #ifndef PIPE_NODIRECT
1591 if ((kn
->kn_data
== 0) && (rpipe
->pipe_state
& PIPE_DIRECTW
))
1592 kn
->kn_data
= rpipe
->pipe_map
.cnt
;
1594 if ((rpipe
->pipe_state
& PIPE_EOF
) ||
1595 (wpipe
== NULL
) || (wpipe
->pipe_state
& PIPE_EOF
)) {
1596 kn
->kn_flags
|= EV_EOF
;
1599 retval
= (kn
->kn_sfflags
& NOTE_LOWAT
) ?
1600 (kn
->kn_data
>= kn
->kn_sdata
) : (kn
->kn_data
> 0);
1610 filt_pipewrite(struct knote
*kn
, long hint
)
1612 struct pipe
*rpipe
= (struct pipe
*)kn
->kn_fp
->f_data
;
1616 * if hint == 0, then we've been called from the kevent
1617 * world directly and do not currently hold the pipe mutex...
1618 * if hint == 1, we're being called back via the KNOTE post
1619 * we made in pipeselwakeup, and we already hold the mutex...
1624 wpipe
= rpipe
->pipe_peer
;
1626 if ((wpipe
== NULL
) || (wpipe
->pipe_state
& PIPE_EOF
)) {
1628 kn
->kn_flags
|= EV_EOF
;
1634 kn
->kn_data
= wpipe
->pipe_buffer
.size
- wpipe
->pipe_buffer
.cnt
;
1636 #ifndef PIPE_NODIRECT
1637 if (wpipe
->pipe_state
& PIPE_DIRECTW
)
1643 return (kn
->kn_data
>= ((kn
->kn_sfflags
& NOTE_LOWAT
) ?
1644 kn
->kn_sdata
: PIPE_BUF
));
1648 fill_pipeinfo(struct pipe
* cpipe
, struct pipe_info
* pinfo
)
1660 error
= mac_check_pipe_stat(active_cred
, cpipe
);
1665 if (cpipe
->pipe_buffer
.buffer
== 0) {
1667 * must be stat'ing the write fd
1669 cpipe
= cpipe
->pipe_peer
;
1675 ub
= &pinfo
->pipe_stat
;
1677 bzero(ub
, sizeof(*ub
));
1678 ub
->st_mode
= S_IFIFO
| S_IRUSR
| S_IWUSR
| S_IRGRP
| S_IWGRP
;
1679 ub
->st_blksize
= cpipe
->pipe_buffer
.size
;
1680 ub
->st_size
= cpipe
->pipe_buffer
.cnt
;
1681 if (ub
->st_blksize
!= 0);
1682 ub
->st_blocks
= (ub
->st_size
+ ub
->st_blksize
- 1) / ub
->st_blksize
;
1685 ub
->st_uid
= kauth_getuid();
1686 ub
->st_gid
= kauth_getgid();
1689 ub
->st_atimespec
.tv_sec
= now
.tv_sec
;
1690 ub
->st_atimespec
.tv_nsec
= now
.tv_usec
* 1000;
1692 ub
->st_mtimespec
.tv_sec
= now
.tv_sec
;
1693 ub
->st_mtimespec
.tv_nsec
= now
.tv_usec
* 1000;
1695 ub
->st_ctimespec
.tv_sec
= now
.tv_sec
;
1696 ub
->st_ctimespec
.tv_nsec
= now
.tv_usec
* 1000;
1699 * Left as 0: st_dev, st_ino, st_nlink, st_rdev, st_flags, st_gen, st_uid, st_gid.
1700 * XXX (st_dev, st_ino) should be unique.
1703 pinfo
->pipe_handle
= (uint64_t)((uintptr_t)cpipe
);
1704 pinfo
->pipe_peerhandle
= (uint64_t)((uintptr_t)(cpipe
->pipe_peer
));
1705 pinfo
->pipe_status
= cpipe
->pipe_state
;