2 * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
4 * @APPLE_LICENSE_HEADER_START@
6 * The contents of this file constitute Original Code as defined in and
7 * are subject to the Apple Public Source License Version 1.1 (the
8 * "License"). You may not use this file except in compliance with the
9 * License. Please obtain a copy of the License at
10 * http://www.apple.com/publicsource and read it before using this file.
12 * This Original Code and all software distributed under the License are
13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17 * License for the specific language governing rights and limitations
20 * @APPLE_LICENSE_HEADER_END@
22 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
25 * The Regents of the University of California. All rights reserved.
27 * Redistribution and use in source and binary forms, with or without
28 * modification, are permitted provided that the following conditions
30 * 1. Redistributions of source code must retain the above copyright
31 * notice, this list of conditions and the following disclaimer.
32 * 2. Redistributions in binary form must reproduce the above copyright
33 * notice, this list of conditions and the following disclaimer in the
34 * documentation and/or other materials provided with the distribution.
35 * 3. All advertising materials mentioning features or use of this software
36 * must display the following acknowledgement:
37 * This product includes software developed by the University of
38 * California, Berkeley and its contributors.
39 * 4. Neither the name of the University nor the names of its contributors
40 * may be used to endorse or promote products derived from this software
41 * without specific prior written permission.
43 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
44 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
45 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
46 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
47 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
48 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
49 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
50 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
51 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
52 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
55 * @(#)vfs_cluster.c 8.10 (Berkeley) 3/28/95
58 #include <sys/param.h>
59 #include <sys/proc_internal.h>
60 #include <sys/buf_internal.h>
61 #include <sys/mount_internal.h>
62 #include <sys/vnode_internal.h>
63 #include <sys/trace.h>
64 #include <sys/malloc.h>
66 #include <sys/kernel.h>
67 #include <sys/resourcevar.h>
68 #include <sys/uio_internal.h>
69 #include <libkern/libkern.h>
70 #include <machine/machine_routines.h>
72 #include <sys/ubc_internal.h>
74 #include <mach/mach_types.h>
75 #include <mach/memory_object_types.h>
76 #include <mach/vm_map.h>
79 #include <vm/vm_kern.h>
80 #include <vm/vm_map.h>
81 #include <vm/vm_pageout.h>
83 #include <sys/kdebug.h>
88 #define CL_COMMIT 0x04
89 #define CL_PAGEOUT 0x10
92 #define CL_NOZERO 0x80
93 #define CL_PAGEIN 0x100
94 #define CL_DEV_MEMORY 0x200
95 #define CL_PRESERVE 0x400
96 #define CL_THROTTLE 0x800
97 #define CL_KEEPCACHED 0x1000
101 u_int io_completed
; /* amount of io that has currently completed */
102 u_int io_issued
; /* amount of io that was successfully issued */
103 int io_error
; /* error code of first error encountered */
104 int io_wanted
; /* someone is sleeping waiting for a change in state */
107 static lck_grp_t
*cl_mtx_grp
;
108 static lck_attr_t
*cl_mtx_attr
;
109 static lck_grp_attr_t
*cl_mtx_grp_attr
;
110 static lck_mtx_t
*cl_mtxp
;
113 static int cluster_io(vnode_t vp
, upl_t upl
, vm_offset_t upl_offset
, off_t f_offset
, int non_rounded_size
,
114 int flags
, buf_t real_bp
, struct clios
*iostate
);
115 static int cluster_iodone(buf_t bp
, void *dummy
);
116 static int cluster_rd_prefetch(vnode_t vp
, off_t f_offset
, u_int size
, off_t filesize
);
117 static int cluster_hard_throttle_on(vnode_t vp
);
119 static int cluster_read_x(vnode_t vp
, struct uio
*uio
, off_t filesize
, int flags
);
120 static int cluster_write_x(vnode_t vp
, struct uio
*uio
, off_t oldEOF
, off_t newEOF
,
121 off_t headOff
, off_t tailOff
, int flags
);
122 static int cluster_nocopy_read(vnode_t vp
, struct uio
*uio
, off_t filesize
);
123 static int cluster_nocopy_write(vnode_t vp
, struct uio
*uio
, off_t newEOF
);
124 static int cluster_phys_read(vnode_t vp
, struct uio
*uio
, off_t filesize
);
125 static int cluster_phys_write(vnode_t vp
, struct uio
*uio
, off_t newEOF
);
126 static int cluster_align_phys_io(vnode_t vp
, struct uio
*uio
, addr64_t usr_paddr
, int xsize
, int flags
);
128 static void cluster_rd_ahead(vnode_t vp
, struct cl_extent
*extent
, off_t filesize
, struct cl_readahead
*ra
);
130 static int cluster_push_x(vnode_t vp
, struct cl_extent
*, off_t EOF
, int flags
);
131 static void cluster_push_EOF(vnode_t vp
, off_t EOF
);
133 static int cluster_try_push(struct cl_writebehind
*, vnode_t vp
, off_t EOF
, int can_delay
, int push_all
);
135 static void sparse_cluster_switch(struct cl_writebehind
*, vnode_t vp
, off_t EOF
);
136 static void sparse_cluster_push(struct cl_writebehind
*, vnode_t vp
, off_t EOF
, int push_all
);
137 static void sparse_cluster_add(struct cl_writebehind
*, vnode_t vp
, struct cl_extent
*, off_t EOF
);
139 static kern_return_t
vfs_drt_mark_pages(void **cmapp
, off_t offset
, u_int length
, int *setcountp
);
140 static kern_return_t
vfs_drt_get_cluster(void **cmapp
, off_t
*offsetp
, u_int
*lengthp
);
141 static kern_return_t
vfs_drt_control(void **cmapp
, int op_type
);
143 int is_file_clean(vnode_t
, off_t
);
146 * throttle the number of async writes that
147 * can be outstanding on a single vnode
148 * before we issue a synchronous write
150 #define HARD_THROTTLE_MAXCNT 0
151 #define HARD_THROTTLE_MAXSIZE (64 * 1024)
153 int hard_throttle_on_root
= 0;
154 struct timeval priority_IO_timestamp_for_root
;
160 * allocate lock group attribute and group
162 cl_mtx_grp_attr
= lck_grp_attr_alloc_init();
163 cl_mtx_grp
= lck_grp_alloc_init("cluster I/O", cl_mtx_grp_attr
);
166 * allocate the lock attribute
168 cl_mtx_attr
= lck_attr_alloc_init();
171 * allocate and initialize mutex's used to protect updates and waits
172 * on the cluster_io context
174 cl_mtxp
= lck_mtx_alloc_init(cl_mtx_grp
, cl_mtx_attr
);
177 panic("cluster_init: failed to allocate cl_mtxp");
182 #define CLW_ALLOCATE 0x01
183 #define CLW_RETURNLOCKED 0x02
185 * if the read ahead context doesn't yet exist,
186 * allocate and initialize it...
187 * the vnode lock serializes multiple callers
188 * during the actual assignment... first one
189 * to grab the lock wins... the other callers
190 * will release the now unnecessary storage
192 * once the context is present, try to grab (but don't block on)
193 * the lock associated with it... if someone
194 * else currently owns it, than the read
195 * will run without read-ahead. this allows
196 * multiple readers to run in parallel and
197 * since there's only 1 read ahead context,
198 * there's no real loss in only allowing 1
199 * reader to have read-ahead enabled.
201 static struct cl_readahead
*
202 cluster_get_rap(vnode_t vp
)
204 struct ubc_info
*ubc
;
205 struct cl_readahead
*rap
;
209 if ((rap
= ubc
->cl_rahead
) == NULL
) {
210 MALLOC_ZONE(rap
, struct cl_readahead
*, sizeof *rap
, M_CLRDAHEAD
, M_WAITOK
);
212 bzero(rap
, sizeof *rap
);
214 lck_mtx_init(&rap
->cl_lockr
, cl_mtx_grp
, cl_mtx_attr
);
218 if (ubc
->cl_rahead
== NULL
)
219 ubc
->cl_rahead
= rap
;
221 lck_mtx_destroy(&rap
->cl_lockr
, cl_mtx_grp
);
222 FREE_ZONE((void *)rap
, sizeof *rap
, M_CLRDAHEAD
);
223 rap
= ubc
->cl_rahead
;
227 if (lck_mtx_try_lock(&rap
->cl_lockr
) == TRUE
)
230 return ((struct cl_readahead
*)NULL
);
235 * if the write behind context doesn't yet exist,
236 * and CLW_ALLOCATE is specified, allocate and initialize it...
237 * the vnode lock serializes multiple callers
238 * during the actual assignment... first one
239 * to grab the lock wins... the other callers
240 * will release the now unnecessary storage
242 * if CLW_RETURNLOCKED is set, grab (blocking if necessary)
243 * the lock associated with the write behind context before
247 static struct cl_writebehind
*
248 cluster_get_wbp(vnode_t vp
, int flags
)
250 struct ubc_info
*ubc
;
251 struct cl_writebehind
*wbp
;
255 if ((wbp
= ubc
->cl_wbehind
) == NULL
) {
257 if ( !(flags
& CLW_ALLOCATE
))
258 return ((struct cl_writebehind
*)NULL
);
260 MALLOC_ZONE(wbp
, struct cl_writebehind
*, sizeof *wbp
, M_CLWRBEHIND
, M_WAITOK
);
262 bzero(wbp
, sizeof *wbp
);
263 lck_mtx_init(&wbp
->cl_lockw
, cl_mtx_grp
, cl_mtx_attr
);
267 if (ubc
->cl_wbehind
== NULL
)
268 ubc
->cl_wbehind
= wbp
;
270 lck_mtx_destroy(&wbp
->cl_lockw
, cl_mtx_grp
);
271 FREE_ZONE((void *)wbp
, sizeof *wbp
, M_CLWRBEHIND
);
272 wbp
= ubc
->cl_wbehind
;
276 if (flags
& CLW_RETURNLOCKED
)
277 lck_mtx_lock(&wbp
->cl_lockw
);
284 cluster_hard_throttle_on(vnode_t vp
)
286 static struct timeval hard_throttle_maxelapsed
= { 0, 200000 };
288 if (vp
->v_mount
->mnt_kern_flag
& MNTK_ROOTDEV
) {
289 struct timeval elapsed
;
291 if (hard_throttle_on_root
)
294 microuptime(&elapsed
);
295 timevalsub(&elapsed
, &priority_IO_timestamp_for_root
);
297 if (timevalcmp(&elapsed
, &hard_throttle_maxelapsed
, <))
305 cluster_iodone(buf_t bp
, __unused
void *dummy
)
318 struct clios
*iostate
;
322 cbp_head
= (buf_t
)(bp
->b_trans_head
);
324 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 20)) | DBG_FUNC_START
,
325 (int)cbp_head
, bp
->b_lblkno
, bp
->b_bcount
, bp
->b_flags
, 0);
327 for (cbp
= cbp_head
; cbp
; cbp
= cbp
->b_trans_next
) {
329 * all I/O requests that are part of this transaction
330 * have to complete before we can process it
332 if ( !(cbp
->b_flags
& B_DONE
)) {
334 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 20)) | DBG_FUNC_END
,
335 (int)cbp_head
, (int)cbp
, cbp
->b_bcount
, cbp
->b_flags
, 0);
345 upl_offset
= cbp
->b_uploffset
;
347 b_flags
= cbp
->b_flags
;
348 real_bp
= cbp
->b_real_bp
;
349 zero_offset
= cbp
->b_validend
;
350 iostate
= (struct clios
*)cbp
->b_iostate
;
353 real_bp
->b_dev
= cbp
->b_dev
;
356 if ((cbp
->b_flags
& B_ERROR
) && error
== 0)
357 error
= cbp
->b_error
;
359 total_resid
+= cbp
->b_resid
;
360 total_size
+= cbp
->b_bcount
;
362 cbp_next
= cbp
->b_trans_next
;
369 cluster_zero(upl
, zero_offset
, PAGE_SIZE
- (zero_offset
& PAGE_MASK
), real_bp
);
375 * someone has issued multiple I/Os asynchrounsly
376 * and is waiting for them to complete (streaming)
378 lck_mtx_lock(cl_mtxp
);
380 if (error
&& iostate
->io_error
== 0)
381 iostate
->io_error
= error
;
383 iostate
->io_completed
+= total_size
;
385 if (iostate
->io_wanted
) {
387 * someone is waiting for the state of
388 * this io stream to change
390 iostate
->io_wanted
= 0;
393 lck_mtx_unlock(cl_mtxp
);
396 wakeup((caddr_t
)&iostate
->io_wanted
);
398 if ((b_flags
& B_NEED_IODONE
) && real_bp
) {
400 real_bp
->b_flags
|= B_ERROR
;
401 real_bp
->b_error
= error
;
403 real_bp
->b_resid
= total_resid
;
405 buf_biodone(real_bp
);
407 if (error
== 0 && total_resid
)
410 if (b_flags
& B_COMMIT_UPL
) {
411 pg_offset
= upl_offset
& PAGE_MASK
;
412 commit_size
= (pg_offset
+ total_size
+ (PAGE_SIZE
- 1)) & ~PAGE_MASK
;
414 if (error
|| (b_flags
& B_NOCACHE
)) {
419 if (b_flags
& B_PAGEIO
) {
420 if (b_flags
& B_READ
)
425 if (b_flags
& B_CACHE
) /* leave pages in the cache unchanged on error */
426 upl_abort_code
= UPL_ABORT_FREE_ON_EMPTY
;
427 else if (page_out
&& (error
!= ENXIO
)) /* transient error */
428 upl_abort_code
= UPL_ABORT_FREE_ON_EMPTY
;
430 upl_abort_code
= UPL_ABORT_FREE_ON_EMPTY
| UPL_ABORT_ERROR
;
432 upl_abort_code
= UPL_ABORT_FREE_ON_EMPTY
| UPL_ABORT_DUMP_PAGES
;
434 ubc_upl_abort_range(upl
, upl_offset
- pg_offset
, commit_size
,
437 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 20)) | DBG_FUNC_END
,
438 (int)upl
, upl_offset
- pg_offset
, commit_size
,
439 0x80000000|upl_abort_code
, 0);
442 int upl_commit_flags
= UPL_COMMIT_FREE_ON_EMPTY
;
444 if ((b_flags
& B_PHYS
) && (b_flags
& B_READ
))
445 upl_commit_flags
|= UPL_COMMIT_SET_DIRTY
;
448 upl_commit_flags
|= UPL_COMMIT_INACTIVATE
;
450 ubc_upl_commit_range(upl
, upl_offset
- pg_offset
, commit_size
,
453 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 20)) | DBG_FUNC_END
,
454 (int)upl
, upl_offset
- pg_offset
, commit_size
,
455 upl_commit_flags
, 0);
458 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 20)) | DBG_FUNC_END
,
459 (int)upl
, upl_offset
, 0, error
, 0);
467 cluster_zero(upl_t upl
, vm_offset_t upl_offset
, int size
, buf_t bp
)
471 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 23)) | DBG_FUNC_START
,
472 upl_offset
, size
, (int)bp
, 0, 0);
474 if (bp
== NULL
|| bp
->b_datap
== 0) {
476 pl
= ubc_upl_pageinfo(upl
);
484 page_index
= upl_offset
/ PAGE_SIZE
;
485 page_offset
= upl_offset
& PAGE_MASK
;
487 zero_addr
= ((addr64_t
)upl_phys_page(pl
, page_index
) << 12) + page_offset
;
488 zero_cnt
= min(PAGE_SIZE
- page_offset
, size
);
490 bzero_phys(zero_addr
, zero_cnt
);
493 upl_offset
+= zero_cnt
;
496 bzero((caddr_t
)((vm_offset_t
)bp
->b_datap
+ upl_offset
), size
);
498 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 23)) | DBG_FUNC_END
,
499 upl_offset
, size
, 0, 0, 0);
504 cluster_io(vnode_t vp
, upl_t upl
, vm_offset_t upl_offset
, off_t f_offset
, int non_rounded_size
,
505 int flags
, buf_t real_bp
, struct clios
*iostate
)
514 buf_t cbp_head
= NULL
;
515 buf_t cbp_tail
= NULL
;
523 int async_throttle
= 0;
528 if (mp
->mnt_devblocksize
> 1) {
530 * round the requested size up so that this I/O ends on a
531 * page boundary in case this is a 'write'... if the filesystem
532 * has blocks allocated to back the page beyond the EOF, we want to
533 * make sure to write out the zero's that are sitting beyond the EOF
534 * so that in case the filesystem doesn't explicitly zero this area
535 * if a hole is created via a lseek/write beyond the current EOF,
536 * it will return zeros when it's read back from the disk. If the
537 * physical allocation doesn't extend for the whole page, we'll
538 * only write/read from the disk up to the end of this allocation
539 * via the extent info returned from the VNOP_BLOCKMAP call.
541 pg_offset
= upl_offset
& PAGE_MASK
;
543 size
= (((non_rounded_size
+ pg_offset
) + (PAGE_SIZE
- 1)) & ~PAGE_MASK
) - pg_offset
;
546 * anyone advertising a blocksize of 1 byte probably
547 * can't deal with us rounding up the request size
548 * AFP is one such filesystem/device
550 size
= non_rounded_size
;
552 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 22)) | DBG_FUNC_START
,
553 (int)f_offset
, size
, upl_offset
, flags
, 0);
555 if (flags
& CL_READ
) {
557 bmap_flags
= VNODE_READ
;
559 max_iosize
= mp
->mnt_maxreadcnt
;
560 max_vectors
= mp
->mnt_segreadcnt
;
563 bmap_flags
= VNODE_WRITE
;
565 max_iosize
= mp
->mnt_maxwritecnt
;
566 max_vectors
= mp
->mnt_segwritecnt
;
568 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 22)) | DBG_FUNC_NONE
, max_iosize
, max_vectors
, mp
->mnt_devblocksize
, 0, 0);
571 * make sure the maximum iosize is a
572 * multiple of the page size
574 max_iosize
&= ~PAGE_MASK
;
576 if (flags
& CL_THROTTLE
) {
577 if ( !(flags
& CL_PAGEOUT
) && cluster_hard_throttle_on(vp
)) {
578 if (max_iosize
> HARD_THROTTLE_MAXSIZE
)
579 max_iosize
= HARD_THROTTLE_MAXSIZE
;
580 async_throttle
= HARD_THROTTLE_MAXCNT
;
582 async_throttle
= VNODE_ASYNC_THROTTLE
;
587 io_flags
|= B_NOCACHE
;
588 if (flags
& (CL_PAGEIN
| CL_PAGEOUT
))
589 io_flags
|= B_PAGEIO
;
590 if (flags
& CL_COMMIT
)
591 io_flags
|= B_COMMIT_UPL
;
592 if (flags
& CL_PRESERVE
)
594 if (flags
& CL_KEEPCACHED
)
597 if ((flags
& CL_READ
) && ((upl_offset
+ non_rounded_size
) & PAGE_MASK
) && (!(flags
& CL_NOZERO
))) {
599 * then we are going to end up
600 * with a page that we can't complete (the file size wasn't a multiple
601 * of PAGE_SIZE and we're trying to read to the end of the file
602 * so we'll go ahead and zero out the portion of the page we can't
603 * read in from the file
605 zero_offset
= upl_offset
+ non_rounded_size
;
612 if (size
> max_iosize
)
613 io_size
= max_iosize
;
617 if ((error
= VNOP_BLOCKMAP(vp
, f_offset
, io_size
, &blkno
, (size_t *)&io_size
, NULL
, bmap_flags
, NULL
))) {
620 if (real_bp
&& (real_bp
->b_blkno
== real_bp
->b_lblkno
))
621 real_bp
->b_blkno
= blkno
;
623 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 24)) | DBG_FUNC_NONE
,
624 (int)f_offset
, (int)blkno
, io_size
, zero_offset
, 0);
628 * vnop_blockmap didn't return an error... however, it did
629 * return an extent size of 0 which means we can't
630 * make forward progress on this I/O... a hole in the
631 * file would be returned as a blkno of -1 with a non-zero io_size
632 * a real extent is returned with a blkno != -1 and a non-zero io_size
637 if ( !(flags
& CL_READ
) && blkno
== -1) {
641 * we're writing into a 'hole'
643 if (flags
& CL_PAGEOUT
) {
645 * if we got here via cluster_pageout
646 * then just error the request and return
647 * the 'hole' should already have been covered
652 if ( !(flags
& CL_COMMIT
)) {
654 * currently writes always request the commit to happen
655 * as part of the io completion... however, if the CL_COMMIT
656 * flag isn't specified, than we can't issue the abort_range
657 * since the call site is going to abort or commit the same upl..
658 * in this case we can only return an error
664 * we can get here if the cluster code happens to
665 * pick up a page that was dirtied via mmap vs
666 * a 'write' and the page targets a 'hole'...
667 * i.e. the writes to the cluster were sparse
668 * and the file was being written for the first time
670 * we can also get here if the filesystem supports
671 * 'holes' that are less than PAGE_SIZE.... because
672 * we can't know if the range in the page that covers
673 * the 'hole' has been dirtied via an mmap or not,
674 * we have to assume the worst and try to push the
675 * entire page to storage.
677 * Try paging out the page individually before
678 * giving up entirely and dumping it (the pageout
679 * path will insure that the zero extent accounting
680 * has been taken care of before we get back into cluster_io)
682 ubc_upl_abort_range(upl
, trunc_page(upl_offset
), PAGE_SIZE
, UPL_ABORT_FREE_ON_EMPTY
);
684 e_offset
= round_page_64(f_offset
+ 1);
686 if (ubc_sync_range(vp
, f_offset
, e_offset
, UBC_PUSHDIRTY
) == 0) {
690 io_size
= e_offset
- f_offset
;
693 upl_offset
+= io_size
;
700 * keep track of how much of the original request
701 * that we've actually completed... non_rounded_size
702 * may go negative due to us rounding the request
703 * to a page size multiple (i.e. size > non_rounded_size)
705 non_rounded_size
-= io_size
;
707 if (non_rounded_size
<= 0) {
709 * we've transferred all of the data in the original
710 * request, but we were unable to complete the tail
711 * of the last page because the file didn't have
712 * an allocation to back that portion... this is ok.
718 lblkno
= (daddr64_t
)(f_offset
/ PAGE_SIZE_64
);
720 * we have now figured out how much I/O we can do - this is in 'io_size'
721 * pg_offset is the starting point in the first page for the I/O
722 * pg_count is the number of full and partial pages that 'io_size' encompasses
724 pg_offset
= upl_offset
& PAGE_MASK
;
726 if (flags
& CL_DEV_MEMORY
) {
728 * currently, can't deal with reading 'holes' in file
735 * treat physical requests as one 'giant' page
739 pg_count
= (io_size
+ pg_offset
+ (PAGE_SIZE
- 1)) / PAGE_SIZE
;
741 if ((flags
& CL_READ
) && blkno
== -1) {
745 * if we're reading and blkno == -1, then we've got a
746 * 'hole' in the file that we need to deal with by zeroing
747 * out the affected area in the upl
749 if (zero_offset
&& io_size
== size
) {
751 * if this upl contains the EOF and it is not a multiple of PAGE_SIZE
752 * than 'zero_offset' will be non-zero
753 * if the 'hole' returned by vnop_blockmap extends all the way to the eof
754 * (indicated by the io_size finishing off the I/O request for this UPL)
755 * than we're not going to issue an I/O for the
756 * last page in this upl... we need to zero both the hole and the tail
757 * of the page beyond the EOF, since the delayed zero-fill won't kick in
759 bytes_to_zero
= (((upl_offset
+ io_size
) + (PAGE_SIZE
- 1)) & ~PAGE_MASK
) - upl_offset
;
763 bytes_to_zero
= io_size
;
765 cluster_zero(upl
, upl_offset
, bytes_to_zero
, real_bp
);
769 * if there is a current I/O chain pending
770 * then the first page of the group we just zero'd
771 * will be handled by the I/O completion if the zero
772 * fill started in the middle of the page
774 pg_count
= (io_size
- pg_offset
) / PAGE_SIZE
;
777 * no pending I/O to pick up that first page
778 * so, we have to make sure it gets committed
780 * set the pg_offset to 0 so that the upl_commit_range
781 * starts with this page
783 pg_count
= (io_size
+ pg_offset
) / PAGE_SIZE
;
786 if (io_size
== size
&& ((upl_offset
+ io_size
) & PAGE_MASK
))
788 * if we're done with the request for this UPL
789 * then we have to make sure to commit the last page
790 * even if we only partially zero-filled it
796 pg_resid
= PAGE_SIZE
- pg_offset
;
800 if (flags
& CL_COMMIT
)
801 ubc_upl_commit_range(upl
,
802 (upl_offset
+ pg_resid
) & ~PAGE_MASK
,
803 pg_count
* PAGE_SIZE
,
804 UPL_COMMIT_CLEAR_DIRTY
| UPL_COMMIT_FREE_ON_EMPTY
);
806 upl_offset
+= io_size
;
810 * keep track of how much of the original request
811 * that we've actually completed... non_rounded_size
812 * may go negative due to us rounding the request
813 * to a page size multiple (i.e. size > non_rounded_size)
815 non_rounded_size
-= io_size
;
817 if (non_rounded_size
<= 0) {
819 * we've transferred all of the data in the original
820 * request, but we were unable to complete the tail
821 * of the last page because the file didn't have
822 * an allocation to back that portion... this is ok.
826 if (cbp_head
&& pg_count
)
831 if (pg_count
> max_vectors
) {
832 if (((pg_count
- max_vectors
) * PAGE_SIZE
) > io_size
) {
833 io_size
= PAGE_SIZE
- pg_offset
;
836 io_size
-= (pg_count
- max_vectors
) * PAGE_SIZE
;
837 pg_count
= max_vectors
;
841 if ( !(mp
->mnt_kern_flag
& MNTK_VIRTUALDEV
))
843 * if we're not targeting a virtual device i.e. a disk image
844 * it's safe to dip into the reserve pool since real devices
845 * can complete this I/O request without requiring additional
846 * bufs from the alloc_io_buf pool
849 else if ((flags
& CL_ASYNC
) && !(flags
& CL_PAGEOUT
))
851 * Throttle the speculative IO
857 cbp
= alloc_io_buf(vp
, priv
);
859 if (flags
& CL_PAGEOUT
) {
862 for (i
= 0; i
< pg_count
; i
++) {
863 if (buf_invalblkno(vp
, lblkno
+ i
, 0) == EBUSY
)
864 panic("BUSY bp found in cluster_io");
867 if (flags
& CL_ASYNC
) {
868 if (buf_setcallback(cbp
, (void *)cluster_iodone
, NULL
))
869 panic("buf_setcallback failed\n");
871 cbp
->b_flags
|= io_flags
;
873 cbp
->b_lblkno
= lblkno
;
874 cbp
->b_blkno
= blkno
;
875 cbp
->b_bcount
= io_size
;
877 if (buf_setupl(cbp
, upl
, upl_offset
))
878 panic("buf_setupl failed\n");
880 cbp
->b_trans_next
= (buf_t
)NULL
;
882 if ((cbp
->b_iostate
= (void *)iostate
))
884 * caller wants to track the state of this
885 * io... bump the amount issued against this stream
887 iostate
->io_issued
+= io_size
;
889 if (flags
& CL_READ
) {
890 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 26)) | DBG_FUNC_NONE
,
891 (int)cbp
->b_lblkno
, (int)cbp
->b_blkno
, upl_offset
, io_size
, 0);
894 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 27)) | DBG_FUNC_NONE
,
895 (int)cbp
->b_lblkno
, (int)cbp
->b_blkno
, upl_offset
, io_size
, 0);
899 cbp_tail
->b_trans_next
= cbp
;
905 (buf_t
)(cbp
->b_trans_head
) = cbp_head
;
908 upl_offset
+= io_size
;
912 * keep track of how much of the original request
913 * that we've actually completed... non_rounded_size
914 * may go negative due to us rounding the request
915 * to a page size multiple (i.e. size > non_rounded_size)
917 non_rounded_size
-= io_size
;
919 if (non_rounded_size
<= 0) {
921 * we've transferred all of the data in the original
922 * request, but we were unable to complete the tail
923 * of the last page because the file didn't have
924 * an allocation to back that portion... this is ok.
928 if ( (!(upl_offset
& PAGE_MASK
) && !(flags
& CL_DEV_MEMORY
) && ((flags
& CL_ASYNC
) || trans_count
> 8)) || size
== 0) {
930 * if we have no more I/O to issue or
931 * the current I/O we've prepared fully
932 * completes the last page in this request
933 * and it's either an ASYNC request or
934 * we've already accumulated more than 8 I/O's into
935 * this transaction and it's not an I/O directed to
936 * special DEVICE memory
937 * then go ahead and issue the I/O
941 cbp_head
->b_flags
|= B_NEED_IODONE
;
942 cbp_head
->b_real_bp
= real_bp
;
944 cbp_head
->b_real_bp
= (buf_t
)NULL
;
948 * we're about to issue the last I/O for this upl
949 * if this was a read to the eof and the eof doesn't
950 * finish on a page boundary, than we need to zero-fill
951 * the rest of the page....
953 cbp_head
->b_validend
= zero_offset
;
955 cbp_head
->b_validend
= 0;
957 if (flags
& CL_THROTTLE
)
958 (void)vnode_waitforwrites(vp
, async_throttle
, 0, 0, (char *)"cluster_io");
960 for (cbp
= cbp_head
; cbp
;) {
963 if ( !(io_flags
& B_READ
))
964 vnode_startwrite(vp
);
966 cbp_next
= cbp
->b_trans_next
;
968 (void) VNOP_STRATEGY(cbp
);
971 if ( !(flags
& CL_ASYNC
)) {
974 for (cbp
= cbp_head
; cbp
; cbp
= cbp
->b_trans_next
)
977 if ((error
= cluster_iodone(cbp_head
, (void *)&dummy
))) {
978 if (((flags
& (CL_PAGEOUT
| CL_KEEPCACHED
)) == CL_PAGEOUT
) && (error
== ENXIO
))
979 error
= 0; /* drop the error */
987 cbp_head
= (buf_t
)NULL
;
988 cbp_tail
= (buf_t
)NULL
;
998 for (cbp
= cbp_head
; cbp
;) {
1001 upl_offset
-= cbp
->b_bcount
;
1002 size
+= cbp
->b_bcount
;
1003 io_size
+= cbp
->b_bcount
;
1005 cbp_next
= cbp
->b_trans_next
;
1010 int need_wakeup
= 0;
1013 * update the error condition for this stream
1014 * since we never really issued the io
1015 * just go ahead and adjust it back
1017 lck_mtx_lock(cl_mtxp
);
1019 if (iostate
->io_error
== 0)
1020 iostate
->io_error
= error
;
1021 iostate
->io_issued
-= io_size
;
1023 if (iostate
->io_wanted
) {
1025 * someone is waiting for the state of
1026 * this io stream to change
1028 iostate
->io_wanted
= 0;
1031 lck_mtx_unlock(cl_mtxp
);
1034 wakeup((caddr_t
)&iostate
->io_wanted
);
1036 pg_offset
= upl_offset
& PAGE_MASK
;
1037 abort_size
= (size
+ pg_offset
+ (PAGE_SIZE
- 1)) & ~PAGE_MASK
;
1039 if (flags
& CL_COMMIT
) {
1042 if (flags
& CL_PRESERVE
) {
1043 ubc_upl_commit_range(upl
, upl_offset
- pg_offset
, abort_size
,
1044 UPL_COMMIT_FREE_ON_EMPTY
);
1046 if ((flags
& CL_PAGEOUT
) && (error
!= ENXIO
)) /* transient error */
1047 upl_abort_code
= UPL_ABORT_FREE_ON_EMPTY
;
1048 else if (flags
& CL_PAGEIN
)
1049 upl_abort_code
= UPL_ABORT_FREE_ON_EMPTY
| UPL_ABORT_ERROR
;
1051 upl_abort_code
= UPL_ABORT_FREE_ON_EMPTY
| UPL_ABORT_DUMP_PAGES
;
1053 ubc_upl_abort_range(upl
, upl_offset
- pg_offset
, abort_size
,
1056 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 28)) | DBG_FUNC_NONE
,
1057 (int)upl
, upl_offset
- pg_offset
, abort_size
, error
, 0);
1060 real_bp
->b_flags
|= B_ERROR
;
1061 real_bp
->b_error
= error
;
1063 buf_biodone(real_bp
);
1068 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 22)) | DBG_FUNC_END
,
1069 (int)f_offset
, size
, upl_offset
, retval
, 0);
1076 cluster_rd_prefetch(vnode_t vp
, off_t f_offset
, u_int size
, off_t filesize
)
1078 int pages_in_prefetch
;
1080 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 49)) | DBG_FUNC_START
,
1081 (int)f_offset
, size
, (int)filesize
, 0, 0);
1083 if (f_offset
>= filesize
) {
1084 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 49)) | DBG_FUNC_END
,
1085 (int)f_offset
, 0, 0, 0, 0);
1088 if (size
> (MAX_UPL_TRANSFER
* PAGE_SIZE
))
1089 size
= (MAX_UPL_TRANSFER
* PAGE_SIZE
);
1091 size
= (size
+ (PAGE_SIZE
- 1)) & ~PAGE_MASK
;
1093 if ((off_t
)size
> (filesize
- f_offset
))
1094 size
= filesize
- f_offset
;
1095 pages_in_prefetch
= (size
+ (PAGE_SIZE
- 1)) / PAGE_SIZE
;
1097 advisory_read(vp
, filesize
, f_offset
, size
);
1099 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 49)) | DBG_FUNC_END
,
1100 (int)f_offset
+ size
, pages_in_prefetch
, 0, 1, 0);
1102 return (pages_in_prefetch
);
1108 cluster_rd_ahead(vnode_t vp
, struct cl_extent
*extent
, off_t filesize
, struct cl_readahead
*rap
)
1112 int size_of_prefetch
;
1115 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 48)) | DBG_FUNC_START
,
1116 (int)extent
->b_addr
, (int)extent
->e_addr
, (int)rap
->cl_lastr
, 0, 0);
1118 if (extent
->b_addr
== rap
->cl_lastr
&& extent
->b_addr
== extent
->e_addr
) {
1119 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 48)) | DBG_FUNC_END
,
1120 rap
->cl_ralen
, (int)rap
->cl_maxra
, (int)rap
->cl_lastr
, 0, 0);
1123 if (rap
->cl_lastr
== -1 || (extent
->b_addr
!= rap
->cl_lastr
&& extent
->b_addr
!= (rap
->cl_lastr
+ 1) &&
1124 (extent
->b_addr
!= (rap
->cl_maxra
+ 1) || rap
->cl_ralen
== 0))) {
1128 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 48)) | DBG_FUNC_END
,
1129 rap
->cl_ralen
, (int)rap
->cl_maxra
, (int)rap
->cl_lastr
, 1, 0);
1133 if (extent
->e_addr
< rap
->cl_maxra
) {
1134 if ((rap
->cl_maxra
- extent
->e_addr
) > (MAX_UPL_TRANSFER
/ 4)) {
1136 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 48)) | DBG_FUNC_END
,
1137 rap
->cl_ralen
, (int)rap
->cl_maxra
, (int)rap
->cl_lastr
, 2, 0);
1141 r_addr
= max(extent
->e_addr
, rap
->cl_maxra
) + 1;
1142 f_offset
= (off_t
)(r_addr
* PAGE_SIZE_64
);
1144 size_of_prefetch
= 0;
1146 ubc_range_op(vp
, f_offset
, f_offset
+ PAGE_SIZE_64
, UPL_ROP_PRESENT
, &size_of_prefetch
);
1148 if (size_of_prefetch
) {
1149 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 48)) | DBG_FUNC_END
,
1150 rap
->cl_ralen
, (int)rap
->cl_maxra
, (int)rap
->cl_lastr
, 3, 0);
1153 if (f_offset
< filesize
) {
1154 daddr64_t read_size
;
1156 rap
->cl_ralen
= rap
->cl_ralen
? min(MAX_UPL_TRANSFER
, rap
->cl_ralen
<< 1) : 1;
1158 read_size
= (extent
->e_addr
+ 1) - extent
->b_addr
;
1160 if (read_size
> rap
->cl_ralen
) {
1161 if (read_size
> MAX_UPL_TRANSFER
)
1162 rap
->cl_ralen
= MAX_UPL_TRANSFER
;
1164 rap
->cl_ralen
= read_size
;
1166 size_of_prefetch
= cluster_rd_prefetch(vp
, f_offset
, rap
->cl_ralen
* PAGE_SIZE
, filesize
);
1168 if (size_of_prefetch
)
1169 rap
->cl_maxra
= (r_addr
+ size_of_prefetch
) - 1;
1171 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 48)) | DBG_FUNC_END
,
1172 rap
->cl_ralen
, (int)rap
->cl_maxra
, (int)rap
->cl_lastr
, 4, 0);
1176 cluster_pageout(vnode_t vp
, upl_t upl
, vm_offset_t upl_offset
, off_t f_offset
,
1177 int size
, off_t filesize
, int flags
)
1183 struct cl_writebehind
*wbp
;
1185 if (vp
->v_mount
->mnt_kern_flag
& MNTK_VIRTUALDEV
)
1187 * if we know we're issuing this I/O to a virtual device (i.e. disk image)
1188 * then we don't want to enforce this throttle... if we do, we can
1189 * potentially deadlock since we're stalling the pageout thread at a time
1190 * when the disk image might need additional memory (which won't be available
1191 * if the pageout thread can't run)... instead we'll just depend on the throttle
1192 * that the pageout thread now has in place to deal with external files
1194 local_flags
= CL_PAGEOUT
;
1196 local_flags
= CL_PAGEOUT
| CL_THROTTLE
;
1198 if ((flags
& UPL_IOSYNC
) == 0)
1199 local_flags
|= CL_ASYNC
;
1200 if ((flags
& UPL_NOCOMMIT
) == 0)
1201 local_flags
|= CL_COMMIT
;
1202 if ((flags
& UPL_KEEPCACHED
))
1203 local_flags
|= CL_KEEPCACHED
;
1206 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 52)) | DBG_FUNC_NONE
,
1207 (int)f_offset
, size
, (int)filesize
, local_flags
, 0);
1210 * If they didn't specify any I/O, then we are done...
1211 * we can't issue an abort because we don't know how
1212 * big the upl really is
1217 if (vp
->v_mount
->mnt_flag
& MNT_RDONLY
) {
1218 if (local_flags
& CL_COMMIT
)
1219 ubc_upl_abort_range(upl
, upl_offset
, size
, UPL_ABORT_FREE_ON_EMPTY
);
1223 * can't page-in from a negative offset
1224 * or if we're starting beyond the EOF
1225 * or if the file offset isn't page aligned
1226 * or the size requested isn't a multiple of PAGE_SIZE
1228 if (f_offset
< 0 || f_offset
>= filesize
||
1229 (f_offset
& PAGE_MASK_64
) || (size
& PAGE_MASK
)) {
1230 if (local_flags
& CL_COMMIT
)
1231 ubc_upl_abort_range(upl
, upl_offset
, size
, UPL_ABORT_FREE_ON_EMPTY
);
1234 max_size
= filesize
- f_offset
;
1236 if (size
< max_size
)
1241 rounded_size
= (io_size
+ (PAGE_SIZE
- 1)) & ~PAGE_MASK
;
1243 if (size
> rounded_size
) {
1244 if (local_flags
& CL_COMMIT
)
1245 ubc_upl_abort_range(upl
, upl_offset
+ rounded_size
, size
- rounded_size
,
1246 UPL_ABORT_FREE_ON_EMPTY
);
1248 if ((wbp
= cluster_get_wbp(vp
, 0)) != NULL
)
1249 wbp
->cl_hasbeenpaged
= 1;
1251 return (cluster_io(vp
, upl
, upl_offset
, f_offset
, io_size
,
1252 local_flags
, (buf_t
)NULL
, (struct clios
*)NULL
));
1256 cluster_pagein(vnode_t vp
, upl_t upl
, vm_offset_t upl_offset
, off_t f_offset
,
1257 int size
, off_t filesize
, int flags
)
1263 int local_flags
= 0;
1265 if (upl
== NULL
|| size
< 0)
1266 panic("cluster_pagein: NULL upl passed in");
1268 if ((flags
& UPL_IOSYNC
) == 0)
1269 local_flags
|= CL_ASYNC
;
1270 if ((flags
& UPL_NOCOMMIT
) == 0)
1271 local_flags
|= CL_COMMIT
;
1274 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 56)) | DBG_FUNC_NONE
,
1275 (int)f_offset
, size
, (int)filesize
, local_flags
, 0);
1278 * can't page-in from a negative offset
1279 * or if we're starting beyond the EOF
1280 * or if the file offset isn't page aligned
1281 * or the size requested isn't a multiple of PAGE_SIZE
1283 if (f_offset
< 0 || f_offset
>= filesize
||
1284 (f_offset
& PAGE_MASK_64
) || (size
& PAGE_MASK
) || (upl_offset
& PAGE_MASK
)) {
1285 if (local_flags
& CL_COMMIT
)
1286 ubc_upl_abort_range(upl
, upl_offset
, size
, UPL_ABORT_FREE_ON_EMPTY
| UPL_ABORT_ERROR
);
1289 max_size
= filesize
- f_offset
;
1291 if (size
< max_size
)
1296 rounded_size
= (io_size
+ (PAGE_SIZE
- 1)) & ~PAGE_MASK
;
1298 if (size
> rounded_size
&& (local_flags
& CL_COMMIT
))
1299 ubc_upl_abort_range(upl
, upl_offset
+ rounded_size
,
1300 size
- rounded_size
, UPL_ABORT_FREE_ON_EMPTY
| UPL_ABORT_ERROR
);
1302 retval
= cluster_io(vp
, upl
, upl_offset
, f_offset
, io_size
,
1303 local_flags
| CL_READ
| CL_PAGEIN
, (buf_t
)NULL
, (struct clios
*)NULL
);
1305 if (retval
== 0 && !(flags
& UPL_NORDAHEAD
) && !(vp
->v_flag
& VRAOFF
)) {
1306 struct cl_readahead
*rap
;
1308 rap
= cluster_get_rap(vp
);
1311 struct cl_extent extent
;
1313 extent
.b_addr
= (daddr64_t
)(f_offset
/ PAGE_SIZE_64
);
1314 extent
.e_addr
= (daddr64_t
)((f_offset
+ ((off_t
)io_size
- 1)) / PAGE_SIZE_64
);
1316 if (rounded_size
== PAGE_SIZE
) {
1318 * we haven't read the last page in of the file yet
1319 * so let's try to read ahead if we're in
1320 * a sequential access pattern
1322 cluster_rd_ahead(vp
, &extent
, filesize
, rap
);
1324 rap
->cl_lastr
= extent
.e_addr
;
1326 lck_mtx_unlock(&rap
->cl_lockr
);
1333 cluster_bp(buf_t bp
)
1338 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 19)) | DBG_FUNC_START
,
1339 (int)bp
, (int)bp
->b_lblkno
, bp
->b_bcount
, bp
->b_flags
, 0);
1341 if (bp
->b_flags
& B_READ
)
1342 flags
= CL_ASYNC
| CL_READ
;
1346 f_offset
= ubc_blktooff(bp
->b_vp
, bp
->b_lblkno
);
1348 return (cluster_io(bp
->b_vp
, bp
->b_upl
, 0, f_offset
, bp
->b_bcount
, flags
, bp
, (struct clios
*)NULL
));
1352 cluster_write(vnode_t vp
, struct uio
*uio
, off_t oldEOF
, off_t newEOF
, off_t headOff
, off_t tailOff
, int xflags
)
1365 if (vp
->v_flag
& VNOCACHE_DATA
)
1366 flags
|= IO_NOCACHE
;
1368 if ( (!(flags
& IO_NOCACHE
)) || (!uio
) || (!UIO_SEG_IS_USER_SPACE(uio
->uio_segflg
))) {
1370 * go do a write through the cache if one of the following is true....
1371 * NOCACHE is not true
1372 * there is no uio structure or it doesn't target USERSPACE
1374 return (cluster_write_x(vp
, uio
, oldEOF
, newEOF
, headOff
, tailOff
, flags
));
1378 if (IS_VALID_UIO_SEGFLG(uio
->uio_segflg
) == 0) {
1379 panic("%s :%d - invalid uio_segflg\n", __FILE__
, __LINE__
);
1381 #endif /* LP64_DEBUG */
1383 while (uio_resid(uio
) && uio
->uio_offset
< newEOF
&& retval
== 0) {
1384 user_size_t iov_len
;
1385 user_addr_t iov_base
;
1388 * we know we have a resid, so this is safe
1389 * skip over any emtpy vectors
1391 uio_update(uio
, (user_size_t
)0);
1393 iov_len
= uio_curriovlen(uio
);
1394 iov_base
= uio_curriovbase(uio
);
1396 upl_size
= PAGE_SIZE
;
1397 upl_flags
= UPL_QUERY_OBJECT_TYPE
;
1399 // LP64todo - fix this!
1400 if ((vm_map_get_upl(current_map(),
1401 (vm_map_offset_t
)(iov_base
& ~((user_addr_t
)PAGE_MASK
)),
1402 &upl_size
, &upl
, NULL
, NULL
, &upl_flags
, 0)) != KERN_SUCCESS
) {
1404 * the user app must have passed in an invalid address
1410 * We check every vector target but if it is physically
1411 * contiguous space, we skip the sanity checks.
1413 if (upl_flags
& UPL_PHYS_CONTIG
) {
1416 zflags
= flags
& ~IO_TAILZEROFILL
;
1417 zflags
|= IO_HEADZEROFILL
;
1419 if (flags
& IO_HEADZEROFILL
) {
1421 * in case we have additional vectors, we don't want to do this again
1423 flags
&= ~IO_HEADZEROFILL
;
1425 if ((retval
= cluster_write_x(vp
, (struct uio
*)0, 0, uio
->uio_offset
, headOff
, 0, zflags
)))
1428 retval
= cluster_phys_write(vp
, uio
, newEOF
);
1430 if (uio_resid(uio
) == 0 && (flags
& IO_TAILZEROFILL
)) {
1431 return (cluster_write_x(vp
, (struct uio
*)0, 0, tailOff
, uio
->uio_offset
, 0, zflags
));
1434 else if ((uio_resid(uio
) < PAGE_SIZE
) || (flags
& (IO_TAILZEROFILL
| IO_HEADZEROFILL
))) {
1436 * we're here because we're don't have a physically contiguous target buffer
1437 * go do a write through the cache if one of the following is true....
1438 * the total xfer size is less than a page...
1439 * we're being asked to ZEROFILL either the head or the tail of the I/O...
1441 return (cluster_write_x(vp
, uio
, oldEOF
, newEOF
, headOff
, tailOff
, flags
));
1443 // LP64todo - fix this!
1444 else if (((int)uio
->uio_offset
& PAGE_MASK
) || (CAST_DOWN(int, iov_base
) & PAGE_MASK
)) {
1445 if (((int)uio
->uio_offset
& PAGE_MASK
) == (CAST_DOWN(int, iov_base
) & PAGE_MASK
)) {
1447 * Bring the file offset write up to a pagesize boundary
1448 * this will also bring the base address to a page boundary
1449 * since they both are currently on the same offset within a page
1450 * note: if we get here, uio->uio_resid is greater than PAGE_SIZE
1451 * so the computed clip_size must always be less than the current uio_resid
1453 clip_size
= (PAGE_SIZE
- (uio
->uio_offset
& PAGE_MASK_64
));
1456 * Fake the resid going into the cluster_write_x call
1457 * and restore it on the way out.
1459 // LP64todo - fix this
1460 prev_resid
= uio_resid(uio
);
1461 uio_setresid(uio
, clip_size
);
1463 retval
= cluster_write_x(vp
, uio
, oldEOF
, newEOF
, headOff
, tailOff
, flags
);
1465 uio_setresid(uio
, prev_resid
- (clip_size
- uio_resid(uio
)));
1468 * can't get both the file offset and the buffer offset aligned to a page boundary
1469 * so fire an I/O through the cache for this entire vector
1471 // LP64todo - fix this
1472 clip_size
= iov_len
;
1473 // LP64todo - fix this
1474 prev_resid
= uio_resid(uio
);
1475 uio_setresid(uio
, clip_size
);
1477 retval
= cluster_write_x(vp
, uio
, oldEOF
, newEOF
, headOff
, tailOff
, flags
);
1479 uio_setresid(uio
, prev_resid
- (clip_size
- uio_resid(uio
)));
1483 * If we come in here, we know the offset into
1484 * the file is on a pagesize boundary and the
1485 * target buffer address is also on a page boundary
1487 max_io_size
= newEOF
- uio
->uio_offset
;
1488 // LP64todo - fix this
1489 clip_size
= uio_resid(uio
);
1490 if (iov_len
< clip_size
)
1491 // LP64todo - fix this!
1492 clip_size
= iov_len
;
1493 if (max_io_size
< clip_size
)
1494 clip_size
= max_io_size
;
1496 if (clip_size
< PAGE_SIZE
) {
1498 * Take care of tail end of write in this vector
1500 // LP64todo - fix this
1501 prev_resid
= uio_resid(uio
);
1502 uio_setresid(uio
, clip_size
);
1504 retval
= cluster_write_x(vp
, uio
, oldEOF
, newEOF
, headOff
, tailOff
, flags
);
1506 uio_setresid(uio
, prev_resid
- (clip_size
- uio_resid(uio
)));
1508 /* round clip_size down to a multiple of pagesize */
1509 clip_size
= clip_size
& ~(PAGE_MASK
);
1510 // LP64todo - fix this
1511 prev_resid
= uio_resid(uio
);
1512 uio_setresid(uio
, clip_size
);
1514 retval
= cluster_nocopy_write(vp
, uio
, newEOF
);
1516 if ((retval
== 0) && uio_resid(uio
))
1517 retval
= cluster_write_x(vp
, uio
, oldEOF
, newEOF
, headOff
, tailOff
, flags
);
1519 uio_setresid(uio
, prev_resid
- (clip_size
- uio_resid(uio
)));
1529 cluster_nocopy_write(vnode_t vp
, struct uio
*uio
, off_t newEOF
)
1532 upl_page_info_t
*pl
;
1533 vm_offset_t upl_offset
;
1537 int upl_needed_size
;
1542 int force_data_sync
;
1544 struct clios iostate
;
1545 struct cl_writebehind
*wbp
;
1548 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 75)) | DBG_FUNC_START
,
1549 (int)uio
->uio_offset
, (int)uio_resid(uio
),
1553 * When we enter this routine, we know
1554 * -- the offset into the file is on a pagesize boundary
1555 * -- the resid is a page multiple
1556 * -- the resid will not exceed iov_len
1559 if ((wbp
= cluster_get_wbp(vp
, CLW_RETURNLOCKED
)) != NULL
) {
1561 cluster_try_push(wbp
, vp
, newEOF
, 0, 1);
1563 lck_mtx_unlock(&wbp
->cl_lockw
);
1565 iostate
.io_completed
= 0;
1566 iostate
.io_issued
= 0;
1567 iostate
.io_error
= 0;
1568 iostate
.io_wanted
= 0;
1570 while (uio_resid(uio
) && uio
->uio_offset
< newEOF
&& error
== 0) {
1571 user_addr_t iov_base
;
1573 io_size
= uio_resid(uio
);
1575 if (io_size
> (MAX_UPL_TRANSFER
* PAGE_SIZE
))
1576 io_size
= MAX_UPL_TRANSFER
* PAGE_SIZE
;
1578 iov_base
= uio_curriovbase(uio
);
1580 // LP64todo - fix this!
1581 upl_offset
= CAST_DOWN(vm_offset_t
, iov_base
) & PAGE_MASK
;
1583 upl_needed_size
= (upl_offset
+ io_size
+ (PAGE_SIZE
-1)) & ~PAGE_MASK
;
1585 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 76)) | DBG_FUNC_START
,
1586 (int)upl_offset
, upl_needed_size
, (int)iov_base
, io_size
, 0);
1588 for (force_data_sync
= 0; force_data_sync
< 3; force_data_sync
++) {
1590 upl_size
= upl_needed_size
;
1591 upl_flags
= UPL_FILE_IO
| UPL_COPYOUT_FROM
| UPL_NO_SYNC
|
1592 UPL_CLEAN_IN_PLACE
| UPL_SET_INTERNAL
| UPL_SET_LITE
| UPL_SET_IO_WIRE
;
1594 // LP64todo - fix this!
1595 kret
= vm_map_get_upl(current_map(),
1596 (vm_map_offset_t
)(iov_base
& ~((user_addr_t
)PAGE_MASK
)),
1604 if (kret
!= KERN_SUCCESS
) {
1605 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 76)) | DBG_FUNC_END
,
1608 * cluster_nocopy_write: failed to get pagelist
1610 * we may have already spun some portion of this request
1611 * off as async requests... we need to wait for the I/O
1612 * to complete before returning
1614 goto wait_for_writes
;
1616 pl
= UPL_GET_INTERNAL_PAGE_LIST(upl
);
1617 pages_in_pl
= upl_size
/ PAGE_SIZE
;
1619 for (i
= 0; i
< pages_in_pl
; i
++) {
1620 if (!upl_valid_page(pl
, i
))
1623 if (i
== pages_in_pl
)
1627 * didn't get all the pages back that we
1628 * needed... release this upl and try again
1630 ubc_upl_abort_range(upl
, (upl_offset
& ~PAGE_MASK
), upl_size
,
1631 UPL_ABORT_FREE_ON_EMPTY
);
1633 if (force_data_sync
>= 3) {
1634 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 76)) | DBG_FUNC_END
,
1635 i
, pages_in_pl
, upl_size
, kret
, 0);
1637 * for some reason, we couldn't acquire a hold on all
1638 * the pages needed in the user's address space
1640 * we may have already spun some portion of this request
1641 * off as async requests... we need to wait for the I/O
1642 * to complete before returning
1644 goto wait_for_writes
;
1648 * Consider the possibility that upl_size wasn't satisfied.
1650 if (upl_size
!= upl_needed_size
)
1651 io_size
= (upl_size
- (int)upl_offset
) & ~PAGE_MASK
;
1653 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 76)) | DBG_FUNC_END
,
1654 (int)upl_offset
, upl_size
, (int)iov_base
, io_size
, 0);
1657 ubc_upl_abort_range(upl
, (upl_offset
& ~PAGE_MASK
), upl_size
,
1658 UPL_ABORT_FREE_ON_EMPTY
);
1660 * we may have already spun some portion of this request
1661 * off as async requests... we need to wait for the I/O
1662 * to complete before returning
1664 goto wait_for_writes
;
1667 * Now look for pages already in the cache
1668 * and throw them away.
1669 * uio->uio_offset is page aligned within the file
1670 * io_size is a multiple of PAGE_SIZE
1672 ubc_range_op(vp
, uio
->uio_offset
, uio
->uio_offset
+ io_size
, UPL_ROP_DUMP
, NULL
);
1675 * we want push out these writes asynchronously so that we can overlap
1676 * the preparation of the next I/O
1677 * if there are already too many outstanding writes
1678 * wait until some complete before issuing the next
1680 lck_mtx_lock(cl_mtxp
);
1682 while ((iostate
.io_issued
- iostate
.io_completed
) > (2 * MAX_UPL_TRANSFER
* PAGE_SIZE
)) {
1683 iostate
.io_wanted
= 1;
1684 msleep((caddr_t
)&iostate
.io_wanted
, cl_mtxp
, PRIBIO
+ 1, "cluster_nocopy_write", 0);
1686 lck_mtx_unlock(cl_mtxp
);
1688 if (iostate
.io_error
) {
1690 * one of the earlier writes we issued ran into a hard error
1691 * don't issue any more writes, cleanup the UPL
1692 * that was just created but not used, then
1693 * go wait for all writes that are part of this stream
1694 * to complete before returning the error to the caller
1696 ubc_upl_abort_range(upl
, (upl_offset
& ~PAGE_MASK
), upl_size
,
1697 UPL_ABORT_FREE_ON_EMPTY
);
1699 goto wait_for_writes
;
1701 io_flag
= CL_ASYNC
| CL_PRESERVE
| CL_COMMIT
| CL_THROTTLE
;
1703 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 77)) | DBG_FUNC_START
,
1704 (int)upl_offset
, (int)uio
->uio_offset
, io_size
, io_flag
, 0);
1706 error
= cluster_io(vp
, upl
, upl_offset
, uio
->uio_offset
,
1707 io_size
, io_flag
, (buf_t
)NULL
, &iostate
);
1709 uio_update(uio
, (user_size_t
)io_size
);
1711 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 77)) | DBG_FUNC_END
,
1712 (int)upl_offset
, (int)uio
->uio_offset
, (int)uio_resid(uio
), error
, 0);
1718 * make sure all async writes issued as part of this stream
1719 * have completed before we return
1721 lck_mtx_lock(cl_mtxp
);
1723 while (iostate
.io_issued
!= iostate
.io_completed
) {
1724 iostate
.io_wanted
= 1;
1725 msleep((caddr_t
)&iostate
.io_wanted
, cl_mtxp
, PRIBIO
+ 1, "cluster_nocopy_write", 0);
1727 lck_mtx_unlock(cl_mtxp
);
1729 if (iostate
.io_error
)
1730 error
= iostate
.io_error
;
1732 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 75)) | DBG_FUNC_END
,
1733 (int)uio
->uio_offset
, (int)uio
->uio_resid
, error
, 4, 0);
1740 cluster_phys_write(vnode_t vp
, struct uio
*uio
, off_t newEOF
)
1742 upl_page_info_t
*pl
;
1745 vm_offset_t upl_offset
;
1749 int upl_needed_size
;
1754 user_addr_t iov_base
;
1756 struct cl_writebehind
*wbp
;
1758 devblocksize
= vp
->v_mount
->mnt_devblocksize
;
1760 * When we enter this routine, we know
1761 * -- the resid will not exceed iov_len
1762 * -- the vector target address is physcially contiguous
1764 if ((wbp
= cluster_get_wbp(vp
, CLW_RETURNLOCKED
)) != NULL
) {
1766 cluster_try_push(wbp
, vp
, newEOF
, 0, 1);
1768 lck_mtx_unlock(&wbp
->cl_lockw
);
1771 if (IS_VALID_UIO_SEGFLG(uio
->uio_segflg
) == 0) {
1772 panic("%s :%d - invalid uio_segflg\n", __FILE__
, __LINE__
);
1774 #endif /* LP64_DEBUG */
1776 // LP64todo - fix this!
1777 io_size
= (int)uio_curriovlen(uio
);
1778 iov_base
= uio_curriovbase(uio
);
1780 upl_offset
= CAST_DOWN(upl_offset_t
, iov_base
) & PAGE_MASK
;
1781 upl_needed_size
= upl_offset
+ io_size
;
1784 upl_size
= upl_needed_size
;
1785 upl_flags
= UPL_FILE_IO
| UPL_COPYOUT_FROM
| UPL_NO_SYNC
|
1786 UPL_CLEAN_IN_PLACE
| UPL_SET_INTERNAL
| UPL_SET_LITE
| UPL_SET_IO_WIRE
;
1788 // LP64todo - fix this!
1789 kret
= vm_map_get_upl(current_map(),
1790 (vm_map_offset_t
)(iov_base
& ~((user_addr_t
)PAGE_MASK
)),
1791 &upl_size
, &upl
, NULL
, &pages_in_pl
, &upl_flags
, 0);
1793 if (kret
!= KERN_SUCCESS
) {
1795 * cluster_phys_write: failed to get pagelist
1796 * note: return kret here
1801 * Consider the possibility that upl_size wasn't satisfied.
1802 * This is a failure in the physical memory case.
1804 if (upl_size
< upl_needed_size
) {
1805 ubc_upl_abort_range(upl
, 0, upl_size
, UPL_ABORT_FREE_ON_EMPTY
);
1808 pl
= ubc_upl_pageinfo(upl
);
1810 src_paddr
= ((addr64_t
)upl_phys_page(pl
, 0) << 12) + (addr64_t
)upl_offset
;
1812 while (((uio
->uio_offset
& (devblocksize
- 1)) || io_size
< devblocksize
) && io_size
) {
1815 head_size
= devblocksize
- (int)(uio
->uio_offset
& (devblocksize
- 1));
1817 if (head_size
> io_size
)
1818 head_size
= io_size
;
1820 error
= cluster_align_phys_io(vp
, uio
, src_paddr
, head_size
, 0);
1823 ubc_upl_abort_range(upl
, 0, upl_size
, UPL_ABORT_FREE_ON_EMPTY
);
1827 upl_offset
+= head_size
;
1828 src_paddr
+= head_size
;
1829 io_size
-= head_size
;
1831 tail_size
= io_size
& (devblocksize
- 1);
1832 io_size
-= tail_size
;
1836 * issue a synchronous write to cluster_io
1838 error
= cluster_io(vp
, upl
, upl_offset
, uio
->uio_offset
,
1839 io_size
, CL_DEV_MEMORY
, (buf_t
)NULL
, (struct clios
*)NULL
);
1843 * The cluster_io write completed successfully,
1844 * update the uio structure
1846 uio_update(uio
, (user_size_t
)io_size
);
1848 src_paddr
+= io_size
;
1851 error
= cluster_align_phys_io(vp
, uio
, src_paddr
, tail_size
, 0);
1854 * just release our hold on the physically contiguous
1855 * region without changing any state
1857 ubc_upl_abort_range(upl
, 0, upl_size
, UPL_ABORT_FREE_ON_EMPTY
);
1864 cluster_write_x(vnode_t vp
, struct uio
*uio
, off_t oldEOF
, off_t newEOF
, off_t headOff
, off_t tailOff
, int flags
)
1866 upl_page_info_t
*pl
;
1868 vm_offset_t upl_offset
= 0;
1881 long long total_size
;
1884 long long zero_cnt1
;
1886 struct cl_extent cl
;
1888 struct cl_writebehind
*wbp
;
1890 if ((wbp
= cluster_get_wbp(vp
, 0)) != NULL
)
1892 if (wbp
->cl_hasbeenpaged
) {
1894 * this vnode had pages cleaned to it by
1895 * the pager which indicates that either
1896 * it's not very 'hot', or the system is
1897 * being overwhelmed by a lot of dirty
1898 * data being delayed in the VM cache...
1899 * in either event, we'll push our remaining
1900 * delayed data at this point... this will
1901 * be more efficient than paging out 1 page at
1902 * a time, and will also act as a throttle
1903 * by delaying this client from writing any
1904 * more data until all his delayed data has
1905 * at least been queued to the uderlying driver.
1907 if (wbp
->cl_number
|| wbp
->cl_scmap
)
1908 cluster_push_EOF(vp
, newEOF
);
1910 wbp
->cl_hasbeenpaged
= 0;
1914 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 40)) | DBG_FUNC_START
,
1915 (int)uio
->uio_offset
, uio_resid(uio
), (int)oldEOF
, (int)newEOF
, 0);
1917 // LP64todo - fix this
1918 io_resid
= uio_resid(uio
);
1920 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 40)) | DBG_FUNC_START
,
1921 0, 0, (int)oldEOF
, (int)newEOF
, 0);
1930 if (flags
& IO_HEADZEROFILL
) {
1932 * some filesystems (HFS is one) don't support unallocated holes within a file...
1933 * so we zero fill the intervening space between the old EOF and the offset
1934 * where the next chunk of real data begins.... ftruncate will also use this
1935 * routine to zero fill to the new EOF when growing a file... in this case, the
1936 * uio structure will not be provided
1939 if (headOff
< uio
->uio_offset
) {
1940 zero_cnt
= uio
->uio_offset
- headOff
;
1943 } else if (headOff
< newEOF
) {
1944 zero_cnt
= newEOF
- headOff
;
1948 if (flags
& IO_TAILZEROFILL
) {
1950 // LP64todo - fix this
1951 zero_off1
= uio
->uio_offset
+ uio_resid(uio
);
1953 if (zero_off1
< tailOff
)
1954 zero_cnt1
= tailOff
- zero_off1
;
1957 if (zero_cnt
== 0 && uio
== (struct uio
*) 0) {
1958 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 40)) | DBG_FUNC_END
,
1959 retval
, 0, 0, 0, 0);
1963 while ((total_size
= (io_resid
+ zero_cnt
+ zero_cnt1
)) && retval
== 0) {
1965 * for this iteration of the loop, figure out where our starting point is
1968 start_offset
= (int)(zero_off
& PAGE_MASK_64
);
1969 upl_f_offset
= zero_off
- start_offset
;
1970 } else if (io_resid
) {
1971 start_offset
= (int)(uio
->uio_offset
& PAGE_MASK_64
);
1972 upl_f_offset
= uio
->uio_offset
- start_offset
;
1974 start_offset
= (int)(zero_off1
& PAGE_MASK_64
);
1975 upl_f_offset
= zero_off1
- start_offset
;
1977 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 46)) | DBG_FUNC_NONE
,
1978 (int)zero_off
, (int)zero_cnt
, (int)zero_off1
, (int)zero_cnt1
, 0);
1980 if (total_size
> (MAX_UPL_TRANSFER
* PAGE_SIZE
))
1981 total_size
= MAX_UPL_TRANSFER
* PAGE_SIZE
;
1983 cl
.b_addr
= (daddr64_t
)(upl_f_offset
/ PAGE_SIZE_64
);
1985 if (uio
&& ((flags
& (IO_NOCACHE
| IO_SYNC
| IO_HEADZEROFILL
| IO_TAILZEROFILL
)) == 0)) {
1987 * assumption... total_size <= io_resid
1988 * because IO_HEADZEROFILL and IO_TAILZEROFILL not set
1990 if ((start_offset
+ total_size
) > (MAX_UPL_TRANSFER
* PAGE_SIZE
))
1991 total_size
-= start_offset
;
1992 xfer_resid
= total_size
;
1994 retval
= cluster_copy_ubc_data(vp
, uio
, &xfer_resid
, 1);
1999 io_resid
-= (total_size
- xfer_resid
);
2000 total_size
= xfer_resid
;
2001 start_offset
= (int)(uio
->uio_offset
& PAGE_MASK_64
);
2002 upl_f_offset
= uio
->uio_offset
- start_offset
;
2004 if (total_size
== 0) {
2007 * the write did not finish on a page boundary
2008 * which will leave upl_f_offset pointing to the
2009 * beginning of the last page written instead of
2010 * the page beyond it... bump it in this case
2011 * so that the cluster code records the last page
2014 upl_f_offset
+= PAGE_SIZE_64
;
2022 * compute the size of the upl needed to encompass
2023 * the requested write... limit each call to cluster_io
2024 * to the maximum UPL size... cluster_io will clip if
2025 * this exceeds the maximum io_size for the device,
2026 * make sure to account for
2027 * a starting offset that's not page aligned
2029 upl_size
= (start_offset
+ total_size
+ (PAGE_SIZE
- 1)) & ~PAGE_MASK
;
2031 if (upl_size
> (MAX_UPL_TRANSFER
* PAGE_SIZE
))
2032 upl_size
= MAX_UPL_TRANSFER
* PAGE_SIZE
;
2034 pages_in_upl
= upl_size
/ PAGE_SIZE
;
2035 io_size
= upl_size
- start_offset
;
2037 if ((long long)io_size
> total_size
)
2038 io_size
= total_size
;
2040 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 41)) | DBG_FUNC_START
, upl_size
, io_size
, total_size
, 0, 0);
2044 * Gather the pages from the buffer cache.
2045 * The UPL_WILL_MODIFY flag lets the UPL subsystem know
2046 * that we intend to modify these pages.
2048 kret
= ubc_create_upl(vp
,
2053 UPL_SET_LITE
| UPL_WILL_MODIFY
);
2054 if (kret
!= KERN_SUCCESS
)
2055 panic("cluster_write: failed to get pagelist");
2057 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 41)) | DBG_FUNC_END
,
2058 (int)upl
, (int)upl_f_offset
, start_offset
, 0, 0);
2060 if (start_offset
&& !upl_valid_page(pl
, 0)) {
2064 * we're starting in the middle of the first page of the upl
2065 * and the page isn't currently valid, so we're going to have
2066 * to read it in first... this is a synchronous operation
2068 read_size
= PAGE_SIZE
;
2070 if ((upl_f_offset
+ read_size
) > newEOF
)
2071 read_size
= newEOF
- upl_f_offset
;
2073 retval
= cluster_io(vp
, upl
, 0, upl_f_offset
, read_size
,
2074 CL_READ
, (buf_t
)NULL
, (struct clios
*)NULL
);
2077 * we had an error during the read which causes us to abort
2078 * the current cluster_write request... before we do, we need
2079 * to release the rest of the pages in the upl without modifying
2080 * there state and mark the failed page in error
2082 ubc_upl_abort_range(upl
, 0, PAGE_SIZE
, UPL_ABORT_DUMP_PAGES
);
2084 if (upl_size
> PAGE_SIZE
)
2085 ubc_upl_abort_range(upl
, 0, upl_size
, UPL_ABORT_FREE_ON_EMPTY
);
2087 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 45)) | DBG_FUNC_NONE
,
2088 (int)upl
, 0, 0, retval
, 0);
2092 if ((start_offset
== 0 || upl_size
> PAGE_SIZE
) && ((start_offset
+ io_size
) & PAGE_MASK
)) {
2094 * the last offset we're writing to in this upl does not end on a page
2095 * boundary... if it's not beyond the old EOF, then we'll also need to
2096 * pre-read this page in if it isn't already valid
2098 upl_offset
= upl_size
- PAGE_SIZE
;
2100 if ((upl_f_offset
+ start_offset
+ io_size
) < oldEOF
&&
2101 !upl_valid_page(pl
, upl_offset
/ PAGE_SIZE
)) {
2104 read_size
= PAGE_SIZE
;
2106 if ((upl_f_offset
+ upl_offset
+ read_size
) > newEOF
)
2107 read_size
= newEOF
- (upl_f_offset
+ upl_offset
);
2109 retval
= cluster_io(vp
, upl
, upl_offset
, upl_f_offset
+ upl_offset
, read_size
,
2110 CL_READ
, (buf_t
)NULL
, (struct clios
*)NULL
);
2113 * we had an error during the read which causes us to abort
2114 * the current cluster_write request... before we do, we
2115 * need to release the rest of the pages in the upl without
2116 * modifying there state and mark the failed page in error
2118 ubc_upl_abort_range(upl
, upl_offset
, PAGE_SIZE
, UPL_ABORT_DUMP_PAGES
);
2120 if (upl_size
> PAGE_SIZE
)
2121 ubc_upl_abort_range(upl
, 0, upl_size
, UPL_ABORT_FREE_ON_EMPTY
);
2123 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 45)) | DBG_FUNC_NONE
,
2124 (int)upl
, 0, 0, retval
, 0);
2129 xfer_resid
= io_size
;
2130 io_offset
= start_offset
;
2132 while (zero_cnt
&& xfer_resid
) {
2134 if (zero_cnt
< (long long)xfer_resid
)
2135 bytes_to_zero
= zero_cnt
;
2137 bytes_to_zero
= xfer_resid
;
2139 if ( !(flags
& (IO_NOZEROVALID
| IO_NOZERODIRTY
))) {
2140 cluster_zero(upl
, io_offset
, bytes_to_zero
, NULL
);
2144 bytes_to_zero
= min(bytes_to_zero
, PAGE_SIZE
- (int)(zero_off
& PAGE_MASK_64
));
2145 zero_pg_index
= (int)((zero_off
- upl_f_offset
) / PAGE_SIZE_64
);
2147 if ( !upl_valid_page(pl
, zero_pg_index
)) {
2148 cluster_zero(upl
, io_offset
, bytes_to_zero
, NULL
);
2150 } else if ((flags
& (IO_NOZERODIRTY
| IO_NOZEROVALID
)) == IO_NOZERODIRTY
&&
2151 !upl_dirty_page(pl
, zero_pg_index
)) {
2152 cluster_zero(upl
, io_offset
, bytes_to_zero
, NULL
);
2155 xfer_resid
-= bytes_to_zero
;
2156 zero_cnt
-= bytes_to_zero
;
2157 zero_off
+= bytes_to_zero
;
2158 io_offset
+= bytes_to_zero
;
2160 if (xfer_resid
&& io_resid
) {
2161 bytes_to_move
= min(io_resid
, xfer_resid
);
2163 retval
= cluster_copy_upl_data(uio
, upl
, io_offset
, bytes_to_move
);
2167 ubc_upl_abort_range(upl
, 0, upl_size
, UPL_ABORT_DUMP_PAGES
| UPL_ABORT_FREE_ON_EMPTY
);
2169 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 45)) | DBG_FUNC_NONE
,
2170 (int)upl
, 0, 0, retval
, 0);
2172 io_resid
-= bytes_to_move
;
2173 xfer_resid
-= bytes_to_move
;
2174 io_offset
+= bytes_to_move
;
2177 while (xfer_resid
&& zero_cnt1
&& retval
== 0) {
2179 if (zero_cnt1
< (long long)xfer_resid
)
2180 bytes_to_zero
= zero_cnt1
;
2182 bytes_to_zero
= xfer_resid
;
2184 if ( !(flags
& (IO_NOZEROVALID
| IO_NOZERODIRTY
))) {
2185 cluster_zero(upl
, io_offset
, bytes_to_zero
, NULL
);
2189 bytes_to_zero
= min(bytes_to_zero
, PAGE_SIZE
- (int)(zero_off1
& PAGE_MASK_64
));
2190 zero_pg_index
= (int)((zero_off1
- upl_f_offset
) / PAGE_SIZE_64
);
2192 if ( !upl_valid_page(pl
, zero_pg_index
)) {
2193 cluster_zero(upl
, io_offset
, bytes_to_zero
, NULL
);
2194 } else if ((flags
& (IO_NOZERODIRTY
| IO_NOZEROVALID
)) == IO_NOZERODIRTY
&&
2195 !upl_dirty_page(pl
, zero_pg_index
)) {
2196 cluster_zero(upl
, io_offset
, bytes_to_zero
, NULL
);
2199 xfer_resid
-= bytes_to_zero
;
2200 zero_cnt1
-= bytes_to_zero
;
2201 zero_off1
+= bytes_to_zero
;
2202 io_offset
+= bytes_to_zero
;
2209 io_size
+= start_offset
;
2211 if ((upl_f_offset
+ io_size
) >= newEOF
&& io_size
< upl_size
) {
2213 * if we're extending the file with this write
2214 * we'll zero fill the rest of the page so that
2215 * if the file gets extended again in such a way as to leave a
2216 * hole starting at this EOF, we'll have zero's in the correct spot
2218 cluster_zero(upl
, io_size
, upl_size
- io_size
, NULL
);
2220 if (flags
& IO_SYNC
)
2222 * if the IO_SYNC flag is set than we need to
2223 * bypass any clusters and immediately issue
2229 * take the lock to protect our accesses
2230 * of the writebehind and sparse cluster state
2232 wbp
= cluster_get_wbp(vp
, CLW_ALLOCATE
| CLW_RETURNLOCKED
);
2235 * calculate the last logical block number
2236 * that this delayed I/O encompassed
2238 cl
.e_addr
= (daddr64_t
)((upl_f_offset
+ (off_t
)upl_size
) / PAGE_SIZE_64
);
2240 if (wbp
->cl_scmap
) {
2242 if ( !(flags
& IO_NOCACHE
)) {
2244 * we've fallen into the sparse
2245 * cluster method of delaying dirty pages
2246 * first, we need to release the upl if we hold one
2247 * since pages in it may be present in the sparse cluster map
2248 * and may span 2 separate buckets there... if they do and
2249 * we happen to have to flush a bucket to make room and it intersects
2250 * this upl, a deadlock may result on page BUSY
2253 ubc_upl_commit_range(upl
, 0, upl_size
,
2254 UPL_COMMIT_SET_DIRTY
| UPL_COMMIT_INACTIVATE
| UPL_COMMIT_FREE_ON_EMPTY
);
2256 sparse_cluster_add(wbp
, vp
, &cl
, newEOF
);
2258 lck_mtx_unlock(&wbp
->cl_lockw
);
2263 * must have done cached writes that fell into
2264 * the sparse cluster mechanism... we've switched
2265 * to uncached writes on the file, so go ahead
2266 * and push whatever's in the sparse map
2267 * and switch back to normal clustering
2269 * see the comment above concerning a possible deadlock...
2272 ubc_upl_commit_range(upl
, 0, upl_size
,
2273 UPL_COMMIT_SET_DIRTY
| UPL_COMMIT_INACTIVATE
| UPL_COMMIT_FREE_ON_EMPTY
);
2275 * setting upl_size to 0 keeps us from committing a
2276 * second time in the start_new_cluster path
2280 sparse_cluster_push(wbp
, vp
, newEOF
, 1);
2284 * no clusters of either type present at this point
2285 * so just go directly to start_new_cluster since
2286 * we know we need to delay this I/O since we've
2287 * already released the pages back into the cache
2288 * to avoid the deadlock with sparse_cluster_push
2290 goto start_new_cluster
;
2294 if (wbp
->cl_number
== 0)
2296 * no clusters currently present
2298 goto start_new_cluster
;
2300 for (cl_index
= 0; cl_index
< wbp
->cl_number
; cl_index
++) {
2302 * check each cluster that we currently hold
2303 * try to merge some or all of this write into
2304 * one or more of the existing clusters... if
2305 * any portion of the write remains, start a
2308 if (cl
.b_addr
>= wbp
->cl_clusters
[cl_index
].b_addr
) {
2310 * the current write starts at or after the current cluster
2312 if (cl
.e_addr
<= (wbp
->cl_clusters
[cl_index
].b_addr
+ MAX_UPL_TRANSFER
)) {
2314 * we have a write that fits entirely
2315 * within the existing cluster limits
2317 if (cl
.e_addr
> wbp
->cl_clusters
[cl_index
].e_addr
)
2319 * update our idea of where the cluster ends
2321 wbp
->cl_clusters
[cl_index
].e_addr
= cl
.e_addr
;
2324 if (cl
.b_addr
< (wbp
->cl_clusters
[cl_index
].b_addr
+ MAX_UPL_TRANSFER
)) {
2326 * we have a write that starts in the middle of the current cluster
2327 * but extends beyond the cluster's limit... we know this because
2328 * of the previous checks
2329 * we'll extend the current cluster to the max
2330 * and update the b_addr for the current write to reflect that
2331 * the head of it was absorbed into this cluster...
2332 * note that we'll always have a leftover tail in this case since
2333 * full absorbtion would have occurred in the clause above
2335 wbp
->cl_clusters
[cl_index
].e_addr
= wbp
->cl_clusters
[cl_index
].b_addr
+ MAX_UPL_TRANSFER
;
2338 daddr64_t start_pg_in_upl
;
2340 start_pg_in_upl
= (daddr64_t
)(upl_f_offset
/ PAGE_SIZE_64
);
2342 if (start_pg_in_upl
< wbp
->cl_clusters
[cl_index
].e_addr
) {
2343 intersection
= (int)((wbp
->cl_clusters
[cl_index
].e_addr
- start_pg_in_upl
) * PAGE_SIZE
);
2345 ubc_upl_commit_range(upl
, upl_offset
, intersection
,
2346 UPL_COMMIT_SET_DIRTY
| UPL_COMMIT_INACTIVATE
| UPL_COMMIT_FREE_ON_EMPTY
);
2347 upl_f_offset
+= intersection
;
2348 upl_offset
+= intersection
;
2349 upl_size
-= intersection
;
2352 cl
.b_addr
= wbp
->cl_clusters
[cl_index
].e_addr
;
2355 * we come here for the case where the current write starts
2356 * beyond the limit of the existing cluster or we have a leftover
2357 * tail after a partial absorbtion
2359 * in either case, we'll check the remaining clusters before
2360 * starting a new one
2364 * the current write starts in front of the cluster we're currently considering
2366 if ((wbp
->cl_clusters
[cl_index
].e_addr
- cl
.b_addr
) <= MAX_UPL_TRANSFER
) {
2368 * we can just merge the new request into
2369 * this cluster and leave it in the cache
2370 * since the resulting cluster is still
2371 * less than the maximum allowable size
2373 wbp
->cl_clusters
[cl_index
].b_addr
= cl
.b_addr
;
2375 if (cl
.e_addr
> wbp
->cl_clusters
[cl_index
].e_addr
) {
2377 * the current write completely
2378 * envelops the existing cluster and since
2379 * each write is limited to at most MAX_UPL_TRANSFER bytes
2380 * we can just use the start and last blocknos of the write
2381 * to generate the cluster limits
2383 wbp
->cl_clusters
[cl_index
].e_addr
= cl
.e_addr
;
2389 * if we were to combine this write with the current cluster
2390 * we would exceed the cluster size limit.... so,
2391 * let's see if there's any overlap of the new I/O with
2392 * the cluster we're currently considering... in fact, we'll
2393 * stretch the cluster out to it's full limit and see if we
2394 * get an intersection with the current write
2397 if (cl
.e_addr
> wbp
->cl_clusters
[cl_index
].e_addr
- MAX_UPL_TRANSFER
) {
2399 * the current write extends into the proposed cluster
2400 * clip the length of the current write after first combining it's
2401 * tail with the newly shaped cluster
2403 wbp
->cl_clusters
[cl_index
].b_addr
= wbp
->cl_clusters
[cl_index
].e_addr
- MAX_UPL_TRANSFER
;
2406 intersection
= (int)((cl
.e_addr
- wbp
->cl_clusters
[cl_index
].b_addr
) * PAGE_SIZE
);
2408 if (intersection
> upl_size
)
2410 * because the current write may consist of a number of pages found in the cache
2411 * which are not part of the UPL, we may have an intersection that exceeds
2412 * the size of the UPL that is also part of this write
2414 intersection
= upl_size
;
2416 ubc_upl_commit_range(upl
, upl_offset
+ (upl_size
- intersection
), intersection
,
2417 UPL_COMMIT_SET_DIRTY
| UPL_COMMIT_INACTIVATE
| UPL_COMMIT_FREE_ON_EMPTY
);
2418 upl_size
-= intersection
;
2420 cl
.e_addr
= wbp
->cl_clusters
[cl_index
].b_addr
;
2423 * if we get here, there was no way to merge
2424 * any portion of this write with this cluster
2425 * or we could only merge part of it which
2426 * will leave a tail...
2427 * we'll check the remaining clusters before starting a new one
2431 if (cl_index
< wbp
->cl_number
)
2433 * we found an existing cluster(s) that we
2434 * could entirely merge this I/O into
2438 if (wbp
->cl_number
< MAX_CLUSTERS
&& !(flags
& IO_NOCACHE
))
2440 * we didn't find an existing cluster to
2441 * merge into, but there's room to start
2444 goto start_new_cluster
;
2447 * no exisitng cluster to merge with and no
2448 * room to start a new one... we'll try
2449 * pushing one of the existing ones... if none of
2450 * them are able to be pushed, we'll switch
2451 * to the sparse cluster mechanism
2452 * cluster_try_push updates cl_number to the
2453 * number of remaining clusters... and
2454 * returns the number of currently unused clusters
2456 int ret_cluster_try_push
= 0;
2457 /* if writes are not deferred, call cluster push immediately */
2458 if (!((unsigned int)vfs_flags(vp
->v_mount
) & MNT_DEFWRITE
)) {
2459 if (flags
& IO_NOCACHE
)
2464 ret_cluster_try_push
= cluster_try_push(wbp
, vp
, newEOF
, can_delay
, 0);
2467 /* execute following regardless writes are deferred or not */
2468 if (ret_cluster_try_push
== 0) {
2470 * no more room in the normal cluster mechanism
2471 * so let's switch to the more expansive but expensive
2472 * sparse mechanism....
2473 * first, we need to release the upl if we hold one
2474 * since pages in it may be present in the sparse cluster map (after the cluster_switch)
2475 * and may span 2 separate buckets there... if they do and
2476 * we happen to have to flush a bucket to make room and it intersects
2477 * this upl, a deadlock may result on page BUSY
2480 ubc_upl_commit_range(upl
, upl_offset
, upl_size
,
2481 UPL_COMMIT_SET_DIRTY
| UPL_COMMIT_INACTIVATE
| UPL_COMMIT_FREE_ON_EMPTY
);
2483 sparse_cluster_switch(wbp
, vp
, newEOF
);
2484 sparse_cluster_add(wbp
, vp
, &cl
, newEOF
);
2486 lck_mtx_unlock(&wbp
->cl_lockw
);
2491 * we pushed one cluster successfully, so we must be sequentially writing this file
2492 * otherwise, we would have failed and fallen into the sparse cluster support
2493 * so let's take the opportunity to push out additional clusters as long as we
2494 * remain below the throttle... this will give us better I/O locality if we're
2495 * in a copy loop (i.e. we won't jump back and forth between the read and write points
2496 * however, we don't want to push so much out that the write throttle kicks in and
2497 * hangs this thread up until some of the I/O completes...
2499 if (!((unsigned int)vfs_flags(vp
->v_mount
) & MNT_DEFWRITE
)) {
2500 while (wbp
->cl_number
&& (vp
->v_numoutput
<= (VNODE_ASYNC_THROTTLE
/ 2)))
2501 cluster_try_push(wbp
, vp
, newEOF
, 0, 0);
2505 wbp
->cl_clusters
[wbp
->cl_number
].b_addr
= cl
.b_addr
;
2506 wbp
->cl_clusters
[wbp
->cl_number
].e_addr
= cl
.e_addr
;
2508 if (flags
& IO_NOCACHE
)
2509 wbp
->cl_clusters
[wbp
->cl_number
].io_nocache
= 1;
2511 wbp
->cl_clusters
[wbp
->cl_number
].io_nocache
= 0;
2515 ubc_upl_commit_range(upl
, upl_offset
, upl_size
,
2516 UPL_COMMIT_SET_DIRTY
| UPL_COMMIT_INACTIVATE
| UPL_COMMIT_FREE_ON_EMPTY
);
2518 lck_mtx_unlock(&wbp
->cl_lockw
);
2523 * we don't hold the vnode lock at this point
2525 * because we had to ask for a UPL that provides currenty non-present pages, the
2526 * UPL has been automatically set to clear the dirty flags (both software and hardware)
2527 * upon committing it... this is not the behavior we want since it's possible for
2528 * pages currently present as part of a mapped file to be dirtied while the I/O is in flight.
2529 * in order to maintain some semblance of coherency with mapped writes
2530 * we need to drop the current upl and pick it back up with COPYOUT_FROM set
2531 * so that we correctly deal with a change in state of the hardware modify bit...
2532 * we do this via cluster_push_x... by passing along the IO_SYNC flag, we force
2533 * cluster_push_x to wait until all the I/Os have completed... cluster_push_x is also
2534 * responsible for generating the correct sized I/O(s)
2536 ubc_upl_commit_range(upl
, 0, upl_size
,
2537 UPL_COMMIT_SET_DIRTY
| UPL_COMMIT_INACTIVATE
| UPL_COMMIT_FREE_ON_EMPTY
);
2539 cl
.e_addr
= (upl_f_offset
+ (off_t
)upl_size
) / PAGE_SIZE_64
;
2541 retval
= cluster_push_x(vp
, &cl
, newEOF
, flags
);
2544 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 40)) | DBG_FUNC_END
,
2545 retval
, 0, io_resid
, 0, 0);
2551 cluster_read(vnode_t vp
, struct uio
*uio
, off_t filesize
, int xflags
)
2564 if (vp
->v_flag
& VNOCACHE_DATA
)
2565 flags
|= IO_NOCACHE
;
2566 if (vp
->v_flag
& VRAOFF
)
2569 if (!((flags
& IO_NOCACHE
) && UIO_SEG_IS_USER_SPACE(uio
->uio_segflg
))) {
2571 * go do a read through the cache if one of the following is true....
2572 * NOCACHE is not true
2573 * the uio request doesn't target USERSPACE
2575 return (cluster_read_x(vp
, uio
, filesize
, flags
));
2579 if (IS_VALID_UIO_SEGFLG(uio
->uio_segflg
) == 0) {
2580 panic("%s :%d - invalid uio_segflg\n", __FILE__
, __LINE__
);
2582 #endif /* LP64_DEBUG */
2584 while (uio_resid(uio
) && uio
->uio_offset
< filesize
&& retval
== 0) {
2585 user_size_t iov_len
;
2586 user_addr_t iov_base
;
2589 * we know we have a resid, so this is safe
2590 * skip over any emtpy vectors
2592 uio_update(uio
, (user_size_t
)0);
2594 iov_len
= uio_curriovlen(uio
);
2595 iov_base
= uio_curriovbase(uio
);
2597 upl_size
= PAGE_SIZE
;
2598 upl_flags
= UPL_QUERY_OBJECT_TYPE
;
2600 // LP64todo - fix this!
2601 if ((vm_map_get_upl(current_map(),
2602 (vm_map_offset_t
)(iov_base
& ~((user_addr_t
)PAGE_MASK
)),
2603 &upl_size
, &upl
, NULL
, NULL
, &upl_flags
, 0)) != KERN_SUCCESS
) {
2605 * the user app must have passed in an invalid address
2611 * We check every vector target but if it is physically
2612 * contiguous space, we skip the sanity checks.
2614 if (upl_flags
& UPL_PHYS_CONTIG
) {
2615 retval
= cluster_phys_read(vp
, uio
, filesize
);
2617 else if (uio_resid(uio
) < PAGE_SIZE
) {
2619 * we're here because we're don't have a physically contiguous target buffer
2620 * go do a read through the cache if
2621 * the total xfer size is less than a page...
2623 return (cluster_read_x(vp
, uio
, filesize
, flags
));
2625 // LP64todo - fix this!
2626 else if (((int)uio
->uio_offset
& PAGE_MASK
) || (CAST_DOWN(int, iov_base
) & PAGE_MASK
)) {
2627 if (((int)uio
->uio_offset
& PAGE_MASK
) == (CAST_DOWN(int, iov_base
) & PAGE_MASK
)) {
2629 * Bring the file offset read up to a pagesize boundary
2630 * this will also bring the base address to a page boundary
2631 * since they both are currently on the same offset within a page
2632 * note: if we get here, uio->uio_resid is greater than PAGE_SIZE
2633 * so the computed clip_size must always be less than the current uio_resid
2635 clip_size
= (PAGE_SIZE
- (int)(uio
->uio_offset
& PAGE_MASK_64
));
2638 * Fake the resid going into the cluster_read_x call
2639 * and restore it on the way out.
2641 prev_resid
= uio_resid(uio
);
2642 // LP64todo - fix this
2643 uio_setresid(uio
, clip_size
);
2645 retval
= cluster_read_x(vp
, uio
, filesize
, flags
);
2647 uio_setresid(uio
, prev_resid
- (clip_size
- uio_resid(uio
)));
2650 * can't get both the file offset and the buffer offset aligned to a page boundary
2651 * so fire an I/O through the cache for this entire vector
2653 // LP64todo - fix this!
2654 clip_size
= iov_len
;
2655 prev_resid
= uio_resid(uio
);
2656 uio_setresid(uio
, clip_size
);
2658 retval
= cluster_read_x(vp
, uio
, filesize
, flags
);
2660 uio_setresid(uio
, prev_resid
- (clip_size
- uio_resid(uio
)));
2664 * If we come in here, we know the offset into
2665 * the file is on a pagesize boundary
2667 max_io_size
= filesize
- uio
->uio_offset
;
2668 // LP64todo - fix this
2669 clip_size
= uio_resid(uio
);
2670 if (iov_len
< clip_size
)
2671 clip_size
= iov_len
;
2672 if (max_io_size
< clip_size
)
2673 clip_size
= (int)max_io_size
;
2675 if (clip_size
< PAGE_SIZE
) {
2677 * Take care of the tail end of the read in this vector.
2679 // LP64todo - fix this
2680 prev_resid
= uio_resid(uio
);
2681 uio_setresid(uio
, clip_size
);
2683 retval
= cluster_read_x(vp
, uio
, filesize
, flags
);
2685 uio_setresid(uio
, prev_resid
- (clip_size
- uio_resid(uio
)));
2687 /* round clip_size down to a multiple of pagesize */
2688 clip_size
= clip_size
& ~(PAGE_MASK
);
2689 // LP64todo - fix this
2690 prev_resid
= uio_resid(uio
);
2691 uio_setresid(uio
, clip_size
);
2693 retval
= cluster_nocopy_read(vp
, uio
, filesize
);
2695 if ((retval
==0) && uio_resid(uio
))
2696 retval
= cluster_read_x(vp
, uio
, filesize
, flags
);
2698 uio_setresid(uio
, prev_resid
- (clip_size
- uio_resid(uio
)));
2707 cluster_read_x(vnode_t vp
, struct uio
*uio
, off_t filesize
, int flags
)
2709 upl_page_info_t
*pl
;
2711 vm_offset_t upl_offset
;
2720 off_t last_ioread_offset
;
2721 off_t last_request_offset
;
2722 u_int size_of_prefetch
;
2727 u_int max_rd_size
= MAX_UPL_TRANSFER
* PAGE_SIZE
;
2728 u_int rd_ahead_enabled
= 1;
2729 u_int prefetch_enabled
= 1;
2730 struct cl_readahead
* rap
;
2731 struct clios iostate
;
2732 struct cl_extent extent
;
2734 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 32)) | DBG_FUNC_START
,
2735 (int)uio
->uio_offset
, uio_resid(uio
), (int)filesize
, 0, 0);
2737 // LP64todo - fix this
2738 last_request_offset
= uio
->uio_offset
+ uio_resid(uio
);
2740 if ((flags
& (IO_RAOFF
|IO_NOCACHE
)) ||
2741 ((last_request_offset
& ~PAGE_MASK_64
) == (uio
->uio_offset
& ~PAGE_MASK_64
))) {
2742 rd_ahead_enabled
= 0;
2745 if (cluster_hard_throttle_on(vp
)) {
2746 rd_ahead_enabled
= 0;
2747 prefetch_enabled
= 0;
2749 max_rd_size
= HARD_THROTTLE_MAXSIZE
;
2751 if ((rap
= cluster_get_rap(vp
)) == NULL
)
2752 rd_ahead_enabled
= 0;
2754 if (last_request_offset
> filesize
)
2755 last_request_offset
= filesize
;
2756 extent
.b_addr
= uio
->uio_offset
/ PAGE_SIZE_64
;
2757 extent
.e_addr
= (last_request_offset
- 1) / PAGE_SIZE_64
;
2759 if (rap
!= NULL
&& rap
->cl_ralen
&& (rap
->cl_lastr
== extent
.b_addr
|| (rap
->cl_lastr
+ 1) == extent
.b_addr
)) {
2761 * determine if we already have a read-ahead in the pipe courtesy of the
2762 * last read systemcall that was issued...
2763 * if so, pick up it's extent to determine where we should start
2764 * with respect to any read-ahead that might be necessary to
2765 * garner all the data needed to complete this read systemcall
2767 last_ioread_offset
= (rap
->cl_maxra
* PAGE_SIZE_64
) + PAGE_SIZE_64
;
2769 if (last_ioread_offset
< uio
->uio_offset
)
2770 last_ioread_offset
= (off_t
)0;
2771 else if (last_ioread_offset
> last_request_offset
)
2772 last_ioread_offset
= last_request_offset
;
2774 last_ioread_offset
= (off_t
)0;
2776 while (uio_resid(uio
) && uio
->uio_offset
< filesize
&& retval
== 0) {
2778 * compute the size of the upl needed to encompass
2779 * the requested read... limit each call to cluster_io
2780 * to the maximum UPL size... cluster_io will clip if
2781 * this exceeds the maximum io_size for the device,
2782 * make sure to account for
2783 * a starting offset that's not page aligned
2785 start_offset
= (int)(uio
->uio_offset
& PAGE_MASK_64
);
2786 upl_f_offset
= uio
->uio_offset
- (off_t
)start_offset
;
2787 max_size
= filesize
- uio
->uio_offset
;
2789 // LP64todo - fix this!
2790 if ((off_t
)((unsigned int)uio_resid(uio
)) < max_size
)
2791 io_size
= uio_resid(uio
);
2795 if (!(flags
& IO_NOCACHE
)) {
2802 * if we keep finding the pages we need already in the cache, then
2803 * don't bother to call cluster_rd_prefetch since it costs CPU cycles
2804 * to determine that we have all the pages we need... once we miss in
2805 * the cache and have issued an I/O, than we'll assume that we're likely
2806 * to continue to miss in the cache and it's to our advantage to try and prefetch
2808 if (last_request_offset
&& last_ioread_offset
&& (size_of_prefetch
= (last_request_offset
- last_ioread_offset
))) {
2809 if ((last_ioread_offset
- uio
->uio_offset
) <= max_rd_size
&& prefetch_enabled
) {
2811 * we've already issued I/O for this request and
2812 * there's still work to do and
2813 * our prefetch stream is running dry, so issue a
2814 * pre-fetch I/O... the I/O latency will overlap
2815 * with the copying of the data
2817 if (size_of_prefetch
> max_rd_size
)
2818 size_of_prefetch
= max_rd_size
;
2820 size_of_prefetch
= cluster_rd_prefetch(vp
, last_ioread_offset
, size_of_prefetch
, filesize
);
2822 last_ioread_offset
+= (off_t
)(size_of_prefetch
* PAGE_SIZE
);
2824 if (last_ioread_offset
> last_request_offset
)
2825 last_ioread_offset
= last_request_offset
;
2829 * limit the size of the copy we're about to do so that
2830 * we can notice that our I/O pipe is running dry and
2831 * get the next I/O issued before it does go dry
2833 if (last_ioread_offset
&& io_size
> ((MAX_UPL_TRANSFER
* PAGE_SIZE
) / 4))
2834 io_resid
= ((MAX_UPL_TRANSFER
* PAGE_SIZE
) / 4);
2838 io_requested
= io_resid
;
2840 retval
= cluster_copy_ubc_data(vp
, uio
, &io_resid
, 0);
2842 io_size
-= (io_requested
- io_resid
);
2844 if (retval
|| io_resid
)
2846 * if we run into a real error or
2847 * a page that is not in the cache
2848 * we need to leave streaming mode
2852 if ((io_size
== 0 || last_ioread_offset
== last_request_offset
) && rd_ahead_enabled
) {
2854 * we're already finished the I/O for this read request
2855 * let's see if we should do a read-ahead
2857 cluster_rd_ahead(vp
, &extent
, filesize
, rap
);
2864 if (extent
.e_addr
< rap
->cl_lastr
)
2866 rap
->cl_lastr
= extent
.e_addr
;
2870 start_offset
= (int)(uio
->uio_offset
& PAGE_MASK_64
);
2871 upl_f_offset
= uio
->uio_offset
- (off_t
)start_offset
;
2872 max_size
= filesize
- uio
->uio_offset
;
2874 if (io_size
> max_rd_size
)
2875 io_size
= max_rd_size
;
2877 upl_size
= (start_offset
+ io_size
+ (PAGE_SIZE
- 1)) & ~PAGE_MASK
;
2879 if (upl_size
> (MAX_UPL_TRANSFER
* PAGE_SIZE
) / 4)
2880 upl_size
= (MAX_UPL_TRANSFER
* PAGE_SIZE
) / 4;
2881 pages_in_upl
= upl_size
/ PAGE_SIZE
;
2883 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 33)) | DBG_FUNC_START
,
2884 (int)upl
, (int)upl_f_offset
, upl_size
, start_offset
, 0);
2886 kret
= ubc_create_upl(vp
,
2892 if (kret
!= KERN_SUCCESS
)
2893 panic("cluster_read: failed to get pagelist");
2895 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 33)) | DBG_FUNC_END
,
2896 (int)upl
, (int)upl_f_offset
, upl_size
, start_offset
, 0);
2899 * scan from the beginning of the upl looking for the first
2900 * non-valid page.... this will become the first page in
2901 * the request we're going to make to 'cluster_io'... if all
2902 * of the pages are valid, we won't call through to 'cluster_io'
2904 for (start_pg
= 0; start_pg
< pages_in_upl
; start_pg
++) {
2905 if (!upl_valid_page(pl
, start_pg
))
2910 * scan from the starting invalid page looking for a valid
2911 * page before the end of the upl is reached, if we
2912 * find one, then it will be the last page of the request to
2915 for (last_pg
= start_pg
; last_pg
< pages_in_upl
; last_pg
++) {
2916 if (upl_valid_page(pl
, last_pg
))
2919 iostate
.io_completed
= 0;
2920 iostate
.io_issued
= 0;
2921 iostate
.io_error
= 0;
2922 iostate
.io_wanted
= 0;
2924 if (start_pg
< last_pg
) {
2926 * we found a range of 'invalid' pages that must be filled
2927 * if the last page in this range is the last page of the file
2928 * we may have to clip the size of it to keep from reading past
2929 * the end of the last physical block associated with the file
2931 upl_offset
= start_pg
* PAGE_SIZE
;
2932 io_size
= (last_pg
- start_pg
) * PAGE_SIZE
;
2934 if ((upl_f_offset
+ upl_offset
+ io_size
) > filesize
)
2935 io_size
= filesize
- (upl_f_offset
+ upl_offset
);
2938 * issue an asynchronous read to cluster_io
2941 error
= cluster_io(vp
, upl
, upl_offset
, upl_f_offset
+ upl_offset
,
2942 io_size
, CL_READ
| CL_ASYNC
, (buf_t
)NULL
, &iostate
);
2946 * if the read completed successfully, or there was no I/O request
2947 * issued, than copy the data into user land via 'cluster_upl_copy_data'
2948 * we'll first add on any 'valid'
2949 * pages that were present in the upl when we acquired it.
2953 for (uio_last
= last_pg
; uio_last
< pages_in_upl
; uio_last
++) {
2954 if (!upl_valid_page(pl
, uio_last
))
2958 * compute size to transfer this round, if uio->uio_resid is
2959 * still non-zero after this attempt, we'll loop around and
2960 * set up for another I/O.
2962 val_size
= (uio_last
* PAGE_SIZE
) - start_offset
;
2964 if (val_size
> max_size
)
2965 val_size
= max_size
;
2967 if (val_size
> uio_resid(uio
))
2968 // LP64todo - fix this
2969 val_size
= uio_resid(uio
);
2971 if (last_ioread_offset
== 0)
2972 last_ioread_offset
= uio
->uio_offset
+ val_size
;
2974 if ((size_of_prefetch
= (last_request_offset
- last_ioread_offset
)) && prefetch_enabled
) {
2976 * if there's still I/O left to do for this request, and...
2977 * we're not in hard throttle mode, then issue a
2978 * pre-fetch I/O... the I/O latency will overlap
2979 * with the copying of the data
2981 size_of_prefetch
= cluster_rd_prefetch(vp
, last_ioread_offset
, size_of_prefetch
, filesize
);
2983 last_ioread_offset
+= (off_t
)(size_of_prefetch
* PAGE_SIZE
);
2985 if (last_ioread_offset
> last_request_offset
)
2986 last_ioread_offset
= last_request_offset
;
2988 } else if ((uio
->uio_offset
+ val_size
) == last_request_offset
) {
2990 * this transfer will finish this request, so...
2991 * let's try to read ahead if we're in
2992 * a sequential access pattern and we haven't
2993 * explicitly disabled it
2995 if (rd_ahead_enabled
)
2996 cluster_rd_ahead(vp
, &extent
, filesize
, rap
);
2999 if (extent
.e_addr
< rap
->cl_lastr
)
3001 rap
->cl_lastr
= extent
.e_addr
;
3004 lck_mtx_lock(cl_mtxp
);
3006 while (iostate
.io_issued
!= iostate
.io_completed
) {
3007 iostate
.io_wanted
= 1;
3008 msleep((caddr_t
)&iostate
.io_wanted
, cl_mtxp
, PRIBIO
+ 1, "cluster_read_x", 0);
3010 lck_mtx_unlock(cl_mtxp
);
3012 if (iostate
.io_error
)
3013 error
= iostate
.io_error
;
3015 retval
= cluster_copy_upl_data(uio
, upl
, start_offset
, val_size
);
3017 if (start_pg
< last_pg
) {
3019 * compute the range of pages that we actually issued an I/O for
3020 * and either commit them as valid if the I/O succeeded
3021 * or abort them if the I/O failed
3023 io_size
= (last_pg
- start_pg
) * PAGE_SIZE
;
3025 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 35)) | DBG_FUNC_START
,
3026 (int)upl
, start_pg
* PAGE_SIZE
, io_size
, error
, 0);
3028 if (error
|| (flags
& IO_NOCACHE
))
3029 ubc_upl_abort_range(upl
, start_pg
* PAGE_SIZE
, io_size
,
3030 UPL_ABORT_DUMP_PAGES
| UPL_ABORT_FREE_ON_EMPTY
);
3032 ubc_upl_commit_range(upl
, start_pg
* PAGE_SIZE
, io_size
,
3033 UPL_COMMIT_CLEAR_DIRTY
|
3034 UPL_COMMIT_FREE_ON_EMPTY
|
3035 UPL_COMMIT_INACTIVATE
);
3037 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 35)) | DBG_FUNC_END
,
3038 (int)upl
, start_pg
* PAGE_SIZE
, io_size
, error
, 0);
3040 if ((last_pg
- start_pg
) < pages_in_upl
) {
3045 * the set of pages that we issued an I/O for did not encompass
3046 * the entire upl... so just release these without modifying
3050 ubc_upl_abort_range(upl
, 0, upl_size
, UPL_ABORT_FREE_ON_EMPTY
);
3052 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 35)) | DBG_FUNC_START
,
3053 (int)upl
, -1, pages_in_upl
- (last_pg
- start_pg
), 0, 0);
3057 * we found some already valid pages at the beginning of
3058 * the upl commit these back to the inactive list with
3061 for (cur_pg
= 0; cur_pg
< start_pg
; cur_pg
++) {
3062 commit_flags
= UPL_COMMIT_FREE_ON_EMPTY
3063 | UPL_COMMIT_INACTIVATE
;
3065 if (upl_dirty_page(pl
, cur_pg
))
3066 commit_flags
|= UPL_COMMIT_SET_DIRTY
;
3068 if ( !(commit_flags
& UPL_COMMIT_SET_DIRTY
) && (flags
& IO_NOCACHE
))
3069 ubc_upl_abort_range(upl
, cur_pg
* PAGE_SIZE
, PAGE_SIZE
,
3070 UPL_ABORT_DUMP_PAGES
| UPL_ABORT_FREE_ON_EMPTY
);
3072 ubc_upl_commit_range(upl
, cur_pg
* PAGE_SIZE
,
3073 PAGE_SIZE
, commit_flags
);
3076 if (last_pg
< uio_last
) {
3078 * we found some already valid pages immediately after the
3079 * pages we issued I/O for, commit these back to the
3080 * inactive list with reference cleared
3082 for (cur_pg
= last_pg
; cur_pg
< uio_last
; cur_pg
++) {
3083 commit_flags
= UPL_COMMIT_FREE_ON_EMPTY
3084 | UPL_COMMIT_INACTIVATE
;
3086 if (upl_dirty_page(pl
, cur_pg
))
3087 commit_flags
|= UPL_COMMIT_SET_DIRTY
;
3089 if ( !(commit_flags
& UPL_COMMIT_SET_DIRTY
) && (flags
& IO_NOCACHE
))
3090 ubc_upl_abort_range(upl
, cur_pg
* PAGE_SIZE
, PAGE_SIZE
,
3091 UPL_ABORT_DUMP_PAGES
| UPL_ABORT_FREE_ON_EMPTY
);
3093 ubc_upl_commit_range(upl
, cur_pg
* PAGE_SIZE
,
3094 PAGE_SIZE
, commit_flags
);
3097 if (uio_last
< pages_in_upl
) {
3099 * there were some invalid pages beyond the valid pages
3100 * that we didn't issue an I/O for, just release them
3103 ubc_upl_abort_range(upl
, uio_last
* PAGE_SIZE
,
3104 (pages_in_upl
- uio_last
) * PAGE_SIZE
, UPL_ABORT_FREE_ON_EMPTY
);
3107 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 35)) | DBG_FUNC_END
,
3108 (int)upl
, -1, -1, 0, 0);
3114 if ( uio_resid(uio
) ) {
3115 if (cluster_hard_throttle_on(vp
)) {
3116 rd_ahead_enabled
= 0;
3117 prefetch_enabled
= 0;
3119 max_rd_size
= HARD_THROTTLE_MAXSIZE
;
3122 rd_ahead_enabled
= 1;
3123 prefetch_enabled
= 1;
3125 max_rd_size
= MAX_UPL_TRANSFER
* PAGE_SIZE
;
3130 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 32)) | DBG_FUNC_END
,
3131 (int)uio
->uio_offset
, uio_resid(uio
), rap
->cl_lastr
, retval
, 0);
3133 lck_mtx_unlock(&rap
->cl_lockr
);
3135 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 32)) | DBG_FUNC_END
,
3136 (int)uio
->uio_offset
, uio_resid(uio
), 0, retval
, 0);
3144 cluster_nocopy_read(vnode_t vp
, struct uio
*uio
, off_t filesize
)
3147 upl_page_info_t
*pl
;
3148 vm_offset_t upl_offset
;
3152 int upl_needed_size
;
3157 int force_data_sync
;
3159 int no_zero_fill
= 0;
3161 struct clios iostate
;
3162 u_int max_rd_size
= MAX_UPL_TRANSFER
* PAGE_SIZE
;
3163 u_int max_rd_ahead
= MAX_UPL_TRANSFER
* PAGE_SIZE
* 2;
3166 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 70)) | DBG_FUNC_START
,
3167 (int)uio
->uio_offset
, uio_resid(uio
), (int)filesize
, 0, 0);
3170 * When we enter this routine, we know
3171 * -- the offset into the file is on a pagesize boundary
3172 * -- the resid is a page multiple
3173 * -- the resid will not exceed iov_len
3176 iostate
.io_completed
= 0;
3177 iostate
.io_issued
= 0;
3178 iostate
.io_error
= 0;
3179 iostate
.io_wanted
= 0;
3181 while (uio_resid(uio
) && uio
->uio_offset
< filesize
&& retval
== 0) {
3182 user_addr_t iov_base
;
3184 if (cluster_hard_throttle_on(vp
)) {
3185 max_rd_size
= HARD_THROTTLE_MAXSIZE
;
3186 max_rd_ahead
= HARD_THROTTLE_MAXSIZE
- 1;
3188 max_rd_size
= MAX_UPL_TRANSFER
* PAGE_SIZE
;
3189 max_rd_ahead
= MAX_UPL_TRANSFER
* PAGE_SIZE
* 8;
3191 max_io_size
= filesize
- uio
->uio_offset
;
3193 // LP64todo - fix this
3194 if (max_io_size
< (off_t
)((unsigned int)uio_resid(uio
)))
3195 io_size
= max_io_size
;
3197 io_size
= uio_resid(uio
);
3200 * First look for pages already in the cache
3201 * and move them to user space.
3203 retval
= cluster_copy_ubc_data(vp
, uio
, &io_size
, 0);
3207 * we may have already spun some portion of this request
3208 * off as async requests... we need to wait for the I/O
3209 * to complete before returning
3211 goto wait_for_reads
;
3214 * If we are already finished with this read, then return
3218 * we may have already spun some portion of this request
3219 * off as async requests... we need to wait for the I/O
3220 * to complete before returning
3222 goto wait_for_reads
;
3224 max_io_size
= io_size
;
3226 if (max_io_size
> max_rd_size
)
3227 max_io_size
= max_rd_size
;
3231 ubc_range_op(vp
, uio
->uio_offset
, uio
->uio_offset
+ max_io_size
, UPL_ROP_ABSENT
, &io_size
);
3235 * we may have already spun some portion of this request
3236 * off as async requests... we need to wait for the I/O
3237 * to complete before returning
3239 goto wait_for_reads
;
3241 iov_base
= uio_curriovbase(uio
);
3243 // LP64todo - fix this!
3244 upl_offset
= CAST_DOWN(vm_offset_t
, iov_base
) & PAGE_MASK
;
3245 upl_needed_size
= (upl_offset
+ io_size
+ (PAGE_SIZE
-1)) & ~PAGE_MASK
;
3247 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 72)) | DBG_FUNC_START
,
3248 (int)upl_offset
, upl_needed_size
, (int)iov_base
, io_size
, 0);
3250 if (upl_offset
== 0 && ((io_size
& PAGE_MASK
) == 0)) {
3252 abort_flag
= UPL_ABORT_DUMP_PAGES
| UPL_ABORT_FREE_ON_EMPTY
;
3255 abort_flag
= UPL_ABORT_FREE_ON_EMPTY
;
3257 for (force_data_sync
= 0; force_data_sync
< 3; force_data_sync
++) {
3259 upl_size
= upl_needed_size
;
3260 upl_flags
= UPL_FILE_IO
| UPL_NO_SYNC
| UPL_SET_INTERNAL
| UPL_SET_LITE
| UPL_SET_IO_WIRE
;
3263 upl_flags
|= UPL_NOZEROFILL
;
3264 if (force_data_sync
)
3265 upl_flags
|= UPL_FORCE_DATA_SYNC
;
3267 // LP64todo - fix this!
3268 kret
= vm_map_create_upl(current_map(),
3269 (vm_map_offset_t
)(iov_base
& ~((user_addr_t
)PAGE_MASK
)),
3270 &upl_size
, &upl
, NULL
, &pages_in_pl
, &upl_flags
);
3272 if (kret
!= KERN_SUCCESS
) {
3273 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 72)) | DBG_FUNC_END
,
3274 (int)upl_offset
, upl_size
, io_size
, kret
, 0);
3276 * cluster_nocopy_read: failed to get pagelist
3278 * we may have already spun some portion of this request
3279 * off as async requests... we need to wait for the I/O
3280 * to complete before returning
3282 goto wait_for_reads
;
3284 pages_in_pl
= upl_size
/ PAGE_SIZE
;
3285 pl
= UPL_GET_INTERNAL_PAGE_LIST(upl
);
3287 for (i
= 0; i
< pages_in_pl
; i
++) {
3288 if (!upl_valid_page(pl
, i
))
3291 if (i
== pages_in_pl
)
3294 ubc_upl_abort_range(upl
, (upl_offset
& ~PAGE_MASK
), upl_size
, abort_flag
);
3296 if (force_data_sync
>= 3) {
3297 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 72)) | DBG_FUNC_END
,
3298 (int)upl_offset
, upl_size
, io_size
, kret
, 0);
3300 goto wait_for_reads
;
3303 * Consider the possibility that upl_size wasn't satisfied.
3305 if (upl_size
!= upl_needed_size
)
3306 io_size
= (upl_size
- (int)upl_offset
) & ~PAGE_MASK
;
3309 ubc_upl_abort_range(upl
, (upl_offset
& ~PAGE_MASK
), upl_size
, abort_flag
);
3310 goto wait_for_reads
;
3312 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 72)) | DBG_FUNC_END
,
3313 (int)upl_offset
, upl_size
, io_size
, kret
, 0);
3316 * request asynchronously so that we can overlap
3317 * the preparation of the next I/O
3318 * if there are already too many outstanding reads
3319 * wait until some have completed before issuing the next read
3321 lck_mtx_lock(cl_mtxp
);
3323 while ((iostate
.io_issued
- iostate
.io_completed
) > max_rd_ahead
) {
3324 iostate
.io_wanted
= 1;
3325 msleep((caddr_t
)&iostate
.io_wanted
, cl_mtxp
, PRIBIO
+ 1, "cluster_nocopy_read", 0);
3327 lck_mtx_unlock(cl_mtxp
);
3329 if (iostate
.io_error
) {
3331 * one of the earlier reads we issued ran into a hard error
3332 * don't issue any more reads, cleanup the UPL
3333 * that was just created but not used, then
3334 * go wait for any other reads to complete before
3335 * returning the error to the caller
3337 ubc_upl_abort_range(upl
, (upl_offset
& ~PAGE_MASK
), upl_size
, abort_flag
);
3339 goto wait_for_reads
;
3341 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 73)) | DBG_FUNC_START
,
3342 (int)upl
, (int)upl_offset
, (int)uio
->uio_offset
, io_size
, 0);
3344 retval
= cluster_io(vp
, upl
, upl_offset
, uio
->uio_offset
, io_size
,
3345 CL_PRESERVE
| CL_COMMIT
| CL_READ
| CL_ASYNC
| CL_NOZERO
,
3346 (buf_t
)NULL
, &iostate
);
3349 * update the uio structure
3351 uio_update(uio
, (user_size_t
)io_size
);
3353 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 73)) | DBG_FUNC_END
,
3354 (int)upl
, (int)uio
->uio_offset
, (int)uio_resid(uio
), retval
, 0);
3360 * make sure all async reads that are part of this stream
3361 * have completed before we return
3363 lck_mtx_lock(cl_mtxp
);
3365 while (iostate
.io_issued
!= iostate
.io_completed
) {
3366 iostate
.io_wanted
= 1;
3367 msleep((caddr_t
)&iostate
.io_wanted
, cl_mtxp
, PRIBIO
+ 1, "cluster_nocopy_read", 0);
3369 lck_mtx_unlock(cl_mtxp
);
3371 if (iostate
.io_error
)
3372 retval
= iostate
.io_error
;
3374 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 70)) | DBG_FUNC_END
,
3375 (int)uio
->uio_offset
, (int)uio_resid(uio
), 6, retval
, 0);
3382 cluster_phys_read(vnode_t vp
, struct uio
*uio
, off_t filesize
)
3384 upl_page_info_t
*pl
;
3386 vm_offset_t upl_offset
;
3390 user_size_t iov_len
;
3391 user_addr_t iov_base
;
3394 int upl_needed_size
;
3398 struct clios iostate
;
3402 devblocksize
= vp
->v_mount
->mnt_devblocksize
;
3404 * When we enter this routine, we know
3405 * -- the resid will not exceed iov_len
3406 * -- the target address is physically contiguous
3410 if (IS_VALID_UIO_SEGFLG(uio
->uio_segflg
) == 0) {
3411 panic("%s :%d - invalid uio_segflg\n", __FILE__
, __LINE__
);
3413 #endif /* LP64_DEBUG */
3415 iov_len
= uio_curriovlen(uio
);
3416 iov_base
= uio_curriovbase(uio
);
3418 max_size
= filesize
- uio
->uio_offset
;
3420 // LP64todo - fix this!
3421 if (max_size
< 0 || (u_int64_t
)max_size
> iov_len
)
3426 // LP64todo - fix this!
3427 upl_offset
= CAST_DOWN(vm_offset_t
, iov_base
) & PAGE_MASK
;
3428 upl_needed_size
= upl_offset
+ io_size
;
3432 upl_size
= upl_needed_size
;
3433 upl_flags
= UPL_FILE_IO
| UPL_NO_SYNC
| UPL_CLEAN_IN_PLACE
| UPL_SET_INTERNAL
| UPL_SET_LITE
| UPL_SET_IO_WIRE
;
3435 kret
= vm_map_get_upl(current_map(),
3436 (vm_map_offset_t
)(iov_base
& ~((user_addr_t
)PAGE_MASK
)),
3437 &upl_size
, &upl
, NULL
, &pages_in_pl
, &upl_flags
, 0);
3439 if (kret
!= KERN_SUCCESS
) {
3441 * cluster_phys_read: failed to get pagelist
3445 if (upl_size
< upl_needed_size
) {
3447 * The upl_size wasn't satisfied.
3449 ubc_upl_abort_range(upl
, 0, upl_size
, UPL_ABORT_FREE_ON_EMPTY
);
3453 pl
= ubc_upl_pageinfo(upl
);
3455 dst_paddr
= ((addr64_t
)upl_phys_page(pl
, 0) << 12) + (addr64_t
)upl_offset
;
3457 while (((uio
->uio_offset
& (devblocksize
- 1)) || io_size
< devblocksize
) && io_size
) {
3460 head_size
= devblocksize
- (int)(uio
->uio_offset
& (devblocksize
- 1));
3462 if (head_size
> io_size
)
3463 head_size
= io_size
;
3465 error
= cluster_align_phys_io(vp
, uio
, dst_paddr
, head_size
, CL_READ
);
3468 ubc_upl_abort_range(upl
, 0, upl_size
, UPL_ABORT_FREE_ON_EMPTY
);
3472 upl_offset
+= head_size
;
3473 dst_paddr
+= head_size
;
3474 io_size
-= head_size
;
3476 tail_size
= io_size
& (devblocksize
- 1);
3477 io_size
-= tail_size
;
3479 iostate
.io_completed
= 0;
3480 iostate
.io_issued
= 0;
3481 iostate
.io_error
= 0;
3482 iostate
.io_wanted
= 0;
3484 while (io_size
&& error
== 0) {
3487 if (io_size
> (MAX_UPL_TRANSFER
* PAGE_SIZE
))
3488 xsize
= MAX_UPL_TRANSFER
* PAGE_SIZE
;
3492 * request asynchronously so that we can overlap
3493 * the preparation of the next I/O... we'll do
3494 * the commit after all the I/O has completed
3495 * since its all issued against the same UPL
3496 * if there are already too many outstanding reads
3497 * wait until some have completed before issuing the next
3499 lck_mtx_lock(cl_mtxp
);
3501 while ((iostate
.io_issued
- iostate
.io_completed
) > (8 * MAX_UPL_TRANSFER
* PAGE_SIZE
)) {
3502 iostate
.io_wanted
= 1;
3503 msleep((caddr_t
)&iostate
.io_wanted
, cl_mtxp
, PRIBIO
+ 1, "cluster_phys_read", 0);
3505 lck_mtx_unlock(cl_mtxp
);
3507 error
= cluster_io(vp
, upl
, upl_offset
, uio
->uio_offset
, xsize
,
3508 CL_READ
| CL_NOZERO
| CL_DEV_MEMORY
| CL_ASYNC
,
3509 (buf_t
)NULL
, &iostate
);
3511 * The cluster_io read was issued successfully,
3512 * update the uio structure
3515 uio_update(uio
, (user_size_t
)xsize
);
3518 upl_offset
+= xsize
;
3523 * make sure all async reads that are part of this stream
3524 * have completed before we proceed
3526 lck_mtx_lock(cl_mtxp
);
3528 while (iostate
.io_issued
!= iostate
.io_completed
) {
3529 iostate
.io_wanted
= 1;
3530 msleep((caddr_t
)&iostate
.io_wanted
, cl_mtxp
, PRIBIO
+ 1, "cluster_phys_read", 0);
3532 lck_mtx_unlock(cl_mtxp
);
3534 if (iostate
.io_error
)
3535 error
= iostate
.io_error
;
3537 if (error
== 0 && tail_size
)
3538 error
= cluster_align_phys_io(vp
, uio
, dst_paddr
, tail_size
, CL_READ
);
3541 * just release our hold on the physically contiguous
3542 * region without changing any state
3544 ubc_upl_abort_range(upl
, 0, upl_size
, UPL_ABORT_FREE_ON_EMPTY
);
3551 * generate advisory I/O's in the largest chunks possible
3552 * the completed pages will be released into the VM cache
3555 advisory_read(vnode_t vp
, off_t filesize
, off_t f_offset
, int resid
)
3557 upl_page_info_t
*pl
;
3559 vm_offset_t upl_offset
;
3573 if ( !UBCINFOEXISTS(vp
))
3576 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 60)) | DBG_FUNC_START
,
3577 (int)f_offset
, resid
, (int)filesize
, 0, 0);
3579 while (resid
&& f_offset
< filesize
&& retval
== 0) {
3581 * compute the size of the upl needed to encompass
3582 * the requested read... limit each call to cluster_io
3583 * to the maximum UPL size... cluster_io will clip if
3584 * this exceeds the maximum io_size for the device,
3585 * make sure to account for
3586 * a starting offset that's not page aligned
3588 start_offset
= (int)(f_offset
& PAGE_MASK_64
);
3589 upl_f_offset
= f_offset
- (off_t
)start_offset
;
3590 max_size
= filesize
- f_offset
;
3592 if (resid
< max_size
)
3597 upl_size
= (start_offset
+ io_size
+ (PAGE_SIZE
- 1)) & ~PAGE_MASK
;
3598 if (upl_size
> (MAX_UPL_TRANSFER
* PAGE_SIZE
))
3599 upl_size
= MAX_UPL_TRANSFER
* PAGE_SIZE
;
3603 * return the number of contiguously present pages in the cache
3604 * starting at upl_f_offset within the file
3606 ubc_range_op(vp
, upl_f_offset
, upl_f_offset
+ upl_size
, UPL_ROP_PRESENT
, &skip_range
);
3610 * skip over pages already present in the cache
3612 io_size
= skip_range
- start_offset
;
3614 f_offset
+= io_size
;
3617 if (skip_range
== upl_size
)
3620 * have to issue some real I/O
3621 * at this point, we know it's starting on a page boundary
3622 * because we've skipped over at least the first page in the request
3625 upl_f_offset
+= skip_range
;
3626 upl_size
-= skip_range
;
3628 pages_in_upl
= upl_size
/ PAGE_SIZE
;
3630 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 61)) | DBG_FUNC_START
,
3631 (int)upl
, (int)upl_f_offset
, upl_size
, start_offset
, 0);
3633 kret
= ubc_create_upl(vp
,
3638 UPL_RET_ONLY_ABSENT
| UPL_SET_LITE
);
3639 if (kret
!= KERN_SUCCESS
)
3644 * before we start marching forward, we must make sure we end on
3645 * a present page, otherwise we will be working with a freed
3648 for (last_pg
= pages_in_upl
- 1; last_pg
>= 0; last_pg
--) {
3649 if (upl_page_present(pl
, last_pg
))
3652 pages_in_upl
= last_pg
+ 1;
3655 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 61)) | DBG_FUNC_END
,
3656 (int)upl
, (int)upl_f_offset
, upl_size
, start_offset
, 0);
3659 for (last_pg
= 0; last_pg
< pages_in_upl
; ) {
3661 * scan from the beginning of the upl looking for the first
3662 * page that is present.... this will become the first page in
3663 * the request we're going to make to 'cluster_io'... if all
3664 * of the pages are absent, we won't call through to 'cluster_io'
3666 for (start_pg
= last_pg
; start_pg
< pages_in_upl
; start_pg
++) {
3667 if (upl_page_present(pl
, start_pg
))
3672 * scan from the starting present page looking for an absent
3673 * page before the end of the upl is reached, if we
3674 * find one, then it will terminate the range of pages being
3675 * presented to 'cluster_io'
3677 for (last_pg
= start_pg
; last_pg
< pages_in_upl
; last_pg
++) {
3678 if (!upl_page_present(pl
, last_pg
))
3682 if (last_pg
> start_pg
) {
3684 * we found a range of pages that must be filled
3685 * if the last page in this range is the last page of the file
3686 * we may have to clip the size of it to keep from reading past
3687 * the end of the last physical block associated with the file
3689 upl_offset
= start_pg
* PAGE_SIZE
;
3690 io_size
= (last_pg
- start_pg
) * PAGE_SIZE
;
3692 if ((upl_f_offset
+ upl_offset
+ io_size
) > filesize
)
3693 io_size
= filesize
- (upl_f_offset
+ upl_offset
);
3696 * issue an asynchronous read to cluster_io
3698 retval
= cluster_io(vp
, upl
, upl_offset
, upl_f_offset
+ upl_offset
, io_size
,
3699 CL_ASYNC
| CL_READ
| CL_COMMIT
| CL_AGE
, (buf_t
)NULL
, (struct clios
*)NULL
);
3705 ubc_upl_abort(upl
, 0);
3707 io_size
= upl_size
- start_offset
;
3709 if (io_size
> resid
)
3711 f_offset
+= io_size
;
3715 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 60)) | DBG_FUNC_END
,
3716 (int)f_offset
, resid
, retval
, 0, 0);
3723 cluster_push(vnode_t vp
, int flags
)
3726 struct cl_writebehind
*wbp
;
3728 if ( !UBCINFOEXISTS(vp
)) {
3729 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 53)) | DBG_FUNC_NONE
, (int)vp
, flags
, 0, -1, 0);
3732 /* return if deferred write is set */
3733 if (((unsigned int)vfs_flags(vp
->v_mount
) & MNT_DEFWRITE
) && (flags
& IO_DEFWRITE
)) {
3736 if ((wbp
= cluster_get_wbp(vp
, CLW_RETURNLOCKED
)) == NULL
) {
3737 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 53)) | DBG_FUNC_NONE
, (int)vp
, flags
, 0, -2, 0);
3740 if (wbp
->cl_number
== 0 && wbp
->cl_scmap
== NULL
) {
3741 lck_mtx_unlock(&wbp
->cl_lockw
);
3743 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 53)) | DBG_FUNC_NONE
, (int)vp
, flags
, 0, -3, 0);
3746 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 53)) | DBG_FUNC_START
,
3747 (int)wbp
->cl_scmap
, wbp
->cl_number
, flags
, 0, 0);
3749 if (wbp
->cl_scmap
) {
3750 sparse_cluster_push(wbp
, vp
, ubc_getsize(vp
), 1);
3754 retval
= cluster_try_push(wbp
, vp
, ubc_getsize(vp
), 0, 1);
3756 lck_mtx_unlock(&wbp
->cl_lockw
);
3758 if (flags
& IO_SYNC
)
3759 (void)vnode_waitforwrites(vp
, 0, 0, 0, (char *)"cluster_push");
3761 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 53)) | DBG_FUNC_END
,
3762 (int)wbp
->cl_scmap
, wbp
->cl_number
, retval
, 0, 0);
3768 __private_extern__
void
3769 cluster_release(struct ubc_info
*ubc
)
3771 struct cl_writebehind
*wbp
;
3772 struct cl_readahead
*rap
;
3774 if ((wbp
= ubc
->cl_wbehind
)) {
3776 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 81)) | DBG_FUNC_START
, (int)ubc
, (int)wbp
->cl_scmap
, wbp
->cl_scdirty
, 0, 0);
3779 vfs_drt_control(&(wbp
->cl_scmap
), 0);
3781 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 81)) | DBG_FUNC_START
, (int)ubc
, 0, 0, 0, 0);
3784 rap
= ubc
->cl_rahead
;
3787 lck_mtx_destroy(&wbp
->cl_lockw
, cl_mtx_grp
);
3788 FREE_ZONE((void *)wbp
, sizeof *wbp
, M_CLWRBEHIND
);
3790 if ((rap
= ubc
->cl_rahead
)) {
3791 lck_mtx_destroy(&rap
->cl_lockr
, cl_mtx_grp
);
3792 FREE_ZONE((void *)rap
, sizeof *rap
, M_CLRDAHEAD
);
3794 ubc
->cl_rahead
= NULL
;
3795 ubc
->cl_wbehind
= NULL
;
3797 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 81)) | DBG_FUNC_END
, (int)ubc
, (int)rap
, (int)wbp
, 0, 0);
3802 cluster_push_EOF(vnode_t vp
, off_t EOF
)
3804 struct cl_writebehind
*wbp
;
3806 wbp
= cluster_get_wbp(vp
, CLW_ALLOCATE
| CLW_RETURNLOCKED
);
3808 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 53)) | DBG_FUNC_START
,
3809 (int)wbp
->cl_scmap
, wbp
->cl_number
, (int)EOF
, 0, 0);
3812 sparse_cluster_push(wbp
, vp
, EOF
, 1);
3814 cluster_try_push(wbp
, vp
, EOF
, 0, 1);
3816 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 53)) | DBG_FUNC_END
,
3817 (int)wbp
->cl_scmap
, wbp
->cl_number
, 0, 0, 0);
3819 lck_mtx_unlock(&wbp
->cl_lockw
);
3824 cluster_try_push(struct cl_writebehind
*wbp
, vnode_t vp
, off_t EOF
, int can_delay
, int push_all
)
3831 struct cl_wextent l_clusters
[MAX_CLUSTERS
];
3834 * the write behind context exists and has
3835 * already been locked...
3837 * make a local 'sorted' copy of the clusters
3838 * and clear wbp->cl_number so that new clusters can
3841 for (cl_index
= 0; cl_index
< wbp
->cl_number
; cl_index
++) {
3842 for (min_index
= -1, cl_index1
= 0; cl_index1
< wbp
->cl_number
; cl_index1
++) {
3843 if (wbp
->cl_clusters
[cl_index1
].b_addr
== wbp
->cl_clusters
[cl_index1
].e_addr
)
3845 if (min_index
== -1)
3846 min_index
= cl_index1
;
3847 else if (wbp
->cl_clusters
[cl_index1
].b_addr
< wbp
->cl_clusters
[min_index
].b_addr
)
3848 min_index
= cl_index1
;
3850 if (min_index
== -1)
3852 l_clusters
[cl_index
].b_addr
= wbp
->cl_clusters
[min_index
].b_addr
;
3853 l_clusters
[cl_index
].e_addr
= wbp
->cl_clusters
[min_index
].e_addr
;
3854 l_clusters
[cl_index
].io_nocache
= wbp
->cl_clusters
[min_index
].io_nocache
;
3856 wbp
->cl_clusters
[min_index
].b_addr
= wbp
->cl_clusters
[min_index
].e_addr
;
3862 if (can_delay
&& cl_len
== MAX_CLUSTERS
) {
3866 * determine if we appear to be writing the file sequentially
3867 * if not, by returning without having pushed any clusters
3868 * we will cause this vnode to be pushed into the sparse cluster mechanism
3869 * used for managing more random I/O patterns
3871 * we know that we've got all clusters currently in use and the next write doesn't fit into one of them...
3872 * that's why we're in try_push with can_delay true...
3874 * check to make sure that all the clusters except the last one are 'full'... and that each cluster
3875 * is adjacent to the next (i.e. we're looking for sequential writes) they were sorted above
3876 * so we can just make a simple pass through, up to, but not including the last one...
3877 * note that e_addr is not inclusive, so it will be equal to the b_addr of the next cluster if they
3880 * we let the last one be partial as long as it was adjacent to the previous one...
3881 * we need to do this to deal with multi-threaded servers that might write an I/O or 2 out
3882 * of order... if this occurs at the tail of the last cluster, we don't want to fall into the sparse cluster world...
3884 for (i
= 0; i
< MAX_CLUSTERS
- 1; i
++) {
3885 if ((l_clusters
[i
].e_addr
- l_clusters
[i
].b_addr
) != MAX_UPL_TRANSFER
)
3887 if (l_clusters
[i
].e_addr
!= l_clusters
[i
+1].b_addr
)
3892 * drop the lock while we're firing off the I/Os...
3893 * this is safe since I'm working off of a private sorted copy
3894 * of the clusters, and I'm going to re-evaluate the public
3895 * state after I retake the lock
3897 lck_mtx_unlock(&wbp
->cl_lockw
);
3899 for (cl_index
= 0; cl_index
< cl_len
; cl_index
++) {
3901 struct cl_extent cl
;
3904 * try to push each cluster in turn...
3906 if (l_clusters
[cl_index
].io_nocache
)
3910 cl
.b_addr
= l_clusters
[cl_index
].b_addr
;
3911 cl
.e_addr
= l_clusters
[cl_index
].e_addr
;
3913 cluster_push_x(vp
, &cl
, EOF
, flags
);
3915 l_clusters
[cl_index
].b_addr
= 0;
3916 l_clusters
[cl_index
].e_addr
= 0;
3923 lck_mtx_lock(&wbp
->cl_lockw
);
3926 if (cl_len
> cl_pushed
) {
3928 * we didn't push all of the clusters, so
3929 * lets try to merge them back in to the vnode
3931 if ((MAX_CLUSTERS
- wbp
->cl_number
) < (cl_len
- cl_pushed
)) {
3933 * we picked up some new clusters while we were trying to
3934 * push the old ones... this can happen because I've dropped
3935 * the vnode lock... the sum of the
3936 * leftovers plus the new cluster count exceeds our ability
3937 * to represent them, so switch to the sparse cluster mechanism
3939 * collect the active public clusters...
3941 sparse_cluster_switch(wbp
, vp
, EOF
);
3943 for (cl_index
= 0, cl_index1
= 0; cl_index
< cl_len
; cl_index
++) {
3944 if (l_clusters
[cl_index
].b_addr
== l_clusters
[cl_index
].e_addr
)
3946 wbp
->cl_clusters
[cl_index1
].b_addr
= l_clusters
[cl_index
].b_addr
;
3947 wbp
->cl_clusters
[cl_index1
].e_addr
= l_clusters
[cl_index
].e_addr
;
3948 wbp
->cl_clusters
[cl_index1
].io_nocache
= l_clusters
[cl_index
].io_nocache
;
3953 * update the cluster count
3955 wbp
->cl_number
= cl_index1
;
3958 * and collect the original clusters that were moved into the
3959 * local storage for sorting purposes
3961 sparse_cluster_switch(wbp
, vp
, EOF
);
3965 * we've got room to merge the leftovers back in
3966 * just append them starting at the next 'hole'
3967 * represented by wbp->cl_number
3969 for (cl_index
= 0, cl_index1
= wbp
->cl_number
; cl_index
< cl_len
; cl_index
++) {
3970 if (l_clusters
[cl_index
].b_addr
== l_clusters
[cl_index
].e_addr
)
3973 wbp
->cl_clusters
[cl_index1
].b_addr
= l_clusters
[cl_index
].b_addr
;
3974 wbp
->cl_clusters
[cl_index1
].e_addr
= l_clusters
[cl_index
].e_addr
;
3975 wbp
->cl_clusters
[cl_index1
].io_nocache
= l_clusters
[cl_index
].io_nocache
;
3980 * update the cluster count
3982 wbp
->cl_number
= cl_index1
;
3985 return(MAX_CLUSTERS
- wbp
->cl_number
);
3991 cluster_push_x(vnode_t vp
, struct cl_extent
*cl
, off_t EOF
, int flags
)
3993 upl_page_info_t
*pl
;
3995 vm_offset_t upl_offset
;
4010 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 51)) | DBG_FUNC_START
,
4011 (int)cl
->b_addr
, (int)cl
->e_addr
, (int)EOF
, flags
, 0);
4013 if ((pages_in_upl
= (int)(cl
->e_addr
- cl
->b_addr
)) == 0) {
4014 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 51)) | DBG_FUNC_END
, 1, 0, 0, 0, 0);
4018 upl_size
= pages_in_upl
* PAGE_SIZE
;
4019 upl_f_offset
= (off_t
)(cl
->b_addr
* PAGE_SIZE_64
);
4021 if (upl_f_offset
+ upl_size
>= EOF
) {
4023 if (upl_f_offset
>= EOF
) {
4025 * must have truncated the file and missed
4026 * clearing a dangling cluster (i.e. it's completely
4027 * beyond the new EOF
4029 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 51)) | DBG_FUNC_END
, 1, 1, 0, 0, 0);
4033 size
= EOF
- upl_f_offset
;
4035 upl_size
= (size
+ (PAGE_SIZE
- 1)) & ~PAGE_MASK
;
4036 pages_in_upl
= upl_size
/ PAGE_SIZE
;
4040 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 41)) | DBG_FUNC_START
, upl_size
, size
, 0, 0, 0);
4043 * by asking for UPL_COPYOUT_FROM and UPL_RET_ONLY_DIRTY, we get the following desirable behavior
4045 * - only pages that are currently dirty are returned... these are the ones we need to clean
4046 * - the hardware dirty bit is cleared when the page is gathered into the UPL... the software dirty bit is set
4047 * - if we have to abort the I/O for some reason, the software dirty bit is left set since we didn't clean the page
4048 * - when we commit the page, the software dirty bit is cleared... the hardware dirty bit is untouched so that if
4049 * someone dirties this page while the I/O is in progress, we don't lose track of the new state
4051 * when the I/O completes, we no longer ask for an explicit clear of the DIRTY state (either soft or hard)
4054 if ((vp
->v_flag
& VNOCACHE_DATA
) || (flags
& IO_NOCACHE
))
4055 upl_flags
= UPL_COPYOUT_FROM
| UPL_RET_ONLY_DIRTY
| UPL_SET_LITE
| UPL_WILL_BE_DUMPED
;
4057 upl_flags
= UPL_COPYOUT_FROM
| UPL_RET_ONLY_DIRTY
| UPL_SET_LITE
;
4059 kret
= ubc_create_upl(vp
,
4065 if (kret
!= KERN_SUCCESS
)
4066 panic("cluster_push: failed to get pagelist");
4068 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 41)) | DBG_FUNC_END
, (int)upl
, upl_f_offset
, 0, 0, 0);
4071 * since we only asked for the dirty pages back
4072 * it's possible that we may only get a few or even none, so...
4073 * before we start marching forward, we must make sure we know
4074 * where the last present page is in the UPL, otherwise we could
4075 * end up working with a freed upl due to the FREE_ON_EMPTY semantics
4076 * employed by commit_range and abort_range.
4078 for (last_pg
= pages_in_upl
- 1; last_pg
>= 0; last_pg
--) {
4079 if (upl_page_present(pl
, last_pg
))
4082 pages_in_upl
= last_pg
+ 1;
4084 if (pages_in_upl
== 0) {
4085 ubc_upl_abort(upl
, 0);
4087 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 51)) | DBG_FUNC_END
, 1, 2, 0, 0, 0);
4091 for (last_pg
= 0; last_pg
< pages_in_upl
; ) {
4093 * find the next dirty page in the UPL
4094 * this will become the first page in the
4095 * next I/O to generate
4097 for (start_pg
= last_pg
; start_pg
< pages_in_upl
; start_pg
++) {
4098 if (upl_dirty_page(pl
, start_pg
))
4100 if (upl_page_present(pl
, start_pg
))
4102 * RET_ONLY_DIRTY will return non-dirty 'precious' pages
4103 * just release these unchanged since we're not going
4104 * to steal them or change their state
4106 ubc_upl_abort_range(upl
, start_pg
* PAGE_SIZE
, PAGE_SIZE
, UPL_ABORT_FREE_ON_EMPTY
);
4108 if (start_pg
>= pages_in_upl
)
4110 * done... no more dirty pages to push
4113 if (start_pg
> last_pg
)
4115 * skipped over some non-dirty pages
4117 size
-= ((start_pg
- last_pg
) * PAGE_SIZE
);
4120 * find a range of dirty pages to write
4122 for (last_pg
= start_pg
; last_pg
< pages_in_upl
; last_pg
++) {
4123 if (!upl_dirty_page(pl
, last_pg
))
4126 upl_offset
= start_pg
* PAGE_SIZE
;
4128 io_size
= min(size
, (last_pg
- start_pg
) * PAGE_SIZE
);
4130 io_flags
= CL_THROTTLE
| CL_COMMIT
;
4132 if ( !(flags
& IO_SYNC
))
4133 io_flags
|= CL_ASYNC
;
4135 retval
= cluster_io(vp
, upl
, upl_offset
, upl_f_offset
+ upl_offset
, io_size
,
4136 io_flags
, (buf_t
)NULL
, (struct clios
*)NULL
);
4138 if (error
== 0 && retval
)
4143 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 51)) | DBG_FUNC_END
, 1, 3, 0, 0, 0);
4150 * sparse_cluster_switch is called with the write behind lock held
4153 sparse_cluster_switch(struct cl_writebehind
*wbp
, vnode_t vp
, off_t EOF
)
4157 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 78)) | DBG_FUNC_START
, (int)vp
, (int)wbp
->cl_scmap
, wbp
->cl_scdirty
, 0, 0);
4159 if (wbp
->cl_scmap
== NULL
)
4160 wbp
->cl_scdirty
= 0;
4162 for (cl_index
= 0; cl_index
< wbp
->cl_number
; cl_index
++) {
4164 struct cl_extent cl
;
4166 for (cl
.b_addr
= wbp
->cl_clusters
[cl_index
].b_addr
; cl
.b_addr
< wbp
->cl_clusters
[cl_index
].e_addr
; cl
.b_addr
++) {
4168 if (ubc_page_op(vp
, (off_t
)(cl
.b_addr
* PAGE_SIZE_64
), 0, 0, &flags
) == KERN_SUCCESS
) {
4169 if (flags
& UPL_POP_DIRTY
) {
4170 cl
.e_addr
= cl
.b_addr
+ 1;
4172 sparse_cluster_add(wbp
, vp
, &cl
, EOF
);
4179 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 78)) | DBG_FUNC_END
, (int)vp
, (int)wbp
->cl_scmap
, wbp
->cl_scdirty
, 0, 0);
4184 * sparse_cluster_push is called with the write behind lock held
4187 sparse_cluster_push(struct cl_writebehind
*wbp
, vnode_t vp
, off_t EOF
, int push_all
)
4189 struct cl_extent cl
;
4193 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 79)) | DBG_FUNC_START
, (int)vp
, (int)wbp
->cl_scmap
, wbp
->cl_scdirty
, push_all
, 0);
4196 vfs_drt_control(&(wbp
->cl_scmap
), 1);
4199 if (vfs_drt_get_cluster(&(wbp
->cl_scmap
), &offset
, &length
) != KERN_SUCCESS
)
4202 cl
.b_addr
= (daddr64_t
)(offset
/ PAGE_SIZE_64
);
4203 cl
.e_addr
= (daddr64_t
)((offset
+ length
) / PAGE_SIZE_64
);
4205 wbp
->cl_scdirty
-= (int)(cl
.e_addr
- cl
.b_addr
);
4207 cluster_push_x(vp
, &cl
, EOF
, 0);
4212 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 79)) | DBG_FUNC_END
, (int)vp
, (int)wbp
->cl_scmap
, wbp
->cl_scdirty
, 0, 0);
4217 * sparse_cluster_add is called with the write behind lock held
4220 sparse_cluster_add(struct cl_writebehind
*wbp
, vnode_t vp
, struct cl_extent
*cl
, off_t EOF
)
4226 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 80)) | DBG_FUNC_START
, (int)wbp
->cl_scmap
, wbp
->cl_scdirty
, (int)cl
->b_addr
, (int)cl
->e_addr
, 0);
4228 offset
= (off_t
)(cl
->b_addr
* PAGE_SIZE_64
);
4229 length
= ((u_int
)(cl
->e_addr
- cl
->b_addr
)) * PAGE_SIZE
;
4231 while (vfs_drt_mark_pages(&(wbp
->cl_scmap
), offset
, length
, &new_dirty
) != KERN_SUCCESS
) {
4233 * no room left in the map
4234 * only a partial update was done
4235 * push out some pages and try again
4237 wbp
->cl_scdirty
+= new_dirty
;
4239 sparse_cluster_push(wbp
, vp
, EOF
, 0);
4241 offset
+= (new_dirty
* PAGE_SIZE_64
);
4242 length
-= (new_dirty
* PAGE_SIZE
);
4244 wbp
->cl_scdirty
+= new_dirty
;
4246 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 80)) | DBG_FUNC_END
, (int)vp
, (int)wbp
->cl_scmap
, wbp
->cl_scdirty
, 0, 0);
4251 cluster_align_phys_io(vnode_t vp
, struct uio
*uio
, addr64_t usr_paddr
, int xsize
, int flags
)
4253 upl_page_info_t
*pl
;
4262 upl_flags
= UPL_SET_LITE
;
4263 if (! (flags
& CL_READ
)) {
4265 * "write" operation: let the UPL subsystem know
4266 * that we intend to modify the buffer cache pages
4269 upl_flags
|= UPL_WILL_MODIFY
;
4272 kret
= ubc_create_upl(vp
,
4273 uio
->uio_offset
& ~PAGE_MASK_64
,
4279 if (kret
!= KERN_SUCCESS
)
4282 if (!upl_valid_page(pl
, 0)) {
4284 * issue a synchronous read to cluster_io
4286 error
= cluster_io(vp
, upl
, 0, uio
->uio_offset
& ~PAGE_MASK_64
, PAGE_SIZE
,
4287 CL_READ
, (buf_t
)NULL
, (struct clios
*)NULL
);
4289 ubc_upl_abort_range(upl
, 0, PAGE_SIZE
, UPL_ABORT_DUMP_PAGES
| UPL_ABORT_FREE_ON_EMPTY
);
4295 ubc_paddr
= ((addr64_t
)upl_phys_page(pl
, 0) << 12) + (addr64_t
)(uio
->uio_offset
& PAGE_MASK_64
);
4298 * NOTE: There is no prototype for the following in BSD. It, and the definitions
4299 * of the defines for cppvPsrc, cppvPsnk, cppvFsnk, and cppvFsrc will be found in
4300 * osfmk/ppc/mappings.h. They are not included here because there appears to be no
4301 * way to do so without exporting them to kexts as well.
4303 if (flags
& CL_READ
)
4304 // copypv(ubc_paddr, usr_paddr, xsize, cppvPsrc | cppvPsnk | cppvFsnk); /* Copy physical to physical and flush the destination */
4305 copypv(ubc_paddr
, usr_paddr
, xsize
, 2 | 1 | 4); /* Copy physical to physical and flush the destination */
4307 // copypv(usr_paddr, ubc_paddr, xsize, cppvPsrc | cppvPsnk | cppvFsrc); /* Copy physical to physical and flush the source */
4308 copypv(usr_paddr
, ubc_paddr
, xsize
, 2 | 1 | 8); /* Copy physical to physical and flush the source */
4310 if ( !(flags
& CL_READ
) || (upl_valid_page(pl
, 0) && upl_dirty_page(pl
, 0))) {
4312 * issue a synchronous write to cluster_io
4314 error
= cluster_io(vp
, upl
, 0, uio
->uio_offset
& ~PAGE_MASK_64
, PAGE_SIZE
,
4315 0, (buf_t
)NULL
, (struct clios
*)NULL
);
4318 uio_update(uio
, (user_size_t
)xsize
);
4321 abort_flags
= UPL_ABORT_FREE_ON_EMPTY
;
4323 abort_flags
= UPL_ABORT_FREE_ON_EMPTY
| UPL_ABORT_DUMP_PAGES
;
4325 ubc_upl_abort_range(upl
, 0, PAGE_SIZE
, abort_flags
);
4333 cluster_copy_upl_data(struct uio
*uio
, upl_t upl
, int upl_offset
, int xsize
)
4340 upl_page_info_t
*pl
;
4342 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 34)) | DBG_FUNC_START
,
4343 (int)uio
->uio_offset
, uio_resid(uio
), upl_offset
, xsize
, 0);
4345 segflg
= uio
->uio_segflg
;
4349 case UIO_USERSPACE32
:
4350 case UIO_USERISPACE32
:
4351 uio
->uio_segflg
= UIO_PHYS_USERSPACE32
;
4355 case UIO_USERISPACE
:
4356 uio
->uio_segflg
= UIO_PHYS_USERSPACE
;
4359 case UIO_USERSPACE64
:
4360 case UIO_USERISPACE64
:
4361 uio
->uio_segflg
= UIO_PHYS_USERSPACE64
;
4364 case UIO_SYSSPACE32
:
4365 uio
->uio_segflg
= UIO_PHYS_SYSSPACE32
;
4369 uio
->uio_segflg
= UIO_PHYS_SYSSPACE
;
4372 case UIO_SYSSPACE64
:
4373 uio
->uio_segflg
= UIO_PHYS_SYSSPACE64
;
4376 pl
= ubc_upl_pageinfo(upl
);
4378 pg_index
= upl_offset
/ PAGE_SIZE
;
4379 pg_offset
= upl_offset
& PAGE_MASK
;
4380 csize
= min(PAGE_SIZE
- pg_offset
, xsize
);
4382 while (xsize
&& retval
== 0) {
4385 paddr
= ((addr64_t
)upl_phys_page(pl
, pg_index
) << 12) + pg_offset
;
4387 retval
= uiomove64(paddr
, csize
, uio
);
4392 csize
= min(PAGE_SIZE
, xsize
);
4394 uio
->uio_segflg
= segflg
;
4396 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 34)) | DBG_FUNC_END
,
4397 (int)uio
->uio_offset
, uio_resid(uio
), retval
, segflg
, 0);
4404 cluster_copy_ubc_data(vnode_t vp
, struct uio
*uio
, int *io_resid
, int mark_dirty
)
4411 memory_object_control_t control
;
4414 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 34)) | DBG_FUNC_START
,
4415 (int)uio
->uio_offset
, uio_resid(uio
), 0, *io_resid
, 0);
4417 control
= ubc_getobject(vp
, UBC_FLAGS_NONE
);
4418 if (control
== MEMORY_OBJECT_CONTROL_NULL
) {
4419 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 34)) | DBG_FUNC_END
,
4420 (int)uio
->uio_offset
, uio_resid(uio
), retval
, 3, 0);
4424 segflg
= uio
->uio_segflg
;
4428 case UIO_USERSPACE32
:
4429 case UIO_USERISPACE32
:
4430 uio
->uio_segflg
= UIO_PHYS_USERSPACE32
;
4433 case UIO_USERSPACE64
:
4434 case UIO_USERISPACE64
:
4435 uio
->uio_segflg
= UIO_PHYS_USERSPACE64
;
4438 case UIO_SYSSPACE32
:
4439 uio
->uio_segflg
= UIO_PHYS_SYSSPACE32
;
4442 case UIO_SYSSPACE64
:
4443 uio
->uio_segflg
= UIO_PHYS_SYSSPACE64
;
4447 case UIO_USERISPACE
:
4448 uio
->uio_segflg
= UIO_PHYS_USERSPACE
;
4452 uio
->uio_segflg
= UIO_PHYS_SYSSPACE
;
4456 if ( (io_size
= *io_resid
) ) {
4457 start_offset
= (int)(uio
->uio_offset
& PAGE_MASK_64
);
4458 xsize
= uio_resid(uio
);
4460 retval
= memory_object_control_uiomove(control
, uio
->uio_offset
- start_offset
,
4461 uio
, start_offset
, io_size
, mark_dirty
);
4462 xsize
-= uio_resid(uio
);
4465 uio
->uio_segflg
= segflg
;
4466 *io_resid
= io_size
;
4468 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW
, 34)) | DBG_FUNC_END
,
4469 (int)uio
->uio_offset
, uio_resid(uio
), retval
, 0x80000000 | segflg
, 0);
4476 is_file_clean(vnode_t vp
, off_t filesize
)
4480 int total_dirty
= 0;
4482 for (f_offset
= 0; f_offset
< filesize
; f_offset
+= PAGE_SIZE_64
) {
4483 if (ubc_page_op(vp
, f_offset
, 0, 0, &flags
) == KERN_SUCCESS
) {
4484 if (flags
& UPL_POP_DIRTY
) {
4498 * Dirty region tracking/clustering mechanism.
4500 * This code (vfs_drt_*) provides a mechanism for tracking and clustering
4501 * dirty regions within a larger space (file). It is primarily intended to
4502 * support clustering in large files with many dirty areas.
4504 * The implementation assumes that the dirty regions are pages.
4506 * To represent dirty pages within the file, we store bit vectors in a
4507 * variable-size circular hash.
4511 * Bitvector size. This determines the number of pages we group in a
4512 * single hashtable entry. Each hashtable entry is aligned to this
4513 * size within the file.
4515 #define DRT_BITVECTOR_PAGES 256
4518 * File offset handling.
4520 * DRT_ADDRESS_MASK is dependent on DRT_BITVECTOR_PAGES;
4521 * the correct formula is (~(DRT_BITVECTOR_PAGES * PAGE_SIZE) - 1)
4523 #define DRT_ADDRESS_MASK (~((1 << 20) - 1))
4524 #define DRT_ALIGN_ADDRESS(addr) ((addr) & DRT_ADDRESS_MASK)
4527 * Hashtable address field handling.
4529 * The low-order bits of the hashtable address are used to conserve
4532 * DRT_HASH_COUNT_MASK must be large enough to store the range
4533 * 0-DRT_BITVECTOR_PAGES inclusive, as well as have one value
4534 * to indicate that the bucket is actually unoccupied.
4536 #define DRT_HASH_GET_ADDRESS(scm, i) ((scm)->scm_hashtable[(i)].dhe_control & DRT_ADDRESS_MASK)
4537 #define DRT_HASH_SET_ADDRESS(scm, i, a) \
4539 (scm)->scm_hashtable[(i)].dhe_control = \
4540 ((scm)->scm_hashtable[(i)].dhe_control & ~DRT_ADDRESS_MASK) | DRT_ALIGN_ADDRESS(a); \
4542 #define DRT_HASH_COUNT_MASK 0x1ff
4543 #define DRT_HASH_GET_COUNT(scm, i) ((scm)->scm_hashtable[(i)].dhe_control & DRT_HASH_COUNT_MASK)
4544 #define DRT_HASH_SET_COUNT(scm, i, c) \
4546 (scm)->scm_hashtable[(i)].dhe_control = \
4547 ((scm)->scm_hashtable[(i)].dhe_control & ~DRT_HASH_COUNT_MASK) | ((c) & DRT_HASH_COUNT_MASK); \
4549 #define DRT_HASH_CLEAR(scm, i) \
4551 (scm)->scm_hashtable[(i)].dhe_control = 0; \
4553 #define DRT_HASH_VACATE(scm, i) DRT_HASH_SET_COUNT((scm), (i), DRT_HASH_COUNT_MASK)
4554 #define DRT_HASH_VACANT(scm, i) (DRT_HASH_GET_COUNT((scm), (i)) == DRT_HASH_COUNT_MASK)
4555 #define DRT_HASH_COPY(oscm, oi, scm, i) \
4557 (scm)->scm_hashtable[(i)].dhe_control = (oscm)->scm_hashtable[(oi)].dhe_control; \
4558 DRT_BITVECTOR_COPY(oscm, oi, scm, i); \
4563 * Hash table moduli.
4565 * Since the hashtable entry's size is dependent on the size of
4566 * the bitvector, and since the hashtable size is constrained to
4567 * both being prime and fitting within the desired allocation
4568 * size, these values need to be manually determined.
4570 * For DRT_BITVECTOR_SIZE = 256, the entry size is 40 bytes.
4572 * The small hashtable allocation is 1024 bytes, so the modulus is 23.
4573 * The large hashtable allocation is 16384 bytes, so the modulus is 401.
4575 #define DRT_HASH_SMALL_MODULUS 23
4576 #define DRT_HASH_LARGE_MODULUS 401
4578 #define DRT_SMALL_ALLOCATION 1024 /* 104 bytes spare */
4579 #define DRT_LARGE_ALLOCATION 16384 /* 344 bytes spare */
4581 /* *** nothing below here has secret dependencies on DRT_BITVECTOR_PAGES *** */
4584 * Hashtable bitvector handling.
4586 * Bitvector fields are 32 bits long.
4589 #define DRT_HASH_SET_BIT(scm, i, bit) \
4590 (scm)->scm_hashtable[(i)].dhe_bitvector[(bit) / 32] |= (1 << ((bit) % 32))
4592 #define DRT_HASH_CLEAR_BIT(scm, i, bit) \
4593 (scm)->scm_hashtable[(i)].dhe_bitvector[(bit) / 32] &= ~(1 << ((bit) % 32))
4595 #define DRT_HASH_TEST_BIT(scm, i, bit) \
4596 ((scm)->scm_hashtable[(i)].dhe_bitvector[(bit) / 32] & (1 << ((bit) % 32)))
4598 #define DRT_BITVECTOR_CLEAR(scm, i) \
4599 bzero(&(scm)->scm_hashtable[(i)].dhe_bitvector[0], (DRT_BITVECTOR_PAGES / 32) * sizeof(u_int32_t))
4601 #define DRT_BITVECTOR_COPY(oscm, oi, scm, i) \
4602 bcopy(&(oscm)->scm_hashtable[(oi)].dhe_bitvector[0], \
4603 &(scm)->scm_hashtable[(i)].dhe_bitvector[0], \
4604 (DRT_BITVECTOR_PAGES / 32) * sizeof(u_int32_t))
4611 struct vfs_drt_hashentry
{
4612 u_int64_t dhe_control
;
4613 u_int32_t dhe_bitvector
[DRT_BITVECTOR_PAGES
/ 32];
4617 * Dirty Region Tracking structure.
4619 * The hashtable is allocated entirely inside the DRT structure.
4621 * The hash is a simple circular prime modulus arrangement, the structure
4622 * is resized from small to large if it overflows.
4625 struct vfs_drt_clustermap
{
4626 u_int32_t scm_magic
; /* sanity/detection */
4627 #define DRT_SCM_MAGIC 0x12020003
4628 u_int32_t scm_modulus
; /* current ring size */
4629 u_int32_t scm_buckets
; /* number of occupied buckets */
4630 u_int32_t scm_lastclean
; /* last entry we cleaned */
4631 u_int32_t scm_iskips
; /* number of slot skips */
4633 struct vfs_drt_hashentry scm_hashtable
[0];
4637 #define DRT_HASH(scm, addr) ((addr) % (scm)->scm_modulus)
4638 #define DRT_HASH_NEXT(scm, addr) (((addr) + 1) % (scm)->scm_modulus)
4641 * Debugging codes and arguments.
4643 #define DRT_DEBUG_EMPTYFREE (FSDBG_CODE(DBG_FSRW, 82)) /* nil */
4644 #define DRT_DEBUG_RETCLUSTER (FSDBG_CODE(DBG_FSRW, 83)) /* offset, length */
4645 #define DRT_DEBUG_ALLOC (FSDBG_CODE(DBG_FSRW, 84)) /* copycount */
4646 #define DRT_DEBUG_INSERT (FSDBG_CODE(DBG_FSRW, 85)) /* offset, iskip */
4647 #define DRT_DEBUG_MARK (FSDBG_CODE(DBG_FSRW, 86)) /* offset, length,
4650 /* 1 (clean, no map) */
4651 /* 2 (map alloc fail) */
4652 /* 3, resid (partial) */
4653 #define DRT_DEBUG_6 (FSDBG_CODE(DBG_FSRW, 87))
4654 #define DRT_DEBUG_SCMDATA (FSDBG_CODE(DBG_FSRW, 88)) /* modulus, buckets,
4655 * lastclean, iskips */
4658 static kern_return_t
vfs_drt_alloc_map(struct vfs_drt_clustermap
**cmapp
);
4659 static kern_return_t
vfs_drt_free_map(struct vfs_drt_clustermap
*cmap
);
4660 static kern_return_t
vfs_drt_search_index(struct vfs_drt_clustermap
*cmap
,
4661 u_int64_t offset
, int *indexp
);
4662 static kern_return_t
vfs_drt_get_index(struct vfs_drt_clustermap
**cmapp
,
4666 static kern_return_t
vfs_drt_do_mark_pages(
4672 static void vfs_drt_trace(
4673 struct vfs_drt_clustermap
*cmap
,
4682 * Allocate and initialise a sparse cluster map.
4684 * Will allocate a new map, resize or compact an existing map.
4686 * XXX we should probably have at least one intermediate map size,
4687 * as the 1:16 ratio seems a bit drastic.
4689 static kern_return_t
4690 vfs_drt_alloc_map(struct vfs_drt_clustermap
**cmapp
)
4692 struct vfs_drt_clustermap
*cmap
, *ocmap
;
4695 int nsize
, i
, active_buckets
, index
, copycount
;
4702 * Decide on the size of the new map.
4704 if (ocmap
== NULL
) {
4705 nsize
= DRT_HASH_SMALL_MODULUS
;
4707 /* count the number of active buckets in the old map */
4709 for (i
= 0; i
< ocmap
->scm_modulus
; i
++) {
4710 if (!DRT_HASH_VACANT(ocmap
, i
) &&
4711 (DRT_HASH_GET_COUNT(ocmap
, i
) != 0))
4715 * If we're currently using the small allocation, check to
4716 * see whether we should grow to the large one.
4718 if (ocmap
->scm_modulus
== DRT_HASH_SMALL_MODULUS
) {
4719 /* if the ring is nearly full */
4720 if (active_buckets
> (DRT_HASH_SMALL_MODULUS
- 5)) {
4721 nsize
= DRT_HASH_LARGE_MODULUS
;
4723 nsize
= DRT_HASH_SMALL_MODULUS
;
4726 /* already using the large modulus */
4727 nsize
= DRT_HASH_LARGE_MODULUS
;
4729 * If the ring is completely full, there's
4730 * nothing useful for us to do. Behave as
4731 * though we had compacted into the new
4734 if (active_buckets
>= DRT_HASH_LARGE_MODULUS
)
4735 return(KERN_SUCCESS
);
4740 * Allocate and initialise the new map.
4743 kret
= kmem_alloc(kernel_map
, (vm_offset_t
*)&cmap
,
4744 (nsize
== DRT_HASH_SMALL_MODULUS
) ? DRT_SMALL_ALLOCATION
: DRT_LARGE_ALLOCATION
);
4745 if (kret
!= KERN_SUCCESS
)
4747 cmap
->scm_magic
= DRT_SCM_MAGIC
;
4748 cmap
->scm_modulus
= nsize
;
4749 cmap
->scm_buckets
= 0;
4750 cmap
->scm_lastclean
= 0;
4751 cmap
->scm_iskips
= 0;
4752 for (i
= 0; i
< cmap
->scm_modulus
; i
++) {
4753 DRT_HASH_CLEAR(cmap
, i
);
4754 DRT_HASH_VACATE(cmap
, i
);
4755 DRT_BITVECTOR_CLEAR(cmap
, i
);
4759 * If there's an old map, re-hash entries from it into the new map.
4762 if (ocmap
!= NULL
) {
4763 for (i
= 0; i
< ocmap
->scm_modulus
; i
++) {
4764 /* skip empty buckets */
4765 if (DRT_HASH_VACANT(ocmap
, i
) ||
4766 (DRT_HASH_GET_COUNT(ocmap
, i
) == 0))
4769 offset
= DRT_HASH_GET_ADDRESS(ocmap
, i
);
4770 kret
= vfs_drt_get_index(&cmap
, offset
, &index
, 1);
4771 if (kret
!= KERN_SUCCESS
) {
4772 /* XXX need to bail out gracefully here */
4773 panic("vfs_drt: new cluster map mysteriously too small");
4776 DRT_HASH_COPY(ocmap
, i
, cmap
, index
);
4781 /* log what we've done */
4782 vfs_drt_trace(cmap
, DRT_DEBUG_ALLOC
, copycount
, 0, 0, 0);
4785 * It's important to ensure that *cmapp always points to
4786 * a valid map, so we must overwrite it before freeing
4790 if (ocmap
!= NULL
) {
4791 /* emit stats into trace buffer */
4792 vfs_drt_trace(ocmap
, DRT_DEBUG_SCMDATA
,
4795 ocmap
->scm_lastclean
,
4798 vfs_drt_free_map(ocmap
);
4800 return(KERN_SUCCESS
);
4805 * Free a sparse cluster map.
4807 static kern_return_t
4808 vfs_drt_free_map(struct vfs_drt_clustermap
*cmap
)
4810 kmem_free(kernel_map
, (vm_offset_t
)cmap
,
4811 (cmap
->scm_modulus
== DRT_HASH_SMALL_MODULUS
) ? DRT_SMALL_ALLOCATION
: DRT_LARGE_ALLOCATION
);
4812 return(KERN_SUCCESS
);
4817 * Find the hashtable slot currently occupied by an entry for the supplied offset.
4819 static kern_return_t
4820 vfs_drt_search_index(struct vfs_drt_clustermap
*cmap
, u_int64_t offset
, int *indexp
)
4824 offset
= DRT_ALIGN_ADDRESS(offset
);
4825 index
= DRT_HASH(cmap
, offset
);
4827 /* traverse the hashtable */
4828 for (i
= 0; i
< cmap
->scm_modulus
; i
++) {
4831 * If the slot is vacant, we can stop.
4833 if (DRT_HASH_VACANT(cmap
, index
))
4837 * If the address matches our offset, we have success.
4839 if (DRT_HASH_GET_ADDRESS(cmap
, index
) == offset
) {
4841 return(KERN_SUCCESS
);
4845 * Move to the next slot, try again.
4847 index
= DRT_HASH_NEXT(cmap
, index
);
4852 return(KERN_FAILURE
);
4856 * Find the hashtable slot for the supplied offset. If we haven't allocated
4857 * one yet, allocate one and populate the address field. Note that it will
4858 * not have a nonzero page count and thus will still technically be free, so
4859 * in the case where we are called to clean pages, the slot will remain free.
4861 static kern_return_t
4862 vfs_drt_get_index(struct vfs_drt_clustermap
**cmapp
, u_int64_t offset
, int *indexp
, int recursed
)
4864 struct vfs_drt_clustermap
*cmap
;
4870 /* look for an existing entry */
4871 kret
= vfs_drt_search_index(cmap
, offset
, indexp
);
4872 if (kret
== KERN_SUCCESS
)
4875 /* need to allocate an entry */
4876 offset
= DRT_ALIGN_ADDRESS(offset
);
4877 index
= DRT_HASH(cmap
, offset
);
4879 /* scan from the index forwards looking for a vacant slot */
4880 for (i
= 0; i
< cmap
->scm_modulus
; i
++) {
4882 if (DRT_HASH_VACANT(cmap
, index
) || DRT_HASH_GET_COUNT(cmap
,index
) == 0) {
4883 cmap
->scm_buckets
++;
4884 if (index
< cmap
->scm_lastclean
)
4885 cmap
->scm_lastclean
= index
;
4886 DRT_HASH_SET_ADDRESS(cmap
, index
, offset
);
4887 DRT_HASH_SET_COUNT(cmap
, index
, 0);
4888 DRT_BITVECTOR_CLEAR(cmap
, index
);
4890 vfs_drt_trace(cmap
, DRT_DEBUG_INSERT
, (int)offset
, i
, 0, 0);
4891 return(KERN_SUCCESS
);
4893 cmap
->scm_iskips
+= i
;
4894 index
= DRT_HASH_NEXT(cmap
, index
);
4898 * We haven't found a vacant slot, so the map is full. If we're not
4899 * already recursed, try reallocating/compacting it.
4902 return(KERN_FAILURE
);
4903 kret
= vfs_drt_alloc_map(cmapp
);
4904 if (kret
== KERN_SUCCESS
) {
4905 /* now try to insert again */
4906 kret
= vfs_drt_get_index(cmapp
, offset
, indexp
, 1);
4912 * Implementation of set dirty/clean.
4914 * In the 'clean' case, not finding a map is OK.
4916 static kern_return_t
4917 vfs_drt_do_mark_pages(
4924 struct vfs_drt_clustermap
*cmap
, **cmapp
;
4926 int i
, index
, pgoff
, pgcount
, setcount
, ecount
;
4928 cmapp
= (struct vfs_drt_clustermap
**)private;
4931 vfs_drt_trace(cmap
, DRT_DEBUG_MARK
| DBG_FUNC_START
, (int)offset
, (int)length
, dirty
, 0);
4933 if (setcountp
!= NULL
)
4936 /* allocate a cluster map if we don't already have one */
4938 /* no cluster map, nothing to clean */
4940 vfs_drt_trace(cmap
, DRT_DEBUG_MARK
| DBG_FUNC_END
, 1, 0, 0, 0);
4941 return(KERN_SUCCESS
);
4943 kret
= vfs_drt_alloc_map(cmapp
);
4944 if (kret
!= KERN_SUCCESS
) {
4945 vfs_drt_trace(cmap
, DRT_DEBUG_MARK
| DBG_FUNC_END
, 2, 0, 0, 0);
4952 * Iterate over the length of the region.
4954 while (length
> 0) {
4956 * Get the hashtable index for this offset.
4958 * XXX this will add blank entries if we are clearing a range
4959 * that hasn't been dirtied.
4961 kret
= vfs_drt_get_index(cmapp
, offset
, &index
, 0);
4962 cmap
= *cmapp
; /* may have changed! */
4963 /* this may be a partial-success return */
4964 if (kret
!= KERN_SUCCESS
) {
4965 if (setcountp
!= NULL
)
4966 *setcountp
= setcount
;
4967 vfs_drt_trace(cmap
, DRT_DEBUG_MARK
| DBG_FUNC_END
, 3, (int)length
, 0, 0);
4973 * Work out how many pages we're modifying in this
4976 pgoff
= (offset
- DRT_ALIGN_ADDRESS(offset
)) / PAGE_SIZE
;
4977 pgcount
= min((length
/ PAGE_SIZE
), (DRT_BITVECTOR_PAGES
- pgoff
));
4980 * Iterate over pages, dirty/clearing as we go.
4982 ecount
= DRT_HASH_GET_COUNT(cmap
, index
);
4983 for (i
= 0; i
< pgcount
; i
++) {
4985 if (!DRT_HASH_TEST_BIT(cmap
, index
, pgoff
+ i
)) {
4986 DRT_HASH_SET_BIT(cmap
, index
, pgoff
+ i
);
4991 if (DRT_HASH_TEST_BIT(cmap
, index
, pgoff
+ i
)) {
4992 DRT_HASH_CLEAR_BIT(cmap
, index
, pgoff
+ i
);
4998 DRT_HASH_SET_COUNT(cmap
, index
, ecount
);
5000 offset
+= pgcount
* PAGE_SIZE
;
5001 length
-= pgcount
* PAGE_SIZE
;
5003 if (setcountp
!= NULL
)
5004 *setcountp
= setcount
;
5006 vfs_drt_trace(cmap
, DRT_DEBUG_MARK
| DBG_FUNC_END
, 0, setcount
, 0, 0);
5008 return(KERN_SUCCESS
);
5012 * Mark a set of pages as dirty/clean.
5014 * This is a public interface.
5017 * Pointer to storage suitable for holding a pointer. Note that
5018 * this must either be NULL or a value set by this function.
5021 * Current file size in bytes.
5024 * Offset of the first page to be marked as dirty, in bytes. Must be
5028 * Length of dirty region, in bytes. Must be a multiple of PAGE_SIZE.
5031 * Number of pages newly marked dirty by this call (optional).
5033 * Returns KERN_SUCCESS if all the pages were successfully marked.
5035 static kern_return_t
5036 vfs_drt_mark_pages(void **cmapp
, off_t offset
, u_int length
, int *setcountp
)
5038 /* XXX size unused, drop from interface */
5039 return(vfs_drt_do_mark_pages(cmapp
, offset
, length
, setcountp
, 1));
5043 static kern_return_t
5044 vfs_drt_unmark_pages(void **cmapp
, off_t offset
, u_int length
)
5046 return(vfs_drt_do_mark_pages(cmapp
, offset
, length
, NULL
, 0));
5051 * Get a cluster of dirty pages.
5053 * This is a public interface.
5056 * Pointer to storage managed by drt_mark_pages. Note that this must
5057 * be NULL or a value set by drt_mark_pages.
5060 * Returns the byte offset into the file of the first page in the cluster.
5063 * Returns the length in bytes of the cluster of dirty pages.
5065 * Returns success if a cluster was found. If KERN_FAILURE is returned, there
5066 * are no dirty pages meeting the minmum size criteria. Private storage will
5067 * be released if there are no more dirty pages left in the map
5070 static kern_return_t
5071 vfs_drt_get_cluster(void **cmapp
, off_t
*offsetp
, u_int
*lengthp
)
5073 struct vfs_drt_clustermap
*cmap
;
5076 int index
, i
, j
, fs
, ls
;
5079 if ((cmapp
== NULL
) || (*cmapp
== NULL
))
5080 return(KERN_FAILURE
);
5083 /* walk the hashtable */
5084 for (offset
= 0, j
= 0; j
< cmap
->scm_modulus
; offset
+= (DRT_BITVECTOR_PAGES
* PAGE_SIZE
), j
++) {
5085 index
= DRT_HASH(cmap
, offset
);
5087 if (DRT_HASH_VACANT(cmap
, index
) || (DRT_HASH_GET_COUNT(cmap
, index
) == 0))
5090 /* scan the bitfield for a string of bits */
5093 for (i
= 0; i
< DRT_BITVECTOR_PAGES
; i
++) {
5094 if (DRT_HASH_TEST_BIT(cmap
, index
, i
)) {
5100 /* didn't find any bits set */
5101 panic("vfs_drt: entry summary count > 0 but no bits set in map");
5103 for (ls
= 0; i
< DRT_BITVECTOR_PAGES
; i
++, ls
++) {
5104 if (!DRT_HASH_TEST_BIT(cmap
, index
, i
))
5108 /* compute offset and length, mark pages clean */
5109 offset
= DRT_HASH_GET_ADDRESS(cmap
, index
) + (PAGE_SIZE
* fs
);
5110 length
= ls
* PAGE_SIZE
;
5111 vfs_drt_do_mark_pages(cmapp
, offset
, length
, NULL
, 0);
5112 cmap
->scm_lastclean
= index
;
5114 /* return successful */
5115 *offsetp
= (off_t
)offset
;
5118 vfs_drt_trace(cmap
, DRT_DEBUG_RETCLUSTER
, (int)offset
, (int)length
, 0, 0);
5119 return(KERN_SUCCESS
);
5122 * We didn't find anything... hashtable is empty
5123 * emit stats into trace buffer and
5126 vfs_drt_trace(cmap
, DRT_DEBUG_SCMDATA
,
5129 cmap
->scm_lastclean
,
5132 vfs_drt_free_map(cmap
);
5135 return(KERN_FAILURE
);
5139 static kern_return_t
5140 vfs_drt_control(void **cmapp
, int op_type
)
5142 struct vfs_drt_clustermap
*cmap
;
5145 if ((cmapp
== NULL
) || (*cmapp
== NULL
))
5146 return(KERN_FAILURE
);
5151 /* emit stats into trace buffer */
5152 vfs_drt_trace(cmap
, DRT_DEBUG_SCMDATA
,
5155 cmap
->scm_lastclean
,
5158 vfs_drt_free_map(cmap
);
5163 cmap
->scm_lastclean
= 0;
5166 return(KERN_SUCCESS
);
5172 * Emit a summary of the state of the clustermap into the trace buffer
5173 * along with some caller-provided data.
5177 vfs_drt_trace(__unused
struct vfs_drt_clustermap
*cmap
, int code
, int arg1
, int arg2
, int arg3
, int arg4
)
5179 KERNEL_DEBUG(code
, arg1
, arg2
, arg3
, arg4
, 0);
5183 vfs_drt_trace(__unused
struct vfs_drt_clustermap
*cmap
, __unused
int code
,
5184 __unused
int arg1
, __unused
int arg2
, __unused
int arg3
,
5192 * Perform basic sanity check on the hash entry summary count
5193 * vs. the actual bits set in the entry.
5196 vfs_drt_sanity(struct vfs_drt_clustermap
*cmap
)
5201 for (index
= 0; index
< cmap
->scm_modulus
; index
++) {
5202 if (DRT_HASH_VACANT(cmap
, index
))
5205 for (bits_on
= 0, i
= 0; i
< DRT_BITVECTOR_PAGES
; i
++) {
5206 if (DRT_HASH_TEST_BIT(cmap
, index
, i
))
5209 if (bits_on
!= DRT_HASH_GET_COUNT(cmap
, index
))
5210 panic("bits_on = %d, index = %d\n", bits_on
, index
);