]> git.saurik.com Git - apple/xnu.git/blob - bsd/vfs/vfs_cluster.c
projects / apple / xnu.git / blob
? search:


3a34e1787f61be54be9aa68585e9f60692a28a70
[apple/xnu.git] / bsd / vfs / vfs_cluster.c
1 /*
2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
29 /*
30 * Copyright (c) 1993
31 * The Regents of the University of California. All rights reserved.
32 *
33 * Redistribution and use in source and binary forms, with or without
34 * modification, are permitted provided that the following conditions
35 * are met:
36 * 1. Redistributions of source code must retain the above copyright
37 * notice, this list of conditions and the following disclaimer.
38 * 2. Redistributions in binary form must reproduce the above copyright
39 * notice, this list of conditions and the following disclaimer in the
40 * documentation and/or other materials provided with the distribution.
41 * 3. All advertising materials mentioning features or use of this software
42 * must display the following acknowledgement:
43 * This product includes software developed by the University of
44 * California, Berkeley and its contributors.
45 * 4. Neither the name of the University nor the names of its contributors
46 * may be used to endorse or promote products derived from this software
47 * without specific prior written permission.
48 *
49 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
50 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
51 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
52 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
53 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
54 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
55 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
56 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
57 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
58 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
59 * SUCH DAMAGE.
60 *
61 * @(#)vfs_cluster.c 8.10 (Berkeley) 3/28/95
62 */
63
64 #include <sys/param.h>
65 #include <sys/proc_internal.h>
66 #include <sys/buf_internal.h>
67 #include <sys/mount_internal.h>
68 #include <sys/vnode_internal.h>
69 #include <sys/trace.h>
70 #include <sys/malloc.h>
71 #include <sys/time.h>
72 #include <sys/kernel.h>
73 #include <sys/resourcevar.h>
74 #include <sys/uio_internal.h>
75 #include <libkern/libkern.h>
76 #include <machine/machine_routines.h>
77
78 #include <sys/ubc_internal.h>
79 #include <vm/vnode_pager.h>
80
81 #include <mach/mach_types.h>
82 #include <mach/memory_object_types.h>
83 #include <mach/vm_map.h>
84 #include <mach/upl.h>
85
86 #include <vm/vm_kern.h>
87 #include <vm/vm_map.h>
88 #include <vm/vm_pageout.h>
89
90 #include <sys/kdebug.h>
91
92 #define CL_READ 0x01
93 #define CL_WRITE 0x02
94 #define CL_ASYNC 0x04
95 #define CL_COMMIT 0x08
96 #define CL_PAGEOUT 0x10
97 #define CL_AGE 0x20
98 #define CL_NOZERO 0x40
99 #define CL_PAGEIN 0x80
100 #define CL_DEV_MEMORY 0x100
101 #define CL_PRESERVE 0x200
102 #define CL_THROTTLE 0x400
103 #define CL_KEEPCACHED 0x800
104 #define CL_DIRECT_IO 0x1000
105 #define CL_PASSIVE 0x2000
106
107
108 struct clios {
109 u_int io_completed; /* amount of io that has currently completed */
110 u_int io_issued; /* amount of io that was successfully issued */
111 int io_error; /* error code of first error encountered */
112 int io_wanted; /* someone is sleeping waiting for a change in state */
113 };
114
115 static lck_grp_t *cl_mtx_grp;
116 static lck_attr_t *cl_mtx_attr;
117 static lck_grp_attr_t *cl_mtx_grp_attr;
118 static lck_mtx_t *cl_mtxp;
119
120
121 #define IO_UNKNOWN 0
122 #define IO_DIRECT 1
123 #define IO_CONTIG 2
124 #define IO_COPY 3
125
126 #define PUSH_DELAY 0x01
127 #define PUSH_ALL 0x02
128 #define PUSH_SYNC 0x04
129
130
131 static void cluster_EOT(buf_t cbp_head, buf_t cbp_tail, int zero_offset);
132 static void cluster_wait_IO(buf_t cbp_head, int async);
133 static void cluster_complete_transaction(buf_t *cbp_head, void *callback_arg, int *retval, int flags, int needwait);
134
135 static int cluster_io_type(struct uio *uio, int *io_type, u_int32_t *io_length, u_int32_t min_length);
136
137 static int cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int non_rounded_size,
138 int flags, buf_t real_bp, struct clios *iostate, int (*)(buf_t, void *), void *callback_arg);
139 static int cluster_iodone(buf_t bp, void *callback_arg);
140 static int cluster_ioerror(upl_t upl, int upl_offset, int abort_size, int error, int io_flags);
141 static int cluster_hard_throttle_on(vnode_t vp);
142
143 static void cluster_syncup(vnode_t vp, off_t newEOF, int (*)(buf_t, void *), void *callback_arg);
144
145 static void cluster_read_upl_release(upl_t upl, int start_pg, int last_pg, int flags);
146 static int cluster_copy_ubc_data_internal(vnode_t vp, struct uio *uio, int *io_resid, int mark_dirty, int take_reference);
147
148 static int cluster_read_copy(vnode_t vp, struct uio *uio, u_int32_t io_req_size, off_t filesize, int flags,
149 int (*)(buf_t, void *), void *callback_arg);
150 static int cluster_read_direct(vnode_t vp, struct uio *uio, off_t filesize, int *read_type, u_int32_t *read_length,
151 int flags, int (*)(buf_t, void *), void *callback_arg);
152 static int cluster_read_contig(vnode_t vp, struct uio *uio, off_t filesize, int *read_type, u_int32_t *read_length,
153 int (*)(buf_t, void *), void *callback_arg, int flags);
154
155 static int cluster_write_copy(vnode_t vp, struct uio *uio, u_int32_t io_req_size, off_t oldEOF, off_t newEOF,
156 off_t headOff, off_t tailOff, int flags, int (*)(buf_t, void *), void *callback_arg);
157 static int cluster_write_direct(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF,
158 int *write_type, u_int32_t *write_length, int flags, int (*)(buf_t, void *), void *callback_arg);
159 static int cluster_write_contig(vnode_t vp, struct uio *uio, off_t newEOF,
160 int *write_type, u_int32_t *write_length, int (*)(buf_t, void *), void *callback_arg, int bflag);
161
162 static int cluster_align_phys_io(vnode_t vp, struct uio *uio, addr64_t usr_paddr, u_int32_t xsize, int flags, int (*)(buf_t, void *), void *callback_arg);
163
164 static int cluster_read_prefetch(vnode_t vp, off_t f_offset, u_int size, off_t filesize, int (*callback)(buf_t, void *), void *callback_arg, int bflag);
165 static void cluster_read_ahead(vnode_t vp, struct cl_extent *extent, off_t filesize, struct cl_readahead *ra, int (*callback)(buf_t, void *), void *callback_arg, int bflag);
166
167 static int cluster_push_now(vnode_t vp, struct cl_extent *, off_t EOF, int flags, int (*)(buf_t, void *), void *callback_arg);
168
169 static int cluster_try_push(struct cl_writebehind *, vnode_t vp, off_t EOF, int push_flag, int (*)(buf_t, void *), void *callback_arg);
170
171 static void sparse_cluster_switch(struct cl_writebehind *, vnode_t vp, off_t EOF, int (*)(buf_t, void *), void *callback_arg);
172 static void sparse_cluster_push(struct cl_writebehind *, vnode_t vp, off_t EOF, int push_flag, int (*)(buf_t, void *), void *callback_arg);
173 static void sparse_cluster_add(struct cl_writebehind *, vnode_t vp, struct cl_extent *, off_t EOF, int (*)(buf_t, void *), void *callback_arg);
174
175 static kern_return_t vfs_drt_mark_pages(void **cmapp, off_t offset, u_int length, u_int *setcountp);
176 static kern_return_t vfs_drt_get_cluster(void **cmapp, off_t *offsetp, u_int *lengthp);
177 static kern_return_t vfs_drt_control(void **cmapp, int op_type);
178
179 int is_file_clean(vnode_t, off_t);
180
181 /*
182 * limit the internal I/O size so that we
183 * can represent it in a 32 bit int
184 */
185 #define MAX_IO_REQUEST_SIZE (1024 * 1024 * 256)
186 #define MAX_IO_CONTIG_SIZE (MAX_UPL_SIZE * PAGE_SIZE)
187 #define MAX_VECTS 16
188 #define MIN_DIRECT_WRITE_SIZE (4 * PAGE_SIZE)
189
190
191 #define MAX_CLUSTER_SIZE(vp) (cluster_max_io_size(vp->v_mount, CL_WRITE))
192 #define MAX_PREFETCH(vp) (cluster_max_io_size(vp->v_mount, CL_READ) * 3);
193
194
195 int speculative_reads_disabled = 0;
196
197 /*
198 * throttle the number of async writes that
199 * can be outstanding on a single vnode
200 * before we issue a synchronous write
201 */
202 #define HARD_THROTTLE_MAXCNT 0
203 #define HARD_THROTTLE_MAXSIZE (64 * 1024)
204
205 int hard_throttle_on_root = 0;
206 struct timeval priority_IO_timestamp_for_root;
207
208
209 void
210 cluster_init(void) {
211 /*
212 * allocate lock group attribute and group
213 */
214 cl_mtx_grp_attr = lck_grp_attr_alloc_init();
215 cl_mtx_grp = lck_grp_alloc_init("cluster I/O", cl_mtx_grp_attr);
216
217 /*
218 * allocate the lock attribute
219 */
220 cl_mtx_attr = lck_attr_alloc_init();
221
222 /*
223 * allocate and initialize mutex's used to protect updates and waits
224 * on the cluster_io context
225 */
226 cl_mtxp = lck_mtx_alloc_init(cl_mtx_grp, cl_mtx_attr);
227
228 if (cl_mtxp == NULL)
229 panic("cluster_init: failed to allocate cl_mtxp");
230 }
231
232
233 uint32_t
234 cluster_max_io_size(mount_t mp, int type)
235 {
236 uint32_t max_io_size;
237 uint32_t segcnt;
238 uint32_t maxcnt;
239
240 switch(type) {
241
242 case CL_READ:
243 segcnt = mp->mnt_segreadcnt;
244 maxcnt = mp->mnt_maxreadcnt;
245 break;
246 case CL_WRITE:
247 segcnt = mp->mnt_segwritecnt;
248 maxcnt = mp->mnt_maxwritecnt;
249 break;
250 default:
251 segcnt = min(mp->mnt_segreadcnt, mp->mnt_segwritecnt);
252 maxcnt = min(mp->mnt_maxreadcnt, mp->mnt_maxwritecnt);
253 break;
254 }
255 if (segcnt > MAX_UPL_SIZE) {
256 /*
257 * don't allow a size beyond the max UPL size we can create
258 */
259 segcnt = MAX_UPL_SIZE;
260 }
261 max_io_size = min((segcnt * PAGE_SIZE), maxcnt);
262
263 if (max_io_size < (MAX_UPL_TRANSFER * PAGE_SIZE)) {
264 /*
265 * don't allow a size smaller than the old fixed limit
266 */
267 max_io_size = (MAX_UPL_TRANSFER * PAGE_SIZE);
268 } else {
269 /*
270 * make sure the size specified is a multiple of PAGE_SIZE
271 */
272 max_io_size &= ~PAGE_MASK;
273 }
274 return (max_io_size);
275 }
276
277
278
279
280 #define CLW_ALLOCATE 0x01
281 #define CLW_RETURNLOCKED 0x02
282 #define CLW_IONOCACHE 0x04
283 #define CLW_IOPASSIVE 0x08
284
285 /*
286 * if the read ahead context doesn't yet exist,
287 * allocate and initialize it...
288 * the vnode lock serializes multiple callers
289 * during the actual assignment... first one
290 * to grab the lock wins... the other callers
291 * will release the now unnecessary storage
292 *
293 * once the context is present, try to grab (but don't block on)
294 * the lock associated with it... if someone
295 * else currently owns it, than the read
296 * will run without read-ahead. this allows
297 * multiple readers to run in parallel and
298 * since there's only 1 read ahead context,
299 * there's no real loss in only allowing 1
300 * reader to have read-ahead enabled.
301 */
302 static struct cl_readahead *
303 cluster_get_rap(vnode_t vp)
304 {
305 struct ubc_info *ubc;
306 struct cl_readahead *rap;
307
308 ubc = vp->v_ubcinfo;
309
310 if ((rap = ubc->cl_rahead) == NULL) {
311 MALLOC_ZONE(rap, struct cl_readahead *, sizeof *rap, M_CLRDAHEAD, M_WAITOK);
312
313 bzero(rap, sizeof *rap);
314 rap->cl_lastr = -1;
315 lck_mtx_init(&rap->cl_lockr, cl_mtx_grp, cl_mtx_attr);
316
317 vnode_lock(vp);
318
319 if (ubc->cl_rahead == NULL)
320 ubc->cl_rahead = rap;
321 else {
322 lck_mtx_destroy(&rap->cl_lockr, cl_mtx_grp);
323 FREE_ZONE((void *)rap, sizeof *rap, M_CLRDAHEAD);
324 rap = ubc->cl_rahead;
325 }
326 vnode_unlock(vp);
327 }
328 if (lck_mtx_try_lock(&rap->cl_lockr) == TRUE)
329 return(rap);
330
331 return ((struct cl_readahead *)NULL);
332 }
333
334
335 /*
336 * if the write behind context doesn't yet exist,
337 * and CLW_ALLOCATE is specified, allocate and initialize it...
338 * the vnode lock serializes multiple callers
339 * during the actual assignment... first one
340 * to grab the lock wins... the other callers
341 * will release the now unnecessary storage
342 *
343 * if CLW_RETURNLOCKED is set, grab (blocking if necessary)
344 * the lock associated with the write behind context before
345 * returning
346 */
347
348 static struct cl_writebehind *
349 cluster_get_wbp(vnode_t vp, int flags)
350 {
351 struct ubc_info *ubc;
352 struct cl_writebehind *wbp;
353
354 ubc = vp->v_ubcinfo;
355
356 if ((wbp = ubc->cl_wbehind) == NULL) {
357
358 if ( !(flags & CLW_ALLOCATE))
359 return ((struct cl_writebehind *)NULL);
360
361 MALLOC_ZONE(wbp, struct cl_writebehind *, sizeof *wbp, M_CLWRBEHIND, M_WAITOK);
362
363 bzero(wbp, sizeof *wbp);
364 lck_mtx_init(&wbp->cl_lockw, cl_mtx_grp, cl_mtx_attr);
365
366 vnode_lock(vp);
367
368 if (ubc->cl_wbehind == NULL)
369 ubc->cl_wbehind = wbp;
370 else {
371 lck_mtx_destroy(&wbp->cl_lockw, cl_mtx_grp);
372 FREE_ZONE((void *)wbp, sizeof *wbp, M_CLWRBEHIND);
373 wbp = ubc->cl_wbehind;
374 }
375 vnode_unlock(vp);
376 }
377 if (flags & CLW_RETURNLOCKED)
378 lck_mtx_lock(&wbp->cl_lockw);
379
380 return (wbp);
381 }
382
383
384 static void
385 cluster_syncup(vnode_t vp, off_t newEOF, int (*callback)(buf_t, void *), void *callback_arg)
386 {
387 struct cl_writebehind *wbp;
388
389 if ((wbp = cluster_get_wbp(vp, 0)) != NULL) {
390
391 if (wbp->cl_number) {
392 lck_mtx_lock(&wbp->cl_lockw);
393
394 cluster_try_push(wbp, vp, newEOF, PUSH_ALL | PUSH_SYNC, callback, callback_arg);
395
396 lck_mtx_unlock(&wbp->cl_lockw);
397 }
398 }
399 }
400
401
402 static int
403 cluster_hard_throttle_on(vnode_t vp)
404 {
405 static struct timeval hard_throttle_maxelapsed = { 0, 200000 };
406
407 if (vp->v_mount->mnt_kern_flag & MNTK_ROOTDEV) {
408 struct timeval elapsed;
409
410 if (hard_throttle_on_root)
411 return(1);
412
413 microuptime(&elapsed);
414 timevalsub(&elapsed, &priority_IO_timestamp_for_root);
415
416 if (timevalcmp(&elapsed, &hard_throttle_maxelapsed, <))
417 return(1);
418 }
419 return(0);
420 }
421
422
423 static int
424 cluster_ioerror(upl_t upl, int upl_offset, int abort_size, int error, int io_flags)
425 {
426 int upl_abort_code = 0;
427 int page_in = 0;
428 int page_out = 0;
429
430 if (io_flags & B_PHYS)
431 /*
432 * direct write of any flavor, or a direct read that wasn't aligned
433 */
434 ubc_upl_commit_range(upl, upl_offset, abort_size, UPL_COMMIT_FREE_ON_EMPTY);
435 else {
436 if (io_flags & B_PAGEIO) {
437 if (io_flags & B_READ)
438 page_in = 1;
439 else
440 page_out = 1;
441 }
442 if (io_flags & B_CACHE)
443 /*
444 * leave pages in the cache unchanged on error
445 */
446 upl_abort_code = UPL_ABORT_FREE_ON_EMPTY;
447 else if (page_out && (error != ENXIO))
448 /*
449 * transient error... leave pages unchanged
450 */
451 upl_abort_code = UPL_ABORT_FREE_ON_EMPTY;
452 else if (page_in)
453 upl_abort_code = UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR;
454 else
455 upl_abort_code = UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_DUMP_PAGES;
456
457 ubc_upl_abort_range(upl, upl_offset, abort_size, upl_abort_code);
458 }
459 return (upl_abort_code);
460 }
461
462
463 static int
464 cluster_iodone(buf_t bp, void *callback_arg)
465 {
466 int b_flags;
467 int error;
468 int total_size;
469 int total_resid;
470 int upl_offset;
471 int zero_offset;
472 int pg_offset = 0;
473 int commit_size = 0;
474 int upl_flags = 0;
475 int transaction_size = 0;
476 upl_t upl;
477 buf_t cbp;
478 buf_t cbp_head;
479 buf_t cbp_next;
480 buf_t real_bp;
481 struct clios *iostate;
482 boolean_t transaction_complete = FALSE;
483
484 cbp_head = (buf_t)(bp->b_trans_head);
485
486 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 20)) | DBG_FUNC_START,
487 (int)cbp_head, bp->b_lblkno, bp->b_bcount, bp->b_flags, 0);
488
489 for (cbp = cbp_head; cbp; cbp = cbp->b_trans_next) {
490 /*
491 * all I/O requests that are part of this transaction
492 * have to complete before we can process it
493 */
494 if ( !(cbp->b_flags & B_DONE)) {
495
496 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 20)) | DBG_FUNC_END,
497 (int)cbp_head, (int)cbp, cbp->b_bcount, cbp->b_flags, 0);
498
499 return 0;
500 }
501 if (cbp->b_flags & B_EOT)
502 transaction_complete = TRUE;
503 }
504 if (transaction_complete == FALSE) {
505 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 20)) | DBG_FUNC_END,
506 (int)cbp_head, 0, 0, 0, 0);
507
508 return 0;
509 }
510 error = 0;
511 total_size = 0;
512 total_resid = 0;
513
514 cbp = cbp_head;
515 upl_offset = cbp->b_uploffset;
516 upl = cbp->b_upl;
517 b_flags = cbp->b_flags;
518 real_bp = cbp->b_real_bp;
519 zero_offset= cbp->b_validend;
520 iostate = (struct clios *)cbp->b_iostate;
521
522 if (real_bp)
523 real_bp->b_dev = cbp->b_dev;
524
525 while (cbp) {
526 if ((cbp->b_flags & B_ERROR) && error == 0)
527 error = cbp->b_error;
528
529 total_resid += cbp->b_resid;
530 total_size += cbp->b_bcount;
531
532 cbp_next = cbp->b_trans_next;
533
534 if (cbp_next == NULL)
535 /*
536 * compute the overall size of the transaction
537 * in case we created one that has 'holes' in it
538 * 'total_size' represents the amount of I/O we
539 * did, not the span of the transaction w/r to the UPL
540 */
541 transaction_size = cbp->b_uploffset + cbp->b_bcount - upl_offset;
542
543 if (cbp != cbp_head)
544 free_io_buf(cbp);
545
546 cbp = cbp_next;
547 }
548 if (error == 0 && total_resid)
549 error = EIO;
550
551 if (error == 0) {
552 int (*cliodone_func)(buf_t, void *) = (int (*)(buf_t, void *))(cbp_head->b_cliodone);
553
554 if (cliodone_func != NULL) {
555 cbp_head->b_bcount = transaction_size;
556
557 error = (*cliodone_func)(cbp_head, callback_arg);
558 }
559 }
560 if (zero_offset)
561 cluster_zero(upl, zero_offset, PAGE_SIZE - (zero_offset & PAGE_MASK), real_bp);
562
563 free_io_buf(cbp_head);
564
565 if (iostate) {
566 int need_wakeup = 0;
567
568 /*
569 * someone has issued multiple I/Os asynchrounsly
570 * and is waiting for them to complete (streaming)
571 */
572 lck_mtx_lock_spin(cl_mtxp);
573
574 if (error && iostate->io_error == 0)
575 iostate->io_error = error;
576
577 iostate->io_completed += total_size;
578
579 if (iostate->io_wanted) {
580 /*
581 * someone is waiting for the state of
582 * this io stream to change
583 */
584 iostate->io_wanted = 0;
585 need_wakeup = 1;
586 }
587 lck_mtx_unlock(cl_mtxp);
588
589 if (need_wakeup)
590 wakeup((caddr_t)&iostate->io_wanted);
591 }
592
593 if (b_flags & B_COMMIT_UPL) {
594
595 pg_offset = upl_offset & PAGE_MASK;
596 commit_size = (pg_offset + transaction_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
597
598 if (error)
599 upl_flags = cluster_ioerror(upl, upl_offset - pg_offset, commit_size, error, b_flags);
600 else {
601 upl_flags = UPL_COMMIT_FREE_ON_EMPTY;
602
603 if ((b_flags & B_PHYS) && (b_flags & B_READ))
604 upl_flags |= UPL_COMMIT_SET_DIRTY;
605
606 if (b_flags & B_AGE)
607 upl_flags |= UPL_COMMIT_INACTIVATE;
608
609 ubc_upl_commit_range(upl, upl_offset - pg_offset, commit_size, upl_flags);
610 }
611 }
612 if ((b_flags & B_NEED_IODONE) && real_bp) {
613 if (error) {
614 real_bp->b_flags |= B_ERROR;
615 real_bp->b_error = error;
616 }
617 real_bp->b_resid = total_resid;
618
619 buf_biodone(real_bp);
620 }
621 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 20)) | DBG_FUNC_END,
622 (int)upl, upl_offset - pg_offset, commit_size, (error << 24) | upl_flags, 0);
623
624 return (error);
625 }
626
627
628 void
629 cluster_zero(upl_t upl, vm_offset_t upl_offset, int size, buf_t bp)
630 {
631
632 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 23)) | DBG_FUNC_START,
633 upl_offset, size, (int)bp, 0, 0);
634
635 if (bp == NULL || bp->b_datap == 0) {
636 upl_page_info_t *pl;
637 addr64_t zero_addr;
638
639 pl = ubc_upl_pageinfo(upl);
640
641 if (upl_device_page(pl) == TRUE) {
642 zero_addr = ((addr64_t)upl_phys_page(pl, 0) << 12) + upl_offset;
643
644 bzero_phys_nc(zero_addr, size);
645 } else {
646 while (size) {
647 int page_offset;
648 int page_index;
649 int zero_cnt;
650
651 page_index = upl_offset / PAGE_SIZE;
652 page_offset = upl_offset & PAGE_MASK;
653
654 zero_addr = ((addr64_t)upl_phys_page(pl, page_index) << 12) + page_offset;
655 zero_cnt = min(PAGE_SIZE - page_offset, size);
656
657 bzero_phys(zero_addr, zero_cnt);
658
659 size -= zero_cnt;
660 upl_offset += zero_cnt;
661 }
662 }
663 } else
664 bzero((caddr_t)((vm_offset_t)bp->b_datap + upl_offset), size);
665
666 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 23)) | DBG_FUNC_END,
667 upl_offset, size, 0, 0, 0);
668 }
669
670
671 static void
672 cluster_EOT(buf_t cbp_head, buf_t cbp_tail, int zero_offset)
673 {
674 cbp_head->b_validend = zero_offset;
675 cbp_tail->b_flags |= B_EOT;
676 }
677
678 static void
679 cluster_wait_IO(buf_t cbp_head, int async)
680 {
681 buf_t cbp;
682
683 if (async) {
684 /*
685 * async callback completion will not normally
686 * generate a wakeup upon I/O completion...
687 * by setting BL_WANTED, we will force a wakeup
688 * to occur as any outstanding I/Os complete...
689 * I/Os already completed will have BL_CALLDONE already
690 * set and we won't block in buf_biowait_callback..
691 * note that we're actually waiting for the bp to have
692 * completed the callback function... only then
693 * can we safely take back ownership of the bp
694 * need the main buf mutex in order to safely
695 * update b_lflags
696 */
697 buf_list_lock();
698
699 for (cbp = cbp_head; cbp; cbp = cbp->b_trans_next)
700 cbp->b_lflags |= BL_WANTED;
701
702 buf_list_unlock();
703 }
704 for (cbp = cbp_head; cbp; cbp = cbp->b_trans_next) {
705 if (async)
706 buf_biowait_callback(cbp);
707 else
708 buf_biowait(cbp);
709 }
710 }
711
712 static void
713 cluster_complete_transaction(buf_t *cbp_head, void *callback_arg, int *retval, int flags, int needwait)
714 {
715 buf_t cbp;
716 int error;
717
718 /*
719 * cluster_complete_transaction will
720 * only be called if we've issued a complete chain in synchronous mode
721 * or, we've already done a cluster_wait_IO on an incomplete chain
722 */
723 if (needwait) {
724 for (cbp = *cbp_head; cbp; cbp = cbp->b_trans_next)
725 buf_biowait(cbp);
726 }
727 error = cluster_iodone(*cbp_head, callback_arg);
728
729 if ( !(flags & CL_ASYNC) && error && *retval == 0) {
730 if (((flags & (CL_PAGEOUT | CL_KEEPCACHED)) != CL_PAGEOUT) || (error != ENXIO))
731 *retval = error;
732 }
733 *cbp_head = (buf_t)NULL;
734 }
735
736
737 static int
738 cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int non_rounded_size,
739 int flags, buf_t real_bp, struct clios *iostate, int (*callback)(buf_t, void *), void *callback_arg)
740 {
741 buf_t cbp;
742 u_int size;
743 u_int io_size;
744 int io_flags;
745 int bmap_flags;
746 int error = 0;
747 int retval = 0;
748 buf_t cbp_head = NULL;
749 buf_t cbp_tail = NULL;
750 int trans_count = 0;
751 int max_trans_count;
752 u_int pg_count;
753 int pg_offset;
754 u_int max_iosize;
755 u_int max_vectors;
756 int priv;
757 int zero_offset = 0;
758 int async_throttle = 0;
759 mount_t mp;
760 vm_offset_t upl_end_offset;
761 boolean_t need_EOT = FALSE;
762
763 /*
764 * we currently don't support buffers larger than a page
765 */
766 if (real_bp && non_rounded_size > PAGE_SIZE)
767 panic("%s(): Called with real buffer of size %d bytes which "
768 "is greater than the maximum allowed size of "
769 "%d bytes (the system PAGE_SIZE).\n",
770 __FUNCTION__, non_rounded_size, PAGE_SIZE);
771
772 mp = vp->v_mount;
773
774 /*
775 * we don't want to do any funny rounding of the size for IO requests
776 * coming through the DIRECT or CONTIGUOUS paths... those pages don't
777 * belong to us... we can't extend (nor do we need to) the I/O to fill
778 * out a page
779 */
780 if (mp->mnt_devblocksize > 1 && !(flags & (CL_DEV_MEMORY | CL_DIRECT_IO))) {
781 /*
782 * round the requested size up so that this I/O ends on a
783 * page boundary in case this is a 'write'... if the filesystem
784 * has blocks allocated to back the page beyond the EOF, we want to
785 * make sure to write out the zero's that are sitting beyond the EOF
786 * so that in case the filesystem doesn't explicitly zero this area
787 * if a hole is created via a lseek/write beyond the current EOF,
788 * it will return zeros when it's read back from the disk. If the
789 * physical allocation doesn't extend for the whole page, we'll
790 * only write/read from the disk up to the end of this allocation
791 * via the extent info returned from the VNOP_BLOCKMAP call.
792 */
793 pg_offset = upl_offset & PAGE_MASK;
794
795 size = (((non_rounded_size + pg_offset) + (PAGE_SIZE - 1)) & ~PAGE_MASK) - pg_offset;
796 } else {
797 /*
798 * anyone advertising a blocksize of 1 byte probably
799 * can't deal with us rounding up the request size
800 * AFP is one such filesystem/device
801 */
802 size = non_rounded_size;
803 }
804 upl_end_offset = upl_offset + size;
805
806 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 22)) | DBG_FUNC_START, (int)f_offset, size, upl_offset, flags, 0);
807
808 /*
809 * Set the maximum transaction size to the maximum desired number of
810 * buffers.
811 */
812 max_trans_count = 8;
813 if (flags & CL_DEV_MEMORY)
814 max_trans_count = 16;
815
816 if (flags & CL_READ) {
817 io_flags = B_READ;
818 bmap_flags = VNODE_READ;
819
820 max_iosize = mp->mnt_maxreadcnt;
821 max_vectors = mp->mnt_segreadcnt;
822 } else {
823 io_flags = B_WRITE;
824 bmap_flags = VNODE_WRITE;
825
826 max_iosize = mp->mnt_maxwritecnt;
827 max_vectors = mp->mnt_segwritecnt;
828 }
829 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 22)) | DBG_FUNC_NONE, max_iosize, max_vectors, mp->mnt_devblocksize, 0, 0);
830
831 /*
832 * make sure the maximum iosize is a
833 * multiple of the page size
834 */
835 max_iosize &= ~PAGE_MASK;
836
837 /*
838 * Ensure the maximum iosize is sensible.
839 */
840 if (!max_iosize)
841 max_iosize = PAGE_SIZE;
842
843 if (flags & CL_THROTTLE) {
844 if ( !(flags & CL_PAGEOUT) && cluster_hard_throttle_on(vp)) {
845 if (max_iosize > HARD_THROTTLE_MAXSIZE)
846 max_iosize = HARD_THROTTLE_MAXSIZE;
847 async_throttle = HARD_THROTTLE_MAXCNT;
848 } else {
849 if ( (flags & CL_DEV_MEMORY) )
850 async_throttle = VNODE_ASYNC_THROTTLE;
851 else {
852 u_int max_cluster;
853 u_int max_cluster_size;
854 u_int max_prefetch;
855
856 max_cluster_size = MAX_CLUSTER_SIZE(vp);
857 max_prefetch = MAX_PREFETCH(vp);
858
859 if (max_iosize > max_cluster_size)
860 max_cluster = max_cluster_size;
861 else
862 max_cluster = max_iosize;
863
864 if (size < max_cluster)
865 max_cluster = size;
866
867 async_throttle = min(VNODE_ASYNC_THROTTLE, (max_prefetch / max_cluster) - 1);
868 }
869 }
870 }
871 if (flags & CL_AGE)
872 io_flags |= B_AGE;
873 if (flags & (CL_PAGEIN | CL_PAGEOUT))
874 io_flags |= B_PAGEIO;
875 if (flags & CL_COMMIT)
876 io_flags |= B_COMMIT_UPL;
877 if (flags & CL_PRESERVE)
878 io_flags |= B_PHYS;
879 if (flags & CL_KEEPCACHED)
880 io_flags |= B_CACHE;
881 if (flags & CL_PASSIVE)
882 io_flags |= B_PASSIVE;
883 if (vp->v_flag & VSYSTEM)
884 io_flags |= B_META;
885
886 if ((flags & CL_READ) && ((upl_offset + non_rounded_size) & PAGE_MASK) && (!(flags & CL_NOZERO))) {
887 /*
888 * then we are going to end up
889 * with a page that we can't complete (the file size wasn't a multiple
890 * of PAGE_SIZE and we're trying to read to the end of the file
891 * so we'll go ahead and zero out the portion of the page we can't
892 * read in from the file
893 */
894 zero_offset = upl_offset + non_rounded_size;
895 }
896 while (size) {
897 daddr64_t blkno;
898 daddr64_t lblkno;
899 u_int io_size_wanted;
900
901 if (size > max_iosize)
902 io_size = max_iosize;
903 else
904 io_size = size;
905
906 io_size_wanted = io_size;
907
908 if ((error = VNOP_BLOCKMAP(vp, f_offset, io_size, &blkno, (size_t *)&io_size, NULL, bmap_flags, NULL)))
909 break;
910
911 if (io_size > io_size_wanted)
912 io_size = io_size_wanted;
913
914 if (real_bp && (real_bp->b_blkno == real_bp->b_lblkno))
915 real_bp->b_blkno = blkno;
916
917 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 24)) | DBG_FUNC_NONE,
918 (int)f_offset, (int)(blkno>>32), (int)blkno, io_size, 0);
919
920 if (io_size == 0) {
921 /*
922 * vnop_blockmap didn't return an error... however, it did
923 * return an extent size of 0 which means we can't
924 * make forward progress on this I/O... a hole in the
925 * file would be returned as a blkno of -1 with a non-zero io_size
926 * a real extent is returned with a blkno != -1 and a non-zero io_size
927 */
928 error = EINVAL;
929 break;
930 }
931 if ( !(flags & CL_READ) && blkno == -1) {
932 off_t e_offset;
933 int pageout_flags;
934
935 /*
936 * we're writing into a 'hole'
937 */
938 if (flags & CL_PAGEOUT) {
939 /*
940 * if we got here via cluster_pageout
941 * then just error the request and return
942 * the 'hole' should already have been covered
943 */
944 error = EINVAL;
945 break;
946 }
947 /*
948 * we can get here if the cluster code happens to
949 * pick up a page that was dirtied via mmap vs
950 * a 'write' and the page targets a 'hole'...
951 * i.e. the writes to the cluster were sparse
952 * and the file was being written for the first time
953 *
954 * we can also get here if the filesystem supports
955 * 'holes' that are less than PAGE_SIZE.... because
956 * we can't know if the range in the page that covers
957 * the 'hole' has been dirtied via an mmap or not,
958 * we have to assume the worst and try to push the
959 * entire page to storage.
960 *
961 * Try paging out the page individually before
962 * giving up entirely and dumping it (the pageout
963 * path will insure that the zero extent accounting
964 * has been taken care of before we get back into cluster_io)
965 *
966 * go direct to vnode_pageout so that we don't have to
967 * unbusy the page from the UPL... we used to do this
968 * so that we could call ubc_sync_range, but that results
969 * in a potential deadlock if someone else races us to acquire
970 * that page and wins and in addition needs one of the pages
971 * we're continuing to hold in the UPL
972 */
973 pageout_flags = UPL_MSYNC | UPL_VNODE_PAGER | UPL_NESTED_PAGEOUT;
974
975 if ( !(flags & CL_ASYNC))
976 pageout_flags |= UPL_IOSYNC;
977 if ( !(flags & CL_COMMIT))
978 pageout_flags |= UPL_NOCOMMIT;
979
980 if (cbp_head) {
981 buf_t last_cbp;
982
983 /*
984 * first we have to wait for the the current outstanding I/Os
985 * to complete... EOT hasn't been set yet on this transaction
986 * so the pages won't be released just because all of the current
987 * I/O linked to this transaction has completed...
988 */
989 cluster_wait_IO(cbp_head, (flags & CL_ASYNC));
990
991 /*
992 * we've got a transcation that
993 * includes the page we're about to push out through vnode_pageout...
994 * find the last bp in the list which will be the one that
995 * includes the head of this page and round it's iosize down
996 * to a page boundary...
997 */
998 for (last_cbp = cbp = cbp_head; cbp->b_trans_next; cbp = cbp->b_trans_next)
999 last_cbp = cbp;
1000
1001 cbp->b_bcount &= ~PAGE_MASK;
1002
1003 if (cbp->b_bcount == 0) {
1004 /*
1005 * this buf no longer has any I/O associated with it
1006 */
1007 free_io_buf(cbp);
1008
1009 if (cbp == cbp_head) {
1010 /*
1011 * the buf we just freed was the only buf in
1012 * this transaction... so there's no I/O to do
1013 */
1014 cbp_head = NULL;
1015 } else {
1016 /*
1017 * remove the buf we just freed from
1018 * the transaction list
1019 */
1020 last_cbp->b_trans_next = NULL;
1021 cbp_tail = last_cbp;
1022 }
1023 }
1024 if (cbp_head) {
1025 /*
1026 * there was more to the current transaction
1027 * than just the page we are pushing out via vnode_pageout...
1028 * mark it as finished and complete it... we've already
1029 * waited for the I/Os to complete above in the call to cluster_wait_IO
1030 */
1031 cluster_EOT(cbp_head, cbp_tail, 0);
1032
1033 cluster_complete_transaction(&cbp_head, callback_arg, &retval, flags, 0);
1034
1035 trans_count = 0;
1036 }
1037 }
1038 if (vnode_pageout(vp, upl, trunc_page(upl_offset), trunc_page_64(f_offset), PAGE_SIZE, pageout_flags, NULL) != PAGER_SUCCESS) {
1039 error = EINVAL;
1040 break;
1041 }
1042 e_offset = round_page_64(f_offset + 1);
1043 io_size = e_offset - f_offset;
1044
1045 f_offset += io_size;
1046 upl_offset += io_size;
1047
1048 if (size >= io_size)
1049 size -= io_size;
1050 else
1051 size = 0;
1052 /*
1053 * keep track of how much of the original request
1054 * that we've actually completed... non_rounded_size
1055 * may go negative due to us rounding the request
1056 * to a page size multiple (i.e. size > non_rounded_size)
1057 */
1058 non_rounded_size -= io_size;
1059
1060 if (non_rounded_size <= 0) {
1061 /*
1062 * we've transferred all of the data in the original
1063 * request, but we were unable to complete the tail
1064 * of the last page because the file didn't have
1065 * an allocation to back that portion... this is ok.
1066 */
1067 size = 0;
1068 }
1069 continue;
1070 }
1071 lblkno = (daddr64_t)(f_offset / PAGE_SIZE_64);
1072 /*
1073 * we have now figured out how much I/O we can do - this is in 'io_size'
1074 * pg_offset is the starting point in the first page for the I/O
1075 * pg_count is the number of full and partial pages that 'io_size' encompasses
1076 */
1077 pg_offset = upl_offset & PAGE_MASK;
1078
1079 if (flags & CL_DEV_MEMORY) {
1080 /*
1081 * treat physical requests as one 'giant' page
1082 */
1083 pg_count = 1;
1084 } else
1085 pg_count = (io_size + pg_offset + (PAGE_SIZE - 1)) / PAGE_SIZE;
1086
1087 if ((flags & CL_READ) && blkno == -1) {
1088 vm_offset_t commit_offset;
1089 int bytes_to_zero;
1090 int complete_transaction_now = 0;
1091
1092 /*
1093 * if we're reading and blkno == -1, then we've got a
1094 * 'hole' in the file that we need to deal with by zeroing
1095 * out the affected area in the upl
1096 */
1097 if (io_size >= (u_int)non_rounded_size) {
1098 /*
1099 * if this upl contains the EOF and it is not a multiple of PAGE_SIZE
1100 * than 'zero_offset' will be non-zero
1101 * if the 'hole' returned by vnop_blockmap extends all the way to the eof
1102 * (indicated by the io_size finishing off the I/O request for this UPL)
1103 * than we're not going to issue an I/O for the
1104 * last page in this upl... we need to zero both the hole and the tail
1105 * of the page beyond the EOF, since the delayed zero-fill won't kick in
1106 */
1107 bytes_to_zero = non_rounded_size;
1108 if (!(flags & CL_NOZERO))
1109 bytes_to_zero = (((upl_offset + io_size) + (PAGE_SIZE - 1)) & ~PAGE_MASK) - upl_offset;
1110
1111 zero_offset = 0;
1112 } else
1113 bytes_to_zero = io_size;
1114
1115 pg_count = 0;
1116
1117 cluster_zero(upl, upl_offset, bytes_to_zero, real_bp);
1118
1119 if (cbp_head) {
1120 int pg_resid;
1121
1122 /*
1123 * if there is a current I/O chain pending
1124 * then the first page of the group we just zero'd
1125 * will be handled by the I/O completion if the zero
1126 * fill started in the middle of the page
1127 */
1128 commit_offset = (upl_offset + (PAGE_SIZE - 1)) & ~PAGE_MASK;
1129
1130 pg_resid = commit_offset - upl_offset;
1131
1132 if (bytes_to_zero >= pg_resid) {
1133 /*
1134 * the last page of the current I/O
1135 * has been completed...
1136 * compute the number of fully zero'd
1137 * pages that are beyond it
1138 * plus the last page if its partial
1139 * and we have no more I/O to issue...
1140 * otherwise a partial page is left
1141 * to begin the next I/O
1142 */
1143 if ((int)io_size >= non_rounded_size)
1144 pg_count = (bytes_to_zero - pg_resid + (PAGE_SIZE - 1)) / PAGE_SIZE;
1145 else
1146 pg_count = (bytes_to_zero - pg_resid) / PAGE_SIZE;
1147
1148 complete_transaction_now = 1;
1149 }
1150 } else {
1151 /*
1152 * no pending I/O to deal with
1153 * so, commit all of the fully zero'd pages
1154 * plus the last page if its partial
1155 * and we have no more I/O to issue...
1156 * otherwise a partial page is left
1157 * to begin the next I/O
1158 */
1159 if ((int)io_size >= non_rounded_size)
1160 pg_count = (pg_offset + bytes_to_zero + (PAGE_SIZE - 1)) / PAGE_SIZE;
1161 else
1162 pg_count = (pg_offset + bytes_to_zero) / PAGE_SIZE;
1163
1164 commit_offset = upl_offset & ~PAGE_MASK;
1165 }
1166 if ( (flags & CL_COMMIT) && pg_count) {
1167 ubc_upl_commit_range(upl, commit_offset, pg_count * PAGE_SIZE,
1168 UPL_COMMIT_CLEAR_DIRTY | UPL_COMMIT_FREE_ON_EMPTY);
1169 }
1170 upl_offset += io_size;
1171 f_offset += io_size;
1172 size -= io_size;
1173
1174 /*
1175 * keep track of how much of the original request
1176 * that we've actually completed... non_rounded_size
1177 * may go negative due to us rounding the request
1178 * to a page size multiple (i.e. size > non_rounded_size)
1179 */
1180 non_rounded_size -= io_size;
1181
1182 if (non_rounded_size <= 0) {
1183 /*
1184 * we've transferred all of the data in the original
1185 * request, but we were unable to complete the tail
1186 * of the last page because the file didn't have
1187 * an allocation to back that portion... this is ok.
1188 */
1189 size = 0;
1190 }
1191 if (cbp_head && (complete_transaction_now || size == 0)) {
1192 cluster_wait_IO(cbp_head, (flags & CL_ASYNC));
1193
1194 cluster_EOT(cbp_head, cbp_tail, size == 0 ? zero_offset : 0);
1195
1196 cluster_complete_transaction(&cbp_head, callback_arg, &retval, flags, 0);
1197
1198 trans_count = 0;
1199 }
1200 continue;
1201 }
1202 if (pg_count > max_vectors) {
1203 if (((pg_count - max_vectors) * PAGE_SIZE) > io_size) {
1204 io_size = PAGE_SIZE - pg_offset;
1205 pg_count = 1;
1206 } else {
1207 io_size -= (pg_count - max_vectors) * PAGE_SIZE;
1208 pg_count = max_vectors;
1209 }
1210 }
1211 /*
1212 * If the transaction is going to reach the maximum number of
1213 * desired elements, truncate the i/o to the nearest page so
1214 * that the actual i/o is initiated after this buffer is
1215 * created and added to the i/o chain.
1216 *
1217 * I/O directed to physically contiguous memory
1218 * doesn't have a requirement to make sure we 'fill' a page
1219 */
1220 if ( !(flags & CL_DEV_MEMORY) && trans_count >= max_trans_count &&
1221 ((upl_offset + io_size) & PAGE_MASK)) {
1222 vm_offset_t aligned_ofs;
1223
1224 aligned_ofs = (upl_offset + io_size) & ~PAGE_MASK;
1225 /*
1226 * If the io_size does not actually finish off even a
1227 * single page we have to keep adding buffers to the
1228 * transaction despite having reached the desired limit.
1229 *
1230 * Eventually we get here with the page being finished
1231 * off (and exceeded) and then we truncate the size of
1232 * this i/o request so that it is page aligned so that
1233 * we can finally issue the i/o on the transaction.
1234 */
1235 if (aligned_ofs > upl_offset) {
1236 io_size = aligned_ofs - upl_offset;
1237 pg_count--;
1238 }
1239 }
1240
1241 if ( !(mp->mnt_kern_flag & MNTK_VIRTUALDEV))
1242 /*
1243 * if we're not targeting a virtual device i.e. a disk image
1244 * it's safe to dip into the reserve pool since real devices
1245 * can complete this I/O request without requiring additional
1246 * bufs from the alloc_io_buf pool
1247 */
1248 priv = 1;
1249 else if ((flags & CL_ASYNC) && !(flags & CL_PAGEOUT))
1250 /*
1251 * Throttle the speculative IO
1252 */
1253 priv = 0;
1254 else
1255 priv = 1;
1256
1257 cbp = alloc_io_buf(vp, priv);
1258
1259 if (flags & CL_PAGEOUT) {
1260 u_int i;
1261
1262 for (i = 0; i < pg_count; i++) {
1263 if (buf_invalblkno(vp, lblkno + i, 0) == EBUSY)
1264 panic("BUSY bp found in cluster_io");
1265 }
1266 }
1267 if (flags & CL_ASYNC) {
1268 if (buf_setcallback(cbp, (void *)cluster_iodone, callback_arg))
1269 panic("buf_setcallback failed\n");
1270 }
1271 cbp->b_cliodone = (void *)callback;
1272 cbp->b_flags |= io_flags;
1273
1274 cbp->b_lblkno = lblkno;
1275 cbp->b_blkno = blkno;
1276 cbp->b_bcount = io_size;
1277
1278 if (buf_setupl(cbp, upl, upl_offset))
1279 panic("buf_setupl failed\n");
1280
1281 cbp->b_trans_next = (buf_t)NULL;
1282
1283 if ((cbp->b_iostate = (void *)iostate))
1284 /*
1285 * caller wants to track the state of this
1286 * io... bump the amount issued against this stream
1287 */
1288 iostate->io_issued += io_size;
1289
1290 if (flags & CL_READ) {
1291 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 26)) | DBG_FUNC_NONE,
1292 (int)cbp->b_lblkno, (int)cbp->b_blkno, upl_offset, io_size, 0);
1293 }
1294 else {
1295 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 27)) | DBG_FUNC_NONE,
1296 (int)cbp->b_lblkno, (int)cbp->b_blkno, upl_offset, io_size, 0);
1297 }
1298
1299 if (cbp_head) {
1300 cbp_tail->b_trans_next = cbp;
1301 cbp_tail = cbp;
1302 } else {
1303 cbp_head = cbp;
1304 cbp_tail = cbp;
1305
1306 if ( (cbp_head->b_real_bp = real_bp) ) {
1307 cbp_head->b_flags |= B_NEED_IODONE;
1308 real_bp = (buf_t)NULL;
1309 }
1310 }
1311 *(buf_t *)(&cbp->b_trans_head) = cbp_head;
1312
1313 trans_count++;
1314
1315 upl_offset += io_size;
1316 f_offset += io_size;
1317 size -= io_size;
1318 /*
1319 * keep track of how much of the original request
1320 * that we've actually completed... non_rounded_size
1321 * may go negative due to us rounding the request
1322 * to a page size multiple (i.e. size > non_rounded_size)
1323 */
1324 non_rounded_size -= io_size;
1325
1326 if (non_rounded_size <= 0) {
1327 /*
1328 * we've transferred all of the data in the original
1329 * request, but we were unable to complete the tail
1330 * of the last page because the file didn't have
1331 * an allocation to back that portion... this is ok.
1332 */
1333 size = 0;
1334 }
1335 if (size == 0) {
1336 /*
1337 * we have no more I/O to issue, so go
1338 * finish the final transaction
1339 */
1340 need_EOT = TRUE;
1341 } else if ( ((flags & CL_DEV_MEMORY) || (upl_offset & PAGE_MASK) == 0) &&
1342 ((flags & CL_ASYNC) || trans_count > max_trans_count) ) {
1343 /*
1344 * I/O directed to physically contiguous memory...
1345 * which doesn't have a requirement to make sure we 'fill' a page
1346 * or...
1347 * the current I/O we've prepared fully
1348 * completes the last page in this request
1349 * and ...
1350 * it's either an ASYNC request or
1351 * we've already accumulated more than 8 I/O's into
1352 * this transaction so mark it as complete so that
1353 * it can finish asynchronously or via the cluster_complete_transaction
1354 * below if the request is synchronous
1355 */
1356 need_EOT = TRUE;
1357 }
1358 if (need_EOT == TRUE)
1359 cluster_EOT(cbp_head, cbp_tail, size == 0 ? zero_offset : 0);
1360
1361 if (flags & CL_THROTTLE)
1362 (void)vnode_waitforwrites(vp, async_throttle, 0, 0, "cluster_io");
1363
1364 if ( !(io_flags & B_READ))
1365 vnode_startwrite(vp);
1366
1367 (void) VNOP_STRATEGY(cbp);
1368
1369 if (need_EOT == TRUE) {
1370 if ( !(flags & CL_ASYNC))
1371 cluster_complete_transaction(&cbp_head, callback_arg, &retval, flags, 1);
1372
1373 need_EOT = FALSE;
1374 trans_count = 0;
1375 cbp_head = NULL;
1376 }
1377 }
1378 if (error) {
1379 int abort_size;
1380
1381 io_size = 0;
1382
1383 if (cbp_head) {
1384 /*
1385 * first wait until all of the outstanding I/O
1386 * for this partial transaction has completed
1387 */
1388 cluster_wait_IO(cbp_head, (flags & CL_ASYNC));
1389
1390 /*
1391 * Rewind the upl offset to the beginning of the
1392 * transaction.
1393 */
1394 upl_offset = cbp_head->b_uploffset;
1395
1396 for (cbp = cbp_head; cbp;) {
1397 buf_t cbp_next;
1398
1399 size += cbp->b_bcount;
1400 io_size += cbp->b_bcount;
1401
1402 cbp_next = cbp->b_trans_next;
1403 free_io_buf(cbp);
1404 cbp = cbp_next;
1405 }
1406 }
1407 if (iostate) {
1408 int need_wakeup = 0;
1409
1410 /*
1411 * update the error condition for this stream
1412 * since we never really issued the io
1413 * just go ahead and adjust it back
1414 */
1415 lck_mtx_lock_spin(cl_mtxp);
1416
1417 if (iostate->io_error == 0)
1418 iostate->io_error = error;
1419 iostate->io_issued -= io_size;
1420
1421 if (iostate->io_wanted) {
1422 /*
1423 * someone is waiting for the state of
1424 * this io stream to change
1425 */
1426 iostate->io_wanted = 0;
1427 need_wakeup = 1;
1428 }
1429 lck_mtx_unlock(cl_mtxp);
1430
1431 if (need_wakeup)
1432 wakeup((caddr_t)&iostate->io_wanted);
1433 }
1434 if (flags & CL_COMMIT) {
1435 int upl_flags;
1436
1437 pg_offset = upl_offset & PAGE_MASK;
1438 abort_size = (upl_end_offset - upl_offset + PAGE_MASK) & ~PAGE_MASK;
1439
1440 upl_flags = cluster_ioerror(upl, upl_offset - pg_offset, abort_size, error, io_flags);
1441
1442 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 28)) | DBG_FUNC_NONE,
1443 (int)upl, upl_offset - pg_offset, abort_size, (error << 24) | upl_flags, 0);
1444 }
1445 if (retval == 0)
1446 retval = error;
1447 } else if (cbp_head)
1448 panic("%s(): cbp_head is not NULL.\n", __FUNCTION__);
1449
1450 if (real_bp) {
1451 /*
1452 * can get here if we either encountered an error
1453 * or we completely zero-filled the request and
1454 * no I/O was issued
1455 */
1456 if (error) {
1457 real_bp->b_flags |= B_ERROR;
1458 real_bp->b_error = error;
1459 }
1460 buf_biodone(real_bp);
1461 }
1462 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 22)) | DBG_FUNC_END, (int)f_offset, size, upl_offset, retval, 0);
1463
1464 return (retval);
1465 }
1466
1467
1468 static int
1469 cluster_read_prefetch(vnode_t vp, off_t f_offset, u_int size, off_t filesize, int (*callback)(buf_t, void *), void *callback_arg, int bflag)
1470 {
1471 int pages_in_prefetch;
1472
1473 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 49)) | DBG_FUNC_START,
1474 (int)f_offset, size, (int)filesize, 0, 0);
1475
1476 if (f_offset >= filesize) {
1477 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 49)) | DBG_FUNC_END,
1478 (int)f_offset, 0, 0, 0, 0);
1479 return(0);
1480 }
1481 if ((off_t)size > (filesize - f_offset))
1482 size = filesize - f_offset;
1483 pages_in_prefetch = (size + (PAGE_SIZE - 1)) / PAGE_SIZE;
1484
1485 advisory_read_ext(vp, filesize, f_offset, size, callback, callback_arg, bflag);
1486
1487 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 49)) | DBG_FUNC_END,
1488 (int)f_offset + size, pages_in_prefetch, 0, 1, 0);
1489
1490 return (pages_in_prefetch);
1491 }
1492
1493
1494
1495 static void
1496 cluster_read_ahead(vnode_t vp, struct cl_extent *extent, off_t filesize, struct cl_readahead *rap, int (*callback)(buf_t, void *), void *callback_arg,
1497 int bflag)
1498 {
1499 daddr64_t r_addr;
1500 off_t f_offset;
1501 int size_of_prefetch;
1502 u_int max_prefetch;
1503
1504
1505 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_START,
1506 (int)extent->b_addr, (int)extent->e_addr, (int)rap->cl_lastr, 0, 0);
1507
1508 if (extent->b_addr == rap->cl_lastr && extent->b_addr == extent->e_addr) {
1509 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_END,
1510 rap->cl_ralen, (int)rap->cl_maxra, (int)rap->cl_lastr, 0, 0);
1511 return;
1512 }
1513 if (rap->cl_lastr == -1 || (extent->b_addr != rap->cl_lastr && extent->b_addr != (rap->cl_lastr + 1))) {
1514 rap->cl_ralen = 0;
1515 rap->cl_maxra = 0;
1516
1517 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_END,
1518 rap->cl_ralen, (int)rap->cl_maxra, (int)rap->cl_lastr, 1, 0);
1519
1520 return;
1521 }
1522 max_prefetch = MAX_PREFETCH(vp);
1523
1524 if (extent->e_addr < rap->cl_maxra) {
1525 if ((rap->cl_maxra - extent->e_addr) > ((max_prefetch / PAGE_SIZE) / 4)) {
1526
1527 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_END,
1528 rap->cl_ralen, (int)rap->cl_maxra, (int)rap->cl_lastr, 2, 0);
1529 return;
1530 }
1531 }
1532 r_addr = max(extent->e_addr, rap->cl_maxra) + 1;
1533 f_offset = (off_t)(r_addr * PAGE_SIZE_64);
1534
1535 size_of_prefetch = 0;
1536
1537 ubc_range_op(vp, f_offset, f_offset + PAGE_SIZE_64, UPL_ROP_PRESENT, &size_of_prefetch);
1538
1539 if (size_of_prefetch) {
1540 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_END,
1541 rap->cl_ralen, (int)rap->cl_maxra, (int)rap->cl_lastr, 3, 0);
1542 return;
1543 }
1544 if (f_offset < filesize) {
1545 daddr64_t read_size;
1546
1547 rap->cl_ralen = rap->cl_ralen ? min(max_prefetch / PAGE_SIZE, rap->cl_ralen << 1) : 1;
1548
1549 read_size = (extent->e_addr + 1) - extent->b_addr;
1550
1551 if (read_size > rap->cl_ralen) {
1552 if (read_size > max_prefetch / PAGE_SIZE)
1553 rap->cl_ralen = max_prefetch / PAGE_SIZE;
1554 else
1555 rap->cl_ralen = read_size;
1556 }
1557 size_of_prefetch = cluster_read_prefetch(vp, f_offset, rap->cl_ralen * PAGE_SIZE, filesize, callback, callback_arg, bflag);
1558
1559 if (size_of_prefetch)
1560 rap->cl_maxra = (r_addr + size_of_prefetch) - 1;
1561 }
1562 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_END,
1563 rap->cl_ralen, (int)rap->cl_maxra, (int)rap->cl_lastr, 4, 0);
1564 }
1565
1566
1567 int
1568 cluster_pageout(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset,
1569 int size, off_t filesize, int flags)
1570 {
1571 return cluster_pageout_ext(vp, upl, upl_offset, f_offset, size, filesize, flags, NULL, NULL);
1572
1573 }
1574
1575
1576 int
1577 cluster_pageout_ext(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset,
1578 int size, off_t filesize, int flags, int (*callback)(buf_t, void *), void *callback_arg)
1579 {
1580 int io_size;
1581 int rounded_size;
1582 off_t max_size;
1583 int local_flags;
1584
1585 if (vp->v_mount->mnt_kern_flag & MNTK_VIRTUALDEV)
1586 /*
1587 * if we know we're issuing this I/O to a virtual device (i.e. disk image)
1588 * then we don't want to enforce this throttle... if we do, we can
1589 * potentially deadlock since we're stalling the pageout thread at a time
1590 * when the disk image might need additional memory (which won't be available
1591 * if the pageout thread can't run)... instead we'll just depend on the throttle
1592 * that the pageout thread now has in place to deal with external files
1593 */
1594 local_flags = CL_PAGEOUT;
1595 else
1596 local_flags = CL_PAGEOUT | CL_THROTTLE;
1597
1598 if ((flags & UPL_IOSYNC) == 0)
1599 local_flags |= CL_ASYNC;
1600 if ((flags & UPL_NOCOMMIT) == 0)
1601 local_flags |= CL_COMMIT;
1602 if ((flags & UPL_KEEPCACHED))
1603 local_flags |= CL_KEEPCACHED;
1604 if (flags & IO_PASSIVE)
1605 local_flags |= CL_PASSIVE;
1606
1607
1608 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 52)) | DBG_FUNC_NONE,
1609 (int)f_offset, size, (int)filesize, local_flags, 0);
1610
1611 /*
1612 * If they didn't specify any I/O, then we are done...
1613 * we can't issue an abort because we don't know how
1614 * big the upl really is
1615 */
1616 if (size <= 0)
1617 return (EINVAL);
1618
1619 if (vp->v_mount->mnt_flag & MNT_RDONLY) {
1620 if (local_flags & CL_COMMIT)
1621 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
1622 return (EROFS);
1623 }
1624 /*
1625 * can't page-in from a negative offset
1626 * or if we're starting beyond the EOF
1627 * or if the file offset isn't page aligned
1628 * or the size requested isn't a multiple of PAGE_SIZE
1629 */
1630 if (f_offset < 0 || f_offset >= filesize ||
1631 (f_offset & PAGE_MASK_64) || (size & PAGE_MASK)) {
1632 if (local_flags & CL_COMMIT)
1633 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
1634 return (EINVAL);
1635 }
1636 max_size = filesize - f_offset;
1637
1638 if (size < max_size)
1639 io_size = size;
1640 else
1641 io_size = max_size;
1642
1643 rounded_size = (io_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
1644
1645 if (size > rounded_size) {
1646 if (local_flags & CL_COMMIT)
1647 ubc_upl_abort_range(upl, upl_offset + rounded_size, size - rounded_size,
1648 UPL_ABORT_FREE_ON_EMPTY);
1649 }
1650 return (cluster_io(vp, upl, upl_offset, f_offset, io_size,
1651 local_flags, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg));
1652 }
1653
1654
1655 int
1656 cluster_pagein(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset,
1657 int size, off_t filesize, int flags)
1658 {
1659 return cluster_pagein_ext(vp, upl, upl_offset, f_offset, size, filesize, flags, NULL, NULL);
1660 }
1661
1662
1663 int
1664 cluster_pagein_ext(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset,
1665 int size, off_t filesize, int flags, int (*callback)(buf_t, void *), void *callback_arg)
1666 {
1667 u_int io_size;
1668 int rounded_size;
1669 off_t max_size;
1670 int retval;
1671 int local_flags = 0;
1672
1673 if (upl == NULL || size < 0)
1674 panic("cluster_pagein: NULL upl passed in");
1675
1676 if ((flags & UPL_IOSYNC) == 0)
1677 local_flags |= CL_ASYNC;
1678 if ((flags & UPL_NOCOMMIT) == 0)
1679 local_flags |= CL_COMMIT;
1680 if (flags & IO_PASSIVE)
1681 local_flags |= CL_PASSIVE;
1682
1683
1684 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 56)) | DBG_FUNC_NONE,
1685 (int)f_offset, size, (int)filesize, local_flags, 0);
1686
1687 /*
1688 * can't page-in from a negative offset
1689 * or if we're starting beyond the EOF
1690 * or if the file offset isn't page aligned
1691 * or the size requested isn't a multiple of PAGE_SIZE
1692 */
1693 if (f_offset < 0 || f_offset >= filesize ||
1694 (f_offset & PAGE_MASK_64) || (size & PAGE_MASK) || (upl_offset & PAGE_MASK)) {
1695 if (local_flags & CL_COMMIT)
1696 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);
1697 return (EINVAL);
1698 }
1699 max_size = filesize - f_offset;
1700
1701 if (size < max_size)
1702 io_size = size;
1703 else
1704 io_size = max_size;
1705
1706 rounded_size = (io_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
1707
1708 if (size > rounded_size && (local_flags & CL_COMMIT))
1709 ubc_upl_abort_range(upl, upl_offset + rounded_size,
1710 size - rounded_size, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);
1711
1712 retval = cluster_io(vp, upl, upl_offset, f_offset, io_size,
1713 local_flags | CL_READ | CL_PAGEIN, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
1714
1715 return (retval);
1716 }
1717
1718
1719 int
1720 cluster_bp(buf_t bp)
1721 {
1722 return cluster_bp_ext(bp, NULL, NULL);
1723 }
1724
1725
1726 int
1727 cluster_bp_ext(buf_t bp, int (*callback)(buf_t, void *), void *callback_arg)
1728 {
1729 off_t f_offset;
1730 int flags;
1731
1732 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 19)) | DBG_FUNC_START,
1733 (int)bp, (int)bp->b_lblkno, bp->b_bcount, bp->b_flags, 0);
1734
1735 if (bp->b_flags & B_READ)
1736 flags = CL_ASYNC | CL_READ;
1737 else
1738 flags = CL_ASYNC;
1739 if (bp->b_flags & B_PASSIVE)
1740 flags |= CL_PASSIVE;
1741
1742 f_offset = ubc_blktooff(bp->b_vp, bp->b_lblkno);
1743
1744 return (cluster_io(bp->b_vp, bp->b_upl, 0, f_offset, bp->b_bcount, flags, bp, (struct clios *)NULL, callback, callback_arg));
1745 }
1746
1747
1748
1749 int
1750 cluster_write(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF, off_t headOff, off_t tailOff, int xflags)
1751 {
1752 return cluster_write_ext(vp, uio, oldEOF, newEOF, headOff, tailOff, xflags, NULL, NULL);
1753 }
1754
1755
1756 int
1757 cluster_write_ext(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF, off_t headOff, off_t tailOff,
1758 int xflags, int (*callback)(buf_t, void *), void *callback_arg)
1759 {
1760 user_ssize_t cur_resid;
1761 int retval = 0;
1762 int flags;
1763 int zflags;
1764 int bflag;
1765 int write_type = IO_COPY;
1766 u_int32_t write_length;
1767
1768 flags = xflags;
1769
1770 if (flags & IO_PASSIVE)
1771 bflag = CL_PASSIVE;
1772 else
1773 bflag = 0;
1774
1775 if (vp->v_flag & VNOCACHE_DATA)
1776 flags |= IO_NOCACHE;
1777
1778 if (uio == NULL) {
1779 /*
1780 * no user data...
1781 * this call is being made to zero-fill some range in the file
1782 */
1783 retval = cluster_write_copy(vp, NULL, (u_int32_t)0, oldEOF, newEOF, headOff, tailOff, flags, callback, callback_arg);
1784
1785 return(retval);
1786 }
1787 /*
1788 * do a write through the cache if one of the following is true....
1789 * NOCACHE is not true and
1790 * the uio request doesn't target USERSPACE
1791 * otherwise, find out if we want the direct or contig variant for
1792 * the first vector in the uio request
1793 */
1794 if ( (flags & IO_NOCACHE) && UIO_SEG_IS_USER_SPACE(uio->uio_segflg) )
1795 retval = cluster_io_type(uio, &write_type, &write_length, MIN_DIRECT_WRITE_SIZE);
1796
1797 if ( (flags & (IO_TAILZEROFILL | IO_HEADZEROFILL)) && write_type == IO_DIRECT)
1798 /*
1799 * must go through the cached variant in this case
1800 */
1801 write_type = IO_COPY;
1802
1803 while ((cur_resid = uio_resid(uio)) && uio->uio_offset < newEOF && retval == 0) {
1804
1805 switch (write_type) {
1806
1807 case IO_COPY:
1808 /*
1809 * make sure the uio_resid isn't too big...
1810 * internally, we want to handle all of the I/O in
1811 * chunk sizes that fit in a 32 bit int
1812 */
1813 if (cur_resid > (user_ssize_t)(MAX_IO_REQUEST_SIZE)) {
1814 /*
1815 * we're going to have to call cluster_write_copy
1816 * more than once...
1817 *
1818 * only want the last call to cluster_write_copy to
1819 * have the IO_TAILZEROFILL flag set and only the
1820 * first call should have IO_HEADZEROFILL
1821 */
1822 zflags = flags & ~IO_TAILZEROFILL;
1823 flags &= ~IO_HEADZEROFILL;
1824
1825 write_length = MAX_IO_REQUEST_SIZE;
1826 } else {
1827 /*
1828 * last call to cluster_write_copy
1829 */
1830 zflags = flags;
1831
1832 write_length = (u_int32_t)cur_resid;
1833 }
1834 retval = cluster_write_copy(vp, uio, write_length, oldEOF, newEOF, headOff, tailOff, zflags, callback, callback_arg);
1835 break;
1836
1837 case IO_CONTIG:
1838 zflags = flags & ~(IO_TAILZEROFILL | IO_HEADZEROFILL);
1839
1840 if (flags & IO_HEADZEROFILL) {
1841 /*
1842 * only do this once per request
1843 */
1844 flags &= ~IO_HEADZEROFILL;
1845
1846 retval = cluster_write_copy(vp, (struct uio *)0, (u_int32_t)0, (off_t)0, uio->uio_offset,
1847 headOff, (off_t)0, zflags | IO_HEADZEROFILL | IO_SYNC, callback, callback_arg);
1848 if (retval)
1849 break;
1850 }
1851 retval = cluster_write_contig(vp, uio, newEOF, &write_type, &write_length, callback, callback_arg, bflag);
1852
1853 if (retval == 0 && (flags & IO_TAILZEROFILL) && uio_resid(uio) == 0) {
1854 /*
1855 * we're done with the data from the user specified buffer(s)
1856 * and we've been requested to zero fill at the tail
1857 * treat this as an IO_HEADZEROFILL which doesn't require a uio
1858 * by rearranging the args and passing in IO_HEADZEROFILL
1859 */
1860 retval = cluster_write_copy(vp, (struct uio *)0, (u_int32_t)0, (off_t)0, tailOff, uio->uio_offset,
1861 (off_t)0, zflags | IO_HEADZEROFILL | IO_SYNC, callback, callback_arg);
1862 }
1863 break;
1864
1865 case IO_DIRECT:
1866 /*
1867 * cluster_write_direct is never called with IO_TAILZEROFILL || IO_HEADZEROFILL
1868 */
1869 retval = cluster_write_direct(vp, uio, oldEOF, newEOF, &write_type, &write_length, flags, callback, callback_arg);
1870 break;
1871
1872 case IO_UNKNOWN:
1873 retval = cluster_io_type(uio, &write_type, &write_length, MIN_DIRECT_WRITE_SIZE);
1874 break;
1875 }
1876 }
1877 return (retval);
1878 }
1879
1880
1881 static int
1882 cluster_write_direct(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF, int *write_type, u_int32_t *write_length,
1883 int flags, int (*callback)(buf_t, void *), void *callback_arg)
1884 {
1885 upl_t upl;
1886 upl_page_info_t *pl;
1887 vm_offset_t upl_offset;
1888 u_int32_t io_req_size;
1889 u_int32_t offset_in_file;
1890 u_int32_t offset_in_iovbase;
1891 u_int32_t io_size;
1892 int io_flag;
1893 int bflag;
1894 vm_size_t upl_size;
1895 vm_size_t upl_needed_size;
1896 mach_msg_type_number_t pages_in_pl;
1897 int upl_flags;
1898 kern_return_t kret;
1899 mach_msg_type_number_t i;
1900 int force_data_sync;
1901 int retval = 0;
1902 int first_IO = 1;
1903 struct clios iostate;
1904 user_addr_t iov_base;
1905 u_int32_t mem_alignment_mask;
1906 u_int32_t devblocksize;
1907 u_int32_t max_upl_size;
1908
1909
1910 max_upl_size = cluster_max_io_size(vp->v_mount, CL_WRITE);
1911
1912 if (flags & IO_PASSIVE)
1913 bflag = CL_PASSIVE;
1914 else
1915 bflag = 0;
1916
1917 /*
1918 * When we enter this routine, we know
1919 * -- the resid will not exceed iov_len
1920 */
1921 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 75)) | DBG_FUNC_START,
1922 (int)uio->uio_offset, *write_length, (int)newEOF, 0, 0);
1923
1924 iostate.io_completed = 0;
1925 iostate.io_issued = 0;
1926 iostate.io_error = 0;
1927 iostate.io_wanted = 0;
1928
1929 mem_alignment_mask = (u_int32_t)vp->v_mount->mnt_alignmentmask;
1930 devblocksize = (u_int32_t)vp->v_mount->mnt_devblocksize;
1931
1932 if (devblocksize == 1) {
1933 /*
1934 * the AFP client advertises a devblocksize of 1
1935 * however, its BLOCKMAP routine maps to physical
1936 * blocks that are PAGE_SIZE in size...
1937 * therefore we can't ask for I/Os that aren't page aligned
1938 * or aren't multiples of PAGE_SIZE in size
1939 * by setting devblocksize to PAGE_SIZE, we re-instate
1940 * the old behavior we had before the mem_alignment_mask
1941 * changes went in...
1942 */
1943 devblocksize = PAGE_SIZE;
1944 }
1945
1946 next_dwrite:
1947 io_req_size = *write_length;
1948 iov_base = uio_curriovbase(uio);
1949
1950 offset_in_file = (u_int32_t)uio->uio_offset & PAGE_MASK;
1951 offset_in_iovbase = (u_int32_t)iov_base & mem_alignment_mask;
1952
1953 if (offset_in_file || offset_in_iovbase) {
1954 /*
1955 * one of the 2 important offsets is misaligned
1956 * so fire an I/O through the cache for this entire vector
1957 */
1958 goto wait_for_dwrites;
1959 }
1960 if (iov_base & (devblocksize - 1)) {
1961 /*
1962 * the offset in memory must be on a device block boundary
1963 * so that we can guarantee that we can generate an
1964 * I/O that ends on a page boundary in cluster_io
1965 */
1966 goto wait_for_dwrites;
1967 }
1968
1969 while (io_req_size >= PAGE_SIZE && uio->uio_offset < newEOF && retval == 0) {
1970
1971 if (first_IO) {
1972 cluster_syncup(vp, newEOF, callback, callback_arg);
1973 first_IO = 0;
1974 }
1975 io_size = io_req_size & ~PAGE_MASK;
1976 iov_base = uio_curriovbase(uio);
1977
1978 if (io_size > max_upl_size)
1979 io_size = max_upl_size;
1980
1981 upl_offset = (vm_offset_t)((u_int32_t)iov_base & PAGE_MASK);
1982 upl_needed_size = (upl_offset + io_size + (PAGE_SIZE -1)) & ~PAGE_MASK;
1983
1984 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 76)) | DBG_FUNC_START,
1985 (int)upl_offset, upl_needed_size, (int)iov_base, io_size, 0);
1986
1987 for (force_data_sync = 0; force_data_sync < 3; force_data_sync++) {
1988 pages_in_pl = 0;
1989 upl_size = upl_needed_size;
1990 upl_flags = UPL_FILE_IO | UPL_COPYOUT_FROM | UPL_NO_SYNC |
1991 UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE;
1992
1993 kret = vm_map_get_upl(current_map(),
1994 (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)),
1995 &upl_size,
1996 &upl,
1997 NULL,
1998 &pages_in_pl,
1999 &upl_flags,
2000 force_data_sync);
2001
2002 if (kret != KERN_SUCCESS) {
2003 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 76)) | DBG_FUNC_END,
2004 0, 0, 0, kret, 0);
2005 /*
2006 * failed to get pagelist
2007 *
2008 * we may have already spun some portion of this request
2009 * off as async requests... we need to wait for the I/O
2010 * to complete before returning
2011 */
2012 goto wait_for_dwrites;
2013 }
2014 pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
2015 pages_in_pl = upl_size / PAGE_SIZE;
2016
2017 for (i = 0; i < pages_in_pl; i++) {
2018 if (!upl_valid_page(pl, i))
2019 break;
2020 }
2021 if (i == pages_in_pl)
2022 break;
2023
2024 /*
2025 * didn't get all the pages back that we
2026 * needed... release this upl and try again
2027 */
2028 ubc_upl_abort(upl, 0);
2029 }
2030 if (force_data_sync >= 3) {
2031 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 76)) | DBG_FUNC_END,
2032 i, pages_in_pl, upl_size, kret, 0);
2033 /*
2034 * for some reason, we couldn't acquire a hold on all
2035 * the pages needed in the user's address space
2036 *
2037 * we may have already spun some portion of this request
2038 * off as async requests... we need to wait for the I/O
2039 * to complete before returning
2040 */
2041 goto wait_for_dwrites;
2042 }
2043
2044 /*
2045 * Consider the possibility that upl_size wasn't satisfied.
2046 */
2047 if (upl_size < upl_needed_size) {
2048 if (upl_size && upl_offset == 0)
2049 io_size = upl_size;
2050 else
2051 io_size = 0;
2052 }
2053 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 76)) | DBG_FUNC_END,
2054 (int)upl_offset, upl_size, (int)iov_base, io_size, 0);
2055
2056 if (io_size == 0) {
2057 ubc_upl_abort(upl, 0);
2058 /*
2059 * we may have already spun some portion of this request
2060 * off as async requests... we need to wait for the I/O
2061 * to complete before returning
2062 */
2063 goto wait_for_dwrites;
2064 }
2065
2066 /*
2067 * Now look for pages already in the cache
2068 * and throw them away.
2069 * uio->uio_offset is page aligned within the file
2070 * io_size is a multiple of PAGE_SIZE
2071 */
2072 ubc_range_op(vp, uio->uio_offset, uio->uio_offset + io_size, UPL_ROP_DUMP, NULL);
2073
2074 /*
2075 * we want push out these writes asynchronously so that we can overlap
2076 * the preparation of the next I/O
2077 * if there are already too many outstanding writes
2078 * wait until some complete before issuing the next
2079 */
2080 lck_mtx_lock(cl_mtxp);
2081
2082 while ((iostate.io_issued - iostate.io_completed) > (2 * max_upl_size)) {
2083
2084 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_START,
2085 iostate.io_issued, iostate.io_completed, 2 * max_upl_size, 0, 0);
2086
2087 iostate.io_wanted = 1;
2088 msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_write_direct", NULL);
2089
2090 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_END,
2091 iostate.io_issued, iostate.io_completed, 2 * max_upl_size, 0, 0);
2092 }
2093 lck_mtx_unlock(cl_mtxp);
2094
2095 if (iostate.io_error) {
2096 /*
2097 * one of the earlier writes we issued ran into a hard error
2098 * don't issue any more writes, cleanup the UPL
2099 * that was just created but not used, then
2100 * go wait for all writes that are part of this stream
2101 * to complete before returning the error to the caller
2102 */
2103 ubc_upl_abort(upl, 0);
2104
2105 goto wait_for_dwrites;
2106 }
2107 io_flag = CL_ASYNC | CL_PRESERVE | CL_COMMIT | CL_THROTTLE | CL_DIRECT_IO | bflag;
2108
2109 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 77)) | DBG_FUNC_START,
2110 (int)upl_offset, (int)uio->uio_offset, io_size, io_flag, 0);
2111
2112 retval = cluster_io(vp, upl, upl_offset, uio->uio_offset,
2113 io_size, io_flag, (buf_t)NULL, &iostate, callback, callback_arg);
2114
2115 /*
2116 * update the uio structure to
2117 * reflect the I/O that we just issued
2118 */
2119 uio_update(uio, (user_size_t)io_size);
2120
2121 io_req_size -= io_size;
2122
2123 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 77)) | DBG_FUNC_END,
2124 (int)upl_offset, (int)uio->uio_offset, io_req_size, retval, 0);
2125
2126 } /* end while */
2127
2128 if (retval == 0 && iostate.io_error == 0 && io_req_size == 0) {
2129
2130 retval = cluster_io_type(uio, write_type, write_length, MIN_DIRECT_WRITE_SIZE);
2131
2132 if (retval == 0 && *write_type == IO_DIRECT) {
2133
2134 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 75)) | DBG_FUNC_NONE,
2135 (int)uio->uio_offset, *write_length, (int)newEOF, 0, 0);
2136
2137 goto next_dwrite;
2138 }
2139 }
2140
2141 wait_for_dwrites:
2142 if (iostate.io_issued) {
2143 /*
2144 * make sure all async writes issued as part of this stream
2145 * have completed before we return
2146 */
2147 lck_mtx_lock(cl_mtxp);
2148
2149 while (iostate.io_issued != iostate.io_completed) {
2150 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_START,
2151 iostate.io_issued, iostate.io_completed, 0, 0, 0);
2152
2153 iostate.io_wanted = 1;
2154 msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_write_direct", NULL);
2155
2156 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_END,
2157 iostate.io_issued, iostate.io_completed, 0, 0, 0);
2158 }
2159 lck_mtx_unlock(cl_mtxp);
2160 }
2161 if (iostate.io_error)
2162 retval = iostate.io_error;
2163
2164 if (io_req_size && retval == 0) {
2165 /*
2166 * we couldn't handle the tail of this request in DIRECT mode
2167 * so fire it through the copy path
2168 *
2169 * note that flags will never have IO_HEADZEROFILL or IO_TAILZEROFILL set
2170 * so we can just pass 0 in for the headOff and tailOff
2171 */
2172 retval = cluster_write_copy(vp, uio, io_req_size, oldEOF, newEOF, (off_t)0, (off_t)0, flags, callback, callback_arg);
2173
2174 *write_type = IO_UNKNOWN;
2175 }
2176 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 75)) | DBG_FUNC_END,
2177 (int)uio->uio_offset, io_req_size, retval, 4, 0);
2178
2179 return (retval);
2180 }
2181
2182
2183 static int
2184 cluster_write_contig(vnode_t vp, struct uio *uio, off_t newEOF, int *write_type, u_int32_t *write_length,
2185 int (*callback)(buf_t, void *), void *callback_arg, int bflag)
2186 {
2187 upl_page_info_t *pl;
2188 addr64_t src_paddr = 0;
2189 upl_t upl[MAX_VECTS];
2190 vm_offset_t upl_offset;
2191 u_int32_t tail_size = 0;
2192 u_int32_t io_size;
2193 u_int32_t xsize;
2194 vm_size_t upl_size;
2195 vm_size_t upl_needed_size;
2196 mach_msg_type_number_t pages_in_pl;
2197 int upl_flags;
2198 kern_return_t kret;
2199 struct clios iostate;
2200 int error = 0;
2201 int cur_upl = 0;
2202 int num_upl = 0;
2203 int n;
2204 user_addr_t iov_base;
2205 u_int32_t devblocksize;
2206 u_int32_t mem_alignment_mask;
2207
2208 /*
2209 * When we enter this routine, we know
2210 * -- the io_req_size will not exceed iov_len
2211 * -- the target address is physically contiguous
2212 */
2213 cluster_syncup(vp, newEOF, callback, callback_arg);
2214
2215 devblocksize = (u_int32_t)vp->v_mount->mnt_devblocksize;
2216 mem_alignment_mask = (u_int32_t)vp->v_mount->mnt_alignmentmask;
2217
2218 iostate.io_completed = 0;
2219 iostate.io_issued = 0;
2220 iostate.io_error = 0;
2221 iostate.io_wanted = 0;
2222
2223 next_cwrite:
2224 io_size = *write_length;
2225
2226 iov_base = uio_curriovbase(uio);
2227
2228 upl_offset = (vm_offset_t)((u_int32_t)iov_base & PAGE_MASK);
2229 upl_needed_size = upl_offset + io_size;
2230
2231 pages_in_pl = 0;
2232 upl_size = upl_needed_size;
2233 upl_flags = UPL_FILE_IO | UPL_COPYOUT_FROM | UPL_NO_SYNC |
2234 UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE;
2235
2236 kret = vm_map_get_upl(current_map(),
2237 (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)),
2238 &upl_size, &upl[cur_upl], NULL, &pages_in_pl, &upl_flags, 0);
2239
2240 if (kret != KERN_SUCCESS) {
2241 /*
2242 * failed to get pagelist
2243 */
2244 error = EINVAL;
2245 goto wait_for_cwrites;
2246 }
2247 num_upl++;
2248
2249 /*
2250 * Consider the possibility that upl_size wasn't satisfied.
2251 */
2252 if (upl_size < upl_needed_size) {
2253 /*
2254 * This is a failure in the physical memory case.
2255 */
2256 error = EINVAL;
2257 goto wait_for_cwrites;
2258 }
2259 pl = ubc_upl_pageinfo(upl[cur_upl]);
2260
2261 src_paddr = ((addr64_t)upl_phys_page(pl, 0) << 12) + (addr64_t)upl_offset;
2262
2263 while (((uio->uio_offset & (devblocksize - 1)) || io_size < devblocksize) && io_size) {
2264 u_int32_t head_size;
2265
2266 head_size = devblocksize - (u_int32_t)(uio->uio_offset & (devblocksize - 1));
2267
2268 if (head_size > io_size)
2269 head_size = io_size;
2270
2271 error = cluster_align_phys_io(vp, uio, src_paddr, head_size, 0, callback, callback_arg);
2272
2273 if (error)
2274 goto wait_for_cwrites;
2275
2276 upl_offset += head_size;
2277 src_paddr += head_size;
2278 io_size -= head_size;
2279
2280 iov_base += head_size;
2281 }
2282 if ((u_int32_t)iov_base & mem_alignment_mask) {
2283 /*
2284 * request doesn't set up on a memory boundary
2285 * the underlying DMA engine can handle...
2286 * return an error instead of going through
2287 * the slow copy path since the intent of this
2288 * path is direct I/O from device memory
2289 */
2290 error = EINVAL;
2291 goto wait_for_cwrites;
2292 }
2293
2294 tail_size = io_size & (devblocksize - 1);
2295 io_size -= tail_size;
2296
2297 while (io_size && error == 0) {
2298
2299 if (io_size > MAX_IO_CONTIG_SIZE)
2300 xsize = MAX_IO_CONTIG_SIZE;
2301 else
2302 xsize = io_size;
2303 /*
2304 * request asynchronously so that we can overlap
2305 * the preparation of the next I/O... we'll do
2306 * the commit after all the I/O has completed
2307 * since its all issued against the same UPL
2308 * if there are already too many outstanding writes
2309 * wait until some have completed before issuing the next
2310 */
2311 if (iostate.io_issued) {
2312 lck_mtx_lock(cl_mtxp);
2313
2314 while ((iostate.io_issued - iostate.io_completed) > (2 * MAX_IO_CONTIG_SIZE)) {
2315
2316 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_START,
2317 iostate.io_issued, iostate.io_completed, 2 * MAX_IO_CONTIG_SIZE, 0, 0);
2318
2319 iostate.io_wanted = 1;
2320 msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_write_contig", NULL);
2321
2322 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_END,
2323 iostate.io_issued, iostate.io_completed, 2 * MAX_IO_CONTIG_SIZE, 0, 0);
2324 }
2325 lck_mtx_unlock(cl_mtxp);
2326 }
2327 if (iostate.io_error) {
2328 /*
2329 * one of the earlier writes we issued ran into a hard error
2330 * don't issue any more writes...
2331 * go wait for all writes that are part of this stream
2332 * to complete before returning the error to the caller
2333 */
2334 goto wait_for_cwrites;
2335 }
2336 /*
2337 * issue an asynchronous write to cluster_io
2338 */
2339 error = cluster_io(vp, upl[cur_upl], upl_offset, uio->uio_offset,
2340 xsize, CL_DEV_MEMORY | CL_ASYNC | bflag, (buf_t)NULL, (struct clios *)&iostate, callback, callback_arg);
2341
2342 if (error == 0) {
2343 /*
2344 * The cluster_io write completed successfully,
2345 * update the uio structure
2346 */
2347 uio_update(uio, (user_size_t)xsize);
2348
2349 upl_offset += xsize;
2350 src_paddr += xsize;
2351 io_size -= xsize;
2352 }
2353 }
2354 if (error == 0 && iostate.io_error == 0 && tail_size == 0 && num_upl < MAX_VECTS) {
2355
2356 error = cluster_io_type(uio, write_type, write_length, 0);
2357
2358 if (error == 0 && *write_type == IO_CONTIG) {
2359 cur_upl++;
2360 goto next_cwrite;
2361 }
2362 } else
2363 *write_type = IO_UNKNOWN;
2364
2365 wait_for_cwrites:
2366 /*
2367 * make sure all async writes that are part of this stream
2368 * have completed before we proceed
2369 */
2370 lck_mtx_lock(cl_mtxp);
2371
2372 while (iostate.io_issued != iostate.io_completed) {
2373 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_START,
2374 iostate.io_issued, iostate.io_completed, 0, 0, 0);
2375
2376 iostate.io_wanted = 1;
2377 msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_write_contig", NULL);
2378
2379 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_END,
2380 iostate.io_issued, iostate.io_completed, 0, 0, 0);
2381 }
2382 lck_mtx_unlock(cl_mtxp);
2383
2384 if (iostate.io_error)
2385 error = iostate.io_error;
2386
2387 if (error == 0 && tail_size)
2388 error = cluster_align_phys_io(vp, uio, src_paddr, tail_size, 0, callback, callback_arg);
2389
2390 for (n = 0; n < num_upl; n++)
2391 /*
2392 * just release our hold on each physically contiguous
2393 * region without changing any state
2394 */
2395 ubc_upl_abort(upl[n], 0);
2396
2397 return (error);
2398 }
2399
2400
2401 static int
2402 cluster_write_copy(vnode_t vp, struct uio *uio, u_int32_t io_req_size, off_t oldEOF, off_t newEOF, off_t headOff,
2403 off_t tailOff, int flags, int (*callback)(buf_t, void *), void *callback_arg)
2404 {
2405 upl_page_info_t *pl;
2406 upl_t upl;
2407 vm_offset_t upl_offset = 0;
2408 vm_size_t upl_size;
2409 off_t upl_f_offset;
2410 int pages_in_upl;
2411 int start_offset;
2412 int xfer_resid;
2413 int io_size;
2414 int io_offset;
2415 int bytes_to_zero;
2416 int bytes_to_move;
2417 kern_return_t kret;
2418 int retval = 0;
2419 int io_resid;
2420 long long total_size;
2421 long long zero_cnt;
2422 off_t zero_off;
2423 long long zero_cnt1;
2424 off_t zero_off1;
2425 struct cl_extent cl;
2426 struct cl_writebehind *wbp;
2427 int bflag;
2428 u_int max_cluster_pgcount;
2429 u_int max_io_size;
2430
2431 if (flags & IO_PASSIVE)
2432 bflag = CL_PASSIVE;
2433 else
2434 bflag = 0;
2435
2436 if (uio) {
2437 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 40)) | DBG_FUNC_START,
2438 (int)uio->uio_offset, io_req_size, (int)oldEOF, (int)newEOF, 0);
2439
2440 io_resid = io_req_size;
2441 } else {
2442 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 40)) | DBG_FUNC_START,
2443 0, 0, (int)oldEOF, (int)newEOF, 0);
2444
2445 io_resid = 0;
2446 }
2447 zero_cnt = 0;
2448 zero_cnt1 = 0;
2449 zero_off = 0;
2450 zero_off1 = 0;
2451
2452 max_cluster_pgcount = MAX_CLUSTER_SIZE(vp) / PAGE_SIZE;
2453 max_io_size = cluster_max_io_size(vp->v_mount, CL_WRITE);
2454
2455 if (flags & IO_HEADZEROFILL) {
2456 /*
2457 * some filesystems (HFS is one) don't support unallocated holes within a file...
2458 * so we zero fill the intervening space between the old EOF and the offset
2459 * where the next chunk of real data begins.... ftruncate will also use this
2460 * routine to zero fill to the new EOF when growing a file... in this case, the
2461 * uio structure will not be provided
2462 */
2463 if (uio) {
2464 if (headOff < uio->uio_offset) {
2465 zero_cnt = uio->uio_offset - headOff;
2466 zero_off = headOff;
2467 }
2468 } else if (headOff < newEOF) {
2469 zero_cnt = newEOF - headOff;
2470 zero_off = headOff;
2471 }
2472 }
2473 if (flags & IO_TAILZEROFILL) {
2474 if (uio) {
2475 zero_off1 = uio->uio_offset + io_req_size;
2476
2477 if (zero_off1 < tailOff)
2478 zero_cnt1 = tailOff - zero_off1;
2479 }
2480 }
2481 if (zero_cnt == 0 && uio == (struct uio *) 0) {
2482 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 40)) | DBG_FUNC_END,
2483 retval, 0, 0, 0, 0);
2484 return (0);
2485 }
2486
2487 while ((total_size = (io_resid + zero_cnt + zero_cnt1)) && retval == 0) {
2488 /*
2489 * for this iteration of the loop, figure out where our starting point is
2490 */
2491 if (zero_cnt) {
2492 start_offset = (int)(zero_off & PAGE_MASK_64);
2493 upl_f_offset = zero_off - start_offset;
2494 } else if (io_resid) {
2495 start_offset = (int)(uio->uio_offset & PAGE_MASK_64);
2496 upl_f_offset = uio->uio_offset - start_offset;
2497 } else {
2498 start_offset = (int)(zero_off1 & PAGE_MASK_64);
2499 upl_f_offset = zero_off1 - start_offset;
2500 }
2501 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 46)) | DBG_FUNC_NONE,
2502 (int)zero_off, (int)zero_cnt, (int)zero_off1, (int)zero_cnt1, 0);
2503
2504 if (total_size > max_io_size)
2505 total_size = max_io_size;
2506
2507 cl.b_addr = (daddr64_t)(upl_f_offset / PAGE_SIZE_64);
2508
2509 if (uio && ((flags & (IO_SYNC | IO_HEADZEROFILL | IO_TAILZEROFILL)) == 0)) {
2510 /*
2511 * assumption... total_size <= io_resid
2512 * because IO_HEADZEROFILL and IO_TAILZEROFILL not set
2513 */
2514 if ((start_offset + total_size) > max_io_size)
2515 total_size -= start_offset;
2516 xfer_resid = total_size;
2517
2518 retval = cluster_copy_ubc_data_internal(vp, uio, &xfer_resid, 1, 1);
2519
2520 if (retval)
2521 break;
2522
2523 io_resid -= (total_size - xfer_resid);
2524 total_size = xfer_resid;
2525 start_offset = (int)(uio->uio_offset & PAGE_MASK_64);
2526 upl_f_offset = uio->uio_offset - start_offset;
2527
2528 if (total_size == 0) {
2529 if (start_offset) {
2530 /*
2531 * the write did not finish on a page boundary
2532 * which will leave upl_f_offset pointing to the
2533 * beginning of the last page written instead of
2534 * the page beyond it... bump it in this case
2535 * so that the cluster code records the last page
2536 * written as dirty
2537 */
2538 upl_f_offset += PAGE_SIZE_64;
2539 }
2540 upl_size = 0;
2541
2542 goto check_cluster;
2543 }
2544 }
2545 /*
2546 * compute the size of the upl needed to encompass
2547 * the requested write... limit each call to cluster_io
2548 * to the maximum UPL size... cluster_io will clip if
2549 * this exceeds the maximum io_size for the device,
2550 * make sure to account for
2551 * a starting offset that's not page aligned
2552 */
2553 upl_size = (start_offset + total_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
2554
2555 if (upl_size > max_io_size)
2556 upl_size = max_io_size;
2557
2558 pages_in_upl = upl_size / PAGE_SIZE;
2559 io_size = upl_size - start_offset;
2560
2561 if ((long long)io_size > total_size)
2562 io_size = total_size;
2563
2564 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 41)) | DBG_FUNC_START, upl_size, io_size, total_size, 0, 0);
2565
2566
2567 /*
2568 * Gather the pages from the buffer cache.
2569 * The UPL_WILL_MODIFY flag lets the UPL subsystem know
2570 * that we intend to modify these pages.
2571 */
2572 kret = ubc_create_upl(vp,
2573 upl_f_offset,
2574 upl_size,
2575 &upl,
2576 &pl,
2577 UPL_SET_LITE | UPL_WILL_MODIFY);
2578 if (kret != KERN_SUCCESS)
2579 panic("cluster_write_copy: failed to get pagelist");
2580
2581 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 41)) | DBG_FUNC_END,
2582 (int)upl, (int)upl_f_offset, start_offset, 0, 0);
2583
2584 if (start_offset && !upl_valid_page(pl, 0)) {
2585 int read_size;
2586
2587 /*
2588 * we're starting in the middle of the first page of the upl
2589 * and the page isn't currently valid, so we're going to have
2590 * to read it in first... this is a synchronous operation
2591 */
2592 read_size = PAGE_SIZE;
2593
2594 if ((upl_f_offset + read_size) > newEOF)
2595 read_size = newEOF - upl_f_offset;
2596
2597 retval = cluster_io(vp, upl, 0, upl_f_offset, read_size,
2598 CL_READ | bflag, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
2599 if (retval) {
2600 /*
2601 * we had an error during the read which causes us to abort
2602 * the current cluster_write request... before we do, we need
2603 * to release the rest of the pages in the upl without modifying
2604 * there state and mark the failed page in error
2605 */
2606 ubc_upl_abort_range(upl, 0, PAGE_SIZE, UPL_ABORT_DUMP_PAGES|UPL_ABORT_FREE_ON_EMPTY);
2607
2608 if (upl_size > PAGE_SIZE)
2609 ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_FREE_ON_EMPTY);
2610
2611 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 45)) | DBG_FUNC_NONE,
2612 (int)upl, 0, 0, retval, 0);
2613 break;
2614 }
2615 }
2616 if ((start_offset == 0 || upl_size > PAGE_SIZE) && ((start_offset + io_size) & PAGE_MASK)) {
2617 /*
2618 * the last offset we're writing to in this upl does not end on a page
2619 * boundary... if it's not beyond the old EOF, then we'll also need to
2620 * pre-read this page in if it isn't already valid
2621 */
2622 upl_offset = upl_size - PAGE_SIZE;
2623
2624 if ((upl_f_offset + start_offset + io_size) < oldEOF &&
2625 !upl_valid_page(pl, upl_offset / PAGE_SIZE)) {
2626 int read_size;
2627
2628 read_size = PAGE_SIZE;
2629
2630 if ((upl_f_offset + upl_offset + read_size) > newEOF)
2631 read_size = newEOF - (upl_f_offset + upl_offset);
2632
2633 retval = cluster_io(vp, upl, upl_offset, upl_f_offset + upl_offset, read_size,
2634 CL_READ | bflag, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
2635 if (retval) {
2636 /*
2637 * we had an error during the read which causes us to abort
2638 * the current cluster_write request... before we do, we
2639 * need to release the rest of the pages in the upl without
2640 * modifying there state and mark the failed page in error
2641 */
2642 ubc_upl_abort_range(upl, upl_offset, PAGE_SIZE, UPL_ABORT_DUMP_PAGES|UPL_ABORT_FREE_ON_EMPTY);
2643
2644 if (upl_size > PAGE_SIZE)
2645 ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_FREE_ON_EMPTY);
2646
2647 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 45)) | DBG_FUNC_NONE,
2648 (int)upl, 0, 0, retval, 0);
2649 break;
2650 }
2651 }
2652 }
2653 xfer_resid = io_size;
2654 io_offset = start_offset;
2655
2656 while (zero_cnt && xfer_resid) {
2657
2658 if (zero_cnt < (long long)xfer_resid)
2659 bytes_to_zero = zero_cnt;
2660 else
2661 bytes_to_zero = xfer_resid;
2662
2663 if ( !(flags & (IO_NOZEROVALID | IO_NOZERODIRTY))) {
2664 cluster_zero(upl, io_offset, bytes_to_zero, NULL);
2665 } else {
2666 int zero_pg_index;
2667
2668 bytes_to_zero = min(bytes_to_zero, PAGE_SIZE - (int)(zero_off & PAGE_MASK_64));
2669 zero_pg_index = (int)((zero_off - upl_f_offset) / PAGE_SIZE_64);
2670
2671 if ( !upl_valid_page(pl, zero_pg_index)) {
2672 cluster_zero(upl, io_offset, bytes_to_zero, NULL);
2673
2674 } else if ((flags & (IO_NOZERODIRTY | IO_NOZEROVALID)) == IO_NOZERODIRTY &&
2675 !upl_dirty_page(pl, zero_pg_index)) {
2676 cluster_zero(upl, io_offset, bytes_to_zero, NULL);
2677 }
2678 }
2679 xfer_resid -= bytes_to_zero;
2680 zero_cnt -= bytes_to_zero;
2681 zero_off += bytes_to_zero;
2682 io_offset += bytes_to_zero;
2683 }
2684 if (xfer_resid && io_resid) {
2685 u_int32_t io_requested;
2686
2687 bytes_to_move = min(io_resid, xfer_resid);
2688 io_requested = bytes_to_move;
2689
2690 retval = cluster_copy_upl_data(uio, upl, io_offset, (int *)&io_requested);
2691
2692 if (retval) {
2693
2694 ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_DUMP_PAGES | UPL_ABORT_FREE_ON_EMPTY);
2695
2696 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 45)) | DBG_FUNC_NONE,
2697 (int)upl, 0, 0, retval, 0);
2698 } else {
2699 io_resid -= bytes_to_move;
2700 xfer_resid -= bytes_to_move;
2701 io_offset += bytes_to_move;
2702 }
2703 }
2704 while (xfer_resid && zero_cnt1 && retval == 0) {
2705
2706 if (zero_cnt1 < (long long)xfer_resid)
2707 bytes_to_zero = zero_cnt1;
2708 else
2709 bytes_to_zero = xfer_resid;
2710
2711 if ( !(flags & (IO_NOZEROVALID | IO_NOZERODIRTY))) {
2712 cluster_zero(upl, io_offset, bytes_to_zero, NULL);
2713 } else {
2714 int zero_pg_index;
2715
2716 bytes_to_zero = min(bytes_to_zero, PAGE_SIZE - (int)(zero_off1 & PAGE_MASK_64));
2717 zero_pg_index = (int)((zero_off1 - upl_f_offset) / PAGE_SIZE_64);
2718
2719 if ( !upl_valid_page(pl, zero_pg_index)) {
2720 cluster_zero(upl, io_offset, bytes_to_zero, NULL);
2721 } else if ((flags & (IO_NOZERODIRTY | IO_NOZEROVALID)) == IO_NOZERODIRTY &&
2722 !upl_dirty_page(pl, zero_pg_index)) {
2723 cluster_zero(upl, io_offset, bytes_to_zero, NULL);
2724 }
2725 }
2726 xfer_resid -= bytes_to_zero;
2727 zero_cnt1 -= bytes_to_zero;
2728 zero_off1 += bytes_to_zero;
2729 io_offset += bytes_to_zero;
2730 }
2731
2732 if (retval == 0) {
2733 int cl_index;
2734 int ret_cluster_try_push;
2735
2736 io_size += start_offset;
2737
2738 if ((upl_f_offset + io_size) >= newEOF && (u_int)io_size < upl_size) {
2739 /*
2740 * if we're extending the file with this write
2741 * we'll zero fill the rest of the page so that
2742 * if the file gets extended again in such a way as to leave a
2743 * hole starting at this EOF, we'll have zero's in the correct spot
2744 */
2745 cluster_zero(upl, io_size, upl_size - io_size, NULL);
2746 }
2747 /*
2748 * release the upl now if we hold one since...
2749 * 1) pages in it may be present in the sparse cluster map
2750 * and may span 2 separate buckets there... if they do and
2751 * we happen to have to flush a bucket to make room and it intersects
2752 * this upl, a deadlock may result on page BUSY
2753 * 2) we're delaying the I/O... from this point forward we're just updating
2754 * the cluster state... no need to hold the pages, so commit them
2755 * 3) IO_SYNC is set...
2756 * because we had to ask for a UPL that provides currenty non-present pages, the
2757 * UPL has been automatically set to clear the dirty flags (both software and hardware)
2758 * upon committing it... this is not the behavior we want since it's possible for
2759 * pages currently present as part of a mapped file to be dirtied while the I/O is in flight.
2760 * we'll pick these pages back up later with the correct behavior specified.
2761 * 4) we don't want to hold pages busy in a UPL and then block on the cluster lock... if a flush
2762 * of this vnode is in progress, we will deadlock if the pages being flushed intersect the pages
2763 * we hold since the flushing context is holding the cluster lock.
2764 */
2765 ubc_upl_commit_range(upl, 0, upl_size,
2766 UPL_COMMIT_SET_DIRTY | UPL_COMMIT_INACTIVATE | UPL_COMMIT_FREE_ON_EMPTY);
2767 check_cluster:
2768 /*
2769 * calculate the last logical block number
2770 * that this delayed I/O encompassed
2771 */
2772 cl.e_addr = (daddr64_t)((upl_f_offset + (off_t)upl_size) / PAGE_SIZE_64);
2773
2774 if (flags & IO_SYNC)
2775 /*
2776 * if the IO_SYNC flag is set than we need to
2777 * bypass any clusters and immediately issue
2778 * the I/O
2779 */
2780 goto issue_io;
2781
2782 /*
2783 * take the lock to protect our accesses
2784 * of the writebehind and sparse cluster state
2785 */
2786 wbp = cluster_get_wbp(vp, CLW_ALLOCATE | CLW_RETURNLOCKED);
2787
2788 if (wbp->cl_scmap) {
2789
2790 if ( !(flags & IO_NOCACHE)) {
2791 /*
2792 * we've fallen into the sparse
2793 * cluster method of delaying dirty pages
2794 */
2795 sparse_cluster_add(wbp, vp, &cl, newEOF, callback, callback_arg);
2796
2797 lck_mtx_unlock(&wbp->cl_lockw);
2798
2799 continue;
2800 }
2801 /*
2802 * must have done cached writes that fell into
2803 * the sparse cluster mechanism... we've switched
2804 * to uncached writes on the file, so go ahead
2805 * and push whatever's in the sparse map
2806 * and switch back to normal clustering
2807 */
2808 wbp->cl_number = 0;
2809
2810 sparse_cluster_push(wbp, vp, newEOF, PUSH_ALL, callback, callback_arg);
2811 /*
2812 * no clusters of either type present at this point
2813 * so just go directly to start_new_cluster since
2814 * we know we need to delay this I/O since we've
2815 * already released the pages back into the cache
2816 * to avoid the deadlock with sparse_cluster_push
2817 */
2818 goto start_new_cluster;
2819 }
2820 if (wbp->cl_number == 0)
2821 /*
2822 * no clusters currently present
2823 */
2824 goto start_new_cluster;
2825
2826 for (cl_index = 0; cl_index < wbp->cl_number; cl_index++) {
2827 /*
2828 * check each cluster that we currently hold
2829 * try to merge some or all of this write into
2830 * one or more of the existing clusters... if
2831 * any portion of the write remains, start a
2832 * new cluster
2833 */
2834 if (cl.b_addr >= wbp->cl_clusters[cl_index].b_addr) {
2835 /*
2836 * the current write starts at or after the current cluster
2837 */
2838 if (cl.e_addr <= (wbp->cl_clusters[cl_index].b_addr + max_cluster_pgcount)) {
2839 /*
2840 * we have a write that fits entirely
2841 * within the existing cluster limits
2842 */
2843 if (cl.e_addr > wbp->cl_clusters[cl_index].e_addr)
2844 /*
2845 * update our idea of where the cluster ends
2846 */
2847 wbp->cl_clusters[cl_index].e_addr = cl.e_addr;
2848 break;
2849 }
2850 if (cl.b_addr < (wbp->cl_clusters[cl_index].b_addr + max_cluster_pgcount)) {
2851 /*
2852 * we have a write that starts in the middle of the current cluster
2853 * but extends beyond the cluster's limit... we know this because
2854 * of the previous checks
2855 * we'll extend the current cluster to the max
2856 * and update the b_addr for the current write to reflect that
2857 * the head of it was absorbed into this cluster...
2858 * note that we'll always have a leftover tail in this case since
2859 * full absorbtion would have occurred in the clause above
2860 */
2861 wbp->cl_clusters[cl_index].e_addr = wbp->cl_clusters[cl_index].b_addr + max_cluster_pgcount;
2862
2863 cl.b_addr = wbp->cl_clusters[cl_index].e_addr;
2864 }
2865 /*
2866 * we come here for the case where the current write starts
2867 * beyond the limit of the existing cluster or we have a leftover
2868 * tail after a partial absorbtion
2869 *
2870 * in either case, we'll check the remaining clusters before
2871 * starting a new one
2872 */
2873 } else {
2874 /*
2875 * the current write starts in front of the cluster we're currently considering
2876 */
2877 if ((wbp->cl_clusters[cl_index].e_addr - cl.b_addr) <= max_cluster_pgcount) {
2878 /*
2879 * we can just merge the new request into
2880 * this cluster and leave it in the cache
2881 * since the resulting cluster is still
2882 * less than the maximum allowable size
2883 */
2884 wbp->cl_clusters[cl_index].b_addr = cl.b_addr;
2885
2886 if (cl.e_addr > wbp->cl_clusters[cl_index].e_addr) {
2887 /*
2888 * the current write completely
2889 * envelops the existing cluster and since
2890 * each write is limited to at most max_cluster_pgcount pages
2891 * we can just use the start and last blocknos of the write
2892 * to generate the cluster limits
2893 */
2894 wbp->cl_clusters[cl_index].e_addr = cl.e_addr;
2895 }
2896 break;
2897 }
2898
2899 /*
2900 * if we were to combine this write with the current cluster
2901 * we would exceed the cluster size limit.... so,
2902 * let's see if there's any overlap of the new I/O with
2903 * the cluster we're currently considering... in fact, we'll
2904 * stretch the cluster out to it's full limit and see if we
2905 * get an intersection with the current write
2906 *
2907 */
2908 if (cl.e_addr > wbp->cl_clusters[cl_index].e_addr - max_cluster_pgcount) {
2909 /*
2910 * the current write extends into the proposed cluster
2911 * clip the length of the current write after first combining it's
2912 * tail with the newly shaped cluster
2913 */
2914 wbp->cl_clusters[cl_index].b_addr = wbp->cl_clusters[cl_index].e_addr - max_cluster_pgcount;
2915
2916 cl.e_addr = wbp->cl_clusters[cl_index].b_addr;
2917 }
2918 /*
2919 * if we get here, there was no way to merge
2920 * any portion of this write with this cluster
2921 * or we could only merge part of it which
2922 * will leave a tail...
2923 * we'll check the remaining clusters before starting a new one
2924 */
2925 }
2926 }
2927 if (cl_index < wbp->cl_number)
2928 /*
2929 * we found an existing cluster(s) that we
2930 * could entirely merge this I/O into
2931 */
2932 goto delay_io;
2933
2934 if (wbp->cl_number < MAX_CLUSTERS)
2935 /*
2936 * we didn't find an existing cluster to
2937 * merge into, but there's room to start
2938 * a new one
2939 */
2940 goto start_new_cluster;
2941
2942 /*
2943 * no exisitng cluster to merge with and no
2944 * room to start a new one... we'll try
2945 * pushing one of the existing ones... if none of
2946 * them are able to be pushed, we'll switch
2947 * to the sparse cluster mechanism
2948 * cluster_try_push updates cl_number to the
2949 * number of remaining clusters... and
2950 * returns the number of currently unused clusters
2951 */
2952 ret_cluster_try_push = 0;
2953
2954 /*
2955 * if writes are not deferred, call cluster push immediately
2956 */
2957 if (!((unsigned int)vfs_flags(vp->v_mount) & MNT_DEFWRITE)) {
2958
2959 ret_cluster_try_push = cluster_try_push(wbp, vp, newEOF, (flags & IO_NOCACHE) ? 0 : PUSH_DELAY, callback, callback_arg);
2960 }
2961
2962 /*
2963 * execute following regardless of writes being deferred or not
2964 */
2965 if (ret_cluster_try_push == 0) {
2966 /*
2967 * no more room in the normal cluster mechanism
2968 * so let's switch to the more expansive but expensive
2969 * sparse mechanism....
2970 */
2971 sparse_cluster_switch(wbp, vp, newEOF, callback, callback_arg);
2972 sparse_cluster_add(wbp, vp, &cl, newEOF, callback, callback_arg);
2973
2974 lck_mtx_unlock(&wbp->cl_lockw);
2975
2976 continue;
2977 }
2978 /*
2979 * we pushed one cluster successfully, so we must be sequentially writing this file
2980 * otherwise, we would have failed and fallen into the sparse cluster support
2981 * so let's take the opportunity to push out additional clusters...
2982 * this will give us better I/O locality if we're in a copy loop
2983 * (i.e. we won't jump back and forth between the read and write points
2984 */
2985 if (!((unsigned int)vfs_flags(vp->v_mount) & MNT_DEFWRITE)) {
2986 while (wbp->cl_number)
2987 cluster_try_push(wbp, vp, newEOF, 0, callback, callback_arg);
2988 }
2989
2990 start_new_cluster:
2991 wbp->cl_clusters[wbp->cl_number].b_addr = cl.b_addr;
2992 wbp->cl_clusters[wbp->cl_number].e_addr = cl.e_addr;
2993
2994 wbp->cl_clusters[wbp->cl_number].io_flags = 0;
2995
2996 if (flags & IO_NOCACHE)
2997 wbp->cl_clusters[wbp->cl_number].io_flags |= CLW_IONOCACHE;
2998
2999 if (bflag & CL_PASSIVE)
3000 wbp->cl_clusters[wbp->cl_number].io_flags |= CLW_IOPASSIVE;
3001
3002 wbp->cl_number++;
3003 delay_io:
3004 lck_mtx_unlock(&wbp->cl_lockw);
3005
3006 continue;
3007 issue_io:
3008 /*
3009 * we don't hold the lock at this point
3010 *
3011 * we've already dropped the current upl, so pick it back up with COPYOUT_FROM set
3012 * so that we correctly deal with a change in state of the hardware modify bit...
3013 * we do this via cluster_push_now... by passing along the IO_SYNC flag, we force
3014 * cluster_push_now to wait until all the I/Os have completed... cluster_push_now is also
3015 * responsible for generating the correct sized I/O(s)
3016 */
3017 retval = cluster_push_now(vp, &cl, newEOF, flags, callback, callback_arg);
3018 }
3019 }
3020 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 40)) | DBG_FUNC_END, retval, 0, io_resid, 0, 0);
3021
3022 return (retval);
3023 }
3024
3025
3026
3027 int
3028 cluster_read(vnode_t vp, struct uio *uio, off_t filesize, int xflags)
3029 {
3030 return cluster_read_ext(vp, uio, filesize, xflags, NULL, NULL);
3031 }
3032
3033
3034 int
3035 cluster_read_ext(vnode_t vp, struct uio *uio, off_t filesize, int xflags, int (*callback)(buf_t, void *), void *callback_arg)
3036 {
3037 int retval = 0;
3038 int flags;
3039 user_ssize_t cur_resid;
3040 u_int32_t io_size;
3041 u_int32_t read_length = 0;
3042 int read_type = IO_COPY;
3043
3044 flags = xflags;
3045
3046 if (vp->v_flag & VNOCACHE_DATA)
3047 flags |= IO_NOCACHE;
3048 if ((vp->v_flag & VRAOFF) || speculative_reads_disabled)
3049 flags |= IO_RAOFF;
3050
3051 /*
3052 * do a read through the cache if one of the following is true....
3053 * NOCACHE is not true
3054 * the uio request doesn't target USERSPACE
3055 * otherwise, find out if we want the direct or contig variant for
3056 * the first vector in the uio request
3057 */
3058 if ( (flags & IO_NOCACHE) && UIO_SEG_IS_USER_SPACE(uio->uio_segflg) )
3059 retval = cluster_io_type(uio, &read_type, &read_length, 0);
3060
3061 while ((cur_resid = uio_resid(uio)) && uio->uio_offset < filesize && retval == 0) {
3062
3063 switch (read_type) {
3064
3065 case IO_COPY:
3066 /*
3067 * make sure the uio_resid isn't too big...
3068 * internally, we want to handle all of the I/O in
3069 * chunk sizes that fit in a 32 bit int
3070 */
3071 if (cur_resid > (user_ssize_t)(MAX_IO_REQUEST_SIZE))
3072 io_size = MAX_IO_REQUEST_SIZE;
3073 else
3074 io_size = (u_int32_t)cur_resid;
3075
3076 retval = cluster_read_copy(vp, uio, io_size, filesize, flags, callback, callback_arg);
3077 break;
3078
3079 case IO_DIRECT:
3080 retval = cluster_read_direct(vp, uio, filesize, &read_type, &read_length, flags, callback, callback_arg);
3081 break;
3082
3083 case IO_CONTIG:
3084 retval = cluster_read_contig(vp, uio, filesize, &read_type, &read_length, callback, callback_arg, flags);
3085 break;
3086
3087 case IO_UNKNOWN:
3088 retval = cluster_io_type(uio, &read_type, &read_length, 0);
3089 break;
3090 }
3091 }
3092 return (retval);
3093 }
3094
3095
3096
3097 static void
3098 cluster_read_upl_release(upl_t upl, int start_pg, int last_pg, int flags)
3099 {
3100 int range;
3101 int abort_flags = UPL_ABORT_FREE_ON_EMPTY;
3102
3103 if ((range = last_pg - start_pg)) {
3104 if ( !(flags & IO_NOCACHE))
3105 abort_flags |= UPL_ABORT_REFERENCE;
3106
3107 ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, range * PAGE_SIZE, abort_flags);
3108 }
3109 }
3110
3111
3112 static int
3113 cluster_read_copy(vnode_t vp, struct uio *uio, u_int32_t io_req_size, off_t filesize, int flags, int (*callback)(buf_t, void *), void *callback_arg)
3114 {
3115 upl_page_info_t *pl;
3116 upl_t upl;
3117 vm_offset_t upl_offset;
3118 u_int32_t upl_size;
3119 off_t upl_f_offset;
3120 int start_offset;
3121 int start_pg;
3122 int last_pg;
3123 int uio_last = 0;
3124 int pages_in_upl;
3125 off_t max_size;
3126 off_t last_ioread_offset;
3127 off_t last_request_offset;
3128 kern_return_t kret;
3129 int error = 0;
3130 int retval = 0;
3131 u_int32_t size_of_prefetch;
3132 u_int32_t xsize;
3133 u_int32_t io_size;
3134 u_int32_t max_rd_size;
3135 u_int32_t max_io_size;
3136 u_int32_t max_prefetch;
3137 u_int rd_ahead_enabled = 1;
3138 u_int prefetch_enabled = 1;
3139 struct cl_readahead * rap;
3140 struct clios iostate;
3141 struct cl_extent extent;
3142 int bflag;
3143 int take_reference = 1;
3144 struct uthread *ut;
3145 int policy = IOPOL_DEFAULT;
3146
3147 policy = current_proc()->p_iopol_disk;
3148
3149 ut = get_bsdthread_info(current_thread());
3150
3151 if (ut->uu_iopol_disk != IOPOL_DEFAULT)
3152 policy = ut->uu_iopol_disk;
3153
3154 if (policy == IOPOL_THROTTLE)
3155 take_reference = 0;
3156
3157 if (flags & IO_PASSIVE)
3158 bflag = CL_PASSIVE;
3159 else
3160 bflag = 0;
3161
3162 max_prefetch = MAX_PREFETCH(vp);
3163 max_rd_size = max_prefetch;
3164 max_io_size = cluster_max_io_size(vp->v_mount, CL_READ);
3165
3166 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 32)) | DBG_FUNC_START,
3167 (int)uio->uio_offset, io_req_size, (int)filesize, flags, 0);
3168
3169 last_request_offset = uio->uio_offset + io_req_size;
3170
3171 if ((flags & (IO_RAOFF|IO_NOCACHE)) || ((last_request_offset & ~PAGE_MASK_64) == (uio->uio_offset & ~PAGE_MASK_64))) {
3172 rd_ahead_enabled = 0;
3173 rap = NULL;
3174 } else {
3175 if (cluster_hard_throttle_on(vp)) {
3176 rd_ahead_enabled = 0;
3177 prefetch_enabled = 0;
3178
3179 max_rd_size = HARD_THROTTLE_MAXSIZE;
3180 }
3181 if ((rap = cluster_get_rap(vp)) == NULL)
3182 rd_ahead_enabled = 0;
3183 }
3184 if (last_request_offset > filesize)
3185 last_request_offset = filesize;
3186 extent.b_addr = uio->uio_offset / PAGE_SIZE_64;
3187 extent.e_addr = (last_request_offset - 1) / PAGE_SIZE_64;
3188
3189 if (rap != NULL && rap->cl_ralen && (rap->cl_lastr == extent.b_addr || (rap->cl_lastr + 1) == extent.b_addr)) {
3190 /*
3191 * determine if we already have a read-ahead in the pipe courtesy of the
3192 * last read systemcall that was issued...
3193 * if so, pick up it's extent to determine where we should start
3194 * with respect to any read-ahead that might be necessary to
3195 * garner all the data needed to complete this read systemcall
3196 */
3197 last_ioread_offset = (rap->cl_maxra * PAGE_SIZE_64) + PAGE_SIZE_64;
3198
3199 if (last_ioread_offset < uio->uio_offset)
3200 last_ioread_offset = (off_t)0;
3201 else if (last_ioread_offset > last_request_offset)
3202 last_ioread_offset = last_request_offset;
3203 } else
3204 last_ioread_offset = (off_t)0;
3205
3206 while (io_req_size && uio->uio_offset < filesize && retval == 0) {
3207 /*
3208 * compute the size of the upl needed to encompass
3209 * the requested read... limit each call to cluster_io
3210 * to the maximum UPL size... cluster_io will clip if
3211 * this exceeds the maximum io_size for the device,
3212 * make sure to account for
3213 * a starting offset that's not page aligned
3214 */
3215 start_offset = (int)(uio->uio_offset & PAGE_MASK_64);
3216 upl_f_offset = uio->uio_offset - (off_t)start_offset;
3217 max_size = filesize - uio->uio_offset;
3218
3219 if ((off_t)(io_req_size) < max_size)
3220 io_size = io_req_size;
3221 else
3222 io_size = max_size;
3223
3224 if (!(flags & IO_NOCACHE)) {
3225
3226 while (io_size) {
3227 u_int32_t io_resid;
3228 u_int32_t io_requested;
3229
3230 /*
3231 * if we keep finding the pages we need already in the cache, then
3232 * don't bother to call cluster_read_prefetch since it costs CPU cycles
3233 * to determine that we have all the pages we need... once we miss in
3234 * the cache and have issued an I/O, than we'll assume that we're likely
3235 * to continue to miss in the cache and it's to our advantage to try and prefetch
3236 */
3237 if (last_request_offset && last_ioread_offset && (size_of_prefetch = (last_request_offset - last_ioread_offset))) {
3238 if ((last_ioread_offset - uio->uio_offset) <= max_rd_size && prefetch_enabled) {
3239 /*
3240 * we've already issued I/O for this request and
3241 * there's still work to do and
3242 * our prefetch stream is running dry, so issue a
3243 * pre-fetch I/O... the I/O latency will overlap
3244 * with the copying of the data
3245 */
3246 if (size_of_prefetch > max_rd_size)
3247 size_of_prefetch = max_rd_size;
3248
3249 size_of_prefetch = cluster_read_prefetch(vp, last_ioread_offset, size_of_prefetch, filesize, callback, callback_arg, bflag);
3250
3251 last_ioread_offset += (off_t)(size_of_prefetch * PAGE_SIZE);
3252
3253 if (last_ioread_offset > last_request_offset)
3254 last_ioread_offset = last_request_offset;
3255 }
3256 }
3257 /*
3258 * limit the size of the copy we're about to do so that
3259 * we can notice that our I/O pipe is running dry and
3260 * get the next I/O issued before it does go dry
3261 */
3262 if (last_ioread_offset && io_size > (max_io_size / 4))
3263 io_resid = (max_io_size / 4);
3264 else
3265 io_resid = io_size;
3266
3267 io_requested = io_resid;
3268
3269 retval = cluster_copy_ubc_data_internal(vp, uio, (int *)&io_resid, 0, take_reference);
3270
3271 xsize = io_requested - io_resid;
3272
3273 io_size -= xsize;
3274 io_req_size -= xsize;
3275
3276 if (retval || io_resid)
3277 /*
3278 * if we run into a real error or
3279 * a page that is not in the cache
3280 * we need to leave streaming mode
3281 */
3282 break;
3283
3284 if ((io_size == 0 || last_ioread_offset == last_request_offset) && rd_ahead_enabled) {
3285 /*
3286 * we're already finished the I/O for this read request
3287 * let's see if we should do a read-ahead
3288 */
3289 cluster_read_ahead(vp, &extent, filesize, rap, callback, callback_arg, bflag);
3290 }
3291 }
3292 if (retval)
3293 break;
3294 if (io_size == 0) {
3295 if (rap != NULL) {
3296 if (extent.e_addr < rap->cl_lastr)
3297 rap->cl_maxra = 0;
3298 rap->cl_lastr = extent.e_addr;
3299 }
3300 break;
3301 }
3302 start_offset = (int)(uio->uio_offset & PAGE_MASK_64);
3303 upl_f_offset = uio->uio_offset - (off_t)start_offset;
3304 max_size = filesize - uio->uio_offset;
3305 }
3306 if (io_size > max_rd_size)
3307 io_size = max_rd_size;
3308
3309 upl_size = (start_offset + io_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
3310
3311 if (flags & IO_NOCACHE) {
3312 if (upl_size > max_io_size)
3313 upl_size = max_io_size;
3314 } else {
3315 if (upl_size > max_io_size / 4)
3316 upl_size = max_io_size / 4;
3317 }
3318 pages_in_upl = upl_size / PAGE_SIZE;
3319
3320 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 33)) | DBG_FUNC_START,
3321 (int)upl, (int)upl_f_offset, upl_size, start_offset, 0);
3322
3323 kret = ubc_create_upl(vp,
3324 upl_f_offset,
3325 upl_size,
3326 &upl,
3327 &pl,
3328 UPL_FILE_IO | UPL_SET_LITE);
3329 if (kret != KERN_SUCCESS)
3330 panic("cluster_read_copy: failed to get pagelist");
3331
3332 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 33)) | DBG_FUNC_END,
3333 (int)upl, (int)upl_f_offset, upl_size, start_offset, 0);
3334
3335 /*
3336 * scan from the beginning of the upl looking for the first
3337 * non-valid page.... this will become the first page in
3338 * the request we're going to make to 'cluster_io'... if all
3339 * of the pages are valid, we won't call through to 'cluster_io'
3340 */
3341 for (start_pg = 0; start_pg < pages_in_upl; start_pg++) {
3342 if (!upl_valid_page(pl, start_pg))
3343 break;
3344 }
3345
3346 /*
3347 * scan from the starting invalid page looking for a valid
3348 * page before the end of the upl is reached, if we
3349 * find one, then it will be the last page of the request to
3350 * 'cluster_io'
3351 */
3352 for (last_pg = start_pg; last_pg < pages_in_upl; last_pg++) {
3353 if (upl_valid_page(pl, last_pg))
3354 break;
3355 }
3356 iostate.io_completed = 0;
3357 iostate.io_issued = 0;
3358 iostate.io_error = 0;
3359 iostate.io_wanted = 0;
3360
3361 if (start_pg < last_pg) {
3362 /*
3363 * we found a range of 'invalid' pages that must be filled
3364 * if the last page in this range is the last page of the file
3365 * we may have to clip the size of it to keep from reading past
3366 * the end of the last physical block associated with the file
3367 */
3368 upl_offset = start_pg * PAGE_SIZE;
3369 io_size = (last_pg - start_pg) * PAGE_SIZE;
3370
3371 if ((upl_f_offset + upl_offset + io_size) > filesize)
3372 io_size = filesize - (upl_f_offset + upl_offset);
3373
3374 /*
3375 * issue an asynchronous read to cluster_io
3376 */
3377
3378 error = cluster_io(vp, upl, upl_offset, upl_f_offset + upl_offset,
3379 io_size, CL_READ | CL_ASYNC | bflag, (buf_t)NULL, &iostate, callback, callback_arg);
3380 }
3381 if (error == 0) {
3382 /*
3383 * if the read completed successfully, or there was no I/O request
3384 * issued, than copy the data into user land via 'cluster_upl_copy_data'
3385 * we'll first add on any 'valid'
3386 * pages that were present in the upl when we acquired it.
3387 */
3388 u_int val_size;
3389
3390 for (uio_last = last_pg; uio_last < pages_in_upl; uio_last++) {
3391 if (!upl_valid_page(pl, uio_last))
3392 break;
3393 }
3394 if (uio_last < pages_in_upl) {
3395 /*
3396 * there were some invalid pages beyond the valid pages
3397 * that we didn't issue an I/O for, just release them
3398 * unchanged now, so that any prefetch/readahed can
3399 * include them
3400 */
3401 ubc_upl_abort_range(upl, uio_last * PAGE_SIZE,
3402 (pages_in_upl - uio_last) * PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
3403 }
3404
3405 /*
3406 * compute size to transfer this round, if io_req_size is
3407 * still non-zero after this attempt, we'll loop around and
3408 * set up for another I/O.
3409 */
3410 val_size = (uio_last * PAGE_SIZE) - start_offset;
3411
3412 if (val_size > max_size)
3413 val_size = max_size;
3414
3415 if (val_size > io_req_size)
3416 val_size = io_req_size;
3417
3418 if ((uio->uio_offset + val_size) > last_ioread_offset)
3419 last_ioread_offset = uio->uio_offset + val_size;
3420
3421 if ((size_of_prefetch = (last_request_offset - last_ioread_offset)) && prefetch_enabled) {
3422
3423 if ((last_ioread_offset - (uio->uio_offset + val_size)) <= upl_size) {
3424 /*
3425 * if there's still I/O left to do for this request, and...
3426 * we're not in hard throttle mode, and...
3427 * we're close to using up the previous prefetch, then issue a
3428 * new pre-fetch I/O... the I/O latency will overlap
3429 * with the copying of the data
3430 */
3431 if (size_of_prefetch > max_rd_size)
3432 size_of_prefetch = max_rd_size;
3433
3434 size_of_prefetch = cluster_read_prefetch(vp, last_ioread_offset, size_of_prefetch, filesize, callback, callback_arg, bflag);
3435
3436 last_ioread_offset += (off_t)(size_of_prefetch * PAGE_SIZE);
3437
3438 if (last_ioread_offset > last_request_offset)
3439 last_ioread_offset = last_request_offset;
3440 }
3441
3442 } else if ((uio->uio_offset + val_size) == last_request_offset) {
3443 /*
3444 * this transfer will finish this request, so...
3445 * let's try to read ahead if we're in
3446 * a sequential access pattern and we haven't
3447 * explicitly disabled it
3448 */
3449 if (rd_ahead_enabled)
3450 cluster_read_ahead(vp, &extent, filesize, rap, callback, callback_arg, bflag);
3451
3452 if (rap != NULL) {
3453 if (extent.e_addr < rap->cl_lastr)
3454 rap->cl_maxra = 0;
3455 rap->cl_lastr = extent.e_addr;
3456 }
3457 }
3458 lck_mtx_lock(cl_mtxp);
3459
3460 while (iostate.io_issued != iostate.io_completed) {
3461 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_START,
3462 iostate.io_issued, iostate.io_completed, 0, 0, 0);
3463
3464 iostate.io_wanted = 1;
3465 msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_read_copy", NULL);
3466
3467 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_END,
3468 iostate.io_issued, iostate.io_completed, 0, 0, 0);
3469 }
3470 lck_mtx_unlock(cl_mtxp);
3471
3472 if (iostate.io_error)
3473 error = iostate.io_error;
3474 else {
3475 u_int32_t io_requested;
3476
3477 io_requested = val_size;
3478
3479 retval = cluster_copy_upl_data(uio, upl, start_offset, (int *)&io_requested);
3480
3481 io_req_size -= (val_size - io_requested);
3482 }
3483 }
3484 if (start_pg < last_pg) {
3485 /*
3486 * compute the range of pages that we actually issued an I/O for
3487 * and either commit them as valid if the I/O succeeded
3488 * or abort them if the I/O failed or we're not supposed to
3489 * keep them in the cache
3490 */
3491 io_size = (last_pg - start_pg) * PAGE_SIZE;
3492
3493 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 35)) | DBG_FUNC_START, (int)upl, start_pg * PAGE_SIZE, io_size, error, 0);
3494
3495 if (error || (flags & IO_NOCACHE))
3496 ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, io_size,
3497 UPL_ABORT_DUMP_PAGES | UPL_ABORT_FREE_ON_EMPTY);
3498 else
3499 ubc_upl_commit_range(upl, start_pg * PAGE_SIZE, io_size,
3500 UPL_COMMIT_CLEAR_DIRTY | UPL_COMMIT_FREE_ON_EMPTY | UPL_COMMIT_INACTIVATE);
3501
3502 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 35)) | DBG_FUNC_END, (int)upl, start_pg * PAGE_SIZE, io_size, error, 0);
3503 }
3504 if ((last_pg - start_pg) < pages_in_upl) {
3505 /*
3506 * the set of pages that we issued an I/O for did not encompass
3507 * the entire upl... so just release these without modifying
3508 * their state
3509 */
3510 if (error)
3511 ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_FREE_ON_EMPTY);
3512 else {
3513
3514 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 35)) | DBG_FUNC_START,
3515 (int)upl, -1, pages_in_upl - (last_pg - start_pg), 0, 0);
3516
3517 /*
3518 * handle any valid pages at the beginning of
3519 * the upl... release these appropriately
3520 */
3521 cluster_read_upl_release(upl, 0, start_pg, flags);
3522
3523 /*
3524 * handle any valid pages immediately after the
3525 * pages we issued I/O for... ... release these appropriately
3526 */
3527 cluster_read_upl_release(upl, last_pg, uio_last, flags);
3528
3529 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 35)) | DBG_FUNC_END, (int)upl, -1, -1, 0, 0);
3530 }
3531 }
3532 if (retval == 0)
3533 retval = error;
3534
3535 if (io_req_size) {
3536 if (cluster_hard_throttle_on(vp)) {
3537 rd_ahead_enabled = 0;
3538 prefetch_enabled = 0;
3539
3540 max_rd_size = HARD_THROTTLE_MAXSIZE;
3541 } else {
3542 if (max_rd_size == HARD_THROTTLE_MAXSIZE) {
3543 /*
3544 * coming out of throttled state
3545 */
3546 if (rap != NULL)
3547 rd_ahead_enabled = 1;
3548 prefetch_enabled = 1;
3549
3550 max_rd_size = max_prefetch;
3551 last_ioread_offset = 0;
3552 }
3553 }
3554 }
3555 }
3556 if (rap != NULL) {
3557 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 32)) | DBG_FUNC_END,
3558 (int)uio->uio_offset, io_req_size, rap->cl_lastr, retval, 0);
3559
3560 lck_mtx_unlock(&rap->cl_lockr);
3561 } else {
3562 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 32)) | DBG_FUNC_END,
3563 (int)uio->uio_offset, io_req_size, 0, retval, 0);
3564 }
3565
3566 return (retval);
3567 }
3568
3569
3570 static int
3571 cluster_read_direct(vnode_t vp, struct uio *uio, off_t filesize, int *read_type, u_int32_t *read_length,
3572 int flags, int (*callback)(buf_t, void *), void *callback_arg)
3573 {
3574 upl_t upl;
3575 upl_page_info_t *pl;
3576 off_t max_io_size;
3577 vm_offset_t upl_offset;
3578 vm_size_t upl_size;
3579 vm_size_t upl_needed_size;
3580 unsigned int pages_in_pl;
3581 int upl_flags;
3582 int bflag;
3583 kern_return_t kret;
3584 unsigned int i;
3585 int force_data_sync;
3586 int retval = 0;
3587 int no_zero_fill = 0;
3588 int abort_flag = 0;
3589 int io_flag = 0;
3590 int misaligned = 0;
3591 struct clios iostate;
3592 user_addr_t iov_base;
3593 u_int32_t io_req_size;
3594 u_int32_t offset_in_file;
3595 u_int32_t offset_in_iovbase;
3596 u_int32_t io_size;
3597 u_int32_t io_min;
3598 u_int32_t xsize;
3599 u_int32_t devblocksize;
3600 u_int32_t mem_alignment_mask;
3601 u_int32_t max_upl_size;
3602 u_int32_t max_rd_size;
3603 u_int32_t max_rd_ahead;
3604
3605
3606 max_upl_size = cluster_max_io_size(vp->v_mount, CL_READ);
3607
3608 max_rd_size = max_upl_size;
3609 max_rd_ahead = max_rd_size * 2;
3610
3611
3612 if (flags & IO_PASSIVE)
3613 bflag = CL_PASSIVE;
3614 else
3615 bflag = 0;
3616
3617 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 70)) | DBG_FUNC_START,
3618 (int)uio->uio_offset, (int)filesize, *read_type, *read_length, 0);
3619
3620 iostate.io_completed = 0;
3621 iostate.io_issued = 0;
3622 iostate.io_error = 0;
3623 iostate.io_wanted = 0;
3624
3625 devblocksize = (u_int32_t)vp->v_mount->mnt_devblocksize;
3626 mem_alignment_mask = (u_int32_t)vp->v_mount->mnt_alignmentmask;
3627
3628 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 70)) | DBG_FUNC_NONE,
3629 (int)devblocksize, (int)mem_alignment_mask, 0, 0, 0);
3630
3631 if (devblocksize == 1) {
3632 /*
3633 * the AFP client advertises a devblocksize of 1
3634 * however, its BLOCKMAP routine maps to physical
3635 * blocks that are PAGE_SIZE in size...
3636 * therefore we can't ask for I/Os that aren't page aligned
3637 * or aren't multiples of PAGE_SIZE in size
3638 * by setting devblocksize to PAGE_SIZE, we re-instate
3639 * the old behavior we had before the mem_alignment_mask
3640 * changes went in...
3641 */
3642 devblocksize = PAGE_SIZE;
3643 }
3644 next_dread:
3645 io_req_size = *read_length;
3646 iov_base = uio_curriovbase(uio);
3647
3648 max_io_size = filesize - uio->uio_offset;
3649
3650 if ((off_t)io_req_size > max_io_size)
3651 io_req_size = max_io_size;
3652
3653 offset_in_file = (u_int32_t)uio->uio_offset & (devblocksize - 1);
3654 offset_in_iovbase = (u_int32_t)iov_base & mem_alignment_mask;
3655
3656 if (offset_in_file || offset_in_iovbase) {
3657 /*
3658 * one of the 2 important offsets is misaligned
3659 * so fire an I/O through the cache for this entire vector
3660 */
3661 misaligned = 1;
3662 }
3663 if (iov_base & (devblocksize - 1)) {
3664 /*
3665 * the offset in memory must be on a device block boundary
3666 * so that we can guarantee that we can generate an
3667 * I/O that ends on a page boundary in cluster_io
3668 */
3669 misaligned = 1;
3670 }
3671 /*
3672 * When we get to this point, we know...
3673 * -- the offset into the file is on a devblocksize boundary
3674 */
3675
3676 while (io_req_size && retval == 0) {
3677 u_int32_t io_start;
3678
3679 if (cluster_hard_throttle_on(vp)) {
3680 max_rd_size = HARD_THROTTLE_MAXSIZE;
3681 max_rd_ahead = HARD_THROTTLE_MAXSIZE - 1;
3682 } else {
3683 max_rd_size = max_upl_size;
3684 max_rd_ahead = max_rd_size * 2;
3685 }
3686 io_start = io_size = io_req_size;
3687
3688 /*
3689 * First look for pages already in the cache
3690 * and move them to user space.
3691 *
3692 * cluster_copy_ubc_data returns the resid
3693 * in io_size
3694 */
3695 retval = cluster_copy_ubc_data_internal(vp, uio, (int *)&io_size, 0, 0);
3696
3697 /*
3698 * calculate the number of bytes actually copied
3699 * starting size - residual
3700 */
3701 xsize = io_start - io_size;
3702
3703 io_req_size -= xsize;
3704
3705 /*
3706 * check to see if we are finished with this request...
3707 */
3708 if (io_req_size == 0 || misaligned) {
3709 /*
3710 * see if there's another uio vector to
3711 * process that's of type IO_DIRECT
3712 *
3713 * break out of while loop to get there
3714 */
3715 break;
3716 }
3717 /*
3718 * assume the request ends on a device block boundary
3719 */
3720 io_min = devblocksize;
3721
3722 /*
3723 * we can handle I/O's in multiples of the device block size
3724 * however, if io_size isn't a multiple of devblocksize we
3725 * want to clip it back to the nearest page boundary since
3726 * we are going to have to go through cluster_read_copy to
3727 * deal with the 'overhang'... by clipping it to a PAGE_SIZE
3728 * multiple, we avoid asking the drive for the same physical
3729 * blocks twice.. once for the partial page at the end of the
3730 * request and a 2nd time for the page we read into the cache
3731 * (which overlaps the end of the direct read) in order to
3732 * get at the overhang bytes
3733 */
3734 if (io_size & (devblocksize - 1)) {
3735 /*
3736 * request does NOT end on a device block boundary
3737 * so clip it back to a PAGE_SIZE boundary
3738 */
3739 io_size &= ~PAGE_MASK;
3740 io_min = PAGE_SIZE;
3741 }
3742 if (retval || io_size < io_min) {
3743 /*
3744 * either an error or we only have the tail left to
3745 * complete via the copy path...
3746 * we may have already spun some portion of this request
3747 * off as async requests... we need to wait for the I/O
3748 * to complete before returning
3749 */
3750 goto wait_for_dreads;
3751 }
3752 if ((xsize = io_size) > max_rd_size)
3753 xsize = max_rd_size;
3754
3755 io_size = 0;
3756
3757 ubc_range_op(vp, uio->uio_offset, uio->uio_offset + xsize, UPL_ROP_ABSENT, (int *)&io_size);
3758
3759 if (io_size == 0) {
3760 /*
3761 * a page must have just come into the cache
3762 * since the first page in this range is no
3763 * longer absent, go back and re-evaluate
3764 */
3765 continue;
3766 }
3767 iov_base = uio_curriovbase(uio);
3768
3769 upl_offset = (vm_offset_t)((u_int32_t)iov_base & PAGE_MASK);
3770 upl_needed_size = (upl_offset + io_size + (PAGE_SIZE -1)) & ~PAGE_MASK;
3771
3772 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 72)) | DBG_FUNC_START,
3773 (int)upl_offset, upl_needed_size, (int)iov_base, io_size, 0);
3774
3775 if (upl_offset == 0 && ((io_size & PAGE_MASK) == 0)) {
3776 no_zero_fill = 1;
3777 abort_flag = UPL_ABORT_DUMP_PAGES | UPL_ABORT_FREE_ON_EMPTY;
3778 } else {
3779 no_zero_fill = 0;
3780 abort_flag = UPL_ABORT_FREE_ON_EMPTY;
3781 }
3782 for (force_data_sync = 0; force_data_sync < 3; force_data_sync++) {
3783 pages_in_pl = 0;
3784 upl_size = upl_needed_size;
3785 upl_flags = UPL_FILE_IO | UPL_NO_SYNC | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE;
3786
3787 if (no_zero_fill)
3788 upl_flags |= UPL_NOZEROFILL;
3789 if (force_data_sync)
3790 upl_flags |= UPL_FORCE_DATA_SYNC;
3791
3792 kret = vm_map_create_upl(current_map(),
3793 (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)),
3794 &upl_size, &upl, NULL, &pages_in_pl, &upl_flags);
3795
3796 if (kret != KERN_SUCCESS) {
3797 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 72)) | DBG_FUNC_END,
3798 (int)upl_offset, upl_size, io_size, kret, 0);
3799 /*
3800 * failed to get pagelist
3801 *
3802 * we may have already spun some portion of this request
3803 * off as async requests... we need to wait for the I/O
3804 * to complete before returning
3805 */
3806 goto wait_for_dreads;
3807 }
3808 pages_in_pl = upl_size / PAGE_SIZE;
3809 pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
3810
3811 for (i = 0; i < pages_in_pl; i++) {
3812 if (!upl_valid_page(pl, i))
3813 break;
3814 }
3815 if (i == pages_in_pl)
3816 break;
3817
3818 ubc_upl_abort(upl, abort_flag);
3819 }
3820 if (force_data_sync >= 3) {
3821 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 72)) | DBG_FUNC_END,
3822 (int)upl_offset, upl_size, io_size, kret, 0);
3823
3824 goto wait_for_dreads;
3825 }
3826 /*
3827 * Consider the possibility that upl_size wasn't satisfied.
3828 */
3829 if (upl_size < upl_needed_size) {
3830 if (upl_size && upl_offset == 0)
3831 io_size = upl_size;
3832 else
3833 io_size = 0;
3834 }
3835 if (io_size == 0) {
3836 ubc_upl_abort(upl, abort_flag);
3837 goto wait_for_dreads;
3838 }
3839 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 72)) | DBG_FUNC_END,
3840 (int)upl_offset, upl_size, io_size, kret, 0);
3841
3842 /*
3843 * request asynchronously so that we can overlap
3844 * the preparation of the next I/O
3845 * if there are already too many outstanding reads
3846 * wait until some have completed before issuing the next read
3847 */
3848 lck_mtx_lock(cl_mtxp);
3849
3850 while ((iostate.io_issued - iostate.io_completed) > max_rd_ahead) {
3851 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_START,
3852 iostate.io_issued, iostate.io_completed, max_rd_ahead, 0, 0);
3853
3854 iostate.io_wanted = 1;
3855 msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_read_direct", NULL);
3856
3857 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_END,
3858 iostate.io_issued, iostate.io_completed, max_rd_ahead, 0, 0);
3859 }
3860 lck_mtx_unlock(cl_mtxp);
3861
3862 if (iostate.io_error) {
3863 /*
3864 * one of the earlier reads we issued ran into a hard error
3865 * don't issue any more reads, cleanup the UPL
3866 * that was just created but not used, then
3867 * go wait for any other reads to complete before
3868 * returning the error to the caller
3869 */
3870 ubc_upl_abort(upl, abort_flag);
3871
3872 goto wait_for_dreads;
3873 }
3874 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 73)) | DBG_FUNC_START,
3875 (int)upl, (int)upl_offset, (int)uio->uio_offset, io_size, 0);
3876
3877 if (no_zero_fill)
3878 io_flag = CL_COMMIT | CL_READ | CL_ASYNC | CL_NOZERO | CL_DIRECT_IO | bflag;
3879 else
3880 io_flag = CL_COMMIT | CL_READ | CL_ASYNC | CL_NOZERO | CL_DIRECT_IO | CL_PRESERVE | bflag;
3881
3882 retval = cluster_io(vp, upl, upl_offset, uio->uio_offset, io_size, io_flag, (buf_t)NULL, &iostate, callback, callback_arg);
3883
3884 /*
3885 * update the uio structure
3886 */
3887 uio_update(uio, (user_size_t)io_size);
3888
3889 io_req_size -= io_size;
3890
3891 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 73)) | DBG_FUNC_END,
3892 (int)upl, (int)uio->uio_offset, io_req_size, retval, 0);
3893
3894 } /* end while */
3895
3896 if (retval == 0 && iostate.io_error == 0 && io_req_size == 0 && uio->uio_offset < filesize) {
3897
3898 retval = cluster_io_type(uio, read_type, read_length, 0);
3899
3900 if (retval == 0 && *read_type == IO_DIRECT) {
3901
3902 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 70)) | DBG_FUNC_NONE,
3903 (int)uio->uio_offset, (int)filesize, *read_type, *read_length, 0);
3904
3905 goto next_dread;
3906 }
3907 }
3908
3909 wait_for_dreads:
3910 if (iostate.io_issued) {
3911 /*
3912 * make sure all async reads that are part of this stream
3913 * have completed before we return
3914 */
3915 lck_mtx_lock(cl_mtxp);
3916
3917 while (iostate.io_issued != iostate.io_completed) {
3918 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_START,
3919 iostate.io_issued, iostate.io_completed, 0, 0, 0);
3920
3921 iostate.io_wanted = 1;
3922 msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_read_direct", NULL);
3923
3924 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_END,
3925 iostate.io_issued, iostate.io_completed, 0, 0, 0);
3926 }
3927 lck_mtx_unlock(cl_mtxp);
3928 }
3929
3930 if (iostate.io_error)
3931 retval = iostate.io_error;
3932
3933 if (io_req_size && retval == 0) {
3934 /*
3935 * we couldn't handle the tail of this request in DIRECT mode
3936 * so fire it through the copy path
3937 */
3938 retval = cluster_read_copy(vp, uio, io_req_size, filesize, flags, callback, callback_arg);
3939
3940 *read_type = IO_UNKNOWN;
3941 }
3942 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 70)) | DBG_FUNC_END,
3943 (int)uio->uio_offset, (int)uio_resid(uio), io_req_size, retval, 0);
3944
3945 return (retval);
3946 }
3947
3948
3949 static int
3950 cluster_read_contig(vnode_t vp, struct uio *uio, off_t filesize, int *read_type, u_int32_t *read_length,
3951 int (*callback)(buf_t, void *), void *callback_arg, int flags)
3952 {
3953 upl_page_info_t *pl;
3954 upl_t upl[MAX_VECTS];
3955 vm_offset_t upl_offset;
3956 addr64_t dst_paddr = 0;
3957 user_addr_t iov_base;
3958 off_t max_size;
3959 vm_size_t upl_size;
3960 vm_size_t upl_needed_size;
3961 mach_msg_type_number_t pages_in_pl;
3962 int upl_flags;
3963 kern_return_t kret;
3964 struct clios iostate;
3965 int error= 0;
3966 int cur_upl = 0;
3967 int num_upl = 0;
3968 int n;
3969 u_int32_t xsize;
3970 u_int32_t io_size;
3971 u_int32_t devblocksize;
3972 u_int32_t mem_alignment_mask;
3973 u_int32_t tail_size = 0;
3974 int bflag;
3975
3976 if (flags & IO_PASSIVE)
3977 bflag = CL_PASSIVE;
3978 else
3979 bflag = 0;
3980
3981 /*
3982 * When we enter this routine, we know
3983 * -- the read_length will not exceed the current iov_len
3984 * -- the target address is physically contiguous for read_length
3985 */
3986 cluster_syncup(vp, filesize, callback, callback_arg);
3987
3988 devblocksize = (u_int32_t)vp->v_mount->mnt_devblocksize;
3989 mem_alignment_mask = (u_int32_t)vp->v_mount->mnt_alignmentmask;
3990
3991 iostate.io_completed = 0;
3992 iostate.io_issued = 0;
3993 iostate.io_error = 0;
3994 iostate.io_wanted = 0;
3995
3996 next_cread:
3997 io_size = *read_length;
3998
3999 max_size = filesize - uio->uio_offset;
4000
4001 if (io_size > max_size)
4002 io_size = max_size;
4003
4004 iov_base = uio_curriovbase(uio);
4005
4006 upl_offset = (vm_offset_t)((u_int32_t)iov_base & PAGE_MASK);
4007 upl_needed_size = upl_offset + io_size;
4008
4009 pages_in_pl = 0;
4010 upl_size = upl_needed_size;
4011 upl_flags = UPL_FILE_IO | UPL_NO_SYNC | UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE;
4012
4013
4014 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 92)) | DBG_FUNC_START,
4015 (int)upl_offset, (int)upl_size, (int)iov_base, io_size, 0);
4016
4017 kret = vm_map_get_upl(current_map(),
4018 (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)),
4019 &upl_size, &upl[cur_upl], NULL, &pages_in_pl, &upl_flags, 0);
4020
4021 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 92)) | DBG_FUNC_END,
4022 (int)upl_offset, upl_size, io_size, kret, 0);
4023
4024 if (kret != KERN_SUCCESS) {
4025 /*
4026 * failed to get pagelist
4027 */
4028 error = EINVAL;
4029 goto wait_for_creads;
4030 }
4031 num_upl++;
4032
4033 if (upl_size < upl_needed_size) {
4034 /*
4035 * The upl_size wasn't satisfied.
4036 */
4037 error = EINVAL;
4038 goto wait_for_creads;
4039 }
4040 pl = ubc_upl_pageinfo(upl[cur_upl]);
4041
4042 dst_paddr = ((addr64_t)upl_phys_page(pl, 0) << 12) + (addr64_t)upl_offset;
4043
4044 while (((uio->uio_offset & (devblocksize - 1)) || io_size < devblocksize) && io_size) {
4045 u_int32_t head_size;
4046
4047 head_size = devblocksize - (u_int32_t)(uio->uio_offset & (devblocksize - 1));
4048
4049 if (head_size > io_size)
4050 head_size = io_size;
4051
4052 error = cluster_align_phys_io(vp, uio, dst_paddr, head_size, CL_READ, callback, callback_arg);
4053
4054 if (error)
4055 goto wait_for_creads;
4056
4057 upl_offset += head_size;
4058 dst_paddr += head_size;
4059 io_size -= head_size;
4060
4061 iov_base += head_size;
4062 }
4063 if ((u_int32_t)iov_base & mem_alignment_mask) {
4064 /*
4065 * request doesn't set up on a memory boundary
4066 * the underlying DMA engine can handle...
4067 * return an error instead of going through
4068 * the slow copy path since the intent of this
4069 * path is direct I/O to device memory
4070 */
4071 error = EINVAL;
4072 goto wait_for_creads;
4073 }
4074
4075 tail_size = io_size & (devblocksize - 1);
4076
4077 io_size -= tail_size;
4078
4079 while (io_size && error == 0) {
4080
4081 if (io_size > MAX_IO_CONTIG_SIZE)
4082 xsize = MAX_IO_CONTIG_SIZE;
4083 else
4084 xsize = io_size;
4085 /*
4086 * request asynchronously so that we can overlap
4087 * the preparation of the next I/O... we'll do
4088 * the commit after all the I/O has completed
4089 * since its all issued against the same UPL
4090 * if there are already too many outstanding reads
4091 * wait until some have completed before issuing the next
4092 */
4093 if (iostate.io_issued) {
4094 lck_mtx_lock(cl_mtxp);
4095
4096 while ((iostate.io_issued - iostate.io_completed) > (2 * MAX_IO_CONTIG_SIZE)) {
4097 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_START,
4098 iostate.io_issued, iostate.io_completed, 2 * MAX_IO_CONTIG_SIZE, 0, 0);
4099
4100 iostate.io_wanted = 1;
4101 msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_read_contig", NULL);
4102
4103 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_END,
4104 iostate.io_issued, iostate.io_completed, 2 * MAX_IO_CONTIG_SIZE, 0, 0);
4105 }
4106 lck_mtx_unlock(cl_mtxp);
4107 }
4108 if (iostate.io_error) {
4109 /*
4110 * one of the earlier reads we issued ran into a hard error
4111 * don't issue any more reads...
4112 * go wait for any other reads to complete before
4113 * returning the error to the caller
4114 */
4115 goto wait_for_creads;
4116 }
4117 error = cluster_io(vp, upl[cur_upl], upl_offset, uio->uio_offset, xsize,
4118 CL_READ | CL_NOZERO | CL_DEV_MEMORY | CL_ASYNC | bflag,
4119 (buf_t)NULL, &iostate, callback, callback_arg);
4120 /*
4121 * The cluster_io read was issued successfully,
4122 * update the uio structure
4123 */
4124 if (error == 0) {
4125 uio_update(uio, (user_size_t)xsize);
4126
4127 dst_paddr += xsize;
4128 upl_offset += xsize;
4129 io_size -= xsize;
4130 }
4131 }
4132 if (error == 0 && iostate.io_error == 0 && tail_size == 0 && num_upl < MAX_VECTS && uio->uio_offset < filesize) {
4133
4134 error = cluster_io_type(uio, read_type, read_length, 0);
4135
4136 if (error == 0 && *read_type == IO_CONTIG) {
4137 cur_upl++;
4138 goto next_cread;
4139 }
4140 } else
4141 *read_type = IO_UNKNOWN;
4142
4143 wait_for_creads:
4144 /*
4145 * make sure all async reads that are part of this stream
4146 * have completed before we proceed
4147 */
4148 lck_mtx_lock(cl_mtxp);
4149
4150 while (iostate.io_issued != iostate.io_completed) {
4151 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_START,
4152 iostate.io_issued, iostate.io_completed, 0, 0, 0);
4153
4154 iostate.io_wanted = 1;
4155 msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_read_contig", NULL);
4156
4157 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_END,
4158 iostate.io_issued, iostate.io_completed, 0, 0, 0);
4159 }
4160 lck_mtx_unlock(cl_mtxp);
4161
4162 if (iostate.io_error)
4163 error = iostate.io_error;
4164
4165 if (error == 0 && tail_size)
4166 error = cluster_align_phys_io(vp, uio, dst_paddr, tail_size, CL_READ, callback, callback_arg);
4167
4168 for (n = 0; n < num_upl; n++)
4169 /*
4170 * just release our hold on each physically contiguous
4171 * region without changing any state
4172 */
4173 ubc_upl_abort(upl[n], 0);
4174
4175 return (error);
4176 }
4177
4178
4179 static int
4180 cluster_io_type(struct uio *uio, int *io_type, u_int32_t *io_length, u_int32_t min_length)
4181 {
4182 user_size_t iov_len;
4183 user_addr_t iov_base = 0;
4184 upl_t upl;
4185 vm_size_t upl_size;
4186 int upl_flags;
4187 int retval = 0;
4188
4189 /*
4190 * skip over any emtpy vectors
4191 */
4192 uio_update(uio, (user_size_t)0);
4193
4194 iov_len = uio_curriovlen(uio);
4195
4196 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 94)) | DBG_FUNC_START, (int)uio, (int)iov_len, 0, 0, 0);
4197
4198 if (iov_len) {
4199 iov_base = uio_curriovbase(uio);
4200 /*
4201 * make sure the size of the vector isn't too big...
4202 * internally, we want to handle all of the I/O in
4203 * chunk sizes that fit in a 32 bit int
4204 */
4205 if (iov_len > (user_size_t)MAX_IO_REQUEST_SIZE)
4206 upl_size = MAX_IO_REQUEST_SIZE;
4207 else
4208 upl_size = (u_int32_t)iov_len;
4209
4210 upl_flags = UPL_QUERY_OBJECT_TYPE;
4211
4212 if ((vm_map_get_upl(current_map(),
4213 (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)),
4214 &upl_size, &upl, NULL, NULL, &upl_flags, 0)) != KERN_SUCCESS) {
4215 /*
4216 * the user app must have passed in an invalid address
4217 */
4218 retval = EFAULT;
4219 }
4220 if (upl_size == 0)
4221 retval = EFAULT;
4222
4223 *io_length = upl_size;
4224
4225 if (upl_flags & UPL_PHYS_CONTIG)
4226 *io_type = IO_CONTIG;
4227 else if (iov_len >= min_length)
4228 *io_type = IO_DIRECT;
4229 else
4230 *io_type = IO_COPY;
4231 } else {
4232 /*
4233 * nothing left to do for this uio
4234 */
4235 *io_length = 0;
4236 *io_type = IO_UNKNOWN;
4237 }
4238 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 94)) | DBG_FUNC_END, (int)iov_base, *io_type, *io_length, retval, 0);
4239
4240 return (retval);
4241 }
4242
4243
4244 /*
4245 * generate advisory I/O's in the largest chunks possible
4246 * the completed pages will be released into the VM cache
4247 */
4248 int
4249 advisory_read(vnode_t vp, off_t filesize, off_t f_offset, int resid)
4250 {
4251 return advisory_read_ext(vp, filesize, f_offset, resid, NULL, NULL, CL_PASSIVE);
4252 }
4253
4254 int
4255 advisory_read_ext(vnode_t vp, off_t filesize, off_t f_offset, int resid, int (*callback)(buf_t, void *), void *callback_arg, int bflag)
4256 {
4257 upl_page_info_t *pl;
4258 upl_t upl;
4259 vm_offset_t upl_offset;
4260 int upl_size;
4261 off_t upl_f_offset;
4262 int start_offset;
4263 int start_pg;
4264 int last_pg;
4265 int pages_in_upl;
4266 off_t max_size;
4267 int io_size;
4268 kern_return_t kret;
4269 int retval = 0;
4270 int issued_io;
4271 int skip_range;
4272 uint32_t max_io_size;
4273
4274
4275 if ( !UBCINFOEXISTS(vp))
4276 return(EINVAL);
4277
4278 if (resid < 0)
4279 return(EINVAL);
4280
4281 max_io_size = cluster_max_io_size(vp->v_mount, CL_READ);
4282
4283 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 60)) | DBG_FUNC_START,
4284 (int)f_offset, resid, (int)filesize, 0, 0);
4285
4286 while (resid && f_offset < filesize && retval == 0) {
4287 /*
4288 * compute the size of the upl needed to encompass
4289 * the requested read... limit each call to cluster_io
4290 * to the maximum UPL size... cluster_io will clip if
4291 * this exceeds the maximum io_size for the device,
4292 * make sure to account for
4293 * a starting offset that's not page aligned
4294 */
4295 start_offset = (int)(f_offset & PAGE_MASK_64);
4296 upl_f_offset = f_offset - (off_t)start_offset;
4297 max_size = filesize - f_offset;
4298
4299 if (resid < max_size)
4300 io_size = resid;
4301 else
4302 io_size = max_size;
4303
4304 upl_size = (start_offset + io_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
4305 if ((uint32_t)upl_size > max_io_size)
4306 upl_size = max_io_size;
4307
4308 skip_range = 0;
4309 /*
4310 * return the number of contiguously present pages in the cache
4311 * starting at upl_f_offset within the file
4312 */
4313 ubc_range_op(vp, upl_f_offset, upl_f_offset + upl_size, UPL_ROP_PRESENT, &skip_range);
4314
4315 if (skip_range) {
4316 /*
4317 * skip over pages already present in the cache
4318 */
4319 io_size = skip_range - start_offset;
4320
4321 f_offset += io_size;
4322 resid -= io_size;
4323
4324 if (skip_range == upl_size)
4325 continue;
4326 /*
4327 * have to issue some real I/O
4328 * at this point, we know it's starting on a page boundary
4329 * because we've skipped over at least the first page in the request
4330 */
4331 start_offset = 0;
4332 upl_f_offset += skip_range;
4333 upl_size -= skip_range;
4334 }
4335 pages_in_upl = upl_size / PAGE_SIZE;
4336
4337 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 61)) | DBG_FUNC_START,
4338 (int)upl, (int)upl_f_offset, upl_size, start_offset, 0);
4339
4340 kret = ubc_create_upl(vp,
4341 upl_f_offset,
4342 upl_size,
4343 &upl,
4344 &pl,
4345 UPL_RET_ONLY_ABSENT | UPL_SET_LITE);
4346 if (kret != KERN_SUCCESS)
4347 return(retval);
4348 issued_io = 0;
4349
4350 /*
4351 * before we start marching forward, we must make sure we end on
4352 * a present page, otherwise we will be working with a freed
4353 * upl
4354 */
4355 for (last_pg = pages_in_upl - 1; last_pg >= 0; last_pg--) {
4356 if (upl_page_present(pl, last_pg))
4357 break;
4358 }
4359 pages_in_upl = last_pg + 1;
4360
4361
4362 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 61)) | DBG_FUNC_END,
4363 (int)upl, (int)upl_f_offset, upl_size, start_offset, 0);
4364
4365
4366 for (last_pg = 0; last_pg < pages_in_upl; ) {
4367 /*
4368 * scan from the beginning of the upl looking for the first
4369 * page that is present.... this will become the first page in
4370 * the request we're going to make to 'cluster_io'... if all
4371 * of the pages are absent, we won't call through to 'cluster_io'
4372 */
4373 for (start_pg = last_pg; start_pg < pages_in_upl; start_pg++) {
4374 if (upl_page_present(pl, start_pg))
4375 break;
4376 }
4377
4378 /*
4379 * scan from the starting present page looking for an absent
4380 * page before the end of the upl is reached, if we
4381 * find one, then it will terminate the range of pages being
4382 * presented to 'cluster_io'
4383 */
4384 for (last_pg = start_pg; last_pg < pages_in_upl; last_pg++) {
4385 if (!upl_page_present(pl, last_pg))
4386 break;
4387 }
4388
4389 if (last_pg > start_pg) {
4390 /*
4391 * we found a range of pages that must be filled
4392 * if the last page in this range is the last page of the file
4393 * we may have to clip the size of it to keep from reading past
4394 * the end of the last physical block associated with the file
4395 */
4396 upl_offset = start_pg * PAGE_SIZE;
4397 io_size = (last_pg - start_pg) * PAGE_SIZE;
4398
4399 if ((upl_f_offset + upl_offset + io_size) > filesize)
4400 io_size = filesize - (upl_f_offset + upl_offset);
4401
4402 /*
4403 * issue an asynchronous read to cluster_io
4404 */
4405 retval = cluster_io(vp, upl, upl_offset, upl_f_offset + upl_offset, io_size,
4406 CL_ASYNC | CL_READ | CL_COMMIT | CL_AGE | bflag, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
4407
4408 issued_io = 1;
4409 }
4410 }
4411 if (issued_io == 0)
4412 ubc_upl_abort(upl, 0);
4413
4414 io_size = upl_size - start_offset;
4415
4416 if (io_size > resid)
4417 io_size = resid;
4418 f_offset += io_size;
4419 resid -= io_size;
4420 }
4421
4422 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 60)) | DBG_FUNC_END,
4423 (int)f_offset, resid, retval, 0, 0);
4424
4425 return(retval);
4426 }
4427
4428
4429 int
4430 cluster_push(vnode_t vp, int flags)
4431 {
4432 return cluster_push_ext(vp, flags, NULL, NULL);
4433 }
4434
4435
4436 int
4437 cluster_push_ext(vnode_t vp, int flags, int (*callback)(buf_t, void *), void *callback_arg)
4438 {
4439 int retval;
4440 struct cl_writebehind *wbp;
4441
4442 if ( !UBCINFOEXISTS(vp)) {
4443 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 53)) | DBG_FUNC_NONE, (int)vp, flags, 0, -1, 0);
4444 return (0);
4445 }
4446 /* return if deferred write is set */
4447 if (((unsigned int)vfs_flags(vp->v_mount) & MNT_DEFWRITE) && (flags & IO_DEFWRITE)) {
4448 return (0);
4449 }
4450 if ((wbp = cluster_get_wbp(vp, CLW_RETURNLOCKED)) == NULL) {
4451 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 53)) | DBG_FUNC_NONE, (int)vp, flags, 0, -2, 0);
4452 return (0);
4453 }
4454 if (wbp->cl_number == 0 && wbp->cl_scmap == NULL) {
4455 lck_mtx_unlock(&wbp->cl_lockw);
4456
4457 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 53)) | DBG_FUNC_NONE, (int)vp, flags, 0, -3, 0);
4458 return(0);
4459 }
4460 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 53)) | DBG_FUNC_START,
4461 (int)wbp->cl_scmap, wbp->cl_number, flags, 0, 0);
4462
4463 if (wbp->cl_scmap) {
4464 sparse_cluster_push(wbp, vp, ubc_getsize(vp), PUSH_ALL | IO_PASSIVE, callback, callback_arg);
4465
4466 retval = 1;
4467 } else
4468 retval = cluster_try_push(wbp, vp, ubc_getsize(vp), PUSH_ALL | IO_PASSIVE, callback, callback_arg);
4469
4470 lck_mtx_unlock(&wbp->cl_lockw);
4471
4472 if (flags & IO_SYNC)
4473 (void)vnode_waitforwrites(vp, 0, 0, 0, "cluster_push");
4474
4475 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 53)) | DBG_FUNC_END,
4476 (int)wbp->cl_scmap, wbp->cl_number, retval, 0, 0);
4477
4478 return (retval);
4479 }
4480
4481
4482 __private_extern__ void
4483 cluster_release(struct ubc_info *ubc)
4484 {
4485 struct cl_writebehind *wbp;
4486 struct cl_readahead *rap;
4487
4488 if ((wbp = ubc->cl_wbehind)) {
4489
4490 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 81)) | DBG_FUNC_START, (int)ubc, (int)wbp->cl_scmap, wbp->cl_scdirty, 0, 0);
4491
4492 if (wbp->cl_scmap)
4493 vfs_drt_control(&(wbp->cl_scmap), 0);
4494 } else {
4495 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 81)) | DBG_FUNC_START, (int)ubc, 0, 0, 0, 0);
4496 }
4497
4498 rap = ubc->cl_rahead;
4499
4500 if (wbp != NULL) {
4501 lck_mtx_destroy(&wbp->cl_lockw, cl_mtx_grp);
4502 FREE_ZONE((void *)wbp, sizeof *wbp, M_CLWRBEHIND);
4503 }
4504 if ((rap = ubc->cl_rahead)) {
4505 lck_mtx_destroy(&rap->cl_lockr, cl_mtx_grp);
4506 FREE_ZONE((void *)rap, sizeof *rap, M_CLRDAHEAD);
4507 }
4508 ubc->cl_rahead = NULL;
4509 ubc->cl_wbehind = NULL;
4510
4511 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 81)) | DBG_FUNC_END, (int)ubc, (int)rap, (int)wbp, 0, 0);
4512 }
4513
4514
4515 static int
4516 cluster_try_push(struct cl_writebehind *wbp, vnode_t vp, off_t EOF, int push_flag, int (*callback)(buf_t, void *), void *callback_arg)
4517 {
4518 int cl_index;
4519 int cl_index1;
4520 int min_index;
4521 int cl_len;
4522 int cl_pushed = 0;
4523 struct cl_wextent l_clusters[MAX_CLUSTERS];
4524 u_int max_cluster_pgcount;
4525
4526
4527 max_cluster_pgcount = MAX_CLUSTER_SIZE(vp) / PAGE_SIZE;
4528 /*
4529 * the write behind context exists and has
4530 * already been locked...
4531 */
4532 if (wbp->cl_number == 0)
4533 /*
4534 * no clusters to push
4535 * return number of empty slots
4536 */
4537 return (MAX_CLUSTERS);
4538
4539 /*
4540 * make a local 'sorted' copy of the clusters
4541 * and clear wbp->cl_number so that new clusters can
4542 * be developed
4543 */
4544 for (cl_index = 0; cl_index < wbp->cl_number; cl_index++) {
4545 for (min_index = -1, cl_index1 = 0; cl_index1 < wbp->cl_number; cl_index1++) {
4546 if (wbp->cl_clusters[cl_index1].b_addr == wbp->cl_clusters[cl_index1].e_addr)
4547 continue;
4548 if (min_index == -1)
4549 min_index = cl_index1;
4550 else if (wbp->cl_clusters[cl_index1].b_addr < wbp->cl_clusters[min_index].b_addr)
4551 min_index = cl_index1;
4552 }
4553 if (min_index == -1)
4554 break;
4555 l_clusters[cl_index].b_addr = wbp->cl_clusters[min_index].b_addr;
4556 l_clusters[cl_index].e_addr = wbp->cl_clusters[min_index].e_addr;
4557 l_clusters[cl_index].io_flags = wbp->cl_clusters[min_index].io_flags;
4558
4559 wbp->cl_clusters[min_index].b_addr = wbp->cl_clusters[min_index].e_addr;
4560 }
4561 wbp->cl_number = 0;
4562
4563 cl_len = cl_index;
4564
4565 if ( (push_flag & PUSH_DELAY) && cl_len == MAX_CLUSTERS ) {
4566 int i;
4567
4568 /*
4569 * determine if we appear to be writing the file sequentially
4570 * if not, by returning without having pushed any clusters
4571 * we will cause this vnode to be pushed into the sparse cluster mechanism
4572 * used for managing more random I/O patterns
4573 *
4574 * we know that we've got all clusters currently in use and the next write doesn't fit into one of them...
4575 * that's why we're in try_push with PUSH_DELAY...
4576 *
4577 * check to make sure that all the clusters except the last one are 'full'... and that each cluster
4578 * is adjacent to the next (i.e. we're looking for sequential writes) they were sorted above
4579 * so we can just make a simple pass through, up to, but not including the last one...
4580 * note that e_addr is not inclusive, so it will be equal to the b_addr of the next cluster if they
4581 * are sequential
4582 *
4583 * we let the last one be partial as long as it was adjacent to the previous one...
4584 * we need to do this to deal with multi-threaded servers that might write an I/O or 2 out
4585 * of order... if this occurs at the tail of the last cluster, we don't want to fall into the sparse cluster world...
4586 */
4587 for (i = 0; i < MAX_CLUSTERS - 1; i++) {
4588 if ((l_clusters[i].e_addr - l_clusters[i].b_addr) != max_cluster_pgcount)
4589 goto dont_try;
4590 if (l_clusters[i].e_addr != l_clusters[i+1].b_addr)
4591 goto dont_try;
4592 }
4593 }
4594 for (cl_index = 0; cl_index < cl_len; cl_index++) {
4595 int flags;
4596 struct cl_extent cl;
4597
4598 /*
4599 * try to push each cluster in turn...
4600 */
4601 if (l_clusters[cl_index].io_flags & CLW_IONOCACHE)
4602 flags = IO_NOCACHE;
4603 else
4604 flags = 0;
4605
4606 if ((l_clusters[cl_index].io_flags & CLW_IOPASSIVE) || (push_flag & IO_PASSIVE))
4607 flags |= IO_PASSIVE;
4608
4609 if (push_flag & PUSH_SYNC)
4610 flags |= IO_SYNC;
4611
4612 cl.b_addr = l_clusters[cl_index].b_addr;
4613 cl.e_addr = l_clusters[cl_index].e_addr;
4614
4615 cluster_push_now(vp, &cl, EOF, flags, callback, callback_arg);
4616
4617 l_clusters[cl_index].b_addr = 0;
4618 l_clusters[cl_index].e_addr = 0;
4619
4620 cl_pushed++;
4621
4622 if ( !(push_flag & PUSH_ALL) )
4623 break;
4624 }
4625 dont_try:
4626 if (cl_len > cl_pushed) {
4627 /*
4628 * we didn't push all of the clusters, so
4629 * lets try to merge them back in to the vnode
4630 */
4631 if ((MAX_CLUSTERS - wbp->cl_number) < (cl_len - cl_pushed)) {
4632 /*
4633 * we picked up some new clusters while we were trying to
4634 * push the old ones... this can happen because I've dropped
4635 * the vnode lock... the sum of the
4636 * leftovers plus the new cluster count exceeds our ability
4637 * to represent them, so switch to the sparse cluster mechanism
4638 *
4639 * collect the active public clusters...
4640 */
4641 sparse_cluster_switch(wbp, vp, EOF, callback, callback_arg);
4642
4643 for (cl_index = 0, cl_index1 = 0; cl_index < cl_len; cl_index++) {
4644 if (l_clusters[cl_index].b_addr == l_clusters[cl_index].e_addr)
4645 continue;
4646 wbp->cl_clusters[cl_index1].b_addr = l_clusters[cl_index].b_addr;
4647 wbp->cl_clusters[cl_index1].e_addr = l_clusters[cl_index].e_addr;
4648 wbp->cl_clusters[cl_index1].io_flags = l_clusters[cl_index].io_flags;
4649
4650 cl_index1++;
4651 }
4652 /*
4653 * update the cluster count
4654 */
4655 wbp->cl_number = cl_index1;
4656
4657 /*
4658 * and collect the original clusters that were moved into the
4659 * local storage for sorting purposes
4660 */
4661 sparse_cluster_switch(wbp, vp, EOF, callback, callback_arg);
4662
4663 } else {
4664 /*
4665 * we've got room to merge the leftovers back in
4666 * just append them starting at the next 'hole'
4667 * represented by wbp->cl_number
4668 */
4669 for (cl_index = 0, cl_index1 = wbp->cl_number; cl_index < cl_len; cl_index++) {
4670 if (l_clusters[cl_index].b_addr == l_clusters[cl_index].e_addr)
4671 continue;
4672
4673 wbp->cl_clusters[cl_index1].b_addr = l_clusters[cl_index].b_addr;
4674 wbp->cl_clusters[cl_index1].e_addr = l_clusters[cl_index].e_addr;
4675 wbp->cl_clusters[cl_index1].io_flags = l_clusters[cl_index].io_flags;
4676
4677 cl_index1++;
4678 }
4679 /*
4680 * update the cluster count
4681 */
4682 wbp->cl_number = cl_index1;
4683 }
4684 }
4685 return (MAX_CLUSTERS - wbp->cl_number);
4686 }
4687
4688
4689
4690 static int
4691 cluster_push_now(vnode_t vp, struct cl_extent *cl, off_t EOF, int flags, int (*callback)(buf_t, void *), void *callback_arg)
4692 {
4693 upl_page_info_t *pl;
4694 upl_t upl;
4695 vm_offset_t upl_offset;
4696 int upl_size;
4697 off_t upl_f_offset;
4698 int pages_in_upl;
4699 int start_pg;
4700 int last_pg;
4701 int io_size;
4702 int io_flags;
4703 int upl_flags;
4704 int bflag;
4705 int size;
4706 int error = 0;
4707 int retval;
4708 kern_return_t kret;
4709
4710 if (flags & IO_PASSIVE)
4711 bflag = CL_PASSIVE;
4712 else
4713 bflag = 0;
4714
4715 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 51)) | DBG_FUNC_START,
4716 (int)cl->b_addr, (int)cl->e_addr, (int)EOF, flags, 0);
4717
4718 if ((pages_in_upl = (int)(cl->e_addr - cl->b_addr)) == 0) {
4719 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 51)) | DBG_FUNC_END, 1, 0, 0, 0, 0);
4720
4721 return (0);
4722 }
4723 upl_size = pages_in_upl * PAGE_SIZE;
4724 upl_f_offset = (off_t)(cl->b_addr * PAGE_SIZE_64);
4725
4726 if (upl_f_offset + upl_size >= EOF) {
4727
4728 if (upl_f_offset >= EOF) {
4729 /*
4730 * must have truncated the file and missed
4731 * clearing a dangling cluster (i.e. it's completely
4732 * beyond the new EOF
4733 */
4734 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 51)) | DBG_FUNC_END, 1, 1, 0, 0, 0);
4735
4736 return(0);
4737 }
4738 size = EOF - upl_f_offset;
4739
4740 upl_size = (size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
4741 pages_in_upl = upl_size / PAGE_SIZE;
4742 } else
4743 size = upl_size;
4744
4745 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 41)) | DBG_FUNC_START, upl_size, size, 0, 0, 0);
4746
4747 /*
4748 * by asking for UPL_COPYOUT_FROM and UPL_RET_ONLY_DIRTY, we get the following desirable behavior
4749 *
4750 * - only pages that are currently dirty are returned... these are the ones we need to clean
4751 * - the hardware dirty bit is cleared when the page is gathered into the UPL... the software dirty bit is set
4752 * - if we have to abort the I/O for some reason, the software dirty bit is left set since we didn't clean the page
4753 * - when we commit the page, the software dirty bit is cleared... the hardware dirty bit is untouched so that if
4754 * someone dirties this page while the I/O is in progress, we don't lose track of the new state
4755 *
4756 * when the I/O completes, we no longer ask for an explicit clear of the DIRTY state (either soft or hard)
4757 */
4758
4759 if ((vp->v_flag & VNOCACHE_DATA) || (flags & IO_NOCACHE))
4760 upl_flags = UPL_COPYOUT_FROM | UPL_RET_ONLY_DIRTY | UPL_SET_LITE | UPL_WILL_BE_DUMPED;
4761 else
4762 upl_flags = UPL_COPYOUT_FROM | UPL_RET_ONLY_DIRTY | UPL_SET_LITE;
4763
4764 kret = ubc_create_upl(vp,
4765 upl_f_offset,
4766 upl_size,
4767 &upl,
4768 &pl,
4769 upl_flags);
4770 if (kret != KERN_SUCCESS)
4771 panic("cluster_push: failed to get pagelist");
4772
4773 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 41)) | DBG_FUNC_END, (int)upl, upl_f_offset, 0, 0, 0);
4774
4775 /*
4776 * since we only asked for the dirty pages back
4777 * it's possible that we may only get a few or even none, so...
4778 * before we start marching forward, we must make sure we know
4779 * where the last present page is in the UPL, otherwise we could
4780 * end up working with a freed upl due to the FREE_ON_EMPTY semantics
4781 * employed by commit_range and abort_range.
4782 */
4783 for (last_pg = pages_in_upl - 1; last_pg >= 0; last_pg--) {
4784 if (upl_page_present(pl, last_pg))
4785 break;
4786 }
4787 pages_in_upl = last_pg + 1;
4788
4789 if (pages_in_upl == 0) {
4790 ubc_upl_abort(upl, 0);
4791
4792 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 51)) | DBG_FUNC_END, 1, 2, 0, 0, 0);
4793 return(0);
4794 }
4795
4796 for (last_pg = 0; last_pg < pages_in_upl; ) {
4797 /*
4798 * find the next dirty page in the UPL
4799 * this will become the first page in the
4800 * next I/O to generate
4801 */
4802 for (start_pg = last_pg; start_pg < pages_in_upl; start_pg++) {
4803 if (upl_dirty_page(pl, start_pg))
4804 break;
4805 if (upl_page_present(pl, start_pg))
4806 /*
4807 * RET_ONLY_DIRTY will return non-dirty 'precious' pages
4808 * just release these unchanged since we're not going
4809 * to steal them or change their state
4810 */
4811 ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
4812 }
4813 if (start_pg >= pages_in_upl)
4814 /*
4815 * done... no more dirty pages to push
4816 */
4817 break;
4818 if (start_pg > last_pg)
4819 /*
4820 * skipped over some non-dirty pages
4821 */
4822 size -= ((start_pg - last_pg) * PAGE_SIZE);
4823
4824 /*
4825 * find a range of dirty pages to write
4826 */
4827 for (last_pg = start_pg; last_pg < pages_in_upl; last_pg++) {
4828 if (!upl_dirty_page(pl, last_pg))
4829 break;
4830 }
4831 upl_offset = start_pg * PAGE_SIZE;
4832
4833 io_size = min(size, (last_pg - start_pg) * PAGE_SIZE);
4834
4835 io_flags = CL_THROTTLE | CL_COMMIT | CL_AGE | bflag;
4836
4837 if ( !(flags & IO_SYNC))
4838 io_flags |= CL_ASYNC;
4839
4840 retval = cluster_io(vp, upl, upl_offset, upl_f_offset + upl_offset, io_size,
4841 io_flags, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
4842
4843 if (error == 0 && retval)
4844 error = retval;
4845
4846 size -= io_size;
4847 }
4848 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 51)) | DBG_FUNC_END, 1, 3, 0, 0, 0);
4849
4850 return(error);
4851 }
4852
4853
4854 /*
4855 * sparse_cluster_switch is called with the write behind lock held
4856 */
4857 static void
4858 sparse_cluster_switch(struct cl_writebehind *wbp, vnode_t vp, off_t EOF, int (*callback)(buf_t, void *), void *callback_arg)
4859 {
4860 int cl_index;
4861
4862 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 78)) | DBG_FUNC_START, (int)vp, (int)wbp->cl_scmap, wbp->cl_scdirty, 0, 0);
4863
4864 if (wbp->cl_scmap == NULL)
4865 wbp->cl_scdirty = 0;
4866
4867 for (cl_index = 0; cl_index < wbp->cl_number; cl_index++) {
4868 int flags;
4869 struct cl_extent cl;
4870
4871 for (cl.b_addr = wbp->cl_clusters[cl_index].b_addr; cl.b_addr < wbp->cl_clusters[cl_index].e_addr; cl.b_addr++) {
4872
4873 if (ubc_page_op(vp, (off_t)(cl.b_addr * PAGE_SIZE_64), 0, NULL, &flags) == KERN_SUCCESS) {
4874 if (flags & UPL_POP_DIRTY) {
4875 cl.e_addr = cl.b_addr + 1;
4876
4877 sparse_cluster_add(wbp, vp, &cl, EOF, callback, callback_arg);
4878 }
4879 }
4880 }
4881 }
4882 wbp->cl_number = 0;
4883
4884 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 78)) | DBG_FUNC_END, (int)vp, (int)wbp->cl_scmap, wbp->cl_scdirty, 0, 0);
4885 }
4886
4887
4888 /*
4889 * sparse_cluster_push is called with the write behind lock held
4890 */
4891 static void
4892 sparse_cluster_push(struct cl_writebehind *wbp, vnode_t vp, off_t EOF, int push_flag, int (*callback)(buf_t, void *), void *callback_arg)
4893 {
4894 struct cl_extent cl;
4895 off_t offset;
4896 u_int length;
4897
4898 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 79)) | DBG_FUNC_START, (int)vp, (int)wbp->cl_scmap, wbp->cl_scdirty, push_flag, 0);
4899
4900 if (push_flag & PUSH_ALL)
4901 vfs_drt_control(&(wbp->cl_scmap), 1);
4902
4903 for (;;) {
4904 if (vfs_drt_get_cluster(&(wbp->cl_scmap), &offset, &length) != KERN_SUCCESS)
4905 break;
4906
4907 cl.b_addr = (daddr64_t)(offset / PAGE_SIZE_64);
4908 cl.e_addr = (daddr64_t)((offset + length) / PAGE_SIZE_64);
4909
4910 wbp->cl_scdirty -= (int)(cl.e_addr - cl.b_addr);
4911
4912 cluster_push_now(vp, &cl, EOF, push_flag & IO_PASSIVE, callback, callback_arg);
4913
4914 if ( !(push_flag & PUSH_ALL) )
4915 break;
4916 }
4917 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 79)) | DBG_FUNC_END, (int)vp, (int)wbp->cl_scmap, wbp->cl_scdirty, 0, 0);
4918 }
4919
4920
4921 /*
4922 * sparse_cluster_add is called with the write behind lock held
4923 */
4924 static void
4925 sparse_cluster_add(struct cl_writebehind *wbp, vnode_t vp, struct cl_extent *cl, off_t EOF, int (*callback)(buf_t, void *), void *callback_arg)
4926 {
4927 u_int new_dirty;
4928 u_int length;
4929 off_t offset;
4930
4931 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 80)) | DBG_FUNC_START, (int)wbp->cl_scmap, wbp->cl_scdirty, (int)cl->b_addr, (int)cl->e_addr, 0);
4932
4933 offset = (off_t)(cl->b_addr * PAGE_SIZE_64);
4934 length = ((u_int)(cl->e_addr - cl->b_addr)) * PAGE_SIZE;
4935
4936 while (vfs_drt_mark_pages(&(wbp->cl_scmap), offset, length, &new_dirty) != KERN_SUCCESS) {
4937 /*
4938 * no room left in the map
4939 * only a partial update was done
4940 * push out some pages and try again
4941 */
4942 wbp->cl_scdirty += new_dirty;
4943
4944 sparse_cluster_push(wbp, vp, EOF, 0, callback, callback_arg);
4945
4946 offset += (new_dirty * PAGE_SIZE_64);
4947 length -= (new_dirty * PAGE_SIZE);
4948 }
4949 wbp->cl_scdirty += new_dirty;
4950
4951 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 80)) | DBG_FUNC_END, (int)vp, (int)wbp->cl_scmap, wbp->cl_scdirty, 0, 0);
4952 }
4953
4954
4955 static int
4956 cluster_align_phys_io(vnode_t vp, struct uio *uio, addr64_t usr_paddr, u_int32_t xsize, int flags, int (*callback)(buf_t, void *), void *callback_arg)
4957 {
4958 upl_page_info_t *pl;
4959 upl_t upl;
4960 addr64_t ubc_paddr;
4961 kern_return_t kret;
4962 int error = 0;
4963 int did_read = 0;
4964 int abort_flags;
4965 int upl_flags;
4966 int bflag;
4967
4968 if (flags & IO_PASSIVE)
4969 bflag = CL_PASSIVE;
4970 else
4971 bflag = 0;
4972
4973 upl_flags = UPL_SET_LITE;
4974
4975 if ( !(flags & CL_READ) ) {
4976 /*
4977 * "write" operation: let the UPL subsystem know
4978 * that we intend to modify the buffer cache pages
4979 * we're gathering.
4980 */
4981 upl_flags |= UPL_WILL_MODIFY;
4982 } else {
4983 /*
4984 * indicate that there is no need to pull the
4985 * mapping for this page... we're only going
4986 * to read from it, not modify it.
4987 */
4988 upl_flags |= UPL_FILE_IO;
4989 }
4990 kret = ubc_create_upl(vp,
4991 uio->uio_offset & ~PAGE_MASK_64,
4992 PAGE_SIZE,
4993 &upl,
4994 &pl,
4995 upl_flags);
4996
4997 if (kret != KERN_SUCCESS)
4998 return(EINVAL);
4999
5000 if (!upl_valid_page(pl, 0)) {
5001 /*
5002 * issue a synchronous read to cluster_io
5003 */
5004 error = cluster_io(vp, upl, 0, uio->uio_offset & ~PAGE_MASK_64, PAGE_SIZE,
5005 CL_READ | bflag, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
5006 if (error) {
5007 ubc_upl_abort_range(upl, 0, PAGE_SIZE, UPL_ABORT_DUMP_PAGES | UPL_ABORT_FREE_ON_EMPTY);
5008
5009 return(error);
5010 }
5011 did_read = 1;
5012 }
5013 ubc_paddr = ((addr64_t)upl_phys_page(pl, 0) << 12) + (addr64_t)(uio->uio_offset & PAGE_MASK_64);
5014
5015 /*
5016 * NOTE: There is no prototype for the following in BSD. It, and the definitions
5017 * of the defines for cppvPsrc, cppvPsnk, cppvFsnk, and cppvFsrc will be found in
5018 * osfmk/ppc/mappings.h. They are not included here because there appears to be no
5019 * way to do so without exporting them to kexts as well.
5020 */
5021 if (flags & CL_READ)
5022 // copypv(ubc_paddr, usr_paddr, xsize, cppvPsrc | cppvPsnk | cppvFsnk); /* Copy physical to physical and flush the destination */
5023 copypv(ubc_paddr, usr_paddr, xsize, 2 | 1 | 4); /* Copy physical to physical and flush the destination */
5024 else
5025 // copypv(usr_paddr, ubc_paddr, xsize, cppvPsrc | cppvPsnk | cppvFsrc); /* Copy physical to physical and flush the source */
5026 copypv(usr_paddr, ubc_paddr, xsize, 2 | 1 | 8); /* Copy physical to physical and flush the source */
5027
5028 if ( !(flags & CL_READ) || (upl_valid_page(pl, 0) && upl_dirty_page(pl, 0))) {
5029 /*
5030 * issue a synchronous write to cluster_io
5031 */
5032 error = cluster_io(vp, upl, 0, uio->uio_offset & ~PAGE_MASK_64, PAGE_SIZE,
5033 bflag, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
5034 }
5035 if (error == 0)
5036 uio_update(uio, (user_size_t)xsize);
5037
5038 if (did_read)
5039 abort_flags = UPL_ABORT_FREE_ON_EMPTY;
5040 else
5041 abort_flags = UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_DUMP_PAGES;
5042
5043 ubc_upl_abort_range(upl, 0, PAGE_SIZE, abort_flags);
5044
5045 return (error);
5046 }
5047
5048
5049
5050 int
5051 cluster_copy_upl_data(struct uio *uio, upl_t upl, int upl_offset, int *io_resid)
5052 {
5053 int pg_offset;
5054 int pg_index;
5055 int csize;
5056 int segflg;
5057 int retval = 0;
5058 int xsize;
5059 upl_page_info_t *pl;
5060
5061 xsize = *io_resid;
5062
5063 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 34)) | DBG_FUNC_START,
5064 (int)uio->uio_offset, upl_offset, xsize, 0, 0);
5065
5066 segflg = uio->uio_segflg;
5067
5068 switch(segflg) {
5069
5070 case UIO_USERSPACE32:
5071 case UIO_USERISPACE32:
5072 uio->uio_segflg = UIO_PHYS_USERSPACE32;
5073 break;
5074
5075 case UIO_USERSPACE:
5076 case UIO_USERISPACE:
5077 uio->uio_segflg = UIO_PHYS_USERSPACE;
5078 break;
5079
5080 case UIO_USERSPACE64:
5081 case UIO_USERISPACE64:
5082 uio->uio_segflg = UIO_PHYS_USERSPACE64;
5083 break;
5084
5085 case UIO_SYSSPACE32:
5086 uio->uio_segflg = UIO_PHYS_SYSSPACE32;
5087 break;
5088
5089 case UIO_SYSSPACE:
5090 uio->uio_segflg = UIO_PHYS_SYSSPACE;
5091 break;
5092
5093 case UIO_SYSSPACE64:
5094 uio->uio_segflg = UIO_PHYS_SYSSPACE64;
5095 break;
5096 }
5097 pl = ubc_upl_pageinfo(upl);
5098
5099 pg_index = upl_offset / PAGE_SIZE;
5100 pg_offset = upl_offset & PAGE_MASK;
5101 csize = min(PAGE_SIZE - pg_offset, xsize);
5102
5103 while (xsize && retval == 0) {
5104 addr64_t paddr;
5105
5106 paddr = ((addr64_t)upl_phys_page(pl, pg_index) << 12) + pg_offset;
5107
5108 retval = uiomove64(paddr, csize, uio);
5109
5110 pg_index += 1;
5111 pg_offset = 0;
5112 xsize -= csize;
5113 csize = min(PAGE_SIZE, xsize);
5114 }
5115 *io_resid = xsize;
5116
5117 uio->uio_segflg = segflg;
5118
5119 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 34)) | DBG_FUNC_END,
5120 (int)uio->uio_offset, xsize, retval, segflg, 0);
5121
5122 return (retval);
5123 }
5124
5125
5126 int
5127 cluster_copy_ubc_data(vnode_t vp, struct uio *uio, int *io_resid, int mark_dirty)
5128 {
5129
5130 return (cluster_copy_ubc_data_internal(vp, uio, io_resid, mark_dirty, 1));
5131 }
5132
5133
5134 static int
5135 cluster_copy_ubc_data_internal(vnode_t vp, struct uio *uio, int *io_resid, int mark_dirty, int take_reference)
5136 {
5137 int segflg;
5138 int io_size;
5139 int xsize;
5140 int start_offset;
5141 int retval = 0;
5142 memory_object_control_t control;
5143
5144 io_size = *io_resid;
5145
5146 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 34)) | DBG_FUNC_START,
5147 (int)uio->uio_offset, 0, io_size, 0, 0);
5148
5149 control = ubc_getobject(vp, UBC_FLAGS_NONE);
5150
5151 if (control == MEMORY_OBJECT_CONTROL_NULL) {
5152 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 34)) | DBG_FUNC_END,
5153 (int)uio->uio_offset, io_size, retval, 3, 0);
5154
5155 return(0);
5156 }
5157 segflg = uio->uio_segflg;
5158
5159 switch(segflg) {
5160
5161 case UIO_USERSPACE32:
5162 case UIO_USERISPACE32:
5163 uio->uio_segflg = UIO_PHYS_USERSPACE32;
5164 break;
5165
5166 case UIO_USERSPACE64:
5167 case UIO_USERISPACE64:
5168 uio->uio_segflg = UIO_PHYS_USERSPACE64;
5169 break;
5170
5171 case UIO_SYSSPACE32:
5172 uio->uio_segflg = UIO_PHYS_SYSSPACE32;
5173 break;
5174
5175 case UIO_SYSSPACE64:
5176 uio->uio_segflg = UIO_PHYS_SYSSPACE64;
5177 break;
5178
5179 case UIO_USERSPACE:
5180 case UIO_USERISPACE:
5181 uio->uio_segflg = UIO_PHYS_USERSPACE;
5182 break;
5183
5184 case UIO_SYSSPACE:
5185 uio->uio_segflg = UIO_PHYS_SYSSPACE;
5186 break;
5187 }
5188
5189 if ( (io_size = *io_resid) ) {
5190 start_offset = (int)(uio->uio_offset & PAGE_MASK_64);
5191 xsize = uio_resid(uio);
5192
5193 retval = memory_object_control_uiomove(control, uio->uio_offset - start_offset, uio,
5194 start_offset, io_size, mark_dirty, take_reference);
5195 xsize -= uio_resid(uio);
5196 io_size -= xsize;
5197 }
5198 uio->uio_segflg = segflg;
5199 *io_resid = io_size;
5200
5201 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 34)) | DBG_FUNC_END,
5202 (int)uio->uio_offset, io_size, retval, 0x80000000 | segflg, 0);
5203
5204 return(retval);
5205 }
5206
5207
5208 int
5209 is_file_clean(vnode_t vp, off_t filesize)
5210 {
5211 off_t f_offset;
5212 int flags;
5213 int total_dirty = 0;
5214
5215 for (f_offset = 0; f_offset < filesize; f_offset += PAGE_SIZE_64) {
5216 if (ubc_page_op(vp, f_offset, 0, NULL, &flags) == KERN_SUCCESS) {
5217 if (flags & UPL_POP_DIRTY) {
5218 total_dirty++;
5219 }
5220 }
5221 }
5222 if (total_dirty)
5223 return(EINVAL);
5224
5225 return (0);
5226 }
5227
5228
5229
5230 /*
5231 * Dirty region tracking/clustering mechanism.
5232 *
5233 * This code (vfs_drt_*) provides a mechanism for tracking and clustering
5234 * dirty regions within a larger space (file). It is primarily intended to
5235 * support clustering in large files with many dirty areas.
5236 *
5237 * The implementation assumes that the dirty regions are pages.
5238 *
5239 * To represent dirty pages within the file, we store bit vectors in a
5240 * variable-size circular hash.
5241 */
5242
5243 /*
5244 * Bitvector size. This determines the number of pages we group in a
5245 * single hashtable entry. Each hashtable entry is aligned to this
5246 * size within the file.
5247 */
5248 #define DRT_BITVECTOR_PAGES 256
5249
5250 /*
5251 * File offset handling.
5252 *
5253 * DRT_ADDRESS_MASK is dependent on DRT_BITVECTOR_PAGES;
5254 * the correct formula is (~(DRT_BITVECTOR_PAGES * PAGE_SIZE) - 1)
5255 */
5256 #define DRT_ADDRESS_MASK (~((1 << 20) - 1))
5257 #define DRT_ALIGN_ADDRESS(addr) ((addr) & DRT_ADDRESS_MASK)
5258
5259 /*
5260 * Hashtable address field handling.
5261 *
5262 * The low-order bits of the hashtable address are used to conserve
5263 * space.
5264 *
5265 * DRT_HASH_COUNT_MASK must be large enough to store the range
5266 * 0-DRT_BITVECTOR_PAGES inclusive, as well as have one value
5267 * to indicate that the bucket is actually unoccupied.
5268 */
5269 #define DRT_HASH_GET_ADDRESS(scm, i) ((scm)->scm_hashtable[(i)].dhe_control & DRT_ADDRESS_MASK)
5270 #define DRT_HASH_SET_ADDRESS(scm, i, a) \
5271 do { \
5272 (scm)->scm_hashtable[(i)].dhe_control = \
5273 ((scm)->scm_hashtable[(i)].dhe_control & ~DRT_ADDRESS_MASK) | DRT_ALIGN_ADDRESS(a); \
5274 } while (0)
5275 #define DRT_HASH_COUNT_MASK 0x1ff
5276 #define DRT_HASH_GET_COUNT(scm, i) ((scm)->scm_hashtable[(i)].dhe_control & DRT_HASH_COUNT_MASK)
5277 #define DRT_HASH_SET_COUNT(scm, i, c) \
5278 do { \
5279 (scm)->scm_hashtable[(i)].dhe_control = \
5280 ((scm)->scm_hashtable[(i)].dhe_control & ~DRT_HASH_COUNT_MASK) | ((c) & DRT_HASH_COUNT_MASK); \
5281 } while (0)
5282 #define DRT_HASH_CLEAR(scm, i) \
5283 do { \
5284 (scm)->scm_hashtable[(i)].dhe_control = 0; \
5285 } while (0)
5286 #define DRT_HASH_VACATE(scm, i) DRT_HASH_SET_COUNT((scm), (i), DRT_HASH_COUNT_MASK)
5287 #define DRT_HASH_VACANT(scm, i) (DRT_HASH_GET_COUNT((scm), (i)) == DRT_HASH_COUNT_MASK)
5288 #define DRT_HASH_COPY(oscm, oi, scm, i) \
5289 do { \
5290 (scm)->scm_hashtable[(i)].dhe_control = (oscm)->scm_hashtable[(oi)].dhe_control; \
5291 DRT_BITVECTOR_COPY(oscm, oi, scm, i); \
5292 } while(0);
5293
5294
5295 /*
5296 * Hash table moduli.
5297 *
5298 * Since the hashtable entry's size is dependent on the size of
5299 * the bitvector, and since the hashtable size is constrained to
5300 * both being prime and fitting within the desired allocation
5301 * size, these values need to be manually determined.
5302 *
5303 * For DRT_BITVECTOR_SIZE = 256, the entry size is 40 bytes.
5304 *
5305 * The small hashtable allocation is 1024 bytes, so the modulus is 23.
5306 * The large hashtable allocation is 16384 bytes, so the modulus is 401.
5307 */
5308 #define DRT_HASH_SMALL_MODULUS 23
5309 #define DRT_HASH_LARGE_MODULUS 401
5310
5311 #define DRT_SMALL_ALLOCATION 1024 /* 104 bytes spare */
5312 #define DRT_LARGE_ALLOCATION 16384 /* 344 bytes spare */
5313
5314 /* *** nothing below here has secret dependencies on DRT_BITVECTOR_PAGES *** */
5315
5316 /*
5317 * Hashtable bitvector handling.
5318 *
5319 * Bitvector fields are 32 bits long.
5320 */
5321
5322 #define DRT_HASH_SET_BIT(scm, i, bit) \
5323 (scm)->scm_hashtable[(i)].dhe_bitvector[(bit) / 32] |= (1 << ((bit) % 32))
5324
5325 #define DRT_HASH_CLEAR_BIT(scm, i, bit) \
5326 (scm)->scm_hashtable[(i)].dhe_bitvector[(bit) / 32] &= ~(1 << ((bit) % 32))
5327
5328 #define DRT_HASH_TEST_BIT(scm, i, bit) \
5329 ((scm)->scm_hashtable[(i)].dhe_bitvector[(bit) / 32] & (1 << ((bit) % 32)))
5330
5331 #define DRT_BITVECTOR_CLEAR(scm, i) \
5332 bzero(&(scm)->scm_hashtable[(i)].dhe_bitvector[0], (DRT_BITVECTOR_PAGES / 32) * sizeof(u_int32_t))
5333
5334 #define DRT_BITVECTOR_COPY(oscm, oi, scm, i) \
5335 bcopy(&(oscm)->scm_hashtable[(oi)].dhe_bitvector[0], \
5336 &(scm)->scm_hashtable[(i)].dhe_bitvector[0], \
5337 (DRT_BITVECTOR_PAGES / 32) * sizeof(u_int32_t))
5338
5339
5340
5341 /*
5342 * Hashtable entry.
5343 */
5344 struct vfs_drt_hashentry {
5345 u_int64_t dhe_control;
5346 u_int32_t dhe_bitvector[DRT_BITVECTOR_PAGES / 32];
5347 };
5348
5349 /*
5350 * Dirty Region Tracking structure.
5351 *
5352 * The hashtable is allocated entirely inside the DRT structure.
5353 *
5354 * The hash is a simple circular prime modulus arrangement, the structure
5355 * is resized from small to large if it overflows.
5356 */
5357
5358 struct vfs_drt_clustermap {
5359 u_int32_t scm_magic; /* sanity/detection */
5360 #define DRT_SCM_MAGIC 0x12020003
5361 u_int32_t scm_modulus; /* current ring size */
5362 u_int32_t scm_buckets; /* number of occupied buckets */
5363 u_int32_t scm_lastclean; /* last entry we cleaned */
5364 u_int32_t scm_iskips; /* number of slot skips */
5365
5366 struct vfs_drt_hashentry scm_hashtable[0];
5367 };
5368
5369
5370 #define DRT_HASH(scm, addr) ((addr) % (scm)->scm_modulus)
5371 #define DRT_HASH_NEXT(scm, addr) (((addr) + 1) % (scm)->scm_modulus)
5372
5373 /*
5374 * Debugging codes and arguments.
5375 */
5376 #define DRT_DEBUG_EMPTYFREE (FSDBG_CODE(DBG_FSRW, 82)) /* nil */
5377 #define DRT_DEBUG_RETCLUSTER (FSDBG_CODE(DBG_FSRW, 83)) /* offset, length */
5378 #define DRT_DEBUG_ALLOC (FSDBG_CODE(DBG_FSRW, 84)) /* copycount */
5379 #define DRT_DEBUG_INSERT (FSDBG_CODE(DBG_FSRW, 85)) /* offset, iskip */
5380 #define DRT_DEBUG_MARK (FSDBG_CODE(DBG_FSRW, 86)) /* offset, length,
5381 * dirty */
5382 /* 0, setcount */
5383 /* 1 (clean, no map) */
5384 /* 2 (map alloc fail) */
5385 /* 3, resid (partial) */
5386 #define DRT_DEBUG_6 (FSDBG_CODE(DBG_FSRW, 87))
5387 #define DRT_DEBUG_SCMDATA (FSDBG_CODE(DBG_FSRW, 88)) /* modulus, buckets,
5388 * lastclean, iskips */
5389
5390
5391 static kern_return_t vfs_drt_alloc_map(struct vfs_drt_clustermap **cmapp);
5392 static kern_return_t vfs_drt_free_map(struct vfs_drt_clustermap *cmap);
5393 static kern_return_t vfs_drt_search_index(struct vfs_drt_clustermap *cmap,
5394 u_int64_t offset, int *indexp);
5395 static kern_return_t vfs_drt_get_index(struct vfs_drt_clustermap **cmapp,
5396 u_int64_t offset,
5397 int *indexp,
5398 int recursed);
5399 static kern_return_t vfs_drt_do_mark_pages(
5400 void **cmapp,
5401 u_int64_t offset,
5402 u_int length,
5403 u_int *setcountp,
5404 int dirty);
5405 static void vfs_drt_trace(
5406 struct vfs_drt_clustermap *cmap,
5407 int code,
5408 int arg1,
5409 int arg2,
5410 int arg3,
5411 int arg4);
5412
5413
5414 /*
5415 * Allocate and initialise a sparse cluster map.
5416 *
5417 * Will allocate a new map, resize or compact an existing map.
5418 *
5419 * XXX we should probably have at least one intermediate map size,
5420 * as the 1:16 ratio seems a bit drastic.
5421 */
5422 static kern_return_t
5423 vfs_drt_alloc_map(struct vfs_drt_clustermap **cmapp)
5424 {
5425 struct vfs_drt_clustermap *cmap, *ocmap;
5426 kern_return_t kret;
5427 u_int64_t offset;
5428 u_int32_t i;
5429 int nsize, active_buckets, index, copycount;
5430
5431 ocmap = NULL;
5432 if (cmapp != NULL)
5433 ocmap = *cmapp;
5434
5435 /*
5436 * Decide on the size of the new map.
5437 */
5438 if (ocmap == NULL) {
5439 nsize = DRT_HASH_SMALL_MODULUS;
5440 } else {
5441 /* count the number of active buckets in the old map */
5442 active_buckets = 0;
5443 for (i = 0; i < ocmap->scm_modulus; i++) {
5444 if (!DRT_HASH_VACANT(ocmap, i) &&
5445 (DRT_HASH_GET_COUNT(ocmap, i) != 0))
5446 active_buckets++;
5447 }
5448 /*
5449 * If we're currently using the small allocation, check to
5450 * see whether we should grow to the large one.
5451 */
5452 if (ocmap->scm_modulus == DRT_HASH_SMALL_MODULUS) {
5453 /* if the ring is nearly full */
5454 if (active_buckets > (DRT_HASH_SMALL_MODULUS - 5)) {
5455 nsize = DRT_HASH_LARGE_MODULUS;
5456 } else {
5457 nsize = DRT_HASH_SMALL_MODULUS;
5458 }
5459 } else {
5460 /* already using the large modulus */
5461 nsize = DRT_HASH_LARGE_MODULUS;
5462 /*
5463 * If the ring is completely full, there's
5464 * nothing useful for us to do. Behave as
5465 * though we had compacted into the new
5466 * array and return.
5467 */
5468 if (active_buckets >= DRT_HASH_LARGE_MODULUS)
5469 return(KERN_SUCCESS);
5470 }
5471 }
5472
5473 /*
5474 * Allocate and initialise the new map.
5475 */
5476
5477 kret = kmem_alloc(kernel_map, (vm_offset_t *)&cmap,
5478 (nsize == DRT_HASH_SMALL_MODULUS) ? DRT_SMALL_ALLOCATION : DRT_LARGE_ALLOCATION);
5479 if (kret != KERN_SUCCESS)
5480 return(kret);
5481 cmap->scm_magic = DRT_SCM_MAGIC;
5482 cmap->scm_modulus = nsize;
5483 cmap->scm_buckets = 0;
5484 cmap->scm_lastclean = 0;
5485 cmap->scm_iskips = 0;
5486 for (i = 0; i < cmap->scm_modulus; i++) {
5487 DRT_HASH_CLEAR(cmap, i);
5488 DRT_HASH_VACATE(cmap, i);
5489 DRT_BITVECTOR_CLEAR(cmap, i);
5490 }
5491
5492 /*
5493 * If there's an old map, re-hash entries from it into the new map.
5494 */
5495 copycount = 0;
5496 if (ocmap != NULL) {
5497 for (i = 0; i < ocmap->scm_modulus; i++) {
5498 /* skip empty buckets */
5499 if (DRT_HASH_VACANT(ocmap, i) ||
5500 (DRT_HASH_GET_COUNT(ocmap, i) == 0))
5501 continue;
5502 /* get new index */
5503 offset = DRT_HASH_GET_ADDRESS(ocmap, i);
5504 kret = vfs_drt_get_index(&cmap, offset, &index, 1);
5505 if (kret != KERN_SUCCESS) {
5506 /* XXX need to bail out gracefully here */
5507 panic("vfs_drt: new cluster map mysteriously too small");
5508 index = 0;
5509 }
5510 /* copy */
5511 DRT_HASH_COPY(ocmap, i, cmap, index);
5512 copycount++;
5513 }
5514 }
5515
5516 /* log what we've done */
5517 vfs_drt_trace(cmap, DRT_DEBUG_ALLOC, copycount, 0, 0, 0);
5518
5519 /*
5520 * It's important to ensure that *cmapp always points to
5521 * a valid map, so we must overwrite it before freeing
5522 * the old map.
5523 */
5524 *cmapp = cmap;
5525 if (ocmap != NULL) {
5526 /* emit stats into trace buffer */
5527 vfs_drt_trace(ocmap, DRT_DEBUG_SCMDATA,
5528 ocmap->scm_modulus,
5529 ocmap->scm_buckets,
5530 ocmap->scm_lastclean,
5531 ocmap->scm_iskips);
5532
5533 vfs_drt_free_map(ocmap);
5534 }
5535 return(KERN_SUCCESS);
5536 }
5537
5538
5539 /*
5540 * Free a sparse cluster map.
5541 */
5542 static kern_return_t
5543 vfs_drt_free_map(struct vfs_drt_clustermap *cmap)
5544 {
5545 kmem_free(kernel_map, (vm_offset_t)cmap,
5546 (cmap->scm_modulus == DRT_HASH_SMALL_MODULUS) ? DRT_SMALL_ALLOCATION : DRT_LARGE_ALLOCATION);
5547 return(KERN_SUCCESS);
5548 }
5549
5550
5551 /*
5552 * Find the hashtable slot currently occupied by an entry for the supplied offset.
5553 */
5554 static kern_return_t
5555 vfs_drt_search_index(struct vfs_drt_clustermap *cmap, u_int64_t offset, int *indexp)
5556 {
5557 int index;
5558 u_int32_t i;
5559
5560 offset = DRT_ALIGN_ADDRESS(offset);
5561 index = DRT_HASH(cmap, offset);
5562
5563 /* traverse the hashtable */
5564 for (i = 0; i < cmap->scm_modulus; i++) {
5565
5566 /*
5567 * If the slot is vacant, we can stop.
5568 */
5569 if (DRT_HASH_VACANT(cmap, index))
5570 break;
5571
5572 /*
5573 * If the address matches our offset, we have success.
5574 */
5575 if (DRT_HASH_GET_ADDRESS(cmap, index) == offset) {
5576 *indexp = index;
5577 return(KERN_SUCCESS);
5578 }
5579
5580 /*
5581 * Move to the next slot, try again.
5582 */
5583 index = DRT_HASH_NEXT(cmap, index);
5584 }
5585 /*
5586 * It's not there.
5587 */
5588 return(KERN_FAILURE);
5589 }
5590
5591 /*
5592 * Find the hashtable slot for the supplied offset. If we haven't allocated
5593 * one yet, allocate one and populate the address field. Note that it will
5594 * not have a nonzero page count and thus will still technically be free, so
5595 * in the case where we are called to clean pages, the slot will remain free.
5596 */
5597 static kern_return_t
5598 vfs_drt_get_index(struct vfs_drt_clustermap **cmapp, u_int64_t offset, int *indexp, int recursed)
5599 {
5600 struct vfs_drt_clustermap *cmap;
5601 kern_return_t kret;
5602 u_int32_t index;
5603 u_int32_t i;
5604
5605 cmap = *cmapp;
5606
5607 /* look for an existing entry */
5608 kret = vfs_drt_search_index(cmap, offset, indexp);
5609 if (kret == KERN_SUCCESS)
5610 return(kret);
5611
5612 /* need to allocate an entry */
5613 offset = DRT_ALIGN_ADDRESS(offset);
5614 index = DRT_HASH(cmap, offset);
5615
5616 /* scan from the index forwards looking for a vacant slot */
5617 for (i = 0; i < cmap->scm_modulus; i++) {
5618 /* slot vacant? */
5619 if (DRT_HASH_VACANT(cmap, index) || DRT_HASH_GET_COUNT(cmap,index) == 0) {
5620 cmap->scm_buckets++;
5621 if (index < cmap->scm_lastclean)
5622 cmap->scm_lastclean = index;
5623 DRT_HASH_SET_ADDRESS(cmap, index, offset);
5624 DRT_HASH_SET_COUNT(cmap, index, 0);
5625 DRT_BITVECTOR_CLEAR(cmap, index);
5626 *indexp = index;
5627 vfs_drt_trace(cmap, DRT_DEBUG_INSERT, (int)offset, i, 0, 0);
5628 return(KERN_SUCCESS);
5629 }
5630 cmap->scm_iskips += i;
5631 index = DRT_HASH_NEXT(cmap, index);
5632 }
5633
5634 /*
5635 * We haven't found a vacant slot, so the map is full. If we're not
5636 * already recursed, try reallocating/compacting it.
5637 */
5638 if (recursed)
5639 return(KERN_FAILURE);
5640 kret = vfs_drt_alloc_map(cmapp);
5641 if (kret == KERN_SUCCESS) {
5642 /* now try to insert again */
5643 kret = vfs_drt_get_index(cmapp, offset, indexp, 1);
5644 }
5645 return(kret);
5646 }
5647
5648 /*
5649 * Implementation of set dirty/clean.
5650 *
5651 * In the 'clean' case, not finding a map is OK.
5652 */
5653 static kern_return_t
5654 vfs_drt_do_mark_pages(
5655 void **private,
5656 u_int64_t offset,
5657 u_int length,
5658 u_int *setcountp,
5659 int dirty)
5660 {
5661 struct vfs_drt_clustermap *cmap, **cmapp;
5662 kern_return_t kret;
5663 int i, index, pgoff, pgcount, setcount, ecount;
5664
5665 cmapp = (struct vfs_drt_clustermap **)private;
5666 cmap = *cmapp;
5667
5668 vfs_drt_trace(cmap, DRT_DEBUG_MARK | DBG_FUNC_START, (int)offset, (int)length, dirty, 0);
5669
5670 if (setcountp != NULL)
5671 *setcountp = 0;
5672
5673 /* allocate a cluster map if we don't already have one */
5674 if (cmap == NULL) {
5675 /* no cluster map, nothing to clean */
5676 if (!dirty) {
5677 vfs_drt_trace(cmap, DRT_DEBUG_MARK | DBG_FUNC_END, 1, 0, 0, 0);
5678 return(KERN_SUCCESS);
5679 }
5680 kret = vfs_drt_alloc_map(cmapp);
5681 if (kret != KERN_SUCCESS) {
5682 vfs_drt_trace(cmap, DRT_DEBUG_MARK | DBG_FUNC_END, 2, 0, 0, 0);
5683 return(kret);
5684 }
5685 }
5686 setcount = 0;
5687
5688 /*
5689 * Iterate over the length of the region.
5690 */
5691 while (length > 0) {
5692 /*
5693 * Get the hashtable index for this offset.
5694 *
5695 * XXX this will add blank entries if we are clearing a range
5696 * that hasn't been dirtied.
5697 */
5698 kret = vfs_drt_get_index(cmapp, offset, &index, 0);
5699 cmap = *cmapp; /* may have changed! */
5700 /* this may be a partial-success return */
5701 if (kret != KERN_SUCCESS) {
5702 if (setcountp != NULL)
5703 *setcountp = setcount;
5704 vfs_drt_trace(cmap, DRT_DEBUG_MARK | DBG_FUNC_END, 3, (int)length, 0, 0);
5705
5706 return(kret);
5707 }
5708
5709 /*
5710 * Work out how many pages we're modifying in this
5711 * hashtable entry.
5712 */
5713 pgoff = (offset - DRT_ALIGN_ADDRESS(offset)) / PAGE_SIZE;
5714 pgcount = min((length / PAGE_SIZE), (DRT_BITVECTOR_PAGES - pgoff));
5715
5716 /*
5717 * Iterate over pages, dirty/clearing as we go.
5718 */
5719 ecount = DRT_HASH_GET_COUNT(cmap, index);
5720 for (i = 0; i < pgcount; i++) {
5721 if (dirty) {
5722 if (!DRT_HASH_TEST_BIT(cmap, index, pgoff + i)) {
5723 DRT_HASH_SET_BIT(cmap, index, pgoff + i);
5724 ecount++;
5725 setcount++;
5726 }
5727 } else {
5728 if (DRT_HASH_TEST_BIT(cmap, index, pgoff + i)) {
5729 DRT_HASH_CLEAR_BIT(cmap, index, pgoff + i);
5730 ecount--;
5731 setcount++;
5732 }
5733 }
5734 }
5735 DRT_HASH_SET_COUNT(cmap, index, ecount);
5736
5737 offset += pgcount * PAGE_SIZE;
5738 length -= pgcount * PAGE_SIZE;
5739 }
5740 if (setcountp != NULL)
5741 *setcountp = setcount;
5742
5743 vfs_drt_trace(cmap, DRT_DEBUG_MARK | DBG_FUNC_END, 0, setcount, 0, 0);
5744
5745 return(KERN_SUCCESS);
5746 }
5747
5748 /*
5749 * Mark a set of pages as dirty/clean.
5750 *
5751 * This is a public interface.
5752 *
5753 * cmapp
5754 * Pointer to storage suitable for holding a pointer. Note that
5755 * this must either be NULL or a value set by this function.
5756 *
5757 * size
5758 * Current file size in bytes.
5759 *
5760 * offset
5761 * Offset of the first page to be marked as dirty, in bytes. Must be
5762 * page-aligned.
5763 *
5764 * length
5765 * Length of dirty region, in bytes. Must be a multiple of PAGE_SIZE.
5766 *
5767 * setcountp
5768 * Number of pages newly marked dirty by this call (optional).
5769 *
5770 * Returns KERN_SUCCESS if all the pages were successfully marked.
5771 */
5772 static kern_return_t
5773 vfs_drt_mark_pages(void **cmapp, off_t offset, u_int length, u_int *setcountp)
5774 {
5775 /* XXX size unused, drop from interface */
5776 return(vfs_drt_do_mark_pages(cmapp, offset, length, setcountp, 1));
5777 }
5778
5779 #if 0
5780 static kern_return_t
5781 vfs_drt_unmark_pages(void **cmapp, off_t offset, u_int length)
5782 {
5783 return(vfs_drt_do_mark_pages(cmapp, offset, length, NULL, 0));
5784 }
5785 #endif
5786
5787 /*
5788 * Get a cluster of dirty pages.
5789 *
5790 * This is a public interface.
5791 *
5792 * cmapp
5793 * Pointer to storage managed by drt_mark_pages. Note that this must
5794 * be NULL or a value set by drt_mark_pages.
5795 *
5796 * offsetp
5797 * Returns the byte offset into the file of the first page in the cluster.
5798 *
5799 * lengthp
5800 * Returns the length in bytes of the cluster of dirty pages.
5801 *
5802 * Returns success if a cluster was found. If KERN_FAILURE is returned, there
5803 * are no dirty pages meeting the minmum size criteria. Private storage will
5804 * be released if there are no more dirty pages left in the map
5805 *
5806 */
5807 static kern_return_t
5808 vfs_drt_get_cluster(void **cmapp, off_t *offsetp, u_int *lengthp)
5809 {
5810 struct vfs_drt_clustermap *cmap;
5811 u_int64_t offset;
5812 u_int length;
5813 u_int32_t j;
5814 int index, i, fs, ls;
5815
5816 /* sanity */
5817 if ((cmapp == NULL) || (*cmapp == NULL))
5818 return(KERN_FAILURE);
5819 cmap = *cmapp;
5820
5821 /* walk the hashtable */
5822 for (offset = 0, j = 0; j < cmap->scm_modulus; offset += (DRT_BITVECTOR_PAGES * PAGE_SIZE), j++) {
5823 index = DRT_HASH(cmap, offset);
5824
5825 if (DRT_HASH_VACANT(cmap, index) || (DRT_HASH_GET_COUNT(cmap, index) == 0))
5826 continue;
5827
5828 /* scan the bitfield for a string of bits */
5829 fs = -1;
5830
5831 for (i = 0; i < DRT_BITVECTOR_PAGES; i++) {
5832 if (DRT_HASH_TEST_BIT(cmap, index, i)) {
5833 fs = i;
5834 break;
5835 }
5836 }
5837 if (fs == -1) {
5838 /* didn't find any bits set */
5839 panic("vfs_drt: entry summary count > 0 but no bits set in map");
5840 }
5841 for (ls = 0; i < DRT_BITVECTOR_PAGES; i++, ls++) {
5842 if (!DRT_HASH_TEST_BIT(cmap, index, i))
5843 break;
5844 }
5845
5846 /* compute offset and length, mark pages clean */
5847 offset = DRT_HASH_GET_ADDRESS(cmap, index) + (PAGE_SIZE * fs);
5848 length = ls * PAGE_SIZE;
5849 vfs_drt_do_mark_pages(cmapp, offset, length, NULL, 0);
5850 cmap->scm_lastclean = index;
5851
5852 /* return successful */
5853 *offsetp = (off_t)offset;
5854 *lengthp = length;
5855
5856 vfs_drt_trace(cmap, DRT_DEBUG_RETCLUSTER, (int)offset, (int)length, 0, 0);
5857 return(KERN_SUCCESS);
5858 }
5859 /*
5860 * We didn't find anything... hashtable is empty
5861 * emit stats into trace buffer and
5862 * then free it
5863 */
5864 vfs_drt_trace(cmap, DRT_DEBUG_SCMDATA,
5865 cmap->scm_modulus,
5866 cmap->scm_buckets,
5867 cmap->scm_lastclean,
5868 cmap->scm_iskips);
5869
5870 vfs_drt_free_map(cmap);
5871 *cmapp = NULL;
5872
5873 return(KERN_FAILURE);
5874 }
5875
5876
5877 static kern_return_t
5878 vfs_drt_control(void **cmapp, int op_type)
5879 {
5880 struct vfs_drt_clustermap *cmap;
5881
5882 /* sanity */
5883 if ((cmapp == NULL) || (*cmapp == NULL))
5884 return(KERN_FAILURE);
5885 cmap = *cmapp;
5886
5887 switch (op_type) {
5888 case 0:
5889 /* emit stats into trace buffer */
5890 vfs_drt_trace(cmap, DRT_DEBUG_SCMDATA,
5891 cmap->scm_modulus,
5892 cmap->scm_buckets,
5893 cmap->scm_lastclean,
5894 cmap->scm_iskips);
5895
5896 vfs_drt_free_map(cmap);
5897 *cmapp = NULL;
5898 break;
5899
5900 case 1:
5901 cmap->scm_lastclean = 0;
5902 break;
5903 }
5904 return(KERN_SUCCESS);
5905 }
5906
5907
5908
5909 /*
5910 * Emit a summary of the state of the clustermap into the trace buffer
5911 * along with some caller-provided data.
5912 */
5913 #if KDEBUG
5914 static void
5915 vfs_drt_trace(__unused struct vfs_drt_clustermap *cmap, int code, int arg1, int arg2, int arg3, int arg4)
5916 {
5917 KERNEL_DEBUG(code, arg1, arg2, arg3, arg4, 0);
5918 }
5919 #else
5920 static void
5921 vfs_drt_trace(__unused struct vfs_drt_clustermap *cmap, __unused int code,
5922 __unused int arg1, __unused int arg2, __unused int arg3,
5923 __unused int arg4)
5924 {
5925 }
5926 #endif
5927
5928 #if 0
5929 /*
5930 * Perform basic sanity check on the hash entry summary count
5931 * vs. the actual bits set in the entry.
5932 */
5933 static void
5934 vfs_drt_sanity(struct vfs_drt_clustermap *cmap)
5935 {
5936 int index, i;
5937 int bits_on;
5938
5939 for (index = 0; index < cmap->scm_modulus; index++) {
5940 if (DRT_HASH_VACANT(cmap, index))
5941 continue;
5942
5943 for (bits_on = 0, i = 0; i < DRT_BITVECTOR_PAGES; i++) {
5944 if (DRT_HASH_TEST_BIT(cmap, index, i))
5945 bits_on++;
5946 }
5947 if (bits_on != DRT_HASH_GET_COUNT(cmap, index))
5948 panic("bits_on = %d, index = %d\n", bits_on, index);
5949 }
5950 }
5951 #endif
mirror of XNU