]> git.saurik.com Git - apple/xnu.git/blob - bsd/vfs/vfs_cluster.c
projects / apple / xnu.git / blob
? search:


c3b24c28d8a21812733a7a03935fba1d7eeb3076
[apple/xnu.git] / bsd / vfs / vfs_cluster.c
1 /*
2 * Copyright (c) 2000-2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
29 /*
30 * Copyright (c) 1993
31 * The Regents of the University of California. All rights reserved.
32 *
33 * Redistribution and use in source and binary forms, with or without
34 * modification, are permitted provided that the following conditions
35 * are met:
36 * 1. Redistributions of source code must retain the above copyright
37 * notice, this list of conditions and the following disclaimer.
38 * 2. Redistributions in binary form must reproduce the above copyright
39 * notice, this list of conditions and the following disclaimer in the
40 * documentation and/or other materials provided with the distribution.
41 * 3. All advertising materials mentioning features or use of this software
42 * must display the following acknowledgement:
43 * This product includes software developed by the University of
44 * California, Berkeley and its contributors.
45 * 4. Neither the name of the University nor the names of its contributors
46 * may be used to endorse or promote products derived from this software
47 * without specific prior written permission.
48 *
49 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
50 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
51 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
52 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
53 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
54 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
55 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
56 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
57 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
58 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
59 * SUCH DAMAGE.
60 *
61 * @(#)vfs_cluster.c 8.10 (Berkeley) 3/28/95
62 */
63
64 #include <sys/param.h>
65 #include <sys/proc_internal.h>
66 #include <sys/buf_internal.h>
67 #include <sys/mount_internal.h>
68 #include <sys/vnode_internal.h>
69 #include <sys/trace.h>
70 #include <sys/malloc.h>
71 #include <sys/time.h>
72 #include <sys/kernel.h>
73 #include <sys/resourcevar.h>
74 #include <sys/uio_internal.h>
75 #include <libkern/libkern.h>
76 #include <machine/machine_routines.h>
77
78 #include <sys/ubc_internal.h>
79 #include <vm/vnode_pager.h>
80
81 #include <mach/mach_types.h>
82 #include <mach/memory_object_types.h>
83 #include <mach/vm_map.h>
84 #include <mach/upl.h>
85
86 #include <vm/vm_kern.h>
87 #include <vm/vm_map.h>
88 #include <vm/vm_pageout.h>
89
90 #include <sys/kdebug.h>
91
92 #define CL_READ 0x01
93 #define CL_WRITE 0x02
94 #define CL_ASYNC 0x04
95 #define CL_COMMIT 0x08
96 #define CL_PAGEOUT 0x10
97 #define CL_AGE 0x20
98 #define CL_NOZERO 0x40
99 #define CL_PAGEIN 0x80
100 #define CL_DEV_MEMORY 0x100
101 #define CL_PRESERVE 0x200
102 #define CL_THROTTLE 0x400
103 #define CL_KEEPCACHED 0x800
104 #define CL_DIRECT_IO 0x1000
105 #define CL_PASSIVE 0x2000
106
107
108 struct clios {
109 u_int io_completed; /* amount of io that has currently completed */
110 u_int io_issued; /* amount of io that was successfully issued */
111 int io_error; /* error code of first error encountered */
112 int io_wanted; /* someone is sleeping waiting for a change in state */
113 };
114
115 static lck_grp_t *cl_mtx_grp;
116 static lck_attr_t *cl_mtx_attr;
117 static lck_grp_attr_t *cl_mtx_grp_attr;
118 static lck_mtx_t *cl_mtxp;
119
120
121 #define IO_UNKNOWN 0
122 #define IO_DIRECT 1
123 #define IO_CONTIG 2
124 #define IO_COPY 3
125
126 #define PUSH_DELAY 0x01
127 #define PUSH_ALL 0x02
128 #define PUSH_SYNC 0x04
129
130
131 static void cluster_EOT(buf_t cbp_head, buf_t cbp_tail, int zero_offset);
132 static void cluster_wait_IO(buf_t cbp_head, int async);
133 static void cluster_complete_transaction(buf_t *cbp_head, void *callback_arg, int *retval, int flags, int needwait);
134
135 static int cluster_io_type(struct uio *uio, int *io_type, u_int32_t *io_length, u_int32_t min_length);
136
137 static int cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int non_rounded_size,
138 int flags, buf_t real_bp, struct clios *iostate, int (*)(buf_t, void *), void *callback_arg);
139 static int cluster_iodone(buf_t bp, void *callback_arg);
140 static int cluster_ioerror(upl_t upl, int upl_offset, int abort_size, int error, int io_flags);
141 static int cluster_hard_throttle_on(vnode_t vp);
142
143 static void cluster_syncup(vnode_t vp, off_t newEOF, int (*)(buf_t, void *), void *callback_arg);
144
145 static void cluster_read_upl_release(upl_t upl, int start_pg, int last_pg, int flags);
146 static int cluster_copy_ubc_data_internal(vnode_t vp, struct uio *uio, int *io_resid, int mark_dirty, int take_reference);
147
148 static int cluster_read_copy(vnode_t vp, struct uio *uio, u_int32_t io_req_size, off_t filesize, int flags,
149 int (*)(buf_t, void *), void *callback_arg);
150 static int cluster_read_direct(vnode_t vp, struct uio *uio, off_t filesize, int *read_type, u_int32_t *read_length,
151 int flags, int (*)(buf_t, void *), void *callback_arg);
152 static int cluster_read_contig(vnode_t vp, struct uio *uio, off_t filesize, int *read_type, u_int32_t *read_length,
153 int (*)(buf_t, void *), void *callback_arg, int flags);
154
155 static int cluster_write_copy(vnode_t vp, struct uio *uio, u_int32_t io_req_size, off_t oldEOF, off_t newEOF,
156 off_t headOff, off_t tailOff, int flags, int (*)(buf_t, void *), void *callback_arg);
157 static int cluster_write_direct(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF,
158 int *write_type, u_int32_t *write_length, int flags, int (*)(buf_t, void *), void *callback_arg);
159 static int cluster_write_contig(vnode_t vp, struct uio *uio, off_t newEOF,
160 int *write_type, u_int32_t *write_length, int (*)(buf_t, void *), void *callback_arg, int bflag);
161
162 static int cluster_align_phys_io(vnode_t vp, struct uio *uio, addr64_t usr_paddr, u_int32_t xsize, int flags, int (*)(buf_t, void *), void *callback_arg);
163
164 static int cluster_read_prefetch(vnode_t vp, off_t f_offset, u_int size, off_t filesize, int (*callback)(buf_t, void *), void *callback_arg, int bflag);
165 static void cluster_read_ahead(vnode_t vp, struct cl_extent *extent, off_t filesize, struct cl_readahead *ra, int (*callback)(buf_t, void *), void *callback_arg, int bflag);
166
167 static int cluster_push_now(vnode_t vp, struct cl_extent *, off_t EOF, int flags, int (*)(buf_t, void *), void *callback_arg);
168
169 static int cluster_try_push(struct cl_writebehind *, vnode_t vp, off_t EOF, int push_flag, int (*)(buf_t, void *), void *callback_arg);
170
171 static void sparse_cluster_switch(struct cl_writebehind *, vnode_t vp, off_t EOF, int (*)(buf_t, void *), void *callback_arg);
172 static void sparse_cluster_push(struct cl_writebehind *, vnode_t vp, off_t EOF, int push_flag, int (*)(buf_t, void *), void *callback_arg);
173 static void sparse_cluster_add(struct cl_writebehind *, vnode_t vp, struct cl_extent *, off_t EOF, int (*)(buf_t, void *), void *callback_arg);
174
175 static kern_return_t vfs_drt_mark_pages(void **cmapp, off_t offset, u_int length, u_int *setcountp);
176 static kern_return_t vfs_drt_get_cluster(void **cmapp, off_t *offsetp, u_int *lengthp);
177 static kern_return_t vfs_drt_control(void **cmapp, int op_type);
178
179 int is_file_clean(vnode_t, off_t);
180
181 /*
182 * limit the internal I/O size so that we
183 * can represent it in a 32 bit int
184 */
185 #define MAX_IO_REQUEST_SIZE (1024 * 1024 * 256)
186 #define MAX_IO_CONTIG_SIZE (MAX_UPL_SIZE * PAGE_SIZE)
187 #define MAX_VECTS 16
188 #define MIN_DIRECT_WRITE_SIZE (4 * PAGE_SIZE)
189
190
191 #define MAX_CLUSTER_SIZE(vp) (cluster_max_io_size(vp->v_mount, CL_WRITE))
192 #define MAX_PREFETCH(vp) (cluster_max_io_size(vp->v_mount, CL_READ) * 3);
193
194
195 int speculative_reads_disabled = 0;
196
197 /*
198 * throttle the number of async writes that
199 * can be outstanding on a single vnode
200 * before we issue a synchronous write
201 */
202 #define HARD_THROTTLE_MAXCNT 0
203 #define HARD_THROTTLE_MAXSIZE (64 * 1024)
204
205 int hard_throttle_on_root = 0;
206 struct timeval priority_IO_timestamp_for_root;
207
208
209 void
210 cluster_init(void) {
211 /*
212 * allocate lock group attribute and group
213 */
214 cl_mtx_grp_attr = lck_grp_attr_alloc_init();
215 cl_mtx_grp = lck_grp_alloc_init("cluster I/O", cl_mtx_grp_attr);
216
217 /*
218 * allocate the lock attribute
219 */
220 cl_mtx_attr = lck_attr_alloc_init();
221
222 /*
223 * allocate and initialize mutex's used to protect updates and waits
224 * on the cluster_io context
225 */
226 cl_mtxp = lck_mtx_alloc_init(cl_mtx_grp, cl_mtx_attr);
227
228 if (cl_mtxp == NULL)
229 panic("cluster_init: failed to allocate cl_mtxp");
230 }
231
232
233 uint32_t
234 cluster_max_io_size(mount_t mp, int type)
235 {
236 uint32_t max_io_size;
237 uint32_t segcnt;
238 uint32_t maxcnt;
239
240 switch(type) {
241
242 case CL_READ:
243 segcnt = mp->mnt_segreadcnt;
244 maxcnt = mp->mnt_maxreadcnt;
245 break;
246 case CL_WRITE:
247 segcnt = mp->mnt_segwritecnt;
248 maxcnt = mp->mnt_maxwritecnt;
249 break;
250 default:
251 segcnt = min(mp->mnt_segreadcnt, mp->mnt_segwritecnt);
252 maxcnt = min(mp->mnt_maxreadcnt, mp->mnt_maxwritecnt);
253 break;
254 }
255 if (segcnt > MAX_UPL_SIZE) {
256 /*
257 * don't allow a size beyond the max UPL size we can create
258 */
259 segcnt = MAX_UPL_SIZE;
260 }
261 max_io_size = min((segcnt * PAGE_SIZE), maxcnt);
262
263 if (max_io_size < (MAX_UPL_TRANSFER * PAGE_SIZE)) {
264 /*
265 * don't allow a size smaller than the old fixed limit
266 */
267 max_io_size = (MAX_UPL_TRANSFER * PAGE_SIZE);
268 } else {
269 /*
270 * make sure the size specified is a multiple of PAGE_SIZE
271 */
272 max_io_size &= ~PAGE_MASK;
273 }
274 return (max_io_size);
275 }
276
277
278
279
280 #define CLW_ALLOCATE 0x01
281 #define CLW_RETURNLOCKED 0x02
282 #define CLW_IONOCACHE 0x04
283 #define CLW_IOPASSIVE 0x08
284
285 /*
286 * if the read ahead context doesn't yet exist,
287 * allocate and initialize it...
288 * the vnode lock serializes multiple callers
289 * during the actual assignment... first one
290 * to grab the lock wins... the other callers
291 * will release the now unnecessary storage
292 *
293 * once the context is present, try to grab (but don't block on)
294 * the lock associated with it... if someone
295 * else currently owns it, than the read
296 * will run without read-ahead. this allows
297 * multiple readers to run in parallel and
298 * since there's only 1 read ahead context,
299 * there's no real loss in only allowing 1
300 * reader to have read-ahead enabled.
301 */
302 static struct cl_readahead *
303 cluster_get_rap(vnode_t vp)
304 {
305 struct ubc_info *ubc;
306 struct cl_readahead *rap;
307
308 ubc = vp->v_ubcinfo;
309
310 if ((rap = ubc->cl_rahead) == NULL) {
311 MALLOC_ZONE(rap, struct cl_readahead *, sizeof *rap, M_CLRDAHEAD, M_WAITOK);
312
313 bzero(rap, sizeof *rap);
314 rap->cl_lastr = -1;
315 lck_mtx_init(&rap->cl_lockr, cl_mtx_grp, cl_mtx_attr);
316
317 vnode_lock(vp);
318
319 if (ubc->cl_rahead == NULL)
320 ubc->cl_rahead = rap;
321 else {
322 lck_mtx_destroy(&rap->cl_lockr, cl_mtx_grp);
323 FREE_ZONE((void *)rap, sizeof *rap, M_CLRDAHEAD);
324 rap = ubc->cl_rahead;
325 }
326 vnode_unlock(vp);
327 }
328 if (lck_mtx_try_lock(&rap->cl_lockr) == TRUE)
329 return(rap);
330
331 return ((struct cl_readahead *)NULL);
332 }
333
334
335 /*
336 * if the write behind context doesn't yet exist,
337 * and CLW_ALLOCATE is specified, allocate and initialize it...
338 * the vnode lock serializes multiple callers
339 * during the actual assignment... first one
340 * to grab the lock wins... the other callers
341 * will release the now unnecessary storage
342 *
343 * if CLW_RETURNLOCKED is set, grab (blocking if necessary)
344 * the lock associated with the write behind context before
345 * returning
346 */
347
348 static struct cl_writebehind *
349 cluster_get_wbp(vnode_t vp, int flags)
350 {
351 struct ubc_info *ubc;
352 struct cl_writebehind *wbp;
353
354 ubc = vp->v_ubcinfo;
355
356 if ((wbp = ubc->cl_wbehind) == NULL) {
357
358 if ( !(flags & CLW_ALLOCATE))
359 return ((struct cl_writebehind *)NULL);
360
361 MALLOC_ZONE(wbp, struct cl_writebehind *, sizeof *wbp, M_CLWRBEHIND, M_WAITOK);
362
363 bzero(wbp, sizeof *wbp);
364 lck_mtx_init(&wbp->cl_lockw, cl_mtx_grp, cl_mtx_attr);
365
366 vnode_lock(vp);
367
368 if (ubc->cl_wbehind == NULL)
369 ubc->cl_wbehind = wbp;
370 else {
371 lck_mtx_destroy(&wbp->cl_lockw, cl_mtx_grp);
372 FREE_ZONE((void *)wbp, sizeof *wbp, M_CLWRBEHIND);
373 wbp = ubc->cl_wbehind;
374 }
375 vnode_unlock(vp);
376 }
377 if (flags & CLW_RETURNLOCKED)
378 lck_mtx_lock(&wbp->cl_lockw);
379
380 return (wbp);
381 }
382
383
384 static void
385 cluster_syncup(vnode_t vp, off_t newEOF, int (*callback)(buf_t, void *), void *callback_arg)
386 {
387 struct cl_writebehind *wbp;
388
389 if ((wbp = cluster_get_wbp(vp, 0)) != NULL) {
390
391 if (wbp->cl_number) {
392 lck_mtx_lock(&wbp->cl_lockw);
393
394 cluster_try_push(wbp, vp, newEOF, PUSH_ALL | PUSH_SYNC, callback, callback_arg);
395
396 lck_mtx_unlock(&wbp->cl_lockw);
397 }
398 }
399 }
400
401
402 static int
403 cluster_hard_throttle_on(vnode_t vp)
404 {
405 static struct timeval hard_throttle_maxelapsed = { 0, 200000 };
406
407 if (vp->v_mount->mnt_kern_flag & MNTK_ROOTDEV) {
408 struct timeval elapsed;
409
410 if (hard_throttle_on_root)
411 return(1);
412
413 microuptime(&elapsed);
414 timevalsub(&elapsed, &priority_IO_timestamp_for_root);
415
416 if (timevalcmp(&elapsed, &hard_throttle_maxelapsed, <))
417 return(1);
418 }
419 return(0);
420 }
421
422
423 static int
424 cluster_ioerror(upl_t upl, int upl_offset, int abort_size, int error, int io_flags)
425 {
426 int upl_abort_code = 0;
427 int page_in = 0;
428 int page_out = 0;
429
430 if (io_flags & B_PHYS)
431 /*
432 * direct write of any flavor, or a direct read that wasn't aligned
433 */
434 ubc_upl_commit_range(upl, upl_offset, abort_size, UPL_COMMIT_FREE_ON_EMPTY);
435 else {
436 if (io_flags & B_PAGEIO) {
437 if (io_flags & B_READ)
438 page_in = 1;
439 else
440 page_out = 1;
441 }
442 if (io_flags & B_CACHE)
443 /*
444 * leave pages in the cache unchanged on error
445 */
446 upl_abort_code = UPL_ABORT_FREE_ON_EMPTY;
447 else if (page_out && (error != ENXIO))
448 /*
449 * transient error... leave pages unchanged
450 */
451 upl_abort_code = UPL_ABORT_FREE_ON_EMPTY;
452 else if (page_in)
453 upl_abort_code = UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR;
454 else
455 upl_abort_code = UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_DUMP_PAGES;
456
457 ubc_upl_abort_range(upl, upl_offset, abort_size, upl_abort_code);
458 }
459 return (upl_abort_code);
460 }
461
462
463 static int
464 cluster_iodone(buf_t bp, void *callback_arg)
465 {
466 int b_flags;
467 int error;
468 int total_size;
469 int total_resid;
470 int upl_offset;
471 int zero_offset;
472 int pg_offset = 0;
473 int commit_size = 0;
474 int upl_flags = 0;
475 int transaction_size = 0;
476 upl_t upl;
477 buf_t cbp;
478 buf_t cbp_head;
479 buf_t cbp_next;
480 buf_t real_bp;
481 struct clios *iostate;
482 boolean_t transaction_complete = FALSE;
483
484 cbp_head = (buf_t)(bp->b_trans_head);
485
486 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 20)) | DBG_FUNC_START,
487 (int)cbp_head, bp->b_lblkno, bp->b_bcount, bp->b_flags, 0);
488
489 for (cbp = cbp_head; cbp; cbp = cbp->b_trans_next) {
490 /*
491 * all I/O requests that are part of this transaction
492 * have to complete before we can process it
493 */
494 if ( !(cbp->b_flags & B_DONE)) {
495
496 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 20)) | DBG_FUNC_END,
497 (int)cbp_head, (int)cbp, cbp->b_bcount, cbp->b_flags, 0);
498
499 return 0;
500 }
501 if (cbp->b_flags & B_EOT)
502 transaction_complete = TRUE;
503 }
504 if (transaction_complete == FALSE) {
505 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 20)) | DBG_FUNC_END,
506 (int)cbp_head, 0, 0, 0, 0);
507
508 return 0;
509 }
510 error = 0;
511 total_size = 0;
512 total_resid = 0;
513
514 cbp = cbp_head;
515 upl_offset = cbp->b_uploffset;
516 upl = cbp->b_upl;
517 b_flags = cbp->b_flags;
518 real_bp = cbp->b_real_bp;
519 zero_offset= cbp->b_validend;
520 iostate = (struct clios *)cbp->b_iostate;
521
522 if (real_bp)
523 real_bp->b_dev = cbp->b_dev;
524
525 while (cbp) {
526 if ((cbp->b_flags & B_ERROR) && error == 0)
527 error = cbp->b_error;
528
529 total_resid += cbp->b_resid;
530 total_size += cbp->b_bcount;
531
532 cbp_next = cbp->b_trans_next;
533
534 if (cbp_next == NULL)
535 /*
536 * compute the overall size of the transaction
537 * in case we created one that has 'holes' in it
538 * 'total_size' represents the amount of I/O we
539 * did, not the span of the transaction w/r to the UPL
540 */
541 transaction_size = cbp->b_uploffset + cbp->b_bcount - upl_offset;
542
543 if (cbp != cbp_head)
544 free_io_buf(cbp);
545
546 cbp = cbp_next;
547 }
548 if (error == 0 && total_resid)
549 error = EIO;
550
551 if (error == 0) {
552 int (*cliodone_func)(buf_t, void *) = (int (*)(buf_t, void *))(cbp_head->b_cliodone);
553
554 if (cliodone_func != NULL) {
555 cbp_head->b_bcount = transaction_size;
556
557 error = (*cliodone_func)(cbp_head, callback_arg);
558 }
559 }
560 if (zero_offset)
561 cluster_zero(upl, zero_offset, PAGE_SIZE - (zero_offset & PAGE_MASK), real_bp);
562
563 free_io_buf(cbp_head);
564
565 if (iostate) {
566 int need_wakeup = 0;
567
568 /*
569 * someone has issued multiple I/Os asynchrounsly
570 * and is waiting for them to complete (streaming)
571 */
572 lck_mtx_lock_spin(cl_mtxp);
573
574 if (error && iostate->io_error == 0)
575 iostate->io_error = error;
576
577 iostate->io_completed += total_size;
578
579 if (iostate->io_wanted) {
580 /*
581 * someone is waiting for the state of
582 * this io stream to change
583 */
584 iostate->io_wanted = 0;
585 need_wakeup = 1;
586 }
587 lck_mtx_unlock(cl_mtxp);
588
589 if (need_wakeup)
590 wakeup((caddr_t)&iostate->io_wanted);
591 }
592
593 if (b_flags & B_COMMIT_UPL) {
594
595 pg_offset = upl_offset & PAGE_MASK;
596 commit_size = (pg_offset + transaction_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
597
598 if (error)
599 upl_flags = cluster_ioerror(upl, upl_offset - pg_offset, commit_size, error, b_flags);
600 else {
601 upl_flags = UPL_COMMIT_FREE_ON_EMPTY;
602
603 if ((b_flags & B_PHYS) && (b_flags & B_READ))
604 upl_flags |= UPL_COMMIT_SET_DIRTY;
605
606 if (b_flags & B_AGE)
607 upl_flags |= UPL_COMMIT_INACTIVATE;
608
609 ubc_upl_commit_range(upl, upl_offset - pg_offset, commit_size, upl_flags);
610 }
611 }
612 if ((b_flags & B_NEED_IODONE) && real_bp) {
613 if (error) {
614 real_bp->b_flags |= B_ERROR;
615 real_bp->b_error = error;
616 }
617 real_bp->b_resid = total_resid;
618
619 buf_biodone(real_bp);
620 }
621 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 20)) | DBG_FUNC_END,
622 (int)upl, upl_offset - pg_offset, commit_size, (error << 24) | upl_flags, 0);
623
624 return (error);
625 }
626
627
628 void
629 cluster_zero(upl_t upl, vm_offset_t upl_offset, int size, buf_t bp)
630 {
631
632 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 23)) | DBG_FUNC_START,
633 upl_offset, size, (int)bp, 0, 0);
634
635 if (bp == NULL || bp->b_datap == 0) {
636 upl_page_info_t *pl;
637 addr64_t zero_addr;
638
639 pl = ubc_upl_pageinfo(upl);
640
641 if (upl_device_page(pl) == TRUE) {
642 zero_addr = ((addr64_t)upl_phys_page(pl, 0) << 12) + upl_offset;
643
644 bzero_phys_nc(zero_addr, size);
645 } else {
646 while (size) {
647 int page_offset;
648 int page_index;
649 int zero_cnt;
650
651 page_index = upl_offset / PAGE_SIZE;
652 page_offset = upl_offset & PAGE_MASK;
653
654 zero_addr = ((addr64_t)upl_phys_page(pl, page_index) << 12) + page_offset;
655 zero_cnt = min(PAGE_SIZE - page_offset, size);
656
657 bzero_phys(zero_addr, zero_cnt);
658
659 size -= zero_cnt;
660 upl_offset += zero_cnt;
661 }
662 }
663 } else
664 bzero((caddr_t)((vm_offset_t)bp->b_datap + upl_offset), size);
665
666 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 23)) | DBG_FUNC_END,
667 upl_offset, size, 0, 0, 0);
668 }
669
670
671 static void
672 cluster_EOT(buf_t cbp_head, buf_t cbp_tail, int zero_offset)
673 {
674 cbp_head->b_validend = zero_offset;
675 cbp_tail->b_flags |= B_EOT;
676 }
677
678 static void
679 cluster_wait_IO(buf_t cbp_head, int async)
680 {
681 buf_t cbp;
682
683 if (async) {
684 /*
685 * async callback completion will not normally
686 * generate a wakeup upon I/O completion...
687 * by setting BL_WANTED, we will force a wakeup
688 * to occur as any outstanding I/Os complete...
689 * I/Os already completed will have BL_CALLDONE already
690 * set and we won't block in buf_biowait_callback..
691 * note that we're actually waiting for the bp to have
692 * completed the callback function... only then
693 * can we safely take back ownership of the bp
694 * need the main buf mutex in order to safely
695 * update b_lflags
696 */
697 buf_list_lock();
698
699 for (cbp = cbp_head; cbp; cbp = cbp->b_trans_next)
700 cbp->b_lflags |= BL_WANTED;
701
702 buf_list_unlock();
703 }
704 for (cbp = cbp_head; cbp; cbp = cbp->b_trans_next) {
705 if (async)
706 buf_biowait_callback(cbp);
707 else
708 buf_biowait(cbp);
709 }
710 }
711
712 static void
713 cluster_complete_transaction(buf_t *cbp_head, void *callback_arg, int *retval, int flags, int needwait)
714 {
715 buf_t cbp;
716 int error;
717
718 /*
719 * cluster_complete_transaction will
720 * only be called if we've issued a complete chain in synchronous mode
721 * or, we've already done a cluster_wait_IO on an incomplete chain
722 */
723 if (needwait) {
724 for (cbp = *cbp_head; cbp; cbp = cbp->b_trans_next)
725 buf_biowait(cbp);
726 }
727 error = cluster_iodone(*cbp_head, callback_arg);
728
729 if ( !(flags & CL_ASYNC) && error && *retval == 0) {
730 if (((flags & (CL_PAGEOUT | CL_KEEPCACHED)) != CL_PAGEOUT) || (error != ENXIO))
731 *retval = error;
732 }
733 *cbp_head = (buf_t)NULL;
734 }
735
736
737 static int
738 cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int non_rounded_size,
739 int flags, buf_t real_bp, struct clios *iostate, int (*callback)(buf_t, void *), void *callback_arg)
740 {
741 buf_t cbp;
742 u_int size;
743 u_int io_size;
744 int io_flags;
745 int bmap_flags;
746 int error = 0;
747 int retval = 0;
748 buf_t cbp_head = NULL;
749 buf_t cbp_tail = NULL;
750 int trans_count = 0;
751 int max_trans_count;
752 u_int pg_count;
753 int pg_offset;
754 u_int max_iosize;
755 u_int max_vectors;
756 int priv;
757 int zero_offset = 0;
758 int async_throttle = 0;
759 mount_t mp;
760 vm_offset_t upl_end_offset;
761 boolean_t need_EOT = FALSE;
762
763 /*
764 * we currently don't support buffers larger than a page
765 */
766 if (real_bp && non_rounded_size > PAGE_SIZE)
767 panic("%s(): Called with real buffer of size %d bytes which "
768 "is greater than the maximum allowed size of "
769 "%d bytes (the system PAGE_SIZE).\n",
770 __FUNCTION__, non_rounded_size, PAGE_SIZE);
771
772 mp = vp->v_mount;
773
774 /*
775 * we don't want to do any funny rounding of the size for IO requests
776 * coming through the DIRECT or CONTIGUOUS paths... those pages don't
777 * belong to us... we can't extend (nor do we need to) the I/O to fill
778 * out a page
779 */
780 if (mp->mnt_devblocksize > 1 && !(flags & (CL_DEV_MEMORY | CL_DIRECT_IO))) {
781 /*
782 * round the requested size up so that this I/O ends on a
783 * page boundary in case this is a 'write'... if the filesystem
784 * has blocks allocated to back the page beyond the EOF, we want to
785 * make sure to write out the zero's that are sitting beyond the EOF
786 * so that in case the filesystem doesn't explicitly zero this area
787 * if a hole is created via a lseek/write beyond the current EOF,
788 * it will return zeros when it's read back from the disk. If the
789 * physical allocation doesn't extend for the whole page, we'll
790 * only write/read from the disk up to the end of this allocation
791 * via the extent info returned from the VNOP_BLOCKMAP call.
792 */
793 pg_offset = upl_offset & PAGE_MASK;
794
795 size = (((non_rounded_size + pg_offset) + (PAGE_SIZE - 1)) & ~PAGE_MASK) - pg_offset;
796 } else {
797 /*
798 * anyone advertising a blocksize of 1 byte probably
799 * can't deal with us rounding up the request size
800 * AFP is one such filesystem/device
801 */
802 size = non_rounded_size;
803 }
804 upl_end_offset = upl_offset + size;
805
806 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 22)) | DBG_FUNC_START, (int)f_offset, size, upl_offset, flags, 0);
807
808 /*
809 * Set the maximum transaction size to the maximum desired number of
810 * buffers.
811 */
812 max_trans_count = 8;
813 if (flags & CL_DEV_MEMORY)
814 max_trans_count = 16;
815
816 if (flags & CL_READ) {
817 io_flags = B_READ;
818 bmap_flags = VNODE_READ;
819
820 max_iosize = mp->mnt_maxreadcnt;
821 max_vectors = mp->mnt_segreadcnt;
822 } else {
823 io_flags = B_WRITE;
824 bmap_flags = VNODE_WRITE;
825
826 max_iosize = mp->mnt_maxwritecnt;
827 max_vectors = mp->mnt_segwritecnt;
828 }
829 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 22)) | DBG_FUNC_NONE, max_iosize, max_vectors, mp->mnt_devblocksize, 0, 0);
830
831 /*
832 * make sure the maximum iosize is a
833 * multiple of the page size
834 */
835 max_iosize &= ~PAGE_MASK;
836
837 /*
838 * Ensure the maximum iosize is sensible.
839 */
840 if (!max_iosize)
841 max_iosize = PAGE_SIZE;
842
843 if (flags & CL_THROTTLE) {
844 if ( !(flags & CL_PAGEOUT) && cluster_hard_throttle_on(vp)) {
845 if (max_iosize > HARD_THROTTLE_MAXSIZE)
846 max_iosize = HARD_THROTTLE_MAXSIZE;
847 async_throttle = HARD_THROTTLE_MAXCNT;
848 } else {
849 if ( (flags & CL_DEV_MEMORY) )
850 async_throttle = VNODE_ASYNC_THROTTLE;
851 else {
852 u_int max_cluster;
853 u_int max_cluster_size;
854 u_int max_prefetch;
855
856 max_cluster_size = MAX_CLUSTER_SIZE(vp);
857 max_prefetch = MAX_PREFETCH(vp);
858
859 if (max_iosize > max_cluster_size)
860 max_cluster = max_cluster_size;
861 else
862 max_cluster = max_iosize;
863
864 if (size < max_cluster)
865 max_cluster = size;
866
867 async_throttle = min(VNODE_ASYNC_THROTTLE, (max_prefetch / max_cluster) - 1);
868 }
869 }
870 }
871 if (flags & CL_AGE)
872 io_flags |= B_AGE;
873 if (flags & (CL_PAGEIN | CL_PAGEOUT))
874 io_flags |= B_PAGEIO;
875 if (flags & CL_COMMIT)
876 io_flags |= B_COMMIT_UPL;
877 if (flags & CL_PRESERVE)
878 io_flags |= B_PHYS;
879 if (flags & CL_KEEPCACHED)
880 io_flags |= B_CACHE;
881 if (flags & CL_PASSIVE)
882 io_flags |= B_PASSIVE;
883 if (vp->v_flag & VSYSTEM)
884 io_flags |= B_META;
885
886 if ((flags & CL_READ) && ((upl_offset + non_rounded_size) & PAGE_MASK) && (!(flags & CL_NOZERO))) {
887 /*
888 * then we are going to end up
889 * with a page that we can't complete (the file size wasn't a multiple
890 * of PAGE_SIZE and we're trying to read to the end of the file
891 * so we'll go ahead and zero out the portion of the page we can't
892 * read in from the file
893 */
894 zero_offset = upl_offset + non_rounded_size;
895 }
896 while (size) {
897 daddr64_t blkno;
898 daddr64_t lblkno;
899 u_int io_size_wanted;
900
901 if (size > max_iosize)
902 io_size = max_iosize;
903 else
904 io_size = size;
905
906 io_size_wanted = io_size;
907
908 if ((error = VNOP_BLOCKMAP(vp, f_offset, io_size, &blkno, (size_t *)&io_size, NULL, bmap_flags, NULL)))
909 break;
910
911 if (io_size > io_size_wanted)
912 io_size = io_size_wanted;
913
914 if (real_bp && (real_bp->b_blkno == real_bp->b_lblkno))
915 real_bp->b_blkno = blkno;
916
917 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 24)) | DBG_FUNC_NONE,
918 (int)f_offset, (int)(blkno>>32), (int)blkno, io_size, 0);
919
920 if (io_size == 0) {
921 /*
922 * vnop_blockmap didn't return an error... however, it did
923 * return an extent size of 0 which means we can't
924 * make forward progress on this I/O... a hole in the
925 * file would be returned as a blkno of -1 with a non-zero io_size
926 * a real extent is returned with a blkno != -1 and a non-zero io_size
927 */
928 error = EINVAL;
929 break;
930 }
931 if ( !(flags & CL_READ) && blkno == -1) {
932 off_t e_offset;
933 int pageout_flags;
934
935 /*
936 * we're writing into a 'hole'
937 */
938 if (flags & CL_PAGEOUT) {
939 /*
940 * if we got here via cluster_pageout
941 * then just error the request and return
942 * the 'hole' should already have been covered
943 */
944 error = EINVAL;
945 break;
946 }
947 /*
948 * we can get here if the cluster code happens to
949 * pick up a page that was dirtied via mmap vs
950 * a 'write' and the page targets a 'hole'...
951 * i.e. the writes to the cluster were sparse
952 * and the file was being written for the first time
953 *
954 * we can also get here if the filesystem supports
955 * 'holes' that are less than PAGE_SIZE.... because
956 * we can't know if the range in the page that covers
957 * the 'hole' has been dirtied via an mmap or not,
958 * we have to assume the worst and try to push the
959 * entire page to storage.
960 *
961 * Try paging out the page individually before
962 * giving up entirely and dumping it (the pageout
963 * path will insure that the zero extent accounting
964 * has been taken care of before we get back into cluster_io)
965 *
966 * go direct to vnode_pageout so that we don't have to
967 * unbusy the page from the UPL... we used to do this
968 * so that we could call ubc_sync_range, but that results
969 * in a potential deadlock if someone else races us to acquire
970 * that page and wins and in addition needs one of the pages
971 * we're continuing to hold in the UPL
972 */
973 pageout_flags = UPL_MSYNC | UPL_VNODE_PAGER | UPL_NESTED_PAGEOUT;
974
975 if ( !(flags & CL_ASYNC))
976 pageout_flags |= UPL_IOSYNC;
977 if ( !(flags & CL_COMMIT))
978 pageout_flags |= UPL_NOCOMMIT;
979
980 if (cbp_head) {
981 buf_t last_cbp;
982
983 /*
984 * first we have to wait for the the current outstanding I/Os
985 * to complete... EOT hasn't been set yet on this transaction
986 * so the pages won't be released just because all of the current
987 * I/O linked to this transaction has completed...
988 */
989 cluster_wait_IO(cbp_head, (flags & CL_ASYNC));
990
991 /*
992 * we've got a transcation that
993 * includes the page we're about to push out through vnode_pageout...
994 * find the last bp in the list which will be the one that
995 * includes the head of this page and round it's iosize down
996 * to a page boundary...
997 */
998 for (last_cbp = cbp = cbp_head; cbp->b_trans_next; cbp = cbp->b_trans_next)
999 last_cbp = cbp;
1000
1001 cbp->b_bcount &= ~PAGE_MASK;
1002
1003 if (cbp->b_bcount == 0) {
1004 /*
1005 * this buf no longer has any I/O associated with it
1006 */
1007 free_io_buf(cbp);
1008
1009 if (cbp == cbp_head) {
1010 /*
1011 * the buf we just freed was the only buf in
1012 * this transaction... so there's no I/O to do
1013 */
1014 cbp_head = NULL;
1015 } else {
1016 /*
1017 * remove the buf we just freed from
1018 * the transaction list
1019 */
1020 last_cbp->b_trans_next = NULL;
1021 cbp_tail = last_cbp;
1022 }
1023 }
1024 if (cbp_head) {
1025 /*
1026 * there was more to the current transaction
1027 * than just the page we are pushing out via vnode_pageout...
1028 * mark it as finished and complete it... we've already
1029 * waited for the I/Os to complete above in the call to cluster_wait_IO
1030 */
1031 cluster_EOT(cbp_head, cbp_tail, 0);
1032
1033 cluster_complete_transaction(&cbp_head, callback_arg, &retval, flags, 0);
1034
1035 trans_count = 0;
1036 }
1037 }
1038 if (vnode_pageout(vp, upl, trunc_page(upl_offset), trunc_page_64(f_offset), PAGE_SIZE, pageout_flags, NULL) != PAGER_SUCCESS) {
1039 error = EINVAL;
1040 break;
1041 }
1042 e_offset = round_page_64(f_offset + 1);
1043 io_size = e_offset - f_offset;
1044
1045 f_offset += io_size;
1046 upl_offset += io_size;
1047
1048 if (size >= io_size)
1049 size -= io_size;
1050 else
1051 size = 0;
1052 /*
1053 * keep track of how much of the original request
1054 * that we've actually completed... non_rounded_size
1055 * may go negative due to us rounding the request
1056 * to a page size multiple (i.e. size > non_rounded_size)
1057 */
1058 non_rounded_size -= io_size;
1059
1060 if (non_rounded_size <= 0) {
1061 /*
1062 * we've transferred all of the data in the original
1063 * request, but we were unable to complete the tail
1064 * of the last page because the file didn't have
1065 * an allocation to back that portion... this is ok.
1066 */
1067 size = 0;
1068 }
1069 continue;
1070 }
1071 lblkno = (daddr64_t)(f_offset / PAGE_SIZE_64);
1072 /*
1073 * we have now figured out how much I/O we can do - this is in 'io_size'
1074 * pg_offset is the starting point in the first page for the I/O
1075 * pg_count is the number of full and partial pages that 'io_size' encompasses
1076 */
1077 pg_offset = upl_offset & PAGE_MASK;
1078
1079 if (flags & CL_DEV_MEMORY) {
1080 /*
1081 * treat physical requests as one 'giant' page
1082 */
1083 pg_count = 1;
1084 } else
1085 pg_count = (io_size + pg_offset + (PAGE_SIZE - 1)) / PAGE_SIZE;
1086
1087 if ((flags & CL_READ) && blkno == -1) {
1088 vm_offset_t commit_offset;
1089 int bytes_to_zero;
1090 int complete_transaction_now = 0;
1091
1092 /*
1093 * if we're reading and blkno == -1, then we've got a
1094 * 'hole' in the file that we need to deal with by zeroing
1095 * out the affected area in the upl
1096 */
1097 if (io_size >= (u_int)non_rounded_size) {
1098 /*
1099 * if this upl contains the EOF and it is not a multiple of PAGE_SIZE
1100 * than 'zero_offset' will be non-zero
1101 * if the 'hole' returned by vnop_blockmap extends all the way to the eof
1102 * (indicated by the io_size finishing off the I/O request for this UPL)
1103 * than we're not going to issue an I/O for the
1104 * last page in this upl... we need to zero both the hole and the tail
1105 * of the page beyond the EOF, since the delayed zero-fill won't kick in
1106 */
1107 bytes_to_zero = non_rounded_size;
1108 if (!(flags & CL_NOZERO))
1109 bytes_to_zero = (((upl_offset + io_size) + (PAGE_SIZE - 1)) & ~PAGE_MASK) - upl_offset;
1110
1111 zero_offset = 0;
1112 } else
1113 bytes_to_zero = io_size;
1114
1115 pg_count = 0;
1116
1117 cluster_zero(upl, upl_offset, bytes_to_zero, real_bp);
1118
1119 if (cbp_head) {
1120 int pg_resid;
1121
1122 /*
1123 * if there is a current I/O chain pending
1124 * then the first page of the group we just zero'd
1125 * will be handled by the I/O completion if the zero
1126 * fill started in the middle of the page
1127 */
1128 commit_offset = (upl_offset + (PAGE_SIZE - 1)) & ~PAGE_MASK;
1129
1130 pg_resid = commit_offset - upl_offset;
1131
1132 if (bytes_to_zero >= pg_resid) {
1133 /*
1134 * the last page of the current I/O
1135 * has been completed...
1136 * compute the number of fully zero'd
1137 * pages that are beyond it
1138 * plus the last page if its partial
1139 * and we have no more I/O to issue...
1140 * otherwise a partial page is left
1141 * to begin the next I/O
1142 */
1143 if ((int)io_size >= non_rounded_size)
1144 pg_count = (bytes_to_zero - pg_resid + (PAGE_SIZE - 1)) / PAGE_SIZE;
1145 else
1146 pg_count = (bytes_to_zero - pg_resid) / PAGE_SIZE;
1147
1148 complete_transaction_now = 1;
1149 }
1150 } else {
1151 /*
1152 * no pending I/O to deal with
1153 * so, commit all of the fully zero'd pages
1154 * plus the last page if its partial
1155 * and we have no more I/O to issue...
1156 * otherwise a partial page is left
1157 * to begin the next I/O
1158 */
1159 if ((int)io_size >= non_rounded_size)
1160 pg_count = (pg_offset + bytes_to_zero + (PAGE_SIZE - 1)) / PAGE_SIZE;
1161 else
1162 pg_count = (pg_offset + bytes_to_zero) / PAGE_SIZE;
1163
1164 commit_offset = upl_offset & ~PAGE_MASK;
1165 }
1166 if ( (flags & CL_COMMIT) && pg_count) {
1167 ubc_upl_commit_range(upl, commit_offset, pg_count * PAGE_SIZE,
1168 UPL_COMMIT_CLEAR_DIRTY | UPL_COMMIT_FREE_ON_EMPTY);
1169 }
1170 upl_offset += io_size;
1171 f_offset += io_size;
1172 size -= io_size;
1173
1174 /*
1175 * keep track of how much of the original request
1176 * that we've actually completed... non_rounded_size
1177 * may go negative due to us rounding the request
1178 * to a page size multiple (i.e. size > non_rounded_size)
1179 */
1180 non_rounded_size -= io_size;
1181
1182 if (non_rounded_size <= 0) {
1183 /*
1184 * we've transferred all of the data in the original
1185 * request, but we were unable to complete the tail
1186 * of the last page because the file didn't have
1187 * an allocation to back that portion... this is ok.
1188 */
1189 size = 0;
1190 }
1191 if (cbp_head && (complete_transaction_now || size == 0)) {
1192 cluster_wait_IO(cbp_head, (flags & CL_ASYNC));
1193
1194 cluster_EOT(cbp_head, cbp_tail, size == 0 ? zero_offset : 0);
1195
1196 cluster_complete_transaction(&cbp_head, callback_arg, &retval, flags, 0);
1197
1198 trans_count = 0;
1199 }
1200 continue;
1201 }
1202 if (pg_count > max_vectors) {
1203 if (((pg_count - max_vectors) * PAGE_SIZE) > io_size) {
1204 io_size = PAGE_SIZE - pg_offset;
1205 pg_count = 1;
1206 } else {
1207 io_size -= (pg_count - max_vectors) * PAGE_SIZE;
1208 pg_count = max_vectors;
1209 }
1210 }
1211 /*
1212 * If the transaction is going to reach the maximum number of
1213 * desired elements, truncate the i/o to the nearest page so
1214 * that the actual i/o is initiated after this buffer is
1215 * created and added to the i/o chain.
1216 *
1217 * I/O directed to physically contiguous memory
1218 * doesn't have a requirement to make sure we 'fill' a page
1219 */
1220 if ( !(flags & CL_DEV_MEMORY) && trans_count >= max_trans_count &&
1221 ((upl_offset + io_size) & PAGE_MASK)) {
1222 vm_offset_t aligned_ofs;
1223
1224 aligned_ofs = (upl_offset + io_size) & ~PAGE_MASK;
1225 /*
1226 * If the io_size does not actually finish off even a
1227 * single page we have to keep adding buffers to the
1228 * transaction despite having reached the desired limit.
1229 *
1230 * Eventually we get here with the page being finished
1231 * off (and exceeded) and then we truncate the size of
1232 * this i/o request so that it is page aligned so that
1233 * we can finally issue the i/o on the transaction.
1234 */
1235 if (aligned_ofs > upl_offset) {
1236 io_size = aligned_ofs - upl_offset;
1237 pg_count--;
1238 }
1239 }
1240
1241 if ( !(mp->mnt_kern_flag & MNTK_VIRTUALDEV))
1242 /*
1243 * if we're not targeting a virtual device i.e. a disk image
1244 * it's safe to dip into the reserve pool since real devices
1245 * can complete this I/O request without requiring additional
1246 * bufs from the alloc_io_buf pool
1247 */
1248 priv = 1;
1249 else if ((flags & CL_ASYNC) && !(flags & CL_PAGEOUT))
1250 /*
1251 * Throttle the speculative IO
1252 */
1253 priv = 0;
1254 else
1255 priv = 1;
1256
1257 cbp = alloc_io_buf(vp, priv);
1258
1259 if (flags & CL_PAGEOUT) {
1260 u_int i;
1261
1262 for (i = 0; i < pg_count; i++) {
1263 if (buf_invalblkno(vp, lblkno + i, 0) == EBUSY)
1264 panic("BUSY bp found in cluster_io");
1265 }
1266 }
1267 if (flags & CL_ASYNC) {
1268 if (buf_setcallback(cbp, (void *)cluster_iodone, callback_arg))
1269 panic("buf_setcallback failed\n");
1270 }
1271 cbp->b_cliodone = (void *)callback;
1272 cbp->b_flags |= io_flags;
1273
1274 cbp->b_lblkno = lblkno;
1275 cbp->b_blkno = blkno;
1276 cbp->b_bcount = io_size;
1277
1278 if (buf_setupl(cbp, upl, upl_offset))
1279 panic("buf_setupl failed\n");
1280
1281 cbp->b_trans_next = (buf_t)NULL;
1282
1283 if ((cbp->b_iostate = (void *)iostate))
1284 /*
1285 * caller wants to track the state of this
1286 * io... bump the amount issued against this stream
1287 */
1288 iostate->io_issued += io_size;
1289
1290 if (flags & CL_READ) {
1291 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 26)) | DBG_FUNC_NONE,
1292 (int)cbp->b_lblkno, (int)cbp->b_blkno, upl_offset, io_size, 0);
1293 }
1294 else {
1295 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 27)) | DBG_FUNC_NONE,
1296 (int)cbp->b_lblkno, (int)cbp->b_blkno, upl_offset, io_size, 0);
1297 }
1298
1299 if (cbp_head) {
1300 cbp_tail->b_trans_next = cbp;
1301 cbp_tail = cbp;
1302 } else {
1303 cbp_head = cbp;
1304 cbp_tail = cbp;
1305
1306 if ( (cbp_head->b_real_bp = real_bp) ) {
1307 cbp_head->b_flags |= B_NEED_IODONE;
1308 real_bp = (buf_t)NULL;
1309 }
1310 }
1311 *(buf_t *)(&cbp->b_trans_head) = cbp_head;
1312
1313 trans_count++;
1314
1315 upl_offset += io_size;
1316 f_offset += io_size;
1317 size -= io_size;
1318 /*
1319 * keep track of how much of the original request
1320 * that we've actually completed... non_rounded_size
1321 * may go negative due to us rounding the request
1322 * to a page size multiple (i.e. size > non_rounded_size)
1323 */
1324 non_rounded_size -= io_size;
1325
1326 if (non_rounded_size <= 0) {
1327 /*
1328 * we've transferred all of the data in the original
1329 * request, but we were unable to complete the tail
1330 * of the last page because the file didn't have
1331 * an allocation to back that portion... this is ok.
1332 */
1333 size = 0;
1334 }
1335 if (size == 0) {
1336 /*
1337 * we have no more I/O to issue, so go
1338 * finish the final transaction
1339 */
1340 need_EOT = TRUE;
1341 } else if ( ((flags & CL_DEV_MEMORY) || (upl_offset & PAGE_MASK) == 0) &&
1342 ((flags & CL_ASYNC) || trans_count > max_trans_count) ) {
1343 /*
1344 * I/O directed to physically contiguous memory...
1345 * which doesn't have a requirement to make sure we 'fill' a page
1346 * or...
1347 * the current I/O we've prepared fully
1348 * completes the last page in this request
1349 * and ...
1350 * it's either an ASYNC request or
1351 * we've already accumulated more than 8 I/O's into
1352 * this transaction so mark it as complete so that
1353 * it can finish asynchronously or via the cluster_complete_transaction
1354 * below if the request is synchronous
1355 */
1356 need_EOT = TRUE;
1357 }
1358 if (need_EOT == TRUE)
1359 cluster_EOT(cbp_head, cbp_tail, size == 0 ? zero_offset : 0);
1360
1361 if (flags & CL_THROTTLE)
1362 (void)vnode_waitforwrites(vp, async_throttle, 0, 0, "cluster_io");
1363
1364 if ( !(io_flags & B_READ))
1365 vnode_startwrite(vp);
1366
1367 (void) VNOP_STRATEGY(cbp);
1368
1369 if (need_EOT == TRUE) {
1370 if ( !(flags & CL_ASYNC))
1371 cluster_complete_transaction(&cbp_head, callback_arg, &retval, flags, 1);
1372
1373 need_EOT = FALSE;
1374 trans_count = 0;
1375 cbp_head = NULL;
1376 }
1377 }
1378 if (error) {
1379 int abort_size;
1380
1381 io_size = 0;
1382
1383 if (cbp_head) {
1384 /*
1385 * first wait until all of the outstanding I/O
1386 * for this partial transaction has completed
1387 */
1388 cluster_wait_IO(cbp_head, (flags & CL_ASYNC));
1389
1390 /*
1391 * Rewind the upl offset to the beginning of the
1392 * transaction.
1393 */
1394 upl_offset = cbp_head->b_uploffset;
1395
1396 for (cbp = cbp_head; cbp;) {
1397 buf_t cbp_next;
1398
1399 size += cbp->b_bcount;
1400 io_size += cbp->b_bcount;
1401
1402 cbp_next = cbp->b_trans_next;
1403 free_io_buf(cbp);
1404 cbp = cbp_next;
1405 }
1406 }
1407 if (iostate) {
1408 int need_wakeup = 0;
1409
1410 /*
1411 * update the error condition for this stream
1412 * since we never really issued the io
1413 * just go ahead and adjust it back
1414 */
1415 lck_mtx_lock_spin(cl_mtxp);
1416
1417 if (iostate->io_error == 0)
1418 iostate->io_error = error;
1419 iostate->io_issued -= io_size;
1420
1421 if (iostate->io_wanted) {
1422 /*
1423 * someone is waiting for the state of
1424 * this io stream to change
1425 */
1426 iostate->io_wanted = 0;
1427 need_wakeup = 1;
1428 }
1429 lck_mtx_unlock(cl_mtxp);
1430
1431 if (need_wakeup)
1432 wakeup((caddr_t)&iostate->io_wanted);
1433 }
1434 if (flags & CL_COMMIT) {
1435 int upl_flags;
1436
1437 pg_offset = upl_offset & PAGE_MASK;
1438 abort_size = (upl_end_offset - upl_offset + PAGE_MASK) & ~PAGE_MASK;
1439
1440 upl_flags = cluster_ioerror(upl, upl_offset - pg_offset, abort_size, error, io_flags);
1441
1442 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 28)) | DBG_FUNC_NONE,
1443 (int)upl, upl_offset - pg_offset, abort_size, (error << 24) | upl_flags, 0);
1444 }
1445 if (retval == 0)
1446 retval = error;
1447 } else if (cbp_head)
1448 panic("%s(): cbp_head is not NULL.\n", __FUNCTION__);
1449
1450 if (real_bp) {
1451 /*
1452 * can get here if we either encountered an error
1453 * or we completely zero-filled the request and
1454 * no I/O was issued
1455 */
1456 if (error) {
1457 real_bp->b_flags |= B_ERROR;
1458 real_bp->b_error = error;
1459 }
1460 buf_biodone(real_bp);
1461 }
1462 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 22)) | DBG_FUNC_END, (int)f_offset, size, upl_offset, retval, 0);
1463
1464 return (retval);
1465 }
1466
1467
1468 static int
1469 cluster_read_prefetch(vnode_t vp, off_t f_offset, u_int size, off_t filesize, int (*callback)(buf_t, void *), void *callback_arg, int bflag)
1470 {
1471 int pages_in_prefetch;
1472
1473 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 49)) | DBG_FUNC_START,
1474 (int)f_offset, size, (int)filesize, 0, 0);
1475
1476 if (f_offset >= filesize) {
1477 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 49)) | DBG_FUNC_END,
1478 (int)f_offset, 0, 0, 0, 0);
1479 return(0);
1480 }
1481 if ((off_t)size > (filesize - f_offset))
1482 size = filesize - f_offset;
1483 pages_in_prefetch = (size + (PAGE_SIZE - 1)) / PAGE_SIZE;
1484
1485 advisory_read_ext(vp, filesize, f_offset, size, callback, callback_arg, bflag);
1486
1487 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 49)) | DBG_FUNC_END,
1488 (int)f_offset + size, pages_in_prefetch, 0, 1, 0);
1489
1490 return (pages_in_prefetch);
1491 }
1492
1493
1494
1495 static void
1496 cluster_read_ahead(vnode_t vp, struct cl_extent *extent, off_t filesize, struct cl_readahead *rap, int (*callback)(buf_t, void *), void *callback_arg,
1497 int bflag)
1498 {
1499 daddr64_t r_addr;
1500 off_t f_offset;
1501 int size_of_prefetch;
1502 u_int max_prefetch;
1503
1504
1505 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_START,
1506 (int)extent->b_addr, (int)extent->e_addr, (int)rap->cl_lastr, 0, 0);
1507
1508 if (extent->b_addr == rap->cl_lastr && extent->b_addr == extent->e_addr) {
1509 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_END,
1510 rap->cl_ralen, (int)rap->cl_maxra, (int)rap->cl_lastr, 0, 0);
1511 return;
1512 }
1513 if (rap->cl_lastr == -1 || (extent->b_addr != rap->cl_lastr && extent->b_addr != (rap->cl_lastr + 1))) {
1514 rap->cl_ralen = 0;
1515 rap->cl_maxra = 0;
1516
1517 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_END,
1518 rap->cl_ralen, (int)rap->cl_maxra, (int)rap->cl_lastr, 1, 0);
1519
1520 return;
1521 }
1522 max_prefetch = MAX_PREFETCH(vp);
1523
1524 if (extent->e_addr < rap->cl_maxra) {
1525 if ((rap->cl_maxra - extent->e_addr) > ((max_prefetch / PAGE_SIZE) / 4)) {
1526
1527 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_END,
1528 rap->cl_ralen, (int)rap->cl_maxra, (int)rap->cl_lastr, 2, 0);
1529 return;
1530 }
1531 }
1532 r_addr = max(extent->e_addr, rap->cl_maxra) + 1;
1533 f_offset = (off_t)(r_addr * PAGE_SIZE_64);
1534
1535 size_of_prefetch = 0;
1536
1537 ubc_range_op(vp, f_offset, f_offset + PAGE_SIZE_64, UPL_ROP_PRESENT, &size_of_prefetch);
1538
1539 if (size_of_prefetch) {
1540 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_END,
1541 rap->cl_ralen, (int)rap->cl_maxra, (int)rap->cl_lastr, 3, 0);
1542 return;
1543 }
1544 if (f_offset < filesize) {
1545 daddr64_t read_size;
1546
1547 rap->cl_ralen = rap->cl_ralen ? min(max_prefetch / PAGE_SIZE, rap->cl_ralen << 1) : 1;
1548
1549 read_size = (extent->e_addr + 1) - extent->b_addr;
1550
1551 if (read_size > rap->cl_ralen) {
1552 if (read_size > max_prefetch / PAGE_SIZE)
1553 rap->cl_ralen = max_prefetch / PAGE_SIZE;
1554 else
1555 rap->cl_ralen = read_size;
1556 }
1557 size_of_prefetch = cluster_read_prefetch(vp, f_offset, rap->cl_ralen * PAGE_SIZE, filesize, callback, callback_arg, bflag);
1558
1559 if (size_of_prefetch)
1560 rap->cl_maxra = (r_addr + size_of_prefetch) - 1;
1561 }
1562 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_END,
1563 rap->cl_ralen, (int)rap->cl_maxra, (int)rap->cl_lastr, 4, 0);
1564 }
1565
1566
1567 int
1568 cluster_pageout(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset,
1569 int size, off_t filesize, int flags)
1570 {
1571 return cluster_pageout_ext(vp, upl, upl_offset, f_offset, size, filesize, flags, NULL, NULL);
1572
1573 }
1574
1575
1576 int
1577 cluster_pageout_ext(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset,
1578 int size, off_t filesize, int flags, int (*callback)(buf_t, void *), void *callback_arg)
1579 {
1580 int io_size;
1581 int rounded_size;
1582 off_t max_size;
1583 int local_flags;
1584
1585 if (vp->v_mount->mnt_kern_flag & MNTK_VIRTUALDEV)
1586 /*
1587 * if we know we're issuing this I/O to a virtual device (i.e. disk image)
1588 * then we don't want to enforce this throttle... if we do, we can
1589 * potentially deadlock since we're stalling the pageout thread at a time
1590 * when the disk image might need additional memory (which won't be available
1591 * if the pageout thread can't run)... instead we'll just depend on the throttle
1592 * that the pageout thread now has in place to deal with external files
1593 */
1594 local_flags = CL_PAGEOUT;
1595 else
1596 local_flags = CL_PAGEOUT | CL_THROTTLE;
1597
1598 if ((flags & UPL_IOSYNC) == 0)
1599 local_flags |= CL_ASYNC;
1600 if ((flags & UPL_NOCOMMIT) == 0)
1601 local_flags |= CL_COMMIT;
1602 if ((flags & UPL_KEEPCACHED))
1603 local_flags |= CL_KEEPCACHED;
1604 if (flags & IO_PASSIVE)
1605 local_flags |= CL_PASSIVE;
1606
1607
1608 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 52)) | DBG_FUNC_NONE,
1609 (int)f_offset, size, (int)filesize, local_flags, 0);
1610
1611 /*
1612 * If they didn't specify any I/O, then we are done...
1613 * we can't issue an abort because we don't know how
1614 * big the upl really is
1615 */
1616 if (size <= 0)
1617 return (EINVAL);
1618
1619 if (vp->v_mount->mnt_flag & MNT_RDONLY) {
1620 if (local_flags & CL_COMMIT)
1621 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
1622 return (EROFS);
1623 }
1624 /*
1625 * can't page-in from a negative offset
1626 * or if we're starting beyond the EOF
1627 * or if the file offset isn't page aligned
1628 * or the size requested isn't a multiple of PAGE_SIZE
1629 */
1630 if (f_offset < 0 || f_offset >= filesize ||
1631 (f_offset & PAGE_MASK_64) || (size & PAGE_MASK)) {
1632 if (local_flags & CL_COMMIT)
1633 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
1634 return (EINVAL);
1635 }
1636 max_size = filesize - f_offset;
1637
1638 if (size < max_size)
1639 io_size = size;
1640 else
1641 io_size = max_size;
1642
1643 rounded_size = (io_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
1644
1645 if (size > rounded_size) {
1646 if (local_flags & CL_COMMIT)
1647 ubc_upl_abort_range(upl, upl_offset + rounded_size, size - rounded_size,
1648 UPL_ABORT_FREE_ON_EMPTY);
1649 }
1650 return (cluster_io(vp, upl, upl_offset, f_offset, io_size,
1651 local_flags, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg));
1652 }
1653
1654
1655 int
1656 cluster_pagein(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset,
1657 int size, off_t filesize, int flags)
1658 {
1659 return cluster_pagein_ext(vp, upl, upl_offset, f_offset, size, filesize, flags, NULL, NULL);
1660 }
1661
1662
1663 int
1664 cluster_pagein_ext(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset,
1665 int size, off_t filesize, int flags, int (*callback)(buf_t, void *), void *callback_arg)
1666 {
1667 u_int io_size;
1668 int rounded_size;
1669 off_t max_size;
1670 int retval;
1671 int local_flags = 0;
1672
1673 if (upl == NULL || size < 0)
1674 panic("cluster_pagein: NULL upl passed in");
1675
1676 if ((flags & UPL_IOSYNC) == 0)
1677 local_flags |= CL_ASYNC;
1678 if ((flags & UPL_NOCOMMIT) == 0)
1679 local_flags |= CL_COMMIT;
1680 if (flags & IO_PASSIVE)
1681 local_flags |= CL_PASSIVE;
1682
1683
1684 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 56)) | DBG_FUNC_NONE,
1685 (int)f_offset, size, (int)filesize, local_flags, 0);
1686
1687 /*
1688 * can't page-in from a negative offset
1689 * or if we're starting beyond the EOF
1690 * or if the file offset isn't page aligned
1691 * or the size requested isn't a multiple of PAGE_SIZE
1692 */
1693 if (f_offset < 0 || f_offset >= filesize ||
1694 (f_offset & PAGE_MASK_64) || (size & PAGE_MASK) || (upl_offset & PAGE_MASK)) {
1695 if (local_flags & CL_COMMIT)
1696 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);
1697 return (EINVAL);
1698 }
1699 max_size = filesize - f_offset;
1700
1701 if (size < max_size)
1702 io_size = size;
1703 else
1704 io_size = max_size;
1705
1706 rounded_size = (io_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
1707
1708 if (size > rounded_size && (local_flags & CL_COMMIT))
1709 ubc_upl_abort_range(upl, upl_offset + rounded_size,
1710 size - rounded_size, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);
1711
1712 retval = cluster_io(vp, upl, upl_offset, f_offset, io_size,
1713 local_flags | CL_READ | CL_PAGEIN, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
1714
1715 return (retval);
1716 }
1717
1718
1719 int
1720 cluster_bp(buf_t bp)
1721 {
1722 return cluster_bp_ext(bp, NULL, NULL);
1723 }
1724
1725
1726 int
1727 cluster_bp_ext(buf_t bp, int (*callback)(buf_t, void *), void *callback_arg)
1728 {
1729 off_t f_offset;
1730 int flags;
1731
1732 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 19)) | DBG_FUNC_START,
1733 (int)bp, (int)bp->b_lblkno, bp->b_bcount, bp->b_flags, 0);
1734
1735 if (bp->b_flags & B_READ)
1736 flags = CL_ASYNC | CL_READ;
1737 else
1738 flags = CL_ASYNC;
1739 if (bp->b_flags & B_PASSIVE)
1740 flags |= CL_PASSIVE;
1741
1742 f_offset = ubc_blktooff(bp->b_vp, bp->b_lblkno);
1743
1744 return (cluster_io(bp->b_vp, bp->b_upl, 0, f_offset, bp->b_bcount, flags, bp, (struct clios *)NULL, callback, callback_arg));
1745 }
1746
1747
1748
1749 int
1750 cluster_write(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF, off_t headOff, off_t tailOff, int xflags)
1751 {
1752 return cluster_write_ext(vp, uio, oldEOF, newEOF, headOff, tailOff, xflags, NULL, NULL);
1753 }
1754
1755
1756 int
1757 cluster_write_ext(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF, off_t headOff, off_t tailOff,
1758 int xflags, int (*callback)(buf_t, void *), void *callback_arg)
1759 {
1760 user_ssize_t cur_resid;
1761 int retval = 0;
1762 int flags;
1763 int zflags;
1764 int bflag;
1765 int write_type = IO_COPY;
1766 u_int32_t write_length;
1767
1768 flags = xflags;
1769
1770 if (flags & IO_PASSIVE)
1771 bflag = CL_PASSIVE;
1772 else
1773 bflag = 0;
1774
1775 if (vp->v_flag & VNOCACHE_DATA)
1776 flags |= IO_NOCACHE;
1777
1778 if (uio == NULL) {
1779 /*
1780 * no user data...
1781 * this call is being made to zero-fill some range in the file
1782 */
1783 retval = cluster_write_copy(vp, NULL, (u_int32_t)0, oldEOF, newEOF, headOff, tailOff, flags, callback, callback_arg);
1784
1785 return(retval);
1786 }
1787 /*
1788 * do a write through the cache if one of the following is true....
1789 * NOCACHE is not true and
1790 * the uio request doesn't target USERSPACE
1791 * otherwise, find out if we want the direct or contig variant for
1792 * the first vector in the uio request
1793 */
1794 if ( (flags & IO_NOCACHE) && UIO_SEG_IS_USER_SPACE(uio->uio_segflg) )
1795 retval = cluster_io_type(uio, &write_type, &write_length, MIN_DIRECT_WRITE_SIZE);
1796
1797 if ( (flags & (IO_TAILZEROFILL | IO_HEADZEROFILL)) && write_type == IO_DIRECT)
1798 /*
1799 * must go through the cached variant in this case
1800 */
1801 write_type = IO_COPY;
1802
1803 while ((cur_resid = uio_resid(uio)) && uio->uio_offset < newEOF && retval == 0) {
1804
1805 switch (write_type) {
1806
1807 case IO_COPY:
1808 /*
1809 * make sure the uio_resid isn't too big...
1810 * internally, we want to handle all of the I/O in
1811 * chunk sizes that fit in a 32 bit int
1812 */
1813 if (cur_resid > (user_ssize_t)(MAX_IO_REQUEST_SIZE)) {
1814 /*
1815 * we're going to have to call cluster_write_copy
1816 * more than once...
1817 *
1818 * only want the last call to cluster_write_copy to
1819 * have the IO_TAILZEROFILL flag set and only the
1820 * first call should have IO_HEADZEROFILL
1821 */
1822 zflags = flags & ~IO_TAILZEROFILL;
1823 flags &= ~IO_HEADZEROFILL;
1824
1825 write_length = MAX_IO_REQUEST_SIZE;
1826 } else {
1827 /*
1828 * last call to cluster_write_copy
1829 */
1830 zflags = flags;
1831
1832 write_length = (u_int32_t)cur_resid;
1833 }
1834 retval = cluster_write_copy(vp, uio, write_length, oldEOF, newEOF, headOff, tailOff, zflags, callback, callback_arg);
1835 break;
1836
1837 case IO_CONTIG:
1838 zflags = flags & ~(IO_TAILZEROFILL | IO_HEADZEROFILL);
1839
1840 if (flags & IO_HEADZEROFILL) {
1841 /*
1842 * only do this once per request
1843 */
1844 flags &= ~IO_HEADZEROFILL;
1845
1846 retval = cluster_write_copy(vp, (struct uio *)0, (u_int32_t)0, (off_t)0, uio->uio_offset,
1847 headOff, (off_t)0, zflags | IO_HEADZEROFILL | IO_SYNC, callback, callback_arg);
1848 if (retval)
1849 break;
1850 }
1851 retval = cluster_write_contig(vp, uio, newEOF, &write_type, &write_length, callback, callback_arg, bflag);
1852
1853 if (retval == 0 && (flags & IO_TAILZEROFILL) && uio_resid(uio) == 0) {
1854 /*
1855 * we're done with the data from the user specified buffer(s)
1856 * and we've been requested to zero fill at the tail
1857 * treat this as an IO_HEADZEROFILL which doesn't require a uio
1858 * by rearranging the args and passing in IO_HEADZEROFILL
1859 */
1860 retval = cluster_write_copy(vp, (struct uio *)0, (u_int32_t)0, (off_t)0, tailOff, uio->uio_offset,
1861 (off_t)0, zflags | IO_HEADZEROFILL | IO_SYNC, callback, callback_arg);
1862 }
1863 break;
1864
1865 case IO_DIRECT:
1866 /*
1867 * cluster_write_direct is never called with IO_TAILZEROFILL || IO_HEADZEROFILL
1868 */
1869 retval = cluster_write_direct(vp, uio, oldEOF, newEOF, &write_type, &write_length, flags, callback, callback_arg);
1870 break;
1871
1872 case IO_UNKNOWN:
1873 retval = cluster_io_type(uio, &write_type, &write_length, MIN_DIRECT_WRITE_SIZE);
1874 break;
1875 }
1876 }
1877 return (retval);
1878 }
1879
1880
1881 static int
1882 cluster_write_direct(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF, int *write_type, u_int32_t *write_length,
1883 int flags, int (*callback)(buf_t, void *), void *callback_arg)
1884 {
1885 upl_t upl;
1886 upl_page_info_t *pl;
1887 vm_offset_t upl_offset;
1888 u_int32_t io_req_size;
1889 u_int32_t offset_in_file;
1890 u_int32_t offset_in_iovbase;
1891 u_int32_t io_size;
1892 int io_flag;
1893 int bflag;
1894 vm_size_t upl_size;
1895 vm_size_t upl_needed_size;
1896 mach_msg_type_number_t pages_in_pl;
1897 int upl_flags;
1898 kern_return_t kret;
1899 mach_msg_type_number_t i;
1900 int force_data_sync;
1901 int retval = 0;
1902 int first_IO = 1;
1903 struct clios iostate;
1904 user_addr_t iov_base;
1905 u_int32_t mem_alignment_mask;
1906 u_int32_t devblocksize;
1907 u_int32_t max_upl_size;
1908
1909
1910 max_upl_size = cluster_max_io_size(vp->v_mount, CL_WRITE);
1911
1912 if (flags & IO_PASSIVE)
1913 bflag = CL_PASSIVE;
1914 else
1915 bflag = 0;
1916
1917 /*
1918 * When we enter this routine, we know
1919 * -- the resid will not exceed iov_len
1920 */
1921 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 75)) | DBG_FUNC_START,
1922 (int)uio->uio_offset, *write_length, (int)newEOF, 0, 0);
1923
1924 iostate.io_completed = 0;
1925 iostate.io_issued = 0;
1926 iostate.io_error = 0;
1927 iostate.io_wanted = 0;
1928
1929 mem_alignment_mask = (u_int32_t)vp->v_mount->mnt_alignmentmask;
1930 devblocksize = (u_int32_t)vp->v_mount->mnt_devblocksize;
1931
1932 if (devblocksize == 1) {
1933 /*
1934 * the AFP client advertises a devblocksize of 1
1935 * however, its BLOCKMAP routine maps to physical
1936 * blocks that are PAGE_SIZE in size...
1937 * therefore we can't ask for I/Os that aren't page aligned
1938 * or aren't multiples of PAGE_SIZE in size
1939 * by setting devblocksize to PAGE_SIZE, we re-instate
1940 * the old behavior we had before the mem_alignment_mask
1941 * changes went in...
1942 */
1943 devblocksize = PAGE_SIZE;
1944 }
1945
1946 next_dwrite:
1947 io_req_size = *write_length;
1948 iov_base = uio_curriovbase(uio);
1949
1950 offset_in_file = (u_int32_t)uio->uio_offset & PAGE_MASK;
1951 offset_in_iovbase = (u_int32_t)iov_base & mem_alignment_mask;
1952
1953 if (offset_in_file || offset_in_iovbase) {
1954 /*
1955 * one of the 2 important offsets is misaligned
1956 * so fire an I/O through the cache for this entire vector
1957 */
1958 goto wait_for_dwrites;
1959 }
1960 if (iov_base & (devblocksize - 1)) {
1961 /*
1962 * the offset in memory must be on a device block boundary
1963 * so that we can guarantee that we can generate an
1964 * I/O that ends on a page boundary in cluster_io
1965 */
1966 goto wait_for_dwrites;
1967 }
1968
1969 while (io_req_size >= PAGE_SIZE && uio->uio_offset < newEOF && retval == 0) {
1970
1971 if (first_IO) {
1972 cluster_syncup(vp, newEOF, callback, callback_arg);
1973 first_IO = 0;
1974 }
1975 io_size = io_req_size & ~PAGE_MASK;
1976 iov_base = uio_curriovbase(uio);
1977
1978 if (io_size > max_upl_size)
1979 io_size = max_upl_size;
1980
1981 upl_offset = (vm_offset_t)((u_int32_t)iov_base & PAGE_MASK);
1982 upl_needed_size = (upl_offset + io_size + (PAGE_SIZE -1)) & ~PAGE_MASK;
1983
1984 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 76)) | DBG_FUNC_START,
1985 (int)upl_offset, upl_needed_size, (int)iov_base, io_size, 0);
1986
1987 for (force_data_sync = 0; force_data_sync < 3; force_data_sync++) {
1988 pages_in_pl = 0;
1989 upl_size = upl_needed_size;
1990 upl_flags = UPL_FILE_IO | UPL_COPYOUT_FROM | UPL_NO_SYNC |
1991 UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE;
1992
1993 kret = vm_map_get_upl(current_map(),
1994 (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)),
1995 &upl_size,
1996 &upl,
1997 NULL,
1998 &pages_in_pl,
1999 &upl_flags,
2000 force_data_sync);
2001
2002 if (kret != KERN_SUCCESS) {
2003 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 76)) | DBG_FUNC_END,
2004 0, 0, 0, kret, 0);
2005 /*
2006 * failed to get pagelist
2007 *
2008 * we may have already spun some portion of this request
2009 * off as async requests... we need to wait for the I/O
2010 * to complete before returning
2011 */
2012 goto wait_for_dwrites;
2013 }
2014 pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
2015 pages_in_pl = upl_size / PAGE_SIZE;
2016
2017 for (i = 0; i < pages_in_pl; i++) {
2018 if (!upl_valid_page(pl, i))
2019 break;
2020 }
2021 if (i == pages_in_pl)
2022 break;
2023
2024 /*
2025 * didn't get all the pages back that we
2026 * needed... release this upl and try again
2027 */
2028 ubc_upl_abort(upl, 0);
2029 }
2030 if (force_data_sync >= 3) {
2031 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 76)) | DBG_FUNC_END,
2032 i, pages_in_pl, upl_size, kret, 0);
2033 /*
2034 * for some reason, we couldn't acquire a hold on all
2035 * the pages needed in the user's address space
2036 *
2037 * we may have already spun some portion of this request
2038 * off as async requests... we need to wait for the I/O
2039 * to complete before returning
2040 */
2041 goto wait_for_dwrites;
2042 }
2043
2044 /*
2045 * Consider the possibility that upl_size wasn't satisfied.
2046 */
2047 if (upl_size < upl_needed_size) {
2048 if (upl_size && upl_offset == 0)
2049 io_size = upl_size;
2050 else
2051 io_size = 0;
2052 }
2053 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 76)) | DBG_FUNC_END,
2054 (int)upl_offset, upl_size, (int)iov_base, io_size, 0);
2055
2056 if (io_size == 0) {
2057 ubc_upl_abort(upl, 0);
2058 /*
2059 * we may have already spun some portion of this request
2060 * off as async requests... we need to wait for the I/O
2061 * to complete before returning
2062 */
2063 goto wait_for_dwrites;
2064 }
2065
2066 /*
2067 * Now look for pages already in the cache
2068 * and throw them away.
2069 * uio->uio_offset is page aligned within the file
2070 * io_size is a multiple of PAGE_SIZE
2071 */
2072 ubc_range_op(vp, uio->uio_offset, uio->uio_offset + io_size, UPL_ROP_DUMP, NULL);
2073
2074 /*
2075 * we want push out these writes asynchronously so that we can overlap
2076 * the preparation of the next I/O
2077 * if there are already too many outstanding writes
2078 * wait until some complete before issuing the next
2079 */
2080 lck_mtx_lock(cl_mtxp);
2081
2082 while ((iostate.io_issued - iostate.io_completed) > (2 * max_upl_size)) {
2083
2084 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_START,
2085 iostate.io_issued, iostate.io_completed, 2 * max_upl_size, 0, 0);
2086
2087 iostate.io_wanted = 1;
2088 msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_write_direct", NULL);
2089
2090 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_END,
2091 iostate.io_issued, iostate.io_completed, 2 * max_upl_size, 0, 0);
2092 }
2093 lck_mtx_unlock(cl_mtxp);
2094
2095 if (iostate.io_error) {
2096 /*
2097 * one of the earlier writes we issued ran into a hard error
2098 * don't issue any more writes, cleanup the UPL
2099 * that was just created but not used, then
2100 * go wait for all writes that are part of this stream
2101 * to complete before returning the error to the caller
2102 */
2103 ubc_upl_abort(upl, 0);
2104
2105 goto wait_for_dwrites;
2106 }
2107 io_flag = CL_ASYNC | CL_PRESERVE | CL_COMMIT | CL_THROTTLE | CL_DIRECT_IO | bflag;
2108
2109 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 77)) | DBG_FUNC_START,
2110 (int)upl_offset, (int)uio->uio_offset, io_size, io_flag, 0);
2111
2112 retval = cluster_io(vp, upl, upl_offset, uio->uio_offset,
2113 io_size, io_flag, (buf_t)NULL, &iostate, callback, callback_arg);
2114
2115 /*
2116 * update the uio structure to
2117 * reflect the I/O that we just issued
2118 */
2119 uio_update(uio, (user_size_t)io_size);
2120
2121 io_req_size -= io_size;
2122
2123 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 77)) | DBG_FUNC_END,
2124 (int)upl_offset, (int)uio->uio_offset, io_req_size, retval, 0);
2125
2126 } /* end while */
2127
2128 if (retval == 0 && iostate.io_error == 0 && io_req_size == 0) {
2129
2130 retval = cluster_io_type(uio, write_type, write_length, MIN_DIRECT_WRITE_SIZE);
2131
2132 if (retval == 0 && *write_type == IO_DIRECT) {
2133
2134 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 75)) | DBG_FUNC_NONE,
2135 (int)uio->uio_offset, *write_length, (int)newEOF, 0, 0);
2136
2137 goto next_dwrite;
2138 }
2139 }
2140
2141 wait_for_dwrites:
2142 if (iostate.io_issued) {
2143 /*
2144 * make sure all async writes issued as part of this stream
2145 * have completed before we return
2146 */
2147 lck_mtx_lock(cl_mtxp);
2148
2149 while (iostate.io_issued != iostate.io_completed) {
2150 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_START,
2151 iostate.io_issued, iostate.io_completed, 0, 0, 0);
2152
2153 iostate.io_wanted = 1;
2154 msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_write_direct", NULL);
2155
2156 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_END,
2157 iostate.io_issued, iostate.io_completed, 0, 0, 0);
2158 }
2159 lck_mtx_unlock(cl_mtxp);
2160 }
2161 if (iostate.io_error)
2162 retval = iostate.io_error;
2163
2164 if (io_req_size && retval == 0) {
2165 /*
2166 * we couldn't handle the tail of this request in DIRECT mode
2167 * so fire it through the copy path
2168 *
2169 * note that flags will never have IO_HEADZEROFILL or IO_TAILZEROFILL set
2170 * so we can just pass 0 in for the headOff and tailOff
2171 */
2172 retval = cluster_write_copy(vp, uio, io_req_size, oldEOF, newEOF, (off_t)0, (off_t)0, flags, callback, callback_arg);
2173
2174 *write_type = IO_UNKNOWN;
2175 }
2176 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 75)) | DBG_FUNC_END,
2177 (int)uio->uio_offset, io_req_size, retval, 4, 0);
2178
2179 return (retval);
2180 }
2181
2182
2183 static int
2184 cluster_write_contig(vnode_t vp, struct uio *uio, off_t newEOF, int *write_type, u_int32_t *write_length,
2185 int (*callback)(buf_t, void *), void *callback_arg, int bflag)
2186 {
2187 upl_page_info_t *pl;
2188 addr64_t src_paddr = 0;
2189 upl_t upl[MAX_VECTS];
2190 vm_offset_t upl_offset;
2191 u_int32_t tail_size = 0;
2192 u_int32_t io_size;
2193 u_int32_t xsize;
2194 vm_size_t upl_size;
2195 vm_size_t upl_needed_size;
2196 mach_msg_type_number_t pages_in_pl;
2197 int upl_flags;
2198 kern_return_t kret;
2199 struct clios iostate;
2200 int error = 0;
2201 int cur_upl = 0;
2202 int num_upl = 0;
2203 int n;
2204 user_addr_t iov_base;
2205 u_int32_t devblocksize;
2206 u_int32_t mem_alignment_mask;
2207
2208 /*
2209 * When we enter this routine, we know
2210 * -- the io_req_size will not exceed iov_len
2211 * -- the target address is physically contiguous
2212 */
2213 cluster_syncup(vp, newEOF, callback, callback_arg);
2214
2215 devblocksize = (u_int32_t)vp->v_mount->mnt_devblocksize;
2216 mem_alignment_mask = (u_int32_t)vp->v_mount->mnt_alignmentmask;
2217
2218 iostate.io_completed = 0;
2219 iostate.io_issued = 0;
2220 iostate.io_error = 0;
2221 iostate.io_wanted = 0;
2222
2223 next_cwrite:
2224 io_size = *write_length;
2225
2226 iov_base = uio_curriovbase(uio);
2227
2228 upl_offset = (vm_offset_t)((u_int32_t)iov_base & PAGE_MASK);
2229 upl_needed_size = upl_offset + io_size;
2230
2231 pages_in_pl = 0;
2232 upl_size = upl_needed_size;
2233 upl_flags = UPL_FILE_IO | UPL_COPYOUT_FROM | UPL_NO_SYNC |
2234 UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE;
2235
2236 kret = vm_map_get_upl(current_map(),
2237 (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)),
2238 &upl_size, &upl[cur_upl], NULL, &pages_in_pl, &upl_flags, 0);
2239
2240 if (kret != KERN_SUCCESS) {
2241 /*
2242 * failed to get pagelist
2243 */
2244 error = EINVAL;
2245 goto wait_for_cwrites;
2246 }
2247 num_upl++;
2248
2249 /*
2250 * Consider the possibility that upl_size wasn't satisfied.
2251 */
2252 if (upl_size < upl_needed_size) {
2253 /*
2254 * This is a failure in the physical memory case.
2255 */
2256 error = EINVAL;
2257 goto wait_for_cwrites;
2258 }
2259 pl = ubc_upl_pageinfo(upl[cur_upl]);
2260
2261 src_paddr = ((addr64_t)upl_phys_page(pl, 0) << 12) + (addr64_t)upl_offset;
2262
2263 while (((uio->uio_offset & (devblocksize - 1)) || io_size < devblocksize) && io_size) {
2264 u_int32_t head_size;
2265
2266 head_size = devblocksize - (u_int32_t)(uio->uio_offset & (devblocksize - 1));
2267
2268 if (head_size > io_size)
2269 head_size = io_size;
2270
2271 error = cluster_align_phys_io(vp, uio, src_paddr, head_size, 0, callback, callback_arg);
2272
2273 if (error)
2274 goto wait_for_cwrites;
2275
2276 upl_offset += head_size;
2277 src_paddr += head_size;
2278 io_size -= head_size;
2279
2280 iov_base += head_size;
2281 }
2282 if ((u_int32_t)iov_base & mem_alignment_mask) {
2283 /*
2284 * request doesn't set up on a memory boundary
2285 * the underlying DMA engine can handle...
2286 * return an error instead of going through
2287 * the slow copy path since the intent of this
2288 * path is direct I/O from device memory
2289 */
2290 error = EINVAL;
2291 goto wait_for_cwrites;
2292 }
2293
2294 tail_size = io_size & (devblocksize - 1);
2295 io_size -= tail_size;
2296
2297 while (io_size && error == 0) {
2298
2299 if (io_size > MAX_IO_CONTIG_SIZE)
2300 xsize = MAX_IO_CONTIG_SIZE;
2301 else
2302 xsize = io_size;
2303 /*
2304 * request asynchronously so that we can overlap
2305 * the preparation of the next I/O... we'll do
2306 * the commit after all the I/O has completed
2307 * since its all issued against the same UPL
2308 * if there are already too many outstanding writes
2309 * wait until some have completed before issuing the next
2310 */
2311 if (iostate.io_issued) {
2312 lck_mtx_lock(cl_mtxp);
2313
2314 while ((iostate.io_issued - iostate.io_completed) > (2 * MAX_IO_CONTIG_SIZE)) {
2315
2316 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_START,
2317 iostate.io_issued, iostate.io_completed, 2 * MAX_IO_CONTIG_SIZE, 0, 0);
2318
2319 iostate.io_wanted = 1;
2320 msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_write_contig", NULL);
2321
2322 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_END,
2323 iostate.io_issued, iostate.io_completed, 2 * MAX_IO_CONTIG_SIZE, 0, 0);
2324 }
2325 lck_mtx_unlock(cl_mtxp);
2326 }
2327 if (iostate.io_error) {
2328 /*
2329 * one of the earlier writes we issued ran into a hard error
2330 * don't issue any more writes...
2331 * go wait for all writes that are part of this stream
2332 * to complete before returning the error to the caller
2333 */
2334 goto wait_for_cwrites;
2335 }
2336 /*
2337 * issue an asynchronous write to cluster_io
2338 */
2339 error = cluster_io(vp, upl[cur_upl], upl_offset, uio->uio_offset,
2340 xsize, CL_DEV_MEMORY | CL_ASYNC | bflag, (buf_t)NULL, (struct clios *)&iostate, callback, callback_arg);
2341
2342 if (error == 0) {
2343 /*
2344 * The cluster_io write completed successfully,
2345 * update the uio structure
2346 */
2347 uio_update(uio, (user_size_t)xsize);
2348
2349 upl_offset += xsize;
2350 src_paddr += xsize;
2351 io_size -= xsize;
2352 }
2353 }
2354 if (error == 0 && iostate.io_error == 0 && tail_size == 0 && num_upl < MAX_VECTS) {
2355
2356 error = cluster_io_type(uio, write_type, write_length, 0);
2357
2358 if (error == 0 && *write_type == IO_CONTIG) {
2359 cur_upl++;
2360 goto next_cwrite;
2361 }
2362 } else
2363 *write_type = IO_UNKNOWN;
2364
2365 wait_for_cwrites:
2366 /*
2367 * make sure all async writes that are part of this stream
2368 * have completed before we proceed
2369 */
2370 lck_mtx_lock(cl_mtxp);
2371
2372 while (iostate.io_issued != iostate.io_completed) {
2373 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_START,
2374 iostate.io_issued, iostate.io_completed, 0, 0, 0);
2375
2376 iostate.io_wanted = 1;
2377 msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_write_contig", NULL);
2378
2379 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_END,
2380 iostate.io_issued, iostate.io_completed, 0, 0, 0);
2381 }
2382 lck_mtx_unlock(cl_mtxp);
2383
2384 if (iostate.io_error)
2385 error = iostate.io_error;
2386
2387 if (error == 0 && tail_size)
2388 error = cluster_align_phys_io(vp, uio, src_paddr, tail_size, 0, callback, callback_arg);
2389
2390 for (n = 0; n < num_upl; n++)
2391 /*
2392 * just release our hold on each physically contiguous
2393 * region without changing any state
2394 */
2395 ubc_upl_abort(upl[n], 0);
2396
2397 return (error);
2398 }
2399
2400
2401 static int
2402 cluster_write_copy(vnode_t vp, struct uio *uio, u_int32_t io_req_size, off_t oldEOF, off_t newEOF, off_t headOff,
2403 off_t tailOff, int flags, int (*callback)(buf_t, void *), void *callback_arg)
2404 {
2405 upl_page_info_t *pl;
2406 upl_t upl;
2407 vm_offset_t upl_offset = 0;
2408 vm_size_t upl_size;
2409 off_t upl_f_offset;
2410 int pages_in_upl;
2411 int start_offset;
2412 int xfer_resid;
2413 int io_size;
2414 int io_offset;
2415 int bytes_to_zero;
2416 int bytes_to_move;
2417 kern_return_t kret;
2418 int retval = 0;
2419 int io_resid;
2420 long long total_size;
2421 long long zero_cnt;
2422 off_t zero_off;
2423 long long zero_cnt1;
2424 off_t zero_off1;
2425 struct cl_extent cl;
2426 int intersection;
2427 struct cl_writebehind *wbp;
2428 int bflag;
2429 u_int max_cluster_pgcount;
2430 u_int max_io_size;
2431
2432 if (flags & IO_PASSIVE)
2433 bflag = CL_PASSIVE;
2434 else
2435 bflag = 0;
2436
2437 if (uio) {
2438 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 40)) | DBG_FUNC_START,
2439 (int)uio->uio_offset, io_req_size, (int)oldEOF, (int)newEOF, 0);
2440
2441 io_resid = io_req_size;
2442 } else {
2443 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 40)) | DBG_FUNC_START,
2444 0, 0, (int)oldEOF, (int)newEOF, 0);
2445
2446 io_resid = 0;
2447 }
2448 zero_cnt = 0;
2449 zero_cnt1 = 0;
2450 zero_off = 0;
2451 zero_off1 = 0;
2452
2453 max_cluster_pgcount = MAX_CLUSTER_SIZE(vp) / PAGE_SIZE;
2454 max_io_size = cluster_max_io_size(vp->v_mount, CL_WRITE);
2455
2456 if (flags & IO_HEADZEROFILL) {
2457 /*
2458 * some filesystems (HFS is one) don't support unallocated holes within a file...
2459 * so we zero fill the intervening space between the old EOF and the offset
2460 * where the next chunk of real data begins.... ftruncate will also use this
2461 * routine to zero fill to the new EOF when growing a file... in this case, the
2462 * uio structure will not be provided
2463 */
2464 if (uio) {
2465 if (headOff < uio->uio_offset) {
2466 zero_cnt = uio->uio_offset - headOff;
2467 zero_off = headOff;
2468 }
2469 } else if (headOff < newEOF) {
2470 zero_cnt = newEOF - headOff;
2471 zero_off = headOff;
2472 }
2473 }
2474 if (flags & IO_TAILZEROFILL) {
2475 if (uio) {
2476 zero_off1 = uio->uio_offset + io_req_size;
2477
2478 if (zero_off1 < tailOff)
2479 zero_cnt1 = tailOff - zero_off1;
2480 }
2481 }
2482 if (zero_cnt == 0 && uio == (struct uio *) 0) {
2483 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 40)) | DBG_FUNC_END,
2484 retval, 0, 0, 0, 0);
2485 return (0);
2486 }
2487
2488 while ((total_size = (io_resid + zero_cnt + zero_cnt1)) && retval == 0) {
2489 /*
2490 * for this iteration of the loop, figure out where our starting point is
2491 */
2492 if (zero_cnt) {
2493 start_offset = (int)(zero_off & PAGE_MASK_64);
2494 upl_f_offset = zero_off - start_offset;
2495 } else if (io_resid) {
2496 start_offset = (int)(uio->uio_offset & PAGE_MASK_64);
2497 upl_f_offset = uio->uio_offset - start_offset;
2498 } else {
2499 start_offset = (int)(zero_off1 & PAGE_MASK_64);
2500 upl_f_offset = zero_off1 - start_offset;
2501 }
2502 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 46)) | DBG_FUNC_NONE,
2503 (int)zero_off, (int)zero_cnt, (int)zero_off1, (int)zero_cnt1, 0);
2504
2505 if (total_size > max_io_size)
2506 total_size = max_io_size;
2507
2508 cl.b_addr = (daddr64_t)(upl_f_offset / PAGE_SIZE_64);
2509
2510 if (uio && ((flags & (IO_SYNC | IO_HEADZEROFILL | IO_TAILZEROFILL)) == 0)) {
2511 /*
2512 * assumption... total_size <= io_resid
2513 * because IO_HEADZEROFILL and IO_TAILZEROFILL not set
2514 */
2515 if ((start_offset + total_size) > max_io_size)
2516 total_size -= start_offset;
2517 xfer_resid = total_size;
2518
2519 retval = cluster_copy_ubc_data_internal(vp, uio, &xfer_resid, 1, 1);
2520
2521 if (retval)
2522 break;
2523
2524 io_resid -= (total_size - xfer_resid);
2525 total_size = xfer_resid;
2526 start_offset = (int)(uio->uio_offset & PAGE_MASK_64);
2527 upl_f_offset = uio->uio_offset - start_offset;
2528
2529 if (total_size == 0) {
2530 if (start_offset) {
2531 /*
2532 * the write did not finish on a page boundary
2533 * which will leave upl_f_offset pointing to the
2534 * beginning of the last page written instead of
2535 * the page beyond it... bump it in this case
2536 * so that the cluster code records the last page
2537 * written as dirty
2538 */
2539 upl_f_offset += PAGE_SIZE_64;
2540 }
2541 upl_size = 0;
2542
2543 goto check_cluster;
2544 }
2545 }
2546 /*
2547 * compute the size of the upl needed to encompass
2548 * the requested write... limit each call to cluster_io
2549 * to the maximum UPL size... cluster_io will clip if
2550 * this exceeds the maximum io_size for the device,
2551 * make sure to account for
2552 * a starting offset that's not page aligned
2553 */
2554 upl_size = (start_offset + total_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
2555
2556 if (upl_size > max_io_size)
2557 upl_size = max_io_size;
2558
2559 pages_in_upl = upl_size / PAGE_SIZE;
2560 io_size = upl_size - start_offset;
2561
2562 if ((long long)io_size > total_size)
2563 io_size = total_size;
2564
2565 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 41)) | DBG_FUNC_START, upl_size, io_size, total_size, 0, 0);
2566
2567
2568 /*
2569 * Gather the pages from the buffer cache.
2570 * The UPL_WILL_MODIFY flag lets the UPL subsystem know
2571 * that we intend to modify these pages.
2572 */
2573 kret = ubc_create_upl(vp,
2574 upl_f_offset,
2575 upl_size,
2576 &upl,
2577 &pl,
2578 UPL_SET_LITE | UPL_WILL_MODIFY);
2579 if (kret != KERN_SUCCESS)
2580 panic("cluster_write_copy: failed to get pagelist");
2581
2582 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 41)) | DBG_FUNC_END,
2583 (int)upl, (int)upl_f_offset, start_offset, 0, 0);
2584
2585 if (start_offset && !upl_valid_page(pl, 0)) {
2586 int read_size;
2587
2588 /*
2589 * we're starting in the middle of the first page of the upl
2590 * and the page isn't currently valid, so we're going to have
2591 * to read it in first... this is a synchronous operation
2592 */
2593 read_size = PAGE_SIZE;
2594
2595 if ((upl_f_offset + read_size) > newEOF)
2596 read_size = newEOF - upl_f_offset;
2597
2598 retval = cluster_io(vp, upl, 0, upl_f_offset, read_size,
2599 CL_READ | bflag, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
2600 if (retval) {
2601 /*
2602 * we had an error during the read which causes us to abort
2603 * the current cluster_write request... before we do, we need
2604 * to release the rest of the pages in the upl without modifying
2605 * there state and mark the failed page in error
2606 */
2607 ubc_upl_abort_range(upl, 0, PAGE_SIZE, UPL_ABORT_DUMP_PAGES);
2608
2609 if (upl_size > PAGE_SIZE)
2610 ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_FREE_ON_EMPTY);
2611
2612 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 45)) | DBG_FUNC_NONE,
2613 (int)upl, 0, 0, retval, 0);
2614 break;
2615 }
2616 }
2617 if ((start_offset == 0 || upl_size > PAGE_SIZE) && ((start_offset + io_size) & PAGE_MASK)) {
2618 /*
2619 * the last offset we're writing to in this upl does not end on a page
2620 * boundary... if it's not beyond the old EOF, then we'll also need to
2621 * pre-read this page in if it isn't already valid
2622 */
2623 upl_offset = upl_size - PAGE_SIZE;
2624
2625 if ((upl_f_offset + start_offset + io_size) < oldEOF &&
2626 !upl_valid_page(pl, upl_offset / PAGE_SIZE)) {
2627 int read_size;
2628
2629 read_size = PAGE_SIZE;
2630
2631 if ((upl_f_offset + upl_offset + read_size) > newEOF)
2632 read_size = newEOF - (upl_f_offset + upl_offset);
2633
2634 retval = cluster_io(vp, upl, upl_offset, upl_f_offset + upl_offset, read_size,
2635 CL_READ | bflag, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
2636 if (retval) {
2637 /*
2638 * we had an error during the read which causes us to abort
2639 * the current cluster_write request... before we do, we
2640 * need to release the rest of the pages in the upl without
2641 * modifying there state and mark the failed page in error
2642 */
2643 ubc_upl_abort_range(upl, upl_offset, PAGE_SIZE, UPL_ABORT_DUMP_PAGES);
2644
2645 if (upl_size > PAGE_SIZE)
2646 ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_FREE_ON_EMPTY);
2647
2648 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 45)) | DBG_FUNC_NONE,
2649 (int)upl, 0, 0, retval, 0);
2650 break;
2651 }
2652 }
2653 }
2654 xfer_resid = io_size;
2655 io_offset = start_offset;
2656
2657 while (zero_cnt && xfer_resid) {
2658
2659 if (zero_cnt < (long long)xfer_resid)
2660 bytes_to_zero = zero_cnt;
2661 else
2662 bytes_to_zero = xfer_resid;
2663
2664 if ( !(flags & (IO_NOZEROVALID | IO_NOZERODIRTY))) {
2665 cluster_zero(upl, io_offset, bytes_to_zero, NULL);
2666 } else {
2667 int zero_pg_index;
2668
2669 bytes_to_zero = min(bytes_to_zero, PAGE_SIZE - (int)(zero_off & PAGE_MASK_64));
2670 zero_pg_index = (int)((zero_off - upl_f_offset) / PAGE_SIZE_64);
2671
2672 if ( !upl_valid_page(pl, zero_pg_index)) {
2673 cluster_zero(upl, io_offset, bytes_to_zero, NULL);
2674
2675 } else if ((flags & (IO_NOZERODIRTY | IO_NOZEROVALID)) == IO_NOZERODIRTY &&
2676 !upl_dirty_page(pl, zero_pg_index)) {
2677 cluster_zero(upl, io_offset, bytes_to_zero, NULL);
2678 }
2679 }
2680 xfer_resid -= bytes_to_zero;
2681 zero_cnt -= bytes_to_zero;
2682 zero_off += bytes_to_zero;
2683 io_offset += bytes_to_zero;
2684 }
2685 if (xfer_resid && io_resid) {
2686 u_int32_t io_requested;
2687
2688 bytes_to_move = min(io_resid, xfer_resid);
2689 io_requested = bytes_to_move;
2690
2691 retval = cluster_copy_upl_data(uio, upl, io_offset, (int *)&io_requested);
2692
2693 if (retval) {
2694
2695 ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_DUMP_PAGES | UPL_ABORT_FREE_ON_EMPTY);
2696
2697 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 45)) | DBG_FUNC_NONE,
2698 (int)upl, 0, 0, retval, 0);
2699 } else {
2700 io_resid -= bytes_to_move;
2701 xfer_resid -= bytes_to_move;
2702 io_offset += bytes_to_move;
2703 }
2704 }
2705 while (xfer_resid && zero_cnt1 && retval == 0) {
2706
2707 if (zero_cnt1 < (long long)xfer_resid)
2708 bytes_to_zero = zero_cnt1;
2709 else
2710 bytes_to_zero = xfer_resid;
2711
2712 if ( !(flags & (IO_NOZEROVALID | IO_NOZERODIRTY))) {
2713 cluster_zero(upl, io_offset, bytes_to_zero, NULL);
2714 } else {
2715 int zero_pg_index;
2716
2717 bytes_to_zero = min(bytes_to_zero, PAGE_SIZE - (int)(zero_off1 & PAGE_MASK_64));
2718 zero_pg_index = (int)((zero_off1 - upl_f_offset) / PAGE_SIZE_64);
2719
2720 if ( !upl_valid_page(pl, zero_pg_index)) {
2721 cluster_zero(upl, io_offset, bytes_to_zero, NULL);
2722 } else if ((flags & (IO_NOZERODIRTY | IO_NOZEROVALID)) == IO_NOZERODIRTY &&
2723 !upl_dirty_page(pl, zero_pg_index)) {
2724 cluster_zero(upl, io_offset, bytes_to_zero, NULL);
2725 }
2726 }
2727 xfer_resid -= bytes_to_zero;
2728 zero_cnt1 -= bytes_to_zero;
2729 zero_off1 += bytes_to_zero;
2730 io_offset += bytes_to_zero;
2731 }
2732
2733 if (retval == 0) {
2734 int cl_index;
2735 int ret_cluster_try_push;
2736
2737 io_size += start_offset;
2738
2739 if ((upl_f_offset + io_size) >= newEOF && (u_int)io_size < upl_size) {
2740 /*
2741 * if we're extending the file with this write
2742 * we'll zero fill the rest of the page so that
2743 * if the file gets extended again in such a way as to leave a
2744 * hole starting at this EOF, we'll have zero's in the correct spot
2745 */
2746 cluster_zero(upl, io_size, upl_size - io_size, NULL);
2747 }
2748 if (flags & IO_SYNC)
2749 /*
2750 * if the IO_SYNC flag is set than we need to
2751 * bypass any clusters and immediately issue
2752 * the I/O
2753 */
2754 goto issue_io;
2755 check_cluster:
2756 /*
2757 * take the lock to protect our accesses
2758 * of the writebehind and sparse cluster state
2759 */
2760 wbp = cluster_get_wbp(vp, CLW_ALLOCATE | CLW_RETURNLOCKED);
2761
2762 /*
2763 * calculate the last logical block number
2764 * that this delayed I/O encompassed
2765 */
2766 cl.e_addr = (daddr64_t)((upl_f_offset + (off_t)upl_size) / PAGE_SIZE_64);
2767
2768 if (wbp->cl_scmap) {
2769
2770 if ( !(flags & IO_NOCACHE)) {
2771 /*
2772 * we've fallen into the sparse
2773 * cluster method of delaying dirty pages
2774 * first, we need to release the upl if we hold one
2775 * since pages in it may be present in the sparse cluster map
2776 * and may span 2 separate buckets there... if they do and
2777 * we happen to have to flush a bucket to make room and it intersects
2778 * this upl, a deadlock may result on page BUSY
2779 */
2780 if (upl_size)
2781 ubc_upl_commit_range(upl, 0, upl_size,
2782 UPL_COMMIT_SET_DIRTY | UPL_COMMIT_INACTIVATE | UPL_COMMIT_FREE_ON_EMPTY);
2783
2784 sparse_cluster_add(wbp, vp, &cl, newEOF, callback, callback_arg);
2785
2786 lck_mtx_unlock(&wbp->cl_lockw);
2787
2788 continue;
2789 }
2790 /*
2791 * must have done cached writes that fell into
2792 * the sparse cluster mechanism... we've switched
2793 * to uncached writes on the file, so go ahead
2794 * and push whatever's in the sparse map
2795 * and switch back to normal clustering
2796 *
2797 * see the comment above concerning a possible deadlock...
2798 */
2799 if (upl_size) {
2800 ubc_upl_commit_range(upl, 0, upl_size,
2801 UPL_COMMIT_SET_DIRTY | UPL_COMMIT_INACTIVATE | UPL_COMMIT_FREE_ON_EMPTY);
2802 /*
2803 * setting upl_size to 0 keeps us from committing a
2804 * second time in the start_new_cluster path
2805 */
2806 upl_size = 0;
2807 }
2808 sparse_cluster_push(wbp, vp, newEOF, PUSH_ALL, callback, callback_arg);
2809
2810 wbp->cl_number = 0;
2811 /*
2812 * no clusters of either type present at this point
2813 * so just go directly to start_new_cluster since
2814 * we know we need to delay this I/O since we've
2815 * already released the pages back into the cache
2816 * to avoid the deadlock with sparse_cluster_push
2817 */
2818 goto start_new_cluster;
2819 }
2820 upl_offset = 0;
2821
2822 if (wbp->cl_number == 0)
2823 /*
2824 * no clusters currently present
2825 */
2826 goto start_new_cluster;
2827
2828 for (cl_index = 0; cl_index < wbp->cl_number; cl_index++) {
2829 /*
2830 * check each cluster that we currently hold
2831 * try to merge some or all of this write into
2832 * one or more of the existing clusters... if
2833 * any portion of the write remains, start a
2834 * new cluster
2835 */
2836 if (cl.b_addr >= wbp->cl_clusters[cl_index].b_addr) {
2837 /*
2838 * the current write starts at or after the current cluster
2839 */
2840 if (cl.e_addr <= (wbp->cl_clusters[cl_index].b_addr + max_cluster_pgcount)) {
2841 /*
2842 * we have a write that fits entirely
2843 * within the existing cluster limits
2844 */
2845 if (cl.e_addr > wbp->cl_clusters[cl_index].e_addr)
2846 /*
2847 * update our idea of where the cluster ends
2848 */
2849 wbp->cl_clusters[cl_index].e_addr = cl.e_addr;
2850 break;
2851 }
2852 if (cl.b_addr < (wbp->cl_clusters[cl_index].b_addr + max_cluster_pgcount)) {
2853 /*
2854 * we have a write that starts in the middle of the current cluster
2855 * but extends beyond the cluster's limit... we know this because
2856 * of the previous checks
2857 * we'll extend the current cluster to the max
2858 * and update the b_addr for the current write to reflect that
2859 * the head of it was absorbed into this cluster...
2860 * note that we'll always have a leftover tail in this case since
2861 * full absorbtion would have occurred in the clause above
2862 */
2863 wbp->cl_clusters[cl_index].e_addr = wbp->cl_clusters[cl_index].b_addr + max_cluster_pgcount;
2864
2865 if (upl_size) {
2866 daddr64_t start_pg_in_upl;
2867
2868 start_pg_in_upl = (daddr64_t)(upl_f_offset / PAGE_SIZE_64);
2869
2870 if (start_pg_in_upl < wbp->cl_clusters[cl_index].e_addr) {
2871 intersection = (int)((wbp->cl_clusters[cl_index].e_addr - start_pg_in_upl) * PAGE_SIZE);
2872
2873 ubc_upl_commit_range(upl, upl_offset, intersection,
2874 UPL_COMMIT_SET_DIRTY | UPL_COMMIT_INACTIVATE | UPL_COMMIT_FREE_ON_EMPTY);
2875 upl_f_offset += intersection;
2876 upl_offset += intersection;
2877 upl_size -= intersection;
2878 }
2879 }
2880 cl.b_addr = wbp->cl_clusters[cl_index].e_addr;
2881 }
2882 /*
2883 * we come here for the case where the current write starts
2884 * beyond the limit of the existing cluster or we have a leftover
2885 * tail after a partial absorbtion
2886 *
2887 * in either case, we'll check the remaining clusters before
2888 * starting a new one
2889 */
2890 } else {
2891 /*
2892 * the current write starts in front of the cluster we're currently considering
2893 */
2894 if ((wbp->cl_clusters[cl_index].e_addr - cl.b_addr) <= max_cluster_pgcount) {
2895 /*
2896 * we can just merge the new request into
2897 * this cluster and leave it in the cache
2898 * since the resulting cluster is still
2899 * less than the maximum allowable size
2900 */
2901 wbp->cl_clusters[cl_index].b_addr = cl.b_addr;
2902
2903 if (cl.e_addr > wbp->cl_clusters[cl_index].e_addr) {
2904 /*
2905 * the current write completely
2906 * envelops the existing cluster and since
2907 * each write is limited to at most max_cluster_pgcount pages
2908 * we can just use the start and last blocknos of the write
2909 * to generate the cluster limits
2910 */
2911 wbp->cl_clusters[cl_index].e_addr = cl.e_addr;
2912 }
2913 break;
2914 }
2915
2916 /*
2917 * if we were to combine this write with the current cluster
2918 * we would exceed the cluster size limit.... so,
2919 * let's see if there's any overlap of the new I/O with
2920 * the cluster we're currently considering... in fact, we'll
2921 * stretch the cluster out to it's full limit and see if we
2922 * get an intersection with the current write
2923 *
2924 */
2925 if (cl.e_addr > wbp->cl_clusters[cl_index].e_addr - max_cluster_pgcount) {
2926 /*
2927 * the current write extends into the proposed cluster
2928 * clip the length of the current write after first combining it's
2929 * tail with the newly shaped cluster
2930 */
2931 wbp->cl_clusters[cl_index].b_addr = wbp->cl_clusters[cl_index].e_addr - max_cluster_pgcount;
2932
2933 if (upl_size) {
2934 intersection = (int)((cl.e_addr - wbp->cl_clusters[cl_index].b_addr) * PAGE_SIZE);
2935
2936 if ((u_int)intersection > upl_size)
2937 /*
2938 * because the current write may consist of a number of pages found in the cache
2939 * which are not part of the UPL, we may have an intersection that exceeds
2940 * the size of the UPL that is also part of this write
2941 */
2942 intersection = upl_size;
2943
2944 ubc_upl_commit_range(upl, upl_offset + (upl_size - intersection), intersection,
2945 UPL_COMMIT_SET_DIRTY | UPL_COMMIT_INACTIVATE | UPL_COMMIT_FREE_ON_EMPTY);
2946 upl_size -= intersection;
2947 }
2948 cl.e_addr = wbp->cl_clusters[cl_index].b_addr;
2949 }
2950 /*
2951 * if we get here, there was no way to merge
2952 * any portion of this write with this cluster
2953 * or we could only merge part of it which
2954 * will leave a tail...
2955 * we'll check the remaining clusters before starting a new one
2956 */
2957 }
2958 }
2959 if (cl_index < wbp->cl_number)
2960 /*
2961 * we found an existing cluster(s) that we
2962 * could entirely merge this I/O into
2963 */
2964 goto delay_io;
2965
2966 if (wbp->cl_number < MAX_CLUSTERS)
2967 /*
2968 * we didn't find an existing cluster to
2969 * merge into, but there's room to start
2970 * a new one
2971 */
2972 goto start_new_cluster;
2973
2974 /*
2975 * no exisitng cluster to merge with and no
2976 * room to start a new one... we'll try
2977 * pushing one of the existing ones... if none of
2978 * them are able to be pushed, we'll switch
2979 * to the sparse cluster mechanism
2980 * cluster_try_push updates cl_number to the
2981 * number of remaining clusters... and
2982 * returns the number of currently unused clusters
2983 */
2984 ret_cluster_try_push = 0;
2985
2986 /*
2987 * if writes are not deferred, call cluster push immediately
2988 */
2989 if (!((unsigned int)vfs_flags(vp->v_mount) & MNT_DEFWRITE)) {
2990
2991 ret_cluster_try_push = cluster_try_push(wbp, vp, newEOF, (flags & IO_NOCACHE) ? 0 : PUSH_DELAY, callback, callback_arg);
2992 }
2993
2994 /*
2995 * execute following regardless of writes being deferred or not
2996 */
2997 if (ret_cluster_try_push == 0) {
2998 /*
2999 * no more room in the normal cluster mechanism
3000 * so let's switch to the more expansive but expensive
3001 * sparse mechanism....
3002 * first, we need to release the upl if we hold one
3003 * since pages in it may be present in the sparse cluster map (after the cluster_switch)
3004 * and may span 2 separate buckets there... if they do and
3005 * we happen to have to flush a bucket to make room and it intersects
3006 * this upl, a deadlock may result on page BUSY
3007 */
3008 if (upl_size)
3009 ubc_upl_commit_range(upl, upl_offset, upl_size,
3010 UPL_COMMIT_SET_DIRTY | UPL_COMMIT_INACTIVATE | UPL_COMMIT_FREE_ON_EMPTY);
3011
3012 sparse_cluster_switch(wbp, vp, newEOF, callback, callback_arg);
3013 sparse_cluster_add(wbp, vp, &cl, newEOF, callback, callback_arg);
3014
3015 lck_mtx_unlock(&wbp->cl_lockw);
3016
3017 continue;
3018 }
3019 /*
3020 * we pushed one cluster successfully, so we must be sequentially writing this file
3021 * otherwise, we would have failed and fallen into the sparse cluster support
3022 * so let's take the opportunity to push out additional clusters...
3023 * this will give us better I/O locality if we're in a copy loop
3024 * (i.e. we won't jump back and forth between the read and write points
3025 */
3026 if (!((unsigned int)vfs_flags(vp->v_mount) & MNT_DEFWRITE)) {
3027 while (wbp->cl_number)
3028 cluster_try_push(wbp, vp, newEOF, 0, callback, callback_arg);
3029 }
3030
3031 start_new_cluster:
3032 wbp->cl_clusters[wbp->cl_number].b_addr = cl.b_addr;
3033 wbp->cl_clusters[wbp->cl_number].e_addr = cl.e_addr;
3034
3035 wbp->cl_clusters[wbp->cl_number].io_flags = 0;
3036
3037 if (flags & IO_NOCACHE)
3038 wbp->cl_clusters[wbp->cl_number].io_flags |= CLW_IONOCACHE;
3039
3040 if (bflag & CL_PASSIVE)
3041 wbp->cl_clusters[wbp->cl_number].io_flags |= CLW_IOPASSIVE;
3042
3043 wbp->cl_number++;
3044 delay_io:
3045 if (upl_size)
3046 ubc_upl_commit_range(upl, upl_offset, upl_size,
3047 UPL_COMMIT_SET_DIRTY | UPL_COMMIT_INACTIVATE | UPL_COMMIT_FREE_ON_EMPTY);
3048
3049 lck_mtx_unlock(&wbp->cl_lockw);
3050
3051 continue;
3052 issue_io:
3053 /*
3054 * we don't hold the vnode lock at this point
3055 *
3056 * because we had to ask for a UPL that provides currenty non-present pages, the
3057 * UPL has been automatically set to clear the dirty flags (both software and hardware)
3058 * upon committing it... this is not the behavior we want since it's possible for
3059 * pages currently present as part of a mapped file to be dirtied while the I/O is in flight.
3060 * in order to maintain some semblance of coherency with mapped writes
3061 * we need to drop the current upl and pick it back up with COPYOUT_FROM set
3062 * so that we correctly deal with a change in state of the hardware modify bit...
3063 * we do this via cluster_push_now... by passing along the IO_SYNC flag, we force
3064 * cluster_push_now to wait until all the I/Os have completed... cluster_push_now is also
3065 * responsible for generating the correct sized I/O(s)
3066 */
3067 ubc_upl_commit_range(upl, 0, upl_size,
3068 UPL_COMMIT_SET_DIRTY | UPL_COMMIT_INACTIVATE | UPL_COMMIT_FREE_ON_EMPTY);
3069
3070 cl.e_addr = (upl_f_offset + (off_t)upl_size) / PAGE_SIZE_64;
3071
3072 retval = cluster_push_now(vp, &cl, newEOF, flags, callback, callback_arg);
3073 }
3074 }
3075 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 40)) | DBG_FUNC_END, retval, 0, io_resid, 0, 0);
3076
3077 return (retval);
3078 }
3079
3080
3081
3082 int
3083 cluster_read(vnode_t vp, struct uio *uio, off_t filesize, int xflags)
3084 {
3085 return cluster_read_ext(vp, uio, filesize, xflags, NULL, NULL);
3086 }
3087
3088
3089 int
3090 cluster_read_ext(vnode_t vp, struct uio *uio, off_t filesize, int xflags, int (*callback)(buf_t, void *), void *callback_arg)
3091 {
3092 int retval = 0;
3093 int flags;
3094 user_ssize_t cur_resid;
3095 u_int32_t io_size;
3096 u_int32_t read_length = 0;
3097 int read_type = IO_COPY;
3098
3099 flags = xflags;
3100
3101 if (vp->v_flag & VNOCACHE_DATA)
3102 flags |= IO_NOCACHE;
3103 if ((vp->v_flag & VRAOFF) || speculative_reads_disabled)
3104 flags |= IO_RAOFF;
3105
3106 /*
3107 * do a read through the cache if one of the following is true....
3108 * NOCACHE is not true
3109 * the uio request doesn't target USERSPACE
3110 * otherwise, find out if we want the direct or contig variant for
3111 * the first vector in the uio request
3112 */
3113 if ( (flags & IO_NOCACHE) && UIO_SEG_IS_USER_SPACE(uio->uio_segflg) )
3114 retval = cluster_io_type(uio, &read_type, &read_length, 0);
3115
3116 while ((cur_resid = uio_resid(uio)) && uio->uio_offset < filesize && retval == 0) {
3117
3118 switch (read_type) {
3119
3120 case IO_COPY:
3121 /*
3122 * make sure the uio_resid isn't too big...
3123 * internally, we want to handle all of the I/O in
3124 * chunk sizes that fit in a 32 bit int
3125 */
3126 if (cur_resid > (user_ssize_t)(MAX_IO_REQUEST_SIZE))
3127 io_size = MAX_IO_REQUEST_SIZE;
3128 else
3129 io_size = (u_int32_t)cur_resid;
3130
3131 retval = cluster_read_copy(vp, uio, io_size, filesize, flags, callback, callback_arg);
3132 break;
3133
3134 case IO_DIRECT:
3135 retval = cluster_read_direct(vp, uio, filesize, &read_type, &read_length, flags, callback, callback_arg);
3136 break;
3137
3138 case IO_CONTIG:
3139 retval = cluster_read_contig(vp, uio, filesize, &read_type, &read_length, callback, callback_arg, flags);
3140 break;
3141
3142 case IO_UNKNOWN:
3143 retval = cluster_io_type(uio, &read_type, &read_length, 0);
3144 break;
3145 }
3146 }
3147 return (retval);
3148 }
3149
3150
3151
3152 static void
3153 cluster_read_upl_release(upl_t upl, int start_pg, int last_pg, int flags)
3154 {
3155 int range;
3156 int abort_flags = UPL_ABORT_FREE_ON_EMPTY;
3157
3158 if ((range = last_pg - start_pg)) {
3159 if ( !(flags & IO_NOCACHE))
3160 abort_flags |= UPL_ABORT_REFERENCE;
3161
3162 ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, range * PAGE_SIZE, abort_flags);
3163 }
3164 }
3165
3166
3167 static int
3168 cluster_read_copy(vnode_t vp, struct uio *uio, u_int32_t io_req_size, off_t filesize, int flags, int (*callback)(buf_t, void *), void *callback_arg)
3169 {
3170 upl_page_info_t *pl;
3171 upl_t upl;
3172 vm_offset_t upl_offset;
3173 u_int32_t upl_size;
3174 off_t upl_f_offset;
3175 int start_offset;
3176 int start_pg;
3177 int last_pg;
3178 int uio_last = 0;
3179 int pages_in_upl;
3180 off_t max_size;
3181 off_t last_ioread_offset;
3182 off_t last_request_offset;
3183 kern_return_t kret;
3184 int error = 0;
3185 int retval = 0;
3186 u_int32_t size_of_prefetch;
3187 u_int32_t xsize;
3188 u_int32_t io_size;
3189 u_int32_t max_rd_size;
3190 u_int32_t max_io_size;
3191 u_int32_t max_prefetch;
3192 u_int rd_ahead_enabled = 1;
3193 u_int prefetch_enabled = 1;
3194 struct cl_readahead * rap;
3195 struct clios iostate;
3196 struct cl_extent extent;
3197 int bflag;
3198 int take_reference = 1;
3199 struct uthread *ut;
3200 int policy = IOPOL_DEFAULT;
3201
3202 policy = current_proc()->p_iopol_disk;
3203
3204 ut = get_bsdthread_info(current_thread());
3205
3206 if (ut->uu_iopol_disk != IOPOL_DEFAULT)
3207 policy = ut->uu_iopol_disk;
3208
3209 if (policy == IOPOL_THROTTLE)
3210 take_reference = 0;
3211
3212 if (flags & IO_PASSIVE)
3213 bflag = CL_PASSIVE;
3214 else
3215 bflag = 0;
3216
3217 max_prefetch = MAX_PREFETCH(vp);
3218 max_rd_size = max_prefetch;
3219 max_io_size = cluster_max_io_size(vp->v_mount, CL_READ);
3220
3221 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 32)) | DBG_FUNC_START,
3222 (int)uio->uio_offset, io_req_size, (int)filesize, flags, 0);
3223
3224 last_request_offset = uio->uio_offset + io_req_size;
3225
3226 if ((flags & (IO_RAOFF|IO_NOCACHE)) || ((last_request_offset & ~PAGE_MASK_64) == (uio->uio_offset & ~PAGE_MASK_64))) {
3227 rd_ahead_enabled = 0;
3228 rap = NULL;
3229 } else {
3230 if (cluster_hard_throttle_on(vp)) {
3231 rd_ahead_enabled = 0;
3232 prefetch_enabled = 0;
3233
3234 max_rd_size = HARD_THROTTLE_MAXSIZE;
3235 }
3236 if ((rap = cluster_get_rap(vp)) == NULL)
3237 rd_ahead_enabled = 0;
3238 }
3239 if (last_request_offset > filesize)
3240 last_request_offset = filesize;
3241 extent.b_addr = uio->uio_offset / PAGE_SIZE_64;
3242 extent.e_addr = (last_request_offset - 1) / PAGE_SIZE_64;
3243
3244 if (rap != NULL && rap->cl_ralen && (rap->cl_lastr == extent.b_addr || (rap->cl_lastr + 1) == extent.b_addr)) {
3245 /*
3246 * determine if we already have a read-ahead in the pipe courtesy of the
3247 * last read systemcall that was issued...
3248 * if so, pick up it's extent to determine where we should start
3249 * with respect to any read-ahead that might be necessary to
3250 * garner all the data needed to complete this read systemcall
3251 */
3252 last_ioread_offset = (rap->cl_maxra * PAGE_SIZE_64) + PAGE_SIZE_64;
3253
3254 if (last_ioread_offset < uio->uio_offset)
3255 last_ioread_offset = (off_t)0;
3256 else if (last_ioread_offset > last_request_offset)
3257 last_ioread_offset = last_request_offset;
3258 } else
3259 last_ioread_offset = (off_t)0;
3260
3261 while (io_req_size && uio->uio_offset < filesize && retval == 0) {
3262 /*
3263 * compute the size of the upl needed to encompass
3264 * the requested read... limit each call to cluster_io
3265 * to the maximum UPL size... cluster_io will clip if
3266 * this exceeds the maximum io_size for the device,
3267 * make sure to account for
3268 * a starting offset that's not page aligned
3269 */
3270 start_offset = (int)(uio->uio_offset & PAGE_MASK_64);
3271 upl_f_offset = uio->uio_offset - (off_t)start_offset;
3272 max_size = filesize - uio->uio_offset;
3273
3274 if ((off_t)(io_req_size) < max_size)
3275 io_size = io_req_size;
3276 else
3277 io_size = max_size;
3278
3279 if (!(flags & IO_NOCACHE)) {
3280
3281 while (io_size) {
3282 u_int32_t io_resid;
3283 u_int32_t io_requested;
3284
3285 /*
3286 * if we keep finding the pages we need already in the cache, then
3287 * don't bother to call cluster_read_prefetch since it costs CPU cycles
3288 * to determine that we have all the pages we need... once we miss in
3289 * the cache and have issued an I/O, than we'll assume that we're likely
3290 * to continue to miss in the cache and it's to our advantage to try and prefetch
3291 */
3292 if (last_request_offset && last_ioread_offset && (size_of_prefetch = (last_request_offset - last_ioread_offset))) {
3293 if ((last_ioread_offset - uio->uio_offset) <= max_rd_size && prefetch_enabled) {
3294 /*
3295 * we've already issued I/O for this request and
3296 * there's still work to do and
3297 * our prefetch stream is running dry, so issue a
3298 * pre-fetch I/O... the I/O latency will overlap
3299 * with the copying of the data
3300 */
3301 if (size_of_prefetch > max_rd_size)
3302 size_of_prefetch = max_rd_size;
3303
3304 size_of_prefetch = cluster_read_prefetch(vp, last_ioread_offset, size_of_prefetch, filesize, callback, callback_arg, bflag);
3305
3306 last_ioread_offset += (off_t)(size_of_prefetch * PAGE_SIZE);
3307
3308 if (last_ioread_offset > last_request_offset)
3309 last_ioread_offset = last_request_offset;
3310 }
3311 }
3312 /*
3313 * limit the size of the copy we're about to do so that
3314 * we can notice that our I/O pipe is running dry and
3315 * get the next I/O issued before it does go dry
3316 */
3317 if (last_ioread_offset && io_size > (max_io_size / 4))
3318 io_resid = (max_io_size / 4);
3319 else
3320 io_resid = io_size;
3321
3322 io_requested = io_resid;
3323
3324 retval = cluster_copy_ubc_data_internal(vp, uio, (int *)&io_resid, 0, take_reference);
3325
3326 xsize = io_requested - io_resid;
3327
3328 io_size -= xsize;
3329 io_req_size -= xsize;
3330
3331 if (retval || io_resid)
3332 /*
3333 * if we run into a real error or
3334 * a page that is not in the cache
3335 * we need to leave streaming mode
3336 */
3337 break;
3338
3339 if ((io_size == 0 || last_ioread_offset == last_request_offset) && rd_ahead_enabled) {
3340 /*
3341 * we're already finished the I/O for this read request
3342 * let's see if we should do a read-ahead
3343 */
3344 cluster_read_ahead(vp, &extent, filesize, rap, callback, callback_arg, bflag);
3345 }
3346 }
3347 if (retval)
3348 break;
3349 if (io_size == 0) {
3350 if (rap != NULL) {
3351 if (extent.e_addr < rap->cl_lastr)
3352 rap->cl_maxra = 0;
3353 rap->cl_lastr = extent.e_addr;
3354 }
3355 break;
3356 }
3357 start_offset = (int)(uio->uio_offset & PAGE_MASK_64);
3358 upl_f_offset = uio->uio_offset - (off_t)start_offset;
3359 max_size = filesize - uio->uio_offset;
3360 }
3361 if (io_size > max_rd_size)
3362 io_size = max_rd_size;
3363
3364 upl_size = (start_offset + io_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
3365
3366 if (flags & IO_NOCACHE) {
3367 if (upl_size > max_io_size)
3368 upl_size = max_io_size;
3369 } else {
3370 if (upl_size > max_io_size / 4)
3371 upl_size = max_io_size / 4;
3372 }
3373 pages_in_upl = upl_size / PAGE_SIZE;
3374
3375 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 33)) | DBG_FUNC_START,
3376 (int)upl, (int)upl_f_offset, upl_size, start_offset, 0);
3377
3378 kret = ubc_create_upl(vp,
3379 upl_f_offset,
3380 upl_size,
3381 &upl,
3382 &pl,
3383 UPL_FILE_IO | UPL_SET_LITE);
3384 if (kret != KERN_SUCCESS)
3385 panic("cluster_read_copy: failed to get pagelist");
3386
3387 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 33)) | DBG_FUNC_END,
3388 (int)upl, (int)upl_f_offset, upl_size, start_offset, 0);
3389
3390 /*
3391 * scan from the beginning of the upl looking for the first
3392 * non-valid page.... this will become the first page in
3393 * the request we're going to make to 'cluster_io'... if all
3394 * of the pages are valid, we won't call through to 'cluster_io'
3395 */
3396 for (start_pg = 0; start_pg < pages_in_upl; start_pg++) {
3397 if (!upl_valid_page(pl, start_pg))
3398 break;
3399 }
3400
3401 /*
3402 * scan from the starting invalid page looking for a valid
3403 * page before the end of the upl is reached, if we
3404 * find one, then it will be the last page of the request to
3405 * 'cluster_io'
3406 */
3407 for (last_pg = start_pg; last_pg < pages_in_upl; last_pg++) {
3408 if (upl_valid_page(pl, last_pg))
3409 break;
3410 }
3411 iostate.io_completed = 0;
3412 iostate.io_issued = 0;
3413 iostate.io_error = 0;
3414 iostate.io_wanted = 0;
3415
3416 if (start_pg < last_pg) {
3417 /*
3418 * we found a range of 'invalid' pages that must be filled
3419 * if the last page in this range is the last page of the file
3420 * we may have to clip the size of it to keep from reading past
3421 * the end of the last physical block associated with the file
3422 */
3423 upl_offset = start_pg * PAGE_SIZE;
3424 io_size = (last_pg - start_pg) * PAGE_SIZE;
3425
3426 if ((upl_f_offset + upl_offset + io_size) > filesize)
3427 io_size = filesize - (upl_f_offset + upl_offset);
3428
3429 /*
3430 * issue an asynchronous read to cluster_io
3431 */
3432
3433 error = cluster_io(vp, upl, upl_offset, upl_f_offset + upl_offset,
3434 io_size, CL_READ | CL_ASYNC | bflag, (buf_t)NULL, &iostate, callback, callback_arg);
3435 }
3436 if (error == 0) {
3437 /*
3438 * if the read completed successfully, or there was no I/O request
3439 * issued, than copy the data into user land via 'cluster_upl_copy_data'
3440 * we'll first add on any 'valid'
3441 * pages that were present in the upl when we acquired it.
3442 */
3443 u_int val_size;
3444
3445 for (uio_last = last_pg; uio_last < pages_in_upl; uio_last++) {
3446 if (!upl_valid_page(pl, uio_last))
3447 break;
3448 }
3449 if (uio_last < pages_in_upl) {
3450 /*
3451 * there were some invalid pages beyond the valid pages
3452 * that we didn't issue an I/O for, just release them
3453 * unchanged now, so that any prefetch/readahed can
3454 * include them
3455 */
3456 ubc_upl_abort_range(upl, uio_last * PAGE_SIZE,
3457 (pages_in_upl - uio_last) * PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
3458 }
3459
3460 /*
3461 * compute size to transfer this round, if io_req_size is
3462 * still non-zero after this attempt, we'll loop around and
3463 * set up for another I/O.
3464 */
3465 val_size = (uio_last * PAGE_SIZE) - start_offset;
3466
3467 if (val_size > max_size)
3468 val_size = max_size;
3469
3470 if (val_size > io_req_size)
3471 val_size = io_req_size;
3472
3473 if ((uio->uio_offset + val_size) > last_ioread_offset)
3474 last_ioread_offset = uio->uio_offset + val_size;
3475
3476 if ((size_of_prefetch = (last_request_offset - last_ioread_offset)) && prefetch_enabled) {
3477
3478 if ((last_ioread_offset - (uio->uio_offset + val_size)) <= upl_size) {
3479 /*
3480 * if there's still I/O left to do for this request, and...
3481 * we're not in hard throttle mode, and...
3482 * we're close to using up the previous prefetch, then issue a
3483 * new pre-fetch I/O... the I/O latency will overlap
3484 * with the copying of the data
3485 */
3486 if (size_of_prefetch > max_rd_size)
3487 size_of_prefetch = max_rd_size;
3488
3489 size_of_prefetch = cluster_read_prefetch(vp, last_ioread_offset, size_of_prefetch, filesize, callback, callback_arg, bflag);
3490
3491 last_ioread_offset += (off_t)(size_of_prefetch * PAGE_SIZE);
3492
3493 if (last_ioread_offset > last_request_offset)
3494 last_ioread_offset = last_request_offset;
3495 }
3496
3497 } else if ((uio->uio_offset + val_size) == last_request_offset) {
3498 /*
3499 * this transfer will finish this request, so...
3500 * let's try to read ahead if we're in
3501 * a sequential access pattern and we haven't
3502 * explicitly disabled it
3503 */
3504 if (rd_ahead_enabled)
3505 cluster_read_ahead(vp, &extent, filesize, rap, callback, callback_arg, bflag);
3506
3507 if (rap != NULL) {
3508 if (extent.e_addr < rap->cl_lastr)
3509 rap->cl_maxra = 0;
3510 rap->cl_lastr = extent.e_addr;
3511 }
3512 }
3513 lck_mtx_lock(cl_mtxp);
3514
3515 while (iostate.io_issued != iostate.io_completed) {
3516 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_START,
3517 iostate.io_issued, iostate.io_completed, 0, 0, 0);
3518
3519 iostate.io_wanted = 1;
3520 msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_read_copy", NULL);
3521
3522 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_END,
3523 iostate.io_issued, iostate.io_completed, 0, 0, 0);
3524 }
3525 lck_mtx_unlock(cl_mtxp);
3526
3527 if (iostate.io_error)
3528 error = iostate.io_error;
3529 else {
3530 u_int32_t io_requested;
3531
3532 io_requested = val_size;
3533
3534 retval = cluster_copy_upl_data(uio, upl, start_offset, (int *)&io_requested);
3535
3536 io_req_size -= (val_size - io_requested);
3537 }
3538 }
3539 if (start_pg < last_pg) {
3540 /*
3541 * compute the range of pages that we actually issued an I/O for
3542 * and either commit them as valid if the I/O succeeded
3543 * or abort them if the I/O failed or we're not supposed to
3544 * keep them in the cache
3545 */
3546 io_size = (last_pg - start_pg) * PAGE_SIZE;
3547
3548 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 35)) | DBG_FUNC_START, (int)upl, start_pg * PAGE_SIZE, io_size, error, 0);
3549
3550 if (error || (flags & IO_NOCACHE))
3551 ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, io_size,
3552 UPL_ABORT_DUMP_PAGES | UPL_ABORT_FREE_ON_EMPTY);
3553 else
3554 ubc_upl_commit_range(upl, start_pg * PAGE_SIZE, io_size,
3555 UPL_COMMIT_CLEAR_DIRTY | UPL_COMMIT_FREE_ON_EMPTY | UPL_COMMIT_INACTIVATE);
3556
3557 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 35)) | DBG_FUNC_END, (int)upl, start_pg * PAGE_SIZE, io_size, error, 0);
3558 }
3559 if ((last_pg - start_pg) < pages_in_upl) {
3560 /*
3561 * the set of pages that we issued an I/O for did not encompass
3562 * the entire upl... so just release these without modifying
3563 * their state
3564 */
3565 if (error)
3566 ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_FREE_ON_EMPTY);
3567 else {
3568
3569 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 35)) | DBG_FUNC_START,
3570 (int)upl, -1, pages_in_upl - (last_pg - start_pg), 0, 0);
3571
3572 /*
3573 * handle any valid pages at the beginning of
3574 * the upl... release these appropriately
3575 */
3576 cluster_read_upl_release(upl, 0, start_pg, flags);
3577
3578 /*
3579 * handle any valid pages immediately after the
3580 * pages we issued I/O for... ... release these appropriately
3581 */
3582 cluster_read_upl_release(upl, last_pg, uio_last, flags);
3583
3584 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 35)) | DBG_FUNC_END, (int)upl, -1, -1, 0, 0);
3585 }
3586 }
3587 if (retval == 0)
3588 retval = error;
3589
3590 if (io_req_size) {
3591 if (cluster_hard_throttle_on(vp)) {
3592 rd_ahead_enabled = 0;
3593 prefetch_enabled = 0;
3594
3595 max_rd_size = HARD_THROTTLE_MAXSIZE;
3596 } else {
3597 if (max_rd_size == HARD_THROTTLE_MAXSIZE) {
3598 /*
3599 * coming out of throttled state
3600 */
3601 if (rap != NULL)
3602 rd_ahead_enabled = 1;
3603 prefetch_enabled = 1;
3604
3605 max_rd_size = max_prefetch;
3606 last_ioread_offset = 0;
3607 }
3608 }
3609 }
3610 }
3611 if (rap != NULL) {
3612 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 32)) | DBG_FUNC_END,
3613 (int)uio->uio_offset, io_req_size, rap->cl_lastr, retval, 0);
3614
3615 lck_mtx_unlock(&rap->cl_lockr);
3616 } else {
3617 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 32)) | DBG_FUNC_END,
3618 (int)uio->uio_offset, io_req_size, 0, retval, 0);
3619 }
3620
3621 return (retval);
3622 }
3623
3624
3625 static int
3626 cluster_read_direct(vnode_t vp, struct uio *uio, off_t filesize, int *read_type, u_int32_t *read_length,
3627 int flags, int (*callback)(buf_t, void *), void *callback_arg)
3628 {
3629 upl_t upl;
3630 upl_page_info_t *pl;
3631 off_t max_io_size;
3632 vm_offset_t upl_offset;
3633 vm_size_t upl_size;
3634 vm_size_t upl_needed_size;
3635 unsigned int pages_in_pl;
3636 int upl_flags;
3637 int bflag;
3638 kern_return_t kret;
3639 unsigned int i;
3640 int force_data_sync;
3641 int retval = 0;
3642 int no_zero_fill = 0;
3643 int abort_flag = 0;
3644 int io_flag = 0;
3645 int misaligned = 0;
3646 struct clios iostate;
3647 user_addr_t iov_base;
3648 u_int32_t io_req_size;
3649 u_int32_t offset_in_file;
3650 u_int32_t offset_in_iovbase;
3651 u_int32_t io_size;
3652 u_int32_t io_min;
3653 u_int32_t xsize;
3654 u_int32_t devblocksize;
3655 u_int32_t mem_alignment_mask;
3656 u_int32_t max_upl_size;
3657 u_int32_t max_rd_size;
3658 u_int32_t max_rd_ahead;
3659
3660
3661 max_upl_size = cluster_max_io_size(vp->v_mount, CL_READ);
3662
3663 max_rd_size = max_upl_size;
3664 max_rd_ahead = max_rd_size * 2;
3665
3666
3667 if (flags & IO_PASSIVE)
3668 bflag = CL_PASSIVE;
3669 else
3670 bflag = 0;
3671
3672 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 70)) | DBG_FUNC_START,
3673 (int)uio->uio_offset, (int)filesize, *read_type, *read_length, 0);
3674
3675 iostate.io_completed = 0;
3676 iostate.io_issued = 0;
3677 iostate.io_error = 0;
3678 iostate.io_wanted = 0;
3679
3680 devblocksize = (u_int32_t)vp->v_mount->mnt_devblocksize;
3681 mem_alignment_mask = (u_int32_t)vp->v_mount->mnt_alignmentmask;
3682
3683 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 70)) | DBG_FUNC_NONE,
3684 (int)devblocksize, (int)mem_alignment_mask, 0, 0, 0);
3685
3686 if (devblocksize == 1) {
3687 /*
3688 * the AFP client advertises a devblocksize of 1
3689 * however, its BLOCKMAP routine maps to physical
3690 * blocks that are PAGE_SIZE in size...
3691 * therefore we can't ask for I/Os that aren't page aligned
3692 * or aren't multiples of PAGE_SIZE in size
3693 * by setting devblocksize to PAGE_SIZE, we re-instate
3694 * the old behavior we had before the mem_alignment_mask
3695 * changes went in...
3696 */
3697 devblocksize = PAGE_SIZE;
3698 }
3699 next_dread:
3700 io_req_size = *read_length;
3701 iov_base = uio_curriovbase(uio);
3702
3703 max_io_size = filesize - uio->uio_offset;
3704
3705 if ((off_t)io_req_size > max_io_size)
3706 io_req_size = max_io_size;
3707
3708 offset_in_file = (u_int32_t)uio->uio_offset & (devblocksize - 1);
3709 offset_in_iovbase = (u_int32_t)iov_base & mem_alignment_mask;
3710
3711 if (offset_in_file || offset_in_iovbase) {
3712 /*
3713 * one of the 2 important offsets is misaligned
3714 * so fire an I/O through the cache for this entire vector
3715 */
3716 misaligned = 1;
3717 }
3718 if (iov_base & (devblocksize - 1)) {
3719 /*
3720 * the offset in memory must be on a device block boundary
3721 * so that we can guarantee that we can generate an
3722 * I/O that ends on a page boundary in cluster_io
3723 */
3724 misaligned = 1;
3725 }
3726 /*
3727 * When we get to this point, we know...
3728 * -- the offset into the file is on a devblocksize boundary
3729 */
3730
3731 while (io_req_size && retval == 0) {
3732 u_int32_t io_start;
3733
3734 if (cluster_hard_throttle_on(vp)) {
3735 max_rd_size = HARD_THROTTLE_MAXSIZE;
3736 max_rd_ahead = HARD_THROTTLE_MAXSIZE - 1;
3737 } else {
3738 max_rd_size = max_upl_size;
3739 max_rd_ahead = max_rd_size * 2;
3740 }
3741 io_start = io_size = io_req_size;
3742
3743 /*
3744 * First look for pages already in the cache
3745 * and move them to user space.
3746 *
3747 * cluster_copy_ubc_data returns the resid
3748 * in io_size
3749 */
3750 retval = cluster_copy_ubc_data_internal(vp, uio, (int *)&io_size, 0, 0);
3751
3752 /*
3753 * calculate the number of bytes actually copied
3754 * starting size - residual
3755 */
3756 xsize = io_start - io_size;
3757
3758 io_req_size -= xsize;
3759
3760 /*
3761 * check to see if we are finished with this request...
3762 */
3763 if (io_req_size == 0 || misaligned) {
3764 /*
3765 * see if there's another uio vector to
3766 * process that's of type IO_DIRECT
3767 *
3768 * break out of while loop to get there
3769 */
3770 break;
3771 }
3772 /*
3773 * assume the request ends on a device block boundary
3774 */
3775 io_min = devblocksize;
3776
3777 /*
3778 * we can handle I/O's in multiples of the device block size
3779 * however, if io_size isn't a multiple of devblocksize we
3780 * want to clip it back to the nearest page boundary since
3781 * we are going to have to go through cluster_read_copy to
3782 * deal with the 'overhang'... by clipping it to a PAGE_SIZE
3783 * multiple, we avoid asking the drive for the same physical
3784 * blocks twice.. once for the partial page at the end of the
3785 * request and a 2nd time for the page we read into the cache
3786 * (which overlaps the end of the direct read) in order to
3787 * get at the overhang bytes
3788 */
3789 if (io_size & (devblocksize - 1)) {
3790 /*
3791 * request does NOT end on a device block boundary
3792 * so clip it back to a PAGE_SIZE boundary
3793 */
3794 io_size &= ~PAGE_MASK;
3795 io_min = PAGE_SIZE;
3796 }
3797 if (retval || io_size < io_min) {
3798 /*
3799 * either an error or we only have the tail left to
3800 * complete via the copy path...
3801 * we may have already spun some portion of this request
3802 * off as async requests... we need to wait for the I/O
3803 * to complete before returning
3804 */
3805 goto wait_for_dreads;
3806 }
3807 if ((xsize = io_size) > max_rd_size)
3808 xsize = max_rd_size;
3809
3810 io_size = 0;
3811
3812 ubc_range_op(vp, uio->uio_offset, uio->uio_offset + xsize, UPL_ROP_ABSENT, (int *)&io_size);
3813
3814 if (io_size == 0) {
3815 /*
3816 * a page must have just come into the cache
3817 * since the first page in this range is no
3818 * longer absent, go back and re-evaluate
3819 */
3820 continue;
3821 }
3822 iov_base = uio_curriovbase(uio);
3823
3824 upl_offset = (vm_offset_t)((u_int32_t)iov_base & PAGE_MASK);
3825 upl_needed_size = (upl_offset + io_size + (PAGE_SIZE -1)) & ~PAGE_MASK;
3826
3827 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 72)) | DBG_FUNC_START,
3828 (int)upl_offset, upl_needed_size, (int)iov_base, io_size, 0);
3829
3830 if (upl_offset == 0 && ((io_size & PAGE_MASK) == 0)) {
3831 no_zero_fill = 1;
3832 abort_flag = UPL_ABORT_DUMP_PAGES | UPL_ABORT_FREE_ON_EMPTY;
3833 } else {
3834 no_zero_fill = 0;
3835 abort_flag = UPL_ABORT_FREE_ON_EMPTY;
3836 }
3837 for (force_data_sync = 0; force_data_sync < 3; force_data_sync++) {
3838 pages_in_pl = 0;
3839 upl_size = upl_needed_size;
3840 upl_flags = UPL_FILE_IO | UPL_NO_SYNC | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE;
3841
3842 if (no_zero_fill)
3843 upl_flags |= UPL_NOZEROFILL;
3844 if (force_data_sync)
3845 upl_flags |= UPL_FORCE_DATA_SYNC;
3846
3847 kret = vm_map_create_upl(current_map(),
3848 (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)),
3849 &upl_size, &upl, NULL, &pages_in_pl, &upl_flags);
3850
3851 if (kret != KERN_SUCCESS) {
3852 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 72)) | DBG_FUNC_END,
3853 (int)upl_offset, upl_size, io_size, kret, 0);
3854 /*
3855 * failed to get pagelist
3856 *
3857 * we may have already spun some portion of this request
3858 * off as async requests... we need to wait for the I/O
3859 * to complete before returning
3860 */
3861 goto wait_for_dreads;
3862 }
3863 pages_in_pl = upl_size / PAGE_SIZE;
3864 pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
3865
3866 for (i = 0; i < pages_in_pl; i++) {
3867 if (!upl_valid_page(pl, i))
3868 break;
3869 }
3870 if (i == pages_in_pl)
3871 break;
3872
3873 ubc_upl_abort(upl, abort_flag);
3874 }
3875 if (force_data_sync >= 3) {
3876 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 72)) | DBG_FUNC_END,
3877 (int)upl_offset, upl_size, io_size, kret, 0);
3878
3879 goto wait_for_dreads;
3880 }
3881 /*
3882 * Consider the possibility that upl_size wasn't satisfied.
3883 */
3884 if (upl_size < upl_needed_size) {
3885 if (upl_size && upl_offset == 0)
3886 io_size = upl_size;
3887 else
3888 io_size = 0;
3889 }
3890 if (io_size == 0) {
3891 ubc_upl_abort(upl, abort_flag);
3892 goto wait_for_dreads;
3893 }
3894 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 72)) | DBG_FUNC_END,
3895 (int)upl_offset, upl_size, io_size, kret, 0);
3896
3897 /*
3898 * request asynchronously so that we can overlap
3899 * the preparation of the next I/O
3900 * if there are already too many outstanding reads
3901 * wait until some have completed before issuing the next read
3902 */
3903 lck_mtx_lock(cl_mtxp);
3904
3905 while ((iostate.io_issued - iostate.io_completed) > max_rd_ahead) {
3906 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_START,
3907 iostate.io_issued, iostate.io_completed, max_rd_ahead, 0, 0);
3908
3909 iostate.io_wanted = 1;
3910 msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_read_direct", NULL);
3911
3912 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_END,
3913 iostate.io_issued, iostate.io_completed, max_rd_ahead, 0, 0);
3914 }
3915 lck_mtx_unlock(cl_mtxp);
3916
3917 if (iostate.io_error) {
3918 /*
3919 * one of the earlier reads we issued ran into a hard error
3920 * don't issue any more reads, cleanup the UPL
3921 * that was just created but not used, then
3922 * go wait for any other reads to complete before
3923 * returning the error to the caller
3924 */
3925 ubc_upl_abort(upl, abort_flag);
3926
3927 goto wait_for_dreads;
3928 }
3929 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 73)) | DBG_FUNC_START,
3930 (int)upl, (int)upl_offset, (int)uio->uio_offset, io_size, 0);
3931
3932 if (no_zero_fill)
3933 io_flag = CL_COMMIT | CL_READ | CL_ASYNC | CL_NOZERO | CL_DIRECT_IO | bflag;
3934 else
3935 io_flag = CL_COMMIT | CL_READ | CL_ASYNC | CL_NOZERO | CL_DIRECT_IO | CL_PRESERVE | bflag;
3936
3937 retval = cluster_io(vp, upl, upl_offset, uio->uio_offset, io_size, io_flag, (buf_t)NULL, &iostate, callback, callback_arg);
3938
3939 /*
3940 * update the uio structure
3941 */
3942 uio_update(uio, (user_size_t)io_size);
3943
3944 io_req_size -= io_size;
3945
3946 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 73)) | DBG_FUNC_END,
3947 (int)upl, (int)uio->uio_offset, io_req_size, retval, 0);
3948
3949 } /* end while */
3950
3951 if (retval == 0 && iostate.io_error == 0 && io_req_size == 0 && uio->uio_offset < filesize) {
3952
3953 retval = cluster_io_type(uio, read_type, read_length, 0);
3954
3955 if (retval == 0 && *read_type == IO_DIRECT) {
3956
3957 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 70)) | DBG_FUNC_NONE,
3958 (int)uio->uio_offset, (int)filesize, *read_type, *read_length, 0);
3959
3960 goto next_dread;
3961 }
3962 }
3963
3964 wait_for_dreads:
3965 if (iostate.io_issued) {
3966 /*
3967 * make sure all async reads that are part of this stream
3968 * have completed before we return
3969 */
3970 lck_mtx_lock(cl_mtxp);
3971
3972 while (iostate.io_issued != iostate.io_completed) {
3973 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_START,
3974 iostate.io_issued, iostate.io_completed, 0, 0, 0);
3975
3976 iostate.io_wanted = 1;
3977 msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_read_direct", NULL);
3978
3979 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_END,
3980 iostate.io_issued, iostate.io_completed, 0, 0, 0);
3981 }
3982 lck_mtx_unlock(cl_mtxp);
3983 }
3984
3985 if (iostate.io_error)
3986 retval = iostate.io_error;
3987
3988 if (io_req_size && retval == 0) {
3989 /*
3990 * we couldn't handle the tail of this request in DIRECT mode
3991 * so fire it through the copy path
3992 */
3993 retval = cluster_read_copy(vp, uio, io_req_size, filesize, flags, callback, callback_arg);
3994
3995 *read_type = IO_UNKNOWN;
3996 }
3997 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 70)) | DBG_FUNC_END,
3998 (int)uio->uio_offset, (int)uio_resid(uio), io_req_size, retval, 0);
3999
4000 return (retval);
4001 }
4002
4003
4004 static int
4005 cluster_read_contig(vnode_t vp, struct uio *uio, off_t filesize, int *read_type, u_int32_t *read_length,
4006 int (*callback)(buf_t, void *), void *callback_arg, int flags)
4007 {
4008 upl_page_info_t *pl;
4009 upl_t upl[MAX_VECTS];
4010 vm_offset_t upl_offset;
4011 addr64_t dst_paddr = 0;
4012 user_addr_t iov_base;
4013 off_t max_size;
4014 vm_size_t upl_size;
4015 vm_size_t upl_needed_size;
4016 mach_msg_type_number_t pages_in_pl;
4017 int upl_flags;
4018 kern_return_t kret;
4019 struct clios iostate;
4020 int error= 0;
4021 int cur_upl = 0;
4022 int num_upl = 0;
4023 int n;
4024 u_int32_t xsize;
4025 u_int32_t io_size;
4026 u_int32_t devblocksize;
4027 u_int32_t mem_alignment_mask;
4028 u_int32_t tail_size = 0;
4029 int bflag;
4030
4031 if (flags & IO_PASSIVE)
4032 bflag = CL_PASSIVE;
4033 else
4034 bflag = 0;
4035
4036 /*
4037 * When we enter this routine, we know
4038 * -- the read_length will not exceed the current iov_len
4039 * -- the target address is physically contiguous for read_length
4040 */
4041 cluster_syncup(vp, filesize, callback, callback_arg);
4042
4043 devblocksize = (u_int32_t)vp->v_mount->mnt_devblocksize;
4044 mem_alignment_mask = (u_int32_t)vp->v_mount->mnt_alignmentmask;
4045
4046 iostate.io_completed = 0;
4047 iostate.io_issued = 0;
4048 iostate.io_error = 0;
4049 iostate.io_wanted = 0;
4050
4051 next_cread:
4052 io_size = *read_length;
4053
4054 max_size = filesize - uio->uio_offset;
4055
4056 if (io_size > max_size)
4057 io_size = max_size;
4058
4059 iov_base = uio_curriovbase(uio);
4060
4061 upl_offset = (vm_offset_t)((u_int32_t)iov_base & PAGE_MASK);
4062 upl_needed_size = upl_offset + io_size;
4063
4064 pages_in_pl = 0;
4065 upl_size = upl_needed_size;
4066 upl_flags = UPL_FILE_IO | UPL_NO_SYNC | UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE;
4067
4068
4069 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 92)) | DBG_FUNC_START,
4070 (int)upl_offset, (int)upl_size, (int)iov_base, io_size, 0);
4071
4072 kret = vm_map_get_upl(current_map(),
4073 (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)),
4074 &upl_size, &upl[cur_upl], NULL, &pages_in_pl, &upl_flags, 0);
4075
4076 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 92)) | DBG_FUNC_END,
4077 (int)upl_offset, upl_size, io_size, kret, 0);
4078
4079 if (kret != KERN_SUCCESS) {
4080 /*
4081 * failed to get pagelist
4082 */
4083 error = EINVAL;
4084 goto wait_for_creads;
4085 }
4086 num_upl++;
4087
4088 if (upl_size < upl_needed_size) {
4089 /*
4090 * The upl_size wasn't satisfied.
4091 */
4092 error = EINVAL;
4093 goto wait_for_creads;
4094 }
4095 pl = ubc_upl_pageinfo(upl[cur_upl]);
4096
4097 dst_paddr = ((addr64_t)upl_phys_page(pl, 0) << 12) + (addr64_t)upl_offset;
4098
4099 while (((uio->uio_offset & (devblocksize - 1)) || io_size < devblocksize) && io_size) {
4100 u_int32_t head_size;
4101
4102 head_size = devblocksize - (u_int32_t)(uio->uio_offset & (devblocksize - 1));
4103
4104 if (head_size > io_size)
4105 head_size = io_size;
4106
4107 error = cluster_align_phys_io(vp, uio, dst_paddr, head_size, CL_READ, callback, callback_arg);
4108
4109 if (error)
4110 goto wait_for_creads;
4111
4112 upl_offset += head_size;
4113 dst_paddr += head_size;
4114 io_size -= head_size;
4115
4116 iov_base += head_size;
4117 }
4118 if ((u_int32_t)iov_base & mem_alignment_mask) {
4119 /*
4120 * request doesn't set up on a memory boundary
4121 * the underlying DMA engine can handle...
4122 * return an error instead of going through
4123 * the slow copy path since the intent of this
4124 * path is direct I/O to device memory
4125 */
4126 error = EINVAL;
4127 goto wait_for_creads;
4128 }
4129
4130 tail_size = io_size & (devblocksize - 1);
4131
4132 io_size -= tail_size;
4133
4134 while (io_size && error == 0) {
4135
4136 if (io_size > MAX_IO_CONTIG_SIZE)
4137 xsize = MAX_IO_CONTIG_SIZE;
4138 else
4139 xsize = io_size;
4140 /*
4141 * request asynchronously so that we can overlap
4142 * the preparation of the next I/O... we'll do
4143 * the commit after all the I/O has completed
4144 * since its all issued against the same UPL
4145 * if there are already too many outstanding reads
4146 * wait until some have completed before issuing the next
4147 */
4148 if (iostate.io_issued) {
4149 lck_mtx_lock(cl_mtxp);
4150
4151 while ((iostate.io_issued - iostate.io_completed) > (2 * MAX_IO_CONTIG_SIZE)) {
4152 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_START,
4153 iostate.io_issued, iostate.io_completed, 2 * MAX_IO_CONTIG_SIZE, 0, 0);
4154
4155 iostate.io_wanted = 1;
4156 msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_read_contig", NULL);
4157
4158 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_END,
4159 iostate.io_issued, iostate.io_completed, 2 * MAX_IO_CONTIG_SIZE, 0, 0);
4160 }
4161 lck_mtx_unlock(cl_mtxp);
4162 }
4163 if (iostate.io_error) {
4164 /*
4165 * one of the earlier reads we issued ran into a hard error
4166 * don't issue any more reads...
4167 * go wait for any other reads to complete before
4168 * returning the error to the caller
4169 */
4170 goto wait_for_creads;
4171 }
4172 error = cluster_io(vp, upl[cur_upl], upl_offset, uio->uio_offset, xsize,
4173 CL_READ | CL_NOZERO | CL_DEV_MEMORY | CL_ASYNC | bflag,
4174 (buf_t)NULL, &iostate, callback, callback_arg);
4175 /*
4176 * The cluster_io read was issued successfully,
4177 * update the uio structure
4178 */
4179 if (error == 0) {
4180 uio_update(uio, (user_size_t)xsize);
4181
4182 dst_paddr += xsize;
4183 upl_offset += xsize;
4184 io_size -= xsize;
4185 }
4186 }
4187 if (error == 0 && iostate.io_error == 0 && tail_size == 0 && num_upl < MAX_VECTS && uio->uio_offset < filesize) {
4188
4189 error = cluster_io_type(uio, read_type, read_length, 0);
4190
4191 if (error == 0 && *read_type == IO_CONTIG) {
4192 cur_upl++;
4193 goto next_cread;
4194 }
4195 } else
4196 *read_type = IO_UNKNOWN;
4197
4198 wait_for_creads:
4199 /*
4200 * make sure all async reads that are part of this stream
4201 * have completed before we proceed
4202 */
4203 lck_mtx_lock(cl_mtxp);
4204
4205 while (iostate.io_issued != iostate.io_completed) {
4206 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_START,
4207 iostate.io_issued, iostate.io_completed, 0, 0, 0);
4208
4209 iostate.io_wanted = 1;
4210 msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_read_contig", NULL);
4211
4212 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_END,
4213 iostate.io_issued, iostate.io_completed, 0, 0, 0);
4214 }
4215 lck_mtx_unlock(cl_mtxp);
4216
4217 if (iostate.io_error)
4218 error = iostate.io_error;
4219
4220 if (error == 0 && tail_size)
4221 error = cluster_align_phys_io(vp, uio, dst_paddr, tail_size, CL_READ, callback, callback_arg);
4222
4223 for (n = 0; n < num_upl; n++)
4224 /*
4225 * just release our hold on each physically contiguous
4226 * region without changing any state
4227 */
4228 ubc_upl_abort(upl[n], 0);
4229
4230 return (error);
4231 }
4232
4233
4234 static int
4235 cluster_io_type(struct uio *uio, int *io_type, u_int32_t *io_length, u_int32_t min_length)
4236 {
4237 user_size_t iov_len;
4238 user_addr_t iov_base = 0;
4239 upl_t upl;
4240 vm_size_t upl_size;
4241 int upl_flags;
4242 int retval = 0;
4243
4244 /*
4245 * skip over any emtpy vectors
4246 */
4247 uio_update(uio, (user_size_t)0);
4248
4249 iov_len = uio_curriovlen(uio);
4250
4251 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 94)) | DBG_FUNC_START, (int)uio, (int)iov_len, 0, 0, 0);
4252
4253 if (iov_len) {
4254 iov_base = uio_curriovbase(uio);
4255 /*
4256 * make sure the size of the vector isn't too big...
4257 * internally, we want to handle all of the I/O in
4258 * chunk sizes that fit in a 32 bit int
4259 */
4260 if (iov_len > (user_size_t)MAX_IO_REQUEST_SIZE)
4261 upl_size = MAX_IO_REQUEST_SIZE;
4262 else
4263 upl_size = (u_int32_t)iov_len;
4264
4265 upl_flags = UPL_QUERY_OBJECT_TYPE;
4266
4267 if ((vm_map_get_upl(current_map(),
4268 (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)),
4269 &upl_size, &upl, NULL, NULL, &upl_flags, 0)) != KERN_SUCCESS) {
4270 /*
4271 * the user app must have passed in an invalid address
4272 */
4273 retval = EFAULT;
4274 }
4275 if (upl_size == 0)
4276 retval = EFAULT;
4277
4278 *io_length = upl_size;
4279
4280 if (upl_flags & UPL_PHYS_CONTIG)
4281 *io_type = IO_CONTIG;
4282 else if (iov_len >= min_length)
4283 *io_type = IO_DIRECT;
4284 else
4285 *io_type = IO_COPY;
4286 } else {
4287 /*
4288 * nothing left to do for this uio
4289 */
4290 *io_length = 0;
4291 *io_type = IO_UNKNOWN;
4292 }
4293 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 94)) | DBG_FUNC_END, (int)iov_base, *io_type, *io_length, retval, 0);
4294
4295 return (retval);
4296 }
4297
4298
4299 /*
4300 * generate advisory I/O's in the largest chunks possible
4301 * the completed pages will be released into the VM cache
4302 */
4303 int
4304 advisory_read(vnode_t vp, off_t filesize, off_t f_offset, int resid)
4305 {
4306 return advisory_read_ext(vp, filesize, f_offset, resid, NULL, NULL, CL_PASSIVE);
4307 }
4308
4309 int
4310 advisory_read_ext(vnode_t vp, off_t filesize, off_t f_offset, int resid, int (*callback)(buf_t, void *), void *callback_arg, int bflag)
4311 {
4312 upl_page_info_t *pl;
4313 upl_t upl;
4314 vm_offset_t upl_offset;
4315 int upl_size;
4316 off_t upl_f_offset;
4317 int start_offset;
4318 int start_pg;
4319 int last_pg;
4320 int pages_in_upl;
4321 off_t max_size;
4322 int io_size;
4323 kern_return_t kret;
4324 int retval = 0;
4325 int issued_io;
4326 int skip_range;
4327 uint32_t max_io_size;
4328
4329
4330 if ( !UBCINFOEXISTS(vp))
4331 return(EINVAL);
4332
4333 max_io_size = cluster_max_io_size(vp->v_mount, CL_READ);
4334
4335 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 60)) | DBG_FUNC_START,
4336 (int)f_offset, resid, (int)filesize, 0, 0);
4337
4338 while (resid && f_offset < filesize && retval == 0) {
4339 /*
4340 * compute the size of the upl needed to encompass
4341 * the requested read... limit each call to cluster_io
4342 * to the maximum UPL size... cluster_io will clip if
4343 * this exceeds the maximum io_size for the device,
4344 * make sure to account for
4345 * a starting offset that's not page aligned
4346 */
4347 start_offset = (int)(f_offset & PAGE_MASK_64);
4348 upl_f_offset = f_offset - (off_t)start_offset;
4349 max_size = filesize - f_offset;
4350
4351 if (resid < max_size)
4352 io_size = resid;
4353 else
4354 io_size = max_size;
4355
4356 upl_size = (start_offset + io_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
4357 if ((uint32_t)upl_size > max_io_size)
4358 upl_size = max_io_size;
4359
4360 skip_range = 0;
4361 /*
4362 * return the number of contiguously present pages in the cache
4363 * starting at upl_f_offset within the file
4364 */
4365 ubc_range_op(vp, upl_f_offset, upl_f_offset + upl_size, UPL_ROP_PRESENT, &skip_range);
4366
4367 if (skip_range) {
4368 /*
4369 * skip over pages already present in the cache
4370 */
4371 io_size = skip_range - start_offset;
4372
4373 f_offset += io_size;
4374 resid -= io_size;
4375
4376 if (skip_range == upl_size)
4377 continue;
4378 /*
4379 * have to issue some real I/O
4380 * at this point, we know it's starting on a page boundary
4381 * because we've skipped over at least the first page in the request
4382 */
4383 start_offset = 0;
4384 upl_f_offset += skip_range;
4385 upl_size -= skip_range;
4386 }
4387 pages_in_upl = upl_size / PAGE_SIZE;
4388
4389 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 61)) | DBG_FUNC_START,
4390 (int)upl, (int)upl_f_offset, upl_size, start_offset, 0);
4391
4392 kret = ubc_create_upl(vp,
4393 upl_f_offset,
4394 upl_size,
4395 &upl,
4396 &pl,
4397 UPL_RET_ONLY_ABSENT | UPL_SET_LITE);
4398 if (kret != KERN_SUCCESS)
4399 return(retval);
4400 issued_io = 0;
4401
4402 /*
4403 * before we start marching forward, we must make sure we end on
4404 * a present page, otherwise we will be working with a freed
4405 * upl
4406 */
4407 for (last_pg = pages_in_upl - 1; last_pg >= 0; last_pg--) {
4408 if (upl_page_present(pl, last_pg))
4409 break;
4410 }
4411 pages_in_upl = last_pg + 1;
4412
4413
4414 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 61)) | DBG_FUNC_END,
4415 (int)upl, (int)upl_f_offset, upl_size, start_offset, 0);
4416
4417
4418 for (last_pg = 0; last_pg < pages_in_upl; ) {
4419 /*
4420 * scan from the beginning of the upl looking for the first
4421 * page that is present.... this will become the first page in
4422 * the request we're going to make to 'cluster_io'... if all
4423 * of the pages are absent, we won't call through to 'cluster_io'
4424 */
4425 for (start_pg = last_pg; start_pg < pages_in_upl; start_pg++) {
4426 if (upl_page_present(pl, start_pg))
4427 break;
4428 }
4429
4430 /*
4431 * scan from the starting present page looking for an absent
4432 * page before the end of the upl is reached, if we
4433 * find one, then it will terminate the range of pages being
4434 * presented to 'cluster_io'
4435 */
4436 for (last_pg = start_pg; last_pg < pages_in_upl; last_pg++) {
4437 if (!upl_page_present(pl, last_pg))
4438 break;
4439 }
4440
4441 if (last_pg > start_pg) {
4442 /*
4443 * we found a range of pages that must be filled
4444 * if the last page in this range is the last page of the file
4445 * we may have to clip the size of it to keep from reading past
4446 * the end of the last physical block associated with the file
4447 */
4448 upl_offset = start_pg * PAGE_SIZE;
4449 io_size = (last_pg - start_pg) * PAGE_SIZE;
4450
4451 if ((upl_f_offset + upl_offset + io_size) > filesize)
4452 io_size = filesize - (upl_f_offset + upl_offset);
4453
4454 /*
4455 * issue an asynchronous read to cluster_io
4456 */
4457 retval = cluster_io(vp, upl, upl_offset, upl_f_offset + upl_offset, io_size,
4458 CL_ASYNC | CL_READ | CL_COMMIT | CL_AGE | bflag, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
4459
4460 issued_io = 1;
4461 }
4462 }
4463 if (issued_io == 0)
4464 ubc_upl_abort(upl, 0);
4465
4466 io_size = upl_size - start_offset;
4467
4468 if (io_size > resid)
4469 io_size = resid;
4470 f_offset += io_size;
4471 resid -= io_size;
4472 }
4473
4474 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 60)) | DBG_FUNC_END,
4475 (int)f_offset, resid, retval, 0, 0);
4476
4477 return(retval);
4478 }
4479
4480
4481 int
4482 cluster_push(vnode_t vp, int flags)
4483 {
4484 return cluster_push_ext(vp, flags, NULL, NULL);
4485 }
4486
4487
4488 int
4489 cluster_push_ext(vnode_t vp, int flags, int (*callback)(buf_t, void *), void *callback_arg)
4490 {
4491 int retval;
4492 struct cl_writebehind *wbp;
4493
4494 if ( !UBCINFOEXISTS(vp)) {
4495 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 53)) | DBG_FUNC_NONE, (int)vp, flags, 0, -1, 0);
4496 return (0);
4497 }
4498 /* return if deferred write is set */
4499 if (((unsigned int)vfs_flags(vp->v_mount) & MNT_DEFWRITE) && (flags & IO_DEFWRITE)) {
4500 return (0);
4501 }
4502 if ((wbp = cluster_get_wbp(vp, CLW_RETURNLOCKED)) == NULL) {
4503 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 53)) | DBG_FUNC_NONE, (int)vp, flags, 0, -2, 0);
4504 return (0);
4505 }
4506 if (wbp->cl_number == 0 && wbp->cl_scmap == NULL) {
4507 lck_mtx_unlock(&wbp->cl_lockw);
4508
4509 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 53)) | DBG_FUNC_NONE, (int)vp, flags, 0, -3, 0);
4510 return(0);
4511 }
4512 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 53)) | DBG_FUNC_START,
4513 (int)wbp->cl_scmap, wbp->cl_number, flags, 0, 0);
4514
4515 if (wbp->cl_scmap) {
4516 sparse_cluster_push(wbp, vp, ubc_getsize(vp), PUSH_ALL | IO_PASSIVE, callback, callback_arg);
4517
4518 retval = 1;
4519 } else
4520 retval = cluster_try_push(wbp, vp, ubc_getsize(vp), PUSH_ALL | IO_PASSIVE, callback, callback_arg);
4521
4522 lck_mtx_unlock(&wbp->cl_lockw);
4523
4524 if (flags & IO_SYNC)
4525 (void)vnode_waitforwrites(vp, 0, 0, 0, "cluster_push");
4526
4527 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 53)) | DBG_FUNC_END,
4528 (int)wbp->cl_scmap, wbp->cl_number, retval, 0, 0);
4529
4530 return (retval);
4531 }
4532
4533
4534 __private_extern__ void
4535 cluster_release(struct ubc_info *ubc)
4536 {
4537 struct cl_writebehind *wbp;
4538 struct cl_readahead *rap;
4539
4540 if ((wbp = ubc->cl_wbehind)) {
4541
4542 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 81)) | DBG_FUNC_START, (int)ubc, (int)wbp->cl_scmap, wbp->cl_scdirty, 0, 0);
4543
4544 if (wbp->cl_scmap)
4545 vfs_drt_control(&(wbp->cl_scmap), 0);
4546 } else {
4547 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 81)) | DBG_FUNC_START, (int)ubc, 0, 0, 0, 0);
4548 }
4549
4550 rap = ubc->cl_rahead;
4551
4552 if (wbp != NULL) {
4553 lck_mtx_destroy(&wbp->cl_lockw, cl_mtx_grp);
4554 FREE_ZONE((void *)wbp, sizeof *wbp, M_CLWRBEHIND);
4555 }
4556 if ((rap = ubc->cl_rahead)) {
4557 lck_mtx_destroy(&rap->cl_lockr, cl_mtx_grp);
4558 FREE_ZONE((void *)rap, sizeof *rap, M_CLRDAHEAD);
4559 }
4560 ubc->cl_rahead = NULL;
4561 ubc->cl_wbehind = NULL;
4562
4563 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 81)) | DBG_FUNC_END, (int)ubc, (int)rap, (int)wbp, 0, 0);
4564 }
4565
4566
4567 static int
4568 cluster_try_push(struct cl_writebehind *wbp, vnode_t vp, off_t EOF, int push_flag, int (*callback)(buf_t, void *), void *callback_arg)
4569 {
4570 int cl_index;
4571 int cl_index1;
4572 int min_index;
4573 int cl_len;
4574 int cl_pushed = 0;
4575 struct cl_wextent l_clusters[MAX_CLUSTERS];
4576 u_int max_cluster_pgcount;
4577
4578
4579 max_cluster_pgcount = MAX_CLUSTER_SIZE(vp) / PAGE_SIZE;
4580 /*
4581 * the write behind context exists and has
4582 * already been locked...
4583 */
4584 if (wbp->cl_number == 0)
4585 /*
4586 * no clusters to push
4587 * return number of empty slots
4588 */
4589 return (MAX_CLUSTERS);
4590
4591 /*
4592 * make a local 'sorted' copy of the clusters
4593 * and clear wbp->cl_number so that new clusters can
4594 * be developed
4595 */
4596 for (cl_index = 0; cl_index < wbp->cl_number; cl_index++) {
4597 for (min_index = -1, cl_index1 = 0; cl_index1 < wbp->cl_number; cl_index1++) {
4598 if (wbp->cl_clusters[cl_index1].b_addr == wbp->cl_clusters[cl_index1].e_addr)
4599 continue;
4600 if (min_index == -1)
4601 min_index = cl_index1;
4602 else if (wbp->cl_clusters[cl_index1].b_addr < wbp->cl_clusters[min_index].b_addr)
4603 min_index = cl_index1;
4604 }
4605 if (min_index == -1)
4606 break;
4607 l_clusters[cl_index].b_addr = wbp->cl_clusters[min_index].b_addr;
4608 l_clusters[cl_index].e_addr = wbp->cl_clusters[min_index].e_addr;
4609 l_clusters[cl_index].io_flags = wbp->cl_clusters[min_index].io_flags;
4610
4611 wbp->cl_clusters[min_index].b_addr = wbp->cl_clusters[min_index].e_addr;
4612 }
4613 wbp->cl_number = 0;
4614
4615 cl_len = cl_index;
4616
4617 if ( (push_flag & PUSH_DELAY) && cl_len == MAX_CLUSTERS ) {
4618 int i;
4619
4620 /*
4621 * determine if we appear to be writing the file sequentially
4622 * if not, by returning without having pushed any clusters
4623 * we will cause this vnode to be pushed into the sparse cluster mechanism
4624 * used for managing more random I/O patterns
4625 *
4626 * we know that we've got all clusters currently in use and the next write doesn't fit into one of them...
4627 * that's why we're in try_push with PUSH_DELAY...
4628 *
4629 * check to make sure that all the clusters except the last one are 'full'... and that each cluster
4630 * is adjacent to the next (i.e. we're looking for sequential writes) they were sorted above
4631 * so we can just make a simple pass through, up to, but not including the last one...
4632 * note that e_addr is not inclusive, so it will be equal to the b_addr of the next cluster if they
4633 * are sequential
4634 *
4635 * we let the last one be partial as long as it was adjacent to the previous one...
4636 * we need to do this to deal with multi-threaded servers that might write an I/O or 2 out
4637 * of order... if this occurs at the tail of the last cluster, we don't want to fall into the sparse cluster world...
4638 */
4639 for (i = 0; i < MAX_CLUSTERS - 1; i++) {
4640 if ((l_clusters[i].e_addr - l_clusters[i].b_addr) != max_cluster_pgcount)
4641 goto dont_try;
4642 if (l_clusters[i].e_addr != l_clusters[i+1].b_addr)
4643 goto dont_try;
4644 }
4645 }
4646 /*
4647 * drop the lock while we're firing off the I/Os...
4648 * this is safe since I'm working off of a private sorted copy
4649 * of the clusters, and I'm going to re-evaluate the public
4650 * state after I retake the lock
4651 *
4652 * we need to drop it to avoid a lock inversion when trying to
4653 * grab pages into the UPL... another thread in 'write' may
4654 * have these pages in its UPL and be blocked trying to
4655 * gain the write-behind lock for this vnode
4656 */
4657 lck_mtx_unlock(&wbp->cl_lockw);
4658
4659 for (cl_index = 0; cl_index < cl_len; cl_index++) {
4660 int flags;
4661 struct cl_extent cl;
4662
4663 /*
4664 * try to push each cluster in turn...
4665 */
4666 if (l_clusters[cl_index].io_flags & CLW_IONOCACHE)
4667 flags = IO_NOCACHE;
4668 else
4669 flags = 0;
4670
4671 if ((l_clusters[cl_index].io_flags & CLW_IOPASSIVE) || (push_flag & IO_PASSIVE))
4672 flags |= IO_PASSIVE;
4673
4674 if (push_flag & PUSH_SYNC)
4675 flags |= IO_SYNC;
4676
4677 cl.b_addr = l_clusters[cl_index].b_addr;
4678 cl.e_addr = l_clusters[cl_index].e_addr;
4679
4680 cluster_push_now(vp, &cl, EOF, flags, callback, callback_arg);
4681
4682 l_clusters[cl_index].b_addr = 0;
4683 l_clusters[cl_index].e_addr = 0;
4684
4685 cl_pushed++;
4686
4687 if ( !(push_flag & PUSH_ALL) )
4688 break;
4689 }
4690 lck_mtx_lock(&wbp->cl_lockw);
4691
4692 dont_try:
4693 if (cl_len > cl_pushed) {
4694 /*
4695 * we didn't push all of the clusters, so
4696 * lets try to merge them back in to the vnode
4697 */
4698 if ((MAX_CLUSTERS - wbp->cl_number) < (cl_len - cl_pushed)) {
4699 /*
4700 * we picked up some new clusters while we were trying to
4701 * push the old ones... this can happen because I've dropped
4702 * the vnode lock... the sum of the
4703 * leftovers plus the new cluster count exceeds our ability
4704 * to represent them, so switch to the sparse cluster mechanism
4705 *
4706 * collect the active public clusters...
4707 */
4708 sparse_cluster_switch(wbp, vp, EOF, callback, callback_arg);
4709
4710 for (cl_index = 0, cl_index1 = 0; cl_index < cl_len; cl_index++) {
4711 if (l_clusters[cl_index].b_addr == l_clusters[cl_index].e_addr)
4712 continue;
4713 wbp->cl_clusters[cl_index1].b_addr = l_clusters[cl_index].b_addr;
4714 wbp->cl_clusters[cl_index1].e_addr = l_clusters[cl_index].e_addr;
4715 wbp->cl_clusters[cl_index1].io_flags = l_clusters[cl_index].io_flags;
4716
4717 cl_index1++;
4718 }
4719 /*
4720 * update the cluster count
4721 */
4722 wbp->cl_number = cl_index1;
4723
4724 /*
4725 * and collect the original clusters that were moved into the
4726 * local storage for sorting purposes
4727 */
4728 sparse_cluster_switch(wbp, vp, EOF, callback, callback_arg);
4729
4730 } else {
4731 /*
4732 * we've got room to merge the leftovers back in
4733 * just append them starting at the next 'hole'
4734 * represented by wbp->cl_number
4735 */
4736 for (cl_index = 0, cl_index1 = wbp->cl_number; cl_index < cl_len; cl_index++) {
4737 if (l_clusters[cl_index].b_addr == l_clusters[cl_index].e_addr)
4738 continue;
4739
4740 wbp->cl_clusters[cl_index1].b_addr = l_clusters[cl_index].b_addr;
4741 wbp->cl_clusters[cl_index1].e_addr = l_clusters[cl_index].e_addr;
4742 wbp->cl_clusters[cl_index1].io_flags = l_clusters[cl_index].io_flags;
4743
4744 cl_index1++;
4745 }
4746 /*
4747 * update the cluster count
4748 */
4749 wbp->cl_number = cl_index1;
4750 }
4751 }
4752 return (MAX_CLUSTERS - wbp->cl_number);
4753 }
4754
4755
4756
4757 static int
4758 cluster_push_now(vnode_t vp, struct cl_extent *cl, off_t EOF, int flags, int (*callback)(buf_t, void *), void *callback_arg)
4759 {
4760 upl_page_info_t *pl;
4761 upl_t upl;
4762 vm_offset_t upl_offset;
4763 int upl_size;
4764 off_t upl_f_offset;
4765 int pages_in_upl;
4766 int start_pg;
4767 int last_pg;
4768 int io_size;
4769 int io_flags;
4770 int upl_flags;
4771 int bflag;
4772 int size;
4773 int error = 0;
4774 int retval;
4775 kern_return_t kret;
4776
4777 if (flags & IO_PASSIVE)
4778 bflag = CL_PASSIVE;
4779 else
4780 bflag = 0;
4781
4782 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 51)) | DBG_FUNC_START,
4783 (int)cl->b_addr, (int)cl->e_addr, (int)EOF, flags, 0);
4784
4785 if ((pages_in_upl = (int)(cl->e_addr - cl->b_addr)) == 0) {
4786 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 51)) | DBG_FUNC_END, 1, 0, 0, 0, 0);
4787
4788 return (0);
4789 }
4790 upl_size = pages_in_upl * PAGE_SIZE;
4791 upl_f_offset = (off_t)(cl->b_addr * PAGE_SIZE_64);
4792
4793 if (upl_f_offset + upl_size >= EOF) {
4794
4795 if (upl_f_offset >= EOF) {
4796 /*
4797 * must have truncated the file and missed
4798 * clearing a dangling cluster (i.e. it's completely
4799 * beyond the new EOF
4800 */
4801 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 51)) | DBG_FUNC_END, 1, 1, 0, 0, 0);
4802
4803 return(0);
4804 }
4805 size = EOF - upl_f_offset;
4806
4807 upl_size = (size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
4808 pages_in_upl = upl_size / PAGE_SIZE;
4809 } else
4810 size = upl_size;
4811
4812 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 41)) | DBG_FUNC_START, upl_size, size, 0, 0, 0);
4813
4814 /*
4815 * by asking for UPL_COPYOUT_FROM and UPL_RET_ONLY_DIRTY, we get the following desirable behavior
4816 *
4817 * - only pages that are currently dirty are returned... these are the ones we need to clean
4818 * - the hardware dirty bit is cleared when the page is gathered into the UPL... the software dirty bit is set
4819 * - if we have to abort the I/O for some reason, the software dirty bit is left set since we didn't clean the page
4820 * - when we commit the page, the software dirty bit is cleared... the hardware dirty bit is untouched so that if
4821 * someone dirties this page while the I/O is in progress, we don't lose track of the new state
4822 *
4823 * when the I/O completes, we no longer ask for an explicit clear of the DIRTY state (either soft or hard)
4824 */
4825
4826 if ((vp->v_flag & VNOCACHE_DATA) || (flags & IO_NOCACHE))
4827 upl_flags = UPL_COPYOUT_FROM | UPL_RET_ONLY_DIRTY | UPL_SET_LITE | UPL_WILL_BE_DUMPED;
4828 else
4829 upl_flags = UPL_COPYOUT_FROM | UPL_RET_ONLY_DIRTY | UPL_SET_LITE;
4830
4831 kret = ubc_create_upl(vp,
4832 upl_f_offset,
4833 upl_size,
4834 &upl,
4835 &pl,
4836 upl_flags);
4837 if (kret != KERN_SUCCESS)
4838 panic("cluster_push: failed to get pagelist");
4839
4840 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 41)) | DBG_FUNC_END, (int)upl, upl_f_offset, 0, 0, 0);
4841
4842 /*
4843 * since we only asked for the dirty pages back
4844 * it's possible that we may only get a few or even none, so...
4845 * before we start marching forward, we must make sure we know
4846 * where the last present page is in the UPL, otherwise we could
4847 * end up working with a freed upl due to the FREE_ON_EMPTY semantics
4848 * employed by commit_range and abort_range.
4849 */
4850 for (last_pg = pages_in_upl - 1; last_pg >= 0; last_pg--) {
4851 if (upl_page_present(pl, last_pg))
4852 break;
4853 }
4854 pages_in_upl = last_pg + 1;
4855
4856 if (pages_in_upl == 0) {
4857 ubc_upl_abort(upl, 0);
4858
4859 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 51)) | DBG_FUNC_END, 1, 2, 0, 0, 0);
4860 return(0);
4861 }
4862
4863 for (last_pg = 0; last_pg < pages_in_upl; ) {
4864 /*
4865 * find the next dirty page in the UPL
4866 * this will become the first page in the
4867 * next I/O to generate
4868 */
4869 for (start_pg = last_pg; start_pg < pages_in_upl; start_pg++) {
4870 if (upl_dirty_page(pl, start_pg))
4871 break;
4872 if (upl_page_present(pl, start_pg))
4873 /*
4874 * RET_ONLY_DIRTY will return non-dirty 'precious' pages
4875 * just release these unchanged since we're not going
4876 * to steal them or change their state
4877 */
4878 ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
4879 }
4880 if (start_pg >= pages_in_upl)
4881 /*
4882 * done... no more dirty pages to push
4883 */
4884 break;
4885 if (start_pg > last_pg)
4886 /*
4887 * skipped over some non-dirty pages
4888 */
4889 size -= ((start_pg - last_pg) * PAGE_SIZE);
4890
4891 /*
4892 * find a range of dirty pages to write
4893 */
4894 for (last_pg = start_pg; last_pg < pages_in_upl; last_pg++) {
4895 if (!upl_dirty_page(pl, last_pg))
4896 break;
4897 }
4898 upl_offset = start_pg * PAGE_SIZE;
4899
4900 io_size = min(size, (last_pg - start_pg) * PAGE_SIZE);
4901
4902 io_flags = CL_THROTTLE | CL_COMMIT | CL_AGE | bflag;
4903
4904 if ( !(flags & IO_SYNC))
4905 io_flags |= CL_ASYNC;
4906
4907 retval = cluster_io(vp, upl, upl_offset, upl_f_offset + upl_offset, io_size,
4908 io_flags, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
4909
4910 if (error == 0 && retval)
4911 error = retval;
4912
4913 size -= io_size;
4914 }
4915 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 51)) | DBG_FUNC_END, 1, 3, 0, 0, 0);
4916
4917 return(error);
4918 }
4919
4920
4921 /*
4922 * sparse_cluster_switch is called with the write behind lock held
4923 */
4924 static void
4925 sparse_cluster_switch(struct cl_writebehind *wbp, vnode_t vp, off_t EOF, int (*callback)(buf_t, void *), void *callback_arg)
4926 {
4927 int cl_index;
4928
4929 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 78)) | DBG_FUNC_START, (int)vp, (int)wbp->cl_scmap, wbp->cl_scdirty, 0, 0);
4930
4931 if (wbp->cl_scmap == NULL)
4932 wbp->cl_scdirty = 0;
4933
4934 for (cl_index = 0; cl_index < wbp->cl_number; cl_index++) {
4935 int flags;
4936 struct cl_extent cl;
4937
4938 for (cl.b_addr = wbp->cl_clusters[cl_index].b_addr; cl.b_addr < wbp->cl_clusters[cl_index].e_addr; cl.b_addr++) {
4939
4940 if (ubc_page_op(vp, (off_t)(cl.b_addr * PAGE_SIZE_64), 0, NULL, &flags) == KERN_SUCCESS) {
4941 if (flags & UPL_POP_DIRTY) {
4942 cl.e_addr = cl.b_addr + 1;
4943
4944 sparse_cluster_add(wbp, vp, &cl, EOF, callback, callback_arg);
4945 }
4946 }
4947 }
4948 }
4949 wbp->cl_number = 0;
4950
4951 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 78)) | DBG_FUNC_END, (int)vp, (int)wbp->cl_scmap, wbp->cl_scdirty, 0, 0);
4952 }
4953
4954
4955 /*
4956 * sparse_cluster_push is called with the write behind lock held
4957 */
4958 static void
4959 sparse_cluster_push(struct cl_writebehind *wbp, vnode_t vp, off_t EOF, int push_flag, int (*callback)(buf_t, void *), void *callback_arg)
4960 {
4961 struct cl_extent cl;
4962 off_t offset;
4963 u_int length;
4964
4965 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 79)) | DBG_FUNC_START, (int)vp, (int)wbp->cl_scmap, wbp->cl_scdirty, push_flag, 0);
4966
4967 if (push_flag & PUSH_ALL)
4968 vfs_drt_control(&(wbp->cl_scmap), 1);
4969
4970 for (;;) {
4971 if (vfs_drt_get_cluster(&(wbp->cl_scmap), &offset, &length) != KERN_SUCCESS)
4972 break;
4973
4974 cl.b_addr = (daddr64_t)(offset / PAGE_SIZE_64);
4975 cl.e_addr = (daddr64_t)((offset + length) / PAGE_SIZE_64);
4976
4977 wbp->cl_scdirty -= (int)(cl.e_addr - cl.b_addr);
4978
4979 /*
4980 * drop the lock while we're firing off the I/Os...
4981 * this is safe since I've already updated the state
4982 * this lock is protecting and I'm going to re-evaluate
4983 * the public state after I retake the lock
4984 *
4985 * we need to drop it to avoid a lock inversion when trying to
4986 * grab pages into the UPL... another thread in 'write' may
4987 * have these pages in its UPL and be blocked trying to
4988 * gain the write-behind lock for this vnode
4989 */
4990 lck_mtx_unlock(&wbp->cl_lockw);
4991
4992 cluster_push_now(vp, &cl, EOF, push_flag & IO_PASSIVE, callback, callback_arg);
4993
4994 lck_mtx_lock(&wbp->cl_lockw);
4995
4996 if ( !(push_flag & PUSH_ALL) )
4997 break;
4998 }
4999 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 79)) | DBG_FUNC_END, (int)vp, (int)wbp->cl_scmap, wbp->cl_scdirty, 0, 0);
5000 }
5001
5002
5003 /*
5004 * sparse_cluster_add is called with the write behind lock held
5005 */
5006 static void
5007 sparse_cluster_add(struct cl_writebehind *wbp, vnode_t vp, struct cl_extent *cl, off_t EOF, int (*callback)(buf_t, void *), void *callback_arg)
5008 {
5009 u_int new_dirty;
5010 u_int length;
5011 off_t offset;
5012
5013 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 80)) | DBG_FUNC_START, (int)wbp->cl_scmap, wbp->cl_scdirty, (int)cl->b_addr, (int)cl->e_addr, 0);
5014
5015 offset = (off_t)(cl->b_addr * PAGE_SIZE_64);
5016 length = ((u_int)(cl->e_addr - cl->b_addr)) * PAGE_SIZE;
5017
5018 while (vfs_drt_mark_pages(&(wbp->cl_scmap), offset, length, &new_dirty) != KERN_SUCCESS) {
5019 /*
5020 * no room left in the map
5021 * only a partial update was done
5022 * push out some pages and try again
5023 */
5024 wbp->cl_scdirty += new_dirty;
5025
5026 sparse_cluster_push(wbp, vp, EOF, 0, callback, callback_arg);
5027
5028 offset += (new_dirty * PAGE_SIZE_64);
5029 length -= (new_dirty * PAGE_SIZE);
5030 }
5031 wbp->cl_scdirty += new_dirty;
5032
5033 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 80)) | DBG_FUNC_END, (int)vp, (int)wbp->cl_scmap, wbp->cl_scdirty, 0, 0);
5034 }
5035
5036
5037 static int
5038 cluster_align_phys_io(vnode_t vp, struct uio *uio, addr64_t usr_paddr, u_int32_t xsize, int flags, int (*callback)(buf_t, void *), void *callback_arg)
5039 {
5040 upl_page_info_t *pl;
5041 upl_t upl;
5042 addr64_t ubc_paddr;
5043 kern_return_t kret;
5044 int error = 0;
5045 int did_read = 0;
5046 int abort_flags;
5047 int upl_flags;
5048 int bflag;
5049
5050 if (flags & IO_PASSIVE)
5051 bflag = CL_PASSIVE;
5052 else
5053 bflag = 0;
5054
5055 upl_flags = UPL_SET_LITE;
5056
5057 if ( !(flags & CL_READ) ) {
5058 /*
5059 * "write" operation: let the UPL subsystem know
5060 * that we intend to modify the buffer cache pages
5061 * we're gathering.
5062 */
5063 upl_flags |= UPL_WILL_MODIFY;
5064 } else {
5065 /*
5066 * indicate that there is no need to pull the
5067 * mapping for this page... we're only going
5068 * to read from it, not modify it.
5069 */
5070 upl_flags |= UPL_FILE_IO;
5071 }
5072 kret = ubc_create_upl(vp,
5073 uio->uio_offset & ~PAGE_MASK_64,
5074 PAGE_SIZE,
5075 &upl,
5076 &pl,
5077 upl_flags);
5078
5079 if (kret != KERN_SUCCESS)
5080 return(EINVAL);
5081
5082 if (!upl_valid_page(pl, 0)) {
5083 /*
5084 * issue a synchronous read to cluster_io
5085 */
5086 error = cluster_io(vp, upl, 0, uio->uio_offset & ~PAGE_MASK_64, PAGE_SIZE,
5087 CL_READ | bflag, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
5088 if (error) {
5089 ubc_upl_abort_range(upl, 0, PAGE_SIZE, UPL_ABORT_DUMP_PAGES | UPL_ABORT_FREE_ON_EMPTY);
5090
5091 return(error);
5092 }
5093 did_read = 1;
5094 }
5095 ubc_paddr = ((addr64_t)upl_phys_page(pl, 0) << 12) + (addr64_t)(uio->uio_offset & PAGE_MASK_64);
5096
5097 /*
5098 * NOTE: There is no prototype for the following in BSD. It, and the definitions
5099 * of the defines for cppvPsrc, cppvPsnk, cppvFsnk, and cppvFsrc will be found in
5100 * osfmk/ppc/mappings.h. They are not included here because there appears to be no
5101 * way to do so without exporting them to kexts as well.
5102 */
5103 if (flags & CL_READ)
5104 // copypv(ubc_paddr, usr_paddr, xsize, cppvPsrc | cppvPsnk | cppvFsnk); /* Copy physical to physical and flush the destination */
5105 copypv(ubc_paddr, usr_paddr, xsize, 2 | 1 | 4); /* Copy physical to physical and flush the destination */
5106 else
5107 // copypv(usr_paddr, ubc_paddr, xsize, cppvPsrc | cppvPsnk | cppvFsrc); /* Copy physical to physical and flush the source */
5108 copypv(usr_paddr, ubc_paddr, xsize, 2 | 1 | 8); /* Copy physical to physical and flush the source */
5109
5110 if ( !(flags & CL_READ) || (upl_valid_page(pl, 0) && upl_dirty_page(pl, 0))) {
5111 /*
5112 * issue a synchronous write to cluster_io
5113 */
5114 error = cluster_io(vp, upl, 0, uio->uio_offset & ~PAGE_MASK_64, PAGE_SIZE,
5115 bflag, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
5116 }
5117 if (error == 0)
5118 uio_update(uio, (user_size_t)xsize);
5119
5120 if (did_read)
5121 abort_flags = UPL_ABORT_FREE_ON_EMPTY;
5122 else
5123 abort_flags = UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_DUMP_PAGES;
5124
5125 ubc_upl_abort_range(upl, 0, PAGE_SIZE, abort_flags);
5126
5127 return (error);
5128 }
5129
5130
5131
5132 int
5133 cluster_copy_upl_data(struct uio *uio, upl_t upl, int upl_offset, int *io_resid)
5134 {
5135 int pg_offset;
5136 int pg_index;
5137 int csize;
5138 int segflg;
5139 int retval = 0;
5140 int xsize;
5141 upl_page_info_t *pl;
5142
5143 xsize = *io_resid;
5144
5145 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 34)) | DBG_FUNC_START,
5146 (int)uio->uio_offset, upl_offset, xsize, 0, 0);
5147
5148 segflg = uio->uio_segflg;
5149
5150 switch(segflg) {
5151
5152 case UIO_USERSPACE32:
5153 case UIO_USERISPACE32:
5154 uio->uio_segflg = UIO_PHYS_USERSPACE32;
5155 break;
5156
5157 case UIO_USERSPACE:
5158 case UIO_USERISPACE:
5159 uio->uio_segflg = UIO_PHYS_USERSPACE;
5160 break;
5161
5162 case UIO_USERSPACE64:
5163 case UIO_USERISPACE64:
5164 uio->uio_segflg = UIO_PHYS_USERSPACE64;
5165 break;
5166
5167 case UIO_SYSSPACE32:
5168 uio->uio_segflg = UIO_PHYS_SYSSPACE32;
5169 break;
5170
5171 case UIO_SYSSPACE:
5172 uio->uio_segflg = UIO_PHYS_SYSSPACE;
5173 break;
5174
5175 case UIO_SYSSPACE64:
5176 uio->uio_segflg = UIO_PHYS_SYSSPACE64;
5177 break;
5178 }
5179 pl = ubc_upl_pageinfo(upl);
5180
5181 pg_index = upl_offset / PAGE_SIZE;
5182 pg_offset = upl_offset & PAGE_MASK;
5183 csize = min(PAGE_SIZE - pg_offset, xsize);
5184
5185 while (xsize && retval == 0) {
5186 addr64_t paddr;
5187
5188 paddr = ((addr64_t)upl_phys_page(pl, pg_index) << 12) + pg_offset;
5189
5190 retval = uiomove64(paddr, csize, uio);
5191
5192 pg_index += 1;
5193 pg_offset = 0;
5194 xsize -= csize;
5195 csize = min(PAGE_SIZE, xsize);
5196 }
5197 *io_resid = xsize;
5198
5199 uio->uio_segflg = segflg;
5200
5201 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 34)) | DBG_FUNC_END,
5202 (int)uio->uio_offset, xsize, retval, segflg, 0);
5203
5204 return (retval);
5205 }
5206
5207
5208 int
5209 cluster_copy_ubc_data(vnode_t vp, struct uio *uio, int *io_resid, int mark_dirty)
5210 {
5211
5212 return (cluster_copy_ubc_data_internal(vp, uio, io_resid, mark_dirty, 1));
5213 }
5214
5215
5216 static int
5217 cluster_copy_ubc_data_internal(vnode_t vp, struct uio *uio, int *io_resid, int mark_dirty, int take_reference)
5218 {
5219 int segflg;
5220 int io_size;
5221 int xsize;
5222 int start_offset;
5223 int retval = 0;
5224 memory_object_control_t control;
5225
5226 io_size = *io_resid;
5227
5228 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 34)) | DBG_FUNC_START,
5229 (int)uio->uio_offset, 0, io_size, 0, 0);
5230
5231 control = ubc_getobject(vp, UBC_FLAGS_NONE);
5232
5233 if (control == MEMORY_OBJECT_CONTROL_NULL) {
5234 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 34)) | DBG_FUNC_END,
5235 (int)uio->uio_offset, io_size, retval, 3, 0);
5236
5237 return(0);
5238 }
5239 segflg = uio->uio_segflg;
5240
5241 switch(segflg) {
5242
5243 case UIO_USERSPACE32:
5244 case UIO_USERISPACE32:
5245 uio->uio_segflg = UIO_PHYS_USERSPACE32;
5246 break;
5247
5248 case UIO_USERSPACE64:
5249 case UIO_USERISPACE64:
5250 uio->uio_segflg = UIO_PHYS_USERSPACE64;
5251 break;
5252
5253 case UIO_SYSSPACE32:
5254 uio->uio_segflg = UIO_PHYS_SYSSPACE32;
5255 break;
5256
5257 case UIO_SYSSPACE64:
5258 uio->uio_segflg = UIO_PHYS_SYSSPACE64;
5259 break;
5260
5261 case UIO_USERSPACE:
5262 case UIO_USERISPACE:
5263 uio->uio_segflg = UIO_PHYS_USERSPACE;
5264 break;
5265
5266 case UIO_SYSSPACE:
5267 uio->uio_segflg = UIO_PHYS_SYSSPACE;
5268 break;
5269 }
5270
5271 if ( (io_size = *io_resid) ) {
5272 start_offset = (int)(uio->uio_offset & PAGE_MASK_64);
5273 xsize = uio_resid(uio);
5274
5275 retval = memory_object_control_uiomove(control, uio->uio_offset - start_offset, uio,
5276 start_offset, io_size, mark_dirty, take_reference);
5277 xsize -= uio_resid(uio);
5278 io_size -= xsize;
5279 }
5280 uio->uio_segflg = segflg;
5281 *io_resid = io_size;
5282
5283 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 34)) | DBG_FUNC_END,
5284 (int)uio->uio_offset, io_size, retval, 0x80000000 | segflg, 0);
5285
5286 return(retval);
5287 }
5288
5289
5290 int
5291 is_file_clean(vnode_t vp, off_t filesize)
5292 {
5293 off_t f_offset;
5294 int flags;
5295 int total_dirty = 0;
5296
5297 for (f_offset = 0; f_offset < filesize; f_offset += PAGE_SIZE_64) {
5298 if (ubc_page_op(vp, f_offset, 0, NULL, &flags) == KERN_SUCCESS) {
5299 if (flags & UPL_POP_DIRTY) {
5300 total_dirty++;
5301 }
5302 }
5303 }
5304 if (total_dirty)
5305 return(EINVAL);
5306
5307 return (0);
5308 }
5309
5310
5311
5312 /*
5313 * Dirty region tracking/clustering mechanism.
5314 *
5315 * This code (vfs_drt_*) provides a mechanism for tracking and clustering
5316 * dirty regions within a larger space (file). It is primarily intended to
5317 * support clustering in large files with many dirty areas.
5318 *
5319 * The implementation assumes that the dirty regions are pages.
5320 *
5321 * To represent dirty pages within the file, we store bit vectors in a
5322 * variable-size circular hash.
5323 */
5324
5325 /*
5326 * Bitvector size. This determines the number of pages we group in a
5327 * single hashtable entry. Each hashtable entry is aligned to this
5328 * size within the file.
5329 */
5330 #define DRT_BITVECTOR_PAGES 256
5331
5332 /*
5333 * File offset handling.
5334 *
5335 * DRT_ADDRESS_MASK is dependent on DRT_BITVECTOR_PAGES;
5336 * the correct formula is (~(DRT_BITVECTOR_PAGES * PAGE_SIZE) - 1)
5337 */
5338 #define DRT_ADDRESS_MASK (~((1 << 20) - 1))
5339 #define DRT_ALIGN_ADDRESS(addr) ((addr) & DRT_ADDRESS_MASK)
5340
5341 /*
5342 * Hashtable address field handling.
5343 *
5344 * The low-order bits of the hashtable address are used to conserve
5345 * space.
5346 *
5347 * DRT_HASH_COUNT_MASK must be large enough to store the range
5348 * 0-DRT_BITVECTOR_PAGES inclusive, as well as have one value
5349 * to indicate that the bucket is actually unoccupied.
5350 */
5351 #define DRT_HASH_GET_ADDRESS(scm, i) ((scm)->scm_hashtable[(i)].dhe_control & DRT_ADDRESS_MASK)
5352 #define DRT_HASH_SET_ADDRESS(scm, i, a) \
5353 do { \
5354 (scm)->scm_hashtable[(i)].dhe_control = \
5355 ((scm)->scm_hashtable[(i)].dhe_control & ~DRT_ADDRESS_MASK) | DRT_ALIGN_ADDRESS(a); \
5356 } while (0)
5357 #define DRT_HASH_COUNT_MASK 0x1ff
5358 #define DRT_HASH_GET_COUNT(scm, i) ((scm)->scm_hashtable[(i)].dhe_control & DRT_HASH_COUNT_MASK)
5359 #define DRT_HASH_SET_COUNT(scm, i, c) \
5360 do { \
5361 (scm)->scm_hashtable[(i)].dhe_control = \
5362 ((scm)->scm_hashtable[(i)].dhe_control & ~DRT_HASH_COUNT_MASK) | ((c) & DRT_HASH_COUNT_MASK); \
5363 } while (0)
5364 #define DRT_HASH_CLEAR(scm, i) \
5365 do { \
5366 (scm)->scm_hashtable[(i)].dhe_control = 0; \
5367 } while (0)
5368 #define DRT_HASH_VACATE(scm, i) DRT_HASH_SET_COUNT((scm), (i), DRT_HASH_COUNT_MASK)
5369 #define DRT_HASH_VACANT(scm, i) (DRT_HASH_GET_COUNT((scm), (i)) == DRT_HASH_COUNT_MASK)
5370 #define DRT_HASH_COPY(oscm, oi, scm, i) \
5371 do { \
5372 (scm)->scm_hashtable[(i)].dhe_control = (oscm)->scm_hashtable[(oi)].dhe_control; \
5373 DRT_BITVECTOR_COPY(oscm, oi, scm, i); \
5374 } while(0);
5375
5376
5377 /*
5378 * Hash table moduli.
5379 *
5380 * Since the hashtable entry's size is dependent on the size of
5381 * the bitvector, and since the hashtable size is constrained to
5382 * both being prime and fitting within the desired allocation
5383 * size, these values need to be manually determined.
5384 *
5385 * For DRT_BITVECTOR_SIZE = 256, the entry size is 40 bytes.
5386 *
5387 * The small hashtable allocation is 1024 bytes, so the modulus is 23.
5388 * The large hashtable allocation is 16384 bytes, so the modulus is 401.
5389 */
5390 #define DRT_HASH_SMALL_MODULUS 23
5391 #define DRT_HASH_LARGE_MODULUS 401
5392
5393 #define DRT_SMALL_ALLOCATION 1024 /* 104 bytes spare */
5394 #define DRT_LARGE_ALLOCATION 16384 /* 344 bytes spare */
5395
5396 /* *** nothing below here has secret dependencies on DRT_BITVECTOR_PAGES *** */
5397
5398 /*
5399 * Hashtable bitvector handling.
5400 *
5401 * Bitvector fields are 32 bits long.
5402 */
5403
5404 #define DRT_HASH_SET_BIT(scm, i, bit) \
5405 (scm)->scm_hashtable[(i)].dhe_bitvector[(bit) / 32] |= (1 << ((bit) % 32))
5406
5407 #define DRT_HASH_CLEAR_BIT(scm, i, bit) \
5408 (scm)->scm_hashtable[(i)].dhe_bitvector[(bit) / 32] &= ~(1 << ((bit) % 32))
5409
5410 #define DRT_HASH_TEST_BIT(scm, i, bit) \
5411 ((scm)->scm_hashtable[(i)].dhe_bitvector[(bit) / 32] & (1 << ((bit) % 32)))
5412
5413 #define DRT_BITVECTOR_CLEAR(scm, i) \
5414 bzero(&(scm)->scm_hashtable[(i)].dhe_bitvector[0], (DRT_BITVECTOR_PAGES / 32) * sizeof(u_int32_t))
5415
5416 #define DRT_BITVECTOR_COPY(oscm, oi, scm, i) \
5417 bcopy(&(oscm)->scm_hashtable[(oi)].dhe_bitvector[0], \
5418 &(scm)->scm_hashtable[(i)].dhe_bitvector[0], \
5419 (DRT_BITVECTOR_PAGES / 32) * sizeof(u_int32_t))
5420
5421
5422
5423 /*
5424 * Hashtable entry.
5425 */
5426 struct vfs_drt_hashentry {
5427 u_int64_t dhe_control;
5428 u_int32_t dhe_bitvector[DRT_BITVECTOR_PAGES / 32];
5429 };
5430
5431 /*
5432 * Dirty Region Tracking structure.
5433 *
5434 * The hashtable is allocated entirely inside the DRT structure.
5435 *
5436 * The hash is a simple circular prime modulus arrangement, the structure
5437 * is resized from small to large if it overflows.
5438 */
5439
5440 struct vfs_drt_clustermap {
5441 u_int32_t scm_magic; /* sanity/detection */
5442 #define DRT_SCM_MAGIC 0x12020003
5443 u_int32_t scm_modulus; /* current ring size */
5444 u_int32_t scm_buckets; /* number of occupied buckets */
5445 u_int32_t scm_lastclean; /* last entry we cleaned */
5446 u_int32_t scm_iskips; /* number of slot skips */
5447
5448 struct vfs_drt_hashentry scm_hashtable[0];
5449 };
5450
5451
5452 #define DRT_HASH(scm, addr) ((addr) % (scm)->scm_modulus)
5453 #define DRT_HASH_NEXT(scm, addr) (((addr) + 1) % (scm)->scm_modulus)
5454
5455 /*
5456 * Debugging codes and arguments.
5457 */
5458 #define DRT_DEBUG_EMPTYFREE (FSDBG_CODE(DBG_FSRW, 82)) /* nil */
5459 #define DRT_DEBUG_RETCLUSTER (FSDBG_CODE(DBG_FSRW, 83)) /* offset, length */
5460 #define DRT_DEBUG_ALLOC (FSDBG_CODE(DBG_FSRW, 84)) /* copycount */
5461 #define DRT_DEBUG_INSERT (FSDBG_CODE(DBG_FSRW, 85)) /* offset, iskip */
5462 #define DRT_DEBUG_MARK (FSDBG_CODE(DBG_FSRW, 86)) /* offset, length,
5463 * dirty */
5464 /* 0, setcount */
5465 /* 1 (clean, no map) */
5466 /* 2 (map alloc fail) */
5467 /* 3, resid (partial) */
5468 #define DRT_DEBUG_6 (FSDBG_CODE(DBG_FSRW, 87))
5469 #define DRT_DEBUG_SCMDATA (FSDBG_CODE(DBG_FSRW, 88)) /* modulus, buckets,
5470 * lastclean, iskips */
5471
5472
5473 static kern_return_t vfs_drt_alloc_map(struct vfs_drt_clustermap **cmapp);
5474 static kern_return_t vfs_drt_free_map(struct vfs_drt_clustermap *cmap);
5475 static kern_return_t vfs_drt_search_index(struct vfs_drt_clustermap *cmap,
5476 u_int64_t offset, int *indexp);
5477 static kern_return_t vfs_drt_get_index(struct vfs_drt_clustermap **cmapp,
5478 u_int64_t offset,
5479 int *indexp,
5480 int recursed);
5481 static kern_return_t vfs_drt_do_mark_pages(
5482 void **cmapp,
5483 u_int64_t offset,
5484 u_int length,
5485 u_int *setcountp,
5486 int dirty);
5487 static void vfs_drt_trace(
5488 struct vfs_drt_clustermap *cmap,
5489 int code,
5490 int arg1,
5491 int arg2,
5492 int arg3,
5493 int arg4);
5494
5495
5496 /*
5497 * Allocate and initialise a sparse cluster map.
5498 *
5499 * Will allocate a new map, resize or compact an existing map.
5500 *
5501 * XXX we should probably have at least one intermediate map size,
5502 * as the 1:16 ratio seems a bit drastic.
5503 */
5504 static kern_return_t
5505 vfs_drt_alloc_map(struct vfs_drt_clustermap **cmapp)
5506 {
5507 struct vfs_drt_clustermap *cmap, *ocmap;
5508 kern_return_t kret;
5509 u_int64_t offset;
5510 u_int32_t i;
5511 int nsize, active_buckets, index, copycount;
5512
5513 ocmap = NULL;
5514 if (cmapp != NULL)
5515 ocmap = *cmapp;
5516
5517 /*
5518 * Decide on the size of the new map.
5519 */
5520 if (ocmap == NULL) {
5521 nsize = DRT_HASH_SMALL_MODULUS;
5522 } else {
5523 /* count the number of active buckets in the old map */
5524 active_buckets = 0;
5525 for (i = 0; i < ocmap->scm_modulus; i++) {
5526 if (!DRT_HASH_VACANT(ocmap, i) &&
5527 (DRT_HASH_GET_COUNT(ocmap, i) != 0))
5528 active_buckets++;
5529 }
5530 /*
5531 * If we're currently using the small allocation, check to
5532 * see whether we should grow to the large one.
5533 */
5534 if (ocmap->scm_modulus == DRT_HASH_SMALL_MODULUS) {
5535 /* if the ring is nearly full */
5536 if (active_buckets > (DRT_HASH_SMALL_MODULUS - 5)) {
5537 nsize = DRT_HASH_LARGE_MODULUS;
5538 } else {
5539 nsize = DRT_HASH_SMALL_MODULUS;
5540 }
5541 } else {
5542 /* already using the large modulus */
5543 nsize = DRT_HASH_LARGE_MODULUS;
5544 /*
5545 * If the ring is completely full, there's
5546 * nothing useful for us to do. Behave as
5547 * though we had compacted into the new
5548 * array and return.
5549 */
5550 if (active_buckets >= DRT_HASH_LARGE_MODULUS)
5551 return(KERN_SUCCESS);
5552 }
5553 }
5554
5555 /*
5556 * Allocate and initialise the new map.
5557 */
5558
5559 kret = kmem_alloc(kernel_map, (vm_offset_t *)&cmap,
5560 (nsize == DRT_HASH_SMALL_MODULUS) ? DRT_SMALL_ALLOCATION : DRT_LARGE_ALLOCATION);
5561 if (kret != KERN_SUCCESS)
5562 return(kret);
5563 cmap->scm_magic = DRT_SCM_MAGIC;
5564 cmap->scm_modulus = nsize;
5565 cmap->scm_buckets = 0;
5566 cmap->scm_lastclean = 0;
5567 cmap->scm_iskips = 0;
5568 for (i = 0; i < cmap->scm_modulus; i++) {
5569 DRT_HASH_CLEAR(cmap, i);
5570 DRT_HASH_VACATE(cmap, i);
5571 DRT_BITVECTOR_CLEAR(cmap, i);
5572 }
5573
5574 /*
5575 * If there's an old map, re-hash entries from it into the new map.
5576 */
5577 copycount = 0;
5578 if (ocmap != NULL) {
5579 for (i = 0; i < ocmap->scm_modulus; i++) {
5580 /* skip empty buckets */
5581 if (DRT_HASH_VACANT(ocmap, i) ||
5582 (DRT_HASH_GET_COUNT(ocmap, i) == 0))
5583 continue;
5584 /* get new index */
5585 offset = DRT_HASH_GET_ADDRESS(ocmap, i);
5586 kret = vfs_drt_get_index(&cmap, offset, &index, 1);
5587 if (kret != KERN_SUCCESS) {
5588 /* XXX need to bail out gracefully here */
5589 panic("vfs_drt: new cluster map mysteriously too small");
5590 index = 0;
5591 }
5592 /* copy */
5593 DRT_HASH_COPY(ocmap, i, cmap, index);
5594 copycount++;
5595 }
5596 }
5597
5598 /* log what we've done */
5599 vfs_drt_trace(cmap, DRT_DEBUG_ALLOC, copycount, 0, 0, 0);
5600
5601 /*
5602 * It's important to ensure that *cmapp always points to
5603 * a valid map, so we must overwrite it before freeing
5604 * the old map.
5605 */
5606 *cmapp = cmap;
5607 if (ocmap != NULL) {
5608 /* emit stats into trace buffer */
5609 vfs_drt_trace(ocmap, DRT_DEBUG_SCMDATA,
5610 ocmap->scm_modulus,
5611 ocmap->scm_buckets,
5612 ocmap->scm_lastclean,
5613 ocmap->scm_iskips);
5614
5615 vfs_drt_free_map(ocmap);
5616 }
5617 return(KERN_SUCCESS);
5618 }
5619
5620
5621 /*
5622 * Free a sparse cluster map.
5623 */
5624 static kern_return_t
5625 vfs_drt_free_map(struct vfs_drt_clustermap *cmap)
5626 {
5627 kmem_free(kernel_map, (vm_offset_t)cmap,
5628 (cmap->scm_modulus == DRT_HASH_SMALL_MODULUS) ? DRT_SMALL_ALLOCATION : DRT_LARGE_ALLOCATION);
5629 return(KERN_SUCCESS);
5630 }
5631
5632
5633 /*
5634 * Find the hashtable slot currently occupied by an entry for the supplied offset.
5635 */
5636 static kern_return_t
5637 vfs_drt_search_index(struct vfs_drt_clustermap *cmap, u_int64_t offset, int *indexp)
5638 {
5639 int index;
5640 u_int32_t i;
5641
5642 offset = DRT_ALIGN_ADDRESS(offset);
5643 index = DRT_HASH(cmap, offset);
5644
5645 /* traverse the hashtable */
5646 for (i = 0; i < cmap->scm_modulus; i++) {
5647
5648 /*
5649 * If the slot is vacant, we can stop.
5650 */
5651 if (DRT_HASH_VACANT(cmap, index))
5652 break;
5653
5654 /*
5655 * If the address matches our offset, we have success.
5656 */
5657 if (DRT_HASH_GET_ADDRESS(cmap, index) == offset) {
5658 *indexp = index;
5659 return(KERN_SUCCESS);
5660 }
5661
5662 /*
5663 * Move to the next slot, try again.
5664 */
5665 index = DRT_HASH_NEXT(cmap, index);
5666 }
5667 /*
5668 * It's not there.
5669 */
5670 return(KERN_FAILURE);
5671 }
5672
5673 /*
5674 * Find the hashtable slot for the supplied offset. If we haven't allocated
5675 * one yet, allocate one and populate the address field. Note that it will
5676 * not have a nonzero page count and thus will still technically be free, so
5677 * in the case where we are called to clean pages, the slot will remain free.
5678 */
5679 static kern_return_t
5680 vfs_drt_get_index(struct vfs_drt_clustermap **cmapp, u_int64_t offset, int *indexp, int recursed)
5681 {
5682 struct vfs_drt_clustermap *cmap;
5683 kern_return_t kret;
5684 u_int32_t index;
5685 u_int32_t i;
5686
5687 cmap = *cmapp;
5688
5689 /* look for an existing entry */
5690 kret = vfs_drt_search_index(cmap, offset, indexp);
5691 if (kret == KERN_SUCCESS)
5692 return(kret);
5693
5694 /* need to allocate an entry */
5695 offset = DRT_ALIGN_ADDRESS(offset);
5696 index = DRT_HASH(cmap, offset);
5697
5698 /* scan from the index forwards looking for a vacant slot */
5699 for (i = 0; i < cmap->scm_modulus; i++) {
5700 /* slot vacant? */
5701 if (DRT_HASH_VACANT(cmap, index) || DRT_HASH_GET_COUNT(cmap,index) == 0) {
5702 cmap->scm_buckets++;
5703 if (index < cmap->scm_lastclean)
5704 cmap->scm_lastclean = index;
5705 DRT_HASH_SET_ADDRESS(cmap, index, offset);
5706 DRT_HASH_SET_COUNT(cmap, index, 0);
5707 DRT_BITVECTOR_CLEAR(cmap, index);
5708 *indexp = index;
5709 vfs_drt_trace(cmap, DRT_DEBUG_INSERT, (int)offset, i, 0, 0);
5710 return(KERN_SUCCESS);
5711 }
5712 cmap->scm_iskips += i;
5713 index = DRT_HASH_NEXT(cmap, index);
5714 }
5715
5716 /*
5717 * We haven't found a vacant slot, so the map is full. If we're not
5718 * already recursed, try reallocating/compacting it.
5719 */
5720 if (recursed)
5721 return(KERN_FAILURE);
5722 kret = vfs_drt_alloc_map(cmapp);
5723 if (kret == KERN_SUCCESS) {
5724 /* now try to insert again */
5725 kret = vfs_drt_get_index(cmapp, offset, indexp, 1);
5726 }
5727 return(kret);
5728 }
5729
5730 /*
5731 * Implementation of set dirty/clean.
5732 *
5733 * In the 'clean' case, not finding a map is OK.
5734 */
5735 static kern_return_t
5736 vfs_drt_do_mark_pages(
5737 void **private,
5738 u_int64_t offset,
5739 u_int length,
5740 u_int *setcountp,
5741 int dirty)
5742 {
5743 struct vfs_drt_clustermap *cmap, **cmapp;
5744 kern_return_t kret;
5745 int i, index, pgoff, pgcount, setcount, ecount;
5746
5747 cmapp = (struct vfs_drt_clustermap **)private;
5748 cmap = *cmapp;
5749
5750 vfs_drt_trace(cmap, DRT_DEBUG_MARK | DBG_FUNC_START, (int)offset, (int)length, dirty, 0);
5751
5752 if (setcountp != NULL)
5753 *setcountp = 0;
5754
5755 /* allocate a cluster map if we don't already have one */
5756 if (cmap == NULL) {
5757 /* no cluster map, nothing to clean */
5758 if (!dirty) {
5759 vfs_drt_trace(cmap, DRT_DEBUG_MARK | DBG_FUNC_END, 1, 0, 0, 0);
5760 return(KERN_SUCCESS);
5761 }
5762 kret = vfs_drt_alloc_map(cmapp);
5763 if (kret != KERN_SUCCESS) {
5764 vfs_drt_trace(cmap, DRT_DEBUG_MARK | DBG_FUNC_END, 2, 0, 0, 0);
5765 return(kret);
5766 }
5767 }
5768 setcount = 0;
5769
5770 /*
5771 * Iterate over the length of the region.
5772 */
5773 while (length > 0) {
5774 /*
5775 * Get the hashtable index for this offset.
5776 *
5777 * XXX this will add blank entries if we are clearing a range
5778 * that hasn't been dirtied.
5779 */
5780 kret = vfs_drt_get_index(cmapp, offset, &index, 0);
5781 cmap = *cmapp; /* may have changed! */
5782 /* this may be a partial-success return */
5783 if (kret != KERN_SUCCESS) {
5784 if (setcountp != NULL)
5785 *setcountp = setcount;
5786 vfs_drt_trace(cmap, DRT_DEBUG_MARK | DBG_FUNC_END, 3, (int)length, 0, 0);
5787
5788 return(kret);
5789 }
5790
5791 /*
5792 * Work out how many pages we're modifying in this
5793 * hashtable entry.
5794 */
5795 pgoff = (offset - DRT_ALIGN_ADDRESS(offset)) / PAGE_SIZE;
5796 pgcount = min((length / PAGE_SIZE), (DRT_BITVECTOR_PAGES - pgoff));
5797
5798 /*
5799 * Iterate over pages, dirty/clearing as we go.
5800 */
5801 ecount = DRT_HASH_GET_COUNT(cmap, index);
5802 for (i = 0; i < pgcount; i++) {
5803 if (dirty) {
5804 if (!DRT_HASH_TEST_BIT(cmap, index, pgoff + i)) {
5805 DRT_HASH_SET_BIT(cmap, index, pgoff + i);
5806 ecount++;
5807 setcount++;
5808 }
5809 } else {
5810 if (DRT_HASH_TEST_BIT(cmap, index, pgoff + i)) {
5811 DRT_HASH_CLEAR_BIT(cmap, index, pgoff + i);
5812 ecount--;
5813 setcount++;
5814 }
5815 }
5816 }
5817 DRT_HASH_SET_COUNT(cmap, index, ecount);
5818
5819 offset += pgcount * PAGE_SIZE;
5820 length -= pgcount * PAGE_SIZE;
5821 }
5822 if (setcountp != NULL)
5823 *setcountp = setcount;
5824
5825 vfs_drt_trace(cmap, DRT_DEBUG_MARK | DBG_FUNC_END, 0, setcount, 0, 0);
5826
5827 return(KERN_SUCCESS);
5828 }
5829
5830 /*
5831 * Mark a set of pages as dirty/clean.
5832 *
5833 * This is a public interface.
5834 *
5835 * cmapp
5836 * Pointer to storage suitable for holding a pointer. Note that
5837 * this must either be NULL or a value set by this function.
5838 *
5839 * size
5840 * Current file size in bytes.
5841 *
5842 * offset
5843 * Offset of the first page to be marked as dirty, in bytes. Must be
5844 * page-aligned.
5845 *
5846 * length
5847 * Length of dirty region, in bytes. Must be a multiple of PAGE_SIZE.
5848 *
5849 * setcountp
5850 * Number of pages newly marked dirty by this call (optional).
5851 *
5852 * Returns KERN_SUCCESS if all the pages were successfully marked.
5853 */
5854 static kern_return_t
5855 vfs_drt_mark_pages(void **cmapp, off_t offset, u_int length, u_int *setcountp)
5856 {
5857 /* XXX size unused, drop from interface */
5858 return(vfs_drt_do_mark_pages(cmapp, offset, length, setcountp, 1));
5859 }
5860
5861 #if 0
5862 static kern_return_t
5863 vfs_drt_unmark_pages(void **cmapp, off_t offset, u_int length)
5864 {
5865 return(vfs_drt_do_mark_pages(cmapp, offset, length, NULL, 0));
5866 }
5867 #endif
5868
5869 /*
5870 * Get a cluster of dirty pages.
5871 *
5872 * This is a public interface.
5873 *
5874 * cmapp
5875 * Pointer to storage managed by drt_mark_pages. Note that this must
5876 * be NULL or a value set by drt_mark_pages.
5877 *
5878 * offsetp
5879 * Returns the byte offset into the file of the first page in the cluster.
5880 *
5881 * lengthp
5882 * Returns the length in bytes of the cluster of dirty pages.
5883 *
5884 * Returns success if a cluster was found. If KERN_FAILURE is returned, there
5885 * are no dirty pages meeting the minmum size criteria. Private storage will
5886 * be released if there are no more dirty pages left in the map
5887 *
5888 */
5889 static kern_return_t
5890 vfs_drt_get_cluster(void **cmapp, off_t *offsetp, u_int *lengthp)
5891 {
5892 struct vfs_drt_clustermap *cmap;
5893 u_int64_t offset;
5894 u_int length;
5895 u_int32_t j;
5896 int index, i, fs, ls;
5897
5898 /* sanity */
5899 if ((cmapp == NULL) || (*cmapp == NULL))
5900 return(KERN_FAILURE);
5901 cmap = *cmapp;
5902
5903 /* walk the hashtable */
5904 for (offset = 0, j = 0; j < cmap->scm_modulus; offset += (DRT_BITVECTOR_PAGES * PAGE_SIZE), j++) {
5905 index = DRT_HASH(cmap, offset);
5906
5907 if (DRT_HASH_VACANT(cmap, index) || (DRT_HASH_GET_COUNT(cmap, index) == 0))
5908 continue;
5909
5910 /* scan the bitfield for a string of bits */
5911 fs = -1;
5912
5913 for (i = 0; i < DRT_BITVECTOR_PAGES; i++) {
5914 if (DRT_HASH_TEST_BIT(cmap, index, i)) {
5915 fs = i;
5916 break;
5917 }
5918 }
5919 if (fs == -1) {
5920 /* didn't find any bits set */
5921 panic("vfs_drt: entry summary count > 0 but no bits set in map");
5922 }
5923 for (ls = 0; i < DRT_BITVECTOR_PAGES; i++, ls++) {
5924 if (!DRT_HASH_TEST_BIT(cmap, index, i))
5925 break;
5926 }
5927
5928 /* compute offset and length, mark pages clean */
5929 offset = DRT_HASH_GET_ADDRESS(cmap, index) + (PAGE_SIZE * fs);
5930 length = ls * PAGE_SIZE;
5931 vfs_drt_do_mark_pages(cmapp, offset, length, NULL, 0);
5932 cmap->scm_lastclean = index;
5933
5934 /* return successful */
5935 *offsetp = (off_t)offset;
5936 *lengthp = length;
5937
5938 vfs_drt_trace(cmap, DRT_DEBUG_RETCLUSTER, (int)offset, (int)length, 0, 0);
5939 return(KERN_SUCCESS);
5940 }
5941 /*
5942 * We didn't find anything... hashtable is empty
5943 * emit stats into trace buffer and
5944 * then free it
5945 */
5946 vfs_drt_trace(cmap, DRT_DEBUG_SCMDATA,
5947 cmap->scm_modulus,
5948 cmap->scm_buckets,
5949 cmap->scm_lastclean,
5950 cmap->scm_iskips);
5951
5952 vfs_drt_free_map(cmap);
5953 *cmapp = NULL;
5954
5955 return(KERN_FAILURE);
5956 }
5957
5958
5959 static kern_return_t
5960 vfs_drt_control(void **cmapp, int op_type)
5961 {
5962 struct vfs_drt_clustermap *cmap;
5963
5964 /* sanity */
5965 if ((cmapp == NULL) || (*cmapp == NULL))
5966 return(KERN_FAILURE);
5967 cmap = *cmapp;
5968
5969 switch (op_type) {
5970 case 0:
5971 /* emit stats into trace buffer */
5972 vfs_drt_trace(cmap, DRT_DEBUG_SCMDATA,
5973 cmap->scm_modulus,
5974 cmap->scm_buckets,
5975 cmap->scm_lastclean,
5976 cmap->scm_iskips);
5977
5978 vfs_drt_free_map(cmap);
5979 *cmapp = NULL;
5980 break;
5981
5982 case 1:
5983 cmap->scm_lastclean = 0;
5984 break;
5985 }
5986 return(KERN_SUCCESS);
5987 }
5988
5989
5990
5991 /*
5992 * Emit a summary of the state of the clustermap into the trace buffer
5993 * along with some caller-provided data.
5994 */
5995 #if KDEBUG
5996 static void
5997 vfs_drt_trace(__unused struct vfs_drt_clustermap *cmap, int code, int arg1, int arg2, int arg3, int arg4)
5998 {
5999 KERNEL_DEBUG(code, arg1, arg2, arg3, arg4, 0);
6000 }
6001 #else
6002 static void
6003 vfs_drt_trace(__unused struct vfs_drt_clustermap *cmap, __unused int code,
6004 __unused int arg1, __unused int arg2, __unused int arg3,
6005 __unused int arg4)
6006 {
6007 }
6008 #endif
6009
6010 #if 0
6011 /*
6012 * Perform basic sanity check on the hash entry summary count
6013 * vs. the actual bits set in the entry.
6014 */
6015 static void
6016 vfs_drt_sanity(struct vfs_drt_clustermap *cmap)
6017 {
6018 int index, i;
6019 int bits_on;
6020
6021 for (index = 0; index < cmap->scm_modulus; index++) {
6022 if (DRT_HASH_VACANT(cmap, index))
6023 continue;
6024
6025 for (bits_on = 0, i = 0; i < DRT_BITVECTOR_PAGES; i++) {
6026 if (DRT_HASH_TEST_BIT(cmap, index, i))
6027 bits_on++;
6028 }
6029 if (bits_on != DRT_HASH_GET_COUNT(cmap, index))
6030 panic("bits_on = %d, index = %d\n", bits_on, index);
6031 }
6032 }
6033 #endif
mirror of XNU