]> git.saurik.com Git - apple/xnu.git/blob - bsd/vfs/vfs_cluster.c
e2b4b14d0137b8927c7bc0150ea9a7da43659714
[apple/xnu.git] / bsd / vfs / vfs_cluster.c
1 /*
2 * Copyright (c) 2000-2004 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * The contents of this file constitute Original Code as defined in and
7 * are subject to the Apple Public Source License Version 1.1 (the
8 * "License"). You may not use this file except in compliance with the
9 * License. Please obtain a copy of the License at
10 * http://www.apple.com/publicsource and read it before using this file.
11 *
12 * This Original Code and all software distributed under the License are
13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17 * License for the specific language governing rights and limitations
18 * under the License.
19 *
20 * @APPLE_LICENSE_HEADER_END@
21 */
22 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
23 /*
24 * Copyright (c) 1993
25 * The Regents of the University of California. All rights reserved.
26 *
27 * Redistribution and use in source and binary forms, with or without
28 * modification, are permitted provided that the following conditions
29 * are met:
30 * 1. Redistributions of source code must retain the above copyright
31 * notice, this list of conditions and the following disclaimer.
32 * 2. Redistributions in binary form must reproduce the above copyright
33 * notice, this list of conditions and the following disclaimer in the
34 * documentation and/or other materials provided with the distribution.
35 * 3. All advertising materials mentioning features or use of this software
36 * must display the following acknowledgement:
37 * This product includes software developed by the University of
38 * California, Berkeley and its contributors.
39 * 4. Neither the name of the University nor the names of its contributors
40 * may be used to endorse or promote products derived from this software
41 * without specific prior written permission.
42 *
43 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
44 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
45 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
46 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
47 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
48 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
49 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
50 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
51 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
52 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
53 * SUCH DAMAGE.
54 *
55 * @(#)vfs_cluster.c 8.10 (Berkeley) 3/28/95
56 */
57
58 #include <sys/param.h>
59 #include <sys/proc_internal.h>
60 #include <sys/buf_internal.h>
61 #include <sys/mount_internal.h>
62 #include <sys/vnode_internal.h>
63 #include <sys/trace.h>
64 #include <sys/malloc.h>
65 #include <sys/time.h>
66 #include <sys/kernel.h>
67 #include <sys/resourcevar.h>
68 #include <sys/uio_internal.h>
69 #include <libkern/libkern.h>
70 #include <machine/machine_routines.h>
71
72 #include <sys/ubc_internal.h>
73
74 #include <mach/mach_types.h>
75 #include <mach/memory_object_types.h>
76 #include <mach/vm_map.h>
77 #include <mach/upl.h>
78
79 #include <vm/vm_kern.h>
80 #include <vm/vm_map.h>
81 #include <vm/vm_pageout.h>
82
83 #include <sys/kdebug.h>
84
85
86 #define CL_READ 0x01
87 #define CL_ASYNC 0x02
88 #define CL_COMMIT 0x04
89 #define CL_PAGEOUT 0x10
90 #define CL_AGE 0x20
91 #define CL_DUMP 0x40
92 #define CL_NOZERO 0x80
93 #define CL_PAGEIN 0x100
94 #define CL_DEV_MEMORY 0x200
95 #define CL_PRESERVE 0x400
96 #define CL_THROTTLE 0x800
97 #define CL_KEEPCACHED 0x1000
98
99
100 struct clios {
101 u_int io_completed; /* amount of io that has currently completed */
102 u_int io_issued; /* amount of io that was successfully issued */
103 int io_error; /* error code of first error encountered */
104 int io_wanted; /* someone is sleeping waiting for a change in state */
105 };
106
107 static lck_grp_t *cl_mtx_grp;
108 static lck_attr_t *cl_mtx_attr;
109 static lck_grp_attr_t *cl_mtx_grp_attr;
110 static lck_mtx_t *cl_mtxp;
111
112
113 static int cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int non_rounded_size,
114 int flags, buf_t real_bp, struct clios *iostate);
115 static int cluster_iodone(buf_t bp, void *dummy);
116 static int cluster_rd_prefetch(vnode_t vp, off_t f_offset, u_int size, off_t filesize);
117 static int cluster_hard_throttle_on(vnode_t vp);
118
119 static int cluster_read_x(vnode_t vp, struct uio *uio, off_t filesize, int flags);
120 static int cluster_write_x(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF,
121 off_t headOff, off_t tailOff, int flags);
122 static int cluster_nocopy_read(vnode_t vp, struct uio *uio, off_t filesize);
123 static int cluster_nocopy_write(vnode_t vp, struct uio *uio, off_t newEOF);
124 static int cluster_phys_read(vnode_t vp, struct uio *uio, off_t filesize);
125 static int cluster_phys_write(vnode_t vp, struct uio *uio, off_t newEOF);
126 static int cluster_align_phys_io(vnode_t vp, struct uio *uio, addr64_t usr_paddr, int xsize, int flags);
127
128 static void cluster_rd_ahead(vnode_t vp, struct cl_extent *extent, off_t filesize, struct cl_readahead *ra);
129
130 static int cluster_push_x(vnode_t vp, struct cl_extent *, off_t EOF, int flags);
131 static void cluster_push_EOF(vnode_t vp, off_t EOF);
132
133 static int cluster_try_push(struct cl_writebehind *, vnode_t vp, off_t EOF, int can_delay, int push_all);
134
135 static void sparse_cluster_switch(struct cl_writebehind *, vnode_t vp, off_t EOF);
136 static void sparse_cluster_push(struct cl_writebehind *, vnode_t vp, off_t EOF, int push_all);
137 static void sparse_cluster_add(struct cl_writebehind *, vnode_t vp, struct cl_extent *, off_t EOF);
138
139 static kern_return_t vfs_drt_mark_pages(void **cmapp, off_t offset, u_int length, int *setcountp);
140 static kern_return_t vfs_drt_get_cluster(void **cmapp, off_t *offsetp, u_int *lengthp);
141 static kern_return_t vfs_drt_control(void **cmapp, int op_type);
142
143 int is_file_clean(vnode_t, off_t);
144
145 /*
146 * throttle the number of async writes that
147 * can be outstanding on a single vnode
148 * before we issue a synchronous write
149 */
150 #define HARD_THROTTLE_MAXCNT 0
151 #define HARD_THROTTLE_MAXSIZE (64 * 1024)
152
153 int hard_throttle_on_root = 0;
154 struct timeval priority_IO_timestamp_for_root;
155
156
157 void
158 cluster_init(void) {
159 /*
160 * allocate lock group attribute and group
161 */
162 cl_mtx_grp_attr = lck_grp_attr_alloc_init();
163 cl_mtx_grp = lck_grp_alloc_init("cluster I/O", cl_mtx_grp_attr);
164
165 /*
166 * allocate the lock attribute
167 */
168 cl_mtx_attr = lck_attr_alloc_init();
169
170 /*
171 * allocate and initialize mutex's used to protect updates and waits
172 * on the cluster_io context
173 */
174 cl_mtxp = lck_mtx_alloc_init(cl_mtx_grp, cl_mtx_attr);
175
176 if (cl_mtxp == NULL)
177 panic("cluster_init: failed to allocate cl_mtxp");
178 }
179
180
181
182 #define CLW_ALLOCATE 0x01
183 #define CLW_RETURNLOCKED 0x02
184 /*
185 * if the read ahead context doesn't yet exist,
186 * allocate and initialize it...
187 * the vnode lock serializes multiple callers
188 * during the actual assignment... first one
189 * to grab the lock wins... the other callers
190 * will release the now unnecessary storage
191 *
192 * once the context is present, try to grab (but don't block on)
193 * the lock associated with it... if someone
194 * else currently owns it, than the read
195 * will run without read-ahead. this allows
196 * multiple readers to run in parallel and
197 * since there's only 1 read ahead context,
198 * there's no real loss in only allowing 1
199 * reader to have read-ahead enabled.
200 */
201 static struct cl_readahead *
202 cluster_get_rap(vnode_t vp)
203 {
204 struct ubc_info *ubc;
205 struct cl_readahead *rap;
206
207 ubc = vp->v_ubcinfo;
208
209 if ((rap = ubc->cl_rahead) == NULL) {
210 MALLOC_ZONE(rap, struct cl_readahead *, sizeof *rap, M_CLRDAHEAD, M_WAITOK);
211
212 bzero(rap, sizeof *rap);
213 rap->cl_lastr = -1;
214 lck_mtx_init(&rap->cl_lockr, cl_mtx_grp, cl_mtx_attr);
215
216 vnode_lock(vp);
217
218 if (ubc->cl_rahead == NULL)
219 ubc->cl_rahead = rap;
220 else {
221 lck_mtx_destroy(&rap->cl_lockr, cl_mtx_grp);
222 FREE_ZONE((void *)rap, sizeof *rap, M_CLRDAHEAD);
223 rap = ubc->cl_rahead;
224 }
225 vnode_unlock(vp);
226 }
227 if (lck_mtx_try_lock(&rap->cl_lockr) == TRUE)
228 return(rap);
229
230 return ((struct cl_readahead *)NULL);
231 }
232
233
234 /*
235 * if the write behind context doesn't yet exist,
236 * and CLW_ALLOCATE is specified, allocate and initialize it...
237 * the vnode lock serializes multiple callers
238 * during the actual assignment... first one
239 * to grab the lock wins... the other callers
240 * will release the now unnecessary storage
241 *
242 * if CLW_RETURNLOCKED is set, grab (blocking if necessary)
243 * the lock associated with the write behind context before
244 * returning
245 */
246
247 static struct cl_writebehind *
248 cluster_get_wbp(vnode_t vp, int flags)
249 {
250 struct ubc_info *ubc;
251 struct cl_writebehind *wbp;
252
253 ubc = vp->v_ubcinfo;
254
255 if ((wbp = ubc->cl_wbehind) == NULL) {
256
257 if ( !(flags & CLW_ALLOCATE))
258 return ((struct cl_writebehind *)NULL);
259
260 MALLOC_ZONE(wbp, struct cl_writebehind *, sizeof *wbp, M_CLWRBEHIND, M_WAITOK);
261
262 bzero(wbp, sizeof *wbp);
263 lck_mtx_init(&wbp->cl_lockw, cl_mtx_grp, cl_mtx_attr);
264
265 vnode_lock(vp);
266
267 if (ubc->cl_wbehind == NULL)
268 ubc->cl_wbehind = wbp;
269 else {
270 lck_mtx_destroy(&wbp->cl_lockw, cl_mtx_grp);
271 FREE_ZONE((void *)wbp, sizeof *wbp, M_CLWRBEHIND);
272 wbp = ubc->cl_wbehind;
273 }
274 vnode_unlock(vp);
275 }
276 if (flags & CLW_RETURNLOCKED)
277 lck_mtx_lock(&wbp->cl_lockw);
278
279 return (wbp);
280 }
281
282
283 static int
284 cluster_hard_throttle_on(vnode_t vp)
285 {
286 static struct timeval hard_throttle_maxelapsed = { 0, 200000 };
287
288 if (vp->v_mount->mnt_kern_flag & MNTK_ROOTDEV) {
289 struct timeval elapsed;
290
291 if (hard_throttle_on_root)
292 return(1);
293
294 microuptime(&elapsed);
295 timevalsub(&elapsed, &priority_IO_timestamp_for_root);
296
297 if (timevalcmp(&elapsed, &hard_throttle_maxelapsed, <))
298 return(1);
299 }
300 return(0);
301 }
302
303
304 static int
305 cluster_iodone(buf_t bp, __unused void *dummy)
306 {
307 int b_flags;
308 int error;
309 int total_size;
310 int total_resid;
311 int upl_offset;
312 int zero_offset;
313 upl_t upl;
314 buf_t cbp;
315 buf_t cbp_head;
316 buf_t cbp_next;
317 buf_t real_bp;
318 struct clios *iostate;
319 int commit_size;
320 int pg_offset;
321
322 cbp_head = (buf_t)(bp->b_trans_head);
323
324 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 20)) | DBG_FUNC_START,
325 (int)cbp_head, bp->b_lblkno, bp->b_bcount, bp->b_flags, 0);
326
327 for (cbp = cbp_head; cbp; cbp = cbp->b_trans_next) {
328 /*
329 * all I/O requests that are part of this transaction
330 * have to complete before we can process it
331 */
332 if ( !(cbp->b_flags & B_DONE)) {
333
334 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 20)) | DBG_FUNC_END,
335 (int)cbp_head, (int)cbp, cbp->b_bcount, cbp->b_flags, 0);
336
337 return 0;
338 }
339 }
340 error = 0;
341 total_size = 0;
342 total_resid = 0;
343
344 cbp = cbp_head;
345 upl_offset = cbp->b_uploffset;
346 upl = cbp->b_upl;
347 b_flags = cbp->b_flags;
348 real_bp = cbp->b_real_bp;
349 zero_offset= cbp->b_validend;
350 iostate = (struct clios *)cbp->b_iostate;
351
352 if (real_bp)
353 real_bp->b_dev = cbp->b_dev;
354
355 while (cbp) {
356 if ((cbp->b_flags & B_ERROR) && error == 0)
357 error = cbp->b_error;
358
359 total_resid += cbp->b_resid;
360 total_size += cbp->b_bcount;
361
362 cbp_next = cbp->b_trans_next;
363
364 free_io_buf(cbp);
365
366 cbp = cbp_next;
367 }
368 if (zero_offset)
369 cluster_zero(upl, zero_offset, PAGE_SIZE - (zero_offset & PAGE_MASK), real_bp);
370
371 if (iostate) {
372 int need_wakeup = 0;
373
374 /*
375 * someone has issued multiple I/Os asynchrounsly
376 * and is waiting for them to complete (streaming)
377 */
378 lck_mtx_lock(cl_mtxp);
379
380 if (error && iostate->io_error == 0)
381 iostate->io_error = error;
382
383 iostate->io_completed += total_size;
384
385 if (iostate->io_wanted) {
386 /*
387 * someone is waiting for the state of
388 * this io stream to change
389 */
390 iostate->io_wanted = 0;
391 need_wakeup = 1;
392 }
393 lck_mtx_unlock(cl_mtxp);
394
395 if (need_wakeup)
396 wakeup((caddr_t)&iostate->io_wanted);
397 }
398 if ((b_flags & B_NEED_IODONE) && real_bp) {
399 if (error) {
400 real_bp->b_flags |= B_ERROR;
401 real_bp->b_error = error;
402 }
403 real_bp->b_resid = total_resid;
404
405 buf_biodone(real_bp);
406 }
407 if (error == 0 && total_resid)
408 error = EIO;
409
410 if (b_flags & B_COMMIT_UPL) {
411 pg_offset = upl_offset & PAGE_MASK;
412 commit_size = (pg_offset + total_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
413
414 if (error || (b_flags & B_NOCACHE)) {
415 int upl_abort_code;
416 int page_in = 0;
417 int page_out = 0;
418
419 if (b_flags & B_PAGEIO) {
420 if (b_flags & B_READ)
421 page_in = 1;
422 else
423 page_out = 1;
424 }
425 if (b_flags & B_CACHE) /* leave pages in the cache unchanged on error */
426 upl_abort_code = UPL_ABORT_FREE_ON_EMPTY;
427 else if (page_out && (error != ENXIO)) /* transient error */
428 upl_abort_code = UPL_ABORT_FREE_ON_EMPTY;
429 else if (page_in)
430 upl_abort_code = UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR;
431 else
432 upl_abort_code = UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_DUMP_PAGES;
433
434 ubc_upl_abort_range(upl, upl_offset - pg_offset, commit_size,
435 upl_abort_code);
436
437 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 20)) | DBG_FUNC_END,
438 (int)upl, upl_offset - pg_offset, commit_size,
439 0x80000000|upl_abort_code, 0);
440
441 } else {
442 int upl_commit_flags = UPL_COMMIT_FREE_ON_EMPTY;
443
444 if ((b_flags & B_PHYS) && (b_flags & B_READ))
445 upl_commit_flags |= UPL_COMMIT_SET_DIRTY;
446
447 if (b_flags & B_AGE)
448 upl_commit_flags |= UPL_COMMIT_INACTIVATE;
449
450 ubc_upl_commit_range(upl, upl_offset - pg_offset, commit_size,
451 upl_commit_flags);
452
453 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 20)) | DBG_FUNC_END,
454 (int)upl, upl_offset - pg_offset, commit_size,
455 upl_commit_flags, 0);
456 }
457 } else {
458 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 20)) | DBG_FUNC_END,
459 (int)upl, upl_offset, 0, error, 0);
460 }
461
462 return (error);
463 }
464
465
466 void
467 cluster_zero(upl_t upl, vm_offset_t upl_offset, int size, buf_t bp)
468 {
469 upl_page_info_t *pl;
470
471 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 23)) | DBG_FUNC_START,
472 upl_offset, size, (int)bp, 0, 0);
473
474 if (bp == NULL || bp->b_datap == 0) {
475
476 pl = ubc_upl_pageinfo(upl);
477
478 while (size) {
479 int page_offset;
480 int page_index;
481 addr64_t zero_addr;
482 int zero_cnt;
483
484 page_index = upl_offset / PAGE_SIZE;
485 page_offset = upl_offset & PAGE_MASK;
486
487 zero_addr = ((addr64_t)upl_phys_page(pl, page_index) << 12) + page_offset;
488 zero_cnt = min(PAGE_SIZE - page_offset, size);
489
490 bzero_phys(zero_addr, zero_cnt);
491
492 size -= zero_cnt;
493 upl_offset += zero_cnt;
494 }
495 } else
496 bzero((caddr_t)((vm_offset_t)bp->b_datap + upl_offset), size);
497
498 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 23)) | DBG_FUNC_END,
499 upl_offset, size, 0, 0, 0);
500 }
501
502
503 static int
504 cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int non_rounded_size,
505 int flags, buf_t real_bp, struct clios *iostate)
506 {
507 buf_t cbp;
508 u_int size;
509 u_int io_size;
510 int io_flags;
511 int bmap_flags;
512 int error = 0;
513 int retval = 0;
514 buf_t cbp_head = NULL;
515 buf_t cbp_tail = NULL;
516 int trans_count = 0;
517 u_int pg_count;
518 int pg_offset;
519 u_int max_iosize;
520 u_int max_vectors;
521 int priv;
522 int zero_offset = 0;
523 int async_throttle = 0;
524 mount_t mp;
525
526 mp = vp->v_mount;
527
528 if (mp->mnt_devblocksize > 1) {
529 /*
530 * round the requested size up so that this I/O ends on a
531 * page boundary in case this is a 'write'... if the filesystem
532 * has blocks allocated to back the page beyond the EOF, we want to
533 * make sure to write out the zero's that are sitting beyond the EOF
534 * so that in case the filesystem doesn't explicitly zero this area
535 * if a hole is created via a lseek/write beyond the current EOF,
536 * it will return zeros when it's read back from the disk. If the
537 * physical allocation doesn't extend for the whole page, we'll
538 * only write/read from the disk up to the end of this allocation
539 * via the extent info returned from the VNOP_BLOCKMAP call.
540 */
541 pg_offset = upl_offset & PAGE_MASK;
542
543 size = (((non_rounded_size + pg_offset) + (PAGE_SIZE - 1)) & ~PAGE_MASK) - pg_offset;
544 } else {
545 /*
546 * anyone advertising a blocksize of 1 byte probably
547 * can't deal with us rounding up the request size
548 * AFP is one such filesystem/device
549 */
550 size = non_rounded_size;
551 }
552 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 22)) | DBG_FUNC_START,
553 (int)f_offset, size, upl_offset, flags, 0);
554
555 if (flags & CL_READ) {
556 io_flags = (B_READ);
557 bmap_flags = VNODE_READ;
558
559 max_iosize = mp->mnt_maxreadcnt;
560 max_vectors = mp->mnt_segreadcnt;
561 } else {
562 io_flags = 0;
563 bmap_flags = VNODE_WRITE;
564
565 max_iosize = mp->mnt_maxwritecnt;
566 max_vectors = mp->mnt_segwritecnt;
567 }
568 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 22)) | DBG_FUNC_NONE, max_iosize, max_vectors, mp->mnt_devblocksize, 0, 0);
569
570 /*
571 * make sure the maximum iosize is a
572 * multiple of the page size
573 */
574 max_iosize &= ~PAGE_MASK;
575
576 if (flags & CL_THROTTLE) {
577 if ( !(flags & CL_PAGEOUT) && cluster_hard_throttle_on(vp)) {
578 if (max_iosize > HARD_THROTTLE_MAXSIZE)
579 max_iosize = HARD_THROTTLE_MAXSIZE;
580 async_throttle = HARD_THROTTLE_MAXCNT;
581 } else
582 async_throttle = VNODE_ASYNC_THROTTLE;
583 }
584 if (flags & CL_AGE)
585 io_flags |= B_AGE;
586 if (flags & CL_DUMP)
587 io_flags |= B_NOCACHE;
588 if (flags & (CL_PAGEIN | CL_PAGEOUT))
589 io_flags |= B_PAGEIO;
590 if (flags & CL_COMMIT)
591 io_flags |= B_COMMIT_UPL;
592 if (flags & CL_PRESERVE)
593 io_flags |= B_PHYS;
594 if (flags & CL_KEEPCACHED)
595 io_flags |= B_CACHE;
596
597 if ((flags & CL_READ) && ((upl_offset + non_rounded_size) & PAGE_MASK) && (!(flags & CL_NOZERO))) {
598 /*
599 * then we are going to end up
600 * with a page that we can't complete (the file size wasn't a multiple
601 * of PAGE_SIZE and we're trying to read to the end of the file
602 * so we'll go ahead and zero out the portion of the page we can't
603 * read in from the file
604 */
605 zero_offset = upl_offset + non_rounded_size;
606 }
607 while (size) {
608 int pg_resid;
609 daddr64_t blkno;
610 daddr64_t lblkno;
611
612 if (size > max_iosize)
613 io_size = max_iosize;
614 else
615 io_size = size;
616
617 if ((error = VNOP_BLOCKMAP(vp, f_offset, io_size, &blkno, (size_t *)&io_size, NULL, bmap_flags, NULL))) {
618 break;
619 }
620 if (real_bp && (real_bp->b_blkno == real_bp->b_lblkno))
621 real_bp->b_blkno = blkno;
622
623 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 24)) | DBG_FUNC_NONE,
624 (int)f_offset, (int)blkno, io_size, zero_offset, 0);
625
626 if (io_size == 0) {
627 /*
628 * vnop_blockmap didn't return an error... however, it did
629 * return an extent size of 0 which means we can't
630 * make forward progress on this I/O... a hole in the
631 * file would be returned as a blkno of -1 with a non-zero io_size
632 * a real extent is returned with a blkno != -1 and a non-zero io_size
633 */
634 error = EINVAL;
635 break;
636 }
637 if ( !(flags & CL_READ) && blkno == -1) {
638 off_t e_offset;
639
640 /*
641 * we're writing into a 'hole'
642 */
643 if (flags & CL_PAGEOUT) {
644 /*
645 * if we got here via cluster_pageout
646 * then just error the request and return
647 * the 'hole' should already have been covered
648 */
649 error = EINVAL;
650 break;
651 }
652 if ( !(flags & CL_COMMIT)) {
653 /*
654 * currently writes always request the commit to happen
655 * as part of the io completion... however, if the CL_COMMIT
656 * flag isn't specified, than we can't issue the abort_range
657 * since the call site is going to abort or commit the same upl..
658 * in this case we can only return an error
659 */
660 error = EINVAL;
661 break;
662 }
663 /*
664 * we can get here if the cluster code happens to
665 * pick up a page that was dirtied via mmap vs
666 * a 'write' and the page targets a 'hole'...
667 * i.e. the writes to the cluster were sparse
668 * and the file was being written for the first time
669 *
670 * we can also get here if the filesystem supports
671 * 'holes' that are less than PAGE_SIZE.... because
672 * we can't know if the range in the page that covers
673 * the 'hole' has been dirtied via an mmap or not,
674 * we have to assume the worst and try to push the
675 * entire page to storage.
676 *
677 * Try paging out the page individually before
678 * giving up entirely and dumping it (the pageout
679 * path will insure that the zero extent accounting
680 * has been taken care of before we get back into cluster_io)
681 */
682 ubc_upl_abort_range(upl, trunc_page(upl_offset), PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
683
684 e_offset = round_page_64(f_offset + 1);
685
686 if (ubc_sync_range(vp, f_offset, e_offset, UBC_PUSHDIRTY) == 0) {
687 error = EINVAL;
688 break;
689 }
690 io_size = e_offset - f_offset;
691
692 f_offset += io_size;
693 upl_offset += io_size;
694
695 if (size >= io_size)
696 size -= io_size;
697 else
698 size = 0;
699 /*
700 * keep track of how much of the original request
701 * that we've actually completed... non_rounded_size
702 * may go negative due to us rounding the request
703 * to a page size multiple (i.e. size > non_rounded_size)
704 */
705 non_rounded_size -= io_size;
706
707 if (non_rounded_size <= 0) {
708 /*
709 * we've transferred all of the data in the original
710 * request, but we were unable to complete the tail
711 * of the last page because the file didn't have
712 * an allocation to back that portion... this is ok.
713 */
714 size = 0;
715 }
716 continue;
717 }
718 lblkno = (daddr64_t)(f_offset / PAGE_SIZE_64);
719 /*
720 * we have now figured out how much I/O we can do - this is in 'io_size'
721 * pg_offset is the starting point in the first page for the I/O
722 * pg_count is the number of full and partial pages that 'io_size' encompasses
723 */
724 pg_offset = upl_offset & PAGE_MASK;
725
726 if (flags & CL_DEV_MEMORY) {
727 /*
728 * currently, can't deal with reading 'holes' in file
729 */
730 if (blkno == -1) {
731 error = EINVAL;
732 break;
733 }
734 /*
735 * treat physical requests as one 'giant' page
736 */
737 pg_count = 1;
738 } else
739 pg_count = (io_size + pg_offset + (PAGE_SIZE - 1)) / PAGE_SIZE;
740
741 if ((flags & CL_READ) && blkno == -1) {
742 int bytes_to_zero;
743
744 /*
745 * if we're reading and blkno == -1, then we've got a
746 * 'hole' in the file that we need to deal with by zeroing
747 * out the affected area in the upl
748 */
749 if (zero_offset && io_size == size) {
750 /*
751 * if this upl contains the EOF and it is not a multiple of PAGE_SIZE
752 * than 'zero_offset' will be non-zero
753 * if the 'hole' returned by vnop_blockmap extends all the way to the eof
754 * (indicated by the io_size finishing off the I/O request for this UPL)
755 * than we're not going to issue an I/O for the
756 * last page in this upl... we need to zero both the hole and the tail
757 * of the page beyond the EOF, since the delayed zero-fill won't kick in
758 */
759 bytes_to_zero = (((upl_offset + io_size) + (PAGE_SIZE - 1)) & ~PAGE_MASK) - upl_offset;
760
761 zero_offset = 0;
762 } else
763 bytes_to_zero = io_size;
764
765 cluster_zero(upl, upl_offset, bytes_to_zero, real_bp);
766
767 if (cbp_head)
768 /*
769 * if there is a current I/O chain pending
770 * then the first page of the group we just zero'd
771 * will be handled by the I/O completion if the zero
772 * fill started in the middle of the page
773 */
774 pg_count = (io_size - pg_offset) / PAGE_SIZE;
775 else {
776 /*
777 * no pending I/O to pick up that first page
778 * so, we have to make sure it gets committed
779 * here.
780 * set the pg_offset to 0 so that the upl_commit_range
781 * starts with this page
782 */
783 pg_count = (io_size + pg_offset) / PAGE_SIZE;
784 pg_offset = 0;
785 }
786 if (io_size == size && ((upl_offset + io_size) & PAGE_MASK))
787 /*
788 * if we're done with the request for this UPL
789 * then we have to make sure to commit the last page
790 * even if we only partially zero-filled it
791 */
792 pg_count++;
793
794 if (pg_count) {
795 if (pg_offset)
796 pg_resid = PAGE_SIZE - pg_offset;
797 else
798 pg_resid = 0;
799
800 if (flags & CL_COMMIT)
801 ubc_upl_commit_range(upl,
802 (upl_offset + pg_resid) & ~PAGE_MASK,
803 pg_count * PAGE_SIZE,
804 UPL_COMMIT_CLEAR_DIRTY | UPL_COMMIT_FREE_ON_EMPTY);
805 }
806 upl_offset += io_size;
807 f_offset += io_size;
808 size -= io_size;
809 /*
810 * keep track of how much of the original request
811 * that we've actually completed... non_rounded_size
812 * may go negative due to us rounding the request
813 * to a page size multiple (i.e. size > non_rounded_size)
814 */
815 non_rounded_size -= io_size;
816
817 if (non_rounded_size <= 0) {
818 /*
819 * we've transferred all of the data in the original
820 * request, but we were unable to complete the tail
821 * of the last page because the file didn't have
822 * an allocation to back that portion... this is ok.
823 */
824 size = 0;
825 }
826 if (cbp_head && pg_count)
827 goto start_io;
828 continue;
829
830 }
831 if (pg_count > max_vectors) {
832 if (((pg_count - max_vectors) * PAGE_SIZE) > io_size) {
833 io_size = PAGE_SIZE - pg_offset;
834 pg_count = 1;
835 } else {
836 io_size -= (pg_count - max_vectors) * PAGE_SIZE;
837 pg_count = max_vectors;
838 }
839 }
840
841 if ( !(mp->mnt_kern_flag & MNTK_VIRTUALDEV))
842 /*
843 * if we're not targeting a virtual device i.e. a disk image
844 * it's safe to dip into the reserve pool since real devices
845 * can complete this I/O request without requiring additional
846 * bufs from the alloc_io_buf pool
847 */
848 priv = 1;
849 else if ((flags & CL_ASYNC) && !(flags & CL_PAGEOUT))
850 /*
851 * Throttle the speculative IO
852 */
853 priv = 0;
854 else
855 priv = 1;
856
857 cbp = alloc_io_buf(vp, priv);
858
859 if (flags & CL_PAGEOUT) {
860 u_int i;
861
862 for (i = 0; i < pg_count; i++) {
863 if (buf_invalblkno(vp, lblkno + i, 0) == EBUSY)
864 panic("BUSY bp found in cluster_io");
865 }
866 }
867 if (flags & CL_ASYNC) {
868 if (buf_setcallback(cbp, (void *)cluster_iodone, NULL))
869 panic("buf_setcallback failed\n");
870 }
871 cbp->b_flags |= io_flags;
872
873 cbp->b_lblkno = lblkno;
874 cbp->b_blkno = blkno;
875 cbp->b_bcount = io_size;
876
877 if (buf_setupl(cbp, upl, upl_offset))
878 panic("buf_setupl failed\n");
879
880 cbp->b_trans_next = (buf_t)NULL;
881
882 if ((cbp->b_iostate = (void *)iostate))
883 /*
884 * caller wants to track the state of this
885 * io... bump the amount issued against this stream
886 */
887 iostate->io_issued += io_size;
888
889 if (flags & CL_READ) {
890 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 26)) | DBG_FUNC_NONE,
891 (int)cbp->b_lblkno, (int)cbp->b_blkno, upl_offset, io_size, 0);
892 }
893 else {
894 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 27)) | DBG_FUNC_NONE,
895 (int)cbp->b_lblkno, (int)cbp->b_blkno, upl_offset, io_size, 0);
896 }
897
898 if (cbp_head) {
899 cbp_tail->b_trans_next = cbp;
900 cbp_tail = cbp;
901 } else {
902 cbp_head = cbp;
903 cbp_tail = cbp;
904 }
905 (buf_t)(cbp->b_trans_head) = cbp_head;
906 trans_count++;
907
908 upl_offset += io_size;
909 f_offset += io_size;
910 size -= io_size;
911 /*
912 * keep track of how much of the original request
913 * that we've actually completed... non_rounded_size
914 * may go negative due to us rounding the request
915 * to a page size multiple (i.e. size > non_rounded_size)
916 */
917 non_rounded_size -= io_size;
918
919 if (non_rounded_size <= 0) {
920 /*
921 * we've transferred all of the data in the original
922 * request, but we were unable to complete the tail
923 * of the last page because the file didn't have
924 * an allocation to back that portion... this is ok.
925 */
926 size = 0;
927 }
928 if ( (!(upl_offset & PAGE_MASK) && !(flags & CL_DEV_MEMORY) && ((flags & CL_ASYNC) || trans_count > 8)) || size == 0) {
929 /*
930 * if we have no more I/O to issue or
931 * the current I/O we've prepared fully
932 * completes the last page in this request
933 * and it's either an ASYNC request or
934 * we've already accumulated more than 8 I/O's into
935 * this transaction and it's not an I/O directed to
936 * special DEVICE memory
937 * then go ahead and issue the I/O
938 */
939 start_io:
940 if (real_bp) {
941 cbp_head->b_flags |= B_NEED_IODONE;
942 cbp_head->b_real_bp = real_bp;
943 } else
944 cbp_head->b_real_bp = (buf_t)NULL;
945
946 if (size == 0) {
947 /*
948 * we're about to issue the last I/O for this upl
949 * if this was a read to the eof and the eof doesn't
950 * finish on a page boundary, than we need to zero-fill
951 * the rest of the page....
952 */
953 cbp_head->b_validend = zero_offset;
954 } else
955 cbp_head->b_validend = 0;
956
957 if (flags & CL_THROTTLE)
958 (void)vnode_waitforwrites(vp, async_throttle, 0, 0, (char *)"cluster_io");
959
960 for (cbp = cbp_head; cbp;) {
961 buf_t cbp_next;
962
963 if ( !(io_flags & B_READ))
964 vnode_startwrite(vp);
965
966 cbp_next = cbp->b_trans_next;
967
968 (void) VNOP_STRATEGY(cbp);
969 cbp = cbp_next;
970 }
971 if ( !(flags & CL_ASYNC)) {
972 int dummy;
973
974 for (cbp = cbp_head; cbp; cbp = cbp->b_trans_next)
975 buf_biowait(cbp);
976
977 if ((error = cluster_iodone(cbp_head, (void *)&dummy))) {
978 if (((flags & (CL_PAGEOUT | CL_KEEPCACHED)) == CL_PAGEOUT) && (error == ENXIO))
979 error = 0; /* drop the error */
980 else {
981 if (retval == 0)
982 retval = error;
983 error = 0;
984 }
985 }
986 }
987 cbp_head = (buf_t)NULL;
988 cbp_tail = (buf_t)NULL;
989
990 trans_count = 0;
991 }
992 }
993 if (error) {
994 int abort_size;
995
996 io_size = 0;
997
998 for (cbp = cbp_head; cbp;) {
999 buf_t cbp_next;
1000
1001 upl_offset -= cbp->b_bcount;
1002 size += cbp->b_bcount;
1003 io_size += cbp->b_bcount;
1004
1005 cbp_next = cbp->b_trans_next;
1006 free_io_buf(cbp);
1007 cbp = cbp_next;
1008 }
1009 if (iostate) {
1010 int need_wakeup = 0;
1011
1012 /*
1013 * update the error condition for this stream
1014 * since we never really issued the io
1015 * just go ahead and adjust it back
1016 */
1017 lck_mtx_lock(cl_mtxp);
1018
1019 if (iostate->io_error == 0)
1020 iostate->io_error = error;
1021 iostate->io_issued -= io_size;
1022
1023 if (iostate->io_wanted) {
1024 /*
1025 * someone is waiting for the state of
1026 * this io stream to change
1027 */
1028 iostate->io_wanted = 0;
1029 need_wakeup = 0;
1030 }
1031 lck_mtx_unlock(cl_mtxp);
1032
1033 if (need_wakeup)
1034 wakeup((caddr_t)&iostate->io_wanted);
1035 }
1036 pg_offset = upl_offset & PAGE_MASK;
1037 abort_size = (size + pg_offset + (PAGE_SIZE - 1)) & ~PAGE_MASK;
1038
1039 if (flags & CL_COMMIT) {
1040 int upl_abort_code;
1041
1042 if (flags & CL_PRESERVE) {
1043 ubc_upl_commit_range(upl, upl_offset - pg_offset, abort_size,
1044 UPL_COMMIT_FREE_ON_EMPTY);
1045 } else {
1046 if ((flags & CL_PAGEOUT) && (error != ENXIO)) /* transient error */
1047 upl_abort_code = UPL_ABORT_FREE_ON_EMPTY;
1048 else if (flags & CL_PAGEIN)
1049 upl_abort_code = UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR;
1050 else
1051 upl_abort_code = UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_DUMP_PAGES;
1052
1053 ubc_upl_abort_range(upl, upl_offset - pg_offset, abort_size,
1054 upl_abort_code);
1055 }
1056 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 28)) | DBG_FUNC_NONE,
1057 (int)upl, upl_offset - pg_offset, abort_size, error, 0);
1058 }
1059 if (real_bp) {
1060 real_bp->b_flags |= B_ERROR;
1061 real_bp->b_error = error;
1062
1063 buf_biodone(real_bp);
1064 }
1065 if (retval == 0)
1066 retval = error;
1067 }
1068 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 22)) | DBG_FUNC_END,
1069 (int)f_offset, size, upl_offset, retval, 0);
1070
1071 return (retval);
1072 }
1073
1074
1075 static int
1076 cluster_rd_prefetch(vnode_t vp, off_t f_offset, u_int size, off_t filesize)
1077 {
1078 int pages_in_prefetch;
1079
1080 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 49)) | DBG_FUNC_START,
1081 (int)f_offset, size, (int)filesize, 0, 0);
1082
1083 if (f_offset >= filesize) {
1084 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 49)) | DBG_FUNC_END,
1085 (int)f_offset, 0, 0, 0, 0);
1086 return(0);
1087 }
1088 if (size > (MAX_UPL_TRANSFER * PAGE_SIZE))
1089 size = (MAX_UPL_TRANSFER * PAGE_SIZE);
1090 else
1091 size = (size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
1092
1093 if ((off_t)size > (filesize - f_offset))
1094 size = filesize - f_offset;
1095 pages_in_prefetch = (size + (PAGE_SIZE - 1)) / PAGE_SIZE;
1096
1097 advisory_read(vp, filesize, f_offset, size);
1098
1099 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 49)) | DBG_FUNC_END,
1100 (int)f_offset + size, pages_in_prefetch, 0, 1, 0);
1101
1102 return (pages_in_prefetch);
1103 }
1104
1105
1106
1107 static void
1108 cluster_rd_ahead(vnode_t vp, struct cl_extent *extent, off_t filesize, struct cl_readahead *rap)
1109 {
1110 daddr64_t r_addr;
1111 off_t f_offset;
1112 int size_of_prefetch;
1113
1114
1115 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_START,
1116 (int)extent->b_addr, (int)extent->e_addr, (int)rap->cl_lastr, 0, 0);
1117
1118 if (extent->b_addr == rap->cl_lastr && extent->b_addr == extent->e_addr) {
1119 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_END,
1120 rap->cl_ralen, (int)rap->cl_maxra, (int)rap->cl_lastr, 0, 0);
1121 return;
1122 }
1123 if (rap->cl_lastr == -1 || (extent->b_addr != rap->cl_lastr && extent->b_addr != (rap->cl_lastr + 1) &&
1124 (extent->b_addr != (rap->cl_maxra + 1) || rap->cl_ralen == 0))) {
1125 rap->cl_ralen = 0;
1126 rap->cl_maxra = 0;
1127
1128 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_END,
1129 rap->cl_ralen, (int)rap->cl_maxra, (int)rap->cl_lastr, 1, 0);
1130
1131 return;
1132 }
1133 if (extent->e_addr < rap->cl_maxra) {
1134 if ((rap->cl_maxra - extent->e_addr) > (MAX_UPL_TRANSFER / 4)) {
1135
1136 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_END,
1137 rap->cl_ralen, (int)rap->cl_maxra, (int)rap->cl_lastr, 2, 0);
1138 return;
1139 }
1140 }
1141 r_addr = max(extent->e_addr, rap->cl_maxra) + 1;
1142 f_offset = (off_t)(r_addr * PAGE_SIZE_64);
1143
1144 size_of_prefetch = 0;
1145
1146 ubc_range_op(vp, f_offset, f_offset + PAGE_SIZE_64, UPL_ROP_PRESENT, &size_of_prefetch);
1147
1148 if (size_of_prefetch) {
1149 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_END,
1150 rap->cl_ralen, (int)rap->cl_maxra, (int)rap->cl_lastr, 3, 0);
1151 return;
1152 }
1153 if (f_offset < filesize) {
1154 daddr64_t read_size;
1155
1156 rap->cl_ralen = rap->cl_ralen ? min(MAX_UPL_TRANSFER, rap->cl_ralen << 1) : 1;
1157
1158 read_size = (extent->e_addr + 1) - extent->b_addr;
1159
1160 if (read_size > rap->cl_ralen) {
1161 if (read_size > MAX_UPL_TRANSFER)
1162 rap->cl_ralen = MAX_UPL_TRANSFER;
1163 else
1164 rap->cl_ralen = read_size;
1165 }
1166 size_of_prefetch = cluster_rd_prefetch(vp, f_offset, rap->cl_ralen * PAGE_SIZE, filesize);
1167
1168 if (size_of_prefetch)
1169 rap->cl_maxra = (r_addr + size_of_prefetch) - 1;
1170 }
1171 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_END,
1172 rap->cl_ralen, (int)rap->cl_maxra, (int)rap->cl_lastr, 4, 0);
1173 }
1174
1175 int
1176 cluster_pageout(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset,
1177 int size, off_t filesize, int flags)
1178 {
1179 int io_size;
1180 int rounded_size;
1181 off_t max_size;
1182 int local_flags;
1183 struct cl_writebehind *wbp;
1184
1185 if (vp->v_mount->mnt_kern_flag & MNTK_VIRTUALDEV)
1186 /*
1187 * if we know we're issuing this I/O to a virtual device (i.e. disk image)
1188 * then we don't want to enforce this throttle... if we do, we can
1189 * potentially deadlock since we're stalling the pageout thread at a time
1190 * when the disk image might need additional memory (which won't be available
1191 * if the pageout thread can't run)... instead we'll just depend on the throttle
1192 * that the pageout thread now has in place to deal with external files
1193 */
1194 local_flags = CL_PAGEOUT;
1195 else
1196 local_flags = CL_PAGEOUT | CL_THROTTLE;
1197
1198 if ((flags & UPL_IOSYNC) == 0)
1199 local_flags |= CL_ASYNC;
1200 if ((flags & UPL_NOCOMMIT) == 0)
1201 local_flags |= CL_COMMIT;
1202 if ((flags & UPL_KEEPCACHED))
1203 local_flags |= CL_KEEPCACHED;
1204
1205
1206 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 52)) | DBG_FUNC_NONE,
1207 (int)f_offset, size, (int)filesize, local_flags, 0);
1208
1209 /*
1210 * If they didn't specify any I/O, then we are done...
1211 * we can't issue an abort because we don't know how
1212 * big the upl really is
1213 */
1214 if (size <= 0)
1215 return (EINVAL);
1216
1217 if (vp->v_mount->mnt_flag & MNT_RDONLY) {
1218 if (local_flags & CL_COMMIT)
1219 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
1220 return (EROFS);
1221 }
1222 /*
1223 * can't page-in from a negative offset
1224 * or if we're starting beyond the EOF
1225 * or if the file offset isn't page aligned
1226 * or the size requested isn't a multiple of PAGE_SIZE
1227 */
1228 if (f_offset < 0 || f_offset >= filesize ||
1229 (f_offset & PAGE_MASK_64) || (size & PAGE_MASK)) {
1230 if (local_flags & CL_COMMIT)
1231 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
1232 return (EINVAL);
1233 }
1234 max_size = filesize - f_offset;
1235
1236 if (size < max_size)
1237 io_size = size;
1238 else
1239 io_size = max_size;
1240
1241 rounded_size = (io_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
1242
1243 if (size > rounded_size) {
1244 if (local_flags & CL_COMMIT)
1245 ubc_upl_abort_range(upl, upl_offset + rounded_size, size - rounded_size,
1246 UPL_ABORT_FREE_ON_EMPTY);
1247 }
1248 if ((wbp = cluster_get_wbp(vp, 0)) != NULL)
1249 wbp->cl_hasbeenpaged = 1;
1250
1251 return (cluster_io(vp, upl, upl_offset, f_offset, io_size,
1252 local_flags, (buf_t)NULL, (struct clios *)NULL));
1253 }
1254
1255 int
1256 cluster_pagein(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset,
1257 int size, off_t filesize, int flags)
1258 {
1259 u_int io_size;
1260 int rounded_size;
1261 off_t max_size;
1262 int retval;
1263 int local_flags = 0;
1264
1265 if (upl == NULL || size < 0)
1266 panic("cluster_pagein: NULL upl passed in");
1267
1268 if ((flags & UPL_IOSYNC) == 0)
1269 local_flags |= CL_ASYNC;
1270 if ((flags & UPL_NOCOMMIT) == 0)
1271 local_flags |= CL_COMMIT;
1272
1273
1274 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 56)) | DBG_FUNC_NONE,
1275 (int)f_offset, size, (int)filesize, local_flags, 0);
1276
1277 /*
1278 * can't page-in from a negative offset
1279 * or if we're starting beyond the EOF
1280 * or if the file offset isn't page aligned
1281 * or the size requested isn't a multiple of PAGE_SIZE
1282 */
1283 if (f_offset < 0 || f_offset >= filesize ||
1284 (f_offset & PAGE_MASK_64) || (size & PAGE_MASK) || (upl_offset & PAGE_MASK)) {
1285 if (local_flags & CL_COMMIT)
1286 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);
1287 return (EINVAL);
1288 }
1289 max_size = filesize - f_offset;
1290
1291 if (size < max_size)
1292 io_size = size;
1293 else
1294 io_size = max_size;
1295
1296 rounded_size = (io_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
1297
1298 if (size > rounded_size && (local_flags & CL_COMMIT))
1299 ubc_upl_abort_range(upl, upl_offset + rounded_size,
1300 size - rounded_size, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);
1301
1302 retval = cluster_io(vp, upl, upl_offset, f_offset, io_size,
1303 local_flags | CL_READ | CL_PAGEIN, (buf_t)NULL, (struct clios *)NULL);
1304
1305 if (retval == 0 && !(flags & UPL_NORDAHEAD) && !(vp->v_flag & VRAOFF)) {
1306 struct cl_readahead *rap;
1307
1308 rap = cluster_get_rap(vp);
1309
1310 if (rap != NULL) {
1311 struct cl_extent extent;
1312
1313 extent.b_addr = (daddr64_t)(f_offset / PAGE_SIZE_64);
1314 extent.e_addr = (daddr64_t)((f_offset + ((off_t)io_size - 1)) / PAGE_SIZE_64);
1315
1316 if (rounded_size == PAGE_SIZE) {
1317 /*
1318 * we haven't read the last page in of the file yet
1319 * so let's try to read ahead if we're in
1320 * a sequential access pattern
1321 */
1322 cluster_rd_ahead(vp, &extent, filesize, rap);
1323 }
1324 rap->cl_lastr = extent.e_addr;
1325
1326 lck_mtx_unlock(&rap->cl_lockr);
1327 }
1328 }
1329 return (retval);
1330 }
1331
1332 int
1333 cluster_bp(buf_t bp)
1334 {
1335 off_t f_offset;
1336 int flags;
1337
1338 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 19)) | DBG_FUNC_START,
1339 (int)bp, (int)bp->b_lblkno, bp->b_bcount, bp->b_flags, 0);
1340
1341 if (bp->b_flags & B_READ)
1342 flags = CL_ASYNC | CL_READ;
1343 else
1344 flags = CL_ASYNC;
1345
1346 f_offset = ubc_blktooff(bp->b_vp, bp->b_lblkno);
1347
1348 return (cluster_io(bp->b_vp, bp->b_upl, 0, f_offset, bp->b_bcount, flags, bp, (struct clios *)NULL));
1349 }
1350
1351 int
1352 cluster_write(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF, off_t headOff, off_t tailOff, int xflags)
1353 {
1354 int prev_resid;
1355 u_int clip_size;
1356 off_t max_io_size;
1357 int upl_size;
1358 int upl_flags;
1359 upl_t upl;
1360 int retval = 0;
1361 int flags;
1362
1363 flags = xflags;
1364
1365 if (vp->v_flag & VNOCACHE_DATA)
1366 flags |= IO_NOCACHE;
1367
1368 if ( (!(flags & IO_NOCACHE)) || (!uio) || (!UIO_SEG_IS_USER_SPACE(uio->uio_segflg))) {
1369 /*
1370 * go do a write through the cache if one of the following is true....
1371 * NOCACHE is not true
1372 * there is no uio structure or it doesn't target USERSPACE
1373 */
1374 return (cluster_write_x(vp, uio, oldEOF, newEOF, headOff, tailOff, flags));
1375 }
1376
1377 #if LP64_DEBUG
1378 if (IS_VALID_UIO_SEGFLG(uio->uio_segflg) == 0) {
1379 panic("%s :%d - invalid uio_segflg\n", __FILE__, __LINE__);
1380 }
1381 #endif /* LP64_DEBUG */
1382
1383 while (uio_resid(uio) && uio->uio_offset < newEOF && retval == 0) {
1384 user_size_t iov_len;
1385 user_addr_t iov_base;
1386
1387 /*
1388 * we know we have a resid, so this is safe
1389 * skip over any emtpy vectors
1390 */
1391 uio_update(uio, (user_size_t)0);
1392
1393 iov_len = uio_curriovlen(uio);
1394 iov_base = uio_curriovbase(uio);
1395
1396 upl_size = PAGE_SIZE;
1397 upl_flags = UPL_QUERY_OBJECT_TYPE;
1398
1399 // LP64todo - fix this!
1400 if ((vm_map_get_upl(current_map(),
1401 (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)),
1402 &upl_size, &upl, NULL, NULL, &upl_flags, 0)) != KERN_SUCCESS) {
1403 /*
1404 * the user app must have passed in an invalid address
1405 */
1406 return (EFAULT);
1407 }
1408
1409 /*
1410 * We check every vector target but if it is physically
1411 * contiguous space, we skip the sanity checks.
1412 */
1413 if (upl_flags & UPL_PHYS_CONTIG) {
1414 int zflags;
1415
1416 zflags = flags & ~IO_TAILZEROFILL;
1417 zflags |= IO_HEADZEROFILL;
1418
1419 if (flags & IO_HEADZEROFILL) {
1420 /*
1421 * in case we have additional vectors, we don't want to do this again
1422 */
1423 flags &= ~IO_HEADZEROFILL;
1424
1425 if ((retval = cluster_write_x(vp, (struct uio *)0, 0, uio->uio_offset, headOff, 0, zflags)))
1426 return(retval);
1427 }
1428 retval = cluster_phys_write(vp, uio, newEOF);
1429
1430 if (uio_resid(uio) == 0 && (flags & IO_TAILZEROFILL)) {
1431 return (cluster_write_x(vp, (struct uio *)0, 0, tailOff, uio->uio_offset, 0, zflags));
1432 }
1433 }
1434 else if ((uio_resid(uio) < PAGE_SIZE) || (flags & (IO_TAILZEROFILL | IO_HEADZEROFILL))) {
1435 /*
1436 * we're here because we're don't have a physically contiguous target buffer
1437 * go do a write through the cache if one of the following is true....
1438 * the total xfer size is less than a page...
1439 * we're being asked to ZEROFILL either the head or the tail of the I/O...
1440 */
1441 return (cluster_write_x(vp, uio, oldEOF, newEOF, headOff, tailOff, flags));
1442 }
1443 // LP64todo - fix this!
1444 else if (((int)uio->uio_offset & PAGE_MASK) || (CAST_DOWN(int, iov_base) & PAGE_MASK)) {
1445 if (((int)uio->uio_offset & PAGE_MASK) == (CAST_DOWN(int, iov_base) & PAGE_MASK)) {
1446 /*
1447 * Bring the file offset write up to a pagesize boundary
1448 * this will also bring the base address to a page boundary
1449 * since they both are currently on the same offset within a page
1450 * note: if we get here, uio->uio_resid is greater than PAGE_SIZE
1451 * so the computed clip_size must always be less than the current uio_resid
1452 */
1453 clip_size = (PAGE_SIZE - (uio->uio_offset & PAGE_MASK_64));
1454
1455 /*
1456 * Fake the resid going into the cluster_write_x call
1457 * and restore it on the way out.
1458 */
1459 // LP64todo - fix this
1460 prev_resid = uio_resid(uio);
1461 uio_setresid(uio, clip_size);
1462
1463 retval = cluster_write_x(vp, uio, oldEOF, newEOF, headOff, tailOff, flags);
1464
1465 uio_setresid(uio, prev_resid - (clip_size - uio_resid(uio)));
1466 } else {
1467 /*
1468 * can't get both the file offset and the buffer offset aligned to a page boundary
1469 * so fire an I/O through the cache for this entire vector
1470 */
1471 // LP64todo - fix this
1472 clip_size = iov_len;
1473 // LP64todo - fix this
1474 prev_resid = uio_resid(uio);
1475 uio_setresid(uio, clip_size);
1476
1477 retval = cluster_write_x(vp, uio, oldEOF, newEOF, headOff, tailOff, flags);
1478
1479 uio_setresid(uio, prev_resid - (clip_size - uio_resid(uio)));
1480 }
1481 } else {
1482 /*
1483 * If we come in here, we know the offset into
1484 * the file is on a pagesize boundary and the
1485 * target buffer address is also on a page boundary
1486 */
1487 max_io_size = newEOF - uio->uio_offset;
1488 // LP64todo - fix this
1489 clip_size = uio_resid(uio);
1490 if (iov_len < clip_size)
1491 // LP64todo - fix this!
1492 clip_size = iov_len;
1493 if (max_io_size < clip_size)
1494 clip_size = max_io_size;
1495
1496 if (clip_size < PAGE_SIZE) {
1497 /*
1498 * Take care of tail end of write in this vector
1499 */
1500 // LP64todo - fix this
1501 prev_resid = uio_resid(uio);
1502 uio_setresid(uio, clip_size);
1503
1504 retval = cluster_write_x(vp, uio, oldEOF, newEOF, headOff, tailOff, flags);
1505
1506 uio_setresid(uio, prev_resid - (clip_size - uio_resid(uio)));
1507 } else {
1508 /* round clip_size down to a multiple of pagesize */
1509 clip_size = clip_size & ~(PAGE_MASK);
1510 // LP64todo - fix this
1511 prev_resid = uio_resid(uio);
1512 uio_setresid(uio, clip_size);
1513
1514 retval = cluster_nocopy_write(vp, uio, newEOF);
1515
1516 if ((retval == 0) && uio_resid(uio))
1517 retval = cluster_write_x(vp, uio, oldEOF, newEOF, headOff, tailOff, flags);
1518
1519 uio_setresid(uio, prev_resid - (clip_size - uio_resid(uio)));
1520 }
1521 } /* end else */
1522 } /* end while */
1523
1524 return(retval);
1525 }
1526
1527
1528 static int
1529 cluster_nocopy_write(vnode_t vp, struct uio *uio, off_t newEOF)
1530 {
1531 upl_t upl;
1532 upl_page_info_t *pl;
1533 vm_offset_t upl_offset;
1534 int io_size;
1535 int io_flag;
1536 int upl_size;
1537 int upl_needed_size;
1538 int pages_in_pl;
1539 int upl_flags;
1540 kern_return_t kret;
1541 int i;
1542 int force_data_sync;
1543 int error = 0;
1544 struct clios iostate;
1545 struct cl_writebehind *wbp;
1546
1547
1548 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 75)) | DBG_FUNC_START,
1549 (int)uio->uio_offset, (int)uio_resid(uio),
1550 (int)newEOF, 0, 0);
1551
1552 /*
1553 * When we enter this routine, we know
1554 * -- the offset into the file is on a pagesize boundary
1555 * -- the resid is a page multiple
1556 * -- the resid will not exceed iov_len
1557 */
1558
1559 if ((wbp = cluster_get_wbp(vp, CLW_RETURNLOCKED)) != NULL) {
1560
1561 cluster_try_push(wbp, vp, newEOF, 0, 1);
1562
1563 lck_mtx_unlock(&wbp->cl_lockw);
1564 }
1565 iostate.io_completed = 0;
1566 iostate.io_issued = 0;
1567 iostate.io_error = 0;
1568 iostate.io_wanted = 0;
1569
1570 while (uio_resid(uio) && uio->uio_offset < newEOF && error == 0) {
1571 user_addr_t iov_base;
1572
1573 io_size = uio_resid(uio);
1574
1575 if (io_size > (MAX_UPL_TRANSFER * PAGE_SIZE))
1576 io_size = MAX_UPL_TRANSFER * PAGE_SIZE;
1577
1578 iov_base = uio_curriovbase(uio);
1579
1580 // LP64todo - fix this!
1581 upl_offset = CAST_DOWN(vm_offset_t, iov_base) & PAGE_MASK;
1582
1583 upl_needed_size = (upl_offset + io_size + (PAGE_SIZE -1)) & ~PAGE_MASK;
1584
1585 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 76)) | DBG_FUNC_START,
1586 (int)upl_offset, upl_needed_size, (int)iov_base, io_size, 0);
1587
1588 for (force_data_sync = 0; force_data_sync < 3; force_data_sync++) {
1589 pages_in_pl = 0;
1590 upl_size = upl_needed_size;
1591 upl_flags = UPL_FILE_IO | UPL_COPYOUT_FROM | UPL_NO_SYNC |
1592 UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE;
1593
1594 // LP64todo - fix this!
1595 kret = vm_map_get_upl(current_map(),
1596 (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)),
1597 &upl_size,
1598 &upl,
1599 NULL,
1600 &pages_in_pl,
1601 &upl_flags,
1602 force_data_sync);
1603
1604 if (kret != KERN_SUCCESS) {
1605 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 76)) | DBG_FUNC_END,
1606 0, 0, 0, kret, 0);
1607 /*
1608 * cluster_nocopy_write: failed to get pagelist
1609 *
1610 * we may have already spun some portion of this request
1611 * off as async requests... we need to wait for the I/O
1612 * to complete before returning
1613 */
1614 goto wait_for_writes;
1615 }
1616 pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
1617 pages_in_pl = upl_size / PAGE_SIZE;
1618
1619 for (i = 0; i < pages_in_pl; i++) {
1620 if (!upl_valid_page(pl, i))
1621 break;
1622 }
1623 if (i == pages_in_pl)
1624 break;
1625
1626 /*
1627 * didn't get all the pages back that we
1628 * needed... release this upl and try again
1629 */
1630 ubc_upl_abort_range(upl, (upl_offset & ~PAGE_MASK), upl_size,
1631 UPL_ABORT_FREE_ON_EMPTY);
1632 }
1633 if (force_data_sync >= 3) {
1634 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 76)) | DBG_FUNC_END,
1635 i, pages_in_pl, upl_size, kret, 0);
1636 /*
1637 * for some reason, we couldn't acquire a hold on all
1638 * the pages needed in the user's address space
1639 *
1640 * we may have already spun some portion of this request
1641 * off as async requests... we need to wait for the I/O
1642 * to complete before returning
1643 */
1644 goto wait_for_writes;
1645 }
1646
1647 /*
1648 * Consider the possibility that upl_size wasn't satisfied.
1649 */
1650 if (upl_size != upl_needed_size)
1651 io_size = (upl_size - (int)upl_offset) & ~PAGE_MASK;
1652
1653 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 76)) | DBG_FUNC_END,
1654 (int)upl_offset, upl_size, (int)iov_base, io_size, 0);
1655
1656 if (io_size == 0) {
1657 ubc_upl_abort_range(upl, (upl_offset & ~PAGE_MASK), upl_size,
1658 UPL_ABORT_FREE_ON_EMPTY);
1659 /*
1660 * we may have already spun some portion of this request
1661 * off as async requests... we need to wait for the I/O
1662 * to complete before returning
1663 */
1664 goto wait_for_writes;
1665 }
1666 /*
1667 * Now look for pages already in the cache
1668 * and throw them away.
1669 * uio->uio_offset is page aligned within the file
1670 * io_size is a multiple of PAGE_SIZE
1671 */
1672 ubc_range_op(vp, uio->uio_offset, uio->uio_offset + io_size, UPL_ROP_DUMP, NULL);
1673
1674 /*
1675 * we want push out these writes asynchronously so that we can overlap
1676 * the preparation of the next I/O
1677 * if there are already too many outstanding writes
1678 * wait until some complete before issuing the next
1679 */
1680 lck_mtx_lock(cl_mtxp);
1681
1682 while ((iostate.io_issued - iostate.io_completed) > (2 * MAX_UPL_TRANSFER * PAGE_SIZE)) {
1683 iostate.io_wanted = 1;
1684 msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_nocopy_write", 0);
1685 }
1686 lck_mtx_unlock(cl_mtxp);
1687
1688 if (iostate.io_error) {
1689 /*
1690 * one of the earlier writes we issued ran into a hard error
1691 * don't issue any more writes, cleanup the UPL
1692 * that was just created but not used, then
1693 * go wait for all writes that are part of this stream
1694 * to complete before returning the error to the caller
1695 */
1696 ubc_upl_abort_range(upl, (upl_offset & ~PAGE_MASK), upl_size,
1697 UPL_ABORT_FREE_ON_EMPTY);
1698
1699 goto wait_for_writes;
1700 }
1701 io_flag = CL_ASYNC | CL_PRESERVE | CL_COMMIT | CL_THROTTLE;
1702
1703 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 77)) | DBG_FUNC_START,
1704 (int)upl_offset, (int)uio->uio_offset, io_size, io_flag, 0);
1705
1706 error = cluster_io(vp, upl, upl_offset, uio->uio_offset,
1707 io_size, io_flag, (buf_t)NULL, &iostate);
1708
1709 uio_update(uio, (user_size_t)io_size);
1710
1711 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 77)) | DBG_FUNC_END,
1712 (int)upl_offset, (int)uio->uio_offset, (int)uio_resid(uio), error, 0);
1713
1714 } /* end while */
1715
1716 wait_for_writes:
1717 /*
1718 * make sure all async writes issued as part of this stream
1719 * have completed before we return
1720 */
1721 lck_mtx_lock(cl_mtxp);
1722
1723 while (iostate.io_issued != iostate.io_completed) {
1724 iostate.io_wanted = 1;
1725 msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_nocopy_write", 0);
1726 }
1727 lck_mtx_unlock(cl_mtxp);
1728
1729 if (iostate.io_error)
1730 error = iostate.io_error;
1731
1732 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 75)) | DBG_FUNC_END,
1733 (int)uio->uio_offset, (int)uio->uio_resid, error, 4, 0);
1734
1735 return (error);
1736 }
1737
1738
1739 static int
1740 cluster_phys_write(vnode_t vp, struct uio *uio, off_t newEOF)
1741 {
1742 upl_page_info_t *pl;
1743 addr64_t src_paddr;
1744 upl_t upl;
1745 vm_offset_t upl_offset;
1746 int tail_size;
1747 int io_size;
1748 int upl_size;
1749 int upl_needed_size;
1750 int pages_in_pl;
1751 int upl_flags;
1752 kern_return_t kret;
1753 int error = 0;
1754 user_addr_t iov_base;
1755 int devblocksize;
1756 struct cl_writebehind *wbp;
1757
1758 devblocksize = vp->v_mount->mnt_devblocksize;
1759 /*
1760 * When we enter this routine, we know
1761 * -- the resid will not exceed iov_len
1762 * -- the vector target address is physcially contiguous
1763 */
1764 if ((wbp = cluster_get_wbp(vp, CLW_RETURNLOCKED)) != NULL) {
1765
1766 cluster_try_push(wbp, vp, newEOF, 0, 1);
1767
1768 lck_mtx_unlock(&wbp->cl_lockw);
1769 }
1770 #if LP64_DEBUG
1771 if (IS_VALID_UIO_SEGFLG(uio->uio_segflg) == 0) {
1772 panic("%s :%d - invalid uio_segflg\n", __FILE__, __LINE__);
1773 }
1774 #endif /* LP64_DEBUG */
1775
1776 // LP64todo - fix this!
1777 io_size = (int)uio_curriovlen(uio);
1778 iov_base = uio_curriovbase(uio);
1779
1780 upl_offset = CAST_DOWN(upl_offset_t, iov_base) & PAGE_MASK;
1781 upl_needed_size = upl_offset + io_size;
1782
1783 pages_in_pl = 0;
1784 upl_size = upl_needed_size;
1785 upl_flags = UPL_FILE_IO | UPL_COPYOUT_FROM | UPL_NO_SYNC |
1786 UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE;
1787
1788 // LP64todo - fix this!
1789 kret = vm_map_get_upl(current_map(),
1790 (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)),
1791 &upl_size, &upl, NULL, &pages_in_pl, &upl_flags, 0);
1792
1793 if (kret != KERN_SUCCESS) {
1794 /*
1795 * cluster_phys_write: failed to get pagelist
1796 * note: return kret here
1797 */
1798 return(EINVAL);
1799 }
1800 /*
1801 * Consider the possibility that upl_size wasn't satisfied.
1802 * This is a failure in the physical memory case.
1803 */
1804 if (upl_size < upl_needed_size) {
1805 ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_FREE_ON_EMPTY);
1806 return(EINVAL);
1807 }
1808 pl = ubc_upl_pageinfo(upl);
1809
1810 src_paddr = ((addr64_t)upl_phys_page(pl, 0) << 12) + (addr64_t)upl_offset;
1811
1812 while (((uio->uio_offset & (devblocksize - 1)) || io_size < devblocksize) && io_size) {
1813 int head_size;
1814
1815 head_size = devblocksize - (int)(uio->uio_offset & (devblocksize - 1));
1816
1817 if (head_size > io_size)
1818 head_size = io_size;
1819
1820 error = cluster_align_phys_io(vp, uio, src_paddr, head_size, 0);
1821
1822 if (error) {
1823 ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_FREE_ON_EMPTY);
1824
1825 return(EINVAL);
1826 }
1827 upl_offset += head_size;
1828 src_paddr += head_size;
1829 io_size -= head_size;
1830 }
1831 tail_size = io_size & (devblocksize - 1);
1832 io_size -= tail_size;
1833
1834 if (io_size) {
1835 /*
1836 * issue a synchronous write to cluster_io
1837 */
1838 error = cluster_io(vp, upl, upl_offset, uio->uio_offset,
1839 io_size, CL_DEV_MEMORY, (buf_t)NULL, (struct clios *)NULL);
1840 }
1841 if (error == 0) {
1842 /*
1843 * The cluster_io write completed successfully,
1844 * update the uio structure
1845 */
1846 uio_update(uio, (user_size_t)io_size);
1847
1848 src_paddr += io_size;
1849
1850 if (tail_size)
1851 error = cluster_align_phys_io(vp, uio, src_paddr, tail_size, 0);
1852 }
1853 /*
1854 * just release our hold on the physically contiguous
1855 * region without changing any state
1856 */
1857 ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_FREE_ON_EMPTY);
1858
1859 return (error);
1860 }
1861
1862
1863 static int
1864 cluster_write_x(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF, off_t headOff, off_t tailOff, int flags)
1865 {
1866 upl_page_info_t *pl;
1867 upl_t upl;
1868 vm_offset_t upl_offset = 0;
1869 int upl_size;
1870 off_t upl_f_offset;
1871 int pages_in_upl;
1872 int start_offset;
1873 int xfer_resid;
1874 int io_size;
1875 int io_offset;
1876 int bytes_to_zero;
1877 int bytes_to_move;
1878 kern_return_t kret;
1879 int retval = 0;
1880 int io_resid;
1881 long long total_size;
1882 long long zero_cnt;
1883 off_t zero_off;
1884 long long zero_cnt1;
1885 off_t zero_off1;
1886 struct cl_extent cl;
1887 int intersection;
1888 struct cl_writebehind *wbp;
1889
1890 if ((wbp = cluster_get_wbp(vp, 0)) != NULL)
1891 {
1892 if (wbp->cl_hasbeenpaged) {
1893 /*
1894 * this vnode had pages cleaned to it by
1895 * the pager which indicates that either
1896 * it's not very 'hot', or the system is
1897 * being overwhelmed by a lot of dirty
1898 * data being delayed in the VM cache...
1899 * in either event, we'll push our remaining
1900 * delayed data at this point... this will
1901 * be more efficient than paging out 1 page at
1902 * a time, and will also act as a throttle
1903 * by delaying this client from writing any
1904 * more data until all his delayed data has
1905 * at least been queued to the uderlying driver.
1906 */
1907 if (wbp->cl_number || wbp->cl_scmap)
1908 cluster_push_EOF(vp, newEOF);
1909
1910 wbp->cl_hasbeenpaged = 0;
1911 }
1912 }
1913 if (uio) {
1914 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 40)) | DBG_FUNC_START,
1915 (int)uio->uio_offset, uio_resid(uio), (int)oldEOF, (int)newEOF, 0);
1916
1917 // LP64todo - fix this
1918 io_resid = uio_resid(uio);
1919 } else {
1920 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 40)) | DBG_FUNC_START,
1921 0, 0, (int)oldEOF, (int)newEOF, 0);
1922
1923 io_resid = 0;
1924 }
1925 zero_cnt = 0;
1926 zero_cnt1 = 0;
1927 zero_off = 0;
1928 zero_off1 = 0;
1929
1930 if (flags & IO_HEADZEROFILL) {
1931 /*
1932 * some filesystems (HFS is one) don't support unallocated holes within a file...
1933 * so we zero fill the intervening space between the old EOF and the offset
1934 * where the next chunk of real data begins.... ftruncate will also use this
1935 * routine to zero fill to the new EOF when growing a file... in this case, the
1936 * uio structure will not be provided
1937 */
1938 if (uio) {
1939 if (headOff < uio->uio_offset) {
1940 zero_cnt = uio->uio_offset - headOff;
1941 zero_off = headOff;
1942 }
1943 } else if (headOff < newEOF) {
1944 zero_cnt = newEOF - headOff;
1945 zero_off = headOff;
1946 }
1947 }
1948 if (flags & IO_TAILZEROFILL) {
1949 if (uio) {
1950 // LP64todo - fix this
1951 zero_off1 = uio->uio_offset + uio_resid(uio);
1952
1953 if (zero_off1 < tailOff)
1954 zero_cnt1 = tailOff - zero_off1;
1955 }
1956 }
1957 if (zero_cnt == 0 && uio == (struct uio *) 0) {
1958 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 40)) | DBG_FUNC_END,
1959 retval, 0, 0, 0, 0);
1960 return (0);
1961 }
1962
1963 while ((total_size = (io_resid + zero_cnt + zero_cnt1)) && retval == 0) {
1964 /*
1965 * for this iteration of the loop, figure out where our starting point is
1966 */
1967 if (zero_cnt) {
1968 start_offset = (int)(zero_off & PAGE_MASK_64);
1969 upl_f_offset = zero_off - start_offset;
1970 } else if (io_resid) {
1971 start_offset = (int)(uio->uio_offset & PAGE_MASK_64);
1972 upl_f_offset = uio->uio_offset - start_offset;
1973 } else {
1974 start_offset = (int)(zero_off1 & PAGE_MASK_64);
1975 upl_f_offset = zero_off1 - start_offset;
1976 }
1977 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 46)) | DBG_FUNC_NONE,
1978 (int)zero_off, (int)zero_cnt, (int)zero_off1, (int)zero_cnt1, 0);
1979
1980 if (total_size > (MAX_UPL_TRANSFER * PAGE_SIZE))
1981 total_size = MAX_UPL_TRANSFER * PAGE_SIZE;
1982
1983 cl.b_addr = (daddr64_t)(upl_f_offset / PAGE_SIZE_64);
1984
1985 if (uio && ((flags & (IO_NOCACHE | IO_SYNC | IO_HEADZEROFILL | IO_TAILZEROFILL)) == 0)) {
1986 /*
1987 * assumption... total_size <= io_resid
1988 * because IO_HEADZEROFILL and IO_TAILZEROFILL not set
1989 */
1990 if ((start_offset + total_size) > (MAX_UPL_TRANSFER * PAGE_SIZE))
1991 total_size -= start_offset;
1992 xfer_resid = total_size;
1993
1994 retval = cluster_copy_ubc_data(vp, uio, &xfer_resid, 1);
1995
1996 if (retval)
1997 break;
1998
1999 io_resid -= (total_size - xfer_resid);
2000 total_size = xfer_resid;
2001 start_offset = (int)(uio->uio_offset & PAGE_MASK_64);
2002 upl_f_offset = uio->uio_offset - start_offset;
2003
2004 if (total_size == 0) {
2005 if (start_offset) {
2006 /*
2007 * the write did not finish on a page boundary
2008 * which will leave upl_f_offset pointing to the
2009 * beginning of the last page written instead of
2010 * the page beyond it... bump it in this case
2011 * so that the cluster code records the last page
2012 * written as dirty
2013 */
2014 upl_f_offset += PAGE_SIZE_64;
2015 }
2016 upl_size = 0;
2017
2018 goto check_cluster;
2019 }
2020 }
2021 /*
2022 * compute the size of the upl needed to encompass
2023 * the requested write... limit each call to cluster_io
2024 * to the maximum UPL size... cluster_io will clip if
2025 * this exceeds the maximum io_size for the device,
2026 * make sure to account for
2027 * a starting offset that's not page aligned
2028 */
2029 upl_size = (start_offset + total_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
2030
2031 if (upl_size > (MAX_UPL_TRANSFER * PAGE_SIZE))
2032 upl_size = MAX_UPL_TRANSFER * PAGE_SIZE;
2033
2034 pages_in_upl = upl_size / PAGE_SIZE;
2035 io_size = upl_size - start_offset;
2036
2037 if ((long long)io_size > total_size)
2038 io_size = total_size;
2039
2040 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 41)) | DBG_FUNC_START, upl_size, io_size, total_size, 0, 0);
2041
2042
2043 /*
2044 * Gather the pages from the buffer cache.
2045 * The UPL_WILL_MODIFY flag lets the UPL subsystem know
2046 * that we intend to modify these pages.
2047 */
2048 kret = ubc_create_upl(vp,
2049 upl_f_offset,
2050 upl_size,
2051 &upl,
2052 &pl,
2053 UPL_SET_LITE | UPL_WILL_MODIFY);
2054 if (kret != KERN_SUCCESS)
2055 panic("cluster_write: failed to get pagelist");
2056
2057 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 41)) | DBG_FUNC_END,
2058 (int)upl, (int)upl_f_offset, start_offset, 0, 0);
2059
2060 if (start_offset && !upl_valid_page(pl, 0)) {
2061 int read_size;
2062
2063 /*
2064 * we're starting in the middle of the first page of the upl
2065 * and the page isn't currently valid, so we're going to have
2066 * to read it in first... this is a synchronous operation
2067 */
2068 read_size = PAGE_SIZE;
2069
2070 if ((upl_f_offset + read_size) > newEOF)
2071 read_size = newEOF - upl_f_offset;
2072
2073 retval = cluster_io(vp, upl, 0, upl_f_offset, read_size,
2074 CL_READ, (buf_t)NULL, (struct clios *)NULL);
2075 if (retval) {
2076 /*
2077 * we had an error during the read which causes us to abort
2078 * the current cluster_write request... before we do, we need
2079 * to release the rest of the pages in the upl without modifying
2080 * there state and mark the failed page in error
2081 */
2082 ubc_upl_abort_range(upl, 0, PAGE_SIZE, UPL_ABORT_DUMP_PAGES);
2083
2084 if (upl_size > PAGE_SIZE)
2085 ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_FREE_ON_EMPTY);
2086
2087 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 45)) | DBG_FUNC_NONE,
2088 (int)upl, 0, 0, retval, 0);
2089 break;
2090 }
2091 }
2092 if ((start_offset == 0 || upl_size > PAGE_SIZE) && ((start_offset + io_size) & PAGE_MASK)) {
2093 /*
2094 * the last offset we're writing to in this upl does not end on a page
2095 * boundary... if it's not beyond the old EOF, then we'll also need to
2096 * pre-read this page in if it isn't already valid
2097 */
2098 upl_offset = upl_size - PAGE_SIZE;
2099
2100 if ((upl_f_offset + start_offset + io_size) < oldEOF &&
2101 !upl_valid_page(pl, upl_offset / PAGE_SIZE)) {
2102 int read_size;
2103
2104 read_size = PAGE_SIZE;
2105
2106 if ((upl_f_offset + upl_offset + read_size) > newEOF)
2107 read_size = newEOF - (upl_f_offset + upl_offset);
2108
2109 retval = cluster_io(vp, upl, upl_offset, upl_f_offset + upl_offset, read_size,
2110 CL_READ, (buf_t)NULL, (struct clios *)NULL);
2111 if (retval) {
2112 /*
2113 * we had an error during the read which causes us to abort
2114 * the current cluster_write request... before we do, we
2115 * need to release the rest of the pages in the upl without
2116 * modifying there state and mark the failed page in error
2117 */
2118 ubc_upl_abort_range(upl, upl_offset, PAGE_SIZE, UPL_ABORT_DUMP_PAGES);
2119
2120 if (upl_size > PAGE_SIZE)
2121 ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_FREE_ON_EMPTY);
2122
2123 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 45)) | DBG_FUNC_NONE,
2124 (int)upl, 0, 0, retval, 0);
2125 break;
2126 }
2127 }
2128 }
2129 xfer_resid = io_size;
2130 io_offset = start_offset;
2131
2132 while (zero_cnt && xfer_resid) {
2133
2134 if (zero_cnt < (long long)xfer_resid)
2135 bytes_to_zero = zero_cnt;
2136 else
2137 bytes_to_zero = xfer_resid;
2138
2139 if ( !(flags & (IO_NOZEROVALID | IO_NOZERODIRTY))) {
2140 cluster_zero(upl, io_offset, bytes_to_zero, NULL);
2141 } else {
2142 int zero_pg_index;
2143
2144 bytes_to_zero = min(bytes_to_zero, PAGE_SIZE - (int)(zero_off & PAGE_MASK_64));
2145 zero_pg_index = (int)((zero_off - upl_f_offset) / PAGE_SIZE_64);
2146
2147 if ( !upl_valid_page(pl, zero_pg_index)) {
2148 cluster_zero(upl, io_offset, bytes_to_zero, NULL);
2149
2150 } else if ((flags & (IO_NOZERODIRTY | IO_NOZEROVALID)) == IO_NOZERODIRTY &&
2151 !upl_dirty_page(pl, zero_pg_index)) {
2152 cluster_zero(upl, io_offset, bytes_to_zero, NULL);
2153 }
2154 }
2155 xfer_resid -= bytes_to_zero;
2156 zero_cnt -= bytes_to_zero;
2157 zero_off += bytes_to_zero;
2158 io_offset += bytes_to_zero;
2159 }
2160 if (xfer_resid && io_resid) {
2161 bytes_to_move = min(io_resid, xfer_resid);
2162
2163 retval = cluster_copy_upl_data(uio, upl, io_offset, bytes_to_move);
2164
2165 if (retval) {
2166
2167 ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_DUMP_PAGES | UPL_ABORT_FREE_ON_EMPTY);
2168
2169 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 45)) | DBG_FUNC_NONE,
2170 (int)upl, 0, 0, retval, 0);
2171 } else {
2172 io_resid -= bytes_to_move;
2173 xfer_resid -= bytes_to_move;
2174 io_offset += bytes_to_move;
2175 }
2176 }
2177 while (xfer_resid && zero_cnt1 && retval == 0) {
2178
2179 if (zero_cnt1 < (long long)xfer_resid)
2180 bytes_to_zero = zero_cnt1;
2181 else
2182 bytes_to_zero = xfer_resid;
2183
2184 if ( !(flags & (IO_NOZEROVALID | IO_NOZERODIRTY))) {
2185 cluster_zero(upl, io_offset, bytes_to_zero, NULL);
2186 } else {
2187 int zero_pg_index;
2188
2189 bytes_to_zero = min(bytes_to_zero, PAGE_SIZE - (int)(zero_off1 & PAGE_MASK_64));
2190 zero_pg_index = (int)((zero_off1 - upl_f_offset) / PAGE_SIZE_64);
2191
2192 if ( !upl_valid_page(pl, zero_pg_index)) {
2193 cluster_zero(upl, io_offset, bytes_to_zero, NULL);
2194 } else if ((flags & (IO_NOZERODIRTY | IO_NOZEROVALID)) == IO_NOZERODIRTY &&
2195 !upl_dirty_page(pl, zero_pg_index)) {
2196 cluster_zero(upl, io_offset, bytes_to_zero, NULL);
2197 }
2198 }
2199 xfer_resid -= bytes_to_zero;
2200 zero_cnt1 -= bytes_to_zero;
2201 zero_off1 += bytes_to_zero;
2202 io_offset += bytes_to_zero;
2203 }
2204
2205 if (retval == 0) {
2206 int cl_index;
2207 int can_delay;
2208
2209 io_size += start_offset;
2210
2211 if ((upl_f_offset + io_size) >= newEOF && io_size < upl_size) {
2212 /*
2213 * if we're extending the file with this write
2214 * we'll zero fill the rest of the page so that
2215 * if the file gets extended again in such a way as to leave a
2216 * hole starting at this EOF, we'll have zero's in the correct spot
2217 */
2218 cluster_zero(upl, io_size, upl_size - io_size, NULL);
2219 }
2220 if (flags & IO_SYNC)
2221 /*
2222 * if the IO_SYNC flag is set than we need to
2223 * bypass any clusters and immediately issue
2224 * the I/O
2225 */
2226 goto issue_io;
2227 check_cluster:
2228 /*
2229 * take the lock to protect our accesses
2230 * of the writebehind and sparse cluster state
2231 */
2232 wbp = cluster_get_wbp(vp, CLW_ALLOCATE | CLW_RETURNLOCKED);
2233
2234 /*
2235 * calculate the last logical block number
2236 * that this delayed I/O encompassed
2237 */
2238 cl.e_addr = (daddr64_t)((upl_f_offset + (off_t)upl_size) / PAGE_SIZE_64);
2239
2240 if (wbp->cl_scmap) {
2241
2242 if ( !(flags & IO_NOCACHE)) {
2243 /*
2244 * we've fallen into the sparse
2245 * cluster method of delaying dirty pages
2246 * first, we need to release the upl if we hold one
2247 * since pages in it may be present in the sparse cluster map
2248 * and may span 2 separate buckets there... if they do and
2249 * we happen to have to flush a bucket to make room and it intersects
2250 * this upl, a deadlock may result on page BUSY
2251 */
2252 if (upl_size)
2253 ubc_upl_commit_range(upl, 0, upl_size,
2254 UPL_COMMIT_SET_DIRTY | UPL_COMMIT_INACTIVATE | UPL_COMMIT_FREE_ON_EMPTY);
2255
2256 sparse_cluster_add(wbp, vp, &cl, newEOF);
2257
2258 lck_mtx_unlock(&wbp->cl_lockw);
2259
2260 continue;
2261 }
2262 /*
2263 * must have done cached writes that fell into
2264 * the sparse cluster mechanism... we've switched
2265 * to uncached writes on the file, so go ahead
2266 * and push whatever's in the sparse map
2267 * and switch back to normal clustering
2268 *
2269 * see the comment above concerning a possible deadlock...
2270 */
2271 if (upl_size) {
2272 ubc_upl_commit_range(upl, 0, upl_size,
2273 UPL_COMMIT_SET_DIRTY | UPL_COMMIT_INACTIVATE | UPL_COMMIT_FREE_ON_EMPTY);
2274 /*
2275 * setting upl_size to 0 keeps us from committing a
2276 * second time in the start_new_cluster path
2277 */
2278 upl_size = 0;
2279 }
2280 sparse_cluster_push(wbp, vp, newEOF, 1);
2281
2282 wbp->cl_number = 0;
2283 /*
2284 * no clusters of either type present at this point
2285 * so just go directly to start_new_cluster since
2286 * we know we need to delay this I/O since we've
2287 * already released the pages back into the cache
2288 * to avoid the deadlock with sparse_cluster_push
2289 */
2290 goto start_new_cluster;
2291 }
2292 upl_offset = 0;
2293
2294 if (wbp->cl_number == 0)
2295 /*
2296 * no clusters currently present
2297 */
2298 goto start_new_cluster;
2299
2300 for (cl_index = 0; cl_index < wbp->cl_number; cl_index++) {
2301 /*
2302 * check each cluster that we currently hold
2303 * try to merge some or all of this write into
2304 * one or more of the existing clusters... if
2305 * any portion of the write remains, start a
2306 * new cluster
2307 */
2308 if (cl.b_addr >= wbp->cl_clusters[cl_index].b_addr) {
2309 /*
2310 * the current write starts at or after the current cluster
2311 */
2312 if (cl.e_addr <= (wbp->cl_clusters[cl_index].b_addr + MAX_UPL_TRANSFER)) {
2313 /*
2314 * we have a write that fits entirely
2315 * within the existing cluster limits
2316 */
2317 if (cl.e_addr > wbp->cl_clusters[cl_index].e_addr)
2318 /*
2319 * update our idea of where the cluster ends
2320 */
2321 wbp->cl_clusters[cl_index].e_addr = cl.e_addr;
2322 break;
2323 }
2324 if (cl.b_addr < (wbp->cl_clusters[cl_index].b_addr + MAX_UPL_TRANSFER)) {
2325 /*
2326 * we have a write that starts in the middle of the current cluster
2327 * but extends beyond the cluster's limit... we know this because
2328 * of the previous checks
2329 * we'll extend the current cluster to the max
2330 * and update the b_addr for the current write to reflect that
2331 * the head of it was absorbed into this cluster...
2332 * note that we'll always have a leftover tail in this case since
2333 * full absorbtion would have occurred in the clause above
2334 */
2335 wbp->cl_clusters[cl_index].e_addr = wbp->cl_clusters[cl_index].b_addr + MAX_UPL_TRANSFER;
2336
2337 if (upl_size) {
2338 daddr64_t start_pg_in_upl;
2339
2340 start_pg_in_upl = (daddr64_t)(upl_f_offset / PAGE_SIZE_64);
2341
2342 if (start_pg_in_upl < wbp->cl_clusters[cl_index].e_addr) {
2343 intersection = (int)((wbp->cl_clusters[cl_index].e_addr - start_pg_in_upl) * PAGE_SIZE);
2344
2345 ubc_upl_commit_range(upl, upl_offset, intersection,
2346 UPL_COMMIT_SET_DIRTY | UPL_COMMIT_INACTIVATE | UPL_COMMIT_FREE_ON_EMPTY);
2347 upl_f_offset += intersection;
2348 upl_offset += intersection;
2349 upl_size -= intersection;
2350 }
2351 }
2352 cl.b_addr = wbp->cl_clusters[cl_index].e_addr;
2353 }
2354 /*
2355 * we come here for the case where the current write starts
2356 * beyond the limit of the existing cluster or we have a leftover
2357 * tail after a partial absorbtion
2358 *
2359 * in either case, we'll check the remaining clusters before
2360 * starting a new one
2361 */
2362 } else {
2363 /*
2364 * the current write starts in front of the cluster we're currently considering
2365 */
2366 if ((wbp->cl_clusters[cl_index].e_addr - cl.b_addr) <= MAX_UPL_TRANSFER) {
2367 /*
2368 * we can just merge the new request into
2369 * this cluster and leave it in the cache
2370 * since the resulting cluster is still
2371 * less than the maximum allowable size
2372 */
2373 wbp->cl_clusters[cl_index].b_addr = cl.b_addr;
2374
2375 if (cl.e_addr > wbp->cl_clusters[cl_index].e_addr) {
2376 /*
2377 * the current write completely
2378 * envelops the existing cluster and since
2379 * each write is limited to at most MAX_UPL_TRANSFER bytes
2380 * we can just use the start and last blocknos of the write
2381 * to generate the cluster limits
2382 */
2383 wbp->cl_clusters[cl_index].e_addr = cl.e_addr;
2384 }
2385 break;
2386 }
2387
2388 /*
2389 * if we were to combine this write with the current cluster
2390 * we would exceed the cluster size limit.... so,
2391 * let's see if there's any overlap of the new I/O with
2392 * the cluster we're currently considering... in fact, we'll
2393 * stretch the cluster out to it's full limit and see if we
2394 * get an intersection with the current write
2395 *
2396 */
2397 if (cl.e_addr > wbp->cl_clusters[cl_index].e_addr - MAX_UPL_TRANSFER) {
2398 /*
2399 * the current write extends into the proposed cluster
2400 * clip the length of the current write after first combining it's
2401 * tail with the newly shaped cluster
2402 */
2403 wbp->cl_clusters[cl_index].b_addr = wbp->cl_clusters[cl_index].e_addr - MAX_UPL_TRANSFER;
2404
2405 if (upl_size) {
2406 intersection = (int)((cl.e_addr - wbp->cl_clusters[cl_index].b_addr) * PAGE_SIZE);
2407
2408 if (intersection > upl_size)
2409 /*
2410 * because the current write may consist of a number of pages found in the cache
2411 * which are not part of the UPL, we may have an intersection that exceeds
2412 * the size of the UPL that is also part of this write
2413 */
2414 intersection = upl_size;
2415
2416 ubc_upl_commit_range(upl, upl_offset + (upl_size - intersection), intersection,
2417 UPL_COMMIT_SET_DIRTY | UPL_COMMIT_INACTIVATE | UPL_COMMIT_FREE_ON_EMPTY);
2418 upl_size -= intersection;
2419 }
2420 cl.e_addr = wbp->cl_clusters[cl_index].b_addr;
2421 }
2422 /*
2423 * if we get here, there was no way to merge
2424 * any portion of this write with this cluster
2425 * or we could only merge part of it which
2426 * will leave a tail...
2427 * we'll check the remaining clusters before starting a new one
2428 */
2429 }
2430 }
2431 if (cl_index < wbp->cl_number)
2432 /*
2433 * we found an existing cluster(s) that we
2434 * could entirely merge this I/O into
2435 */
2436 goto delay_io;
2437
2438 if (wbp->cl_number < MAX_CLUSTERS && !(flags & IO_NOCACHE))
2439 /*
2440 * we didn't find an existing cluster to
2441 * merge into, but there's room to start
2442 * a new one
2443 */
2444 goto start_new_cluster;
2445
2446 /*
2447 * no exisitng cluster to merge with and no
2448 * room to start a new one... we'll try
2449 * pushing one of the existing ones... if none of
2450 * them are able to be pushed, we'll switch
2451 * to the sparse cluster mechanism
2452 * cluster_try_push updates cl_number to the
2453 * number of remaining clusters... and
2454 * returns the number of currently unused clusters
2455 */
2456 int ret_cluster_try_push = 0;
2457 /* if writes are not deferred, call cluster push immediately */
2458 if (!((unsigned int)vfs_flags(vp->v_mount) & MNT_DEFWRITE)) {
2459 if (flags & IO_NOCACHE)
2460 can_delay = 0;
2461 else
2462 can_delay = 1;
2463
2464 ret_cluster_try_push = cluster_try_push(wbp, vp, newEOF, can_delay, 0);
2465 }
2466
2467 /* execute following regardless writes are deferred or not */
2468 if (ret_cluster_try_push == 0) {
2469 /*
2470 * no more room in the normal cluster mechanism
2471 * so let's switch to the more expansive but expensive
2472 * sparse mechanism....
2473 * first, we need to release the upl if we hold one
2474 * since pages in it may be present in the sparse cluster map (after the cluster_switch)
2475 * and may span 2 separate buckets there... if they do and
2476 * we happen to have to flush a bucket to make room and it intersects
2477 * this upl, a deadlock may result on page BUSY
2478 */
2479 if (upl_size)
2480 ubc_upl_commit_range(upl, upl_offset, upl_size,
2481 UPL_COMMIT_SET_DIRTY | UPL_COMMIT_INACTIVATE | UPL_COMMIT_FREE_ON_EMPTY);
2482
2483 sparse_cluster_switch(wbp, vp, newEOF);
2484 sparse_cluster_add(wbp, vp, &cl, newEOF);
2485
2486 lck_mtx_unlock(&wbp->cl_lockw);
2487
2488 continue;
2489 }
2490 /*
2491 * we pushed one cluster successfully, so we must be sequentially writing this file
2492 * otherwise, we would have failed and fallen into the sparse cluster support
2493 * so let's take the opportunity to push out additional clusters as long as we
2494 * remain below the throttle... this will give us better I/O locality if we're
2495 * in a copy loop (i.e. we won't jump back and forth between the read and write points
2496 * however, we don't want to push so much out that the write throttle kicks in and
2497 * hangs this thread up until some of the I/O completes...
2498 */
2499 if (!((unsigned int)vfs_flags(vp->v_mount) & MNT_DEFWRITE)) {
2500 while (wbp->cl_number && (vp->v_numoutput <= (VNODE_ASYNC_THROTTLE / 2)))
2501 cluster_try_push(wbp, vp, newEOF, 0, 0);
2502 }
2503
2504 start_new_cluster:
2505 wbp->cl_clusters[wbp->cl_number].b_addr = cl.b_addr;
2506 wbp->cl_clusters[wbp->cl_number].e_addr = cl.e_addr;
2507
2508 if (flags & IO_NOCACHE)
2509 wbp->cl_clusters[wbp->cl_number].io_nocache = 1;
2510 else
2511 wbp->cl_clusters[wbp->cl_number].io_nocache = 0;
2512 wbp->cl_number++;
2513 delay_io:
2514 if (upl_size)
2515 ubc_upl_commit_range(upl, upl_offset, upl_size,
2516 UPL_COMMIT_SET_DIRTY | UPL_COMMIT_INACTIVATE | UPL_COMMIT_FREE_ON_EMPTY);
2517
2518 lck_mtx_unlock(&wbp->cl_lockw);
2519
2520 continue;
2521 issue_io:
2522 /*
2523 * we don't hold the vnode lock at this point
2524 *
2525 * because we had to ask for a UPL that provides currenty non-present pages, the
2526 * UPL has been automatically set to clear the dirty flags (both software and hardware)
2527 * upon committing it... this is not the behavior we want since it's possible for
2528 * pages currently present as part of a mapped file to be dirtied while the I/O is in flight.
2529 * in order to maintain some semblance of coherency with mapped writes
2530 * we need to drop the current upl and pick it back up with COPYOUT_FROM set
2531 * so that we correctly deal with a change in state of the hardware modify bit...
2532 * we do this via cluster_push_x... by passing along the IO_SYNC flag, we force
2533 * cluster_push_x to wait until all the I/Os have completed... cluster_push_x is also
2534 * responsible for generating the correct sized I/O(s)
2535 */
2536 ubc_upl_commit_range(upl, 0, upl_size,
2537 UPL_COMMIT_SET_DIRTY | UPL_COMMIT_INACTIVATE | UPL_COMMIT_FREE_ON_EMPTY);
2538
2539 cl.e_addr = (upl_f_offset + (off_t)upl_size) / PAGE_SIZE_64;
2540
2541 retval = cluster_push_x(vp, &cl, newEOF, flags);
2542 }
2543 }
2544 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 40)) | DBG_FUNC_END,
2545 retval, 0, io_resid, 0, 0);
2546
2547 return (retval);
2548 }
2549
2550 int
2551 cluster_read(vnode_t vp, struct uio *uio, off_t filesize, int xflags)
2552 {
2553 int prev_resid;
2554 u_int clip_size;
2555 off_t max_io_size;
2556 int upl_size;
2557 int upl_flags;
2558 upl_t upl;
2559 int retval = 0;
2560 int flags;
2561
2562 flags = xflags;
2563
2564 if (vp->v_flag & VNOCACHE_DATA)
2565 flags |= IO_NOCACHE;
2566 if (vp->v_flag & VRAOFF)
2567 flags |= IO_RAOFF;
2568
2569 if (!((flags & IO_NOCACHE) && UIO_SEG_IS_USER_SPACE(uio->uio_segflg))) {
2570 /*
2571 * go do a read through the cache if one of the following is true....
2572 * NOCACHE is not true
2573 * the uio request doesn't target USERSPACE
2574 */
2575 return (cluster_read_x(vp, uio, filesize, flags));
2576 }
2577
2578 #if LP64_DEBUG
2579 if (IS_VALID_UIO_SEGFLG(uio->uio_segflg) == 0) {
2580 panic("%s :%d - invalid uio_segflg\n", __FILE__, __LINE__);
2581 }
2582 #endif /* LP64_DEBUG */
2583
2584 while (uio_resid(uio) && uio->uio_offset < filesize && retval == 0) {
2585 user_size_t iov_len;
2586 user_addr_t iov_base;
2587
2588 /*
2589 * we know we have a resid, so this is safe
2590 * skip over any emtpy vectors
2591 */
2592 uio_update(uio, (user_size_t)0);
2593
2594 iov_len = uio_curriovlen(uio);
2595 iov_base = uio_curriovbase(uio);
2596
2597 upl_size = PAGE_SIZE;
2598 upl_flags = UPL_QUERY_OBJECT_TYPE;
2599
2600 // LP64todo - fix this!
2601 if ((vm_map_get_upl(current_map(),
2602 (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)),
2603 &upl_size, &upl, NULL, NULL, &upl_flags, 0)) != KERN_SUCCESS) {
2604 /*
2605 * the user app must have passed in an invalid address
2606 */
2607 return (EFAULT);
2608 }
2609
2610 /*
2611 * We check every vector target but if it is physically
2612 * contiguous space, we skip the sanity checks.
2613 */
2614 if (upl_flags & UPL_PHYS_CONTIG) {
2615 retval = cluster_phys_read(vp, uio, filesize);
2616 }
2617 else if (uio_resid(uio) < PAGE_SIZE) {
2618 /*
2619 * we're here because we're don't have a physically contiguous target buffer
2620 * go do a read through the cache if
2621 * the total xfer size is less than a page...
2622 */
2623 return (cluster_read_x(vp, uio, filesize, flags));
2624 }
2625 // LP64todo - fix this!
2626 else if (((int)uio->uio_offset & PAGE_MASK) || (CAST_DOWN(int, iov_base) & PAGE_MASK)) {
2627 if (((int)uio->uio_offset & PAGE_MASK) == (CAST_DOWN(int, iov_base) & PAGE_MASK)) {
2628 /*
2629 * Bring the file offset read up to a pagesize boundary
2630 * this will also bring the base address to a page boundary
2631 * since they both are currently on the same offset within a page
2632 * note: if we get here, uio->uio_resid is greater than PAGE_SIZE
2633 * so the computed clip_size must always be less than the current uio_resid
2634 */
2635 clip_size = (PAGE_SIZE - (int)(uio->uio_offset & PAGE_MASK_64));
2636
2637 /*
2638 * Fake the resid going into the cluster_read_x call
2639 * and restore it on the way out.
2640 */
2641 prev_resid = uio_resid(uio);
2642 // LP64todo - fix this
2643 uio_setresid(uio, clip_size);
2644
2645 retval = cluster_read_x(vp, uio, filesize, flags);
2646
2647 uio_setresid(uio, prev_resid - (clip_size - uio_resid(uio)));
2648 } else {
2649 /*
2650 * can't get both the file offset and the buffer offset aligned to a page boundary
2651 * so fire an I/O through the cache for this entire vector
2652 */
2653 // LP64todo - fix this!
2654 clip_size = iov_len;
2655 prev_resid = uio_resid(uio);
2656 uio_setresid(uio, clip_size);
2657
2658 retval = cluster_read_x(vp, uio, filesize, flags);
2659
2660 uio_setresid(uio, prev_resid - (clip_size - uio_resid(uio)));
2661 }
2662 } else {
2663 /*
2664 * If we come in here, we know the offset into
2665 * the file is on a pagesize boundary
2666 */
2667 max_io_size = filesize - uio->uio_offset;
2668 // LP64todo - fix this
2669 clip_size = uio_resid(uio);
2670 if (iov_len < clip_size)
2671 clip_size = iov_len;
2672 if (max_io_size < clip_size)
2673 clip_size = (int)max_io_size;
2674
2675 if (clip_size < PAGE_SIZE) {
2676 /*
2677 * Take care of the tail end of the read in this vector.
2678 */
2679 // LP64todo - fix this
2680 prev_resid = uio_resid(uio);
2681 uio_setresid(uio, clip_size);
2682
2683 retval = cluster_read_x(vp, uio, filesize, flags);
2684
2685 uio_setresid(uio, prev_resid - (clip_size - uio_resid(uio)));
2686 } else {
2687 /* round clip_size down to a multiple of pagesize */
2688 clip_size = clip_size & ~(PAGE_MASK);
2689 // LP64todo - fix this
2690 prev_resid = uio_resid(uio);
2691 uio_setresid(uio, clip_size);
2692
2693 retval = cluster_nocopy_read(vp, uio, filesize);
2694
2695 if ((retval==0) && uio_resid(uio))
2696 retval = cluster_read_x(vp, uio, filesize, flags);
2697
2698 uio_setresid(uio, prev_resid - (clip_size - uio_resid(uio)));
2699 }
2700 } /* end else */
2701 } /* end while */
2702
2703 return(retval);
2704 }
2705
2706 static int
2707 cluster_read_x(vnode_t vp, struct uio *uio, off_t filesize, int flags)
2708 {
2709 upl_page_info_t *pl;
2710 upl_t upl;
2711 vm_offset_t upl_offset;
2712 int upl_size;
2713 off_t upl_f_offset;
2714 int start_offset;
2715 int start_pg;
2716 int last_pg;
2717 int uio_last = 0;
2718 int pages_in_upl;
2719 off_t max_size;
2720 off_t last_ioread_offset;
2721 off_t last_request_offset;
2722 u_int size_of_prefetch;
2723 u_int io_size;
2724 kern_return_t kret;
2725 int error = 0;
2726 int retval = 0;
2727 u_int max_rd_size = MAX_UPL_TRANSFER * PAGE_SIZE;
2728 u_int rd_ahead_enabled = 1;
2729 u_int prefetch_enabled = 1;
2730 struct cl_readahead * rap;
2731 struct clios iostate;
2732 struct cl_extent extent;
2733
2734 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 32)) | DBG_FUNC_START,
2735 (int)uio->uio_offset, uio_resid(uio), (int)filesize, 0, 0);
2736
2737 // LP64todo - fix this
2738 last_request_offset = uio->uio_offset + uio_resid(uio);
2739
2740 if ((flags & (IO_RAOFF|IO_NOCACHE)) ||
2741 ((last_request_offset & ~PAGE_MASK_64) == (uio->uio_offset & ~PAGE_MASK_64))) {
2742 rd_ahead_enabled = 0;
2743 rap = NULL;
2744 } else {
2745 if (cluster_hard_throttle_on(vp)) {
2746 rd_ahead_enabled = 0;
2747 prefetch_enabled = 0;
2748
2749 max_rd_size = HARD_THROTTLE_MAXSIZE;
2750 }
2751 if ((rap = cluster_get_rap(vp)) == NULL)
2752 rd_ahead_enabled = 0;
2753 }
2754 if (last_request_offset > filesize)
2755 last_request_offset = filesize;
2756 extent.b_addr = uio->uio_offset / PAGE_SIZE_64;
2757 extent.e_addr = (last_request_offset - 1) / PAGE_SIZE_64;
2758
2759 if (rap != NULL && rap->cl_ralen && (rap->cl_lastr == extent.b_addr || (rap->cl_lastr + 1) == extent.b_addr)) {
2760 /*
2761 * determine if we already have a read-ahead in the pipe courtesy of the
2762 * last read systemcall that was issued...
2763 * if so, pick up it's extent to determine where we should start
2764 * with respect to any read-ahead that might be necessary to
2765 * garner all the data needed to complete this read systemcall
2766 */
2767 last_ioread_offset = (rap->cl_maxra * PAGE_SIZE_64) + PAGE_SIZE_64;
2768
2769 if (last_ioread_offset < uio->uio_offset)
2770 last_ioread_offset = (off_t)0;
2771 else if (last_ioread_offset > last_request_offset)
2772 last_ioread_offset = last_request_offset;
2773 } else
2774 last_ioread_offset = (off_t)0;
2775
2776 while (uio_resid(uio) && uio->uio_offset < filesize && retval == 0) {
2777 /*
2778 * compute the size of the upl needed to encompass
2779 * the requested read... limit each call to cluster_io
2780 * to the maximum UPL size... cluster_io will clip if
2781 * this exceeds the maximum io_size for the device,
2782 * make sure to account for
2783 * a starting offset that's not page aligned
2784 */
2785 start_offset = (int)(uio->uio_offset & PAGE_MASK_64);
2786 upl_f_offset = uio->uio_offset - (off_t)start_offset;
2787 max_size = filesize - uio->uio_offset;
2788
2789 // LP64todo - fix this!
2790 if ((off_t)((unsigned int)uio_resid(uio)) < max_size)
2791 io_size = uio_resid(uio);
2792 else
2793 io_size = max_size;
2794
2795 if (!(flags & IO_NOCACHE)) {
2796
2797 while (io_size) {
2798 u_int io_resid;
2799 u_int io_requested;
2800
2801 /*
2802 * if we keep finding the pages we need already in the cache, then
2803 * don't bother to call cluster_rd_prefetch since it costs CPU cycles
2804 * to determine that we have all the pages we need... once we miss in
2805 * the cache and have issued an I/O, than we'll assume that we're likely
2806 * to continue to miss in the cache and it's to our advantage to try and prefetch
2807 */
2808 if (last_request_offset && last_ioread_offset && (size_of_prefetch = (last_request_offset - last_ioread_offset))) {
2809 if ((last_ioread_offset - uio->uio_offset) <= max_rd_size && prefetch_enabled) {
2810 /*
2811 * we've already issued I/O for this request and
2812 * there's still work to do and
2813 * our prefetch stream is running dry, so issue a
2814 * pre-fetch I/O... the I/O latency will overlap
2815 * with the copying of the data
2816 */
2817 if (size_of_prefetch > max_rd_size)
2818 size_of_prefetch = max_rd_size;
2819
2820 size_of_prefetch = cluster_rd_prefetch(vp, last_ioread_offset, size_of_prefetch, filesize);
2821
2822 last_ioread_offset += (off_t)(size_of_prefetch * PAGE_SIZE);
2823
2824 if (last_ioread_offset > last_request_offset)
2825 last_ioread_offset = last_request_offset;
2826 }
2827 }
2828 /*
2829 * limit the size of the copy we're about to do so that
2830 * we can notice that our I/O pipe is running dry and
2831 * get the next I/O issued before it does go dry
2832 */
2833 if (last_ioread_offset && io_size > ((MAX_UPL_TRANSFER * PAGE_SIZE) / 4))
2834 io_resid = ((MAX_UPL_TRANSFER * PAGE_SIZE) / 4);
2835 else
2836 io_resid = io_size;
2837
2838 io_requested = io_resid;
2839
2840 retval = cluster_copy_ubc_data(vp, uio, &io_resid, 0);
2841
2842 io_size -= (io_requested - io_resid);
2843
2844 if (retval || io_resid)
2845 /*
2846 * if we run into a real error or
2847 * a page that is not in the cache
2848 * we need to leave streaming mode
2849 */
2850 break;
2851
2852 if ((io_size == 0 || last_ioread_offset == last_request_offset) && rd_ahead_enabled) {
2853 /*
2854 * we're already finished the I/O for this read request
2855 * let's see if we should do a read-ahead
2856 */
2857 cluster_rd_ahead(vp, &extent, filesize, rap);
2858 }
2859 }
2860 if (retval)
2861 break;
2862 if (io_size == 0) {
2863 if (rap != NULL) {
2864 if (extent.e_addr < rap->cl_lastr)
2865 rap->cl_maxra = 0;
2866 rap->cl_lastr = extent.e_addr;
2867 }
2868 break;
2869 }
2870 start_offset = (int)(uio->uio_offset & PAGE_MASK_64);
2871 upl_f_offset = uio->uio_offset - (off_t)start_offset;
2872 max_size = filesize - uio->uio_offset;
2873 }
2874 if (io_size > max_rd_size)
2875 io_size = max_rd_size;
2876
2877 upl_size = (start_offset + io_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
2878
2879 if (upl_size > (MAX_UPL_TRANSFER * PAGE_SIZE) / 4)
2880 upl_size = (MAX_UPL_TRANSFER * PAGE_SIZE) / 4;
2881 pages_in_upl = upl_size / PAGE_SIZE;
2882
2883 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 33)) | DBG_FUNC_START,
2884 (int)upl, (int)upl_f_offset, upl_size, start_offset, 0);
2885
2886 kret = ubc_create_upl(vp,
2887 upl_f_offset,
2888 upl_size,
2889 &upl,
2890 &pl,
2891 UPL_SET_LITE);
2892 if (kret != KERN_SUCCESS)
2893 panic("cluster_read: failed to get pagelist");
2894
2895 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 33)) | DBG_FUNC_END,
2896 (int)upl, (int)upl_f_offset, upl_size, start_offset, 0);
2897
2898 /*
2899 * scan from the beginning of the upl looking for the first
2900 * non-valid page.... this will become the first page in
2901 * the request we're going to make to 'cluster_io'... if all
2902 * of the pages are valid, we won't call through to 'cluster_io'
2903 */
2904 for (start_pg = 0; start_pg < pages_in_upl; start_pg++) {
2905 if (!upl_valid_page(pl, start_pg))
2906 break;
2907 }
2908
2909 /*
2910 * scan from the starting invalid page looking for a valid
2911 * page before the end of the upl is reached, if we
2912 * find one, then it will be the last page of the request to
2913 * 'cluster_io'
2914 */
2915 for (last_pg = start_pg; last_pg < pages_in_upl; last_pg++) {
2916 if (upl_valid_page(pl, last_pg))
2917 break;
2918 }
2919 iostate.io_completed = 0;
2920 iostate.io_issued = 0;
2921 iostate.io_error = 0;
2922 iostate.io_wanted = 0;
2923
2924 if (start_pg < last_pg) {
2925 /*
2926 * we found a range of 'invalid' pages that must be filled
2927 * if the last page in this range is the last page of the file
2928 * we may have to clip the size of it to keep from reading past
2929 * the end of the last physical block associated with the file
2930 */
2931 upl_offset = start_pg * PAGE_SIZE;
2932 io_size = (last_pg - start_pg) * PAGE_SIZE;
2933
2934 if ((upl_f_offset + upl_offset + io_size) > filesize)
2935 io_size = filesize - (upl_f_offset + upl_offset);
2936
2937 /*
2938 * issue an asynchronous read to cluster_io
2939 */
2940
2941 error = cluster_io(vp, upl, upl_offset, upl_f_offset + upl_offset,
2942 io_size, CL_READ | CL_ASYNC, (buf_t)NULL, &iostate);
2943 }
2944 if (error == 0) {
2945 /*
2946 * if the read completed successfully, or there was no I/O request
2947 * issued, than copy the data into user land via 'cluster_upl_copy_data'
2948 * we'll first add on any 'valid'
2949 * pages that were present in the upl when we acquired it.
2950 */
2951 u_int val_size;
2952
2953 for (uio_last = last_pg; uio_last < pages_in_upl; uio_last++) {
2954 if (!upl_valid_page(pl, uio_last))
2955 break;
2956 }
2957 /*
2958 * compute size to transfer this round, if uio->uio_resid is
2959 * still non-zero after this attempt, we'll loop around and
2960 * set up for another I/O.
2961 */
2962 val_size = (uio_last * PAGE_SIZE) - start_offset;
2963
2964 if (val_size > max_size)
2965 val_size = max_size;
2966
2967 if (val_size > uio_resid(uio))
2968 // LP64todo - fix this
2969 val_size = uio_resid(uio);
2970
2971 if (last_ioread_offset == 0)
2972 last_ioread_offset = uio->uio_offset + val_size;
2973
2974 if ((size_of_prefetch = (last_request_offset - last_ioread_offset)) && prefetch_enabled) {
2975 /*
2976 * if there's still I/O left to do for this request, and...
2977 * we're not in hard throttle mode, then issue a
2978 * pre-fetch I/O... the I/O latency will overlap
2979 * with the copying of the data
2980 */
2981 size_of_prefetch = cluster_rd_prefetch(vp, last_ioread_offset, size_of_prefetch, filesize);
2982
2983 last_ioread_offset += (off_t)(size_of_prefetch * PAGE_SIZE);
2984
2985 if (last_ioread_offset > last_request_offset)
2986 last_ioread_offset = last_request_offset;
2987
2988 } else if ((uio->uio_offset + val_size) == last_request_offset) {
2989 /*
2990 * this transfer will finish this request, so...
2991 * let's try to read ahead if we're in
2992 * a sequential access pattern and we haven't
2993 * explicitly disabled it
2994 */
2995 if (rd_ahead_enabled)
2996 cluster_rd_ahead(vp, &extent, filesize, rap);
2997
2998 if (rap != NULL) {
2999 if (extent.e_addr < rap->cl_lastr)
3000 rap->cl_maxra = 0;
3001 rap->cl_lastr = extent.e_addr;
3002 }
3003 }
3004 lck_mtx_lock(cl_mtxp);
3005
3006 while (iostate.io_issued != iostate.io_completed) {
3007 iostate.io_wanted = 1;
3008 msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_read_x", 0);
3009 }
3010 lck_mtx_unlock(cl_mtxp);
3011
3012 if (iostate.io_error)
3013 error = iostate.io_error;
3014 else
3015 retval = cluster_copy_upl_data(uio, upl, start_offset, val_size);
3016 }
3017 if (start_pg < last_pg) {
3018 /*
3019 * compute the range of pages that we actually issued an I/O for
3020 * and either commit them as valid if the I/O succeeded
3021 * or abort them if the I/O failed
3022 */
3023 io_size = (last_pg - start_pg) * PAGE_SIZE;
3024
3025 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 35)) | DBG_FUNC_START,
3026 (int)upl, start_pg * PAGE_SIZE, io_size, error, 0);
3027
3028 if (error || (flags & IO_NOCACHE))
3029 ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, io_size,
3030 UPL_ABORT_DUMP_PAGES | UPL_ABORT_FREE_ON_EMPTY);
3031 else
3032 ubc_upl_commit_range(upl, start_pg * PAGE_SIZE, io_size,
3033 UPL_COMMIT_CLEAR_DIRTY |
3034 UPL_COMMIT_FREE_ON_EMPTY |
3035 UPL_COMMIT_INACTIVATE);
3036
3037 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 35)) | DBG_FUNC_END,
3038 (int)upl, start_pg * PAGE_SIZE, io_size, error, 0);
3039 }
3040 if ((last_pg - start_pg) < pages_in_upl) {
3041 int cur_pg;
3042 int commit_flags;
3043
3044 /*
3045 * the set of pages that we issued an I/O for did not encompass
3046 * the entire upl... so just release these without modifying
3047 * their state
3048 */
3049 if (error)
3050 ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_FREE_ON_EMPTY);
3051 else {
3052 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 35)) | DBG_FUNC_START,
3053 (int)upl, -1, pages_in_upl - (last_pg - start_pg), 0, 0);
3054
3055 if (start_pg) {
3056 /*
3057 * we found some already valid pages at the beginning of
3058 * the upl commit these back to the inactive list with
3059 * reference cleared
3060 */
3061 for (cur_pg = 0; cur_pg < start_pg; cur_pg++) {
3062 commit_flags = UPL_COMMIT_FREE_ON_EMPTY
3063 | UPL_COMMIT_INACTIVATE;
3064
3065 if (upl_dirty_page(pl, cur_pg))
3066 commit_flags |= UPL_COMMIT_SET_DIRTY;
3067
3068 if ( !(commit_flags & UPL_COMMIT_SET_DIRTY) && (flags & IO_NOCACHE))
3069 ubc_upl_abort_range(upl, cur_pg * PAGE_SIZE, PAGE_SIZE,
3070 UPL_ABORT_DUMP_PAGES | UPL_ABORT_FREE_ON_EMPTY);
3071 else
3072 ubc_upl_commit_range(upl, cur_pg * PAGE_SIZE,
3073 PAGE_SIZE, commit_flags);
3074 }
3075 }
3076 if (last_pg < uio_last) {
3077 /*
3078 * we found some already valid pages immediately after the
3079 * pages we issued I/O for, commit these back to the
3080 * inactive list with reference cleared
3081 */
3082 for (cur_pg = last_pg; cur_pg < uio_last; cur_pg++) {
3083 commit_flags = UPL_COMMIT_FREE_ON_EMPTY
3084 | UPL_COMMIT_INACTIVATE;
3085
3086 if (upl_dirty_page(pl, cur_pg))
3087 commit_flags |= UPL_COMMIT_SET_DIRTY;
3088
3089 if ( !(commit_flags & UPL_COMMIT_SET_DIRTY) && (flags & IO_NOCACHE))
3090 ubc_upl_abort_range(upl, cur_pg * PAGE_SIZE, PAGE_SIZE,
3091 UPL_ABORT_DUMP_PAGES | UPL_ABORT_FREE_ON_EMPTY);
3092 else
3093 ubc_upl_commit_range(upl, cur_pg * PAGE_SIZE,
3094 PAGE_SIZE, commit_flags);
3095 }
3096 }
3097 if (uio_last < pages_in_upl) {
3098 /*
3099 * there were some invalid pages beyond the valid pages
3100 * that we didn't issue an I/O for, just release them
3101 * unchanged
3102 */
3103 ubc_upl_abort_range(upl, uio_last * PAGE_SIZE,
3104 (pages_in_upl - uio_last) * PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
3105 }
3106
3107 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 35)) | DBG_FUNC_END,
3108 (int)upl, -1, -1, 0, 0);
3109 }
3110 }
3111 if (retval == 0)
3112 retval = error;
3113
3114 if ( uio_resid(uio) ) {
3115 if (cluster_hard_throttle_on(vp)) {
3116 rd_ahead_enabled = 0;
3117 prefetch_enabled = 0;
3118
3119 max_rd_size = HARD_THROTTLE_MAXSIZE;
3120 } else {
3121 if (rap != NULL)
3122 rd_ahead_enabled = 1;
3123 prefetch_enabled = 1;
3124
3125 max_rd_size = MAX_UPL_TRANSFER * PAGE_SIZE;
3126 }
3127 }
3128 }
3129 if (rap != NULL) {
3130 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 32)) | DBG_FUNC_END,
3131 (int)uio->uio_offset, uio_resid(uio), rap->cl_lastr, retval, 0);
3132
3133 lck_mtx_unlock(&rap->cl_lockr);
3134 } else {
3135 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 32)) | DBG_FUNC_END,
3136 (int)uio->uio_offset, uio_resid(uio), 0, retval, 0);
3137 }
3138
3139 return (retval);
3140 }
3141
3142
3143 static int
3144 cluster_nocopy_read(vnode_t vp, struct uio *uio, off_t filesize)
3145 {
3146 upl_t upl;
3147 upl_page_info_t *pl;
3148 vm_offset_t upl_offset;
3149 off_t max_io_size;
3150 int io_size;
3151 int upl_size;
3152 int upl_needed_size;
3153 int pages_in_pl;
3154 int upl_flags;
3155 kern_return_t kret;
3156 int i;
3157 int force_data_sync;
3158 int retval = 0;
3159 int no_zero_fill = 0;
3160 int abort_flag = 0;
3161 struct clios iostate;
3162 u_int max_rd_size = MAX_UPL_TRANSFER * PAGE_SIZE;
3163 u_int max_rd_ahead = MAX_UPL_TRANSFER * PAGE_SIZE * 2;
3164
3165
3166 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 70)) | DBG_FUNC_START,
3167 (int)uio->uio_offset, uio_resid(uio), (int)filesize, 0, 0);
3168
3169 /*
3170 * When we enter this routine, we know
3171 * -- the offset into the file is on a pagesize boundary
3172 * -- the resid is a page multiple
3173 * -- the resid will not exceed iov_len
3174 */
3175
3176 iostate.io_completed = 0;
3177 iostate.io_issued = 0;
3178 iostate.io_error = 0;
3179 iostate.io_wanted = 0;
3180
3181 while (uio_resid(uio) && uio->uio_offset < filesize && retval == 0) {
3182 user_addr_t iov_base;
3183
3184 if (cluster_hard_throttle_on(vp)) {
3185 max_rd_size = HARD_THROTTLE_MAXSIZE;
3186 max_rd_ahead = HARD_THROTTLE_MAXSIZE - 1;
3187 } else {
3188 max_rd_size = MAX_UPL_TRANSFER * PAGE_SIZE;
3189 max_rd_ahead = MAX_UPL_TRANSFER * PAGE_SIZE * 8;
3190 }
3191 max_io_size = filesize - uio->uio_offset;
3192
3193 // LP64todo - fix this
3194 if (max_io_size < (off_t)((unsigned int)uio_resid(uio)))
3195 io_size = max_io_size;
3196 else
3197 io_size = uio_resid(uio);
3198
3199 /*
3200 * First look for pages already in the cache
3201 * and move them to user space.
3202 */
3203 retval = cluster_copy_ubc_data(vp, uio, &io_size, 0);
3204
3205 if (retval) {
3206 /*
3207 * we may have already spun some portion of this request
3208 * off as async requests... we need to wait for the I/O
3209 * to complete before returning
3210 */
3211 goto wait_for_reads;
3212 }
3213 /*
3214 * If we are already finished with this read, then return
3215 */
3216 if (io_size == 0) {
3217 /*
3218 * we may have already spun some portion of this request
3219 * off as async requests... we need to wait for the I/O
3220 * to complete before returning
3221 */
3222 goto wait_for_reads;
3223 }
3224 max_io_size = io_size;
3225
3226 if (max_io_size > max_rd_size)
3227 max_io_size = max_rd_size;
3228
3229 io_size = 0;
3230
3231 ubc_range_op(vp, uio->uio_offset, uio->uio_offset + max_io_size, UPL_ROP_ABSENT, &io_size);
3232
3233 if (io_size == 0)
3234 /*
3235 * we may have already spun some portion of this request
3236 * off as async requests... we need to wait for the I/O
3237 * to complete before returning
3238 */
3239 goto wait_for_reads;
3240
3241 iov_base = uio_curriovbase(uio);
3242
3243 // LP64todo - fix this!
3244 upl_offset = CAST_DOWN(vm_offset_t, iov_base) & PAGE_MASK;
3245 upl_needed_size = (upl_offset + io_size + (PAGE_SIZE -1)) & ~PAGE_MASK;
3246
3247 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 72)) | DBG_FUNC_START,
3248 (int)upl_offset, upl_needed_size, (int)iov_base, io_size, 0);
3249
3250 if (upl_offset == 0 && ((io_size & PAGE_MASK) == 0)) {
3251 no_zero_fill = 1;
3252 abort_flag = UPL_ABORT_DUMP_PAGES | UPL_ABORT_FREE_ON_EMPTY;
3253 } else {
3254 no_zero_fill = 0;
3255 abort_flag = UPL_ABORT_FREE_ON_EMPTY;
3256 }
3257 for (force_data_sync = 0; force_data_sync < 3; force_data_sync++) {
3258 pages_in_pl = 0;
3259 upl_size = upl_needed_size;
3260 upl_flags = UPL_FILE_IO | UPL_NO_SYNC | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE;
3261
3262 if (no_zero_fill)
3263 upl_flags |= UPL_NOZEROFILL;
3264 if (force_data_sync)
3265 upl_flags |= UPL_FORCE_DATA_SYNC;
3266
3267 // LP64todo - fix this!
3268 kret = vm_map_create_upl(current_map(),
3269 (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)),
3270 &upl_size, &upl, NULL, &pages_in_pl, &upl_flags);
3271
3272 if (kret != KERN_SUCCESS) {
3273 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 72)) | DBG_FUNC_END,
3274 (int)upl_offset, upl_size, io_size, kret, 0);
3275 /*
3276 * cluster_nocopy_read: failed to get pagelist
3277 *
3278 * we may have already spun some portion of this request
3279 * off as async requests... we need to wait for the I/O
3280 * to complete before returning
3281 */
3282 goto wait_for_reads;
3283 }
3284 pages_in_pl = upl_size / PAGE_SIZE;
3285 pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
3286
3287 for (i = 0; i < pages_in_pl; i++) {
3288 if (!upl_valid_page(pl, i))
3289 break;
3290 }
3291 if (i == pages_in_pl)
3292 break;
3293
3294 ubc_upl_abort_range(upl, (upl_offset & ~PAGE_MASK), upl_size, abort_flag);
3295 }
3296 if (force_data_sync >= 3) {
3297 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 72)) | DBG_FUNC_END,
3298 (int)upl_offset, upl_size, io_size, kret, 0);
3299
3300 goto wait_for_reads;
3301 }
3302 /*
3303 * Consider the possibility that upl_size wasn't satisfied.
3304 */
3305 if (upl_size != upl_needed_size)
3306 io_size = (upl_size - (int)upl_offset) & ~PAGE_MASK;
3307
3308 if (io_size == 0) {
3309 ubc_upl_abort_range(upl, (upl_offset & ~PAGE_MASK), upl_size, abort_flag);
3310 goto wait_for_reads;
3311 }
3312 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 72)) | DBG_FUNC_END,
3313 (int)upl_offset, upl_size, io_size, kret, 0);
3314
3315 /*
3316 * request asynchronously so that we can overlap
3317 * the preparation of the next I/O
3318 * if there are already too many outstanding reads
3319 * wait until some have completed before issuing the next read
3320 */
3321 lck_mtx_lock(cl_mtxp);
3322
3323 while ((iostate.io_issued - iostate.io_completed) > max_rd_ahead) {
3324 iostate.io_wanted = 1;
3325 msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_nocopy_read", 0);
3326 }
3327 lck_mtx_unlock(cl_mtxp);
3328
3329 if (iostate.io_error) {
3330 /*
3331 * one of the earlier reads we issued ran into a hard error
3332 * don't issue any more reads, cleanup the UPL
3333 * that was just created but not used, then
3334 * go wait for any other reads to complete before
3335 * returning the error to the caller
3336 */
3337 ubc_upl_abort_range(upl, (upl_offset & ~PAGE_MASK), upl_size, abort_flag);
3338
3339 goto wait_for_reads;
3340 }
3341 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 73)) | DBG_FUNC_START,
3342 (int)upl, (int)upl_offset, (int)uio->uio_offset, io_size, 0);
3343
3344 retval = cluster_io(vp, upl, upl_offset, uio->uio_offset, io_size,
3345 CL_PRESERVE | CL_COMMIT | CL_READ | CL_ASYNC | CL_NOZERO,
3346 (buf_t)NULL, &iostate);
3347
3348 /*
3349 * update the uio structure
3350 */
3351 uio_update(uio, (user_size_t)io_size);
3352
3353 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 73)) | DBG_FUNC_END,
3354 (int)upl, (int)uio->uio_offset, (int)uio_resid(uio), retval, 0);
3355
3356 } /* end while */
3357
3358 wait_for_reads:
3359 /*
3360 * make sure all async reads that are part of this stream
3361 * have completed before we return
3362 */
3363 lck_mtx_lock(cl_mtxp);
3364
3365 while (iostate.io_issued != iostate.io_completed) {
3366 iostate.io_wanted = 1;
3367 msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_nocopy_read", 0);
3368 }
3369 lck_mtx_unlock(cl_mtxp);
3370
3371 if (iostate.io_error)
3372 retval = iostate.io_error;
3373
3374 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 70)) | DBG_FUNC_END,
3375 (int)uio->uio_offset, (int)uio_resid(uio), 6, retval, 0);
3376
3377 return (retval);
3378 }
3379
3380
3381 static int
3382 cluster_phys_read(vnode_t vp, struct uio *uio, off_t filesize)
3383 {
3384 upl_page_info_t *pl;
3385 upl_t upl;
3386 vm_offset_t upl_offset;
3387 addr64_t dst_paddr;
3388 off_t max_size;
3389 int io_size;
3390 user_size_t iov_len;
3391 user_addr_t iov_base;
3392 int tail_size;
3393 int upl_size;
3394 int upl_needed_size;
3395 int pages_in_pl;
3396 int upl_flags;
3397 kern_return_t kret;
3398 struct clios iostate;
3399 int error;
3400 int devblocksize;
3401
3402 devblocksize = vp->v_mount->mnt_devblocksize;
3403 /*
3404 * When we enter this routine, we know
3405 * -- the resid will not exceed iov_len
3406 * -- the target address is physically contiguous
3407 */
3408
3409 #if LP64_DEBUG
3410 if (IS_VALID_UIO_SEGFLG(uio->uio_segflg) == 0) {
3411 panic("%s :%d - invalid uio_segflg\n", __FILE__, __LINE__);
3412 }
3413 #endif /* LP64_DEBUG */
3414
3415 iov_len = uio_curriovlen(uio);
3416 iov_base = uio_curriovbase(uio);
3417
3418 max_size = filesize - uio->uio_offset;
3419
3420 // LP64todo - fix this!
3421 if (max_size < 0 || (u_int64_t)max_size > iov_len)
3422 io_size = iov_len;
3423 else
3424 io_size = max_size;
3425
3426 // LP64todo - fix this!
3427 upl_offset = CAST_DOWN(vm_offset_t, iov_base) & PAGE_MASK;
3428 upl_needed_size = upl_offset + io_size;
3429
3430 error = 0;
3431 pages_in_pl = 0;
3432 upl_size = upl_needed_size;
3433 upl_flags = UPL_FILE_IO | UPL_NO_SYNC | UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE;
3434
3435 kret = vm_map_get_upl(current_map(),
3436 (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)),
3437 &upl_size, &upl, NULL, &pages_in_pl, &upl_flags, 0);
3438
3439 if (kret != KERN_SUCCESS) {
3440 /*
3441 * cluster_phys_read: failed to get pagelist
3442 */
3443 return(EINVAL);
3444 }
3445 if (upl_size < upl_needed_size) {
3446 /*
3447 * The upl_size wasn't satisfied.
3448 */
3449 ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_FREE_ON_EMPTY);
3450
3451 return(EINVAL);
3452 }
3453 pl = ubc_upl_pageinfo(upl);
3454
3455 dst_paddr = ((addr64_t)upl_phys_page(pl, 0) << 12) + (addr64_t)upl_offset;
3456
3457 while (((uio->uio_offset & (devblocksize - 1)) || io_size < devblocksize) && io_size) {
3458 int head_size;
3459
3460 head_size = devblocksize - (int)(uio->uio_offset & (devblocksize - 1));
3461
3462 if (head_size > io_size)
3463 head_size = io_size;
3464
3465 error = cluster_align_phys_io(vp, uio, dst_paddr, head_size, CL_READ);
3466
3467 if (error) {
3468 ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_FREE_ON_EMPTY);
3469
3470 return(EINVAL);
3471 }
3472 upl_offset += head_size;
3473 dst_paddr += head_size;
3474 io_size -= head_size;
3475 }
3476 tail_size = io_size & (devblocksize - 1);
3477 io_size -= tail_size;
3478
3479 iostate.io_completed = 0;
3480 iostate.io_issued = 0;
3481 iostate.io_error = 0;
3482 iostate.io_wanted = 0;
3483
3484 while (io_size && error == 0) {
3485 int xsize;
3486
3487 if (io_size > (MAX_UPL_TRANSFER * PAGE_SIZE))
3488 xsize = MAX_UPL_TRANSFER * PAGE_SIZE;
3489 else
3490 xsize = io_size;
3491 /*
3492 * request asynchronously so that we can overlap
3493 * the preparation of the next I/O... we'll do
3494 * the commit after all the I/O has completed
3495 * since its all issued against the same UPL
3496 * if there are already too many outstanding reads
3497 * wait until some have completed before issuing the next
3498 */
3499 lck_mtx_lock(cl_mtxp);
3500
3501 while ((iostate.io_issued - iostate.io_completed) > (8 * MAX_UPL_TRANSFER * PAGE_SIZE)) {
3502 iostate.io_wanted = 1;
3503 msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_phys_read", 0);
3504 }
3505 lck_mtx_unlock(cl_mtxp);
3506
3507 error = cluster_io(vp, upl, upl_offset, uio->uio_offset, xsize,
3508 CL_READ | CL_NOZERO | CL_DEV_MEMORY | CL_ASYNC,
3509 (buf_t)NULL, &iostate);
3510 /*
3511 * The cluster_io read was issued successfully,
3512 * update the uio structure
3513 */
3514 if (error == 0) {
3515 uio_update(uio, (user_size_t)xsize);
3516
3517 dst_paddr += xsize;
3518 upl_offset += xsize;
3519 io_size -= xsize;
3520 }
3521 }
3522 /*
3523 * make sure all async reads that are part of this stream
3524 * have completed before we proceed
3525 */
3526 lck_mtx_lock(cl_mtxp);
3527
3528 while (iostate.io_issued != iostate.io_completed) {
3529 iostate.io_wanted = 1;
3530 msleep((caddr_t)&iostate.io_wanted, cl_mtxp, PRIBIO + 1, "cluster_phys_read", 0);
3531 }
3532 lck_mtx_unlock(cl_mtxp);
3533
3534 if (iostate.io_error)
3535 error = iostate.io_error;
3536
3537 if (error == 0 && tail_size)
3538 error = cluster_align_phys_io(vp, uio, dst_paddr, tail_size, CL_READ);
3539
3540 /*
3541 * just release our hold on the physically contiguous
3542 * region without changing any state
3543 */
3544 ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_FREE_ON_EMPTY);
3545
3546 return (error);
3547 }
3548
3549
3550 /*
3551 * generate advisory I/O's in the largest chunks possible
3552 * the completed pages will be released into the VM cache
3553 */
3554 int
3555 advisory_read(vnode_t vp, off_t filesize, off_t f_offset, int resid)
3556 {
3557 upl_page_info_t *pl;
3558 upl_t upl;
3559 vm_offset_t upl_offset;
3560 int upl_size;
3561 off_t upl_f_offset;
3562 int start_offset;
3563 int start_pg;
3564 int last_pg;
3565 int pages_in_upl;
3566 off_t max_size;
3567 int io_size;
3568 kern_return_t kret;
3569 int retval = 0;
3570 int issued_io;
3571 int skip_range;
3572
3573 if ( !UBCINFOEXISTS(vp))
3574 return(EINVAL);
3575
3576 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 60)) | DBG_FUNC_START,
3577 (int)f_offset, resid, (int)filesize, 0, 0);
3578
3579 while (resid && f_offset < filesize && retval == 0) {
3580 /*
3581 * compute the size of the upl needed to encompass
3582 * the requested read... limit each call to cluster_io
3583 * to the maximum UPL size... cluster_io will clip if
3584 * this exceeds the maximum io_size for the device,
3585 * make sure to account for
3586 * a starting offset that's not page aligned
3587 */
3588 start_offset = (int)(f_offset & PAGE_MASK_64);
3589 upl_f_offset = f_offset - (off_t)start_offset;
3590 max_size = filesize - f_offset;
3591
3592 if (resid < max_size)
3593 io_size = resid;
3594 else
3595 io_size = max_size;
3596
3597 upl_size = (start_offset + io_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
3598 if (upl_size > (MAX_UPL_TRANSFER * PAGE_SIZE))
3599 upl_size = MAX_UPL_TRANSFER * PAGE_SIZE;
3600
3601 skip_range = 0;
3602 /*
3603 * return the number of contiguously present pages in the cache
3604 * starting at upl_f_offset within the file
3605 */
3606 ubc_range_op(vp, upl_f_offset, upl_f_offset + upl_size, UPL_ROP_PRESENT, &skip_range);
3607
3608 if (skip_range) {
3609 /*
3610 * skip over pages already present in the cache
3611 */
3612 io_size = skip_range - start_offset;
3613
3614 f_offset += io_size;
3615 resid -= io_size;
3616
3617 if (skip_range == upl_size)
3618 continue;
3619 /*
3620 * have to issue some real I/O
3621 * at this point, we know it's starting on a page boundary
3622 * because we've skipped over at least the first page in the request
3623 */
3624 start_offset = 0;
3625 upl_f_offset += skip_range;
3626 upl_size -= skip_range;
3627 }
3628 pages_in_upl = upl_size / PAGE_SIZE;
3629
3630 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 61)) | DBG_FUNC_START,
3631 (int)upl, (int)upl_f_offset, upl_size, start_offset, 0);
3632
3633 kret = ubc_create_upl(vp,
3634 upl_f_offset,
3635 upl_size,
3636 &upl,
3637 &pl,
3638 UPL_RET_ONLY_ABSENT | UPL_SET_LITE);
3639 if (kret != KERN_SUCCESS)
3640 return(retval);
3641 issued_io = 0;
3642
3643 /*
3644 * before we start marching forward, we must make sure we end on
3645 * a present page, otherwise we will be working with a freed
3646 * upl
3647 */
3648 for (last_pg = pages_in_upl - 1; last_pg >= 0; last_pg--) {
3649 if (upl_page_present(pl, last_pg))
3650 break;
3651 }
3652 pages_in_upl = last_pg + 1;
3653
3654
3655 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 61)) | DBG_FUNC_END,
3656 (int)upl, (int)upl_f_offset, upl_size, start_offset, 0);
3657
3658
3659 for (last_pg = 0; last_pg < pages_in_upl; ) {
3660 /*
3661 * scan from the beginning of the upl looking for the first
3662 * page that is present.... this will become the first page in
3663 * the request we're going to make to 'cluster_io'... if all
3664 * of the pages are absent, we won't call through to 'cluster_io'
3665 */
3666 for (start_pg = last_pg; start_pg < pages_in_upl; start_pg++) {
3667 if (upl_page_present(pl, start_pg))
3668 break;
3669 }
3670
3671 /*
3672 * scan from the starting present page looking for an absent
3673 * page before the end of the upl is reached, if we
3674 * find one, then it will terminate the range of pages being
3675 * presented to 'cluster_io'
3676 */
3677 for (last_pg = start_pg; last_pg < pages_in_upl; last_pg++) {
3678 if (!upl_page_present(pl, last_pg))
3679 break;
3680 }
3681
3682 if (last_pg > start_pg) {
3683 /*
3684 * we found a range of pages that must be filled
3685 * if the last page in this range is the last page of the file
3686 * we may have to clip the size of it to keep from reading past
3687 * the end of the last physical block associated with the file
3688 */
3689 upl_offset = start_pg * PAGE_SIZE;
3690 io_size = (last_pg - start_pg) * PAGE_SIZE;
3691
3692 if ((upl_f_offset + upl_offset + io_size) > filesize)
3693 io_size = filesize - (upl_f_offset + upl_offset);
3694
3695 /*
3696 * issue an asynchronous read to cluster_io
3697 */
3698 retval = cluster_io(vp, upl, upl_offset, upl_f_offset + upl_offset, io_size,
3699 CL_ASYNC | CL_READ | CL_COMMIT | CL_AGE, (buf_t)NULL, (struct clios *)NULL);
3700
3701 issued_io = 1;
3702 }
3703 }
3704 if (issued_io == 0)
3705 ubc_upl_abort(upl, 0);
3706
3707 io_size = upl_size - start_offset;
3708
3709 if (io_size > resid)
3710 io_size = resid;
3711 f_offset += io_size;
3712 resid -= io_size;
3713 }
3714
3715 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 60)) | DBG_FUNC_END,
3716 (int)f_offset, resid, retval, 0, 0);
3717
3718 return(retval);
3719 }
3720
3721
3722 int
3723 cluster_push(vnode_t vp, int flags)
3724 {
3725 int retval;
3726 struct cl_writebehind *wbp;
3727
3728 if ( !UBCINFOEXISTS(vp)) {
3729 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 53)) | DBG_FUNC_NONE, (int)vp, flags, 0, -1, 0);
3730 return (0);
3731 }
3732 /* return if deferred write is set */
3733 if (((unsigned int)vfs_flags(vp->v_mount) & MNT_DEFWRITE) && (flags & IO_DEFWRITE)) {
3734 return (0);
3735 }
3736 if ((wbp = cluster_get_wbp(vp, CLW_RETURNLOCKED)) == NULL) {
3737 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 53)) | DBG_FUNC_NONE, (int)vp, flags, 0, -2, 0);
3738 return (0);
3739 }
3740 if (wbp->cl_number == 0 && wbp->cl_scmap == NULL) {
3741 lck_mtx_unlock(&wbp->cl_lockw);
3742
3743 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 53)) | DBG_FUNC_NONE, (int)vp, flags, 0, -3, 0);
3744 return(0);
3745 }
3746 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 53)) | DBG_FUNC_START,
3747 (int)wbp->cl_scmap, wbp->cl_number, flags, 0, 0);
3748
3749 if (wbp->cl_scmap) {
3750 sparse_cluster_push(wbp, vp, ubc_getsize(vp), 1);
3751
3752 retval = 1;
3753 } else
3754 retval = cluster_try_push(wbp, vp, ubc_getsize(vp), 0, 1);
3755
3756 lck_mtx_unlock(&wbp->cl_lockw);
3757
3758 if (flags & IO_SYNC)
3759 (void)vnode_waitforwrites(vp, 0, 0, 0, (char *)"cluster_push");
3760
3761 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 53)) | DBG_FUNC_END,
3762 (int)wbp->cl_scmap, wbp->cl_number, retval, 0, 0);
3763
3764 return (retval);
3765 }
3766
3767
3768 __private_extern__ void
3769 cluster_release(struct ubc_info *ubc)
3770 {
3771 struct cl_writebehind *wbp;
3772 struct cl_readahead *rap;
3773
3774 if ((wbp = ubc->cl_wbehind)) {
3775
3776 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 81)) | DBG_FUNC_START, (int)ubc, (int)wbp->cl_scmap, wbp->cl_scdirty, 0, 0);
3777
3778 if (wbp->cl_scmap)
3779 vfs_drt_control(&(wbp->cl_scmap), 0);
3780 } else {
3781 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 81)) | DBG_FUNC_START, (int)ubc, 0, 0, 0, 0);
3782 }
3783
3784 rap = ubc->cl_rahead;
3785
3786 if (wbp != NULL) {
3787 lck_mtx_destroy(&wbp->cl_lockw, cl_mtx_grp);
3788 FREE_ZONE((void *)wbp, sizeof *wbp, M_CLWRBEHIND);
3789 }
3790 if ((rap = ubc->cl_rahead)) {
3791 lck_mtx_destroy(&rap->cl_lockr, cl_mtx_grp);
3792 FREE_ZONE((void *)rap, sizeof *rap, M_CLRDAHEAD);
3793 }
3794 ubc->cl_rahead = NULL;
3795 ubc->cl_wbehind = NULL;
3796
3797 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 81)) | DBG_FUNC_END, (int)ubc, (int)rap, (int)wbp, 0, 0);
3798 }
3799
3800
3801 static void
3802 cluster_push_EOF(vnode_t vp, off_t EOF)
3803 {
3804 struct cl_writebehind *wbp;
3805
3806 wbp = cluster_get_wbp(vp, CLW_ALLOCATE | CLW_RETURNLOCKED);
3807
3808 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 53)) | DBG_FUNC_START,
3809 (int)wbp->cl_scmap, wbp->cl_number, (int)EOF, 0, 0);
3810
3811 if (wbp->cl_scmap)
3812 sparse_cluster_push(wbp, vp, EOF, 1);
3813 else
3814 cluster_try_push(wbp, vp, EOF, 0, 1);
3815
3816 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 53)) | DBG_FUNC_END,
3817 (int)wbp->cl_scmap, wbp->cl_number, 0, 0, 0);
3818
3819 lck_mtx_unlock(&wbp->cl_lockw);
3820 }
3821
3822
3823 static int
3824 cluster_try_push(struct cl_writebehind *wbp, vnode_t vp, off_t EOF, int can_delay, int push_all)
3825 {
3826 int cl_index;
3827 int cl_index1;
3828 int min_index;
3829 int cl_len;
3830 int cl_pushed = 0;
3831 struct cl_wextent l_clusters[MAX_CLUSTERS];
3832
3833 /*
3834 * the write behind context exists and has
3835 * already been locked...
3836 *
3837 * make a local 'sorted' copy of the clusters
3838 * and clear wbp->cl_number so that new clusters can
3839 * be developed
3840 */
3841 for (cl_index = 0; cl_index < wbp->cl_number; cl_index++) {
3842 for (min_index = -1, cl_index1 = 0; cl_index1 < wbp->cl_number; cl_index1++) {
3843 if (wbp->cl_clusters[cl_index1].b_addr == wbp->cl_clusters[cl_index1].e_addr)
3844 continue;
3845 if (min_index == -1)
3846 min_index = cl_index1;
3847 else if (wbp->cl_clusters[cl_index1].b_addr < wbp->cl_clusters[min_index].b_addr)
3848 min_index = cl_index1;
3849 }
3850 if (min_index == -1)
3851 break;
3852 l_clusters[cl_index].b_addr = wbp->cl_clusters[min_index].b_addr;
3853 l_clusters[cl_index].e_addr = wbp->cl_clusters[min_index].e_addr;
3854 l_clusters[cl_index].io_nocache = wbp->cl_clusters[min_index].io_nocache;
3855
3856 wbp->cl_clusters[min_index].b_addr = wbp->cl_clusters[min_index].e_addr;
3857 }
3858 wbp->cl_number = 0;
3859
3860 cl_len = cl_index;
3861
3862 if (can_delay && cl_len == MAX_CLUSTERS) {
3863 int i;
3864
3865 /*
3866 * determine if we appear to be writing the file sequentially
3867 * if not, by returning without having pushed any clusters
3868 * we will cause this vnode to be pushed into the sparse cluster mechanism
3869 * used for managing more random I/O patterns
3870 *
3871 * we know that we've got all clusters currently in use and the next write doesn't fit into one of them...
3872 * that's why we're in try_push with can_delay true...
3873 *
3874 * check to make sure that all the clusters except the last one are 'full'... and that each cluster
3875 * is adjacent to the next (i.e. we're looking for sequential writes) they were sorted above
3876 * so we can just make a simple pass through, up to, but not including the last one...
3877 * note that e_addr is not inclusive, so it will be equal to the b_addr of the next cluster if they
3878 * are sequential
3879 *
3880 * we let the last one be partial as long as it was adjacent to the previous one...
3881 * we need to do this to deal with multi-threaded servers that might write an I/O or 2 out
3882 * of order... if this occurs at the tail of the last cluster, we don't want to fall into the sparse cluster world...
3883 */
3884 for (i = 0; i < MAX_CLUSTERS - 1; i++) {
3885 if ((l_clusters[i].e_addr - l_clusters[i].b_addr) != MAX_UPL_TRANSFER)
3886 goto dont_try;
3887 if (l_clusters[i].e_addr != l_clusters[i+1].b_addr)
3888 goto dont_try;
3889 }
3890 }
3891 /*
3892 * drop the lock while we're firing off the I/Os...
3893 * this is safe since I'm working off of a private sorted copy
3894 * of the clusters, and I'm going to re-evaluate the public
3895 * state after I retake the lock
3896 */
3897 lck_mtx_unlock(&wbp->cl_lockw);
3898
3899 for (cl_index = 0; cl_index < cl_len; cl_index++) {
3900 int flags;
3901 struct cl_extent cl;
3902
3903 /*
3904 * try to push each cluster in turn...
3905 */
3906 if (l_clusters[cl_index].io_nocache)
3907 flags = IO_NOCACHE;
3908 else
3909 flags = 0;
3910 cl.b_addr = l_clusters[cl_index].b_addr;
3911 cl.e_addr = l_clusters[cl_index].e_addr;
3912
3913 cluster_push_x(vp, &cl, EOF, flags);
3914
3915 l_clusters[cl_index].b_addr = 0;
3916 l_clusters[cl_index].e_addr = 0;
3917
3918 cl_pushed++;
3919
3920 if (push_all == 0)
3921 break;
3922 }
3923 lck_mtx_lock(&wbp->cl_lockw);
3924
3925 dont_try:
3926 if (cl_len > cl_pushed) {
3927 /*
3928 * we didn't push all of the clusters, so
3929 * lets try to merge them back in to the vnode
3930 */
3931 if ((MAX_CLUSTERS - wbp->cl_number) < (cl_len - cl_pushed)) {
3932 /*
3933 * we picked up some new clusters while we were trying to
3934 * push the old ones... this can happen because I've dropped
3935 * the vnode lock... the sum of the
3936 * leftovers plus the new cluster count exceeds our ability
3937 * to represent them, so switch to the sparse cluster mechanism
3938 *
3939 * collect the active public clusters...
3940 */
3941 sparse_cluster_switch(wbp, vp, EOF);
3942
3943 for (cl_index = 0, cl_index1 = 0; cl_index < cl_len; cl_index++) {
3944 if (l_clusters[cl_index].b_addr == l_clusters[cl_index].e_addr)
3945 continue;
3946 wbp->cl_clusters[cl_index1].b_addr = l_clusters[cl_index].b_addr;
3947 wbp->cl_clusters[cl_index1].e_addr = l_clusters[cl_index].e_addr;
3948 wbp->cl_clusters[cl_index1].io_nocache = l_clusters[cl_index].io_nocache;
3949
3950 cl_index1++;
3951 }
3952 /*
3953 * update the cluster count
3954 */
3955 wbp->cl_number = cl_index1;
3956
3957 /*
3958 * and collect the original clusters that were moved into the
3959 * local storage for sorting purposes
3960 */
3961 sparse_cluster_switch(wbp, vp, EOF);
3962
3963 } else {
3964 /*
3965 * we've got room to merge the leftovers back in
3966 * just append them starting at the next 'hole'
3967 * represented by wbp->cl_number
3968 */
3969 for (cl_index = 0, cl_index1 = wbp->cl_number; cl_index < cl_len; cl_index++) {
3970 if (l_clusters[cl_index].b_addr == l_clusters[cl_index].e_addr)
3971 continue;
3972
3973 wbp->cl_clusters[cl_index1].b_addr = l_clusters[cl_index].b_addr;
3974 wbp->cl_clusters[cl_index1].e_addr = l_clusters[cl_index].e_addr;
3975 wbp->cl_clusters[cl_index1].io_nocache = l_clusters[cl_index].io_nocache;
3976
3977 cl_index1++;
3978 }
3979 /*
3980 * update the cluster count
3981 */
3982 wbp->cl_number = cl_index1;
3983 }
3984 }
3985 return(MAX_CLUSTERS - wbp->cl_number);
3986 }
3987
3988
3989
3990 static int
3991 cluster_push_x(vnode_t vp, struct cl_extent *cl, off_t EOF, int flags)
3992 {
3993 upl_page_info_t *pl;
3994 upl_t upl;
3995 vm_offset_t upl_offset;
3996 int upl_size;
3997 off_t upl_f_offset;
3998 int pages_in_upl;
3999 int start_pg;
4000 int last_pg;
4001 int io_size;
4002 int io_flags;
4003 int upl_flags;
4004 int size;
4005 int error = 0;
4006 int retval;
4007 kern_return_t kret;
4008
4009
4010 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 51)) | DBG_FUNC_START,
4011 (int)cl->b_addr, (int)cl->e_addr, (int)EOF, flags, 0);
4012
4013 if ((pages_in_upl = (int)(cl->e_addr - cl->b_addr)) == 0) {
4014 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 51)) | DBG_FUNC_END, 1, 0, 0, 0, 0);
4015
4016 return (0);
4017 }
4018 upl_size = pages_in_upl * PAGE_SIZE;
4019 upl_f_offset = (off_t)(cl->b_addr * PAGE_SIZE_64);
4020
4021 if (upl_f_offset + upl_size >= EOF) {
4022
4023 if (upl_f_offset >= EOF) {
4024 /*
4025 * must have truncated the file and missed
4026 * clearing a dangling cluster (i.e. it's completely
4027 * beyond the new EOF
4028 */
4029 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 51)) | DBG_FUNC_END, 1, 1, 0, 0, 0);
4030
4031 return(0);
4032 }
4033 size = EOF - upl_f_offset;
4034
4035 upl_size = (size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
4036 pages_in_upl = upl_size / PAGE_SIZE;
4037 } else
4038 size = upl_size;
4039
4040 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 41)) | DBG_FUNC_START, upl_size, size, 0, 0, 0);
4041
4042 /*
4043 * by asking for UPL_COPYOUT_FROM and UPL_RET_ONLY_DIRTY, we get the following desirable behavior
4044 *
4045 * - only pages that are currently dirty are returned... these are the ones we need to clean
4046 * - the hardware dirty bit is cleared when the page is gathered into the UPL... the software dirty bit is set
4047 * - if we have to abort the I/O for some reason, the software dirty bit is left set since we didn't clean the page
4048 * - when we commit the page, the software dirty bit is cleared... the hardware dirty bit is untouched so that if
4049 * someone dirties this page while the I/O is in progress, we don't lose track of the new state
4050 *
4051 * when the I/O completes, we no longer ask for an explicit clear of the DIRTY state (either soft or hard)
4052 */
4053
4054 if ((vp->v_flag & VNOCACHE_DATA) || (flags & IO_NOCACHE))
4055 upl_flags = UPL_COPYOUT_FROM | UPL_RET_ONLY_DIRTY | UPL_SET_LITE | UPL_WILL_BE_DUMPED;
4056 else
4057 upl_flags = UPL_COPYOUT_FROM | UPL_RET_ONLY_DIRTY | UPL_SET_LITE;
4058
4059 kret = ubc_create_upl(vp,
4060 upl_f_offset,
4061 upl_size,
4062 &upl,
4063 &pl,
4064 upl_flags);
4065 if (kret != KERN_SUCCESS)
4066 panic("cluster_push: failed to get pagelist");
4067
4068 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 41)) | DBG_FUNC_END, (int)upl, upl_f_offset, 0, 0, 0);
4069
4070 /*
4071 * since we only asked for the dirty pages back
4072 * it's possible that we may only get a few or even none, so...
4073 * before we start marching forward, we must make sure we know
4074 * where the last present page is in the UPL, otherwise we could
4075 * end up working with a freed upl due to the FREE_ON_EMPTY semantics
4076 * employed by commit_range and abort_range.
4077 */
4078 for (last_pg = pages_in_upl - 1; last_pg >= 0; last_pg--) {
4079 if (upl_page_present(pl, last_pg))
4080 break;
4081 }
4082 pages_in_upl = last_pg + 1;
4083
4084 if (pages_in_upl == 0) {
4085 ubc_upl_abort(upl, 0);
4086
4087 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 51)) | DBG_FUNC_END, 1, 2, 0, 0, 0);
4088 return(0);
4089 }
4090
4091 for (last_pg = 0; last_pg < pages_in_upl; ) {
4092 /*
4093 * find the next dirty page in the UPL
4094 * this will become the first page in the
4095 * next I/O to generate
4096 */
4097 for (start_pg = last_pg; start_pg < pages_in_upl; start_pg++) {
4098 if (upl_dirty_page(pl, start_pg))
4099 break;
4100 if (upl_page_present(pl, start_pg))
4101 /*
4102 * RET_ONLY_DIRTY will return non-dirty 'precious' pages
4103 * just release these unchanged since we're not going
4104 * to steal them or change their state
4105 */
4106 ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
4107 }
4108 if (start_pg >= pages_in_upl)
4109 /*
4110 * done... no more dirty pages to push
4111 */
4112 break;
4113 if (start_pg > last_pg)
4114 /*
4115 * skipped over some non-dirty pages
4116 */
4117 size -= ((start_pg - last_pg) * PAGE_SIZE);
4118
4119 /*
4120 * find a range of dirty pages to write
4121 */
4122 for (last_pg = start_pg; last_pg < pages_in_upl; last_pg++) {
4123 if (!upl_dirty_page(pl, last_pg))
4124 break;
4125 }
4126 upl_offset = start_pg * PAGE_SIZE;
4127
4128 io_size = min(size, (last_pg - start_pg) * PAGE_SIZE);
4129
4130 io_flags = CL_THROTTLE | CL_COMMIT;
4131
4132 if ( !(flags & IO_SYNC))
4133 io_flags |= CL_ASYNC;
4134
4135 retval = cluster_io(vp, upl, upl_offset, upl_f_offset + upl_offset, io_size,
4136 io_flags, (buf_t)NULL, (struct clios *)NULL);
4137
4138 if (error == 0 && retval)
4139 error = retval;
4140
4141 size -= io_size;
4142 }
4143 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 51)) | DBG_FUNC_END, 1, 3, 0, 0, 0);
4144
4145 return(error);
4146 }
4147
4148
4149 /*
4150 * sparse_cluster_switch is called with the write behind lock held
4151 */
4152 static void
4153 sparse_cluster_switch(struct cl_writebehind *wbp, vnode_t vp, off_t EOF)
4154 {
4155 int cl_index;
4156
4157 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 78)) | DBG_FUNC_START, (int)vp, (int)wbp->cl_scmap, wbp->cl_scdirty, 0, 0);
4158
4159 if (wbp->cl_scmap == NULL)
4160 wbp->cl_scdirty = 0;
4161
4162 for (cl_index = 0; cl_index < wbp->cl_number; cl_index++) {
4163 int flags;
4164 struct cl_extent cl;
4165
4166 for (cl.b_addr = wbp->cl_clusters[cl_index].b_addr; cl.b_addr < wbp->cl_clusters[cl_index].e_addr; cl.b_addr++) {
4167
4168 if (ubc_page_op(vp, (off_t)(cl.b_addr * PAGE_SIZE_64), 0, 0, &flags) == KERN_SUCCESS) {
4169 if (flags & UPL_POP_DIRTY) {
4170 cl.e_addr = cl.b_addr + 1;
4171
4172 sparse_cluster_add(wbp, vp, &cl, EOF);
4173 }
4174 }
4175 }
4176 }
4177 wbp->cl_number = 0;
4178
4179 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 78)) | DBG_FUNC_END, (int)vp, (int)wbp->cl_scmap, wbp->cl_scdirty, 0, 0);
4180 }
4181
4182
4183 /*
4184 * sparse_cluster_push is called with the write behind lock held
4185 */
4186 static void
4187 sparse_cluster_push(struct cl_writebehind *wbp, vnode_t vp, off_t EOF, int push_all)
4188 {
4189 struct cl_extent cl;
4190 off_t offset;
4191 u_int length;
4192
4193 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 79)) | DBG_FUNC_START, (int)vp, (int)wbp->cl_scmap, wbp->cl_scdirty, push_all, 0);
4194
4195 if (push_all)
4196 vfs_drt_control(&(wbp->cl_scmap), 1);
4197
4198 for (;;) {
4199 if (vfs_drt_get_cluster(&(wbp->cl_scmap), &offset, &length) != KERN_SUCCESS)
4200 break;
4201
4202 cl.b_addr = (daddr64_t)(offset / PAGE_SIZE_64);
4203 cl.e_addr = (daddr64_t)((offset + length) / PAGE_SIZE_64);
4204
4205 wbp->cl_scdirty -= (int)(cl.e_addr - cl.b_addr);
4206
4207 cluster_push_x(vp, &cl, EOF, 0);
4208
4209 if (push_all == 0)
4210 break;
4211 }
4212 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 79)) | DBG_FUNC_END, (int)vp, (int)wbp->cl_scmap, wbp->cl_scdirty, 0, 0);
4213 }
4214
4215
4216 /*
4217 * sparse_cluster_add is called with the write behind lock held
4218 */
4219 static void
4220 sparse_cluster_add(struct cl_writebehind *wbp, vnode_t vp, struct cl_extent *cl, off_t EOF)
4221 {
4222 u_int new_dirty;
4223 u_int length;
4224 off_t offset;
4225
4226 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 80)) | DBG_FUNC_START, (int)wbp->cl_scmap, wbp->cl_scdirty, (int)cl->b_addr, (int)cl->e_addr, 0);
4227
4228 offset = (off_t)(cl->b_addr * PAGE_SIZE_64);
4229 length = ((u_int)(cl->e_addr - cl->b_addr)) * PAGE_SIZE;
4230
4231 while (vfs_drt_mark_pages(&(wbp->cl_scmap), offset, length, &new_dirty) != KERN_SUCCESS) {
4232 /*
4233 * no room left in the map
4234 * only a partial update was done
4235 * push out some pages and try again
4236 */
4237 wbp->cl_scdirty += new_dirty;
4238
4239 sparse_cluster_push(wbp, vp, EOF, 0);
4240
4241 offset += (new_dirty * PAGE_SIZE_64);
4242 length -= (new_dirty * PAGE_SIZE);
4243 }
4244 wbp->cl_scdirty += new_dirty;
4245
4246 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 80)) | DBG_FUNC_END, (int)vp, (int)wbp->cl_scmap, wbp->cl_scdirty, 0, 0);
4247 }
4248
4249
4250 static int
4251 cluster_align_phys_io(vnode_t vp, struct uio *uio, addr64_t usr_paddr, int xsize, int flags)
4252 {
4253 upl_page_info_t *pl;
4254 upl_t upl;
4255 addr64_t ubc_paddr;
4256 kern_return_t kret;
4257 int error = 0;
4258 int did_read = 0;
4259 int abort_flags;
4260 int upl_flags;
4261
4262 upl_flags = UPL_SET_LITE;
4263 if (! (flags & CL_READ)) {
4264 /*
4265 * "write" operation: let the UPL subsystem know
4266 * that we intend to modify the buffer cache pages
4267 * we're gathering.
4268 */
4269 upl_flags |= UPL_WILL_MODIFY;
4270 }
4271
4272 kret = ubc_create_upl(vp,
4273 uio->uio_offset & ~PAGE_MASK_64,
4274 PAGE_SIZE,
4275 &upl,
4276 &pl,
4277 upl_flags);
4278
4279 if (kret != KERN_SUCCESS)
4280 return(EINVAL);
4281
4282 if (!upl_valid_page(pl, 0)) {
4283 /*
4284 * issue a synchronous read to cluster_io
4285 */
4286 error = cluster_io(vp, upl, 0, uio->uio_offset & ~PAGE_MASK_64, PAGE_SIZE,
4287 CL_READ, (buf_t)NULL, (struct clios *)NULL);
4288 if (error) {
4289 ubc_upl_abort_range(upl, 0, PAGE_SIZE, UPL_ABORT_DUMP_PAGES | UPL_ABORT_FREE_ON_EMPTY);
4290
4291 return(error);
4292 }
4293 did_read = 1;
4294 }
4295 ubc_paddr = ((addr64_t)upl_phys_page(pl, 0) << 12) + (addr64_t)(uio->uio_offset & PAGE_MASK_64);
4296
4297 /*
4298 * NOTE: There is no prototype for the following in BSD. It, and the definitions
4299 * of the defines for cppvPsrc, cppvPsnk, cppvFsnk, and cppvFsrc will be found in
4300 * osfmk/ppc/mappings.h. They are not included here because there appears to be no
4301 * way to do so without exporting them to kexts as well.
4302 */
4303 if (flags & CL_READ)
4304 // copypv(ubc_paddr, usr_paddr, xsize, cppvPsrc | cppvPsnk | cppvFsnk); /* Copy physical to physical and flush the destination */
4305 copypv(ubc_paddr, usr_paddr, xsize, 2 | 1 | 4); /* Copy physical to physical and flush the destination */
4306 else
4307 // copypv(usr_paddr, ubc_paddr, xsize, cppvPsrc | cppvPsnk | cppvFsrc); /* Copy physical to physical and flush the source */
4308 copypv(usr_paddr, ubc_paddr, xsize, 2 | 1 | 8); /* Copy physical to physical and flush the source */
4309
4310 if ( !(flags & CL_READ) || (upl_valid_page(pl, 0) && upl_dirty_page(pl, 0))) {
4311 /*
4312 * issue a synchronous write to cluster_io
4313 */
4314 error = cluster_io(vp, upl, 0, uio->uio_offset & ~PAGE_MASK_64, PAGE_SIZE,
4315 0, (buf_t)NULL, (struct clios *)NULL);
4316 }
4317 if (error == 0)
4318 uio_update(uio, (user_size_t)xsize);
4319
4320 if (did_read)
4321 abort_flags = UPL_ABORT_FREE_ON_EMPTY;
4322 else
4323 abort_flags = UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_DUMP_PAGES;
4324
4325 ubc_upl_abort_range(upl, 0, PAGE_SIZE, abort_flags);
4326
4327 return (error);
4328 }
4329
4330
4331
4332 int
4333 cluster_copy_upl_data(struct uio *uio, upl_t upl, int upl_offset, int xsize)
4334 {
4335 int pg_offset;
4336 int pg_index;
4337 int csize;
4338 int segflg;
4339 int retval = 0;
4340 upl_page_info_t *pl;
4341
4342 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 34)) | DBG_FUNC_START,
4343 (int)uio->uio_offset, uio_resid(uio), upl_offset, xsize, 0);
4344
4345 segflg = uio->uio_segflg;
4346
4347 switch(segflg) {
4348
4349 case UIO_USERSPACE32:
4350 case UIO_USERISPACE32:
4351 uio->uio_segflg = UIO_PHYS_USERSPACE32;
4352 break;
4353
4354 case UIO_USERSPACE:
4355 case UIO_USERISPACE:
4356 uio->uio_segflg = UIO_PHYS_USERSPACE;
4357 break;
4358
4359 case UIO_USERSPACE64:
4360 case UIO_USERISPACE64:
4361 uio->uio_segflg = UIO_PHYS_USERSPACE64;
4362 break;
4363
4364 case UIO_SYSSPACE32:
4365 uio->uio_segflg = UIO_PHYS_SYSSPACE32;
4366 break;
4367
4368 case UIO_SYSSPACE:
4369 uio->uio_segflg = UIO_PHYS_SYSSPACE;
4370 break;
4371
4372 case UIO_SYSSPACE64:
4373 uio->uio_segflg = UIO_PHYS_SYSSPACE64;
4374 break;
4375 }
4376 pl = ubc_upl_pageinfo(upl);
4377
4378 pg_index = upl_offset / PAGE_SIZE;
4379 pg_offset = upl_offset & PAGE_MASK;
4380 csize = min(PAGE_SIZE - pg_offset, xsize);
4381
4382 while (xsize && retval == 0) {
4383 addr64_t paddr;
4384
4385 paddr = ((addr64_t)upl_phys_page(pl, pg_index) << 12) + pg_offset;
4386
4387 retval = uiomove64(paddr, csize, uio);
4388
4389 pg_index += 1;
4390 pg_offset = 0;
4391 xsize -= csize;
4392 csize = min(PAGE_SIZE, xsize);
4393 }
4394 uio->uio_segflg = segflg;
4395
4396 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 34)) | DBG_FUNC_END,
4397 (int)uio->uio_offset, uio_resid(uio), retval, segflg, 0);
4398
4399 return (retval);
4400 }
4401
4402
4403 int
4404 cluster_copy_ubc_data(vnode_t vp, struct uio *uio, int *io_resid, int mark_dirty)
4405 {
4406 int segflg;
4407 int io_size;
4408 int xsize;
4409 int start_offset;
4410 int retval = 0;
4411 memory_object_control_t control;
4412
4413
4414 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 34)) | DBG_FUNC_START,
4415 (int)uio->uio_offset, uio_resid(uio), 0, *io_resid, 0);
4416
4417 control = ubc_getobject(vp, UBC_FLAGS_NONE);
4418 if (control == MEMORY_OBJECT_CONTROL_NULL) {
4419 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 34)) | DBG_FUNC_END,
4420 (int)uio->uio_offset, uio_resid(uio), retval, 3, 0);
4421
4422 return(0);
4423 }
4424 segflg = uio->uio_segflg;
4425
4426 switch(segflg) {
4427
4428 case UIO_USERSPACE32:
4429 case UIO_USERISPACE32:
4430 uio->uio_segflg = UIO_PHYS_USERSPACE32;
4431 break;
4432
4433 case UIO_USERSPACE64:
4434 case UIO_USERISPACE64:
4435 uio->uio_segflg = UIO_PHYS_USERSPACE64;
4436 break;
4437
4438 case UIO_SYSSPACE32:
4439 uio->uio_segflg = UIO_PHYS_SYSSPACE32;
4440 break;
4441
4442 case UIO_SYSSPACE64:
4443 uio->uio_segflg = UIO_PHYS_SYSSPACE64;
4444 break;
4445
4446 case UIO_USERSPACE:
4447 case UIO_USERISPACE:
4448 uio->uio_segflg = UIO_PHYS_USERSPACE;
4449 break;
4450
4451 case UIO_SYSSPACE:
4452 uio->uio_segflg = UIO_PHYS_SYSSPACE;
4453 break;
4454 }
4455
4456 if ( (io_size = *io_resid) ) {
4457 start_offset = (int)(uio->uio_offset & PAGE_MASK_64);
4458 xsize = uio_resid(uio);
4459
4460 retval = memory_object_control_uiomove(control, uio->uio_offset - start_offset,
4461 uio, start_offset, io_size, mark_dirty);
4462 xsize -= uio_resid(uio);
4463 io_size -= xsize;
4464 }
4465 uio->uio_segflg = segflg;
4466 *io_resid = io_size;
4467
4468 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 34)) | DBG_FUNC_END,
4469 (int)uio->uio_offset, uio_resid(uio), retval, 0x80000000 | segflg, 0);
4470
4471 return(retval);
4472 }
4473
4474
4475 int
4476 is_file_clean(vnode_t vp, off_t filesize)
4477 {
4478 off_t f_offset;
4479 int flags;
4480 int total_dirty = 0;
4481
4482 for (f_offset = 0; f_offset < filesize; f_offset += PAGE_SIZE_64) {
4483 if (ubc_page_op(vp, f_offset, 0, 0, &flags) == KERN_SUCCESS) {
4484 if (flags & UPL_POP_DIRTY) {
4485 total_dirty++;
4486 }
4487 }
4488 }
4489 if (total_dirty)
4490 return(EINVAL);
4491
4492 return (0);
4493 }
4494
4495
4496
4497 /*
4498 * Dirty region tracking/clustering mechanism.
4499 *
4500 * This code (vfs_drt_*) provides a mechanism for tracking and clustering
4501 * dirty regions within a larger space (file). It is primarily intended to
4502 * support clustering in large files with many dirty areas.
4503 *
4504 * The implementation assumes that the dirty regions are pages.
4505 *
4506 * To represent dirty pages within the file, we store bit vectors in a
4507 * variable-size circular hash.
4508 */
4509
4510 /*
4511 * Bitvector size. This determines the number of pages we group in a
4512 * single hashtable entry. Each hashtable entry is aligned to this
4513 * size within the file.
4514 */
4515 #define DRT_BITVECTOR_PAGES 256
4516
4517 /*
4518 * File offset handling.
4519 *
4520 * DRT_ADDRESS_MASK is dependent on DRT_BITVECTOR_PAGES;
4521 * the correct formula is (~(DRT_BITVECTOR_PAGES * PAGE_SIZE) - 1)
4522 */
4523 #define DRT_ADDRESS_MASK (~((1 << 20) - 1))
4524 #define DRT_ALIGN_ADDRESS(addr) ((addr) & DRT_ADDRESS_MASK)
4525
4526 /*
4527 * Hashtable address field handling.
4528 *
4529 * The low-order bits of the hashtable address are used to conserve
4530 * space.
4531 *
4532 * DRT_HASH_COUNT_MASK must be large enough to store the range
4533 * 0-DRT_BITVECTOR_PAGES inclusive, as well as have one value
4534 * to indicate that the bucket is actually unoccupied.
4535 */
4536 #define DRT_HASH_GET_ADDRESS(scm, i) ((scm)->scm_hashtable[(i)].dhe_control & DRT_ADDRESS_MASK)
4537 #define DRT_HASH_SET_ADDRESS(scm, i, a) \
4538 do { \
4539 (scm)->scm_hashtable[(i)].dhe_control = \
4540 ((scm)->scm_hashtable[(i)].dhe_control & ~DRT_ADDRESS_MASK) | DRT_ALIGN_ADDRESS(a); \
4541 } while (0)
4542 #define DRT_HASH_COUNT_MASK 0x1ff
4543 #define DRT_HASH_GET_COUNT(scm, i) ((scm)->scm_hashtable[(i)].dhe_control & DRT_HASH_COUNT_MASK)
4544 #define DRT_HASH_SET_COUNT(scm, i, c) \
4545 do { \
4546 (scm)->scm_hashtable[(i)].dhe_control = \
4547 ((scm)->scm_hashtable[(i)].dhe_control & ~DRT_HASH_COUNT_MASK) | ((c) & DRT_HASH_COUNT_MASK); \
4548 } while (0)
4549 #define DRT_HASH_CLEAR(scm, i) \
4550 do { \
4551 (scm)->scm_hashtable[(i)].dhe_control = 0; \
4552 } while (0)
4553 #define DRT_HASH_VACATE(scm, i) DRT_HASH_SET_COUNT((scm), (i), DRT_HASH_COUNT_MASK)
4554 #define DRT_HASH_VACANT(scm, i) (DRT_HASH_GET_COUNT((scm), (i)) == DRT_HASH_COUNT_MASK)
4555 #define DRT_HASH_COPY(oscm, oi, scm, i) \
4556 do { \
4557 (scm)->scm_hashtable[(i)].dhe_control = (oscm)->scm_hashtable[(oi)].dhe_control; \
4558 DRT_BITVECTOR_COPY(oscm, oi, scm, i); \
4559 } while(0);
4560
4561
4562 /*
4563 * Hash table moduli.
4564 *
4565 * Since the hashtable entry's size is dependent on the size of
4566 * the bitvector, and since the hashtable size is constrained to
4567 * both being prime and fitting within the desired allocation
4568 * size, these values need to be manually determined.
4569 *
4570 * For DRT_BITVECTOR_SIZE = 256, the entry size is 40 bytes.
4571 *
4572 * The small hashtable allocation is 1024 bytes, so the modulus is 23.
4573 * The large hashtable allocation is 16384 bytes, so the modulus is 401.
4574 */
4575 #define DRT_HASH_SMALL_MODULUS 23
4576 #define DRT_HASH_LARGE_MODULUS 401
4577
4578 #define DRT_SMALL_ALLOCATION 1024 /* 104 bytes spare */
4579 #define DRT_LARGE_ALLOCATION 16384 /* 344 bytes spare */
4580
4581 /* *** nothing below here has secret dependencies on DRT_BITVECTOR_PAGES *** */
4582
4583 /*
4584 * Hashtable bitvector handling.
4585 *
4586 * Bitvector fields are 32 bits long.
4587 */
4588
4589 #define DRT_HASH_SET_BIT(scm, i, bit) \
4590 (scm)->scm_hashtable[(i)].dhe_bitvector[(bit) / 32] |= (1 << ((bit) % 32))
4591
4592 #define DRT_HASH_CLEAR_BIT(scm, i, bit) \
4593 (scm)->scm_hashtable[(i)].dhe_bitvector[(bit) / 32] &= ~(1 << ((bit) % 32))
4594
4595 #define DRT_HASH_TEST_BIT(scm, i, bit) \
4596 ((scm)->scm_hashtable[(i)].dhe_bitvector[(bit) / 32] & (1 << ((bit) % 32)))
4597
4598 #define DRT_BITVECTOR_CLEAR(scm, i) \
4599 bzero(&(scm)->scm_hashtable[(i)].dhe_bitvector[0], (DRT_BITVECTOR_PAGES / 32) * sizeof(u_int32_t))
4600
4601 #define DRT_BITVECTOR_COPY(oscm, oi, scm, i) \
4602 bcopy(&(oscm)->scm_hashtable[(oi)].dhe_bitvector[0], \
4603 &(scm)->scm_hashtable[(i)].dhe_bitvector[0], \
4604 (DRT_BITVECTOR_PAGES / 32) * sizeof(u_int32_t))
4605
4606
4607
4608 /*
4609 * Hashtable entry.
4610 */
4611 struct vfs_drt_hashentry {
4612 u_int64_t dhe_control;
4613 u_int32_t dhe_bitvector[DRT_BITVECTOR_PAGES / 32];
4614 };
4615
4616 /*
4617 * Dirty Region Tracking structure.
4618 *
4619 * The hashtable is allocated entirely inside the DRT structure.
4620 *
4621 * The hash is a simple circular prime modulus arrangement, the structure
4622 * is resized from small to large if it overflows.
4623 */
4624
4625 struct vfs_drt_clustermap {
4626 u_int32_t scm_magic; /* sanity/detection */
4627 #define DRT_SCM_MAGIC 0x12020003
4628 u_int32_t scm_modulus; /* current ring size */
4629 u_int32_t scm_buckets; /* number of occupied buckets */
4630 u_int32_t scm_lastclean; /* last entry we cleaned */
4631 u_int32_t scm_iskips; /* number of slot skips */
4632
4633 struct vfs_drt_hashentry scm_hashtable[0];
4634 };
4635
4636
4637 #define DRT_HASH(scm, addr) ((addr) % (scm)->scm_modulus)
4638 #define DRT_HASH_NEXT(scm, addr) (((addr) + 1) % (scm)->scm_modulus)
4639
4640 /*
4641 * Debugging codes and arguments.
4642 */
4643 #define DRT_DEBUG_EMPTYFREE (FSDBG_CODE(DBG_FSRW, 82)) /* nil */
4644 #define DRT_DEBUG_RETCLUSTER (FSDBG_CODE(DBG_FSRW, 83)) /* offset, length */
4645 #define DRT_DEBUG_ALLOC (FSDBG_CODE(DBG_FSRW, 84)) /* copycount */
4646 #define DRT_DEBUG_INSERT (FSDBG_CODE(DBG_FSRW, 85)) /* offset, iskip */
4647 #define DRT_DEBUG_MARK (FSDBG_CODE(DBG_FSRW, 86)) /* offset, length,
4648 * dirty */
4649 /* 0, setcount */
4650 /* 1 (clean, no map) */
4651 /* 2 (map alloc fail) */
4652 /* 3, resid (partial) */
4653 #define DRT_DEBUG_6 (FSDBG_CODE(DBG_FSRW, 87))
4654 #define DRT_DEBUG_SCMDATA (FSDBG_CODE(DBG_FSRW, 88)) /* modulus, buckets,
4655 * lastclean, iskips */
4656
4657
4658 static kern_return_t vfs_drt_alloc_map(struct vfs_drt_clustermap **cmapp);
4659 static kern_return_t vfs_drt_free_map(struct vfs_drt_clustermap *cmap);
4660 static kern_return_t vfs_drt_search_index(struct vfs_drt_clustermap *cmap,
4661 u_int64_t offset, int *indexp);
4662 static kern_return_t vfs_drt_get_index(struct vfs_drt_clustermap **cmapp,
4663 u_int64_t offset,
4664 int *indexp,
4665 int recursed);
4666 static kern_return_t vfs_drt_do_mark_pages(
4667 void **cmapp,
4668 u_int64_t offset,
4669 u_int length,
4670 int *setcountp,
4671 int dirty);
4672 static void vfs_drt_trace(
4673 struct vfs_drt_clustermap *cmap,
4674 int code,
4675 int arg1,
4676 int arg2,
4677 int arg3,
4678 int arg4);
4679
4680
4681 /*
4682 * Allocate and initialise a sparse cluster map.
4683 *
4684 * Will allocate a new map, resize or compact an existing map.
4685 *
4686 * XXX we should probably have at least one intermediate map size,
4687 * as the 1:16 ratio seems a bit drastic.
4688 */
4689 static kern_return_t
4690 vfs_drt_alloc_map(struct vfs_drt_clustermap **cmapp)
4691 {
4692 struct vfs_drt_clustermap *cmap, *ocmap;
4693 kern_return_t kret;
4694 u_int64_t offset;
4695 int nsize, i, active_buckets, index, copycount;
4696
4697 ocmap = NULL;
4698 if (cmapp != NULL)
4699 ocmap = *cmapp;
4700
4701 /*
4702 * Decide on the size of the new map.
4703 */
4704 if (ocmap == NULL) {
4705 nsize = DRT_HASH_SMALL_MODULUS;
4706 } else {
4707 /* count the number of active buckets in the old map */
4708 active_buckets = 0;
4709 for (i = 0; i < ocmap->scm_modulus; i++) {
4710 if (!DRT_HASH_VACANT(ocmap, i) &&
4711 (DRT_HASH_GET_COUNT(ocmap, i) != 0))
4712 active_buckets++;
4713 }
4714 /*
4715 * If we're currently using the small allocation, check to
4716 * see whether we should grow to the large one.
4717 */
4718 if (ocmap->scm_modulus == DRT_HASH_SMALL_MODULUS) {
4719 /* if the ring is nearly full */
4720 if (active_buckets > (DRT_HASH_SMALL_MODULUS - 5)) {
4721 nsize = DRT_HASH_LARGE_MODULUS;
4722 } else {
4723 nsize = DRT_HASH_SMALL_MODULUS;
4724 }
4725 } else {
4726 /* already using the large modulus */
4727 nsize = DRT_HASH_LARGE_MODULUS;
4728 /*
4729 * If the ring is completely full, there's
4730 * nothing useful for us to do. Behave as
4731 * though we had compacted into the new
4732 * array and return.
4733 */
4734 if (active_buckets >= DRT_HASH_LARGE_MODULUS)
4735 return(KERN_SUCCESS);
4736 }
4737 }
4738
4739 /*
4740 * Allocate and initialise the new map.
4741 */
4742
4743 kret = kmem_alloc(kernel_map, (vm_offset_t *)&cmap,
4744 (nsize == DRT_HASH_SMALL_MODULUS) ? DRT_SMALL_ALLOCATION : DRT_LARGE_ALLOCATION);
4745 if (kret != KERN_SUCCESS)
4746 return(kret);
4747 cmap->scm_magic = DRT_SCM_MAGIC;
4748 cmap->scm_modulus = nsize;
4749 cmap->scm_buckets = 0;
4750 cmap->scm_lastclean = 0;
4751 cmap->scm_iskips = 0;
4752 for (i = 0; i < cmap->scm_modulus; i++) {
4753 DRT_HASH_CLEAR(cmap, i);
4754 DRT_HASH_VACATE(cmap, i);
4755 DRT_BITVECTOR_CLEAR(cmap, i);
4756 }
4757
4758 /*
4759 * If there's an old map, re-hash entries from it into the new map.
4760 */
4761 copycount = 0;
4762 if (ocmap != NULL) {
4763 for (i = 0; i < ocmap->scm_modulus; i++) {
4764 /* skip empty buckets */
4765 if (DRT_HASH_VACANT(ocmap, i) ||
4766 (DRT_HASH_GET_COUNT(ocmap, i) == 0))
4767 continue;
4768 /* get new index */
4769 offset = DRT_HASH_GET_ADDRESS(ocmap, i);
4770 kret = vfs_drt_get_index(&cmap, offset, &index, 1);
4771 if (kret != KERN_SUCCESS) {
4772 /* XXX need to bail out gracefully here */
4773 panic("vfs_drt: new cluster map mysteriously too small");
4774 }
4775 /* copy */
4776 DRT_HASH_COPY(ocmap, i, cmap, index);
4777 copycount++;
4778 }
4779 }
4780
4781 /* log what we've done */
4782 vfs_drt_trace(cmap, DRT_DEBUG_ALLOC, copycount, 0, 0, 0);
4783
4784 /*
4785 * It's important to ensure that *cmapp always points to
4786 * a valid map, so we must overwrite it before freeing
4787 * the old map.
4788 */
4789 *cmapp = cmap;
4790 if (ocmap != NULL) {
4791 /* emit stats into trace buffer */
4792 vfs_drt_trace(ocmap, DRT_DEBUG_SCMDATA,
4793 ocmap->scm_modulus,
4794 ocmap->scm_buckets,
4795 ocmap->scm_lastclean,
4796 ocmap->scm_iskips);
4797
4798 vfs_drt_free_map(ocmap);
4799 }
4800 return(KERN_SUCCESS);
4801 }
4802
4803
4804 /*
4805 * Free a sparse cluster map.
4806 */
4807 static kern_return_t
4808 vfs_drt_free_map(struct vfs_drt_clustermap *cmap)
4809 {
4810 kmem_free(kernel_map, (vm_offset_t)cmap,
4811 (cmap->scm_modulus == DRT_HASH_SMALL_MODULUS) ? DRT_SMALL_ALLOCATION : DRT_LARGE_ALLOCATION);
4812 return(KERN_SUCCESS);
4813 }
4814
4815
4816 /*
4817 * Find the hashtable slot currently occupied by an entry for the supplied offset.
4818 */
4819 static kern_return_t
4820 vfs_drt_search_index(struct vfs_drt_clustermap *cmap, u_int64_t offset, int *indexp)
4821 {
4822 int index, i;
4823
4824 offset = DRT_ALIGN_ADDRESS(offset);
4825 index = DRT_HASH(cmap, offset);
4826
4827 /* traverse the hashtable */
4828 for (i = 0; i < cmap->scm_modulus; i++) {
4829
4830 /*
4831 * If the slot is vacant, we can stop.
4832 */
4833 if (DRT_HASH_VACANT(cmap, index))
4834 break;
4835
4836 /*
4837 * If the address matches our offset, we have success.
4838 */
4839 if (DRT_HASH_GET_ADDRESS(cmap, index) == offset) {
4840 *indexp = index;
4841 return(KERN_SUCCESS);
4842 }
4843
4844 /*
4845 * Move to the next slot, try again.
4846 */
4847 index = DRT_HASH_NEXT(cmap, index);
4848 }
4849 /*
4850 * It's not there.
4851 */
4852 return(KERN_FAILURE);
4853 }
4854
4855 /*
4856 * Find the hashtable slot for the supplied offset. If we haven't allocated
4857 * one yet, allocate one and populate the address field. Note that it will
4858 * not have a nonzero page count and thus will still technically be free, so
4859 * in the case where we are called to clean pages, the slot will remain free.
4860 */
4861 static kern_return_t
4862 vfs_drt_get_index(struct vfs_drt_clustermap **cmapp, u_int64_t offset, int *indexp, int recursed)
4863 {
4864 struct vfs_drt_clustermap *cmap;
4865 kern_return_t kret;
4866 int index, i;
4867
4868 cmap = *cmapp;
4869
4870 /* look for an existing entry */
4871 kret = vfs_drt_search_index(cmap, offset, indexp);
4872 if (kret == KERN_SUCCESS)
4873 return(kret);
4874
4875 /* need to allocate an entry */
4876 offset = DRT_ALIGN_ADDRESS(offset);
4877 index = DRT_HASH(cmap, offset);
4878
4879 /* scan from the index forwards looking for a vacant slot */
4880 for (i = 0; i < cmap->scm_modulus; i++) {
4881 /* slot vacant? */
4882 if (DRT_HASH_VACANT(cmap, index) || DRT_HASH_GET_COUNT(cmap,index) == 0) {
4883 cmap->scm_buckets++;
4884 if (index < cmap->scm_lastclean)
4885 cmap->scm_lastclean = index;
4886 DRT_HASH_SET_ADDRESS(cmap, index, offset);
4887 DRT_HASH_SET_COUNT(cmap, index, 0);
4888 DRT_BITVECTOR_CLEAR(cmap, index);
4889 *indexp = index;
4890 vfs_drt_trace(cmap, DRT_DEBUG_INSERT, (int)offset, i, 0, 0);
4891 return(KERN_SUCCESS);
4892 }
4893 cmap->scm_iskips += i;
4894 index = DRT_HASH_NEXT(cmap, index);
4895 }
4896
4897 /*
4898 * We haven't found a vacant slot, so the map is full. If we're not
4899 * already recursed, try reallocating/compacting it.
4900 */
4901 if (recursed)
4902 return(KERN_FAILURE);
4903 kret = vfs_drt_alloc_map(cmapp);
4904 if (kret == KERN_SUCCESS) {
4905 /* now try to insert again */
4906 kret = vfs_drt_get_index(cmapp, offset, indexp, 1);
4907 }
4908 return(kret);
4909 }
4910
4911 /*
4912 * Implementation of set dirty/clean.
4913 *
4914 * In the 'clean' case, not finding a map is OK.
4915 */
4916 static kern_return_t
4917 vfs_drt_do_mark_pages(
4918 void **private,
4919 u_int64_t offset,
4920 u_int length,
4921 int *setcountp,
4922 int dirty)
4923 {
4924 struct vfs_drt_clustermap *cmap, **cmapp;
4925 kern_return_t kret;
4926 int i, index, pgoff, pgcount, setcount, ecount;
4927
4928 cmapp = (struct vfs_drt_clustermap **)private;
4929 cmap = *cmapp;
4930
4931 vfs_drt_trace(cmap, DRT_DEBUG_MARK | DBG_FUNC_START, (int)offset, (int)length, dirty, 0);
4932
4933 if (setcountp != NULL)
4934 *setcountp = 0;
4935
4936 /* allocate a cluster map if we don't already have one */
4937 if (cmap == NULL) {
4938 /* no cluster map, nothing to clean */
4939 if (!dirty) {
4940 vfs_drt_trace(cmap, DRT_DEBUG_MARK | DBG_FUNC_END, 1, 0, 0, 0);
4941 return(KERN_SUCCESS);
4942 }
4943 kret = vfs_drt_alloc_map(cmapp);
4944 if (kret != KERN_SUCCESS) {
4945 vfs_drt_trace(cmap, DRT_DEBUG_MARK | DBG_FUNC_END, 2, 0, 0, 0);
4946 return(kret);
4947 }
4948 }
4949 setcount = 0;
4950
4951 /*
4952 * Iterate over the length of the region.
4953 */
4954 while (length > 0) {
4955 /*
4956 * Get the hashtable index for this offset.
4957 *
4958 * XXX this will add blank entries if we are clearing a range
4959 * that hasn't been dirtied.
4960 */
4961 kret = vfs_drt_get_index(cmapp, offset, &index, 0);
4962 cmap = *cmapp; /* may have changed! */
4963 /* this may be a partial-success return */
4964 if (kret != KERN_SUCCESS) {
4965 if (setcountp != NULL)
4966 *setcountp = setcount;
4967 vfs_drt_trace(cmap, DRT_DEBUG_MARK | DBG_FUNC_END, 3, (int)length, 0, 0);
4968
4969 return(kret);
4970 }
4971
4972 /*
4973 * Work out how many pages we're modifying in this
4974 * hashtable entry.
4975 */
4976 pgoff = (offset - DRT_ALIGN_ADDRESS(offset)) / PAGE_SIZE;
4977 pgcount = min((length / PAGE_SIZE), (DRT_BITVECTOR_PAGES - pgoff));
4978
4979 /*
4980 * Iterate over pages, dirty/clearing as we go.
4981 */
4982 ecount = DRT_HASH_GET_COUNT(cmap, index);
4983 for (i = 0; i < pgcount; i++) {
4984 if (dirty) {
4985 if (!DRT_HASH_TEST_BIT(cmap, index, pgoff + i)) {
4986 DRT_HASH_SET_BIT(cmap, index, pgoff + i);
4987 ecount++;
4988 setcount++;
4989 }
4990 } else {
4991 if (DRT_HASH_TEST_BIT(cmap, index, pgoff + i)) {
4992 DRT_HASH_CLEAR_BIT(cmap, index, pgoff + i);
4993 ecount--;
4994 setcount++;
4995 }
4996 }
4997 }
4998 DRT_HASH_SET_COUNT(cmap, index, ecount);
4999
5000 offset += pgcount * PAGE_SIZE;
5001 length -= pgcount * PAGE_SIZE;
5002 }
5003 if (setcountp != NULL)
5004 *setcountp = setcount;
5005
5006 vfs_drt_trace(cmap, DRT_DEBUG_MARK | DBG_FUNC_END, 0, setcount, 0, 0);
5007
5008 return(KERN_SUCCESS);
5009 }
5010
5011 /*
5012 * Mark a set of pages as dirty/clean.
5013 *
5014 * This is a public interface.
5015 *
5016 * cmapp
5017 * Pointer to storage suitable for holding a pointer. Note that
5018 * this must either be NULL or a value set by this function.
5019 *
5020 * size
5021 * Current file size in bytes.
5022 *
5023 * offset
5024 * Offset of the first page to be marked as dirty, in bytes. Must be
5025 * page-aligned.
5026 *
5027 * length
5028 * Length of dirty region, in bytes. Must be a multiple of PAGE_SIZE.
5029 *
5030 * setcountp
5031 * Number of pages newly marked dirty by this call (optional).
5032 *
5033 * Returns KERN_SUCCESS if all the pages were successfully marked.
5034 */
5035 static kern_return_t
5036 vfs_drt_mark_pages(void **cmapp, off_t offset, u_int length, int *setcountp)
5037 {
5038 /* XXX size unused, drop from interface */
5039 return(vfs_drt_do_mark_pages(cmapp, offset, length, setcountp, 1));
5040 }
5041
5042 #if 0
5043 static kern_return_t
5044 vfs_drt_unmark_pages(void **cmapp, off_t offset, u_int length)
5045 {
5046 return(vfs_drt_do_mark_pages(cmapp, offset, length, NULL, 0));
5047 }
5048 #endif
5049
5050 /*
5051 * Get a cluster of dirty pages.
5052 *
5053 * This is a public interface.
5054 *
5055 * cmapp
5056 * Pointer to storage managed by drt_mark_pages. Note that this must
5057 * be NULL or a value set by drt_mark_pages.
5058 *
5059 * offsetp
5060 * Returns the byte offset into the file of the first page in the cluster.
5061 *
5062 * lengthp
5063 * Returns the length in bytes of the cluster of dirty pages.
5064 *
5065 * Returns success if a cluster was found. If KERN_FAILURE is returned, there
5066 * are no dirty pages meeting the minmum size criteria. Private storage will
5067 * be released if there are no more dirty pages left in the map
5068 *
5069 */
5070 static kern_return_t
5071 vfs_drt_get_cluster(void **cmapp, off_t *offsetp, u_int *lengthp)
5072 {
5073 struct vfs_drt_clustermap *cmap;
5074 u_int64_t offset;
5075 u_int length;
5076 int index, i, j, fs, ls;
5077
5078 /* sanity */
5079 if ((cmapp == NULL) || (*cmapp == NULL))
5080 return(KERN_FAILURE);
5081 cmap = *cmapp;
5082
5083 /* walk the hashtable */
5084 for (offset = 0, j = 0; j < cmap->scm_modulus; offset += (DRT_BITVECTOR_PAGES * PAGE_SIZE), j++) {
5085 index = DRT_HASH(cmap, offset);
5086
5087 if (DRT_HASH_VACANT(cmap, index) || (DRT_HASH_GET_COUNT(cmap, index) == 0))
5088 continue;
5089
5090 /* scan the bitfield for a string of bits */
5091 fs = -1;
5092
5093 for (i = 0; i < DRT_BITVECTOR_PAGES; i++) {
5094 if (DRT_HASH_TEST_BIT(cmap, index, i)) {
5095 fs = i;
5096 break;
5097 }
5098 }
5099 if (fs == -1) {
5100 /* didn't find any bits set */
5101 panic("vfs_drt: entry summary count > 0 but no bits set in map");
5102 }
5103 for (ls = 0; i < DRT_BITVECTOR_PAGES; i++, ls++) {
5104 if (!DRT_HASH_TEST_BIT(cmap, index, i))
5105 break;
5106 }
5107
5108 /* compute offset and length, mark pages clean */
5109 offset = DRT_HASH_GET_ADDRESS(cmap, index) + (PAGE_SIZE * fs);
5110 length = ls * PAGE_SIZE;
5111 vfs_drt_do_mark_pages(cmapp, offset, length, NULL, 0);
5112 cmap->scm_lastclean = index;
5113
5114 /* return successful */
5115 *offsetp = (off_t)offset;
5116 *lengthp = length;
5117
5118 vfs_drt_trace(cmap, DRT_DEBUG_RETCLUSTER, (int)offset, (int)length, 0, 0);
5119 return(KERN_SUCCESS);
5120 }
5121 /*
5122 * We didn't find anything... hashtable is empty
5123 * emit stats into trace buffer and
5124 * then free it
5125 */
5126 vfs_drt_trace(cmap, DRT_DEBUG_SCMDATA,
5127 cmap->scm_modulus,
5128 cmap->scm_buckets,
5129 cmap->scm_lastclean,
5130 cmap->scm_iskips);
5131
5132 vfs_drt_free_map(cmap);
5133 *cmapp = NULL;
5134
5135 return(KERN_FAILURE);
5136 }
5137
5138
5139 static kern_return_t
5140 vfs_drt_control(void **cmapp, int op_type)
5141 {
5142 struct vfs_drt_clustermap *cmap;
5143
5144 /* sanity */
5145 if ((cmapp == NULL) || (*cmapp == NULL))
5146 return(KERN_FAILURE);
5147 cmap = *cmapp;
5148
5149 switch (op_type) {
5150 case 0:
5151 /* emit stats into trace buffer */
5152 vfs_drt_trace(cmap, DRT_DEBUG_SCMDATA,
5153 cmap->scm_modulus,
5154 cmap->scm_buckets,
5155 cmap->scm_lastclean,
5156 cmap->scm_iskips);
5157
5158 vfs_drt_free_map(cmap);
5159 *cmapp = NULL;
5160 break;
5161
5162 case 1:
5163 cmap->scm_lastclean = 0;
5164 break;
5165 }
5166 return(KERN_SUCCESS);
5167 }
5168
5169
5170
5171 /*
5172 * Emit a summary of the state of the clustermap into the trace buffer
5173 * along with some caller-provided data.
5174 */
5175 #if KDEBUG
5176 static void
5177 vfs_drt_trace(__unused struct vfs_drt_clustermap *cmap, int code, int arg1, int arg2, int arg3, int arg4)
5178 {
5179 KERNEL_DEBUG(code, arg1, arg2, arg3, arg4, 0);
5180 }
5181 #else
5182 static void
5183 vfs_drt_trace(__unused struct vfs_drt_clustermap *cmap, __unused int code,
5184 __unused int arg1, __unused int arg2, __unused int arg3,
5185 __unused int arg4)
5186 {
5187 }
5188 #endif
5189
5190 #if 0
5191 /*
5192 * Perform basic sanity check on the hash entry summary count
5193 * vs. the actual bits set in the entry.
5194 */
5195 static void
5196 vfs_drt_sanity(struct vfs_drt_clustermap *cmap)
5197 {
5198 int index, i;
5199 int bits_on;
5200
5201 for (index = 0; index < cmap->scm_modulus; index++) {
5202 if (DRT_HASH_VACANT(cmap, index))
5203 continue;
5204
5205 for (bits_on = 0, i = 0; i < DRT_BITVECTOR_PAGES; i++) {
5206 if (DRT_HASH_TEST_BIT(cmap, index, i))
5207 bits_on++;
5208 }
5209 if (bits_on != DRT_HASH_GET_COUNT(cmap, index))
5210 panic("bits_on = %d, index = %d\n", bits_on, index);
5211 }
5212 }
5213 #endif