]> git.saurik.com Git - apple/xnu.git/blob - bsd/vfs/vfs_cluster.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-1699.22.81.tar.gz
[apple/xnu.git] / bsd / vfs / vfs_cluster.c
1 /*
2 * Copyright (c) 2000-2008 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
29 /*
30 * Copyright (c) 1993
31 * The Regents of the University of California. All rights reserved.
32 *
33 * Redistribution and use in source and binary forms, with or without
34 * modification, are permitted provided that the following conditions
35 * are met:
36 * 1. Redistributions of source code must retain the above copyright
37 * notice, this list of conditions and the following disclaimer.
38 * 2. Redistributions in binary form must reproduce the above copyright
39 * notice, this list of conditions and the following disclaimer in the
40 * documentation and/or other materials provided with the distribution.
41 * 3. All advertising materials mentioning features or use of this software
42 * must display the following acknowledgement:
43 * This product includes software developed by the University of
44 * California, Berkeley and its contributors.
45 * 4. Neither the name of the University nor the names of its contributors
46 * may be used to endorse or promote products derived from this software
47 * without specific prior written permission.
48 *
49 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
50 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
51 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
52 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
53 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
54 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
55 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
56 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
57 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
58 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
59 * SUCH DAMAGE.
60 *
61 * @(#)vfs_cluster.c 8.10 (Berkeley) 3/28/95
62 */
63
64 #include <sys/param.h>
65 #include <sys/proc_internal.h>
66 #include <sys/buf_internal.h>
67 #include <sys/mount_internal.h>
68 #include <sys/vnode_internal.h>
69 #include <sys/trace.h>
70 #include <sys/malloc.h>
71 #include <sys/time.h>
72 #include <sys/kernel.h>
73 #include <sys/resourcevar.h>
74 #include <sys/uio_internal.h>
75 #include <libkern/libkern.h>
76 #include <machine/machine_routines.h>
77
78 #include <sys/ubc_internal.h>
79 #include <vm/vnode_pager.h>
80
81 #include <mach/mach_types.h>
82 #include <mach/memory_object_types.h>
83 #include <mach/vm_map.h>
84 #include <mach/upl.h>
85 #include <kern/task.h>
86
87 #include <vm/vm_kern.h>
88 #include <vm/vm_map.h>
89 #include <vm/vm_pageout.h>
90
91 #include <sys/kdebug.h>
92 #include <libkern/OSAtomic.h>
93
94 #include <sys/sdt.h>
95
96 #if 0
97 #undef KERNEL_DEBUG
98 #define KERNEL_DEBUG KERNEL_DEBUG_CONSTANT
99 #endif
100
101
102 #define CL_READ 0x01
103 #define CL_WRITE 0x02
104 #define CL_ASYNC 0x04
105 #define CL_COMMIT 0x08
106 #define CL_PAGEOUT 0x10
107 #define CL_AGE 0x20
108 #define CL_NOZERO 0x40
109 #define CL_PAGEIN 0x80
110 #define CL_DEV_MEMORY 0x100
111 #define CL_PRESERVE 0x200
112 #define CL_THROTTLE 0x400
113 #define CL_KEEPCACHED 0x800
114 #define CL_DIRECT_IO 0x1000
115 #define CL_PASSIVE 0x2000
116 #define CL_IOSTREAMING 0x4000
117 #define CL_CLOSE 0x8000
118 #define CL_ENCRYPTED 0x10000
119
120 #define MAX_VECTOR_UPL_ELEMENTS 8
121 #define MAX_VECTOR_UPL_SIZE (2 * MAX_UPL_SIZE) * PAGE_SIZE
122
123 extern upl_t vector_upl_create(vm_offset_t);
124 extern boolean_t vector_upl_is_valid(upl_t);
125 extern boolean_t vector_upl_set_subupl(upl_t,upl_t, u_int32_t);
126 extern void vector_upl_set_pagelist(upl_t);
127 extern void vector_upl_set_iostate(upl_t, upl_t, vm_offset_t, u_int32_t);
128
129 struct clios {
130 lck_mtx_t io_mtxp;
131 u_int io_completed; /* amount of io that has currently completed */
132 u_int io_issued; /* amount of io that was successfully issued */
133 int io_error; /* error code of first error encountered */
134 int io_wanted; /* someone is sleeping waiting for a change in state */
135 };
136
137 static lck_grp_t *cl_mtx_grp;
138 static lck_attr_t *cl_mtx_attr;
139 static lck_grp_attr_t *cl_mtx_grp_attr;
140 static lck_mtx_t *cl_transaction_mtxp;
141
142
143 #define IO_UNKNOWN 0
144 #define IO_DIRECT 1
145 #define IO_CONTIG 2
146 #define IO_COPY 3
147
148 #define PUSH_DELAY 0x01
149 #define PUSH_ALL 0x02
150 #define PUSH_SYNC 0x04
151
152
153 static void cluster_EOT(buf_t cbp_head, buf_t cbp_tail, int zero_offset);
154 static void cluster_wait_IO(buf_t cbp_head, int async);
155 static void cluster_complete_transaction(buf_t *cbp_head, void *callback_arg, int *retval, int flags, int needwait);
156
157 static int cluster_io_type(struct uio *uio, int *io_type, u_int32_t *io_length, u_int32_t min_length);
158
159 static int cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int non_rounded_size,
160 int flags, buf_t real_bp, struct clios *iostate, int (*)(buf_t, void *), void *callback_arg);
161 static int cluster_iodone(buf_t bp, void *callback_arg);
162 static int cluster_ioerror(upl_t upl, int upl_offset, int abort_size, int error, int io_flags);
163 static int cluster_hard_throttle_on(vnode_t vp, uint32_t);
164
165 static void cluster_iostate_wait(struct clios *iostate, u_int target, const char *wait_name);
166
167 static void cluster_syncup(vnode_t vp, off_t newEOF, int (*)(buf_t, void *), void *callback_arg);
168
169 static void cluster_read_upl_release(upl_t upl, int start_pg, int last_pg, int take_reference);
170 static int cluster_copy_ubc_data_internal(vnode_t vp, struct uio *uio, int *io_resid, int mark_dirty, int take_reference);
171
172 static int cluster_read_copy(vnode_t vp, struct uio *uio, u_int32_t io_req_size, off_t filesize, int flags,
173 int (*)(buf_t, void *), void *callback_arg);
174 static int cluster_read_direct(vnode_t vp, struct uio *uio, off_t filesize, int *read_type, u_int32_t *read_length,
175 int flags, int (*)(buf_t, void *), void *callback_arg);
176 static int cluster_read_contig(vnode_t vp, struct uio *uio, off_t filesize, int *read_type, u_int32_t *read_length,
177 int (*)(buf_t, void *), void *callback_arg, int flags);
178
179 static int cluster_write_copy(vnode_t vp, struct uio *uio, u_int32_t io_req_size, off_t oldEOF, off_t newEOF,
180 off_t headOff, off_t tailOff, int flags, int (*)(buf_t, void *), void *callback_arg);
181 static int cluster_write_direct(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF,
182 int *write_type, u_int32_t *write_length, int flags, int (*)(buf_t, void *), void *callback_arg);
183 static int cluster_write_contig(vnode_t vp, struct uio *uio, off_t newEOF,
184 int *write_type, u_int32_t *write_length, int (*)(buf_t, void *), void *callback_arg, int bflag);
185
186 static int cluster_align_phys_io(vnode_t vp, struct uio *uio, addr64_t usr_paddr, u_int32_t xsize, int flags, int (*)(buf_t, void *), void *callback_arg);
187
188 static int cluster_read_prefetch(vnode_t vp, off_t f_offset, u_int size, off_t filesize, int (*callback)(buf_t, void *), void *callback_arg, int bflag);
189 static void cluster_read_ahead(vnode_t vp, struct cl_extent *extent, off_t filesize, struct cl_readahead *ra, int (*callback)(buf_t, void *), void *callback_arg, int bflag);
190
191 static int cluster_push_now(vnode_t vp, struct cl_extent *, off_t EOF, int flags, int (*)(buf_t, void *), void *callback_arg);
192
193 static int cluster_try_push(struct cl_writebehind *, vnode_t vp, off_t EOF, int push_flag, int flags, int (*)(buf_t, void *), void *callback_arg);
194
195 static void sparse_cluster_switch(struct cl_writebehind *, vnode_t vp, off_t EOF, int (*)(buf_t, void *), void *callback_arg);
196 static void sparse_cluster_push(void **cmapp, vnode_t vp, off_t EOF, int push_flag, int io_flags, int (*)(buf_t, void *), void *callback_arg);
197 static void sparse_cluster_add(void **cmapp, vnode_t vp, struct cl_extent *, off_t EOF, int (*)(buf_t, void *), void *callback_arg);
198
199 static kern_return_t vfs_drt_mark_pages(void **cmapp, off_t offset, u_int length, u_int *setcountp);
200 static kern_return_t vfs_drt_get_cluster(void **cmapp, off_t *offsetp, u_int *lengthp);
201 static kern_return_t vfs_drt_control(void **cmapp, int op_type);
202
203
204 /*
205 * limit the internal I/O size so that we
206 * can represent it in a 32 bit int
207 */
208 #define MAX_IO_REQUEST_SIZE (1024 * 1024 * 512)
209 #define MAX_IO_CONTIG_SIZE (MAX_UPL_SIZE * PAGE_SIZE)
210 #define MAX_VECTS 16
211 #define MIN_DIRECT_WRITE_SIZE (4 * PAGE_SIZE)
212
213 #define WRITE_THROTTLE 6
214 #define WRITE_THROTTLE_SSD 2
215 #define WRITE_BEHIND 1
216 #define WRITE_BEHIND_SSD 1
217 #define PREFETCH 3
218 #define PREFETCH_SSD 2
219
220 #define IO_SCALE(vp, base) (vp->v_mount->mnt_ioscale * base)
221 #define MAX_CLUSTER_SIZE(vp) (cluster_max_io_size(vp->v_mount, CL_WRITE))
222 #define MAX_PREFETCH(vp, size, is_ssd) (size * IO_SCALE(vp, (is_ssd && !ignore_is_ssd) ? PREFETCH_SSD : PREFETCH))
223
224 int ignore_is_ssd = 0;
225 int speculative_reads_disabled = 0;
226 uint32_t speculative_prefetch_max = (MAX_UPL_SIZE * 3);
227
228 /*
229 * throttle the number of async writes that
230 * can be outstanding on a single vnode
231 * before we issue a synchronous write
232 */
233 #define HARD_THROTTLE_MAXCNT 0
234 #define HARD_THROTTLE_MAXSIZE (32 * 1024)
235
236 int hard_throttle_on_root = 0;
237 struct timeval priority_IO_timestamp_for_root;
238
239
240 void
241 cluster_init(void) {
242 /*
243 * allocate lock group attribute and group
244 */
245 cl_mtx_grp_attr = lck_grp_attr_alloc_init();
246 cl_mtx_grp = lck_grp_alloc_init("cluster I/O", cl_mtx_grp_attr);
247
248 /*
249 * allocate the lock attribute
250 */
251 cl_mtx_attr = lck_attr_alloc_init();
252
253 cl_transaction_mtxp = lck_mtx_alloc_init(cl_mtx_grp, cl_mtx_attr);
254
255 if (cl_transaction_mtxp == NULL)
256 panic("cluster_init: failed to allocate cl_transaction_mtxp");
257 }
258
259
260 uint32_t
261 cluster_max_io_size(mount_t mp, int type)
262 {
263 uint32_t max_io_size;
264 uint32_t segcnt;
265 uint32_t maxcnt;
266
267 switch(type) {
268
269 case CL_READ:
270 segcnt = mp->mnt_segreadcnt;
271 maxcnt = mp->mnt_maxreadcnt;
272 break;
273 case CL_WRITE:
274 segcnt = mp->mnt_segwritecnt;
275 maxcnt = mp->mnt_maxwritecnt;
276 break;
277 default:
278 segcnt = min(mp->mnt_segreadcnt, mp->mnt_segwritecnt);
279 maxcnt = min(mp->mnt_maxreadcnt, mp->mnt_maxwritecnt);
280 break;
281 }
282 if (segcnt > MAX_UPL_SIZE) {
283 /*
284 * don't allow a size beyond the max UPL size we can create
285 */
286 segcnt = MAX_UPL_SIZE;
287 }
288 max_io_size = min((segcnt * PAGE_SIZE), maxcnt);
289
290 if (max_io_size < (MAX_UPL_TRANSFER * PAGE_SIZE)) {
291 /*
292 * don't allow a size smaller than the old fixed limit
293 */
294 max_io_size = (MAX_UPL_TRANSFER * PAGE_SIZE);
295 } else {
296 /*
297 * make sure the size specified is a multiple of PAGE_SIZE
298 */
299 max_io_size &= ~PAGE_MASK;
300 }
301 return (max_io_size);
302 }
303
304
305
306
307 #define CLW_ALLOCATE 0x01
308 #define CLW_RETURNLOCKED 0x02
309 #define CLW_IONOCACHE 0x04
310 #define CLW_IOPASSIVE 0x08
311
312 /*
313 * if the read ahead context doesn't yet exist,
314 * allocate and initialize it...
315 * the vnode lock serializes multiple callers
316 * during the actual assignment... first one
317 * to grab the lock wins... the other callers
318 * will release the now unnecessary storage
319 *
320 * once the context is present, try to grab (but don't block on)
321 * the lock associated with it... if someone
322 * else currently owns it, than the read
323 * will run without read-ahead. this allows
324 * multiple readers to run in parallel and
325 * since there's only 1 read ahead context,
326 * there's no real loss in only allowing 1
327 * reader to have read-ahead enabled.
328 */
329 static struct cl_readahead *
330 cluster_get_rap(vnode_t vp)
331 {
332 struct ubc_info *ubc;
333 struct cl_readahead *rap;
334
335 ubc = vp->v_ubcinfo;
336
337 if ((rap = ubc->cl_rahead) == NULL) {
338 MALLOC_ZONE(rap, struct cl_readahead *, sizeof *rap, M_CLRDAHEAD, M_WAITOK);
339
340 bzero(rap, sizeof *rap);
341 rap->cl_lastr = -1;
342 lck_mtx_init(&rap->cl_lockr, cl_mtx_grp, cl_mtx_attr);
343
344 vnode_lock(vp);
345
346 if (ubc->cl_rahead == NULL)
347 ubc->cl_rahead = rap;
348 else {
349 lck_mtx_destroy(&rap->cl_lockr, cl_mtx_grp);
350 FREE_ZONE((void *)rap, sizeof *rap, M_CLRDAHEAD);
351 rap = ubc->cl_rahead;
352 }
353 vnode_unlock(vp);
354 }
355 if (lck_mtx_try_lock(&rap->cl_lockr) == TRUE)
356 return(rap);
357
358 return ((struct cl_readahead *)NULL);
359 }
360
361
362 /*
363 * if the write behind context doesn't yet exist,
364 * and CLW_ALLOCATE is specified, allocate and initialize it...
365 * the vnode lock serializes multiple callers
366 * during the actual assignment... first one
367 * to grab the lock wins... the other callers
368 * will release the now unnecessary storage
369 *
370 * if CLW_RETURNLOCKED is set, grab (blocking if necessary)
371 * the lock associated with the write behind context before
372 * returning
373 */
374
375 static struct cl_writebehind *
376 cluster_get_wbp(vnode_t vp, int flags)
377 {
378 struct ubc_info *ubc;
379 struct cl_writebehind *wbp;
380
381 ubc = vp->v_ubcinfo;
382
383 if ((wbp = ubc->cl_wbehind) == NULL) {
384
385 if ( !(flags & CLW_ALLOCATE))
386 return ((struct cl_writebehind *)NULL);
387
388 MALLOC_ZONE(wbp, struct cl_writebehind *, sizeof *wbp, M_CLWRBEHIND, M_WAITOK);
389
390 bzero(wbp, sizeof *wbp);
391 lck_mtx_init(&wbp->cl_lockw, cl_mtx_grp, cl_mtx_attr);
392
393 vnode_lock(vp);
394
395 if (ubc->cl_wbehind == NULL)
396 ubc->cl_wbehind = wbp;
397 else {
398 lck_mtx_destroy(&wbp->cl_lockw, cl_mtx_grp);
399 FREE_ZONE((void *)wbp, sizeof *wbp, M_CLWRBEHIND);
400 wbp = ubc->cl_wbehind;
401 }
402 vnode_unlock(vp);
403 }
404 if (flags & CLW_RETURNLOCKED)
405 lck_mtx_lock(&wbp->cl_lockw);
406
407 return (wbp);
408 }
409
410
411 static void
412 cluster_syncup(vnode_t vp, off_t newEOF, int (*callback)(buf_t, void *), void *callback_arg)
413 {
414 struct cl_writebehind *wbp;
415
416 if ((wbp = cluster_get_wbp(vp, 0)) != NULL) {
417
418 if (wbp->cl_number) {
419 lck_mtx_lock(&wbp->cl_lockw);
420
421 cluster_try_push(wbp, vp, newEOF, PUSH_ALL | PUSH_SYNC, 0, callback, callback_arg);
422
423 lck_mtx_unlock(&wbp->cl_lockw);
424 }
425 }
426 }
427
428
429 static int
430 cluster_hard_throttle_on(vnode_t vp, uint32_t hard_throttle)
431 {
432 struct uthread *ut;
433
434 if (hard_throttle) {
435 static struct timeval hard_throttle_maxelapsed = { 0, 200000 };
436
437 if (vp->v_mount->mnt_kern_flag & MNTK_ROOTDEV) {
438 struct timeval elapsed;
439
440 if (hard_throttle_on_root)
441 return(1);
442
443 microuptime(&elapsed);
444 timevalsub(&elapsed, &priority_IO_timestamp_for_root);
445
446 if (timevalcmp(&elapsed, &hard_throttle_maxelapsed, <))
447 return(1);
448 }
449 }
450 if (throttle_get_io_policy(&ut) == IOPOL_THROTTLE) {
451 if (throttle_io_will_be_throttled(-1, vp->v_mount)) {
452 return(1);
453 }
454 }
455 return(0);
456 }
457
458
459 static void
460 cluster_iostate_wait(struct clios *iostate, u_int target, const char *wait_name)
461 {
462
463 lck_mtx_lock(&iostate->io_mtxp);
464
465 while ((iostate->io_issued - iostate->io_completed) > target) {
466
467 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_START,
468 iostate->io_issued, iostate->io_completed, target, 0, 0);
469
470 iostate->io_wanted = 1;
471 msleep((caddr_t)&iostate->io_wanted, &iostate->io_mtxp, PRIBIO + 1, wait_name, NULL);
472
473 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 95)) | DBG_FUNC_END,
474 iostate->io_issued, iostate->io_completed, target, 0, 0);
475 }
476 lck_mtx_unlock(&iostate->io_mtxp);
477 }
478
479
480 static int
481 cluster_ioerror(upl_t upl, int upl_offset, int abort_size, int error, int io_flags)
482 {
483 int upl_abort_code = 0;
484 int page_in = 0;
485 int page_out = 0;
486
487 if ((io_flags & (B_PHYS | B_CACHE)) == (B_PHYS | B_CACHE))
488 /*
489 * direct write of any flavor, or a direct read that wasn't aligned
490 */
491 ubc_upl_commit_range(upl, upl_offset, abort_size, UPL_COMMIT_FREE_ON_EMPTY);
492 else {
493 if (io_flags & B_PAGEIO) {
494 if (io_flags & B_READ)
495 page_in = 1;
496 else
497 page_out = 1;
498 }
499 if (io_flags & B_CACHE)
500 /*
501 * leave pages in the cache unchanged on error
502 */
503 upl_abort_code = UPL_ABORT_FREE_ON_EMPTY;
504 else if (page_out && (error != ENXIO))
505 /*
506 * transient error... leave pages unchanged
507 */
508 upl_abort_code = UPL_ABORT_FREE_ON_EMPTY;
509 else if (page_in)
510 upl_abort_code = UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR;
511 else
512 upl_abort_code = UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_DUMP_PAGES;
513
514 ubc_upl_abort_range(upl, upl_offset, abort_size, upl_abort_code);
515 }
516 return (upl_abort_code);
517 }
518
519
520 static int
521 cluster_iodone(buf_t bp, void *callback_arg)
522 {
523 int b_flags;
524 int error;
525 int total_size;
526 int total_resid;
527 int upl_offset;
528 int zero_offset;
529 int pg_offset = 0;
530 int commit_size = 0;
531 int upl_flags = 0;
532 int transaction_size = 0;
533 upl_t upl;
534 buf_t cbp;
535 buf_t cbp_head;
536 buf_t cbp_next;
537 buf_t real_bp;
538 struct clios *iostate;
539 boolean_t transaction_complete = FALSE;
540
541 cbp_head = (buf_t)(bp->b_trans_head);
542
543 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 20)) | DBG_FUNC_START,
544 cbp_head, bp->b_lblkno, bp->b_bcount, bp->b_flags, 0);
545
546 if (cbp_head->b_trans_next || !(cbp_head->b_flags & B_EOT)) {
547 boolean_t need_wakeup = FALSE;
548
549 lck_mtx_lock_spin(cl_transaction_mtxp);
550
551 bp->b_flags |= B_TDONE;
552
553 if (bp->b_flags & B_TWANTED) {
554 CLR(bp->b_flags, B_TWANTED);
555 need_wakeup = TRUE;
556 }
557 for (cbp = cbp_head; cbp; cbp = cbp->b_trans_next) {
558 /*
559 * all I/O requests that are part of this transaction
560 * have to complete before we can process it
561 */
562 if ( !(cbp->b_flags & B_TDONE)) {
563
564 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 20)) | DBG_FUNC_END,
565 cbp_head, cbp, cbp->b_bcount, cbp->b_flags, 0);
566
567 lck_mtx_unlock(cl_transaction_mtxp);
568
569 if (need_wakeup == TRUE)
570 wakeup(bp);
571
572 return 0;
573 }
574 if (cbp->b_flags & B_EOT)
575 transaction_complete = TRUE;
576 }
577 lck_mtx_unlock(cl_transaction_mtxp);
578
579 if (need_wakeup == TRUE)
580 wakeup(bp);
581
582 if (transaction_complete == FALSE) {
583 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 20)) | DBG_FUNC_END,
584 cbp_head, 0, 0, 0, 0);
585 return 0;
586 }
587 }
588 error = 0;
589 total_size = 0;
590 total_resid = 0;
591
592 cbp = cbp_head;
593 upl_offset = cbp->b_uploffset;
594 upl = cbp->b_upl;
595 b_flags = cbp->b_flags;
596 real_bp = cbp->b_real_bp;
597 zero_offset= cbp->b_validend;
598 iostate = (struct clios *)cbp->b_iostate;
599
600 if (real_bp)
601 real_bp->b_dev = cbp->b_dev;
602
603 while (cbp) {
604 if ((cbp->b_flags & B_ERROR) && error == 0)
605 error = cbp->b_error;
606
607 total_resid += cbp->b_resid;
608 total_size += cbp->b_bcount;
609
610 cbp_next = cbp->b_trans_next;
611
612 if (cbp_next == NULL)
613 /*
614 * compute the overall size of the transaction
615 * in case we created one that has 'holes' in it
616 * 'total_size' represents the amount of I/O we
617 * did, not the span of the transaction w/r to the UPL
618 */
619 transaction_size = cbp->b_uploffset + cbp->b_bcount - upl_offset;
620
621 if (cbp != cbp_head)
622 free_io_buf(cbp);
623
624 cbp = cbp_next;
625 }
626 if (error == 0 && total_resid)
627 error = EIO;
628
629 if (error == 0) {
630 int (*cliodone_func)(buf_t, void *) = (int (*)(buf_t, void *))(cbp_head->b_cliodone);
631
632 if (cliodone_func != NULL) {
633 cbp_head->b_bcount = transaction_size;
634
635 error = (*cliodone_func)(cbp_head, callback_arg);
636 }
637 }
638 if (zero_offset)
639 cluster_zero(upl, zero_offset, PAGE_SIZE - (zero_offset & PAGE_MASK), real_bp);
640
641 free_io_buf(cbp_head);
642
643 if (iostate) {
644 int need_wakeup = 0;
645
646 /*
647 * someone has issued multiple I/Os asynchrounsly
648 * and is waiting for them to complete (streaming)
649 */
650 lck_mtx_lock_spin(&iostate->io_mtxp);
651
652 if (error && iostate->io_error == 0)
653 iostate->io_error = error;
654
655 iostate->io_completed += total_size;
656
657 if (iostate->io_wanted) {
658 /*
659 * someone is waiting for the state of
660 * this io stream to change
661 */
662 iostate->io_wanted = 0;
663 need_wakeup = 1;
664 }
665 lck_mtx_unlock(&iostate->io_mtxp);
666
667 if (need_wakeup)
668 wakeup((caddr_t)&iostate->io_wanted);
669 }
670
671 if (b_flags & B_COMMIT_UPL) {
672
673 pg_offset = upl_offset & PAGE_MASK;
674 commit_size = (pg_offset + transaction_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
675
676 if (error)
677 upl_flags = cluster_ioerror(upl, upl_offset - pg_offset, commit_size, error, b_flags);
678 else {
679 upl_flags = UPL_COMMIT_FREE_ON_EMPTY;
680
681 if ((b_flags & B_PHYS) && (b_flags & B_READ))
682 upl_flags |= UPL_COMMIT_SET_DIRTY;
683
684 if (b_flags & B_AGE)
685 upl_flags |= UPL_COMMIT_INACTIVATE;
686
687 ubc_upl_commit_range(upl, upl_offset - pg_offset, commit_size, upl_flags);
688 }
689 }
690 if (real_bp) {
691 if (error) {
692 real_bp->b_flags |= B_ERROR;
693 real_bp->b_error = error;
694 }
695 real_bp->b_resid = total_resid;
696
697 buf_biodone(real_bp);
698 }
699 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 20)) | DBG_FUNC_END,
700 upl, upl_offset - pg_offset, commit_size, (error << 24) | upl_flags, 0);
701
702 return (error);
703 }
704
705
706 uint32_t
707 cluster_hard_throttle_limit(vnode_t vp, uint32_t *limit, uint32_t hard_throttle)
708 {
709 if (cluster_hard_throttle_on(vp, hard_throttle)) {
710 *limit = HARD_THROTTLE_MAXSIZE;
711 return 1;
712 }
713 return 0;
714 }
715
716
717 void
718 cluster_zero(upl_t upl, upl_offset_t upl_offset, int size, buf_t bp)
719 {
720
721 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 23)) | DBG_FUNC_START,
722 upl_offset, size, bp, 0, 0);
723
724 if (bp == NULL || bp->b_datap == 0) {
725 upl_page_info_t *pl;
726 addr64_t zero_addr;
727
728 pl = ubc_upl_pageinfo(upl);
729
730 if (upl_device_page(pl) == TRUE) {
731 zero_addr = ((addr64_t)upl_phys_page(pl, 0) << 12) + upl_offset;
732
733 bzero_phys_nc(zero_addr, size);
734 } else {
735 while (size) {
736 int page_offset;
737 int page_index;
738 int zero_cnt;
739
740 page_index = upl_offset / PAGE_SIZE;
741 page_offset = upl_offset & PAGE_MASK;
742
743 zero_addr = ((addr64_t)upl_phys_page(pl, page_index) << 12) + page_offset;
744 zero_cnt = min(PAGE_SIZE - page_offset, size);
745
746 bzero_phys(zero_addr, zero_cnt);
747
748 size -= zero_cnt;
749 upl_offset += zero_cnt;
750 }
751 }
752 } else
753 bzero((caddr_t)((vm_offset_t)bp->b_datap + upl_offset), size);
754
755 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 23)) | DBG_FUNC_END,
756 upl_offset, size, 0, 0, 0);
757 }
758
759
760 static void
761 cluster_EOT(buf_t cbp_head, buf_t cbp_tail, int zero_offset)
762 {
763 cbp_head->b_validend = zero_offset;
764 cbp_tail->b_flags |= B_EOT;
765 }
766
767 static void
768 cluster_wait_IO(buf_t cbp_head, int async)
769 {
770 buf_t cbp;
771
772 if (async) {
773 /*
774 * async callback completion will not normally
775 * generate a wakeup upon I/O completion...
776 * by setting B_TWANTED, we will force a wakeup
777 * to occur as any outstanding I/Os complete...
778 * I/Os already completed will have B_TDONE already
779 * set and we won't cause us to block
780 * note that we're actually waiting for the bp to have
781 * completed the callback function... only then
782 * can we safely take back ownership of the bp
783 */
784 lck_mtx_lock_spin(cl_transaction_mtxp);
785
786 for (cbp = cbp_head; cbp; cbp = cbp->b_trans_next)
787 cbp->b_flags |= B_TWANTED;
788
789 lck_mtx_unlock(cl_transaction_mtxp);
790 }
791 for (cbp = cbp_head; cbp; cbp = cbp->b_trans_next) {
792
793 if (async) {
794 while (!ISSET(cbp->b_flags, B_TDONE)) {
795
796 lck_mtx_lock_spin(cl_transaction_mtxp);
797
798 if (!ISSET(cbp->b_flags, B_TDONE)) {
799 DTRACE_IO1(wait__start, buf_t, cbp);
800 (void) msleep(cbp, cl_transaction_mtxp, PDROP | (PRIBIO+1), "cluster_wait_IO", NULL);
801 DTRACE_IO1(wait__done, buf_t, cbp);
802 } else
803 lck_mtx_unlock(cl_transaction_mtxp);
804 }
805 } else
806 buf_biowait(cbp);
807 }
808 }
809
810 static void
811 cluster_complete_transaction(buf_t *cbp_head, void *callback_arg, int *retval, int flags, int needwait)
812 {
813 buf_t cbp;
814 int error;
815
816 /*
817 * cluster_complete_transaction will
818 * only be called if we've issued a complete chain in synchronous mode
819 * or, we've already done a cluster_wait_IO on an incomplete chain
820 */
821 if (needwait) {
822 for (cbp = *cbp_head; cbp; cbp = cbp->b_trans_next)
823 buf_biowait(cbp);
824 }
825 /*
826 * we've already waited on all of the I/Os in this transaction,
827 * so mark all of the buf_t's in this transaction as B_TDONE
828 * so that cluster_iodone sees the transaction as completed
829 */
830 for (cbp = *cbp_head; cbp; cbp = cbp->b_trans_next)
831 cbp->b_flags |= B_TDONE;
832
833 error = cluster_iodone(*cbp_head, callback_arg);
834
835 if ( !(flags & CL_ASYNC) && error && *retval == 0) {
836 if (((flags & (CL_PAGEOUT | CL_KEEPCACHED)) != CL_PAGEOUT) || (error != ENXIO))
837 *retval = error;
838 }
839 *cbp_head = (buf_t)NULL;
840 }
841
842
843 static int
844 cluster_io(vnode_t vp, upl_t upl, vm_offset_t upl_offset, off_t f_offset, int non_rounded_size,
845 int flags, buf_t real_bp, struct clios *iostate, int (*callback)(buf_t, void *), void *callback_arg)
846 {
847 buf_t cbp;
848 u_int size;
849 u_int io_size;
850 int io_flags;
851 int bmap_flags;
852 int error = 0;
853 int retval = 0;
854 buf_t cbp_head = NULL;
855 buf_t cbp_tail = NULL;
856 int trans_count = 0;
857 int max_trans_count;
858 u_int pg_count;
859 int pg_offset;
860 u_int max_iosize;
861 u_int max_vectors;
862 int priv;
863 int zero_offset = 0;
864 int async_throttle = 0;
865 mount_t mp;
866 vm_offset_t upl_end_offset;
867 boolean_t need_EOT = FALSE;
868
869 /*
870 * we currently don't support buffers larger than a page
871 */
872 if (real_bp && non_rounded_size > PAGE_SIZE)
873 panic("%s(): Called with real buffer of size %d bytes which "
874 "is greater than the maximum allowed size of "
875 "%d bytes (the system PAGE_SIZE).\n",
876 __FUNCTION__, non_rounded_size, PAGE_SIZE);
877
878 mp = vp->v_mount;
879
880 /*
881 * we don't want to do any funny rounding of the size for IO requests
882 * coming through the DIRECT or CONTIGUOUS paths... those pages don't
883 * belong to us... we can't extend (nor do we need to) the I/O to fill
884 * out a page
885 */
886 if (mp->mnt_devblocksize > 1 && !(flags & (CL_DEV_MEMORY | CL_DIRECT_IO))) {
887 /*
888 * round the requested size up so that this I/O ends on a
889 * page boundary in case this is a 'write'... if the filesystem
890 * has blocks allocated to back the page beyond the EOF, we want to
891 * make sure to write out the zero's that are sitting beyond the EOF
892 * so that in case the filesystem doesn't explicitly zero this area
893 * if a hole is created via a lseek/write beyond the current EOF,
894 * it will return zeros when it's read back from the disk. If the
895 * physical allocation doesn't extend for the whole page, we'll
896 * only write/read from the disk up to the end of this allocation
897 * via the extent info returned from the VNOP_BLOCKMAP call.
898 */
899 pg_offset = upl_offset & PAGE_MASK;
900
901 size = (((non_rounded_size + pg_offset) + (PAGE_SIZE - 1)) & ~PAGE_MASK) - pg_offset;
902 } else {
903 /*
904 * anyone advertising a blocksize of 1 byte probably
905 * can't deal with us rounding up the request size
906 * AFP is one such filesystem/device
907 */
908 size = non_rounded_size;
909 }
910 upl_end_offset = upl_offset + size;
911
912 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 22)) | DBG_FUNC_START, (int)f_offset, size, upl_offset, flags, 0);
913
914 /*
915 * Set the maximum transaction size to the maximum desired number of
916 * buffers.
917 */
918 max_trans_count = 8;
919 if (flags & CL_DEV_MEMORY)
920 max_trans_count = 16;
921
922 if (flags & CL_READ) {
923 io_flags = B_READ;
924 bmap_flags = VNODE_READ;
925
926 max_iosize = mp->mnt_maxreadcnt;
927 max_vectors = mp->mnt_segreadcnt;
928 } else {
929 io_flags = B_WRITE;
930 bmap_flags = VNODE_WRITE;
931
932 max_iosize = mp->mnt_maxwritecnt;
933 max_vectors = mp->mnt_segwritecnt;
934 }
935 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 22)) | DBG_FUNC_NONE, max_iosize, max_vectors, mp->mnt_devblocksize, 0, 0);
936
937 /*
938 * make sure the maximum iosize is a
939 * multiple of the page size
940 */
941 max_iosize &= ~PAGE_MASK;
942
943 /*
944 * Ensure the maximum iosize is sensible.
945 */
946 if (!max_iosize)
947 max_iosize = PAGE_SIZE;
948
949 if (flags & CL_THROTTLE) {
950 if ( !(flags & CL_PAGEOUT) && cluster_hard_throttle_on(vp, 1)) {
951 if (max_iosize > HARD_THROTTLE_MAXSIZE)
952 max_iosize = HARD_THROTTLE_MAXSIZE;
953 async_throttle = HARD_THROTTLE_MAXCNT;
954 } else {
955 if ( (flags & CL_DEV_MEMORY) )
956 async_throttle = IO_SCALE(vp, VNODE_ASYNC_THROTTLE);
957 else {
958 u_int max_cluster;
959 u_int max_cluster_size;
960 u_int scale;
961
962 max_cluster_size = MAX_CLUSTER_SIZE(vp);
963
964 if (max_iosize > max_cluster_size)
965 max_cluster = max_cluster_size;
966 else
967 max_cluster = max_iosize;
968
969 if (size < max_cluster)
970 max_cluster = size;
971
972 if ((vp->v_mount->mnt_kern_flag & MNTK_SSD) && !ignore_is_ssd)
973 scale = WRITE_THROTTLE_SSD;
974 else
975 scale = WRITE_THROTTLE;
976
977 if (flags & CL_CLOSE)
978 scale += MAX_CLUSTERS;
979
980 async_throttle = min(IO_SCALE(vp, VNODE_ASYNC_THROTTLE), ((scale * max_cluster_size) / max_cluster) - 1);
981 }
982 }
983 }
984 if (flags & CL_AGE)
985 io_flags |= B_AGE;
986 if (flags & (CL_PAGEIN | CL_PAGEOUT))
987 io_flags |= B_PAGEIO;
988 if (flags & (CL_IOSTREAMING))
989 io_flags |= B_IOSTREAMING;
990 if (flags & CL_COMMIT)
991 io_flags |= B_COMMIT_UPL;
992 if (flags & CL_DIRECT_IO)
993 io_flags |= B_PHYS;
994 if (flags & (CL_PRESERVE | CL_KEEPCACHED))
995 io_flags |= B_CACHE;
996 if (flags & CL_PASSIVE)
997 io_flags |= B_PASSIVE;
998 if (flags & CL_ENCRYPTED)
999 io_flags |= B_ENCRYPTED_IO;
1000 if (vp->v_flag & VSYSTEM)
1001 io_flags |= B_META;
1002
1003 if ((flags & CL_READ) && ((upl_offset + non_rounded_size) & PAGE_MASK) && (!(flags & CL_NOZERO))) {
1004 /*
1005 * then we are going to end up
1006 * with a page that we can't complete (the file size wasn't a multiple
1007 * of PAGE_SIZE and we're trying to read to the end of the file
1008 * so we'll go ahead and zero out the portion of the page we can't
1009 * read in from the file
1010 */
1011 zero_offset = upl_offset + non_rounded_size;
1012 }
1013 while (size) {
1014 daddr64_t blkno;
1015 daddr64_t lblkno;
1016 u_int io_size_wanted;
1017 size_t io_size_tmp;
1018
1019 if (size > max_iosize)
1020 io_size = max_iosize;
1021 else
1022 io_size = size;
1023
1024 io_size_wanted = io_size;
1025 io_size_tmp = (size_t)io_size;
1026
1027 if ((error = VNOP_BLOCKMAP(vp, f_offset, io_size, &blkno, &io_size_tmp, NULL, bmap_flags, NULL)))
1028 break;
1029
1030 if (io_size_tmp > io_size_wanted)
1031 io_size = io_size_wanted;
1032 else
1033 io_size = (u_int)io_size_tmp;
1034
1035 if (real_bp && (real_bp->b_blkno == real_bp->b_lblkno))
1036 real_bp->b_blkno = blkno;
1037
1038 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 24)) | DBG_FUNC_NONE,
1039 (int)f_offset, (int)(blkno>>32), (int)blkno, io_size, 0);
1040
1041 if (io_size == 0) {
1042 /*
1043 * vnop_blockmap didn't return an error... however, it did
1044 * return an extent size of 0 which means we can't
1045 * make forward progress on this I/O... a hole in the
1046 * file would be returned as a blkno of -1 with a non-zero io_size
1047 * a real extent is returned with a blkno != -1 and a non-zero io_size
1048 */
1049 error = EINVAL;
1050 break;
1051 }
1052 if ( !(flags & CL_READ) && blkno == -1) {
1053 off_t e_offset;
1054 int pageout_flags;
1055
1056 if (upl_get_internal_vectorupl(upl))
1057 panic("Vector UPLs should not take this code-path\n");
1058 /*
1059 * we're writing into a 'hole'
1060 */
1061 if (flags & CL_PAGEOUT) {
1062 /*
1063 * if we got here via cluster_pageout
1064 * then just error the request and return
1065 * the 'hole' should already have been covered
1066 */
1067 error = EINVAL;
1068 break;
1069 }
1070 /*
1071 * we can get here if the cluster code happens to
1072 * pick up a page that was dirtied via mmap vs
1073 * a 'write' and the page targets a 'hole'...
1074 * i.e. the writes to the cluster were sparse
1075 * and the file was being written for the first time
1076 *
1077 * we can also get here if the filesystem supports
1078 * 'holes' that are less than PAGE_SIZE.... because
1079 * we can't know if the range in the page that covers
1080 * the 'hole' has been dirtied via an mmap or not,
1081 * we have to assume the worst and try to push the
1082 * entire page to storage.
1083 *
1084 * Try paging out the page individually before
1085 * giving up entirely and dumping it (the pageout
1086 * path will insure that the zero extent accounting
1087 * has been taken care of before we get back into cluster_io)
1088 *
1089 * go direct to vnode_pageout so that we don't have to
1090 * unbusy the page from the UPL... we used to do this
1091 * so that we could call ubc_sync_range, but that results
1092 * in a potential deadlock if someone else races us to acquire
1093 * that page and wins and in addition needs one of the pages
1094 * we're continuing to hold in the UPL
1095 */
1096 pageout_flags = UPL_MSYNC | UPL_VNODE_PAGER | UPL_NESTED_PAGEOUT;
1097
1098 if ( !(flags & CL_ASYNC))
1099 pageout_flags |= UPL_IOSYNC;
1100 if ( !(flags & CL_COMMIT))
1101 pageout_flags |= UPL_NOCOMMIT;
1102
1103 if (cbp_head) {
1104 buf_t last_cbp;
1105
1106 /*
1107 * first we have to wait for the the current outstanding I/Os
1108 * to complete... EOT hasn't been set yet on this transaction
1109 * so the pages won't be released just because all of the current
1110 * I/O linked to this transaction has completed...
1111 */
1112 cluster_wait_IO(cbp_head, (flags & CL_ASYNC));
1113
1114 /*
1115 * we've got a transcation that
1116 * includes the page we're about to push out through vnode_pageout...
1117 * find the last bp in the list which will be the one that
1118 * includes the head of this page and round it's iosize down
1119 * to a page boundary...
1120 */
1121 for (last_cbp = cbp = cbp_head; cbp->b_trans_next; cbp = cbp->b_trans_next)
1122 last_cbp = cbp;
1123
1124 cbp->b_bcount &= ~PAGE_MASK;
1125
1126 if (cbp->b_bcount == 0) {
1127 /*
1128 * this buf no longer has any I/O associated with it
1129 */
1130 free_io_buf(cbp);
1131
1132 if (cbp == cbp_head) {
1133 /*
1134 * the buf we just freed was the only buf in
1135 * this transaction... so there's no I/O to do
1136 */
1137 cbp_head = NULL;
1138 } else {
1139 /*
1140 * remove the buf we just freed from
1141 * the transaction list
1142 */
1143 last_cbp->b_trans_next = NULL;
1144 cbp_tail = last_cbp;
1145 }
1146 }
1147 if (cbp_head) {
1148 /*
1149 * there was more to the current transaction
1150 * than just the page we are pushing out via vnode_pageout...
1151 * mark it as finished and complete it... we've already
1152 * waited for the I/Os to complete above in the call to cluster_wait_IO
1153 */
1154 cluster_EOT(cbp_head, cbp_tail, 0);
1155
1156 cluster_complete_transaction(&cbp_head, callback_arg, &retval, flags, 0);
1157
1158 trans_count = 0;
1159 }
1160 }
1161 if (vnode_pageout(vp, upl, trunc_page(upl_offset), trunc_page_64(f_offset), PAGE_SIZE, pageout_flags, NULL) != PAGER_SUCCESS) {
1162 error = EINVAL;
1163 }
1164 e_offset = round_page_64(f_offset + 1);
1165 io_size = e_offset - f_offset;
1166
1167 f_offset += io_size;
1168 upl_offset += io_size;
1169
1170 if (size >= io_size)
1171 size -= io_size;
1172 else
1173 size = 0;
1174 /*
1175 * keep track of how much of the original request
1176 * that we've actually completed... non_rounded_size
1177 * may go negative due to us rounding the request
1178 * to a page size multiple (i.e. size > non_rounded_size)
1179 */
1180 non_rounded_size -= io_size;
1181
1182 if (non_rounded_size <= 0) {
1183 /*
1184 * we've transferred all of the data in the original
1185 * request, but we were unable to complete the tail
1186 * of the last page because the file didn't have
1187 * an allocation to back that portion... this is ok.
1188 */
1189 size = 0;
1190 }
1191 if (error) {
1192 if (size == 0)
1193 flags &= ~CL_COMMIT;
1194 break;
1195 }
1196 continue;
1197 }
1198 lblkno = (daddr64_t)(f_offset / PAGE_SIZE_64);
1199 /*
1200 * we have now figured out how much I/O we can do - this is in 'io_size'
1201 * pg_offset is the starting point in the first page for the I/O
1202 * pg_count is the number of full and partial pages that 'io_size' encompasses
1203 */
1204 pg_offset = upl_offset & PAGE_MASK;
1205
1206 if (flags & CL_DEV_MEMORY) {
1207 /*
1208 * treat physical requests as one 'giant' page
1209 */
1210 pg_count = 1;
1211 } else
1212 pg_count = (io_size + pg_offset + (PAGE_SIZE - 1)) / PAGE_SIZE;
1213
1214 if ((flags & CL_READ) && blkno == -1) {
1215 vm_offset_t commit_offset;
1216 int bytes_to_zero;
1217 int complete_transaction_now = 0;
1218
1219 /*
1220 * if we're reading and blkno == -1, then we've got a
1221 * 'hole' in the file that we need to deal with by zeroing
1222 * out the affected area in the upl
1223 */
1224 if (io_size >= (u_int)non_rounded_size) {
1225 /*
1226 * if this upl contains the EOF and it is not a multiple of PAGE_SIZE
1227 * than 'zero_offset' will be non-zero
1228 * if the 'hole' returned by vnop_blockmap extends all the way to the eof
1229 * (indicated by the io_size finishing off the I/O request for this UPL)
1230 * than we're not going to issue an I/O for the
1231 * last page in this upl... we need to zero both the hole and the tail
1232 * of the page beyond the EOF, since the delayed zero-fill won't kick in
1233 */
1234 bytes_to_zero = non_rounded_size;
1235 if (!(flags & CL_NOZERO))
1236 bytes_to_zero = (((upl_offset + io_size) + (PAGE_SIZE - 1)) & ~PAGE_MASK) - upl_offset;
1237
1238 zero_offset = 0;
1239 } else
1240 bytes_to_zero = io_size;
1241
1242 pg_count = 0;
1243
1244 cluster_zero(upl, upl_offset, bytes_to_zero, real_bp);
1245
1246 if (cbp_head) {
1247 int pg_resid;
1248
1249 /*
1250 * if there is a current I/O chain pending
1251 * then the first page of the group we just zero'd
1252 * will be handled by the I/O completion if the zero
1253 * fill started in the middle of the page
1254 */
1255 commit_offset = (upl_offset + (PAGE_SIZE - 1)) & ~PAGE_MASK;
1256
1257 pg_resid = commit_offset - upl_offset;
1258
1259 if (bytes_to_zero >= pg_resid) {
1260 /*
1261 * the last page of the current I/O
1262 * has been completed...
1263 * compute the number of fully zero'd
1264 * pages that are beyond it
1265 * plus the last page if its partial
1266 * and we have no more I/O to issue...
1267 * otherwise a partial page is left
1268 * to begin the next I/O
1269 */
1270 if ((int)io_size >= non_rounded_size)
1271 pg_count = (bytes_to_zero - pg_resid + (PAGE_SIZE - 1)) / PAGE_SIZE;
1272 else
1273 pg_count = (bytes_to_zero - pg_resid) / PAGE_SIZE;
1274
1275 complete_transaction_now = 1;
1276 }
1277 } else {
1278 /*
1279 * no pending I/O to deal with
1280 * so, commit all of the fully zero'd pages
1281 * plus the last page if its partial
1282 * and we have no more I/O to issue...
1283 * otherwise a partial page is left
1284 * to begin the next I/O
1285 */
1286 if ((int)io_size >= non_rounded_size)
1287 pg_count = (pg_offset + bytes_to_zero + (PAGE_SIZE - 1)) / PAGE_SIZE;
1288 else
1289 pg_count = (pg_offset + bytes_to_zero) / PAGE_SIZE;
1290
1291 commit_offset = upl_offset & ~PAGE_MASK;
1292 }
1293 if ( (flags & CL_COMMIT) && pg_count) {
1294 ubc_upl_commit_range(upl, commit_offset, pg_count * PAGE_SIZE,
1295 UPL_COMMIT_CLEAR_DIRTY | UPL_COMMIT_FREE_ON_EMPTY);
1296 }
1297 upl_offset += io_size;
1298 f_offset += io_size;
1299 size -= io_size;
1300
1301 /*
1302 * keep track of how much of the original request
1303 * that we've actually completed... non_rounded_size
1304 * may go negative due to us rounding the request
1305 * to a page size multiple (i.e. size > non_rounded_size)
1306 */
1307 non_rounded_size -= io_size;
1308
1309 if (non_rounded_size <= 0) {
1310 /*
1311 * we've transferred all of the data in the original
1312 * request, but we were unable to complete the tail
1313 * of the last page because the file didn't have
1314 * an allocation to back that portion... this is ok.
1315 */
1316 size = 0;
1317 }
1318 if (cbp_head && (complete_transaction_now || size == 0)) {
1319 cluster_wait_IO(cbp_head, (flags & CL_ASYNC));
1320
1321 cluster_EOT(cbp_head, cbp_tail, size == 0 ? zero_offset : 0);
1322
1323 cluster_complete_transaction(&cbp_head, callback_arg, &retval, flags, 0);
1324
1325 trans_count = 0;
1326 }
1327 continue;
1328 }
1329 if (pg_count > max_vectors) {
1330 if (((pg_count - max_vectors) * PAGE_SIZE) > io_size) {
1331 io_size = PAGE_SIZE - pg_offset;
1332 pg_count = 1;
1333 } else {
1334 io_size -= (pg_count - max_vectors) * PAGE_SIZE;
1335 pg_count = max_vectors;
1336 }
1337 }
1338 /*
1339 * If the transaction is going to reach the maximum number of
1340 * desired elements, truncate the i/o to the nearest page so
1341 * that the actual i/o is initiated after this buffer is
1342 * created and added to the i/o chain.
1343 *
1344 * I/O directed to physically contiguous memory
1345 * doesn't have a requirement to make sure we 'fill' a page
1346 */
1347 if ( !(flags & CL_DEV_MEMORY) && trans_count >= max_trans_count &&
1348 ((upl_offset + io_size) & PAGE_MASK)) {
1349 vm_offset_t aligned_ofs;
1350
1351 aligned_ofs = (upl_offset + io_size) & ~PAGE_MASK;
1352 /*
1353 * If the io_size does not actually finish off even a
1354 * single page we have to keep adding buffers to the
1355 * transaction despite having reached the desired limit.
1356 *
1357 * Eventually we get here with the page being finished
1358 * off (and exceeded) and then we truncate the size of
1359 * this i/o request so that it is page aligned so that
1360 * we can finally issue the i/o on the transaction.
1361 */
1362 if (aligned_ofs > upl_offset) {
1363 io_size = aligned_ofs - upl_offset;
1364 pg_count--;
1365 }
1366 }
1367
1368 if ( !(mp->mnt_kern_flag & MNTK_VIRTUALDEV))
1369 /*
1370 * if we're not targeting a virtual device i.e. a disk image
1371 * it's safe to dip into the reserve pool since real devices
1372 * can complete this I/O request without requiring additional
1373 * bufs from the alloc_io_buf pool
1374 */
1375 priv = 1;
1376 else if ((flags & CL_ASYNC) && !(flags & CL_PAGEOUT))
1377 /*
1378 * Throttle the speculative IO
1379 */
1380 priv = 0;
1381 else
1382 priv = 1;
1383
1384 cbp = alloc_io_buf(vp, priv);
1385
1386 if (flags & CL_PAGEOUT) {
1387 u_int i;
1388
1389 for (i = 0; i < pg_count; i++) {
1390 if (buf_invalblkno(vp, lblkno + i, 0) == EBUSY)
1391 panic("BUSY bp found in cluster_io");
1392 }
1393 }
1394 if (flags & CL_ASYNC) {
1395 if (buf_setcallback(cbp, (void *)cluster_iodone, callback_arg))
1396 panic("buf_setcallback failed\n");
1397 }
1398 cbp->b_cliodone = (void *)callback;
1399 cbp->b_flags |= io_flags;
1400
1401 cbp->b_lblkno = lblkno;
1402 cbp->b_blkno = blkno;
1403 cbp->b_bcount = io_size;
1404
1405 if (buf_setupl(cbp, upl, upl_offset))
1406 panic("buf_setupl failed\n");
1407
1408 cbp->b_trans_next = (buf_t)NULL;
1409
1410 if ((cbp->b_iostate = (void *)iostate))
1411 /*
1412 * caller wants to track the state of this
1413 * io... bump the amount issued against this stream
1414 */
1415 iostate->io_issued += io_size;
1416
1417 if (flags & CL_READ) {
1418 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 26)) | DBG_FUNC_NONE,
1419 (int)cbp->b_lblkno, (int)cbp->b_blkno, upl_offset, io_size, 0);
1420 }
1421 else {
1422 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 27)) | DBG_FUNC_NONE,
1423 (int)cbp->b_lblkno, (int)cbp->b_blkno, upl_offset, io_size, 0);
1424 }
1425
1426 if (cbp_head) {
1427 cbp_tail->b_trans_next = cbp;
1428 cbp_tail = cbp;
1429 } else {
1430 cbp_head = cbp;
1431 cbp_tail = cbp;
1432
1433 if ( (cbp_head->b_real_bp = real_bp) )
1434 real_bp = (buf_t)NULL;
1435 }
1436 *(buf_t *)(&cbp->b_trans_head) = cbp_head;
1437
1438 trans_count++;
1439
1440 upl_offset += io_size;
1441 f_offset += io_size;
1442 size -= io_size;
1443 /*
1444 * keep track of how much of the original request
1445 * that we've actually completed... non_rounded_size
1446 * may go negative due to us rounding the request
1447 * to a page size multiple (i.e. size > non_rounded_size)
1448 */
1449 non_rounded_size -= io_size;
1450
1451 if (non_rounded_size <= 0) {
1452 /*
1453 * we've transferred all of the data in the original
1454 * request, but we were unable to complete the tail
1455 * of the last page because the file didn't have
1456 * an allocation to back that portion... this is ok.
1457 */
1458 size = 0;
1459 }
1460 if (size == 0) {
1461 /*
1462 * we have no more I/O to issue, so go
1463 * finish the final transaction
1464 */
1465 need_EOT = TRUE;
1466 } else if ( ((flags & CL_DEV_MEMORY) || (upl_offset & PAGE_MASK) == 0) &&
1467 ((flags & CL_ASYNC) || trans_count > max_trans_count) ) {
1468 /*
1469 * I/O directed to physically contiguous memory...
1470 * which doesn't have a requirement to make sure we 'fill' a page
1471 * or...
1472 * the current I/O we've prepared fully
1473 * completes the last page in this request
1474 * and ...
1475 * it's either an ASYNC request or
1476 * we've already accumulated more than 8 I/O's into
1477 * this transaction so mark it as complete so that
1478 * it can finish asynchronously or via the cluster_complete_transaction
1479 * below if the request is synchronous
1480 */
1481 need_EOT = TRUE;
1482 }
1483 if (need_EOT == TRUE)
1484 cluster_EOT(cbp_head, cbp_tail, size == 0 ? zero_offset : 0);
1485
1486 if (flags & CL_THROTTLE)
1487 (void)vnode_waitforwrites(vp, async_throttle, 0, 0, "cluster_io");
1488
1489 if ( !(io_flags & B_READ))
1490 vnode_startwrite(vp);
1491
1492 (void) VNOP_STRATEGY(cbp);
1493
1494 if (need_EOT == TRUE) {
1495 if ( !(flags & CL_ASYNC))
1496 cluster_complete_transaction(&cbp_head, callback_arg, &retval, flags, 1);
1497
1498 need_EOT = FALSE;
1499 trans_count = 0;
1500 cbp_head = NULL;
1501 }
1502 }
1503 if (error) {
1504 int abort_size;
1505
1506 io_size = 0;
1507
1508 if (cbp_head) {
1509 /*
1510 * first wait until all of the outstanding I/O
1511 * for this partial transaction has completed
1512 */
1513 cluster_wait_IO(cbp_head, (flags & CL_ASYNC));
1514
1515 /*
1516 * Rewind the upl offset to the beginning of the
1517 * transaction.
1518 */
1519 upl_offset = cbp_head->b_uploffset;
1520
1521 for (cbp = cbp_head; cbp;) {
1522 buf_t cbp_next;
1523
1524 size += cbp->b_bcount;
1525 io_size += cbp->b_bcount;
1526
1527 cbp_next = cbp->b_trans_next;
1528 free_io_buf(cbp);
1529 cbp = cbp_next;
1530 }
1531 }
1532 if (iostate) {
1533 int need_wakeup = 0;
1534
1535 /*
1536 * update the error condition for this stream
1537 * since we never really issued the io
1538 * just go ahead and adjust it back
1539 */
1540 lck_mtx_lock_spin(&iostate->io_mtxp);
1541
1542 if (iostate->io_error == 0)
1543 iostate->io_error = error;
1544 iostate->io_issued -= io_size;
1545
1546 if (iostate->io_wanted) {
1547 /*
1548 * someone is waiting for the state of
1549 * this io stream to change
1550 */
1551 iostate->io_wanted = 0;
1552 need_wakeup = 1;
1553 }
1554 lck_mtx_unlock(&iostate->io_mtxp);
1555
1556 if (need_wakeup)
1557 wakeup((caddr_t)&iostate->io_wanted);
1558 }
1559 if (flags & CL_COMMIT) {
1560 int upl_flags;
1561
1562 pg_offset = upl_offset & PAGE_MASK;
1563 abort_size = (upl_end_offset - upl_offset + PAGE_MASK) & ~PAGE_MASK;
1564
1565 upl_flags = cluster_ioerror(upl, upl_offset - pg_offset, abort_size, error, io_flags);
1566
1567 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 28)) | DBG_FUNC_NONE,
1568 upl, upl_offset - pg_offset, abort_size, (error << 24) | upl_flags, 0);
1569 }
1570 if (retval == 0)
1571 retval = error;
1572 } else if (cbp_head)
1573 panic("%s(): cbp_head is not NULL.\n", __FUNCTION__);
1574
1575 if (real_bp) {
1576 /*
1577 * can get here if we either encountered an error
1578 * or we completely zero-filled the request and
1579 * no I/O was issued
1580 */
1581 if (error) {
1582 real_bp->b_flags |= B_ERROR;
1583 real_bp->b_error = error;
1584 }
1585 buf_biodone(real_bp);
1586 }
1587 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 22)) | DBG_FUNC_END, (int)f_offset, size, upl_offset, retval, 0);
1588
1589 return (retval);
1590 }
1591
1592 #define reset_vector_run_state() \
1593 issueVectorUPL = vector_upl_offset = vector_upl_index = vector_upl_iosize = vector_upl_size = 0;
1594
1595 static int
1596 vector_cluster_io(vnode_t vp, upl_t vector_upl, vm_offset_t vector_upl_offset, off_t v_upl_uio_offset, int vector_upl_iosize,
1597 int io_flag, buf_t real_bp, struct clios *iostate, int (*callback)(buf_t, void *), void *callback_arg)
1598 {
1599 vector_upl_set_pagelist(vector_upl);
1600
1601 if(io_flag & CL_READ) {
1602 if(vector_upl_offset == 0 && ((vector_upl_iosize & PAGE_MASK)==0))
1603 io_flag &= ~CL_PRESERVE; /*don't zero fill*/
1604 else
1605 io_flag |= CL_PRESERVE; /*zero fill*/
1606 }
1607 return (cluster_io(vp, vector_upl, vector_upl_offset, v_upl_uio_offset, vector_upl_iosize, io_flag, real_bp, iostate, callback, callback_arg));
1608
1609 }
1610
1611 static int
1612 cluster_read_prefetch(vnode_t vp, off_t f_offset, u_int size, off_t filesize, int (*callback)(buf_t, void *), void *callback_arg, int bflag)
1613 {
1614 int pages_in_prefetch;
1615
1616 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 49)) | DBG_FUNC_START,
1617 (int)f_offset, size, (int)filesize, 0, 0);
1618
1619 if (f_offset >= filesize) {
1620 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 49)) | DBG_FUNC_END,
1621 (int)f_offset, 0, 0, 0, 0);
1622 return(0);
1623 }
1624 if ((off_t)size > (filesize - f_offset))
1625 size = filesize - f_offset;
1626 pages_in_prefetch = (size + (PAGE_SIZE - 1)) / PAGE_SIZE;
1627
1628 advisory_read_ext(vp, filesize, f_offset, size, callback, callback_arg, bflag);
1629
1630 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 49)) | DBG_FUNC_END,
1631 (int)f_offset + size, pages_in_prefetch, 0, 1, 0);
1632
1633 return (pages_in_prefetch);
1634 }
1635
1636
1637
1638 static void
1639 cluster_read_ahead(vnode_t vp, struct cl_extent *extent, off_t filesize, struct cl_readahead *rap, int (*callback)(buf_t, void *), void *callback_arg,
1640 int bflag)
1641 {
1642 daddr64_t r_addr;
1643 off_t f_offset;
1644 int size_of_prefetch;
1645 u_int max_prefetch;
1646
1647
1648 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_START,
1649 (int)extent->b_addr, (int)extent->e_addr, (int)rap->cl_lastr, 0, 0);
1650
1651 if (extent->b_addr == rap->cl_lastr && extent->b_addr == extent->e_addr) {
1652 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_END,
1653 rap->cl_ralen, (int)rap->cl_maxra, (int)rap->cl_lastr, 0, 0);
1654 return;
1655 }
1656 if (rap->cl_lastr == -1 || (extent->b_addr != rap->cl_lastr && extent->b_addr != (rap->cl_lastr + 1))) {
1657 rap->cl_ralen = 0;
1658 rap->cl_maxra = 0;
1659
1660 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_END,
1661 rap->cl_ralen, (int)rap->cl_maxra, (int)rap->cl_lastr, 1, 0);
1662
1663 return;
1664 }
1665 max_prefetch = MAX_PREFETCH(vp, cluster_max_io_size(vp->v_mount, CL_READ), (vp->v_mount->mnt_kern_flag & MNTK_SSD));
1666
1667 if ((max_prefetch / PAGE_SIZE) > speculative_prefetch_max)
1668 max_prefetch = (speculative_prefetch_max * PAGE_SIZE);
1669
1670 if (max_prefetch <= PAGE_SIZE) {
1671 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_END,
1672 rap->cl_ralen, (int)rap->cl_maxra, (int)rap->cl_lastr, 6, 0);
1673 return;
1674 }
1675 if (extent->e_addr < rap->cl_maxra) {
1676 if ((rap->cl_maxra - extent->e_addr) > ((max_prefetch / PAGE_SIZE) / 4)) {
1677
1678 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_END,
1679 rap->cl_ralen, (int)rap->cl_maxra, (int)rap->cl_lastr, 2, 0);
1680 return;
1681 }
1682 }
1683 r_addr = max(extent->e_addr, rap->cl_maxra) + 1;
1684 f_offset = (off_t)(r_addr * PAGE_SIZE_64);
1685
1686 size_of_prefetch = 0;
1687
1688 ubc_range_op(vp, f_offset, f_offset + PAGE_SIZE_64, UPL_ROP_PRESENT, &size_of_prefetch);
1689
1690 if (size_of_prefetch) {
1691 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_END,
1692 rap->cl_ralen, (int)rap->cl_maxra, (int)rap->cl_lastr, 3, 0);
1693 return;
1694 }
1695 if (f_offset < filesize) {
1696 daddr64_t read_size;
1697
1698 rap->cl_ralen = rap->cl_ralen ? min(max_prefetch / PAGE_SIZE, rap->cl_ralen << 1) : 1;
1699
1700 read_size = (extent->e_addr + 1) - extent->b_addr;
1701
1702 if (read_size > rap->cl_ralen) {
1703 if (read_size > max_prefetch / PAGE_SIZE)
1704 rap->cl_ralen = max_prefetch / PAGE_SIZE;
1705 else
1706 rap->cl_ralen = read_size;
1707 }
1708 size_of_prefetch = cluster_read_prefetch(vp, f_offset, rap->cl_ralen * PAGE_SIZE, filesize, callback, callback_arg, bflag);
1709
1710 if (size_of_prefetch)
1711 rap->cl_maxra = (r_addr + size_of_prefetch) - 1;
1712 }
1713 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 48)) | DBG_FUNC_END,
1714 rap->cl_ralen, (int)rap->cl_maxra, (int)rap->cl_lastr, 4, 0);
1715 }
1716
1717
1718 int
1719 cluster_pageout(vnode_t vp, upl_t upl, upl_offset_t upl_offset, off_t f_offset,
1720 int size, off_t filesize, int flags)
1721 {
1722 return cluster_pageout_ext(vp, upl, upl_offset, f_offset, size, filesize, flags, NULL, NULL);
1723
1724 }
1725
1726
1727 int
1728 cluster_pageout_ext(vnode_t vp, upl_t upl, upl_offset_t upl_offset, off_t f_offset,
1729 int size, off_t filesize, int flags, int (*callback)(buf_t, void *), void *callback_arg)
1730 {
1731 int io_size;
1732 int rounded_size;
1733 off_t max_size;
1734 int local_flags;
1735
1736 local_flags = CL_PAGEOUT | CL_THROTTLE;
1737
1738 if ((flags & UPL_IOSYNC) == 0)
1739 local_flags |= CL_ASYNC;
1740 if ((flags & UPL_NOCOMMIT) == 0)
1741 local_flags |= CL_COMMIT;
1742 if ((flags & UPL_KEEPCACHED))
1743 local_flags |= CL_KEEPCACHED;
1744 if (flags & UPL_PAGING_ENCRYPTED)
1745 local_flags |= CL_ENCRYPTED;
1746
1747
1748 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 52)) | DBG_FUNC_NONE,
1749 (int)f_offset, size, (int)filesize, local_flags, 0);
1750
1751 /*
1752 * If they didn't specify any I/O, then we are done...
1753 * we can't issue an abort because we don't know how
1754 * big the upl really is
1755 */
1756 if (size <= 0)
1757 return (EINVAL);
1758
1759 if (vp->v_mount->mnt_flag & MNT_RDONLY) {
1760 if (local_flags & CL_COMMIT)
1761 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
1762 return (EROFS);
1763 }
1764 /*
1765 * can't page-in from a negative offset
1766 * or if we're starting beyond the EOF
1767 * or if the file offset isn't page aligned
1768 * or the size requested isn't a multiple of PAGE_SIZE
1769 */
1770 if (f_offset < 0 || f_offset >= filesize ||
1771 (f_offset & PAGE_MASK_64) || (size & PAGE_MASK)) {
1772 if (local_flags & CL_COMMIT)
1773 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY);
1774 return (EINVAL);
1775 }
1776 max_size = filesize - f_offset;
1777
1778 if (size < max_size)
1779 io_size = size;
1780 else
1781 io_size = max_size;
1782
1783 rounded_size = (io_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
1784
1785 if (size > rounded_size) {
1786 if (local_flags & CL_COMMIT)
1787 ubc_upl_abort_range(upl, upl_offset + rounded_size, size - rounded_size,
1788 UPL_ABORT_FREE_ON_EMPTY);
1789 }
1790 return (cluster_io(vp, upl, upl_offset, f_offset, io_size,
1791 local_flags, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg));
1792 }
1793
1794
1795 int
1796 cluster_pagein(vnode_t vp, upl_t upl, upl_offset_t upl_offset, off_t f_offset,
1797 int size, off_t filesize, int flags)
1798 {
1799 return cluster_pagein_ext(vp, upl, upl_offset, f_offset, size, filesize, flags, NULL, NULL);
1800 }
1801
1802
1803 int
1804 cluster_pagein_ext(vnode_t vp, upl_t upl, upl_offset_t upl_offset, off_t f_offset,
1805 int size, off_t filesize, int flags, int (*callback)(buf_t, void *), void *callback_arg)
1806 {
1807 u_int io_size;
1808 int rounded_size;
1809 off_t max_size;
1810 int retval;
1811 int local_flags = 0;
1812
1813 if (upl == NULL || size < 0)
1814 panic("cluster_pagein: NULL upl passed in");
1815
1816 if ((flags & UPL_IOSYNC) == 0)
1817 local_flags |= CL_ASYNC;
1818 if ((flags & UPL_NOCOMMIT) == 0)
1819 local_flags |= CL_COMMIT;
1820 if (flags & UPL_IOSTREAMING)
1821 local_flags |= CL_IOSTREAMING;
1822 if (flags & UPL_PAGING_ENCRYPTED)
1823 local_flags |= CL_ENCRYPTED;
1824
1825
1826 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 56)) | DBG_FUNC_NONE,
1827 (int)f_offset, size, (int)filesize, local_flags, 0);
1828
1829 /*
1830 * can't page-in from a negative offset
1831 * or if we're starting beyond the EOF
1832 * or if the file offset isn't page aligned
1833 * or the size requested isn't a multiple of PAGE_SIZE
1834 */
1835 if (f_offset < 0 || f_offset >= filesize ||
1836 (f_offset & PAGE_MASK_64) || (size & PAGE_MASK) || (upl_offset & PAGE_MASK)) {
1837 if (local_flags & CL_COMMIT)
1838 ubc_upl_abort_range(upl, upl_offset, size, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);
1839 return (EINVAL);
1840 }
1841 max_size = filesize - f_offset;
1842
1843 if (size < max_size)
1844 io_size = size;
1845 else
1846 io_size = max_size;
1847
1848 rounded_size = (io_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
1849
1850 if (size > rounded_size && (local_flags & CL_COMMIT))
1851 ubc_upl_abort_range(upl, upl_offset + rounded_size,
1852 size - rounded_size, UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_ERROR);
1853
1854 retval = cluster_io(vp, upl, upl_offset, f_offset, io_size,
1855 local_flags | CL_READ | CL_PAGEIN, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
1856
1857 return (retval);
1858 }
1859
1860
1861 int
1862 cluster_bp(buf_t bp)
1863 {
1864 return cluster_bp_ext(bp, NULL, NULL);
1865 }
1866
1867
1868 int
1869 cluster_bp_ext(buf_t bp, int (*callback)(buf_t, void *), void *callback_arg)
1870 {
1871 off_t f_offset;
1872 int flags;
1873
1874 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 19)) | DBG_FUNC_START,
1875 bp, (int)bp->b_lblkno, bp->b_bcount, bp->b_flags, 0);
1876
1877 if (bp->b_flags & B_READ)
1878 flags = CL_ASYNC | CL_READ;
1879 else
1880 flags = CL_ASYNC;
1881 if (bp->b_flags & B_PASSIVE)
1882 flags |= CL_PASSIVE;
1883
1884 f_offset = ubc_blktooff(bp->b_vp, bp->b_lblkno);
1885
1886 return (cluster_io(bp->b_vp, bp->b_upl, 0, f_offset, bp->b_bcount, flags, bp, (struct clios *)NULL, callback, callback_arg));
1887 }
1888
1889
1890
1891 int
1892 cluster_write(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF, off_t headOff, off_t tailOff, int xflags)
1893 {
1894 return cluster_write_ext(vp, uio, oldEOF, newEOF, headOff, tailOff, xflags, NULL, NULL);
1895 }
1896
1897
1898 int
1899 cluster_write_ext(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF, off_t headOff, off_t tailOff,
1900 int xflags, int (*callback)(buf_t, void *), void *callback_arg)
1901 {
1902 user_ssize_t cur_resid;
1903 int retval = 0;
1904 int flags;
1905 int zflags;
1906 int bflag;
1907 int write_type = IO_COPY;
1908 u_int32_t write_length;
1909
1910 flags = xflags;
1911
1912 if (flags & IO_PASSIVE)
1913 bflag = CL_PASSIVE;
1914 else
1915 bflag = 0;
1916
1917 if (vp->v_flag & VNOCACHE_DATA)
1918 flags |= IO_NOCACHE;
1919
1920 if (uio == NULL) {
1921 /*
1922 * no user data...
1923 * this call is being made to zero-fill some range in the file
1924 */
1925 retval = cluster_write_copy(vp, NULL, (u_int32_t)0, oldEOF, newEOF, headOff, tailOff, flags, callback, callback_arg);
1926
1927 return(retval);
1928 }
1929 /*
1930 * do a write through the cache if one of the following is true....
1931 * NOCACHE is not true or NODIRECT is true
1932 * the uio request doesn't target USERSPACE
1933 * otherwise, find out if we want the direct or contig variant for
1934 * the first vector in the uio request
1935 */
1936 if ( ((flags & (IO_NOCACHE | IO_NODIRECT)) == IO_NOCACHE) && UIO_SEG_IS_USER_SPACE(uio->uio_segflg) )
1937 retval = cluster_io_type(uio, &write_type, &write_length, MIN_DIRECT_WRITE_SIZE);
1938
1939 if ( (flags & (IO_TAILZEROFILL | IO_HEADZEROFILL)) && write_type == IO_DIRECT)
1940 /*
1941 * must go through the cached variant in this case
1942 */
1943 write_type = IO_COPY;
1944
1945 while ((cur_resid = uio_resid(uio)) && uio->uio_offset < newEOF && retval == 0) {
1946
1947 switch (write_type) {
1948
1949 case IO_COPY:
1950 /*
1951 * make sure the uio_resid isn't too big...
1952 * internally, we want to handle all of the I/O in
1953 * chunk sizes that fit in a 32 bit int
1954 */
1955 if (cur_resid > (user_ssize_t)(MAX_IO_REQUEST_SIZE)) {
1956 /*
1957 * we're going to have to call cluster_write_copy
1958 * more than once...
1959 *
1960 * only want the last call to cluster_write_copy to
1961 * have the IO_TAILZEROFILL flag set and only the
1962 * first call should have IO_HEADZEROFILL
1963 */
1964 zflags = flags & ~IO_TAILZEROFILL;
1965 flags &= ~IO_HEADZEROFILL;
1966
1967 write_length = MAX_IO_REQUEST_SIZE;
1968 } else {
1969 /*
1970 * last call to cluster_write_copy
1971 */
1972 zflags = flags;
1973
1974 write_length = (u_int32_t)cur_resid;
1975 }
1976 retval = cluster_write_copy(vp, uio, write_length, oldEOF, newEOF, headOff, tailOff, zflags, callback, callback_arg);
1977 break;
1978
1979 case IO_CONTIG:
1980 zflags = flags & ~(IO_TAILZEROFILL | IO_HEADZEROFILL);
1981
1982 if (flags & IO_HEADZEROFILL) {
1983 /*
1984 * only do this once per request
1985 */
1986 flags &= ~IO_HEADZEROFILL;
1987
1988 retval = cluster_write_copy(vp, (struct uio *)0, (u_int32_t)0, (off_t)0, uio->uio_offset,
1989 headOff, (off_t)0, zflags | IO_HEADZEROFILL | IO_SYNC, callback, callback_arg);
1990 if (retval)
1991 break;
1992 }
1993 retval = cluster_write_contig(vp, uio, newEOF, &write_type, &write_length, callback, callback_arg, bflag);
1994
1995 if (retval == 0 && (flags & IO_TAILZEROFILL) && uio_resid(uio) == 0) {
1996 /*
1997 * we're done with the data from the user specified buffer(s)
1998 * and we've been requested to zero fill at the tail
1999 * treat this as an IO_HEADZEROFILL which doesn't require a uio
2000 * by rearranging the args and passing in IO_HEADZEROFILL
2001 */
2002 retval = cluster_write_copy(vp, (struct uio *)0, (u_int32_t)0, (off_t)0, tailOff, uio->uio_offset,
2003 (off_t)0, zflags | IO_HEADZEROFILL | IO_SYNC, callback, callback_arg);
2004 }
2005 break;
2006
2007 case IO_DIRECT:
2008 /*
2009 * cluster_write_direct is never called with IO_TAILZEROFILL || IO_HEADZEROFILL
2010 */
2011 retval = cluster_write_direct(vp, uio, oldEOF, newEOF, &write_type, &write_length, flags, callback, callback_arg);
2012 break;
2013
2014 case IO_UNKNOWN:
2015 retval = cluster_io_type(uio, &write_type, &write_length, MIN_DIRECT_WRITE_SIZE);
2016 break;
2017 }
2018 /*
2019 * in case we end up calling cluster_write_copy (from cluster_write_direct)
2020 * multiple times to service a multi-vector request that is not aligned properly
2021 * we need to update the oldEOF so that we
2022 * don't zero-fill the head of a page if we've successfully written
2023 * data to that area... 'cluster_write_copy' will zero-fill the head of a
2024 * page that is beyond the oldEOF if the write is unaligned... we only
2025 * want that to happen for the very first page of the cluster_write,
2026 * NOT the first page of each vector making up a multi-vector write.
2027 */
2028 if (uio->uio_offset > oldEOF)
2029 oldEOF = uio->uio_offset;
2030 }
2031 return (retval);
2032 }
2033
2034
2035 static int
2036 cluster_write_direct(vnode_t vp, struct uio *uio, off_t oldEOF, off_t newEOF, int *write_type, u_int32_t *write_length,
2037 int flags, int (*callback)(buf_t, void *), void *callback_arg)
2038 {
2039 upl_t upl;
2040 upl_page_info_t *pl;
2041 vm_offset_t upl_offset;
2042 vm_offset_t vector_upl_offset = 0;
2043 u_int32_t io_req_size;
2044 u_int32_t offset_in_file;
2045 u_int32_t offset_in_iovbase;
2046 u_int32_t io_size;
2047 int io_flag = 0;
2048 upl_size_t upl_size, vector_upl_size = 0;
2049 vm_size_t upl_needed_size;
2050 mach_msg_type_number_t pages_in_pl;
2051 int upl_flags;
2052 kern_return_t kret;
2053 mach_msg_type_number_t i;
2054 int force_data_sync;
2055 int retval = 0;
2056 int first_IO = 1;
2057 struct clios iostate;
2058 user_addr_t iov_base;
2059 u_int32_t mem_alignment_mask;
2060 u_int32_t devblocksize;
2061 u_int32_t max_upl_size;
2062
2063 u_int32_t vector_upl_iosize = 0;
2064 int issueVectorUPL = 0,useVectorUPL = (uio->uio_iovcnt > 1);
2065 off_t v_upl_uio_offset = 0;
2066 int vector_upl_index=0;
2067 upl_t vector_upl = NULL;
2068
2069
2070 /*
2071 * When we enter this routine, we know
2072 * -- the resid will not exceed iov_len
2073 */
2074 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 75)) | DBG_FUNC_START,
2075 (int)uio->uio_offset, *write_length, (int)newEOF, 0, 0);
2076
2077 max_upl_size = cluster_max_io_size(vp->v_mount, CL_WRITE);
2078
2079 io_flag = CL_ASYNC | CL_PRESERVE | CL_COMMIT | CL_THROTTLE | CL_DIRECT_IO;
2080
2081 if (flags & IO_PASSIVE)
2082 io_flag |= CL_PASSIVE;
2083
2084 iostate.io_completed = 0;
2085 iostate.io_issued = 0;
2086 iostate.io_error = 0;
2087 iostate.io_wanted = 0;
2088
2089 lck_mtx_init(&iostate.io_mtxp, cl_mtx_grp, cl_mtx_attr);
2090
2091 mem_alignment_mask = (u_int32_t)vp->v_mount->mnt_alignmentmask;
2092 devblocksize = (u_int32_t)vp->v_mount->mnt_devblocksize;
2093
2094 if (devblocksize == 1) {
2095 /*
2096 * the AFP client advertises a devblocksize of 1
2097 * however, its BLOCKMAP routine maps to physical
2098 * blocks that are PAGE_SIZE in size...
2099 * therefore we can't ask for I/Os that aren't page aligned
2100 * or aren't multiples of PAGE_SIZE in size
2101 * by setting devblocksize to PAGE_SIZE, we re-instate
2102 * the old behavior we had before the mem_alignment_mask
2103 * changes went in...
2104 */
2105 devblocksize = PAGE_SIZE;
2106 }
2107
2108 next_dwrite:
2109 io_req_size = *write_length;
2110 iov_base = uio_curriovbase(uio);
2111
2112 offset_in_file = (u_int32_t)uio->uio_offset & PAGE_MASK;
2113 offset_in_iovbase = (u_int32_t)iov_base & mem_alignment_mask;
2114
2115 if (offset_in_file || offset_in_iovbase) {
2116 /*
2117 * one of the 2 important offsets is misaligned
2118 * so fire an I/O through the cache for this entire vector
2119 */
2120 goto wait_for_dwrites;
2121 }
2122 if (iov_base & (devblocksize - 1)) {
2123 /*
2124 * the offset in memory must be on a device block boundary
2125 * so that we can guarantee that we can generate an
2126 * I/O that ends on a page boundary in cluster_io
2127 */
2128 goto wait_for_dwrites;
2129 }
2130
2131 while (io_req_size >= PAGE_SIZE && uio->uio_offset < newEOF && retval == 0) {
2132
2133 if (first_IO) {
2134 cluster_syncup(vp, newEOF, callback, callback_arg);
2135 first_IO = 0;
2136 }
2137 io_size = io_req_size & ~PAGE_MASK;
2138 iov_base = uio_curriovbase(uio);
2139
2140 if (io_size > max_upl_size)
2141 io_size = max_upl_size;
2142
2143 if(useVectorUPL && (iov_base & PAGE_MASK)) {
2144 /*
2145 * We have an iov_base that's not page-aligned.
2146 * Issue all I/O's that have been collected within
2147 * this Vectored UPL.
2148 */
2149 if(vector_upl_index) {
2150 retval = vector_cluster_io(vp, vector_upl, vector_upl_offset, v_upl_uio_offset, vector_upl_iosize, io_flag, (buf_t)NULL, &iostate, callback, callback_arg);
2151 reset_vector_run_state();
2152 }
2153
2154 /*
2155 * After this point, if we are using the Vector UPL path and the base is
2156 * not page-aligned then the UPL with that base will be the first in the vector UPL.
2157 */
2158 }
2159
2160 upl_offset = (vm_offset_t)((u_int32_t)iov_base & PAGE_MASK);
2161 upl_needed_size = (upl_offset + io_size + (PAGE_SIZE -1)) & ~PAGE_MASK;
2162
2163 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 76)) | DBG_FUNC_START,
2164 (int)upl_offset, upl_needed_size, (int)iov_base, io_size, 0);
2165
2166 for (force_data_sync = 0; force_data_sync < 3; force_data_sync++) {
2167 pages_in_pl = 0;
2168 upl_size = upl_needed_size;
2169 upl_flags = UPL_FILE_IO | UPL_COPYOUT_FROM | UPL_NO_SYNC |
2170 UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE;
2171
2172 kret = vm_map_get_upl(current_map(),
2173 (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)),
2174 &upl_size,
2175 &upl,
2176 NULL,
2177 &pages_in_pl,
2178 &upl_flags,
2179 force_data_sync);
2180
2181 if (kret != KERN_SUCCESS) {
2182 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 76)) | DBG_FUNC_END,
2183 0, 0, 0, kret, 0);
2184 /*
2185 * failed to get pagelist
2186 *
2187 * we may have already spun some portion of this request
2188 * off as async requests... we need to wait for the I/O
2189 * to complete before returning
2190 */
2191 goto wait_for_dwrites;
2192 }
2193 pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
2194 pages_in_pl = upl_size / PAGE_SIZE;
2195
2196 for (i = 0; i < pages_in_pl; i++) {
2197 if (!upl_valid_page(pl, i))
2198 break;
2199 }
2200 if (i == pages_in_pl)
2201 break;
2202
2203 /*
2204 * didn't get all the pages back that we
2205 * needed... release this upl and try again
2206 */
2207 ubc_upl_abort(upl, 0);
2208 }
2209 if (force_data_sync >= 3) {
2210 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 76)) | DBG_FUNC_END,
2211 i, pages_in_pl, upl_size, kret, 0);
2212 /*
2213 * for some reason, we couldn't acquire a hold on all
2214 * the pages needed in the user's address space
2215 *
2216 * we may have already spun some portion of this request
2217 * off as async requests... we need to wait for the I/O
2218 * to complete before returning
2219 */
2220 goto wait_for_dwrites;
2221 }
2222
2223 /*
2224 * Consider the possibility that upl_size wasn't satisfied.
2225 */
2226 if (upl_size < upl_needed_size) {
2227 if (upl_size && upl_offset == 0)
2228 io_size = upl_size;
2229 else
2230 io_size = 0;
2231 }
2232 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 76)) | DBG_FUNC_END,
2233 (int)upl_offset, upl_size, (int)iov_base, io_size, 0);
2234
2235 if (io_size == 0) {
2236 ubc_upl_abort(upl, 0);
2237 /*
2238 * we may have already spun some portion of this request
2239 * off as async requests... we need to wait for the I/O
2240 * to complete before returning
2241 */
2242 goto wait_for_dwrites;
2243 }
2244
2245 if(useVectorUPL) {
2246 vm_offset_t end_off = ((iov_base + io_size) & PAGE_MASK);
2247 if(end_off)
2248 issueVectorUPL = 1;
2249 /*
2250 * After this point, if we are using a vector UPL, then
2251 * either all the UPL elements end on a page boundary OR
2252 * this UPL is the last element because it does not end
2253 * on a page boundary.
2254 */
2255 }
2256
2257 /*
2258 * Now look for pages already in the cache
2259 * and throw them away.
2260 * uio->uio_offset is page aligned within the file
2261 * io_size is a multiple of PAGE_SIZE
2262 */
2263 ubc_range_op(vp, uio->uio_offset, uio->uio_offset + io_size, UPL_ROP_DUMP, NULL);
2264
2265 /*
2266 * we want push out these writes asynchronously so that we can overlap
2267 * the preparation of the next I/O
2268 * if there are already too many outstanding writes
2269 * wait until some complete before issuing the next
2270 */
2271 if (iostate.io_issued > iostate.io_completed)
2272 cluster_iostate_wait(&iostate, max_upl_size * IO_SCALE(vp, 2), "cluster_write_direct");
2273
2274 if (iostate.io_error) {
2275 /*
2276 * one of the earlier writes we issued ran into a hard error
2277 * don't issue any more writes, cleanup the UPL
2278 * that was just created but not used, then
2279 * go wait for all writes that are part of this stream
2280 * to complete before returning the error to the caller
2281 */
2282 ubc_upl_abort(upl, 0);
2283
2284 goto wait_for_dwrites;
2285 }
2286
2287 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 77)) | DBG_FUNC_START,
2288 (int)upl_offset, (int)uio->uio_offset, io_size, io_flag, 0);
2289
2290 if(!useVectorUPL)
2291 retval = cluster_io(vp, upl, upl_offset, uio->uio_offset,
2292 io_size, io_flag, (buf_t)NULL, &iostate, callback, callback_arg);
2293
2294 else {
2295 if(!vector_upl_index) {
2296 vector_upl = vector_upl_create(upl_offset);
2297 v_upl_uio_offset = uio->uio_offset;
2298 vector_upl_offset = upl_offset;
2299 }
2300
2301 vector_upl_set_subupl(vector_upl,upl,upl_size);
2302 vector_upl_set_iostate(vector_upl, upl, vector_upl_size, upl_size);
2303 vector_upl_index++;
2304 vector_upl_iosize += io_size;
2305 vector_upl_size += upl_size;
2306
2307 if(issueVectorUPL || vector_upl_index == MAX_VECTOR_UPL_ELEMENTS || vector_upl_size >= MAX_VECTOR_UPL_SIZE) {
2308 retval = vector_cluster_io(vp, vector_upl, vector_upl_offset, v_upl_uio_offset, vector_upl_iosize, io_flag, (buf_t)NULL, &iostate, callback, callback_arg);
2309 reset_vector_run_state();
2310 }
2311 }
2312
2313 /*
2314 * update the uio structure to
2315 * reflect the I/O that we just issued
2316 */
2317 uio_update(uio, (user_size_t)io_size);
2318
2319 /*
2320 * in case we end up calling through to cluster_write_copy to finish
2321 * the tail of this request, we need to update the oldEOF so that we
2322 * don't zero-fill the head of a page if we've successfully written
2323 * data to that area... 'cluster_write_copy' will zero-fill the head of a
2324 * page that is beyond the oldEOF if the write is unaligned... we only
2325 * want that to happen for the very first page of the cluster_write,
2326 * NOT the first page of each vector making up a multi-vector write.
2327 */
2328 if (uio->uio_offset > oldEOF)
2329 oldEOF = uio->uio_offset;
2330
2331 io_req_size -= io_size;
2332
2333 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 77)) | DBG_FUNC_END,
2334 (int)upl_offset, (int)uio->uio_offset, io_req_size, retval, 0);
2335
2336 } /* end while */
2337
2338 if (retval == 0 && iostate.io_error == 0 && io_req_size == 0) {
2339
2340 retval = cluster_io_type(uio, write_type, write_length, MIN_DIRECT_WRITE_SIZE);
2341
2342 if (retval == 0 && *write_type == IO_DIRECT) {
2343
2344 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 75)) | DBG_FUNC_NONE,
2345 (int)uio->uio_offset, *write_length, (int)newEOF, 0, 0);
2346
2347 goto next_dwrite;
2348 }
2349 }
2350
2351 wait_for_dwrites:
2352
2353 if (retval == 0 && iostate.io_error == 0 && useVectorUPL && vector_upl_index) {
2354 retval = vector_cluster_io(vp, vector_upl, vector_upl_offset, v_upl_uio_offset, vector_upl_iosize, io_flag, (buf_t)NULL, &iostate, callback, callback_arg);
2355 reset_vector_run_state();
2356 }
2357
2358 if (iostate.io_issued > iostate.io_completed) {
2359 /*
2360 * make sure all async writes issued as part of this stream
2361 * have completed before we return
2362 */
2363 cluster_iostate_wait(&iostate, 0, "cluster_write_direct");
2364 }
2365 if (iostate.io_error)
2366 retval = iostate.io_error;
2367
2368 lck_mtx_destroy(&iostate.io_mtxp, cl_mtx_grp);
2369
2370 if (io_req_size && retval == 0) {
2371 /*
2372 * we couldn't handle the tail of this request in DIRECT mode
2373 * so fire it through the copy path
2374 *
2375 * note that flags will never have IO_HEADZEROFILL or IO_TAILZEROFILL set
2376 * so we can just pass 0 in for the headOff and tailOff
2377 */
2378 if (uio->uio_offset > oldEOF)
2379 oldEOF = uio->uio_offset;
2380
2381 retval = cluster_write_copy(vp, uio, io_req_size, oldEOF, newEOF, (off_t)0, (off_t)0, flags, callback, callback_arg);
2382
2383 *write_type = IO_UNKNOWN;
2384 }
2385 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 75)) | DBG_FUNC_END,
2386 (int)uio->uio_offset, io_req_size, retval, 4, 0);
2387
2388 return (retval);
2389 }
2390
2391
2392 static int
2393 cluster_write_contig(vnode_t vp, struct uio *uio, off_t newEOF, int *write_type, u_int32_t *write_length,
2394 int (*callback)(buf_t, void *), void *callback_arg, int bflag)
2395 {
2396 upl_page_info_t *pl;
2397 addr64_t src_paddr = 0;
2398 upl_t upl[MAX_VECTS];
2399 vm_offset_t upl_offset;
2400 u_int32_t tail_size = 0;
2401 u_int32_t io_size;
2402 u_int32_t xsize;
2403 upl_size_t upl_size;
2404 vm_size_t upl_needed_size;
2405 mach_msg_type_number_t pages_in_pl;
2406 int upl_flags;
2407 kern_return_t kret;
2408 struct clios iostate;
2409 int error = 0;
2410 int cur_upl = 0;
2411 int num_upl = 0;
2412 int n;
2413 user_addr_t iov_base;
2414 u_int32_t devblocksize;
2415 u_int32_t mem_alignment_mask;
2416
2417 /*
2418 * When we enter this routine, we know
2419 * -- the io_req_size will not exceed iov_len
2420 * -- the target address is physically contiguous
2421 */
2422 cluster_syncup(vp, newEOF, callback, callback_arg);
2423
2424 devblocksize = (u_int32_t)vp->v_mount->mnt_devblocksize;
2425 mem_alignment_mask = (u_int32_t)vp->v_mount->mnt_alignmentmask;
2426
2427 iostate.io_completed = 0;
2428 iostate.io_issued = 0;
2429 iostate.io_error = 0;
2430 iostate.io_wanted = 0;
2431
2432 lck_mtx_init(&iostate.io_mtxp, cl_mtx_grp, cl_mtx_attr);
2433
2434 next_cwrite:
2435 io_size = *write_length;
2436
2437 iov_base = uio_curriovbase(uio);
2438
2439 upl_offset = (vm_offset_t)((u_int32_t)iov_base & PAGE_MASK);
2440 upl_needed_size = upl_offset + io_size;
2441
2442 pages_in_pl = 0;
2443 upl_size = upl_needed_size;
2444 upl_flags = UPL_FILE_IO | UPL_COPYOUT_FROM | UPL_NO_SYNC |
2445 UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE;
2446
2447 kret = vm_map_get_upl(current_map(),
2448 (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)),
2449 &upl_size, &upl[cur_upl], NULL, &pages_in_pl, &upl_flags, 0);
2450
2451 if (kret != KERN_SUCCESS) {
2452 /*
2453 * failed to get pagelist
2454 */
2455 error = EINVAL;
2456 goto wait_for_cwrites;
2457 }
2458 num_upl++;
2459
2460 /*
2461 * Consider the possibility that upl_size wasn't satisfied.
2462 */
2463 if (upl_size < upl_needed_size) {
2464 /*
2465 * This is a failure in the physical memory case.
2466 */
2467 error = EINVAL;
2468 goto wait_for_cwrites;
2469 }
2470 pl = ubc_upl_pageinfo(upl[cur_upl]);
2471
2472 src_paddr = ((addr64_t)upl_phys_page(pl, 0) << 12) + (addr64_t)upl_offset;
2473
2474 while (((uio->uio_offset & (devblocksize - 1)) || io_size < devblocksize) && io_size) {
2475 u_int32_t head_size;
2476
2477 head_size = devblocksize - (u_int32_t)(uio->uio_offset & (devblocksize - 1));
2478
2479 if (head_size > io_size)
2480 head_size = io_size;
2481
2482 error = cluster_align_phys_io(vp, uio, src_paddr, head_size, 0, callback, callback_arg);
2483
2484 if (error)
2485 goto wait_for_cwrites;
2486
2487 upl_offset += head_size;
2488 src_paddr += head_size;
2489 io_size -= head_size;
2490
2491 iov_base += head_size;
2492 }
2493 if ((u_int32_t)iov_base & mem_alignment_mask) {
2494 /*
2495 * request doesn't set up on a memory boundary
2496 * the underlying DMA engine can handle...
2497 * return an error instead of going through
2498 * the slow copy path since the intent of this
2499 * path is direct I/O from device memory
2500 */
2501 error = EINVAL;
2502 goto wait_for_cwrites;
2503 }
2504
2505 tail_size = io_size & (devblocksize - 1);
2506 io_size -= tail_size;
2507
2508 while (io_size && error == 0) {
2509
2510 if (io_size > MAX_IO_CONTIG_SIZE)
2511 xsize = MAX_IO_CONTIG_SIZE;
2512 else
2513 xsize = io_size;
2514 /*
2515 * request asynchronously so that we can overlap
2516 * the preparation of the next I/O... we'll do
2517 * the commit after all the I/O has completed
2518 * since its all issued against the same UPL
2519 * if there are already too many outstanding writes
2520 * wait until some have completed before issuing the next
2521 */
2522 if (iostate.io_issued > iostate.io_completed)
2523 cluster_iostate_wait(&iostate, MAX_IO_CONTIG_SIZE * IO_SCALE(vp, 2), "cluster_write_contig");
2524
2525 if (iostate.io_error) {
2526 /*
2527 * one of the earlier writes we issued ran into a hard error
2528 * don't issue any more writes...
2529 * go wait for all writes that are part of this stream
2530 * to complete before returning the error to the caller
2531 */
2532 goto wait_for_cwrites;
2533 }
2534 /*
2535 * issue an asynchronous write to cluster_io
2536 */
2537 error = cluster_io(vp, upl[cur_upl], upl_offset, uio->uio_offset,
2538 xsize, CL_DEV_MEMORY | CL_ASYNC | bflag, (buf_t)NULL, (struct clios *)&iostate, callback, callback_arg);
2539
2540 if (error == 0) {
2541 /*
2542 * The cluster_io write completed successfully,
2543 * update the uio structure
2544 */
2545 uio_update(uio, (user_size_t)xsize);
2546
2547 upl_offset += xsize;
2548 src_paddr += xsize;
2549 io_size -= xsize;
2550 }
2551 }
2552 if (error == 0 && iostate.io_error == 0 && tail_size == 0 && num_upl < MAX_VECTS) {
2553
2554 error = cluster_io_type(uio, write_type, write_length, 0);
2555
2556 if (error == 0 && *write_type == IO_CONTIG) {
2557 cur_upl++;
2558 goto next_cwrite;
2559 }
2560 } else
2561 *write_type = IO_UNKNOWN;
2562
2563 wait_for_cwrites:
2564 /*
2565 * make sure all async writes that are part of this stream
2566 * have completed before we proceed
2567 */
2568 if (iostate.io_issued > iostate.io_completed)
2569 cluster_iostate_wait(&iostate, 0, "cluster_write_contig");
2570
2571 if (iostate.io_error)
2572 error = iostate.io_error;
2573
2574 lck_mtx_destroy(&iostate.io_mtxp, cl_mtx_grp);
2575
2576 if (error == 0 && tail_size)
2577 error = cluster_align_phys_io(vp, uio, src_paddr, tail_size, 0, callback, callback_arg);
2578
2579 for (n = 0; n < num_upl; n++)
2580 /*
2581 * just release our hold on each physically contiguous
2582 * region without changing any state
2583 */
2584 ubc_upl_abort(upl[n], 0);
2585
2586 return (error);
2587 }
2588
2589
2590 /*
2591 * need to avoid a race between an msync of a range of pages dirtied via mmap
2592 * vs a filesystem such as HFS deciding to write a 'hole' to disk via cluster_write's
2593 * zerofill mechanism before it has seen the VNOP_PAGEOUTs for the pages being msync'd
2594 *
2595 * we should never force-zero-fill pages that are already valid in the cache...
2596 * the entire page contains valid data (either from disk, zero-filled or dirtied
2597 * via an mmap) so we can only do damage by trying to zero-fill
2598 *
2599 */
2600 static int
2601 cluster_zero_range(upl_t upl, upl_page_info_t *pl, int flags, int io_offset, off_t zero_off, off_t upl_f_offset, int bytes_to_zero)
2602 {
2603 int zero_pg_index;
2604 boolean_t need_cluster_zero = TRUE;
2605
2606 if ((flags & (IO_NOZEROVALID | IO_NOZERODIRTY))) {
2607
2608 bytes_to_zero = min(bytes_to_zero, PAGE_SIZE - (int)(zero_off & PAGE_MASK_64));
2609 zero_pg_index = (int)((zero_off - upl_f_offset) / PAGE_SIZE_64);
2610
2611 if (upl_valid_page(pl, zero_pg_index)) {
2612 /*
2613 * never force zero valid pages - dirty or clean
2614 * we'll leave these in the UPL for cluster_write_copy to deal with
2615 */
2616 need_cluster_zero = FALSE;
2617 }
2618 }
2619 if (need_cluster_zero == TRUE)
2620 cluster_zero(upl, io_offset, bytes_to_zero, NULL);
2621
2622 return (bytes_to_zero);
2623 }
2624
2625
2626 static int
2627 cluster_write_copy(vnode_t vp, struct uio *uio, u_int32_t io_req_size, off_t oldEOF, off_t newEOF, off_t headOff,
2628 off_t tailOff, int flags, int (*callback)(buf_t, void *), void *callback_arg)
2629 {
2630 upl_page_info_t *pl;
2631 upl_t upl;
2632 vm_offset_t upl_offset = 0;
2633 vm_size_t upl_size;
2634 off_t upl_f_offset;
2635 int pages_in_upl;
2636 int start_offset;
2637 int xfer_resid;
2638 int io_size;
2639 int io_offset;
2640 int bytes_to_zero;
2641 int bytes_to_move;
2642 kern_return_t kret;
2643 int retval = 0;
2644 int io_resid;
2645 long long total_size;
2646 long long zero_cnt;
2647 off_t zero_off;
2648 long long zero_cnt1;
2649 off_t zero_off1;
2650 off_t write_off = 0;
2651 int write_cnt = 0;
2652 boolean_t first_pass = FALSE;
2653 struct cl_extent cl;
2654 struct cl_writebehind *wbp;
2655 int bflag;
2656 u_int max_cluster_pgcount;
2657 u_int max_io_size;
2658
2659 if (uio) {
2660 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 40)) | DBG_FUNC_START,
2661 (int)uio->uio_offset, io_req_size, (int)oldEOF, (int)newEOF, 0);
2662
2663 io_resid = io_req_size;
2664 } else {
2665 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 40)) | DBG_FUNC_START,
2666 0, 0, (int)oldEOF, (int)newEOF, 0);
2667
2668 io_resid = 0;
2669 }
2670 if (flags & IO_PASSIVE)
2671 bflag = CL_PASSIVE;
2672 else
2673 bflag = 0;
2674
2675 zero_cnt = 0;
2676 zero_cnt1 = 0;
2677 zero_off = 0;
2678 zero_off1 = 0;
2679
2680 max_cluster_pgcount = MAX_CLUSTER_SIZE(vp) / PAGE_SIZE;
2681 max_io_size = cluster_max_io_size(vp->v_mount, CL_WRITE);
2682
2683 if (flags & IO_HEADZEROFILL) {
2684 /*
2685 * some filesystems (HFS is one) don't support unallocated holes within a file...
2686 * so we zero fill the intervening space between the old EOF and the offset
2687 * where the next chunk of real data begins.... ftruncate will also use this
2688 * routine to zero fill to the new EOF when growing a file... in this case, the
2689 * uio structure will not be provided
2690 */
2691 if (uio) {
2692 if (headOff < uio->uio_offset) {
2693 zero_cnt = uio->uio_offset - headOff;
2694 zero_off = headOff;
2695 }
2696 } else if (headOff < newEOF) {
2697 zero_cnt = newEOF - headOff;
2698 zero_off = headOff;
2699 }
2700 } else {
2701 if (uio && uio->uio_offset > oldEOF) {
2702 zero_off = uio->uio_offset & ~PAGE_MASK_64;
2703
2704 if (zero_off >= oldEOF) {
2705 zero_cnt = uio->uio_offset - zero_off;
2706
2707 flags |= IO_HEADZEROFILL;
2708 }
2709 }
2710 }
2711 if (flags & IO_TAILZEROFILL) {
2712 if (uio) {
2713 zero_off1 = uio->uio_offset + io_req_size;
2714
2715 if (zero_off1 < tailOff)
2716 zero_cnt1 = tailOff - zero_off1;
2717 }
2718 } else {
2719 if (uio && newEOF > oldEOF) {
2720 zero_off1 = uio->uio_offset + io_req_size;
2721
2722 if (zero_off1 == newEOF && (zero_off1 & PAGE_MASK_64)) {
2723 zero_cnt1 = PAGE_SIZE_64 - (zero_off1 & PAGE_MASK_64);
2724
2725 flags |= IO_TAILZEROFILL;
2726 }
2727 }
2728 }
2729 if (zero_cnt == 0 && uio == (struct uio *) 0) {
2730 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 40)) | DBG_FUNC_END,
2731 retval, 0, 0, 0, 0);
2732 return (0);
2733 }
2734 if (uio) {
2735 write_off = uio->uio_offset;
2736 write_cnt = uio_resid(uio);
2737 /*
2738 * delay updating the sequential write info
2739 * in the control block until we've obtained
2740 * the lock for it
2741 */
2742 first_pass = TRUE;
2743 }
2744 while ((total_size = (io_resid + zero_cnt + zero_cnt1)) && retval == 0) {
2745 /*
2746 * for this iteration of the loop, figure out where our starting point is
2747 */
2748 if (zero_cnt) {
2749 start_offset = (int)(zero_off & PAGE_MASK_64);
2750 upl_f_offset = zero_off - start_offset;
2751 } else if (io_resid) {
2752 start_offset = (int)(uio->uio_offset & PAGE_MASK_64);
2753 upl_f_offset = uio->uio_offset - start_offset;
2754 } else {
2755 start_offset = (int)(zero_off1 & PAGE_MASK_64);
2756 upl_f_offset = zero_off1 - start_offset;
2757 }
2758 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 46)) | DBG_FUNC_NONE,
2759 (int)zero_off, (int)zero_cnt, (int)zero_off1, (int)zero_cnt1, 0);
2760
2761 if (total_size > max_io_size)
2762 total_size = max_io_size;
2763
2764 cl.b_addr = (daddr64_t)(upl_f_offset / PAGE_SIZE_64);
2765
2766 if (uio && ((flags & (IO_SYNC | IO_HEADZEROFILL | IO_TAILZEROFILL)) == 0)) {
2767 /*
2768 * assumption... total_size <= io_resid
2769 * because IO_HEADZEROFILL and IO_TAILZEROFILL not set
2770 */
2771 if ((start_offset + total_size) > max_io_size)
2772 total_size = max_io_size - start_offset;
2773 xfer_resid = total_size;
2774
2775 retval = cluster_copy_ubc_data_internal(vp, uio, &xfer_resid, 1, 1);
2776
2777 if (retval)
2778 break;
2779
2780 io_resid -= (total_size - xfer_resid);
2781 total_size = xfer_resid;
2782 start_offset = (int)(uio->uio_offset & PAGE_MASK_64);
2783 upl_f_offset = uio->uio_offset - start_offset;
2784
2785 if (total_size == 0) {
2786 if (start_offset) {
2787 /*
2788 * the write did not finish on a page boundary
2789 * which will leave upl_f_offset pointing to the
2790 * beginning of the last page written instead of
2791 * the page beyond it... bump it in this case
2792 * so that the cluster code records the last page
2793 * written as dirty
2794 */
2795 upl_f_offset += PAGE_SIZE_64;
2796 }
2797 upl_size = 0;
2798
2799 goto check_cluster;
2800 }
2801 }
2802 /*
2803 * compute the size of the upl needed to encompass
2804 * the requested write... limit each call to cluster_io
2805 * to the maximum UPL size... cluster_io will clip if
2806 * this exceeds the maximum io_size for the device,
2807 * make sure to account for
2808 * a starting offset that's not page aligned
2809 */
2810 upl_size = (start_offset + total_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
2811
2812 if (upl_size > max_io_size)
2813 upl_size = max_io_size;
2814
2815 pages_in_upl = upl_size / PAGE_SIZE;
2816 io_size = upl_size - start_offset;
2817
2818 if ((long long)io_size > total_size)
2819 io_size = total_size;
2820
2821 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 41)) | DBG_FUNC_START, upl_size, io_size, total_size, 0, 0);
2822
2823
2824 /*
2825 * Gather the pages from the buffer cache.
2826 * The UPL_WILL_MODIFY flag lets the UPL subsystem know
2827 * that we intend to modify these pages.
2828 */
2829 kret = ubc_create_upl(vp,
2830 upl_f_offset,
2831 upl_size,
2832 &upl,
2833 &pl,
2834 UPL_SET_LITE | (( uio!=NULL && (uio->uio_flags & UIO_FLAGS_IS_COMPRESSED_FILE)) ? 0 : UPL_WILL_MODIFY));
2835 if (kret != KERN_SUCCESS)
2836 panic("cluster_write_copy: failed to get pagelist");
2837
2838 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 41)) | DBG_FUNC_END,
2839 upl, (int)upl_f_offset, start_offset, 0, 0);
2840
2841 if (start_offset && upl_f_offset < oldEOF && !upl_valid_page(pl, 0)) {
2842 int read_size;
2843
2844 /*
2845 * we're starting in the middle of the first page of the upl
2846 * and the page isn't currently valid, so we're going to have
2847 * to read it in first... this is a synchronous operation
2848 */
2849 read_size = PAGE_SIZE;
2850
2851 if ((upl_f_offset + read_size) > oldEOF)
2852 read_size = oldEOF - upl_f_offset;
2853
2854 retval = cluster_io(vp, upl, 0, upl_f_offset, read_size,
2855 CL_READ | bflag, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
2856 if (retval) {
2857 /*
2858 * we had an error during the read which causes us to abort
2859 * the current cluster_write request... before we do, we need
2860 * to release the rest of the pages in the upl without modifying
2861 * there state and mark the failed page in error
2862 */
2863 ubc_upl_abort_range(upl, 0, PAGE_SIZE, UPL_ABORT_DUMP_PAGES|UPL_ABORT_FREE_ON_EMPTY);
2864
2865 if (upl_size > PAGE_SIZE)
2866 ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_FREE_ON_EMPTY);
2867
2868 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 45)) | DBG_FUNC_NONE,
2869 upl, 0, 0, retval, 0);
2870 break;
2871 }
2872 }
2873 if ((start_offset == 0 || upl_size > PAGE_SIZE) && ((start_offset + io_size) & PAGE_MASK)) {
2874 /*
2875 * the last offset we're writing to in this upl does not end on a page
2876 * boundary... if it's not beyond the old EOF, then we'll also need to
2877 * pre-read this page in if it isn't already valid
2878 */
2879 upl_offset = upl_size - PAGE_SIZE;
2880
2881 if ((upl_f_offset + start_offset + io_size) < oldEOF &&
2882 !upl_valid_page(pl, upl_offset / PAGE_SIZE)) {
2883 int read_size;
2884
2885 read_size = PAGE_SIZE;
2886
2887 if ((off_t)(upl_f_offset + upl_offset + read_size) > oldEOF)
2888 read_size = oldEOF - (upl_f_offset + upl_offset);
2889
2890 retval = cluster_io(vp, upl, upl_offset, upl_f_offset + upl_offset, read_size,
2891 CL_READ | bflag, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
2892 if (retval) {
2893 /*
2894 * we had an error during the read which causes us to abort
2895 * the current cluster_write request... before we do, we
2896 * need to release the rest of the pages in the upl without
2897 * modifying there state and mark the failed page in error
2898 */
2899 ubc_upl_abort_range(upl, upl_offset, PAGE_SIZE, UPL_ABORT_DUMP_PAGES|UPL_ABORT_FREE_ON_EMPTY);
2900
2901 if (upl_size > PAGE_SIZE)
2902 ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_FREE_ON_EMPTY);
2903
2904 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 45)) | DBG_FUNC_NONE,
2905 upl, 0, 0, retval, 0);
2906 break;
2907 }
2908 }
2909 }
2910 xfer_resid = io_size;
2911 io_offset = start_offset;
2912
2913 while (zero_cnt && xfer_resid) {
2914
2915 if (zero_cnt < (long long)xfer_resid)
2916 bytes_to_zero = zero_cnt;
2917 else
2918 bytes_to_zero = xfer_resid;
2919
2920 bytes_to_zero = cluster_zero_range(upl, pl, flags, io_offset, zero_off, upl_f_offset, bytes_to_zero);
2921
2922 xfer_resid -= bytes_to_zero;
2923 zero_cnt -= bytes_to_zero;
2924 zero_off += bytes_to_zero;
2925 io_offset += bytes_to_zero;
2926 }
2927 if (xfer_resid && io_resid) {
2928 u_int32_t io_requested;
2929
2930 bytes_to_move = min(io_resid, xfer_resid);
2931 io_requested = bytes_to_move;
2932
2933 retval = cluster_copy_upl_data(uio, upl, io_offset, (int *)&io_requested);
2934
2935 if (retval) {
2936 ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_DUMP_PAGES | UPL_ABORT_FREE_ON_EMPTY);
2937
2938 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 45)) | DBG_FUNC_NONE,
2939 upl, 0, 0, retval, 0);
2940 } else {
2941 io_resid -= bytes_to_move;
2942 xfer_resid -= bytes_to_move;
2943 io_offset += bytes_to_move;
2944 }
2945 }
2946 while (xfer_resid && zero_cnt1 && retval == 0) {
2947
2948 if (zero_cnt1 < (long long)xfer_resid)
2949 bytes_to_zero = zero_cnt1;
2950 else
2951 bytes_to_zero = xfer_resid;
2952
2953 bytes_to_zero = cluster_zero_range(upl, pl, flags, io_offset, zero_off1, upl_f_offset, bytes_to_zero);
2954
2955 xfer_resid -= bytes_to_zero;
2956 zero_cnt1 -= bytes_to_zero;
2957 zero_off1 += bytes_to_zero;
2958 io_offset += bytes_to_zero;
2959 }
2960 if (retval == 0) {
2961 int cl_index;
2962 int ret_cluster_try_push;
2963
2964 io_size += start_offset;
2965
2966 if ((upl_f_offset + io_size) >= newEOF && (u_int)io_size < upl_size) {
2967 /*
2968 * if we're extending the file with this write
2969 * we'll zero fill the rest of the page so that
2970 * if the file gets extended again in such a way as to leave a
2971 * hole starting at this EOF, we'll have zero's in the correct spot
2972 */
2973 cluster_zero(upl, io_size, upl_size - io_size, NULL);
2974 }
2975 /*
2976 * release the upl now if we hold one since...
2977 * 1) pages in it may be present in the sparse cluster map
2978 * and may span 2 separate buckets there... if they do and
2979 * we happen to have to flush a bucket to make room and it intersects
2980 * this upl, a deadlock may result on page BUSY
2981 * 2) we're delaying the I/O... from this point forward we're just updating
2982 * the cluster state... no need to hold the pages, so commit them
2983 * 3) IO_SYNC is set...
2984 * because we had to ask for a UPL that provides currenty non-present pages, the
2985 * UPL has been automatically set to clear the dirty flags (both software and hardware)
2986 * upon committing it... this is not the behavior we want since it's possible for
2987 * pages currently present as part of a mapped file to be dirtied while the I/O is in flight.
2988 * we'll pick these pages back up later with the correct behavior specified.
2989 * 4) we don't want to hold pages busy in a UPL and then block on the cluster lock... if a flush
2990 * of this vnode is in progress, we will deadlock if the pages being flushed intersect the pages
2991 * we hold since the flushing context is holding the cluster lock.
2992 */
2993 ubc_upl_commit_range(upl, 0, upl_size,
2994 UPL_COMMIT_SET_DIRTY | UPL_COMMIT_INACTIVATE | UPL_COMMIT_FREE_ON_EMPTY);
2995 check_cluster:
2996 /*
2997 * calculate the last logical block number
2998 * that this delayed I/O encompassed
2999 */
3000 cl.e_addr = (daddr64_t)((upl_f_offset + (off_t)upl_size) / PAGE_SIZE_64);
3001
3002 if (flags & IO_SYNC) {
3003 /*
3004 * if the IO_SYNC flag is set than we need to
3005 * bypass any clusters and immediately issue
3006 * the I/O
3007 */
3008 goto issue_io;
3009 }
3010 /*
3011 * take the lock to protect our accesses
3012 * of the writebehind and sparse cluster state
3013 */
3014 wbp = cluster_get_wbp(vp, CLW_ALLOCATE | CLW_RETURNLOCKED);
3015
3016 if (wbp->cl_scmap) {
3017
3018 if ( !(flags & IO_NOCACHE)) {
3019 /*
3020 * we've fallen into the sparse
3021 * cluster method of delaying dirty pages
3022 */
3023 sparse_cluster_add(&(wbp->cl_scmap), vp, &cl, newEOF, callback, callback_arg);
3024
3025 lck_mtx_unlock(&wbp->cl_lockw);
3026
3027 continue;
3028 }
3029 /*
3030 * must have done cached writes that fell into
3031 * the sparse cluster mechanism... we've switched
3032 * to uncached writes on the file, so go ahead
3033 * and push whatever's in the sparse map
3034 * and switch back to normal clustering
3035 */
3036 wbp->cl_number = 0;
3037
3038 sparse_cluster_push(&(wbp->cl_scmap), vp, newEOF, PUSH_ALL, 0, callback, callback_arg);
3039 /*
3040 * no clusters of either type present at this point
3041 * so just go directly to start_new_cluster since
3042 * we know we need to delay this I/O since we've
3043 * already released the pages back into the cache
3044 * to avoid the deadlock with sparse_cluster_push
3045 */
3046 goto start_new_cluster;
3047 }
3048 if (first_pass) {
3049 if (write_off == wbp->cl_last_write)
3050 wbp->cl_seq_written += write_cnt;
3051 else
3052 wbp->cl_seq_written = write_cnt;
3053
3054 wbp->cl_last_write = write_off + write_cnt;
3055
3056 first_pass = FALSE;
3057 }
3058 if (wbp->cl_number == 0)
3059 /*
3060 * no clusters currently present
3061 */
3062 goto start_new_cluster;
3063
3064 for (cl_index = 0; cl_index < wbp->cl_number; cl_index++) {
3065 /*
3066 * check each cluster that we currently hold
3067 * try to merge some or all of this write into
3068 * one or more of the existing clusters... if
3069 * any portion of the write remains, start a
3070 * new cluster
3071 */
3072 if (cl.b_addr >= wbp->cl_clusters[cl_index].b_addr) {
3073 /*
3074 * the current write starts at or after the current cluster
3075 */
3076 if (cl.e_addr <= (wbp->cl_clusters[cl_index].b_addr + max_cluster_pgcount)) {
3077 /*
3078 * we have a write that fits entirely
3079 * within the existing cluster limits
3080 */
3081 if (cl.e_addr > wbp->cl_clusters[cl_index].e_addr)
3082 /*
3083 * update our idea of where the cluster ends
3084 */
3085 wbp->cl_clusters[cl_index].e_addr = cl.e_addr;
3086 break;
3087 }
3088 if (cl.b_addr < (wbp->cl_clusters[cl_index].b_addr + max_cluster_pgcount)) {
3089 /*
3090 * we have a write that starts in the middle of the current cluster
3091 * but extends beyond the cluster's limit... we know this because
3092 * of the previous checks
3093 * we'll extend the current cluster to the max
3094 * and update the b_addr for the current write to reflect that
3095 * the head of it was absorbed into this cluster...
3096 * note that we'll always have a leftover tail in this case since
3097 * full absorbtion would have occurred in the clause above
3098 */
3099 wbp->cl_clusters[cl_index].e_addr = wbp->cl_clusters[cl_index].b_addr + max_cluster_pgcount;
3100
3101 cl.b_addr = wbp->cl_clusters[cl_index].e_addr;
3102 }
3103 /*
3104 * we come here for the case where the current write starts
3105 * beyond the limit of the existing cluster or we have a leftover
3106 * tail after a partial absorbtion
3107 *
3108 * in either case, we'll check the remaining clusters before
3109 * starting a new one
3110 */
3111 } else {
3112 /*
3113 * the current write starts in front of the cluster we're currently considering
3114 */
3115 if ((wbp->cl_clusters[cl_index].e_addr - cl.b_addr) <= max_cluster_pgcount) {
3116 /*
3117 * we can just merge the new request into
3118 * this cluster and leave it in the cache
3119 * since the resulting cluster is still
3120 * less than the maximum allowable size
3121 */
3122 wbp->cl_clusters[cl_index].b_addr = cl.b_addr;
3123
3124 if (cl.e_addr > wbp->cl_clusters[cl_index].e_addr) {
3125 /*
3126 * the current write completely
3127 * envelops the existing cluster and since
3128 * each write is limited to at most max_cluster_pgcount pages
3129 * we can just use the start and last blocknos of the write
3130 * to generate the cluster limits
3131 */
3132 wbp->cl_clusters[cl_index].e_addr = cl.e_addr;
3133 }
3134 break;
3135 }
3136
3137 /*
3138 * if we were to combine this write with the current cluster
3139 * we would exceed the cluster size limit.... so,
3140 * let's see if there's any overlap of the new I/O with
3141 * the cluster we're currently considering... in fact, we'll
3142 * stretch the cluster out to it's full limit and see if we
3143 * get an intersection with the current write
3144 *
3145 */
3146 if (cl.e_addr > wbp->cl_clusters[cl_index].e_addr - max_cluster_pgcount) {
3147 /*
3148 * the current write extends into the proposed cluster
3149 * clip the length of the current write after first combining it's
3150 * tail with the newly shaped cluster
3151 */
3152 wbp->cl_clusters[cl_index].b_addr = wbp->cl_clusters[cl_index].e_addr - max_cluster_pgcount;
3153
3154 cl.e_addr = wbp->cl_clusters[cl_index].b_addr;
3155 }
3156 /*
3157 * if we get here, there was no way to merge
3158 * any portion of this write with this cluster
3159 * or we could only merge part of it which
3160 * will leave a tail...
3161 * we'll check the remaining clusters before starting a new one
3162 */
3163 }
3164 }
3165 if (cl_index < wbp->cl_number)
3166 /*
3167 * we found an existing cluster(s) that we
3168 * could entirely merge this I/O into
3169 */
3170 goto delay_io;
3171
3172 if (!((unsigned int)vfs_flags(vp->v_mount) & MNT_DEFWRITE) &&
3173 wbp->cl_number == MAX_CLUSTERS &&
3174 wbp->cl_seq_written >= (MAX_CLUSTERS * (max_cluster_pgcount * PAGE_SIZE))) {
3175 uint32_t n;
3176
3177 if (vp->v_mount->mnt_kern_flag & MNTK_SSD)
3178 n = WRITE_BEHIND_SSD;
3179 else
3180 n = WRITE_BEHIND;
3181
3182 while (n--)
3183 cluster_try_push(wbp, vp, newEOF, 0, 0, callback, callback_arg);
3184 }
3185 if (wbp->cl_number < MAX_CLUSTERS) {
3186 /*
3187 * we didn't find an existing cluster to
3188 * merge into, but there's room to start
3189 * a new one
3190 */
3191 goto start_new_cluster;
3192 }
3193 /*
3194 * no exisitng cluster to merge with and no
3195 * room to start a new one... we'll try
3196 * pushing one of the existing ones... if none of
3197 * them are able to be pushed, we'll switch
3198 * to the sparse cluster mechanism
3199 * cluster_try_push updates cl_number to the
3200 * number of remaining clusters... and
3201 * returns the number of currently unused clusters
3202 */
3203 ret_cluster_try_push = 0;
3204
3205 /*
3206 * if writes are not deferred, call cluster push immediately
3207 */
3208 if (!((unsigned int)vfs_flags(vp->v_mount) & MNT_DEFWRITE)) {
3209
3210 ret_cluster_try_push = cluster_try_push(wbp, vp, newEOF, (flags & IO_NOCACHE) ? 0 : PUSH_DELAY, 0, callback, callback_arg);
3211 }
3212
3213 /*
3214 * execute following regardless of writes being deferred or not
3215 */
3216 if (ret_cluster_try_push == 0) {
3217 /*
3218 * no more room in the normal cluster mechanism
3219 * so let's switch to the more expansive but expensive
3220 * sparse mechanism....
3221 */
3222 sparse_cluster_switch(wbp, vp, newEOF, callback, callback_arg);
3223 sparse_cluster_add(&(wbp->cl_scmap), vp, &cl, newEOF, callback, callback_arg);
3224
3225 lck_mtx_unlock(&wbp->cl_lockw);
3226
3227 continue;
3228 }
3229 start_new_cluster:
3230 wbp->cl_clusters[wbp->cl_number].b_addr = cl.b_addr;
3231 wbp->cl_clusters[wbp->cl_number].e_addr = cl.e_addr;
3232
3233 wbp->cl_clusters[wbp->cl_number].io_flags = 0;
3234
3235 if (flags & IO_NOCACHE)
3236 wbp->cl_clusters[wbp->cl_number].io_flags |= CLW_IONOCACHE;
3237
3238 if (bflag & CL_PASSIVE)
3239 wbp->cl_clusters[wbp->cl_number].io_flags |= CLW_IOPASSIVE;
3240
3241 wbp->cl_number++;
3242 delay_io:
3243 lck_mtx_unlock(&wbp->cl_lockw);
3244
3245 continue;
3246 issue_io:
3247 /*
3248 * we don't hold the lock at this point
3249 *
3250 * we've already dropped the current upl, so pick it back up with COPYOUT_FROM set
3251 * so that we correctly deal with a change in state of the hardware modify bit...
3252 * we do this via cluster_push_now... by passing along the IO_SYNC flag, we force
3253 * cluster_push_now to wait until all the I/Os have completed... cluster_push_now is also
3254 * responsible for generating the correct sized I/O(s)
3255 */
3256 retval = cluster_push_now(vp, &cl, newEOF, flags, callback, callback_arg);
3257 }
3258 }
3259 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 40)) | DBG_FUNC_END, retval, 0, io_resid, 0, 0);
3260
3261 return (retval);
3262 }
3263
3264
3265
3266 int
3267 cluster_read(vnode_t vp, struct uio *uio, off_t filesize, int xflags)
3268 {
3269 return cluster_read_ext(vp, uio, filesize, xflags, NULL, NULL);
3270 }
3271
3272
3273 int
3274 cluster_read_ext(vnode_t vp, struct uio *uio, off_t filesize, int xflags, int (*callback)(buf_t, void *), void *callback_arg)
3275 {
3276 int retval = 0;
3277 int flags;
3278 user_ssize_t cur_resid;
3279 u_int32_t io_size;
3280 u_int32_t read_length = 0;
3281 int read_type = IO_COPY;
3282
3283 flags = xflags;
3284
3285 if (vp->v_flag & VNOCACHE_DATA)
3286 flags |= IO_NOCACHE;
3287 if ((vp->v_flag & VRAOFF) || speculative_reads_disabled)
3288 flags |= IO_RAOFF;
3289
3290 /*
3291 * do a read through the cache if one of the following is true....
3292 * NOCACHE is not true
3293 * the uio request doesn't target USERSPACE
3294 * otherwise, find out if we want the direct or contig variant for
3295 * the first vector in the uio request
3296 */
3297 if ( (flags & IO_NOCACHE) && UIO_SEG_IS_USER_SPACE(uio->uio_segflg) )
3298 retval = cluster_io_type(uio, &read_type, &read_length, 0);
3299
3300 while ((cur_resid = uio_resid(uio)) && uio->uio_offset < filesize && retval == 0) {
3301
3302 switch (read_type) {
3303
3304 case IO_COPY:
3305 /*
3306 * make sure the uio_resid isn't too big...
3307 * internally, we want to handle all of the I/O in
3308 * chunk sizes that fit in a 32 bit int
3309 */
3310 if (cur_resid > (user_ssize_t)(MAX_IO_REQUEST_SIZE))
3311 io_size = MAX_IO_REQUEST_SIZE;
3312 else
3313 io_size = (u_int32_t)cur_resid;
3314
3315 retval = cluster_read_copy(vp, uio, io_size, filesize, flags, callback, callback_arg);
3316 break;
3317
3318 case IO_DIRECT:
3319 retval = cluster_read_direct(vp, uio, filesize, &read_type, &read_length, flags, callback, callback_arg);
3320 break;
3321
3322 case IO_CONTIG:
3323 retval = cluster_read_contig(vp, uio, filesize, &read_type, &read_length, callback, callback_arg, flags);
3324 break;
3325
3326 case IO_UNKNOWN:
3327 retval = cluster_io_type(uio, &read_type, &read_length, 0);
3328 break;
3329 }
3330 }
3331 return (retval);
3332 }
3333
3334
3335
3336 static void
3337 cluster_read_upl_release(upl_t upl, int start_pg, int last_pg, int take_reference)
3338 {
3339 int range;
3340 int abort_flags = UPL_ABORT_FREE_ON_EMPTY;
3341
3342 if ((range = last_pg - start_pg)) {
3343 if (take_reference)
3344 abort_flags |= UPL_ABORT_REFERENCE;
3345
3346 ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, range * PAGE_SIZE, abort_flags);
3347 }
3348 }
3349
3350
3351 static int
3352 cluster_read_copy(vnode_t vp, struct uio *uio, u_int32_t io_req_size, off_t filesize, int flags, int (*callback)(buf_t, void *), void *callback_arg)
3353 {
3354 upl_page_info_t *pl;
3355 upl_t upl;
3356 vm_offset_t upl_offset;
3357 u_int32_t upl_size;
3358 off_t upl_f_offset;
3359 int start_offset;
3360 int start_pg;
3361 int last_pg;
3362 int uio_last = 0;
3363 int pages_in_upl;
3364 off_t max_size;
3365 off_t last_ioread_offset;
3366 off_t last_request_offset;
3367 kern_return_t kret;
3368 int error = 0;
3369 int retval = 0;
3370 u_int32_t size_of_prefetch;
3371 u_int32_t xsize;
3372 u_int32_t io_size;
3373 u_int32_t max_rd_size;
3374 u_int32_t max_io_size;
3375 u_int32_t max_prefetch;
3376 u_int rd_ahead_enabled = 1;
3377 u_int prefetch_enabled = 1;
3378 struct cl_readahead * rap;
3379 struct clios iostate;
3380 struct cl_extent extent;
3381 int bflag;
3382 int take_reference = 1;
3383 #if CONFIG_EMBEDDED
3384 struct uthread *ut;
3385 #endif /* CONFIG_EMBEDDED */
3386 int policy = IOPOL_DEFAULT;
3387 boolean_t iolock_inited = FALSE;
3388
3389 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 32)) | DBG_FUNC_START,
3390 (int)uio->uio_offset, io_req_size, (int)filesize, flags, 0);
3391
3392 #if !CONFIG_EMBEDDED
3393 policy = proc_get_task_selfdiskacc();
3394 #else /* !CONFIG_EMBEDDED */
3395 policy = current_proc()->p_iopol_disk;
3396
3397 ut = get_bsdthread_info(current_thread());
3398
3399 if (ut->uu_iopol_disk != IOPOL_DEFAULT)
3400 policy = ut->uu_iopol_disk;
3401 #endif /* !CONFIG_EMBEDDED */
3402
3403 if (policy == IOPOL_THROTTLE || (flags & IO_NOCACHE))
3404 take_reference = 0;
3405
3406 if (flags & IO_PASSIVE)
3407 bflag = CL_PASSIVE;
3408 else
3409 bflag = 0;
3410
3411 max_io_size = cluster_max_io_size(vp->v_mount, CL_READ);
3412 max_prefetch = MAX_PREFETCH(vp, max_io_size, (vp->v_mount->mnt_kern_flag & MNTK_SSD));
3413 max_rd_size = max_prefetch;
3414
3415 last_request_offset = uio->uio_offset + io_req_size;
3416
3417 if (last_request_offset > filesize)
3418 last_request_offset = filesize;
3419
3420 if ((flags & (IO_RAOFF|IO_NOCACHE)) || ((last_request_offset & ~PAGE_MASK_64) == (uio->uio_offset & ~PAGE_MASK_64))) {
3421 rd_ahead_enabled = 0;
3422 rap = NULL;
3423 } else {
3424 if (cluster_hard_throttle_on(vp, 1)) {
3425 rd_ahead_enabled = 0;
3426 prefetch_enabled = 0;
3427
3428 max_rd_size = HARD_THROTTLE_MAXSIZE;
3429 } else if (policy == IOPOL_THROTTLE) {
3430 rd_ahead_enabled = 0;
3431 prefetch_enabled = 0;
3432 }
3433 if ((rap = cluster_get_rap(vp)) == NULL)
3434 rd_ahead_enabled = 0;
3435 else {
3436 extent.b_addr = uio->uio_offset / PAGE_SIZE_64;
3437 extent.e_addr = (last_request_offset - 1) / PAGE_SIZE_64;
3438 }
3439 }
3440 if (rap != NULL && rap->cl_ralen && (rap->cl_lastr == extent.b_addr || (rap->cl_lastr + 1) == extent.b_addr)) {
3441 /*
3442 * determine if we already have a read-ahead in the pipe courtesy of the
3443 * last read systemcall that was issued...
3444 * if so, pick up it's extent to determine where we should start
3445 * with respect to any read-ahead that might be necessary to
3446 * garner all the data needed to complete this read systemcall
3447 */
3448 last_ioread_offset = (rap->cl_maxra * PAGE_SIZE_64) + PAGE_SIZE_64;
3449
3450 if (last_ioread_offset < uio->uio_offset)
3451 last_ioread_offset = (off_t)0;
3452 else if (last_ioread_offset > last_request_offset)
3453 last_ioread_offset = last_request_offset;
3454 } else
3455 last_ioread_offset = (off_t)0;
3456
3457 while (io_req_size && uio->uio_offset < filesize && retval == 0) {
3458
3459 max_size = filesize - uio->uio_offset;
3460
3461 if ((off_t)(io_req_size) < max_size)
3462 io_size = io_req_size;
3463 else
3464 io_size = max_size;
3465
3466 if (!(flags & IO_NOCACHE)) {
3467
3468 while (io_size) {
3469 u_int32_t io_resid;
3470 u_int32_t io_requested;
3471
3472 /*
3473 * if we keep finding the pages we need already in the cache, then
3474 * don't bother to call cluster_read_prefetch since it costs CPU cycles
3475 * to determine that we have all the pages we need... once we miss in
3476 * the cache and have issued an I/O, than we'll assume that we're likely
3477 * to continue to miss in the cache and it's to our advantage to try and prefetch
3478 */
3479 if (last_request_offset && last_ioread_offset && (size_of_prefetch = (last_request_offset - last_ioread_offset))) {
3480 if ((last_ioread_offset - uio->uio_offset) <= max_rd_size && prefetch_enabled) {
3481 /*
3482 * we've already issued I/O for this request and
3483 * there's still work to do and
3484 * our prefetch stream is running dry, so issue a
3485 * pre-fetch I/O... the I/O latency will overlap
3486 * with the copying of the data
3487 */
3488 if (size_of_prefetch > max_rd_size)
3489 size_of_prefetch = max_rd_size;
3490
3491 size_of_prefetch = cluster_read_prefetch(vp, last_ioread_offset, size_of_prefetch, filesize, callback, callback_arg, bflag);
3492
3493 last_ioread_offset += (off_t)(size_of_prefetch * PAGE_SIZE);
3494
3495 if (last_ioread_offset > last_request_offset)
3496 last_ioread_offset = last_request_offset;
3497 }
3498 }
3499 /*
3500 * limit the size of the copy we're about to do so that
3501 * we can notice that our I/O pipe is running dry and
3502 * get the next I/O issued before it does go dry
3503 */
3504 if (last_ioread_offset && io_size > (max_io_size / 4))
3505 io_resid = (max_io_size / 4);
3506 else
3507 io_resid = io_size;
3508
3509 io_requested = io_resid;
3510
3511 retval = cluster_copy_ubc_data_internal(vp, uio, (int *)&io_resid, 0, take_reference);
3512
3513 xsize = io_requested - io_resid;
3514
3515 io_size -= xsize;
3516 io_req_size -= xsize;
3517
3518 if (retval || io_resid)
3519 /*
3520 * if we run into a real error or
3521 * a page that is not in the cache
3522 * we need to leave streaming mode
3523 */
3524 break;
3525
3526 if (rd_ahead_enabled && (io_size == 0 || last_ioread_offset == last_request_offset)) {
3527 /*
3528 * we're already finished the I/O for this read request
3529 * let's see if we should do a read-ahead
3530 */
3531 cluster_read_ahead(vp, &extent, filesize, rap, callback, callback_arg, bflag);
3532 }
3533 }
3534 if (retval)
3535 break;
3536 if (io_size == 0) {
3537 if (rap != NULL) {
3538 if (extent.e_addr < rap->cl_lastr)
3539 rap->cl_maxra = 0;
3540 rap->cl_lastr = extent.e_addr;
3541 }
3542 break;
3543 }
3544 /*
3545 * recompute max_size since cluster_copy_ubc_data_internal
3546 * may have advanced uio->uio_offset
3547 */
3548 max_size = filesize - uio->uio_offset;
3549 }
3550 /*
3551 * compute the size of the upl needed to encompass
3552 * the requested read... limit each call to cluster_io
3553 * to the maximum UPL size... cluster_io will clip if
3554 * this exceeds the maximum io_size for the device,
3555 * make sure to account for
3556 * a starting offset that's not page aligned
3557 */
3558 start_offset = (int)(uio->uio_offset & PAGE_MASK_64);
3559 upl_f_offset = uio->uio_offset - (off_t)start_offset;
3560
3561 if (io_size > max_rd_size)
3562 io_size = max_rd_size;
3563
3564 upl_size = (start_offset + io_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
3565
3566 if (flags & IO_NOCACHE) {
3567 if (upl_size > max_io_size)
3568 upl_size = max_io_size;
3569 } else {
3570 if (upl_size > max_io_size / 4)
3571 upl_size = max_io_size / 4;
3572 }
3573 pages_in_upl = upl_size / PAGE_SIZE;
3574
3575 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 33)) | DBG_FUNC_START,
3576 upl, (int)upl_f_offset, upl_size, start_offset, 0);
3577
3578 kret = ubc_create_upl(vp,
3579 upl_f_offset,
3580 upl_size,
3581 &upl,
3582 &pl,
3583 UPL_FILE_IO | UPL_SET_LITE);
3584 if (kret != KERN_SUCCESS)
3585 panic("cluster_read_copy: failed to get pagelist");
3586
3587 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 33)) | DBG_FUNC_END,
3588 upl, (int)upl_f_offset, upl_size, start_offset, 0);
3589
3590 /*
3591 * scan from the beginning of the upl looking for the first
3592 * non-valid page.... this will become the first page in
3593 * the request we're going to make to 'cluster_io'... if all
3594 * of the pages are valid, we won't call through to 'cluster_io'
3595 */
3596 for (start_pg = 0; start_pg < pages_in_upl; start_pg++) {
3597 if (!upl_valid_page(pl, start_pg))
3598 break;
3599 }
3600
3601 /*
3602 * scan from the starting invalid page looking for a valid
3603 * page before the end of the upl is reached, if we
3604 * find one, then it will be the last page of the request to
3605 * 'cluster_io'
3606 */
3607 for (last_pg = start_pg; last_pg < pages_in_upl; last_pg++) {
3608 if (upl_valid_page(pl, last_pg))
3609 break;
3610 }
3611 iostate.io_completed = 0;
3612 iostate.io_issued = 0;
3613 iostate.io_error = 0;
3614 iostate.io_wanted = 0;
3615
3616 if (start_pg < last_pg) {
3617 /*
3618 * we found a range of 'invalid' pages that must be filled
3619 * if the last page in this range is the last page of the file
3620 * we may have to clip the size of it to keep from reading past
3621 * the end of the last physical block associated with the file
3622 */
3623 if (iolock_inited == FALSE) {
3624 lck_mtx_init(&iostate.io_mtxp, cl_mtx_grp, cl_mtx_attr);
3625
3626 iolock_inited = TRUE;
3627 }
3628 upl_offset = start_pg * PAGE_SIZE;
3629 io_size = (last_pg - start_pg) * PAGE_SIZE;
3630
3631 if ((off_t)(upl_f_offset + upl_offset + io_size) > filesize)
3632 io_size = filesize - (upl_f_offset + upl_offset);
3633
3634 /*
3635 * issue an asynchronous read to cluster_io
3636 */
3637
3638 error = cluster_io(vp, upl, upl_offset, upl_f_offset + upl_offset,
3639 io_size, CL_READ | CL_ASYNC | bflag, (buf_t)NULL, &iostate, callback, callback_arg);
3640
3641 if (rap) {
3642 if (extent.e_addr < rap->cl_maxra) {
3643 /*
3644 * we've just issued a read for a block that should have been
3645 * in the cache courtesy of the read-ahead engine... something
3646 * has gone wrong with the pipeline, so reset the read-ahead
3647 * logic which will cause us to restart from scratch
3648 */
3649 rap->cl_maxra = 0;
3650 }
3651 }
3652 }
3653 if (error == 0) {
3654 /*
3655 * if the read completed successfully, or there was no I/O request
3656 * issued, than copy the data into user land via 'cluster_upl_copy_data'
3657 * we'll first add on any 'valid'
3658 * pages that were present in the upl when we acquired it.
3659 */
3660 u_int val_size;
3661
3662 for (uio_last = last_pg; uio_last < pages_in_upl; uio_last++) {
3663 if (!upl_valid_page(pl, uio_last))
3664 break;
3665 }
3666 if (uio_last < pages_in_upl) {
3667 /*
3668 * there were some invalid pages beyond the valid pages
3669 * that we didn't issue an I/O for, just release them
3670 * unchanged now, so that any prefetch/readahed can
3671 * include them
3672 */
3673 ubc_upl_abort_range(upl, uio_last * PAGE_SIZE,
3674 (pages_in_upl - uio_last) * PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
3675 }
3676
3677 /*
3678 * compute size to transfer this round, if io_req_size is
3679 * still non-zero after this attempt, we'll loop around and
3680 * set up for another I/O.
3681 */
3682 val_size = (uio_last * PAGE_SIZE) - start_offset;
3683
3684 if (val_size > max_size)
3685 val_size = max_size;
3686
3687 if (val_size > io_req_size)
3688 val_size = io_req_size;
3689
3690 if ((uio->uio_offset + val_size) > last_ioread_offset)
3691 last_ioread_offset = uio->uio_offset + val_size;
3692
3693 if ((size_of_prefetch = (last_request_offset - last_ioread_offset)) && prefetch_enabled) {
3694
3695 if ((last_ioread_offset - (uio->uio_offset + val_size)) <= upl_size) {
3696 /*
3697 * if there's still I/O left to do for this request, and...
3698 * we're not in hard throttle mode, and...
3699 * we're close to using up the previous prefetch, then issue a
3700 * new pre-fetch I/O... the I/O latency will overlap
3701 * with the copying of the data
3702 */
3703 if (size_of_prefetch > max_rd_size)
3704 size_of_prefetch = max_rd_size;
3705
3706 size_of_prefetch = cluster_read_prefetch(vp, last_ioread_offset, size_of_prefetch, filesize, callback, callback_arg, bflag);
3707
3708 last_ioread_offset += (off_t)(size_of_prefetch * PAGE_SIZE);
3709
3710 if (last_ioread_offset > last_request_offset)
3711 last_ioread_offset = last_request_offset;
3712 }
3713
3714 } else if ((uio->uio_offset + val_size) == last_request_offset) {
3715 /*
3716 * this transfer will finish this request, so...
3717 * let's try to read ahead if we're in
3718 * a sequential access pattern and we haven't
3719 * explicitly disabled it
3720 */
3721 if (rd_ahead_enabled)
3722 cluster_read_ahead(vp, &extent, filesize, rap, callback, callback_arg, bflag);
3723
3724 if (rap != NULL) {
3725 if (extent.e_addr < rap->cl_lastr)
3726 rap->cl_maxra = 0;
3727 rap->cl_lastr = extent.e_addr;
3728 }
3729 }
3730 if (iostate.io_issued > iostate.io_completed)
3731 cluster_iostate_wait(&iostate, 0, "cluster_read_copy");
3732
3733 if (iostate.io_error)
3734 error = iostate.io_error;
3735 else {
3736 u_int32_t io_requested;
3737
3738 io_requested = val_size;
3739
3740 retval = cluster_copy_upl_data(uio, upl, start_offset, (int *)&io_requested);
3741
3742 io_req_size -= (val_size - io_requested);
3743 }
3744 } else {
3745 if (iostate.io_issued > iostate.io_completed)
3746 cluster_iostate_wait(&iostate, 0, "cluster_read_copy");
3747 }
3748 if (start_pg < last_pg) {
3749 /*
3750 * compute the range of pages that we actually issued an I/O for
3751 * and either commit them as valid if the I/O succeeded
3752 * or abort them if the I/O failed or we're not supposed to
3753 * keep them in the cache
3754 */
3755 io_size = (last_pg - start_pg) * PAGE_SIZE;
3756
3757 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 35)) | DBG_FUNC_START, upl, start_pg * PAGE_SIZE, io_size, error, 0);
3758
3759 if (error || (flags & IO_NOCACHE))
3760 ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, io_size,
3761 UPL_ABORT_DUMP_PAGES | UPL_ABORT_FREE_ON_EMPTY);
3762 else {
3763 int commit_flags = UPL_COMMIT_CLEAR_DIRTY | UPL_COMMIT_FREE_ON_EMPTY;
3764
3765 if (take_reference)
3766 commit_flags |= UPL_COMMIT_INACTIVATE;
3767 else
3768 commit_flags |= UPL_COMMIT_SPECULATE;
3769
3770 ubc_upl_commit_range(upl, start_pg * PAGE_SIZE, io_size, commit_flags);
3771 }
3772 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 35)) | DBG_FUNC_END, upl, start_pg * PAGE_SIZE, io_size, error, 0);
3773 }
3774 if ((last_pg - start_pg) < pages_in_upl) {
3775 /*
3776 * the set of pages that we issued an I/O for did not encompass
3777 * the entire upl... so just release these without modifying
3778 * their state
3779 */
3780 if (error)
3781 ubc_upl_abort_range(upl, 0, upl_size, UPL_ABORT_FREE_ON_EMPTY);
3782 else {
3783
3784 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 35)) | DBG_FUNC_START,
3785 upl, -1, pages_in_upl - (last_pg - start_pg), 0, 0);
3786
3787 /*
3788 * handle any valid pages at the beginning of
3789 * the upl... release these appropriately
3790 */
3791 cluster_read_upl_release(upl, 0, start_pg, take_reference);
3792
3793 /*
3794 * handle any valid pages immediately after the
3795 * pages we issued I/O for... ... release these appropriately
3796 */
3797 cluster_read_upl_release(upl, last_pg, uio_last, take_reference);
3798
3799 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 35)) | DBG_FUNC_END, upl, -1, -1, 0, 0);
3800 }
3801 }
3802 if (retval == 0)
3803 retval = error;
3804
3805 if (io_req_size) {
3806 if (cluster_hard_throttle_on(vp, 1)) {
3807 rd_ahead_enabled = 0;
3808 prefetch_enabled = 0;
3809
3810 max_rd_size = HARD_THROTTLE_MAXSIZE;
3811 } else {
3812 if (max_rd_size == HARD_THROTTLE_MAXSIZE) {
3813 /*
3814 * coming out of throttled state
3815 */
3816 if (policy != IOPOL_THROTTLE) {
3817 if (rap != NULL)
3818 rd_ahead_enabled = 1;
3819 prefetch_enabled = 1;
3820 }
3821 max_rd_size = max_prefetch;
3822 last_ioread_offset = 0;
3823 }
3824 }
3825 }
3826 }
3827 if (iolock_inited == TRUE) {
3828 if (iostate.io_issued > iostate.io_completed) {
3829 /*
3830 * cluster_io returned an error after it
3831 * had already issued some I/O. we need
3832 * to wait for that I/O to complete before
3833 * we can destroy the iostate mutex...
3834 * 'retval' already contains the early error
3835 * so no need to pick it up from iostate.io_error
3836 */
3837 cluster_iostate_wait(&iostate, 0, "cluster_read_copy");
3838 }
3839 lck_mtx_destroy(&iostate.io_mtxp, cl_mtx_grp);
3840 }
3841 if (rap != NULL) {
3842 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 32)) | DBG_FUNC_END,
3843 (int)uio->uio_offset, io_req_size, rap->cl_lastr, retval, 0);
3844
3845 lck_mtx_unlock(&rap->cl_lockr);
3846 } else {
3847 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 32)) | DBG_FUNC_END,
3848 (int)uio->uio_offset, io_req_size, 0, retval, 0);
3849 }
3850
3851 return (retval);
3852 }
3853
3854
3855 static int
3856 cluster_read_direct(vnode_t vp, struct uio *uio, off_t filesize, int *read_type, u_int32_t *read_length,
3857 int flags, int (*callback)(buf_t, void *), void *callback_arg)
3858 {
3859 upl_t upl;
3860 upl_page_info_t *pl;
3861 off_t max_io_size;
3862 vm_offset_t upl_offset, vector_upl_offset = 0;
3863 upl_size_t upl_size, vector_upl_size = 0;
3864 vm_size_t upl_needed_size;
3865 unsigned int pages_in_pl;
3866 int upl_flags;
3867 kern_return_t kret;
3868 unsigned int i;
3869 int force_data_sync;
3870 int retval = 0;
3871 int no_zero_fill = 0;
3872 int io_flag = 0;
3873 int misaligned = 0;
3874 struct clios iostate;
3875 user_addr_t iov_base;
3876 u_int32_t io_req_size;
3877 u_int32_t offset_in_file;
3878 u_int32_t offset_in_iovbase;
3879 u_int32_t io_size;
3880 u_int32_t io_min;
3881 u_int32_t xsize;
3882 u_int32_t devblocksize;
3883 u_int32_t mem_alignment_mask;
3884 u_int32_t max_upl_size;
3885 u_int32_t max_rd_size;
3886 u_int32_t max_rd_ahead;
3887 boolean_t strict_uncached_IO = FALSE;
3888
3889 u_int32_t vector_upl_iosize = 0;
3890 int issueVectorUPL = 0,useVectorUPL = (uio->uio_iovcnt > 1);
3891 off_t v_upl_uio_offset = 0;
3892 int vector_upl_index=0;
3893 upl_t vector_upl = NULL;
3894
3895 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 70)) | DBG_FUNC_START,
3896 (int)uio->uio_offset, (int)filesize, *read_type, *read_length, 0);
3897
3898 max_upl_size = cluster_max_io_size(vp->v_mount, CL_READ);
3899
3900 max_rd_size = max_upl_size;
3901 max_rd_ahead = max_rd_size * IO_SCALE(vp, 2);
3902
3903 io_flag = CL_COMMIT | CL_READ | CL_ASYNC | CL_NOZERO | CL_DIRECT_IO;
3904
3905 if (flags & IO_PASSIVE)
3906 io_flag |= CL_PASSIVE;
3907
3908 iostate.io_completed = 0;
3909 iostate.io_issued = 0;
3910 iostate.io_error = 0;
3911 iostate.io_wanted = 0;
3912
3913 lck_mtx_init(&iostate.io_mtxp, cl_mtx_grp, cl_mtx_attr);
3914
3915 devblocksize = (u_int32_t)vp->v_mount->mnt_devblocksize;
3916 mem_alignment_mask = (u_int32_t)vp->v_mount->mnt_alignmentmask;
3917
3918 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 70)) | DBG_FUNC_NONE,
3919 (int)devblocksize, (int)mem_alignment_mask, 0, 0, 0);
3920
3921 if (devblocksize == 1) {
3922 /*
3923 * the AFP client advertises a devblocksize of 1
3924 * however, its BLOCKMAP routine maps to physical
3925 * blocks that are PAGE_SIZE in size...
3926 * therefore we can't ask for I/Os that aren't page aligned
3927 * or aren't multiples of PAGE_SIZE in size
3928 * by setting devblocksize to PAGE_SIZE, we re-instate
3929 * the old behavior we had before the mem_alignment_mask
3930 * changes went in...
3931 */
3932 devblocksize = PAGE_SIZE;
3933 }
3934
3935 strict_uncached_IO = ubc_strict_uncached_IO(vp);
3936
3937 next_dread:
3938 io_req_size = *read_length;
3939 iov_base = uio_curriovbase(uio);
3940
3941 max_io_size = filesize - uio->uio_offset;
3942
3943 if ((off_t)io_req_size > max_io_size)
3944 io_req_size = max_io_size;
3945
3946 offset_in_file = (u_int32_t)uio->uio_offset & (devblocksize - 1);
3947 offset_in_iovbase = (u_int32_t)iov_base & mem_alignment_mask;
3948
3949 if (offset_in_file || offset_in_iovbase) {
3950 /*
3951 * one of the 2 important offsets is misaligned
3952 * so fire an I/O through the cache for this entire vector
3953 */
3954 misaligned = 1;
3955 }
3956 if (iov_base & (devblocksize - 1)) {
3957 /*
3958 * the offset in memory must be on a device block boundary
3959 * so that we can guarantee that we can generate an
3960 * I/O that ends on a page boundary in cluster_io
3961 */
3962 misaligned = 1;
3963 }
3964 /*
3965 * When we get to this point, we know...
3966 * -- the offset into the file is on a devblocksize boundary
3967 */
3968
3969 while (io_req_size && retval == 0) {
3970 u_int32_t io_start;
3971
3972 if (cluster_hard_throttle_on(vp, 1)) {
3973 max_rd_size = HARD_THROTTLE_MAXSIZE;
3974 max_rd_ahead = HARD_THROTTLE_MAXSIZE - 1;
3975 } else {
3976 max_rd_size = max_upl_size;
3977 max_rd_ahead = max_rd_size * IO_SCALE(vp, 2);
3978 }
3979 io_start = io_size = io_req_size;
3980
3981 /*
3982 * First look for pages already in the cache
3983 * and move them to user space.
3984 *
3985 * cluster_copy_ubc_data returns the resid
3986 * in io_size
3987 */
3988 if (strict_uncached_IO == FALSE) {
3989 retval = cluster_copy_ubc_data_internal(vp, uio, (int *)&io_size, 0, 0);
3990 }
3991 /*
3992 * calculate the number of bytes actually copied
3993 * starting size - residual
3994 */
3995 xsize = io_start - io_size;
3996
3997 io_req_size -= xsize;
3998
3999 if(useVectorUPL && (xsize || (iov_base & PAGE_MASK))) {
4000 /*
4001 * We found something in the cache or we have an iov_base that's not
4002 * page-aligned.
4003 *
4004 * Issue all I/O's that have been collected within this Vectored UPL.
4005 */
4006 if(vector_upl_index) {
4007 retval = vector_cluster_io(vp, vector_upl, vector_upl_offset, v_upl_uio_offset, vector_upl_iosize, io_flag, (buf_t)NULL, &iostate, callback, callback_arg);
4008 reset_vector_run_state();
4009 }
4010
4011 if(xsize)
4012 useVectorUPL = 0;
4013
4014 /*
4015 * After this point, if we are using the Vector UPL path and the base is
4016 * not page-aligned then the UPL with that base will be the first in the vector UPL.
4017 */
4018 }
4019
4020 /*
4021 * check to see if we are finished with this request...
4022 */
4023 if (io_req_size == 0 || misaligned) {
4024 /*
4025 * see if there's another uio vector to
4026 * process that's of type IO_DIRECT
4027 *
4028 * break out of while loop to get there
4029 */
4030 break;
4031 }
4032 /*
4033 * assume the request ends on a device block boundary
4034 */
4035 io_min = devblocksize;
4036
4037 /*
4038 * we can handle I/O's in multiples of the device block size
4039 * however, if io_size isn't a multiple of devblocksize we
4040 * want to clip it back to the nearest page boundary since
4041 * we are going to have to go through cluster_read_copy to
4042 * deal with the 'overhang'... by clipping it to a PAGE_SIZE
4043 * multiple, we avoid asking the drive for the same physical
4044 * blocks twice.. once for the partial page at the end of the
4045 * request and a 2nd time for the page we read into the cache
4046 * (which overlaps the end of the direct read) in order to
4047 * get at the overhang bytes
4048 */
4049 if (io_size & (devblocksize - 1)) {
4050 /*
4051 * request does NOT end on a device block boundary
4052 * so clip it back to a PAGE_SIZE boundary
4053 */
4054 io_size &= ~PAGE_MASK;
4055 io_min = PAGE_SIZE;
4056 }
4057 if (retval || io_size < io_min) {
4058 /*
4059 * either an error or we only have the tail left to
4060 * complete via the copy path...
4061 * we may have already spun some portion of this request
4062 * off as async requests... we need to wait for the I/O
4063 * to complete before returning
4064 */
4065 goto wait_for_dreads;
4066 }
4067
4068 if (strict_uncached_IO == FALSE) {
4069
4070 if ((xsize = io_size) > max_rd_size)
4071 xsize = max_rd_size;
4072
4073 io_size = 0;
4074
4075 ubc_range_op(vp, uio->uio_offset, uio->uio_offset + xsize, UPL_ROP_ABSENT, (int *)&io_size);
4076
4077 if (io_size == 0) {
4078 /*
4079 * a page must have just come into the cache
4080 * since the first page in this range is no
4081 * longer absent, go back and re-evaluate
4082 */
4083 continue;
4084 }
4085 }
4086
4087 iov_base = uio_curriovbase(uio);
4088
4089 upl_offset = (vm_offset_t)((u_int32_t)iov_base & PAGE_MASK);
4090 upl_needed_size = (upl_offset + io_size + (PAGE_SIZE -1)) & ~PAGE_MASK;
4091
4092 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 72)) | DBG_FUNC_START,
4093 (int)upl_offset, upl_needed_size, (int)iov_base, io_size, 0);
4094
4095 if (upl_offset == 0 && ((io_size & PAGE_MASK) == 0))
4096 no_zero_fill = 1;
4097 else
4098 no_zero_fill = 0;
4099
4100 for (force_data_sync = 0; force_data_sync < 3; force_data_sync++) {
4101 pages_in_pl = 0;
4102 upl_size = upl_needed_size;
4103 upl_flags = UPL_FILE_IO | UPL_NO_SYNC | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE;
4104
4105 if (no_zero_fill)
4106 upl_flags |= UPL_NOZEROFILL;
4107 if (force_data_sync)
4108 upl_flags |= UPL_FORCE_DATA_SYNC;
4109
4110 kret = vm_map_create_upl(current_map(),
4111 (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)),
4112 &upl_size, &upl, NULL, &pages_in_pl, &upl_flags);
4113
4114 if (kret != KERN_SUCCESS) {
4115 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 72)) | DBG_FUNC_END,
4116 (int)upl_offset, upl_size, io_size, kret, 0);
4117 /*
4118 * failed to get pagelist
4119 *
4120 * we may have already spun some portion of this request
4121 * off as async requests... we need to wait for the I/O
4122 * to complete before returning
4123 */
4124 goto wait_for_dreads;
4125 }
4126 pages_in_pl = upl_size / PAGE_SIZE;
4127 pl = UPL_GET_INTERNAL_PAGE_LIST(upl);
4128
4129 for (i = 0; i < pages_in_pl; i++) {
4130 if (!upl_page_present(pl, i))
4131 break;
4132 }
4133 if (i == pages_in_pl)
4134 break;
4135
4136 ubc_upl_abort(upl, 0);
4137 }
4138 if (force_data_sync >= 3) {
4139 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 72)) | DBG_FUNC_END,
4140 (int)upl_offset, upl_size, io_size, kret, 0);
4141
4142 goto wait_for_dreads;
4143 }
4144 /*
4145 * Consider the possibility that upl_size wasn't satisfied.
4146 */
4147 if (upl_size < upl_needed_size) {
4148 if (upl_size && upl_offset == 0)
4149 io_size = upl_size;
4150 else
4151 io_size = 0;
4152 }
4153 if (io_size == 0) {
4154 ubc_upl_abort(upl, 0);
4155 goto wait_for_dreads;
4156 }
4157 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 72)) | DBG_FUNC_END,
4158 (int)upl_offset, upl_size, io_size, kret, 0);
4159
4160 if(useVectorUPL) {
4161 vm_offset_t end_off = ((iov_base + io_size) & PAGE_MASK);
4162 if(end_off)
4163 issueVectorUPL = 1;
4164 /*
4165 * After this point, if we are using a vector UPL, then
4166 * either all the UPL elements end on a page boundary OR
4167 * this UPL is the last element because it does not end
4168 * on a page boundary.
4169 */
4170 }
4171
4172 /*
4173 * request asynchronously so that we can overlap
4174 * the preparation of the next I/O
4175 * if there are already too many outstanding reads
4176 * wait until some have completed before issuing the next read
4177 */
4178 if (iostate.io_issued > iostate.io_completed)
4179 cluster_iostate_wait(&iostate, max_rd_ahead, "cluster_read_direct");
4180
4181 if (iostate.io_error) {
4182 /*
4183 * one of the earlier reads we issued ran into a hard error
4184 * don't issue any more reads, cleanup the UPL
4185 * that was just created but not used, then
4186 * go wait for any other reads to complete before
4187 * returning the error to the caller
4188 */
4189 ubc_upl_abort(upl, 0);
4190
4191 goto wait_for_dreads;
4192 }
4193 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 73)) | DBG_FUNC_START,
4194 upl, (int)upl_offset, (int)uio->uio_offset, io_size, 0);
4195
4196
4197 if(!useVectorUPL) {
4198 if (no_zero_fill)
4199 io_flag &= ~CL_PRESERVE;
4200 else
4201 io_flag |= CL_PRESERVE;
4202
4203 retval = cluster_io(vp, upl, upl_offset, uio->uio_offset, io_size, io_flag, (buf_t)NULL, &iostate, callback, callback_arg);
4204
4205 } else {
4206
4207 if(!vector_upl_index) {
4208 vector_upl = vector_upl_create(upl_offset);
4209 v_upl_uio_offset = uio->uio_offset;
4210 vector_upl_offset = upl_offset;
4211 }
4212
4213 vector_upl_set_subupl(vector_upl,upl, upl_size);
4214 vector_upl_set_iostate(vector_upl, upl, vector_upl_size, upl_size);
4215 vector_upl_index++;
4216 vector_upl_size += upl_size;
4217 vector_upl_iosize += io_size;
4218
4219 if(issueVectorUPL || vector_upl_index == MAX_VECTOR_UPL_ELEMENTS || vector_upl_size >= MAX_VECTOR_UPL_SIZE) {
4220 retval = vector_cluster_io(vp, vector_upl, vector_upl_offset, v_upl_uio_offset, vector_upl_iosize, io_flag, (buf_t)NULL, &iostate, callback, callback_arg);
4221 reset_vector_run_state();
4222 }
4223 }
4224 /*
4225 * update the uio structure
4226 */
4227 uio_update(uio, (user_size_t)io_size);
4228
4229 io_req_size -= io_size;
4230
4231 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 73)) | DBG_FUNC_END,
4232 upl, (int)uio->uio_offset, io_req_size, retval, 0);
4233
4234 } /* end while */
4235
4236 if (retval == 0 && iostate.io_error == 0 && io_req_size == 0 && uio->uio_offset < filesize) {
4237
4238 retval = cluster_io_type(uio, read_type, read_length, 0);
4239
4240 if (retval == 0 && *read_type == IO_DIRECT) {
4241
4242 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 70)) | DBG_FUNC_NONE,
4243 (int)uio->uio_offset, (int)filesize, *read_type, *read_length, 0);
4244
4245 goto next_dread;
4246 }
4247 }
4248
4249 wait_for_dreads:
4250
4251 if(retval == 0 && iostate.io_error == 0 && useVectorUPL && vector_upl_index) {
4252 retval = vector_cluster_io(vp, vector_upl, vector_upl_offset, v_upl_uio_offset, vector_upl_iosize, io_flag, (buf_t)NULL, &iostate, callback, callback_arg);
4253 reset_vector_run_state();
4254 }
4255 /*
4256 * make sure all async reads that are part of this stream
4257 * have completed before we return
4258 */
4259 if (iostate.io_issued > iostate.io_completed)
4260 cluster_iostate_wait(&iostate, 0, "cluster_read_direct");
4261
4262 if (iostate.io_error)
4263 retval = iostate.io_error;
4264
4265 lck_mtx_destroy(&iostate.io_mtxp, cl_mtx_grp);
4266
4267 if (io_req_size && retval == 0) {
4268 /*
4269 * we couldn't handle the tail of this request in DIRECT mode
4270 * so fire it through the copy path
4271 */
4272 retval = cluster_read_copy(vp, uio, io_req_size, filesize, flags, callback, callback_arg);
4273
4274 *read_type = IO_UNKNOWN;
4275 }
4276 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 70)) | DBG_FUNC_END,
4277 (int)uio->uio_offset, (int)uio_resid(uio), io_req_size, retval, 0);
4278
4279 return (retval);
4280 }
4281
4282
4283 static int
4284 cluster_read_contig(vnode_t vp, struct uio *uio, off_t filesize, int *read_type, u_int32_t *read_length,
4285 int (*callback)(buf_t, void *), void *callback_arg, int flags)
4286 {
4287 upl_page_info_t *pl;
4288 upl_t upl[MAX_VECTS];
4289 vm_offset_t upl_offset;
4290 addr64_t dst_paddr = 0;
4291 user_addr_t iov_base;
4292 off_t max_size;
4293 upl_size_t upl_size;
4294 vm_size_t upl_needed_size;
4295 mach_msg_type_number_t pages_in_pl;
4296 int upl_flags;
4297 kern_return_t kret;
4298 struct clios iostate;
4299 int error= 0;
4300 int cur_upl = 0;
4301 int num_upl = 0;
4302 int n;
4303 u_int32_t xsize;
4304 u_int32_t io_size;
4305 u_int32_t devblocksize;
4306 u_int32_t mem_alignment_mask;
4307 u_int32_t tail_size = 0;
4308 int bflag;
4309
4310 if (flags & IO_PASSIVE)
4311 bflag = CL_PASSIVE;
4312 else
4313 bflag = 0;
4314
4315 /*
4316 * When we enter this routine, we know
4317 * -- the read_length will not exceed the current iov_len
4318 * -- the target address is physically contiguous for read_length
4319 */
4320 cluster_syncup(vp, filesize, callback, callback_arg);
4321
4322 devblocksize = (u_int32_t)vp->v_mount->mnt_devblocksize;
4323 mem_alignment_mask = (u_int32_t)vp->v_mount->mnt_alignmentmask;
4324
4325 iostate.io_completed = 0;
4326 iostate.io_issued = 0;
4327 iostate.io_error = 0;
4328 iostate.io_wanted = 0;
4329
4330 lck_mtx_init(&iostate.io_mtxp, cl_mtx_grp, cl_mtx_attr);
4331
4332 next_cread:
4333 io_size = *read_length;
4334
4335 max_size = filesize - uio->uio_offset;
4336
4337 if (io_size > max_size)
4338 io_size = max_size;
4339
4340 iov_base = uio_curriovbase(uio);
4341
4342 upl_offset = (vm_offset_t)((u_int32_t)iov_base & PAGE_MASK);
4343 upl_needed_size = upl_offset + io_size;
4344
4345 pages_in_pl = 0;
4346 upl_size = upl_needed_size;
4347 upl_flags = UPL_FILE_IO | UPL_NO_SYNC | UPL_CLEAN_IN_PLACE | UPL_SET_INTERNAL | UPL_SET_LITE | UPL_SET_IO_WIRE;
4348
4349
4350 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 92)) | DBG_FUNC_START,
4351 (int)upl_offset, (int)upl_size, (int)iov_base, io_size, 0);
4352
4353 kret = vm_map_get_upl(current_map(),
4354 (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)),
4355 &upl_size, &upl[cur_upl], NULL, &pages_in_pl, &upl_flags, 0);
4356
4357 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 92)) | DBG_FUNC_END,
4358 (int)upl_offset, upl_size, io_size, kret, 0);
4359
4360 if (kret != KERN_SUCCESS) {
4361 /*
4362 * failed to get pagelist
4363 */
4364 error = EINVAL;
4365 goto wait_for_creads;
4366 }
4367 num_upl++;
4368
4369 if (upl_size < upl_needed_size) {
4370 /*
4371 * The upl_size wasn't satisfied.
4372 */
4373 error = EINVAL;
4374 goto wait_for_creads;
4375 }
4376 pl = ubc_upl_pageinfo(upl[cur_upl]);
4377
4378 dst_paddr = ((addr64_t)upl_phys_page(pl, 0) << 12) + (addr64_t)upl_offset;
4379
4380 while (((uio->uio_offset & (devblocksize - 1)) || io_size < devblocksize) && io_size) {
4381 u_int32_t head_size;
4382
4383 head_size = devblocksize - (u_int32_t)(uio->uio_offset & (devblocksize - 1));
4384
4385 if (head_size > io_size)
4386 head_size = io_size;
4387
4388 error = cluster_align_phys_io(vp, uio, dst_paddr, head_size, CL_READ, callback, callback_arg);
4389
4390 if (error)
4391 goto wait_for_creads;
4392
4393 upl_offset += head_size;
4394 dst_paddr += head_size;
4395 io_size -= head_size;
4396
4397 iov_base += head_size;
4398 }
4399 if ((u_int32_t)iov_base & mem_alignment_mask) {
4400 /*
4401 * request doesn't set up on a memory boundary
4402 * the underlying DMA engine can handle...
4403 * return an error instead of going through
4404 * the slow copy path since the intent of this
4405 * path is direct I/O to device memory
4406 */
4407 error = EINVAL;
4408 goto wait_for_creads;
4409 }
4410
4411 tail_size = io_size & (devblocksize - 1);
4412
4413 io_size -= tail_size;
4414
4415 while (io_size && error == 0) {
4416
4417 if (io_size > MAX_IO_CONTIG_SIZE)
4418 xsize = MAX_IO_CONTIG_SIZE;
4419 else
4420 xsize = io_size;
4421 /*
4422 * request asynchronously so that we can overlap
4423 * the preparation of the next I/O... we'll do
4424 * the commit after all the I/O has completed
4425 * since its all issued against the same UPL
4426 * if there are already too many outstanding reads
4427 * wait until some have completed before issuing the next
4428 */
4429 if (iostate.io_issued > iostate.io_completed)
4430 cluster_iostate_wait(&iostate, MAX_IO_CONTIG_SIZE * IO_SCALE(vp, 2), "cluster_read_contig");
4431
4432 if (iostate.io_error) {
4433 /*
4434 * one of the earlier reads we issued ran into a hard error
4435 * don't issue any more reads...
4436 * go wait for any other reads to complete before
4437 * returning the error to the caller
4438 */
4439 goto wait_for_creads;
4440 }
4441 error = cluster_io(vp, upl[cur_upl], upl_offset, uio->uio_offset, xsize,
4442 CL_READ | CL_NOZERO | CL_DEV_MEMORY | CL_ASYNC | bflag,
4443 (buf_t)NULL, &iostate, callback, callback_arg);
4444 /*
4445 * The cluster_io read was issued successfully,
4446 * update the uio structure
4447 */
4448 if (error == 0) {
4449 uio_update(uio, (user_size_t)xsize);
4450
4451 dst_paddr += xsize;
4452 upl_offset += xsize;
4453 io_size -= xsize;
4454 }
4455 }
4456 if (error == 0 && iostate.io_error == 0 && tail_size == 0 && num_upl < MAX_VECTS && uio->uio_offset < filesize) {
4457
4458 error = cluster_io_type(uio, read_type, read_length, 0);
4459
4460 if (error == 0 && *read_type == IO_CONTIG) {
4461 cur_upl++;
4462 goto next_cread;
4463 }
4464 } else
4465 *read_type = IO_UNKNOWN;
4466
4467 wait_for_creads:
4468 /*
4469 * make sure all async reads that are part of this stream
4470 * have completed before we proceed
4471 */
4472 if (iostate.io_issued > iostate.io_completed)
4473 cluster_iostate_wait(&iostate, 0, "cluster_read_contig");
4474
4475 if (iostate.io_error)
4476 error = iostate.io_error;
4477
4478 lck_mtx_destroy(&iostate.io_mtxp, cl_mtx_grp);
4479
4480 if (error == 0 && tail_size)
4481 error = cluster_align_phys_io(vp, uio, dst_paddr, tail_size, CL_READ, callback, callback_arg);
4482
4483 for (n = 0; n < num_upl; n++)
4484 /*
4485 * just release our hold on each physically contiguous
4486 * region without changing any state
4487 */
4488 ubc_upl_abort(upl[n], 0);
4489
4490 return (error);
4491 }
4492
4493
4494 static int
4495 cluster_io_type(struct uio *uio, int *io_type, u_int32_t *io_length, u_int32_t min_length)
4496 {
4497 user_size_t iov_len;
4498 user_addr_t iov_base = 0;
4499 upl_t upl;
4500 upl_size_t upl_size;
4501 int upl_flags;
4502 int retval = 0;
4503
4504 /*
4505 * skip over any emtpy vectors
4506 */
4507 uio_update(uio, (user_size_t)0);
4508
4509 iov_len = uio_curriovlen(uio);
4510
4511 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 94)) | DBG_FUNC_START, uio, (int)iov_len, 0, 0, 0);
4512
4513 if (iov_len) {
4514 iov_base = uio_curriovbase(uio);
4515 /*
4516 * make sure the size of the vector isn't too big...
4517 * internally, we want to handle all of the I/O in
4518 * chunk sizes that fit in a 32 bit int
4519 */
4520 if (iov_len > (user_size_t)MAX_IO_REQUEST_SIZE)
4521 upl_size = MAX_IO_REQUEST_SIZE;
4522 else
4523 upl_size = (u_int32_t)iov_len;
4524
4525 upl_flags = UPL_QUERY_OBJECT_TYPE;
4526
4527 if ((vm_map_get_upl(current_map(),
4528 (vm_map_offset_t)(iov_base & ~((user_addr_t)PAGE_MASK)),
4529 &upl_size, &upl, NULL, NULL, &upl_flags, 0)) != KERN_SUCCESS) {
4530 /*
4531 * the user app must have passed in an invalid address
4532 */
4533 retval = EFAULT;
4534 }
4535 if (upl_size == 0)
4536 retval = EFAULT;
4537
4538 *io_length = upl_size;
4539
4540 if (upl_flags & UPL_PHYS_CONTIG)
4541 *io_type = IO_CONTIG;
4542 else if (iov_len >= min_length)
4543 *io_type = IO_DIRECT;
4544 else
4545 *io_type = IO_COPY;
4546 } else {
4547 /*
4548 * nothing left to do for this uio
4549 */
4550 *io_length = 0;
4551 *io_type = IO_UNKNOWN;
4552 }
4553 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 94)) | DBG_FUNC_END, iov_base, *io_type, *io_length, retval, 0);
4554
4555 return (retval);
4556 }
4557
4558
4559 /*
4560 * generate advisory I/O's in the largest chunks possible
4561 * the completed pages will be released into the VM cache
4562 */
4563 int
4564 advisory_read(vnode_t vp, off_t filesize, off_t f_offset, int resid)
4565 {
4566 return advisory_read_ext(vp, filesize, f_offset, resid, NULL, NULL, CL_PASSIVE);
4567 }
4568
4569 int
4570 advisory_read_ext(vnode_t vp, off_t filesize, off_t f_offset, int resid, int (*callback)(buf_t, void *), void *callback_arg, int bflag)
4571 {
4572 upl_page_info_t *pl;
4573 upl_t upl;
4574 vm_offset_t upl_offset;
4575 int upl_size;
4576 off_t upl_f_offset;
4577 int start_offset;
4578 int start_pg;
4579 int last_pg;
4580 int pages_in_upl;
4581 off_t max_size;
4582 int io_size;
4583 kern_return_t kret;
4584 int retval = 0;
4585 int issued_io;
4586 int skip_range;
4587 uint32_t max_io_size;
4588
4589
4590 if ( !UBCINFOEXISTS(vp))
4591 return(EINVAL);
4592
4593 if (resid < 0)
4594 return(EINVAL);
4595
4596 max_io_size = cluster_max_io_size(vp->v_mount, CL_READ);
4597
4598 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 60)) | DBG_FUNC_START,
4599 (int)f_offset, resid, (int)filesize, 0, 0);
4600
4601 while (resid && f_offset < filesize && retval == 0) {
4602 /*
4603 * compute the size of the upl needed to encompass
4604 * the requested read... limit each call to cluster_io
4605 * to the maximum UPL size... cluster_io will clip if
4606 * this exceeds the maximum io_size for the device,
4607 * make sure to account for
4608 * a starting offset that's not page aligned
4609 */
4610 start_offset = (int)(f_offset & PAGE_MASK_64);
4611 upl_f_offset = f_offset - (off_t)start_offset;
4612 max_size = filesize - f_offset;
4613
4614 if (resid < max_size)
4615 io_size = resid;
4616 else
4617 io_size = max_size;
4618
4619 upl_size = (start_offset + io_size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
4620 if ((uint32_t)upl_size > max_io_size)
4621 upl_size = max_io_size;
4622
4623 skip_range = 0;
4624 /*
4625 * return the number of contiguously present pages in the cache
4626 * starting at upl_f_offset within the file
4627 */
4628 ubc_range_op(vp, upl_f_offset, upl_f_offset + upl_size, UPL_ROP_PRESENT, &skip_range);
4629
4630 if (skip_range) {
4631 /*
4632 * skip over pages already present in the cache
4633 */
4634 io_size = skip_range - start_offset;
4635
4636 f_offset += io_size;
4637 resid -= io_size;
4638
4639 if (skip_range == upl_size)
4640 continue;
4641 /*
4642 * have to issue some real I/O
4643 * at this point, we know it's starting on a page boundary
4644 * because we've skipped over at least the first page in the request
4645 */
4646 start_offset = 0;
4647 upl_f_offset += skip_range;
4648 upl_size -= skip_range;
4649 }
4650 pages_in_upl = upl_size / PAGE_SIZE;
4651
4652 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 61)) | DBG_FUNC_START,
4653 upl, (int)upl_f_offset, upl_size, start_offset, 0);
4654
4655 kret = ubc_create_upl(vp,
4656 upl_f_offset,
4657 upl_size,
4658 &upl,
4659 &pl,
4660 UPL_RET_ONLY_ABSENT | UPL_SET_LITE);
4661 if (kret != KERN_SUCCESS)
4662 return(retval);
4663 issued_io = 0;
4664
4665 /*
4666 * before we start marching forward, we must make sure we end on
4667 * a present page, otherwise we will be working with a freed
4668 * upl
4669 */
4670 for (last_pg = pages_in_upl - 1; last_pg >= 0; last_pg--) {
4671 if (upl_page_present(pl, last_pg))
4672 break;
4673 }
4674 pages_in_upl = last_pg + 1;
4675
4676
4677 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 61)) | DBG_FUNC_END,
4678 upl, (int)upl_f_offset, upl_size, start_offset, 0);
4679
4680
4681 for (last_pg = 0; last_pg < pages_in_upl; ) {
4682 /*
4683 * scan from the beginning of the upl looking for the first
4684 * page that is present.... this will become the first page in
4685 * the request we're going to make to 'cluster_io'... if all
4686 * of the pages are absent, we won't call through to 'cluster_io'
4687 */
4688 for (start_pg = last_pg; start_pg < pages_in_upl; start_pg++) {
4689 if (upl_page_present(pl, start_pg))
4690 break;
4691 }
4692
4693 /*
4694 * scan from the starting present page looking for an absent
4695 * page before the end of the upl is reached, if we
4696 * find one, then it will terminate the range of pages being
4697 * presented to 'cluster_io'
4698 */
4699 for (last_pg = start_pg; last_pg < pages_in_upl; last_pg++) {
4700 if (!upl_page_present(pl, last_pg))
4701 break;
4702 }
4703
4704 if (last_pg > start_pg) {
4705 /*
4706 * we found a range of pages that must be filled
4707 * if the last page in this range is the last page of the file
4708 * we may have to clip the size of it to keep from reading past
4709 * the end of the last physical block associated with the file
4710 */
4711 upl_offset = start_pg * PAGE_SIZE;
4712 io_size = (last_pg - start_pg) * PAGE_SIZE;
4713
4714 if ((off_t)(upl_f_offset + upl_offset + io_size) > filesize)
4715 io_size = filesize - (upl_f_offset + upl_offset);
4716
4717 /*
4718 * issue an asynchronous read to cluster_io
4719 */
4720 retval = cluster_io(vp, upl, upl_offset, upl_f_offset + upl_offset, io_size,
4721 CL_ASYNC | CL_READ | CL_COMMIT | CL_AGE | bflag, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
4722
4723 issued_io = 1;
4724 }
4725 }
4726 if (issued_io == 0)
4727 ubc_upl_abort(upl, 0);
4728
4729 io_size = upl_size - start_offset;
4730
4731 if (io_size > resid)
4732 io_size = resid;
4733 f_offset += io_size;
4734 resid -= io_size;
4735 }
4736
4737 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 60)) | DBG_FUNC_END,
4738 (int)f_offset, resid, retval, 0, 0);
4739
4740 return(retval);
4741 }
4742
4743
4744 int
4745 cluster_push(vnode_t vp, int flags)
4746 {
4747 return cluster_push_ext(vp, flags, NULL, NULL);
4748 }
4749
4750
4751 int
4752 cluster_push_ext(vnode_t vp, int flags, int (*callback)(buf_t, void *), void *callback_arg)
4753 {
4754 int retval;
4755 int my_sparse_wait = 0;
4756 struct cl_writebehind *wbp;
4757
4758 if ( !UBCINFOEXISTS(vp)) {
4759 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 53)) | DBG_FUNC_NONE, vp, flags, 0, -1, 0);
4760 return (0);
4761 }
4762 /* return if deferred write is set */
4763 if (((unsigned int)vfs_flags(vp->v_mount) & MNT_DEFWRITE) && (flags & IO_DEFWRITE)) {
4764 return (0);
4765 }
4766 if ((wbp = cluster_get_wbp(vp, CLW_RETURNLOCKED)) == NULL) {
4767 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 53)) | DBG_FUNC_NONE, vp, flags, 0, -2, 0);
4768 return (0);
4769 }
4770 if (wbp->cl_number == 0 && wbp->cl_scmap == NULL) {
4771 lck_mtx_unlock(&wbp->cl_lockw);
4772
4773 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 53)) | DBG_FUNC_NONE, vp, flags, 0, -3, 0);
4774 return(0);
4775 }
4776 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 53)) | DBG_FUNC_START,
4777 wbp->cl_scmap, wbp->cl_number, flags, 0, 0);
4778
4779 /*
4780 * if we have an fsync in progress, we don't want to allow any additional
4781 * sync/fsync/close(s) to occur until it finishes.
4782 * note that its possible for writes to continue to occur to this file
4783 * while we're waiting and also once the fsync starts to clean if we're
4784 * in the sparse map case
4785 */
4786 while (wbp->cl_sparse_wait) {
4787 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 97)) | DBG_FUNC_START, vp, 0, 0, 0, 0);
4788
4789 msleep((caddr_t)&wbp->cl_sparse_wait, &wbp->cl_lockw, PRIBIO + 1, "cluster_push_ext", NULL);
4790
4791 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 97)) | DBG_FUNC_END, vp, 0, 0, 0, 0);
4792 }
4793 if (flags & IO_SYNC) {
4794 my_sparse_wait = 1;
4795 wbp->cl_sparse_wait = 1;
4796
4797 /*
4798 * this is an fsync (or equivalent)... we must wait for any existing async
4799 * cleaning operations to complete before we evaulate the current state
4800 * and finish cleaning... this insures that all writes issued before this
4801 * fsync actually get cleaned to the disk before this fsync returns
4802 */
4803 while (wbp->cl_sparse_pushes) {
4804 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 98)) | DBG_FUNC_START, vp, 0, 0, 0, 0);
4805
4806 msleep((caddr_t)&wbp->cl_sparse_pushes, &wbp->cl_lockw, PRIBIO + 1, "cluster_push_ext", NULL);
4807
4808 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 98)) | DBG_FUNC_END, vp, 0, 0, 0, 0);
4809 }
4810 }
4811 if (wbp->cl_scmap) {
4812 void *scmap;
4813
4814 if (wbp->cl_sparse_pushes < SPARSE_PUSH_LIMIT) {
4815
4816 scmap = wbp->cl_scmap;
4817 wbp->cl_scmap = NULL;
4818
4819 wbp->cl_sparse_pushes++;
4820
4821 lck_mtx_unlock(&wbp->cl_lockw);
4822
4823 sparse_cluster_push(&scmap, vp, ubc_getsize(vp), PUSH_ALL, flags | IO_PASSIVE, callback, callback_arg);
4824
4825 lck_mtx_lock(&wbp->cl_lockw);
4826
4827 wbp->cl_sparse_pushes--;
4828
4829 if (wbp->cl_sparse_wait && wbp->cl_sparse_pushes == 0)
4830 wakeup((caddr_t)&wbp->cl_sparse_pushes);
4831 } else {
4832 sparse_cluster_push(&(wbp->cl_scmap), vp, ubc_getsize(vp), PUSH_ALL, flags | IO_PASSIVE, callback, callback_arg);
4833 }
4834 retval = 1;
4835 } else {
4836 retval = cluster_try_push(wbp, vp, ubc_getsize(vp), PUSH_ALL, flags | IO_PASSIVE, callback, callback_arg);
4837 }
4838 lck_mtx_unlock(&wbp->cl_lockw);
4839
4840 if (flags & IO_SYNC)
4841 (void)vnode_waitforwrites(vp, 0, 0, 0, "cluster_push");
4842
4843 if (my_sparse_wait) {
4844 /*
4845 * I'm the owner of the serialization token
4846 * clear it and wakeup anyone that is waiting
4847 * for me to finish
4848 */
4849 lck_mtx_lock(&wbp->cl_lockw);
4850
4851 wbp->cl_sparse_wait = 0;
4852 wakeup((caddr_t)&wbp->cl_sparse_wait);
4853
4854 lck_mtx_unlock(&wbp->cl_lockw);
4855 }
4856 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 53)) | DBG_FUNC_END,
4857 wbp->cl_scmap, wbp->cl_number, retval, 0, 0);
4858
4859 return (retval);
4860 }
4861
4862
4863 __private_extern__ void
4864 cluster_release(struct ubc_info *ubc)
4865 {
4866 struct cl_writebehind *wbp;
4867 struct cl_readahead *rap;
4868
4869 if ((wbp = ubc->cl_wbehind)) {
4870
4871 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 81)) | DBG_FUNC_START, ubc, wbp->cl_scmap, 0, 0, 0);
4872
4873 if (wbp->cl_scmap)
4874 vfs_drt_control(&(wbp->cl_scmap), 0);
4875 } else {
4876 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 81)) | DBG_FUNC_START, ubc, 0, 0, 0, 0);
4877 }
4878
4879 rap = ubc->cl_rahead;
4880
4881 if (wbp != NULL) {
4882 lck_mtx_destroy(&wbp->cl_lockw, cl_mtx_grp);
4883 FREE_ZONE((void *)wbp, sizeof *wbp, M_CLWRBEHIND);
4884 }
4885 if ((rap = ubc->cl_rahead)) {
4886 lck_mtx_destroy(&rap->cl_lockr, cl_mtx_grp);
4887 FREE_ZONE((void *)rap, sizeof *rap, M_CLRDAHEAD);
4888 }
4889 ubc->cl_rahead = NULL;
4890 ubc->cl_wbehind = NULL;
4891
4892 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 81)) | DBG_FUNC_END, ubc, rap, wbp, 0, 0);
4893 }
4894
4895
4896 static int
4897 cluster_try_push(struct cl_writebehind *wbp, vnode_t vp, off_t EOF, int push_flag, int io_flags, int (*callback)(buf_t, void *), void *callback_arg)
4898 {
4899 int cl_index;
4900 int cl_index1;
4901 int min_index;
4902 int cl_len;
4903 int cl_pushed = 0;
4904 struct cl_wextent l_clusters[MAX_CLUSTERS];
4905 u_int max_cluster_pgcount;
4906
4907
4908 max_cluster_pgcount = MAX_CLUSTER_SIZE(vp) / PAGE_SIZE;
4909 /*
4910 * the write behind context exists and has
4911 * already been locked...
4912 */
4913 if (wbp->cl_number == 0)
4914 /*
4915 * no clusters to push
4916 * return number of empty slots
4917 */
4918 return (MAX_CLUSTERS);
4919
4920 /*
4921 * make a local 'sorted' copy of the clusters
4922 * and clear wbp->cl_number so that new clusters can
4923 * be developed
4924 */
4925 for (cl_index = 0; cl_index < wbp->cl_number; cl_index++) {
4926 for (min_index = -1, cl_index1 = 0; cl_index1 < wbp->cl_number; cl_index1++) {
4927 if (wbp->cl_clusters[cl_index1].b_addr == wbp->cl_clusters[cl_index1].e_addr)
4928 continue;
4929 if (min_index == -1)
4930 min_index = cl_index1;
4931 else if (wbp->cl_clusters[cl_index1].b_addr < wbp->cl_clusters[min_index].b_addr)
4932 min_index = cl_index1;
4933 }
4934 if (min_index == -1)
4935 break;
4936
4937 l_clusters[cl_index].b_addr = wbp->cl_clusters[min_index].b_addr;
4938 l_clusters[cl_index].e_addr = wbp->cl_clusters[min_index].e_addr;
4939 l_clusters[cl_index].io_flags = wbp->cl_clusters[min_index].io_flags;
4940
4941 wbp->cl_clusters[min_index].b_addr = wbp->cl_clusters[min_index].e_addr;
4942 }
4943 wbp->cl_number = 0;
4944
4945 cl_len = cl_index;
4946
4947 if ( (push_flag & PUSH_DELAY) && cl_len == MAX_CLUSTERS ) {
4948 int i;
4949
4950 /*
4951 * determine if we appear to be writing the file sequentially
4952 * if not, by returning without having pushed any clusters
4953 * we will cause this vnode to be pushed into the sparse cluster mechanism
4954 * used for managing more random I/O patterns
4955 *
4956 * we know that we've got all clusters currently in use and the next write doesn't fit into one of them...
4957 * that's why we're in try_push with PUSH_DELAY...
4958 *
4959 * check to make sure that all the clusters except the last one are 'full'... and that each cluster
4960 * is adjacent to the next (i.e. we're looking for sequential writes) they were sorted above
4961 * so we can just make a simple pass through, up to, but not including the last one...
4962 * note that e_addr is not inclusive, so it will be equal to the b_addr of the next cluster if they
4963 * are sequential
4964 *
4965 * we let the last one be partial as long as it was adjacent to the previous one...
4966 * we need to do this to deal with multi-threaded servers that might write an I/O or 2 out
4967 * of order... if this occurs at the tail of the last cluster, we don't want to fall into the sparse cluster world...
4968 */
4969 for (i = 0; i < MAX_CLUSTERS - 1; i++) {
4970 if ((l_clusters[i].e_addr - l_clusters[i].b_addr) != max_cluster_pgcount)
4971 goto dont_try;
4972 if (l_clusters[i].e_addr != l_clusters[i+1].b_addr)
4973 goto dont_try;
4974 }
4975 }
4976 for (cl_index = 0; cl_index < cl_len; cl_index++) {
4977 int flags;
4978 struct cl_extent cl;
4979
4980 flags = io_flags & (IO_PASSIVE|IO_CLOSE);
4981
4982 /*
4983 * try to push each cluster in turn...
4984 */
4985 if (l_clusters[cl_index].io_flags & CLW_IONOCACHE)
4986 flags |= IO_NOCACHE;
4987
4988 if (l_clusters[cl_index].io_flags & CLW_IOPASSIVE)
4989 flags |= IO_PASSIVE;
4990
4991 if (push_flag & PUSH_SYNC)
4992 flags |= IO_SYNC;
4993
4994 cl.b_addr = l_clusters[cl_index].b_addr;
4995 cl.e_addr = l_clusters[cl_index].e_addr;
4996
4997 cluster_push_now(vp, &cl, EOF, flags, callback, callback_arg);
4998
4999 l_clusters[cl_index].b_addr = 0;
5000 l_clusters[cl_index].e_addr = 0;
5001
5002 cl_pushed++;
5003
5004 if ( !(push_flag & PUSH_ALL) )
5005 break;
5006 }
5007 dont_try:
5008 if (cl_len > cl_pushed) {
5009 /*
5010 * we didn't push all of the clusters, so
5011 * lets try to merge them back in to the vnode
5012 */
5013 if ((MAX_CLUSTERS - wbp->cl_number) < (cl_len - cl_pushed)) {
5014 /*
5015 * we picked up some new clusters while we were trying to
5016 * push the old ones... this can happen because I've dropped
5017 * the vnode lock... the sum of the
5018 * leftovers plus the new cluster count exceeds our ability
5019 * to represent them, so switch to the sparse cluster mechanism
5020 *
5021 * collect the active public clusters...
5022 */
5023 sparse_cluster_switch(wbp, vp, EOF, callback, callback_arg);
5024
5025 for (cl_index = 0, cl_index1 = 0; cl_index < cl_len; cl_index++) {
5026 if (l_clusters[cl_index].b_addr == l_clusters[cl_index].e_addr)
5027 continue;
5028 wbp->cl_clusters[cl_index1].b_addr = l_clusters[cl_index].b_addr;
5029 wbp->cl_clusters[cl_index1].e_addr = l_clusters[cl_index].e_addr;
5030 wbp->cl_clusters[cl_index1].io_flags = l_clusters[cl_index].io_flags;
5031
5032 cl_index1++;
5033 }
5034 /*
5035 * update the cluster count
5036 */
5037 wbp->cl_number = cl_index1;
5038
5039 /*
5040 * and collect the original clusters that were moved into the
5041 * local storage for sorting purposes
5042 */
5043 sparse_cluster_switch(wbp, vp, EOF, callback, callback_arg);
5044
5045 } else {
5046 /*
5047 * we've got room to merge the leftovers back in
5048 * just append them starting at the next 'hole'
5049 * represented by wbp->cl_number
5050 */
5051 for (cl_index = 0, cl_index1 = wbp->cl_number; cl_index < cl_len; cl_index++) {
5052 if (l_clusters[cl_index].b_addr == l_clusters[cl_index].e_addr)
5053 continue;
5054
5055 wbp->cl_clusters[cl_index1].b_addr = l_clusters[cl_index].b_addr;
5056 wbp->cl_clusters[cl_index1].e_addr = l_clusters[cl_index].e_addr;
5057 wbp->cl_clusters[cl_index1].io_flags = l_clusters[cl_index].io_flags;
5058
5059 cl_index1++;
5060 }
5061 /*
5062 * update the cluster count
5063 */
5064 wbp->cl_number = cl_index1;
5065 }
5066 }
5067 return (MAX_CLUSTERS - wbp->cl_number);
5068 }
5069
5070
5071
5072 static int
5073 cluster_push_now(vnode_t vp, struct cl_extent *cl, off_t EOF, int flags, int (*callback)(buf_t, void *), void *callback_arg)
5074 {
5075 upl_page_info_t *pl;
5076 upl_t upl;
5077 vm_offset_t upl_offset;
5078 int upl_size;
5079 off_t upl_f_offset;
5080 int pages_in_upl;
5081 int start_pg;
5082 int last_pg;
5083 int io_size;
5084 int io_flags;
5085 int upl_flags;
5086 int bflag;
5087 int size;
5088 int error = 0;
5089 int retval;
5090 kern_return_t kret;
5091
5092 if (flags & IO_PASSIVE)
5093 bflag = CL_PASSIVE;
5094 else
5095 bflag = 0;
5096
5097 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 51)) | DBG_FUNC_START,
5098 (int)cl->b_addr, (int)cl->e_addr, (int)EOF, flags, 0);
5099
5100 if ((pages_in_upl = (int)(cl->e_addr - cl->b_addr)) == 0) {
5101 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 51)) | DBG_FUNC_END, 1, 0, 0, 0, 0);
5102
5103 return (0);
5104 }
5105 upl_size = pages_in_upl * PAGE_SIZE;
5106 upl_f_offset = (off_t)(cl->b_addr * PAGE_SIZE_64);
5107
5108 if (upl_f_offset + upl_size >= EOF) {
5109
5110 if (upl_f_offset >= EOF) {
5111 /*
5112 * must have truncated the file and missed
5113 * clearing a dangling cluster (i.e. it's completely
5114 * beyond the new EOF
5115 */
5116 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 51)) | DBG_FUNC_END, 1, 1, 0, 0, 0);
5117
5118 return(0);
5119 }
5120 size = EOF - upl_f_offset;
5121
5122 upl_size = (size + (PAGE_SIZE - 1)) & ~PAGE_MASK;
5123 pages_in_upl = upl_size / PAGE_SIZE;
5124 } else
5125 size = upl_size;
5126
5127 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 41)) | DBG_FUNC_START, upl_size, size, 0, 0, 0);
5128
5129 /*
5130 * by asking for UPL_COPYOUT_FROM and UPL_RET_ONLY_DIRTY, we get the following desirable behavior
5131 *
5132 * - only pages that are currently dirty are returned... these are the ones we need to clean
5133 * - the hardware dirty bit is cleared when the page is gathered into the UPL... the software dirty bit is set
5134 * - if we have to abort the I/O for some reason, the software dirty bit is left set since we didn't clean the page
5135 * - when we commit the page, the software dirty bit is cleared... the hardware dirty bit is untouched so that if
5136 * someone dirties this page while the I/O is in progress, we don't lose track of the new state
5137 *
5138 * when the I/O completes, we no longer ask for an explicit clear of the DIRTY state (either soft or hard)
5139 */
5140
5141 if ((vp->v_flag & VNOCACHE_DATA) || (flags & IO_NOCACHE))
5142 upl_flags = UPL_COPYOUT_FROM | UPL_RET_ONLY_DIRTY | UPL_SET_LITE | UPL_WILL_BE_DUMPED;
5143 else
5144 upl_flags = UPL_COPYOUT_FROM | UPL_RET_ONLY_DIRTY | UPL_SET_LITE;
5145
5146 kret = ubc_create_upl(vp,
5147 upl_f_offset,
5148 upl_size,
5149 &upl,
5150 &pl,
5151 upl_flags);
5152 if (kret != KERN_SUCCESS)
5153 panic("cluster_push: failed to get pagelist");
5154
5155 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 41)) | DBG_FUNC_END, upl, upl_f_offset, 0, 0, 0);
5156
5157 /*
5158 * since we only asked for the dirty pages back
5159 * it's possible that we may only get a few or even none, so...
5160 * before we start marching forward, we must make sure we know
5161 * where the last present page is in the UPL, otherwise we could
5162 * end up working with a freed upl due to the FREE_ON_EMPTY semantics
5163 * employed by commit_range and abort_range.
5164 */
5165 for (last_pg = pages_in_upl - 1; last_pg >= 0; last_pg--) {
5166 if (upl_page_present(pl, last_pg))
5167 break;
5168 }
5169 pages_in_upl = last_pg + 1;
5170
5171 if (pages_in_upl == 0) {
5172 ubc_upl_abort(upl, 0);
5173
5174 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 51)) | DBG_FUNC_END, 1, 2, 0, 0, 0);
5175 return(0);
5176 }
5177
5178 for (last_pg = 0; last_pg < pages_in_upl; ) {
5179 /*
5180 * find the next dirty page in the UPL
5181 * this will become the first page in the
5182 * next I/O to generate
5183 */
5184 for (start_pg = last_pg; start_pg < pages_in_upl; start_pg++) {
5185 if (upl_dirty_page(pl, start_pg))
5186 break;
5187 if (upl_page_present(pl, start_pg))
5188 /*
5189 * RET_ONLY_DIRTY will return non-dirty 'precious' pages
5190 * just release these unchanged since we're not going
5191 * to steal them or change their state
5192 */
5193 ubc_upl_abort_range(upl, start_pg * PAGE_SIZE, PAGE_SIZE, UPL_ABORT_FREE_ON_EMPTY);
5194 }
5195 if (start_pg >= pages_in_upl)
5196 /*
5197 * done... no more dirty pages to push
5198 */
5199 break;
5200 if (start_pg > last_pg)
5201 /*
5202 * skipped over some non-dirty pages
5203 */
5204 size -= ((start_pg - last_pg) * PAGE_SIZE);
5205
5206 /*
5207 * find a range of dirty pages to write
5208 */
5209 for (last_pg = start_pg; last_pg < pages_in_upl; last_pg++) {
5210 if (!upl_dirty_page(pl, last_pg))
5211 break;
5212 }
5213 upl_offset = start_pg * PAGE_SIZE;
5214
5215 io_size = min(size, (last_pg - start_pg) * PAGE_SIZE);
5216
5217 io_flags = CL_THROTTLE | CL_COMMIT | CL_AGE | bflag;
5218
5219 if ( !(flags & IO_SYNC))
5220 io_flags |= CL_ASYNC;
5221
5222 if (flags & IO_CLOSE)
5223 io_flags |= CL_CLOSE;
5224
5225 retval = cluster_io(vp, upl, upl_offset, upl_f_offset + upl_offset, io_size,
5226 io_flags, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
5227
5228 if (error == 0 && retval)
5229 error = retval;
5230
5231 size -= io_size;
5232 }
5233 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 51)) | DBG_FUNC_END, 1, 3, 0, 0, 0);
5234
5235 return(error);
5236 }
5237
5238
5239 /*
5240 * sparse_cluster_switch is called with the write behind lock held
5241 */
5242 static void
5243 sparse_cluster_switch(struct cl_writebehind *wbp, vnode_t vp, off_t EOF, int (*callback)(buf_t, void *), void *callback_arg)
5244 {
5245 int cl_index;
5246
5247 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 78)) | DBG_FUNC_START, vp, wbp->cl_scmap, 0, 0, 0);
5248
5249 for (cl_index = 0; cl_index < wbp->cl_number; cl_index++) {
5250 int flags;
5251 struct cl_extent cl;
5252
5253 for (cl.b_addr = wbp->cl_clusters[cl_index].b_addr; cl.b_addr < wbp->cl_clusters[cl_index].e_addr; cl.b_addr++) {
5254
5255 if (ubc_page_op(vp, (off_t)(cl.b_addr * PAGE_SIZE_64), 0, NULL, &flags) == KERN_SUCCESS) {
5256 if (flags & UPL_POP_DIRTY) {
5257 cl.e_addr = cl.b_addr + 1;
5258
5259 sparse_cluster_add(&(wbp->cl_scmap), vp, &cl, EOF, callback, callback_arg);
5260 }
5261 }
5262 }
5263 }
5264 wbp->cl_number = 0;
5265
5266 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 78)) | DBG_FUNC_END, vp, wbp->cl_scmap, 0, 0, 0);
5267 }
5268
5269
5270 /*
5271 * sparse_cluster_push must be called with the write-behind lock held if the scmap is
5272 * still associated with the write-behind context... however, if the scmap has been disassociated
5273 * from the write-behind context (the cluster_push case), the wb lock is not held
5274 */
5275 static void
5276 sparse_cluster_push(void **scmap, vnode_t vp, off_t EOF, int push_flag, int io_flags, int (*callback)(buf_t, void *), void *callback_arg)
5277 {
5278 struct cl_extent cl;
5279 off_t offset;
5280 u_int length;
5281
5282 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 79)) | DBG_FUNC_START, vp, (*scmap), 0, push_flag, 0);
5283
5284 if (push_flag & PUSH_ALL)
5285 vfs_drt_control(scmap, 1);
5286
5287 for (;;) {
5288 if (vfs_drt_get_cluster(scmap, &offset, &length) != KERN_SUCCESS)
5289 break;
5290
5291 cl.b_addr = (daddr64_t)(offset / PAGE_SIZE_64);
5292 cl.e_addr = (daddr64_t)((offset + length) / PAGE_SIZE_64);
5293
5294 cluster_push_now(vp, &cl, EOF, io_flags & (IO_PASSIVE|IO_CLOSE), callback, callback_arg);
5295
5296 if ( !(push_flag & PUSH_ALL) )
5297 break;
5298 }
5299 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 79)) | DBG_FUNC_END, vp, (*scmap), 0, 0, 0);
5300 }
5301
5302
5303 /*
5304 * sparse_cluster_add is called with the write behind lock held
5305 */
5306 static void
5307 sparse_cluster_add(void **scmap, vnode_t vp, struct cl_extent *cl, off_t EOF, int (*callback)(buf_t, void *), void *callback_arg)
5308 {
5309 u_int new_dirty;
5310 u_int length;
5311 off_t offset;
5312
5313 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 80)) | DBG_FUNC_START, (*scmap), 0, cl->b_addr, (int)cl->e_addr, 0);
5314
5315 offset = (off_t)(cl->b_addr * PAGE_SIZE_64);
5316 length = ((u_int)(cl->e_addr - cl->b_addr)) * PAGE_SIZE;
5317
5318 while (vfs_drt_mark_pages(scmap, offset, length, &new_dirty) != KERN_SUCCESS) {
5319 /*
5320 * no room left in the map
5321 * only a partial update was done
5322 * push out some pages and try again
5323 */
5324 sparse_cluster_push(scmap, vp, EOF, 0, 0, callback, callback_arg);
5325
5326 offset += (new_dirty * PAGE_SIZE_64);
5327 length -= (new_dirty * PAGE_SIZE);
5328 }
5329 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 80)) | DBG_FUNC_END, vp, (*scmap), 0, 0, 0);
5330 }
5331
5332
5333 static int
5334 cluster_align_phys_io(vnode_t vp, struct uio *uio, addr64_t usr_paddr, u_int32_t xsize, int flags, int (*callback)(buf_t, void *), void *callback_arg)
5335 {
5336 upl_page_info_t *pl;
5337 upl_t upl;
5338 addr64_t ubc_paddr;
5339 kern_return_t kret;
5340 int error = 0;
5341 int did_read = 0;
5342 int abort_flags;
5343 int upl_flags;
5344 int bflag;
5345
5346 if (flags & IO_PASSIVE)
5347 bflag = CL_PASSIVE;
5348 else
5349 bflag = 0;
5350
5351 upl_flags = UPL_SET_LITE;
5352
5353 if ( !(flags & CL_READ) ) {
5354 /*
5355 * "write" operation: let the UPL subsystem know
5356 * that we intend to modify the buffer cache pages
5357 * we're gathering.
5358 */
5359 upl_flags |= UPL_WILL_MODIFY;
5360 } else {
5361 /*
5362 * indicate that there is no need to pull the
5363 * mapping for this page... we're only going
5364 * to read from it, not modify it.
5365 */
5366 upl_flags |= UPL_FILE_IO;
5367 }
5368 kret = ubc_create_upl(vp,
5369 uio->uio_offset & ~PAGE_MASK_64,
5370 PAGE_SIZE,
5371 &upl,
5372 &pl,
5373 upl_flags);
5374
5375 if (kret != KERN_SUCCESS)
5376 return(EINVAL);
5377
5378 if (!upl_valid_page(pl, 0)) {
5379 /*
5380 * issue a synchronous read to cluster_io
5381 */
5382 error = cluster_io(vp, upl, 0, uio->uio_offset & ~PAGE_MASK_64, PAGE_SIZE,
5383 CL_READ | bflag, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
5384 if (error) {
5385 ubc_upl_abort_range(upl, 0, PAGE_SIZE, UPL_ABORT_DUMP_PAGES | UPL_ABORT_FREE_ON_EMPTY);
5386
5387 return(error);
5388 }
5389 did_read = 1;
5390 }
5391 ubc_paddr = ((addr64_t)upl_phys_page(pl, 0) << 12) + (addr64_t)(uio->uio_offset & PAGE_MASK_64);
5392
5393 /*
5394 * NOTE: There is no prototype for the following in BSD. It, and the definitions
5395 * of the defines for cppvPsrc, cppvPsnk, cppvFsnk, and cppvFsrc will be found in
5396 * osfmk/ppc/mappings.h. They are not included here because there appears to be no
5397 * way to do so without exporting them to kexts as well.
5398 */
5399 if (flags & CL_READ)
5400 // copypv(ubc_paddr, usr_paddr, xsize, cppvPsrc | cppvPsnk | cppvFsnk); /* Copy physical to physical and flush the destination */
5401 copypv(ubc_paddr, usr_paddr, xsize, 2 | 1 | 4); /* Copy physical to physical and flush the destination */
5402 else
5403 // copypv(usr_paddr, ubc_paddr, xsize, cppvPsrc | cppvPsnk | cppvFsrc); /* Copy physical to physical and flush the source */
5404 copypv(usr_paddr, ubc_paddr, xsize, 2 | 1 | 8); /* Copy physical to physical and flush the source */
5405
5406 if ( !(flags & CL_READ) || (upl_valid_page(pl, 0) && upl_dirty_page(pl, 0))) {
5407 /*
5408 * issue a synchronous write to cluster_io
5409 */
5410 error = cluster_io(vp, upl, 0, uio->uio_offset & ~PAGE_MASK_64, PAGE_SIZE,
5411 bflag, (buf_t)NULL, (struct clios *)NULL, callback, callback_arg);
5412 }
5413 if (error == 0)
5414 uio_update(uio, (user_size_t)xsize);
5415
5416 if (did_read)
5417 abort_flags = UPL_ABORT_FREE_ON_EMPTY;
5418 else
5419 abort_flags = UPL_ABORT_FREE_ON_EMPTY | UPL_ABORT_DUMP_PAGES;
5420
5421 ubc_upl_abort_range(upl, 0, PAGE_SIZE, abort_flags);
5422
5423 return (error);
5424 }
5425
5426
5427
5428 int
5429 cluster_copy_upl_data(struct uio *uio, upl_t upl, int upl_offset, int *io_resid)
5430 {
5431 int pg_offset;
5432 int pg_index;
5433 int csize;
5434 int segflg;
5435 int retval = 0;
5436 int xsize;
5437 upl_page_info_t *pl;
5438
5439 xsize = *io_resid;
5440
5441 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 34)) | DBG_FUNC_START,
5442 (int)uio->uio_offset, upl_offset, xsize, 0, 0);
5443
5444 segflg = uio->uio_segflg;
5445
5446 switch(segflg) {
5447
5448 case UIO_USERSPACE32:
5449 case UIO_USERISPACE32:
5450 uio->uio_segflg = UIO_PHYS_USERSPACE32;
5451 break;
5452
5453 case UIO_USERSPACE:
5454 case UIO_USERISPACE:
5455 uio->uio_segflg = UIO_PHYS_USERSPACE;
5456 break;
5457
5458 case UIO_USERSPACE64:
5459 case UIO_USERISPACE64:
5460 uio->uio_segflg = UIO_PHYS_USERSPACE64;
5461 break;
5462
5463 case UIO_SYSSPACE:
5464 uio->uio_segflg = UIO_PHYS_SYSSPACE;
5465 break;
5466
5467 }
5468 pl = ubc_upl_pageinfo(upl);
5469
5470 pg_index = upl_offset / PAGE_SIZE;
5471 pg_offset = upl_offset & PAGE_MASK;
5472 csize = min(PAGE_SIZE - pg_offset, xsize);
5473
5474 while (xsize && retval == 0) {
5475 addr64_t paddr;
5476
5477 paddr = ((addr64_t)upl_phys_page(pl, pg_index) << 12) + pg_offset;
5478
5479 retval = uiomove64(paddr, csize, uio);
5480
5481 pg_index += 1;
5482 pg_offset = 0;
5483 xsize -= csize;
5484 csize = min(PAGE_SIZE, xsize);
5485 }
5486 *io_resid = xsize;
5487
5488 uio->uio_segflg = segflg;
5489
5490 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 34)) | DBG_FUNC_END,
5491 (int)uio->uio_offset, xsize, retval, segflg, 0);
5492
5493 return (retval);
5494 }
5495
5496
5497 int
5498 cluster_copy_ubc_data(vnode_t vp, struct uio *uio, int *io_resid, int mark_dirty)
5499 {
5500
5501 return (cluster_copy_ubc_data_internal(vp, uio, io_resid, mark_dirty, 1));
5502 }
5503
5504
5505 static int
5506 cluster_copy_ubc_data_internal(vnode_t vp, struct uio *uio, int *io_resid, int mark_dirty, int take_reference)
5507 {
5508 int segflg;
5509 int io_size;
5510 int xsize;
5511 int start_offset;
5512 int retval = 0;
5513 memory_object_control_t control;
5514
5515 io_size = *io_resid;
5516
5517 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 34)) | DBG_FUNC_START,
5518 (int)uio->uio_offset, io_size, mark_dirty, take_reference, 0);
5519
5520 control = ubc_getobject(vp, UBC_FLAGS_NONE);
5521
5522 if (control == MEMORY_OBJECT_CONTROL_NULL) {
5523 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 34)) | DBG_FUNC_END,
5524 (int)uio->uio_offset, io_size, retval, 3, 0);
5525
5526 return(0);
5527 }
5528 segflg = uio->uio_segflg;
5529
5530 switch(segflg) {
5531
5532 case UIO_USERSPACE32:
5533 case UIO_USERISPACE32:
5534 uio->uio_segflg = UIO_PHYS_USERSPACE32;
5535 break;
5536
5537 case UIO_USERSPACE64:
5538 case UIO_USERISPACE64:
5539 uio->uio_segflg = UIO_PHYS_USERSPACE64;
5540 break;
5541
5542 case UIO_USERSPACE:
5543 case UIO_USERISPACE:
5544 uio->uio_segflg = UIO_PHYS_USERSPACE;
5545 break;
5546
5547 case UIO_SYSSPACE:
5548 uio->uio_segflg = UIO_PHYS_SYSSPACE;
5549 break;
5550 }
5551
5552 if ( (io_size = *io_resid) ) {
5553 start_offset = (int)(uio->uio_offset & PAGE_MASK_64);
5554 xsize = uio_resid(uio);
5555
5556 retval = memory_object_control_uiomove(control, uio->uio_offset - start_offset, uio,
5557 start_offset, io_size, mark_dirty, take_reference);
5558 xsize -= uio_resid(uio);
5559 io_size -= xsize;
5560 }
5561 uio->uio_segflg = segflg;
5562 *io_resid = io_size;
5563
5564 KERNEL_DEBUG((FSDBG_CODE(DBG_FSRW, 34)) | DBG_FUNC_END,
5565 (int)uio->uio_offset, io_size, retval, 0x80000000 | segflg, 0);
5566
5567 return(retval);
5568 }
5569
5570
5571 int
5572 is_file_clean(vnode_t vp, off_t filesize)
5573 {
5574 off_t f_offset;
5575 int flags;
5576 int total_dirty = 0;
5577
5578 for (f_offset = 0; f_offset < filesize; f_offset += PAGE_SIZE_64) {
5579 if (ubc_page_op(vp, f_offset, 0, NULL, &flags) == KERN_SUCCESS) {
5580 if (flags & UPL_POP_DIRTY) {
5581 total_dirty++;
5582 }
5583 }
5584 }
5585 if (total_dirty)
5586 return(EINVAL);
5587
5588 return (0);
5589 }
5590
5591
5592
5593 /*
5594 * Dirty region tracking/clustering mechanism.
5595 *
5596 * This code (vfs_drt_*) provides a mechanism for tracking and clustering
5597 * dirty regions within a larger space (file). It is primarily intended to
5598 * support clustering in large files with many dirty areas.
5599 *
5600 * The implementation assumes that the dirty regions are pages.
5601 *
5602 * To represent dirty pages within the file, we store bit vectors in a
5603 * variable-size circular hash.
5604 */
5605
5606 /*
5607 * Bitvector size. This determines the number of pages we group in a
5608 * single hashtable entry. Each hashtable entry is aligned to this
5609 * size within the file.
5610 */
5611 #define DRT_BITVECTOR_PAGES 256
5612
5613 /*
5614 * File offset handling.
5615 *
5616 * DRT_ADDRESS_MASK is dependent on DRT_BITVECTOR_PAGES;
5617 * the correct formula is (~(DRT_BITVECTOR_PAGES * PAGE_SIZE) - 1)
5618 */
5619 #define DRT_ADDRESS_MASK (~((1 << 20) - 1))
5620 #define DRT_ALIGN_ADDRESS(addr) ((addr) & DRT_ADDRESS_MASK)
5621
5622 /*
5623 * Hashtable address field handling.
5624 *
5625 * The low-order bits of the hashtable address are used to conserve
5626 * space.
5627 *
5628 * DRT_HASH_COUNT_MASK must be large enough to store the range
5629 * 0-DRT_BITVECTOR_PAGES inclusive, as well as have one value
5630 * to indicate that the bucket is actually unoccupied.
5631 */
5632 #define DRT_HASH_GET_ADDRESS(scm, i) ((scm)->scm_hashtable[(i)].dhe_control & DRT_ADDRESS_MASK)
5633 #define DRT_HASH_SET_ADDRESS(scm, i, a) \
5634 do { \
5635 (scm)->scm_hashtable[(i)].dhe_control = \
5636 ((scm)->scm_hashtable[(i)].dhe_control & ~DRT_ADDRESS_MASK) | DRT_ALIGN_ADDRESS(a); \
5637 } while (0)
5638 #define DRT_HASH_COUNT_MASK 0x1ff
5639 #define DRT_HASH_GET_COUNT(scm, i) ((scm)->scm_hashtable[(i)].dhe_control & DRT_HASH_COUNT_MASK)
5640 #define DRT_HASH_SET_COUNT(scm, i, c) \
5641 do { \
5642 (scm)->scm_hashtable[(i)].dhe_control = \
5643 ((scm)->scm_hashtable[(i)].dhe_control & ~DRT_HASH_COUNT_MASK) | ((c) & DRT_HASH_COUNT_MASK); \
5644 } while (0)
5645 #define DRT_HASH_CLEAR(scm, i) \
5646 do { \
5647 (scm)->scm_hashtable[(i)].dhe_control = 0; \
5648 } while (0)
5649 #define DRT_HASH_VACATE(scm, i) DRT_HASH_SET_COUNT((scm), (i), DRT_HASH_COUNT_MASK)
5650 #define DRT_HASH_VACANT(scm, i) (DRT_HASH_GET_COUNT((scm), (i)) == DRT_HASH_COUNT_MASK)
5651 #define DRT_HASH_COPY(oscm, oi, scm, i) \
5652 do { \
5653 (scm)->scm_hashtable[(i)].dhe_control = (oscm)->scm_hashtable[(oi)].dhe_control; \
5654 DRT_BITVECTOR_COPY(oscm, oi, scm, i); \
5655 } while(0);
5656
5657
5658 /*
5659 * Hash table moduli.
5660 *
5661 * Since the hashtable entry's size is dependent on the size of
5662 * the bitvector, and since the hashtable size is constrained to
5663 * both being prime and fitting within the desired allocation
5664 * size, these values need to be manually determined.
5665 *
5666 * For DRT_BITVECTOR_SIZE = 256, the entry size is 40 bytes.
5667 *
5668 * The small hashtable allocation is 1024 bytes, so the modulus is 23.
5669 * The large hashtable allocation is 16384 bytes, so the modulus is 401.
5670 */
5671 #define DRT_HASH_SMALL_MODULUS 23
5672 #define DRT_HASH_LARGE_MODULUS 401
5673
5674 /*
5675 * Physical memory required before the large hash modulus is permitted.
5676 *
5677 * On small memory systems, the large hash modulus can lead to phsyical
5678 * memory starvation, so we avoid using it there.
5679 */
5680 #define DRT_HASH_LARGE_MEMORY_REQUIRED (1024LL * 1024LL * 1024LL) /* 1GiB */
5681
5682 #define DRT_SMALL_ALLOCATION 1024 /* 104 bytes spare */
5683 #define DRT_LARGE_ALLOCATION 16384 /* 344 bytes spare */
5684
5685 /* *** nothing below here has secret dependencies on DRT_BITVECTOR_PAGES *** */
5686
5687 /*
5688 * Hashtable bitvector handling.
5689 *
5690 * Bitvector fields are 32 bits long.
5691 */
5692
5693 #define DRT_HASH_SET_BIT(scm, i, bit) \
5694 (scm)->scm_hashtable[(i)].dhe_bitvector[(bit) / 32] |= (1 << ((bit) % 32))
5695
5696 #define DRT_HASH_CLEAR_BIT(scm, i, bit) \
5697 (scm)->scm_hashtable[(i)].dhe_bitvector[(bit) / 32] &= ~(1 << ((bit) % 32))
5698
5699 #define DRT_HASH_TEST_BIT(scm, i, bit) \
5700 ((scm)->scm_hashtable[(i)].dhe_bitvector[(bit) / 32] & (1 << ((bit) % 32)))
5701
5702 #define DRT_BITVECTOR_CLEAR(scm, i) \
5703 bzero(&(scm)->scm_hashtable[(i)].dhe_bitvector[0], (DRT_BITVECTOR_PAGES / 32) * sizeof(u_int32_t))
5704
5705 #define DRT_BITVECTOR_COPY(oscm, oi, scm, i) \
5706 bcopy(&(oscm)->scm_hashtable[(oi)].dhe_bitvector[0], \
5707 &(scm)->scm_hashtable[(i)].dhe_bitvector[0], \
5708 (DRT_BITVECTOR_PAGES / 32) * sizeof(u_int32_t))
5709
5710
5711
5712 /*
5713 * Hashtable entry.
5714 */
5715 struct vfs_drt_hashentry {
5716 u_int64_t dhe_control;
5717 u_int32_t dhe_bitvector[DRT_BITVECTOR_PAGES / 32];
5718 };
5719
5720 /*
5721 * Dirty Region Tracking structure.
5722 *
5723 * The hashtable is allocated entirely inside the DRT structure.
5724 *
5725 * The hash is a simple circular prime modulus arrangement, the structure
5726 * is resized from small to large if it overflows.
5727 */
5728
5729 struct vfs_drt_clustermap {
5730 u_int32_t scm_magic; /* sanity/detection */
5731 #define DRT_SCM_MAGIC 0x12020003
5732 u_int32_t scm_modulus; /* current ring size */
5733 u_int32_t scm_buckets; /* number of occupied buckets */
5734 u_int32_t scm_lastclean; /* last entry we cleaned */
5735 u_int32_t scm_iskips; /* number of slot skips */
5736
5737 struct vfs_drt_hashentry scm_hashtable[0];
5738 };
5739
5740
5741 #define DRT_HASH(scm, addr) ((addr) % (scm)->scm_modulus)
5742 #define DRT_HASH_NEXT(scm, addr) (((addr) + 1) % (scm)->scm_modulus)
5743
5744 /*
5745 * Debugging codes and arguments.
5746 */
5747 #define DRT_DEBUG_EMPTYFREE (FSDBG_CODE(DBG_FSRW, 82)) /* nil */
5748 #define DRT_DEBUG_RETCLUSTER (FSDBG_CODE(DBG_FSRW, 83)) /* offset, length */
5749 #define DRT_DEBUG_ALLOC (FSDBG_CODE(DBG_FSRW, 84)) /* copycount */
5750 #define DRT_DEBUG_INSERT (FSDBG_CODE(DBG_FSRW, 85)) /* offset, iskip */
5751 #define DRT_DEBUG_MARK (FSDBG_CODE(DBG_FSRW, 86)) /* offset, length,
5752 * dirty */
5753 /* 0, setcount */
5754 /* 1 (clean, no map) */
5755 /* 2 (map alloc fail) */
5756 /* 3, resid (partial) */
5757 #define DRT_DEBUG_6 (FSDBG_CODE(DBG_FSRW, 87))
5758 #define DRT_DEBUG_SCMDATA (FSDBG_CODE(DBG_FSRW, 88)) /* modulus, buckets,
5759 * lastclean, iskips */
5760
5761
5762 static kern_return_t vfs_drt_alloc_map(struct vfs_drt_clustermap **cmapp);
5763 static kern_return_t vfs_drt_free_map(struct vfs_drt_clustermap *cmap);
5764 static kern_return_t vfs_drt_search_index(struct vfs_drt_clustermap *cmap,
5765 u_int64_t offset, int *indexp);
5766 static kern_return_t vfs_drt_get_index(struct vfs_drt_clustermap **cmapp,
5767 u_int64_t offset,
5768 int *indexp,
5769 int recursed);
5770 static kern_return_t vfs_drt_do_mark_pages(
5771 void **cmapp,
5772 u_int64_t offset,
5773 u_int length,
5774 u_int *setcountp,
5775 int dirty);
5776 static void vfs_drt_trace(
5777 struct vfs_drt_clustermap *cmap,
5778 int code,
5779 int arg1,
5780 int arg2,
5781 int arg3,
5782 int arg4);
5783
5784
5785 /*
5786 * Allocate and initialise a sparse cluster map.
5787 *
5788 * Will allocate a new map, resize or compact an existing map.
5789 *
5790 * XXX we should probably have at least one intermediate map size,
5791 * as the 1:16 ratio seems a bit drastic.
5792 */
5793 static kern_return_t
5794 vfs_drt_alloc_map(struct vfs_drt_clustermap **cmapp)
5795 {
5796 struct vfs_drt_clustermap *cmap, *ocmap;
5797 kern_return_t kret;
5798 u_int64_t offset;
5799 u_int32_t i;
5800 int nsize, active_buckets, index, copycount;
5801
5802 ocmap = NULL;
5803 if (cmapp != NULL)
5804 ocmap = *cmapp;
5805
5806 /*
5807 * Decide on the size of the new map.
5808 */
5809 if (ocmap == NULL) {
5810 nsize = DRT_HASH_SMALL_MODULUS;
5811 } else {
5812 /* count the number of active buckets in the old map */
5813 active_buckets = 0;
5814 for (i = 0; i < ocmap->scm_modulus; i++) {
5815 if (!DRT_HASH_VACANT(ocmap, i) &&
5816 (DRT_HASH_GET_COUNT(ocmap, i) != 0))
5817 active_buckets++;
5818 }
5819 /*
5820 * If we're currently using the small allocation, check to
5821 * see whether we should grow to the large one.
5822 */
5823 if (ocmap->scm_modulus == DRT_HASH_SMALL_MODULUS) {
5824 /*
5825 * If the ring is nearly full and we are allowed to
5826 * use the large modulus, upgrade.
5827 */
5828 if ((active_buckets > (DRT_HASH_SMALL_MODULUS - 5)) &&
5829 (max_mem >= DRT_HASH_LARGE_MEMORY_REQUIRED)) {
5830 nsize = DRT_HASH_LARGE_MODULUS;
5831 } else {
5832 nsize = DRT_HASH_SMALL_MODULUS;
5833 }
5834 } else {
5835 /* already using the large modulus */
5836 nsize = DRT_HASH_LARGE_MODULUS;
5837 /*
5838 * If the ring is completely full, there's
5839 * nothing useful for us to do. Behave as
5840 * though we had compacted into the new
5841 * array and return.
5842 */
5843 if (active_buckets >= DRT_HASH_LARGE_MODULUS)
5844 return(KERN_SUCCESS);
5845 }
5846 }
5847
5848 /*
5849 * Allocate and initialise the new map.
5850 */
5851
5852 kret = kmem_alloc(kernel_map, (vm_offset_t *)&cmap,
5853 (nsize == DRT_HASH_SMALL_MODULUS) ? DRT_SMALL_ALLOCATION : DRT_LARGE_ALLOCATION);
5854 if (kret != KERN_SUCCESS)
5855 return(kret);
5856 cmap->scm_magic = DRT_SCM_MAGIC;
5857 cmap->scm_modulus = nsize;
5858 cmap->scm_buckets = 0;
5859 cmap->scm_lastclean = 0;
5860 cmap->scm_iskips = 0;
5861 for (i = 0; i < cmap->scm_modulus; i++) {
5862 DRT_HASH_CLEAR(cmap, i);
5863 DRT_HASH_VACATE(cmap, i);
5864 DRT_BITVECTOR_CLEAR(cmap, i);
5865 }
5866
5867 /*
5868 * If there's an old map, re-hash entries from it into the new map.
5869 */
5870 copycount = 0;
5871 if (ocmap != NULL) {
5872 for (i = 0; i < ocmap->scm_modulus; i++) {
5873 /* skip empty buckets */
5874 if (DRT_HASH_VACANT(ocmap, i) ||
5875 (DRT_HASH_GET_COUNT(ocmap, i) == 0))
5876 continue;
5877 /* get new index */
5878 offset = DRT_HASH_GET_ADDRESS(ocmap, i);
5879 kret = vfs_drt_get_index(&cmap, offset, &index, 1);
5880 if (kret != KERN_SUCCESS) {
5881 /* XXX need to bail out gracefully here */
5882 panic("vfs_drt: new cluster map mysteriously too small");
5883 index = 0;
5884 }
5885 /* copy */
5886 DRT_HASH_COPY(ocmap, i, cmap, index);
5887 copycount++;
5888 }
5889 }
5890
5891 /* log what we've done */
5892 vfs_drt_trace(cmap, DRT_DEBUG_ALLOC, copycount, 0, 0, 0);
5893
5894 /*
5895 * It's important to ensure that *cmapp always points to
5896 * a valid map, so we must overwrite it before freeing
5897 * the old map.
5898 */
5899 *cmapp = cmap;
5900 if (ocmap != NULL) {
5901 /* emit stats into trace buffer */
5902 vfs_drt_trace(ocmap, DRT_DEBUG_SCMDATA,
5903 ocmap->scm_modulus,
5904 ocmap->scm_buckets,
5905 ocmap->scm_lastclean,
5906 ocmap->scm_iskips);
5907
5908 vfs_drt_free_map(ocmap);
5909 }
5910 return(KERN_SUCCESS);
5911 }
5912
5913
5914 /*
5915 * Free a sparse cluster map.
5916 */
5917 static kern_return_t
5918 vfs_drt_free_map(struct vfs_drt_clustermap *cmap)
5919 {
5920 kmem_free(kernel_map, (vm_offset_t)cmap,
5921 (cmap->scm_modulus == DRT_HASH_SMALL_MODULUS) ? DRT_SMALL_ALLOCATION : DRT_LARGE_ALLOCATION);
5922 return(KERN_SUCCESS);
5923 }
5924
5925
5926 /*
5927 * Find the hashtable slot currently occupied by an entry for the supplied offset.
5928 */
5929 static kern_return_t
5930 vfs_drt_search_index(struct vfs_drt_clustermap *cmap, u_int64_t offset, int *indexp)
5931 {
5932 int index;
5933 u_int32_t i;
5934
5935 offset = DRT_ALIGN_ADDRESS(offset);
5936 index = DRT_HASH(cmap, offset);
5937
5938 /* traverse the hashtable */
5939 for (i = 0; i < cmap->scm_modulus; i++) {
5940
5941 /*
5942 * If the slot is vacant, we can stop.
5943 */
5944 if (DRT_HASH_VACANT(cmap, index))
5945 break;
5946
5947 /*
5948 * If the address matches our offset, we have success.
5949 */
5950 if (DRT_HASH_GET_ADDRESS(cmap, index) == offset) {
5951 *indexp = index;
5952 return(KERN_SUCCESS);
5953 }
5954
5955 /*
5956 * Move to the next slot, try again.
5957 */
5958 index = DRT_HASH_NEXT(cmap, index);
5959 }
5960 /*
5961 * It's not there.
5962 */
5963 return(KERN_FAILURE);
5964 }
5965
5966 /*
5967 * Find the hashtable slot for the supplied offset. If we haven't allocated
5968 * one yet, allocate one and populate the address field. Note that it will
5969 * not have a nonzero page count and thus will still technically be free, so
5970 * in the case where we are called to clean pages, the slot will remain free.
5971 */
5972 static kern_return_t
5973 vfs_drt_get_index(struct vfs_drt_clustermap **cmapp, u_int64_t offset, int *indexp, int recursed)
5974 {
5975 struct vfs_drt_clustermap *cmap;
5976 kern_return_t kret;
5977 u_int32_t index;
5978 u_int32_t i;
5979
5980 cmap = *cmapp;
5981
5982 /* look for an existing entry */
5983 kret = vfs_drt_search_index(cmap, offset, indexp);
5984 if (kret == KERN_SUCCESS)
5985 return(kret);
5986
5987 /* need to allocate an entry */
5988 offset = DRT_ALIGN_ADDRESS(offset);
5989 index = DRT_HASH(cmap, offset);
5990
5991 /* scan from the index forwards looking for a vacant slot */
5992 for (i = 0; i < cmap->scm_modulus; i++) {
5993 /* slot vacant? */
5994 if (DRT_HASH_VACANT(cmap, index) || DRT_HASH_GET_COUNT(cmap,index) == 0) {
5995 cmap->scm_buckets++;
5996 if (index < cmap->scm_lastclean)
5997 cmap->scm_lastclean = index;
5998 DRT_HASH_SET_ADDRESS(cmap, index, offset);
5999 DRT_HASH_SET_COUNT(cmap, index, 0);
6000 DRT_BITVECTOR_CLEAR(cmap, index);
6001 *indexp = index;
6002 vfs_drt_trace(cmap, DRT_DEBUG_INSERT, (int)offset, i, 0, 0);
6003 return(KERN_SUCCESS);
6004 }
6005 cmap->scm_iskips += i;
6006 index = DRT_HASH_NEXT(cmap, index);
6007 }
6008
6009 /*
6010 * We haven't found a vacant slot, so the map is full. If we're not
6011 * already recursed, try reallocating/compacting it.
6012 */
6013 if (recursed)
6014 return(KERN_FAILURE);
6015 kret = vfs_drt_alloc_map(cmapp);
6016 if (kret == KERN_SUCCESS) {
6017 /* now try to insert again */
6018 kret = vfs_drt_get_index(cmapp, offset, indexp, 1);
6019 }
6020 return(kret);
6021 }
6022
6023 /*
6024 * Implementation of set dirty/clean.
6025 *
6026 * In the 'clean' case, not finding a map is OK.
6027 */
6028 static kern_return_t
6029 vfs_drt_do_mark_pages(
6030 void **private,
6031 u_int64_t offset,
6032 u_int length,
6033 u_int *setcountp,
6034 int dirty)
6035 {
6036 struct vfs_drt_clustermap *cmap, **cmapp;
6037 kern_return_t kret;
6038 int i, index, pgoff, pgcount, setcount, ecount;
6039
6040 cmapp = (struct vfs_drt_clustermap **)private;
6041 cmap = *cmapp;
6042
6043 vfs_drt_trace(cmap, DRT_DEBUG_MARK | DBG_FUNC_START, (int)offset, (int)length, dirty, 0);
6044
6045 if (setcountp != NULL)
6046 *setcountp = 0;
6047
6048 /* allocate a cluster map if we don't already have one */
6049 if (cmap == NULL) {
6050 /* no cluster map, nothing to clean */
6051 if (!dirty) {
6052 vfs_drt_trace(cmap, DRT_DEBUG_MARK | DBG_FUNC_END, 1, 0, 0, 0);
6053 return(KERN_SUCCESS);
6054 }
6055 kret = vfs_drt_alloc_map(cmapp);
6056 if (kret != KERN_SUCCESS) {
6057 vfs_drt_trace(cmap, DRT_DEBUG_MARK | DBG_FUNC_END, 2, 0, 0, 0);
6058 return(kret);
6059 }
6060 }
6061 setcount = 0;
6062
6063 /*
6064 * Iterate over the length of the region.
6065 */
6066 while (length > 0) {
6067 /*
6068 * Get the hashtable index for this offset.
6069 *
6070 * XXX this will add blank entries if we are clearing a range
6071 * that hasn't been dirtied.
6072 */
6073 kret = vfs_drt_get_index(cmapp, offset, &index, 0);
6074 cmap = *cmapp; /* may have changed! */
6075 /* this may be a partial-success return */
6076 if (kret != KERN_SUCCESS) {
6077 if (setcountp != NULL)
6078 *setcountp = setcount;
6079 vfs_drt_trace(cmap, DRT_DEBUG_MARK | DBG_FUNC_END, 3, (int)length, 0, 0);
6080
6081 return(kret);
6082 }
6083
6084 /*
6085 * Work out how many pages we're modifying in this
6086 * hashtable entry.
6087 */
6088 pgoff = (offset - DRT_ALIGN_ADDRESS(offset)) / PAGE_SIZE;
6089 pgcount = min((length / PAGE_SIZE), (DRT_BITVECTOR_PAGES - pgoff));
6090
6091 /*
6092 * Iterate over pages, dirty/clearing as we go.
6093 */
6094 ecount = DRT_HASH_GET_COUNT(cmap, index);
6095 for (i = 0; i < pgcount; i++) {
6096 if (dirty) {
6097 if (!DRT_HASH_TEST_BIT(cmap, index, pgoff + i)) {
6098 DRT_HASH_SET_BIT(cmap, index, pgoff + i);
6099 ecount++;
6100 setcount++;
6101 }
6102 } else {
6103 if (DRT_HASH_TEST_BIT(cmap, index, pgoff + i)) {
6104 DRT_HASH_CLEAR_BIT(cmap, index, pgoff + i);
6105 ecount--;
6106 setcount++;
6107 }
6108 }
6109 }
6110 DRT_HASH_SET_COUNT(cmap, index, ecount);
6111
6112 offset += pgcount * PAGE_SIZE;
6113 length -= pgcount * PAGE_SIZE;
6114 }
6115 if (setcountp != NULL)
6116 *setcountp = setcount;
6117
6118 vfs_drt_trace(cmap, DRT_DEBUG_MARK | DBG_FUNC_END, 0, setcount, 0, 0);
6119
6120 return(KERN_SUCCESS);
6121 }
6122
6123 /*
6124 * Mark a set of pages as dirty/clean.
6125 *
6126 * This is a public interface.
6127 *
6128 * cmapp
6129 * Pointer to storage suitable for holding a pointer. Note that
6130 * this must either be NULL or a value set by this function.
6131 *
6132 * size
6133 * Current file size in bytes.
6134 *
6135 * offset
6136 * Offset of the first page to be marked as dirty, in bytes. Must be
6137 * page-aligned.
6138 *
6139 * length
6140 * Length of dirty region, in bytes. Must be a multiple of PAGE_SIZE.
6141 *
6142 * setcountp
6143 * Number of pages newly marked dirty by this call (optional).
6144 *
6145 * Returns KERN_SUCCESS if all the pages were successfully marked.
6146 */
6147 static kern_return_t
6148 vfs_drt_mark_pages(void **cmapp, off_t offset, u_int length, u_int *setcountp)
6149 {
6150 /* XXX size unused, drop from interface */
6151 return(vfs_drt_do_mark_pages(cmapp, offset, length, setcountp, 1));
6152 }
6153
6154 #if 0
6155 static kern_return_t
6156 vfs_drt_unmark_pages(void **cmapp, off_t offset, u_int length)
6157 {
6158 return(vfs_drt_do_mark_pages(cmapp, offset, length, NULL, 0));
6159 }
6160 #endif
6161
6162 /*
6163 * Get a cluster of dirty pages.
6164 *
6165 * This is a public interface.
6166 *
6167 * cmapp
6168 * Pointer to storage managed by drt_mark_pages. Note that this must
6169 * be NULL or a value set by drt_mark_pages.
6170 *
6171 * offsetp
6172 * Returns the byte offset into the file of the first page in the cluster.
6173 *
6174 * lengthp
6175 * Returns the length in bytes of the cluster of dirty pages.
6176 *
6177 * Returns success if a cluster was found. If KERN_FAILURE is returned, there
6178 * are no dirty pages meeting the minmum size criteria. Private storage will
6179 * be released if there are no more dirty pages left in the map
6180 *
6181 */
6182 static kern_return_t
6183 vfs_drt_get_cluster(void **cmapp, off_t *offsetp, u_int *lengthp)
6184 {
6185 struct vfs_drt_clustermap *cmap;
6186 u_int64_t offset;
6187 u_int length;
6188 u_int32_t j;
6189 int index, i, fs, ls;
6190
6191 /* sanity */
6192 if ((cmapp == NULL) || (*cmapp == NULL))
6193 return(KERN_FAILURE);
6194 cmap = *cmapp;
6195
6196 /* walk the hashtable */
6197 for (offset = 0, j = 0; j < cmap->scm_modulus; offset += (DRT_BITVECTOR_PAGES * PAGE_SIZE), j++) {
6198 index = DRT_HASH(cmap, offset);
6199
6200 if (DRT_HASH_VACANT(cmap, index) || (DRT_HASH_GET_COUNT(cmap, index) == 0))
6201 continue;
6202
6203 /* scan the bitfield for a string of bits */
6204 fs = -1;
6205
6206 for (i = 0; i < DRT_BITVECTOR_PAGES; i++) {
6207 if (DRT_HASH_TEST_BIT(cmap, index, i)) {
6208 fs = i;
6209 break;
6210 }
6211 }
6212 if (fs == -1) {
6213 /* didn't find any bits set */
6214 panic("vfs_drt: entry summary count > 0 but no bits set in map");
6215 }
6216 for (ls = 0; i < DRT_BITVECTOR_PAGES; i++, ls++) {
6217 if (!DRT_HASH_TEST_BIT(cmap, index, i))
6218 break;
6219 }
6220
6221 /* compute offset and length, mark pages clean */
6222 offset = DRT_HASH_GET_ADDRESS(cmap, index) + (PAGE_SIZE * fs);
6223 length = ls * PAGE_SIZE;
6224 vfs_drt_do_mark_pages(cmapp, offset, length, NULL, 0);
6225 cmap->scm_lastclean = index;
6226
6227 /* return successful */
6228 *offsetp = (off_t)offset;
6229 *lengthp = length;
6230
6231 vfs_drt_trace(cmap, DRT_DEBUG_RETCLUSTER, (int)offset, (int)length, 0, 0);
6232 return(KERN_SUCCESS);
6233 }
6234 /*
6235 * We didn't find anything... hashtable is empty
6236 * emit stats into trace buffer and
6237 * then free it
6238 */
6239 vfs_drt_trace(cmap, DRT_DEBUG_SCMDATA,
6240 cmap->scm_modulus,
6241 cmap->scm_buckets,
6242 cmap->scm_lastclean,
6243 cmap->scm_iskips);
6244
6245 vfs_drt_free_map(cmap);
6246 *cmapp = NULL;
6247
6248 return(KERN_FAILURE);
6249 }
6250
6251
6252 static kern_return_t
6253 vfs_drt_control(void **cmapp, int op_type)
6254 {
6255 struct vfs_drt_clustermap *cmap;
6256
6257 /* sanity */
6258 if ((cmapp == NULL) || (*cmapp == NULL))
6259 return(KERN_FAILURE);
6260 cmap = *cmapp;
6261
6262 switch (op_type) {
6263 case 0:
6264 /* emit stats into trace buffer */
6265 vfs_drt_trace(cmap, DRT_DEBUG_SCMDATA,
6266 cmap->scm_modulus,
6267 cmap->scm_buckets,
6268 cmap->scm_lastclean,
6269 cmap->scm_iskips);
6270
6271 vfs_drt_free_map(cmap);
6272 *cmapp = NULL;
6273 break;
6274
6275 case 1:
6276 cmap->scm_lastclean = 0;
6277 break;
6278 }
6279 return(KERN_SUCCESS);
6280 }
6281
6282
6283
6284 /*
6285 * Emit a summary of the state of the clustermap into the trace buffer
6286 * along with some caller-provided data.
6287 */
6288 #if KDEBUG
6289 static void
6290 vfs_drt_trace(__unused struct vfs_drt_clustermap *cmap, int code, int arg1, int arg2, int arg3, int arg4)
6291 {
6292 KERNEL_DEBUG(code, arg1, arg2, arg3, arg4, 0);
6293 }
6294 #else
6295 static void
6296 vfs_drt_trace(__unused struct vfs_drt_clustermap *cmap, __unused int code,
6297 __unused int arg1, __unused int arg2, __unused int arg3,
6298 __unused int arg4)
6299 {
6300 }
6301 #endif
6302
6303 #if 0
6304 /*
6305 * Perform basic sanity check on the hash entry summary count
6306 * vs. the actual bits set in the entry.
6307 */
6308 static void
6309 vfs_drt_sanity(struct vfs_drt_clustermap *cmap)
6310 {
6311 int index, i;
6312 int bits_on;
6313
6314 for (index = 0; index < cmap->scm_modulus; index++) {
6315 if (DRT_HASH_VACANT(cmap, index))
6316 continue;
6317
6318 for (bits_on = 0, i = 0; i < DRT_BITVECTOR_PAGES; i++) {
6319 if (DRT_HASH_TEST_BIT(cmap, index, i))
6320 bits_on++;
6321 }
6322 if (bits_on != DRT_HASH_GET_COUNT(cmap, index))
6323 panic("bits_on = %d, index = %d\n", bits_on, index);
6324 }
6325 }
6326 #endif
mirror of XNU