]> git.saurik.com Git - apple/xnu.git/blob - bsd/miscfs/specfs/spec_vnops.c
projects / apple / xnu.git / blob
? search:


625e9736fb7b87cf4a1e7dadd1d11e36e152bf00
[apple/xnu.git] / bsd / miscfs / specfs / spec_vnops.c
1 /*
2 * Copyright (c) 2000-2016 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
29 /*
30 * Copyright (c) 1989, 1993, 1995
31 * The Regents of the University of California. All rights reserved.
32 *
33 * Redistribution and use in source and binary forms, with or without
34 * modification, are permitted provided that the following conditions
35 * are met:
36 * 1. Redistributions of source code must retain the above copyright
37 * notice, this list of conditions and the following disclaimer.
38 * 2. Redistributions in binary form must reproduce the above copyright
39 * notice, this list of conditions and the following disclaimer in the
40 * documentation and/or other materials provided with the distribution.
41 * 3. All advertising materials mentioning features or use of this software
42 * must display the following acknowledgement:
43 * This product includes software developed by the University of
44 * California, Berkeley and its contributors.
45 * 4. Neither the name of the University nor the names of its contributors
46 * may be used to endorse or promote products derived from this software
47 * without specific prior written permission.
48 *
49 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
50 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
51 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
52 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
53 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
54 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
55 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
56 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
57 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
58 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
59 * SUCH DAMAGE.
60 *
61 * @(#)spec_vnops.c 8.14 (Berkeley) 5/21/95
62 */
63
64 #include <sys/param.h>
65 #include <sys/proc_internal.h>
66 #include <sys/kauth.h>
67 #include <sys/systm.h>
68 #include <sys/kernel.h>
69 #include <sys/conf.h>
70 #include <sys/buf_internal.h>
71 #include <sys/mount_internal.h>
72 #include <sys/vnode_internal.h>
73 #include <sys/file_internal.h>
74 #include <sys/namei.h>
75 #include <sys/stat.h>
76 #include <sys/errno.h>
77 #include <sys/ioctl.h>
78 #include <sys/file.h>
79 #include <sys/user.h>
80 #include <sys/malloc.h>
81 #include <sys/disk.h>
82 #include <sys/uio_internal.h>
83 #include <sys/resource.h>
84 #include <machine/machine_routines.h>
85 #include <miscfs/specfs/specdev.h>
86 #include <vfs/vfs_support.h>
87 #include <vfs/vfs_disk_conditioner.h>
88
89 #include <kern/assert.h>
90 #include <kern/task.h>
91 #include <kern/sched_prim.h>
92 #include <kern/thread.h>
93 #include <kern/policy_internal.h>
94 #include <kern/timer_call.h>
95 #include <kern/waitq.h>
96
97 #include <pexpert/pexpert.h>
98
99 #include <sys/kdebug.h>
100 #include <libkern/section_keywords.h>
101
102 /* XXX following three prototypes should be in a header file somewhere */
103 extern dev_t chrtoblk(dev_t dev);
104 extern boolean_t iskmemdev(dev_t dev);
105 extern int bpfkqfilter(dev_t dev, struct knote *kn);
106 extern int ptsd_kqfilter(dev_t, struct knote *);
107
108 struct vnode *speclisth[SPECHSZ];
109
110 /* symbolic sleep message strings for devices */
111 char devopn[] = "devopn";
112 char devio[] = "devio";
113 char devwait[] = "devwait";
114 char devin[] = "devin";
115 char devout[] = "devout";
116 char devioc[] = "devioc";
117 char devcls[] = "devcls";
118
119 #define VOPFUNC int (*)(void *)
120
121 int (**spec_vnodeop_p)(void *);
122 struct vnodeopv_entry_desc spec_vnodeop_entries[] = {
123 { &vnop_default_desc, (VOPFUNC)vn_default_error },
124 { &vnop_lookup_desc, (VOPFUNC)spec_lookup }, /* lookup */
125 { &vnop_create_desc, (VOPFUNC)err_create }, /* create */
126 { &vnop_mknod_desc, (VOPFUNC)err_mknod }, /* mknod */
127 { &vnop_open_desc, (VOPFUNC)spec_open }, /* open */
128 { &vnop_close_desc, (VOPFUNC)spec_close }, /* close */
129 { &vnop_access_desc, (VOPFUNC)spec_access }, /* access */
130 { &vnop_getattr_desc, (VOPFUNC)spec_getattr }, /* getattr */
131 { &vnop_setattr_desc, (VOPFUNC)spec_setattr }, /* setattr */
132 { &vnop_read_desc, (VOPFUNC)spec_read }, /* read */
133 { &vnop_write_desc, (VOPFUNC)spec_write }, /* write */
134 { &vnop_ioctl_desc, (VOPFUNC)spec_ioctl }, /* ioctl */
135 { &vnop_select_desc, (VOPFUNC)spec_select }, /* select */
136 { &vnop_revoke_desc, (VOPFUNC)nop_revoke }, /* revoke */
137 { &vnop_mmap_desc, (VOPFUNC)err_mmap }, /* mmap */
138 { &vnop_fsync_desc, (VOPFUNC)spec_fsync }, /* fsync */
139 { &vnop_remove_desc, (VOPFUNC)err_remove }, /* remove */
140 { &vnop_link_desc, (VOPFUNC)err_link }, /* link */
141 { &vnop_rename_desc, (VOPFUNC)err_rename }, /* rename */
142 { &vnop_mkdir_desc, (VOPFUNC)err_mkdir }, /* mkdir */
143 { &vnop_rmdir_desc, (VOPFUNC)err_rmdir }, /* rmdir */
144 { &vnop_symlink_desc, (VOPFUNC)err_symlink }, /* symlink */
145 { &vnop_readdir_desc, (VOPFUNC)err_readdir }, /* readdir */
146 { &vnop_readlink_desc, (VOPFUNC)err_readlink }, /* readlink */
147 { &vnop_inactive_desc, (VOPFUNC)nop_inactive }, /* inactive */
148 { &vnop_reclaim_desc, (VOPFUNC)nop_reclaim }, /* reclaim */
149 { &vnop_strategy_desc, (VOPFUNC)spec_strategy }, /* strategy */
150 { &vnop_pathconf_desc, (VOPFUNC)spec_pathconf }, /* pathconf */
151 { &vnop_advlock_desc, (VOPFUNC)err_advlock }, /* advlock */
152 { &vnop_bwrite_desc, (VOPFUNC)spec_bwrite }, /* bwrite */
153 { &vnop_pagein_desc, (VOPFUNC)err_pagein }, /* Pagein */
154 { &vnop_pageout_desc, (VOPFUNC)err_pageout }, /* Pageout */
155 { &vnop_copyfile_desc, (VOPFUNC)err_copyfile }, /* Copyfile */
156 { &vnop_blktooff_desc, (VOPFUNC)spec_blktooff }, /* blktooff */
157 { &vnop_offtoblk_desc, (VOPFUNC)spec_offtoblk }, /* offtoblk */
158 { &vnop_blockmap_desc, (VOPFUNC)spec_blockmap }, /* blockmap */
159 { (struct vnodeop_desc*)NULL, (int(*)(void *))NULL }
160 };
161 struct vnodeopv_desc spec_vnodeop_opv_desc =
162 { &spec_vnodeop_p, spec_vnodeop_entries };
163
164
165 static void set_blocksize(vnode_t, dev_t);
166
167 #define LOWPRI_TIER1_WINDOW_MSECS 25
168 #define LOWPRI_TIER2_WINDOW_MSECS 100
169 #define LOWPRI_TIER3_WINDOW_MSECS 500
170
171 #define LOWPRI_TIER1_IO_PERIOD_MSECS 40
172 #define LOWPRI_TIER2_IO_PERIOD_MSECS 85
173 #define LOWPRI_TIER3_IO_PERIOD_MSECS 200
174
175 #define LOWPRI_TIER1_IO_PERIOD_SSD_MSECS 5
176 #define LOWPRI_TIER2_IO_PERIOD_SSD_MSECS 15
177 #define LOWPRI_TIER3_IO_PERIOD_SSD_MSECS 25
178
179
180 int throttle_windows_msecs[THROTTLE_LEVEL_END + 1] = {
181 0,
182 LOWPRI_TIER1_WINDOW_MSECS,
183 LOWPRI_TIER2_WINDOW_MSECS,
184 LOWPRI_TIER3_WINDOW_MSECS,
185 };
186
187 int throttle_io_period_msecs[THROTTLE_LEVEL_END + 1] = {
188 0,
189 LOWPRI_TIER1_IO_PERIOD_MSECS,
190 LOWPRI_TIER2_IO_PERIOD_MSECS,
191 LOWPRI_TIER3_IO_PERIOD_MSECS,
192 };
193
194 int throttle_io_period_ssd_msecs[THROTTLE_LEVEL_END + 1] = {
195 0,
196 LOWPRI_TIER1_IO_PERIOD_SSD_MSECS,
197 LOWPRI_TIER2_IO_PERIOD_SSD_MSECS,
198 LOWPRI_TIER3_IO_PERIOD_SSD_MSECS,
199 };
200
201
202 int throttled_count[THROTTLE_LEVEL_END + 1];
203
204 struct _throttle_io_info_t {
205 lck_mtx_t throttle_lock;
206
207 struct timeval throttle_last_write_timestamp;
208 struct timeval throttle_min_timer_deadline;
209 struct timeval throttle_window_start_timestamp[THROTTLE_LEVEL_END + 1]; /* window starts at both the beginning and completion of an I/O */
210 struct timeval throttle_last_IO_timestamp[THROTTLE_LEVEL_END + 1];
211 pid_t throttle_last_IO_pid[THROTTLE_LEVEL_END + 1];
212 struct timeval throttle_start_IO_period_timestamp[THROTTLE_LEVEL_END + 1];
213 int32_t throttle_inflight_count[THROTTLE_LEVEL_END + 1];
214
215 TAILQ_HEAD( , uthread) throttle_uthlist[THROTTLE_LEVEL_END + 1]; /* Lists of throttled uthreads */
216 int throttle_next_wake_level;
217
218 thread_call_t throttle_timer_call;
219 int32_t throttle_timer_ref;
220 int32_t throttle_timer_active;
221
222 int32_t throttle_io_count;
223 int32_t throttle_io_count_begin;
224 int *throttle_io_periods;
225 uint32_t throttle_io_period_num;
226
227 int32_t throttle_refcnt;
228 int32_t throttle_alloc;
229 int32_t throttle_disabled;
230 int32_t throttle_is_fusion_with_priority;
231 };
232
233 struct _throttle_io_info_t _throttle_io_info[LOWPRI_MAX_NUM_DEV];
234
235
236 int lowpri_throttle_enabled = 1;
237
238
239 static void throttle_info_end_io_internal(struct _throttle_io_info_t *info, int throttle_level);
240 static int throttle_info_update_internal(struct _throttle_io_info_t *info, uthread_t ut, int flags, boolean_t isssd, boolean_t inflight, struct bufattr *bap);
241 static int throttle_get_thread_throttle_level(uthread_t ut);
242 static int throttle_get_thread_throttle_level_internal(uthread_t ut, int io_tier);
243 void throttle_info_mount_reset_period(mount_t mp, int isssd);
244
245 /*
246 * Trivial lookup routine that always fails.
247 */
248 int
249 spec_lookup(struct vnop_lookup_args *ap)
250 {
251
252 *ap->a_vpp = NULL;
253 return (ENOTDIR);
254 }
255
256 static void
257 set_blocksize(struct vnode *vp, dev_t dev)
258 {
259 int (*size)(dev_t);
260 int rsize;
261
262 if ((major(dev) < nblkdev) && (size = bdevsw[major(dev)].d_psize)) {
263 rsize = (*size)(dev);
264 if (rsize <= 0) /* did size fail? */
265 vp->v_specsize = DEV_BSIZE;
266 else
267 vp->v_specsize = rsize;
268 }
269 else
270 vp->v_specsize = DEV_BSIZE;
271 }
272
273 void
274 set_fsblocksize(struct vnode *vp)
275 {
276
277 if (vp->v_type == VBLK) {
278 dev_t dev = (dev_t)vp->v_rdev;
279 int maj = major(dev);
280
281 if ((u_int)maj >= (u_int)nblkdev)
282 return;
283
284 vnode_lock(vp);
285 set_blocksize(vp, dev);
286 vnode_unlock(vp);
287 }
288
289 }
290
291
292 /*
293 * Open a special file.
294 */
295 int
296 spec_open(struct vnop_open_args *ap)
297 {
298 struct proc *p = vfs_context_proc(ap->a_context);
299 kauth_cred_t cred = vfs_context_ucred(ap->a_context);
300 struct vnode *vp = ap->a_vp;
301 dev_t bdev, dev = (dev_t)vp->v_rdev;
302 int maj = major(dev);
303 int error;
304
305 /*
306 * Don't allow open if fs is mounted -nodev.
307 */
308 if (vp->v_mount && (vp->v_mount->mnt_flag & MNT_NODEV))
309 return (ENXIO);
310
311 switch (vp->v_type) {
312
313 case VCHR:
314 if ((u_int)maj >= (u_int)nchrdev)
315 return (ENXIO);
316 if (cred != FSCRED && (ap->a_mode & FWRITE)) {
317 /*
318 * When running in very secure mode, do not allow
319 * opens for writing of any disk character devices.
320 */
321 if (securelevel >= 2 && isdisk(dev, VCHR))
322 return (EPERM);
323
324 /* Never allow writing to /dev/mem or /dev/kmem */
325 if (iskmemdev(dev))
326 return (EPERM);
327 /*
328 * When running in secure mode, do not allow opens for
329 * writing of character devices whose corresponding block
330 * devices are currently mounted.
331 */
332 if (securelevel >= 1) {
333 if ((bdev = chrtoblk(dev)) != NODEV && check_mountedon(bdev, VBLK, &error))
334 return (error);
335 }
336 }
337
338 devsw_lock(dev, S_IFCHR);
339 error = (*cdevsw[maj].d_open)(dev, ap->a_mode, S_IFCHR, p);
340
341 if (error == 0) {
342 vp->v_specinfo->si_opencount++;
343 }
344
345 devsw_unlock(dev, S_IFCHR);
346
347 if (error == 0 && cdevsw[maj].d_type == D_DISK && !vp->v_un.vu_specinfo->si_initted) {
348 int isssd = 0;
349 uint64_t throttle_mask = 0;
350 uint32_t devbsdunit = 0;
351
352 if (VNOP_IOCTL(vp, DKIOCGETTHROTTLEMASK, (caddr_t)&throttle_mask, 0, NULL) == 0) {
353
354 if (throttle_mask != 0 &&
355 VNOP_IOCTL(vp, DKIOCISSOLIDSTATE, (caddr_t)&isssd, 0, ap->a_context) == 0) {
356 /*
357 * as a reasonable approximation, only use the lowest bit of the mask
358 * to generate a disk unit number
359 */
360 devbsdunit = num_trailing_0(throttle_mask);
361
362 vnode_lock(vp);
363
364 vp->v_un.vu_specinfo->si_isssd = isssd;
365 vp->v_un.vu_specinfo->si_devbsdunit = devbsdunit;
366 vp->v_un.vu_specinfo->si_throttle_mask = throttle_mask;
367 vp->v_un.vu_specinfo->si_throttleable = 1;
368 vp->v_un.vu_specinfo->si_initted = 1;
369
370 vnode_unlock(vp);
371 }
372 }
373 if (vp->v_un.vu_specinfo->si_initted == 0) {
374 vnode_lock(vp);
375 vp->v_un.vu_specinfo->si_initted = 1;
376 vnode_unlock(vp);
377 }
378 }
379 return (error);
380
381 case VBLK:
382 if ((u_int)maj >= (u_int)nblkdev)
383 return (ENXIO);
384 /*
385 * When running in very secure mode, do not allow
386 * opens for writing of any disk block devices.
387 */
388 if (securelevel >= 2 && cred != FSCRED &&
389 (ap->a_mode & FWRITE) && bdevsw[maj].d_type == D_DISK)
390 return (EPERM);
391 /*
392 * Do not allow opens of block devices that are
393 * currently mounted.
394 */
395 if ( (error = vfs_mountedon(vp)) )
396 return (error);
397
398 devsw_lock(dev, S_IFBLK);
399 error = (*bdevsw[maj].d_open)(dev, ap->a_mode, S_IFBLK, p);
400 if (!error) {
401 vp->v_specinfo->si_opencount++;
402 }
403 devsw_unlock(dev, S_IFBLK);
404
405 if (!error) {
406 u_int64_t blkcnt;
407 u_int32_t blksize;
408 int setsize = 0;
409 u_int32_t size512 = 512;
410
411
412 if (!VNOP_IOCTL(vp, DKIOCGETBLOCKSIZE, (caddr_t)&blksize, 0, ap->a_context)) {
413 /* Switch to 512 byte sectors (temporarily) */
414
415 if (!VNOP_IOCTL(vp, DKIOCSETBLOCKSIZE, (caddr_t)&size512, FWRITE, ap->a_context)) {
416 /* Get the number of 512 byte physical blocks. */
417 if (!VNOP_IOCTL(vp, DKIOCGETBLOCKCOUNT, (caddr_t)&blkcnt, 0, ap->a_context)) {
418 setsize = 1;
419 }
420 }
421 /* If it doesn't set back, we can't recover */
422 if (VNOP_IOCTL(vp, DKIOCSETBLOCKSIZE, (caddr_t)&blksize, FWRITE, ap->a_context))
423 error = ENXIO;
424 }
425
426
427 vnode_lock(vp);
428 set_blocksize(vp, dev);
429
430 /*
431 * Cache the size in bytes of the block device for later
432 * use by spec_write().
433 */
434 if (setsize)
435 vp->v_specdevsize = blkcnt * (u_int64_t)size512;
436 else
437 vp->v_specdevsize = (u_int64_t)0; /* Default: Can't get */
438
439 vnode_unlock(vp);
440
441 }
442 return(error);
443 default:
444 panic("spec_open type");
445 }
446 return (0);
447 }
448
449 /*
450 * Vnode op for read
451 */
452 int
453 spec_read(struct vnop_read_args *ap)
454 {
455 struct vnode *vp = ap->a_vp;
456 struct uio *uio = ap->a_uio;
457 struct buf *bp;
458 daddr64_t bn, nextbn;
459 long bsize, bscale;
460 int devBlockSize=0;
461 int n, on;
462 int error = 0;
463 dev_t dev;
464
465 #if DIAGNOSTIC
466 if (uio->uio_rw != UIO_READ)
467 panic("spec_read mode");
468 if (UIO_SEG_IS_USER_SPACE(uio->uio_segflg))
469 panic("spec_read proc");
470 #endif
471 if (uio_resid(uio) == 0)
472 return (0);
473
474 switch (vp->v_type) {
475
476 case VCHR:
477 {
478 struct _throttle_io_info_t *throttle_info = NULL;
479 int thread_throttle_level;
480 if (cdevsw[major(vp->v_rdev)].d_type == D_DISK && vp->v_un.vu_specinfo->si_throttleable) {
481 throttle_info = &_throttle_io_info[vp->v_un.vu_specinfo->si_devbsdunit];
482 thread_throttle_level = throttle_info_update_internal(throttle_info, NULL, 0, vp->v_un.vu_specinfo->si_isssd, TRUE, NULL);
483 }
484 error = (*cdevsw[major(vp->v_rdev)].d_read)
485 (vp->v_rdev, uio, ap->a_ioflag);
486
487 if (throttle_info) {
488 throttle_info_end_io_internal(throttle_info, thread_throttle_level);
489 }
490
491 return (error);
492 }
493
494 case VBLK:
495 if (uio->uio_offset < 0)
496 return (EINVAL);
497
498 dev = vp->v_rdev;
499
500 devBlockSize = vp->v_specsize;
501
502 if (devBlockSize > PAGE_SIZE)
503 return (EINVAL);
504
505 bscale = PAGE_SIZE / devBlockSize;
506 bsize = bscale * devBlockSize;
507
508 do {
509 on = uio->uio_offset % bsize;
510
511 bn = (daddr64_t)((uio->uio_offset / devBlockSize) &~ (bscale - 1));
512
513 if (vp->v_speclastr + bscale == bn) {
514 nextbn = bn + bscale;
515 error = buf_breadn(vp, bn, (int)bsize, &nextbn,
516 (int *)&bsize, 1, NOCRED, &bp);
517 } else
518 error = buf_bread(vp, bn, (int)bsize, NOCRED, &bp);
519
520 vnode_lock(vp);
521 vp->v_speclastr = bn;
522 vnode_unlock(vp);
523
524 n = bsize - buf_resid(bp);
525 if ((on > n) || error) {
526 if (!error)
527 error = EINVAL;
528 buf_brelse(bp);
529 return (error);
530 }
531 n = min((unsigned)(n - on), uio_resid(uio));
532
533 error = uiomove((char *)buf_dataptr(bp) + on, n, uio);
534 if (n + on == bsize)
535 buf_markaged(bp);
536 buf_brelse(bp);
537 } while (error == 0 && uio_resid(uio) > 0 && n != 0);
538 return (error);
539
540 default:
541 panic("spec_read type");
542 }
543 /* NOTREACHED */
544
545 return (0);
546 }
547
548 /*
549 * Vnode op for write
550 */
551 int
552 spec_write(struct vnop_write_args *ap)
553 {
554 struct vnode *vp = ap->a_vp;
555 struct uio *uio = ap->a_uio;
556 struct buf *bp;
557 daddr64_t bn;
558 int bsize, blkmask, bscale;
559 int io_sync;
560 int devBlockSize=0;
561 int n, on;
562 int error = 0;
563 dev_t dev;
564
565 #if DIAGNOSTIC
566 if (uio->uio_rw != UIO_WRITE)
567 panic("spec_write mode");
568 if (UIO_SEG_IS_USER_SPACE(uio->uio_segflg))
569 panic("spec_write proc");
570 #endif
571
572 switch (vp->v_type) {
573
574 case VCHR:
575 {
576 struct _throttle_io_info_t *throttle_info = NULL;
577 int thread_throttle_level;
578 if (cdevsw[major(vp->v_rdev)].d_type == D_DISK && vp->v_un.vu_specinfo->si_throttleable) {
579 throttle_info = &_throttle_io_info[vp->v_un.vu_specinfo->si_devbsdunit];
580
581 thread_throttle_level = throttle_info_update_internal(throttle_info, NULL, 0, vp->v_un.vu_specinfo->si_isssd, TRUE, NULL);
582
583 microuptime(&throttle_info->throttle_last_write_timestamp);
584 }
585 error = (*cdevsw[major(vp->v_rdev)].d_write)
586 (vp->v_rdev, uio, ap->a_ioflag);
587
588 if (throttle_info) {
589 throttle_info_end_io_internal(throttle_info, thread_throttle_level);
590 }
591
592 return (error);
593 }
594
595 case VBLK:
596 if (uio_resid(uio) == 0)
597 return (0);
598 if (uio->uio_offset < 0)
599 return (EINVAL);
600
601 io_sync = (ap->a_ioflag & IO_SYNC);
602
603 dev = (vp->v_rdev);
604
605 devBlockSize = vp->v_specsize;
606 if (devBlockSize > PAGE_SIZE)
607 return(EINVAL);
608
609 bscale = PAGE_SIZE / devBlockSize;
610 blkmask = bscale - 1;
611 bsize = bscale * devBlockSize;
612
613
614 do {
615 bn = (daddr64_t)((uio->uio_offset / devBlockSize) &~ blkmask);
616 on = uio->uio_offset % bsize;
617
618 n = min((unsigned)(bsize - on), uio_resid(uio));
619
620 /*
621 * Use buf_getblk() as an optimization IFF:
622 *
623 * 1) We are reading exactly a block on a block
624 * aligned boundary
625 * 2) We know the size of the device from spec_open
626 * 3) The read doesn't span the end of the device
627 *
628 * Otherwise, we fall back on buf_bread().
629 */
630 if (n == bsize &&
631 vp->v_specdevsize != (u_int64_t)0 &&
632 (uio->uio_offset + (u_int64_t)n) > vp->v_specdevsize) {
633 /* reduce the size of the read to what is there */
634 n = (uio->uio_offset + (u_int64_t)n) - vp->v_specdevsize;
635 }
636
637 if (n == bsize)
638 bp = buf_getblk(vp, bn, bsize, 0, 0, BLK_WRITE);
639 else
640 error = (int)buf_bread(vp, bn, bsize, NOCRED, &bp);
641
642 /* Translate downstream error for upstream, if needed */
643 if (!error)
644 error = (int)buf_error(bp);
645 if (error) {
646 buf_brelse(bp);
647 return (error);
648 }
649 n = min(n, bsize - buf_resid(bp));
650
651 error = uiomove((char *)buf_dataptr(bp) + on, n, uio);
652 if (error) {
653 buf_brelse(bp);
654 return (error);
655 }
656 buf_markaged(bp);
657
658 if (io_sync)
659 error = buf_bwrite(bp);
660 else {
661 if ((n + on) == bsize)
662 error = buf_bawrite(bp);
663 else
664 error = buf_bdwrite(bp);
665 }
666 } while (error == 0 && uio_resid(uio) > 0 && n != 0);
667 return (error);
668
669 default:
670 panic("spec_write type");
671 }
672 /* NOTREACHED */
673
674 return (0);
675 }
676
677 /*
678 * Device ioctl operation.
679 */
680 int
681 spec_ioctl(struct vnop_ioctl_args *ap)
682 {
683 proc_t p = vfs_context_proc(ap->a_context);
684 dev_t dev = ap->a_vp->v_rdev;
685 int retval = 0;
686
687 KERNEL_DEBUG_CONSTANT(FSDBG_CODE(DBG_IOCTL, 0) | DBG_FUNC_START,
688 dev, ap->a_command, ap->a_fflag, ap->a_vp->v_type, 0);
689
690 switch (ap->a_vp->v_type) {
691
692 case VCHR:
693 retval = (*cdevsw[major(dev)].d_ioctl)(dev, ap->a_command, ap->a_data,
694 ap->a_fflag, p);
695 break;
696
697 case VBLK:
698 retval = (*bdevsw[major(dev)].d_ioctl)(dev, ap->a_command, ap->a_data, ap->a_fflag, p);
699 if (!retval && ap->a_command == DKIOCSETBLOCKSIZE)
700 ap->a_vp->v_specsize = *(uint32_t *)ap->a_data;
701 break;
702
703 default:
704 panic("spec_ioctl");
705 /* NOTREACHED */
706 }
707 KERNEL_DEBUG_CONSTANT(FSDBG_CODE(DBG_IOCTL, 0) | DBG_FUNC_END,
708 dev, ap->a_command, ap->a_fflag, retval, 0);
709
710 return (retval);
711 }
712
713 int
714 spec_select(struct vnop_select_args *ap)
715 {
716 proc_t p = vfs_context_proc(ap->a_context);
717 dev_t dev;
718
719 switch (ap->a_vp->v_type) {
720
721 default:
722 return (1); /* XXX */
723
724 case VCHR:
725 dev = ap->a_vp->v_rdev;
726 return (*cdevsw[major(dev)].d_select)(dev, ap->a_which, ap->a_wql, p);
727 }
728 }
729
730 static int filt_specattach(struct knote *kn, struct kevent_internal_s *kev);
731
732 int
733 spec_kqfilter(vnode_t vp, struct knote *kn, struct kevent_internal_s *kev)
734 {
735 dev_t dev;
736
737 assert(vnode_ischr(vp));
738
739 dev = vnode_specrdev(vp);
740
741 #if NETWORKING
742 /*
743 * Try a bpf device, as defined in bsd/net/bpf.c
744 * If it doesn't error out the attach, then it
745 * claimed it. Otherwise, fall through and try
746 * other attaches.
747 */
748 int32_t tmp_flags = kn->kn_flags;
749 int64_t tmp_data = kn->kn_data;
750 int res;
751
752 res = bpfkqfilter(dev, kn);
753 if ((kn->kn_flags & EV_ERROR) == 0) {
754 return res;
755 }
756 kn->kn_flags = tmp_flags;
757 kn->kn_data = tmp_data;
758 #endif
759
760 if (major(dev) > nchrdev) {
761 knote_set_error(kn, ENXIO);
762 return 0;
763 }
764
765 kn->kn_vnode_kqok = !!(cdevsw_flags[major(dev)] & CDEVSW_SELECT_KQUEUE);
766 kn->kn_vnode_use_ofst = !!(cdevsw_flags[major(dev)] & CDEVSW_USE_OFFSET);
767
768 if (cdevsw_flags[major(dev)] & CDEVSW_IS_PTS) {
769 kn->kn_filtid = EVFILTID_PTSD;
770 return ptsd_kqfilter(dev, kn);
771 } else if (cdevsw[major(dev)].d_type == D_TTY &&
772 !(cdevsw_flags[major(dev)] & CDEVSW_IS_PTC) &&
773 kn->kn_vnode_kqok) {
774 /*
775 * TTYs from drivers that use struct ttys use their own filter
776 * routines. The PTC driver doesn't use the tty for character
777 * counts, so it must go through the select fallback.
778 */
779 kn->kn_filtid = EVFILTID_TTY;
780 return knote_fops(kn)->f_attach(kn, kev);
781 }
782
783 /* Try to attach to other char special devices */
784 return filt_specattach(kn, kev);
785 }
786
787 /*
788 * Synch buffers associated with a block device
789 */
790 int
791 spec_fsync_internal(vnode_t vp, int waitfor, __unused vfs_context_t context)
792 {
793 if (vp->v_type == VCHR)
794 return (0);
795 /*
796 * Flush all dirty buffers associated with a block device.
797 */
798 buf_flushdirtyblks(vp, (waitfor == MNT_WAIT || waitfor == MNT_DWAIT), 0, "spec_fsync");
799
800 return (0);
801 }
802
803 int
804 spec_fsync(struct vnop_fsync_args *ap)
805 {
806 return spec_fsync_internal(ap->a_vp, ap->a_waitfor, ap->a_context);
807 }
808
809
810 /*
811 * Just call the device strategy routine
812 */
813 void throttle_init(void);
814
815
816 #if 0
817 #define DEBUG_ALLOC_THROTTLE_INFO(format, debug_info, args...) \
818 do { \
819 if ((debug_info)->alloc) \
820 printf("%s: "format, __FUNCTION__, ## args); \
821 } while(0)
822
823 #else
824 #define DEBUG_ALLOC_THROTTLE_INFO(format, debug_info, args...)
825 #endif
826
827
828 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier1_window_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_windows_msecs[THROTTLE_LEVEL_TIER1], 0, "");
829 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier2_window_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_windows_msecs[THROTTLE_LEVEL_TIER2], 0, "");
830 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier3_window_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_windows_msecs[THROTTLE_LEVEL_TIER3], 0, "");
831
832 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier1_io_period_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_msecs[THROTTLE_LEVEL_TIER1], 0, "");
833 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier2_io_period_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_msecs[THROTTLE_LEVEL_TIER2], 0, "");
834 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier3_io_period_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_msecs[THROTTLE_LEVEL_TIER3], 0, "");
835
836 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier1_io_period_ssd_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_ssd_msecs[THROTTLE_LEVEL_TIER1], 0, "");
837 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier2_io_period_ssd_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_ssd_msecs[THROTTLE_LEVEL_TIER2], 0, "");
838 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier3_io_period_ssd_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_ssd_msecs[THROTTLE_LEVEL_TIER3], 0, "");
839
840 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_enabled, CTLFLAG_RW | CTLFLAG_LOCKED, &lowpri_throttle_enabled, 0, "");
841
842
843 static lck_grp_t *throttle_lock_grp;
844 static lck_attr_t *throttle_lock_attr;
845 static lck_grp_attr_t *throttle_lock_grp_attr;
846
847
848 /*
849 * throttled I/O helper function
850 * convert the index of the lowest set bit to a device index
851 */
852 int
853 num_trailing_0(uint64_t n)
854 {
855 /*
856 * since in most cases the number of trailing 0s is very small,
857 * we simply counting sequentially from the lowest bit
858 */
859 if (n == 0)
860 return sizeof(n) * 8;
861 int count = 0;
862 while (!ISSET(n, 1)) {
863 n >>= 1;
864 ++count;
865 }
866 return count;
867 }
868
869
870 /*
871 * Release the reference and if the item was allocated and this is the last
872 * reference then free it.
873 *
874 * This routine always returns the old value.
875 */
876 static int
877 throttle_info_rel(struct _throttle_io_info_t *info)
878 {
879 SInt32 oldValue = OSDecrementAtomic(&info->throttle_refcnt);
880
881 DEBUG_ALLOC_THROTTLE_INFO("refcnt = %d info = %p\n",
882 info, (int)(oldValue -1), info );
883
884 /* The reference count just went negative, very bad */
885 if (oldValue == 0)
886 panic("throttle info ref cnt went negative!");
887
888 /*
889 * Once reference count is zero, no one else should be able to take a
890 * reference
891 */
892 if ((info->throttle_refcnt == 0) && (info->throttle_alloc)) {
893 DEBUG_ALLOC_THROTTLE_INFO("Freeing info = %p\n", info);
894
895 lck_mtx_destroy(&info->throttle_lock, throttle_lock_grp);
896 FREE(info, M_TEMP);
897 }
898 return oldValue;
899 }
900
901
902 /*
903 * Just take a reference on the throttle info structure.
904 *
905 * This routine always returns the old value.
906 */
907 static SInt32
908 throttle_info_ref(struct _throttle_io_info_t *info)
909 {
910 SInt32 oldValue = OSIncrementAtomic(&info->throttle_refcnt);
911
912 DEBUG_ALLOC_THROTTLE_INFO("refcnt = %d info = %p\n",
913 info, (int)(oldValue -1), info );
914 /* Allocated items should never have a reference of zero */
915 if (info->throttle_alloc && (oldValue == 0))
916 panic("Taking a reference without calling create throttle info!\n");
917
918 return oldValue;
919 }
920
921 /*
922 * on entry the throttle_lock is held...
923 * this function is responsible for taking
924 * and dropping the reference on the info
925 * structure which will keep it from going
926 * away while the timer is running if it
927 * happens to have been dynamically allocated by
928 * a network fileystem kext which is now trying
929 * to free it
930 */
931 static uint32_t
932 throttle_timer_start(struct _throttle_io_info_t *info, boolean_t update_io_count, int wakelevel)
933 {
934 struct timeval elapsed;
935 struct timeval now;
936 struct timeval period;
937 uint64_t elapsed_msecs;
938 int throttle_level;
939 int level;
940 int msecs;
941 boolean_t throttled = FALSE;
942 boolean_t need_timer = FALSE;
943
944 microuptime(&now);
945
946 if (update_io_count == TRUE) {
947 info->throttle_io_count_begin = info->throttle_io_count;
948 info->throttle_io_period_num++;
949
950 while (wakelevel >= THROTTLE_LEVEL_THROTTLED)
951 info->throttle_start_IO_period_timestamp[wakelevel--] = now;
952
953 info->throttle_min_timer_deadline = now;
954
955 msecs = info->throttle_io_periods[THROTTLE_LEVEL_THROTTLED];
956 period.tv_sec = msecs / 1000;
957 period.tv_usec = (msecs % 1000) * 1000;
958
959 timevaladd(&info->throttle_min_timer_deadline, &period);
960 }
961 for (throttle_level = THROTTLE_LEVEL_START; throttle_level < THROTTLE_LEVEL_END; throttle_level++) {
962
963 elapsed = now;
964 timevalsub(&elapsed, &info->throttle_window_start_timestamp[throttle_level]);
965 elapsed_msecs = (uint64_t)elapsed.tv_sec * (uint64_t)1000 + (elapsed.tv_usec / 1000);
966
967 for (level = throttle_level + 1; level <= THROTTLE_LEVEL_END; level++) {
968
969 if (!TAILQ_EMPTY(&info->throttle_uthlist[level])) {
970
971 if (elapsed_msecs < (uint64_t)throttle_windows_msecs[level] || info->throttle_inflight_count[throttle_level]) {
972 /*
973 * we had an I/O occur at a higher priority tier within
974 * this tier's throttle window
975 */
976 throttled = TRUE;
977 }
978 /*
979 * we assume that the windows are the same or longer
980 * as we drop through the throttling tiers... thus
981 * we can stop looking once we run into a tier with
982 * threads to schedule regardless of whether it's
983 * still in its throttling window or not
984 */
985 break;
986 }
987 }
988 if (throttled == TRUE)
989 break;
990 }
991 if (throttled == TRUE) {
992 uint64_t deadline = 0;
993 struct timeval target;
994 struct timeval min_target;
995
996 /*
997 * we've got at least one tier still in a throttled window
998 * so we need a timer running... compute the next deadline
999 * and schedule it
1000 */
1001 for (level = throttle_level+1; level <= THROTTLE_LEVEL_END; level++) {
1002
1003 if (TAILQ_EMPTY(&info->throttle_uthlist[level]))
1004 continue;
1005
1006 target = info->throttle_start_IO_period_timestamp[level];
1007
1008 msecs = info->throttle_io_periods[level];
1009 period.tv_sec = msecs / 1000;
1010 period.tv_usec = (msecs % 1000) * 1000;
1011
1012 timevaladd(&target, &period);
1013
1014 if (need_timer == FALSE || timevalcmp(&target, &min_target, <)) {
1015 min_target = target;
1016 need_timer = TRUE;
1017 }
1018 }
1019 if (timevalcmp(&info->throttle_min_timer_deadline, &now, >)) {
1020 if (timevalcmp(&info->throttle_min_timer_deadline, &min_target, >))
1021 min_target = info->throttle_min_timer_deadline;
1022 }
1023
1024 if (info->throttle_timer_active) {
1025 if (thread_call_cancel(info->throttle_timer_call) == FALSE) {
1026 /*
1027 * couldn't kill the timer because it's already
1028 * been dispatched, so don't try to start a new
1029 * one... once we drop the lock, the timer will
1030 * proceed and eventually re-run this function
1031 */
1032 need_timer = FALSE;
1033 } else
1034 info->throttle_timer_active = 0;
1035 }
1036 if (need_timer == TRUE) {
1037 /*
1038 * This is defined as an int (32-bit) rather than a 64-bit
1039 * value because it would need a really big period in the
1040 * order of ~500 days to overflow this. So, we let this be
1041 * 32-bit which allows us to use the clock_interval_to_deadline()
1042 * routine.
1043 */
1044 int target_msecs;
1045
1046 if (info->throttle_timer_ref == 0) {
1047 /*
1048 * take a reference for the timer
1049 */
1050 throttle_info_ref(info);
1051
1052 info->throttle_timer_ref = 1;
1053 }
1054 elapsed = min_target;
1055 timevalsub(&elapsed, &now);
1056 target_msecs = elapsed.tv_sec * 1000 + elapsed.tv_usec / 1000;
1057
1058 if (target_msecs <= 0) {
1059 /*
1060 * we may have computed a deadline slightly in the past
1061 * due to various factors... if so, just set the timer
1062 * to go off in the near future (we don't need to be precise)
1063 */
1064 target_msecs = 1;
1065 }
1066 clock_interval_to_deadline(target_msecs, 1000000, &deadline);
1067
1068 thread_call_enter_delayed(info->throttle_timer_call, deadline);
1069 info->throttle_timer_active = 1;
1070 }
1071 }
1072 return (throttle_level);
1073 }
1074
1075
1076 static void
1077 throttle_timer(struct _throttle_io_info_t *info)
1078 {
1079 uthread_t ut, utlist;
1080 struct timeval elapsed;
1081 struct timeval now;
1082 uint64_t elapsed_msecs;
1083 int throttle_level;
1084 int level;
1085 int wake_level;
1086 caddr_t wake_address = NULL;
1087 boolean_t update_io_count = FALSE;
1088 boolean_t need_wakeup = FALSE;
1089 boolean_t need_release = FALSE;
1090
1091 ut = NULL;
1092 lck_mtx_lock(&info->throttle_lock);
1093
1094 info->throttle_timer_active = 0;
1095 microuptime(&now);
1096
1097 elapsed = now;
1098 timevalsub(&elapsed, &info->throttle_start_IO_period_timestamp[THROTTLE_LEVEL_THROTTLED]);
1099 elapsed_msecs = (uint64_t)elapsed.tv_sec * (uint64_t)1000 + (elapsed.tv_usec / 1000);
1100
1101 if (elapsed_msecs >= (uint64_t)info->throttle_io_periods[THROTTLE_LEVEL_THROTTLED]) {
1102
1103 wake_level = info->throttle_next_wake_level;
1104
1105 for (level = THROTTLE_LEVEL_START; level < THROTTLE_LEVEL_END; level++) {
1106
1107 elapsed = now;
1108 timevalsub(&elapsed, &info->throttle_start_IO_period_timestamp[wake_level]);
1109 elapsed_msecs = (uint64_t)elapsed.tv_sec * (uint64_t)1000 + (elapsed.tv_usec / 1000);
1110
1111 if (elapsed_msecs >= (uint64_t)info->throttle_io_periods[wake_level] && !TAILQ_EMPTY(&info->throttle_uthlist[wake_level])) {
1112 /*
1113 * we're closing out the current IO period...
1114 * if we have a waiting thread, wake it up
1115 * after we have reset the I/O window info
1116 */
1117 need_wakeup = TRUE;
1118 update_io_count = TRUE;
1119
1120 info->throttle_next_wake_level = wake_level - 1;
1121
1122 if (info->throttle_next_wake_level == THROTTLE_LEVEL_START)
1123 info->throttle_next_wake_level = THROTTLE_LEVEL_END;
1124
1125 break;
1126 }
1127 wake_level--;
1128
1129 if (wake_level == THROTTLE_LEVEL_START)
1130 wake_level = THROTTLE_LEVEL_END;
1131 }
1132 }
1133 if (need_wakeup == TRUE) {
1134 if (!TAILQ_EMPTY(&info->throttle_uthlist[wake_level])) {
1135
1136 ut = (uthread_t)TAILQ_FIRST(&info->throttle_uthlist[wake_level]);
1137 TAILQ_REMOVE(&info->throttle_uthlist[wake_level], ut, uu_throttlelist);
1138 ut->uu_on_throttlelist = THROTTLE_LEVEL_NONE;
1139 ut->uu_is_throttled = FALSE;
1140
1141 wake_address = (caddr_t)&ut->uu_on_throttlelist;
1142 }
1143 } else
1144 wake_level = THROTTLE_LEVEL_START;
1145
1146 throttle_level = throttle_timer_start(info, update_io_count, wake_level);
1147
1148 if (wake_address != NULL)
1149 wakeup(wake_address);
1150
1151 for (level = THROTTLE_LEVEL_THROTTLED; level <= throttle_level; level++) {
1152
1153 TAILQ_FOREACH_SAFE(ut, &info->throttle_uthlist[level], uu_throttlelist, utlist) {
1154
1155 TAILQ_REMOVE(&info->throttle_uthlist[level], ut, uu_throttlelist);
1156 ut->uu_on_throttlelist = THROTTLE_LEVEL_NONE;
1157 ut->uu_is_throttled = FALSE;
1158
1159 wakeup(&ut->uu_on_throttlelist);
1160 }
1161 }
1162 if (info->throttle_timer_active == 0 && info->throttle_timer_ref) {
1163 info->throttle_timer_ref = 0;
1164 need_release = TRUE;
1165 }
1166 lck_mtx_unlock(&info->throttle_lock);
1167
1168 if (need_release == TRUE)
1169 throttle_info_rel(info);
1170 }
1171
1172
1173 static int
1174 throttle_add_to_list(struct _throttle_io_info_t *info, uthread_t ut, int mylevel, boolean_t insert_tail)
1175 {
1176 boolean_t start_timer = FALSE;
1177 int level = THROTTLE_LEVEL_START;
1178
1179 if (TAILQ_EMPTY(&info->throttle_uthlist[mylevel])) {
1180 info->throttle_start_IO_period_timestamp[mylevel] = info->throttle_last_IO_timestamp[mylevel];
1181 start_timer = TRUE;
1182 }
1183
1184 if (insert_tail == TRUE)
1185 TAILQ_INSERT_TAIL(&info->throttle_uthlist[mylevel], ut, uu_throttlelist);
1186 else
1187 TAILQ_INSERT_HEAD(&info->throttle_uthlist[mylevel], ut, uu_throttlelist);
1188
1189 ut->uu_on_throttlelist = mylevel;
1190
1191 if (start_timer == TRUE) {
1192 /* we may need to start or rearm the timer */
1193 level = throttle_timer_start(info, FALSE, THROTTLE_LEVEL_START);
1194
1195 if (level == THROTTLE_LEVEL_END) {
1196 if (ut->uu_on_throttlelist >= THROTTLE_LEVEL_THROTTLED) {
1197 TAILQ_REMOVE(&info->throttle_uthlist[ut->uu_on_throttlelist], ut, uu_throttlelist);
1198
1199 ut->uu_on_throttlelist = THROTTLE_LEVEL_NONE;
1200 }
1201 }
1202 }
1203 return (level);
1204 }
1205
1206 static void
1207 throttle_init_throttle_window(void)
1208 {
1209 int throttle_window_size;
1210
1211 /*
1212 * The hierarchy of throttle window values is as follows:
1213 * - Global defaults
1214 * - Device tree properties
1215 * - Boot-args
1216 * All values are specified in msecs.
1217 */
1218
1219 /* Override global values with device-tree properties */
1220 if (PE_get_default("kern.io_throttle_window_tier1", &throttle_window_size, sizeof(throttle_window_size)))
1221 throttle_windows_msecs[THROTTLE_LEVEL_TIER1] = throttle_window_size;
1222
1223 if (PE_get_default("kern.io_throttle_window_tier2", &throttle_window_size, sizeof(throttle_window_size)))
1224 throttle_windows_msecs[THROTTLE_LEVEL_TIER2] = throttle_window_size;
1225
1226 if (PE_get_default("kern.io_throttle_window_tier3", &throttle_window_size, sizeof(throttle_window_size)))
1227 throttle_windows_msecs[THROTTLE_LEVEL_TIER3] = throttle_window_size;
1228
1229 /* Override with boot-args */
1230 if (PE_parse_boot_argn("io_throttle_window_tier1", &throttle_window_size, sizeof(throttle_window_size)))
1231 throttle_windows_msecs[THROTTLE_LEVEL_TIER1] = throttle_window_size;
1232
1233 if (PE_parse_boot_argn("io_throttle_window_tier2", &throttle_window_size, sizeof(throttle_window_size)))
1234 throttle_windows_msecs[THROTTLE_LEVEL_TIER2] = throttle_window_size;
1235
1236 if (PE_parse_boot_argn("io_throttle_window_tier3", &throttle_window_size, sizeof(throttle_window_size)))
1237 throttle_windows_msecs[THROTTLE_LEVEL_TIER3] = throttle_window_size;
1238 }
1239
1240 static void
1241 throttle_init_throttle_period(struct _throttle_io_info_t *info, boolean_t isssd)
1242 {
1243 int throttle_period_size;
1244
1245 /*
1246 * The hierarchy of throttle period values is as follows:
1247 * - Global defaults
1248 * - Device tree properties
1249 * - Boot-args
1250 * All values are specified in msecs.
1251 */
1252
1253 /* Assign global defaults */
1254 if ((isssd == TRUE) && (info->throttle_is_fusion_with_priority == 0))
1255 info->throttle_io_periods = &throttle_io_period_ssd_msecs[0];
1256 else
1257 info->throttle_io_periods = &throttle_io_period_msecs[0];
1258
1259 /* Override global values with device-tree properties */
1260 if (PE_get_default("kern.io_throttle_period_tier1", &throttle_period_size, sizeof(throttle_period_size)))
1261 info->throttle_io_periods[THROTTLE_LEVEL_TIER1] = throttle_period_size;
1262
1263 if (PE_get_default("kern.io_throttle_period_tier2", &throttle_period_size, sizeof(throttle_period_size)))
1264 info->throttle_io_periods[THROTTLE_LEVEL_TIER2] = throttle_period_size;
1265
1266 if (PE_get_default("kern.io_throttle_period_tier3", &throttle_period_size, sizeof(throttle_period_size)))
1267 info->throttle_io_periods[THROTTLE_LEVEL_TIER3] = throttle_period_size;
1268
1269 /* Override with boot-args */
1270 if (PE_parse_boot_argn("io_throttle_period_tier1", &throttle_period_size, sizeof(throttle_period_size)))
1271 info->throttle_io_periods[THROTTLE_LEVEL_TIER1] = throttle_period_size;
1272
1273 if (PE_parse_boot_argn("io_throttle_period_tier2", &throttle_period_size, sizeof(throttle_period_size)))
1274 info->throttle_io_periods[THROTTLE_LEVEL_TIER2] = throttle_period_size;
1275
1276 if (PE_parse_boot_argn("io_throttle_period_tier3", &throttle_period_size, sizeof(throttle_period_size)))
1277 info->throttle_io_periods[THROTTLE_LEVEL_TIER3] = throttle_period_size;
1278
1279 }
1280
1281 #if CONFIG_IOSCHED
1282 extern void vm_io_reprioritize_init(void);
1283 int iosched_enabled = 1;
1284 #endif
1285
1286 void
1287 throttle_init(void)
1288 {
1289 struct _throttle_io_info_t *info;
1290 int i;
1291 int level;
1292 #if CONFIG_IOSCHED
1293 int iosched;
1294 #endif
1295 /*
1296 * allocate lock group attribute and group
1297 */
1298 throttle_lock_grp_attr = lck_grp_attr_alloc_init();
1299 throttle_lock_grp = lck_grp_alloc_init("throttle I/O", throttle_lock_grp_attr);
1300
1301 /* Update throttle parameters based on device tree configuration */
1302 throttle_init_throttle_window();
1303
1304 /*
1305 * allocate the lock attribute
1306 */
1307 throttle_lock_attr = lck_attr_alloc_init();
1308
1309 for (i = 0; i < LOWPRI_MAX_NUM_DEV; i++) {
1310 info = &_throttle_io_info[i];
1311
1312 lck_mtx_init(&info->throttle_lock, throttle_lock_grp, throttle_lock_attr);
1313 info->throttle_timer_call = thread_call_allocate((thread_call_func_t)throttle_timer, (thread_call_param_t)info);
1314
1315 for (level = 0; level <= THROTTLE_LEVEL_END; level++) {
1316 TAILQ_INIT(&info->throttle_uthlist[level]);
1317 info->throttle_last_IO_pid[level] = 0;
1318 info->throttle_inflight_count[level] = 0;
1319 }
1320 info->throttle_next_wake_level = THROTTLE_LEVEL_END;
1321 info->throttle_disabled = 0;
1322 info->throttle_is_fusion_with_priority = 0;
1323 }
1324 #if CONFIG_IOSCHED
1325 if (PE_parse_boot_argn("iosched", &iosched, sizeof(iosched))) {
1326 iosched_enabled = iosched;
1327 }
1328 if (iosched_enabled) {
1329 /* Initialize I/O Reprioritization mechanism */
1330 vm_io_reprioritize_init();
1331 }
1332 #endif
1333 }
1334
1335 void
1336 sys_override_io_throttle(int flag)
1337 {
1338 if (flag == THROTTLE_IO_ENABLE)
1339 lowpri_throttle_enabled = 1;
1340
1341 if (flag == THROTTLE_IO_DISABLE)
1342 lowpri_throttle_enabled = 0;
1343 }
1344
1345 int rethrottle_wakeups = 0;
1346
1347 /*
1348 * the uu_rethrottle_lock is used to synchronize this function
1349 * with "throttle_lowpri_io" which is where a throttled thread
1350 * will block... that function will grab this lock before beginning
1351 * it's decision making process concerning the need to block, and
1352 * hold it through the assert_wait. When that thread is awakened
1353 * for any reason (timer or rethrottle), it will reacquire the
1354 * uu_rethrottle_lock before determining if it really is ok for
1355 * it to now run. This is the point at which the thread could
1356 * enter a different throttling queue and reblock or return from
1357 * the throttle w/o having waited out it's entire throttle if
1358 * the rethrottle has now moved it out of any currently
1359 * active throttle window.
1360 *
1361 *
1362 * NOTES:
1363 * 1 - This may be called with the task lock held.
1364 * 2 - This may be called with preemption and interrupts disabled
1365 * in the kqueue wakeup path so we can't take the throttle_lock which is a mutex
1366 * 3 - This cannot safely dereference uu_throttle_info, as it may
1367 * get deallocated out from under us
1368 */
1369
1370 void
1371 rethrottle_thread(uthread_t ut)
1372 {
1373 /*
1374 * If uthread doesn't have throttle state, then there's no chance
1375 * of it needing a rethrottle.
1376 */
1377 if (ut->uu_throttle_info == NULL)
1378 return;
1379
1380 boolean_t s = ml_set_interrupts_enabled(FALSE);
1381 lck_spin_lock(&ut->uu_rethrottle_lock);
1382
1383 if (ut->uu_is_throttled == FALSE)
1384 ut->uu_was_rethrottled = TRUE;
1385 else {
1386 int my_new_level = throttle_get_thread_throttle_level(ut);
1387
1388 if (my_new_level != ut->uu_on_throttlelist) {
1389 /*
1390 * ut is currently blocked (as indicated by
1391 * ut->uu_is_throttled == TRUE)
1392 * and we're changing it's throttle level, so
1393 * we need to wake it up.
1394 */
1395 ut->uu_is_throttled = FALSE;
1396 wakeup(&ut->uu_on_throttlelist);
1397
1398 rethrottle_wakeups++;
1399 KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_FSRW, 102)), thread_tid(ut->uu_thread), ut->uu_on_throttlelist, my_new_level, 0, 0);
1400 }
1401 }
1402 lck_spin_unlock(&ut->uu_rethrottle_lock);
1403 ml_set_interrupts_enabled(s);
1404 }
1405
1406
1407 /*
1408 * KPI routine
1409 *
1410 * Create and take a reference on a throttle info structure and return a
1411 * pointer for the file system to use when calling throttle_info_update.
1412 * Calling file system must have a matching release for every create.
1413 */
1414 void *
1415 throttle_info_create(void)
1416 {
1417 struct _throttle_io_info_t *info;
1418 int level;
1419
1420 MALLOC(info, struct _throttle_io_info_t *, sizeof(*info), M_TEMP, M_ZERO | M_WAITOK);
1421 /* Should never happen but just in case */
1422 if (info == NULL)
1423 return NULL;
1424 /* Mark that this one was allocated and needs to be freed */
1425 DEBUG_ALLOC_THROTTLE_INFO("Creating info = %p\n", info, info );
1426 info->throttle_alloc = TRUE;
1427
1428 lck_mtx_init(&info->throttle_lock, throttle_lock_grp, throttle_lock_attr);
1429 info->throttle_timer_call = thread_call_allocate((thread_call_func_t)throttle_timer, (thread_call_param_t)info);
1430
1431 for (level = 0; level <= THROTTLE_LEVEL_END; level++) {
1432 TAILQ_INIT(&info->throttle_uthlist[level]);
1433 }
1434 info->throttle_next_wake_level = THROTTLE_LEVEL_END;
1435
1436 /* Take a reference */
1437 OSIncrementAtomic(&info->throttle_refcnt);
1438 return info;
1439 }
1440
1441 /*
1442 * KPI routine
1443 *
1444 * Release the throttle info pointer if all the reference are gone. Should be
1445 * called to release reference taken by throttle_info_create
1446 */
1447 void
1448 throttle_info_release(void *throttle_info)
1449 {
1450 DEBUG_ALLOC_THROTTLE_INFO("Releaseing info = %p\n",
1451 (struct _throttle_io_info_t *)throttle_info,
1452 (struct _throttle_io_info_t *)throttle_info);
1453 if (throttle_info) /* Just to be careful */
1454 throttle_info_rel(throttle_info);
1455 }
1456
1457 /*
1458 * KPI routine
1459 *
1460 * File Systems that create an info structure, need to call this routine in
1461 * their mount routine (used by cluster code). File Systems that call this in
1462 * their mount routines must call throttle_info_mount_rel in their unmount
1463 * routines.
1464 */
1465 void
1466 throttle_info_mount_ref(mount_t mp, void *throttle_info)
1467 {
1468 if ((throttle_info == NULL) || (mp == NULL))
1469 return;
1470 throttle_info_ref(throttle_info);
1471
1472 /*
1473 * We already have a reference release it before adding the new one
1474 */
1475 if (mp->mnt_throttle_info)
1476 throttle_info_rel(mp->mnt_throttle_info);
1477 mp->mnt_throttle_info = throttle_info;
1478 }
1479
1480 /*
1481 * Private KPI routine
1482 *
1483 * return a handle for accessing throttle_info given a throttle_mask. The
1484 * handle must be released by throttle_info_rel_by_mask
1485 */
1486 int
1487 throttle_info_ref_by_mask(uint64_t throttle_mask, throttle_info_handle_t *throttle_info_handle)
1488 {
1489 int dev_index;
1490 struct _throttle_io_info_t *info;
1491
1492 if (throttle_info_handle == NULL)
1493 return EINVAL;
1494
1495 dev_index = num_trailing_0(throttle_mask);
1496 info = &_throttle_io_info[dev_index];
1497 throttle_info_ref(info);
1498 *(struct _throttle_io_info_t**)throttle_info_handle = info;
1499
1500 return 0;
1501 }
1502
1503 /*
1504 * Private KPI routine
1505 *
1506 * release the handle obtained by throttle_info_ref_by_mask
1507 */
1508 void
1509 throttle_info_rel_by_mask(throttle_info_handle_t throttle_info_handle)
1510 {
1511 /*
1512 * for now the handle is just a pointer to _throttle_io_info_t
1513 */
1514 throttle_info_rel((struct _throttle_io_info_t*)throttle_info_handle);
1515 }
1516
1517 /*
1518 * KPI routine
1519 *
1520 * File Systems that throttle_info_mount_ref, must call this routine in their
1521 * umount routine.
1522 */
1523 void
1524 throttle_info_mount_rel(mount_t mp)
1525 {
1526 if (mp->mnt_throttle_info)
1527 throttle_info_rel(mp->mnt_throttle_info);
1528 mp->mnt_throttle_info = NULL;
1529 }
1530
1531 /*
1532 * Reset throttling periods for the given mount point
1533 *
1534 * private interface used by disk conditioner to reset
1535 * throttling periods when 'is_ssd' status changes
1536 */
1537 void
1538 throttle_info_mount_reset_period(mount_t mp, int isssd)
1539 {
1540 struct _throttle_io_info_t *info;
1541
1542 if (mp == NULL)
1543 info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
1544 else if (mp->mnt_throttle_info == NULL)
1545 info = &_throttle_io_info[mp->mnt_devbsdunit];
1546 else
1547 info = mp->mnt_throttle_info;
1548
1549 throttle_init_throttle_period(info, isssd);
1550 }
1551
1552 void
1553 throttle_info_get_last_io_time(mount_t mp, struct timeval *tv)
1554 {
1555 struct _throttle_io_info_t *info;
1556
1557 if (mp == NULL)
1558 info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
1559 else if (mp->mnt_throttle_info == NULL)
1560 info = &_throttle_io_info[mp->mnt_devbsdunit];
1561 else
1562 info = mp->mnt_throttle_info;
1563
1564 *tv = info->throttle_last_write_timestamp;
1565 }
1566
1567 void
1568 update_last_io_time(mount_t mp)
1569 {
1570 struct _throttle_io_info_t *info;
1571
1572 if (mp == NULL)
1573 info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
1574 else if (mp->mnt_throttle_info == NULL)
1575 info = &_throttle_io_info[mp->mnt_devbsdunit];
1576 else
1577 info = mp->mnt_throttle_info;
1578
1579 microuptime(&info->throttle_last_write_timestamp);
1580 if (mp != NULL)
1581 mp->mnt_last_write_completed_timestamp = info->throttle_last_write_timestamp;
1582 }
1583
1584 int
1585 throttle_get_io_policy(uthread_t *ut)
1586 {
1587 if (ut != NULL)
1588 *ut = get_bsdthread_info(current_thread());
1589
1590 return (proc_get_effective_thread_policy(current_thread(), TASK_POLICY_IO));
1591 }
1592
1593 int
1594 throttle_get_passive_io_policy(uthread_t *ut)
1595 {
1596 if (ut != NULL)
1597 *ut = get_bsdthread_info(current_thread());
1598
1599 return (proc_get_effective_thread_policy(current_thread(), TASK_POLICY_PASSIVE_IO));
1600 }
1601
1602
1603 static int
1604 throttle_get_thread_throttle_level(uthread_t ut)
1605 {
1606 uthread_t *ut_p = (ut == NULL) ? &ut : NULL;
1607 int io_tier = throttle_get_io_policy(ut_p);
1608
1609 return throttle_get_thread_throttle_level_internal(ut, io_tier);
1610 }
1611
1612 /*
1613 * Return a throttle level given an existing I/O tier (such as returned by throttle_get_io_policy)
1614 */
1615 static int
1616 throttle_get_thread_throttle_level_internal(uthread_t ut, int io_tier) {
1617 int thread_throttle_level = io_tier;
1618 int user_idle_level;
1619
1620 assert(ut != NULL);
1621
1622 /* Bootcache misses should always be throttled */
1623 if (ut->uu_throttle_bc == TRUE)
1624 thread_throttle_level = THROTTLE_LEVEL_TIER3;
1625
1626 /*
1627 * Issue tier3 I/O as tier2 when the user is idle
1628 * to allow maintenance tasks to make more progress.
1629 *
1630 * Assume any positive idle level is enough... for now it's
1631 * only ever 0 or 128 but this is not defined anywhere.
1632 */
1633 if (thread_throttle_level >= THROTTLE_LEVEL_TIER3) {
1634 user_idle_level = timer_get_user_idle_level();
1635 if (user_idle_level > 0) {
1636 thread_throttle_level--;
1637 }
1638 }
1639
1640 return (thread_throttle_level);
1641 }
1642
1643 /*
1644 * I/O will be throttled if either of the following are true:
1645 * - Higher tiers have in-flight I/O
1646 * - The time delta since the last start/completion of a higher tier is within the throttle window interval
1647 *
1648 * In-flight I/O is bookended by throttle_info_update_internal/throttle_info_end_io_internal
1649 */
1650 static int
1651 throttle_io_will_be_throttled_internal(void * throttle_info, int * mylevel, int * throttling_level)
1652 {
1653 struct _throttle_io_info_t *info = throttle_info;
1654 struct timeval elapsed;
1655 struct timeval now;
1656 uint64_t elapsed_msecs;
1657 int thread_throttle_level;
1658 int throttle_level;
1659
1660 if ((thread_throttle_level = throttle_get_thread_throttle_level(NULL)) < THROTTLE_LEVEL_THROTTLED)
1661 return (THROTTLE_DISENGAGED);
1662
1663 microuptime(&now);
1664
1665 for (throttle_level = THROTTLE_LEVEL_START; throttle_level < thread_throttle_level; throttle_level++) {
1666 if (info->throttle_inflight_count[throttle_level]) {
1667 break;
1668 }
1669 elapsed = now;
1670 timevalsub(&elapsed, &info->throttle_window_start_timestamp[throttle_level]);
1671 elapsed_msecs = (uint64_t)elapsed.tv_sec * (uint64_t)1000 + (elapsed.tv_usec / 1000);
1672
1673 if (elapsed_msecs < (uint64_t)throttle_windows_msecs[thread_throttle_level])
1674 break;
1675 }
1676 if (throttle_level >= thread_throttle_level) {
1677 /*
1678 * we're beyond all of the throttle windows
1679 * that affect the throttle level of this thread,
1680 * so go ahead and treat as normal I/O
1681 */
1682 return (THROTTLE_DISENGAGED);
1683 }
1684 if (mylevel)
1685 *mylevel = thread_throttle_level;
1686 if (throttling_level)
1687 *throttling_level = throttle_level;
1688
1689 if (info->throttle_io_count != info->throttle_io_count_begin) {
1690 /*
1691 * we've already issued at least one throttleable I/O
1692 * in the current I/O window, so avoid issuing another one
1693 */
1694 return (THROTTLE_NOW);
1695 }
1696 /*
1697 * we're in the throttle window, so
1698 * cut the I/O size back
1699 */
1700 return (THROTTLE_ENGAGED);
1701 }
1702
1703 /*
1704 * If we have a mount point and it has a throttle info pointer then
1705 * use it to do the check, otherwise use the device unit number to find
1706 * the correct throttle info array element.
1707 */
1708 int
1709 throttle_io_will_be_throttled(__unused int lowpri_window_msecs, mount_t mp)
1710 {
1711 struct _throttle_io_info_t *info;
1712
1713 /*
1714 * Should we just return zero if no mount point
1715 */
1716 if (mp == NULL)
1717 info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
1718 else if (mp->mnt_throttle_info == NULL)
1719 info = &_throttle_io_info[mp->mnt_devbsdunit];
1720 else
1721 info = mp->mnt_throttle_info;
1722
1723 if (info->throttle_is_fusion_with_priority) {
1724 uthread_t ut = get_bsdthread_info(current_thread());
1725 if (ut->uu_lowpri_window == 0)
1726 return (THROTTLE_DISENGAGED);
1727 }
1728
1729 if (info->throttle_disabled)
1730 return (THROTTLE_DISENGAGED);
1731 else
1732 return throttle_io_will_be_throttled_internal(info, NULL, NULL);
1733 }
1734
1735 /*
1736 * Routine to increment I/O throttling counters maintained in the proc
1737 */
1738
1739 static void
1740 throttle_update_proc_stats(pid_t throttling_pid, int count)
1741 {
1742 proc_t throttling_proc;
1743 proc_t throttled_proc = current_proc();
1744
1745 /* The throttled_proc is always the current proc; so we are not concerned with refs */
1746 OSAddAtomic64(count, &(throttled_proc->was_throttled));
1747
1748 /* The throttling pid might have exited by now */
1749 throttling_proc = proc_find(throttling_pid);
1750 if (throttling_proc != PROC_NULL) {
1751 OSAddAtomic64(count, &(throttling_proc->did_throttle));
1752 proc_rele(throttling_proc);
1753 }
1754 }
1755
1756 /*
1757 * Block until woken up by the throttle timer or by a rethrottle call.
1758 * As long as we hold the throttle_lock while querying the throttle tier, we're
1759 * safe against seeing an old throttle tier after a rethrottle.
1760 */
1761 uint32_t
1762 throttle_lowpri_io(int sleep_amount)
1763 {
1764 uthread_t ut;
1765 struct _throttle_io_info_t *info;
1766 int throttle_type = 0;
1767 int mylevel = 0;
1768 int throttling_level = THROTTLE_LEVEL_NONE;
1769 int sleep_cnt = 0;
1770 uint32_t throttle_io_period_num = 0;
1771 boolean_t insert_tail = TRUE;
1772 boolean_t s;
1773
1774 ut = get_bsdthread_info(current_thread());
1775
1776 if (ut->uu_lowpri_window == 0)
1777 return (0);
1778
1779 info = ut->uu_throttle_info;
1780
1781 if (info == NULL) {
1782 ut->uu_throttle_bc = FALSE;
1783 ut->uu_lowpri_window = 0;
1784 return (0);
1785 }
1786 lck_mtx_lock(&info->throttle_lock);
1787 assert(ut->uu_on_throttlelist < THROTTLE_LEVEL_THROTTLED);
1788
1789 if (sleep_amount == 0)
1790 goto done;
1791
1792 if (sleep_amount == 1 && ut->uu_throttle_bc == FALSE)
1793 sleep_amount = 0;
1794
1795 throttle_io_period_num = info->throttle_io_period_num;
1796
1797 ut->uu_was_rethrottled = FALSE;
1798
1799 while ( (throttle_type = throttle_io_will_be_throttled_internal(info, &mylevel, &throttling_level)) ) {
1800
1801 if (throttle_type == THROTTLE_ENGAGED) {
1802 if (sleep_amount == 0)
1803 break;
1804 if (info->throttle_io_period_num < throttle_io_period_num)
1805 break;
1806 if ((info->throttle_io_period_num - throttle_io_period_num) >= (uint32_t)sleep_amount)
1807 break;
1808 }
1809 /*
1810 * keep the same position in the list if "rethrottle_thread" changes our throttle level and
1811 * then puts us back to the original level before we get a chance to run
1812 */
1813 if (ut->uu_on_throttlelist >= THROTTLE_LEVEL_THROTTLED && ut->uu_on_throttlelist != mylevel) {
1814 /*
1815 * must have been awakened via "rethrottle_thread" (the timer pulls us off the list)
1816 * and we've changed our throttling level, so pull ourselves off of the appropriate list
1817 * and make sure we get put on the tail of the new list since we're starting anew w/r to
1818 * the throttling engine
1819 */
1820 TAILQ_REMOVE(&info->throttle_uthlist[ut->uu_on_throttlelist], ut, uu_throttlelist);
1821 ut->uu_on_throttlelist = THROTTLE_LEVEL_NONE;
1822 insert_tail = TRUE;
1823 }
1824 if (ut->uu_on_throttlelist < THROTTLE_LEVEL_THROTTLED) {
1825 if (throttle_add_to_list(info, ut, mylevel, insert_tail) == THROTTLE_LEVEL_END)
1826 goto done;
1827 }
1828 assert(throttling_level >= THROTTLE_LEVEL_START && throttling_level <= THROTTLE_LEVEL_END);
1829
1830 s = ml_set_interrupts_enabled(FALSE);
1831 lck_spin_lock(&ut->uu_rethrottle_lock);
1832
1833 /*
1834 * this is the critical section w/r to our interaction
1835 * with "rethrottle_thread"
1836 */
1837 if (ut->uu_was_rethrottled == TRUE) {
1838
1839 lck_spin_unlock(&ut->uu_rethrottle_lock);
1840 ml_set_interrupts_enabled(s);
1841 lck_mtx_yield(&info->throttle_lock);
1842
1843 KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_FSRW, 103)), thread_tid(ut->uu_thread), ut->uu_on_throttlelist, 0, 0, 0);
1844
1845 ut->uu_was_rethrottled = FALSE;
1846 continue;
1847 }
1848 KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_THROTTLE, PROCESS_THROTTLED)) | DBG_FUNC_NONE,
1849 info->throttle_last_IO_pid[throttling_level], throttling_level, proc_selfpid(), mylevel, 0);
1850
1851 if (sleep_cnt == 0) {
1852 KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_FSRW, 97)) | DBG_FUNC_START,
1853 throttle_windows_msecs[mylevel], info->throttle_io_periods[mylevel], info->throttle_io_count, 0, 0);
1854 throttled_count[mylevel]++;
1855 }
1856 ut->uu_wmesg = "throttle_lowpri_io";
1857
1858 assert_wait((caddr_t)&ut->uu_on_throttlelist, THREAD_UNINT);
1859
1860 ut->uu_is_throttled = TRUE;
1861 lck_spin_unlock(&ut->uu_rethrottle_lock);
1862 ml_set_interrupts_enabled(s);
1863
1864 lck_mtx_unlock(&info->throttle_lock);
1865
1866 thread_block(THREAD_CONTINUE_NULL);
1867
1868 ut->uu_wmesg = NULL;
1869
1870 ut->uu_is_throttled = FALSE;
1871 ut->uu_was_rethrottled = FALSE;
1872
1873 lck_mtx_lock(&info->throttle_lock);
1874
1875 sleep_cnt++;
1876
1877 if (sleep_amount == 0)
1878 insert_tail = FALSE;
1879 else if (info->throttle_io_period_num < throttle_io_period_num ||
1880 (info->throttle_io_period_num - throttle_io_period_num) >= (uint32_t)sleep_amount) {
1881 insert_tail = FALSE;
1882 sleep_amount = 0;
1883 }
1884 }
1885 done:
1886 if (ut->uu_on_throttlelist >= THROTTLE_LEVEL_THROTTLED) {
1887 TAILQ_REMOVE(&info->throttle_uthlist[ut->uu_on_throttlelist], ut, uu_throttlelist);
1888 ut->uu_on_throttlelist = THROTTLE_LEVEL_NONE;
1889 }
1890 lck_mtx_unlock(&info->throttle_lock);
1891
1892 if (sleep_cnt) {
1893 KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_FSRW, 97)) | DBG_FUNC_END,
1894 throttle_windows_msecs[mylevel], info->throttle_io_periods[mylevel], info->throttle_io_count, 0, 0);
1895 /*
1896 * We update the stats for the last pid which opened a throttle window for the throttled thread.
1897 * This might not be completely accurate since the multiple throttles seen by the lower tier pid
1898 * might have been caused by various higher prio pids. However, updating these stats accurately
1899 * means doing a proc_find while holding the throttle lock which leads to deadlock.
1900 */
1901 throttle_update_proc_stats(info->throttle_last_IO_pid[throttling_level], sleep_cnt);
1902 }
1903
1904 ut->uu_throttle_info = NULL;
1905 ut->uu_throttle_bc = FALSE;
1906 ut->uu_lowpri_window = 0;
1907
1908 throttle_info_rel(info);
1909
1910 return (sleep_cnt);
1911 }
1912
1913 /*
1914 * KPI routine
1915 *
1916 * set a kernel thread's IO policy. policy can be:
1917 * IOPOL_NORMAL, IOPOL_THROTTLE, IOPOL_PASSIVE, IOPOL_UTILITY, IOPOL_STANDARD
1918 *
1919 * explanations about these policies are in the man page of setiopolicy_np
1920 */
1921 void throttle_set_thread_io_policy(int policy)
1922 {
1923 proc_set_thread_policy(current_thread(), TASK_POLICY_INTERNAL, TASK_POLICY_IOPOL, policy);
1924 }
1925
1926 void throttle_info_reset_window(uthread_t ut)
1927 {
1928 struct _throttle_io_info_t *info;
1929
1930 if (ut == NULL)
1931 ut = get_bsdthread_info(current_thread());
1932
1933 if ( (info = ut->uu_throttle_info) ) {
1934 throttle_info_rel(info);
1935
1936 ut->uu_throttle_info = NULL;
1937 ut->uu_lowpri_window = 0;
1938 ut->uu_throttle_bc = FALSE;
1939 }
1940 }
1941
1942 static
1943 void throttle_info_set_initial_window(uthread_t ut, struct _throttle_io_info_t *info, boolean_t BC_throttle, boolean_t isssd)
1944 {
1945 if (lowpri_throttle_enabled == 0 || info->throttle_disabled)
1946 return;
1947
1948 if (info->throttle_io_periods == 0) {
1949 throttle_init_throttle_period(info, isssd);
1950 }
1951 if (ut->uu_throttle_info == NULL) {
1952
1953 ut->uu_throttle_info = info;
1954 throttle_info_ref(info);
1955 DEBUG_ALLOC_THROTTLE_INFO("updating info = %p\n", info, info );
1956
1957 ut->uu_lowpri_window = 1;
1958 ut->uu_throttle_bc = BC_throttle;
1959 }
1960 }
1961
1962 /*
1963 * Update inflight IO count and throttling window
1964 * Should be called when an IO is done
1965 *
1966 * Only affects IO that was sent through spec_strategy
1967 */
1968 void throttle_info_end_io(buf_t bp) {
1969 mount_t mp;
1970 struct bufattr *bap;
1971 struct _throttle_io_info_t *info;
1972 int io_tier;
1973
1974 bap = &bp->b_attr;
1975 if (!ISSET(bap->ba_flags, BA_STRATEGY_TRACKED_IO)) {
1976 return;
1977 }
1978 CLR(bap->ba_flags, BA_STRATEGY_TRACKED_IO);
1979
1980 mp = buf_vnode(bp)->v_mount;
1981 if (mp != NULL) {
1982 info = &_throttle_io_info[mp->mnt_devbsdunit];
1983 } else {
1984 info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
1985 }
1986
1987 io_tier = GET_BUFATTR_IO_TIER(bap);
1988 if (ISSET(bap->ba_flags, BA_IO_TIER_UPGRADE)) {
1989 io_tier--;
1990 }
1991
1992 throttle_info_end_io_internal(info, io_tier);
1993 }
1994
1995 /*
1996 * Decrement inflight count initially incremented by throttle_info_update_internal
1997 */
1998 static
1999 void throttle_info_end_io_internal(struct _throttle_io_info_t *info, int throttle_level) {
2000 if (throttle_level == THROTTLE_LEVEL_NONE) {
2001 return;
2002 }
2003
2004 microuptime(&info->throttle_window_start_timestamp[throttle_level]);
2005 OSDecrementAtomic(&info->throttle_inflight_count[throttle_level]);
2006 assert(info->throttle_inflight_count[throttle_level] >= 0);
2007 }
2008
2009 /*
2010 * If inflight is TRUE and bap is NULL then the caller is responsible for calling
2011 * throttle_info_end_io_internal to avoid leaking in-flight I/O.
2012 */
2013 static
2014 int throttle_info_update_internal(struct _throttle_io_info_t *info, uthread_t ut, int flags, boolean_t isssd, boolean_t inflight, struct bufattr *bap)
2015 {
2016 int thread_throttle_level;
2017
2018 if (lowpri_throttle_enabled == 0 || info->throttle_disabled)
2019 return THROTTLE_LEVEL_NONE;
2020
2021 if (ut == NULL)
2022 ut = get_bsdthread_info(current_thread());
2023
2024 if (bap && inflight && !ut->uu_throttle_bc) {
2025 thread_throttle_level = GET_BUFATTR_IO_TIER(bap);
2026 if (ISSET(bap->ba_flags, BA_IO_TIER_UPGRADE)) {
2027 thread_throttle_level--;
2028 }
2029 } else {
2030 thread_throttle_level = throttle_get_thread_throttle_level(ut);
2031 }
2032
2033 if (thread_throttle_level != THROTTLE_LEVEL_NONE) {
2034 if(!ISSET(flags, B_PASSIVE)) {
2035 info->throttle_last_IO_pid[thread_throttle_level] = proc_selfpid();
2036 if (inflight && !ut->uu_throttle_bc) {
2037 if (NULL != bap) {
2038 SET(bap->ba_flags, BA_STRATEGY_TRACKED_IO);
2039 }
2040 OSIncrementAtomic(&info->throttle_inflight_count[thread_throttle_level]);
2041 } else {
2042 microuptime(&info->throttle_window_start_timestamp[thread_throttle_level]);
2043 }
2044 KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_THROTTLE, OPEN_THROTTLE_WINDOW)) | DBG_FUNC_NONE,
2045 current_proc()->p_pid, thread_throttle_level, 0, 0, 0);
2046 }
2047 microuptime(&info->throttle_last_IO_timestamp[thread_throttle_level]);
2048 }
2049
2050
2051 if (thread_throttle_level >= THROTTLE_LEVEL_THROTTLED) {
2052 /*
2053 * I'd really like to do the IOSleep here, but
2054 * we may be holding all kinds of filesystem related locks
2055 * and the pages for this I/O marked 'busy'...
2056 * we don't want to cause a normal task to block on
2057 * one of these locks while we're throttling a task marked
2058 * for low priority I/O... we'll mark the uthread and
2059 * do the delay just before we return from the system
2060 * call that triggered this I/O or from vnode_pagein
2061 */
2062 OSAddAtomic(1, &info->throttle_io_count);
2063
2064 throttle_info_set_initial_window(ut, info, FALSE, isssd);
2065 }
2066
2067 return thread_throttle_level;
2068 }
2069
2070 void *throttle_info_update_by_mount(mount_t mp)
2071 {
2072 struct _throttle_io_info_t *info;
2073 uthread_t ut;
2074 boolean_t isssd = FALSE;
2075
2076 ut = get_bsdthread_info(current_thread());
2077
2078 if (mp != NULL) {
2079 if (disk_conditioner_mount_is_ssd(mp))
2080 isssd = TRUE;
2081 info = &_throttle_io_info[mp->mnt_devbsdunit];
2082 } else
2083 info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
2084
2085 if (!ut->uu_lowpri_window)
2086 throttle_info_set_initial_window(ut, info, FALSE, isssd);
2087
2088 return info;
2089 }
2090
2091
2092 /*
2093 * KPI routine
2094 *
2095 * this is usually called before every I/O, used for throttled I/O
2096 * book keeping. This routine has low overhead and does not sleep
2097 */
2098 void throttle_info_update(void *throttle_info, int flags)
2099 {
2100 if (throttle_info)
2101 throttle_info_update_internal(throttle_info, NULL, flags, FALSE, FALSE, NULL);
2102 }
2103
2104 /*
2105 * KPI routine
2106 *
2107 * this is usually called before every I/O, used for throttled I/O
2108 * book keeping. This routine has low overhead and does not sleep
2109 */
2110 void throttle_info_update_by_mask(void *throttle_info_handle, int flags)
2111 {
2112 void *throttle_info = throttle_info_handle;
2113
2114 /*
2115 * for now we only use the lowest bit of the throttle mask, so the
2116 * handle is the same as the throttle_info. Later if we store a
2117 * set of throttle infos in the handle, we will want to loop through
2118 * them and call throttle_info_update in a loop
2119 */
2120 throttle_info_update(throttle_info, flags);
2121 }
2122 /*
2123 * KPI routine
2124 *
2125 * This routine marks the throttle info as disabled. Used for mount points which
2126 * support I/O scheduling.
2127 */
2128
2129 void throttle_info_disable_throttle(int devno, boolean_t isfusion)
2130 {
2131 struct _throttle_io_info_t *info;
2132
2133 if (devno < 0 || devno >= LOWPRI_MAX_NUM_DEV)
2134 panic("Illegal devno (%d) passed into throttle_info_disable_throttle()", devno);
2135
2136 info = &_throttle_io_info[devno];
2137 // don't disable software throttling on devices that are part of a fusion device
2138 // and override the software throttle periods to use HDD periods
2139 if (isfusion) {
2140 info->throttle_is_fusion_with_priority = isfusion;
2141 throttle_init_throttle_period(info, FALSE);
2142 }
2143 info->throttle_disabled = !info->throttle_is_fusion_with_priority;
2144 return;
2145 }
2146
2147
2148 /*
2149 * KPI routine (private)
2150 * Called to determine if this IO is being throttled to this level so that it can be treated specially
2151 */
2152 int throttle_info_io_will_be_throttled(void * throttle_info, int policy)
2153 {
2154 struct _throttle_io_info_t *info = throttle_info;
2155 struct timeval elapsed;
2156 uint64_t elapsed_msecs;
2157 int throttle_level;
2158 int thread_throttle_level;
2159
2160 switch (policy) {
2161
2162 case IOPOL_THROTTLE:
2163 thread_throttle_level = THROTTLE_LEVEL_TIER3;
2164 break;
2165 case IOPOL_UTILITY:
2166 thread_throttle_level = THROTTLE_LEVEL_TIER2;
2167 break;
2168 case IOPOL_STANDARD:
2169 thread_throttle_level = THROTTLE_LEVEL_TIER1;
2170 break;
2171 default:
2172 thread_throttle_level = THROTTLE_LEVEL_TIER0;
2173 break;
2174 }
2175 for (throttle_level = THROTTLE_LEVEL_START; throttle_level < thread_throttle_level; throttle_level++) {
2176 if (info->throttle_inflight_count[throttle_level]) {
2177 break;
2178 }
2179
2180 microuptime(&elapsed);
2181 timevalsub(&elapsed, &info->throttle_window_start_timestamp[throttle_level]);
2182 elapsed_msecs = (uint64_t)elapsed.tv_sec * (uint64_t)1000 + (elapsed.tv_usec / 1000);
2183
2184 if (elapsed_msecs < (uint64_t)throttle_windows_msecs[thread_throttle_level])
2185 break;
2186 }
2187 if (throttle_level >= thread_throttle_level) {
2188 /*
2189 * we're beyond all of the throttle windows
2190 * so go ahead and treat as normal I/O
2191 */
2192 return (THROTTLE_DISENGAGED);
2193 }
2194 /*
2195 * we're in the throttle window
2196 */
2197 return (THROTTLE_ENGAGED);
2198 }
2199
2200 int throttle_lowpri_window(void)
2201 {
2202 struct uthread *ut = get_bsdthread_info(current_thread());
2203 return ut->uu_lowpri_window;
2204 }
2205
2206
2207 #if CONFIG_IOSCHED
2208 int upl_get_cached_tier(void *);
2209 #endif
2210
2211 int
2212 spec_strategy(struct vnop_strategy_args *ap)
2213 {
2214 buf_t bp;
2215 int bflags;
2216 int io_tier;
2217 int passive;
2218 dev_t bdev;
2219 uthread_t ut;
2220 mount_t mp;
2221 struct bufattr *bap;
2222 int strategy_ret;
2223 struct _throttle_io_info_t *throttle_info;
2224 boolean_t isssd = FALSE;
2225 boolean_t inflight = FALSE;
2226 boolean_t upgrade = FALSE;
2227 int code = 0;
2228
2229 #if !CONFIG_EMBEDDED
2230 proc_t curproc = current_proc();
2231 #endif /* !CONFIG_EMBEDDED */
2232
2233 bp = ap->a_bp;
2234 bdev = buf_device(bp);
2235 mp = buf_vnode(bp)->v_mount;
2236 bap = &bp->b_attr;
2237
2238 #if CONFIG_IOSCHED
2239 if (bp->b_flags & B_CLUSTER) {
2240
2241 io_tier = upl_get_cached_tier(bp->b_upl);
2242
2243 if (io_tier == -1)
2244 io_tier = throttle_get_io_policy(&ut);
2245 #if DEVELOPMENT || DEBUG
2246 else {
2247 int my_io_tier = throttle_get_io_policy(&ut);
2248
2249 if (io_tier != my_io_tier)
2250 KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_THROTTLE, IO_TIER_UPL_MISMATCH)) | DBG_FUNC_NONE, buf_kernel_addrperm_addr(bp), my_io_tier, io_tier, 0, 0);
2251 }
2252 #endif
2253 } else
2254 io_tier = throttle_get_io_policy(&ut);
2255 #else
2256 io_tier = throttle_get_io_policy(&ut);
2257 #endif
2258 passive = throttle_get_passive_io_policy(&ut);
2259
2260 /*
2261 * Mark if the I/O was upgraded by throttle_get_thread_throttle_level
2262 * while preserving the original issued tier (throttle_get_io_policy
2263 * does not return upgraded tiers)
2264 */
2265 if (mp && io_tier > throttle_get_thread_throttle_level_internal(ut, io_tier)) {
2266 #if CONFIG_IOSCHED
2267 if (!(mp->mnt_ioflags & MNT_IOFLAGS_IOSCHED_SUPPORTED)) {
2268 upgrade = TRUE;
2269 }
2270 #else /* CONFIG_IOSCHED */
2271 upgrade = TRUE;
2272 #endif /* CONFIG_IOSCHED */
2273 }
2274
2275 if (bp->b_flags & B_META)
2276 bap->ba_flags |= BA_META;
2277
2278 #if CONFIG_IOSCHED
2279 /*
2280 * For I/O Scheduling, we currently do not have a way to track and expedite metadata I/Os.
2281 * To ensure we dont get into priority inversions due to metadata I/Os, we use the following rules:
2282 * For metadata reads, ceil all I/Os to IOSCHED_METADATA_TIER & mark them passive if the I/O tier was upgraded
2283 * For metadata writes, unconditionally mark them as IOSCHED_METADATA_TIER and passive
2284 */
2285 if (bap->ba_flags & BA_META) {
2286 if (mp && (mp->mnt_ioflags & MNT_IOFLAGS_IOSCHED_SUPPORTED)) {
2287 if (bp->b_flags & B_READ) {
2288 if (io_tier > IOSCHED_METADATA_TIER) {
2289 io_tier = IOSCHED_METADATA_TIER;
2290 passive = 1;
2291 }
2292 } else {
2293 io_tier = IOSCHED_METADATA_TIER;
2294 passive = 1;
2295 }
2296 }
2297 }
2298 #endif /* CONFIG_IOSCHED */
2299
2300 SET_BUFATTR_IO_TIER(bap, io_tier);
2301
2302 if (passive) {
2303 bp->b_flags |= B_PASSIVE;
2304 bap->ba_flags |= BA_PASSIVE;
2305 }
2306
2307 #if !CONFIG_EMBEDDED
2308 if ((curproc != NULL) && ((curproc->p_flag & P_DELAYIDLESLEEP) == P_DELAYIDLESLEEP))
2309 bap->ba_flags |= BA_DELAYIDLESLEEP;
2310 #endif /* !CONFIG_EMBEDDED */
2311
2312 bflags = bp->b_flags;
2313
2314 if (((bflags & B_READ) == 0) && ((bflags & B_ASYNC) == 0))
2315 bufattr_markquickcomplete(bap);
2316
2317 if (bflags & B_READ)
2318 code |= DKIO_READ;
2319 if (bflags & B_ASYNC)
2320 code |= DKIO_ASYNC;
2321
2322 if (bap->ba_flags & BA_META)
2323 code |= DKIO_META;
2324 else if (bflags & B_PAGEIO)
2325 code |= DKIO_PAGING;
2326
2327 if (io_tier != 0)
2328 code |= DKIO_THROTTLE;
2329
2330 code |= ((io_tier << DKIO_TIER_SHIFT) & DKIO_TIER_MASK);
2331
2332 if (bflags & B_PASSIVE)
2333 code |= DKIO_PASSIVE;
2334
2335 if (bap->ba_flags & BA_NOCACHE)
2336 code |= DKIO_NOCACHE;
2337
2338 if (upgrade) {
2339 code |= DKIO_TIER_UPGRADE;
2340 SET(bap->ba_flags, BA_IO_TIER_UPGRADE);
2341 }
2342
2343 if (kdebug_enable) {
2344 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_COMMON, FSDBG_CODE(DBG_DKRW, code) | DBG_FUNC_NONE,
2345 buf_kernel_addrperm_addr(bp), bdev, (int)buf_blkno(bp), buf_count(bp), 0);
2346 }
2347
2348 thread_update_io_stats(current_thread(), buf_count(bp), code);
2349
2350 if (mp != NULL) {
2351 if (disk_conditioner_mount_is_ssd(mp))
2352 isssd = TRUE;
2353 /*
2354 * Partially initialized mounts don't have a final devbsdunit and should not be tracked.
2355 * Verify that devbsdunit is initialized (non-zero) or that 0 is the correct initialized value
2356 * (mnt_throttle_mask is initialized and num_trailing_0 would be 0)
2357 */
2358 if (mp->mnt_devbsdunit || (mp->mnt_throttle_mask != LOWPRI_MAX_NUM_DEV - 1 && mp->mnt_throttle_mask & 0x1)) {
2359 inflight = TRUE;
2360 }
2361 throttle_info = &_throttle_io_info[mp->mnt_devbsdunit];
2362
2363 } else
2364 throttle_info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
2365
2366 throttle_info_update_internal(throttle_info, ut, bflags, isssd, inflight, bap);
2367
2368 if ((bflags & B_READ) == 0) {
2369 microuptime(&throttle_info->throttle_last_write_timestamp);
2370
2371 if (mp) {
2372 mp->mnt_last_write_issued_timestamp = throttle_info->throttle_last_write_timestamp;
2373 INCR_PENDING_IO(buf_count(bp), mp->mnt_pending_write_size);
2374 }
2375 } else if (mp) {
2376 INCR_PENDING_IO(buf_count(bp), mp->mnt_pending_read_size);
2377 }
2378 /*
2379 * The BootCache may give us special information about
2380 * the IO, so it returns special values that we check
2381 * for here.
2382 *
2383 * IO_SATISFIED_BY_CACHE
2384 * The read has been satisfied by the boot cache. Don't
2385 * throttle the thread unnecessarily.
2386 *
2387 * IO_SHOULD_BE_THROTTLED
2388 * The boot cache is playing back a playlist and this IO
2389 * cut through. Throttle it so we're not cutting through
2390 * the boot cache too often.
2391 *
2392 * Note that typical strategy routines are defined with
2393 * a void return so we'll get garbage here. In the
2394 * unlikely case the garbage matches our special return
2395 * value, it's not a big deal since we're only adjusting
2396 * the throttling delay.
2397 */
2398 #define IO_SATISFIED_BY_CACHE ((int)0xcafefeed)
2399 #define IO_SHOULD_BE_THROTTLED ((int)0xcafebeef)
2400 typedef int strategy_fcn_ret_t(struct buf *bp);
2401
2402 strategy_ret = (*(strategy_fcn_ret_t*)bdevsw[major(bdev)].d_strategy)(bp);
2403
2404 // disk conditioner needs to track when this I/O actually starts
2405 // which means track it after `strategy` which may include delays
2406 // from inflight I/Os
2407 microuptime(&bp->b_timestamp_tv);
2408
2409 if (IO_SATISFIED_BY_CACHE == strategy_ret) {
2410 /*
2411 * If this was a throttled IO satisfied by the boot cache,
2412 * don't delay the thread.
2413 */
2414 throttle_info_reset_window(ut);
2415
2416 } else if (IO_SHOULD_BE_THROTTLED == strategy_ret) {
2417 /*
2418 * If the boot cache indicates this IO should be throttled,
2419 * delay the thread.
2420 */
2421 throttle_info_set_initial_window(ut, throttle_info, TRUE, isssd);
2422 }
2423 return (0);
2424 }
2425
2426
2427 /*
2428 * This is a noop, simply returning what one has been given.
2429 */
2430 int
2431 spec_blockmap(__unused struct vnop_blockmap_args *ap)
2432 {
2433 return (ENOTSUP);
2434 }
2435
2436
2437 /*
2438 * Device close routine
2439 */
2440 int
2441 spec_close(struct vnop_close_args *ap)
2442 {
2443 struct vnode *vp = ap->a_vp;
2444 dev_t dev = vp->v_rdev;
2445 int error = 0;
2446 int flags = ap->a_fflag;
2447 struct proc *p = vfs_context_proc(ap->a_context);
2448 struct session *sessp;
2449
2450 switch (vp->v_type) {
2451
2452 case VCHR:
2453 /*
2454 * Hack: a tty device that is a controlling terminal
2455 * has a reference from the session structure.
2456 * We cannot easily tell that a character device is
2457 * a controlling terminal, unless it is the closing
2458 * process' controlling terminal. In that case,
2459 * if the reference count is 1 (this is the very
2460 * last close)
2461 */
2462 sessp = proc_session(p);
2463 devsw_lock(dev, S_IFCHR);
2464 if (sessp != SESSION_NULL) {
2465 if (vp == sessp->s_ttyvp && vcount(vp) == 1) {
2466 struct tty *tp = TTY_NULL;
2467
2468 devsw_unlock(dev, S_IFCHR);
2469 session_lock(sessp);
2470 if (vp == sessp->s_ttyvp) {
2471 tp = SESSION_TP(sessp);
2472 sessp->s_ttyvp = NULL;
2473 sessp->s_ttyvid = 0;
2474 sessp->s_ttyp = TTY_NULL;
2475 sessp->s_ttypgrpid = NO_PID;
2476 }
2477 session_unlock(sessp);
2478
2479 if (tp != TTY_NULL) {
2480 /*
2481 * We may have won a race with a proc_exit
2482 * of the session leader, the winner
2483 * clears the flag (even if not set)
2484 */
2485 tty_lock(tp);
2486 ttyclrpgrphup(tp);
2487 tty_unlock(tp);
2488
2489 ttyfree(tp);
2490 }
2491 devsw_lock(dev, S_IFCHR);
2492 }
2493 session_rele(sessp);
2494 }
2495
2496 if (--vp->v_specinfo->si_opencount < 0)
2497 panic("negative open count (c, %u, %u)", major(dev), minor(dev));
2498
2499 /*
2500 * close on last reference or on vnode revoke call
2501 */
2502 if (vcount(vp) == 0 || (flags & IO_REVOKE) != 0)
2503 error = cdevsw[major(dev)].d_close(dev, flags, S_IFCHR, p);
2504
2505 devsw_unlock(dev, S_IFCHR);
2506 break;
2507
2508 case VBLK:
2509 /*
2510 * If there is more than one outstanding open, don't
2511 * send the close to the device.
2512 */
2513 devsw_lock(dev, S_IFBLK);
2514 if (vcount(vp) > 1) {
2515 vp->v_specinfo->si_opencount--;
2516 devsw_unlock(dev, S_IFBLK);
2517 return (0);
2518 }
2519 devsw_unlock(dev, S_IFBLK);
2520
2521 /*
2522 * On last close of a block device (that isn't mounted)
2523 * we must invalidate any in core blocks, so that
2524 * we can, for instance, change floppy disks.
2525 */
2526 if ((error = spec_fsync_internal(vp, MNT_WAIT, ap->a_context)))
2527 return (error);
2528
2529 error = buf_invalidateblks(vp, BUF_WRITE_DATA, 0, 0);
2530 if (error)
2531 return (error);
2532
2533 devsw_lock(dev, S_IFBLK);
2534
2535 if (--vp->v_specinfo->si_opencount < 0)
2536 panic("negative open count (b, %u, %u)", major(dev), minor(dev));
2537
2538 if (vcount(vp) == 0)
2539 error = bdevsw[major(dev)].d_close(dev, flags, S_IFBLK, p);
2540
2541 devsw_unlock(dev, S_IFBLK);
2542 break;
2543
2544 default:
2545 panic("spec_close: not special");
2546 return(EBADF);
2547 }
2548
2549 return error;
2550 }
2551
2552 /*
2553 * Return POSIX pathconf information applicable to special devices.
2554 */
2555 int
2556 spec_pathconf(struct vnop_pathconf_args *ap)
2557 {
2558
2559 switch (ap->a_name) {
2560 case _PC_LINK_MAX:
2561 *ap->a_retval = LINK_MAX;
2562 return (0);
2563 case _PC_MAX_CANON:
2564 *ap->a_retval = MAX_CANON;
2565 return (0);
2566 case _PC_MAX_INPUT:
2567 *ap->a_retval = MAX_INPUT;
2568 return (0);
2569 case _PC_PIPE_BUF:
2570 *ap->a_retval = PIPE_BUF;
2571 return (0);
2572 case _PC_CHOWN_RESTRICTED:
2573 *ap->a_retval = 200112; /* _POSIX_CHOWN_RESTRICTED */
2574 return (0);
2575 case _PC_VDISABLE:
2576 *ap->a_retval = _POSIX_VDISABLE;
2577 return (0);
2578 default:
2579 return (EINVAL);
2580 }
2581 /* NOTREACHED */
2582 }
2583
2584 /*
2585 * Special device failed operation
2586 */
2587 int
2588 spec_ebadf(__unused void *dummy)
2589 {
2590
2591 return (EBADF);
2592 }
2593
2594 /* Blktooff derives file offset from logical block number */
2595 int
2596 spec_blktooff(struct vnop_blktooff_args *ap)
2597 {
2598 struct vnode *vp = ap->a_vp;
2599
2600 switch (vp->v_type) {
2601 case VCHR:
2602 *ap->a_offset = (off_t)-1; /* failure */
2603 return (ENOTSUP);
2604
2605 case VBLK:
2606 printf("spec_blktooff: not implemented for VBLK\n");
2607 *ap->a_offset = (off_t)-1; /* failure */
2608 return (ENOTSUP);
2609
2610 default:
2611 panic("spec_blktooff type");
2612 }
2613 /* NOTREACHED */
2614
2615 return (0);
2616 }
2617
2618 /* Offtoblk derives logical block number from file offset */
2619 int
2620 spec_offtoblk(struct vnop_offtoblk_args *ap)
2621 {
2622 struct vnode *vp = ap->a_vp;
2623
2624 switch (vp->v_type) {
2625 case VCHR:
2626 *ap->a_lblkno = (daddr64_t)-1; /* failure */
2627 return (ENOTSUP);
2628
2629 case VBLK:
2630 printf("spec_offtoblk: not implemented for VBLK\n");
2631 *ap->a_lblkno = (daddr64_t)-1; /* failure */
2632 return (ENOTSUP);
2633
2634 default:
2635 panic("spec_offtoblk type");
2636 }
2637 /* NOTREACHED */
2638
2639 return (0);
2640 }
2641
2642 static void filt_specdetach(struct knote *kn);
2643 static int filt_specevent(struct knote *kn, long hint);
2644 static int filt_spectouch(struct knote *kn, struct kevent_internal_s *kev);
2645 static int filt_specprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev);
2646 static unsigned filt_specpeek(struct knote *kn);
2647
2648 SECURITY_READ_ONLY_EARLY(struct filterops) spec_filtops = {
2649 .f_isfd = 1,
2650 .f_attach = filt_specattach,
2651 .f_detach = filt_specdetach,
2652 .f_event = filt_specevent,
2653 .f_touch = filt_spectouch,
2654 .f_process = filt_specprocess,
2655 .f_peek = filt_specpeek
2656 };
2657
2658
2659 /*
2660 * Given a waitq that is assumed to be embedded within a selinfo structure,
2661 * return the containing selinfo structure. While 'wq' is not really a queue
2662 * element, this macro simply does the offset_of calculation to get back to a
2663 * containing struct given the struct type and member name.
2664 */
2665 #define selinfo_from_waitq(wq) \
2666 qe_element((wq), struct selinfo, si_waitq)
2667
2668 static int
2669 spec_knote_select_and_link(struct knote *kn)
2670 {
2671 uthread_t uth;
2672 vfs_context_t ctx;
2673 vnode_t vp;
2674 struct waitq_set *old_wqs;
2675 uint64_t rsvd, rsvd_arg;
2676 uint64_t *rlptr = NULL;
2677 struct selinfo *si = NULL;
2678 int selres = 0;
2679
2680 uth = get_bsdthread_info(current_thread());
2681
2682 ctx = vfs_context_current();
2683 vp = (vnode_t)kn->kn_fp->f_fglob->fg_data;
2684
2685 int error = vnode_getwithvid(vp, kn->kn_hookid);
2686 if (error != 0) {
2687 knote_set_error(kn, ENOENT);
2688 return 0;
2689 }
2690
2691 /*
2692 * This function may be called many times to link or re-link the
2693 * underlying vnode to the kqueue. If we've already linked the two,
2694 * we will have a valid kn_hook_data which ties us to the underlying
2695 * device's waitq via a the waitq's prepost table object. However,
2696 * devices can abort any select action by calling selthreadclear().
2697 * This is OK because the table object will be invalidated by the
2698 * driver (through a call to selthreadclear), so any attempt to access
2699 * the associated waitq will fail because the table object is invalid.
2700 *
2701 * Even if we've already registered, we need to pass a pointer
2702 * to a reserved link structure. Otherwise, selrecord() will
2703 * infer that we're in the second pass of select() and won't
2704 * actually do anything!
2705 */
2706 rsvd = rsvd_arg = waitq_link_reserve(NULL);
2707 rlptr = (void *)&rsvd_arg;
2708
2709 /*
2710 * Trick selrecord() into hooking kqueue's wait queue set
2711 * set into device's selinfo wait queue
2712 */
2713 old_wqs = uth->uu_wqset;
2714 uth->uu_wqset = &(knote_get_kq(kn)->kq_wqs);
2715 selres = VNOP_SELECT(vp, knote_get_seltype(kn), 0, rlptr, ctx);
2716 uth->uu_wqset = old_wqs;
2717
2718 /*
2719 * make sure to cleanup the reserved link - this guards against
2720 * drivers that may not actually call selrecord().
2721 */
2722 waitq_link_release(rsvd);
2723 if (rsvd != rsvd_arg) {
2724 /* the driver / handler called selrecord() */
2725 struct waitq *wq;
2726 memcpy(&wq, rlptr, sizeof(void *));
2727
2728 /*
2729 * The waitq is part of the selinfo structure managed by the
2730 * driver. For certain drivers, we want to hook the knote into
2731 * the selinfo structure's si_note field so selwakeup can call
2732 * KNOTE.
2733 */
2734 si = selinfo_from_waitq(wq);
2735
2736 /*
2737 * The waitq_get_prepost_id() function will (potentially)
2738 * allocate a prepost table object for the waitq and return
2739 * the table object's ID to us. It will also set the
2740 * waitq_prepost_id field within the waitq structure.
2741 *
2742 * We can just overwrite kn_hook_data because it's simply a
2743 * table ID used to grab a reference when needed.
2744 *
2745 * We have a reference on the vnode, so we know that the
2746 * device won't go away while we get this ID.
2747 */
2748 kn->kn_hook_data = waitq_get_prepost_id(wq);
2749 } else {
2750 assert(selres != 0);
2751 }
2752
2753 vnode_put(vp);
2754
2755 return selres;
2756 }
2757
2758 static void filt_spec_common(struct knote *kn, int selres)
2759 {
2760 if (kn->kn_vnode_use_ofst) {
2761 if (kn->kn_fp->f_fglob->fg_offset >= (uint32_t)selres) {
2762 kn->kn_data = 0;
2763 } else {
2764 kn->kn_data = ((uint32_t)selres) - kn->kn_fp->f_fglob->fg_offset;
2765 }
2766 } else {
2767 kn->kn_data = selres;
2768 }
2769 }
2770
2771 static int
2772 filt_specattach(struct knote *kn, __unused struct kevent_internal_s *kev)
2773 {
2774 vnode_t vp;
2775 dev_t dev;
2776
2777 vp = (vnode_t)kn->kn_fp->f_fglob->fg_data; /* Already have iocount, and vnode is alive */
2778
2779 assert(vnode_ischr(vp));
2780
2781 dev = vnode_specrdev(vp);
2782
2783 /*
2784 * For a few special kinds of devices, we can attach knotes with
2785 * no restrictions because their "select" vectors return the amount
2786 * of data available. Others require an explicit NOTE_LOWAT with
2787 * data of 1, indicating that the caller doesn't care about actual
2788 * data counts, just an indication that the device has data.
2789 */
2790 if (!kn->kn_vnode_kqok &&
2791 ((kn->kn_sfflags & NOTE_LOWAT) == 0 || kn->kn_sdata != 1)) {
2792 knote_set_error(kn, EINVAL);
2793 return 0;
2794 }
2795
2796 /*
2797 * This forces the select fallback to call through VNOP_SELECT and hook
2798 * up selinfo on every filter routine.
2799 *
2800 * Pseudo-terminal controllers are opted out of native kevent support --
2801 * remove this when they get their own EVFILTID.
2802 */
2803 if (cdevsw_flags[major(dev)] & CDEVSW_IS_PTC) {
2804 kn->kn_vnode_kqok = 0;
2805 }
2806
2807 kn->kn_filtid = EVFILTID_SPEC;
2808 kn->kn_hook_data = 0;
2809 kn->kn_hookid = vnode_vid(vp);
2810
2811 knote_markstayactive(kn);
2812 return spec_knote_select_and_link(kn);
2813 }
2814
2815 static void
2816 filt_specdetach(struct knote *kn)
2817 {
2818 knote_clearstayactive(kn);
2819
2820 /*
2821 * This is potentially tricky: the device's selinfo waitq that was
2822 * tricked into being part of this knote's waitq set may not be a part
2823 * of any other set, and the device itself may have revoked the memory
2824 * in which the waitq was held. We use the knote's kn_hook_data field
2825 * to keep the ID of the waitq's prepost table object. This
2826 * object keeps a pointer back to the waitq, and gives us a safe way
2827 * to decouple the dereferencing of driver allocated memory: if the
2828 * driver goes away (taking the waitq with it) then the prepost table
2829 * object will be invalidated. The waitq details are handled in the
2830 * waitq API invoked here.
2831 */
2832 if (kn->kn_hook_data) {
2833 waitq_unlink_by_prepost_id(kn->kn_hook_data, &(knote_get_kq(kn)->kq_wqs));
2834 kn->kn_hook_data = 0;
2835 }
2836 }
2837
2838 static int
2839 filt_specevent(struct knote *kn, __unused long hint)
2840 {
2841 /*
2842 * Nothing should call knote or knote_vanish on this knote.
2843 */
2844 panic("filt_specevent(%p)", kn);
2845 return 0;
2846 }
2847
2848 static int
2849 filt_spectouch(struct knote *kn, struct kevent_internal_s *kev)
2850 {
2851 kn->kn_sdata = kev->data;
2852 kn->kn_sfflags = kev->fflags;
2853 if ((kn->kn_status & KN_UDATA_SPECIFIC) == 0)
2854 kn->kn_udata = kev->udata;
2855
2856 if (kev->flags & EV_ENABLE) {
2857 return spec_knote_select_and_link(kn);
2858 }
2859
2860 return 0;
2861 }
2862
2863 static int
2864 filt_specprocess(struct knote *kn, struct filt_process_s *data, struct kevent_internal_s *kev)
2865 {
2866 #pragma unused(data)
2867 vnode_t vp;
2868 uthread_t uth;
2869 vfs_context_t ctx;
2870 int res;
2871 int selres;
2872 int error;
2873
2874 uth = get_bsdthread_info(current_thread());
2875 ctx = vfs_context_current();
2876 vp = (vnode_t)kn->kn_fp->f_fglob->fg_data;
2877
2878 /* FIXME JMM - locking against touches? */
2879
2880 error = vnode_getwithvid(vp, kn->kn_hookid);
2881 if (error != 0) {
2882 kn->kn_flags |= (EV_EOF | EV_ONESHOT);
2883 *kev = kn->kn_kevent;
2884 return 1;
2885 }
2886
2887 selres = spec_knote_select_and_link(kn);
2888 filt_spec_common(kn, selres);
2889
2890 vnode_put(vp);
2891
2892 res = ((kn->kn_sfflags & NOTE_LOWAT) != 0) ?
2893 (kn->kn_data >= kn->kn_sdata) : kn->kn_data;
2894
2895 if (res) {
2896 *kev = kn->kn_kevent;
2897 if (kn->kn_flags & EV_CLEAR) {
2898 kn->kn_fflags = 0;
2899 kn->kn_data = 0;
2900 }
2901 }
2902
2903 return res;
2904 }
2905
2906 static unsigned
2907 filt_specpeek(struct knote *kn)
2908 {
2909 int selres = 0;
2910
2911 selres = spec_knote_select_and_link(kn);
2912 filt_spec_common(kn, selres);
2913
2914 return kn->kn_data;
2915 }
2916
mirror of XNU