]> git.saurik.com Git - apple/xnu.git/blob - bsd/miscfs/specfs/spec_vnops.c
projects / apple / xnu.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
xnu-7195.81.3.tar.gz
[apple/xnu.git] / bsd / miscfs / specfs / spec_vnops.c
1 /*
2 * Copyright (c) 2000-2019 Apple Computer, Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
29 /*
30 * Copyright (c) 1989, 1993, 1995
31 * The Regents of the University of California. All rights reserved.
32 *
33 * Redistribution and use in source and binary forms, with or without
34 * modification, are permitted provided that the following conditions
35 * are met:
36 * 1. Redistributions of source code must retain the above copyright
37 * notice, this list of conditions and the following disclaimer.
38 * 2. Redistributions in binary form must reproduce the above copyright
39 * notice, this list of conditions and the following disclaimer in the
40 * documentation and/or other materials provided with the distribution.
41 * 3. All advertising materials mentioning features or use of this software
42 * must display the following acknowledgement:
43 * This product includes software developed by the University of
44 * California, Berkeley and its contributors.
45 * 4. Neither the name of the University nor the names of its contributors
46 * may be used to endorse or promote products derived from this software
47 * without specific prior written permission.
48 *
49 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
50 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
51 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
52 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
53 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
54 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
55 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
56 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
57 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
58 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
59 * SUCH DAMAGE.
60 *
61 * @(#)spec_vnops.c 8.14 (Berkeley) 5/21/95
62 */
63
64 #include <sys/param.h>
65 #include <sys/proc_internal.h>
66 #include <sys/kauth.h>
67 #include <sys/systm.h>
68 #include <sys/kernel.h>
69 #include <sys/conf.h>
70 #include <sys/buf_internal.h>
71 #include <sys/mount_internal.h>
72 #include <sys/vnode_internal.h>
73 #include <sys/file_internal.h>
74 #include <sys/namei.h>
75 #include <sys/stat.h>
76 #include <sys/errno.h>
77 #include <sys/ioctl.h>
78 #include <sys/file.h>
79 #include <sys/user.h>
80 #include <sys/malloc.h>
81 #include <sys/disk.h>
82 #include <sys/uio_internal.h>
83 #include <sys/resource.h>
84 #include <machine/machine_routines.h>
85 #include <miscfs/specfs/specdev.h>
86 #include <vfs/vfs_support.h>
87 #include <vfs/vfs_disk_conditioner.h>
88
89 #include <kern/assert.h>
90 #include <kern/task.h>
91 #include <kern/sched_prim.h>
92 #include <kern/thread.h>
93 #include <kern/policy_internal.h>
94 #include <kern/timer_call.h>
95 #include <kern/waitq.h>
96
97 #include <pexpert/pexpert.h>
98 #include <IOKit/IOBSD.h>
99
100 #include <sys/kdebug.h>
101 #include <libkern/section_keywords.h>
102
103 /* XXX following three prototypes should be in a header file somewhere */
104 extern dev_t chrtoblk(dev_t dev);
105 extern boolean_t iskmemdev(dev_t dev);
106 extern int bpfkqfilter(dev_t dev, struct knote *kn);
107 extern int ptsd_kqfilter(dev_t, struct knote *);
108 extern int ptmx_kqfilter(dev_t, struct knote *);
109 #if CONFIG_PHYS_WRITE_ACCT
110 uint64_t kernel_pm_writes; // to track the sync writes occuring during power management transitions
111 #endif /* CONFIG_PHYS_WRITE_ACCT */
112
113
114 struct vnode *speclisth[SPECHSZ];
115
116 /* symbolic sleep message strings for devices */
117 char devopn[] = "devopn";
118 char devio[] = "devio";
119 char devwait[] = "devwait";
120 char devin[] = "devin";
121 char devout[] = "devout";
122 char devioc[] = "devioc";
123 char devcls[] = "devcls";
124
125 #define VOPFUNC int (*)(void *)
126
127 int(**spec_vnodeop_p)(void *);
128 const struct vnodeopv_entry_desc spec_vnodeop_entries[] = {
129 { .opve_op = &vnop_default_desc, .opve_impl = (VOPFUNC)vn_default_error },
130 { .opve_op = &vnop_lookup_desc, .opve_impl = (VOPFUNC)spec_lookup }, /* lookup */
131 { .opve_op = &vnop_create_desc, .opve_impl = (VOPFUNC)err_create }, /* create */
132 { .opve_op = &vnop_mknod_desc, .opve_impl = (VOPFUNC)err_mknod }, /* mknod */
133 { .opve_op = &vnop_open_desc, .opve_impl = (VOPFUNC)spec_open }, /* open */
134 { .opve_op = &vnop_close_desc, .opve_impl = (VOPFUNC)spec_close }, /* close */
135 { .opve_op = &vnop_access_desc, .opve_impl = (VOPFUNC)spec_access }, /* access */
136 { .opve_op = &vnop_getattr_desc, .opve_impl = (VOPFUNC)spec_getattr }, /* getattr */
137 { .opve_op = &vnop_setattr_desc, .opve_impl = (VOPFUNC)spec_setattr }, /* setattr */
138 { .opve_op = &vnop_read_desc, .opve_impl = (VOPFUNC)spec_read }, /* read */
139 { .opve_op = &vnop_write_desc, .opve_impl = (VOPFUNC)spec_write }, /* write */
140 { .opve_op = &vnop_ioctl_desc, .opve_impl = (VOPFUNC)spec_ioctl }, /* ioctl */
141 { .opve_op = &vnop_select_desc, .opve_impl = (VOPFUNC)spec_select }, /* select */
142 { .opve_op = &vnop_revoke_desc, .opve_impl = (VOPFUNC)nop_revoke }, /* revoke */
143 { .opve_op = &vnop_mmap_desc, .opve_impl = (VOPFUNC)err_mmap }, /* mmap */
144 { .opve_op = &vnop_fsync_desc, .opve_impl = (VOPFUNC)spec_fsync }, /* fsync */
145 { .opve_op = &vnop_remove_desc, .opve_impl = (VOPFUNC)err_remove }, /* remove */
146 { .opve_op = &vnop_link_desc, .opve_impl = (VOPFUNC)err_link }, /* link */
147 { .opve_op = &vnop_rename_desc, .opve_impl = (VOPFUNC)err_rename }, /* rename */
148 { .opve_op = &vnop_mkdir_desc, .opve_impl = (VOPFUNC)err_mkdir }, /* mkdir */
149 { .opve_op = &vnop_rmdir_desc, .opve_impl = (VOPFUNC)err_rmdir }, /* rmdir */
150 { .opve_op = &vnop_symlink_desc, .opve_impl = (VOPFUNC)err_symlink }, /* symlink */
151 { .opve_op = &vnop_readdir_desc, .opve_impl = (VOPFUNC)err_readdir }, /* readdir */
152 { .opve_op = &vnop_readlink_desc, .opve_impl = (VOPFUNC)err_readlink }, /* readlink */
153 { .opve_op = &vnop_inactive_desc, .opve_impl = (VOPFUNC)nop_inactive }, /* inactive */
154 { .opve_op = &vnop_reclaim_desc, .opve_impl = (VOPFUNC)nop_reclaim }, /* reclaim */
155 { .opve_op = &vnop_strategy_desc, .opve_impl = (VOPFUNC)spec_strategy }, /* strategy */
156 { .opve_op = &vnop_pathconf_desc, .opve_impl = (VOPFUNC)spec_pathconf }, /* pathconf */
157 { .opve_op = &vnop_advlock_desc, .opve_impl = (VOPFUNC)err_advlock }, /* advlock */
158 { .opve_op = &vnop_bwrite_desc, .opve_impl = (VOPFUNC)spec_bwrite }, /* bwrite */
159 { .opve_op = &vnop_pagein_desc, .opve_impl = (VOPFUNC)err_pagein }, /* Pagein */
160 { .opve_op = &vnop_pageout_desc, .opve_impl = (VOPFUNC)err_pageout }, /* Pageout */
161 { .opve_op = &vnop_copyfile_desc, .opve_impl = (VOPFUNC)err_copyfile }, /* Copyfile */
162 { .opve_op = &vnop_blktooff_desc, .opve_impl = (VOPFUNC)spec_blktooff }, /* blktooff */
163 { .opve_op = &vnop_offtoblk_desc, .opve_impl = (VOPFUNC)spec_offtoblk }, /* offtoblk */
164 { .opve_op = &vnop_blockmap_desc, .opve_impl = (VOPFUNC)spec_blockmap }, /* blockmap */
165 { .opve_op = (struct vnodeop_desc*)NULL, .opve_impl = (int (*)(void *))NULL }
166 };
167 const struct vnodeopv_desc spec_vnodeop_opv_desc =
168 { .opv_desc_vector_p = &spec_vnodeop_p, .opv_desc_ops = spec_vnodeop_entries };
169
170
171 static void set_blocksize(vnode_t, dev_t);
172
173 #define LOWPRI_TIER1_WINDOW_MSECS 25
174 #define LOWPRI_TIER2_WINDOW_MSECS 100
175 #define LOWPRI_TIER3_WINDOW_MSECS 500
176
177 #define LOWPRI_TIER1_IO_PERIOD_MSECS 40
178 #define LOWPRI_TIER2_IO_PERIOD_MSECS 85
179 #define LOWPRI_TIER3_IO_PERIOD_MSECS 200
180
181 #define LOWPRI_TIER1_IO_PERIOD_SSD_MSECS 5
182 #define LOWPRI_TIER2_IO_PERIOD_SSD_MSECS 15
183 #define LOWPRI_TIER3_IO_PERIOD_SSD_MSECS 25
184
185
186 int throttle_windows_msecs[THROTTLE_LEVEL_END + 1] = {
187 0,
188 LOWPRI_TIER1_WINDOW_MSECS,
189 LOWPRI_TIER2_WINDOW_MSECS,
190 LOWPRI_TIER3_WINDOW_MSECS,
191 };
192
193 int throttle_io_period_msecs[THROTTLE_LEVEL_END + 1] = {
194 0,
195 LOWPRI_TIER1_IO_PERIOD_MSECS,
196 LOWPRI_TIER2_IO_PERIOD_MSECS,
197 LOWPRI_TIER3_IO_PERIOD_MSECS,
198 };
199
200 int throttle_io_period_ssd_msecs[THROTTLE_LEVEL_END + 1] = {
201 0,
202 LOWPRI_TIER1_IO_PERIOD_SSD_MSECS,
203 LOWPRI_TIER2_IO_PERIOD_SSD_MSECS,
204 LOWPRI_TIER3_IO_PERIOD_SSD_MSECS,
205 };
206
207
208 int throttled_count[THROTTLE_LEVEL_END + 1];
209
210 struct _throttle_io_info_t {
211 lck_mtx_t throttle_lock;
212
213 struct timeval throttle_last_write_timestamp;
214 struct timeval throttle_min_timer_deadline;
215 struct timeval throttle_window_start_timestamp[THROTTLE_LEVEL_END + 1]; /* window starts at both the beginning and completion of an I/O */
216 struct timeval throttle_last_IO_timestamp[THROTTLE_LEVEL_END + 1];
217 pid_t throttle_last_IO_pid[THROTTLE_LEVEL_END + 1];
218 struct timeval throttle_start_IO_period_timestamp[THROTTLE_LEVEL_END + 1];
219 int32_t throttle_inflight_count[THROTTLE_LEVEL_END + 1];
220
221 TAILQ_HEAD(, uthread) throttle_uthlist[THROTTLE_LEVEL_END + 1]; /* Lists of throttled uthreads */
222 int throttle_next_wake_level;
223
224 thread_call_t throttle_timer_call;
225 int32_t throttle_timer_ref;
226 int32_t throttle_timer_active;
227
228 int32_t throttle_io_count;
229 int32_t throttle_io_count_begin;
230 int *throttle_io_periods;
231 uint32_t throttle_io_period_num;
232
233 int32_t throttle_refcnt;
234 int32_t throttle_alloc;
235 int32_t throttle_disabled;
236 int32_t throttle_is_fusion_with_priority;
237 };
238
239 struct _throttle_io_info_t _throttle_io_info[LOWPRI_MAX_NUM_DEV];
240
241
242 int lowpri_throttle_enabled = 1;
243
244
245 static void throttle_info_end_io_internal(struct _throttle_io_info_t *info, int throttle_level);
246 static int throttle_info_update_internal(struct _throttle_io_info_t *info, uthread_t ut, int flags, boolean_t isssd, boolean_t inflight, struct bufattr *bap);
247 static int throttle_get_thread_throttle_level(uthread_t ut);
248 static int throttle_get_thread_throttle_level_internal(uthread_t ut, int io_tier);
249 void throttle_info_mount_reset_period(mount_t mp, int isssd);
250
251 /*
252 * Trivial lookup routine that always fails.
253 */
254 int
255 spec_lookup(struct vnop_lookup_args *ap)
256 {
257 *ap->a_vpp = NULL;
258 return ENOTDIR;
259 }
260
261 static void
262 set_blocksize(struct vnode *vp, dev_t dev)
263 {
264 int (*size)(dev_t);
265 int rsize;
266
267 if ((major(dev) < nblkdev) && (size = bdevsw[major(dev)].d_psize)) {
268 rsize = (*size)(dev);
269 if (rsize <= 0) { /* did size fail? */
270 vp->v_specsize = DEV_BSIZE;
271 } else {
272 vp->v_specsize = rsize;
273 }
274 } else {
275 vp->v_specsize = DEV_BSIZE;
276 }
277 }
278
279 void
280 set_fsblocksize(struct vnode *vp)
281 {
282 if (vp->v_type == VBLK) {
283 dev_t dev = (dev_t)vp->v_rdev;
284 int maj = major(dev);
285
286 if ((u_int)maj >= (u_int)nblkdev) {
287 return;
288 }
289
290 vnode_lock(vp);
291 set_blocksize(vp, dev);
292 vnode_unlock(vp);
293 }
294 }
295
296
297 /*
298 * Open a special file.
299 */
300 int
301 spec_open(struct vnop_open_args *ap)
302 {
303 static const char *OPEN_MOUNTED_ENTITLEMENT = "com.apple.private.vfs.open-mounted";
304
305 struct proc *p = vfs_context_proc(ap->a_context);
306 kauth_cred_t cred = vfs_context_ucred(ap->a_context);
307 struct vnode *vp = ap->a_vp;
308 dev_t bdev, dev = (dev_t)vp->v_rdev;
309 int maj = major(dev);
310 int error;
311
312 /*
313 * Don't allow open if fs is mounted -nodev.
314 */
315 if (vp->v_mount && (vp->v_mount->mnt_flag & MNT_NODEV)) {
316 return ENXIO;
317 }
318
319 switch (vp->v_type) {
320 case VCHR:
321 if ((u_int)maj >= (u_int)nchrdev) {
322 return ENXIO;
323 }
324 if (cred != FSCRED && (ap->a_mode & FWRITE)) {
325 #if 0
326 /*
327 * When running in very secure mode, do not allow
328 * opens for writing of any disk character devices.
329 */
330 if (securelevel >= 2 && isdisk(dev, VCHR)) {
331 return EPERM;
332 }
333 #endif
334
335 /* Never allow writing to /dev/mem or /dev/kmem */
336 if (iskmemdev(dev)) {
337 return EPERM;
338 }
339 /*
340 * When running in secure mode, do not allow opens for
341 * writing of character devices whose corresponding block
342 * devices are currently mounted.
343 */
344 if (securelevel >= 1) {
345 if ((bdev = chrtoblk(dev)) != NODEV && check_mountedon(bdev, VBLK, &error)) {
346 return error;
347 }
348 }
349 }
350
351 devsw_lock(dev, S_IFCHR);
352 error = (*cdevsw[maj].d_open)(dev, ap->a_mode, S_IFCHR, p);
353
354 if (error == 0) {
355 vp->v_specinfo->si_opencount++;
356 }
357
358 devsw_unlock(dev, S_IFCHR);
359
360 if (error == 0 && cdevsw[maj].d_type == D_DISK && !vp->v_un.vu_specinfo->si_initted) {
361 int isssd = 0;
362 uint64_t throttle_mask = 0;
363 uint32_t devbsdunit = 0;
364
365 if (VNOP_IOCTL(vp, DKIOCGETTHROTTLEMASK, (caddr_t)&throttle_mask, 0, NULL) == 0) {
366 if (throttle_mask != 0 &&
367 VNOP_IOCTL(vp, DKIOCISSOLIDSTATE, (caddr_t)&isssd, 0, ap->a_context) == 0) {
368 /*
369 * as a reasonable approximation, only use the lowest bit of the mask
370 * to generate a disk unit number
371 */
372 devbsdunit = num_trailing_0(throttle_mask);
373
374 vnode_lock(vp);
375
376 vp->v_un.vu_specinfo->si_isssd = isssd ? 1 : 0;
377 vp->v_un.vu_specinfo->si_devbsdunit = devbsdunit;
378 vp->v_un.vu_specinfo->si_throttle_mask = throttle_mask;
379 vp->v_un.vu_specinfo->si_throttleable = 1;
380 vp->v_un.vu_specinfo->si_initted = 1;
381
382 vnode_unlock(vp);
383 }
384 }
385 if (vp->v_un.vu_specinfo->si_initted == 0) {
386 vnode_lock(vp);
387 vp->v_un.vu_specinfo->si_initted = 1;
388 vnode_unlock(vp);
389 }
390 }
391 return error;
392
393 case VBLK:
394 if ((u_int)maj >= (u_int)nblkdev) {
395 return ENXIO;
396 }
397 /*
398 * When running in very secure mode, do not allow
399 * opens for writing of any disk block devices.
400 */
401 if (securelevel >= 2 && cred != FSCRED &&
402 (ap->a_mode & FWRITE) && bdevsw[maj].d_type == D_DISK) {
403 return EPERM;
404 }
405 /*
406 * Do not allow opens of block devices that are
407 * currently mounted.
408 */
409 if (!IOTaskHasEntitlement(current_task(), OPEN_MOUNTED_ENTITLEMENT)) {
410 if ((error = vfs_mountedon(vp))) {
411 return error;
412 }
413 }
414
415 devsw_lock(dev, S_IFBLK);
416 error = (*bdevsw[maj].d_open)(dev, ap->a_mode, S_IFBLK, p);
417 if (!error) {
418 vp->v_specinfo->si_opencount++;
419 }
420 devsw_unlock(dev, S_IFBLK);
421
422 if (!error) {
423 u_int64_t blkcnt;
424 u_int32_t blksize;
425 int setsize = 0;
426 u_int32_t size512 = 512;
427
428
429 if (!VNOP_IOCTL(vp, DKIOCGETBLOCKSIZE, (caddr_t)&blksize, 0, ap->a_context)) {
430 /* Switch to 512 byte sectors (temporarily) */
431
432 if (!VNOP_IOCTL(vp, DKIOCSETBLOCKSIZE, (caddr_t)&size512, FWRITE, ap->a_context)) {
433 /* Get the number of 512 byte physical blocks. */
434 if (!VNOP_IOCTL(vp, DKIOCGETBLOCKCOUNT, (caddr_t)&blkcnt, 0, ap->a_context)) {
435 setsize = 1;
436 }
437 }
438 /* If it doesn't set back, we can't recover */
439 if (VNOP_IOCTL(vp, DKIOCSETBLOCKSIZE, (caddr_t)&blksize, FWRITE, ap->a_context)) {
440 error = ENXIO;
441 }
442 }
443
444
445 vnode_lock(vp);
446 set_blocksize(vp, dev);
447
448 /*
449 * Cache the size in bytes of the block device for later
450 * use by spec_write().
451 */
452 if (setsize) {
453 vp->v_specdevsize = blkcnt * (u_int64_t)size512;
454 } else {
455 vp->v_specdevsize = (u_int64_t)0; /* Default: Can't get */
456 }
457 vnode_unlock(vp);
458 }
459 return error;
460 default:
461 panic("spec_open type");
462 }
463 return 0;
464 }
465
466 /*
467 * Vnode op for read
468 */
469 int
470 spec_read(struct vnop_read_args *ap)
471 {
472 struct vnode *vp = ap->a_vp;
473 struct uio *uio = ap->a_uio;
474 struct buf *bp;
475 daddr64_t bn, nextbn;
476 long bscale;
477 int devBlockSize = 0;
478 size_t bsize, n, on;
479 int error = 0;
480 dev_t dev;
481
482 #if DIAGNOSTIC
483 if (uio->uio_rw != UIO_READ) {
484 panic("spec_read mode");
485 }
486 if (UIO_SEG_IS_USER_SPACE(uio->uio_segflg)) {
487 panic("spec_read proc");
488 }
489 #endif
490 if (uio_resid(uio) == 0) {
491 return 0;
492 }
493
494 switch (vp->v_type) {
495 case VCHR:
496 {
497 struct _throttle_io_info_t *throttle_info = NULL;
498 int thread_throttle_level;
499 uint64_t blkno = 0;
500 uint32_t iolen = 0;
501 int ddisk = 0;
502 int ktrace_code = DKIO_READ;
503 devBlockSize = vp->v_specsize;
504 uintptr_t our_id;
505
506 if (cdevsw[major(vp->v_rdev)].d_type == D_DISK) {
507 ddisk = 1;
508 }
509
510 if (ddisk && vp->v_un.vu_specinfo->si_throttleable) {
511 throttle_info = &_throttle_io_info[vp->v_un.vu_specinfo->si_devbsdunit];
512 thread_throttle_level = throttle_info_update_internal(throttle_info, NULL, 0, vp->v_un.vu_specinfo->si_isssd, TRUE, NULL);
513 }
514
515 if (kdebug_enable && ddisk) {
516 if (devBlockSize == 0) {
517 devBlockSize = 512; // default sector size
518 }
519
520 if (uio_offset(uio) && devBlockSize) {
521 blkno = ((uint64_t) uio_offset(uio) / ((uint64_t)devBlockSize));
522 }
523 iolen = (int) uio_resid(uio);
524 our_id = (uintptr_t)thread_tid(current_thread());
525 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_COMMON,
526 (FSDBG_CODE(DBG_DKRW, ktrace_code)) | DBG_FUNC_NONE, our_id,
527 vp->v_rdev, blkno, iolen, 0);
528 }
529
530 error = (*cdevsw[major(vp->v_rdev)].d_read)
531 (vp->v_rdev, uio, ap->a_ioflag);
532
533
534 if (kdebug_enable && ddisk) {
535 uint32_t residual = (uint32_t)uio_resid(uio);
536 ktrace_code |= DKIO_DONE;
537 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_COMMON,
538 (FSDBG_CODE(DBG_DKRW, ktrace_code)) | DBG_FUNC_NONE, our_id,
539 (uintptr_t)VM_KERNEL_ADDRPERM(vp), residual, error, 0);
540 }
541
542 if (throttle_info) {
543 throttle_info_end_io_internal(throttle_info, thread_throttle_level);
544 }
545
546 return error;
547 }
548
549 case VBLK:
550 if (uio->uio_offset < 0) {
551 return EINVAL;
552 }
553
554 dev = vp->v_rdev;
555
556 devBlockSize = vp->v_specsize;
557
558 if (devBlockSize > PAGE_SIZE) {
559 return EINVAL;
560 }
561
562 bscale = PAGE_SIZE / devBlockSize;
563 bsize = bscale * devBlockSize;
564
565 do {
566 on = uio->uio_offset % bsize;
567
568 bn = (daddr64_t)((uio->uio_offset / devBlockSize) & ~(bscale - 1));
569
570 if (vp->v_speclastr + bscale == bn) {
571 nextbn = bn + bscale;
572 error = buf_breadn(vp, bn, (int)bsize, &nextbn,
573 (int *)&bsize, 1, NOCRED, &bp);
574 } else {
575 error = buf_bread(vp, bn, (int)bsize, NOCRED, &bp);
576 }
577
578 vnode_lock(vp);
579 vp->v_speclastr = bn;
580 vnode_unlock(vp);
581
582 n = bsize - buf_resid(bp);
583 if ((on > n) || error) {
584 if (!error) {
585 error = EINVAL;
586 }
587 buf_brelse(bp);
588 return error;
589 }
590 n = MIN((n - on), (size_t)uio_resid(uio));
591
592 error = uiomove((char *)buf_dataptr(bp) + on, (int)n, uio);
593 if (n + on == bsize) {
594 buf_markaged(bp);
595 }
596 buf_brelse(bp);
597 } while (error == 0 && uio_resid(uio) > 0 && n != 0);
598 return error;
599
600 default:
601 panic("spec_read type");
602 }
603 /* NOTREACHED */
604
605 return 0;
606 }
607
608 /*
609 * Vnode op for write
610 */
611 int
612 spec_write(struct vnop_write_args *ap)
613 {
614 struct vnode *vp = ap->a_vp;
615 struct uio *uio = ap->a_uio;
616 struct buf *bp;
617 daddr64_t bn;
618 int blkmask, bscale;
619 int io_sync;
620 int devBlockSize = 0;
621 size_t bsize, n, on;
622 int error = 0;
623 dev_t dev;
624
625 #if DIAGNOSTIC
626 if (uio->uio_rw != UIO_WRITE) {
627 panic("spec_write mode");
628 }
629 if (UIO_SEG_IS_USER_SPACE(uio->uio_segflg)) {
630 panic("spec_write proc");
631 }
632 #endif
633
634 switch (vp->v_type) {
635 case VCHR:
636 {
637 struct _throttle_io_info_t *throttle_info = NULL;
638 int thread_throttle_level;
639 dev = vp->v_rdev;
640 devBlockSize = vp->v_specsize;
641 uint32_t iolen = 0;
642 uint64_t blkno = 0;
643 int ddisk = 0;
644 int ktrace_code = 0; // write is implied; read must be OR'd in.
645 uintptr_t our_id;
646
647 if (cdevsw[major(dev)].d_type == D_DISK) {
648 ddisk = 1;
649 }
650
651 if (ddisk && vp->v_un.vu_specinfo->si_throttleable) {
652 throttle_info = &_throttle_io_info[vp->v_un.vu_specinfo->si_devbsdunit];
653
654 thread_throttle_level = throttle_info_update_internal(throttle_info, NULL, 0, vp->v_un.vu_specinfo->si_isssd, TRUE, NULL);
655
656 microuptime(&throttle_info->throttle_last_write_timestamp);
657 }
658
659 if (kdebug_enable && ddisk) {
660 if (devBlockSize == 0) {
661 devBlockSize = 512; // default sector size
662 }
663 if ((uio_offset(uio) != 0) && devBlockSize) {
664 blkno = ((uint64_t)uio_offset(uio)) / ((uint64_t)devBlockSize);
665 }
666 iolen = (int)uio_resid(uio);
667 our_id = (uintptr_t)thread_tid(current_thread());
668 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_COMMON,
669 (FSDBG_CODE(DBG_DKRW, ktrace_code)) | DBG_FUNC_NONE, our_id,
670 vp->v_rdev, blkno, iolen, 0);
671 }
672 error = (*cdevsw[major(vp->v_rdev)].d_write)
673 (vp->v_rdev, uio, ap->a_ioflag);
674
675 if (kdebug_enable && ddisk) {
676 //emit the I/O completion
677 uint32_t residual = (uint32_t)uio_resid(uio);
678 ktrace_code |= DKIO_DONE;
679 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_COMMON,
680 (FSDBG_CODE(DBG_DKRW, ktrace_code)) | DBG_FUNC_NONE, our_id,
681 (uintptr_t)VM_KERNEL_ADDRPERM(vp), residual, error, 0);
682 }
683
684 if (throttle_info) {
685 throttle_info_end_io_internal(throttle_info, thread_throttle_level);
686 }
687
688 return error;
689 }
690
691 case VBLK:
692 if (uio_resid(uio) == 0) {
693 return 0;
694 }
695 if (uio->uio_offset < 0) {
696 return EINVAL;
697 }
698
699 io_sync = (ap->a_ioflag & IO_SYNC);
700
701 dev = (vp->v_rdev);
702
703 devBlockSize = vp->v_specsize;
704 if (devBlockSize > PAGE_SIZE) {
705 return EINVAL;
706 }
707
708 bscale = PAGE_SIZE / devBlockSize;
709 blkmask = bscale - 1;
710 bsize = bscale * devBlockSize;
711
712
713 do {
714 bn = (daddr64_t)((uio->uio_offset / devBlockSize) & ~blkmask);
715 on = uio->uio_offset % bsize;
716
717 n = MIN((bsize - on), (size_t)uio_resid(uio));
718
719 /*
720 * Use buf_getblk() as an optimization IFF:
721 *
722 * 1) We are reading exactly a block on a block
723 * aligned boundary
724 * 2) We know the size of the device from spec_open
725 * 3) The read doesn't span the end of the device
726 *
727 * Otherwise, we fall back on buf_bread().
728 */
729 if (n == bsize &&
730 vp->v_specdevsize != (u_int64_t)0 &&
731 (uio->uio_offset + (u_int64_t)n) > vp->v_specdevsize) {
732 /* reduce the size of the read to what is there */
733 n = (uio->uio_offset + (u_int64_t)n) - vp->v_specdevsize;
734 }
735
736 if (n == bsize) {
737 bp = buf_getblk(vp, bn, (int)bsize, 0, 0, BLK_WRITE);
738 } else {
739 error = (int)buf_bread(vp, bn, (int)bsize, NOCRED, &bp);
740 }
741
742 /* Translate downstream error for upstream, if needed */
743 if (!error) {
744 error = (int)buf_error(bp);
745 }
746 if (error) {
747 buf_brelse(bp);
748 return error;
749 }
750 n = MIN(n, bsize - buf_resid(bp));
751
752 error = uiomove((char *)buf_dataptr(bp) + on, (int)n, uio);
753 if (error) {
754 buf_brelse(bp);
755 return error;
756 }
757 buf_markaged(bp);
758
759 if (io_sync) {
760 error = buf_bwrite(bp);
761 } else {
762 if ((n + on) == bsize) {
763 error = buf_bawrite(bp);
764 } else {
765 error = buf_bdwrite(bp);
766 }
767 }
768 } while (error == 0 && uio_resid(uio) > 0 && n != 0);
769 return error;
770
771 default:
772 panic("spec_write type");
773 }
774 /* NOTREACHED */
775
776 return 0;
777 }
778
779 /*
780 * Device ioctl operation.
781 */
782 int
783 spec_ioctl(struct vnop_ioctl_args *ap)
784 {
785 proc_t p = vfs_context_proc(ap->a_context);
786 dev_t dev = ap->a_vp->v_rdev;
787 int retval = 0;
788
789 KERNEL_DEBUG_CONSTANT(FSDBG_CODE(DBG_IOCTL, 0) | DBG_FUNC_START,
790 dev, ap->a_command, ap->a_fflag, ap->a_vp->v_type, 0);
791
792 switch (ap->a_vp->v_type) {
793 case VCHR:
794 retval = (*cdevsw[major(dev)].d_ioctl)(dev, ap->a_command, ap->a_data,
795 ap->a_fflag, p);
796 break;
797
798 case VBLK:
799 retval = (*bdevsw[major(dev)].d_ioctl)(dev, ap->a_command, ap->a_data, ap->a_fflag, p);
800 if (!retval && ap->a_command == DKIOCSETBLOCKSIZE) {
801 ap->a_vp->v_specsize = *(uint32_t *)ap->a_data;
802 }
803 break;
804
805 default:
806 panic("spec_ioctl");
807 /* NOTREACHED */
808 }
809 KERNEL_DEBUG_CONSTANT(FSDBG_CODE(DBG_IOCTL, 0) | DBG_FUNC_END,
810 dev, ap->a_command, ap->a_fflag, retval, 0);
811
812 return retval;
813 }
814
815 int
816 spec_select(struct vnop_select_args *ap)
817 {
818 proc_t p = vfs_context_proc(ap->a_context);
819 dev_t dev;
820
821 switch (ap->a_vp->v_type) {
822 default:
823 return 1; /* XXX */
824
825 case VCHR:
826 dev = ap->a_vp->v_rdev;
827 return (*cdevsw[major(dev)].d_select)(dev, ap->a_which, ap->a_wql, p);
828 }
829 }
830
831 static int filt_specattach(struct knote *kn, struct kevent_qos_s *kev);
832
833 int
834 spec_kqfilter(vnode_t vp, struct knote *kn, struct kevent_qos_s *kev)
835 {
836 dev_t dev;
837
838 assert(vnode_ischr(vp));
839
840 dev = vnode_specrdev(vp);
841
842 #if NETWORKING
843 /*
844 * Try a bpf device, as defined in bsd/net/bpf.c
845 * If it doesn't error out the attach, then it
846 * claimed it. Otherwise, fall through and try
847 * other attaches.
848 */
849 int32_t tmp_flags = kn->kn_flags;
850 int64_t tmp_sdata = kn->kn_sdata;
851 int res;
852
853 res = bpfkqfilter(dev, kn);
854 if ((kn->kn_flags & EV_ERROR) == 0) {
855 return res;
856 }
857 kn->kn_flags = tmp_flags;
858 kn->kn_sdata = tmp_sdata;
859 #endif
860
861 if (major(dev) > nchrdev) {
862 knote_set_error(kn, ENXIO);
863 return 0;
864 }
865
866 kn->kn_vnode_kqok = !!(cdevsw_flags[major(dev)] & CDEVSW_SELECT_KQUEUE);
867 kn->kn_vnode_use_ofst = !!(cdevsw_flags[major(dev)] & CDEVSW_USE_OFFSET);
868
869 if (cdevsw_flags[major(dev)] & CDEVSW_IS_PTS) {
870 kn->kn_filtid = EVFILTID_PTSD;
871 return ptsd_kqfilter(dev, kn);
872 } else if (cdevsw_flags[major(dev)] & CDEVSW_IS_PTC) {
873 kn->kn_filtid = EVFILTID_PTMX;
874 return ptmx_kqfilter(dev, kn);
875 } else if (cdevsw[major(dev)].d_type == D_TTY && kn->kn_vnode_kqok) {
876 /*
877 * TTYs from drivers that use struct ttys use their own filter
878 * routines. The PTC driver doesn't use the tty for character
879 * counts, so it must go through the select fallback.
880 */
881 kn->kn_filtid = EVFILTID_TTY;
882 return knote_fops(kn)->f_attach(kn, kev);
883 }
884
885 /* Try to attach to other char special devices */
886 return filt_specattach(kn, kev);
887 }
888
889 /*
890 * Synch buffers associated with a block device
891 */
892 int
893 spec_fsync_internal(vnode_t vp, int waitfor, __unused vfs_context_t context)
894 {
895 if (vp->v_type == VCHR) {
896 return 0;
897 }
898 /*
899 * Flush all dirty buffers associated with a block device.
900 */
901 buf_flushdirtyblks(vp, (waitfor == MNT_WAIT || waitfor == MNT_DWAIT), 0, "spec_fsync");
902
903 return 0;
904 }
905
906 int
907 spec_fsync(struct vnop_fsync_args *ap)
908 {
909 return spec_fsync_internal(ap->a_vp, ap->a_waitfor, ap->a_context);
910 }
911
912
913 /*
914 * Just call the device strategy routine
915 */
916 void throttle_init(void);
917
918
919 #if 0
920 #define DEBUG_ALLOC_THROTTLE_INFO(format, debug_info, args...) \
921 do { \
922 if ((debug_info)->alloc) \
923 printf("%s: "format, __FUNCTION__, ## args); \
924 } while(0)
925
926 #else
927 #define DEBUG_ALLOC_THROTTLE_INFO(format, debug_info, args...)
928 #endif
929
930
931 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier1_window_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_windows_msecs[THROTTLE_LEVEL_TIER1], 0, "");
932 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier2_window_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_windows_msecs[THROTTLE_LEVEL_TIER2], 0, "");
933 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier3_window_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_windows_msecs[THROTTLE_LEVEL_TIER3], 0, "");
934
935 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier1_io_period_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_msecs[THROTTLE_LEVEL_TIER1], 0, "");
936 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier2_io_period_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_msecs[THROTTLE_LEVEL_TIER2], 0, "");
937 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier3_io_period_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_msecs[THROTTLE_LEVEL_TIER3], 0, "");
938
939 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier1_io_period_ssd_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_ssd_msecs[THROTTLE_LEVEL_TIER1], 0, "");
940 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier2_io_period_ssd_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_ssd_msecs[THROTTLE_LEVEL_TIER2], 0, "");
941 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_tier3_io_period_ssd_msecs, CTLFLAG_RW | CTLFLAG_LOCKED, &throttle_io_period_ssd_msecs[THROTTLE_LEVEL_TIER3], 0, "");
942
943 SYSCTL_INT(_debug, OID_AUTO, lowpri_throttle_enabled, CTLFLAG_RW | CTLFLAG_LOCKED, &lowpri_throttle_enabled, 0, "");
944
945
946 static lck_grp_t *throttle_lock_grp;
947 static lck_attr_t *throttle_lock_attr;
948 static lck_grp_attr_t *throttle_lock_grp_attr;
949
950
951 /*
952 * throttled I/O helper function
953 * convert the index of the lowest set bit to a device index
954 */
955 int
956 num_trailing_0(uint64_t n)
957 {
958 /*
959 * since in most cases the number of trailing 0s is very small,
960 * we simply counting sequentially from the lowest bit
961 */
962 if (n == 0) {
963 return sizeof(n) * 8;
964 }
965 int count = 0;
966 while (!ISSET(n, 1)) {
967 n >>= 1;
968 ++count;
969 }
970 return count;
971 }
972
973
974 /*
975 * Release the reference and if the item was allocated and this is the last
976 * reference then free it.
977 *
978 * This routine always returns the old value.
979 */
980 static int
981 throttle_info_rel(struct _throttle_io_info_t *info)
982 {
983 SInt32 oldValue = OSDecrementAtomic(&info->throttle_refcnt);
984
985 DEBUG_ALLOC_THROTTLE_INFO("refcnt = %d info = %p\n",
986 info, (int)(oldValue - 1), info );
987
988 /* The reference count just went negative, very bad */
989 if (oldValue == 0) {
990 panic("throttle info ref cnt went negative!");
991 }
992
993 /*
994 * Once reference count is zero, no one else should be able to take a
995 * reference
996 */
997 if ((info->throttle_refcnt == 0) && (info->throttle_alloc)) {
998 DEBUG_ALLOC_THROTTLE_INFO("Freeing info = %p\n", info);
999
1000 lck_mtx_destroy(&info->throttle_lock, throttle_lock_grp);
1001 FREE(info, M_TEMP);
1002 }
1003 return oldValue;
1004 }
1005
1006
1007 /*
1008 * Just take a reference on the throttle info structure.
1009 *
1010 * This routine always returns the old value.
1011 */
1012 static SInt32
1013 throttle_info_ref(struct _throttle_io_info_t *info)
1014 {
1015 SInt32 oldValue = OSIncrementAtomic(&info->throttle_refcnt);
1016
1017 DEBUG_ALLOC_THROTTLE_INFO("refcnt = %d info = %p\n",
1018 info, (int)(oldValue - 1), info );
1019 /* Allocated items should never have a reference of zero */
1020 if (info->throttle_alloc && (oldValue == 0)) {
1021 panic("Taking a reference without calling create throttle info!\n");
1022 }
1023
1024 return oldValue;
1025 }
1026
1027 /*
1028 * on entry the throttle_lock is held...
1029 * this function is responsible for taking
1030 * and dropping the reference on the info
1031 * structure which will keep it from going
1032 * away while the timer is running if it
1033 * happens to have been dynamically allocated by
1034 * a network fileystem kext which is now trying
1035 * to free it
1036 */
1037 static uint32_t
1038 throttle_timer_start(struct _throttle_io_info_t *info, boolean_t update_io_count, int wakelevel)
1039 {
1040 struct timeval elapsed;
1041 struct timeval now;
1042 struct timeval period;
1043 uint64_t elapsed_msecs;
1044 int throttle_level;
1045 int level;
1046 int msecs;
1047 boolean_t throttled = FALSE;
1048 boolean_t need_timer = FALSE;
1049
1050 microuptime(&now);
1051
1052 if (update_io_count == TRUE) {
1053 info->throttle_io_count_begin = info->throttle_io_count;
1054 info->throttle_io_period_num++;
1055
1056 while (wakelevel >= THROTTLE_LEVEL_THROTTLED) {
1057 info->throttle_start_IO_period_timestamp[wakelevel--] = now;
1058 }
1059
1060 info->throttle_min_timer_deadline = now;
1061
1062 msecs = info->throttle_io_periods[THROTTLE_LEVEL_THROTTLED];
1063 period.tv_sec = msecs / 1000;
1064 period.tv_usec = (msecs % 1000) * 1000;
1065
1066 timevaladd(&info->throttle_min_timer_deadline, &period);
1067 }
1068 for (throttle_level = THROTTLE_LEVEL_START; throttle_level < THROTTLE_LEVEL_END; throttle_level++) {
1069 elapsed = now;
1070 timevalsub(&elapsed, &info->throttle_window_start_timestamp[throttle_level]);
1071 elapsed_msecs = (uint64_t)elapsed.tv_sec * (uint64_t)1000 + (elapsed.tv_usec / 1000);
1072
1073 for (level = throttle_level + 1; level <= THROTTLE_LEVEL_END; level++) {
1074 if (!TAILQ_EMPTY(&info->throttle_uthlist[level])) {
1075 if (elapsed_msecs < (uint64_t)throttle_windows_msecs[level] || info->throttle_inflight_count[throttle_level]) {
1076 /*
1077 * we had an I/O occur at a higher priority tier within
1078 * this tier's throttle window
1079 */
1080 throttled = TRUE;
1081 }
1082 /*
1083 * we assume that the windows are the same or longer
1084 * as we drop through the throttling tiers... thus
1085 * we can stop looking once we run into a tier with
1086 * threads to schedule regardless of whether it's
1087 * still in its throttling window or not
1088 */
1089 break;
1090 }
1091 }
1092 if (throttled == TRUE) {
1093 break;
1094 }
1095 }
1096 if (throttled == TRUE) {
1097 uint64_t deadline = 0;
1098 struct timeval target;
1099 struct timeval min_target;
1100
1101 /*
1102 * we've got at least one tier still in a throttled window
1103 * so we need a timer running... compute the next deadline
1104 * and schedule it
1105 */
1106 for (level = throttle_level + 1; level <= THROTTLE_LEVEL_END; level++) {
1107 if (TAILQ_EMPTY(&info->throttle_uthlist[level])) {
1108 continue;
1109 }
1110
1111 target = info->throttle_start_IO_period_timestamp[level];
1112
1113 msecs = info->throttle_io_periods[level];
1114 period.tv_sec = msecs / 1000;
1115 period.tv_usec = (msecs % 1000) * 1000;
1116
1117 timevaladd(&target, &period);
1118
1119 if (need_timer == FALSE || timevalcmp(&target, &min_target, <)) {
1120 min_target = target;
1121 need_timer = TRUE;
1122 }
1123 }
1124 if (timevalcmp(&info->throttle_min_timer_deadline, &now, >)) {
1125 if (timevalcmp(&info->throttle_min_timer_deadline, &min_target, >)) {
1126 min_target = info->throttle_min_timer_deadline;
1127 }
1128 }
1129
1130 if (info->throttle_timer_active) {
1131 if (thread_call_cancel(info->throttle_timer_call) == FALSE) {
1132 /*
1133 * couldn't kill the timer because it's already
1134 * been dispatched, so don't try to start a new
1135 * one... once we drop the lock, the timer will
1136 * proceed and eventually re-run this function
1137 */
1138 need_timer = FALSE;
1139 } else {
1140 info->throttle_timer_active = 0;
1141 }
1142 }
1143 if (need_timer == TRUE) {
1144 /*
1145 * This is defined as an int (32-bit) rather than a 64-bit
1146 * value because it would need a really big period in the
1147 * order of ~500 days to overflow this. So, we let this be
1148 * 32-bit which allows us to use the clock_interval_to_deadline()
1149 * routine.
1150 */
1151 int target_msecs;
1152
1153 if (info->throttle_timer_ref == 0) {
1154 /*
1155 * take a reference for the timer
1156 */
1157 throttle_info_ref(info);
1158
1159 info->throttle_timer_ref = 1;
1160 }
1161 elapsed = min_target;
1162 timevalsub(&elapsed, &now);
1163 target_msecs = (int)(elapsed.tv_sec * 1000 + elapsed.tv_usec / 1000);
1164
1165 if (target_msecs <= 0) {
1166 /*
1167 * we may have computed a deadline slightly in the past
1168 * due to various factors... if so, just set the timer
1169 * to go off in the near future (we don't need to be precise)
1170 */
1171 target_msecs = 1;
1172 }
1173 clock_interval_to_deadline(target_msecs, 1000000, &deadline);
1174
1175 thread_call_enter_delayed(info->throttle_timer_call, deadline);
1176 info->throttle_timer_active = 1;
1177 }
1178 }
1179 return throttle_level;
1180 }
1181
1182
1183 static void
1184 throttle_timer(struct _throttle_io_info_t *info)
1185 {
1186 uthread_t ut, utlist;
1187 struct timeval elapsed;
1188 struct timeval now;
1189 uint64_t elapsed_msecs;
1190 int throttle_level;
1191 int level;
1192 int wake_level;
1193 caddr_t wake_address = NULL;
1194 boolean_t update_io_count = FALSE;
1195 boolean_t need_wakeup = FALSE;
1196 boolean_t need_release = FALSE;
1197
1198 ut = NULL;
1199 lck_mtx_lock(&info->throttle_lock);
1200
1201 info->throttle_timer_active = 0;
1202 microuptime(&now);
1203
1204 elapsed = now;
1205 timevalsub(&elapsed, &info->throttle_start_IO_period_timestamp[THROTTLE_LEVEL_THROTTLED]);
1206 elapsed_msecs = (uint64_t)elapsed.tv_sec * (uint64_t)1000 + (elapsed.tv_usec / 1000);
1207
1208 if (elapsed_msecs >= (uint64_t)info->throttle_io_periods[THROTTLE_LEVEL_THROTTLED]) {
1209 wake_level = info->throttle_next_wake_level;
1210
1211 for (level = THROTTLE_LEVEL_START; level < THROTTLE_LEVEL_END; level++) {
1212 elapsed = now;
1213 timevalsub(&elapsed, &info->throttle_start_IO_period_timestamp[wake_level]);
1214 elapsed_msecs = (uint64_t)elapsed.tv_sec * (uint64_t)1000 + (elapsed.tv_usec / 1000);
1215
1216 if (elapsed_msecs >= (uint64_t)info->throttle_io_periods[wake_level] && !TAILQ_EMPTY(&info->throttle_uthlist[wake_level])) {
1217 /*
1218 * we're closing out the current IO period...
1219 * if we have a waiting thread, wake it up
1220 * after we have reset the I/O window info
1221 */
1222 need_wakeup = TRUE;
1223 update_io_count = TRUE;
1224
1225 info->throttle_next_wake_level = wake_level - 1;
1226
1227 if (info->throttle_next_wake_level == THROTTLE_LEVEL_START) {
1228 info->throttle_next_wake_level = THROTTLE_LEVEL_END;
1229 }
1230
1231 break;
1232 }
1233 wake_level--;
1234
1235 if (wake_level == THROTTLE_LEVEL_START) {
1236 wake_level = THROTTLE_LEVEL_END;
1237 }
1238 }
1239 }
1240 if (need_wakeup == TRUE) {
1241 if (!TAILQ_EMPTY(&info->throttle_uthlist[wake_level])) {
1242 ut = (uthread_t)TAILQ_FIRST(&info->throttle_uthlist[wake_level]);
1243 TAILQ_REMOVE(&info->throttle_uthlist[wake_level], ut, uu_throttlelist);
1244 ut->uu_on_throttlelist = THROTTLE_LEVEL_NONE;
1245 ut->uu_is_throttled = false;
1246
1247 wake_address = (caddr_t)&ut->uu_on_throttlelist;
1248 }
1249 } else {
1250 wake_level = THROTTLE_LEVEL_START;
1251 }
1252
1253 throttle_level = throttle_timer_start(info, update_io_count, wake_level);
1254
1255 if (wake_address != NULL) {
1256 wakeup(wake_address);
1257 }
1258
1259 for (level = THROTTLE_LEVEL_THROTTLED; level <= throttle_level; level++) {
1260 TAILQ_FOREACH_SAFE(ut, &info->throttle_uthlist[level], uu_throttlelist, utlist) {
1261 TAILQ_REMOVE(&info->throttle_uthlist[level], ut, uu_throttlelist);
1262 ut->uu_on_throttlelist = THROTTLE_LEVEL_NONE;
1263 ut->uu_is_throttled = false;
1264
1265 wakeup(&ut->uu_on_throttlelist);
1266 }
1267 }
1268 if (info->throttle_timer_active == 0 && info->throttle_timer_ref) {
1269 info->throttle_timer_ref = 0;
1270 need_release = TRUE;
1271 }
1272 lck_mtx_unlock(&info->throttle_lock);
1273
1274 if (need_release == TRUE) {
1275 throttle_info_rel(info);
1276 }
1277 }
1278
1279
1280 static int
1281 throttle_add_to_list(struct _throttle_io_info_t *info, uthread_t ut, int mylevel, boolean_t insert_tail)
1282 {
1283 boolean_t start_timer = FALSE;
1284 int level = THROTTLE_LEVEL_START;
1285
1286 if (TAILQ_EMPTY(&info->throttle_uthlist[mylevel])) {
1287 info->throttle_start_IO_period_timestamp[mylevel] = info->throttle_last_IO_timestamp[mylevel];
1288 start_timer = TRUE;
1289 }
1290
1291 if (insert_tail == TRUE) {
1292 TAILQ_INSERT_TAIL(&info->throttle_uthlist[mylevel], ut, uu_throttlelist);
1293 } else {
1294 TAILQ_INSERT_HEAD(&info->throttle_uthlist[mylevel], ut, uu_throttlelist);
1295 }
1296
1297 ut->uu_on_throttlelist = (int8_t)mylevel;
1298
1299 if (start_timer == TRUE) {
1300 /* we may need to start or rearm the timer */
1301 level = throttle_timer_start(info, FALSE, THROTTLE_LEVEL_START);
1302
1303 if (level == THROTTLE_LEVEL_END) {
1304 if (ut->uu_on_throttlelist >= THROTTLE_LEVEL_THROTTLED) {
1305 TAILQ_REMOVE(&info->throttle_uthlist[ut->uu_on_throttlelist], ut, uu_throttlelist);
1306
1307 ut->uu_on_throttlelist = THROTTLE_LEVEL_NONE;
1308 }
1309 }
1310 }
1311 return level;
1312 }
1313
1314 static void
1315 throttle_init_throttle_window(void)
1316 {
1317 int throttle_window_size;
1318
1319 /*
1320 * The hierarchy of throttle window values is as follows:
1321 * - Global defaults
1322 * - Device tree properties
1323 * - Boot-args
1324 * All values are specified in msecs.
1325 */
1326
1327 /* Override global values with device-tree properties */
1328 if (PE_get_default("kern.io_throttle_window_tier1", &throttle_window_size, sizeof(throttle_window_size))) {
1329 throttle_windows_msecs[THROTTLE_LEVEL_TIER1] = throttle_window_size;
1330 }
1331
1332 if (PE_get_default("kern.io_throttle_window_tier2", &throttle_window_size, sizeof(throttle_window_size))) {
1333 throttle_windows_msecs[THROTTLE_LEVEL_TIER2] = throttle_window_size;
1334 }
1335
1336 if (PE_get_default("kern.io_throttle_window_tier3", &throttle_window_size, sizeof(throttle_window_size))) {
1337 throttle_windows_msecs[THROTTLE_LEVEL_TIER3] = throttle_window_size;
1338 }
1339
1340 /* Override with boot-args */
1341 if (PE_parse_boot_argn("io_throttle_window_tier1", &throttle_window_size, sizeof(throttle_window_size))) {
1342 throttle_windows_msecs[THROTTLE_LEVEL_TIER1] = throttle_window_size;
1343 }
1344
1345 if (PE_parse_boot_argn("io_throttle_window_tier2", &throttle_window_size, sizeof(throttle_window_size))) {
1346 throttle_windows_msecs[THROTTLE_LEVEL_TIER2] = throttle_window_size;
1347 }
1348
1349 if (PE_parse_boot_argn("io_throttle_window_tier3", &throttle_window_size, sizeof(throttle_window_size))) {
1350 throttle_windows_msecs[THROTTLE_LEVEL_TIER3] = throttle_window_size;
1351 }
1352 }
1353
1354 static void
1355 throttle_init_throttle_period(struct _throttle_io_info_t *info, boolean_t isssd)
1356 {
1357 int throttle_period_size;
1358
1359 /*
1360 * The hierarchy of throttle period values is as follows:
1361 * - Global defaults
1362 * - Device tree properties
1363 * - Boot-args
1364 * All values are specified in msecs.
1365 */
1366
1367 /* Assign global defaults */
1368 if ((isssd == TRUE) && (info->throttle_is_fusion_with_priority == 0)) {
1369 info->throttle_io_periods = &throttle_io_period_ssd_msecs[0];
1370 } else {
1371 info->throttle_io_periods = &throttle_io_period_msecs[0];
1372 }
1373
1374 /* Override global values with device-tree properties */
1375 if (PE_get_default("kern.io_throttle_period_tier1", &throttle_period_size, sizeof(throttle_period_size))) {
1376 info->throttle_io_periods[THROTTLE_LEVEL_TIER1] = throttle_period_size;
1377 }
1378
1379 if (PE_get_default("kern.io_throttle_period_tier2", &throttle_period_size, sizeof(throttle_period_size))) {
1380 info->throttle_io_periods[THROTTLE_LEVEL_TIER2] = throttle_period_size;
1381 }
1382
1383 if (PE_get_default("kern.io_throttle_period_tier3", &throttle_period_size, sizeof(throttle_period_size))) {
1384 info->throttle_io_periods[THROTTLE_LEVEL_TIER3] = throttle_period_size;
1385 }
1386
1387 /* Override with boot-args */
1388 if (PE_parse_boot_argn("io_throttle_period_tier1", &throttle_period_size, sizeof(throttle_period_size))) {
1389 info->throttle_io_periods[THROTTLE_LEVEL_TIER1] = throttle_period_size;
1390 }
1391
1392 if (PE_parse_boot_argn("io_throttle_period_tier2", &throttle_period_size, sizeof(throttle_period_size))) {
1393 info->throttle_io_periods[THROTTLE_LEVEL_TIER2] = throttle_period_size;
1394 }
1395
1396 if (PE_parse_boot_argn("io_throttle_period_tier3", &throttle_period_size, sizeof(throttle_period_size))) {
1397 info->throttle_io_periods[THROTTLE_LEVEL_TIER3] = throttle_period_size;
1398 }
1399 }
1400
1401 #if CONFIG_IOSCHED
1402 extern void vm_io_reprioritize_init(void);
1403 int iosched_enabled = 1;
1404 #endif
1405
1406 void
1407 throttle_init(void)
1408 {
1409 struct _throttle_io_info_t *info;
1410 int i;
1411 int level;
1412 #if CONFIG_IOSCHED
1413 int iosched;
1414 #endif
1415 /*
1416 * allocate lock group attribute and group
1417 */
1418 throttle_lock_grp_attr = lck_grp_attr_alloc_init();
1419 throttle_lock_grp = lck_grp_alloc_init("throttle I/O", throttle_lock_grp_attr);
1420
1421 /* Update throttle parameters based on device tree configuration */
1422 throttle_init_throttle_window();
1423
1424 /*
1425 * allocate the lock attribute
1426 */
1427 throttle_lock_attr = lck_attr_alloc_init();
1428
1429 for (i = 0; i < LOWPRI_MAX_NUM_DEV; i++) {
1430 info = &_throttle_io_info[i];
1431
1432 lck_mtx_init(&info->throttle_lock, throttle_lock_grp, throttle_lock_attr);
1433 info->throttle_timer_call = thread_call_allocate((thread_call_func_t)throttle_timer, (thread_call_param_t)info);
1434
1435 for (level = 0; level <= THROTTLE_LEVEL_END; level++) {
1436 TAILQ_INIT(&info->throttle_uthlist[level]);
1437 info->throttle_last_IO_pid[level] = 0;
1438 info->throttle_inflight_count[level] = 0;
1439 }
1440 info->throttle_next_wake_level = THROTTLE_LEVEL_END;
1441 info->throttle_disabled = 0;
1442 info->throttle_is_fusion_with_priority = 0;
1443 }
1444 #if CONFIG_IOSCHED
1445 if (PE_parse_boot_argn("iosched", &iosched, sizeof(iosched))) {
1446 iosched_enabled = iosched;
1447 }
1448 if (iosched_enabled) {
1449 /* Initialize I/O Reprioritization mechanism */
1450 vm_io_reprioritize_init();
1451 }
1452 #endif
1453 }
1454
1455 void
1456 sys_override_io_throttle(boolean_t enable_override)
1457 {
1458 if (enable_override) {
1459 lowpri_throttle_enabled = 0;
1460 } else {
1461 lowpri_throttle_enabled = 1;
1462 }
1463 }
1464
1465 int rethrottle_wakeups = 0;
1466
1467 /*
1468 * the uu_rethrottle_lock is used to synchronize this function
1469 * with "throttle_lowpri_io" which is where a throttled thread
1470 * will block... that function will grab this lock before beginning
1471 * it's decision making process concerning the need to block, and
1472 * hold it through the assert_wait. When that thread is awakened
1473 * for any reason (timer or rethrottle), it will reacquire the
1474 * uu_rethrottle_lock before determining if it really is ok for
1475 * it to now run. This is the point at which the thread could
1476 * enter a different throttling queue and reblock or return from
1477 * the throttle w/o having waited out it's entire throttle if
1478 * the rethrottle has now moved it out of any currently
1479 * active throttle window.
1480 *
1481 *
1482 * NOTES:
1483 * 1 - This may be called with the task lock held.
1484 * 2 - This may be called with preemption and interrupts disabled
1485 * in the kqueue wakeup path so we can't take the throttle_lock which is a mutex
1486 * 3 - This cannot safely dereference uu_throttle_info, as it may
1487 * get deallocated out from under us
1488 */
1489
1490 void
1491 rethrottle_thread(uthread_t ut)
1492 {
1493 /*
1494 * If uthread doesn't have throttle state, then there's no chance
1495 * of it needing a rethrottle.
1496 */
1497 if (ut->uu_throttle_info == NULL) {
1498 return;
1499 }
1500
1501 boolean_t s = ml_set_interrupts_enabled(FALSE);
1502 lck_spin_lock(&ut->uu_rethrottle_lock);
1503
1504 if (!ut->uu_is_throttled) {
1505 ut->uu_was_rethrottled = true;
1506 } else {
1507 int my_new_level = throttle_get_thread_throttle_level(ut);
1508
1509 if (my_new_level != ut->uu_on_throttlelist) {
1510 /*
1511 * ut is currently blocked (as indicated by
1512 * ut->uu_is_throttled == true)
1513 * and we're changing it's throttle level, so
1514 * we need to wake it up.
1515 */
1516 ut->uu_is_throttled = false;
1517 wakeup(&ut->uu_on_throttlelist);
1518
1519 rethrottle_wakeups++;
1520 KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_FSRW, 102)), thread_tid(ut->uu_thread), ut->uu_on_throttlelist, my_new_level, 0, 0);
1521 }
1522 }
1523 lck_spin_unlock(&ut->uu_rethrottle_lock);
1524 ml_set_interrupts_enabled(s);
1525 }
1526
1527
1528 /*
1529 * KPI routine
1530 *
1531 * Create and take a reference on a throttle info structure and return a
1532 * pointer for the file system to use when calling throttle_info_update.
1533 * Calling file system must have a matching release for every create.
1534 */
1535 void *
1536 throttle_info_create(void)
1537 {
1538 struct _throttle_io_info_t *info;
1539 int level;
1540
1541 MALLOC(info, struct _throttle_io_info_t *, sizeof(*info), M_TEMP, M_ZERO | M_WAITOK);
1542 /* Should never happen but just in case */
1543 if (info == NULL) {
1544 return NULL;
1545 }
1546 /* Mark that this one was allocated and needs to be freed */
1547 DEBUG_ALLOC_THROTTLE_INFO("Creating info = %p\n", info, info );
1548 info->throttle_alloc = TRUE;
1549
1550 lck_mtx_init(&info->throttle_lock, throttle_lock_grp, throttle_lock_attr);
1551 info->throttle_timer_call = thread_call_allocate((thread_call_func_t)throttle_timer, (thread_call_param_t)info);
1552
1553 for (level = 0; level <= THROTTLE_LEVEL_END; level++) {
1554 TAILQ_INIT(&info->throttle_uthlist[level]);
1555 }
1556 info->throttle_next_wake_level = THROTTLE_LEVEL_END;
1557
1558 /* Take a reference */
1559 OSIncrementAtomic(&info->throttle_refcnt);
1560 return info;
1561 }
1562
1563 /*
1564 * KPI routine
1565 *
1566 * Release the throttle info pointer if all the reference are gone. Should be
1567 * called to release reference taken by throttle_info_create
1568 */
1569 void
1570 throttle_info_release(void *throttle_info)
1571 {
1572 DEBUG_ALLOC_THROTTLE_INFO("Releaseing info = %p\n",
1573 (struct _throttle_io_info_t *)throttle_info,
1574 (struct _throttle_io_info_t *)throttle_info);
1575 if (throttle_info) { /* Just to be careful */
1576 throttle_info_rel(throttle_info);
1577 }
1578 }
1579
1580 /*
1581 * KPI routine
1582 *
1583 * File Systems that create an info structure, need to call this routine in
1584 * their mount routine (used by cluster code). File Systems that call this in
1585 * their mount routines must call throttle_info_mount_rel in their unmount
1586 * routines.
1587 */
1588 void
1589 throttle_info_mount_ref(mount_t mp, void *throttle_info)
1590 {
1591 if ((throttle_info == NULL) || (mp == NULL)) {
1592 return;
1593 }
1594 throttle_info_ref(throttle_info);
1595
1596 /*
1597 * We already have a reference release it before adding the new one
1598 */
1599 if (mp->mnt_throttle_info) {
1600 throttle_info_rel(mp->mnt_throttle_info);
1601 }
1602 mp->mnt_throttle_info = throttle_info;
1603 }
1604
1605 /*
1606 * Private KPI routine
1607 *
1608 * return a handle for accessing throttle_info given a throttle_mask. The
1609 * handle must be released by throttle_info_rel_by_mask
1610 */
1611 int
1612 throttle_info_ref_by_mask(uint64_t throttle_mask, throttle_info_handle_t *throttle_info_handle)
1613 {
1614 int dev_index;
1615 struct _throttle_io_info_t *info;
1616
1617 /*
1618 * The 'throttle_mask' is not expected to be 0 otherwise num_trailing_0()
1619 * would return value of 64 and this will cause '_throttle_io_info' to
1620 * go out of bounds as '_throttle_io_info' is only LOWPRI_MAX_NUM_DEV (64)
1621 * elements long.
1622 */
1623 if (throttle_info_handle == NULL || throttle_mask == 0) {
1624 return EINVAL;
1625 }
1626
1627 dev_index = num_trailing_0(throttle_mask);
1628 info = &_throttle_io_info[dev_index];
1629 throttle_info_ref(info);
1630 *(struct _throttle_io_info_t**)throttle_info_handle = info;
1631
1632 return 0;
1633 }
1634
1635 /*
1636 * Private KPI routine
1637 *
1638 * release the handle obtained by throttle_info_ref_by_mask
1639 */
1640 void
1641 throttle_info_rel_by_mask(throttle_info_handle_t throttle_info_handle)
1642 {
1643 /*
1644 * for now the handle is just a pointer to _throttle_io_info_t
1645 */
1646 throttle_info_rel((struct _throttle_io_info_t*)throttle_info_handle);
1647 }
1648
1649 /*
1650 * KPI routine
1651 *
1652 * File Systems that throttle_info_mount_ref, must call this routine in their
1653 * umount routine.
1654 */
1655 void
1656 throttle_info_mount_rel(mount_t mp)
1657 {
1658 if (mp->mnt_throttle_info) {
1659 throttle_info_rel(mp->mnt_throttle_info);
1660 }
1661 mp->mnt_throttle_info = NULL;
1662 }
1663
1664 /*
1665 * Reset throttling periods for the given mount point
1666 *
1667 * private interface used by disk conditioner to reset
1668 * throttling periods when 'is_ssd' status changes
1669 */
1670 void
1671 throttle_info_mount_reset_period(mount_t mp, int isssd)
1672 {
1673 struct _throttle_io_info_t *info;
1674
1675 if (mp == NULL) {
1676 info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
1677 } else if (mp->mnt_throttle_info == NULL) {
1678 info = &_throttle_io_info[mp->mnt_devbsdunit];
1679 } else {
1680 info = mp->mnt_throttle_info;
1681 }
1682
1683 throttle_init_throttle_period(info, isssd);
1684 }
1685
1686 void
1687 throttle_info_get_last_io_time(mount_t mp, struct timeval *tv)
1688 {
1689 struct _throttle_io_info_t *info;
1690
1691 if (mp == NULL) {
1692 info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
1693 } else if (mp->mnt_throttle_info == NULL) {
1694 info = &_throttle_io_info[mp->mnt_devbsdunit];
1695 } else {
1696 info = mp->mnt_throttle_info;
1697 }
1698
1699 *tv = info->throttle_last_write_timestamp;
1700 }
1701
1702 void
1703 update_last_io_time(mount_t mp)
1704 {
1705 struct _throttle_io_info_t *info;
1706
1707 if (mp == NULL) {
1708 info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
1709 } else if (mp->mnt_throttle_info == NULL) {
1710 info = &_throttle_io_info[mp->mnt_devbsdunit];
1711 } else {
1712 info = mp->mnt_throttle_info;
1713 }
1714
1715 microuptime(&info->throttle_last_write_timestamp);
1716 if (mp != NULL) {
1717 mp->mnt_last_write_completed_timestamp = info->throttle_last_write_timestamp;
1718 }
1719 }
1720
1721 int
1722 throttle_get_io_policy(uthread_t *ut)
1723 {
1724 if (ut != NULL) {
1725 *ut = get_bsdthread_info(current_thread());
1726 }
1727
1728 return proc_get_effective_thread_policy(current_thread(), TASK_POLICY_IO);
1729 }
1730
1731 int
1732 throttle_get_passive_io_policy(uthread_t *ut)
1733 {
1734 if (ut != NULL) {
1735 *ut = get_bsdthread_info(current_thread());
1736 }
1737
1738 return proc_get_effective_thread_policy(current_thread(), TASK_POLICY_PASSIVE_IO);
1739 }
1740
1741
1742 static int
1743 throttle_get_thread_throttle_level(uthread_t ut)
1744 {
1745 uthread_t *ut_p = (ut == NULL) ? &ut : NULL;
1746 int io_tier = throttle_get_io_policy(ut_p);
1747
1748 return throttle_get_thread_throttle_level_internal(ut, io_tier);
1749 }
1750
1751 /*
1752 * Return a throttle level given an existing I/O tier (such as returned by throttle_get_io_policy)
1753 */
1754 static int
1755 throttle_get_thread_throttle_level_internal(uthread_t ut, int io_tier)
1756 {
1757 int thread_throttle_level = io_tier;
1758 int user_idle_level;
1759
1760 assert(ut != NULL);
1761
1762 /* Bootcache misses should always be throttled */
1763 if (ut->uu_throttle_bc) {
1764 thread_throttle_level = THROTTLE_LEVEL_TIER3;
1765 }
1766
1767 /*
1768 * Issue tier3 I/O as tier2 when the user is idle
1769 * to allow maintenance tasks to make more progress.
1770 *
1771 * Assume any positive idle level is enough... for now it's
1772 * only ever 0 or 128 but this is not defined anywhere.
1773 */
1774 if (thread_throttle_level >= THROTTLE_LEVEL_TIER3) {
1775 user_idle_level = timer_get_user_idle_level();
1776 if (user_idle_level > 0) {
1777 thread_throttle_level--;
1778 }
1779 }
1780
1781 return thread_throttle_level;
1782 }
1783
1784 /*
1785 * I/O will be throttled if either of the following are true:
1786 * - Higher tiers have in-flight I/O
1787 * - The time delta since the last start/completion of a higher tier is within the throttle window interval
1788 *
1789 * In-flight I/O is bookended by throttle_info_update_internal/throttle_info_end_io_internal
1790 */
1791 static int
1792 throttle_io_will_be_throttled_internal(void * throttle_info, int * mylevel, int * throttling_level)
1793 {
1794 struct _throttle_io_info_t *info = throttle_info;
1795 struct timeval elapsed;
1796 struct timeval now;
1797 uint64_t elapsed_msecs;
1798 int thread_throttle_level;
1799 int throttle_level;
1800
1801 if ((thread_throttle_level = throttle_get_thread_throttle_level(NULL)) < THROTTLE_LEVEL_THROTTLED) {
1802 return THROTTLE_DISENGAGED;
1803 }
1804
1805 microuptime(&now);
1806
1807 for (throttle_level = THROTTLE_LEVEL_START; throttle_level < thread_throttle_level; throttle_level++) {
1808 if (info->throttle_inflight_count[throttle_level]) {
1809 break;
1810 }
1811 elapsed = now;
1812 timevalsub(&elapsed, &info->throttle_window_start_timestamp[throttle_level]);
1813 elapsed_msecs = (uint64_t)elapsed.tv_sec * (uint64_t)1000 + (elapsed.tv_usec / 1000);
1814
1815 if (elapsed_msecs < (uint64_t)throttle_windows_msecs[thread_throttle_level]) {
1816 break;
1817 }
1818 }
1819 if (throttle_level >= thread_throttle_level) {
1820 /*
1821 * we're beyond all of the throttle windows
1822 * that affect the throttle level of this thread,
1823 * so go ahead and treat as normal I/O
1824 */
1825 return THROTTLE_DISENGAGED;
1826 }
1827 if (mylevel) {
1828 *mylevel = thread_throttle_level;
1829 }
1830 if (throttling_level) {
1831 *throttling_level = throttle_level;
1832 }
1833
1834 if (info->throttle_io_count != info->throttle_io_count_begin) {
1835 /*
1836 * we've already issued at least one throttleable I/O
1837 * in the current I/O window, so avoid issuing another one
1838 */
1839 return THROTTLE_NOW;
1840 }
1841 /*
1842 * we're in the throttle window, so
1843 * cut the I/O size back
1844 */
1845 return THROTTLE_ENGAGED;
1846 }
1847
1848 /*
1849 * If we have a mount point and it has a throttle info pointer then
1850 * use it to do the check, otherwise use the device unit number to find
1851 * the correct throttle info array element.
1852 */
1853 int
1854 throttle_io_will_be_throttled(__unused int lowpri_window_msecs, mount_t mp)
1855 {
1856 struct _throttle_io_info_t *info;
1857
1858 /*
1859 * Should we just return zero if no mount point
1860 */
1861 if (mp == NULL) {
1862 info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
1863 } else if (mp->mnt_throttle_info == NULL) {
1864 info = &_throttle_io_info[mp->mnt_devbsdunit];
1865 } else {
1866 info = mp->mnt_throttle_info;
1867 }
1868
1869 if (info->throttle_is_fusion_with_priority) {
1870 uthread_t ut = get_bsdthread_info(current_thread());
1871 if (ut->uu_lowpri_window == 0) {
1872 return THROTTLE_DISENGAGED;
1873 }
1874 }
1875
1876 if (info->throttle_disabled) {
1877 return THROTTLE_DISENGAGED;
1878 } else {
1879 return throttle_io_will_be_throttled_internal(info, NULL, NULL);
1880 }
1881 }
1882
1883 /*
1884 * Routine to increment I/O throttling counters maintained in the proc
1885 */
1886
1887 static void
1888 throttle_update_proc_stats(pid_t throttling_pid, int count)
1889 {
1890 proc_t throttling_proc;
1891 proc_t throttled_proc = current_proc();
1892
1893 /* The throttled_proc is always the current proc; so we are not concerned with refs */
1894 OSAddAtomic64(count, &(throttled_proc->was_throttled));
1895
1896 /* The throttling pid might have exited by now */
1897 throttling_proc = proc_find(throttling_pid);
1898 if (throttling_proc != PROC_NULL) {
1899 OSAddAtomic64(count, &(throttling_proc->did_throttle));
1900 proc_rele(throttling_proc);
1901 }
1902 }
1903
1904 /*
1905 * Block until woken up by the throttle timer or by a rethrottle call.
1906 * As long as we hold the throttle_lock while querying the throttle tier, we're
1907 * safe against seeing an old throttle tier after a rethrottle.
1908 */
1909 uint32_t
1910 throttle_lowpri_io(int sleep_amount)
1911 {
1912 uthread_t ut;
1913 struct _throttle_io_info_t *info;
1914 int throttle_type = 0;
1915 int mylevel = 0;
1916 int throttling_level = THROTTLE_LEVEL_NONE;
1917 int sleep_cnt = 0;
1918 uint32_t throttle_io_period_num = 0;
1919 boolean_t insert_tail = TRUE;
1920 boolean_t s;
1921
1922 ut = get_bsdthread_info(current_thread());
1923
1924 if (ut->uu_lowpri_window == 0) {
1925 return 0;
1926 }
1927
1928 info = ut->uu_throttle_info;
1929
1930 if (info == NULL) {
1931 ut->uu_throttle_bc = false;
1932 ut->uu_lowpri_window = 0;
1933 return 0;
1934 }
1935 lck_mtx_lock(&info->throttle_lock);
1936 assert(ut->uu_on_throttlelist < THROTTLE_LEVEL_THROTTLED);
1937
1938 if (sleep_amount == 0) {
1939 goto done;
1940 }
1941
1942 if (sleep_amount == 1 && !ut->uu_throttle_bc) {
1943 sleep_amount = 0;
1944 }
1945
1946 throttle_io_period_num = info->throttle_io_period_num;
1947
1948 ut->uu_was_rethrottled = false;
1949
1950 while ((throttle_type = throttle_io_will_be_throttled_internal(info, &mylevel, &throttling_level))) {
1951 if (throttle_type == THROTTLE_ENGAGED) {
1952 if (sleep_amount == 0) {
1953 break;
1954 }
1955 if (info->throttle_io_period_num < throttle_io_period_num) {
1956 break;
1957 }
1958 if ((info->throttle_io_period_num - throttle_io_period_num) >= (uint32_t)sleep_amount) {
1959 break;
1960 }
1961 }
1962 /*
1963 * keep the same position in the list if "rethrottle_thread" changes our throttle level and
1964 * then puts us back to the original level before we get a chance to run
1965 */
1966 if (ut->uu_on_throttlelist >= THROTTLE_LEVEL_THROTTLED && ut->uu_on_throttlelist != mylevel) {
1967 /*
1968 * must have been awakened via "rethrottle_thread" (the timer pulls us off the list)
1969 * and we've changed our throttling level, so pull ourselves off of the appropriate list
1970 * and make sure we get put on the tail of the new list since we're starting anew w/r to
1971 * the throttling engine
1972 */
1973 TAILQ_REMOVE(&info->throttle_uthlist[ut->uu_on_throttlelist], ut, uu_throttlelist);
1974 ut->uu_on_throttlelist = THROTTLE_LEVEL_NONE;
1975 insert_tail = TRUE;
1976 }
1977 if (ut->uu_on_throttlelist < THROTTLE_LEVEL_THROTTLED) {
1978 if (throttle_add_to_list(info, ut, mylevel, insert_tail) == THROTTLE_LEVEL_END) {
1979 goto done;
1980 }
1981 }
1982 assert(throttling_level >= THROTTLE_LEVEL_START && throttling_level <= THROTTLE_LEVEL_END);
1983
1984 s = ml_set_interrupts_enabled(FALSE);
1985 lck_spin_lock(&ut->uu_rethrottle_lock);
1986
1987 /*
1988 * this is the critical section w/r to our interaction
1989 * with "rethrottle_thread"
1990 */
1991 if (ut->uu_was_rethrottled) {
1992 lck_spin_unlock(&ut->uu_rethrottle_lock);
1993 ml_set_interrupts_enabled(s);
1994 lck_mtx_yield(&info->throttle_lock);
1995
1996 KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_FSRW, 103)), thread_tid(ut->uu_thread), ut->uu_on_throttlelist, 0, 0, 0);
1997
1998 ut->uu_was_rethrottled = false;
1999 continue;
2000 }
2001 KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_THROTTLE, PROCESS_THROTTLED)) | DBG_FUNC_NONE,
2002 info->throttle_last_IO_pid[throttling_level], throttling_level, proc_selfpid(), mylevel, 0);
2003
2004 if (sleep_cnt == 0) {
2005 KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_FSRW, 97)) | DBG_FUNC_START,
2006 throttle_windows_msecs[mylevel], info->throttle_io_periods[mylevel], info->throttle_io_count, 0, 0);
2007 throttled_count[mylevel]++;
2008 }
2009 ut->uu_wmesg = "throttle_lowpri_io";
2010
2011 assert_wait((caddr_t)&ut->uu_on_throttlelist, THREAD_UNINT);
2012
2013 ut->uu_is_throttled = true;
2014 lck_spin_unlock(&ut->uu_rethrottle_lock);
2015 ml_set_interrupts_enabled(s);
2016
2017 lck_mtx_unlock(&info->throttle_lock);
2018
2019 thread_block(THREAD_CONTINUE_NULL);
2020
2021 ut->uu_wmesg = NULL;
2022
2023 ut->uu_is_throttled = false;
2024 ut->uu_was_rethrottled = false;
2025
2026 lck_mtx_lock(&info->throttle_lock);
2027
2028 sleep_cnt++;
2029
2030 if (sleep_amount == 0) {
2031 insert_tail = FALSE;
2032 } else if (info->throttle_io_period_num < throttle_io_period_num ||
2033 (info->throttle_io_period_num - throttle_io_period_num) >= (uint32_t)sleep_amount) {
2034 insert_tail = FALSE;
2035 sleep_amount = 0;
2036 }
2037 }
2038 done:
2039 if (ut->uu_on_throttlelist >= THROTTLE_LEVEL_THROTTLED) {
2040 TAILQ_REMOVE(&info->throttle_uthlist[ut->uu_on_throttlelist], ut, uu_throttlelist);
2041 ut->uu_on_throttlelist = THROTTLE_LEVEL_NONE;
2042 }
2043 lck_mtx_unlock(&info->throttle_lock);
2044
2045 if (sleep_cnt) {
2046 KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_FSRW, 97)) | DBG_FUNC_END,
2047 throttle_windows_msecs[mylevel], info->throttle_io_periods[mylevel], info->throttle_io_count, 0, 0);
2048 /*
2049 * We update the stats for the last pid which opened a throttle window for the throttled thread.
2050 * This might not be completely accurate since the multiple throttles seen by the lower tier pid
2051 * might have been caused by various higher prio pids. However, updating these stats accurately
2052 * means doing a proc_find while holding the throttle lock which leads to deadlock.
2053 */
2054 throttle_update_proc_stats(info->throttle_last_IO_pid[throttling_level], sleep_cnt);
2055 }
2056
2057 ut->uu_throttle_info = NULL;
2058 ut->uu_throttle_bc = false;
2059 ut->uu_lowpri_window = 0;
2060
2061 throttle_info_rel(info);
2062
2063 return sleep_cnt;
2064 }
2065
2066 /*
2067 * returns TRUE if the throttle_lowpri_io called with the same sleep_amount would've slept
2068 * This function mimics the most of the throttle_lowpri_io checks but without actual sleeping
2069 */
2070 int
2071 throttle_lowpri_io_will_be_throttled(int sleep_amount)
2072 {
2073 if (sleep_amount == 0) {
2074 return FALSE;
2075 }
2076
2077 uthread_t ut = get_bsdthread_info(current_thread());
2078 if (ut->uu_lowpri_window == 0) {
2079 return FALSE;
2080 }
2081
2082 struct _throttle_io_info_t *info = ut->uu_throttle_info;
2083 if (info == NULL) {
2084 return FALSE;
2085 }
2086
2087 lck_mtx_lock(&info->throttle_lock);
2088 assert(ut->uu_on_throttlelist < THROTTLE_LEVEL_THROTTLED);
2089
2090 if (sleep_amount == 1 && !ut->uu_throttle_bc) {
2091 sleep_amount = 0;
2092 }
2093
2094 int result = FALSE;
2095
2096 int throttle_type = throttle_io_will_be_throttled_internal(info, NULL, NULL);
2097 if (throttle_type > THROTTLE_DISENGAGED) {
2098 result = TRUE;
2099 if ((throttle_type == THROTTLE_ENGAGED) && (sleep_amount == 0)) {
2100 result = FALSE;
2101 }
2102 }
2103
2104 lck_mtx_unlock(&info->throttle_lock);
2105
2106 return result;
2107 }
2108
2109
2110 /*
2111 * KPI routine
2112 *
2113 * set a kernel thread's IO policy. policy can be:
2114 * IOPOL_NORMAL, IOPOL_THROTTLE, IOPOL_PASSIVE, IOPOL_UTILITY, IOPOL_STANDARD
2115 *
2116 * explanations about these policies are in the man page of setiopolicy_np
2117 */
2118 void
2119 throttle_set_thread_io_policy(int policy)
2120 {
2121 proc_set_thread_policy(current_thread(), TASK_POLICY_INTERNAL, TASK_POLICY_IOPOL, policy);
2122 }
2123
2124 int
2125 throttle_get_thread_effective_io_policy()
2126 {
2127 return proc_get_effective_thread_policy(current_thread(), TASK_POLICY_IO);
2128 }
2129
2130 void
2131 throttle_info_reset_window(uthread_t ut)
2132 {
2133 struct _throttle_io_info_t *info;
2134
2135 if (ut == NULL) {
2136 ut = get_bsdthread_info(current_thread());
2137 }
2138
2139 if ((info = ut->uu_throttle_info)) {
2140 throttle_info_rel(info);
2141
2142 ut->uu_throttle_info = NULL;
2143 ut->uu_lowpri_window = 0;
2144 ut->uu_throttle_bc = false;
2145 }
2146 }
2147
2148 static
2149 void
2150 throttle_info_set_initial_window(uthread_t ut, struct _throttle_io_info_t *info, boolean_t BC_throttle, boolean_t isssd)
2151 {
2152 if (lowpri_throttle_enabled == 0 || info->throttle_disabled) {
2153 return;
2154 }
2155
2156 if (info->throttle_io_periods == 0) {
2157 throttle_init_throttle_period(info, isssd);
2158 }
2159 if (ut->uu_throttle_info == NULL) {
2160 ut->uu_throttle_info = info;
2161 throttle_info_ref(info);
2162 DEBUG_ALLOC_THROTTLE_INFO("updating info = %p\n", info, info );
2163
2164 ut->uu_lowpri_window = 1;
2165 ut->uu_throttle_bc = BC_throttle;
2166 }
2167 }
2168
2169 /*
2170 * Update inflight IO count and throttling window
2171 * Should be called when an IO is done
2172 *
2173 * Only affects IO that was sent through spec_strategy
2174 */
2175 void
2176 throttle_info_end_io(buf_t bp)
2177 {
2178 mount_t mp;
2179 struct bufattr *bap;
2180 struct _throttle_io_info_t *info;
2181 int io_tier;
2182
2183 bap = &bp->b_attr;
2184 if (!ISSET(bap->ba_flags, BA_STRATEGY_TRACKED_IO)) {
2185 return;
2186 }
2187 CLR(bap->ba_flags, BA_STRATEGY_TRACKED_IO);
2188
2189 mp = buf_vnode(bp)->v_mount;
2190 if (mp != NULL) {
2191 info = &_throttle_io_info[mp->mnt_devbsdunit];
2192 } else {
2193 info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
2194 }
2195
2196 io_tier = GET_BUFATTR_IO_TIER(bap);
2197 if (ISSET(bap->ba_flags, BA_IO_TIER_UPGRADE)) {
2198 io_tier--;
2199 }
2200
2201 throttle_info_end_io_internal(info, io_tier);
2202 }
2203
2204 /*
2205 * Decrement inflight count initially incremented by throttle_info_update_internal
2206 */
2207 static
2208 void
2209 throttle_info_end_io_internal(struct _throttle_io_info_t *info, int throttle_level)
2210 {
2211 if (throttle_level == THROTTLE_LEVEL_NONE) {
2212 return;
2213 }
2214
2215 microuptime(&info->throttle_window_start_timestamp[throttle_level]);
2216 OSDecrementAtomic(&info->throttle_inflight_count[throttle_level]);
2217 assert(info->throttle_inflight_count[throttle_level] >= 0);
2218 }
2219
2220 /*
2221 * If inflight is TRUE and bap is NULL then the caller is responsible for calling
2222 * throttle_info_end_io_internal to avoid leaking in-flight I/O.
2223 */
2224 static
2225 int
2226 throttle_info_update_internal(struct _throttle_io_info_t *info, uthread_t ut, int flags, boolean_t isssd, boolean_t inflight, struct bufattr *bap)
2227 {
2228 int thread_throttle_level;
2229
2230 if (lowpri_throttle_enabled == 0 || info->throttle_disabled) {
2231 return THROTTLE_LEVEL_NONE;
2232 }
2233
2234 if (ut == NULL) {
2235 ut = get_bsdthread_info(current_thread());
2236 }
2237
2238 if (bap && inflight && !ut->uu_throttle_bc) {
2239 thread_throttle_level = GET_BUFATTR_IO_TIER(bap);
2240 if (ISSET(bap->ba_flags, BA_IO_TIER_UPGRADE)) {
2241 thread_throttle_level--;
2242 }
2243 } else {
2244 thread_throttle_level = throttle_get_thread_throttle_level(ut);
2245 }
2246
2247 if (thread_throttle_level != THROTTLE_LEVEL_NONE) {
2248 if (!ISSET(flags, B_PASSIVE)) {
2249 info->throttle_last_IO_pid[thread_throttle_level] = proc_selfpid();
2250 if (inflight && !ut->uu_throttle_bc) {
2251 if (NULL != bap) {
2252 SET(bap->ba_flags, BA_STRATEGY_TRACKED_IO);
2253 }
2254 OSIncrementAtomic(&info->throttle_inflight_count[thread_throttle_level]);
2255 } else {
2256 microuptime(&info->throttle_window_start_timestamp[thread_throttle_level]);
2257 }
2258 KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_THROTTLE, OPEN_THROTTLE_WINDOW)) | DBG_FUNC_NONE,
2259 current_proc()->p_pid, thread_throttle_level, 0, 0, 0);
2260 }
2261 microuptime(&info->throttle_last_IO_timestamp[thread_throttle_level]);
2262 }
2263
2264
2265 if (thread_throttle_level >= THROTTLE_LEVEL_THROTTLED) {
2266 /*
2267 * I'd really like to do the IOSleep here, but
2268 * we may be holding all kinds of filesystem related locks
2269 * and the pages for this I/O marked 'busy'...
2270 * we don't want to cause a normal task to block on
2271 * one of these locks while we're throttling a task marked
2272 * for low priority I/O... we'll mark the uthread and
2273 * do the delay just before we return from the system
2274 * call that triggered this I/O or from vnode_pagein
2275 */
2276 OSAddAtomic(1, &info->throttle_io_count);
2277
2278 throttle_info_set_initial_window(ut, info, FALSE, isssd);
2279 }
2280
2281 return thread_throttle_level;
2282 }
2283
2284 void *
2285 throttle_info_update_by_mount(mount_t mp)
2286 {
2287 struct _throttle_io_info_t *info;
2288 uthread_t ut;
2289 boolean_t isssd = FALSE;
2290
2291 ut = get_bsdthread_info(current_thread());
2292
2293 if (mp != NULL) {
2294 if (disk_conditioner_mount_is_ssd(mp)) {
2295 isssd = TRUE;
2296 }
2297 info = &_throttle_io_info[mp->mnt_devbsdunit];
2298 } else {
2299 info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
2300 }
2301
2302 if (!ut->uu_lowpri_window) {
2303 throttle_info_set_initial_window(ut, info, FALSE, isssd);
2304 }
2305
2306 return info;
2307 }
2308
2309
2310 /*
2311 * KPI routine
2312 *
2313 * this is usually called before every I/O, used for throttled I/O
2314 * book keeping. This routine has low overhead and does not sleep
2315 */
2316 void
2317 throttle_info_update(void *throttle_info, int flags)
2318 {
2319 if (throttle_info) {
2320 throttle_info_update_internal(throttle_info, NULL, flags, FALSE, FALSE, NULL);
2321 }
2322 }
2323
2324 /*
2325 * KPI routine
2326 *
2327 * this is usually called before every I/O, used for throttled I/O
2328 * book keeping. This routine has low overhead and does not sleep
2329 */
2330 void
2331 throttle_info_update_by_mask(void *throttle_info_handle, int flags)
2332 {
2333 void *throttle_info = throttle_info_handle;
2334
2335 /*
2336 * for now we only use the lowest bit of the throttle mask, so the
2337 * handle is the same as the throttle_info. Later if we store a
2338 * set of throttle infos in the handle, we will want to loop through
2339 * them and call throttle_info_update in a loop
2340 */
2341 throttle_info_update(throttle_info, flags);
2342 }
2343 /*
2344 * KPI routine
2345 *
2346 * This routine marks the throttle info as disabled. Used for mount points which
2347 * support I/O scheduling.
2348 */
2349
2350 void
2351 throttle_info_disable_throttle(int devno, boolean_t isfusion)
2352 {
2353 struct _throttle_io_info_t *info;
2354
2355 if (devno < 0 || devno >= LOWPRI_MAX_NUM_DEV) {
2356 panic("Illegal devno (%d) passed into throttle_info_disable_throttle()", devno);
2357 }
2358
2359 info = &_throttle_io_info[devno];
2360 // don't disable software throttling on devices that are part of a fusion device
2361 // and override the software throttle periods to use HDD periods
2362 if (isfusion) {
2363 info->throttle_is_fusion_with_priority = isfusion;
2364 throttle_init_throttle_period(info, FALSE);
2365 }
2366 info->throttle_disabled = !info->throttle_is_fusion_with_priority;
2367 return;
2368 }
2369
2370
2371 /*
2372 * KPI routine (private)
2373 * Called to determine if this IO is being throttled to this level so that it can be treated specially
2374 */
2375 int
2376 throttle_info_io_will_be_throttled(void * throttle_info, int policy)
2377 {
2378 struct _throttle_io_info_t *info = throttle_info;
2379 struct timeval elapsed;
2380 uint64_t elapsed_msecs;
2381 int throttle_level;
2382 int thread_throttle_level;
2383
2384 switch (policy) {
2385 case IOPOL_THROTTLE:
2386 thread_throttle_level = THROTTLE_LEVEL_TIER3;
2387 break;
2388 case IOPOL_UTILITY:
2389 thread_throttle_level = THROTTLE_LEVEL_TIER2;
2390 break;
2391 case IOPOL_STANDARD:
2392 thread_throttle_level = THROTTLE_LEVEL_TIER1;
2393 break;
2394 default:
2395 thread_throttle_level = THROTTLE_LEVEL_TIER0;
2396 break;
2397 }
2398 for (throttle_level = THROTTLE_LEVEL_START; throttle_level < thread_throttle_level; throttle_level++) {
2399 if (info->throttle_inflight_count[throttle_level]) {
2400 break;
2401 }
2402
2403 microuptime(&elapsed);
2404 timevalsub(&elapsed, &info->throttle_window_start_timestamp[throttle_level]);
2405 elapsed_msecs = (uint64_t)elapsed.tv_sec * (uint64_t)1000 + (elapsed.tv_usec / 1000);
2406
2407 if (elapsed_msecs < (uint64_t)throttle_windows_msecs[thread_throttle_level]) {
2408 break;
2409 }
2410 }
2411 if (throttle_level >= thread_throttle_level) {
2412 /*
2413 * we're beyond all of the throttle windows
2414 * so go ahead and treat as normal I/O
2415 */
2416 return THROTTLE_DISENGAGED;
2417 }
2418 /*
2419 * we're in the throttle window
2420 */
2421 return THROTTLE_ENGAGED;
2422 }
2423
2424 int
2425 throttle_lowpri_window(void)
2426 {
2427 struct uthread *ut = get_bsdthread_info(current_thread());
2428 return ut->uu_lowpri_window;
2429 }
2430
2431
2432 #if CONFIG_IOSCHED
2433 int upl_get_cached_tier(void *);
2434 #endif
2435
2436 #if CONFIG_PHYS_WRITE_ACCT
2437 extern thread_t pm_sync_thread;
2438 #endif /* CONFIG_PHYS_WRITE_ACCT */
2439
2440 int
2441 spec_strategy(struct vnop_strategy_args *ap)
2442 {
2443 buf_t bp;
2444 int bflags;
2445 int io_tier;
2446 int passive;
2447 dev_t bdev;
2448 uthread_t ut;
2449 mount_t mp;
2450 struct bufattr *bap;
2451 int strategy_ret;
2452 struct _throttle_io_info_t *throttle_info;
2453 boolean_t isssd = FALSE;
2454 boolean_t inflight = FALSE;
2455 boolean_t upgrade = FALSE;
2456 int code = 0;
2457
2458 #if CONFIG_DELAY_IDLE_SLEEP
2459 proc_t curproc = current_proc();
2460 #endif /* CONFIG_DELAY_IDLE_SLEEP */
2461
2462 bp = ap->a_bp;
2463 bdev = buf_device(bp);
2464 mp = buf_vnode(bp)->v_mount;
2465 bap = &bp->b_attr;
2466
2467 #if CONFIG_PHYS_WRITE_ACCT
2468 if (current_thread() == pm_sync_thread) {
2469 OSAddAtomic64(buf_count(bp), (SInt64 *)&(kernel_pm_writes));
2470 }
2471 #endif /* CONFIG_PHYS_WRITE_ACCT */
2472
2473 #if CONFIG_IOSCHED
2474 if (bp->b_flags & B_CLUSTER) {
2475 io_tier = upl_get_cached_tier(bp->b_upl);
2476
2477 if (io_tier == -1) {
2478 io_tier = throttle_get_io_policy(&ut);
2479 }
2480 #if DEVELOPMENT || DEBUG
2481 else {
2482 int my_io_tier = throttle_get_io_policy(&ut);
2483
2484 if (io_tier != my_io_tier) {
2485 KERNEL_DEBUG_CONSTANT((FSDBG_CODE(DBG_THROTTLE, IO_TIER_UPL_MISMATCH)) | DBG_FUNC_NONE, buf_kernel_addrperm_addr(bp), my_io_tier, io_tier, 0, 0);
2486 }
2487 }
2488 #endif
2489 } else {
2490 io_tier = throttle_get_io_policy(&ut);
2491 }
2492 #else
2493 io_tier = throttle_get_io_policy(&ut);
2494 #endif
2495 passive = throttle_get_passive_io_policy(&ut);
2496
2497 /*
2498 * Mark if the I/O was upgraded by throttle_get_thread_throttle_level
2499 * while preserving the original issued tier (throttle_get_io_policy
2500 * does not return upgraded tiers)
2501 */
2502 if (mp && io_tier > throttle_get_thread_throttle_level_internal(ut, io_tier)) {
2503 #if CONFIG_IOSCHED
2504 if (!(mp->mnt_ioflags & MNT_IOFLAGS_IOSCHED_SUPPORTED)) {
2505 upgrade = TRUE;
2506 }
2507 #else /* CONFIG_IOSCHED */
2508 upgrade = TRUE;
2509 #endif /* CONFIG_IOSCHED */
2510 }
2511
2512 if (bp->b_flags & B_META) {
2513 bap->ba_flags |= BA_META;
2514 }
2515
2516 #if CONFIG_IOSCHED
2517 /*
2518 * For I/O Scheduling, we currently do not have a way to track and expedite metadata I/Os.
2519 * To ensure we dont get into priority inversions due to metadata I/Os, we use the following rules:
2520 * For metadata reads, ceil all I/Os to IOSCHED_METADATA_TIER & mark them passive if the I/O tier was upgraded
2521 * For metadata writes, unconditionally mark them as IOSCHED_METADATA_TIER and passive
2522 */
2523 if (bap->ba_flags & BA_META) {
2524 if ((mp && (mp->mnt_ioflags & MNT_IOFLAGS_IOSCHED_SUPPORTED)) || (bap->ba_flags & BA_IO_SCHEDULED)) {
2525 if (bp->b_flags & B_READ) {
2526 if (io_tier > IOSCHED_METADATA_TIER) {
2527 io_tier = IOSCHED_METADATA_TIER;
2528 passive = 1;
2529 }
2530 } else {
2531 io_tier = IOSCHED_METADATA_TIER;
2532 passive = 1;
2533 }
2534 }
2535 }
2536 #endif /* CONFIG_IOSCHED */
2537
2538 SET_BUFATTR_IO_TIER(bap, io_tier);
2539
2540 if (passive) {
2541 bp->b_flags |= B_PASSIVE;
2542 bap->ba_flags |= BA_PASSIVE;
2543 }
2544
2545 #if CONFIG_DELAY_IDLE_SLEEP
2546 if ((curproc != NULL) && ((curproc->p_flag & P_DELAYIDLESLEEP) == P_DELAYIDLESLEEP)) {
2547 bap->ba_flags |= BA_DELAYIDLESLEEP;
2548 }
2549 #endif /* CONFIG_DELAY_IDLE_SLEEP */
2550
2551 bflags = bp->b_flags;
2552
2553 if (((bflags & B_READ) == 0) && ((bflags & B_ASYNC) == 0)) {
2554 bufattr_markquickcomplete(bap);
2555 }
2556
2557 if (bflags & B_READ) {
2558 code |= DKIO_READ;
2559 }
2560 if (bflags & B_ASYNC) {
2561 code |= DKIO_ASYNC;
2562 }
2563
2564 if (bap->ba_flags & BA_META) {
2565 code |= DKIO_META;
2566 } else if (bflags & B_PAGEIO) {
2567 code |= DKIO_PAGING;
2568 }
2569
2570 if (io_tier != 0) {
2571 code |= DKIO_THROTTLE;
2572 }
2573
2574 code |= ((io_tier << DKIO_TIER_SHIFT) & DKIO_TIER_MASK);
2575
2576 if (bflags & B_PASSIVE) {
2577 code |= DKIO_PASSIVE;
2578 }
2579
2580 if (bap->ba_flags & BA_NOCACHE) {
2581 code |= DKIO_NOCACHE;
2582 }
2583
2584 if (upgrade) {
2585 code |= DKIO_TIER_UPGRADE;
2586 SET(bap->ba_flags, BA_IO_TIER_UPGRADE);
2587 }
2588
2589 if (kdebug_enable) {
2590 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_COMMON, FSDBG_CODE(DBG_DKRW, code) | DBG_FUNC_NONE,
2591 buf_kernel_addrperm_addr(bp), bdev, buf_blkno(bp), buf_count(bp), 0);
2592 }
2593
2594 thread_update_io_stats(current_thread(), buf_count(bp), code);
2595
2596 if (mp != NULL) {
2597 if (disk_conditioner_mount_is_ssd(mp)) {
2598 isssd = TRUE;
2599 }
2600 /*
2601 * Partially initialized mounts don't have a final devbsdunit and should not be tracked.
2602 * Verify that devbsdunit is initialized (non-zero) or that 0 is the correct initialized value
2603 * (mnt_throttle_mask is initialized and num_trailing_0 would be 0)
2604 */
2605 if (mp->mnt_devbsdunit || (mp->mnt_throttle_mask != LOWPRI_MAX_NUM_DEV - 1 && mp->mnt_throttle_mask & 0x1)) {
2606 inflight = TRUE;
2607 }
2608 throttle_info = &_throttle_io_info[mp->mnt_devbsdunit];
2609 } else {
2610 throttle_info = &_throttle_io_info[LOWPRI_MAX_NUM_DEV - 1];
2611 }
2612
2613 throttle_info_update_internal(throttle_info, ut, bflags, isssd, inflight, bap);
2614
2615 if ((bflags & B_READ) == 0) {
2616 microuptime(&throttle_info->throttle_last_write_timestamp);
2617
2618 if (mp) {
2619 mp->mnt_last_write_issued_timestamp = throttle_info->throttle_last_write_timestamp;
2620 INCR_PENDING_IO(buf_count(bp), mp->mnt_pending_write_size);
2621 }
2622 } else if (mp) {
2623 INCR_PENDING_IO(buf_count(bp), mp->mnt_pending_read_size);
2624 }
2625 /*
2626 * The BootCache may give us special information about
2627 * the IO, so it returns special values that we check
2628 * for here.
2629 *
2630 * IO_SATISFIED_BY_CACHE
2631 * The read has been satisfied by the boot cache. Don't
2632 * throttle the thread unnecessarily.
2633 *
2634 * IO_SHOULD_BE_THROTTLED
2635 * The boot cache is playing back a playlist and this IO
2636 * cut through. Throttle it so we're not cutting through
2637 * the boot cache too often.
2638 *
2639 * Note that typical strategy routines are defined with
2640 * a void return so we'll get garbage here. In the
2641 * unlikely case the garbage matches our special return
2642 * value, it's not a big deal since we're only adjusting
2643 * the throttling delay.
2644 */
2645 #define IO_SATISFIED_BY_CACHE ((int)0xcafefeed)
2646 #define IO_SHOULD_BE_THROTTLED ((int)0xcafebeef)
2647 typedef int strategy_fcn_ret_t(struct buf *bp);
2648
2649 strategy_ret = (*(strategy_fcn_ret_t*)bdevsw[major(bdev)].d_strategy)(bp);
2650
2651 // disk conditioner needs to track when this I/O actually starts
2652 // which means track it after `strategy` which may include delays
2653 // from inflight I/Os
2654 microuptime(&bp->b_timestamp_tv);
2655
2656 if (IO_SATISFIED_BY_CACHE == strategy_ret) {
2657 /*
2658 * If this was a throttled IO satisfied by the boot cache,
2659 * don't delay the thread.
2660 */
2661 throttle_info_reset_window(ut);
2662 } else if (IO_SHOULD_BE_THROTTLED == strategy_ret) {
2663 /*
2664 * If the boot cache indicates this IO should be throttled,
2665 * delay the thread.
2666 */
2667 throttle_info_set_initial_window(ut, throttle_info, TRUE, isssd);
2668 }
2669 return 0;
2670 }
2671
2672
2673 /*
2674 * This is a noop, simply returning what one has been given.
2675 */
2676 int
2677 spec_blockmap(__unused struct vnop_blockmap_args *ap)
2678 {
2679 return ENOTSUP;
2680 }
2681
2682
2683 /*
2684 * Device close routine
2685 */
2686 int
2687 spec_close(struct vnop_close_args *ap)
2688 {
2689 struct vnode *vp = ap->a_vp;
2690 dev_t dev = vp->v_rdev;
2691 int error = 0;
2692 int flags = ap->a_fflag;
2693 struct proc *p = vfs_context_proc(ap->a_context);
2694 struct session *sessp;
2695
2696 switch (vp->v_type) {
2697 case VCHR:
2698 /*
2699 * Hack: a tty device that is a controlling terminal
2700 * has a reference from the session structure.
2701 * We cannot easily tell that a character device is
2702 * a controlling terminal, unless it is the closing
2703 * process' controlling terminal. In that case,
2704 * if the reference count is 1 (this is the very
2705 * last close)
2706 */
2707 sessp = proc_session(p);
2708 devsw_lock(dev, S_IFCHR);
2709 if (sessp != SESSION_NULL) {
2710 if (vp == sessp->s_ttyvp && vcount(vp) == 1) {
2711 struct tty *tp = TTY_NULL;
2712
2713 devsw_unlock(dev, S_IFCHR);
2714 session_lock(sessp);
2715 if (vp == sessp->s_ttyvp) {
2716 tp = SESSION_TP(sessp);
2717 sessp->s_ttyvp = NULL;
2718 sessp->s_ttyvid = 0;
2719 sessp->s_ttyp = TTY_NULL;
2720 sessp->s_ttypgrpid = NO_PID;
2721 }
2722 session_unlock(sessp);
2723
2724 if (tp != TTY_NULL) {
2725 ttyfree(tp);
2726 }
2727 devsw_lock(dev, S_IFCHR);
2728 }
2729 session_rele(sessp);
2730 }
2731
2732 if (--vp->v_specinfo->si_opencount < 0) {
2733 panic("negative open count (c, %u, %u)", major(dev), minor(dev));
2734 }
2735
2736 /*
2737 * close on last reference or on vnode revoke call
2738 */
2739 if (vcount(vp) == 0 || (flags & IO_REVOKE) != 0) {
2740 error = cdevsw[major(dev)].d_close(dev, flags, S_IFCHR, p);
2741 }
2742
2743 devsw_unlock(dev, S_IFCHR);
2744 break;
2745
2746 case VBLK:
2747 /*
2748 * If there is more than one outstanding open, don't
2749 * send the close to the device.
2750 */
2751 devsw_lock(dev, S_IFBLK);
2752 if (vcount(vp) > 1) {
2753 vp->v_specinfo->si_opencount--;
2754 devsw_unlock(dev, S_IFBLK);
2755 return 0;
2756 }
2757 devsw_unlock(dev, S_IFBLK);
2758
2759 /*
2760 * On last close of a block device (that isn't mounted)
2761 * we must invalidate any in core blocks, so that
2762 * we can, for instance, change floppy disks.
2763 */
2764 if ((error = spec_fsync_internal(vp, MNT_WAIT, ap->a_context))) {
2765 return error;
2766 }
2767
2768 error = buf_invalidateblks(vp, BUF_WRITE_DATA, 0, 0);
2769 if (error) {
2770 return error;
2771 }
2772
2773 devsw_lock(dev, S_IFBLK);
2774
2775 if (--vp->v_specinfo->si_opencount < 0) {
2776 panic("negative open count (b, %u, %u)", major(dev), minor(dev));
2777 }
2778
2779 if (vcount(vp) == 0) {
2780 error = bdevsw[major(dev)].d_close(dev, flags, S_IFBLK, p);
2781 }
2782
2783 devsw_unlock(dev, S_IFBLK);
2784 break;
2785
2786 default:
2787 panic("spec_close: not special");
2788 return EBADF;
2789 }
2790
2791 return error;
2792 }
2793
2794 /*
2795 * Return POSIX pathconf information applicable to special devices.
2796 */
2797 int
2798 spec_pathconf(struct vnop_pathconf_args *ap)
2799 {
2800 switch (ap->a_name) {
2801 case _PC_LINK_MAX:
2802 *ap->a_retval = LINK_MAX;
2803 return 0;
2804 case _PC_MAX_CANON:
2805 *ap->a_retval = MAX_CANON;
2806 return 0;
2807 case _PC_MAX_INPUT:
2808 *ap->a_retval = MAX_INPUT;
2809 return 0;
2810 case _PC_PIPE_BUF:
2811 *ap->a_retval = PIPE_BUF;
2812 return 0;
2813 case _PC_CHOWN_RESTRICTED:
2814 *ap->a_retval = 200112; /* _POSIX_CHOWN_RESTRICTED */
2815 return 0;
2816 case _PC_VDISABLE:
2817 *ap->a_retval = _POSIX_VDISABLE;
2818 return 0;
2819 default:
2820 return EINVAL;
2821 }
2822 /* NOTREACHED */
2823 }
2824
2825 /*
2826 * Special device failed operation
2827 */
2828 int
2829 spec_ebadf(__unused void *dummy)
2830 {
2831 return EBADF;
2832 }
2833
2834 /* Blktooff derives file offset from logical block number */
2835 int
2836 spec_blktooff(struct vnop_blktooff_args *ap)
2837 {
2838 struct vnode *vp = ap->a_vp;
2839
2840 switch (vp->v_type) {
2841 case VCHR:
2842 *ap->a_offset = (off_t)-1; /* failure */
2843 return ENOTSUP;
2844
2845 case VBLK:
2846 printf("spec_blktooff: not implemented for VBLK\n");
2847 *ap->a_offset = (off_t)-1; /* failure */
2848 return ENOTSUP;
2849
2850 default:
2851 panic("spec_blktooff type");
2852 }
2853 /* NOTREACHED */
2854
2855 return 0;
2856 }
2857
2858 /* Offtoblk derives logical block number from file offset */
2859 int
2860 spec_offtoblk(struct vnop_offtoblk_args *ap)
2861 {
2862 struct vnode *vp = ap->a_vp;
2863
2864 switch (vp->v_type) {
2865 case VCHR:
2866 *ap->a_lblkno = (daddr64_t)-1; /* failure */
2867 return ENOTSUP;
2868
2869 case VBLK:
2870 printf("spec_offtoblk: not implemented for VBLK\n");
2871 *ap->a_lblkno = (daddr64_t)-1; /* failure */
2872 return ENOTSUP;
2873
2874 default:
2875 panic("spec_offtoblk type");
2876 }
2877 /* NOTREACHED */
2878
2879 return 0;
2880 }
2881
2882 static void filt_specdetach(struct knote *kn);
2883 static int filt_specevent(struct knote *kn, long hint);
2884 static int filt_spectouch(struct knote *kn, struct kevent_qos_s *kev);
2885 static int filt_specprocess(struct knote *kn, struct kevent_qos_s *kev);
2886 static int filt_specpeek(struct knote *kn);
2887
2888 SECURITY_READ_ONLY_EARLY(struct filterops) spec_filtops = {
2889 .f_isfd = 1,
2890 .f_attach = filt_specattach,
2891 .f_detach = filt_specdetach,
2892 .f_event = filt_specevent,
2893 .f_touch = filt_spectouch,
2894 .f_process = filt_specprocess,
2895 .f_peek = filt_specpeek
2896 };
2897
2898
2899 /*
2900 * Given a waitq that is assumed to be embedded within a selinfo structure,
2901 * return the containing selinfo structure. While 'wq' is not really a queue
2902 * element, this macro simply does the offset_of calculation to get back to a
2903 * containing struct given the struct type and member name.
2904 */
2905 #define selinfo_from_waitq(wq) \
2906 qe_element((wq), struct selinfo, si_waitq)
2907
2908 static int
2909 spec_knote_select_and_link(struct knote *kn)
2910 {
2911 uthread_t uth;
2912 vfs_context_t ctx;
2913 vnode_t vp;
2914 struct waitq_set *old_wqs;
2915 uint64_t rsvd, rsvd_arg;
2916 uint64_t *rlptr = NULL;
2917 struct selinfo *si = NULL;
2918 int selres = 0;
2919
2920 uth = get_bsdthread_info(current_thread());
2921
2922 ctx = vfs_context_current();
2923 vp = (vnode_t)kn->kn_fp->fp_glob->fg_data;
2924
2925 int error = vnode_getwithvid(vp, vnode_vid(vp));
2926 if (error != 0) {
2927 knote_set_error(kn, ENOENT);
2928 return 0;
2929 }
2930
2931 /*
2932 * This function may be called many times to link or re-link the
2933 * underlying vnode to the kqueue. If we've already linked the two,
2934 * we will have a valid kn_hook_waitqid which ties us to the underlying
2935 * device's waitq via a the waitq's prepost table object. However,
2936 * devices can abort any select action by calling selthreadclear().
2937 * This is OK because the table object will be invalidated by the
2938 * driver (through a call to selthreadclear), so any attempt to access
2939 * the associated waitq will fail because the table object is invalid.
2940 *
2941 * Even if we've already registered, we need to pass a pointer
2942 * to a reserved link structure. Otherwise, selrecord() will
2943 * infer that we're in the second pass of select() and won't
2944 * actually do anything!
2945 */
2946 rsvd = rsvd_arg = waitq_link_reserve(NULL);
2947 rlptr = (void *)&rsvd_arg;
2948
2949 /*
2950 * Trick selrecord() into hooking kqueue's wait queue set into the device's
2951 * selinfo wait queue.
2952 */
2953 old_wqs = uth->uu_wqset;
2954 uth->uu_wqset = &(knote_get_kq(kn)->kq_wqs);
2955
2956 /*
2957 * Be sure that the waitq set is linked
2958 * before calling select to avoid possible
2959 * allocation under spinlocks.
2960 */
2961 waitq_set_lazy_init_link(uth->uu_wqset);
2962
2963 /*
2964 * Now these are the laws of VNOP_SELECT, as old and as true as the sky,
2965 * And the device that shall keep it may prosper, but the device that shall
2966 * break it must receive ENODEV:
2967 *
2968 * 1. Take a lock to protect against other selects on the same vnode.
2969 * 2. Return 1 if data is ready to be read.
2970 * 3. Return 0 and call `selrecord` on a handy `selinfo` structure if there
2971 * is no data.
2972 * 4. Call `selwakeup` when the vnode has an active `selrecord` and data
2973 * can be read or written (depending on the seltype).
2974 * 5. If there's a `selrecord` and no corresponding `selwakeup`, but the
2975 * vnode is going away, call `selthreadclear`.
2976 */
2977 selres = VNOP_SELECT(vp, knote_get_seltype(kn), 0, rlptr, ctx);
2978 uth->uu_wqset = old_wqs;
2979
2980 /*
2981 * Make sure to cleanup the reserved link - this guards against
2982 * drivers that may not actually call selrecord().
2983 */
2984 waitq_link_release(rsvd);
2985 if (rsvd != rsvd_arg) {
2986 /* The driver / handler called selrecord() */
2987 struct waitq *wq;
2988 memcpy(&wq, rlptr, sizeof(void *));
2989
2990 /*
2991 * The waitq is part of the selinfo structure managed by the
2992 * driver. For certain drivers, we want to hook the knote into
2993 * the selinfo structure's si_note field so selwakeup can call
2994 * KNOTE.
2995 */
2996 si = selinfo_from_waitq(wq);
2997
2998 /*
2999 * The waitq_get_prepost_id() function will (potentially)
3000 * allocate a prepost table object for the waitq and return
3001 * the table object's ID to us. It will also set the
3002 * waitq_prepost_id field within the waitq structure.
3003 *
3004 * We can just overwrite kn_hook_waitqid because it's simply a
3005 * table ID used to grab a reference when needed.
3006 *
3007 * We have a reference on the vnode, so we know that the
3008 * device won't go away while we get this ID.
3009 *
3010 * Note: on 32bit this field is 32bit only.
3011 */
3012 kn->kn_hook_waitqid = (typeof(kn->kn_hook_waitqid))waitq_get_prepost_id(wq);
3013 } else if (selres == 0) {
3014 /*
3015 * The device indicated that there's no data to read, but didn't call
3016 * `selrecord`. Nothing will be notified of changes to this vnode, so
3017 * return an error back to user space, to make it clear that the knote
3018 * is not attached.
3019 */
3020 knote_set_error(kn, ENODEV);
3021 }
3022
3023 vnode_put(vp);
3024
3025 return selres;
3026 }
3027
3028 static int
3029 filt_spec_common(struct knote *kn, struct kevent_qos_s *kev, int selres)
3030 {
3031 int64_t data;
3032 int ret;
3033
3034 if (kn->kn_vnode_use_ofst) {
3035 if (kn->kn_fp->fp_glob->fg_offset >= (uint32_t)selres) {
3036 data = 0;
3037 } else {
3038 data = ((uint32_t)selres) - kn->kn_fp->fp_glob->fg_offset;
3039 }
3040 } else {
3041 data = selres;
3042 }
3043
3044 ret = data >= knote_low_watermark(kn);
3045
3046 if (ret && kev) {
3047 knote_fill_kevent(kn, kev, data);
3048 }
3049
3050 return ret;
3051 }
3052
3053 static int
3054 filt_specattach(struct knote *kn, __unused struct kevent_qos_s *kev)
3055 {
3056 vnode_t vp;
3057 dev_t dev;
3058
3059 vp = (vnode_t)kn->kn_fp->fp_glob->fg_data; /* Already have iocount, and vnode is alive */
3060
3061 assert(vnode_ischr(vp));
3062
3063 dev = vnode_specrdev(vp);
3064
3065 /*
3066 * For a few special kinds of devices, we can attach knotes with
3067 * no restrictions because their "select" vectors return the amount
3068 * of data available. Others require an explicit NOTE_LOWAT with
3069 * data of 1, indicating that the caller doesn't care about actual
3070 * data counts, just an indication that the device has data.
3071 */
3072 if (!kn->kn_vnode_kqok &&
3073 ((kn->kn_sfflags & NOTE_LOWAT) == 0 || kn->kn_sdata != 1)) {
3074 knote_set_error(kn, EINVAL);
3075 return 0;
3076 }
3077
3078 /*
3079 * This forces the select fallback to call through VNOP_SELECT and hook
3080 * up selinfo on every filter routine.
3081 *
3082 * Pseudo-terminal controllers are opted out of native kevent support --
3083 * remove this when they get their own EVFILTID.
3084 */
3085 if (cdevsw_flags[major(dev)] & CDEVSW_IS_PTC) {
3086 kn->kn_vnode_kqok = 0;
3087 }
3088
3089 kn->kn_filtid = EVFILTID_SPEC;
3090 kn->kn_hook_waitqid = 0;
3091
3092 knote_markstayactive(kn);
3093 return spec_knote_select_and_link(kn);
3094 }
3095
3096 static void
3097 filt_specdetach(struct knote *kn)
3098 {
3099 knote_clearstayactive(kn);
3100
3101 /*
3102 * This is potentially tricky: the device's selinfo waitq that was
3103 * tricked into being part of this knote's waitq set may not be a part
3104 * of any other set, and the device itself may have revoked the memory
3105 * in which the waitq was held. We use the knote's kn_hook_waitqid field
3106 * to keep the ID of the waitq's prepost table object. This
3107 * object keeps a pointer back to the waitq, and gives us a safe way
3108 * to decouple the dereferencing of driver allocated memory: if the
3109 * driver goes away (taking the waitq with it) then the prepost table
3110 * object will be invalidated. The waitq details are handled in the
3111 * waitq API invoked here.
3112 */
3113 if (kn->kn_hook_waitqid) {
3114 waitq_unlink_by_prepost_id(kn->kn_hook_waitqid, &(knote_get_kq(kn)->kq_wqs));
3115 kn->kn_hook_waitqid = 0;
3116 }
3117 }
3118
3119 static int
3120 filt_specevent(struct knote *kn, __unused long hint)
3121 {
3122 /*
3123 * Nothing should call knote or knote_vanish on this knote.
3124 */
3125 panic("filt_specevent(%p)", kn);
3126 return 0;
3127 }
3128
3129 static int
3130 filt_spectouch(struct knote *kn, struct kevent_qos_s *kev)
3131 {
3132 kn->kn_sdata = kev->data;
3133 kn->kn_sfflags = kev->fflags;
3134
3135 if (kev->flags & EV_ENABLE) {
3136 return spec_knote_select_and_link(kn);
3137 }
3138
3139 return 0;
3140 }
3141
3142 static int
3143 filt_specprocess(struct knote *kn, struct kevent_qos_s *kev)
3144 {
3145 vnode_t vp;
3146 uthread_t uth;
3147 vfs_context_t ctx;
3148 int res;
3149 int selres;
3150 int error;
3151
3152 uth = get_bsdthread_info(current_thread());
3153 ctx = vfs_context_current();
3154 vp = (vnode_t)kn->kn_fp->fp_glob->fg_data;
3155
3156 error = vnode_getwithvid(vp, vnode_vid(vp));
3157 if (error != 0) {
3158 kn->kn_flags |= (EV_EOF | EV_ONESHOT);
3159 knote_fill_kevent(kn, kev, 0);
3160 return 1;
3161 }
3162
3163 selres = spec_knote_select_and_link(kn);
3164 res = filt_spec_common(kn, kev, selres);
3165
3166 vnode_put(vp);
3167
3168 return res;
3169 }
3170
3171 static int
3172 filt_specpeek(struct knote *kn)
3173 {
3174 int selres = 0;
3175
3176 selres = spec_knote_select_and_link(kn);
3177 return filt_spec_common(kn, NULL, selres);
3178 }
mirror of XNU