bsd/vfs/vfs_disk_conditioner.c

   1 /*
   2  * Copyright (c) 2016 Apple Computer, Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <sys/fsctl.h>
  30 #include <stdbool.h>
  31 #include <sys/time.h>
  32 #include <sys/buf.h>
  33 #include <sys/mount_internal.h>
  34 #include <sys/vnode_internal.h>
  35 #include <sys/buf_internal.h>
  36
  37 #include <kern/kalloc.h>
  38
  39 #include <sys/kauth.h>
  40 #include <IOKit/IOBSD.h>
  41
  42 #include <vfs/vfs_disk_conditioner.h>
  43
  44 #define DISK_CONDITIONER_SET_ENTITLEMENT "com.apple.private.dmc.set"
  45
  46 // number of total blocks for a mount
  47 #define BLK_MAX(mp) ((mp->mnt_vfsstat.f_blocks * mp->mnt_vfsstat.f_bsize) / (mp->mnt_devblocksize))
  48
  49 // approx. time to spin up an idle HDD
  50 #define DISK_SPINUP_SEC (8)
  51
  52 // idle period until assumed disk spin down
  53 #define DISK_IDLE_SEC (10 * 60)
  54
  55 struct saved_mount_fields {
  56         uint32_t        mnt_maxreadcnt;         /* Max. byte count for read */
  57         uint32_t        mnt_maxwritecnt;        /* Max. byte count for write */
  58         uint32_t        mnt_segreadcnt;         /* Max. segment count for read */
  59         uint32_t        mnt_segwritecnt;        /* Max. segment count for write */
  60         uint32_t        mnt_ioqueue_depth;      /* the maxiumum number of commands a device can accept */
  61         uint32_t        mnt_ioscale;            /* scale the various throttles/limits imposed on the amount of I/O in flight */
  62 };
  63
  64 struct _disk_conditioner_info_t {
  65         disk_conditioner_info dcinfo; // all the original data from fsctl
  66         struct saved_mount_fields mnt_fields; // fields to restore in mount_t when conditioner is disabled
  67
  68         daddr64_t last_blkno; // approx. last transfered block for simulating seek times
  69         struct timeval last_io_timestamp; // the last time an I/O completed
  70 };
  71
  72 void disk_conditioner_delay(buf_t, int, int, uint64_t);
  73 void disk_conditioner_unmount(mount_t mp);
  74
  75 extern void throttle_info_mount_reset_period(mount_t, int isssd);
  76
  77 static double
  78 weighted_scale_factor(double scale)
  79 {
  80         // 0 to 1 increasing quickly from 0. This weights smaller blkdiffs higher to add a type of minimum latency
  81         // I would like to use log(10) / 2.0 + 1, but using different approximation due to no math library
  82         // y = (x-1)^3 + 1
  83         double x_m1 = scale - 1;
  84         return x_m1 * x_m1 * x_m1 + 1;
  85 }
  86
  87 void
  88 disk_conditioner_delay(buf_t bp, int extents, int total_size, uint64_t already_elapsed_usec)
  89 {
  90         mount_t mp;
  91         uint64_t delay_usec;
  92         daddr64_t blkdiff;
  93         daddr64_t last_blkno;
  94         double access_time_scale;
  95         struct _disk_conditioner_info_t *internal_info = NULL;
  96         disk_conditioner_info *info = NULL;
  97         struct timeval elapsed;
  98         struct timeval start;
  99         vnode_t vp;
 100
 101         vp = buf_vnode(bp);
 102         if (!vp) {
 103                 return;
 104         }
 105
 106         mp = vp->v_mount;
 107         if (!mp) {
 108                 return;
 109         }
 110
 111         internal_info = mp->mnt_disk_conditioner_info;
 112         if (!internal_info || !internal_info->dcinfo.enabled) {
 113                 return;
 114         }
 115         info = &(internal_info->dcinfo);
 116
 117         if (!info->is_ssd) {
 118                 // calculate approximate seek time based on difference in block number
 119                 last_blkno = internal_info->last_blkno;
 120                 blkdiff = bp->b_blkno > last_blkno ? bp->b_blkno - last_blkno : last_blkno - bp->b_blkno;
 121                 internal_info->last_blkno = bp->b_blkno + bp->b_bcount;
 122         } else {
 123                 blkdiff = BLK_MAX(mp);
 124         }
 125
 126         // scale access time by (distance in blocks from previous I/O / maximum blocks)
 127         access_time_scale = weighted_scale_factor((double)blkdiff / BLK_MAX(mp));
 128         // most cases should pass in extents==1 for optimal delay calculation, otherwise just multiply delay by extents
 129         delay_usec = (uint64_t)(((uint64_t)extents * info->access_time_usec) * access_time_scale);
 130
 131         if (info->read_throughput_mbps && (bp->b_flags & B_READ)) {
 132                 delay_usec += (uint64_t)(total_size / ((double)(info->read_throughput_mbps * 1024 * 1024 / 8) / USEC_PER_SEC));
 133         } else if (info->write_throughput_mbps && !(bp->b_flags & B_READ)) {
 134                 delay_usec += (uint64_t)(total_size / ((double)(info->write_throughput_mbps * 1024 * 1024 / 8) / USEC_PER_SEC));
 135         }
 136
 137         // try simulating disk spinup based on time since last I/O
 138         if (!info->is_ssd) {
 139                 microuptime(&elapsed);
 140                 timevalsub(&elapsed, &internal_info->last_io_timestamp);
 141                 // avoid this delay right after boot (assuming last_io_timestamp is 0 and disk is already spinning)
 142                 if (elapsed.tv_sec > DISK_IDLE_SEC && internal_info->last_io_timestamp.tv_sec != 0) {
 143                         delay_usec += DISK_SPINUP_SEC * USEC_PER_SEC;
 144                 }
 145         }
 146
 147         if (delay_usec <= already_elapsed_usec) {
 148                 microuptime(&internal_info->last_io_timestamp);
 149                 return;
 150         }
 151
 152         delay_usec -= already_elapsed_usec;
 153
 154         while (delay_usec) {
 155                 microuptime(&start);
 156                 delay(delay_usec);
 157                 microuptime(&elapsed);
 158                 timevalsub(&elapsed, &start);
 159                 if (elapsed.tv_sec * USEC_PER_SEC < delay_usec) {
 160                         delay_usec -= elapsed.tv_sec * USEC_PER_SEC;
 161                 } else {
 162                         break;
 163                 }
 164                 if ((uint64_t)elapsed.tv_usec < delay_usec) {
 165                         delay_usec -= elapsed.tv_usec;
 166                 } else {
 167                         break;
 168                 }
 169         }
 170
 171         microuptime(&internal_info->last_io_timestamp);
 172 }
 173
 174 int
 175 disk_conditioner_get_info(mount_t mp, disk_conditioner_info *uinfo)
 176 {
 177         struct _disk_conditioner_info_t *info;
 178
 179         if (!mp) {
 180                 return EINVAL;
 181         }
 182
 183         info = mp->mnt_disk_conditioner_info;
 184
 185         if (info) {
 186                 memcpy(uinfo, &(info->dcinfo), sizeof(disk_conditioner_info));
 187         }
 188
 189         return 0;
 190 }
 191
 192 static inline void
 193 disk_conditioner_restore_mount_fields(mount_t mp, struct saved_mount_fields *mnt_fields) {
 194         mp->mnt_maxreadcnt = mnt_fields->mnt_maxreadcnt;
 195         mp->mnt_maxwritecnt = mnt_fields->mnt_maxwritecnt;
 196         mp->mnt_segreadcnt = mnt_fields->mnt_segreadcnt;
 197         mp->mnt_segwritecnt = mnt_fields->mnt_segwritecnt;
 198         mp->mnt_ioqueue_depth = mnt_fields->mnt_ioqueue_depth;
 199         mp->mnt_ioscale = mnt_fields->mnt_ioscale;
 200 }
 201
 202 int
 203 disk_conditioner_set_info(mount_t mp, disk_conditioner_info *uinfo)
 204 {
 205         struct _disk_conditioner_info_t *internal_info;
 206         disk_conditioner_info *info;
 207         struct saved_mount_fields *mnt_fields;
 208
 209         if (!kauth_cred_issuser(kauth_cred_get()) || !IOTaskHasEntitlement(current_task(), DISK_CONDITIONER_SET_ENTITLEMENT)) {
 210                 return EPERM;
 211         }
 212
 213         if (!mp) {
 214                 return EINVAL;
 215         }
 216
 217         mount_lock(mp);
 218
 219         internal_info = mp->mnt_disk_conditioner_info;
 220         if (!internal_info) {
 221                 internal_info = mp->mnt_disk_conditioner_info = kalloc(sizeof(struct _disk_conditioner_info_t));
 222                 bzero(internal_info, sizeof(struct _disk_conditioner_info_t));
 223                 mnt_fields = &(internal_info->mnt_fields);
 224
 225                 /* save mount_t fields for restoration later */
 226                 mnt_fields->mnt_maxreadcnt = mp->mnt_maxreadcnt;
 227                 mnt_fields->mnt_maxwritecnt = mp->mnt_maxwritecnt;
 228                 mnt_fields->mnt_segreadcnt = mp->mnt_segreadcnt;
 229                 mnt_fields->mnt_segwritecnt = mp->mnt_segwritecnt;
 230                 mnt_fields->mnt_ioqueue_depth = mp->mnt_ioqueue_depth;
 231                 mnt_fields->mnt_ioscale = mp->mnt_ioscale;
 232         }
 233
 234         info = &(internal_info->dcinfo);
 235         mnt_fields = &(internal_info->mnt_fields);
 236
 237         if (!uinfo->enabled && info->enabled) {
 238                 /* disk conditioner is being disabled when already enabled */
 239                 disk_conditioner_restore_mount_fields(mp, mnt_fields);
 240         }
 241
 242         memcpy(info, uinfo, sizeof(disk_conditioner_info));
 243
 244         /* scale back based on hardware advertised limits */
 245         if (uinfo->ioqueue_depth == 0 || uinfo->ioqueue_depth > mnt_fields->mnt_ioqueue_depth) {
 246                 info->ioqueue_depth = mnt_fields->mnt_ioqueue_depth;
 247         }
 248         if (uinfo->maxreadcnt == 0 || uinfo->maxreadcnt > mnt_fields->mnt_maxreadcnt) {
 249                 info->maxreadcnt = mnt_fields->mnt_maxreadcnt;
 250         }
 251         if (uinfo->maxwritecnt == 0 || uinfo->maxwritecnt > mnt_fields->mnt_maxwritecnt) {
 252                 info->maxwritecnt = mnt_fields->mnt_maxwritecnt;
 253         }
 254         if (uinfo->segreadcnt == 0 || uinfo->segreadcnt > mnt_fields->mnt_segreadcnt) {
 255                 info->segreadcnt = mnt_fields->mnt_segreadcnt;
 256         }
 257         if (uinfo->segwritecnt == 0 || uinfo->segwritecnt > mnt_fields->mnt_segwritecnt) {
 258                 info->segwritecnt = mnt_fields->mnt_segwritecnt;
 259         }
 260
 261         if (uinfo->enabled) {
 262                 mp->mnt_maxreadcnt = info->maxreadcnt;
 263                 mp->mnt_maxwritecnt = info->maxwritecnt;
 264                 mp->mnt_segreadcnt = info->segreadcnt;
 265                 mp->mnt_segwritecnt = info->segwritecnt;
 266                 mp->mnt_ioqueue_depth = info->ioqueue_depth;
 267                 mp->mnt_ioscale = MNT_IOSCALE(info->ioqueue_depth);
 268         }
 269
 270         mount_unlock(mp);
 271
 272         microuptime(&internal_info->last_io_timestamp);
 273
 274         // make sure throttling picks up the new periods
 275         throttle_info_mount_reset_period(mp, info->is_ssd);
 276
 277         return 0;
 278 }
 279
 280 void
 281 disk_conditioner_unmount(mount_t mp)
 282 {
 283         struct _disk_conditioner_info_t *internal_info = mp->mnt_disk_conditioner_info;
 284
 285         if (!internal_info) {
 286                 return;
 287         }
 288
 289         if (internal_info->dcinfo.enabled) {
 290                 disk_conditioner_restore_mount_fields(mp, &(internal_info->mnt_fields));
 291         }
 292         mp->mnt_disk_conditioner_info = NULL;
 293         kfree(internal_info, sizeof(struct _disk_conditioner_info_t));
 294 }
 295
 296 boolean_t
 297 disk_conditioner_mount_is_ssd(mount_t mp)
 298 {
 299         struct _disk_conditioner_info_t *internal_info = mp->mnt_disk_conditioner_info;
 300
 301         if (!internal_info || !internal_info->dcinfo.enabled) {
 302                 return !!(mp->mnt_kern_flag & MNTK_SSD);
 303         }
 304
 305         return internal_info->dcinfo.is_ssd;
 306 }