bsd/vfs/vfs_io_compression_stats.c

   1 /*
   2  * Copyright (c) 2020 Apple Inc. All rights reserved.
   3  *
   4  * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
   5  *
   6  * This file contains Original Code and/or Modifications of Original Code
   7  * as defined in and that are subject to the Apple Public Source License
   8  * Version 2.0 (the 'License'). You may not use this file except in
   9  * compliance with the License. The rights granted to you under the License
  10  * may not be used to create, or enable the creation or redistribution of,
  11  * unlawful or unlicensed copies of an Apple operating system, or to
  12  * circumvent, violate, or enable the circumvention or violation of, any
  13  * terms of an Apple operating system software license agreement.
  14  *
  15  * Please obtain a copy of the License at
  16  * http://www.opensource.apple.com/apsl/ and read it before using this file.
  17  *
  18  * The Original Code and all software distributed under the License are
  19  * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
  20  * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
  21  * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
  22  * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
  23  * Please see the License for the specific language governing rights and
  24  * limitations under the License.
  25  *
  26  * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
  27  */
  28
  29 #include <kern/cpu_data.h>
  30 #include <kern/cpu_number.h>
  31 #include <kern/host.h>
  32
  33 #include <mach/host_priv.h>
  34 #include <mach/host_special_ports.h>
  35 #include <mach/host_info.h>
  36 #include <mach/iocompressionstats_notification_server.h>
  37 #include <mach/mach_host.h>
  38
  39 #include <sys/mount_internal.h>
  40 #include <sys/param.h>
  41 #include <sys/sysctl.h>
  42 #include <sys/vnode.h>
  43 #include <sys/vnode_internal.h>
  44
  45 #include <vfs/vfs_io_compression_stats.h>
  46
  47 #include <vm/lz4.h>
  48 #include <vm/vm_compressor_algorithms.h>
  49 #include <vm/vm_protos.h>
  50
  51
  52 int io_compression_stats_enable = 0;
  53 int io_compression_stats_block_size = IO_COMPRESSION_STATS_DEFAULT_BLOCK_SIZE;
  54
  55 #define LZ4_SCRATCH_ALIGN (64)
  56 typedef struct {
  57         uint8_t lz4state[lz4_encode_scratch_size]__attribute((aligned(LZ4_SCRATCH_ALIGN)));
  58 } lz4_encode_scratch_t;
  59
  60 lz4_encode_scratch_t **per_cpu_scratch_buf;
  61 uint8_t **per_cpu_compression_buf;
  62 uint32_t io_compression_stats_cpu_count;
  63 char *vnpath_scratch_buf;
  64
  65 LCK_GRP_DECLARE(io_compression_stats_lckgrp, "io_compression_stats");
  66 LCK_RW_DECLARE(io_compression_stats_lock, &io_compression_stats_lckgrp);
  67 LCK_MTX_DECLARE(iocs_store_buffer_lock, &io_compression_stats_lckgrp);
  68
  69 typedef enum io_compression_stats_allocate_type {
  70         IO_COMPRESSION_STATS_NEW_ALLOC = 0,
  71         IO_COMPRESSION_STATS_RESIZE = 1
  72 } io_compression_stats_alloc_type_t;
  73
  74 static void io_compression_stats_deallocate_compression_buffers(void);
  75
  76 struct iocs_store_buffer iocs_store_buffer = {
  77         .buffer = 0,
  78         .current_position = 0,
  79         .marked_point = 0
  80 };
  81
  82 int iocs_sb_bytes_since_last_mark = 0;
  83 int iocs_sb_bytes_since_last_notification = 0;
  84
  85 ZONE_DECLARE(io_compression_stats_zone, "io_compression_stats",
  86     sizeof(struct io_compression_stats), ZC_NOENCRYPT | ZC_NOGC | ZC_ZFREE_CLEARMEM);
  87
  88 static int
  89 io_compression_stats_allocate_compression_buffers(io_compression_stats_alloc_type_t alloc_type, uint32_t block_size)
  90 {
  91         int err = 0;
  92         host_basic_info_data_t hinfo;
  93         mach_msg_type_number_t count = HOST_BASIC_INFO_COUNT;
  94 #define BSD_HOST 1
  95         host_info((host_t)BSD_HOST, HOST_BASIC_INFO, (host_info_t)&hinfo, &count);
  96
  97         io_compression_stats_cpu_count = hinfo.max_cpus;
  98         if (alloc_type == IO_COMPRESSION_STATS_NEW_ALLOC) {
  99                 assert(per_cpu_scratch_buf == NULL);
 100                 per_cpu_scratch_buf = kheap_alloc(KHEAP_DEFAULT, sizeof(lz4_encode_scratch_t *) * io_compression_stats_cpu_count, Z_ZERO);
 101                 if (per_cpu_scratch_buf == NULL) {
 102                         err = ENOMEM;
 103                         goto out;
 104                 }
 105                 assert(per_cpu_compression_buf == NULL);
 106                 per_cpu_compression_buf = kheap_alloc(KHEAP_DEFAULT, sizeof(uint8_t *) * io_compression_stats_cpu_count, Z_ZERO);
 107                 if (per_cpu_compression_buf == NULL) {
 108                         err = ENOMEM;
 109                         goto out;
 110                 }
 111         }
 112         for (uint32_t cpu = 0; cpu < io_compression_stats_cpu_count; cpu++) {
 113                 if (alloc_type == IO_COMPRESSION_STATS_NEW_ALLOC) {
 114                         per_cpu_scratch_buf[cpu] = kheap_alloc(KHEAP_DEFAULT, sizeof(lz4_encode_scratch_t), Z_ZERO);
 115                         if (per_cpu_scratch_buf[cpu] == NULL) {
 116                                 err = ENOMEM;
 117                                 goto out;
 118                         }
 119                 } else {
 120                         kheap_free_addr(KHEAP_DEFAULT, per_cpu_compression_buf[cpu]);
 121                 }
 122                 per_cpu_compression_buf[cpu] = kheap_alloc(KHEAP_DEFAULT, block_size, Z_ZERO);
 123                 if (per_cpu_compression_buf[cpu] == NULL) {
 124                         err = ENOMEM;
 125                         goto out;
 126                 }
 127         }
 128         bzero(&iocs_store_buffer, sizeof(struct iocs_store_buffer));
 129         iocs_store_buffer.buffer = kheap_alloc(KHEAP_DEFAULT, IOCS_STORE_BUFFER_SIZE, Z_ZERO);
 130         if (iocs_store_buffer.buffer == NULL) {
 131                 err = ENOMEM;
 132                 goto out;
 133         }
 134         iocs_store_buffer.current_position = 0;
 135         iocs_store_buffer.marked_point = 0;
 136
 137         assert(vnpath_scratch_buf == NULL);
 138         vnpath_scratch_buf = kheap_alloc(KHEAP_DEFAULT, MAXPATHLEN, Z_ZERO);
 139         if (vnpath_scratch_buf == NULL) {
 140                 err = ENOMEM;
 141                 goto out;
 142         }
 143
 144 out:
 145         if (err) {
 146                 /* In case of any error, irrespective of whether it is new alloc or resize,
 147                  *  dellocate all buffers and fail */
 148                 io_compression_stats_deallocate_compression_buffers();
 149         }
 150         return err;
 151 }
 152
 153 static void
 154 io_compression_stats_deallocate_compression_buffers()
 155 {
 156         uint32_t cpu;
 157         if (per_cpu_compression_buf != NULL) {
 158                 for (cpu = 0; cpu < io_compression_stats_cpu_count; cpu++) {
 159                         if (per_cpu_compression_buf[cpu] != NULL) {
 160                                 kheap_free_addr(KHEAP_DEFAULT, per_cpu_compression_buf[cpu]);
 161                                 per_cpu_compression_buf[cpu] = NULL;
 162                         }
 163                 }
 164                 kheap_free_addr(KHEAP_DEFAULT, per_cpu_compression_buf);
 165                 per_cpu_compression_buf = NULL;
 166         }
 167
 168         if (per_cpu_scratch_buf != NULL) {
 169                 for (cpu = 0; cpu < io_compression_stats_cpu_count; cpu++) {
 170                         if (per_cpu_scratch_buf[cpu] != NULL) {
 171                                 kheap_free_addr(KHEAP_DEFAULT, per_cpu_scratch_buf[cpu]);
 172                                 per_cpu_scratch_buf[cpu] = NULL;
 173                         }
 174                 }
 175                 kheap_free_addr(KHEAP_DEFAULT, per_cpu_scratch_buf);
 176                 per_cpu_scratch_buf = NULL;
 177         }
 178
 179         if (iocs_store_buffer.buffer != NULL) {
 180                 kheap_free_addr(KHEAP_DEFAULT, iocs_store_buffer.buffer);
 181                 bzero(&iocs_store_buffer, sizeof(struct iocs_store_buffer));
 182         }
 183
 184         iocs_sb_bytes_since_last_mark = 0;
 185         iocs_sb_bytes_since_last_notification = 0;
 186
 187         if (vnpath_scratch_buf != NULL) {
 188                 kheap_free_addr(KHEAP_DEFAULT, vnpath_scratch_buf);
 189                 vnpath_scratch_buf = NULL;
 190         }
 191 }
 192
 193
 194 static int
 195 sysctl_io_compression_stats_enable SYSCTL_HANDLER_ARGS
 196 {
 197 #pragma unused (arg1, arg2, oidp)
 198
 199         int error = 0;
 200         int enable = 0;
 201
 202         error = SYSCTL_OUT(req, &io_compression_stats_enable, sizeof(int));
 203
 204         if (error || !req->newptr) {
 205                 return error;
 206         }
 207
 208         error = SYSCTL_IN(req, &enable, sizeof(int));
 209         if (error) {
 210                 return error;
 211         }
 212
 213         if (!((enable == 1) || (enable == 0))) {
 214                 return EINVAL;
 215         }
 216
 217         lck_rw_lock_exclusive(&io_compression_stats_lock);
 218         lck_mtx_lock(&iocs_store_buffer_lock);
 219         if ((io_compression_stats_enable == 0) && (enable == 1)) {
 220                 /* Enabling collection of stats. Allocate appropriate buffers */
 221                 error = io_compression_stats_allocate_compression_buffers(IO_COMPRESSION_STATS_NEW_ALLOC, io_compression_stats_block_size);
 222                 if (error == 0) {
 223                         io_compression_stats_enable = enable;
 224                         io_compression_stats_dbg("SUCCESS: setting io_compression_stats_enable to %d", io_compression_stats_enable);
 225                 } else {
 226                         io_compression_stats_dbg("FAILED: setting io_compression_stats_enable to %d", io_compression_stats_enable);
 227                 }
 228         } else if ((io_compression_stats_enable == 1) && (enable == 0)) {
 229                 io_compression_stats_deallocate_compression_buffers();
 230                 io_compression_stats_enable = 0;
 231                 io_compression_stats_dbg("SUCCESS: setting io_compression_stats_enable to %d", io_compression_stats_enable);
 232         }
 233         lck_mtx_unlock(&iocs_store_buffer_lock);
 234         lck_rw_unlock_exclusive(&io_compression_stats_lock);
 235
 236         return error;
 237 }
 238 SYSCTL_PROC(_vfs, OID_AUTO, io_compression_stats_enable, CTLTYPE_INT | CTLFLAG_RW, 0, 0, &sysctl_io_compression_stats_enable, "I", "");
 239
 240 static int
 241 sysctl_io_compression_block_size SYSCTL_HANDLER_ARGS
 242 {
 243 #pragma unused (arg1, arg2, oidp)
 244
 245         int error = 0;
 246         int block_size = io_compression_stats_block_size;
 247
 248         error = SYSCTL_OUT(req, &block_size, sizeof(int));
 249
 250         if (error || !req->newptr) {
 251                 return error;
 252         }
 253
 254         error = SYSCTL_IN(req, &block_size, sizeof(int));
 255         if (error) {
 256                 return error;
 257         }
 258
 259         if (block_size < IO_COMPRESSION_STATS_MIN_BLOCK_SIZE || block_size > IO_COMPRESSION_STATS_MAX_BLOCK_SIZE) {
 260                 return EINVAL;
 261         }
 262
 263         lck_rw_lock_exclusive(&io_compression_stats_lock);
 264
 265         if (io_compression_stats_block_size != block_size) {
 266                 if (io_compression_stats_enable == 1) {
 267                         /* IO compression stats is enabled, rellocate buffers. */
 268                         error = io_compression_stats_allocate_compression_buffers(IO_COMPRESSION_STATS_RESIZE, block_size);
 269                         if (error == 0) {
 270                                 io_compression_stats_block_size = block_size;
 271                                 io_compression_stats_dbg("SUCCESS: setting io_compression_stats_block_size to %d", io_compression_stats_block_size);
 272                         } else {
 273                                 /* Failed to allocate buffers, disable IO compression stats */
 274                                 io_compression_stats_enable = 0;
 275                                 io_compression_stats_dbg("Failed: setting io_compression_stats_block_size to %d", io_compression_stats_block_size);
 276                         }
 277                 } else {
 278                         /* IO compression stats is disabled, only set the io_compression_stats_block_size */
 279                         io_compression_stats_block_size = block_size;
 280                         io_compression_stats_dbg("SUCCESS: setting io_compression_stats_block_size to %d", io_compression_stats_block_size);
 281                 }
 282         }
 283         lck_rw_unlock_exclusive(&io_compression_stats_lock);
 284
 285
 286         return error;
 287 }
 288 SYSCTL_PROC(_vfs, OID_AUTO, io_compression_stats_block_size, CTLTYPE_INT | CTLFLAG_RW, 0, 0, &sysctl_io_compression_block_size, "I", "");
 289
 290
 291 static int32_t
 292 iocs_compress_block(uint8_t *block_ptr, uint32_t block_size)
 293 {
 294         disable_preemption();
 295
 296         uint32_t current_cpu = cpu_number();
 297         if (!(current_cpu < io_compression_stats_cpu_count)) {
 298                 enable_preemption();
 299                 return -1;
 300         }
 301
 302         lz4_encode_scratch_t *scratch_buf = per_cpu_scratch_buf[current_cpu];
 303         uint8_t *dest_buf = per_cpu_compression_buf[current_cpu];
 304
 305         int compressed_block_size = (int) lz4raw_encode_buffer(dest_buf, block_size,
 306             block_ptr, block_size, (lz4_hash_entry_t *) scratch_buf);
 307
 308         enable_preemption();
 309
 310         return compressed_block_size;
 311 }
 312 /*
 313  * Compress buf in chunks of io_compression_stats_block_size
 314  */
 315 static uint32_t
 316 iocs_compress_buffer(vnode_t vn, uint8_t *buf_ptr, uint32_t buf_size)
 317 {
 318         uint32_t offset;
 319         uint32_t compressed_size = 0;
 320         int block_size = io_compression_stats_block_size;
 321         int block_stats_scaling_factor = block_size / IOCS_BLOCK_NUM_SIZE_BUCKETS;
 322
 323         for (offset = 0; offset < buf_size; offset += block_size) {
 324                 int current_block_size = min(block_size, buf_size - offset);
 325                 int current_compressed_block_size = iocs_compress_block(buf_ptr + offset, current_block_size);
 326
 327                 if (current_compressed_block_size == 0) {
 328                         compressed_size += current_block_size;
 329                         vnode_updateiocompressionblockstats(vn, current_block_size / block_stats_scaling_factor);
 330                 } else if (current_compressed_block_size != -1) {
 331                         compressed_size += current_compressed_block_size;
 332                         vnode_updateiocompressionblockstats(vn, current_compressed_block_size / block_stats_scaling_factor);
 333                 }
 334         }
 335
 336         return compressed_size;
 337 }
 338
 339 static uint32_t
 340 log2down(uint32_t x)
 341 {
 342         return 31 - __builtin_clz(x);
 343 }
 344
 345 /*
 346  * Once we get the IO compression stats for the entire buffer, we update buffer_size_compressibility_dist,
 347  * which helps us observe distribution across various io sizes and compression factors.
 348  * The goal of next two functions is to get the index in this buffer_size_compressibility_dist table.
 349  */
 350
 351 /*
 352  * Maps IO size to a bucket between 0 - IO_COMPRESSION_STATS_MAX_SIZE_BUCKET
 353  * for size < 4096 returns 0 and size > 1MB returns IO_COMPRESSION_STATS_MAX_SIZE_BUCKET (9).
 354  * For IO sizes in-between we arrive at the index based on log2 function.
 355  * sizes 4097 - 8192 => index = 1,
 356  * sizes 8193 - 16384 => index = 2, and so on
 357  */
 358 #define SIZE_COMPRESSION_DIST_SIZE_BUCKET_MIN   4096
 359 #define SIZE_COMPRESSION_DIST_SIZE_BUCKET_MAX   (1024 * 1024)
 360 static uint32_t
 361 get_buffer_size_bucket(uint32_t size)
 362 {
 363         if (size <= SIZE_COMPRESSION_DIST_SIZE_BUCKET_MIN) {
 364                 return 0;
 365         }
 366         if (size > SIZE_COMPRESSION_DIST_SIZE_BUCKET_MAX) {
 367                 return IOCS_BUFFER_MAX_BUCKET;
 368         }
 369 #define IOCS_INDEX_MAP_OFFSET 11
 370         return log2down(size - 1) - IOCS_INDEX_MAP_OFFSET;
 371 }
 372
 373 /*
 374  * Maps compression factor to a bucket between 0 - IO_COMPRESSION_STATS_MAX_COMPRESSION_BUCKET
 375  */
 376 static uint32_t
 377 get_buffer_compressibility_bucket(uint32_t uncompressed_size, uint32_t compressed_size)
 378 {
 379         int saved_space_pc = (uncompressed_size - compressed_size) * 100 / uncompressed_size;
 380
 381         if (saved_space_pc < 0) {
 382                 saved_space_pc = 0;
 383         }
 384
 385         /* saved_space_pc lies bw 0 - 100. log2(saved_space_pc) lies bw 0 - 6 */
 386         return log2down(saved_space_pc);
 387 }
 388
 389 void
 390 io_compression_stats(buf_t bp)
 391 {
 392         uint8_t *buf_ptr = NULL;
 393         int bflags = bp->b_flags;
 394         uint32_t compressed_size = 0;
 395         uint32_t buf_cnt = buf_count(bp);
 396         uint64_t duration = 0;
 397         caddr_t vaddr = NULL;
 398         vnode_t vn = buf_vnode(bp);
 399         int err = 0;
 400
 401         if ((io_compression_stats_enable != 1) || (bflags & B_READ) || (buf_cnt <= 0)) {
 402                 return;
 403         }
 404
 405         if (!lck_rw_try_lock_shared(&io_compression_stats_lock)) {
 406                 /* sysctl modifying IO compression stats parameters is in progress.
 407                  *  Don't block, since malloc might be in progress. */
 408                 return;
 409         }
 410         /* re-check io_compression_stats_enable with lock */
 411         if (io_compression_stats_enable != 1) {
 412                 goto out;
 413         }
 414
 415         err = buf_map(bp, &vaddr);
 416         if (!err) {
 417                 buf_ptr = (uint8_t *) vaddr;
 418         }
 419
 420         if (buf_ptr != NULL) {
 421                 int64_t start = mach_absolute_time();
 422                 compressed_size = iocs_compress_buffer(vn, buf_ptr, buf_cnt);
 423                 absolutetime_to_nanoseconds(mach_absolute_time() - start, &duration);
 424
 425                 if (compressed_size != 0) {
 426                         vnode_updateiocompressionbufferstats(vn, buf_cnt, compressed_size,
 427                             get_buffer_size_bucket(buf_cnt),
 428                             get_buffer_compressibility_bucket(buf_cnt, compressed_size));
 429                 }
 430         }
 431
 432         KDBG_RELEASE(FSDBG_CODE(DBG_VFS, DBG_VFS_IO_COMPRESSION_STATS) | DBG_FUNC_NONE,
 433             duration, io_compression_stats_block_size, compressed_size, buf_cnt, 0);
 434
 435 out:
 436         lck_rw_unlock_shared(&io_compression_stats_lock);
 437         if (buf_ptr != NULL) {
 438                 buf_unmap(bp);
 439         }
 440 }
 441
 442 static void
 443 iocs_notify_user(void)
 444 {
 445         mach_port_t user_port = MACH_PORT_NULL;
 446         kern_return_t kr = host_get_iocompressionstats_port(host_priv_self(), &user_port);
 447         if ((kr != KERN_SUCCESS) || !IPC_PORT_VALID(user_port)) {
 448                 return;
 449         }
 450         iocompressionstats_notification(user_port, 0);
 451         ipc_port_release_send(user_port);
 452 }
 453 static void
 454 construct_iocs_sbe_from_vnode(struct vnode *vp, struct iocs_store_buffer_entry *iocs_sbe)
 455 {
 456         int path_len = MAXPATHLEN;
 457
 458         vn_getpath(vp, vnpath_scratch_buf, &path_len);
 459         /*
 460          * Total path length is path_len, we can copy out IOCS_SBE_PATH_LEN bytes. We are interested
 461          * in first segment of the path to try and figure out the process writing to the file, and we are
 462          * interested in the last segment to figure out extention. So, in cases where
 463          * IOCS_SBE_PATH_LEN < path_len, lets copy out first IOCS_PATH_START_BYTES_TO_COPY bytes and
 464          * last IOCS_PATH_END_BYTES_TO_COPY (last segment includes the null character).
 465          */
 466         if (path_len > IOCS_SBE_PATH_LEN) {
 467                 strncpy(iocs_sbe->path_name, vnpath_scratch_buf, IOCS_PATH_START_BYTES_TO_COPY);
 468                 strncpy(iocs_sbe->path_name + IOCS_PATH_START_BYTES_TO_COPY,
 469                     vnpath_scratch_buf + path_len - IOCS_PATH_END_BYTES_TO_COPY,
 470                     IOCS_PATH_END_BYTES_TO_COPY);
 471         } else {
 472                 strncpy(iocs_sbe->path_name, vnpath_scratch_buf, IOCS_SBE_PATH_LEN);
 473         }
 474         memcpy(&iocs_sbe->iocs, vp->io_compression_stats, sizeof(struct io_compression_stats));
 475 }
 476 void
 477 vnode_iocs_record_and_free(struct vnode *vp)
 478 {
 479         int notify = 0;
 480         struct iocs_store_buffer_entry *iocs_sbe = NULL;
 481
 482         if (!lck_mtx_try_lock(&iocs_store_buffer_lock)) {
 483                 goto out;
 484         }
 485
 486         if (iocs_store_buffer.buffer == NULL) {
 487                 goto release;
 488         }
 489
 490         assert(iocs_store_buffer.current_position + sizeof(struct iocs_store_buffer_entry) <= IOCS_STORE_BUFFER_SIZE);
 491
 492         iocs_sbe = (struct iocs_store_buffer_entry *)(iocs_store_buffer.buffer + iocs_store_buffer.current_position);
 493
 494         construct_iocs_sbe_from_vnode(vp, iocs_sbe);
 495
 496         iocs_store_buffer.current_position += sizeof(struct iocs_store_buffer_entry);
 497
 498         if (iocs_store_buffer.current_position + sizeof(struct iocs_store_buffer_entry) > IOCS_STORE_BUFFER_SIZE) {
 499                 /* We've reached end of the buffer, move back to the top */
 500                 iocs_store_buffer.current_position = 0;
 501         }
 502
 503         iocs_sb_bytes_since_last_mark += sizeof(struct iocs_store_buffer_entry);
 504         iocs_sb_bytes_since_last_notification += sizeof(struct iocs_store_buffer_entry);
 505
 506         if ((iocs_sb_bytes_since_last_mark > IOCS_STORE_BUFFER_NOTIFY_AT) &&
 507             (iocs_sb_bytes_since_last_notification > IOCS_STORE_BUFFER_NOTIFICATION_INTERVAL)) {
 508                 notify = 1;
 509                 iocs_sb_bytes_since_last_notification = 0;
 510         }
 511
 512 release:
 513         lck_mtx_unlock(&iocs_store_buffer_lock);
 514 out:
 515         /* We need to free io_compression_stats whether or not we were able to record it */
 516         bzero(vp->io_compression_stats, sizeof(struct io_compression_stats));
 517         zfree(io_compression_stats_zone, vp->io_compression_stats);
 518         vp->io_compression_stats = NULL;
 519         if (notify) {
 520                 iocs_notify_user();
 521         }
 522 }
 523
 524 struct vnode_iocs_context {
 525         struct sysctl_req *addr;
 526         int current_ptr;
 527 };
 528
 529 static int
 530 vnode_iocs_callback(struct vnode *vp, void *vctx)
 531 {
 532         struct vnode_iocs_context *ctx = vctx;
 533         struct sysctl_req *req = ctx->addr;
 534         int current_ptr = ctx->current_ptr;
 535
 536         if (current_ptr + sizeof(struct iocs_store_buffer_entry) < req->oldlen) {
 537                 if (vp->io_compression_stats != NULL) {
 538                         construct_iocs_sbe_from_vnode(vp, (struct iocs_store_buffer_entry *) (req->oldptr + current_ptr));
 539                         current_ptr += sizeof(struct iocs_store_buffer_entry);
 540                 }
 541         } else {
 542                 return VNODE_RETURNED_DONE;
 543         }
 544         ctx->current_ptr = current_ptr;
 545
 546         return VNODE_RETURNED;
 547 }
 548
 549 static int
 550 vfs_iocs_callback(mount_t mp, void *arg)
 551 {
 552         if (mp->mnt_flag & MNT_LOCAL) {
 553                 vnode_iterate(mp, VNODE_ITERATE_ALL, vnode_iocs_callback, arg);
 554         }
 555
 556         return VFS_RETURNED;
 557 }
 558
 559 extern long numvnodes;
 560
 561 static int
 562 sysctl_io_compression_dump_stats SYSCTL_HANDLER_ARGS
 563 {
 564 #pragma unused (arg1, arg2, oidp)
 565
 566         int32_t error = 0;
 567         uint32_t inp_flag = 0;
 568         uint32_t ret_len;
 569
 570         if (io_compression_stats_enable == 0) {
 571                 error = EINVAL;
 572                 goto out;
 573         }
 574
 575         if ((req->newptr != USER_ADDR_NULL) && (req->newlen == sizeof(uint32_t))) {
 576                 error = SYSCTL_IN(req, &inp_flag, sizeof(uint32_t));
 577                 if (error) {
 578                         goto out;
 579                 }
 580                 switch (inp_flag) {
 581                 case IOCS_SYSCTL_LIVE:
 582                 case IOCS_SYSCTL_STORE_BUFFER_RD_ONLY:
 583                 case IOCS_SYSCTL_STORE_BUFFER_MARK:
 584                         break;
 585                 default:
 586                         error = EINVAL;
 587                         goto out;
 588                 }
 589         } else {
 590                 error = EINVAL;
 591                 goto out;
 592         }
 593
 594         if (req->oldptr == USER_ADDR_NULL) {
 595                 /* Query to figure out size of the buffer */
 596                 if (inp_flag & IOCS_SYSCTL_LIVE) {
 597                         req->oldidx = numvnodes * sizeof(struct iocs_store_buffer_entry);
 598                 } else {
 599                         /* Buffer size for archived case, let's keep it
 600                          * simple and return IOCS store buffer size */
 601                         req->oldidx = IOCS_STORE_BUFFER_SIZE;
 602                 }
 603                 goto out;
 604         }
 605
 606         if (inp_flag & IOCS_SYSCTL_LIVE) {
 607                 struct vnode_iocs_context ctx;
 608
 609                 bzero(&ctx, sizeof(struct vnode_iocs_context));
 610                 ctx.addr = req;
 611                 vfs_iterate(0, vfs_iocs_callback, &ctx);
 612                 req->oldidx = ctx.current_ptr;
 613                 goto out;
 614         }
 615
 616         /* reading from store buffer */
 617         lck_mtx_lock(&iocs_store_buffer_lock);
 618
 619         if (iocs_store_buffer.buffer == NULL) {
 620                 error = EINVAL;
 621                 goto release;
 622         }
 623         if (iocs_sb_bytes_since_last_mark == 0) {
 624                 req->oldidx = 0;
 625                 goto release;
 626         }
 627
 628         int expected_size = 0;
 629         /* Dry run to figure out amount of space required to copy out the
 630          * iocs_store_buffer.buffer */
 631         if (iocs_store_buffer.marked_point < iocs_store_buffer.current_position) {
 632                 expected_size = iocs_store_buffer.current_position - iocs_store_buffer.marked_point;
 633         } else {
 634                 expected_size = IOCS_STORE_BUFFER_SIZE - iocs_store_buffer.marked_point;
 635                 expected_size += iocs_store_buffer.current_position;
 636         }
 637
 638         if (req->oldlen < expected_size) {
 639                 error = ENOMEM;
 640                 req->oldidx = 0;
 641                 goto release;
 642         }
 643
 644         if (iocs_store_buffer.marked_point < iocs_store_buffer.current_position) {
 645                 error = copyout(iocs_store_buffer.buffer + iocs_store_buffer.marked_point,
 646                     req->oldptr,
 647                     iocs_store_buffer.current_position - iocs_store_buffer.marked_point);
 648                 if (error) {
 649                         req->oldidx = 0;
 650                         goto release;
 651                 }
 652                 ret_len = iocs_store_buffer.current_position - iocs_store_buffer.marked_point;
 653         } else {
 654                 error = copyout(iocs_store_buffer.buffer + iocs_store_buffer.marked_point,
 655                     req->oldptr,
 656                     IOCS_STORE_BUFFER_SIZE - iocs_store_buffer.marked_point);
 657                 if (error) {
 658                         req->oldidx = 0;
 659                         goto release;
 660                 }
 661                 ret_len = IOCS_STORE_BUFFER_SIZE - iocs_store_buffer.marked_point;
 662
 663                 error = copyout(iocs_store_buffer.buffer,
 664                     req->oldptr + ret_len,
 665                     iocs_store_buffer.current_position);
 666                 if (error) {
 667                         req->oldidx = 0;
 668                         goto release;
 669                 }
 670                 ret_len += iocs_store_buffer.current_position;
 671         }
 672
 673         req->oldidx = ret_len;
 674         if ((ret_len != 0) && (inp_flag & IOCS_SYSCTL_STORE_BUFFER_MARK)) {
 675                 iocs_sb_bytes_since_last_mark = 0;
 676                 iocs_store_buffer.marked_point = iocs_store_buffer.current_position;
 677         }
 678 release:
 679         lck_mtx_unlock(&iocs_store_buffer_lock);
 680
 681 out:
 682         return error;
 683 }
 684 SYSCTL_PROC(_vfs, OID_AUTO, io_compression_dump_stats, CTLFLAG_WR | CTLTYPE_NODE, 0, 0, sysctl_io_compression_dump_stats, "-", "");
 685
 686 errno_t
 687 vnode_updateiocompressionblockstats(vnode_t vp, uint32_t size_bucket)
 688 {
 689         if (vp == NULL) {
 690                 return EINVAL;
 691         }
 692
 693         if (size_bucket >= IOCS_BLOCK_NUM_SIZE_BUCKETS) {
 694                 return EINVAL;
 695         }
 696
 697         if (vp->io_compression_stats == NULL) {
 698                 io_compression_stats_t iocs = (io_compression_stats_t)zalloc_flags(io_compression_stats_zone, Z_ZERO);
 699                 if (iocs == NULL) {
 700                         return ENOMEM;
 701                 }
 702                 vnode_lock_spin(vp);
 703                 /* Re-check with lock */
 704                 if (vp->io_compression_stats == NULL) {
 705                         vp->io_compression_stats = iocs;
 706                 } else {
 707                         zfree(io_compression_stats_zone, iocs);
 708                 }
 709                 vnode_unlock(vp);
 710         }
 711         OSIncrementAtomic((SInt32 *)&vp->io_compression_stats->block_compressed_size_dist[size_bucket]);
 712
 713         return 0;
 714 }
 715 errno_t
 716 vnode_updateiocompressionbufferstats(__unused vnode_t vp, __unused uint64_t uncompressed_size, __unused uint64_t compressed_size, __unused uint32_t size_bucket, __unused uint32_t compression_bucket)
 717 {
 718         if (vp == NULL) {
 719                 return EINVAL;
 720         }
 721
 722         /* vnode_updateiocompressionblockstats will always be called before vnode_updateiocompressionbufferstats.
 723          * Hence vp->io_compression_stats should already be allocated */
 724         if (vp->io_compression_stats == NULL) {
 725                 return EINVAL;
 726         }
 727
 728         if ((size_bucket >= IOCS_BUFFER_NUM_SIZE_BUCKETS) || (compression_bucket >= IOCS_BUFFER_NUM_COMPRESSION_BUCKETS)) {
 729                 return EINVAL;
 730         }
 731
 732         OSAddAtomic64(uncompressed_size, &vp->io_compression_stats->uncompressed_size);
 733         OSAddAtomic64(compressed_size, &vp->io_compression_stats->compressed_size);
 734
 735         OSIncrementAtomic((SInt32 *)&vp->io_compression_stats->buffer_size_compression_dist[size_bucket][compression_bucket]);
 736
 737         return 0;
 738 }