iosim.tproj/iosim.c

   1 #include <stdio.h>
   2 #include <stdlib.h>
   3 #include <fcntl.h>
   4 #include <unistd.h>
   5 #include <getopt.h>
   6 #include <string.h>
   7 #include <sys/resource.h>
   8 #include <pthread.h>
   9 #include <sys/time.h>
  10 #include <sys/stat.h>
  11 #include <sys/types.h>
  12 #include <math.h>
  13 #include <signal.h>
  14 #include <libkern/OSAtomic.h>
  15 #include <limits.h>
  16 #include <errno.h>
  17
  18 #define IO_MODE_SEQ             0
  19 #define IO_MODE_RANDOM          1
  20
  21 #define WORKLOAD_TYPE_RO        0
  22 #define WORKLOAD_TYPE_WO        1
  23 #define WORKLOAD_TYPE_RW        2
  24
  25 #define MAX_THREADS             1000
  26 #define MAX_FILENAME            64
  27 #define MAX_ITERATIONS          10000
  28 #define LATENCY_BIN_SIZE        500
  29 #define LATENCY_BINS            11
  30 #define LOW_LATENCY_BIN_SIZE    50
  31 #define LOW_LATENCY_BINS        11
  32 #define THROUGHPUT_INTERVAL     5000
  33 #define DEFAULT_FILE_SIZE       (262144)
  34 #define BLOCKSIZE               1024
  35 #define MAX_CMD_SIZE            256
  36 #define PG_MASK                 ~(0xFFF)
  37
  38 int burst_count = 10;                   /* Unit: Number ; Desc.: I/O Burst Count */
  39 int inter_burst_duration = 0;           /* Unit: msecs  ; Desc.: I/O Inter-Burst Duration (-1: Random value [0,100]) */
  40 int inter_io_delay_ms = 0;              /* Unit: msecs  ; Desc.: Inter I/O Delay */
  41 int thread_count = 1;                   /* Unit: Number ; Desc.: Thread Count */
  42 int workload_type = WORKLOAD_TYPE_RO;   /* Unit: 0/1/2  ; Desc.: Workload Type */
  43 int io_size = 4096;                     /* Unit: Bytes  ; Desc.: I/O Unit Size */
  44 int sync_frequency_ms = 0;              /* Unit: msecs  ; Desc.: Sync thread frequency (0: Indicates no sync) */
  45 int io_mode = 0;                        /* Unit: 0/1    ; Desc.: I/O Mode (Seq./Rand.) */
  46 int test_duration = 0;                  /* Unit: secs   ; Desc.: Total Test Duration (0 indicates wait for Ctrl+C signal) */
  47 int io_tier = 0;                        /* Unit: 0/1/2/3; Desc.: I/O Tier */
  48 int file_size = DEFAULT_FILE_SIZE;      /* Unit: pages  ; Desc.: File Size in 4096 byte blocks */
  49 int cached_io_flag = 0;                 /* Unit: 0/1    ; Desc.: I/O Caching behavior (no-cached/cached) */
  50 char *user_fname;
  51 int user_specified_file = 0;
  52
  53 int64_t total_io_count;
  54 int64_t total_io_size;
  55 int64_t total_io_time;
  56 int64_t total_burst_count;
  57 int64_t latency_histogram[LATENCY_BINS];
  58 int64_t burst_latency_histogram[LATENCY_BINS];
  59 int64_t low_latency_histogram[LOW_LATENCY_BINS];
  60 int64_t throughput_histogram[MAX_ITERATIONS];
  61 int64_t throughput_index;
  62
  63 void print_usage(void);
  64 void print_data_percentage(int percent);
  65 void print_stats(void);
  66 unsigned int find_io_bin(int64_t latency, int latency_bin_size, int latency_bins);
  67 void signalHandler(int sig);
  68 void perform_io(int fd, char *buf, int size, int type);
  69 void *sync_routine(void *arg);
  70 void *calculate_throughput(void *arg);
  71 void *io_routine(void *arg);
  72 void validate_option(int value, int min, int max, char *option, char *units);
  73 void print_test_setup(int value, char *option, char *units, char *comment);
  74 void setup_process_io_policy(int io_tier);
  75 void print_latency_histogram(int64_t *data, int latency_bins, int latency_bin_size);
  76
  77 void
  78 print_usage(void)
  79 {
  80         printf("Usage: ./iosim [options]\n");
  81         printf("Options:\n");
  82         printf("-c: (number)  Burst Count. No. of I/Os performed in an I/O burst\n");
  83         printf("-i: (msecs)   Inter Burst Duration. Amount of time the thread sleeps between bursts (-1 indicates random durations between 0-100 msecs)\n");
  84         printf("-d: (msecs)   Inter I/O delay. Amount of time between issuing I/Os\n");
  85         printf("-t: (number)  Thread count\n");
  86         printf("-f: (0/1/2 :  Read-Only/Write-Only/Mixed RW) Workload Type\n");
  87         printf("-m: (0/1   :  Sequential/Random) I/O pattern\n");
  88         printf("-j: (number)  Size of I/O in bytes\n");
  89         printf("-s: (msecs)   Frequency of sync() calls\n");
  90         printf("-x: (secs)    Test duration (0 indicates that the tool would wait for a Ctrl-C)\n");
  91         printf("-l: (0/1/2/3) I/O Tier\n");
  92         printf("-z: (number)  File Size in pages (1 page = 4096 bytes) \n");
  93         printf("-n: (string)  File name used for tests (the tool would create files if this option is not specified)\n");
  94         printf("-a: (0/1   :  Non-cached/Cached) I/O Caching behavior\n");
  95 }
  96
  97 void print_data_percentage(int percent)
  98 {
  99         int count = (int)(round(percent / 5.0));
 100         int spaces = 20 - count;
 101         printf("| ");
 102         for(; count > 0; count--)
 103                 printf("*");
 104         for(; spaces > 0; spaces--)
 105                 printf(" ");
 106         printf("|");
 107 }
 108
 109 void print_latency_histogram(int64_t *data, int latency_bins, int latency_bin_size)
 110 {
 111         double percentage;
 112         char label[MAX_FILENAME];
 113         int i;
 114
 115         for (i = 0; i < latency_bins; i++) {
 116                 if (i == (latency_bins - 1))
 117                         snprintf(label, MAX_FILENAME, "> %d usecs", i * latency_bin_size);
 118                 else
 119                         snprintf(label, MAX_FILENAME, "%d - %d usecs", i * latency_bin_size, (i+1) * latency_bin_size);
 120                 printf("%25s ", label);
 121                 percentage = ((double)data[i] * 100.0) / (double)total_io_count;
 122                 print_data_percentage((int)percentage);
 123                 printf(" %.2lf%%\n", percentage);
 124         }
 125         printf("\n");
 126 }
 127
 128 void print_stats()
 129 {
 130         int i;
 131         double percentage;
 132         char label[MAX_FILENAME];
 133
 134         printf("I/O Statistics:\n");
 135
 136         printf("Total I/Os      : %lld\n", total_io_count);
 137         printf("Avg. Latency    : %.2lf usecs\n", ((double)total_io_time) / ((double)total_io_count));
 138
 139         printf("Low Latency Histogram: \n");
 140         print_latency_histogram(low_latency_histogram, LOW_LATENCY_BINS, LOW_LATENCY_BIN_SIZE);
 141         printf("Latency Histogram: \n");
 142         print_latency_histogram(latency_histogram, LATENCY_BINS, LATENCY_BIN_SIZE);
 143         printf("Burst Avg. Latency Histogram: \n");
 144         print_latency_histogram(burst_latency_histogram, LATENCY_BINS, LATENCY_BIN_SIZE);
 145
 146         printf("Throughput Timeline: \n");
 147
 148         int64_t max_throughput = 0;
 149         for (i = 0; i < throughput_index; i++) {
 150                 if (max_throughput < throughput_histogram[i])
 151                         max_throughput = throughput_histogram[i];
 152         }
 153
 154         for (i = 0; i < throughput_index; i++) {
 155                 snprintf(label, MAX_FILENAME, "T=%d msecs", (i+1) * THROUGHPUT_INTERVAL);
 156                 printf("%25s ", label);
 157                 percentage = ((double)throughput_histogram[i] * 100) / (double)max_throughput;
 158                 print_data_percentage((int)percentage);
 159                 printf("%.2lf MBps\n", ((double)throughput_histogram[i] / 1048576.0) / ((double)THROUGHPUT_INTERVAL / 1000.0));
 160         }
 161         printf("\n");
 162 }
 163
 164 unsigned int find_io_bin(int64_t latency, int latency_bin_size, int latency_bins)
 165 {
 166         int bin = (int) (latency / latency_bin_size);
 167         if (bin >= latency_bins)
 168                 bin = latency_bins - 1;
 169         return bin;
 170 }
 171
 172 void signalHandler(int sig)
 173 {
 174         printf("\n");
 175         print_stats();
 176         exit(0);
 177 }
 178
 179
 180 void perform_io(int fd, char *buf, int size, int type)
 181 {
 182         long ret;
 183
 184         if (type == WORKLOAD_TYPE_RW)
 185                 type = (rand() % 2) ? WORKLOAD_TYPE_WO : WORKLOAD_TYPE_RO;
 186
 187         while(size > 0) {
 188
 189                 if (type == WORKLOAD_TYPE_RO)
 190                         ret = read(fd, buf, size);
 191                 else
 192                         ret = write(fd, buf, size);
 193
 194                 if (ret == 0) {
 195                         if (lseek(fd, 0, SEEK_SET) < 0) {
 196                                 perror("lseek() to reset file offset to zero failed!\n");
 197                                 goto error;
 198                         }
 199                 }
 200
 201                 if (ret < 0) {
 202                         perror("read/write syscall failed!\n");
 203                         goto error;
 204                 }
 205                 size -= ret;
 206         }
 207
 208         return;
 209
 210 error:
 211         print_stats();
 212         exit(1);
 213 }
 214
 215 void *sync_routine(void *arg)
 216 {
 217         while(1) {
 218                 usleep(sync_frequency_ms * 1000);
 219                 sync();
 220         }
 221         pthread_exit(NULL);
 222 }
 223
 224 void *calculate_throughput(void *arg)
 225 {
 226         int64_t prev_total_io_size = 0;
 227         int64_t size;
 228
 229         while(1) {
 230                 usleep(THROUGHPUT_INTERVAL * 1000);
 231                 size = total_io_size - prev_total_io_size;
 232                 throughput_histogram[throughput_index] = size;
 233                 prev_total_io_size = total_io_size;
 234                 throughput_index++;
 235         }
 236         pthread_exit(NULL);
 237 }
 238
 239 void *io_routine(void *arg)
 240 {
 241         struct timeval start_tv;
 242         struct timeval end_tv;
 243         int64_t elapsed;
 244         int64_t burst_elapsed;
 245         char *data;
 246         char test_filename[MAX_FILENAME];
 247         struct stat filestat;
 248         int i, fd, io_thread_id;
 249
 250         io_thread_id = (int)arg;
 251         if (user_specified_file)
 252                 strlcpy(test_filename, user_fname, MAX_FILENAME);
 253         else
 254                 snprintf(test_filename, MAX_FILENAME, "iosim-%d-%d", (int)getpid(), io_thread_id);
 255
 256         if (0 > (fd = open(test_filename, O_RDWR, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH))) {
 257                 printf("Error opening file %s!\n", test_filename);
 258                 exit(1);
 259         }
 260
 261         if (fstat(fd, &filestat) < 0) {
 262                 printf("Error stat()ing file %s!\n", test_filename);
 263                 exit(1);
 264         }
 265
 266         if (filestat.st_size < io_size) {
 267                 printf("%s: File size (%lld) smaller than I/O size (%d)!\n", test_filename, filestat.st_size, io_size);
 268                 exit(1);
 269         }
 270
 271         if (!cached_io_flag)
 272                 fcntl(fd, F_NOCACHE, 1);
 273
 274         fcntl(fd, F_RDAHEAD, 0);
 275
 276         if(!(data = (char *)calloc(io_size, 1))) {
 277                 perror("Error allocating buffers for I/O!\n");
 278                 exit(1);
 279         }
 280         memset(data, '\0', io_size);
 281
 282         while(1) {
 283                 burst_elapsed = 0;
 284
 285                 for(i = 0; i < burst_count; i++) {
 286                         if (io_mode == IO_MODE_RANDOM) {
 287                                 if (lseek(fd, (rand() % (filestat.st_size - io_size)) & PG_MASK, SEEK_SET) < 0) {
 288                                         perror("Error lseek()ing to random location in file!\n");
 289                                         exit(1);
 290                                 }
 291                         }
 292
 293                         gettimeofday(&start_tv, NULL);
 294                         perform_io(fd, data, io_size, workload_type);
 295                         gettimeofday(&end_tv, NULL);
 296
 297                         OSAtomicIncrement64(&total_io_count);
 298                         OSAtomicAdd64(io_size, &total_io_size);
 299                         elapsed = ((end_tv.tv_sec - start_tv.tv_sec) * 1000000)  + (end_tv.tv_usec - start_tv.tv_usec);
 300                         OSAtomicAdd64(elapsed, &total_io_time);
 301                         OSAtomicIncrement64(&(latency_histogram[find_io_bin(elapsed, LATENCY_BIN_SIZE, LATENCY_BINS)]));
 302                         OSAtomicIncrement64(&(low_latency_histogram[find_io_bin(elapsed, LOW_LATENCY_BIN_SIZE, LOW_LATENCY_BINS)]));
 303                         burst_elapsed += elapsed;
 304
 305                         if (inter_io_delay_ms)
 306                                 usleep(inter_io_delay_ms * 1000);
 307                 }
 308
 309                 burst_elapsed /= burst_count;
 310                 OSAtomicIncrement64(&(burst_latency_histogram[find_io_bin(burst_elapsed, LATENCY_BIN_SIZE, LATENCY_BINS)]));
 311                 OSAtomicIncrement64(&total_burst_count);
 312
 313                 if(inter_burst_duration == -1)
 314                         usleep((rand() % 100) * 1000);
 315                 else
 316                         usleep(inter_burst_duration * 1000);
 317         }
 318
 319         free(data);
 320         close(fd);
 321         pthread_exit(NULL);
 322 }
 323
 324 void validate_option(int value, int min, int max, char *option, char *units)
 325 {
 326         if (value < min || value > max) {
 327                 printf("Illegal option value %d for %s (Min value: %d %s, Max value: %d %s).\n", value, option, min, units, max, units);
 328                 exit(1);
 329         }
 330 }
 331
 332 void print_test_setup(int value, char *option, char *units, char *comment)
 333 {
 334         if (comment == NULL)
 335                 printf("%32s: %16d %-16s\n", option, value, units);
 336         else
 337                 printf("%32s: %16d %-16s (%s)\n", option, value, units, comment);
 338 }
 339
 340 void setup_process_io_policy(int io_tier)
 341 {
 342         switch(io_tier)
 343         {
 344                 case 0:
 345                         if (setiopolicy_np(IOPOL_TYPE_DISK, IOPOL_SCOPE_PROCESS, IOPOL_IMPORTANT))
 346                                 goto iopol_error;
 347                         break;
 348                 case 1:
 349                         if (setiopolicy_np(IOPOL_TYPE_DISK, IOPOL_SCOPE_PROCESS, IOPOL_STANDARD))
 350                                 goto iopol_error;
 351                         break;
 352                 case 2:
 353                         if (setiopolicy_np(IOPOL_TYPE_DISK, IOPOL_SCOPE_PROCESS, IOPOL_UTILITY))
 354                                 goto iopol_error;
 355                         break;
 356                 case 3:
 357                         if (setiopolicy_np(IOPOL_TYPE_DISK, IOPOL_SCOPE_PROCESS, IOPOL_THROTTLE))
 358                                 goto iopol_error;
 359                         break;
 360         }
 361         return;
 362
 363 iopol_error:
 364         printf("Error setting process-wide I/O policy to %d\n", io_tier);
 365         exit(1);
 366 }
 367
 368 int main(int argc, char *argv[])
 369 {
 370         int i, option = 0;
 371         pthread_t thread_list[MAX_THREADS];
 372         pthread_t sync_thread;
 373         pthread_t throughput_thread;
 374         char fname[MAX_FILENAME];
 375
 376         while((option = getopt(argc, argv,"hc:i:d:t:f:m:j:s:x:l:z:n:a:")) != -1) {
 377                 switch(option) {
 378                         case 'c':
 379                                 burst_count = atoi(optarg);
 380                                 validate_option(burst_count, 0, INT_MAX, "Burst Count", "I/Os");
 381                                 break;
 382                         case 'i':
 383                                 inter_burst_duration = atoi(optarg);
 384                                 validate_option(inter_burst_duration, -1, INT_MAX, "Inter Burst duration", "msecs");
 385                                 break;
 386                         case 'd':
 387                                 inter_io_delay_ms = atoi(optarg);
 388                                 validate_option(inter_io_delay_ms, 0, INT_MAX, "Inter I/O Delay", "msecs");
 389                                 break;
 390                         case 't':
 391                                 thread_count = atoi(optarg);
 392                                 validate_option(thread_count, 0, MAX_THREADS, "Thread Count", "Threads");
 393                                 break;
 394                         case 'f':
 395                                 workload_type = atoi(optarg);
 396                                 validate_option(workload_type, 0, 2, "Workload Type", "");
 397                                 break;
 398                         case 'm':
 399                                 io_mode = atoi(optarg);
 400                                 validate_option(io_mode, 0, 1, "I/O Mode", "");
 401                                 break;
 402                         case 'j':
 403                                 io_size = atoi(optarg);
 404                                 validate_option(io_size, 0, INT_MAX, "I/O Size", "Bytes");
 405                                 break;
 406                         case 'h':
 407                                 print_usage();
 408                                 exit(1);
 409                         case 's':
 410                                 sync_frequency_ms = atoi(optarg);
 411                                 validate_option(sync_frequency_ms, 0, INT_MAX, "Sync. Frequency", "msecs");
 412                                 break;
 413                         case 'x':
 414                                 test_duration = atoi(optarg);
 415                                 validate_option(test_duration, 0, INT_MAX, "Test duration", "secs");
 416                                 break;
 417                         case 'l':
 418                                 io_tier = atoi(optarg);
 419                                 validate_option(io_tier, 0, 3, "I/O Tier", "");
 420                                 break;
 421                         case 'z':
 422                                 file_size = atoi(optarg);
 423                                 validate_option(file_size, 0, INT_MAX, "File Size", "bytes");
 424                                 break;
 425                         case 'n':
 426                                 user_fname = optarg;
 427                                 user_specified_file = 1;
 428                                 break;
 429                         case 'a':
 430                                 cached_io_flag = atoi(optarg);
 431                                 validate_option(cached_io_flag, 0, 1, "I/Os cached/no-cached", "");
 432                                 break;
 433                         default:
 434                                 printf("Unknown option %c\n", option);
 435                                 print_usage();
 436                                 exit(1);
 437                 }
 438         }
 439
 440         printf("***********************TEST SETUP*************************\n");
 441
 442         print_test_setup(burst_count, "Burst Count", "I/Os", 0);
 443         print_test_setup(inter_burst_duration, "Inter Burst duration", "msecs", "-1 indicates random burst duration");
 444         print_test_setup(inter_io_delay_ms, "Inter I/O Delay", "msecs", 0);
 445         print_test_setup(thread_count, "Thread Count", "Threads", 0);
 446         print_test_setup(workload_type, "Workload Type", "", "0:R 1:W 2:RW");
 447         print_test_setup(io_mode, "I/O Mode", "", "0:Seq. 1:Rnd");
 448         print_test_setup(io_size, "I/O Size", "Bytes", 0);
 449         print_test_setup(sync_frequency_ms, "Sync. Frequency", "msecs", "0 indicates no sync. thread");
 450         print_test_setup(test_duration, "Test duration", "secs", "0 indicates tool waits for Ctrl+C");
 451         print_test_setup(io_tier, "I/O Tier", "", 0);
 452         print_test_setup(cached_io_flag, "I/O Caching", "", "0 indicates non-cached I/Os");
 453         print_test_setup(0, "File read-aheads", "", "0 indicates read-aheads disabled");
 454
 455         printf("**********************************************************\n");
 456
 457         if (user_specified_file == 0) {
 458                 char dd_command[MAX_CMD_SIZE];
 459                 for (i=0; i < thread_count; i++) {
 460                         snprintf(fname, MAX_FILENAME, "iosim-%d-%d", (int)getpid(), i);
 461                         snprintf(dd_command, MAX_CMD_SIZE, "dd if=/dev/urandom of=%s bs=4096 count=%d", fname, file_size);
 462                         printf("Creating file %s of size %lld...\n", fname, ((int64_t)file_size * 4096));
 463                         system(dd_command);
 464                 }
 465         } else {
 466                 printf("Using user specified file %s for all threads...\n", user_fname);
 467         }
 468         system("purge");
 469         setup_process_io_policy(io_tier);
 470
 471         printf("**********************************************************\n");
 472         printf("Creating threads and generating workload...\n");
 473
 474         signal(SIGINT, signalHandler);
 475         signal(SIGALRM, signalHandler);
 476
 477         for(i=0; i < thread_count; i++) {
 478                 if (pthread_create(&thread_list[i], NULL, io_routine, i) < 0) {
 479                         perror("Could not create I/O thread!\n");
 480                         exit(1);
 481                 }
 482         }
 483
 484         if (sync_frequency_ms) {
 485                 if (pthread_create(&sync_thread, NULL, sync_routine, NULL) < 0) {
 486                         perror("Could not create sync thread!\n");
 487                         exit(1);
 488                 }
 489         }
 490
 491         if (pthread_create(&throughput_thread, NULL, calculate_throughput, NULL) < 0) {
 492                 perror("Could not throughput calculation thread!\n");
 493                 exit(1);
 494         }
 495
 496         /* All threads are now initialized */
 497         if (test_duration)
 498                 alarm(test_duration);
 499
 500         for(i=0; i < thread_count; i++)
 501                 pthread_join(thread_list[i], NULL);
 502
 503         if (sync_frequency_ms)
 504                 pthread_join(sync_thread, NULL);
 505
 506         pthread_join(throughput_thread, NULL);
 507
 508         pthread_exit(0);
 509 }