src/aof.c

   1 #include "redis.h"
   2 #include "bio.h"
   3
   4 #include <signal.h>
   5 #include <fcntl.h>
   6 #include <sys/stat.h>
   7 #include <sys/types.h>
   8 #include <sys/time.h>
   9 #include <sys/resource.h>
  10 #include <sys/wait.h>
  11
  12 void aofUpdateCurrentSize(void);
  13
  14 /* Called when the user switches from "appendonly yes" to "appendonly no"
  15  * at runtime using the CONFIG command. */
  16 void stopAppendOnly(void) {
  17     flushAppendOnlyFile();
  18     aof_fsync(server.appendfd);
  19     close(server.appendfd);
  20
  21     server.appendfd = -1;
  22     server.appendseldb = -1;
  23     server.appendonly = 0;
  24     /* rewrite operation in progress? kill it, wait child exit */
  25     if (server.bgrewritechildpid != -1) {
  26         int statloc;
  27
  28         if (kill(server.bgrewritechildpid,SIGKILL) != -1)
  29             wait3(&statloc,0,NULL);
  30         /* reset the buffer accumulating changes while the child saves */
  31         sdsfree(server.bgrewritebuf);
  32         server.bgrewritebuf = sdsempty();
  33         server.bgrewritechildpid = -1;
  34     }
  35 }
  36
  37 /* Called when the user switches from "appendonly no" to "appendonly yes"
  38  * at runtime using the CONFIG command. */
  39 int startAppendOnly(void) {
  40     server.appendonly = 1;
  41     server.lastfsync = time(NULL);
  42     server.appendfd = open(server.appendfilename,O_WRONLY|O_APPEND|O_CREAT,0644);
  43     if (server.appendfd == -1) {
  44         redisLog(REDIS_WARNING,"Used tried to switch on AOF via CONFIG, but I can't open the AOF file: %s",strerror(errno));
  45         return REDIS_ERR;
  46     }
  47     if (rewriteAppendOnlyFileBackground() == REDIS_ERR) {
  48         server.appendonly = 0;
  49         close(server.appendfd);
  50         redisLog(REDIS_WARNING,"Used tried to switch on AOF via CONFIG, I can't trigger a background AOF rewrite operation. Check the above logs for more info about the error.",strerror(errno));
  51         return REDIS_ERR;
  52     }
  53     return REDIS_OK;
  54 }
  55
  56 /* Write the append only file buffer on disk.
  57  *
  58  * Since we are required to write the AOF before replying to the client,
  59  * and the only way the client socket can get a write is entering when the
  60  * the event loop, we accumulate all the AOF writes in a memory
  61  * buffer and write it on disk using this function just before entering
  62  * the event loop again. */
  63 void flushAppendOnlyFile(void) {
  64     ssize_t nwritten;
  65
  66     if (sdslen(server.aofbuf) == 0) return;
  67
  68     /* We want to perform a single write. This should be guaranteed atomic
  69      * at least if the filesystem we are writing is a real physical one.
  70      * While this will save us against the server being killed I don't think
  71      * there is much to do about the whole server stopping for power problems
  72      * or alike */
  73     nwritten = write(server.appendfd,server.aofbuf,sdslen(server.aofbuf));
  74     if (nwritten != (signed)sdslen(server.aofbuf)) {
  75         /* Ooops, we are in troubles. The best thing to do for now is
  76          * aborting instead of giving the illusion that everything is
  77          * working as expected. */
  78         if (nwritten == -1) {
  79             redisLog(REDIS_WARNING,"Exiting on error writing to the append-only file: %s",strerror(errno));
  80         } else {
  81             redisLog(REDIS_WARNING,"Exiting on short write while writing to the append-only file: %s",strerror(errno));
  82         }
  83         exit(1);
  84     }
  85     server.appendonly_current_size += nwritten;
  86
  87     /* Re-use AOF buffer when it is small enough. The maximum comes from the
  88      * arena size of 4k minus some overhead (but is otherwise arbitrary). */
  89     if ((sdslen(server.aofbuf)+sdsavail(server.aofbuf)) < 4000) {
  90         sdsclear(server.aofbuf);
  91     } else {
  92         sdsfree(server.aofbuf);
  93         server.aofbuf = sdsempty();
  94     }
  95
  96     /* Don't fsync if no-appendfsync-on-rewrite is set to yes and there are
  97      * children doing I/O in the background. */
  98     if (server.no_appendfsync_on_rewrite &&
  99         (server.bgrewritechildpid != -1 || server.bgsavechildpid != -1))
 100             return;
 101
 102     /* Perform the fsync if needed. */
 103     if (server.appendfsync == APPENDFSYNC_ALWAYS ||
 104         (server.appendfsync == APPENDFSYNC_EVERYSEC &&
 105          server.unixtime > server.lastfsync))
 106     {
 107         /* aof_fsync is defined as fdatasync() for Linux in order to avoid
 108          * flushing metadata. */
 109         aof_fsync(server.appendfd); /* Let's try to get this data on the disk */
 110         server.lastfsync = server.unixtime;
 111     }
 112 }
 113
 114 sds catAppendOnlyGenericCommand(sds dst, int argc, robj **argv) {
 115     char buf[32];
 116     int len, j;
 117     robj *o;
 118
 119     buf[0] = '*';
 120     len = 1+ll2string(buf+1,sizeof(buf)-1,argc);
 121     buf[len++] = '\r';
 122     buf[len++] = '\n';
 123     dst = sdscatlen(dst,buf,len);
 124
 125     for (j = 0; j < argc; j++) {
 126         o = getDecodedObject(argv[j]);
 127         buf[0] = '$';
 128         len = 1+ll2string(buf+1,sizeof(buf)-1,sdslen(o->ptr));
 129         buf[len++] = '\r';
 130         buf[len++] = '\n';
 131         dst = sdscatlen(dst,buf,len);
 132         dst = sdscatlen(dst,o->ptr,sdslen(o->ptr));
 133         dst = sdscatlen(dst,"\r\n",2);
 134         decrRefCount(o);
 135     }
 136     return dst;
 137 }
 138
 139 sds catAppendOnlyExpireAtCommand(sds buf, robj *key, robj *seconds) {
 140     int argc = 3;
 141     long when;
 142     robj *argv[3];
 143
 144     /* Make sure we can use strtol */
 145     seconds = getDecodedObject(seconds);
 146     when = time(NULL)+strtol(seconds->ptr,NULL,10);
 147     decrRefCount(seconds);
 148
 149     argv[0] = createStringObject("EXPIREAT",8);
 150     argv[1] = key;
 151     argv[2] = createObject(REDIS_STRING,
 152         sdscatprintf(sdsempty(),"%ld",when));
 153     buf = catAppendOnlyGenericCommand(buf, argc, argv);
 154     decrRefCount(argv[0]);
 155     decrRefCount(argv[2]);
 156     return buf;
 157 }
 158
 159 void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int argc) {
 160     sds buf = sdsempty();
 161     robj *tmpargv[3];
 162
 163     /* The DB this command was targetting is not the same as the last command
 164      * we appendend. To issue a SELECT command is needed. */
 165     if (dictid != server.appendseldb) {
 166         char seldb[64];
 167
 168         snprintf(seldb,sizeof(seldb),"%d",dictid);
 169         buf = sdscatprintf(buf,"*2\r\n$6\r\nSELECT\r\n$%lu\r\n%s\r\n",
 170             (unsigned long)strlen(seldb),seldb);
 171         server.appendseldb = dictid;
 172     }
 173
 174     if (cmd->proc == expireCommand) {
 175         /* Translate EXPIRE into EXPIREAT */
 176         buf = catAppendOnlyExpireAtCommand(buf,argv[1],argv[2]);
 177     } else if (cmd->proc == setexCommand) {
 178         /* Translate SETEX to SET and EXPIREAT */
 179         tmpargv[0] = createStringObject("SET",3);
 180         tmpargv[1] = argv[1];
 181         tmpargv[2] = argv[3];
 182         buf = catAppendOnlyGenericCommand(buf,3,tmpargv);
 183         decrRefCount(tmpargv[0]);
 184         buf = catAppendOnlyExpireAtCommand(buf,argv[1],argv[2]);
 185     } else {
 186         buf = catAppendOnlyGenericCommand(buf,argc,argv);
 187     }
 188
 189     /* Append to the AOF buffer. This will be flushed on disk just before
 190      * of re-entering the event loop, so before the client will get a
 191      * positive reply about the operation performed. */
 192     server.aofbuf = sdscatlen(server.aofbuf,buf,sdslen(buf));
 193
 194     /* If a background append only file rewriting is in progress we want to
 195      * accumulate the differences between the child DB and the current one
 196      * in a buffer, so that when the child process will do its work we
 197      * can append the differences to the new append only file. */
 198     if (server.bgrewritechildpid != -1)
 199         server.bgrewritebuf = sdscatlen(server.bgrewritebuf,buf,sdslen(buf));
 200
 201     sdsfree(buf);
 202 }
 203
 204 /* In Redis commands are always executed in the context of a client, so in
 205  * order to load the append only file we need to create a fake client. */
 206 struct redisClient *createFakeClient(void) {
 207     struct redisClient *c = zmalloc(sizeof(*c));
 208
 209     selectDb(c,0);
 210     c->fd = -1;
 211     c->querybuf = sdsempty();
 212     c->argc = 0;
 213     c->argv = NULL;
 214     c->bufpos = 0;
 215     c->flags = 0;
 216     /* We set the fake client as a slave waiting for the synchronization
 217      * so that Redis will not try to send replies to this client. */
 218     c->replstate = REDIS_REPL_WAIT_BGSAVE_START;
 219     c->reply = listCreate();
 220     c->watched_keys = listCreate();
 221     listSetFreeMethod(c->reply,decrRefCount);
 222     listSetDupMethod(c->reply,dupClientReplyValue);
 223     initClientMultiState(c);
 224     return c;
 225 }
 226
 227 void freeFakeClient(struct redisClient *c) {
 228     sdsfree(c->querybuf);
 229     listRelease(c->reply);
 230     listRelease(c->watched_keys);
 231     freeClientMultiState(c);
 232     zfree(c);
 233 }
 234
 235 /* Replay the append log file. On error REDIS_OK is returned. On non fatal
 236  * error (the append only file is zero-length) REDIS_ERR is returned. On
 237  * fatal error an error message is logged and the program exists. */
 238 int loadAppendOnlyFile(char *filename) {
 239     struct redisClient *fakeClient;
 240     FILE *fp = fopen(filename,"r");
 241     struct redis_stat sb;
 242     int appendonly = server.appendonly;
 243     long loops = 0;
 244
 245     if (fp && redis_fstat(fileno(fp),&sb) != -1 && sb.st_size == 0) {
 246         server.appendonly_current_size = 0;
 247         fclose(fp);
 248         return REDIS_ERR;
 249     }
 250
 251     if (fp == NULL) {
 252         redisLog(REDIS_WARNING,"Fatal error: can't open the append log file for reading: %s",strerror(errno));
 253         exit(1);
 254     }
 255
 256     /* Temporarily disable AOF, to prevent EXEC from feeding a MULTI
 257      * to the same file we're about to read. */
 258     server.appendonly = 0;
 259
 260     fakeClient = createFakeClient();
 261     startLoading(fp);
 262
 263     while(1) {
 264         int argc, j;
 265         unsigned long len;
 266         robj **argv;
 267         char buf[128];
 268         sds argsds;
 269         struct redisCommand *cmd;
 270
 271         /* Serve the clients from time to time */
 272         if (!(loops++ % 1000)) {
 273             loadingProgress(ftello(fp));
 274             aeProcessEvents(server.el, AE_FILE_EVENTS|AE_DONT_WAIT);
 275         }
 276
 277         if (fgets(buf,sizeof(buf),fp) == NULL) {
 278             if (feof(fp))
 279                 break;
 280             else
 281                 goto readerr;
 282         }
 283         if (buf[0] != '*') goto fmterr;
 284         argc = atoi(buf+1);
 285         argv = zmalloc(sizeof(robj*)*argc);
 286         for (j = 0; j < argc; j++) {
 287             if (fgets(buf,sizeof(buf),fp) == NULL) goto readerr;
 288             if (buf[0] != '$') goto fmterr;
 289             len = strtol(buf+1,NULL,10);
 290             argsds = sdsnewlen(NULL,len);
 291             if (len && fread(argsds,len,1,fp) == 0) goto fmterr;
 292             argv[j] = createObject(REDIS_STRING,argsds);
 293             if (fread(buf,2,1,fp) == 0) goto fmterr; /* discard CRLF */
 294         }
 295
 296         /* Command lookup */
 297         cmd = lookupCommand(argv[0]->ptr);
 298         if (!cmd) {
 299             redisLog(REDIS_WARNING,"Unknown command '%s' reading the append only file", argv[0]->ptr);
 300             exit(1);
 301         }
 302         /* Run the command in the context of a fake client */
 303         fakeClient->argc = argc;
 304         fakeClient->argv = argv;
 305         cmd->proc(fakeClient);
 306
 307         /* The fake client should not have a reply */
 308         redisAssert(fakeClient->bufpos == 0 && listLength(fakeClient->reply) == 0);
 309         /* The fake client should never get blocked */
 310         redisAssert((fakeClient->flags & REDIS_BLOCKED) == 0);
 311
 312         /* Clean up. Command code may have changed argv/argc so we use the
 313          * argv/argc of the client instead of the local variables. */
 314         for (j = 0; j < fakeClient->argc; j++)
 315             decrRefCount(fakeClient->argv[j]);
 316         zfree(fakeClient->argv);
 317     }
 318
 319     /* This point can only be reached when EOF is reached without errors.
 320      * If the client is in the middle of a MULTI/EXEC, log error and quit. */
 321     if (fakeClient->flags & REDIS_MULTI) goto readerr;
 322
 323     fclose(fp);
 324     freeFakeClient(fakeClient);
 325     server.appendonly = appendonly;
 326     stopLoading();
 327     aofUpdateCurrentSize();
 328     server.auto_aofrewrite_base_size = server.appendonly_current_size;
 329     return REDIS_OK;
 330
 331 readerr:
 332     if (feof(fp)) {
 333         redisLog(REDIS_WARNING,"Unexpected end of file reading the append only file");
 334     } else {
 335         redisLog(REDIS_WARNING,"Unrecoverable error reading the append only file: %s", strerror(errno));
 336     }
 337     exit(1);
 338 fmterr:
 339     redisLog(REDIS_WARNING,"Bad file format reading the append only file: make a backup of your AOF file, then use ./redis-check-aof --fix <filename>");
 340     exit(1);
 341 }
 342
 343 /* Write a sequence of commands able to fully rebuild the dataset into
 344  * "filename". Used both by REWRITEAOF and BGREWRITEAOF. */
 345 int rewriteAppendOnlyFile(char *filename) {
 346     dictIterator *di = NULL;
 347     dictEntry *de;
 348     FILE *fp;
 349     char tmpfile[256];
 350     int j;
 351     time_t now = time(NULL);
 352
 353     /* Note that we have to use a different temp name here compared to the
 354      * one used by rewriteAppendOnlyFileBackground() function. */
 355     snprintf(tmpfile,256,"temp-rewriteaof-%d.aof", (int) getpid());
 356     fp = fopen(tmpfile,"w");
 357     if (!fp) {
 358         redisLog(REDIS_WARNING, "Failed rewriting the append only file: %s", strerror(errno));
 359         return REDIS_ERR;
 360     }
 361     for (j = 0; j < server.dbnum; j++) {
 362         char selectcmd[] = "*2\r\n$6\r\nSELECT\r\n";
 363         redisDb *db = server.db+j;
 364         dict *d = db->dict;
 365         if (dictSize(d) == 0) continue;
 366         di = dictGetSafeIterator(d);
 367         if (!di) {
 368             fclose(fp);
 369             return REDIS_ERR;
 370         }
 371
 372         /* SELECT the new DB */
 373         if (fwrite(selectcmd,sizeof(selectcmd)-1,1,fp) == 0) goto werr;
 374         if (fwriteBulkLongLong(fp,j) == 0) goto werr;
 375
 376         /* Iterate this DB writing every entry */
 377         while((de = dictNext(di)) != NULL) {
 378             sds keystr;
 379             robj key, *o;
 380             time_t expiretime;
 381
 382             keystr = dictGetEntryKey(de);
 383             o = dictGetEntryVal(de);
 384             initStaticStringObject(key,keystr);
 385
 386             expiretime = getExpire(db,&key);
 387
 388             /* Save the key and associated value */
 389             if (o->type == REDIS_STRING) {
 390                 /* Emit a SET command */
 391                 char cmd[]="*3\r\n$3\r\nSET\r\n";
 392                 if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
 393                 /* Key and value */
 394                 if (fwriteBulkObject(fp,&key) == 0) goto werr;
 395                 if (fwriteBulkObject(fp,o) == 0) goto werr;
 396             } else if (o->type == REDIS_LIST) {
 397                 /* Emit the RPUSHes needed to rebuild the list */
 398                 char cmd[]="*3\r\n$5\r\nRPUSH\r\n";
 399                 if (o->encoding == REDIS_ENCODING_ZIPLIST) {
 400                     unsigned char *zl = o->ptr;
 401                     unsigned char *p = ziplistIndex(zl,0);
 402                     unsigned char *vstr;
 403                     unsigned int vlen;
 404                     long long vlong;
 405
 406                     while(ziplistGet(p,&vstr,&vlen,&vlong)) {
 407                         if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
 408                         if (fwriteBulkObject(fp,&key) == 0) goto werr;
 409                         if (vstr) {
 410                             if (fwriteBulkString(fp,(char*)vstr,vlen) == 0)
 411                                 goto werr;
 412                         } else {
 413                             if (fwriteBulkLongLong(fp,vlong) == 0)
 414                                 goto werr;
 415                         }
 416                         p = ziplistNext(zl,p);
 417                     }
 418                 } else if (o->encoding == REDIS_ENCODING_LINKEDLIST) {
 419                     list *list = o->ptr;
 420                     listNode *ln;
 421                     listIter li;
 422
 423                     listRewind(list,&li);
 424                     while((ln = listNext(&li))) {
 425                         robj *eleobj = listNodeValue(ln);
 426
 427                         if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
 428                         if (fwriteBulkObject(fp,&key) == 0) goto werr;
 429                         if (fwriteBulkObject(fp,eleobj) == 0) goto werr;
 430                     }
 431                 } else {
 432                     redisPanic("Unknown list encoding");
 433                 }
 434             } else if (o->type == REDIS_SET) {
 435                 char cmd[]="*3\r\n$4\r\nSADD\r\n";
 436
 437                 /* Emit the SADDs needed to rebuild the set */
 438                 if (o->encoding == REDIS_ENCODING_INTSET) {
 439                     int ii = 0;
 440                     int64_t llval;
 441                     while(intsetGet(o->ptr,ii++,&llval)) {
 442                         if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
 443                         if (fwriteBulkObject(fp,&key) == 0) goto werr;
 444                         if (fwriteBulkLongLong(fp,llval) == 0) goto werr;
 445                     }
 446                 } else if (o->encoding == REDIS_ENCODING_HT) {
 447                     dictIterator *di = dictGetIterator(o->ptr);
 448                     dictEntry *de;
 449                     while((de = dictNext(di)) != NULL) {
 450                         robj *eleobj = dictGetEntryKey(de);
 451                         if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
 452                         if (fwriteBulkObject(fp,&key) == 0) goto werr;
 453                         if (fwriteBulkObject(fp,eleobj) == 0) goto werr;
 454                     }
 455                     dictReleaseIterator(di);
 456                 } else {
 457                     redisPanic("Unknown set encoding");
 458                 }
 459             } else if (o->type == REDIS_ZSET) {
 460                 /* Emit the ZADDs needed to rebuild the sorted set */
 461                 char cmd[]="*4\r\n$4\r\nZADD\r\n";
 462
 463                 if (o->encoding == REDIS_ENCODING_ZIPLIST) {
 464                     unsigned char *zl = o->ptr;
 465                     unsigned char *eptr, *sptr;
 466                     unsigned char *vstr;
 467                     unsigned int vlen;
 468                     long long vll;
 469                     double score;
 470
 471                     eptr = ziplistIndex(zl,0);
 472                     redisAssert(eptr != NULL);
 473                     sptr = ziplistNext(zl,eptr);
 474                     redisAssert(sptr != NULL);
 475
 476                     while (eptr != NULL) {
 477                         redisAssert(ziplistGet(eptr,&vstr,&vlen,&vll));
 478                         score = zzlGetScore(sptr);
 479
 480                         if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
 481                         if (fwriteBulkObject(fp,&key) == 0) goto werr;
 482                         if (fwriteBulkDouble(fp,score) == 0) goto werr;
 483                         if (vstr != NULL) {
 484                             if (fwriteBulkString(fp,(char*)vstr,vlen) == 0)
 485                                 goto werr;
 486                         } else {
 487                             if (fwriteBulkLongLong(fp,vll) == 0)
 488                                 goto werr;
 489                         }
 490                         zzlNext(zl,&eptr,&sptr);
 491                     }
 492                 } else if (o->encoding == REDIS_ENCODING_SKIPLIST) {
 493                     zset *zs = o->ptr;
 494                     dictIterator *di = dictGetIterator(zs->dict);
 495                     dictEntry *de;
 496
 497                     while((de = dictNext(di)) != NULL) {
 498                         robj *eleobj = dictGetEntryKey(de);
 499                         double *score = dictGetEntryVal(de);
 500
 501                         if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
 502                         if (fwriteBulkObject(fp,&key) == 0) goto werr;
 503                         if (fwriteBulkDouble(fp,*score) == 0) goto werr;
 504                         if (fwriteBulkObject(fp,eleobj) == 0) goto werr;
 505                     }
 506                     dictReleaseIterator(di);
 507                 } else {
 508                     redisPanic("Unknown sorted set encoding");
 509                 }
 510             } else if (o->type == REDIS_HASH) {
 511                 char cmd[]="*4\r\n$4\r\nHSET\r\n";
 512
 513                 /* Emit the HSETs needed to rebuild the hash */
 514                 if (o->encoding == REDIS_ENCODING_ZIPMAP) {
 515                     unsigned char *p = zipmapRewind(o->ptr);
 516                     unsigned char *field, *val;
 517                     unsigned int flen, vlen;
 518
 519                     while((p = zipmapNext(p,&field,&flen,&val,&vlen)) != NULL) {
 520                         if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
 521                         if (fwriteBulkObject(fp,&key) == 0) goto werr;
 522                         if (fwriteBulkString(fp,(char*)field,flen) == 0)
 523                             goto werr;
 524                         if (fwriteBulkString(fp,(char*)val,vlen) == 0)
 525                             goto werr;
 526                     }
 527                 } else {
 528                     dictIterator *di = dictGetIterator(o->ptr);
 529                     dictEntry *de;
 530
 531                     while((de = dictNext(di)) != NULL) {
 532                         robj *field = dictGetEntryKey(de);
 533                         robj *val = dictGetEntryVal(de);
 534
 535                         if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
 536                         if (fwriteBulkObject(fp,&key) == 0) goto werr;
 537                         if (fwriteBulkObject(fp,field) == 0) goto werr;
 538                         if (fwriteBulkObject(fp,val) == 0) goto werr;
 539                     }
 540                     dictReleaseIterator(di);
 541                 }
 542             } else {
 543                 redisPanic("Unknown object type");
 544             }
 545             /* Save the expire time */
 546             if (expiretime != -1) {
 547                 char cmd[]="*3\r\n$8\r\nEXPIREAT\r\n";
 548                 /* If this key is already expired skip it */
 549                 if (expiretime < now) continue;
 550                 if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
 551                 if (fwriteBulkObject(fp,&key) == 0) goto werr;
 552                 if (fwriteBulkLongLong(fp,expiretime) == 0) goto werr;
 553             }
 554         }
 555         dictReleaseIterator(di);
 556     }
 557
 558     /* Make sure data will not remain on the OS's output buffers */
 559     fflush(fp);
 560     aof_fsync(fileno(fp));
 561     fclose(fp);
 562
 563     /* Use RENAME to make sure the DB file is changed atomically only
 564      * if the generate DB file is ok. */
 565     if (rename(tmpfile,filename) == -1) {
 566         redisLog(REDIS_WARNING,"Error moving temp append only file on the final destination: %s", strerror(errno));
 567         unlink(tmpfile);
 568         return REDIS_ERR;
 569     }
 570     redisLog(REDIS_NOTICE,"SYNC append only file rewrite performed");
 571     return REDIS_OK;
 572
 573 werr:
 574     fclose(fp);
 575     unlink(tmpfile);
 576     redisLog(REDIS_WARNING,"Write error writing append only file on disk: %s", strerror(errno));
 577     if (di) dictReleaseIterator(di);
 578     return REDIS_ERR;
 579 }
 580
 581 /* This is how rewriting of the append only file in background works:
 582  *
 583  * 1) The user calls BGREWRITEAOF
 584  * 2) Redis calls this function, that forks():
 585  *    2a) the child rewrite the append only file in a temp file.
 586  *    2b) the parent accumulates differences in server.bgrewritebuf.
 587  * 3) When the child finished '2a' exists.
 588  * 4) The parent will trap the exit code, if it's OK, will append the
 589  *    data accumulated into server.bgrewritebuf into the temp file, and
 590  *    finally will rename(2) the temp file in the actual file name.
 591  *    The the new file is reopened as the new append only file. Profit!
 592  */
 593 int rewriteAppendOnlyFileBackground(void) {
 594     pid_t childpid;
 595     long long start;
 596
 597     if (server.bgrewritechildpid != -1) return REDIS_ERR;
 598     start = ustime();
 599     if ((childpid = fork()) == 0) {
 600         char tmpfile[256];
 601
 602         /* Child */
 603         if (server.ipfd > 0) close(server.ipfd);
 604         if (server.sofd > 0) close(server.sofd);
 605         snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) getpid());
 606         if (rewriteAppendOnlyFile(tmpfile) == REDIS_OK) {
 607             _exit(0);
 608         } else {
 609             _exit(1);
 610         }
 611     } else {
 612         /* Parent */
 613         server.stat_fork_time = ustime()-start;
 614         if (childpid == -1) {
 615             redisLog(REDIS_WARNING,
 616                 "Can't rewrite append only file in background: fork: %s",
 617                 strerror(errno));
 618             return REDIS_ERR;
 619         }
 620         redisLog(REDIS_NOTICE,
 621             "Background append only file rewriting started by pid %d",childpid);
 622         server.bgrewritechildpid = childpid;
 623         updateDictResizePolicy();
 624         /* We set appendseldb to -1 in order to force the next call to the
 625          * feedAppendOnlyFile() to issue a SELECT command, so the differences
 626          * accumulated by the parent into server.bgrewritebuf will start
 627          * with a SELECT statement and it will be safe to merge. */
 628         server.appendseldb = -1;
 629         return REDIS_OK;
 630     }
 631     return REDIS_OK; /* unreached */
 632 }
 633
 634 void bgrewriteaofCommand(redisClient *c) {
 635     if (server.bgrewritechildpid != -1) {
 636         addReplyError(c,"Background append only file rewriting already in progress");
 637     } else if (server.bgsavechildpid != -1) {
 638         server.aofrewrite_scheduled = 1;
 639         addReplyStatus(c,"Background append only file rewriting scheduled");
 640     } else if (rewriteAppendOnlyFileBackground() == REDIS_OK) {
 641         addReplyStatus(c,"Background append only file rewriting started");
 642     } else {
 643         addReply(c,shared.err);
 644     }
 645 }
 646
 647 void aofRemoveTempFile(pid_t childpid) {
 648     char tmpfile[256];
 649
 650     snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) childpid);
 651     unlink(tmpfile);
 652 }
 653
 654 /* Update the server.appendonly_current_size filed explicitly using stat(2)
 655  * to check the size of the file. This is useful after a rewrite or after
 656  * a restart, normally the size is updated just adding the write length
 657  * to the current lenght, that is much faster. */
 658 void aofUpdateCurrentSize(void) {
 659     struct redis_stat sb;
 660
 661     if (redis_fstat(server.appendfd,&sb) == -1) {
 662         redisLog(REDIS_WARNING,"Unable to check the AOF length: %s",
 663             strerror(errno));
 664     } else {
 665         server.appendonly_current_size = sb.st_size;
 666     }
 667 }
 668
 669 /* A background append only file rewriting (BGREWRITEAOF) terminated its work.
 670  * Handle this. */
 671 void backgroundRewriteDoneHandler(int exitcode, int bysignal) {
 672     if (!bysignal && exitcode == 0) {
 673         int newfd, oldfd;
 674         int nwritten;
 675         char tmpfile[256];
 676         long long now = ustime();
 677
 678         redisLog(REDIS_NOTICE,
 679             "Background AOF rewrite terminated with success");
 680
 681         /* Flush the differences accumulated by the parent to the
 682          * rewritten AOF. */
 683         snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof",
 684             (int)server.bgrewritechildpid);
 685         newfd = open(tmpfile,O_WRONLY|O_APPEND);
 686         if (newfd == -1) {
 687             redisLog(REDIS_WARNING,
 688                 "Unable to open the temporary AOF produced by the child: %s", strerror(errno));
 689             goto cleanup;
 690         }
 691
 692         nwritten = write(newfd,server.bgrewritebuf,sdslen(server.bgrewritebuf));
 693         if (nwritten != (signed)sdslen(server.bgrewritebuf)) {
 694             if (nwritten == -1) {
 695                 redisLog(REDIS_WARNING,
 696                     "Error trying to flush the parent diff to the rewritten AOF: %s", strerror(errno));
 697             } else {
 698                 redisLog(REDIS_WARNING,
 699                     "Short write trying to flush the parent diff to the rewritten AOF: %s", strerror(errno));
 700             }
 701             close(newfd);
 702             goto cleanup;
 703         }
 704
 705         redisLog(REDIS_NOTICE,
 706             "Parent diff successfully flushed to the rewritten AOF (%lu bytes)", nwritten);
 707
 708         /* The only remaining thing to do is to rename the temporary file to
 709          * the configured file and switch the file descriptor used to do AOF
 710          * writes. We don't want close(2) or rename(2) calls to block the
 711          * server on old file deletion.
 712          *
 713          * There are two possible scenarios:
 714          *
 715          * 1) AOF is DISABLED and this was a one time rewrite. The temporary
 716          * file will be renamed to the configured file. When this file already
 717          * exists, it will be unlinked, which may block the server.
 718          *
 719          * 2) AOF is ENABLED and the rewritten AOF will immediately start
 720          * receiving writes. After the temporary file is renamed to the
 721          * configured file, the original AOF file descriptor will be closed.
 722          * Since this will be the last reference to that file, closing it
 723          * causes the underlying file to be unlinked, which may block the
 724          * server.
 725          *
 726          * To mitigate the blocking effect of the unlink operation (either
 727          * caused by rename(2) in scenario 1, or by close(2) in scenario 2), we
 728          * use a background thread to take care of this. First, we
 729          * make scenario 1 identical to scenario 2 by opening the target file
 730          * when it exists. The unlink operation after the rename(2) will then
 731          * be executed upon calling close(2) for its descriptor. Everything to
 732          * guarantee atomicity for this switch has already happened by then, so
 733          * we don't care what the outcome or duration of that close operation
 734          * is, as long as the file descriptor is released again. */
 735         if (server.appendfd == -1) {
 736             /* AOF disabled */
 737
 738              /* Don't care if this fails: oldfd will be -1 and we handle that.
 739               * One notable case of -1 return is if the old file does
 740               * not exist. */
 741              oldfd = open(server.appendfilename,O_RDONLY|O_NONBLOCK);
 742         } else {
 743             /* AOF enabled */
 744             oldfd = -1; /* We'll set this to the current AOF filedes later. */
 745         }
 746
 747         /* Rename the temporary file. This will not unlink the target file if
 748          * it exists, because we reference it with "oldfd". */
 749         if (rename(tmpfile,server.appendfilename) == -1) {
 750             redisLog(REDIS_WARNING,
 751                 "Error trying to rename the temporary AOF: %s", strerror(errno));
 752             close(newfd);
 753             if (oldfd != -1) close(oldfd);
 754             goto cleanup;
 755         }
 756
 757         if (server.appendfd == -1) {
 758             /* AOF disabled, we don't need to set the AOF file descriptor
 759              * to this new file, so we can close it. */
 760             close(newfd);
 761         } else {
 762             /* AOF enabled, replace the old fd with the new one. */
 763             oldfd = server.appendfd;
 764             server.appendfd = newfd;
 765             if (server.appendfsync != APPENDFSYNC_NO) aof_fsync(newfd);
 766             server.appendseldb = -1; /* Make sure SELECT is re-issued */
 767             aofUpdateCurrentSize();
 768             server.auto_aofrewrite_base_size = server.appendonly_current_size;
 769         }
 770
 771         redisLog(REDIS_NOTICE, "Background AOF rewrite successful");
 772
 773         /* Asynchronously close the overwritten AOF. */
 774         if (oldfd != -1) bioCreateBackgroundJob(REDIS_BIO_CLOSE_FILE,(void*)(long)oldfd,NULL,NULL);
 775
 776         redisLog(REDIS_VERBOSE,
 777             "Background AOF rewrite signal handler took %lldus", ustime()-now);
 778     } else if (!bysignal && exitcode != 0) {
 779         redisLog(REDIS_WARNING,
 780             "Background AOF rewrite terminated with error");
 781     } else {
 782         redisLog(REDIS_WARNING,
 783             "Background AOF rewrite terminated by signal %d", bysignal);
 784     }
 785
 786 cleanup:
 787     sdsfree(server.bgrewritebuf);
 788     server.bgrewritebuf = sdsempty();
 789     aofRemoveTempFile(server.bgrewritechildpid);
 790     server.bgrewritechildpid = -1;
 791 }