Two fixes for replication: Slave performs the AOF rewrite at the right point. Non...

[redis.git] / src / aof.c

diff --git a/src/aof.c b/src/aof.c
index 51054b296c2359701e33c0610bd9a715346be91b..cd409a0b376d85d2df9bdc6e583abd8e3b6aacb9 100644 (file)
--- a/src/aof.c
+++ b/src/aof.c
@@ -3,6 +3,10 @@
 #include <signal.h>
 #include <fcntl.h>
 #include <sys/stat.h>
 #include <signal.h>
 #include <fcntl.h>
 #include <sys/stat.h>

+#include <sys/types.h>
+#include <sys/time.h>
+#include <sys/resource.h>
+#include <sys/wait.h>

 
 /* Called when the user switches from "appendonly yes" to "appendonly no"
  * at runtime using the CONFIG command. */
 
 /* Called when the user switches from "appendonly yes" to "appendonly no"
  * at runtime using the CONFIG command. */


@@ -185,11 +189,13 @@ struct redisClient *createFakeClient(void) {
     c->querybuf = sdsempty();
     c->argc = 0;
     c->argv = NULL;
     c->querybuf = sdsempty();
     c->argc = 0;
     c->argv = NULL;

+    c->bufpos = 0;

     c->flags = 0;
     /* We set the fake client as a slave waiting for the synchronization
      * so that Redis will not try to send replies to this client. */
     c->replstate = REDIS_REPL_WAIT_BGSAVE_START;
     c->reply = listCreate();
     c->flags = 0;
     /* We set the fake client as a slave waiting for the synchronization
      * so that Redis will not try to send replies to this client. */
     c->replstate = REDIS_REPL_WAIT_BGSAVE_START;
     c->reply = listCreate();

+    c->watched_keys = listCreate();

     listSetFreeMethod(c->reply,decrRefCount);
     listSetDupMethod(c->reply,dupClientReplyValue);
     initClientMultiState(c);
     listSetFreeMethod(c->reply,decrRefCount);
     listSetDupMethod(c->reply,dupClientReplyValue);
     initClientMultiState(c);


@@ -199,6 +205,7 @@ struct redisClient *createFakeClient(void) {
 void freeFakeClient(struct redisClient *c) {
     sdsfree(c->querybuf);
     listRelease(c->reply);
 void freeFakeClient(struct redisClient *c) {
     sdsfree(c->querybuf);
     listRelease(c->reply);

+    listRelease(c->watched_keys);

     freeClientMultiState(c);
     zfree(c);
 }
     freeClientMultiState(c);
     zfree(c);
 }


@@ -211,9 +218,12 @@ int loadAppendOnlyFile(char *filename) {
     FILE *fp = fopen(filename,"r");
     struct redis_stat sb;
     int appendonly = server.appendonly;
     FILE *fp = fopen(filename,"r");
     struct redis_stat sb;
     int appendonly = server.appendonly;

+    long loops = 0;

 
 

-    if (redis_fstat(fileno(fp),&sb) != -1 && sb.st_size == 0)
+    if (fp && redis_fstat(fileno(fp),&sb) != -1 && sb.st_size == 0) {
+        fclose(fp);

         return REDIS_ERR;
         return REDIS_ERR;

+    }

 
     if (fp == NULL) {
         redisLog(REDIS_WARNING,"Fatal error: can't open the append log file for reading: %s",strerror(errno));
 
     if (fp == NULL) {
         redisLog(REDIS_WARNING,"Fatal error: can't open the append log file for reading: %s",strerror(errno));


@@ -225,6 +235,8 @@ int loadAppendOnlyFile(char *filename) {
     server.appendonly = 0;
 
     fakeClient = createFakeClient();
     server.appendonly = 0;
 
     fakeClient = createFakeClient();

+    startLoading(fp);
+

     while(1) {
         int argc, j;
         unsigned long len;
     while(1) {
         int argc, j;
         unsigned long len;


@@ -232,7 +244,12 @@ int loadAppendOnlyFile(char *filename) {
         char buf[128];
         sds argsds;
         struct redisCommand *cmd;
         char buf[128];
         sds argsds;
         struct redisCommand *cmd;

-        int force_swapout;
+
+        /* Serve the clients from time to time */
+        if (!(loops++ % 1000)) {
+            loadingProgress(ftello(fp));
+            aeProcessEvents(server.el, AE_FILE_EVENTS|AE_DONT_WAIT);
+        }

 
         if (fgets(buf,sizeof(buf),fp) == NULL) {
             if (feof(fp))
 
         if (fgets(buf,sizeof(buf),fp) == NULL) {
             if (feof(fp))


@@ -259,29 +276,19 @@ int loadAppendOnlyFile(char *filename) {
             redisLog(REDIS_WARNING,"Unknown command '%s' reading the append only file", argv[0]->ptr);
             exit(1);
         }
             redisLog(REDIS_WARNING,"Unknown command '%s' reading the append only file", argv[0]->ptr);
             exit(1);
         }

-        /* Try object encoding */
-        if (cmd->flags & REDIS_CMD_BULK)
-            argv[argc-1] = tryObjectEncoding(argv[argc-1]);

         /* Run the command in the context of a fake client */
         fakeClient->argc = argc;
         fakeClient->argv = argv;
         cmd->proc(fakeClient);
         /* Run the command in the context of a fake client */
         fakeClient->argc = argc;
         fakeClient->argv = argv;
         cmd->proc(fakeClient);

-        /* Discard the reply objects list from the fake client */
-        while(listLength(fakeClient->reply))
-            listDelNode(fakeClient->reply,listFirst(fakeClient->reply));
-        /* Clean up, ready for the next command */
-        for (j = 0; j < argc; j++) decrRefCount(argv[j]);
-        zfree(argv);
-        /* Handle swapping while loading big datasets when VM is on */
-        force_swapout = 0;
-        if ((zmalloc_used_memory() - server.vm_max_memory) > 1024*1024*32)
-            force_swapout = 1;
-
-        if (server.vm_enabled && force_swapout) {
-            while (zmalloc_used_memory() > server.vm_max_memory) {
-                if (vmSwapOneObjectBlocking() == REDIS_ERR) break;
-            }
-        }
+
+        /* The fake client should not have a reply */
+        redisAssert(fakeClient->bufpos == 0 && listLength(fakeClient->reply) == 0);
+
+        /* Clean up. Command code may have changed argv/argc so we use the
+         * argv/argc of the client instead of the local variables. */
+        for (j = 0; j < fakeClient->argc; j++)
+            decrRefCount(fakeClient->argv[j]);
+        zfree(fakeClient->argv);

     }
 
     /* This point can only be reached when EOF is reached without errors.
     }
 
     /* This point can only be reached when EOF is reached without errors.


@@ -291,6 +298,7 @@ int loadAppendOnlyFile(char *filename) {
     fclose(fp);
     freeFakeClient(fakeClient);
     server.appendonly = appendonly;
     fclose(fp);
     freeFakeClient(fakeClient);
     server.appendonly = appendonly;

+    stopLoading();

     return REDIS_OK;
 
 readerr:
     return REDIS_OK;
 
 readerr:


@@ -301,59 +309,10 @@ readerr:
     }
     exit(1);
 fmterr:
     }
     exit(1);
 fmterr:

-    redisLog(REDIS_WARNING,"Bad file format reading the append only file");
+    redisLog(REDIS_WARNING,"Bad file format reading the append only file: make a backup of your AOF file, then use ./redis-check-aof --fix <filename>");

     exit(1);
 }
 
     exit(1);
 }
 

-/* Write binary-safe string into a file in the bulkformat
- * $<count>\r\n<payload>\r\n */
-int fwriteBulkString(FILE *fp, char *s, unsigned long len) {
-    char cbuf[128];
-    int clen;
-    cbuf[0] = '$';
-    clen = 1+ll2string(cbuf+1,sizeof(cbuf)-1,len);
-    cbuf[clen++] = '\r';
-    cbuf[clen++] = '\n';
-    if (fwrite(cbuf,clen,1,fp) == 0) return 0;
-    if (len > 0 && fwrite(s,len,1,fp) == 0) return 0;
-    if (fwrite("\r\n",2,1,fp) == 0) return 0;
-    return 1;
-}
-
-/* Write a double value in bulk format $<count>\r\n<payload>\r\n */
-int fwriteBulkDouble(FILE *fp, double d) {
-    char buf[128], dbuf[128];
-
-    snprintf(dbuf,sizeof(dbuf),"%.17g\r\n",d);
-    snprintf(buf,sizeof(buf),"$%lu\r\n",(unsigned long)strlen(dbuf)-2);
-    if (fwrite(buf,strlen(buf),1,fp) == 0) return 0;
-    if (fwrite(dbuf,strlen(dbuf),1,fp) == 0) return 0;
-    return 1;
-}
-
-/* Write a long value in bulk format $<count>\r\n<payload>\r\n */
-int fwriteBulkLongLong(FILE *fp, long long l) {
-    char bbuf[128], lbuf[128];
-    unsigned int blen, llen;
-    llen = ll2string(lbuf,32,l);
-    blen = snprintf(bbuf,sizeof(bbuf),"$%u\r\n%s\r\n",llen,lbuf);
-    if (fwrite(bbuf,blen,1,fp) == 0) return 0;
-    return 1;
-}
-
-/* Delegate writing an object to writing a bulk string or bulk long long. */
-int fwriteBulkObject(FILE *fp, robj *obj) {
-    /* Avoid using getDecodedObject to help copy-on-write (we are often
-     * in a child process when this function is called). */
-    if (obj->encoding == REDIS_ENCODING_INT) {
-        return fwriteBulkLongLong(fp,(long)obj->ptr);
-    } else if (obj->encoding == REDIS_ENCODING_RAW) {
-        return fwriteBulkString(fp,obj->ptr,sdslen(obj->ptr));
-    } else {
-        redisPanic("Unknown string encoding");
-    }
-}
-

 /* Write a sequence of commands able to fully rebuild the dataset into
  * "filename". Used both by REWRITEAOF and BGREWRITEAOF. */
 int rewriteAppendOnlyFile(char *filename) {
 /* Write a sequence of commands able to fully rebuild the dataset into
  * "filename". Used both by REWRITEAOF and BGREWRITEAOF. */
 int rewriteAppendOnlyFile(char *filename) {


@@ -389,25 +348,14 @@ int rewriteAppendOnlyFile(char *filename) {
 
         /* Iterate this DB writing every entry */
         while((de = dictNext(di)) != NULL) {
 
         /* Iterate this DB writing every entry */
         while((de = dictNext(di)) != NULL) {

-            sds keystr = dictGetEntryKey(de);
+            sds keystr;

             robj key, *o;
             time_t expiretime;
             robj key, *o;
             time_t expiretime;

-            int swapped;

 
             keystr = dictGetEntryKey(de);
             o = dictGetEntryVal(de);
             initStaticStringObject(key,keystr);
 
             keystr = dictGetEntryKey(de);
             o = dictGetEntryVal(de);
             initStaticStringObject(key,keystr);

-            /* If the value for this key is swapped, load a preview in memory.
-             * We use a "swapped" flag to remember if we need to free the
-             * value object instead to just increment the ref count anyway
-             * in order to avoid copy-on-write of pages if we are forked() */
-            if (!server.vm_enabled || o->storage == REDIS_VM_MEMORY ||
-                o->storage == REDIS_VM_SWAPPING) {
-                swapped = 0;
-            } else {
-                o = vmPreviewObject(o);
-                swapped = 1;
-            }
+

             expiretime = getExpire(db,&key);
 
             /* Save the key and associated value */
             expiretime = getExpire(db,&key);
 
             /* Save the key and associated value */


@@ -457,37 +405,81 @@ int rewriteAppendOnlyFile(char *filename) {
                     redisPanic("Unknown list encoding");
                 }
             } else if (o->type == REDIS_SET) {
                     redisPanic("Unknown list encoding");
                 }
             } else if (o->type == REDIS_SET) {

-                /* Emit the SADDs needed to rebuild the set */
-                dict *set = o->ptr;
-                dictIterator *di = dictGetIterator(set);
-                dictEntry *de;
-
-                while((de = dictNext(di)) != NULL) {
-                    char cmd[]="*3\r\n$4\r\nSADD\r\n";
-                    robj *eleobj = dictGetEntryKey(de);
+                char cmd[]="*3\r\n$4\r\nSADD\r\n";

 
 

-                    if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
-                    if (fwriteBulkObject(fp,&key) == 0) goto werr;
-                    if (fwriteBulkObject(fp,eleobj) == 0) goto werr;
+                /* Emit the SADDs needed to rebuild the set */
+                if (o->encoding == REDIS_ENCODING_INTSET) {
+                    int ii = 0;
+                    int64_t llval;
+                    while(intsetGet(o->ptr,ii++,&llval)) {
+                        if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
+                        if (fwriteBulkObject(fp,&key) == 0) goto werr;
+                        if (fwriteBulkLongLong(fp,llval) == 0) goto werr;
+                    }
+                } else if (o->encoding == REDIS_ENCODING_HT) {
+                    dictIterator *di = dictGetIterator(o->ptr);
+                    dictEntry *de;
+                    while((de = dictNext(di)) != NULL) {
+                        robj *eleobj = dictGetEntryKey(de);
+                        if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
+                        if (fwriteBulkObject(fp,&key) == 0) goto werr;
+                        if (fwriteBulkObject(fp,eleobj) == 0) goto werr;
+                    }
+                    dictReleaseIterator(di);
+                } else {
+                    redisPanic("Unknown set encoding");

                 }
                 }

-                dictReleaseIterator(di);

             } else if (o->type == REDIS_ZSET) {
                 /* Emit the ZADDs needed to rebuild the sorted set */
             } else if (o->type == REDIS_ZSET) {
                 /* Emit the ZADDs needed to rebuild the sorted set */

-                zset *zs = o->ptr;
-                dictIterator *di = dictGetIterator(zs->dict);
-                dictEntry *de;
-
-                while((de = dictNext(di)) != NULL) {
-                    char cmd[]="*4\r\n$4\r\nZADD\r\n";
-                    robj *eleobj = dictGetEntryKey(de);
-                    double *score = dictGetEntryVal(de);
-
-                    if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
-                    if (fwriteBulkObject(fp,&key) == 0) goto werr;
-                    if (fwriteBulkDouble(fp,*score) == 0) goto werr;
-                    if (fwriteBulkObject(fp,eleobj) == 0) goto werr;
+                char cmd[]="*4\r\n$4\r\nZADD\r\n";
+
+                if (o->encoding == REDIS_ENCODING_ZIPLIST) {
+                    unsigned char *zl = o->ptr;
+                    unsigned char *eptr, *sptr;
+                    unsigned char *vstr;
+                    unsigned int vlen;
+                    long long vll;
+                    double score;
+
+                    eptr = ziplistIndex(zl,0);
+                    redisAssert(eptr != NULL);
+                    sptr = ziplistNext(zl,eptr);
+                    redisAssert(sptr != NULL);
+
+                    while (eptr != NULL) {
+                        redisAssert(ziplistGet(eptr,&vstr,&vlen,&vll));
+                        score = zzlGetScore(sptr);
+
+                        if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
+                        if (fwriteBulkObject(fp,&key) == 0) goto werr;
+                        if (fwriteBulkDouble(fp,score) == 0) goto werr;
+                        if (vstr != NULL) {
+                            if (fwriteBulkString(fp,(char*)vstr,vlen) == 0)
+                                goto werr;
+                        } else {
+                            if (fwriteBulkLongLong(fp,vll) == 0)
+                                goto werr;
+                        }
+                        zzlNext(zl,&eptr,&sptr);
+                    }
+                } else if (o->encoding == REDIS_ENCODING_SKIPLIST) {
+                    zset *zs = o->ptr;
+                    dictIterator *di = dictGetIterator(zs->dict);
+                    dictEntry *de;
+
+                    while((de = dictNext(di)) != NULL) {
+                        robj *eleobj = dictGetEntryKey(de);
+                        double *score = dictGetEntryVal(de);
+
+                        if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
+                        if (fwriteBulkObject(fp,&key) == 0) goto werr;
+                        if (fwriteBulkDouble(fp,*score) == 0) goto werr;
+                        if (fwriteBulkObject(fp,eleobj) == 0) goto werr;
+                    }
+                    dictReleaseIterator(di);
+                } else {
+                    redisPanic("Unknown sorted set encoding");

                 }
                 }

-                dictReleaseIterator(di);

             } else if (o->type == REDIS_HASH) {
                 char cmd[]="*4\r\n$4\r\nHSET\r\n";
 
             } else if (o->type == REDIS_HASH) {
                 char cmd[]="*4\r\n$4\r\nHSET\r\n";
 


@@ -500,10 +492,10 @@ int rewriteAppendOnlyFile(char *filename) {
                     while((p = zipmapNext(p,&field,&flen,&val,&vlen)) != NULL) {
                         if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
                         if (fwriteBulkObject(fp,&key) == 0) goto werr;
                     while((p = zipmapNext(p,&field,&flen,&val,&vlen)) != NULL) {
                         if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
                         if (fwriteBulkObject(fp,&key) == 0) goto werr;

-                        if (fwriteBulkString(fp,(char*)field,flen) == -1)
-                            return -1;
-                        if (fwriteBulkString(fp,(char*)val,vlen) == -1)
-                            return -1;
+                        if (fwriteBulkString(fp,(char*)field,flen) == 0)
+                            goto werr;
+                        if (fwriteBulkString(fp,(char*)val,vlen) == 0)
+                            goto werr;

                     }
                 } else {
                     dictIterator *di = dictGetIterator(o->ptr);
                     }
                 } else {
                     dictIterator *di = dictGetIterator(o->ptr);


@@ -515,8 +507,8 @@ int rewriteAppendOnlyFile(char *filename) {
 
                         if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
                         if (fwriteBulkObject(fp,&key) == 0) goto werr;
 
                         if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
                         if (fwriteBulkObject(fp,&key) == 0) goto werr;

-                        if (fwriteBulkObject(fp,field) == -1) return -1;
-                        if (fwriteBulkObject(fp,val) == -1) return -1;
+                        if (fwriteBulkObject(fp,field) == 0) goto werr;
+                        if (fwriteBulkObject(fp,val) == 0) goto werr;

                     }
                     dictReleaseIterator(di);
                 }
                     }
                     dictReleaseIterator(di);
                 }


@@ -532,7 +524,6 @@ int rewriteAppendOnlyFile(char *filename) {
                 if (fwriteBulkObject(fp,&key) == 0) goto werr;
                 if (fwriteBulkLongLong(fp,expiretime) == 0) goto werr;
             }
                 if (fwriteBulkObject(fp,&key) == 0) goto werr;
                 if (fwriteBulkLongLong(fp,expiretime) == 0) goto werr;
             }

-            if (swapped) decrRefCount(o);

         }
         dictReleaseIterator(di);
     }
         }
         dictReleaseIterator(di);
     }


@@ -574,15 +565,20 @@ werr:
  */
 int rewriteAppendOnlyFileBackground(void) {
     pid_t childpid;
  */
 int rewriteAppendOnlyFileBackground(void) {
     pid_t childpid;

+    long long start;

 
     if (server.bgrewritechildpid != -1) return REDIS_ERR;
 
     if (server.bgrewritechildpid != -1) return REDIS_ERR;

-    if (server.vm_enabled) waitEmptyIOJobsQueue();
+    if (server.ds_enabled != 0) {
+        redisLog(REDIS_WARNING,"BGREWRITEAOF called with diskstore enabled: AOF is not supported when diskstore is enabled. Operation not performed.");
+        return REDIS_ERR;
+    }
+    start = ustime();

     if ((childpid = fork()) == 0) {
     if ((childpid = fork()) == 0) {

-        /* Child */

         char tmpfile[256];
 
         char tmpfile[256];
 

-        if (server.vm_enabled) vmReopenSwapFile();
-        close(server.fd);
+        /* Child */
+        if (server.ipfd > 0) close(server.ipfd);
+        if (server.sofd > 0) close(server.sofd);

         snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) getpid());
         if (rewriteAppendOnlyFile(tmpfile) == REDIS_OK) {
             _exit(0);
         snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) getpid());
         if (rewriteAppendOnlyFile(tmpfile) == REDIS_OK) {
             _exit(0);


@@ -591,6 +587,7 @@ int rewriteAppendOnlyFileBackground(void) {
         }
     } else {
         /* Parent */
         }
     } else {
         /* Parent */

+        server.stat_fork_time = ustime()-start;

         if (childpid == -1) {
             redisLog(REDIS_WARNING,
                 "Can't rewrite append only file in background: fork: %s",
         if (childpid == -1) {
             redisLog(REDIS_WARNING,
                 "Can't rewrite append only file in background: fork: %s",


@@ -613,12 +610,11 @@ int rewriteAppendOnlyFileBackground(void) {
 
 void bgrewriteaofCommand(redisClient *c) {
     if (server.bgrewritechildpid != -1) {
 
 void bgrewriteaofCommand(redisClient *c) {
     if (server.bgrewritechildpid != -1) {

-        addReplySds(c,sdsnew("-ERR background append only file rewriting already in progress\r\n"));
+        addReplyError(c,"Background append only file rewriting already in progress");

         return;
     }
     if (rewriteAppendOnlyFileBackground() == REDIS_OK) {
         return;
     }
     if (rewriteAppendOnlyFileBackground() == REDIS_OK) {

-        char *status = "+Background append only file rewriting started\r\n";
-        addReplySds(c,sdsnew(status));
+        addReplyStatus(c,"Background append only file rewriting started");

     } else {
         addReply(c,shared.err);
     }
     } else {
         addReply(c,shared.err);
     }


@@ -633,10 +629,7 @@ void aofRemoveTempFile(pid_t childpid) {
 
 /* A background append only file rewriting (BGREWRITEAOF) terminated its work.
  * Handle this. */
 
 /* A background append only file rewriting (BGREWRITEAOF) terminated its work.
  * Handle this. */

-void backgroundRewriteDoneHandler(int statloc) {
-    int exitcode = WEXITSTATUS(statloc);
-    int bysignal = WIFSIGNALED(statloc);
-
+void backgroundRewriteDoneHandler(int exitcode, int bysignal) {

     if (!bysignal && exitcode == 0) {
         int fd;
         char tmpfile[256];
     if (!bysignal && exitcode == 0) {
         int fd;
         char tmpfile[256];


@@ -684,7 +677,7 @@ void backgroundRewriteDoneHandler(int statloc) {
     } else {
         redisLog(REDIS_WARNING,
             "Background append only file rewriting terminated by signal %d",
     } else {
         redisLog(REDIS_WARNING,
             "Background append only file rewriting terminated by signal %d",

-            WTERMSIG(statloc));
+            bysignal);

     }
 cleanup:
     sdsfree(server.bgrewritebuf);
     }
 cleanup:
     sdsfree(server.bgrewritebuf);