diskstore removed

[redis.git] / src / aof.c

diff --git a/src/aof.c b/src/aof.c
index f5d04a62b4d493d1cbb82b5b7f2b98a55adf0a10..ac6b97915e26b5f61ef85b10afee377ab9f4c31d 100644 (file)
--- a/src/aof.c
+++ b/src/aof.c
@@ -8,6 +8,8 @@
 #include <sys/resource.h>
 #include <sys/wait.h>
 
 #include <sys/resource.h>
 #include <sys/wait.h>
 

+void aofUpdateCurrentSize(void);
+

 /* Called when the user switches from "appendonly yes" to "appendonly no"
  * at runtime using the CONFIG command. */
 void stopAppendOnly(void) {
 /* Called when the user switches from "appendonly yes" to "appendonly no"
  * at runtime using the CONFIG command. */
 void stopAppendOnly(void) {


@@ -19,15 +21,15 @@ void stopAppendOnly(void) {
     server.appendseldb = -1;
     server.appendonly = 0;
     /* rewrite operation in progress? kill it, wait child exit */
     server.appendseldb = -1;
     server.appendonly = 0;
     /* rewrite operation in progress? kill it, wait child exit */

-    if (server.bgsavechildpid != -1) {
+    if (server.bgrewritechildpid != -1) {

         int statloc;
 
         int statloc;
 

-        if (kill(server.bgsavechildpid,SIGKILL) != -1)
+        if (kill(server.bgrewritechildpid,SIGKILL) != -1)

             wait3(&statloc,0,NULL);
         /* reset the buffer accumulating changes while the child saves */
         sdsfree(server.bgrewritebuf);
         server.bgrewritebuf = sdsempty();
             wait3(&statloc,0,NULL);
         /* reset the buffer accumulating changes while the child saves */
         sdsfree(server.bgrewritebuf);
         server.bgrewritebuf = sdsempty();

-        server.bgsavechildpid = -1;
+        server.bgrewritechildpid = -1;

     }
 }
 
     }
 }
 


@@ -82,6 +84,7 @@ void flushAppendOnlyFile(void) {
     }
     sdsfree(server.aofbuf);
     server.aofbuf = sdsempty();
     }
     sdsfree(server.aofbuf);
     server.aofbuf = sdsempty();

+    server.appendonly_current_size += nwritten;

 
     /* Don't Fsync if no-appendfsync-on-rewrite is set to yes and we have
      * childs performing heavy I/O on disk. */
 
     /* Don't Fsync if no-appendfsync-on-rewrite is set to yes and we have
      * childs performing heavy I/O on disk. */


@@ -220,8 +223,11 @@ int loadAppendOnlyFile(char *filename) {
     int appendonly = server.appendonly;
     long loops = 0;
 
     int appendonly = server.appendonly;
     long loops = 0;
 

-    if (redis_fstat(fileno(fp),&sb) != -1 && sb.st_size == 0)
+    if (fp && redis_fstat(fileno(fp),&sb) != -1 && sb.st_size == 0) {
+        server.appendonly_current_size = 0;
+        fclose(fp);

         return REDIS_ERR;
         return REDIS_ERR;

+    }

 
     if (fp == NULL) {
         redisLog(REDIS_WARNING,"Fatal error: can't open the append log file for reading: %s",strerror(errno));
 
     if (fp == NULL) {
         redisLog(REDIS_WARNING,"Fatal error: can't open the append log file for reading: %s",strerror(errno));


@@ -282,9 +288,11 @@ int loadAppendOnlyFile(char *filename) {
         /* The fake client should not have a reply */
         redisAssert(fakeClient->bufpos == 0 && listLength(fakeClient->reply) == 0);
 
         /* The fake client should not have a reply */
         redisAssert(fakeClient->bufpos == 0 && listLength(fakeClient->reply) == 0);
 

-        /* Clean up, ready for the next command */
-        for (j = 0; j < argc; j++) decrRefCount(argv[j]);
-        zfree(argv);
+        /* Clean up. Command code may have changed argv/argc so we use the
+         * argv/argc of the client instead of the local variables. */
+        for (j = 0; j < fakeClient->argc; j++)
+            decrRefCount(fakeClient->argv[j]);
+        zfree(fakeClient->argv);

     }
 
     /* This point can only be reached when EOF is reached without errors.
     }
 
     /* This point can only be reached when EOF is reached without errors.


@@ -295,6 +303,8 @@ int loadAppendOnlyFile(char *filename) {
     freeFakeClient(fakeClient);
     server.appendonly = appendonly;
     stopLoading();
     freeFakeClient(fakeClient);
     server.appendonly = appendonly;
     stopLoading();

+    aofUpdateCurrentSize();
+    server.auto_aofrewrite_base_size = server.appendonly_current_size;

     return REDIS_OK;
 
 readerr:
     return REDIS_OK;
 
 readerr:


@@ -332,7 +342,7 @@ int rewriteAppendOnlyFile(char *filename) {
         redisDb *db = server.db+j;
         dict *d = db->dict;
         if (dictSize(d) == 0) continue;
         redisDb *db = server.db+j;
         dict *d = db->dict;
         if (dictSize(d) == 0) continue;

-        di = dictGetIterator(d);
+        di = dictGetSafeIterator(d);

         if (!di) {
             fclose(fp);
             return REDIS_ERR;
         if (!di) {
             fclose(fp);
             return REDIS_ERR;


@@ -344,7 +354,7 @@ int rewriteAppendOnlyFile(char *filename) {
 
         /* Iterate this DB writing every entry */
         while((de = dictNext(di)) != NULL) {
 
         /* Iterate this DB writing every entry */
         while((de = dictNext(di)) != NULL) {

-            sds keystr = dictGetEntryKey(de);
+            sds keystr;

             robj key, *o;
             time_t expiretime;
 
             robj key, *o;
             time_t expiretime;
 


@@ -427,21 +437,55 @@ int rewriteAppendOnlyFile(char *filename) {
                 }
             } else if (o->type == REDIS_ZSET) {
                 /* Emit the ZADDs needed to rebuild the sorted set */
                 }
             } else if (o->type == REDIS_ZSET) {
                 /* Emit the ZADDs needed to rebuild the sorted set */

-                zset *zs = o->ptr;
-                dictIterator *di = dictGetIterator(zs->dict);
-                dictEntry *de;
-
-                while((de = dictNext(di)) != NULL) {
-                    char cmd[]="*4\r\n$4\r\nZADD\r\n";
-                    robj *eleobj = dictGetEntryKey(de);
-                    double *score = dictGetEntryVal(de);
-
-                    if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
-                    if (fwriteBulkObject(fp,&key) == 0) goto werr;
-                    if (fwriteBulkDouble(fp,*score) == 0) goto werr;
-                    if (fwriteBulkObject(fp,eleobj) == 0) goto werr;
+                char cmd[]="*4\r\n$4\r\nZADD\r\n";
+
+                if (o->encoding == REDIS_ENCODING_ZIPLIST) {
+                    unsigned char *zl = o->ptr;
+                    unsigned char *eptr, *sptr;
+                    unsigned char *vstr;
+                    unsigned int vlen;
+                    long long vll;
+                    double score;
+
+                    eptr = ziplistIndex(zl,0);
+                    redisAssert(eptr != NULL);
+                    sptr = ziplistNext(zl,eptr);
+                    redisAssert(sptr != NULL);
+
+                    while (eptr != NULL) {
+                        redisAssert(ziplistGet(eptr,&vstr,&vlen,&vll));
+                        score = zzlGetScore(sptr);
+
+                        if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
+                        if (fwriteBulkObject(fp,&key) == 0) goto werr;
+                        if (fwriteBulkDouble(fp,score) == 0) goto werr;
+                        if (vstr != NULL) {
+                            if (fwriteBulkString(fp,(char*)vstr,vlen) == 0)
+                                goto werr;
+                        } else {
+                            if (fwriteBulkLongLong(fp,vll) == 0)
+                                goto werr;
+                        }
+                        zzlNext(zl,&eptr,&sptr);
+                    }
+                } else if (o->encoding == REDIS_ENCODING_SKIPLIST) {
+                    zset *zs = o->ptr;
+                    dictIterator *di = dictGetIterator(zs->dict);
+                    dictEntry *de;
+
+                    while((de = dictNext(di)) != NULL) {
+                        robj *eleobj = dictGetEntryKey(de);
+                        double *score = dictGetEntryVal(de);
+
+                        if (fwrite(cmd,sizeof(cmd)-1,1,fp) == 0) goto werr;
+                        if (fwriteBulkObject(fp,&key) == 0) goto werr;
+                        if (fwriteBulkDouble(fp,*score) == 0) goto werr;
+                        if (fwriteBulkObject(fp,eleobj) == 0) goto werr;
+                    }
+                    dictReleaseIterator(di);
+                } else {
+                    redisPanic("Unknown sorted set encoding");

                 }
                 }

-                dictReleaseIterator(di);

             } else if (o->type == REDIS_HASH) {
                 char cmd[]="*4\r\n$4\r\nHSET\r\n";
 
             } else if (o->type == REDIS_HASH) {
                 char cmd[]="*4\r\n$4\r\nHSET\r\n";
 


@@ -527,13 +571,14 @@ werr:
  */
 int rewriteAppendOnlyFileBackground(void) {
     pid_t childpid;
  */
 int rewriteAppendOnlyFileBackground(void) {
     pid_t childpid;

+    long long start;

 
     if (server.bgrewritechildpid != -1) return REDIS_ERR;
 
     if (server.bgrewritechildpid != -1) return REDIS_ERR;

-    redisAssert(server.ds_enabled == 0);
+    start = ustime();

     if ((childpid = fork()) == 0) {
     if ((childpid = fork()) == 0) {

-        /* Child */

         char tmpfile[256];
 
         char tmpfile[256];
 

+        /* Child */

         if (server.ipfd > 0) close(server.ipfd);
         if (server.sofd > 0) close(server.sofd);
         snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) getpid());
         if (server.ipfd > 0) close(server.ipfd);
         if (server.sofd > 0) close(server.sofd);
         snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) getpid());


@@ -544,6 +589,7 @@ int rewriteAppendOnlyFileBackground(void) {
         }
     } else {
         /* Parent */
         }
     } else {
         /* Parent */

+        server.stat_fork_time = ustime()-start;

         if (childpid == -1) {
             redisLog(REDIS_WARNING,
                 "Can't rewrite append only file in background: fork: %s",
         if (childpid == -1) {
             redisLog(REDIS_WARNING,
                 "Can't rewrite append only file in background: fork: %s",


@@ -567,9 +613,10 @@ int rewriteAppendOnlyFileBackground(void) {
 void bgrewriteaofCommand(redisClient *c) {
     if (server.bgrewritechildpid != -1) {
         addReplyError(c,"Background append only file rewriting already in progress");
 void bgrewriteaofCommand(redisClient *c) {
     if (server.bgrewritechildpid != -1) {
         addReplyError(c,"Background append only file rewriting already in progress");

-        return;
-    }
-    if (rewriteAppendOnlyFileBackground() == REDIS_OK) {
+    } else if (server.bgsavechildpid != -1) {
+        server.aofrewrite_scheduled = 1;
+        addReplyStatus(c,"Background append only file rewriting scheduled");
+    } else if (rewriteAppendOnlyFileBackground() == REDIS_OK) {

         addReplyStatus(c,"Background append only file rewriting started");
     } else {
         addReply(c,shared.err);
         addReplyStatus(c,"Background append only file rewriting started");
     } else {
         addReply(c,shared.err);


@@ -583,12 +630,24 @@ void aofRemoveTempFile(pid_t childpid) {
     unlink(tmpfile);
 }
 
     unlink(tmpfile);
 }
 

+/* Update the server.appendonly_current_size filed explicitly using stat(2)
+ * to check the size of the file. This is useful after a rewrite or after
+ * a restart, normally the size is updated just adding the write length
+ * to the current lenght, that is much faster. */
+void aofUpdateCurrentSize(void) {
+    struct redis_stat sb;
+
+    if (redis_fstat(server.appendfd,&sb) == -1) {
+        redisLog(REDIS_WARNING,"Unable to check the AOF length: %s",
+            strerror(errno));
+    } else {
+        server.appendonly_current_size = sb.st_size;
+    }
+}
+

 /* A background append only file rewriting (BGREWRITEAOF) terminated its work.
  * Handle this. */
 /* A background append only file rewriting (BGREWRITEAOF) terminated its work.
  * Handle this. */

-void backgroundRewriteDoneHandler(int statloc) {
-    int exitcode = WEXITSTATUS(statloc);
-    int bysignal = WIFSIGNALED(statloc);
-
+void backgroundRewriteDoneHandler(int exitcode, int bysignal) {

     if (!bysignal && exitcode == 0) {
         int fd;
         char tmpfile[256];
     if (!bysignal && exitcode == 0) {
         int fd;
         char tmpfile[256];


@@ -626,6 +685,8 @@ void backgroundRewriteDoneHandler(int statloc) {
             if (server.appendfsync != APPENDFSYNC_NO) aof_fsync(fd);
             server.appendseldb = -1; /* Make sure it will issue SELECT */
             redisLog(REDIS_NOTICE,"The new append only file was selected for future appends.");
             if (server.appendfsync != APPENDFSYNC_NO) aof_fsync(fd);
             server.appendseldb = -1; /* Make sure it will issue SELECT */
             redisLog(REDIS_NOTICE,"The new append only file was selected for future appends.");

+            aofUpdateCurrentSize();
+            server.auto_aofrewrite_base_size = server.appendonly_current_size;

         } else {
             /* If append only is disabled we just generate a dump in this
              * format. Why not? */
         } else {
             /* If append only is disabled we just generate a dump in this
              * format. Why not? */


@@ -636,7 +697,7 @@ void backgroundRewriteDoneHandler(int statloc) {
     } else {
         redisLog(REDIS_WARNING,
             "Background append only file rewriting terminated by signal %d",
     } else {
         redisLog(REDIS_WARNING,
             "Background append only file rewriting terminated by signal %d",

-            WTERMSIG(statloc));
+            bysignal);

     }
 cleanup:
     sdsfree(server.bgrewritebuf);
     }
 cleanup:
     sdsfree(server.bgrewritebuf);