X-Git-Url: https://git.saurik.com/redis.git/blobdiff_plain/b508aeb994ffebd93fa5c3238057dfa2101d0424..19951d965aa78402fa3030af05a68b81fc4e9522:/src/aof.c

diff --git a/src/aof.c b/src/aof.c
index 66516f78..bfa8163d 100644
--- a/src/aof.c
+++ b/src/aof.c
@@ -181,21 +181,38 @@ sds catAppendOnlyGenericCommand(sds dst, int argc, robj **argv) {
     return dst;
 }
 
-sds catAppendOnlyExpireAtCommand(sds buf, robj *key, robj *seconds) {
-    int argc = 3;
-    long when;
+/* Create the sds representation of an PEXPIREAT command, using
+ * 'seconds' as time to live and 'cmd' to understand what command
+ * we are translating into a PEXPIREAT.
+ *
+ * This command is used in order to translate EXPIRE and PEXPIRE commands
+ * into PEXPIREAT command so that we retain precision in the append only
+ * file, and the time is always absolute and not relative. */
+sds catAppendOnlyExpireAtCommand(sds buf, struct redisCommand *cmd, robj *key, robj *seconds) {
+    long long when;
     robj *argv[3];
 
     /* Make sure we can use strtol */
     seconds = getDecodedObject(seconds);
-    when = time(NULL)+strtol(seconds->ptr,NULL,10);
+    when = strtoll(seconds->ptr,NULL,10);
+    /* Convert argument into milliseconds for EXPIRE, SETEX, EXPIREAT */
+    if (cmd->proc == expireCommand || cmd->proc == setexCommand ||
+        cmd->proc == expireatCommand)
+    {
+        when *= 1000;
+    }
+    /* Convert into absolute time for EXPIRE, PEXPIRE, SETEX, PSETEX */
+    if (cmd->proc == expireCommand || cmd->proc == pexpireCommand ||
+        cmd->proc == setexCommand || cmd->proc == psetexCommand)
+    {
+        when += mstime();
+    }
     decrRefCount(seconds);
 
-    argv[0] = createStringObject("EXPIREAT",8);
+    argv[0] = createStringObject("PEXPIREAT",9);
     argv[1] = key;
-    argv[2] = createObject(REDIS_STRING,
-        sdscatprintf(sdsempty(),"%ld",when));
-    buf = catAppendOnlyGenericCommand(buf, argc, argv);
+    argv[2] = createStringObjectFromLongLong(when);
+    buf = catAppendOnlyGenericCommand(buf, 3, argv);
     decrRefCount(argv[0]);
     decrRefCount(argv[2]);
     return buf;
@@ -216,18 +233,22 @@ void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int a
         server.appendseldb = dictid;
     }
 
-    if (cmd->proc == expireCommand) {
-        /* Translate EXPIRE into EXPIREAT */
-        buf = catAppendOnlyExpireAtCommand(buf,argv[1],argv[2]);
-    } else if (cmd->proc == setexCommand) {
-        /* Translate SETEX to SET and EXPIREAT */
+    if (cmd->proc == expireCommand || cmd->proc == pexpireCommand ||
+        cmd->proc == expireatCommand) {
+        /* Translate EXPIRE/PEXPIRE/EXPIREAT into PEXPIREAT */
+        buf = catAppendOnlyExpireAtCommand(buf,cmd,argv[1],argv[2]);
+    } else if (cmd->proc == setexCommand || cmd->proc == psetexCommand) {
+        /* Translate SETEX/PSETEX to SET and PEXPIREAT */
         tmpargv[0] = createStringObject("SET",3);
         tmpargv[1] = argv[1];
         tmpargv[2] = argv[3];
         buf = catAppendOnlyGenericCommand(buf,3,tmpargv);
         decrRefCount(tmpargv[0]);
-        buf = catAppendOnlyExpireAtCommand(buf,argv[1],argv[2]);
+        buf = catAppendOnlyExpireAtCommand(buf,cmd,argv[1],argv[2]);
     } else {
+        /* All the other commands don't need translation or need the
+         * same translation already operated in the command vector
+         * for the replication itself. */
         buf = catAppendOnlyGenericCommand(buf,argc,argv);
     }
 
@@ -401,8 +422,236 @@ int rioWriteBulkObject(rio *r, robj *obj) {
     }
 }
 
+/* Emit the commands needed to rebuild a list object.
+ * The function returns 0 on error, 1 on success. */
+int rewriteListObject(rio *r, robj *key, robj *o) {
+    long long count = 0, items = listTypeLength(o);
+
+    if (o->encoding == REDIS_ENCODING_ZIPLIST) {
+        unsigned char *zl = o->ptr;
+        unsigned char *p = ziplistIndex(zl,0);
+        unsigned char *vstr;
+        unsigned int vlen;
+        long long vlong;
+
+        while(ziplistGet(p,&vstr,&vlen,&vlong)) {
+            if (count == 0) {
+                int cmd_items = (items > REDIS_AOFREWRITE_ITEMS_PER_CMD) ?
+                    REDIS_AOFREWRITE_ITEMS_PER_CMD : items;
+
+                if (rioWriteBulkCount(r,'*',2+cmd_items) == 0) return 0;
+                if (rioWriteBulkString(r,"RPUSH",5) == 0) return 0;
+                if (rioWriteBulkObject(r,key) == 0) return 0;
+            }
+            if (vstr) {
+                if (rioWriteBulkString(r,(char*)vstr,vlen) == 0) return 0;
+            } else {
+                if (rioWriteBulkLongLong(r,vlong) == 0) return 0;
+            }
+            p = ziplistNext(zl,p);
+            if (++count == REDIS_AOFREWRITE_ITEMS_PER_CMD) count = 0;
+            items--;
+        }
+    } else if (o->encoding == REDIS_ENCODING_LINKEDLIST) {
+        list *list = o->ptr;
+        listNode *ln;
+        listIter li;
+
+        listRewind(list,&li);
+        while((ln = listNext(&li))) {
+            robj *eleobj = listNodeValue(ln);
+
+            if (count == 0) {
+                int cmd_items = (items > REDIS_AOFREWRITE_ITEMS_PER_CMD) ?
+                    REDIS_AOFREWRITE_ITEMS_PER_CMD : items;
+
+                if (rioWriteBulkCount(r,'*',2+cmd_items) == 0) return 0;
+                if (rioWriteBulkString(r,"RPUSH",5) == 0) return 0;
+                if (rioWriteBulkObject(r,key) == 0) return 0;
+            }
+            if (rioWriteBulkObject(r,eleobj) == 0) return 0;
+            if (++count == REDIS_AOFREWRITE_ITEMS_PER_CMD) count = 0;
+            items--;
+        }
+    } else {
+        redisPanic("Unknown list encoding");
+    }
+    return 1;
+}
+
+/* Emit the commands needed to rebuild a set object.
+ * The function returns 0 on error, 1 on success. */
+int rewriteSetObject(rio *r, robj *key, robj *o) {
+    long long count = 0, items = setTypeSize(o);
+
+    if (o->encoding == REDIS_ENCODING_INTSET) {
+        int ii = 0;
+        int64_t llval;
+
+        while(intsetGet(o->ptr,ii++,&llval)) {
+            if (count == 0) {
+                int cmd_items = (items > REDIS_AOFREWRITE_ITEMS_PER_CMD) ?
+                    REDIS_AOFREWRITE_ITEMS_PER_CMD : items;
+
+                if (rioWriteBulkCount(r,'*',2+cmd_items) == 0) return 0;
+                if (rioWriteBulkString(r,"SADD",4) == 0) return 0;
+                if (rioWriteBulkObject(r,key) == 0) return 0;
+            }
+            if (rioWriteBulkLongLong(r,llval) == 0) return 0;
+            if (++count == REDIS_AOFREWRITE_ITEMS_PER_CMD) count = 0;
+            items--;
+        }
+    } else if (o->encoding == REDIS_ENCODING_HT) {
+        dictIterator *di = dictGetIterator(o->ptr);
+        dictEntry *de;
+
+        while((de = dictNext(di)) != NULL) {
+            robj *eleobj = dictGetKey(de);
+            if (count == 0) {
+                int cmd_items = (items > REDIS_AOFREWRITE_ITEMS_PER_CMD) ?
+                    REDIS_AOFREWRITE_ITEMS_PER_CMD : items;
+
+                if (rioWriteBulkCount(r,'*',2+cmd_items) == 0) return 0;
+                if (rioWriteBulkString(r,"SADD",4) == 0) return 0;
+                if (rioWriteBulkObject(r,key) == 0) return 0;
+            }
+            if (rioWriteBulkObject(r,eleobj) == 0) return 0;
+            if (++count == REDIS_AOFREWRITE_ITEMS_PER_CMD) count = 0;
+            items--;
+        }
+        dictReleaseIterator(di);
+    } else {
+        redisPanic("Unknown set encoding");
+    }
+    return 1;
+}
+
+/* Emit the commands needed to rebuild a sorted set object.
+ * The function returns 0 on error, 1 on success. */
+int rewriteSortedSetObject(rio *r, robj *key, robj *o) {
+    long long count = 0, items = zsetLength(o);
+
+    if (o->encoding == REDIS_ENCODING_ZIPLIST) {
+        unsigned char *zl = o->ptr;
+        unsigned char *eptr, *sptr;
+        unsigned char *vstr;
+        unsigned int vlen;
+        long long vll;
+        double score;
+
+        eptr = ziplistIndex(zl,0);
+        redisAssert(eptr != NULL);
+        sptr = ziplistNext(zl,eptr);
+        redisAssert(sptr != NULL);
+
+        while (eptr != NULL) {
+            redisAssert(ziplistGet(eptr,&vstr,&vlen,&vll));
+            score = zzlGetScore(sptr);
+
+            if (count == 0) {
+                int cmd_items = (items > REDIS_AOFREWRITE_ITEMS_PER_CMD) ?
+                    REDIS_AOFREWRITE_ITEMS_PER_CMD : items;
+
+                if (rioWriteBulkCount(r,'*',2+cmd_items*2) == 0) return 0;
+                if (rioWriteBulkString(r,"ZADD",4) == 0) return 0;
+                if (rioWriteBulkObject(r,key) == 0) return 0;
+            }
+            if (rioWriteBulkDouble(r,score) == 0) return 0;
+            if (vstr != NULL) {
+                if (rioWriteBulkString(r,(char*)vstr,vlen) == 0) return 0;
+            } else {
+                if (rioWriteBulkLongLong(r,vll) == 0) return 0;
+            }
+            zzlNext(zl,&eptr,&sptr);
+            if (++count == REDIS_AOFREWRITE_ITEMS_PER_CMD) count = 0;
+            items--;
+        }
+    } else if (o->encoding == REDIS_ENCODING_SKIPLIST) {
+        zset *zs = o->ptr;
+        dictIterator *di = dictGetIterator(zs->dict);
+        dictEntry *de;
+
+        while((de = dictNext(di)) != NULL) {
+            robj *eleobj = dictGetKey(de);
+            double *score = dictGetVal(de);
+
+            if (count == 0) {
+                int cmd_items = (items > REDIS_AOFREWRITE_ITEMS_PER_CMD) ?
+                    REDIS_AOFREWRITE_ITEMS_PER_CMD : items;
+
+                if (rioWriteBulkCount(r,'*',2+cmd_items*2) == 0) return 0;
+                if (rioWriteBulkString(r,"ZADD",4) == 0) return 0;
+                if (rioWriteBulkObject(r,key) == 0) return 0;
+            }
+            if (rioWriteBulkDouble(r,*score) == 0) return 0;
+            if (rioWriteBulkObject(r,eleobj) == 0) return 0;
+            if (++count == REDIS_AOFREWRITE_ITEMS_PER_CMD) count = 0;
+            items--;
+        }
+        dictReleaseIterator(di);
+    } else {
+        redisPanic("Unknown sorted zset encoding");
+    }
+    return 1;
+}
+
+/* Emit the commands needed to rebuild a hash object.
+ * The function returns 0 on error, 1 on success. */
+int rewriteHashObject(rio *r, robj *key, robj *o) {
+    long long count = 0, items = hashTypeLength(o);
+
+    if (o->encoding == REDIS_ENCODING_ZIPMAP) {
+        unsigned char *p = zipmapRewind(o->ptr);
+        unsigned char *field, *val;
+        unsigned int flen, vlen;
+
+        while((p = zipmapNext(p,&field,&flen,&val,&vlen)) != NULL) {
+            if (count == 0) {
+                int cmd_items = (items > REDIS_AOFREWRITE_ITEMS_PER_CMD) ?
+                    REDIS_AOFREWRITE_ITEMS_PER_CMD : items;
+
+                if (rioWriteBulkCount(r,'*',2+cmd_items*2) == 0) return 0;
+                if (rioWriteBulkString(r,"HMSET",5) == 0) return 0;
+                if (rioWriteBulkObject(r,key) == 0) return 0;
+            }
+            if (rioWriteBulkString(r,(char*)field,flen) == 0) return 0;
+            if (rioWriteBulkString(r,(char*)val,vlen) == 0) return 0;
+            if (++count == REDIS_AOFREWRITE_ITEMS_PER_CMD) count = 0;
+            items--;
+        }
+    } else {
+        dictIterator *di = dictGetIterator(o->ptr);
+        dictEntry *de;
+
+        while((de = dictNext(di)) != NULL) {
+            robj *field = dictGetKey(de);
+            robj *val = dictGetVal(de);
+
+            if (count == 0) {
+                int cmd_items = (items > REDIS_AOFREWRITE_ITEMS_PER_CMD) ?
+                    REDIS_AOFREWRITE_ITEMS_PER_CMD : items;
+
+                if (rioWriteBulkCount(r,'*',2+cmd_items*2) == 0) return 0;
+                if (rioWriteBulkString(r,"HMSET",5) == 0) return 0;
+                if (rioWriteBulkObject(r,key) == 0) return 0;
+            }
+            if (rioWriteBulkObject(r,field) == 0) return 0;
+            if (rioWriteBulkObject(r,val) == 0) return 0;
+            if (++count == REDIS_AOFREWRITE_ITEMS_PER_CMD) count = 0;
+            items--;
+        }
+        dictReleaseIterator(di);
+    }
+    return 1;
+}
+
 /* Write a sequence of commands able to fully rebuild the dataset into
- * "filename". Used both by REWRITEAOF and BGREWRITEAOF. */
+ * "filename". Used both by REWRITEAOF and BGREWRITEAOF.
+ *
+ * In order to minimize the number of commands needed in the rewritten
+ * log Redis uses variadic commands when possible, such as RPUSH, SADD
+ * and ZADD. However at max REDIS_AOFREWRITE_ITEMS_PER_CMD items per time
+ * are inserted using a single command. */
 int rewriteAppendOnlyFile(char *filename) {
     dictIterator *di = NULL;
     dictEntry *de;
@@ -410,7 +659,7 @@ int rewriteAppendOnlyFile(char *filename) {
     FILE *fp;
     char tmpfile[256];
     int j;
-    time_t now = time(NULL);
+    long long now = mstime();
 
     /* Note that we have to use a different temp name here compared to the
      * one used by rewriteAppendOnlyFileBackground() function. */
@@ -441,10 +690,10 @@ int rewriteAppendOnlyFile(char *filename) {
         while((de = dictNext(di)) != NULL) {
             sds keystr;
             robj key, *o;
-            time_t expiretime;
+            long long expiretime;
 
-            keystr = dictGetEntryKey(de);
-            o = dictGetEntryVal(de);
+            keystr = dictGetKey(de);
+            o = dictGetVal(de);
             initStaticStringObject(key,keystr);
 
             expiretime = getExpire(db,&key);
@@ -458,157 +707,19 @@ int rewriteAppendOnlyFile(char *filename) {
                 if (rioWriteBulkObject(&aof,&key) == 0) goto werr;
                 if (rioWriteBulkObject(&aof,o) == 0) goto werr;
             } else if (o->type == REDIS_LIST) {
-                /* Emit the RPUSHes needed to rebuild the list */
-                char cmd[]="*3\r\n$5\r\nRPUSH\r\n";
-                if (o->encoding == REDIS_ENCODING_ZIPLIST) {
-                    unsigned char *zl = o->ptr;
-                    unsigned char *p = ziplistIndex(zl,0);
-                    unsigned char *vstr;
-                    unsigned int vlen;
-                    long long vlong;
-
-                    while(ziplistGet(p,&vstr,&vlen,&vlong)) {
-                        if (rioWrite(&aof,cmd,sizeof(cmd)-1) == 0) goto werr;
-                        if (rioWriteBulkObject(&aof,&key) == 0) goto werr;
-                        if (vstr) {
-                            if (rioWriteBulkString(&aof,(char*)vstr,vlen) == 0)
-                                goto werr;
-                        } else {
-                            if (rioWriteBulkLongLong(&aof,vlong) == 0)
-                                goto werr;
-                        }
-                        p = ziplistNext(zl,p);
-                    }
-                } else if (o->encoding == REDIS_ENCODING_LINKEDLIST) {
-                    list *list = o->ptr;
-                    listNode *ln;
-                    listIter li;
-
-                    listRewind(list,&li);
-                    while((ln = listNext(&li))) {
-                        robj *eleobj = listNodeValue(ln);
-
-                        if (rioWrite(&aof,cmd,sizeof(cmd)-1) == 0) goto werr;
-                        if (rioWriteBulkObject(&aof,&key) == 0) goto werr;
-                        if (rioWriteBulkObject(&aof,eleobj) == 0) goto werr;
-                    }
-                } else {
-                    redisPanic("Unknown list encoding");
-                }
+                if (rewriteListObject(&aof,&key,o) == 0) goto werr;
             } else if (o->type == REDIS_SET) {
-                char cmd[]="*3\r\n$4\r\nSADD\r\n";
-
-                /* Emit the SADDs needed to rebuild the set */
-                if (o->encoding == REDIS_ENCODING_INTSET) {
-                    int ii = 0;
-                    int64_t llval;
-                    while(intsetGet(o->ptr,ii++,&llval)) {
-                        if (rioWrite(&aof,cmd,sizeof(cmd)-1) == 0) goto werr;
-                        if (rioWriteBulkObject(&aof,&key) == 0) goto werr;
-                        if (rioWriteBulkLongLong(&aof,llval) == 0) goto werr;
-                    }
-                } else if (o->encoding == REDIS_ENCODING_HT) {
-                    dictIterator *di = dictGetIterator(o->ptr);
-                    dictEntry *de;
-                    while((de = dictNext(di)) != NULL) {
-                        robj *eleobj = dictGetEntryKey(de);
-                        if (rioWrite(&aof,cmd,sizeof(cmd)-1) == 0) goto werr;
-                        if (rioWriteBulkObject(&aof,&key) == 0) goto werr;
-                        if (rioWriteBulkObject(&aof,eleobj) == 0) goto werr;
-                    }
-                    dictReleaseIterator(di);
-                } else {
-                    redisPanic("Unknown set encoding");
-                }
+                if (rewriteSetObject(&aof,&key,o) == 0) goto werr;
             } else if (o->type == REDIS_ZSET) {
-                /* Emit the ZADDs needed to rebuild the sorted set */
-                char cmd[]="*4\r\n$4\r\nZADD\r\n";
-
-                if (o->encoding == REDIS_ENCODING_ZIPLIST) {
-                    unsigned char *zl = o->ptr;
-                    unsigned char *eptr, *sptr;
-                    unsigned char *vstr;
-                    unsigned int vlen;
-                    long long vll;
-                    double score;
-
-                    eptr = ziplistIndex(zl,0);
-                    redisAssert(eptr != NULL);
-                    sptr = ziplistNext(zl,eptr);
-                    redisAssert(sptr != NULL);
-
-                    while (eptr != NULL) {
-                        redisAssert(ziplistGet(eptr,&vstr,&vlen,&vll));
-                        score = zzlGetScore(sptr);
-
-                        if (rioWrite(&aof,cmd,sizeof(cmd)-1) == 0) goto werr;
-                        if (rioWriteBulkObject(&aof,&key) == 0) goto werr;
-                        if (rioWriteBulkDouble(&aof,score) == 0) goto werr;
-                        if (vstr != NULL) {
-                            if (rioWriteBulkString(&aof,(char*)vstr,vlen) == 0)
-                                goto werr;
-                        } else {
-                            if (rioWriteBulkLongLong(&aof,vll) == 0)
-                                goto werr;
-                        }
-                        zzlNext(zl,&eptr,&sptr);
-                    }
-                } else if (o->encoding == REDIS_ENCODING_SKIPLIST) {
-                    zset *zs = o->ptr;
-                    dictIterator *di = dictGetIterator(zs->dict);
-                    dictEntry *de;
-
-                    while((de = dictNext(di)) != NULL) {
-                        robj *eleobj = dictGetEntryKey(de);
-                        double *score = dictGetEntryVal(de);
-
-                        if (rioWrite(&aof,cmd,sizeof(cmd)-1) == 0) goto werr;
-                        if (rioWriteBulkObject(&aof,&key) == 0) goto werr;
-                        if (rioWriteBulkDouble(&aof,*score) == 0) goto werr;
-                        if (rioWriteBulkObject(&aof,eleobj) == 0) goto werr;
-                    }
-                    dictReleaseIterator(di);
-                } else {
-                    redisPanic("Unknown sorted set encoding");
-                }
+                if (rewriteSortedSetObject(&aof,&key,o) == 0) goto werr;
             } else if (o->type == REDIS_HASH) {
-                char cmd[]="*4\r\n$4\r\nHSET\r\n";
-
-                /* Emit the HSETs needed to rebuild the hash */
-                if (o->encoding == REDIS_ENCODING_ZIPMAP) {
-                    unsigned char *p = zipmapRewind(o->ptr);
-                    unsigned char *field, *val;
-                    unsigned int flen, vlen;
-
-                    while((p = zipmapNext(p,&field,&flen,&val,&vlen)) != NULL) {
-                        if (rioWrite(&aof,cmd,sizeof(cmd)-1) == 0) goto werr;
-                        if (rioWriteBulkObject(&aof,&key) == 0) goto werr;
-                        if (rioWriteBulkString(&aof,(char*)field,flen) == 0)
-                            goto werr;
-                        if (rioWriteBulkString(&aof,(char*)val,vlen) == 0)
-                            goto werr;
-                    }
-                } else {
-                    dictIterator *di = dictGetIterator(o->ptr);
-                    dictEntry *de;
-
-                    while((de = dictNext(di)) != NULL) {
-                        robj *field = dictGetEntryKey(de);
-                        robj *val = dictGetEntryVal(de);
-
-                        if (rioWrite(&aof,cmd,sizeof(cmd)-1) == 0) goto werr;
-                        if (rioWriteBulkObject(&aof,&key) == 0) goto werr;
-                        if (rioWriteBulkObject(&aof,field) == 0) goto werr;
-                        if (rioWriteBulkObject(&aof,val) == 0) goto werr;
-                    }
-                    dictReleaseIterator(di);
-                }
+                if (rewriteHashObject(&aof,&key,o) == 0) goto werr;
             } else {
                 redisPanic("Unknown object type");
             }
             /* Save the expire time */
             if (expiretime != -1) {
-                char cmd[]="*3\r\n$8\r\nEXPIREAT\r\n";
+                char cmd[]="*3\r\n$9\r\nPEXPIREAT\r\n";
                 /* If this key is already expired skip it */
                 if (expiretime < now) continue;
                 if (rioWrite(&aof,cmd,sizeof(cmd)-1) == 0) goto werr;