Issue 22 fixed

diff --git a/Makefile b/Makefile
index cb09c911f6d6e6f97fcfece77156cc78301b820f..8a397bddc5d4bd5b14902fde2436a43fac569816 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -6,7 +6,7 @@ DEBUG?= -g
 CFLAGS?= -std=c99 -pedantic -O2 -Wall -W -DSDS_ABORT_ON_OOM
 CCOPT= $(CFLAGS)
 
 CFLAGS?= -std=c99 -pedantic -O2 -Wall -W -DSDS_ABORT_ON_OOM
 CCOPT= $(CFLAGS)
 

-OBJ = adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o
+OBJ = adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o

 BENCHOBJ = ae.o anet.o benchmark.o sds.o adlist.o zmalloc.o
 CLIOBJ = anet.o sds.o adlist.o redis-cli.o zmalloc.o
 
 BENCHOBJ = ae.o anet.o benchmark.o sds.o adlist.o zmalloc.o
 CLIOBJ = anet.o sds.o adlist.o redis-cli.o zmalloc.o
 

diff --git a/TODO b/TODO
index d729196c1b88ba73a9bf565ae0223338e0fcff8f..657ae026c5cf4465fb62ac66752203e513019fbc 100644 (file)
--- a/TODO
+++ b/TODO
@@ -16,3 +16,4 @@
 This command should be smart and don't use too much memory, that is, take two connections at the same time against the two servers and perform the comparison key by key. Probably the initial "KEYS *" is unavoidable.
 
 - objects sharing, "objectsharing yes", "objectsharingpool 1024"
 This command should be smart and don't use too much memory, that is, take two connections at the same time against the two servers and perform the comparison key by key. Probably the initial "KEYS *" is unavoidable.
 
 - objects sharing, "objectsharing yes", "objectsharingpool 1024"

+- if in-memory values compression will be implemented, make sure to implement this so that addReply() is able to handle compressed objects, just creating an uncompressed version on the fly and adding this to the output queue instead of the original one. When insetad we need to look at the object string value (SORT BY for example), call a function that will turn the object into an uncompresed one.

diff --git a/redis.c b/redis.c
index 50b7e49b5bf638ddd18813e2a54052a0aac96972..f91616157df99fb3528b9141d805bb0b03a838e0 100644 (file)
--- a/redis.c
+++ b/redis.c
@@ -56,6 +56,7 @@
 #include "dict.h"   /* Hash tables */
 #include "adlist.h" /* Linked lists */
 #include "zmalloc.h" /* total memory usage aware version of malloc/free */
 #include "dict.h"   /* Hash tables */
 #include "adlist.h" /* Linked lists */
 #include "zmalloc.h" /* total memory usage aware version of malloc/free */

+#include "lzf.h"

 
 /* Error codes */
 #define REDIS_OK                0
 
 /* Error codes */
 #define REDIS_OK                0


@@ -116,7 +117,7 @@
 #define REDIS_RDB_ENC_INT8 0        /* 8 bit signed integer */
 #define REDIS_RDB_ENC_INT16 1       /* 16 bit signed integer */
 #define REDIS_RDB_ENC_INT32 2       /* 32 bit signed integer */
 #define REDIS_RDB_ENC_INT8 0        /* 8 bit signed integer */
 #define REDIS_RDB_ENC_INT16 1       /* 16 bit signed integer */
 #define REDIS_RDB_ENC_INT32 2       /* 32 bit signed integer */

-#define REDIS_RDB_ENC_FLZ 3         /* string compressed with FASTLZ */
+#define REDIS_RDB_ENC_LZF 3         /* string compressed with FASTLZ */

 
 /* Client flags */
 #define REDIS_CLOSE 1       /* This client connection should be closed ASAP */
 
 /* Client flags */
 #define REDIS_CLOSE 1       /* This client connection should be closed ASAP */


@@ -1645,12 +1646,41 @@ int rdbTryIntegerEncoding(sds s, unsigned char *enc) {
     }
 }
 
     }
 }
 

+static int rdbSaveLzfStringObject(FILE *fp, robj *obj) {
+    unsigned int comprlen, outlen;
+    unsigned char byte;
+    void *out;
+
+    /* We require at least four bytes compression for this to be worth it */
+    outlen = sdslen(obj->ptr)-4;
+    if (outlen <= 0) return 0;
+    if ((out = zmalloc(outlen)) == NULL) return 0;
+    comprlen = lzf_compress(obj->ptr, sdslen(obj->ptr), out, outlen);
+    if (comprlen == 0) {
+        free(out);
+        return 0;
+    }
+    /* Data compressed! Let's save it on disk */
+    byte = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_LZF;
+    if (fwrite(&byte,1,1,fp) == 0) goto writeerr;
+    if (rdbSaveLen(fp,comprlen) == -1) goto writeerr;
+    if (rdbSaveLen(fp,sdslen(obj->ptr)) == -1) goto writeerr;
+    if (fwrite(out,comprlen,1,fp) == 0) goto writeerr;
+    free(out);
+    return comprlen;
+
+writeerr:
+    free(out);
+    return -1;
+}
+

 /* Save a string objet as [len][data] on disk. If the object is a string
  * representation of an integer value we try to safe it in a special form */
 static int rdbSaveStringObject(FILE *fp, robj *obj) {
     size_t len = sdslen(obj->ptr);
     int enclen;
 
 /* Save a string objet as [len][data] on disk. If the object is a string
  * representation of an integer value we try to safe it in a special form */
 static int rdbSaveStringObject(FILE *fp, robj *obj) {
     size_t len = sdslen(obj->ptr);
     int enclen;
 

+    /* Try integer encoding */

     if (len <= 11) {
         unsigned char buf[5];
         if ((enclen = rdbTryIntegerEncoding(obj->ptr,buf)) > 0) {
     if (len <= 11) {
         unsigned char buf[5];
         if ((enclen = rdbTryIntegerEncoding(obj->ptr,buf)) > 0) {


@@ -1658,6 +1688,19 @@ static int rdbSaveStringObject(FILE *fp, robj *obj) {
             return 0;
         }
     }
             return 0;
         }
     }

+
+    /* Try LZF compression - under 20 bytes it's unable to compress even
+     * aaaaaaaaaaaaaaaaaa so to try is just useful to make the CPU hot */
+    if (len > 20) {
+        int retval;
+
+        retval = rdbSaveLzfStringObject(fp,obj);
+        if (retval == -1) return -1;
+        if (retval > 0) return 0;
+        /* retval == 0 means data can't be compressed, save the old way */
+    }
+
+    /* Store verbatim */

     if (rdbSaveLen(fp,len) == -1) return -1;
     if (len && fwrite(obj->ptr,len,1,fp) == 0) return -1;
     return 0;
     if (rdbSaveLen(fp,len) == -1) return -1;
     if (len && fwrite(obj->ptr,len,1,fp) == 0) return -1;
     return 0;


@@ -2123,7 +2166,7 @@ static void selectCommand(redisClient *c) {
     int id = atoi(c->argv[1]->ptr);
     
     if (selectDb(c,id) == REDIS_ERR) {
     int id = atoi(c->argv[1]->ptr);
     
     if (selectDb(c,id) == REDIS_ERR) {

-        addReplySds(c,"-ERR invalid DB index\r\n");
+        addReplySds(c,sdsnew("-ERR invalid DB index\r\n"));

     } else {
         addReply(c,shared.ok);
     }
     } else {
         addReply(c,shared.ok);
     }