Integer encoding implemented in dump file. Doc updated

diff --git a/TODO b/TODO
index 6e5317ba5c8c45f759f6ff6a32dae522c4c5b0dd..85304b96a2f7c11bb1333cf30aa934fd83fafed3 100644 (file)
--- a/TODO
+++ b/TODO
@@ -7,7 +7,16 @@
 - maxclients directive
 - check 'server.dirty' everywere
 - replication automated tests
 - maxclients directive
 - check 'server.dirty' everywere
 - replication automated tests

-- a command, or an external tool, to perform the MD5SUM of the whole dataset, so that if the dataset between two servers is identical, so will be the MD5SUM
+- a command, or an external tool, to perform the SHA1SUM of the whole dataset, so that if the dataset between two servers is identical, so will be the MD5SUM
+- an external tool able to perform the 'difference' between two Redis servers. It's like 'diff', but against Redis servers, and the output is the set of commands needed to turn the first server into the second, suitable to be sent via netcat.
+
+   $ ./redis-diff 192.168.1.1 192.168.1.2 > diff.txt
+   $ cat diff.txt | nc 192.168.1.1 6379
+   $ ./redis-diff 192.168.1.1 192.168.1.2
+   $ # No output now the servers are identical
+
+This command should be smart and don't use too much memory, that is, take two connections at the same time against the two servers and perform the comparison key by key. Probably the initial "KEYS *" is unavoidable.
+

 - objects sharing, "objectsharing yes", "objectsharingpool 1024"
 
 * Include Lua and Perl bindings
 - objects sharing, "objectsharing yes", "objectsharingpool 1024"
 
 * Include Lua and Perl bindings

diff --git a/doc/Credits.html b/doc/Credits.html
index b00cfa264279d7c81665b02b54928e6b97d2c72f..751cce781a3bc3e0241e1a1bbf4afb5829ae3905 100644 (file)
--- a/doc/Credits.html
+++ b/doc/Credits.html
@@ -26,7 +26,9 @@
                 </div>
 
                 <div class="narrow">
                 </div>
 
                 <div class="narrow">

-                    <h1><a name="Credits">Credits</a></h1><ul><li> The Redis server was designed and written by <a href="http://invece.org" target="_blank">Salvatore Sanfilippo (aka antirez)</a></li><li> The Ruby client library was written by <a href="http://brainspl.at/" target="_blank">Ezra Zygmuntowicz (aka ezmobius)</a></li><li> The Python and PHP client libraries were written by <a href="http://qix.it" target="_blank">Ludovico Magnocavallo (aka ludo)</a></li><li> The Erlang client library was written by <a href="http://www.adroll.com/" target="_blank">Valentino Volonghi of Adroll</a></li><li> <b>brettbender</b> found and fixed a but in sds.c that caused the server to crash at least on 64 bit systems, and anyway to be buggy since we used the same vararg thing against vsprintf without to call va_start and va_end every time.</li></ul>
+                    <h1><a name="Credits">Credits</a></h1><ul><li> The Redis server was designed and written by <a href="http://invece.org" target="_blank">Salvatore Sanfilippo (aka antirez)</a></li><li> <a href="http://brainspl.at/" target="_blank">Ezra Zygmuntowicz (aka ezmobius)</a> - Ruby client lib initial version and hacking</li><li> <a href="http://qix.it" target="_blank">Ludovico Magnocavallo (aka ludo)</a> - Python clinet lib</li><li> <a href="http://www.adroll.com/" target="_blank">Valentino Volonghi of Adroll</a> - Erlang client lib</li><li> <b>brettbender</b> - found and fixed a but in sds.c that caused the server to crash at least on 64 bit systems, and anyway to be buggy since we used the same vararg thing against vsprintf without to call va_start and va_end every time.</li><li> <a href="http://www.rot13.org/~dpavlin" target="_blank">Dobrica Pavlinusic</a> - Perl client lib</li><li> Brian Hammond - AUTH command implementation</li><li> <a href="http://www.clorophilla.net/" target="_blank">Daniele Alessandri</a> - Lua client lib</li><li> Corey Stup - C99 cleanups</li><li> Taylor Weibley - Ruby client hacking </li></ul>
+p.s. sorry to take this file in sync is hard in this early days. Please drop me an email if I forgot to add your name here!
+

                 </div>
         
             </div>
                 </div>
         
             </div>

diff --git a/redis.c b/redis.c
index 003193f4d3e66a4292eee86d69026cee238009ee..b77432f4d866e661cfa72bf61fa786b1bf5247c9 100644 (file)
--- a/redis.c
+++ b/redis.c
@@ -1509,6 +1509,7 @@ static int rdbSaveType(FILE *fp, unsigned char type) {
     return 0;
 }
 
     return 0;
 }
 

+/* check rdbLoadLen() comments for more info */

 static int rdbSaveLen(FILE *fp, uint32_t len) {
     unsigned char buf[2];
 
 static int rdbSaveLen(FILE *fp, uint32_t len) {
     unsigned char buf[2];
 


@@ -1531,9 +1532,57 @@ static int rdbSaveLen(FILE *fp, uint32_t len) {
     return 0;
 }
 
     return 0;
 }
 

+/* String objects in the form "2391" "-100" without any space and with a
+ * range of values that can fit in an 8, 16 or 32 bit signed value can be
+ * encoded as integers to save space */
+int rdbTryIntegerEncoding(sds s, unsigned char *enc) {
+    long long value;
+    char *endptr, buf[32];
+
+    /* Check if it's possible to encode this value as a number */
+    value = strtoll(s, &endptr, 10);
+    if (endptr[0] != '\0') return 0;
+    snprintf(buf,32,"%lld",value);
+
+    /* If the number converted back into a string is not identical
+     * then it's not possible to encode the string as integer */
+    if (strlen(buf) != sdslen(s) || memcmp(buf,s,sdslen(s))) return 0;
+
+    /* Finally check if it fits in our ranges */
+    if (value >= -(1<<7) && value <= (1<<7)-1) {
+        enc[0] = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_INT8;
+        enc[1] = value&0xFF;
+        return 2;
+    } else if (value >= -(1<<15) && value <= (1<<15)-1) {
+        enc[0] = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_INT16;
+        enc[1] = value&0xFF;
+        enc[2] = (value>>8)&0xFF;
+        return 3;
+    } else if (value >= -((long long)1<<31) && value <= ((long long)1<<31)-1) {
+        enc[0] = (REDIS_RDB_ENCVAL<<6)|REDIS_RDB_ENC_INT32;
+        enc[1] = value&0xFF;
+        enc[2] = (value>>8)&0xFF;
+        enc[3] = (value>>16)&0xFF;
+        enc[4] = (value>>24)&0xFF;
+        return 5;
+    } else {
+        return 0;
+    }
+}
+
+/* Save a string objet as [len][data] on disk. If the object is a string
+ * representation of an integer value we try to safe it in a special form */

 static int rdbSaveStringObject(FILE *fp, robj *obj) {
     size_t len = sdslen(obj->ptr);
 static int rdbSaveStringObject(FILE *fp, robj *obj) {
     size_t len = sdslen(obj->ptr);

+    int enclen;

 
 

+    if (len <= 11) {
+        unsigned char buf[5];
+        if ((enclen = rdbTryIntegerEncoding(obj->ptr,buf)) > 0) {
+            if (fwrite(buf,enclen,1,fp) == 0) return -1;
+            return 0;
+        }
+    }

     if (rdbSaveLen(fp,len) == -1) return -1;
     if (len && fwrite(obj->ptr,len,1,fp) == 0) return -1;
     return 0;
     if (rdbSaveLen(fp,len) == -1) return -1;
     if (len && fwrite(obj->ptr,len,1,fp) == 0) return -1;
     return 0;


@@ -1664,10 +1713,16 @@ static int rdbLoadType(FILE *fp) {
     return type;
 }
 
     return type;
 }
 

-static uint32_t rdbLoadLen(FILE *fp, int rdbver) {
+/* Load an encoded length from the DB, see the REDIS_RDB_* defines on the top
+ * of this file for a description of how this are stored on disk.
+ *
+ * isencoded is set to 1 if the readed length is not actually a length but
+ * an "encoding type", check the above comments for more info */
+static uint32_t rdbLoadLen(FILE *fp, int rdbver, int *isencoded) {

     unsigned char buf[2];
     uint32_t len;
 
     unsigned char buf[2];
     uint32_t len;
 

+    if (isencoded) *isencoded = 0;

     if (rdbver == 0) {
         if (fread(&len,4,1,fp) == 0) return REDIS_RDB_LENERR;
         return ntohl(len);
     if (rdbver == 0) {
         if (fread(&len,4,1,fp) == 0) return REDIS_RDB_LENERR;
         return ntohl(len);


@@ -1678,7 +1733,11 @@ static uint32_t rdbLoadLen(FILE *fp, int rdbver) {
         type = (buf[0]&0xC0)>>6;
         if (type == REDIS_RDB_6BITLEN) {
             /* Read a 6 bit len */
         type = (buf[0]&0xC0)>>6;
         if (type == REDIS_RDB_6BITLEN) {
             /* Read a 6 bit len */

-            return buf[0];
+            return buf[0]&0x3F;
+        } else if (type == REDIS_RDB_ENCVAL) {
+            /* Read a 6 bit len encoding type */
+            if (isencoded) *isencoded = 1;
+            return buf[0]&0x3F;

         } else if (type == REDIS_RDB_14BITLEN) {
             /* Read a 14 bit len */
             if (fread(buf+1,1,1,fp) == 0) return REDIS_RDB_LENERR;
         } else if (type == REDIS_RDB_14BITLEN) {
             /* Read a 14 bit len */
             if (fread(buf+1,1,1,fp) == 0) return REDIS_RDB_LENERR;


@@ -1691,10 +1750,47 @@ static uint32_t rdbLoadLen(FILE *fp, int rdbver) {
     }
 }
 
     }
 }
 

-static robj *rdbLoadStringObject(FILE*fp,int rdbver) {
-    uint32_t len = rdbLoadLen(fp,rdbver);
+static robj *rdbLoadIntegerObject(FILE *fp, int enctype) {
+    unsigned char enc[4];
+    long long val;
+
+    if (enctype == REDIS_RDB_ENC_INT8) {
+        if (fread(enc,1,1,fp) == 0) return NULL;
+        val = (signed char)enc[0];
+    } else if (enctype == REDIS_RDB_ENC_INT16) {
+        uint16_t v;
+        if (fread(enc,2,1,fp) == 0) return NULL;
+        v = enc[0]|(enc[1]<<8);
+        val = (int16_t)v;
+    } else if (enctype == REDIS_RDB_ENC_INT32) {
+        uint32_t v;
+        if (fread(enc,4,1,fp) == 0) return NULL;
+        v = enc[0]|(enc[1]<<8)|(enc[2]<<16)|(enc[3]<<24);
+        val = (int32_t)v;
+    } else {
+        val = 0; /* anti-warning */
+        assert(0!=0);
+    }
+    return createObject(REDIS_STRING,sdscatprintf(sdsempty(),"%lld",val));
+}
+
+static robj *rdbLoadStringObject(FILE*fp, int rdbver) {
+    int isencoded;
+    uint32_t len;

     sds val;
 
     sds val;
 

+    len = rdbLoadLen(fp,rdbver,&isencoded);
+    if (isencoded) {
+        switch(len) {
+        case REDIS_RDB_ENC_INT8:
+        case REDIS_RDB_ENC_INT16:
+        case REDIS_RDB_ENC_INT32:
+            return rdbLoadIntegerObject(fp,len);
+        default:
+            assert(0!=0);
+        }
+    }
+

     if (len == REDIS_RDB_LENERR) return NULL;
     val = sdsnewlen(NULL,len);
     if (len && fread(val,len,1,fp) == 0) {
     if (len == REDIS_RDB_LENERR) return NULL;
     val = sdsnewlen(NULL,len);
     if (len && fread(val,len,1,fp) == 0) {


@@ -1736,7 +1832,8 @@ static int rdbLoad(char *filename) {
         if (type == REDIS_EOF) break;
         /* Handle SELECT DB opcode as a special case */
         if (type == REDIS_SELECTDB) {
         if (type == REDIS_EOF) break;
         /* Handle SELECT DB opcode as a special case */
         if (type == REDIS_SELECTDB) {

-            if ((dbid = rdbLoadLen(fp,rdbver)) == REDIS_RDB_LENERR) goto eoferr;
+            if ((dbid = rdbLoadLen(fp,rdbver,NULL)) == REDIS_RDB_LENERR)
+                goto eoferr;

             if (dbid >= (unsigned)server.dbnum) {
                 redisLog(REDIS_WARNING,"FATAL: Data file was created with a Redis server configured to handle more than %d databases. Exiting\n", server.dbnum);
                 exit(1);
             if (dbid >= (unsigned)server.dbnum) {
                 redisLog(REDIS_WARNING,"FATAL: Data file was created with a Redis server configured to handle more than %d databases. Exiting\n", server.dbnum);
                 exit(1);


@@ -1754,7 +1851,7 @@ static int rdbLoad(char *filename) {
             /* Read list/set value */
             uint32_t listlen;
 
             /* Read list/set value */
             uint32_t listlen;
 

-            if ((listlen = rdbLoadLen(fp,rdbver)) == REDIS_RDB_LENERR)
+            if ((listlen = rdbLoadLen(fp,rdbver,NULL)) == REDIS_RDB_LENERR)

                 goto eoferr;
             o = (type == REDIS_LIST) ? createListObject() : createSetObject();
             /* Load every single element of the list/set */
                 goto eoferr;
             o = (type == REDIS_LIST) ? createListObject() : createSetObject();
             /* Load every single element of the list/set */


@@ -1785,8 +1882,8 @@ static int rdbLoad(char *filename) {
     return REDIS_OK;
 
 eoferr: /* unexpected end of file is handled here with a fatal exit */
     return REDIS_OK;
 
 eoferr: /* unexpected end of file is handled here with a fatal exit */

-    decrRefCount(keyobj);
-    redisLog(REDIS_WARNING,"Short read loading DB. Unrecoverable error, exiting now.");
+    if (keyobj) decrRefCount(keyobj);
+    redisLog(REDIS_WARNING,"Short read or OOM loading DB. Unrecoverable error, exiting now.");

     exit(1);
     return REDIS_ERR; /* Just to avoid warning */
 }
     exit(1);
     return REDIS_ERR; /* Just to avoid warning */
 }